Creating a neural network in pytorch from scratch

[This tutorial is under development......Please don't use it right now]

import torch
import torch.nn as nn
from torch.optim import SGD
import matplotlib.pyplot as plt

# Create a neural network which can learn to multiply numbers
x = [[1,4], [2,5], [3,1], [7,3], [8,2], [9,5], [11,3]]
y = [[4], [10], [3], [10], [16], [45], [33]]

# convert lists into tensor objects (it is a good practice to convert tensor elements in float)
# objects since it will be multiplied by float weights
x = torch.tensor(x).float()
y = torch.tensor(y).float()

device = 'cuda' if torch.cuda.is_available() else 'cpu'
x = x.to(device)
y = y.to(device)


# create a neural network which can learn multiplication operation
class MulNeuralNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.input_to_deep_layer = nn.Linear(2,8) # 2 input features and 8 output features
        self.deep_layer_activation = nn.ReLU() # relu activation, other activation functions are sigmod, softmax, tanh
        self.deep_layer_to_output_layer = nn.Linear(8,1)

    def forward(self, x):
        '''
        It is mandatory to use forward function name in pytorch since it is a reserved name in PyTorch
        for forward propogation
        '''
        x = self.input_to_deep_layer(x)
        x = self.deep_layer_activation(x)
        x = self.deep_layer_to_output_layer(x)
        return x

mynet = MulNeuralNet().to(device)
loss_func = nn.MSELoss()
y_pred = mynet(x)
loss_value = loss_func(y_pred,y)

opt = SGD(mynet.parameters(), lr = 0.001)
loss_track = []
for _ in range(30):
    opt.zero_grad()
    loss_value = loss_func(mynet(x),y)
    loss_value.backward()
    opt.step()
    loss_track.append(loss_value)

plt.plot(loss_track)
plt.title('Loss vs Epochs')
plt.xlabel('epochs')
plt.ylabel('loss value')
plt.show()

Output:

As you can see in above code data size was very small 

x = [[1,4], [2,5], [3,1], [7,3], [8,2], [9,5], [11,3]]
y = [[4], [10], [3], [10], [16], [45], [33]]

But in real life problems your dataset may have millions of data points and it may not be possible to process all millions of data points in each epoch due to computational resource constraint.

Second point: While training your neural net it is better to sample your data well and use some of it's portion in each batch which is representative of data.

 

import torch
import torch.nn as nn
from torch.optim import SGD
from torch.utils.data import Dataset, DataLoader
import matplotlib.pyplot as plt
import random

# generate synthetic data
X = []
Y = []
for _ in range(10):
    x1 = random.randint(1,20)
    x2 = random.randint(1,20)
    X.append([x1, x2])
    Y.append([x1 + x2])

# convert lists into tensor objects (it is a good practice to convert tensor elements in float)
# objects since it will be multiplied by float weights
X = torch.tensor(X).float()
Y = torch.tensor(Y).float()

device = 'cuda' if torch.cuda.is_available() else 'cpu'
X = X.to(device)
Y = Y.to(device)

class ThisDataset(Dataset):
    def __init__(self,x,y):
        self.x = torch.tensor(x).float()
        self.y = torch.tensor(y).float()
    def __len__(self):
        return len(self.x)
    def __getitem__(self, ix):
        return self.x[ix], self.y[ix]
ds = ThisDataset(X, Y)

# create a neural network which can learn multiplication operation
class MulNeuralNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.input_to_deep_layer = nn.Linear(2,8) # 2 input features and 8 output features
        self.deep_layer_activation = nn.ReLU() # relu activation, other activation functions are sigmod, softmax, tanh
        self.deep_layer_to_output_layer = nn.Linear(8,1)

    def forward(self, x):
        '''
        It is mandatory to use forward function name in pytorch since it is a reserved name in PyTorch
        for forward propogation
        '''
        x = self.input_to_deep_layer(x)
        x = self.deep_layer_activation(x)
        x = self.deep_layer_to_output_layer(x)
        return x

mynet = MulNeuralNet().to(device)
loss_func = nn.MSELoss()

opt = SGD(mynet.parameters(), lr = 0.001)
loss_track = []
dl = DataLoader(ds, batch_size=5, shuffle=True)
for _ in range(100):
    for data in dl:
        x, y = data
        opt.zero_grad()
        loss_value = loss_func(mynet(x),y)
        loss_value.backward()
        opt.step()
        loss_track.append(loss_value)

val_x = [[3,15]]
val_x = torch.tensor(val_x).float().to(device)
print(mynet(val_x))

Output:

tensor([[17.9682]], device='cuda:0', grad_fn=<AddmmBackward>)

Using a custom loss function in pytorch:

90% of the time you will be using loss functions given in pytorch library but as complexity of problems increases like object detection problem or generative adverserial network etc, you may need to define your own custom functions

In next section i'll be explaining instead of using nn.MSELoss function how you can use your own custom implementation

def custom_mean_squared_error(y_pred, y):
    loss = (y_pred-y)**2
    loss = loss.mean()
    return loss

This is how you use custom loss functions (see complete code below)

import torch
import torch.nn as nn
from torch.optim import SGD
from torch.utils.data import Dataset, DataLoader
import matplotlib.pyplot as plt
import random

# generate synthetic data
X = []
Y = []
for _ in range(10):
    x1 = random.randint(1,20)
    x2 = random.randint(1,20)
    X.append([x1, x2])
    Y.append([x1 + x2])

# convert lists into tensor objects (it is a good practice to convert tensor elements in float)
# objects since it will be multiplied by float weights
X = torch.tensor(X).float()
Y = torch.tensor(Y).float()

device = 'cuda' if torch.cuda.is_available() else 'cpu'
X = X.to(device)
Y = Y.to(device)

class ThisDataset(Dataset):
    def __init__(self,x,y):
        self.x = torch.tensor(x).float()
        self.y = torch.tensor(y).float()
    def __len__(self):
        return len(self.x)
    def __getitem__(self, ix):
        return self.x[ix], self.y[ix]
ds = ThisDataset(X, Y)

# create a neural network which can learn multiplication operation
class MulNeuralNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.input_to_deep_layer = nn.Linear(2,8) # 2 input features and 8 output features
        self.deep_layer_activation = nn.ReLU() # relu activation, other activation functions are sigmod, softmax, tanh
        self.deep_layer_to_output_layer = nn.Linear(8,1)

    def forward(self, x):
        '''
        It is mandatory to use forward function name in pytorch since it is a reserved name in PyTorch
        for forward propogation
        '''
        x = self.input_to_deep_layer(x)
        x = self.deep_layer_activation(x)
        x = self.deep_layer_to_output_layer(x)
        return x

mynet = MulNeuralNet().to(device)

def custom_mean_squared_error(y_pred, y):
    loss = (y_pred-y)**2
    loss = loss.mean()
    return loss

loss_func = custom_mean_squared_error

opt = SGD(mynet.parameters(), lr = 0.001)
loss_track = []
dl = DataLoader(ds, batch_size=5, shuffle=True)
for _ in range(100):
    for data in dl:
        x, y = data
        opt.zero_grad()
        loss_value = loss_func(mynet(x),y)
        loss_value.backward()
        opt.step()
        loss_track.append(loss_value)

val_x = [[7,22]]
val_x = torch.tensor(val_x).float().to(device)
print(mynet(val_x))