Skip to content

Commit de30e87

Browse files
loveunkloveunk
committed
test multiple activations
1 parent 3066693 commit de30e87

File tree

4 files changed

+131
-65
lines changed

4 files changed

+131
-65
lines changed

‎.gitignore

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
1+
__pycache__
12
.idea
23
*.h5
34
.ipynb_checkpoints
5+
.vscode
46
tensorflow/introduction-to-tensorflow-for-AI-ML-DL/h-or-s/
57
tensorflow/introduction-to-tensorflow-for-AI-ML-DL/*.zip
68
tensorflow/introduction-to-tensorflow-for-AI-ML-DL/cats-v-dogs
7-
tensorflow/introduction-to-tensorflow-for-AI-ML-DL/sign-language-mnist
9+
tensorflow/introduction-to-tensorflow-for-AI-ML-DL/sign-language-mnist
10+
pytorch/samples/data
11+
pytorch/samples/runs

‎pytorch/samples/MnistClassification.py

+86-40
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,41 @@
22
import torch.nn as nn
33
import torch.nn.functional as F
44
from torchvision import datasets, transforms
5+
from torch.utils.tensorboard import SummaryWriter
6+
# from DynamicRELU import DYReLU2
7+
8+
tensorboard_on = False
9+
if tensorboard_on:
10+
writer = SummaryWriter()
11+
512

613
class MyConvNet(nn.Module):
7-
def __init__(self):
14+
def __init__(self, relu, relustr, **kwargs):
815
super(MyConvNet, self).__init__()
916
self.conv1 = nn.Conv2d(1, 20, 5, 1)
1017
self.conv2 = nn.Conv2d(20, 50, 5, 1)
1118
self.fc1 = nn.Linear(4*4*50, 500)
1219
self.fc2 = nn.Linear(500, 10)
13-
20+
if relustr == 'dyrelu':
21+
self.relu1 = relu(20, 20)
22+
self.relu2 = relu(50, 50)
23+
self.relu3 = relu(500, 500)
24+
else:
25+
self.relu1 = relu()
26+
self.relu2 = relu()
27+
self.relu3 = relu()
28+
1429
def forward(self, x):
15-
x = self.conv1(x) # 28x28 -> 24x24
16-
x = F.relu(x) # 24x24
17-
x = F.max_pool2d(x, 2, 2) # 24x24 -> 12x12
18-
x = self.conv2(x) # 12x12 -> 8x8
19-
x = F.relu(x) # 8x8
20-
x = F.max_pool2d(x, 2, 2) # 8x8 -> 4x4
21-
x = torch.flatten(x, 1)
22-
x = self.fc1(x)
23-
x = F.relu(x)
24-
x = self.fc2(x)
30+
x = self.conv1(x) # 28x28x1 -> 24x24x20
31+
x = self.relu1(x) # 24x24x20
32+
x = F.max_pool2d(x, 2, 2) # 24x24x20 -> 12x12x20
33+
x = self.conv2(x) # 12x12x20 -> 8x8x50
34+
x = self.relu2(x) # 8x8x50
35+
x = F.max_pool2d(x, 2, 2) # 8x8x50 -> 4x4x50
36+
x = torch.flatten(x, 1) # 4x4x50 -> 4*4*50
37+
x = self.fc1(x) # 4*4*50 -> 500
38+
# x = self.relu3(x) # 500 -> 500
39+
x = self.fc2(x) # 500 -> 10
2540
return F.log_softmax(x, dim=1)
2641

2742

@@ -38,12 +53,16 @@ def train(model, device, train_loader, optimizer, epoch):
3853
loss.backward()
3954
optimizer.step()
4055

41-
if batch_idx % 100 == 0:
42-
print("Train epoch: {}, iteration: {}, Loss: {}".format(
43-
epoch, batch_idx, loss.item()
44-
))
56+
if tensorboard_on:
57+
writer.add_scalar('Loss/train',
58+
loss.item(),
59+
epoch * len(train_loader) + batch_idx)
60+
61+
# if batch_idx % 100 == 0:
62+
print("Epoch: {}, train loss: {}, ".format(epoch, loss.item()), end='')
63+
4564

46-
def test(model, device, test_loader):
65+
def test(model, device, test_loader, epoch):
4766
model.eval()
4867
total_loss = 0
4968
correct = 0.
@@ -57,40 +76,67 @@ def test(model, device, test_loader):
5776

5877
total_loss /= len(test_loader.dataset)
5978
acc = correct / len(test_loader.dataset) * 100.
60-
print("Test loss: {}, accuracy: {}".format(total_loss, acc))
79+
print("test loss: {}, accuracy: {}".format(total_loss, acc))
80+
81+
if tensorboard_on:
82+
writer.add_scalar('Loss/test', total_loss, epoch)
83+
writer.add_scalar('Accuracy/test', acc, epoch)
84+
6185

6286
def main():
63-
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
64-
batch_size = 32
87+
batch_size = 128
88+
lr = 0.01
89+
momentum = 0.9
90+
epochs = 15
91+
schd_step = 7
92+
relus = {'relu': nn.ReLU,
93+
'lrelu': nn.LeakyReLU,
94+
'rrelu': nn.RReLU,
95+
'prelu': nn.PReLU,
96+
'relu6': nn.ReLU6,
97+
'elu': nn.ELU,
98+
'selu': nn.SELU,
99+
# dyrelu': DYReLU2
100+
}
101+
relu_kwargs = [{}, {}, {}, {}, {}, {}, {}, {}]
65102

66-
kwargs = {'num_workers': 1, 'pin_memory': True} if torch.cuda.is_available() else {}
103+
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
104+
kwargs = {'num_workers': 1, 'pin_memory': True} \
105+
if torch.cuda.is_available() else {}
67106
train_dataloader = torch.utils.data.DataLoader(
68-
datasets.MNIST('./data', train=True, download=True,
69-
transform=transforms.Compose([
70-
transforms.ToTensor(),
71-
transforms.Normalize((0.1307,), (0.3081,))
72-
])),
107+
datasets.MNIST(
108+
'./data', train=True, download=True,
109+
transform=transforms.Compose([
110+
transforms.ToTensor(),
111+
transforms.Normalize((0.1307,), (0.3081,))
112+
])),
73113
batch_size=batch_size, shuffle=True, **kwargs)
74114
test_dataloader = torch.utils.data.DataLoader(
75-
datasets.MNIST('./data', train=False, download=True,
76-
transform=transforms.Compose([
77-
transforms.ToTensor(),
78-
transforms.Normalize((0.1307,), (0.3081,))
79-
])),
115+
datasets.MNIST(
116+
'./data', train=False, download=True,
117+
transform=transforms.Compose([
118+
transforms.ToTensor(),
119+
transforms.Normalize((0.1307,), (0.3081,))
120+
])),
80121
batch_size=batch_size, shuffle=True, **kwargs)
81122

82-
lr = 1e-2
83-
momentum = 0.5
84-
epochs = 10
123+
for i, (relustr, relu) in enumerate(relus.items()):
124+
print('--------------------- {} ---------------------'.format(relustr))
125+
model = MyConvNet(relu, relustr, **relu_kwargs[i]).to(device)
126+
optimizer = torch.optim.SGD(
127+
model.parameters(), lr=lr, momentum=momentum)
128+
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, schd_step)
85129

86-
model = MyConvNet().to(device)
87-
optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum)
130+
for epoch in range(epochs):
131+
train(model, device, train_dataloader, optimizer, epoch)
132+
test(model, device, test_dataloader, epoch)
133+
scheduler.step()
88134

89-
for epoch in range(epochs):
90-
train(model, device, train_dataloader, optimizer, epoch)
91-
test(model, device, test_dataloader)
135+
# torch.save(model.state_dict(), 'mnist_cnn.pt')
92136

93-
torch.save(model.state_dict(), 'mnist_cnn.pt')
94137

95138
if __name__ == '__main__':
96139
main()
140+
141+
if tensorboard_on:
142+
writer.close()

‎pytorch/samples/ResNetFinetune.py

+34-20
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@
22
from torchvision import models, datasets, transforms
33
import copy
44

5+
56
def set_parameter_requires_grad(model, feature_extract):
67
for param in model.parameters():
78
param.requires_grad = feature_extract
89

9-
def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
10+
11+
def initialize_model(model_name, num_classes, feature_extract,
12+
use_pretrained=True):
1013
if model_name == 'resnet':
1114
model_ft = models.resnet18(pretrained=use_pretrained)
1215
set_parameter_requires_grad(model_ft, feature_extract)
@@ -17,29 +20,34 @@ def initialize_model(model_name, num_classes, feature_extract, use_pretrained=Tr
1720
else:
1821
raise NotImplementedError
1922

23+
2024
def dataloader(batch_size):
21-
kwargs = {'num_workers': 1, 'pin_memory': True} if torch.cuda.is_available() else {}
25+
kwargs = {'num_workers': 1, 'pin_memory': True} \
26+
if torch.cuda.is_available() else {}
2227

2328
train_loader = torch.utils.data.DataLoader(
2429
datasets.MNIST('./data', train=True, download=True,
25-
transform=transforms.Compose([
26-
transforms.ToTensor(),
27-
transforms.Normalize((0.1307,), (0.3081,))
28-
])),
30+
transform=transforms.Compose([
31+
transforms.ToTensor(),
32+
transforms.Normalize((0.1307,), (0.3081,))
33+
])),
2934
batch_size=batch_size, shuffle=True, **kwargs)
3035
test_loader = torch.utils.data.DataLoader(
3136
datasets.MNIST('./data', train=False, download=True,
32-
transform=transforms.Compose([
33-
transforms.ToTensor(),
34-
transforms.Normalize((0.1307,), (0.3081,))
35-
])),
37+
transform=transforms.Compose([
38+
transforms.ToTensor(),
39+
transforms.Normalize((0.1307,), (0.3081,))
40+
])),
3641
batch_size=batch_size, shuffle=True, **kwargs)
3742

38-
return {'train':train_loader, 'test':test_loader}
43+
return {'train': train_loader, 'test': test_loader}
3944

40-
def train_model(model, epochs, batch_size, loss_fn, optimizer, device, dataloaders):
45+
46+
def train_model(model, epochs, batch_size, loss_fn, optimizer, device,
47+
dataloaders):
4148
val_acc_history = []
4249
best_model_wts = copy.deepcopy(model.state_dict())
50+
best_acc = 0
4351

4452
for epoch in range(epochs):
4553
for phase in ['train', 'test']:
@@ -52,7 +60,7 @@ def train_model(model, epochs, batch_size, loss_fn, optimizer, device, dataloade
5260
for data, target in dataloaders[phase]:
5361
data, targets = data.to(device), target.to(device)
5462
# resnet accepts images with 3 channels
55-
data = data.repeat(1,3,1,1)
63+
data = data.repeat(1, 3, 1, 1)
5664

5765
with torch.autograd.set_grad_enabled(phase == 'train'):
5866
outputs = model(data)
@@ -64,12 +72,14 @@ def train_model(model, epochs, batch_size, loss_fn, optimizer, device, dataloade
6472
loss.backward()
6573
optimizer.step()
6674
running_loss += loss.item() * data.size(0)
67-
running_corrects += torch.sum(preds.cpu().view(-1) == targets.cpu().view(-1))
75+
cnt = torch.sum(preds.cpu().view(-1) == targets.cpu().view(-1))
76+
running_corrects += cnt
6877

6978
epoch_loss = running_loss / len(dataloaders[phase].dataset)
70-
epoch_acc = running_corrects / len(dataloaders[phase].dataset)
79+
epoch_acc = running_corrects / len(dataloaders[phase].dataset)
7180

72-
print('Epoch: {}, phase: {}, loss: {}, acc: {}'.format(epoch, phase, epoch_loss, epoch_acc))
81+
print('Epoch: {}, phase: {}, loss: {}, acc: {}'.format(
82+
epoch, phase, epoch_loss, epoch_acc))
7383

7484
if phase == 'val' and epoch_acc > best_acc:
7585
best_acc = epoch_acc
@@ -79,6 +89,7 @@ def train_model(model, epochs, batch_size, loss_fn, optimizer, device, dataloade
7989
model.load_state_dict(best_model_wts)
8090
return model, val_acc_history
8191

92+
8293
def main():
8394
classes = 10
8495
epochs = 10
@@ -91,12 +102,15 @@ def main():
91102
# print(model_ft.fc.weight.requires_grad)
92103

93104
dataloaders = dataloader(batch_size)
94-
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
95-
model_ft.parameters()), lr=0.01, momentum=0.9)
105+
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
106+
model_ft.parameters()),
107+
lr=0.01,
108+
momentum=0.9)
96109
loss_fn = torch.nn.CrossEntropyLoss()
97110

98-
train_model(model_ft.to(device), epochs, batch_size,
111+
train_model(model_ft.to(device), epochs, batch_size,
99112
loss_fn, optimizer, device, dataloaders)
100113

114+
101115
if __name__ == "__main__":
102-
main()
116+
main()

‎pytorch/samples/TwoLayerFCSample.py

+6-4
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,24 @@
88
x = torch.randn(N, D_in).cuda()
99
y = torch.randn(N, D_out).cuda()
1010

11+
1112
class TwoLayerNet(torch.nn.Module):
1213
def __init__(self, D_in, H, D_out):
1314
super(TwoLayerNet, self).__init__()
1415
self.l1 = torch.nn.Linear(D_in, H)
1516
self.l2 = torch.nn.ReLU()
1617
self.l3 = torch.nn.Linear(H, D_out)
17-
18+
1819
def forward(self, x):
1920
x = self.l1(x)
2021
x = self.l2(x)
2122
x = self.l3(x)
2223
return x
2324

24-
#torch.nn.init.normal_(model[0].weight)
25-
#torch.nn.init.normal_(model[2].weight)
25+
# torch.nn.init.normal_(model[0].weight)
26+
# torch.nn.init.normal_(model[2].weight)
27+
28+
2629
model = TwoLayerNet(D_in, H, D_out).cuda()
2730

2831
loss_fn = torch.nn.MSELoss(reduction="sum")
@@ -42,4 +45,3 @@ def forward(self, x):
4245

4346
# update weights
4447
optimizer.step()
45-

0 commit comments

Comments
 (0)