使用FashionMNIST数据集， 对10个类别的“时装”图像进行分类，数据集中的样例图如下，其中每个小图对应一个样本。数据集中包含已经预先划分好的训练集和测试集，其中训练集共60,000张图像，测试集共10,000张图像。每张图像均为单通道黑白图像，大小为28*28pixel，分属10个类别。
fashion-mnist
本次学习的内容将按照一个完整的深度学习流程进行，主要步骤如下：
- 首先导入基础的包
- 配置训练环境和超参数
- 数据读入和加载
- 数据可视化查看，确认数据读入无误
- 构建模型
- 训练函数和验证函数封装
- 训练模型，查看训练集和验证集loss，准确率
- 保存模型
1.导入基础的包
# 1.导入基础的包
import os
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
- 配置GPU
由于自己的电脑是cpu，没有gpu，所以这里的gpu的内容注释掉了
# 2. 配置GPU，这里有两种方式
## 方案一：使用os.environ
# os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# 方案二：使用“device”，后续对要使用GPU的变量用.to(device)即可
device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")
## 配置其他超参数，如batch_size, num_workers, learning rate, 以及总的epochs
batch_size =128
num_workers = 0 # 对于Windows用户，这里应设置为0，否则会出现多线程错误
lr = 1e-4
epochs = 20
- 首先设置数据变换
# 3. 首先设置数据变换
from torchvision import transforms
image_size = 28
data_transform = transforms.Compose([
transforms.ToPILImage(),
# 这一步取决于后续的数据读取方式，如果使用内置数据集读取方式则不需要
transforms.Resize(image_size),
transforms.ToTensor()
])
- 数据读取，构建dataset类
## 读取方式一：使用torchvision自带数据集，下载可能需要一段时间，这种方式只适合特定案例，大部分都需要自己构建数据集
# from torchvision import datasets
# train_data = datasets.FashionMNIST(root='./', train=True, download=True, transform=data_transform)
# test_data = datasets.FashionMNIST(root='./', train=False, download=True, transform=data_transform)
## 4. 读取方式二：读入csv格式的数据，自行构建Dataset类
class FMDataset(Dataset):
def __init__(self, df, transform=None):
# 向类中传入外部参数，并定义样本集
self.df = df
self.transform = transform
self.images = df.iloc[:, 1:].values.astype(np.uint8)
self.labels = df.iloc[:, 0].values
def __len__(self):
# 返回数据集的样本数
return len(self.images)
def __getitem__(self, idx):
# 读取样本中的元素，进行一定变换，并返回所需要的数据
image = self.images[idx].reshape(28, 28, 1)
label = int(self.labels[idx])
if self.transform is not None:
image = self.transform(image)
else:
image = torch.tensor(image/255, dtype=torch.float)
label = torch.tensor(label, dtype=torch.long)
return image, label
# 数据读取
train_df = pd.read_csv('/Users/anker/Desktop/python_code/datasets/fashionMNIST/fashion-mnist_train.csv')
test_df = pd.read_csv('/Users/anker/Desktop/python_code/datasets/fashionMNIST/fashion-mnist_test.csv')
# 按dataset构建训练集和测试集
train_data = FMDataset(train_df, data_transform)
test_data = FMDataset(test_df, data_transform)
# 定义DataLoader类，以便在训练和测试时加载数据
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)
test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False, num_workers=num_workers)
5.对读入的数据进行可视化，验证读入数据是否准确
# 5.对读入的数据进行可视化，验证读入数据是否准确
%matplotlib inline
import matplotlib.pyplot as plt
image, label = next(iter(train_loader))
print(image.shape, label.shape)
print(label[0])
plt.imshow(image[0][0], cmap='gray')
# 输出
torch.Size([128, 1, 28, 28]) torch.Size([128])
tensor(2)
示例.png
- 模型设计
# 6. 模型设计，由于任务较为简单，这里我们手搭一个CNN，而不考虑当下各种模型的复杂结构，模型构建完成后，将模型放到GPU上用于训练。
from turtle import forward
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv = nn.Sequential(
nn.Conv2d(1, 32, 5),
nn.ReLU(),
nn.MaxPool2d(2, stride=2),
nn.Dropout(0.3),
nn.Conv2d(32, 64, 5),
nn.ReLU(),
nn.MaxPool2d(2, stride=2),
nn.Dropout(0.3)
)
self.fc = nn.Sequential(
nn.Linear(64*4*4, 512),
nn.ReLU(),
nn.Linear(512, 10)
)
def forward(self, x):
x = self.conv(x)
x = x.view(-1, 64*4*4)
x = self.fc(x)
# x = nn.functional.normalize(x)
return x
model = Net()
# model = model.cuda() # 实现模型从cpu到gpu的迁移
# model = nn.DataParallel(model).cuda() # 多卡训练时的写法
- 设定损失函数
# 7. 设定损失函数，
# 使用torch.nn模块自带的CrossEntropy损失
# PyTorch会自动把整数型的label转为one-hot型，用于计算CE loss
# 这里需要确保label是从0开始的，同时模型不加softmax层（使用logits计算）,这也说明了PyTorch训练中各个部分不是独立的，需要通盘考虑
criterion = nn.CrossEntropyLoss()
- 设定优化器
# 8. 设定优化器(对反向传播及梯度管理)，常使用的是Adam优化器
optimizer = optim.Adam(model.parameters(), lr=0.001)
- 对训练函数和测试分别封装成函数
# 9. 对训练函数和测试分别封装成函数，方便后续调用
# 训练和测试的主要区别在于：1.模型状态的设置 2.是否需要初始优化器 3.是否需要将loss传回网络 4.是否需要每步更新optimizer
# 测试或验证可以计算分类的准确率
def train(epoch):
model.train()
train_loss = 0
for data, label in train_loader:
# data, label = data.cuda(), label.cuda() # 将数据放到gpu
# 清零梯度
optimizer.zero_grad()
output = model(data)
loss = criterion(output, label) # 计算loss
loss.backward() # 反向传播
optimizer.step() # 参数更新
train_loss = train_loss + loss.item() * data.size(0)
train_loss = train_loss / len(train_loader.dataset)
print('epoch: {}\t training loss: {:.6f}'.format(epoch, train_loss))
# 验证集函数封装
def val(epoch):
model.eval()
val_loss = 0
gt_labels = []
pred_labels = []
with torch.no_grad():
for data, label in test_loader:
# data, label = data.cuda(), label.cuda() # 将数据放到gpu
output = model(data)
preds = torch.argmax(output, 1)
# gt_labels.append(label.cuda().data.numpy()) # 将数据从gpu中拿出来
gt_labels.append(label.cpu().data.numpy()) # 将数据从cpu中拿出来,存储起来
pred_labels.append(preds.cpu().data.numpy())
# 计算loss
loss = criterion(output, label)
val_loss = val_loss + loss.item() * data.size(0)
val_loss = val_loss / len(test_loader.dataset)
gt_labels, pred_labels = np.concatenate(gt_labels), np.concatenate(pred_labels)
acc = np.sum(gt_labels == pred_labels) / len(pred_labels)
print('epoch:{}\t Calidation loss: {:.6f}, Accuracy:{:.6f}'.format(epoch, val_loss, acc))
#验证代码是否正确
# val(1)
- 训练模型，查看loss
# 10. 训练模型，查看loss
from pathlib import Path
import os
for epoch in range(1, epochs):
train(epoch)
val(epoch)
# 模型保存
save_path = Path('./FahionModel_' + str(epoch) + '.pkl')
torch.save(model, save_path)
print('model saved {}'.format(save_path))
# 输出，这里只训练了10个epoch
epoch: 1 training loss: 0.669471
epoch: 1 Calidation loss: 0.463060, Accuracy: 0.834800
model saved as Fashion_model_1.pkl
epoch: 2 training loss: 0.436001
epoch: 2 Calidation loss: 0.365704, Accuracy: 0.868500
model saved as Fashion_model_2.pkl
epoch: 3 training loss: 0.371728
epoch: 3 Calidation loss: 0.321203, Accuracy: 0.883000
model saved as Fashion_model_3.pkl
epoch: 4 training loss: 0.336750
epoch: 4 Calidation loss: 0.286281, Accuracy: 0.894400
model saved as Fashion_model_4.pkl
epoch: 5 training loss: 0.315759
epoch: 5 Calidation loss: 0.287050, Accuracy: 0.897600
model saved as Fashion_model_5.pkl
epoch: 6 training loss: 0.297339
epoch: 6 Calidation loss: 0.262994, Accuracy: 0.905900
model saved as Fashion_model_6.pkl
epoch: 7 training loss: 0.282930
epoch: 7 Calidation loss: 0.248433, Accuracy: 0.909600
model saved as Fashion_model_7.pkl
epoch: 8 training loss: 0.267312
epoch: 8 Calidation loss: 0.240321, Accuracy: 0.912300
model saved as Fashion_model_8.pkl
epoch: 9 training loss: 0.256241
epoch: 9 Calidation loss: 0.235803, Accuracy: 0.912100
model saved as Fashion_model_9.pkl
epoch: 10 training loss: 0.246015
epoch: 10 Calidation loss: 0.243302, Accuracy: 0.910300
model saved as Fashion_model_10.pkl
以上就是一个深度学习模型的完整训练过程，实际在工程中还会有推理的部分，后续再说明。
