深度学习

基本操作

构建数据库

dataset = torchvision.datasets.CIFAR10('data_cifari10', train=False, transform=torchvision.transforms.ToTensor(), download=True)

模型

# 例子 prtrained表示是否预先训练过
vgg16_false = torchvision.models.vgg16(pretrained=False)
vgg16_true = torchvision.models.vgg16(pretrained=True)

保存模型

# 方式1，模型结构+模型参数
torch.save(vgg16, '../../vgg16.pth')

# 方式2，模型参数
torch.save(vgg16.state_dic(), 'vgg16.pth')

加载模型

# 方式1，加载模型
vgg16 = torch.load('vgg16.pth')
# 方式2，加载模型
vgg16 = torchvision.model.vgg16(pretrained=False)
vgg16.load_state_dic(torch.load('vgg16.pth'))

整体模型

'''
Time : 2022/2/5 16:05 
File : train.py 
'''
import torch
import torchvision
from tensorboardX import SummaryWriter
from torch.utils.data import DataLoader
from model import *

# 准备数据集
train_data = torchvision.datasets.CIFAR10(root='data_cifari10', train=True, transform=torchvision.transforms.ToTensor()
                                          ,download=True)
test_data = torchvision.datasets.CIFAR10(root='data_cifari10', train=False, transform=torchvision.transforms.ToTensor()
                                          ,download=True)

# 数据集长度
train_data_size = len(train_data)
test_data_size = len(test_data)
print('训练数据集为：{}'.format(train_data_size))
print('测试数据集为：{}'.format(test_data_size))

# 理用dataloader加载数据集
train_loader = DataLoader(train_data, batch_size=64)
test_loader = DataLoader(test_data, batch_size=64)

# 创建网络模型
model = Cifar()

# 创建损失函数
loss_fn = nn.CrossEntropyLoss()

# 优化器
learning_rate = 1e-2
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

# 设置训练网络的一些参数
total_train_step = 0    # 记录训练次数
total_test_step = 0     # 记录测试次数
epoch = 10              # 训练轮数

# 添加tensorboard
writer = SummaryWriter("logs_train")

for i in range(epoch):
    print("--------第 {} 轮训练开始--------".format(i+1))

    # 训练步数开始
    for data in train_loader:
        imgs, targets = data
        outputs = model(imgs)
        loss = loss_fn(outputs, targets)

        # 优化器优化模型
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        total_train_step += 1
        if total_train_step % 100 == 0:
            print("训练次数：{} ，Loss：{}".format(total_train_step, loss.item()))
            writer.add_scalar("train_loss", loss.item(), total_train_step)

    # 测试步骤开始
    total_test_loss = 0
    with torch.no_grad():
        for data in test_loader:
            imgs, targets = data
            outputs = model(imgs)
            loss = loss_fn(outputs, targets)
            total_test_loss = total_test_loss + loss.item()
    print("整体测试集上的Loss：{}".format(total_test_loss))
    writer.add_scalar("test_loss", total_test_loss, total_test_step)
    total_test_step += 1

writer.close()

Anaconda

# 创建环境
conda create -n [name] python=3.9
# 删除环境
conda remove -n [name] --all
# 查看所有环境
conda info --envs
# 激活环境
conda activate [name]
# 添加镜像源
conda config --add
# 显示所有下载通道
conda config --show channels

pytorch

dataset

from torch.utils.data import Dataset

class XXX(Dataset)

tensorboard

from torch.utils.tensorboard import SummaryWriter / from tensorboardX import SummaryWriter

writer = SummaryWriter("logs")

writer.add_images
()  # 加图片 array格式，PIL.open()读取，用np.array()转
writer.add_scalar() # 加函数

writer.close()

torchvision库部分

datasets

import torchvision

trian_set = torchvision.datasets.xxxx[数据集](root='xxx/xxx', train=True,download=True)
test_set = torchvision.datasets.xxxx[数据集](root='xxx/xxx', train=False,download=True)

transforms

创建->调用->输入->输入->输出

from torchvision import transforms

# 例子
img_path = 'xxx'
img = Image.open(img_path) 

# ToTensor  转化为tensor类型
trans_totensor = transforms.ToTensor()
img_tensor = trans_totensor(img)

# Normalize
trans_norm = transforms.Normalize([0.5, 0.5, 0.5],[0.5, 0.5, 0.5])
img_norm = trans_norm(img)  #此处img为tensor类型

# Resize  修改尺寸
trans_resize = transforms.Resize((512, 512))
img_resize = trans_resize(img)  #此处img为PIL类型

# Compose 参数是列表[transforms.类型1, transforms.类型2, ...] 合并
trans_compose = transforms.Compose(
  [transforms.ToTensor(), transforms.Resize((512, 512))]
  )

DataLoader

import torchvision
from torch.utils.data import DataLoader

test_data = torchvision.datasets.xxxx[数据集](root='xxx/xxx', train=False,download=True)
# 数据集，batch_size一次拿取的次数，shuffle按序，drop_last最后的图片不足batch_size是否drop
test_loader = DataLoader(dataset=test_data, batch_size=4, shuffle=Ture, num_workers=0, drop_last=False)

torch.nn

import torch.nn as nn
import torch.nn.functional as F

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5)  #自写 
        self.conv2 = nn.Conv2d(20, 20, 5) #自写

    def forward(self, x):
        x = F.relu(self.conv1(x)) #自写
        return F.relu(self.conv2(x))  #自写

jupyter notebook

1. 在conda虚拟环境中conda install ipykernel下载ipy内核，或者在任意环境下conda install -n myenv ipykernel安装内核
2. 在虚拟环境下python -m ipykernel install --user --name=xxx将虚拟环境配置到jupyter notebook
3. 安装jupyter notebook
4. 启动

numpy

常用库是import numpy as np
随机一维np.arange(10)，随机一维矩阵
随机生成指定纬度矩阵：np.random.randn(2,3)生成2*3

pandas

1. 创建DataFrame df = pd.DataFrame(dict, index=)
2. 保存至excledf = pd.to_excle()
3. 读取excledf = pd.read_excle()参数包括header=1第二行开始读取，index_col=ID，skiprows=3,usecols='C:F/None',dtype=['':,'':,'':]跳过行数，从第几列到第几列

表示几行几列：df.shape
行数列表：df.columns
显示头几行：df.head()前三行df.head(3)
显示尾几行：df.tail()
索引ID列的第i个：df['ID'].at[i]

设置第几个标签列为index：df.set_index(xx,inplace=True)
pd.Series(li,index=[],name='')往表中加入一列/行数据为li，名字为name

from datetime import date,timedelta年月日库Date=date[2018,1,1]

import pandas as pd
# 读取数据
data = pd.read_csv(r"path")

data.columns  # 显示行数据
data.index  # 显示列数据

data.loc[2:5] # 取行（3~4行）
data.iloc[:,2:5]  # 取列（3~4列）
data['列']  # 取列

# 保存数据
data.to_csv('path') # data要是dataframe格式

# 补充
pd.dataframe([...]/{...},index='',columns='')

pd.array([])  #转化为数组
pd.sum[[a,b],axis=0].tolist() #数列或数组a、b逐一相加

图片读取库

PIL库

图片类型为PILImage格式

from PIL import Image
img_path = 'xxx'
img = Image.open(img_path)

爬虫

webdriver

from selenium import webdriver
option = webdriver.EdgeOptions()
option.binary_location = r'...\msedge.exe'  #浏览器exe路径
option.add_argument('headless')        # add_argument属性指定浏览器开启隐式
driver = webdriver.Edge(r'....\msedgedriver.exe', options=option) #driver路径
driver.get('www.xxx.com') #网页路径

html = driver.get_source
url = driver.current_url

urllib

import urllib.request
def getUrl(baseurl):
    url = baseurl
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
        (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62'
    }
    req = urllib.request.Request(url=url, headers=headers)
    response = urllib.request.urlopen(req)
    print(response.read().decode('utf-8'))

BeautifulSoup

from bs4 import BeatifulSoup
soup = BeautifulSoup(html, 'html.parser') # 解析html

soup.find_all()  # for k in soup.find_all('div', {'class': 'player'}): # 或...=...

soup.find()  # X = k.find('a').get_text()

文件操作

Json、txt

import json

dic = {}  

js = json.dumps(dic)   
file = open('test.txt', 'w')  
file.write(js)  
file.close()  

import json

file = open('test.txt', 'r') 
js = file.read()
# json.load()：是从文件中加载内容并转换成json；
# json.loads()：是将字符串转换成json
dic = json.loads(js)   
print(dic) 
file.close() 

dic = {} 

fw = open("test.txt",'w+')
fw.write(str(dic))      #把字典转化为str
fw.close()

fr = open("test.txt",'r+')
dic = eval(fr.read())   #读取的str转换为字典
print(dic)
fr.close()

正则、re模块

https://www.cnblogs.com/fozero/p/7868687.html

import re
re.compile('...')

re.search()  # 在一个字符串中搜索匹配正则表达式的第一个位置,返回match对象
re.match()  # 从一个字符串的开始位置起匹配正则表达式,返回match对象
re.findall()  # 搜索字符串,以列表类型返回全部能匹配的子串
re.split()  # 将一个字符串按照正则表达式匹配结果进行分割,返回列表类型
re.finditer() # 搜索字符串,返回一个匹配结果的迭代类型,每个迭代元素是match对象
re.sub()  # 在一个字符串中替换所有匹配正则表达式的子串,返回替换后的字符串

其它

os库

import os
from os import system

# 关机
os.system("shutdown -s -t  60 ")

# 重启
system("shutdown -r -t 100")

# path + '...'
os.path.jion('...')