From 646cb3884c3a4564ce66716cbaa83d3432e88ba1 Mon Sep 17 00:00:00 2001 From: gpl Date: Sat, 28 Oct 2023 11:22:24 +0800 Subject: [PATCH] DL --- src/DeepLearning/2.1.1数据样本分析.py | 81 +++++++++++ .../2.1.2数据集长宽分析.py | 42 ++++++ src/DeepLearning/2.1.3.数据增强处理.py | 49 +++++++ .../2.1.4数据集平衡处理.py | 23 +++ .../2.1.5图像均值和方差.py | 28 ++++ src/DeepLearning/2.2.1生成数据集.py | 40 ++++++ src/DeepLearning/3.训练模型.py | 124 ++++++++++++++++ .../4.1单张图片模型测试.py | 77 ++++++++++ src/DeepLearning/5.1.1迁移学习.py | 135 ++++++++++++++++++ src/DeepLearning/5.1.2画loss曲线.py | 74 ++++++++++ src/DeepLearning/5.3_1模型验证.py | 75 ++++++++++ src/DeepLearning/5.3_2模型验证.py | 78 ++++++++++ 12 files changed, 826 insertions(+) create mode 100644 src/DeepLearning/2.1.1数据样本分析.py create mode 100644 src/DeepLearning/2.1.2数据集长宽分析.py create mode 100644 src/DeepLearning/2.1.3.数据增强处理.py create mode 100644 src/DeepLearning/2.1.4数据集平衡处理.py create mode 100644 src/DeepLearning/2.1.5图像均值和方差.py create mode 100644 src/DeepLearning/2.2.1生成数据集.py create mode 100644 src/DeepLearning/3.训练模型.py create mode 100644 src/DeepLearning/4.1单张图片模型测试.py create mode 100644 src/DeepLearning/5.1.1迁移学习.py create mode 100644 src/DeepLearning/5.1.2画loss曲线.py create mode 100644 src/DeepLearning/5.3_1模型验证.py create mode 100644 src/DeepLearning/5.3_2模型验证.py diff --git a/src/DeepLearning/2.1.1数据样本分析.py b/src/DeepLearning/2.1.1数据样本分析.py new file mode 100644 index 0000000..1475019 --- /dev/null +++ b/src/DeepLearning/2.1.1数据样本分析.py @@ -0,0 +1,81 @@ +''' + 数据样本分析 + 画出数据量条形图 + 画出图像分辨率散点图 +''' +import os +import PIL.Image as Image +import matplotlib.pyplot as plt +import numpy as np + +def plot_resolution(dataset_root_path): + img_size_list = [] # 存储图片长宽数据 + for root, dirs, files in os.walk(dataset_root_path): + for file_i in files: + file_i_full_path = os.path.join(root, file_i) + img_i = Image.open(file_i_full_path) + img_i_size = img_i.size # 获取单张图像的长宽 + img_size_list.append(img_i_size) + + print(img_size_list) # + + width_list = [img_size_list[i][0] for i in range(len(img_size_list))]#提取所有图片的宽度信息，构建一个新的列表 + height_list = [img_size_list[i][1] for i in range(len(img_size_list))]#提取所有图片的高度信息，构建一个新的列表 + + # print(width_list) # 640 + # print(height_list) # 346 + + plt.rcParams["font.sans-serif"] = ["SimHei"] # 设置中文字体 + plt.rcParams["font.size"] = 8 + plt.rcParams["axes.unicode_minus"] = False # 该语句解决图像中的“-”负号的乱码问题 + + plt.scatter(width_list, height_list, s=1) + plt.xlabel("宽") + plt.ylabel("高") + plt.title("图像宽高分布") + plt.show() + + +# 画出条形图 +def plot_bar(dataset_root_path): + + file_name_list = [] + file_num_list = [] + for root, dirs, files in os.walk(dataset_root_path): + if len(dirs) != 0: + for dir_i in dirs: + file_name_list.append(dir_i) + file_num_list.append(len(files)) + + file_num_list = file_num_list[1:] + # 求均值，并把均值以横线形式显示出来 + mean = np.mean(file_num_list) + print("mean = ", mean) + + bar_positions = np.arange(len(file_name_list)) + + fig, ax = plt.subplots() # 定义画的区间和子画 + ax.bar(bar_positions, file_num_list, 0.5) # 画柱图，参数：柱间的距离，柱的值，柱的宽度 + + ax.plot(bar_positions, [mean for i in bar_positions], color="red") # 显示平均值 + + plt.rcParams["font.sans-serif"] = ["SimHei"] # 设置中文字体 + plt.rcParams["font.size"] = 8 + plt.rcParams["axes.unicode_minus"] = False # 该语句解决图像中的“-”负号的乱码问题 + + ax.set_xticks(bar_positions) # 设置x轴的刻度 + ax.set_xticklabels(file_name_list, rotation=90) # 设置x轴的标签 + ax.set_ylabel("类别数量") + ax.set_title("数据分布图") + plt.show() + +if __name__ == '__main__': + + dataset_root_path = "dataset" + + plot_resolution(dataset_root_path) + + # plot_bar(dataset_root_path) + + + diff --git a/src/DeepLearning/2.1.2数据集长宽分析.py b/src/DeepLearning/2.1.2数据集长宽分析.py new file mode 100644 index 0000000..a396070 --- /dev/null +++ b/src/DeepLearning/2.1.2数据集长宽分析.py @@ -0,0 +1,42 @@ +from PIL import Image +import os + +dataset_root_path = "dataset" + +min = 200 # 短边 +max = 2000 # 长边 +ratio = 0.5 # 短边 / 长边 + +delete_list = [] # 所有图片的长宽数据 +for root,dirs,files in os.walk(dataset_root_path): + for file_i in files: + file_i_full_path = os.path.join(root, file_i)#构建当前文件的完整路径 + img_i = Image.open(file_i_full_path)#将其加载为一个图像对象 + img_i_size = img_i.size # 获取单张图像的长宽 + + # 删除单边过短的图片 + if img_i_size[0] max or img_i_size[1] > max: + print(file_i_full_path, " 不满足要求") + delete_list.append(file_i_full_path) + + # 删除宽高比例不当的图片 + long = img_i_size[0] if img_i_size[0] > img_i_size[1] else img_i_size[1] + short = img_i_size[0] if img_i_size[0] < img_i_size[1] else img_i_size[1] + + if short / long < ratio: + print(file_i_full_path, " 不满足要求",img_i_size[0],img_i_size[1]) + delete_list.append(file_i_full_path) + + +# print(delete_list) +for file_i in delete_list: + try: + print("正在删除",file_i) + os.remove(file_i) + except: + pass diff --git a/src/DeepLearning/2.1.3.数据增强处理.py b/src/DeepLearning/2.1.3.数据增强处理.py new file mode 100644 index 0000000..d997473 --- /dev/null +++ b/src/DeepLearning/2.1.3.数据增强处理.py @@ -0,0 +1,49 @@ +''' + 使用翻转进行数据增强 +''' +import os +import cv2 +import numpy as np + +# 水平翻转 +def Horizontal(image): + return cv2.flip(image,1,dst=None) #水平镜像 + +# 垂直翻转 +def Vertical(image): + return cv2.flip(image,0,dst=None) #垂直镜像 + +if __name__ == '__main__': + from_root = r"dataset" + save_root = r"enhance_dataset" + + threshold = 200 + + for a,b,c in os.walk(from_root): + for file_i in c: + file_i_path = os.path.join(a,file_i) + + split = os.path.split(file_i_path) + dir_loc = os.path.split(split[0])[1] + save_path = os.path.join(save_root,dir_loc) + + print(file_i_path) + print(save_path) + + if os.path.isdir(save_path) == False: + os.makedirs(save_path) + + img_i = cv2.imdecode(np.fromfile(file_i_path, dtype=np.uint8),-1) # 读取图片 + + cv2.imencode('.jpg', img_i)[1].tofile(os.path.join(save_path, file_i[:-5] + "_original.jpg")) # 保存图片 + + if len(c) < threshold: + + img_horizontal = Horizontal(img_i) + cv2.imencode('.jpg', img_horizontal)[1].tofile(os.path.join(save_path, file_i[:-5] + "_horizontal.jpg")) # 保存图片 + + img_vertical = Vertical(img_i) + cv2.imencode('.jpg', img_vertical)[1].tofile(os.path.join(save_path, file_i[:-5] + "_vertical.jpg")) # 保存图片 + + else: + pass diff --git a/src/DeepLearning/2.1.4数据集平衡处理.py b/src/DeepLearning/2.1.4数据集平衡处理.py new file mode 100644 index 0000000..50db109 --- /dev/null +++ b/src/DeepLearning/2.1.4数据集平衡处理.py @@ -0,0 +1,23 @@ +import os +import random + +img_root = r"enhance_dataset" +threshold = 300 + +for a,b,c in os.walk(img_root): + if len(c) > threshold: + delete_list = [] + for file_i in c: + file_i_full_path = os.path.join(a,file_i) + delete_list.append(file_i_full_path) + + random.shuffle(delete_list) + + print(delete_list) + delete_list = delete_list[threshold:] + for file_delete_i in delete_list: + os.remove(file_delete_i) + print("将会删除",file_delete_i) + + + diff --git a/src/DeepLearning/2.1.5图像均值和方差.py b/src/DeepLearning/2.1.5图像均值和方差.py new file mode 100644 index 0000000..b9b3d1b --- /dev/null +++ b/src/DeepLearning/2.1.5图像均值和方差.py @@ -0,0 +1,28 @@ +from torchvision.datasets import ImageFolder +import torch +from torchvision import transforms as T +from tqdm import tqdm + +transform = T.Compose([ + T.RandomResizedCrop(224), + T.ToTensor(), +]) + +def getStat(train_data): + train_loader = torch.utils.data.DataLoader( + train_data, batch_size=1, shuffle=False, num_workers=0, pin_memory=True) + + mean = torch.zeros(3) + std = torch.zeros(3) + for X, _ in tqdm(train_loader): + for d in range(3): + mean[d] += X[:, d, :, :].mean() # N, C, H ,W + std[d] += X[:, d, :, :].std() + mean.div_(len(train_data)) + std.div_(len(train_data)) + return list(mean.numpy()), list(std.numpy()) + + +if __name__ == '__main__': + train_dataset = ImageFolder(root=r'enhance_dataset', transform=transform) + print(getStat(train_dataset)) \ No newline at end of file diff --git a/src/DeepLearning/2.2.1生成数据集.py b/src/DeepLearning/2.2.1生成数据集.py new file mode 100644 index 0000000..98011af --- /dev/null +++ b/src/DeepLearning/2.2.1生成数据集.py @@ -0,0 +1,40 @@ +import os +import random + +train_ratio = 0.9 #训练集比例 +test_ratio = 1-train_ratio #测试集比例 + +rootdata = r"dataset" + +train_list, test_list = [],[] + +class_flag = -1#初始类别标签 +t=1 +for a,b,c in os.walk(rootdata):#目录路径，子目录名，文件名 + # if t==1: + # # print(a) + # print(b) + # print(len(c)) + # t=-1 + + for i in range(0, int(len(c)*train_ratio)):#根据训练集比例确定训练集的文件数量 + train_data = os.path.join(a, c[i])+'\t'+str(class_flag)+'\n' + # print('666'+train_data) + train_list.append(train_data) + + for i in range(int(len(c) * train_ratio), len(c)): + test_data = os.path.join(a, c[i]) + '\t' + str(class_flag)+'\n' + test_list.append(test_data) + + class_flag += 1 + +random.shuffle(train_list)#随机打乱训练集列表 train_list 中的元素顺序 +random.shuffle(test_list) + +with open('train.txt','w',encoding='UTF-8') as f: + for train_img in train_list: + f.write(str(train_img))#将训练集中的文件路径和类别标签写入文件 'train.txt'，将其转换为字符串形式并写入文件。 + +with open('test.txt','w',encoding='UTF-8') as f: + for test_img in test_list: + f.write(test_img) \ No newline at end of file diff --git a/src/DeepLearning/3.训练模型.py b/src/DeepLearning/3.训练模型.py new file mode 100644 index 0000000..de7f7ed --- /dev/null +++ b/src/DeepLearning/3.训练模型.py @@ -0,0 +1,124 @@ +import time +import torch +from torch import nn +from torch.utils.data import DataLoader +from utils import LoadData,WriteData + +from torchvision.models import resnet18 + +def train(dataloader, model, loss_fn, optimizer,device): + size = len(dataloader.dataset) + avg_loss = 0 + # 从数据加载器中读取batch（一次读取多少张，即批次数），X(图片数据)，y（图片真实标签）。 + for batch, (X, y) in enumerate(dataloader):#固定格式：batch：第几批数据，不是批次大小，（X，y）：数值用括号 + # 将数据存到显卡 + X, y = X.to(device), y.to(device) + # 得到预测的结果pred + pred = model(X) + loss = loss_fn(pred, y) + avg_loss += loss + # 反向传播，更新模型参数 + optimizer.zero_grad() + loss.backward() + optimizer.step() + # 每训练10次，输出一次当前信息 + if batch % 10 == 0: + loss, current = loss.item(), batch * len(X) + print(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") + + # 当一个epoch完了后返回平均 loss + avg_loss /= size + avg_loss = avg_loss.detach().cpu().numpy() + return avg_loss + + +def validate(dataloader, model, loss_fn, device): + size = len(dataloader.dataset) + # 将模型转为验证模式 + model.eval() + # 初始化test_loss 和 correct， 用来统计每次的误差 + test_loss, correct = 0, 0 + # 测试时模型参数不用更新，所以no_gard() + # 非训练， 推理期用到 + with torch.no_grad(): + # 加载数据加载器，得到里面的X（图片数据）和y(真实标签） + for X, y in dataloader: + # 将数据转到GPU + X, y = X.to(device), y.to(device) + # 将图片传入到模型当中就，得到预测的值pred + pred = model(X) + # 计算预测值pred和真实值y的差距 + test_loss += loss_fn(pred, y).item() + # 统计预测正确的个数(针对分类) + correct += (pred.argmax(1) == y).type(torch.float).sum().item() + test_loss /= size + correct /= size + print(f"correct = {correct}, Test Error: \n Accuracy: {(100 * correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + return correct, test_loss + + +if __name__=='__main__': + batch_size = 20#设置批次大小，每次训练时用于更新模型的样本数量 + + # # 给训练集和测试集分别创建一个数据集加载器 + train_data = LoadData("train.txt", True)#true为训练集 + valid_data = LoadData("test.txt", False)#false为测试集 + + #批量加载数据，num_workers并行工作线程数， + train_dataloader = DataLoader(dataset=train_data, num_workers=4, pin_memory=True, batch_size=batch_size, shuffle=True) + valid_dataloader = DataLoader(dataset=valid_data, num_workers=4, pin_memory=True, batch_size=batch_size) + + # 如果显卡可用，则用显卡进行训练 + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using {device} device") + + #创建一个ResNet-18模型实例 + model = resnet18(num_classes=55) + + model = model.to(device) + state_dict = torch.load(r"output/resnet18_no_pretrain_best.pth") + # print("state_dict = ",state_dict) + model.load_state_dict(state_dict) + + # 定义损失函数，计算相差多少，交叉熵， + loss_fn = nn.CrossEntropyLoss() + + # 定义优化器，随机梯度下降法 + learning_rate = 1e-3 + optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) + + #设置训练总轮数，初始化变量loss用于跟踪最低的平均损失值 + epochs = 50 + loss_ = 10 + save_root = "output/" + + + for t in range(epochs): + print(f"Epoch {t + 1}\n-------------------------------") + time_start = time.time() + avg_loss = train(train_dataloader, model, loss_fn, optimizer, device) + time_end = time.time() + print(f"train time: {(time_end - time_start)}") + # (dataloader, model, loss_fn, device)jif + #调用 validate 函数进行验证集上的性能评估，计算验证集的准确率和损失 + val_accuracy, val_loss = validate(valid_dataloader, model,loss_fn, device) + # 将训练和验证的结果写入文件写入数据 + WriteData(save_root + "resnet18_no_pretrain.txt", + "epoch", t,#epoch数 + "train_loss", avg_loss,#训练损失 + "val_loss", val_loss,#验证损失 + "val_accuracy", val_accuracy)#验证准确率 + #如果当前epoch是5的倍数，保存模型参数文件，以便后续恢复或继续训练。 + if t % 5 == 0: + torch.save(model.state_dict(), save_root + "resnet18_no_pretrain_epoch" + str(t) + "_loss_" + str(avg_loss) + ".pth") + + #每个epoch都保存模型参数文件，以便后续分析和恢复。 + torch.save(model.state_dict(), save_root + "resnet18_no_pretrain_last.pth") + + #如果当前epoch的平均损失小于 loss_，则更新 loss_ 为当前损失值，并保存当前模型参数作为最佳模型。 + if avg_loss < loss_: + loss_ = avg_loss + torch.save(model.state_dict(), save_root + "resnet18_no_pretrain_best.pth") + + torch.cuda.empty_cache() + diff --git a/src/DeepLearning/4.1单张图片模型测试.py b/src/DeepLearning/4.1单张图片模型测试.py new file mode 100644 index 0000000..d41a1f8 --- /dev/null +++ b/src/DeepLearning/4.1单张图片模型测试.py @@ -0,0 +1,77 @@ +''' + 单图测试 +''' + +import torch +from torchvision.models import resnet18 +from PIL import Image +import torchvision.transforms as transforms +import os + +transform_BZ= transforms.Normalize( + mean=[0.46402064, 0.45047238, 0.37801373], # 取决于数据集 + std=[0.2007732, 0.196271, 0.19854763] +) + + +def padding_black(img,img_size = 512): # 如果尺寸太小可以扩充 + w, h = img.size + scale = img_size / max(w, h) + img_fg = img.resize([int(x) for x in [w * scale, h * scale]]) + size_fg = img_fg.size + size_bg = img_size + img_bg = Image.new("RGB", (size_bg, size_bg)) + img_bg.paste(img_fg, ((size_bg - size_fg[0]) // 2, + (size_bg - size_fg[1]) // 2)) + img = img_bg + return img + +if __name__=='__main__': + + img_path = r'enhance_dataset/有害垃圾_药片/img_药片_7_vertical.jpg' + + #创建了一个图像转换操作的组合，该组合将应用于输入图像。 + #这些操作包括将图像调整大小为512x512像素，将图像转换为张量 + val_tf = transforms.Compose([ ##简单把图片压缩了变成Tensor模式 + transforms.Resize(512), + transforms.ToTensor(), + transform_BZ # 标准化操作 + ]) + + + + # 如果显卡可用，则用显卡进行训练 + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using {device} device") + + finetune_net = resnet18(num_classes=55).to(device) + #创建ResNet-18深度学习模型 + #分类成55个不同的类别 + + state_dict = torch.load(r"output/resnet18_no_pretrain_best.pth") + # print("state_dict = ",state_dict) + finetune_net.load_state_dict(state_dict) + #最后，这一行将模型设置为评估模式，这表示模型将在推理模式下运行，不进行梯度计算。 + #这对于推断和预测任务非常有用，因为在这些情况下不需要计算梯度。模型在评估模式下运行时，通常更加高效。 + finetune_net.eval() + with torch.no_grad(): + + # finetune_net.to(device) + img = Image.open(img_path) # 打开图片 + img = img.convert('RGB') # 转换为RGB 格式 + img = padding_black(img) + img = val_tf(img) + img_tensor = torch.unsqueeze(img, 0) # N,C,H,W, ; C,H,W + + img_tensor = img_tensor.to(device) + result = finetune_net(img_tensor) + # print("result = ",result.argmax(1)) + + + id = result.argmax(1).item() + + file_list=[] + for a,b,c in os.walk("dataset"): + if len(b) != 0: + file_list = b + print("预测结果为：",file_list[id]) diff --git a/src/DeepLearning/5.1.1迁移学习.py b/src/DeepLearning/5.1.1迁移学习.py new file mode 100644 index 0000000..0ceea71 --- /dev/null +++ b/src/DeepLearning/5.1.1迁移学习.py @@ -0,0 +1,135 @@ +''' + 加载pytorch自带的模型，从头训练自己的数据 +''' +import time +import torch +from torch import nn +from torch.utils.data import DataLoader +from utils import LoadData,WriteData + +from torchvision.models import resnet18, resnet34,resnet50, resnet101, resnet152 ,mobilenet_v2 # ResNet系列 + +def train(dataloader, model, loss_fn, optimizer,device): + size = len(dataloader.dataset) + avg_loss = 0 + # 从数据加载器中读取batch（一次读取多少张，即批次数），X(图片数据)，y（图片真实标签）。 + for batch, (X, y) in enumerate(dataloader):#固定格式：batch：第几批数据，不是批次大小，（X，y）：数值用括号 + + # print(size) + # 将数据存到显卡 + X, y = X.to(device), y.to(device) + # 得到预测的结果pred + pred = model(X) + loss = loss_fn(pred, y) + avg_loss += loss + # 反向传播，更新模型参数 + optimizer.zero_grad() + loss.backward() + optimizer.step() + # 每训练10次，输出一次当前信息 + if batch % 10 == 0: + loss, current = loss.item(), batch * len(X) + print(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") + + # 当一个epoch完了后返回平均 loss + avg_loss /= size + avg_loss = avg_loss.detach().cpu().numpy() + return avg_loss + + +def validate(dataloader, model, loss_fn, device): + size = len(dataloader.dataset) + # 将模型转为验证模式 + model.eval() + # 初始化test_loss 和 correct， 用来统计每次的误差 + test_loss, correct = 0, 0 + # 测试时模型参数不用更新，所以no_gard() + # 非训练， 推理期用到 + with torch.no_grad(): + # 加载数据加载器，得到里面的X（图片数据）和y(真实标签） + + for X, y in dataloader: + # 将数据转到GPU + X, y = X.to(device), y.to(device) + # 将图片传入到模型当中就，得到预测的值pred + pred = model(X) + # 计算预测值pred和真实值y的差距 + test_loss += loss_fn(pred, y).item() + # 统计预测正确的个数(针对分类) + correct += (pred.argmax(1) == y).type(torch.float).sum().item() + test_loss /= size + correct /= size + print(f"correct = {correct}, Test Error: \n Accuracy: {(100 * correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + return correct, test_loss + + +if __name__=='__main__': + batch_size = 32 + + # # 给训练集和测试集分别创建一个数据集加载器 + train_data = LoadData("train.txt", True) + valid_data = LoadData("test.txt", False) + + + train_dataloader = DataLoader(dataset=train_data, num_workers=4, pin_memory=True, batch_size=batch_size, shuffle=True) + valid_dataloader = DataLoader(dataset=valid_data, num_workers=4, pin_memory=True, batch_size=batch_size) + + # 如果显卡可用，则用显卡进行训练 + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using {device} device") + + + finetune_net = resnet18(pretrained=True) + + + finetune_net.fc = nn.Linear(finetune_net.fc.in_features, 55) + nn.init.xavier_normal_(finetune_net.fc.weight) + + parms_1x = [value for name, value in finetune_net.named_parameters() + if name not in ["fc.weight", "fc.bias"]] + # 最后一层10倍学习率 + parms_10x = [value for name, value in finetune_net.named_parameters() + if name in ["fc.weight", "fc.bias"]] + + finetune_net = finetune_net.to(device) + # 定义损失函数，计算相差多少，交叉熵， + loss_fn = nn.CrossEntropyLoss() + + # 定义优化器，用来训练时候优化模型参数，随机梯度下降法 + learning_rate = 1e-4 + optimizer = torch.optim.Adam([ + { + 'params': parms_1x + }, + { + 'params': parms_10x, + 'lr': learning_rate * 10 + }], lr=learning_rate) + + epochs = 50 + loss_ = 10 + save_root = "output/" + + + + for t in range(epochs): + print(f"Epoch {t + 1}\n-------------------------------") + time_start = time.time() + avg_loss = train(train_dataloader, finetune_net, loss_fn, optimizer, device) + time_end = time.time() + print(f"train time: {(time_end - time_start)}") + + val_accuracy, val_loss = validate(valid_dataloader, finetune_net,loss_fn, device) + # 写入数据 + WriteData(save_root + "resnet18_e.txt", + "epoch", t, + "train_loss", avg_loss, + "val_loss", val_loss, + "val_accuracy", val_accuracy) + if t % 5 == 0: + torch.save(finetune_net.state_dict(), save_root + "resnet18_e_epoch" + str(t) + "_loss_" + str(avg_loss) + ".pth") + torch.save(finetune_net.state_dict(), save_root + "resnet18_e_last.pth") + if avg_loss < loss_: + loss_ = avg_loss + torch.save(finetune_net.state_dict(), save_root + "resnet18_e_best.pth") + diff --git a/src/DeepLearning/5.1.2画loss曲线.py b/src/DeepLearning/5.1.2画loss曲线.py new file mode 100644 index 0000000..64b42de --- /dev/null +++ b/src/DeepLearning/5.1.2画loss曲线.py @@ -0,0 +1,74 @@ +# -*-coding:utf-8-*- +from matplotlib import pyplot as plt +import numpy as np + + +def ReadData(data_loc): + epoch_list = [] + train_loss_list = [] + test_loss_list = [] + test_accuracy_list = [] + + # open(data_loc,"r").readlines() + with open(data_loc, "r") as f: + linedata = f.readlines() + + for line_i in linedata: + data = line_i.split('\t') + print("data = ", data) + epoch_i , train_loss_i,test_loss_i,test_accuracy_i =data[1], data[3],data[5],data[7] + epoch_list.append(int(epoch_i)) + train_loss_list.append(float(train_loss_i)) + test_loss_list.append(float(test_loss_i)) + test_accuracy_list.append(float(test_accuracy_i)) + + # print(epoch_list) + # print(train_loss_list) + # print(test_loss_list) + # print(test_accuracy_list) + return epoch_list, train_loss_list ,test_loss_list,test_accuracy_list + + + +def DrawLoss(train_loss_list,train_loss_list_2): + plt.style.use('dark_background') + plt.title("Loss") + plt.xlabel("epoch") + plt.ylabel("loss") + + train_loss_list = train_loss_list[:10] + + epoch_list = [i for i in range(len(train_loss_list))] + + p1, = plt.plot(epoch_list, train_loss_list, linewidth=3) + p2, = plt.plot(epoch_list, train_loss_list_2, linewidth=3) + + plt.legend([p1, p2], ["with pretrain", "no pretrain"]) + plt.show() + +def DrawAcc(train_loss_list,train_loss_list_2): + plt.style.use('dark_background') + plt.title("Accuracy") + plt.xlabel("epoch") + plt.ylabel("accuracy") + + train_loss_list = train_loss_list[:10] + + epoch_list = [i for i in range(len(train_loss_list))] + + p1, = plt.plot(epoch_list, train_loss_list, linewidth=3) + p2, = plt.plot(epoch_list, train_loss_list_2, linewidth=3) + + plt.legend([p1, p2], ["with pretrain", "no pretrain"]) + plt.show() + +if __name__ == '__main__': + data_1_loc = "output/resnet18.txt" + data_2_loc = "output/resnet18_no_pretrain.txt" + + _, train_loss_list ,test_loss_list,test_accuracy_list = ReadData(data_1_loc) + _, train_loss_list_2 ,test_loss_list_2,test_accuracy_list_2 = ReadData(data_2_loc) + + DrawLoss(train_loss_list,train_loss_list_2) + + DrawAcc(test_accuracy_list,test_accuracy_list_2) diff --git a/src/DeepLearning/5.3_1模型验证.py b/src/DeepLearning/5.3_1模型验证.py new file mode 100644 index 0000000..779dfcd --- /dev/null +++ b/src/DeepLearning/5.3_1模型验证.py @@ -0,0 +1,75 @@ + +import torch +from torch.utils.data import DataLoader +from utils import LoadData,WriteData +import torch.nn as nn +from torchvision.models import resnet18 +from tqdm import tqdm + +import os + +import pandas as pd + + +def test(dataloader, model, device): + pred_list = [] + # 将模型转为验证模式 + model.eval() + # 测试时模型参数不用更新，所以no_gard() + # 非训练， 推理期用到 + with torch.no_grad(): + # 加载数据加载器，得到里面的X（图片数据）和y(真实标签） + for X, y in tqdm(dataloader): + # 将数据转到GPU + X, y = X.to(device), y.to(device) + # 将图片传入到模型当中就，得到预测的值pred + pred = model(X) + pred_softmax = torch.softmax(pred, 1).cpu().numpy() + pred_list.append(pred_softmax.tolist()[0]) + return pred_list + + +if __name__=='__main__': + batch_size = 1 + + # # 给训练集和测试集分别创建一个数据集加载器 + test_data = LoadData("test.txt", False) + + test_dataloader = DataLoader(dataset=test_data, num_workers=4, pin_memory=True, batch_size=batch_size) + + # 如果显卡可用，则用显卡进行训练 + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using {device} device") + + + model = resnet18(num_classes=55) + model.load_state_dict(torch.load("output/resnet18_e_best.pth")) + model.to(device) + + # 定义损失函数，计算相差多少，交叉熵， + # loss_fn = nn.CrossEntropyLoss() + + ''' + 获取结果 + ''' + # 获取模型输出 + pred_list = test(test_dataloader, model,device) + print("pred_list = ", pred_list) + + ''' + 获取文件夹列表 + ''' + file_name_list = [] + data_root = r"enhance_dataset" + for a,b,c in os.walk(data_root): + if len(b) != 0: + print(b) + file_name_list = b + + df_pred = pd.DataFrame(data=pred_list, columns=file_name_list) + print(df_pred) + df_pred.to_csv('pred_result.csv', encoding='gbk', index=False) + + + + diff --git a/src/DeepLearning/5.3_2模型验证.py b/src/DeepLearning/5.3_2模型验证.py new file mode 100644 index 0000000..4579120 --- /dev/null +++ b/src/DeepLearning/5.3_2模型验证.py @@ -0,0 +1,78 @@ +from sklearn.metrics import * # pip install scikit-learn +import matplotlib.pyplot as plt # pip install matplotlib +import pandas as pd # pip install pandas +import os + +''' +读取数据 + +需要读取模型输出的标签（predict_label）以及原本的标签（true_label） + +''' +target_loc = "./test.txt" # 真实标签所在的文件 +target_data = pd.read_csv(target_loc, sep="\t", names=["loc","type"]) +true_label = [i for i in target_data["type"]] + + +predict_loc = "./pred_result.csv" # 3.ModelEvaluate.py生成的文件 + +predict_data = pd.read_csv(predict_loc,encoding="GBK")#,index_col=0) + +predict_label = predict_data.to_numpy().argmax(axis=1) + +predict_score = predict_data.to_numpy().max(axis=1) + +''' + 常用指标：精度，查准率，召回率，F1-Score +''' +# 精度，准确率， 预测正确的占所有样本种的比例 +accuracy = accuracy_score(true_label, predict_label) +print("精度: ",accuracy) + +# 查准率P（准确率），precision(查准率)=TP/(TP+FP) + +precision = precision_score(true_label, predict_label, labels=None, pos_label=1, average='macro') # 'micro', 'macro', 'weighted' +print("查准率P: ",precision) + +# 查全率R（召回率），原本为对的，预测正确的比例；recall(查全率)=TP/(TP+FN) +recall = recall_score(true_label, predict_label, average='macro') # 'micro', 'macro', 'weighted' +print("召回率: ",recall) + +# F1-Score +f1 = f1_score(true_label, predict_label, average='macro') # 'micro', 'macro', 'weighted' +print("F1 Score: ",f1) + + +''' +混淆矩阵 +''' +label_names = [] +data_root = r"./enhance_dataset" +for a,b,c in os.walk(data_root): + if len(b) != 0: + print(b) + label_names = b + +confusion = confusion_matrix(true_label, predict_label, labels=[i for i in range(len(label_names))]) + + +plt.matshow(confusion, cmap=plt.cm.Oranges) # Greens, Blues, Oranges, Reds + +plt.rcParams["font.sans-serif"] = ["SimHei"] # 设置中文字体 +plt.rcParams["font.size"] = 8 +plt.rcParams["axes.unicode_minus"] = False # 该语句解决图像中的“-”负号的乱码问题 + +plt.colorbar() +for i in range(len(confusion)): + for j in range(len(confusion)): + plt.annotate(confusion[j,i], xy=(i, j), horizontalalignment='center', verticalalignment='center') +plt.ylabel('True label') +plt.xlabel('Predicted label') + + +plt.xticks(range(len(label_names)), label_names,rotation=270) +plt.yticks(range(len(label_names)), label_names) +plt.title("Confusion Matrix") +plt.show() + +