目录
- 最近帮朋友做一个图片数据分类的项目,借鉴了大佬的代码,修修改改,话不多说直接上代码!
一、图片转batch文件

import numpy as np
from PIL import Image
import operator
from os import listdir
import sys
import pickle
import random
data = {}
list1 = []
list2 = []
list3 = []
def img_tra():
for k in range(0, num):
currentpath = folder + "/" + imglist[k]
im = Image.open(currentpath)
x_s = 32
y_s = 32
out = im.resize((x_s, y_s), Image.ANTIALIAS)
out.save(folder_ad + "/" + str(imglist[k]))
def addWord(theIndex, word, adder):
theIndex.setdefault(word, []).append(adder)
def seplabel(fname):
filestr = fname.split(".")[0]
label = int(filestr.split("_")[0])
return fname
def mkcf():
global data
global list1
global list2
global list3
for k in range(0, num):
currentpath = folder_ad + "/" + imglist[k]
im = Image.open(currentpath)
im = im.convert('L')
with open(binpath, 'a') as f:
for i in range(0, 32):
for j in range(0, 32):
cl = im.getpixel((i, j))
print(cl)
list1.append(cl[0])
for i in range(0, 32):
for j in range(0, 32):
cl = im.getpixel((i, j))
list1.append(cl[1])
for i in range(0, 32):
for j in range(0, 32):
cl = im.getpixel((i, j))
list1.append(cl[2])
list2.append(list1)
list1 = []
f.close()
print("image" + str(k + 1) + "saved.")
list3.append(imglist[k].encode('utf-8'))
arr2 = np.array(list2, dtype=np.uint8)
data['batch_label'.encode('utf-8')] = 'batch_1'.encode('utf-8')
data.setdefault('labels'.encode('utf-8'), label)
data.setdefault('data'.encode('utf-8'), arr2)
data.setdefault('filenames'.encode('utf-8'), list3)
output = open(binpath, 'wb')
pickle.dump(data, output)
output.close()
name = 'test'
folder_ad = r"D:\lhh\python\demo\图片制作数据集\shujuji\size54_50pics\{}".format(name)
imglist = listdir(folder_ad)
num = len(imglist)
label = []
for i in range(0, num):
label.append(seplabel(imglist[i]))
binpath = r"D:\lhh\python\demo\图片制作数据集\{}batch".format(name)
mkcf()
- 生成batch文件

二、batch文件转json对应图片
- 读取batch文件,做成对应的json文件导入

import torch
import torchvision
import numpy as np
import cv2
import os
import json
def unpickle(file):
import pickle
with open(file, 'rb') as fo:
dict = pickle.load(fo, encoding='bytes')
return dict
anno_loc = r'D:\lhh\python\demo\图片制作数据集\shujuji\annotations'
if os.path.exists(anno_loc) == False:
os.mkdir(anno_loc)
train_filenames = []
train_annotations = []
test_filenames = []
test_annotations= []
def eye_annotations(file_dir):
print('creat train_img annotations')
for i in range(1,5):
data_name = file_dir + '/' + 'data_batch_' + str(i)
data_dict = unpickle(data_name)
print(data_name + ' is processing')
for j in range(10):
img_name = str(data_dict[b'labels'][j])
img_annotations = data_dict[b'labels'][j]
train_filenames.append(img_name)
train_annotations.append(img_annotations)
print(data_name + ' is done')
test_data_name = file_dir + '/test_batch'
print(test_data_name + ' is processing')
test_dict = unpickle(test_data_name)
for m in range(10):
testimg_name = str(test_dict[b'labels'][m])
testimg_annotations = test_dict[b'labels'][m]
test_filenames.append(testimg_name)
test_annotations.append(testimg_annotations)
print(test_data_name + ' is done')
print('Finish file processing')
if __name__ == '__main__':
file_dir = 'D:\lhh\python\demo\图片制作数据集\shujuji'
eye_annotations(file_dir)
train_annot_dict = {
'images': train_filenames,
'categories': train_annotations
}
test_annot_dict = {
'images':test_filenames,
'categories':test_annotations
}
train_json = json.dumps(train_annot_dict)
print(train_json)
train_file = open(r'D:\lhh\python\demo\图片制作数据集\shujuji\annotations\eye_train.json', 'w')
train_file.write(train_json)
train_file.close()
test_json =json.dumps(test_annot_dict)
test_file = open(r'D:\lhh\python\demo\图片制作数据集\shujuji\annotations\eye_test.json','w')
test_file.write(test_json)
test_file.close()
print('annotations have writen to json file')
三、模型训练
'''
import json
import matplotlib.pyplot as plt
import numpy as np
from torch.utils.data import Dataset,DataLoader
'''
import torch
import torchvision
from torch.optim import lr_scheduler
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.utils.data import DataLoader
import torchvision.datasets as dsets
import torchvision.transforms as trans
import time
import matplotlib.pyplot as plt
import numpy as np
from torch.utils.data import Dataset
import json
class EYEIMG(Dataset):
def __init__(self, root, train=True, transform=None, target_transform=None):
super(EYEIMG, self).__init__()
self.train = train
self.transform = transform
self.target_transform = target_transform
if self.train:
file_annotation = root + '/annotations/eye_train.json'
img_folder = root + '/data/train_eye/'
else:
file_annotation = root + '/annotations/eye_test.json'
img_folder = root + '/data/test_eye/'
fp = open(file_annotation, 'r')
data_dict = json.load(fp)
assert len(data_dict['images']) == len(data_dict['categories'])
num_data = len(data_dict['images'])
self.filenames = []
self.labels = []
self.img_folder = img_folder
for i in range(num_data):
self.filenames.append(data_dict['images'][i])
self.labels.append(data_dict['categories'][i])
def __getitem__(self, index):
img_name = self.img_folder + self.filenames[index]
label = self.labels[index]
img = plt.imread(img_name)
img = self.transform(img)
return img, label
def __len__(self):
return len(self.filenames)
BATCH_SIZE = 10
nepochs = 40
LR = 0.001
loss_func = nn.CrossEntropyLoss()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
def get_mean_std(dataset, ratio=0.01):
dataloader = DataLoader(dataset, batch_size=int(len(dataset)), shuffle=True, )
train = iter(dataloader).next()[0]
mean = np.mean(train.numpy(), axis=(0, 1, 2, 3))
std = np.std(train.numpy(), axis=(0, 1, 2, 3))
return mean, std
train_set = EYEIMG(root=r'D:\lhh\python\demo\图片制作数据集\shujuji',
train=True,
transform=trans.ToTensor())
test_set = EYEIMG(root=r'D:\lhh\python\demo\图片制作数据集\shujuji',
train=False,
transform=trans.ToTensor())
mean_train, std_train = get_mean_std(train_set)
mean_test, std_test = get_mean_std(test_set)
print(mean_train, std_train, mean_test, std_test)
n_train_samples = 40
train_set = EYEIMG(root=r'D:\lhh\python\demo\图片制作数据集\shujuji',
transform=trans.Compose([
trans.ToPILImage(),
trans.RandomHorizontalFlip(),
trans.ToTensor(),
trans.Normalize(mean_train, std_train)
]))
train_dl = DataLoader(train_set,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=0)
test_set = EYEIMG(root='D:\lhh\python\demo\图片制作数据集\shujuji',
transform=trans.Compose([
trans.ToPILImage(),
trans.ToTensor(),
trans.Normalize(mean_test, std_test)
]))
test_dl = DataLoader(test_set,
batch_size=BATCH_SIZE,
num_workers=0)
train_data = DataLoader(train_set,
batch_size=5,
shuffle=True,
)
def imshow(img):
img = img / 2 + 0.5
npimg = img.numpy()
plt.figure(figsize=(64, 64))
plt.imshow(np.transpose(npimg, (1, 2, 0)))
plt.xticks(())
plt.yticks(())
dataiter = iter(train_data)
images, labels = dataiter.next()
imshow(torchvision.utils.make_grid(images))
classes = ('0', '1', '2', '3')
print(labels)
print(' '.join('%5s' % classes[int(labels[j].split('_')[0])] for j in range(5)))
def eval(model, loss_func, dataloader):
model.eval()
loss, accuracy = 0, 0
with torch.no_grad():
for batch_x, batch_y in dataloader:
batch_x, batch_y = batch_x.to(device), batch_y
logits = model(batch_x)
batch_y = [int(i.split('_')[0]) for i in batch_y]
batch_y = torch.Tensor(batch_y).long()
error = loss_func(logits, batch_y)
loss += error.item()
probs, pred_y = logits.data.max(dim=1)
accuracy += (pred_y == batch_y.data).float().sum() / batch_y.size(0)
loss /= len(dataloader)
accuracy = accuracy * 100.0 / len(dataloader)
return loss, accuracy
def train_epoch(model, loss_func, optimizer, dataloader):
model.train()
for batch_x, batch_y in dataloader:
optimizer.zero_grad()
batch_x = batch_x.cpu()
batch_y = [int(i.split('_')[0]) for i in batch_y]
batch_y = torch.Tensor(batch_y).long()
logits = model(batch_x)
error = loss_func(logits, batch_y)
error.backward()
optimizer.step()
def conv3x3(in_features, out_features):
return nn.Conv2d(in_features, out_features, kernel_size=3, padding=1)
class VGG(nn.Module):
def __init__(self):
super(VGG, self).__init__()
self.features = nn.Sequential(
conv3x3(1, 64),
nn.BatchNorm2d(64),
nn.ReLU(),
conv3x3(64, 64),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.MaxPool2d(2),
conv3x3(64, 128),
nn.BatchNorm2d(128),
nn.ReLU(),
conv3x3(128, 128),
nn.BatchNorm2d(128),
nn.ReLU(),
nn.MaxPool2d(2),
conv3x3(128, 256),
nn.BatchNorm2d(256),
nn.ReLU(),
conv3x3(256, 256),
nn.BatchNorm2d(256),
nn.ReLU(),
conv3x3(256, 256),
nn.BatchNorm2d(256),
nn.ReLU(),
conv3x3(256, 256),
nn.BatchNorm2d(256),
nn.ReLU(),
nn.MaxPool2d(2),
conv3x3(256, 512),
nn.BatchNorm2d(512),
nn.ReLU(),
conv3x3(512, 512),
nn.BatchNorm2d(512),
nn.ReLU(),
conv3x3(512, 512),
nn.BatchNorm2d(512),
nn.ReLU(),
conv3x3(512, 512),
nn.BatchNorm2d(512),
nn.ReLU(),
nn.MaxPool2d(2),
conv3x3(512, 512),
nn.BatchNorm2d(512),
nn.ReLU(),
conv3x3(512, 512),
nn.BatchNorm2d(512),
nn.ReLU(),
conv3x3(512, 512),
nn.BatchNorm2d(512),
nn.ReLU(),
conv3x3(512, 512),
nn.BatchNorm2d(512),
nn.ReLU(),
nn.MaxPool2d(2),
)
self.classifier = nn.Sequential(
nn.Linear(512, 4096),
nn.ReLU(),
nn.Dropout(),
nn.Linear(4096, 4096),
nn.ReLU(),
nn.Dropout(),
nn.Linear(4096, 10),
)
def forward(self, x):
out = self.features(x)
out = out.view(out.size(0), -1)
out = self.classifier(out)
return out
vgg19 = VGG().to(device)
print(vgg19)
optimizer = torch.optim.Adam(vgg19.parameters(), lr=LR)
scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[40], gamma=0.1)
learn_history = []
print('开始训练VGG19……')
for epoch in range(nepochs):
since = time.time()
train_epoch(vgg19, loss_func, optimizer, train_dl)
if (epoch) % 5 == 0:
tr_loss, tr_acc = eval(vgg19, loss_func, train_dl)
te_loss, te_acc = eval(vgg19, loss_func, test_dl)
learn_history.append((tr_loss, tr_acc, te_loss, te_acc))
now = time.time()
print('[{}/{}, {} seconds]|\t 训练误差: {}, 训练正确率: {}\t |\t 测试误差:{}, 测试正确率: {}'.format(
epoch + 1, nepochs, '%.2f' % (now - since), '%.2f' % (tr_loss), '%.2f' % (tr_acc), '%.2f' % (te_loss),
'%.2f' % (te_acc)))
最终结果
