完成图像风格迁移

ImageProcess/settings.py

@@ -36,6 +36,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'basic',
+    'advance'
 ]
 
 MIDDLEWARE = [

ImageProcess/urls.py

@@ -18,10 +18,12 @@ from django.urls import path, include, re_path
 from django.views.static import serve  # 导入相关静态文件处理的views控制包
 
 from ImageProcess.settings import MEDIA_ROOT
-from basic import urls
+from advance import urls as aurl
+from basic import urls as burl
 
 urlpatterns = [
     re_path(r'^media/(?P<path>.*)$', serve, {"document_root": MEDIA_ROOT}),
     path('admin/', admin.site.urls),
-    path('basic/', include(urls)),
+    path('basic/', include(burl)),
+    path('advance/', include(aurl))
 ]

advance/An_error_try/README.md

@@ -0,0 +1,10 @@
+这是对于图像风格迁移之中的`固定风格任意内容的迁移`的学习 \
+由于我的机器没有合适的GPU，所以很难进行多次模型的训练（因为一次时间耗费太久）\
+最终仅仅训练出两个模型（model文件夹中）， 一个使用了梵高的一幅画作为style，另一个是毕加索的一副画。（style文件夹中） \
+但是存在两个问题：
+
+- 图片在转换后大致风格和style是一致的，但是会出现一些奇怪的白色斑点，由于我对深度学习了解有限，这个问题难以解决
+-
+
+图片的格式在转换完成之后是tensor格式，在保存的时候，不论是转换成numpy格式使用opencv保存，还是直接使用torch的一个名为save_image的函数保存，都会出现一些图片的颜色变化，导致风格变化。同样，这个问题由于我的知识有限，难以解决 \
+而且因为机器的原因，没有足够的时间来测试。。。。

advance/An_error_try/models.py

@@ -0,0 +1,49 @@
+from torch import nn
+
+
+def ConvLayer(in_channels, out_channels, kernel_size=3, stride=1,
+              upsample=None, instance_norm=True, relu=True):
+    layers = []
+    if upsample:
+        layers.append(nn.Upsample(mode='nearest', scale_factor=upsample))
+    layers.append(nn.ReflectionPad2d(kernel_size // 2))
+    layers.append(nn.Conv2d(in_channels, out_channels, kernel_size, stride))
+    if instance_norm:
+        layers.append(nn.InstanceNorm2d(out_channels))
+    if relu:
+        layers.append(nn.ReLU())
+    return layers
+
+
+class ResidualBlock(nn.Module):
+    def __init__(self, channels):
+        super(ResidualBlock, self).__init__()
+        self.conv = nn.Sequential(
+            *ConvLayer(channels, channels, kernel_size=3, stride=1),
+            *ConvLayer(channels, channels, kernel_size=3, stride=1, relu=False)
+        )
+
+    def forward(self, x):
+        return self.conv(x) + x
+
+
+class TransformNet(nn.Module):
+    def __init__(self, base=32):
+        super(TransformNet, self).__init__()
+        self.downsampling = nn.Sequential(
+            *ConvLayer(3, base, kernel_size=9),
+            *ConvLayer(base, base * 2, kernel_size=3, stride=2),
+            *ConvLayer(base * 2, base * 4, kernel_size=3, stride=2),
+        )
+        self.residuals = nn.Sequential(*[ResidualBlock(base * 4) for i in range(5)])
+        self.upsampling = nn.Sequential(
+            *ConvLayer(base * 4, base * 2, kernel_size=3, upsample=2),
+            *ConvLayer(base * 2, base, kernel_size=3, upsample=2),
+            *ConvLayer(base, 3, kernel_size=9, instance_norm=False, relu=False),
+        )
+
+    def forward(self, X):
+        y = self.downsampling(X)
+        y = self.residuals(y)
+        y = self.upsampling(y)
+        return y

advance/An_error_try/transfer.py

@@ -0,0 +1,53 @@
+from torchvision import utils as vutils
+
+from models import *
+from utils import *
+
+CORE = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+device = torch.device(CORE)
+
+transform_net = TransformNet(32).to(device)
+transform_net.load_state_dict(
+    torch.load('./model/Picasso.pth', map_location=CORE))
+
+style_img = read_image('./style/style-Picasso.jpg').to(device)
+
+content_img = read_image('./img/content.jpg').to(device)
+output_img = transform_net(content_img)
+
+plt.figure(figsize=(18, 6))
+
+plt.subplot(1, 3, 1)
+imshow(style_img, title='Style Image')
+
+plt.subplot(1, 3, 2)
+imshow(content_img, title='Content Image')
+
+plt.subplot(1, 3, 3)
+imshow(output_img.detach(), title='Output Image')
+
+plt.show()
+
+# def save_image_tensor2cv2(input_tensor: torch.Tensor, filename):
+#     """
+#     将tensor保存为cv2格式
+#     :param input_tensor: 要保存的tensor
+#     :param filename: 保存的文件名
+#     """
+#     assert (len(input_tensor.shape) == 4 and input_tensor.shape[0] == 1)
+#     # 复制一份
+#     input_tensor = input_tensor.clone().detach()
+#     # 到cpu
+#     input_tensor = input_tensor.to(torch.device('cpu'))
+#     # 反归一化
+#     # input_tensor = unnormalize(input_tensor)
+#     # 去掉批次维度
+#     input_tensor = input_tensor.squeeze()
+#     # 从[0,1]转化为[0,255]，再从CHW转为HWC，最后转为cv2
+#     input_tensor = input_tensor.mul_(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).type(torch.uint8).numpy()
+#     # RGB转BRG
+#     input_tensor = cv2.cvtColor(input_tensor, cv2.COLOR_RGB2BGR)
+#     cv2.imwrite(filename, input_tensor)
+
+vutils.save_image(output_img.detach(), './result/Picasso.jpg')

advance/An_error_try/utils.py

@@ -0,0 +1,99 @@
+import cv2
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+import torchvision.transforms as transforms
+from PIL import Image
+
+cnn_normalization_mean = [0.485, 0.456, 0.406]
+cnn_normalization_std = [0.229, 0.224, 0.225]
+tensor_normalizer = transforms.Normalize(mean=cnn_normalization_mean, std=cnn_normalization_std)
+epsilon = 1e-5
+
+
+def preprocess_image(image, target_width=None):
+    """输入 PIL.Image 对象，输出标准化后的四维 tensor"""
+    if target_width:
+        t = transforms.Compose([
+            transforms.Resize(target_width),
+            transforms.CenterCrop(target_width),
+            transforms.ToTensor(),
+            tensor_normalizer,
+        ])
+    else:
+        t = transforms.Compose([
+            transforms.ToTensor(),
+            tensor_normalizer,
+        ])
+    return t(image).unsqueeze(0)
+
+
+def image_to_tensor(image, target_width=None):
+    """输入 OpenCV 图像，范围 0~255，BGR 顺序，输出标准化后的四维 tensor"""
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    image = Image.fromarray(image)
+    return preprocess_image(image, target_width)
+
+
+def read_image(path, target_width=None):
+    """输入图像路径，输出标准化后的四维 tensor"""
+    image = Image.open(path)
+    return preprocess_image(image, target_width)
+
+
+def recover_image(tensor):
+    """输入 GPU 上的四维 tensor，输出 0~255 范围的三维 numpy 矩阵，RGB 顺序"""
+    image = tensor.detach().cpu().numpy()
+    image = image * np.array(cnn_normalization_std).reshape((1, 3, 1, 1)) + \
+            np.array(cnn_normalization_mean).reshape((1, 3, 1, 1))
+    return (image.transpose(0, 2, 3, 1) * 255.).clip(0, 255).astype(np.uint8)[0]
+
+
+def recover_tensor(tensor):
+    m = torch.tensor(cnn_normalization_mean).view(1, 3, 1, 1).to(tensor.device)
+    s = torch.tensor(cnn_normalization_std).view(1, 3, 1, 1).to(tensor.device)
+    tensor = tensor * s + m
+    return tensor.clamp(0, 1)
+
+
+def imshow(tensor, title=None):
+    """输入 GPU 上的四维 tensor，然后绘制该图像"""
+    image = recover_image(tensor)
+    print(image.shape)
+    plt.imshow(image)
+    if title is not None:
+        plt.title(title)
+
+
+def mean_std(features):
+    """输入 VGG16 计算的四个特征，输出每张特征图的均值和标准差，长度为1920"""
+    mean_std_features = []
+    for x in features:
+        x = x.view(*x.shape[:2], -1)
+        x = torch.cat([x.mean(-1), torch.sqrt(x.var(-1) + epsilon)], dim=-1)
+        n = x.shape[0]
+        x2 = x.view(n, 2, -1).transpose(2, 1).contiguous().view(n, -1)  # 【mean, ..., std, ...] to [mean, std, ...]
+        mean_std_features.append(x2)
+    mean_std_features = torch.cat(mean_std_features, dim=-1)
+    return mean_std_features
+
+
+class Smooth:
+    # 对输入的数据进行滑动平均
+    def __init__(self, windowsize=100):
+        self.window_size = windowsize
+        self.data = np.zeros((self.window_size, 1), dtype=np.float32)
+        self.index = 0
+
+    def __iadd__(self, x):
+        if self.index == 0:
+            self.data[:] = x
+        self.data[self.index % self.window_size] = x
+        self.index += 1
+        return self
+
+    def __float__(self):
+        return float(self.data.mean())
+
+    def __format__(self, f):
+        return self.__float__().__format__(f)

advance/admin.py

@@ -0,0 +1 @@
+# Register your models here.

advance/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class AdvanceConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'advance'

advance/tests.py

@@ -0,0 +1 @@
+# Create your tests here.

advance/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+    path('transfer', views.transfer)
+]

advance/utils.py

@@ -0,0 +1,40 @@
+import cv2
+
+
+def style_transfer(pathIn='',
+                   pathOut='',
+                   model='',
+                   width=None,
+                   jpg_quality=80):
+    """
+    pathIn: 原始图片的路径
+    pathOut: 风格化图片的保存路径
+    model: 预训练模型的路径
+    width: 设置风格化图片的宽度，默认为None, 即原始图片尺寸
+    jpg_quality: 0-100，设置输出图片的质量，默认80，越大图片质量越好
+    """
+
+    # 读入原始图片，调整图片至所需尺寸，然后获取图片的宽度和高度
+    img = cv2.imread(pathIn)
+    (h, w) = img.shape[:2]
+    if width is not None:
+        img = cv2.resize(img, (width, round(width * h / w)), interpolation=cv2.INTER_CUBIC)
+        (h, w) = img.shape[:2]
+
+    net = cv2.dnn.readNetFromTorch(model)
+
+    # 将图片构建成一个blob：设置图片尺寸，将各通道像素值减去平均值（比如ImageNet所有训练样本各通道统计平均值）
+    # 然后执行一次前馈网络计算，并输出计算所需的时间
+    blob = cv2.dnn.blobFromImage(img, 1.0, (w, h), (103.939, 116.779, 123.680), swapRB=False, crop=False)
+    net.setInput(blob)
+    output = net.forward()
+
+    # reshape输出结果, 将减去的平均值加回来，并交换各颜色通道
+    output = output.reshape((3, output.shape[2], output.shape[3]))
+    output[0] += 103.939
+    output[1] += 116.779
+    output[2] += 123.680
+    output = output.transpose(1, 2, 0)
+
+    # 输出风格化后的图片
+    cv2.imwrite(pathOut, output, [int(cv2.IMWRITE_JPEG_QUALITY), jpg_quality])

advance/views.py

@@ -0,0 +1,28 @@
+import json
+
+from django.http import HttpResponse
+from django.views.decorators.csrf import csrf_exempt
+
+from basic.helper import get_image_name
+from .utils import *
+
+PREFIX = './media/'
+DEFAULT_FORMAT = '.jpg'
+
+SUPPORT_MODEL = ['candy', 'composition_vii', 'feathers', 'la_muse', 'mosaic',
+                 'starry_night', 'the_scream', 'the_wave', 'udnie']
+
+
+@csrf_exempt
+def transfer(request):
+    if request.method == 'POST':
+        para = json.loads(request.body)
+        image = para['img']
+        my_type = para['model']
+        if my_type not in SUPPORT_MODEL:
+            return HttpResponse('不支持的风格转换')
+        model = 'advance/models/' + para['type'] + '.t7'
+        filename = get_image_name() + DEFAULT_FORMAT
+        style_transfer(PREFIX + image, PREFIX + filename, model)
+        return HttpResponse(filename)
+    return HttpResponse('请使用POST方法')

basic/helper.py

@@ -574,7 +574,10 @@ def process_stop_choice(img, my_min, my_max):
     return output
 
 
-def save_image(img):
+def save_image(img, prefix=None):
     img_name = get_image_name() + DEFAULT_FORMAT
-    cv2.imwrite(PREFIX + img_name, img)
+    if prefix is None:
+        cv2.imwrite(PREFIX + img_name, img)
+    else:
+        cv2.imwrite(prefix + img_name, img)
     return img_name