预先训练的Tensorflow模型RGB -> RGBY信道扩展

Question

我正在蛋白质分析项目上工作。我们收到的图片*的蛋白质有4个过滤器(红色，绿色，蓝色和黄色)。每个RGBY信道都包含独特的数据，因为不同的细胞结构可以用不同的滤波器看到。

这样做的目的是使用经过预先培训的网络，例如VGG19，并将信道数从默认的3条扩展到4条。

(我的应用程序，我不允许在10个声誉之前直接添加图像，请按“运行代码片段”按钮来可视化)：

<img src="https://i.stack.imgur.com/TZKka.png" alt="Italian Trulli">

​

图片:将RGB扩展到RGBY的VGG模型

Y通道应该是现有预训练信道的副本。这样就有可能利用预先训练过的权重。

是否有人知道如何扩展预先训练过的网络？

*作者的拼贴-阿伦尼娅从卡格尔，“蛋白质图谱-探索和基线”内核。

Answer 1

使用layer.get_weights()和layer.set_weights()函数的Keras。

为4层VGG (设置输入shape=(width, height, 4))创建一个模板结构.然后将3通道RGB模型的权重加载到4通道作为RGBB.

下面是执行该过程的代码。在顺序VGG的情况下，唯一需要修改的层是第一卷积层。后续层的结构与信道数无关。

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from keras.applications.vgg19 import VGG19
from keras.models import Model

vgg19 = VGG19(weights='imagenet')
vgg19.summary() # To check which layers will be omitted in 'pretrained' model

# Load part of the VGG without the top layers into 'pretrained' model
pretrained = Model(inputs=vgg19.input, outputs=vgg19.get_layer('block5_pool').output)
pretrained.summary()

#%% Prepare model template with 4 input channels
config = pretrained.get_config() # run config['layers'][i] for reference
                                 # to restore layer-by layer structure

from keras.layers import Input, Conv2D, MaxPooling2D
from keras import optimizers

# For training from scratch change kernel_initializer to e.g.'VarianceScaling'
inputs = Input(shape=(224, 224, 4), name='input_17')
# block 1
x = Conv2D(64, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block1_conv1')(inputs)
x = Conv2D(64, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block1_conv2')(x)
x = MaxPooling2D(pool_size=(2, 2), name='block1_pool')(x)

# block 2
x = Conv2D(128, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block2_conv1')(x)
x = Conv2D(128, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block2_conv2')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2,2), name='block2_pool')(x)

# block 3
x = Conv2D(256, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block3_conv1')(x)
x = Conv2D(256, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block3_conv2')(x)
x = Conv2D(256, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block3_conv3')(x)
x = Conv2D(256, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block3_conv4')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2,2), name='block3_pool')(x)

# block 4
x = Conv2D(512, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block4_conv1')(x)
x = Conv2D(512, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block4_conv2')(x)
x = Conv2D(512, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block4_conv3')(x)
x = Conv2D(512, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block4_conv4')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2,2), name='block4_pool')(x)

# block 5
x = Conv2D(512, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block5_conv1')(x)
x = Conv2D(512, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block5_conv2')(x)
x = Conv2D(512, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block5_conv3')(x)
x = Conv2D(512, (3,3), padding='same', activation='relu', kernel_initializer='zeros', name='block5_conv4')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2,2), name='block5_pool')(x)

vgg_template = Model(inputs=inputs, outputs=x)

vgg_template.compile(optimizer=optimizers.RMSprop(lr=2e-4),
                     loss='categorical_crossentropy',
                     metrics=['acc'])


#%% Rewrite the weight loading/modification function
import numpy as np

layers_to_modify = ['block1_conv1'] # Turns out the only layer that changes
                                    # shape due to 4th channel is the first
                                    # convolution layer.

for layer in pretrained.layers: # pretrained Model and template have the same
                                # layers, so it doesn't matter which to 
                                # iterate over.

    if layer.get_weights() != []: # Skip input, pooling and no weights layers

        target_layer = vgg_template.get_layer(name=layer.name)

        if layer.name in layers_to_modify:

            kernels = layer.get_weights()[0]
            biases  = layer.get_weights()[1]

            kernels_extra_channel = np.concatenate((kernels,
                                                    kernels[:,:,-1:,:]),
                                                    axis=-2) # For channels_last

            target_layer.set_weights([kernels_extra_channel, biases])

        else:
            target_layer.set_weights(layer.get_weights())


#%% Save 4 channel model populated with weights for futher use    

vgg_template.save('vgg19_modified_clear.hdf5')

Answer 2

除了RGBY的情况外，下面的代码片段通常通过复制或根据需要删除层的权重和/或偏差向量维度的来工作。请参考numpy 文档关于numpy.resize所做的事情:在最初的问题中，它将B通道的权重复制到Y通道(或者更一般地复制到任何更高的维度)。

import numpy as np
import tensorflow as tf
...

model = ...  # your RGBY model is here
pretrained_model = tf.keras.models.load_model(...)  # pretrained RGB model

# the following assumes that the layers match with the two models and
# only the shapes of weights and/or biases are different
for pretrained_layer, layer in zip(pretrained_model.layers, model.layers):
    pretrained = pretrained_layer.get_weights()
    target = layer.get_weights()
    if len(pretrained) == 0:  # skip input, pooling and other no weights layers
        continue
    try:  
        # set the pretrained weights as is whenever possible
        layer.set_weights(pretrained)
    except:
        # numpy.resize to the rescue whenever there is a shape mismatch
        for idx, (l1, l2) in enumerate(zip(pretrained, target)):
            target[idx] = np.resize(l1, l2.shape)

        layer.set_weights(target)