找不到超参数优化Python中的泄漏ReLU

Question

我正在使用一个超参数字典和一个用于神经网络中的超参数优化的函数，如下所示：

from tensorflow.keras.layers import LeakyReLU

parameters=[
    {
        "name": "learning_rate",
        "type": "range",
        "bounds": [0.001, 0.5],
        "log_scale": True,
    },
    {
        "name": "dropout_rate",
        "type": "range",
        "bounds": [0.01, 0.9],
        "log_scale": True,
    },
    {
        "name": "num_hidden_layers",
        "type": "range",
        "bounds": [1, 7],
        "value_type": "int"
    },
    {
        "name": "neurons_per_layer",
        "type": "range",
        "bounds": [1, 300],
        "value_type": "int"
    },
    {
        "name": "batch_size",
        "type": "choice",
        "values": [8, 10, 16, 20, 30],
    },
    
    {
        "name": "activation",
        "type": "choice",
        "values": [ 'LeakyReLU(alpha=0.3)', 'relu'],
    },
    {
        "name": "optimizer",
        "type": "choice",
        "values": ['adam', 'rms', 'sgd'],
    },
]

# This returns a multi-layer-perceptron model in Keras.
def get_keras_model(num_hidden_layers, 
                    num_neurons_per_layer, 
                    dropout_rate, 
                    activation):
    # create the MLP model.
    
    # define the layers.
    inputs = tf.keras.Input(shape=(train_dataset.shape[1],))  # input layer.
    x = layers.Dropout(dropout_rate)(inputs) # dropout on the weights.
    
    # Add the hidden layers.
    for i in range(num_hidden_layers):
        x = layers.Dense(num_neurons_per_layer, 
                         activation=activation)(x)
        x = layers.Dropout(dropout_rate)(x)
    
    # output layer.
    outputs = layers.Dense(1, activation='linear')(x)
    
    model = tf.keras.Model(inputs=inputs, outputs=outputs)
    return model
    

# This function takes in the hyperparameters and returns a score (Cross validation). 
# Returns the mean of the validation loss based on which we decide which algorithm has the best hyperparameters
def keras_mlp_cv_score(parameterization, weight=None):
    
    model = get_keras_model(parameterization.get('num_hidden_layers'),
                            parameterization.get('neurons_per_layer'),
                            parameterization.get('dropout_rate'),
                            parameterization.get('activation'))
    
    opt = parameterization.get('optimizer')
    opt = opt.lower()
    
    learning_rate = parameterization.get('learning_rate')
    
    if opt == 'adam':
        optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
    elif opt == 'rms':
        optimizer = tf.keras.optimizers.RMSprop(learning_rate=learning_rate)
    else:
        optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)
        
    act = parameterization.get('activation')
    act = act.lower()
    
    if act == 'leakyrelu': 
        activation = ""
        get_keras_model.add(tf.layers.leakyReLU())
    
    
    NUM_EPOCHS = 100
    
    # Specify the training configuration.
    model.compile(optimizer=optimizer,
                  loss=tf.keras.losses.MeanSquaredError(),
                  metrics=['mae', 'mse'] )

    data = X_train
    labels = y_train.values
    
    early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)

    
    # fit the model using a 20% validation set. with a patience of 10 to avoid overfitting
    res = model.fit(data, labels, epochs=NUM_EPOCHS, batch_size=parameterization.get('batch_size'),
                    validation_split=0.2, callbacks=[early_stop, tfdocs.modeling.EpochDots()])
    
    # look at the last 10 epochs. Get the mean and standard deviation of the validation score.
    last10_scores = np.array(res.history['val_loss'][-10:])
    mean = last10_scores.mean()
    sem = last10_scores.std()
    
    # If the model didn't converge then set a high loss.
    if np.isnan(mean):
        return 9999.0, 0.0
    
    return mean, sem

但无论我如何使用LeakyReLU，它都会抛出未找到激活函数的错误。我还尝试了tf.nn.leaky_relu请帮助我将LeakyReLU正确地合并到我的代码中。

Answer 1

你写道：

act = act.lower()    

if act == 'LeakyReLU': 
    ...

这个测试将总是错误的，因为'LeakyReLU'有一些大写字母，而act从来没有，因此它永远不会向您的模型添加LeakyRelu层。

尝试：

act = act.lower()    

if act == 'leakyrelu': 
    ...

另外，正确的语法是tf.keras.layers.LeakyReLU() (Leaky ReLU in tensorflow documentation)。

如果需要，请使用错误的堆栈跟踪更新您的问题。

请尝试以下操作：

def get_keras_model(num_hidden_layers, 
                num_neurons_per_layer, 
                dropout_rate, 
                activation):
    # create the MLP model.

    # define the layers.
    inputs = tf.keras.Input(shape=(train_dataset.shape[1],))  # input layer.
    x = layers.Dropout(dropout_rate)(inputs) # dropout on the weights.

    leakyrelu =  activation.lower() == 'leakyrelu'

    # Add the hidden layers.
    for i in range(num_hidden_layers):
        x = layers.Dense(num_neurons_per_layer, 
                     activation=activation if not leakyrelu else None)(x)
        if leakyrelu:
            x = LeakyReLU()(x)
        x = layers.Dropout(dropout_rate)(x)    


def keras_mlp_cv_score(parameterization, weight=None):

    model = get_keras_model(parameterization.get('num_hidden_layers'),
                        parameterization.get('neurons_per_layer'),
                        parameterization.get('dropout_rate'),
                        parameterization.get('activation'))

    # No need for "if act == 'leakyrelu':" etc anymore.
....