Aller au contenu

TP Module 4 : Les méthodes d'optimisation des réseaux de neurones

Import libs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
import tensorflow as tf
from tensorflow import keras

print(tf.__version__)
print(keras.__version__)

import numpy as np
import random
import os
import datetime

# Load the TensorBoard notebook extension
%load_ext tensorboard

# freeze de l'aléatoire, pour avoir des expériences reproductibles.
RANDOM_SEED = 42

os.environ['PYTHONHASHSEED'] = str(RANDOM_SEED)
random.seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)
os.environ['TF_DETERMINISTIC_OPS'] = '1'
tf.random.set_seed(RANDOM_SEED)

2.2.0-rc3
2.3.0-tf


1
!nvidia-smi

Wed Apr 22 12:29:55 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 440.64.00    Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   42C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+


 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
from tensorflow.keras import models
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Input
from tensorflow.keras.layers import Activation
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import MaxPooling2D

from sklearn.model_selection import train_test_split

import matplotlib.pyplot as plt
import pandas as pd

Import dataset

Le but ici sera de voir l'impact des différentes méthodes d'optimisation.

Pour se faire on se basera sur le même dataset pour toute la suite du TP, avec un entraînement de \(20\) époques et une batchsize de \(64\).

1
2
3
(X_train,y_train), (X_test,y_test)  = tf.keras.datasets.fashion_mnist.load_data()

print(X_train.shape, y_train.shape)

Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz
32768/29515 [=================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz
26427392/26421880 [==============================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz
8192/5148 [===============================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz
4423680/4422102 [==============================] - 0s 0us/step
(60000, 28, 28) (60000,)


1
[name for name in dir(keras.datasets) if not name.startswith("_")]

['boston_housing',
 'cifar10',
 'cifar100',
 'fashion_mnist',
 'imdb',
 'mnist',
 'reuters']

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
class_names = ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat",
               "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"]

n_rows = 4
n_cols = 10
plt.figure(figsize=(n_cols * 2, n_rows * 2))
for row in range(n_rows):
    for col in range(n_cols):
        index = n_cols * row + col
        plt.subplot(n_rows, n_cols, index + 1)
        plt.imshow(X_train[index, : , :, ], cmap="binary", interpolation="nearest")
        plt.axis('off')
        plt.title(class_names[y_train[index]], fontsize=12)
plt.subplots_adjust(wspace=0.2, hspace=0.5)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
X_train = X_train.reshape(-1, 28, 28, 1).astype('float32')
X_test = X_test.reshape(-1, 28, 28, 1).astype('float32')


X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, random_state=RANDOM_SEED)

X_test = X_test/255
X_train = X_train/255
X_valid = X_valid/255

y_train_oh = tf.keras.utils.to_categorical(y_train, num_classes=10)
y_test_oh = tf.keras.utils.to_categorical(y_test, num_classes=10)
y_valid_oh = tf.keras.utils.to_categorical(y_valid, num_classes=10)

1
2
3
4
comp = pd.DataFrame()
comp['run'] = []
comp['Perte'] = []
comp['Précision'] = []

CNN basique

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
def base_model(optimizer = tf.keras.optimizers.SGD(lr=0.001)):
  model = keras.models.Sequential([
                                 Input(shape=(28,28,1)),
                                 Conv2D(32, (3, 3), activation='relu', padding='same'),
                                 Conv2D(64, (3, 3), activation='relu', padding='same'),
                                 MaxPooling2D((2, 2)),
                                 Conv2D(128, (3, 3), activation='relu', padding='same'),
                                 Conv2D(128, (3, 3), activation='relu', padding='same'),
                                 MaxPooling2D((2, 2)),
                                 Flatten(),
                                 Dense(256),
                                 Activation('relu'),
                                 Dense(128),
                                 Activation('relu'),
                                 Dense(10, activation='softmax')
                                 ])

  model.compile(loss = 'categorical_crossentropy',
                optimizer=optimizer,
                metrics=['accuracy'])

  return model

1
2
model = base_model()
model.summary()

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d (Conv2D)              (None, 28, 28, 32)        320       
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 28, 28, 64)        18496     
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 14, 14, 64)        0         
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 14, 14, 128)       73856     
_________________________________________________________________
conv2d_3 (Conv2D)            (None, 14, 14, 128)       147584    
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 7, 7, 128)         0         
_________________________________________________________________
flatten (Flatten)            (None, 6272)              0         
_________________________________________________________________
dense (Dense)                (None, 256)               1605888   
_________________________________________________________________
activation (Activation)      (None, 256)               0         
_________________________________________________________________
dense_1 (Dense)              (None, 128)               32896     
_________________________________________________________________
activation_1 (Activation)    (None, 128)               0         
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1290      
=================================================================
Total params: 1,880,330
Trainable params: 1,880,330
Non-trainable params: 0
_________________________________________________________________


1
2
3
4
5
6
7
8
logdir = os.path.join("logs", datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)

history = model.fit(X_train, y_train_oh,
                    epochs = 20,
                    batch_size=64,
                    validation_data=(X_valid, y_valid_oh),
                    callbacks=[tensorboard_callback])

Epoch 1/20
704/704 [==============================] - 12s 17ms/step - loss: 2.2813 - accuracy: 0.2722 - val_loss: 2.2517 - val_accuracy: 0.3401
Epoch 2/20
704/704 [==============================] - 12s 17ms/step - loss: 2.0288 - accuracy: 0.4274 - val_loss: 1.3395 - val_accuracy: 0.6017
Epoch 3/20
704/704 [==============================] - 12s 17ms/step - loss: 0.9132 - accuracy: 0.6698 - val_loss: 0.8187 - val_accuracy: 0.6958
Epoch 4/20
704/704 [==============================] - 12s 17ms/step - loss: 0.7399 - accuracy: 0.7263 - val_loss: 0.8537 - val_accuracy: 0.6747
Epoch 5/20
704/704 [==============================] - 12s 17ms/step - loss: 0.6871 - accuracy: 0.7485 - val_loss: 0.9972 - val_accuracy: 0.6795
Epoch 6/20
704/704 [==============================] - 12s 17ms/step - loss: 0.6479 - accuracy: 0.7618 - val_loss: 0.7312 - val_accuracy: 0.7510
Epoch 7/20
704/704 [==============================] - 12s 17ms/step - loss: 0.6215 - accuracy: 0.7725 - val_loss: 0.6519 - val_accuracy: 0.7645
Epoch 8/20
704/704 [==============================] - 12s 17ms/step - loss: 0.5956 - accuracy: 0.7811 - val_loss: 0.8025 - val_accuracy: 0.7280
Epoch 9/20
704/704 [==============================] - 12s 17ms/step - loss: 0.5774 - accuracy: 0.7881 - val_loss: 0.8302 - val_accuracy: 0.7277
Epoch 10/20
704/704 [==============================] - 12s 17ms/step - loss: 0.5556 - accuracy: 0.7972 - val_loss: 0.6401 - val_accuracy: 0.7703
Epoch 11/20
704/704 [==============================] - 12s 17ms/step - loss: 0.5423 - accuracy: 0.8014 - val_loss: 0.6411 - val_accuracy: 0.7772
Epoch 12/20
704/704 [==============================] - 12s 17ms/step - loss: 0.5246 - accuracy: 0.8095 - val_loss: 0.5534 - val_accuracy: 0.8013
Epoch 13/20
704/704 [==============================] - 12s 17ms/step - loss: 0.5135 - accuracy: 0.8144 - val_loss: 0.6555 - val_accuracy: 0.7576
Epoch 14/20
704/704 [==============================] - 12s 17ms/step - loss: 0.5013 - accuracy: 0.8187 - val_loss: 0.6696 - val_accuracy: 0.7626
Epoch 15/20
704/704 [==============================] - 12s 17ms/step - loss: 0.4895 - accuracy: 0.8221 - val_loss: 0.4851 - val_accuracy: 0.8303
Epoch 16/20
704/704 [==============================] - 12s 17ms/step - loss: 0.4808 - accuracy: 0.8256 - val_loss: 0.7087 - val_accuracy: 0.7554
Epoch 17/20
704/704 [==============================] - 12s 17ms/step - loss: 0.4699 - accuracy: 0.8297 - val_loss: 0.4808 - val_accuracy: 0.8287
Epoch 18/20
704/704 [==============================] - 12s 17ms/step - loss: 0.4602 - accuracy: 0.8333 - val_loss: 0.5784 - val_accuracy: 0.7922
Epoch 19/20
704/704 [==============================] - 12s 17ms/step - loss: 0.4519 - accuracy: 0.8368 - val_loss: 0.6801 - val_accuracy: 0.7607
Epoch 20/20
704/704 [==============================] - 12s 17ms/step - loss: 0.4463 - accuracy: 0.8390 - val_loss: 0.4517 - val_accuracy: 0.8411


1
%tensorboard --logdir logs

1
2
from tensorboard import notebook
notebook.list() # View open TensorBoard instances

Known TensorBoard instances:
  - port 6006: logdir logs (started 0:04:21 ago; pid 312)


1
2
3
# Control TensorBoard display. If no port is provided, 
# the most recently launched TensorBoard is used
notebook.display(port=6006, height=1000) 

1
2
3
4
5
6
7
from sklearn.metrics import classification_report

predictions = model.predict(X_test, batch_size=64)

print(classification_report(y_test_oh.argmax(axis=1),
                            predictions.argmax(axis=1),
                            target_names=class_names))

              precision    recall  f1-score   support

 T-shirt/top       0.87      0.65      0.75      1000
     Trouser       0.98      0.93      0.95      1000
    Pullover       0.75      0.71      0.73      1000
       Dress       0.73      0.92      0.81      1000
        Coat       0.79      0.60      0.68      1000
      Sandal       0.94      0.96      0.95      1000
       Shirt       0.53      0.71      0.60      1000
     Sneaker       0.93      0.92      0.92      1000
         Bag       0.95      0.95      0.95      1000
  Ankle boot       0.94      0.95      0.95      1000

    accuracy                           0.83     10000
   macro avg       0.84      0.83      0.83     10000
weighted avg       0.84      0.83      0.83     10000



1
2
3
4
pd.DataFrame(history.history).plot(figsize=(12,8))
plt.grid(True)
plt.gca().set_ylim(0,2)
plt.show()

1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'basic_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 3ms/step - loss: 0.4696 - accuracy: 0.8282
         run     Perte  Précision
0  basic_CNN  0.469638     0.8282

Régularisation \(L_{1}\), \(L_{2}\)

1
kernel_regularizer=tf.keras.regularizers.l1(0.01)

1
kernel_regularizer=tf.keras.regularizers.l1(0.01)

BatchNorm

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
model = keras.models.Sequential([
                                 Input(shape=(28, 28, 1)),
                                 Conv2D(32, (3, 3), activation='relu', padding='same'),
                                 Conv2D(64, (3, 3), activation='relu', padding='same',use_bias=False),
                                 BatchNormalization(),
                                 MaxPooling2D((2, 2)),
                                 Conv2D(128, (3, 3), activation='relu', padding='same'),
                                 Conv2D(128, (3, 3), activation='relu', padding='same',use_bias=False),
                                 BatchNormalization(),
                                 MaxPooling2D((2, 2)),
                                 Flatten(),
                                 Dense(256),
                                 Activation('relu'),
                                 Dense(128),
                                 Activation('relu'),
                                 Dense(10, activation='softmax')
                                 ])

1
2
3
4
5
model.summary()

model.compile(loss = 'categorical_crossentropy',
              optimizer=tf.keras.optimizers.SGD(lr=0.001),
              metrics=['accuracy'])

Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_4 (Conv2D)            (None, 28, 28, 32)        320       
_________________________________________________________________
conv2d_5 (Conv2D)            (None, 28, 28, 64)        18432     
_________________________________________________________________
batch_normalization (BatchNo (None, 28, 28, 64)        256       
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 14, 14, 64)        0         
_________________________________________________________________
conv2d_6 (Conv2D)            (None, 14, 14, 128)       73856     
_________________________________________________________________
conv2d_7 (Conv2D)            (None, 14, 14, 128)       147456    
_________________________________________________________________
batch_normalization_1 (Batch (None, 14, 14, 128)       512       
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 7, 7, 128)         0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 6272)              0         
_________________________________________________________________
dense_3 (Dense)              (None, 256)               1605888   
_________________________________________________________________
activation_2 (Activation)    (None, 256)               0         
_________________________________________________________________
dense_4 (Dense)              (None, 128)               32896     
_________________________________________________________________
activation_3 (Activation)    (None, 128)               0         
_________________________________________________________________
dense_5 (Dense)              (None, 10)                1290      
=================================================================
Total params: 1,880,906
Trainable params: 1,880,522
Non-trainable params: 384
_________________________________________________________________


1
2
bn1 = model.layers[2]
[(var.name, var.trainable) for var in bn1.variables]

[('batch_normalization_6/gamma:0', True),
 ('batch_normalization_6/beta:0', True),
 ('batch_normalization_6/moving_mean:0', False),
 ('batch_normalization_6/moving_variance:0', False)]

1
2
3
4
5
6
7
8
logdir = os.path.join("logs", datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)

history = model.fit(X_train, y_train_oh,
                    epochs = 20,
                    batch_size=64,
                    validation_data=(X_valid, y_valid_oh),
                    callbacks=[tensorboard_callback])

Epoch 1/20
704/704 [==============================] - 12s 18ms/step - loss: 0.5792 - accuracy: 0.8023 - val_loss: 0.5035 - val_accuracy: 0.8353
Epoch 2/20
704/704 [==============================] - 12s 17ms/step - loss: 0.3677 - accuracy: 0.8712 - val_loss: 0.3615 - val_accuracy: 0.8709
Epoch 3/20
704/704 [==============================] - 12s 17ms/step - loss: 0.3160 - accuracy: 0.8882 - val_loss: 0.3245 - val_accuracy: 0.8847
Epoch 4/20
704/704 [==============================] - 12s 17ms/step - loss: 0.2830 - accuracy: 0.9002 - val_loss: 0.3664 - val_accuracy: 0.8675
Epoch 5/20
704/704 [==============================] - 12s 17ms/step - loss: 0.2602 - accuracy: 0.9089 - val_loss: 0.3474 - val_accuracy: 0.8724
Epoch 6/20
704/704 [==============================] - 12s 17ms/step - loss: 0.2390 - accuracy: 0.9157 - val_loss: 0.2883 - val_accuracy: 0.8957
Epoch 7/20
704/704 [==============================] - 12s 17ms/step - loss: 0.2242 - accuracy: 0.9219 - val_loss: 0.2869 - val_accuracy: 0.8966
Epoch 8/20
704/704 [==============================] - 12s 17ms/step - loss: 0.2103 - accuracy: 0.9268 - val_loss: 0.3699 - val_accuracy: 0.8709
Epoch 9/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1958 - accuracy: 0.9315 - val_loss: 0.2748 - val_accuracy: 0.8993
Epoch 10/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1852 - accuracy: 0.9350 - val_loss: 0.3592 - val_accuracy: 0.8777
Epoch 11/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1737 - accuracy: 0.9402 - val_loss: 0.3234 - val_accuracy: 0.8891
Epoch 12/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1633 - accuracy: 0.9440 - val_loss: 0.2610 - val_accuracy: 0.9066
Epoch 13/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1535 - accuracy: 0.9479 - val_loss: 0.3963 - val_accuracy: 0.8635
Epoch 14/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1447 - accuracy: 0.9512 - val_loss: 0.2610 - val_accuracy: 0.9045
Epoch 15/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1357 - accuracy: 0.9552 - val_loss: 0.2761 - val_accuracy: 0.9040
Epoch 16/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1272 - accuracy: 0.9590 - val_loss: 0.2643 - val_accuracy: 0.9065
Epoch 17/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1200 - accuracy: 0.9614 - val_loss: 0.2600 - val_accuracy: 0.9096
Epoch 18/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1128 - accuracy: 0.9644 - val_loss: 0.2628 - val_accuracy: 0.9067
Epoch 19/20
704/704 [==============================] - 12s 17ms/step - loss: 0.1055 - accuracy: 0.9674 - val_loss: 0.3656 - val_accuracy: 0.8784
Epoch 20/20
704/704 [==============================] - 12s 17ms/step - loss: 0.0993 - accuracy: 0.9695 - val_loss: 0.2529 - val_accuracy: 0.9127


1
2
3
4
pd.DataFrame(history.history).plot(figsize=(12,8))
plt.grid(True)
plt.gca().set_ylim(0,1)
plt.show()

1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'bn_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 3ms/step - loss: 0.2919 - accuracy: 0.8975
         run     Perte  Précision
0  basic_CNN  0.469638     0.8282
1     bn_CNN  0.291927     0.8975

La question de savoir si oui ou non la couche de BatchNorm devrait êêtre placée avant ou après la fonction d'activation fait toujours débat.

Dans l'article d'origine, et les architectures classiques, elle est placée avant.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
model = keras.models.Sequential([
                                 Input(shape=(28, 28, 1)),
                                 Conv2D(32, (3, 3), activation='relu', padding='same'),
                                 Conv2D(64, (3, 3), padding='same',use_bias=False),
                                 BatchNormalization(),
                                 Activation('relu'),
                                 MaxPooling2D((2, 2)),
                                 Conv2D(128, (3, 3), activation='relu', padding='same'),
                                 Conv2D(128, (3, 3), padding='same',use_bias=False),
                                 BatchNormalization(),
                                 Activation('relu'),
                                 MaxPooling2D((2, 2)),
                                 Flatten(),
                                 Dense(256),
                                 Activation('relu'),
                                 Dense(128),
                                 Activation('relu'),
                                 Dense(10, activation='softmax')
                                 ])

1
2
3
4
5
model.summary()

model.compile(loss = 'categorical_crossentropy',
              optimizer=tf.keras.optimizers.SGD(lr=0.001),
              metrics=['accuracy'])

Model: "sequential_6"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_30 (Conv2D)           (None, 28, 28, 32)        320       
_________________________________________________________________
conv2d_31 (Conv2D)           (None, 28, 28, 64)        18432     
_________________________________________________________________
batch_normalization_8 (Batch (None, 28, 28, 64)        256       
_________________________________________________________________
activation_15 (Activation)   (None, 28, 28, 64)        0         
_________________________________________________________________
max_pooling2d_15 (MaxPooling (None, 14, 14, 64)        0         
_________________________________________________________________
conv2d_32 (Conv2D)           (None, 14, 14, 128)       73856     
_________________________________________________________________
conv2d_33 (Conv2D)           (None, 14, 14, 128)       147456    
_________________________________________________________________
batch_normalization_9 (Batch (None, 14, 14, 128)       512       
_________________________________________________________________
activation_16 (Activation)   (None, 14, 14, 128)       0         
_________________________________________________________________
max_pooling2d_16 (MaxPooling (None, 7, 7, 128)         0         
_________________________________________________________________
flatten_6 (Flatten)          (None, 6272)              0         
_________________________________________________________________
dense_18 (Dense)             (None, 256)               1605888   
_________________________________________________________________
activation_17 (Activation)   (None, 256)               0         
_________________________________________________________________
dense_19 (Dense)             (None, 128)               32896     
_________________________________________________________________
activation_18 (Activation)   (None, 128)               0         
_________________________________________________________________
dense_20 (Dense)             (None, 10)                1290      
=================================================================
Total params: 1,880,906
Trainable params: 1,880,522
Non-trainable params: 384
_________________________________________________________________


1
2
3
4
history = model.fit(X_train, y_train_oh,
                   epochs = 20,
                   batch_size=64,
                   validation_data=(X_valid, y_valid_oh))

Epoch 1/20
704/704 [==============================] - 8s 11ms/step - loss: 0.6268 - accuracy: 0.7898 - val_loss: 0.5258 - val_accuracy: 0.8177
Epoch 2/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4040 - accuracy: 0.8602 - val_loss: 0.4041 - val_accuracy: 0.8544
Epoch 3/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3529 - accuracy: 0.8765 - val_loss: 0.3803 - val_accuracy: 0.8637
Epoch 4/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3196 - accuracy: 0.8883 - val_loss: 0.3971 - val_accuracy: 0.8502
Epoch 5/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2974 - accuracy: 0.8965 - val_loss: 0.4005 - val_accuracy: 0.8489
Epoch 6/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2782 - accuracy: 0.9032 - val_loss: 0.3119 - val_accuracy: 0.8881
Epoch 7/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2632 - accuracy: 0.9089 - val_loss: 0.3238 - val_accuracy: 0.8802
Epoch 8/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2502 - accuracy: 0.9135 - val_loss: 0.3797 - val_accuracy: 0.8636
Epoch 9/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2380 - accuracy: 0.9186 - val_loss: 0.2796 - val_accuracy: 0.8986
Epoch 10/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2284 - accuracy: 0.9210 - val_loss: 0.3247 - val_accuracy: 0.8877
Epoch 11/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2185 - accuracy: 0.9255 - val_loss: 0.3409 - val_accuracy: 0.8800
Epoch 12/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2090 - accuracy: 0.9276 - val_loss: 0.2644 - val_accuracy: 0.9037
Epoch 13/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2011 - accuracy: 0.9326 - val_loss: 0.3488 - val_accuracy: 0.8706
Epoch 14/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1933 - accuracy: 0.9345 - val_loss: 0.2609 - val_accuracy: 0.9050
Epoch 15/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1857 - accuracy: 0.9362 - val_loss: 0.2720 - val_accuracy: 0.9045
Epoch 16/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1772 - accuracy: 0.9404 - val_loss: 0.2502 - val_accuracy: 0.9070
Epoch 17/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1721 - accuracy: 0.9414 - val_loss: 0.2536 - val_accuracy: 0.9100
Epoch 18/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1662 - accuracy: 0.9442 - val_loss: 0.3002 - val_accuracy: 0.8953
Epoch 19/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1595 - accuracy: 0.9472 - val_loss: 0.4275 - val_accuracy: 0.8586
Epoch 20/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1539 - accuracy: 0.9488 - val_loss: 0.2485 - val_accuracy: 0.9110


1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'bn2_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 3ms/step - loss: 0.2717 - accuracy: 0.9046
         run     Perte  Précision
0  basic_CNN  0.469638     0.8282
1     bn_CNN  0.291927     0.8975
2    bn2_CNN  0.271721     0.9046

De façon classique, l'architecture moderne en ce qui concerne les couches convolutives est la suivante :

1
2
3
4
Conv2D(filtre, (kernel_size, kernel_size), padding='same',use_bias=False),
BatchNormalization(),
Activation('relu'),
MaxPooling2D((kernel_size, kernel_size)),

Dropout

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
model = keras.models.Sequential([
                                 Input(shape=(28, 28, 1)),
                                 Conv2D(32, (3, 3), activation='relu', padding='same'),
                                 Conv2D(64, (3, 3), activation='relu', padding='same'),
                                 MaxPooling2D((2, 2)),
                                 Conv2D(128, (3, 3), activation='relu', padding='same'),
                                 Conv2D(128, (3, 3), activation='relu', padding='same'),
                                 MaxPooling2D((2, 2)),
                                 Flatten(),
                                 Dense(256),
                                 Dropout(0.25),
                                 Activation('relu'),
                                 Dense(128),
                                 Dropout(0.25),
                                 Activation('relu'),
                                 Dense(10),
                                 Activation('softmax')
                                 ])

1
2
3
4
5
model.summary()

model.compile(loss = tf.keras.losses.CategoricalCrossentropy(),
              optimizer=tf.keras.optimizers.SGD(lr=0.001),
              metrics=['accuracy'])

Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_4 (Conv2D)            (None, 28, 28, 32)        320       
_________________________________________________________________
conv2d_5 (Conv2D)            (None, 28, 28, 64)        18496     
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 14, 14, 64)        0         
_________________________________________________________________
conv2d_6 (Conv2D)            (None, 14, 14, 128)       73856     
_________________________________________________________________
conv2d_7 (Conv2D)            (None, 14, 14, 128)       147584    
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 7, 7, 128)         0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 6272)              0         
_________________________________________________________________
dense_3 (Dense)              (None, 256)               1605888   
_________________________________________________________________
dropout_2 (Dropout)          (None, 256)               0         
_________________________________________________________________
activation_2 (Activation)    (None, 256)               0         
_________________________________________________________________
dense_4 (Dense)              (None, 128)               32896     
_________________________________________________________________
dropout_3 (Dropout)          (None, 128)               0         
_________________________________________________________________
activation_3 (Activation)    (None, 128)               0         
_________________________________________________________________
dense_5 (Dense)              (None, 10)                1290      
_________________________________________________________________
activation_4 (Activation)    (None, 10)                0         
=================================================================
Total params: 1,880,330
Trainable params: 1,880,330
Non-trainable params: 0
_________________________________________________________________


1
2
3
4
history = model.fit(X_train, y_train_oh,
                   epochs = 20,
                   batch_size=64,
                   validation_data=(X_valid, y_valid_oh))

Epoch 1/20
704/704 [==============================] - 8s 12ms/step - loss: 2.2616 - accuracy: 0.2487 - val_loss: 2.1819 - val_accuracy: 0.4624
Epoch 2/20
704/704 [==============================] - 8s 12ms/step - loss: 1.7692 - accuracy: 0.4208 - val_loss: 1.0943 - val_accuracy: 0.6203
Epoch 3/20
704/704 [==============================] - 8s 11ms/step - loss: 1.1026 - accuracy: 0.5836 - val_loss: 0.8671 - val_accuracy: 0.6748
Epoch 4/20
704/704 [==============================] - 8s 11ms/step - loss: 0.9212 - accuracy: 0.6552 - val_loss: 0.7761 - val_accuracy: 0.7034
Epoch 5/20
704/704 [==============================] - 8s 11ms/step - loss: 0.8448 - accuracy: 0.6886 - val_loss: 0.9521 - val_accuracy: 0.6486
Epoch 6/20
704/704 [==============================] - 8s 11ms/step - loss: 0.7920 - accuracy: 0.7104 - val_loss: 0.6777 - val_accuracy: 0.7539
Epoch 7/20
704/704 [==============================] - 8s 11ms/step - loss: 0.7587 - accuracy: 0.7256 - val_loss: 0.6720 - val_accuracy: 0.7416
Epoch 8/20
704/704 [==============================] - 8s 11ms/step - loss: 0.7186 - accuracy: 0.7380 - val_loss: 0.7903 - val_accuracy: 0.7137
Epoch 9/20
704/704 [==============================] - 8s 11ms/step - loss: 0.6936 - accuracy: 0.7513 - val_loss: 0.6363 - val_accuracy: 0.7635
Epoch 10/20
704/704 [==============================] - 8s 11ms/step - loss: 0.6714 - accuracy: 0.7574 - val_loss: 0.5762 - val_accuracy: 0.7931
Epoch 11/20
704/704 [==============================] - 8s 11ms/step - loss: 0.6538 - accuracy: 0.7611 - val_loss: 0.5720 - val_accuracy: 0.7921
Epoch 12/20
704/704 [==============================] - 8s 11ms/step - loss: 0.6329 - accuracy: 0.7705 - val_loss: 0.5508 - val_accuracy: 0.8018
Epoch 13/20
704/704 [==============================] - 8s 11ms/step - loss: 0.6165 - accuracy: 0.7768 - val_loss: 0.5731 - val_accuracy: 0.7837
Epoch 14/20
704/704 [==============================] - 8s 11ms/step - loss: 0.6003 - accuracy: 0.7841 - val_loss: 0.5497 - val_accuracy: 0.7943
Epoch 15/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5880 - accuracy: 0.7877 - val_loss: 0.5172 - val_accuracy: 0.8163
Epoch 16/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5769 - accuracy: 0.7917 - val_loss: 0.5066 - val_accuracy: 0.8160
Epoch 17/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5662 - accuracy: 0.7942 - val_loss: 0.4985 - val_accuracy: 0.8170
Epoch 18/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5553 - accuracy: 0.7999 - val_loss: 0.4834 - val_accuracy: 0.8309
Epoch 19/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5469 - accuracy: 0.8033 - val_loss: 0.5423 - val_accuracy: 0.7934
Epoch 20/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5345 - accuracy: 0.8060 - val_loss: 0.5119 - val_accuracy: 0.8133


1
2
3
4
pd.DataFrame(history.history).plot(figsize=(12,8))
plt.grid(True)
plt.gca().set_ylim(0,2)
plt.show()

1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'drop_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 4ms/step - loss: 0.5324 - accuracy: 0.7998

Remarque : il n'est pas toujours nécéssaire ou mêême utile de laisser la dernière fonction d'activation 

1
Activation('softmax')
dans le modèle.

Dans certains cas, il est plus utilse de s'arrêêter à la dernière couche dense 

1
Dense(num_classes)
Dans ce cas là, la fonction de perte doit être modifiée en 
1
loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True).

Monte Carlo Dropout

Si l'on se souvient de la partie théorique, le dropout est désactivé lorsque l'on fait de l'inférence. Toutefois, il est possible de le réactiver pour avoir une simulation de Monte Carlo pour la prédiction, en moyennant plusieurs prédictions pour la même observation.

1
2
3
4
5
# On fait 100 prédictions sur le dataset de test
y_probas = np.stack([model(X_test, training=True)
                     for sample in range(100)])
# on prend alors la moyenne suivant la première dimension
y_proba = y_probas.mean(axis=0)

Avant de développer le code précedent, voyons une prédiction classique sur la première observation de X_test.

1
np.round(model.predict(X_test[:1]), 2)

array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.26, 0.  , 0.7 ]],
      dtype=float32)

Le modèle est semble certain que cette image appartient à la dernière classe "ankle boot". Mais si l'on réactive le dropout durant l'inférence, il y a des prédictions, où il n'en est plus aussi certain. Pour la \(1\)-ère prédiction, le score pour la \(9\)-ième classe n'est plus le même.

1
np.round(y_probas[:, :1], 2)

array([[[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.29, 0.  , 0.67]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.12, 0.  , 0.86]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.32, 0.  , 0.42, 0.  , 0.26]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.32, 0.  , 0.56]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.24, 0.  , 0.74]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.18, 0.  , 0.79]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.11, 0.  , 0.39, 0.01, 0.49]],

       [[0.01, 0.  , 0.  , 0.  , 0.  , 0.28, 0.  , 0.39, 0.01, 0.29]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.13, 0.  , 0.8 ]],

       [[0.  , 0.  , 0.03, 0.  , 0.  , 0.09, 0.01, 0.45, 0.01, 0.4 ]],

       [[0.01, 0.  , 0.  , 0.  , 0.  , 0.14, 0.  , 0.32, 0.  , 0.52]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.09, 0.  , 0.88]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.29, 0.  , 0.68]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.18, 0.  , 0.82]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.01, 0.5 , 0.  , 0.46]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.18, 0.  , 0.22, 0.  , 0.59]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.52, 0.  , 0.43]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.59, 0.  , 0.38]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.62, 0.  , 0.34]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.58, 0.02, 0.27]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.61, 0.  , 0.35]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.93]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.23, 0.  , 0.27, 0.  , 0.49]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.33, 0.  , 0.62]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.7 , 0.  , 0.28]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.19, 0.  , 0.78]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.43, 0.  , 0.54]],

       [[0.01, 0.  , 0.02, 0.02, 0.  , 0.06, 0.  , 0.6 , 0.  , 0.28]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.02, 0.  , 0.94]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.08, 0.  , 0.32, 0.  , 0.6 ]],

       [[0.  , 0.  , 0.14, 0.  , 0.  , 0.06, 0.01, 0.36, 0.06, 0.37]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.16, 0.  , 0.78]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.23, 0.  , 0.76]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.24, 0.  , 0.74]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.25, 0.  , 0.71]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.07, 0.  , 0.92]],

       [[0.  , 0.  , 0.01, 0.  , 0.  , 0.06, 0.  , 0.08, 0.01, 0.83]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.06, 0.01, 0.9 ]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.1 , 0.  , 0.35, 0.01, 0.55]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.1 , 0.01, 0.82]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.27, 0.  , 0.71]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.59, 0.  , 0.33]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.08, 0.  , 0.07, 0.  , 0.85]],

       [[0.01, 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.06, 0.01, 0.85]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.1 , 0.  , 0.89]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.4 , 0.01, 0.53]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.35, 0.  , 0.59]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.1 , 0.  , 0.87]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.32, 0.  , 0.63]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.14, 0.  , 0.82]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.14, 0.  , 0.82]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.28, 0.  , 0.7 ]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.09, 0.  , 0.36, 0.  , 0.55]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.52, 0.  , 0.41]],

       [[0.  , 0.  , 0.01, 0.  , 0.  , 0.02, 0.  , 0.06, 0.01, 0.9 ]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.46, 0.  , 0.53]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.12, 0.  , 0.82]],

       [[0.  , 0.  , 0.01, 0.  , 0.  , 0.03, 0.  , 0.52, 0.01, 0.44]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.08, 0.  , 0.04, 0.  , 0.87]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.35, 0.  , 0.63]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.62, 0.  , 0.35]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.09, 0.  , 0.12, 0.  , 0.78]],

       [[0.  , 0.  , 0.02, 0.01, 0.  , 0.03, 0.01, 0.28, 0.02, 0.63]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.08, 0.  , 0.19, 0.  , 0.72]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.44, 0.  , 0.5 ]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.52, 0.  , 0.36]],

       [[0.01, 0.  , 0.01, 0.  , 0.  , 0.13, 0.  , 0.02, 0.  , 0.84]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.35, 0.01, 0.52]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.18, 0.  , 0.74]],

       [[0.  , 0.  , 0.01, 0.  , 0.  , 0.01, 0.01, 0.37, 0.03, 0.57]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.14, 0.  , 0.27, 0.01, 0.58]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.13, 0.  , 0.82]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.41, 0.  , 0.56]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.2 , 0.  , 0.79]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.08, 0.  , 0.92]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.52, 0.  , 0.45]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.11, 0.  , 0.28, 0.  , 0.6 ]],

       [[0.  , 0.  , 0.01, 0.  , 0.  , 0.16, 0.  , 0.13, 0.  , 0.71]],

       [[0.  , 0.  , 0.02, 0.  , 0.  , 0.02, 0.  , 0.15, 0.01, 0.79]],

       [[0.  , 0.  , 0.01, 0.  , 0.  , 0.04, 0.  , 0.19, 0.01, 0.75]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.12, 0.  , 0.87]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.66, 0.  , 0.33]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.42, 0.  , 0.51]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.35, 0.  , 0.2 , 0.  , 0.44]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.08, 0.01, 0.87]],

       [[0.  , 0.  , 0.  , 0.01, 0.  , 0.12, 0.  , 0.24, 0.  , 0.61]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.23, 0.  , 0.76]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.3 , 0.  , 0.68]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.39, 0.  , 0.6 ]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.11, 0.  , 0.87]],

       [[0.  , 0.01, 0.  , 0.  , 0.  , 0.03, 0.  , 0.61, 0.  , 0.35]],

       [[0.  , 0.  , 0.01, 0.  , 0.  , 0.01, 0.  , 0.65, 0.  , 0.32]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.46, 0.  , 0.51]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.23, 0.  , 0.75]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.15, 0.  , 0.17, 0.  , 0.67]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.22, 0.  , 0.76]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.51, 0.07, 0.36]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.17, 0.  , 0.81]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.96]],

       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.46, 0.03, 0.51]]],
      dtype=float32)

Si l'on fait la moyenne sur la première dimension, on voit que le modèle n'est plus aussi certain que l'image appartienne à la classe \(9\).

1
np.round(y_proba[:1], 2)

array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.29, 0.  , 0.64]],
      dtype=float32)

D'ailleurs, on voit que l'écart type pour certaine classe semble assez important. Dans le cas où l'on développe des modèles pour des systèmes critiques (finance, santé), c'est important de prendre ces statistiques. 

1
2
y_std = y_probas.std(axis=0)
np.round(y_std[:1], 2)

array([[0.  , 0.  , 0.01, 0.  , 0.  , 0.06, 0.  , 0.18, 0.01, 0.19]],
      dtype=float32)

1
2
y_pred = np.argmax(y_proba, axis=1)
y_pred

array([9, 2, 1, ..., 8, 1, 5])

On voit d'ailleurs que en faisant cela, on gagne légèrement en précision (j'etais à accuracy: 0.7998, lors de ce run).

1
2
accuracy = np.sum(y_pred == y_test) / len(y_test)
accuracy

0.8031

1
2
3
class MCDropout(keras.layers.Dropout):
    def call(self, inputs):
        return super().call(inputs, training=True)

Initilisation de Xavier et de He

Par défault, Keras utilise glorot_uniform.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
model = keras.models.Sequential([
                                 Input(shape=(28, 28, 1)),
                                 Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),
                                 Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),
                                 MaxPooling2D((2, 2)),
                                 Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),
                                 Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),
                                 MaxPooling2D((2, 2)),
                                 Flatten(),
                                 Dense(256, kernel_initializer="he_normal"),
                                 Activation('relu'),
                                 Dense(128, kernel_initializer="he_normal"),
                                 Activation('relu'),
                                 Dense(10, activation='softmax')
                                 ])

model.summary()

model.compile(loss = 'categorical_crossentropy',
              optimizer=tf.keras.optimizers.SGD(lr=0.001),
              metrics=['accuracy'])

Model: "sequential_10"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_46 (Conv2D)           (None, 28, 28, 32)        320       
_________________________________________________________________
conv2d_47 (Conv2D)           (None, 28, 28, 64)        18496     
_________________________________________________________________
max_pooling2d_23 (MaxPooling (None, 14, 14, 64)        0         
_________________________________________________________________
conv2d_48 (Conv2D)           (None, 14, 14, 128)       73856     
_________________________________________________________________
conv2d_49 (Conv2D)           (None, 14, 14, 128)       147584    
_________________________________________________________________
max_pooling2d_24 (MaxPooling (None, 7, 7, 128)         0         
_________________________________________________________________
flatten_10 (Flatten)         (None, 6272)              0         
_________________________________________________________________
dense_30 (Dense)             (None, 256)               1605888   
_________________________________________________________________
activation_25 (Activation)   (None, 256)               0         
_________________________________________________________________
dense_31 (Dense)             (None, 128)               32896     
_________________________________________________________________
activation_26 (Activation)   (None, 128)               0         
_________________________________________________________________
dense_32 (Dense)             (None, 10)                1290      
=================================================================
Total params: 1,880,330
Trainable params: 1,880,330
Non-trainable params: 0
_________________________________________________________________


1
2
3
4
history = model.fit(X_train, y_train_oh,
                   epochs = 20,
                   batch_size=64,
                   validation_data=(X_valid, y_valid_oh))

Epoch 1/20
704/704 [==============================] - 8s 11ms/step - loss: 0.9376 - accuracy: 0.7017 - val_loss: 0.8291 - val_accuracy: 0.7122
Epoch 2/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5761 - accuracy: 0.7957 - val_loss: 0.6167 - val_accuracy: 0.7787
Epoch 3/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5070 - accuracy: 0.8194 - val_loss: 0.5451 - val_accuracy: 0.8119
Epoch 4/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4684 - accuracy: 0.8348 - val_loss: 0.5113 - val_accuracy: 0.8159
Epoch 5/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4397 - accuracy: 0.8440 - val_loss: 0.7333 - val_accuracy: 0.7603
Epoch 6/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4164 - accuracy: 0.8517 - val_loss: 0.4440 - val_accuracy: 0.8388
Epoch 7/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3999 - accuracy: 0.8582 - val_loss: 0.4860 - val_accuracy: 0.8289
Epoch 8/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3859 - accuracy: 0.8648 - val_loss: 0.5647 - val_accuracy: 0.8170
Epoch 9/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3722 - accuracy: 0.8696 - val_loss: 0.3680 - val_accuracy: 0.8666
Epoch 10/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3602 - accuracy: 0.8736 - val_loss: 0.3614 - val_accuracy: 0.8700
Epoch 11/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3507 - accuracy: 0.8770 - val_loss: 0.4713 - val_accuracy: 0.8329
Epoch 12/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3408 - accuracy: 0.8793 - val_loss: 0.3694 - val_accuracy: 0.8679
Epoch 13/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3336 - accuracy: 0.8826 - val_loss: 0.4376 - val_accuracy: 0.8385
Epoch 14/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3274 - accuracy: 0.8843 - val_loss: 0.4386 - val_accuracy: 0.8431
Epoch 15/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3193 - accuracy: 0.8873 - val_loss: 0.3434 - val_accuracy: 0.8777
Epoch 16/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3133 - accuracy: 0.8890 - val_loss: 0.3469 - val_accuracy: 0.8743
Epoch 17/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3082 - accuracy: 0.8922 - val_loss: 0.3196 - val_accuracy: 0.8871
Epoch 18/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3027 - accuracy: 0.8933 - val_loss: 0.4351 - val_accuracy: 0.8531
Epoch 19/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2970 - accuracy: 0.8960 - val_loss: 0.5215 - val_accuracy: 0.8220
Epoch 20/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2927 - accuracy: 0.8971 - val_loss: 0.3065 - val_accuracy: 0.8895


1
2
3
4
pd.DataFrame(history.history).plot(figsize=(12,8))
plt.grid(True)
plt.gca().set_ylim(0,2)
plt.show()

1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'he_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 4ms/step - loss: 0.3282 - accuracy: 0.8817
         run     Perte  Précision
0  basic_CNN  0.469638     0.8282
1     bn_CNN  0.291927     0.8975
2    bn2_CNN  0.271721     0.9046
3   drop_CNN  0.497585     0.8187
4     he_CNN  0.328177     0.8817

Optimisation de la SGD

1
[name for name in dir(keras.optimizers) if not name.startswith("_")]

['Adadelta',
 'Adagrad',
 'Adam',
 'Adamax',
 'Ftrl',
 'Nadam',
 'Optimizer',
 'RMSprop',
 'SGD',
 'deserialize',
 'get',
 'schedules',
 'serialize']

LRD

1
2
3
4
5
6
def exponential_decay(lr0, s):
  def exponential_decay_fn(epoch):
    return lr0 * 0.1**(epoch / s)
  return exponential_decay_fn

exponential_decay_fn = exponential_decay(lr0=0.001,s=20)

1
2
3
4
5
6
rng = [i for i in range(100)]
y = [exponential_decay_fn(x) for x in rng]
plt.plot(rng, [exponential_decay_fn(x) for x in rng])
plt.xlabel("Epoch")
plt.ylabel("Learning Rate")
print(y[0], y[-1])

1
lr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn, verbose=True)

Momentum

1
2
3
4
5
6
7
8
optimizer = keras.optimizers.SGD(lr=0.001, momentum=0.9)

model = base_model(optimizer = optimizer)

history = model.fit(X_train, y_train_oh,
                   epochs = 20,
                   batch_size=64,
                   validation_data=(X_valid, y_valid_oh))

Epoch 1/20
704/704 [==============================] - 8s 11ms/step - loss: 1.0444 - accuracy: 0.6311 - val_loss: 0.7022 - val_accuracy: 0.7455
Epoch 2/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5792 - accuracy: 0.7872 - val_loss: 0.5715 - val_accuracy: 0.7930
Epoch 3/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4858 - accuracy: 0.8243 - val_loss: 0.4480 - val_accuracy: 0.8421
Epoch 4/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4330 - accuracy: 0.8420 - val_loss: 0.4316 - val_accuracy: 0.8421
Epoch 5/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4002 - accuracy: 0.8547 - val_loss: 0.3932 - val_accuracy: 0.8547
Epoch 6/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3704 - accuracy: 0.8641 - val_loss: 0.3739 - val_accuracy: 0.8673
Epoch 7/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3496 - accuracy: 0.8721 - val_loss: 0.3562 - val_accuracy: 0.8701
Epoch 8/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3295 - accuracy: 0.8794 - val_loss: 0.3587 - val_accuracy: 0.8668
Epoch 9/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3139 - accuracy: 0.8833 - val_loss: 0.3395 - val_accuracy: 0.8777
Epoch 10/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2992 - accuracy: 0.8905 - val_loss: 0.3219 - val_accuracy: 0.8833
Epoch 11/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2871 - accuracy: 0.8945 - val_loss: 0.3207 - val_accuracy: 0.8809
Epoch 12/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2777 - accuracy: 0.8969 - val_loss: 0.2976 - val_accuracy: 0.8944
Epoch 13/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2629 - accuracy: 0.9030 - val_loss: 0.2999 - val_accuracy: 0.8936
Epoch 14/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2529 - accuracy: 0.9064 - val_loss: 0.3000 - val_accuracy: 0.8899
Epoch 15/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2465 - accuracy: 0.9086 - val_loss: 0.2856 - val_accuracy: 0.8971
Epoch 16/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2376 - accuracy: 0.9117 - val_loss: 0.2839 - val_accuracy: 0.8968
Epoch 17/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2288 - accuracy: 0.9150 - val_loss: 0.2913 - val_accuracy: 0.8975
Epoch 18/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2223 - accuracy: 0.9172 - val_loss: 0.2755 - val_accuracy: 0.9008
Epoch 19/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2154 - accuracy: 0.9207 - val_loss: 0.2833 - val_accuracy: 0.8977
Epoch 20/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2071 - accuracy: 0.9232 - val_loss: 0.2737 - val_accuracy: 0.9009


1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'momentum_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 4ms/step - loss: 0.2860 - accuracy: 0.8988
            run     Perte  Précision
0     basic_CNN  0.469638     0.8282
1        bn_CNN  0.291927     0.8975
2       bn2_CNN  0.271721     0.9046
3      drop_CNN  0.497585     0.8187
4        he_CNN  0.328177     0.8817
5  momentum_CNN  0.286005     0.8988

Nesterov

1
2
3
4
5
6
7
8
optimizer = keras.optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True)

model = base_model(optimizer = optimizer)

history = model.fit(X_train, y_train_oh,
                   epochs = 20,
                   batch_size=64,
                   validation_data=(X_valid, y_valid_oh))

Epoch 1/20
704/704 [==============================] - 8s 11ms/step - loss: 1.1436 - accuracy: 0.6146 - val_loss: 0.8565 - val_accuracy: 0.6578
Epoch 2/20
704/704 [==============================] - 8s 11ms/step - loss: 0.5587 - accuracy: 0.7964 - val_loss: 0.6265 - val_accuracy: 0.7754
Epoch 3/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4691 - accuracy: 0.8315 - val_loss: 0.4570 - val_accuracy: 0.8379
Epoch 4/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4151 - accuracy: 0.8498 - val_loss: 0.4584 - val_accuracy: 0.8299
Epoch 5/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3811 - accuracy: 0.8628 - val_loss: 0.4670 - val_accuracy: 0.8242
Epoch 6/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3538 - accuracy: 0.8719 - val_loss: 0.3482 - val_accuracy: 0.8751
Epoch 7/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3340 - accuracy: 0.8784 - val_loss: 0.3706 - val_accuracy: 0.8655
Epoch 8/20
704/704 [==============================] - 8s 11ms/step - loss: 0.3146 - accuracy: 0.8867 - val_loss: 0.4191 - val_accuracy: 0.8507
Epoch 9/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2984 - accuracy: 0.8911 - val_loss: 0.3340 - val_accuracy: 0.8815
Epoch 10/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2862 - accuracy: 0.8955 - val_loss: 0.3138 - val_accuracy: 0.8869
Epoch 11/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2735 - accuracy: 0.9001 - val_loss: 0.3191 - val_accuracy: 0.8863
Epoch 12/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2645 - accuracy: 0.9021 - val_loss: 0.2970 - val_accuracy: 0.8955
Epoch 13/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2520 - accuracy: 0.9074 - val_loss: 0.2954 - val_accuracy: 0.8971
Epoch 14/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2436 - accuracy: 0.9106 - val_loss: 0.2855 - val_accuracy: 0.8978
Epoch 15/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2349 - accuracy: 0.9136 - val_loss: 0.2758 - val_accuracy: 0.8995
Epoch 16/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2244 - accuracy: 0.9176 - val_loss: 0.2763 - val_accuracy: 0.9023
Epoch 17/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2181 - accuracy: 0.9193 - val_loss: 0.2817 - val_accuracy: 0.9017
Epoch 18/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2117 - accuracy: 0.9224 - val_loss: 0.2909 - val_accuracy: 0.8964
Epoch 19/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2040 - accuracy: 0.9258 - val_loss: 0.2865 - val_accuracy: 0.8957
Epoch 20/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1961 - accuracy: 0.9279 - val_loss: 0.2796 - val_accuracy: 0.8991


1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'nesterov_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 4ms/step - loss: 0.2970 - accuracy: 0.8942
            run     Perte  Précision
0     basic_CNN  0.469638     0.8282
1        bn_CNN  0.291927     0.8975
2       bn2_CNN  0.271721     0.9046
3      drop_CNN  0.497585     0.8187
4        he_CNN  0.328177     0.8817
5  momentum_CNN  0.286005     0.8988
6  nesterov_CNN  0.296967     0.8942

RMSProp

1
2
3
4
5
6
7
8
optimizer = keras.optimizers.RMSprop(lr=0.001, rho=0.9)

model = base_model(optimizer = optimizer)

history = model.fit(X_train, y_train_oh,
                   epochs = 20,
                   batch_size=64,
                   validation_data=(X_valid, y_valid_oh))

Epoch 1/20
704/704 [==============================] - 8s 12ms/step - loss: 0.4338 - accuracy: 0.8420 - val_loss: 0.4256 - val_accuracy: 0.8446
Epoch 2/20
704/704 [==============================] - 8s 12ms/step - loss: 0.2444 - accuracy: 0.9113 - val_loss: 0.2724 - val_accuracy: 0.9021
Epoch 3/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1981 - accuracy: 0.9287 - val_loss: 0.2215 - val_accuracy: 0.9189
Epoch 4/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1690 - accuracy: 0.9393 - val_loss: 0.2297 - val_accuracy: 0.9157
Epoch 5/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1486 - accuracy: 0.9460 - val_loss: 0.2638 - val_accuracy: 0.9139
Epoch 6/20
704/704 [==============================] - 9s 12ms/step - loss: 0.1311 - accuracy: 0.9536 - val_loss: 0.2867 - val_accuracy: 0.9127
Epoch 7/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1175 - accuracy: 0.9585 - val_loss: 0.2541 - val_accuracy: 0.9213
Epoch 8/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1087 - accuracy: 0.9622 - val_loss: 0.4688 - val_accuracy: 0.9194
Epoch 9/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1003 - accuracy: 0.9652 - val_loss: 0.3328 - val_accuracy: 0.9269
Epoch 10/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0962 - accuracy: 0.9668 - val_loss: 0.4793 - val_accuracy: 0.9161
Epoch 11/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0906 - accuracy: 0.9695 - val_loss: 0.3913 - val_accuracy: 0.9321
Epoch 12/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0894 - accuracy: 0.9706 - val_loss: 0.3182 - val_accuracy: 0.9229
Epoch 13/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0881 - accuracy: 0.9705 - val_loss: 0.4237 - val_accuracy: 0.9284
Epoch 14/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0834 - accuracy: 0.9727 - val_loss: 0.5571 - val_accuracy: 0.9282
Epoch 15/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0817 - accuracy: 0.9743 - val_loss: 0.9855 - val_accuracy: 0.9236
Epoch 16/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0817 - accuracy: 0.9750 - val_loss: 0.3483 - val_accuracy: 0.9257
Epoch 17/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0835 - accuracy: 0.9747 - val_loss: 0.5525 - val_accuracy: 0.9230
Epoch 18/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0858 - accuracy: 0.9732 - val_loss: 0.3046 - val_accuracy: 0.9223
Epoch 19/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0793 - accuracy: 0.9755 - val_loss: 0.4696 - val_accuracy: 0.9265
Epoch 20/20
704/704 [==============================] - 8s 12ms/step - loss: 0.0825 - accuracy: 0.9747 - val_loss: 0.5379 - val_accuracy: 0.9200


1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'RMS_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 3ms/step - loss: 0.6125 - accuracy: 0.9108
            run     Perte  Précision
0     basic_CNN  0.469638     0.8282
1        bn_CNN  0.291927     0.8975
2       bn2_CNN  0.271721     0.9046
3      drop_CNN  0.497585     0.8187
4        he_CNN  0.328177     0.8817
5  momentum_CNN  0.286005     0.8988
6  nesterov_CNN  0.296967     0.8942
7       RMS_CNN  0.612474     0.9108

Adam

1
2
3
4
5
6
7
8
optimizer = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999)

model = base_model(optimizer = optimizer)

history = model.fit(X_train, y_train_oh,
                   epochs = 20,
                   batch_size=64,
                   validation_data=(X_valid, y_valid_oh))

Epoch 1/20
704/704 [==============================] - 8s 11ms/step - loss: 0.4215 - accuracy: 0.8475 - val_loss: 0.2967 - val_accuracy: 0.8911
Epoch 2/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2484 - accuracy: 0.9099 - val_loss: 0.2287 - val_accuracy: 0.9171
Epoch 3/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1963 - accuracy: 0.9278 - val_loss: 0.2135 - val_accuracy: 0.9187
Epoch 4/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1601 - accuracy: 0.9408 - val_loss: 0.2227 - val_accuracy: 0.9195
Epoch 5/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1320 - accuracy: 0.9501 - val_loss: 0.2325 - val_accuracy: 0.9224
Epoch 6/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1037 - accuracy: 0.9602 - val_loss: 0.2395 - val_accuracy: 0.9205
Epoch 7/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0832 - accuracy: 0.9691 - val_loss: 0.2608 - val_accuracy: 0.9260
Epoch 8/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0642 - accuracy: 0.9770 - val_loss: 0.2667 - val_accuracy: 0.9267
Epoch 9/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0513 - accuracy: 0.9810 - val_loss: 0.3028 - val_accuracy: 0.9234
Epoch 10/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0434 - accuracy: 0.9842 - val_loss: 0.2953 - val_accuracy: 0.9257
Epoch 11/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0370 - accuracy: 0.9870 - val_loss: 0.3495 - val_accuracy: 0.9269
Epoch 12/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0355 - accuracy: 0.9873 - val_loss: 0.3201 - val_accuracy: 0.9251
Epoch 13/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0265 - accuracy: 0.9910 - val_loss: 0.3979 - val_accuracy: 0.9211
Epoch 14/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0255 - accuracy: 0.9915 - val_loss: 0.4307 - val_accuracy: 0.9284
Epoch 15/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0256 - accuracy: 0.9910 - val_loss: 0.4208 - val_accuracy: 0.9189
Epoch 16/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0282 - accuracy: 0.9905 - val_loss: 0.3714 - val_accuracy: 0.9237
Epoch 17/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0189 - accuracy: 0.9936 - val_loss: 0.4384 - val_accuracy: 0.9300
Epoch 18/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0178 - accuracy: 0.9933 - val_loss: 0.4159 - val_accuracy: 0.9265
Epoch 19/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0229 - accuracy: 0.9922 - val_loss: 0.3970 - val_accuracy: 0.9282
Epoch 20/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0219 - accuracy: 0.9926 - val_loss: 0.4309 - val_accuracy: 0.9231


1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'Adam_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 4ms/step - loss: 0.5018 - accuracy: 0.9172
            run     Perte  Précision
0     basic_CNN  0.469638     0.8282
1        bn_CNN  0.291927     0.8975
2       bn2_CNN  0.271721     0.9046
3      drop_CNN  0.497585     0.8187
4        he_CNN  0.328177     0.8817
5  momentum_CNN  0.286005     0.8988
6  nesterov_CNN  0.296967     0.8942
7       RMS_CNN  0.612474     0.9108
8      Adam_CNN  0.501753     0.9172

BatchNorm + He + Adam

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
model = keras.models.Sequential([
                                 Input(shape=(28, 28, 1)),
                                 Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),
                                 Conv2D(64, (3, 3), padding='same',use_bias=False,  kernel_initializer="he_normal"),
                                 BatchNormalization(),
                                 MaxPooling2D((2, 2)),
                                 Activation('relu'),
                                 Conv2D(128, (3, 3), activation='relu', padding='same',  kernel_initializer="he_normal"),
                                 Conv2D(128, (3, 3), padding='same',use_bias=False,  kernel_initializer="he_normal"),
                                 BatchNormalization(),
                                 MaxPooling2D((2, 2)),
                                 Activation('relu'),
                                 Flatten(),
                                 Dense(256,  kernel_initializer="he_normal"),
                                 Activation('relu'),
                                 Dense(128,  kernel_initializer="he_normal"),
                                 Activation('relu'),
                                 Dense(10, activation='softmax',  kernel_initializer="he_normal")
                                 ])

model.compile(loss = 'categorical_crossentropy',
              optimizer=tf.keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999),
              metrics=['accuracy'])

history = model.fit(X_train, y_train_oh,
                   epochs = 20,
                   batch_size=64,
                   validation_data=(X_valid, y_valid_oh))

Epoch 1/20
704/704 [==============================] - 8s 12ms/step - loss: 0.4872 - accuracy: 0.8331 - val_loss: 0.3376 - val_accuracy: 0.8719
Epoch 2/20
704/704 [==============================] - 8s 11ms/step - loss: 0.2807 - accuracy: 0.8990 - val_loss: 0.2903 - val_accuracy: 0.8987
Epoch 3/20
704/704 [==============================] - 8s 12ms/step - loss: 0.2239 - accuracy: 0.9176 - val_loss: 0.2428 - val_accuracy: 0.9114
Epoch 4/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1906 - accuracy: 0.9300 - val_loss: 0.2191 - val_accuracy: 0.9189
Epoch 5/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1688 - accuracy: 0.9381 - val_loss: 0.2799 - val_accuracy: 0.9035
Epoch 6/20
704/704 [==============================] - 8s 11ms/step - loss: 0.1460 - accuracy: 0.9471 - val_loss: 0.2471 - val_accuracy: 0.9162
Epoch 7/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1245 - accuracy: 0.9546 - val_loss: 0.2247 - val_accuracy: 0.9223
Epoch 8/20
704/704 [==============================] - 8s 12ms/step - loss: 0.1045 - accuracy: 0.9620 - val_loss: 0.2052 - val_accuracy: 0.9289
Epoch 9/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0879 - accuracy: 0.9678 - val_loss: 0.2891 - val_accuracy: 0.9127
Epoch 10/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0720 - accuracy: 0.9737 - val_loss: 0.2324 - val_accuracy: 0.9281
Epoch 11/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0601 - accuracy: 0.9784 - val_loss: 0.2793 - val_accuracy: 0.9283
Epoch 12/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0480 - accuracy: 0.9830 - val_loss: 0.2750 - val_accuracy: 0.9293
Epoch 13/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0399 - accuracy: 0.9854 - val_loss: 0.2626 - val_accuracy: 0.9333
Epoch 14/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0368 - accuracy: 0.9859 - val_loss: 0.2972 - val_accuracy: 0.9299
Epoch 15/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0356 - accuracy: 0.9871 - val_loss: 0.2719 - val_accuracy: 0.9324
Epoch 16/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0281 - accuracy: 0.9899 - val_loss: 0.3408 - val_accuracy: 0.9231
Epoch 17/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0249 - accuracy: 0.9912 - val_loss: 0.3601 - val_accuracy: 0.9246
Epoch 18/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0269 - accuracy: 0.9900 - val_loss: 0.3535 - val_accuracy: 0.9291
Epoch 19/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0186 - accuracy: 0.9935 - val_loss: 0.3342 - val_accuracy: 0.9309
Epoch 20/20
704/704 [==============================] - 8s 11ms/step - loss: 0.0218 - accuracy: 0.9926 - val_loss: 0.3540 - val_accuracy: 0.9288


1
2
3
4
pd.DataFrame(history.history).plot(figsize=(12,8))
plt.grid(True)
plt.gca().set_ylim(0,1)
plt.show()

1
2
3
4
5
loss, accuracy = model.evaluate(X_test,
                                y_test_oh)

comp = comp.append({'run': 'Bn_He_Adam_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 4ms/step - loss: 0.4062 - accuracy: 0.9219

Keras Tuner et HiPlot

Savoir

  • quel optimiseur utiliser,
  • quel taux d'apprentissage fixer,
  • quel taux de dropout fixer,
  • le nombre de neurones denses,
  • ou de filtres dans une couche convolutive,

tout cela peut être submergeant.

On pout toutefois en automatiser une certaine partie, en espérant que le choix fait sera alors optimal.

1
!pip install hiplot

1
!pip install keras-tuner

1
2
from kerastuner import HyperModel
from kerastuner.tuners import Hyperband

L'objet de base dans Keras-Tuner est la classe HyperModel, et sa variable hyperparamètre défini par la hp. On a plusieurs façon de l'appeler.

  • Faire un choix entre plusieurs possibilités déjà fixées, avec une valeur par défaut. 

    1
2
3
4
    filters=hp.Choice('num_filters_1',
                  values=[32, 64, 128],
                  default=64,
                  )

  • Définir un intervalle de réels \([\mathrm{min \_ value}, \mathrm{max\_value}]\) avec un incrément step, et une valeur par défaut. 

    1
2
3
4
5
6
    rate=hp.Float('dropout_1',
               min_value=0.0,
               max_value=0.5,
               default=0.25,
               step=0.05
               )

  • Définir un intervalle d'entiers \([\mathrm{min\_value}, \mathrm{max\_value}]\) avec un incrément step, et une valeur par défaut. 

    1
2
3
4
5
6
    units=hp.Int('units',
              min_value=32,
              max_value=512,
              step=32,
              default=128
              )


 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
class CNNHyperModel(HyperModel):

    def __init__(self, input_shape, num_classes):
        self.input_shape = input_shape
        self.num_classes = num_classes

    def build(self, hp):
        model = tf.keras.Sequential([
                                     Conv2D(hp.Choice('num_filters_1',
                                                      values=[32, 64],
                                                      default=64
                                                      ), 
                                            (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),

                                     Conv2D(hp.Choice('num_filters_2',
                                                      values=[32, 64],
                                                      default=64
                                                      ), 
                                            (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),

                                     MaxPooling2D((2, 2)),
                                     Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),
                                     Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer="he_normal"),
                                     MaxPooling2D((2, 2)),

                                     Flatten(),
                                     Dense(hp.Int('units',
                                                  min_value=32,
                                                  max_value=512,
                                                  step=32,
                                                  default=256
                                                  ), 
                                           kernel_initializer="he_normal"),

                                     Activation('relu'),
                                     Dense(128, kernel_initializer="he_normal"),
                                     Activation('relu'),

                                     Dense(self.num_classes, 
                                           activation='softmax')
                                     ])


        model.compile(
            optimizer=tf.keras.optimizers.Adam(
                hp.Float(
                    'learning_rate',
                    min_value=1e-4,
                    max_value=1e-2,
                    sampling='LOG',
                    default=1e-3
                )
            ),
            loss='categorical_crossentropy',
            metrics=['accuracy']
        )
        return model

hypermodel = CNNHyperModel(input_shape=(28, 28, 1), num_classes=10)

Une fois notre hypermodèle défini, on instancie un tuner en définissant : - quelle méthode d'optimisation on utilisera( ici RandomSearch), - L'objectif d'optimisation, - Le repertoire directory où seront stockées les résultats.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
from kerastuner.tuners import RandomSearch

tuner = RandomSearch(
    hypermodel,
    objective='val_accuracy',
    seed=RANDOM_SEED,
    max_trials=20,
    directory='random_search',
    project_name='fashion_mnist'
)

  • Le paramètre objective est la fonction à optimiser. Le tuner déduit s'il s'agit d'un problème de maximisation ou de minimisation en fonction de sa valeur.

  • Ensuite, la variable max_trials représente le nombre de combinaisons d'hyperparamètres qui seront testées par le tuner, tandis que la variable execution_per_trial est le nombre de modèles qui devraient être construits et adaptés à chaque essai pour des raisons de robustesse.

  • project_name lui est le nom du dossier dans lequel les données de visualisation seront enregistrées

Le nombre de tuners, ie de méthode de recherche prédéfinies, est au nombre de 3 (https://keras-team.github.io/keras-tuner/documentation/tuners/), on

  • RandomSearch, qui effectue une recherche aléatoire,
  • BayesianOptimization, pour un optimisation Bayesienne,
  • Hyperband, qui utilise l'algorithme HyperBand, méthode d'optimisation via les bandits manchots à \(n\) bras.

L'espace des hyperparamètres est alors visualisable via la commande suivante.

1
tuner.search_space_summary()

Search space summary
|-Default search space size: 4

num_filters_1 (Choice)
|-default: 64
|-ordered: True
|-values: [32, 64]

num_filters_2 (Choice)
|-default: 64
|-ordered: True
|-values: [32, 64]

units (Int)
|-default: 256
|-max_value: 512
|-min_value: 32
|-sampling: None
|-step: 32

learning_rate (Float)
|-default: 0.001
|-max_value: 0.01
|-min_value: 0.0001
|-sampling: log
|-step: None

De la même façon que l'on entraîne un modèle via la commande .fit(), on lance la recherche des hyperparamètres optimaux via la commance .search().

1
2
3
4
5
6
7
8
9
num_of_epochs = 10

callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)
tuner.search(X_train,
             y_train_oh,
             epochs=num_of_epochs,
             batch_size=64,
             validation_data=(X_valid, y_valid_oh),
             callbacks=[callback])

Epoch 1/10
704/704 [==============================] - 8s 11ms/step - loss: 0.4168 - accuracy: 0.8494 - val_loss: 0.2970 - val_accuracy: 0.8934
Epoch 2/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2591 - accuracy: 0.9068 - val_loss: 0.2509 - val_accuracy: 0.9111
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2048 - accuracy: 0.9256 - val_loss: 0.2268 - val_accuracy: 0.9153
Epoch 4/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1666 - accuracy: 0.9387 - val_loss: 0.2306 - val_accuracy: 0.9139
Epoch 5/10
704/704 [==============================] - 7s 11ms/step - loss: 0.1309 - accuracy: 0.9516 - val_loss: 0.2405 - val_accuracy: 0.9124
Epoch 6/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1030 - accuracy: 0.9617 - val_loss: 0.2344 - val_accuracy: 0.9180

Trial complete

Trial summary
|-Trial ID: ad549b3b4b8b6172b82ea67ee5fad689
|-Score: 0.9179999828338623
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0003497444477863496
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 224 
Epoch 1/10
704/704 [==============================] - 9s 13ms/step - loss: 0.4549 - accuracy: 0.8366 - val_loss: 0.3342 - val_accuracy: 0.8806
Epoch 2/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2940 - accuracy: 0.8945 - val_loss: 0.2841 - val_accuracy: 0.8981
Epoch 3/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2436 - accuracy: 0.9111 - val_loss: 0.2525 - val_accuracy: 0.9068
Epoch 4/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2096 - accuracy: 0.9232 - val_loss: 0.2480 - val_accuracy: 0.9079
Epoch 5/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1848 - accuracy: 0.9315 - val_loss: 0.2710 - val_accuracy: 0.8982
Epoch 6/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1603 - accuracy: 0.9408 - val_loss: 0.2396 - val_accuracy: 0.9149
Epoch 7/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1393 - accuracy: 0.9492 - val_loss: 0.2223 - val_accuracy: 0.9186
Epoch 8/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1185 - accuracy: 0.9577 - val_loss: 0.2200 - val_accuracy: 0.9227
Epoch 9/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1012 - accuracy: 0.9627 - val_loss: 0.2714 - val_accuracy: 0.9123
Epoch 10/10
704/704 [==============================] - 9s 12ms/step - loss: 0.0859 - accuracy: 0.9691 - val_loss: 0.2429 - val_accuracy: 0.9205

Trial complete

Trial summary
|-Trial ID: 8279e19274848d0d493e7c40cac4713c
|-Score: 0.9226666688919067
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00013607174450468629
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 288 
Epoch 1/10
704/704 [==============================] - 9s 12ms/step - loss: 0.4173 - accuracy: 0.8495 - val_loss: 0.2987 - val_accuracy: 0.8889
Epoch 2/10
704/704 [==============================] - 8s 12ms/step - loss: 0.2552 - accuracy: 0.9057 - val_loss: 0.2484 - val_accuracy: 0.9107
Epoch 3/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2046 - accuracy: 0.9243 - val_loss: 0.2410 - val_accuracy: 0.9133
Epoch 4/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1704 - accuracy: 0.9368 - val_loss: 0.2299 - val_accuracy: 0.9167
Epoch 5/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1447 - accuracy: 0.9464 - val_loss: 0.2436 - val_accuracy: 0.9174
Epoch 6/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1228 - accuracy: 0.9547 - val_loss: 0.2702 - val_accuracy: 0.9129
Epoch 7/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1011 - accuracy: 0.9632 - val_loss: 0.2520 - val_accuracy: 0.9222

Trial complete

Trial summary
|-Trial ID: 9fa074003b81dc588b82694213d0eff8
|-Score: 0.9222000241279602
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.001715074355925934
|-num_filters_1: 64
|-num_filters_2: 32
|-units: 512 
Epoch 1/10
704/704 [==============================] - 8s 11ms/step - loss: 0.4543 - accuracy: 0.8377 - val_loss: 0.3411 - val_accuracy: 0.8769
Epoch 2/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2938 - accuracy: 0.8938 - val_loss: 0.2950 - val_accuracy: 0.8941
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2420 - accuracy: 0.9123 - val_loss: 0.2572 - val_accuracy: 0.9071
Epoch 4/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2049 - accuracy: 0.9256 - val_loss: 0.2439 - val_accuracy: 0.9099
Epoch 5/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1768 - accuracy: 0.9360 - val_loss: 0.2637 - val_accuracy: 0.9029
Epoch 6/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1513 - accuracy: 0.9450 - val_loss: 0.2431 - val_accuracy: 0.9137
Epoch 7/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1290 - accuracy: 0.9537 - val_loss: 0.2405 - val_accuracy: 0.9125
Epoch 8/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1094 - accuracy: 0.9614 - val_loss: 0.2179 - val_accuracy: 0.9225
Epoch 9/10
704/704 [==============================] - 8s 11ms/step - loss: 0.0903 - accuracy: 0.9676 - val_loss: 0.2728 - val_accuracy: 0.9088
Epoch 10/10
704/704 [==============================] - 8s 11ms/step - loss: 0.0733 - accuracy: 0.9739 - val_loss: 0.2495 - val_accuracy: 0.9229

Trial complete

Trial summary
|-Trial ID: ad4279e13d037a77eb612f9367544ae9
|-Score: 0.9228666424751282
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00012165541012624621
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 480 
Epoch 1/10
704/704 [==============================] - 8s 11ms/step - loss: 0.3942 - accuracy: 0.8578 - val_loss: 0.2787 - val_accuracy: 0.8969
Epoch 2/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2472 - accuracy: 0.9094 - val_loss: 0.2325 - val_accuracy: 0.9128
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1933 - accuracy: 0.9278 - val_loss: 0.2098 - val_accuracy: 0.9207
Epoch 4/10
704/704 [==============================] - 7s 11ms/step - loss: 0.1571 - accuracy: 0.9410 - val_loss: 0.2531 - val_accuracy: 0.9041
Epoch 5/10
704/704 [==============================] - 7s 10ms/step - loss: 0.1258 - accuracy: 0.9524 - val_loss: 0.2383 - val_accuracy: 0.9174
Epoch 6/10
704/704 [==============================] - 7s 11ms/step - loss: 0.0988 - accuracy: 0.9629 - val_loss: 0.2387 - val_accuracy: 0.9184

Trial complete

Trial summary
|-Trial ID: 0d06d8240a239bf91872c87cdfffd081
|-Score: 0.9206666946411133
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0009749332269823054
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 192 
Epoch 1/10
704/704 [==============================] - 9s 12ms/step - loss: 0.4211 - accuracy: 0.8496 - val_loss: 0.3052 - val_accuracy: 0.8866
Epoch 2/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2540 - accuracy: 0.9084 - val_loss: 0.2378 - val_accuracy: 0.9118
Epoch 3/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2002 - accuracy: 0.9262 - val_loss: 0.2149 - val_accuracy: 0.9195
Epoch 4/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1636 - accuracy: 0.9400 - val_loss: 0.2114 - val_accuracy: 0.9220
Epoch 5/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1310 - accuracy: 0.9507 - val_loss: 0.2307 - val_accuracy: 0.9196
Epoch 6/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1057 - accuracy: 0.9606 - val_loss: 0.2163 - val_accuracy: 0.9247
Epoch 7/10
704/704 [==============================] - 9s 12ms/step - loss: 0.0809 - accuracy: 0.9695 - val_loss: 0.2183 - val_accuracy: 0.9293

Trial complete

Trial summary
|-Trial ID: 888c4d90d81bb8a15db36715a329abe5
|-Score: 0.9293333292007446
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0006752863927347823
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 256 
Epoch 1/10
704/704 [==============================] - 8s 11ms/step - loss: 0.4065 - accuracy: 0.8545 - val_loss: 0.2793 - val_accuracy: 0.8965
Epoch 2/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2476 - accuracy: 0.9104 - val_loss: 0.2366 - val_accuracy: 0.9121
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2009 - accuracy: 0.9257 - val_loss: 0.2230 - val_accuracy: 0.9177
Epoch 4/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1716 - accuracy: 0.9367 - val_loss: 0.2302 - val_accuracy: 0.9198
Epoch 5/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1524 - accuracy: 0.9441 - val_loss: 0.2771 - val_accuracy: 0.9065
Epoch 6/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1278 - accuracy: 0.9526 - val_loss: 0.2299 - val_accuracy: 0.9222

Trial complete

Trial summary
|-Trial ID: 3e5e38f8970e988856d96a81dd0cb384
|-Score: 0.9222000241279602
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0023360487671257027
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 384 
Epoch 1/10
704/704 [==============================] - 8s 12ms/step - loss: 0.4540 - accuracy: 0.8346 - val_loss: 0.3190 - val_accuracy: 0.8851
Epoch 2/10
704/704 [==============================] - 8s 12ms/step - loss: 0.2785 - accuracy: 0.8996 - val_loss: 0.2580 - val_accuracy: 0.9079
Epoch 3/10
704/704 [==============================] - 8s 12ms/step - loss: 0.2256 - accuracy: 0.9175 - val_loss: 0.2406 - val_accuracy: 0.9104
Epoch 4/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1921 - accuracy: 0.9306 - val_loss: 0.2271 - val_accuracy: 0.9160
Epoch 5/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1627 - accuracy: 0.9394 - val_loss: 0.2319 - val_accuracy: 0.9171
Epoch 6/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1382 - accuracy: 0.9488 - val_loss: 0.2194 - val_accuracy: 0.9218
Epoch 7/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1126 - accuracy: 0.9588 - val_loss: 0.2146 - val_accuracy: 0.9239
Epoch 8/10
704/704 [==============================] - 9s 12ms/step - loss: 0.0916 - accuracy: 0.9668 - val_loss: 0.2299 - val_accuracy: 0.9251
Epoch 9/10
704/704 [==============================] - 8s 12ms/step - loss: 0.0734 - accuracy: 0.9732 - val_loss: 0.3007 - val_accuracy: 0.9144
Epoch 10/10
704/704 [==============================] - 8s 12ms/step - loss: 0.0611 - accuracy: 0.9776 - val_loss: 0.2920 - val_accuracy: 0.9189

Trial complete

Trial summary
|-Trial ID: 8b8f262dc35d2c9bec969b0c7126eb8d
|-Score: 0.9251333475112915
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00036276965095376624
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 64 
Epoch 1/10
704/704 [==============================] - 9s 12ms/step - loss: 0.4737 - accuracy: 0.8366 - val_loss: 0.3217 - val_accuracy: 0.8825
Epoch 2/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2988 - accuracy: 0.8895 - val_loss: 0.3082 - val_accuracy: 0.8886
Epoch 3/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2627 - accuracy: 0.9015 - val_loss: 0.2628 - val_accuracy: 0.9033
Epoch 4/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2367 - accuracy: 0.9128 - val_loss: 0.2531 - val_accuracy: 0.9063
Epoch 5/10
704/704 [==============================] - 8s 12ms/step - loss: 0.2206 - accuracy: 0.9181 - val_loss: 0.2952 - val_accuracy: 0.8993
Epoch 6/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2109 - accuracy: 0.9216 - val_loss: 0.2781 - val_accuracy: 0.8996
Epoch 7/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1948 - accuracy: 0.9280 - val_loss: 0.2531 - val_accuracy: 0.9113
Epoch 8/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1826 - accuracy: 0.9320 - val_loss: 0.2671 - val_accuracy: 0.9127
Epoch 9/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1887 - accuracy: 0.9316 - val_loss: 0.2836 - val_accuracy: 0.9081
Epoch 10/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1691 - accuracy: 0.9387 - val_loss: 0.2690 - val_accuracy: 0.9123

Trial complete

Trial summary
|-Trial ID: e63dadbd65de215a3097342f65d4ca6b
|-Score: 0.9126666784286499
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.003964886305018653
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 224 
Epoch 1/10
704/704 [==============================] - 8s 11ms/step - loss: 0.4152 - accuracy: 0.8523 - val_loss: 0.2942 - val_accuracy: 0.8907
Epoch 2/10
704/704 [==============================] - 7s 11ms/step - loss: 0.2526 - accuracy: 0.9078 - val_loss: 0.2415 - val_accuracy: 0.9092
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1968 - accuracy: 0.9270 - val_loss: 0.2263 - val_accuracy: 0.9163
Epoch 4/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1632 - accuracy: 0.9407 - val_loss: 0.2260 - val_accuracy: 0.9151
Epoch 5/10
704/704 [==============================] - 7s 11ms/step - loss: 0.1297 - accuracy: 0.9519 - val_loss: 0.2575 - val_accuracy: 0.9107
Epoch 6/10
704/704 [==============================] - 7s 11ms/step - loss: 0.1058 - accuracy: 0.9604 - val_loss: 0.2581 - val_accuracy: 0.9118
Epoch 7/10
704/704 [==============================] - 7s 11ms/step - loss: 0.0838 - accuracy: 0.9694 - val_loss: 0.2470 - val_accuracy: 0.9201

Trial complete

Trial summary
|-Trial ID: c5a660c5b9f51d34c7738dac5b546771
|-Score: 0.9201333522796631
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0010277342442932575
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 160 
Epoch 1/10
704/704 [==============================] - 9s 13ms/step - loss: 0.4259 - accuracy: 0.8470 - val_loss: 0.3122 - val_accuracy: 0.8875
Epoch 2/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2672 - accuracy: 0.9037 - val_loss: 0.2500 - val_accuracy: 0.9089
Epoch 3/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2149 - accuracy: 0.9222 - val_loss: 0.2419 - val_accuracy: 0.9112
Epoch 4/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1790 - accuracy: 0.9345 - val_loss: 0.2294 - val_accuracy: 0.9140
Epoch 5/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1480 - accuracy: 0.9460 - val_loss: 0.2373 - val_accuracy: 0.9131
Epoch 6/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1224 - accuracy: 0.9550 - val_loss: 0.2487 - val_accuracy: 0.9127
Epoch 7/10
704/704 [==============================] - 9s 12ms/step - loss: 0.0965 - accuracy: 0.9646 - val_loss: 0.2306 - val_accuracy: 0.9225

Trial complete

Trial summary
|-Trial ID: 0bf67cf27d543ea733d47f39075cf186
|-Score: 0.9224666953086853
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00025071332274895686
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 384 
Epoch 1/10
704/704 [==============================] - 8s 11ms/step - loss: 0.4525 - accuracy: 0.8384 - val_loss: 0.3305 - val_accuracy: 0.8824
Epoch 2/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2939 - accuracy: 0.8955 - val_loss: 0.2951 - val_accuracy: 0.8953
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2412 - accuracy: 0.9121 - val_loss: 0.2610 - val_accuracy: 0.9045
Epoch 4/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2081 - accuracy: 0.9243 - val_loss: 0.2400 - val_accuracy: 0.9121
Epoch 5/10
704/704 [==============================] - 7s 11ms/step - loss: 0.1785 - accuracy: 0.9348 - val_loss: 0.2951 - val_accuracy: 0.8959
Epoch 6/10
704/704 [==============================] - 7s 11ms/step - loss: 0.1545 - accuracy: 0.9441 - val_loss: 0.2476 - val_accuracy: 0.9129
Epoch 7/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1317 - accuracy: 0.9523 - val_loss: 0.2356 - val_accuracy: 0.9157
Epoch 8/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1106 - accuracy: 0.9619 - val_loss: 0.2424 - val_accuracy: 0.9171
Epoch 9/10
704/704 [==============================] - 7s 11ms/step - loss: 0.0930 - accuracy: 0.9662 - val_loss: 0.2706 - val_accuracy: 0.9117
Epoch 10/10
704/704 [==============================] - 7s 11ms/step - loss: 0.0744 - accuracy: 0.9736 - val_loss: 0.2584 - val_accuracy: 0.9219

Trial complete

Trial summary
|-Trial ID: 6a1861dcde046721d62be37072693fe1
|-Score: 0.9218666553497314
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00014527180385656086
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 224 
Epoch 1/10
704/704 [==============================] - 8s 11ms/step - loss: 0.5618 - accuracy: 0.8244 - val_loss: 0.3947 - val_accuracy: 0.8569
Epoch 2/10
704/704 [==============================] - 8s 11ms/step - loss: 0.3342 - accuracy: 0.8777 - val_loss: 0.3224 - val_accuracy: 0.8801
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2999 - accuracy: 0.8881 - val_loss: 0.2986 - val_accuracy: 0.8887
Epoch 4/10
704/704 [==============================] - 7s 11ms/step - loss: 0.2818 - accuracy: 0.8957 - val_loss: 0.2983 - val_accuracy: 0.8874
Epoch 5/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2721 - accuracy: 0.8990 - val_loss: 0.3596 - val_accuracy: 0.8762
Epoch 6/10
704/704 [==============================] - 7s 11ms/step - loss: 0.2709 - accuracy: 0.8992 - val_loss: 0.3247 - val_accuracy: 0.8817
Epoch 7/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2687 - accuracy: 0.9011 - val_loss: 0.3011 - val_accuracy: 0.8930

Trial complete

Trial summary
|-Trial ID: c284d54100b20a32b6aa92ddafcb7c29
|-Score: 0.8930000066757202
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.006678619769811224
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 224 
Epoch 1/10
704/704 [==============================] - 8s 11ms/step - loss: 0.4399 - accuracy: 0.8412 - val_loss: 0.3216 - val_accuracy: 0.8828
Epoch 2/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2793 - accuracy: 0.8999 - val_loss: 0.2713 - val_accuracy: 0.9025
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2268 - accuracy: 0.9173 - val_loss: 0.2415 - val_accuracy: 0.9102
Epoch 4/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1911 - accuracy: 0.9298 - val_loss: 0.2375 - val_accuracy: 0.9106
Epoch 5/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1609 - accuracy: 0.9408 - val_loss: 0.2751 - val_accuracy: 0.9025
Epoch 6/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1366 - accuracy: 0.9488 - val_loss: 0.2382 - val_accuracy: 0.9157
Epoch 7/10
704/704 [==============================] - 8s 11ms/step - loss: 0.1105 - accuracy: 0.9593 - val_loss: 0.2396 - val_accuracy: 0.9187

Trial complete

Trial summary
|-Trial ID: 932a4440c0a7e4b8e0f629c87a4f1789
|-Score: 0.918666660785675
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0002449483373216793
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 224 
Epoch 1/10
704/704 [==============================] - 9s 13ms/step - loss: 0.4237 - accuracy: 0.8497 - val_loss: 0.2860 - val_accuracy: 0.8974
Epoch 2/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2604 - accuracy: 0.9061 - val_loss: 0.2438 - val_accuracy: 0.9117
Epoch 3/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2069 - accuracy: 0.9225 - val_loss: 0.2246 - val_accuracy: 0.9173
Epoch 4/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1730 - accuracy: 0.9359 - val_loss: 0.2231 - val_accuracy: 0.9191
Epoch 5/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1485 - accuracy: 0.9436 - val_loss: 0.2548 - val_accuracy: 0.9167
Epoch 6/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1267 - accuracy: 0.9520 - val_loss: 0.2448 - val_accuracy: 0.9167
Epoch 7/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1045 - accuracy: 0.9610 - val_loss: 0.2277 - val_accuracy: 0.9247

Trial complete

Trial summary
|-Trial ID: 0c621859edde556edb80c57cf0c8266b
|-Score: 0.9247333407402039
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.001453129317743269
|-num_filters_1: 64
|-num_filters_2: 64
|-units: 96 
Epoch 1/10
704/704 [==============================] - 9s 13ms/step - loss: 0.5476 - accuracy: 0.8286 - val_loss: 0.3346 - val_accuracy: 0.8709
Epoch 2/10
704/704 [==============================] - 9s 13ms/step - loss: 0.3074 - accuracy: 0.8883 - val_loss: 0.3067 - val_accuracy: 0.8888
Epoch 3/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2723 - accuracy: 0.9013 - val_loss: 0.3265 - val_accuracy: 0.8818
Epoch 4/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2559 - accuracy: 0.9060 - val_loss: 0.2848 - val_accuracy: 0.8953
Epoch 5/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2425 - accuracy: 0.9114 - val_loss: 0.2858 - val_accuracy: 0.8973
Epoch 6/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2384 - accuracy: 0.9137 - val_loss: 0.2825 - val_accuracy: 0.9000
Epoch 7/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2212 - accuracy: 0.9194 - val_loss: 0.3535 - val_accuracy: 0.8835
Epoch 8/10
704/704 [==============================] - 9s 12ms/step - loss: 0.2159 - accuracy: 0.9218 - val_loss: 0.3222 - val_accuracy: 0.8905
Epoch 9/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2137 - accuracy: 0.9227 - val_loss: 0.3173 - val_accuracy: 0.9015

Trial complete

Trial summary
|-Trial ID: c54d5d40604c49d5387cec2703cf3750
|-Score: 0.9014666676521301
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.005704251931830293
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 480 
Epoch 1/10
704/704 [==============================] - 8s 12ms/step - loss: 0.4362 - accuracy: 0.8432 - val_loss: 0.3162 - val_accuracy: 0.8859
Epoch 2/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2736 - accuracy: 0.9010 - val_loss: 0.2606 - val_accuracy: 0.9046
Epoch 3/10
704/704 [==============================] - 8s 11ms/step - loss: 0.2196 - accuracy: 0.9194 - val_loss: 0.2443 - val_accuracy: 0.9090
Epoch 4/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1840 - accuracy: 0.9330 - val_loss: 0.2176 - val_accuracy: 0.9209
Epoch 5/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1509 - accuracy: 0.9437 - val_loss: 0.2442 - val_accuracy: 0.9140
Epoch 6/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1269 - accuracy: 0.9519 - val_loss: 0.2352 - val_accuracy: 0.9165
Epoch 7/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1004 - accuracy: 0.9631 - val_loss: 0.2251 - val_accuracy: 0.9240

Trial complete

Trial summary
|-Trial ID: 822f72617459e63dd80979cd498fca21
|-Score: 0.9240000247955322
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0003623815491328145
|-num_filters_1: 64
|-num_filters_2: 32
|-units: 96 
Epoch 1/10
704/704 [==============================] - 8s 12ms/step - loss: 0.4912 - accuracy: 0.8245 - val_loss: 0.3394 - val_accuracy: 0.8806
Epoch 2/10
704/704 [==============================] - 8s 12ms/step - loss: 0.3076 - accuracy: 0.8901 - val_loss: 0.3014 - val_accuracy: 0.8927
Epoch 3/10
704/704 [==============================] - 8s 12ms/step - loss: 0.2532 - accuracy: 0.9088 - val_loss: 0.2668 - val_accuracy: 0.9023
Epoch 4/10
704/704 [==============================] - 8s 12ms/step - loss: 0.2165 - accuracy: 0.9220 - val_loss: 0.2599 - val_accuracy: 0.9049
Epoch 5/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1894 - accuracy: 0.9314 - val_loss: 0.2814 - val_accuracy: 0.8993
Epoch 6/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1650 - accuracy: 0.9392 - val_loss: 0.2432 - val_accuracy: 0.9114
Epoch 7/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1430 - accuracy: 0.9474 - val_loss: 0.2406 - val_accuracy: 0.9134
Epoch 8/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1213 - accuracy: 0.9561 - val_loss: 0.2293 - val_accuracy: 0.9198
Epoch 9/10
704/704 [==============================] - 8s 12ms/step - loss: 0.1034 - accuracy: 0.9616 - val_loss: 0.2832 - val_accuracy: 0.9069
Epoch 10/10
704/704 [==============================] - 8s 12ms/step - loss: 0.0859 - accuracy: 0.9685 - val_loss: 0.2390 - val_accuracy: 0.9210

Trial complete

Trial summary
|-Trial ID: 63e62b8aedcfffad94d17a4acb85075b
|-Score: 0.9210000038146973
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00011510936029076842
|-num_filters_1: 64
|-num_filters_2: 32
|-units: 256 
Epoch 1/10
704/704 [==============================] - 9s 13ms/step - loss: 0.4175 - accuracy: 0.8509 - val_loss: 0.3105 - val_accuracy: 0.8843
Epoch 2/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2631 - accuracy: 0.9051 - val_loss: 0.2478 - val_accuracy: 0.9088
Epoch 3/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2082 - accuracy: 0.9236 - val_loss: 0.2309 - val_accuracy: 0.9131
Epoch 4/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1709 - accuracy: 0.9375 - val_loss: 0.2193 - val_accuracy: 0.9191
Epoch 5/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1375 - accuracy: 0.9492 - val_loss: 0.2253 - val_accuracy: 0.9181
Epoch 6/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1088 - accuracy: 0.9598 - val_loss: 0.2426 - val_accuracy: 0.9189
Epoch 7/10
704/704 [==============================] - 9s 13ms/step - loss: 0.0838 - accuracy: 0.9689 - val_loss: 0.2514 - val_accuracy: 0.9168

Trial complete

Trial summary
|-Trial ID: ff63b04ae658949175a2fa79197ae8a3
|-Score: 0.9191333055496216
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0003021384939866327
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 416 
Epoch 1/10
704/704 [==============================] - 9s 13ms/step - loss: 0.4488 - accuracy: 0.8391 - val_loss: 0.3204 - val_accuracy: 0.8839
Epoch 2/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2875 - accuracy: 0.8973 - val_loss: 0.2801 - val_accuracy: 0.8988
Epoch 3/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2347 - accuracy: 0.9142 - val_loss: 0.2502 - val_accuracy: 0.9074
Epoch 4/10
704/704 [==============================] - 9s 13ms/step - loss: 0.2011 - accuracy: 0.9279 - val_loss: 0.2469 - val_accuracy: 0.9082
Epoch 5/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1717 - accuracy: 0.9368 - val_loss: 0.2625 - val_accuracy: 0.9028
Epoch 6/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1460 - accuracy: 0.9471 - val_loss: 0.2437 - val_accuracy: 0.9103
Epoch 7/10
704/704 [==============================] - 9s 13ms/step - loss: 0.1233 - accuracy: 0.9546 - val_loss: 0.2213 - val_accuracy: 0.9224
Epoch 8/10
704/704 [==============================] - 9s 12ms/step - loss: 0.1042 - accuracy: 0.9624 - val_loss: 0.2427 - val_accuracy: 0.9189
Epoch 9/10
704/704 [==============================] - 9s 12ms/step - loss: 0.0852 - accuracy: 0.9692 - val_loss: 0.2911 - val_accuracy: 0.9087
Epoch 10/10
704/704 [==============================] - 9s 12ms/step - loss: 0.0690 - accuracy: 0.9748 - val_loss: 0.2558 - val_accuracy: 0.9183

Trial complete

Trial summary
|-Trial ID: 9e654a297fcccf88ea1724cb684f6542
|-Score: 0.9223999977111816
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00014948684422968163
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 288 
INFO:tensorflow:Oracle triggered exit


1
tuner.results_summary()

Results summary
|-Results in random_search/fashion_mnist
|-Showing 10 best trials
|-Objective(name='val_accuracy', direction='max')

Trial summary
|-Trial ID: 888c4d90d81bb8a15db36715a329abe5
|-Score: 0.9293333292007446
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0006752863927347823
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 256

Trial summary
|-Trial ID: 8b8f262dc35d2c9bec969b0c7126eb8d
|-Score: 0.9251333475112915
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00036276965095376624
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 64

Trial summary
|-Trial ID: 0c621859edde556edb80c57cf0c8266b
|-Score: 0.9247333407402039
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.001453129317743269
|-num_filters_1: 64
|-num_filters_2: 64
|-units: 96

Trial summary
|-Trial ID: 822f72617459e63dd80979cd498fca21
|-Score: 0.9240000247955322
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0003623815491328145
|-num_filters_1: 64
|-num_filters_2: 32
|-units: 96

Trial summary
|-Trial ID: ad4279e13d037a77eb612f9367544ae9
|-Score: 0.9228666424751282
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00012165541012624621
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 480

Trial summary
|-Trial ID: 8279e19274848d0d493e7c40cac4713c
|-Score: 0.9226666688919067
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00013607174450468629
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 288

Trial summary
|-Trial ID: 0bf67cf27d543ea733d47f39075cf186
|-Score: 0.9224666953086853
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00025071332274895686
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 384

Trial summary
|-Trial ID: 9e654a297fcccf88ea1724cb684f6542
|-Score: 0.9223999977111816
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.00014948684422968163
|-num_filters_1: 32
|-num_filters_2: 64
|-units: 288

Trial summary
|-Trial ID: 9fa074003b81dc588b82694213d0eff8
|-Score: 0.9222000241279602
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.001715074355925934
|-num_filters_1: 64
|-num_filters_2: 32
|-units: 512

Trial summary
|-Trial ID: 3e5e38f8970e988856d96a81dd0cb384
|-Score: 0.9222000241279602
|-Best step: 0

Hyperparameters:
|-learning_rate: 0.0023360487671257027
|-num_filters_1: 32
|-num_filters_2: 32
|-units: 384 
1
best_model = tuner.get_best_models(num_models=1)[0]

1
2
3
4
5
loss, accuracy = best_model.evaluate(X_test,
                                     y_test_oh)

comp = comp.append({'run': 'Optim_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 4ms/step - loss: 0.2551 - accuracy: 0.9246
               run     Perte  Précision
0        basic_CNN  0.469638     0.8282
1           bn_CNN  0.291927     0.8975
2          bn2_CNN  0.271721     0.9046
3         drop_CNN  0.497585     0.8187
4           he_CNN  0.328177     0.8817
5     momentum_CNN  0.286005     0.8988
6     nesterov_CNN  0.296967     0.8942
7          RMS_CNN  0.612474     0.9108
8         Adam_CNN  0.501753     0.9172
9   Bn_He_Adam_CNN  0.395749     0.9219
10       Optim_CNN  0.255052     0.9246
11      Optim2_CNN  0.584973     0.9308
12       Optim_CNN  0.255052     0.9246


 1
 2
 3
 4
 5
 6
 7
 8
 9
10
import json

vis_data = []
rootdir = 'random_search/fashion_mnist'
for subdirs, dirs, files in os.walk(rootdir):
    for file in files:
        if file.endswith("trial.json"):
          with open(subdirs + '/' + file, 'r') as json_file:
            data = json_file.read()
          vis_data.append(json.loads(data))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
import hiplot as hip

data = [{'num_filters_1': vis_data[idx]['hyperparameters']['values']['num_filters_1'],
         'num_filters_2': vis_data[idx]['hyperparameters']['values']['num_filters_2'],
         'units': vis_data[idx]['hyperparameters']['values']['units'], 
         'learning_rate': vis_data[idx]['hyperparameters']['values']['learning_rate'], 
         'loss': vis_data[idx]['metrics']['metrics']['loss']['observations'][0]['value'],  
         'val_loss': vis_data[idx]['metrics']['metrics']['val_loss']['observations'][0]['value'], 
         'accuracy': vis_data[idx]['metrics']['metrics']['accuracy']['observations'][0]['value'],
         'val_accuracy': vis_data[idx]['metrics']['metrics']['val_accuracy']['observations'][0]['value']} for idx in range(num_of_epochs)]

hip.Experiment.from_iterable(data).display()

1
2
lr0 = tuner.oracle.get_best_trials(num_trials=1)[0].hyperparameters.values['learning_rate']
print(lr0)

0.0006752863927347823


 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
def exponential_decay(lr0,step):
    def exponential_decay_fn(epoch):
        return lr0*0.1**(epoch/step)
    return exponential_decay_fn

exponential_decay_fn = exponential_decay(lr0 = lr0, step = 20)

callbacks_fit = [keras.callbacks.EarlyStopping(
                 # Stop training when `val_loss` is no longer improving
                 monitor='val_loss',
                 # "no longer improving" being defined as "no better than 1e-2 less"
                 min_delta=1e-2,
                 # "no longer improving" being further defined as "for at least 2 epochs"
                 patience=10,
                 verbose=1),
                 keras.callbacks.LearningRateScheduler(exponential_decay_fn),
                 keras.callbacks.ModelCheckpoint('./weights.{epoch:02d}-.hdf5',
                                                 monitor='val_loss',
                                                 verbose=1,
                                                 save_best_only=True,
                                                 save_weights_only=False,
                                                 mode='auto',
                                                 save_freq='epoch')]

1
2
3
4
5
6
7
8
history = best_model.fit(X_train,
                         y_train_oh,
                         validation_data=(X_valid, y_valid_oh),
                         batch_size=64,
                         epochs=100,
                         initial_epoch=10,
                         callbacks=callbacks_fit,
                         verbose=2)

1
2
3
4
5
loss, accuracy = best_model.evaluate(X_test,
                                     y_test_oh)

comp = comp.append({'run': 'Optim2_CNN', 'Perte' : loss, 'Précision' : accuracy}, ignore_index=True)
print(comp)

313/313 [==============================] - 1s 4ms/step - loss: 0.5850 - accuracy: 0.9308
               run     Perte  Précision
0        basic_CNN  0.469638     0.8282
1           bn_CNN  0.291927     0.8975
2          bn2_CNN  0.271721     0.9046
3         drop_CNN  0.497585     0.8187
4           he_CNN  0.328177     0.8817
5     momentum_CNN  0.286005     0.8988
6     nesterov_CNN  0.296967     0.8942
7          RMS_CNN  0.612474     0.9108
8         Adam_CNN  0.501753     0.9172
9   Bn_He_Adam_CNN  0.395749     0.9219
10       Optim_CNN  0.255052     0.9246
11      Optim2_CNN  0.584973     0.9308

Comparaison

1
2
3
4
5
6
# import plotly.express as px
# fig = px.scatter(comp, 
#                 x="Précision", 
#                 y="Perte",
#                 color="run")
# fig.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
        <script type="text/javascript">

                window.PLOTLYENV=window.PLOTLYENV || {};

            if (document.getElementById("d3a19371-4742-4764-85ae-aa0c67d28833")) {
                Plotly.newPlot(
                    'd3a19371-4742-4764-85ae-aa0c67d28833',
                    [{"hoverlabel": {"namelength": 0}, "hovertemplate": "run=basic_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=basic_CNN", "marker": {"color": "#636efa", "symbol": "circle"}, "mode": "markers", "name": "run=basic_CNN", "showlegend": true, "type": "scatter", "x": [0.8281999826431274], "xaxis": "x", "y": [0.46963778138160706], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=bn_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=bn_CNN", "marker": {"color": "#EF553B", "symbol": "circle"}, "mode": "markers", "name": "run=bn_CNN", "showlegend": true, "type": "scatter", "x": [0.8974999785423279], "xaxis": "x", "y": [0.29192668199539185], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=bn2_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=bn2_CNN", "marker": {"color": "#00cc96", "symbol": "circle"}, "mode": "markers", "name": "run=bn2_CNN", "showlegend": true, "type": "scatter", "x": [0.9046000242233276], "xaxis": "x", "y": [0.27172139286994934], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=drop_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=drop_CNN", "marker": {"color": "#ab63fa", "symbol": "circle"}, "mode": "markers", "name": "run=drop_CNN", "showlegend": true, "type": "scatter", "x": [0.8187000155448914], "xaxis": "x", "y": [0.49758535623550415], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=he_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=he_CNN", "marker": {"color": "#FFA15A", "symbol": "circle"}, "mode": "markers", "name": "run=he_CNN", "showlegend": true, "type": "scatter", "x": [0.8816999793052673], "xaxis": "x", "y": [0.32817697525024414], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=momentum_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=momentum_CNN", "marker": {"color": "#19d3f3", "symbol": "circle"}, "mode": "markers", "name": "run=momentum_CNN", "showlegend": true, "type": "scatter", "x": [0.8988000154495239], "xaxis": "x", "y": [0.2860051989555359], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=nesterov_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=nesterov_CNN", "marker": {"color": "#FF6692", "symbol": "circle"}, "mode": "markers", "name": "run=nesterov_CNN", "showlegend": true, "type": "scatter", "x": [0.8942000269889832], "xaxis": "x", "y": [0.29696694016456604], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=RMS_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=RMS_CNN", "marker": {"color": "#B6E880", "symbol": "circle"}, "mode": "markers", "name": "run=RMS_CNN", "showlegend": true, "type": "scatter", "x": [0.9107999801635742], "xaxis": "x", "y": [0.6124737858772278], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=Adam_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=Adam_CNN", "marker": {"color": "#FF97FF", "symbol": "circle"}, "mode": "markers", "name": "run=Adam_CNN", "showlegend": true, "type": "scatter", "x": [0.9172000288963318], "xaxis": "x", "y": [0.5017529129981995], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=Bn_He_Adam_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=Bn_He_Adam_CNN", "marker": {"color": "#FECB52", "symbol": "circle"}, "mode": "markers", "name": "run=Bn_He_Adam_CNN", "showlegend": true, "type": "scatter", "x": [0.9218999743461609], "xaxis": "x", "y": [0.39574897289276123], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=Optim_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=Optim_CNN", "marker": {"color": "#636efa", "symbol": "circle"}, "mode": "markers", "name": "run=Optim_CNN", "showlegend": true, "type": "scatter", "x": [0.9246000051498413, 0.9246000051498413], "xaxis": "x", "y": [0.25505155324935913, 0.25505155324935913], "yaxis": "y"}, {"hoverlabel": {"namelength": 0}, "hovertemplate": "run=Optim2_CNN<br>Pr\u00e9cision=%{x}<br>Perte=%{y}", "legendgroup": "run=Optim2_CNN", "marker": {"color": "#EF553B", "symbol": "circle"}, "mode": "markers", "name": "run=Optim2_CNN", "showlegend": true, "type": "scatter", "x": [0.9308000206947327], "xaxis": "x", "y": [0.5849732756614685], "yaxis": "y"}],
                    {"legend": {"tracegroupgap": 0}, "margin": {"t": 60}, "template": {"data": {"bar": [{"error_x": {"color": "#2a3f5f"}, "error_y": {"color": "#2a3f5f"}, "marker": {"line": {"color": "#E5ECF6", "width": 0.5}}, "type": "bar"}], "barpolar": [{"marker": {"line": {"color": "#E5ECF6", "width": 0.5}}, "type": "barpolar"}], "carpet": [{"aaxis": {"endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f"}, "baxis": {"endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f"}, "type": "carpet"}], "choropleth": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "type": "choropleth"}], "contour": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "type": "contour"}], "contourcarpet": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "type": "contourcarpet"}], "heatmap": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "type": "heatmap"}], "heatmapgl": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "type": "heatmapgl"}], "histogram": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "histogram"}], "histogram2d": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "type": "histogram2d"}], "histogram2dcontour": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "type": "histogram2dcontour"}], "mesh3d": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "type": "mesh3d"}], "parcoords": [{"line": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "parcoords"}], "pie": [{"automargin": true, "type": "pie"}], "scatter": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scatter"}], "scatter3d": [{"line": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scatter3d"}], "scattercarpet": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scattercarpet"}], "scattergeo": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scattergeo"}], "scattergl": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scattergl"}], "scattermapbox": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scattermapbox"}], "scatterpolar": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scatterpolar"}], "scatterpolargl": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scatterpolargl"}], "scatterternary": [{"marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "type": "scatterternary"}], "surface": [{"colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "type": "surface"}], "table": [{"cells": {"fill": {"color": "#EBF0F8"}, "line": {"color": "white"}}, "header": {"fill": {"color": "#C8D4E3"}, "line": {"color": "white"}}, "type": "table"}]}, "layout": {"annotationdefaults": {"arrowcolor": "#2a3f5f", "arrowhead": 0, "arrowwidth": 1}, "coloraxis": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "colorscale": {"diverging": [[0, "#8e0152"], [0.1, "#c51b7d"], [0.2, "#de77ae"], [0.3, "#f1b6da"], [0.4, "#fde0ef"], [0.5, "#f7f7f7"], [0.6, "#e6f5d0"], [0.7, "#b8e186"], [0.8, "#7fbc41"], [0.9, "#4d9221"], [1, "#276419"]], "sequential": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "sequentialminus": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]]}, "colorway": ["#636efa", "#EF553B", "#00cc96", "#ab63fa", "#FFA15A", "#19d3f3", "#FF6692", "#B6E880", "#FF97FF", "#FECB52"], "font": {"color": "#2a3f5f"}, "geo": {"bgcolor": "white", "lakecolor": "white", "landcolor": "#E5ECF6", "showlakes": true, "showland": true, "subunitcolor": "white"}, "hoverlabel": {"align": "left"}, "hovermode": "closest", "mapbox": {"style": "light"}, "paper_bgcolor": "white", "plot_bgcolor": "#E5ECF6", "polar": {"angularaxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}, "bgcolor": "#E5ECF6", "radialaxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}}, "scene": {"xaxis": {"backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white"}, "yaxis": {"backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white"}, "zaxis": {"backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white"}}, "shapedefaults": {"line": {"color": "#2a3f5f"}}, "ternary": {"aaxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}, "baxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}, "bgcolor": "#E5ECF6", "caxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}}, "title": {"x": 0.05}, "xaxis": {"automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": {"standoff": 15}, "zerolinecolor": "white", "zerolinewidth": 2}, "yaxis": {"automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": {"standoff": 15}, "zerolinecolor": "white", "zerolinewidth": 2}}}, "xaxis": {"anchor": "y", "domain": [0.0, 1.0], "title": {"text": "Pr\u00e9cision"}}, "yaxis": {"anchor": "x", "domain": [0.0, 1.0], "title": {"text": "Perte"}}},
                    {"responsive": true}
                ).then(function(){

var gd = document.getElementById('d3a19371-4742-4764-85ae-aa0c67d28833'); var x = new MutationObserver(function (mutations, observer) {{ var display = window.getComputedStyle(gd).display; if (!display || display === 'none') {{ console.log([gd, 'removed!']); Plotly.purge(gd); observer.disconnect(); }} }});

// Listen for the removal of the full notebook cells var notebookContainer = gd.closest('#notebook-container'); if (notebookContainer) {{ x.observe(notebookContainer, {childList: true}); }}

// Listen for the clearing of the current output cell var outputEl = gd.closest('.output'); if (outputEl) {{ x.observe(outputEl, {childList: true}); }}

1
2
3
4
                    })
            };

        </script>