Image Classification#
We show in this tutorial how to use DRAGON for image classification task. We need to create a search space with two graphs, one treating 2D data, and a second one treating 1D data.
Loading the dataset#
[1]:
from sklearn.datasets import load_digits
import matplotlib.pyplot as plt
digits = load_digits()
_, axes = plt.subplots(nrows=1, ncols=4, figsize=(10, 3))
for ax, image, label in zip(axes, digits.images, digits.target):
ax.set_axis_off()
ax.imshow(image, cmap=plt.cm.gray_r, interpolation="nearest")
ax.set_title("Training: %i" % label)
[2]:
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
digits.images, digits.target, test_size=0.5, shuffle=False
)
print(f"X_train: {X_train.shape}, y_train: {y_train.shape}, X_val: {X_test.shape}, y_val: {y_test.shape}")
X_train: (898, 8, 8), y_train: (898,), X_val: (899, 8, 8), y_val: (899,)
Defining the Loss function#
DNN definition#
[3]:
import torch
import torch.nn as nn
import numpy as np
import os
class ClassificationDNN(nn.Module):
def __init__(self, args, input_shape) -> None:
super().__init__()
self.input_shape = input_shape
self.dag_2d = args['2D Dag']
self.dag_2d.set(self.input_shape)
flat_shape = (np.prod(self.dag_2d.output_shape),)
self.dag_1d = args['1D Dag']
self.dag_1d.set(flat_shape)
self.output = args["Out"]
self.output.set(self.dag_1d.output_shape)
def forward(self, X, **kwargs):
out_2d = self.dag_2d(X)
flat = nn.Flatten()(out_2d)
out_1d = self.dag_1d(flat)
out = self.output(out_1d)
return out
def save(self, path):
if not os.path.exists(path):
os.makedirs(path)
full_path = os.path.join(path, "best_model.pth")
torch.save(self.state_dict(), full_path)
Search Space Definition#
[4]:
from dragon.search_space.bricks_variables import mlp_var, dropout, identity_var, operations_var, mlp_const_var, conv_2d, pooling_2d, dag_var, node_var
from dragon.search_space.base_variables import ArrayVar
from dragon.search_operators.base_neighborhoods import ArrayInterval
candidate_operations_2d = operations_var("2D Candidate operations", size=10,
candidates=[mlp_var("MLP"), identity_var("Identity"), dropout('Dropout'), conv_2d('Conv 2d', max_out=8), pooling_2d("Pooling")])
dag_2d = dag_var("2D Dag", candidate_operations_2d)
candidate_operations_1d = operations_var("1D Candidate operations", size=10,
candidates=[mlp_var("MLP"), identity_var("Identity"), dropout('Dropout')])
dag_1d = dag_var("1D Dag", candidate_operations_1d)
out = node_var("Out", operation=mlp_const_var('Operation', 10), activation_function=nn.Softmax())
search_space = ArrayVar(dag_2d, dag_1d, out, label="Search Space", neighbor=ArrayInterval())
p1, p2 = search_space.random(2)
DNN Training#
[5]:
import numpy as np
from skorch import NeuralNetClassifier
from sklearn.metrics import accuracy_score
from dragon.utils.tools import set_seed
def train_and_predict(args, *kwargs, verbose=False):
set_seed(0)
labels = [e.label for e in search_space]
args = dict(zip(labels, args))
model = ClassificationDNN(args, input_shape=(8,8,1))
trainer = NeuralNetClassifier(
model,
max_epochs=10,
lr=0.01,
iterator_train__shuffle=True,
verbose=verbose,
)
trainer.fit(np.expand_dims(X_train.astype(np.float32), axis=-1), y_train.astype(np.int64))
y_pred = trainer.predict(np.expand_dims(X_test.astype(np.float32), axis=-1))
acc = accuracy_score(y_test, y_pred)
return - acc, model # We are optimizing a minimization problem
loss_1, model_1 = train_and_predict(p1,verbose=True)
loss_2, model_2 = train_and_predict(p2,verbose=True)
print("P1 ==> accuracy: ", np.round(-loss_1*100,2), "%\n")
print("P2 ==> accuracy: ", np.round(-loss_2*100,2), "%")
epoch train_loss valid_acc valid_loss dur
------- ------------ ----------- ------------ ------
1 2.3249 0.1000 2.3259 0.0336
2 2.3235 0.1000 2.3237 0.0210
3 2.3207 0.1000 2.3218 0.0212
4 2.3209 0.1000 2.3199 0.0240
5 2.3121 0.1000 2.3184 0.0223
6 2.3185 0.1000 2.3170 0.0210
7 2.3173 0.1000 2.3156 0.0211
8 2.3161 0.1000 2.3144 0.0212
9 2.3121 0.1000 2.3133 0.0211
10 2.3121 0.1000 2.3124 0.0210
epoch train_loss valid_acc valid_loss dur
------- ------------ ----------- ------------ ------
1 2.4227 0.0889 2.3395 0.0177
2 2.2958 0.1111 2.2995 0.0177
3 2.2492 0.1944 2.2670 0.0181
4 2.2149 0.2056 2.2351 0.0178
5 2.1758 0.2556 2.2047 0.0188
6 2.1443 0.3111 2.1734 0.0187
7 2.1081 0.3111 2.1463 0.0188
8 2.0804 0.3944 2.1196 0.0186
9 2.0501 0.4056 2.0949 0.0180
10 2.0219 0.4278 2.0678 0.0183
P1 ==> accuracy: 10.12 %
P2 ==> accuracy: 47.72 %
Implementing an optimization strategy#
[6]:
import time
from dragon.search_algorithm.ssea import SteadyStateEA
search_algorithm = SteadyStateEA(search_space, n_iterations=20, population_size=5, selection_size=3, evaluation=train_and_predict, save_dir="save/test_image/")
start_time = time.time()
min_loss = search_algorithm.run()
end_time = time.time() - start_time
print(f"Best score: {np.round(-min_loss*100,2)}%\nComputation time: {np.round(end_time,2)} seconds")
2024-11-27 09:25:22,127 | WARNING | Install mpi4py if you want to use the distributed version.
2024-11-27 09:25:22,133 | INFO | The whole population has been created (size = 5), 5 have been randomy initialized.
2024-11-27 09:25:23,786 | INFO | Best found! -0.8309232480533927 < inf
2024-11-27 09:25:34,229 | INFO | Best found! -0.8865406006674083 < -0.8309232480533927
2024-11-27 09:26:04,620 | INFO | All models have been at least evaluated once, t = 5 < 20.
2024-11-27 09:26:04,622 | INFO | After initialisation, it remains 15 iterations.
2024-11-27 09:26:04,689 | INFO | Evolving 2 and 0 to 6 and 7
2024-11-27 09:26:08,960 | INFO | Replacing 1 by 7, removing save/test_image//x_1.pkl
2024-11-27 09:26:12,941 | INFO | Replacing 7 by 6, removing save/test_image//x_7.pkl
2024-11-27 09:26:13,073 | INFO | Evolving 3 and 0 to 8 and 9
2024-11-27 09:26:18,886 | INFO | Replacing 2 by 9, removing save/test_image//x_2.pkl
2024-11-27 09:26:24,914 | INFO | Replacing 6 by 8, removing save/test_image//x_6.pkl
2024-11-27 09:26:24,999 | INFO | Evolving 9 and 3 to 10 and 11
2024-11-27 09:26:37,646 | INFO | Replacing 8 by 11, removing save/test_image//x_8.pkl
2024-11-27 09:26:37,653 | INFO | Best found! -0.9065628476084538 < -0.8865406006674083
2024-11-27 09:26:44,856 | INFO | Replacing 0 by 10, removing save/test_image//x_0.pkl
2024-11-27 09:26:44,955 | INFO | Evolving 9 and 11 to 12 and 13
2024-11-27 09:26:58,134 | INFO | Replacing 10 by 13, removing save/test_image//x_10.pkl
2024-11-27 09:26:58,140 | INFO | Best found! -0.917686318131257 < -0.9065628476084538
2024-11-27 09:27:02,307 | INFO | Replacing 4 by 12, removing save/test_image//x_4.pkl
2024-11-27 09:27:02,383 | INFO | Evolving 11 and 12 to 14 and 15
2024-11-27 09:27:05,641 | INFO | Replacing 9 by 15, removing save/test_image//x_9.pkl
2024-11-27 09:27:13,295 | INFO | Replacing 12 by 14, removing save/test_image//x_12.pkl
2024-11-27 09:27:13,385 | INFO | Evolving 13 and 3 to 16 and 17
2024-11-27 09:27:20,587 | INFO | Replacing 15 by 17, removing save/test_image//x_15.pkl
2024-11-27 09:27:24,845 | INFO | Replacing 3 by 16, removing save/test_image//x_3.pkl
2024-11-27 09:27:24,847 | INFO | Best found! -0.9199110122358176 < -0.917686318131257
2024-11-27 09:27:24,937 | INFO | Evolving 14 and 17 to 18 and 19
2024-11-27 09:27:31,999 | INFO | Replacing 11 by 19, removing save/test_image//x_11.pkl
2024-11-27 09:27:32,002 | INFO | Best found! -0.9232480533926585 < -0.9199110122358176
2024-11-27 09:27:38,326 | INFO | Replacing 17 by 18, removing save/test_image//x_17.pkl
2024-11-27 09:27:38,384 | INFO | Evolving 19 and 18 to 20 and 21
2024-11-27 09:27:43,798 | INFO | Replacing 14 by 21, removing save/test_image//x_14.pkl
2024-11-27 09:27:43,801 | INFO | Best found! -0.9243604004449388 < -0.9232480533926585
2024-11-27 09:27:43,809 | INFO | Search algorithm is done. Min Loss = -0.9243604004449388
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[6], line 8
6 min_loss = search_algorithm.run()
7 end_time = time.time() - start_time
----> 8 print(f"Best score: {np.round(-min_loss*100,2)}%\nComputation time: {np.round(end_time,2)} seconds")
TypeError: bad operand type for unary -: 'NoneType'
Starting with a completely random sets of DNNs, we managed in a few minutes to converge towards an accuracy higher than 92%.
[ ]:
from dragon.utils.plot_functions import load_archi
set_seed(0)
best_args =load_archi('save/test_image/best_model/x.pkl')
labels = [e.label for e in search_space]
best_args = dict(zip(labels, best_args))
model = ClassificationDNN(best_args, (8,8,1))
model.load_state_dict(torch.load('save/test_image/best_model/best_model.pth'))
model = NeuralNetClassifier(
model,
max_epochs=1,
lr=0.0001,
iterator_train__shuffle=True,
verbose=False,
)
model.fit(np.expand_dims(X_train.astype(np.float32), axis=-1), y_train.astype(np.int64))
y_pred = model.predict(np.expand_dims(X_test.astype(np.float32), axis=-1))
acc = accuracy_score(y_test, y_pred)
print("Final accuracy: ", np.round(acc*100,2), "%\n")
Final accuracy: 75.75 %
[ ]:
from sklearn import metrics
disp = metrics.ConfusionMatrixDisplay.from_predictions(y_test, y_pred)
disp.figure_.suptitle("Confusion Matrix")
plt.show()
[ ]:
import graphviz
from dragon.utils.plot_functions import draw_cell, str_operations
def draw_graph(n_dag2, m_dag2, n_dag1, m_dag1, output_file, act="Identity()", name="MNIST"):
G = graphviz.Digraph(output_file, format='pdf',
node_attr={'nodesep': '0.02', 'shape': 'box', 'rankstep': '0.02', 'fontsize': '20', "fontname": "sans-serif"})
G, g_nodes = draw_cell(G, n_dag2, m_dag2, "#ffa600", [], name_input=name,
color_input="#ef5675")
G.node("Flatten", style="rounded,filled", color="black", fillcolor="#CE1C4E", fontcolor="#ECECEC")
G.edge(g_nodes[-1], "Flatten")
G, g_nodes = draw_cell(G, n_dag1, m_dag1, "#ffa600", g_nodes, name_input=["Flatten"],
color_input="#ef5675")
G.node(','.join(["MLP", "10", act]), style="rounded,filled", color="black", fillcolor="#ef5675", fontcolor="#ECECEC")
G.edge(g_nodes[-1], ','.join(["MLP", "10", act]))
return G
m_dag2 = best_args['2D Dag'].matrix
n_dag2 = str_operations(best_args["2D Dag"].operations)
m_dag1 = best_args['1D Dag'].matrix
n_dag1 = str_operations(best_args["1D Dag"].operations)
graph = draw_graph(n_dag2, m_dag2, n_dag1, m_dag1, "save/test_image/best_archi")
print(f'Model giving a score of {np.round(acc*100,2)}%:')
graph
Model giving a score of 75.75%:
