Image Classification#
We show in this tutorial how to use DRAGON for image classification task. We need to create a search space with two graphs, one treating 2D data, and a second one treating 1D data.
Loading the dataset#
[1]:
from sklearn.datasets import load_digits
import matplotlib.pyplot as plt
digits = load_digits()
_, axes = plt.subplots(nrows=1, ncols=4, figsize=(10, 3))
for ax, image, label in zip(axes, digits.images, digits.target):
ax.set_axis_off()
ax.imshow(image, cmap=plt.cm.gray_r, interpolation="nearest")
ax.set_title("Training: %i" % label)
[2]:
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
digits.images, digits.target, test_size=0.5, shuffle=False
)
print(f"X_train: {X_train.shape}, y_train: {y_train.shape}, X_val: {X_test.shape}, y_val: {y_test.shape}")
X_train: (898, 8, 8), y_train: (898,), X_val: (899, 8, 8), y_val: (899,)
Defining the Loss function#
DNN definition#
[3]:
import torch
import torch.nn as nn
import numpy as np
import os
class ClassificationDNN(nn.Module):
def __init__(self, args, input_shape) -> None:
super().__init__()
self.input_shape = input_shape
self.dag_2d = args['2D Dag']
self.dag_2d.set(self.input_shape)
flat_shape = (np.prod(self.dag_2d.output_shape),)
self.dag_1d = args['1D Dag']
self.dag_1d.set(flat_shape)
self.output = args["Out"]
self.output.set(self.dag_1d.output_shape)
def forward(self, X, **kwargs):
out_2d = self.dag_2d(X)
flat = nn.Flatten()(out_2d)
out_1d = self.dag_1d(flat)
out = self.output(out_1d)
return out
def save(self, path):
if not os.path.exists(path):
os.makedirs(path)
full_path = os.path.join(path, "best_model.pth")
torch.save(self.state_dict(), full_path)
Search Space Definition#
[4]:
from dragon.search_space.bricks_variables import mlp_var, dropout, identity_var, operations_var, mlp_const_var, conv_2d, pooling_2d, dag_var, node_var
from dragon.search_space.zellij_variables import ArrayVar
from dragon.search_algorithm.zellij_neighborhoods import ArrayInterval
candidate_operations_2d = operations_var("2D Candidate operations", size=10,
candidates=[mlp_var("MLP"), identity_var("Identity"), dropout('Dropout'), conv_2d('Conv 2d', max_out=8), pooling_2d("Pooling")])
dag_2d = dag_var("2D Dag", candidate_operations_2d)
candidate_operations_1d = operations_var("1D Candidate operations", size=10,
candidates=[mlp_var("MLP"), identity_var("Identity"), dropout('Dropout')])
dag_1d = dag_var("1D Dag", candidate_operations_1d)
out = node_var("Out", operation=mlp_const_var('Operation', 10), activation_function=nn.Softmax())
search_space = ArrayVar(dag_2d, dag_1d, out, label="Search Space", neighbor=ArrayInterval())
p1, p2 = search_space.random(2)
DNN Training#
[5]:
import numpy as np
from skorch import NeuralNetClassifier
from sklearn.metrics import accuracy_score
from dragon.utils.tools import set_seed
def train_and_predict(args, *kwargs, verbose=False):
set_seed(0)
labels = [e.label for e in search_space]
args = dict(zip(labels, args))
model = ClassificationDNN(args, input_shape=(8,8,1))
trainer = NeuralNetClassifier(
model,
max_epochs=10,
lr=0.01,
iterator_train__shuffle=True,
verbose=verbose,
)
trainer.fit(np.expand_dims(X_train.astype(np.float32), axis=-1), y_train.astype(np.int64))
y_pred = trainer.predict(np.expand_dims(X_test.astype(np.float32), axis=-1))
acc = accuracy_score(y_test, y_pred)
return - acc, model # We are optimizing a minimization problem
loss_1, model_1 = train_and_predict(p1,verbose=True)
loss_2, model_2 = train_and_predict(p2,verbose=True)
print("P1 ==> accuracy: ", np.round(-loss_1*100,2), "%\n")
print("P2 ==> accuracy: ", np.round(-loss_2*100,2), "%")
epoch train_loss valid_acc valid_loss dur
------- ------------ ----------- ------------ ------
1 13.9172 0.1000 14.3481 1.8579
2 14.4547 0.1000 14.3481 1.2120
3 14.2327 0.1000 14.3481 1.8822
4 14.3437 0.1000 14.3481 2.0864
5 14.2993 0.1000 14.3481 2.2426
6 14.1883 0.1000 14.3481 2.3807
7 14.2549 0.1000 14.3481 2.4584
8 14.3437 0.1000 14.3481 2.1397
9 14.1217 0.1000 14.3481 2.5125
10 14.3659 0.1000 14.3481 1.8389
epoch train_loss valid_acc valid_loss dur
------- ------------ ----------- ------------ ------
1 1.8868 0.6556 1.2829 0.0937
2 0.9898 0.7556 0.9188 0.0923
3 0.6783 0.7667 0.6940 0.0932
4 0.4561 0.7833 0.6939 0.1000
5 0.3409 0.7778 0.6125 0.0944
6 0.3530 0.8167 0.4716 0.0962
7 0.2087 0.8333 0.4719 0.0969
8 0.1831 0.8833 0.3127 0.0945
9 0.1612 0.8833 0.3414 0.1132
10 0.1426 0.8833 0.3131 0.1101
P1 ==> accuracy: 10.12 %
P2 ==> accuracy: 91.88 %
Implementing an optimization strategy#
[6]:
import time
from dragon.search_algorithm.ssea import SteadyStateEA
search_algorithm = SteadyStateEA(search_space, n_iterations=20, population_size=5, selection_size=3, evaluation=train_and_predict, save_dir="save/test_image/")
start_time = time.time()
min_loss = search_algorithm.run()
end_time = time.time() - start_time
print(f"Best score: {np.round(-min_loss*100,2)}%\nComputation time: {np.round(end_time,2)} seconds")
2024-09-30 08:39:29,676 | WARNING | Install mpi4py if you want to use the distributed version.
2024-09-30 08:39:29,682 | INFO | The whole population has been created (size = 5), 5 have been randomy initialized.
2024-09-30 08:39:29,682 | INFO | We start by evaluating the whole population (size=5)
2024-09-30 08:39:41,290 | INFO | Best found ! -0.10122358175750834 < inf
2024-09-30 08:39:58,064 | INFO | Best found ! -0.8665183537263627 < -0.10122358175750834
2024-09-30 08:40:15,949 | INFO | All models have been at least evaluated once, t = 5 < 20.
2024-09-30 08:40:15,974 | INFO | Evolving 3 and 0 to 6 and 7
2024-09-30 08:40:19,888 | INFO | Replacing 1 by 5
2024-09-30 08:40:31,751 | INFO | Evolving 4 and 2 to 8 and 9
2024-09-30 08:40:48,030 | INFO | Replacing 0 by 7
2024-09-30 08:40:59,333 | INFO | Replacing 4 by 8
2024-09-30 08:40:59,349 | INFO | Best found ! -0.8943270300333704 < -0.8665183537263627
2024-09-30 08:40:59,412 | INFO | Evolving 8 and 2 to 10 and 11
2024-09-30 08:41:09,931 | INFO | Replacing 7 by 9
2024-09-30 08:41:09,946 | INFO | Best found ! -0.899888765294772 < -0.8943270300333704
2024-09-30 08:41:20,802 | INFO | Replacing 3 by 10
2024-09-30 08:41:20,849 | INFO | Evolving 9 and 10 to 12 and 13
2024-09-30 08:41:31,183 | INFO | Replacing 5 by 11
2024-09-30 08:41:31,190 | INFO | Best found ! -0.9043381535038932 < -0.899888765294772
2024-09-30 08:41:41,613 | INFO | Replacing 2 by 12
2024-09-30 08:41:41,662 | INFO | Evolving 12 and 11 to 14 and 15
2024-09-30 08:41:52,058 | INFO | Replacing 8 by 13
2024-09-30 08:42:02,788 | INFO | Replacing 10 by 14
2024-09-30 08:42:02,800 | INFO | Best found ! -0.9121245828698554 < -0.9043381535038932
2024-09-30 08:42:02,860 | INFO | Evolving 14 and 13 to 16 and 17
2024-09-30 08:42:13,682 | INFO | Replacing 9 by 15
2024-09-30 08:42:24,225 | INFO | Replacing 12 by 16
2024-09-30 08:42:24,306 | INFO | Evolving 15 and 14 to 18 and 19
2024-09-30 08:43:11,096 | INFO | Replacing 11 by 18
2024-09-30 08:43:11,165 | INFO | Evolving 16 and 15 to 20 and 21
2024-09-30 08:43:32,883 | INFO | Replacing 13 by 20
2024-09-30 08:43:32,893 | INFO | Best found ! -0.9132369299221357 < -0.9121245828698554
Best score: 91.32%
Computation time: 243.24 seconds
Starting with a completely random sets of DNNs, we managed in a few minutes to converge towards an accuracy higher than 92%.
[ ]:
from dragon.utils.plot_functions import load_archi
set_seed(0)
best_args =load_archi('save/test_image/best_model/x.pkl')
labels = [e.label for e in search_space]
best_args = dict(zip(labels, best_args))
model = ClassificationDNN(best_args, (8,8,1))
model.load_state_dict(torch.load('save/test_image/best_model/best_model.pth'))
model = NeuralNetClassifier(
model,
max_epochs=1,
lr=0.0001,
iterator_train__shuffle=True,
verbose=False,
)
model.fit(np.expand_dims(X_train.astype(np.float32), axis=-1), y_train.astype(np.int64))
y_pred = model.predict(np.expand_dims(X_test.astype(np.float32), axis=-1))
acc = accuracy_score(y_test, y_pred)
print("Final accuracy: ", np.round(acc*100,2), "%\n")
Final accuracy: 75.75 %
[ ]:
from sklearn import metrics
disp = metrics.ConfusionMatrixDisplay.from_predictions(y_test, y_pred)
disp.figure_.suptitle("Confusion Matrix")
plt.show()
[ ]:
import graphviz
from dragon.utils.plot_functions import draw_cell, str_operations
def draw_graph(n_dag2, m_dag2, n_dag1, m_dag1, output_file, act="Identity()", name="MNIST"):
G = graphviz.Digraph(output_file, format='pdf',
node_attr={'nodesep': '0.02', 'shape': 'box', 'rankstep': '0.02', 'fontsize': '20', "fontname": "sans-serif"})
G, g_nodes = draw_cell(G, n_dag2, m_dag2, "#ffa600", [], name_input=name,
color_input="#ef5675")
G.node("Flatten", style="rounded,filled", color="black", fillcolor="#CE1C4E", fontcolor="#ECECEC")
G.edge(g_nodes[-1], "Flatten")
G, g_nodes = draw_cell(G, n_dag1, m_dag1, "#ffa600", g_nodes, name_input=["Flatten"],
color_input="#ef5675")
G.node(','.join(["MLP", "10", act]), style="rounded,filled", color="black", fillcolor="#ef5675", fontcolor="#ECECEC")
G.edge(g_nodes[-1], ','.join(["MLP", "10", act]))
return G
m_dag2 = best_args['2D Dag'].matrix
n_dag2 = str_operations(best_args["2D Dag"].operations)
m_dag1 = best_args['1D Dag'].matrix
n_dag1 = str_operations(best_args["1D Dag"].operations)
graph = draw_graph(n_dag2, m_dag2, n_dag1, m_dag1, "save/test_image/best_archi")
print(f'Model giving a score of {np.round(acc*100,2)}%:')
graph
Model giving a score of 75.75%:
