DAG neighborhoods implementation

DAG neighborhoods implementation#

class HpInterval(neighborhood=None, variable=None)[source]

Bases: VarNeighborhood

Addon, used to determine the neighbor of an HpVar. Mutate the operation if it is not a constant and the hyperparameters.

Parameters:

variable (HpVar, default=None) – Targeted Variable

Examples

>>> from dragon.search_space.bricks import MLP
>>> from dragon.search_space.base_variables import Constant, IntVar
>>> from dragon.search_space.dag_variables import HpVar
>>> from dragon.search_operators.base_neighborhoods import ConstantInterval, IntInterval
>>> from dragon.search_algorithmdag_neighborhoods import HpInterval
>>> mlp = Constant("MLP operation", MLP, neighbor=ConstantInterval())
>>> hp = {"out_channels": IntVar("out_channels", 1, 10, neighbor=IntInterval(2))}
>>> mlp_var = HpVar("MLP var", mlp, hyperparameters=hp, neighbor=HpInterval())
>>> print(mlp_var)
HpVar(MLP var,
>>> test_mlp = mlp_var.random()
>>> print(test_mlp)
[<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 2}]
>>> mlp_var.neighbor(test_mlp[0], test_mlp[1])
[<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 1}]
property neighborhood
property target
class CatHpInterval(neighborhood=None, variable=None)[source]

Bases: VarNeighborhood

Addon, used to determine the neighbor of a CatVar of candidates operations. Given a probability neighborhood, draw a neighbor of the current operation, or draw a complete new operation

Parameters:

  • variable (CatVar, default=None) – Targeted Variable

  • neighborhood (float < 1, default=0.9) – Probability of drawing a neighbor instead of changing the whole operation.

Examples

>>> from dragon.search_space.bricks import MLP, LayerNorm1d, BatchNorm1d
>>> from dragon.search_space.base_variables import Constant, IntVar, CatVar
>>> from dragon.search_space.dag_variables import HpVar
>>> from dragon.search_operators.base_neighborhoods import ConstantInterval, IntInterval, CatInterval
>>> from dragon.search_algorithmdag_neighborhoods import HpInterval, CatHpInterval
>>> mlp = Constant("MLP operation", MLP, neighbor=ConstantInterval())
>>> hp = {"out_channels": IntVar("out_channels", 1, 10, neighbor=IntInterval(2))}
>>> mlp_var = HpVar("MLP var", mlp, hyperparameters=hp, neighbor=HpInterval())
>>> norm = CatVar("1d norm layers", features=[LayerNorm1d, BatchNorm1d], neighbor=CatInterval())
>>> norm_var = HpVar("Norm var", norm, hyperparameters={}, neighbor=HpInterval())
>>> candidates=CatVar("Candidates", features=[mlp_var, norm_var],neighbor=CatHpInterval(neighborhood=0.4))
>>> print(candidates)
CatVar(Candidates, [HpVar(MLP var, , HpVar(Norm var, ])
>>> test_candidates = candidates.random()
>>> print(test_candidates)
[<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 2}]
>>> candidates.neighbor(test_candidates[0], test_candidates[1], size=10)
[[<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 2}], [<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 1}], [<class 'dragon.search_space.bricks.normalization.BatchNorm1d'>, {}], [<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 1}], [<class 'dragon.search_space.bricks.normalization.LayerNorm1d'>, {}], [<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 3}], [<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 1}], [<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 2}], [<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 4}], [<class 'dragon.search_space.bricks.basics.MLP'>, {'out_channels': 3}]]
property neighborhood
property target
class NodeInterval(neighborhood=None, variable=None)[source]

Bases: VarNeighborhood

Addon, used to determine the neighbor of a Node. Change the combiner and/or the operation and/or the hyperparameters and/or the activation function.

Parameters:

variable (CatVar, default=None) – Targeted Variable

Examples

>>> from dragon.search_space.dag_variables import NodeVariable, HpVar
>>> from dragon.search_space.bricks import MLP
>>> from dragon.search_space.base_variables import Constant, IntVar, CatVar
>>> from dragon.search_space.bricks_variables import activation_var
>>> from dragon.search_operators.base_neighborhoods import ConstantInterval, IntInterval, CatInterval
>>> from dragon.search_algorithmdag_neighborhoods import NodeInterval, HpInterval
>>> combiner = CatVar("Combiner", features = ['add', 'mul'], neighbor=CatInterval())
>>> operation = HpVar("Operation", Constant("MLP operation", MLP, neighbor=ConstantInterval()),
...                   hyperparameters={"out_channels": IntVar("out_channels", 1, 10, neighbor=IntInterval(1))}, neighbor=HpInterval())
>>> node = NodeVariable(label="Node variable",
...                     combiner=combiner,
...                     operation=operation,
...                     activation_function=activation_var("Activation"), neighbor=NodeInterval())
>>> print(node)
Combiner: CatVar(Combiner, ['add', 'mul']) - Operation: HpVar(Operation,  - Act. Function: CatVar(Activation, [ReLU(), LeakyReLU(negative_slope=0.01), Identity(), Sigmoid(), Tanh(), ELU(alpha=1.0), GELU(approximate='none'), SiLU()])
>>> test_node = node.random()
>>> print(test_node)

(combiner) mul – (name) <class ‘dragon.search_space.bricks.basics.MLP’> – (hp) {‘out_channels’: 6} – (activation) LeakyReLU(negative_slope=0.01) – >>> neighbor = node.neighbor(test_node) >>> print(‘Neighbor: ‘, neighbor) Neighbor: (combiner) add – (name) <class ‘dragon.search_space.bricks.basics.MLP’> – (hp) {‘out_channels’: 6} – (activation) LeakyReLU(negative_slope=0.01) – >>> neighbor.set((3,)) >>> print(‘Neighbor after setting: ‘, neighbor) Neighbor after setting: (input shape) (3,) – (combiner) add – (op) MLP( (linear): Linear(in_features=3, out_features=6, bias=True) ) – (activation) LeakyReLU(negative_slope=0.01) – (output shape) (6,) >>> node.neighbor(neighbor)

(input shape) (3,) – (combiner) mul – (op) MLP( (linear): Linear(in_features=3, out_features=5, bias=True) ) – (activation) LeakyReLU(negative_slope=0.01) – (output shape) (5,)

property neighborhood
property target
class EvoDagInterval(neighborhood=None, variable=None, nb_mutations=None)[source]

Bases: VarNeighborhood

NodeInterval

Addon, used to determine the neighbor of an EvoDagVariable. May perform several modifications such as adding / deleting nodes, changing the nodes content, adding/removing connections.

Parameters:

variable (EvoDagVariable, default=None) – Targeted Variable.

Examples

>>> from dragon.search_space.dag_variables import HpVar, NodeVariable, EvoDagVariable
>>> from dragon.search_space.bricks import MLP, MaxPooling1D, AVGPooling1D
>>> from dragon.search_space.base_variables import Constant, IntVar, CatVar, DynamicBlock
>>> from dragon.search_space.bricks_variables import activation_var
>>> from dragon.search_algorithmdag_neighborhoods import CatHpInterval, EvoDagInterval, NodeInterval, HpInterval
>>> from dragon.search_operators.base_neighborhoods import ConstantInterval, IntInterval, CatInterval, DynamicBlockInterval
>>> mlp = HpVar("Operation", Constant("MLP operation", MLP, neighbor=ConstantInterval()), hyperparameters={"out_channels": IntVar("out_channels", 1, 10, neighbor=IntInterval(5))}, neighbor=HpInterval())
>>> pooling = HpVar("Operation", CatVar("Pooling operation", [MaxPooling1D, AVGPooling1D], neighbor=CatInterval()), hyperparameters={"pool_size": IntVar("pool_size", 1, 5, neighbor=IntInterval(2))}, neighbor=HpInterval())
>>> candidates = NodeVariable(label = "Candidates",
...                           combiner=CatVar("Combiner", features=['add', 'concat'], neighbor=CatInterval()),
...                           operation=CatVar("Candidates", [mlp, pooling], neighbor=CatHpInterval(0.4)),
...                           activation_function=activation_var("Activation"), neighbor=NodeInterval())
>>> operations = DynamicBlock("Operations", candidates, repeat=5, neighbor=DynamicBlockInterval(2))
>>> dag = EvoDagVariable(label="DAG", operations=operations, neighbor=EvoDagInterval())
>>> print(dag)
EvoDagVariable(DAG,             - Operations:
DynamicBlock(Operations, Combiner: CatVar(Combiner, ['add', 'concat']) - Operation: CatVar(Candidates, [HpVar(Operation, , HpVar(Operation, ]) - Act. Function: CatVar(Activation, [ReLU(), LeakyReLU(negative_slope=0.01), Identity(), Sigmoid(), Tanh(), ELU(alpha=1.0), GELU(approximate='none'), SiLU()])
>>> test_dag = dag.random()
>>> print(test_dag)
NODES: [
(combiner) add -- (name) <class 'dragon.search_space.bricks.basics.Identity'> -- (hp) {} -- (activation) Identity() -- ,
(combiner) add -- (name) <class 'dragon.search_space.bricks.pooling.MaxPooling1D'> -- (hp) {'pool_size': 3} -- (activation) Sigmoid() -- ,
(combiner) concat -- (name) <class 'dragon.search_space.bricks.pooling.MaxPooling1D'> -- (hp) {'pool_size': 4} -- (activation) Identity() -- ] | MATRIX:[[0, 1, 1], [0, 0, 1], [0, 0, 0]]
>>> neighbor = dag.neighbor(test_dag, 3)
>>> print(neighbor)
[NODES: [
(combiner) add -- (name) <class 'dragon.search_space.bricks.basics.Identity'> -- (hp) {} -- (activation) Identity() -- ,
(combiner) add -- (name) <class 'dragon.search_space.bricks.basics.MLP'> -- (hp) {'out_channels': 6} -- (activation) ReLU() -- ,
(combiner) concat -- (name) <class 'dragon.search_space.bricks.pooling.AVGPooling1D'> -- (hp) {'pool_size': 3} -- (activation) Sigmoid() -- ] | MATRIX:[[0, 1, 1], [0, 0, 1], [0, 0, 0]], NODES: [
(combiner) add -- (name) <class 'dragon.search_space.bricks.basics.Identity'> -- (hp) {} -- (activation) Identity() -- ,
(combiner) concat -- (name) <class 'dragon.search_space.bricks.pooling.AVGPooling1D'> -- (hp) {'pool_size': 3} -- (activation) Sigmoid() -- ,
(combiner) concat -- (name) <class 'dragon.search_space.bricks.pooling.MaxPooling1D'> -- (hp) {'pool_size': 4} -- (activation) Identity() -- ] | MATRIX:[[0, 1, 1], [0, 0, 1], [0, 0, 0]], NODES: [
(combiner) add -- (name) <class 'dragon.search_space.bricks.basics.Identity'> -- (hp) {} -- (activation) Identity() -- ,
(combiner) concat -- (name) <class 'dragon.search_space.bricks.pooling.AVGPooling1D'> -- (hp) {'pool_size': 3} -- (activation) Sigmoid() -- ,
(combiner) concat -- (name) <class 'dragon.search_space.bricks.pooling.MaxPooling1D'> -- (hp) {'pool_size': 4} -- (activation) Identity() -- ] | MATRIX:[[0, 1, 0], [0, 0, 1], [0, 0, 0]]]
>>> neighbor[0].set((3,))
>>> print('First neighbor after setting: ', neighbor)
First neighbor after setting:  [ModuleList(
(0):
(input shape) (3,) -- (combiner) add -- (op) Identity() -- (activation) Identity() -- (output shape) (3,)
(1):
(input shape) (3,) -- (combiner) add -- (op) MLP(
    (linear): Linear(in_features=3, out_features=6, bias=True)
) -- (activation) ReLU() -- (output shape) (6,)
(2):
(input shape) (9,) -- (combiner) concat -- (op) AVGPooling1D(
    (pooling): AvgPool1d(kernel_size=(3,), stride=(3,), padding=(0,))
) -- (activation) Sigmoid() -- (output shape) (3,)
), NODES: [
(input shape) (3,) -- (combiner) add -- (op) Identity() -- (activation) Identity() -- (output shape) (3,),
(input shape) (9,) -- (combiner) concat -- (op) AVGPooling1D(
(pooling): AvgPool1d(kernel_size=(3,), stride=(3,), padding=(0,))
) -- (activation) Sigmoid() -- (output shape) (3,),
(combiner) concat -- (name) <class 'dragon.search_space.bricks.pooling.MaxPooling1D'> -- (hp) {'pool_size': 4} -- (activation) Identity() -- ] | MATRIX:[[0, 1, 1], [0, 0, 1], [0, 0, 0]], NODES: [
(input shape) (3,) -- (combiner) add -- (op) Identity() -- (activation) Identity() -- (output shape) (3,),
(input shape) (9,) -- (combiner) concat -- (op) AVGPooling1D(
(pooling): AvgPool1d(kernel_size=(3,), stride=(3,), padding=(0,))
) -- (activation) Sigmoid() -- (output shape) (3,),
(combiner) concat -- (name) <class 'dragon.search_space.bricks.pooling.MaxPooling1D'> -- (hp) {'pool_size': 4} -- (activation) Identity() -- ] | MATRIX:[[0, 1, 0], [0, 0, 1], [0, 0, 0]]]
>>> dag.neighbor(neighbor[0])
NODES: [
(input shape) (3,) -- (combiner) add -- (op) Identity() -- (activation) Identity() -- (output shape) (3,),
(input shape) (3,) -- (combiner) concat -- (op) MLP(
(linear): Linear(in_features=3, out_features=9, bias=True)
) -- (activation) ReLU() -- (output shape) (9,),
(input shape) (12,) -- (combiner) concat -- (op) AVGPooling1D(
(pooling): AvgPool1d(kernel_size=(3,), stride=(3,), padding=(0,))
) -- (activation) Sigmoid() -- (output shape) (4,)] | MATRIX:[[0, 1, 1], [0, 0, 1], [0, 0, 0]]
property neighborhood
property target
modification(modif, idx, inter)[source]