mcode-neural 0.5.0__py3-none-any.whl

mcode_neural/network.py

@@ -0,0 +1,447 @@
+# region  F I L E
+# <copyright file="mcode-neural/src/mcode_neural/network.py" company="MicroCODE Incorporated">Copyright © 2026 MicroCODE, Inc. Troy, MI</copyright><author>Timothy J. McGuire</author>
+# region  M O D U L E
+# region  D O C U M E N T A T I O N
+"""
+Project:  MicroCODE Python Packages
+Customer: Internal + MIT xPRO Course
+@module   'mcode_neural.network'
+@memberof mcode_neural
+@created  November 2026
+@author   Timothy McGuire, MicroCODE, Inc.
+@description >
+MicroCODE Shared Network Library
+
+LICENSE:
+--------
+MIT License: MicroCODE.mcode-neural
+
+Copyright (c) 2026 Timothy McGuire, MicroCODE, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+DESCRIPTION:
+------------
+This module implements what every MicroCODE network is: 'NETWORK_KINDS', how
+each kind of network's layers are found and read, a PyTorch module's or our
+NumPy single neuron's; and 'Network', one network of a Model, a view of its
+module holding its 'Layer's. A Model holds one or more Networks, each named
+for the Model attribute that holds it, e.g. an autoencoder's "encoder" and
+"decoder", and each declaring its 'source', what it reads: the data, or an
+earlier network. The Model's own 'forward()' is the one record of how its
+networks combine; building a Model runs it once, with hooks, to find each
+network's layers and shapes, and fails naming the network when its declared
+source does not fit.
+
+
+REFERENCES:
+-----------
+1. MIT xPRO Course: AI and Deep Learning: Neural Network Applications
+
+2. PyTorch -- torch.nn.Module.register_forward_hook
+   https://pytorch.org/docs/stable/generated/torch.nn.Module.html
+"""
+
+# endregion
+
+# region  I M P O R T S
+
+import numpy as np
+import torch
+
+from .layer import (
+    LAYER_KINDS,
+    LAYER_MODULES,
+    LAYER_PASS_MODULES,
+    NEURON_MODULES,
+    RESHAPE_MODULES,
+    Layer,
+    _scored,
+)
+
+# endregion
+
+# region  C O N S T A N T S
+
+# how every network of one kind is found and read, keyed by the 'network_kind' its Model declares:
+#   networks(model): the Model's Networks, in the order its 'sources' lists them
+#   parameters(layer): one of its Layers' (weight, bias), NumPy arrays read live
+#   z(layer, X): that Layer's z for every sample of X, float64, (N, neurons, ...)
+#   inputs(layer, X): what that Layer's weight module reads for every sample of X, float64, (N, *its in_shape)
+#   outputs(network, X, name): what the network's module computes for X, float64, (N, *out_shape);
+#                              'name' is the caller's name for X, for the messages
+#   count(network): its count of learned values
+#   device(model): the device the Model trains on, whose memory a run records; None for NumPy
+NETWORK_KINDS = {
+    # any PyTorch module, its layers found by a hooked pass of the Model's 'forward()'
+    'torch': {
+        'networks': lambda model: _probed_networks(model),
+        'parameters': lambda layer: _module_parameters(layer),
+        'z': lambda layer, X: _hooked(layer, X, lambda inputs, output: output),
+        'inputs': lambda layer, X: _hooked(layer, X, lambda inputs, output: inputs[0]),
+        'outputs': lambda network, X, name: _module_outputs(network, X, name),
+        'count': lambda network: sum(p.numel() for p in network.module.parameters()),
+        'device': lambda model: next(model.parameters()).device,
+    },
+    # our single NumPy neuron: one network of one Layer, 'w' as its one row of weights
+    'numpy': {
+        'networks': lambda model: _neuron_networks(model),
+        'parameters': lambda layer: (
+            layer.model.w.reshape(1, -1),
+            np.array([layer.model.w_0]),
+        ),
+        'z': lambda layer, X: (X @ layer.model.w + layer.model.w_0).reshape(-1, 1),
+        'inputs': lambda layer, X: np.asarray(X, dtype=np.float64),
+        'outputs': lambda network, X, name: network.model.outputs(X).reshape(-1, 1),
+        'count': lambda network: int(network.model.w.size + 1),
+        'device': lambda model: None,
+    },
+}
+
+# endregion
+
+# region  C L A S S E S
+
+
+class Network:
+    """
+    One network of a Model: a view of the module a Model attribute holds, or a single neuron's
+    'w', with its Layers in the order they run. It holds no weights of its own, so it always
+    reads the trained values.
+
+    Args:
+        model:      the Model it belongs to.
+        index:      its position in 'model.network'.
+        name:       the Model attribute that holds it, e.g. "net", "encoder", or "decoder"; "neuron"
+                    for a single neuron, which has no module.
+        kind:       how its layers are found and read, a key of 'NETWORK_KINDS': "torch" or "numpy".
+        source:     what it reads, a tuple of "data" or an earlier network's name, e.g. ("encoder",).
+        module:     the nn.Module it views; None for a single neuron.
+        in_shape:   one sample's shape entering it, e.g. (784,) or (1, 28, 28).
+        out_shape:  one sample's shape leaving it, e.g. (32,).
+        layer_rows: each of its layers' 'kind', 'activation', 'module', 'in_shape', and
+                    'out_shape', in the order they run, as 'Layer' takes them.
+
+    Note:
+        Models build their own networks; read them from 'model.network'. The Model's
+        'forward()' is the one record of how its networks combine; 'source' describes it, and
+        building the Model checks each network's 'in_shape' against its source's shape.
+
+    Example:
+        >>> decoder = model.network[1]
+        >>> decoder.name, decoder.source
+        ('decoder', ('encoder',))
+        >>> decoder.layer[0].neuron[3].bias
+    """
+
+    def __init__(
+        self,
+        model,
+        index,
+        name,
+        kind,
+        source,
+        module,
+        in_shape,
+        out_shape,
+        layer_rows,
+    ):
+        self.model = model
+        self.index = index
+        self.name = name
+        self.kind = kind
+        self.source = source
+        self.module = module
+        self.in_shape = in_shape
+        self.out_shape = out_shape
+        self.layer = tuple(Layer(self, i, **row) for i, row in enumerate(layer_rows))
+
+    @property
+    def parameters(self):
+        """Its count of learned values, every value of its module's parameters."""
+        return NETWORK_KINDS[self.kind]['count'](self)
+
+    @property
+    def label(self):
+        """Its position, as the code reaches it, e.g. "network[1]"."""
+        return f'network[{self.index}]'
+
+    def outputs(self, X):
+        """
+        Returns what it computes for every sample, scored in eval() without gradients, its Model's mode left as it was.
+
+        Args:
+            X:  its input samples, one per row: a tensor or NumPy array of shape (N, ...), each
+                shaped to 'in_shape', e.g. the data for an encoder, latent codes for a decoder.
+
+        Returns:
+            a float64 NumPy array of shape (N, *out_shape): a network's logits, an encoder's
+            latent codes, or a single neuron's a.
+        """
+        return self._outputs(X, 'X')
+
+    def _outputs(self, X, name):
+        """Returns 'outputs(X)', its messages naming X by the caller's 'name', e.g. "Z"."""
+        return NETWORK_KINDS[self.kind]['outputs'](self, X, name)
+
+    def _layer_parameters(self, layer):
+        """Returns one of its Layers' (weight, bias), as NumPy arrays."""
+        return NETWORK_KINDS[self.kind]['parameters'](layer)
+
+    def _layer_z(self, layer, X):
+        """Returns one of its Layers' z for every sample of X, as a NumPy array."""
+        return NETWORK_KINDS[self.kind]['z'](layer, X)
+
+    def _layer_inputs(self, layer, X):
+        """Returns what one of its Layers' weight module reads for every sample of X, as a NumPy array."""
+        return NETWORK_KINDS[self.kind]['inputs'](layer, X)
+
+
+# endregion
+
+# region  P R I V A T E   F U N C T I O N S
+
+
+def _recorder(log):
+    """Returns a forward hook that appends (module, shape in, shape out) of one sample to 'log'."""
+
+    def record(module, inputs, output):
+        log.append((module, tuple(inputs[0].shape[1:]), tuple(output.shape[1:])))
+
+    return record
+
+
+def _probed_networks(model):
+    """
+    Returns a PyTorch Model's Networks, found by one zero sample run through its 'forward()'.
+
+    Args:
+        model:  a PyTorch Model, its networks built and named in 'model.sources'. Hooks on each
+                network and each innermost module record what runs, in order; the sample runs in
+                eval() without gradients, its mode restored, so no Dropout draws a random number,
+                and every hook is removed after.
+    """
+    parameter = next(model.parameters())
+    x = torch.zeros(1, *model.in_shape, dtype=parameter.dtype, device=parameter.device)
+    runs = {name: [] for name in model.sources}
+    records = {name: [] for name in model.sources}
+    handles = []
+    for name in model.sources:
+        network = getattr(model, name)
+        handles.append(network.register_forward_hook(_recorder(runs[name])))
+        for module in network.modules():
+            if next(module.children(), None) is None:
+                handles.append(module.register_forward_hook(_recorder(records[name])))
+    is_training = model.training
+    model.eval()
+    try:
+        with torch.no_grad():
+            model(x)
+    finally:
+        model.train(is_training)
+        for handle in handles:
+            handle.remove()
+    for name in model.sources:
+        _check_runs(model, name, runs[name], records[name])
+    shapes = {name: runs[name][0][1:] for name in model.sources}
+    _check_sources(model, shapes)
+    last = list(model.sources)[-1]
+    return tuple(
+        Network(
+            model,
+            index,
+            name,
+            'torch',
+            source,
+            getattr(model, name),
+            *shapes[name],
+            _layer_rows(model, name, records[name], is_last=name == last),
+        )
+        for index, (name, source) in enumerate(model.sources.items())
+    )
+
+
+def _check_runs(model, name, runs, records):
+    """
+    Raises a ValueError naming the network unless it ran exactly once, each of its modules once.
+
+    Args:
+        model:      the Model probed.
+        name:       the network's name, a key of 'model.sources'.
+        runs:       the network's own records, one per time it ran.
+        records:    its innermost modules' records, in the order they ran.
+    """
+    if not runs:
+        raise ValueError(
+            f"'model.{name}' is in '{type(model).__name__}.sources' but never runs in 'forward()'"
+        )
+    modules = [module for module, _, _ in records]
+    if len(runs) > 1 or len(set(map(id, modules))) < len(modules):
+        raise ValueError(
+            f"'model.{name}' runs more than once in one pass; a network or module run twice is not supported yet"
+        )
+
+
+def _check_sources(model, shapes):
+    """
+    Raises a TypeError or ValueError naming the network unless each declared source fits it.
+
+    Args:
+        model:  the Model probed; 'model.sources' maps each network to a tuple of one source,
+                "data" or an earlier network's name.
+        shapes: each network's (shape in, shape out) of one sample, by name.
+    """
+    owner = f"'{type(model).__name__}.sources'"
+    names = list(model.sources)
+    for i, (name, source) in enumerate(model.sources.items()):
+        if not isinstance(source, tuple):
+            raise TypeError(
+                f"{owner}['{name}'] must be a tuple of sources, e.g. ('data',), got {source!r}"
+            )
+        if len(source) != 1:
+            raise ValueError(
+                f"{owner} says '{name}' reads {source}; a network of several sources is not supported yet"
+            )
+        (read,) = source
+        if read == 'data':
+            expected = tuple(model.in_shape)
+        elif read in names[:i]:
+            expected = shapes[read][1]
+        else:
+            raise ValueError(
+                f"{owner} says '{name}' reads '{read}', neither 'data' nor a network before it, {names[:i]}"
+            )
+        if shapes[name][0] != expected:
+            raise ValueError(
+                f"{owner} says '{name}' reads '{read}', one sample {expected}, but it reads {shapes[name][0]}"
+            )
+
+
+def _layer_rows(model, name, records, is_last):
+    """
+    Returns one network's layers, as 'Layer' takes them: each weight module with the modules after it.
+
+    Args:
+        model:      the Model probed.
+        name:       the network's name, for the messages.
+        records:    its innermost modules' (module, shape in, shape out), in the order they ran.
+        is_last:    True for the Model's last network, whose last layer takes the Model's output
+                    activation when no module applies it, as the loss does.
+    """
+    first = type(records[0][0])
+    if first not in LAYER_MODULES:
+        raise ValueError(
+            f"'model.{name}' must start with a weight module, one of"
+            f' {[module.__name__ for module in LAYER_MODULES]}, got {first.__name__}'
+        )
+    rows = []
+    for module, in_shape, out_shape in records:
+        kind = type(module)
+        # a weight module starts a layer; every other module belongs to the layer before it
+        if kind in LAYER_MODULES:
+            rows.append(
+                {
+                    'kind': LAYER_MODULES[kind],
+                    'activation': None,
+                    'module': [],
+                    'in_shape': in_shape,
+                }
+            )
+        elif kind in NEURON_MODULES:
+            if rows[-1]['activation'] is None:
+                rows[-1]['activation'] = NEURON_MODULES[kind]
+        elif not isinstance(module, LAYER_PASS_MODULES + RESHAPE_MODULES):
+            raise ValueError(
+                f"'model.{name}' holds a {kind.__name__}: not a weight module, a 'NEURON_KINDS' activation,"
+                ' a pool, a dropout, or a reshape'
+            )
+        rows[-1]['module'].append(module)
+        if not isinstance(module, RESHAPE_MODULES):
+            rows[-1]['out_shape'] = out_shape
+    # the loss applies a network's output activation, so its last layer has no module for it
+    if is_last and rows[-1]['activation'] is None:
+        rows[-1]['activation'] = model.output_activation_name
+    for row in rows:
+        row['module'] = tuple(row['module'])
+        if row['activation'] is None:
+            row['activation'] = 'linear'
+    return rows
+
+
+def _module_parameters(layer):
+    """Returns a PyTorch layer's (weight, bias), its weight module's, as NumPy arrays on the CPU."""
+    weight, bias = LAYER_KINDS[layer.kind]['parameters'](layer.module[0])
+    return weight.detach().cpu().numpy(), bias.detach().cpu().numpy()
+
+
+def _hooked(layer, X, pick):
+    """
+    Returns what a PyTorch layer's weight module reads or computes for every sample, as the
+    Model's 'forward()' runs it, scored by '_scored()', as a float64 NumPy array.
+
+    Args:
+        layer:  a Layer of a PyTorch Model; the build proved its weight module runs once per pass.
+        X:      the input samples, as the Model's 'outputs()' takes them.
+        pick:   'pick(inputs, output)' returns the tensor to keep from one hooked call, e.g. its
+                output, the layer's z, or 'inputs[0]', what it reads.
+    """
+    model = layer.model
+    captured = []
+    handle = layer.module[0].register_forward_hook(
+        lambda module, inputs, output: captured.append(pick(inputs, output).cpu())
+    )
+    try:
+        _scored(model, model, X, 'X', model.in_shape)
+    finally:
+        handle.remove()
+    return torch.cat(captured).double().numpy()
+
+
+def _module_outputs(network, X, name):
+    """Returns a PyTorch network's module run on X, scored by '_scored()', as a float64 NumPy array."""
+    return (
+        _scored(network.model, network.module, X, name, network.in_shape)
+        .cpu()
+        .double()
+        .numpy()
+    )
+
+
+def _neuron_networks(model):
+    """Returns a single neuron Model's one Network, of one Linear Layer of one Neuron."""
+    ((name, source),) = model.sources.items()
+    shape = (len(model.w),)
+    row = {
+        'kind': 'linear',
+        'activation': model.activation_name,
+        'module': (),
+        'in_shape': shape,
+        'out_shape': (1,),
+    }
+    return (Network(model, 0, name, 'numpy', source, None, shape, (1,), (row,)),)
+
+
+# endregion
+
+# region  P U B L I C   F U N C T I O N S
+
+# endregion
+# endregion
+# endregion

mcode_neural/neuron.py

@@ -0,0 +1,233 @@
+# region  F I L E
+# <copyright file="mcode-neural/src/mcode_neural/neuron.py" company="MicroCODE Incorporated">Copyright © 2026 MicroCODE, Inc. Troy, MI</copyright><author>Timothy J. McGuire</author>
+# region  M O D U L E
+# region  D O C U M E N T A T I O N
+"""
+Project:  MicroCODE Python Packages
+Customer: Internal + MIT xPRO Course
+@module   'mcode_neural.neuron'
+@memberof mcode_neural
+@created  October 2026
+@author   Timothy McGuire, MicroCODE, Inc.
+@description >
+MicroCODE Shared Neuron Library
+
+LICENSE:
+--------
+MIT License: MicroCODE.mcode-neural
+
+Copyright (c) 2026 Timothy McGuire, MicroCODE, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+DESCRIPTION:
+------------
+This module implements what every MicroCODE neuron is: 'NEURON_KINDS', each
+activation's label, PyTorch module, range, NumPy activation, and slope; and
+'Neuron', one output of a model's 'Layer', a view of its trained weights;
+and 'NeuronOptimizer', the gradient descent a single neuron trains with.
+Our single neuron models, 'NeuronModel' and its subclasses, are in 'model.py'
+with every Model.
+
+
+REFERENCES:
+-----------
+1. MIT xPRO Course: AI and Deep Learning: Neural Network Applications
+
+2. NumPy -- numpy.ndarray
+   https://numpy.org/doc/stable/reference/arrays.ndarray.html
+"""
+
+# endregion
+
+# region  I M P O R T S
+
+import math
+
+import numbers
+
+import numpy as np
+
+import torch
+
+from torch import nn
+
+# endregion
+
+# region  C O N S T A N T S
+
+# added to a sigmoid's denominator, so it never divides by 0, as the course's neurons do
+NON_ZERO = 1e-8
+# what every neuron of one kind shares, keyed by its activation's name:
+#   label: its name as the views print it; module: the PyTorch module that applies it
+#   range: the (low, high) its outputs can reach
+#   activation(z): its NumPy activation of a layer's z, (N, neurons, ...); softmax spans the neurons
+#   slope(a): da/dz at its activation a, the term every weight's gradient is scaled by
+#   has_curve: whether one neuron's a is a curve of its own z alone, so a view can draw it
+NEURON_KINDS = {
+    'linear': {
+        'label': 'Linear',
+        'module': nn.Identity,
+        'range': (-math.inf, math.inf),
+        'activation': lambda z: z,
+        'slope': lambda a: 1.0,
+        'has_curve': True,
+    },
+    'sigmoid': {
+        'label': 'Sigmoid',
+        'module': nn.Sigmoid,
+        'range': (0.0, 1.0),
+        'activation': lambda z: 1 / (1 + np.exp(-z) + NON_ZERO),
+        'slope': lambda a: a * (1 - a),
+        'has_curve': True,
+    },
+    'tanh': {
+        'label': 'Tanh',
+        'module': nn.Tanh,
+        'range': (-1.0, 1.0),
+        'activation': lambda z: np.tanh(z),
+        'slope': lambda a: 1 - a**2,
+        'has_curve': True,
+    },
+    'relu': {
+        'label': 'ReLU',
+        'module': nn.ReLU,
+        'range': (0.0, math.inf),
+        'activation': lambda z: np.maximum(z, 0.0),
+        'slope': lambda a: (a > 0) * 1.0,
+        'has_curve': True,
+    },
+    'softmax': {
+        'label': 'Softmax',
+        'module': nn.Softmax,
+        'range': (0.0, 1.0),
+        'activation': lambda z: torch.softmax(torch.as_tensor(z), dim=1).numpy(),
+        'slope': lambda a: a * (1 - a),
+        'has_curve': False,
+    },
+}
+
+# endregion
+
+# region  C L A S S E S
+
+
+class Neuron:
+    """
+    One neuron of a model's 'Layer': a view of one of its outputs, holding no weights of its own,
+    so it always reads the trained values.
+
+    Args:
+        layer:  the 'Layer' it belongs to.
+        index:  its position in 'layer.neuron', its output's row of the layer's weights.
+
+    Note:
+        Layers build their own neurons; read them from 'model.layer[k].neuron[j]', or
+        'model.network[i].layer[k].neuron[j]' on a Model of several networks.
+
+    Example:
+        >>> neuron = model.layer[0].neuron[5]
+        >>> z, a = neuron.outputs(X)
+    """
+
+    def __init__(self, layer, index):
+        self.layer = layer
+        self.index = index
+
+    @property
+    def kind(self):
+        """Its activation, its layer's, a key of 'NEURON_KINDS', e.g. "relu"."""
+        return self.layer.activation
+
+    @property
+    def weight(self):
+        """Its weights: a Linear layer's row, a conv filter (in_channels, h, w), or a single neuron's 'w'."""
+        return self.layer.neuron_weights(self.index)
+
+    @property
+    def bias(self):
+        """Its bias, a float."""
+        return float(self.layer.bias[self.index])
+
+    @property
+    def label(self):
+        """Its position, as the code reaches it, e.g. "layer[2].neuron[5]" or "network[1].layer[0].neuron[3]"."""
+        return f'{self.layer.label}.neuron[{self.index}]'
+
+    def outputs(self, X):
+        """
+        Returns its (z, a) for every sample, its column of 'layer.outputs(X)'.
+
+        Args:
+            X:  the input samples, as the model's 'outputs()' or 'classify()' takes them.
+
+        Returns:
+            (z, a) float64 NumPy arrays: (N,) for a Linear neuron, (N, h, w) for a conv neuron's map.
+        """
+        z, a = self.layer.outputs(X)
+        return z[:, self.index], a[:, self.index]
+
+
+class NeuronOptimizer:
+    """
+    Plain gradient descent for a single neuron model: the optimizer 'train_model()' takes for a NeuronModel.
+
+    Args:
+        lr: the learning rate, the step size of each update, a number above 0.
+
+    Note:
+        It holds the two attributes a PyTorch optimizer documents, 'defaults' and
+        'param_groups', so a run's "train" event records it as any optimizer:
+        {"name": "NeuronOptimizer", "lr": 0.01, "defaults": {}}. Each step moves the neuron
+        by the mean of its batch's per-sample updates, so a 'batch_size' of 1 is the course's
+        per-sample loop.
+
+    Example:
+        >>> neuron = train_model(neuron, NeuronOptimizer(lr=0.01), None, num_epochs=10, batch_size=1)
+    """
+
+    def __init__(self, lr):
+        if isinstance(lr, bool) or not isinstance(lr, numbers.Real) or not lr > 0:
+            raise ValueError(f"'lr' must be a number above 0, got {lr!r}")
+        self.defaults = {'lr': lr}
+        self.param_groups = [{'lr': lr}]
+
+    def step(self, model, grad_loss):
+        """
+        Moves the neuron's weights one step down its loss: 'model.update(grad_loss, lr)'.
+
+        Args:
+            model:      the NeuronModel, its 'gradient()' cached for this batch.
+            grad_loss:  dL/da of each sample of the batch, 'model.loss_gradient(a, y)'.
+        """
+        model.update(grad_loss, self.param_groups[0]['lr'])
+
+
+# endregion
+
+# region  P R I V A T E   F U N C T I O N S
+
+# endregion
+
+# region  P U B L I C   F U N C T I O N S
+
+# endregion
+# endregion
+# endregion