mcode-neural 0.5.0__py3-none-any.whl

mcode_neural/layer.py

@@ -0,0 +1,589 @@
+# region  F I L E
+# <copyright file="mcode-neural/src/mcode_neural/layer.py" company="MicroCODE Incorporated">Copyright © 2026 MicroCODE, Inc. Troy, MI</copyright><author>Timothy J. McGuire</author>
+# region  M O D U L E
+# region  D O C U M E N T A T I O N
+"""
+Project:  MicroCODE Python Packages
+Customer: Internal + MIT xPRO Course
+@module   'mcode_neural.layer'
+@memberof mcode_neural
+@created  October 2026
+@author   Timothy McGuire, MicroCODE, Inc.
+@description >
+MicroCODE Shared Layer Library
+
+LICENSE:
+--------
+MIT License: MicroCODE.mcode-neural
+
+Copyright (c) 2026 Timothy McGuire, MicroCODE, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+DESCRIPTION:
+------------
+This module implements what every MicroCODE layer is: 'LAYER_KINDS', each
+weight module's kind, its fields, how it indexes its neurons, and how its
+weights are read; and 'Layer', one weight module of a Network with the modules
+after it, a view of its trained weights holding its 'Neuron's. It also builds
+and checks the networks' stacks: hidden 'Linear' layers, each followed by a
+ReLU (and a Dropout when asked), then a 'Linear' output layer, after any
+convolution stages, 'Conv2d' / 'ReLU' / 'MaxPool2d', and an autoencoder's
+mirrored decoder. The Networks that hold the layers are in 'network.py'; the
+Models, 'FeedforwardModel' and its subclasses and 'AutoencoderModel', are in
+'model.py' with every Model.
+
+
+REFERENCES:
+-----------
+1. MIT xPRO Course: AI and Deep Learning: Neural Network Applications
+
+2. PyTorch -- torch.nn (Linear, ReLU, Dropout, Sequential, Identity, Sigmoid, Tanh,
+   Conv2d, MaxPool2d, Flatten, Unflatten, ConvTranspose2d)
+   https://pytorch.org/docs/stable/nn.html
+"""
+
+# endregion
+
+# region  I M P O R T S
+
+import math
+import numbers
+from collections.abc import Sequence
+from itertools import pairwise
+
+import numpy as np
+import torch
+
+from torch import nn
+
+from .data import DATA_KINDS
+from .neuron import NEURON_KINDS, Neuron
+
+# endregion
+
+# region  C O N S T A N T S
+
+# the keys of each 'convolution' entry, in the order the "create" event records them:
+#   channels: the number of feature maps the stage's Conv2d outputs
+#   kernel: the Conv2d's square kernel width, odd, so padding keeps the image's height and width
+#   pool: the MaxPool2d's square window, dividing height and width by it; 1 adds no pooling
+CONV_STAGE_KEYS = ('channels', 'kernel', 'pool')
+
+# what every layer of one kind shares, keyed by the kind of its weight module:
+#   module: the PyTorch module that holds its weights
+#   fields: the module's settings, in the order the internals table labels them
+#   neuron_axis: the axis of the module's 'weight' that indexes its neurons
+#   parameters(module): the module's (weight, bias) tensors
+LAYER_KINDS = {
+    'linear': {
+        'module': nn.Linear,
+        'fields': ('in_features', 'out_features'),
+        'neuron_axis': 0,
+        'parameters': lambda module: (module.weight, module.bias),
+    },
+    'conv': {
+        'module': nn.Conv2d,
+        'fields': ('in_channels', 'out_channels', 'kernel_size', 'stride', 'padding'),
+        'neuron_axis': 0,
+        'parameters': lambda module: (module.weight, module.bias),
+    },
+    'deconv': {
+        'module': nn.ConvTranspose2d,
+        'fields': ('in_channels', 'out_channels', 'kernel_size', 'stride'),
+        # a ConvTranspose2d's weight is (in_channels, out_channels, ...), its outputs second
+        'neuron_axis': 1,
+        'parameters': lambda module: (module.weight, module.bias),
+    },
+}
+
+# a name for each of one neuron's inputs, keyed by the length of its 'input_layout()':
+#   1: a row of inputs, "in[k]"
+#   3: channels of rows and columns, a conv filter or an image, "in[c] (r, s)"
+INPUT_NAMES = {
+    1: lambda layout: [f'in[{k}]' for k in range(layout[0])],
+    3: lambda layout: [f'in[{c}] ({r}, {s})' for c, r, s in np.ndindex(*layout)],
+}
+
+# each weight module's layer kind, and each activation module's neuron kind, by the module's type
+LAYER_MODULES = {row['module']: kind for kind, row in LAYER_KINDS.items()}
+NEURON_MODULES = {row['module']: kind for kind, row in NEURON_KINDS.items()}
+# the modules that reshape a sample without changing a value, so a layer's shape out is read before them
+RESHAPE_MODULES = (nn.Flatten, nn.Unflatten)
+# the modules a layer holds after its weight module that neither weigh nor activate
+LAYER_PASS_MODULES = (nn.Dropout, nn.MaxPool2d, nn.AdaptiveAvgPool2d)
+
+# endregion
+
+# region  C L A S S E S
+
+
+class Layer:
+    """
+    One layer of a Network: a view of one weight module, or a single neuron's 'w', with the modules
+    that follow it up to the next weight module. It holds no weights of its own, so it always
+    reads the trained values.
+
+    Args:
+        network:    the Network it belongs to.
+        index:      its position in 'network.layer', in the order its weight modules run.
+        kind:       its weight module's kind, a key of 'LAYER_KINDS': "linear", "conv", or "deconv".
+        activation: its neurons' kind, a key of 'NEURON_KINDS', e.g. "relu".
+        module:     the nn modules in it, its weight module first, e.g. (Linear, ReLU, Dropout); ()
+                    for a single neuron.
+        in_shape:   one sample's shape entering its weight module, e.g. (784,) or (1, 28, 28).
+        out_shape:  one sample's shape leaving it, read before a Flatten or Unflatten, e.g. (64,)
+                    or (8, 14, 14).
+
+    Note:
+        Networks build their own layers; read them from 'model.network[k].layer', or from
+        'model.layer' on a Model of one network. A MaxPool2d, Dropout, Flatten, or Unflatten
+        belongs to the layer before it; its 'outputs()' are read before them.
+
+    Example:
+        >>> layer = model.layer[1]
+        >>> z, a = layer.outputs(X)
+        >>> layer.neuron[5].bias
+    """
+
+    def __init__(
+        self,
+        network,
+        index,
+        kind,
+        activation,
+        module,
+        in_shape,
+        out_shape,
+    ):
+        self.network = network
+        self.index = index
+        self.kind = kind
+        self.activation = activation
+        self.module = module
+        self.in_shape = in_shape
+        self.out_shape = out_shape
+        # one neuron per output, as its weight module indexes them
+        count = self.weight.shape[LAYER_KINDS[kind]['neuron_axis']]
+        self.neuron = tuple(Neuron(self, j) for j in range(count))
+
+    @property
+    def model(self):
+        """The Model its Network belongs to."""
+        return self.network.model
+
+    @property
+    def weight(self):
+        """Its weights, a NumPy array read from its Network on each access, e.g. (out_features, in_features)."""
+        return self.network._layer_parameters(self)[0]
+
+    @property
+    def bias(self):
+        """Its biases, a NumPy array of one per neuron, read from its Network on each access."""
+        return self.network._layer_parameters(self)[1]
+
+    def neuron_weights(self, j):
+        """Returns neuron j's weights, taken on its kind's 'neuron_axis', e.g. one row of a Linear layer."""
+        return np.take(self.weight, j, axis=LAYER_KINDS[self.kind]['neuron_axis'])
+
+    def input_layout(self):
+        """
+        Returns the shape one neuron's inputs, and so its weights, are laid out in: a conv filter's
+        (in_channels, kh, kw); the first Linear layer of a network that reads the data, when one
+        sample of the data is an image, the image's (c, h, w); any other Linear layer's (in_features,).
+        """
+        if self._reads_data() and not DATA_KINDS[self.model.data.kind]['has_features']:
+            sample = next(iter(self.model.data.datasets.values()))[0][0].shape
+            if len(sample) == 3:
+                return tuple(sample)
+        return tuple(self.neuron[0].weight.shape)
+
+    def input_names(self):
+        """
+        Returns a name for each of one neuron's weights, in its order: the data's feature names for
+        the first Linear layer of a network that reads data that names them, else by its
+        'input_layout()': "in[k]" for a row, or "in[c] (r, s)" for a conv filter or an image.
+        """
+        if self._reads_data() and DATA_KINDS[self.model.data.kind]['has_features']:
+            return list(self.model.data.feature)
+        layout = self.input_layout()
+        return INPUT_NAMES[len(layout)](layout)
+
+    def inputs(self, X):
+        """
+        Returns every sample's input to its weight module, the x its neurons weigh, scored as
+        'outputs()' scores, its train / eval mode left as it was.
+
+        Args:
+            X:  the input samples, as the model's 'outputs()' or 'classify()' takes them.
+
+        Returns:
+            a float64 NumPy array of shape (N, *in_shape), e.g. (N, 3) for a first Linear layer
+            of three features, or (N, 8, 14, 14) for a conv layer reading 8 maps.
+        """
+        # how a layer's inputs are read is its Network's kind, in NETWORK_KINDS
+        return np.asarray(self.network._layer_inputs(self, X), dtype=np.float64)
+
+    def outputs(self, X):
+        """
+        Returns every sample's z, its weight module's output, and a, its activation of z, scored as
+        the model scores, its train / eval mode left as it was.
+
+        Args:
+            X:  the input samples, as the model's 'outputs()' or 'classify()' takes them.
+
+        Returns:
+            (z, a) float64 NumPy arrays of shape (N, neurons, ...), e.g. (N, 64), or (N, 8, 28, 28)
+            for a conv layer, whose MaxPool2d follows its activation.
+        """
+        z = np.asarray(self.network._layer_z(self, X), dtype=np.float64)
+        return z, NEURON_KINDS[self.activation]['activation'](z)
+
+    @property
+    def parameters(self):
+        """Its count of learned values, its weights and biases."""
+        return int(self.weight.size + self.bias.size)
+
+    @property
+    def label(self):
+        """Its position, as the code reaches it: "layer[2]" on a Model of one network, else "network[1].layer[0]"."""
+        if len(self.model.network) == 1:
+            return f'layer[{self.index}]'
+        return f'{self.network.label}.layer[{self.index}]'
+
+    def _reads_data(self):
+        """Returns whether it is the first Linear layer of a network that reads the data."""
+        return (
+            self.network.source == ('data',)
+            and self.index == 0
+            and self.kind == 'linear'
+        )
+
+
+# endregion
+
+# region  P R I V A T E   F U N C T I O N S
+
+
+def _check_width(name, width):
+    """
+    Raises a ValueError naming 'name' unless 'width' is a whole number of at least 1.
+
+    Args:
+        name:   the field being checked, as the caller wrote it, e.g. "hidden_size[1]".
+        width:  the layer width to check; NumPy integers are whole numbers, bools are not.
+    """
+    if isinstance(width, bool) or not isinstance(width, numbers.Integral) or width < 1:
+        raise ValueError(
+            f"'{name}' must be a whole number of at least 1, got {width!r}"
+        )
+
+
+def _check_sizes(name, sizes):
+    """
+    Raises a TypeError naming 'name' unless 'sizes' is a sequence of widths, each checked by '_check_width()'.
+
+    Args:
+        name:   the argument being checked, e.g. "hidden_size".
+        sizes:  the layer widths, in order, e.g. (64, 32); () is no layers.
+    """
+    if not isinstance(sizes, Sequence):
+        raise TypeError(
+            f"'{name}' must be a sequence of widths, e.g. (64, 32), got {sizes!r}"
+        )
+    for i, width in enumerate(sizes):
+        _check_width(f'{name}[{i}]', width)
+
+
+def _check_dropout(dropout):
+    """
+    Raises a ValueError unless 'dropout' is a number at least 0 and below 1.
+
+    Args:
+        dropout:    the probability of zeroing each hidden activation while training.
+    """
+    if not isinstance(dropout, numbers.Real) or not 0 <= dropout < 1:
+        raise ValueError(
+            f"'dropout' must be a number at least 0 and below 1, got {dropout!r}"
+        )
+
+
+def _stack(sizes, dropout):
+    """
+    Returns the layers of one stack: Linear, ReLU, and (when asked) Dropout between each pair of
+    widths, then a last Linear with no activation.
+
+    Args:
+        sizes:      every width in order, input first and output last, e.g. (784, 128, 64, 2).
+        dropout:    the Dropout rate after each hidden ReLU; 0 adds no Dropout layers.
+
+    Returns:
+        a list of layers in the course's order, for 'nn.Sequential(*layers)'.
+    """
+    layers = []
+    # each hidden layer reads the width of the layer before it
+    for size_in, size_out in pairwise(sizes[:-1]):
+        layers += [nn.Linear(size_in, size_out), nn.ReLU()]
+        if dropout:
+            layers.append(nn.Dropout(dropout))
+    return layers + [nn.Linear(sizes[-2], sizes[-1])]
+
+
+def _check_conv(in_features, in_shape, convolution, is_mirrored):
+    """
+    Raises a TypeError or ValueError naming the field unless 'convolution' and 'in_shape' build a network that runs.
+
+    Args:
+        in_features:    the network's input width, already checked; 'in_shape' must hold as many values.
+        in_shape:       one image's (channels, height, width); None with no conv stages.
+        convolution:    each stage a dict of exactly 'CONV_STAGE_KEYS', each value a whole number of
+                        at least 1, its kernel odd.
+        is_mirrored:    True for an autoencoder, whose decoder must rebuild every pool exactly, so
+                        each pool must divide the height and width it pools.
+    """
+    if not isinstance(convolution, Sequence):
+        raise TypeError(
+            f"'convolution' must be a sequence of stages, e.g. ({{'channels': 8, 'kernel': 3, 'pool': 2}},), got {convolution!r}"
+        )
+    for i, stage in enumerate(convolution):
+        if not isinstance(stage, dict):
+            raise TypeError(
+                f"'convolution[{i}]' must be a dict of {list(CONV_STAGE_KEYS)}, got {stage!r}"
+            )
+        if set(stage) != set(CONV_STAGE_KEYS):
+            raise ValueError(
+                f"'convolution[{i}]' must have exactly the keys {list(CONV_STAGE_KEYS)}, got {list(stage)}"
+            )
+        for key in CONV_STAGE_KEYS:
+            _check_width(f"convolution[{i}]['{key}']", stage[key])
+        # an even kernel cannot be padded evenly, so the stage would change the image's size
+        if stage['kernel'] % 2 == 0:
+            raise ValueError(
+                f"'convolution[{i}]['kernel']' must be odd, so the image keeps its height and width, got {stage['kernel']}"
+            )
+    if not convolution:
+        if in_shape is not None:
+            raise ValueError(
+                f"'in_shape' is read only with 'convolution'; got in_shape={in_shape!r} and no stages"
+            )
+        return
+    if in_shape is None:
+        raise ValueError(
+            "'convolution' needs 'in_shape', one image's (channels, height, width), e.g. (1, 28, 28)"
+        )
+    if not isinstance(in_shape, Sequence):
+        raise TypeError(
+            f"'in_shape' must be a sequence, (channels, height, width), got {in_shape!r}"
+        )
+    if len(in_shape) != 3:
+        raise ValueError(
+            f"'in_shape' must be (channels, height, width), got {tuple(in_shape)}"
+        )
+    for i, size in enumerate(in_shape):
+        _check_width(f'in_shape[{i}]', size)
+    if math.prod(in_shape) != in_features:
+        raise ValueError(
+            f"'in_shape' {tuple(in_shape)} holds {math.prod(in_shape)} values, not 'in_features' {in_features}"
+        )
+    shapes = [tuple(in_shape), *_stage_shapes(in_shape, convolution)]
+    for i, (before, after) in enumerate(pairwise(shapes)):
+        pool = convolution[i]['pool']
+        if 0 in after:
+            raise ValueError(
+                f"'convolution[{i}]['pool']' {pool} pools a {before[1]} x {before[2]} image to nothing"
+            )
+        # ConvTranspose2d multiplies by the pool, so a floored height or width would come back short
+        if is_mirrored and (before[1] % pool or before[2] % pool):
+            raise ValueError(
+                f"'convolution[{i}]['pool']' {pool} must divide the {before[1]} x {before[2]} image it pools,"
+                ' so the decoder rebuilds it'
+            )
+
+
+def _conv_arguments(in_shape, convolution):
+    """
+    Returns the conv arguments as the "create" event records them, already checked by '_check_conv()'.
+
+    Args:
+        in_shape:       one image's (channels, height, width), or None.
+        convolution:    the stages, each a dict of 'CONV_STAGE_KEYS'.
+
+    Returns:
+        {"in_shape": a tuple of ints or None, "convolution": a tuple of dicts, each in 'CONV_STAGE_KEYS' order}.
+    """
+    return {
+        'in_shape': None if in_shape is None else tuple(int(size) for size in in_shape),
+        'convolution': tuple(
+            {key: int(stage[key]) for key in CONV_STAGE_KEYS} for stage in convolution
+        ),
+    }
+
+
+def _sample_shape(in_features, in_shape):
+    """Returns one sample's shape as the network reads it: 'in_shape', or (in_features,) when it is None."""
+    if in_shape is None:
+        return (int(in_features),)
+    return in_shape
+
+
+def _stage_shapes(in_shape, convolution):
+    """
+    Returns each conv stage's output (channels, height, width), in order; [] with no stages.
+
+    Args:
+        in_shape:       one image's (channels, height, width); unread with no stages.
+        convolution:    the stages; each keeps the height and width through its Conv2d, then its
+                        MaxPool2d floors them divided by its pool.
+    """
+    shapes = []
+    shape = in_shape
+    for stage in convolution:
+        shape = (
+            stage['channels'],
+            shape[1] // stage['pool'],
+            shape[2] // stage['pool'],
+        )
+        shapes.append(shape)
+    return shapes
+
+
+def _conv_stack(in_shape, convolution):
+    """
+    Returns the conv layers: Conv2d, ReLU, and MaxPool2d (none for a pool of 1) per stage, then a Flatten; [] with no stages.
+
+    Args:
+        in_shape:       one image's (channels, height, width); its channels feed the first Conv2d.
+        convolution:    the stages, each a dict of 'CONV_STAGE_KEYS'.
+    """
+    if not convolution:
+        return []
+    layers = []
+    channels = in_shape[0]
+    for stage in convolution:
+        kernel = stage['kernel']
+        layers += [
+            nn.Conv2d(channels, stage['channels'], kernel, padding=kernel // 2),
+            nn.ReLU(),
+        ]
+        if stage['pool'] > 1:
+            layers.append(nn.MaxPool2d(stage['pool']))
+        channels = stage['channels']
+    return layers + [nn.Flatten()]
+
+
+def _deconv_stack(in_shape, convolution):
+    """
+    Returns the decoder's conv layers, after its last Linear: a ReLU and an Unflatten to the last
+    stage's maps, then one ConvTranspose2d per stage in reverse, its kernel and stride the stage's
+    pool, a ReLU between each and none after the last; [] with no stages.
+
+    Args:
+        in_shape:       one image's (channels, height, width), which the last ConvTranspose2d outputs.
+        convolution:    the encoder's stages, each a dict of 'CONV_STAGE_KEYS'.
+    """
+    if not convolution:
+        return []
+    shapes = [in_shape, *_stage_shapes(in_shape, convolution)]
+    layers = [nn.ReLU(), nn.Unflatten(1, shapes[-1])]
+    for before, stage in zip(shapes[-2::-1], convolution[::-1]):
+        pool = stage['pool']
+        layers += [
+            nn.ConvTranspose2d(stage['channels'], before[0], pool, stride=pool),
+            nn.ReLU(),
+        ]
+    # the output style reads the last ConvTranspose2d's values, which a ReLU would clip at 0
+    return layers[:-1]
+
+
+def _shaped(inputs, shape, subject):
+    """
+    Returns 'inputs' reshaped to (N, *shape), each sample as the network reads it.
+
+    Args:
+        inputs:     the samples, a tensor of shape (N, ...), e.g. (N, 1, 28, 28) images or (N, 784) rows.
+        shape:      one sample's shape as the network reads it, e.g. (1, 28, 28), or (784,) for a
+                    Linear network.
+        subject:    what the samples are, for the message, e.g. "'X'".
+    """
+    size = math.prod(shape)
+    if inputs.shape[1:].numel() != size:
+        raise ValueError(
+            f'{subject} must hold {size} values per sample, read as {tuple(shape)}, got shape {tuple(inputs.shape)}'
+        )
+    return inputs.reshape(len(inputs), *shape)
+
+
+def _scored(model, layers, X, name, shape):
+    """
+    Returns 'layers(X)' for every sample, scored in eval() without gradients, the model's mode left as it was.
+
+    Args:
+        model:  the network 'layers' belong to; its mode is set and restored.
+        layers: the module to run, the whole model or one of its stacks, e.g. 'model.encoder'.
+        X:      the samples, one per row: a tensor or NumPy array of shape (N, ...).
+        name:   the caller's name for X, e.g. "Z", for the messages.
+        shape:  one sample's shape as 'layers' read it, e.g. 'model.in_shape', or (latent_size,)
+                for a decoder; each sample is reshaped to it, as 'train_model()' feeds the network.
+
+    Returns:
+        the outputs tensor, on the CPU.
+
+    Note:
+        X is scored in chunks of the model's scoring batch size, its latest training's
+        'batch_size', each chunk moved to the model's device alone and its outputs back to the
+        CPU, so a phase of any size fits the GPU memory one training batch did.
+    """
+    from .model import _score_batch_size
+
+    # PyTorch's own errors for these name neither 'X' nor the shape it needs
+    if not isinstance(X, (torch.Tensor, np.ndarray)):
+        raise TypeError(
+            f"'{name}' must be a tensor or NumPy array, got {type(X).__name__}"
+        )
+    if X.ndim < 2:
+        raise ValueError(
+            f"'{name}' must hold one row per sample, got shape {tuple(X.shape)}"
+        )
+    # the samples take the model's own dtype here, and its device one chunk at a time
+    parameter = next(model.parameters())
+    inputs = torch.as_tensor(X, dtype=parameter.dtype)
+    # each sample shaped as train_model() feeds the network; an empty X keeps its 0 rows
+    inputs = _shaped(inputs, shape, f"'{name}'")
+    is_training = model.training
+    # Dropout is off to score; the caller's mode comes back even when the forward pass fails
+    model.eval()
+    try:
+        with torch.no_grad():
+            outputs = [
+                layers(chunk.to(parameter.device)).cpu()
+                for chunk in inputs.split(_score_batch_size(model))
+            ]
+    finally:
+        model.train(is_training)
+    return torch.cat(outputs)
+
+
+# endregion
+
+# region  P U B L I C   F U N C T I O N S
+
+# endregion
+# endregion
+# endregion