zeroth-learn 0.1.0__py3-none-any.whl

zeroth/__init__.py

@@ -0,0 +1,5 @@
+from zeroth.model import (Model, ModelConfig, ZerothOrderModel, FirstOrderModel,
+                          ZerothOrderModelConfig, FirstOrderModelConfig)
+from zeroth.experiment import Experiment, ExperimentConfig, VariationConfig
+
+from zeroth.data import Data

zeroth/experiment.py

@@ -0,0 +1,134 @@
+from zeroth.utils.dataclasses_utils import config_serializer, generate_param_map, get_name, set_value_by_path
+from zeroth.model import ModelConfig, Model
+from zeroth.plot_losses import plot_losses
+from zeroth.data import Data
+
+from dataclasses import dataclass, replace
+from typing import Callable
+
+import pandas as pd
+import itertools
+import json
+import os
+
+
+
+
+@dataclass(frozen=True)
+class VariationConfig:
+    param: str
+    values: list
+
+
+@dataclass(frozen=True)
+class ExperimentConfig:
+    name: str
+    title: str
+    base_model: ModelConfig
+    variations: list[VariationConfig]
+    create_data: Callable
+    plot_dimension : int
+
+    def instantiate(self):
+        return Experiment(self)
+
+
+class Experiment:
+    """Manages the full lifecycle of a deep learning experiment.
+
+    It handles data loading, model instantiation, training loops, and results visualization.
+
+    Attributes:
+        name (str): Name of the experiment
+        title (str): Title of the graphs
+        models (list[Model]): List of models to train/compare
+        data (Data): The dataset wrapper.
+    """
+
+    def __init__(self, config: ExperimentConfig):
+        self.config = config
+        self.name: str = config.name
+        self.title: str = config.title
+        self.base_model_config: ModelConfig = config.base_model
+        self.models: list[Model] = generate_models(config.base_model, config.variations)
+        self.data: Data = config.create_data()
+        self.plot_dimension: int = config.plot_dimension
+
+        self.save_dir = os.path.join("results", self.name)
+
+    def launch(self, do_train, do_test, nb_print_train, do_plot_train, do_save):
+        """
+        Executes the experiment pipeline.
+
+        Args:
+            do_train (bool): Whether to run the training loop.
+            do_test (bool): Whether to run evaluation on test set.
+            nb_print_train (int): Number of logs to print during training.
+            do_plot_train (bool): If True, plots loss curves after training.
+            do_save (bool): if True, saves the plots and dataframes
+        """
+        print(f"### Launching Experiment : {self.name} ###")
+        if do_train:
+            self.train(nb_print=nb_print_train, do_plot=do_plot_train, do_save=do_save)
+        if do_test:
+            self.test()
+        if do_save:
+            self.save_df()
+
+    def train(self, nb_print: int, do_plot: bool, do_save: bool):
+        for model in self.models:
+            model.train(self.data, nb_print)
+
+        if do_plot:
+            plot_path = None
+            if do_save:
+                os.makedirs(self.save_dir, exist_ok=True)
+                plot_path = os.path.join(self.save_dir, "training_losses.png")
+
+            plot_losses(dimension=self.plot_dimension, models=self.models, title=self.title, save_path=plot_path)
+
+    def test(self):
+        for model in self.models:
+            model.test(self.data)
+
+    def save_df(self):
+        """
+        saves the models parameters and their args
+        """
+        os.makedirs(self.save_dir, exist_ok=True)
+        print(f"    Saving results to: {self.save_dir}")
+
+        data = [model.id | {"test_loss": model.test_loss, "test_accuracy": model.test_accuracy}
+                for model in self.models]
+
+        df = pd.DataFrame(data)
+        df.to_csv(os.path.join(self.save_dir, "models_accuracy.csv"), index_label="iteration")
+
+        config_path = os.path.join(self.save_dir, "config.json")
+        with open(config_path, "w") as f:
+            json.dump(self.config, f, default=config_serializer, indent=4)
+
+
+def generate_models(base_model: ModelConfig, variations: list[VariationConfig]) -> list[Model]:
+
+    models = []
+
+    names = [v.param for v in variations]
+    values = [v.values for v in variations]
+
+    # key: param  value: path
+    PARAM_MAP = generate_param_map(base_model)
+
+    for combination in itertools.product(*values):
+        id_ = {}
+        for key, val in zip(names, combination):
+            id_[key] = get_name(val)
+        current_model = base_model
+        for param_key, value in zip(names, combination):
+            path = PARAM_MAP[param_key]
+            current_model = set_value_by_path(current_model, path, value)
+
+        current_model = replace(current_model, id=id_)
+        models.append(current_model.instantiate())
+
+    return models

zeroth/losses.py

@@ -0,0 +1,121 @@
+from abc import ABC, abstractmethod
+import numpy as np
+
+
+class Loss(ABC):
+    @staticmethod
+    @abstractmethod
+    def compute_loss(Y_pred: np.ndarray, Y_true: np.ndarray) -> float:
+        """
+        :param Y_pred: shape (out, batch)
+        :param Y_true: shape (out, batch)
+        :return: avg loss shape: float
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def compute_batch_losses(Y_pred: np.ndarray, Y_true: np.ndarray) -> np.ndarray:
+        """
+        :param Y_pred: shape (out, batch)
+        :param Y_true: shape (out, batch)
+        :return: batch loss shape (batch, )
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def compute_perturbed_losses(pY_pred: np.ndarray, Y_true: np.ndarray) -> np.ndarray:
+        """
+        :param pY_pred: (T, out, batch)
+        :param Y_true: (out, batch)
+        :return: perturbed loss (nb_params, T)
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def compute_gradient_wrt_preactivation(last_layer, Y_pred: np.ndarray, Y_true: np.ndarray) -> np.ndarray:
+        """
+        :param last_layer: last_layer of the network
+        :param Y_pred: shape (out, batch)
+        :param Y_true: shape (out, batch)
+        :return: batch loss shape (batch, )
+        """
+        pass
+
+    @abstractmethod
+    def compute_losses_for_zeroth_order(self, pY_pred: np.ndarray, Y_true: np.ndarray) -> tuple[float, np.ndarray]:
+        pass
+
+
+    @abstractmethod
+    def compute_losses_for_first_order(self, last_layer, Y_pred: np.ndarray, Y_true: np.ndarray):
+        pass
+
+
+class MSE(Loss):
+    name = "MSE"
+
+    @staticmethod
+    def compute_loss(Y_pred, Y_true) -> float:
+        return np.mean((Y_pred - Y_true) ** 2, axis=(0, 1))
+
+    @staticmethod
+    def compute_batch_losses(Y_pred, Y_true):
+        return np.mean((Y_pred - Y_true) ** 2, axis=0)
+
+    @staticmethod
+    def compute_perturbed_losses(pY_pred, Y_true):
+        return np.mean((pY_pred - Y_true) ** 2, axis=1)
+
+    @staticmethod
+    def compute_gradient_wrt_activation(Y_pred, Y_true):
+        return 2 * np.mean(Y_pred - Y_true, axis=0)
+
+    @staticmethod
+    def compute_gradient_wrt_preactivation(last_layer, Y_pred, Y_true):
+        dL_dA = 2 * (Y_pred - Y_true) / Y_true.size
+        dL_dZ = dL_dA * last_layer.df(last_layer.Z)
+        return dL_dZ
+
+    def compute_losses_for_zeroth_order(self, pY_pred, Y_true):
+        return (self.compute_loss(pY_pred[0], Y_true),
+                self.compute_perturbed_losses(pY_pred, Y_true))
+
+    def compute_losses_for_first_order(self, last_layer, Y_pred, Y_true):
+        return self.compute_loss(Y_pred, Y_true), self.compute_gradient_wrt_preactivation(last_layer, Y_pred, Y_true)
+
+
+class CrossEntropy(Loss):
+    name = "CrossEntropy"
+
+    @staticmethod
+    def compute_loss(Y_pred: np.ndarray, Y_true: np.ndarray) -> float:
+        idx = np.arange(Y_pred.shape[1])
+        return - np.mean(np.log(1e-8 + Y_pred[Y_true, idx]))
+
+    @staticmethod
+    def compute_batch_losses(Y_pred: np.ndarray, Y_true: np.ndarray):
+        idx = np.arange(Y_pred.shape[1])
+        return - np.log(1e-8 + Y_pred[Y_true, idx])
+
+    @staticmethod
+    def compute_perturbed_losses(pY_pred: np.ndarray, Y_true: np.ndarray):
+        idx = np.arange(pY_pred.shape[2])
+        return - np.mean(np.log(1e-8 + pY_pred[:, Y_true, idx]), axis=1)
+
+    @staticmethod
+    def compute_gradient_wrt_preactivation(last_layer, Y_pred: np.ndarray, Y_true: np.ndarray):
+        dZ = Y_pred.copy()
+        batch_size = Y_pred.shape[1]
+        dZ[Y_true[0], np.arange(batch_size)] -= 1.0
+        return dZ
+
+    def compute_losses_for_zeroth_order(self, pY_pred: np.ndarray, Y_true: np.ndarray):
+        avg_loss = self.compute_loss(pY_pred[0], Y_true)
+        p_loss = self.compute_perturbed_losses(pY_pred, Y_true)
+        return avg_loss, p_loss
+
+    def compute_losses_for_first_order(self, last_layer, Y_pred: np.ndarray, Y_true: np.ndarray) -> tuple[float, np.ndarray]:
+        return self.compute_loss(Y_pred, Y_true), self.compute_gradient_wrt_preactivation(last_layer, Y_pred, Y_true)

zeroth/model.py

@@ -0,0 +1,144 @@
+from dataclasses import dataclass
+from matplotlib.axes import Axes
+from typing import Callable
+from abc import ABC
+
+import pandas as pd
+import numpy as np
+
+from zeroth.zeroth_order import ZerothOrderNeuralNetwork, ZerothOrderOptimizerConfig, GradientEstimatorConfig, \
+    GradientEstimator
+from zeroth.first_order import FirstOrderNeuralNetwork, FirstOrderOptimizerConfig
+from zeroth.abstract import BlackBox, NeuralNetworkConfig, Optimizer
+from zeroth.losses import Loss
+from zeroth.data import Data
+
+
+@dataclass(frozen=True)
+class ModelConfig:
+    """
+    name (str): Name of the model (used for display and saving).
+    loss (Loss): The loss class.
+    metric (Callable): Function (Y_pred, Y_true) -> score (e.g., accuracy).
+    batch_size (int): Number of samples per gradient update.
+    plot_results (Callable): Function to visualize test results.
+    nb_epochs (int): Number of passes through the entire dataset.
+    """
+    name: str
+    id: dict
+    loss: Loss
+    metric: Callable
+    batch_size: int
+    nb_epochs: int
+
+    def instantiate(self):
+        raise NotImplementedError
+
+
+@dataclass(frozen=True)
+class FirstOrderModelConfig(ModelConfig):
+    neural_network_config: NeuralNetworkConfig
+    optimizer_config: FirstOrderOptimizerConfig
+
+    def instantiate(self):
+        return FirstOrderModel(self)
+
+
+@dataclass(frozen=True)
+class ZerothOrderModelConfig(ModelConfig):
+    neural_network_config: NeuralNetworkConfig
+    optimizer_config: ZerothOrderOptimizerConfig
+    gradient_estimator_config: GradientEstimatorConfig
+
+    def instantiate(self):
+        return ZerothOrderModel(self)
+
+
+class Model(ABC):
+    """
+    Base class orchestrating the training and testing loop.
+
+    This class abstracts the abstract logic for training
+    regardless of the underlying engine (Backpropagation or zeroth_order).
+    """
+
+    def __init__(self, config: ModelConfig):
+
+        self.name: str = config.name
+        self.id: dict = config.id
+        self.loss: Loss = config.loss
+        self.metric: Callable = config.metric
+        self.batch_size: int = config.batch_size
+        self.nb_epochs: int = config.nb_epochs
+        self.neural_network: BlackBox | None = None
+        self.optimizer: Optimizer | None = None
+
+        self.train_loss: np.ndarray = np.array([])
+        self.test_loss: float | None = None
+        self.test_accuracy: float | None = None
+
+    def train(self, data: Data, nb_print: int = 0):
+        """Runs the training loop over the dataset.
+
+        Args:
+            data (Data): The dataset object containing train/test sets.
+            nb_print (int): Number of progress updates to print per epoch.
+
+        Returns:
+            np.ndarray: Array of loss values recorded at each step (for plotting).
+        """
+
+        nb_batches = data.nb_data // self.batch_size
+
+        self.train_loss = np.zeros(self.nb_epochs * nb_batches, dtype=np.float64)
+
+        print_indexes = np.linspace(0, nb_batches - 1, nb_print).astype(int)
+        print(f"    Training {self.id} Model")
+        for epoch_idx in range(self.nb_epochs):
+            print(f"        epoch n°{epoch_idx + 1} out of {self.nb_epochs}")
+            data.prepare_data(self.batch_size)
+            for batch_idx in range(nb_batches):
+                X_train, Y_train = data.X_train[batch_idx], data.Y_train[batch_idx]
+                avg_loss = self.optimizer.do_descent(self.neural_network, self.loss, X_train, Y_train)
+                self.train_loss[epoch_idx * nb_batches + batch_idx] = avg_loss
+
+                if batch_idx in print_indexes:
+                    print(f"            batch n°{batch_idx + 1} out of {nb_batches}, "
+                          f"loss : {np.round(self.train_loss[epoch_idx * nb_batches + batch_idx], 3)}")
+            self.test(data)
+
+    def plot_loss(self, ax: Axes, label: str, smooth_span: int = 50):
+        ax.plot(self.train_loss, alpha=0.25, linewidth=1.0)
+        smooth = self.smooth_curve(self.train_loss, smooth_span)
+        ax.plot(smooth, label=label, linewidth=2.5)
+
+    @staticmethod
+    def smooth_curve(loss: np.ndarray, smooth_span: int) -> np.ndarray:
+        return np.exp(pd.Series(np.log(loss)).ewm(span=smooth_span, adjust=True).mean())
+
+    def test(self, data):
+        X_test, Y_true = data.X_test, data.Y_test  # (in, batch), (out, batch)
+        Y_pred = self.neural_network.forward(X_test)  # (out, batch)
+
+        self.test_accuracy = self.metric(Y_pred, Y_true)
+        self.test_loss = self.loss.compute_loss(Y_pred, Y_true)
+
+        print(f"    {self.id} accuracy : {self.test_accuracy}, loss : {self.test_loss}")
+
+
+class FirstOrderModel(Model):
+    def __init__(self, config: FirstOrderModelConfig):
+        super().__init__(config)
+
+        self.neural_network = FirstOrderNeuralNetwork(config.neural_network_config)
+        self.optimizer = config.optimizer_config.instantiate()
+
+
+class ZerothOrderModel(Model):
+    def __init__(self, config: ZerothOrderModelConfig):
+        super().__init__(config)
+
+        self.neural_network: ZerothOrderNeuralNetwork = ZerothOrderNeuralNetwork(config.neural_network_config)
+        nb_params = self.neural_network.params.nb_params
+        self.gradient_estimator: GradientEstimator = config.gradient_estimator_config.instantiate(nb_params)
+        self.optimizer: Optimizer = config.optimizer_config.instantiate(self.gradient_estimator)

zeroth/plot_losses.py

@@ -0,0 +1,190 @@
+from zeroth.model import Model
+
+from matplotlib.axes import Axes
+from cycler import cycler
+
+import matplotlib.pyplot as plt
+
+def set_style():
+    plt.rcParams.update({
+        "font.family": "serif",
+        "font.serif": ["Times New Roman", "Times", "DejaVu Serif"],
+        "font.size": 10,
+        "axes.labelsize": 10,
+        "axes.titlesize": 11,
+        "axes.linewidth": 0.8,
+        "axes.spines.top": False,
+        "axes.spines.right": False,
+        "xtick.labelsize": 9,
+        "ytick.labelsize": 9,
+        "xtick.direction": "out",
+        "ytick.direction": "out",
+        "xtick.major.size": 3,
+        "ytick.major.size": 3,
+        "lines.linewidth": 2.2,
+        "legend.fontsize": 9,
+        "legend.frameon": False,
+        "grid.color": "0.85",
+        "grid.linewidth": 0.6,
+        "grid.linestyle": "-",
+        "savefig.dpi": 300,
+        "savefig.bbox": "tight",
+    })
+    plt.rcParams["axes.prop_cycle"] = cycler(color=[
+        "#4477AA", "#EE6677", "#228833",
+        "#CCBB44", "#66CCEE", "#AA3377"
+    ])
+
+
+def format_ax(ax: Axes):
+    ax.set_axisbelow(True)
+    ax.set_yscale('log')
+    ax.grid(True, which="major", axis="y")
+    ax.grid(False, axis="x")
+    ax.spines['top'].set_visible(False)
+    ax.spines['right'].set_visible(False)
+
+
+def plot_0d(models: list[Model], title: str, smooth_span: int = 50):
+    """
+    Plots a single graph overlaying multiple models that share the same hyperparameters.
+    """
+    fig, ax = plt.subplots(figsize=(5.5, 3.5))
+
+    for model in models:
+        others = [f"{k}={v}" for k, v in model.id.items()]
+        label = ", ".join(others)
+        model.plot_loss(ax, label, smooth_span)
+
+        format_ax(ax)
+
+    plt.subplots_adjust(left=0.1, right=0.95, top=0.9, bottom=0.2)
+    fig.suptitle(title, fontweight='bold', fontsize=12)
+    ax.set_xlabel("Training steps")
+    ax.set_ylabel("Training loss")
+    format_ax(ax)
+
+    handles, labels = ax.get_legend_handles_labels()
+
+    if handles:
+        fig.legend(handles, labels, loc='lower center', ncol=len(handles),
+                   bbox_to_anchor=(0.5, 0), frameon=False, fontsize=9)
+
+
+def plot_1d(models: list[Model], title: str, key: str, smooth_span: int = 50):
+    """
+    Plots a row of subplots, varying one hyperparameter (key) across columns.
+    """
+
+    cols = list(dict.fromkeys([m.id[key] for m in models]))
+    n_models = len(cols)
+    fig, axs = plt.subplots(1, n_models, figsize=(4.5 * n_models, 3.5), sharey=True)
+
+    for i, val in enumerate(cols):
+        ax = axs[i]
+        cell_models = [m for m in models if m.id[key] == val]
+
+        for model in cell_models:
+            others = [f"{k}={v}" for k, v in model.id.items()
+                      if k != key]
+            label = ", ".join(others)
+            model.plot_loss(ax, label, smooth_span)
+
+        format_ax(ax)
+
+        ax.set_title(f"{key} = {val}")
+
+
+    fig.text(0.5, 0.1, "Training steps", ha='center', fontsize=10)
+    fig.text(0.01, 0.5, "Training loss", va='center', rotation='vertical', fontsize=10)
+
+    plt.subplots_adjust(left=0.05, right=0.96, top=0.85, bottom=0.2, wspace=0.10, hspace=0.18)
+    fig.suptitle(title, fontweight='bold', fontsize=12)
+
+    handles, labels = axs[0].get_legend_handles_labels()
+
+    if handles:
+        fig.legend(handles, labels, loc='lower center', ncol=len(handles),
+                   bbox_to_anchor=(0.5, 0), frameon=False, fontsize=9)
+
+
+def plot_2d_grid(models: list[Model], title: str, row_key: str, col_key: str, smooth_span: int = 50):
+    """
+    Plots a grid of subplots varying two hyperparameters: one across rows, one across columns.
+
+    Args:
+        models (list): List of model objects.
+        title (str): The title of the plot.
+        row_key (str): The hyperparameter key changing across rows.
+        col_key (str): The hyperparameter key changing across columns.
+        smooth_span (int): The span for the EWM average.
+    """
+    rows = list(dict.fromkeys([m.id[row_key] for m in models]))
+    cols = list(dict.fromkeys([m.id[col_key] for m in models]))
+
+    fig, axs = plt.subplots(len(rows), len(cols),
+                            figsize=(4.5 * len(cols), 3.5 * len(rows)),
+                            sharex=True, sharey=True, squeeze=False)
+
+    for i, r_val in enumerate(rows):
+        for j, c_val in enumerate(cols):
+            ax = axs[i, j]
+            cell_models = [m for m in models if m.id[row_key] == r_val and m.id[col_key] == c_val]
+
+            for model in cell_models:
+                others = [f"{k}={v}" for k, v in model.id.items() if k not in [row_key, col_key]]
+                label = ", ".join(others)
+                model.plot_loss(ax, label, smooth_span)
+
+            format_ax(ax)
+
+
+            if i == 0:
+                ax.set_title(f"{col_key} = {c_val}")
+
+            if j == len(cols) - 1:
+                ax.text(1.02, 0.5, f"{row_key} = {r_val}",
+                        transform=ax.transAxes, rotation=-90,
+                        va="center", ha="left")
+
+    plt.subplots_adjust(left=0.06, right=0.96, top=0.90, bottom=0.12, wspace=0.10, hspace=0.18)
+    fig.suptitle(title, fontsize=14, fontweight='bold', y=0.98)
+
+    handles, labels = axs[0, 0].get_legend_handles_labels()
+    if handles:
+        fig.legend(handles, labels, loc='lower center', ncol=len(handles),
+                   bbox_to_anchor=(0.5, 0.02), frameon=False, fontsize=9)
+
+    fig.text(0.5, 0.07, "Training steps", ha='center', fontsize=10)
+    fig.text(0.02, 0.5, "Training loss", va='center', rotation='vertical', fontsize=10)
+
+
+
+def plot_losses(dimension: int, models: list, title: str, save_path: str = None, smooth_span: int = 100):
+    """
+    Main entry point for plotting. Automatically detects if the plot should be 0D, 1D, or 2D
+    based on the number of variation parameters.
+
+    Args:
+        dimension (int): dimension of the plot
+        models (list): List of model objects.
+        title (str): The title of the plot.
+        save_path (str, optional): File path to save the figure (e.g., 'plot.png').
+        smooth_span (int): EWM span for smoothing. Defaults to 50.
+    """
+    set_style()
+
+    keys = list(models[0].id.keys())
+
+    if dimension == 0:
+        plot_0d(models, title, smooth_span)
+    elif dimension == 1:
+        plot_1d(models, title, keys[0], smooth_span)
+    else:
+        plot_2d_grid(models, title, keys[0], keys[1], smooth_span)
+
+    if save_path:
+        plt.savefig(save_path, dpi=300, bbox_inches="tight")
+        print(f"Plot saved to {save_path}")
+
+    plt.show()

zeroth_learn-0.1.0.dist-info/METADATA

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: zeroth-learn
+Version: 0.1.0
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: matplotlib
+Requires-Dist: cycler

zeroth_learn-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+zeroth/__init__.py,sha256=Zi4bPQQUNwI9sXGkaQFEZA84w0a4je3vdJJEIAsH3kU,260
+zeroth/experiment.py,sha256=Ca60CyNjarDWTCaamvW_2WINAO6SZE4HQsPNIsM_HWA,4320
+zeroth/losses.py,sha256=V9w_o9asXIWtEx_V7dWeurlv-jXGYjsjIc2zT1AQxK0,4071
+zeroth/model.py,sha256=PufxZTHkDb1xe0ExjVy-MAhkR0LYm-ft-o0xlQSMPDc,5355
+zeroth/plot_losses.py,sha256=fT3ipCWtgeBSmhALIn6IkHKcyKRH2ErCrSiX7FhI-nA,6503
+zeroth_learn-0.1.0.dist-info/METADATA,sha256=DsSHGh2eGfTnA0UPg7aY7ir131FqGI2g58Zq0jQmcag,147
+zeroth_learn-0.1.0.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
+zeroth_learn-0.1.0.dist-info/top_level.txt,sha256=ZNQ6QAwK_P_ev_5dpk-HUbpu49Lu25HJ6kbv6f0mOyo,7
+zeroth_learn-0.1.0.dist-info/RECORD,,

zeroth_learn-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

zeroth_learn-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+zeroth