optiml 1.5__tar.gz → 1.7__tar.gz

optiml-1.7/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Donato Meoli
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

optiml-1.7/PKG-INFO

@@ -0,0 +1,203 @@
+Metadata-Version: 2.4
+Name: optiml
+Version: 1.7
+Summary: Optimizers for/and sklearn compatible Machine Learning models
+Home-page: https://github.com/dmeoli/optiml
+Author: Donato Meoli
+Author-email: donato.meoli@outlook.com
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: autograd
+Requires-Dist: casadi
+Requires-Dist: cvxopt
+Requires-Dist: cvxpy
+Requires-Dist: matplotlib
+Requires-Dist: numpy
+Requires-Dist: pytest
+Requires-Dist: qpsolvers
+Requires-Dist: quadprog
+Requires-Dist: scikit-learn
+Requires-Dist: scipy
+Requires-Dist: wurlitzer
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# OptiML
+
+[![CI](https://github.com/dmeoli/optiml/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/dmeoli/optiml/actions/workflows/ci.yml)
+[![Coverage Status](https://coveralls.io/repos/github/dmeoli/optiml/badge.svg?branch=master)](https://coveralls.io/github/dmeoli/optiml?branch=master)
+[![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue)](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue)
+[![PyPI Version](https://img.shields.io/pypi/v/optiml.svg?color=blue)](https://pypi.org/project/optiml/)
+[![PyPI Downloads](https://img.shields.io/pypi/dm/optiml.svg)](https://pypistats.org/packages/optiml)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/dmeoli/optiml/master)
+[![Documentation](https://img.shields.io/badge/docs-online-blue)](https://dmeoli.github.io/optiml)
+
+OptiML is a sklearn compatible implementation of *Support Vector Machines* and *Deep Neural Networks*, both with some of
+the most successful features according to the state of the art.
+
+This work was motivated by the possibility of being able to solve the optimization problem deriving from the
+mathematical formulation of these models through a wide range of optimization algorithms object of study and developed
+for the Numerical Methods and Optimization course @ [Department of Computer Science](https://www.di.unipi.it/en/) @
+[University of Pisa](https://www.unipi.it/index.php/english) under the supervision of
+prof. [Antonio Frangioni](http://pages.di.unipi.it/frangio/).
+
+## Contents
+
+- Numerical Optimization
+    - Unconstrained Optimization
+        - Line Search Methods
+            - 1st Order Methods
+                - [x] Steepest Gradient Descent
+                - [x] Conjugate Gradient
+                    - [x] Fletcher–Reeves formula
+                    - [x] Polak–Ribière formula
+                    - [x] Hestenes-Stiefel formula
+                    - [x] Dai-Yuan formula
+            - 2nd Order Methods
+                - [x] Newton
+                - Quasi-Newton
+                    - [x] BFGS
+                    - [x] L-BFGS
+        - Stochastic Methods
+            - [x] Stochastic Gradient Descent
+                - [x] Momentum
+                    - [x] Polyak
+                    - [x] Nesterov
+            - [x] Adam
+                - [x] Momentum
+                    - [x] Polyak
+                    - [x] Nesterov
+            - [x] AMSGrad
+                - [x] Momentum
+                    - [x] Polyak
+                    - [x] Nesterov
+            - [x] AdaMax
+                - [x] Momentum
+                    - [x] Polyak
+                    - [x] Nesterov
+            - [x] AdaGrad
+            - [x] AdaDelta
+            - [x] RMSProp
+                - [x] Momentum
+                    - [x] Polyak
+                    - [x] Nesterov
+            - [x] Schedules
+                - Step size
+                    - [x] Decaying
+                    - [x] Linear Annealing
+                    - [x] Repeater
+                - Momentum
+                    - [x] Sutskever Blend
+        - [x] Proximal Bundle with [cvxpy](https://github.com/cvxgrp/cvxpy) interface to
+          [ecos](https://github.com/embotech/ecos), [osqp](https://github.com/oxfordcontrol/osqp),
+          [scs](https://github.com/cvxgrp/scs),
+          [etc](https://www.cvxpy.org/tutorial/advanced/index.html#choosing-a-solver).
+    - Constrained Quadratic Optimization
+        - Box-Constrained Quadratic Methods
+            - [x] Projected Gradient
+            - [x] Frank-Wolfe or Conditional Gradient
+            - [x] Active Set
+            - [x] Interior Point
+        - [x] Lagrangian Dual
+        - [x] Augmented Lagrangian Dual
+
+- Machine Learning
+    - [x] Support Vector Machines
+        - Formulations
+            - Primal
+            - Wolfe Dual
+            - Lagrangian Dual
+        - [x] Support Vector Classifier
+            - Losses
+                - [x] Hinge (L1 Loss) ![l1_svc_loss](notebooks/optimization/tex/img/l1_svc_loss.png)
+                - [x] Squared Hinge (L2 Loss) ![l2_svc_loss](notebooks/optimization/tex/img/l2_svc_loss.png)
+        - [x] Support Vector Regression
+            - Losses
+                - [x] Epsilon-insensitive (L1 Loss) ![l1_svr_loss](notebooks/optimization/tex/img/l1_svr_loss.png)
+                - [x] Squared Epsilon-insensitive (L2 Loss) ![l2_svr_loss](notebooks/optimization/tex/img/l2_svr_loss.png)
+        - Kernels
+            - [x] Linear
+
+                |                                                SVC                                                 |                                                SVR                                                 |
+                |:--------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------:|
+                | ![linear_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/linear_dual_l1_svc_hyperplane.png) | ![linear_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/linear_dual_l1_svr_hyperplane.png) |
+
+            - [x] Polynomial
+
+                |     SVC      | SVR          |
+                |    :----:    |    :----:    |
+                | ![poly_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/poly_dual_l1_svc_hyperplane.png) | ![poly_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/poly_dual_l1_svr_hyperplane.png) |              
+
+            - [x] Gaussian
+
+                |                                                  SVC                                                   |                                                  SVR                                                   |
+                |:------------------------------------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------:|
+                | ![gaussian_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/gaussian_dual_l1_svc_hyperplane.png) | ![gaussian_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/gaussian_dual_l1_svr_hyperplane.png) |
+
+            - [x] Laplacian
+
+                |                                                   SVC                                                    |                                                   SVR                                                    |
+                |:--------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------:|
+                | ![laplacian_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/laplacian_dual_l1_svc_hyperplane.png) | ![laplacian_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/laplacian_dual_l1_svr_hyperplane.png) |
+
+            - [x] Sigmoid
+        - Optimizers (ad hoc)
+            - [x] Sequential Minimal Optimization (SMO)
+            - [x] QP solver with [qpsolvers](https://github.com/stephane-caron/qpsolvers) interface to
+              [cvxopt](https://github.com/cvxopt/cvxopt), [quadprog](https://github.com/rmcgibbo/quadprog),
+              [qpOASES](https://github.com/coin-or/qpOASES), [etc](https://github.com/stephane-caron/qpsolvers#solvers).
+    - [x] Neural Networks
+        - [x] Neural Network Classifier
+        - [x] Neural Network Regressor
+        - Losses
+            - [x] Mean Absolute Error (L1 Loss)
+            - [x] Mean Squared Error (L2 Loss)
+            - [x] Binary Cross Entropy
+            - [x] Categorical Cross Entropy
+            - [x] Sparse Categorical Cross Entropy
+        - Regularizers
+            - [x] L1 or Lasso
+            - [x] L2 or Ridge or Tikhonov
+        - Activations
+            - [x] Linear
+            - [x] Sigmoid
+            - [x] Tanh
+            - [x] ReLU
+            - [x] SoftMax
+        - Layers
+            - [x] Fully Connected
+        - Initializers
+            - [x] Xavier or Glorot (normal and uniform)
+            - [x] He (normal and uniform)
+
+## Install
+
+```
+pip install optiml
+```
+
+## Documentation
+
+The full API reference is available at [dmeoli.github.io/optiml](https://dmeoli.github.io/optiml),
+automatically built from the source docstrings and published to GitHub Pages on every push to `master`.
+
+It can also be built locally with [Sphinx](https://www.sphinx-doc.org):
+
+```
+pip install -r docs/requirements.txt
+sphinx-build -b html docs docs/_build/html
+```
+
+## License [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+This software is released under the MIT License. See the [LICENSE](LICENSE) file for details.

{optiml-1.5 → optiml-1.7}/README.md

@@ -1,18 +1,21 @@
 # OptiML
-[![Build Status](https://travis-ci.com/dmeoli/optiml.svg?branch=master)](https://travis-ci.com/dmeoli/optiml) 
-[![Coverage Status](https://coveralls.io/repos/github/dmeoli/optiml/badge.svg?branch=master)](https://coveralls.io/github/dmeoli/optiml?branch=master) 
-[![Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9-blue)](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9-blue) 
+
+[![CI](https://github.com/dmeoli/optiml/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/dmeoli/optiml/actions/workflows/ci.yml)
+[![Coverage Status](https://coveralls.io/repos/github/dmeoli/optiml/badge.svg?branch=master)](https://coveralls.io/github/dmeoli/optiml?branch=master)
+[![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue)](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue)
 [![PyPI Version](https://img.shields.io/pypi/v/optiml.svg?color=blue)](https://pypi.org/project/optiml/)
 [![PyPI Downloads](https://img.shields.io/pypi/dm/optiml.svg)](https://pypistats.org/packages/optiml)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/dmeoli/optiml/master)
+[![Documentation](https://img.shields.io/badge/docs-online-blue)](https://dmeoli.github.io/optiml)
 
-OptiML is a sklearn compatible implementation of *Support Vector Machines* and *Deep Neural Networks*, 
-both with some of the most successful features according to the state of the art.
+OptiML is a sklearn compatible implementation of *Support Vector Machines* and *Deep Neural Networks*, both with some of
+the most successful features according to the state of the art.
 
-This work was motivated by the possibility of being able to solve the optimization problem deriving from the mathematical 
-formulation of these models through a wide range of optimization algorithms object of study and developed for the 
-Numerical Methods and Optimization course  @ [Department of Computer Science](https://www.di.unipi.it/en/) @ 
-[University of Pisa](https://www.unipi.it/index.php/english) under the supervision of prof. [Antonio Frangioni](http://pages.di.unipi.it/frangio/).
+This work was motivated by the possibility of being able to solve the optimization problem deriving from the
+mathematical formulation of these models through a wide range of optimization algorithms object of study and developed
+for the Numerical Methods and Optimization course @ [Department of Computer Science](https://www.di.unipi.it/en/) @
+[University of Pisa](https://www.unipi.it/index.php/english) under the supervision of
+prof. [Antonio Frangioni](http://pages.di.unipi.it/frangio/).
 
 ## Contents
 
@@ -30,7 +33,7 @@ Numerical Methods and Optimization course  @ [Department of Computer Science](ht
                 - [x] Newton
                 - Quasi-Newton
                     - [x] BFGS
-                    - [ ] L-BFGS
+                    - [x] L-BFGS
         - Stochastic Methods
             - [x] Stochastic Gradient Descent
                 - [x] Momentum
@@ -61,9 +64,10 @@ Numerical Methods and Optimization course  @ [Department of Computer Science](ht
                     - [x] Repeater
                 - Momentum
                     - [x] Sutskever Blend
-        - [x] Proximal Bundle with [cvxpy](https://github.com/cvxgrp/cvxpy) interface to 
-          [ecos](https://github.com/embotech/ecos), [osqp](https://github.com/oxfordcontrol/osqp), 
-          [scs](https://github.com/cvxgrp/scs), [etc](https://www.cvxpy.org/tutorial/advanced/index.html#choosing-a-solver).
+        - [x] Proximal Bundle with [cvxpy](https://github.com/cvxgrp/cvxpy) interface to
+          [ecos](https://github.com/embotech/ecos), [osqp](https://github.com/oxfordcontrol/osqp),
+          [scs](https://github.com/cvxgrp/scs),
+          [etc](https://www.cvxpy.org/tutorial/advanced/index.html#choosing-a-solver).
     - Constrained Quadratic Optimization
         - Box-Constrained Quadratic Methods
             - [x] Projected Gradient
@@ -89,35 +93,35 @@ Numerical Methods and Optimization course  @ [Department of Computer Science](ht
                 - [x] Squared Epsilon-insensitive (L2 Loss) ![l2_svr_loss](notebooks/optimization/tex/img/l2_svr_loss.png)
         - Kernels
             - [x] Linear
-                
-                | SVC          | SVR          |
-                |    :----:    |    :----:    |
+
+                |                                                SVC                                                 |                                                SVR                                                 |
+                |:--------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------:|
                 | ![linear_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/linear_dual_l1_svc_hyperplane.png) | ![linear_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/linear_dual_l1_svr_hyperplane.png) |
-                
+
             - [x] Polynomial
-                 
-                | SVC          | SVR          |
+
+                |     SVC      | SVR          |
                 |    :----:    |    :----:    |
                 | ![poly_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/poly_dual_l1_svc_hyperplane.png) | ![poly_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/poly_dual_l1_svr_hyperplane.png) |              
-                
+
             - [x] Gaussian
-                
-                | SVC          | SVR          |
-                |    :----:    |    :----:    |
+
+                |                                                  SVC                                                   |                                                  SVR                                                   |
+                |:------------------------------------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------:|
                 | ![gaussian_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/gaussian_dual_l1_svc_hyperplane.png) | ![gaussian_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/gaussian_dual_l1_svr_hyperplane.png) |
-                
+
             - [x] Laplacian
-              
-                | SVC          | SVR          |
-                |    :----:    |    :----:    |
+
+                |                                                   SVC                                                    |                                                   SVR                                                    |
+                |:--------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------:|
                 | ![laplacian_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/laplacian_dual_l1_svc_hyperplane.png) | ![laplacian_dual_l1_svc_hyperplane](notebooks/optimization/tex/img/laplacian_dual_l1_svr_hyperplane.png) |
-            
+
             - [x] Sigmoid
         - Optimizers (ad hoc)
             - [x] Sequential Minimal Optimization (SMO)
-            - [x] QP solver with [qpsolvers](https://github.com/stephane-caron/qpsolvers) interface to 
-            [cvxopt](https://github.com/cvxopt/cvxopt), [quadprog](https://github.com/rmcgibbo/quadprog), 
-            [qpOASES](https://github.com/coin-or/qpOASES), [etc](https://github.com/stephane-caron/qpsolvers#solvers).
+            - [x] QP solver with [qpsolvers](https://github.com/stephane-caron/qpsolvers) interface to
+              [cvxopt](https://github.com/cvxopt/cvxopt), [quadprog](https://github.com/rmcgibbo/quadprog),
+              [qpOASES](https://github.com/coin-or/qpOASES), [etc](https://github.com/stephane-caron/qpsolvers#solvers).
     - [x] Neural Networks
         - [x] Neural Network Classifier
         - [x] Neural Network Regressor
@@ -148,6 +152,18 @@ Numerical Methods and Optimization course  @ [Department of Computer Science](ht
 pip install optiml
 ```
 
+## Documentation
+
+The full API reference is available at [dmeoli.github.io/optiml](https://dmeoli.github.io/optiml),
+automatically built from the source docstrings and published to GitHub Pages on every push to `master`.
+
+It can also be built locally with [Sphinx](https://www.sphinx-doc.org):
+
+```
+pip install -r docs/requirements.txt
+sphinx-build -b html docs docs/_build/html
+```
+
 ## License [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 This software is released under the MIT License. See the [LICENSE](LICENSE) file for details.

{optiml-1.5 → optiml-1.7}/optiml/ml/neural_network/_base.py

@@ -18,6 +18,97 @@ from ...opti.unconstrained.stochastic import StochasticOptimizer, StochasticGrad
 
 
 class NeuralNetwork(BaseEstimator, Layer, ABC):
+    """
+    Base abstract class for all feed-forward neural network estimators.
+    It chains a sequence of layers, performs forward/backward propagation
+    and trains the network parameters by minimizing the given loss with
+    the chosen optimizer.
+
+    Parameters
+    ----------
+
+    layers : tuple of `Layer` instances, default=()
+        The ordered sequence of layers composing the network.
+
+    loss : `NeuralNetworkLoss` subclass, default=mean_squared_error
+        Specifies the loss function to minimize.
+
+    optimizer : `Optimizer` subclass, default=StochasticGradientDescent
+        The solver for optimization. It can be a subclass of the
+        `LineSearchOptimizer`, the `ProximalBundle` method or a subclass
+        of the `StochasticOptimizer`.
+
+    learning_rate : float, default=0.01
+        The initial learning rate used for weight update. It controls the
+        step-size in updating the weights. Only used when ``optimizer`` is a
+        subclass of `StochasticOptimizer`.
+
+    max_iter : int, default=1000
+        Maximum number of iterations. The solver iterates until convergence
+        (determined by ``tol``) or this number of iterations. If the optimizer
+        is a subclass of `StochasticOptimizer`, this value determines the number
+        of epochs, not the number of gradient steps.
+
+    momentum_type : {'none', 'polyak', 'nesterov'}, default='none'
+        Momentum type used for weight update. Only used when ``optimizer`` is
+        a subclass of `StochasticMomentumOptimizer`.
+
+    momentum : float, default=0.9
+        Momentum for weight update. Should be between 0 and 1. Only used when
+        ``optimizer`` is a subclass of `StochasticMomentumOptimizer`.
+
+    tol : float, default=1e-4
+        Tolerance for stopping criterion.
+
+    validation_split : float, default=0.
+        The proportion of training data to set aside as validation set for
+        early stopping. Must be between 0 and 1. Only used when ``optimizer``
+        is a subclass of `StochasticOptimizer`.
+
+    batch_size : int, default=None
+        Size of mini batches for stochastic optimizers.
+        Only used when ``optimizer`` is a subclass of `StochasticOptimizer`.
+
+    max_f_eval : int, default=15000
+        Maximum number of loss function calls. Only used when ``optimizer``
+        is a subclass of `LineSearchOptimizer`.
+
+    early_stopping : bool, default=False
+        Whether to use early stopping to terminate training when the
+        monitored score/loss does not improve by at least ``tol`` for
+        ``patience`` consecutive epochs.
+        Only used when ``optimizer`` is a subclass of `StochasticOptimizer`.
+
+    patience : int, default=5
+        Maximum number of epochs to not meet ``tol`` improvement.
+        Only used when ``optimizer`` is a subclass of `StochasticOptimizer`.
+
+    shuffle : bool, default=True
+        Whether to shuffle samples for batch sampling in each iteration. Only
+        used when the ``optimizer`` is a subclass of `StochasticOptimizer`.
+
+    random_state : int, RandomState instance or None, default=None
+        Controls the pseudo random number generation for the train-validation
+        split and for shuffling the data in batch sampling.
+        Pass an int for reproducible output across multiple function calls.
+
+    mu : float, default=1
+        Mu parameter for the proximal bundle method.
+        Only used when ``optimizer`` is `ProximalBundle`. Must be strictly positive.
+
+    master_solver : string, default='clarabel'
+        Master solver for the proximal bundle method for the CVXPY interface.
+        Only used when ``optimizer`` is `ProximalBundle`.
+
+    master_verbose : bool or int, default=False
+        Controls the verbosity of the CVXPY interface.
+        Only used when ``optimizer`` is `ProximalBundle`.
+
+    verbose : bool or int, default=False
+        Controls the verbosity of progress messages to stdout. Use a boolean value
+        to switch on/off or an int value to show progress each ``verbose`` time
+        optimization steps.
+    """
 
     def __init__(self,
                  layers=(),
@@ -36,7 +127,7 @@ class NeuralNetwork(BaseEstimator, Layer, ABC):
                  shuffle=True,
                  random_state=None,
                  mu=1,
-                 master_solver='ecos',
+                 master_solver='clarabel',
                  master_verbose=False,
                  verbose=False):
         self.layers = layers
@@ -277,6 +368,11 @@ class NeuralNetwork(BaseEstimator, Layer, ABC):
 
 
 class NeuralNetworkClassifier(ClassifierMixin, NeuralNetwork):
+    """
+    Feed-forward neural network for classification. The output layer must be
+    sigmoid (binary/multi-label) or softmax (multi-class), consistently with
+    the chosen loss function.
+    """
 
     def _store_train_val_info(self, opt, X_batch, y_batch, X_val, y_val):
         super(NeuralNetworkClassifier, self)._store_train_val_info(opt, X_batch, y_batch, X_val, y_val)
@@ -331,6 +427,11 @@ class NeuralNetworkClassifier(ClassifierMixin, NeuralNetwork):
 
 
 class NeuralNetworkRegressor(RegressorMixin, NeuralNetwork):
+    """
+    Feed-forward neural network for regression. The output layer must be
+    linear or, for regression between 0 and 1, sigmoid. The number of output
+    neurons must equal the number of targets.
+    """
 
     def _store_train_val_info(self, opt, X_batch, y_batch, X_val, y_val):
         super(NeuralNetworkRegressor, self)._store_train_val_info(opt, X_batch, y_batch, X_val, y_val)

{optiml-1.5 → optiml-1.7}/optiml/ml/neural_network/activations.py

@@ -5,6 +5,10 @@ from autograd.scipy.special import expit
 
 
 class Activation(ABC):
+    """
+    Base abstract class for all activation functions. Subclasses must
+    implement ``function`` and its element-wise derivative ``jacobian``.
+    """
 
     def function(self, x):
         raise NotImplementedError
@@ -17,6 +21,7 @@ class Activation(ABC):
 
 
 class Linear(Activation):
+    r"""Identity (linear) activation function :math:`f(x) = x`."""
 
     def function(self, x):
         return x
@@ -26,6 +31,7 @@ class Linear(Activation):
 
 
 class ReLU(Activation):
+    r"""Rectified linear unit activation function :math:`f(x) = \max(0, x)`."""
 
     def function(self, x):
         return np.maximum(0., x)
@@ -35,6 +41,7 @@ class ReLU(Activation):
 
 
 class Tanh(Activation):
+    r"""Hyperbolic tangent activation function :math:`f(x) = \tanh(x)`."""
 
     def function(self, x):
         return np.tanh(x)
@@ -44,6 +51,7 @@ class Tanh(Activation):
 
 
 class Sigmoid(Activation):
+    r"""Logistic sigmoid activation function :math:`f(x) = \frac{1}{1 + e^{-x}}`."""
 
     def function(self, x):
         return expit(x)
@@ -54,6 +62,7 @@ class Sigmoid(Activation):
 
 
 class SoftMax(Activation):
+    r"""Softmax activation function :math:`f(x)_i = \frac{e^{x_i}}{\sum_j e^{x_j}}`."""
 
     def function(self, x, axis=-1):
         exps = np.exp(x - np.max(x, axis=axis, keepdims=True))

optiml-1.7/optiml/ml/neural_network/initializers.py

@@ -0,0 +1,66 @@
+import numpy as np
+
+
+def truncated_normal(shape, mean=0., std=1., random_state=None):
+    truncated = 2 * std + mean
+    return np.clip(np.random.RandomState(random_state).normal(size=shape, loc=mean, scale=std), -truncated, truncated)
+
+
+def glorot_normal(shape, random_state=None):
+    r"""Glorot normal initializer, also called Xavier normal initializer.
+    It draws samples from a truncated normal distribution centered on 0
+    with
+
+    .. math::
+
+        \text{std} = \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}}
+
+    where ``fan_in`` is the number of input units in the weight tensor
+    and ``fan_out`` is the number of output units in the weight tensor."""
+    fan_in, fan_out = shape[0], shape[1]
+    std = np.sqrt(2. / (fan_in + fan_out))
+    return truncated_normal(shape=shape, mean=0., std=std, random_state=random_state)
+
+
+def glorot_uniform(shape, random_state=None):
+    r"""Glorot uniform initializer, also called Xavier uniform initializer.
+    It draws samples from a uniform distribution within
+    :math:`[-\text{limit}, \text{limit}]` where
+
+    .. math::
+
+        \text{limit} = \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}}
+
+    where ``fan_in`` is the number of input units in the weight tensor
+    and ``fan_out`` is the number of output units in the weight tensor."""
+    fan_in, fan_out = shape[0], shape[1]
+    limit = np.sqrt(6. / (fan_in + fan_out))
+    return np.random.RandomState(random_state).uniform(size=shape, low=-limit, high=limit)
+
+
+def he_normal(shape, random_state=None):
+    r"""He normal initializer. It draws samples from a truncated normal
+    distribution centered on 0 with
+
+    .. math::
+
+        \text{std} = \sqrt{\frac{2}{\text{fan\_in}}}
+
+    where ``fan_in`` is the number of input units in the weight tensor."""
+    fan_in, fan_out = shape[0], shape[1]
+    std = np.sqrt(2. / fan_in)
+    return truncated_normal(shape=shape, mean=0., std=std, random_state=random_state)
+
+
+def he_uniform(shape, random_state=None):
+    r"""He uniform variance scaling initializer. It draws samples from
+    a uniform distribution within :math:`[-\text{limit}, \text{limit}]` where
+
+    .. math::
+
+        \text{limit} = \sqrt{\frac{6}{\text{fan\_in}}}
+
+    where ``fan_in`` is the number of input units in the weight tensor."""
+    fan_in, fan_out = shape[0], shape[1]
+    limit = np.sqrt(6. / fan_in)
+    return np.random.RandomState(random_state).uniform(size=shape, low=-limit, high=limit)