mlpack 4.6.2__cp313-cp313-win_amd64.whl → 4.7.0__cp313-cp313-win_amd64.whl

mlpack/include/mlpack/methods/ann/layer/dropconnect_impl.hpp

@@ -22,35 +22,35 @@
 namespace mlpack {
 
 template<typename MatType>
-DropConnectType<MatType>::DropConnectType() :
+DropConnect<MatType>::DropConnect() :
     Layer<MatType>(),
     ratio(0.5),
     scale(2.0),
-    baseLayer(new LinearType<MatType>(0))
+    baseLayer(new Linear<MatType>(0))
 {
   // Nothing to do here.
 }
 
 template<typename MatType>
-DropConnectType<MatType>::DropConnectType(
+DropConnect<MatType>::DropConnect(
     const size_t outSize,
     const double ratio) :
     Layer<MatType>(),
     ratio(ratio),
     scale(1.0 / (1 - ratio)),
-    baseLayer(new LinearType<MatType>(outSize))
+    baseLayer(new Linear<MatType>(outSize))
 {
   // Nothing to do.
 }
 
 template<typename MatType>
-DropConnectType<MatType>::~DropConnectType()
+DropConnect<MatType>::~DropConnect()
 {
   delete baseLayer;
 }
 
 template<typename MatType>
-DropConnectType<MatType>::DropConnectType(const DropConnectType& other) :
+DropConnect<MatType>::DropConnect(const DropConnect& other) :
     Layer<MatType>(other),
     ratio(other.ratio),
     scale(other.scale),
@@ -60,7 +60,7 @@ DropConnectType<MatType>::DropConnectType(const DropConnectType& other) :
 }
 
 template<typename MatType>
-DropConnectType<MatType>::DropConnectType(DropConnectType&& other) :
+DropConnect<MatType>::DropConnect(DropConnect&& other) :
     Layer<MatType>(std::move(other)),
     ratio(std::move(other.ratio)),
     scale(std::move(other.scale)),
@@ -70,8 +70,8 @@ DropConnectType<MatType>::DropConnectType(DropConnectType&& other) :
 }
 
 template<typename MatType>
-DropConnectType<MatType>&
-DropConnectType<MatType>::operator=(const DropConnectType& other)
+DropConnect<MatType>&
+DropConnect<MatType>::operator=(const DropConnect& other)
 {
   if (&other != this)
   {
@@ -85,8 +85,8 @@ DropConnectType<MatType>::operator=(const DropConnectType& other)
 }
 
 template<typename MatType>
-DropConnectType<MatType>&
-DropConnectType<MatType>::operator=(DropConnectType&& other)
+DropConnect<MatType>&
+DropConnect<MatType>::operator=(DropConnect&& other)
 {
   if (&other != this)
   {
@@ -100,7 +100,7 @@ DropConnectType<MatType>::operator=(DropConnectType&& other)
 }
 
 template<typename MatType>
-void DropConnectType<MatType>::Forward(const MatType& input, MatType& output)
+void DropConnect<MatType>::Forward(const MatType& input, MatType& output)
 {
   // The DropConnect mask will not be multiplied in testing mode.
   if (!this->training)
@@ -114,18 +114,18 @@ void DropConnectType<MatType>::Forward(const MatType& input, MatType& output)
 
     // Scale with input / (1 - ratio) and set values to zero with
     // probability ratio.
-    mask.randu(denoise.n_rows, denoise.n_cols);
-    mask.transform([&](double val) { return (val > ratio); });
+    mask = conv_to<MatType>::from(
+        randu<MatType>(denoise.n_rows, denoise.n_cols) > ElemType(ratio));
 
     baseLayer->Parameters() = denoise % mask;
     baseLayer->Forward(input, output);
 
-    output = output * scale;
+    output = output * ElemType(scale);
   }
 }
 
 template<typename MatType>
-void DropConnectType<MatType>::Backward(
+void DropConnect<MatType>::Backward(
     const MatType& input,
     const MatType& output,
     const MatType& gy,
@@ -135,7 +135,7 @@ void DropConnectType<MatType>::Backward(
 }
 
 template<typename MatType>
-void DropConnectType<MatType>::Gradient(
+void DropConnect<MatType>::Gradient(
     const MatType& input,
     const MatType& error,
     MatType& gradient)
@@ -147,7 +147,7 @@ void DropConnectType<MatType>::Gradient(
 }
 
 template<typename MatType>
-void DropConnectType<MatType>::ComputeOutputDimensions()
+void DropConnect<MatType>::ComputeOutputDimensions()
 {
   // Propagate input dimensions to the base layer.
   baseLayer->InputDimensions() = this->inputDimensions;
@@ -155,14 +155,14 @@ void DropConnectType<MatType>::ComputeOutputDimensions()
 }
 
 template<typename MatType>
-void DropConnectType<MatType>::SetWeights(const MatType& weightsIn)
+void DropConnect<MatType>::SetWeights(const MatType& weightsIn)
 {
   baseLayer->SetWeights(weightsIn);
 }
 
 template<typename MatType>
 template<typename Archive>
-void DropConnectType<MatType>::serialize(
+void DropConnect<MatType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
   ar(cereal::base_class<Layer<MatType>>(this));

mlpack/include/mlpack/methods/ann/layer/dropout.hpp

@@ -44,30 +44,33 @@ namespace mlpack {
  *    computation.
  */
 template<typename MatType = arma::mat>
-class DropoutType : public Layer<MatType>
+class Dropout : public Layer<MatType>
 {
  public:
+  // Convenience typedef to access the element type of the weights and data.
+  using ElemType = typename MatType::elem_type;
+
   /**
    * Create the Dropout object using the specified ratio parameter.
    *
    * @param ratio The probability of setting a value to zero.
    */
-  DropoutType(const double ratio = 0.5);
+  Dropout(const double ratio = 0.5);
 
-  //! Clone the DropoutType object. This handles polymorphism correctly.
-  DropoutType* Clone() const { return new DropoutType(*this); }
+  // Clone the Dropout object. This handles polymorphism correctly.
+  Dropout* Clone() const { return new Dropout(*this); }
 
   // Virtual destructor.
-  virtual ~DropoutType() { }
+  virtual ~Dropout() { }
 
-  //! Copy the given DropoutType.
-  DropoutType(const DropoutType& other);
-  //! Take ownership of the given DropoutType.
-  DropoutType(DropoutType&& other);
-  //! Copy the given DropoutType.
-  DropoutType& operator=(const DropoutType& other);
-  //! Take ownership of the given DropoutType.
-  DropoutType& operator=(DropoutType&& other);
+  //! Copy the given Dropout.
+  Dropout(const Dropout& other);
+  //! Take ownership of the given Dropout.
+  Dropout(Dropout&& other);
+  //! Copy the given Dropout.
+  Dropout& operator=(const Dropout& other);
+  //! Take ownership of the given Dropout.
+  Dropout& operator=(Dropout&& other);
 
   /**
    * Ordinary feed forward pass of the dropout layer.
@@ -115,12 +118,7 @@ class DropoutType : public Layer<MatType>
 
   //! The scale fraction.
   double scale;
-}; // class DropoutType
-
-// Convenience typedefs.
-
-// Standard Dropout layer.
-using Dropout = DropoutType<arma::mat>;
+}; // class Dropout
 
 } // namespace mlpack
 

mlpack/include/mlpack/methods/ann/layer/dropout_impl.hpp

@@ -19,7 +19,7 @@
 namespace mlpack {
 
 template<typename MatType>
-DropoutType<MatType>::DropoutType(
+Dropout<MatType>::Dropout(
     const double ratio) :
     Layer<MatType>(),
     ratio(ratio),
@@ -29,7 +29,7 @@ DropoutType<MatType>::DropoutType(
 }
 
 template<typename MatType>
-DropoutType<MatType>::DropoutType(const DropoutType& other) :
+Dropout<MatType>::Dropout(const Dropout& other) :
     Layer<MatType>(other),
     ratio(other.ratio),
     scale(other.scale)
@@ -38,7 +38,7 @@ DropoutType<MatType>::DropoutType(const DropoutType& other) :
 }
 
 template<typename MatType>
-DropoutType<MatType>::DropoutType(DropoutType&& other) :
+Dropout<MatType>::Dropout(Dropout&& other) :
     Layer<MatType>(std::move(other)),
     ratio(std::move(other.ratio)),
     scale(std::move(other.scale))
@@ -47,8 +47,8 @@ DropoutType<MatType>::DropoutType(DropoutType&& other) :
 }
 
 template<typename MatType>
-DropoutType<MatType>&
-DropoutType<MatType>::operator=(const DropoutType& other)
+Dropout<MatType>&
+Dropout<MatType>::operator=(const Dropout& other)
 {
   if (&other != this)
   {
@@ -61,8 +61,8 @@ DropoutType<MatType>::operator=(const DropoutType& other)
 }
 
 template<typename MatType>
-DropoutType<MatType>&
-DropoutType<MatType>::operator=(DropoutType&& other)
+Dropout<MatType>&
+Dropout<MatType>::operator=(Dropout&& other)
 {
   if (&other != this)
   {
@@ -75,7 +75,7 @@ DropoutType<MatType>::operator=(DropoutType&& other)
 }
 
 template<typename MatType>
-void DropoutType<MatType>::Forward(const MatType& input, MatType& output)
+void Dropout<MatType>::Forward(const MatType& input, MatType& output)
 {
   // The dropout mask will not be multiplied in testing mode.
   if (!this->training)
@@ -86,32 +86,25 @@ void DropoutType<MatType>::Forward(const MatType& input, MatType& output)
   {
     // Scale with input / (1 - ratio) and set values to zero with probability
     // 'ratio'.
-    mask.randu(input.n_rows, input.n_cols);
-    #pragma omp parallel for collapse(2)
-    for (size_t i = 0; i < input.n_rows; ++i)
-    {
-      for (size_t j = 0; j < input.n_cols; ++j)
-      {
-        mask(i, j) = (mask(i, j) > this->ratio) ? 1.0 : 0.0;
-      }
-    }
-    output = input % mask * this->scale;
+    mask = conv_to<MatType>::from(
+        randu<MatType>(input.n_rows, input.n_cols) > ElemType(ratio));
+    output = input % mask * ElemType(scale);
   }
 }
 
 template<typename MatType>
-void DropoutType<MatType>::Backward(
+void Dropout<MatType>::Backward(
     const MatType& /* input */,
     const MatType& /* output */,
     const MatType& gy,
     MatType& g)
 {
-  g = gy % mask * scale;
+  g = gy % mask * ElemType(scale);
 }
 
 template<typename MatType>
 template<typename Archive>
-void DropoutType<MatType>::serialize(
+void Dropout<MatType>::serialize(
     Archive& ar,
     const uint32_t /* version */)
 {

mlpack/include/mlpack/methods/ann/layer/elu.hpp

@@ -104,15 +104,18 @@ namespace mlpack {
  *     type to differ from the input type (Default: arma::mat).
  */
 template <typename MatType = arma::mat>
-class ELUType : public Layer<MatType>
+class ELU : public Layer<MatType>
 {
  public:
+  // Convenience typedef to access the element type of the weights and data.
+  using ElemType = typename MatType::elem_type;
+
   /**
    * Create the ELU object.
    *
    * NOTE: Use this constructor for SELU activation function.
    */
-  ELUType();
+  ELU();
 
   /**
    * Create the ELU object using the specified parameter. The non zero
@@ -122,25 +125,25 @@ class ELUType : public Layer<MatType>
    * @note Use this constructor for ELU activation function.
    * @param alpha Scale parameter for the negative factor.
    */
-  ELUType(const double alpha);
+  ELU(const double alpha);
 
-  //! Clone the ELUType object. This handles polymorphism correctly.
-  ELUType* Clone() const { return new ELUType(*this); }
+  //! Clone the ELU object. This handles polymorphism correctly.
+  ELU* Clone() const { return new ELU(*this); }
 
   // Virtual destructor.
-  virtual ~ELUType() { }
+  virtual ~ELU() { }
 
   // Copy constructor.
-  ELUType(const ELUType& other);
+  ELU(const ELU& other);
 
   // Move Constructor.
-  ELUType(ELUType&& other);
+  ELU(ELU&& other);
 
   // Copy assignment operator.
-  ELUType& operator=(const ELUType& other);
+  ELU& operator=(const ELU& other);
 
   // Move assignement operator.
-  ELUType& operator=(ELUType&& other);
+  ELU& operator=(ELU&& other);
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -181,27 +184,22 @@ class ELUType : public Layer<MatType>
   void serialize(Archive& ar, const uint32_t /* version */);
 
  private:
-  //! Locally stored first derivative of the activation function.
-  MatType derivative;
-
-  //! ELU Hyperparameter (0 < alpha)
-  //! SELU parameter fixed to 1.6732632423543774 for normalized inputs.
+  // ELU Hyperparameter (0 < alpha)
+  // SELU parameter fixed to 1.6732632423543774 for normalized inputs.
   double alpha;
 
-  //! Lambda parameter used for multiplication of ELU function.
-  //! For ELU activation function, lambda = 1.
-  //! For SELU activation function, lambda = 1.0507009873554802 for normalized
-  //! inputs.
+  // Lambda parameter used for multiplication of ELU function.
+  // For ELU activation function, lambda = 1.
+  // For SELU activation function, lambda = 1.0507009873554802 for normalized
+  // inputs.
   double lambda;
-}; // class ELUType
+}; // class ELU
 
 // Convenience typedefs.
 
-// ELU layer.
-using ELU = ELUType<arma::mat>;
-
-// SELU layer.
-using SELU = ELUType<arma::mat>;
+// SELU layer (implemented as a class for template deduction).
+template<typename MatType = arma::mat>
+class SELU : public ELU<MatType> { };
 
 } // namespace mlpack
 

mlpack/include/mlpack/methods/ann/layer/elu_impl.hpp

@@ -26,7 +26,7 @@ namespace mlpack {
 // This constructor is called for SELU activation function.  The values of
 // alpha and lambda are constant for normalized inputs.
 template<typename MatType>
-ELUType<MatType>::ELUType() :
+ELU<MatType>::ELU() :
     Layer<MatType>(),
     alpha(1.6732632423543774),
     lambda(1.0507009873554802)
@@ -37,7 +37,7 @@ ELUType<MatType>::ELUType() :
 // This constructor is called for ELU activation function. The value of lambda
 // is fixed and equal to 1. 'alpha' is a hyperparameter.
 template<typename MatType>
-ELUType<MatType>::ELUType(const double alpha) :
+ELU<MatType>::ELU(const double alpha) :
     Layer<MatType>(),
     alpha(alpha),
     lambda(1)
@@ -46,7 +46,7 @@ ELUType<MatType>::ELUType(const double alpha) :
 }
 
 template<typename MatType>
-ELUType<MatType>::ELUType(const ELUType& other) :
+ELU<MatType>::ELU(const ELU& other) :
     Layer<MatType>(other),
     alpha(other.alpha),
     lambda(other.lambda)
@@ -55,8 +55,8 @@ ELUType<MatType>::ELUType(const ELUType& other) :
 }
 
 template<typename MatType>
-ELUType<MatType>::ELUType(
-    ELUType&& other) :
+ELU<MatType>::ELU(
+    ELU&& other) :
     Layer<MatType>(std::move(other)),
     alpha(std::move(other.alpha)),
     lambda(std::move(other.lambda))
@@ -65,8 +65,8 @@ ELUType<MatType>::ELUType(
 }
 
 template<typename MatType>
-ELUType<MatType>&
-ELUType<MatType>::operator=(const ELUType& other)
+ELU<MatType>&
+ELU<MatType>::operator=(const ELU& other)
 {
   if (&other != this)
   {
@@ -79,8 +79,8 @@ ELUType<MatType>::operator=(const ELUType& other)
 }
 
 template<typename MatType>
-ELUType<MatType>&
-ELUType<MatType>::operator=(ELUType&& other)
+ELU<MatType>&
+ELU<MatType>::operator=(ELU&& other)
 {
   if (&other != this)
   {
@@ -93,39 +93,32 @@ ELUType<MatType>::operator=(ELUType&& other)
 }
 
 template<typename MatType>
-void ELUType<MatType>::Forward(
+void ELU<MatType>::Forward(
     const MatType& input, MatType& output)
 {
   for (size_t i = 0; i < input.n_elem; ++i)
   {
-    if (input(i) < DBL_MAX)
-    {
-      output(i) = (input(i) > 0) ? lambda * input(i) : lambda * alpha *
-          (std::exp(input(i)) - 1);
-    }
-  }
-
-  if (this->training)
-  {
-    derivative.set_size(arma::size(input));
-    for (size_t i = 0; i < input.n_elem; ++i)
-      derivative(i) = (input(i) > 0) ? lambda : output(i) + lambda * alpha;
+    output(i) = (input(i) >= 0) ? ElemType(lambda) * input(i) :
+        ElemType(lambda) * ElemType(alpha) * (std::exp(input(i)) - 1);
   }
 }
 
 template<typename MatType>
-void ELUType<MatType>::Backward(
-    const MatType& /* input */,
-    const MatType& /* output */,
+void ELU<MatType>::Backward(
+    const MatType& input,
+    const MatType& output,
     const MatType& gy,
     MatType& g)
 {
-  g = gy % derivative;
+  for (size_t i = 0; i < input.n_elem; ++i)
+  {
+    g(i) = gy(i) * ((input(i) >= 0) ? lambda : output(i) + lambda * alpha);
+  }
 }
 
 template<typename MatType>
 template<typename Archive>
-void ELUType<MatType>::serialize(
+void ELU<MatType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
   ar(cereal::base_class<Layer<MatType>>(this));

mlpack/include/mlpack/methods/ann/layer/embedding.hpp

@@ -0,0 +1,160 @@
+/**
+ * @file methods/ann/layer/embedding.hpp
+ * @author Marcus Edel
+ * @author Kumar Utkarsh
+ * @author Ryan Curtin
+ *
+ * Definition of the Embedding (embedding) layer class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_EMBEDDING_HPP
+#define MLPACK_METHODS_ANN_LAYER_EMBEDDING_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/methods/ann/regularizer/no_regularizer.hpp>
+
+#include "layer.hpp"
+
+namespace mlpack {
+
+/**
+ * The Embedding class stores word embeddings and retrieves them using tokens.
+ * It must always be the first layer of any network it is used in.
+ *
+ * The input to the Embedding class is a matrix of shape (sequenceLength,
+ * batchSize). The matrix consists of tokens which are used to lookup the table
+ * (i.e. weights) to find the embeddings of those tokens.
+ *
+ * The input shape : (sequenceLength, batchSize).
+ * The output shape : (sequenceLength * embeddingSize, batchSize).
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template<
+    typename MatType = arma::mat,
+    typename RegularizerType = NoRegularizer
+>
+class Embedding : public Layer<MatType>
+{
+ public:
+  using CubeType = typename GetCubeType<MatType>::type;
+
+  // Create an Embedding layer.
+  Embedding();
+
+  /**
+   * Create the Embedding layer object with the specified number of output
+   * dimensions.  The vocabulary size (number of possible inputs) and the
+   * embedding size (dimensionality of the output) must be given.
+   *
+   * @param vocabSize The size of the input vocabulary (number of different
+   *     possible input values).
+   * @param embeddingSize Number of dimensions to use for the embedding.
+   * @param regularizer The regularizer to use, optional (default: no
+   *     regularizer).
+   */
+  Embedding(const size_t vocabSize,
+            const size_t embeddingSize,
+            RegularizerType regularizer = RegularizerType());
+
+  virtual ~Embedding() { }
+
+  // Clone the Embedding object. This handles polymorphism correctly.
+  Embedding* Clone() const { return new Embedding(*this); }
+
+  // Copy the other Embedding layer (but not weights).
+  Embedding(const Embedding& layer);
+
+  // Take ownership of the members of the other Embedding layer (but not
+  // weights).
+  Embedding(Embedding&& layer);
+
+  // Copy the other Embedding layer (but not weights).
+  Embedding& operator=(const Embedding& layer);
+
+  // Take ownership of the members of the other Embedding layer (but not
+  // weights).
+  Embedding& operator=(Embedding&& layer);
+
+  /**
+   * Reset the layer parameter (weights and bias). The method is called to
+   * assign the allocated memory to the internal learnable parameters.
+   */
+  void SetWeights(const MatType& weightsIn);
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  void Forward(const MatType& input, MatType& output);
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards trough f. Using the results from the feed
+   * forward pass.
+   *
+   * @param * (input) The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  void Backward(const MatType& /* input */,
+                const MatType& /* output */,
+                const MatType& gy,
+                MatType& g);
+
+  /**
+   * Calculate the gradient using the output delta and the input activation.
+   *
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
+   */
+  void Gradient(const MatType& input,
+                const MatType& error,
+                MatType& gradient);
+
+  // Get the parameters.
+  const MatType& Parameters() const { return weights; }
+  // Modify the parameters.
+  MatType& Parameters() { return weights; }
+
+  // Get the size of the weights.
+  size_t WeightSize() const { return vocabSize * embeddingSize; }
+
+  // Compute the output dimensions of the layer given `InputDimensions()`.
+  void ComputeOutputDimensions();
+
+  // Serialize the layer.
+  template<typename Archive>
+  void serialize(Archive& ar, const uint32_t /* version */);
+
+ private:
+  // Locally-stored size of the vocabulary.
+  size_t vocabSize;
+
+  // Locally-stored length of each embedding vector.
+  size_t embeddingSize;
+
+  // Locally-stored weight object.
+  MatType weights;
+
+  // Locally-stored regularizer object.
+  RegularizerType regularizer;
+}; // class Embedding
+
+} // namespace mlpack
+
+// Include implementation.
+#include "embedding_impl.hpp"
+
+#endif