mlpack 4.6.2__cp38-cp38-win_amd64.whl → 4.7.0__cp38-cp38-win_amd64.whl

mlpack/include/mlpack/methods/ann/layer/mean_pooling_impl.hpp

@@ -20,14 +20,14 @@
 namespace mlpack {
 
 template<typename MatType>
-MeanPoolingType<MatType>::MeanPoolingType() :
+MeanPooling<MatType>::MeanPooling() :
     Layer<MatType>()
 {
   // Nothing to do here.
 }
 
 template<typename MatType>
-MeanPoolingType<MatType>::MeanPoolingType(
+MeanPooling<MatType>::MeanPooling(
     const size_t kernelWidth,
     const size_t kernelHeight,
     const size_t strideWidth,
@@ -45,8 +45,8 @@ MeanPoolingType<MatType>::MeanPoolingType(
 }
 
 template<typename MatType>
-MeanPoolingType<MatType>::MeanPoolingType(
-    const MeanPoolingType& other) :
+MeanPooling<MatType>::MeanPooling(
+    const MeanPooling& other) :
     Layer<MatType>(other),
     kernelWidth(other.kernelWidth),
     kernelHeight(other.kernelHeight),
@@ -59,8 +59,8 @@ MeanPoolingType<MatType>::MeanPoolingType(
 }
 
 template<typename MatType>
-MeanPoolingType<MatType>::MeanPoolingType(
-    MeanPoolingType&& other) :
+MeanPooling<MatType>::MeanPooling(
+    MeanPooling&& other) :
     Layer<MatType>(std::move(other)),
     kernelWidth(std::move(other.kernelWidth)),
     kernelHeight(std::move(other.kernelHeight)),
@@ -73,8 +73,8 @@ MeanPoolingType<MatType>::MeanPoolingType(
 }
 
 template<typename MatType>
-MeanPoolingType<MatType>&
-MeanPoolingType<MatType>::operator=(const MeanPoolingType& other)
+MeanPooling<MatType>&
+MeanPooling<MatType>::operator=(const MeanPooling& other)
 {
   if (&other != this)
   {
@@ -91,8 +91,8 @@ MeanPoolingType<MatType>::operator=(const MeanPoolingType& other)
 }
 
 template<typename MatType>
-MeanPoolingType<MatType>&
-MeanPoolingType<MatType>::operator=(MeanPoolingType&& other)
+MeanPooling<MatType>&
+MeanPooling<MatType>::operator=(MeanPooling&& other)
 {
   if (&other != this)
   {
@@ -109,7 +109,7 @@ MeanPoolingType<MatType>::operator=(MeanPoolingType&& other)
 }
 
 template<typename MatType>
-void MeanPoolingType<MatType>::Forward(
+void MeanPooling<MatType>::Forward(
     const MatType& input, MatType& output)
 {
   // Create Alias of input as 2D image as input is 1D vector.
@@ -127,7 +127,7 @@ void MeanPoolingType<MatType>::Forward(
 }
 
 template<typename MatType>
-void MeanPoolingType<MatType>::Backward(
+void MeanPooling<MatType>::Backward(
   const MatType& input,
   const MatType& /* output */,
   const MatType& gy,
@@ -154,7 +154,7 @@ void MeanPoolingType<MatType>::Backward(
 }
 
 template<typename MatType>
-void MeanPoolingType<MatType>::ComputeOutputDimensions()
+void MeanPooling<MatType>::ComputeOutputDimensions()
 {
   this->outputDimensions = this->inputDimensions;
 
@@ -184,7 +184,7 @@ void MeanPoolingType<MatType>::ComputeOutputDimensions()
 
 template<typename MatType>
 template<typename Archive>
-void MeanPoolingType<MatType>::serialize(
+void MeanPooling<MatType>::serialize(
     Archive& ar,
     const uint32_t /* version */)
 {
@@ -199,7 +199,7 @@ void MeanPoolingType<MatType>::serialize(
 }
 
 template<typename MatType>
-void MeanPoolingType<MatType>::PoolingOperation(
+void MeanPooling<MatType>::PoolingOperation(
     const CubeType& input,
     CubeType& output)
 {
@@ -242,7 +242,7 @@ void MeanPoolingType<MatType>::PoolingOperation(
 }
 
 template<typename MatType>
-void MeanPoolingType<MatType>::Unpooling(
+void MeanPooling<MatType>::Unpooling(
     const MatType& error,
     MatType& output)
 {

mlpack/include/mlpack/methods/ann/layer/multi_layer.hpp

@@ -78,10 +78,10 @@ class MultiLayer : public Layer<MatType>
    * @param start Index of first layer to pass data through.
    * @param end Index of last layer to pass data through.
    */
-  void Forward(const MatType& input,
-               MatType& output,
-               const size_t start,
-               const size_t end);
+  void PartialForward(const MatType& input,
+                      MatType& output,
+                      const size_t start,
+                      const size_t end);
 
   /**
    * Perform a backward pass with the given data.  `gy` is expected to be the
@@ -164,9 +164,24 @@ class MultiLayer : public Layer<MatType>
    * @param args The layer parameter.
    */
   template <typename LayerType, typename... Args>
-  void Add(Args... args)
+  void Add(Args&&... args)
   {
-    network.push_back(new LayerType(args...));
+    network.push_back(new LayerType(std::forward<Args>(args)...));
+    layerOutputs.push_back(MatType());
+    layerDeltas.push_back(MatType());
+    layerGradients.push_back(MatType());
+  }
+
+  /**
+   * Add a new module to the model, using the MatType of this layer.
+   *
+   * @param args The layer parameter.
+   */
+  template<template<typename...> typename LayerType,
+           typename... Args>
+  void Add(Args&&... args)
+  {
+    network.push_back(new LayerType<MatType>(std::forward<Args>(args)...));
     layerOutputs.push_back(MatType());
     layerDeltas.push_back(MatType());
     layerGradients.push_back(MatType());
@@ -177,6 +192,7 @@ class MultiLayer : public Layer<MatType>
    *
    * @param layer The Layer to be added to the model.
    */
+  [[deprecated("Will be removed in mlpack 5.0.0.  Pass a reference instead.")]]
   void Add(Layer<MatType>* layer)
   {
     network.push_back(layer);
@@ -185,6 +201,20 @@ class MultiLayer : public Layer<MatType>
     layerGradients.push_back(MatType());
   }
 
+  template<typename LayerType>
+  void Add(LayerType&& layer,
+           typename std::enable_if_t<
+               !std::is_pointer_v<std::remove_reference_t<LayerType>>>* = 0)
+  {
+    using NewLayerType =
+        typename std::remove_cv_t<std::remove_reference_t<LayerType>>;
+
+    network.push_back(new NewLayerType(std::forward<LayerType>(layer)));
+    layerOutputs.push_back(MatType());
+    layerDeltas.push_back(MatType());
+    layerGradients.push_back(MatType());
+  }
+
   //! Get the network (series of layers) held by this MultiLayer.
   const std::vector<Layer<MatType>*>& Network() const
   {

mlpack/include/mlpack/methods/ann/layer/multi_layer_impl.hpp

@@ -81,6 +81,8 @@ MultiLayer<MatType>& MultiLayer<MatType>::operator=(const MultiLayer& other)
   {
     Layer<MatType>::operator=(other);
 
+    for (size_t i = 0; i < network.size(); ++i)
+      delete network[i];
     network.clear();
     layerOutputs.clear();
     layerDeltas.clear();
@@ -120,6 +122,8 @@ MultiLayer<MatType>& MultiLayer<MatType>::operator=(MultiLayer&& other)
     totalInputSize = std::move(other.totalInputSize);
     totalOutputSize = std::move(other.totalOutputSize);
 
+    for (size_t i = 0; i < network.size(); ++i)
+      delete network[i];
     network = std::move(other.network);
 
     layerOutputs.resize(network.size(), MatType());
@@ -138,11 +142,11 @@ template<typename MatType>
 void MultiLayer<MatType>::Forward(
     const MatType& input, MatType& output)
 {
-  Forward(input, output, 0, network.size() - 1);
+  PartialForward(input, output, 0, network.size() - 1);
 }
 
 template<typename MatType>
-void MultiLayer<MatType>::Forward(
+void MultiLayer<MatType>::PartialForward(
     const MatType& input,
     MatType& output,
     const size_t start,

mlpack/include/mlpack/methods/ann/layer/multihead_attention.hpp

@@ -66,14 +66,17 @@ template <
     typename MatType = arma::mat,
     typename RegularizerType = NoRegularizer
 >
-class MultiheadAttentionType : public Layer<MatType>
+class MultiheadAttention : public Layer<MatType>
 {
  public:
+  // Convenience typedefs.
+  using ElemType = typename MatType::elem_type;
   using CubeType = typename GetCubeType<MatType>::type;
+
   /**
    * Default constructor.
    */
-  MultiheadAttentionType();
+  MultiheadAttention();
 
   /**
    * Create the MultiheadAttention object using the specified modules.
@@ -87,17 +90,17 @@ class MultiheadAttentionType : public Layer<MatType>
    * @param selfAttention Use self-attention; source key, query, and value all
    *     come from the same inputs
    */
-  MultiheadAttentionType(const size_t tgtSeqLen,
-                         const size_t numHeads,
-                         const MatType& attnMask = MatType(),
-                         const MatType& keyPaddingMask = MatType(),
-                         const bool selfAttention = false);
+  MultiheadAttention(const size_t tgtSeqLen,
+                     const size_t numHeads,
+                     const CubeType& attnMask = CubeType(),
+                     const MatType& keyPaddingMask = MatType(),
+                     const bool selfAttention = false);
 
-  //! Clone the MultiheadAttentionType object. This handles polymorphism
+  //! Clone the MultiheadAttention object. This handles polymorphism
   //! correctly.
-  MultiheadAttentionType* Clone() const override
+  MultiheadAttention* Clone() const override
   {
-    return new MultiheadAttentionType(*this);
+    return new MultiheadAttention(*this);
   }
 
   /**
@@ -175,9 +178,9 @@ class MultiheadAttentionType : public Layer<MatType>
   size_t& NumHeads() { return numHeads; }
 
   //! Get the two dimensional Attention Mask.  Contains values 0 or 1.
-  MatType const& AttentionMask() const { return attnMask; }
+  CubeType const& AttentionMask() const { return attnMask; }
   //! Modify the two dimensional Attention Mask.  Should take values 0 or 1.
-  MatType& AttentionMask() { return attnMask; }
+  CubeType& AttentionMask() { return attnMask; }
 
   //! Get Key Padding Mask.  Contains values 0 or 1.
   MatType const& KeyPaddingMask() const { return keyPaddingMask; }
@@ -265,8 +268,11 @@ class MultiheadAttentionType : public Layer<MatType>
   }
 
  private:
-  //! Element Type of the output.
-  using ElemType = typename MatType::elem_type;
+  static void MaskedForwardSoftmax(CubeType& scores,
+                                   const size_t numHeads,
+                                   const size_t batchSize,
+                                   const CubeType& attnMask,
+                                   const MatType& keyPaddingMask);
 
   //! Target sequence length.
   size_t tgtSeqLen;
@@ -283,11 +289,12 @@ class MultiheadAttentionType : public Layer<MatType>
   //! Dimensionality of each head.
   size_t headDim;
 
-  //! Two dimensional Attention Mask of shape (tgtSeqLen, srcSeqLen).  Takes
-  //! the values [-Inf, 0]
-  MatType attnMask;
+  //! Two dimensional Attention Mask of shape (srcSeqLen, tgtSeqLen, batchSize).
+  //! Takes the values [-Inf, 0]
+  CubeType attnMask;
 
-  //! Key Padding Mask.  Takes the values [-Inf, 0]
+  //! Key Padding Mask.  The shape of keyPaddingMask : (srcSeqLen, batchSize)
+  //! the values [-Inf, 0]
   MatType keyPaddingMask;
 
   //! Whether or not self-attention is used (source key, value, and query all
@@ -337,15 +344,12 @@ class MultiheadAttentionType : public Layer<MatType>
   CubeType attnOut;
 
   //! Softmax layer to represent the probabilities of next sequence.
-  SoftmaxType<MatType> softmax;
+  Softmax<MatType> softmax;
 
   //! Locally-stored regularizer object.
   RegularizerType regularizer;
 }; // class MultiheadAttention
 
-// Standard MultiheadAttention layer using no regularization.
-using MultiheadAttention = MultiheadAttentionType<arma::mat, NoRegularizer>;
-
 } // namespace mlpack
 
 // Include implementation.