mlpack 4.6.2__cp313-cp313-win_amd64.whl → 4.7.0__cp313-cp313-win_amd64.whl

mlpack/include/mlpack/core/data/load_impl.hpp

@@ -17,171 +17,48 @@
 // In case it hasn't already been included.
 #include "load.hpp"
 
-#include <algorithm>
-#include <exception>
-
-#include "extension.hpp"
-#include "string_algorithms.hpp"
-
 namespace mlpack {
-namespace data {
-
-// The following functions are kept for backward compatibility,
-// Please remove them when we release mlpack 5.
-template<typename eT>
-bool Load(const std::string& filename,
-          arma::Mat<eT>& matrix,
-          const bool fatal,
-          const bool transpose,
-          const FileType inputLoadType)
-{
-  MatrixOptions opts;
-  opts.Fatal() = fatal;
-  opts.NoTranspose() = !transpose;
-  opts.Format() = inputLoadType;
-
-  return Load(filename, matrix, opts);
-}
-
-// For loading data into sparse matrix
-template <typename eT>
-bool Load(const std::string& filename,
-          arma::SpMat<eT>& matrix,
-          const bool fatal,
-          const bool transpose,
-          const FileType inputLoadType)
-{
-  MatrixOptions opts;
-  opts.Fatal() = fatal;
-  opts.NoTranspose() = !transpose;
-  opts.Format() = inputLoadType;
-
-  return Load(filename, matrix, opts);
-}
-
-// For loading data into a column vector
-template <typename eT>
-bool Load(const std::string& filename,
-          arma::Col<eT>& vec,
-          const bool fatal)
-{
-  DataOptions opts;
-  opts.Fatal() = fatal;
-  return Load(filename, vec, opts);
-}
-
-// For loading data into a raw vector
-template <typename eT>
-bool Load(const std::string& filename,
-          arma::Row<eT>& rowvec,
-          const bool fatal)
-{
-  DataOptions opts;
-  opts.Fatal() = fatal;
-  return Load(filename, rowvec, opts);
-}
-
-// Load with mappings.  Unfortunately we have to implement this ourselves.
-template<typename eT, typename PolicyType>
-bool Load(const std::string& filename,
-          arma::Mat<eT>& matrix,
-          DatasetMapper<PolicyType>& info,
-          const bool fatal,
-          const bool transpose)
-{
-  TextOptions opts;
-  opts.Fatal() = fatal;
-  opts.NoTranspose() = !transpose;
-  opts.Categorical() = true;
-
-  if constexpr (std::is_same_v<PolicyType, data::IncrementPolicy>)
-  {
-    opts.DatasetInfo() = info;
-  }
-  else if constexpr (std::is_same_v<PolicyType, data::MissingPolicy>)
-  {
-    opts.MissingPolicy() = true;
-    opts.DatasetMissingPolicy() = info;
-  }
-
-  bool success = Load(filename, matrix, opts);
-
-  if constexpr (std::is_same_v<PolicyType, data::IncrementPolicy>)
-  {
-    info = opts.DatasetInfo();
-  }
-  else if constexpr (std::is_same_v<PolicyType, data::MissingPolicy>)
-  {
-    info = opts.DatasetMissingPolicy();
-  }
-
-  return success;
-}
 
 template<typename MatType, typename DataOptionsType>
 bool Load(const std::string& filename,
           MatType& matrix,
           const DataOptionsType& opts,
-          std::enable_if_t<IsArma<MatType>::value ||
-              IsSparseMat<MatType>::value>*,
-          std::enable_if_t<!std::is_same_v<DataOptionsType, bool>>*)
+          const typename std::enable_if_t<
+              IsDataOptions<DataOptionsType>::value>*)
 {
   DataOptionsType tmpOpts(opts);
-  return Load(filename, matrix, tmpOpts);
+  return Load(filename, matrix, tmpOpts, false);
 }
 
-template<typename MatType>
-bool LoadMatrix(const std::string& filename,
-                MatType& matrix,
-                std::fstream& stream,
-                TextOptions& txtOpts)
+template<typename eT, typename DataOptionsType>
+bool Load(const std::vector<std::string>& files,
+          arma::Mat<eT>& matrix,
+          const DataOptionsType& opts,
+          const typename std::enable_if_t<
+              IsDataOptions<DataOptionsType>::value>*)
 {
-  bool success = false;
-  if constexpr (IsSparseMat<MatType>::value)
-  {
-    success = LoadSparse(filename, matrix, txtOpts, stream);
-  }
-  else if (txtOpts.Categorical() ||
-      (txtOpts.Format() == FileType::ARFFASCII))
-  {
-    success = LoadCategorical(filename, matrix, txtOpts);
-  }
-  else if constexpr (IsCol<MatType>::value)
-  {
-    success = LoadCol(filename, matrix, txtOpts, stream);
-  }
-  else if constexpr (IsRow<MatType>::value)
-  {
-    success = LoadRow(filename, matrix, txtOpts, stream);
-  }
-  else if constexpr (IsDense<MatType>::value)
-  {
-    success = LoadDense(filename, matrix, txtOpts, stream);
-  }
-  else
-  {
-    if (txtOpts.Fatal())
-      Log::Fatal << "data::Load(): unknown matrix-like type given!"
-          << std::endl;
-    else
-      Log::Warn << "data::Load(): unknown matrix-like type given!"
-          << std::endl;
-
-    return false;
-  }
-  return success;
+  DataOptionsType tmpOpts(opts);
+  return Load(files, matrix, tmpOpts, false);
 }
 
-template<typename MatType, typename DataOptionsType>
+template<typename ObjectType, typename DataOptionsType>
 bool Load(const std::string& filename,
-          MatType& matrix,
+          ObjectType& matrix,
           DataOptionsType& opts,
-          std::enable_if_t<IsArma<MatType>::value ||
-              IsSparseMat<MatType>::value>*,
-          std::enable_if_t<!std::is_same_v<DataOptionsType, bool>>*)
+          const bool copyBack,
+          const typename std::enable_if_t<
+              IsDataOptions<DataOptionsType>::value>*)
 {
   Timer::Start("loading_data");
 
+  static_assert(!IsArma<ObjectType>::value || !IsSparseMat<ObjectType>::value
+      || !HasSerialize<ObjectType>::value, "mlpack can load Armadillo"
+      " matrices or serialized mlpack models only; please use a known type.");
+  const bool isMatrixType = IsArma<ObjectType>::value ||
+      IsSparseMat<ObjectType>::value;
+  const bool isSerializable = HasSerialize<ObjectType>::value;
+  const bool isSparseMatrixType = IsSparseMat<ObjectType>::value;
+
   std::fstream stream;
   bool success = OpenFile(filename, opts, true, stream);
   if (!success)
@@ -190,43 +67,65 @@ bool Load(const std::string& filename,
     return false;
   }
 
-  success = DetectFileType<MatType>(filename, opts, true, &stream);
+  success = DetectFileType<ObjectType>(filename, opts, true, &stream);
   if (!success)
   {
     Timer::Stop("loading_data");
     return false;
   }
+  const bool isImageFormat = (opts.Format() == FileType::PNG ||
+      opts.Format() == FileType::JPG || opts.Format() == FileType::PNM ||
+      opts.Format() == FileType::BMP || opts.Format() == FileType::GIF ||
+      opts.Format() == FileType::PSD || opts.Format() == FileType::TGA ||
+      opts.Format() == FileType::PIC || opts.Format() == FileType::ImageType);
 
-  if constexpr (IsArma<MatType>::value || IsSparseMat<MatType>::value)
+  if constexpr (isMatrixType)
   {
-    TextOptions txtOpts(std::move(opts));
-    success = LoadMatrix(filename, matrix, stream, txtOpts);
-    opts = std::move(txtOpts);
+    if (isImageFormat)
+    {
+      if constexpr (isSparseMatrixType)
+      {
+        return HandleError("Cannot load image data into a sparse matrix. "
+        "Please use dense matrix instead.", opts);
+      }
+      else
+      {
+        ImageOptions imgOpts(std::move(opts));
+        std::vector<std::string> files;
+        files.push_back(filename);
+        success = LoadImage(files, matrix, imgOpts);
+        if (copyBack)
+          opts = std::move(imgOpts);
+      }
+    }
+    else
+    {
+      TextOptions txtOpts(std::move(opts));
+      success = LoadNumeric(filename, matrix, stream, txtOpts);
+      if (copyBack)
+        opts = std::move(txtOpts);
+    }
+  }
+  else if constexpr (isSerializable)
+  {
+    success = LoadModel(matrix, opts, stream);
   }
   else
   {
-    if (opts.Fatal())
-      Log::Fatal << "DataOptionsType is unknown!  Please use a known type "
-          << "or provide specific overloads." << std::endl;
-    else
-      Log::Warn << "DataOptionsType is unknown!  Please use a known type "
-          << "or provide specific overloads." << std::endl;
-    return false;
+    return HandleError("DataOptionsType is unknown!  Please use a known type "
+        "or provide specific overloads.", opts);
   }
 
   if (!success)
   {
     Timer::Stop("loading_data");
-    if (opts.Fatal())
-      Log::Fatal << "Loading from '" << filename << "' failed." << std::endl;
-    else
-      Log::Warn << "Loading from '" << filename << "' failed." << std::endl;
-
-    return false;
+    std::stringstream oss;
+    oss << "Loading from '" << filename << "' failed.";
+    return HandleError(oss, opts);
   }
   else
   {
-    if constexpr (IsArma<MatType>::value)
+    if constexpr (IsArma<ObjectType>::value)
     {
       Log::Info << "Size is " << matrix.n_rows << " x "
           << matrix.n_cols << ".\n";
@@ -238,115 +137,42 @@ bool Load(const std::string& filename,
   return success;
 }
 
-template<typename MatType>
-bool LoadDense(const std::string& filename,
-               MatType& matrix,
-               TextOptions& opts,
-               std::fstream& stream)
-{
-  bool success;
-  if (opts.Format() != FileType::RawBinary)
-    Log::Info << "Loading '" << filename << "' as "
-        << opts.FileTypeToString() << ".  " << std::flush;
-
-  // We can't use the stream if the type is HDF5.
-  if (opts.Format() == FileType::HDF5Binary)
-  {
-    success = LoadHDF5(filename, matrix, opts);
-  }
-  else if (opts.Format() == FileType::CSVASCII)
-  {
-    success = LoadCSVASCII(filename, matrix, opts);
-
-    if (matrix.col(0).is_zero())
-      Log::Warn << "data::Load(): the first line in '" << filename << "' was "
-          << "loaded as all zeros; if the first row is headers, specify "
-          << "`HasHeaders() = true` in the given DataOptions." << std::endl;
-  }
-  else
-  {
-    if (opts.Format() == FileType::RawBinary)
-      Log::Warn << "Loading '" << filename << "' as "
-        << opts.FileTypeToString() << "; "
-        << "but this may not be the actual filetype!" << std::endl;
-
-    success = matrix.load(stream, ToArmaFileType(opts.Format()));
-    if (!opts.NoTranspose())
-      inplace_trans(matrix);
-  }
-  return success;
-}
-
-template <typename eT>
-bool LoadSparse(const std::string& filename,
-                arma::SpMat<eT>& matrix,
-                TextOptions& opts,
-                std::fstream& stream)
+template<typename eT, typename DataOptionsType>
+bool Load(const std::vector<std::string>& files,
+          arma::Mat<eT>& matrix,
+          DataOptionsType& opts,
+          const bool copyBack,
+          const typename std::enable_if_t<
+              IsDataOptions<DataOptionsType>::value>*)
 {
-  bool success;
-  // There is still a small amount of differentiation that needs to be done:
-  // if we got a text type, it could be a coordinate list.  We will make an
-  // educated guess based on the shape of the input.
-  if (opts.Format() == FileType::RawASCII)
+  bool success = false;
+  if (files.empty())
   {
-    // Get the number of columns in the file.  If it is the right shape, we
-    // will assume it is sparse.
-    const size_t cols = CountCols(stream);
-    if (cols == 3)
-    {
-      // We have the right number of columns, so assume the type is a
-      // coordinate list.
-      opts.Format() = FileType::CoordASCII;
-    }
+    return HandleError("Load(): given set of filenames is empty;"
+        " loading failed.", opts);
   }
 
-  // Filter out invalid types.
-  if ((opts.Format() == FileType::PGMBinary) ||
-      (opts.Format() == FileType::PPMBinary) ||
-      (opts.Format() == FileType::ArmaASCII) ||
-      (opts.Format() == FileType::RawBinary))
-  {
-    if (opts.Fatal())
-      Log::Fatal << "Cannot load '" << filename << "' with type "
-          << opts.FileTypeToString() << " into a sparse matrix; format is "
-          << "only supported for dense matrices." << std::endl;
-    else
-      Log::Warn << "Cannot load '" << filename << "' with type "
-          << opts.FileTypeToString() << " into a sparse matrix; format is "
-          << "only supported for dense matrices; load failed." << std::endl;
+  DetectFromExtension<arma::Mat<eT>>(files.back(), opts);
+  const bool isImageFormat = (opts.Format() == FileType::PNG ||
+      opts.Format() == FileType::JPG || opts.Format() == FileType::PNM ||
+      opts.Format() == FileType::BMP || opts.Format() == FileType::GIF ||
+      opts.Format() == FileType::PSD || opts.Format() == FileType::TGA ||
+      opts.Format() == FileType::PIC || opts.Format() == FileType::ImageType);
 
-    return false;
-  }
-  else if (opts.Format() == FileType::CSVASCII)
+  if (isImageFormat)
   {
-    // Armadillo sparse matrices can't load CSVs, so we have to load a separate
-    // matrix to do that.  If the CSV has three columns, we assume it's a
-    // coordinate list.
-    arma::Mat<eT> dense;
-    success = dense.load(stream, ToArmaFileType(opts.Format()));
-    if (dense.n_cols == 3)
-    {
-      arma::umat locations = arma::conv_to<arma::umat>::from(
-          dense.cols(0, 1).t());
-      matrix = arma::SpMat<eT>(locations, dense.col(2));
-    }
-    else
-    {
-      matrix = arma::conv_to<arma::SpMat<eT>>::from(dense);
-    }
+    ImageOptions imgOpts(std::move(opts));
+    success = LoadImage(files, matrix, imgOpts);
+    if (copyBack)
+      opts = std::move(imgOpts);
   }
   else
   {
-    success = matrix.load(stream, ToArmaFileType(opts.Format()));
-  }
-
-  if (!opts.NoTranspose())
-  {
-    // It seems that there is no direct way to use inplace_trans() on
-    // sparse matrices.
-    matrix = matrix.t();
+    TextOptions txtOpts(std::move(opts));
+    success = LoadNumericMultifile(files, matrix, txtOpts);
+    if (copyBack)
+      opts = std::move(txtOpts);
   }
-
   return success;
 }
 
@@ -393,14 +219,10 @@ bool LoadCategorical(const std::string& filename,
   {
     // The type is unknown.
     Timer::Stop("loading_data");
-    if (opts.Fatal())
-      Log::Fatal << "Unable to detect type of '" << filename << "'; "
-          << "Incorrect extension?" << std::endl;
-    else
-      Log::Warn << "Unable to detect type of '" << filename << "'; load failed."
-          << " Incorrect extension?" << std::endl;
-
-    return false;
+    std::stringstream oss;
+    oss << "Unable to detect type of '" << filename << "'; "
+          << "Incorrect extension?";
+    return HandleError(oss, opts);
   }
 
   Log::Info << "Size is " << matrix.n_rows << " x " << matrix.n_cols << ".\n";
@@ -410,7 +232,6 @@ bool LoadCategorical(const std::string& filename,
   return true;
 }
 
-} // namespace data
 } // namespace mlpack
 
 #endif

mlpack/include/mlpack/core/data/load_model.hpp

@@ -0,0 +1,62 @@
+/**
+ * @file core/data/load_model.hpp
+ * @author Ryan Curtin
+ * @author Omar Shrit
+ *
+ * Intenal implementation of model-specific Load() function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_DATA_LOAD_MODEL_HPP
+#define MLPACK_CORE_DATA_LOAD_MODEL_HPP
+
+#include <cereal/archives/xml.hpp>
+#include <cereal/archives/binary.hpp>
+#include <cereal/archives/json.hpp>
+
+#include "text_options.hpp"
+
+namespace mlpack {
+
+template<typename Object>
+bool LoadModel(Object& objectToSerialize,
+               DataOptionsBase<PlainDataOptions>& opts,
+               std::fstream& stream)
+{
+  try
+  {
+    if (opts.Format() == FileType::XML)
+    {
+      cereal::XMLInputArchive ar(stream);
+      ar(cereal::make_nvp("model", objectToSerialize));
+    }
+    else if (opts.Format() == FileType::JSON)
+    {
+     cereal::JSONInputArchive ar(stream);
+     ar(cereal::make_nvp("model", objectToSerialize));
+    }
+    else if (opts.Format() == FileType::BIN)
+    {
+      cereal::BinaryInputArchive ar(stream);
+      ar(cereal::make_nvp("model", objectToSerialize));
+    }
+
+    return true;
+  }
+  catch (cereal::Exception& e)
+  {
+    if (opts.Fatal())
+      Log::Fatal << e.what() << std::endl;
+    else
+      Log::Warn << e.what() << std::endl;
+
+    return false;
+  }
+}
+
+} // namespace mlpack
+
+#endif