flowyml 1.1.0__py3-none-any.whl

flowyml/storage/materializers/pandas.py

@@ -0,0 +1,142 @@
+"""Pandas materializer for DataFrame serialization."""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from flowyml.storage.materializers.base import BaseMaterializer, register_materializer
+
+try:
+    import pandas as pd
+
+    PANDAS_AVAILABLE = True
+except ImportError:
+    PANDAS_AVAILABLE = False
+
+
+if PANDAS_AVAILABLE:
+
+    class PandasMaterializer(BaseMaterializer):
+        """Materializer for Pandas DataFrames and Series."""
+
+        def save(self, obj: Any, path: Path) -> None:
+            """Save Pandas object to path.
+
+            Args:
+                obj: Pandas DataFrame or Series
+                path: Directory path where object should be saved
+            """
+            path.mkdir(parents=True, exist_ok=True)
+
+            if isinstance(obj, pd.DataFrame):
+                # Save DataFrame as parquet (efficient) or CSV (fallback)
+                try:
+                    data_path = path / "data.parquet"
+                    obj.to_parquet(data_path, index=True)
+                    format_used = "parquet"
+                except Exception:
+                    data_path = path / "data.csv"
+                    obj.to_csv(data_path, index=True)
+                    format_used = "csv"
+
+                # Save metadata
+                metadata = {
+                    "type": "pandas_dataframe",
+                    "format": format_used,
+                    "shape": list(obj.shape),
+                    "columns": obj.columns.tolist(),
+                    "dtypes": {col: str(dtype) for col, dtype in obj.dtypes.items()},
+                    "index_name": obj.index.name,
+                }
+
+                with open(path / "metadata.json", "w") as f:
+                    json.dump(metadata, f, indent=2)
+
+            elif isinstance(obj, pd.Series):
+                # Save Series
+                try:
+                    data_path = path / "data.parquet"
+                    obj.to_frame().to_parquet(data_path, index=True)
+                    format_used = "parquet"
+                except Exception:
+                    data_path = path / "data.csv"
+                    obj.to_csv(data_path, index=True, header=True)
+                    format_used = "csv"
+
+                metadata = {
+                    "type": "pandas_series",
+                    "format": format_used,
+                    "shape": len(obj),
+                    "name": obj.name,
+                    "dtype": str(obj.dtype),
+                    "index_name": obj.index.name,
+                }
+
+                with open(path / "metadata.json", "w") as f:
+                    json.dump(metadata, f, indent=2)
+
+        def load(self, path: Path) -> Any:
+            """Load Pandas object from path.
+
+            Args:
+                path: Directory path from which to load object
+
+            Returns:
+                Loaded Pandas DataFrame or Series
+            """
+            # Load metadata
+            metadata_path = path / "metadata.json"
+            if metadata_path.exists():
+                with open(metadata_path) as f:
+                    metadata = json.load(f)
+            else:
+                metadata = {}
+
+            obj_type = metadata.get("type", "pandas_dataframe")
+            format_used = metadata.get("format", "parquet")
+
+            if obj_type == "pandas_dataframe":
+                if format_used == "parquet":
+                    data_path = path / "data.parquet"
+                    return pd.read_parquet(data_path)
+                else:
+                    data_path = path / "data.csv"
+                    return pd.read_csv(data_path, index_col=0)
+
+            elif obj_type == "pandas_series":
+                if format_used == "parquet":
+                    data_path = path / "data.parquet"
+                    df = pd.read_parquet(data_path)
+                    return df.iloc[:, 0]
+                else:
+                    data_path = path / "data.csv"
+                    series = pd.read_csv(data_path, index_col=0)
+                    if isinstance(series, pd.DataFrame):
+                        series = series.iloc[:, 0]
+                    return series
+
+            else:
+                raise ValueError(f"Unknown Pandas object type: {obj_type}")
+
+        @classmethod
+        def supported_types(cls) -> list[type]:
+            """Return Pandas types supported by this materializer."""
+            return [pd.DataFrame, pd.Series]
+
+    # Auto-register
+    register_materializer(PandasMaterializer)
+
+else:
+    # Placeholder when Pandas not available
+    class PandasMaterializer(BaseMaterializer):
+        """Placeholder materializer when Pandas is not installed."""
+
+        def save(self, obj: Any, path: Path) -> None:
+            raise ImportError("Pandas is not installed. Install with: pip install pandas")
+
+        def load(self, path: Path) -> Any:
+            raise ImportError("Pandas is not installed. Install with: pip install pandas")
+
+        @classmethod
+        def supported_types(cls) -> list[type]:
+            return []

flowyml/storage/materializers/pytorch.py

@@ -0,0 +1,135 @@
+"""PyTorch materializer for model serialization."""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from flowyml.storage.materializers.base import BaseMaterializer, register_materializer
+import contextlib
+import builtins
+
+try:
+    import torch
+    import torch.nn as nn
+
+    # Verify PyTorch has expected attributes
+    _ = nn.Module
+    _ = torch.Tensor
+    PYTORCH_AVAILABLE = True
+except (ImportError, AttributeError):
+    PYTORCH_AVAILABLE = False
+    torch = None
+    nn = None
+
+
+if PYTORCH_AVAILABLE:
+
+    class PyTorchMaterializer(BaseMaterializer):
+        """Materializer for PyTorch models and tensors."""
+
+        def save(self, obj: Any, path: Path) -> None:
+            """Save PyTorch object to path.
+
+            Args:
+                obj: PyTorch model or tensor
+                path: Directory path where object should be saved
+            """
+            path.mkdir(parents=True, exist_ok=True)
+
+            if isinstance(obj, nn.Module):
+                # Save model
+                model_path = path / "model.pt"
+                torch.save(obj.state_dict(), model_path)
+
+                # Save model architecture info
+                metadata = {
+                    "type": "pytorch_model",
+                    "class_name": obj.__class__.__name__,
+                    "module": obj.__class__.__module__,
+                }
+
+                # Try to capture model architecture
+                with contextlib.suppress(builtins.BaseException):
+                    metadata["architecture"] = str(obj)
+
+                with open(path / "metadata.json", "w") as f:
+                    json.dump(metadata, f, indent=2)
+
+            elif isinstance(obj, torch.Tensor):
+                # Save tensor
+                tensor_path = path / "tensor.pt"
+                torch.save(obj, tensor_path)
+
+                metadata = {
+                    "type": "pytorch_tensor",
+                    "shape": list(obj.shape),
+                    "dtype": str(obj.dtype),
+                    "device": str(obj.device),
+                }
+
+                with open(path / "metadata.json", "w") as f:
+                    json.dump(metadata, f, indent=2)
+
+            else:
+                # Generic PyTorch object
+                obj_path = path / "object.pt"
+                torch.save(obj, obj_path)
+
+                metadata = {"type": "pytorch_object"}
+                with open(path / "metadata.json", "w") as f:
+                    json.dump(metadata, f, indent=2)
+
+        def load(self, path: Path) -> Any:
+            """Load PyTorch object from path.
+
+            Args:
+                path: Directory path from which to load object
+
+            Returns:
+                Loaded PyTorch object
+            """
+            # Load metadata
+            metadata_path = path / "metadata.json"
+            if metadata_path.exists():
+                with open(metadata_path) as f:
+                    metadata = json.load(f)
+            else:
+                metadata = {}
+
+            obj_type = metadata.get("type", "pytorch_object")
+
+            if obj_type == "pytorch_model":
+                # Load model state dict only (user needs to provide architecture)
+                model_path = path / "model.pt"
+                return torch.load(model_path, weights_only=True)
+
+            elif obj_type == "pytorch_tensor":
+                tensor_path = path / "tensor.pt"
+                return torch.load(tensor_path, weights_only=True)
+
+            else:
+                obj_path = path / "object.pt"
+                return torch.load(obj_path, weights_only=False)
+
+        @classmethod
+        def supported_types(cls) -> list[type]:
+            """Return PyTorch types supported by this materializer."""
+            return [nn.Module, torch.Tensor]
+
+    # Auto-register
+    register_materializer(PyTorchMaterializer)
+
+else:
+    # Placeholder when PyTorch not available
+    class PyTorchMaterializer(BaseMaterializer):
+        """Placeholder materializer when PyTorch is not installed."""
+
+        def save(self, obj: Any, path: Path) -> None:
+            raise ImportError("PyTorch is not installed. Install with: pip install torch")
+
+        def load(self, path: Path) -> Any:
+            raise ImportError("PyTorch is not installed. Install with: pip install torch")
+
+        @classmethod
+        def supported_types(cls) -> list[type]:
+            return []

flowyml/storage/materializers/sklearn.py

@@ -0,0 +1,110 @@
+"""Scikit-learn materializer for model serialization."""
+
+import json
+import pickle
+from pathlib import Path
+from typing import Any
+
+from flowyml.storage.materializers.base import BaseMaterializer, register_materializer
+import contextlib
+import builtins
+
+try:
+    from sklearn.base import BaseEstimator
+
+    SKLEARN_AVAILABLE = True
+except ImportError:
+    SKLEARN_AVAILABLE = False
+
+
+if SKLEARN_AVAILABLE:
+
+    class SklearnMaterializer(BaseMaterializer):
+        """Materializer for scikit-learn models."""
+
+        def save(self, obj: Any, path: Path) -> None:
+            """Save scikit-learn model to path.
+
+            Args:
+                obj: Scikit-learn model
+                path: Directory path where object should be saved
+            """
+            path.mkdir(parents=True, exist_ok=True)
+
+            # Save model using pickle
+            model_path = path / "model.pkl"
+            with open(model_path, "wb") as f:
+                pickle.dump(obj, f)
+
+            # Save metadata
+            metadata = {
+                "type": "sklearn_model",
+                "class_name": obj.__class__.__name__,
+                "module": obj.__class__.__module__,
+            }
+
+            # Capture model parameters
+            with contextlib.suppress(builtins.BaseException):
+                metadata["params"] = obj.get_params()
+
+            # Capture feature importance if available
+            if hasattr(obj, "feature_importances_"):
+                with contextlib.suppress(builtins.BaseException):
+                    metadata["feature_importances"] = obj.feature_importances_.tolist()
+
+            # Capture number of features
+            if hasattr(obj, "n_features_in_"):
+                metadata["n_features"] = int(obj.n_features_in_)
+
+            # Capture feature names if available
+            if hasattr(obj, "feature_names_in_"):
+                with contextlib.suppress(builtins.BaseException):
+                    metadata["feature_names"] = obj.feature_names_in_.tolist()
+
+            # Capture classes if classifier
+            if hasattr(obj, "classes_"):
+                with contextlib.suppress(builtins.BaseException):
+                    metadata["classes"] = obj.classes_.tolist()
+
+            with open(path / "metadata.json", "w") as f:
+                json.dump(metadata, f, indent=2, default=str)
+
+        def load(self, path: Path) -> Any:
+            """Load scikit-learn model from path.
+
+            Args:
+                path: Directory path from which to load object
+
+            Returns:
+                Loaded scikit-learn model
+            """
+            model_path = path / "model.pkl"
+
+            if not model_path.exists():
+                raise FileNotFoundError(f"Model file not found at {model_path}")
+
+            with open(model_path, "rb") as f:
+                return pickle.load(f)
+
+        @classmethod
+        def supported_types(cls) -> list[type]:
+            """Return scikit-learn types supported by this materializer."""
+            return [BaseEstimator]
+
+    # Auto-register
+    register_materializer(SklearnMaterializer)
+
+else:
+    # Placeholder when scikit-learn not available
+    class SklearnMaterializer(BaseMaterializer):
+        """Placeholder materializer when scikit-learn is not installed."""
+
+        def save(self, obj: Any, path: Path) -> None:
+            raise ImportError("Scikit-learn is not installed. Install with: pip install scikit-learn")
+
+        def load(self, path: Path) -> Any:
+            raise ImportError("Scikit-learn is not installed. Install with: pip install scikit-learn")
+
+        @classmethod
+        def supported_types(cls) -> list[type]:
+            return []

flowyml/storage/materializers/tensorflow.py

@@ -0,0 +1,152 @@
+"""TensorFlow materializer for model serialization."""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from flowyml.storage.materializers.base import BaseMaterializer, register_materializer
+import contextlib
+import builtins
+
+try:
+    import tensorflow as tf
+
+    # Verify TensorFlow has expected attributes
+    _ = tf.keras.Model
+    _ = tf.Tensor
+    _ = tf.Variable
+    TENSORFLOW_AVAILABLE = True
+except (ImportError, AttributeError):
+    TENSORFLOW_AVAILABLE = False
+    tf = None
+
+
+if TENSORFLOW_AVAILABLE:
+
+    class TensorFlowMaterializer(BaseMaterializer):
+        """Materializer for TensorFlow/Keras models."""
+
+        def save(self, obj: Any, path: Path) -> None:
+            """Save TensorFlow object to path.
+
+            Args:
+                obj: TensorFlow model or tensor
+                path: Directory path where object should be saved
+            """
+            path.mkdir(parents=True, exist_ok=True)
+
+            if isinstance(obj, tf.keras.Model):
+                # Save Keras model in SavedModel format
+                model_path = path / "saved_model"
+                obj.save(model_path, save_format="tf")
+
+                # Save metadata
+                metadata = {
+                    "type": "tensorflow_keras_model",
+                    "class_name": obj.__class__.__name__,
+                    "input_shape": str(obj.input_shape) if hasattr(obj, "input_shape") else None,
+                    "output_shape": str(obj.output_shape) if hasattr(obj, "output_shape") else None,
+                }
+
+                # Try to get model config
+                with contextlib.suppress(builtins.BaseException):
+                    metadata["config"] = obj.get_config()
+
+                with open(path / "metadata.json", "w") as f:
+                    json.dump(metadata, f, indent=2, default=str)
+
+            elif isinstance(obj, tf.Tensor):
+                # Save tensor as numpy array
+                import numpy as np
+
+                tensor_path = path / "tensor.npy"
+                np.save(tensor_path, obj.numpy())
+
+                metadata = {
+                    "type": "tensorflow_tensor",
+                    "shape": list(obj.shape),
+                    "dtype": str(obj.dtype),
+                }
+
+                with open(path / "metadata.json", "w") as f:
+                    json.dump(metadata, f, indent=2)
+
+            elif isinstance(obj, tf.Variable):
+                # Save variable
+                import numpy as np
+
+                var_path = path / "variable.npy"
+                np.save(var_path, obj.numpy())
+
+                metadata = {
+                    "type": "tensorflow_variable",
+                    "shape": list(obj.shape),
+                    "dtype": str(obj.dtype),
+                    "name": obj.name,
+                }
+
+                with open(path / "metadata.json", "w") as f:
+                    json.dump(metadata, f, indent=2)
+
+        def load(self, path: Path) -> Any:
+            """Load TensorFlow object from path.
+
+            Args:
+                path: Directory path from which to load object
+
+            Returns:
+                Loaded TensorFlow object
+            """
+            # Load metadata
+            metadata_path = path / "metadata.json"
+            if metadata_path.exists():
+                with open(metadata_path) as f:
+                    metadata = json.load(f)
+            else:
+                metadata = {}
+
+            obj_type = metadata.get("type", "tensorflow_object")
+
+            if obj_type == "tensorflow_keras_model":
+                model_path = path / "saved_model"
+                return tf.keras.models.load_model(model_path)
+
+            elif obj_type == "tensorflow_tensor":
+                import numpy as np
+
+                tensor_path = path / "tensor.npy"
+                array = np.load(tensor_path)
+                return tf.convert_to_tensor(array)
+
+            elif obj_type == "tensorflow_variable":
+                import numpy as np
+
+                var_path = path / "variable.npy"
+                array = np.load(var_path)
+                return tf.Variable(array, name=metadata.get("name"))
+
+            else:
+                raise ValueError(f"Unknown TensorFlow object type: {obj_type}")
+
+        @classmethod
+        def supported_types(cls) -> list[type]:
+            """Return TensorFlow types supported by this materializer."""
+            return [tf.keras.Model, tf.Tensor, tf.Variable]
+
+    # Auto-register
+    register_materializer(TensorFlowMaterializer)
+
+else:
+    # Placeholder when TensorFlow not available
+    class TensorFlowMaterializer(BaseMaterializer):
+        """Placeholder materializer when TensorFlow is not installed."""
+
+        def save(self, obj: Any, path: Path) -> None:
+            raise ImportError("TensorFlow is not installed. Install with: pip install tensorflow")
+
+        def load(self, path: Path) -> Any:
+            raise ImportError("TensorFlow is not installed. Install with: pip install tensorflow")
+
+        @classmethod
+        def supported_types(cls) -> list[type]:
+            return []