flowyml 1.1.0__py3-none-any.whl

flowyml/core/templates.py

@@ -0,0 +1,226 @@
+"""Pre-built pipeline templates."""
+
+from typing import Any
+from flowyml.core.pipeline import Pipeline
+
+
+class PipelineTemplate:
+    """Base class for pipeline templates."""
+
+    @staticmethod
+    def create(name: str, **kwargs) -> Pipeline:
+        """Create a pipeline from this template."""
+        raise NotImplementedError()
+
+
+class MLTrainingTemplate(PipelineTemplate):
+    """Standard ML training pipeline template.
+
+    Steps:
+    1. Load data
+    2. Preprocess
+    3. Train model
+    4. Evaluate
+    5. Save model
+    """
+
+    @staticmethod
+    def create(
+        name: str = "ml_training",
+        data_loader: Any | None = None,
+        preprocessor: Any | None = None,
+        trainer: Any | None = None,
+        evaluator: Any | None = None,
+        model_saver: Any | None = None,
+        **ctx_params,
+    ) -> Pipeline:
+        """Create an ML training pipeline.
+
+        Args:
+            name: Pipeline name
+            data_loader: Function to load data
+            preprocessor: Function to preprocess data
+            trainer: Function to train model
+            evaluator: Function to evaluate model
+            model_saver: Function to save model
+            **ctx_params: Additional context parameters
+        """
+        from flowyml.core.pipeline import Pipeline
+        from flowyml.core.step import step
+        from flowyml.core.context import context
+
+        ctx = context(**ctx_params)
+        pipeline = Pipeline(name, context=ctx)
+
+        if data_loader:
+            load_step = step(name="load_data", outputs=["dataset"])(data_loader)
+            pipeline.add_step(load_step)
+
+        if preprocessor:
+            preprocess_step = step(
+                name="preprocess",
+                inputs=["dataset"],
+                outputs=["processed_data"],
+            )(preprocessor)
+            pipeline.add_step(preprocess_step)
+
+        if trainer:
+            train_step = step(
+                name="train",
+                inputs=["processed_data"],
+                outputs=["model"],
+            )(trainer)
+            pipeline.add_step(train_step)
+
+        if evaluator:
+            eval_step = step(
+                name="evaluate",
+                inputs=["model", "processed_data"],
+                outputs=["metrics"],
+            )(evaluator)
+            pipeline.add_step(eval_step)
+
+        if model_saver:
+            save_step = step(
+                name="save_model",
+                inputs=["model"],
+            )(model_saver)
+            pipeline.add_step(save_step)
+
+        return pipeline
+
+
+class DataPipelineTemplate(PipelineTemplate):
+    """Data processing pipeline template.
+
+    Steps:
+    1. Extract
+    2. Transform
+    3. Load (ETL)
+    """
+
+    @staticmethod
+    def create(
+        name: str = "etl_pipeline",
+        extractor: Any | None = None,
+        transformer: Any | None = None,
+        loader: Any | None = None,
+        **ctx_params,
+    ) -> Pipeline:
+        """Create an ETL pipeline."""
+        from flowyml.core.pipeline import Pipeline
+        from flowyml.core.step import step
+        from flowyml.core.context import context
+
+        ctx = context(**ctx_params)
+        pipeline = Pipeline(name, context=ctx)
+
+        if extractor:
+            extract_step = step(name="extract", outputs=["raw_data"])(extractor)
+            pipeline.add_step(extract_step)
+
+        if transformer:
+            transform_step = step(
+                name="transform",
+                inputs=["raw_data"],
+                outputs=["transformed_data"],
+            )(transformer)
+            pipeline.add_step(transform_step)
+
+        if loader:
+            load_step = step(
+                name="load",
+                inputs=["transformed_data"],
+            )(loader)
+            pipeline.add_step(load_step)
+
+        return pipeline
+
+
+class ABTestPipelineTemplate(PipelineTemplate):
+    """A/B testing pipeline template.
+
+    Runs multiple model variants and compares results.
+    """
+
+    @staticmethod
+    def create(
+        name: str = "ab_test",
+        data_loader: Any | None = None,
+        model_a_trainer: Any | None = None,
+        model_b_trainer: Any | None = None,
+        comparator: Any | None = None,
+        **ctx_params,
+    ) -> Pipeline:
+        """Create an A/B test pipeline."""
+        from flowyml.core.pipeline import Pipeline
+        from flowyml.core.step import step
+        from flowyml.core.context import context
+
+        ctx = context(**ctx_params)
+        pipeline = Pipeline(name, context=ctx)
+
+        if data_loader:
+            load_step = step(name="load_data", outputs=["dataset"])(data_loader)
+            pipeline.add_step(load_step)
+
+        if model_a_trainer:
+            train_a = step(
+                name="train_model_a",
+                inputs=["dataset"],
+                outputs=["model_a", "metrics_a"],
+            )(model_a_trainer)
+            pipeline.add_step(train_a)
+
+        if model_b_trainer:
+            train_b = step(
+                name="train_model_b",
+                inputs=["dataset"],
+                outputs=["model_b", "metrics_b"],
+            )(model_b_trainer)
+            pipeline.add_step(train_b)
+
+        if comparator:
+            compare = step(
+                name="compare",
+                inputs=["metrics_a", "metrics_b"],
+                outputs=["winner"],
+            )(comparator)
+            pipeline.add_step(compare)
+
+        return pipeline
+
+
+# Template registry
+TEMPLATES = {
+    "ml_training": MLTrainingTemplate,
+    "etl": DataPipelineTemplate,
+    "data_pipeline": DataPipelineTemplate,
+    "ab_test": ABTestPipelineTemplate,
+}
+
+
+def create_from_template(template_name: str, **kwargs) -> Pipeline:
+    """Create a pipeline from a template.
+
+    Args:
+        template_name: Name of the template
+        **kwargs: Template-specific arguments
+
+    Returns:
+        Configured pipeline
+
+    Examples:
+        >>> pipeline = create_from_template("ml_training", data_loader=load_data, trainer=train_model)
+    """
+    if template_name not in TEMPLATES:
+        available = ", ".join(TEMPLATES.keys())
+        raise ValueError(f"Unknown template '{template_name}'. Available: {available}")
+
+    template_class = TEMPLATES[template_name]
+    return template_class.create(**kwargs)
+
+
+def list_templates() -> list[str]:
+    """List available templates."""
+    return list(TEMPLATES.keys())

flowyml/core/versioning.py

@@ -0,0 +1,217 @@
+"""Pipeline versioning system."""
+
+import json
+import hashlib
+from pathlib import Path
+from typing import Any, Never
+from datetime import datetime
+from dataclasses import dataclass, asdict
+
+
+@dataclass
+class PipelineVersion:
+    """Represents a pipeline version."""
+
+    version: str
+    pipeline_name: str
+    created_at: str
+    steps: list[str]
+    step_hashes: dict[str, str]
+    context_params: dict[str, Any]
+    metadata: dict[str, Any]
+
+
+class VersionedPipeline:
+    """Pipeline with version control.
+
+    Tracks changes between versions and allows comparison.
+
+    Examples:
+        >>> from flowyml import VersionedPipeline, step
+        >>> pipeline = VersionedPipeline("training")
+        >>> pipeline.version = "v1.0.0"
+        >>> pipeline.add_step(load_data)
+        >>> pipeline.add_step(train_model)
+        >>> pipeline.save_version()
+        >>> # Make changes
+        >>> pipeline.add_step(evaluate)
+        >>> pipeline.version = "v1.1.0"
+        >>> pipeline.save_version()
+        >>> # Compare versions
+        >>> diff = pipeline.compare_with("v1.0.0")
+    """
+
+    def __init__(
+        self,
+        name: str,
+        version: str = "v0.1.0",
+        versions_dir: str = ".flowyml/versions",
+    ):
+        from flowyml.core.pipeline import Pipeline
+
+        self.name = name
+        self._version = version
+        self.pipeline = Pipeline(name)
+
+        # Version storage
+        self.versions_dir = Path(versions_dir) / name
+        self.versions_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load version history
+        self.versions: dict[str, PipelineVersion] = {}
+        self._load_versions()
+
+    @property
+    def version(self) -> str:
+        """Get current version."""
+        return self._version
+
+    @version.setter
+    def version(self, value: str) -> None:
+        """Set version."""
+        self._version = value
+
+    def add_step(self, step):
+        """Add a step to the pipeline."""
+        self.pipeline.add_step(step)
+        return self
+
+    def _compute_step_hash(self, step) -> str:
+        """Compute hash of step definition."""
+        # Hash based on source code
+        if hasattr(step, "source_code") and step.source_code:
+            return hashlib.md5(step.source_code.encode()).hexdigest()
+        # Fallback to name
+        return hashlib.md5(step.name.encode()).hexdigest()
+
+    def save_version(self, metadata: dict[str, Any] | None = None):
+        """Save current version."""
+        # Compute step hashes
+        step_hashes = {}
+        step_names = []
+
+        for step in self.pipeline.steps:
+            step_names.append(step.name)
+            step_hashes[step.name] = self._compute_step_hash(step)
+
+        # Create version record
+        version_data = PipelineVersion(
+            version=self._version,
+            pipeline_name=self.name,
+            created_at=datetime.now().isoformat(),
+            steps=step_names,
+            step_hashes=step_hashes,
+            context_params=self.pipeline.context._params if hasattr(self.pipeline.context, "_params") else {},
+            metadata=metadata or {},
+        )
+
+        # Save to disk
+        version_file = self.versions_dir / f"{self._version}.json"
+        with open(version_file, "w") as f:
+            json.dump(asdict(version_data), f, indent=2)
+
+        self.versions[self._version] = version_data
+
+        return version_data
+
+    def _load_versions(self) -> None:
+        """Load version history."""
+        for version_file in self.versions_dir.glob("*.json"):
+            with open(version_file) as f:
+                data = json.load(f)
+                version = data["version"]
+                self.versions[version] = PipelineVersion(**data)
+
+    def list_versions(self) -> list[str]:
+        """List all saved versions."""
+        return sorted(self.versions.keys())
+
+    def get_version(self, version: str) -> PipelineVersion | None:
+        """Get specific version details."""
+        return self.versions.get(version)
+
+    def compare_with(self, other_version: str) -> dict[str, Any]:
+        """Compare current pipeline with another version.
+
+        Returns:
+            Dictionary with differences
+        """
+        if other_version not in self.versions:
+            raise ValueError(f"Version {other_version} not found")
+
+        current_steps = {s.name: self._compute_step_hash(s) for s in self.pipeline.steps}
+        other = self.versions[other_version]
+
+        # Find differences
+        added_steps = set(current_steps.keys()) - set(other.steps)
+        removed_steps = set(other.steps) - set(current_steps.keys())
+
+        # Modified steps (same name, different hash)
+        modified_steps = []
+        for step_name in set(current_steps.keys()) & set(other.steps):
+            if current_steps[step_name] != other.step_hashes.get(step_name):
+                modified_steps.append(step_name)
+
+        comparison = {
+            "current_version": self._version,
+            "compared_to": other_version,
+            "added_steps": list(added_steps),
+            "removed_steps": list(removed_steps),
+            "modified_steps": modified_steps,
+            "step_order_changed": current_steps.keys() != other.steps,
+            "context_changes": self._compare_dicts(
+                self.pipeline.context._params if hasattr(self.pipeline.context, "_params") else {},
+                other.context_params,
+            ),
+        }
+
+        return comparison
+
+    def _compare_dicts(self, d1: dict, d2: dict) -> dict[str, Any]:
+        """Compare two dictionaries."""
+        added = set(d1.keys()) - set(d2.keys())
+        removed = set(d2.keys()) - set(d1.keys())
+        modified = {k for k in set(d1.keys()) & set(d2.keys()) if d1[k] != d2[k]}
+
+        return {
+            "added": {k: d1[k] for k in added},
+            "removed": {k: d2[k] for k in removed},
+            "modified": {k: {"old": d2[k], "new": d1[k]} for k in modified},
+        }
+
+    def display_comparison(self, other_version: str) -> None:
+        """Display comparison in readable format."""
+        diff = self.compare_with(other_version)
+
+        if diff["added_steps"]:
+            pass
+
+        if diff["removed_steps"]:
+            pass
+
+        if diff["modified_steps"]:
+            pass
+
+        if diff["step_order_changed"]:
+            pass
+
+        changes = diff["context_changes"]
+        if any([changes["added"], changes["removed"], changes["modified"]]):
+            if changes["added"]:
+                pass
+            if changes["removed"]:
+                pass
+            if changes["modified"]:
+                pass
+
+    def rollback(self, version: str) -> Never:
+        """Rollback to a previous version (not implemented - would need to reconstruct pipeline)."""
+        raise NotImplementedError("Rollback requires pipeline reconstruction from saved state")
+
+    def run(self, *args, **kwargs):
+        """Run the pipeline."""
+        return self.pipeline.run(*args, **kwargs)
+
+    def __getattr__(self, name):
+        """Delegate to underlying pipeline."""
+        return getattr(self.pipeline, name)

flowyml/integrations/__init__.py

@@ -0,0 +1 @@
+"""flowyml Integrations."""

flowyml/integrations/keras.py

@@ -0,0 +1,134 @@
+"""Keras integration for flowyml."""
+
+from pathlib import Path
+from datetime import datetime
+
+try:
+    from tensorflow import keras
+except ImportError:
+    try:
+        import keras
+    except ImportError:
+        keras = None
+
+from flowyml.tracking.experiment import Experiment
+from flowyml.storage.metadata import SQLiteMetadataStore
+
+
+class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
+    """Keras callback for flowyml tracking.
+
+    Automatically logs:
+    - Training metrics (loss, accuracy, etc.)
+    - Model checkpoints (optional)
+    - Model architecture
+    - Training parameters
+    """
+
+    def __init__(
+        self,
+        experiment_name: str,
+        run_name: str | None = None,
+        log_model: bool = True,
+        log_every_epoch: bool = True,
+        metadata_store: SQLiteMetadataStore | None = None,
+    ):
+        """Args:
+        experiment_name: Name of the experiment
+        run_name: Optional run name (defaults to timestamp)
+        log_model: Whether to save the model as an artifact
+        log_every_epoch: Whether to log metrics every epoch
+        metadata_store: Optional metadata store override.
+        """
+        if keras is None:
+            raise ImportError("Keras is not installed. Please install tensorflow or keras.")
+
+        super().__init__()
+        self.experiment_name = experiment_name
+        self.run_name = run_name or datetime.now().strftime("run_%Y%m%d_%H%M%S")
+        self.log_model = log_model
+        self.log_every_epoch = log_every_epoch
+
+        self.metadata_store = metadata_store or SQLiteMetadataStore()
+
+        # Initialize experiment
+        self.experiment = Experiment(experiment_name)
+
+        # Track params
+        self.params_logged = False
+
+    def on_train_begin(self, logs=None) -> None:
+        """Log initial parameters."""
+        if not self.params_logged:
+            params = {
+                "optimizer": str(self.model.optimizer.get_config()),
+                "loss": str(self.model.loss),
+                "metrics": [str(m) for m in self.model.metrics_names],
+                "epochs": self.params.get("epochs"),
+                "batch_size": self.params.get("batch_size"),
+                "samples": self.params.get("samples"),
+            }
+
+            # Log architecture
+            model_json = self.model.to_json()
+
+            self.metadata_store.log_experiment_run(
+                experiment_id=self.experiment_name,
+                run_id=self.run_name,
+                parameters=params,
+            )
+
+            # Save architecture as artifact
+            self.metadata_store.save_artifact(
+                artifact_id=f"{self.run_name}_model_arch",
+                metadata={
+                    "name": "model_architecture",
+                    "type": "json",
+                    "run_id": self.run_name,
+                    "value": model_json,
+                    "created_at": datetime.now().isoformat(),
+                },
+            )
+
+            self.params_logged = True
+
+    def on_epoch_end(self, epoch, logs=None) -> None:
+        """Log metrics at the end of each epoch."""
+        if self.log_every_epoch and logs:
+            # Log metrics to DB
+            for k, v in logs.items():
+                self.metadata_store.save_metric(
+                    run_id=self.run_name,
+                    name=k,
+                    value=float(v),
+                    step=epoch,
+                )
+
+            # Update experiment run
+            self.metadata_store.log_experiment_run(
+                experiment_id=self.experiment_name,
+                run_id=self.run_name,
+                metrics=logs,
+            )
+
+    def on_train_end(self, logs=None) -> None:
+        """Save model at the end of training."""
+        if self.log_model:
+            # Create artifacts directory
+            # Create artifacts directory
+            artifact_dir = Path(f".flowyml/artifacts/{self.run_name}")
+            artifact_dir.mkdir(parents=True, exist_ok=True)
+
+            model_path = artifact_dir / "model.keras"
+            self.model.save(model_path)
+
+            self.metadata_store.save_artifact(
+                artifact_id=f"{self.run_name}_model",
+                metadata={
+                    "name": "trained_model",
+                    "type": "keras_model",
+                    "run_id": self.run_name,
+                    "path": str(model_path.resolve()),
+                    "created_at": datetime.now().isoformat(),
+                },
+            )

flowyml/monitoring/__init__.py

@@ -0,0 +1 @@
+"""flowyml Monitoring module."""

flowyml/monitoring/alerts.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Never
+from datetime import datetime
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class AlertLevel(Enum):
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+    CRITICAL = "critical"
+
+
+@dataclass
+class Alert:
+    title: str
+    message: str
+    level: AlertLevel
+    timestamp: datetime = field(default_factory=datetime.now)
+    metadata: dict[str, Any] | None = None
+
+
+class AlertHandler:
+    def handle(self, alert: Alert) -> Never:
+        raise NotImplementedError
+
+
+class ConsoleAlertHandler(AlertHandler):
+    def handle(self, alert: Alert) -> None:
+        # Simple ANSI colors if supported
+        if alert.level == AlertLevel.ERROR or alert.level == AlertLevel.CRITICAL or alert.level == AlertLevel.WARNING:
+            pass
+
+
+class AlertManager:
+    def __init__(self):
+        self.handlers: list[AlertHandler] = [ConsoleAlertHandler()]
+        self.history: list[Alert] = []
+
+    def add_handler(self, handler: AlertHandler) -> None:
+        self.handlers.append(handler)
+
+    def send_alert(self, title: str, message: str, level: AlertLevel = AlertLevel.INFO, metadata: dict = None) -> None:
+        alert = Alert(title=title, message=message, level=level, metadata=metadata)
+        self.history.append(alert)
+        for handler in self.handlers:
+            try:
+                handler.handle(alert)
+            except Exception as e:
+                logger.error(f"Failed to handle alert: {e}")
+
+
+# Global instance
+alert_manager = AlertManager()