flowyml 1.1.0__py3-none-any.whl

flowyml/core/advanced_cache.py

@@ -0,0 +1,274 @@
+"""Advanced caching strategies."""
+
+import hashlib
+import pickle
+from pathlib import Path
+from typing import Any
+from collections.abc import Callable
+from datetime import datetime, timedelta
+
+
+class ContentBasedCache:
+    """Content-based caching using input hashing.
+
+    Caches based on actual input content, not just step name.
+    """
+
+    def __init__(self, cache_dir: str = ".flowyml/cache"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def _compute_hash(self, *args, **kwargs) -> str:
+        """Compute hash of inputs."""
+        # Serialize inputs
+        try:
+            content = pickle.dumps((args, kwargs))
+            return hashlib.sha256(content).hexdigest()
+        except Exception:
+            # Fallback to str representation
+            content = str((args, kwargs)).encode()
+            return hashlib.sha256(content).hexdigest()
+
+    def get(self, step_name: str, *args, **kwargs) -> Any | None:
+        """Get cached result if exists."""
+        content_hash = self._compute_hash(*args, **kwargs)
+        cache_key = f"{step_name}_{content_hash}"
+        cache_file = self.cache_dir / f"{cache_key}.pkl"
+
+        if cache_file.exists():
+            with open(cache_file, "rb") as f:
+                cached_data = pickle.load(f)
+
+            # Check if still valid
+            if "result" in cached_data:
+                return cached_data["result"]
+
+        return None
+
+    def set_value(self, step_name: str, result: Any, *args, **kwargs) -> None:
+        """Cache a result."""
+        content_hash = self._compute_hash(*args, **kwargs)
+        cache_key = f"{step_name}_{content_hash}"
+        cache_file = self.cache_dir / f"{cache_key}.pkl"
+
+        cached_data = {
+            "result": result,
+            "cached_at": datetime.now().isoformat(),
+            "inputs_hash": content_hash,
+        }
+
+        with open(cache_file, "wb") as f:
+            pickle.dump(cached_data, f)
+
+    def invalidate(self, step_name: str | None = None) -> None:
+        """Invalidate cache entries."""
+        pattern = f"{step_name}_*.pkl" if step_name else "*.pkl"
+
+        for cache_file in self.cache_dir.glob(pattern):
+            cache_file.unlink()
+
+
+class SharedCache:
+    """Shared cache across different pipeline runs.
+
+    Allows cache reuse across multiple executions.
+    """
+
+    def __init__(self, cache_dir: str = ".flowyml/shared_cache"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.index_file = self.cache_dir / "index.json"
+        self._load_index()
+
+    def _load_index(self) -> None:
+        """Load cache index."""
+        import json
+
+        if self.index_file.exists():
+            with open(self.index_file) as f:
+                self.index = json.load(f)
+        else:
+            self.index = {}
+
+    def _save_index(self) -> None:
+        """Save cache index."""
+        import json
+
+        with open(self.index_file, "w") as f:
+            json.dump(self.index, f, indent=2)
+
+    def get(self, cache_key: str) -> Any | None:
+        """Get from shared cache."""
+        if cache_key in self.index:
+            cache_file = self.cache_dir / f"{cache_key}.pkl"
+            if cache_file.exists():
+                with open(cache_file, "rb") as f:
+                    return pickle.load(f)
+        return None
+
+    def set_value(self, cache_key: str, value: Any, metadata: dict | None = None) -> None:
+        """Set shared cache entry."""
+        cache_file = self.cache_dir / f"{cache_key}.pkl"
+
+        with open(cache_file, "wb") as f:
+            pickle.dump(value, f)
+
+        self.index[cache_key] = {
+            "cached_at": datetime.now().isoformat(),
+            "metadata": metadata or {},
+        }
+        self._save_index()
+
+    def list_keys(self) -> list:
+        """List all cache keys."""
+        return list(self.index.keys())
+
+
+class SmartCache:
+    """Smart cache with TTL and automatic invalidation.
+
+    Features:
+    - Time-to-live (TTL)
+    - Size limits
+    - LRU eviction
+    """
+
+    def __init__(
+        self,
+        cache_dir: str = ".flowyml/smart_cache",
+        ttl_seconds: int = 3600,
+        max_size_mb: int = 1000,
+    ):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.ttl_seconds = ttl_seconds
+        self.max_size_mb = max_size_mb
+
+    def get(self, key: str) -> Any | None:
+        """Get cached value if not expired."""
+        cache_file = self.cache_dir / f"{key}.pkl"
+        metadata_file = self.cache_dir / f"{key}.meta"
+
+        if not cache_file.exists() or not metadata_file.exists():
+            return None
+
+        # Check TTL
+        import json
+
+        with open(metadata_file) as f:
+            metadata = json.load(f)
+
+        cached_at = datetime.fromisoformat(metadata["cached_at"])
+        if datetime.now() - cached_at > timedelta(seconds=self.ttl_seconds):
+            # Expired
+            cache_file.unlink()
+            metadata_file.unlink()
+            return None
+
+        # Update access time
+        metadata["last_accessed"] = datetime.now().isoformat()
+        with open(metadata_file, "w") as f:
+            json.dump(metadata, f)
+
+        with open(cache_file, "rb") as f:
+            return pickle.load(f)
+
+    def set_value(self, key: str, value: Any) -> None:
+        """Set cached value."""
+        import json
+
+        # Check size limits
+        self._evict_if_needed()
+
+        cache_file = self.cache_dir / f"{key}.pkl"
+        metadata_file = self.cache_dir / f"{key}.meta"
+
+        with open(cache_file, "wb") as f:
+            pickle.dump(value, f)
+
+        metadata = {
+            "cached_at": datetime.now().isoformat(),
+            "last_accessed": datetime.now().isoformat(),
+            "size_bytes": cache_file.stat().st_size,
+        }
+
+        with open(metadata_file, "w") as f:
+            json.dump(metadata, f)
+
+    def _evict_if_needed(self) -> None:
+        """Evict old entries if cache is too large."""
+        total_size = sum(f.stat().st_size for f in self.cache_dir.glob("*.pkl"))
+        max_size_bytes = self.max_size_mb * 1024 * 1024
+
+        if total_size > max_size_bytes:
+            # LRU eviction
+            import json
+
+            entries = []
+            for meta_file in self.cache_dir.glob("*.meta"):
+                with open(meta_file) as f:
+                    metadata = json.load(f)
+                    entries.append(
+                        (
+                            meta_file.stem,
+                            datetime.fromisoformat(metadata["last_accessed"]),
+                        ),
+                    )
+
+            # Sort by access time
+            entries.sort(key=lambda x: x[1])
+
+            # Remove oldest entries until under limit
+            for key, _ in entries:
+                cache_file = self.cache_dir / f"{key}.pkl"
+                meta_file = self.cache_dir / f"{key}.meta"
+
+                if cache_file.exists():
+                    cache_file.unlink()
+                if meta_file.exists():
+                    meta_file.unlink()
+
+                # Recalculate size
+                total_size = sum(f.stat().st_size for f in self.cache_dir.glob("*.pkl"))
+                if total_size <= max_size_bytes:
+                    break
+
+
+def memoize(ttl_seconds: int | None = None):
+    """Memoization decorator for functions.
+
+    Args:
+        ttl_seconds: Time-to-live for cached results
+
+    Example:
+        >>> @memoize(ttl_seconds=3600)
+        ... def expensive_function(x):
+        ...     return x**2
+    """
+    cache = {}
+    cache_time = {}
+
+    def decorator(func: Callable):
+        def wrapper(*args, **kwargs):
+            # Create cache key
+            key = (args, tuple(sorted(kwargs.items())))
+
+            # Check if cached and valid
+            if key in cache:
+                if ttl_seconds is None:
+                    return cache[key]
+
+                elapsed = (datetime.now() - cache_time[key]).total_seconds()
+                if elapsed < ttl_seconds:
+                    return cache[key]
+
+            # Compute and cache
+            result = func(*args, **kwargs)
+            cache[key] = result
+            cache_time[key] = datetime.now()
+
+            return result
+
+        return wrapper
+
+    return decorator

flowyml/core/approval.py

@@ -0,0 +1,64 @@
+"""Human-in-the-loop approval step."""
+
+from collections.abc import Callable
+from flowyml.core.step import Step
+
+
+class ApprovalStep(Step):
+    """A step that pauses execution until manual approval is granted.
+
+    This is useful for:
+    - Reviewing LLM outputs before proceeding
+    - Cost control (approving expensive operations)
+    - Safety checks
+
+    The step will poll for approval status or wait for a signal.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        approver: str | None = None,
+        timeout_seconds: int = 3600,  # 1 hour default
+        auto_approve_if: Callable | None = None,
+    ):
+        super().__init__(name)
+        self.approver = approver
+        self.timeout_seconds = timeout_seconds
+        self.auto_approve_if = auto_approve_if
+
+    def __call__(self, *args, **kwargs):
+        """Execute the approval logic."""
+        # Check auto-approval condition
+        if self.auto_approve_if and self.auto_approve_if(*args, **kwargs):
+            return args[0] if args else None
+
+        # In a real implementation, this would:
+        # 1. Create an 'Approval Request' in the DB
+        # 2. Send a notification (Slack/Email)
+        # 3. Poll DB for status change
+
+        # For this local version, we'll simulate a simple CLI prompt if interactive,
+        # or just fail if non-interactive (safety first).
+
+        # Check if we are in an interactive terminal
+        import sys
+
+        if sys.stdin.isatty():
+            response = input("   Approve execution? [y/N]: ")
+            if response.lower() == "y":
+                return args[0] if args else None
+            else:
+                raise RuntimeError(f"Step '{self.name}' was rejected by user.")
+        else:
+            # Non-interactive mode - check for a file or env var?
+            # For now, we'll just raise an error saying manual intervention needed
+            # In a real system, this would block/suspend the workflow state.
+            raise RuntimeError(
+                "Manual approval required but running in non-interactive mode. Implement persistent state storage to handle async approvals.",
+            )
+
+
+def approval(name: str = "approval", **kwargs):
+    """Decorator/helper to create an approval step."""
+    return ApprovalStep(name, **kwargs)

flowyml/core/cache.py

@@ -0,0 +1,203 @@
+"""Cache Module - Intelligent caching strategies for pipeline steps."""
+
+import json
+import pickle
+import hashlib
+from pathlib import Path
+from typing import Any
+from datetime import datetime
+from dataclasses import dataclass, asdict
+
+
+@dataclass
+class CacheEntry:
+    """A cache entry with metadata."""
+
+    key: str
+    value: Any
+    created_at: datetime
+    step_name: str
+    code_hash: str
+    input_hash: str | None = None
+    size_bytes: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        data = asdict(self)
+        data["created_at"] = self.created_at.isoformat()
+        return data
+
+
+class CacheStrategy:
+    """Base class for caching strategies."""
+
+    def get_key(self, step_name: str, inputs: dict[str, Any], code_hash: str) -> str:
+        """Generate cache key."""
+        raise NotImplementedError
+
+    def should_cache(self, step_name: str) -> bool:
+        """Determine if step should be cached."""
+        return True
+
+
+class CodeHashCache(CacheStrategy):
+    """Cache based on function code hash."""
+
+    def get_key(self, step_name: str, inputs: dict[str, Any], code_hash: str) -> str:
+        return f"{step_name}:{code_hash}"
+
+
+class InputHashCache(CacheStrategy):
+    """Cache based on input hash."""
+
+    def get_key(self, step_name: str, inputs: dict[str, Any], code_hash: str) -> str:
+        input_str = json.dumps(inputs, sort_keys=True, default=str)
+        input_hash = hashlib.sha256(input_str.encode()).hexdigest()[:16]
+        return f"{step_name}:{code_hash}:{input_hash}"
+
+
+class CacheStore:
+    """Local cache storage for pipeline steps."""
+
+    def __init__(self, cache_dir: str = ".flowyml/cache"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.metadata_file = self.cache_dir / "metadata.json"
+        self.metadata = self._load_metadata()
+
+        # Statistics
+        self.hits = 0
+        self.misses = 0
+
+    def _load_metadata(self) -> dict[str, dict[str, Any]]:
+        """Load cache metadata."""
+        if self.metadata_file.exists():
+            with open(self.metadata_file) as f:
+                return json.load(f)
+        return {}
+
+    def _save_metadata(self) -> None:
+        """Save cache metadata."""
+        with open(self.metadata_file, "w") as f:
+            json.dump(self.metadata, f, indent=2)
+
+    def _get_cache_path(self, key: str) -> Path:
+        """Get path for cache file."""
+        # Use hash to avoid filesystem issues with long keys
+        key_hash = hashlib.sha256(key.encode()).hexdigest()
+        return self.cache_dir / f"{key_hash}.pkl"
+
+    def get(self, key: str) -> Any | None:
+        """Retrieve value from cache.
+
+        Args:
+            key: Cache key
+
+        Returns:
+            Cached value or None if not found
+        """
+        cache_path = self._get_cache_path(key)
+
+        if not cache_path.exists():
+            self.misses += 1
+            return None
+
+        try:
+            with open(cache_path, "rb") as f:
+                value = pickle.load(f)
+            self.hits += 1
+            return value
+        except Exception:
+            self.misses += 1
+            return None
+
+    def set_value(self, key: str, value: Any, step_name: str, code_hash: str, input_hash: str | None = None) -> None:
+        """Store value in cache.
+
+        Args:
+            key: Cache key
+            value: Value to cache
+            step_name: Name of the step
+            code_hash: Hash of step code
+            input_hash: Hash of inputs (optional)
+        """
+        cache_path = self._get_cache_path(key)
+
+        try:
+            with open(cache_path, "wb") as f:
+                pickle.dump(value, f)
+
+            # Update metadata
+            size_bytes = cache_path.stat().st_size
+            self.metadata[key] = {
+                "step_name": step_name,
+                "code_hash": code_hash,
+                "input_hash": input_hash,
+                "created_at": datetime.now().isoformat(),
+                "size_bytes": size_bytes,
+                "file": str(cache_path.name),
+            }
+            self._save_metadata()
+
+        except Exception:
+            pass
+
+    def invalidate(self, key: str | None = None, step_name: str | None = None) -> None:
+        """Invalidate cache entries.
+
+        Args:
+            key: Specific cache key to invalidate
+            step_name: Invalidate all entries for a step
+        """
+        if key:
+            cache_path = self._get_cache_path(key)
+            if cache_path.exists():
+                cache_path.unlink()
+            if key in self.metadata:
+                del self.metadata[key]
+
+        elif step_name:
+            keys_to_remove = [k for k, v in self.metadata.items() if v["step_name"] == step_name]
+            for k in keys_to_remove:
+                cache_path = self._get_cache_path(k)
+                if cache_path.exists():
+                    cache_path.unlink()
+                del self.metadata[k]
+
+        self._save_metadata()
+
+    def clear(self) -> None:
+        """Clear all cache entries."""
+        for cache_file in self.cache_dir.glob("*.pkl"):
+            cache_file.unlink()
+        self.metadata = {}
+        self._save_metadata()
+
+    def stats(self) -> dict[str, Any]:
+        """Get cache statistics.
+
+        Returns:
+            Dictionary with cache statistics
+        """
+        total_size = sum(v["size_bytes"] for v in self.metadata.values())
+        total_entries = len(self.metadata)
+
+        by_step = {}
+        for entry in self.metadata.values():
+            step = entry["step_name"]
+            if step not in by_step:
+                by_step[step] = {"count": 0, "size_bytes": 0}
+            by_step[step]["count"] += 1
+            by_step[step]["size_bytes"] += entry["size_bytes"]
+
+        total_requests = self.hits + self.misses
+        hit_rate = self.hits / total_requests if total_requests > 0 else 0
+
+        return {
+            "total_entries": total_entries,
+            "total_size_mb": total_size / (1024 * 1024),
+            "hits": self.hits,
+            "misses": self.misses,
+            "hit_rate": hit_rate,
+            "by_step": by_step,
+        }

flowyml/core/checkpoint.py

@@ -0,0 +1,148 @@
+"""Pipeline checkpointing for resumable execution."""
+
+import json
+import pickle
+from pathlib import Path
+from typing import Any
+from datetime import datetime
+
+
+class PipelineCheckpoint:
+    """Save and restore pipeline execution state.
+
+    Allows resuming failed pipelines from the last successful step.
+
+    Examples:
+        >>> checkpoint = PipelineCheckpoint(run_id="run_123")
+        >>> # Save state after each step
+        >>> checkpoint.save_step_state("step1", outputs)
+        >>> # Resume from checkpoint
+        >>> state = checkpoint.load()
+        >>> last_step = state["last_completed_step"]
+    """
+
+    def __init__(
+        self,
+        run_id: str,
+        checkpoint_dir: str = ".flowyml/checkpoints",
+    ):
+        self.run_id = run_id
+        self.checkpoint_dir = Path(checkpoint_dir)
+        self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+        self.checkpoint_file = self.checkpoint_dir / f"{run_id}.json"
+        self.state_dir = self.checkpoint_dir / run_id
+        self.state_dir.mkdir(exist_ok=True)
+
+    def save_step_state(
+        self,
+        step_name: str,
+        outputs: Any,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Save state after completing a step."""
+        # Save outputs
+        output_file = self.state_dir / f"{step_name}.pkl"
+        with open(output_file, "wb") as f:
+            pickle.dump(outputs, f)
+
+        # Update checkpoint metadata
+        checkpoint_data = self.load() if self.checkpoint_file.exists() else {}
+
+        checkpoint_data.update(
+            {
+                "run_id": self.run_id,
+                "last_completed_step": step_name,
+                "last_update": datetime.now().isoformat(),
+                "completed_steps": checkpoint_data.get("completed_steps", []) + [step_name],
+                "step_metadata": checkpoint_data.get("step_metadata", {}),
+            },
+        )
+
+        if metadata:
+            checkpoint_data["step_metadata"][step_name] = metadata
+
+        # Save checkpoint
+        with open(self.checkpoint_file, "w") as f:
+            json.dump(checkpoint_data, f, indent=2)
+
+    def load_step_state(self, step_name: str) -> Any:
+        """Load state for a specific step."""
+        output_file = self.state_dir / f"{step_name}.pkl"
+        if not output_file.exists():
+            raise FileNotFoundError(f"No checkpoint found for step: {step_name}")
+
+        with open(output_file, "rb") as f:
+            return pickle.load(f)
+
+    def load(self) -> dict[str, Any]:
+        """Load checkpoint metadata."""
+        if not self.checkpoint_file.exists():
+            return {}
+
+        with open(self.checkpoint_file) as f:
+            return json.load(f)
+
+    def exists(self) -> bool:
+        """Check if checkpoint exists."""
+        return self.checkpoint_file.exists()
+
+    def get_completed_steps(self) -> list:
+        """Get list of completed steps."""
+        data = self.load()
+        return data.get("completed_steps", [])
+
+    def clear(self) -> None:
+        """Clear checkpoint data."""
+        if self.checkpoint_file.exists():
+            self.checkpoint_file.unlink()
+
+        # Clear state files
+        for file in self.state_dir.glob("*.pkl"):
+            file.unlink()
+
+    def resume_point(self) -> str | None:
+        """Get the resume point (last completed step)."""
+        data = self.load()
+        return data.get("last_completed_step")
+
+
+def checkpoint_enabled_pipeline(pipeline, run_id: str):
+    """Wrap a pipeline to enable checkpointing.
+
+    This is a decorator-style wrapper that adds checkpoint functionality.
+    """
+    checkpoint = PipelineCheckpoint(run_id)
+
+    # Store original run method
+    original_run = pipeline.run
+
+    def run_with_checkpoints(*args, **kwargs):
+        """Modified run method with checkpointing."""
+        if checkpoint.exists():
+            response = input("Resume from checkpoint? [y/N]: ")
+
+            if response.lower() == "y":
+                # Load completed steps
+                checkpoint.get_completed_steps()
+
+                # In a real implementation, we would modify the execution
+                # to skip completed steps. For now, just notify.
+
+        # Run the pipeline
+        result = original_run(*args, **kwargs)
+
+        # Save final checkpoint
+        if result.success:
+            checkpoint.save_step_state(
+                "pipeline_complete",
+                result.outputs,
+                metadata={"duration": result.duration_seconds},
+            )
+
+        return result
+
+    # Replace run method
+    pipeline.run = run_with_checkpoints
+
+    return pipeline