flowyml 1.1.0__py3-none-any.whl

flowyml/utils/git.py

@@ -0,0 +1,319 @@
+"""Git integration utilities for experiment tracking."""
+
+import subprocess
+from pathlib import Path
+from typing import Any
+
+
+class GitInfo:
+    """Git repository information."""
+
+    def __init__(
+        self,
+        commit_hash: str | None = None,
+        branch: str | None = None,
+        is_dirty: bool = False,
+        remote_url: str | None = None,
+        author: str | None = None,
+        commit_message: str | None = None,
+        commit_time: str | None = None,
+    ):
+        self.commit_hash = commit_hash
+        self.branch = branch
+        self.is_dirty = is_dirty
+        self.remote_url = remote_url
+        self.author = author
+        self.commit_message = commit_message
+        self.commit_time = commit_time
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "commit_hash": self.commit_hash,
+            "branch": self.branch,
+            "is_dirty": self.is_dirty,
+            "remote_url": self.remote_url,
+            "author": self.author,
+            "commit_message": self.commit_message,
+            "commit_time": self.commit_time,
+        }
+
+    @property
+    def is_available(self) -> bool:
+        """Check if git info is available."""
+        return self.commit_hash is not None
+
+
+def run_git_command(command: list, cwd: Path | None = None) -> str | None:
+    """Run a git command and return output.
+
+    Args:
+        command: Git command as list of strings
+        cwd: Working directory
+
+    Returns:
+        Command output or None if failed
+    """
+    try:
+        result = subprocess.run(
+            ["git"] + command,
+            cwd=cwd or Path.cwd(),
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if result.returncode == 0:
+            return result.stdout.strip()
+        return None
+    except (subprocess.TimeoutExpired, FileNotFoundError, Exception):
+        return None
+
+
+def is_git_repo(path: Path | None = None) -> bool:
+    """Check if directory is a git repository.
+
+    Args:
+        path: Directory to check
+
+    Returns:
+        True if directory is a git repository
+    """
+    result = run_git_command(["rev-parse", "--git-dir"], cwd=path)
+    return result is not None
+
+
+def get_commit_hash(path: Path | None = None) -> str | None:
+    """Get current commit hash.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        Commit hash or None
+    """
+    return run_git_command(["rev-parse", "HEAD"], cwd=path)
+
+
+def get_short_commit_hash(path: Path | None = None) -> str | None:
+    """Get short commit hash.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        Short commit hash or None
+    """
+    return run_git_command(["rev-parse", "--short", "HEAD"], cwd=path)
+
+
+def get_branch_name(path: Path | None = None) -> str | None:
+    """Get current branch name.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        Branch name or None
+    """
+    return run_git_command(["rev-parse", "--abbrev-ref", "HEAD"], cwd=path)
+
+
+def is_dirty(path: Path | None = None) -> bool:
+    """Check if repository has uncommitted changes.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        True if repository is dirty
+    """
+    result = run_git_command(["status", "--porcelain"], cwd=path)
+    return bool(result)
+
+
+def get_remote_url(path: Path | None = None, remote: str = "origin") -> str | None:
+    """Get remote repository URL.
+
+    Args:
+        path: Repository path
+        remote: Remote name
+
+    Returns:
+        Remote URL or None
+    """
+    return run_git_command(["config", "--get", f"remote.{remote}.url"], cwd=path)
+
+
+def get_commit_author(path: Path | None = None) -> str | None:
+    """Get author of current commit.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        Commit author or None
+    """
+    return run_git_command(["log", "-1", "--format=%an <%ae>"], cwd=path)
+
+
+def get_commit_message(path: Path | None = None) -> str | None:
+    """Get message of current commit.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        Commit message or None
+    """
+    return run_git_command(["log", "-1", "--format=%s"], cwd=path)
+
+
+def get_commit_time(path: Path | None = None) -> str | None:
+    """Get timestamp of current commit.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        Commit timestamp or None
+    """
+    return run_git_command(["log", "-1", "--format=%ci"], cwd=path)
+
+
+def get_diff(path: Path | None = None, staged: bool = False) -> str | None:
+    """Get diff of uncommitted changes.
+
+    Args:
+        path: Repository path
+        staged: Get staged changes only
+
+    Returns:
+        Diff output or None
+    """
+    command = ["diff"]
+    if staged:
+        command.append("--cached")
+
+    return run_git_command(command, cwd=path)
+
+
+def get_git_info(path: Path | None = None) -> GitInfo:
+    """Get comprehensive git information.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        GitInfo object with repository information
+    """
+    if not is_git_repo(path):
+        return GitInfo()
+
+    return GitInfo(
+        commit_hash=get_commit_hash(path),
+        branch=get_branch_name(path),
+        is_dirty=is_dirty(path),
+        remote_url=get_remote_url(path),
+        author=get_commit_author(path),
+        commit_message=get_commit_message(path),
+        commit_time=get_commit_time(path),
+    )
+
+
+def save_git_snapshot(output_dir: Path, path: Path | None = None) -> None:
+    """Save git repository snapshot.
+
+    Args:
+        output_dir: Directory to save snapshot
+        path: Repository path
+    """
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Save git info
+    git_info = get_git_info(path)
+    if git_info.is_available:
+        import json
+
+        with open(output_dir / "git_info.json", "w") as f:
+            json.dump(git_info.to_dict(), f, indent=2)
+
+    # Save diff if dirty
+    if git_info.is_dirty:
+        diff = get_diff(path)
+        if diff:
+            with open(output_dir / "git_diff.patch", "w") as f:
+                f.write(diff)
+
+
+def get_file_commit_history(
+    file_path: str,
+    max_count: int = 10,
+    path: Path | None = None,
+) -> list[dict[str, str]]:
+    """Get commit history for a specific file.
+
+    Args:
+        file_path: Path to file
+        max_count: Maximum number of commits to return
+        path: Repository path
+
+    Returns:
+        List of commit dictionaries
+    """
+    if not is_git_repo(path):
+        return []
+
+    log_format = "%H|%an|%ae|%ci|%s"
+    result = run_git_command(
+        ["log", f"--max-count={max_count}", f"--format={log_format}", "--", file_path],
+        cwd=path,
+    )
+
+    if not result:
+        return []
+
+    commits = []
+    for line in result.split("\n"):
+        if not line:
+            continue
+
+        parts = line.split("|", 4)
+        if len(parts) == 5:
+            commits.append(
+                {
+                    "hash": parts[0],
+                    "author_name": parts[1],
+                    "author_email": parts[2],
+                    "timestamp": parts[3],
+                    "message": parts[4],
+                },
+            )
+
+    return commits
+
+
+def get_tags(path: Path | None = None) -> list[str]:
+    """Get list of git tags.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        List of tag names
+    """
+    result = run_git_command(["tag", "--list"], cwd=path)
+    if result:
+        return [tag for tag in result.split("\n") if tag]
+    return []
+
+
+def get_current_tag(path: Path | None = None) -> str | None:
+    """Get tag pointing to current commit.
+
+    Args:
+        path: Repository path
+
+    Returns:
+        Tag name or None
+    """
+    return run_git_command(["describe", "--exact-match", "--tags"], cwd=path)

flowyml/utils/logging.py

@@ -0,0 +1,61 @@
+"""Logging utilities for flowyml."""
+
+import logging
+import sys
+from pathlib import Path
+
+
+def setup_logger(
+    name: str = "flowyml",
+    level: int = logging.INFO,
+    log_file: str = None,
+) -> logging.Logger:
+    """Setup logger for flowyml.
+
+    Args:
+        name: Logger name
+        level: Logging level
+        log_file: Optional log file path
+
+    Returns:
+        Configured logger
+    """
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+
+    # Console handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(level)
+    console_format = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    )
+    console_handler.setFormatter(console_format)
+    logger.addHandler(console_handler)
+
+    # File handler if specified
+    if log_file:
+        log_path = Path(log_file)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setLevel(level)
+        file_handler.setFormatter(console_format)
+        logger.addHandler(file_handler)
+
+    return logger
+
+
+# Default logger
+logger = setup_logger()
+
+
+def get_logger(name: str = "flowyml") -> logging.Logger:
+    """Get a logger by name.
+
+    Args:
+        name: Logger name
+
+    Returns:
+        Logger instance
+    """
+    return logging.getLogger(name)

flowyml/utils/performance.py

@@ -0,0 +1,314 @@
+"""Performance optimization utilities."""
+
+import functools
+from typing import Any
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
+import multiprocessing as mp
+
+
+class LazyValue:
+    """Lazy evaluation wrapper.
+
+    Computes value only when accessed.
+
+    Examples:
+        >>> lazy = LazyValue(lambda: expensive_computation())
+        >>> # Not computed yet
+        >>> result = lazy.value  # Now computed
+    """
+
+    def __init__(self, compute_func: Callable):
+        self._compute_func = compute_func
+        self._value = None
+        self._computed = False
+
+    @property
+    def value(self):
+        """Get the computed value."""
+        if not self._computed:
+            self._value = self._compute_func()
+            self._computed = True
+        return self._value
+
+    def __repr__(self):
+        if self._computed:
+            return f"LazyValue(computed={self._value})"
+        return "LazyValue(not computed)"
+
+
+def lazy_property(func):
+    """Decorator for lazy property evaluation.
+
+    Example:
+        >>> class Model:
+        ...     @lazy_property
+        ...     def expensive_data(self):
+        ...         return load_large_dataset()
+    """
+    attr_name = "_lazy_" + func.__name__
+
+    @property
+    @functools.wraps(func)
+    def wrapper(self):
+        if not hasattr(self, attr_name):
+            setattr(self, attr_name, func(self))
+        return getattr(self, attr_name)
+
+    return wrapper
+
+
+class ParallelExecutor:
+    """Smart parallelization for pipeline steps.
+
+    Automatically determines best parallelization strategy.
+
+    Examples:
+        >>> executor = ParallelExecutor(max_workers=4)
+        >>> results = executor.map_items(process_item, items)
+    """
+
+    def __init__(
+        self,
+        max_workers: int | None = None,
+        mode: str = "thread",  # 'thread' or 'process'
+    ):
+        self.max_workers = max_workers or mp.cpu_count()
+        self.mode = mode
+
+    def map_items(self, func: Callable, items: list) -> list:
+        """Parallel map operation.
+
+        Args:
+            func: Function to apply
+            items: Items to process
+
+        Returns:
+            List of results
+        """
+        executor_class = ProcessPoolExecutor if self.mode == "process" else ThreadPoolExecutor
+
+        with executor_class(max_workers=self.max_workers) as executor:
+            results = list(executor.map(func, items))
+
+        return results
+
+    def submit(self, func: Callable, *args, **kwargs):
+        """Submit a single task."""
+        executor = ProcessPoolExecutor(max_workers=1) if self.mode == "process" else ThreadPoolExecutor(max_workers=1)
+
+        future = executor.submit(func, *args, **kwargs)
+        return future
+
+
+class IncrementalComputation:
+    """Incremental computation for data processing.
+
+    Processes data in chunks and caches intermediate results.
+
+    Examples:
+        >>> computer = IncrementalComputation(chunk_size=1000)
+        >>> result = computer.compute(large_dataset, process_chunk)
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 1000,
+        cache_dir: str = ".flowyml/incremental",
+    ):
+        self.chunk_size = chunk_size
+        from pathlib import Path
+
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def compute(
+        self,
+        data: list,
+        func: Callable,
+        aggregate_func: Callable | None = None,
+    ) -> Any:
+        """Compute incrementally over data chunks.
+
+        Args:
+            data: Input data
+            func: Function to apply to each chunk
+            aggregate_func: Optional function to aggregate chunk results
+
+        Returns:
+            Aggregated result
+        """
+        results = []
+
+        # Process in chunks
+        for i in range(0, len(data), self.chunk_size):
+            chunk = data[i : i + self.chunk_size]
+
+            # Check cache
+            cache_key = f"chunk_{i}"
+            cache_file = self.cache_dir / f"{cache_key}.pkl"
+
+            if cache_file.exists():
+                import pickle
+
+                with open(cache_file, "rb") as f:
+                    chunk_result = pickle.load(f)
+            else:
+                # Compute
+                chunk_result = func(chunk)
+
+                # Cache result
+                import pickle
+
+                with open(cache_file, "wb") as f:
+                    pickle.dump(chunk_result, f)
+
+            results.append(chunk_result)
+
+        # Aggregate
+        if aggregate_func:
+            return aggregate_func(results)
+        return results
+
+
+class GPUResourceManager:
+    """GPU resource management and allocation.
+
+    Helps manage GPU memory and device placement.
+
+    Examples:
+        >>> gpu = GPUResourceManager()
+        >>> if gpu.has_gpu():
+        ...     with gpu.allocate_device(0):
+        ...         # Run GPU operations
+        ...         pass
+    """
+
+    def __init__(self):
+        self._check_gpu_availability()
+
+    def _check_gpu_availability(self) -> None:
+        """Check for GPU availability."""
+        try:
+            import torch
+
+            self.has_torch = True
+            self.torch_available = torch.cuda.is_available()
+            self.torch_device_count = torch.cuda.device_count() if self.torch_available else 0
+        except ImportError:
+            self.has_torch = False
+            self.torch_available = False
+            self.torch_device_count = 0
+
+        try:
+            import tensorflow as tf
+
+            self.has_tf = True
+            gpus = tf.config.list_physical_devices("GPU")
+            self.tf_available = len(gpus) > 0
+            self.tf_device_count = len(gpus)
+        except ImportError:
+            self.has_tf = False
+            self.tf_available = False
+            self.tf_device_count = 0
+
+    def has_gpu(self) -> bool:
+        """Check if GPU is available."""
+        return self.torch_available or self.tf_available
+
+    def get_device_count(self) -> int:
+        """Get number of available GPUs."""
+        return max(self.torch_device_count, self.tf_device_count)
+
+    def allocate_device(self, device_id: int = 0):
+        """Context manager to allocate specific GPU."""
+        import os
+
+        class DeviceContext:
+            def __enter__(ctx_self):
+                os.environ["CUDA_VISIBLE_DEVICES"] = str(device_id)
+                return device_id
+
+            def __exit__(ctx_self, *args):
+                # Reset
+                if "CUDA_VISIBLE_DEVICES" in os.environ:
+                    del os.environ["CUDA_VISIBLE_DEVICES"]
+
+        return DeviceContext()
+
+    def get_memory_info(self, device_id: int = 0) -> dict:
+        """Get GPU memory information."""
+        if not self.has_gpu():
+            return {"available": False}
+
+        info = {"available": True}
+
+        if self.has_torch:
+            import torch
+
+            if self.torch_available:
+                info["torch"] = {
+                    "allocated": torch.cuda.memory_allocated(device_id),
+                    "cached": torch.cuda.memory_reserved(device_id),
+                }
+
+        if self.has_tf and self.tf_available:
+            # TensorFlow memory info
+            pass
+
+        return info
+
+    def clear_cache(self, device_id: int = 0) -> None:
+        """Clear GPU cache."""
+        if self.has_torch and self.torch_available:
+            import torch
+
+            torch.cuda.empty_cache()
+
+
+def optimize_dataframe(df, inplace: bool = False):
+    """Optimize pandas DataFrame memory usage.
+
+    Args:
+        df: DataFrame to optimize
+        inplace: Whether to modify inplace
+
+    Returns:
+        Optimized DataFrame
+    """
+    import pandas as pd
+
+    if not inplace:
+        df = df.copy()
+
+    # Optimize integers
+    for col in df.select_dtypes(include=["int"]).columns:
+        df[col] = pd.to_numeric(df[col], downcast="integer")
+
+    # Optimize floats
+    for col in df.select_dtypes(include=["float"]).columns:
+        df[col] = pd.to_numeric(df[col], downcast="float")
+
+    # Convert objects to categories if appropriate
+    for col in df.select_dtypes(include=["object"]).columns:
+        num_unique = df[col].nunique()
+        num_total = len(df[col])
+
+        if num_unique / num_total < 0.5:  # Less than 50% unique
+            df[col] = df[col].astype("category")
+
+    return df
+
+
+def batch_iterator(items: list, batch_size: int):
+    """Iterate over items in batches.
+
+    Args:
+        items: List of items
+        batch_size: Size of each batch
+
+    Yields:
+        Batches of items
+    """
+    for i in range(0, len(items), batch_size):
+        yield items[i : i + batch_size]