mcli-framework 7.10.1__py3-none-any.whl → 7.10.2__py3-none-any.whl

mcli/lib/custom_commands.py

@@ -107,15 +107,25 @@ class CustomCommandManager:
         """
         Load all custom commands from the commands directory.
 
+        Automatically filters out test commands (starting with 'test_' or 'test-')
+        unless MCLI_INCLUDE_TEST_COMMANDS=true is set.
+
         Returns:
             List of command data dictionaries
         """
         commands = []
+        include_test = os.environ.get('MCLI_INCLUDE_TEST_COMMANDS', 'false').lower() == 'true'
+
         for command_file in self.commands_dir.glob("*.json"):
             # Skip the lockfile
             if command_file.name == "commands.lock.json":
                 continue
 
+            # Skip test commands unless explicitly included
+            if not include_test and command_file.stem.startswith(('test_', 'test-')):
+                logger.debug(f"Skipping test command: {command_file.name}")
+                continue
+
             command_data = self.load_command(command_file)
             if command_data:
                 commands.append(command_data)

mcli/lib/optional_deps.py

@@ -0,0 +1,240 @@
+"""
+Utilities for graceful handling of optional dependencies.
+
+This module provides helper functions and decorators to handle optional
+dependencies gracefully, with clear error messages when features are unavailable.
+"""
+
+import functools
+from typing import Any, Callable, Dict, Optional, Tuple
+
+from mcli.lib.logger.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class OptionalDependency:
+    """
+    Container for an optional dependency with availability tracking.
+
+    Example:
+        >>> ollama = OptionalDependency("ollama")
+        >>> if ollama.available:
+        ...     client = ollama.module.Client()
+    """
+
+    def __init__(
+        self,
+        module_name: str,
+        import_name: Optional[str] = None,
+        install_hint: Optional[str] = None,
+    ):
+        """
+        Initialize optional dependency handler.
+
+        Args:
+            module_name: Name of the module to import (e.g., "ollama")
+            import_name: Alternative import name if different from module_name
+            install_hint: Custom installation instruction
+        """
+        self.module_name = module_name
+        self.import_name = import_name or module_name
+        self.install_hint = install_hint or f"pip install {module_name}"
+        self.module: Optional[Any] = None
+        self.available = False
+        self.error: Optional[Exception] = None
+
+        self._try_import()
+
+    def _try_import(self):
+        """Attempt to import the module."""
+        try:
+            self.module = __import__(self.import_name)
+            self.available = True
+            logger.debug(f"Optional dependency '{self.module_name}' is available")
+        except ImportError as e:
+            self.available = False
+            self.error = e
+            logger.debug(f"Optional dependency '{self.module_name}' is not available: {e}")
+
+    def require(self, feature_name: Optional[str] = None) -> Any:
+        """
+        Require the dependency to be available, raising an error if not.
+
+        Args:
+            feature_name: Name of the feature requiring this dependency
+
+        Returns:
+            The imported module
+
+        Raises:
+            ImportError: If the dependency is not available
+        """
+        if not self.available:
+            feature_msg = f" for {feature_name}" if feature_name else ""
+            raise ImportError(
+                f"'{self.module_name}' is required{feature_msg} but not installed.\n"
+                f"Install it with: {self.install_hint}"
+            )
+        return self.module
+
+    def __getattr__(self, name: str) -> Any:
+        """Allow direct attribute access to the module."""
+        if not self.available:
+            raise ImportError(
+                f"Cannot access '{name}' from '{self.module_name}' - module not installed.\n"
+                f"Install it with: {self.install_hint}"
+            )
+        return getattr(self.module, name)
+
+
+def optional_import(
+    module_name: str, import_name: Optional[str] = None, install_hint: Optional[str] = None
+) -> Tuple[Optional[Any], bool]:
+    """
+    Try to import an optional dependency.
+
+    Args:
+        module_name: Name of the module to import
+        import_name: Alternative import name if different from module_name
+        install_hint: Custom installation instruction
+
+    Returns:
+        Tuple of (module, available) where module is None if unavailable
+
+    Example:
+        >>> ollama, OLLAMA_AVAILABLE = optional_import("ollama")
+        >>> if OLLAMA_AVAILABLE:
+        ...     client = ollama.Client()
+    """
+    dep = OptionalDependency(module_name, import_name, install_hint)
+    return (dep.module, dep.available)
+
+
+def require_dependency(
+    module_name: str, feature_name: str, install_hint: Optional[str] = None
+) -> Any:
+    """
+    Require a dependency, raising clear error if not available.
+
+    Args:
+        module_name: Name of the module to import
+        feature_name: Name of the feature requiring this dependency
+        install_hint: Custom installation instruction
+
+    Returns:
+        The imported module
+
+    Raises:
+        ImportError: If the dependency is not available
+
+    Example:
+        >>> streamlit = require_dependency("streamlit", "dashboard")
+    """
+    dep = OptionalDependency(module_name, install_hint=install_hint)
+    return dep.require(feature_name)
+
+
+def requires(*dependencies: str, install_all_hint: Optional[str] = None):
+    """
+    Decorator to mark a function as requiring specific dependencies.
+
+    Args:
+        *dependencies: Module names required by the function
+        install_all_hint: Custom installation instruction for all dependencies
+
+    Raises:
+        ImportError: If any required dependency is not available
+
+    Example:
+        >>> @requires("torch", "transformers")
+        ... def train_model():
+        ...     import torch
+        ...     import transformers
+        ...     # training code
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            missing = []
+            for dep_name in dependencies:
+                dep = OptionalDependency(dep_name)
+                if not dep.available:
+                    missing.append(dep_name)
+
+            if missing:
+                if install_all_hint:
+                    hint = install_all_hint
+                else:
+                    hint = f"pip install {' '.join(missing)}"
+
+                raise ImportError(
+                    f"Function '{func.__name__}' requires missing dependencies: {', '.join(missing)}\n"
+                    f"Install them with: {hint}"
+                )
+
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+# Common optional dependencies registry
+OPTIONAL_DEPS: Dict[str, OptionalDependency] = {}
+
+
+def register_optional_dependency(
+    module_name: str, import_name: Optional[str] = None, install_hint: Optional[str] = None
+) -> OptionalDependency:
+    """
+    Register and cache an optional dependency.
+
+    Args:
+        module_name: Name of the module to import
+        import_name: Alternative import name if different from module_name
+        install_hint: Custom installation instruction
+
+    Returns:
+        OptionalDependency instance
+    """
+    if module_name not in OPTIONAL_DEPS:
+        OPTIONAL_DEPS[module_name] = OptionalDependency(module_name, import_name, install_hint)
+    return OPTIONAL_DEPS[module_name]
+
+
+def check_dependencies(*module_names: str) -> Dict[str, bool]:
+    """
+    Check availability of multiple dependencies.
+
+    Args:
+        *module_names: Module names to check
+
+    Returns:
+        Dictionary mapping module names to availability status
+
+    Example:
+        >>> status = check_dependencies("torch", "transformers", "streamlit")
+        >>> print(status)
+        {'torch': True, 'transformers': False, 'streamlit': True}
+    """
+    return {
+        name: OptionalDependency(name).available for name in module_names
+    }
+
+
+# Pre-register common optional dependencies
+_COMMON_DEPS = {
+    "ollama": ("ollama", "pip install ollama"),
+    "streamlit": ("streamlit", "pip install streamlit"),
+    "torch": ("torch", "pip install torch"),
+    "transformers": ("transformers", "pip install transformers"),
+    "mlflow": ("mlflow", "pip install mlflow"),
+    "plotly": ("plotly", "pip install plotly"),
+    "pandas": ("pandas", "pip install pandas"),
+    "numpy": ("numpy", "pip install numpy"),
+}
+
+for module_name, (import_name, hint) in _COMMON_DEPS.items():
+    register_optional_dependency(module_name, import_name, hint)

mcli/workflow/git_commit/ai_service.py

@@ -2,11 +2,13 @@ import json
 import logging
 from typing import Any, Dict, Optional
 
-import ollama
-
 from mcli.lib.logger.logger import get_logger
+from mcli.lib.optional_deps import optional_import
 from mcli.lib.toml.toml import read_from_toml
 
+# Gracefully handle optional ollama dependency
+ollama, OLLAMA_AVAILABLE = optional_import("ollama")
+
 logger = get_logger(__name__)
 
 
@@ -204,6 +206,15 @@ Generate ONLY the commit message, nothing else:"""
     def generate_commit_message(self, changes: Dict[str, Any], diff_content: str) -> str:
         """Generate an AI-powered commit message"""
         try:
+            # Check if ollama is available
+            if not OLLAMA_AVAILABLE:
+                logger.warning(
+                    "Ollama is not installed. Install it with: pip install ollama\n"
+                    "Falling back to rule-based commit message generation."
+                )
+                analysis = self._analyze_file_patterns(changes)
+                return self._generate_fallback_message(changes, analysis)
+
             # Analyze the changes first
             analysis = self._analyze_file_patterns(changes)
 

mcli/workflow/notebook/converter.py

@@ -0,0 +1,375 @@
+"""
+Converter for transforming between MCLI workflow JSON and notebook format.
+
+This module provides bidirectional conversion between:
+1. Legacy MCLI workflow JSON format (single code field)
+2. New Jupyter-compatible notebook format (multi-cell)
+"""
+
+import json
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from mcli.lib.logger.logger import get_logger
+
+from .schema import (
+    CellLanguage,
+    CellType,
+    MCLIMetadata,
+    NotebookCell,
+    NotebookMetadata,
+    WorkflowNotebook,
+)
+
+logger = get_logger()
+
+
+class WorkflowConverter:
+    """Convert between workflow JSON and notebook formats."""
+
+    @staticmethod
+    def _split_code_into_cells(code: str, language: str = "python") -> List[NotebookCell]:
+        """
+        Split a monolithic code block into logical cells.
+
+        This attempts to intelligently split code based on:
+        - Comment markers like # %% or # CELL
+        - Function/class definitions
+        - Major logical blocks
+        """
+        cells = []
+
+        # First, try to split by cell markers (VSCode/Jupyter style)
+        cell_marker_pattern = r"^#\s*%%|^#\s*<cell>|^#\s*CELL"
+        segments = re.split(cell_marker_pattern, code, flags=re.MULTILINE)
+
+        if len(segments) > 1:
+            # Found cell markers
+            for i, segment in enumerate(segments):
+                if segment.strip():
+                    cells.append(
+                        NotebookCell(
+                            cell_type=CellType.CODE,
+                            source=segment.strip() + "\n",
+                            metadata={"language": language},
+                        )
+                    )
+        else:
+            # No cell markers, try to split intelligently by blank lines or major blocks
+            lines = code.split("\n")
+            current_cell_lines = []
+
+            for i, line in enumerate(lines):
+                current_cell_lines.append(line)
+
+                # Split on double blank lines or before major definitions
+                next_line = lines[i + 1] if i + 1 < len(lines) else ""
+                is_double_blank = line.strip() == "" and next_line.strip() == ""
+                is_major_def = (
+                    next_line.strip().startswith("def ")
+                    or next_line.strip().startswith("class ")
+                    or next_line.strip().startswith("@")
+                )
+
+                if (is_double_blank or is_major_def) and len(current_cell_lines) > 3:
+                    cell_code = "\n".join(current_cell_lines).strip()
+                    if cell_code:
+                        cells.append(
+                            NotebookCell(
+                                cell_type=CellType.CODE,
+                                source=cell_code + "\n",
+                                metadata={"language": language},
+                            )
+                        )
+                    current_cell_lines = []
+
+            # Add remaining lines as final cell
+            if current_cell_lines:
+                cell_code = "\n".join(current_cell_lines).strip()
+                if cell_code:
+                    cells.append(
+                        NotebookCell(
+                            cell_type=CellType.CODE,
+                            source=cell_code + "\n",
+                            metadata={"language": language},
+                        )
+                    )
+
+        # If no cells were created, add the entire code as one cell
+        if not cells and code.strip():
+            cells.append(
+                NotebookCell(
+                    cell_type=CellType.CODE,
+                    source=code,
+                    metadata={"language": language},
+                )
+            )
+
+        return cells
+
+    @classmethod
+    def workflow_to_notebook(
+        cls, workflow_data: Dict[str, Any], add_description: bool = True
+    ) -> WorkflowNotebook:
+        """
+        Convert legacy workflow JSON to notebook format.
+
+        Args:
+            workflow_data: Legacy workflow JSON data
+            add_description: Add description as markdown cell
+
+        Returns:
+            WorkflowNotebook instance
+        """
+        # Extract metadata
+        name = workflow_data.get("name", "untitled")
+        description = workflow_data.get("description", "")
+        group = workflow_data.get("group")
+        version = workflow_data.get("version", "1.0")
+        language = workflow_data.get("language", "python")
+        created_at = workflow_data.get("created_at")
+        updated_at = workflow_data.get("updated_at")
+        extra_metadata = workflow_data.get("metadata", {})
+
+        # Create MCLI metadata
+        mcli_metadata = MCLIMetadata(
+            name=name,
+            description=description,
+            group=group,
+            version=version,
+            language=CellLanguage(language),
+            created_at=created_at,
+            updated_at=updated_at,
+            extra=extra_metadata,
+        )
+
+        # Create notebook metadata
+        notebook_metadata = NotebookMetadata(mcli=mcli_metadata)
+
+        # Create notebook
+        notebook = WorkflowNotebook(metadata=notebook_metadata)
+
+        # Add description as markdown cell if present
+        if add_description and description:
+            notebook.add_markdown_cell(f"# {name}\n\n{description}")
+
+        # Extract and split code into cells
+        code = workflow_data.get("code", "")
+        if code:
+            cells = cls._split_code_into_cells(code, language)
+            notebook.cells.extend(cells)
+
+        return notebook
+
+    @staticmethod
+    def notebook_to_workflow(notebook: WorkflowNotebook) -> Dict[str, Any]:
+        """
+        Convert notebook format to legacy workflow JSON.
+
+        Args:
+            notebook: WorkflowNotebook instance
+
+        Returns:
+            Legacy workflow JSON data
+        """
+        mcli_meta = notebook.metadata.mcli
+
+        # Combine all code cells into single code field
+        code_parts = []
+        for cell in notebook.cells:
+            if cell.cell_type == CellType.CODE:
+                code_parts.append(cell.source_text)
+
+        # Join with cell markers for potential round-trip conversion
+        combined_code = "\n# %%\n".join(code_parts)
+
+        # Build workflow data
+        workflow_data = {
+            "name": mcli_meta.name,
+            "description": mcli_meta.description,
+            "version": mcli_meta.version,
+            "language": mcli_meta.language.value,
+            "code": combined_code,
+        }
+
+        # Add optional fields
+        if mcli_meta.group:
+            workflow_data["group"] = mcli_meta.group
+        if mcli_meta.created_at:
+            workflow_data["created_at"] = mcli_meta.created_at
+        if mcli_meta.updated_at:
+            workflow_data["updated_at"] = mcli_meta.updated_at
+        else:
+            workflow_data["updated_at"] = datetime.utcnow().isoformat() + "Z"
+
+        if mcli_meta.extra:
+            workflow_data["metadata"] = mcli_meta.extra
+
+        return workflow_data
+
+    @classmethod
+    def load_workflow_json(cls, path: Union[str, Path]) -> Dict[str, Any]:
+        """Load workflow JSON from file."""
+        path = Path(path)
+        with open(path, "r") as f:
+            return json.load(f)
+
+    @classmethod
+    def save_workflow_json(cls, data: Dict[str, Any], path: Union[str, Path]) -> None:
+        """Save workflow JSON to file."""
+        path = Path(path)
+        with open(path, "w") as f:
+            json.dump(data, f, indent=2)
+
+    @classmethod
+    def load_notebook_json(cls, path: Union[str, Path]) -> WorkflowNotebook:
+        """Load notebook from JSON file."""
+        path = Path(path)
+        with open(path, "r") as f:
+            data = json.load(f)
+
+        # Check if it's a notebook or legacy workflow format
+        if "nbformat" in data:
+            # It's already a notebook
+            return WorkflowNotebook.from_dict(data)
+        else:
+            # It's a legacy workflow, convert it
+            logger.info(f"Converting legacy workflow to notebook format: {path}")
+            return cls.workflow_to_notebook(data)
+
+    @classmethod
+    def save_notebook_json(cls, notebook: WorkflowNotebook, path: Union[str, Path]) -> None:
+        """Save notebook to JSON file."""
+        path = Path(path)
+        data = notebook.to_dict()
+        with open(path, "w") as f:
+            json.dump(data, f, indent=2)
+
+    @classmethod
+    def convert_file_to_notebook(
+        cls, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None
+    ) -> Path:
+        """
+        Convert a workflow JSON file to notebook format.
+
+        Args:
+            input_path: Path to legacy workflow JSON
+            output_path: Optional output path (defaults to same path)
+
+        Returns:
+            Path to the converted notebook file
+        """
+        input_path = Path(input_path)
+        output_path = Path(output_path) if output_path else input_path
+
+        # Load legacy workflow
+        workflow_data = cls.load_workflow_json(input_path)
+
+        # Convert to notebook
+        notebook = cls.workflow_to_notebook(workflow_data)
+
+        # Save notebook
+        cls.save_notebook_json(notebook, output_path)
+
+        logger.info(f"Converted {input_path} to notebook format at {output_path}")
+        return output_path
+
+    @classmethod
+    def convert_file_to_workflow(
+        cls, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None
+    ) -> Path:
+        """
+        Convert a notebook file to legacy workflow JSON format.
+
+        Args:
+            input_path: Path to notebook JSON
+            output_path: Optional output path (defaults to same path)
+
+        Returns:
+            Path to the converted workflow file
+        """
+        input_path = Path(input_path)
+        output_path = Path(output_path) if output_path else input_path
+
+        # Load notebook
+        notebook = cls.load_notebook_json(input_path)
+
+        # Convert to workflow
+        workflow_data = cls.notebook_to_workflow(notebook)
+
+        # Save workflow
+        cls.save_workflow_json(workflow_data, output_path)
+
+        logger.info(f"Converted {input_path} to workflow format at {output_path}")
+        return output_path
+
+    @classmethod
+    def migrate_directory(
+        cls, directory: Union[str, Path], backup: bool = True, in_place: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Migrate all workflow JSON files in a directory to notebook format.
+
+        Args:
+            directory: Directory containing workflow JSON files
+            backup: Create backup files before conversion
+            in_place: Convert files in place (vs creating new files)
+
+        Returns:
+            Dictionary with migration results
+        """
+        directory = Path(directory)
+        results = {
+            "total": 0,
+            "converted": 0,
+            "failed": 0,
+            "skipped": 0,
+            "files": [],
+        }
+
+        for json_file in directory.glob("*.json"):
+            # Skip lockfile and already-converted notebooks
+            if json_file.name == "commands.lock.json":
+                continue
+
+            try:
+                # Load and check if already a notebook
+                with open(json_file, "r") as f:
+                    data = json.load(f)
+
+                results["total"] += 1
+
+                if "nbformat" in data:
+                    # Already a notebook
+                    results["skipped"] += 1
+                    logger.debug(f"Skipping {json_file.name} - already a notebook")
+                    continue
+
+                # Backup if requested
+                if backup:
+                    backup_path = json_file.with_suffix(".json.bak")
+                    cls.save_workflow_json(data, backup_path)
+                    logger.debug(f"Created backup: {backup_path}")
+
+                # Convert to notebook
+                if in_place:
+                    output_path = json_file
+                else:
+                    output_path = json_file.with_stem(f"{json_file.stem}.notebook")
+
+                cls.convert_file_to_notebook(json_file, output_path)
+
+                results["converted"] += 1
+                results["files"].append(str(json_file))
+
+            except Exception as e:
+                logger.error(f"Failed to convert {json_file}: {e}")
+                results["failed"] += 1
+
+        logger.info(
+            f"Migration complete: {results['converted']} converted, "
+            f"{results['skipped']} skipped, {results['failed']} failed"
+        )
+        return results