comfygit-core 0.2.0__py3-none-any.whl

comfygit_core/analyzers/custom_node_scanner.py

@@ -0,0 +1,109 @@
+"""Simple custom node scanner for finding dependencies in nodes."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class NodeDependencies:
+    """Dependencies found in a custom node."""
+    requirements_file: Path | None = None
+    requirements: list[str] | None = None
+    pyproject_file: Path | None = None
+    has_install_script: bool = False
+
+
+class CustomNodeScanner:
+    """Scans custom nodes for dependencies and metadata."""
+
+    def scan_node(self, node_path: Path) -> NodeDependencies:
+        """Scan a custom node directory for dependencies.
+        
+        Args:
+            node_path: Path to the custom node directory
+            
+        Returns:
+            NodeDependencies with found requirements
+        """
+        result = NodeDependencies()
+
+        if not node_path.exists() or not node_path.is_dir():
+            return result
+
+        # Look for requirements.txt (primary source)
+        req_file = node_path / "requirements.txt"
+        if req_file.exists():
+            result.requirements_file = req_file
+            result.requirements = self._read_requirements(req_file)
+
+        # Check for pyproject.toml (secondary)
+        pyproject_file = node_path / "pyproject.toml"
+        if pyproject_file.exists():
+            result.pyproject_file = pyproject_file
+
+        # Check for install scripts
+        for script_name in ["install.py", "install.sh", "setup.py"]:
+            if (node_path / script_name).exists():
+                result.has_install_script = True
+                break
+
+        return result
+
+    def _read_requirements(self, req_file: Path) -> list[str]:
+        """Read and parse requirements.txt file.
+
+        Strips inline comments (e.g., 'gdown # comment') to ensure PEP 508 compliance.
+
+        Args:
+            req_file: Path to requirements.txt
+
+        Returns:
+            List of requirement strings with inline comments removed
+        """
+        requirements = []
+
+        try:
+            with open(req_file, encoding='utf-8') as f:
+                for line in f:
+                    line = line.strip()
+                    # Skip full-line comments and empty lines
+                    if line and not line.startswith('#'):
+                        # Handle -r includes (just note them for now)
+                        if line.startswith('-r '):
+                            # TODO: Handle recursive requirements
+                            continue
+
+                        # Strip inline comments (everything after #)
+                        # Example: "gdown # supports downloading" -> "gdown"
+                        if '#' in line:
+                            line = line.split('#', 1)[0].strip()
+
+                        # Only add if there's content after stripping
+                        if line:
+                            requirements.append(line)
+        except Exception:
+            # Return empty list on any read error
+            pass
+
+        return requirements
+
+    def find_all_requirements(self, node_path: Path) -> list[Path]:
+        """Find all requirements files in a node (including subdirectories).
+        
+        Args:
+            node_path: Path to the custom node directory
+            
+        Returns:
+            List of paths to requirements files
+        """
+        if not node_path.exists() or not node_path.is_dir():
+            return []
+
+        # Look for requirements files
+        patterns = ["requirements*.txt", "requirements/*.txt"]
+        req_files = []
+
+        for pattern in patterns:
+            req_files.extend(node_path.glob(pattern))
+
+        return req_files

comfygit_core/analyzers/git_change_parser.py

@@ -0,0 +1,156 @@
+"""Parse git changes in pyproject.toml files to extract what was modified."""
+
+from pathlib import Path
+from typing import Any
+
+import tomlkit
+
+from ..logging.logging_config import get_logger
+from ..models.environment import GitStatus
+from ..utils.dependency_parser import compare_dependency_sets, extract_all_dependencies
+from ..utils.git import git_show
+
+logger = get_logger(__name__)
+
+
+class GitChangeParser:
+    """Parse git changes to pyproject.toml to identify what was modified."""
+
+    def __init__(self, repo_path: Path):
+        """Initialize the parser.
+
+        Args:
+            repo_path: Path to git repository (.cec directory)
+        """
+        self.repo_path = repo_path
+
+    def parse_changes(self, current_config: dict) -> dict[str, Any]:
+        """Parse git changes comparing HEAD to current.
+
+        Args:
+            current_config: Current pyproject.toml configuration
+
+        Returns:
+            Dict with changes categorized by type
+        """
+        changes = {
+            'nodes_added': [],
+            'nodes_removed': [],
+            'dependencies_added': [],
+            'dependencies_removed': [],
+            'dependencies_updated': [],
+            'constraints_added': [],
+            'constraints_removed': [],
+        }
+
+        try:
+            # Get the last committed version
+            committed_content = git_show(
+                self.repo_path,
+                "HEAD",
+                Path("pyproject.toml"),
+                is_text=True
+            )
+
+            if not committed_content:
+                return changes
+
+            committed_config = tomlkit.loads(committed_content)
+
+            # Compare each category
+            self._compare_nodes(committed_config, current_config, changes)
+            self._compare_dependencies(committed_config, current_config, changes)
+            self._compare_constraints(committed_config, current_config, changes)
+
+        except (ValueError, OSError) as e:
+            # No previous commit or file doesn't exist in HEAD
+            logger.debug(f"Could not get previous pyproject.toml: {e}")
+            # This is fine - might be the first commit
+
+        return changes
+
+
+    def update_git_status(self, status: GitStatus, current_config: dict) -> None:
+        """Update a GitStatus object with parsed changes.
+
+        Args:
+            status: GitStatus object to update
+            current_config: Current pyproject.toml configuration
+        """
+        changes = self.parse_changes(current_config)
+
+        status.nodes_added = changes['nodes_added']
+        status.nodes_removed = changes['nodes_removed']
+        status.dependencies_added = changes['dependencies_added']
+        status.dependencies_removed = changes['dependencies_removed']
+        status.dependencies_updated = changes['dependencies_updated']
+        status.constraints_added = changes['constraints_added']
+        status.constraints_removed = changes['constraints_removed']
+
+    def _compare_nodes(self, old_config: dict, new_config: dict, changes: dict) -> None:
+        """Compare custom nodes between configs."""
+        old_nodes = old_config.get('tool', {}).get('comfygit', {}).get('nodes', {})
+        new_nodes = new_config.get('tool', {}).get('comfygit', {}).get('nodes', {})
+
+        # Flatten old nodes (handle legacy 'development' section)
+        old_flat = self._flatten_nodes(old_nodes)
+        new_flat = self._flatten_nodes(new_nodes)
+
+        old_keys = set(old_flat.keys())
+        new_keys = set(new_flat.keys())
+
+        for key in new_keys - old_keys:
+            node_data = new_flat[key]
+            node_name = node_data.get('name', key)
+            is_development = node_data.get('version') == 'dev'
+            changes['nodes_added'].append({
+                'name': node_name,
+                'is_development': is_development
+            })
+
+        for key in old_keys - new_keys:
+            node_data = old_flat[key]
+            node_name = node_data.get('name', key)
+            is_development = node_data.get('version') == 'dev'
+            changes['nodes_removed'].append({
+                'name': node_name,
+                'is_development': is_development
+            })
+
+    def _flatten_nodes(self, nodes_config: dict) -> dict:
+        """Flatten nodes, handling legacy 'development' section."""
+        flat = {}
+        for key, value in nodes_config.items():
+            if key == 'development' and isinstance(value, dict):
+                # Legacy development section - flatten it
+                for dev_key, dev_value in value.items():
+                    if isinstance(dev_value, dict):
+                        flat[dev_key] = dev_value
+            elif isinstance(value, dict) and 'name' in value:
+                # Regular node
+                flat[key] = value
+        return flat
+
+    def _compare_dependencies(self, old_config: dict, new_config: dict, changes: dict) -> None:
+        """Compare Python dependencies using existing utilities."""
+        old_deps = extract_all_dependencies(old_config)
+        new_deps = extract_all_dependencies(new_config)
+
+        dep_changes = compare_dependency_sets(old_deps, new_deps)
+        changes['dependencies_added'] = dep_changes.get('added', [])
+        changes['dependencies_removed'] = dep_changes.get('removed', [])
+        changes['dependencies_updated'] = dep_changes.get('updated', [])
+
+    def _compare_constraints(self, old_config: dict, new_config: dict, changes: dict) -> None:
+        """Compare UV constraint dependencies."""
+        old_constraints = old_config.get('tool', {}).get('uv', {}).get('constraint-dependencies', [])
+        new_constraints = new_config.get('tool', {}).get('uv', {}).get('constraint-dependencies', [])
+
+        old_set = set(old_constraints)
+        new_set = set(new_constraints)
+
+        changes['constraints_added'] = list(new_set - old_set)
+        changes['constraints_removed'] = list(old_set - new_set)
+
+    # Workflow tracking comparison removed - all workflows are auto-managed
+

comfygit_core/analyzers/model_scanner.py

@@ -0,0 +1,318 @@
+"""ModelScanner - Model file discovery, hashing, and indexing operations."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+
+from ..configs.model_config import ModelConfig
+from ..logging.logging_config import get_logger
+from ..models.exceptions import ComfyDockError
+from ..models.shared import ModelInfo
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..repositories.model_repository import ModelRepository
+
+logger = get_logger(__name__)
+
+
+class ModelProcessResult(Enum):
+    """Result of processing a single model file."""
+    ADDED = "added"
+    UPDATED_PATH = "updated_path"
+    SKIPPED_DUPLICATE = "skipped_duplicate"
+    SKIPPED_SAME_FILE = "skipped_same"
+    COLLISION_RESOLVED = "collision_resolved"
+
+
+# Extensions that are definitely not model files
+EXCLUDED_EXTENSIONS = {
+    '.txt', '.md',
+    '.lock', '.gitignore', '.gitattributes',
+    '.log', '.html', '.xml',
+    '.py', '.js', '.ts', '.sh', '.bat', '.ps1',
+    '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg',
+    '.mp4', '.avi', '.mov', '.webm', '.mp3', '.wav',
+    '.zip', '.tar', '.gz', '.rar', '.7z',
+    '.pdf', '.doc', '.docx', '.xls', '.xlsx',
+    '.DS_Store', '.env',
+}
+
+# Minimum file size in bytes
+MIN_MODEL_SIZE = 8
+
+# Minimum file size for model files (8 bytes)
+
+
+@dataclass
+class ScanResult:
+    """Result of model scanning operation."""
+    scanned_count: int
+    added_count: int
+    updated_count: int
+    skipped_count: int
+    error_count: int
+    errors: list[str]
+    removed_count: int = 0
+
+
+class ModelScanProgress:
+    """Callback protocol for model scan progress updates."""
+
+    def on_scan_start(self, total_files: int) -> None:
+        """Called when scan starts with total file count."""
+        pass
+
+    def on_file_processed(self, current: int, total: int, filename: str) -> None:
+        """Called after each file is processed."""
+        pass
+
+    def on_scan_complete(self, result: ScanResult) -> None:
+        """Called when scan completes."""
+        pass
+
+
+class ModelScanner:
+    """Model file discovery, hashing, and indexing operations."""
+
+    def __init__(self, index_manager: ModelRepository, model_config: ModelConfig | None = None):
+        """Initialize ModelScanner.
+
+        Args:
+            index_manager: ModelIndexManager for database operations
+            model_config: ModelConfig for extension filtering, loads default if None
+        """
+        self.index_manager = index_manager
+        self.model_config = model_config or ModelConfig.load()
+        self.quiet = False
+
+    def scan_directory(self, models_dir: Path, quiet: bool = False, progress: ModelScanProgress | None = None) -> ScanResult:
+        """Scan single models directory for all model files.
+
+        Args:
+            models_dir: Path to models directory to scan
+            quiet: Suppress logging
+            progress: Optional progress callback
+
+        Returns:
+            ScanResult with operation statistics
+        """
+        self.quiet = quiet
+
+        if not models_dir.exists():
+            raise ComfyDockError(f"Models directory does not exist: {models_dir}")
+        if not models_dir.is_dir():
+            raise ComfyDockError(f"Models path is not a directory: {models_dir}")
+
+        if not self.quiet:
+            logger.info(f"Scanning models directory: {models_dir}")
+
+        # Get existing locations from this directory to check for changes (mtime optimization)
+        existing_locations = {loc['relative_path']: loc for loc in self.index_manager.get_all_locations(models_dir)}
+
+        # Find all potential model files
+        model_files = self._find_model_files(models_dir) or []
+
+        result = ScanResult(len(model_files), 0, 0, 0, 0, [])
+
+        # Notify progress of scan start
+        if progress:
+            progress.on_scan_start(len(model_files))
+
+        # Process each model file
+        for idx, file_path in enumerate(model_files, 1):
+            try:
+                relative_path = str(file_path.relative_to(models_dir))
+                file_stat = file_path.stat()
+
+                # Check if file has changed
+                existing = existing_locations.get(relative_path)
+                if existing and existing['mtime'] == file_stat.st_mtime:
+                    result.skipped_count += 1
+                    if progress:
+                        progress.on_file_processed(idx, len(model_files), file_path.name)
+                    continue
+
+                # Process the model file
+                process_result = self._process_model_file(file_path, models_dir)
+                self._update_result_counters(result, process_result)
+
+                # Notify progress after processing
+                if progress:
+                    progress.on_file_processed(idx, len(model_files), file_path.name)
+
+            except Exception as e:
+                error_msg = f"Error processing {file_path}: {e}"
+                logger.error(error_msg)
+                result.errors.append(error_msg)
+                result.error_count += 1
+
+        # Clean up stale locations
+        removed_count = self.index_manager.clean_stale_locations(models_dir)
+        result.removed_count = removed_count
+
+        if removed_count > 0 and not self.quiet:
+            logger.info(f"Cleaned up {removed_count} stale locations")
+
+        if not self.quiet:
+            logger.info(f"Scan complete: {result.added_count} added, {result.updated_count} updated, {result.skipped_count} skipped")
+
+        # Notify progress of completion
+        if progress:
+            progress.on_scan_complete(result)
+
+        return result
+
+    def _process_model_file(self, file_path: Path, models_dir: Path) -> ModelProcessResult:
+        """Process a model file and add it to the index.
+
+        Args:
+            file_path: Path to the model file
+            models_dir: Base models directory
+
+        Returns:
+            Result of the processing operation
+        """
+        try:
+            # Get file info
+            file_stat = file_path.stat()
+            relative_path = str(file_path.relative_to(models_dir))
+            filename = file_path.name
+
+            # Calculate hash
+            short_hash = self.index_manager.calculate_short_hash(file_path)
+
+            # Check if model already exists
+            if self.index_manager.has_model(short_hash):
+                # Model exists, just add/update the location
+                self.index_manager.add_location(short_hash, models_dir, relative_path, filename, file_stat.st_mtime)
+                if not self.quiet:
+                    logger.debug(f"Updated location for existing model: {relative_path}")
+                return ModelProcessResult.UPDATED_PATH
+            else:
+                # New model - add to both tables
+                self.index_manager.ensure_model(short_hash, file_stat.st_size)
+                self.index_manager.add_location(short_hash, models_dir, relative_path, filename, file_stat.st_mtime)
+                if not self.quiet:
+                    logger.debug(f"Added new model: {relative_path}")
+                return ModelProcessResult.ADDED
+
+        except Exception as e:
+            logger.error(f"Error processing model file {file_path}: {e}")
+            raise
+
+    def _update_result_counters(self, result: ScanResult, process_result: ModelProcessResult) -> None:
+        """Update ScanResult counters based on processing result."""
+        match process_result:
+            case ModelProcessResult.ADDED:
+                result.added_count += 1
+            case ModelProcessResult.UPDATED_PATH:
+                result.updated_count += 1
+            case ModelProcessResult.SKIPPED_SAME_FILE:
+                result.skipped_count += 1
+
+    def _find_model_files(self, path: Path) -> list[Path]:
+        """Find and filter valid model files in directory."""
+        model_files = []
+        total_found = 0
+        skipped_hidden = 0
+        skipped_not_file = 0
+        skipped_small = 0
+        skipped_validation = 0
+        skipped_error = 0
+
+        for file_path in path.rglob("*"):
+            total_found += 1
+
+            # Skip hidden directories (check only relative path parts, not absolute)
+            try:
+                relative_path = file_path.relative_to(path)
+                if any(part.startswith('.') for part in relative_path.parts):
+                    skipped_hidden += 1
+                    continue
+            except ValueError:
+                # File not under base path - skip
+                skipped_error += 1
+                continue
+
+            try:
+                if not file_path.is_file() or file_path.is_symlink():
+                    skipped_not_file += 1
+                    continue
+
+                # Skip small files
+                file_size = file_path.stat().st_size
+                if file_size < MIN_MODEL_SIZE:
+                    skipped_small += 1
+                    if not self.quiet:
+                        logger.debug(f"Skipped (too small: {file_size} bytes): {file_path.name}")
+                    continue
+
+                # Apply config-based validation
+                if not self._is_valid_model_file(file_path, path):
+                    skipped_validation += 1
+                    if not self.quiet:
+                        logger.debug(f"Skipped (validation failed): {file_path.relative_to(path)}")
+                    continue
+
+                if not self.quiet:
+                    logger.debug(f"Found valid model: {file_path.relative_to(path)}")
+                model_files.append(file_path)
+
+            except (OSError, PermissionError) as e:
+                skipped_error += 1
+                if not self.quiet:
+                    logger.debug(f"Skipped (error: {e}): {file_path.name}")
+                continue
+
+        if not self.quiet:
+            logger.debug(
+                f"File scan summary: {total_found} total, {len(model_files)} valid, "
+                f"{skipped_hidden} hidden, {skipped_not_file} not-file, "
+                f"{skipped_small} small, {skipped_validation} validation-failed, {skipped_error} errors"
+            )
+        return model_files
+
+    def _is_valid_model_file(self, file_path: Path, base_dir: Path) -> bool:
+        """Check if file is valid based on directory-specific rules."""
+
+        # Always exclude obviously non-model files
+        if file_path.suffix.lower() in EXCLUDED_EXTENSIONS:
+            if not self.quiet:
+                logger.debug(f"  Excluded extension {file_path.suffix}: {file_path.name}")
+            return False
+
+        # Get the relative path to determine directory structure
+        try:
+            relative_path = file_path.relative_to(base_dir)
+            if len(relative_path.parts) > 0:
+                # First directory after base is the model type directory
+                model_dir = relative_path.parts[0]
+
+                if self.model_config.is_standard_directory(model_dir):
+                    # Standard directory - use specific extensions
+                    valid_extensions = self.model_config.get_extensions_for_directory(model_dir)
+                    is_valid = file_path.suffix.lower() in valid_extensions
+                    if not is_valid and not self.quiet:
+                        logger.debug(
+                            f"  Invalid extension for {model_dir}/: {file_path.suffix} "
+                            f"(valid: {valid_extensions}) - {file_path.name}"
+                        )
+                    return is_valid
+                else:
+                    # Non-standard directory - be permissive (already excluded obvious non-models)
+                    if not self.quiet:
+                        logger.debug(f"  Non-standard directory {model_dir}/, allowing: {file_path.name}")
+                    return True
+        except ValueError as e:
+            # File not under base_dir? Shouldn't happen with rglob
+            if not self.quiet:
+                logger.debug(f"  ValueError getting relative path: {e} - {file_path}")
+            return False
+
+        # Fallback: check against default extensions
+        is_valid = file_path.suffix.lower() in self.model_config.default_extensions
+        if not is_valid and not self.quiet:
+            logger.debug(f"  Not in default extensions: {file_path.suffix} - {file_path.name}")
+        return is_valid

comfygit_core/analyzers/node_classifier.py

@@ -0,0 +1,58 @@
+"""Node classification service for workflow analysis."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from ..logging.logging_config import get_logger
+from ..configs.comfyui_builtin_nodes import COMFYUI_BUILTIN_NODES
+
+if TYPE_CHECKING:
+    from ..configs.model_config import ModelConfig
+    from ..models.workflow import Workflow, WorkflowNode
+
+logger = get_logger(__name__)
+
+@dataclass
+class NodeClassifierResultMulti:
+    builtin_nodes: list[WorkflowNode]
+    custom_nodes: list[WorkflowNode]
+    
+class NodeClassifier:
+    """Service for classifying and categorizing workflow nodes."""
+
+    def __init__(self):
+        self.builtin_nodes = set(COMFYUI_BUILTIN_NODES["all_builtin_nodes"])
+
+    def get_custom_node_types(self, workflow: Workflow) -> set[str]:
+        """Get custom node types from workflow."""
+        return workflow.node_types - self.builtin_nodes
+
+    def get_model_loader_nodes(self, workflow: Workflow, model_config: ModelConfig) -> list[WorkflowNode]:
+        """Get model loader nodes from workflow."""
+        return [node for node in workflow.nodes.values() if model_config.is_model_loader_node(node.type)]
+    
+    @staticmethod
+    def classify_single_node(node: WorkflowNode) -> str:
+        """Classify a single node by type."""
+        all_builtin_nodes = set(COMFYUI_BUILTIN_NODES["all_builtin_nodes"])
+        if node.type in all_builtin_nodes:
+            return "builtin"
+        return "custom"
+    
+    @staticmethod
+    def classify_nodes(workflow: Workflow) -> NodeClassifierResultMulti:
+        """Classify all nodes by type."""
+        all_builtin_nodes = set(COMFYUI_BUILTIN_NODES["all_builtin_nodes"])
+        builtin_nodes: list[WorkflowNode] = []
+        custom_nodes: list[WorkflowNode] = []
+
+        for node in workflow.nodes.values():
+            if node.type in all_builtin_nodes:
+                builtin_nodes.append(node)
+            else:
+                custom_nodes.append(node)
+
+        return NodeClassifierResultMulti(builtin_nodes, custom_nodes)