kubiya-control-plane-api 0.9.15__py3-none-any.whl

control_plane_api/worker/skills/loaders/filesystem_loader.py

@@ -0,0 +1,357 @@
+"""
+Filesystem Skill Loader
+
+Discovers skills from filesystem directories by scanning for skill.yaml files.
+"""
+import yaml
+import importlib.util
+import sys
+import json
+from pathlib import Path
+from typing import List, Type, Dict, Any
+import structlog
+
+from .base import BaseSkillLoader
+from control_plane_api.worker.skills.registry import LoadedSkill, SkillSource
+
+logger = structlog.get_logger()
+
+# Try to import jsonschema for validation (optional dependency)
+try:
+    import jsonschema
+    JSONSCHEMA_AVAILABLE = True
+except ImportError:
+    JSONSCHEMA_AVAILABLE = False
+    logger.warning(
+        "jsonschema_not_available",
+        message="Install jsonschema package for skill.yaml validation: pip install jsonschema"
+    )
+
+
+def validate_skill_yaml(manifest: Dict[str, Any], yaml_path: Path) -> None:
+    """
+    Issue #6 Fix: Validate skill.yaml structure against JSON schema.
+
+    Args:
+        manifest: Parsed YAML manifest
+        yaml_path: Path to the YAML file for error reporting
+
+    Raises:
+        ValueError: If validation fails
+    """
+    if not JSONSCHEMA_AVAILABLE:
+        logger.debug(
+            "skipping_skill_yaml_validation",
+            yaml_path=str(yaml_path),
+            reason="jsonschema not installed",
+        )
+        return
+
+    # Load schema
+    schema_path = Path(__file__).parent.parent / "skill_yaml_schema.json"
+    if not schema_path.exists():
+        logger.warning(
+            "skill_yaml_schema_not_found",
+            schema_path=str(schema_path),
+            yaml_path=str(yaml_path),
+        )
+        return
+
+    try:
+        with open(schema_path) as f:
+            schema = json.load(f)
+
+        # Validate
+        jsonschema.validate(instance=manifest, schema=schema)
+
+        logger.debug(
+            "skill_yaml_validation_passed",
+            yaml_path=str(yaml_path),
+            skill_name=manifest.get("metadata", {}).get("name", "unknown"),
+        )
+
+    except jsonschema.ValidationError as e:
+        # Build helpful error message
+        error_path = " -> ".join(str(p) for p in e.path) if e.path else "root"
+        error_msg = (
+            f"Invalid skill.yaml at {yaml_path}\n"
+            f"Location: {error_path}\n"
+            f"Error: {e.message}\n\n"
+            f"Schema requirement: {e.schema.get('description', 'No description')}\n"
+        )
+
+        # Add suggestions based on common errors
+        if "required" in e.message.lower():
+            error_msg += f"\nMissing required fields. Check the skill.yaml schema.\n"
+        elif "enum" in str(e.schema):
+            allowed = e.schema.get("enum", [])
+            error_msg += f"\nAllowed values: {allowed}\n"
+
+        logger.error(
+            "skill_yaml_validation_failed",
+            yaml_path=str(yaml_path),
+            error_path=error_path,
+            error_message=e.message,
+            schema_path=str(schema_path),
+        )
+
+        raise ValueError(error_msg)
+
+    except Exception as e:
+        logger.error(
+            "skill_yaml_validation_error",
+            yaml_path=str(yaml_path),
+            error=str(e),
+            exc_info=True,
+        )
+        raise ValueError(f"Failed to validate skill.yaml at {yaml_path}: {str(e)}")
+
+
+class FilesystemSkillLoader(BaseSkillLoader):
+    """
+    Load skills from filesystem directories.
+
+    Scans for skill.yaml files in configured paths.
+    """
+
+    def __init__(self, search_paths: List[Path]):
+        self.search_paths = [Path(p) for p in search_paths]
+        self.logger = structlog.get_logger()
+
+    def discover(self) -> List[LoadedSkill]:
+        """
+        Discover skills by scanning filesystem for skill.yaml files.
+        """
+        skills = []
+
+        for search_path in self.search_paths:
+            if not search_path.exists():
+                self.logger.debug("search_path_not_found", path=str(search_path))
+                continue
+
+            # Find all skill.yaml files
+            for skill_yaml in search_path.rglob("skill.yaml"):
+                try:
+                    skill = self._load_skill_from_yaml(skill_yaml)
+                    skills.append(skill)
+                    self.logger.info(
+                        "skill_discovered",
+                        name=skill.name,
+                        path=str(skill_yaml.parent),
+                    )
+                except Exception as e:
+                    self.logger.error(
+                        "failed_to_load_skill",
+                        path=str(skill_yaml),
+                        error=str(e),
+                        exc_info=True,
+                    )
+
+        return skills
+
+    def _load_skill_from_yaml(self, yaml_path: Path) -> LoadedSkill:
+        """Load a skill from a skill.yaml file."""
+        with open(yaml_path) as f:
+            manifest = yaml.safe_load(f)
+
+        # Issue #6 Fix: Validate skill.yaml before parsing
+        validate_skill_yaml(manifest, yaml_path)
+
+        skill_dir = yaml_path.parent
+        metadata = manifest.get("metadata", {})
+        spec = manifest.get("spec", {})
+
+        # Load implementations
+        implementations = {}
+        impl_configs = spec.get("implementations", {})
+
+        if not impl_configs:
+            raise ValueError(f"No implementations defined in {yaml_path}")
+
+        # Track failed implementations for better error reporting
+        failed_implementations = {}
+
+        for runtime, impl_info in impl_configs.items():
+            try:
+                # Check if this is a builtin runtime implementation
+                if impl_info.get("builtin", False):
+                    # Builtin implementations (like Claude Code SDK tools)
+                    # Store metadata about builtin tools instead of Python class
+                    implementations[runtime] = {
+                        "builtin": True,
+                        "tools": impl_info.get("tools", []),
+                    }
+                    self.logger.debug(
+                        "builtin_implementation_registered",
+                        skill=metadata["name"],
+                        runtime=runtime,
+                        tools=impl_info.get("tools", []),
+                    )
+                    continue
+
+                module_name = impl_info["module"]
+                class_name = impl_info["class"]
+
+                # Import the module
+                impl_class = self._import_class_from_skill(
+                    skill_dir, module_name, class_name
+                )
+                implementations[runtime] = impl_class
+
+                self.logger.info(
+                    "implementation_loaded_successfully",
+                    skill=metadata["name"],
+                    runtime=runtime,
+                    class_name=class_name,
+                    module_path=str(skill_dir / f"{module_name}.py"),
+                )
+            except FileNotFoundError as e:
+                # Issue #3 Fix: Fail loudly with actionable error
+                error_detail = str(e)
+                failed_implementations[runtime] = error_detail
+                self.logger.error(
+                    "implementation_file_not_found",
+                    skill=metadata["name"],
+                    runtime=runtime,
+                    error=error_detail,
+                    module_name=module_name,
+                    expected_path=str(skill_dir / f"{module_name}.py"),
+                    recovery_suggestion=f"Create the file '{module_name}.py' in {skill_dir} or update the skill.yaml to reference an existing module.",
+                )
+            except AttributeError as e:
+                # Issue #3 Fix: Class not found in module
+                error_detail = str(e)
+                failed_implementations[runtime] = error_detail
+                self.logger.error(
+                    "implementation_class_not_found",
+                    skill=metadata["name"],
+                    runtime=runtime,
+                    error=error_detail,
+                    class_name=class_name,
+                    module_name=module_name,
+                    recovery_suggestion=f"Ensure class '{class_name}' exists in '{module_name}.py' or update the skill.yaml with the correct class name.",
+                )
+            except ImportError as e:
+                # Issue #3 Fix: Module import failed
+                error_detail = str(e)
+                failed_implementations[runtime] = error_detail
+                self.logger.error(
+                    "implementation_import_failed",
+                    skill=metadata["name"],
+                    runtime=runtime,
+                    error=error_detail,
+                    module_name=module_name,
+                    recovery_suggestion=f"Check that '{module_name}.py' is valid Python and all dependencies are installed. Error: {error_detail}",
+                )
+            except Exception as e:
+                # Issue #3 Fix: Generic implementation failure
+                error_detail = str(e)
+                failed_implementations[runtime] = error_detail
+                self.logger.error(
+                    "implementation_load_failed",
+                    skill=metadata["name"],
+                    runtime=runtime,
+                    error=error_detail,
+                    error_type=type(e).__name__,
+                    recovery_suggestion="Check the implementation file for syntax errors or missing dependencies.",
+                    exc_info=True,
+                )
+
+        # Issue #3 Fix: Fail loudly if no valid implementations
+        if not implementations:
+            error_summary = "\n".join([
+                f"  - {runtime}: {error}"
+                for runtime, error in failed_implementations.items()
+            ])
+            raise ValueError(
+                f"Failed to load any implementations for skill '{metadata['name']}' from {yaml_path}.\n"
+                f"All {len(impl_configs)} runtime(s) failed:\n{error_summary}\n\n"
+                f"Recovery steps:\n"
+                f"1. Check that implementation files exist in {skill_dir}\n"
+                f"2. Verify module and class names in skill.yaml match actual files\n"
+                f"3. Ensure all dependencies are installed\n"
+                f"4. Check implementation files for syntax errors"
+            )
+
+        return LoadedSkill(
+            name=metadata["name"],
+            version=metadata.get("version", "0.0.0"),
+            source=self._get_source_for_path(yaml_path),
+            skill_type=spec.get("type", "custom"),
+            manifest=manifest,
+            implementations=implementations,
+            metadata={
+                "path": str(skill_dir),
+                "yaml_path": str(yaml_path),
+            },
+        )
+
+    def _import_class_from_skill(
+        self, skill_dir: Path, module_name: str, class_name: str
+    ) -> Type:
+        """
+        Dynamically import a class from a skill's module.
+
+        Args:
+            skill_dir: Directory containing the skill
+            module_name: Module name (e.g., "implementation" or "agno_impl")
+            class_name: Class name to import
+
+        Returns:
+            The imported class
+        """
+        # Convert module name to file path
+        module_file = skill_dir / f"{module_name}.py"
+
+        if not module_file.exists():
+            raise FileNotFoundError(f"Module file not found: {module_file}")
+
+        # Create unique module name to avoid collisions
+        unique_module_name = f"skill_{skill_dir.name}_{module_name}"
+
+        # Load the module
+        spec = importlib.util.spec_from_file_location(unique_module_name, module_file)
+        if spec is None or spec.loader is None:
+            raise ImportError(f"Could not load module spec from {module_file}")
+
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[unique_module_name] = module
+        spec.loader.exec_module(module)
+
+        # Get the class
+        if not hasattr(module, class_name):
+            raise AttributeError(
+                f"Class '{class_name}' not found in module {module_file}"
+            )
+
+        return getattr(module, class_name)
+
+    def _get_source_for_path(self, yaml_path: Path) -> SkillSource:
+        """Determine source type based on path."""
+        path_str = str(yaml_path.resolve())
+
+        # Check if it's in user workspace (.kubiya/skills in current dir)
+        if ".kubiya/skills" in path_str and Path.cwd() in yaml_path.parents:
+            return SkillSource.USER_WORKSPACE
+
+        # Check if it's in global user directory (~/.kubiya/skills)
+        global_skills = Path.home() / ".kubiya/skills"
+        if global_skills in yaml_path.parents:
+            return SkillSource.USER_GLOBAL
+
+        # Default to builtin
+        return SkillSource.BUILTIN
+
+    def get_source_type(self) -> SkillSource:
+        return SkillSource.USER_WORKSPACE
+
+    def load_skill(self, skill_id: str) -> LoadedSkill:
+        """Load a specific skill by searching for its directory."""
+        for search_path in self.search_paths:
+            skill_path = search_path / skill_id / "skill.yaml"
+            if skill_path.exists():
+                return self._load_skill_from_yaml(skill_path)
+
+        raise FileNotFoundError(
+            f"Skill '{skill_id}' not found in search paths: {self.search_paths}"
+        )

control_plane_api/worker/skills/registry.py

@@ -0,0 +1,208 @@
+"""
+Worker-side Skill Registry
+
+Central registry for all skills available to the worker.
+Discovers and tracks skills from multiple sources (filesystem, API, packages).
+"""
+from typing import Dict, List, Optional, Type, Any, Union
+from dataclasses import dataclass
+from enum import Enum
+import structlog
+
+logger = structlog.get_logger()
+
+
+class SkillSource(str, Enum):
+    """Source where skill was loaded from."""
+    CONTROL_PLANE_API = "api"           # Centralized skills from DB
+    USER_WORKSPACE = "workspace"        # .kubiya/skills/ in project
+    USER_GLOBAL = "global"              # ~/.kubiya/skills/
+    PYTHON_PACKAGE = "package"          # pip installed packages
+    GIT_REPOSITORY = "git"              # Git repos
+    BUILTIN = "builtin"                 # Built-in skills
+
+
+@dataclass
+class LoadedSkill:
+    """A loaded and ready-to-use skill."""
+    name: str
+    version: str
+    source: SkillSource
+    skill_type: str  # e.g., "shell", "file_system", "custom"
+    manifest: Dict[str, Any]  # Parsed skill.yaml or metadata
+    implementations: Dict[str, Union[Type, Dict[str, Any]]]  # Runtime -> class or builtin config
+    metadata: Dict[str, Any]  # Additional metadata (path, etc.)
+
+
+class SkillRegistry:
+    """
+    Central registry for all skills on worker side.
+
+    Responsibilities:
+    - Track loaded skills
+    - Resolve dependencies
+    - Handle versioning
+    - Match skills to runtimes
+    """
+
+    def __init__(self):
+        self._skills: Dict[str, LoadedSkill] = {}
+        self._skills_by_type: Dict[str, List[LoadedSkill]] = {}
+        self.logger = structlog.get_logger()
+
+    def register(self, skill: LoadedSkill) -> None:
+        """Register a loaded skill."""
+        skill_key = f"{skill.name}:{skill.version}"
+
+        if skill_key in self._skills:
+            self.logger.warning(
+                "skill_already_registered",
+                name=skill.name,
+                version=skill.version,
+                existing_source=self._skills[skill_key].source,
+                new_source=skill.source,
+            )
+            # Source priority: builtin < global < workspace < git < package < api
+            if self._should_replace(self._skills[skill_key].source, skill.source):
+                self._skills[skill_key] = skill
+                self.logger.info("skill_replaced", name=skill.name)
+        else:
+            self._skills[skill_key] = skill
+            self.logger.info(
+                "skill_registered",
+                name=skill.name,
+                version=skill.version,
+                source=skill.source,
+                skill_type=skill.skill_type,
+            )
+
+        # Index by type for faster lookup
+        if skill.skill_type not in self._skills_by_type:
+            self._skills_by_type[skill.skill_type] = []
+
+        # Remove old version if exists
+        self._skills_by_type[skill.skill_type] = [
+            s for s in self._skills_by_type[skill.skill_type]
+            if s.name != skill.name
+        ]
+        self._skills_by_type[skill.skill_type].append(skill)
+
+    def get(self, name: str, version: Optional[str] = None) -> Optional[LoadedSkill]:
+        """
+        Get a skill by name and optional version.
+
+        If version not specified, returns latest version.
+        """
+        if version:
+            return self._skills.get(f"{name}:{version}")
+
+        # Get latest version
+        matching = [s for s in self._skills.values() if s.name == name]
+        if not matching:
+            return None
+
+        # Sort by semantic version
+        try:
+            from packaging import version as pkg_version
+            return max(matching, key=lambda s: pkg_version.parse(s.version))
+        except:
+            # Fallback to simple comparison if packaging not available
+            return matching[0]
+
+    def get_by_type(self, skill_type: str) -> Optional[LoadedSkill]:
+        """
+        Get a skill by type (e.g., "shell", "file_system").
+
+        Returns the first registered skill of that type.
+        Useful for built-in skills where type maps 1:1 to skill.
+        """
+        skills = self._skills_by_type.get(skill_type, [])
+        return skills[0] if skills else None
+
+    def list_skills(self, source: Optional[SkillSource] = None) -> List[LoadedSkill]:
+        """List all registered skills, optionally filtered by source."""
+        skills = list(self._skills.values())
+        if source:
+            skills = [s for s in skills if s.source == source]
+        return skills
+
+    def get_implementation_for_runtime(
+        self, skill: LoadedSkill, runtime_type: str
+    ) -> Optional[Union[Type, Dict[str, Any]]]:
+        """
+        Get the appropriate implementation class for a specific runtime.
+
+        Falls back to 'default' implementation if runtime-specific not available.
+
+        Returns:
+            Either a Python class (Type) or a dict with builtin config
+        """
+        # Check runtime-specific implementation
+        impl = skill.implementations.get(runtime_type)
+        if impl:
+            return impl
+
+        # Fall back to default implementation
+        return skill.implementations.get("default")
+
+    def resolve_dependencies(self, skill: LoadedSkill) -> List[str]:
+        """
+        Resolve skill dependencies.
+
+        Returns list of missing dependency names.
+        """
+        missing = []
+        dependencies = skill.manifest.get("spec", {}).get("dependencies", [])
+
+        for dep in dependencies:
+            dep_name = dep["name"]
+            dep_version = dep.get("version", ">=0.0.0")
+
+            resolved = self.get(dep_name)
+            if not resolved:
+                if not dep.get("optional", False):
+                    missing.append(f"{dep_name} {dep_version}")
+            else:
+                # Check version constraint
+                try:
+                    from packaging import version, specifiers
+                    spec = specifiers.SpecifierSet(dep_version)
+                    if not spec.contains(resolved.version):
+                        missing.append(
+                            f"{dep_name} {dep_version} (found {resolved.version})"
+                        )
+                except:
+                    # Skip version check if packaging not available
+                    pass
+
+        return missing
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get registry statistics."""
+        return {
+            "total_skills": len(self._skills),
+            "skills_by_source": {
+                source.value: len([s for s in self._skills.values() if s.source == source])
+                for source in SkillSource
+            },
+            "skills_by_type": {
+                skill_type: len(skills)
+                for skill_type, skills in self._skills_by_type.items()
+            },
+        }
+
+    def _should_replace(self, existing: SkillSource, new: SkillSource) -> bool:
+        """Determine if new source should replace existing based on priority."""
+        priority = {
+            SkillSource.BUILTIN: 1,
+            SkillSource.USER_GLOBAL: 2,
+            SkillSource.USER_WORKSPACE: 3,
+            SkillSource.GIT_REPOSITORY: 4,
+            SkillSource.PYTHON_PACKAGE: 5,
+            SkillSource.CONTROL_PLANE_API: 6,
+        }
+        return priority.get(new, 0) > priority.get(existing, 0)
+
+
+# Global registry instance
+skill_registry = SkillRegistry()

control_plane_api/worker/tests/__init__.py

@@ -0,0 +1 @@
+"""Test suite for worker files"""

control_plane_api/worker/tests/conftest.py

@@ -0,0 +1,12 @@
+"""
+Pytest configuration for worker tests.
+Sets up sys.path to allow 'runtimes' imports.
+"""
+
+import sys
+from pathlib import Path
+
+# Add the worker directory to sys.path so that 'from runtimes import ...' works
+worker_dir = Path(__file__).parent.parent
+if str(worker_dir) not in sys.path:
+    sys.path.insert(0, str(worker_dir))