kweaver-dolphin 0.1.0__py3-none-any.whl

dolphin/lib/skillkits/resource/skill_cache.py

@@ -0,0 +1,215 @@
+"""TTL/LRU cache implementation for ResourceSkillkit.
+
+This module provides caching mechanisms for skill metadata (Level 1)
+to optimize performance and reduce disk I/O.
+"""
+
+import time
+from collections import OrderedDict
+from dataclasses import dataclass
+from threading import RLock
+from typing import Generic, TypeVar, Optional, Dict, Any
+
+T = TypeVar("T")
+
+
+@dataclass
+class CacheEntry(Generic[T]):
+    """A single cache entry with TTL support.
+
+    Attributes:
+        value: The cached value
+        created_at: Unix timestamp when entry was created
+        last_accessed: Unix timestamp when entry was last accessed
+    """
+
+    value: T
+    created_at: float
+    last_accessed: float
+
+
+class TTLLRUCache(Generic[T]):
+    """Thread-safe TTL + LRU cache implementation.
+
+    This cache combines Time-To-Live (TTL) expiration with Least Recently Used (LRU)
+    eviction policy. Entries expire after TTL seconds, and when the cache is full,
+    the least recently used entries are evicted first.
+
+    Attributes:
+        ttl_seconds: Time-to-live for cache entries in seconds
+        max_size: Maximum number of entries in cache
+    """
+
+    def __init__(self, ttl_seconds: int = 300, max_size: int = 100):
+        """Initialize the cache.
+
+        Args:
+            ttl_seconds: TTL for entries (default 5 minutes)
+            max_size: Maximum cache size (default 100 entries)
+        """
+        self._ttl_seconds = ttl_seconds
+        self._max_size = max_size
+        self._cache: OrderedDict[str, CacheEntry[T]] = OrderedDict()
+        self._lock = RLock()
+
+    def get(self, key: str) -> Optional[T]:
+        """Get a value from cache.
+
+        Args:
+            key: The cache key
+
+        Returns:
+            The cached value, or None if not found or expired
+        """
+        with self._lock:
+            entry = self._cache.get(key)
+            if entry is None:
+                return None
+
+            # Check TTL expiration
+            if self._is_expired(entry):
+                del self._cache[key]
+                return None
+
+            # Update last accessed time and move to end (most recently used)
+            entry.last_accessed = time.time()
+            self._cache.move_to_end(key)
+            return entry.value
+
+    def set(self, key: str, value: T) -> None:
+        """Set a value in cache.
+
+        Args:
+            key: The cache key
+            value: The value to cache
+        """
+        with self._lock:
+            now = time.time()
+
+            # If key exists, update it
+            if key in self._cache:
+                self._cache[key] = CacheEntry(
+                    value=value, created_at=now, last_accessed=now
+                )
+                self._cache.move_to_end(key)
+            else:
+                # Evict if at capacity
+                while len(self._cache) >= self._max_size:
+                    self._cache.popitem(last=False)  # Remove oldest (LRU)
+
+                self._cache[key] = CacheEntry(
+                    value=value, created_at=now, last_accessed=now
+                )
+
+    def delete(self, key: str) -> bool:
+        """Delete an entry from cache.
+
+        Args:
+            key: The cache key to delete
+
+        Returns:
+            True if entry was deleted, False if not found
+        """
+        with self._lock:
+            if key in self._cache:
+                del self._cache[key]
+                return True
+            return False
+
+    def clear(self) -> None:
+        """Clear all entries from cache."""
+        with self._lock:
+            self._cache.clear()
+
+    def cleanup_expired(self) -> int:
+        """Remove all expired entries from cache.
+
+        Returns:
+            Number of entries removed
+        """
+        with self._lock:
+            expired_keys = [
+                key for key, entry in self._cache.items() if self._is_expired(entry)
+            ]
+            for key in expired_keys:
+                del self._cache[key]
+            return len(expired_keys)
+
+    def _is_expired(self, entry: CacheEntry[T]) -> bool:
+        """Check if a cache entry is expired.
+
+        Args:
+            entry: The cache entry to check
+
+        Returns:
+            True if expired, False otherwise
+        """
+        return time.time() - entry.created_at > self._ttl_seconds
+
+    def __len__(self) -> int:
+        """Return the number of entries in cache."""
+        with self._lock:
+            return len(self._cache)
+
+    def __contains__(self, key: str) -> bool:
+        """Check if key exists and is not expired."""
+        return self.get(key) is not None
+
+    def keys(self) -> list:
+        """Return list of all non-expired keys."""
+        with self._lock:
+            return [key for key, entry in self._cache.items() if not self._is_expired(entry)]
+
+    def stats(self) -> Dict[str, Any]:
+        """Get cache statistics.
+
+        Returns:
+            Dictionary with cache stats
+        """
+        with self._lock:
+            expired_count = sum(
+                1 for entry in self._cache.values() if self._is_expired(entry)
+            )
+            return {
+                "size": len(self._cache),
+                "max_size": self._max_size,
+                "ttl_seconds": self._ttl_seconds,
+                "expired_entries": expired_count,
+                "active_entries": len(self._cache) - expired_count,
+            }
+
+
+class SkillMetaCache(TTLLRUCache):
+    """Specialized cache for Level 1 skill metadata.
+
+    This cache stores SkillMeta objects for quick access during
+    system prompt generation.
+    """
+
+    def __init__(self, ttl_seconds: int = 300, max_size: int = 100):
+        """Initialize the skill metadata cache.
+
+        Args:
+            ttl_seconds: TTL for entries (default 5 minutes)
+            max_size: Maximum cache size (default 100 skills)
+        """
+        super().__init__(ttl_seconds=ttl_seconds, max_size=max_size)
+
+
+class SkillContentCache(TTLLRUCache):
+    """Specialized cache for Level 2 skill content.
+
+    This cache stores SkillContent objects for skills that have
+    been fully loaded. Note that Level 2 content persistence is
+    primarily handled via history bucket, this cache is for
+    internal optimization.
+    """
+
+    def __init__(self, ttl_seconds: int = 600, max_size: int = 50):
+        """Initialize the skill content cache.
+
+        Args:
+            ttl_seconds: TTL for entries (default 10 minutes)
+            max_size: Maximum cache size (default 50 skills)
+        """
+        super().__init__(ttl_seconds=ttl_seconds, max_size=max_size)

dolphin/lib/skillkits/resource/skill_loader.py

@@ -0,0 +1,395 @@
+"""SKILL.md loader for ResourceSkillkit.
+
+This module provides the SkillLoader class for loading and parsing
+SKILL.md files with support for the three-level progressive loading:
+- Level 1: Metadata (name, description)
+- Level 2: Full SKILL.md content
+- Level 3: Resource files (scripts/, references/)
+"""
+
+import re
+import os
+from pathlib import Path
+from typing import Optional, List, Tuple, Dict, Any
+
+import yaml
+
+from .models.skill_meta import SkillMeta, SkillContent
+from .models.skill_config import ResourceSkillConfig
+from .skill_validator import SkillValidator, ValidationResult, resolve_safe_path
+from dolphin.core.logging.logger import get_logger
+
+logger = get_logger("resource_skillkit")
+
+
+class SkillLoaderError(Exception):
+    """Exception raised for skill loading errors."""
+
+    pass
+
+
+class SkillLoader:
+    """Loader for SKILL.md files supporting progressive loading.
+
+    This loader handles:
+    - Scanning directories for skill packages
+    - Parsing YAML frontmatter and markdown body
+    - Level 1/2/3 content loading
+    - Validation and error handling
+    """
+
+    # Regex pattern for YAML frontmatter
+    FRONTMATTER_PATTERN = re.compile(
+        r"^---\s*\r?\n(.*?)\r?\n---\s*\r?\n(.*)$",
+        re.DOTALL
+    )
+
+    def __init__(self, config: Optional[ResourceSkillConfig] = None):
+        """Initialize the loader.
+
+        Args:
+            config: Optional configuration, uses defaults if not provided
+        """
+        self.config = config or ResourceSkillConfig()
+        self.validator = SkillValidator(self.config)
+
+    def scan_directories(self, base_path: Optional[Path] = None) -> List[SkillMeta]:
+        """Scan configured directories for skill packages.
+
+        Scans all configured directories and returns Level 1 metadata
+        for each valid skill found. Higher priority directories take
+        precedence for duplicate skill names.
+
+        Args:
+            base_path: Optional base path for resolving relative directories
+
+        Returns:
+            List of SkillMeta objects for found skills
+        """
+        skills: Dict[str, SkillMeta] = {}  # name -> meta, for deduplication
+        directories = self.config.get_resolved_directories(base_path)
+
+        for directory in directories:
+            if not directory.exists():
+                logger.debug(f"Skill directory does not exist: {directory}")
+                continue
+
+            if not directory.is_dir():
+                logger.warning(f"Skill path is not a directory: {directory}")
+                continue
+
+            max_depth = max(1, int(self.config.max_scan_depth))
+            for item in self._iter_skill_dirs(directory, max_depth=max_depth):
+                try:
+                    meta = self.load_metadata(item)
+                    if meta and meta.name not in skills:
+                        # First occurrence wins (higher priority directory)
+                        skills[meta.name] = meta
+                        logger.debug(f"Found skill: {meta.name} at {item}")
+                except Exception as e:
+                    logger.warning(f"Failed to load skill from {item}: {e}")
+
+        return list(skills.values())
+
+    def _iter_skill_dirs(self, root: Path, max_depth: int) -> List[Path]:
+        """Iterate directories that contain SKILL.md under root, up to max_depth."""
+        found: List[Path] = []
+        stack: List[Tuple[Path, int]] = [(root, 0)]
+
+        while stack:
+            current, depth = stack.pop()
+            skill_md = current / "SKILL.md"
+            if skill_md.is_file():
+                found.append(current)
+                continue
+
+            if depth >= max_depth:
+                continue
+
+            try:
+                subdirs = [
+                    p
+                    for p in current.iterdir()
+                    if p.is_dir()
+                    and not p.name.startswith(".")
+                    and p.name not in {"__pycache__", "scripts", "references"}
+                ]
+            except PermissionError:
+                continue
+
+            for sub in sorted(subdirs, key=lambda p: p.name, reverse=True):
+                stack.append((sub, depth + 1))
+
+        return sorted(found, key=lambda p: str(p))
+
+    def load_metadata(self, skill_dir: Path) -> Optional[SkillMeta]:
+        """Load Level 1 metadata from a skill directory.
+
+        Args:
+            skill_dir: Path to the skill directory
+
+        Returns:
+            SkillMeta if successful, None if failed
+        """
+        skill_md = skill_dir / "SKILL.md"
+
+        if not skill_md.is_file():
+            logger.warning(f"SKILL.md not found in {skill_dir}")
+            return None
+
+        # Security: reject symlinks to prevent path traversal
+        if skill_md.is_symlink():
+            logger.warning(f"SKILL.md is a symlink, skipping: {skill_md}")
+            return None
+
+        try:
+            size_validation = self.validator.validate_size(skill_md)
+            if not size_validation.is_valid:
+                logger.warning(
+                    f"SKILL.md too large in {skill_dir}: {size_validation.errors}"
+                )
+                return None
+
+            content = skill_md.read_text(encoding="utf-8")
+            frontmatter, _ = self._parse_frontmatter(content)
+
+            if not frontmatter:
+                logger.warning(f"No frontmatter found in {skill_md}")
+                return None
+
+            # Validate frontmatter
+            validation = self.validator.validate_frontmatter(frontmatter)
+            if not validation.is_valid:
+                logger.warning(
+                    f"Invalid frontmatter in {skill_md}: {validation.errors}"
+                )
+                return None
+
+            for warning in validation.warnings:
+                logger.debug(f"Frontmatter warning for {skill_md}: {warning}")
+
+            return SkillMeta(
+                name=frontmatter.get("name", ""),
+                description=frontmatter.get("description", ""),
+                base_path=str(skill_dir.resolve()),
+                version=frontmatter.get("version"),
+                tags=frontmatter.get("tags", []),
+            )
+
+        except Exception as e:
+            logger.error(f"Error loading metadata from {skill_md}: {e}")
+            return None
+
+    def load_content(self, skill_dir: Path) -> Optional[SkillContent]:
+        """Load Level 2 full content from a skill directory.
+
+        Args:
+            skill_dir: Path to the skill directory
+
+        Returns:
+            SkillContent if successful, None if failed
+        """
+        skill_md = skill_dir / "SKILL.md"
+
+        if not skill_md.is_file():
+            logger.warning(f"SKILL.md not found in {skill_dir}")
+            return None
+
+        # Security: reject symlinks to prevent path traversal
+        if skill_md.is_symlink():
+            logger.warning(f"SKILL.md is a symlink, skipping: {skill_md}")
+            return None
+
+        try:
+            size_validation = self.validator.validate_size(skill_dir)
+            if not size_validation.is_valid:
+                logger.warning(
+                    f"Skill package too large in {skill_dir}: {size_validation.errors}"
+                )
+                return None
+
+            content = skill_md.read_text(encoding="utf-8")
+            frontmatter, body = self._parse_frontmatter(content)
+
+            if frontmatter is None:
+                # No frontmatter, treat entire content as body
+                frontmatter = {}
+                body = content
+
+            # List available scripts and references
+            scripts = self._list_directory_files(skill_dir / "scripts")
+            references = self._list_directory_files(skill_dir / "references")
+
+            return SkillContent(
+                frontmatter=frontmatter,
+                body=body.strip(),
+                available_scripts=scripts,
+                available_references=references,
+            )
+
+        except Exception as e:
+            logger.error(f"Error loading content from {skill_md}: {e}")
+            return None
+
+    def load_resource(
+        self, skill_dir: Path, resource_path: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """Load Level 3 resource file content.
+
+        Args:
+            skill_dir: Path to the skill directory
+            resource_path: Relative path to resource file (e.g., "scripts/etl.py")
+
+        Returns:
+            Tuple of (content, error_message). Content is None if error.
+        """
+        full_path, error = resolve_safe_path(resource_path, skill_dir)
+        if error:
+            return None, error
+        if full_path is None:
+            return None, f"Invalid path: '{resource_path}'"
+
+        # Validate file type
+        validation = self.validator.validate_file_type(full_path)
+        if not validation.is_valid:
+            return None, validation.errors[0]
+
+        # Validate size
+        size_validation = self.validator.validate_size(full_path)
+        if not size_validation.is_valid:
+            return None, size_validation.errors[0]
+
+        try:
+            flags = os.O_RDONLY
+            nofollow = getattr(os, "O_NOFOLLOW", 0)
+            if nofollow:
+                flags |= nofollow
+            fd = os.open(str(full_path), flags)
+            with os.fdopen(fd, "r", encoding="utf-8") as f:
+                return f.read(), None
+        except UnicodeDecodeError:
+            return None, f"Cannot read '{resource_path}': not a text file"
+        except OSError as e:
+            return None, f"Error reading '{resource_path}': {e}"
+        except Exception as e:
+            return None, f"Error reading '{resource_path}': {e}"
+
+    def _parse_frontmatter(self, content: str) -> Tuple[Optional[Dict[str, Any]], str]:
+        """Parse YAML frontmatter from SKILL.md content.
+
+        Args:
+            content: The full SKILL.md file content
+
+        Returns:
+            Tuple of (frontmatter_dict, body_content)
+            frontmatter_dict is None if no valid frontmatter found
+        """
+        match = self.FRONTMATTER_PATTERN.match(content)
+
+        if not match:
+            return None, content
+
+        yaml_content = match.group(1)
+        body = match.group(2)
+
+        try:
+            frontmatter = yaml.safe_load(yaml_content)
+            if not isinstance(frontmatter, dict):
+                logger.warning("Frontmatter is not a valid YAML dictionary")
+                return None, content
+            return frontmatter, body
+        except yaml.YAMLError as e:
+            logger.warning(f"Failed to parse YAML frontmatter: {e}")
+            return None, content
+
+    def _list_directory_files(
+        self, directory: Path, max_depth: int = 2
+    ) -> List[str]:
+        """List files in a directory recursively.
+
+        Args:
+            directory: Directory to list
+            max_depth: Maximum recursion depth
+
+        Returns:
+            List of relative file paths
+        """
+        if not directory.is_dir():
+            return []
+
+        files = []
+        base_name = directory.name
+
+        def _scan(current_dir: Path, depth: int, prefix: str):
+            if depth > max_depth:
+                return
+
+            try:
+                for item in sorted(current_dir.iterdir()):
+                    rel_path = f"{prefix}/{item.name}" if prefix else item.name
+
+                    if item.is_file():
+                        # Validate file type
+                        if self.validator.validate_file_type(item).is_valid:
+                            files.append(f"{base_name}/{rel_path}")
+                    elif item.is_dir() and not item.name.startswith("."):
+                        _scan(item, depth + 1, rel_path)
+            except PermissionError:
+                logger.debug(f"Permission denied accessing {current_dir}")
+
+        _scan(directory, 1, "")
+        return files
+
+    def find_skill_by_name(
+        self, name: str, base_path: Optional[Path] = None
+    ) -> Optional[Path]:
+        """Find a skill directory by name.
+
+        Args:
+            name: The skill name to find
+            base_path: Optional base path for resolving directories
+
+        Returns:
+            Path to skill directory if found, None otherwise
+        """
+        directories = self.config.get_resolved_directories(base_path)
+
+        for directory in directories:
+            if not directory.is_dir():
+                continue
+
+            skill_dir = directory / name
+            if skill_dir.is_dir() and (skill_dir / "SKILL.md").is_file():
+                return skill_dir
+
+        return None
+
+
+def truncate_content(content: str, max_tokens: int, chars_per_token: int = 4) -> str:
+    """Truncate content to approximately max_tokens.
+
+    Args:
+        content: The content to truncate
+        max_tokens: Maximum number of tokens
+        chars_per_token: Estimated characters per token
+
+    Returns:
+        Truncated content with notice if truncated
+    """
+    max_chars = max_tokens * chars_per_token
+
+    if len(content) <= max_chars:
+        return content
+
+    # Find a good break point (end of line near limit)
+    truncated = content[:max_chars]
+    last_newline = truncated.rfind("\n")
+
+    if last_newline > max_chars * 0.8:  # If newline is in last 20%
+        truncated = truncated[:last_newline]
+
+    return (
+        truncated
+        + "\n\n---\n"
+        + "*[Content truncated. Use `_load_skill_resource()` to load specific files.]*"
+    )