skillpool 4.3.0__py3-none-any.whl

skillpool/materializer/budget_cropper.py

@@ -0,0 +1,178 @@
+"""BudgetCropper — 按 token 预算裁剪 markdown 内容。
+
+策略优先级（从低到高删除）：
+1. Version History（最低优先级）
+2. Checklist 的 medium 项
+3. Checklist 的 low 项
+4. Description（截断）
+5. Header + Dimension + Weight + Veto + Schema（最高优先级，保留）
+"""
+
+from __future__ import annotations
+
+import re
+
+
+class BudgetCropper:
+    """按 token 预算裁剪 markdown 内容。
+
+    Args:
+        max_tokens: 最大 token 数限制，默认 4096。
+    """
+
+    def __init__(self, max_tokens: int = 4096):
+        self.max_tokens = max_tokens
+
+    def crop(self, markdown: str) -> str:
+        """裁剪 markdown 到 max_tokens 以内。"""
+        if self.estimate_tokens(markdown) <= self.max_tokens:
+            return markdown
+
+        # 策略 1: 删除 Version History
+        markdown = self._remove_section(markdown, "## Version History")
+        if self.estimate_tokens(markdown) <= self.max_tokens:
+            return markdown
+
+        # 策略 2: 删除 Checklist 的 medium 项
+        markdown = self._remove_checklist_items(markdown, "medium")
+        if self.estimate_tokens(markdown) <= self.max_tokens:
+            return markdown
+
+        # 策略 3: 删除 Checklist 的 low 项
+        markdown = self._remove_checklist_items(markdown, "low")
+        if self.estimate_tokens(markdown) <= self.max_tokens:
+            return markdown
+
+        # 策略 4: 截断 Description
+        markdown = self._truncate_description(markdown)
+        if self.estimate_tokens(markdown) <= self.max_tokens:
+            return markdown
+
+        # 最后手段：硬截断
+        return self._hard_truncate(markdown)
+
+    def estimate_tokens(self, text: str) -> int:
+        """估算 token 数。简单实现：len(text) // 4。"""
+        return len(text) // 4
+
+    def _remove_section(self, markdown: str, header_prefix: str) -> str:
+        """删除指定 section（标题匹配：检查标题行是否包含 header_prefix）。"""
+        lines = markdown.split("\n")
+        result = []
+        in_section = False
+        section_level = 0
+
+        for line in lines:
+            header_match = re.match(r"^(#{1,6})\s+", line)
+            if header_match:
+                current_level = len(header_match.group(1))
+                if in_section:
+                    if current_level <= section_level:
+                        in_section = False
+                    else:
+                        continue
+                # Match section by checking if header text contains the prefix
+                if header_prefix.lower() in line.lower():
+                    in_section = True
+                    section_level = current_level
+                    continue
+            if in_section:
+                continue
+            result.append(line)
+
+        return "\n".join(result)
+
+    def _remove_checklist_items(self, markdown: str, priority: str) -> str:
+        """删除 Checklist 中指定优先级的项。"""
+        lines = markdown.split("\n")
+        result = []
+        in_checklist = False
+        checklist_level = 0
+
+        for line in lines:
+            header_match = re.match(r"^(#{1,6})\s+", line)
+            if header_match:
+                current_level = len(header_match.group(1))
+                if in_checklist and current_level <= checklist_level:
+                    in_checklist = False
+                if "checklist" in line.lower() or "检查项" in line:
+                    in_checklist = True
+                    checklist_level = current_level
+                result.append(line)
+                continue
+
+            if in_checklist:
+                if re.match(r"^\s*[-*]\s+", line):
+                    # Match [medium], [low], (medium), (low) formats
+                    if re.search(rf"[\[(]{priority}[\])]", line, re.IGNORECASE):
+                        continue
+                    # Match "priority: medium" format
+                    if re.search(rf"priority:\s*{priority}", line, re.IGNORECASE):
+                        continue
+
+            result.append(line)
+
+        return "\n".join(result)
+
+    def _truncate_description(self, markdown: str) -> str:
+        """截断 Description 部分。"""
+        lines = markdown.split("\n")
+        result = []
+        in_description = False
+        description_level = 0
+        description_lines = []
+
+        for line in lines:
+            header_match = re.match(r"^(#{1,6})\s+", line)
+            if header_match:
+                current_level = len(header_match.group(1))
+                if in_description and current_level <= description_level:
+                    truncated = self._truncate_lines(description_lines, 200)
+                    result.extend(truncated)
+                    in_description = False
+                    description_lines = []
+                if "Description" in line or "概述" in line or "简介" in line:
+                    in_description = True
+                    description_level = current_level
+                    result.append(line)
+                    continue
+                result.append(line)
+                continue
+
+            if in_description:
+                description_lines.append(line)
+            else:
+                result.append(line)
+
+        if in_description and description_lines:
+            truncated = self._truncate_lines(description_lines, 200)
+            result.extend(truncated)
+
+        return "\n".join(result)
+
+    def _truncate_lines(self, lines: list, max_chars: int) -> list:
+        """截断行列表到指定字符数。"""
+        result = []
+        total = 0
+        for line in lines:
+            if total + len(line) > max_chars:
+                remaining = max_chars - total
+                if remaining > 20:
+                    result.append(line[:remaining] + "...")
+                break
+            result.append(line)
+            total += len(line)
+        return result
+
+    def _hard_truncate(self, markdown: str) -> str:
+        """硬截断到 max_tokens * 4 字符。"""
+        max_chars = self.max_tokens * 4
+        if len(markdown) <= max_chars:
+            return markdown
+        truncated = markdown[:max_chars]
+        last_period = truncated.rfind(".")
+        last_newline = truncated.rfind("\n")
+        cut_point = max(last_period, last_newline)
+        if cut_point > max_chars * 0.8:
+            return truncated[: cut_point + 1] + "\n\n... (truncated)"
+        return truncated + "\n\n... (truncated)"

skillpool/materializer/csdf_loader.py

@@ -0,0 +1,114 @@
+"""Shared CSDF loading utility for SkillPool MCP and LazySkillLoader.
+
+Extracted from mcp_server.py and lazy_loader.py to eliminate code
+duplication. Both modules use this single implementation.
+
+Part of SkillPool — independent infrastructure, shared by all agents.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def load_csdf(
+    skill_id: str,
+    skills_dir: Path,
+) -> dict[str, Any] | None:
+    """Load CSDF data for a skill by ID.
+
+    Tries three lookup strategies in order:
+    1. Exact YAML match: {skills_dir}/{skill_id}.yaml
+    2. Prefix YAML match: {skills_dir}/{skill_id}_*.yaml
+    3. Directory-based: {skills_dir}/{skill_id}/SKILL.md
+
+    Args:
+        skill_id: Skill identifier (e.g., "S09", "scaffold-docs").
+        skills_dir: Path to the skills directory.
+
+    Returns:
+        Dict with CSDF data, or None if not found.
+    """
+    # 1. Exact match
+    exact = skills_dir / f"{skill_id}.yaml"
+    if exact.exists():
+        return _parse_yaml(exact)
+
+    # 2. Prefix match (e.g., S09-resilience-degradation.yaml)
+    for p in skills_dir.glob(f"{skill_id}-*.yaml"):
+        return _parse_yaml(p)
+
+    # 3. Directory-based skill
+    skill_dir = skills_dir / skill_id
+    if skill_dir.is_dir():
+        skill_md = skill_dir / "SKILL.md"
+        if skill_md.exists():
+            return _parse_directory_skill(skill_id, skill_md, skill_dir)
+
+    return None
+
+
+def _parse_yaml(path: Path) -> dict[str, Any] | None:
+    """Parse a CSDF YAML file."""
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+        if data and isinstance(data, dict):
+            return data
+    except (yaml.YAMLError, OSError):
+        pass
+    return None
+
+
+def _parse_directory_skill(
+    skill_id: str,
+    skill_md: Path,
+    skill_dir: Path,
+) -> dict[str, Any]:
+    """Parse a directory-based skill from SKILL.md frontmatter."""
+    try:
+        text = skill_md.read_text(encoding="utf-8")
+    except OSError:
+        return {"id": skill_id, "name": skill_id, "type": "directory"}
+
+    # Parse YAML frontmatter
+    frontmatter: dict[str, Any] = {"id": skill_id, "name": skill_id, "type": "directory"}
+    if text.startswith("---"):
+        end = text.find("---", 3)
+        if end > 0:
+            try:
+                fm = yaml.safe_load(text[3:end])
+                if isinstance(fm, dict):
+                    frontmatter.update(fm)
+            except yaml.YAMLError:
+                pass
+
+    frontmatter["id"] = frontmatter.get("id", skill_id)
+    frontmatter["name"] = frontmatter.get("name", skill_id)
+    frontmatter["type"] = "directory"
+    frontmatter["_skill_dir"] = str(skill_dir)
+
+    # Store the markdown body (content after frontmatter) for directory-based skills
+    # This allows skill_definition() to return the full SKILL.md content
+    if text.startswith("---"):
+        end = text.find("---", 3)
+        if end > 0:
+            # Extract body after the closing ---
+            body = text[end + 3 :].lstrip("\n")
+            if body:
+                frontmatter["_markdown_body"] = body
+
+    # Merge manifest.yaml if present (contains synergies, dependencies, etc.)
+    manifest_path = skill_dir / "manifest.yaml"
+    if manifest_path.exists():
+        manifest_data = _parse_yaml(manifest_path)
+        if manifest_data and isinstance(manifest_data, dict):
+            # manifest fields override frontmatter defaults, but don't clobber id/type
+            for k, v in manifest_data.items():
+                if k not in ("id", "type"):
+                    frontmatter.setdefault(k, v)
+
+    return frontmatter

skillpool/materializer/lazy_loader.py

@@ -0,0 +1,265 @@
+"""LazySkillLoader — tiered skill loading for token-efficient context delivery.
+
+Three loading tiers:
+  L0 (metadata):  id, name, version, dimension, weight, tags  — ~50 tokens
+  L1 (summary):   L0 + description + checklist summary        — ~200 tokens
+  L2 (full def):  complete SKILL.md via Materializer           — full token cost
+
+L0 reads YAML frontmatter only (no materialization).
+L1 adds the description field from the CSDF dict.
+L2 runs the full Materializer.materialize() pipeline.
+
+Uses shared csdf_loader for CSDF loading (eliminates duplication with mcp_server).
+
+Part of SkillPool — independent infrastructure, shared by all agents.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from pathlib import Path
+from typing import Optional
+
+
+from skillpool.config import get_data_dir
+from skillpool.materializer import Materializer
+from skillpool.materializer.csdf_loader import load_csdf
+from skillpool.materializer.models import MaterializationResult
+from skillpool.profile import CLAUDE_CODE_PROFILE, AgentCapabilityProfile
+
+_SKILLS_DIR = get_data_dir() / "skills"
+
+_VALID_TIERS = {"L0", "L1", "L2"}
+
+logger = logging.getLogger("skillpool.lazy_loader")
+
+
+class LazySkillLoader:
+    """Tiered skill loader with in-memory cache and thread safety.
+
+    Usage:
+        loader = LazySkillLoader()
+        meta = loader.load("S09", tier="L0")       # cheap metadata
+        summary = loader.upgrade("S09", "L0", "L1") # add description
+        full = loader.upgrade("S09", "L1", "L2")    # full materialization
+    """
+
+    def __init__(
+        self,
+        profile: Optional[AgentCapabilityProfile] = None,
+        skills_dir: Optional[Path] = None,
+    ):
+        self._profile = profile or CLAUDE_CODE_PROFILE
+        self._skills_dir = skills_dir or _SKILLS_DIR
+        # Cache: {skill_id: {"L0": dict, "L1": dict, "L2": dict}}
+        self._cache: dict[str, dict[str, dict]] = {}
+        # Track file modification times for cache invalidation
+        self._mtimes: dict[str, float] = {}
+        self._lock = threading.Lock()
+
+    def load(self, skill_id: str, tier: str = "L0") -> dict:
+        """Load a skill at the specified tier.
+
+        Args:
+            skill_id: Skill identifier (e.g., "S09", "scaffold-docs")
+            tier: Loading tier — "L0", "L1", or "L2"
+
+        Returns:
+            Dict with skill data at the requested tier.
+            Includes a "_tier" key indicating the loaded tier.
+
+        Raises:
+            ValueError: If tier is invalid or skill_id not found.
+        """
+        # Part of SkillPool — independent infrastructure, shared by all agents
+        self._validate_tier(tier)
+
+        with self._lock:
+            # Check for cache invalidation (file modified)
+            self._check_invalidation(skill_id)
+
+            # Return cached if available
+            if skill_id in self._cache and tier in self._cache[skill_id]:
+                return self._cache[skill_id][tier]
+
+        # Load CSDF outside lock (I/O bound)
+        csdf = load_csdf(skill_id, self._skills_dir)
+        if csdf is None:
+            raise ValueError(f"Skill not found: {skill_id}")
+
+        with self._lock:
+            # Ensure cache entry exists
+            if skill_id not in self._cache:
+                self._cache[skill_id] = {}
+
+            # Build tiers incrementally
+            self._ensure_tiers(skill_id, csdf, tier)
+
+            return self._cache[skill_id][tier]
+
+    def preload(self, skill_ids: list[str], tier: str = "L0") -> dict[str, dict]:
+        """Batch-load multiple skills at the specified tier.
+
+        Args:
+            skill_ids: List of skill identifiers
+            tier: Loading tier for all skills
+
+        Returns:
+            Dict mapping skill_id to its loaded data. Skills that fail
+            to load are omitted from the result.
+        """
+        # Part of SkillPool — independent infrastructure, shared by all agents
+        self._validate_tier(tier)
+        results = {}
+        for sid in skill_ids:
+            try:
+                results[sid] = self.load(sid, tier=tier)
+            except ValueError:
+                logger.debug("preload skipped missing skill: %s", sid)
+                continue
+        return results
+
+    def upgrade(self, skill_id: str, from_tier: str, to_tier: str) -> dict:
+        """Load more detail for an already-loaded skill.
+
+        Args:
+            skill_id: Skill identifier
+            from_tier: Current tier (must already be loaded)
+            to_tier: Target tier (must be higher than from_tier)
+
+        Returns:
+            Dict with skill data at the target tier.
+
+        Raises:
+            ValueError: If skill not cached, from_tier not loaded,
+                        or invalid tier ordering.
+        """
+        # Part of SkillPool — independent infrastructure, shared by all agents
+        self._validate_tier(from_tier)
+        self._validate_tier(to_tier)
+
+        tier_order = {"L0": 0, "L1": 1, "L2": 2}
+        if tier_order[to_tier] <= tier_order[from_tier]:
+            raise ValueError(f"to_tier ({to_tier}) must be higher than from_tier ({from_tier})")
+
+        with self._lock:
+            if skill_id not in self._cache:
+                raise ValueError(f"Skill {skill_id} not in cache — call load() first")
+
+            if from_tier not in self._cache[skill_id]:
+                raise ValueError(f"Skill {skill_id} not loaded at {from_tier} — load it first")
+
+        # Re-load at the higher tier (uses cache for lower tiers)
+        return self.load(skill_id, tier=to_tier)
+
+    def clear_cache(self, skill_id: str | None = None) -> None:
+        """Clear cached data for a skill, or all skills if skill_id is None."""
+        with self._lock:
+            if skill_id is None:
+                self._cache.clear()
+                self._mtimes.clear()
+            else:
+                self._cache.pop(skill_id, None)
+                self._mtimes.pop(skill_id, None)
+
+    # ── Internal helpers ──────────────────────────────────────────
+
+    def _check_invalidation(self, skill_id: str) -> None:
+        """Check if cached data is stale (file modified since last load).
+
+        Must be called with self._lock held.
+        """
+        # Try to find the source file
+        yaml_path = self._skills_dir / f"{skill_id}.yaml"
+        if not yaml_path.exists():
+            for p in self._skills_dir.glob(f"{skill_id}_*.yaml"):
+                yaml_path = p
+                break
+
+        if not yaml_path.exists():
+            # Directory-based skill
+            md_path = self._skills_dir / skill_id / "SKILL.md"
+            if md_path.exists():
+                yaml_path = md_path
+
+        if yaml_path.exists():
+            try:
+                mtime = yaml_path.stat().st_mtime
+                if skill_id in self._mtimes and self._mtimes[skill_id] != mtime:
+                    logger.info("Cache invalidated for %s (file modified)", skill_id)
+                    self._cache.pop(skill_id, None)
+                self._mtimes[skill_id] = mtime
+            except OSError:
+                pass
+
+    def _ensure_tiers(self, skill_id: str, csdf: dict, target_tier: str) -> None:
+        """Build cache entries up to the target tier, starting from L0.
+
+        Must be called with self._lock held.
+        """
+        cache = self._cache[skill_id]
+
+        # L0: metadata only (~50 tokens)
+        if "L0" not in cache:
+            cache["L0"] = {
+                "id": csdf.get("id", skill_id),
+                "name": csdf.get("name", ""),
+                "version": csdf.get("version", ""),
+                "dimension": csdf.get("dimension", ""),
+                "weight": csdf.get("weight", 0),
+                "tags": csdf.get("tags", []),
+                "_tier": "L0",
+            }
+
+        if target_tier == "L0":
+            return
+
+        # L1: L0 + description + checklist summary (~200 tokens)
+        if "L1" not in cache:
+            l0 = cache["L0"]
+            checklist_summary = []
+            for item in csdf.get("checklist", []):
+                if isinstance(item, dict):
+                    checklist_summary.append(
+                        {
+                            "id": item.get("id", ""),
+                            "description": item.get("description", ""),
+                            "severity": item.get("severity", ""),
+                        }
+                    )
+            cache["L1"] = {
+                **l0,
+                "description": csdf.get("description", ""),
+                "checklist_summary": checklist_summary,
+                "_tier": "L1",
+            }
+
+        if target_tier == "L1":
+            return
+
+        # L2: full materialization via Materializer
+        if "L2" not in cache:
+            mat = Materializer(profile=self._profile)
+            result: MaterializationResult = mat.materialize(csdf_dict=csdf)
+            l2_data = {
+                **cache["L1"],
+                "_tier": "L2",
+            }
+            # Preserve raw markdown body for directory-based skills
+            if "_markdown_body" in csdf:
+                l2_data["_markdown_body"] = csdf["_markdown_body"]
+            if result.status == "success" and result.skill is not None:
+                l2_data["markdown"] = result.skill.markdown
+                l2_data["token_count"] = result.skill.token_count
+            else:
+                logger.warning("L2 materialization failed for %s: %s", skill_id, result.errors)
+                l2_data["markdown"] = ""
+                l2_data["token_count"] = 0
+                l2_data["_materialization_errors"] = result.errors
+            cache["L2"] = l2_data
+
+    @staticmethod
+    def _validate_tier(tier: str) -> None:
+        if tier not in _VALID_TIERS:
+            raise ValueError(f"Invalid tier '{tier}'; must be one of {sorted(_VALID_TIERS)}")

skillpool/materializer/lifecycle_filter.py

@@ -0,0 +1,93 @@
+"""LifecycleFilter — 根据 Skill 生命周期状态过滤 markdown 内容。
+
+根据 CSDF 中的 lifecycle_state 字段，对 markdown 内容添加状态标记、
+降级警告或截断处理。
+"""
+
+from __future__ import annotations
+
+from skillpool.lifecycle import SkillLifecycleState, parse_state
+
+
+class LifecycleFilter:
+    """根据 Skill 生命周期状态过滤内容。
+
+    Args:
+        strict: True 时，REJECTED 清空内容，ARCHIVED 截断内容。
+    """
+
+    def __init__(self, strict: bool = True):
+        self.strict = strict
+
+    def filter(self, markdown: str, csdf: dict) -> str:
+        """过滤 markdown 内容。
+
+        Args:
+            markdown: 原始 markdown 文本。
+            csdf: CSDF 字典，需包含 lifecycle_state 字段。
+
+        Returns:
+            过滤后的 markdown 文本。
+        """
+        state = self._resolve_state(csdf)
+        return self._apply_filter(markdown, state)
+
+    def _resolve_state(self, csdf: dict) -> SkillLifecycleState:
+        """从 CSDF 解析生命周期状态，默认 ACTIVE。"""
+        raw = csdf.get("lifecycle_state")
+        if raw is None:
+            return SkillLifecycleState.ACTIVE
+        state = parse_state(str(raw))
+        if state is None:
+            return SkillLifecycleState.ACTIVE
+        return state
+
+    def _apply_filter(self, markdown: str, state: SkillLifecycleState) -> str:
+        """根据状态应用过滤规则。"""
+        if state == SkillLifecycleState.DRAFT:
+            return self._prepend_warning(markdown, "[DRAFT] 开发中")
+        elif state == SkillLifecycleState.PROPOSED:
+            return self._prepend_warning(markdown, "[DRAFT] 开发中")
+        elif state == SkillLifecycleState.UNDER_REVIEW:
+            return self._prepend_warning(markdown, "[REVIEW] 审核中")
+        elif state == SkillLifecycleState.APPROVED:
+            return markdown
+        elif state == SkillLifecycleState.ACTIVE:
+            return markdown
+        elif state == SkillLifecycleState.REJECTED:
+            return self._handle_rejected(markdown)
+        elif state == SkillLifecycleState.DEPRECATED:
+            return self._handle_deprecated(markdown)
+        elif state == SkillLifecycleState.ARCHIVED:
+            return self._handle_archived(markdown)
+        elif state == SkillLifecycleState.REMOVED:
+            return ""
+        return markdown
+
+    def _prepend_warning(self, markdown: str, warning: str) -> str:
+        """在 markdown 顶部添加警告。"""
+        return f"> ⚠️ {warning}\n\n{markdown}"
+
+    def _handle_rejected(self, markdown: str) -> str:
+        """处理 REJECTED 状态。"""
+        warning = "[REJECTED] 已否决"
+        if self.strict:
+            return f"> ⚠️ {warning}\n"
+        return self._prepend_warning(markdown, warning)
+
+    def _handle_deprecated(self, markdown: str) -> str:
+        """处理 DEPRECATED 状态，添加替代建议。"""
+        replacement = "请查看替代方案或联系维护者"
+        warning = f"[DEPRECATED] 已弃用 — {replacement}"
+        return self._prepend_warning(markdown, warning)
+
+    def _handle_archived(self, markdown: str) -> str:
+        """处理 ARCHIVED 状态。"""
+        warning = "[ARCHIVED] 已归档"
+        if self.strict:
+            # 截断：只保留前 200 字符
+            truncated = markdown[:200]
+            if len(markdown) > 200:
+                truncated += "\n\n... (内容已截断)"
+            return f"> ⚠️ {warning}\n\n{truncated}"
+        return self._prepend_warning(markdown, warning)