pythonclaw 0.2.0__py3-none-any.whl

pythonclaw/core/skill_loader.py

@@ -0,0 +1,322 @@
+"""
+Skill discovery and loading for PythonClaw.
+
+Three-tier progressive disclosure
+----------------------------------
+Inspired by Claude's Agent Skills architecture, skills are loaded on demand
+in three tiers to minimise context-window usage:
+
+  Level 1 — Metadata  (always loaded at startup, ~100 tokens per skill)
+      YAML frontmatter from every SKILL.md (``name`` + ``description``).
+      Injected into the system prompt as a "skill catalog" so the LLM
+      knows what capabilities exist and when to activate them.
+
+  Level 2 — Core instructions  (loaded when the LLM triggers ``use_skill``)
+      The body of SKILL.md: workflows, rules, step-by-step guidance.
+
+  Level 3 — Extended resources  (loaded as needed via ``read_file`` / ``run_command``)
+      Scripts, schemas, reference docs, templates, CSV data — anything
+      bundled in the skill folder that SKILL.md references.
+
+SKILL.md format  (Claude-compatible)
+--------------------------------------
+    ---
+    name: calculator
+    description: >
+      Performs basic arithmetic.  Use when the user asks to calculate
+      math expressions, additions, multiplications, etc.
+    ---
+    # Calculator
+
+    ## Instructions
+    Run `python {skill_path}/calc.py "expression"` ...
+
+    ## Resources
+    - `calc.py` — arithmetic script
+
+Directory layout (both flat and categorised)
+----------------------------------------------
+    <skills_dir>/
+        <skill_name>/              — flat (Claude-style)
+            SKILL.md
+            *.py / *.sh
+
+    <skills_dir>/
+        <category>/                — categorised (PythonClaw-style)
+            CATEGORY.md            — optional category description
+            <skill_name>/
+                SKILL.md
+                *.py / *.sh
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+from .utils import parse_frontmatter
+
+logger = logging.getLogger(__name__)
+
+
+# ── Data classes ─────────────────────────────────────────────────────────────
+
+class SkillMetadata:
+    """Level 1 — lightweight metadata for a single skill."""
+
+    __slots__ = ("name", "description", "path", "category")
+
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        path: str,
+        category: str = "",
+    ) -> None:
+        self.name = name
+        self.description = description
+        self.path = path
+        self.category = category
+
+
+class Skill:
+    """Level 2 — a fully loaded skill including its instruction text."""
+
+    def __init__(self, metadata: SkillMetadata, instructions: str) -> None:
+        self.metadata = metadata
+        self.instructions = instructions
+
+    @property
+    def name(self) -> str:
+        return self.metadata.name
+
+    @property
+    def description(self) -> str:
+        return self.metadata.description
+
+
+# ── Registry ─────────────────────────────────────────────────────────────────
+
+class SkillRegistry:
+    """
+    Scans skill directories and provides three-tier progressive loading.
+
+    Supports both flat layouts (``skills/calculator/SKILL.md``) and
+    categorised layouts (``skills/math/calculator/SKILL.md``).
+    """
+
+    def __init__(self, skills_dirs: list[str] | None = None) -> None:
+        self.skills_dirs: list[str] = list(skills_dirs) if skills_dirs else []
+        self._cache: list[SkillMetadata] | None = None
+
+    def invalidate(self) -> None:
+        """Clear the discovery cache so new skills are picked up on next call."""
+        self._cache = None
+
+    # ── Level 1: Metadata discovery ──────────────────────────────────────
+
+    def discover(self) -> list[SkillMetadata]:
+        """
+        Scan all configured directories and return metadata for every skill.
+
+        Results are cached after the first call.  This is the **Level 1**
+        operation — only YAML frontmatter is read (name + description).
+        """
+        if self._cache is not None:
+            return self._cache
+
+        skills: list[SkillMetadata] = []
+        seen_names: set[str] = set()
+
+        for s_dir in self.skills_dirs:
+            if not os.path.isdir(s_dir):
+                continue
+            self._scan_dir(s_dir, skills, seen_names)
+
+        self._cache = skills
+        return skills
+
+    def _scan_dir(
+        self,
+        base_dir: str,
+        out: list[SkillMetadata],
+        seen: set[str],
+    ) -> None:
+        """Recursively find SKILL.md files up to 2 levels deep."""
+        for entry in sorted(os.listdir(base_dir)):
+            if entry.startswith(("__", ".")):
+                continue
+            entry_path = os.path.join(base_dir, entry)
+            if not os.path.isdir(entry_path):
+                continue
+
+            skill_md = os.path.join(entry_path, "SKILL.md")
+            if os.path.isfile(skill_md):
+                # Flat layout: skills/<skill>/SKILL.md
+                meta = self._read_metadata(skill_md, entry_path, category="")
+                if meta and meta.name not in seen:
+                    out.append(meta)
+                    seen.add(meta.name)
+            else:
+                # Categorised layout: skills/<category>/<skill>/SKILL.md
+                category_name = entry
+                for sub_entry in sorted(os.listdir(entry_path)):
+                    if sub_entry.startswith(("__", ".")):
+                        continue
+                    sub_path = os.path.join(entry_path, sub_entry)
+                    sub_md = os.path.join(sub_path, "SKILL.md")
+                    if os.path.isdir(sub_path) and os.path.isfile(sub_md):
+                        meta = self._read_metadata(
+                            sub_md, sub_path, category=category_name
+                        )
+                        if meta and meta.name not in seen:
+                            out.append(meta)
+                            seen.add(meta.name)
+
+    @staticmethod
+    def _read_metadata(
+        md_path: str,
+        skill_dir: str,
+        category: str,
+    ) -> SkillMetadata | None:
+        try:
+            with open(md_path, "r", encoding="utf-8") as f:
+                content = f.read()
+            meta, _ = parse_frontmatter(content)
+            name = meta.get("name", os.path.basename(skill_dir))
+            description = meta.get("description", "No description.")
+            return SkillMetadata(
+                name=name,
+                description=description,
+                path=os.path.abspath(skill_dir),
+                category=category,
+            )
+        except OSError as exc:
+            logger.warning("Could not read skill at '%s': %s", md_path, exc)
+            return None
+
+    # ── Level 2: Full instruction loading ────────────────────────────────
+
+    def load_skill(self, name: str) -> Skill | None:
+        """
+        Load the full SKILL.md body for a skill by name (**Level 2**).
+
+        The ``{skill_path}`` placeholder in the instruction text is
+        replaced with the skill's absolute directory path.
+        """
+        for meta in self.discover():
+            if meta.name != name:
+                continue
+            md_path = os.path.join(meta.path, "SKILL.md")
+            try:
+                with open(md_path, "r", encoding="utf-8") as f:
+                    content = f.read()
+                _, instructions = parse_frontmatter(content)
+                instructions = instructions.replace("{skill_path}", meta.path)
+                return Skill(metadata=meta, instructions=instructions)
+            except OSError as exc:
+                logger.error("Error loading skill '%s': %s", name, exc)
+                return None
+        return None
+
+    # ── Level 3: Resource discovery ──────────────────────────────────────
+
+    def list_resources(self, name: str) -> list[str]:
+        """
+        List bundled resource files for a skill (**Level 3** discovery).
+
+        Returns relative filenames (e.g. ``["calc.py", "REFERENCE.md"]``)
+        excluding SKILL.md itself.
+        """
+        for meta in self.discover():
+            if meta.name != name:
+                continue
+            try:
+                return sorted(
+                    f
+                    for f in os.listdir(meta.path)
+                    if f != "SKILL.md"
+                    and not f.startswith(("__", "."))
+                    and os.path.isfile(os.path.join(meta.path, f))
+                )
+            except OSError:
+                return []
+        return []
+
+    def get_resource_path(self, skill_name: str, resource: str) -> str | None:
+        """Return the absolute path to a resource file inside a skill folder."""
+        for meta in self.discover():
+            if meta.name != skill_name:
+                continue
+            full = os.path.join(meta.path, resource)
+            if os.path.isfile(full):
+                return full
+            return None
+        return None
+
+    # ── Catalog builder (for system prompt injection) ────────────────────
+
+    def build_catalog(self) -> str:
+        """
+        Build a compact skill catalog string for the system prompt.
+
+        Groups skills by category and formats them as a bulleted list.
+        This is what the LLM sees at **Level 1** — all available skills
+        and their descriptions, so it can decide when to trigger them.
+        """
+        skills = self.discover()
+        if not skills:
+            return "(no skills installed)"
+
+        # Group by category
+        groups: dict[str, list[SkillMetadata]] = {}
+        for s in skills:
+            groups.setdefault(s.category or "general", []).append(s)
+
+        lines: list[str] = []
+        for cat in sorted(groups):
+            if cat != "general":
+                lines.append(f"\n  **{cat}**")
+            for s in groups[cat]:
+                lines.append(f"  - `{s.name}`: {s.description}")
+
+        return "\n".join(lines)
+
+
+# ── Module-level convenience functions ───────────────────────────────────────
+
+def load_skill_by_name(
+    skill_name: str,
+    skills_dirs: list[str] | None = None,
+) -> Skill | None:
+    """Load a skill by name (Level 2)."""
+    return SkillRegistry(skills_dirs).load_skill(skill_name)
+
+
+def search_skills(
+    query: str,
+    skills_dirs: list[str] | None = None,
+) -> list[dict]:
+    """Search skills by keyword match in name or description."""
+    q = query.lower()
+    return [
+        {"name": s.name, "description": s.description, "category": s.category}
+        for s in SkillRegistry(skills_dirs).discover()
+        if q in s.name.lower() or q in s.description.lower()
+    ]
+
+
+def list_skills_in_category(
+    category: str,
+    skills_dirs: list[str] | None = None,
+) -> list[dict]:
+    """List skills in a specific category (backward compat)."""
+    return [
+        {
+            "name": s.name,
+            "description": s.description,
+            "path_name": os.path.basename(s.path),
+        }
+        for s in SkillRegistry(skills_dirs).discover()
+        if s.category == category
+    ]

pythonclaw/core/skillhub.py

@@ -0,0 +1,290 @@
+"""
+SkillHub marketplace client for PythonClaw.
+
+Integrates with https://www.skillhub.club/ API to search, browse,
+and install community skills directly into the local skill directory.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import ssl
+import urllib.error
+import urllib.request
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+API_BASE = "https://www.skillhub.club/api/v1"
+SKILL_PAGE_BASE = "https://www.skillhub.club/skills"
+
+
+def _get_ssl_ctx() -> ssl.SSLContext:
+    """Build an SSL context; use unverified fallback for macOS cert issues."""
+    try:
+        import certifi
+        return ssl.create_default_context(cafile=certifi.where())
+    except ImportError:
+        pass
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+def _api_key() -> str:
+    """Read SkillHub API key from config or environment."""
+    from .. import config
+    return config.get_str("skillhub", "apiKey", env="SKILLHUB_API_KEY")
+
+
+def _api_request(
+    method: str,
+    path: str,
+    *,
+    body: dict | None = None,
+    params: dict | None = None,
+) -> dict:
+    """Make an authenticated request to the SkillHub API."""
+    api_key = _api_key()
+
+    url = f"{API_BASE}{path}"
+    if params:
+        qs = "&".join(f"{k}={urllib.request.quote(str(v))}" for k, v in params.items() if v is not None)
+        if qs:
+            url = f"{url}?{qs}"
+
+    headers: dict[str, str] = {"Content-Type": "application/json"}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    data = json.dumps(body).encode("utf-8") if body else None
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+
+    try:
+        with urllib.request.urlopen(req, timeout=15, context=_get_ssl_ctx()) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        err_body = exc.read().decode("utf-8", errors="replace")
+        logger.warning("SkillHub API error %s: %s", exc.code, err_body)
+        raise RuntimeError(f"SkillHub API error ({exc.code}): {err_body}") from exc
+    except Exception as exc:
+        raise RuntimeError(f"SkillHub request failed: {exc}") from exc
+
+
+def search(query: str, *, limit: int = 10, category: str | None = None) -> list[dict]:
+    """Search SkillHub for skills matching a query."""
+    body: dict[str, Any] = {"query": query, "limit": limit, "method": "hybrid"}
+    if category:
+        body["category"] = category
+    result = _api_request("POST", "/skills/search", body=body)
+    return result.get("results", result.get("skills", []))
+
+
+def browse(*, limit: int = 20, sort: str = "score", category: str | None = None) -> list[dict]:
+    """Browse the SkillHub catalog."""
+    params: dict[str, Any] = {"limit": limit, "sort": sort}
+    if category:
+        params["category"] = category
+    result = _api_request("GET", "/skills/catalog", params=params)
+    return result.get("results", result.get("skills", []))
+
+
+def get_skill_detail(skill_id: str) -> dict | None:
+    """Fetch full detail for a skill, including SKILL.md content.
+
+    Tries the API first, falls back to scraping the skill page.
+    """
+    try:
+        result = _api_request("GET", f"/skills/{skill_id}")
+        skill = result.get("skill", result)
+        return {
+            "id": skill.get("slug", skill.get("id", skill_id)),
+            "name": skill.get("name", ""),
+            "description": skill.get("description", ""),
+            "skill_md": skill.get("skill_md_raw", skill.get("skill_md", "")),
+            "category": skill.get("category", ""),
+            "author": skill.get("author", ""),
+            "score": skill.get("composite_score", skill.get("simple_score", "")),
+            "stars": skill.get("github_stars", ""),
+            "source_url": skill.get("repo_url", f"{SKILL_PAGE_BASE}/{skill_id}"),
+        }
+    except Exception:
+        pass
+
+    return _scrape_skill_page(skill_id)
+
+
+def _scrape_skill_page(skill_id: str) -> dict | None:
+    """Fallback: scrape the skill page for SKILL.md content."""
+    url = f"{SKILL_PAGE_BASE}/{skill_id}"
+    req = urllib.request.Request(url, headers={"User-Agent": "PythonClaw/1.0"})
+    try:
+        with urllib.request.urlopen(req, timeout=15, context=_get_ssl_ctx()) as resp:
+            html = resp.read().decode("utf-8")
+    except Exception as exc:
+        logger.warning("Failed to fetch skill page %s: %s", url, exc)
+        return None
+
+    skill_md = _extract_skill_md(html)
+    if not skill_md:
+        return None
+
+    name_match = re.search(r"<h1[^>]*>(.*?)</h1>", html, re.DOTALL)
+    name = name_match.group(1).strip() if name_match else skill_id.split("-")[-1]
+    name = re.sub(r"<[^>]+>", "", name).strip()
+
+    desc = ""
+    fm_match = re.search(r"^description:\s*>?\s*(.+?)$", skill_md, re.MULTILINE)
+    if fm_match:
+        desc = fm_match.group(1).strip()
+
+    return {
+        "id": skill_id,
+        "name": name,
+        "description": desc,
+        "skill_md": skill_md,
+        "source_url": url,
+    }
+
+
+def _extract_skill_md(html: str) -> str | None:
+    """Extract SKILL.md content from a skill page HTML."""
+    # SkillHub renders skill content inside a prose-skill container.
+    # The frontmatter appears as <hr/> then <h2>name: ... description: ...
+    # followed by the instruction body.
+
+    m = re.search(
+        r'class="[^"]*prose-skill[^"]*"[^>]*>(.*?)(?:</div>\s*<div|$)',
+        html,
+        re.DOTALL,
+    )
+    if not m:
+        m = re.search(r'<hr/>\s*<h2>name:\s*\S+', html)
+        if m:
+            start = m.start()
+            end_markers = ['Content curated', 'User Rating', 'USER RATING', 'Grade ']
+            end = len(html)
+            for marker in end_markers:
+                pos = html.find(marker, start)
+                if pos != -1 and pos < end:
+                    end = pos
+            raw = html[start:end]
+        else:
+            return None
+    else:
+        raw = m.group(1)
+
+    raw = re.sub(r"<[^>]+>", "\n", raw)
+    raw = raw.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&")
+    raw = raw.replace("&quot;", '"').replace("&#39;", "'")
+
+    lines = []
+    for line in raw.split("\n"):
+        stripped = line.strip()
+        if stripped:
+            lines.append(stripped)
+
+    text = "\n".join(lines)
+
+    # Reconstruct frontmatter
+    fm_match = re.search(r'name:\s*(\S+)\s+description:\s*(.+?)(?=\n\w|\n#|\Z)', text, re.DOTALL)
+    if fm_match:
+        name = fm_match.group(1).strip()
+        desc = fm_match.group(2).strip()
+        body_start = fm_match.end()
+        body = text[body_start:].strip()
+        return f"---\nname: {name}\ndescription: >\n  {desc}\n---\n\n{body}"
+
+    return text if len(text) > 50 else None
+
+
+def install_skill(
+    skill_id: str,
+    *,
+    target_dir: str | None = None,
+    skill_md_override: str | None = None,
+) -> str:
+    """Download and install a skill from SkillHub into the local skills directory.
+
+    Returns the path to the installed skill directory.
+    """
+    if target_dir is None:
+        target_dir = os.path.join("context", "skills")
+
+    detail = None
+    if not skill_md_override:
+        detail = get_skill_detail(skill_id)
+        if not detail:
+            raise RuntimeError(f"Could not fetch skill '{skill_id}' from SkillHub.")
+
+    skill_md = skill_md_override or detail.get("skill_md", "")
+    if not skill_md:
+        raise RuntimeError(f"No SKILL.md content found for '{skill_id}'.")
+
+    skill_name = _derive_skill_name(skill_id, skill_md, detail)
+    safe_name = re.sub(r"[^a-zA-Z0-9_-]", "_", skill_name).strip("_")
+    if not safe_name:
+        safe_name = "imported_skill"
+
+    category = "skillhub"
+    skill_dir = os.path.join(target_dir, category, safe_name)
+    os.makedirs(skill_dir, exist_ok=True)
+
+    md_path = os.path.join(skill_dir, "SKILL.md")
+
+    if not skill_md.startswith("---"):
+        skill_md = f"---\nname: {safe_name}\ndescription: Imported from SkillHub ({skill_id})\n---\n\n{skill_md}"
+
+    with open(md_path, "w", encoding="utf-8") as f:
+        f.write(skill_md + "\n")
+
+    source_url = detail.get("source_url", f"{SKILL_PAGE_BASE}/{skill_id}") if detail else f"{SKILL_PAGE_BASE}/{skill_id}"
+    meta_path = os.path.join(skill_dir, ".skillhub.json")
+    with open(meta_path, "w", encoding="utf-8") as f:
+        json.dump({"id": skill_id, "source": source_url, "installed_by": "pythonclaw"}, f, indent=2)
+
+    return skill_dir
+
+
+def _derive_skill_name(skill_id: str, skill_md: str, detail: dict | None) -> str:
+    """Extract a reasonable skill name from available data."""
+    name_match = re.search(r"^name:\s*(.+)$", skill_md, re.MULTILINE)
+    if name_match:
+        return name_match.group(1).strip()
+    if detail and detail.get("name"):
+        return detail["name"]
+    parts = skill_id.rsplit("-", 1)
+    return parts[-1] if parts else skill_id
+
+
+def format_search_results(results: list[dict]) -> str:
+    """Format search results for CLI display."""
+    if not results:
+        return "No skills found."
+
+    lines = []
+    for i, r in enumerate(results, 1):
+        name = r.get("name", r.get("title", "???"))
+        desc = r.get("description", "")[:80]
+        sid = r.get("id", r.get("slug", ""))
+        score = r.get("score", r.get("ai_score", ""))
+        stars = r.get("stars", "")
+
+        header = f"  {i}. {name}"
+        if score:
+            header += f"  (score: {score})"
+        if stars:
+            header += f"  ★{stars}"
+
+        lines.append(header)
+        if desc:
+            lines.append(f"     {desc}")
+        if sid:
+            lines.append(f"     ID: {sid}")
+        lines.append("")
+
+    return "\n".join(lines)