memstack-skill-loader 3.5.0__py3-none-any.whl

memstack_skill_loader/dashboard.py

@@ -0,0 +1,360 @@
+"""Local HTTP dashboard server for MemStack usage analytics."""
+
+import json
+import os
+import pickle
+import re
+import sys
+import webbrowser
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+
+from .config import load_config
+from .license import is_pro_exclusive
+import sqlite3
+from .stats import DB_PATH, get_dashboard_data, get_project_details, get_skill_fire_counts
+
+_HTML_PATH = Path(__file__).parent / "dashboard.html"
+
+
+def _load_ignore_set() -> frozenset[str]:
+    """Load disabled skill names from .memstack-ignore in CWD."""
+    ignore_path = Path.cwd() / ".memstack-ignore"
+    if not ignore_path.exists():
+        return frozenset()
+    try:
+        lines = ignore_path.read_text(encoding="utf-8").splitlines()
+        return frozenset(
+            line.strip().lower()
+            for line in lines
+            if line.strip() and not line.strip().startswith("#")
+        )
+    except Exception:
+        return frozenset()
+
+
+_CATEGORY_MAP = {
+    "automation": "Automation", "cron-scheduler": "Automation", "n8n-workflow": "Automation",
+    "api-integration": "Automation", "webhook-receiver": "Automation",
+    "business": "Business", "quill": "Business", "scan": "Business", "governor": "Business",
+    "content": "Content", "content-pipeline": "Content", "humanize": "Content",
+    "deployment": "Deployment", "railway-deploy": "Deployment", "docker-deploy": "Deployment",
+    "vps-deploy": "Deployment",
+    "development": "Development", "forge": "Development", "shard": "Development",
+    "state": "Development", "work": "Development", "verify": "Development",
+    "project": "Development", "familiar": "Development",
+    "compress": "Core", "diary": "Core", "echo": "Core", "grimoire": "Core",
+    "sight": "Core", "token-optimization": "Core",
+    "marketing": "Marketing", "seo-geo": "Marketing",
+    "product": "Product",
+    "security": "Security", "advanced-security": "Security", "env-manager-pro": "Security",
+}
+
+
+def _derive_category(slug: str) -> str:
+    """Derive a category from a skill slug."""
+    return _CATEGORY_MAP.get(slug, "Other")
+
+
+def _strip_non_ascii(text: str) -> str:
+    """Strip non-ASCII characters (emoji, unicode) from text."""
+    return re.sub(r'[^\x00-\x7F]+', '', text).strip()
+
+
+def _get_skills_data() -> dict:
+    """Build the full skill catalog with fire counts and status."""
+    config = load_config()
+    pkl_path = config.resolved_vector_db_path / "tfidf_index.pkl"
+
+    if not pkl_path.exists():
+        return {
+            "project": os.path.basename(os.getcwd()),
+            "skills": [],
+            "total": 0,
+            "free_count": 0,
+            "pro_count": 0,
+            "disabled_count": 0,
+            "error": "No skill index found. Run reindex_skills first.",
+        }
+
+    try:
+        with open(pkl_path, "rb") as f:
+            index = pickle.load(f)
+    except Exception:
+        return {
+            "project": os.path.basename(os.getcwd()),
+            "skills": [],
+            "total": 0,
+            "free_count": 0,
+            "pro_count": 0,
+            "disabled_count": 0,
+            "error": "Failed to load skill index.",
+        }
+
+    skills_raw = index.get("skills", [])
+    fire_counts = get_skill_fire_counts()
+    ignored = _load_ignore_set()
+
+    skills = []
+    free_count = 0
+    pro_count = 0
+    disabled_count = 0
+
+    for s in skills_raw:
+        slug = s.get("slug", s.get("name", ""))
+        is_pro = is_pro_exclusive(slug)
+        name_raw = s.get("name", "")
+        name_ascii = _strip_non_ascii(name_raw).lower()
+        enabled = (slug.lower() not in ignored
+                   and name_raw.lower() not in ignored
+                   and name_ascii not in ignored)
+        fires = fire_counts.get(s.get("name", ""), 0)
+
+        if is_pro:
+            pro_count += 1
+        else:
+            free_count += 1
+        if not enabled:
+            disabled_count += 1
+
+        skills.append({
+            "name": s.get("name", ""),
+            "slug": slug,
+            "description": s.get("description", ""),
+            "source_label": s.get("source_label", ""),
+            "category": _derive_category(slug),
+            "is_pro": is_pro,
+            "enabled": enabled,
+            "fire_count": fires,
+        })
+
+    return {
+        "project": os.path.basename(os.getcwd()),
+        "skills": skills,
+        "total": len(skills),
+        "free_count": free_count,
+        "pro_count": pro_count,
+        "disabled_count": disabled_count,
+        # Display counts — use fixed values since index may not have all Pro stubs
+        "display_free_count": 85,
+        "display_pro_count": 29,
+        "display_total": 114,
+    }
+
+
+def _toggle_skill(skill_name: str, action: str) -> dict:
+    """Enable or disable a skill by updating .memstack-ignore."""
+    ignore_path = Path.cwd() / ".memstack-ignore"
+
+    # Validate action
+    if action not in ("enable", "disable"):
+        return {"success": False, "error": "Invalid action. Use 'enable' or 'disable'."}
+
+    # Validate skill exists in index
+    config = load_config()
+    pkl_path = config.resolved_vector_db_path / "tfidf_index.pkl"
+    if pkl_path.exists():
+        try:
+            with open(pkl_path, "rb") as f:
+                index = pickle.load(f)
+            known = {s.get("name", "").lower() for s in index.get("skills", [])}
+            known |= {s.get("slug", "").lower() for s in index.get("skills", [])}
+            if skill_name.lower() not in known:
+                return {"success": False, "error": f"Skill '{skill_name}' not found in index."}
+        except Exception:
+            pass
+
+    # Read existing ignore file
+    existing_lines: list[str] = []
+    if ignore_path.exists():
+        try:
+            existing_lines = ignore_path.read_text(encoding="utf-8").splitlines()
+        except Exception:
+            pass
+
+    name_lower = skill_name.lower()
+    name_lower_ascii = _strip_non_ascii(skill_name).lower()
+
+    if action == "disable":
+        # Check if already disabled
+        active = [
+            line.strip().lower()
+            for line in existing_lines
+            if line.strip() and not line.strip().startswith("#")
+        ]
+        if name_lower in active or name_lower_ascii in active:
+            return {"success": True, "skill": skill_name, "enabled": False,
+                    "disabled_count": len(active)}
+        # Strip emoji/unicode — write only ASCII to .memstack-ignore
+        existing_lines.append(_strip_non_ascii(skill_name))
+        ignore_path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = ignore_path.with_suffix(".tmp")
+        tmp.write_text("\n".join(existing_lines) + "\n", encoding="utf-8")
+        tmp.replace(ignore_path)
+        ignored = _load_ignore_set()
+        return {"success": True, "skill": skill_name, "enabled": False,
+                "disabled_count": len(ignored)}
+
+    else:  # enable
+        if not ignore_path.exists():
+            return {"success": True, "skill": skill_name, "enabled": True,
+                    "disabled_count": 0}
+        new_lines = [
+            line for line in existing_lines
+            if line.strip().lower() != name_lower and line.strip().lower() != name_lower_ascii
+        ]
+        if len(new_lines) == len(existing_lines):
+            return {"success": True, "skill": skill_name, "enabled": True,
+                    "disabled_count": len(_load_ignore_set())}
+        if not any(l.strip() and not l.strip().startswith("#") for l in new_lines):
+            try:
+                ignore_path.unlink()
+            except Exception:
+                pass
+            return {"success": True, "skill": skill_name, "enabled": True,
+                    "disabled_count": 0}
+        tmp = ignore_path.with_suffix(".tmp")
+        tmp.write_text("\n".join(new_lines) + "\n", encoding="utf-8")
+        tmp.replace(ignore_path)
+        ignored = _load_ignore_set()
+        return {"success": True, "skill": skill_name, "enabled": True,
+                "disabled_count": len(ignored)}
+
+
+def _get_category_skills() -> dict:
+    """Return skill fire counts grouped by category."""
+    if not DB_PATH.exists():
+        return {}
+    try:
+        conn = sqlite3.connect(str(DB_PATH), timeout=5)
+        rows = conn.execute(
+            """
+            SELECT COALESCE(category, 'uncategorized') as cat, skill_name, COUNT(*) as fires
+            FROM skill_fires
+            GROUP BY cat, skill_name
+            ORDER BY cat, fires DESC
+            """,
+        ).fetchall()
+        conn.close()
+    except Exception:
+        return {}
+
+    result: dict[str, list[dict]] = {}
+    for cat, name, fires in rows:
+        result.setdefault(cat, []).append({"name": name, "fires": fires})
+    return result
+
+
+def _get_diary_entries() -> list[dict]:
+    """Read diary entries from known locations."""
+    entries = []
+    diary_dirs = [
+        Path.home() / ".memstack" / "diary",
+        Path.cwd() / "memory" / "sessions",
+    ]
+    for diary_dir in diary_dirs:
+        if not diary_dir.exists():
+            continue
+        for f in sorted(diary_dir.glob("*.md"), reverse=True):
+            try:
+                content = f.read_text(encoding="utf-8", errors="replace")
+                # Extract date from filename (e.g., 2026-04-04-multi-agent-session.md)
+                name = f.stem
+                date_part = name[:10] if len(name) >= 10 else name
+                # Extract project from first line or filename
+                first_line = content.split("\n", 1)[0].strip("#").strip()
+                entries.append({
+                    "date": date_part,
+                    "filename": f.name,
+                    "title": first_line or name,
+                    "content": content,
+                    "source": str(diary_dir),
+                })
+            except Exception:
+                continue
+    return entries
+
+
+class _Handler(BaseHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass  # suppress default logging
+
+    def do_GET(self):
+        if self.path == "/":
+            body = _HTML_PATH.read_bytes()
+            self._respond(200, "text/html", body)
+        elif self.path == "/api/stats":
+            body = json.dumps(get_dashboard_data()).encode()
+            self._respond(200, "application/json", body)
+        elif self.path == "/api/skills":
+            body = json.dumps(_get_skills_data()).encode()
+            self._respond(200, "application/json", body)
+        elif self.path == "/api/diary":
+            body = json.dumps(_get_diary_entries()).encode()
+            self._respond(200, "application/json", body)
+        elif self.path == "/api/projects":
+            body = json.dumps(get_project_details()).encode()
+            self._respond(200, "application/json", body)
+        elif self.path == "/api/category-skills":
+            body = json.dumps(_get_category_skills()).encode()
+            self._respond(200, "application/json", body)
+        else:
+            self.send_error(404)
+
+    def do_POST(self):
+        if self.path == "/api/skills/toggle":
+            content_length = int(self.headers.get("Content-Length", 0))
+            if content_length == 0:
+                body = json.dumps({"success": False, "error": "Request body required."}).encode()
+                self._respond(400, "application/json", body)
+                return
+            raw = self.rfile.read(content_length)
+            try:
+                data = json.loads(raw)
+            except (json.JSONDecodeError, ValueError):
+                body = json.dumps({"success": False, "error": "Invalid JSON body."}).encode()
+                self._respond(400, "application/json", body)
+                return
+
+            skill = data.get("skill", "").strip()
+            action = data.get("action", "").strip()
+
+            if not skill:
+                body = json.dumps({"success": False, "error": "Missing 'skill' field."}).encode()
+                self._respond(400, "application/json", body)
+                return
+
+            if action not in ("enable", "disable"):
+                body = json.dumps({"success": False, "error": "Invalid action. Use 'enable' or 'disable'."}).encode()
+                self._respond(400, "application/json", body)
+                return
+
+            result = _toggle_skill(skill, action)
+            status = 200 if result.get("success") else 400
+            body = json.dumps(result).encode()
+            self._respond(status, "application/json", body)
+        else:
+            self.send_error(404)
+
+    def _respond(self, status: int, content_type: str, body: bytes):
+        self.send_response(status)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.end_headers()
+        self.wfile.write(body)
+
+
+def start_dashboard(port: int = 3333, open_browser: bool = True):
+    """Start the dashboard HTTP server (blocking)."""
+    httpd = HTTPServer(("127.0.0.1", port), _Handler)
+    url = f"http://localhost:{port}"
+    print(f"MemStack\u2122 Dashboard running at {url}", file=sys.stderr)
+    if open_browser:
+        webbrowser.open(url)
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        print("\nDashboard stopped.", file=sys.stderr)
+    finally:
+        httpd.server_close()

memstack_skill_loader/indexer.py

@@ -0,0 +1,240 @@
+"""Skill indexer — reads SKILL.md files and builds LanceDB vector index."""
+
+import sys
+from pathlib import Path
+
+import lancedb
+
+from .config import Config, load_config
+
+
+def _parse_frontmatter(content: str) -> tuple[dict, str]:
+    """Extract YAML frontmatter and body from markdown content."""
+    if not content.startswith("---"):
+        return {}, content
+
+    end = content.find("---", 3)
+    if end == -1:
+        return {}, content
+
+    frontmatter_text = content[3:end].strip()
+    body = content[end + 3:].strip()
+
+    metadata = {}
+    for line in frontmatter_text.split("\n"):
+        line = line.strip()
+        if ":" in line:
+            key, _, value = line.partition(":")
+            value = value.strip().strip('"').strip("'")
+            metadata[key.strip()] = value
+
+    return metadata, body
+
+
+def _display_name_from_filename(dirname: str) -> str:
+    """Convert 'railway-deploy' to 'Railway Deploy'."""
+    return dirname.replace("-", " ").replace("_", " ").title()
+
+
+def _display_name_from_meta_name(meta_name: str) -> str:
+    """Convert 'memstack-codebase-index' to 'Codebase Index'."""
+    parts = meta_name.split("-")
+    if parts[0] == "memstack":
+        parts = parts[1:]
+    return " ".join(parts).title()
+
+
+def _display_name_from_h1(body: str) -> str:
+    """Extract display name from the first H1 heading, before any em-dash."""
+    for line in body.split("\n"):
+        line = line.strip()
+        if line.startswith("# "):
+            heading = line[2:].strip()
+            # Strip subtitle after em-dash (e.g., "Codebase Index — Scanning...")
+            if " — " in heading:
+                heading = heading.split(" — ")[0].strip()
+            elif " - " in heading:
+                heading = heading.split(" - ")[0].strip()
+            return heading
+    return ""
+
+
+def discover_skills(config: Config) -> list[dict]:
+    """Walk skill sources and parse all SKILL.md files."""
+    skills = []
+
+    for source in config.skill_sources:
+        source_path = Path(source.path).expanduser()
+        if not source_path.exists():
+            print(f"Warning: skill source not found: {source_path}", file=sys.stderr)
+            continue
+
+        pattern = source.pattern
+        for skill_file in sorted(source_path.glob(pattern)):
+            content = skill_file.read_text(encoding="utf-8", errors="replace")
+            metadata, body = _parse_frontmatter(content)
+
+            # Primary: H1 heading (human-readable)
+            display_name = _display_name_from_h1(body)
+            # Fallback: frontmatter name (machine ID → title case)
+            if not display_name:
+                meta_name = metadata.get("name", "")
+                if meta_name:
+                    display_name = _display_name_from_meta_name(meta_name)
+            # Last resort: directory name
+            if not display_name:
+                display_name = _display_name_from_filename(skill_file.parent.name)
+
+            description = metadata.get("description", "")
+            if not description:
+                for line in body.split("\n"):
+                    line = line.strip().lstrip("#").strip()
+                    if line and not line.startswith("*"):
+                        description = line
+                        break
+
+            skills.append({
+                "name": display_name,
+                "slug": skill_file.parent.name,
+                "description": description,
+                "filename": skill_file.name,
+                "filepath": str(skill_file.resolve()),
+                "source_label": source.label,
+                "content": content,
+            })
+
+    return skills
+
+
+def _build_tfidf_index(skills: list[dict], texts: list[str], config: Config) -> None:
+    """Build a TF-IDF index and save as pickle for fast search.
+
+    Pickle is used here because sklearn's TfidfVectorizer and sparse matrices
+    cannot be serialized with JSON. The pickle is generated and consumed by
+    this same codebase (indexer writes, tfidf_search reads).
+    """
+    import pickle
+    from sklearn.feature_extraction.text import TfidfVectorizer
+
+    vectorizer = TfidfVectorizer(
+        stop_words="english",
+        ngram_range=(1, 2),
+        max_features=5000,
+        sublinear_tf=True,
+    )
+    matrix = vectorizer.fit_transform(texts)
+
+    # Store skill metadata (without vectors) alongside the TF-IDF index
+    skill_meta = [
+        {
+            "name": s["name"],
+            "slug": s["slug"],
+            "description": s["description"],
+            "filename": s["filename"],
+            "filepath": s["filepath"],
+            "source_label": s["source_label"],
+            "content": s["content"],
+        }
+        for s in skills
+    ]
+
+    pkl_path = config.resolved_vector_db_path / "tfidf_index.pkl"
+    with open(pkl_path, "wb") as f:
+        pickle.dump({"vectorizer": vectorizer, "matrix": matrix, "skills": skill_meta}, f)
+
+    # TF-IDF index saved silently
+
+
+def build_index(config: Config | None = None) -> int:
+    """Build/rebuild the LanceDB vector index. Returns number of skills indexed."""
+    if config is None:
+        config = load_config()
+
+    skills = discover_skills(config)
+    if not skills:
+        print("No skills found to index.", file=sys.stderr)
+        return 0
+
+    import os
+    import time
+    import logging
+    import warnings
+
+    # Suppress all noisy output before importing ML libraries
+    os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
+    os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    os.environ["SAFETENSORS_FAST_GPU"] = "0"
+    os.environ["TQDM_DISABLE"] = "1"
+    warnings.filterwarnings("ignore")
+    logging.getLogger("transformers").setLevel(logging.ERROR)
+    logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
+    logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
+    logging.getLogger("safetensors").setLevel(logging.ERROR)
+
+    print("Loading embedding model (first run may take 30-60 seconds)...", file=sys.stderr)
+
+    # Suppress safetensors LOAD REPORT, tqdm progress bars, and HF Hub warnings
+    # by redirecting at the OS file-descriptor level (C extensions bypass Python
+    # sys.stdout/stderr). Import is inside the block because it also emits warnings.
+    _null_fd = os.open(os.devnull, os.O_WRONLY)
+    _saved_stdout_fd = os.dup(1)
+    _saved_stderr_fd = os.dup(2)
+    os.dup2(_null_fd, 1)
+    os.dup2(_null_fd, 2)
+    try:
+        from sentence_transformers import SentenceTransformer
+        model = SentenceTransformer(config.embedding_model)
+    finally:
+        os.dup2(_saved_stdout_fd, 1)
+        os.dup2(_saved_stderr_fd, 2)
+        os.close(_saved_stdout_fd)
+        os.close(_saved_stderr_fd)
+        os.close(_null_fd)
+
+    print(f"Indexing {len(skills)} skills...", file=sys.stderr)
+    t0 = time.time()
+
+    texts = [f"{s['name']} {s['description']}" for s in skills]
+    vectors = model.encode(texts, show_progress_bar=False)
+
+    records = []
+    for i, skill in enumerate(skills):
+        records.append({
+            "name": skill["name"],
+            "description": skill["description"],
+            "filename": skill["filename"],
+            "filepath": skill["filepath"],
+            "source_label": skill["source_label"],
+            "content": skill["content"],
+            "vector": vectors[i].tolist(),
+        })
+
+    db_path = str(config.resolved_vector_db_path)
+    db = lancedb.connect(db_path)
+
+    try:
+        db.drop_table("skills")
+    except Exception:
+        pass
+
+    db.create_table("skills", data=records)
+    elapsed = time.time() - t0
+    print(f"Done! {len(records)} skills indexed in {elapsed:.1f}s", file=sys.stderr)
+
+    # Also build TF-IDF index for fast search (no PyTorch needed at search time)
+    _build_tfidf_index(skills, texts, config)
+
+    return len(records)
+
+
+def main():
+    """CLI entry point for rebuilding the index."""
+    config = load_config()
+    count = build_index(config)
+    if count == 0:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()