coding-os 0.3.2__py3-none-any.whl

scripts/generate_manifest.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Generate core/scaffold_manifest.json by initializing fresh sandboxes.
+
+For each (agent, template) combination we care about, runs `cos init`
+into a temp directory with a fixed placeholder name, then records the
+set of relative paths produced. Doctor compares this set against a
+live project to detect missing files.
+
+Path-only (no hashes) — placeholder substitution produces different
+content per project name, so hash diffs would be noisy. Hash drift is
+a follow-up feature.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+MANIFEST_PATH = REPO_ROOT / "src" / "core" / "scaffold_manifest.json"
+FIXTURE_NAME = "cos-manifest-fixture"
+
+
+def _discover_sections() -> list[tuple[str, str, list[str]]]:
+    """Compute (section_id, agent, templates) tuples from live registries.
+
+    Produces (for each adapter):
+      - <agent>_base                — base scaffold only
+      - <agent>_<stack>             — base + that stack
+
+    Adding a new stack (templates/<id>/stack.yaml) or adapter
+    (adapters/<id>/adapter.yaml) automatically shows up here — no edits
+    to this script are needed.
+    """
+    # Late imports so the script stays usable if cli imports break.
+    sys.path.insert(0, str(REPO_ROOT))
+    from cli.adapter_registry import load_adapter_registry
+    from cli.stack_registry import load_stack_registry
+
+    adapters = load_adapter_registry(REPO_ROOT / "src" / "adapters")
+    stacks = load_stack_registry(REPO_ROOT / "src" / "templates")
+
+    sections: list[tuple[str, str, list[str]]] = []
+    for agent_id in sorted(adapters):
+        sections.append((f"{agent_id}_base", agent_id, []))
+        for stack_id in sorted(stacks.keys()):
+            sections.append((f"{agent_id}_{stack_id}", agent_id, [stack_id]))
+    return sections
+
+
+# Legacy constant retained for back-compat with any external caller.
+# Computed lazily because the registries may change between invocations.
+SECTIONS: list[tuple[str, str, list[str]]] = _discover_sections()
+
+# Runtime state files — single source of truth lives in cli/doctor.py.
+sys.path.insert(0, str(REPO_ROOT))
+from cli.doctor import IGNORED_PREFIXES, RUNTIME_PATHS
+
+
+def _scaffold(agent: str, templates: list[str], target: Path) -> None:
+    cmd = [
+        sys.executable,
+        "-m",
+        "cli.main",
+        "init",
+        "--agent",
+        agent,
+        "--project-dir",
+        str(target.parent),
+        "--name",
+        target.name,
+        "--no-git",
+        "--force",
+        "--no-register",
+        # Manifest captures scaffolded file paths only; the RAG index lives in the
+        # gitignored runtime DB (excluded by _collect_paths). Skipping it turns a
+        # ~5-min 27-section run (embedding-model load per section) into ~1 min.
+        "--no-index",
+    ]
+    for t in templates:
+        cmd.extend(["--template", t])
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(REPO_ROOT)
+    subprocess.run(
+        cmd,
+        cwd=str(REPO_ROOT),
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+        check=True,
+    )
+
+
+def _collect_paths(root: Path) -> list[str]:
+    """Return sorted relative paths, excluding runtime state and VCS/build dirs."""
+    paths = []
+    for p in root.rglob("*"):
+        if not p.is_file():
+            continue
+        rel = p.relative_to(root).as_posix()
+        if rel in RUNTIME_PATHS:
+            continue
+        if any(rel.startswith(prefix) for prefix in IGNORED_PREFIXES):
+            continue
+        # Tool-cache artifacts (gitignored) that a dev checkout may carry into
+        # a scaffolded project; never part of the committed scaffold contract,
+        # so a clean-clone regen never produces them.
+        if "__pycache__/" in rel or rel.endswith(".pyc") or ".ruff_cache/" in rel:
+            continue
+        paths.append(rel)
+    return sorted(paths)
+
+
+def build_manifest() -> dict:
+    manifest: dict = {
+        "version": 1,
+        "fixture_name": FIXTURE_NAME,
+        "sections": {},
+    }
+    with tempfile.TemporaryDirectory(prefix="cos-manifest-") as tmp:
+        tmp_dir = Path(tmp)
+        for section_id, agent, templates in SECTIONS:
+            target = tmp_dir / section_id / FIXTURE_NAME
+            target.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                _scaffold(agent, templates, target)
+            except subprocess.CalledProcessError as exc:
+                print(
+                    f"[manifest] ABORTING — {section_id} init failed; "
+                    f"manifest NOT written.\n"
+                    f"  stderr: {exc.stderr}\n",
+                    file=sys.stderr,
+                )
+                raise
+            paths = _collect_paths(target)
+            manifest["sections"][section_id] = {
+                "agent": agent,
+                "templates": templates,
+                "paths": paths,
+                "count": len(paths),
+            }
+            print(f"[manifest] {section_id}: {len(paths)} files")
+            shutil.rmtree(target.parent)
+    return manifest
+
+
+def main() -> int:
+    manifest = build_manifest()
+    MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True)
+    MANIFEST_PATH.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+    total = sum(s["count"] for s in manifest["sections"].values())
+    print(
+        f"[manifest] wrote {MANIFEST_PATH.relative_to(REPO_ROOT)} "
+        f"({len(manifest['sections'])} sections, {total} files)"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/golden_sections.py

@@ -0,0 +1,20 @@
+"""SSOT for the golden snapshot matrix: (section_id, agent, [stacks]).
+
+Imported by src/scripts/capture_golden.py (the writer) and
+tests/test_golden_parity.py (the asserter) so the captured set and the
+asserted set can never drift — a fixture captured but not asserted (or
+vice versa) would be drift-blind CI.
+"""
+
+from __future__ import annotations
+
+SECTIONS: list[tuple[str, str, list[str]]] = [
+    ("claude_base", "claude", []),
+    ("claude_django", "claude", ["django"]),
+    ("claude_nextjs", "claude", ["nextjs"]),
+    ("claude_node-express", "claude", ["node-express"]),
+    ("claude_vue-nuxt", "claude", ["vue-nuxt"]),
+    ("codex_base", "codex", []),
+    ("codex_django", "codex", ["django"]),
+    ("codex_nextjs", "codex", ["nextjs"]),
+]

scripts/graph_demo.py

@@ -0,0 +1,161 @@
+"""Demo — index a folder and open the graph_os viewer in a browser.
+
+Usage:
+    uv run --extra rag python scripts/graph_demo.py              # index the repo
+    uv run --extra rag python scripts/graph_demo.py --path docs  # just docs/
+    uv run --extra rag python scripts/graph_demo.py --serve      # local HTTP server
+
+No external services; everything runs locally in SQLite + a static HTML
+file. Opens the viewer in your default browser at the end.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import tempfile
+import time
+import webbrowser
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+sys.path.insert(0, str(REPO_ROOT / "core"))
+sys.path.insert(0, str(REPO_ROOT / "core" / "thinking_os"))
+
+from database import init_db  # type: ignore
+
+from graph_os.backends.sqlite_backend import SqliteBackend
+from graph_os.extractors import (
+    code_python,
+    code_shell,
+    code_ts,
+    code_yaml,
+    contracts,
+    md_links,
+    task_deps,
+)
+from graph_os.ingest import walk_local
+from graph_os.viewer import build_view
+
+
+def _extractors_for(path: str) -> list:
+    if path.endswith(".py"):
+        return [code_python.extract, contracts.extract]
+    if path.endswith((".ts", ".tsx")):
+        return [code_ts.extract, contracts.extract]
+    if path.endswith(".sh"):
+        return [code_shell.extract]
+    if path.endswith((".yaml", ".yml")):
+        return [code_yaml.extract]
+    if path.endswith(".md"):
+        if "/tasks/" in path:
+            return [task_deps.extract, md_links.extract]
+        return [md_links.extract]
+    if path.endswith(".go"):
+        return [contracts.extract]
+    return []
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--path", default=str(REPO_ROOT))
+    parser.add_argument("--out", default=str(REPO_ROOT / ".coding-os" / "graph-viz.html"))
+    parser.add_argument("--db", default=None)
+    parser.add_argument("--max-files", type=int, default=2000)
+    parser.add_argument("--no-open", action="store_true")
+    parser.add_argument("--serve", action="store_true")
+    parser.add_argument("--port", type=int, default=0)
+    args = parser.parse_args()
+
+    db_path = args.db or tempfile.NamedTemporaryFile(suffix=".db", delete=False).name
+    print(f"[demo] init DB at {db_path}")
+    conn = init_db(db_path)
+    backend = SqliteBackend(conn=conn)
+
+    target = Path(args.path).resolve()
+    print(f"[demo] walking {target} ...")
+    plan = walk_local(target, max_files=args.max_files)
+    print(f"[demo] collected {len(plan.files)} files")
+
+    started = time.monotonic()
+    indexed = skipped = errors = 0
+    for file_path in plan.files:
+        try:
+            rel = str(file_path.relative_to(target))
+        except ValueError:
+            rel = str(file_path)
+        extractors = _extractors_for(rel)
+        if not extractors:
+            skipped += 1
+            continue
+        try:
+            content = file_path.read_text(encoding="utf-8")
+        except (UnicodeDecodeError, OSError):
+            errors += 1
+            continue
+        for extractor in extractors:
+            try:
+                result = extractor(rel, content)
+                backend.bulk_upsert(result.nodes, result.edges)
+            except Exception as exc:
+                errors += 1
+                print(f"[demo]   ! {rel}: {extractor.__name__} -> {exc}")
+        indexed += 1
+    elapsed_ms = int((time.monotonic() - started) * 1000)
+    node_count = backend.count_nodes()
+    edge_count = backend.count_edges()
+    print(
+        f"[demo] indexed {indexed} files (skipped {skipped}, errors {errors}) "
+        f"in {elapsed_ms} ms -> {node_count} nodes, {edge_count} edges"
+    )
+
+    out_path = Path(args.out).resolve()
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    build_view(backend, out_path, title=f"graph_os @ {target.name}")
+    print(f"[demo] HTML written -> {out_path}")
+
+    if args.serve:
+        _serve(out_path, port=args.port, open_browser=not args.no_open)
+    elif not args.no_open:
+        webbrowser.open(out_path.as_uri())
+        print("[demo] opened in default browser (file:// URL)")
+    else:
+        print(f"[demo] open it manually: {out_path.as_uri()}")
+    return 0
+
+
+def _serve(path: Path, *, port: int, open_browser: bool) -> None:
+    """Tiny static server - binds to 127.0.0.1 only."""
+    from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+    os.chdir(path.parent)
+
+    class _Handler(SimpleHTTPRequestHandler):
+        def log_message(self, *_args, **_kwargs):
+            pass
+
+    actual_port = port or _pick_free_port()
+    httpd = HTTPServer(("127.0.0.1", actual_port), _Handler)
+    url = f"http://127.0.0.1:{actual_port}/{path.name}"
+    print(f"[demo] serving at {url}")
+    print("[demo] Ctrl-C to stop")
+    if open_browser:
+        webbrowser.open(url)
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        print("\n[demo] bye")
+        httpd.shutdown()
+
+
+def _pick_free_port() -> int:
+    import socket
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/install-git-hooks.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Install git pre-commit + commit-msg hooks for the human persona.
+#
+# Covers the enforcement gap where a human editing files directly bypasses
+# ALL PreToolUse:Write|Edit hooks (thinking_os gate, doc-anchor, bad-patterns,
+# task-frontmatter, etc.) plus the agent-side commit-message contract.
+#
+# Install: bash src/scripts/install-git-hooks.sh
+# Uninstall: rm .git/hooks/{pre-commit,commit-msg}
+set -euo pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || true)"
+if [[ -z "$REPO_ROOT" ]]; then
+  REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+fi
+
+if [[ ! -d "${REPO_ROOT}/.git" ]]; then
+  echo "ERROR: not a git repository root: ${REPO_ROOT}" >&2
+  exit 1
+fi
+
+mkdir -p "${REPO_ROOT}/.git/hooks"
+
+install_hook() {
+  local name="$1" src="$2"
+  if [[ ! -f "$src" ]]; then
+    echo "ERROR: source hook body not found: $src" >&2
+    exit 1
+  fi
+  local dst="${REPO_ROOT}/.git/hooks/${name}"
+  cp "$src" "$dst"
+  chmod +x "$dst"
+  echo "Installed: ${dst}"
+}
+
+install_hook prepare-commit-msg "${REPO_ROOT}/src/scripts/_prepare_commit_msg_body.sh"
+install_hook pre-commit  "${REPO_ROOT}/src/scripts/_pre_commit_body.sh"
+install_hook commit-msg  "${REPO_ROOT}/src/scripts/_commit_msg_body.sh"
+install_hook post-commit "${REPO_ROOT}/src/scripts/_post_commit_body.sh"
+
+echo ""
+echo "Coverage:"
+echo "  prepare-commit-msg → stamp the active task id (TASK-NNN) into the subject so history links the commit"
+echo "  pre-commit  → block-bad-patterns, validate-task-frontmatter, block-migration-conflict"
+echo "  commit-msg  → title ≤100 chars · body ≤3 lines · no attribution / USER / Persian quotes"
+echo "  post-commit → append committed code files + sha to the task's Work Log"
+echo "Uninstall: rm ${REPO_ROOT}/.git/hooks/{prepare-commit-msg,pre-commit,commit-msg,post-commit}"

scripts/migrate_embeddings_minilm_to_bge_m3.py

@@ -0,0 +1,84 @@
+"""CLI — re-embed the existing `embeddings` table with BGE-M3.
+
+INPUT:        --batch-size · --max-batches · --status-only · --checkpoint (resume).
+OUTPUT:       per-run JSON report on stdout (rows re-embedded / remaining / status).
+DEPENDENCIES: src/core/thinking_os/migrator_embeddings.py, $COS_DB_PATH.
+NOTES:        Resumable via --checkpoint; per-batch progress delegated to
+              migrator_embeddings.run_until_idle.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
+    )
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--db", type=str, default=None, help="Override DB path.")
+    parser.add_argument(
+        "--target-model",
+        type=str,
+        default="BAAI/bge-m3",
+        help="Target model for re-embedding.",
+    )
+    parser.add_argument("--batch-size", type=int, default=256, help="Rows per batch.")
+    parser.add_argument(
+        "--checkpoint",
+        type=str,
+        default=".coding-os/.embedding-migration.json",
+        help="Checkpoint file path.",
+    )
+    parser.add_argument(
+        "--max-batches",
+        type=int,
+        default=None,
+        help="Optional cap (default: run until idle).",
+    )
+    parser.add_argument(
+        "--status-only",
+        action="store_true",
+        help="Report status without running any batches.",
+    )
+    args = parser.parse_args()
+
+    # script lives at src/scripts/ → src is two parents up; the package is
+    # src/core/thinking_os. The `core/` form predated the src/ migration and
+    # left this one level short, so the migrator could never import — the
+    # embedding migration stalled and the doctor checkpoint never cleared. TASK-409.
+    thinking_os = Path(__file__).resolve().parent.parent / "core" / "thinking_os"
+    sys.path.insert(0, str(thinking_os))
+    import migrator_embeddings  # type: ignore
+    from database import init_db  # type: ignore
+
+    conn = init_db(args.db)
+    try:
+        if args.status_only:
+            report = migrator_embeddings.migration_status(
+                conn,
+                target_model=args.target_model,
+                checkpoint_path=args.checkpoint,
+            )
+        else:
+            report = migrator_embeddings.run_until_idle(
+                conn,
+                target_model=args.target_model,
+                batch_size=args.batch_size,
+                checkpoint_path=args.checkpoint,
+                max_batches=args.max_batches,
+            )
+        print(json.dumps(report, indent=2, default=str))
+    finally:
+        conn.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())