coding-os 0.3.2__py3-none-any.whl

core/graph_os/bench/__init__.py

@@ -0,0 +1,17 @@
+"""Scale benchmark fixtures + regression gate (I.13).
+
+The bench module is intentionally small: it generates fixture corpora
+of configurable size, runs the indexer + a set of read queries against
+the backend, and records the measured numbers in a machine-readable
+shape that `docs/benchmarks/graph_os.md` pulls in.
+"""
+
+from .fixtures import build_mixed_corpus, build_python_corpus
+from .harness import BenchResult, run_benchmark
+
+__all__ = [
+    "BenchResult",
+    "build_mixed_corpus",
+    "build_python_corpus",
+    "run_benchmark",
+]

core/graph_os/bench/fixtures.py

@@ -0,0 +1,56 @@
+"""Deterministic fixture generators (I.13).
+
+DEPENDS:  stdlib only.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def build_python_corpus(root: Path, *, count: int) -> list[Path]:
+    """Generate `count` small Python modules that import each other."""
+    root.mkdir(parents=True, exist_ok=True)
+    paths: list[Path] = []
+    for i in range(count):
+        target = root / f"mod_{i:04d}.py"
+        imports = ""
+        if i > 0:
+            imports = f"from mod_{(i - 1):04d} import helper\n"
+        body = (
+            f"{imports}"
+            f"def helper(x: int) -> int:\n"
+            f'    """stable docstring for mod_{i:04d}."""\n'
+            f"    return x + {i}\n\n"
+            f"class Thing{i}:\n"
+            f"    def run(self, n: int) -> int:\n"
+            f"        return helper(n)\n"
+        )
+        target.write_text(body, encoding="utf-8")
+        paths.append(target)
+    return paths
+
+
+def build_mixed_corpus(root: Path, *, size: int = 100) -> list[Path]:
+    """Generate a heterogeneous corpus for contracts + docs coverage."""
+    paths: list[Path] = []
+    (root / "docs").mkdir(parents=True, exist_ok=True)
+    (root / "backend").mkdir(parents=True, exist_ok=True)
+    for i in range(size):
+        (root / "docs" / f"spec_{i}.md").write_text(
+            f"# Spec {i}\n\nSee [other](./spec_{(i + 1) % size}.md).\n",
+            encoding="utf-8",
+        )
+        (root / "backend" / f"route_{i}.py").write_text(
+            f"from fastapi import APIRouter\n"
+            f"router = APIRouter()\n\n"
+            f"@router.get('/items/{i}')\n"
+            f"def get_item_{i}(): return {{'i': {i}}}\n",
+            encoding="utf-8",
+        )
+    paths.extend((root / "docs").glob("*.md"))
+    paths.extend((root / "backend").glob("*.py"))
+    return paths
+
+
+__all__ = ["build_mixed_corpus", "build_python_corpus"]

core/graph_os/bench/harness.py

@@ -0,0 +1,95 @@
+"""Benchmark harness (I.13).
+
+DEPENDS:  extractors, backend, stdlib time.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterable
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+from ..backend import GraphBackend
+from ..extractors import code_python, contracts, md_links
+
+
+@dataclass
+class BenchResult:
+    corpus_size: int
+    index_duration_ms: int
+    query_duration_ms: int
+    nodes_written: int
+    edges_written: int
+    backend_id: str
+    meta: dict = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def run_benchmark(
+    backend: GraphBackend,
+    files: Iterable[Path],
+    *,
+    query_samples: int = 10,
+) -> BenchResult:
+    """Index the given files and time a small read-path sample."""
+    file_list = list(files)
+    t0 = time.monotonic()
+    nodes_total = 0
+    edges_total = 0
+    for path in file_list:
+        suffix = path.suffix.lower()
+        source = path.read_text(encoding="utf-8")
+        for extractor in _extractors_for(suffix):
+            result = extractor(str(path), source)
+            n, e = backend.bulk_upsert(result.nodes, result.edges)
+            nodes_total += n
+            edges_total += e
+    index_ms = int((time.monotonic() - t0) * 1000)
+
+    t1 = time.monotonic()
+    for _ in range(query_samples):
+        backend.list_edges(limit=50)
+    query_ms = int((time.monotonic() - t1) * 1000)
+
+    return BenchResult(
+        corpus_size=len(file_list),
+        index_duration_ms=index_ms,
+        query_duration_ms=query_ms,
+        nodes_written=nodes_total,
+        edges_written=edges_total,
+        backend_id=backend.backend_id,
+    )
+
+
+def assert_within_budget(
+    result: BenchResult,
+    *,
+    per_file_budget_ms: float = 50.0,
+    per_query_budget_ms: float = 50.0,
+    query_samples: int = 10,
+) -> None:
+    """Raise AssertionError when a run exceeds the regression budget."""
+    if result.corpus_size == 0:
+        return
+    avg_index_ms = result.index_duration_ms / result.corpus_size
+    avg_query_ms = result.query_duration_ms / max(query_samples, 1)
+    assert avg_index_ms <= per_file_budget_ms, (
+        f"indexing regressed: {avg_index_ms:.1f} ms/file > {per_file_budget_ms}"
+    )
+    assert avg_query_ms <= per_query_budget_ms, (
+        f"query regressed: {avg_query_ms:.1f} ms/query > {per_query_budget_ms}"
+    )
+
+
+def _extractors_for(suffix: str):
+    if suffix == ".py":
+        return (code_python.extract, contracts.extract)
+    if suffix == ".md":
+        return (md_links.extract,)
+    return ()
+
+
+__all__ = ["BenchResult", "assert_within_budget", "run_benchmark"]

core/graph_os/bench/persian_precision.py

@@ -0,0 +1,142 @@
+"""Persian / multilingual retrieval precision harness.
+
+DEPENDS:  embeddings module + SQLite.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+import sys
+import tempfile
+from pathlib import Path
+
+_HERE = Path(__file__).resolve()
+_CORE_DIR = _HERE.parent.parent.parent
+_TOS_DIR = _CORE_DIR / "thinking_os"
+for _p in (_CORE_DIR, _TOS_DIR):
+    if _p.exists() and str(_p) not in sys.path:
+        sys.path.insert(0, str(_p))
+
+
+FIXTURE: list[tuple[str, str]] = [
+    ("doc:authentication", "User authentication flow: login, OAuth, JWT tokens"),
+    ("doc:auth-persian", "جریان احراز هویت کاربر: ورود، OAuth، توکن‌های JWT"),
+    ("doc:payment", "Payment processing: stripe integration, commission split"),
+    ("doc:payment-persian", "پردازش پرداخت: یکپارچه‌سازی استرایپ، تقسیم کارمزد"),
+    ("doc:commission-fa", "محاسبه کمیسیون فروش برای مشاور املاک"),
+    ("doc:search", "Full-text search across products with FTS5"),
+    ("doc:search-persian", "جستجوی متن کامل در محصولات با استفاده از FTS5"),
+    ("doc:cache", "Redis cache layer with invalidation hooks"),
+    ("doc:cache-persian", "لایه کش ردیس با قلاب‌های باطل‌سازی"),
+    ("doc:analytics", "Product view analytics pipeline"),
+    ("doc:analytics-persian", "خط لوله آمار بازدید محصول"),
+    ("doc:misc", "General utility functions for date parsing"),
+]
+
+
+QUERIES: list[tuple[str, str]] = [
+    ("احراز هویت کاربر", "doc:auth-persian"),
+    ("تقسیم کارمزد", "doc:commission-fa"),
+    ("جستجوی متن کامل", "doc:search-persian"),
+    ("کش ردیس", "doc:cache-persian"),
+    ("آمار محصول", "doc:analytics-persian"),
+]
+
+
+def _seed_fixture(conn: sqlite3.Connection, model_name: str) -> None:
+    import embeddings  # type: ignore
+
+    cursor = conn.cursor()
+    cursor.execute(
+        "CREATE TABLE IF NOT EXISTS fixture (id INTEGER PRIMARY KEY, uid TEXT UNIQUE, content TEXT)"
+    )
+    for uid, content in FIXTURE:
+        cursor.execute(
+            "INSERT OR IGNORE INTO fixture (uid, content) VALUES (?, ?)",
+            (uid, content),
+        )
+    conn.commit()
+    rows = cursor.execute("SELECT id, content FROM fixture").fetchall()
+    for row_id, content in rows:
+        embeddings.upsert_embedding(conn, "fixture", int(row_id), content, model_name=model_name)
+
+
+def _score(conn: sqlite3.Connection, model_name: str) -> dict:
+    import embeddings  # type: ignore
+
+    cursor = conn.cursor()
+    hits_at_1 = hits_at_3 = 0
+    results = []
+    for query, expected_uid in QUERIES:
+        vec = embeddings.embed_text(query, model_name=model_name)
+        if vec is None:
+            results.append({"query": query, "skipped": "embed failed"})
+            continue
+        candidates = cursor.execute(
+            "SELECT e.source_id, e.embedding, f.uid FROM embeddings e "
+            "JOIN fixture f ON e.source_id = f.id WHERE e.model_name = ?",
+            (model_name,),
+        ).fetchall()
+        vectors = [row[1] for row in candidates]
+        scores = embeddings.cosine_similarity(vec, vectors)
+        ranked = sorted(
+            zip((row[2] for row in candidates), scores),
+            key=lambda pair: pair[1],
+            reverse=True,
+        )
+        if ranked and ranked[0][0] == expected_uid:
+            hits_at_1 += 1
+        if any(uid == expected_uid for uid, _ in ranked[:3]):
+            hits_at_3 += 1
+        results.append(
+            {
+                "query": query,
+                "expected": expected_uid,
+                "top_3": [{"uid": u, "score": round(s, 4)} for u, s in ranked[:3]],
+            }
+        )
+    total = len(QUERIES) or 1
+    return {
+        "model_name": model_name,
+        "precision_at_1": hits_at_1 / total,
+        "precision_at_3": hits_at_3 / total,
+        "details": results,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", default=None)
+    parser.add_argument("--output", default=None)
+    args = parser.parse_args()
+
+    import embeddings  # type: ignore
+    from database import init_db  # type: ignore
+
+    model_name = args.model or embeddings.active_model_name()
+    if not embeddings.is_available():
+        print(json.dumps({"status": "skipped", "reason": "embeddings unavailable"}))
+        return 0
+
+    db_path = Path(tempfile.NamedTemporaryFile(suffix=".db", delete=False).name)
+    conn = init_db(str(db_path))
+    try:
+        _seed_fixture(conn, model_name=model_name)
+        report = _score(conn, model_name=model_name)
+    finally:
+        conn.close()
+        db_path.unlink(missing_ok=True)
+
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+    if args.output:
+        Path(args.output).write_text(
+            json.dumps(report, indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

core/graph_os/bench/scale_500k.py

@@ -0,0 +1,120 @@
+"""500k-symbol benchmark harness.
+
+DEPENDS:  graph_os.bench.fixtures + harness, SqliteBackend (or Kuzu
+          via COS_GRAPH_BACKEND=kuzu).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+_HERE = Path(__file__).resolve()
+_CORE_DIR = _HERE.parent.parent.parent
+_TOS_DIR = _CORE_DIR / "thinking_os"
+for _p in (_CORE_DIR, _TOS_DIR):
+    if _p.exists() and str(_p) not in sys.path:
+        sys.path.insert(0, str(_p))
+
+
+def _build_shard(shard_dir: Path, *, start: int, count: int) -> list[Path]:
+    shard_dir.mkdir(parents=True, exist_ok=True)
+    paths: list[Path] = []
+    for i in range(start, start + count):
+        target = shard_dir / f"m_{i:07d}.py"
+        body = (
+            f"# generated fixture {i}\n"
+            f"def fn_{i:07d}(x: int) -> int:\n"
+            f'    """deterministic body"""\n'
+            f"    return x + {i}\n\n"
+            f"class Thing{i}:\n"
+            f"    def run(self, n: int) -> int:\n"
+            f"        return fn_{i:07d}(n)\n"
+        )
+        target.write_text(body, encoding="utf-8")
+        paths.append(target)
+    return paths
+
+
+def generate_corpus(root: Path, *, count: int, shard_size: int = 2000) -> list[Path]:
+    """Produce `count` files split across shard directories."""
+    produced: list[Path] = []
+    for shard_index in range((count + shard_size - 1) // shard_size):
+        shard_start = shard_index * shard_size
+        shard_count = min(shard_size, count - shard_start)
+        shard_dir = root / f"shard_{shard_index:04d}"
+        produced.extend(_build_shard(shard_dir, start=shard_start, count=shard_count))
+    return produced
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--count", type=int, default=500_000)
+    parser.add_argument("--corpus-dir", default=None)
+    parser.add_argument("--output", default=None)
+    parser.add_argument("--cleanup", action="store_true")
+    parser.add_argument("--limit-for-dryrun", type=int, default=None)
+    args = parser.parse_args()
+
+    import shutil
+    import tempfile
+
+    from database import init_db  # type: ignore
+
+    from graph_os.backends.sqlite_backend import SqliteBackend  # type: ignore
+    from graph_os.bench import run_benchmark  # type: ignore
+
+    count = args.limit_for_dryrun or args.count
+    corpus_root = (
+        Path(args.corpus_dir)
+        if args.corpus_dir
+        else Path(tempfile.mkdtemp(prefix="cos-bench-500k-"))
+    )
+    print(f"[bench] generating {count} files under {corpus_root} ...")
+    gen_started = time.monotonic()
+    paths = generate_corpus(corpus_root, count=count)
+    gen_elapsed = int((time.monotonic() - gen_started) * 1000)
+    print(f"[bench] generated {len(paths)} files in {gen_elapsed} ms")
+
+    db_path = Path(tempfile.NamedTemporaryFile(suffix=".db", delete=False).name)
+    conn = init_db(str(db_path))
+    backend = SqliteBackend(conn=conn)
+
+    print("[bench] running benchmark ...")
+    bench_started = time.monotonic()
+    result = run_benchmark(backend, paths)
+    bench_elapsed = int((time.monotonic() - bench_started) * 1000)
+
+    report = {
+        "corpus_size": len(paths),
+        "generation_duration_ms": gen_elapsed,
+        "benchmark_duration_ms": bench_elapsed,
+        "index_duration_ms": result.index_duration_ms,
+        "query_duration_ms": result.query_duration_ms,
+        "nodes_written": result.nodes_written,
+        "edges_written": result.edges_written,
+        "backend": result.backend_id,
+        "hardware": {
+            "uname": os.uname().sysname if hasattr(os, "uname") else "unknown",
+            "cpu_count": os.cpu_count(),
+        },
+    }
+    conn.close()
+
+    if args.output:
+        Path(args.output).write_text(json.dumps(report, indent=2, default=str), encoding="utf-8")
+    print(json.dumps(report, indent=2, default=str))
+
+    if args.cleanup:
+        shutil.rmtree(corpus_root, ignore_errors=True)
+        db_path.unlink(missing_ok=True)
+        print(f"[bench] cleaned up {corpus_root}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

core/graph_os/bench/token_cost.py

@@ -0,0 +1,170 @@
+"""Graph-envelope token cost vs naive file-read baseline (TASK-483).
+
+Measures, per documented graph workflow, the tokens of the graph envelope
+(``meta.tokens_estimated`` — a chars/4 heuristic, NOT a real tokenizer)
+against the tokens an agent would spend reading the source files to answer
+the same structural question by hand. Reuses the existing bench fixtures +
+``run_benchmark``; adds no parallel corpus and no second token estimator.
+
+The numbers are heuristic bands, not a contract: the estimator is chars/4
+(ASCII) and the naive baseline is a deliberate "read everything the query
+would otherwise force you to read" proxy. The only CI gate is gross
+regression (graph envelope > 2x its naive baseline).
+
+DEPENDS: graph_os.bench fixtures/harness, graph_os.tools.graph, _shared,
+thinking_os.database.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import tempfile
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+from ..backends.sqlite_backend import SqliteBackend
+from . import build_mixed_corpus, build_python_corpus, run_benchmark
+
+# Workflows named in the graph-first rule (.claude/rules/meta-graph-first.md)
+# that this bench measures. Order is the report order.
+WORKFLOWS = (
+    "references",
+    "rename_plan",
+    "contracts",
+    "communities+export",
+    "detect_changes",
+)
+
+
+@dataclass
+class TokenCostRow:
+    workflow: str
+    graph_tokens: int
+    naive_tokens: int
+    ratio: float  # naive / graph — naive tokens one graph token replaces
+    savings_pct: float  # (1 - graph/naive) * 100
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def _token_estimator():
+    from ..tools.graph import _envelope_module
+
+    return _envelope_module()._estimate_tokens
+
+
+def _fresh_conn(db_path: str):
+    try:
+        import database  # type: ignore
+    except ImportError:
+        thinking = Path(__file__).resolve().parent.parent.parent / "thinking_os"
+        if str(thinking) not in sys.path:
+            sys.path.insert(0, str(thinking))
+        import database  # type: ignore
+    return database.init_db(db_path)
+
+
+def _envelope_tokens(envelope: dict, estimate) -> int:
+    if isinstance(envelope, dict):
+        data = envelope.get("data")
+        if isinstance(data, dict):
+            meta = data.get("meta")
+            if isinstance(meta, dict) and "tokens_estimated" in meta:
+                return int(meta["tokens_estimated"])
+    # Fail envelope or unexpected shape — fall back to estimating the
+    # serialized response so the row is never silently zero.
+    return max(1, estimate(json.dumps(envelope, default=str)))
+
+
+def _highest_degree_function_uid(backend: SqliteBackend) -> str:
+    best_uid, best_degree = "", -1
+    for node in backend.sample_nodes("function", limit=50):
+        degree = len(backend.list_edges(target_uid=node.uid, limit=200))
+        if degree > best_degree:
+            best_uid, best_degree = node.uid, degree
+    return best_uid
+
+
+def measure_token_cost(*, python_count: int = 40, mixed_size: int = 30) -> list[TokenCostRow]:
+    """Build the standard bench corpora, index them, and return per-workflow
+    graph-envelope tokens vs the naive read-everything baseline."""
+    from ..tools import graph as graph_tools
+
+    estimate = _token_estimator()
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        py_files = build_python_corpus(root / "py", count=python_count)
+        mixed_files = build_mixed_corpus(root / "mixed", size=mixed_size)
+        all_files = py_files + mixed_files
+
+        conn = _fresh_conn(str(root / "graph.db"))
+        try:
+            backend = SqliteBackend(conn=conn)
+            run_benchmark(backend, all_files)
+            backend.link_external_stubs()
+            backend.link_import_bindings()
+
+            # Naive baselines: tokens an agent spends reading source by hand.
+            all_text = "\n".join(p.read_text(encoding="utf-8") for p in all_files)
+            route_text = "\n".join(
+                p.read_text(encoding="utf-8") for p in all_files if "route_" in p.name
+            )
+            all_tokens = max(1, estimate(all_text))
+            route_tokens = max(1, estimate(route_text)) if route_text else all_tokens
+
+            prev_singleton = graph_tools._BACKEND_SINGLETON
+            graph_tools._BACKEND_SINGLETON = backend
+            try:
+                uid = _highest_degree_function_uid(backend)
+                envelopes = {
+                    "references": [graph_tools.cos_graph_references(uid)],
+                    "rename_plan": [graph_tools.cos_graph_rename_plan(uid, "renamed_symbol")],
+                    "contracts": [graph_tools.cos_graph_contracts()],
+                    "communities+export": [
+                        graph_tools.cos_graph_communities(),
+                        graph_tools.cos_graph_export(),
+                    ],
+                    "detect_changes": [
+                        graph_tools.cos_graph_detect_changes(files=[str(p) for p in all_files])
+                    ],
+                }
+            finally:
+                graph_tools._BACKEND_SINGLETON = prev_singleton
+        finally:
+            conn.close()
+
+    naive_for = {
+        "references": all_tokens,
+        "rename_plan": all_tokens,
+        "contracts": route_tokens,
+        "communities+export": all_tokens,
+        "detect_changes": all_tokens,
+    }
+    rows: list[TokenCostRow] = []
+    for workflow in WORKFLOWS:
+        graph_tokens = max(1, sum(_envelope_tokens(e, estimate) for e in envelopes[workflow]))
+        naive_tokens = naive_for[workflow]
+        rows.append(
+            TokenCostRow(
+                workflow=workflow,
+                graph_tokens=graph_tokens,
+                naive_tokens=naive_tokens,
+                ratio=round(naive_tokens / graph_tokens, 2),
+                savings_pct=round((1 - graph_tokens / naive_tokens) * 100, 1),
+            )
+        )
+    return rows
+
+
+def report_json(rows: list[TokenCostRow] | None = None) -> str:
+    rows = rows if rows is not None else measure_token_cost()
+    return json.dumps([r.to_dict() for r in rows], indent=2)
+
+
+__all__ = ["TokenCostRow", "WORKFLOWS", "measure_token_cost", "report_json"]
+
+
+if __name__ == "__main__":
+    print(report_json())