boring-agent-memory 0.2.2__py3-none-any.whl

boring_agent_memory/cli.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+from collections.abc import Mapping
+from pathlib import Path
+
+from . import __version__
+from .canonical import list_canonical_sources, verify_canonical_source
+from .config import load_config
+from .eval import evaluate_gates, run_eval
+from .index import build_index, update_index
+from .query import query_memory
+from .schema import DEFAULT_DB_PATH, connect, fts5_available, init_db, schema_version
+from .server import serve_stdio
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="bam", description="Boring Agent Memory")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    parser.add_argument("--db", default=None, help="SQLite database path")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    init_parser = subparsers.add_parser("init", help="Initialize the local database")
+    init_parser.add_argument("--json", action="store_true")
+
+    build_parser = subparsers.add_parser("build", help="Build a local BM25 index")
+    build_parser.add_argument("--config", help="YAML, TOML, or JSON memory config")
+    build_parser.add_argument("--include", action="append", default=[], help="File or directory to index")
+    build_parser.add_argument("--exclude", action="append", default=[], help="Glob to exclude")
+    build_parser.add_argument("--workspace", default=None, help="Workspace root for relative includes")
+    build_parser.add_argument("--source-type", default=None)
+    build_parser.add_argument("--max-bytes", type=int, default=None)
+    build_parser.add_argument("--chunk-size", type=int, default=None)
+    build_parser.add_argument("--json", action="store_true")
+
+    update_parser = subparsers.add_parser("update", help="Incrementally update a local index")
+    update_parser.add_argument("--config", help="YAML, TOML, or JSON memory config")
+    update_parser.add_argument("--include", action="append", default=[], help="File or directory to index")
+    update_parser.add_argument("--exclude", action="append", default=[], help="Glob to exclude")
+    update_parser.add_argument("--workspace", default=None, help="Workspace root for relative includes")
+    update_parser.add_argument("--source-type", default=None)
+    update_parser.add_argument("--max-bytes", type=int, default=None)
+    update_parser.add_argument("--chunk-size", type=int, default=None)
+    update_parser.add_argument("--dry-run", action="store_true")
+    update_parser.add_argument("--json", action="store_true")
+
+    query_parser = subparsers.add_parser("query", help="Query the local memory index")
+    query_parser.add_argument("query")
+    query_parser.add_argument("--limit", type=int, default=5)
+    query_parser.add_argument("--source-type", default=None)
+    query_parser.add_argument("--workspace", default=None)
+    query_parser.add_argument("--json", action="store_true")
+
+    status_parser = subparsers.add_parser("status", help="Show index status")
+    status_parser.add_argument("--json", action="store_true")
+
+    inspect_parser = subparsers.add_parser("inspect", help="Inspect indexed canonical sources")
+    inspect_parser.add_argument("source_path", nargs="?", help="Canonical source path to verify")
+    inspect_parser.add_argument("--limit", type=int, default=50)
+    inspect_parser.add_argument("--json", action="store_true")
+
+    health_parser = subparsers.add_parser("health", help="Run local health checks")
+    health_parser.add_argument("--json", action="store_true")
+
+    eval_parser = subparsers.add_parser("eval", help="Run deterministic memory retrieval evals")
+    eval_parser.add_argument(
+        "--fixture",
+        "--fixtures",
+        dest="fixtures",
+        default="evals/fixtures",
+        help="Directory containing eval fixture files",
+    )
+    eval_parser.add_argument(
+        "--golden",
+        default="evals/golden.jsonl",
+        help="JSONL file with golden eval cases",
+    )
+    eval_parser.add_argument("--limit", type=int, default=3)
+    eval_parser.add_argument("--min-recall-at-1", type=float, default=None)
+    eval_parser.add_argument("--min-recall-at-3", type=float, default=None)
+    eval_parser.add_argument("--min-source-accuracy", type=float, default=None)
+    eval_parser.add_argument("--min-snippet-term-rate", type=float, default=None)
+    eval_parser.add_argument("--min-stale-detection-rate", type=float, default=None)
+    eval_parser.add_argument("--max-privacy-leaks", type=int, default=None)
+    eval_parser.add_argument("--json", action="store_true")
+
+    serve_parser = subparsers.add_parser("serve", help="Serve a tiny JSON-lines query interface")
+    serve_parser.add_argument("--stdio", action="store_true", help="Read JSON lines from stdin")
+
+    args = parser.parse_args(argv)
+    db_path = Path(args.db).expanduser() if args.db else DEFAULT_DB_PATH
+
+    if args.command == "init":
+        conn = connect(db_path)
+        init_db(conn)
+        ok = fts5_available(conn)
+        conn.close()
+        return emit({"database": db_path.as_posix(), "fts5": ok}, args.json)
+
+    if args.command in {"build", "update"}:
+        try:
+            config = load_config(args.config) if args.config else None
+        except (OSError, ValueError) as exc:
+            parser.error(str(exc))
+        db_path = Path(args.db).expanduser() if args.db else (config.index_path if config else DEFAULT_DB_PATH)
+        includes = [*(config.include if config else ()), *args.include]
+        excludes = [*(config.exclude if config else ()), *args.exclude]
+        workspace = args.workspace or (config.workspace if config else None)
+        source_type = args.source_type or (config.source_type if config else "file")
+        max_bytes = args.max_bytes if args.max_bytes is not None else (config.max_bytes if config else 512 * 1024)
+        chunk_size = args.chunk_size if args.chunk_size is not None else (config.chunk_size if config else 1600)
+        if not includes:
+            parser.error(f"{args.command} requires --include or --config with include paths")
+        stats: dict[str, object]
+        if args.command == "build":
+            try:
+                stats = dict(
+                    build_index(
+                        db_path=db_path,
+                        includes=list(includes),
+                        excludes=list(excludes),
+                        workspace=workspace,
+                        source_type=source_type,
+                        max_bytes=max_bytes,
+                        chunk_size=chunk_size,
+                    )
+                )
+            except (OSError, ValueError, sqlite3.DatabaseError) as exc:
+                parser.error(str(exc))
+        else:
+            try:
+                stats = update_index(
+                    db_path=db_path,
+                    includes=list(includes),
+                    excludes=list(excludes),
+                    workspace=workspace,
+                    source_type=source_type,
+                    max_bytes=max_bytes,
+                    chunk_size=chunk_size,
+                    dry_run=args.dry_run,
+                )
+            except (OSError, ValueError, sqlite3.DatabaseError) as exc:
+                parser.error(str(exc))
+        return emit({"database": db_path.as_posix(), **stats}, args.json)
+
+    if args.command == "query":
+        try:
+            results = [
+                result.to_dict()
+                for result in query_memory(
+                    db_path=db_path,
+                    query=args.query,
+                    limit=args.limit,
+                    source_type=args.source_type,
+                    workspace=args.workspace,
+                )
+            ]
+        except FileNotFoundError as exc:
+            parser.error(str(exc))
+        if args.json:
+            print(json.dumps({"query": args.query, "results": results}, indent=2, ensure_ascii=False))
+        else:
+            print_results(results)
+        return 0
+
+    if args.command == "status":
+        return emit(status(db_path), args.json)
+
+    if args.command == "inspect":
+        try:
+            if args.source_path:
+                return emit(verify_canonical_source(db_path, args.source_path), args.json)
+            payload: dict[str, object] = {
+                "database": db_path.as_posix(),
+                "sources": list_canonical_sources(db_path, limit=args.limit),
+            }
+        except FileNotFoundError as exc:
+            parser.error(str(exc))
+        return emit(payload, args.json)
+
+    if args.command == "health":
+        health = status(db_path)
+        health["sqlite_fts5"] = _sqlite_fts5_health()
+        health["ok"] = health["sqlite_fts5"] and health["database_exists"]
+        return emit(health, args.json)
+
+    if args.command == "eval":
+        try:
+            report = run_eval(fixture_dir=args.fixtures, golden_path=args.golden, limit=args.limit)
+        except FileNotFoundError as exc:
+            parser.error(f"eval fixtures not found: {exc}; run from a repository checkout or pass --fixtures and --golden")
+        failures = evaluate_gates(
+            report,
+            min_recall_at_1=args.min_recall_at_1,
+            min_recall_at_3=args.min_recall_at_3,
+            min_source_accuracy=args.min_source_accuracy,
+            min_snippet_term_rate=args.min_snippet_term_rate,
+            min_stale_detection_rate=args.min_stale_detection_rate,
+            max_privacy_leaks=args.max_privacy_leaks,
+        )
+        report["passed"] = not failures
+        report["failures"] = failures
+        if args.json:
+            print(json.dumps(report, indent=2, ensure_ascii=False))
+        else:
+            print_eval_report(report)
+        return 0 if not failures else 1
+
+    if args.command == "serve":
+        if not args.stdio:
+            parser.error("serve currently requires --stdio")
+        return serve_stdio(db_path)
+
+    parser.error("unknown command")
+    return 2
+
+
+def status(db_path: Path) -> dict[str, object]:
+    if not db_path.exists():
+        return {
+            "database": db_path.as_posix(),
+            "database_exists": False,
+            "records": 0,
+            "documents": 0,
+            "chunks": 0,
+            "schema_version": None,
+            "config_fingerprint": None,
+            "source_types": {},
+        }
+    conn = connect(db_path)
+    init_db(conn)
+    documents = conn.execute("SELECT count(*) FROM documents").fetchone()[0]
+    chunks = conn.execute("SELECT count(*) FROM chunks").fetchone()[0]
+    rows = conn.execute(
+        "SELECT source_type, count(*) AS count FROM documents GROUP BY source_type ORDER BY source_type"
+    ).fetchall()
+    metadata = conn.execute("SELECT config_fingerprint FROM index_metadata WHERE singleton = 1").fetchone()
+    version = schema_version(conn)
+    conn.close()
+    return {
+        "database": db_path.as_posix(),
+        "database_exists": True,
+        "records": documents,
+        "documents": documents,
+        "chunks": chunks,
+        "schema_version": version,
+        "config_fingerprint": metadata["config_fingerprint"] if metadata else None,
+        "source_types": {row["source_type"]: row["count"] for row in rows},
+    }
+
+
+def _sqlite_fts5_health() -> bool:
+    conn = sqlite3.connect(":memory:")
+    try:
+        return fts5_available(conn)
+    finally:
+        conn.close()
+
+
+def emit(payload: Mapping[str, object], as_json: bool) -> int:
+    if as_json:
+        print(json.dumps(payload, indent=2, ensure_ascii=False))
+    else:
+        for key, value in payload.items():
+            print(f"{key}: {value}")
+    return 0
+
+
+def print_results(results: list[dict[str, object]]) -> None:
+    for result in results:
+        print(f"{result['title']}  score={result['score']}")
+        print(f"  {result['source_path']}")
+        print(f"  {result['snippet']}")
+
+
+def print_eval_report(report: dict[str, object]) -> None:
+    metrics = report["metrics"]
+    assert isinstance(metrics, dict)
+    print(f"cases: {metrics['cases']}")
+    print(f"recall_at_1: {metrics['recall_at_1']:.3f}")
+    print(f"recall_at_3: {metrics['recall_at_3']:.3f}")
+    print(f"mrr: {metrics['mrr']:.3f}")
+    print(f"source_accuracy: {metrics['source_accuracy']:.3f}")
+    print(f"snippet_term_rate: {metrics['snippet_term_rate']:.3f}")
+    print(f"privacy_leak_count: {metrics['privacy_leak_count']}")
+    print(f"stale_detection_rate: {metrics['stale_detection_rate']:.3f}")
+    failures = report.get("failures", [])
+    if failures:
+        print("failures:")
+        assert isinstance(failures, list)
+        for failure in failures:
+            print(f"- {failure}")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

boring_agent_memory/config.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import json
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .chunking import DEFAULT_CHUNK_SIZE
+from .schema import DEFAULT_DB_PATH
+
+
+@dataclass(frozen=True)
+class MemoryConfig:
+    index_path: Path = DEFAULT_DB_PATH
+    include: tuple[str, ...] = ()
+    exclude: tuple[str, ...] = ()
+    workspace: Path | None = None
+    source_type: str = "file"
+    max_file_size_kb: int = 512
+    chunk_size: int = DEFAULT_CHUNK_SIZE
+
+    @property
+    def max_bytes(self) -> int:
+        return self.max_file_size_kb * 1024
+
+
+def load_config(path: Path | str) -> MemoryConfig:
+    config_path = Path(path).expanduser()
+    raw = _load_mapping(config_path)
+    base_dir = config_path.resolve().parent
+    privacy_value = raw.get("privacy")
+    privacy: dict[str, Any] = privacy_value if isinstance(privacy_value, dict) else {}
+
+    index_path = _path_value(raw.get("index_path", raw.get("database")), DEFAULT_DB_PATH, base_dir)
+    workspace = _optional_path_value(raw.get("workspace"), base_dir)
+    if "max_bytes" in raw:
+        max_file_size_kb = max(1, int(raw["max_bytes"]) // 1024)
+    else:
+        max_file_size_kb = int(privacy.get("max_file_size_kb", raw.get("max_file_size_kb", 512)))
+
+    return MemoryConfig(
+        index_path=index_path,
+        include=tuple(_list_value(raw.get("include"))),
+        exclude=tuple(_list_value(raw.get("exclude"))),
+        workspace=workspace,
+        source_type=str(raw.get("source_type", "file")),
+        max_file_size_kb=max_file_size_kb,
+        chunk_size=int(raw.get("chunk_size", DEFAULT_CHUNK_SIZE)),
+    )
+
+
+def _load_mapping(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    suffix = path.suffix.lower()
+    if suffix == ".json":
+        data = json.loads(text)
+    elif suffix == ".toml":
+        data = tomllib.loads(text)
+    elif suffix in {".yaml", ".yml"}:
+        data = _parse_simple_yaml(text)
+    else:
+        raise ValueError(f"Unsupported config format: {path.suffix}")
+    if not isinstance(data, dict):
+        raise ValueError("Config root must be a mapping")  # noqa: TRY004
+    return data
+
+
+def _parse_simple_yaml(text: str) -> dict[str, Any]:
+    """Parse the small YAML subset used by BAM examples without a dependency."""
+    data: dict[str, Any] = {}
+    current_key: str | None = None
+
+    for line_number, raw_line in enumerate(text.splitlines(), start=1):
+        line = _strip_comment(raw_line).rstrip()
+        if not line.strip():
+            continue
+
+        stripped = line.strip()
+        if not line.startswith(" ") and ":" in stripped and not stripped.startswith("- "):
+            key, value = stripped.split(":", 1)
+            key = key.strip()
+            value = value.strip()
+            if value:
+                data[key] = _scalar(value)
+                current_key = None
+            else:
+                data[key] = {}
+                current_key = key
+            continue
+
+        if stripped.startswith("- ") and current_key:
+            if data[current_key] == {}:
+                data[current_key] = []
+            if not isinstance(data[current_key], list):
+                raise ValueError(f"unsupported YAML on line {line_number}: {current_key} mixes keys and list items")
+            data[current_key].append(_scalar(stripped[2:].strip()))
+            continue
+
+        if current_key and isinstance(data.get(current_key), dict) and ":" in stripped:
+            key, value = stripped.split(":", 1)
+            data[current_key][key.strip()] = _scalar(value.strip())
+            continue
+
+        raise ValueError(f"unsupported YAML on line {line_number}: {stripped}")
+
+    return data
+
+
+def _strip_comment(line: str) -> str:
+    quote = ""
+    for index, char in enumerate(line):
+        starts_token = index == 0 or line[index - 1].isspace()
+        if quote:
+            if char == quote:
+                quote = ""
+        elif char in {"'", '"'} and starts_token:
+            quote = char
+        elif char == "#" and starts_token:
+            return line[:index]
+    return line
+
+
+def _scalar(value: str) -> Any:
+    value = value.strip()
+    if value[:1] in {"[", "{"}:
+        raise ValueError(f"unsupported YAML flow syntax: {value}; use a block list")
+    if value[:1] in {"'", '"'}:
+        inner = value[1:-1]
+        if len(value) < 2 or value[-1] != value[0] or "''" in inner or (value[0] == '"' and "\\" in inner):
+            raise ValueError(f"unsupported YAML quoting: {value}")
+        return inner
+    if value.lower() in {"true", "false"}:
+        return value.lower() == "true"
+    try:
+        return int(value)
+    except ValueError:
+        return value
+
+
+def _path_value(value: Any, default: Path, base_dir: Path) -> Path:
+    if value is None:
+        return default
+    path = Path(str(value)).expanduser()
+    return path if path.is_absolute() else base_dir / path
+
+
+def _optional_path_value(value: Any, base_dir: Path) -> Path | None:
+    if value in {None, ""}:
+        return None
+    path = Path(str(value)).expanduser()
+    return path if path.is_absolute() else base_dir / path
+
+
+def _list_value(value: Any) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, list):
+        return [str(item) for item in value]
+    raise ValueError("Expected a string or list")