cortexm 0.3.0__py3-none-any.whl

cortexm/cli.py

@@ -0,0 +1,295 @@
+"""cortexm — the Context-M command line.
+
+    cortexm serve                    # MCP server (stdio JSON-RPC)
+    cortexm stats [--db PATH]
+    cortexm verify [--db PATH]       # integrity audit (hashes + vectors)
+    cortexm consolidate [--db PATH]
+    cortexm migrate --from mem0 --path mem0.db [--db PATH]
+    cortexm cost --memories 1000000  # μ=0 cost calculator
+    cortexm bench --buckets 128k,1m  # BEAM-style benchmark
+    cortexm export-schema [--db PATH]
+    cortexm git log|branches|diff|blame
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def _memory(args):
+    from cortexm.api.memory import Memory
+    from cortexm.config import Config
+    cfg = Config.from_env()
+    if getattr(args, "db", None):
+        cfg.db_path = args.db
+    return Memory(cfg)
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="cortexm",
+                                 description="Context-M memory fabric CLI")
+    sub = ap.add_subparsers(dest="cmd")
+
+    p = sub.add_parser("serve", help="run the MCP server on stdio")
+    p.add_argument("--db", default=None)
+
+    p = sub.add_parser("serve-rest", help="run the REST API server (HTTP)")
+    p.add_argument("--db", default=":memory:")
+    p.add_argument("--host", default="0.0.0.0")
+    p.add_argument("--port", type=int, default=8900)
+    p.add_argument("--pii", default=None, choices=[None, "off", "redact",
+                                                    "block", "tag"])
+    p.add_argument("--admin-key", default=None,
+                   help="mint an admin API key at boot (printed once)")
+    p.add_argument("--sparql-port", type=int, default=None,
+                   help="co-host a SPARQL endpoint on this port "
+                        "(shares one Memory instance with the REST API)")
+    p.add_argument("--sparql-host", default="0.0.0.0",
+                   help="bind SPARQL endpoint to this host")
+    p.add_argument("--sparql-user-id", default=None,
+                   help="scope SPARQL queries to a single user")
+
+    for name, help_ in (("stats", "memory statistics"),
+                        ("verify", "integrity audit"),
+                        ("consolidate", "lifecycle consolidation"),
+                        ("export-schema", "federation schema report")):
+        p = sub.add_parser(name, help=help_)
+        p.add_argument("--db", default=None)
+
+    # extend consolidate with toggles for each pass
+    # (the loop above already created 'consolidate'; we fetch it
+    # by name and add extra flags)
+    con = [a for a in sub.choices.values() if a.prog.endswith("consolidate")][0]
+    con.add_argument("--dry-run", action="store_true",
+                     help="don't commit changes — just report what would happen")
+    con.add_argument("--no-lifecycle", action="store_true",
+                     help="skip the lifecycle pass (short→long promotion / decay)")
+    con.add_argument("--no-dreaming", action="store_true",
+                     help="skip the Aeon dreaming pass (merge triples, retire stale, "
+                          "defrag palace, retrain prefetcher)")
+    con.add_argument("--user-id", default=None,
+                     help="scope the dreaming pass to one user")
+
+    p = sub.add_parser("keys", help="API key management (RBAC)")
+    p.add_argument("op", choices=["create", "list", "revoke"])
+    p.add_argument("--db", default=None)
+    p.add_argument("--role", default="reader",
+                   choices=["admin", "operator", "reader", "auditor"])
+    p.add_argument("--label", default="")
+    p.add_argument("--id", default=None, help="key id to revoke")
+
+    p = sub.add_parser("audit", help="audit log tail / verify / export")
+    p.add_argument("op", choices=["tail", "verify", "export"])
+    p.add_argument("--db", default=None)
+    p.add_argument("-n", type=int, default=30)
+    p.add_argument("--out", default="audit.jsonl",
+                   help="export path (op=export)")
+
+    p = sub.add_parser("snapshot", help="atomic backup with manifest")
+    p.add_argument("--db", default=None)
+    p.add_argument("--path", required=True)
+
+    p = sub.add_parser("erase", help="GDPR right-to-erasure")
+    p.add_argument("--db", default=None)
+    p.add_argument("--user-id", required=True)
+
+    p = sub.add_parser("governance", help="governance ops")
+    p.add_argument("op", choices=["retention", "state-at"])
+    p.add_argument("--db", default=None)
+    p.add_argument("--days", type=int, default=365)
+    p.add_argument("--when", default=None,
+                   help="ISO datetime for state-at")
+    p.add_argument("--user-id", default=None)
+
+    p = sub.add_parser("migrate", help="import from mem0/zep/chroma")
+    p.add_argument("--from", dest="source", required=True,
+                   choices=["mem0", "zep", "chroma"])
+    p.add_argument("--path", required=True)
+    p.add_argument("--db", default=None)
+    p.add_argument("--user-id", default="migrated")
+
+    p = sub.add_parser("cost", help="μ=0 cost calculator")
+    p.add_argument("--memories", type=int, default=1_000_000)
+    p.add_argument("--ingest-per-memory", type=float, default=0.001,
+                   help="competitor LLM-extract cost per memory (USD)")
+
+    p = sub.add_parser("bench", help="BEAM-style benchmark")
+    p.add_argument("--buckets", default="128k")
+    p.add_argument("--micro", action="store_true")
+
+    p = sub.add_parser("git", help="Memory Git operations")
+    p.add_argument("op", choices=["log", "branches", "diff", "blame"])
+    p.add_argument("rest", nargs="*")
+    p.add_argument("--db", default=None)
+
+    args = ap.parse_args(argv)
+    if not args.cmd:
+        ap.print_help()
+        return 0
+
+    if args.cmd == "serve":
+        from cortexm.mcp.server import serve
+        serve(args.db)
+        return 0
+
+    if args.cmd == "serve-rest":
+        from cortexm.server.rest import main as rest_main
+        # argparse passthrough for the REST server
+        rest_argv = ["--db", args.db, "--host", args.host,
+                     "--port", str(args.port)]
+        if args.pii:
+            rest_argv += ["--pii", args.pii]
+        if args.admin_key:
+            rest_argv += ["--admin-key", args.admin_key]
+        if getattr(args, "sparql_port", None):
+            rest_argv += ["--sparql-port", str(args.sparql_port),
+                          "--sparql-host", args.sparql_host]
+            if args.sparql_user_id:
+                rest_argv += ["--sparql-user-id", args.sparql_user_id]
+        import sys as _sys
+        _sys.argv = ["contextm-serve"] + rest_argv
+        rest_main()
+        return 0
+
+    if args.cmd == "keys":
+        m = _memory(args)
+        try:
+            if args.op == "create":
+                out = m.keys.create(args.role, label=args.label)
+                print(json.dumps(out, indent=2))
+            elif args.op == "list":
+                print(json.dumps({"keys": m.keys.list_keys()}, indent=2))
+            elif args.op == "revoke":
+                print(json.dumps({"revoked": m.keys.revoke(args.id or "")}))
+        finally:
+            m.close()
+        return 0
+
+    if args.cmd == "audit":
+        m = _memory(args)
+        try:
+            if args.op == "verify":
+                print(json.dumps(m.audit_log.verify(), indent=2))
+            elif args.op == "export":
+                n = m.audit_log.export_jsonl(args.out)
+                print(json.dumps({"exported": n, "path": args.out}))
+            else:
+                print(json.dumps({"events": m.audit_log.tail(args.n)}, indent=2))
+        finally:
+            m.close()
+        return 0
+
+    if args.cmd == "snapshot":
+        m = _memory(args)
+        try:
+            print(json.dumps(m.governance.snapshot(args.path), indent=2))
+        finally:
+            m.close()
+        return 0
+
+    if args.cmd == "erase":
+        m = _memory(args)
+        try:
+            print(json.dumps(m.governance.erase_user(args.user_id), indent=2))
+        finally:
+            m.close()
+        return 0
+
+    if args.cmd == "governance":
+        m = _memory(args)
+        try:
+            if args.op == "retention":
+                out = m.governance.apply_retention(args.days,
+                                                    user_id=args.user_id)
+            else:
+                out = {"facts": m.governance.state_at(args.when or "",
+                                                       user_id=args.user_id)}
+            print(json.dumps(out, indent=2, default=str))
+        finally:
+            m.close()
+        return 0
+
+    if args.cmd == "cost":
+        print(cost_report(args.memories, args.ingest_per_memory))
+        return 0
+
+    if args.cmd == "bench":
+        from cortexm.bench.run import main as bench_main
+        argv = ["--buckets", args.buckets]
+        if args.micro:
+            argv.append("--micro")
+        bench_main()
+        return 0
+
+    m = _memory(args)
+    try:
+        if args.cmd == "stats":
+            print(json.dumps(m.stats(), indent=2, default=str))
+        elif args.cmd == "verify":
+            print(json.dumps(m.verify_integrity(), indent=2, default=str))
+        elif args.cmd == "consolidate":
+            print(json.dumps(m.consolidate(
+                dry_run=getattr(args, "dry_run", False),
+                lifecycle=not getattr(args, "no_lifecycle", False),
+                dreaming=not getattr(args, "no_dreaming", False),
+                user_id=getattr(args, "user_id", None),
+            ), indent=2, default=str))
+        elif args.cmd == "export-schema":
+            print(json.dumps(m.export_schema_report(), indent=2, default=str))
+        elif args.cmd == "migrate":
+            from cortexm.migrate.importers import MIGRATORS
+            out = MIGRATORS[args.source](m, args.path, user_id=args.user_id)
+            print(json.dumps(out, indent=2))
+        elif args.cmd == "git":
+            if args.op == "log":
+                for c in m.log():
+                    print(f"{c['id'][:8]} {c['ts']} {c['message']} "
+                          f"({json.loads(c['parents'])})")
+            elif args.op == "branches":
+                for b in m.branches():
+                    print(f"{b['name']}: {b['head'][:8]}")
+            elif args.op == "diff":
+                if len(args.rest) < 2:
+                    print("usage: cortexm git diff <commitA> <commitB>")
+                    return 2
+                print(json.dumps(m.diff(args.rest[0], args.rest[1]),
+                                 indent=2))
+            elif args.op == "blame":
+                if not args.rest:
+                    print("usage: cortexm git blame <subject> [relation]")
+                    return 2
+                rel = args.rest[1] if len(args.rest) > 1 else None
+                for row in m.blame(args.rest[0], rel):
+                    print(f"{row['commit'][:8] if row['commit'] else '--------'} "
+                          f"{row['recorded_at'][:10]} {row['fact']} "
+                          f"{'[active]' if row['active'] else '[retired]'}")
+        return 0
+    finally:
+        m.close()
+
+
+def cost_report(memories: int, competitor_per_memory: float) -> str:
+    """The μ=0 cost asymmetry (Section: MOAT 6)."""
+    ours = memories * 0.00001      # deterministic CPU-only ingest
+    theirs = memories * competitor_per_memory
+    lines = [
+        "Context-M μ=0 Cost Calculator",
+        "============================",
+        f"memories:                 {memories:,}",
+        f"cortex-m ingest (CPU):    ${ours:,.2f}",
+        f"LLM-in-loop ingest:       ${theirs:,.2f}",
+        f"cost advantage:           {theirs / max(ours, 1e-9):,.0f}x",
+        "",
+        "storage tiers (per million memories):",
+        "  int8   770 MB  baseline",
+        "  binary  96 MB  edge (Raspberry Pi 5 → 10M memories)",
+        "  rabitq  96 MB  ultra-edge",
+        "  pq       8 MB  cloud",
+    ]
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cortexm/cognition/__init__.py

@@ -0,0 +1,53 @@
+"""HMS-style Cognition Engine — background self-organization.
+
+The standout feature of holographic-memory (HMS): a background thread
+that surfaces patterns, builds abstractions, detects knowledge gaps,
+hypothesizes fillers, and finds analogies across domains.
+
+This is the active part of memory — it turns a passive store into a
+self-organizing knowledge base. When a user says "Alice works at Google"
+and later "Alice moved to Mountain View," the engine should hypothesize
+(Alice, lives_in, Mountain_View) and flag it for confirmation.
+
+Architecture:
+  PatternScanner   — surfaces structural regularities across triples
+  AbstractionEngine — bundles atom vectors into prototypes when N
+                       entities share a relation pattern
+  GapDetector      — finds missing relations by comparing an entity's
+                       profile to peers
+  HypothesisEngine — proposes fillers for gaps via Hopfield cleanup
+  AnalogyDetector  — finds structurally isomorphic domains via
+                       bipartite relation mapping
+
+Unlike HMS (which runs in a background thread), we trigger the engine
+from `cortexm consolidate` — deterministic, auditable, no surprise
+writes. Output is HYPOTHESIZED_BY edges in the Trace with confidence
+< 0.5; never active in retrieval unless explicitly promoted.
+
+The 5 stages run in a fixed pipeline so a single consolidate() call
+produces a full self-organization sweep. Each stage reads the prior
+stage's outputs via the Trace, so the engine is composable.
+"""
+
+from cortexm.cognition.scanner import PatternScanner
+from cortexm.cognition.abstraction import AbstractionEngine
+from cortexm.cognition.gaps import GapDetector, HypothesisEngine
+from cortexm.cognition.analogy import AnalogyDetector
+from cortexm.cognition.engine import (
+    CognitionEngine,
+    run_cognition_pass,
+    HYPOTHESIZED_BY,
+    PROMOTED_FROM,
+)
+
+__all__ = [
+    "PatternScanner",
+    "AbstractionEngine",
+    "GapDetector",
+    "HypothesisEngine",
+    "AnalogyDetector",
+    "CognitionEngine",
+    "run_cognition_pass",
+    "HYPOTHESIZED_BY",
+    "PROMOTED_FROM",
+]

cortexm/cognition/abstraction.py

@@ -0,0 +1,192 @@
+"""AbstractionEngine — bundles atom vectors into prototype categories.
+
+Second stage of the HMS cognition engine. Consumes the patterns
+surfaced by PatternScanner and builds prototype categories:
+
+  - When N entities share a relation pattern (e.g. works_at + lives_in),
+    create a prototype "person" category and tag those entities as
+    members.
+  - When N values appear for the same relation across subjects
+    (value_fanout), those values become "category centroids" — e.g.
+    if (Google, Stripe, Anthropic) all appear as works_at values,
+    they form a "tech_company" prototype.
+
+The prototype is stored as a derived fact in the Trace:
+  (proto:<name>, member_of, <entity>)
+  with provenance.kind = "abstraction" and confidence < 0.5
+
+This stage does NOT delete facts — it adds new derived facts that
+link entities to abstractions. The derived facts are tagged
+`is_derived=1` so they don't pollute the active fact count.
+
+Prototype vectors are also computed in the VSA palace (superposition
+of members' holograms) so retrieval can match against the prototype
+directly — but this is the palace's responsibility, not the
+abstraction engine's. We only emit the membership edges here.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any
+
+from cortexm.cognition.scanner import Pattern, ScanResult
+from cortexm.trace.store import TraceStore
+from cortexm.util import iso, new_id
+
+
+@dataclass
+class Abstraction:
+    """A prototype category discovered by AbstractionEngine."""
+    name: str                # e.g. "person" or "tech_company"
+    kind: str                 # "subject_role" | "value_cluster"
+    members: list[str] = field(default_factory=list)
+    prototype_relations: list[str] = field(default_factory=list)
+    confidence: float = 0.0
+
+
+@dataclass
+class AbstractionResult:
+    abstractions: list[Abstraction] = field(default_factory=list)
+    membership_edges_added: int = 0
+    duration_ms: float = 0.0
+
+
+class AbstractionEngine:
+    """Builds prototype categories from surfaced patterns."""
+
+    def __init__(self, store: TraceStore,
+                 min_members: int = 2,
+                 min_co_occur_support: int = 2) -> None:
+        self.store = store
+        self.min_members = min_members
+        self.min_co_occur_support = min_co_occur_support
+
+    def run(self, scan: ScanResult, *,
+            dry_run: bool = False,
+            commit_id: str | None = None,
+            user_id: str | None = None) -> AbstractionResult:
+        """Build abstractions from scan results."""
+        import time
+        t0 = time.perf_counter()
+
+        # subject-role abstractions: subjects with the same set of
+        # relations (e.g. {works_at, lives_in, has_skill}) form a
+        # "person" prototype. Group subjects by frozenset(relations).
+        rel_groups: dict[frozenset, list[str]] = defaultdict(list)
+        for p in scan.patterns:
+            if p.kind != "subject_fanout":
+                continue
+            if p.support < self.min_co_occur_support:
+                continue
+            rels = frozenset(p.payload.get("relations", []))
+            if len(rels) >= 2:
+                rel_groups[rels].append(p.payload["subject"])
+
+        # value-cluster abstractions: values that appear for the same
+        # relation across N subjects form a "value_cluster" prototype
+        # (e.g. tech_company for {Google, Stripe, Anthropic, OpenAI})
+        val_groups: dict[tuple[str, list[str]], list[str]] = defaultdict(list)
+        for p in scan.patterns:
+            if p.kind != "value_fanout":
+                continue
+            if p.support < self.min_members:
+                continue
+            # which relation does this value cluster under? We don't
+            # know directly from the pattern — look it up.
+            val = p.payload["value"]
+            subjs = p.payload["subjects"]
+            # find the relation(s) that produce this value across subjects
+            rel_rows = self.store.conn.execute(
+                "SELECT DISTINCT relation FROM facts WHERE value=? "
+                "AND is_active=1 AND quarantined=0 LIMIT 3",
+                (val,)).fetchall()
+            for r in rel_rows:
+                rel = r[0]
+                key = (rel, [val])
+                val_groups[key].extend(subjs)
+
+        abstractions: list[Abstraction] = []
+        # synthesize subject-role abstractions
+        proto_idx = 0
+        for rels, members in rel_groups.items():
+            if len(members) < self.min_members:
+                continue
+            name = f"proto:role_{proto_idx}"
+            proto_idx += 1
+            abstractions.append(Abstraction(
+                name=name,
+                kind="subject_role",
+                members=members,
+                prototype_relations=sorted(rels),
+                confidence=min(0.49, 0.10 + 0.05 * len(members))))
+
+        # synthesize value-cluster abstractions
+        cluster_idx = 0
+        seen_clusters: set[frozenset] = set()
+        for (rel, vals), members in val_groups.items():
+            # collapse duplicates (same values for same relation)
+            key = frozenset(vals + [rel])
+            if key in seen_clusters:
+                continue
+            seen_clusters.add(key)
+            # collect unique members (subjects that have these values)
+            unique_members = sorted(set(members))
+            if len(unique_members) < self.min_members:
+                continue
+            name = f"proto:value_cluster_{cluster_idx}"
+            cluster_idx += 1
+            abstractions.append(Abstraction(
+                name=name,
+                kind="value_cluster",
+                members=unique_members,
+                prototype_relations=[rel],
+                confidence=min(0.49, 0.10 + 0.05 * len(unique_members))))
+
+        # emit membership edges as derived facts
+        n_added = 0
+        if not dry_run and abstractions:
+            ts = iso(_now())
+            for ab in abstractions:
+                for member in ab.members:
+                    fid = new_id()
+                    self.store.conn.execute(
+                        "INSERT INTO facts "
+                        "(id, subject, relation, value, valid_from, "
+                        " tx_from, confidence, user_id, memory_type, "
+                        " is_derived, is_active, birth_commit, provenance) "
+                        "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
+                        (fid, ab.name, "member_of", member, ts, ts,
+                         ab.confidence, user_id or "default",
+                         "long_term", 1, 1,
+                         commit_id,
+                         _prov_json(ab)))
+                    n_added += 1
+            if commit_id:
+                self.store.update_commit_n_facts(commit_id, n_added)
+
+        return AbstractionResult(
+            abstractions=abstractions,
+            membership_edges_added=n_added,
+            duration_ms=(time.perf_counter() - t0) * 1000.0)
+
+
+def _now():
+    from datetime import datetime, timezone
+    return datetime.now(timezone.utc)
+
+
+def _prov_json(ab: Abstraction) -> str:
+    import json
+    return json.dumps({
+        "kind": "abstraction",
+        "abstraction_kind": ab.kind,
+        "abstraction_name": ab.name,
+        "n_members": len(ab.members),
+        "prototype_relations": ab.prototype_relations,
+        "generated_by": "cognition.abstraction",
+    })
+
+
+__all__ = ["AbstractionEngine", "Abstraction", "AbstractionResult"]