continuityos 0.7.0__py3-none-any.whl

continuityos/__init__.py

@@ -0,0 +1,14 @@
+"""ContinuityOS — durable, hybrid memory + continuity layer for AI agents and humans.
+
+Memory: structural (namespaces + keyword FTS) + semantic (vector cosine) recall.
+Continuity: canon, frontiers, open loops, checkpoints, anti-drift doctor, handoff.
+Local-first. No data leaves the machine.
+"""
+from .memory import Memory, MemoryItem
+from .continuity import Continuity
+from .agents import Council, Actor
+from .twin import Twin
+from .control import ControlPlane
+from . import fork
+__all__ = ["Memory", "MemoryItem", "Continuity", "Council", "Actor", "Twin", "ControlPlane", "fork"]
+__version__ = "0.7.0"

continuityos/agents.py

@@ -0,0 +1,76 @@
+"""Multi-agent council — many agents (and the human) on one ContinuityOS.
+
+Generalized from the Continuity OS AGENT_COUNCIL / INTERNAL_AGENTS canon:
+every actor has an authority level (1..5); namespaces have a minimum write
+level; the human operator is Sovereign (5). Internal roles are stable
+attention functions (Archivist / Builder / Critic / Steward), not personas.
+Every memory written through the council is tagged with its author + authority,
+so a swarm can share one memory without overwriting trust.
+"""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+from .memory import Memory
+
+# authority levels
+SOVEREIGN, STEWARD, BUILDER, CRITIC, READER = 5, 4, 3, 2, 1
+LEVEL_NAME = {5:"sovereign",4:"steward",3:"builder",2:"critic",1:"reader"}
+
+# internal roles = stable attention functions (from INTERNAL_AGENTS canon)
+ROLES = {
+    "archivist": "Read everything; extract the delta without drowning in raw archive.",
+    "builder":   "Make the change; turn intent into concrete artifacts.",
+    "critic":    "Challenge; surface risks, gaps, and false trails before they ship.",
+    "steward":   "Maintain integrity; enforce canon, anti-drift, and closure.",
+}
+
+# minimum authority level required to WRITE into a namespace
+NAMESPACE_MIN_WRITE = {
+    "canon": SOVEREIGN,      # only the human changes non-negotiable truths
+    "frontier": STEWARD,
+    "checkpoint": BUILDER,
+    "loop": BUILDER,
+    "rules": STEWARD,
+    "default": CRITIC,       # most namespaces: critic+ may write
+}
+
+@dataclass
+class Actor:
+    name: str
+    authority: int = BUILDER
+    role: str = "builder"
+
+class Council:
+    def __init__(self, memory: Optional[Memory] = None, db: str = "continuityos.db"):
+        self.m = memory or Memory(db)
+        self.actors: Dict[str, Actor] = {}
+
+    def register(self, name: str, authority: int = BUILDER, role: str = "builder") -> Actor:
+        a = Actor(name=name, authority=int(authority), role=role)
+        self.actors[name] = a
+        return a
+
+    def can_write(self, actor: str, namespace: str) -> bool:
+        a = self.actors.get(actor)
+        if not a:
+            return False
+        need = NAMESPACE_MIN_WRITE.get(namespace, NAMESPACE_MIN_WRITE["default"])
+        return a.authority >= need
+
+    def remember(self, actor: str, text: str, namespace: str = "notes",
+                 tags: Optional[List[str]] = None) -> int:
+        a = self.actors.get(actor)
+        if not a:
+            raise PermissionError(f"unknown actor '{actor}'")
+        if not self.can_write(actor, namespace):
+            need = NAMESPACE_MIN_WRITE.get(namespace, NAMESPACE_MIN_WRITE["default"])
+            raise PermissionError(
+                f"{actor} (L{a.authority}/{LEVEL_NAME.get(a.authority)}) cannot write "
+                f"[{namespace}] — needs L{need}/{LEVEL_NAME.get(need)}")
+        tags = (tags or []) + [f"by:{actor}", f"role:{a.role}", f"auth:{a.authority}"]
+        return self.m.remember(text, namespace=namespace, tags=tags,
+                               meta={"author": actor, "authority": a.authority, "role": a.role})
+
+    def roster(self) -> List[Dict]:
+        return [{"name":a.name,"authority":a.authority,"level":LEVEL_NAME.get(a.authority),"role":a.role}
+                for a in self.actors.values()]

continuityos/api.py

@@ -0,0 +1,36 @@
+"""Tiny stdlib HTTP API (no FastAPI dependency) so `pip install continuityos` stays light.
+  POST /remember {text,namespace?,tags?}   GET /recall?q=..&k=..   GET /namespaces
+"""
+from __future__ import annotations
+import json
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import urlparse, parse_qs
+from .memory import Memory
+
+def run(db: str, host: str = "127.0.0.1", port: int = 8077):
+    mem = Memory(db)
+    class H(BaseHTTPRequestHandler):
+        def _j(self, code, obj):
+            b = json.dumps(obj, ensure_ascii=False).encode()
+            self.send_response(code); self.send_header("Content-Type","application/json; charset=utf-8")
+            self.send_header("Content-Length",str(len(b))); self.end_headers(); self.wfile.write(b)
+        def log_message(self,*a): pass
+        def do_GET(self):
+            u = urlparse(self.path); qs = parse_qs(u.query)
+            if u.path == "/recall":
+                q = (qs.get("q") or [""])[0]; k = int((qs.get("k") or ["5"])[0])
+                ns = (qs.get("namespace") or [None])[0]
+                return self._j(200, {"hits":[h.to_dict() for h in mem.recall(q,k=k,namespace=ns)]})
+            if u.path == "/namespaces":
+                return self._j(200, {"namespaces": mem.namespaces(), "count": mem.count()})
+            if u.path in ("/","/health"):
+                return self._j(200, {"ok":True,"product":"ContinuityOS","count":mem.count()})
+            self._j(404, {"error":"not found"})
+        def do_POST(self):
+            n = int(self.headers.get("Content-Length",0)); body = json.loads(self.rfile.read(n) or b"{}")
+            if self.path == "/remember":
+                rid = mem.remember(body["text"], namespace=body.get("namespace","notes"), tags=body.get("tags"))
+                return self._j(200, {"id":rid})
+            self._j(404, {"error":"not found"})
+    print(f"ContinuityOS API on http://{host}:{port}")
+    ThreadingHTTPServer((host,port), H).serve_forever()

continuityos/cli.py

@@ -0,0 +1,102 @@
+"""ContinuityOS CLI.
+Memory:     cos remember | recall | namespaces
+Continuity: cos canon | frontier | loop | checkpoint | doctor | handoff
+Twin:       cos predict | alignment
+Serve:      cos serve (MCP stdio) | cos api (HTTP)
+"""
+from __future__ import annotations
+import argparse, os, json, sys
+from .memory import Memory
+from .continuity import Continuity
+from .twin import Twin
+
+def _db(a): return a.db or os.path.expanduser("~/.continuityos/memory.db")
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="cos", description="ContinuityOS — durable memory + continuity for agents & humans")
+    ap.add_argument("--db", default=None)
+    s = ap.add_subparsers(dest="cmd", required=True)
+    r = s.add_parser("remember"); r.add_argument("text"); r.add_argument("-n","--namespace",default="notes"); r.add_argument("-t","--tags",default="")
+    q = s.add_parser("recall"); q.add_argument("query"); q.add_argument("-k",type=int,default=5); q.add_argument("-n","--namespace",default=None)
+    s.add_parser("namespaces")
+    cn = s.add_parser("canon"); cn.add_argument("text", nargs="?"); 
+    fr = s.add_parser("frontier"); fr.add_argument("kind", nargs="?", choices=["trunk","cash","lab","parked"]); fr.add_argument("item", nargs="?")
+    lp = s.add_parser("loop"); lp.add_argument("text", nargs="?"); lp.add_argument("--close", type=int, default=None)
+    cp = s.add_parser("checkpoint"); cp.add_argument("--summary",required=True); cp.add_argument("--next",required=True,dest="nxt"); cp.add_argument("--proof",default="")
+    s.add_parser("doctor")
+    s.add_parser("handoff")
+    s.add_parser("boot")
+    bc = s.add_parser("close"); bc.add_argument("--summary",required=True); bc.add_argument("--next",required=True,dest="nxt"); bc.add_argument("--proof",default="")
+    s.add_parser("compress")
+    s.add_parser("serve")
+    pa = s.add_parser("api"); pa.add_argument("--host",default="127.0.0.1"); pa.add_argument("--port",type=int,default=8077)
+    pr = s.add_parser("predict", help="Digital-twin: likely stance on a situation, grounded in recorded rules and precedent")
+    pr.add_argument("situation")
+    al = s.add_parser("alignment", help="Check a proposed action against canon/rules; flags conflicts with non-negotiable rules")
+    al.add_argument("action")
+    a = ap.parse_args(argv)
+
+    if a.cmd == "serve":
+        from . import mcp_server; sys.argv = ["mcp","--db",_db(a)]; return mcp_server.main()
+    if a.cmd == "api":
+        from . import api; return api.run(_db(a), a.host, a.port)
+
+    db = _db(a); 
+    try:
+        from .embedders import FastEmbedEmbedder
+        m = Memory(db, embedder=FastEmbedEmbedder())
+    except Exception:
+        m = Memory(db)
+    c = Continuity(memory=m)
+    t = Twin(memory=m)
+    if a.cmd == "remember":
+        tags=[t.strip() for t in a.tags.split(",") if t.strip()]
+        print("stored #%d in [%s]" % (m.remember(a.text,namespace=a.namespace,tags=tags), a.namespace))
+    elif a.cmd == "recall":
+        for h in m.recall(a.query,k=a.k,namespace=a.namespace):
+            print("%.3f [%s] %s  (%s)" % (h.score,h.namespace,h.text,h.why))
+    elif a.cmd == "namespaces":
+        print(json.dumps(m.namespaces(),ensure_ascii=False,indent=2))
+    elif a.cmd == "canon":
+        if a.text: print("canon #%d" % c.add_canon(a.text))
+        else:
+            for r in c._dump("canon"): print("- "+r["text"])
+    elif a.cmd == "frontier":
+        if a.kind and a.item: print("set %s -> %s (#%d)" % (a.kind,a.item,c.set_frontier(a.kind,a.item)))
+        else: print(json.dumps(c.frontiers(),ensure_ascii=False,indent=2))
+    elif a.cmd == "loop":
+        if a.close is not None: c.close_loop(a.close); print("closed loop #%d" % a.close)
+        elif a.text: print("loop #%d opened" % c.add_loop(a.text))
+        else:
+            for l in c.open_loops(): print("[#%d] %s" % (l["id"],l["text"]))
+    elif a.cmd == "checkpoint":
+        print("checkpoint #%d" % c.checkpoint(summary=a.summary,next_action=a.nxt,proof=a.proof))
+    elif a.cmd == "doctor":
+        d=c.doctor(); print("%s  %d/%d" % ("✅ healthy" if d["healthy"] else "⚠ drift", d["passed"], d["total"]))
+        for ch in d["checks"]: print("  %s %s — %s" % ("✓" if ch["ok"] else "✗", ch["check"], ch["detail"]))
+    elif a.cmd == "handoff":
+        print(c.handoff())
+    elif a.cmd == "boot":
+        # start of session: show handoff + doctor (the boot ritual)
+        print(c.handoff()); print("\n--- doctor ---")
+        d=c.doctor(); print("%s %d/%d" % ("OK" if d["healthy"] else "DRIFT", d["passed"], d["total"]))
+        for ch in d["checks"]:
+            if not ch["ok"]: print("  ! %s — %s" % (ch["check"], ch["detail"]))
+    elif a.cmd == "close":
+        # end of session: checkpoint + doctor (closure beats branching)
+        cid=c.checkpoint(summary=a.summary, next_action=a.nxt, proof=a.proof)
+        print("checkpoint #%d" % cid); d=c.doctor()
+        print("doctor: %s %d/%d" % ("OK" if d["healthy"] else "DRIFT", d["passed"], d["total"]))
+    elif a.cmd == "compress":
+        # weekly compression: report counts per namespace to spot bloat
+        print("namespace sizes (compress candidates):")
+        for ns in m.namespaces(): print("  %-12s %d" % (ns["namespace"], ns["count"]))
+        ol=c.open_loops()
+        print("open loops: %d (close stale ones with: cos loop --close <id>)" % len(ol))
+    elif a.cmd == "predict":
+        print(json.dumps(t.predict(a.situation), ensure_ascii=False, indent=2))
+    elif a.cmd == "alignment":
+        print(json.dumps(t.alignment(a.action), ensure_ascii=False, indent=2))
+
+if __name__ == "__main__":
+    main()

continuityos/continuity.py

@@ -0,0 +1,117 @@
+"""Continuity layer — ContinuityOS is more than a memory store.
+
+A continuity OS keeps the *thread* between sessions (and between versions of you
+and versions of the model): slow truths (canon), live state (frontiers + open
+loops), session checkpoints, anti-drift checks, and handoff packs. All of it is
+just structured memory, so it shares the same local store and hybrid recall.
+
+Reserved namespaces: `canon` (non-negotiable truths/rules), `frontier`
+(trunk/cash/lab focus), `loop` (open loops), `checkpoint` (session deltas).
+"""
+from __future__ import annotations
+import time, json
+from typing import List, Dict, Any, Optional
+from .memory import Memory
+
+FRONTIER_KINDS = ("trunk", "cash", "lab", "parked")
+
+class Continuity:
+    def __init__(self, memory: Optional[Memory] = None, db: str = "continuityos.db"):
+        self.m = memory or Memory(db)
+
+    # ---- canon: slow truths ----
+    def add_canon(self, text: str, tags: Optional[List[str]] = None) -> int:
+        return self.m.remember(text, namespace="canon", tags=tags or [])
+
+    def canon(self) -> List[Dict[str, Any]]:
+        return [r.to_dict() for r in self.m.recall("", k=100, namespace="canon")] or \
+               self._dump("canon")
+
+    # ---- frontiers: 1 trunk + 1 cash + 1 lab discipline ----
+    def set_frontier(self, kind: str, item: str) -> int:
+        kind = kind.lower()
+        if kind not in FRONTIER_KINDS:
+            raise ValueError(f"kind must be one of {FRONTIER_KINDS}")
+        # supersede previous frontier of same kind (keep history via meta)
+        return self.m.remember(item, namespace="frontier", tags=[kind],
+                               meta={"kind": kind, "ts": time.time()})
+
+    def frontiers(self) -> Dict[str, Optional[str]]:
+        rows = self._dump("frontier")
+        latest: Dict[str, Dict[str, Any]] = {}
+        for r in rows:
+            k = (r["meta"] or {}).get("kind") or (r["tags"][0] if r["tags"] else "parked")
+            ts = (r["meta"] or {}).get("ts", 0)
+            if k not in latest or ts >= latest[k]["meta"].get("ts", 0):
+                latest[k] = r
+        return {k: latest[k]["text"] for k in latest}
+
+    # ---- open loops ----
+    def add_loop(self, text: str, tags: Optional[List[str]] = None) -> int:
+        return self.m.remember(text, namespace="loop", tags=(tags or []) + ["open"],
+                               meta={"open": True, "ts": time.time()})
+
+    def close_loop(self, loop_id: int) -> bool:
+        return self.m.forget(loop_id)
+
+    def open_loops(self) -> List[Dict[str, Any]]:
+        return [r for r in self._dump("loop")]
+
+    # ---- checkpoints: every session ends with delta + next + proof ----
+    def checkpoint(self, summary: str, next_action: str, proof: str = "") -> int:
+        text = f"DELTA: {summary} | NEXT: {next_action}" + (f" | PROOF: {proof}" if proof else "")
+        return self.m.remember(text, namespace="checkpoint",
+                               tags=["checkpoint"], meta={"ts": time.time(),
+                               "summary": summary, "next": next_action, "proof": proof})
+
+    def last_checkpoint(self) -> Optional[Dict[str, Any]]:
+        rows = sorted(self._dump("checkpoint"), key=lambda r: (r["meta"] or {}).get("ts", 0), reverse=True)
+        return rows[0] if rows else None
+
+    # ---- anti-drift doctor ----
+    def doctor(self, max_open_loops: int = 7, checkpoint_stale_hours: float = 48) -> Dict[str, Any]:
+        fr = self.frontiers()
+        loops = self.open_loops()
+        last = self.last_checkpoint()
+        now = time.time()
+        checks = []
+        def chk(ok, name, detail): checks.append({"ok": bool(ok), "check": name, "detail": detail})
+        chk("cash" in fr, "cash_frontier_set", fr.get("cash", "— not set"))
+        chk("trunk" in fr, "trunk_set", fr.get("trunk", "— not set"))
+        chk(len(loops) <= max_open_loops, "open_loops_bounded", f"{len(loops)} open (max {max_open_loops})")
+        if last:
+            age_h = (now - (last["meta"] or {}).get("ts", now)) / 3600
+            chk(age_h <= checkpoint_stale_hours, "checkpoint_fresh", f"{age_h:.1f}h old")
+            chk(bool((last["meta"] or {}).get("proof")), "has_proof", (last["meta"] or {}).get("proof") or "— no proof")
+        else:
+            chk(False, "checkpoint_fresh", "no checkpoint yet")
+            chk(False, "has_proof", "no checkpoint yet")
+        # L6 autopoiesis — self-maintenance invariants (system "alive")
+        chk(self.m.count() > 0, "memory_persists", f"{self.m.count()} memories")
+        chk(len(self._dump("canon")) > 0, "identity_persists", "canon present" )
+        chk(len(loops) > 0, "purpose_persists", f"{len(loops)} open loop(s)")
+        passed = sum(1 for c in checks if c["ok"])
+        return {"healthy": passed == len(checks), "passed": passed, "total": len(checks), "checks": checks}
+
+    # ---- handoff pack: context for the next session / agent ----
+    def handoff(self) -> str:
+        fr = self.frontiers()
+        loops = self.open_loops()
+        last = self.last_checkpoint()
+        canon = self._dump("canon")
+        out = ["# ContinuityOS handoff pack"]
+        out.append("\n## Canon (non-negotiable)")
+        out += [f"- {c['text']}" for c in canon[:12]] or ["- (none)"]
+        out.append("\n## Frontiers")
+        out += [f"- {k}: {v}" for k, v in fr.items()] or ["- (none)"]
+        out.append("\n## Open loops")
+        out += [f"- [#{l['id']}] {l['text']}" for l in loops[:20]] or ["- (none)"]
+        out.append("\n## Last checkpoint")
+        out.append(f"- {last['text']}" if last else "- (none)")
+        return "\n".join(out)
+
+    def _dump(self, namespace: str) -> List[Dict[str, Any]]:
+        rows = self.m.store.all_with_vecs(namespace=namespace)
+        import json as _j
+        return [{"id": r["id"], "text": r["text"], "namespace": r["namespace"],
+                 "tags": _j.loads(r["tags"]), "meta": _j.loads(r["meta"])} for r in rows]

continuityos/control.py

@@ -0,0 +1,54 @@
+"""L5 Control Plane — operator actions over memory (from Twin admin-control-plane spec).
+
+corrections, redaction (privacy), rollback to a checkpoint, and a transparency
+export (what is stored about whom). Every control action is itself recorded.
+"""
+from __future__ import annotations
+import time, json
+from typing import List, Dict, Any, Optional
+from .memory import Memory
+
+class ControlPlane:
+    def __init__(self, memory: Optional[Memory] = None, db: str = "continuityos.db"):
+        self.m = memory or Memory(db)
+
+    def correct(self, item_id: int, new_text: str, namespace: str = "notes") -> int:
+        """Supersede a memory: forget the old, store the corrected one, log it."""
+        old = self.m.store.get(item_id)
+        self.m.forget(item_id)
+        rid = self.m.remember(new_text, namespace=namespace, tags=["corrected"],
+                              meta={"corrects": item_id})
+        self.m.remember(f"correction of #{item_id} -> #{rid}", namespace="control",
+                        tags=["control","correct"], meta={"ts": time.time()})
+        return rid
+
+    def redact(self, query: str, namespace: Optional[str] = None) -> int:
+        """Privacy: delete memories matching a query. Returns count removed."""
+        hits = self.m.recall(query, k=100, namespace=namespace)
+        n = 0
+        for h in hits:
+            if h.score > 0.2:
+                self.m.forget(h.id); n += 1
+        self.m.remember(f"redacted {n} memories matching '{query}'", namespace="control",
+                        tags=["control","redact"], meta={"ts": time.time(), "count": n})
+        return n
+
+    def rollback(self, checkpoint_id: int) -> Dict[str, Any]:
+        """Revert state created AFTER a checkpoint: forget loop/frontier/checkpoint items newer than it."""
+        cp = self.m.store.get(checkpoint_id)
+        if not cp:
+            return {"ok": False, "error": "checkpoint not found"}
+        cutoff = cp["created_at"]
+        removed = 0
+        for ns in ("loop", "frontier", "checkpoint", "notes"):
+            for r in self.m.store.all_with_vecs(namespace=ns):
+                if r["created_at"] > cutoff:
+                    self.m.forget(r["id"]); removed += 1
+        self.m.remember(f"rollback to checkpoint #{checkpoint_id}, removed {removed} newer items",
+                        namespace="control", tags=["control","rollback"], meta={"ts": time.time()})
+        return {"ok": True, "checkpoint": checkpoint_id, "removed": removed}
+
+    def export(self) -> Dict[str, Any]:
+        """Transparency: what is stored, by namespace (consent / data-subject view)."""
+        return {"namespaces": self.m.namespaces(), "total": self.m.count(),
+                "note": "All data is local. Use redact() to remove, correct() to fix."}

continuityos/dedupe.py

@@ -0,0 +1,131 @@
+"""ContinuityOS memory de-duplication — MinHash + SimHash LSH (pure-python, no deps).
+
+Applies guide_lsh_deduplication: collapses near-duplicate memories into CompactDigest
+records so the long-term store doesn't bloat with restated facts. Parameters match the
+guide: MinHash Jaccard >= 0.55 (n_perm=128), SimHash Hamming <= 3 (strict) / <= 4 (soft).
+
+Usage:
+    from continuityos.dedupe import find_near_duplicates, compact_digest
+    groups = find_near_duplicates([(m.id, m.text) for m in all_memories])
+    for g in groups: digest = compact_digest(g)
+Read-only by design: returns duplicate groups + digests; the caller decides what to drop.
+"""
+from __future__ import annotations
+import re, hashlib
+from typing import List, Tuple, Dict
+
+_WORD = re.compile(r"\w+", re.UNICODE)
+N_PERM = 128
+JACCARD_THRESH = 0.55
+HAMMING_STRICT = 3
+HAMMING_SOFT = 4
+_MERSENNE = (1 << 61) - 1
+
+
+def _shingles(text: str, k: int = 3) -> set:
+    toks = [t.lower() for t in _WORD.findall(text)]
+    if len(toks) < k:
+        return set(toks)
+    return {" ".join(toks[i:i + k]) for i in range(len(toks) - k + 1)}
+
+
+def _h(s: str, seed: int) -> int:
+    return int(hashlib.blake2b(s.encode("utf-8"), digest_size=8,
+                               salt=seed.to_bytes(8, "little")).hexdigest(), 16)
+
+
+def minhash(text: str, n_perm: int = N_PERM) -> Tuple[int, ...]:
+    """MinHash signature (n_perm permutations)."""
+    sh = _shingles(text)
+    if not sh:
+        return tuple([0] * n_perm)
+    sig = []
+    for p in range(n_perm):
+        sig.append(min(_h(s, p) % _MERSENNE for s in sh))
+    return tuple(sig)
+
+
+def jaccard(a: Tuple[int, ...], b: Tuple[int, ...]) -> float:
+    if not a or not b:
+        return 0.0
+    return sum(1 for x, y in zip(a, b) if x == y) / len(a)
+
+
+def simhash(text: str, bits: int = 64) -> int:
+    """SimHash fingerprint."""
+    v = [0] * bits
+    for s in _shingles(text, k=2):
+        hv = _h(s, 1)
+        for i in range(bits):
+            v[i] += 1 if (hv >> i) & 1 else -1
+    out = 0
+    for i in range(bits):
+        if v[i] > 0:
+            out |= (1 << i)
+    return out
+
+
+def hamming(a: int, b: int) -> int:
+    return bin(a ^ b).count("1")
+
+
+def find_near_duplicates(items: List[Tuple[str, str]],
+                         jaccard_thresh: float = JACCARD_THRESH,
+                         hamming_thresh: int = HAMMING_STRICT) -> List[List[str]]:
+    """items = [(id, text), ...] -> list of duplicate groups (each a list of ids).
+    Two-stage gate (guide): SimHash Hamming candidate -> MinHash Jaccard confirm."""
+    sims = {i: simhash(t) for i, t in items}
+    mins = {i: minhash(t) for i, t in items}
+    ids = [i for i, _ in items]
+    parent: Dict[str, str] = {i: i for i in ids}
+
+    def find(x):
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]; x = parent[x]
+        return x
+
+    def union(a, b):
+        ra, rb = find(a), find(b)
+        if ra != rb:
+            parent[rb] = ra
+
+    for ai in range(len(ids)):
+        for bi in range(ai + 1, len(ids)):
+            a, b = ids[ai], ids[bi]
+            if hamming(sims[a], sims[b]) <= hamming_thresh and \
+               jaccard(mins[a], mins[b]) >= jaccard_thresh:
+                union(a, b)
+    groups: Dict[str, List[str]] = {}
+    for i in ids:
+        groups.setdefault(find(i), []).append(i)
+    return [g for g in groups.values() if len(g) > 1]
+
+
+def compact_digest(group_items: List[Tuple[str, str]]) -> dict:
+    """Collapse a duplicate group into one CompactDigest (guide format):
+    core summary = the longest/most-informative text, back-trace ids preserved."""
+    canonical = max(group_items, key=lambda it: len(it[1]))
+    return {
+        "core_summary": canonical[1],
+        "canonical_id": canonical[0],
+        "back_trace_ids": [i for i, _ in group_items],
+        "merged_count": len(group_items),
+    }
+
+
+# NOTE: LSH (MinHash/SimHash) is the NEAR-EXACT layer — it collapses the same fact
+# restated with minor edits (punctuation, whitespace, a swapped word). Semantic
+# paraphrases ("X causes Y" vs "Y is caused by X") are NOT near-duplicates here by
+# design; those are handled by ContinuityOS vector recall. Two cheap, complementary layers.
+if __name__ == "__main__":
+    demo = [
+        ("m1", "The arena uses GCP spot preemption which causes periodic reboots and bot flapping."),
+        ("m2", "The arena uses GCP spot preemption which causes periodic reboots and bot flapping"),
+        ("m3", "The arena uses GCP spot-preemption, which causes periodic reboots and bot flapping."),
+        ("m4", "Grid trading is market-neutral and works best in flat regimes."),
+        ("m5", "Completely unrelated note about coffee and morning routines and tea."),
+    ]
+    groups = find_near_duplicates(demo)
+    print("duplicate groups:", groups)  # expect [['m1','m2','m3']]
+    for g in groups:
+        print("digest:", compact_digest([(i, t) for i, t in demo if i in g]))

continuityos/embed.py

@@ -0,0 +1,43 @@
+"""Pluggable embeddings.
+
+Default `HashingEmbedder` is dependency-free and deterministic: char n-gram
+hashing into a fixed-dim L2-normalized vector. Good enough for local semantic
+recall and fully offline. For production-grade semantics, pass any callable
+that maps str->list[float] (e.g. a sentence-transformers model) as `embedder`.
+"""
+from __future__ import annotations
+import math, re, hashlib
+from typing import List
+
+_TOKEN = re.compile(r"[\w]+", re.UNICODE)
+
+def _stable_hash(s: str) -> int:
+    # deterministic across processes/runs (unlike builtin hash() for str)
+    return int.from_bytes(hashlib.blake2b(s.encode("utf-8"), digest_size=8).digest(), "big")
+
+def _ngrams(text: str, n: int = 3):
+    toks = _TOKEN.findall((text or "").lower())
+    for t in toks:                       # word-level tokens
+        yield t
+    joined = " ".join(toks)
+    for i in range(len(joined) - n + 1):  # char n-grams (morphology / typos / multilingual)
+        yield joined[i:i+n]
+
+class HashingEmbedder:
+    """Deterministic offline embedder. dim defaults to 256."""
+    def __init__(self, dim: int = 256):
+        self.dim = dim
+
+    def __call__(self, text: str) -> List[float]:
+        vec = [0.0] * self.dim
+        for g in _ngrams(text):
+            h = _stable_hash(g)
+            idx = h % self.dim
+            sign = 1.0 if (h >> 16) & 1 else -1.0
+            vec[idx] += sign
+        norm = math.sqrt(sum(v * v for v in vec)) or 1.0
+        return [v / norm for v in vec]
+
+def cosine(a: List[float], b: List[float]) -> float:
+    # vectors are L2-normalized -> dot product == cosine
+    return sum(x * y for x, y in zip(a, b))

continuityos/embedders.py

@@ -0,0 +1,47 @@
+"""Optional production-grade embedders (pluggable into Memory(embedder=...)).
+
+Default stays the dependency-free offline HashingEmbedder. These give real
+semantic recall:
+
+  pip install "continuityos[fast]"   # fastembed: ONNX, no torch, small + fast
+  pip install "continuityos[st]"     # sentence-transformers: widest model choice
+
+  from continuityos import Memory
+  from continuityos.embedders import FastEmbedEmbedder
+  m = Memory("memory.db", embedder=FastEmbedEmbedder())
+
+All embedders return L2-normalized vectors so cosine == dot product.
+"""
+from __future__ import annotations
+import math
+from typing import List
+
+def _l2(v: List[float]) -> List[float]:
+    n = math.sqrt(sum(x * x for x in v)) or 1.0
+    return [x / n for x in v]
+
+class FastEmbedEmbedder:
+    """ONNX embedder via `fastembed` (no torch). Default model bge-small-en-v1.5 (384-dim)."""
+    def __init__(self, model: str = "BAAI/bge-small-en-v1.5"):
+        try:
+            from fastembed import TextEmbedding
+        except Exception as e:
+            raise ImportError('FastEmbedEmbedder needs:  pip install "continuityos[fast]"') from e
+        self._model = TextEmbedding(model_name=model)
+
+    def __call__(self, text: str) -> List[float]:
+        vec = list(next(iter(self._model.embed([text or ""]))))
+        return _l2([float(x) for x in vec])
+
+class SentenceTransformerEmbedder:
+    """sentence-transformers embedder. Default all-MiniLM-L6-v2 (384-dim)."""
+    def __init__(self, model: str = "all-MiniLM-L6-v2"):
+        try:
+            from sentence_transformers import SentenceTransformer
+        except Exception as e:
+            raise ImportError('SentenceTransformerEmbedder needs:  pip install "continuityos[st]"') from e
+        self._model = SentenceTransformer(model)
+
+    def __call__(self, text: str) -> List[float]:
+        v = self._model.encode(text or "", normalize_embeddings=True)
+        return [float(x) for x in v]