codemem-mcp 0.1.0__py3-none-any.whl

codemem/__init__.py

@@ -0,0 +1,2 @@
+"""codemem: system-coding-memory. One memory for every project on every machine."""
+__version__ = "0.1.0"

codemem/cli.py

@@ -0,0 +1,126 @@
+"""codemem command line: serve, backfill, gitea, docs, seed, scan, embed, reindex, backup, sync."""
+import argparse, gzip, shutil, sys, time
+from pathlib import Path
+from . import config
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="codemem", description=__doc__)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("serve", help="run the MCP server + web UI")
+    b = sub.add_parser("backfill", help="ingest all commits from every bare repo on the git server")
+    b.add_argument("repos", nargs="*")
+    sub.add_parser("gitea", help="pull descriptions/topics/urls from Gitea")
+    sub.add_parser("docs", help="(re)ingest markdown knowledge sources")
+    sub.add_parser("seed", help="seed howto notes and known assets")
+    s = sub.add_parser("scan", help="scan local directories for projects")
+    s.add_argument("roots", nargs="*", help="directories (default: registered scan roots, else config defaults)")
+    e = sub.add_parser("embed", help="embed index rows that lack vectors")
+    e.add_argument("--all", action="store_true", help="keep going until nothing is pending")
+    sub.add_parser("reindex", help="rebuild the FTS index from source tables (drops embeddings)")
+    bk = sub.add_parser("backup", help="consistent gzip copy of the database")
+    bk.add_argument("--dest", default=str(config.BACKUP_DIR))
+    bk.add_argument("--keep", type=int, default=30)
+    sub.add_parser("sync", help="docs + gitea + scan + embed: the timer job")
+    d = sub.add_parser("describe", help="draft descriptions for own projects that lack one (Ollama, tagged auto-described)")
+    d.add_argument("--dry-run", action="store_true")
+    dc = sub.add_parser("discover", help="mine own repos for reusable assets (tagged auto-discovered) and shared code links")
+    dc.add_argument("repos", nargs="*"); dc.add_argument("--no-describe", action="store_true", help="skip model drafts for files without a docstring")
+    rv = sub.add_parser("review", help="targeted model review: near-duplicate confirmation, thin descriptions on trusted assets")
+    rv.add_argument("--stage", type=int, choices=[1, 2], default=0, help="run one stage only (default both)")
+    rv.add_argument("--limit", type=int, default=40); rv.add_argument("--dry-run", action="store_true")
+    rv.add_argument("--machine", default="", help="stage 2: review every Python asset from this machine (source shipped by its agent)")
+    dn = sub.add_parser("delete-note", help="delete one note by id, with its index and embedding rows")
+    dn.add_argument("ids", nargs="+", type=int)
+    pg = sub.add_parser("purge", help="remove a project and everything attached to it (locations, assets, notes, commits, links)")
+    pg.add_argument("names", nargs="+")
+    sub.add_parser("trust", help="recompute trust scores for all projects and assets")
+    sub.add_parser("stats")
+    a = ap.parse_args(argv)
+
+    if a.cmd == "serve":
+        from .server import run; run()
+    elif a.cmd == "backfill":
+        from .gitsync import backfill
+        t = time.time(); n = backfill(a.repos or None); print(f"{n} new commits in {time.time()-t:.1f}s")
+    elif a.cmd == "gitea":
+        from .gitsync import gitea_sync; gitea_sync()
+    elif a.cmd == "docs":
+        from .knowledge import ingest_docs; ingest_docs()
+    elif a.cmd == "seed":
+        from .knowledge import seed; seed()
+    elif a.cmd == "scan":
+        from .scan import scan_local; print(scan_local(a.roots or None))
+    elif a.cmd == "embed":
+        from .search import embed_pending
+        total = 0
+        while True:
+            n = embed_pending(); total += n; print(f"  embedded {n}")
+            if not n or not a.all:
+                break
+        print(f"{total} embedded")
+    elif a.cmd == "reindex":
+        from .store import reindex_all; reindex_all(); print("reindexed")
+    elif a.cmd == "backup":
+        backup(Path(a.dest), a.keep)
+    elif a.cmd == "sync":
+        from .knowledge import ingest_docs, seed
+        from .gitsync import gitea_sync, backfill
+        from .scan import scan_local
+        from .search import embed_pending
+        print("docs"); ingest_docs()
+        print("seed"); seed()
+        print("gitea"); gitea_sync()
+        print("backfill (catches anything the hook missed)"); backfill(log=lambda *_: None)
+        print("scan"); print(" ", scan_local())
+        from .discover import discover
+        print("discover"); print(" ", discover(describe=True, log=lambda *_: None))
+        from .trust import compute_all
+        print("trust"); compute_all()
+        print("embed"); 
+        while embed_pending(): pass
+        print("done")
+    elif a.cmd == "discover":
+        from .discover import discover; print(discover(a.repos or None, describe=not a.no_describe))
+    elif a.cmd == "review":
+        from .review import stage1, stage2
+        if a.stage in (0, 1): stage1(a.limit, a.dry_run)
+        if a.stage in (0, 2): stage2(a.limit, a.dry_run, machine=a.machine)
+        from .trust import compute_all; compute_all()
+    elif a.cmd == "delete-note":
+        from .store import delete_note
+        for i in a.ids:
+            n = delete_note(i)
+            print(f"deleted {i}: [{n['kind']}] {n['title'][:70]}" if n else f"no note {i}")
+    elif a.cmd == "purge":
+        from .store import purge_project
+        for n in a.names: print(purge_project(n))
+    elif a.cmd == "trust":
+        from .trust import compute_all; compute_all()
+    elif a.cmd == "describe":
+        from .describe import describe_all; print(f"{describe_all(dry=a.dry_run)} described")
+    elif a.cmd == "stats":
+        import json
+        from .server import stats; print(json.dumps(stats(), indent=1))
+
+
+def backup(dest: Path, keep: int):
+    import sqlite3
+    from .db import connect
+    dest.mkdir(parents=True, exist_ok=True)
+    stamp = time.strftime("%Y%m%d")
+    tmp = dest / f"codemem-{stamp}.db"
+    dst = sqlite3.connect(str(tmp))
+    connect().backup(dst)
+    dst.close()
+    with open(tmp, "rb") as f, gzip.open(f"{tmp}.gz", "wb") as g:
+        shutil.copyfileobj(f, g)
+    tmp.unlink()
+    old = sorted(dest.glob("codemem-*.db.gz"))[:-keep]
+    for o in old:
+        o.unlink()
+    print(f"backup -> {tmp}.gz ({len(old)} pruned)")
+
+
+if __name__ == "__main__":
+    main()

codemem/config.py

@@ -0,0 +1,62 @@
+"""All tunables in one place. Every value can be overridden by an environment variable."""
+import os, socket
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent
+DATA_DIR = Path(os.environ.get("CODEMEM_DATA", Path.home() / ".codemem"))
+DB_PATH = Path(os.environ.get("CODEMEM_DB", DATA_DIR / "codemem.db"))
+BACKUP_DIR = Path(os.environ.get("CODEMEM_BACKUP_DIR", DATA_DIR / "backups"))
+HOST = os.environ.get("CODEMEM_HOST", "0.0.0.0")
+PORT = int(os.environ.get("CODEMEM_PORT", "8055"))
+MACHINE = os.environ.get("CODEMEM_MACHINE", socket.gethostname())
+
+# The git host: a directory of bare repos served over SSH. codemem reads them directly, so it
+# normally runs on the same machine. GIT_SSH_HOST is what clients put in their remote URLs
+# (user@host) and is used to recognise our own remotes; empty means "same host, current user".
+GIT_ROOT = Path(os.environ.get("GIT_ROOT", "/srv/git"))
+GIT_SSH_HOST = os.environ.get("CODEMEM_GIT_SSH_HOST", socket.gethostname())
+PUSH_LOG = GIT_ROOT / "logs" / "push.log"
+GITEA_URL = os.environ.get("GITEA_URL", "http://127.0.0.1:3000").rstrip("/")
+GITEA_PUBLIC_URL = os.environ.get("GITEA_PUBLIC_URL", GITEA_URL).rstrip("/")
+GITEA_TOKEN_FILE = Path(os.environ.get("GITEA_TOKEN_FILE", GIT_ROOT / ".gitea-token"))
+
+# GitHub/GitLab owners that are YOU (comma list). A clone whose remote owner is not listed here is
+# vendor code. Empty (the default) disables owner-based vendor detection; mark vendor clones by hand.
+OWN_REMOTE_OWNERS = {o.strip().lower() for o in os.environ.get("CODEMEM_OWN_OWNERS", "").split(",") if o.strip()}
+# Names/paths that must never enter codemem at all. A regex matched case-insensitively against project
+# names, remotes and scan paths, e.g. r"(^|[/\\_\-\s])(secret-project|other)([/\\_\-\s.]|$)" for whole
+# path segments or name tokens. Empty (the default) excludes nothing. Purge existing rows with
+# `codemem purge <project>`.
+EXCLUDE_PATTERN = os.environ.get("CODEMEM_EXCLUDE", "")
+
+# Default audience for new projects. "unrestricted" = personal work, no content filtering applied.
+DEFAULT_AUDIENCE = os.environ.get("CODEMEM_DEFAULT_AUDIENCE", "unrestricted")
+
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434").rstrip("/")
+EMBED_MODEL = os.environ.get("CODEMEM_EMBED_MODEL", "nomic-embed-text")
+EMBED_ENABLED = os.environ.get("CODEMEM_EMBED", "1") not in ("0", "false", "no")
+# Local model used to draft descriptions and run the targeted code review.
+DESCRIBE_MODEL = os.environ.get("CODEMEM_DESCRIBE_MODEL", "qwen3-coder:30b")
+
+# Optional markdown table of code-review grades: rows like `| ... | `project` | ... | ... | B |`.
+# Feeds the trust score's review signal. Empty = no review signal.
+REVIEW_TABLE = os.environ.get("CODEMEM_REVIEW_TABLE", "")
+
+# Markdown that gets ingested as knowledge. Colon-separated globs, re-hashed on every run.
+# Add the docs of your other infrastructure repos here so howto() can answer from them.
+DOC_SOURCES = [s for s in os.environ.get("CODEMEM_DOC_SOURCES", ":".join([
+    str(HERE / "README.md"),
+    str(HERE / "docs" / "*.md"),
+])).split(":") if s]
+
+# Default scan roots for THIS machine. Other machines add theirs via add_scan_root.
+DEFAULT_SCAN_ROOTS = [s for s in os.environ.get("CODEMEM_SCAN_ROOTS", ":".join([
+    str(Path.home() / "projects"),
+])).split(":") if s]
+
+SKIP_DIRS = {".git", "node_modules", "venv", ".venv", "env", "__pycache__", ".cache", "dist",
+             "build", ".next", "site-packages", ".tox", ".mypy_cache",
+             "target", ".idea", ".vscode", "checkpoints", "models", "outputs", "output", "data",
+             "backup", "backups", "archive", "archives", "old", "_old", "mirror", "mirrors",
+             "PackageCache", "Library", "Temp", "Logs", "obj", "bin", ".godot", ".import"}   # Unity/Godot/.NET caches
+SCAN_MAX_DEPTH = 3

codemem/db.py

@@ -0,0 +1,199 @@
+"""SQLite schema, connection, and the single FTS5 index that spans every record type."""
+import json, sqlite3, threading, time
+from contextlib import contextmanager
+from . import config
+
+SCHEMA = """
+PRAGMA journal_mode=WAL;
+CREATE TABLE IF NOT EXISTS project (
+  id INTEGER PRIMARY KEY, name TEXT UNIQUE NOT NULL,
+  description TEXT DEFAULT '', purpose TEXT DEFAULT '', status TEXT DEFAULT 'active',
+  audience TEXT DEFAULT 'unrestricted', origin TEXT DEFAULT 'own', visibility TEXT DEFAULT 'private', maturity TEXT DEFAULT '', maturity_note TEXT DEFAULT '',
+  trust INTEGER, trust_breakdown TEXT DEFAULT '', verified_at TEXT, verified_note TEXT DEFAULT '',
+  tags TEXT DEFAULT '', languages TEXT DEFAULT '',
+  remote_url TEXT DEFAULT '', gitea_url TEXT DEFAULT '', github_url TEXT DEFAULT '',
+  first_commit TEXT, last_commit TEXT, commit_count INTEGER DEFAULT 0,
+  created_at TEXT, updated_at TEXT);
+CREATE TABLE IF NOT EXISTS location (
+  id INTEGER PRIMARY KEY, project_id INTEGER REFERENCES project(id) ON DELETE CASCADE,
+  machine TEXT NOT NULL, path TEXT NOT NULL, is_git INTEGER DEFAULT 0,
+  remote_url TEXT DEFAULT '', branch TEXT DEFAULT '', dirty INTEGER DEFAULT 0,
+  last_local_commit TEXT, file_count INTEGER, languages TEXT DEFAULT '',
+  key_files TEXT DEFAULT '', readme_head TEXT DEFAULT '', last_scanned TEXT,
+  UNIQUE(machine, path));
+CREATE TABLE IF NOT EXISTS asset (
+  id INTEGER PRIMARY KEY, project_id INTEGER REFERENCES project(id) ON DELETE SET NULL,
+  name TEXT NOT NULL, kind TEXT NOT NULL, path TEXT DEFAULT '', machine TEXT DEFAULT '',
+  description TEXT DEFAULT '', usage TEXT DEFAULT '', tags TEXT DEFAULT '',
+  maturity TEXT DEFAULT '', maturity_note TEXT DEFAULT '',
+  last_changed TEXT, change_count INTEGER, blob_hash TEXT DEFAULT '', size INTEGER, symbols TEXT DEFAULT '',
+  imports TEXT DEFAULT '', func_hashes TEXT DEFAULT '', signatures TEXT DEFAULT '', imported_by INTEGER DEFAULT 0,
+  review TEXT DEFAULT '', reviewed_at TEXT, source_head TEXT DEFAULT '',
+  trust INTEGER, trust_breakdown TEXT DEFAULT '', verified_at TEXT, verified_note TEXT DEFAULT '',
+  created_at TEXT, updated_at TEXT, UNIQUE(name, kind));
+CREATE TABLE IF NOT EXISTS note (
+  id INTEGER PRIMARY KEY, project_id INTEGER REFERENCES project(id) ON DELETE SET NULL,
+  kind TEXT NOT NULL, title TEXT NOT NULL, body TEXT DEFAULT '', tags TEXT DEFAULT '',
+  machine TEXT DEFAULT '', session_id TEXT DEFAULT '', path TEXT DEFAULT '', created_at TEXT);
+CREATE TABLE IF NOT EXISTS "commit" (
+  id INTEGER PRIMARY KEY, project_id INTEGER REFERENCES project(id) ON DELETE CASCADE,
+  hash TEXT NOT NULL, author TEXT, date TEXT, message TEXT, files TEXT DEFAULT '',
+  ref TEXT DEFAULT '', pushed_at TEXT, pushed_by TEXT, pushed_from TEXT,
+  UNIQUE(project_id, hash));
+CREATE TABLE IF NOT EXISTS link (
+  id INTEGER PRIMARY KEY, from_kind TEXT, from_id INTEGER, to_kind TEXT, to_id INTEGER,
+  relation TEXT NOT NULL, note TEXT DEFAULT '', created_at TEXT,
+  UNIQUE(from_kind, from_id, to_kind, to_id, relation));
+CREATE TABLE IF NOT EXISTS doc (
+  id INTEGER PRIMARY KEY, source TEXT NOT NULL UNIQUE, title TEXT, body TEXT, hash TEXT,
+  updated_at TEXT);
+CREATE TABLE IF NOT EXISTS scan_root (
+  id INTEGER PRIMARY KEY, machine TEXT NOT NULL, path TEXT NOT NULL, enabled INTEGER DEFAULT 1,
+  note TEXT DEFAULT '', last_scanned TEXT, UNIQUE(machine, path));
+CREATE TABLE IF NOT EXISTS embedding (
+  kind TEXT NOT NULL, ref_id INTEGER NOT NULL, model TEXT NOT NULL, hash TEXT, vec BLOB,
+  PRIMARY KEY(kind, ref_id));
+CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
+  kind UNINDEXED, ref_id UNINDEXED, project UNINDEXED, audience UNINDEXED, maturity UNINDEXED, origin UNINDEXED, title, body, tags,
+  tokenize='porter unicode61');
+CREATE INDEX IF NOT EXISTS idx_commit_date ON "commit"(date);
+CREATE INDEX IF NOT EXISTS idx_commit_project ON "commit"(project_id, date);
+CREATE INDEX IF NOT EXISTS idx_note_project ON note(project_id, created_at);
+"""
+
+_lock = threading.RLock()
+_conn = None
+
+
+def now():
+    return time.strftime("%Y-%m-%dT%H:%M:%S%z")
+
+
+def connect():
+    global _conn
+    with _lock:
+        if _conn is None:
+            config.DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+            _conn = sqlite3.connect(str(config.DB_PATH), check_same_thread=False, timeout=30)
+            _conn.row_factory = sqlite3.Row
+            _conn.execute("PRAGMA foreign_keys=ON")
+            _conn.executescript(SCHEMA)
+            _migrate(_conn)
+        return _conn
+
+
+# Controlled vocabulary for maturity. Free text is accepted but these are what filters and the UI expect.
+MATURITY = {
+    "authoritative": "the one to use; maintained and trusted",
+    "usable": "works, reuse with normal care",
+    "experimental": "unproven; may be worth building on",
+    "antiquated": "works but superseded or dated; prefer something newer",
+    "sunset": "being retired; do not build on it",
+    "broken": "does not work as is",
+    "junk": "not worth reusing; kept for reference only",
+}
+MATURITY_RANK = {"authoritative": 1.6, "usable": 1.2, "experimental": 1.0, "": 1.0,
+                 "antiquated": 0.7, "sunset": 0.6, "broken": 0.5, "junk": 0.35}
+
+
+def _migrate(c):
+    """Add columns introduced after the first release; recreate the FTS table if its shape changed."""
+    def cols(t):
+        return {r[1] for r in c.execute(f'PRAGMA table_info("{t}")')}
+    for table in ("project", "asset"):
+        for col in ("maturity", "maturity_note"):
+            if col not in cols(table):
+                c.execute(f'ALTER TABLE "{table}" ADD COLUMN {col} TEXT DEFAULT \'\'')
+    for col, typ in (("last_changed", "TEXT"), ("change_count", "INTEGER"), ("blob_hash", "TEXT DEFAULT ''"), ("size", "INTEGER"), ("symbols", "TEXT DEFAULT ''"),
+                     ("imports", "TEXT DEFAULT ''"), ("func_hashes", "TEXT DEFAULT ''"), ("signatures", "TEXT DEFAULT ''"), ("imported_by", "INTEGER DEFAULT 0"),
+                     ("review", "TEXT DEFAULT ''"), ("reviewed_at", "TEXT"), ("source_head", "TEXT DEFAULT ''")):
+        if col not in cols("asset"):
+            c.execute(f"ALTER TABLE asset ADD COLUMN {col} {typ}")
+    for table in ("project", "asset"):
+        for col, typ in (("trust", "INTEGER"), ("trust_breakdown", "TEXT DEFAULT ''"), ("verified_at", "TEXT"), ("verified_note", "TEXT DEFAULT ''")):
+            if col not in cols(table):
+                c.execute(f'ALTER TABLE "{table}" ADD COLUMN {col} {typ}')
+    if "visibility" not in cols("project"):
+        c.execute("ALTER TABLE project ADD COLUMN visibility TEXT DEFAULT 'private'")
+    if "origin" not in cols("project"):
+        c.execute("ALTER TABLE project ADD COLUMN origin TEXT DEFAULT 'own'")
+    # 2026-09-06: the default audience label changed from 'personal' to 'unrestricted' (see docs/USER_GUIDE.md).
+    c.execute("UPDATE project SET audience='unrestricted' WHERE audience='personal'")
+    c.execute("UPDATE search_index SET audience='unrestricted' WHERE audience='personal'")
+    if "maturity" not in cols("search_index") or "origin" not in cols("search_index"):
+        c.execute("DROP TABLE search_index")
+        c.executescript(SCHEMA)
+        c.commit()
+        from .store import reindex_all
+        reindex_all(keep_embeddings=True)
+    c.commit()
+
+
+@contextmanager
+def tx():
+    """Serialised write transaction. SQLite is fine with this at our scale."""
+    c = connect()
+    with _lock:
+        try:
+            yield c
+            c.commit()
+        except Exception:
+            c.rollback()
+            raise
+
+
+def q(sql, params=()):
+    with _lock:
+        return [dict(r) for r in connect().execute(sql, params).fetchall()]
+
+
+def one(sql, params=()):
+    rows = q(sql, params)
+    return rows[0] if rows else None
+
+
+# ---- search index maintenance ------------------------------------------------
+# Every record type writes itself into search_index through index_item, so search
+# needs no joins and new kinds need no schema change.
+
+def index_item(c, kind, ref_id, title, body, tags="", project="", maturity=None):
+    """Index one record. Audience is inherited from the project so filtering needs no joins."""
+    audience, pmat, origin = "", "", "own"
+    if project:
+        r = c.execute("SELECT audience, maturity, origin FROM project WHERE name=? COLLATE NOCASE", (project,)).fetchone()
+        audience, pmat, origin = ((r[0] or ""), (r[1] or ""), (r[2] or "own")) if r else ("", "", "own")
+    if maturity is None:
+        maturity = pmat  # records inherit their project's rating unless they carry their own
+    c.execute("DELETE FROM search_index WHERE kind=? AND ref_id=?", (kind, ref_id))
+    c.execute("INSERT INTO search_index(kind, ref_id, project, audience, maturity, origin, title, body, tags) VALUES (?,?,?,?,?,?,?,?,?)",
+              (kind, ref_id, project or "", audience, maturity or "", origin, title or "", (body or "")[:20000], tags or ""))
+    if not KEEP_EMBEDDINGS:
+        c.execute("DELETE FROM embedding WHERE kind=? AND ref_id=?", (kind, ref_id))
+
+
+KEEP_EMBEDDINGS = False
+
+
+def unindex(c, kind, ref_id):
+    c.execute("DELETE FROM search_index WHERE kind=? AND ref_id=?", (kind, ref_id))
+    c.execute("DELETE FROM embedding WHERE kind=? AND ref_id=?", (kind, ref_id))
+
+
+def rebuild_index():
+    """Drop and recreate search_index from the source tables."""
+    from .store import reindex_all
+    reindex_all()
+
+
+def tags_norm(tags):
+    if not tags:
+        return ""
+    if isinstance(tags, str):
+        parts = [t.strip() for t in tags.replace(";", ",").split(",")]
+    else:
+        parts = [str(t).strip() for t in tags]
+    return ",".join(sorted({p.lower() for p in parts if p}))
+
+
+def dumps(x):
+    return json.dumps(x, ensure_ascii=False, default=str)

codemem/describe.py

@@ -0,0 +1,71 @@
+"""Draft a description and purpose for projects that lack one, using a local Ollama model.
+
+Only touches own projects whose description is empty, tiny, or just the name. Everything it writes
+is tagged `auto-described` so a human-written replacement is easy to spot and the draft is never
+mistaken for a considered statement. Re-running skips projects already tagged.
+"""
+import json, urllib.request
+from pathlib import Path
+from . import config
+from .db import q, one
+from .store import upsert_project
+
+MODEL = config.DESCRIBE_MODEL
+PROMPT = """You are cataloguing a developer's personal projects. From the evidence below, write:
+1. "description": one sentence, max 25 words, what the project IS (tool/app/library/experiment) and what it does. No marketing words.
+2. "purpose": one sentence, max 25 words, why someone would reach for it, or what problem it solved.
+If the evidence is too thin, say so in the description ("Unclear: ...") rather than guessing.
+Answer with JSON only: {{"description": "...", "purpose": "..."}}
+
+Project name: {name}
+Languages: {languages}
+Key files: {key_files}
+Path: {path}
+README / CLAUDE.md head:
+{readme}
+Recent commit subjects:
+{commits}
+"""
+
+
+def candidates():
+    return q("""SELECT p.id, p.name, p.description, p.tags, l.path, l.languages, l.key_files, l.readme_head
+                FROM project p LEFT JOIN location l ON l.project_id=p.id AND l.machine=?
+                WHERE p.origin='own' AND (p.description='' OR length(p.description)<25 OR p.description=p.name COLLATE NOCASE)
+                AND (',' || p.tags || ',') NOT LIKE '%,auto-described,%' GROUP BY p.id ORDER BY p.name""", (config.MACHINE,))
+
+
+def ask(prompt):
+    req = urllib.request.Request(f"{config.OLLAMA_URL}/api/chat", data=json.dumps({
+        "model": MODEL, "stream": False, "format": "json", "options": {"temperature": 0.2, "num_ctx": 8192},
+        "messages": [{"role": "user", "content": prompt}]}).encode(), headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=300) as r:
+        return json.loads(json.load(r)["message"]["content"])
+
+
+def describe_all(log=print, dry=False, limit=100):
+    done = 0
+    for c in candidates()[:limit]:
+        commits = q('SELECT substr(message,1,90) AS m FROM "commit" WHERE project_id=? ORDER BY date DESC LIMIT 8', (c["id"],))
+        readme = c["readme_head"] or ""
+        if c["path"] and not readme:
+            for f in ("README.md", "CLAUDE.md"):
+                fp = Path(c["path"]) / f
+                if fp.exists():
+                    readme = "\n".join(fp.read_text(errors="replace").splitlines()[:25])[:2000]; break
+        if not readme and not commits and not c["path"]:
+            log(f"  {c['name']}: no evidence, skipped"); continue
+        prompt = PROMPT.format(name=c["name"], languages=c["languages"] or "?", key_files=c["key_files"] or "?",
+                               path=c["path"] or "(bare repo only)", readme=readme or "(none)",
+                               commits="\n".join(x["m"].splitlines()[0] for x in commits) or "(none)")
+        try:
+            out = ask(prompt)
+        except Exception as e:
+            log(f"  {c['name']}: model error {e}"); continue
+        desc, purpose = (out.get("description") or "").strip()[:300], (out.get("purpose") or "").strip()[:300]
+        log(f"  {c['name']}: {desc}")
+        if not dry and desc:
+            tags = ",".join(t for t in [c["tags"], "auto-described"] if t)
+            upsert_project(c["name"], description=desc, purpose=purpose, tags=tags)
+            done += 1
+    return done

codemem/discover.py

@@ -0,0 +1,180 @@
+"""Asset discovery on the server, and ingestion of assets posted by remote agents.
+
+Server side: mine every own repo (bare repo in GIT_ROOT, else the local working copy) with the
+shared rules in discover_core. Remote side: client/codemem_agent.py runs the same rules on its
+machine and posts assets inside its scan payload; ingest_assets() below stores them identically.
+
+Then link retooling across ALL projects and machines from what is stored: identical blob hashes,
+and Python files whose symbol sets overlap by half or more, become `shares-code-with` links.
+"""
+import json, urllib.request
+from collections import defaultdict
+from pathlib import Path
+from . import config
+from .db import q, one, tx
+from .store import upsert_asset, add_link, get_project
+from .discover_core import scan_bare, scan_worktree, asset_name
+
+DESCRIBE_MODEL = config.DESCRIBE_MODEL
+
+
+def draft_description(path, text_head):
+    prompt = ("One sentence, max 25 words, plain: what does this file do and when would someone reuse it? "
+              "If unclear, start with 'Unclear:'. Answer JSON {\"description\": \"...\"}\n\nFile: " + path + "\n\n" + text_head[:4000])
+    try:
+        req = urllib.request.Request(f"{config.OLLAMA_URL}/api/chat", data=json.dumps({
+            "model": DESCRIBE_MODEL, "stream": False, "format": "json", "options": {"temperature": 0.2, "num_ctx": 8192},
+            "messages": [{"role": "user", "content": prompt}]}).encode(), headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(req, timeout=300) as r:
+            return (json.loads(json.load(r)["message"]["content"]).get("description") or "").strip()[:300]
+    except Exception:
+        return ""
+
+
+def ingest_assets(project, assets, machine="", base_path="", describe=True):
+    """Store asset dicts (from scan_bare/scan_worktree or a remote payload) for a project.
+    Existing assets at the same path are refreshed; human descriptions are never overwritten."""
+    created = updated = drafted = 0
+    for a in assets:
+        path = a["path"].replace("\\", "/")
+        existing = one("SELECT * FROM asset WHERE project_id=? AND (path=? OR path LIKE ?)", (project["id"], path, f"%/{path}"))
+        fields = {"last_changed": a.get("last_changed"), "change_count": a.get("change_count"), "blob_hash": a.get("blob_hash") or "",
+                  "size": a.get("size"), "symbols": ",".join(a.get("symbols") or [])[:2000],
+                  "imports": ",".join(a.get("imports") or [])[:1000], "func_hashes": json.dumps(a.get("func_hashes") or [])[:20000],
+                  "signatures": "\n".join(a.get("signatures") or [])[:6000]}
+        if a.get("source_head"):
+            fields["source_head"] = a["source_head"][:16000]
+        if existing:
+            if not existing["usage"] and a.get("usage"):
+                fields["usage"] = a["usage"]
+            if base_path and not existing["machine"]:
+                fields["machine"] = machine; fields["path"] = f"{base_path}/{path}"
+            upsert_asset(existing["name"], existing["kind"], **fields)
+            updated += 1
+        else:
+            desc = a.get("description") or ""
+            tags = "auto-discovered"
+            if not desc and describe and a.get("head"):
+                desc = draft_description(path, a["head"])
+                if desc:
+                    tags += ",auto-described"; drafted += 1
+            upsert_asset(asset_name(path, project["name"]), a["kind"], project=project["name"],
+                         path=f"{base_path}/{path}" if base_path else path, machine=machine,
+                         description=desc or f"Unclear: no docstring in {path}", usage=a.get("usage") or "", tags=tags, **fields)
+            created += 1
+    return {"created": created, "updated": updated, "drafted": drafted}
+
+
+def repo_for_project(p):
+    bare = config.GIT_ROOT / f"{p['name']}.git"
+    if bare.is_dir():
+        return bare, True
+    loc = one("SELECT path FROM location WHERE project_id=? AND machine=? AND is_git=1", (p["id"], config.MACHINE))
+    if loc and Path(loc["path"], ".git").exists():
+        return Path(loc["path"]), False
+    return None, None
+
+
+def discover(names=None, describe=True, log=print):
+    projects = q("SELECT * FROM project WHERE origin='own'" + (f" AND name IN ({','.join('?' * len(names))})" if names else ""), names or ())
+    tot = defaultdict(int)
+    for p in projects:
+        repo, bare = repo_for_project(p)
+        if not repo:
+            continue
+        local = one("SELECT path FROM location WHERE project_id=? AND machine=?", (p["id"], config.MACHINE))
+        assets = list(scan_bare(repo) if bare else scan_worktree(repo))
+        if describe:
+            _attach_heads(repo, bare, assets)
+        r = ingest_assets(p, assets, machine=config.MACHINE if local else "", base_path=local["path"] if local else "", describe=describe)
+        for k, v in r.items():
+            tot[k] += v
+        log(f"  {p['name']}: {len(assets)} candidates")
+    tot["links"] = link_shared_code(log)
+    tot["projects"] = len(projects)
+    return dict(tot)
+
+
+def _attach_heads(repo, bare, assets):
+    """For files without a description, fetch the first 60 lines so the model can draft one."""
+    from .discover_core import _git
+    for a in assets:
+        if a.get("description"):
+            continue
+        if bare:
+            raw = _git(repo, "cat-file", "-p", a["blob_hash"], binary=True)
+        else:
+            try:
+                raw = (Path(repo) / a["path"]).read_bytes()
+            except OSError:
+                raw = b""
+        a["head"] = "\n".join(raw.decode("utf-8", "replace").splitlines()[:60])
+
+
+def link_shared_code(log=print):
+    """Cross-project, cross-machine: identical blobs and >=50% overlapping symbol sets."""
+    rows = q("""SELECT a.path, a.blob_hash, a.symbols, a.func_hashes, p.name AS project FROM asset a JOIN project p ON p.id=a.project_id
+                WHERE p.origin='own' AND a.tags LIKE '%auto-discovered%'""")
+    pairs = defaultdict(set)
+    byblob = defaultdict(list)
+    for r in rows:
+        if r["blob_hash"]:
+            byblob[r["blob_hash"]].append(r)
+    for locs in byblob.values():
+        projs = sorted({r["project"] for r in locs})
+        for i, a in enumerate(projs):
+            for b in projs[i + 1:]:
+                fa = next(r["path"] for r in locs if r["project"] == a); fb = next(r["path"] for r in locs if r["project"] == b)
+                pairs[(a, b)].add(f"identical: {Path(fa).name} = {Path(fb).name}" if Path(fa).name == Path(fb).name else f"identical: {fa} = {fb}")
+    syms = [(r["project"], r["path"], frozenset(r["symbols"].split(","))) for r in rows if r["symbols"] and r["symbols"].count(",") >= 4]
+    for i, (pa, fa, sa) in enumerate(syms):
+        for pb, fb, sb in syms[i + 1:]:
+            if pa == pb:
+                continue
+            j = len(sa & sb) / len(sa | sb)
+            if j >= 0.5:
+                a, b = sorted([pa, pb])
+                pairs[(a, b)].add(f"similar ({j:.0%} same functions): {fa} ~ {fb}")
+    # function level: the same normalized body in two different projects (renames and docstrings ignored)
+    byfunc = defaultdict(list)
+    for r in rows:
+        if not r["func_hashes"]:
+            continue
+        try:
+            for f in json.loads(r["func_hashes"]):
+                if f.get("lines", 0) >= 8:
+                    byfunc[f["hash"]].append((r["project"], r["path"], f["name"], f["lines"]))
+        except ValueError:
+            pass
+    for hits in byfunc.values():
+        projs = sorted({h[0] for h in hits})
+        if len(projs) < 2:
+            continue
+        for i, a in enumerate(projs):
+            for b in projs[i + 1:]:
+                ha = next(h for h in hits if h[0] == a); hb = next(h for h in hits if h[0] == b)
+                if any(n.startswith("identical:") and Path(ha[1]).name in n for n in pairs[(a, b)]):
+                    continue  # whole file already reported
+                pairs[(a, b)].add(f"function {ha[2]} ({ha[3]} lines): {ha[1]} = {hb[1]}" + (f" as {hb[2]}" if hb[2] != ha[2] else ""))
+    # Keep model verdicts ("model: ...") from review stage 1 across rebuilds; only the computed parts are regenerated.
+    kept = {}
+    for l in q("SELECT from_id, to_id, note FROM link WHERE relation='shares-code-with'"):
+        parts = [x for x in (l["note"] or "").split("; ") if x.startswith("model:")]
+        if parts:
+            kept[(l["from_id"], l["to_id"])] = parts
+    with tx() as c:
+        c.execute("DELETE FROM link WHERE relation='shares-code-with'")
+    n = 0
+    seen = set()
+    for (a, b), files in pairs.items():
+        pa, pb = get_project(a), get_project(b)
+        if pa and pb:
+            key = (pa["id"], pb["id"]); seen.add(key)
+            note = "; ".join(kept.get(key, []) + sorted(files))[:2000]
+            add_link("project", pa["id"], "project", pb["id"], "shares-code-with", note)
+            n += 1
+    for key, parts in kept.items():   # verdict pairs whose computed evidence vanished still keep the verdict
+        if key not in seen:
+            add_link("project", key[0], "project", key[1], "shares-code-with", "; ".join(parts)[:2000]); n += 1
+    log(f"  shared code: {n} project pairs linked")
+    return n