sessionmem 1.6.0 → 1.8.0

package/dist/core/retrieve/temporalIntent.js

@@ -0,0 +1,59 @@
+const DAY = 86_400_000;
+function range(start, end) {
+    return { kind: "past", start, end };
+}
+export function detectTemporalIntent(query, now) {
+    const text = query.normalize("NFKC").toLowerCase();
+    // A dated or historical clause takes precedence over a current-state cue.
+    const iso = /\b(\d{4})-(\d{2})(?:-(\d{2}))?\b/.exec(text);
+    if (iso) {
+        const year = Number(iso[1]), month = Number(iso[2]) - 1, day = Number(iso[3] ?? 1);
+        const start = Date.UTC(year, month, day);
+        const date = new Date(start);
+        if (date.getUTCFullYear() === year && date.getUTCMonth() === month && date.getUTCDate() === day) {
+            return range(start, iso[3] ? start + DAY : Date.UTC(year, month + 1, 1));
+        }
+    }
+    const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
+    if (/\byesterday\b/.test(text))
+        return range(today - DAY, today);
+    const relative = /\b(?:last|previous) (week|month|year)\b/.exec(text);
+    if (relative) {
+        const year = now.getUTCFullYear(), month = now.getUTCMonth();
+        if (relative[1] === "month")
+            return range(Date.UTC(year, month - 1, 1), Date.UTC(year, month, 1));
+        if (relative[1] === "year")
+            return range(Date.UTC(year - 1, 0, 1), Date.UTC(year, 0, 1));
+        // Calendar weeks start on Monday, in UTC.
+        const monday = today - ((now.getUTCDay() + 6) % 7) * DAY;
+        return range(monday - 7 * DAY, monday);
+    }
+    const ago = /\b(\d{1,3}) (day|week)s? ago\b/.exec(text);
+    if (ago) {
+        const start = today - Number(ago[1]) * (ago[2] === "week" ? 7 : 1) * DAY;
+        return range(start, start + (ago[2] === "week" ? 7 : 1) * DAY);
+    }
+    const year = /\b(?:in|during) ((?:19|20)\d{2})\b/.exec(text);
+    if (year)
+        return range(Date.UTC(Number(year[1]), 0, 1), Date.UTC(Number(year[1]) + 1, 0, 1));
+    if (/\b(before|previously|formerly|prior|used to|no longer)\b/.test(text))
+        return { kind: "past", previous: true };
+    if (/\b(when (?:did|was|were)|in the past|historical|history)\b/.test(text))
+        return { kind: "past" };
+    if (/\b(current|currently|now|latest|today|at present)\b/.test(text))
+        return { kind: "current" };
+    return { kind: "none" };
+}
+/** A bounded signal, separate from lexical relevance. Missing dates stay neutral. */
+export function temporalRelevance(intent, timestamp, supersededFraction, now) {
+    const time = Date.parse(timestamp);
+    if (intent.kind === "current") {
+        const freshness = Number.isFinite(time) ? 1 / (1 + Math.max(0, now.getTime() - time) / (30 * DAY)) : 0.5;
+        return (1 - supersededFraction + freshness) / 2;
+    }
+    if (intent.start !== undefined && intent.end !== undefined && Number.isFinite(time)) {
+        const distance = time < intent.start ? intent.start - time : time >= intent.end ? time - intent.end + 1 : 0;
+        return 1 / (1 + distance / (30 * DAY));
+    }
+    return intent.previous ? supersededFraction : 0.5;
+}

package/dist/core/schema/migrations/016_semantic_embeddings.sql

@@ -0,0 +1,12 @@
+-- Separate learned vectors from legacy hash embeddings. Models never mix.
+CREATE TABLE semantic_embeddings (
+  memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE ON UPDATE CASCADE,
+  model TEXT NOT NULL,
+  dimension INTEGER NOT NULL CHECK (dimension > 0),
+  vector BLOB NOT NULL,
+  PRIMARY KEY (memory_id, model)
+);
+CREATE TRIGGER semantic_embeddings_content_update AFTER UPDATE OF content ON memories
+WHEN old.content IS NOT new.content BEGIN
+  DELETE FROM semantic_embeddings WHERE memory_id = new.id;
+END;

package/dist/core/storage/entityLinkRepo.js

@@ -0,0 +1,47 @@
+import { queryEntityNames } from "../extract/knowledge.js";
+// Explicit spelling aliases only. No fuzzy matching or inferred coreference.
+const ALIASES = [["postgres", "postgresql"], ["node", "node.js"], ["k8s", "kubernetes"]];
+/** Link literal query names to existing entities, then weight rare names more. */
+export function linkedEntityScores(db, projectId, query, now, includeHistory, hubDamping = false) {
+    const names = new Set(queryEntityNames(query));
+    for (const group of ALIASES)
+        if (group.some(name => names.has(name)))
+            for (const name of group)
+                names.add(name);
+    const rows = db.prepare(`SELECT DISTINCT e.memory_id, e.kind, e.name FROM memory_entities e
+    JOIN memories m ON m.id = e.memory_id
+    WHERE m.project_id = ? AND (m.expires_at IS NULL OR m.expires_at > ?)
+      AND (? = 1 OR m.id NOT IN (SELECT memory_id FROM superseded_memories))
+      AND e.name IN (SELECT value FROM json_each(?)) ORDER BY e.kind, e.name, e.memory_id`)
+        .all(projectId, now.toISOString(), Number(includeHistory), JSON.stringify([...names]));
+    const frequencies = new Map();
+    const key = (row) => `${row.kind}:${row.name}`;
+    for (const row of rows)
+        frequencies.set(key(row), (frequencies.get(key(row)) ?? 0) + 1);
+    if (hubDamping) {
+        // Match at most eight stored names, preferring specific spans. Exact and
+        // explicit alias matches have similarity 1; no entity model is implied.
+        const selected = new Set([...new Set(rows.map(row => row.name))]
+            .sort((a, b) => b.length - a.length || a.localeCompare(b)).slice(0, 8));
+        const boosts = new Map();
+        for (const row of rows) {
+            if (!selected.has(row.name))
+                continue;
+            const degree = frequencies.get(key(row));
+            const boost = 0.5 / (1 + 0.001 * (degree - 1) ** 2);
+            boosts.set(row.memory_id, Math.max(boosts.get(row.memory_id) ?? 0, boost));
+        }
+        return new Map([...boosts].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])));
+    }
+    const total = db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE project_id = ?
+    AND (expires_at IS NULL OR expires_at > ?)
+    AND (? = 1 OR id NOT IN (SELECT memory_id FROM superseded_memories))`)
+        .get(projectId, now.toISOString(), Number(includeHistory)).n;
+    const weights = new Map([...frequencies].map(([name, count]) => [name, 1 + Math.log((total + 1) / (count + 1))]));
+    const denominator = [...weights.values()].reduce((a, b) => a + b, 0);
+    const scores = new Map();
+    for (const row of rows)
+        scores.set(row.memory_id, (scores.get(row.memory_id) ?? 0) + weights.get(key(row)) / denominator);
+    // Keep scores for all existing lexical/dense candidates. Only admission is capped.
+    return new Map([...scores].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])));
+}

package/dist/core/storage/memorySearchRepo.js

@@ -91,6 +91,17 @@ function mapRows(rows) {
         };
     });
 }
+/** Hydrate a bounded semantic candidate list under the same visibility rules. */
+export function searchMemoryCandidatesByIds(db, projectId, ids, now, includeHistory = false) {
+    if (!ids.length)
+        return [];
+    const rows = db.prepare(`SELECT * FROM memories WHERE project_id = ?
+    AND id IN (${ids.map(() => "?").join(",")})
+    AND (expires_at IS NULL OR expires_at > ?)
+    AND (? = 1 OR id NOT IN (SELECT memory_id FROM superseded_memories))`)
+        .all(projectId, ...ids, now.toISOString(), Number(includeHistory));
+    return mapRows(rows);
+}
 export function searchMemoryCandidates(db, projectId, now = new Date(), includeHistory = false) {
     const rows = getSearchStatements(db).searchCandidates.all({
         projectId,
@@ -107,13 +118,15 @@ const QUERY_STOP_WORDS = new Set(("a an and are as at be been being but by can c
     "where which who whom why will with would you your yours yourself yourselves " +
     "s t don doesn didn isn wasn weren won wouldn couldn shouldn").split(" "));
 /** Match any meaningful query term; question grammar must not exclude evidence. */
-function sanitizeFtsQuery(queryText) {
+export function lexicalQueryTerms(queryText) {
     const tokens = [...new Set(queryText.normalize("NFKC").toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [])];
     const terms = tokens.filter((token) => !QUERY_STOP_WORDS.has(token));
     // Preserve literal searches consisting entirely of stop words. Tokens contain
     // only letters/numbers, so FTS operators and punctuation cannot become syntax.
-    return (terms.length ? terms : tokens)
-        .slice(0, 64)
+    return (terms.length ? terms : tokens).slice(0, 64);
+}
+function sanitizeFtsQuery(queryText) {
+    return lexicalQueryTerms(queryText)
         .map((token) => `"${token}"`)
         .join(" OR ");
 }

package/dist/core/storage/semanticSearchRepo.js

@@ -0,0 +1,56 @@
+import { searchMemoryCandidatesByIds } from "./memorySearchRepo.js";
+function normalize(vector, dimension) {
+    if (vector.length !== dimension || !vector.every(Number.isFinite))
+        throw new Error("Invalid local embedding");
+    const norm = Math.hypot(...vector);
+    if (!norm)
+        throw new Error("Empty local embedding");
+    return vector.map(v => v / norm);
+}
+/** Backfill from stored text after normal capture policy; never raw capture input. */
+export async function indexSemanticMemories(db, projectId, embedder, memoryId) {
+    const rows = db.prepare(`SELECT m.id, m.content FROM memories m
+    WHERE m.project_id = ? AND (? IS NULL OR m.id = ?) AND NOT EXISTS (SELECT 1 FROM semantic_embeddings s
+      WHERE s.memory_id = m.id AND s.model = ? AND s.dimension = ?)`)
+        .all(projectId, memoryId ?? null, memoryId ?? null, embedder.id, embedder.dimension);
+    const insert = db.prepare(`INSERT OR REPLACE INTO semantic_embeddings(memory_id, model, dimension, vector)
+    SELECT id, @model, @dimension, @vector FROM memories
+    WHERE id = @id AND project_id = @project AND content = @content`);
+    for (const row of rows) {
+        const vector = normalize(await embedder.embed(row.content), embedder.dimension);
+        const bytes = Buffer.alloc(vector.length * 4);
+        vector.forEach((v, i) => bytes.writeFloatLE(v, i * 4));
+        // A concurrent edit/delete during inference must not install stale vectors.
+        insert.run({ ...row, model: embedder.id, dimension: embedder.dimension, vector: bytes, project: projectId });
+    }
+}
+/** Exact local cosine scan. Appropriate to small project stores; no ANN service. */
+export async function searchSemanticMemories(db, projectId, query, now, includeHistory, embedder) {
+    await indexSemanticMemories(db, projectId, embedder);
+    const vector = normalize(await embedder.embed(query), embedder.dimension);
+    const rows = db.prepare(`SELECT s.memory_id, s.vector FROM semantic_embeddings s JOIN memories m ON m.id = s.memory_id
+    WHERE m.project_id = @project AND s.model = @model AND s.dimension = @dimension
+      AND (m.expires_at IS NULL OR m.expires_at > @now)
+      AND (@history = 1 OR m.id NOT IN (SELECT memory_id FROM superseded_memories))`)
+        .all({ project: projectId, model: embedder.id, dimension: embedder.dimension, now: now.toISOString(), history: Number(includeHistory) });
+    const scored = [];
+    for (const row of rows) {
+        if (row.vector.length !== vector.length * 4)
+            continue;
+        let similarity = 0, norm = 0;
+        for (let i = 0; i < vector.length; i++) {
+            const value = row.vector.readFloatLE(i * 4);
+            similarity += value * vector[i];
+            norm += value * value;
+        }
+        if (Number.isFinite(similarity) && norm > 0)
+            scored.push({ id: row.memory_id, similarity: Math.max(-1, Math.min(1, similarity / Math.sqrt(norm))) });
+    }
+    scored.sort((a, b) => b.similarity - a.similarity || a.id.localeCompare(b.id));
+    const top = scored.slice(0, 50);
+    const candidates = new Map(searchMemoryCandidatesByIds(db, projectId, top.map(r => r.id), now, includeHistory).map(c => [c.id, c]));
+    return top.flatMap(row => {
+        const candidate = candidates.get(row.id);
+        return candidate ? [{ candidate, similarity: row.similarity }] : [];
+    });
+}

package/dist/core/storage/temporalRepo.js

@@ -0,0 +1,13 @@
+/** Use claim observation times when available, otherwise the original capture time. */
+export function memoryTemporalMetadata(db, projectId, ids) {
+    const rows = db.prepare(`SELECT m.id, m.created_at, MAX(c.observed_at) AS observed_at,
+    COUNT(c.claim_key) AS claims, COUNT(s.claim_key) AS superseded
+    FROM memories m LEFT JOIN memory_claims c ON c.memory_id = m.id
+    LEFT JOIN memory_supersessions s ON s.supersedes = c.memory_id AND s.claim_key = c.claim_key
+    WHERE m.project_id = ? AND m.id IN (SELECT value FROM json_each(?)) GROUP BY m.id`)
+        .all(projectId, JSON.stringify(ids));
+    return new Map(rows.map(row => [row.id, {
+            timestamp: row.observed_at ?? row.created_at,
+            supersededFraction: row.claims ? row.superseded / row.claims : 0,
+        }]));
+}

package/dist/core/storage/timelineRepo.js

@@ -0,0 +1,27 @@
+import { DomainError } from "../api/errors.js";
+/** Small SQL projections, stable ordering, and no access-count writes. */
+export function memoryTimeline(db, input, now) {
+    const visible = `project_id = @projectId AND (expires_at IS NULL OR expires_at > @now)
+    AND (@history = 1 OR id NOT IN (SELECT memory_id FROM superseded_memories))`;
+    const bindings = { projectId: input.projectId, now: now.toISOString(), history: Number(input.includeHistory) };
+    // Project only enough text for the one-line summary, never embeddings or bodies.
+    const select = `SELECT id, substr(content, 1, 1000) AS content, kind, created_at FROM memories WHERE ${visible}`;
+    if (!input.memoryId) {
+        const rows = db.prepare(`${select} AND created_at >= @fromDate AND created_at <= @toDate
+      ORDER BY created_at, id LIMIT @limit`).all({ ...bindings,
+            fromDate: new Date(input.fromDate).toISOString(), toDate: new Date(input.toDate).toISOString(), limit: input.limit + 1, });
+        return { rows: rows.slice(0, input.limit), hasMore: rows.length > input.limit };
+    }
+    const anchor = db.prepare(`${select} AND id = @memoryId`).get({ ...bindings, memoryId: input.memoryId });
+    if (!anchor)
+        throw new DomainError("NOT_FOUND", `Memory not found: ${input.memoryId}`);
+    const params = { ...bindings, date: anchor.created_at, id: anchor.id, limit: input.limit + 1 };
+    const before = db.prepare(`${select} AND (created_at < @date OR (created_at = @date AND id < @id))
+    ORDER BY created_at DESC, id DESC LIMIT @limit`).all(params);
+    const after = db.prepare(`${select} AND (created_at > @date OR (created_at = @date AND id > @id))
+    ORDER BY created_at, id LIMIT @limit`).all(params);
+    const left = Math.min(before.length, Math.max(Math.floor((input.limit - 1) / 2), input.limit - 1 - after.length));
+    const right = Math.min(after.length, input.limit - 1 - left);
+    return { rows: [...before.slice(0, left).reverse(), anchor, ...after.slice(0, right)],
+        hasMore: before.length + after.length + 1 > input.limit };
+}

package/docs/architecture.md

@@ -70,7 +70,19 @@ When an agent or `sessionmem search` submits a query, the core engine searches t
 
 The ranker divides each positive BM25 score by the best score for that query. If the query contains known entity names in this project, relevance is `0.90 * base relevance + 0.10 * entity overlap`. Overlap is the fraction of distinct matched query names linked to the candidate memory; names are matched as literal token sequences (up to eight tokens), including lowercase questions. With no known query entities, relevance is unchanged. This reranks the existing candidates; it does not expand the FTS candidate pool or traverse relations. The outer relevance, recency, and importance weights stay 0.60, 0.25, and 0.15. Sparse lexical matches retain the recency/importance fallback union. If there are no lexical matches, base relevance uses the previous hash-vector ranking. Project scope and expiry checks apply to both paths. Automatic migrations preserve memory bodies and IDs.
 
-Existing public request fields and all response fields are unchanged; `includeHistory` is an optional addition. The `semantic` field still reports hash cosine similarity. The legacy `score.raw.semantic` and `score.weighted.semantic` fields carry the relevance component used for ranking, including entity overlap when query entities match. Scores are relative to the current query's candidates, not calibrated probabilities. Retrieval needs no API keys, downloaded models, or new dependencies.
+Existing public request fields and all response fields are unchanged; `includeHistory` is an optional addition. With default retrieval, the `semantic` field reports hash cosine similarity. The legacy `score.raw.semantic` and `score.weighted.semantic` fields carry the relevance component used for ranking, including entity overlap when query entities match. Scores are relative to the current query's candidates, not calibrated probabilities. Default retrieval needs no API keys, downloaded models, or new dependencies.
+
+With `semanticSearch=true` and the optional local packages installed, migration
+016 stores versioned learned vectors separately from hash embeddings. Retrieval
+unions up to 50 exact-cosine semantic candidates with lexical candidates, blends
+normalized BM25 and positive cosine 2:1, and retains the existing entity and outer
+scoring weights. With no lexical match it uses cosine relevance. The `semantic`
+field remains the legacy hash diagnostic, matching `embeddingDim` and
+`embeddingVersion`; learned cosine contributes to ranking. The same project,
+expiry, and supersession checks apply. Edits invalidate vectors and deletes
+cascade; missing vectors are backfilled from stored redacted content. This path
+does not change contradiction rules. Model loading/inference errors restore the
+default retrieval path. See [installation and limitations](local-semantic.md).
 
 ## Injection flow
 

package/docs/automatic-memory.md

@@ -0,0 +1,86 @@
+# Automatic capture and recall
+
+Install once and restart the host so it loads its MCP configuration and guidance.
+The installed flow does not ask the user to run scans, recall commands or save
+commands during ordinary agent work. Host tool permissions must allow the memory
+tools. `sessionmem doctor --adapter <name>` checks the configured flow and gives
+repair hints for missing hooks, stale instructions, disabled MCP wiring and stale
+watcher checkpoints.
+
+## Host audit and behavior
+
+| Host | Capture after install | Session-start recall | Recall on prompts |
+| --- | --- | --- | --- |
+| Claude Code | PostToolUse captures Bash/Edit/Write/MultiEdit; SessionEnd summarizes; login watcher also imports transcripts | SessionStart hook catches up transcripts and injects context | UserPromptSubmit hook retrieves context for the submitted prompt |
+| Codex | Login-started watcher imports local rollout transcripts | Global AGENTS.md and MCP instructions tell the agent to search before work and fetch selected details | Instructions tell the agent to search for a new topic |
+| Cursor | Login-started watcher imports supported local composer SQLite records | Project always-apply rule and global MCP instructions tell the agent to search before work | Same instruction-driven topic recall |
+| Windsurf | Agent calls storeMemory/batchStoreMemory under global rules; no transcript importer | Global rules and MCP instructions | Instruction-driven topic recall |
+| Cline | Agent calls storeMemory/batchStoreMemory under installed rules; no transcript importer | Installed rules and MCP instructions | Instruction-driven topic recall |
+| Antigravity, QCoder adapters | Agent tool saves when the host loads the configured MCP server and guidance; no transcript importer | Instruction-driven; existing adapter file paths are unverified against the native products | Instruction-driven; host loading must be verified |
+| Generic MCP | Agent tool saves when the host reads project AGENTS.md or MCP instructions; no transcript importer | Instruction-driven, dependent on host support | Instruction-driven topic recall |
+
+Only Claude Code has deterministic host hooks in this integration. Instruction-
+driven recall is configured by install, but doctor cannot prove that a model
+executed the instructions or that an unverified native host loaded a guessed
+configuration path. Doctor marks Antigravity/QCoder automation unestablished and
+gives a native-settings verification hint. Its output identifies this distinction. The tests simulate
+an agent following the installed instructions; they do not launch proprietary
+host interfaces or claim that every model obeys instructions.
+
+The watcher starts immediately at install and at login through launchd on macOS,
+a user systemd service on Linux, or Task Scheduler on Windows. It polls supported
+transcripts every five seconds by default. Capture requires readable transcript
+files in a supported format. A missing directory is valid before the first
+session; an unsupported format or unreadable source is a diagnostic failure.
+No user action is needed to import later sessions while the service is healthy.
+
+Claude Code SessionStart and a new MCP server connection also perform an
+incremental catch-up. A session opened immediately after another session can
+therefore recall newly written transcript events before the next watcher poll.
+The MCP server advertises the startup, new-topic recall and decision-save flow
+in its protocol instructions for all hosts. Existing user prose is preserved
+when managed guidance blocks are updated.
+
+Cursor rules are project scoped. The old home-level `.cursor/rules` location did
+not establish a rule for arbitrary projects. Install now writes the rule in the
+current project, repairs `alwaysApply:true`, and supplies the same flow through
+the global MCP server's instructions in other projects. A host that ignores MCP
+instructions needs an active native rule in that project.
+
+Plain-terminal install detection recognizes Claude Code, Codex, Cursor and
+Windsurf CLI executables when on PATH, plus the documented native Claude paths.
+Other hosts still need selection at setup with `--adapter`. Configuration
+directories alone are not treated as evidence of an installed host.
+
+Cline's installed MCP entry preapproves the local search, detail and capture tools
+so these calls do not require a separate user action each time. Deletion and
+external operations are excluded. Other hosts retain their own permission rules.
+
+## Diagnosis and repair
+
+Doctor separately reports hook definitions, MCP startup and tool availability,
+current agent instructions, capture mode and recall mode. For hookless hosts it
+says that recall is instruction-driven and execution is unverified. For hosts
+without transcript support it says that capture consists of agent-driven saves,
+not background transcript import. Hook and guidance configuration are checked
+against the current installed version. Failed guidance writes make install fail
+instead of printing a successful setup message.
+
+Doctor's MCP probe bypasses startup capture so diagnosis does not import
+transcripts or alter the user's memories. Its isolated storage canary rolls back.
+The watcher-health check requires a recent, successful checkpoint for the selected
+database, rather than assuming that a registered service is running.
+
+## CI evidence
+
+`tests/integration/cli/automatic-memory-flow.spec.ts` runs in the CI OS matrix.
+It uses a temporary home and project. The OS supervisor commands are simulated,
+but the registered watcher command performs real transcript ingestion.
+
+The Claude case installs the actual hook definitions, feeds capture events,
+ends the session, starts a new session, and verifies the startup and prompt
+context envelopes contain the stored database decision. The Codex and Cursor
+cases create real-format transcript fixtures, install and run the automatic
+watcher sweep, run doctor, then open a new MCP session. A simulated agent follows
+the installed instructions and verifies selected full details reach its context.
+No manual import or scan command is used.

package/docs/benchmark.md

@@ -5,6 +5,22 @@
 [benchmarks/README.md](../benchmarks/README.md) for the corpus license, selection
 rules, metric definitions, and detailed reproduction instructions.
 
+## Compact tools, temporal retrieval and entity fusion
+
+See the [retrieval evaluation](../benchmarks/retrieval-gaps.md) for fresh 1.7.0
+baselines, per-category before/after tables, complete regression artifacts and
+measured compact-tool token costs. Temporal and source-informed additive experiments remain off by default. Entity
+coverage improves default BM25; optional hybrid results are preserved.
+
+## Local semantic and competitor comparison
+
+The [measured comparison](../COMPARISON.md) records a new unchanged-main run,
+local raw-turn retrieval variants from claude-mem, Mem0 OSS, Graphiti and cognee,
+and the optional MiniLM hybrid. It includes exact commands and commits, complete
+per-question archives, latency costs and regressions. These component scores do
+not evaluate full capture/graph/answer pipelines. The comparisons below are
+earlier changes on earlier commits; their timings are not the new baseline.
+
 ## Entity overlap comparison
 
 The baseline was run first on current main at `81f987c`. A fixed 10% entity

package/docs/local-semantic.md

@@ -0,0 +1,58 @@
+# Optional local semantic retrieval
+
+The default is BM25 with entity-link coverage. Active MiniLM retains the existing
+entity-overlap fusion because stronger coverage lowered its hit rate and recall.
+To add a learned semantic candidate pool,
+install the optional packages beside sessionmem and enable it:
+
+```bash
+ONNXRUNTIME_NODE_INSTALL=skip npm install sessionmem @huggingface/tokenizers@0.2.0 onnxruntime-node@1.30.0 @xcidos/genesis-memory-model@0.1.0-alpha.1
+npx sessionmem config set semanticSearch true
+```
+
+For a global sessionmem installation, use `npm install -g` for the same packages.
+`ONNXRUNTIME_NODE_INSTALL=skip` skips optional CUDA downloads; the npm package
+already contains the CPU runtime. On PowerShell set
+`$env:ONNXRUNTIME_NODE_INSTALL = 'skip'` before the install command.
+Developers should also use that variable with `npm ci` to avoid CUDA downloads.
+Disable semantic retrieval with `sessionmem config set semanticSearch false`.
+
+Installation fetches packages and weights. Capture and retrieval only load files
+from these installed packages; they never download a model, contact a model
+server, or send memory text over a network. Missing packages, checksum mismatches,
+unsupported native runtimes, and inference errors fall back to BM25 and emit one
+diagnostic per service instance. Restart after installing or repairing packages.
+Before loading ONNX Runtime, sessionmem sets the process-wide
+`ORT_DISABLE_TELEMETRY=1` to suppress its native POSIX telemetry uploader.
+The benchmark's `--semantic` mode instead fails if the model cannot load, to
+prevent accidentally publishing fallback scores as semantic results.
+
+The encoder is sentence-transformers/all-MiniLM-L6-v2: 384 dimensions, mean pooling,
+L2 normalization, and at most 256 WordPiece tokens. Its weights and tokenizer come
+from the pinned npm distribution above, whose provenance names upstream revision
+`c9745ed1d9f207416be6d2e6f8de32d1f16199bf`. Both files are SHA-256 checked before
+loading. The model is by the sentence-transformers contributors, distributed under
+[Apache-2.0](https://www.apache.org/licenses/LICENSE-2.0); see
+[model information](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2)
+and the package's provenance README. No competitor code or prompt is included.
+
+Learned vectors are a separate SQLite BLOB index, keyed by memory and complete
+model/tokenizer/pooling version. Content edits invalidate vectors; deletes cascade.
+Only stored text is embedded, after the configured capture redaction policy.
+Capture through `storeMemory` indexes it immediately. Existing memories are
+embedded as stored; other write paths are backfilled
+on the next semantic query, so the first query on a large existing store can be
+slow. Model initialization and indexing cost are distinct from warm retrieval.
+
+Queries perform an exact local cosine scan, take up to 50 semantic candidates,
+and union them with FTS candidates. Project, expiry, and supersession filters
+apply to both channels; `includeHistory` explicitly includes superseded rows.
+Normalized BM25 relevance and positive cosine are blended 2:1 before the existing
+entity, recency, importance, and access signals. With no lexical match, cosine
+supplies relevance. This weighting follows the measured weakness of dense-only
+retrieval; it is not a held-out optimum. Read the measured regressions in
+[the comparison](../COMPARISON.md).
+
+Exact scans are intended for small project stores and scale linearly with the
+number of vectors. Long memories are truncated to the encoder's token window;
+no new chunking or model-based contradiction resolution is implemented.

package/docs/research/automatic-capture.md

@@ -0,0 +1,62 @@
+# Automatic transcript capture
+
+Research reviewed on October 18, 2026, before implementation. This change adopts
+approaches only. No competitor source code was copied or adapted.
+
+| Project and primary sources | Observed approach | Takeaway for sessionmem |
+| --- | --- | --- |
+| [claude-mem hooks](https://github.com/thedotmack/claude-mem/blob/main/plugin/hooks/hooks.json), [worker service](https://github.com/thedotmack/claude-mem/blob/main/docs/public/architecture/worker-service.mdx) | SessionStart starts a worker; async tool and stop hooks enqueue capture. The worker exposes health checks and separate start, stop, status and log controls. | Start capture automatically, keep explicit controls, and diagnose capture independently of registration. Keep the existing Claude Code hooks. |
+| [Mem0 Claude Code plugin](https://github.com/mem0ai/mem0/blob/main/integrations/claude-code-plugin/README.md), [OpenClaw integration](https://github.com/mem0ai/mem0/blob/main/integrations/openclaw/README.md) | Hooks save local evidence; detached workers flush batches and survive host exit. Capture has pause/resume controls. The OpenClaw integration filters and redacts automatic capture. The previously indexed `mem0-plugin/scripts/auto_capture.py` path is now absent; the current plugin README was reviewed instead. | Keep capture local and redacted, preserve restart checkpoints, and expose enable/disable/status. This change keeps the existing offline summarizer. |
+| [Cognee Claude Code integration](https://github.com/topoteretes/cognee-integrations/blob/main/integrations/claude-code/README.md) | SessionStart bootstraps a background watcher, write hooks stage capture, and detached final-sync workers handle exit. Persisted markers coordinate retries and avoid duplicate work. Logs and diagnostics expose failures. | Reuse the existing persistent transcript checkpoints and event IDs. Log failures and make stale capture visible in doctor. |
+| [Engram installation](https://github.com/Gentleman-Programming/engram/blob/main/docs/INSTALLATION.md), [daemon control discussion](https://github.com/Gentleman-Programming/engram/issues/1462) | Local memory tooling also uses a background service. The daemon discussion describes stale executable paths and service/session process collisions as operational problems. The discussion is a reported problem, not an implemented guarantee. | Pin absolute runtime and entry-point paths, refresh one stable registration, and validate the registered paths. |
+
+Sessionmem uses OS login supervision rather than requiring a host hook to start
+capture. This covers Codex and Cursor installations without wired capture hooks.
+The OS implementations share an `enable`, `disable`, `status` interface. They
+update one stable per-user registration and start it immediately. The watcher
+records completed sweeps separately from the OS registration so doctor can report
+capture that has stopped or is failing.
+
+The npm binary is a JavaScript entry point. Services invoke its absolute path
+through the absolute Node executable running install. They do not use npm shims,
+`npx`, an interactive shell, or PATH lookup. Changing Node installations requires
+`sessionmem autostart enable` again.
+
+Native definitions were also checked with `systemd-analyze verify`, Python's
+plist parser, and the [Microsoft Task Scheduler schema](https://learn.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-schema).
+The task uses schema version 1.3, unlimited execution time, and the schema's
+maximum 255 restart attempts at one-minute intervals. Windows status and stop
+checks use [numeric HRESULTs](https://learn.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-error-and-success-constants)
+rather than localized error text.
+
+## Validation boundaries
+
+CI mocks the OS supervisor commands, tests all three native definitions and
+command sequences, and runs the real watcher against synthetic transcripts.
+The packaged-install smoke test uses that same test-only supervisor. It does not
+validate the live login behavior of launchd, systemd or Task Scheduler.
+
+Manual verification required before release:
+
+- macOS: install from a normal GUI login as an ordinary user. Confirm the plist
+  in `~/Library/LaunchAgents`, `launchctl print gui/<uid>/io.sessionmem.watch`,
+  immediate capture, and capture after logout/login with the terminal closed.
+  Repeat install and confirm one loaded job. Check paths containing spaces.
+  Disable, enable and uninstall; verify the job stops and the plist is removed.
+- Linux: use a desktop/login with systemd's user manager. Confirm
+  `systemctl --user is-enabled sessionmem-watch.service` and `is-active`, capture
+  after login with no terminal, and restart after killing the watcher. Repeat
+  install and check one unit. Disable/enable and uninstall must stop capture and
+  remove the unit. Also verify install reports a useful failure without a user
+  bus; it must not claim capture succeeded.
+- Windows: install without elevation as the current user. Confirm one Scheduled
+  Task with that user's logon trigger, immediate capture, capture after logging
+  out/in, paths with spaces/non-ASCII characters, and no interactive prompt.
+  Confirm unlimited runtime, restart after a failing process, and battery
+  settings. Repeat install, disable/enable, and uninstall; verify no old task or
+  watcher remains. Test on a non-English Windows installation as well.
+- Each OS: append a real Codex or Cursor conversation and confirm imported
+  memories, a fresh checkpoint and timestamped `logs/watch.log` output. Move or
+  remove the registered runtime/entry point and check doctor's binary-path fix
+  hint. Stop capture long enough to stale the checkpoint and check the capture
+  failure hint. Uninstall must preserve memories unless `--purge` is supplied.