memorykit 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
memorykit/provider.py ADDED
@@ -0,0 +1,2908 @@
1
+ #!/usr/bin/env python3
2
+ """Validate durable memories and invoke an optional MemPalace provider."""
3
+
4
+ # Governing decisions. adrkit scans only the first 8192 bytes of a file for
5
+ # markers, and this module is far larger, so file-level declarations belong
6
+ # here in the header rather than beside the code they explain.
7
+ # @adr 0002
8
+ # @adr 0003
9
+ # @adr 0007
10
+
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import contextlib
15
+ import hashlib
16
+ import json
17
+ import os
18
+ import re
19
+ import shutil
20
+ import subprocess
21
+ import sys
22
+ import tempfile
23
+ import time
24
+ import uuid
25
+ from dataclasses import dataclass
26
+ from datetime import datetime, timezone
27
+ from pathlib import Path
28
+
29
+ SCHEMA = "context-kit/memory-v1"
30
+ MAX_BYTES = 32 * 1024
31
+ MAX_MEMORY_LINES = 220
32
+ MAX_HANDOFF_LINES = 300
33
+ MAX_HANDOFF_ITEMS = 25
34
+ MAX_CUES = 3
35
+ MAX_STATE_REASON_CHARS = 1000
36
+ PROVIDER_BACKUP_RETENTION = 1
37
+ PROJECT_SLUG_PREFIX_LENGTH = 31
38
+ ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,95}$")
39
+ HASH_RE = re.compile(r"^[0-9a-f]{64}$")
40
+ COMMIT_RE = re.compile(r"^[0-9a-fA-F]{7,64}$")
41
+ REPOSITORY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$")
42
+ FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$")
43
+ PLACEHOLDER_RE = re.compile(r"\{\{[^{}\n]+\}\}")
44
+ LIST_ITEM_RE = re.compile(r"^(?:[-*+] |\d+[.)] )")
45
+ TOKEN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*")
46
+ MEMORY_TYPES = {"fact", "decision", "procedure", "constraint", "episode"}
47
+ SCOPES = {"project"}
48
+ FRESHNESS_STATES = {"current", "stale", "superseded", "revoked"}
49
+ REVIEW_STATES = {"proposed", "accepted", "rejected"}
50
+ STATE_SCHEMA = "context-kit/memory-state-v1"
51
+ RECEIPT_SCHEMA = "context-kit/memory-provider-receipt-v1"
52
+ CANDIDATE_SCHEMA = "context-kit/memory-candidate-v1"
53
+ WAKE_SCHEMA = "context-kit/memory-wake-v1"
54
+ # Session mining recognizes GitHub Copilot CLI event logs
55
+ # (`~/.copilot/session-state/<session-id>/events.jsonl`).
56
+ SESSION_PRODUCER = "github-copilot-cli"
57
+ # `session_id` arrives from the session log and is used to name a candidate
58
+ # file, so it is validated as a single safe path component before use.
59
+ SESSION_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
60
+ MAX_TURN_CHARS = 2000
61
+ MAX_CANDIDATE_TURNS = 400
62
+ # A wake digest primes a session, so it competes with real work for context.
63
+ # MemPalace targets 600-900 tokens for the same job; this is a comparable
64
+ # character budget with a hard record cap so one verbose store cannot flood it.
65
+ MAX_WAKE_RECORDS = 12
66
+ MAX_WAKE_CHARS = 2400
67
+ # Detection only. Each pattern names a high-signal credential shape so a
68
+ # finding can be reported precisely and, with --redact, masked in place.
69
+ SECRET_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
70
+ ("aws-access-key-id", re.compile(r"AKIA[0-9A-Z]{16}")),
71
+ ("github-token", re.compile(r"gh[pousr]_[A-Za-z0-9]{36,}")),
72
+ ("slack-token", re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}")),
73
+ ("private-key-block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
74
+ (
75
+ "json-web-token",
76
+ re.compile(r"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"),
77
+ ),
78
+ (
79
+ "assigned-credential",
80
+ re.compile(
81
+ r"(?i)\b(?:api[_-]?key|secret|password|passwd|access[_-]?token|bearer)\b"
82
+ r"\s*[:=]\s*[\"']?[A-Za-z0-9/+_.-]{16,}"
83
+ ),
84
+ ),
85
+ )
86
+ PROJECTION_MARKER_SCHEMA = "context-kit/memory-provider-projection-v1"
87
+ PROJECTION_MARKER_NAME = ".context-kit-projection.json"
88
+ STATE_SEQUENCE_WIDTH = 20
89
+ STATE_LOCK_TIMEOUT_SECONDS = 5.0
90
+ STATE_LOCK_STALE_SECONDS = 300.0
91
+ STATE_SEQUENCE_RE = re.compile(r"^(\d{20})-[0-9a-f]{32}\.json$")
92
+ REVIEW_TRANSITIONS = {
93
+ "proposed": {"accepted", "rejected"},
94
+ "accepted": {"rejected"},
95
+ "rejected": {"accepted"},
96
+ }
97
+ FRESHNESS_TRANSITIONS = {
98
+ "current": {"stale", "superseded", "revoked"},
99
+ "stale": {"current", "superseded", "revoked"},
100
+ "superseded": set(),
101
+ "revoked": set(),
102
+ }
103
+ # The adapter is verified against this MemPalace release; see
104
+ # skills/memory-workflows/references/provider-mempalace.md for the
105
+ # compatibility matrix and how `doctor` reports drift.
106
+ MEMPALACE_TESTED_VERSION = (3, 6, 0)
107
+ MEMPALACE_TESTED_RELEASE_LINE = "3.6.x"
108
+ # The first-party `rag` provider is this repository's own `indexkit` plugin.
109
+ # `CONTEXT_KIT_INDEXKIT_HOME` (indexkit >= 0.4.0) separates venv resolution
110
+ # from index-data location, which is what lets this adapter redirect index data
111
+ # into a project-isolated store without relocating the shared venv.
112
+ RAG_TESTED_VERSION = (0, 6, 1)
113
+ RAG_TESTED_RELEASE_LINE = "0.6.x"
114
+ RAG_INDEX_NAME = "memory"
115
+ # One record spans several indexed chunks, so ask for more chunks than the
116
+ # caller asked for records and deduplicate back down.
117
+ RAG_CHUNKS_PER_RECORD = 4
118
+ SEMVER_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)")
119
+ REQUIRED_FIELDS = (
120
+ "schema",
121
+ "id",
122
+ "type",
123
+ "scope",
124
+ "repository",
125
+ "branch",
126
+ "head",
127
+ "observed_at",
128
+ "captured_at",
129
+ "freshness",
130
+ "review",
131
+ "source",
132
+ "source_hash",
133
+ )
134
+ MEMORY_HEADINGS = (
135
+ "## Primary Memory",
136
+ "## Cue Anchors",
137
+ "## Evidence",
138
+ "## Supersedes",
139
+ "## Review Notes",
140
+ )
141
+ HANDOFF_FIELDS = (
142
+ "schema",
143
+ "generated_at",
144
+ "repository",
145
+ "worktree",
146
+ "branch",
147
+ "head",
148
+ "base_ref",
149
+ "base_commit",
150
+ "worktree_state",
151
+ )
152
+ HANDOFF_HEADINGS = (
153
+ "## Scope",
154
+ "## Verified Facts",
155
+ "## Decisions",
156
+ "## Changed Files",
157
+ "## Completed Work",
158
+ "## Unresolved Items",
159
+ "## Next Steps",
160
+ "## Validation State",
161
+ "## Provenance and Freshness",
162
+ )
163
+
164
+
165
+ class Refusal(ValueError):
166
+ """An invalid input or unsafe provider request."""
167
+
168
+
169
+ @dataclass(frozen=True)
170
+ class CapabilityProbe:
171
+ """One exact-argv help surface the adapter depends on."""
172
+
173
+ name: str
174
+ argv: tuple[str, ...]
175
+ contract: str
176
+ required_tokens: tuple[str, ...] = ()
177
+
178
+
179
+ @dataclass(frozen=True)
180
+ class ProviderSpec:
181
+ """Everything that differs between external providers.
182
+
183
+ Keeping the differences declarative means `sync-provider`, `search`, and
184
+ `doctor` share one projection, staging, swap, marker, and receipt path
185
+ regardless of which provider is configured.
186
+ """
187
+
188
+ name: str
189
+ # Live store directory under providers/<name>/<project-key>/. This is the
190
+ # unit that is atomically swapped, so it must be a directory the adapter
191
+ # owns exclusively.
192
+ store_dirname: str
193
+ backup_prefix: str
194
+ bin_env: str
195
+ executable: str
196
+ install_hint: str
197
+ tested_version: tuple[int, int, int]
198
+ tested_release_line: str
199
+ capabilities: tuple[CapabilityProbe, ...]
200
+ # Pre-rename names still honored so an install predating a rename keeps
201
+ # resolving. Declarative for the same reason the rest of this class is:
202
+ # the migration is provider data, not branching in the resolver.
203
+ legacy_bin_env: str | None = None
204
+ legacy_executables: tuple[str, ...] = ()
205
+
206
+ def index_argv(self, projection: Path, project_key: str) -> list[str]:
207
+ raise NotImplementedError
208
+
209
+ def search_argv(self, query: str, results: int) -> list[str]:
210
+ raise NotImplementedError
211
+
212
+ def store_env(self, store: Path) -> dict[str, str]:
213
+ """Environment that points the provider at `store` for this call only."""
214
+ raise NotImplementedError
215
+
216
+
217
+ @dataclass(frozen=True)
218
+ class MemPalaceSpec(ProviderSpec):
219
+ def index_argv(self, projection: Path, project_key: str) -> list[str]:
220
+ return ["mine", str(projection), "--wing", project_key]
221
+
222
+ def search_argv(self, query: str, results: int) -> list[str]:
223
+ return ["search", query, "--results", str(results)]
224
+
225
+ def store_env(self, store: Path) -> dict[str, str]:
226
+ return {"MEMPALACE_PALACE_PATH": str(store)}
227
+
228
+
229
+ @dataclass(frozen=True)
230
+ class RagSpec(ProviderSpec):
231
+ def index_argv(self, projection: Path, project_key: str) -> list[str]:
232
+ # The store is already project-isolated by `store_env`, so a constant
233
+ # index name keeps the on-disk layout readable.
234
+ return ["index", str(projection), "--name", RAG_INDEX_NAME]
235
+
236
+ def search_argv(self, query: str, results: int) -> list[str]:
237
+ return ["query", query, "--name", RAG_INDEX_NAME, "--k", str(results), "--json"]
238
+
239
+ def store_env(self, store: Path) -> dict[str, str]:
240
+ # CONTEXT_KIT_DATA relocates *index data* into the isolated store.
241
+ # The home variable pins the venv to its normal location so redirecting
242
+ # data does not make the launcher look for a venv that only exists in
243
+ # the shared indexkit home.
244
+ #
245
+ # Both names are exported. `_bundled_executable` can legitimately
246
+ # resolve a pre-rename `bin/rag` launcher, and that launcher reads only
247
+ # `CONTEXT_KIT_LOCAL_RAG_HOME`. Setting just the new name would leave it
248
+ # falling back to CONTEXT_KIT_DATA — the isolated store — where no venv
249
+ # exists, so every provider call through an old sibling would fail.
250
+ home = str(_indexkit_home())
251
+ return {
252
+ "CONTEXT_KIT_DATA": str(store),
253
+ "CONTEXT_KIT_INDEXKIT_HOME": home,
254
+ "CONTEXT_KIT_LOCAL_RAG_HOME": home,
255
+ }
256
+
257
+
258
+ PROVIDER_SPECS: dict[str, ProviderSpec] = {
259
+ "mempalace": MemPalaceSpec(
260
+ name="mempalace",
261
+ store_dirname="palace",
262
+ backup_prefix="palace-backup-",
263
+ bin_env="CONTEXT_KIT_MEMPALACE_BIN",
264
+ executable="mempalace",
265
+ install_hint="install it separately with `uv tool install mempalace`",
266
+ tested_version=MEMPALACE_TESTED_VERSION,
267
+ tested_release_line=MEMPALACE_TESTED_RELEASE_LINE,
268
+ # Each probe mirrors an exact argv the adapter actually invokes.
269
+ # `wake-up` is deliberately absent: `wake` is built from local
270
+ # records for every provider, so requiring it would refuse an
271
+ # install over a command this adapter never calls.
272
+ # Probing `--help` (never the mutating command itself) lets `doctor`
273
+ # catch upstream CLI drift without importing provider internals or
274
+ # writing to a store.
275
+ capabilities=(
276
+ CapabilityProbe(
277
+ name="capture",
278
+ argv=("mine", "--help"),
279
+ contract="mine <dir> --wing <project-key>",
280
+ required_tokens=("--wing",),
281
+ ),
282
+ CapabilityProbe(
283
+ name="search",
284
+ argv=("search", "--help"),
285
+ contract="search <query> --results <n>",
286
+ required_tokens=("--results",),
287
+ ),
288
+ ),
289
+ ),
290
+ "rag": RagSpec(
291
+ name="rag",
292
+ store_dirname="store",
293
+ backup_prefix="store-backup-",
294
+ bin_env="CONTEXT_KIT_INDEXKIT_BIN",
295
+ executable="indexkit",
296
+ legacy_bin_env="CONTEXT_KIT_RAG_BIN",
297
+ legacy_executables=("rag",),
298
+ install_hint=(
299
+ "install the context-kit `indexkit` plugin and run "
300
+ "`bash plugins/indexkit/scripts/bootstrap.sh`"
301
+ ),
302
+ tested_version=RAG_TESTED_VERSION,
303
+ tested_release_line=RAG_TESTED_RELEASE_LINE,
304
+ capabilities=(
305
+ CapabilityProbe(
306
+ name="capture",
307
+ argv=("index", "--help"),
308
+ contract="index <dir> --name <index>",
309
+ required_tokens=("--name",),
310
+ ),
311
+ CapabilityProbe(
312
+ name="search",
313
+ argv=("query", "--help"),
314
+ contract="query <text> --name <index> --k <n> --json",
315
+ required_tokens=("--name", "--k", "--json"),
316
+ ),
317
+ ),
318
+ ),
319
+ }
320
+ PROVIDERS = ("none", *sorted(PROVIDER_SPECS))
321
+
322
+
323
+ @dataclass(frozen=True)
324
+ class Config:
325
+ provider: str
326
+ home: Path
327
+ project: str | None
328
+ auto_capture: bool
329
+ recall_on_start: bool = False
330
+
331
+ @property
332
+ def project_slug(self) -> str:
333
+ if not self.project:
334
+ raise Refusal(
335
+ "set CONTEXT_KIT_MEMORY_PROJECT (or pass --project) to isolate memory"
336
+ )
337
+ if not REPOSITORY_RE.fullmatch(self.project):
338
+ raise Refusal("memory project must be a concrete owner/name identity")
339
+ prefix = re.sub(r"[^A-Za-z0-9._-]+", "-", self.project).strip("-").lower()
340
+ digest = hashlib.sha256(self.project.encode("utf-8")).hexdigest()
341
+ return f"{prefix[:PROJECT_SLUG_PREFIX_LENGTH]}-{digest}"
342
+
343
+ @property
344
+ def spec(self) -> ProviderSpec:
345
+ try:
346
+ return PROVIDER_SPECS[self.provider]
347
+ except KeyError:
348
+ raise Refusal(
349
+ f"operation requires an external provider; configured: {self.provider}"
350
+ ) from None
351
+
352
+ @property
353
+ def provider_root(self) -> Path:
354
+ return self.home / "providers" / self.spec.name / self.project_slug
355
+
356
+ @property
357
+ def store_path(self) -> Path:
358
+ """The project-isolated directory swapped atomically on reconciliation."""
359
+ return self.provider_root / self.spec.store_dirname
360
+
361
+ @property
362
+ def palace_path(self) -> Path:
363
+ # Retained name for the MemPalace layout; `store_path` is the
364
+ # provider-neutral accessor used by the shared reconciliation path.
365
+ return self.home / "providers" / "mempalace" / self.project_slug / "palace"
366
+
367
+ @property
368
+ def records_path(self) -> Path:
369
+ return self.home / "records" / self.project_slug
370
+
371
+ @property
372
+ def states_path(self) -> Path:
373
+ return self.home / "states" / self.project_slug
374
+
375
+ @property
376
+ def receipts_path(self) -> Path:
377
+ return self.home / "receipts" / self.project_slug
378
+
379
+ @property
380
+ def candidates_path(self) -> Path:
381
+ """Reviewable session extractions. Never active memory."""
382
+ return self.home / "candidates" / self.project_slug
383
+
384
+
385
+ def _first_env(*names: str) -> str | None:
386
+ for name in names:
387
+ value = os.environ.get(name)
388
+ if value:
389
+ return value
390
+ return None
391
+
392
+
393
+ def _truthy(value: str | None) -> bool:
394
+ return bool(value and value.strip().lower() in {"1", "true", "yes", "on"})
395
+
396
+
397
+ def _indexkit_home() -> Path:
398
+ """Where indexkit keeps its bootstrapped venv.
399
+
400
+ Resolution is deliberately not a naive read of `CLAUDE_PLUGIN_DATA`: that
401
+ variable is *plugin-scoped*, so inside this plugin it points at memory's
402
+ own data directory. Both Claude Code and GitHub Copilot CLI lay plugin data
403
+ out as `<root>/<plugin>`, and `memory` hard-depends on `indexkit`, so the
404
+ dependency's home is a **sibling** of ours. Verified on a real Copilot
405
+ install: `~/.copilot/plugin-data/context-kit/{memory,indexkit}`.
406
+
407
+ The sibling is used only when it actually exists, so a wrong guess degrades
408
+ to the documented default rather than pointing at an empty directory.
409
+
410
+ Read from the *ambient* environment, before this adapter redirects
411
+ `CONTEXT_KIT_DATA` at the provider store.
412
+
413
+ The engine was renamed from `local-rag` to `indexkit` (ADR-0007). A host
414
+ that installed the plugin before the rename still has a `local-rag` data
415
+ directory, and a user may still export `CONTEXT_KIT_LOCAL_RAG_HOME`, so both
416
+ names are accepted with the current one preferred.
417
+ """
418
+ configured = _first_env(
419
+ "CONTEXT_KIT_INDEXKIT_HOME",
420
+ "CONTEXT_KIT_LOCAL_RAG_HOME",
421
+ "CONTEXT_KIT_DATA",
422
+ "PRODUCTIVITY_SKILLS_DATA",
423
+ )
424
+ if configured:
425
+ return Path(configured).expanduser()
426
+ plugin_data = _first_env("CLAUDE_PLUGIN_DATA")
427
+ if plugin_data:
428
+ root = Path(plugin_data).expanduser().parent
429
+ for name in ("indexkit", "local-rag"):
430
+ sibling = root / name
431
+ if (sibling / "venv").is_dir() or (sibling / "pyproject.sha").is_file():
432
+ return sibling
433
+ default_root = Path.home() / ".claude/plugins/data"
434
+ legacy = default_root / "local-rag"
435
+ if not (default_root / "indexkit" / "venv").is_dir() and (legacy / "venv").is_dir():
436
+ return legacy
437
+ return default_root / "indexkit"
438
+
439
+
440
+ def _config(args: argparse.Namespace) -> Config:
441
+ provider = (
442
+ getattr(args, "provider", None)
443
+ or _first_env(
444
+ "CONTEXT_KIT_MEMORY_PROVIDER",
445
+ "CLAUDE_PLUGIN_OPTION_PROVIDER",
446
+ )
447
+ or "none"
448
+ ).lower()
449
+ if provider not in PROVIDERS:
450
+ raise Refusal("memory provider must be one of: " + ", ".join(PROVIDERS))
451
+
452
+ home_value = (
453
+ getattr(args, "home", None)
454
+ or _first_env(
455
+ "CONTEXT_KIT_MEMORY_HOME",
456
+ "CLAUDE_PLUGIN_OPTION_MEMORY_HOME",
457
+ )
458
+ or "~/.local/share/context-kit/memory"
459
+ )
460
+ home = Path(home_value).expanduser().resolve()
461
+ project = getattr(args, "project", None) or _first_env(
462
+ "CONTEXT_KIT_MEMORY_PROJECT",
463
+ "CLAUDE_PLUGIN_OPTION_PROJECT",
464
+ )
465
+ auto_value = _first_env(
466
+ "CONTEXT_KIT_MEMORY_AUTO_CAPTURE",
467
+ "CLAUDE_PLUGIN_OPTION_AUTO_CAPTURE",
468
+ )
469
+ recall_value = _first_env(
470
+ "CONTEXT_KIT_MEMORY_RECALL_ON_START",
471
+ "CLAUDE_PLUGIN_OPTION_RECALL_ON_START",
472
+ )
473
+ return Config(
474
+ provider=provider,
475
+ home=home,
476
+ project=project,
477
+ auto_capture=_truthy(auto_value),
478
+ recall_on_start=_truthy(recall_value),
479
+ )
480
+
481
+
482
+ def _parse_frontmatter(text: str) -> tuple[dict[str, str], list[str]]:
483
+ lines = text.splitlines()
484
+ if not lines or lines[0].strip() != "---":
485
+ raise Refusal("artifact must start with flat YAML frontmatter")
486
+ fields: dict[str, str] = {}
487
+ closing = None
488
+ for index, line in enumerate(lines[1:], start=1):
489
+ if line.strip() == "---":
490
+ closing = index
491
+ break
492
+ if not line or line[0].isspace() or ":" not in line:
493
+ raise Refusal(
494
+ "frontmatter must contain only flat non-empty key/value fields"
495
+ )
496
+ key, value = line.split(":", 1)
497
+ key = key.strip()
498
+ value = value.strip()
499
+ if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
500
+ value = value[1:-1]
501
+ if not key or not value or key in fields:
502
+ raise Refusal("frontmatter contains an empty or duplicate field")
503
+ fields[key] = value
504
+ if closing is None:
505
+ raise Refusal("frontmatter is missing its closing delimiter")
506
+ return fields, lines[closing + 1 :]
507
+
508
+
509
+ def _validate_timestamp(value: str, field: str) -> None:
510
+ try:
511
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
512
+ except ValueError as exc:
513
+ raise Refusal(f"{field} must be an ISO 8601 timestamp") from exc
514
+ if parsed.tzinfo is None or parsed.utcoffset() is None:
515
+ raise Refusal(f"{field} must include a timezone")
516
+
517
+
518
+ def _build_sections(
519
+ body: list[str],
520
+ headings: tuple[str, ...],
521
+ found: list[tuple[str, int]],
522
+ ) -> dict[str, list[str]]:
523
+ found_headings = [heading for heading, _position in found]
524
+ if found_headings != list(headings):
525
+ raise Refusal("artifact headings are missing, reordered, or unexpected")
526
+ positions = [position for _heading, position in found]
527
+ result: dict[str, list[str]] = {}
528
+ for index, heading in enumerate(headings):
529
+ end = positions[index + 1] if index + 1 < len(positions) else len(body)
530
+ result[heading] = body[positions[index] + 1 : end]
531
+ return result
532
+
533
+
534
+ def _memory_sections(body: list[str]) -> dict[str, list[str]]:
535
+ found: list[tuple[str, int]] = []
536
+ fence: tuple[str, int] | None = None
537
+ for index, line in enumerate(body):
538
+ fence_match = FENCE_RE.match(line)
539
+ if fence is not None:
540
+ if fence_match:
541
+ marker, suffix = fence_match.groups()
542
+ if (
543
+ marker[0] == fence[0]
544
+ and len(marker) >= fence[1]
545
+ and not suffix.strip()
546
+ ):
547
+ fence = None
548
+ continue
549
+ if fence_match:
550
+ marker, _suffix = fence_match.groups()
551
+ fence = (marker[0], len(marker))
552
+ continue
553
+ if line.startswith("## "):
554
+ found.append((line, index))
555
+ return _build_sections(body, MEMORY_HEADINGS, found)
556
+
557
+
558
+ def _handoff_sections(body: list[str]) -> dict[str, list[str]]:
559
+ found = []
560
+ for index, line in enumerate(body):
561
+ if line.startswith("## "):
562
+ found.append((f"## {line[3:].strip()}", index))
563
+ return _build_sections(body, HANDOFF_HEADINGS, found)
564
+
565
+
566
+ def _nonempty_section(lines: list[str], heading: str) -> str:
567
+ text = "\n".join(lines).strip()
568
+ if not text or text == "- None.":
569
+ raise Refusal(f"{heading} must not be empty")
570
+ return text
571
+
572
+
573
+ def _required_section(lines: list[str], heading: str) -> str:
574
+ text = "\n".join(lines).strip()
575
+ if not text:
576
+ raise Refusal(f"{heading} must not be empty; use '- None.'")
577
+ return text
578
+
579
+
580
+ def _validate_branch(value: str) -> None:
581
+ try:
582
+ result = subprocess.run(
583
+ ["git", "check-ref-format", "--branch", value],
584
+ capture_output=True,
585
+ check=False,
586
+ text=True,
587
+ timeout=5.0,
588
+ )
589
+ except FileNotFoundError as exc:
590
+ raise Refusal("git is required to validate branch provenance") from exc
591
+ except subprocess.TimeoutExpired as exc:
592
+ raise Refusal("Git branch validation timed out") from exc
593
+ if result.returncode != 0:
594
+ raise Refusal("branch must be a valid concrete Git branch name")
595
+
596
+
597
+ def _read_bounded(
598
+ path: Path, *, max_lines: int = MAX_MEMORY_LINES
599
+ ) -> tuple[bytes, str]:
600
+ if not path.is_file():
601
+ raise Refusal(f"artifact is not a file: {path}")
602
+ raw = path.read_bytes()
603
+ if len(raw) > MAX_BYTES:
604
+ raise Refusal(f"artifact exceeds {MAX_BYTES} bytes")
605
+ try:
606
+ text = raw.decode("utf-8")
607
+ except UnicodeDecodeError as exc:
608
+ raise Refusal("artifact must be UTF-8") from exc
609
+ if len(text.splitlines()) > max_lines:
610
+ raise Refusal(f"artifact exceeds {max_lines} lines")
611
+ return raw, text
612
+
613
+
614
+ def validate_memory(path: Path, *, verify_source: bool = True) -> dict[str, object]:
615
+ raw, text = _read_bounded(path)
616
+ fields, body = _parse_frontmatter(text)
617
+ missing = [field for field in REQUIRED_FIELDS if field not in fields]
618
+ extras = sorted(set(fields) - set(REQUIRED_FIELDS))
619
+ if missing or extras:
620
+ raise Refusal(
621
+ f"memory frontmatter mismatch; missing={missing or 'none'} "
622
+ f"unexpected={extras or 'none'}"
623
+ )
624
+ if fields["schema"] != SCHEMA:
625
+ raise Refusal(f"schema must be {SCHEMA}")
626
+ if not ID_RE.fullmatch(fields["id"]):
627
+ raise Refusal("memory id must be lowercase and use letters, numbers, ._-")
628
+ if fields["type"] not in MEMORY_TYPES:
629
+ raise Refusal(f"memory type must be one of {sorted(MEMORY_TYPES)}")
630
+ if fields["scope"] not in SCOPES:
631
+ raise Refusal("memory scope must be 'project'")
632
+ if fields["freshness"] not in FRESHNESS_STATES:
633
+ raise Refusal(f"freshness must be one of {sorted(FRESHNESS_STATES)}")
634
+ if fields["review"] not in REVIEW_STATES:
635
+ raise Refusal(f"review must be one of {sorted(REVIEW_STATES)}")
636
+ if not HASH_RE.fullmatch(fields["source_hash"]):
637
+ raise Refusal("source_hash must be a lowercase SHA-256 digest")
638
+ _validate_timestamp(fields["observed_at"], "observed_at")
639
+ _validate_timestamp(fields["captured_at"], "captured_at")
640
+ if not REPOSITORY_RE.fullmatch(fields["repository"]):
641
+ raise Refusal("repository must be a concrete owner/name identity")
642
+ _validate_branch(fields["branch"])
643
+ if not COMMIT_RE.fullmatch(fields["head"]):
644
+ raise Refusal("head must be a 7-64 character hexadecimal commit")
645
+
646
+ sections = _memory_sections(body)
647
+ primary = _nonempty_section(sections["## Primary Memory"], "Primary Memory")
648
+ if len(primary) > 600:
649
+ raise Refusal("Primary Memory must be at most 600 characters")
650
+ _nonempty_section(sections["## Evidence"], "Evidence")
651
+
652
+ cue_lines = [line.strip() for line in sections["## Cue Anchors"] if line.strip()]
653
+ if cue_lines == ["- None."]:
654
+ cues: list[str] = []
655
+ else:
656
+ if not cue_lines or any(
657
+ not line.startswith("- ") or line == "- None." for line in cue_lines
658
+ ):
659
+ raise Refusal("Cue Anchors must contain only bullets or '- None.'")
660
+ cues = [line[2:].strip() for line in cue_lines]
661
+ if len(cues) > MAX_CUES:
662
+ raise Refusal(f"Cue Anchors may contain at most {MAX_CUES} entries")
663
+ if any(not cue or len(cue) > 120 for cue in cues):
664
+ raise Refusal("each cue anchor must contain 1..120 characters")
665
+ _required_section(sections["## Supersedes"], "Supersedes")
666
+ _nonempty_section(sections["## Review Notes"], "Review Notes")
667
+
668
+ source = Path(fields["source"]).expanduser()
669
+ if verify_source and source.is_file():
670
+ actual = hashlib.sha256(source.read_bytes()).hexdigest()
671
+ if actual != fields["source_hash"]:
672
+ raise Refusal("source_hash does not match the referenced source file")
673
+ return {
674
+ **fields,
675
+ "artifact_hash": hashlib.sha256(raw).hexdigest(),
676
+ "primary_memory": primary,
677
+ "cue_anchors": cues,
678
+ }
679
+
680
+
681
+ def validate_handoff(path: Path) -> dict[str, object]:
682
+ raw, text = _read_bounded(path, max_lines=MAX_HANDOFF_LINES)
683
+ if PLACEHOLDER_RE.search(text):
684
+ raise Refusal("handoff contains unresolved {{...}} template placeholders")
685
+ fields, body = _parse_frontmatter(text)
686
+ missing = [field for field in HANDOFF_FIELDS if field not in fields]
687
+ if missing:
688
+ raise Refusal(f"handoff is missing required fields: {', '.join(missing)}")
689
+ if fields["schema"] != "context-kit/handoff-v1":
690
+ raise Refusal("handoff schema must be context-kit/handoff-v1")
691
+ _validate_timestamp(fields["generated_at"], "generated_at")
692
+ for field in ("head", "base_commit"):
693
+ if not COMMIT_RE.fullmatch(fields[field]):
694
+ raise Refusal(f"{field} must be a 7-64 character hexadecimal commit")
695
+ if fields["worktree_state"] not in {"clean", "dirty"}:
696
+ raise Refusal("handoff worktree_state must be clean or dirty")
697
+ titles = [line.strip() for line in body if line.startswith("# ")]
698
+ if titles != ["# Context Handoff"]:
699
+ raise Refusal("handoff must contain exactly one '# Context Handoff' title")
700
+ sections = _handoff_sections(body)
701
+ for heading, lines in sections.items():
702
+ _required_section(lines, heading.removeprefix("## "))
703
+ item_count = sum(
704
+ bool(LIST_ITEM_RE.match(line)) for line in lines if line.strip()
705
+ )
706
+ if item_count > MAX_HANDOFF_ITEMS:
707
+ raise Refusal(
708
+ f"{heading.removeprefix('## ')} has {item_count} list items; "
709
+ f"maximum is {MAX_HANDOFF_ITEMS}"
710
+ )
711
+ return {**fields, "artifact_hash": hashlib.sha256(raw).hexdigest()}
712
+
713
+
714
+ def _git(repo: Path, *argv: str) -> str:
715
+ try:
716
+ result = subprocess.run(
717
+ ["git", "-C", str(repo), *argv],
718
+ capture_output=True,
719
+ check=False,
720
+ text=True,
721
+ timeout=20.0,
722
+ )
723
+ except subprocess.TimeoutExpired as exc:
724
+ raise Refusal("git context check timed out") from exc
725
+ if result.returncode != 0:
726
+ error = result.stderr.strip()
727
+ raise Refusal(f"cannot establish repository context: {error or 'git failed'}")
728
+ return result.stdout.strip()
729
+
730
+
731
+ def _normalize_repository(remote: str) -> str:
732
+ value = remote.strip()
733
+ if value.startswith("git@") and ":" in value:
734
+ value = value.split(":", 1)[1]
735
+ elif "://" in value:
736
+ value = value.split("://", 1)[1]
737
+ if "@" in value.split("/", 1)[0]:
738
+ value = value.split("@", 1)[1]
739
+ value = value.split("/", 1)[1] if "/" in value else value
740
+ value = value.rstrip("/")
741
+ if value.endswith(".git"):
742
+ value = value[:-4]
743
+ parts = [part for part in value.split("/") if part]
744
+ if len(parts) < 2:
745
+ raise Refusal("cannot normalize the repository remote to owner/name")
746
+ return "/".join(parts[-2:])
747
+
748
+
749
+ def _assert_project_matches(metadata: dict[str, object], config: Config) -> None:
750
+ project = config.project
751
+ if not project:
752
+ config.project_slug
753
+ if metadata["repository"] != project:
754
+ raise Refusal(
755
+ "artifact repository does not match configured memory project: "
756
+ f"artifact={metadata['repository']!r} project={project!r}"
757
+ )
758
+
759
+
760
+ def _assert_handoff_current(metadata: dict[str, object], repo: Path) -> None:
761
+ root = Path(_git(repo, "rev-parse", "--show-toplevel")).resolve()
762
+ remote = _normalize_repository(_git(root, "remote", "get-url", "origin"))
763
+ branch = _git(root, "branch", "--show-current")
764
+ head = _git(root, "rev-parse", "HEAD")
765
+ base_commit = _git(root, "merge-base", "HEAD", metadata["base_ref"])
766
+ worktree_state = "dirty" if _git(root, "status", "--porcelain") else "clean"
767
+ checks = {
768
+ "repository": remote,
769
+ "branch": branch,
770
+ "head": head,
771
+ "base_commit": base_commit,
772
+ "worktree_state": worktree_state,
773
+ }
774
+ differences = [
775
+ f"{field}: saved={metadata[field]!r} current={current!r}"
776
+ for field, current in checks.items()
777
+ if metadata[field] != current
778
+ ]
779
+ if differences:
780
+ raise Refusal(
781
+ "handoff is mismatched or stale; validate/resume it before archival: "
782
+ + "; ".join(differences)
783
+ )
784
+
785
+
786
+ def _write_once(destination: Path, raw: bytes) -> str:
787
+ destination.parent.mkdir(parents=True, exist_ok=True)
788
+ with tempfile.NamedTemporaryFile(
789
+ dir=destination.parent,
790
+ prefix=f".{destination.name}.",
791
+ delete=False,
792
+ ) as handle:
793
+ handle.write(raw)
794
+ temporary = Path(handle.name)
795
+ os.chmod(temporary, 0o600)
796
+ try:
797
+ os.link(temporary, destination)
798
+ except FileExistsError:
799
+ if destination.read_bytes() == raw:
800
+ return "unchanged"
801
+ raise Refusal(f"refusing to overwrite a different artifact: {destination}")
802
+ finally:
803
+ temporary.unlink(missing_ok=True)
804
+ return "created"
805
+
806
+
807
+ def _utc_timestamp() -> str:
808
+ return datetime.now(timezone.utc).isoformat(timespec="microseconds")
809
+
810
+
811
+ def _new_write_once_path(directory: Path, suffix: str) -> Path:
812
+ directory.mkdir(parents=True, exist_ok=True)
813
+ stamp = _utc_timestamp().replace("-", "").replace(":", "").replace("+", "p")
814
+ return directory / f"{stamp}-{os.getpid()}-{uuid.uuid4().hex}{suffix}"
815
+
816
+
817
+ def _write_json_once(directory: Path, payload: dict[str, object]) -> Path:
818
+ raw = (json.dumps(payload, sort_keys=True, indent=2) + "\n").encode("utf-8")
819
+ destination = _new_write_once_path(directory, ".json")
820
+ if _write_once(destination, raw) != "created":
821
+ raise Refusal(f"refusing to reuse a generated write-once path: {destination}")
822
+ return destination
823
+
824
+
825
+ def _initial_state(metadata: dict[str, object]) -> dict[str, str]:
826
+ return {
827
+ "review": str(metadata["review"]),
828
+ "freshness": str(metadata["freshness"]),
829
+ }
830
+
831
+
832
+ def _event_paths(config: Config, record_id: str) -> list[Path]:
833
+ directory = config.states_path / record_id
834
+ paths = list(directory.glob("*.json")) if directory.exists() else []
835
+ sequenced = [path for path in paths if STATE_SEQUENCE_RE.fullmatch(path.name)]
836
+ legacy = [path for path in paths if path not in sequenced]
837
+ if sequenced and legacy:
838
+ raise Refusal(
839
+ "mixed legacy and sequenced state events; migrate legacy events before "
840
+ f"recording more state for {record_id}"
841
+ )
842
+ if sequenced:
843
+ return sorted(
844
+ sequenced, key=lambda path: int(STATE_SEQUENCE_RE.fullmatch(path.name)[1])
845
+ )
846
+ return sorted(legacy)
847
+
848
+
849
+ @contextlib.contextmanager
850
+ def _state_lock(config: Config, record_id: str):
851
+ """Serialize state transitions without making the evidence artifact writable."""
852
+ parent = config.states_path / record_id
853
+ parent.mkdir(parents=True, exist_ok=True)
854
+ lock = parent / ".lock"
855
+ token = uuid.uuid4().hex
856
+ owner = lock / "owner.json"
857
+ deadline = time.monotonic() + STATE_LOCK_TIMEOUT_SECONDS
858
+ while True:
859
+ try:
860
+ lock.mkdir()
861
+ except FileExistsError:
862
+ if _reclaim_stale_lock(lock):
863
+ continue
864
+ if time.monotonic() >= deadline:
865
+ raise Refusal(f"state transition is busy for record: {record_id}")
866
+ time.sleep(0.02)
867
+ continue
868
+ try:
869
+ _write_once(
870
+ owner,
871
+ json.dumps(
872
+ {
873
+ "pid": os.getpid(),
874
+ "token": token,
875
+ "acquired_at": _utc_timestamp(),
876
+ },
877
+ sort_keys=True,
878
+ ).encode("utf-8"),
879
+ )
880
+ except (OSError, Refusal):
881
+ shutil.rmtree(lock, ignore_errors=True)
882
+ raise
883
+ break
884
+ try:
885
+ yield
886
+ finally:
887
+ try:
888
+ payload = json.loads(owner.read_text(encoding="utf-8"))
889
+ except (OSError, json.JSONDecodeError):
890
+ payload = {}
891
+ if payload.get("token") == token:
892
+ owner.unlink(missing_ok=True)
893
+ lock.rmdir()
894
+
895
+
896
+ def _reclaim_stale_lock(lock: Path) -> bool:
897
+ """Reclaim only a dead POSIX owner, or a conservatively old non-POSIX lock."""
898
+ owner = lock / "owner.json"
899
+ try:
900
+ payload = json.loads(owner.read_text(encoding="utf-8"))
901
+ except (OSError, json.JSONDecodeError):
902
+ return False
903
+ pid = payload.get("pid")
904
+ reclaim = False
905
+ if os.name == "posix" and isinstance(pid, int) and pid > 0:
906
+ try:
907
+ os.kill(pid, 0)
908
+ except ProcessLookupError:
909
+ reclaim = True
910
+ except PermissionError:
911
+ return False
912
+ elif os.name != "posix":
913
+ try:
914
+ reclaim = time.time() - lock.stat().st_mtime >= STATE_LOCK_STALE_SECONDS
915
+ except OSError:
916
+ return False
917
+ if not reclaim:
918
+ return False
919
+ retired = lock.with_name(f".lock-stale-{uuid.uuid4().hex}")
920
+ try:
921
+ os.replace(lock, retired)
922
+ except OSError:
923
+ return False
924
+ shutil.rmtree(retired, ignore_errors=True)
925
+ return True
926
+
927
+
928
+ def _validate_state_event(
929
+ path: Path,
930
+ *,
931
+ metadata: dict[str, object],
932
+ config: Config,
933
+ state: dict[str, str],
934
+ ) -> dict[str, object]:
935
+ try:
936
+ payload = json.loads(path.read_text(encoding="utf-8"))
937
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
938
+ raise Refusal(f"invalid state event {path}: {exc}") from exc
939
+ required = {
940
+ "schema",
941
+ "event_id",
942
+ "record_id",
943
+ "record_hash",
944
+ "project",
945
+ "project_key",
946
+ "timestamp",
947
+ "prior_review",
948
+ "prior_freshness",
949
+ "effective_review",
950
+ "effective_freshness",
951
+ "reason",
952
+ }
953
+ sequenced = STATE_SEQUENCE_RE.fullmatch(path.name)
954
+ expected = required | {"sequence"} if sequenced else required
955
+ if not isinstance(payload, dict) or set(payload) != expected:
956
+ raise Refusal(f"state event has an invalid schema: {path}")
957
+ if payload["schema"] != STATE_SCHEMA:
958
+ raise Refusal(f"state event has an unsupported schema: {path}")
959
+ if (
960
+ payload["record_id"] != metadata["id"]
961
+ or payload["record_hash"] != metadata["artifact_hash"]
962
+ ):
963
+ raise Refusal(f"state event does not bind the exact record: {path}")
964
+ if (
965
+ payload["project"] != config.project
966
+ or payload["project_key"] != config.project_slug
967
+ ):
968
+ raise Refusal(f"state event belongs to another project: {path}")
969
+ if not isinstance(payload["event_id"], str) or not payload["event_id"]:
970
+ raise Refusal(f"state event is missing an event_id: {path}")
971
+ if sequenced and (
972
+ not isinstance(payload["sequence"], int)
973
+ or payload["sequence"] != int(sequenced[1])
974
+ ):
975
+ raise Refusal(f"state event sequence does not match its filename: {path}")
976
+ if not isinstance(payload["reason"], str) or not payload["reason"].strip():
977
+ raise Refusal(f"state event is missing a reason: {path}")
978
+ if len(payload["reason"]) > MAX_STATE_REASON_CHARS:
979
+ raise Refusal(f"state event reason is too long: {path}")
980
+ for key in ("timestamp",):
981
+ if not isinstance(payload[key], str):
982
+ raise Refusal(f"state event has a non-string {key}: {path}")
983
+ _validate_timestamp(payload[key], key)
984
+ for key, allowed in (
985
+ ("prior_review", REVIEW_STATES),
986
+ ("effective_review", REVIEW_STATES),
987
+ ("prior_freshness", FRESHNESS_STATES),
988
+ ("effective_freshness", FRESHNESS_STATES),
989
+ ):
990
+ if not isinstance(payload[key], str) or payload[key] not in allowed:
991
+ raise Refusal(f"state event has invalid {key}: {path}")
992
+ if (
993
+ payload["prior_review"] != state["review"]
994
+ or payload["prior_freshness"] != state["freshness"]
995
+ ):
996
+ raise Refusal(f"state event prior state does not match its history: {path}")
997
+ _validate_transition(
998
+ state,
999
+ {
1000
+ "review": str(payload["effective_review"]),
1001
+ "freshness": str(payload["effective_freshness"]),
1002
+ },
1003
+ )
1004
+ return payload
1005
+
1006
+
1007
+ def _validate_transition(previous: dict[str, str], next_state: dict[str, str]) -> None:
1008
+ if previous == next_state:
1009
+ raise Refusal("state event must change review or freshness")
1010
+ if (
1011
+ previous["review"] != next_state["review"]
1012
+ and next_state["review"] not in REVIEW_TRANSITIONS[previous["review"]]
1013
+ ):
1014
+ raise Refusal(
1015
+ f"invalid review transition: {previous['review']} -> {next_state['review']}"
1016
+ )
1017
+ if (
1018
+ previous["freshness"] != next_state["freshness"]
1019
+ and next_state["freshness"] not in FRESHNESS_TRANSITIONS[previous["freshness"]]
1020
+ ):
1021
+ raise Refusal(
1022
+ "invalid freshness transition: "
1023
+ f"{previous['freshness']} -> {next_state['freshness']}"
1024
+ )
1025
+
1026
+
1027
+ def effective_state(metadata: dict[str, object], config: Config) -> dict[str, str]:
1028
+ """Resolve immutable initial frontmatter plus append-only state events."""
1029
+ state = _initial_state(metadata)
1030
+ for path in _event_paths(config, str(metadata["id"])):
1031
+ event = _validate_state_event(
1032
+ path, metadata=metadata, config=config, state=state
1033
+ )
1034
+ state = {
1035
+ "review": str(event["effective_review"]),
1036
+ "freshness": str(event["effective_freshness"]),
1037
+ }
1038
+ return state
1039
+
1040
+
1041
+ def _load_record(
1042
+ path: Path, config: Config
1043
+ ) -> tuple[dict[str, object], dict[str, str]]:
1044
+ metadata = validate_memory(path, verify_source=False)
1045
+ _assert_project_matches(metadata, config)
1046
+ return metadata, effective_state(metadata, config)
1047
+
1048
+
1049
+ def _is_active(state: dict[str, str]) -> bool:
1050
+ return state == {"review": "accepted", "freshness": "current"}
1051
+
1052
+
1053
+ def _source_state(metadata: dict[str, object]) -> str:
1054
+ source = Path(str(metadata["source"])).expanduser()
1055
+ if not source.is_file():
1056
+ return "unavailable"
1057
+ actual = hashlib.sha256(source.read_bytes()).hexdigest()
1058
+ return "verified" if actual == metadata["source_hash"] else "drifted"
1059
+
1060
+
1061
+ def _active_projection(
1062
+ config: Config,
1063
+ ) -> tuple[list[tuple[Path, dict[str, object]]], list[dict[str, str]]]:
1064
+ included: list[tuple[Path, dict[str, object]]] = []
1065
+ excluded: list[dict[str, str]] = []
1066
+ for path in (
1067
+ sorted(config.records_path.glob("*.md")) if config.records_path.exists() else []
1068
+ ):
1069
+ try:
1070
+ metadata, state = _load_record(path, config)
1071
+ if _is_active(state):
1072
+ included.append((path, metadata))
1073
+ else:
1074
+ excluded.append(
1075
+ {
1076
+ "id": str(metadata["id"]),
1077
+ "review": state["review"],
1078
+ "freshness": state["freshness"],
1079
+ }
1080
+ )
1081
+ except Refusal as exc:
1082
+ excluded.append({"artifact": str(path), "error": str(exc)})
1083
+ return included, excluded
1084
+
1085
+
1086
+ def _projection_hash(records: list[tuple[Path, dict[str, object]]]) -> str:
1087
+ digest = hashlib.sha256()
1088
+ for _, metadata in records:
1089
+ digest.update(str(metadata["id"]).encode("utf-8"))
1090
+ digest.update(b"\0")
1091
+ digest.update(str(metadata["artifact_hash"]).encode("ascii"))
1092
+ digest.update(b"\n")
1093
+ return digest.hexdigest()
1094
+
1095
+
1096
+ def _ledger_hash(config: Config) -> str:
1097
+ """Bind provider authority to all local record and state history, not its shape."""
1098
+ digest = hashlib.sha256()
1099
+ paths: list[Path] = []
1100
+ if config.records_path.exists():
1101
+ paths.extend(config.records_path.glob("*.md"))
1102
+ if config.states_path.exists():
1103
+ paths.extend(
1104
+ path
1105
+ for path in config.states_path.rglob("*.json")
1106
+ if ".lock" not in path.relative_to(config.states_path).parts
1107
+ )
1108
+ for path in sorted(paths, key=lambda item: str(item.relative_to(config.home))):
1109
+ digest.update(str(path.relative_to(config.home)).encode("utf-8"))
1110
+ digest.update(b"\0")
1111
+ digest.update(hashlib.sha256(path.read_bytes()).digest())
1112
+ digest.update(b"\n")
1113
+ return digest.hexdigest()
1114
+
1115
+
1116
+ def _materialize_projection(
1117
+ records: list[tuple[Path, dict[str, object]]], destination: Path
1118
+ ) -> None:
1119
+ destination.mkdir(parents=True, exist_ok=True)
1120
+ if any(destination.iterdir()):
1121
+ raise Refusal(
1122
+ f"refusing to reuse a non-empty projection directory: {destination}"
1123
+ )
1124
+ for path, metadata in records:
1125
+ target = destination / f"{metadata['id']}.md"
1126
+ shutil.copyfile(path, target)
1127
+ os.chmod(target, 0o600)
1128
+
1129
+
1130
+ def _provider_receipt(
1131
+ config: Config,
1132
+ *,
1133
+ provider_version: str,
1134
+ operation: str,
1135
+ artifact_hash: str | None,
1136
+ argv: list[str],
1137
+ outcome: str,
1138
+ detail: str,
1139
+ projection_hash: str | None = None,
1140
+ backup_path: Path | None = None,
1141
+ recovery_status: str = "not-needed",
1142
+ ) -> Path:
1143
+ payload: dict[str, object] = {
1144
+ "schema": RECEIPT_SCHEMA,
1145
+ "receipt_id": uuid.uuid4().hex,
1146
+ "timestamp": _utc_timestamp(),
1147
+ "provider": config.spec.name,
1148
+ "provider_version": provider_version,
1149
+ "project": config.project,
1150
+ "project_key": config.project_slug,
1151
+ "store_path": str(config.store_path),
1152
+ "operation": operation,
1153
+ "artifact_hash": artifact_hash,
1154
+ "projection_hash": projection_hash,
1155
+ "argv": argv,
1156
+ "outcome": outcome,
1157
+ "detail": detail,
1158
+ "backup_path": str(backup_path) if backup_path else None,
1159
+ "recovery_status": recovery_status,
1160
+ }
1161
+ if config.spec.name == "mempalace":
1162
+ # Retained for continuity with receipts written before the adapter
1163
+ # supported more than one provider.
1164
+ payload["palace_path"] = str(config.store_path)
1165
+ return _write_json_once(config.receipts_path, payload)
1166
+
1167
+
1168
+ def _provider_executable(spec: ProviderSpec) -> str:
1169
+ bin_envs = [spec.bin_env]
1170
+ if spec.legacy_bin_env:
1171
+ bin_envs.append(spec.legacy_bin_env)
1172
+ for bin_env in bin_envs:
1173
+ override = _first_env(bin_env)
1174
+ if not override:
1175
+ continue
1176
+ candidate = Path(override).expanduser()
1177
+ if not candidate.is_absolute():
1178
+ raise Refusal(f"{bin_env} must be an absolute path")
1179
+ if not candidate.is_file() or not os.access(candidate, os.X_OK):
1180
+ raise Refusal(
1181
+ f"configured {spec.name} executable is not runnable: {candidate}"
1182
+ )
1183
+ return str(candidate)
1184
+ for name in (spec.executable, *spec.legacy_executables):
1185
+ executable = shutil.which(name)
1186
+ if executable:
1187
+ return executable
1188
+ bundled = _bundled_executable(spec)
1189
+ if bundled is not None:
1190
+ return str(bundled)
1191
+ raise Refusal(
1192
+ f"{spec.name} provider selected but `{spec.executable}` is not installed; "
1193
+ f"{spec.install_hint}"
1194
+ )
1195
+
1196
+
1197
+ def _indexkit_root() -> Path | None:
1198
+ """The sibling indexkit plugin root, when deployed alongside `memory`.
1199
+
1200
+ `memory` hard-depends on `indexkit`, so both are installed together and
1201
+ the dependency's launcher and bootstrap are reachable without a PATH entry
1202
+ or extra configuration. `local-rag` is the pre-rename directory (ADR-0007).
1203
+
1204
+ The search walks ancestors rather than indexing a fixed depth because this
1205
+ module has two deployment shapes (ADR-0009): `plugins/memory/src/memorykit/`
1206
+ inside the catalog, and `site-packages/memorykit/` from a `pip install`. A
1207
+ hardcoded `parents[N]` is correct for at most one of them, and the failure
1208
+ is silent — a wrong guess resolves to a directory that simply has no
1209
+ bootstrap script, which is indistinguishable from "no sibling installed".
1210
+ Only a directory that actually contains the launcher counts, so a packaged
1211
+ install finds nothing and degrades to PATH resolution, which is the right
1212
+ answer there.
1213
+ """
1214
+ for ancestor in Path(__file__).resolve().parents[1:5]:
1215
+ for name in ("indexkit", "local-rag"):
1216
+ candidate = ancestor / name
1217
+ if (candidate / "scripts" / "bootstrap.sh").is_file():
1218
+ return candidate
1219
+ return None
1220
+
1221
+
1222
+ def _bundled_executable(spec: ProviderSpec) -> Path | None:
1223
+ """Resolve a sibling context-kit plugin launcher that is not on PATH."""
1224
+ if spec.name != "rag":
1225
+ return None
1226
+ root = _indexkit_root()
1227
+ if root is None:
1228
+ return None
1229
+ for name in (spec.executable, *spec.legacy_executables):
1230
+ candidate = root / "bin" / name
1231
+ if candidate.is_file() and os.access(candidate, os.X_OK):
1232
+ return candidate
1233
+ return None
1234
+
1235
+
1236
+ def _rag_runtime_status(*, bootstrap: bool = False) -> dict[str, object]:
1237
+ """Report whether the indexkit venv is usable, optionally building it.
1238
+
1239
+ Claude Code and GitHub Copilot CLI both bootstrap indexkit from its
1240
+ `SessionStart` hook, but APM does not deploy hooks and any host may have a
1241
+ stale or half-built venv, so readiness is checked explicitly here rather
1242
+ than surfacing later as an opaque provider failure. A venv built from
1243
+ different project metadata is reported as loudly as a missing one, because
1244
+ it otherwise runs stale code silently.
1245
+ """
1246
+ root = _indexkit_root()
1247
+ if root is None:
1248
+ return {
1249
+ "status": "unknown",
1250
+ "detail": "the indexkit plugin was not found next to memory; "
1251
+ "runtime readiness could not be checked",
1252
+ }
1253
+ script = root / "scripts" / "bootstrap.sh"
1254
+ env = os.environ.copy()
1255
+ # Both names, for the same reason as `RagSpec.store_env`: a pre-rename
1256
+ # sibling ships a bootstrap that reads only `CONTEXT_KIT_LOCAL_RAG_HOME`.
1257
+ home = str(_indexkit_home())
1258
+ env["CONTEXT_KIT_INDEXKIT_HOME"] = home
1259
+ env["CONTEXT_KIT_LOCAL_RAG_HOME"] = home
1260
+ if bootstrap:
1261
+ try:
1262
+ built = subprocess.run(
1263
+ ["bash", str(script)],
1264
+ capture_output=True,
1265
+ check=False,
1266
+ timeout=900.0,
1267
+ env=env,
1268
+ )
1269
+ except (subprocess.TimeoutExpired, OSError) as exc:
1270
+ raise Refusal(f"indexkit bootstrap could not run: {exc}") from exc
1271
+ if built.returncode != 0:
1272
+ detail = built.stderr.decode("utf-8", errors="replace").strip()
1273
+ raise Refusal(f"indexkit bootstrap failed: {detail or 'no error output'}")
1274
+ try:
1275
+ checked = subprocess.run(
1276
+ ["bash", str(script), "--check"],
1277
+ capture_output=True,
1278
+ check=False,
1279
+ timeout=60.0,
1280
+ env=env,
1281
+ )
1282
+ except (subprocess.TimeoutExpired, OSError) as exc:
1283
+ return {"status": "unknown", "detail": f"readiness check failed: {exc}"}
1284
+ report: dict[str, object] = {}
1285
+ for line in checked.stdout.decode("utf-8", errors="replace").splitlines():
1286
+ key, separator, value = line.partition("=")
1287
+ if separator:
1288
+ report[key.strip()] = value.strip()
1289
+ report.setdefault("status", "unknown")
1290
+ return report
1291
+
1292
+
1293
+ def _probe_capability(
1294
+ executable: str,
1295
+ config: Config,
1296
+ spec: ProviderSpec,
1297
+ probe: CapabilityProbe,
1298
+ *,
1299
+ timeout: float = 10.0,
1300
+ ) -> dict[str, object]:
1301
+ report: dict[str, object] = {
1302
+ "name": probe.name,
1303
+ "command": " ".join(probe.argv),
1304
+ "contract": probe.contract,
1305
+ }
1306
+ try:
1307
+ result = subprocess.run(
1308
+ [executable, *probe.argv],
1309
+ capture_output=True,
1310
+ check=False,
1311
+ timeout=timeout,
1312
+ env=_provider_env(config),
1313
+ )
1314
+ except subprocess.TimeoutExpired:
1315
+ report["status"] = "timeout"
1316
+ report["detail"] = f"probe timed out after {timeout:g}s"
1317
+ return report
1318
+ except OSError as exc:
1319
+ report["status"] = "error"
1320
+ report["detail"] = str(exc)
1321
+ return report
1322
+ output = "\n".join(
1323
+ (
1324
+ result.stdout.decode("utf-8", errors="replace"),
1325
+ result.stderr.decode("utf-8", errors="replace"),
1326
+ )
1327
+ )
1328
+ if result.returncode != 0:
1329
+ report["status"] = "missing"
1330
+ report["detail"] = f"`{' '.join(probe.argv)}` exited {result.returncode}"
1331
+ return report
1332
+ missing_tokens = [token for token in probe.required_tokens if token not in output]
1333
+ if missing_tokens:
1334
+ report["status"] = "incompatible"
1335
+ report["detail"] = f"missing option(s): {', '.join(missing_tokens)}"
1336
+ return report
1337
+ report["status"] = "ok"
1338
+ return report
1339
+
1340
+
1341
+ def _parse_provider_version(raw: str) -> tuple[int, int, int] | None:
1342
+ match = SEMVER_RE.search(raw)
1343
+ if not match:
1344
+ return None
1345
+ major, minor, patch = (int(part) for part in match.groups())
1346
+ return (major, minor, patch)
1347
+
1348
+
1349
+ def _provider_version_status(
1350
+ parsed: tuple[int, int, int] | None, spec: ProviderSpec
1351
+ ) -> str:
1352
+ if parsed is None:
1353
+ return "unknown"
1354
+ if parsed[:2] == spec.tested_version[:2]:
1355
+ return "tested"
1356
+ if parsed[:2] < spec.tested_version[:2]:
1357
+ return "older-than-tested"
1358
+ return "newer-than-tested"
1359
+
1360
+
1361
+ def _provider_env(config: Config, store_path: Path | None = None) -> dict[str, str]:
1362
+ env = os.environ.copy()
1363
+ path = store_path or config.store_path
1364
+ path.parent.mkdir(parents=True, exist_ok=True)
1365
+ env.update(config.spec.store_env(path))
1366
+ return env
1367
+
1368
+
1369
+ def _run_provider(
1370
+ config: Config,
1371
+ argv: list[str],
1372
+ *,
1373
+ input_bytes: bytes | None = None,
1374
+ timeout: float = 120.0,
1375
+ store_path: Path | None = None,
1376
+ ) -> subprocess.CompletedProcess[bytes]:
1377
+ spec = config.spec
1378
+ executable = _provider_executable(spec)
1379
+ try:
1380
+ result = subprocess.run(
1381
+ [executable, *argv],
1382
+ input=input_bytes,
1383
+ capture_output=True,
1384
+ check=False,
1385
+ timeout=timeout,
1386
+ env=_provider_env(config, store_path),
1387
+ )
1388
+ except subprocess.TimeoutExpired as exc:
1389
+ raise Refusal(f"{spec.name} command timed out after {timeout:g}s") from exc
1390
+ if result.returncode != 0:
1391
+ error = result.stderr.decode("utf-8", errors="replace").strip()
1392
+ raise Refusal(
1393
+ f"{spec.name} exited {result.returncode}: {error or 'no error output'}"
1394
+ )
1395
+ return result
1396
+
1397
+
1398
+ def _write_stdout(raw: bytes) -> None:
1399
+ sys.stdout.write(raw.decode("utf-8", errors="replace"))
1400
+
1401
+
1402
+ def _provider_version(config: Config) -> tuple[str, str]:
1403
+ executable = _provider_executable(config.spec)
1404
+ version = _run_provider(config, ["--version"], timeout=20.0)
1405
+ return executable, version.stdout.decode("utf-8", errors="replace").strip()
1406
+
1407
+
1408
+ def _capture_memory(args: argparse.Namespace, config: Config) -> int:
1409
+ source = Path(args.artifact).expanduser().resolve()
1410
+ metadata = validate_memory(source)
1411
+ _assert_project_matches(metadata, config)
1412
+ raw = source.read_bytes()
1413
+ destination = config.records_path / f"{metadata['id']}.md"
1414
+ state = _write_once(destination, raw)
1415
+ archived = False
1416
+ archive_reason = "provider not selected"
1417
+ archive_outcome = "not-selected"
1418
+ receipt: Path | None = None
1419
+ effective = effective_state(metadata, config)
1420
+ if config.provider != "none" and not args.local_only:
1421
+ if _is_active(effective):
1422
+ archive_reason = (
1423
+ "eligible for provider indexing but pending explicit "
1424
+ "sync-provider --apply"
1425
+ )
1426
+ archive_outcome = "pending-sync"
1427
+ receipt = _provider_receipt(
1428
+ config,
1429
+ operation="capture",
1430
+ provider_version="not-invoked",
1431
+ artifact_hash=str(metadata["artifact_hash"]),
1432
+ argv=[],
1433
+ outcome="pending-sync",
1434
+ detail=archive_reason,
1435
+ )
1436
+ else:
1437
+ archive_reason = (
1438
+ "not provider eligible; explicit sync will exclude this record "
1439
+ f"(review={effective['review']}, freshness={effective['freshness']})"
1440
+ )
1441
+ archive_outcome = "skipped"
1442
+ receipt = _provider_receipt(
1443
+ config,
1444
+ provider_version="not-invoked",
1445
+ operation="capture",
1446
+ artifact_hash=str(metadata["artifact_hash"]),
1447
+ argv=[],
1448
+ outcome="skipped",
1449
+ detail=archive_reason,
1450
+ )
1451
+ elif args.local_only:
1452
+ archive_reason = "skipped: --local-only"
1453
+ archive_outcome = "skipped"
1454
+ print(
1455
+ json.dumps(
1456
+ {
1457
+ "status": state,
1458
+ "artifact": str(destination),
1459
+ "project": config.project_slug,
1460
+ "provider": config.provider,
1461
+ "provider_archived": archived,
1462
+ "provider_archive": {
1463
+ "outcome": archive_outcome,
1464
+ "reason": archive_reason,
1465
+ "receipt": str(receipt) if receipt else None,
1466
+ },
1467
+ "provider_reconciliation": (
1468
+ "required: run sync-provider --apply"
1469
+ if config.provider != "none" and _is_active(effective)
1470
+ else "not-required"
1471
+ ),
1472
+ "effective_state": effective,
1473
+ }
1474
+ )
1475
+ )
1476
+ return 0
1477
+
1478
+
1479
+ def _archive_handoff(args: argparse.Namespace, config: Config) -> int:
1480
+ source = Path(args.artifact).expanduser().resolve()
1481
+ metadata = validate_handoff(source)
1482
+ _assert_project_matches(metadata, config)
1483
+ _assert_handoff_current(metadata, Path(args.repo).expanduser().resolve())
1484
+ raw = source.read_bytes()
1485
+ name = (
1486
+ f"handoff-{metadata['generated_at'][:10]}-{metadata['artifact_hash'][:12]}.md"
1487
+ )
1488
+ destination = config.home / "handoffs" / config.project_slug / name
1489
+ state = _write_once(destination, raw)
1490
+ archived = False
1491
+ archive_reason = "provider not selected"
1492
+ receipt: Path | None = None
1493
+ if config.provider != "none" and not args.local_only:
1494
+ archive_reason = (
1495
+ "skipped: handoffs are local historical evidence, not active memory"
1496
+ )
1497
+ receipt = _provider_receipt(
1498
+ config,
1499
+ provider_version="not-invoked",
1500
+ operation="archive-handoff",
1501
+ artifact_hash=str(metadata["artifact_hash"]),
1502
+ argv=[],
1503
+ outcome="skipped",
1504
+ detail=archive_reason,
1505
+ )
1506
+ elif args.local_only:
1507
+ archive_reason = "skipped: --local-only"
1508
+ print(
1509
+ json.dumps(
1510
+ {
1511
+ "status": state,
1512
+ "artifact": str(destination),
1513
+ "project": config.project_slug,
1514
+ "provider": config.provider,
1515
+ "provider_archived": archived,
1516
+ "provider_archive": {
1517
+ "outcome": "skipped",
1518
+ "reason": archive_reason,
1519
+ "receipt": str(receipt) if receipt else None,
1520
+ },
1521
+ "provider_reconciliation": "not-required: handoffs are not active memory",
1522
+ "saved_head": metadata["head"],
1523
+ }
1524
+ )
1525
+ )
1526
+ return 0
1527
+
1528
+
1529
+ def _local_search(
1530
+ args: argparse.Namespace,
1531
+ config: Config,
1532
+ *,
1533
+ annotations: dict[str, object] | None = None,
1534
+ ) -> int:
1535
+ terms = {term.lower() for term in TOKEN_RE.findall(args.query)}
1536
+ if not terms:
1537
+ raise Refusal("search query must contain at least one searchable term")
1538
+ results: list[dict[str, object]] = []
1539
+ invalid_records: list[dict[str, str]] = []
1540
+ inactive_records: list[dict[str, str]] = []
1541
+ for path in (
1542
+ sorted(config.records_path.glob("*.md")) if config.records_path.exists() else []
1543
+ ):
1544
+ try:
1545
+ metadata, state = _load_record(path, config)
1546
+ except Refusal as exc:
1547
+ invalid_records.append({"artifact": str(path), "error": str(exc)})
1548
+ continue
1549
+ if not _is_active(state) and not args.include_inactive:
1550
+ inactive_records.append(
1551
+ {
1552
+ "id": str(metadata["id"]),
1553
+ "review": state["review"],
1554
+ "freshness": state["freshness"],
1555
+ }
1556
+ )
1557
+ continue
1558
+ primary = metadata["primary_memory"]
1559
+ cues = metadata["cue_anchors"]
1560
+ primary_text = primary.lower()
1561
+ cue_text = " ".join(cues).lower()
1562
+ source_state = _source_state(metadata)
1563
+ primary_matches = sum(term in primary_text for term in terms)
1564
+ cue_matches = sum(term in cue_text for term in terms)
1565
+ score = primary_matches * 2 + cue_matches
1566
+ if score:
1567
+ results.append(
1568
+ {
1569
+ "id": metadata["id"],
1570
+ "type": metadata["type"],
1571
+ "freshness": state["freshness"],
1572
+ "review": state["review"],
1573
+ "primary_memory": primary,
1574
+ "cue_anchors": cues,
1575
+ "source": metadata["source"],
1576
+ "source_hash": metadata["source_hash"],
1577
+ "source_state": source_state,
1578
+ "score": score,
1579
+ }
1580
+ )
1581
+ results.sort(key=lambda item: (-int(item["score"]), str(item["id"])))
1582
+ payload: dict[str, object] = {
1583
+ "provider": "local",
1584
+ "project": config.project_slug,
1585
+ "records": results[: args.results],
1586
+ "invalid_records": invalid_records,
1587
+ "inactive_records": inactive_records,
1588
+ "include_inactive": args.include_inactive,
1589
+ }
1590
+ # A degraded provider search annotates the local result rather than
1591
+ # silently presenting lexical hits as semantic recall.
1592
+ payload.update(annotations or {})
1593
+ print(json.dumps(payload))
1594
+ return 0
1595
+
1596
+
1597
+ def _provider_search(args: argparse.Namespace, config: Config) -> int:
1598
+ """Run a reconciled provider query. MemPalace streams; rag is enriched."""
1599
+ spec = config.spec
1600
+ if spec.name != "rag":
1601
+ result = _run_provider(config, spec.search_argv(args.query, args.results))
1602
+ _write_stdout(result.stdout)
1603
+ return 0
1604
+
1605
+ records, _ = _active_projection(config)
1606
+ by_id = {str(metadata["id"]): metadata for _, metadata in records}
1607
+ # rag indexes each markdown section separately, so one record yields hits
1608
+ # for Primary Memory, Cue Anchors, Evidence, and so on. Over-fetch chunks
1609
+ # so that deduplicating back to records can still fill the requested
1610
+ # number of *records* rather than being consumed by one verbose memory.
1611
+ chunk_budget = min(50, max(args.results * RAG_CHUNKS_PER_RECORD, args.results))
1612
+ result = _run_provider(config, spec.search_argv(args.query, chunk_budget))
1613
+ raw = result.stdout.decode("utf-8", errors="replace").strip()
1614
+ try:
1615
+ hits = json.loads(raw) if raw else []
1616
+ except json.JSONDecodeError as exc:
1617
+ raise Refusal(f"rag returned output that is not valid JSON: {exc}") from exc
1618
+ if not isinstance(hits, list):
1619
+ raise Refusal("rag returned an unexpected JSON shape; expected a list")
1620
+
1621
+ found: list[dict[str, object]] = []
1622
+ seen: dict[str, dict[str, object]] = {}
1623
+ unmatched: list[str] = []
1624
+ for hit in hits:
1625
+ if not isinstance(hit, dict):
1626
+ continue
1627
+ # The projection materializes each record as `<record-id>.md`, so a hit
1628
+ # path maps back to exactly one record id.
1629
+ path = str(hit.get("path", ""))
1630
+ record_id = path[:-3] if path.endswith(".md") else path
1631
+ metadata = by_id.get(record_id)
1632
+ if metadata is None:
1633
+ if path not in unmatched:
1634
+ unmatched.append(path)
1635
+ continue
1636
+ if record_id in seen:
1637
+ # Keep the strongest chunk for a record; count the rest.
1638
+ entry = seen[record_id]
1639
+ entry["matched_chunks"] = int(entry["matched_chunks"]) + 1
1640
+ continue
1641
+ state = effective_state(metadata, config)
1642
+ entry = {
1643
+ "id": metadata["id"],
1644
+ "type": metadata["type"],
1645
+ "freshness": state["freshness"],
1646
+ "review": state["review"],
1647
+ "primary_memory": metadata["primary_memory"],
1648
+ "cue_anchors": metadata["cue_anchors"],
1649
+ "source": metadata["source"],
1650
+ "source_hash": metadata["source_hash"],
1651
+ "source_state": _source_state(metadata),
1652
+ "score": hit.get("score"),
1653
+ "retrieval_mode": hit.get("retrieval_mode"),
1654
+ "heading": hit.get("heading"),
1655
+ "matched_chunks": 1,
1656
+ }
1657
+ seen[record_id] = entry
1658
+ found.append(entry)
1659
+ if len(found) >= args.results:
1660
+ break
1661
+ print(
1662
+ json.dumps(
1663
+ {
1664
+ "provider": spec.name,
1665
+ "project": config.project_slug,
1666
+ "records": found,
1667
+ # A hit the adapter cannot bind to a current record is reported,
1668
+ # never dropped: it means the index is ahead of the ledger.
1669
+ "unmatched_hits": sorted(unmatched),
1670
+ "include_inactive": False,
1671
+ }
1672
+ )
1673
+ )
1674
+ return 0
1675
+
1676
+
1677
+ def _search(args: argparse.Namespace, config: Config) -> int:
1678
+ if config.provider == "none":
1679
+ return _local_search(args, config)
1680
+ if args.include_inactive:
1681
+ raise Refusal(
1682
+ "--include-inactive is available only for local audit search; "
1683
+ "the provider index is active-only"
1684
+ )
1685
+ records, _ = _active_projection(config)
1686
+ projection_hash = _projection_hash(records)
1687
+ # Reconciliation is a correctness gate, not an availability problem, so it
1688
+ # refuses outright and is deliberately outside the degradation path below.
1689
+ if not _has_current_provider_projection(config, projection_hash):
1690
+ raise Refusal(
1691
+ "provider index is not reconciled with accepted/current records; "
1692
+ "run sync-provider --apply or use --provider none for local recall"
1693
+ )
1694
+ try:
1695
+ return _provider_search(args, config)
1696
+ except Refusal as exc:
1697
+ return _local_search(
1698
+ args,
1699
+ config,
1700
+ annotations={
1701
+ "degraded_from": config.provider,
1702
+ "degraded_reason": str(exc),
1703
+ "degraded_detail": (
1704
+ "semantic recall was unavailable; these are lexical matches "
1705
+ "over primary memories and cue anchors only"
1706
+ ),
1707
+ },
1708
+ )
1709
+
1710
+
1711
+ def _wake_digest(config: Config) -> dict[str, object]:
1712
+ """Build a bounded session-priming digest from local records.
1713
+
1714
+ Provider-neutral by construction: records are the system of record and a
1715
+ provider store is a rebuildable projection of them, so the digest is
1716
+ identical whether the configured provider is `none`, `rag`, or
1717
+ `mempalace`. That also makes it a stable payload for lifecycle hooks.
1718
+ """
1719
+ records, _ = _active_projection(config)
1720
+ inactive = 0
1721
+ if config.records_path.exists():
1722
+ for path in sorted(config.records_path.glob("*.md")):
1723
+ try:
1724
+ metadata, state = _load_record(path, config)
1725
+ except Refusal:
1726
+ continue
1727
+ if not _is_active(state):
1728
+ inactive += 1
1729
+
1730
+ # Most recently observed first; ties break on id so output is stable.
1731
+ ordered = sorted(
1732
+ records,
1733
+ key=lambda item: (str(item[1]["observed_at"]), str(item[1]["id"])),
1734
+ reverse=True,
1735
+ )
1736
+ by_type: dict[str, int] = {}
1737
+ for _, metadata in ordered:
1738
+ kind = str(metadata["type"])
1739
+ by_type[kind] = by_type.get(kind, 0) + 1
1740
+
1741
+ surfaced: list[dict[str, object]] = []
1742
+ attention: list[dict[str, str]] = []
1743
+ used = 0
1744
+ truncated = False
1745
+ for _, metadata in ordered:
1746
+ if len(surfaced) >= MAX_WAKE_RECORDS:
1747
+ truncated = True
1748
+ break
1749
+ primary = str(metadata["primary_memory"])
1750
+ cost = len(primary) + len(str(metadata["id"])) + len(str(metadata["type"])) + 8
1751
+ if used + cost > MAX_WAKE_CHARS and surfaced:
1752
+ truncated = True
1753
+ break
1754
+ used += cost
1755
+ # Drift is only computed for surfaced records: hashing every source on
1756
+ # every session start would be unbounded work. `audit` sweeps the store.
1757
+ state = _source_state(metadata)
1758
+ if state != "verified":
1759
+ attention.append(
1760
+ {
1761
+ "id": str(metadata["id"]),
1762
+ "issue": state,
1763
+ "source": str(metadata["source"]),
1764
+ }
1765
+ )
1766
+ surfaced.append(
1767
+ {
1768
+ "id": metadata["id"],
1769
+ "type": metadata["type"],
1770
+ "primary_memory": primary,
1771
+ "observed_at": metadata["observed_at"],
1772
+ "source": metadata["source"],
1773
+ "source_state": state,
1774
+ }
1775
+ )
1776
+
1777
+ lines: list[str] = []
1778
+ if surfaced:
1779
+ lines.append(f"## Durable project memory — {config.project}")
1780
+ lines.append("")
1781
+ lines.append(
1782
+ f"{len(records)} reviewed memor{'y' if len(records) == 1 else 'ies'} "
1783
+ "for this project. These are leads, not current truth: open the "
1784
+ "cited source before relying on one."
1785
+ )
1786
+ lines.append("")
1787
+ for entry in surfaced:
1788
+ flag = "" if entry["source_state"] == "verified" else " [source changed]"
1789
+ lines.append(
1790
+ f"- [{entry['type']}] {entry['primary_memory']} "
1791
+ f"({entry['id']}){flag}"
1792
+ )
1793
+ if truncated:
1794
+ lines.append("")
1795
+ lines.append(
1796
+ f"Showing {len(surfaced)} of {len(records)}; "
1797
+ "run `search` for the rest."
1798
+ )
1799
+ if attention:
1800
+ lines.append("")
1801
+ lines.append(
1802
+ f"{len(attention)} surfaced record(s) cite a source that changed "
1803
+ "or is missing. Run `audit` before trusting them."
1804
+ )
1805
+
1806
+ digest: dict[str, object] = {
1807
+ "schema": WAKE_SCHEMA,
1808
+ "project": config.project,
1809
+ "project_key": config.project_slug,
1810
+ "provider": config.provider,
1811
+ "generated_at": _utc_timestamp(),
1812
+ "counts": {
1813
+ "active": len(records),
1814
+ "inactive": inactive,
1815
+ "by_type": by_type,
1816
+ "surfaced": len(surfaced),
1817
+ },
1818
+ "memories": surfaced,
1819
+ "attention": attention,
1820
+ "truncated": truncated,
1821
+ "context": "\n".join(lines),
1822
+ }
1823
+ if config.provider != "none":
1824
+ digest["reconciled"] = _has_current_provider_projection(
1825
+ config, _projection_hash(records)
1826
+ )
1827
+ return digest
1828
+
1829
+
1830
+ def _wake(args: argparse.Namespace, config: Config) -> int:
1831
+ digest = _wake_digest(config)
1832
+ if getattr(args, "format", "json") == "text":
1833
+ context = str(digest["context"])
1834
+ if context:
1835
+ print(context)
1836
+ return 0
1837
+ print(json.dumps(digest))
1838
+ return 0
1839
+
1840
+
1841
+ def _audit(args: argparse.Namespace, config: Config) -> int:
1842
+ """Sweep every record and report source drift. Proposes; never deletes.
1843
+
1844
+ MemPalace's `sync` prunes drawers whose sources are gitignored, deleted, or
1845
+ moved. Deleting is wrong here: a record's evidence is the reason it can be
1846
+ trusted later, and a moved file is not proof the decision was wrong. This
1847
+ reports drift and suggests a state transition the reviewer can apply with
1848
+ `record-state`.
1849
+ """
1850
+ findings: list[dict[str, object]] = []
1851
+ invalid: list[dict[str, str]] = []
1852
+ counts = {"verified": 0, "drifted": 0, "unavailable": 0, "invalid": 0}
1853
+ paths = (
1854
+ sorted(config.records_path.glob("*.md")) if config.records_path.exists() else []
1855
+ )
1856
+ for path in paths:
1857
+ try:
1858
+ metadata, state = _load_record(path, config)
1859
+ except Refusal as exc:
1860
+ counts["invalid"] += 1
1861
+ invalid.append({"artifact": str(path), "error": str(exc)})
1862
+ continue
1863
+ source_state = _source_state(metadata)
1864
+ counts[source_state] = counts.get(source_state, 0) + 1
1865
+ if source_state == "verified":
1866
+ continue
1867
+ active = _is_active(state)
1868
+ findings.append(
1869
+ {
1870
+ "id": metadata["id"],
1871
+ "source": metadata["source"],
1872
+ "source_state": source_state,
1873
+ "review": state["review"],
1874
+ "freshness": state["freshness"],
1875
+ "active": active,
1876
+ # Only an active record misleads recall, so only it needs action.
1877
+ "suggested": (
1878
+ "record-state {id} --freshness stale --reason "
1879
+ '"Cited source {issue}; re-verify before relying on it."'.format(
1880
+ id=metadata["id"],
1881
+ issue=(
1882
+ "changed since capture"
1883
+ if source_state == "drifted"
1884
+ else "is no longer present"
1885
+ ),
1886
+ )
1887
+ if active
1888
+ else "none: already inactive"
1889
+ ),
1890
+ }
1891
+ )
1892
+ findings.sort(key=lambda item: (not item["active"], str(item["id"])))
1893
+ actionable = [f for f in findings if f["active"]]
1894
+ print(
1895
+ json.dumps(
1896
+ {
1897
+ "project": config.project_slug,
1898
+ "records_examined": len(paths),
1899
+ "counts": counts,
1900
+ "findings": findings[: args.limit],
1901
+ "actionable": len(actionable),
1902
+ "invalid_records": invalid,
1903
+ "truncated": len(findings) > args.limit,
1904
+ "note": (
1905
+ "audit reports and proposes; it never edits or deletes a "
1906
+ "record. Apply a suggestion with `record-state`, then "
1907
+ "re-run `sync-provider --apply` if a provider is configured."
1908
+ ),
1909
+ }
1910
+ )
1911
+ )
1912
+ return 0
1913
+
1914
+
1915
+ def _doctor(args: argparse.Namespace, config: Config) -> int:
1916
+ result: dict[str, object] = {
1917
+ "provider": config.provider,
1918
+ "home": str(config.home),
1919
+ "project": config.project,
1920
+ "auto_capture": config.auto_capture,
1921
+ }
1922
+ if config.provider == "none":
1923
+ result.update(
1924
+ {
1925
+ "status": "ready",
1926
+ "mode": "local",
1927
+ "records_path": str(config.home / "records" / config.project_slug),
1928
+ }
1929
+ )
1930
+ print(json.dumps(result))
1931
+ return 0
1932
+ config.project_slug
1933
+ spec = config.spec
1934
+ executable = _provider_executable(spec)
1935
+ if spec.name == "rag":
1936
+ # The venv only backs the bundled `bin/rag` launcher. A user-supplied
1937
+ # executable (CONTEXT_KIT_INDEXKIT_BIN or one on PATH) manages its own
1938
+ # runtime, so gating on our bootstrap would be wrong there.
1939
+ bundled = _bundled_executable(spec)
1940
+ if bundled is not None and str(bundled) == executable:
1941
+ # Checked before the version and capability probes: without a
1942
+ # usable venv those fail with an opaque launcher error instead of
1943
+ # the actionable "run the bootstrap" answer.
1944
+ runtime = _rag_runtime_status(
1945
+ bootstrap=bool(getattr(args, "bootstrap", False))
1946
+ )
1947
+ result["runtime"] = runtime
1948
+ status = str(runtime.get("status"))
1949
+ if status not in {"ready", "unknown"}:
1950
+ command = runtime.get("bootstrap_command") or (
1951
+ "bash plugins/indexkit/scripts/bootstrap.sh"
1952
+ )
1953
+ raise Refusal(
1954
+ f"the indexkit runtime is {status} "
1955
+ f"({runtime.get('detail', 'no detail')}); run: {command} "
1956
+ "— Claude Code and GitHub Copilot CLI bootstrap this from "
1957
+ "the indexkit SessionStart hook, but APM does not deploy "
1958
+ "hooks and an existing venv can be stale. Re-run `doctor "
1959
+ "--bootstrap` to build it now."
1960
+ )
1961
+ version_result = _run_provider(config, ["--version"], timeout=20.0)
1962
+ raw_version = version_result.stdout.decode("utf-8", errors="replace").strip()
1963
+ parsed_version = _parse_provider_version(raw_version)
1964
+ version_status = _provider_version_status(parsed_version, spec)
1965
+ capabilities = [
1966
+ _probe_capability(executable, config, spec, probe)
1967
+ for probe in spec.capabilities
1968
+ ]
1969
+ missing = [c for c in capabilities if c["status"] != "ok"]
1970
+ compatibility: dict[str, object] = {
1971
+ "detected_version": raw_version,
1972
+ "parsed_version": (
1973
+ ".".join(str(part) for part in parsed_version) if parsed_version else None
1974
+ ),
1975
+ "version_status": version_status,
1976
+ "tested_release_line": spec.tested_release_line,
1977
+ "tested_version": ".".join(str(part) for part in spec.tested_version),
1978
+ "executable": executable,
1979
+ "store_path": str(config.store_path),
1980
+ "capabilities": capabilities,
1981
+ }
1982
+ if spec.name == "mempalace":
1983
+ compatibility["palace_path"] = str(config.store_path)
1984
+ result["compatibility"] = compatibility
1985
+ if missing:
1986
+ summary = "; ".join(
1987
+ f"{c['name']} ({c['command']}): {c['detail']}" for c in missing
1988
+ )
1989
+ raise Refusal(
1990
+ f"{spec.name} CLI is missing required capabilities for this adapter "
1991
+ f"(tested against {spec.tested_release_line}, detected "
1992
+ f"{raw_version or 'unknown version'}): {summary}"
1993
+ )
1994
+ # A patch/minor version different from the tested line is not on its own
1995
+ # a reason to block: only missing/incompatible capabilities are fatal.
1996
+ result.update(
1997
+ {
1998
+ "status": "ready",
1999
+ "executable": executable,
2000
+ "store_path": str(config.store_path),
2001
+ "version": raw_version,
2002
+ }
2003
+ )
2004
+ if spec.name == "mempalace":
2005
+ result["palace_path"] = str(config.store_path)
2006
+ print(json.dumps(result))
2007
+ return 0
2008
+
2009
+
2010
+ def _record_review(config: Config) -> int:
2011
+ results: list[dict[str, str]] = []
2012
+ for path in (
2013
+ sorted(config.records_path.glob("*.md")) if config.records_path.exists() else []
2014
+ ):
2015
+ try:
2016
+ metadata, state = _load_record(path, config)
2017
+ results.append(
2018
+ {
2019
+ "id": metadata["id"],
2020
+ "artifact": str(path),
2021
+ "freshness": state["freshness"],
2022
+ "review": state["review"],
2023
+ "source_state": _source_state(metadata),
2024
+ "active": str(_is_active(state)).lower(),
2025
+ }
2026
+ )
2027
+ except Refusal as exc:
2028
+ results.append(
2029
+ {
2030
+ "artifact": str(path),
2031
+ "source_state": "invalid-or-stale",
2032
+ "error": str(exc),
2033
+ }
2034
+ )
2035
+ print(
2036
+ json.dumps(
2037
+ {
2038
+ "project": config.project_slug,
2039
+ "records": results,
2040
+ "audit": True,
2041
+ "include_inactive": True,
2042
+ }
2043
+ )
2044
+ )
2045
+ return 0
2046
+
2047
+
2048
+ def _record_state(args: argparse.Namespace, config: Config) -> int:
2049
+ if not ID_RE.fullmatch(args.record_id):
2050
+ raise Refusal("record id must be lowercase and use letters, numbers, ._-")
2051
+ path = config.records_path / f"{args.record_id}.md"
2052
+ if not path.is_file():
2053
+ raise Refusal(f"record does not exist in this project: {args.record_id}")
2054
+ reason = args.reason.strip()
2055
+ if not reason:
2056
+ raise Refusal("--reason must not be empty")
2057
+ if len(reason) > MAX_STATE_REASON_CHARS:
2058
+ raise Refusal(f"--reason must not exceed {MAX_STATE_REASON_CHARS} characters")
2059
+ with _state_lock(config, args.record_id):
2060
+ metadata, current = _load_record(path, config)
2061
+ requested = {
2062
+ "review": args.review if args.review is not None else current["review"],
2063
+ "freshness": (
2064
+ args.freshness if args.freshness is not None else current["freshness"]
2065
+ ),
2066
+ }
2067
+ _validate_transition(current, requested)
2068
+ existing = _event_paths(config, args.record_id)
2069
+ if existing and not STATE_SEQUENCE_RE.fullmatch(existing[0].name):
2070
+ raise Refusal(
2071
+ "legacy timestamp-named state events are replayed read-only; "
2072
+ "migrate them before recording a new transition"
2073
+ )
2074
+ sequence = (
2075
+ int(STATE_SEQUENCE_RE.fullmatch(existing[-1].name)[1]) + 1
2076
+ if existing
2077
+ else 1
2078
+ )
2079
+ payload: dict[str, object] = {
2080
+ "schema": STATE_SCHEMA,
2081
+ "event_id": uuid.uuid4().hex,
2082
+ "record_id": metadata["id"],
2083
+ "record_hash": metadata["artifact_hash"],
2084
+ "project": config.project,
2085
+ "project_key": config.project_slug,
2086
+ "timestamp": _utc_timestamp(),
2087
+ "prior_review": current["review"],
2088
+ "prior_freshness": current["freshness"],
2089
+ "effective_review": requested["review"],
2090
+ "effective_freshness": requested["freshness"],
2091
+ "reason": reason,
2092
+ "sequence": sequence,
2093
+ }
2094
+ raw = (json.dumps(payload, sort_keys=True, indent=2) + "\n").encode("utf-8")
2095
+ event = (
2096
+ config.states_path
2097
+ / args.record_id
2098
+ / f"{sequence:0{STATE_SEQUENCE_WIDTH}d}-{uuid.uuid4().hex}.json"
2099
+ )
2100
+ if _write_once(event, raw) != "created":
2101
+ raise Refusal(f"refusing to reuse a generated state event path: {event}")
2102
+ print(
2103
+ json.dumps(
2104
+ {
2105
+ "status": "created",
2106
+ "event": str(event),
2107
+ "record": str(path),
2108
+ "record_hash": metadata["artifact_hash"],
2109
+ "project": config.project_slug,
2110
+ "prior_state": current,
2111
+ "effective_state": requested,
2112
+ "provider_reconciliation": (
2113
+ "required before provider recall"
2114
+ if config.provider == "mempalace"
2115
+ else "not-applicable"
2116
+ ),
2117
+ }
2118
+ )
2119
+ )
2120
+ return 0
2121
+
2122
+
2123
+ def _read_receipts(config: Config) -> list[dict[str, object]]:
2124
+ receipts: list[dict[str, object]] = []
2125
+ if not config.receipts_path.exists():
2126
+ return receipts
2127
+ for path in sorted(config.receipts_path.glob("*.json")):
2128
+ try:
2129
+ payload = json.loads(path.read_text(encoding="utf-8"))
2130
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
2131
+ continue
2132
+ if (
2133
+ isinstance(payload, dict)
2134
+ and payload.get("schema") == RECEIPT_SCHEMA
2135
+ and payload.get("project") == config.project
2136
+ and payload.get("project_key") == config.project_slug
2137
+ ):
2138
+ receipts.append(payload)
2139
+ return receipts
2140
+
2141
+
2142
+ def _write_projection_marker(
2143
+ stage: Path,
2144
+ config: Config,
2145
+ projection_hash: str,
2146
+ ledger_hash: str,
2147
+ provider_version: str,
2148
+ ) -> None:
2149
+ raw = (
2150
+ json.dumps(
2151
+ {
2152
+ "schema": PROJECTION_MARKER_SCHEMA,
2153
+ "project": config.project,
2154
+ "project_key": config.project_slug,
2155
+ "projection_hash": projection_hash,
2156
+ "ledger_hash": ledger_hash,
2157
+ "provider_version": provider_version,
2158
+ "applied_at": _utc_timestamp(),
2159
+ },
2160
+ sort_keys=True,
2161
+ indent=2,
2162
+ )
2163
+ + "\n"
2164
+ ).encode("utf-8")
2165
+ _write_once(stage / PROJECTION_MARKER_NAME, raw)
2166
+
2167
+
2168
+ def _has_current_provider_projection(config: Config, projection_hash: str) -> bool:
2169
+ marker = config.store_path / PROJECTION_MARKER_NAME
2170
+ try:
2171
+ payload = json.loads(marker.read_text(encoding="utf-8"))
2172
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
2173
+ return False
2174
+ if not isinstance(payload, dict) or set(payload) != {
2175
+ "schema",
2176
+ "project",
2177
+ "project_key",
2178
+ "projection_hash",
2179
+ "ledger_hash",
2180
+ "provider_version",
2181
+ "applied_at",
2182
+ }:
2183
+ return False
2184
+ try:
2185
+ _validate_timestamp(str(payload["applied_at"]), "applied_at")
2186
+ except Refusal:
2187
+ return False
2188
+ return (
2189
+ payload["schema"] == PROJECTION_MARKER_SCHEMA
2190
+ and payload["project"] == config.project
2191
+ and payload["project_key"] == config.project_slug
2192
+ and payload["projection_hash"] == projection_hash
2193
+ and payload["ledger_hash"] == _ledger_hash(config)
2194
+ and isinstance(payload["provider_version"], str)
2195
+ )
2196
+
2197
+
2198
+ def _prune_provider_backups(
2199
+ parent: Path, keep: Path | None, prefix: str = "palace-backup-"
2200
+ ) -> list[str]:
2201
+ backups = [
2202
+ path for path in parent.glob(f"{prefix}*") if path.is_dir() and path != keep
2203
+ ]
2204
+ if keep is None:
2205
+ backups.sort(
2206
+ key=lambda path: (path.stat().st_mtime_ns, path.name), reverse=True
2207
+ )
2208
+ backups = backups[PROVIDER_BACKUP_RETENTION:]
2209
+ removed: list[str] = []
2210
+ for path in backups:
2211
+ shutil.rmtree(path)
2212
+ removed.append(str(path))
2213
+ return sorted(removed)
2214
+
2215
+
2216
+ def _sync_provider(args: argparse.Namespace, config: Config) -> int:
2217
+ if config.provider == "none":
2218
+ raise Refusal(
2219
+ "sync-provider requires an external provider; choose one of: "
2220
+ + ", ".join(sorted(PROVIDER_SPECS))
2221
+ )
2222
+ spec = config.spec
2223
+ records, excluded = _active_projection(config)
2224
+ projection_hash = _projection_hash(records)
2225
+ ledger_hash = _ledger_hash(config)
2226
+ plan: dict[str, object] = {
2227
+ "project": config.project_slug,
2228
+ "provider": spec.name,
2229
+ "store_path": str(config.store_path),
2230
+ "active_record_ids": [str(metadata["id"]) for _, metadata in records],
2231
+ "excluded_records": excluded,
2232
+ "projection_hash": projection_hash,
2233
+ "apply": bool(args.apply),
2234
+ }
2235
+ if spec.name == "mempalace":
2236
+ plan["palace_path"] = str(config.store_path)
2237
+ if not args.apply:
2238
+ plan["status"] = "dry-run"
2239
+ plan["safety"] = (
2240
+ "apply builds a fresh project-isolated store, preserves a backup, "
2241
+ f"then swaps only after {spec.name} succeeds"
2242
+ )
2243
+ print(json.dumps(plan))
2244
+ return 0
2245
+ if os.name != "posix":
2246
+ raise Refusal(
2247
+ "safe provider replacement is supported only on POSIX; "
2248
+ "dry-run was not applied and no store was changed"
2249
+ )
2250
+
2251
+ parent = config.store_path.parent
2252
+ parent.mkdir(parents=True, exist_ok=True)
2253
+ projection = Path(tempfile.mkdtemp(prefix=".projection-", dir=parent))
2254
+ stage = parent / f".store-rebuild-{uuid.uuid4().hex}"
2255
+ backup: Path | None = None
2256
+ executable = ""
2257
+ version = "unknown"
2258
+ argv: list[str] = []
2259
+ recovery_status = "not-needed"
2260
+ try:
2261
+ _materialize_projection(records, projection)
2262
+ stage.mkdir(mode=0o700)
2263
+ executable, version = _provider_version(config)
2264
+ argv = [executable, *spec.index_argv(projection, config.project_slug)]
2265
+ _run_provider(
2266
+ config,
2267
+ argv[1:],
2268
+ timeout=300.0,
2269
+ store_path=stage,
2270
+ )
2271
+ if not stage.is_dir():
2272
+ raise Refusal(f"{spec.name} did not leave a valid staged store")
2273
+ _write_projection_marker(
2274
+ stage,
2275
+ config,
2276
+ projection_hash,
2277
+ ledger_hash,
2278
+ version,
2279
+ )
2280
+ if config.store_path.exists():
2281
+ backup = parent / f"{spec.backup_prefix}{uuid.uuid4().hex}"
2282
+ os.replace(config.store_path, backup)
2283
+ try:
2284
+ os.replace(stage, config.store_path)
2285
+ except OSError:
2286
+ if backup is not None and not config.store_path.exists():
2287
+ os.replace(backup, config.store_path)
2288
+ backup = None
2289
+ recovery_status = "restored-to-live-store"
2290
+ elif backup is not None:
2291
+ recovery_status = "backup-preserved"
2292
+ raise
2293
+ except (OSError, Refusal) as exc:
2294
+ receipt = _provider_receipt(
2295
+ config,
2296
+ provider_version=version,
2297
+ operation="sync-provider",
2298
+ artifact_hash=None,
2299
+ argv=argv,
2300
+ outcome="failed",
2301
+ detail=str(exc),
2302
+ projection_hash=projection_hash,
2303
+ backup_path=backup,
2304
+ recovery_status=recovery_status,
2305
+ )
2306
+ raise Refusal(
2307
+ f"provider synchronization failed; receipt={receipt}: {exc}"
2308
+ ) from exc
2309
+ finally:
2310
+ shutil.rmtree(projection, ignore_errors=True)
2311
+ if stage.exists() and stage != config.store_path:
2312
+ shutil.rmtree(stage, ignore_errors=True)
2313
+ receipt = _provider_receipt(
2314
+ config,
2315
+ provider_version=version,
2316
+ operation="sync-provider",
2317
+ artifact_hash=None,
2318
+ argv=argv,
2319
+ outcome="success",
2320
+ detail=f"reconciled {len(records)} accepted/current records",
2321
+ projection_hash=projection_hash,
2322
+ backup_path=backup,
2323
+ )
2324
+ try:
2325
+ removed_backups = _prune_provider_backups(parent, backup, spec.backup_prefix)
2326
+ except OSError as exc:
2327
+ raise Refusal(
2328
+ "provider synchronized but backup retention failed; "
2329
+ f"receipt={receipt}: {exc}"
2330
+ ) from exc
2331
+ plan.update(
2332
+ {
2333
+ "status": "synchronized",
2334
+ "backup_path": str(backup) if backup else None,
2335
+ "removed_backups": removed_backups,
2336
+ "receipt": str(receipt),
2337
+ }
2338
+ )
2339
+ print(json.dumps(plan))
2340
+ return 0
2341
+
2342
+
2343
+ def _extract_copilot_session(raw: bytes) -> dict[str, object] | None:
2344
+ """Extract the human-visible conversation from a Copilot CLI event log.
2345
+
2346
+ Copilot records *all* session activity in one event stream, so most events
2347
+ are not conversation. Attribution matters more than volume here: a subagent
2348
+ task prompt is written by the orchestrating model, not by the person, and
2349
+ storing it as a user turn silently misattributes authorship.
2350
+
2351
+ Measured across a real 115-session corpus, only 24 of 729 `user.message`
2352
+ events were human-authored; 611 carried `parentAgentTaskId` (subagent task
2353
+ prompts) and 94 carried a `source` (generated skill/agent/command/system
2354
+ context). Every distinct `source` value observed was generated context, so
2355
+ the presence of the field — not a specific prefix — is the reliable signal.
2356
+
2357
+ Returns None when the input is not a recognized Copilot session. A
2358
+ recognized session with no conversational turns returns an empty turn list
2359
+ rather than None, so the caller can record it as empty instead of falling
2360
+ back to raw event JSON.
2361
+ """
2362
+ text = raw.decode("utf-8", errors="replace")
2363
+ session_id: str | None = None
2364
+ started_at: str | None = None
2365
+ producer: str | None = None
2366
+ turns: list[dict[str, str]] = []
2367
+ dropped: dict[str, int] = {
2368
+ "user_subagent_prompt": 0,
2369
+ "user_generated_context": 0,
2370
+ "user_empty": 0,
2371
+ "assistant_tool_nested": 0,
2372
+ "assistant_subagent": 0,
2373
+ "assistant_empty": 0,
2374
+ "non_conversational_event": 0,
2375
+ "unparsable_line": 0,
2376
+ }
2377
+
2378
+ for line in text.splitlines():
2379
+ line = line.strip()
2380
+ if not line:
2381
+ continue
2382
+ try:
2383
+ event = json.loads(line)
2384
+ except json.JSONDecodeError:
2385
+ dropped["unparsable_line"] += 1
2386
+ continue
2387
+ if not isinstance(event, dict):
2388
+ dropped["unparsable_line"] += 1
2389
+ continue
2390
+ kind = event.get("type", "")
2391
+ data = event.get("data", {})
2392
+ if not isinstance(data, dict):
2393
+ dropped["non_conversational_event"] += 1
2394
+ continue
2395
+
2396
+ if kind == "session.start":
2397
+ candidate_id = data.get("sessionId")
2398
+ if isinstance(candidate_id, str) and candidate_id:
2399
+ session_id = candidate_id
2400
+ start_time = data.get("startTime")
2401
+ if isinstance(start_time, str) and start_time:
2402
+ started_at = start_time
2403
+ produced_by = data.get("producer")
2404
+ if isinstance(produced_by, str) and produced_by:
2405
+ producer = produced_by
2406
+ continue
2407
+
2408
+ if session_id is None:
2409
+ # Nothing before a recognized session.start is trusted.
2410
+ dropped["non_conversational_event"] += 1
2411
+ continue
2412
+
2413
+ if kind == "user.message":
2414
+ if data.get("parentAgentTaskId"):
2415
+ dropped["user_subagent_prompt"] += 1
2416
+ continue
2417
+ if "source" in data:
2418
+ # Generated context: skill-, agent-, command-, or system.
2419
+ # A human turn carries no source at all.
2420
+ dropped["user_generated_context"] += 1
2421
+ continue
2422
+ role = "user"
2423
+ elif kind == "assistant.message":
2424
+ if data.get("parentToolCallId"):
2425
+ dropped["assistant_tool_nested"] += 1
2426
+ continue
2427
+ if data.get("parentAgentTaskId"):
2428
+ dropped["assistant_subagent"] += 1
2429
+ continue
2430
+ role = "assistant"
2431
+ else:
2432
+ dropped["non_conversational_event"] += 1
2433
+ continue
2434
+
2435
+ # `content` is what the person actually wrote. `transformedContent`
2436
+ # is post-expansion, and reasoning fields are never retained.
2437
+ content = data.get("content")
2438
+ if not isinstance(content, str) or not content.strip():
2439
+ dropped[f"{role}_empty"] += 1
2440
+ continue
2441
+ turns.append({"role": role, "content": content.strip()})
2442
+
2443
+ if session_id is None:
2444
+ return None
2445
+ return {
2446
+ "session_id": session_id,
2447
+ "started_at": started_at,
2448
+ "producer": producer,
2449
+ "turns": turns,
2450
+ "dropped": dropped,
2451
+ }
2452
+
2453
+
2454
+ def _scan_secrets(text: str) -> list[dict[str, object]]:
2455
+ findings: list[dict[str, object]] = []
2456
+ for name, pattern in SECRET_PATTERNS:
2457
+ count = len(pattern.findall(text))
2458
+ if count:
2459
+ findings.append({"pattern": name, "matches": count})
2460
+ return findings
2461
+
2462
+
2463
+ def _redact_secrets(text: str) -> tuple[str, int]:
2464
+ redactions = 0
2465
+ for name, pattern in SECRET_PATTERNS:
2466
+ text, count = pattern.subn(f"[redacted:{name}]", text)
2467
+ redactions += count
2468
+ return text, redactions
2469
+
2470
+
2471
+ def _render_turns(turns: list[dict[str, str]]) -> tuple[list[str], int, int]:
2472
+ lines: list[str] = []
2473
+ truncated = 0
2474
+ omitted = max(0, len(turns) - MAX_CANDIDATE_TURNS)
2475
+ for index, turn in enumerate(turns[:MAX_CANDIDATE_TURNS], start=1):
2476
+ content = turn["content"]
2477
+ if len(content) > MAX_TURN_CHARS:
2478
+ content = content[:MAX_TURN_CHARS]
2479
+ truncated += 1
2480
+ content += f"\n\n[truncated at {MAX_TURN_CHARS} characters]"
2481
+ lines.append(f"### {index}. {turn['role']}")
2482
+ lines.append("")
2483
+ lines.extend(content.splitlines())
2484
+ lines.append("")
2485
+ if omitted:
2486
+ # A reviewer must never mistake a sliced transcript for a complete one.
2487
+ lines.append(
2488
+ f"### [{omitted} further turn(s) omitted at the "
2489
+ f"{MAX_CANDIDATE_TURNS}-turn candidate limit]"
2490
+ )
2491
+ lines.append("")
2492
+ return lines, truncated, omitted
2493
+
2494
+
2495
+ def _propose_from_session(args: argparse.Namespace, config: Config) -> int:
2496
+ """Extract reviewable candidates from Copilot CLI sessions.
2497
+
2498
+ This deliberately proposes rather than captures. A transcript is not an
2499
+ atomic memory, so authoring a `memory-v1` record from a candidate stays an
2500
+ explicit judgment step; nothing here can enter active recall on its own.
2501
+ """
2502
+ root = Path(args.path).expanduser().resolve()
2503
+ if root.is_file():
2504
+ logs = [root]
2505
+ elif root.is_dir():
2506
+ logs = sorted(root.glob("*/events.jsonl")) or sorted(root.glob("events.jsonl"))
2507
+ else:
2508
+ raise Refusal(f"session path does not exist: {root}")
2509
+ if not logs:
2510
+ raise Refusal(f"no Copilot `events.jsonl` logs found under {root}")
2511
+
2512
+ repo = Path(args.repo).expanduser().resolve()
2513
+ repository = _normalize_repository(_git(repo, "remote", "get-url", "origin"))
2514
+ _assert_project_matches({"repository": repository, "scope": "project"}, config)
2515
+ branch = _git(repo, "rev-parse", "--abbrev-ref", "HEAD")
2516
+ head = _git(repo, "rev-parse", "HEAD")
2517
+ if branch == "HEAD":
2518
+ # Detached checkouts (CI PR builds, a worktree at a tag, bisect) have no
2519
+ # branch anchor, and a project record requires one. Say so plainly
2520
+ # instead of refusing with a generic name-format error.
2521
+ raise Refusal(
2522
+ f"{repo} is in a detached HEAD state, so there is no branch anchor "
2523
+ "for a project record. Check out a named branch, or pass --repo "
2524
+ "pointing at a checkout that is on one."
2525
+ )
2526
+ _validate_branch(branch)
2527
+
2528
+ planned: list[dict[str, object]] = []
2529
+ written: list[str] = []
2530
+ blocked: list[dict[str, object]] = []
2531
+ skipped: list[dict[str, object]] = []
2532
+
2533
+ for log in logs:
2534
+ raw = log.read_bytes()
2535
+ extracted = _extract_copilot_session(raw)
2536
+ if extracted is None:
2537
+ skipped.append({"source": str(log), "reason": "not-a-copilot-session"})
2538
+ continue
2539
+ turns = list(extracted["turns"]) # type: ignore[arg-type]
2540
+ if not turns:
2541
+ skipped.append({"source": str(log), "reason": "no-conversational-turns"})
2542
+ continue
2543
+ session_id = str(extracted["session_id"])
2544
+ if not SESSION_ID_RE.fullmatch(session_id):
2545
+ # `session_id` names the candidate file. An id like `../../outside`
2546
+ # would escape the project-isolated directory, so an unsafe value
2547
+ # is refused loudly rather than sanitized into something plausible.
2548
+ skipped.append(
2549
+ {
2550
+ "source": str(log),
2551
+ "reason": "unsafe-session-id",
2552
+ "session_id": session_id[:80],
2553
+ }
2554
+ )
2555
+ continue
2556
+
2557
+ body_lines, truncated, omitted = _render_turns(turns)
2558
+ transcript = "\n".join(body_lines)
2559
+ findings = _scan_secrets(transcript)
2560
+ redactions = 0
2561
+ if findings:
2562
+ if not args.redact:
2563
+ blocked.append(
2564
+ {
2565
+ "source": str(log),
2566
+ "reason": "possible-credentials",
2567
+ "findings": findings,
2568
+ }
2569
+ )
2570
+ continue
2571
+ transcript, redactions = _redact_secrets(transcript)
2572
+
2573
+ source_hash = hashlib.sha256(raw).hexdigest()
2574
+ observed_at = extracted["started_at"] or _utc_timestamp()
2575
+ try:
2576
+ _validate_timestamp(str(observed_at), "observed_at")
2577
+ except Refusal:
2578
+ observed_at = _utc_timestamp()
2579
+ entry: dict[str, object] = {
2580
+ "session_id": session_id,
2581
+ "source": str(log),
2582
+ "source_hash": source_hash,
2583
+ "turns": len(turns),
2584
+ "turns_written": len(turns) - omitted,
2585
+ "omitted_turns": omitted,
2586
+ "dropped": extracted["dropped"],
2587
+ "truncated_turns": truncated,
2588
+ "redactions": redactions,
2589
+ }
2590
+ if args.dry_run:
2591
+ planned.append(entry)
2592
+ continue
2593
+
2594
+ name = f"{session_id}-{source_hash[:12]}.md"
2595
+ destination = config.candidates_path / name
2596
+ # Defense in depth: the id was validated above, so a path that still
2597
+ # resolves outside the project directory is a bug, not user error.
2598
+ if destination.parent.resolve() != config.candidates_path.resolve():
2599
+ raise Refusal(
2600
+ f"refusing a candidate path outside the project store: {destination}"
2601
+ )
2602
+ document = "\n".join(
2603
+ [
2604
+ "---",
2605
+ f"schema: {CANDIDATE_SCHEMA}",
2606
+ f"session_id: {session_id}",
2607
+ "scope: project",
2608
+ f"repository: {repository}",
2609
+ f"branch: {branch}",
2610
+ f"head: {head}",
2611
+ f"producer: {extracted['producer'] or SESSION_PRODUCER}",
2612
+ f"observed_at: {observed_at}",
2613
+ f"extracted_at: {_utc_timestamp()}",
2614
+ f"source: {log}",
2615
+ f"source_hash: {source_hash}",
2616
+ f"turns: {len(turns) - omitted}",
2617
+ f"turns_extracted: {len(turns)}",
2618
+ f"omitted_turns: {omitted}",
2619
+ f"redactions: {redactions}",
2620
+ "review: candidate",
2621
+ "---",
2622
+ "",
2623
+ "## Provenance",
2624
+ "",
2625
+ f"- Extracted from `{log}` (SHA-256 `{source_hash}`).",
2626
+ "- Only top-level human and assistant turns are retained. Subagent",
2627
+ " prompts, generated skill/agent/command context, tool-nested",
2628
+ " messages, and model reasoning are excluded by construction.",
2629
+ "",
2630
+ "## Dropped Events",
2631
+ "",
2632
+ *(
2633
+ f"- {reason}: {count}"
2634
+ for reason, count in sorted(
2635
+ dict(extracted["dropped"]).items() # type: ignore[arg-type]
2636
+ )
2637
+ if count
2638
+ ),
2639
+ "",
2640
+ "## Transcript",
2641
+ "",
2642
+ transcript,
2643
+ "",
2644
+ "## Review Notes",
2645
+ "",
2646
+ "- This is a candidate, not a memory. To retain anything here,",
2647
+ " author a `context-kit/memory-v1` record whose `source` is the",
2648
+ " session log above and whose `source_hash` matches, mark it",
2649
+ " `review: proposed`, then promote it with `record-state` only",
2650
+ " after checking the evidence.",
2651
+ "",
2652
+ ]
2653
+ )
2654
+ state = _write_once(destination, document.encode("utf-8"))
2655
+ entry["artifact"] = str(destination)
2656
+ entry["status"] = state
2657
+ written.append(str(destination))
2658
+ planned.append(entry)
2659
+
2660
+ print(
2661
+ json.dumps(
2662
+ {
2663
+ "status": "dry-run" if args.dry_run else "extracted",
2664
+ "project": config.project_slug,
2665
+ "repository": repository,
2666
+ "branch": branch,
2667
+ "head": head,
2668
+ "logs_examined": len(logs),
2669
+ "candidates": planned,
2670
+ "written": written,
2671
+ "blocked": blocked,
2672
+ "skipped": skipped,
2673
+ "note": (
2674
+ "candidates are proposals for review; nothing here enters "
2675
+ "active recall until an explicit memory-v1 record is "
2676
+ "captured and accepted"
2677
+ ),
2678
+ }
2679
+ )
2680
+ )
2681
+ return 0
2682
+
2683
+
2684
+ def _hook_recall(config: Config) -> int:
2685
+ """Emit reviewed memory as session-start context.
2686
+
2687
+ Both Claude Code and GitHub Copilot CLI run a plugin's `hooks/hooks.json`
2688
+ and honor an `additionalContext` string, so this is the one lifecycle point
2689
+ where durable memory pays for itself without the agent asking: reviewed,
2690
+ accepted, current records prime the session.
2691
+
2692
+ Recall is read-only, so it is gated on its own switch rather than on
2693
+ `auto_capture`, which governs writing.
2694
+ """
2695
+ if not config.recall_on_start:
2696
+ print("{}")
2697
+ return 0
2698
+ try:
2699
+ digest = _wake_digest(config)
2700
+ except Refusal:
2701
+ # A hook must never break a session over a memory problem.
2702
+ print("{}")
2703
+ return 0
2704
+ context = str(digest["context"])
2705
+ if not context:
2706
+ print("{}")
2707
+ return 0
2708
+ print(json.dumps({"additionalContext": context}))
2709
+ return 0
2710
+
2711
+
2712
+ def _run_hook(event: str, config: Config, payload: bytes) -> int:
2713
+ if event == "session-start":
2714
+ return _hook_recall(config)
2715
+ if not config.auto_capture:
2716
+ print("{}")
2717
+ return 0
2718
+ try:
2719
+ decoded = json.loads(payload)
2720
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
2721
+ raise Refusal(f"hook payload must be valid JSON: {exc}") from exc
2722
+ if not isinstance(decoded, dict):
2723
+ raise Refusal("hook payload must be a JSON object")
2724
+ pending_dir = config.home / "pending-hooks" / config.project_slug
2725
+ pending = _new_write_once_path(pending_dir, f"-{event}.json")
2726
+ if _write_once(pending, payload) != "created":
2727
+ raise Refusal(f"refusing to reuse a generated hook payload path: {pending}")
2728
+ print(
2729
+ json.dumps(
2730
+ {
2731
+ "status": "queued-for-review",
2732
+ "event": event,
2733
+ "pending": str(pending),
2734
+ "provider_invoked": False,
2735
+ }
2736
+ )
2737
+ )
2738
+ return 0
2739
+
2740
+
2741
+ def _add_config_args(parser: argparse.ArgumentParser) -> None:
2742
+ parser.add_argument("--provider", choices=PROVIDERS)
2743
+ parser.add_argument("--home")
2744
+ parser.add_argument("--project")
2745
+
2746
+
2747
+ def _parser() -> argparse.ArgumentParser:
2748
+ parser = argparse.ArgumentParser(
2749
+ # No `prog=`, deliberately: argparse then derives it from argv[0], so
2750
+ # help text names whichever entry point the user actually invoked —
2751
+ # `memorykit` from the installed console script, `memory-provider.py`
2752
+ # from the plugin launcher. A hardcoded name would be wrong for one of
2753
+ # the two deployment shapes and would print copy-paste-broken usage.
2754
+ description="Validate context-kit memories and invoke an optional provider.",
2755
+ )
2756
+ sub = parser.add_subparsers(dest="command", required=True)
2757
+
2758
+ validate = sub.add_parser("validate")
2759
+ validate.add_argument("artifact")
2760
+
2761
+ capture = sub.add_parser("capture")
2762
+ capture.add_argument("artifact")
2763
+ capture.add_argument("--local-only", action="store_true")
2764
+ _add_config_args(capture)
2765
+
2766
+ archive = sub.add_parser("archive-handoff")
2767
+ archive.add_argument("artifact")
2768
+ archive.add_argument("--local-only", action="store_true")
2769
+ archive.add_argument(
2770
+ "--repo",
2771
+ default=".",
2772
+ help="Current repository used to enforce handoff freshness.",
2773
+ )
2774
+ _add_config_args(archive)
2775
+
2776
+ search = sub.add_parser("search")
2777
+ search.add_argument("query")
2778
+ search.add_argument("--results", type=int, default=8)
2779
+ search.add_argument(
2780
+ "--include-inactive",
2781
+ action="store_true",
2782
+ help="Audit local proposed, rejected, stale, superseded, and revoked records.",
2783
+ )
2784
+ _add_config_args(search)
2785
+
2786
+ wake = sub.add_parser("wake")
2787
+ wake.add_argument(
2788
+ "--format",
2789
+ choices=("json", "text"),
2790
+ default="json",
2791
+ help="`text` prints only the injectable context block.",
2792
+ )
2793
+ _add_config_args(wake)
2794
+
2795
+ doctor = sub.add_parser("doctor")
2796
+ doctor.add_argument(
2797
+ "--bootstrap",
2798
+ action="store_true",
2799
+ help="Build the indexkit runtime if it is missing or stale (rag provider).",
2800
+ )
2801
+ _add_config_args(doctor)
2802
+
2803
+ audit = sub.add_parser("audit")
2804
+ audit.add_argument(
2805
+ "--limit",
2806
+ type=int,
2807
+ default=50,
2808
+ help="Maximum findings to print (default 50).",
2809
+ )
2810
+ _add_config_args(audit)
2811
+
2812
+ review = sub.add_parser("review")
2813
+ review.add_argument(
2814
+ "--include-inactive",
2815
+ action="store_true",
2816
+ help="Explicitly document audit intent; review already reports all records.",
2817
+ )
2818
+ _add_config_args(review)
2819
+
2820
+ state = sub.add_parser("record-state")
2821
+ state.add_argument("record_id")
2822
+ state.add_argument("--review", choices=sorted(REVIEW_STATES))
2823
+ state.add_argument("--freshness", choices=sorted(FRESHNESS_STATES))
2824
+ state.add_argument("--reason", required=True)
2825
+ _add_config_args(state)
2826
+
2827
+ sync = sub.add_parser("sync-provider")
2828
+ sync.add_argument(
2829
+ "--apply",
2830
+ action="store_true",
2831
+ help="Build, validate, back up, and replace the project-isolated active palace.",
2832
+ )
2833
+ _add_config_args(sync)
2834
+
2835
+ propose = sub.add_parser("propose-from-session")
2836
+ propose.add_argument("path", help="A Copilot session directory or events.jsonl.")
2837
+ propose.add_argument(
2838
+ "--repo",
2839
+ default=".",
2840
+ help="Repository supplying the project, branch, and HEAD anchors.",
2841
+ )
2842
+ propose.add_argument(
2843
+ "--write",
2844
+ dest="dry_run",
2845
+ action="store_false",
2846
+ help="Write candidate artifacts. Without this the run is a dry run.",
2847
+ )
2848
+ propose.add_argument(
2849
+ "--redact",
2850
+ action="store_true",
2851
+ help="Mask detected credential-shaped spans instead of refusing.",
2852
+ )
2853
+ propose.set_defaults(dry_run=True)
2854
+ _add_config_args(propose)
2855
+
2856
+ hook = sub.add_parser("hook")
2857
+ hook.add_argument(
2858
+ "event",
2859
+ choices=("session-start", "stop", "precompact", "session-end"),
2860
+ )
2861
+ _add_config_args(hook)
2862
+ return parser
2863
+
2864
+
2865
+ def main(argv: list[str] | None = None) -> int:
2866
+ args = _parser().parse_args(argv)
2867
+ try:
2868
+ if args.command == "validate":
2869
+ metadata = validate_memory(Path(args.artifact).expanduser().resolve())
2870
+ print(json.dumps({"status": "valid", "id": metadata["id"]}))
2871
+ return 0
2872
+
2873
+ config = _config(args)
2874
+ if args.command == "capture":
2875
+ return _capture_memory(args, config)
2876
+ if args.command == "archive-handoff":
2877
+ return _archive_handoff(args, config)
2878
+ if args.command == "search":
2879
+ if not 1 <= args.results <= 50:
2880
+ raise Refusal("--results must be between 1 and 50")
2881
+ return _search(args, config)
2882
+ if args.command == "wake":
2883
+ return _wake(args, config)
2884
+ if args.command == "doctor":
2885
+ return _doctor(args, config)
2886
+ if args.command == "audit":
2887
+ if args.limit < 1:
2888
+ raise Refusal("--limit must be at least 1")
2889
+ return _audit(args, config)
2890
+ if args.command == "review":
2891
+ return _record_review(config)
2892
+ if args.command == "record-state":
2893
+ return _record_state(args, config)
2894
+ if args.command == "sync-provider":
2895
+ return _sync_provider(args, config)
2896
+ if args.command == "propose-from-session":
2897
+ return _propose_from_session(args, config)
2898
+ if args.command == "hook":
2899
+ payload = sys.stdin.buffer.read()
2900
+ return _run_hook(args.event, config, payload)
2901
+ except (OSError, Refusal) as exc:
2902
+ print(json.dumps({"status": "refused", "error": str(exc)}), file=sys.stderr)
2903
+ return 2
2904
+ return 2
2905
+
2906
+
2907
+ if __name__ == "__main__":
2908
+ raise SystemExit(main())