memorykit 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memorykit/__init__.py +24 -0
- memorykit/__main__.py +13 -0
- memorykit/mcp.py +373 -0
- memorykit/provider.py +2908 -0
- memorykit-0.6.0.dist-info/METADATA +292 -0
- memorykit-0.6.0.dist-info/RECORD +9 -0
- memorykit-0.6.0.dist-info/WHEEL +4 -0
- memorykit-0.6.0.dist-info/entry_points.txt +3 -0
- memorykit-0.6.0.dist-info/licenses/LICENSE +21 -0
memorykit/provider.py
ADDED
|
@@ -0,0 +1,2908 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Validate durable memories and invoke an optional MemPalace provider."""
|
|
3
|
+
|
|
4
|
+
# Governing decisions. adrkit scans only the first 8192 bytes of a file for
|
|
5
|
+
# markers, and this module is far larger, so file-level declarations belong
|
|
6
|
+
# here in the header rather than beside the code they explain.
|
|
7
|
+
# @adr 0002
|
|
8
|
+
# @adr 0003
|
|
9
|
+
# @adr 0007
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import contextlib
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import shutil
|
|
20
|
+
import subprocess
|
|
21
|
+
import sys
|
|
22
|
+
import tempfile
|
|
23
|
+
import time
|
|
24
|
+
import uuid
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from datetime import datetime, timezone
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
SCHEMA = "context-kit/memory-v1"
|
|
30
|
+
MAX_BYTES = 32 * 1024
|
|
31
|
+
MAX_MEMORY_LINES = 220
|
|
32
|
+
MAX_HANDOFF_LINES = 300
|
|
33
|
+
MAX_HANDOFF_ITEMS = 25
|
|
34
|
+
MAX_CUES = 3
|
|
35
|
+
MAX_STATE_REASON_CHARS = 1000
|
|
36
|
+
PROVIDER_BACKUP_RETENTION = 1
|
|
37
|
+
PROJECT_SLUG_PREFIX_LENGTH = 31
|
|
38
|
+
ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,95}$")
|
|
39
|
+
HASH_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
40
|
+
COMMIT_RE = re.compile(r"^[0-9a-fA-F]{7,64}$")
|
|
41
|
+
REPOSITORY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$")
|
|
42
|
+
FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$")
|
|
43
|
+
PLACEHOLDER_RE = re.compile(r"\{\{[^{}\n]+\}\}")
|
|
44
|
+
LIST_ITEM_RE = re.compile(r"^(?:[-*+] |\d+[.)] )")
|
|
45
|
+
TOKEN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*")
|
|
46
|
+
MEMORY_TYPES = {"fact", "decision", "procedure", "constraint", "episode"}
|
|
47
|
+
SCOPES = {"project"}
|
|
48
|
+
FRESHNESS_STATES = {"current", "stale", "superseded", "revoked"}
|
|
49
|
+
REVIEW_STATES = {"proposed", "accepted", "rejected"}
|
|
50
|
+
STATE_SCHEMA = "context-kit/memory-state-v1"
|
|
51
|
+
RECEIPT_SCHEMA = "context-kit/memory-provider-receipt-v1"
|
|
52
|
+
CANDIDATE_SCHEMA = "context-kit/memory-candidate-v1"
|
|
53
|
+
WAKE_SCHEMA = "context-kit/memory-wake-v1"
|
|
54
|
+
# Session mining recognizes GitHub Copilot CLI event logs
|
|
55
|
+
# (`~/.copilot/session-state/<session-id>/events.jsonl`).
|
|
56
|
+
SESSION_PRODUCER = "github-copilot-cli"
|
|
57
|
+
# `session_id` arrives from the session log and is used to name a candidate
|
|
58
|
+
# file, so it is validated as a single safe path component before use.
|
|
59
|
+
SESSION_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
|
60
|
+
MAX_TURN_CHARS = 2000
|
|
61
|
+
MAX_CANDIDATE_TURNS = 400
|
|
62
|
+
# A wake digest primes a session, so it competes with real work for context.
|
|
63
|
+
# MemPalace targets 600-900 tokens for the same job; this is a comparable
|
|
64
|
+
# character budget with a hard record cap so one verbose store cannot flood it.
|
|
65
|
+
MAX_WAKE_RECORDS = 12
|
|
66
|
+
MAX_WAKE_CHARS = 2400
|
|
67
|
+
# Detection only. Each pattern names a high-signal credential shape so a
|
|
68
|
+
# finding can be reported precisely and, with --redact, masked in place.
|
|
69
|
+
SECRET_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
|
|
70
|
+
("aws-access-key-id", re.compile(r"AKIA[0-9A-Z]{16}")),
|
|
71
|
+
("github-token", re.compile(r"gh[pousr]_[A-Za-z0-9]{36,}")),
|
|
72
|
+
("slack-token", re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}")),
|
|
73
|
+
("private-key-block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
|
|
74
|
+
(
|
|
75
|
+
"json-web-token",
|
|
76
|
+
re.compile(r"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"),
|
|
77
|
+
),
|
|
78
|
+
(
|
|
79
|
+
"assigned-credential",
|
|
80
|
+
re.compile(
|
|
81
|
+
r"(?i)\b(?:api[_-]?key|secret|password|passwd|access[_-]?token|bearer)\b"
|
|
82
|
+
r"\s*[:=]\s*[\"']?[A-Za-z0-9/+_.-]{16,}"
|
|
83
|
+
),
|
|
84
|
+
),
|
|
85
|
+
)
|
|
86
|
+
PROJECTION_MARKER_SCHEMA = "context-kit/memory-provider-projection-v1"
|
|
87
|
+
PROJECTION_MARKER_NAME = ".context-kit-projection.json"
|
|
88
|
+
STATE_SEQUENCE_WIDTH = 20
|
|
89
|
+
STATE_LOCK_TIMEOUT_SECONDS = 5.0
|
|
90
|
+
STATE_LOCK_STALE_SECONDS = 300.0
|
|
91
|
+
STATE_SEQUENCE_RE = re.compile(r"^(\d{20})-[0-9a-f]{32}\.json$")
|
|
92
|
+
REVIEW_TRANSITIONS = {
|
|
93
|
+
"proposed": {"accepted", "rejected"},
|
|
94
|
+
"accepted": {"rejected"},
|
|
95
|
+
"rejected": {"accepted"},
|
|
96
|
+
}
|
|
97
|
+
FRESHNESS_TRANSITIONS = {
|
|
98
|
+
"current": {"stale", "superseded", "revoked"},
|
|
99
|
+
"stale": {"current", "superseded", "revoked"},
|
|
100
|
+
"superseded": set(),
|
|
101
|
+
"revoked": set(),
|
|
102
|
+
}
|
|
103
|
+
# The adapter is verified against this MemPalace release; see
|
|
104
|
+
# skills/memory-workflows/references/provider-mempalace.md for the
|
|
105
|
+
# compatibility matrix and how `doctor` reports drift.
|
|
106
|
+
MEMPALACE_TESTED_VERSION = (3, 6, 0)
|
|
107
|
+
MEMPALACE_TESTED_RELEASE_LINE = "3.6.x"
|
|
108
|
+
# The first-party `rag` provider is this repository's own `indexkit` plugin.
|
|
109
|
+
# `CONTEXT_KIT_INDEXKIT_HOME` (indexkit >= 0.4.0) separates venv resolution
|
|
110
|
+
# from index-data location, which is what lets this adapter redirect index data
|
|
111
|
+
# into a project-isolated store without relocating the shared venv.
|
|
112
|
+
RAG_TESTED_VERSION = (0, 6, 1)
|
|
113
|
+
RAG_TESTED_RELEASE_LINE = "0.6.x"
|
|
114
|
+
RAG_INDEX_NAME = "memory"
|
|
115
|
+
# One record spans several indexed chunks, so ask for more chunks than the
|
|
116
|
+
# caller asked for records and deduplicate back down.
|
|
117
|
+
RAG_CHUNKS_PER_RECORD = 4
|
|
118
|
+
SEMVER_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)")
|
|
119
|
+
REQUIRED_FIELDS = (
|
|
120
|
+
"schema",
|
|
121
|
+
"id",
|
|
122
|
+
"type",
|
|
123
|
+
"scope",
|
|
124
|
+
"repository",
|
|
125
|
+
"branch",
|
|
126
|
+
"head",
|
|
127
|
+
"observed_at",
|
|
128
|
+
"captured_at",
|
|
129
|
+
"freshness",
|
|
130
|
+
"review",
|
|
131
|
+
"source",
|
|
132
|
+
"source_hash",
|
|
133
|
+
)
|
|
134
|
+
MEMORY_HEADINGS = (
|
|
135
|
+
"## Primary Memory",
|
|
136
|
+
"## Cue Anchors",
|
|
137
|
+
"## Evidence",
|
|
138
|
+
"## Supersedes",
|
|
139
|
+
"## Review Notes",
|
|
140
|
+
)
|
|
141
|
+
HANDOFF_FIELDS = (
|
|
142
|
+
"schema",
|
|
143
|
+
"generated_at",
|
|
144
|
+
"repository",
|
|
145
|
+
"worktree",
|
|
146
|
+
"branch",
|
|
147
|
+
"head",
|
|
148
|
+
"base_ref",
|
|
149
|
+
"base_commit",
|
|
150
|
+
"worktree_state",
|
|
151
|
+
)
|
|
152
|
+
HANDOFF_HEADINGS = (
|
|
153
|
+
"## Scope",
|
|
154
|
+
"## Verified Facts",
|
|
155
|
+
"## Decisions",
|
|
156
|
+
"## Changed Files",
|
|
157
|
+
"## Completed Work",
|
|
158
|
+
"## Unresolved Items",
|
|
159
|
+
"## Next Steps",
|
|
160
|
+
"## Validation State",
|
|
161
|
+
"## Provenance and Freshness",
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class Refusal(ValueError):
|
|
166
|
+
"""An invalid input or unsafe provider request."""
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@dataclass(frozen=True)
|
|
170
|
+
class CapabilityProbe:
|
|
171
|
+
"""One exact-argv help surface the adapter depends on."""
|
|
172
|
+
|
|
173
|
+
name: str
|
|
174
|
+
argv: tuple[str, ...]
|
|
175
|
+
contract: str
|
|
176
|
+
required_tokens: tuple[str, ...] = ()
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@dataclass(frozen=True)
|
|
180
|
+
class ProviderSpec:
|
|
181
|
+
"""Everything that differs between external providers.
|
|
182
|
+
|
|
183
|
+
Keeping the differences declarative means `sync-provider`, `search`, and
|
|
184
|
+
`doctor` share one projection, staging, swap, marker, and receipt path
|
|
185
|
+
regardless of which provider is configured.
|
|
186
|
+
"""
|
|
187
|
+
|
|
188
|
+
name: str
|
|
189
|
+
# Live store directory under providers/<name>/<project-key>/. This is the
|
|
190
|
+
# unit that is atomically swapped, so it must be a directory the adapter
|
|
191
|
+
# owns exclusively.
|
|
192
|
+
store_dirname: str
|
|
193
|
+
backup_prefix: str
|
|
194
|
+
bin_env: str
|
|
195
|
+
executable: str
|
|
196
|
+
install_hint: str
|
|
197
|
+
tested_version: tuple[int, int, int]
|
|
198
|
+
tested_release_line: str
|
|
199
|
+
capabilities: tuple[CapabilityProbe, ...]
|
|
200
|
+
# Pre-rename names still honored so an install predating a rename keeps
|
|
201
|
+
# resolving. Declarative for the same reason the rest of this class is:
|
|
202
|
+
# the migration is provider data, not branching in the resolver.
|
|
203
|
+
legacy_bin_env: str | None = None
|
|
204
|
+
legacy_executables: tuple[str, ...] = ()
|
|
205
|
+
|
|
206
|
+
def index_argv(self, projection: Path, project_key: str) -> list[str]:
|
|
207
|
+
raise NotImplementedError
|
|
208
|
+
|
|
209
|
+
def search_argv(self, query: str, results: int) -> list[str]:
|
|
210
|
+
raise NotImplementedError
|
|
211
|
+
|
|
212
|
+
def store_env(self, store: Path) -> dict[str, str]:
|
|
213
|
+
"""Environment that points the provider at `store` for this call only."""
|
|
214
|
+
raise NotImplementedError
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@dataclass(frozen=True)
|
|
218
|
+
class MemPalaceSpec(ProviderSpec):
|
|
219
|
+
def index_argv(self, projection: Path, project_key: str) -> list[str]:
|
|
220
|
+
return ["mine", str(projection), "--wing", project_key]
|
|
221
|
+
|
|
222
|
+
def search_argv(self, query: str, results: int) -> list[str]:
|
|
223
|
+
return ["search", query, "--results", str(results)]
|
|
224
|
+
|
|
225
|
+
def store_env(self, store: Path) -> dict[str, str]:
|
|
226
|
+
return {"MEMPALACE_PALACE_PATH": str(store)}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@dataclass(frozen=True)
|
|
230
|
+
class RagSpec(ProviderSpec):
|
|
231
|
+
def index_argv(self, projection: Path, project_key: str) -> list[str]:
|
|
232
|
+
# The store is already project-isolated by `store_env`, so a constant
|
|
233
|
+
# index name keeps the on-disk layout readable.
|
|
234
|
+
return ["index", str(projection), "--name", RAG_INDEX_NAME]
|
|
235
|
+
|
|
236
|
+
def search_argv(self, query: str, results: int) -> list[str]:
|
|
237
|
+
return ["query", query, "--name", RAG_INDEX_NAME, "--k", str(results), "--json"]
|
|
238
|
+
|
|
239
|
+
def store_env(self, store: Path) -> dict[str, str]:
|
|
240
|
+
# CONTEXT_KIT_DATA relocates *index data* into the isolated store.
|
|
241
|
+
# The home variable pins the venv to its normal location so redirecting
|
|
242
|
+
# data does not make the launcher look for a venv that only exists in
|
|
243
|
+
# the shared indexkit home.
|
|
244
|
+
#
|
|
245
|
+
# Both names are exported. `_bundled_executable` can legitimately
|
|
246
|
+
# resolve a pre-rename `bin/rag` launcher, and that launcher reads only
|
|
247
|
+
# `CONTEXT_KIT_LOCAL_RAG_HOME`. Setting just the new name would leave it
|
|
248
|
+
# falling back to CONTEXT_KIT_DATA — the isolated store — where no venv
|
|
249
|
+
# exists, so every provider call through an old sibling would fail.
|
|
250
|
+
home = str(_indexkit_home())
|
|
251
|
+
return {
|
|
252
|
+
"CONTEXT_KIT_DATA": str(store),
|
|
253
|
+
"CONTEXT_KIT_INDEXKIT_HOME": home,
|
|
254
|
+
"CONTEXT_KIT_LOCAL_RAG_HOME": home,
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
PROVIDER_SPECS: dict[str, ProviderSpec] = {
|
|
259
|
+
"mempalace": MemPalaceSpec(
|
|
260
|
+
name="mempalace",
|
|
261
|
+
store_dirname="palace",
|
|
262
|
+
backup_prefix="palace-backup-",
|
|
263
|
+
bin_env="CONTEXT_KIT_MEMPALACE_BIN",
|
|
264
|
+
executable="mempalace",
|
|
265
|
+
install_hint="install it separately with `uv tool install mempalace`",
|
|
266
|
+
tested_version=MEMPALACE_TESTED_VERSION,
|
|
267
|
+
tested_release_line=MEMPALACE_TESTED_RELEASE_LINE,
|
|
268
|
+
# Each probe mirrors an exact argv the adapter actually invokes.
|
|
269
|
+
# `wake-up` is deliberately absent: `wake` is built from local
|
|
270
|
+
# records for every provider, so requiring it would refuse an
|
|
271
|
+
# install over a command this adapter never calls.
|
|
272
|
+
# Probing `--help` (never the mutating command itself) lets `doctor`
|
|
273
|
+
# catch upstream CLI drift without importing provider internals or
|
|
274
|
+
# writing to a store.
|
|
275
|
+
capabilities=(
|
|
276
|
+
CapabilityProbe(
|
|
277
|
+
name="capture",
|
|
278
|
+
argv=("mine", "--help"),
|
|
279
|
+
contract="mine <dir> --wing <project-key>",
|
|
280
|
+
required_tokens=("--wing",),
|
|
281
|
+
),
|
|
282
|
+
CapabilityProbe(
|
|
283
|
+
name="search",
|
|
284
|
+
argv=("search", "--help"),
|
|
285
|
+
contract="search <query> --results <n>",
|
|
286
|
+
required_tokens=("--results",),
|
|
287
|
+
),
|
|
288
|
+
),
|
|
289
|
+
),
|
|
290
|
+
"rag": RagSpec(
|
|
291
|
+
name="rag",
|
|
292
|
+
store_dirname="store",
|
|
293
|
+
backup_prefix="store-backup-",
|
|
294
|
+
bin_env="CONTEXT_KIT_INDEXKIT_BIN",
|
|
295
|
+
executable="indexkit",
|
|
296
|
+
legacy_bin_env="CONTEXT_KIT_RAG_BIN",
|
|
297
|
+
legacy_executables=("rag",),
|
|
298
|
+
install_hint=(
|
|
299
|
+
"install the context-kit `indexkit` plugin and run "
|
|
300
|
+
"`bash plugins/indexkit/scripts/bootstrap.sh`"
|
|
301
|
+
),
|
|
302
|
+
tested_version=RAG_TESTED_VERSION,
|
|
303
|
+
tested_release_line=RAG_TESTED_RELEASE_LINE,
|
|
304
|
+
capabilities=(
|
|
305
|
+
CapabilityProbe(
|
|
306
|
+
name="capture",
|
|
307
|
+
argv=("index", "--help"),
|
|
308
|
+
contract="index <dir> --name <index>",
|
|
309
|
+
required_tokens=("--name",),
|
|
310
|
+
),
|
|
311
|
+
CapabilityProbe(
|
|
312
|
+
name="search",
|
|
313
|
+
argv=("query", "--help"),
|
|
314
|
+
contract="query <text> --name <index> --k <n> --json",
|
|
315
|
+
required_tokens=("--name", "--k", "--json"),
|
|
316
|
+
),
|
|
317
|
+
),
|
|
318
|
+
),
|
|
319
|
+
}
|
|
320
|
+
PROVIDERS = ("none", *sorted(PROVIDER_SPECS))
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
@dataclass(frozen=True)
|
|
324
|
+
class Config:
|
|
325
|
+
provider: str
|
|
326
|
+
home: Path
|
|
327
|
+
project: str | None
|
|
328
|
+
auto_capture: bool
|
|
329
|
+
recall_on_start: bool = False
|
|
330
|
+
|
|
331
|
+
@property
|
|
332
|
+
def project_slug(self) -> str:
|
|
333
|
+
if not self.project:
|
|
334
|
+
raise Refusal(
|
|
335
|
+
"set CONTEXT_KIT_MEMORY_PROJECT (or pass --project) to isolate memory"
|
|
336
|
+
)
|
|
337
|
+
if not REPOSITORY_RE.fullmatch(self.project):
|
|
338
|
+
raise Refusal("memory project must be a concrete owner/name identity")
|
|
339
|
+
prefix = re.sub(r"[^A-Za-z0-9._-]+", "-", self.project).strip("-").lower()
|
|
340
|
+
digest = hashlib.sha256(self.project.encode("utf-8")).hexdigest()
|
|
341
|
+
return f"{prefix[:PROJECT_SLUG_PREFIX_LENGTH]}-{digest}"
|
|
342
|
+
|
|
343
|
+
@property
|
|
344
|
+
def spec(self) -> ProviderSpec:
|
|
345
|
+
try:
|
|
346
|
+
return PROVIDER_SPECS[self.provider]
|
|
347
|
+
except KeyError:
|
|
348
|
+
raise Refusal(
|
|
349
|
+
f"operation requires an external provider; configured: {self.provider}"
|
|
350
|
+
) from None
|
|
351
|
+
|
|
352
|
+
@property
|
|
353
|
+
def provider_root(self) -> Path:
|
|
354
|
+
return self.home / "providers" / self.spec.name / self.project_slug
|
|
355
|
+
|
|
356
|
+
@property
|
|
357
|
+
def store_path(self) -> Path:
|
|
358
|
+
"""The project-isolated directory swapped atomically on reconciliation."""
|
|
359
|
+
return self.provider_root / self.spec.store_dirname
|
|
360
|
+
|
|
361
|
+
@property
|
|
362
|
+
def palace_path(self) -> Path:
|
|
363
|
+
# Retained name for the MemPalace layout; `store_path` is the
|
|
364
|
+
# provider-neutral accessor used by the shared reconciliation path.
|
|
365
|
+
return self.home / "providers" / "mempalace" / self.project_slug / "palace"
|
|
366
|
+
|
|
367
|
+
@property
|
|
368
|
+
def records_path(self) -> Path:
|
|
369
|
+
return self.home / "records" / self.project_slug
|
|
370
|
+
|
|
371
|
+
@property
|
|
372
|
+
def states_path(self) -> Path:
|
|
373
|
+
return self.home / "states" / self.project_slug
|
|
374
|
+
|
|
375
|
+
@property
|
|
376
|
+
def receipts_path(self) -> Path:
|
|
377
|
+
return self.home / "receipts" / self.project_slug
|
|
378
|
+
|
|
379
|
+
@property
|
|
380
|
+
def candidates_path(self) -> Path:
|
|
381
|
+
"""Reviewable session extractions. Never active memory."""
|
|
382
|
+
return self.home / "candidates" / self.project_slug
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _first_env(*names: str) -> str | None:
|
|
386
|
+
for name in names:
|
|
387
|
+
value = os.environ.get(name)
|
|
388
|
+
if value:
|
|
389
|
+
return value
|
|
390
|
+
return None
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _truthy(value: str | None) -> bool:
|
|
394
|
+
return bool(value and value.strip().lower() in {"1", "true", "yes", "on"})
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _indexkit_home() -> Path:
|
|
398
|
+
"""Where indexkit keeps its bootstrapped venv.
|
|
399
|
+
|
|
400
|
+
Resolution is deliberately not a naive read of `CLAUDE_PLUGIN_DATA`: that
|
|
401
|
+
variable is *plugin-scoped*, so inside this plugin it points at memory's
|
|
402
|
+
own data directory. Both Claude Code and GitHub Copilot CLI lay plugin data
|
|
403
|
+
out as `<root>/<plugin>`, and `memory` hard-depends on `indexkit`, so the
|
|
404
|
+
dependency's home is a **sibling** of ours. Verified on a real Copilot
|
|
405
|
+
install: `~/.copilot/plugin-data/context-kit/{memory,indexkit}`.
|
|
406
|
+
|
|
407
|
+
The sibling is used only when it actually exists, so a wrong guess degrades
|
|
408
|
+
to the documented default rather than pointing at an empty directory.
|
|
409
|
+
|
|
410
|
+
Read from the *ambient* environment, before this adapter redirects
|
|
411
|
+
`CONTEXT_KIT_DATA` at the provider store.
|
|
412
|
+
|
|
413
|
+
The engine was renamed from `local-rag` to `indexkit` (ADR-0007). A host
|
|
414
|
+
that installed the plugin before the rename still has a `local-rag` data
|
|
415
|
+
directory, and a user may still export `CONTEXT_KIT_LOCAL_RAG_HOME`, so both
|
|
416
|
+
names are accepted with the current one preferred.
|
|
417
|
+
"""
|
|
418
|
+
configured = _first_env(
|
|
419
|
+
"CONTEXT_KIT_INDEXKIT_HOME",
|
|
420
|
+
"CONTEXT_KIT_LOCAL_RAG_HOME",
|
|
421
|
+
"CONTEXT_KIT_DATA",
|
|
422
|
+
"PRODUCTIVITY_SKILLS_DATA",
|
|
423
|
+
)
|
|
424
|
+
if configured:
|
|
425
|
+
return Path(configured).expanduser()
|
|
426
|
+
plugin_data = _first_env("CLAUDE_PLUGIN_DATA")
|
|
427
|
+
if plugin_data:
|
|
428
|
+
root = Path(plugin_data).expanduser().parent
|
|
429
|
+
for name in ("indexkit", "local-rag"):
|
|
430
|
+
sibling = root / name
|
|
431
|
+
if (sibling / "venv").is_dir() or (sibling / "pyproject.sha").is_file():
|
|
432
|
+
return sibling
|
|
433
|
+
default_root = Path.home() / ".claude/plugins/data"
|
|
434
|
+
legacy = default_root / "local-rag"
|
|
435
|
+
if not (default_root / "indexkit" / "venv").is_dir() and (legacy / "venv").is_dir():
|
|
436
|
+
return legacy
|
|
437
|
+
return default_root / "indexkit"
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _config(args: argparse.Namespace) -> Config:
|
|
441
|
+
provider = (
|
|
442
|
+
getattr(args, "provider", None)
|
|
443
|
+
or _first_env(
|
|
444
|
+
"CONTEXT_KIT_MEMORY_PROVIDER",
|
|
445
|
+
"CLAUDE_PLUGIN_OPTION_PROVIDER",
|
|
446
|
+
)
|
|
447
|
+
or "none"
|
|
448
|
+
).lower()
|
|
449
|
+
if provider not in PROVIDERS:
|
|
450
|
+
raise Refusal("memory provider must be one of: " + ", ".join(PROVIDERS))
|
|
451
|
+
|
|
452
|
+
home_value = (
|
|
453
|
+
getattr(args, "home", None)
|
|
454
|
+
or _first_env(
|
|
455
|
+
"CONTEXT_KIT_MEMORY_HOME",
|
|
456
|
+
"CLAUDE_PLUGIN_OPTION_MEMORY_HOME",
|
|
457
|
+
)
|
|
458
|
+
or "~/.local/share/context-kit/memory"
|
|
459
|
+
)
|
|
460
|
+
home = Path(home_value).expanduser().resolve()
|
|
461
|
+
project = getattr(args, "project", None) or _first_env(
|
|
462
|
+
"CONTEXT_KIT_MEMORY_PROJECT",
|
|
463
|
+
"CLAUDE_PLUGIN_OPTION_PROJECT",
|
|
464
|
+
)
|
|
465
|
+
auto_value = _first_env(
|
|
466
|
+
"CONTEXT_KIT_MEMORY_AUTO_CAPTURE",
|
|
467
|
+
"CLAUDE_PLUGIN_OPTION_AUTO_CAPTURE",
|
|
468
|
+
)
|
|
469
|
+
recall_value = _first_env(
|
|
470
|
+
"CONTEXT_KIT_MEMORY_RECALL_ON_START",
|
|
471
|
+
"CLAUDE_PLUGIN_OPTION_RECALL_ON_START",
|
|
472
|
+
)
|
|
473
|
+
return Config(
|
|
474
|
+
provider=provider,
|
|
475
|
+
home=home,
|
|
476
|
+
project=project,
|
|
477
|
+
auto_capture=_truthy(auto_value),
|
|
478
|
+
recall_on_start=_truthy(recall_value),
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _parse_frontmatter(text: str) -> tuple[dict[str, str], list[str]]:
|
|
483
|
+
lines = text.splitlines()
|
|
484
|
+
if not lines or lines[0].strip() != "---":
|
|
485
|
+
raise Refusal("artifact must start with flat YAML frontmatter")
|
|
486
|
+
fields: dict[str, str] = {}
|
|
487
|
+
closing = None
|
|
488
|
+
for index, line in enumerate(lines[1:], start=1):
|
|
489
|
+
if line.strip() == "---":
|
|
490
|
+
closing = index
|
|
491
|
+
break
|
|
492
|
+
if not line or line[0].isspace() or ":" not in line:
|
|
493
|
+
raise Refusal(
|
|
494
|
+
"frontmatter must contain only flat non-empty key/value fields"
|
|
495
|
+
)
|
|
496
|
+
key, value = line.split(":", 1)
|
|
497
|
+
key = key.strip()
|
|
498
|
+
value = value.strip()
|
|
499
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
|
500
|
+
value = value[1:-1]
|
|
501
|
+
if not key or not value or key in fields:
|
|
502
|
+
raise Refusal("frontmatter contains an empty or duplicate field")
|
|
503
|
+
fields[key] = value
|
|
504
|
+
if closing is None:
|
|
505
|
+
raise Refusal("frontmatter is missing its closing delimiter")
|
|
506
|
+
return fields, lines[closing + 1 :]
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _validate_timestamp(value: str, field: str) -> None:
|
|
510
|
+
try:
|
|
511
|
+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
512
|
+
except ValueError as exc:
|
|
513
|
+
raise Refusal(f"{field} must be an ISO 8601 timestamp") from exc
|
|
514
|
+
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
|
515
|
+
raise Refusal(f"{field} must include a timezone")
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _build_sections(
|
|
519
|
+
body: list[str],
|
|
520
|
+
headings: tuple[str, ...],
|
|
521
|
+
found: list[tuple[str, int]],
|
|
522
|
+
) -> dict[str, list[str]]:
|
|
523
|
+
found_headings = [heading for heading, _position in found]
|
|
524
|
+
if found_headings != list(headings):
|
|
525
|
+
raise Refusal("artifact headings are missing, reordered, or unexpected")
|
|
526
|
+
positions = [position for _heading, position in found]
|
|
527
|
+
result: dict[str, list[str]] = {}
|
|
528
|
+
for index, heading in enumerate(headings):
|
|
529
|
+
end = positions[index + 1] if index + 1 < len(positions) else len(body)
|
|
530
|
+
result[heading] = body[positions[index] + 1 : end]
|
|
531
|
+
return result
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _memory_sections(body: list[str]) -> dict[str, list[str]]:
|
|
535
|
+
found: list[tuple[str, int]] = []
|
|
536
|
+
fence: tuple[str, int] | None = None
|
|
537
|
+
for index, line in enumerate(body):
|
|
538
|
+
fence_match = FENCE_RE.match(line)
|
|
539
|
+
if fence is not None:
|
|
540
|
+
if fence_match:
|
|
541
|
+
marker, suffix = fence_match.groups()
|
|
542
|
+
if (
|
|
543
|
+
marker[0] == fence[0]
|
|
544
|
+
and len(marker) >= fence[1]
|
|
545
|
+
and not suffix.strip()
|
|
546
|
+
):
|
|
547
|
+
fence = None
|
|
548
|
+
continue
|
|
549
|
+
if fence_match:
|
|
550
|
+
marker, _suffix = fence_match.groups()
|
|
551
|
+
fence = (marker[0], len(marker))
|
|
552
|
+
continue
|
|
553
|
+
if line.startswith("## "):
|
|
554
|
+
found.append((line, index))
|
|
555
|
+
return _build_sections(body, MEMORY_HEADINGS, found)
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def _handoff_sections(body: list[str]) -> dict[str, list[str]]:
|
|
559
|
+
found = []
|
|
560
|
+
for index, line in enumerate(body):
|
|
561
|
+
if line.startswith("## "):
|
|
562
|
+
found.append((f"## {line[3:].strip()}", index))
|
|
563
|
+
return _build_sections(body, HANDOFF_HEADINGS, found)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _nonempty_section(lines: list[str], heading: str) -> str:
|
|
567
|
+
text = "\n".join(lines).strip()
|
|
568
|
+
if not text or text == "- None.":
|
|
569
|
+
raise Refusal(f"{heading} must not be empty")
|
|
570
|
+
return text
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _required_section(lines: list[str], heading: str) -> str:
|
|
574
|
+
text = "\n".join(lines).strip()
|
|
575
|
+
if not text:
|
|
576
|
+
raise Refusal(f"{heading} must not be empty; use '- None.'")
|
|
577
|
+
return text
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _validate_branch(value: str) -> None:
|
|
581
|
+
try:
|
|
582
|
+
result = subprocess.run(
|
|
583
|
+
["git", "check-ref-format", "--branch", value],
|
|
584
|
+
capture_output=True,
|
|
585
|
+
check=False,
|
|
586
|
+
text=True,
|
|
587
|
+
timeout=5.0,
|
|
588
|
+
)
|
|
589
|
+
except FileNotFoundError as exc:
|
|
590
|
+
raise Refusal("git is required to validate branch provenance") from exc
|
|
591
|
+
except subprocess.TimeoutExpired as exc:
|
|
592
|
+
raise Refusal("Git branch validation timed out") from exc
|
|
593
|
+
if result.returncode != 0:
|
|
594
|
+
raise Refusal("branch must be a valid concrete Git branch name")
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def _read_bounded(
|
|
598
|
+
path: Path, *, max_lines: int = MAX_MEMORY_LINES
|
|
599
|
+
) -> tuple[bytes, str]:
|
|
600
|
+
if not path.is_file():
|
|
601
|
+
raise Refusal(f"artifact is not a file: {path}")
|
|
602
|
+
raw = path.read_bytes()
|
|
603
|
+
if len(raw) > MAX_BYTES:
|
|
604
|
+
raise Refusal(f"artifact exceeds {MAX_BYTES} bytes")
|
|
605
|
+
try:
|
|
606
|
+
text = raw.decode("utf-8")
|
|
607
|
+
except UnicodeDecodeError as exc:
|
|
608
|
+
raise Refusal("artifact must be UTF-8") from exc
|
|
609
|
+
if len(text.splitlines()) > max_lines:
|
|
610
|
+
raise Refusal(f"artifact exceeds {max_lines} lines")
|
|
611
|
+
return raw, text
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
def validate_memory(path: Path, *, verify_source: bool = True) -> dict[str, object]:
|
|
615
|
+
raw, text = _read_bounded(path)
|
|
616
|
+
fields, body = _parse_frontmatter(text)
|
|
617
|
+
missing = [field for field in REQUIRED_FIELDS if field not in fields]
|
|
618
|
+
extras = sorted(set(fields) - set(REQUIRED_FIELDS))
|
|
619
|
+
if missing or extras:
|
|
620
|
+
raise Refusal(
|
|
621
|
+
f"memory frontmatter mismatch; missing={missing or 'none'} "
|
|
622
|
+
f"unexpected={extras or 'none'}"
|
|
623
|
+
)
|
|
624
|
+
if fields["schema"] != SCHEMA:
|
|
625
|
+
raise Refusal(f"schema must be {SCHEMA}")
|
|
626
|
+
if not ID_RE.fullmatch(fields["id"]):
|
|
627
|
+
raise Refusal("memory id must be lowercase and use letters, numbers, ._-")
|
|
628
|
+
if fields["type"] not in MEMORY_TYPES:
|
|
629
|
+
raise Refusal(f"memory type must be one of {sorted(MEMORY_TYPES)}")
|
|
630
|
+
if fields["scope"] not in SCOPES:
|
|
631
|
+
raise Refusal("memory scope must be 'project'")
|
|
632
|
+
if fields["freshness"] not in FRESHNESS_STATES:
|
|
633
|
+
raise Refusal(f"freshness must be one of {sorted(FRESHNESS_STATES)}")
|
|
634
|
+
if fields["review"] not in REVIEW_STATES:
|
|
635
|
+
raise Refusal(f"review must be one of {sorted(REVIEW_STATES)}")
|
|
636
|
+
if not HASH_RE.fullmatch(fields["source_hash"]):
|
|
637
|
+
raise Refusal("source_hash must be a lowercase SHA-256 digest")
|
|
638
|
+
_validate_timestamp(fields["observed_at"], "observed_at")
|
|
639
|
+
_validate_timestamp(fields["captured_at"], "captured_at")
|
|
640
|
+
if not REPOSITORY_RE.fullmatch(fields["repository"]):
|
|
641
|
+
raise Refusal("repository must be a concrete owner/name identity")
|
|
642
|
+
_validate_branch(fields["branch"])
|
|
643
|
+
if not COMMIT_RE.fullmatch(fields["head"]):
|
|
644
|
+
raise Refusal("head must be a 7-64 character hexadecimal commit")
|
|
645
|
+
|
|
646
|
+
sections = _memory_sections(body)
|
|
647
|
+
primary = _nonempty_section(sections["## Primary Memory"], "Primary Memory")
|
|
648
|
+
if len(primary) > 600:
|
|
649
|
+
raise Refusal("Primary Memory must be at most 600 characters")
|
|
650
|
+
_nonempty_section(sections["## Evidence"], "Evidence")
|
|
651
|
+
|
|
652
|
+
cue_lines = [line.strip() for line in sections["## Cue Anchors"] if line.strip()]
|
|
653
|
+
if cue_lines == ["- None."]:
|
|
654
|
+
cues: list[str] = []
|
|
655
|
+
else:
|
|
656
|
+
if not cue_lines or any(
|
|
657
|
+
not line.startswith("- ") or line == "- None." for line in cue_lines
|
|
658
|
+
):
|
|
659
|
+
raise Refusal("Cue Anchors must contain only bullets or '- None.'")
|
|
660
|
+
cues = [line[2:].strip() for line in cue_lines]
|
|
661
|
+
if len(cues) > MAX_CUES:
|
|
662
|
+
raise Refusal(f"Cue Anchors may contain at most {MAX_CUES} entries")
|
|
663
|
+
if any(not cue or len(cue) > 120 for cue in cues):
|
|
664
|
+
raise Refusal("each cue anchor must contain 1..120 characters")
|
|
665
|
+
_required_section(sections["## Supersedes"], "Supersedes")
|
|
666
|
+
_nonempty_section(sections["## Review Notes"], "Review Notes")
|
|
667
|
+
|
|
668
|
+
source = Path(fields["source"]).expanduser()
|
|
669
|
+
if verify_source and source.is_file():
|
|
670
|
+
actual = hashlib.sha256(source.read_bytes()).hexdigest()
|
|
671
|
+
if actual != fields["source_hash"]:
|
|
672
|
+
raise Refusal("source_hash does not match the referenced source file")
|
|
673
|
+
return {
|
|
674
|
+
**fields,
|
|
675
|
+
"artifact_hash": hashlib.sha256(raw).hexdigest(),
|
|
676
|
+
"primary_memory": primary,
|
|
677
|
+
"cue_anchors": cues,
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def validate_handoff(path: Path) -> dict[str, object]:
|
|
682
|
+
raw, text = _read_bounded(path, max_lines=MAX_HANDOFF_LINES)
|
|
683
|
+
if PLACEHOLDER_RE.search(text):
|
|
684
|
+
raise Refusal("handoff contains unresolved {{...}} template placeholders")
|
|
685
|
+
fields, body = _parse_frontmatter(text)
|
|
686
|
+
missing = [field for field in HANDOFF_FIELDS if field not in fields]
|
|
687
|
+
if missing:
|
|
688
|
+
raise Refusal(f"handoff is missing required fields: {', '.join(missing)}")
|
|
689
|
+
if fields["schema"] != "context-kit/handoff-v1":
|
|
690
|
+
raise Refusal("handoff schema must be context-kit/handoff-v1")
|
|
691
|
+
_validate_timestamp(fields["generated_at"], "generated_at")
|
|
692
|
+
for field in ("head", "base_commit"):
|
|
693
|
+
if not COMMIT_RE.fullmatch(fields[field]):
|
|
694
|
+
raise Refusal(f"{field} must be a 7-64 character hexadecimal commit")
|
|
695
|
+
if fields["worktree_state"] not in {"clean", "dirty"}:
|
|
696
|
+
raise Refusal("handoff worktree_state must be clean or dirty")
|
|
697
|
+
titles = [line.strip() for line in body if line.startswith("# ")]
|
|
698
|
+
if titles != ["# Context Handoff"]:
|
|
699
|
+
raise Refusal("handoff must contain exactly one '# Context Handoff' title")
|
|
700
|
+
sections = _handoff_sections(body)
|
|
701
|
+
for heading, lines in sections.items():
|
|
702
|
+
_required_section(lines, heading.removeprefix("## "))
|
|
703
|
+
item_count = sum(
|
|
704
|
+
bool(LIST_ITEM_RE.match(line)) for line in lines if line.strip()
|
|
705
|
+
)
|
|
706
|
+
if item_count > MAX_HANDOFF_ITEMS:
|
|
707
|
+
raise Refusal(
|
|
708
|
+
f"{heading.removeprefix('## ')} has {item_count} list items; "
|
|
709
|
+
f"maximum is {MAX_HANDOFF_ITEMS}"
|
|
710
|
+
)
|
|
711
|
+
return {**fields, "artifact_hash": hashlib.sha256(raw).hexdigest()}
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _git(repo: Path, *argv: str) -> str:
|
|
715
|
+
try:
|
|
716
|
+
result = subprocess.run(
|
|
717
|
+
["git", "-C", str(repo), *argv],
|
|
718
|
+
capture_output=True,
|
|
719
|
+
check=False,
|
|
720
|
+
text=True,
|
|
721
|
+
timeout=20.0,
|
|
722
|
+
)
|
|
723
|
+
except subprocess.TimeoutExpired as exc:
|
|
724
|
+
raise Refusal("git context check timed out") from exc
|
|
725
|
+
if result.returncode != 0:
|
|
726
|
+
error = result.stderr.strip()
|
|
727
|
+
raise Refusal(f"cannot establish repository context: {error or 'git failed'}")
|
|
728
|
+
return result.stdout.strip()
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _normalize_repository(remote: str) -> str:
|
|
732
|
+
value = remote.strip()
|
|
733
|
+
if value.startswith("git@") and ":" in value:
|
|
734
|
+
value = value.split(":", 1)[1]
|
|
735
|
+
elif "://" in value:
|
|
736
|
+
value = value.split("://", 1)[1]
|
|
737
|
+
if "@" in value.split("/", 1)[0]:
|
|
738
|
+
value = value.split("@", 1)[1]
|
|
739
|
+
value = value.split("/", 1)[1] if "/" in value else value
|
|
740
|
+
value = value.rstrip("/")
|
|
741
|
+
if value.endswith(".git"):
|
|
742
|
+
value = value[:-4]
|
|
743
|
+
parts = [part for part in value.split("/") if part]
|
|
744
|
+
if len(parts) < 2:
|
|
745
|
+
raise Refusal("cannot normalize the repository remote to owner/name")
|
|
746
|
+
return "/".join(parts[-2:])
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def _assert_project_matches(metadata: dict[str, object], config: Config) -> None:
|
|
750
|
+
project = config.project
|
|
751
|
+
if not project:
|
|
752
|
+
config.project_slug
|
|
753
|
+
if metadata["repository"] != project:
|
|
754
|
+
raise Refusal(
|
|
755
|
+
"artifact repository does not match configured memory project: "
|
|
756
|
+
f"artifact={metadata['repository']!r} project={project!r}"
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def _assert_handoff_current(metadata: dict[str, object], repo: Path) -> None:
|
|
761
|
+
root = Path(_git(repo, "rev-parse", "--show-toplevel")).resolve()
|
|
762
|
+
remote = _normalize_repository(_git(root, "remote", "get-url", "origin"))
|
|
763
|
+
branch = _git(root, "branch", "--show-current")
|
|
764
|
+
head = _git(root, "rev-parse", "HEAD")
|
|
765
|
+
base_commit = _git(root, "merge-base", "HEAD", metadata["base_ref"])
|
|
766
|
+
worktree_state = "dirty" if _git(root, "status", "--porcelain") else "clean"
|
|
767
|
+
checks = {
|
|
768
|
+
"repository": remote,
|
|
769
|
+
"branch": branch,
|
|
770
|
+
"head": head,
|
|
771
|
+
"base_commit": base_commit,
|
|
772
|
+
"worktree_state": worktree_state,
|
|
773
|
+
}
|
|
774
|
+
differences = [
|
|
775
|
+
f"{field}: saved={metadata[field]!r} current={current!r}"
|
|
776
|
+
for field, current in checks.items()
|
|
777
|
+
if metadata[field] != current
|
|
778
|
+
]
|
|
779
|
+
if differences:
|
|
780
|
+
raise Refusal(
|
|
781
|
+
"handoff is mismatched or stale; validate/resume it before archival: "
|
|
782
|
+
+ "; ".join(differences)
|
|
783
|
+
)
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def _write_once(destination: Path, raw: bytes) -> str:
|
|
787
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
788
|
+
with tempfile.NamedTemporaryFile(
|
|
789
|
+
dir=destination.parent,
|
|
790
|
+
prefix=f".{destination.name}.",
|
|
791
|
+
delete=False,
|
|
792
|
+
) as handle:
|
|
793
|
+
handle.write(raw)
|
|
794
|
+
temporary = Path(handle.name)
|
|
795
|
+
os.chmod(temporary, 0o600)
|
|
796
|
+
try:
|
|
797
|
+
os.link(temporary, destination)
|
|
798
|
+
except FileExistsError:
|
|
799
|
+
if destination.read_bytes() == raw:
|
|
800
|
+
return "unchanged"
|
|
801
|
+
raise Refusal(f"refusing to overwrite a different artifact: {destination}")
|
|
802
|
+
finally:
|
|
803
|
+
temporary.unlink(missing_ok=True)
|
|
804
|
+
return "created"
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def _utc_timestamp() -> str:
|
|
808
|
+
return datetime.now(timezone.utc).isoformat(timespec="microseconds")
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def _new_write_once_path(directory: Path, suffix: str) -> Path:
|
|
812
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
813
|
+
stamp = _utc_timestamp().replace("-", "").replace(":", "").replace("+", "p")
|
|
814
|
+
return directory / f"{stamp}-{os.getpid()}-{uuid.uuid4().hex}{suffix}"
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def _write_json_once(directory: Path, payload: dict[str, object]) -> Path:
|
|
818
|
+
raw = (json.dumps(payload, sort_keys=True, indent=2) + "\n").encode("utf-8")
|
|
819
|
+
destination = _new_write_once_path(directory, ".json")
|
|
820
|
+
if _write_once(destination, raw) != "created":
|
|
821
|
+
raise Refusal(f"refusing to reuse a generated write-once path: {destination}")
|
|
822
|
+
return destination
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def _initial_state(metadata: dict[str, object]) -> dict[str, str]:
|
|
826
|
+
return {
|
|
827
|
+
"review": str(metadata["review"]),
|
|
828
|
+
"freshness": str(metadata["freshness"]),
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def _event_paths(config: Config, record_id: str) -> list[Path]:
|
|
833
|
+
directory = config.states_path / record_id
|
|
834
|
+
paths = list(directory.glob("*.json")) if directory.exists() else []
|
|
835
|
+
sequenced = [path for path in paths if STATE_SEQUENCE_RE.fullmatch(path.name)]
|
|
836
|
+
legacy = [path for path in paths if path not in sequenced]
|
|
837
|
+
if sequenced and legacy:
|
|
838
|
+
raise Refusal(
|
|
839
|
+
"mixed legacy and sequenced state events; migrate legacy events before "
|
|
840
|
+
f"recording more state for {record_id}"
|
|
841
|
+
)
|
|
842
|
+
if sequenced:
|
|
843
|
+
return sorted(
|
|
844
|
+
sequenced, key=lambda path: int(STATE_SEQUENCE_RE.fullmatch(path.name)[1])
|
|
845
|
+
)
|
|
846
|
+
return sorted(legacy)
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
@contextlib.contextmanager
|
|
850
|
+
def _state_lock(config: Config, record_id: str):
|
|
851
|
+
"""Serialize state transitions without making the evidence artifact writable."""
|
|
852
|
+
parent = config.states_path / record_id
|
|
853
|
+
parent.mkdir(parents=True, exist_ok=True)
|
|
854
|
+
lock = parent / ".lock"
|
|
855
|
+
token = uuid.uuid4().hex
|
|
856
|
+
owner = lock / "owner.json"
|
|
857
|
+
deadline = time.monotonic() + STATE_LOCK_TIMEOUT_SECONDS
|
|
858
|
+
while True:
|
|
859
|
+
try:
|
|
860
|
+
lock.mkdir()
|
|
861
|
+
except FileExistsError:
|
|
862
|
+
if _reclaim_stale_lock(lock):
|
|
863
|
+
continue
|
|
864
|
+
if time.monotonic() >= deadline:
|
|
865
|
+
raise Refusal(f"state transition is busy for record: {record_id}")
|
|
866
|
+
time.sleep(0.02)
|
|
867
|
+
continue
|
|
868
|
+
try:
|
|
869
|
+
_write_once(
|
|
870
|
+
owner,
|
|
871
|
+
json.dumps(
|
|
872
|
+
{
|
|
873
|
+
"pid": os.getpid(),
|
|
874
|
+
"token": token,
|
|
875
|
+
"acquired_at": _utc_timestamp(),
|
|
876
|
+
},
|
|
877
|
+
sort_keys=True,
|
|
878
|
+
).encode("utf-8"),
|
|
879
|
+
)
|
|
880
|
+
except (OSError, Refusal):
|
|
881
|
+
shutil.rmtree(lock, ignore_errors=True)
|
|
882
|
+
raise
|
|
883
|
+
break
|
|
884
|
+
try:
|
|
885
|
+
yield
|
|
886
|
+
finally:
|
|
887
|
+
try:
|
|
888
|
+
payload = json.loads(owner.read_text(encoding="utf-8"))
|
|
889
|
+
except (OSError, json.JSONDecodeError):
|
|
890
|
+
payload = {}
|
|
891
|
+
if payload.get("token") == token:
|
|
892
|
+
owner.unlink(missing_ok=True)
|
|
893
|
+
lock.rmdir()
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _reclaim_stale_lock(lock: Path) -> bool:
|
|
897
|
+
"""Reclaim only a dead POSIX owner, or a conservatively old non-POSIX lock."""
|
|
898
|
+
owner = lock / "owner.json"
|
|
899
|
+
try:
|
|
900
|
+
payload = json.loads(owner.read_text(encoding="utf-8"))
|
|
901
|
+
except (OSError, json.JSONDecodeError):
|
|
902
|
+
return False
|
|
903
|
+
pid = payload.get("pid")
|
|
904
|
+
reclaim = False
|
|
905
|
+
if os.name == "posix" and isinstance(pid, int) and pid > 0:
|
|
906
|
+
try:
|
|
907
|
+
os.kill(pid, 0)
|
|
908
|
+
except ProcessLookupError:
|
|
909
|
+
reclaim = True
|
|
910
|
+
except PermissionError:
|
|
911
|
+
return False
|
|
912
|
+
elif os.name != "posix":
|
|
913
|
+
try:
|
|
914
|
+
reclaim = time.time() - lock.stat().st_mtime >= STATE_LOCK_STALE_SECONDS
|
|
915
|
+
except OSError:
|
|
916
|
+
return False
|
|
917
|
+
if not reclaim:
|
|
918
|
+
return False
|
|
919
|
+
retired = lock.with_name(f".lock-stale-{uuid.uuid4().hex}")
|
|
920
|
+
try:
|
|
921
|
+
os.replace(lock, retired)
|
|
922
|
+
except OSError:
|
|
923
|
+
return False
|
|
924
|
+
shutil.rmtree(retired, ignore_errors=True)
|
|
925
|
+
return True
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _validate_state_event(
|
|
929
|
+
path: Path,
|
|
930
|
+
*,
|
|
931
|
+
metadata: dict[str, object],
|
|
932
|
+
config: Config,
|
|
933
|
+
state: dict[str, str],
|
|
934
|
+
) -> dict[str, object]:
|
|
935
|
+
try:
|
|
936
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
937
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
938
|
+
raise Refusal(f"invalid state event {path}: {exc}") from exc
|
|
939
|
+
required = {
|
|
940
|
+
"schema",
|
|
941
|
+
"event_id",
|
|
942
|
+
"record_id",
|
|
943
|
+
"record_hash",
|
|
944
|
+
"project",
|
|
945
|
+
"project_key",
|
|
946
|
+
"timestamp",
|
|
947
|
+
"prior_review",
|
|
948
|
+
"prior_freshness",
|
|
949
|
+
"effective_review",
|
|
950
|
+
"effective_freshness",
|
|
951
|
+
"reason",
|
|
952
|
+
}
|
|
953
|
+
sequenced = STATE_SEQUENCE_RE.fullmatch(path.name)
|
|
954
|
+
expected = required | {"sequence"} if sequenced else required
|
|
955
|
+
if not isinstance(payload, dict) or set(payload) != expected:
|
|
956
|
+
raise Refusal(f"state event has an invalid schema: {path}")
|
|
957
|
+
if payload["schema"] != STATE_SCHEMA:
|
|
958
|
+
raise Refusal(f"state event has an unsupported schema: {path}")
|
|
959
|
+
if (
|
|
960
|
+
payload["record_id"] != metadata["id"]
|
|
961
|
+
or payload["record_hash"] != metadata["artifact_hash"]
|
|
962
|
+
):
|
|
963
|
+
raise Refusal(f"state event does not bind the exact record: {path}")
|
|
964
|
+
if (
|
|
965
|
+
payload["project"] != config.project
|
|
966
|
+
or payload["project_key"] != config.project_slug
|
|
967
|
+
):
|
|
968
|
+
raise Refusal(f"state event belongs to another project: {path}")
|
|
969
|
+
if not isinstance(payload["event_id"], str) or not payload["event_id"]:
|
|
970
|
+
raise Refusal(f"state event is missing an event_id: {path}")
|
|
971
|
+
if sequenced and (
|
|
972
|
+
not isinstance(payload["sequence"], int)
|
|
973
|
+
or payload["sequence"] != int(sequenced[1])
|
|
974
|
+
):
|
|
975
|
+
raise Refusal(f"state event sequence does not match its filename: {path}")
|
|
976
|
+
if not isinstance(payload["reason"], str) or not payload["reason"].strip():
|
|
977
|
+
raise Refusal(f"state event is missing a reason: {path}")
|
|
978
|
+
if len(payload["reason"]) > MAX_STATE_REASON_CHARS:
|
|
979
|
+
raise Refusal(f"state event reason is too long: {path}")
|
|
980
|
+
for key in ("timestamp",):
|
|
981
|
+
if not isinstance(payload[key], str):
|
|
982
|
+
raise Refusal(f"state event has a non-string {key}: {path}")
|
|
983
|
+
_validate_timestamp(payload[key], key)
|
|
984
|
+
for key, allowed in (
|
|
985
|
+
("prior_review", REVIEW_STATES),
|
|
986
|
+
("effective_review", REVIEW_STATES),
|
|
987
|
+
("prior_freshness", FRESHNESS_STATES),
|
|
988
|
+
("effective_freshness", FRESHNESS_STATES),
|
|
989
|
+
):
|
|
990
|
+
if not isinstance(payload[key], str) or payload[key] not in allowed:
|
|
991
|
+
raise Refusal(f"state event has invalid {key}: {path}")
|
|
992
|
+
if (
|
|
993
|
+
payload["prior_review"] != state["review"]
|
|
994
|
+
or payload["prior_freshness"] != state["freshness"]
|
|
995
|
+
):
|
|
996
|
+
raise Refusal(f"state event prior state does not match its history: {path}")
|
|
997
|
+
_validate_transition(
|
|
998
|
+
state,
|
|
999
|
+
{
|
|
1000
|
+
"review": str(payload["effective_review"]),
|
|
1001
|
+
"freshness": str(payload["effective_freshness"]),
|
|
1002
|
+
},
|
|
1003
|
+
)
|
|
1004
|
+
return payload
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def _validate_transition(previous: dict[str, str], next_state: dict[str, str]) -> None:
|
|
1008
|
+
if previous == next_state:
|
|
1009
|
+
raise Refusal("state event must change review or freshness")
|
|
1010
|
+
if (
|
|
1011
|
+
previous["review"] != next_state["review"]
|
|
1012
|
+
and next_state["review"] not in REVIEW_TRANSITIONS[previous["review"]]
|
|
1013
|
+
):
|
|
1014
|
+
raise Refusal(
|
|
1015
|
+
f"invalid review transition: {previous['review']} -> {next_state['review']}"
|
|
1016
|
+
)
|
|
1017
|
+
if (
|
|
1018
|
+
previous["freshness"] != next_state["freshness"]
|
|
1019
|
+
and next_state["freshness"] not in FRESHNESS_TRANSITIONS[previous["freshness"]]
|
|
1020
|
+
):
|
|
1021
|
+
raise Refusal(
|
|
1022
|
+
"invalid freshness transition: "
|
|
1023
|
+
f"{previous['freshness']} -> {next_state['freshness']}"
|
|
1024
|
+
)
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def effective_state(metadata: dict[str, object], config: Config) -> dict[str, str]:
|
|
1028
|
+
"""Resolve immutable initial frontmatter plus append-only state events."""
|
|
1029
|
+
state = _initial_state(metadata)
|
|
1030
|
+
for path in _event_paths(config, str(metadata["id"])):
|
|
1031
|
+
event = _validate_state_event(
|
|
1032
|
+
path, metadata=metadata, config=config, state=state
|
|
1033
|
+
)
|
|
1034
|
+
state = {
|
|
1035
|
+
"review": str(event["effective_review"]),
|
|
1036
|
+
"freshness": str(event["effective_freshness"]),
|
|
1037
|
+
}
|
|
1038
|
+
return state
|
|
1039
|
+
|
|
1040
|
+
|
|
1041
|
+
def _load_record(
|
|
1042
|
+
path: Path, config: Config
|
|
1043
|
+
) -> tuple[dict[str, object], dict[str, str]]:
|
|
1044
|
+
metadata = validate_memory(path, verify_source=False)
|
|
1045
|
+
_assert_project_matches(metadata, config)
|
|
1046
|
+
return metadata, effective_state(metadata, config)
|
|
1047
|
+
|
|
1048
|
+
|
|
1049
|
+
def _is_active(state: dict[str, str]) -> bool:
|
|
1050
|
+
return state == {"review": "accepted", "freshness": "current"}
|
|
1051
|
+
|
|
1052
|
+
|
|
1053
|
+
def _source_state(metadata: dict[str, object]) -> str:
|
|
1054
|
+
source = Path(str(metadata["source"])).expanduser()
|
|
1055
|
+
if not source.is_file():
|
|
1056
|
+
return "unavailable"
|
|
1057
|
+
actual = hashlib.sha256(source.read_bytes()).hexdigest()
|
|
1058
|
+
return "verified" if actual == metadata["source_hash"] else "drifted"
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _active_projection(
|
|
1062
|
+
config: Config,
|
|
1063
|
+
) -> tuple[list[tuple[Path, dict[str, object]]], list[dict[str, str]]]:
|
|
1064
|
+
included: list[tuple[Path, dict[str, object]]] = []
|
|
1065
|
+
excluded: list[dict[str, str]] = []
|
|
1066
|
+
for path in (
|
|
1067
|
+
sorted(config.records_path.glob("*.md")) if config.records_path.exists() else []
|
|
1068
|
+
):
|
|
1069
|
+
try:
|
|
1070
|
+
metadata, state = _load_record(path, config)
|
|
1071
|
+
if _is_active(state):
|
|
1072
|
+
included.append((path, metadata))
|
|
1073
|
+
else:
|
|
1074
|
+
excluded.append(
|
|
1075
|
+
{
|
|
1076
|
+
"id": str(metadata["id"]),
|
|
1077
|
+
"review": state["review"],
|
|
1078
|
+
"freshness": state["freshness"],
|
|
1079
|
+
}
|
|
1080
|
+
)
|
|
1081
|
+
except Refusal as exc:
|
|
1082
|
+
excluded.append({"artifact": str(path), "error": str(exc)})
|
|
1083
|
+
return included, excluded
|
|
1084
|
+
|
|
1085
|
+
|
|
1086
|
+
def _projection_hash(records: list[tuple[Path, dict[str, object]]]) -> str:
|
|
1087
|
+
digest = hashlib.sha256()
|
|
1088
|
+
for _, metadata in records:
|
|
1089
|
+
digest.update(str(metadata["id"]).encode("utf-8"))
|
|
1090
|
+
digest.update(b"\0")
|
|
1091
|
+
digest.update(str(metadata["artifact_hash"]).encode("ascii"))
|
|
1092
|
+
digest.update(b"\n")
|
|
1093
|
+
return digest.hexdigest()
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
def _ledger_hash(config: Config) -> str:
|
|
1097
|
+
"""Bind provider authority to all local record and state history, not its shape."""
|
|
1098
|
+
digest = hashlib.sha256()
|
|
1099
|
+
paths: list[Path] = []
|
|
1100
|
+
if config.records_path.exists():
|
|
1101
|
+
paths.extend(config.records_path.glob("*.md"))
|
|
1102
|
+
if config.states_path.exists():
|
|
1103
|
+
paths.extend(
|
|
1104
|
+
path
|
|
1105
|
+
for path in config.states_path.rglob("*.json")
|
|
1106
|
+
if ".lock" not in path.relative_to(config.states_path).parts
|
|
1107
|
+
)
|
|
1108
|
+
for path in sorted(paths, key=lambda item: str(item.relative_to(config.home))):
|
|
1109
|
+
digest.update(str(path.relative_to(config.home)).encode("utf-8"))
|
|
1110
|
+
digest.update(b"\0")
|
|
1111
|
+
digest.update(hashlib.sha256(path.read_bytes()).digest())
|
|
1112
|
+
digest.update(b"\n")
|
|
1113
|
+
return digest.hexdigest()
|
|
1114
|
+
|
|
1115
|
+
|
|
1116
|
+
def _materialize_projection(
|
|
1117
|
+
records: list[tuple[Path, dict[str, object]]], destination: Path
|
|
1118
|
+
) -> None:
|
|
1119
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
1120
|
+
if any(destination.iterdir()):
|
|
1121
|
+
raise Refusal(
|
|
1122
|
+
f"refusing to reuse a non-empty projection directory: {destination}"
|
|
1123
|
+
)
|
|
1124
|
+
for path, metadata in records:
|
|
1125
|
+
target = destination / f"{metadata['id']}.md"
|
|
1126
|
+
shutil.copyfile(path, target)
|
|
1127
|
+
os.chmod(target, 0o600)
|
|
1128
|
+
|
|
1129
|
+
|
|
1130
|
+
def _provider_receipt(
|
|
1131
|
+
config: Config,
|
|
1132
|
+
*,
|
|
1133
|
+
provider_version: str,
|
|
1134
|
+
operation: str,
|
|
1135
|
+
artifact_hash: str | None,
|
|
1136
|
+
argv: list[str],
|
|
1137
|
+
outcome: str,
|
|
1138
|
+
detail: str,
|
|
1139
|
+
projection_hash: str | None = None,
|
|
1140
|
+
backup_path: Path | None = None,
|
|
1141
|
+
recovery_status: str = "not-needed",
|
|
1142
|
+
) -> Path:
|
|
1143
|
+
payload: dict[str, object] = {
|
|
1144
|
+
"schema": RECEIPT_SCHEMA,
|
|
1145
|
+
"receipt_id": uuid.uuid4().hex,
|
|
1146
|
+
"timestamp": _utc_timestamp(),
|
|
1147
|
+
"provider": config.spec.name,
|
|
1148
|
+
"provider_version": provider_version,
|
|
1149
|
+
"project": config.project,
|
|
1150
|
+
"project_key": config.project_slug,
|
|
1151
|
+
"store_path": str(config.store_path),
|
|
1152
|
+
"operation": operation,
|
|
1153
|
+
"artifact_hash": artifact_hash,
|
|
1154
|
+
"projection_hash": projection_hash,
|
|
1155
|
+
"argv": argv,
|
|
1156
|
+
"outcome": outcome,
|
|
1157
|
+
"detail": detail,
|
|
1158
|
+
"backup_path": str(backup_path) if backup_path else None,
|
|
1159
|
+
"recovery_status": recovery_status,
|
|
1160
|
+
}
|
|
1161
|
+
if config.spec.name == "mempalace":
|
|
1162
|
+
# Retained for continuity with receipts written before the adapter
|
|
1163
|
+
# supported more than one provider.
|
|
1164
|
+
payload["palace_path"] = str(config.store_path)
|
|
1165
|
+
return _write_json_once(config.receipts_path, payload)
|
|
1166
|
+
|
|
1167
|
+
|
|
1168
|
+
def _provider_executable(spec: ProviderSpec) -> str:
|
|
1169
|
+
bin_envs = [spec.bin_env]
|
|
1170
|
+
if spec.legacy_bin_env:
|
|
1171
|
+
bin_envs.append(spec.legacy_bin_env)
|
|
1172
|
+
for bin_env in bin_envs:
|
|
1173
|
+
override = _first_env(bin_env)
|
|
1174
|
+
if not override:
|
|
1175
|
+
continue
|
|
1176
|
+
candidate = Path(override).expanduser()
|
|
1177
|
+
if not candidate.is_absolute():
|
|
1178
|
+
raise Refusal(f"{bin_env} must be an absolute path")
|
|
1179
|
+
if not candidate.is_file() or not os.access(candidate, os.X_OK):
|
|
1180
|
+
raise Refusal(
|
|
1181
|
+
f"configured {spec.name} executable is not runnable: {candidate}"
|
|
1182
|
+
)
|
|
1183
|
+
return str(candidate)
|
|
1184
|
+
for name in (spec.executable, *spec.legacy_executables):
|
|
1185
|
+
executable = shutil.which(name)
|
|
1186
|
+
if executable:
|
|
1187
|
+
return executable
|
|
1188
|
+
bundled = _bundled_executable(spec)
|
|
1189
|
+
if bundled is not None:
|
|
1190
|
+
return str(bundled)
|
|
1191
|
+
raise Refusal(
|
|
1192
|
+
f"{spec.name} provider selected but `{spec.executable}` is not installed; "
|
|
1193
|
+
f"{spec.install_hint}"
|
|
1194
|
+
)
|
|
1195
|
+
|
|
1196
|
+
|
|
1197
|
+
def _indexkit_root() -> Path | None:
|
|
1198
|
+
"""The sibling indexkit plugin root, when deployed alongside `memory`.
|
|
1199
|
+
|
|
1200
|
+
`memory` hard-depends on `indexkit`, so both are installed together and
|
|
1201
|
+
the dependency's launcher and bootstrap are reachable without a PATH entry
|
|
1202
|
+
or extra configuration. `local-rag` is the pre-rename directory (ADR-0007).
|
|
1203
|
+
|
|
1204
|
+
The search walks ancestors rather than indexing a fixed depth because this
|
|
1205
|
+
module has two deployment shapes (ADR-0009): `plugins/memory/src/memorykit/`
|
|
1206
|
+
inside the catalog, and `site-packages/memorykit/` from a `pip install`. A
|
|
1207
|
+
hardcoded `parents[N]` is correct for at most one of them, and the failure
|
|
1208
|
+
is silent — a wrong guess resolves to a directory that simply has no
|
|
1209
|
+
bootstrap script, which is indistinguishable from "no sibling installed".
|
|
1210
|
+
Only a directory that actually contains the launcher counts, so a packaged
|
|
1211
|
+
install finds nothing and degrades to PATH resolution, which is the right
|
|
1212
|
+
answer there.
|
|
1213
|
+
"""
|
|
1214
|
+
for ancestor in Path(__file__).resolve().parents[1:5]:
|
|
1215
|
+
for name in ("indexkit", "local-rag"):
|
|
1216
|
+
candidate = ancestor / name
|
|
1217
|
+
if (candidate / "scripts" / "bootstrap.sh").is_file():
|
|
1218
|
+
return candidate
|
|
1219
|
+
return None
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
def _bundled_executable(spec: ProviderSpec) -> Path | None:
|
|
1223
|
+
"""Resolve a sibling context-kit plugin launcher that is not on PATH."""
|
|
1224
|
+
if spec.name != "rag":
|
|
1225
|
+
return None
|
|
1226
|
+
root = _indexkit_root()
|
|
1227
|
+
if root is None:
|
|
1228
|
+
return None
|
|
1229
|
+
for name in (spec.executable, *spec.legacy_executables):
|
|
1230
|
+
candidate = root / "bin" / name
|
|
1231
|
+
if candidate.is_file() and os.access(candidate, os.X_OK):
|
|
1232
|
+
return candidate
|
|
1233
|
+
return None
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
def _rag_runtime_status(*, bootstrap: bool = False) -> dict[str, object]:
|
|
1237
|
+
"""Report whether the indexkit venv is usable, optionally building it.
|
|
1238
|
+
|
|
1239
|
+
Claude Code and GitHub Copilot CLI both bootstrap indexkit from its
|
|
1240
|
+
`SessionStart` hook, but APM does not deploy hooks and any host may have a
|
|
1241
|
+
stale or half-built venv, so readiness is checked explicitly here rather
|
|
1242
|
+
than surfacing later as an opaque provider failure. A venv built from
|
|
1243
|
+
different project metadata is reported as loudly as a missing one, because
|
|
1244
|
+
it otherwise runs stale code silently.
|
|
1245
|
+
"""
|
|
1246
|
+
root = _indexkit_root()
|
|
1247
|
+
if root is None:
|
|
1248
|
+
return {
|
|
1249
|
+
"status": "unknown",
|
|
1250
|
+
"detail": "the indexkit plugin was not found next to memory; "
|
|
1251
|
+
"runtime readiness could not be checked",
|
|
1252
|
+
}
|
|
1253
|
+
script = root / "scripts" / "bootstrap.sh"
|
|
1254
|
+
env = os.environ.copy()
|
|
1255
|
+
# Both names, for the same reason as `RagSpec.store_env`: a pre-rename
|
|
1256
|
+
# sibling ships a bootstrap that reads only `CONTEXT_KIT_LOCAL_RAG_HOME`.
|
|
1257
|
+
home = str(_indexkit_home())
|
|
1258
|
+
env["CONTEXT_KIT_INDEXKIT_HOME"] = home
|
|
1259
|
+
env["CONTEXT_KIT_LOCAL_RAG_HOME"] = home
|
|
1260
|
+
if bootstrap:
|
|
1261
|
+
try:
|
|
1262
|
+
built = subprocess.run(
|
|
1263
|
+
["bash", str(script)],
|
|
1264
|
+
capture_output=True,
|
|
1265
|
+
check=False,
|
|
1266
|
+
timeout=900.0,
|
|
1267
|
+
env=env,
|
|
1268
|
+
)
|
|
1269
|
+
except (subprocess.TimeoutExpired, OSError) as exc:
|
|
1270
|
+
raise Refusal(f"indexkit bootstrap could not run: {exc}") from exc
|
|
1271
|
+
if built.returncode != 0:
|
|
1272
|
+
detail = built.stderr.decode("utf-8", errors="replace").strip()
|
|
1273
|
+
raise Refusal(f"indexkit bootstrap failed: {detail or 'no error output'}")
|
|
1274
|
+
try:
|
|
1275
|
+
checked = subprocess.run(
|
|
1276
|
+
["bash", str(script), "--check"],
|
|
1277
|
+
capture_output=True,
|
|
1278
|
+
check=False,
|
|
1279
|
+
timeout=60.0,
|
|
1280
|
+
env=env,
|
|
1281
|
+
)
|
|
1282
|
+
except (subprocess.TimeoutExpired, OSError) as exc:
|
|
1283
|
+
return {"status": "unknown", "detail": f"readiness check failed: {exc}"}
|
|
1284
|
+
report: dict[str, object] = {}
|
|
1285
|
+
for line in checked.stdout.decode("utf-8", errors="replace").splitlines():
|
|
1286
|
+
key, separator, value = line.partition("=")
|
|
1287
|
+
if separator:
|
|
1288
|
+
report[key.strip()] = value.strip()
|
|
1289
|
+
report.setdefault("status", "unknown")
|
|
1290
|
+
return report
|
|
1291
|
+
|
|
1292
|
+
|
|
1293
|
+
def _probe_capability(
|
|
1294
|
+
executable: str,
|
|
1295
|
+
config: Config,
|
|
1296
|
+
spec: ProviderSpec,
|
|
1297
|
+
probe: CapabilityProbe,
|
|
1298
|
+
*,
|
|
1299
|
+
timeout: float = 10.0,
|
|
1300
|
+
) -> dict[str, object]:
|
|
1301
|
+
report: dict[str, object] = {
|
|
1302
|
+
"name": probe.name,
|
|
1303
|
+
"command": " ".join(probe.argv),
|
|
1304
|
+
"contract": probe.contract,
|
|
1305
|
+
}
|
|
1306
|
+
try:
|
|
1307
|
+
result = subprocess.run(
|
|
1308
|
+
[executable, *probe.argv],
|
|
1309
|
+
capture_output=True,
|
|
1310
|
+
check=False,
|
|
1311
|
+
timeout=timeout,
|
|
1312
|
+
env=_provider_env(config),
|
|
1313
|
+
)
|
|
1314
|
+
except subprocess.TimeoutExpired:
|
|
1315
|
+
report["status"] = "timeout"
|
|
1316
|
+
report["detail"] = f"probe timed out after {timeout:g}s"
|
|
1317
|
+
return report
|
|
1318
|
+
except OSError as exc:
|
|
1319
|
+
report["status"] = "error"
|
|
1320
|
+
report["detail"] = str(exc)
|
|
1321
|
+
return report
|
|
1322
|
+
output = "\n".join(
|
|
1323
|
+
(
|
|
1324
|
+
result.stdout.decode("utf-8", errors="replace"),
|
|
1325
|
+
result.stderr.decode("utf-8", errors="replace"),
|
|
1326
|
+
)
|
|
1327
|
+
)
|
|
1328
|
+
if result.returncode != 0:
|
|
1329
|
+
report["status"] = "missing"
|
|
1330
|
+
report["detail"] = f"`{' '.join(probe.argv)}` exited {result.returncode}"
|
|
1331
|
+
return report
|
|
1332
|
+
missing_tokens = [token for token in probe.required_tokens if token not in output]
|
|
1333
|
+
if missing_tokens:
|
|
1334
|
+
report["status"] = "incompatible"
|
|
1335
|
+
report["detail"] = f"missing option(s): {', '.join(missing_tokens)}"
|
|
1336
|
+
return report
|
|
1337
|
+
report["status"] = "ok"
|
|
1338
|
+
return report
|
|
1339
|
+
|
|
1340
|
+
|
|
1341
|
+
def _parse_provider_version(raw: str) -> tuple[int, int, int] | None:
|
|
1342
|
+
match = SEMVER_RE.search(raw)
|
|
1343
|
+
if not match:
|
|
1344
|
+
return None
|
|
1345
|
+
major, minor, patch = (int(part) for part in match.groups())
|
|
1346
|
+
return (major, minor, patch)
|
|
1347
|
+
|
|
1348
|
+
|
|
1349
|
+
def _provider_version_status(
|
|
1350
|
+
parsed: tuple[int, int, int] | None, spec: ProviderSpec
|
|
1351
|
+
) -> str:
|
|
1352
|
+
if parsed is None:
|
|
1353
|
+
return "unknown"
|
|
1354
|
+
if parsed[:2] == spec.tested_version[:2]:
|
|
1355
|
+
return "tested"
|
|
1356
|
+
if parsed[:2] < spec.tested_version[:2]:
|
|
1357
|
+
return "older-than-tested"
|
|
1358
|
+
return "newer-than-tested"
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
def _provider_env(config: Config, store_path: Path | None = None) -> dict[str, str]:
|
|
1362
|
+
env = os.environ.copy()
|
|
1363
|
+
path = store_path or config.store_path
|
|
1364
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1365
|
+
env.update(config.spec.store_env(path))
|
|
1366
|
+
return env
|
|
1367
|
+
|
|
1368
|
+
|
|
1369
|
+
def _run_provider(
|
|
1370
|
+
config: Config,
|
|
1371
|
+
argv: list[str],
|
|
1372
|
+
*,
|
|
1373
|
+
input_bytes: bytes | None = None,
|
|
1374
|
+
timeout: float = 120.0,
|
|
1375
|
+
store_path: Path | None = None,
|
|
1376
|
+
) -> subprocess.CompletedProcess[bytes]:
|
|
1377
|
+
spec = config.spec
|
|
1378
|
+
executable = _provider_executable(spec)
|
|
1379
|
+
try:
|
|
1380
|
+
result = subprocess.run(
|
|
1381
|
+
[executable, *argv],
|
|
1382
|
+
input=input_bytes,
|
|
1383
|
+
capture_output=True,
|
|
1384
|
+
check=False,
|
|
1385
|
+
timeout=timeout,
|
|
1386
|
+
env=_provider_env(config, store_path),
|
|
1387
|
+
)
|
|
1388
|
+
except subprocess.TimeoutExpired as exc:
|
|
1389
|
+
raise Refusal(f"{spec.name} command timed out after {timeout:g}s") from exc
|
|
1390
|
+
if result.returncode != 0:
|
|
1391
|
+
error = result.stderr.decode("utf-8", errors="replace").strip()
|
|
1392
|
+
raise Refusal(
|
|
1393
|
+
f"{spec.name} exited {result.returncode}: {error or 'no error output'}"
|
|
1394
|
+
)
|
|
1395
|
+
return result
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def _write_stdout(raw: bytes) -> None:
|
|
1399
|
+
sys.stdout.write(raw.decode("utf-8", errors="replace"))
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
def _provider_version(config: Config) -> tuple[str, str]:
|
|
1403
|
+
executable = _provider_executable(config.spec)
|
|
1404
|
+
version = _run_provider(config, ["--version"], timeout=20.0)
|
|
1405
|
+
return executable, version.stdout.decode("utf-8", errors="replace").strip()
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def _capture_memory(args: argparse.Namespace, config: Config) -> int:
|
|
1409
|
+
source = Path(args.artifact).expanduser().resolve()
|
|
1410
|
+
metadata = validate_memory(source)
|
|
1411
|
+
_assert_project_matches(metadata, config)
|
|
1412
|
+
raw = source.read_bytes()
|
|
1413
|
+
destination = config.records_path / f"{metadata['id']}.md"
|
|
1414
|
+
state = _write_once(destination, raw)
|
|
1415
|
+
archived = False
|
|
1416
|
+
archive_reason = "provider not selected"
|
|
1417
|
+
archive_outcome = "not-selected"
|
|
1418
|
+
receipt: Path | None = None
|
|
1419
|
+
effective = effective_state(metadata, config)
|
|
1420
|
+
if config.provider != "none" and not args.local_only:
|
|
1421
|
+
if _is_active(effective):
|
|
1422
|
+
archive_reason = (
|
|
1423
|
+
"eligible for provider indexing but pending explicit "
|
|
1424
|
+
"sync-provider --apply"
|
|
1425
|
+
)
|
|
1426
|
+
archive_outcome = "pending-sync"
|
|
1427
|
+
receipt = _provider_receipt(
|
|
1428
|
+
config,
|
|
1429
|
+
operation="capture",
|
|
1430
|
+
provider_version="not-invoked",
|
|
1431
|
+
artifact_hash=str(metadata["artifact_hash"]),
|
|
1432
|
+
argv=[],
|
|
1433
|
+
outcome="pending-sync",
|
|
1434
|
+
detail=archive_reason,
|
|
1435
|
+
)
|
|
1436
|
+
else:
|
|
1437
|
+
archive_reason = (
|
|
1438
|
+
"not provider eligible; explicit sync will exclude this record "
|
|
1439
|
+
f"(review={effective['review']}, freshness={effective['freshness']})"
|
|
1440
|
+
)
|
|
1441
|
+
archive_outcome = "skipped"
|
|
1442
|
+
receipt = _provider_receipt(
|
|
1443
|
+
config,
|
|
1444
|
+
provider_version="not-invoked",
|
|
1445
|
+
operation="capture",
|
|
1446
|
+
artifact_hash=str(metadata["artifact_hash"]),
|
|
1447
|
+
argv=[],
|
|
1448
|
+
outcome="skipped",
|
|
1449
|
+
detail=archive_reason,
|
|
1450
|
+
)
|
|
1451
|
+
elif args.local_only:
|
|
1452
|
+
archive_reason = "skipped: --local-only"
|
|
1453
|
+
archive_outcome = "skipped"
|
|
1454
|
+
print(
|
|
1455
|
+
json.dumps(
|
|
1456
|
+
{
|
|
1457
|
+
"status": state,
|
|
1458
|
+
"artifact": str(destination),
|
|
1459
|
+
"project": config.project_slug,
|
|
1460
|
+
"provider": config.provider,
|
|
1461
|
+
"provider_archived": archived,
|
|
1462
|
+
"provider_archive": {
|
|
1463
|
+
"outcome": archive_outcome,
|
|
1464
|
+
"reason": archive_reason,
|
|
1465
|
+
"receipt": str(receipt) if receipt else None,
|
|
1466
|
+
},
|
|
1467
|
+
"provider_reconciliation": (
|
|
1468
|
+
"required: run sync-provider --apply"
|
|
1469
|
+
if config.provider != "none" and _is_active(effective)
|
|
1470
|
+
else "not-required"
|
|
1471
|
+
),
|
|
1472
|
+
"effective_state": effective,
|
|
1473
|
+
}
|
|
1474
|
+
)
|
|
1475
|
+
)
|
|
1476
|
+
return 0
|
|
1477
|
+
|
|
1478
|
+
|
|
1479
|
+
def _archive_handoff(args: argparse.Namespace, config: Config) -> int:
|
|
1480
|
+
source = Path(args.artifact).expanduser().resolve()
|
|
1481
|
+
metadata = validate_handoff(source)
|
|
1482
|
+
_assert_project_matches(metadata, config)
|
|
1483
|
+
_assert_handoff_current(metadata, Path(args.repo).expanduser().resolve())
|
|
1484
|
+
raw = source.read_bytes()
|
|
1485
|
+
name = (
|
|
1486
|
+
f"handoff-{metadata['generated_at'][:10]}-{metadata['artifact_hash'][:12]}.md"
|
|
1487
|
+
)
|
|
1488
|
+
destination = config.home / "handoffs" / config.project_slug / name
|
|
1489
|
+
state = _write_once(destination, raw)
|
|
1490
|
+
archived = False
|
|
1491
|
+
archive_reason = "provider not selected"
|
|
1492
|
+
receipt: Path | None = None
|
|
1493
|
+
if config.provider != "none" and not args.local_only:
|
|
1494
|
+
archive_reason = (
|
|
1495
|
+
"skipped: handoffs are local historical evidence, not active memory"
|
|
1496
|
+
)
|
|
1497
|
+
receipt = _provider_receipt(
|
|
1498
|
+
config,
|
|
1499
|
+
provider_version="not-invoked",
|
|
1500
|
+
operation="archive-handoff",
|
|
1501
|
+
artifact_hash=str(metadata["artifact_hash"]),
|
|
1502
|
+
argv=[],
|
|
1503
|
+
outcome="skipped",
|
|
1504
|
+
detail=archive_reason,
|
|
1505
|
+
)
|
|
1506
|
+
elif args.local_only:
|
|
1507
|
+
archive_reason = "skipped: --local-only"
|
|
1508
|
+
print(
|
|
1509
|
+
json.dumps(
|
|
1510
|
+
{
|
|
1511
|
+
"status": state,
|
|
1512
|
+
"artifact": str(destination),
|
|
1513
|
+
"project": config.project_slug,
|
|
1514
|
+
"provider": config.provider,
|
|
1515
|
+
"provider_archived": archived,
|
|
1516
|
+
"provider_archive": {
|
|
1517
|
+
"outcome": "skipped",
|
|
1518
|
+
"reason": archive_reason,
|
|
1519
|
+
"receipt": str(receipt) if receipt else None,
|
|
1520
|
+
},
|
|
1521
|
+
"provider_reconciliation": "not-required: handoffs are not active memory",
|
|
1522
|
+
"saved_head": metadata["head"],
|
|
1523
|
+
}
|
|
1524
|
+
)
|
|
1525
|
+
)
|
|
1526
|
+
return 0
|
|
1527
|
+
|
|
1528
|
+
|
|
1529
|
+
def _local_search(
|
|
1530
|
+
args: argparse.Namespace,
|
|
1531
|
+
config: Config,
|
|
1532
|
+
*,
|
|
1533
|
+
annotations: dict[str, object] | None = None,
|
|
1534
|
+
) -> int:
|
|
1535
|
+
terms = {term.lower() for term in TOKEN_RE.findall(args.query)}
|
|
1536
|
+
if not terms:
|
|
1537
|
+
raise Refusal("search query must contain at least one searchable term")
|
|
1538
|
+
results: list[dict[str, object]] = []
|
|
1539
|
+
invalid_records: list[dict[str, str]] = []
|
|
1540
|
+
inactive_records: list[dict[str, str]] = []
|
|
1541
|
+
for path in (
|
|
1542
|
+
sorted(config.records_path.glob("*.md")) if config.records_path.exists() else []
|
|
1543
|
+
):
|
|
1544
|
+
try:
|
|
1545
|
+
metadata, state = _load_record(path, config)
|
|
1546
|
+
except Refusal as exc:
|
|
1547
|
+
invalid_records.append({"artifact": str(path), "error": str(exc)})
|
|
1548
|
+
continue
|
|
1549
|
+
if not _is_active(state) and not args.include_inactive:
|
|
1550
|
+
inactive_records.append(
|
|
1551
|
+
{
|
|
1552
|
+
"id": str(metadata["id"]),
|
|
1553
|
+
"review": state["review"],
|
|
1554
|
+
"freshness": state["freshness"],
|
|
1555
|
+
}
|
|
1556
|
+
)
|
|
1557
|
+
continue
|
|
1558
|
+
primary = metadata["primary_memory"]
|
|
1559
|
+
cues = metadata["cue_anchors"]
|
|
1560
|
+
primary_text = primary.lower()
|
|
1561
|
+
cue_text = " ".join(cues).lower()
|
|
1562
|
+
source_state = _source_state(metadata)
|
|
1563
|
+
primary_matches = sum(term in primary_text for term in terms)
|
|
1564
|
+
cue_matches = sum(term in cue_text for term in terms)
|
|
1565
|
+
score = primary_matches * 2 + cue_matches
|
|
1566
|
+
if score:
|
|
1567
|
+
results.append(
|
|
1568
|
+
{
|
|
1569
|
+
"id": metadata["id"],
|
|
1570
|
+
"type": metadata["type"],
|
|
1571
|
+
"freshness": state["freshness"],
|
|
1572
|
+
"review": state["review"],
|
|
1573
|
+
"primary_memory": primary,
|
|
1574
|
+
"cue_anchors": cues,
|
|
1575
|
+
"source": metadata["source"],
|
|
1576
|
+
"source_hash": metadata["source_hash"],
|
|
1577
|
+
"source_state": source_state,
|
|
1578
|
+
"score": score,
|
|
1579
|
+
}
|
|
1580
|
+
)
|
|
1581
|
+
results.sort(key=lambda item: (-int(item["score"]), str(item["id"])))
|
|
1582
|
+
payload: dict[str, object] = {
|
|
1583
|
+
"provider": "local",
|
|
1584
|
+
"project": config.project_slug,
|
|
1585
|
+
"records": results[: args.results],
|
|
1586
|
+
"invalid_records": invalid_records,
|
|
1587
|
+
"inactive_records": inactive_records,
|
|
1588
|
+
"include_inactive": args.include_inactive,
|
|
1589
|
+
}
|
|
1590
|
+
# A degraded provider search annotates the local result rather than
|
|
1591
|
+
# silently presenting lexical hits as semantic recall.
|
|
1592
|
+
payload.update(annotations or {})
|
|
1593
|
+
print(json.dumps(payload))
|
|
1594
|
+
return 0
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
def _provider_search(args: argparse.Namespace, config: Config) -> int:
|
|
1598
|
+
"""Run a reconciled provider query. MemPalace streams; rag is enriched."""
|
|
1599
|
+
spec = config.spec
|
|
1600
|
+
if spec.name != "rag":
|
|
1601
|
+
result = _run_provider(config, spec.search_argv(args.query, args.results))
|
|
1602
|
+
_write_stdout(result.stdout)
|
|
1603
|
+
return 0
|
|
1604
|
+
|
|
1605
|
+
records, _ = _active_projection(config)
|
|
1606
|
+
by_id = {str(metadata["id"]): metadata for _, metadata in records}
|
|
1607
|
+
# rag indexes each markdown section separately, so one record yields hits
|
|
1608
|
+
# for Primary Memory, Cue Anchors, Evidence, and so on. Over-fetch chunks
|
|
1609
|
+
# so that deduplicating back to records can still fill the requested
|
|
1610
|
+
# number of *records* rather than being consumed by one verbose memory.
|
|
1611
|
+
chunk_budget = min(50, max(args.results * RAG_CHUNKS_PER_RECORD, args.results))
|
|
1612
|
+
result = _run_provider(config, spec.search_argv(args.query, chunk_budget))
|
|
1613
|
+
raw = result.stdout.decode("utf-8", errors="replace").strip()
|
|
1614
|
+
try:
|
|
1615
|
+
hits = json.loads(raw) if raw else []
|
|
1616
|
+
except json.JSONDecodeError as exc:
|
|
1617
|
+
raise Refusal(f"rag returned output that is not valid JSON: {exc}") from exc
|
|
1618
|
+
if not isinstance(hits, list):
|
|
1619
|
+
raise Refusal("rag returned an unexpected JSON shape; expected a list")
|
|
1620
|
+
|
|
1621
|
+
found: list[dict[str, object]] = []
|
|
1622
|
+
seen: dict[str, dict[str, object]] = {}
|
|
1623
|
+
unmatched: list[str] = []
|
|
1624
|
+
for hit in hits:
|
|
1625
|
+
if not isinstance(hit, dict):
|
|
1626
|
+
continue
|
|
1627
|
+
# The projection materializes each record as `<record-id>.md`, so a hit
|
|
1628
|
+
# path maps back to exactly one record id.
|
|
1629
|
+
path = str(hit.get("path", ""))
|
|
1630
|
+
record_id = path[:-3] if path.endswith(".md") else path
|
|
1631
|
+
metadata = by_id.get(record_id)
|
|
1632
|
+
if metadata is None:
|
|
1633
|
+
if path not in unmatched:
|
|
1634
|
+
unmatched.append(path)
|
|
1635
|
+
continue
|
|
1636
|
+
if record_id in seen:
|
|
1637
|
+
# Keep the strongest chunk for a record; count the rest.
|
|
1638
|
+
entry = seen[record_id]
|
|
1639
|
+
entry["matched_chunks"] = int(entry["matched_chunks"]) + 1
|
|
1640
|
+
continue
|
|
1641
|
+
state = effective_state(metadata, config)
|
|
1642
|
+
entry = {
|
|
1643
|
+
"id": metadata["id"],
|
|
1644
|
+
"type": metadata["type"],
|
|
1645
|
+
"freshness": state["freshness"],
|
|
1646
|
+
"review": state["review"],
|
|
1647
|
+
"primary_memory": metadata["primary_memory"],
|
|
1648
|
+
"cue_anchors": metadata["cue_anchors"],
|
|
1649
|
+
"source": metadata["source"],
|
|
1650
|
+
"source_hash": metadata["source_hash"],
|
|
1651
|
+
"source_state": _source_state(metadata),
|
|
1652
|
+
"score": hit.get("score"),
|
|
1653
|
+
"retrieval_mode": hit.get("retrieval_mode"),
|
|
1654
|
+
"heading": hit.get("heading"),
|
|
1655
|
+
"matched_chunks": 1,
|
|
1656
|
+
}
|
|
1657
|
+
seen[record_id] = entry
|
|
1658
|
+
found.append(entry)
|
|
1659
|
+
if len(found) >= args.results:
|
|
1660
|
+
break
|
|
1661
|
+
print(
|
|
1662
|
+
json.dumps(
|
|
1663
|
+
{
|
|
1664
|
+
"provider": spec.name,
|
|
1665
|
+
"project": config.project_slug,
|
|
1666
|
+
"records": found,
|
|
1667
|
+
# A hit the adapter cannot bind to a current record is reported,
|
|
1668
|
+
# never dropped: it means the index is ahead of the ledger.
|
|
1669
|
+
"unmatched_hits": sorted(unmatched),
|
|
1670
|
+
"include_inactive": False,
|
|
1671
|
+
}
|
|
1672
|
+
)
|
|
1673
|
+
)
|
|
1674
|
+
return 0
|
|
1675
|
+
|
|
1676
|
+
|
|
1677
|
+
def _search(args: argparse.Namespace, config: Config) -> int:
|
|
1678
|
+
if config.provider == "none":
|
|
1679
|
+
return _local_search(args, config)
|
|
1680
|
+
if args.include_inactive:
|
|
1681
|
+
raise Refusal(
|
|
1682
|
+
"--include-inactive is available only for local audit search; "
|
|
1683
|
+
"the provider index is active-only"
|
|
1684
|
+
)
|
|
1685
|
+
records, _ = _active_projection(config)
|
|
1686
|
+
projection_hash = _projection_hash(records)
|
|
1687
|
+
# Reconciliation is a correctness gate, not an availability problem, so it
|
|
1688
|
+
# refuses outright and is deliberately outside the degradation path below.
|
|
1689
|
+
if not _has_current_provider_projection(config, projection_hash):
|
|
1690
|
+
raise Refusal(
|
|
1691
|
+
"provider index is not reconciled with accepted/current records; "
|
|
1692
|
+
"run sync-provider --apply or use --provider none for local recall"
|
|
1693
|
+
)
|
|
1694
|
+
try:
|
|
1695
|
+
return _provider_search(args, config)
|
|
1696
|
+
except Refusal as exc:
|
|
1697
|
+
return _local_search(
|
|
1698
|
+
args,
|
|
1699
|
+
config,
|
|
1700
|
+
annotations={
|
|
1701
|
+
"degraded_from": config.provider,
|
|
1702
|
+
"degraded_reason": str(exc),
|
|
1703
|
+
"degraded_detail": (
|
|
1704
|
+
"semantic recall was unavailable; these are lexical matches "
|
|
1705
|
+
"over primary memories and cue anchors only"
|
|
1706
|
+
),
|
|
1707
|
+
},
|
|
1708
|
+
)
|
|
1709
|
+
|
|
1710
|
+
|
|
1711
|
+
def _wake_digest(config: Config) -> dict[str, object]:
|
|
1712
|
+
"""Build a bounded session-priming digest from local records.
|
|
1713
|
+
|
|
1714
|
+
Provider-neutral by construction: records are the system of record and a
|
|
1715
|
+
provider store is a rebuildable projection of them, so the digest is
|
|
1716
|
+
identical whether the configured provider is `none`, `rag`, or
|
|
1717
|
+
`mempalace`. That also makes it a stable payload for lifecycle hooks.
|
|
1718
|
+
"""
|
|
1719
|
+
records, _ = _active_projection(config)
|
|
1720
|
+
inactive = 0
|
|
1721
|
+
if config.records_path.exists():
|
|
1722
|
+
for path in sorted(config.records_path.glob("*.md")):
|
|
1723
|
+
try:
|
|
1724
|
+
metadata, state = _load_record(path, config)
|
|
1725
|
+
except Refusal:
|
|
1726
|
+
continue
|
|
1727
|
+
if not _is_active(state):
|
|
1728
|
+
inactive += 1
|
|
1729
|
+
|
|
1730
|
+
# Most recently observed first; ties break on id so output is stable.
|
|
1731
|
+
ordered = sorted(
|
|
1732
|
+
records,
|
|
1733
|
+
key=lambda item: (str(item[1]["observed_at"]), str(item[1]["id"])),
|
|
1734
|
+
reverse=True,
|
|
1735
|
+
)
|
|
1736
|
+
by_type: dict[str, int] = {}
|
|
1737
|
+
for _, metadata in ordered:
|
|
1738
|
+
kind = str(metadata["type"])
|
|
1739
|
+
by_type[kind] = by_type.get(kind, 0) + 1
|
|
1740
|
+
|
|
1741
|
+
surfaced: list[dict[str, object]] = []
|
|
1742
|
+
attention: list[dict[str, str]] = []
|
|
1743
|
+
used = 0
|
|
1744
|
+
truncated = False
|
|
1745
|
+
for _, metadata in ordered:
|
|
1746
|
+
if len(surfaced) >= MAX_WAKE_RECORDS:
|
|
1747
|
+
truncated = True
|
|
1748
|
+
break
|
|
1749
|
+
primary = str(metadata["primary_memory"])
|
|
1750
|
+
cost = len(primary) + len(str(metadata["id"])) + len(str(metadata["type"])) + 8
|
|
1751
|
+
if used + cost > MAX_WAKE_CHARS and surfaced:
|
|
1752
|
+
truncated = True
|
|
1753
|
+
break
|
|
1754
|
+
used += cost
|
|
1755
|
+
# Drift is only computed for surfaced records: hashing every source on
|
|
1756
|
+
# every session start would be unbounded work. `audit` sweeps the store.
|
|
1757
|
+
state = _source_state(metadata)
|
|
1758
|
+
if state != "verified":
|
|
1759
|
+
attention.append(
|
|
1760
|
+
{
|
|
1761
|
+
"id": str(metadata["id"]),
|
|
1762
|
+
"issue": state,
|
|
1763
|
+
"source": str(metadata["source"]),
|
|
1764
|
+
}
|
|
1765
|
+
)
|
|
1766
|
+
surfaced.append(
|
|
1767
|
+
{
|
|
1768
|
+
"id": metadata["id"],
|
|
1769
|
+
"type": metadata["type"],
|
|
1770
|
+
"primary_memory": primary,
|
|
1771
|
+
"observed_at": metadata["observed_at"],
|
|
1772
|
+
"source": metadata["source"],
|
|
1773
|
+
"source_state": state,
|
|
1774
|
+
}
|
|
1775
|
+
)
|
|
1776
|
+
|
|
1777
|
+
lines: list[str] = []
|
|
1778
|
+
if surfaced:
|
|
1779
|
+
lines.append(f"## Durable project memory — {config.project}")
|
|
1780
|
+
lines.append("")
|
|
1781
|
+
lines.append(
|
|
1782
|
+
f"{len(records)} reviewed memor{'y' if len(records) == 1 else 'ies'} "
|
|
1783
|
+
"for this project. These are leads, not current truth: open the "
|
|
1784
|
+
"cited source before relying on one."
|
|
1785
|
+
)
|
|
1786
|
+
lines.append("")
|
|
1787
|
+
for entry in surfaced:
|
|
1788
|
+
flag = "" if entry["source_state"] == "verified" else " [source changed]"
|
|
1789
|
+
lines.append(
|
|
1790
|
+
f"- [{entry['type']}] {entry['primary_memory']} "
|
|
1791
|
+
f"({entry['id']}){flag}"
|
|
1792
|
+
)
|
|
1793
|
+
if truncated:
|
|
1794
|
+
lines.append("")
|
|
1795
|
+
lines.append(
|
|
1796
|
+
f"Showing {len(surfaced)} of {len(records)}; "
|
|
1797
|
+
"run `search` for the rest."
|
|
1798
|
+
)
|
|
1799
|
+
if attention:
|
|
1800
|
+
lines.append("")
|
|
1801
|
+
lines.append(
|
|
1802
|
+
f"{len(attention)} surfaced record(s) cite a source that changed "
|
|
1803
|
+
"or is missing. Run `audit` before trusting them."
|
|
1804
|
+
)
|
|
1805
|
+
|
|
1806
|
+
digest: dict[str, object] = {
|
|
1807
|
+
"schema": WAKE_SCHEMA,
|
|
1808
|
+
"project": config.project,
|
|
1809
|
+
"project_key": config.project_slug,
|
|
1810
|
+
"provider": config.provider,
|
|
1811
|
+
"generated_at": _utc_timestamp(),
|
|
1812
|
+
"counts": {
|
|
1813
|
+
"active": len(records),
|
|
1814
|
+
"inactive": inactive,
|
|
1815
|
+
"by_type": by_type,
|
|
1816
|
+
"surfaced": len(surfaced),
|
|
1817
|
+
},
|
|
1818
|
+
"memories": surfaced,
|
|
1819
|
+
"attention": attention,
|
|
1820
|
+
"truncated": truncated,
|
|
1821
|
+
"context": "\n".join(lines),
|
|
1822
|
+
}
|
|
1823
|
+
if config.provider != "none":
|
|
1824
|
+
digest["reconciled"] = _has_current_provider_projection(
|
|
1825
|
+
config, _projection_hash(records)
|
|
1826
|
+
)
|
|
1827
|
+
return digest
|
|
1828
|
+
|
|
1829
|
+
|
|
1830
|
+
def _wake(args: argparse.Namespace, config: Config) -> int:
|
|
1831
|
+
digest = _wake_digest(config)
|
|
1832
|
+
if getattr(args, "format", "json") == "text":
|
|
1833
|
+
context = str(digest["context"])
|
|
1834
|
+
if context:
|
|
1835
|
+
print(context)
|
|
1836
|
+
return 0
|
|
1837
|
+
print(json.dumps(digest))
|
|
1838
|
+
return 0
|
|
1839
|
+
|
|
1840
|
+
|
|
1841
|
+
def _audit(args: argparse.Namespace, config: Config) -> int:
|
|
1842
|
+
"""Sweep every record and report source drift. Proposes; never deletes.
|
|
1843
|
+
|
|
1844
|
+
MemPalace's `sync` prunes drawers whose sources are gitignored, deleted, or
|
|
1845
|
+
moved. Deleting is wrong here: a record's evidence is the reason it can be
|
|
1846
|
+
trusted later, and a moved file is not proof the decision was wrong. This
|
|
1847
|
+
reports drift and suggests a state transition the reviewer can apply with
|
|
1848
|
+
`record-state`.
|
|
1849
|
+
"""
|
|
1850
|
+
findings: list[dict[str, object]] = []
|
|
1851
|
+
invalid: list[dict[str, str]] = []
|
|
1852
|
+
counts = {"verified": 0, "drifted": 0, "unavailable": 0, "invalid": 0}
|
|
1853
|
+
paths = (
|
|
1854
|
+
sorted(config.records_path.glob("*.md")) if config.records_path.exists() else []
|
|
1855
|
+
)
|
|
1856
|
+
for path in paths:
|
|
1857
|
+
try:
|
|
1858
|
+
metadata, state = _load_record(path, config)
|
|
1859
|
+
except Refusal as exc:
|
|
1860
|
+
counts["invalid"] += 1
|
|
1861
|
+
invalid.append({"artifact": str(path), "error": str(exc)})
|
|
1862
|
+
continue
|
|
1863
|
+
source_state = _source_state(metadata)
|
|
1864
|
+
counts[source_state] = counts.get(source_state, 0) + 1
|
|
1865
|
+
if source_state == "verified":
|
|
1866
|
+
continue
|
|
1867
|
+
active = _is_active(state)
|
|
1868
|
+
findings.append(
|
|
1869
|
+
{
|
|
1870
|
+
"id": metadata["id"],
|
|
1871
|
+
"source": metadata["source"],
|
|
1872
|
+
"source_state": source_state,
|
|
1873
|
+
"review": state["review"],
|
|
1874
|
+
"freshness": state["freshness"],
|
|
1875
|
+
"active": active,
|
|
1876
|
+
# Only an active record misleads recall, so only it needs action.
|
|
1877
|
+
"suggested": (
|
|
1878
|
+
"record-state {id} --freshness stale --reason "
|
|
1879
|
+
'"Cited source {issue}; re-verify before relying on it."'.format(
|
|
1880
|
+
id=metadata["id"],
|
|
1881
|
+
issue=(
|
|
1882
|
+
"changed since capture"
|
|
1883
|
+
if source_state == "drifted"
|
|
1884
|
+
else "is no longer present"
|
|
1885
|
+
),
|
|
1886
|
+
)
|
|
1887
|
+
if active
|
|
1888
|
+
else "none: already inactive"
|
|
1889
|
+
),
|
|
1890
|
+
}
|
|
1891
|
+
)
|
|
1892
|
+
findings.sort(key=lambda item: (not item["active"], str(item["id"])))
|
|
1893
|
+
actionable = [f for f in findings if f["active"]]
|
|
1894
|
+
print(
|
|
1895
|
+
json.dumps(
|
|
1896
|
+
{
|
|
1897
|
+
"project": config.project_slug,
|
|
1898
|
+
"records_examined": len(paths),
|
|
1899
|
+
"counts": counts,
|
|
1900
|
+
"findings": findings[: args.limit],
|
|
1901
|
+
"actionable": len(actionable),
|
|
1902
|
+
"invalid_records": invalid,
|
|
1903
|
+
"truncated": len(findings) > args.limit,
|
|
1904
|
+
"note": (
|
|
1905
|
+
"audit reports and proposes; it never edits or deletes a "
|
|
1906
|
+
"record. Apply a suggestion with `record-state`, then "
|
|
1907
|
+
"re-run `sync-provider --apply` if a provider is configured."
|
|
1908
|
+
),
|
|
1909
|
+
}
|
|
1910
|
+
)
|
|
1911
|
+
)
|
|
1912
|
+
return 0
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _doctor(args: argparse.Namespace, config: Config) -> int:
|
|
1916
|
+
result: dict[str, object] = {
|
|
1917
|
+
"provider": config.provider,
|
|
1918
|
+
"home": str(config.home),
|
|
1919
|
+
"project": config.project,
|
|
1920
|
+
"auto_capture": config.auto_capture,
|
|
1921
|
+
}
|
|
1922
|
+
if config.provider == "none":
|
|
1923
|
+
result.update(
|
|
1924
|
+
{
|
|
1925
|
+
"status": "ready",
|
|
1926
|
+
"mode": "local",
|
|
1927
|
+
"records_path": str(config.home / "records" / config.project_slug),
|
|
1928
|
+
}
|
|
1929
|
+
)
|
|
1930
|
+
print(json.dumps(result))
|
|
1931
|
+
return 0
|
|
1932
|
+
config.project_slug
|
|
1933
|
+
spec = config.spec
|
|
1934
|
+
executable = _provider_executable(spec)
|
|
1935
|
+
if spec.name == "rag":
|
|
1936
|
+
# The venv only backs the bundled `bin/rag` launcher. A user-supplied
|
|
1937
|
+
# executable (CONTEXT_KIT_INDEXKIT_BIN or one on PATH) manages its own
|
|
1938
|
+
# runtime, so gating on our bootstrap would be wrong there.
|
|
1939
|
+
bundled = _bundled_executable(spec)
|
|
1940
|
+
if bundled is not None and str(bundled) == executable:
|
|
1941
|
+
# Checked before the version and capability probes: without a
|
|
1942
|
+
# usable venv those fail with an opaque launcher error instead of
|
|
1943
|
+
# the actionable "run the bootstrap" answer.
|
|
1944
|
+
runtime = _rag_runtime_status(
|
|
1945
|
+
bootstrap=bool(getattr(args, "bootstrap", False))
|
|
1946
|
+
)
|
|
1947
|
+
result["runtime"] = runtime
|
|
1948
|
+
status = str(runtime.get("status"))
|
|
1949
|
+
if status not in {"ready", "unknown"}:
|
|
1950
|
+
command = runtime.get("bootstrap_command") or (
|
|
1951
|
+
"bash plugins/indexkit/scripts/bootstrap.sh"
|
|
1952
|
+
)
|
|
1953
|
+
raise Refusal(
|
|
1954
|
+
f"the indexkit runtime is {status} "
|
|
1955
|
+
f"({runtime.get('detail', 'no detail')}); run: {command} "
|
|
1956
|
+
"— Claude Code and GitHub Copilot CLI bootstrap this from "
|
|
1957
|
+
"the indexkit SessionStart hook, but APM does not deploy "
|
|
1958
|
+
"hooks and an existing venv can be stale. Re-run `doctor "
|
|
1959
|
+
"--bootstrap` to build it now."
|
|
1960
|
+
)
|
|
1961
|
+
version_result = _run_provider(config, ["--version"], timeout=20.0)
|
|
1962
|
+
raw_version = version_result.stdout.decode("utf-8", errors="replace").strip()
|
|
1963
|
+
parsed_version = _parse_provider_version(raw_version)
|
|
1964
|
+
version_status = _provider_version_status(parsed_version, spec)
|
|
1965
|
+
capabilities = [
|
|
1966
|
+
_probe_capability(executable, config, spec, probe)
|
|
1967
|
+
for probe in spec.capabilities
|
|
1968
|
+
]
|
|
1969
|
+
missing = [c for c in capabilities if c["status"] != "ok"]
|
|
1970
|
+
compatibility: dict[str, object] = {
|
|
1971
|
+
"detected_version": raw_version,
|
|
1972
|
+
"parsed_version": (
|
|
1973
|
+
".".join(str(part) for part in parsed_version) if parsed_version else None
|
|
1974
|
+
),
|
|
1975
|
+
"version_status": version_status,
|
|
1976
|
+
"tested_release_line": spec.tested_release_line,
|
|
1977
|
+
"tested_version": ".".join(str(part) for part in spec.tested_version),
|
|
1978
|
+
"executable": executable,
|
|
1979
|
+
"store_path": str(config.store_path),
|
|
1980
|
+
"capabilities": capabilities,
|
|
1981
|
+
}
|
|
1982
|
+
if spec.name == "mempalace":
|
|
1983
|
+
compatibility["palace_path"] = str(config.store_path)
|
|
1984
|
+
result["compatibility"] = compatibility
|
|
1985
|
+
if missing:
|
|
1986
|
+
summary = "; ".join(
|
|
1987
|
+
f"{c['name']} ({c['command']}): {c['detail']}" for c in missing
|
|
1988
|
+
)
|
|
1989
|
+
raise Refusal(
|
|
1990
|
+
f"{spec.name} CLI is missing required capabilities for this adapter "
|
|
1991
|
+
f"(tested against {spec.tested_release_line}, detected "
|
|
1992
|
+
f"{raw_version or 'unknown version'}): {summary}"
|
|
1993
|
+
)
|
|
1994
|
+
# A patch/minor version different from the tested line is not on its own
|
|
1995
|
+
# a reason to block: only missing/incompatible capabilities are fatal.
|
|
1996
|
+
result.update(
|
|
1997
|
+
{
|
|
1998
|
+
"status": "ready",
|
|
1999
|
+
"executable": executable,
|
|
2000
|
+
"store_path": str(config.store_path),
|
|
2001
|
+
"version": raw_version,
|
|
2002
|
+
}
|
|
2003
|
+
)
|
|
2004
|
+
if spec.name == "mempalace":
|
|
2005
|
+
result["palace_path"] = str(config.store_path)
|
|
2006
|
+
print(json.dumps(result))
|
|
2007
|
+
return 0
|
|
2008
|
+
|
|
2009
|
+
|
|
2010
|
+
def _record_review(config: Config) -> int:
|
|
2011
|
+
results: list[dict[str, str]] = []
|
|
2012
|
+
for path in (
|
|
2013
|
+
sorted(config.records_path.glob("*.md")) if config.records_path.exists() else []
|
|
2014
|
+
):
|
|
2015
|
+
try:
|
|
2016
|
+
metadata, state = _load_record(path, config)
|
|
2017
|
+
results.append(
|
|
2018
|
+
{
|
|
2019
|
+
"id": metadata["id"],
|
|
2020
|
+
"artifact": str(path),
|
|
2021
|
+
"freshness": state["freshness"],
|
|
2022
|
+
"review": state["review"],
|
|
2023
|
+
"source_state": _source_state(metadata),
|
|
2024
|
+
"active": str(_is_active(state)).lower(),
|
|
2025
|
+
}
|
|
2026
|
+
)
|
|
2027
|
+
except Refusal as exc:
|
|
2028
|
+
results.append(
|
|
2029
|
+
{
|
|
2030
|
+
"artifact": str(path),
|
|
2031
|
+
"source_state": "invalid-or-stale",
|
|
2032
|
+
"error": str(exc),
|
|
2033
|
+
}
|
|
2034
|
+
)
|
|
2035
|
+
print(
|
|
2036
|
+
json.dumps(
|
|
2037
|
+
{
|
|
2038
|
+
"project": config.project_slug,
|
|
2039
|
+
"records": results,
|
|
2040
|
+
"audit": True,
|
|
2041
|
+
"include_inactive": True,
|
|
2042
|
+
}
|
|
2043
|
+
)
|
|
2044
|
+
)
|
|
2045
|
+
return 0
|
|
2046
|
+
|
|
2047
|
+
|
|
2048
|
+
def _record_state(args: argparse.Namespace, config: Config) -> int:
|
|
2049
|
+
if not ID_RE.fullmatch(args.record_id):
|
|
2050
|
+
raise Refusal("record id must be lowercase and use letters, numbers, ._-")
|
|
2051
|
+
path = config.records_path / f"{args.record_id}.md"
|
|
2052
|
+
if not path.is_file():
|
|
2053
|
+
raise Refusal(f"record does not exist in this project: {args.record_id}")
|
|
2054
|
+
reason = args.reason.strip()
|
|
2055
|
+
if not reason:
|
|
2056
|
+
raise Refusal("--reason must not be empty")
|
|
2057
|
+
if len(reason) > MAX_STATE_REASON_CHARS:
|
|
2058
|
+
raise Refusal(f"--reason must not exceed {MAX_STATE_REASON_CHARS} characters")
|
|
2059
|
+
with _state_lock(config, args.record_id):
|
|
2060
|
+
metadata, current = _load_record(path, config)
|
|
2061
|
+
requested = {
|
|
2062
|
+
"review": args.review if args.review is not None else current["review"],
|
|
2063
|
+
"freshness": (
|
|
2064
|
+
args.freshness if args.freshness is not None else current["freshness"]
|
|
2065
|
+
),
|
|
2066
|
+
}
|
|
2067
|
+
_validate_transition(current, requested)
|
|
2068
|
+
existing = _event_paths(config, args.record_id)
|
|
2069
|
+
if existing and not STATE_SEQUENCE_RE.fullmatch(existing[0].name):
|
|
2070
|
+
raise Refusal(
|
|
2071
|
+
"legacy timestamp-named state events are replayed read-only; "
|
|
2072
|
+
"migrate them before recording a new transition"
|
|
2073
|
+
)
|
|
2074
|
+
sequence = (
|
|
2075
|
+
int(STATE_SEQUENCE_RE.fullmatch(existing[-1].name)[1]) + 1
|
|
2076
|
+
if existing
|
|
2077
|
+
else 1
|
|
2078
|
+
)
|
|
2079
|
+
payload: dict[str, object] = {
|
|
2080
|
+
"schema": STATE_SCHEMA,
|
|
2081
|
+
"event_id": uuid.uuid4().hex,
|
|
2082
|
+
"record_id": metadata["id"],
|
|
2083
|
+
"record_hash": metadata["artifact_hash"],
|
|
2084
|
+
"project": config.project,
|
|
2085
|
+
"project_key": config.project_slug,
|
|
2086
|
+
"timestamp": _utc_timestamp(),
|
|
2087
|
+
"prior_review": current["review"],
|
|
2088
|
+
"prior_freshness": current["freshness"],
|
|
2089
|
+
"effective_review": requested["review"],
|
|
2090
|
+
"effective_freshness": requested["freshness"],
|
|
2091
|
+
"reason": reason,
|
|
2092
|
+
"sequence": sequence,
|
|
2093
|
+
}
|
|
2094
|
+
raw = (json.dumps(payload, sort_keys=True, indent=2) + "\n").encode("utf-8")
|
|
2095
|
+
event = (
|
|
2096
|
+
config.states_path
|
|
2097
|
+
/ args.record_id
|
|
2098
|
+
/ f"{sequence:0{STATE_SEQUENCE_WIDTH}d}-{uuid.uuid4().hex}.json"
|
|
2099
|
+
)
|
|
2100
|
+
if _write_once(event, raw) != "created":
|
|
2101
|
+
raise Refusal(f"refusing to reuse a generated state event path: {event}")
|
|
2102
|
+
print(
|
|
2103
|
+
json.dumps(
|
|
2104
|
+
{
|
|
2105
|
+
"status": "created",
|
|
2106
|
+
"event": str(event),
|
|
2107
|
+
"record": str(path),
|
|
2108
|
+
"record_hash": metadata["artifact_hash"],
|
|
2109
|
+
"project": config.project_slug,
|
|
2110
|
+
"prior_state": current,
|
|
2111
|
+
"effective_state": requested,
|
|
2112
|
+
"provider_reconciliation": (
|
|
2113
|
+
"required before provider recall"
|
|
2114
|
+
if config.provider == "mempalace"
|
|
2115
|
+
else "not-applicable"
|
|
2116
|
+
),
|
|
2117
|
+
}
|
|
2118
|
+
)
|
|
2119
|
+
)
|
|
2120
|
+
return 0
|
|
2121
|
+
|
|
2122
|
+
|
|
2123
|
+
def _read_receipts(config: Config) -> list[dict[str, object]]:
|
|
2124
|
+
receipts: list[dict[str, object]] = []
|
|
2125
|
+
if not config.receipts_path.exists():
|
|
2126
|
+
return receipts
|
|
2127
|
+
for path in sorted(config.receipts_path.glob("*.json")):
|
|
2128
|
+
try:
|
|
2129
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
2130
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
2131
|
+
continue
|
|
2132
|
+
if (
|
|
2133
|
+
isinstance(payload, dict)
|
|
2134
|
+
and payload.get("schema") == RECEIPT_SCHEMA
|
|
2135
|
+
and payload.get("project") == config.project
|
|
2136
|
+
and payload.get("project_key") == config.project_slug
|
|
2137
|
+
):
|
|
2138
|
+
receipts.append(payload)
|
|
2139
|
+
return receipts
|
|
2140
|
+
|
|
2141
|
+
|
|
2142
|
+
def _write_projection_marker(
|
|
2143
|
+
stage: Path,
|
|
2144
|
+
config: Config,
|
|
2145
|
+
projection_hash: str,
|
|
2146
|
+
ledger_hash: str,
|
|
2147
|
+
provider_version: str,
|
|
2148
|
+
) -> None:
|
|
2149
|
+
raw = (
|
|
2150
|
+
json.dumps(
|
|
2151
|
+
{
|
|
2152
|
+
"schema": PROJECTION_MARKER_SCHEMA,
|
|
2153
|
+
"project": config.project,
|
|
2154
|
+
"project_key": config.project_slug,
|
|
2155
|
+
"projection_hash": projection_hash,
|
|
2156
|
+
"ledger_hash": ledger_hash,
|
|
2157
|
+
"provider_version": provider_version,
|
|
2158
|
+
"applied_at": _utc_timestamp(),
|
|
2159
|
+
},
|
|
2160
|
+
sort_keys=True,
|
|
2161
|
+
indent=2,
|
|
2162
|
+
)
|
|
2163
|
+
+ "\n"
|
|
2164
|
+
).encode("utf-8")
|
|
2165
|
+
_write_once(stage / PROJECTION_MARKER_NAME, raw)
|
|
2166
|
+
|
|
2167
|
+
|
|
2168
|
+
def _has_current_provider_projection(config: Config, projection_hash: str) -> bool:
|
|
2169
|
+
marker = config.store_path / PROJECTION_MARKER_NAME
|
|
2170
|
+
try:
|
|
2171
|
+
payload = json.loads(marker.read_text(encoding="utf-8"))
|
|
2172
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
2173
|
+
return False
|
|
2174
|
+
if not isinstance(payload, dict) or set(payload) != {
|
|
2175
|
+
"schema",
|
|
2176
|
+
"project",
|
|
2177
|
+
"project_key",
|
|
2178
|
+
"projection_hash",
|
|
2179
|
+
"ledger_hash",
|
|
2180
|
+
"provider_version",
|
|
2181
|
+
"applied_at",
|
|
2182
|
+
}:
|
|
2183
|
+
return False
|
|
2184
|
+
try:
|
|
2185
|
+
_validate_timestamp(str(payload["applied_at"]), "applied_at")
|
|
2186
|
+
except Refusal:
|
|
2187
|
+
return False
|
|
2188
|
+
return (
|
|
2189
|
+
payload["schema"] == PROJECTION_MARKER_SCHEMA
|
|
2190
|
+
and payload["project"] == config.project
|
|
2191
|
+
and payload["project_key"] == config.project_slug
|
|
2192
|
+
and payload["projection_hash"] == projection_hash
|
|
2193
|
+
and payload["ledger_hash"] == _ledger_hash(config)
|
|
2194
|
+
and isinstance(payload["provider_version"], str)
|
|
2195
|
+
)
|
|
2196
|
+
|
|
2197
|
+
|
|
2198
|
+
def _prune_provider_backups(
|
|
2199
|
+
parent: Path, keep: Path | None, prefix: str = "palace-backup-"
|
|
2200
|
+
) -> list[str]:
|
|
2201
|
+
backups = [
|
|
2202
|
+
path for path in parent.glob(f"{prefix}*") if path.is_dir() and path != keep
|
|
2203
|
+
]
|
|
2204
|
+
if keep is None:
|
|
2205
|
+
backups.sort(
|
|
2206
|
+
key=lambda path: (path.stat().st_mtime_ns, path.name), reverse=True
|
|
2207
|
+
)
|
|
2208
|
+
backups = backups[PROVIDER_BACKUP_RETENTION:]
|
|
2209
|
+
removed: list[str] = []
|
|
2210
|
+
for path in backups:
|
|
2211
|
+
shutil.rmtree(path)
|
|
2212
|
+
removed.append(str(path))
|
|
2213
|
+
return sorted(removed)
|
|
2214
|
+
|
|
2215
|
+
|
|
2216
|
+
def _sync_provider(args: argparse.Namespace, config: Config) -> int:
|
|
2217
|
+
if config.provider == "none":
|
|
2218
|
+
raise Refusal(
|
|
2219
|
+
"sync-provider requires an external provider; choose one of: "
|
|
2220
|
+
+ ", ".join(sorted(PROVIDER_SPECS))
|
|
2221
|
+
)
|
|
2222
|
+
spec = config.spec
|
|
2223
|
+
records, excluded = _active_projection(config)
|
|
2224
|
+
projection_hash = _projection_hash(records)
|
|
2225
|
+
ledger_hash = _ledger_hash(config)
|
|
2226
|
+
plan: dict[str, object] = {
|
|
2227
|
+
"project": config.project_slug,
|
|
2228
|
+
"provider": spec.name,
|
|
2229
|
+
"store_path": str(config.store_path),
|
|
2230
|
+
"active_record_ids": [str(metadata["id"]) for _, metadata in records],
|
|
2231
|
+
"excluded_records": excluded,
|
|
2232
|
+
"projection_hash": projection_hash,
|
|
2233
|
+
"apply": bool(args.apply),
|
|
2234
|
+
}
|
|
2235
|
+
if spec.name == "mempalace":
|
|
2236
|
+
plan["palace_path"] = str(config.store_path)
|
|
2237
|
+
if not args.apply:
|
|
2238
|
+
plan["status"] = "dry-run"
|
|
2239
|
+
plan["safety"] = (
|
|
2240
|
+
"apply builds a fresh project-isolated store, preserves a backup, "
|
|
2241
|
+
f"then swaps only after {spec.name} succeeds"
|
|
2242
|
+
)
|
|
2243
|
+
print(json.dumps(plan))
|
|
2244
|
+
return 0
|
|
2245
|
+
if os.name != "posix":
|
|
2246
|
+
raise Refusal(
|
|
2247
|
+
"safe provider replacement is supported only on POSIX; "
|
|
2248
|
+
"dry-run was not applied and no store was changed"
|
|
2249
|
+
)
|
|
2250
|
+
|
|
2251
|
+
parent = config.store_path.parent
|
|
2252
|
+
parent.mkdir(parents=True, exist_ok=True)
|
|
2253
|
+
projection = Path(tempfile.mkdtemp(prefix=".projection-", dir=parent))
|
|
2254
|
+
stage = parent / f".store-rebuild-{uuid.uuid4().hex}"
|
|
2255
|
+
backup: Path | None = None
|
|
2256
|
+
executable = ""
|
|
2257
|
+
version = "unknown"
|
|
2258
|
+
argv: list[str] = []
|
|
2259
|
+
recovery_status = "not-needed"
|
|
2260
|
+
try:
|
|
2261
|
+
_materialize_projection(records, projection)
|
|
2262
|
+
stage.mkdir(mode=0o700)
|
|
2263
|
+
executable, version = _provider_version(config)
|
|
2264
|
+
argv = [executable, *spec.index_argv(projection, config.project_slug)]
|
|
2265
|
+
_run_provider(
|
|
2266
|
+
config,
|
|
2267
|
+
argv[1:],
|
|
2268
|
+
timeout=300.0,
|
|
2269
|
+
store_path=stage,
|
|
2270
|
+
)
|
|
2271
|
+
if not stage.is_dir():
|
|
2272
|
+
raise Refusal(f"{spec.name} did not leave a valid staged store")
|
|
2273
|
+
_write_projection_marker(
|
|
2274
|
+
stage,
|
|
2275
|
+
config,
|
|
2276
|
+
projection_hash,
|
|
2277
|
+
ledger_hash,
|
|
2278
|
+
version,
|
|
2279
|
+
)
|
|
2280
|
+
if config.store_path.exists():
|
|
2281
|
+
backup = parent / f"{spec.backup_prefix}{uuid.uuid4().hex}"
|
|
2282
|
+
os.replace(config.store_path, backup)
|
|
2283
|
+
try:
|
|
2284
|
+
os.replace(stage, config.store_path)
|
|
2285
|
+
except OSError:
|
|
2286
|
+
if backup is not None and not config.store_path.exists():
|
|
2287
|
+
os.replace(backup, config.store_path)
|
|
2288
|
+
backup = None
|
|
2289
|
+
recovery_status = "restored-to-live-store"
|
|
2290
|
+
elif backup is not None:
|
|
2291
|
+
recovery_status = "backup-preserved"
|
|
2292
|
+
raise
|
|
2293
|
+
except (OSError, Refusal) as exc:
|
|
2294
|
+
receipt = _provider_receipt(
|
|
2295
|
+
config,
|
|
2296
|
+
provider_version=version,
|
|
2297
|
+
operation="sync-provider",
|
|
2298
|
+
artifact_hash=None,
|
|
2299
|
+
argv=argv,
|
|
2300
|
+
outcome="failed",
|
|
2301
|
+
detail=str(exc),
|
|
2302
|
+
projection_hash=projection_hash,
|
|
2303
|
+
backup_path=backup,
|
|
2304
|
+
recovery_status=recovery_status,
|
|
2305
|
+
)
|
|
2306
|
+
raise Refusal(
|
|
2307
|
+
f"provider synchronization failed; receipt={receipt}: {exc}"
|
|
2308
|
+
) from exc
|
|
2309
|
+
finally:
|
|
2310
|
+
shutil.rmtree(projection, ignore_errors=True)
|
|
2311
|
+
if stage.exists() and stage != config.store_path:
|
|
2312
|
+
shutil.rmtree(stage, ignore_errors=True)
|
|
2313
|
+
receipt = _provider_receipt(
|
|
2314
|
+
config,
|
|
2315
|
+
provider_version=version,
|
|
2316
|
+
operation="sync-provider",
|
|
2317
|
+
artifact_hash=None,
|
|
2318
|
+
argv=argv,
|
|
2319
|
+
outcome="success",
|
|
2320
|
+
detail=f"reconciled {len(records)} accepted/current records",
|
|
2321
|
+
projection_hash=projection_hash,
|
|
2322
|
+
backup_path=backup,
|
|
2323
|
+
)
|
|
2324
|
+
try:
|
|
2325
|
+
removed_backups = _prune_provider_backups(parent, backup, spec.backup_prefix)
|
|
2326
|
+
except OSError as exc:
|
|
2327
|
+
raise Refusal(
|
|
2328
|
+
"provider synchronized but backup retention failed; "
|
|
2329
|
+
f"receipt={receipt}: {exc}"
|
|
2330
|
+
) from exc
|
|
2331
|
+
plan.update(
|
|
2332
|
+
{
|
|
2333
|
+
"status": "synchronized",
|
|
2334
|
+
"backup_path": str(backup) if backup else None,
|
|
2335
|
+
"removed_backups": removed_backups,
|
|
2336
|
+
"receipt": str(receipt),
|
|
2337
|
+
}
|
|
2338
|
+
)
|
|
2339
|
+
print(json.dumps(plan))
|
|
2340
|
+
return 0
|
|
2341
|
+
|
|
2342
|
+
|
|
2343
|
+
def _extract_copilot_session(raw: bytes) -> dict[str, object] | None:
|
|
2344
|
+
"""Extract the human-visible conversation from a Copilot CLI event log.
|
|
2345
|
+
|
|
2346
|
+
Copilot records *all* session activity in one event stream, so most events
|
|
2347
|
+
are not conversation. Attribution matters more than volume here: a subagent
|
|
2348
|
+
task prompt is written by the orchestrating model, not by the person, and
|
|
2349
|
+
storing it as a user turn silently misattributes authorship.
|
|
2350
|
+
|
|
2351
|
+
Measured across a real 115-session corpus, only 24 of 729 `user.message`
|
|
2352
|
+
events were human-authored; 611 carried `parentAgentTaskId` (subagent task
|
|
2353
|
+
prompts) and 94 carried a `source` (generated skill/agent/command/system
|
|
2354
|
+
context). Every distinct `source` value observed was generated context, so
|
|
2355
|
+
the presence of the field — not a specific prefix — is the reliable signal.
|
|
2356
|
+
|
|
2357
|
+
Returns None when the input is not a recognized Copilot session. A
|
|
2358
|
+
recognized session with no conversational turns returns an empty turn list
|
|
2359
|
+
rather than None, so the caller can record it as empty instead of falling
|
|
2360
|
+
back to raw event JSON.
|
|
2361
|
+
"""
|
|
2362
|
+
text = raw.decode("utf-8", errors="replace")
|
|
2363
|
+
session_id: str | None = None
|
|
2364
|
+
started_at: str | None = None
|
|
2365
|
+
producer: str | None = None
|
|
2366
|
+
turns: list[dict[str, str]] = []
|
|
2367
|
+
dropped: dict[str, int] = {
|
|
2368
|
+
"user_subagent_prompt": 0,
|
|
2369
|
+
"user_generated_context": 0,
|
|
2370
|
+
"user_empty": 0,
|
|
2371
|
+
"assistant_tool_nested": 0,
|
|
2372
|
+
"assistant_subagent": 0,
|
|
2373
|
+
"assistant_empty": 0,
|
|
2374
|
+
"non_conversational_event": 0,
|
|
2375
|
+
"unparsable_line": 0,
|
|
2376
|
+
}
|
|
2377
|
+
|
|
2378
|
+
for line in text.splitlines():
|
|
2379
|
+
line = line.strip()
|
|
2380
|
+
if not line:
|
|
2381
|
+
continue
|
|
2382
|
+
try:
|
|
2383
|
+
event = json.loads(line)
|
|
2384
|
+
except json.JSONDecodeError:
|
|
2385
|
+
dropped["unparsable_line"] += 1
|
|
2386
|
+
continue
|
|
2387
|
+
if not isinstance(event, dict):
|
|
2388
|
+
dropped["unparsable_line"] += 1
|
|
2389
|
+
continue
|
|
2390
|
+
kind = event.get("type", "")
|
|
2391
|
+
data = event.get("data", {})
|
|
2392
|
+
if not isinstance(data, dict):
|
|
2393
|
+
dropped["non_conversational_event"] += 1
|
|
2394
|
+
continue
|
|
2395
|
+
|
|
2396
|
+
if kind == "session.start":
|
|
2397
|
+
candidate_id = data.get("sessionId")
|
|
2398
|
+
if isinstance(candidate_id, str) and candidate_id:
|
|
2399
|
+
session_id = candidate_id
|
|
2400
|
+
start_time = data.get("startTime")
|
|
2401
|
+
if isinstance(start_time, str) and start_time:
|
|
2402
|
+
started_at = start_time
|
|
2403
|
+
produced_by = data.get("producer")
|
|
2404
|
+
if isinstance(produced_by, str) and produced_by:
|
|
2405
|
+
producer = produced_by
|
|
2406
|
+
continue
|
|
2407
|
+
|
|
2408
|
+
if session_id is None:
|
|
2409
|
+
# Nothing before a recognized session.start is trusted.
|
|
2410
|
+
dropped["non_conversational_event"] += 1
|
|
2411
|
+
continue
|
|
2412
|
+
|
|
2413
|
+
if kind == "user.message":
|
|
2414
|
+
if data.get("parentAgentTaskId"):
|
|
2415
|
+
dropped["user_subagent_prompt"] += 1
|
|
2416
|
+
continue
|
|
2417
|
+
if "source" in data:
|
|
2418
|
+
# Generated context: skill-, agent-, command-, or system.
|
|
2419
|
+
# A human turn carries no source at all.
|
|
2420
|
+
dropped["user_generated_context"] += 1
|
|
2421
|
+
continue
|
|
2422
|
+
role = "user"
|
|
2423
|
+
elif kind == "assistant.message":
|
|
2424
|
+
if data.get("parentToolCallId"):
|
|
2425
|
+
dropped["assistant_tool_nested"] += 1
|
|
2426
|
+
continue
|
|
2427
|
+
if data.get("parentAgentTaskId"):
|
|
2428
|
+
dropped["assistant_subagent"] += 1
|
|
2429
|
+
continue
|
|
2430
|
+
role = "assistant"
|
|
2431
|
+
else:
|
|
2432
|
+
dropped["non_conversational_event"] += 1
|
|
2433
|
+
continue
|
|
2434
|
+
|
|
2435
|
+
# `content` is what the person actually wrote. `transformedContent`
|
|
2436
|
+
# is post-expansion, and reasoning fields are never retained.
|
|
2437
|
+
content = data.get("content")
|
|
2438
|
+
if not isinstance(content, str) or not content.strip():
|
|
2439
|
+
dropped[f"{role}_empty"] += 1
|
|
2440
|
+
continue
|
|
2441
|
+
turns.append({"role": role, "content": content.strip()})
|
|
2442
|
+
|
|
2443
|
+
if session_id is None:
|
|
2444
|
+
return None
|
|
2445
|
+
return {
|
|
2446
|
+
"session_id": session_id,
|
|
2447
|
+
"started_at": started_at,
|
|
2448
|
+
"producer": producer,
|
|
2449
|
+
"turns": turns,
|
|
2450
|
+
"dropped": dropped,
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
|
|
2454
|
+
def _scan_secrets(text: str) -> list[dict[str, object]]:
|
|
2455
|
+
findings: list[dict[str, object]] = []
|
|
2456
|
+
for name, pattern in SECRET_PATTERNS:
|
|
2457
|
+
count = len(pattern.findall(text))
|
|
2458
|
+
if count:
|
|
2459
|
+
findings.append({"pattern": name, "matches": count})
|
|
2460
|
+
return findings
|
|
2461
|
+
|
|
2462
|
+
|
|
2463
|
+
def _redact_secrets(text: str) -> tuple[str, int]:
|
|
2464
|
+
redactions = 0
|
|
2465
|
+
for name, pattern in SECRET_PATTERNS:
|
|
2466
|
+
text, count = pattern.subn(f"[redacted:{name}]", text)
|
|
2467
|
+
redactions += count
|
|
2468
|
+
return text, redactions
|
|
2469
|
+
|
|
2470
|
+
|
|
2471
|
+
def _render_turns(turns: list[dict[str, str]]) -> tuple[list[str], int, int]:
|
|
2472
|
+
lines: list[str] = []
|
|
2473
|
+
truncated = 0
|
|
2474
|
+
omitted = max(0, len(turns) - MAX_CANDIDATE_TURNS)
|
|
2475
|
+
for index, turn in enumerate(turns[:MAX_CANDIDATE_TURNS], start=1):
|
|
2476
|
+
content = turn["content"]
|
|
2477
|
+
if len(content) > MAX_TURN_CHARS:
|
|
2478
|
+
content = content[:MAX_TURN_CHARS]
|
|
2479
|
+
truncated += 1
|
|
2480
|
+
content += f"\n\n[truncated at {MAX_TURN_CHARS} characters]"
|
|
2481
|
+
lines.append(f"### {index}. {turn['role']}")
|
|
2482
|
+
lines.append("")
|
|
2483
|
+
lines.extend(content.splitlines())
|
|
2484
|
+
lines.append("")
|
|
2485
|
+
if omitted:
|
|
2486
|
+
# A reviewer must never mistake a sliced transcript for a complete one.
|
|
2487
|
+
lines.append(
|
|
2488
|
+
f"### [{omitted} further turn(s) omitted at the "
|
|
2489
|
+
f"{MAX_CANDIDATE_TURNS}-turn candidate limit]"
|
|
2490
|
+
)
|
|
2491
|
+
lines.append("")
|
|
2492
|
+
return lines, truncated, omitted
|
|
2493
|
+
|
|
2494
|
+
|
|
2495
|
+
def _propose_from_session(args: argparse.Namespace, config: Config) -> int:
|
|
2496
|
+
"""Extract reviewable candidates from Copilot CLI sessions.
|
|
2497
|
+
|
|
2498
|
+
This deliberately proposes rather than captures. A transcript is not an
|
|
2499
|
+
atomic memory, so authoring a `memory-v1` record from a candidate stays an
|
|
2500
|
+
explicit judgment step; nothing here can enter active recall on its own.
|
|
2501
|
+
"""
|
|
2502
|
+
root = Path(args.path).expanduser().resolve()
|
|
2503
|
+
if root.is_file():
|
|
2504
|
+
logs = [root]
|
|
2505
|
+
elif root.is_dir():
|
|
2506
|
+
logs = sorted(root.glob("*/events.jsonl")) or sorted(root.glob("events.jsonl"))
|
|
2507
|
+
else:
|
|
2508
|
+
raise Refusal(f"session path does not exist: {root}")
|
|
2509
|
+
if not logs:
|
|
2510
|
+
raise Refusal(f"no Copilot `events.jsonl` logs found under {root}")
|
|
2511
|
+
|
|
2512
|
+
repo = Path(args.repo).expanduser().resolve()
|
|
2513
|
+
repository = _normalize_repository(_git(repo, "remote", "get-url", "origin"))
|
|
2514
|
+
_assert_project_matches({"repository": repository, "scope": "project"}, config)
|
|
2515
|
+
branch = _git(repo, "rev-parse", "--abbrev-ref", "HEAD")
|
|
2516
|
+
head = _git(repo, "rev-parse", "HEAD")
|
|
2517
|
+
if branch == "HEAD":
|
|
2518
|
+
# Detached checkouts (CI PR builds, a worktree at a tag, bisect) have no
|
|
2519
|
+
# branch anchor, and a project record requires one. Say so plainly
|
|
2520
|
+
# instead of refusing with a generic name-format error.
|
|
2521
|
+
raise Refusal(
|
|
2522
|
+
f"{repo} is in a detached HEAD state, so there is no branch anchor "
|
|
2523
|
+
"for a project record. Check out a named branch, or pass --repo "
|
|
2524
|
+
"pointing at a checkout that is on one."
|
|
2525
|
+
)
|
|
2526
|
+
_validate_branch(branch)
|
|
2527
|
+
|
|
2528
|
+
planned: list[dict[str, object]] = []
|
|
2529
|
+
written: list[str] = []
|
|
2530
|
+
blocked: list[dict[str, object]] = []
|
|
2531
|
+
skipped: list[dict[str, object]] = []
|
|
2532
|
+
|
|
2533
|
+
for log in logs:
|
|
2534
|
+
raw = log.read_bytes()
|
|
2535
|
+
extracted = _extract_copilot_session(raw)
|
|
2536
|
+
if extracted is None:
|
|
2537
|
+
skipped.append({"source": str(log), "reason": "not-a-copilot-session"})
|
|
2538
|
+
continue
|
|
2539
|
+
turns = list(extracted["turns"]) # type: ignore[arg-type]
|
|
2540
|
+
if not turns:
|
|
2541
|
+
skipped.append({"source": str(log), "reason": "no-conversational-turns"})
|
|
2542
|
+
continue
|
|
2543
|
+
session_id = str(extracted["session_id"])
|
|
2544
|
+
if not SESSION_ID_RE.fullmatch(session_id):
|
|
2545
|
+
# `session_id` names the candidate file. An id like `../../outside`
|
|
2546
|
+
# would escape the project-isolated directory, so an unsafe value
|
|
2547
|
+
# is refused loudly rather than sanitized into something plausible.
|
|
2548
|
+
skipped.append(
|
|
2549
|
+
{
|
|
2550
|
+
"source": str(log),
|
|
2551
|
+
"reason": "unsafe-session-id",
|
|
2552
|
+
"session_id": session_id[:80],
|
|
2553
|
+
}
|
|
2554
|
+
)
|
|
2555
|
+
continue
|
|
2556
|
+
|
|
2557
|
+
body_lines, truncated, omitted = _render_turns(turns)
|
|
2558
|
+
transcript = "\n".join(body_lines)
|
|
2559
|
+
findings = _scan_secrets(transcript)
|
|
2560
|
+
redactions = 0
|
|
2561
|
+
if findings:
|
|
2562
|
+
if not args.redact:
|
|
2563
|
+
blocked.append(
|
|
2564
|
+
{
|
|
2565
|
+
"source": str(log),
|
|
2566
|
+
"reason": "possible-credentials",
|
|
2567
|
+
"findings": findings,
|
|
2568
|
+
}
|
|
2569
|
+
)
|
|
2570
|
+
continue
|
|
2571
|
+
transcript, redactions = _redact_secrets(transcript)
|
|
2572
|
+
|
|
2573
|
+
source_hash = hashlib.sha256(raw).hexdigest()
|
|
2574
|
+
observed_at = extracted["started_at"] or _utc_timestamp()
|
|
2575
|
+
try:
|
|
2576
|
+
_validate_timestamp(str(observed_at), "observed_at")
|
|
2577
|
+
except Refusal:
|
|
2578
|
+
observed_at = _utc_timestamp()
|
|
2579
|
+
entry: dict[str, object] = {
|
|
2580
|
+
"session_id": session_id,
|
|
2581
|
+
"source": str(log),
|
|
2582
|
+
"source_hash": source_hash,
|
|
2583
|
+
"turns": len(turns),
|
|
2584
|
+
"turns_written": len(turns) - omitted,
|
|
2585
|
+
"omitted_turns": omitted,
|
|
2586
|
+
"dropped": extracted["dropped"],
|
|
2587
|
+
"truncated_turns": truncated,
|
|
2588
|
+
"redactions": redactions,
|
|
2589
|
+
}
|
|
2590
|
+
if args.dry_run:
|
|
2591
|
+
planned.append(entry)
|
|
2592
|
+
continue
|
|
2593
|
+
|
|
2594
|
+
name = f"{session_id}-{source_hash[:12]}.md"
|
|
2595
|
+
destination = config.candidates_path / name
|
|
2596
|
+
# Defense in depth: the id was validated above, so a path that still
|
|
2597
|
+
# resolves outside the project directory is a bug, not user error.
|
|
2598
|
+
if destination.parent.resolve() != config.candidates_path.resolve():
|
|
2599
|
+
raise Refusal(
|
|
2600
|
+
f"refusing a candidate path outside the project store: {destination}"
|
|
2601
|
+
)
|
|
2602
|
+
document = "\n".join(
|
|
2603
|
+
[
|
|
2604
|
+
"---",
|
|
2605
|
+
f"schema: {CANDIDATE_SCHEMA}",
|
|
2606
|
+
f"session_id: {session_id}",
|
|
2607
|
+
"scope: project",
|
|
2608
|
+
f"repository: {repository}",
|
|
2609
|
+
f"branch: {branch}",
|
|
2610
|
+
f"head: {head}",
|
|
2611
|
+
f"producer: {extracted['producer'] or SESSION_PRODUCER}",
|
|
2612
|
+
f"observed_at: {observed_at}",
|
|
2613
|
+
f"extracted_at: {_utc_timestamp()}",
|
|
2614
|
+
f"source: {log}",
|
|
2615
|
+
f"source_hash: {source_hash}",
|
|
2616
|
+
f"turns: {len(turns) - omitted}",
|
|
2617
|
+
f"turns_extracted: {len(turns)}",
|
|
2618
|
+
f"omitted_turns: {omitted}",
|
|
2619
|
+
f"redactions: {redactions}",
|
|
2620
|
+
"review: candidate",
|
|
2621
|
+
"---",
|
|
2622
|
+
"",
|
|
2623
|
+
"## Provenance",
|
|
2624
|
+
"",
|
|
2625
|
+
f"- Extracted from `{log}` (SHA-256 `{source_hash}`).",
|
|
2626
|
+
"- Only top-level human and assistant turns are retained. Subagent",
|
|
2627
|
+
" prompts, generated skill/agent/command context, tool-nested",
|
|
2628
|
+
" messages, and model reasoning are excluded by construction.",
|
|
2629
|
+
"",
|
|
2630
|
+
"## Dropped Events",
|
|
2631
|
+
"",
|
|
2632
|
+
*(
|
|
2633
|
+
f"- {reason}: {count}"
|
|
2634
|
+
for reason, count in sorted(
|
|
2635
|
+
dict(extracted["dropped"]).items() # type: ignore[arg-type]
|
|
2636
|
+
)
|
|
2637
|
+
if count
|
|
2638
|
+
),
|
|
2639
|
+
"",
|
|
2640
|
+
"## Transcript",
|
|
2641
|
+
"",
|
|
2642
|
+
transcript,
|
|
2643
|
+
"",
|
|
2644
|
+
"## Review Notes",
|
|
2645
|
+
"",
|
|
2646
|
+
"- This is a candidate, not a memory. To retain anything here,",
|
|
2647
|
+
" author a `context-kit/memory-v1` record whose `source` is the",
|
|
2648
|
+
" session log above and whose `source_hash` matches, mark it",
|
|
2649
|
+
" `review: proposed`, then promote it with `record-state` only",
|
|
2650
|
+
" after checking the evidence.",
|
|
2651
|
+
"",
|
|
2652
|
+
]
|
|
2653
|
+
)
|
|
2654
|
+
state = _write_once(destination, document.encode("utf-8"))
|
|
2655
|
+
entry["artifact"] = str(destination)
|
|
2656
|
+
entry["status"] = state
|
|
2657
|
+
written.append(str(destination))
|
|
2658
|
+
planned.append(entry)
|
|
2659
|
+
|
|
2660
|
+
print(
|
|
2661
|
+
json.dumps(
|
|
2662
|
+
{
|
|
2663
|
+
"status": "dry-run" if args.dry_run else "extracted",
|
|
2664
|
+
"project": config.project_slug,
|
|
2665
|
+
"repository": repository,
|
|
2666
|
+
"branch": branch,
|
|
2667
|
+
"head": head,
|
|
2668
|
+
"logs_examined": len(logs),
|
|
2669
|
+
"candidates": planned,
|
|
2670
|
+
"written": written,
|
|
2671
|
+
"blocked": blocked,
|
|
2672
|
+
"skipped": skipped,
|
|
2673
|
+
"note": (
|
|
2674
|
+
"candidates are proposals for review; nothing here enters "
|
|
2675
|
+
"active recall until an explicit memory-v1 record is "
|
|
2676
|
+
"captured and accepted"
|
|
2677
|
+
),
|
|
2678
|
+
}
|
|
2679
|
+
)
|
|
2680
|
+
)
|
|
2681
|
+
return 0
|
|
2682
|
+
|
|
2683
|
+
|
|
2684
|
+
def _hook_recall(config: Config) -> int:
|
|
2685
|
+
"""Emit reviewed memory as session-start context.
|
|
2686
|
+
|
|
2687
|
+
Both Claude Code and GitHub Copilot CLI run a plugin's `hooks/hooks.json`
|
|
2688
|
+
and honor an `additionalContext` string, so this is the one lifecycle point
|
|
2689
|
+
where durable memory pays for itself without the agent asking: reviewed,
|
|
2690
|
+
accepted, current records prime the session.
|
|
2691
|
+
|
|
2692
|
+
Recall is read-only, so it is gated on its own switch rather than on
|
|
2693
|
+
`auto_capture`, which governs writing.
|
|
2694
|
+
"""
|
|
2695
|
+
if not config.recall_on_start:
|
|
2696
|
+
print("{}")
|
|
2697
|
+
return 0
|
|
2698
|
+
try:
|
|
2699
|
+
digest = _wake_digest(config)
|
|
2700
|
+
except Refusal:
|
|
2701
|
+
# A hook must never break a session over a memory problem.
|
|
2702
|
+
print("{}")
|
|
2703
|
+
return 0
|
|
2704
|
+
context = str(digest["context"])
|
|
2705
|
+
if not context:
|
|
2706
|
+
print("{}")
|
|
2707
|
+
return 0
|
|
2708
|
+
print(json.dumps({"additionalContext": context}))
|
|
2709
|
+
return 0
|
|
2710
|
+
|
|
2711
|
+
|
|
2712
|
+
def _run_hook(event: str, config: Config, payload: bytes) -> int:
|
|
2713
|
+
if event == "session-start":
|
|
2714
|
+
return _hook_recall(config)
|
|
2715
|
+
if not config.auto_capture:
|
|
2716
|
+
print("{}")
|
|
2717
|
+
return 0
|
|
2718
|
+
try:
|
|
2719
|
+
decoded = json.loads(payload)
|
|
2720
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
2721
|
+
raise Refusal(f"hook payload must be valid JSON: {exc}") from exc
|
|
2722
|
+
if not isinstance(decoded, dict):
|
|
2723
|
+
raise Refusal("hook payload must be a JSON object")
|
|
2724
|
+
pending_dir = config.home / "pending-hooks" / config.project_slug
|
|
2725
|
+
pending = _new_write_once_path(pending_dir, f"-{event}.json")
|
|
2726
|
+
if _write_once(pending, payload) != "created":
|
|
2727
|
+
raise Refusal(f"refusing to reuse a generated hook payload path: {pending}")
|
|
2728
|
+
print(
|
|
2729
|
+
json.dumps(
|
|
2730
|
+
{
|
|
2731
|
+
"status": "queued-for-review",
|
|
2732
|
+
"event": event,
|
|
2733
|
+
"pending": str(pending),
|
|
2734
|
+
"provider_invoked": False,
|
|
2735
|
+
}
|
|
2736
|
+
)
|
|
2737
|
+
)
|
|
2738
|
+
return 0
|
|
2739
|
+
|
|
2740
|
+
|
|
2741
|
+
def _add_config_args(parser: argparse.ArgumentParser) -> None:
|
|
2742
|
+
parser.add_argument("--provider", choices=PROVIDERS)
|
|
2743
|
+
parser.add_argument("--home")
|
|
2744
|
+
parser.add_argument("--project")
|
|
2745
|
+
|
|
2746
|
+
|
|
2747
|
+
def _parser() -> argparse.ArgumentParser:
|
|
2748
|
+
parser = argparse.ArgumentParser(
|
|
2749
|
+
# No `prog=`, deliberately: argparse then derives it from argv[0], so
|
|
2750
|
+
# help text names whichever entry point the user actually invoked —
|
|
2751
|
+
# `memorykit` from the installed console script, `memory-provider.py`
|
|
2752
|
+
# from the plugin launcher. A hardcoded name would be wrong for one of
|
|
2753
|
+
# the two deployment shapes and would print copy-paste-broken usage.
|
|
2754
|
+
description="Validate context-kit memories and invoke an optional provider.",
|
|
2755
|
+
)
|
|
2756
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
2757
|
+
|
|
2758
|
+
validate = sub.add_parser("validate")
|
|
2759
|
+
validate.add_argument("artifact")
|
|
2760
|
+
|
|
2761
|
+
capture = sub.add_parser("capture")
|
|
2762
|
+
capture.add_argument("artifact")
|
|
2763
|
+
capture.add_argument("--local-only", action="store_true")
|
|
2764
|
+
_add_config_args(capture)
|
|
2765
|
+
|
|
2766
|
+
archive = sub.add_parser("archive-handoff")
|
|
2767
|
+
archive.add_argument("artifact")
|
|
2768
|
+
archive.add_argument("--local-only", action="store_true")
|
|
2769
|
+
archive.add_argument(
|
|
2770
|
+
"--repo",
|
|
2771
|
+
default=".",
|
|
2772
|
+
help="Current repository used to enforce handoff freshness.",
|
|
2773
|
+
)
|
|
2774
|
+
_add_config_args(archive)
|
|
2775
|
+
|
|
2776
|
+
search = sub.add_parser("search")
|
|
2777
|
+
search.add_argument("query")
|
|
2778
|
+
search.add_argument("--results", type=int, default=8)
|
|
2779
|
+
search.add_argument(
|
|
2780
|
+
"--include-inactive",
|
|
2781
|
+
action="store_true",
|
|
2782
|
+
help="Audit local proposed, rejected, stale, superseded, and revoked records.",
|
|
2783
|
+
)
|
|
2784
|
+
_add_config_args(search)
|
|
2785
|
+
|
|
2786
|
+
wake = sub.add_parser("wake")
|
|
2787
|
+
wake.add_argument(
|
|
2788
|
+
"--format",
|
|
2789
|
+
choices=("json", "text"),
|
|
2790
|
+
default="json",
|
|
2791
|
+
help="`text` prints only the injectable context block.",
|
|
2792
|
+
)
|
|
2793
|
+
_add_config_args(wake)
|
|
2794
|
+
|
|
2795
|
+
doctor = sub.add_parser("doctor")
|
|
2796
|
+
doctor.add_argument(
|
|
2797
|
+
"--bootstrap",
|
|
2798
|
+
action="store_true",
|
|
2799
|
+
help="Build the indexkit runtime if it is missing or stale (rag provider).",
|
|
2800
|
+
)
|
|
2801
|
+
_add_config_args(doctor)
|
|
2802
|
+
|
|
2803
|
+
audit = sub.add_parser("audit")
|
|
2804
|
+
audit.add_argument(
|
|
2805
|
+
"--limit",
|
|
2806
|
+
type=int,
|
|
2807
|
+
default=50,
|
|
2808
|
+
help="Maximum findings to print (default 50).",
|
|
2809
|
+
)
|
|
2810
|
+
_add_config_args(audit)
|
|
2811
|
+
|
|
2812
|
+
review = sub.add_parser("review")
|
|
2813
|
+
review.add_argument(
|
|
2814
|
+
"--include-inactive",
|
|
2815
|
+
action="store_true",
|
|
2816
|
+
help="Explicitly document audit intent; review already reports all records.",
|
|
2817
|
+
)
|
|
2818
|
+
_add_config_args(review)
|
|
2819
|
+
|
|
2820
|
+
state = sub.add_parser("record-state")
|
|
2821
|
+
state.add_argument("record_id")
|
|
2822
|
+
state.add_argument("--review", choices=sorted(REVIEW_STATES))
|
|
2823
|
+
state.add_argument("--freshness", choices=sorted(FRESHNESS_STATES))
|
|
2824
|
+
state.add_argument("--reason", required=True)
|
|
2825
|
+
_add_config_args(state)
|
|
2826
|
+
|
|
2827
|
+
sync = sub.add_parser("sync-provider")
|
|
2828
|
+
sync.add_argument(
|
|
2829
|
+
"--apply",
|
|
2830
|
+
action="store_true",
|
|
2831
|
+
help="Build, validate, back up, and replace the project-isolated active palace.",
|
|
2832
|
+
)
|
|
2833
|
+
_add_config_args(sync)
|
|
2834
|
+
|
|
2835
|
+
propose = sub.add_parser("propose-from-session")
|
|
2836
|
+
propose.add_argument("path", help="A Copilot session directory or events.jsonl.")
|
|
2837
|
+
propose.add_argument(
|
|
2838
|
+
"--repo",
|
|
2839
|
+
default=".",
|
|
2840
|
+
help="Repository supplying the project, branch, and HEAD anchors.",
|
|
2841
|
+
)
|
|
2842
|
+
propose.add_argument(
|
|
2843
|
+
"--write",
|
|
2844
|
+
dest="dry_run",
|
|
2845
|
+
action="store_false",
|
|
2846
|
+
help="Write candidate artifacts. Without this the run is a dry run.",
|
|
2847
|
+
)
|
|
2848
|
+
propose.add_argument(
|
|
2849
|
+
"--redact",
|
|
2850
|
+
action="store_true",
|
|
2851
|
+
help="Mask detected credential-shaped spans instead of refusing.",
|
|
2852
|
+
)
|
|
2853
|
+
propose.set_defaults(dry_run=True)
|
|
2854
|
+
_add_config_args(propose)
|
|
2855
|
+
|
|
2856
|
+
hook = sub.add_parser("hook")
|
|
2857
|
+
hook.add_argument(
|
|
2858
|
+
"event",
|
|
2859
|
+
choices=("session-start", "stop", "precompact", "session-end"),
|
|
2860
|
+
)
|
|
2861
|
+
_add_config_args(hook)
|
|
2862
|
+
return parser
|
|
2863
|
+
|
|
2864
|
+
|
|
2865
|
+
def main(argv: list[str] | None = None) -> int:
|
|
2866
|
+
args = _parser().parse_args(argv)
|
|
2867
|
+
try:
|
|
2868
|
+
if args.command == "validate":
|
|
2869
|
+
metadata = validate_memory(Path(args.artifact).expanduser().resolve())
|
|
2870
|
+
print(json.dumps({"status": "valid", "id": metadata["id"]}))
|
|
2871
|
+
return 0
|
|
2872
|
+
|
|
2873
|
+
config = _config(args)
|
|
2874
|
+
if args.command == "capture":
|
|
2875
|
+
return _capture_memory(args, config)
|
|
2876
|
+
if args.command == "archive-handoff":
|
|
2877
|
+
return _archive_handoff(args, config)
|
|
2878
|
+
if args.command == "search":
|
|
2879
|
+
if not 1 <= args.results <= 50:
|
|
2880
|
+
raise Refusal("--results must be between 1 and 50")
|
|
2881
|
+
return _search(args, config)
|
|
2882
|
+
if args.command == "wake":
|
|
2883
|
+
return _wake(args, config)
|
|
2884
|
+
if args.command == "doctor":
|
|
2885
|
+
return _doctor(args, config)
|
|
2886
|
+
if args.command == "audit":
|
|
2887
|
+
if args.limit < 1:
|
|
2888
|
+
raise Refusal("--limit must be at least 1")
|
|
2889
|
+
return _audit(args, config)
|
|
2890
|
+
if args.command == "review":
|
|
2891
|
+
return _record_review(config)
|
|
2892
|
+
if args.command == "record-state":
|
|
2893
|
+
return _record_state(args, config)
|
|
2894
|
+
if args.command == "sync-provider":
|
|
2895
|
+
return _sync_provider(args, config)
|
|
2896
|
+
if args.command == "propose-from-session":
|
|
2897
|
+
return _propose_from_session(args, config)
|
|
2898
|
+
if args.command == "hook":
|
|
2899
|
+
payload = sys.stdin.buffer.read()
|
|
2900
|
+
return _run_hook(args.event, config, payload)
|
|
2901
|
+
except (OSError, Refusal) as exc:
|
|
2902
|
+
print(json.dumps({"status": "refused", "error": str(exc)}), file=sys.stderr)
|
|
2903
|
+
return 2
|
|
2904
|
+
return 2
|
|
2905
|
+
|
|
2906
|
+
|
|
2907
|
+
if __name__ == "__main__":
|
|
2908
|
+
raise SystemExit(main())
|