rememberstack 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. rememberstack/__init__.py +9 -0
  2. rememberstack/adapters/__init__.py +42 -0
  3. rememberstack/adapters/codex_writer.py +221 -0
  4. rememberstack/adapters/markitdown_converter.py +42 -0
  5. rememberstack/adapters/openrouter.py +136 -0
  6. rememberstack/adapters/selfhost/__init__.py +54 -0
  7. rememberstack/adapters/selfhost/forget.py +66 -0
  8. rememberstack/adapters/selfhost/git.py +374 -0
  9. rememberstack/adapters/selfhost/lance.py +328 -0
  10. rememberstack/adapters/selfhost/minio.py +279 -0
  11. rememberstack/adapters/selfhost/mounts.py +249 -0
  12. rememberstack/adapters/selfhost/object_store.py +130 -0
  13. rememberstack/adapters/selfhost/projection.py +80 -0
  14. rememberstack/adapters/selfhost/queue.py +137 -0
  15. rememberstack/adapters/selfhost/telemetry.py +45 -0
  16. rememberstack/adapters/selfhost/watcher.py +70 -0
  17. rememberstack/adapters/testing/__init__.py +15 -0
  18. rememberstack/adapters/testing/cost_meter.py +13 -0
  19. rememberstack/adapters/testing/model_provider.py +83 -0
  20. rememberstack/adapters/testing/queue.py +43 -0
  21. rememberstack/adapters/testing/telemetry.py +22 -0
  22. rememberstack/client.py +19 -0
  23. rememberstack/core/__init__.py +127 -0
  24. rememberstack/core/blockizer.py +189 -0
  25. rememberstack/core/chunker.py +216 -0
  26. rememberstack/core/consumption_skill.py +275 -0
  27. rememberstack/core/conversion.py +76 -0
  28. rememberstack/core/core_manifest.py +598 -0
  29. rememberstack/core/extension_packs.py +124 -0
  30. rememberstack/core/forget.py +17 -0
  31. rememberstack/core/knowledge_authored.py +276 -0
  32. rememberstack/core/knowledge_compile.py +215 -0
  33. rememberstack/core/knowledge_fact_sheet.py +210 -0
  34. rememberstack/core/knowledge_hashing.py +68 -0
  35. rememberstack/core/knowledge_planner.py +64 -0
  36. rememberstack/core/knowledge_writer.py +175 -0
  37. rememberstack/core/ranking.py +200 -0
  38. rememberstack/core/recipe_linter.py +149 -0
  39. rememberstack/core/section_snap.py +209 -0
  40. rememberstack/core/storage_routing.py +27 -0
  41. rememberstack/eval/__init__.py +53 -0
  42. rememberstack/eval/consumption.py +141 -0
  43. rememberstack/eval/contradiction.py +184 -0
  44. rememberstack/eval/harness.py +136 -0
  45. rememberstack/eval/lifecycle.py +400 -0
  46. rememberstack/eval/operational_scale.py +49 -0
  47. rememberstack/eval/resolution.py +255 -0
  48. rememberstack/eval/retrieval_spikes.py +50 -0
  49. rememberstack/eval/skeleton.py +231 -0
  50. rememberstack/llm/__init__.py +1 -0
  51. rememberstack/model/__init__.py +589 -0
  52. rememberstack/model/adjudication.py +100 -0
  53. rememberstack/model/auth.py +27 -0
  54. rememberstack/model/blocks.py +30 -0
  55. rememberstack/model/chunks.py +190 -0
  56. rememberstack/model/claims.py +162 -0
  57. rememberstack/model/client.py +98 -0
  58. rememberstack/model/clustering.py +54 -0
  59. rememberstack/model/component_version.py +124 -0
  60. rememberstack/model/consumption.py +88 -0
  61. rememberstack/model/conversion.py +31 -0
  62. rememberstack/model/deployment.py +53 -0
  63. rememberstack/model/documents.py +168 -0
  64. rememberstack/model/envelope.py +513 -0
  65. rememberstack/model/evaluation.py +72 -0
  66. rememberstack/model/forget.py +143 -0
  67. rememberstack/model/git.py +13 -0
  68. rememberstack/model/knowledge.py +840 -0
  69. rememberstack/model/knowledge_authored.py +325 -0
  70. rememberstack/model/knowledge_planner.py +431 -0
  71. rememberstack/model/lifecycle.py +42 -0
  72. rememberstack/model/model_provider.py +78 -0
  73. rememberstack/model/mounts.py +24 -0
  74. rememberstack/model/object_store.py +21 -0
  75. rememberstack/model/operational_scale.py +59 -0
  76. rememberstack/model/operations.py +153 -0
  77. rememberstack/model/processing.py +228 -0
  78. rememberstack/model/queue.py +73 -0
  79. rememberstack/model/recipes.py +83 -0
  80. rememberstack/model/relations.py +79 -0
  81. rememberstack/model/resolution.py +83 -0
  82. rememberstack/model/retrieval_spikes.py +62 -0
  83. rememberstack/model/sections.py +120 -0
  84. rememberstack/model/telemetry.py +30 -0
  85. rememberstack/ports/__init__.py +29 -0
  86. rememberstack/ports/auth.py +16 -0
  87. rememberstack/ports/connector.py +23 -0
  88. rememberstack/ports/cost_meter.py +17 -0
  89. rememberstack/ports/forget.py +20 -0
  90. rememberstack/ports/git.py +20 -0
  91. rememberstack/ports/model_provider.py +28 -0
  92. rememberstack/ports/mounts.py +16 -0
  93. rememberstack/ports/object_store.py +27 -0
  94. rememberstack/ports/p1_index.py +92 -0
  95. rememberstack/ports/purge.py +93 -0
  96. rememberstack/ports/queue.py +23 -0
  97. rememberstack/ports/telemetry.py +21 -0
  98. rememberstack/profiles/__init__.py +22 -0
  99. rememberstack/profiles/selfhost.py +324 -0
  100. rememberstack/profiles/selfhost_forget.py +158 -0
  101. rememberstack/profiles/selfhost_operations.py +95 -0
  102. rememberstack/py.typed +1 -0
  103. rememberstack/spine/__init__.py +93 -0
  104. rememberstack/spine/admission.py +26 -0
  105. rememberstack/spine/backfill.py +168 -0
  106. rememberstack/spine/catalog_contract.py +742 -0
  107. rememberstack/spine/chunk_catalog.py +237 -0
  108. rememberstack/spine/claim_catalog.py +298 -0
  109. rememberstack/spine/clustering.py +740 -0
  110. rememberstack/spine/component_versions.py +208 -0
  111. rememberstack/spine/consumption.py +81 -0
  112. rememberstack/spine/deployment_bootstrap.py +445 -0
  113. rememberstack/spine/document_catalog.py +621 -0
  114. rememberstack/spine/entity_registry.py +205 -0
  115. rememberstack/spine/extension_packs.py +220 -0
  116. rememberstack/spine/fact_catalog.py +571 -0
  117. rememberstack/spine/forget.py +1753 -0
  118. rememberstack/spine/knowledge.py +5467 -0
  119. rememberstack/spine/lifecycle.py +1071 -0
  120. rememberstack/spine/migrations/__init__.py +1 -0
  121. rememberstack/spine/migrations/_helpers.py +153 -0
  122. rememberstack/spine/migrations/env.py +58 -0
  123. rememberstack/spine/migrations/script.py.mako +27 -0
  124. rememberstack/spine/migrations/versions/__init__.py +1 -0
  125. rememberstack/spine/migrations/versions/p0_02_0001_extensions_enums.py +189 -0
  126. rememberstack/spine/migrations/versions/p0_02_0002_infrastructure_registries.py +321 -0
  127. rememberstack/spine/migrations/versions/p0_02_0003_entities_evaluation_e0_e1.py +631 -0
  128. rememberstack/spine/migrations/versions/p0_02_0004_claims_facts_evidence.py +411 -0
  129. rememberstack/spine/migrations/versions/p0_02_0005_projection_knowledge_retrieval.py +391 -0
  130. rememberstack/spine/migrations/versions/p0_02_0006_partitions_views.py +158 -0
  131. rememberstack/spine/migrations/versions/p2_06_0007_invalidated_outcome.py +26 -0
  132. rememberstack/spine/migrations/versions/p3_01_0008_document_version_target.py +58 -0
  133. rememberstack/spine/migrations/versions/p3_05_0009_reconcile_stage.py +27 -0
  134. rememberstack/spine/migrations/versions/p3_07_0010_lifecycle_eval_suite.py +25 -0
  135. rememberstack/spine/migrations/versions/p4_01_0011_survivor_view_rewrite.py +57 -0
  136. rememberstack/spine/migrations/versions/p6_02_0012_knowledge_compile_recovery.py +58 -0
  137. rememberstack/spine/migrations/versions/p6_04_0013_knowledge_writer_ledger.py +46 -0
  138. rememberstack/spine/migrations/versions/p6_05_0014_knowledge_planner_runtime.py +217 -0
  139. rememberstack/spine/migrations/versions/p6_06_0015_authored_dispatch_runtime.py +38 -0
  140. rememberstack/spine/migrations/versions/p7_02_0016_operational_eval_suite.py +19 -0
  141. rememberstack/spine/migrations/versions/p7_05_0017_hard_forget.py +55 -0
  142. rememberstack/spine/observation_adjudication.py +778 -0
  143. rememberstack/spine/operations.py +298 -0
  144. rememberstack/spine/projection.py +662 -0
  145. rememberstack/spine/recipes.py +276 -0
  146. rememberstack/spine/resolver.py +763 -0
  147. rememberstack/spine/review.py +650 -0
  148. rememberstack/spine/settings.py +22 -0
  149. rememberstack/spine/supersession.py +510 -0
  150. rememberstack/spine/sync.py +128 -0
  151. rememberstack/spine/work_ledger.py +816 -0
  152. rememberstack/surfaces/__init__.py +110 -0
  153. rememberstack/surfaces/cli.py +447 -0
  154. rememberstack/surfaces/consumption_skill.py +87 -0
  155. rememberstack/surfaces/graph_queries.py +698 -0
  156. rememberstack/surfaces/http_api.py +377 -0
  157. rememberstack/surfaces/mcp.py +67 -0
  158. rememberstack/surfaces/query_engine.py +1591 -0
  159. rememberstack/surfaces/recipe_executor.py +185 -0
  160. rememberstack/surfaces/recipe_surface.py +219 -0
  161. rememberstack/surfaces/remote_mcp.py +133 -0
  162. rememberstack/surfaces/sdk.py +324 -0
  163. rememberstack/workers/__init__.py +155 -0
  164. rememberstack/workers/base.py +312 -0
  165. rememberstack/workers/e0.py +577 -0
  166. rememberstack/workers/e1.py +425 -0
  167. rememberstack/workers/e2.py +525 -0
  168. rememberstack/workers/e3.py +434 -0
  169. rememberstack/workers/forget.py +299 -0
  170. rememberstack/workers/knowledge_authored.py +146 -0
  171. rememberstack/workers/knowledge_driver.py +735 -0
  172. rememberstack/workers/knowledge_fact_sheet.py +123 -0
  173. rememberstack/workers/knowledge_planner.py +325 -0
  174. rememberstack/workers/knowledge_writer.py +393 -0
  175. rememberstack/workers/operations.py +42 -0
  176. rememberstack/workers/p1.py +234 -0
  177. rememberstack/workers/p2.py +513 -0
  178. rememberstack/workers/p2_analytics.py +276 -0
  179. rememberstack/workers/p3.py +673 -0
  180. rememberstack/workers/reconcile.py +485 -0
  181. rememberstack/workers/sync.py +168 -0
  182. rememberstack-0.1.0.dist-info/METADATA +213 -0
  183. rememberstack-0.1.0.dist-info/RECORD +186 -0
  184. rememberstack-0.1.0.dist-info/WHEEL +4 -0
  185. rememberstack-0.1.0.dist-info/entry_points.txt +2 -0
  186. rememberstack-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,189 @@
1
+ """The deterministic blockizer: document.md → the block sequence (D57, e1 §2).
2
+
3
+ One shared code path regardless of which converter produced the Markdown. The
4
+ parser profile is pinned (GFM: CommonMark + the table extension, via
5
+ markdown-it-py); normalization order is fixed (join hard-wrapped lines → NFC →
6
+ collapse internal whitespace → hash). Determinism is regression-tested by the
7
+ golden corpus in CI per `BLOCKIZER_VERSION` — drift is a version bump, never a
8
+ silent change.
9
+ """
10
+
11
+ import hashlib
12
+ from typing import Final
13
+ import unicodedata
14
+
15
+ from markdown_it import MarkdownIt
16
+ from markdown_it.token import Token
17
+
18
+ from rememberstack.model import Block
19
+ from rememberstack.model import BlockType
20
+
21
+ BLOCKIZER_VERSION: Final = "blockizer-2026.07:markdown-it-py-4:gfm-tables"
22
+ """Pins the parser library generation and enabled-extension set (e1 §2)."""
23
+
24
+ _ATOMIC_CONTAINERS: Final = {
25
+ "table_open": ("table_close", BlockType.TABLE),
26
+ "blockquote_open": ("blockquote_close", BlockType.QUOTE),
27
+ }
28
+ _LIST_OPENERS: Final = frozenset({"bullet_list_open", "ordered_list_open"})
29
+
30
+
31
+ def blockize(*, document_md: str) -> tuple[Block, ...]:
32
+ """Derive the deterministic block sequence from a document.md rendering.
33
+
34
+ Blocks are CommonMark block-level elements: paragraphs, headings, list
35
+ items, atomic tables, code fences, and block quotes. Offsets slice
36
+ `document_md` exactly; the hash is over the normalized text, so a pure
37
+ reflow (hard-wrap change) does not change identity.
38
+ """
39
+ tokens = _parser().parse(document_md)
40
+ line_offsets = _line_offsets(document_md=document_md)
41
+ blocks: list[Block] = []
42
+ index = 0
43
+ while index < len(tokens):
44
+ token = tokens[index]
45
+ consumed, block = _emit(
46
+ tokens=tokens,
47
+ index=index,
48
+ document_md=document_md,
49
+ line_offsets=line_offsets,
50
+ ordinal=len(blocks),
51
+ )
52
+ if block is not None:
53
+ blocks.append(block)
54
+ index += consumed
55
+ del token
56
+ return tuple(blocks)
57
+
58
+
59
+ def normalized_block_text(*, raw: str) -> str:
60
+ """Apply the fixed normalization order: join lines → NFC → collapse spaces."""
61
+ joined = " ".join(line.strip() for line in raw.splitlines())
62
+ composed = unicodedata.normalize("NFC", joined)
63
+ return " ".join(composed.split())
64
+
65
+
66
+ def block_hash(*, raw: str) -> str:
67
+ """The block identity: sha256 over the normalized text."""
68
+ digest = hashlib.sha256(normalized_block_text(raw=raw).encode("utf-8"))
69
+ return digest.hexdigest()
70
+
71
+
72
+ def _parser() -> MarkdownIt:
73
+ """The pinned GFM-profile parser: CommonMark plus the table extension."""
74
+ return MarkdownIt("commonmark").enable("table")
75
+
76
+
77
+ def _line_offsets(*, document_md: str) -> tuple[int, ...]:
78
+ """Char offset of each line start, plus the end-of-document sentinel."""
79
+ offsets = [0]
80
+ for line in document_md.splitlines(keepends=True):
81
+ offsets.append(offsets[-1] + len(line))
82
+ return tuple(offsets)
83
+
84
+
85
+ def _emit(
86
+ *,
87
+ tokens: list[Token],
88
+ index: int,
89
+ document_md: str,
90
+ line_offsets: tuple[int, ...],
91
+ ordinal: int,
92
+ ) -> tuple[int, Block | None]:
93
+ """Emit at most one block starting at tokens[index]; return tokens consumed.
94
+
95
+ List containers recurse into their items; each top-level list item is one
96
+ atomic block whose span includes any nested sub-list (deliberate: emitting
97
+ nested items separately would create overlapping spans, and chunks require
98
+ non-overlapping whole-block runs — e1 §4). Other containers and leaves map
99
+ directly. Unknown structural tokens are skipped one at a time — the golden
100
+ corpus locks the observable result.
101
+ """
102
+ token = tokens[index]
103
+ if token.type in _LIST_OPENERS:
104
+ return 1, None # items are emitted individually; the wrapper is not a block
105
+ if token.type == "list_item_open":
106
+ close = _matching_close(
107
+ tokens=tokens, index=index, close_type="list_item_close"
108
+ )
109
+ return (
110
+ close - index + 1,
111
+ _block_from_lines(
112
+ token=token,
113
+ block_type=BlockType.LIST_ITEM,
114
+ document_md=document_md,
115
+ line_offsets=line_offsets,
116
+ ordinal=ordinal,
117
+ ),
118
+ )
119
+ if token.type in _ATOMIC_CONTAINERS:
120
+ close_type, block_type = _ATOMIC_CONTAINERS[token.type]
121
+ close = _matching_close(tokens=tokens, index=index, close_type=close_type)
122
+ return (
123
+ close - index + 1,
124
+ _block_from_lines(
125
+ token=token,
126
+ block_type=block_type,
127
+ document_md=document_md,
128
+ line_offsets=line_offsets,
129
+ ordinal=ordinal,
130
+ ),
131
+ )
132
+ if token.type == "heading_open":
133
+ return 3, _block_from_lines(
134
+ token=token,
135
+ block_type=BlockType.HEADING,
136
+ document_md=document_md,
137
+ line_offsets=line_offsets,
138
+ ordinal=ordinal,
139
+ )
140
+ if token.type == "paragraph_open":
141
+ return 3, _block_from_lines(
142
+ token=token,
143
+ block_type=BlockType.PARAGRAPH,
144
+ document_md=document_md,
145
+ line_offsets=line_offsets,
146
+ ordinal=ordinal,
147
+ )
148
+ if token.type in ("fence", "code_block"):
149
+ return 1, _block_from_lines(
150
+ token=token,
151
+ block_type=BlockType.CODE,
152
+ document_md=document_md,
153
+ line_offsets=line_offsets,
154
+ ordinal=ordinal,
155
+ )
156
+ return 1, None
157
+
158
+
159
+ def _matching_close(*, tokens: list[Token], index: int, close_type: str) -> int:
160
+ """Index of the container's matching close token at the same nesting level."""
161
+ level = tokens[index].level
162
+ for probe in range(index + 1, len(tokens)):
163
+ if tokens[probe].type == close_type and tokens[probe].level == level:
164
+ return probe
165
+ raise ValueError(f"unbalanced container at token {index}: no {close_type}")
166
+
167
+
168
+ def _block_from_lines(
169
+ *,
170
+ token: Token,
171
+ block_type: BlockType,
172
+ document_md: str,
173
+ line_offsets: tuple[int, ...],
174
+ ordinal: int,
175
+ ) -> Block:
176
+ """Build the block record from a token's source line map."""
177
+ if token.map is None:
178
+ raise ValueError(f"token {token.type} carries no source map")
179
+ start_line, end_line = token.map
180
+ char_start = line_offsets[start_line]
181
+ char_end = line_offsets[end_line]
182
+ raw = document_md[char_start:char_end].rstrip("\r\n")
183
+ return Block(
184
+ ordinal=ordinal,
185
+ type=block_type,
186
+ char_start=char_start,
187
+ char_end=char_start + len(raw),
188
+ block_hash=block_hash(raw=raw),
189
+ )
@@ -0,0 +1,216 @@
1
+ """The deterministic E1 chunker: anchor-stabilized packing of whole blocks (D58).
2
+
3
+ A chunk is an ordered run of whole blocks within one section, packed to a
4
+ token budget. Boundaries are stabilized by content anchors: a block whose hash
5
+ satisfies the anchor predicate forces a boundary before it, so an early edit
6
+ perturbs packing only up to the next anchor instead of rippling through the
7
+ document (e1 §4). Chunks never overlap and never split a block; an oversized
8
+ block becomes its own oversized chunk rather than being cut.
9
+
10
+ chunks = f(blocks, sections, budget, anchors, CHUNKER_VERSION): a parameter
11
+ change is a version bump and a cheap repack of existing atoms.
12
+ """
13
+
14
+ import hashlib
15
+ from typing import Final
16
+
17
+ from pydantic import BaseModel
18
+ from pydantic import ConfigDict
19
+ from pydantic import Field
20
+
21
+ from rememberstack.model import Block
22
+ from rememberstack.model import PackedChunk
23
+ from rememberstack.model import SectionSpan
24
+
25
+ CHUNKER_VERSION: Final = "e1-chunker-2026.07b:whitespace-tokens:anchored:owner-runs"
26
+ """Pins the packing algorithm and the token counter; the full packing
27
+ generation additionally encodes the parameter values — see `chunker_version`."""
28
+
29
+
30
+ class ChunkerParams(BaseModel):
31
+ """The bound packing parameters (e1 §4). Values are starting points to
32
+ measure (spike 3), never committed constants; changing them is a
33
+ CHUNKER_VERSION bump."""
34
+
35
+ model_config = ConfigDict(frozen=True, extra="forbid")
36
+
37
+ token_budget: int = Field(default=400, ge=1)
38
+ anchor_modulus: int = Field(default=24, ge=1)
39
+ anchor_min_gap_tokens: int = Field(default=200, ge=0)
40
+
41
+
42
+ def chunker_version(*, params: "ChunkerParams") -> str:
43
+ """The complete packing-generation identity: algorithm + parameter values.
44
+
45
+ chunks = f(blocks, sections, budget, anchors, version) — so changing any
46
+ parameter re-keys packing automatically instead of silently replaying
47
+ rows produced under different numbers (D58).
48
+ """
49
+ return (
50
+ f"{CHUNKER_VERSION}"
51
+ f":b{params.token_budget}"
52
+ f"-m{params.anchor_modulus}"
53
+ f"-g{params.anchor_min_gap_tokens}"
54
+ )
55
+
56
+
57
+ def pack_blocks(
58
+ *,
59
+ blocks: tuple[Block, ...],
60
+ sections: tuple[SectionSpan, ...],
61
+ document_md: str,
62
+ params: ChunkerParams,
63
+ ) -> tuple[PackedChunk, ...]:
64
+ """Pack the block grid into chunks, deepest-owner run by run.
65
+
66
+ Every block belongs to exactly one DEEPEST section (D57): a leaf owns
67
+ its whole range, and a parent directly owns the blocks none of its
68
+ children cover (content before the first child, and gaps between
69
+ children — the snap assigns those to the parent, never to a child).
70
+ Each contiguous run of same-owner blocks packs independently, so no
71
+ block is chunked twice and none is silently dropped (Codex review: a
72
+ leaf-only walk lost every parent's direct content). Within a run,
73
+ blocks accumulate greedily to the token budget; a chunk boundary is
74
+ forced before every anchor block, and a block that alone exceeds the
75
+ budget ships as its own oversized chunk. Sections are never crossed
76
+ (§3 makes the partition well-defined).
77
+ """
78
+ chunks: list[PackedChunk] = []
79
+ for section, run_start, run_end in _owner_runs(sections=sections):
80
+ run_blocks = tuple(
81
+ block for block in blocks if run_start <= block.ordinal <= run_end
82
+ )
83
+ chunks.extend(
84
+ _pack_section(
85
+ section=section,
86
+ blocks=run_blocks,
87
+ document_md=document_md,
88
+ params=params,
89
+ first_ordinal=len(chunks),
90
+ )
91
+ )
92
+ return tuple(chunks)
93
+
94
+
95
+ def _owner_runs(
96
+ *, sections: tuple[SectionSpan, ...]
97
+ ) -> tuple[tuple[SectionSpan, int, int], ...]:
98
+ """Every contiguous block run with its deepest-owning section, in
99
+ document order: a section's runs are its range minus its direct
100
+ children's ranges."""
101
+ runs: list[tuple[SectionSpan, int, int]] = []
102
+ for section in sections:
103
+ prefix = f"{section.node_path}."
104
+ child_ranges = sorted(
105
+ (other.block_start, other.block_end)
106
+ for other in sections
107
+ if other.node_path.startswith(prefix)
108
+ and "." not in other.node_path.removeprefix(prefix)
109
+ )
110
+ cursor = section.block_start
111
+ for child_start, child_end in child_ranges:
112
+ if child_start > cursor:
113
+ runs.append((section, cursor, child_start - 1))
114
+ cursor = max(cursor, child_end + 1)
115
+ if cursor <= section.block_end:
116
+ runs.append((section, cursor, section.block_end))
117
+ return tuple(sorted(runs, key=lambda run: run[1]))
118
+
119
+
120
+ def chunk_content_hash(*, block_hashes: tuple[str, ...]) -> str:
121
+ """The chunk identity: sha256 over the ordered block-hash sequence (D58)."""
122
+ digest = hashlib.sha256("\n".join(block_hashes).encode("utf-8"))
123
+ return digest.hexdigest()
124
+
125
+
126
+ def extraction_input_hash(
127
+ *,
128
+ own_block_hashes: tuple[str, ...],
129
+ neighbor_block_hashes: tuple[str, ...],
130
+ header_facts: tuple[str, ...],
131
+ extractor_version: str,
132
+ structurer_version: str,
133
+ ) -> str:
134
+ """The D56 reuse key: stable inputs of the E2 bundle, no LLM output.
135
+
136
+ Own blocks + neighbor blocks + deterministic document metadata + the
137
+ extractor and structurer versions. Prefixes, summaries, and section paths
138
+ are carried forward on reuse, never keyed — so an unchanged key within a
139
+ lineage means the prior claims are re-attached instead of re-extracted.
140
+ """
141
+ payload = "\x1e".join(
142
+ (
143
+ "\n".join(own_block_hashes),
144
+ "\n".join(neighbor_block_hashes),
145
+ "\n".join(header_facts),
146
+ extractor_version,
147
+ structurer_version,
148
+ )
149
+ )
150
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
151
+
152
+
153
+ def count_tokens(*, text: str) -> int:
154
+ """The pinned token counter: whitespace tokens (part of CHUNKER_VERSION)."""
155
+ return len(text.split())
156
+
157
+
158
+ def is_anchor(*, block_hash: str, params: ChunkerParams) -> bool:
159
+ """The anchor predicate's hash half: uint64(block_hash) mod M == 0 (e1 §4)."""
160
+ return int(block_hash[:16], 16) % params.anchor_modulus == 0
161
+
162
+
163
+ def _pack_section(
164
+ *,
165
+ section: SectionSpan,
166
+ blocks: tuple[Block, ...],
167
+ document_md: str,
168
+ params: ChunkerParams,
169
+ first_ordinal: int,
170
+ ) -> tuple[PackedChunk, ...]:
171
+ """Pack one section's blocks; boundaries never leave the section."""
172
+ chunks: list[PackedChunk] = []
173
+ run: list[tuple[Block, int]] = []
174
+ run_tokens = 0
175
+ tokens_since_anchor = params.anchor_min_gap_tokens # first anchor never suppressed
176
+
177
+ def flush() -> None:
178
+ nonlocal run, run_tokens
179
+ if not run:
180
+ return
181
+ run_blocks = tuple(block for block, _ in run)
182
+ chunks.append(
183
+ PackedChunk(
184
+ ordinal=first_ordinal + len(chunks),
185
+ section_id=section.section_id,
186
+ block_start=run_blocks[0].ordinal,
187
+ block_end=run_blocks[-1].ordinal,
188
+ char_start=run_blocks[0].char_start,
189
+ char_end=run_blocks[-1].char_end,
190
+ chunk_content_hash=chunk_content_hash(
191
+ block_hashes=tuple(block.block_hash for block in run_blocks)
192
+ ),
193
+ token_count=run_tokens,
194
+ )
195
+ )
196
+ run = []
197
+ run_tokens = 0
198
+
199
+ for block in blocks:
200
+ tokens = count_tokens(text=document_md[block.char_start : block.char_end])
201
+ anchored = (
202
+ is_anchor(block_hash=block.block_hash, params=params)
203
+ and tokens_since_anchor >= params.anchor_min_gap_tokens
204
+ )
205
+ if anchored:
206
+ flush() # packing after an anchor is independent of everything before
207
+ tokens_since_anchor = 0
208
+ elif run and run_tokens + tokens > params.token_budget:
209
+ flush() # budget boundary: the block starts the next run
210
+ run.append((block, tokens))
211
+ run_tokens += tokens
212
+ tokens_since_anchor += tokens
213
+ if run_tokens > params.token_budget:
214
+ flush() # a single oversized block ships as its own oversized chunk
215
+ flush()
216
+ return tuple(chunks)
@@ -0,0 +1,275 @@
1
+ """Pure renderer for the versioned D51 agent-consumption skill."""
2
+
3
+ import json
4
+ from typing import Final
5
+
6
+ from rememberstack.model import ConsumptionRecipe
7
+ from rememberstack.model import ConsumptionScope
8
+ from rememberstack.model import ConsumptionSkillContext
9
+ from rememberstack.model import PublishedMounts
10
+ from rememberstack.model import RenderedConsumptionSkill
11
+
12
+ CONSUMPTION_SKILL_VERSION: Final = "1.0.0"
13
+
14
+
15
+ def render_consumption_skill(
16
+ *, context: ConsumptionSkillContext
17
+ ) -> RenderedConsumptionSkill:
18
+ """Render one complete ``SKILL.md`` from typed deployment state."""
19
+ deployment = context.deployment
20
+ sections = (
21
+ _header(),
22
+ _deployment(context=context),
23
+ _default_motion(
24
+ knowledge_page_count=deployment.knowledge_page_count,
25
+ recipes=context.recipes,
26
+ ),
27
+ _grains(),
28
+ _testimony(recipes=context.recipes),
29
+ _time_and_media(),
30
+ _envelope(),
31
+ _mounts(mounts=context.mounts),
32
+ _recipes(recipes=context.recipes),
33
+ _working_rules(),
34
+ )
35
+ return RenderedConsumptionSkill(
36
+ deployment_id=deployment.deployment_id,
37
+ version=CONSUMPTION_SKILL_VERSION,
38
+ content="\n\n".join(sections).rstrip() + "\n",
39
+ )
40
+
41
+
42
+ def _header() -> str:
43
+ """The stable skill identity and revision."""
44
+ return (
45
+ "---\n"
46
+ "name: rememberstack\n"
47
+ "description: Use one configured RememberStack deployment without "
48
+ "mixing facts, testimony, and compiled knowledge.\n"
49
+ "---\n\n"
50
+ "# Use RememberStack\n\n"
51
+ f"Skill revision: `{CONSUMPTION_SKILL_VERSION}`. Follow these instructions "
52
+ "when a task depends on the configured memory."
53
+ )
54
+
55
+
56
+ def _deployment(*, context: ConsumptionSkillContext) -> str:
57
+ """Render deployment identity, language, scopes, and current K state."""
58
+ deployment = context.deployment
59
+ description = (
60
+ f"\n- Purpose: {_literal(value=deployment.description)}"
61
+ if deployment.description
62
+ else ""
63
+ )
64
+ scopes = _scope_lines(scopes=deployment.scopes)
65
+ knowledge_state = (
66
+ "known empty: no K pages are registered"
67
+ if deployment.knowledge_page_count == 0
68
+ else f"{deployment.knowledge_page_count} K page(s) are registered"
69
+ )
70
+ return (
71
+ "## This deployment\n\n"
72
+ f"- Name: {_literal(value=deployment.name)}\n"
73
+ f"- Slug: `{deployment.slug}`\n"
74
+ f"- Deployment id: `{deployment.deployment_id}`\n"
75
+ f"- Default language: `{deployment.default_language}`"
76
+ f"{description}\n"
77
+ f"- Plane K state: {knowledge_state}.\n"
78
+ f"- Special-purpose scopes:\n{scopes}"
79
+ )
80
+
81
+
82
+ def _default_motion(
83
+ *, knowledge_page_count: int, recipes: tuple[ConsumptionRecipe, ...]
84
+ ) -> str:
85
+ """Teach the one progressive-disclosure motion and honest empty-K fallback."""
86
+ orientation_route = (
87
+ "Read the knowledge checkout or use the active `pages_about` orientation "
88
+ "recipe."
89
+ if any(recipe.name == "pages_about" for recipe in recipes)
90
+ else "Read the knowledge checkout, or use an orientation-intent recipe if "
91
+ "one is enabled."
92
+ )
93
+ empty_instruction = (
94
+ "This deployment currently has no K pages. The orientation attempt is still "
95
+ "correct, but an empty/`known_empty` result means: fall back to the P3 corpus "
96
+ "tree when mounted, or to search when unmounted. Never invent the missing "
97
+ "summary."
98
+ if knowledge_page_count == 0
99
+ else "If K returns `known_empty`, fall back to P3 or search; never turn an "
100
+ "empty orientation layer into an invented summary."
101
+ )
102
+ return (
103
+ "## Default motion: orient, verify, audit\n\n"
104
+ f"1. **Orient on plane K.** {orientation_route} K is cheap, pre-paid "
105
+ "synthesis, not live-confirmed truth.\n"
106
+ "2. **Verify on the spine.** For anything load-bearing, query the fact layer "
107
+ "(relations or observations). A fact lookup re-checks live PostgreSQL state.\n"
108
+ "3. **Audit on evidence.** When the stakes or ambiguity demand it, hydrate "
109
+ "the fact to claims, source spans, documents, and finally the original.\n\n"
110
+ f"{empty_instruction}"
111
+ )
112
+
113
+
114
+ def _grains() -> str:
115
+ """Teach the claim-to-fact-to-compiled terminology ladder."""
116
+ return (
117
+ "## Keep the grains separate\n\n"
118
+ "- A **claim** is immutable testimony: what one source asserted. It is "
119
+ "evidence grain and may be stale, superseded, or contradicted.\n"
120
+ "- A **relation** links two entities; an **observation** records a value or "
121
+ "statement about one entity. Together they are the **fact layer**: the "
122
+ "system's adjudicated, validity-filtered holdings. Questions of the form "
123
+ '"is this true now?" go here by default.\n'
124
+ "- A **compiled K page** is pre-paid synthesis. It is compiled grain and "
125
+ "must be read with its compile time, stale flag, and open-flag count.\n"
126
+ "- A **core belief** is a stricter configured K tier, not a new source of "
127
+ "truth.\n\n"
128
+ "Never blend evidence and facts into one unlabeled answer. If a task asks "
129
+ "both what someone said and what the system believes, return separate "
130
+ "evidence-grain and fact-grain parts."
131
+ )
132
+
133
+
134
+ def _testimony(*, recipes: tuple[ConsumptionRecipe, ...]) -> str:
135
+ """Teach current testimony, historical opt-in, and withdrawn support."""
136
+ recipe_names = {recipe.name for recipe in recipes}
137
+ history_surface = (
138
+ "This deployment enables `claims_as_of`; use it only for assertion "
139
+ "history, never for current truth."
140
+ if "claims_as_of" in recipe_names
141
+ else "This deployment does not enable a `claims_as_of` recipe. Its current "
142
+ "query surfaces do not expose superseded testimony, so do not attempt an "
143
+ "undeclared history option."
144
+ )
145
+ return (
146
+ "## Testimony currency and shaky support\n\n"
147
+ "Claim search defaults to **current testimony**. Claims left behind by a "
148
+ "living document's newer version or by a newer extraction generation are "
149
+ "history, not current search results. `claims_as_of` means **what sources "
150
+ "asserted as of a past system time**; it never means what is true now. "
151
+ f"{history_surface}\n\n"
152
+ "A fact with `support: withdrawn` has lost all current-testimony support "
153
+ "because a toolchain re-read did not re-derive it. It still stands while "
154
+ "review is open, but it is shaky: report the caveat, inspect its transcript "
155
+ "and evidence, and do not make it load-bearing without verification."
156
+ )
157
+
158
+
159
+ def _time_and_media() -> str:
160
+ """Keep the three media/time coordinates and derivation labels distinct."""
161
+ return (
162
+ "## Time and media\n\n"
163
+ 'Do not collapse these into "the timestamp":\n\n'
164
+ "- a source locator such as `start_ms` says **where in a file** evidence "
165
+ "occurs;\n"
166
+ "- `valid_from` / `valid_until` say **when a fact held in the world**;\n"
167
+ "- `ingested_at` / `believed_at` say **when the system knew it**.\n\n"
168
+ "For media-derived evidence, read `evidence_mode`: `source_expression` is "
169
+ "rendered speech/text, `model_observation` is what a model reports seeing, "
170
+ "and `model_interpretation` is the model's interpretation. When tone or a "
171
+ "visual detail matters, follow the source locator to the raw interval or "
172
+ "region. The transcript is a map, not the territory."
173
+ )
174
+
175
+
176
+ def _envelope() -> str:
177
+ """Teach response honesty fields and the distinct negative reactions."""
178
+ return (
179
+ "## Read the whole response envelope\n\n"
180
+ "Check `grain`, applied `valid_at`/`believed_at`, identity regime, per-store "
181
+ "freshness, truncation/continuation, and `dropped_by_hydration`. A result "
182
+ "inside a live contradiction group must include or point to its co-members; "
183
+ "report the competing sides instead of silently picking one.\n\n"
184
+ "Negative results require different moves:\n\n"
185
+ "- `unknown_entity`: widen resolution or search;\n"
186
+ "- `known_empty`: the entity exists but no matching result is known within "
187
+ "the stated freshness;\n"
188
+ "- `boundary`: re-plan using the named workaround.\n\n"
189
+ "Hard-forgotten material is intentionally indistinguishable from content "
190
+ "that never existed."
191
+ )
192
+
193
+
194
+ def _mounts(*, mounts: PublishedMounts | None) -> str:
195
+ """Render exact mount paths or the unmounted parity rule."""
196
+ if mounts is None:
197
+ availability = (
198
+ "No mounts are available in this harness. Use API, CLI, or MCP for "
199
+ "orientation, readable artifacts, and query operations."
200
+ )
201
+ else:
202
+ availability = (
203
+ "The four read-only mounts are available:\n\n"
204
+ f"- P3 corpus tree: `{mounts.p3}`\n"
205
+ f"- E0 artifacts: `{mounts.artifacts}`\n"
206
+ f"- raw originals (off the navigation path; audited): `{mounts.raw}`\n"
207
+ f"- plane K checkout: `{mounts.knowledge}`"
208
+ )
209
+ return (
210
+ "## Filesystem first\n\n"
211
+ f"{availability}\n\n"
212
+ "When mounts exist, prefer them for navigation, reading, and grep. Reserve "
213
+ "API/CLI/MCP for operations with no filesystem equivalent: semantic search, "
214
+ "graph traversal, temporal as-of queries, hydration, transcripts, and "
215
+ "deltas. Start in P3 or K, not raw. Follow an explicit raw pointer only "
216
+ "when the original is needed, and use the deployment's audited raw-access "
217
+ "mechanism."
218
+ )
219
+
220
+
221
+ def _recipes(*, recipes: tuple[ConsumptionRecipe, ...]) -> str:
222
+ """Render only this deployment's latest active recipe versions."""
223
+ if not recipes:
224
+ rows = "No recipes are enabled. Use the primitive API directly."
225
+ else:
226
+ rows = "\n".join(
227
+ f"- `{recipe.name}` — `{recipe.output_grain}` / "
228
+ f"`{recipe.answer_intent}`: {_one_line(value=recipe.description)}"
229
+ for recipe in recipes
230
+ )
231
+ return (
232
+ "## Enabled recipes and surfaces\n\n"
233
+ f"{rows}\n\n"
234
+ "Discover the current set with `remember query list`, `GET /recipes`, or MCP "
235
+ "tool listing. Run one with `remember query run <name> --arg key=value`, "
236
+ "`POST /recipe/<name>`, or the same-named MCP tool. Recipe grain and intent "
237
+ "are part of the contract; a recipe adds no capability beyond its primitive "
238
+ "chain."
239
+ )
240
+
241
+
242
+ def _working_rules() -> str:
243
+ """End with a compact operational checklist for the consuming agent."""
244
+ return (
245
+ "## Before acting on a memory answer\n\n"
246
+ "1. Did I use facts, not claims, for a current-truth question?\n"
247
+ "2. Did I keep fact, evidence, and compiled grains labeled separately?\n"
248
+ "3. Did I inspect freshness, truncation, contradictions, and withdrawn "
249
+ "support?\n"
250
+ "4. Did I verify load-bearing K statements on the spine?\n"
251
+ "5. Did I hydrate to evidence or raw source when the stakes required it?"
252
+ )
253
+
254
+
255
+ def _scope_lines(*, scopes: tuple[ConsumptionScope, ...]) -> str:
256
+ """Render special-purpose scope rows without inventing a default K page."""
257
+ if not scopes:
258
+ return " - none registered"
259
+ return "\n".join(
260
+ f" - `{scope.slug}` ({_one_line(value=scope.name)})"
261
+ + (f" at `{scope.git_path}`" if scope.git_path else "")
262
+ + (f": {_one_line(value=scope.description)}" if scope.description else "")
263
+ for scope in scopes
264
+ )
265
+
266
+
267
+ def _literal(*, value: str) -> str:
268
+ """Render deployment-controlled prose as one explicit JSON string literal."""
269
+ literal = json.dumps(value, ensure_ascii=False).replace("`", "'")
270
+ return f"`{literal}`"
271
+
272
+
273
+ def _one_line(*, value: str) -> str:
274
+ """Collapse deployment-controlled display text to one Markdown-safe line."""
275
+ return " ".join(value.split()).replace("`", "'")