rememberstack 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. rememberstack/__init__.py +9 -0
  2. rememberstack/adapters/__init__.py +42 -0
  3. rememberstack/adapters/codex_writer.py +221 -0
  4. rememberstack/adapters/markitdown_converter.py +42 -0
  5. rememberstack/adapters/openrouter.py +136 -0
  6. rememberstack/adapters/selfhost/__init__.py +54 -0
  7. rememberstack/adapters/selfhost/forget.py +66 -0
  8. rememberstack/adapters/selfhost/git.py +374 -0
  9. rememberstack/adapters/selfhost/lance.py +328 -0
  10. rememberstack/adapters/selfhost/minio.py +279 -0
  11. rememberstack/adapters/selfhost/mounts.py +249 -0
  12. rememberstack/adapters/selfhost/object_store.py +130 -0
  13. rememberstack/adapters/selfhost/projection.py +80 -0
  14. rememberstack/adapters/selfhost/queue.py +137 -0
  15. rememberstack/adapters/selfhost/telemetry.py +45 -0
  16. rememberstack/adapters/selfhost/watcher.py +70 -0
  17. rememberstack/adapters/testing/__init__.py +15 -0
  18. rememberstack/adapters/testing/cost_meter.py +13 -0
  19. rememberstack/adapters/testing/model_provider.py +83 -0
  20. rememberstack/adapters/testing/queue.py +43 -0
  21. rememberstack/adapters/testing/telemetry.py +22 -0
  22. rememberstack/client.py +19 -0
  23. rememberstack/core/__init__.py +127 -0
  24. rememberstack/core/blockizer.py +189 -0
  25. rememberstack/core/chunker.py +216 -0
  26. rememberstack/core/consumption_skill.py +275 -0
  27. rememberstack/core/conversion.py +76 -0
  28. rememberstack/core/core_manifest.py +598 -0
  29. rememberstack/core/extension_packs.py +124 -0
  30. rememberstack/core/forget.py +17 -0
  31. rememberstack/core/knowledge_authored.py +276 -0
  32. rememberstack/core/knowledge_compile.py +215 -0
  33. rememberstack/core/knowledge_fact_sheet.py +210 -0
  34. rememberstack/core/knowledge_hashing.py +68 -0
  35. rememberstack/core/knowledge_planner.py +64 -0
  36. rememberstack/core/knowledge_writer.py +175 -0
  37. rememberstack/core/ranking.py +200 -0
  38. rememberstack/core/recipe_linter.py +149 -0
  39. rememberstack/core/section_snap.py +209 -0
  40. rememberstack/core/storage_routing.py +27 -0
  41. rememberstack/eval/__init__.py +53 -0
  42. rememberstack/eval/consumption.py +141 -0
  43. rememberstack/eval/contradiction.py +184 -0
  44. rememberstack/eval/harness.py +136 -0
  45. rememberstack/eval/lifecycle.py +400 -0
  46. rememberstack/eval/operational_scale.py +49 -0
  47. rememberstack/eval/resolution.py +255 -0
  48. rememberstack/eval/retrieval_spikes.py +50 -0
  49. rememberstack/eval/skeleton.py +231 -0
  50. rememberstack/llm/__init__.py +1 -0
  51. rememberstack/model/__init__.py +589 -0
  52. rememberstack/model/adjudication.py +100 -0
  53. rememberstack/model/auth.py +27 -0
  54. rememberstack/model/blocks.py +30 -0
  55. rememberstack/model/chunks.py +190 -0
  56. rememberstack/model/claims.py +162 -0
  57. rememberstack/model/client.py +98 -0
  58. rememberstack/model/clustering.py +54 -0
  59. rememberstack/model/component_version.py +124 -0
  60. rememberstack/model/consumption.py +88 -0
  61. rememberstack/model/conversion.py +31 -0
  62. rememberstack/model/deployment.py +53 -0
  63. rememberstack/model/documents.py +168 -0
  64. rememberstack/model/envelope.py +513 -0
  65. rememberstack/model/evaluation.py +72 -0
  66. rememberstack/model/forget.py +143 -0
  67. rememberstack/model/git.py +13 -0
  68. rememberstack/model/knowledge.py +840 -0
  69. rememberstack/model/knowledge_authored.py +325 -0
  70. rememberstack/model/knowledge_planner.py +431 -0
  71. rememberstack/model/lifecycle.py +42 -0
  72. rememberstack/model/model_provider.py +78 -0
  73. rememberstack/model/mounts.py +24 -0
  74. rememberstack/model/object_store.py +21 -0
  75. rememberstack/model/operational_scale.py +59 -0
  76. rememberstack/model/operations.py +153 -0
  77. rememberstack/model/processing.py +228 -0
  78. rememberstack/model/queue.py +73 -0
  79. rememberstack/model/recipes.py +83 -0
  80. rememberstack/model/relations.py +79 -0
  81. rememberstack/model/resolution.py +83 -0
  82. rememberstack/model/retrieval_spikes.py +62 -0
  83. rememberstack/model/sections.py +120 -0
  84. rememberstack/model/telemetry.py +30 -0
  85. rememberstack/ports/__init__.py +29 -0
  86. rememberstack/ports/auth.py +16 -0
  87. rememberstack/ports/connector.py +23 -0
  88. rememberstack/ports/cost_meter.py +17 -0
  89. rememberstack/ports/forget.py +20 -0
  90. rememberstack/ports/git.py +20 -0
  91. rememberstack/ports/model_provider.py +28 -0
  92. rememberstack/ports/mounts.py +16 -0
  93. rememberstack/ports/object_store.py +27 -0
  94. rememberstack/ports/p1_index.py +92 -0
  95. rememberstack/ports/purge.py +93 -0
  96. rememberstack/ports/queue.py +23 -0
  97. rememberstack/ports/telemetry.py +21 -0
  98. rememberstack/profiles/__init__.py +22 -0
  99. rememberstack/profiles/selfhost.py +324 -0
  100. rememberstack/profiles/selfhost_forget.py +158 -0
  101. rememberstack/profiles/selfhost_operations.py +95 -0
  102. rememberstack/py.typed +1 -0
  103. rememberstack/spine/__init__.py +93 -0
  104. rememberstack/spine/admission.py +26 -0
  105. rememberstack/spine/backfill.py +168 -0
  106. rememberstack/spine/catalog_contract.py +742 -0
  107. rememberstack/spine/chunk_catalog.py +237 -0
  108. rememberstack/spine/claim_catalog.py +298 -0
  109. rememberstack/spine/clustering.py +740 -0
  110. rememberstack/spine/component_versions.py +208 -0
  111. rememberstack/spine/consumption.py +81 -0
  112. rememberstack/spine/deployment_bootstrap.py +445 -0
  113. rememberstack/spine/document_catalog.py +621 -0
  114. rememberstack/spine/entity_registry.py +205 -0
  115. rememberstack/spine/extension_packs.py +220 -0
  116. rememberstack/spine/fact_catalog.py +571 -0
  117. rememberstack/spine/forget.py +1753 -0
  118. rememberstack/spine/knowledge.py +5467 -0
  119. rememberstack/spine/lifecycle.py +1071 -0
  120. rememberstack/spine/migrations/__init__.py +1 -0
  121. rememberstack/spine/migrations/_helpers.py +153 -0
  122. rememberstack/spine/migrations/env.py +58 -0
  123. rememberstack/spine/migrations/script.py.mako +27 -0
  124. rememberstack/spine/migrations/versions/__init__.py +1 -0
  125. rememberstack/spine/migrations/versions/p0_02_0001_extensions_enums.py +189 -0
  126. rememberstack/spine/migrations/versions/p0_02_0002_infrastructure_registries.py +321 -0
  127. rememberstack/spine/migrations/versions/p0_02_0003_entities_evaluation_e0_e1.py +631 -0
  128. rememberstack/spine/migrations/versions/p0_02_0004_claims_facts_evidence.py +411 -0
  129. rememberstack/spine/migrations/versions/p0_02_0005_projection_knowledge_retrieval.py +391 -0
  130. rememberstack/spine/migrations/versions/p0_02_0006_partitions_views.py +158 -0
  131. rememberstack/spine/migrations/versions/p2_06_0007_invalidated_outcome.py +26 -0
  132. rememberstack/spine/migrations/versions/p3_01_0008_document_version_target.py +58 -0
  133. rememberstack/spine/migrations/versions/p3_05_0009_reconcile_stage.py +27 -0
  134. rememberstack/spine/migrations/versions/p3_07_0010_lifecycle_eval_suite.py +25 -0
  135. rememberstack/spine/migrations/versions/p4_01_0011_survivor_view_rewrite.py +57 -0
  136. rememberstack/spine/migrations/versions/p6_02_0012_knowledge_compile_recovery.py +58 -0
  137. rememberstack/spine/migrations/versions/p6_04_0013_knowledge_writer_ledger.py +46 -0
  138. rememberstack/spine/migrations/versions/p6_05_0014_knowledge_planner_runtime.py +217 -0
  139. rememberstack/spine/migrations/versions/p6_06_0015_authored_dispatch_runtime.py +38 -0
  140. rememberstack/spine/migrations/versions/p7_02_0016_operational_eval_suite.py +19 -0
  141. rememberstack/spine/migrations/versions/p7_05_0017_hard_forget.py +55 -0
  142. rememberstack/spine/observation_adjudication.py +778 -0
  143. rememberstack/spine/operations.py +298 -0
  144. rememberstack/spine/projection.py +662 -0
  145. rememberstack/spine/recipes.py +276 -0
  146. rememberstack/spine/resolver.py +763 -0
  147. rememberstack/spine/review.py +650 -0
  148. rememberstack/spine/settings.py +22 -0
  149. rememberstack/spine/supersession.py +510 -0
  150. rememberstack/spine/sync.py +128 -0
  151. rememberstack/spine/work_ledger.py +816 -0
  152. rememberstack/surfaces/__init__.py +110 -0
  153. rememberstack/surfaces/cli.py +447 -0
  154. rememberstack/surfaces/consumption_skill.py +87 -0
  155. rememberstack/surfaces/graph_queries.py +698 -0
  156. rememberstack/surfaces/http_api.py +377 -0
  157. rememberstack/surfaces/mcp.py +67 -0
  158. rememberstack/surfaces/query_engine.py +1591 -0
  159. rememberstack/surfaces/recipe_executor.py +185 -0
  160. rememberstack/surfaces/recipe_surface.py +219 -0
  161. rememberstack/surfaces/remote_mcp.py +133 -0
  162. rememberstack/surfaces/sdk.py +324 -0
  163. rememberstack/workers/__init__.py +155 -0
  164. rememberstack/workers/base.py +312 -0
  165. rememberstack/workers/e0.py +577 -0
  166. rememberstack/workers/e1.py +425 -0
  167. rememberstack/workers/e2.py +525 -0
  168. rememberstack/workers/e3.py +434 -0
  169. rememberstack/workers/forget.py +299 -0
  170. rememberstack/workers/knowledge_authored.py +146 -0
  171. rememberstack/workers/knowledge_driver.py +735 -0
  172. rememberstack/workers/knowledge_fact_sheet.py +123 -0
  173. rememberstack/workers/knowledge_planner.py +325 -0
  174. rememberstack/workers/knowledge_writer.py +393 -0
  175. rememberstack/workers/operations.py +42 -0
  176. rememberstack/workers/p1.py +234 -0
  177. rememberstack/workers/p2.py +513 -0
  178. rememberstack/workers/p2_analytics.py +276 -0
  179. rememberstack/workers/p3.py +673 -0
  180. rememberstack/workers/reconcile.py +485 -0
  181. rememberstack/workers/sync.py +168 -0
  182. rememberstack-0.1.0.dist-info/METADATA +213 -0
  183. rememberstack-0.1.0.dist-info/RECORD +186 -0
  184. rememberstack-0.1.0.dist-info/WHEEL +4 -0
  185. rememberstack-0.1.0.dist-info/entry_points.txt +2 -0
  186. rememberstack-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,200 @@
1
+ """Pure rank operators (retrieval §3: `fuse`, `rerank`): the D9 fusion math.
2
+
3
+ These are the fusion and reranking stages as *pure* functions — no spine, no
4
+ ports, no I/O — so the same code fuses an agent's ad-hoc channel set and a
5
+ recipe's fixed one, and every stage is inspectable rather than a black box.
6
+
7
+ **Reciprocal-rank fusion (RRF).** When several channels each return a ranked
8
+ list — semantic search, BM25, FTS — they cannot be compared by raw score:
9
+ a cosine distance and a BM25 score are different units, and two embedding
10
+ families are not on the same scale either. RRF sidesteps that by scoring an
11
+ item purely on its *ranks*: an item that placed 1st in one channel and 3rd in
12
+ another scores ``1/(k+1) + 1/(k+3)``. The constant ``k`` (≈60, a starting
13
+ default retained after the WP-5.6 relevance grid could not distinguish the
14
+ tested values) damps the top ranks so one channel cannot dominate on a single
15
+ high placement. Items no channel returned score zero and drop out. The result
16
+ is one order that rewards agreement across channels without ever comparing
17
+ incomparable scores.
18
+
19
+ **Reranking** is the same currency read differently: given a set of items each
20
+ carrying a named signal (graph distance to a focal entity, evidence count),
21
+ reorder by that signal. It is a stage, not a verdict — the caller sees the
22
+ signal value on every item and can chain another stage on top.
23
+ """
24
+
25
+ from collections.abc import Callable
26
+ from collections.abc import Sequence
27
+ from typing import Final
28
+ from uuid import UUID
29
+
30
+ from rememberstack.model import RankedItem
31
+
32
+ DEFAULT_RRF_K: Final = 60
33
+ """The conventional starting default for RRF damping (D9/S46).
34
+
35
+ WP-5.6 retained it because the small canary grid did not distinguish the
36
+ tested k values; it was not empirically selected over them.
37
+ """
38
+
39
+ DEFAULT_GRAPH_DISTANCE_WEIGHT: Final = 0.10
40
+ """Smallest tested nonzero proximity bonus on WP-5.6's best plateau."""
41
+
42
+ DEFAULT_EVIDENCE_COUNT_WEIGHT: Final = 0.10
43
+ """Smallest tested nonzero corroboration bonus on WP-5.6's best plateau."""
44
+
45
+
46
+ def reciprocal_rank_fusion(
47
+ *, rankings: Sequence[Sequence[UUID]], k: int = DEFAULT_RRF_K
48
+ ) -> tuple[RankedItem, ...]:
49
+ """Fuse several ranked id lists into one RRF-scored order (D9/S46).
50
+
51
+ Each inner sequence is one channel's ranking, best first. An item's
52
+ score is the sum over channels of ``1/(k + rank)`` (rank 1-based), so
53
+ agreement across channels wins without ever comparing raw scores. Only
54
+ an item's BEST rank within a channel counts — a channel that lists an id
55
+ twice contributes once, so one channel cannot forge cross-channel
56
+ agreement. Ties break on the id's own order for determinism. `k` must be
57
+ positive.
58
+ """
59
+ if k < 1:
60
+ raise ValueError("the RRF constant k must be at least 1")
61
+ scores: dict[UUID, float] = {}
62
+ contributions: dict[UUID, dict[str, float]] = {}
63
+ for channel, ranking in enumerate(rankings):
64
+ seen: set[UUID] = set()
65
+ for rank, item_id in enumerate(ranking, start=1):
66
+ if item_id in seen:
67
+ continue # a duplicate in one channel counts only at its best rank
68
+ seen.add(item_id)
69
+ increment = 1.0 / (k + rank)
70
+ scores[item_id] = scores.get(item_id, 0.0) + increment
71
+ contributions.setdefault(item_id, {})[f"channel_{channel}"] = increment
72
+ ordered = sorted(scores, key=lambda item_id: (-scores[item_id], item_id.bytes))
73
+ return tuple(
74
+ RankedItem(
75
+ item_id=item_id, score=scores[item_id], signals=contributions[item_id]
76
+ )
77
+ for item_id in ordered
78
+ )
79
+
80
+
81
+ def rerank_by_signal(
82
+ *, items: Sequence[RankedItem], signal: str, ascending: bool = False
83
+ ) -> tuple[RankedItem, ...]:
84
+ """Reorder items by one named signal each already carries (D9/S46/S48).
85
+
86
+ `graph_distance` reranks ascending (nearer the focal entity is more
87
+ relevant); `evidence_count` reranks descending (more corroboration
88
+ first). An item missing the signal sorts last in either direction
89
+ rather than raising — a partial signal is a weaker rerank, not an
90
+ error — and keeps its incoming `score` rather than being stamped with a
91
+ non-finite sentinel that would not survive JSON. An item that HAS the
92
+ signal takes it as its new `score`; `signals` is preserved throughout so
93
+ a later stage can read every contribution.
94
+ """
95
+ sentinel = float("inf") if ascending else float("-inf")
96
+
97
+ def key(item: RankedItem) -> tuple[float, bytes]:
98
+ value = item.signals.get(signal, sentinel)
99
+ return (value if ascending else -value, item.item_id.bytes)
100
+
101
+ return tuple(
102
+ item.model_copy(update={"score": item.signals[signal]})
103
+ if signal in item.signals
104
+ else item
105
+ for item in sorted(items, key=key)
106
+ )
107
+
108
+
109
+ def rerank_by_weighted_signals(
110
+ *,
111
+ items: Sequence[RankedItem],
112
+ graph_distance_weight: float = DEFAULT_GRAPH_DISTANCE_WEIGHT,
113
+ evidence_count_weight: float = DEFAULT_EVIDENCE_COUNT_WEIGHT,
114
+ ) -> tuple[RankedItem, ...]:
115
+ """Blend normalized RRF, proximity, and support without unit confusion.
116
+
117
+ The incoming RRF score remains the base signal. Graph distance becomes a
118
+ closeness score (nearer is larger), and evidence count becomes a normalized
119
+ corroboration score. Missing optional signals contribute zero. WP-5.6
120
+ exercised the two bonuses on a small canary grid; deterministic ids break
121
+ ties. Normalization is relative to this candidate set, so absolute scores
122
+ are not comparable across separate calls or pages.
123
+ """
124
+ if graph_distance_weight < 0 or evidence_count_weight < 0:
125
+ raise ValueError("rerank weights must be non-negative")
126
+ if not items:
127
+ return ()
128
+ base = _relative_to_best(values=tuple(item.score for item in items))
129
+ graph = _normalized_optional(
130
+ items=items, signal="graph_distance", higher_is_better=False
131
+ )
132
+ evidence = _normalized_optional(
133
+ items=items, signal="evidence_count", higher_is_better=True
134
+ )
135
+ weighted = tuple(
136
+ base[index]
137
+ + graph_distance_weight * graph[index]
138
+ + evidence_count_weight * evidence[index]
139
+ for index in range(len(items))
140
+ )
141
+ rescored = tuple(
142
+ item.model_copy(
143
+ update={
144
+ "score": weighted[index],
145
+ "signals": {
146
+ **item.signals,
147
+ "rrf_score": item.score,
148
+ "rrf_normalized": base[index],
149
+ "graph_proximity_normalized": graph[index],
150
+ "evidence_support_normalized": evidence[index],
151
+ "weighted_relevance": weighted[index],
152
+ },
153
+ }
154
+ )
155
+ for index, item in enumerate(items)
156
+ )
157
+ return tuple(sorted(rescored, key=lambda item: (-item.score, item.item_id.bytes)))
158
+
159
+
160
+ def _normalized_optional(
161
+ *, items: Sequence[RankedItem], signal: str, higher_is_better: bool
162
+ ) -> tuple[float, ...]:
163
+ """Normalize one optional signal, leaving missing values at zero."""
164
+ present = tuple(item.signals[signal] for item in items if signal in item.signals)
165
+ if not present:
166
+ return tuple(0.0 for _ in items)
167
+ normalized = _normalizer(values=present, higher_is_better=higher_is_better)
168
+ return tuple(
169
+ normalized(item.signals[signal]) if signal in item.signals else 0.0
170
+ for item in items
171
+ )
172
+
173
+
174
+ def _normalized(
175
+ *, values: tuple[float, ...], higher_is_better: bool
176
+ ) -> tuple[float, ...]:
177
+ """Map a complete numeric signal to [0, 1]."""
178
+ normalizer = _normalizer(values=values, higher_is_better=higher_is_better)
179
+ return tuple(normalizer(value) for value in values)
180
+
181
+
182
+ def _relative_to_best(*, values: tuple[float, ...]) -> tuple[float, ...]:
183
+ """Preserve close positive RRF differences instead of stretching them."""
184
+ high = max(values)
185
+ if high <= 0:
186
+ return _normalized(values=values, higher_is_better=True)
187
+ return tuple(max(value, 0.0) / high for value in values)
188
+
189
+
190
+ def _normalizer(
191
+ *, values: tuple[float, ...], higher_is_better: bool
192
+ ) -> Callable[[float], float]:
193
+ """Build a compact min-max normalizer for one candidate set."""
194
+ low = min(values)
195
+ high = max(values)
196
+ if high == low:
197
+ return lambda _value: 1.0
198
+ if higher_is_better:
199
+ return lambda value: (value - low) / (high - low)
200
+ return lambda value: (high - value) / (high - low)
@@ -0,0 +1,149 @@
1
+ """The recipe registration linter (D50/D41): grain semantics, mechanically.
2
+
3
+ A recipe declares two enums — `output_grain` and `answer_intent` — and a
4
+ `chain` of primitive ops. The database CHECK enforces the headline bar
5
+ (`current_facts` ⇒ `fact` grain); this linter enforces the *chain-level*
6
+ rules the DB cannot see, so a registration that would let a recipe lie about
7
+ what it returns is rejected before it ever reaches a surface:
8
+
9
+ - **`current_facts` may ride only validity-filtered fact primitives.** This
10
+ is the D41 bar in full: a recipe that answers "what holds now" must compose
11
+ lookups/aggregates that filter both temporal clocks — never a claims search,
12
+ which is evidence ("what a source *asserted*"), not fact.
13
+ - **The chain's terminal grain must match `output_grain`.** A recipe that
14
+ ends on a claims search cannot advertise `fact`; one that ends on a K-page
15
+ read cannot advertise `evidence`. The grain a caller reads is the grain the
16
+ last step actually produces.
17
+ - **Each intent implies a shape.** `assertion_history` is evidence-grain,
18
+ `change_feed` ends on the delta, `audit` ends on a decision trail. These
19
+ keep the MCP tool a caller sees honest about what it will get back.
20
+
21
+ `fuse` produces an **evidence-grade ordering**, not the confirmed records
22
+ themselves: its output is a ranking of candidate ids still to be hydrated
23
+ (matching the `fuse` primitive's own grain), so a recipe that ends on a fuse
24
+ is evidence, never fact. The op vocabulary the linter accepts is exactly the
25
+ set the executor can run — a chain never lints only to fail at execution.
26
+ """
27
+
28
+ from dataclasses import dataclass
29
+
30
+ from rememberstack.model import Grain
31
+ from rememberstack.model import Recipe
32
+ from rememberstack.model import RecipeAnswerIntent
33
+
34
+
35
+ class RecipeLintError(Exception):
36
+ """A recipe registration the linter rejected, with the reason stated."""
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class _OpSpec:
41
+ """What a chain op produces, for the mechanical grain checks."""
42
+
43
+ grain: Grain # the D49 grain this op's envelope carries
44
+ validity_filtered: bool # filters BOTH clocks to "now" (the current_facts bar)
45
+ min_inputs: int = 0 # prior steps this op must consume
46
+
47
+
48
+ # The op vocabulary a recipe chain may compose — exactly the set the executor
49
+ # implements, so a lint-clean chain always runs. `validity_filtered` is the
50
+ # strict current-instant test (both clocks): only the point-in-time lookups
51
+ # qualify. `aggregate` is NOT one of them — its forms span history (timeline)
52
+ # or count live-but-expired rows — so it can never sit in a `current_facts`
53
+ # recipe. `fuse` returns an evidence-grade ranking (candidates to hydrate).
54
+ _OPS: dict[str, _OpSpec] = {
55
+ "lookup_relations": _OpSpec(Grain.FACT, validity_filtered=True),
56
+ "lookup_observations": _OpSpec(Grain.FACT, validity_filtered=True),
57
+ "aggregate": _OpSpec(Grain.FACT, validity_filtered=False),
58
+ "search_claims": _OpSpec(Grain.EVIDENCE, validity_filtered=False),
59
+ "hydrate_relation": _OpSpec(Grain.COMPOSITE, validity_filtered=False),
60
+ "transcript": _OpSpec(Grain.COMPOSITE, validity_filtered=False),
61
+ "delta": _OpSpec(Grain.COMPOSITE, validity_filtered=False),
62
+ "pages_about": _OpSpec(Grain.COMPILED, validity_filtered=False),
63
+ "fuse": _OpSpec(Grain.EVIDENCE, validity_filtered=False, min_inputs=1),
64
+ }
65
+
66
+ KNOWN_OPS = frozenset(_OPS)
67
+ """The primitive ops a recipe chain may name — exactly the executor's set."""
68
+
69
+
70
+ def lint_recipe(recipe: Recipe) -> None:
71
+ """Validate a recipe against the D50/D41 grain rules, or raise.
72
+
73
+ Runs before every registration: a chain that would let a recipe
74
+ misreport its grain, or a `current_facts` recipe that reaches for
75
+ evidence, never becomes a row. Raises `RecipeLintError` naming the first
76
+ violation; returns None when the recipe is well-formed.
77
+ """
78
+ _check_ops_and_inputs(recipe)
79
+ terminal_grain = _OPS[recipe.chain[-1].op].grain
80
+ if terminal_grain != recipe.output_grain:
81
+ raise RecipeLintError(
82
+ f"recipe {recipe.name!r} declares output_grain"
83
+ f" {recipe.output_grain.value!r} but its chain ends on a"
84
+ f" {terminal_grain.value!r}-grain op"
85
+ )
86
+ _check_intent(recipe)
87
+
88
+
89
+ def _check_ops_and_inputs(recipe: Recipe) -> None:
90
+ """Every op is known, and every input references an earlier step."""
91
+ for index, step in enumerate(recipe.chain):
92
+ spec = _OPS.get(step.op)
93
+ if spec is None:
94
+ raise RecipeLintError(
95
+ f"recipe {recipe.name!r} step {index} names unknown op"
96
+ f" {step.op!r}; known ops: {', '.join(sorted(KNOWN_OPS))}"
97
+ )
98
+ if len(step.inputs) < spec.min_inputs:
99
+ raise RecipeLintError(
100
+ f"recipe {recipe.name!r} step {index} ({step.op}) needs at"
101
+ f" least {spec.min_inputs} input(s)"
102
+ )
103
+ for referenced in step.inputs:
104
+ if not 0 <= referenced < index:
105
+ raise RecipeLintError(
106
+ f"recipe {recipe.name!r} step {index} references step"
107
+ f" {referenced}, which is not an earlier step"
108
+ )
109
+
110
+
111
+ def _check_intent(recipe: Recipe) -> None:
112
+ """The answer_intent → chain-shape rules (the mechanical grain bar)."""
113
+ intent = recipe.answer_intent
114
+ if intent is RecipeAnswerIntent.CURRENT_FACTS:
115
+ if recipe.output_grain is not Grain.FACT:
116
+ raise RecipeLintError(
117
+ f"recipe {recipe.name!r} answers current_facts but is not"
118
+ " fact-grain (the D41 bar)"
119
+ )
120
+ for index, step in enumerate(recipe.chain):
121
+ spec = _OPS[step.op]
122
+ if not (spec.validity_filtered and spec.grain is Grain.FACT):
123
+ raise RecipeLintError(
124
+ f"recipe {recipe.name!r} answers current_facts but step"
125
+ f" {index} ({step.op}) is not a validity-filtered fact"
126
+ " primitive — 'what holds now' never rides evidence or a"
127
+ " history-spanning aggregate (D41)"
128
+ )
129
+ elif intent is RecipeAnswerIntent.ASSERTION_HISTORY:
130
+ if recipe.output_grain is not Grain.EVIDENCE:
131
+ raise RecipeLintError(
132
+ f"recipe {recipe.name!r} answers assertion_history but is not"
133
+ " evidence-grain — 'what sources asserted' is evidence (D41)"
134
+ )
135
+ elif intent is RecipeAnswerIntent.CHANGE_FEED:
136
+ if recipe.chain[-1].op != "delta":
137
+ raise RecipeLintError(
138
+ f"recipe {recipe.name!r} answers change_feed but does not end"
139
+ " on the delta primitive"
140
+ )
141
+ elif intent is RecipeAnswerIntent.AUDIT:
142
+ if recipe.chain[-1].op not in {"transcript", "hydrate_relation"}:
143
+ raise RecipeLintError(
144
+ f"recipe {recipe.name!r} answers audit but does not end on a"
145
+ " decision trail (transcript or hydrate_relation)"
146
+ )
147
+ # ORIENTATION is deliberately shape-permissive: a shaped overview may be a
148
+ # fact aggregate, a K page, or a bundle — its honesty is the grain match
149
+ # already checked, not a fixed op.
@@ -0,0 +1,209 @@
1
+ """The deterministic section snap (D57, e1 §3): LLM spans → block partition.
2
+
3
+ The structurer LLM proposes section boundaries as character spans, and being
4
+ an LLM it can propose anything — overlaps, gaps, reversed spans, offsets past
5
+ the end of the document, absurd nesting. Chunks are runs of whole blocks that
6
+ must never cross a section, so sections MUST be unions of whole blocks; this
7
+ module is the deterministic layer that makes that true for every possible
8
+ input ("LLM proposes, a deterministic layer disposes").
9
+
10
+ The algorithm is pure and total: proposed spans + the block grid in, a
11
+ well-formed section tree out, never an exception. Malformed input degrades to
12
+ a coarser but correct partition; the worst case is the synthetic root alone.
13
+ """
14
+
15
+ from collections.abc import Sequence
16
+ from typing import Final
17
+
18
+ from rememberstack.model import Block
19
+ from rememberstack.model import ProposedSection
20
+ from rememberstack.model import SnappedSection
21
+
22
+ SECTION_ROLES: Final = frozenset(
23
+ {
24
+ "body",
25
+ "abstract",
26
+ "introduction",
27
+ "results",
28
+ "methods",
29
+ "discussion",
30
+ "conclusion",
31
+ "references",
32
+ "appendix",
33
+ "table",
34
+ "figure_caption",
35
+ "nav",
36
+ "boilerplate",
37
+ "legal",
38
+ }
39
+ )
40
+ """The section_role enum (D39): anything else the LLM invents becomes body."""
41
+
42
+ _MAX_DEPTH: Final = 16
43
+ """Nesting deeper than this is flattened away — the blocks stay with the
44
+ deepest surviving ancestor. A guard against pathological LLM recursion, not a
45
+ semantic limit; real documents never approach it."""
46
+
47
+
48
+ def snap_sections(
49
+ *,
50
+ proposed: Sequence[ProposedSection],
51
+ blocks: Sequence[Block],
52
+ title: str | None,
53
+ markdown_chars: int,
54
+ ) -> tuple[SnappedSection, ...]:
55
+ """Normalize a proposed section tree onto the block grid (e1 §3).
56
+
57
+ The five steps, in order: (1) every proposed start snaps BACKWARD to the
58
+ start of the block containing it; (2) siblings sort by snapped start,
59
+ longer proposed span first, emission order last, and siblings sharing a
60
+ snapped start collapse into one — the longer span wins and adopts the
61
+ loser's children; (3) siblings tile forward — each ends where the next
62
+ begins, the last at its parent's end, so blocks before the first child
63
+ are the parent's direct content; (4) children clip to their parent's
64
+ range and empty sections are pruned; (5) the root always spans the whole
65
+ block sequence.
66
+
67
+ Returns depth-first document order, root first. The empty document gets
68
+ the lone root with the empty block range ``0..-1``.
69
+ """
70
+ root_title = title or ""
71
+ last_block = len(blocks) - 1
72
+ root = SnappedSection(
73
+ node_path="0",
74
+ parent_path=None,
75
+ title=root_title,
76
+ role="body",
77
+ block_start=0,
78
+ block_end=last_block,
79
+ char_start=0,
80
+ char_end=markdown_chars,
81
+ summary="",
82
+ ordinal=0,
83
+ )
84
+ if last_block < 0:
85
+ return (root,)
86
+ output: list[SnappedSection] = [root]
87
+ _snap_level(
88
+ nodes=proposed,
89
+ parent_path="0",
90
+ parent_start=0,
91
+ parent_end=last_block,
92
+ blocks=blocks,
93
+ depth=1,
94
+ output=output,
95
+ )
96
+ return tuple(output)
97
+
98
+
99
+ def _snap_level(
100
+ *,
101
+ nodes: Sequence[ProposedSection],
102
+ parent_path: str,
103
+ parent_start: int,
104
+ parent_end: int,
105
+ blocks: Sequence[Block],
106
+ depth: int,
107
+ output: list[SnappedSection],
108
+ ) -> None:
109
+ """Snap one sibling level into the parent's block range, then recurse."""
110
+ if depth >= _MAX_DEPTH:
111
+ return
112
+ candidates = _ordered_candidates(
113
+ nodes=nodes, parent_start=parent_start, parent_end=parent_end, blocks=blocks
114
+ )
115
+ for index, (start, node, adopted) in enumerate(candidates):
116
+ end = (
117
+ candidates[index + 1][0] - 1 # tile: end where the next begins
118
+ if index + 1 < len(candidates)
119
+ else parent_end
120
+ )
121
+ if end < start:
122
+ continue # empty after tiling/clipping: pruned
123
+ path = f"{parent_path}.{index}"
124
+ output.append(
125
+ SnappedSection(
126
+ node_path=path,
127
+ parent_path=parent_path,
128
+ title=node.title,
129
+ role=_sanitize_role(role=node.role),
130
+ block_start=start,
131
+ block_end=end,
132
+ char_start=blocks[start].char_start,
133
+ char_end=blocks[end].char_end,
134
+ summary=node.summary,
135
+ ordinal=len(output),
136
+ )
137
+ )
138
+ _snap_level(
139
+ nodes=(*node.children, *adopted),
140
+ parent_path=path,
141
+ parent_start=start,
142
+ parent_end=end,
143
+ blocks=blocks,
144
+ depth=depth + 1,
145
+ output=output,
146
+ )
147
+
148
+
149
+ def _ordered_candidates(
150
+ *,
151
+ nodes: Sequence[ProposedSection],
152
+ parent_start: int,
153
+ parent_end: int,
154
+ blocks: Sequence[Block],
155
+ ) -> list[tuple[int, ProposedSection, tuple[ProposedSection, ...]]]:
156
+ """Snap starts, order siblings, collapse same-start ties (steps 1–2).
157
+
158
+ Yields ``(snapped_start, winner, adopted_children)`` with strictly
159
+ increasing starts; a tie's loser is dropped and its children are adopted
160
+ by the winner so a duplicated heading cannot erase a subtree. Step 4's
161
+ clipping is applied here where it prunes whole nodes: a zero-length
162
+ proposal, and a proposal whose char span lies entirely outside its
163
+ parent's char range, are empty after clipping — pruned, never inflated
164
+ into a real section by the start clamp + tiling (Codex review).
165
+ """
166
+ parent_char_start = blocks[parent_start].char_start
167
+ parent_char_end = blocks[parent_end].char_end
168
+ snapped: list[tuple[int, int, int, ProposedSection]] = []
169
+ for emission_index, node in enumerate(nodes):
170
+ if node.char_end <= node.char_start:
171
+ continue # zero-length (or reversed): pruned
172
+ if node.char_end <= parent_char_start or node.char_start >= parent_char_end:
173
+ continue # no overlap with the parent: empty after clipping
174
+ start = _snap_start(char=node.char_start, blocks=blocks)
175
+ start = max(start, parent_start)
176
+ if start > parent_end:
177
+ continue # entirely outside the parent: pruned by clipping
178
+ proposed_length = node.char_end - node.char_start
179
+ snapped.append((start, -proposed_length, emission_index, node))
180
+ snapped.sort(key=lambda entry: entry[:3])
181
+ candidates: list[tuple[int, ProposedSection, tuple[ProposedSection, ...]]] = []
182
+ for start, _, _, node in snapped:
183
+ if candidates and candidates[-1][0] == start:
184
+ winner_start, winner, adopted = candidates[-1]
185
+ candidates[-1] = (winner_start, winner, (*adopted, *node.children))
186
+ continue
187
+ candidates.append((start, node, ()))
188
+ return candidates
189
+
190
+
191
+ def _snap_start(*, char: int, blocks: Sequence[Block]) -> int:
192
+ """Step 1: the ordinal of the block containing ``char``, snapping backward.
193
+
194
+ A char before the first block clamps to block 0; a char past the last
195
+ block's end clamps to the last block. Chars falling between blocks (in
196
+ inter-block whitespace) belong to the preceding block — backward snap.
197
+ """
198
+ if char <= blocks[0].char_start:
199
+ return 0
200
+ for ordinal in range(len(blocks) - 1, -1, -1):
201
+ if blocks[ordinal].char_start <= char:
202
+ return ordinal
203
+ return 0
204
+
205
+
206
+ def _sanitize_role(*, role: str) -> str:
207
+ """An invented role name degrades to body — the enum is the contract."""
208
+ normalized = role.strip().lower()
209
+ return normalized if normalized in SECTION_ROLES else "body"
@@ -0,0 +1,27 @@
1
+ """Storage-class routing for raw originals (D51 guardrail 3).
2
+
3
+ Which originals stay cheap to read and which go cold is a *policy* decision
4
+ about the corpus, not a storage-provider mechanic — so it lives here, in
5
+ pure logic, and both the ingest path (which routes at the write) and the
6
+ provider adapters (which apply the class) depend on it rather than on each
7
+ other.
8
+
9
+ Media a multimodal harness actually reads stays hot: for a video, an audio
10
+ file, or a photo *input*, the original IS the artifact — conversion yields
11
+ only a lossy transcript or description. Text and office originals are kept
12
+ for audit and re-conversion, so they go cold. Routing at the write is what
13
+ kills the grep-the-archive cost bug at the source rather than on the bill.
14
+ """
15
+
16
+ from typing import Final
17
+
18
+ HOT_MIME_PREFIXES: Final = ("video/", "audio/", "image/")
19
+ """Originals a harness reads directly — the bytes themselves are the value."""
20
+
21
+ HOT: Final = "hot"
22
+ COLD: Final = "cold"
23
+
24
+
25
+ def storage_class_for(*, mime: str) -> str:
26
+ """Route one original's storage class by mime (per-deployment policy)."""
27
+ return HOT if mime.startswith(HOT_MIME_PREFIXES) else COLD
@@ -0,0 +1,53 @@
1
+ """Evaluation package: the D22 harness and the golden suites."""
2
+
3
+ from rememberstack.eval.consumption import make_retrieval_evaluator
4
+ from rememberstack.eval.consumption import make_s58_evaluator
5
+ from rememberstack.eval.consumption import S58_CANARIES
6
+ from rememberstack.eval.consumption import seed_s58_canaries
7
+ from rememberstack.eval.contradiction import CONTRADICTION_PRECISION_FLOOR
8
+ from rememberstack.eval.contradiction import CONTRADICTION_RECALL_FLOOR
9
+ from rememberstack.eval.contradiction import run_contradiction_suite
10
+ from rememberstack.eval.contradiction import seed_contradiction_cases
11
+ from rememberstack.eval.harness import CaseEvaluator
12
+ from rememberstack.eval.harness import EvalHarness
13
+ from rememberstack.eval.lifecycle import flag_rate_by_extractor
14
+ from rememberstack.eval.lifecycle import register_lifecycle_evaluator
15
+ from rememberstack.eval.lifecycle import run_lifecycle_suite
16
+ from rememberstack.eval.operational_scale import OPERATIONAL_SCALE_VERSION
17
+ from rememberstack.eval.operational_scale import record_operational_scale_report
18
+ from rememberstack.eval.resolution import PRECISION_FLOOR
19
+ from rememberstack.eval.resolution import RECALL_FLOOR
20
+ from rememberstack.eval.resolution import run_resolution_suite
21
+ from rememberstack.eval.resolution import seed_synthetic_golden_pairs
22
+ from rememberstack.eval.retrieval_spikes import record_retrieval_spike_report
23
+ from rememberstack.eval.retrieval_spikes import RETRIEVAL_SPIKE_VERSION
24
+ from rememberstack.eval.skeleton import make_skeleton_evaluator
25
+ from rememberstack.eval.skeleton import seed_skeleton_canaries
26
+ from rememberstack.eval.skeleton import SKELETON_CANARIES
27
+
28
+ __all__ = (
29
+ "CONTRADICTION_PRECISION_FLOOR",
30
+ "CONTRADICTION_RECALL_FLOOR",
31
+ "CaseEvaluator",
32
+ "make_retrieval_evaluator",
33
+ "make_s58_evaluator",
34
+ "run_contradiction_suite",
35
+ "flag_rate_by_extractor",
36
+ "register_lifecycle_evaluator",
37
+ "run_lifecycle_suite",
38
+ "seed_contradiction_cases",
39
+ "seed_s58_canaries",
40
+ "S58_CANARIES",
41
+ "EvalHarness",
42
+ "OPERATIONAL_SCALE_VERSION",
43
+ "PRECISION_FLOOR",
44
+ "RECALL_FLOOR",
45
+ "record_retrieval_spike_report",
46
+ "record_operational_scale_report",
47
+ "RETRIEVAL_SPIKE_VERSION",
48
+ "run_resolution_suite",
49
+ "seed_synthetic_golden_pairs",
50
+ "SKELETON_CANARIES",
51
+ "make_skeleton_evaluator",
52
+ "seed_skeleton_canaries",
53
+ )