rememberstack 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. rememberstack/__init__.py +9 -0
  2. rememberstack/adapters/__init__.py +42 -0
  3. rememberstack/adapters/codex_writer.py +221 -0
  4. rememberstack/adapters/markitdown_converter.py +42 -0
  5. rememberstack/adapters/openrouter.py +136 -0
  6. rememberstack/adapters/selfhost/__init__.py +54 -0
  7. rememberstack/adapters/selfhost/forget.py +66 -0
  8. rememberstack/adapters/selfhost/git.py +374 -0
  9. rememberstack/adapters/selfhost/lance.py +328 -0
  10. rememberstack/adapters/selfhost/minio.py +279 -0
  11. rememberstack/adapters/selfhost/mounts.py +249 -0
  12. rememberstack/adapters/selfhost/object_store.py +130 -0
  13. rememberstack/adapters/selfhost/projection.py +80 -0
  14. rememberstack/adapters/selfhost/queue.py +137 -0
  15. rememberstack/adapters/selfhost/telemetry.py +45 -0
  16. rememberstack/adapters/selfhost/watcher.py +70 -0
  17. rememberstack/adapters/testing/__init__.py +15 -0
  18. rememberstack/adapters/testing/cost_meter.py +13 -0
  19. rememberstack/adapters/testing/model_provider.py +83 -0
  20. rememberstack/adapters/testing/queue.py +43 -0
  21. rememberstack/adapters/testing/telemetry.py +22 -0
  22. rememberstack/client.py +19 -0
  23. rememberstack/core/__init__.py +127 -0
  24. rememberstack/core/blockizer.py +189 -0
  25. rememberstack/core/chunker.py +216 -0
  26. rememberstack/core/consumption_skill.py +275 -0
  27. rememberstack/core/conversion.py +76 -0
  28. rememberstack/core/core_manifest.py +598 -0
  29. rememberstack/core/extension_packs.py +124 -0
  30. rememberstack/core/forget.py +17 -0
  31. rememberstack/core/knowledge_authored.py +276 -0
  32. rememberstack/core/knowledge_compile.py +215 -0
  33. rememberstack/core/knowledge_fact_sheet.py +210 -0
  34. rememberstack/core/knowledge_hashing.py +68 -0
  35. rememberstack/core/knowledge_planner.py +64 -0
  36. rememberstack/core/knowledge_writer.py +175 -0
  37. rememberstack/core/ranking.py +200 -0
  38. rememberstack/core/recipe_linter.py +149 -0
  39. rememberstack/core/section_snap.py +209 -0
  40. rememberstack/core/storage_routing.py +27 -0
  41. rememberstack/eval/__init__.py +53 -0
  42. rememberstack/eval/consumption.py +141 -0
  43. rememberstack/eval/contradiction.py +184 -0
  44. rememberstack/eval/harness.py +136 -0
  45. rememberstack/eval/lifecycle.py +400 -0
  46. rememberstack/eval/operational_scale.py +49 -0
  47. rememberstack/eval/resolution.py +255 -0
  48. rememberstack/eval/retrieval_spikes.py +50 -0
  49. rememberstack/eval/skeleton.py +231 -0
  50. rememberstack/llm/__init__.py +1 -0
  51. rememberstack/model/__init__.py +589 -0
  52. rememberstack/model/adjudication.py +100 -0
  53. rememberstack/model/auth.py +27 -0
  54. rememberstack/model/blocks.py +30 -0
  55. rememberstack/model/chunks.py +190 -0
  56. rememberstack/model/claims.py +162 -0
  57. rememberstack/model/client.py +98 -0
  58. rememberstack/model/clustering.py +54 -0
  59. rememberstack/model/component_version.py +124 -0
  60. rememberstack/model/consumption.py +88 -0
  61. rememberstack/model/conversion.py +31 -0
  62. rememberstack/model/deployment.py +53 -0
  63. rememberstack/model/documents.py +168 -0
  64. rememberstack/model/envelope.py +513 -0
  65. rememberstack/model/evaluation.py +72 -0
  66. rememberstack/model/forget.py +143 -0
  67. rememberstack/model/git.py +13 -0
  68. rememberstack/model/knowledge.py +840 -0
  69. rememberstack/model/knowledge_authored.py +325 -0
  70. rememberstack/model/knowledge_planner.py +431 -0
  71. rememberstack/model/lifecycle.py +42 -0
  72. rememberstack/model/model_provider.py +78 -0
  73. rememberstack/model/mounts.py +24 -0
  74. rememberstack/model/object_store.py +21 -0
  75. rememberstack/model/operational_scale.py +59 -0
  76. rememberstack/model/operations.py +153 -0
  77. rememberstack/model/processing.py +228 -0
  78. rememberstack/model/queue.py +73 -0
  79. rememberstack/model/recipes.py +83 -0
  80. rememberstack/model/relations.py +79 -0
  81. rememberstack/model/resolution.py +83 -0
  82. rememberstack/model/retrieval_spikes.py +62 -0
  83. rememberstack/model/sections.py +120 -0
  84. rememberstack/model/telemetry.py +30 -0
  85. rememberstack/ports/__init__.py +29 -0
  86. rememberstack/ports/auth.py +16 -0
  87. rememberstack/ports/connector.py +23 -0
  88. rememberstack/ports/cost_meter.py +17 -0
  89. rememberstack/ports/forget.py +20 -0
  90. rememberstack/ports/git.py +20 -0
  91. rememberstack/ports/model_provider.py +28 -0
  92. rememberstack/ports/mounts.py +16 -0
  93. rememberstack/ports/object_store.py +27 -0
  94. rememberstack/ports/p1_index.py +92 -0
  95. rememberstack/ports/purge.py +93 -0
  96. rememberstack/ports/queue.py +23 -0
  97. rememberstack/ports/telemetry.py +21 -0
  98. rememberstack/profiles/__init__.py +22 -0
  99. rememberstack/profiles/selfhost.py +324 -0
  100. rememberstack/profiles/selfhost_forget.py +158 -0
  101. rememberstack/profiles/selfhost_operations.py +95 -0
  102. rememberstack/py.typed +1 -0
  103. rememberstack/spine/__init__.py +93 -0
  104. rememberstack/spine/admission.py +26 -0
  105. rememberstack/spine/backfill.py +168 -0
  106. rememberstack/spine/catalog_contract.py +742 -0
  107. rememberstack/spine/chunk_catalog.py +237 -0
  108. rememberstack/spine/claim_catalog.py +298 -0
  109. rememberstack/spine/clustering.py +740 -0
  110. rememberstack/spine/component_versions.py +208 -0
  111. rememberstack/spine/consumption.py +81 -0
  112. rememberstack/spine/deployment_bootstrap.py +445 -0
  113. rememberstack/spine/document_catalog.py +621 -0
  114. rememberstack/spine/entity_registry.py +205 -0
  115. rememberstack/spine/extension_packs.py +220 -0
  116. rememberstack/spine/fact_catalog.py +571 -0
  117. rememberstack/spine/forget.py +1753 -0
  118. rememberstack/spine/knowledge.py +5467 -0
  119. rememberstack/spine/lifecycle.py +1071 -0
  120. rememberstack/spine/migrations/__init__.py +1 -0
  121. rememberstack/spine/migrations/_helpers.py +153 -0
  122. rememberstack/spine/migrations/env.py +58 -0
  123. rememberstack/spine/migrations/script.py.mako +27 -0
  124. rememberstack/spine/migrations/versions/__init__.py +1 -0
  125. rememberstack/spine/migrations/versions/p0_02_0001_extensions_enums.py +189 -0
  126. rememberstack/spine/migrations/versions/p0_02_0002_infrastructure_registries.py +321 -0
  127. rememberstack/spine/migrations/versions/p0_02_0003_entities_evaluation_e0_e1.py +631 -0
  128. rememberstack/spine/migrations/versions/p0_02_0004_claims_facts_evidence.py +411 -0
  129. rememberstack/spine/migrations/versions/p0_02_0005_projection_knowledge_retrieval.py +391 -0
  130. rememberstack/spine/migrations/versions/p0_02_0006_partitions_views.py +158 -0
  131. rememberstack/spine/migrations/versions/p2_06_0007_invalidated_outcome.py +26 -0
  132. rememberstack/spine/migrations/versions/p3_01_0008_document_version_target.py +58 -0
  133. rememberstack/spine/migrations/versions/p3_05_0009_reconcile_stage.py +27 -0
  134. rememberstack/spine/migrations/versions/p3_07_0010_lifecycle_eval_suite.py +25 -0
  135. rememberstack/spine/migrations/versions/p4_01_0011_survivor_view_rewrite.py +57 -0
  136. rememberstack/spine/migrations/versions/p6_02_0012_knowledge_compile_recovery.py +58 -0
  137. rememberstack/spine/migrations/versions/p6_04_0013_knowledge_writer_ledger.py +46 -0
  138. rememberstack/spine/migrations/versions/p6_05_0014_knowledge_planner_runtime.py +217 -0
  139. rememberstack/spine/migrations/versions/p6_06_0015_authored_dispatch_runtime.py +38 -0
  140. rememberstack/spine/migrations/versions/p7_02_0016_operational_eval_suite.py +19 -0
  141. rememberstack/spine/migrations/versions/p7_05_0017_hard_forget.py +55 -0
  142. rememberstack/spine/observation_adjudication.py +778 -0
  143. rememberstack/spine/operations.py +298 -0
  144. rememberstack/spine/projection.py +662 -0
  145. rememberstack/spine/recipes.py +276 -0
  146. rememberstack/spine/resolver.py +763 -0
  147. rememberstack/spine/review.py +650 -0
  148. rememberstack/spine/settings.py +22 -0
  149. rememberstack/spine/supersession.py +510 -0
  150. rememberstack/spine/sync.py +128 -0
  151. rememberstack/spine/work_ledger.py +816 -0
  152. rememberstack/surfaces/__init__.py +110 -0
  153. rememberstack/surfaces/cli.py +447 -0
  154. rememberstack/surfaces/consumption_skill.py +87 -0
  155. rememberstack/surfaces/graph_queries.py +698 -0
  156. rememberstack/surfaces/http_api.py +377 -0
  157. rememberstack/surfaces/mcp.py +67 -0
  158. rememberstack/surfaces/query_engine.py +1591 -0
  159. rememberstack/surfaces/recipe_executor.py +185 -0
  160. rememberstack/surfaces/recipe_surface.py +219 -0
  161. rememberstack/surfaces/remote_mcp.py +133 -0
  162. rememberstack/surfaces/sdk.py +324 -0
  163. rememberstack/workers/__init__.py +155 -0
  164. rememberstack/workers/base.py +312 -0
  165. rememberstack/workers/e0.py +577 -0
  166. rememberstack/workers/e1.py +425 -0
  167. rememberstack/workers/e2.py +525 -0
  168. rememberstack/workers/e3.py +434 -0
  169. rememberstack/workers/forget.py +299 -0
  170. rememberstack/workers/knowledge_authored.py +146 -0
  171. rememberstack/workers/knowledge_driver.py +735 -0
  172. rememberstack/workers/knowledge_fact_sheet.py +123 -0
  173. rememberstack/workers/knowledge_planner.py +325 -0
  174. rememberstack/workers/knowledge_writer.py +393 -0
  175. rememberstack/workers/operations.py +42 -0
  176. rememberstack/workers/p1.py +234 -0
  177. rememberstack/workers/p2.py +513 -0
  178. rememberstack/workers/p2_analytics.py +276 -0
  179. rememberstack/workers/p3.py +673 -0
  180. rememberstack/workers/reconcile.py +485 -0
  181. rememberstack/workers/sync.py +168 -0
  182. rememberstack-0.1.0.dist-info/METADATA +213 -0
  183. rememberstack-0.1.0.dist-info/RECORD +186 -0
  184. rememberstack-0.1.0.dist-info/WHEEL +4 -0
  185. rememberstack-0.1.0.dist-info/entry_points.txt +2 -0
  186. rememberstack-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,485 @@
1
+ """Reconciliation, finalization, and deletion grains (lifecycle §3–5, §8).
2
+
3
+ One flow for both lifecycle problems: when a version's chain completes, the
4
+ reconcile stage diffs the lineage's testimony, transitions currency, recounts
5
+ the touched facts, applies the §4 zero-support policy, and emits the
6
+ fact-level `evidence_changed` delta. The policy's two branches never mix:
7
+
8
+ - **the source acted** (a living edit removed content, a deletion) → close
9
+ per shape, recorded and reversible, no flag;
10
+ - **only our transcription changed** (an extractor bump did not re-derive a
11
+ claim from the unchanged file) → flag `support_withdrawn` for review; this
12
+ is the flag's only trigger.
13
+
14
+ Watched lineages defer source-acted closure to their sync cycle's
15
+ FINALIZATION (the retract-timing barrier): an intra-cycle move resolves as a
16
+ support swap, never retract-then-reassert. The `CycleFinalizer` runs that
17
+ job; `DeletionService` is the operator's grain (§8) through the same
18
+ cascade.
19
+ """
20
+
21
+ from uuid import NAMESPACE_URL
22
+ from uuid import UUID
23
+ from uuid import uuid5
24
+
25
+ from rememberstack.core import chunker_version as chunker_version_of
26
+ from rememberstack.core import ChunkerParams
27
+ from rememberstack.model import ClaimedWork
28
+ from rememberstack.model import CurrencyTransition
29
+ from rememberstack.model import NonRetryableHandlerError
30
+ from rememberstack.model import ReconciliationDelta
31
+ from rememberstack.ports.cost_meter import CostMeterPort
32
+ from rememberstack.spine.lifecycle import LifecycleCatalog
33
+ from rememberstack.spine.review import ReviewQueue
34
+ from rememberstack.workers.base import HandlerOutcome
35
+ from rememberstack.workers.e1 import E2_EXTRACTOR_VERSION
36
+
37
+ RECONCILE_VERSION = "reconcile-2026.07"
38
+ """The reconcile stage's component version (D12 idempotency key member)."""
39
+
40
+
41
+ class ReconcileHandler:
42
+ """The reconcile stage: one completed version's basis change, settled."""
43
+
44
+ def __init__(
45
+ self,
46
+ *,
47
+ catalog: LifecycleCatalog,
48
+ review_queue: ReviewQueue,
49
+ extractor_version: str = E2_EXTRACTOR_VERSION,
50
+ chunker_version: str | None = None,
51
+ ) -> None:
52
+ """Bind the handler to the lifecycle catalog and the review queue.
53
+
54
+ ``chunker_version`` names the packing generation of the completing
55
+ basis (the same parameters the composing profile gave the chunk
56
+ stage); it defaults to the default parameters' generation.
57
+ """
58
+ self._catalog = catalog
59
+ self._review_queue = review_queue
60
+ self._extractor_version = extractor_version
61
+ self._chunker_version = chunker_version or chunker_version_of(
62
+ params=ChunkerParams()
63
+ )
64
+
65
+ def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome:
66
+ """Diff → transition → recount → policy → emit, idempotently.
67
+
68
+ The work row's processing_id is the run's `reconciliation_id`: a
69
+ retried attempt re-emits every ledger row, closure, flag, and
70
+ trigger as a no-op.
71
+ """
72
+ del meter
73
+ version_id = _payload_uuid(work=work, field="version_id")
74
+ representation_id = _payload_uuid(work=work, field="representation_id")
75
+ context = self._catalog.reconciliation_context(version_id=version_id)
76
+ deployment_id = work.deployment_id
77
+ reconciliation_id = work.processing_id
78
+
79
+ source_acted: tuple[CurrencyTransition, ...] = ()
80
+ if (
81
+ context["versioning_mode"] == "living"
82
+ and context["current_version_id"] is not None
83
+ ):
84
+ source_acted = self._catalog.stale_for_supersession(
85
+ deployment_id=deployment_id,
86
+ doc_id=context["doc_id"], # type: ignore[arg-type]
87
+ current_version_id=context["current_version_id"], # type: ignore[arg-type]
88
+ )
89
+ transcription = self._catalog.stale_for_reextraction(
90
+ version_id=version_id,
91
+ representation_id=representation_id,
92
+ chunker_version=self._chunker_version,
93
+ extractor_version=self._extractor_version,
94
+ )
95
+ transitions = (*source_acted, *transcription)
96
+ applied = self._catalog.apply_transitions(
97
+ deployment_id=deployment_id,
98
+ reconciliation_id=reconciliation_id,
99
+ transitions=transitions,
100
+ )
101
+ # retry recovery (Codex review): a crash between the currency
102
+ # transaction and the steps below must not orphan the run — union
103
+ # what the ledger already holds under this reconciliation_id, since
104
+ # a retry recomputes an empty stale set (the cache already flipped)
105
+ recorded = self._catalog.recorded_transitions(
106
+ reconciliation_id=reconciliation_id
107
+ )
108
+ seen = {(t.claim_id, t.reason, t.became_current) for t in transitions}
109
+ for prior in recorded:
110
+ key = (prior.claim_id, prior.reason, prior.became_current)
111
+ if key not in seen:
112
+ seen.add(key)
113
+ transitions = (*transitions, *(prior,))
114
+ if prior.reason == "reextracted":
115
+ transcription = (*transcription, prior)
116
+ else:
117
+ source_acted = (*source_acted, prior)
118
+
119
+ claim_ids = tuple({transition.claim_id for transition in transitions})
120
+ relation_ids = self._catalog.affected_relation_ids(claim_ids=claim_ids)
121
+ observation_ids = self._catalog.affected_observation_ids(claim_ids=claim_ids)
122
+ changed_relations, changed_observations = self._catalog.recount(
123
+ relation_ids=relation_ids, observation_ids=observation_ids
124
+ )
125
+
126
+ zero_relations = self._catalog.open_zero_support_relations(
127
+ relation_ids=relation_ids
128
+ )
129
+ zero_observations = self._catalog.open_zero_support_observations(
130
+ observation_ids=observation_ids
131
+ )
132
+ source_claims = tuple({t.claim_id for t in source_acted})
133
+ source_relations = set(
134
+ self._catalog.affected_relation_ids(claim_ids=source_claims)
135
+ )
136
+ source_observations = set(
137
+ self._catalog.affected_observation_ids(claim_ids=source_claims)
138
+ )
139
+
140
+ closed_relations: tuple[UUID, ...] = ()
141
+ closed_observations: tuple[UUID, ...] = ()
142
+ if context["sync_cycle_id"] is None:
143
+ # not cycle-stamped (uploads, direct API ingest): the source
144
+ # acted and there is no move-vs-retract ambiguity — close now
145
+ closed_relations = self._catalog.close_relations(
146
+ deployment_id=deployment_id,
147
+ relation_ids=tuple(
148
+ fact for fact in zero_relations if fact in source_relations
149
+ ),
150
+ boundary=context["current_source_modified_at"],
151
+ reconciliation_id=reconciliation_id,
152
+ )
153
+ closed_observations = self._catalog.close_observations(
154
+ deployment_id=deployment_id,
155
+ observation_ids=tuple(
156
+ fact for fact in zero_observations if fact in source_observations
157
+ ),
158
+ reconciliation_id=reconciliation_id,
159
+ )
160
+ # else: closure waits for the cycle-finalization barrier (§5) —
161
+ # an intra-cycle move must land as a support swap, never a retract
162
+
163
+ flags = self._flag_transcription_only(
164
+ deployment_id=deployment_id,
165
+ transcription=transcription,
166
+ zero_relations=tuple(
167
+ fact for fact in zero_relations if fact not in source_relations
168
+ ),
169
+ zero_observations=tuple(
170
+ fact for fact in zero_observations if fact not in source_observations
171
+ ),
172
+ )
173
+
174
+ self._catalog.emit_evidence_changed(
175
+ deployment_id=deployment_id,
176
+ delta=ReconciliationDelta(
177
+ reconciliation_id=reconciliation_id,
178
+ transitions=applied,
179
+ # the stale-storm guard: only facts whose STATE moved — a
180
+ # re-extraction that changes no fact state stales nothing
181
+ recounted_relations=changed_relations,
182
+ recounted_observations=changed_observations,
183
+ relations_closed=closed_relations,
184
+ observations_closed=closed_observations,
185
+ flags_raised=flags,
186
+ ),
187
+ )
188
+ return HandlerOutcome()
189
+
190
+ def _flag_transcription_only(
191
+ self,
192
+ *,
193
+ deployment_id: UUID,
194
+ transcription: tuple[CurrencyTransition, ...],
195
+ zero_relations: tuple[UUID, ...],
196
+ zero_observations: tuple[UUID, ...],
197
+ ) -> tuple[UUID, ...]:
198
+ """§4's second branch: transcription-only zero support → one flag each.
199
+
200
+ The event carries no information about the world (the file still
201
+ says what it said), so no mechanical verdict is derivable — a
202
+ reviewer decides. Idempotent: an already-open flag is never stacked.
203
+ """
204
+ flagged: list[UUID] = []
205
+ for fact_kind, fact_ids in (
206
+ ("relation", zero_relations),
207
+ ("observation", zero_observations),
208
+ ):
209
+ for fact_id in fact_ids:
210
+ if self._review_queue.has_open_support_withdrawn(fact_id=fact_id):
211
+ continue
212
+ withdrawn = self._withdrawn_claim(
213
+ fact_kind=fact_kind, fact_id=fact_id, transcription=transcription
214
+ )
215
+ if withdrawn is None:
216
+ continue
217
+ self._review_queue.flag_support_withdrawn(
218
+ deployment_id=deployment_id,
219
+ fact_kind=fact_kind,
220
+ fact_id=fact_id,
221
+ claim_id=withdrawn.claim_id,
222
+ diff={
223
+ "reason": "reextracted",
224
+ "from_extractor_version": withdrawn.from_extractor_version,
225
+ "to_extractor_version": self._extractor_version,
226
+ # the full superseding basis, for exact attribution
227
+ # when a non-extractor coordinate caused the bump
228
+ "to_chunker_version": self._chunker_version,
229
+ },
230
+ )
231
+ flagged.append(fact_id)
232
+ return tuple(flagged)
233
+
234
+ def _withdrawn_claim(
235
+ self,
236
+ *,
237
+ fact_kind: str,
238
+ fact_id: UUID,
239
+ transcription: tuple[CurrencyTransition, ...],
240
+ ) -> CurrencyTransition | None:
241
+ """The transitioned claim whose withdrawal starved this fact."""
242
+ for transition in transcription:
243
+ affected = (
244
+ self._catalog.affected_relation_ids(claim_ids=(transition.claim_id,))
245
+ if fact_kind == "relation"
246
+ else self._catalog.affected_observation_ids(
247
+ claim_ids=(transition.claim_id,)
248
+ )
249
+ )
250
+ if fact_id in affected:
251
+ return transition
252
+ return None
253
+
254
+
255
+ class CycleFinalizer:
256
+ """The retract-timing barrier's second half: per-cycle retraction (§5).
257
+
258
+ Runs after every lineage a cycle observed has finished its chain:
259
+ evaluates source-acted zero-support closure for observed lineages and
260
+ runs the deletion cascade for lineages whose source deletion the cycle
261
+ recorded. Lineages still extracting defer the whole cycle — the
262
+ recorded grace, visible as (completed_at set, finalized_at null).
263
+ """
264
+
265
+ def __init__(self, *, catalog: LifecycleCatalog) -> None:
266
+ """Bind the finalizer to the lifecycle catalog."""
267
+ self._catalog = catalog
268
+
269
+ def finalize_ready(self, *, deployment_id: UUID) -> tuple[UUID, ...]:
270
+ """Finalize every ready cycle; returns the cycles this call won.
271
+
272
+ The claim is atomic and FIRST (two finalizer instances never both
273
+ evaluate one cycle); every cascade re-derives from current state
274
+ under a derived, stable reconciliation id, so a crash mid-cycle
275
+ leaves a brief, visible, self-healing gap rather than duplicates.
276
+ Source-tombstoned lineages are swept deployment-wide on every pass
277
+ for the same reason. A LOSSY cycle (per-item failures) skips
278
+ absence-based closure — its observation set is incomplete; the next
279
+ healthy cycle of the same source covers it.
280
+ """
281
+ finalized: list[UUID] = []
282
+ for cycle_id, failed_items in self._catalog.cycles_ready_to_finalize(
283
+ deployment_id=deployment_id
284
+ ):
285
+ if not self._catalog.claim_finalization(cycle_id=cycle_id):
286
+ continue # another finalizer won this cycle
287
+ if failed_items == 0:
288
+ for doc_id in self._catalog.cycle_lineages(cycle_id=cycle_id):
289
+ self._close_lineage_zero_support(
290
+ deployment_id=deployment_id, cycle_id=cycle_id, doc_id=doc_id
291
+ )
292
+ finalized.append(cycle_id)
293
+ for doc_id in self._catalog.tombstoned_lineages_needing_cascade(
294
+ deployment_id=deployment_id
295
+ ):
296
+ cascade_lineage_removal(
297
+ catalog=self._catalog,
298
+ deployment_id=deployment_id,
299
+ doc_id=doc_id,
300
+ reconciliation_id=_derived_run_id(
301
+ kind="finalize-delete", doc_id=doc_id
302
+ ),
303
+ )
304
+ return tuple(finalized)
305
+
306
+ def _close_lineage_zero_support(
307
+ self, *, deployment_id: UUID, cycle_id: UUID, doc_id: UUID
308
+ ) -> None:
309
+ """§4 source-acted closure for one observed lineage, cycle-scoped."""
310
+ claim_ids = self._catalog.lineage_claim_ids(
311
+ deployment_id=deployment_id, doc_id=doc_id
312
+ )
313
+ relation_ids = self._catalog.affected_relation_ids(claim_ids=claim_ids)
314
+ observation_ids = self._catalog.affected_observation_ids(claim_ids=claim_ids)
315
+ reconciliation_id = _derived_run_id(
316
+ kind="finalize", cycle_id=cycle_id, doc_id=doc_id
317
+ )
318
+ closed_relations = self._catalog.close_relations(
319
+ deployment_id=deployment_id,
320
+ relation_ids=self._catalog.open_zero_support_relations(
321
+ relation_ids=relation_ids
322
+ ),
323
+ boundary=self._catalog.closure_boundary(doc_id=doc_id),
324
+ reconciliation_id=reconciliation_id,
325
+ )
326
+ closed_observations = self._catalog.close_observations(
327
+ deployment_id=deployment_id,
328
+ observation_ids=self._catalog.open_zero_support_observations(
329
+ observation_ids=observation_ids
330
+ ),
331
+ reconciliation_id=reconciliation_id,
332
+ )
333
+ self._catalog.emit_evidence_changed(
334
+ deployment_id=deployment_id,
335
+ delta=ReconciliationDelta(
336
+ reconciliation_id=reconciliation_id,
337
+ relations_closed=closed_relations,
338
+ observations_closed=closed_observations,
339
+ ),
340
+ )
341
+
342
+
343
+ class DeletionService:
344
+ """The §8 deletion grains: version, lineage — one uniform cascade.
345
+
346
+ Deleting removes the document's contribution: currency ends, counts
347
+ recompute, solely-supported facts close (recorded, reversible, no
348
+ flag). Claims are retained as history — forgotten ≠ deleted; only
349
+ hard-forget (§13) scrubs content.
350
+ """
351
+
352
+ def __init__(self, *, catalog: LifecycleCatalog) -> None:
353
+ """Bind the service to the lifecycle catalog."""
354
+ self._catalog = catalog
355
+
356
+ def delete_version(self, *, version_id: UUID) -> ReconciliationDelta:
357
+ """End one version's testimony; the lineage continues (§8)."""
358
+ info = self._catalog.delete_version(version_id=version_id)
359
+ deployment_id: UUID = info["deployment_id"] # type: ignore[assignment]
360
+ doc_id: UUID = info["doc_id"] # type: ignore[assignment]
361
+ context = self._catalog.reconciliation_context(version_id=version_id)
362
+ # ONLY the deleted version's exclusive testimony ends — a snapshot
363
+ # lineage's other versions keep their currency (Codex review)
364
+ transitions: tuple[CurrencyTransition, ...] = (
365
+ self._catalog.stale_for_version_deletion(
366
+ deployment_id=deployment_id, version_id=version_id
367
+ )
368
+ )
369
+ if context["current_version_id"] is not None:
370
+ # "the lineage continues": the repointed predecessor's testimony
371
+ # is the current basis again — its claims regain currency
372
+ transitions = (
373
+ *transitions,
374
+ *self._catalog.regained_by_current_version(
375
+ deployment_id=deployment_id,
376
+ doc_id=doc_id,
377
+ current_version_id=context["current_version_id"], # type: ignore[arg-type]
378
+ ),
379
+ )
380
+ return _cascade(
381
+ catalog=self._catalog,
382
+ deployment_id=deployment_id,
383
+ transitions=transitions,
384
+ reconciliation_id=_derived_run_id(kind="delete-version", id_=version_id),
385
+ boundary=self._catalog.closure_boundary(doc_id=doc_id),
386
+ )
387
+
388
+ def delete_lineage(
389
+ self, *, deployment_id: UUID, doc_id: UUID
390
+ ) -> ReconciliationDelta:
391
+ """Remove a lineage's whole contribution (operator grain, §8)."""
392
+ self._catalog.delete_lineage(doc_id=doc_id)
393
+ return cascade_lineage_removal(
394
+ catalog=self._catalog,
395
+ deployment_id=deployment_id,
396
+ doc_id=doc_id,
397
+ reconciliation_id=_derived_run_id(kind="delete-lineage", id_=doc_id),
398
+ )
399
+
400
+
401
+ def cascade_lineage_removal(
402
+ *,
403
+ catalog: LifecycleCatalog,
404
+ deployment_id: UUID,
405
+ doc_id: UUID,
406
+ reconciliation_id: UUID,
407
+ ) -> ReconciliationDelta:
408
+ """The uniform lineage-removal cascade (§8): currency → recount → close."""
409
+ transitions = catalog.stale_for_deletion(deployment_id=deployment_id, doc_id=doc_id)
410
+ recorded = catalog.recorded_transitions(reconciliation_id=reconciliation_id)
411
+ seen = {(item.claim_id, item.reason, item.became_current) for item in transitions}
412
+ transitions = (
413
+ *transitions,
414
+ *(
415
+ item
416
+ for item in recorded
417
+ if (item.claim_id, item.reason, item.became_current) not in seen
418
+ ),
419
+ )
420
+ return _cascade(
421
+ catalog=catalog,
422
+ deployment_id=deployment_id,
423
+ transitions=transitions,
424
+ reconciliation_id=reconciliation_id,
425
+ boundary=None,
426
+ )
427
+
428
+
429
+ def _cascade(
430
+ *,
431
+ catalog: LifecycleCatalog,
432
+ deployment_id: UUID,
433
+ transitions: tuple[CurrencyTransition, ...],
434
+ reconciliation_id: UUID,
435
+ boundary: object,
436
+ ) -> ReconciliationDelta:
437
+ """Apply one source-acted basis change end to end, idempotently."""
438
+ applied = catalog.apply_transitions(
439
+ deployment_id=deployment_id,
440
+ reconciliation_id=reconciliation_id,
441
+ transitions=transitions,
442
+ )
443
+ claim_ids = tuple({transition.claim_id for transition in transitions})
444
+ relation_ids = catalog.affected_relation_ids(claim_ids=claim_ids)
445
+ observation_ids = catalog.affected_observation_ids(claim_ids=claim_ids)
446
+ catalog.recount(relation_ids=relation_ids, observation_ids=observation_ids)
447
+ closed_relations = catalog.close_relations(
448
+ deployment_id=deployment_id,
449
+ relation_ids=catalog.open_zero_support_relations(relation_ids=relation_ids),
450
+ boundary=boundary,
451
+ reconciliation_id=reconciliation_id,
452
+ )
453
+ closed_observations = catalog.close_observations(
454
+ deployment_id=deployment_id,
455
+ observation_ids=catalog.open_zero_support_observations(
456
+ observation_ids=observation_ids
457
+ ),
458
+ reconciliation_id=reconciliation_id,
459
+ )
460
+ delta = ReconciliationDelta(
461
+ reconciliation_id=reconciliation_id,
462
+ transitions=applied,
463
+ recounted_relations=relation_ids,
464
+ recounted_observations=observation_ids,
465
+ relations_closed=closed_relations,
466
+ observations_closed=closed_observations,
467
+ )
468
+ catalog.emit_evidence_changed(deployment_id=deployment_id, delta=delta)
469
+ return delta
470
+
471
+
472
+ def _derived_run_id(*, kind: str, **parts: object) -> UUID:
473
+ """A stable reconciliation id for non-queued runs (retry-idempotent)."""
474
+ suffix = ":".join(str(value) for value in parts.values())
475
+ return uuid5(NAMESPACE_URL, f"rememberstack:{kind}:{suffix}")
476
+
477
+
478
+ def _payload_uuid(*, work: ClaimedWork, field: str) -> UUID:
479
+ """Read a required UUID from the claimed payload; absence is non-retryable."""
480
+ value = (work.payload or {}).get(field)
481
+ if not isinstance(value, str):
482
+ raise NonRetryableHandlerError(
483
+ f"stage {work.stage} work {work.processing_id} carries no {field!r} payload"
484
+ )
485
+ return UUID(value)
@@ -0,0 +1,168 @@
1
+ """The watch loop (lifecycle §2, D55): one recorded poll cycle per pass.
2
+
3
+ The efficiency ladder's cheapest exits live here: unchanged revision → no
4
+ fetch; a file being actively edited coalesces to one ingested version per
5
+ stability window (the debounce discipline); identical bytes are the
6
+ content-hash no-op downstream. Deletion observations tombstone the lineage
7
+ loudly; the cascade is the delete worker's job.
8
+ """
9
+
10
+ from collections.abc import Mapping
11
+ from datetime import datetime
12
+ from datetime import timedelta
13
+ from datetime import UTC
14
+ from typing import Final
15
+ from uuid import UUID
16
+
17
+ from pydantic import Field
18
+ from pydantic_settings import BaseSettings
19
+ from pydantic_settings import SettingsConfigDict
20
+
21
+ from rememberstack.model import DocumentUpload
22
+ from rememberstack.model import ProcessingLane
23
+ from rememberstack.model import SourceItem
24
+ from rememberstack.model import SyncCycleSummary
25
+ from rememberstack.ports.connector import WatchedSourcePort
26
+ from rememberstack.spine.sync import SyncCatalog
27
+ from rememberstack.workers.e0 import UploadIngestor
28
+
29
+ DEFAULT_VERSIONING_MODE_BY_KIND: Final = {
30
+ # spike 6's starting heuristic: edit-in-place leans living;
31
+ # replace-whole-file and rolling logs lean archival (snapshot)
32
+ "watched_directory": "living",
33
+ "google_drive": "living",
34
+ "upload": "snapshot",
35
+ }
36
+
37
+
38
+ class SyncSettings(BaseSettings):
39
+ """The watch loop's knobs (starting points, D22)."""
40
+
41
+ model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_SYNC_")
42
+
43
+ debounce_quiet_seconds: float = Field(default=120.0, ge=0.0)
44
+ lane: ProcessingLane = ProcessingLane.STEADY
45
+
46
+
47
+ class SyncCycleRunner:
48
+ """Run one recorded poll cycle for one (deployment, source kind)."""
49
+
50
+ def __init__(
51
+ self, *, catalog: SyncCatalog, ingestor: UploadIngestor, settings: SyncSettings
52
+ ) -> None:
53
+ """Bind the runner to the cycle catalog, ingest path, and settings."""
54
+ self._catalog = catalog
55
+ self._ingestor = ingestor
56
+ self._settings = settings
57
+
58
+ def run_cycle(
59
+ self, *, deployment_id: UUID, source_kind: str, source: WatchedSourcePort
60
+ ) -> SyncCycleSummary:
61
+ """One poll pass: observe, debounce, ingest, tombstone, record.
62
+
63
+ The cycle row always reaches ``completed_at`` — one bad item (or a
64
+ failed poll) must not strand a cycle that already stamped versions,
65
+ because an eternally open cycle can never pass reconciliation's
66
+ finalization barrier. Lost items are counted on the row instead.
67
+ """
68
+ known = self._catalog.known_revisions(
69
+ deployment_id=deployment_id, source_kind=source_kind
70
+ )
71
+ cycle_id = self._catalog.open_cycle(
72
+ deployment_id=deployment_id, source_kind=source_kind
73
+ )
74
+ items: tuple[SourceItem, ...] = ()
75
+ ingested: list[UUID] = []
76
+ deletions: list[UUID] = []
77
+ unchanged = 0
78
+ debounced = 0
79
+ failed = 0
80
+ now = datetime.now(tz=UTC)
81
+ quiet = timedelta(seconds=self._settings.debounce_quiet_seconds)
82
+ versioning_mode = DEFAULT_VERSIONING_MODE_BY_KIND.get(source_kind, "snapshot")
83
+ try:
84
+ items = source.poll(known=known)
85
+ for item in items:
86
+ try:
87
+ outcome = self._observe_item(
88
+ deployment_id=deployment_id,
89
+ source_kind=source_kind,
90
+ cycle_id=cycle_id,
91
+ source=source,
92
+ item=item,
93
+ known=known,
94
+ now=now,
95
+ quiet=quiet,
96
+ versioning_mode=versioning_mode,
97
+ )
98
+ except Exception: # noqa: BLE001 — the pass must stay recorded
99
+ failed += 1
100
+ continue
101
+ kind, ref = outcome
102
+ if kind == "ingested" and ref is not None:
103
+ ingested.append(ref)
104
+ elif kind == "deleted" and ref is not None:
105
+ deletions.append(ref)
106
+ elif kind == "unchanged":
107
+ unchanged += 1
108
+ elif kind == "debounced":
109
+ debounced += 1
110
+ finally:
111
+ self._catalog.complete_cycle(
112
+ cycle_id=cycle_id, observed=len(items), failed=failed
113
+ )
114
+ return SyncCycleSummary(
115
+ cycle_id=cycle_id,
116
+ observed=len(items),
117
+ ingested=tuple(ingested),
118
+ unchanged=unchanged,
119
+ debounced=debounced,
120
+ deletions_observed=tuple(deletions),
121
+ failed=failed,
122
+ )
123
+
124
+ def _observe_item(
125
+ self,
126
+ *,
127
+ deployment_id: UUID,
128
+ source_kind: str,
129
+ cycle_id: UUID,
130
+ source: WatchedSourcePort,
131
+ item: SourceItem,
132
+ known: Mapping[str, str],
133
+ now: datetime,
134
+ quiet: timedelta,
135
+ versioning_mode: str,
136
+ ) -> tuple[str, UUID | None]:
137
+ """Route one observed item down the efficiency ladder."""
138
+ if item.deleted:
139
+ lineage = self._catalog.observe_deletion(
140
+ deployment_id=deployment_id,
141
+ source_kind=source_kind,
142
+ source_ref=item.source_ref,
143
+ cycle_id=cycle_id,
144
+ )
145
+ return ("deleted", lineage)
146
+ if known.get(item.source_ref) == item.revision:
147
+ return ("unchanged", None) # revision no-op: no fetch, no bytes moved
148
+ if now - item.modified_at < quiet:
149
+ return ("debounced", None) # actively edited: coalesce to a later cycle
150
+ content = source.fetch(source_ref=item.source_ref)
151
+ result = self._ingestor.ingest_observed(
152
+ deployment_id=deployment_id,
153
+ source_kind=source_kind,
154
+ source_ref=item.source_ref,
155
+ upload=DocumentUpload(
156
+ filename=item.filename or item.source_ref,
157
+ mime=item.mime,
158
+ content=content,
159
+ ),
160
+ versioning_mode=versioning_mode,
161
+ source_modified_at=item.modified_at,
162
+ source_version_ref=item.revision,
163
+ sync_cycle_id=cycle_id,
164
+ lane=self._settings.lane,
165
+ )
166
+ if result.created:
167
+ return ("ingested", result.version_id)
168
+ return ("unchanged", None) # content-hash no-op (revision churn, same bytes)