rememberstack 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rememberstack/__init__.py +9 -0
- rememberstack/adapters/__init__.py +42 -0
- rememberstack/adapters/codex_writer.py +221 -0
- rememberstack/adapters/markitdown_converter.py +42 -0
- rememberstack/adapters/openrouter.py +136 -0
- rememberstack/adapters/selfhost/__init__.py +54 -0
- rememberstack/adapters/selfhost/forget.py +66 -0
- rememberstack/adapters/selfhost/git.py +374 -0
- rememberstack/adapters/selfhost/lance.py +328 -0
- rememberstack/adapters/selfhost/minio.py +279 -0
- rememberstack/adapters/selfhost/mounts.py +249 -0
- rememberstack/adapters/selfhost/object_store.py +130 -0
- rememberstack/adapters/selfhost/projection.py +80 -0
- rememberstack/adapters/selfhost/queue.py +137 -0
- rememberstack/adapters/selfhost/telemetry.py +45 -0
- rememberstack/adapters/selfhost/watcher.py +70 -0
- rememberstack/adapters/testing/__init__.py +15 -0
- rememberstack/adapters/testing/cost_meter.py +13 -0
- rememberstack/adapters/testing/model_provider.py +83 -0
- rememberstack/adapters/testing/queue.py +43 -0
- rememberstack/adapters/testing/telemetry.py +22 -0
- rememberstack/client.py +19 -0
- rememberstack/core/__init__.py +127 -0
- rememberstack/core/blockizer.py +189 -0
- rememberstack/core/chunker.py +216 -0
- rememberstack/core/consumption_skill.py +275 -0
- rememberstack/core/conversion.py +76 -0
- rememberstack/core/core_manifest.py +598 -0
- rememberstack/core/extension_packs.py +124 -0
- rememberstack/core/forget.py +17 -0
- rememberstack/core/knowledge_authored.py +276 -0
- rememberstack/core/knowledge_compile.py +215 -0
- rememberstack/core/knowledge_fact_sheet.py +210 -0
- rememberstack/core/knowledge_hashing.py +68 -0
- rememberstack/core/knowledge_planner.py +64 -0
- rememberstack/core/knowledge_writer.py +175 -0
- rememberstack/core/ranking.py +200 -0
- rememberstack/core/recipe_linter.py +149 -0
- rememberstack/core/section_snap.py +209 -0
- rememberstack/core/storage_routing.py +27 -0
- rememberstack/eval/__init__.py +53 -0
- rememberstack/eval/consumption.py +141 -0
- rememberstack/eval/contradiction.py +184 -0
- rememberstack/eval/harness.py +136 -0
- rememberstack/eval/lifecycle.py +400 -0
- rememberstack/eval/operational_scale.py +49 -0
- rememberstack/eval/resolution.py +255 -0
- rememberstack/eval/retrieval_spikes.py +50 -0
- rememberstack/eval/skeleton.py +231 -0
- rememberstack/llm/__init__.py +1 -0
- rememberstack/model/__init__.py +589 -0
- rememberstack/model/adjudication.py +100 -0
- rememberstack/model/auth.py +27 -0
- rememberstack/model/blocks.py +30 -0
- rememberstack/model/chunks.py +190 -0
- rememberstack/model/claims.py +162 -0
- rememberstack/model/client.py +98 -0
- rememberstack/model/clustering.py +54 -0
- rememberstack/model/component_version.py +124 -0
- rememberstack/model/consumption.py +88 -0
- rememberstack/model/conversion.py +31 -0
- rememberstack/model/deployment.py +53 -0
- rememberstack/model/documents.py +168 -0
- rememberstack/model/envelope.py +513 -0
- rememberstack/model/evaluation.py +72 -0
- rememberstack/model/forget.py +143 -0
- rememberstack/model/git.py +13 -0
- rememberstack/model/knowledge.py +840 -0
- rememberstack/model/knowledge_authored.py +325 -0
- rememberstack/model/knowledge_planner.py +431 -0
- rememberstack/model/lifecycle.py +42 -0
- rememberstack/model/model_provider.py +78 -0
- rememberstack/model/mounts.py +24 -0
- rememberstack/model/object_store.py +21 -0
- rememberstack/model/operational_scale.py +59 -0
- rememberstack/model/operations.py +153 -0
- rememberstack/model/processing.py +228 -0
- rememberstack/model/queue.py +73 -0
- rememberstack/model/recipes.py +83 -0
- rememberstack/model/relations.py +79 -0
- rememberstack/model/resolution.py +83 -0
- rememberstack/model/retrieval_spikes.py +62 -0
- rememberstack/model/sections.py +120 -0
- rememberstack/model/telemetry.py +30 -0
- rememberstack/ports/__init__.py +29 -0
- rememberstack/ports/auth.py +16 -0
- rememberstack/ports/connector.py +23 -0
- rememberstack/ports/cost_meter.py +17 -0
- rememberstack/ports/forget.py +20 -0
- rememberstack/ports/git.py +20 -0
- rememberstack/ports/model_provider.py +28 -0
- rememberstack/ports/mounts.py +16 -0
- rememberstack/ports/object_store.py +27 -0
- rememberstack/ports/p1_index.py +92 -0
- rememberstack/ports/purge.py +93 -0
- rememberstack/ports/queue.py +23 -0
- rememberstack/ports/telemetry.py +21 -0
- rememberstack/profiles/__init__.py +22 -0
- rememberstack/profiles/selfhost.py +324 -0
- rememberstack/profiles/selfhost_forget.py +158 -0
- rememberstack/profiles/selfhost_operations.py +95 -0
- rememberstack/py.typed +1 -0
- rememberstack/spine/__init__.py +93 -0
- rememberstack/spine/admission.py +26 -0
- rememberstack/spine/backfill.py +168 -0
- rememberstack/spine/catalog_contract.py +742 -0
- rememberstack/spine/chunk_catalog.py +237 -0
- rememberstack/spine/claim_catalog.py +298 -0
- rememberstack/spine/clustering.py +740 -0
- rememberstack/spine/component_versions.py +208 -0
- rememberstack/spine/consumption.py +81 -0
- rememberstack/spine/deployment_bootstrap.py +445 -0
- rememberstack/spine/document_catalog.py +621 -0
- rememberstack/spine/entity_registry.py +205 -0
- rememberstack/spine/extension_packs.py +220 -0
- rememberstack/spine/fact_catalog.py +571 -0
- rememberstack/spine/forget.py +1753 -0
- rememberstack/spine/knowledge.py +5467 -0
- rememberstack/spine/lifecycle.py +1071 -0
- rememberstack/spine/migrations/__init__.py +1 -0
- rememberstack/spine/migrations/_helpers.py +153 -0
- rememberstack/spine/migrations/env.py +58 -0
- rememberstack/spine/migrations/script.py.mako +27 -0
- rememberstack/spine/migrations/versions/__init__.py +1 -0
- rememberstack/spine/migrations/versions/p0_02_0001_extensions_enums.py +189 -0
- rememberstack/spine/migrations/versions/p0_02_0002_infrastructure_registries.py +321 -0
- rememberstack/spine/migrations/versions/p0_02_0003_entities_evaluation_e0_e1.py +631 -0
- rememberstack/spine/migrations/versions/p0_02_0004_claims_facts_evidence.py +411 -0
- rememberstack/spine/migrations/versions/p0_02_0005_projection_knowledge_retrieval.py +391 -0
- rememberstack/spine/migrations/versions/p0_02_0006_partitions_views.py +158 -0
- rememberstack/spine/migrations/versions/p2_06_0007_invalidated_outcome.py +26 -0
- rememberstack/spine/migrations/versions/p3_01_0008_document_version_target.py +58 -0
- rememberstack/spine/migrations/versions/p3_05_0009_reconcile_stage.py +27 -0
- rememberstack/spine/migrations/versions/p3_07_0010_lifecycle_eval_suite.py +25 -0
- rememberstack/spine/migrations/versions/p4_01_0011_survivor_view_rewrite.py +57 -0
- rememberstack/spine/migrations/versions/p6_02_0012_knowledge_compile_recovery.py +58 -0
- rememberstack/spine/migrations/versions/p6_04_0013_knowledge_writer_ledger.py +46 -0
- rememberstack/spine/migrations/versions/p6_05_0014_knowledge_planner_runtime.py +217 -0
- rememberstack/spine/migrations/versions/p6_06_0015_authored_dispatch_runtime.py +38 -0
- rememberstack/spine/migrations/versions/p7_02_0016_operational_eval_suite.py +19 -0
- rememberstack/spine/migrations/versions/p7_05_0017_hard_forget.py +55 -0
- rememberstack/spine/observation_adjudication.py +778 -0
- rememberstack/spine/operations.py +298 -0
- rememberstack/spine/projection.py +662 -0
- rememberstack/spine/recipes.py +276 -0
- rememberstack/spine/resolver.py +763 -0
- rememberstack/spine/review.py +650 -0
- rememberstack/spine/settings.py +22 -0
- rememberstack/spine/supersession.py +510 -0
- rememberstack/spine/sync.py +128 -0
- rememberstack/spine/work_ledger.py +816 -0
- rememberstack/surfaces/__init__.py +110 -0
- rememberstack/surfaces/cli.py +447 -0
- rememberstack/surfaces/consumption_skill.py +87 -0
- rememberstack/surfaces/graph_queries.py +698 -0
- rememberstack/surfaces/http_api.py +377 -0
- rememberstack/surfaces/mcp.py +67 -0
- rememberstack/surfaces/query_engine.py +1591 -0
- rememberstack/surfaces/recipe_executor.py +185 -0
- rememberstack/surfaces/recipe_surface.py +219 -0
- rememberstack/surfaces/remote_mcp.py +133 -0
- rememberstack/surfaces/sdk.py +324 -0
- rememberstack/workers/__init__.py +155 -0
- rememberstack/workers/base.py +312 -0
- rememberstack/workers/e0.py +577 -0
- rememberstack/workers/e1.py +425 -0
- rememberstack/workers/e2.py +525 -0
- rememberstack/workers/e3.py +434 -0
- rememberstack/workers/forget.py +299 -0
- rememberstack/workers/knowledge_authored.py +146 -0
- rememberstack/workers/knowledge_driver.py +735 -0
- rememberstack/workers/knowledge_fact_sheet.py +123 -0
- rememberstack/workers/knowledge_planner.py +325 -0
- rememberstack/workers/knowledge_writer.py +393 -0
- rememberstack/workers/operations.py +42 -0
- rememberstack/workers/p1.py +234 -0
- rememberstack/workers/p2.py +513 -0
- rememberstack/workers/p2_analytics.py +276 -0
- rememberstack/workers/p3.py +673 -0
- rememberstack/workers/reconcile.py +485 -0
- rememberstack/workers/sync.py +168 -0
- rememberstack-0.1.0.dist-info/METADATA +213 -0
- rememberstack-0.1.0.dist-info/RECORD +186 -0
- rememberstack-0.1.0.dist-info/WHEEL +4 -0
- rememberstack-0.1.0.dist-info/entry_points.txt +2 -0
- rememberstack-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
"""Reconciliation, finalization, and deletion grains (lifecycle §3–5, §8).
|
|
2
|
+
|
|
3
|
+
One flow for both lifecycle problems: when a version's chain completes, the
|
|
4
|
+
reconcile stage diffs the lineage's testimony, transitions currency, recounts
|
|
5
|
+
the touched facts, applies the §4 zero-support policy, and emits the
|
|
6
|
+
fact-level `evidence_changed` delta. The policy's two branches never mix:
|
|
7
|
+
|
|
8
|
+
- **the source acted** (a living edit removed content, a deletion) → close
|
|
9
|
+
per shape, recorded and reversible, no flag;
|
|
10
|
+
- **only our transcription changed** (an extractor bump did not re-derive a
|
|
11
|
+
claim from the unchanged file) → flag `support_withdrawn` for review; this
|
|
12
|
+
is the flag's only trigger.
|
|
13
|
+
|
|
14
|
+
Watched lineages defer source-acted closure to their sync cycle's
|
|
15
|
+
FINALIZATION (the retract-timing barrier): an intra-cycle move resolves as a
|
|
16
|
+
support swap, never retract-then-reassert. The `CycleFinalizer` runs that
|
|
17
|
+
job; `DeletionService` is the operator's grain (§8) through the same
|
|
18
|
+
cascade.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from uuid import NAMESPACE_URL
|
|
22
|
+
from uuid import UUID
|
|
23
|
+
from uuid import uuid5
|
|
24
|
+
|
|
25
|
+
from rememberstack.core import chunker_version as chunker_version_of
|
|
26
|
+
from rememberstack.core import ChunkerParams
|
|
27
|
+
from rememberstack.model import ClaimedWork
|
|
28
|
+
from rememberstack.model import CurrencyTransition
|
|
29
|
+
from rememberstack.model import NonRetryableHandlerError
|
|
30
|
+
from rememberstack.model import ReconciliationDelta
|
|
31
|
+
from rememberstack.ports.cost_meter import CostMeterPort
|
|
32
|
+
from rememberstack.spine.lifecycle import LifecycleCatalog
|
|
33
|
+
from rememberstack.spine.review import ReviewQueue
|
|
34
|
+
from rememberstack.workers.base import HandlerOutcome
|
|
35
|
+
from rememberstack.workers.e1 import E2_EXTRACTOR_VERSION
|
|
36
|
+
|
|
37
|
+
RECONCILE_VERSION = "reconcile-2026.07"
|
|
38
|
+
"""The reconcile stage's component version (D12 idempotency key member)."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ReconcileHandler:
|
|
42
|
+
"""The reconcile stage: one completed version's basis change, settled."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
*,
|
|
47
|
+
catalog: LifecycleCatalog,
|
|
48
|
+
review_queue: ReviewQueue,
|
|
49
|
+
extractor_version: str = E2_EXTRACTOR_VERSION,
|
|
50
|
+
chunker_version: str | None = None,
|
|
51
|
+
) -> None:
|
|
52
|
+
"""Bind the handler to the lifecycle catalog and the review queue.
|
|
53
|
+
|
|
54
|
+
``chunker_version`` names the packing generation of the completing
|
|
55
|
+
basis (the same parameters the composing profile gave the chunk
|
|
56
|
+
stage); it defaults to the default parameters' generation.
|
|
57
|
+
"""
|
|
58
|
+
self._catalog = catalog
|
|
59
|
+
self._review_queue = review_queue
|
|
60
|
+
self._extractor_version = extractor_version
|
|
61
|
+
self._chunker_version = chunker_version or chunker_version_of(
|
|
62
|
+
params=ChunkerParams()
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def handle(self, *, work: ClaimedWork, meter: CostMeterPort) -> HandlerOutcome:
|
|
66
|
+
"""Diff → transition → recount → policy → emit, idempotently.
|
|
67
|
+
|
|
68
|
+
The work row's processing_id is the run's `reconciliation_id`: a
|
|
69
|
+
retried attempt re-emits every ledger row, closure, flag, and
|
|
70
|
+
trigger as a no-op.
|
|
71
|
+
"""
|
|
72
|
+
del meter
|
|
73
|
+
version_id = _payload_uuid(work=work, field="version_id")
|
|
74
|
+
representation_id = _payload_uuid(work=work, field="representation_id")
|
|
75
|
+
context = self._catalog.reconciliation_context(version_id=version_id)
|
|
76
|
+
deployment_id = work.deployment_id
|
|
77
|
+
reconciliation_id = work.processing_id
|
|
78
|
+
|
|
79
|
+
source_acted: tuple[CurrencyTransition, ...] = ()
|
|
80
|
+
if (
|
|
81
|
+
context["versioning_mode"] == "living"
|
|
82
|
+
and context["current_version_id"] is not None
|
|
83
|
+
):
|
|
84
|
+
source_acted = self._catalog.stale_for_supersession(
|
|
85
|
+
deployment_id=deployment_id,
|
|
86
|
+
doc_id=context["doc_id"], # type: ignore[arg-type]
|
|
87
|
+
current_version_id=context["current_version_id"], # type: ignore[arg-type]
|
|
88
|
+
)
|
|
89
|
+
transcription = self._catalog.stale_for_reextraction(
|
|
90
|
+
version_id=version_id,
|
|
91
|
+
representation_id=representation_id,
|
|
92
|
+
chunker_version=self._chunker_version,
|
|
93
|
+
extractor_version=self._extractor_version,
|
|
94
|
+
)
|
|
95
|
+
transitions = (*source_acted, *transcription)
|
|
96
|
+
applied = self._catalog.apply_transitions(
|
|
97
|
+
deployment_id=deployment_id,
|
|
98
|
+
reconciliation_id=reconciliation_id,
|
|
99
|
+
transitions=transitions,
|
|
100
|
+
)
|
|
101
|
+
# retry recovery (Codex review): a crash between the currency
|
|
102
|
+
# transaction and the steps below must not orphan the run — union
|
|
103
|
+
# what the ledger already holds under this reconciliation_id, since
|
|
104
|
+
# a retry recomputes an empty stale set (the cache already flipped)
|
|
105
|
+
recorded = self._catalog.recorded_transitions(
|
|
106
|
+
reconciliation_id=reconciliation_id
|
|
107
|
+
)
|
|
108
|
+
seen = {(t.claim_id, t.reason, t.became_current) for t in transitions}
|
|
109
|
+
for prior in recorded:
|
|
110
|
+
key = (prior.claim_id, prior.reason, prior.became_current)
|
|
111
|
+
if key not in seen:
|
|
112
|
+
seen.add(key)
|
|
113
|
+
transitions = (*transitions, *(prior,))
|
|
114
|
+
if prior.reason == "reextracted":
|
|
115
|
+
transcription = (*transcription, prior)
|
|
116
|
+
else:
|
|
117
|
+
source_acted = (*source_acted, prior)
|
|
118
|
+
|
|
119
|
+
claim_ids = tuple({transition.claim_id for transition in transitions})
|
|
120
|
+
relation_ids = self._catalog.affected_relation_ids(claim_ids=claim_ids)
|
|
121
|
+
observation_ids = self._catalog.affected_observation_ids(claim_ids=claim_ids)
|
|
122
|
+
changed_relations, changed_observations = self._catalog.recount(
|
|
123
|
+
relation_ids=relation_ids, observation_ids=observation_ids
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
zero_relations = self._catalog.open_zero_support_relations(
|
|
127
|
+
relation_ids=relation_ids
|
|
128
|
+
)
|
|
129
|
+
zero_observations = self._catalog.open_zero_support_observations(
|
|
130
|
+
observation_ids=observation_ids
|
|
131
|
+
)
|
|
132
|
+
source_claims = tuple({t.claim_id for t in source_acted})
|
|
133
|
+
source_relations = set(
|
|
134
|
+
self._catalog.affected_relation_ids(claim_ids=source_claims)
|
|
135
|
+
)
|
|
136
|
+
source_observations = set(
|
|
137
|
+
self._catalog.affected_observation_ids(claim_ids=source_claims)
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
closed_relations: tuple[UUID, ...] = ()
|
|
141
|
+
closed_observations: tuple[UUID, ...] = ()
|
|
142
|
+
if context["sync_cycle_id"] is None:
|
|
143
|
+
# not cycle-stamped (uploads, direct API ingest): the source
|
|
144
|
+
# acted and there is no move-vs-retract ambiguity — close now
|
|
145
|
+
closed_relations = self._catalog.close_relations(
|
|
146
|
+
deployment_id=deployment_id,
|
|
147
|
+
relation_ids=tuple(
|
|
148
|
+
fact for fact in zero_relations if fact in source_relations
|
|
149
|
+
),
|
|
150
|
+
boundary=context["current_source_modified_at"],
|
|
151
|
+
reconciliation_id=reconciliation_id,
|
|
152
|
+
)
|
|
153
|
+
closed_observations = self._catalog.close_observations(
|
|
154
|
+
deployment_id=deployment_id,
|
|
155
|
+
observation_ids=tuple(
|
|
156
|
+
fact for fact in zero_observations if fact in source_observations
|
|
157
|
+
),
|
|
158
|
+
reconciliation_id=reconciliation_id,
|
|
159
|
+
)
|
|
160
|
+
# else: closure waits for the cycle-finalization barrier (§5) —
|
|
161
|
+
# an intra-cycle move must land as a support swap, never a retract
|
|
162
|
+
|
|
163
|
+
flags = self._flag_transcription_only(
|
|
164
|
+
deployment_id=deployment_id,
|
|
165
|
+
transcription=transcription,
|
|
166
|
+
zero_relations=tuple(
|
|
167
|
+
fact for fact in zero_relations if fact not in source_relations
|
|
168
|
+
),
|
|
169
|
+
zero_observations=tuple(
|
|
170
|
+
fact for fact in zero_observations if fact not in source_observations
|
|
171
|
+
),
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
self._catalog.emit_evidence_changed(
|
|
175
|
+
deployment_id=deployment_id,
|
|
176
|
+
delta=ReconciliationDelta(
|
|
177
|
+
reconciliation_id=reconciliation_id,
|
|
178
|
+
transitions=applied,
|
|
179
|
+
# the stale-storm guard: only facts whose STATE moved — a
|
|
180
|
+
# re-extraction that changes no fact state stales nothing
|
|
181
|
+
recounted_relations=changed_relations,
|
|
182
|
+
recounted_observations=changed_observations,
|
|
183
|
+
relations_closed=closed_relations,
|
|
184
|
+
observations_closed=closed_observations,
|
|
185
|
+
flags_raised=flags,
|
|
186
|
+
),
|
|
187
|
+
)
|
|
188
|
+
return HandlerOutcome()
|
|
189
|
+
|
|
190
|
+
def _flag_transcription_only(
|
|
191
|
+
self,
|
|
192
|
+
*,
|
|
193
|
+
deployment_id: UUID,
|
|
194
|
+
transcription: tuple[CurrencyTransition, ...],
|
|
195
|
+
zero_relations: tuple[UUID, ...],
|
|
196
|
+
zero_observations: tuple[UUID, ...],
|
|
197
|
+
) -> tuple[UUID, ...]:
|
|
198
|
+
"""§4's second branch: transcription-only zero support → one flag each.
|
|
199
|
+
|
|
200
|
+
The event carries no information about the world (the file still
|
|
201
|
+
says what it said), so no mechanical verdict is derivable — a
|
|
202
|
+
reviewer decides. Idempotent: an already-open flag is never stacked.
|
|
203
|
+
"""
|
|
204
|
+
flagged: list[UUID] = []
|
|
205
|
+
for fact_kind, fact_ids in (
|
|
206
|
+
("relation", zero_relations),
|
|
207
|
+
("observation", zero_observations),
|
|
208
|
+
):
|
|
209
|
+
for fact_id in fact_ids:
|
|
210
|
+
if self._review_queue.has_open_support_withdrawn(fact_id=fact_id):
|
|
211
|
+
continue
|
|
212
|
+
withdrawn = self._withdrawn_claim(
|
|
213
|
+
fact_kind=fact_kind, fact_id=fact_id, transcription=transcription
|
|
214
|
+
)
|
|
215
|
+
if withdrawn is None:
|
|
216
|
+
continue
|
|
217
|
+
self._review_queue.flag_support_withdrawn(
|
|
218
|
+
deployment_id=deployment_id,
|
|
219
|
+
fact_kind=fact_kind,
|
|
220
|
+
fact_id=fact_id,
|
|
221
|
+
claim_id=withdrawn.claim_id,
|
|
222
|
+
diff={
|
|
223
|
+
"reason": "reextracted",
|
|
224
|
+
"from_extractor_version": withdrawn.from_extractor_version,
|
|
225
|
+
"to_extractor_version": self._extractor_version,
|
|
226
|
+
# the full superseding basis, for exact attribution
|
|
227
|
+
# when a non-extractor coordinate caused the bump
|
|
228
|
+
"to_chunker_version": self._chunker_version,
|
|
229
|
+
},
|
|
230
|
+
)
|
|
231
|
+
flagged.append(fact_id)
|
|
232
|
+
return tuple(flagged)
|
|
233
|
+
|
|
234
|
+
def _withdrawn_claim(
|
|
235
|
+
self,
|
|
236
|
+
*,
|
|
237
|
+
fact_kind: str,
|
|
238
|
+
fact_id: UUID,
|
|
239
|
+
transcription: tuple[CurrencyTransition, ...],
|
|
240
|
+
) -> CurrencyTransition | None:
|
|
241
|
+
"""The transitioned claim whose withdrawal starved this fact."""
|
|
242
|
+
for transition in transcription:
|
|
243
|
+
affected = (
|
|
244
|
+
self._catalog.affected_relation_ids(claim_ids=(transition.claim_id,))
|
|
245
|
+
if fact_kind == "relation"
|
|
246
|
+
else self._catalog.affected_observation_ids(
|
|
247
|
+
claim_ids=(transition.claim_id,)
|
|
248
|
+
)
|
|
249
|
+
)
|
|
250
|
+
if fact_id in affected:
|
|
251
|
+
return transition
|
|
252
|
+
return None
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
class CycleFinalizer:
|
|
256
|
+
"""The retract-timing barrier's second half: per-cycle retraction (§5).
|
|
257
|
+
|
|
258
|
+
Runs after every lineage a cycle observed has finished its chain:
|
|
259
|
+
evaluates source-acted zero-support closure for observed lineages and
|
|
260
|
+
runs the deletion cascade for lineages whose source deletion the cycle
|
|
261
|
+
recorded. Lineages still extracting defer the whole cycle — the
|
|
262
|
+
recorded grace, visible as (completed_at set, finalized_at null).
|
|
263
|
+
"""
|
|
264
|
+
|
|
265
|
+
def __init__(self, *, catalog: LifecycleCatalog) -> None:
|
|
266
|
+
"""Bind the finalizer to the lifecycle catalog."""
|
|
267
|
+
self._catalog = catalog
|
|
268
|
+
|
|
269
|
+
def finalize_ready(self, *, deployment_id: UUID) -> tuple[UUID, ...]:
|
|
270
|
+
"""Finalize every ready cycle; returns the cycles this call won.
|
|
271
|
+
|
|
272
|
+
The claim is atomic and FIRST (two finalizer instances never both
|
|
273
|
+
evaluate one cycle); every cascade re-derives from current state
|
|
274
|
+
under a derived, stable reconciliation id, so a crash mid-cycle
|
|
275
|
+
leaves a brief, visible, self-healing gap rather than duplicates.
|
|
276
|
+
Source-tombstoned lineages are swept deployment-wide on every pass
|
|
277
|
+
for the same reason. A LOSSY cycle (per-item failures) skips
|
|
278
|
+
absence-based closure — its observation set is incomplete; the next
|
|
279
|
+
healthy cycle of the same source covers it.
|
|
280
|
+
"""
|
|
281
|
+
finalized: list[UUID] = []
|
|
282
|
+
for cycle_id, failed_items in self._catalog.cycles_ready_to_finalize(
|
|
283
|
+
deployment_id=deployment_id
|
|
284
|
+
):
|
|
285
|
+
if not self._catalog.claim_finalization(cycle_id=cycle_id):
|
|
286
|
+
continue # another finalizer won this cycle
|
|
287
|
+
if failed_items == 0:
|
|
288
|
+
for doc_id in self._catalog.cycle_lineages(cycle_id=cycle_id):
|
|
289
|
+
self._close_lineage_zero_support(
|
|
290
|
+
deployment_id=deployment_id, cycle_id=cycle_id, doc_id=doc_id
|
|
291
|
+
)
|
|
292
|
+
finalized.append(cycle_id)
|
|
293
|
+
for doc_id in self._catalog.tombstoned_lineages_needing_cascade(
|
|
294
|
+
deployment_id=deployment_id
|
|
295
|
+
):
|
|
296
|
+
cascade_lineage_removal(
|
|
297
|
+
catalog=self._catalog,
|
|
298
|
+
deployment_id=deployment_id,
|
|
299
|
+
doc_id=doc_id,
|
|
300
|
+
reconciliation_id=_derived_run_id(
|
|
301
|
+
kind="finalize-delete", doc_id=doc_id
|
|
302
|
+
),
|
|
303
|
+
)
|
|
304
|
+
return tuple(finalized)
|
|
305
|
+
|
|
306
|
+
def _close_lineage_zero_support(
|
|
307
|
+
self, *, deployment_id: UUID, cycle_id: UUID, doc_id: UUID
|
|
308
|
+
) -> None:
|
|
309
|
+
"""§4 source-acted closure for one observed lineage, cycle-scoped."""
|
|
310
|
+
claim_ids = self._catalog.lineage_claim_ids(
|
|
311
|
+
deployment_id=deployment_id, doc_id=doc_id
|
|
312
|
+
)
|
|
313
|
+
relation_ids = self._catalog.affected_relation_ids(claim_ids=claim_ids)
|
|
314
|
+
observation_ids = self._catalog.affected_observation_ids(claim_ids=claim_ids)
|
|
315
|
+
reconciliation_id = _derived_run_id(
|
|
316
|
+
kind="finalize", cycle_id=cycle_id, doc_id=doc_id
|
|
317
|
+
)
|
|
318
|
+
closed_relations = self._catalog.close_relations(
|
|
319
|
+
deployment_id=deployment_id,
|
|
320
|
+
relation_ids=self._catalog.open_zero_support_relations(
|
|
321
|
+
relation_ids=relation_ids
|
|
322
|
+
),
|
|
323
|
+
boundary=self._catalog.closure_boundary(doc_id=doc_id),
|
|
324
|
+
reconciliation_id=reconciliation_id,
|
|
325
|
+
)
|
|
326
|
+
closed_observations = self._catalog.close_observations(
|
|
327
|
+
deployment_id=deployment_id,
|
|
328
|
+
observation_ids=self._catalog.open_zero_support_observations(
|
|
329
|
+
observation_ids=observation_ids
|
|
330
|
+
),
|
|
331
|
+
reconciliation_id=reconciliation_id,
|
|
332
|
+
)
|
|
333
|
+
self._catalog.emit_evidence_changed(
|
|
334
|
+
deployment_id=deployment_id,
|
|
335
|
+
delta=ReconciliationDelta(
|
|
336
|
+
reconciliation_id=reconciliation_id,
|
|
337
|
+
relations_closed=closed_relations,
|
|
338
|
+
observations_closed=closed_observations,
|
|
339
|
+
),
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class DeletionService:
|
|
344
|
+
"""The §8 deletion grains: version, lineage — one uniform cascade.
|
|
345
|
+
|
|
346
|
+
Deleting removes the document's contribution: currency ends, counts
|
|
347
|
+
recompute, solely-supported facts close (recorded, reversible, no
|
|
348
|
+
flag). Claims are retained as history — forgotten ≠ deleted; only
|
|
349
|
+
hard-forget (§13) scrubs content.
|
|
350
|
+
"""
|
|
351
|
+
|
|
352
|
+
def __init__(self, *, catalog: LifecycleCatalog) -> None:
|
|
353
|
+
"""Bind the service to the lifecycle catalog."""
|
|
354
|
+
self._catalog = catalog
|
|
355
|
+
|
|
356
|
+
def delete_version(self, *, version_id: UUID) -> ReconciliationDelta:
|
|
357
|
+
"""End one version's testimony; the lineage continues (§8)."""
|
|
358
|
+
info = self._catalog.delete_version(version_id=version_id)
|
|
359
|
+
deployment_id: UUID = info["deployment_id"] # type: ignore[assignment]
|
|
360
|
+
doc_id: UUID = info["doc_id"] # type: ignore[assignment]
|
|
361
|
+
context = self._catalog.reconciliation_context(version_id=version_id)
|
|
362
|
+
# ONLY the deleted version's exclusive testimony ends — a snapshot
|
|
363
|
+
# lineage's other versions keep their currency (Codex review)
|
|
364
|
+
transitions: tuple[CurrencyTransition, ...] = (
|
|
365
|
+
self._catalog.stale_for_version_deletion(
|
|
366
|
+
deployment_id=deployment_id, version_id=version_id
|
|
367
|
+
)
|
|
368
|
+
)
|
|
369
|
+
if context["current_version_id"] is not None:
|
|
370
|
+
# "the lineage continues": the repointed predecessor's testimony
|
|
371
|
+
# is the current basis again — its claims regain currency
|
|
372
|
+
transitions = (
|
|
373
|
+
*transitions,
|
|
374
|
+
*self._catalog.regained_by_current_version(
|
|
375
|
+
deployment_id=deployment_id,
|
|
376
|
+
doc_id=doc_id,
|
|
377
|
+
current_version_id=context["current_version_id"], # type: ignore[arg-type]
|
|
378
|
+
),
|
|
379
|
+
)
|
|
380
|
+
return _cascade(
|
|
381
|
+
catalog=self._catalog,
|
|
382
|
+
deployment_id=deployment_id,
|
|
383
|
+
transitions=transitions,
|
|
384
|
+
reconciliation_id=_derived_run_id(kind="delete-version", id_=version_id),
|
|
385
|
+
boundary=self._catalog.closure_boundary(doc_id=doc_id),
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
def delete_lineage(
|
|
389
|
+
self, *, deployment_id: UUID, doc_id: UUID
|
|
390
|
+
) -> ReconciliationDelta:
|
|
391
|
+
"""Remove a lineage's whole contribution (operator grain, §8)."""
|
|
392
|
+
self._catalog.delete_lineage(doc_id=doc_id)
|
|
393
|
+
return cascade_lineage_removal(
|
|
394
|
+
catalog=self._catalog,
|
|
395
|
+
deployment_id=deployment_id,
|
|
396
|
+
doc_id=doc_id,
|
|
397
|
+
reconciliation_id=_derived_run_id(kind="delete-lineage", id_=doc_id),
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def cascade_lineage_removal(
|
|
402
|
+
*,
|
|
403
|
+
catalog: LifecycleCatalog,
|
|
404
|
+
deployment_id: UUID,
|
|
405
|
+
doc_id: UUID,
|
|
406
|
+
reconciliation_id: UUID,
|
|
407
|
+
) -> ReconciliationDelta:
|
|
408
|
+
"""The uniform lineage-removal cascade (§8): currency → recount → close."""
|
|
409
|
+
transitions = catalog.stale_for_deletion(deployment_id=deployment_id, doc_id=doc_id)
|
|
410
|
+
recorded = catalog.recorded_transitions(reconciliation_id=reconciliation_id)
|
|
411
|
+
seen = {(item.claim_id, item.reason, item.became_current) for item in transitions}
|
|
412
|
+
transitions = (
|
|
413
|
+
*transitions,
|
|
414
|
+
*(
|
|
415
|
+
item
|
|
416
|
+
for item in recorded
|
|
417
|
+
if (item.claim_id, item.reason, item.became_current) not in seen
|
|
418
|
+
),
|
|
419
|
+
)
|
|
420
|
+
return _cascade(
|
|
421
|
+
catalog=catalog,
|
|
422
|
+
deployment_id=deployment_id,
|
|
423
|
+
transitions=transitions,
|
|
424
|
+
reconciliation_id=reconciliation_id,
|
|
425
|
+
boundary=None,
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _cascade(
|
|
430
|
+
*,
|
|
431
|
+
catalog: LifecycleCatalog,
|
|
432
|
+
deployment_id: UUID,
|
|
433
|
+
transitions: tuple[CurrencyTransition, ...],
|
|
434
|
+
reconciliation_id: UUID,
|
|
435
|
+
boundary: object,
|
|
436
|
+
) -> ReconciliationDelta:
|
|
437
|
+
"""Apply one source-acted basis change end to end, idempotently."""
|
|
438
|
+
applied = catalog.apply_transitions(
|
|
439
|
+
deployment_id=deployment_id,
|
|
440
|
+
reconciliation_id=reconciliation_id,
|
|
441
|
+
transitions=transitions,
|
|
442
|
+
)
|
|
443
|
+
claim_ids = tuple({transition.claim_id for transition in transitions})
|
|
444
|
+
relation_ids = catalog.affected_relation_ids(claim_ids=claim_ids)
|
|
445
|
+
observation_ids = catalog.affected_observation_ids(claim_ids=claim_ids)
|
|
446
|
+
catalog.recount(relation_ids=relation_ids, observation_ids=observation_ids)
|
|
447
|
+
closed_relations = catalog.close_relations(
|
|
448
|
+
deployment_id=deployment_id,
|
|
449
|
+
relation_ids=catalog.open_zero_support_relations(relation_ids=relation_ids),
|
|
450
|
+
boundary=boundary,
|
|
451
|
+
reconciliation_id=reconciliation_id,
|
|
452
|
+
)
|
|
453
|
+
closed_observations = catalog.close_observations(
|
|
454
|
+
deployment_id=deployment_id,
|
|
455
|
+
observation_ids=catalog.open_zero_support_observations(
|
|
456
|
+
observation_ids=observation_ids
|
|
457
|
+
),
|
|
458
|
+
reconciliation_id=reconciliation_id,
|
|
459
|
+
)
|
|
460
|
+
delta = ReconciliationDelta(
|
|
461
|
+
reconciliation_id=reconciliation_id,
|
|
462
|
+
transitions=applied,
|
|
463
|
+
recounted_relations=relation_ids,
|
|
464
|
+
recounted_observations=observation_ids,
|
|
465
|
+
relations_closed=closed_relations,
|
|
466
|
+
observations_closed=closed_observations,
|
|
467
|
+
)
|
|
468
|
+
catalog.emit_evidence_changed(deployment_id=deployment_id, delta=delta)
|
|
469
|
+
return delta
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _derived_run_id(*, kind: str, **parts: object) -> UUID:
|
|
473
|
+
"""A stable reconciliation id for non-queued runs (retry-idempotent)."""
|
|
474
|
+
suffix = ":".join(str(value) for value in parts.values())
|
|
475
|
+
return uuid5(NAMESPACE_URL, f"rememberstack:{kind}:{suffix}")
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _payload_uuid(*, work: ClaimedWork, field: str) -> UUID:
|
|
479
|
+
"""Read a required UUID from the claimed payload; absence is non-retryable."""
|
|
480
|
+
value = (work.payload or {}).get(field)
|
|
481
|
+
if not isinstance(value, str):
|
|
482
|
+
raise NonRetryableHandlerError(
|
|
483
|
+
f"stage {work.stage} work {work.processing_id} carries no {field!r} payload"
|
|
484
|
+
)
|
|
485
|
+
return UUID(value)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""The watch loop (lifecycle §2, D55): one recorded poll cycle per pass.
|
|
2
|
+
|
|
3
|
+
The efficiency ladder's cheapest exits live here: unchanged revision → no
|
|
4
|
+
fetch; a file being actively edited coalesces to one ingested version per
|
|
5
|
+
stability window (the debounce discipline); identical bytes are the
|
|
6
|
+
content-hash no-op downstream. Deletion observations tombstone the lineage
|
|
7
|
+
loudly; the cascade is the delete worker's job.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from datetime import timedelta
|
|
13
|
+
from datetime import UTC
|
|
14
|
+
from typing import Final
|
|
15
|
+
from uuid import UUID
|
|
16
|
+
|
|
17
|
+
from pydantic import Field
|
|
18
|
+
from pydantic_settings import BaseSettings
|
|
19
|
+
from pydantic_settings import SettingsConfigDict
|
|
20
|
+
|
|
21
|
+
from rememberstack.model import DocumentUpload
|
|
22
|
+
from rememberstack.model import ProcessingLane
|
|
23
|
+
from rememberstack.model import SourceItem
|
|
24
|
+
from rememberstack.model import SyncCycleSummary
|
|
25
|
+
from rememberstack.ports.connector import WatchedSourcePort
|
|
26
|
+
from rememberstack.spine.sync import SyncCatalog
|
|
27
|
+
from rememberstack.workers.e0 import UploadIngestor
|
|
28
|
+
|
|
29
|
+
DEFAULT_VERSIONING_MODE_BY_KIND: Final = {
|
|
30
|
+
# spike 6's starting heuristic: edit-in-place leans living;
|
|
31
|
+
# replace-whole-file and rolling logs lean archival (snapshot)
|
|
32
|
+
"watched_directory": "living",
|
|
33
|
+
"google_drive": "living",
|
|
34
|
+
"upload": "snapshot",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class SyncSettings(BaseSettings):
|
|
39
|
+
"""The watch loop's knobs (starting points, D22)."""
|
|
40
|
+
|
|
41
|
+
model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_SYNC_")
|
|
42
|
+
|
|
43
|
+
debounce_quiet_seconds: float = Field(default=120.0, ge=0.0)
|
|
44
|
+
lane: ProcessingLane = ProcessingLane.STEADY
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class SyncCycleRunner:
|
|
48
|
+
"""Run one recorded poll cycle for one (deployment, source kind)."""
|
|
49
|
+
|
|
50
|
+
def __init__(
|
|
51
|
+
self, *, catalog: SyncCatalog, ingestor: UploadIngestor, settings: SyncSettings
|
|
52
|
+
) -> None:
|
|
53
|
+
"""Bind the runner to the cycle catalog, ingest path, and settings."""
|
|
54
|
+
self._catalog = catalog
|
|
55
|
+
self._ingestor = ingestor
|
|
56
|
+
self._settings = settings
|
|
57
|
+
|
|
58
|
+
def run_cycle(
|
|
59
|
+
self, *, deployment_id: UUID, source_kind: str, source: WatchedSourcePort
|
|
60
|
+
) -> SyncCycleSummary:
|
|
61
|
+
"""One poll pass: observe, debounce, ingest, tombstone, record.
|
|
62
|
+
|
|
63
|
+
The cycle row always reaches ``completed_at`` — one bad item (or a
|
|
64
|
+
failed poll) must not strand a cycle that already stamped versions,
|
|
65
|
+
because an eternally open cycle can never pass reconciliation's
|
|
66
|
+
finalization barrier. Lost items are counted on the row instead.
|
|
67
|
+
"""
|
|
68
|
+
known = self._catalog.known_revisions(
|
|
69
|
+
deployment_id=deployment_id, source_kind=source_kind
|
|
70
|
+
)
|
|
71
|
+
cycle_id = self._catalog.open_cycle(
|
|
72
|
+
deployment_id=deployment_id, source_kind=source_kind
|
|
73
|
+
)
|
|
74
|
+
items: tuple[SourceItem, ...] = ()
|
|
75
|
+
ingested: list[UUID] = []
|
|
76
|
+
deletions: list[UUID] = []
|
|
77
|
+
unchanged = 0
|
|
78
|
+
debounced = 0
|
|
79
|
+
failed = 0
|
|
80
|
+
now = datetime.now(tz=UTC)
|
|
81
|
+
quiet = timedelta(seconds=self._settings.debounce_quiet_seconds)
|
|
82
|
+
versioning_mode = DEFAULT_VERSIONING_MODE_BY_KIND.get(source_kind, "snapshot")
|
|
83
|
+
try:
|
|
84
|
+
items = source.poll(known=known)
|
|
85
|
+
for item in items:
|
|
86
|
+
try:
|
|
87
|
+
outcome = self._observe_item(
|
|
88
|
+
deployment_id=deployment_id,
|
|
89
|
+
source_kind=source_kind,
|
|
90
|
+
cycle_id=cycle_id,
|
|
91
|
+
source=source,
|
|
92
|
+
item=item,
|
|
93
|
+
known=known,
|
|
94
|
+
now=now,
|
|
95
|
+
quiet=quiet,
|
|
96
|
+
versioning_mode=versioning_mode,
|
|
97
|
+
)
|
|
98
|
+
except Exception: # noqa: BLE001 — the pass must stay recorded
|
|
99
|
+
failed += 1
|
|
100
|
+
continue
|
|
101
|
+
kind, ref = outcome
|
|
102
|
+
if kind == "ingested" and ref is not None:
|
|
103
|
+
ingested.append(ref)
|
|
104
|
+
elif kind == "deleted" and ref is not None:
|
|
105
|
+
deletions.append(ref)
|
|
106
|
+
elif kind == "unchanged":
|
|
107
|
+
unchanged += 1
|
|
108
|
+
elif kind == "debounced":
|
|
109
|
+
debounced += 1
|
|
110
|
+
finally:
|
|
111
|
+
self._catalog.complete_cycle(
|
|
112
|
+
cycle_id=cycle_id, observed=len(items), failed=failed
|
|
113
|
+
)
|
|
114
|
+
return SyncCycleSummary(
|
|
115
|
+
cycle_id=cycle_id,
|
|
116
|
+
observed=len(items),
|
|
117
|
+
ingested=tuple(ingested),
|
|
118
|
+
unchanged=unchanged,
|
|
119
|
+
debounced=debounced,
|
|
120
|
+
deletions_observed=tuple(deletions),
|
|
121
|
+
failed=failed,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
def _observe_item(
|
|
125
|
+
self,
|
|
126
|
+
*,
|
|
127
|
+
deployment_id: UUID,
|
|
128
|
+
source_kind: str,
|
|
129
|
+
cycle_id: UUID,
|
|
130
|
+
source: WatchedSourcePort,
|
|
131
|
+
item: SourceItem,
|
|
132
|
+
known: Mapping[str, str],
|
|
133
|
+
now: datetime,
|
|
134
|
+
quiet: timedelta,
|
|
135
|
+
versioning_mode: str,
|
|
136
|
+
) -> tuple[str, UUID | None]:
|
|
137
|
+
"""Route one observed item down the efficiency ladder."""
|
|
138
|
+
if item.deleted:
|
|
139
|
+
lineage = self._catalog.observe_deletion(
|
|
140
|
+
deployment_id=deployment_id,
|
|
141
|
+
source_kind=source_kind,
|
|
142
|
+
source_ref=item.source_ref,
|
|
143
|
+
cycle_id=cycle_id,
|
|
144
|
+
)
|
|
145
|
+
return ("deleted", lineage)
|
|
146
|
+
if known.get(item.source_ref) == item.revision:
|
|
147
|
+
return ("unchanged", None) # revision no-op: no fetch, no bytes moved
|
|
148
|
+
if now - item.modified_at < quiet:
|
|
149
|
+
return ("debounced", None) # actively edited: coalesce to a later cycle
|
|
150
|
+
content = source.fetch(source_ref=item.source_ref)
|
|
151
|
+
result = self._ingestor.ingest_observed(
|
|
152
|
+
deployment_id=deployment_id,
|
|
153
|
+
source_kind=source_kind,
|
|
154
|
+
source_ref=item.source_ref,
|
|
155
|
+
upload=DocumentUpload(
|
|
156
|
+
filename=item.filename or item.source_ref,
|
|
157
|
+
mime=item.mime,
|
|
158
|
+
content=content,
|
|
159
|
+
),
|
|
160
|
+
versioning_mode=versioning_mode,
|
|
161
|
+
source_modified_at=item.modified_at,
|
|
162
|
+
source_version_ref=item.revision,
|
|
163
|
+
sync_cycle_id=cycle_id,
|
|
164
|
+
lane=self._settings.lane,
|
|
165
|
+
)
|
|
166
|
+
if result.created:
|
|
167
|
+
return ("ingested", result.version_id)
|
|
168
|
+
return ("unchanged", None) # content-hash no-op (revision churn, same bytes)
|