rememberstack 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rememberstack/__init__.py +9 -0
- rememberstack/adapters/__init__.py +42 -0
- rememberstack/adapters/codex_writer.py +221 -0
- rememberstack/adapters/markitdown_converter.py +42 -0
- rememberstack/adapters/openrouter.py +136 -0
- rememberstack/adapters/selfhost/__init__.py +54 -0
- rememberstack/adapters/selfhost/forget.py +66 -0
- rememberstack/adapters/selfhost/git.py +374 -0
- rememberstack/adapters/selfhost/lance.py +328 -0
- rememberstack/adapters/selfhost/minio.py +279 -0
- rememberstack/adapters/selfhost/mounts.py +249 -0
- rememberstack/adapters/selfhost/object_store.py +130 -0
- rememberstack/adapters/selfhost/projection.py +80 -0
- rememberstack/adapters/selfhost/queue.py +137 -0
- rememberstack/adapters/selfhost/telemetry.py +45 -0
- rememberstack/adapters/selfhost/watcher.py +70 -0
- rememberstack/adapters/testing/__init__.py +15 -0
- rememberstack/adapters/testing/cost_meter.py +13 -0
- rememberstack/adapters/testing/model_provider.py +83 -0
- rememberstack/adapters/testing/queue.py +43 -0
- rememberstack/adapters/testing/telemetry.py +22 -0
- rememberstack/client.py +19 -0
- rememberstack/core/__init__.py +127 -0
- rememberstack/core/blockizer.py +189 -0
- rememberstack/core/chunker.py +216 -0
- rememberstack/core/consumption_skill.py +275 -0
- rememberstack/core/conversion.py +76 -0
- rememberstack/core/core_manifest.py +598 -0
- rememberstack/core/extension_packs.py +124 -0
- rememberstack/core/forget.py +17 -0
- rememberstack/core/knowledge_authored.py +276 -0
- rememberstack/core/knowledge_compile.py +215 -0
- rememberstack/core/knowledge_fact_sheet.py +210 -0
- rememberstack/core/knowledge_hashing.py +68 -0
- rememberstack/core/knowledge_planner.py +64 -0
- rememberstack/core/knowledge_writer.py +175 -0
- rememberstack/core/ranking.py +200 -0
- rememberstack/core/recipe_linter.py +149 -0
- rememberstack/core/section_snap.py +209 -0
- rememberstack/core/storage_routing.py +27 -0
- rememberstack/eval/__init__.py +53 -0
- rememberstack/eval/consumption.py +141 -0
- rememberstack/eval/contradiction.py +184 -0
- rememberstack/eval/harness.py +136 -0
- rememberstack/eval/lifecycle.py +400 -0
- rememberstack/eval/operational_scale.py +49 -0
- rememberstack/eval/resolution.py +255 -0
- rememberstack/eval/retrieval_spikes.py +50 -0
- rememberstack/eval/skeleton.py +231 -0
- rememberstack/llm/__init__.py +1 -0
- rememberstack/model/__init__.py +589 -0
- rememberstack/model/adjudication.py +100 -0
- rememberstack/model/auth.py +27 -0
- rememberstack/model/blocks.py +30 -0
- rememberstack/model/chunks.py +190 -0
- rememberstack/model/claims.py +162 -0
- rememberstack/model/client.py +98 -0
- rememberstack/model/clustering.py +54 -0
- rememberstack/model/component_version.py +124 -0
- rememberstack/model/consumption.py +88 -0
- rememberstack/model/conversion.py +31 -0
- rememberstack/model/deployment.py +53 -0
- rememberstack/model/documents.py +168 -0
- rememberstack/model/envelope.py +513 -0
- rememberstack/model/evaluation.py +72 -0
- rememberstack/model/forget.py +143 -0
- rememberstack/model/git.py +13 -0
- rememberstack/model/knowledge.py +840 -0
- rememberstack/model/knowledge_authored.py +325 -0
- rememberstack/model/knowledge_planner.py +431 -0
- rememberstack/model/lifecycle.py +42 -0
- rememberstack/model/model_provider.py +78 -0
- rememberstack/model/mounts.py +24 -0
- rememberstack/model/object_store.py +21 -0
- rememberstack/model/operational_scale.py +59 -0
- rememberstack/model/operations.py +153 -0
- rememberstack/model/processing.py +228 -0
- rememberstack/model/queue.py +73 -0
- rememberstack/model/recipes.py +83 -0
- rememberstack/model/relations.py +79 -0
- rememberstack/model/resolution.py +83 -0
- rememberstack/model/retrieval_spikes.py +62 -0
- rememberstack/model/sections.py +120 -0
- rememberstack/model/telemetry.py +30 -0
- rememberstack/ports/__init__.py +29 -0
- rememberstack/ports/auth.py +16 -0
- rememberstack/ports/connector.py +23 -0
- rememberstack/ports/cost_meter.py +17 -0
- rememberstack/ports/forget.py +20 -0
- rememberstack/ports/git.py +20 -0
- rememberstack/ports/model_provider.py +28 -0
- rememberstack/ports/mounts.py +16 -0
- rememberstack/ports/object_store.py +27 -0
- rememberstack/ports/p1_index.py +92 -0
- rememberstack/ports/purge.py +93 -0
- rememberstack/ports/queue.py +23 -0
- rememberstack/ports/telemetry.py +21 -0
- rememberstack/profiles/__init__.py +22 -0
- rememberstack/profiles/selfhost.py +324 -0
- rememberstack/profiles/selfhost_forget.py +158 -0
- rememberstack/profiles/selfhost_operations.py +95 -0
- rememberstack/py.typed +1 -0
- rememberstack/spine/__init__.py +93 -0
- rememberstack/spine/admission.py +26 -0
- rememberstack/spine/backfill.py +168 -0
- rememberstack/spine/catalog_contract.py +742 -0
- rememberstack/spine/chunk_catalog.py +237 -0
- rememberstack/spine/claim_catalog.py +298 -0
- rememberstack/spine/clustering.py +740 -0
- rememberstack/spine/component_versions.py +208 -0
- rememberstack/spine/consumption.py +81 -0
- rememberstack/spine/deployment_bootstrap.py +445 -0
- rememberstack/spine/document_catalog.py +621 -0
- rememberstack/spine/entity_registry.py +205 -0
- rememberstack/spine/extension_packs.py +220 -0
- rememberstack/spine/fact_catalog.py +571 -0
- rememberstack/spine/forget.py +1753 -0
- rememberstack/spine/knowledge.py +5467 -0
- rememberstack/spine/lifecycle.py +1071 -0
- rememberstack/spine/migrations/__init__.py +1 -0
- rememberstack/spine/migrations/_helpers.py +153 -0
- rememberstack/spine/migrations/env.py +58 -0
- rememberstack/spine/migrations/script.py.mako +27 -0
- rememberstack/spine/migrations/versions/__init__.py +1 -0
- rememberstack/spine/migrations/versions/p0_02_0001_extensions_enums.py +189 -0
- rememberstack/spine/migrations/versions/p0_02_0002_infrastructure_registries.py +321 -0
- rememberstack/spine/migrations/versions/p0_02_0003_entities_evaluation_e0_e1.py +631 -0
- rememberstack/spine/migrations/versions/p0_02_0004_claims_facts_evidence.py +411 -0
- rememberstack/spine/migrations/versions/p0_02_0005_projection_knowledge_retrieval.py +391 -0
- rememberstack/spine/migrations/versions/p0_02_0006_partitions_views.py +158 -0
- rememberstack/spine/migrations/versions/p2_06_0007_invalidated_outcome.py +26 -0
- rememberstack/spine/migrations/versions/p3_01_0008_document_version_target.py +58 -0
- rememberstack/spine/migrations/versions/p3_05_0009_reconcile_stage.py +27 -0
- rememberstack/spine/migrations/versions/p3_07_0010_lifecycle_eval_suite.py +25 -0
- rememberstack/spine/migrations/versions/p4_01_0011_survivor_view_rewrite.py +57 -0
- rememberstack/spine/migrations/versions/p6_02_0012_knowledge_compile_recovery.py +58 -0
- rememberstack/spine/migrations/versions/p6_04_0013_knowledge_writer_ledger.py +46 -0
- rememberstack/spine/migrations/versions/p6_05_0014_knowledge_planner_runtime.py +217 -0
- rememberstack/spine/migrations/versions/p6_06_0015_authored_dispatch_runtime.py +38 -0
- rememberstack/spine/migrations/versions/p7_02_0016_operational_eval_suite.py +19 -0
- rememberstack/spine/migrations/versions/p7_05_0017_hard_forget.py +55 -0
- rememberstack/spine/observation_adjudication.py +778 -0
- rememberstack/spine/operations.py +298 -0
- rememberstack/spine/projection.py +662 -0
- rememberstack/spine/recipes.py +276 -0
- rememberstack/spine/resolver.py +763 -0
- rememberstack/spine/review.py +650 -0
- rememberstack/spine/settings.py +22 -0
- rememberstack/spine/supersession.py +510 -0
- rememberstack/spine/sync.py +128 -0
- rememberstack/spine/work_ledger.py +816 -0
- rememberstack/surfaces/__init__.py +110 -0
- rememberstack/surfaces/cli.py +447 -0
- rememberstack/surfaces/consumption_skill.py +87 -0
- rememberstack/surfaces/graph_queries.py +698 -0
- rememberstack/surfaces/http_api.py +377 -0
- rememberstack/surfaces/mcp.py +67 -0
- rememberstack/surfaces/query_engine.py +1591 -0
- rememberstack/surfaces/recipe_executor.py +185 -0
- rememberstack/surfaces/recipe_surface.py +219 -0
- rememberstack/surfaces/remote_mcp.py +133 -0
- rememberstack/surfaces/sdk.py +324 -0
- rememberstack/workers/__init__.py +155 -0
- rememberstack/workers/base.py +312 -0
- rememberstack/workers/e0.py +577 -0
- rememberstack/workers/e1.py +425 -0
- rememberstack/workers/e2.py +525 -0
- rememberstack/workers/e3.py +434 -0
- rememberstack/workers/forget.py +299 -0
- rememberstack/workers/knowledge_authored.py +146 -0
- rememberstack/workers/knowledge_driver.py +735 -0
- rememberstack/workers/knowledge_fact_sheet.py +123 -0
- rememberstack/workers/knowledge_planner.py +325 -0
- rememberstack/workers/knowledge_writer.py +393 -0
- rememberstack/workers/operations.py +42 -0
- rememberstack/workers/p1.py +234 -0
- rememberstack/workers/p2.py +513 -0
- rememberstack/workers/p2_analytics.py +276 -0
- rememberstack/workers/p3.py +673 -0
- rememberstack/workers/reconcile.py +485 -0
- rememberstack/workers/sync.py +168 -0
- rememberstack-0.1.0.dist-info/METADATA +213 -0
- rememberstack-0.1.0.dist-info/RECORD +186 -0
- rememberstack-0.1.0.dist-info/WHEEL +4 -0
- rememberstack-0.1.0.dist-info/entry_points.txt +2 -0
- rememberstack-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,1753 @@
|
|
|
1
|
+
"""PostgreSQL materialization, admission, and inventory for D74."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
import json
|
|
5
|
+
from uuid import UUID
|
|
6
|
+
|
|
7
|
+
from sqlalchemy import text
|
|
8
|
+
from sqlalchemy import TextClause
|
|
9
|
+
from sqlalchemy.engine import Connection
|
|
10
|
+
from sqlalchemy.engine import Engine
|
|
11
|
+
from sqlalchemy.engine import RowMapping
|
|
12
|
+
|
|
13
|
+
from rememberstack.core import source_identity_hash
|
|
14
|
+
from rememberstack.model import ComponentVersionConflictError
|
|
15
|
+
from rememberstack.model import EnqueueOutcome
|
|
16
|
+
from rememberstack.model import EnqueueWork
|
|
17
|
+
from rememberstack.model import ForgetInProgressError
|
|
18
|
+
from rememberstack.model import ForgetManifest
|
|
19
|
+
from rememberstack.model import ForgetManifestConflictError
|
|
20
|
+
from rememberstack.model import ForgetManifestNotFoundError
|
|
21
|
+
from rememberstack.model import ForgetManifestRecord
|
|
22
|
+
from rememberstack.model import ForgetManifestStatus
|
|
23
|
+
from rememberstack.model import ForgetTargetNotFoundError
|
|
24
|
+
from rememberstack.model import ForgottenSourceError
|
|
25
|
+
from rememberstack.model import ObjectKey
|
|
26
|
+
from rememberstack.model import PipelineStage
|
|
27
|
+
from rememberstack.model import ProcessingTarget
|
|
28
|
+
from rememberstack.spine.admission import active_forget_id_on
|
|
29
|
+
from rememberstack.spine.work_ledger import enqueue_on
|
|
30
|
+
|
|
31
|
+
HARD_FORGET_COMPONENT_VERSION = "hard-forget-v1"
|
|
32
|
+
"""The deterministic D74 coordinator generation recorded on its one work row."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ForgetCatalog:
|
|
36
|
+
"""Own local forget admission, inventory, progress, and exact ingest guards."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, *, engine: Engine) -> None:
|
|
39
|
+
"""Bind the guard catalog to the authoritative spine database."""
|
|
40
|
+
self._engine = engine
|
|
41
|
+
|
|
42
|
+
def assert_available(self, *, deployment_id: UUID) -> None:
|
|
43
|
+
"""Refuse traffic while a preparing or accepted forget is incomplete."""
|
|
44
|
+
with self._engine.connect() as connection:
|
|
45
|
+
forget_id = active_forget_id_on(
|
|
46
|
+
connection=connection, deployment_id=deployment_id
|
|
47
|
+
)
|
|
48
|
+
if forget_id is not None:
|
|
49
|
+
raise ForgetInProgressError(
|
|
50
|
+
f"deployment {deployment_id} is honoring forget_id {forget_id}"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
def prepare(
|
|
54
|
+
self, *, deployment_id: UUID, doc_id: UUID, forget_id: UUID
|
|
55
|
+
) -> ForgetManifestRecord:
|
|
56
|
+
"""Establish one deployment-wide preparing barrier idempotently."""
|
|
57
|
+
with self._engine.begin() as connection:
|
|
58
|
+
connection.execute(_LOCK_DEPLOYMENT, {"deployment_id": deployment_id})
|
|
59
|
+
active_id = active_forget_id_on(
|
|
60
|
+
connection=connection, deployment_id=deployment_id
|
|
61
|
+
)
|
|
62
|
+
if active_id is not None:
|
|
63
|
+
active = (
|
|
64
|
+
connection.execute(
|
|
65
|
+
_SELECT_BY_ID,
|
|
66
|
+
{"deployment_id": deployment_id, "forget_id": active_id},
|
|
67
|
+
)
|
|
68
|
+
.mappings()
|
|
69
|
+
.one()
|
|
70
|
+
)
|
|
71
|
+
if active["forget_id"] == forget_id and active["doc_id"] == doc_id:
|
|
72
|
+
return _record(row=active)
|
|
73
|
+
raise ForgetInProgressError(
|
|
74
|
+
f"deployment {deployment_id} is already honoring forget_id"
|
|
75
|
+
f" {active_id}"
|
|
76
|
+
)
|
|
77
|
+
existing = (
|
|
78
|
+
connection.execute(
|
|
79
|
+
_SELECT_BY_DOC, {"deployment_id": deployment_id, "doc_id": doc_id}
|
|
80
|
+
)
|
|
81
|
+
.mappings()
|
|
82
|
+
.first()
|
|
83
|
+
)
|
|
84
|
+
if existing is not None:
|
|
85
|
+
if existing["forget_id"] != forget_id:
|
|
86
|
+
raise ForgetManifestConflictError(
|
|
87
|
+
f"document {doc_id} already belongs to forget_id"
|
|
88
|
+
f" {existing['forget_id']}"
|
|
89
|
+
)
|
|
90
|
+
return _record(row=existing)
|
|
91
|
+
owned = (
|
|
92
|
+
connection.execute(
|
|
93
|
+
_SELECT_DOCUMENT, {"deployment_id": deployment_id, "doc_id": doc_id}
|
|
94
|
+
)
|
|
95
|
+
.mappings()
|
|
96
|
+
.first()
|
|
97
|
+
)
|
|
98
|
+
if owned is None:
|
|
99
|
+
raise ForgetTargetNotFoundError(
|
|
100
|
+
f"deployment {deployment_id} does not own document {doc_id}"
|
|
101
|
+
)
|
|
102
|
+
row = (
|
|
103
|
+
connection.execute(
|
|
104
|
+
_INSERT_PREPARING,
|
|
105
|
+
{
|
|
106
|
+
"forget_id": forget_id,
|
|
107
|
+
"deployment_id": deployment_id,
|
|
108
|
+
"doc_id": doc_id,
|
|
109
|
+
},
|
|
110
|
+
)
|
|
111
|
+
.mappings()
|
|
112
|
+
.one()
|
|
113
|
+
)
|
|
114
|
+
return _record(row=row)
|
|
115
|
+
|
|
116
|
+
def ordinary_work_is_drained(self, *, deployment_id: UUID) -> bool:
|
|
117
|
+
"""Return whether every already-running non-forget handler has finished."""
|
|
118
|
+
with self._engine.connect() as connection:
|
|
119
|
+
return bool(
|
|
120
|
+
connection.execute(
|
|
121
|
+
_ORDINARY_WORK_DRAINED, {"deployment_id": deployment_id}
|
|
122
|
+
).scalar_one()
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
def record_for_doc(
|
|
126
|
+
self, *, deployment_id: UUID, doc_id: UUID
|
|
127
|
+
) -> ForgetManifestRecord | None:
|
|
128
|
+
"""Return existing local progress for an idempotent request retry."""
|
|
129
|
+
with self._engine.connect() as connection:
|
|
130
|
+
row = (
|
|
131
|
+
connection.execute(
|
|
132
|
+
_SELECT_BY_DOC, {"deployment_id": deployment_id, "doc_id": doc_id}
|
|
133
|
+
)
|
|
134
|
+
.mappings()
|
|
135
|
+
.first()
|
|
136
|
+
)
|
|
137
|
+
return _record(row=row) if row is not None else None
|
|
138
|
+
|
|
139
|
+
def preparing_record(self, *, deployment_id: UUID) -> ForgetManifestRecord | None:
|
|
140
|
+
"""Return the single crash-recoverable preparing request, when present."""
|
|
141
|
+
with self._engine.connect() as connection:
|
|
142
|
+
row = (
|
|
143
|
+
connection.execute(_SELECT_PREPARING, {"deployment_id": deployment_id})
|
|
144
|
+
.mappings()
|
|
145
|
+
.one_or_none()
|
|
146
|
+
)
|
|
147
|
+
return _record(row=row) if row is not None else None
|
|
148
|
+
|
|
149
|
+
def materialize_portable(self, *, manifest: ForgetManifest) -> None:
|
|
150
|
+
"""Recreate or validate the local barrier from portable restore intent."""
|
|
151
|
+
parameters = {
|
|
152
|
+
"forget_id": manifest.forget_id,
|
|
153
|
+
"deployment_id": manifest.deployment_id,
|
|
154
|
+
"doc_id": manifest.doc_id,
|
|
155
|
+
"schema_version": manifest.schema_version,
|
|
156
|
+
"manifest_hash": manifest.sha256(),
|
|
157
|
+
"manifest": manifest.canonical_bytes().decode("utf-8"),
|
|
158
|
+
"source_identity_hash": manifest.source_identity_hash,
|
|
159
|
+
"content_hashes": list(manifest.content_hashes),
|
|
160
|
+
}
|
|
161
|
+
with self._engine.begin() as connection:
|
|
162
|
+
connection.execute(
|
|
163
|
+
_LOCK_DEPLOYMENT, {"deployment_id": manifest.deployment_id}
|
|
164
|
+
)
|
|
165
|
+
by_document = (
|
|
166
|
+
connection.execute(
|
|
167
|
+
_SELECT_BY_DOC,
|
|
168
|
+
{
|
|
169
|
+
"deployment_id": manifest.deployment_id,
|
|
170
|
+
"doc_id": manifest.doc_id,
|
|
171
|
+
},
|
|
172
|
+
)
|
|
173
|
+
.mappings()
|
|
174
|
+
.one_or_none()
|
|
175
|
+
)
|
|
176
|
+
if (
|
|
177
|
+
by_document is not None
|
|
178
|
+
and by_document["forget_id"] != manifest.forget_id
|
|
179
|
+
):
|
|
180
|
+
raise ForgetManifestConflictError(
|
|
181
|
+
f"document {manifest.doc_id} already belongs to forget_id"
|
|
182
|
+
f" {by_document['forget_id']}"
|
|
183
|
+
)
|
|
184
|
+
row = (
|
|
185
|
+
connection.execute(
|
|
186
|
+
_SELECT_BY_ID_FOR_UPDATE,
|
|
187
|
+
{
|
|
188
|
+
"deployment_id": manifest.deployment_id,
|
|
189
|
+
"forget_id": manifest.forget_id,
|
|
190
|
+
},
|
|
191
|
+
)
|
|
192
|
+
.mappings()
|
|
193
|
+
.one_or_none()
|
|
194
|
+
)
|
|
195
|
+
if row is None:
|
|
196
|
+
connection.execute(_INSERT_PORTABLE, parameters)
|
|
197
|
+
return
|
|
198
|
+
if row["doc_id"] != manifest.doc_id:
|
|
199
|
+
raise ForgetManifestConflictError(
|
|
200
|
+
f"forget_id {manifest.forget_id} targets a different document"
|
|
201
|
+
)
|
|
202
|
+
if row["manifest"] is not None:
|
|
203
|
+
_require_same_manifest(row=row, manifest=manifest)
|
|
204
|
+
return
|
|
205
|
+
connection.execute(_MATERIALIZE_PREPARING, parameters)
|
|
206
|
+
|
|
207
|
+
def blocking_k_paths(self, *, deployment_id: UUID, doc_id: UUID) -> tuple[str, ...]:
|
|
208
|
+
"""Return owner-controlled paths whose synced citations still reach a lineage."""
|
|
209
|
+
with self._engine.connect() as connection:
|
|
210
|
+
values = connection.execute(
|
|
211
|
+
_BLOCKING_K_PATHS, {"deployment_id": deployment_id, "doc_id": doc_id}
|
|
212
|
+
).scalars()
|
|
213
|
+
return tuple(sorted({str(value) for value in values}))
|
|
214
|
+
|
|
215
|
+
def k_paths_for_artifacts(
|
|
216
|
+
self, *, deployment_id: UUID, artifact_ids: tuple[UUID, ...]
|
|
217
|
+
) -> tuple[str, ...]:
|
|
218
|
+
"""Return exact body and curation paths nominated for Git history purge."""
|
|
219
|
+
if not artifact_ids:
|
|
220
|
+
return ()
|
|
221
|
+
with self._engine.connect() as connection:
|
|
222
|
+
values = connection.execute(
|
|
223
|
+
_K_PATHS_FOR_ARTIFACTS,
|
|
224
|
+
{"deployment_id": deployment_id, "artifact_ids": list(artifact_ids)},
|
|
225
|
+
).scalars()
|
|
226
|
+
return tuple(sorted({str(value) for value in values if value is not None}))
|
|
227
|
+
|
|
228
|
+
def inventory_and_store_manifest(
|
|
229
|
+
self,
|
|
230
|
+
*,
|
|
231
|
+
deployment_id: UUID,
|
|
232
|
+
doc_id: UUID,
|
|
233
|
+
forget_id: UUID,
|
|
234
|
+
requested_at: datetime,
|
|
235
|
+
) -> ForgetManifest:
|
|
236
|
+
"""Freeze one repeatable-read inventory and commit it before portable append."""
|
|
237
|
+
with self._engine.connect().execution_options(
|
|
238
|
+
isolation_level="REPEATABLE READ"
|
|
239
|
+
) as connection:
|
|
240
|
+
with connection.begin():
|
|
241
|
+
row = (
|
|
242
|
+
connection.execute(
|
|
243
|
+
_SELECT_BY_ID_FOR_UPDATE,
|
|
244
|
+
{"deployment_id": deployment_id, "forget_id": forget_id},
|
|
245
|
+
)
|
|
246
|
+
.mappings()
|
|
247
|
+
.first()
|
|
248
|
+
)
|
|
249
|
+
if row is None or row["doc_id"] != doc_id:
|
|
250
|
+
raise ForgetManifestNotFoundError(
|
|
251
|
+
f"preparing forget_id {forget_id} does not exist"
|
|
252
|
+
)
|
|
253
|
+
if row["manifest"] is not None:
|
|
254
|
+
return _manifest(row=row)
|
|
255
|
+
document = (
|
|
256
|
+
connection.execute(
|
|
257
|
+
_SELECT_DOCUMENT,
|
|
258
|
+
{"deployment_id": deployment_id, "doc_id": doc_id},
|
|
259
|
+
)
|
|
260
|
+
.mappings()
|
|
261
|
+
.first()
|
|
262
|
+
)
|
|
263
|
+
if document is None:
|
|
264
|
+
raise ForgetTargetNotFoundError(
|
|
265
|
+
f"deployment {deployment_id} does not own document {doc_id}"
|
|
266
|
+
)
|
|
267
|
+
manifest = ForgetManifest(
|
|
268
|
+
forget_id=forget_id,
|
|
269
|
+
deployment_id=deployment_id,
|
|
270
|
+
doc_id=doc_id,
|
|
271
|
+
requested_at=requested_at,
|
|
272
|
+
source_identity_hash=(
|
|
273
|
+
source_identity_hash(
|
|
274
|
+
deployment_id=deployment_id,
|
|
275
|
+
source_kind=str(document["source_kind"]),
|
|
276
|
+
source_ref=str(document["source_ref"]),
|
|
277
|
+
)
|
|
278
|
+
if document["source_ref"] is not None
|
|
279
|
+
else None
|
|
280
|
+
),
|
|
281
|
+
content_hashes=_text_values(
|
|
282
|
+
connection=connection,
|
|
283
|
+
statement=_CONTENT_HASHES,
|
|
284
|
+
deployment_id=deployment_id,
|
|
285
|
+
doc_id=doc_id,
|
|
286
|
+
),
|
|
287
|
+
chunk_ids=_uuid_values(
|
|
288
|
+
connection=connection,
|
|
289
|
+
statement=_CHUNK_IDS,
|
|
290
|
+
deployment_id=deployment_id,
|
|
291
|
+
doc_id=doc_id,
|
|
292
|
+
),
|
|
293
|
+
claim_ids=_uuid_values(
|
|
294
|
+
connection=connection,
|
|
295
|
+
statement=_CLAIM_IDS,
|
|
296
|
+
deployment_id=deployment_id,
|
|
297
|
+
doc_id=doc_id,
|
|
298
|
+
),
|
|
299
|
+
mention_ids=_uuid_values(
|
|
300
|
+
connection=connection,
|
|
301
|
+
statement=_MENTION_IDS,
|
|
302
|
+
deployment_id=deployment_id,
|
|
303
|
+
doc_id=doc_id,
|
|
304
|
+
),
|
|
305
|
+
resolved_entity_ids=_uuid_values(
|
|
306
|
+
connection=connection,
|
|
307
|
+
statement=_RESOLVED_ENTITY_IDS,
|
|
308
|
+
deployment_id=deployment_id,
|
|
309
|
+
doc_id=doc_id,
|
|
310
|
+
),
|
|
311
|
+
fact_ids=_uuid_values(
|
|
312
|
+
connection=connection,
|
|
313
|
+
statement=_EXCLUSIVE_FACT_IDS,
|
|
314
|
+
deployment_id=deployment_id,
|
|
315
|
+
doc_id=doc_id,
|
|
316
|
+
),
|
|
317
|
+
entity_ids=_uuid_values(
|
|
318
|
+
connection=connection,
|
|
319
|
+
statement=_EXCLUSIVE_ENTITY_IDS,
|
|
320
|
+
deployment_id=deployment_id,
|
|
321
|
+
doc_id=doc_id,
|
|
322
|
+
),
|
|
323
|
+
object_keys=tuple(
|
|
324
|
+
ObjectKey(value)
|
|
325
|
+
for value in _text_values(
|
|
326
|
+
connection=connection,
|
|
327
|
+
statement=_OBJECT_KEYS,
|
|
328
|
+
deployment_id=deployment_id,
|
|
329
|
+
doc_id=doc_id,
|
|
330
|
+
)
|
|
331
|
+
),
|
|
332
|
+
projection_prefixes=tuple(
|
|
333
|
+
ObjectKey(value)
|
|
334
|
+
for value in _text_values(
|
|
335
|
+
connection=connection,
|
|
336
|
+
statement=_PROJECTION_PREFIXES,
|
|
337
|
+
deployment_id=deployment_id,
|
|
338
|
+
doc_id=doc_id,
|
|
339
|
+
)
|
|
340
|
+
),
|
|
341
|
+
k_artifact_ids=_uuid_values(
|
|
342
|
+
connection=connection,
|
|
343
|
+
statement=_K_ARTIFACT_IDS,
|
|
344
|
+
deployment_id=deployment_id,
|
|
345
|
+
doc_id=doc_id,
|
|
346
|
+
),
|
|
347
|
+
)
|
|
348
|
+
connection.execute(
|
|
349
|
+
_STORE_MANIFEST,
|
|
350
|
+
{
|
|
351
|
+
"deployment_id": deployment_id,
|
|
352
|
+
"forget_id": forget_id,
|
|
353
|
+
"schema_version": manifest.schema_version,
|
|
354
|
+
"manifest_hash": manifest.sha256(),
|
|
355
|
+
"manifest": manifest.canonical_bytes().decode("utf-8"),
|
|
356
|
+
"source_identity_hash": manifest.source_identity_hash,
|
|
357
|
+
"content_hashes": list(manifest.content_hashes),
|
|
358
|
+
},
|
|
359
|
+
)
|
|
360
|
+
return manifest
|
|
361
|
+
|
|
362
|
+
def cancel_unstored_preparation(
|
|
363
|
+
self, *, deployment_id: UUID, forget_id: UUID
|
|
364
|
+
) -> bool:
|
|
365
|
+
"""Reopen admission only before immutable manifest bytes have been stored."""
|
|
366
|
+
with self._engine.begin() as connection:
|
|
367
|
+
return (
|
|
368
|
+
connection.execute(
|
|
369
|
+
_DELETE_UNSTORED_PREPARING,
|
|
370
|
+
{"deployment_id": deployment_id, "forget_id": forget_id},
|
|
371
|
+
).rowcount
|
|
372
|
+
== 1
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
def accept_and_enqueue(self, *, manifest: ForgetManifest) -> EnqueueOutcome:
|
|
376
|
+
"""Acknowledge portable append and atomically create the one worker row."""
|
|
377
|
+
with self._engine.begin() as connection:
|
|
378
|
+
row = (
|
|
379
|
+
connection.execute(
|
|
380
|
+
_SELECT_BY_ID_FOR_UPDATE,
|
|
381
|
+
{
|
|
382
|
+
"deployment_id": manifest.deployment_id,
|
|
383
|
+
"forget_id": manifest.forget_id,
|
|
384
|
+
},
|
|
385
|
+
)
|
|
386
|
+
.mappings()
|
|
387
|
+
.first()
|
|
388
|
+
)
|
|
389
|
+
if row is None:
|
|
390
|
+
raise ForgetManifestNotFoundError(
|
|
391
|
+
f"stored forget_id {manifest.forget_id} does not exist"
|
|
392
|
+
)
|
|
393
|
+
_require_same_manifest(row=row, manifest=manifest)
|
|
394
|
+
connection.execute(
|
|
395
|
+
_REGISTER_FORGETTER,
|
|
396
|
+
{
|
|
397
|
+
"deployment_id": manifest.deployment_id,
|
|
398
|
+
"version": HARD_FORGET_COMPONENT_VERSION,
|
|
399
|
+
},
|
|
400
|
+
)
|
|
401
|
+
definition_matches = bool(
|
|
402
|
+
connection.execute(
|
|
403
|
+
_FORGETTER_DEFINITION_MATCHES,
|
|
404
|
+
{
|
|
405
|
+
"deployment_id": manifest.deployment_id,
|
|
406
|
+
"version": HARD_FORGET_COMPONENT_VERSION,
|
|
407
|
+
},
|
|
408
|
+
).scalar_one()
|
|
409
|
+
)
|
|
410
|
+
if not definition_matches:
|
|
411
|
+
raise ComponentVersionConflictError(
|
|
412
|
+
"hard-forget component version has a conflicting definition"
|
|
413
|
+
)
|
|
414
|
+
connection.execute(
|
|
415
|
+
_MARK_ACCEPTED,
|
|
416
|
+
{
|
|
417
|
+
"deployment_id": manifest.deployment_id,
|
|
418
|
+
"forget_id": manifest.forget_id,
|
|
419
|
+
},
|
|
420
|
+
)
|
|
421
|
+
return enqueue_on(
|
|
422
|
+
connection=connection,
|
|
423
|
+
work=EnqueueWork(
|
|
424
|
+
deployment_id=manifest.deployment_id,
|
|
425
|
+
target_kind=ProcessingTarget.DOCUMENT,
|
|
426
|
+
target_id=manifest.doc_id,
|
|
427
|
+
stage=PipelineStage.HARD_FORGET,
|
|
428
|
+
component_version=HARD_FORGET_COMPONENT_VERSION,
|
|
429
|
+
content_hash=manifest.sha256(),
|
|
430
|
+
lane=None,
|
|
431
|
+
payload={"forget_id": str(manifest.forget_id)},
|
|
432
|
+
),
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
def manifest_for(self, *, deployment_id: UUID, forget_id: UUID) -> ForgetManifest:
|
|
436
|
+
"""Return one accepted worker's exact immutable local manifest."""
|
|
437
|
+
with self._engine.connect() as connection:
|
|
438
|
+
row = (
|
|
439
|
+
connection.execute(
|
|
440
|
+
_SELECT_BY_ID,
|
|
441
|
+
{"deployment_id": deployment_id, "forget_id": forget_id},
|
|
442
|
+
)
|
|
443
|
+
.mappings()
|
|
444
|
+
.first()
|
|
445
|
+
)
|
|
446
|
+
if row is None or row["manifest"] is None:
|
|
447
|
+
raise ForgetManifestNotFoundError(
|
|
448
|
+
f"accepted forget_id {forget_id} does not exist"
|
|
449
|
+
)
|
|
450
|
+
return _manifest(row=row)
|
|
451
|
+
|
|
452
|
+
def scrub_postgres(self, *, manifest: ForgetManifest) -> None:
|
|
453
|
+
"""Idempotently remove lineage-owned source-bearing PostgreSQL payloads."""
|
|
454
|
+
parameters = {
|
|
455
|
+
"deployment_id": manifest.deployment_id,
|
|
456
|
+
"doc_id": manifest.doc_id,
|
|
457
|
+
"forget_id": manifest.forget_id,
|
|
458
|
+
"chunk_ids": list(manifest.chunk_ids),
|
|
459
|
+
"claim_ids": list(manifest.claim_ids),
|
|
460
|
+
"mention_ids": list(manifest.mention_ids),
|
|
461
|
+
"resolved_entity_ids": list(manifest.resolved_entity_ids),
|
|
462
|
+
"fact_ids": list(manifest.fact_ids),
|
|
463
|
+
"entity_ids": list(manifest.entity_ids),
|
|
464
|
+
"k_artifact_ids": list(manifest.k_artifact_ids),
|
|
465
|
+
"content_hashes": list(manifest.content_hashes),
|
|
466
|
+
}
|
|
467
|
+
with self._engine.begin() as connection:
|
|
468
|
+
row = (
|
|
469
|
+
connection.execute(
|
|
470
|
+
_SELECT_BY_ID_FOR_UPDATE,
|
|
471
|
+
{
|
|
472
|
+
"deployment_id": manifest.deployment_id,
|
|
473
|
+
"forget_id": manifest.forget_id,
|
|
474
|
+
},
|
|
475
|
+
)
|
|
476
|
+
.mappings()
|
|
477
|
+
.first()
|
|
478
|
+
)
|
|
479
|
+
if row is None:
|
|
480
|
+
raise ForgetManifestNotFoundError(
|
|
481
|
+
f"forget_id {manifest.forget_id} does not exist"
|
|
482
|
+
)
|
|
483
|
+
_require_same_manifest(row=row, manifest=manifest)
|
|
484
|
+
for statement in _POSTGRES_SCRUB:
|
|
485
|
+
connection.execute(statement, parameters)
|
|
486
|
+
|
|
487
|
+
def verify_postgres_scrubbed(self, *, manifest: ForgetManifest) -> None:
|
|
488
|
+
"""Raise visibly if any nominated source-bearing PostgreSQL row remains."""
|
|
489
|
+
with self._engine.connect() as connection:
|
|
490
|
+
remaining = int(
|
|
491
|
+
connection.execute(
|
|
492
|
+
_VERIFY_POSTGRES_SCRUB,
|
|
493
|
+
{
|
|
494
|
+
"deployment_id": manifest.deployment_id,
|
|
495
|
+
"doc_id": manifest.doc_id,
|
|
496
|
+
"chunk_ids": list(manifest.chunk_ids),
|
|
497
|
+
"claim_ids": list(manifest.claim_ids),
|
|
498
|
+
"mention_ids": list(manifest.mention_ids),
|
|
499
|
+
"resolved_entity_ids": list(manifest.resolved_entity_ids),
|
|
500
|
+
"fact_ids": list(manifest.fact_ids),
|
|
501
|
+
"entity_ids": list(manifest.entity_ids),
|
|
502
|
+
"k_artifact_ids": list(manifest.k_artifact_ids),
|
|
503
|
+
"content_hashes": list(manifest.content_hashes),
|
|
504
|
+
},
|
|
505
|
+
).scalar_one()
|
|
506
|
+
)
|
|
507
|
+
if remaining:
|
|
508
|
+
raise RuntimeError(
|
|
509
|
+
f"forget_id {manifest.forget_id} PostgreSQL verification found"
|
|
510
|
+
f" {remaining} source-bearing row groups"
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
def mark_complete(self, *, manifest: ForgetManifest) -> None:
|
|
514
|
+
"""Open admission only after every store and PostgreSQL verification succeeds."""
|
|
515
|
+
with self._engine.begin() as connection:
|
|
516
|
+
updated = connection.execute(
|
|
517
|
+
_MARK_COMPLETE,
|
|
518
|
+
{
|
|
519
|
+
"deployment_id": manifest.deployment_id,
|
|
520
|
+
"forget_id": manifest.forget_id,
|
|
521
|
+
"manifest_hash": manifest.sha256(),
|
|
522
|
+
},
|
|
523
|
+
).rowcount
|
|
524
|
+
if updated != 1:
|
|
525
|
+
raise ForgetManifestConflictError(
|
|
526
|
+
f"forget_id {manifest.forget_id} was not accepted with these bytes"
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
def guard_ingest(
|
|
530
|
+
self,
|
|
531
|
+
*,
|
|
532
|
+
deployment_id: UUID,
|
|
533
|
+
source_kind: str,
|
|
534
|
+
source_ref: str,
|
|
535
|
+
content_hash: str,
|
|
536
|
+
) -> None:
|
|
537
|
+
"""Refuse active traffic and any completed manifest's exact identity."""
|
|
538
|
+
self.assert_available(deployment_id=deployment_id)
|
|
539
|
+
identity_hash = source_identity_hash(
|
|
540
|
+
deployment_id=deployment_id, source_kind=source_kind, source_ref=source_ref
|
|
541
|
+
)
|
|
542
|
+
with self._engine.connect() as connection:
|
|
543
|
+
forget_id = connection.execute(
|
|
544
|
+
_FORGOTTEN_INGEST,
|
|
545
|
+
{
|
|
546
|
+
"deployment_id": deployment_id,
|
|
547
|
+
"source_identity_hash": identity_hash,
|
|
548
|
+
"content_hash": content_hash,
|
|
549
|
+
},
|
|
550
|
+
).scalar_one_or_none()
|
|
551
|
+
if forget_id is not None:
|
|
552
|
+
raise ForgottenSourceError(
|
|
553
|
+
f"ingest matches irreversible forget_id {forget_id}"
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def _record(*, row: RowMapping) -> ForgetManifestRecord:
|
|
558
|
+
"""Validate one database row as the typed progress boundary."""
|
|
559
|
+
return ForgetManifestRecord(
|
|
560
|
+
forget_id=row["forget_id"],
|
|
561
|
+
deployment_id=row["deployment_id"],
|
|
562
|
+
doc_id=row["doc_id"],
|
|
563
|
+
manifest=_manifest(row=row) if row["manifest"] is not None else None,
|
|
564
|
+
manifest_hash=row["manifest_hash"],
|
|
565
|
+
status=ForgetManifestStatus(str(row["status"])),
|
|
566
|
+
prepared_at=row["prepared_at"],
|
|
567
|
+
accepted_at=row["accepted_at"],
|
|
568
|
+
completed_at=row["completed_at"],
|
|
569
|
+
last_verified_at=row["last_verified_at"],
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _manifest(*, row: RowMapping) -> ForgetManifest:
|
|
574
|
+
"""Validate the immutable JSON materialization from one progress row."""
|
|
575
|
+
return ForgetManifest.model_validate_json(
|
|
576
|
+
json.dumps(row["manifest"], ensure_ascii=False, separators=(",", ":"))
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _require_same_manifest(*, row: RowMapping, manifest: ForgetManifest) -> None:
|
|
581
|
+
"""Reject local identity reuse with bytes unlike the portable append."""
|
|
582
|
+
stored = _manifest(row=row)
|
|
583
|
+
if stored.canonical_bytes() != manifest.canonical_bytes():
|
|
584
|
+
raise ForgetManifestConflictError(
|
|
585
|
+
f"forget_id {manifest.forget_id} has different local manifest bytes"
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _uuid_values(
|
|
590
|
+
*, connection: Connection, statement: TextClause, deployment_id: UUID, doc_id: UUID
|
|
591
|
+
) -> tuple[UUID, ...]:
|
|
592
|
+
"""Return one query's distinct UUID column in canonical lexical order."""
|
|
593
|
+
values = connection.execute(
|
|
594
|
+
statement, {"deployment_id": deployment_id, "doc_id": doc_id}
|
|
595
|
+
).scalars()
|
|
596
|
+
return tuple(sorted({UUID(str(value)) for value in values}, key=str))
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def _text_values(
|
|
600
|
+
*, connection: Connection, statement: TextClause, deployment_id: UUID, doc_id: UUID
|
|
601
|
+
) -> tuple[str, ...]:
|
|
602
|
+
"""Return one query's non-null text column sorted and duplicate-free."""
|
|
603
|
+
values = connection.execute(
|
|
604
|
+
statement, {"deployment_id": deployment_id, "doc_id": doc_id}
|
|
605
|
+
).scalars()
|
|
606
|
+
return tuple(sorted({str(value) for value in values if value is not None}))
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
_LOCK_DEPLOYMENT = text(
|
|
610
|
+
"SELECT pg_advisory_xact_lock(hashtextextended('hard-forget:' ||"
|
|
611
|
+
" CAST(:deployment_id AS text), 0))"
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
_SELECT_DOCUMENT = text(
|
|
615
|
+
"""
|
|
616
|
+
SELECT doc_id, source_kind, source_ref
|
|
617
|
+
FROM documents
|
|
618
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
619
|
+
"""
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
_INSERT_PREPARING = text(
|
|
623
|
+
"""
|
|
624
|
+
INSERT INTO forget_manifests (
|
|
625
|
+
forget_id, deployment_id, doc_id, schema_version, status
|
|
626
|
+
) VALUES (:forget_id, :deployment_id, :doc_id, 1, 'preparing')
|
|
627
|
+
RETURNING forget_id, deployment_id, doc_id, manifest_hash, manifest,
|
|
628
|
+
status, prepared_at, accepted_at, completed_at, last_verified_at
|
|
629
|
+
"""
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
_SELECT_BY_ID = text(
|
|
633
|
+
"""
|
|
634
|
+
SELECT forget_id, deployment_id, doc_id, manifest_hash, manifest,
|
|
635
|
+
status, prepared_at, accepted_at, completed_at, last_verified_at
|
|
636
|
+
FROM forget_manifests
|
|
637
|
+
WHERE deployment_id = :deployment_id AND forget_id = :forget_id
|
|
638
|
+
"""
|
|
639
|
+
)
|
|
640
|
+
|
|
641
|
+
_SELECT_BY_ID_FOR_UPDATE = text(
|
|
642
|
+
"""
|
|
643
|
+
SELECT forget_id, deployment_id, doc_id, manifest_hash, manifest,
|
|
644
|
+
status, prepared_at, accepted_at, completed_at, last_verified_at
|
|
645
|
+
FROM forget_manifests
|
|
646
|
+
WHERE deployment_id = :deployment_id AND forget_id = :forget_id
|
|
647
|
+
FOR UPDATE
|
|
648
|
+
"""
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
_SELECT_BY_DOC = text(
|
|
652
|
+
"""
|
|
653
|
+
SELECT forget_id, deployment_id, doc_id, manifest_hash, manifest,
|
|
654
|
+
status, prepared_at, accepted_at, completed_at, last_verified_at
|
|
655
|
+
FROM forget_manifests
|
|
656
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
657
|
+
"""
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
_SELECT_PREPARING = text(
|
|
661
|
+
"""
|
|
662
|
+
SELECT forget_id, deployment_id, doc_id, manifest_hash, manifest,
|
|
663
|
+
status, prepared_at, accepted_at, completed_at, last_verified_at
|
|
664
|
+
FROM forget_manifests
|
|
665
|
+
WHERE deployment_id = :deployment_id AND status = 'preparing'
|
|
666
|
+
"""
|
|
667
|
+
)
|
|
668
|
+
|
|
669
|
+
_INSERT_PORTABLE = text(
|
|
670
|
+
"""
|
|
671
|
+
INSERT INTO forget_manifests (
|
|
672
|
+
forget_id, deployment_id, doc_id, schema_version, manifest_hash, manifest,
|
|
673
|
+
source_identity_hash, content_hashes, status, accepted_at
|
|
674
|
+
) VALUES (
|
|
675
|
+
:forget_id, :deployment_id, :doc_id, :schema_version, :manifest_hash,
|
|
676
|
+
CAST(:manifest AS jsonb), :source_identity_hash, :content_hashes,
|
|
677
|
+
'accepted', now()
|
|
678
|
+
)
|
|
679
|
+
"""
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
_MATERIALIZE_PREPARING = text(
|
|
683
|
+
"""
|
|
684
|
+
UPDATE forget_manifests
|
|
685
|
+
SET schema_version = :schema_version,
|
|
686
|
+
manifest_hash = :manifest_hash,
|
|
687
|
+
manifest = CAST(:manifest AS jsonb),
|
|
688
|
+
source_identity_hash = :source_identity_hash,
|
|
689
|
+
content_hashes = :content_hashes,
|
|
690
|
+
status = 'accepted',
|
|
691
|
+
accepted_at = COALESCE(accepted_at, now())
|
|
692
|
+
WHERE deployment_id = :deployment_id
|
|
693
|
+
AND forget_id = :forget_id
|
|
694
|
+
AND doc_id = :doc_id
|
|
695
|
+
AND status = 'preparing'
|
|
696
|
+
AND manifest IS NULL
|
|
697
|
+
"""
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
_ORDINARY_WORK_DRAINED = text(
|
|
701
|
+
"""
|
|
702
|
+
SELECT NOT EXISTS (
|
|
703
|
+
SELECT 1
|
|
704
|
+
FROM processing_state
|
|
705
|
+
WHERE deployment_id = :deployment_id
|
|
706
|
+
AND status = 'running'
|
|
707
|
+
AND stage <> 'hard_forget'
|
|
708
|
+
)
|
|
709
|
+
"""
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
_CONTENT_HASHES = text(
|
|
713
|
+
"""
|
|
714
|
+
SELECT DISTINCT content_hash
|
|
715
|
+
FROM document_versions
|
|
716
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
717
|
+
ORDER BY content_hash
|
|
718
|
+
"""
|
|
719
|
+
)
|
|
720
|
+
|
|
721
|
+
_CHUNK_IDS = text(
|
|
722
|
+
"""
|
|
723
|
+
SELECT DISTINCT chunk_id
|
|
724
|
+
FROM chunks
|
|
725
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
726
|
+
ORDER BY chunk_id
|
|
727
|
+
"""
|
|
728
|
+
)
|
|
729
|
+
|
|
730
|
+
_CLAIM_IDS = text(
|
|
731
|
+
"""
|
|
732
|
+
SELECT DISTINCT claim_id
|
|
733
|
+
FROM claims
|
|
734
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
735
|
+
ORDER BY claim_id
|
|
736
|
+
"""
|
|
737
|
+
)
|
|
738
|
+
|
|
739
|
+
_MENTION_IDS = text(
|
|
740
|
+
"""
|
|
741
|
+
SELECT DISTINCT mention_id
|
|
742
|
+
FROM mentions
|
|
743
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
744
|
+
ORDER BY mention_id
|
|
745
|
+
"""
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
_RESOLVED_ENTITY_IDS = text(
|
|
749
|
+
"""
|
|
750
|
+
SELECT DISTINCT decision.entity_id
|
|
751
|
+
FROM resolution_decisions decision
|
|
752
|
+
JOIN mentions mention ON mention.mention_id = decision.mention_id
|
|
753
|
+
WHERE decision.deployment_id = :deployment_id
|
|
754
|
+
AND mention.deployment_id = :deployment_id
|
|
755
|
+
AND mention.doc_id = :doc_id
|
|
756
|
+
ORDER BY decision.entity_id
|
|
757
|
+
"""
|
|
758
|
+
)
|
|
759
|
+
|
|
760
|
+
_EXCLUSIVE_FACT_IDS = text(
|
|
761
|
+
"""
|
|
762
|
+
SELECT fact_id
|
|
763
|
+
FROM (
|
|
764
|
+
SELECT relation_id AS fact_id
|
|
765
|
+
FROM relations relation
|
|
766
|
+
WHERE relation.deployment_id = :deployment_id
|
|
767
|
+
AND EXISTS (
|
|
768
|
+
SELECT 1 FROM relation_evidence target
|
|
769
|
+
WHERE target.deployment_id = :deployment_id
|
|
770
|
+
AND target.relation_id = relation.relation_id
|
|
771
|
+
AND target.doc_id = :doc_id
|
|
772
|
+
)
|
|
773
|
+
AND NOT EXISTS (
|
|
774
|
+
SELECT 1
|
|
775
|
+
FROM relation_evidence other
|
|
776
|
+
JOIN claims claim
|
|
777
|
+
ON claim.deployment_id = other.deployment_id
|
|
778
|
+
AND claim.claim_id = other.claim_id
|
|
779
|
+
JOIN documents document
|
|
780
|
+
ON document.deployment_id = other.deployment_id
|
|
781
|
+
AND document.doc_id = other.doc_id
|
|
782
|
+
WHERE other.deployment_id = :deployment_id
|
|
783
|
+
AND other.relation_id = relation.relation_id
|
|
784
|
+
AND other.doc_id <> :doc_id
|
|
785
|
+
AND claim.is_current_testimony
|
|
786
|
+
AND document.deleted_at IS NULL
|
|
787
|
+
)
|
|
788
|
+
UNION
|
|
789
|
+
SELECT observation_id AS fact_id
|
|
790
|
+
FROM observations observation
|
|
791
|
+
WHERE observation.deployment_id = :deployment_id
|
|
792
|
+
AND EXISTS (
|
|
793
|
+
SELECT 1 FROM observation_evidence target
|
|
794
|
+
WHERE target.deployment_id = :deployment_id
|
|
795
|
+
AND target.observation_id = observation.observation_id
|
|
796
|
+
AND target.doc_id = :doc_id
|
|
797
|
+
)
|
|
798
|
+
AND NOT EXISTS (
|
|
799
|
+
SELECT 1
|
|
800
|
+
FROM observation_evidence other
|
|
801
|
+
JOIN claims claim
|
|
802
|
+
ON claim.deployment_id = other.deployment_id
|
|
803
|
+
AND claim.claim_id = other.claim_id
|
|
804
|
+
JOIN documents document
|
|
805
|
+
ON document.deployment_id = other.deployment_id
|
|
806
|
+
AND document.doc_id = other.doc_id
|
|
807
|
+
WHERE other.deployment_id = :deployment_id
|
|
808
|
+
AND other.observation_id = observation.observation_id
|
|
809
|
+
AND other.doc_id <> :doc_id
|
|
810
|
+
AND claim.is_current_testimony
|
|
811
|
+
AND document.deleted_at IS NULL
|
|
812
|
+
)
|
|
813
|
+
) exclusive
|
|
814
|
+
ORDER BY fact_id
|
|
815
|
+
"""
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
_EXCLUSIVE_ENTITY_IDS = text(
|
|
819
|
+
"""
|
|
820
|
+
SELECT DISTINCT target.entity_id
|
|
821
|
+
FROM resolution_decisions target
|
|
822
|
+
JOIN mentions mention ON mention.mention_id = target.mention_id
|
|
823
|
+
WHERE target.deployment_id = :deployment_id
|
|
824
|
+
AND mention.deployment_id = :deployment_id
|
|
825
|
+
AND mention.doc_id = :doc_id
|
|
826
|
+
AND target.superseded_by IS NULL
|
|
827
|
+
AND NOT EXISTS (
|
|
828
|
+
SELECT 1
|
|
829
|
+
FROM resolution_decisions other
|
|
830
|
+
JOIN mentions other_mention ON other_mention.mention_id = other.mention_id
|
|
831
|
+
JOIN claims other_claim
|
|
832
|
+
ON other_claim.deployment_id = other_mention.deployment_id
|
|
833
|
+
AND other_claim.claim_id = other_mention.claim_id
|
|
834
|
+
JOIN documents other_document
|
|
835
|
+
ON other_document.deployment_id = other_mention.deployment_id
|
|
836
|
+
AND other_document.doc_id = other_mention.doc_id
|
|
837
|
+
WHERE other.deployment_id = :deployment_id
|
|
838
|
+
AND other.entity_id = target.entity_id
|
|
839
|
+
AND other.superseded_by IS NULL
|
|
840
|
+
AND other_mention.doc_id <> :doc_id
|
|
841
|
+
AND other_claim.is_current_testimony
|
|
842
|
+
AND other_document.deleted_at IS NULL
|
|
843
|
+
)
|
|
844
|
+
ORDER BY target.entity_id
|
|
845
|
+
"""
|
|
846
|
+
)
|
|
847
|
+
|
|
848
|
+
_K_ARTIFACT_IDS = text(
|
|
849
|
+
"""
|
|
850
|
+
SELECT artifact_id
|
|
851
|
+
FROM knowledge_artifacts
|
|
852
|
+
WHERE deployment_id = :deployment_id
|
|
853
|
+
ORDER BY artifact_id::text
|
|
854
|
+
"""
|
|
855
|
+
)
|
|
856
|
+
|
|
857
|
+
_BLOCKING_K_PATHS = text(
|
|
858
|
+
"""
|
|
859
|
+
WITH target_relations AS (
|
|
860
|
+
SELECT DISTINCT relation_id
|
|
861
|
+
FROM relation_evidence
|
|
862
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
863
|
+
), affected AS (
|
|
864
|
+
SELECT DISTINCT artifact_id
|
|
865
|
+
FROM knowledge_artifact_evidence
|
|
866
|
+
WHERE deployment_id = :deployment_id
|
|
867
|
+
AND (
|
|
868
|
+
doc_id = :doc_id
|
|
869
|
+
OR claim_lineage_id = :doc_id
|
|
870
|
+
OR relation_id IN (SELECT relation_id FROM target_relations)
|
|
871
|
+
)
|
|
872
|
+
)
|
|
873
|
+
SELECT path
|
|
874
|
+
FROM (
|
|
875
|
+
SELECT artifact.git_path AS path
|
|
876
|
+
FROM knowledge_artifacts artifact
|
|
877
|
+
WHERE artifact.deployment_id = :deployment_id
|
|
878
|
+
AND artifact.artifact_id IN (SELECT artifact_id FROM affected)
|
|
879
|
+
AND artifact.page_kind = 'authored'
|
|
880
|
+
UNION
|
|
881
|
+
SELECT artifact.curation_path AS path
|
|
882
|
+
FROM knowledge_artifacts artifact
|
|
883
|
+
WHERE artifact.deployment_id = :deployment_id
|
|
884
|
+
AND artifact.artifact_id IN (SELECT artifact_id FROM affected)
|
|
885
|
+
AND artifact.page_kind = 'compiled'
|
|
886
|
+
AND artifact.curation_path IS NOT NULL
|
|
887
|
+
) blocking
|
|
888
|
+
ORDER BY path
|
|
889
|
+
"""
|
|
890
|
+
)
|
|
891
|
+
|
|
892
|
+
_K_PATHS_FOR_ARTIFACTS = text(
|
|
893
|
+
"""
|
|
894
|
+
SELECT path
|
|
895
|
+
FROM (
|
|
896
|
+
SELECT git_path AS path
|
|
897
|
+
FROM knowledge_artifacts
|
|
898
|
+
WHERE deployment_id = :deployment_id
|
|
899
|
+
AND artifact_id = ANY(:artifact_ids)
|
|
900
|
+
UNION
|
|
901
|
+
SELECT curation_path AS path
|
|
902
|
+
FROM knowledge_artifacts
|
|
903
|
+
WHERE deployment_id = :deployment_id
|
|
904
|
+
AND artifact_id = ANY(:artifact_ids)
|
|
905
|
+
AND curation_path IS NOT NULL
|
|
906
|
+
) paths
|
|
907
|
+
ORDER BY path
|
|
908
|
+
"""
|
|
909
|
+
)
|
|
910
|
+
|
|
911
|
+
_OBJECT_KEYS = text(
|
|
912
|
+
"""
|
|
913
|
+
WITH target_artifacts AS (
|
|
914
|
+
SELECT artifact_id
|
|
915
|
+
FROM knowledge_artifacts
|
|
916
|
+
WHERE deployment_id = :deployment_id
|
|
917
|
+
), keys AS (
|
|
918
|
+
SELECT object.raw_uri AS object_key
|
|
919
|
+
FROM document_versions version
|
|
920
|
+
JOIN content_objects object
|
|
921
|
+
ON object.deployment_id = version.deployment_id
|
|
922
|
+
AND object.content_hash = version.content_hash
|
|
923
|
+
WHERE version.deployment_id = :deployment_id
|
|
924
|
+
AND version.doc_id = :doc_id
|
|
925
|
+
AND NOT EXISTS (
|
|
926
|
+
SELECT 1
|
|
927
|
+
FROM document_versions other_version
|
|
928
|
+
JOIN documents other_document
|
|
929
|
+
ON other_document.deployment_id = other_version.deployment_id
|
|
930
|
+
AND other_document.doc_id = other_version.doc_id
|
|
931
|
+
WHERE other_version.deployment_id = version.deployment_id
|
|
932
|
+
AND other_version.content_hash = version.content_hash
|
|
933
|
+
AND other_version.doc_id <> :doc_id
|
|
934
|
+
AND other_version.deleted_at IS NULL
|
|
935
|
+
AND other_document.deleted_at IS NULL
|
|
936
|
+
)
|
|
937
|
+
UNION ALL
|
|
938
|
+
SELECT uri.object_key
|
|
939
|
+
FROM document_representations representation
|
|
940
|
+
JOIN document_versions version
|
|
941
|
+
ON version.deployment_id = representation.deployment_id
|
|
942
|
+
AND version.version_id = representation.version_id
|
|
943
|
+
CROSS JOIN LATERAL (
|
|
944
|
+
VALUES (representation.markdown_uri), (representation.pageindex_uri),
|
|
945
|
+
(representation.conversion_uri), (representation.blocks_uri),
|
|
946
|
+
(representation.meta_uri)
|
|
947
|
+
) uri(object_key)
|
|
948
|
+
WHERE version.deployment_id = :deployment_id AND version.doc_id = :doc_id
|
|
949
|
+
UNION ALL
|
|
950
|
+
SELECT compilation.session_transcript_uri
|
|
951
|
+
FROM knowledge_compilations compilation
|
|
952
|
+
WHERE compilation.deployment_id = :deployment_id
|
|
953
|
+
AND compilation.artifact_id IN (SELECT artifact_id FROM target_artifacts)
|
|
954
|
+
UNION ALL
|
|
955
|
+
SELECT run.session_transcript_uri
|
|
956
|
+
FROM knowledge_plan_runs run
|
|
957
|
+
WHERE run.deployment_id = :deployment_id
|
|
958
|
+
)
|
|
959
|
+
SELECT DISTINCT object_key
|
|
960
|
+
FROM keys
|
|
961
|
+
WHERE object_key IS NOT NULL
|
|
962
|
+
ORDER BY object_key
|
|
963
|
+
"""
|
|
964
|
+
)
|
|
965
|
+
|
|
966
|
+
_PROJECTION_PREFIXES = text(
|
|
967
|
+
"""
|
|
968
|
+
SELECT DISTINCT gcs_uri
|
|
969
|
+
FROM projection_snapshots
|
|
970
|
+
WHERE deployment_id = :deployment_id
|
|
971
|
+
AND plane IN ('P2_graph', 'P3_corpusfs')
|
|
972
|
+
ORDER BY gcs_uri
|
|
973
|
+
"""
|
|
974
|
+
)
|
|
975
|
+
|
|
976
|
+
_STORE_MANIFEST = text(
|
|
977
|
+
"""
|
|
978
|
+
UPDATE forget_manifests
|
|
979
|
+
SET schema_version = :schema_version,
|
|
980
|
+
manifest_hash = :manifest_hash,
|
|
981
|
+
manifest = CAST(:manifest AS jsonb),
|
|
982
|
+
source_identity_hash = :source_identity_hash,
|
|
983
|
+
content_hashes = :content_hashes
|
|
984
|
+
WHERE deployment_id = :deployment_id
|
|
985
|
+
AND forget_id = :forget_id
|
|
986
|
+
AND status = 'preparing'
|
|
987
|
+
AND manifest IS NULL
|
|
988
|
+
"""
|
|
989
|
+
)
|
|
990
|
+
|
|
991
|
+
_DELETE_UNSTORED_PREPARING = text(
|
|
992
|
+
"""
|
|
993
|
+
DELETE FROM forget_manifests
|
|
994
|
+
WHERE deployment_id = :deployment_id
|
|
995
|
+
AND forget_id = :forget_id
|
|
996
|
+
AND status = 'preparing'
|
|
997
|
+
AND manifest IS NULL
|
|
998
|
+
"""
|
|
999
|
+
)
|
|
1000
|
+
|
|
1001
|
+
_REGISTER_FORGETTER = text(
|
|
1002
|
+
"""
|
|
1003
|
+
INSERT INTO pipeline_component_versions (
|
|
1004
|
+
deployment_id, component, version, params, notes
|
|
1005
|
+
) VALUES (
|
|
1006
|
+
:deployment_id, 'forgetter', :version, '{"schema_version": 1}'::jsonb,
|
|
1007
|
+
'D74 deterministic hard-forget coordinator'
|
|
1008
|
+
)
|
|
1009
|
+
ON CONFLICT (deployment_id, component, version) DO NOTHING
|
|
1010
|
+
"""
|
|
1011
|
+
)
|
|
1012
|
+
|
|
1013
|
+
_FORGETTER_DEFINITION_MATCHES = text(
|
|
1014
|
+
"""
|
|
1015
|
+
SELECT params = '{"schema_version": 1}'::jsonb
|
|
1016
|
+
AND model_name IS NULL
|
|
1017
|
+
AND prompt_hash IS NULL
|
|
1018
|
+
AND embedding_dim IS NULL
|
|
1019
|
+
AND notes = 'D74 deterministic hard-forget coordinator'
|
|
1020
|
+
FROM pipeline_component_versions
|
|
1021
|
+
WHERE deployment_id = :deployment_id
|
|
1022
|
+
AND component = 'forgetter'
|
|
1023
|
+
AND version = :version
|
|
1024
|
+
"""
|
|
1025
|
+
)
|
|
1026
|
+
|
|
1027
|
+
_MARK_ACCEPTED = text(
|
|
1028
|
+
"""
|
|
1029
|
+
UPDATE forget_manifests
|
|
1030
|
+
SET status = CASE WHEN status = 'complete' THEN status ELSE 'accepted' END,
|
|
1031
|
+
accepted_at = COALESCE(accepted_at, now())
|
|
1032
|
+
WHERE deployment_id = :deployment_id AND forget_id = :forget_id
|
|
1033
|
+
"""
|
|
1034
|
+
)
|
|
1035
|
+
|
|
1036
|
+
_POSTGRES_SCRUB = (
|
|
1037
|
+
text(
|
|
1038
|
+
"""
|
|
1039
|
+
UPDATE documents
|
|
1040
|
+
SET current_version_id = NULL,
|
|
1041
|
+
document_entity_id = NULL,
|
|
1042
|
+
source_ref = NULL,
|
|
1043
|
+
source_uri = NULL,
|
|
1044
|
+
title = NULL,
|
|
1045
|
+
deleted_at = COALESCE(deleted_at, now())
|
|
1046
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1047
|
+
"""
|
|
1048
|
+
),
|
|
1049
|
+
text(
|
|
1050
|
+
"""
|
|
1051
|
+
UPDATE document_versions
|
|
1052
|
+
SET current_representation_id = NULL,
|
|
1053
|
+
source_version_ref = NULL,
|
|
1054
|
+
source_modified_at = NULL,
|
|
1055
|
+
published_at = NULL,
|
|
1056
|
+
language = NULL,
|
|
1057
|
+
status = 'deleted',
|
|
1058
|
+
error = NULL,
|
|
1059
|
+
superseded_at = NULL,
|
|
1060
|
+
deleted_at = COALESCE(deleted_at, now())
|
|
1061
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1062
|
+
"""
|
|
1063
|
+
),
|
|
1064
|
+
# The eval schema predates D74 and has no lineage-provenance column. A rare
|
|
1065
|
+
# hard-forget therefore drops deployment-scoped free-text eval fixtures
|
|
1066
|
+
# instead of guessing semantically which prose came from the lineage.
|
|
1067
|
+
text(
|
|
1068
|
+
"""
|
|
1069
|
+
DELETE FROM eval_runs WHERE deployment_id = :deployment_id
|
|
1070
|
+
"""
|
|
1071
|
+
),
|
|
1072
|
+
text(
|
|
1073
|
+
"""
|
|
1074
|
+
DELETE FROM canary_cases WHERE deployment_id = :deployment_id
|
|
1075
|
+
"""
|
|
1076
|
+
),
|
|
1077
|
+
text(
|
|
1078
|
+
"""
|
|
1079
|
+
DELETE FROM golden_claim_labels WHERE deployment_id = :deployment_id
|
|
1080
|
+
"""
|
|
1081
|
+
),
|
|
1082
|
+
text(
|
|
1083
|
+
"""
|
|
1084
|
+
DELETE FROM golden_pairs WHERE deployment_id = :deployment_id
|
|
1085
|
+
"""
|
|
1086
|
+
),
|
|
1087
|
+
text(
|
|
1088
|
+
"""
|
|
1089
|
+
DELETE FROM knowledge_plan_decisions decision
|
|
1090
|
+
WHERE decision.deployment_id = :deployment_id
|
|
1091
|
+
"""
|
|
1092
|
+
),
|
|
1093
|
+
text(
|
|
1094
|
+
"""
|
|
1095
|
+
DELETE FROM knowledge_plan_runs run
|
|
1096
|
+
WHERE run.deployment_id = :deployment_id
|
|
1097
|
+
"""
|
|
1098
|
+
),
|
|
1099
|
+
text(
|
|
1100
|
+
"""
|
|
1101
|
+
DELETE FROM knowledge_refresh_queue refresh
|
|
1102
|
+
WHERE refresh.deployment_id = :deployment_id
|
|
1103
|
+
AND (
|
|
1104
|
+
refresh.artifact_id = ANY(:k_artifact_ids)
|
|
1105
|
+
OR refresh.scope_id IN (
|
|
1106
|
+
SELECT artifact.scope_id
|
|
1107
|
+
FROM knowledge_artifacts artifact
|
|
1108
|
+
WHERE artifact.deployment_id = :deployment_id
|
|
1109
|
+
AND artifact.artifact_id = ANY(:k_artifact_ids)
|
|
1110
|
+
AND artifact.scope_id IS NOT NULL
|
|
1111
|
+
)
|
|
1112
|
+
OR refresh.payload::text LIKE '%' || CAST(:doc_id AS text) || '%'
|
|
1113
|
+
OR EXISTS (
|
|
1114
|
+
SELECT 1
|
|
1115
|
+
FROM unnest(
|
|
1116
|
+
CAST(:claim_ids AS uuid[]) || CAST(:mention_ids AS uuid[])
|
|
1117
|
+
|| CAST(:fact_ids AS uuid[])
|
|
1118
|
+
|| CAST(:resolved_entity_ids AS uuid[])
|
|
1119
|
+
|| CAST(:k_artifact_ids AS uuid[])
|
|
1120
|
+
) AS ids(value)
|
|
1121
|
+
WHERE refresh.payload::text LIKE '%'
|
|
1122
|
+
|| CAST(ids.value AS text) || '%'
|
|
1123
|
+
)
|
|
1124
|
+
)
|
|
1125
|
+
"""
|
|
1126
|
+
),
|
|
1127
|
+
text(
|
|
1128
|
+
"""
|
|
1129
|
+
DELETE FROM knowledge_dispatches dispatch
|
|
1130
|
+
WHERE dispatch.deployment_id = :deployment_id
|
|
1131
|
+
AND (
|
|
1132
|
+
dispatch.payload::text LIKE '%' || CAST(:doc_id AS text) || '%'
|
|
1133
|
+
OR EXISTS (
|
|
1134
|
+
SELECT 1
|
|
1135
|
+
FROM unnest(
|
|
1136
|
+
CAST(:claim_ids AS uuid[]) || CAST(:mention_ids AS uuid[])
|
|
1137
|
+
|| CAST(:fact_ids AS uuid[])
|
|
1138
|
+
|| CAST(:resolved_entity_ids AS uuid[])
|
|
1139
|
+
|| CAST(:k_artifact_ids AS uuid[])
|
|
1140
|
+
) AS ids(value)
|
|
1141
|
+
WHERE dispatch.payload::text LIKE '%'
|
|
1142
|
+
|| CAST(ids.value AS text) || '%'
|
|
1143
|
+
)
|
|
1144
|
+
)
|
|
1145
|
+
"""
|
|
1146
|
+
),
|
|
1147
|
+
text(
|
|
1148
|
+
"""
|
|
1149
|
+
DELETE FROM knowledge_artifact_evidence
|
|
1150
|
+
WHERE deployment_id = :deployment_id
|
|
1151
|
+
AND (doc_id = :doc_id OR claim_lineage_id = :doc_id
|
|
1152
|
+
OR relation_id = ANY(:fact_ids))
|
|
1153
|
+
"""
|
|
1154
|
+
),
|
|
1155
|
+
text(
|
|
1156
|
+
"""
|
|
1157
|
+
DELETE FROM knowledge_compilations
|
|
1158
|
+
WHERE deployment_id = :deployment_id
|
|
1159
|
+
AND artifact_id = ANY(:k_artifact_ids)
|
|
1160
|
+
"""
|
|
1161
|
+
),
|
|
1162
|
+
text(
|
|
1163
|
+
"""
|
|
1164
|
+
UPDATE knowledge_artifacts
|
|
1165
|
+
SET page_summary = NULL,
|
|
1166
|
+
content_hash = NULL,
|
|
1167
|
+
inputs_hash = NULL,
|
|
1168
|
+
last_compiled_at = NULL,
|
|
1169
|
+
status = CASE WHEN page_kind = 'compiled' THEN 'stale' ELSE status END
|
|
1170
|
+
WHERE deployment_id = :deployment_id
|
|
1171
|
+
AND artifact_id = ANY(:k_artifact_ids)
|
|
1172
|
+
"""
|
|
1173
|
+
),
|
|
1174
|
+
text(
|
|
1175
|
+
"""
|
|
1176
|
+
UPDATE relation_adjudications
|
|
1177
|
+
SET triggering_claim_id = NULL, features = NULL
|
|
1178
|
+
WHERE deployment_id = :deployment_id
|
|
1179
|
+
AND triggering_claim_id = ANY(:claim_ids)
|
|
1180
|
+
"""
|
|
1181
|
+
),
|
|
1182
|
+
text(
|
|
1183
|
+
"""
|
|
1184
|
+
UPDATE observation_adjudications
|
|
1185
|
+
SET triggering_claim_id = NULL, features = NULL
|
|
1186
|
+
WHERE deployment_id = :deployment_id
|
|
1187
|
+
AND triggering_claim_id = ANY(:claim_ids)
|
|
1188
|
+
"""
|
|
1189
|
+
),
|
|
1190
|
+
text(
|
|
1191
|
+
"""
|
|
1192
|
+
UPDATE relation_adjudications
|
|
1193
|
+
SET superseded_by = NULL
|
|
1194
|
+
WHERE superseded_by IN (
|
|
1195
|
+
SELECT adjudication_id FROM relation_adjudications
|
|
1196
|
+
WHERE deployment_id = :deployment_id
|
|
1197
|
+
AND (relation_id = ANY(:fact_ids)
|
|
1198
|
+
OR related_relation_id = ANY(:fact_ids))
|
|
1199
|
+
)
|
|
1200
|
+
"""
|
|
1201
|
+
),
|
|
1202
|
+
text(
|
|
1203
|
+
"""
|
|
1204
|
+
DELETE FROM relation_adjudications
|
|
1205
|
+
WHERE deployment_id = :deployment_id
|
|
1206
|
+
AND (relation_id = ANY(:fact_ids)
|
|
1207
|
+
OR related_relation_id = ANY(:fact_ids))
|
|
1208
|
+
"""
|
|
1209
|
+
),
|
|
1210
|
+
text(
|
|
1211
|
+
"""
|
|
1212
|
+
UPDATE observation_adjudications
|
|
1213
|
+
SET superseded_by = NULL
|
|
1214
|
+
WHERE superseded_by IN (
|
|
1215
|
+
SELECT adjudication_id FROM observation_adjudications
|
|
1216
|
+
WHERE deployment_id = :deployment_id
|
|
1217
|
+
AND (observation_id = ANY(:fact_ids)
|
|
1218
|
+
OR related_observation_id = ANY(:fact_ids))
|
|
1219
|
+
)
|
|
1220
|
+
"""
|
|
1221
|
+
),
|
|
1222
|
+
text(
|
|
1223
|
+
"""
|
|
1224
|
+
DELETE FROM observation_adjudications
|
|
1225
|
+
WHERE deployment_id = :deployment_id
|
|
1226
|
+
AND (observation_id = ANY(:fact_ids)
|
|
1227
|
+
OR related_observation_id = ANY(:fact_ids))
|
|
1228
|
+
"""
|
|
1229
|
+
),
|
|
1230
|
+
text(
|
|
1231
|
+
"""
|
|
1232
|
+
DELETE FROM relation_evidence
|
|
1233
|
+
WHERE deployment_id = :deployment_id
|
|
1234
|
+
AND (doc_id = :doc_id OR relation_id = ANY(:fact_ids))
|
|
1235
|
+
"""
|
|
1236
|
+
),
|
|
1237
|
+
text(
|
|
1238
|
+
"""
|
|
1239
|
+
DELETE FROM observation_evidence
|
|
1240
|
+
WHERE deployment_id = :deployment_id
|
|
1241
|
+
AND (doc_id = :doc_id OR observation_id = ANY(:fact_ids))
|
|
1242
|
+
"""
|
|
1243
|
+
),
|
|
1244
|
+
text(
|
|
1245
|
+
"""
|
|
1246
|
+
DELETE FROM relations
|
|
1247
|
+
WHERE deployment_id = :deployment_id AND relation_id = ANY(:fact_ids)
|
|
1248
|
+
"""
|
|
1249
|
+
),
|
|
1250
|
+
text(
|
|
1251
|
+
"""
|
|
1252
|
+
DELETE FROM observations
|
|
1253
|
+
WHERE deployment_id = :deployment_id AND observation_id = ANY(:fact_ids)
|
|
1254
|
+
"""
|
|
1255
|
+
),
|
|
1256
|
+
text(
|
|
1257
|
+
"""
|
|
1258
|
+
DELETE FROM aliases
|
|
1259
|
+
WHERE deployment_id = :deployment_id AND entity_id = ANY(:entity_ids)
|
|
1260
|
+
"""
|
|
1261
|
+
),
|
|
1262
|
+
# Generic-identifier guards are derived from source lemmas but carry no
|
|
1263
|
+
# lineage provenance. Rebuilding this small cache is safer than retaining
|
|
1264
|
+
# an unprovable surface form after an irreversible forget.
|
|
1265
|
+
text(
|
|
1266
|
+
"""
|
|
1267
|
+
DELETE FROM generic_identifier_guard
|
|
1268
|
+
WHERE deployment_id = :deployment_id
|
|
1269
|
+
"""
|
|
1270
|
+
),
|
|
1271
|
+
text(
|
|
1272
|
+
"""
|
|
1273
|
+
UPDATE merge_events
|
|
1274
|
+
SET trigger_lemmas = '{}',
|
|
1275
|
+
evidence = NULL,
|
|
1276
|
+
pre_merge_membership_snapshot = '{}'::jsonb
|
|
1277
|
+
WHERE deployment_id = :deployment_id
|
|
1278
|
+
AND (
|
|
1279
|
+
survivor_id = ANY(:resolved_entity_ids)
|
|
1280
|
+
OR absorbed_id = ANY(:resolved_entity_ids)
|
|
1281
|
+
OR EXISTS (
|
|
1282
|
+
SELECT 1 FROM unnest(CAST(:mention_ids AS uuid[])) AS ids(value)
|
|
1283
|
+
WHERE pre_merge_membership_snapshot::text LIKE '%'
|
|
1284
|
+
|| CAST(ids.value AS text) || '%'
|
|
1285
|
+
)
|
|
1286
|
+
)
|
|
1287
|
+
"""
|
|
1288
|
+
),
|
|
1289
|
+
text(
|
|
1290
|
+
"""
|
|
1291
|
+
UPDATE resolution_exclusions
|
|
1292
|
+
SET reason = NULL
|
|
1293
|
+
WHERE deployment_id = :deployment_id
|
|
1294
|
+
AND (
|
|
1295
|
+
entity_id_low = ANY(:resolved_entity_ids)
|
|
1296
|
+
OR entity_id_high = ANY(:resolved_entity_ids)
|
|
1297
|
+
)
|
|
1298
|
+
"""
|
|
1299
|
+
),
|
|
1300
|
+
text(
|
|
1301
|
+
"""
|
|
1302
|
+
UPDATE entities
|
|
1303
|
+
SET canonical_name = '',
|
|
1304
|
+
normalized_name = '',
|
|
1305
|
+
status = 'retired',
|
|
1306
|
+
merged_into = NULL,
|
|
1307
|
+
type_confidence = NULL,
|
|
1308
|
+
profile_summary = NULL,
|
|
1309
|
+
profile_embedding_ref = NULL,
|
|
1310
|
+
mention_count = 0,
|
|
1311
|
+
graph_degree = 0,
|
|
1312
|
+
updated_at = now()
|
|
1313
|
+
WHERE deployment_id = :deployment_id AND entity_id = ANY(:entity_ids)
|
|
1314
|
+
"""
|
|
1315
|
+
),
|
|
1316
|
+
text(
|
|
1317
|
+
"""
|
|
1318
|
+
DELETE FROM resolution_decisions
|
|
1319
|
+
WHERE deployment_id = :deployment_id
|
|
1320
|
+
AND mention_id = ANY(:mention_ids)
|
|
1321
|
+
"""
|
|
1322
|
+
),
|
|
1323
|
+
text(
|
|
1324
|
+
"""
|
|
1325
|
+
DELETE FROM mentions
|
|
1326
|
+
WHERE deployment_id = :deployment_id
|
|
1327
|
+
AND (doc_id = :doc_id OR mention_id = ANY(:mention_ids))
|
|
1328
|
+
"""
|
|
1329
|
+
),
|
|
1330
|
+
text(
|
|
1331
|
+
"""
|
|
1332
|
+
DELETE FROM grounding_audits
|
|
1333
|
+
WHERE deployment_id = :deployment_id AND claim_id = ANY(:claim_ids)
|
|
1334
|
+
"""
|
|
1335
|
+
),
|
|
1336
|
+
text(
|
|
1337
|
+
"""
|
|
1338
|
+
DELETE FROM claim_extraction_decisions
|
|
1339
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1340
|
+
"""
|
|
1341
|
+
),
|
|
1342
|
+
text(
|
|
1343
|
+
"""
|
|
1344
|
+
DELETE FROM chunk_claims
|
|
1345
|
+
WHERE deployment_id = :deployment_id
|
|
1346
|
+
AND (chunk_id = ANY(:chunk_ids) OR claim_id = ANY(:claim_ids))
|
|
1347
|
+
"""
|
|
1348
|
+
),
|
|
1349
|
+
text(
|
|
1350
|
+
"""
|
|
1351
|
+
DELETE FROM claims
|
|
1352
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1353
|
+
"""
|
|
1354
|
+
),
|
|
1355
|
+
text(
|
|
1356
|
+
"""
|
|
1357
|
+
DELETE FROM chunks
|
|
1358
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1359
|
+
"""
|
|
1360
|
+
),
|
|
1361
|
+
text(
|
|
1362
|
+
"""
|
|
1363
|
+
DELETE FROM document_sections
|
|
1364
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1365
|
+
"""
|
|
1366
|
+
),
|
|
1367
|
+
text(
|
|
1368
|
+
"""
|
|
1369
|
+
DELETE FROM document_representations
|
|
1370
|
+
WHERE deployment_id = :deployment_id
|
|
1371
|
+
AND version_id IN (
|
|
1372
|
+
SELECT version_id FROM document_versions
|
|
1373
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1374
|
+
)
|
|
1375
|
+
"""
|
|
1376
|
+
),
|
|
1377
|
+
text(
|
|
1378
|
+
"""
|
|
1379
|
+
DELETE FROM document_crossrefs
|
|
1380
|
+
WHERE deployment_id = :deployment_id AND from_doc_id = :doc_id
|
|
1381
|
+
"""
|
|
1382
|
+
),
|
|
1383
|
+
text(
|
|
1384
|
+
"""
|
|
1385
|
+
DELETE FROM document_crossrefs
|
|
1386
|
+
WHERE deployment_id = :deployment_id AND to_doc_id = :doc_id
|
|
1387
|
+
"""
|
|
1388
|
+
),
|
|
1389
|
+
text(
|
|
1390
|
+
"""
|
|
1391
|
+
UPDATE content_objects object
|
|
1392
|
+
SET mime = 'application/x-forgotten',
|
|
1393
|
+
byte_size = NULL,
|
|
1394
|
+
raw_uri = 'forgotten/' || object.content_hash,
|
|
1395
|
+
purged_at = COALESCE(object.purged_at, now())
|
|
1396
|
+
WHERE object.deployment_id = :deployment_id
|
|
1397
|
+
AND object.content_hash = ANY(:content_hashes)
|
|
1398
|
+
AND NOT EXISTS (
|
|
1399
|
+
SELECT 1
|
|
1400
|
+
FROM document_versions other_version
|
|
1401
|
+
JOIN documents other_document
|
|
1402
|
+
ON other_document.deployment_id = other_version.deployment_id
|
|
1403
|
+
AND other_document.doc_id = other_version.doc_id
|
|
1404
|
+
WHERE other_version.deployment_id = object.deployment_id
|
|
1405
|
+
AND other_version.content_hash = object.content_hash
|
|
1406
|
+
AND other_version.doc_id <> :doc_id
|
|
1407
|
+
AND other_version.deleted_at IS NULL
|
|
1408
|
+
AND other_document.deleted_at IS NULL
|
|
1409
|
+
)
|
|
1410
|
+
"""
|
|
1411
|
+
),
|
|
1412
|
+
text(
|
|
1413
|
+
"""
|
|
1414
|
+
UPDATE processing_state
|
|
1415
|
+
SET payload = NULL, last_error = NULL
|
|
1416
|
+
WHERE deployment_id = :deployment_id
|
|
1417
|
+
AND stage <> 'hard_forget'
|
|
1418
|
+
AND (
|
|
1419
|
+
target_id = :doc_id
|
|
1420
|
+
OR target_id = ANY(:chunk_ids)
|
|
1421
|
+
OR target_id = ANY(:claim_ids)
|
|
1422
|
+
OR target_id = ANY(:mention_ids)
|
|
1423
|
+
OR target_id = ANY(:fact_ids)
|
|
1424
|
+
OR target_id = ANY(:entity_ids)
|
|
1425
|
+
OR target_id = ANY(:resolved_entity_ids)
|
|
1426
|
+
OR content_hash = ANY(:content_hashes)
|
|
1427
|
+
OR payload::text LIKE '%' || CAST(:doc_id AS text) || '%'
|
|
1428
|
+
OR EXISTS (
|
|
1429
|
+
SELECT 1
|
|
1430
|
+
FROM unnest(
|
|
1431
|
+
CAST(:chunk_ids AS uuid[]) || CAST(:claim_ids AS uuid[])
|
|
1432
|
+
|| CAST(:mention_ids AS uuid[]) || CAST(:fact_ids AS uuid[])
|
|
1433
|
+
|| CAST(:resolved_entity_ids AS uuid[])
|
|
1434
|
+
|| CAST(:k_artifact_ids AS uuid[])
|
|
1435
|
+
) AS ids(value)
|
|
1436
|
+
WHERE payload::text LIKE '%' || CAST(ids.value AS text) || '%'
|
|
1437
|
+
)
|
|
1438
|
+
)
|
|
1439
|
+
"""
|
|
1440
|
+
),
|
|
1441
|
+
text(
|
|
1442
|
+
"""
|
|
1443
|
+
DELETE FROM review_queue review
|
|
1444
|
+
WHERE review.deployment_id = :deployment_id
|
|
1445
|
+
AND (
|
|
1446
|
+
review.candidate::text LIKE '%' || CAST(:doc_id AS text) || '%'
|
|
1447
|
+
OR EXISTS (
|
|
1448
|
+
SELECT 1
|
|
1449
|
+
FROM unnest(
|
|
1450
|
+
CAST(:chunk_ids AS uuid[]) || CAST(:claim_ids AS uuid[])
|
|
1451
|
+
|| CAST(:mention_ids AS uuid[]) || CAST(:fact_ids AS uuid[])
|
|
1452
|
+
|| CAST(:resolved_entity_ids AS uuid[])
|
|
1453
|
+
|| CAST(:k_artifact_ids AS uuid[])
|
|
1454
|
+
) AS ids(value)
|
|
1455
|
+
WHERE review.candidate::text LIKE '%'
|
|
1456
|
+
|| CAST(ids.value AS text) || '%'
|
|
1457
|
+
)
|
|
1458
|
+
)
|
|
1459
|
+
"""
|
|
1460
|
+
),
|
|
1461
|
+
)
|
|
1462
|
+
|
|
1463
|
+
_VERIFY_POSTGRES_SCRUB = text(
|
|
1464
|
+
"""
|
|
1465
|
+
SELECT count(*)
|
|
1466
|
+
FROM (
|
|
1467
|
+
SELECT 1 FROM documents
|
|
1468
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1469
|
+
AND (source_ref IS NOT NULL OR source_uri IS NOT NULL OR title IS NOT NULL
|
|
1470
|
+
OR current_version_id IS NOT NULL OR document_entity_id IS NOT NULL
|
|
1471
|
+
OR deleted_at IS NULL)
|
|
1472
|
+
UNION ALL
|
|
1473
|
+
SELECT 1 FROM document_representations representation
|
|
1474
|
+
JOIN document_versions version
|
|
1475
|
+
ON version.deployment_id = representation.deployment_id
|
|
1476
|
+
AND version.version_id = representation.version_id
|
|
1477
|
+
WHERE version.deployment_id = :deployment_id AND version.doc_id = :doc_id
|
|
1478
|
+
UNION ALL
|
|
1479
|
+
SELECT 1 FROM document_versions
|
|
1480
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1481
|
+
AND (current_representation_id IS NOT NULL
|
|
1482
|
+
OR source_version_ref IS NOT NULL
|
|
1483
|
+
OR source_modified_at IS NOT NULL
|
|
1484
|
+
OR published_at IS NOT NULL
|
|
1485
|
+
OR language IS NOT NULL
|
|
1486
|
+
OR status <> 'deleted'
|
|
1487
|
+
OR error IS NOT NULL
|
|
1488
|
+
OR superseded_at IS NOT NULL
|
|
1489
|
+
OR deleted_at IS NULL)
|
|
1490
|
+
UNION ALL
|
|
1491
|
+
SELECT 1 FROM document_sections
|
|
1492
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1493
|
+
UNION ALL
|
|
1494
|
+
SELECT 1 FROM chunks
|
|
1495
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1496
|
+
UNION ALL
|
|
1497
|
+
SELECT 1 FROM chunk_claims
|
|
1498
|
+
WHERE deployment_id = :deployment_id
|
|
1499
|
+
AND (chunk_id = ANY(:chunk_ids) OR claim_id = ANY(:claim_ids))
|
|
1500
|
+
UNION ALL
|
|
1501
|
+
SELECT 1 FROM claims
|
|
1502
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1503
|
+
UNION ALL
|
|
1504
|
+
SELECT 1 FROM mentions
|
|
1505
|
+
WHERE deployment_id = :deployment_id
|
|
1506
|
+
AND (doc_id = :doc_id OR mention_id = ANY(:mention_ids))
|
|
1507
|
+
UNION ALL
|
|
1508
|
+
SELECT 1 FROM resolution_decisions
|
|
1509
|
+
WHERE deployment_id = :deployment_id AND mention_id = ANY(:mention_ids)
|
|
1510
|
+
UNION ALL
|
|
1511
|
+
SELECT 1 FROM claim_extraction_decisions
|
|
1512
|
+
WHERE deployment_id = :deployment_id AND doc_id = :doc_id
|
|
1513
|
+
UNION ALL
|
|
1514
|
+
SELECT 1 FROM grounding_audits
|
|
1515
|
+
WHERE deployment_id = :deployment_id AND claim_id = ANY(:claim_ids)
|
|
1516
|
+
UNION ALL
|
|
1517
|
+
SELECT 1 FROM relation_evidence
|
|
1518
|
+
WHERE deployment_id = :deployment_id
|
|
1519
|
+
AND (doc_id = :doc_id OR relation_id = ANY(:fact_ids))
|
|
1520
|
+
UNION ALL
|
|
1521
|
+
SELECT 1 FROM observation_evidence
|
|
1522
|
+
WHERE deployment_id = :deployment_id
|
|
1523
|
+
AND (doc_id = :doc_id OR observation_id = ANY(:fact_ids))
|
|
1524
|
+
UNION ALL
|
|
1525
|
+
SELECT 1 FROM relations
|
|
1526
|
+
WHERE deployment_id = :deployment_id AND relation_id = ANY(:fact_ids)
|
|
1527
|
+
UNION ALL
|
|
1528
|
+
SELECT 1 FROM observations
|
|
1529
|
+
WHERE deployment_id = :deployment_id AND observation_id = ANY(:fact_ids)
|
|
1530
|
+
UNION ALL
|
|
1531
|
+
SELECT 1 FROM aliases
|
|
1532
|
+
WHERE deployment_id = :deployment_id AND entity_id = ANY(:entity_ids)
|
|
1533
|
+
UNION ALL
|
|
1534
|
+
SELECT 1 FROM generic_identifier_guard
|
|
1535
|
+
WHERE deployment_id = :deployment_id
|
|
1536
|
+
UNION ALL
|
|
1537
|
+
SELECT 1 FROM merge_events
|
|
1538
|
+
WHERE deployment_id = :deployment_id
|
|
1539
|
+
AND (
|
|
1540
|
+
survivor_id = ANY(:resolved_entity_ids)
|
|
1541
|
+
OR absorbed_id = ANY(:resolved_entity_ids)
|
|
1542
|
+
OR EXISTS (
|
|
1543
|
+
SELECT 1 FROM unnest(CAST(:mention_ids AS uuid[])) AS ids(value)
|
|
1544
|
+
WHERE pre_merge_membership_snapshot::text LIKE '%'
|
|
1545
|
+
|| CAST(ids.value AS text) || '%'
|
|
1546
|
+
)
|
|
1547
|
+
)
|
|
1548
|
+
AND (
|
|
1549
|
+
cardinality(trigger_lemmas) > 0
|
|
1550
|
+
OR evidence IS NOT NULL
|
|
1551
|
+
OR pre_merge_membership_snapshot <> '{}'::jsonb
|
|
1552
|
+
)
|
|
1553
|
+
UNION ALL
|
|
1554
|
+
SELECT 1 FROM resolution_exclusions
|
|
1555
|
+
WHERE deployment_id = :deployment_id
|
|
1556
|
+
AND reason IS NOT NULL
|
|
1557
|
+
AND (
|
|
1558
|
+
entity_id_low = ANY(:resolved_entity_ids)
|
|
1559
|
+
OR entity_id_high = ANY(:resolved_entity_ids)
|
|
1560
|
+
)
|
|
1561
|
+
UNION ALL
|
|
1562
|
+
SELECT 1 FROM entities
|
|
1563
|
+
WHERE deployment_id = :deployment_id AND entity_id = ANY(:entity_ids)
|
|
1564
|
+
AND (status <> 'retired' OR canonical_name <> '' OR normalized_name <> ''
|
|
1565
|
+
OR profile_summary IS NOT NULL OR profile_embedding_ref IS NOT NULL)
|
|
1566
|
+
UNION ALL
|
|
1567
|
+
SELECT 1 FROM knowledge_artifact_evidence
|
|
1568
|
+
WHERE deployment_id = :deployment_id
|
|
1569
|
+
AND (doc_id = :doc_id OR claim_lineage_id = :doc_id
|
|
1570
|
+
OR relation_id = ANY(:fact_ids))
|
|
1571
|
+
UNION ALL
|
|
1572
|
+
SELECT 1 FROM knowledge_compilations
|
|
1573
|
+
WHERE deployment_id = :deployment_id
|
|
1574
|
+
AND artifact_id = ANY(:k_artifact_ids)
|
|
1575
|
+
UNION ALL
|
|
1576
|
+
SELECT 1 FROM knowledge_artifacts
|
|
1577
|
+
WHERE deployment_id = :deployment_id
|
|
1578
|
+
AND artifact_id = ANY(:k_artifact_ids)
|
|
1579
|
+
AND (
|
|
1580
|
+
page_summary IS NOT NULL OR content_hash IS NOT NULL
|
|
1581
|
+
OR inputs_hash IS NOT NULL OR last_compiled_at IS NOT NULL
|
|
1582
|
+
OR (page_kind = 'compiled' AND status <> 'stale')
|
|
1583
|
+
)
|
|
1584
|
+
UNION ALL
|
|
1585
|
+
SELECT 1 FROM document_crossrefs
|
|
1586
|
+
WHERE deployment_id = :deployment_id
|
|
1587
|
+
AND (from_doc_id = :doc_id OR to_doc_id = :doc_id)
|
|
1588
|
+
UNION ALL
|
|
1589
|
+
SELECT 1 FROM relation_adjudications
|
|
1590
|
+
WHERE deployment_id = :deployment_id
|
|
1591
|
+
AND (
|
|
1592
|
+
triggering_claim_id = ANY(:claim_ids)
|
|
1593
|
+
OR relation_id = ANY(:fact_ids)
|
|
1594
|
+
OR related_relation_id = ANY(:fact_ids)
|
|
1595
|
+
)
|
|
1596
|
+
UNION ALL
|
|
1597
|
+
SELECT 1 FROM observation_adjudications
|
|
1598
|
+
WHERE deployment_id = :deployment_id
|
|
1599
|
+
AND (
|
|
1600
|
+
triggering_claim_id = ANY(:claim_ids)
|
|
1601
|
+
OR observation_id = ANY(:fact_ids)
|
|
1602
|
+
OR related_observation_id = ANY(:fact_ids)
|
|
1603
|
+
)
|
|
1604
|
+
UNION ALL
|
|
1605
|
+
SELECT 1 FROM knowledge_plan_decisions decision
|
|
1606
|
+
WHERE decision.deployment_id = :deployment_id
|
|
1607
|
+
UNION ALL
|
|
1608
|
+
SELECT 1 FROM knowledge_plan_runs run
|
|
1609
|
+
WHERE run.deployment_id = :deployment_id
|
|
1610
|
+
UNION ALL
|
|
1611
|
+
SELECT 1 FROM knowledge_refresh_queue refresh
|
|
1612
|
+
WHERE refresh.deployment_id = :deployment_id
|
|
1613
|
+
AND (
|
|
1614
|
+
refresh.artifact_id = ANY(:k_artifact_ids)
|
|
1615
|
+
OR refresh.scope_id IN (
|
|
1616
|
+
SELECT artifact.scope_id
|
|
1617
|
+
FROM knowledge_artifacts artifact
|
|
1618
|
+
WHERE artifact.deployment_id = :deployment_id
|
|
1619
|
+
AND artifact.artifact_id = ANY(:k_artifact_ids)
|
|
1620
|
+
AND artifact.scope_id IS NOT NULL
|
|
1621
|
+
)
|
|
1622
|
+
OR refresh.payload::text LIKE '%' || CAST(:doc_id AS text) || '%'
|
|
1623
|
+
OR EXISTS (
|
|
1624
|
+
SELECT 1
|
|
1625
|
+
FROM unnest(
|
|
1626
|
+
CAST(:claim_ids AS uuid[]) || CAST(:mention_ids AS uuid[])
|
|
1627
|
+
|| CAST(:fact_ids AS uuid[])
|
|
1628
|
+
|| CAST(:resolved_entity_ids AS uuid[])
|
|
1629
|
+
|| CAST(:k_artifact_ids AS uuid[])
|
|
1630
|
+
) AS ids(value)
|
|
1631
|
+
WHERE refresh.payload::text LIKE '%'
|
|
1632
|
+
|| CAST(ids.value AS text) || '%'
|
|
1633
|
+
)
|
|
1634
|
+
)
|
|
1635
|
+
UNION ALL
|
|
1636
|
+
SELECT 1 FROM knowledge_dispatches dispatch
|
|
1637
|
+
WHERE dispatch.deployment_id = :deployment_id
|
|
1638
|
+
AND (
|
|
1639
|
+
dispatch.payload::text LIKE '%' || CAST(:doc_id AS text) || '%'
|
|
1640
|
+
OR EXISTS (
|
|
1641
|
+
SELECT 1
|
|
1642
|
+
FROM unnest(
|
|
1643
|
+
CAST(:claim_ids AS uuid[]) || CAST(:mention_ids AS uuid[])
|
|
1644
|
+
|| CAST(:fact_ids AS uuid[])
|
|
1645
|
+
|| CAST(:resolved_entity_ids AS uuid[])
|
|
1646
|
+
|| CAST(:k_artifact_ids AS uuid[])
|
|
1647
|
+
) AS ids(value)
|
|
1648
|
+
WHERE dispatch.payload::text LIKE '%'
|
|
1649
|
+
|| CAST(ids.value AS text) || '%'
|
|
1650
|
+
)
|
|
1651
|
+
)
|
|
1652
|
+
UNION ALL
|
|
1653
|
+
SELECT 1 FROM eval_runs WHERE deployment_id = :deployment_id
|
|
1654
|
+
UNION ALL
|
|
1655
|
+
SELECT 1 FROM canary_cases WHERE deployment_id = :deployment_id
|
|
1656
|
+
UNION ALL
|
|
1657
|
+
SELECT 1 FROM golden_claim_labels WHERE deployment_id = :deployment_id
|
|
1658
|
+
UNION ALL
|
|
1659
|
+
SELECT 1 FROM golden_pairs WHERE deployment_id = :deployment_id
|
|
1660
|
+
UNION ALL
|
|
1661
|
+
SELECT 1 FROM processing_state
|
|
1662
|
+
WHERE deployment_id = :deployment_id
|
|
1663
|
+
AND stage <> 'hard_forget'
|
|
1664
|
+
AND (payload IS NOT NULL OR last_error IS NOT NULL)
|
|
1665
|
+
AND (
|
|
1666
|
+
target_id = :doc_id
|
|
1667
|
+
OR target_id = ANY(:chunk_ids)
|
|
1668
|
+
OR target_id = ANY(:claim_ids)
|
|
1669
|
+
OR target_id = ANY(:mention_ids)
|
|
1670
|
+
OR target_id = ANY(:fact_ids)
|
|
1671
|
+
OR target_id = ANY(:entity_ids)
|
|
1672
|
+
OR target_id = ANY(:resolved_entity_ids)
|
|
1673
|
+
OR content_hash = ANY(:content_hashes)
|
|
1674
|
+
OR payload::text LIKE '%' || CAST(:doc_id AS text) || '%'
|
|
1675
|
+
OR EXISTS (
|
|
1676
|
+
SELECT 1
|
|
1677
|
+
FROM unnest(
|
|
1678
|
+
CAST(:chunk_ids AS uuid[]) || CAST(:claim_ids AS uuid[])
|
|
1679
|
+
|| CAST(:mention_ids AS uuid[]) || CAST(:fact_ids AS uuid[])
|
|
1680
|
+
|| CAST(:resolved_entity_ids AS uuid[])
|
|
1681
|
+
|| CAST(:k_artifact_ids AS uuid[])
|
|
1682
|
+
) AS ids(value)
|
|
1683
|
+
WHERE payload::text LIKE '%' || CAST(ids.value AS text) || '%'
|
|
1684
|
+
)
|
|
1685
|
+
)
|
|
1686
|
+
UNION ALL
|
|
1687
|
+
SELECT 1 FROM review_queue review
|
|
1688
|
+
WHERE review.deployment_id = :deployment_id
|
|
1689
|
+
AND (
|
|
1690
|
+
review.candidate::text LIKE '%' || CAST(:doc_id AS text) || '%'
|
|
1691
|
+
OR EXISTS (
|
|
1692
|
+
SELECT 1
|
|
1693
|
+
FROM unnest(
|
|
1694
|
+
CAST(:chunk_ids AS uuid[]) || CAST(:claim_ids AS uuid[])
|
|
1695
|
+
|| CAST(:mention_ids AS uuid[]) || CAST(:fact_ids AS uuid[])
|
|
1696
|
+
|| CAST(:resolved_entity_ids AS uuid[])
|
|
1697
|
+
|| CAST(:k_artifact_ids AS uuid[])
|
|
1698
|
+
) AS ids(value)
|
|
1699
|
+
WHERE review.candidate::text LIKE '%'
|
|
1700
|
+
|| CAST(ids.value AS text) || '%'
|
|
1701
|
+
)
|
|
1702
|
+
)
|
|
1703
|
+
UNION ALL
|
|
1704
|
+
SELECT 1 FROM content_objects object
|
|
1705
|
+
WHERE object.deployment_id = :deployment_id
|
|
1706
|
+
AND object.content_hash = ANY(:content_hashes)
|
|
1707
|
+
AND NOT EXISTS (
|
|
1708
|
+
SELECT 1
|
|
1709
|
+
FROM document_versions other_version
|
|
1710
|
+
JOIN documents other_document
|
|
1711
|
+
ON other_document.deployment_id = other_version.deployment_id
|
|
1712
|
+
AND other_document.doc_id = other_version.doc_id
|
|
1713
|
+
WHERE other_version.deployment_id = object.deployment_id
|
|
1714
|
+
AND other_version.content_hash = object.content_hash
|
|
1715
|
+
AND other_version.doc_id <> :doc_id
|
|
1716
|
+
AND other_version.deleted_at IS NULL
|
|
1717
|
+
AND other_document.deleted_at IS NULL
|
|
1718
|
+
)
|
|
1719
|
+
AND (object.mime <> 'application/x-forgotten'
|
|
1720
|
+
OR object.byte_size IS NOT NULL
|
|
1721
|
+
OR object.raw_uri <> 'forgotten/' || object.content_hash
|
|
1722
|
+
OR object.purged_at IS NULL)
|
|
1723
|
+
) residual
|
|
1724
|
+
"""
|
|
1725
|
+
)
|
|
1726
|
+
|
|
1727
|
+
_MARK_COMPLETE = text(
|
|
1728
|
+
"""
|
|
1729
|
+
UPDATE forget_manifests
|
|
1730
|
+
SET status = 'complete', completed_at = COALESCE(completed_at, now()),
|
|
1731
|
+
last_verified_at = now()
|
|
1732
|
+
WHERE deployment_id = :deployment_id
|
|
1733
|
+
AND forget_id = :forget_id
|
|
1734
|
+
AND manifest_hash = :manifest_hash
|
|
1735
|
+
AND status IN ('accepted', 'complete')
|
|
1736
|
+
"""
|
|
1737
|
+
)
|
|
1738
|
+
|
|
1739
|
+
|
|
1740
|
+
_FORGOTTEN_INGEST = text(
|
|
1741
|
+
"""
|
|
1742
|
+
SELECT forget_id
|
|
1743
|
+
FROM forget_manifests
|
|
1744
|
+
WHERE deployment_id = :deployment_id
|
|
1745
|
+
AND status = 'complete'
|
|
1746
|
+
AND (
|
|
1747
|
+
source_identity_hash = :source_identity_hash
|
|
1748
|
+
OR :content_hash = ANY(content_hashes)
|
|
1749
|
+
)
|
|
1750
|
+
ORDER BY completed_at, forget_id
|
|
1751
|
+
LIMIT 1
|
|
1752
|
+
"""
|
|
1753
|
+
)
|