rememberstack 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. rememberstack/__init__.py +9 -0
  2. rememberstack/adapters/__init__.py +42 -0
  3. rememberstack/adapters/codex_writer.py +221 -0
  4. rememberstack/adapters/markitdown_converter.py +42 -0
  5. rememberstack/adapters/openrouter.py +136 -0
  6. rememberstack/adapters/selfhost/__init__.py +54 -0
  7. rememberstack/adapters/selfhost/forget.py +66 -0
  8. rememberstack/adapters/selfhost/git.py +374 -0
  9. rememberstack/adapters/selfhost/lance.py +328 -0
  10. rememberstack/adapters/selfhost/minio.py +279 -0
  11. rememberstack/adapters/selfhost/mounts.py +249 -0
  12. rememberstack/adapters/selfhost/object_store.py +130 -0
  13. rememberstack/adapters/selfhost/projection.py +80 -0
  14. rememberstack/adapters/selfhost/queue.py +137 -0
  15. rememberstack/adapters/selfhost/telemetry.py +45 -0
  16. rememberstack/adapters/selfhost/watcher.py +70 -0
  17. rememberstack/adapters/testing/__init__.py +15 -0
  18. rememberstack/adapters/testing/cost_meter.py +13 -0
  19. rememberstack/adapters/testing/model_provider.py +83 -0
  20. rememberstack/adapters/testing/queue.py +43 -0
  21. rememberstack/adapters/testing/telemetry.py +22 -0
  22. rememberstack/client.py +19 -0
  23. rememberstack/core/__init__.py +127 -0
  24. rememberstack/core/blockizer.py +189 -0
  25. rememberstack/core/chunker.py +216 -0
  26. rememberstack/core/consumption_skill.py +275 -0
  27. rememberstack/core/conversion.py +76 -0
  28. rememberstack/core/core_manifest.py +598 -0
  29. rememberstack/core/extension_packs.py +124 -0
  30. rememberstack/core/forget.py +17 -0
  31. rememberstack/core/knowledge_authored.py +276 -0
  32. rememberstack/core/knowledge_compile.py +215 -0
  33. rememberstack/core/knowledge_fact_sheet.py +210 -0
  34. rememberstack/core/knowledge_hashing.py +68 -0
  35. rememberstack/core/knowledge_planner.py +64 -0
  36. rememberstack/core/knowledge_writer.py +175 -0
  37. rememberstack/core/ranking.py +200 -0
  38. rememberstack/core/recipe_linter.py +149 -0
  39. rememberstack/core/section_snap.py +209 -0
  40. rememberstack/core/storage_routing.py +27 -0
  41. rememberstack/eval/__init__.py +53 -0
  42. rememberstack/eval/consumption.py +141 -0
  43. rememberstack/eval/contradiction.py +184 -0
  44. rememberstack/eval/harness.py +136 -0
  45. rememberstack/eval/lifecycle.py +400 -0
  46. rememberstack/eval/operational_scale.py +49 -0
  47. rememberstack/eval/resolution.py +255 -0
  48. rememberstack/eval/retrieval_spikes.py +50 -0
  49. rememberstack/eval/skeleton.py +231 -0
  50. rememberstack/llm/__init__.py +1 -0
  51. rememberstack/model/__init__.py +589 -0
  52. rememberstack/model/adjudication.py +100 -0
  53. rememberstack/model/auth.py +27 -0
  54. rememberstack/model/blocks.py +30 -0
  55. rememberstack/model/chunks.py +190 -0
  56. rememberstack/model/claims.py +162 -0
  57. rememberstack/model/client.py +98 -0
  58. rememberstack/model/clustering.py +54 -0
  59. rememberstack/model/component_version.py +124 -0
  60. rememberstack/model/consumption.py +88 -0
  61. rememberstack/model/conversion.py +31 -0
  62. rememberstack/model/deployment.py +53 -0
  63. rememberstack/model/documents.py +168 -0
  64. rememberstack/model/envelope.py +513 -0
  65. rememberstack/model/evaluation.py +72 -0
  66. rememberstack/model/forget.py +143 -0
  67. rememberstack/model/git.py +13 -0
  68. rememberstack/model/knowledge.py +840 -0
  69. rememberstack/model/knowledge_authored.py +325 -0
  70. rememberstack/model/knowledge_planner.py +431 -0
  71. rememberstack/model/lifecycle.py +42 -0
  72. rememberstack/model/model_provider.py +78 -0
  73. rememberstack/model/mounts.py +24 -0
  74. rememberstack/model/object_store.py +21 -0
  75. rememberstack/model/operational_scale.py +59 -0
  76. rememberstack/model/operations.py +153 -0
  77. rememberstack/model/processing.py +228 -0
  78. rememberstack/model/queue.py +73 -0
  79. rememberstack/model/recipes.py +83 -0
  80. rememberstack/model/relations.py +79 -0
  81. rememberstack/model/resolution.py +83 -0
  82. rememberstack/model/retrieval_spikes.py +62 -0
  83. rememberstack/model/sections.py +120 -0
  84. rememberstack/model/telemetry.py +30 -0
  85. rememberstack/ports/__init__.py +29 -0
  86. rememberstack/ports/auth.py +16 -0
  87. rememberstack/ports/connector.py +23 -0
  88. rememberstack/ports/cost_meter.py +17 -0
  89. rememberstack/ports/forget.py +20 -0
  90. rememberstack/ports/git.py +20 -0
  91. rememberstack/ports/model_provider.py +28 -0
  92. rememberstack/ports/mounts.py +16 -0
  93. rememberstack/ports/object_store.py +27 -0
  94. rememberstack/ports/p1_index.py +92 -0
  95. rememberstack/ports/purge.py +93 -0
  96. rememberstack/ports/queue.py +23 -0
  97. rememberstack/ports/telemetry.py +21 -0
  98. rememberstack/profiles/__init__.py +22 -0
  99. rememberstack/profiles/selfhost.py +324 -0
  100. rememberstack/profiles/selfhost_forget.py +158 -0
  101. rememberstack/profiles/selfhost_operations.py +95 -0
  102. rememberstack/py.typed +1 -0
  103. rememberstack/spine/__init__.py +93 -0
  104. rememberstack/spine/admission.py +26 -0
  105. rememberstack/spine/backfill.py +168 -0
  106. rememberstack/spine/catalog_contract.py +742 -0
  107. rememberstack/spine/chunk_catalog.py +237 -0
  108. rememberstack/spine/claim_catalog.py +298 -0
  109. rememberstack/spine/clustering.py +740 -0
  110. rememberstack/spine/component_versions.py +208 -0
  111. rememberstack/spine/consumption.py +81 -0
  112. rememberstack/spine/deployment_bootstrap.py +445 -0
  113. rememberstack/spine/document_catalog.py +621 -0
  114. rememberstack/spine/entity_registry.py +205 -0
  115. rememberstack/spine/extension_packs.py +220 -0
  116. rememberstack/spine/fact_catalog.py +571 -0
  117. rememberstack/spine/forget.py +1753 -0
  118. rememberstack/spine/knowledge.py +5467 -0
  119. rememberstack/spine/lifecycle.py +1071 -0
  120. rememberstack/spine/migrations/__init__.py +1 -0
  121. rememberstack/spine/migrations/_helpers.py +153 -0
  122. rememberstack/spine/migrations/env.py +58 -0
  123. rememberstack/spine/migrations/script.py.mako +27 -0
  124. rememberstack/spine/migrations/versions/__init__.py +1 -0
  125. rememberstack/spine/migrations/versions/p0_02_0001_extensions_enums.py +189 -0
  126. rememberstack/spine/migrations/versions/p0_02_0002_infrastructure_registries.py +321 -0
  127. rememberstack/spine/migrations/versions/p0_02_0003_entities_evaluation_e0_e1.py +631 -0
  128. rememberstack/spine/migrations/versions/p0_02_0004_claims_facts_evidence.py +411 -0
  129. rememberstack/spine/migrations/versions/p0_02_0005_projection_knowledge_retrieval.py +391 -0
  130. rememberstack/spine/migrations/versions/p0_02_0006_partitions_views.py +158 -0
  131. rememberstack/spine/migrations/versions/p2_06_0007_invalidated_outcome.py +26 -0
  132. rememberstack/spine/migrations/versions/p3_01_0008_document_version_target.py +58 -0
  133. rememberstack/spine/migrations/versions/p3_05_0009_reconcile_stage.py +27 -0
  134. rememberstack/spine/migrations/versions/p3_07_0010_lifecycle_eval_suite.py +25 -0
  135. rememberstack/spine/migrations/versions/p4_01_0011_survivor_view_rewrite.py +57 -0
  136. rememberstack/spine/migrations/versions/p6_02_0012_knowledge_compile_recovery.py +58 -0
  137. rememberstack/spine/migrations/versions/p6_04_0013_knowledge_writer_ledger.py +46 -0
  138. rememberstack/spine/migrations/versions/p6_05_0014_knowledge_planner_runtime.py +217 -0
  139. rememberstack/spine/migrations/versions/p6_06_0015_authored_dispatch_runtime.py +38 -0
  140. rememberstack/spine/migrations/versions/p7_02_0016_operational_eval_suite.py +19 -0
  141. rememberstack/spine/migrations/versions/p7_05_0017_hard_forget.py +55 -0
  142. rememberstack/spine/observation_adjudication.py +778 -0
  143. rememberstack/spine/operations.py +298 -0
  144. rememberstack/spine/projection.py +662 -0
  145. rememberstack/spine/recipes.py +276 -0
  146. rememberstack/spine/resolver.py +763 -0
  147. rememberstack/spine/review.py +650 -0
  148. rememberstack/spine/settings.py +22 -0
  149. rememberstack/spine/supersession.py +510 -0
  150. rememberstack/spine/sync.py +128 -0
  151. rememberstack/spine/work_ledger.py +816 -0
  152. rememberstack/surfaces/__init__.py +110 -0
  153. rememberstack/surfaces/cli.py +447 -0
  154. rememberstack/surfaces/consumption_skill.py +87 -0
  155. rememberstack/surfaces/graph_queries.py +698 -0
  156. rememberstack/surfaces/http_api.py +377 -0
  157. rememberstack/surfaces/mcp.py +67 -0
  158. rememberstack/surfaces/query_engine.py +1591 -0
  159. rememberstack/surfaces/recipe_executor.py +185 -0
  160. rememberstack/surfaces/recipe_surface.py +219 -0
  161. rememberstack/surfaces/remote_mcp.py +133 -0
  162. rememberstack/surfaces/sdk.py +324 -0
  163. rememberstack/workers/__init__.py +155 -0
  164. rememberstack/workers/base.py +312 -0
  165. rememberstack/workers/e0.py +577 -0
  166. rememberstack/workers/e1.py +425 -0
  167. rememberstack/workers/e2.py +525 -0
  168. rememberstack/workers/e3.py +434 -0
  169. rememberstack/workers/forget.py +299 -0
  170. rememberstack/workers/knowledge_authored.py +146 -0
  171. rememberstack/workers/knowledge_driver.py +735 -0
  172. rememberstack/workers/knowledge_fact_sheet.py +123 -0
  173. rememberstack/workers/knowledge_planner.py +325 -0
  174. rememberstack/workers/knowledge_writer.py +393 -0
  175. rememberstack/workers/operations.py +42 -0
  176. rememberstack/workers/p1.py +234 -0
  177. rememberstack/workers/p2.py +513 -0
  178. rememberstack/workers/p2_analytics.py +276 -0
  179. rememberstack/workers/p3.py +673 -0
  180. rememberstack/workers/reconcile.py +485 -0
  181. rememberstack/workers/sync.py +168 -0
  182. rememberstack-0.1.0.dist-info/METADATA +213 -0
  183. rememberstack-0.1.0.dist-info/RECORD +186 -0
  184. rememberstack-0.1.0.dist-info/WHEEL +4 -0
  185. rememberstack-0.1.0.dist-info/entry_points.txt +2 -0
  186. rememberstack-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,324 @@
1
+ """Executable self-host composition for the WP-0.4c Compose quickstart."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ from pathlib import Path
7
+ import sys
8
+ from typing import Self
9
+ from typing import TYPE_CHECKING
10
+ from uuid import UUID
11
+
12
+ from alembic import command
13
+ from alembic.config import Config
14
+ from pydantic import Field
15
+ from pydantic_settings import BaseSettings
16
+ from pydantic_settings import SettingsConfigDict
17
+ import sqlalchemy
18
+ from sqlalchemy import text
19
+ from sqlalchemy.engine import Engine
20
+ from sqlalchemy.engine import make_url
21
+
22
+ from rememberstack.adapters import OpenRouterModelProvider
23
+ from rememberstack.adapters import OpenRouterSettings
24
+ from rememberstack.adapters.selfhost import LocalFSForgetManifestStore
25
+ from rememberstack.adapters.selfhost import MinIOObjectStore
26
+ from rememberstack.adapters.selfhost import MinIOSettings
27
+ from rememberstack.model import DeploymentBootstrapInput
28
+ from rememberstack.model import PipelineStage
29
+ from rememberstack.spine import DeploymentBootstrapper
30
+ from rememberstack.spine import RecipeRegistry
31
+ from rememberstack.spine import seed_canonical_recipes
32
+ from rememberstack.spine.settings import load_database_settings
33
+
34
+ if TYPE_CHECKING:
35
+ from fastapi import FastAPI
36
+
37
+ from rememberstack.adapters.selfhost import SelfHostWorkerLoop
38
+
39
+ _SUPPORTED_WORKER_STAGES = (PipelineStage.CONVERT, PipelineStage.STRUCTURE)
40
+
41
+
42
+ class SelfHostSettings(BaseSettings):
43
+ """One fresh self-host deployment's profile and process settings."""
44
+
45
+ model_config = SettingsConfigDict(
46
+ env_prefix="REMEMBERSTACK_SELFHOST_", extra="ignore"
47
+ )
48
+
49
+ deployment_id: UUID
50
+ deployment_slug: str = Field(default="local", min_length=1)
51
+ deployment_name: str = Field(default="Local memory", min_length=1)
52
+ default_language: str = Field(default="en", min_length=1)
53
+ raw_bucket_name: str = Field(default="remember-raw", min_length=1)
54
+ artifacts_bucket_name: str = Field(default="remember-artifacts", min_length=1)
55
+ corpusfs_bucket_name: str = Field(default="remember-corpusfs", min_length=1)
56
+ lance_root: Path = Path("/var/lib/rememberstack/lance")
57
+ forget_manifest_root: Path = Path("/var/lib/rememberstack/forget-manifests")
58
+ migration_config: Path = Path("alembic.ini")
59
+ api_host: str = "0.0.0.0"
60
+ api_port: int = Field(default=8000, ge=1, le=65_535)
61
+ worker_rate_per_s: float = Field(default=20.0, gt=0)
62
+ worker_burst: float = Field(default=20.0, ge=1)
63
+ worker_fallback_poll_s: float = Field(default=5.0, gt=0)
64
+ worker_session_s: float = Field(default=3_600.0, gt=0)
65
+
66
+
67
+ class _FreshDeploymentReadiness:
68
+ """Fail closed if a fresh quickstart sees portable forget history.
69
+
70
+ WP-0.4c establishes a fresh-deployment Compose skeleton. It must never
71
+ silently serve a restored deployment whose D74 manifests require the full
72
+ hard-forget recovery composition; finding any manifest stops startup.
73
+ """
74
+
75
+ def __init__(self, *, store: LocalFSForgetManifestStore) -> None:
76
+ """Bind the separately durable manifest root."""
77
+ self._store = store
78
+
79
+ def ensure_ready(self, *, deployment_id: UUID) -> tuple[UUID, ...]:
80
+ """Accept an empty root and refuse every non-empty restore."""
81
+ manifests = self._store.manifests(deployment_id=deployment_id)
82
+ if manifests:
83
+ raise RuntimeError(
84
+ "the Compose quickstart found portable hard-forget manifests;"
85
+ " restore requires the complete D74 self-host recovery profile"
86
+ )
87
+ return ()
88
+
89
+
90
+ class SelfHostProfile:
91
+ """Compose the existing API and E0 workers over PostgreSQL, MinIO, and Lance."""
92
+
93
+ def __init__(
94
+ self,
95
+ *,
96
+ settings: SelfHostSettings,
97
+ engine: Engine,
98
+ raw_store: MinIOObjectStore,
99
+ artifact_store: MinIOObjectStore,
100
+ corpusfs_store: MinIOObjectStore,
101
+ model_provider: OpenRouterModelProvider,
102
+ ) -> None:
103
+ """Retain one dependency graph for an API, setup, or worker process."""
104
+ self._settings = settings
105
+ self._engine = engine
106
+ self._raw_store = raw_store
107
+ self._artifact_store = artifact_store
108
+ self._corpusfs_store = corpusfs_store
109
+ self._model_provider = model_provider
110
+
111
+ @classmethod
112
+ def from_settings(cls) -> Self:
113
+ """Load every external value through its typed settings boundary."""
114
+ profile_settings = SelfHostSettings.model_validate({})
115
+ minio_settings = MinIOSettings.model_validate({})
116
+ return cls(
117
+ settings=profile_settings,
118
+ engine=sqlalchemy.create_engine(
119
+ load_database_settings().sqlalchemy_url(), pool_pre_ping=True
120
+ ),
121
+ raw_store=MinIOObjectStore(
122
+ bucket=profile_settings.raw_bucket_name, settings=minio_settings
123
+ ),
124
+ artifact_store=MinIOObjectStore(
125
+ bucket=profile_settings.artifacts_bucket_name, settings=minio_settings
126
+ ),
127
+ corpusfs_store=MinIOObjectStore(
128
+ bucket=profile_settings.corpusfs_bucket_name, settings=minio_settings
129
+ ),
130
+ model_provider=OpenRouterModelProvider(
131
+ settings=OpenRouterSettings.model_validate({})
132
+ ),
133
+ )
134
+
135
+ def close(self) -> None:
136
+ """Dispose this process's explicitly owned database pool."""
137
+ self._engine.dispose()
138
+
139
+ def setup(self) -> None:
140
+ """Apply migrations, provision buckets, bootstrap core rows, and seed recipes."""
141
+ migration = Config(str(self._settings.migration_config))
142
+ migration.set_main_option(
143
+ "sqlalchemy.url", load_database_settings().sqlalchemy_url()
144
+ )
145
+ command.upgrade(config=migration, revision="head")
146
+ self._raw_store.ensure_bucket()
147
+ self._artifact_store.ensure_bucket()
148
+ self._corpusfs_store.ensure_bucket()
149
+ self._settings.forget_manifest_root.mkdir(parents=True, exist_ok=True)
150
+ DeploymentBootstrapper(engine=self._engine).bootstrap_deployment(
151
+ deployment_input=DeploymentBootstrapInput(
152
+ deployment_id=self._settings.deployment_id,
153
+ slug=self._settings.deployment_slug,
154
+ name=self._settings.deployment_name,
155
+ default_language=self._settings.default_language,
156
+ raw_bucket=f"s3://{self._settings.raw_bucket_name}",
157
+ artifacts_bucket=f"s3://{self._settings.artifacts_bucket_name}",
158
+ corpusfs_bucket=f"s3://{self._settings.corpusfs_bucket_name}",
159
+ )
160
+ )
161
+ seed_canonical_recipes(
162
+ registry=RecipeRegistry(engine=self._engine),
163
+ deployment_id=self._settings.deployment_id,
164
+ )
165
+
166
+ def api(self) -> FastAPI:
167
+ """Build the existing HTTP surface over this self-host dependency graph."""
168
+ from rememberstack.adapters.selfhost.lance import LanceChunkIndex
169
+ from rememberstack.spine import DocumentCatalog
170
+ from rememberstack.spine import ForgetCatalog
171
+ from rememberstack.surfaces import build_api
172
+ from rememberstack.surfaces import QueryEngine
173
+ from rememberstack.surfaces import RecipeExecutor
174
+ from rememberstack.surfaces import RecipeSurface
175
+ from rememberstack.workers.e0 import UploadIngestor
176
+
177
+ query_engine = QueryEngine(
178
+ engine=self._engine,
179
+ search_index=LanceChunkIndex(root=self._settings.lance_root),
180
+ model_provider=self._model_provider,
181
+ embedding_model="qwen/qwen3-embedding-8b",
182
+ )
183
+ app = build_api(
184
+ engine=query_engine,
185
+ deployment_id=self._settings.deployment_id,
186
+ admission=ForgetCatalog(engine=self._engine),
187
+ readiness=_FreshDeploymentReadiness(
188
+ store=LocalFSForgetManifestStore(
189
+ root=self._settings.forget_manifest_root
190
+ )
191
+ ),
192
+ surface=RecipeSurface(
193
+ registry=RecipeRegistry(engine=self._engine),
194
+ executor=RecipeExecutor(query_engine=query_engine),
195
+ deployment_id=self._settings.deployment_id,
196
+ ),
197
+ ingest=UploadIngestor(
198
+ catalog=DocumentCatalog(engine=self._engine),
199
+ raw_store=self._raw_store,
200
+ admission=ForgetCatalog(engine=self._engine),
201
+ ),
202
+ )
203
+
204
+ @app.get("/healthz", include_in_schema=False)
205
+ def healthz() -> dict[str, str]:
206
+ """Prove the process can reach its authoritative PostgreSQL spine."""
207
+ with self._engine.connect() as connection:
208
+ connection.execute(text("SELECT 1")).scalar_one()
209
+ return {"status": "ok"}
210
+
211
+ return app
212
+
213
+ def worker_loop(self, *, stage: PipelineStage) -> SelfHostWorkerLoop:
214
+ """Build one E0 route's ordinary LISTEN/NOTIFY worker loop."""
215
+ from rememberstack.adapters.selfhost import SelfHostTaskQueue
216
+ from rememberstack.adapters.selfhost import SelfHostWorkerLoop
217
+ from rememberstack.adapters.selfhost import TokenBucket
218
+ from rememberstack.core import ConversionRouter
219
+ from rememberstack.core import MarkdownPassthroughConverter
220
+ from rememberstack.model import ProcessingLane
221
+ from rememberstack.spine import DocumentCatalog
222
+ from rememberstack.spine import WorkLedger
223
+ from rememberstack.spine import WorkLedgerSettings
224
+ from rememberstack.workers import ConvertHandler
225
+ from rememberstack.workers import HandlerRegistry
226
+ from rememberstack.workers import StructureHandler
227
+ from rememberstack.workers import Worker
228
+
229
+ if stage not in _SUPPORTED_WORKER_STAGES:
230
+ raise ValueError(f"the Compose skeleton has no handler for stage {stage}")
231
+ catalog = DocumentCatalog(engine=self._engine)
232
+ registry = HandlerRegistry()
233
+ if stage is PipelineStage.CONVERT:
234
+ registry.register(
235
+ stage=stage,
236
+ handler=ConvertHandler(
237
+ catalog=catalog,
238
+ raw_store=self._raw_store,
239
+ artifact_store=self._artifact_store,
240
+ router=ConversionRouter(
241
+ routes={"text/markdown": MarkdownPassthroughConverter()}
242
+ ),
243
+ ),
244
+ )
245
+ else:
246
+ registry.register(
247
+ stage=stage,
248
+ handler=StructureHandler(
249
+ catalog=catalog,
250
+ artifact_store=self._artifact_store,
251
+ model_provider=self._model_provider,
252
+ ),
253
+ )
254
+ ledger = WorkLedger(engine=self._engine, settings=WorkLedgerSettings())
255
+ return SelfHostWorkerLoop(
256
+ worker=Worker(
257
+ ledger=ledger, registry=registry, queue=SelfHostTaskQueue(ledger=ledger)
258
+ ),
259
+ deployment_id=self._settings.deployment_id,
260
+ stage=stage,
261
+ lane=ProcessingLane.STEADY,
262
+ bucket=TokenBucket(
263
+ rate_per_s=self._settings.worker_rate_per_s,
264
+ capacity=self._settings.worker_burst,
265
+ ),
266
+ database_url=_psycopg_url(),
267
+ fallback_poll_s=self._settings.worker_fallback_poll_s,
268
+ )
269
+
270
+ def run_worker(self, *, stage: PipelineStage) -> None:
271
+ """Run one configured E0 route until the process is stopped or fails."""
272
+ loop = self.worker_loop(stage=stage)
273
+ while True:
274
+ loop.run_for(duration_s=self._settings.worker_session_s)
275
+
276
+
277
+ def create_api() -> FastAPI:
278
+ """Uvicorn factory for the self-host API process."""
279
+ return SelfHostProfile.from_settings().api()
280
+
281
+
282
+ def main(argv: list[str] | None = None) -> int:
283
+ """Run setup, API, or one E0 worker process for Docker Compose."""
284
+ parser = argparse.ArgumentParser(description="rememberstack self-host profile")
285
+ subparsers = parser.add_subparsers(dest="command", required=True)
286
+ subparsers.add_parser("setup", help="migrate and bootstrap the deployment")
287
+ subparsers.add_parser("api", help="serve the deployment HTTP API")
288
+ worker = subparsers.add_parser("worker", help="run one E0 worker route")
289
+ worker.add_argument(
290
+ "--stage",
291
+ choices=tuple(stage.value for stage in _SUPPORTED_WORKER_STAGES),
292
+ required=True,
293
+ )
294
+ args = parser.parse_args(argv)
295
+ settings = SelfHostSettings.model_validate({})
296
+ if args.command == "api":
297
+ import uvicorn
298
+
299
+ uvicorn.run(
300
+ create_api(),
301
+ host=settings.api_host,
302
+ port=settings.api_port,
303
+ access_log=True,
304
+ )
305
+ return 0
306
+ profile = SelfHostProfile.from_settings()
307
+ try:
308
+ if args.command == "setup":
309
+ profile.setup()
310
+ return 0
311
+ profile.run_worker(stage=PipelineStage(args.stage))
312
+ return 0
313
+ finally:
314
+ profile.close()
315
+
316
+
317
+ def _psycopg_url() -> str:
318
+ """Remove SQLAlchemy's driver suffix for psycopg's native connection parser."""
319
+ url = make_url(load_database_settings().sqlalchemy_url())
320
+ return url.set(drivername="postgresql").render_as_string(hide_password=False)
321
+
322
+
323
+ if __name__ == "__main__":
324
+ sys.exit(main())
@@ -0,0 +1,158 @@
1
+ """One explicit self-host composition for D74 hard-forget and readiness."""
2
+
3
+ from datetime import datetime
4
+ from pathlib import Path
5
+ from typing import Self
6
+ from uuid import UUID
7
+
8
+ from sqlalchemy.engine import Engine
9
+
10
+ from rememberstack.adapters.selfhost import LanceChunkIndex
11
+ from rememberstack.adapters.selfhost import LocalFSForgetManifestStore
12
+ from rememberstack.adapters.selfhost import LocalFSObjectStore
13
+ from rememberstack.adapters.selfhost import LocalGitRepository
14
+ from rememberstack.adapters.selfhost import SelfHostProjectionPurger
15
+ from rememberstack.model import ForgetManifest
16
+ from rememberstack.model import PipelineStage
17
+ from rememberstack.spine import ForgetCatalog
18
+ from rememberstack.spine import LifecycleCatalog
19
+ from rememberstack.spine import ProjectionCatalog
20
+ from rememberstack.workers import CorpusFsBuilder
21
+ from rememberstack.workers import DeletionService
22
+ from rememberstack.workers import GraphRebuildWorker
23
+ from rememberstack.workers import HandlerRegistry
24
+ from rememberstack.workers import HardForgetHandler
25
+ from rememberstack.workers import HardForgetReadiness
26
+ from rememberstack.workers import HardForgetService
27
+ from rememberstack.workers import KnowledgeCommitDriver
28
+ from rememberstack.workers import KnowledgeCycleForgetRebuilder
29
+ from rememberstack.workers import ProjectionPairForgetRebuilder
30
+
31
+
32
+ class SelfHostHardForget:
33
+ """The self-host request, worker registration, admission, and startup gate."""
34
+
35
+ def __init__(
36
+ self,
37
+ *,
38
+ catalog: ForgetCatalog,
39
+ service: HardForgetService,
40
+ handler: HardForgetHandler,
41
+ readiness: HardForgetReadiness,
42
+ ) -> None:
43
+ """Retain one shared coordinator graph for every self-host surface."""
44
+ self._catalog = catalog
45
+ self._service = service
46
+ self._handler = handler
47
+ self._readiness = readiness
48
+
49
+ @classmethod
50
+ def compose(
51
+ cls,
52
+ *,
53
+ engine: Engine,
54
+ manifest_root: Path,
55
+ object_roots: tuple[Path, ...],
56
+ snapshot_root: Path,
57
+ lance_root: Path,
58
+ p2_cache_root: Path,
59
+ mount_root: Path,
60
+ knowledge_repository: Path,
61
+ knowledge_author_name: str,
62
+ knowledge_author_email: str,
63
+ rebuild_workdir: Path,
64
+ knowledge_driver: KnowledgeCommitDriver,
65
+ ) -> Self:
66
+ """Compose only existing production workers and selected self-host stores."""
67
+ if not object_roots:
68
+ raise ValueError("hard-forget requires at least one ordinary object root")
69
+ catalog = ForgetCatalog(engine=engine)
70
+ manifest_store = LocalFSForgetManifestStore(root=manifest_root)
71
+ object_purgers = tuple(LocalFSObjectStore(root=root) for root in object_roots)
72
+ snapshot_store = LocalFSObjectStore(root=snapshot_root)
73
+ projection_catalog = ProjectionCatalog(engine=engine)
74
+ k_git = LocalGitRepository(
75
+ repository=knowledge_repository,
76
+ path_catalog=catalog,
77
+ author_name=knowledge_author_name,
78
+ author_email=knowledge_author_email,
79
+ )
80
+ service = HardForgetService(
81
+ catalog=catalog, manifest_store=manifest_store, k_git=k_git
82
+ )
83
+ handler = HardForgetHandler(
84
+ catalog=catalog,
85
+ deletion=DeletionService(catalog=LifecycleCatalog(engine=engine)),
86
+ object_purgers=object_purgers,
87
+ p1=LanceChunkIndex(root=lance_root),
88
+ projection_rebuilder=ProjectionPairForgetRebuilder(
89
+ graph=GraphRebuildWorker(
90
+ catalog=projection_catalog, snapshot_store=snapshot_store
91
+ ),
92
+ corpus=CorpusFsBuilder(
93
+ catalog=projection_catalog, snapshot_store=snapshot_store
94
+ ),
95
+ workdir=rebuild_workdir,
96
+ ),
97
+ projection_purger=SelfHostProjectionPurger(
98
+ object_purger=snapshot_store,
99
+ catalog=projection_catalog,
100
+ p2_cache_root=p2_cache_root,
101
+ mount_root=mount_root,
102
+ ),
103
+ knowledge_rebuilder=KnowledgeCycleForgetRebuilder(driver=knowledge_driver),
104
+ k_git=k_git,
105
+ )
106
+ readiness = HardForgetReadiness(
107
+ catalog=catalog,
108
+ manifest_store=manifest_store,
109
+ request_service=service,
110
+ handler=handler,
111
+ )
112
+ return cls(
113
+ catalog=catalog, service=service, handler=handler, readiness=readiness
114
+ )
115
+
116
+ def register(self, *, registry: HandlerRegistry) -> None:
117
+ """Install the one unlaned hard-forget worker handler."""
118
+ registry.register(stage=PipelineStage.HARD_FORGET, handler=self._handler)
119
+
120
+ def request(
121
+ self,
122
+ *,
123
+ deployment_id: UUID,
124
+ doc_id: UUID,
125
+ forget_id: UUID,
126
+ requested_at: datetime,
127
+ ) -> ForgetManifest:
128
+ """Submit one lineage request through the crash-safe acceptance cut."""
129
+ return self._service.request(
130
+ deployment_id=deployment_id,
131
+ doc_id=doc_id,
132
+ forget_id=forget_id,
133
+ requested_at=requested_at,
134
+ )
135
+
136
+ def ensure_ready(self, *, deployment_id: UUID) -> tuple[UUID, ...]:
137
+ """Re-honor every portable manifest before serving begins."""
138
+ return self._readiness.ensure_ready(deployment_id=deployment_id)
139
+
140
+ def assert_available(self, *, deployment_id: UUID) -> None:
141
+ """Implement the shared public/mount admission perimeter."""
142
+ self._catalog.assert_available(deployment_id=deployment_id)
143
+
144
+ def guard_ingest(
145
+ self,
146
+ *,
147
+ deployment_id: UUID,
148
+ source_kind: str,
149
+ source_ref: str,
150
+ content_hash: str,
151
+ ) -> None:
152
+ """Implement E0's before-bytes barrier and irreversible ingest guard."""
153
+ self._catalog.guard_ingest(
154
+ deployment_id=deployment_id,
155
+ source_kind=source_kind,
156
+ source_ref=source_ref,
157
+ content_hash=content_hash,
158
+ )
@@ -0,0 +1,95 @@
1
+ """Self-host composition root for the optional local operational commands."""
2
+
3
+ from datetime import datetime
4
+ from pathlib import Path
5
+ from typing import Self
6
+ from uuid import UUID
7
+
8
+ import sqlalchemy
9
+ from sqlalchemy.engine import Engine
10
+
11
+ from rememberstack.adapters.selfhost import LocalFSObjectStore
12
+ from rememberstack.adapters.selfhost import SelfHostTaskQueue
13
+ from rememberstack.model import DeadLetterReplayResult
14
+ from rememberstack.model import OperationalReport
15
+ from rememberstack.model import ProcessingLane
16
+ from rememberstack.spine import ForgetCatalog
17
+ from rememberstack.spine import OperationalCatalog
18
+ from rememberstack.spine import OperationalSettings
19
+ from rememberstack.spine import ProjectionCatalog
20
+ from rememberstack.spine import WorkLedger
21
+ from rememberstack.spine import WorkLedgerSettings
22
+ from rememberstack.spine.settings import load_database_settings
23
+ from rememberstack.workers import CorpusFsBuilder
24
+ from rememberstack.workers import DeadLetterReplayer
25
+ from rememberstack.workers import GraphRebuildWorker
26
+
27
+
28
+ class SelfHostOperations:
29
+ """Compose local adapters around one explicitly owned database engine."""
30
+
31
+ def __init__(self, *, engine: Engine) -> None:
32
+ """Take ownership of an engine created for one CLI invocation."""
33
+ self._engine = engine
34
+
35
+ @classmethod
36
+ def from_settings(cls) -> Self:
37
+ """Create the local composition from the typed database setting."""
38
+ return cls(
39
+ engine=sqlalchemy.create_engine(load_database_settings().sqlalchemy_url())
40
+ )
41
+
42
+ def close(self) -> None:
43
+ """Dispose the command-owned connection pool."""
44
+ self._engine.dispose()
45
+
46
+ def inspect(self, *, deployment_id: UUID) -> OperationalReport:
47
+ """Build one bounded typed report."""
48
+ return OperationalCatalog(
49
+ engine=self._engine, settings=OperationalSettings()
50
+ ).inspect(deployment_id=deployment_id)
51
+
52
+ def replay(
53
+ self,
54
+ *,
55
+ deployment_id: UUID,
56
+ processing_id: UUID,
57
+ attempt_allowance: int,
58
+ lane: ProcessingLane | None,
59
+ not_before: datetime | None,
60
+ ) -> DeadLetterReplayResult:
61
+ """Compose the authoritative replay transition with local delivery."""
62
+ ForgetCatalog(engine=self._engine).assert_available(deployment_id=deployment_id)
63
+ ledger = WorkLedger(engine=self._engine, settings=WorkLedgerSettings())
64
+ return DeadLetterReplayer(
65
+ ledger=ledger, queue=SelfHostTaskQueue(ledger=ledger)
66
+ ).replay(
67
+ deployment_id=deployment_id,
68
+ processing_id=processing_id,
69
+ attempt_allowance=attempt_allowance,
70
+ lane=lane,
71
+ not_before=not_before,
72
+ )
73
+
74
+ def rebuild(
75
+ self,
76
+ *,
77
+ plane: str,
78
+ deployment_id: UUID,
79
+ snapshot_root: Path,
80
+ workdir: Path,
81
+ version: str,
82
+ ) -> dict[str, object]:
83
+ """Invoke the existing whole-rebuild implementation for P2 or P3."""
84
+ ForgetCatalog(engine=self._engine).assert_available(deployment_id=deployment_id)
85
+ catalog = ProjectionCatalog(engine=self._engine)
86
+ store = LocalFSObjectStore(root=snapshot_root)
87
+ if plane == "p2":
88
+ return GraphRebuildWorker(catalog=catalog, snapshot_store=store).rebuild(
89
+ deployment_id=deployment_id, workdir=workdir, version=version
90
+ )
91
+ if plane == "p3":
92
+ return CorpusFsBuilder(catalog=catalog, snapshot_store=store).build(
93
+ deployment_id=deployment_id, version=version
94
+ )
95
+ raise ValueError(f"unknown projection plane {plane!r}")
rememberstack/py.typed ADDED
@@ -0,0 +1 @@
1
+
@@ -0,0 +1,93 @@
1
+ """Postgres spine package."""
2
+
3
+ from rememberstack.spine.backfill import BackfillFinalizer
4
+ from rememberstack.spine.backfill import BackfillSeeder
5
+ from rememberstack.spine.backfill import BackfillSeederSettings
6
+ from rememberstack.spine.chunk_catalog import ChunkCatalog
7
+ from rememberstack.spine.claim_catalog import ClaimCatalog
8
+ from rememberstack.spine.clustering import EntityClusterer
9
+ from rememberstack.spine.component_versions import ComponentVersionRegistrar
10
+ from rememberstack.spine.consumption import ConsumptionCatalog
11
+ from rememberstack.spine.consumption import ConsumptionDeploymentNotFoundError
12
+ from rememberstack.spine.deployment_bootstrap import DeploymentBootstrapper
13
+ from rememberstack.spine.document_catalog import DocumentCatalog
14
+ from rememberstack.spine.entity_registry import EntityRegistry
15
+ from rememberstack.spine.entity_registry import T0_RESOLVER_VERSION
16
+ from rememberstack.spine.extension_packs import install_pack
17
+ from rememberstack.spine.extension_packs import PackAnchorError
18
+ from rememberstack.spine.extension_packs import PackConflictError
19
+ from rememberstack.spine.fact_catalog import FactCatalog
20
+ from rememberstack.spine.forget import ForgetCatalog
21
+ from rememberstack.spine.knowledge import KnowledgeCommitBusyError
22
+ from rememberstack.spine.knowledge import KnowledgeCompilationError
23
+ from rememberstack.spine.knowledge import KnowledgeCompileContextMissingError
24
+ from rememberstack.spine.knowledge import KnowledgeControlPlane
25
+ from rememberstack.spine.knowledge import KnowledgeDispatchUnavailableError
26
+ from rememberstack.spine.lifecycle import LifecycleCatalog
27
+ from rememberstack.spine.observation_adjudication import OBSERVATION_ADJUDICATOR_VERSION
28
+ from rememberstack.spine.observation_adjudication import ObservationAdjudicator
29
+ from rememberstack.spine.observation_adjudication import ObservationSettings
30
+ from rememberstack.spine.operations import error_class_from_traceback
31
+ from rememberstack.spine.operations import OperationalCatalog
32
+ from rememberstack.spine.operations import OperationalSettings
33
+ from rememberstack.spine.projection import ProjectionCatalog
34
+ from rememberstack.spine.recipes import CANONICAL_RECIPES
35
+ from rememberstack.spine.recipes import RecipeRegistry
36
+ from rememberstack.spine.recipes import seed_canonical_recipes
37
+ from rememberstack.spine.resolver import CascadeResolver
38
+ from rememberstack.spine.resolver import RESOLVER_VERSION
39
+ from rememberstack.spine.resolver import seed_resolver_version
40
+ from rememberstack.spine.review import ReviewQueue
41
+ from rememberstack.spine.supersession import ADJUDICATOR_VERSION
42
+ from rememberstack.spine.supersession import SupersessionAdjudicator
43
+ from rememberstack.spine.supersession import SupersessionSettings
44
+ from rememberstack.spine.sync import SyncCatalog
45
+ from rememberstack.spine.work_ledger import WorkLedger
46
+ from rememberstack.spine.work_ledger import WorkLedgerSettings
47
+
48
+ __all__ = (
49
+ "BackfillFinalizer",
50
+ "BackfillSeeder",
51
+ "BackfillSeederSettings",
52
+ "ChunkCatalog",
53
+ "ClaimCatalog",
54
+ "EntityRegistry",
55
+ "PackAnchorError",
56
+ "PackConflictError",
57
+ "install_pack",
58
+ "ADJUDICATOR_VERSION",
59
+ "CascadeResolver",
60
+ "SupersessionAdjudicator",
61
+ "SupersessionSettings",
62
+ "SyncCatalog",
63
+ "FactCatalog",
64
+ "ForgetCatalog",
65
+ "OBSERVATION_ADJUDICATOR_VERSION",
66
+ "ObservationAdjudicator",
67
+ "ObservationSettings",
68
+ "OperationalCatalog",
69
+ "OperationalSettings",
70
+ "error_class_from_traceback",
71
+ "RESOLVER_VERSION",
72
+ "LifecycleCatalog",
73
+ "KnowledgeCompilationError",
74
+ "KnowledgeCommitBusyError",
75
+ "KnowledgeCompileContextMissingError",
76
+ "KnowledgeControlPlane",
77
+ "KnowledgeDispatchUnavailableError",
78
+ "ProjectionCatalog",
79
+ "CANONICAL_RECIPES",
80
+ "RecipeRegistry",
81
+ "seed_canonical_recipes",
82
+ "ReviewQueue",
83
+ "seed_resolver_version",
84
+ "T0_RESOLVER_VERSION",
85
+ "ComponentVersionRegistrar",
86
+ "ConsumptionCatalog",
87
+ "ConsumptionDeploymentNotFoundError",
88
+ "EntityClusterer",
89
+ "DocumentCatalog",
90
+ "DeploymentBootstrapper",
91
+ "WorkLedger",
92
+ "WorkLedgerSettings",
93
+ )