dandelion-memory 1.0.0rc15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. benchmarks/__init__.py +3 -0
  2. benchmarks/adapters/__init__.py +29 -0
  3. benchmarks/adapters/ablations.py +406 -0
  4. benchmarks/adapters/base.py +72 -0
  5. benchmarks/adapters/controlled/__init__.py +13 -0
  6. benchmarks/adapters/controlled/core_archival.py +115 -0
  7. benchmarks/adapters/controlled/extract_resolve.py +130 -0
  8. benchmarks/adapters/controlled/hybrid_rag.py +120 -0
  9. benchmarks/adapters/controlled/no_late_binding.py +113 -0
  10. benchmarks/adapters/dandelion.py +94 -0
  11. benchmarks/adapters/native/__init__.py +31 -0
  12. benchmarks/adapters/native/base.py +45 -0
  13. benchmarks/adapters/native/dandelion_native.py +164 -0
  14. benchmarks/adapters/native/dandelion_native_b0.py +165 -0
  15. benchmarks/adapters/native/dandelion_native_b1.py +170 -0
  16. benchmarks/adapters/native/dandelion_native_b2.py +169 -0
  17. benchmarks/adapters/native/dandelion_native_b3.py +171 -0
  18. benchmarks/adapters/native/dandelion_native_c4.py +261 -0
  19. benchmarks/adapters/native/dandelion_native_c4_1.py +281 -0
  20. benchmarks/adapters/native/dandelion_native_c4_2.py +435 -0
  21. benchmarks/adapters/native/event_evidence_b2.py +616 -0
  22. benchmarks/adapters/native/evidence_b2.py +609 -0
  23. benchmarks/adapters/native/letta_native.py +253 -0
  24. benchmarks/adapters/native/mem0_native.py +219 -0
  25. benchmarks/adapters/native/store_vector_b0.py +792 -0
  26. benchmarks/adapters/native/store_vector_b2.py +808 -0
  27. benchmarks/agent_adapter_stdio_acceptance.py +86 -0
  28. benchmarks/bootstrap_cross_stack.py +218 -0
  29. benchmarks/canonical_profile_instrumentation.py +167 -0
  30. benchmarks/canonical_public_baseline.py +288 -0
  31. benchmarks/cli.py +525 -0
  32. benchmarks/collector_public30_offline.py +1653 -0
  33. benchmarks/datasets/__init__.py +45 -0
  34. benchmarks/datasets/conversational_generator.py +238 -0
  35. benchmarks/datasets/generator.py +283 -0
  36. benchmarks/datasets/holdout_topology_generator.py +290 -0
  37. benchmarks/datasets/holdout_topology_generator_v2.py +294 -0
  38. benchmarks/datasets/multi_world.py +1643 -0
  39. benchmarks/datasets/scale_generator.py +289 -0
  40. benchmarks/datasets/schema.py +57 -0
  41. benchmarks/deep_collision_stress.py +199 -0
  42. benchmarks/evaluators/__init__.py +12 -0
  43. benchmarks/evaluators/manifest.py +142 -0
  44. benchmarks/evaluators/metrics.py +277 -0
  45. benchmarks/evaluators/runner.py +258 -0
  46. benchmarks/f1_step0/build_dev_measurement_manifest_v1.py +127 -0
  47. benchmarks/f1_step0/build_dev_measurement_manifest_v2.py +127 -0
  48. benchmarks/f1_step0/build_dev_measurement_manifest_v3.py +146 -0
  49. benchmarks/f1_step0/build_ledger.py +143 -0
  50. benchmarks/f1_step0/build_ledger_v2.py +296 -0
  51. benchmarks/f1_step0/compare_dev52_readiness_v1.py +106 -0
  52. benchmarks/f1_step0/locator_public_smoke.py +172 -0
  53. benchmarks/f1_step0/prepare_dev52_owner_v1.py +133 -0
  54. benchmarks/f1_step0/prepare_dev52_owner_v2.py +133 -0
  55. benchmarks/f1_step0/public_identity_probe.py +191 -0
  56. benchmarks/f1_step0/public_response_mapper.py +101 -0
  57. benchmarks/f1_step0/real_source_public_preflight.py +244 -0
  58. benchmarks/f1_step0/run_exact_fusion_dev_experiment_v1.py +83 -0
  59. benchmarks/f1_step0/scorer.py +110 -0
  60. benchmarks/f1_step0/source_aware_dev_measurement_v1.py +627 -0
  61. benchmarks/f1_step0/source_aware_dev_measurement_v2.py +659 -0
  62. benchmarks/f1_step0/source_aware_dev_measurement_v3.py +776 -0
  63. benchmarks/f1_step0/test_public_response_mapper.py +110 -0
  64. benchmarks/f1_step0/test_real_source_public_preflight.py +25 -0
  65. benchmarks/f1_step0/test_scorer.py +158 -0
  66. benchmarks/f1_step0/test_source_aware_dev_measurement.py +189 -0
  67. benchmarks/f1_step0/test_source_aware_dev_measurement_v2.py +263 -0
  68. benchmarks/f1_step0/test_source_aware_dev_measurement_v3.py +334 -0
  69. benchmarks/f1_step0/typed_readback_public_smoke.py +181 -0
  70. benchmarks/forensic_c4_1_dev.py +258 -0
  71. benchmarks/forensic_c4_dev.py +255 -0
  72. benchmarks/forensic_deep25_c4_1.py +478 -0
  73. benchmarks/legacy_migration_acceptance.py +29 -0
  74. benchmarks/longmemeval_actor.py +497 -0
  75. benchmarks/longmemeval_agent_judge.py +593 -0
  76. benchmarks/longmemeval_budget.py +139 -0
  77. benchmarks/longmemeval_full.py +410 -0
  78. benchmarks/longmemeval_judge.py +327 -0
  79. benchmarks/longmemeval_pilot.py +316 -0
  80. benchmarks/longmemeval_s.py +318 -0
  81. benchmarks/memory_bench.py +248 -0
  82. benchmarks/memory_formation/__init__.py +1 -0
  83. benchmarks/memory_formation/audit.py +801 -0
  84. benchmarks/memory_formation/real_audit.py +688 -0
  85. benchmarks/memory_session_acceptance.py +279 -0
  86. benchmarks/native_codex_host.py +513 -0
  87. benchmarks/owner_dense_acceptance.py +126 -0
  88. benchmarks/pilot_seeker_vault.py +266 -0
  89. benchmarks/prepare_public30_ripgrep_range_owner.py +115 -0
  90. benchmarks/profile_canonical.py +373 -0
  91. benchmarks/profile_canonical_lanes.py +405 -0
  92. benchmarks/profile_search_chunks.py +351 -0
  93. benchmarks/public_ranges_faq_proof.py +121 -0
  94. benchmarks/public_repos/build_history_scope_v3.py +330 -0
  95. benchmarks/public_repos/build_oracle.py +108 -0
  96. benchmarks/public_repos/build_oracle_v2.py +169 -0
  97. benchmarks/public_repos/build_oracle_v3.py +176 -0
  98. benchmarks/public_repos/map_public_v2.py +296 -0
  99. benchmarks/public_repos/project_prompts_v2.py +31 -0
  100. benchmarks/public_repos/project_prompts_v3.py +29 -0
  101. benchmarks/public_repos/ready_admin_v2.py +214 -0
  102. benchmarks/public_repos/score_public_v2.py +258 -0
  103. benchmarks/public_repos/test_map_public_v2.py +171 -0
  104. benchmarks/public_repos/test_public_repo_v2.py +44 -0
  105. benchmarks/public_repos/test_public_repo_v3.py +46 -0
  106. benchmarks/public_repos/test_ready_admin_v2.py +117 -0
  107. benchmarks/public_repos/test_score_public_v2.py +163 -0
  108. benchmarks/public_repos/test_wire_v2.py +292 -0
  109. benchmarks/public_repos/wire_v2.py +603 -0
  110. benchmarks/real_blocker_fix_acceptance.py +155 -0
  111. benchmarks/real_scan_rescan_acceptance.py +41 -0
  112. benchmarks/recall_v2_20k_smoke.py +112 -0
  113. benchmarks/recall_v2_e1_retrieve.py +70 -0
  114. benchmarks/recall_v2_e1_score.py +271 -0
  115. benchmarks/recall_v2_e2_prepare.py +69 -0
  116. benchmarks/recall_v2_e2_public_score.py +91 -0
  117. benchmarks/recall_v2_e2_score.py +108 -0
  118. benchmarks/recall_v2_navigation_mcp_replay.py +308 -0
  119. benchmarks/recall_v2_reader_capture.py +118 -0
  120. benchmarks/recall_v2_reader_replay.py +136 -0
  121. benchmarks/run_blind_h10_holdout.py +284 -0
  122. benchmarks/run_blind_h9_holdout.py +275 -0
  123. benchmarks/run_c4_2_nightly_scale.py +446 -0
  124. benchmarks/run_context_scale_simulation.py +189 -0
  125. benchmarks/run_conversational_bench.py +170 -0
  126. benchmarks/run_h11_evaluation.py +316 -0
  127. benchmarks/run_h11_v2_evaluation.py +431 -0
  128. benchmarks/run_native_fixed12.py +469 -0
  129. benchmarks/run_native_preflight.py +280 -0
  130. benchmarks/run_scale_stress.py +225 -0
  131. benchmarks/solanazation_brownfield_01/build_oracle.py +970 -0
  132. benchmarks/solanazation_brownfield_01/check_level_a.py +78 -0
  133. benchmarks/solanazation_brownfield_01/debug_scope.py +13 -0
  134. benchmarks/solanazation_brownfield_01/evaluate_bootstrap_02.py +570 -0
  135. benchmarks/solanazation_brownfield_01/evaluate_first_pass.py +306 -0
  136. benchmarks/solanazation_brownfield_01/generate_dryrun_report.py +222 -0
  137. benchmarks/solanazation_brownfield_01/generate_manifests.py +146 -0
  138. benchmarks/solanazation_brownfield_01/inspect_misses.py +29 -0
  139. benchmarks/solanazation_brownfield_01/run_miss_forensic.py +206 -0
  140. benchmarks/solanazation_brownfield_01/test_facts.py +15 -0
  141. benchmarks/statistics/__init__.py +15 -0
  142. benchmarks/statistics/significance.py +326 -0
  143. benchmarks/test_native_mcp_preflight.py +229 -0
  144. benchmarks/test_prepare_public30_range_owner.py +35 -0
  145. benchmarks/test_public30_source_reference.py +698 -0
  146. benchmarks/test_recall_v2_e1_reader_input.py +89 -0
  147. benchmarks/test_recall_v2_navigation_mcp_replay.py +155 -0
  148. benchmarks/test_recall_v2_reader_replay.py +41 -0
  149. dandelion/__init__.py +3 -0
  150. dandelion/__main__.py +11 -0
  151. dandelion/agent_adapter.py +2369 -0
  152. dandelion/bootstrap/__init__.py +872 -0
  153. dandelion/bootstrap/alias_policy.py +50 -0
  154. dandelion/bootstrap/authority.py +225 -0
  155. dandelion/bootstrap/canonical_history.py +412 -0
  156. dandelion/bootstrap/classifier.py +403 -0
  157. dandelion/bootstrap/component_graph.py +280 -0
  158. dandelion/bootstrap/discovery.py +279 -0
  159. dandelion/bootstrap/domain_induction.py +180 -0
  160. dandelion/bootstrap/domain_topology.py +673 -0
  161. dandelion/bootstrap/evidence_envelope.py +132 -0
  162. dandelion/bootstrap/extractors/base.py +171 -0
  163. dandelion/bootstrap/extractors/exact.py +1234 -0
  164. dandelion/bootstrap/extractors/generic.py +100 -0
  165. dandelion/bootstrap/extractors/git_temporal.py +111 -0
  166. dandelion/bootstrap/extractors/registry.py +345 -0
  167. dandelion/bootstrap/extractors/relations.py +89 -0
  168. dandelion/bootstrap/extractors/semantic.py +424 -0
  169. dandelion/bootstrap/extractors/status.py +218 -0
  170. dandelion/bootstrap/git_sweeper.py +135 -0
  171. dandelion/bootstrap/manifest.py +246 -0
  172. dandelion/bootstrap/quarantine.py +264 -0
  173. dandelion/bootstrap/report.py +259 -0
  174. dandelion/bootstrap/root_resolver.py +221 -0
  175. dandelion/bootstrap/tech_profiler.py +125 -0
  176. dandelion/bootstrap/topology.py +75 -0
  177. dandelion/bootstrap/topology_index.py +110 -0
  178. dandelion/bootstrap/transaction.py +315 -0
  179. dandelion/canonical_bootstrap.py +257 -0
  180. dandelion/canonical_memory.py +4130 -0
  181. dandelion/captured_input.py +295 -0
  182. dandelion/cli.py +767 -0
  183. dandelion/config.py +40 -0
  184. dandelion/derived_index.py +745 -0
  185. dandelion/domains.py +706 -0
  186. dandelion/embedder.py +229 -0
  187. dandelion/event_evidence.py +571 -0
  188. dandelion/evidence.py +812 -0
  189. dandelion/evidence_response.py +453 -0
  190. dandelion/evidence_workspace.py +2360 -0
  191. dandelion/exact_context.py +129 -0
  192. dandelion/hydrator.py +124 -0
  193. dandelion/inspector.py +442 -0
  194. dandelion/inspector_assets/dandelion-mark.svg +33 -0
  195. dandelion/inspector_assets/dandelion_graph.js +361 -0
  196. dandelion/inspector_assets/inspector.css +223 -0
  197. dandelion/inspector_assets/inspector.html +135 -0
  198. dandelion/inspector_assets/inspector.js +1059 -0
  199. dandelion/legacy_import.py +431 -0
  200. dandelion/live_source_grants.py +133 -0
  201. dandelion/local_owner.py +114 -0
  202. dandelion/memory_client.py +252 -0
  203. dandelion/memory_contract.py +107 -0
  204. dandelion/memory_owner.py +560 -0
  205. dandelion/reader_providers.py +118 -0
  206. dandelion/recall_v2.py +767 -0
  207. dandelion/reconciler.py +219 -0
  208. dandelion/request_budget.py +125 -0
  209. dandelion/retained_basis.py +223 -0
  210. dandelion/retrieval.py +485 -0
  211. dandelion/router.py +405 -0
  212. dandelion/search_terms.py +53 -0
  213. dandelion/security.py +124 -0
  214. dandelion/server.py +810 -0
  215. dandelion/source_content_fts.py +219 -0
  216. dandelion/source_delivery.py +89 -0
  217. dandelion/source_history.py +241 -0
  218. dandelion/source_identity.py +88 -0
  219. dandelion/source_navigator.py +5 -0
  220. dandelion/source_reader.py +444 -0
  221. dandelion/source_references.py +379 -0
  222. dandelion/source_sections.py +265 -0
  223. dandelion/store_kv.py +684 -0
  224. dandelion/store_vector.py +1289 -0
  225. dandelion/studio.py +325 -0
  226. dandelion/wal.py +227 -0
  227. dandelion_memory-1.0.0rc15.dist-info/METADATA +48 -0
  228. dandelion_memory-1.0.0rc15.dist-info/RECORD +232 -0
  229. dandelion_memory-1.0.0rc15.dist-info/WHEEL +5 -0
  230. dandelion_memory-1.0.0rc15.dist-info/entry_points.txt +5 -0
  231. dandelion_memory-1.0.0rc15.dist-info/licenses/LICENSE +21 -0
  232. dandelion_memory-1.0.0rc15.dist-info/top_level.txt +2 -0
@@ -0,0 +1,431 @@
1
+ """Frozen-copy importer from legacy JSON/Qdrant files into canonical SQLite.
2
+
3
+ This module is intentionally not wired to public CLI or MCP tools. Callers
4
+ must provide a copied, quiescent source workspace; the importer verifies the
5
+ source hash manifest again immediately before its one SQLite transaction.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import json
12
+ import pickle
13
+ import re
14
+ import sqlite3
15
+ from dataclasses import dataclass
16
+ from pathlib import Path
17
+ from typing import Any, Mapping
18
+
19
+ from dandelion.canonical_memory import CanonicalMemory, REF_RE
20
+
21
+
22
+ _SENSITIVE_SECURITY_EVIDENCE = re.compile(
23
+ r"(?:secret|token|password|passphrase|credential|api[_-]?key|private[_-]?key|bearer|auth(?:orization)?|env(?:ironment)?)",
24
+ re.I,
25
+ )
26
+ _SAFE_PUBLIC_FACT_KEY = re.compile(
27
+ r"(?:^|[._-])(?:map_[wh]|width|height|package_id|package_name|port|host|url|version)(?:$|[._-])",
28
+ re.I,
29
+ )
30
+
31
+
32
+ def _classify_security(
33
+ explicit: object,
34
+ *,
35
+ key: str = "",
36
+ value: str = "",
37
+ source: str = "",
38
+ ) -> tuple[str, str]:
39
+ """Classify unlabeled legacy records without exposing their content.
40
+
41
+ An explicit legacy label is authoritative. Missing labels are public only
42
+ for a narrow set of ordinary configuration identifiers; credential-like
43
+ names, values, or sources fail closed.
44
+ """
45
+ if isinstance(explicit, str) and explicit:
46
+ return explicit, "explicit_label"
47
+ evidence = " ".join((key, value, source))
48
+ if _SENSITIVE_SECURITY_EVIDENCE.search(evidence):
49
+ return "RESTRICTED", "sensitive_evidence_fail_closed"
50
+ if _SAFE_PUBLIC_FACT_KEY.search(key):
51
+ return "PUBLIC", "safe_key_evidence"
52
+ return "RESTRICTED", "insufficient_public_evidence_fail_closed"
53
+
54
+
55
+ class LegacyImportError(RuntimeError):
56
+ """A frozen legacy source cannot be imported safely."""
57
+
58
+
59
+ class LegacyImportSourceChanged(LegacyImportError):
60
+ """A source file changed after preparation and before atomic apply."""
61
+
62
+
63
+ class LegacyImportCollision(LegacyImportError):
64
+ """A legacy stable ID would overwrite incompatible canonical state."""
65
+
66
+
67
+ class LegacyImportUnsupported(LegacyImportError):
68
+ """A source shape cannot be claimed losslessly by the bounded importer."""
69
+
70
+
71
+ @dataclass(frozen=True)
72
+ class PreparedLegacyImport:
73
+ """Validated source rows plus a values-free source manifest."""
74
+
75
+ source_workspace: Path
76
+ source_digest: str
77
+ manifest: Mapping[str, Any]
78
+ facts: tuple[Mapping[str, Any], ...]
79
+ domains: tuple[Mapping[str, Any], ...]
80
+ memories: tuple[Mapping[str, Any], ...]
81
+
82
+
83
+ class LegacyImporter:
84
+ """Prepare outside SQLite, then atomically apply to ``CanonicalMemory``."""
85
+
86
+ _FACTS_FILE = "facts.json"
87
+ _QDRANT_META = "qdrant/meta.json"
88
+ _QDRANT_STORAGE = "qdrant/collection/dandelion_petals/storage.sqlite"
89
+ _OPTIONAL_FILES = ("wal.jsonl",)
90
+
91
+ def __init__(self, source_workspace: Path | str, canonical: CanonicalMemory) -> None:
92
+ self.source_workspace = Path(source_workspace).resolve()
93
+ self.canonical = canonical
94
+ if self.source_workspace == canonical.workspace:
95
+ raise LegacyImportError("Legacy import source must be a frozen copy, not the target workspace")
96
+ self.source_dir = self.source_workspace / ".dandelion"
97
+
98
+ def preview(self) -> dict[str, Any]:
99
+ """Validate the copy and return a values-free manifest without mutation."""
100
+ return dict(self.prepare().manifest)
101
+
102
+ def prepare(self) -> PreparedLegacyImport:
103
+ """Read all supported source records outside the canonical transaction."""
104
+ source_files = self._source_files()
105
+ source_digest = _digest(source_files)
106
+ facts_doc = _load_json(self.source_dir / self._FACTS_FILE)
107
+ facts, domains, reverse_refs = self._prepare_facts(facts_doc)
108
+ memories = self._prepare_memories()
109
+ self._validate_refs(facts, reverse_refs, memories)
110
+ counts = {
111
+ "facts": len(facts),
112
+ "fact_revisions": sum(len(item["revisions"]) for item in facts),
113
+ "domains": len(domains),
114
+ "memories": len(memories),
115
+ "active_memories": sum(1 for item in memories if item["active"]),
116
+ "inactive_memories": sum(1 for item in memories if not item["active"]),
117
+ "memory_refs": sum(len(item["refs"]) for item in memories),
118
+ "reverse_ref_edges": sum(len(values) for values in reverse_refs.values()),
119
+ }
120
+ status_records = sum(
121
+ 1 for item in memories
122
+ if item.get("metadata", {}).get("payload", {}).get("memory_kind") == "status"
123
+ )
124
+ classified_records = [*facts, *memories]
125
+ classification_counts: dict[str, int] = {}
126
+ classification_reasons: dict[str, int] = {}
127
+ for item in classified_records:
128
+ security_class = str(item["security_class"])
129
+ reason = str(item["security_classification_reason"])
130
+ classification_counts[security_class] = classification_counts.get(security_class, 0) + 1
131
+ classification_reasons[reason] = classification_reasons.get(reason, 0) + 1
132
+ unsupported = []
133
+ if status_records:
134
+ unsupported.append({
135
+ "kind": "structured_claim_materialization",
136
+ "records": status_records,
137
+ "preserved_as": "legacy_memory_metadata",
138
+ "query_parity": "DEFERRED",
139
+ })
140
+ manifest = {
141
+ "version": 1,
142
+ "import_id": f"legacy-{source_digest[:32]}",
143
+ "source_digest": source_digest,
144
+ "source_workspace_hash": _sha256(str(self.source_workspace).encode("utf-8")),
145
+ "files": source_files,
146
+ "counts": counts,
147
+ "security_classification": {
148
+ "counts": dict(sorted(classification_counts.items())),
149
+ "reasons": dict(sorted(classification_reasons.items())),
150
+ },
151
+ "unsupported": unsupported,
152
+ }
153
+ return PreparedLegacyImport(
154
+ source_workspace=self.source_workspace,
155
+ source_digest=source_digest,
156
+ manifest=manifest,
157
+ facts=tuple(facts),
158
+ domains=tuple(domains),
159
+ memories=tuple(memories),
160
+ )
161
+
162
+ def apply(self, prepared: PreparedLegacyImport | None = None) -> dict[str, Any]:
163
+ """Recheck the frozen copy, then atomically persist canonical rows/outbox."""
164
+ prepared = prepared or self.prepare()
165
+ if prepared.source_workspace != self.source_workspace:
166
+ raise LegacyImportError("Prepared import belongs to another source workspace")
167
+ current_files = self._source_files()
168
+ if current_files != prepared.manifest["files"] or _digest(current_files) != prepared.source_digest:
169
+ raise LegacyImportSourceChanged("Legacy import source changed after prepare")
170
+ try:
171
+ return self.canonical.apply_legacy_import(
172
+ source_digest=prepared.source_digest,
173
+ manifest=prepared.manifest,
174
+ facts=prepared.facts,
175
+ domains=prepared.domains,
176
+ memories=prepared.memories,
177
+ )
178
+ except (ValueError, sqlite3.IntegrityError) as exc:
179
+ raise LegacyImportCollision("Legacy import collision; canonical state was not changed") from exc
180
+
181
+ def _source_files(self) -> dict[str, dict[str, Any]]:
182
+ if not self.source_dir.is_dir():
183
+ raise LegacyImportError("Frozen source is missing its .dandelion directory")
184
+ names = [self._FACTS_FILE, self._QDRANT_META, self._QDRANT_STORAGE]
185
+ names.extend(name for name in self._OPTIONAL_FILES if (self.source_dir / name).is_file())
186
+ files: dict[str, dict[str, Any]] = {}
187
+ for name in names:
188
+ path = self.source_dir / name
189
+ if not path.is_file():
190
+ raise LegacyImportUnsupported(f"Required frozen source file is missing: {name}")
191
+ files[name] = {"sha256": _sha256(path.read_bytes()), "bytes": path.stat().st_size}
192
+ return dict(sorted(files.items()))
193
+
194
+ def _prepare_facts(
195
+ self,
196
+ document: Mapping[str, Any],
197
+ ) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]], dict[str, tuple[str, ...]]]:
198
+ if not isinstance(document, dict):
199
+ raise LegacyImportUnsupported("facts.json root must be an object")
200
+ raw_domains = document.get("__domains__", {})
201
+ raw_refs = document.get("__fact_refs__", {})
202
+ if not isinstance(raw_domains, dict) or not isinstance(raw_refs, dict):
203
+ raise LegacyImportUnsupported("Legacy domains and reverse refs must be objects")
204
+
205
+ domains: list[Mapping[str, Any]] = []
206
+ for name, metadata in sorted(raw_domains.items()):
207
+ if not isinstance(name, str) or not isinstance(metadata, dict):
208
+ raise LegacyImportUnsupported("Legacy domain metadata is malformed")
209
+ domains.append({
210
+ "name": name,
211
+ "canonical_id": f"legacy-domain-{_sha256(name.encode('utf-8'))[:32]}",
212
+ "metadata": metadata,
213
+ "content_hash": _hash_json(metadata),
214
+ })
215
+
216
+ facts: list[Mapping[str, Any]] = []
217
+ for key, raw in sorted(document.items()):
218
+ if key in {"__domains__", "__fact_refs__"}:
219
+ continue
220
+ if not isinstance(key, str) or not isinstance(raw, dict):
221
+ raise LegacyImportUnsupported("Only structured legacy fact records are supported")
222
+ revisions = raw.get("revisions")
223
+ if not isinstance(revisions, list) or not revisions:
224
+ raise LegacyImportUnsupported("Legacy fact revisions are required for lossless import")
225
+ prepared_revisions: list[Mapping[str, Any]] = []
226
+ for position, revision in enumerate(revisions, start=1):
227
+ if not isinstance(revision, dict):
228
+ raise LegacyImportUnsupported("Legacy fact revision is malformed")
229
+ try:
230
+ number = int(revision["rev"])
231
+ value = revision["value"]
232
+ created_at = float(revision["created_at"])
233
+ valid_from = float(revision["valid_from"])
234
+ valid_until = revision.get("valid_until")
235
+ valid_until = float(valid_until) if valid_until is not None else None
236
+ source = str(revision["source"])
237
+ confidence = float(revision["confidence"])
238
+ except (KeyError, TypeError, ValueError) as exc:
239
+ raise LegacyImportUnsupported("Legacy fact revision fields are malformed") from exc
240
+ if number != position or not isinstance(value, str) or valid_until is not None and valid_until < valid_from:
241
+ raise LegacyImportUnsupported("Legacy revision sequence or value type is unsupported")
242
+ prepared_revisions.append({
243
+ "revision": number,
244
+ "value": value,
245
+ "source": source,
246
+ "confidence": confidence,
247
+ "valid_from": valid_from,
248
+ "valid_until": valid_until,
249
+ "metadata": revision,
250
+ "created_at": created_at,
251
+ })
252
+ try:
253
+ current_revision = int(raw["revision"])
254
+ except (KeyError, TypeError, ValueError) as exc:
255
+ raise LegacyImportUnsupported("Legacy fact current revision is malformed") from exc
256
+ if current_revision != len(prepared_revisions):
257
+ raise LegacyImportUnsupported("Legacy fact current revision does not match revision history")
258
+ security_class, security_reason = _classify_security(
259
+ raw.get("security_class"), key=key,
260
+ value=" ".join(str(item["value"]) for item in prepared_revisions),
261
+ source=" ".join(str(item["source"]) for item in prepared_revisions),
262
+ )
263
+ facts.append({
264
+ "key": key,
265
+ "current_revision": current_revision,
266
+ "security_class": security_class,
267
+ "security_classification_reason": security_reason,
268
+ "metadata": raw,
269
+ "revisions": prepared_revisions,
270
+ "content_hash": _hash_json(raw),
271
+ })
272
+
273
+ reverse_refs: dict[str, tuple[str, ...]] = {}
274
+ for key, memory_ids in raw_refs.items():
275
+ if not isinstance(key, str) or not isinstance(memory_ids, list) or not all(isinstance(item, str) for item in memory_ids):
276
+ raise LegacyImportUnsupported("Legacy reverse reference is malformed")
277
+ reverse_refs[key] = tuple(memory_ids)
278
+ return facts, domains, reverse_refs
279
+
280
+ def _prepare_memories(self) -> list[Mapping[str, Any]]:
281
+ storage = self.source_dir / self._QDRANT_STORAGE
282
+ try:
283
+ database = sqlite3.connect(f"file:{storage}?mode=ro", uri=True)
284
+ rows = database.execute("SELECT point FROM points").fetchall()
285
+ except sqlite3.Error as exc:
286
+ raise LegacyImportUnsupported("Frozen Qdrant point storage cannot be read") from exc
287
+ finally:
288
+ try:
289
+ database.close()
290
+ except UnboundLocalError:
291
+ pass
292
+
293
+ memories: list[Mapping[str, Any]] = []
294
+ seen_ids: set[str] = set()
295
+ for (serialized_point,) in rows:
296
+ try:
297
+ # Qdrant local storage in this project serializes PointStruct
298
+ # with pickle. The caller's frozen COPY is therefore trusted
299
+ # local input; arbitrary/untrusted pickle is unsupported.
300
+ point = pickle.loads(serialized_point)
301
+ point_id = str(point.id)
302
+ payload = point.payload
303
+ except Exception as exc:
304
+ raise LegacyImportUnsupported("Frozen Qdrant point encoding is unsupported") from exc
305
+ if not point_id or point_id in seen_ids or not isinstance(payload, dict):
306
+ raise LegacyImportUnsupported("Frozen Qdrant point identity or payload is malformed")
307
+ seen_ids.add(point_id)
308
+ required = ("domain", "template", "is_active", "created_at", "supersedes")
309
+ if any(field not in payload for field in required):
310
+ raise LegacyImportUnsupported("Frozen Qdrant point is missing required semantic fields")
311
+ if not isinstance(payload["domain"], str) or not isinstance(payload["template"], str):
312
+ raise LegacyImportUnsupported("Frozen Qdrant semantic fields have unsupported types")
313
+ if not isinstance(payload["is_active"], bool):
314
+ raise LegacyImportUnsupported("Frozen Qdrant active state must be boolean")
315
+ try:
316
+ created_at = float(payload["created_at"])
317
+ except (TypeError, ValueError) as exc:
318
+ raise LegacyImportUnsupported("Frozen Qdrant created_at is malformed") from exc
319
+ refs = payload.get("referenced_facts", [])
320
+ if not isinstance(refs, list) or not all(isinstance(item, str) for item in refs):
321
+ raise LegacyImportUnsupported("Frozen Qdrant referenced_facts is malformed")
322
+ if payload.get("supersedes") is not None and not isinstance(payload.get("supersedes"), str):
323
+ raise LegacyImportUnsupported("Frozen Qdrant supersedes is malformed")
324
+ _assert_json(payload)
325
+ typed_refs = _typed_refs(payload["template"], refs)
326
+ metadata = {"payload": payload, "vector_dimensions": len(point.vector)}
327
+ security_class, security_reason = _classify_security(
328
+ payload.get("security_class"), key=f"{payload.get('domain', '')}.{payload.get('source_component', '')}",
329
+ value=str(payload["template"]), source=str(payload.get("source", "legacy_qdrant")),
330
+ )
331
+ memories.append({
332
+ "id": point_id,
333
+ "domain": payload["domain"],
334
+ "template": payload["template"],
335
+ "source": payload.get("source") if isinstance(payload.get("source"), str) else "legacy_qdrant",
336
+ "authority": _payload_authority(payload),
337
+ "security_class": security_class,
338
+ "security_classification_reason": security_reason,
339
+ "active": payload["is_active"],
340
+ "created_at": created_at,
341
+ "supersedes": payload.get("supersedes"),
342
+ "refs": typed_refs,
343
+ "metadata": metadata,
344
+ "content_hash": _hash_json({"id": point_id, "payload": payload}),
345
+ })
346
+ return memories
347
+
348
+ @staticmethod
349
+ def _validate_refs(
350
+ facts: list[Mapping[str, Any]],
351
+ reverse_refs: Mapping[str, tuple[str, ...]],
352
+ memories: list[Mapping[str, Any]],
353
+ ) -> None:
354
+ fact_keys = {item["key"] for item in facts}
355
+ memory_ids = {item["id"] for item in memories}
356
+ for fact_key, ids in reverse_refs.items():
357
+ if fact_key not in fact_keys or any(memory_id not in memory_ids for memory_id in ids):
358
+ raise LegacyImportUnsupported("Legacy reverse reference is unresolved")
359
+ expected_reverse: dict[str, set[str]] = {}
360
+ for memory in memories:
361
+ seen_ref_modes: set[tuple[str, str]] = set()
362
+ for ref in memory["refs"]:
363
+ if ref["fact_key"] not in fact_keys:
364
+ raise LegacyImportUnsupported("Semantic reference targets an absent legacy fact")
365
+ identity = (ref["fact_key"], ref["mode"])
366
+ if identity in seen_ref_modes:
367
+ raise LegacyImportUnsupported("Multiple selectors for one legacy fact are unsupported")
368
+ seen_ref_modes.add(identity)
369
+ expected_reverse.setdefault(ref["fact_key"], set()).add(memory["id"])
370
+ for fact_key in set(reverse_refs) | set(expected_reverse):
371
+ if set(reverse_refs.get(fact_key, ())) != expected_reverse.get(fact_key, set()):
372
+ raise LegacyImportUnsupported("Legacy reverse index and semantic references disagree")
373
+
374
+
375
+ def _typed_refs(template: str, explicit_refs: list[str]) -> list[Mapping[str, Any]]:
376
+ selectors = {match.group(1): match.group(2) for match in REF_RE.finditer(template)}
377
+ result: list[Mapping[str, Any]] = []
378
+ for key in explicit_refs:
379
+ selector = selectors.get(key)
380
+ result.append({
381
+ "fact_key": key,
382
+ "mode": "snapshot" if selector else "live",
383
+ "pinned_revision": int(selector) if selector else None,
384
+ })
385
+ return result
386
+
387
+
388
+ def _payload_authority(payload: Mapping[str, Any]) -> str:
389
+ provenance = payload.get("provenance")
390
+ if isinstance(provenance, dict) and isinstance(provenance.get("authority"), str):
391
+ return provenance["authority"]
392
+ return "LEGACY_IMPORT"
393
+
394
+
395
+ def _load_json(path: Path) -> Mapping[str, Any]:
396
+ try:
397
+ value = json.loads(path.read_text(encoding="utf-8"))
398
+ except (OSError, json.JSONDecodeError) as exc:
399
+ raise LegacyImportUnsupported("Frozen facts.json cannot be read") from exc
400
+ if not isinstance(value, dict):
401
+ raise LegacyImportUnsupported("facts.json root must be an object")
402
+ return value
403
+
404
+
405
+ def _assert_json(value: Any) -> None:
406
+ try:
407
+ json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
408
+ except (TypeError, ValueError) as exc:
409
+ raise LegacyImportUnsupported("Frozen Qdrant payload is not JSON data") from exc
410
+
411
+
412
+ def _hash_json(value: Any) -> str:
413
+ return _sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8"))
414
+
415
+
416
+ def _sha256(value: bytes) -> str:
417
+ return hashlib.sha256(value).hexdigest()
418
+
419
+
420
+ def _digest(files: Mapping[str, Mapping[str, Any]]) -> str:
421
+ return _hash_json(files)
422
+
423
+
424
+ __all__ = [
425
+ "LegacyImportCollision",
426
+ "LegacyImportError",
427
+ "LegacyImporter",
428
+ "LegacyImportSourceChanged",
429
+ "LegacyImportUnsupported",
430
+ "PreparedLegacyImport",
431
+ ]
@@ -0,0 +1,133 @@
1
+ """Fresh SQLite grant projections; cached parsing never authorizes a source."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import re
8
+ import threading
9
+ from collections import OrderedDict
10
+ from copy import deepcopy
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ from dandelion.source_reader import safe_relative
15
+ from dandelion.request_budget import check_deadline
16
+
17
+
18
+ def _parse_catalog(catalog: dict, policy: str | None, owner_policy: str | None) -> dict:
19
+ if not isinstance(catalog, dict):
20
+ return {}
21
+ return {
22
+ path: entry for path, entry in sorted(catalog.items())
23
+ if isinstance(path, str) and safe_relative(path) is not None
24
+ and isinstance(entry, dict)
25
+ and entry.get("source_policy") in (None, "PUBLIC_RANGES_V1")
26
+ and entry.get("source_policy") == policy
27
+ and entry.get("source_policy") in (None, owner_policy)
28
+ and re.fullmatch(r"[0-9a-f]{64}", str(entry.get("sha256", "")))
29
+ and isinstance(entry.get("line_count"), int)
30
+ and not isinstance(entry.get("line_count"), bool)
31
+ }
32
+
33
+
34
+ # Cache decoded grant fields only, never the much larger capture-locator manifest.
35
+ # Every lookup still reads the actual latest row and its live withdrawal state.
36
+ _projection_cache: OrderedDict[tuple[str, str | None], tuple[dict, Any, Any]] = OrderedDict()
37
+ _projection_cache_lock = threading.Lock()
38
+
39
+
40
+ def _reject_non_json_constant(value: str) -> None:
41
+ raise ValueError("Invalid JSON constant")
42
+
43
+
44
+ def _manifest_projection(raw: str, owner_policy: str | None) -> tuple[dict, Any, Any] | None:
45
+ key = (hashlib.sha256(raw.encode("utf-8")).hexdigest(), owner_policy)
46
+ with _projection_cache_lock:
47
+ cached = _projection_cache.get(key)
48
+ if cached is not None:
49
+ _projection_cache.move_to_end(key)
50
+ return cached
51
+ check_deadline()
52
+ try:
53
+ manifest = json.loads(raw, parse_constant=_reject_non_json_constant)
54
+ except (ValueError, TypeError, RecursionError):
55
+ return None
56
+ if not isinstance(manifest, dict):
57
+ return None
58
+ catalog = manifest.get("source_catalog")
59
+ policy = manifest.get("source_policy")
60
+ projection = manifest.get("source_content_projection")
61
+ if (not isinstance(catalog, dict) or policy is not None and not isinstance(policy, str)
62
+ or projection is not None and not isinstance(projection, str)):
63
+ return None
64
+ parsed = (_parse_catalog(catalog, policy, owner_policy), policy, projection)
65
+ check_deadline()
66
+ # Larger catalogs remain usable without retention; unrelated manifest size
67
+ # does not prevent reuse of a small grant projection.
68
+ if len(json.dumps((catalog, policy, projection), ensure_ascii=True).encode("ascii")) <= 2 * 1024 * 1024:
69
+ with _projection_cache_lock:
70
+ _projection_cache[key] = parsed
71
+ _projection_cache.move_to_end(key)
72
+ while len(_projection_cache) > 4:
73
+ _projection_cache.popitem(last=False)
74
+ return parsed
75
+
76
+
77
+ def live_source_grants(owner: Any, requested: Path | None = None, *,
78
+ paths: list[str] | None = None) -> list[dict[str, Any]]:
79
+ """Read each actual latest row, including same-digest edits and revocations.
80
+
81
+ Projection reuse binds to the full current manifest bytes, not its mutable
82
+ plan digest or row identity. JSON decoding keeps duplicate-key last values.
83
+ Invalid latest JSON yields no grant, never a previous committed row.
84
+ """
85
+ check_deadline()
86
+ if requested is None:
87
+ selection = (
88
+ "SELECT b.source_workspace,b.manifest_json,b.plan_digest "
89
+ "FROM canonical_bootstrap_runs b WHERE b.plan_digest=("
90
+ "SELECT b2.plan_digest FROM canonical_bootstrap_runs b2 "
91
+ "WHERE b2.source_workspace=b.source_workspace "
92
+ "ORDER BY b2.committed_at DESC,b2.plan_digest DESC LIMIT 1)"
93
+ )
94
+ parameters = ()
95
+ else:
96
+ # Sort row identities before fetching a potentially large manifest.
97
+ selection = (
98
+ "SELECT source_workspace,manifest_json,plan_digest "
99
+ "FROM canonical_bootstrap_runs WHERE rowid=("
100
+ "SELECT rowid FROM canonical_bootstrap_runs WHERE source_workspace=? "
101
+ "ORDER BY committed_at DESC,plan_digest DESC LIMIT 1)"
102
+ )
103
+ parameters = (str(requested),)
104
+ sql = (
105
+ "SELECT b.source_workspace,b.plan_digest,b.manifest_json FROM ("
106
+ + selection + ") b "
107
+ "WHERE NOT EXISTS (SELECT 1 FROM canonical_source_withdrawals w "
108
+ "WHERE w.source_workspace=b.source_workspace) "
109
+ "ORDER BY b.source_workspace"
110
+ )
111
+ with owner._connect() as db:
112
+ rows = db.execute(sql, parameters).fetchall()
113
+ check_deadline()
114
+ grants = []
115
+ for raw_root, digest, raw in rows:
116
+ check_deadline()
117
+ try:
118
+ root = Path(raw_root).resolve(strict=True)
119
+ if not root.is_dir() or requested is not None and root != requested:
120
+ continue
121
+ parsed = _manifest_projection(raw, owner.source_policy)
122
+ if parsed is None:
123
+ continue
124
+ catalog, policy, projection = parsed
125
+ check_deadline()
126
+ selected = catalog if paths is None else {path: catalog[path] for path in paths if path in catalog}
127
+ grants.append({"lineage_root": str(root), "plan_digest": digest,
128
+ "source_policy": policy, "source_content_projection": projection,
129
+ "source_catalog": deepcopy(selected)})
130
+ except (OSError, RuntimeError, ValueError, TypeError):
131
+ continue
132
+ check_deadline()
133
+ return grants
@@ -0,0 +1,114 @@
1
+ """Explicit lexical owner lifetime for a local MCP host, not a detached daemon."""
2
+
3
+ from contextlib import contextmanager
4
+ from pathlib import Path
5
+ import threading
6
+ import time
7
+
8
+ from dandelion.memory_client import MemoryClient, MemoryClientError, OwnerUnavailable
9
+ from dandelion.memory_owner import MemoryOwner, OwnerAlreadyRunning, OwnerDescriptorError, MemoryOwnerError
10
+
11
+
12
+ class LocalOwnerSession:
13
+ """A workspace lease, independent of which connected MCP process hosts it."""
14
+
15
+ def __init__(self, workspace: Path, *, lifecycle_v1: bool, access_mode: str):
16
+ self.workspace = workspace.resolve()
17
+ if not self.workspace.is_dir():
18
+ raise ValueError("local owner requires an existing workspace")
19
+ self.lifecycle_v1 = lifecycle_v1
20
+ self.access_mode = access_mode
21
+ self.owner = None
22
+ self.lock = threading.RLock()
23
+
24
+ def _start(self, owner):
25
+ from dandelion.canonical_memory import CanonicalMemory
26
+ from dandelion.derived_index import CanonicalDerivedService
27
+
28
+ try:
29
+ canonical = CanonicalMemory(self.workspace, lifecycle_v1_enabled=self.lifecycle_v1,
30
+ access_mode=self.access_mode)
31
+ owner.service = CanonicalDerivedService(canonical, None)
32
+ owner.start()
33
+ except Exception:
34
+ owner.abort_start()
35
+ raise
36
+ self.owner = owner
37
+ return MemoryClient(owner.descriptor)
38
+
39
+ def connect(self, *, deadline=None):
40
+ deadline = deadline if deadline is not None else time.monotonic() + 10
41
+ with self.lock:
42
+ while time.monotonic() < deadline:
43
+ if self.owner is not None and self.owner.is_running:
44
+ return MemoryClient(self.owner.descriptor)
45
+ try:
46
+ client = MemoryClient.for_workspace(
47
+ self.workspace, timeout_seconds=deadline - time.monotonic())
48
+ status = client.status()
49
+ except (OwnerUnavailable, OwnerDescriptorError):
50
+ contender = MemoryOwner(self.workspace, None)
51
+ try:
52
+ contender.acquire_guard()
53
+ except OwnerAlreadyRunning:
54
+ # A different adapter may be initializing or taking over.
55
+ time.sleep(min(0.1, max(0, deadline - time.monotonic())))
56
+ continue
57
+ return self._start(contender)
58
+ if (status.get("owner_id") != client.descriptor.owner_id
59
+ or Path(status["workspace"]).resolve() != self.workspace
60
+ or status.get("access_mode", "strict") != self.access_mode
61
+ or (self.lifecycle_v1 and not status.get("lifecycle_v1_enabled"))):
62
+ raise ValueError("existing owner does not match requested workspace/mode/capabilities")
63
+ return client
64
+ raise OwnerUnavailable("existing memory owner did not become ready")
65
+
66
+ def client(self):
67
+ return _LocalMemoryClient(self, self.connect().descriptor)
68
+
69
+ def close(self):
70
+ with self.lock:
71
+ if self.owner is not None:
72
+ self.owner.stop()
73
+
74
+
75
+ class _LocalMemoryClient(MemoryClient):
76
+ """Rebind before dispatch; never replay an operation after a transport failure."""
77
+
78
+ def __init__(self, session, descriptor):
79
+ super().__init__(descriptor)
80
+ self.session = session
81
+
82
+ def call(self, operation, payload=None, **kwargs):
83
+ requested_timeout = kwargs.get("_timeout_seconds")
84
+ timeout = min(self.timeout_seconds, requested_timeout if requested_timeout is not None else self.timeout_seconds)
85
+ if timeout <= 0:
86
+ raise ValueError("request timeout must be positive")
87
+ requested_deadline = kwargs.get("_deadline")
88
+ deadline = min(requested_deadline if requested_deadline is not None else float("inf"),
89
+ time.monotonic() + timeout)
90
+ if time.monotonic() >= deadline:
91
+ raise OwnerUnavailable("Memory owner request deadline exceeded")
92
+ try:
93
+ descriptor = self.session.connect(deadline=deadline).descriptor
94
+ except (ValueError, MemoryOwnerError) as exc:
95
+ raise MemoryClientError("Local owner binding failed: " + str(exc)) from exc
96
+ # Use a request-local client: simultaneous calls must not exchange identities.
97
+ self.descriptor = descriptor
98
+ kwargs["_timeout_seconds"] = deadline - time.monotonic()
99
+ if kwargs["_timeout_seconds"] <= 0:
100
+ raise OwnerUnavailable("Memory owner request deadline exceeded")
101
+ client = MemoryClient(descriptor, timeout_seconds=self.timeout_seconds,
102
+ max_message_bytes=self.max_message_bytes)
103
+ return client.call(operation, payload, **kwargs)
104
+
105
+
106
+ @contextmanager
107
+ def local_owner(workspace: Path, *, lifecycle_v1: bool, access_mode: str):
108
+ """Enable local ownership and safe pre-dispatch takeover for this connection."""
109
+ session = LocalOwnerSession(workspace, lifecycle_v1=lifecycle_v1, access_mode=access_mode)
110
+ try:
111
+ session.connect()
112
+ yield session
113
+ finally:
114
+ session.close()