dandelion-memory 1.0.0rc15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchmarks/__init__.py +3 -0
- benchmarks/adapters/__init__.py +29 -0
- benchmarks/adapters/ablations.py +406 -0
- benchmarks/adapters/base.py +72 -0
- benchmarks/adapters/controlled/__init__.py +13 -0
- benchmarks/adapters/controlled/core_archival.py +115 -0
- benchmarks/adapters/controlled/extract_resolve.py +130 -0
- benchmarks/adapters/controlled/hybrid_rag.py +120 -0
- benchmarks/adapters/controlled/no_late_binding.py +113 -0
- benchmarks/adapters/dandelion.py +94 -0
- benchmarks/adapters/native/__init__.py +31 -0
- benchmarks/adapters/native/base.py +45 -0
- benchmarks/adapters/native/dandelion_native.py +164 -0
- benchmarks/adapters/native/dandelion_native_b0.py +165 -0
- benchmarks/adapters/native/dandelion_native_b1.py +170 -0
- benchmarks/adapters/native/dandelion_native_b2.py +169 -0
- benchmarks/adapters/native/dandelion_native_b3.py +171 -0
- benchmarks/adapters/native/dandelion_native_c4.py +261 -0
- benchmarks/adapters/native/dandelion_native_c4_1.py +281 -0
- benchmarks/adapters/native/dandelion_native_c4_2.py +435 -0
- benchmarks/adapters/native/event_evidence_b2.py +616 -0
- benchmarks/adapters/native/evidence_b2.py +609 -0
- benchmarks/adapters/native/letta_native.py +253 -0
- benchmarks/adapters/native/mem0_native.py +219 -0
- benchmarks/adapters/native/store_vector_b0.py +792 -0
- benchmarks/adapters/native/store_vector_b2.py +808 -0
- benchmarks/agent_adapter_stdio_acceptance.py +86 -0
- benchmarks/bootstrap_cross_stack.py +218 -0
- benchmarks/canonical_profile_instrumentation.py +167 -0
- benchmarks/canonical_public_baseline.py +288 -0
- benchmarks/cli.py +525 -0
- benchmarks/collector_public30_offline.py +1653 -0
- benchmarks/datasets/__init__.py +45 -0
- benchmarks/datasets/conversational_generator.py +238 -0
- benchmarks/datasets/generator.py +283 -0
- benchmarks/datasets/holdout_topology_generator.py +290 -0
- benchmarks/datasets/holdout_topology_generator_v2.py +294 -0
- benchmarks/datasets/multi_world.py +1643 -0
- benchmarks/datasets/scale_generator.py +289 -0
- benchmarks/datasets/schema.py +57 -0
- benchmarks/deep_collision_stress.py +199 -0
- benchmarks/evaluators/__init__.py +12 -0
- benchmarks/evaluators/manifest.py +142 -0
- benchmarks/evaluators/metrics.py +277 -0
- benchmarks/evaluators/runner.py +258 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v1.py +127 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v2.py +127 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v3.py +146 -0
- benchmarks/f1_step0/build_ledger.py +143 -0
- benchmarks/f1_step0/build_ledger_v2.py +296 -0
- benchmarks/f1_step0/compare_dev52_readiness_v1.py +106 -0
- benchmarks/f1_step0/locator_public_smoke.py +172 -0
- benchmarks/f1_step0/prepare_dev52_owner_v1.py +133 -0
- benchmarks/f1_step0/prepare_dev52_owner_v2.py +133 -0
- benchmarks/f1_step0/public_identity_probe.py +191 -0
- benchmarks/f1_step0/public_response_mapper.py +101 -0
- benchmarks/f1_step0/real_source_public_preflight.py +244 -0
- benchmarks/f1_step0/run_exact_fusion_dev_experiment_v1.py +83 -0
- benchmarks/f1_step0/scorer.py +110 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v1.py +627 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v2.py +659 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v3.py +776 -0
- benchmarks/f1_step0/test_public_response_mapper.py +110 -0
- benchmarks/f1_step0/test_real_source_public_preflight.py +25 -0
- benchmarks/f1_step0/test_scorer.py +158 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement.py +189 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement_v2.py +263 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement_v3.py +334 -0
- benchmarks/f1_step0/typed_readback_public_smoke.py +181 -0
- benchmarks/forensic_c4_1_dev.py +258 -0
- benchmarks/forensic_c4_dev.py +255 -0
- benchmarks/forensic_deep25_c4_1.py +478 -0
- benchmarks/legacy_migration_acceptance.py +29 -0
- benchmarks/longmemeval_actor.py +497 -0
- benchmarks/longmemeval_agent_judge.py +593 -0
- benchmarks/longmemeval_budget.py +139 -0
- benchmarks/longmemeval_full.py +410 -0
- benchmarks/longmemeval_judge.py +327 -0
- benchmarks/longmemeval_pilot.py +316 -0
- benchmarks/longmemeval_s.py +318 -0
- benchmarks/memory_bench.py +248 -0
- benchmarks/memory_formation/__init__.py +1 -0
- benchmarks/memory_formation/audit.py +801 -0
- benchmarks/memory_formation/real_audit.py +688 -0
- benchmarks/memory_session_acceptance.py +279 -0
- benchmarks/native_codex_host.py +513 -0
- benchmarks/owner_dense_acceptance.py +126 -0
- benchmarks/pilot_seeker_vault.py +266 -0
- benchmarks/prepare_public30_ripgrep_range_owner.py +115 -0
- benchmarks/profile_canonical.py +373 -0
- benchmarks/profile_canonical_lanes.py +405 -0
- benchmarks/profile_search_chunks.py +351 -0
- benchmarks/public_ranges_faq_proof.py +121 -0
- benchmarks/public_repos/build_history_scope_v3.py +330 -0
- benchmarks/public_repos/build_oracle.py +108 -0
- benchmarks/public_repos/build_oracle_v2.py +169 -0
- benchmarks/public_repos/build_oracle_v3.py +176 -0
- benchmarks/public_repos/map_public_v2.py +296 -0
- benchmarks/public_repos/project_prompts_v2.py +31 -0
- benchmarks/public_repos/project_prompts_v3.py +29 -0
- benchmarks/public_repos/ready_admin_v2.py +214 -0
- benchmarks/public_repos/score_public_v2.py +258 -0
- benchmarks/public_repos/test_map_public_v2.py +171 -0
- benchmarks/public_repos/test_public_repo_v2.py +44 -0
- benchmarks/public_repos/test_public_repo_v3.py +46 -0
- benchmarks/public_repos/test_ready_admin_v2.py +117 -0
- benchmarks/public_repos/test_score_public_v2.py +163 -0
- benchmarks/public_repos/test_wire_v2.py +292 -0
- benchmarks/public_repos/wire_v2.py +603 -0
- benchmarks/real_blocker_fix_acceptance.py +155 -0
- benchmarks/real_scan_rescan_acceptance.py +41 -0
- benchmarks/recall_v2_20k_smoke.py +112 -0
- benchmarks/recall_v2_e1_retrieve.py +70 -0
- benchmarks/recall_v2_e1_score.py +271 -0
- benchmarks/recall_v2_e2_prepare.py +69 -0
- benchmarks/recall_v2_e2_public_score.py +91 -0
- benchmarks/recall_v2_e2_score.py +108 -0
- benchmarks/recall_v2_navigation_mcp_replay.py +308 -0
- benchmarks/recall_v2_reader_capture.py +118 -0
- benchmarks/recall_v2_reader_replay.py +136 -0
- benchmarks/run_blind_h10_holdout.py +284 -0
- benchmarks/run_blind_h9_holdout.py +275 -0
- benchmarks/run_c4_2_nightly_scale.py +446 -0
- benchmarks/run_context_scale_simulation.py +189 -0
- benchmarks/run_conversational_bench.py +170 -0
- benchmarks/run_h11_evaluation.py +316 -0
- benchmarks/run_h11_v2_evaluation.py +431 -0
- benchmarks/run_native_fixed12.py +469 -0
- benchmarks/run_native_preflight.py +280 -0
- benchmarks/run_scale_stress.py +225 -0
- benchmarks/solanazation_brownfield_01/build_oracle.py +970 -0
- benchmarks/solanazation_brownfield_01/check_level_a.py +78 -0
- benchmarks/solanazation_brownfield_01/debug_scope.py +13 -0
- benchmarks/solanazation_brownfield_01/evaluate_bootstrap_02.py +570 -0
- benchmarks/solanazation_brownfield_01/evaluate_first_pass.py +306 -0
- benchmarks/solanazation_brownfield_01/generate_dryrun_report.py +222 -0
- benchmarks/solanazation_brownfield_01/generate_manifests.py +146 -0
- benchmarks/solanazation_brownfield_01/inspect_misses.py +29 -0
- benchmarks/solanazation_brownfield_01/run_miss_forensic.py +206 -0
- benchmarks/solanazation_brownfield_01/test_facts.py +15 -0
- benchmarks/statistics/__init__.py +15 -0
- benchmarks/statistics/significance.py +326 -0
- benchmarks/test_native_mcp_preflight.py +229 -0
- benchmarks/test_prepare_public30_range_owner.py +35 -0
- benchmarks/test_public30_source_reference.py +698 -0
- benchmarks/test_recall_v2_e1_reader_input.py +89 -0
- benchmarks/test_recall_v2_navigation_mcp_replay.py +155 -0
- benchmarks/test_recall_v2_reader_replay.py +41 -0
- dandelion/__init__.py +3 -0
- dandelion/__main__.py +11 -0
- dandelion/agent_adapter.py +2369 -0
- dandelion/bootstrap/__init__.py +872 -0
- dandelion/bootstrap/alias_policy.py +50 -0
- dandelion/bootstrap/authority.py +225 -0
- dandelion/bootstrap/canonical_history.py +412 -0
- dandelion/bootstrap/classifier.py +403 -0
- dandelion/bootstrap/component_graph.py +280 -0
- dandelion/bootstrap/discovery.py +279 -0
- dandelion/bootstrap/domain_induction.py +180 -0
- dandelion/bootstrap/domain_topology.py +673 -0
- dandelion/bootstrap/evidence_envelope.py +132 -0
- dandelion/bootstrap/extractors/base.py +171 -0
- dandelion/bootstrap/extractors/exact.py +1234 -0
- dandelion/bootstrap/extractors/generic.py +100 -0
- dandelion/bootstrap/extractors/git_temporal.py +111 -0
- dandelion/bootstrap/extractors/registry.py +345 -0
- dandelion/bootstrap/extractors/relations.py +89 -0
- dandelion/bootstrap/extractors/semantic.py +424 -0
- dandelion/bootstrap/extractors/status.py +218 -0
- dandelion/bootstrap/git_sweeper.py +135 -0
- dandelion/bootstrap/manifest.py +246 -0
- dandelion/bootstrap/quarantine.py +264 -0
- dandelion/bootstrap/report.py +259 -0
- dandelion/bootstrap/root_resolver.py +221 -0
- dandelion/bootstrap/tech_profiler.py +125 -0
- dandelion/bootstrap/topology.py +75 -0
- dandelion/bootstrap/topology_index.py +110 -0
- dandelion/bootstrap/transaction.py +315 -0
- dandelion/canonical_bootstrap.py +257 -0
- dandelion/canonical_memory.py +4130 -0
- dandelion/captured_input.py +295 -0
- dandelion/cli.py +767 -0
- dandelion/config.py +40 -0
- dandelion/derived_index.py +745 -0
- dandelion/domains.py +706 -0
- dandelion/embedder.py +229 -0
- dandelion/event_evidence.py +571 -0
- dandelion/evidence.py +812 -0
- dandelion/evidence_response.py +453 -0
- dandelion/evidence_workspace.py +2360 -0
- dandelion/exact_context.py +129 -0
- dandelion/hydrator.py +124 -0
- dandelion/inspector.py +442 -0
- dandelion/inspector_assets/dandelion-mark.svg +33 -0
- dandelion/inspector_assets/dandelion_graph.js +361 -0
- dandelion/inspector_assets/inspector.css +223 -0
- dandelion/inspector_assets/inspector.html +135 -0
- dandelion/inspector_assets/inspector.js +1059 -0
- dandelion/legacy_import.py +431 -0
- dandelion/live_source_grants.py +133 -0
- dandelion/local_owner.py +114 -0
- dandelion/memory_client.py +252 -0
- dandelion/memory_contract.py +107 -0
- dandelion/memory_owner.py +560 -0
- dandelion/reader_providers.py +118 -0
- dandelion/recall_v2.py +767 -0
- dandelion/reconciler.py +219 -0
- dandelion/request_budget.py +125 -0
- dandelion/retained_basis.py +223 -0
- dandelion/retrieval.py +485 -0
- dandelion/router.py +405 -0
- dandelion/search_terms.py +53 -0
- dandelion/security.py +124 -0
- dandelion/server.py +810 -0
- dandelion/source_content_fts.py +219 -0
- dandelion/source_delivery.py +89 -0
- dandelion/source_history.py +241 -0
- dandelion/source_identity.py +88 -0
- dandelion/source_navigator.py +5 -0
- dandelion/source_reader.py +444 -0
- dandelion/source_references.py +379 -0
- dandelion/source_sections.py +265 -0
- dandelion/store_kv.py +684 -0
- dandelion/store_vector.py +1289 -0
- dandelion/studio.py +325 -0
- dandelion/wal.py +227 -0
- dandelion_memory-1.0.0rc15.dist-info/METADATA +48 -0
- dandelion_memory-1.0.0rc15.dist-info/RECORD +232 -0
- dandelion_memory-1.0.0rc15.dist-info/WHEEL +5 -0
- dandelion_memory-1.0.0rc15.dist-info/entry_points.txt +5 -0
- dandelion_memory-1.0.0rc15.dist-info/licenses/LICENSE +21 -0
- dandelion_memory-1.0.0rc15.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
"""Frozen-copy importer from legacy JSON/Qdrant files into canonical SQLite.
|
|
2
|
+
|
|
3
|
+
This module is intentionally not wired to public CLI or MCP tools. Callers
|
|
4
|
+
must provide a copied, quiescent source workspace; the importer verifies the
|
|
5
|
+
source hash manifest again immediately before its one SQLite transaction.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import pickle
|
|
13
|
+
import re
|
|
14
|
+
import sqlite3
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Mapping
|
|
18
|
+
|
|
19
|
+
from dandelion.canonical_memory import CanonicalMemory, REF_RE
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
_SENSITIVE_SECURITY_EVIDENCE = re.compile(
|
|
23
|
+
r"(?:secret|token|password|passphrase|credential|api[_-]?key|private[_-]?key|bearer|auth(?:orization)?|env(?:ironment)?)",
|
|
24
|
+
re.I,
|
|
25
|
+
)
|
|
26
|
+
_SAFE_PUBLIC_FACT_KEY = re.compile(
|
|
27
|
+
r"(?:^|[._-])(?:map_[wh]|width|height|package_id|package_name|port|host|url|version)(?:$|[._-])",
|
|
28
|
+
re.I,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _classify_security(
|
|
33
|
+
explicit: object,
|
|
34
|
+
*,
|
|
35
|
+
key: str = "",
|
|
36
|
+
value: str = "",
|
|
37
|
+
source: str = "",
|
|
38
|
+
) -> tuple[str, str]:
|
|
39
|
+
"""Classify unlabeled legacy records without exposing their content.
|
|
40
|
+
|
|
41
|
+
An explicit legacy label is authoritative. Missing labels are public only
|
|
42
|
+
for a narrow set of ordinary configuration identifiers; credential-like
|
|
43
|
+
names, values, or sources fail closed.
|
|
44
|
+
"""
|
|
45
|
+
if isinstance(explicit, str) and explicit:
|
|
46
|
+
return explicit, "explicit_label"
|
|
47
|
+
evidence = " ".join((key, value, source))
|
|
48
|
+
if _SENSITIVE_SECURITY_EVIDENCE.search(evidence):
|
|
49
|
+
return "RESTRICTED", "sensitive_evidence_fail_closed"
|
|
50
|
+
if _SAFE_PUBLIC_FACT_KEY.search(key):
|
|
51
|
+
return "PUBLIC", "safe_key_evidence"
|
|
52
|
+
return "RESTRICTED", "insufficient_public_evidence_fail_closed"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class LegacyImportError(RuntimeError):
|
|
56
|
+
"""A frozen legacy source cannot be imported safely."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class LegacyImportSourceChanged(LegacyImportError):
|
|
60
|
+
"""A source file changed after preparation and before atomic apply."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class LegacyImportCollision(LegacyImportError):
|
|
64
|
+
"""A legacy stable ID would overwrite incompatible canonical state."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class LegacyImportUnsupported(LegacyImportError):
|
|
68
|
+
"""A source shape cannot be claimed losslessly by the bounded importer."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class PreparedLegacyImport:
|
|
73
|
+
"""Validated source rows plus a values-free source manifest."""
|
|
74
|
+
|
|
75
|
+
source_workspace: Path
|
|
76
|
+
source_digest: str
|
|
77
|
+
manifest: Mapping[str, Any]
|
|
78
|
+
facts: tuple[Mapping[str, Any], ...]
|
|
79
|
+
domains: tuple[Mapping[str, Any], ...]
|
|
80
|
+
memories: tuple[Mapping[str, Any], ...]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class LegacyImporter:
|
|
84
|
+
"""Prepare outside SQLite, then atomically apply to ``CanonicalMemory``."""
|
|
85
|
+
|
|
86
|
+
_FACTS_FILE = "facts.json"
|
|
87
|
+
_QDRANT_META = "qdrant/meta.json"
|
|
88
|
+
_QDRANT_STORAGE = "qdrant/collection/dandelion_petals/storage.sqlite"
|
|
89
|
+
_OPTIONAL_FILES = ("wal.jsonl",)
|
|
90
|
+
|
|
91
|
+
def __init__(self, source_workspace: Path | str, canonical: CanonicalMemory) -> None:
|
|
92
|
+
self.source_workspace = Path(source_workspace).resolve()
|
|
93
|
+
self.canonical = canonical
|
|
94
|
+
if self.source_workspace == canonical.workspace:
|
|
95
|
+
raise LegacyImportError("Legacy import source must be a frozen copy, not the target workspace")
|
|
96
|
+
self.source_dir = self.source_workspace / ".dandelion"
|
|
97
|
+
|
|
98
|
+
def preview(self) -> dict[str, Any]:
|
|
99
|
+
"""Validate the copy and return a values-free manifest without mutation."""
|
|
100
|
+
return dict(self.prepare().manifest)
|
|
101
|
+
|
|
102
|
+
def prepare(self) -> PreparedLegacyImport:
|
|
103
|
+
"""Read all supported source records outside the canonical transaction."""
|
|
104
|
+
source_files = self._source_files()
|
|
105
|
+
source_digest = _digest(source_files)
|
|
106
|
+
facts_doc = _load_json(self.source_dir / self._FACTS_FILE)
|
|
107
|
+
facts, domains, reverse_refs = self._prepare_facts(facts_doc)
|
|
108
|
+
memories = self._prepare_memories()
|
|
109
|
+
self._validate_refs(facts, reverse_refs, memories)
|
|
110
|
+
counts = {
|
|
111
|
+
"facts": len(facts),
|
|
112
|
+
"fact_revisions": sum(len(item["revisions"]) for item in facts),
|
|
113
|
+
"domains": len(domains),
|
|
114
|
+
"memories": len(memories),
|
|
115
|
+
"active_memories": sum(1 for item in memories if item["active"]),
|
|
116
|
+
"inactive_memories": sum(1 for item in memories if not item["active"]),
|
|
117
|
+
"memory_refs": sum(len(item["refs"]) for item in memories),
|
|
118
|
+
"reverse_ref_edges": sum(len(values) for values in reverse_refs.values()),
|
|
119
|
+
}
|
|
120
|
+
status_records = sum(
|
|
121
|
+
1 for item in memories
|
|
122
|
+
if item.get("metadata", {}).get("payload", {}).get("memory_kind") == "status"
|
|
123
|
+
)
|
|
124
|
+
classified_records = [*facts, *memories]
|
|
125
|
+
classification_counts: dict[str, int] = {}
|
|
126
|
+
classification_reasons: dict[str, int] = {}
|
|
127
|
+
for item in classified_records:
|
|
128
|
+
security_class = str(item["security_class"])
|
|
129
|
+
reason = str(item["security_classification_reason"])
|
|
130
|
+
classification_counts[security_class] = classification_counts.get(security_class, 0) + 1
|
|
131
|
+
classification_reasons[reason] = classification_reasons.get(reason, 0) + 1
|
|
132
|
+
unsupported = []
|
|
133
|
+
if status_records:
|
|
134
|
+
unsupported.append({
|
|
135
|
+
"kind": "structured_claim_materialization",
|
|
136
|
+
"records": status_records,
|
|
137
|
+
"preserved_as": "legacy_memory_metadata",
|
|
138
|
+
"query_parity": "DEFERRED",
|
|
139
|
+
})
|
|
140
|
+
manifest = {
|
|
141
|
+
"version": 1,
|
|
142
|
+
"import_id": f"legacy-{source_digest[:32]}",
|
|
143
|
+
"source_digest": source_digest,
|
|
144
|
+
"source_workspace_hash": _sha256(str(self.source_workspace).encode("utf-8")),
|
|
145
|
+
"files": source_files,
|
|
146
|
+
"counts": counts,
|
|
147
|
+
"security_classification": {
|
|
148
|
+
"counts": dict(sorted(classification_counts.items())),
|
|
149
|
+
"reasons": dict(sorted(classification_reasons.items())),
|
|
150
|
+
},
|
|
151
|
+
"unsupported": unsupported,
|
|
152
|
+
}
|
|
153
|
+
return PreparedLegacyImport(
|
|
154
|
+
source_workspace=self.source_workspace,
|
|
155
|
+
source_digest=source_digest,
|
|
156
|
+
manifest=manifest,
|
|
157
|
+
facts=tuple(facts),
|
|
158
|
+
domains=tuple(domains),
|
|
159
|
+
memories=tuple(memories),
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
def apply(self, prepared: PreparedLegacyImport | None = None) -> dict[str, Any]:
|
|
163
|
+
"""Recheck the frozen copy, then atomically persist canonical rows/outbox."""
|
|
164
|
+
prepared = prepared or self.prepare()
|
|
165
|
+
if prepared.source_workspace != self.source_workspace:
|
|
166
|
+
raise LegacyImportError("Prepared import belongs to another source workspace")
|
|
167
|
+
current_files = self._source_files()
|
|
168
|
+
if current_files != prepared.manifest["files"] or _digest(current_files) != prepared.source_digest:
|
|
169
|
+
raise LegacyImportSourceChanged("Legacy import source changed after prepare")
|
|
170
|
+
try:
|
|
171
|
+
return self.canonical.apply_legacy_import(
|
|
172
|
+
source_digest=prepared.source_digest,
|
|
173
|
+
manifest=prepared.manifest,
|
|
174
|
+
facts=prepared.facts,
|
|
175
|
+
domains=prepared.domains,
|
|
176
|
+
memories=prepared.memories,
|
|
177
|
+
)
|
|
178
|
+
except (ValueError, sqlite3.IntegrityError) as exc:
|
|
179
|
+
raise LegacyImportCollision("Legacy import collision; canonical state was not changed") from exc
|
|
180
|
+
|
|
181
|
+
def _source_files(self) -> dict[str, dict[str, Any]]:
|
|
182
|
+
if not self.source_dir.is_dir():
|
|
183
|
+
raise LegacyImportError("Frozen source is missing its .dandelion directory")
|
|
184
|
+
names = [self._FACTS_FILE, self._QDRANT_META, self._QDRANT_STORAGE]
|
|
185
|
+
names.extend(name for name in self._OPTIONAL_FILES if (self.source_dir / name).is_file())
|
|
186
|
+
files: dict[str, dict[str, Any]] = {}
|
|
187
|
+
for name in names:
|
|
188
|
+
path = self.source_dir / name
|
|
189
|
+
if not path.is_file():
|
|
190
|
+
raise LegacyImportUnsupported(f"Required frozen source file is missing: {name}")
|
|
191
|
+
files[name] = {"sha256": _sha256(path.read_bytes()), "bytes": path.stat().st_size}
|
|
192
|
+
return dict(sorted(files.items()))
|
|
193
|
+
|
|
194
|
+
def _prepare_facts(
|
|
195
|
+
self,
|
|
196
|
+
document: Mapping[str, Any],
|
|
197
|
+
) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]], dict[str, tuple[str, ...]]]:
|
|
198
|
+
if not isinstance(document, dict):
|
|
199
|
+
raise LegacyImportUnsupported("facts.json root must be an object")
|
|
200
|
+
raw_domains = document.get("__domains__", {})
|
|
201
|
+
raw_refs = document.get("__fact_refs__", {})
|
|
202
|
+
if not isinstance(raw_domains, dict) or not isinstance(raw_refs, dict):
|
|
203
|
+
raise LegacyImportUnsupported("Legacy domains and reverse refs must be objects")
|
|
204
|
+
|
|
205
|
+
domains: list[Mapping[str, Any]] = []
|
|
206
|
+
for name, metadata in sorted(raw_domains.items()):
|
|
207
|
+
if not isinstance(name, str) or not isinstance(metadata, dict):
|
|
208
|
+
raise LegacyImportUnsupported("Legacy domain metadata is malformed")
|
|
209
|
+
domains.append({
|
|
210
|
+
"name": name,
|
|
211
|
+
"canonical_id": f"legacy-domain-{_sha256(name.encode('utf-8'))[:32]}",
|
|
212
|
+
"metadata": metadata,
|
|
213
|
+
"content_hash": _hash_json(metadata),
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
facts: list[Mapping[str, Any]] = []
|
|
217
|
+
for key, raw in sorted(document.items()):
|
|
218
|
+
if key in {"__domains__", "__fact_refs__"}:
|
|
219
|
+
continue
|
|
220
|
+
if not isinstance(key, str) or not isinstance(raw, dict):
|
|
221
|
+
raise LegacyImportUnsupported("Only structured legacy fact records are supported")
|
|
222
|
+
revisions = raw.get("revisions")
|
|
223
|
+
if not isinstance(revisions, list) or not revisions:
|
|
224
|
+
raise LegacyImportUnsupported("Legacy fact revisions are required for lossless import")
|
|
225
|
+
prepared_revisions: list[Mapping[str, Any]] = []
|
|
226
|
+
for position, revision in enumerate(revisions, start=1):
|
|
227
|
+
if not isinstance(revision, dict):
|
|
228
|
+
raise LegacyImportUnsupported("Legacy fact revision is malformed")
|
|
229
|
+
try:
|
|
230
|
+
number = int(revision["rev"])
|
|
231
|
+
value = revision["value"]
|
|
232
|
+
created_at = float(revision["created_at"])
|
|
233
|
+
valid_from = float(revision["valid_from"])
|
|
234
|
+
valid_until = revision.get("valid_until")
|
|
235
|
+
valid_until = float(valid_until) if valid_until is not None else None
|
|
236
|
+
source = str(revision["source"])
|
|
237
|
+
confidence = float(revision["confidence"])
|
|
238
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
239
|
+
raise LegacyImportUnsupported("Legacy fact revision fields are malformed") from exc
|
|
240
|
+
if number != position or not isinstance(value, str) or valid_until is not None and valid_until < valid_from:
|
|
241
|
+
raise LegacyImportUnsupported("Legacy revision sequence or value type is unsupported")
|
|
242
|
+
prepared_revisions.append({
|
|
243
|
+
"revision": number,
|
|
244
|
+
"value": value,
|
|
245
|
+
"source": source,
|
|
246
|
+
"confidence": confidence,
|
|
247
|
+
"valid_from": valid_from,
|
|
248
|
+
"valid_until": valid_until,
|
|
249
|
+
"metadata": revision,
|
|
250
|
+
"created_at": created_at,
|
|
251
|
+
})
|
|
252
|
+
try:
|
|
253
|
+
current_revision = int(raw["revision"])
|
|
254
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
255
|
+
raise LegacyImportUnsupported("Legacy fact current revision is malformed") from exc
|
|
256
|
+
if current_revision != len(prepared_revisions):
|
|
257
|
+
raise LegacyImportUnsupported("Legacy fact current revision does not match revision history")
|
|
258
|
+
security_class, security_reason = _classify_security(
|
|
259
|
+
raw.get("security_class"), key=key,
|
|
260
|
+
value=" ".join(str(item["value"]) for item in prepared_revisions),
|
|
261
|
+
source=" ".join(str(item["source"]) for item in prepared_revisions),
|
|
262
|
+
)
|
|
263
|
+
facts.append({
|
|
264
|
+
"key": key,
|
|
265
|
+
"current_revision": current_revision,
|
|
266
|
+
"security_class": security_class,
|
|
267
|
+
"security_classification_reason": security_reason,
|
|
268
|
+
"metadata": raw,
|
|
269
|
+
"revisions": prepared_revisions,
|
|
270
|
+
"content_hash": _hash_json(raw),
|
|
271
|
+
})
|
|
272
|
+
|
|
273
|
+
reverse_refs: dict[str, tuple[str, ...]] = {}
|
|
274
|
+
for key, memory_ids in raw_refs.items():
|
|
275
|
+
if not isinstance(key, str) or not isinstance(memory_ids, list) or not all(isinstance(item, str) for item in memory_ids):
|
|
276
|
+
raise LegacyImportUnsupported("Legacy reverse reference is malformed")
|
|
277
|
+
reverse_refs[key] = tuple(memory_ids)
|
|
278
|
+
return facts, domains, reverse_refs
|
|
279
|
+
|
|
280
|
+
def _prepare_memories(self) -> list[Mapping[str, Any]]:
|
|
281
|
+
storage = self.source_dir / self._QDRANT_STORAGE
|
|
282
|
+
try:
|
|
283
|
+
database = sqlite3.connect(f"file:{storage}?mode=ro", uri=True)
|
|
284
|
+
rows = database.execute("SELECT point FROM points").fetchall()
|
|
285
|
+
except sqlite3.Error as exc:
|
|
286
|
+
raise LegacyImportUnsupported("Frozen Qdrant point storage cannot be read") from exc
|
|
287
|
+
finally:
|
|
288
|
+
try:
|
|
289
|
+
database.close()
|
|
290
|
+
except UnboundLocalError:
|
|
291
|
+
pass
|
|
292
|
+
|
|
293
|
+
memories: list[Mapping[str, Any]] = []
|
|
294
|
+
seen_ids: set[str] = set()
|
|
295
|
+
for (serialized_point,) in rows:
|
|
296
|
+
try:
|
|
297
|
+
# Qdrant local storage in this project serializes PointStruct
|
|
298
|
+
# with pickle. The caller's frozen COPY is therefore trusted
|
|
299
|
+
# local input; arbitrary/untrusted pickle is unsupported.
|
|
300
|
+
point = pickle.loads(serialized_point)
|
|
301
|
+
point_id = str(point.id)
|
|
302
|
+
payload = point.payload
|
|
303
|
+
except Exception as exc:
|
|
304
|
+
raise LegacyImportUnsupported("Frozen Qdrant point encoding is unsupported") from exc
|
|
305
|
+
if not point_id or point_id in seen_ids or not isinstance(payload, dict):
|
|
306
|
+
raise LegacyImportUnsupported("Frozen Qdrant point identity or payload is malformed")
|
|
307
|
+
seen_ids.add(point_id)
|
|
308
|
+
required = ("domain", "template", "is_active", "created_at", "supersedes")
|
|
309
|
+
if any(field not in payload for field in required):
|
|
310
|
+
raise LegacyImportUnsupported("Frozen Qdrant point is missing required semantic fields")
|
|
311
|
+
if not isinstance(payload["domain"], str) or not isinstance(payload["template"], str):
|
|
312
|
+
raise LegacyImportUnsupported("Frozen Qdrant semantic fields have unsupported types")
|
|
313
|
+
if not isinstance(payload["is_active"], bool):
|
|
314
|
+
raise LegacyImportUnsupported("Frozen Qdrant active state must be boolean")
|
|
315
|
+
try:
|
|
316
|
+
created_at = float(payload["created_at"])
|
|
317
|
+
except (TypeError, ValueError) as exc:
|
|
318
|
+
raise LegacyImportUnsupported("Frozen Qdrant created_at is malformed") from exc
|
|
319
|
+
refs = payload.get("referenced_facts", [])
|
|
320
|
+
if not isinstance(refs, list) or not all(isinstance(item, str) for item in refs):
|
|
321
|
+
raise LegacyImportUnsupported("Frozen Qdrant referenced_facts is malformed")
|
|
322
|
+
if payload.get("supersedes") is not None and not isinstance(payload.get("supersedes"), str):
|
|
323
|
+
raise LegacyImportUnsupported("Frozen Qdrant supersedes is malformed")
|
|
324
|
+
_assert_json(payload)
|
|
325
|
+
typed_refs = _typed_refs(payload["template"], refs)
|
|
326
|
+
metadata = {"payload": payload, "vector_dimensions": len(point.vector)}
|
|
327
|
+
security_class, security_reason = _classify_security(
|
|
328
|
+
payload.get("security_class"), key=f"{payload.get('domain', '')}.{payload.get('source_component', '')}",
|
|
329
|
+
value=str(payload["template"]), source=str(payload.get("source", "legacy_qdrant")),
|
|
330
|
+
)
|
|
331
|
+
memories.append({
|
|
332
|
+
"id": point_id,
|
|
333
|
+
"domain": payload["domain"],
|
|
334
|
+
"template": payload["template"],
|
|
335
|
+
"source": payload.get("source") if isinstance(payload.get("source"), str) else "legacy_qdrant",
|
|
336
|
+
"authority": _payload_authority(payload),
|
|
337
|
+
"security_class": security_class,
|
|
338
|
+
"security_classification_reason": security_reason,
|
|
339
|
+
"active": payload["is_active"],
|
|
340
|
+
"created_at": created_at,
|
|
341
|
+
"supersedes": payload.get("supersedes"),
|
|
342
|
+
"refs": typed_refs,
|
|
343
|
+
"metadata": metadata,
|
|
344
|
+
"content_hash": _hash_json({"id": point_id, "payload": payload}),
|
|
345
|
+
})
|
|
346
|
+
return memories
|
|
347
|
+
|
|
348
|
+
@staticmethod
|
|
349
|
+
def _validate_refs(
|
|
350
|
+
facts: list[Mapping[str, Any]],
|
|
351
|
+
reverse_refs: Mapping[str, tuple[str, ...]],
|
|
352
|
+
memories: list[Mapping[str, Any]],
|
|
353
|
+
) -> None:
|
|
354
|
+
fact_keys = {item["key"] for item in facts}
|
|
355
|
+
memory_ids = {item["id"] for item in memories}
|
|
356
|
+
for fact_key, ids in reverse_refs.items():
|
|
357
|
+
if fact_key not in fact_keys or any(memory_id not in memory_ids for memory_id in ids):
|
|
358
|
+
raise LegacyImportUnsupported("Legacy reverse reference is unresolved")
|
|
359
|
+
expected_reverse: dict[str, set[str]] = {}
|
|
360
|
+
for memory in memories:
|
|
361
|
+
seen_ref_modes: set[tuple[str, str]] = set()
|
|
362
|
+
for ref in memory["refs"]:
|
|
363
|
+
if ref["fact_key"] not in fact_keys:
|
|
364
|
+
raise LegacyImportUnsupported("Semantic reference targets an absent legacy fact")
|
|
365
|
+
identity = (ref["fact_key"], ref["mode"])
|
|
366
|
+
if identity in seen_ref_modes:
|
|
367
|
+
raise LegacyImportUnsupported("Multiple selectors for one legacy fact are unsupported")
|
|
368
|
+
seen_ref_modes.add(identity)
|
|
369
|
+
expected_reverse.setdefault(ref["fact_key"], set()).add(memory["id"])
|
|
370
|
+
for fact_key in set(reverse_refs) | set(expected_reverse):
|
|
371
|
+
if set(reverse_refs.get(fact_key, ())) != expected_reverse.get(fact_key, set()):
|
|
372
|
+
raise LegacyImportUnsupported("Legacy reverse index and semantic references disagree")
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _typed_refs(template: str, explicit_refs: list[str]) -> list[Mapping[str, Any]]:
|
|
376
|
+
selectors = {match.group(1): match.group(2) for match in REF_RE.finditer(template)}
|
|
377
|
+
result: list[Mapping[str, Any]] = []
|
|
378
|
+
for key in explicit_refs:
|
|
379
|
+
selector = selectors.get(key)
|
|
380
|
+
result.append({
|
|
381
|
+
"fact_key": key,
|
|
382
|
+
"mode": "snapshot" if selector else "live",
|
|
383
|
+
"pinned_revision": int(selector) if selector else None,
|
|
384
|
+
})
|
|
385
|
+
return result
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _payload_authority(payload: Mapping[str, Any]) -> str:
|
|
389
|
+
provenance = payload.get("provenance")
|
|
390
|
+
if isinstance(provenance, dict) and isinstance(provenance.get("authority"), str):
|
|
391
|
+
return provenance["authority"]
|
|
392
|
+
return "LEGACY_IMPORT"
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _load_json(path: Path) -> Mapping[str, Any]:
|
|
396
|
+
try:
|
|
397
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
398
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
399
|
+
raise LegacyImportUnsupported("Frozen facts.json cannot be read") from exc
|
|
400
|
+
if not isinstance(value, dict):
|
|
401
|
+
raise LegacyImportUnsupported("facts.json root must be an object")
|
|
402
|
+
return value
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _assert_json(value: Any) -> None:
|
|
406
|
+
try:
|
|
407
|
+
json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
408
|
+
except (TypeError, ValueError) as exc:
|
|
409
|
+
raise LegacyImportUnsupported("Frozen Qdrant payload is not JSON data") from exc
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _hash_json(value: Any) -> str:
|
|
413
|
+
return _sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8"))
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _sha256(value: bytes) -> str:
|
|
417
|
+
return hashlib.sha256(value).hexdigest()
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _digest(files: Mapping[str, Mapping[str, Any]]) -> str:
|
|
421
|
+
return _hash_json(files)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
__all__ = [
|
|
425
|
+
"LegacyImportCollision",
|
|
426
|
+
"LegacyImportError",
|
|
427
|
+
"LegacyImporter",
|
|
428
|
+
"LegacyImportSourceChanged",
|
|
429
|
+
"LegacyImportUnsupported",
|
|
430
|
+
"PreparedLegacyImport",
|
|
431
|
+
]
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Fresh SQLite grant projections; cached parsing never authorizes a source."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
import threading
|
|
9
|
+
from collections import OrderedDict
|
|
10
|
+
from copy import deepcopy
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from dandelion.source_reader import safe_relative
|
|
15
|
+
from dandelion.request_budget import check_deadline
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _parse_catalog(catalog: dict, policy: str | None, owner_policy: str | None) -> dict:
|
|
19
|
+
if not isinstance(catalog, dict):
|
|
20
|
+
return {}
|
|
21
|
+
return {
|
|
22
|
+
path: entry for path, entry in sorted(catalog.items())
|
|
23
|
+
if isinstance(path, str) and safe_relative(path) is not None
|
|
24
|
+
and isinstance(entry, dict)
|
|
25
|
+
and entry.get("source_policy") in (None, "PUBLIC_RANGES_V1")
|
|
26
|
+
and entry.get("source_policy") == policy
|
|
27
|
+
and entry.get("source_policy") in (None, owner_policy)
|
|
28
|
+
and re.fullmatch(r"[0-9a-f]{64}", str(entry.get("sha256", "")))
|
|
29
|
+
and isinstance(entry.get("line_count"), int)
|
|
30
|
+
and not isinstance(entry.get("line_count"), bool)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# Cache decoded grant fields only, never the much larger capture-locator manifest.
|
|
35
|
+
# Every lookup still reads the actual latest row and its live withdrawal state.
|
|
36
|
+
_projection_cache: OrderedDict[tuple[str, str | None], tuple[dict, Any, Any]] = OrderedDict()
|
|
37
|
+
_projection_cache_lock = threading.Lock()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _reject_non_json_constant(value: str) -> None:
|
|
41
|
+
raise ValueError("Invalid JSON constant")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _manifest_projection(raw: str, owner_policy: str | None) -> tuple[dict, Any, Any] | None:
|
|
45
|
+
key = (hashlib.sha256(raw.encode("utf-8")).hexdigest(), owner_policy)
|
|
46
|
+
with _projection_cache_lock:
|
|
47
|
+
cached = _projection_cache.get(key)
|
|
48
|
+
if cached is not None:
|
|
49
|
+
_projection_cache.move_to_end(key)
|
|
50
|
+
return cached
|
|
51
|
+
check_deadline()
|
|
52
|
+
try:
|
|
53
|
+
manifest = json.loads(raw, parse_constant=_reject_non_json_constant)
|
|
54
|
+
except (ValueError, TypeError, RecursionError):
|
|
55
|
+
return None
|
|
56
|
+
if not isinstance(manifest, dict):
|
|
57
|
+
return None
|
|
58
|
+
catalog = manifest.get("source_catalog")
|
|
59
|
+
policy = manifest.get("source_policy")
|
|
60
|
+
projection = manifest.get("source_content_projection")
|
|
61
|
+
if (not isinstance(catalog, dict) or policy is not None and not isinstance(policy, str)
|
|
62
|
+
or projection is not None and not isinstance(projection, str)):
|
|
63
|
+
return None
|
|
64
|
+
parsed = (_parse_catalog(catalog, policy, owner_policy), policy, projection)
|
|
65
|
+
check_deadline()
|
|
66
|
+
# Larger catalogs remain usable without retention; unrelated manifest size
|
|
67
|
+
# does not prevent reuse of a small grant projection.
|
|
68
|
+
if len(json.dumps((catalog, policy, projection), ensure_ascii=True).encode("ascii")) <= 2 * 1024 * 1024:
|
|
69
|
+
with _projection_cache_lock:
|
|
70
|
+
_projection_cache[key] = parsed
|
|
71
|
+
_projection_cache.move_to_end(key)
|
|
72
|
+
while len(_projection_cache) > 4:
|
|
73
|
+
_projection_cache.popitem(last=False)
|
|
74
|
+
return parsed
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def live_source_grants(owner: Any, requested: Path | None = None, *,
|
|
78
|
+
paths: list[str] | None = None) -> list[dict[str, Any]]:
|
|
79
|
+
"""Read each actual latest row, including same-digest edits and revocations.
|
|
80
|
+
|
|
81
|
+
Projection reuse binds to the full current manifest bytes, not its mutable
|
|
82
|
+
plan digest or row identity. JSON decoding keeps duplicate-key last values.
|
|
83
|
+
Invalid latest JSON yields no grant, never a previous committed row.
|
|
84
|
+
"""
|
|
85
|
+
check_deadline()
|
|
86
|
+
if requested is None:
|
|
87
|
+
selection = (
|
|
88
|
+
"SELECT b.source_workspace,b.manifest_json,b.plan_digest "
|
|
89
|
+
"FROM canonical_bootstrap_runs b WHERE b.plan_digest=("
|
|
90
|
+
"SELECT b2.plan_digest FROM canonical_bootstrap_runs b2 "
|
|
91
|
+
"WHERE b2.source_workspace=b.source_workspace "
|
|
92
|
+
"ORDER BY b2.committed_at DESC,b2.plan_digest DESC LIMIT 1)"
|
|
93
|
+
)
|
|
94
|
+
parameters = ()
|
|
95
|
+
else:
|
|
96
|
+
# Sort row identities before fetching a potentially large manifest.
|
|
97
|
+
selection = (
|
|
98
|
+
"SELECT source_workspace,manifest_json,plan_digest "
|
|
99
|
+
"FROM canonical_bootstrap_runs WHERE rowid=("
|
|
100
|
+
"SELECT rowid FROM canonical_bootstrap_runs WHERE source_workspace=? "
|
|
101
|
+
"ORDER BY committed_at DESC,plan_digest DESC LIMIT 1)"
|
|
102
|
+
)
|
|
103
|
+
parameters = (str(requested),)
|
|
104
|
+
sql = (
|
|
105
|
+
"SELECT b.source_workspace,b.plan_digest,b.manifest_json FROM ("
|
|
106
|
+
+ selection + ") b "
|
|
107
|
+
"WHERE NOT EXISTS (SELECT 1 FROM canonical_source_withdrawals w "
|
|
108
|
+
"WHERE w.source_workspace=b.source_workspace) "
|
|
109
|
+
"ORDER BY b.source_workspace"
|
|
110
|
+
)
|
|
111
|
+
with owner._connect() as db:
|
|
112
|
+
rows = db.execute(sql, parameters).fetchall()
|
|
113
|
+
check_deadline()
|
|
114
|
+
grants = []
|
|
115
|
+
for raw_root, digest, raw in rows:
|
|
116
|
+
check_deadline()
|
|
117
|
+
try:
|
|
118
|
+
root = Path(raw_root).resolve(strict=True)
|
|
119
|
+
if not root.is_dir() or requested is not None and root != requested:
|
|
120
|
+
continue
|
|
121
|
+
parsed = _manifest_projection(raw, owner.source_policy)
|
|
122
|
+
if parsed is None:
|
|
123
|
+
continue
|
|
124
|
+
catalog, policy, projection = parsed
|
|
125
|
+
check_deadline()
|
|
126
|
+
selected = catalog if paths is None else {path: catalog[path] for path in paths if path in catalog}
|
|
127
|
+
grants.append({"lineage_root": str(root), "plan_digest": digest,
|
|
128
|
+
"source_policy": policy, "source_content_projection": projection,
|
|
129
|
+
"source_catalog": deepcopy(selected)})
|
|
130
|
+
except (OSError, RuntimeError, ValueError, TypeError):
|
|
131
|
+
continue
|
|
132
|
+
check_deadline()
|
|
133
|
+
return grants
|
dandelion/local_owner.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Explicit lexical owner lifetime for a local MCP host, not a detached daemon."""
|
|
2
|
+
|
|
3
|
+
from contextlib import contextmanager
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
from dandelion.memory_client import MemoryClient, MemoryClientError, OwnerUnavailable
|
|
9
|
+
from dandelion.memory_owner import MemoryOwner, OwnerAlreadyRunning, OwnerDescriptorError, MemoryOwnerError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class LocalOwnerSession:
|
|
13
|
+
"""A workspace lease, independent of which connected MCP process hosts it."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, workspace: Path, *, lifecycle_v1: bool, access_mode: str):
|
|
16
|
+
self.workspace = workspace.resolve()
|
|
17
|
+
if not self.workspace.is_dir():
|
|
18
|
+
raise ValueError("local owner requires an existing workspace")
|
|
19
|
+
self.lifecycle_v1 = lifecycle_v1
|
|
20
|
+
self.access_mode = access_mode
|
|
21
|
+
self.owner = None
|
|
22
|
+
self.lock = threading.RLock()
|
|
23
|
+
|
|
24
|
+
def _start(self, owner):
|
|
25
|
+
from dandelion.canonical_memory import CanonicalMemory
|
|
26
|
+
from dandelion.derived_index import CanonicalDerivedService
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
canonical = CanonicalMemory(self.workspace, lifecycle_v1_enabled=self.lifecycle_v1,
|
|
30
|
+
access_mode=self.access_mode)
|
|
31
|
+
owner.service = CanonicalDerivedService(canonical, None)
|
|
32
|
+
owner.start()
|
|
33
|
+
except Exception:
|
|
34
|
+
owner.abort_start()
|
|
35
|
+
raise
|
|
36
|
+
self.owner = owner
|
|
37
|
+
return MemoryClient(owner.descriptor)
|
|
38
|
+
|
|
39
|
+
def connect(self, *, deadline=None):
|
|
40
|
+
deadline = deadline if deadline is not None else time.monotonic() + 10
|
|
41
|
+
with self.lock:
|
|
42
|
+
while time.monotonic() < deadline:
|
|
43
|
+
if self.owner is not None and self.owner.is_running:
|
|
44
|
+
return MemoryClient(self.owner.descriptor)
|
|
45
|
+
try:
|
|
46
|
+
client = MemoryClient.for_workspace(
|
|
47
|
+
self.workspace, timeout_seconds=deadline - time.monotonic())
|
|
48
|
+
status = client.status()
|
|
49
|
+
except (OwnerUnavailable, OwnerDescriptorError):
|
|
50
|
+
contender = MemoryOwner(self.workspace, None)
|
|
51
|
+
try:
|
|
52
|
+
contender.acquire_guard()
|
|
53
|
+
except OwnerAlreadyRunning:
|
|
54
|
+
# A different adapter may be initializing or taking over.
|
|
55
|
+
time.sleep(min(0.1, max(0, deadline - time.monotonic())))
|
|
56
|
+
continue
|
|
57
|
+
return self._start(contender)
|
|
58
|
+
if (status.get("owner_id") != client.descriptor.owner_id
|
|
59
|
+
or Path(status["workspace"]).resolve() != self.workspace
|
|
60
|
+
or status.get("access_mode", "strict") != self.access_mode
|
|
61
|
+
or (self.lifecycle_v1 and not status.get("lifecycle_v1_enabled"))):
|
|
62
|
+
raise ValueError("existing owner does not match requested workspace/mode/capabilities")
|
|
63
|
+
return client
|
|
64
|
+
raise OwnerUnavailable("existing memory owner did not become ready")
|
|
65
|
+
|
|
66
|
+
def client(self):
|
|
67
|
+
return _LocalMemoryClient(self, self.connect().descriptor)
|
|
68
|
+
|
|
69
|
+
def close(self):
|
|
70
|
+
with self.lock:
|
|
71
|
+
if self.owner is not None:
|
|
72
|
+
self.owner.stop()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _LocalMemoryClient(MemoryClient):
|
|
76
|
+
"""Rebind before dispatch; never replay an operation after a transport failure."""
|
|
77
|
+
|
|
78
|
+
def __init__(self, session, descriptor):
|
|
79
|
+
super().__init__(descriptor)
|
|
80
|
+
self.session = session
|
|
81
|
+
|
|
82
|
+
def call(self, operation, payload=None, **kwargs):
|
|
83
|
+
requested_timeout = kwargs.get("_timeout_seconds")
|
|
84
|
+
timeout = min(self.timeout_seconds, requested_timeout if requested_timeout is not None else self.timeout_seconds)
|
|
85
|
+
if timeout <= 0:
|
|
86
|
+
raise ValueError("request timeout must be positive")
|
|
87
|
+
requested_deadline = kwargs.get("_deadline")
|
|
88
|
+
deadline = min(requested_deadline if requested_deadline is not None else float("inf"),
|
|
89
|
+
time.monotonic() + timeout)
|
|
90
|
+
if time.monotonic() >= deadline:
|
|
91
|
+
raise OwnerUnavailable("Memory owner request deadline exceeded")
|
|
92
|
+
try:
|
|
93
|
+
descriptor = self.session.connect(deadline=deadline).descriptor
|
|
94
|
+
except (ValueError, MemoryOwnerError) as exc:
|
|
95
|
+
raise MemoryClientError("Local owner binding failed: " + str(exc)) from exc
|
|
96
|
+
# Use a request-local client: simultaneous calls must not exchange identities.
|
|
97
|
+
self.descriptor = descriptor
|
|
98
|
+
kwargs["_timeout_seconds"] = deadline - time.monotonic()
|
|
99
|
+
if kwargs["_timeout_seconds"] <= 0:
|
|
100
|
+
raise OwnerUnavailable("Memory owner request deadline exceeded")
|
|
101
|
+
client = MemoryClient(descriptor, timeout_seconds=self.timeout_seconds,
|
|
102
|
+
max_message_bytes=self.max_message_bytes)
|
|
103
|
+
return client.call(operation, payload, **kwargs)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@contextmanager
|
|
107
|
+
def local_owner(workspace: Path, *, lifecycle_v1: bool, access_mode: str):
|
|
108
|
+
"""Enable local ownership and safe pre-dispatch takeover for this connection."""
|
|
109
|
+
session = LocalOwnerSession(workspace, lifecycle_v1=lifecycle_v1, access_mode=access_mode)
|
|
110
|
+
try:
|
|
111
|
+
session.connect()
|
|
112
|
+
yield session
|
|
113
|
+
finally:
|
|
114
|
+
session.close()
|