crprotocol 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. crp/__init__.py +126 -0
  2. crp/__main__.py +8 -0
  3. crp/_typing.py +27 -0
  4. crp/_version.py +5 -0
  5. crp/adapters.py +31 -0
  6. crp/advanced/__init__.py +40 -0
  7. crp/advanced/auto_ingest.py +400 -0
  8. crp/advanced/cqs.py +235 -0
  9. crp/advanced/cross_window.py +477 -0
  10. crp/advanced/curator.py +265 -0
  11. crp/advanced/feedback.py +146 -0
  12. crp/advanced/hierarchical.py +211 -0
  13. crp/advanced/meta_learning.py +401 -0
  14. crp/advanced/parallel.py +98 -0
  15. crp/advanced/review_cycle.py +329 -0
  16. crp/advanced/scale_mode.py +129 -0
  17. crp/advanced/source_grounding.py +207 -0
  18. crp/ckf/__init__.py +35 -0
  19. crp/ckf/community.py +377 -0
  20. crp/ckf/fabric.py +445 -0
  21. crp/ckf/gc.py +175 -0
  22. crp/ckf/graph_walk.py +87 -0
  23. crp/ckf/merge.py +133 -0
  24. crp/ckf/pattern_query.py +122 -0
  25. crp/ckf/pubsub.py +128 -0
  26. crp/ckf/semantic.py +207 -0
  27. crp/cli/__init__.py +7 -0
  28. crp/cli/main.py +329 -0
  29. crp/cli/sidecar.py +929 -0
  30. crp/cli/startup.py +272 -0
  31. crp/continuation/__init__.py +103 -0
  32. crp/continuation/completion.py +348 -0
  33. crp/continuation/degradation.py +157 -0
  34. crp/continuation/document_map.py +160 -0
  35. crp/continuation/flow.py +109 -0
  36. crp/continuation/gap.py +419 -0
  37. crp/continuation/manager.py +484 -0
  38. crp/continuation/quality_monitor.py +179 -0
  39. crp/continuation/stitch.py +419 -0
  40. crp/continuation/trigger.py +142 -0
  41. crp/continuation/voice.py +157 -0
  42. crp/core/__init__.py +69 -0
  43. crp/core/batch.py +77 -0
  44. crp/core/circuit_breaker.py +116 -0
  45. crp/core/config.py +377 -0
  46. crp/core/context_tools.py +540 -0
  47. crp/core/dispatch_router.py +3977 -0
  48. crp/core/errors.py +128 -0
  49. crp/core/extraction_facade.py +384 -0
  50. crp/core/facilitator.py +713 -0
  51. crp/core/idempotency.py +215 -0
  52. crp/core/orchestrator.py +1435 -0
  53. crp/core/relay_strategies.py +613 -0
  54. crp/core/security_manager.py +140 -0
  55. crp/core/session.py +134 -0
  56. crp/core/task_intent.py +36 -0
  57. crp/core/window.py +363 -0
  58. crp/envelope/__init__.py +30 -0
  59. crp/envelope/builder.py +288 -0
  60. crp/envelope/decomposer.py +236 -0
  61. crp/envelope/formatter.py +168 -0
  62. crp/envelope/packer.py +211 -0
  63. crp/envelope/reranker.py +209 -0
  64. crp/envelope/scoring.py +310 -0
  65. crp/extraction/__init__.py +45 -0
  66. crp/extraction/complexity.py +96 -0
  67. crp/extraction/contradiction.py +132 -0
  68. crp/extraction/pipeline.py +360 -0
  69. crp/extraction/quality_gate.py +237 -0
  70. crp/extraction/stage1_regex.py +173 -0
  71. crp/extraction/stage2_statistical.py +244 -0
  72. crp/extraction/stage3_gliner.py +210 -0
  73. crp/extraction/stage4_uie.py +183 -0
  74. crp/extraction/stage5_discourse.py +175 -0
  75. crp/extraction/stage6_llm.py +178 -0
  76. crp/extraction/structured_output.py +219 -0
  77. crp/extraction/types.py +299 -0
  78. crp/license_guard.py +722 -0
  79. crp/observability/__init__.py +30 -0
  80. crp/observability/audit.py +118 -0
  81. crp/observability/events.py +233 -0
  82. crp/observability/metrics.py +264 -0
  83. crp/observability/quality.py +135 -0
  84. crp/observability/structured_logging.py +81 -0
  85. crp/observability/telemetry.py +117 -0
  86. crp/provenance/__init__.py +314 -0
  87. crp/provenance/_embeddings.py +97 -0
  88. crp/provenance/_types.py +378 -0
  89. crp/provenance/attribution_scorer.py +252 -0
  90. crp/provenance/claim_detector.py +229 -0
  91. crp/provenance/contradiction_detector.py +243 -0
  92. crp/provenance/distortion_detector.py +397 -0
  93. crp/provenance/entailment_verifier.py +358 -0
  94. crp/provenance/fabrication_detector.py +203 -0
  95. crp/provenance/hallucination_scorer.py +320 -0
  96. crp/provenance/omission_analyzer.py +106 -0
  97. crp/provenance/provenance_chain.py +205 -0
  98. crp/provenance/report_generator.py +440 -0
  99. crp/providers/__init__.py +43 -0
  100. crp/providers/anthropic.py +270 -0
  101. crp/providers/base.py +135 -0
  102. crp/providers/custom.py +63 -0
  103. crp/providers/diagnostic.py +251 -0
  104. crp/providers/llamacpp.py +224 -0
  105. crp/providers/manager.py +139 -0
  106. crp/providers/ollama.py +243 -0
  107. crp/providers/openai.py +628 -0
  108. crp/providers/tokenizers.py +48 -0
  109. crp/py.typed +0 -0
  110. crp/resources/__init__.py +53 -0
  111. crp/resources/adaptive_allocator.py +525 -0
  112. crp/resources/cost_model.py +388 -0
  113. crp/resources/overhead_manager.py +217 -0
  114. crp/resources/resource_manager.py +262 -0
  115. crp/schemas/__init__.py +20 -0
  116. crp/schemas/cost-estimate.json +33 -0
  117. crp/schemas/crp-error.json +43 -0
  118. crp/schemas/envelope-preview.json +40 -0
  119. crp/schemas/persisted-state-header.json +27 -0
  120. crp/schemas/quality-report.json +94 -0
  121. crp/schemas/session-handle.json +33 -0
  122. crp/schemas/session-status.json +57 -0
  123. crp/schemas/stream-event.json +18 -0
  124. crp/schemas/task-intent.json +42 -0
  125. crp/security/__init__.py +93 -0
  126. crp/security/audit_trail.py +392 -0
  127. crp/security/binding.py +192 -0
  128. crp/security/compliance.py +813 -0
  129. crp/security/consent.py +593 -0
  130. crp/security/embedding_defense.py +161 -0
  131. crp/security/encryption.py +202 -0
  132. crp/security/injection.py +335 -0
  133. crp/security/integrity.py +267 -0
  134. crp/security/privacy.py +662 -0
  135. crp/security/quarantine.py +249 -0
  136. crp/security/rbac.py +221 -0
  137. crp/security/validation.py +164 -0
  138. crp/state/__init__.py +31 -0
  139. crp/state/cold_storage.py +258 -0
  140. crp/state/compaction.py +263 -0
  141. crp/state/critical_state.py +104 -0
  142. crp/state/event_log.py +313 -0
  143. crp/state/fact.py +189 -0
  144. crp/state/serialization.py +189 -0
  145. crp/state/session_cleanup.py +77 -0
  146. crp/state/snapshot.py +290 -0
  147. crp/state/warm_store.py +346 -0
  148. crprotocol-2.0.0.dist-info/METADATA +1295 -0
  149. crprotocol-2.0.0.dist-info/RECORD +153 -0
  150. crprotocol-2.0.0.dist-info/WHEEL +4 -0
  151. crprotocol-2.0.0.dist-info/entry_points.txt +2 -0
  152. crprotocol-2.0.0.dist-info/licenses/LICENSE.md +170 -0
  153. crprotocol-2.0.0.dist-info/licenses/NOTICE +18 -0
@@ -0,0 +1,1435 @@
1
+ # Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
2
+ # Licensed under Elastic License 2.0 — see LICENSE.md for details.
3
+ """CRP orchestrator — full-featured dispatch with all subsystems (§2.5, §6.5).
4
+
5
+ Capabilities:
6
+ - Envelope-based dispatch with 6-phase fact packing
7
+ - Graduated 6-stage extraction pipeline on outputs + ingestion
8
+ - WarmStateStore with CriticalState / StructuralState
9
+ - CKF (Contextual Knowledge Fabric) 4-mode retrieval
10
+ - Multi-window continuation with 3-way termination
11
+ - Token measurement & budget validation
12
+ - Message assembly per Axiom 4 (Model Ignorance)
13
+ - QualityReport with real extraction & security stats
14
+ - Session status & cost estimation
15
+ - Budget cap enforcement
16
+ - Streaming dispatch (§6.10.5)
17
+ - Zero-LLM ingestion with graduated extraction (§2.5)
18
+ - State export (§2.5)
19
+ - Injection detection (advisory, §7.5)
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import atexit
25
+ import hashlib
26
+ import json
27
+ import logging
28
+ import os
29
+ import re
30
+ import threading
31
+ import time
32
+ import uuid
33
+ from collections import deque
34
+ from collections.abc import Generator
35
+ from concurrent.futures import ThreadPoolExecutor
36
+ from dataclasses import dataclass, field
37
+ from typing import Any
38
+
39
+ from crp.core.config import ConfigurationResolver, CRPConfig
40
+ from crp.core.errors import (
41
+ BudgetExhaustedError,
42
+ ProviderError,
43
+ RateLimitExceededError,
44
+ SessionClosedError,
45
+ SessionExpiredError,
46
+ ValidationError,
47
+ )
48
+ from crp.core.session import (
49
+ CostEstimate,
50
+ EnvelopePreview,
51
+ QualityReport,
52
+ RemainingBudget,
53
+ SecurityFlags,
54
+ SessionHandle,
55
+ SessionStatus,
56
+ )
57
+ from crp.core.task_intent import TaskIntent
58
+ from crp.core.window import (
59
+ WindowDAG,
60
+ WindowMetrics,
61
+ WindowNode,
62
+ WindowState,
63
+ compute_envelope_budget,
64
+ resolve_generation_reserve,
65
+ )
66
+ from crp.providers.base import LLMProvider
67
+
68
+ # Deferred imports to avoid circular dependency chains.
69
+ # These modules import from crp.core.task_intent (which is in crp.core),
70
+ # creating a cycle via crp.core.__init__ → crp.core.orchestrator.
71
+ # We import them lazily at class init or first use instead.
72
+ from typing import TYPE_CHECKING
73
+
74
+ if TYPE_CHECKING:
75
+ from crp.advanced.source_grounding import SourceGroundingEngine
76
+ from crp.continuation.manager import ContinuationConfig, ContinuationManager
77
+ from crp.envelope.builder import EnvelopeResult, EnvelopeState
78
+ from crp.extraction.pipeline import ExtractionPipeline
79
+ from crp.extraction.quality_gate import run_quality_gate as _run_quality_gate
80
+ from crp.extraction.types import ExtractionResult as PipelineExtractionResult, Fact
81
+ from crp.security.audit_trail import ComplianceAuditTrail
82
+ from crp.security.binding import SessionBindingManager
83
+ from crp.security.consent import ConsentManager, ProcessingRecordKeeper, HumanOversightController
84
+ from crp.security.compliance import ComplianceReporter, RiskClassifier
85
+ from crp.security.embedding_defense import EmbeddingDefense
86
+ from crp.security.encryption import StateEncryptor
87
+ from crp.security.injection import InjectionDetector
88
+ from crp.security.integrity import FactIntegrityChain
89
+ from crp.security.privacy import PIIScanner, RetentionManager, DataLineageTracker
90
+ from crp.security.quarantine import IngestQuarantine
91
+ from crp.security.rbac import RBACEnforcer
92
+ from crp.security.validation import InputValidator
93
+ from crp.state.warm_store import WarmStateStore, WarmStoreConfig
94
+
95
+ from crp.core.dispatch_router import (
96
+ DispatchMixin,
97
+ StreamEvent,
98
+ ExtractionProgress,
99
+ WindowSummary,
100
+ _classify_quality_tier,
101
+ _safe_provider_error,
102
+ assemble_messages,
103
+ )
104
+ from crp.core.extraction_facade import ExtractionMixin, ExtractionResult
105
+
106
+ logger = logging.getLogger("crp.orchestrator")
107
+
108
+
109
+ # ---------------------------------------------------------------------------
110
+ # ContinuationInfo (kept here — only used in orchestrator context)
111
+ # ---------------------------------------------------------------------------
112
+
113
+
114
+ @dataclass
115
+ class ContinuationInfo:
116
+ """Emitted when a continuation window is triggered."""
117
+
118
+ continuation_index: int = 0
119
+ reason: str = ""
120
+
121
+
122
+ # ---------------------------------------------------------------------------
123
+ # Auto-detection: zero-config provider resolution
124
+ # ---------------------------------------------------------------------------
125
+
126
+ def _auto_detect_provider(model: str | None = None) -> LLMProvider:
127
+ """Auto-detect the best available LLM provider.
128
+
129
+ Resolution order:
130
+ 1. If ``model`` is given and matches a known provider pattern, use that.
131
+ 2. If ``OPENAI_API_KEY`` is set → OpenAIAdapter.
132
+ 3. If ``ANTHROPIC_API_KEY`` is set → AnthropicAdapter.
133
+ 4. If Ollama is running locally (``OLLAMA_HOST`` or localhost:11434) → OllamaAdapter.
134
+ 5. Raise a helpful error.
135
+ """
136
+ import os
137
+
138
+ # Model-name heuristics
139
+ if model:
140
+ lower = model.lower()
141
+ if any(lower.startswith(p) for p in ("gpt-", "o1", "o3", "o4")):
142
+ from crp.providers.openai import OpenAIAdapter
143
+ return OpenAIAdapter(model=model)
144
+ if lower.startswith("claude"):
145
+ from crp.providers.anthropic import AnthropicAdapter
146
+ return AnthropicAdapter(model=model)
147
+ # Any other model name → try Ollama
148
+ from crp.providers.ollama import OllamaAdapter
149
+ return OllamaAdapter(model=model)
150
+
151
+ # Environment-based detection
152
+ if os.environ.get("OPENAI_API_KEY"):
153
+ from crp.providers.openai import OpenAIAdapter
154
+ return OpenAIAdapter()
155
+
156
+ if os.environ.get("ANTHROPIC_API_KEY"):
157
+ from crp.providers.anthropic import AnthropicAdapter
158
+ return AnthropicAdapter()
159
+
160
+ # Try Ollama on localhost
161
+ ollama_host = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
162
+ try:
163
+ import urllib.request
164
+ req = urllib.request.Request(f"{ollama_host}/api/tags", method="GET")
165
+ with urllib.request.urlopen(req, timeout=2):
166
+ from crp.providers.ollama import OllamaAdapter
167
+ return OllamaAdapter()
168
+ except Exception:
169
+ pass
170
+
171
+ raise ValueError(
172
+ "No LLM provider detected. Either:\n"
173
+ " 1. Pass provider=... explicitly:\n"
174
+ " from crp.providers import OpenAIAdapter\n"
175
+ " client = crp.Client(provider=OpenAIAdapter())\n"
176
+ " 2. Pass model=... for auto-detection:\n"
177
+ " client = crp.Client(model='gpt-4o')\n"
178
+ " 3. Set an API key environment variable:\n"
179
+ " OPENAI_API_KEY=sk-... or ANTHROPIC_API_KEY=...\n"
180
+ " 4. Start Ollama locally:\n"
181
+ " ollama serve && ollama pull llama3.1"
182
+ )
183
+
184
+
185
+ # ---------------------------------------------------------------------------
186
+ # Helpers
187
+ # ---------------------------------------------------------------------------
188
+
189
+ _UUID_RE = re.compile(r"^[0-9a-fA-F\-]{36}$")
190
+
191
+
192
+ # ---------------------------------------------------------------------------
193
+ # Orchestrator
194
+ # ---------------------------------------------------------------------------
195
+
196
+ class CRPOrchestrator(DispatchMixin, ExtractionMixin):
197
+ """Central dispatcher managing window lifecycle and protocol operations.
198
+
199
+ Integrates ALL CRP subsystems:
200
+ - ExtractionPipeline (graduated 6-stage extraction on outputs + ingestion)
201
+ - WarmStateStore (persistent fact accumulation, ranking, aging)
202
+ - CKF (ContextualKnowledgeFabric — 4-mode retrieval)
203
+ - Envelope builder (6-phase budget-aware fact packing)
204
+ - ContinuationManager (3-way termination, gap analysis, stitching)
205
+ - InjectionDetector (advisory security scanning)
206
+ """
207
+
208
+ def __init__(
209
+ self,
210
+ provider: LLMProvider | None = None,
211
+ config: CRPConfig | None = None,
212
+ *,
213
+ llm: LLMProvider | None = None,
214
+ model: str | None = None,
215
+ **init_kwargs: Any,
216
+ ) -> None:
217
+ # Deferred imports (avoid circular: orchestrator → ckf → extraction → core)
218
+ from crp.ckf.fabric import CKFConfig, ContextualKnowledgeFabric
219
+ from crp.continuation.manager import ContinuationConfig
220
+ from crp.extraction.pipeline import ExtractionPipeline
221
+ from crp.state.warm_store import WarmStateStore, WarmStoreConfig
222
+
223
+ # Accept 'llm' as alias for 'provider' (backwards compat with docs)
224
+ resolved_provider = provider or llm
225
+ if resolved_provider is None:
226
+ resolved_provider = _auto_detect_provider(model=model)
227
+
228
+ self._provider = resolved_provider
229
+
230
+ # Resolve configuration
231
+ if config is not None:
232
+ self._config = config
233
+ else:
234
+ self._config = ConfigurationResolver().resolve(**init_kwargs)
235
+
236
+ # Session
237
+ timeout = self._config.session_timeout
238
+ now = time.time()
239
+ self._session = SessionHandle(
240
+ expires_at=now + timeout,
241
+ )
242
+ self._closed = False
243
+
244
+ # Wire structured logging (§audit M2)
245
+ from crp.observability.structured_logging import (
246
+ configure_structured_logging,
247
+ set_session_context,
248
+ )
249
+ configure_structured_logging()
250
+ set_session_context(self._session.session_id)
251
+
252
+ # Thread safety — RLock guards all mutable state (§audit C1)
253
+ self._lock = threading.RLock()
254
+
255
+ # Configurable thread pool for async dispatch (§audit C5)
256
+ max_workers = self._config.get("max_threads", 2)
257
+ self._executor = ThreadPoolExecutor(
258
+ max_workers=max(1, int(max_workers)),
259
+ thread_name_prefix="crp-dispatch",
260
+ )
261
+
262
+ # Circuit breaker — prevent cascading failures on provider errors (§audit H4)
263
+ from crp.core.circuit_breaker import CircuitBreaker, CircuitBreakerConfig
264
+ self._circuit_breaker = CircuitBreaker(CircuitBreakerConfig())
265
+
266
+ # Counters
267
+ self._windows_completed = 0
268
+ self._total_input_tokens = 0
269
+ self._total_output_tokens = 0
270
+ self._continuation_windows_total = 0
271
+
272
+ # DAG
273
+ self._dag = WindowDAG()
274
+
275
+ # ── Subsystems ─────────────────────────────────────────
276
+ # Resource manager — centralized tracking (§audit R1)
277
+ from crp.resources.resource_manager import MODEL_ESTIMATES, ResourceManager
278
+ self._resource_manager = ResourceManager(
279
+ budget_mb=self._config.get("memory_budget_mb", 512),
280
+ )
281
+ for model_name, est_mb in MODEL_ESTIMATES.items():
282
+ self._resource_manager.register_model(model_name, est_mb)
283
+
284
+ # Overhead manager — feature shedding cascade (§audit R2)
285
+ from crp.resources.overhead_manager import OverheadBudgetManager
286
+ self._overhead_manager = OverheadBudgetManager(
287
+ max_overhead_pct=self._config.get("overhead_cap_pct", 15.0),
288
+ )
289
+
290
+ # Adaptive resource allocator — dynamic pipeline tuning (§resource-alloc)
291
+ from crp.resources.adaptive_allocator import AdaptiveAllocator
292
+ self._adaptive_allocator = AdaptiveAllocator(
293
+ resource_manager=self._resource_manager,
294
+ overhead_manager=self._overhead_manager,
295
+ overhead_cap_pct=self._config.get("overhead_cap_pct", 15.0),
296
+ idle_model_timeout_s=self._config.get("idle_model_timeout_s", 300.0),
297
+ )
298
+
299
+ # WarmStateStore — Tier 2 in-memory fact storage
300
+ self._warm_store = WarmStateStore(WarmStoreConfig(
301
+ max_facts=10_000,
302
+ ))
303
+
304
+ # Extraction pipeline — graduated 6-stage
305
+ # Extraction pipeline — graduated 6-stage
306
+ self._extraction = ExtractionPipeline(
307
+ enable_stage_3=self._config.get("enable_stage_3", True),
308
+ enable_stage_4=self._config.get("enable_stage_4", True),
309
+ enable_stage_5=self._config.get("enable_stage_5", True),
310
+ enable_stage_6=self._config.get("enable_stage_6", False),
311
+ )
312
+
313
+ # CKF — Contextual Knowledge Fabric (4-mode retrieval)
314
+ self._ckf = ContextualKnowledgeFabric(CKFConfig(
315
+ max_facts=10_000,
316
+ ))
317
+
318
+ # Eagerly load cross-encoder model to avoid cold-start delay
319
+ from crp.envelope.reranker import preload_cross_encoder
320
+ preload_cross_encoder()
321
+
322
+ # Injection detector (advisory, never blocks)
323
+ # ── Security subsystems — delegated to SecurityManager (§audit4 CQ-C1) ──
324
+ from crp.core.security_manager import SecurityManager
325
+ self._security = SecurityManager(
326
+ session_id=self._session.session_id,
327
+ session_key=b"", # session_binding creates its own key
328
+ config=self._config,
329
+ )
330
+ # Backward-compatible attribute delegation
331
+ self._injection_detector = self._security.injection_detector
332
+ self._input_validator = self._security.input_validator
333
+ self._session_binding = self._security.session_binding
334
+ self._rbac = self._security.rbac
335
+ self._integrity_chain = self._security.integrity_chain
336
+ self._encryptor = self._security.encryptor
337
+ self._quarantine = self._security.quarantine
338
+ self._embedding_defense = self._security.embedding_defense
339
+ self._pii_scanner = self._security.pii_scanner
340
+ self._retention_manager = self._security.retention_manager
341
+ self._erasure_manager = self._security.erasure_manager
342
+ self._lineage_tracker = self._security.lineage_tracker
343
+ self._consent_manager = self._security.consent_manager
344
+ self._processing_records = self._security.processing_records
345
+ self._human_oversight = self._security.human_oversight
346
+ self._compliance_audit = self._security.compliance_audit
347
+ self._risk_classifier = self._security.risk_classifier
348
+ self._compliance_reporter = self._security.compliance_reporter
349
+
350
+ # Source grounding — store original passages alongside facts (§17)
351
+ from crp.advanced.source_grounding import SourceGroundingEngine
352
+ self._source_grounding = SourceGroundingEngine()
353
+
354
+ # Meta-learning scaffolds for small models (§19)
355
+ from crp.advanced.meta_learning import MetaLearningEngine, MetaLearningConfig
356
+ self._meta_learning = MetaLearningEngine(
357
+ dispatch_fn=self._make_meta_dispatch_fn(),
358
+ config=MetaLearningConfig(),
359
+ )
360
+
361
+ # Decision Provenance Engine — claim attribution & audit (§7.14.3)
362
+ from crp.provenance import DecisionProvenanceEngine, ProvenanceConfig
363
+ self._provenance_engine = DecisionProvenanceEngine(
364
+ config=ProvenanceConfig(),
365
+ )
366
+
367
+ # LLM context curator — progressive understanding synthesis (§18)
368
+ # V6 fix: Wire dispatch function at init time, not lazily during
369
+ # dispatch. This avoids race conditions and stale provider references.
370
+ from crp.advanced.curator import LLMContextCurator, CurationConfig
371
+
372
+ def _curator_dispatch(sys_prompt: str, task: str, **kw):
373
+ try:
374
+ output, _ = self._provider.generate_chat(
375
+ [
376
+ {"role": "system", "content": sys_prompt},
377
+ {"role": "user", "content": task},
378
+ ],
379
+ **kw,
380
+ )
381
+ return (output, {})
382
+ except Exception as exc:
383
+ logger.debug("Curator dispatch failed: %s", exc)
384
+ return ("", {})
385
+
386
+ self._curator = LLMContextCurator(
387
+ dispatch_fn=_curator_dispatch,
388
+ config=CurationConfig(),
389
+ )
390
+
391
+ # ── Wire embedding function globally (§5B.3, GAP A fix) ──
392
+ # This enables: lazy fact embeddings, semantic gap analysis,
393
+ # CKF semantic mode, and cosine similarity for fulfillment scoring.
394
+ from crp.envelope.decomposer import get_embedding_fn
395
+ from crp.state.fact import set_embedding_function
396
+ self._embedding_fn = get_embedding_fn()
397
+ if self._embedding_fn is not None:
398
+ set_embedding_function(self._embedding_fn)
399
+ logger.info("Embedding function wired (all-MiniLM-L6-v2)")
400
+
401
+ # Continuation manager — created per dispatch (stateful per task)
402
+ self._continuation_config = ContinuationConfig(
403
+ max_continuations=self._config.max_continuations,
404
+ l3_extractor=self._make_l3_extractor(),
405
+ embedding_fn=self._embedding_fn,
406
+ )
407
+
408
+ # Extraction history for quality gate anomaly detection (§audit4 REL-M1)
409
+ self._extraction_history: deque[PipelineExtractionResult] = deque(maxlen=20)
410
+
411
+ # ── Observability subsystems (§9) ─────────────────────
412
+ # EventEmitter — protocol-wide event bus for all pipeline stages
413
+ from crp.observability.events import EventEmitter
414
+ self._emitter = EventEmitter()
415
+ self._emitter.start()
416
+ self._emitter.emit("session.created", {
417
+ "session_id": self._session.session_id,
418
+ "model": self._provider.model_name,
419
+ "context_window": self._provider.context_window_size(),
420
+ })
421
+
422
+ # TelemetryWriter — per-window JSONL telemetry (optional file sink)
423
+ from crp.observability.telemetry import TelemetryWriter, WindowTelemetry
424
+ telemetry_path = self._config.get("telemetry_path", "")
425
+ self._telemetry_writer: TelemetryWriter | None = (
426
+ TelemetryWriter(telemetry_path) if telemetry_path else None
427
+ )
428
+
429
+ # ── Advanced modules (previously disconnected) ────────
430
+ # CQSDetector — detect context hunger in LLM output (§CQS)
431
+ from crp.advanced.cqs import CQSDetector
432
+ self._cqs_detector = CQSDetector()
433
+
434
+ # CrossWindowValidator — consistency validation across windows (§cross-window)
435
+ from crp.advanced.cross_window import CrossWindowValidator
436
+
437
+ def _cross_window_dispatch(sys_prompt: str, task: str, **kw) -> tuple[str, Any]:
438
+ try:
439
+ output, _ = self._provider.generate_chat(
440
+ [{"role": "system", "content": sys_prompt},
441
+ {"role": "user", "content": task}],
442
+ **kw,
443
+ )
444
+ return output, {}
445
+ except Exception:
446
+ return "", {}
447
+
448
+ self._cross_window_validator = CrossWindowValidator(
449
+ dispatch_fn=_cross_window_dispatch,
450
+ embedding_fn=self._embedding_fn,
451
+ )
452
+
453
+ # FeedbackLoop — fact confidence adjustment from user/system feedback (§feedback)
454
+ from crp.advanced.feedback import FeedbackLoop
455
+ self._feedback_loop = FeedbackLoop()
456
+
457
+ # ParallelFanOut — parallel multi-task dispatch (§parallel)
458
+ from crp.advanced.parallel import ParallelFanOut
459
+
460
+ def _fanout_dispatch(sys_prompt: str, task: str, **kw) -> tuple[str, Any]:
461
+ try:
462
+ output, _ = self._provider.generate_chat(
463
+ [{"role": "system", "content": sys_prompt},
464
+ {"role": "user", "content": task}],
465
+ **kw,
466
+ )
467
+ return output, {}
468
+ except Exception:
469
+ return "", {}
470
+
471
+ def _fanout_extract(text) -> list[dict[str, Any]]:
472
+ result = self._extraction.extract(text, source_window_id="parallel-fanout")
473
+ if not result.facts:
474
+ return []
475
+ return [{"id": f.id, "text": f.text, "confidence": f.confidence} for f in result.facts]
476
+
477
+ self._parallel_fanout = ParallelFanOut(
478
+ dispatch_fn=_fanout_dispatch,
479
+ extract_fn=_fanout_extract,
480
+ max_concurrent=self._config.get("parallel_max_concurrent", 4),
481
+ )
482
+
483
+ # ReviewCycleManager — periodic quality review during long generation (§review)
484
+ from crp.advanced.review_cycle import ReviewCycleManager
485
+ self._review_cycle = ReviewCycleManager(
486
+ dispatch_fn=_cross_window_dispatch,
487
+ model_review_capability=self._meta_learning.assess_model_capability()
488
+ if hasattr(self._meta_learning, "assess_model_capability") else 1,
489
+ )
490
+
491
+ # ScaleModeSelector — processing mode selection based on task size (§scale)
492
+ from crp.advanced.scale_mode import ScaleModeSelector
493
+ self._scale_mode = ScaleModeSelector(
494
+ context_window=self._provider.context_window_size(),
495
+ )
496
+
497
+ # ── Provider manager with fallback (§05, §F3 fix) ────
498
+ from crp.providers.manager import LLMProviderManager
499
+ self._provider_manager = LLMProviderManager(self._provider)
500
+
501
+ # ---------- Compliance audit: session created (§7.14) ----------
502
+ from crp.security.audit_trail import ComplianceEventType
503
+ self._compliance_audit.record(
504
+ ComplianceEventType.SESSION_CREATED,
505
+ session_id=self._session.session_id,
506
+ data={
507
+ "protocol_version": self._session.protocol_version,
508
+ "model": self._provider.model_name,
509
+ "context_window": self._provider.context_window_size(),
510
+ },
511
+ )
512
+
513
+ # Lock immutable fields after session creation
514
+ self._config.lock()
515
+
516
+ # Register atexit handler for graceful shutdown (§audit H6)
517
+ atexit.register(self._atexit_close)
518
+
519
+ # ------------------------------------------------------------------
520
+ # L3 LLM-assisted requirement extraction (§5B.1)
521
+ # ------------------------------------------------------------------
522
+
523
+ def _make_l3_extractor(self):
524
+ """Create L3 extractor callback that uses the LLM provider.
525
+
526
+ Returns a callable that takes task_intent (str) and returns
527
+ a list of Requirement objects. Used by ContinuationManager for
528
+ LLM-assisted requirement discovery beyond L1 regex / L2 semantic.
529
+
530
+ V4 fix: Budget-aware — caps input tokens, tracks overhead in
531
+ telemetry, and limits max_tokens to avoid unbounded LLM cost.
532
+ This is an analytical side-call (not content generation) so a
533
+ full envelope pipeline is unnecessary, but we enforce the budget
534
+ formula and log the interaction.
535
+ """
536
+ def _extract_via_llm(task_intent: str):
537
+ from crp.continuation.gap import Requirement
538
+
539
+ if self._provider is None:
540
+ return []
541
+ try:
542
+ # Budget-aware: cap task intent to fit within a modest budget
543
+ context_window = self._provider.context_window_size()
544
+ # Reserve 512 for output, ~100 for system prompt overhead
545
+ max_input_chars = min(2000, int(context_window * 3.3 * 0.5))
546
+ truncated_intent = task_intent[:max_input_chars]
547
+
548
+ prompt = (
549
+ "Analyze the following task and list the distinct requirements "
550
+ "that must be fulfilled. Return each requirement on its own line "
551
+ "prefixed with '- '. Be specific and concise.\n\n"
552
+ f"Task: {truncated_intent}"
553
+ )
554
+ messages = [
555
+ {"role": "system", "content": "You are a task analyst. Extract requirements."},
556
+ {"role": "user", "content": prompt},
557
+ ]
558
+ # Track token cost for telemetry
559
+ input_tokens = self._provider.count_tokens(prompt) + self._provider.count_tokens(messages[0]["content"])
560
+ logger.debug("L3 extractor: %d input tokens (analytical side-call)", input_tokens)
561
+
562
+ output, _ = self._provider.generate_chat(messages, max_tokens=512)
563
+ requirements: list[Requirement] = []
564
+ for line in output.splitlines():
565
+ line = line.strip()
566
+ if line.startswith("- ") and len(line) > 4:
567
+ requirements.append(Requirement(
568
+ text=line[2:].strip(),
569
+ level=3,
570
+ category="llm_extracted",
571
+ weight=1.0,
572
+ ))
573
+ logger.debug("L3 extractor: %d requirements extracted", len(requirements))
574
+ return requirements
575
+ except Exception as exc:
576
+ logger.debug("L3 extractor failed (non-fatal): %s", exc)
577
+ return [] # L3 failure is non-fatal (§5B.1)
578
+
579
+ return _extract_via_llm
580
+
581
+ # ------------------------------------------------------------------
582
+ # Meta-learning dispatch function (§19 ORC support)
583
+ # ------------------------------------------------------------------
584
+
585
+ def _make_meta_dispatch_fn(self):
586
+ """Create a dispatch callback for MetaLearningEngine.
587
+
588
+ MetaLearningEngine.dispatch_fn signature: (system_prompt, user_prompt) -> (output, meta).
589
+ Used for ORC (Orchestrated Reasoning Chains) LLM-assisted decomposition.
590
+
591
+ V5 fix: Budget-aware — caps prompt to half context window, logs
592
+ token cost, and stores extracted facts from meta-learning output
593
+ back into the warm store so they're not ephemeral.
594
+ """
595
+ def _meta_dispatch(system_prompt: str, user_prompt: str):
596
+ if self._provider is None:
597
+ return ("", {})
598
+ try:
599
+ # Budget-aware: cap user prompt to fit within context window
600
+ context_window = self._provider.context_window_size()
601
+ sys_tokens = self._provider.count_tokens(system_prompt)
602
+ max_user_tokens = context_window - sys_tokens - 1024 # reserve for output
603
+ user_tokens = self._provider.count_tokens(user_prompt)
604
+ if user_tokens > max_user_tokens > 0:
605
+ # Truncate proportionally
606
+ ratio = max_user_tokens / user_tokens
607
+ user_prompt = user_prompt[:int(len(user_prompt) * ratio)]
608
+
609
+ messages = [
610
+ {"role": "system", "content": system_prompt},
611
+ {"role": "user", "content": user_prompt},
612
+ ]
613
+ logger.debug(
614
+ "Meta-learning dispatch: %d+%d input tokens (ORC side-call)",
615
+ sys_tokens, self._provider.count_tokens(user_prompt),
616
+ )
617
+ output, finish_reason = self._provider.generate_chat(messages, max_tokens=1024)
618
+
619
+ # Store extracted facts from meta-learning output so they're
620
+ # not ephemeral (they feed back into CKF and warm store)
621
+ if output and len(output) > 50:
622
+ try:
623
+ meta_result = self._extraction.extract(
624
+ output, source_window_id="meta-learning",
625
+ )
626
+ if meta_result.facts:
627
+ self._warm_store.add_facts(meta_result.facts)
628
+ self._ckf.store(meta_result.facts, window_id="meta-learning")
629
+ logger.debug(
630
+ "Meta-learning: %d facts extracted and stored",
631
+ len(meta_result.facts),
632
+ )
633
+ except Exception as exc:
634
+ logger.debug("Meta-learning extraction failed: %s", exc)
635
+
636
+ return (output, {"finish_reason": finish_reason})
637
+ except Exception as exc:
638
+ logger.debug("Meta-learning dispatch failed: %s", exc)
639
+ return ("", {})
640
+
641
+ return _meta_dispatch
642
+
643
+ # ------------------------------------------------------------------
644
+ # Guards
645
+ # ------------------------------------------------------------------
646
+
647
+ def _check_session(self) -> None:
648
+ """Raise if session is closed or expired."""
649
+ if self._closed:
650
+ raise SessionClosedError()
651
+ if self._session.is_expired:
652
+ raise SessionExpiredError()
653
+
654
+ def _check_budget(self, input_tokens: int) -> None:
655
+ """Raise BudgetExhaustedError if caps would be exceeded."""
656
+ max_w = self._config.max_windows_per_session
657
+ if max_w and self._windows_completed >= max_w:
658
+ raise BudgetExhaustedError(
659
+ "Max windows per session exceeded",
660
+ limit=max_w,
661
+ used=self._windows_completed,
662
+ )
663
+ max_in = self._config.max_total_input_tokens
664
+ if max_in and (self._total_input_tokens + input_tokens) > max_in:
665
+ raise BudgetExhaustedError(
666
+ "Max total input tokens exceeded",
667
+ limit=max_in,
668
+ used=self._total_input_tokens,
669
+ requested=input_tokens,
670
+ )
671
+
672
+ # ------------------------------------------------------------------
673
+ # Internal: extraction + fact storage
674
+ def session_status(self) -> SessionStatus:
675
+ """Get live session metrics."""
676
+ self._check_session()
677
+
678
+ # RBAC permission check (§7.10)
679
+ from crp.security.rbac import Permission
680
+ perm_result = self._rbac.check_permission(Permission.READ_STATE)
681
+ if not perm_result.allowed:
682
+ raise RateLimitExceededError(perm_result.reason)
683
+
684
+ remaining = None
685
+ max_w = self._config.max_windows_per_session
686
+ max_in = self._config.max_total_input_tokens
687
+ max_out = self._config.max_total_output_tokens
688
+ if max_w or max_in or max_out:
689
+ remaining = RemainingBudget(
690
+ windows_remaining=(max_w - self._windows_completed) if max_w else None,
691
+ input_tokens_remaining=(max_in - self._total_input_tokens) if max_in else None,
692
+ output_tokens_remaining=(max_out - self._total_output_tokens) if max_out else None,
693
+ )
694
+
695
+ # Overhead from continuation windows
696
+ total = self._windows_completed
697
+ overhead = self._continuation_windows_total / total if total > 0 else 0.0
698
+
699
+ # USD cost — compute from provider pricing if available
700
+ cost: float | None = None
701
+ in_cost, out_cost = self._provider.cost_per_1k_tokens()
702
+ if in_cost > 0 or out_cost > 0:
703
+ cost = (
704
+ (self._total_input_tokens / 1000 * in_cost)
705
+ + (self._total_output_tokens / 1000 * out_cost)
706
+ )
707
+
708
+ return SessionStatus(
709
+ session_id=self._session.session_id,
710
+ windows_completed=self._windows_completed,
711
+ total_input_tokens=self._total_input_tokens,
712
+ total_output_tokens=self._total_output_tokens,
713
+ facts_in_warm_state=self._warm_store.fact_count,
714
+ overhead_ratio=overhead,
715
+ remaining_budget=remaining,
716
+ total_cost=cost,
717
+ )
718
+
719
+ def estimate_session(
720
+ self,
721
+ system_prompt: str = "",
722
+ task_input: str = "",
723
+ *,
724
+ planned_dispatches: int = 1,
725
+ avg_output_tokens: int | None = None,
726
+ ) -> CostEstimate:
727
+ """Pre-flight cost estimation WITHOUT executing LLM calls.
728
+
729
+ Args:
730
+ system_prompt: Representative system prompt for token estimation.
731
+ task_input: Representative task input for token estimation.
732
+ planned_dispatches: Number of dispatches planned (default: 1).
733
+ avg_output_tokens: Expected average output tokens per dispatch.
734
+ If None, assumes model uses full generation reserve.
735
+
736
+ Returns:
737
+ CostEstimate with token estimates and USD cost (if pricing known).
738
+ """
739
+ self._check_session()
740
+
741
+ # RBAC permission check (§7.10)
742
+ from crp.security.rbac import Permission
743
+ perm_result = self._rbac.check_permission(Permission.READ_STATE)
744
+ if not perm_result.allowed:
745
+ raise RateLimitExceededError(perm_result.reason)
746
+
747
+ s_tokens = self._provider.count_tokens(system_prompt) if system_prompt else 0
748
+ t_tokens = self._provider.count_tokens(task_input) if task_input else 0
749
+ context_window = self._provider.context_window_size()
750
+ g = resolve_generation_reserve(
751
+ None, self._provider.max_output_tokens, context_window,
752
+ is_thinking_model=getattr(self._provider, "is_thinking_model", False),
753
+ )
754
+
755
+ estimated_input_per = s_tokens + t_tokens
756
+ estimated_output_per = avg_output_tokens or g
757
+ total_input = estimated_input_per * planned_dispatches
758
+ total_output = estimated_output_per * planned_dispatches
759
+
760
+ # USD cost estimation via provider pricing
761
+ cost_usd: float | None = None
762
+ in_cost, out_cost = self._provider.cost_per_1k_tokens()
763
+ if in_cost > 0 or out_cost > 0:
764
+ cost_usd = (total_input / 1000 * in_cost) + (total_output / 1000 * out_cost)
765
+
766
+ confidence = "medium"
767
+ if planned_dispatches > 10:
768
+ confidence = "low"
769
+ elif planned_dispatches > 1 and system_prompt and task_input:
770
+ confidence = "high"
771
+
772
+ return CostEstimate(
773
+ estimated_windows=planned_dispatches,
774
+ estimated_input_tokens=total_input,
775
+ estimated_output_tokens=total_output,
776
+ estimated_cost_usd=cost_usd,
777
+ confidence=confidence,
778
+ )
779
+
780
+ def preview_envelope(
781
+ self, system_prompt: str, task_input: str
782
+ ) -> EnvelopePreview:
783
+ """Inspect envelope contents WITHOUT dispatching."""
784
+ self._check_session()
785
+
786
+ # RBAC permission check (§7.10)
787
+ from crp.security.rbac import Permission
788
+ perm_result = self._rbac.check_permission(Permission.READ_ENVELOPE)
789
+ if not perm_result.allowed:
790
+ raise RateLimitExceededError(perm_result.reason)
791
+
792
+ s_tokens = self._provider.count_tokens(system_prompt)
793
+ t_tokens = self._provider.count_tokens(task_input)
794
+ context_window = self._provider.context_window_size()
795
+ g = resolve_generation_reserve(
796
+ None, self._provider.max_output_tokens, context_window,
797
+ is_thinking_model=getattr(self._provider, "is_thinking_model", False),
798
+ )
799
+
800
+ # Build a real envelope preview from accumulated facts
801
+ envelope_result = self._build_envelope(system_prompt, task_input, g)
802
+
803
+ return EnvelopePreview(
804
+ total_tokens=s_tokens + t_tokens + envelope_result.envelope_tokens,
805
+ envelope_tokens=envelope_result.envelope_tokens,
806
+ generation_reserve=g,
807
+ facts_included=envelope_result.facts_included,
808
+ facts_available=self._warm_store.fact_count,
809
+ saturation=envelope_result.saturation,
810
+ )
811
+
812
+ # ------------------------------------------------------------------
813
+ # Configuration
814
+ # ------------------------------------------------------------------
815
+
816
+ def configure(self, **kwargs: Any) -> None:
817
+ """Runtime configuration changes (mutable fields only)."""
818
+ self._check_session()
819
+
820
+ # RBAC permission check (§7.10)
821
+ from crp.security.rbac import Permission
822
+ perm_result = self._rbac.check_permission(Permission.CONFIGURE)
823
+ if not perm_result.allowed:
824
+ raise RateLimitExceededError(perm_result.reason)
825
+
826
+ from crp.ckf.fabric import CKFConfig, ContextualKnowledgeFabric
827
+ from crp.state.warm_store import WarmStateStore, WarmStoreConfig
828
+
829
+ self._config.update(kwargs)
830
+
831
+ # ------------------------------------------------------------------
832
+ # State management
833
+ # ------------------------------------------------------------------
834
+
835
+ def reset_session(self) -> None:
836
+ """Clear warm state, CKF, event log, DAG. Keeps session open."""
837
+ self._check_session()
838
+
839
+ # ---------- Compliance imports (§7.14) ----------
840
+ from crp.security.audit_trail import ComplianceEventType
841
+
842
+ # ---------- Compliance audit: session reset (§7.14) ----------
843
+ self._compliance_audit.record(
844
+ ComplianceEventType.DATA_DELETED,
845
+ session_id=self._session.session_id,
846
+ data={
847
+ "operation": "reset_session",
848
+ "facts_deleted": self._warm_store.fact_count,
849
+ "windows_reset": self._windows_completed,
850
+ },
851
+ )
852
+
853
+ # RBAC permission check (§7.10) — destructive, requires ADMIN
854
+ from crp.security.rbac import Permission
855
+ perm_result = self._rbac.check_permission(Permission.MANAGE_SESSIONS)
856
+ if not perm_result.allowed:
857
+ self._compliance_audit.record(
858
+ ComplianceEventType.RBAC_DENIED,
859
+ session_id=self._session.session_id,
860
+ data={"operation": "reset_session", "reason": perm_result.reason},
861
+ )
862
+ raise RateLimitExceededError(perm_result.reason)
863
+ from crp.ckf.fabric import CKFConfig, ContextualKnowledgeFabric
864
+ from crp.security.integrity import FactIntegrityChain
865
+ from crp.security.quarantine import IngestQuarantine
866
+ from crp.state.warm_store import WarmStateStore, WarmStoreConfig
867
+
868
+ self._windows_completed = 0
869
+ self._total_input_tokens = 0
870
+ self._total_output_tokens = 0
871
+ self._continuation_windows_total = 0
872
+ self._dag.clear()
873
+ # Reset all subsystems
874
+ self._warm_store = WarmStateStore(WarmStoreConfig(max_facts=10_000))
875
+ self._ckf = ContextualKnowledgeFabric(CKFConfig(max_facts=10_000))
876
+ self._extraction_history.clear()
877
+ # Reset security subsystems
878
+ self._integrity_chain = FactIntegrityChain(
879
+ session_key=self._session_binding.session_key,
880
+ )
881
+ self._quarantine = IngestQuarantine()
882
+ self._rbac.reset_limits()
883
+ logger.info("Session %s reset (all subsystems)", self._session.session_id)
884
+
885
+ # ------------------------------------------------------------------
886
+ # Public API: Observability (§9 — EventEmitter access)
887
+ # ------------------------------------------------------------------
888
+
889
+ @property
890
+ def emitter(self):
891
+ """Access the protocol event bus for subscribing to events."""
892
+ return self._emitter
893
+
894
+ def on(self, event_type: str, listener) -> None:
895
+ """Subscribe to a protocol event (convenience wrapper).
896
+
897
+ Usage::
898
+
899
+ def on_dispatch(event):
900
+ print(f"Dispatch completed: {event.data}")
901
+
902
+ orch.on("dispatch.completed", on_dispatch)
903
+ """
904
+ self._emitter.on(event_type, listener)
905
+
906
+ # ------------------------------------------------------------------
907
+ # Public API: Feedback Loop (§feedback — fact confidence adjustment)
908
+ # ------------------------------------------------------------------
909
+
910
+ @property
911
+ def feedback(self):
912
+ """Access the feedback loop for fact confidence adjustments."""
913
+ return self._feedback_loop
914
+
915
+ def boost_fact(self, fact_id: str, delta: float = 0.1, reason: str = "") -> None:
916
+ """Boost a fact's confidence (positive feedback)."""
917
+ self._feedback_loop.boost_confidence(fact_id, delta, reason)
918
+ self._emitter.emit("fact.boosted", {"fact_id": fact_id, "delta": delta})
919
+
920
+ def penalize_fact(self, fact_id: str, delta: float = -0.2, reason: str = "") -> None:
921
+ """Penalize a fact's confidence (negative feedback)."""
922
+ self._feedback_loop.penalize_confidence(fact_id, delta, reason)
923
+ self._emitter.emit("fact.penalized", {"fact_id": fact_id, "delta": delta})
924
+
925
+ def reject_fact(self, fact_id: str, reason: str = "") -> None:
926
+ """Reject a fact entirely (user override)."""
927
+ self._feedback_loop.reject_fact(fact_id, reason)
928
+ self._emitter.emit("fact.rejected", {"fact_id": fact_id, "reason": reason})
929
+
930
+ # ------------------------------------------------------------------
931
+ # Public API: Provider manager (§05 — multi-provider routing)
932
+ # ------------------------------------------------------------------
933
+
934
+ def register_provider(self, provider: LLMProvider) -> None:
935
+ """Register an additional LLM provider for fallback routing."""
936
+ self._provider_manager.register(provider)
937
+ self._emitter.emit("provider.connected", {
938
+ "model": provider.model_name,
939
+ })
940
+
941
+ # ------------------------------------------------------------------
942
+ # Public API: Parallel fan-out (§parallel)
943
+ # ------------------------------------------------------------------
944
+
945
+ @property
946
+ def parallel(self):
947
+ """Access the parallel fan-out engine for multi-task dispatch."""
948
+ return self._parallel_fanout
949
+
950
+ # ------------------------------------------------------------------
951
+ # Session close
952
+ # ------------------------------------------------------------------
953
+
954
+ def _atexit_close(self) -> None:
955
+ """Safety-net close called by atexit — never raises (§audit H6)."""
956
+ try:
957
+ # Suppress logging during shutdown to avoid I/O-on-closed-file errors
958
+ logging.disable(logging.CRITICAL)
959
+ # Use timeout to prevent deadlock if lock is held (§audit3 ORCH-C5)
960
+ acquired = self._lock.acquire(timeout=5.0)
961
+ if acquired:
962
+ try:
963
+ self._close_locked()
964
+ finally:
965
+ self._lock.release()
966
+ except Exception: # noqa: BLE001
967
+ pass
968
+ finally:
969
+ logging.disable(logging.NOTSET)
970
+
971
+ def close(self) -> None:
972
+ """Close session — flush warm→cold, persist CKF, zero keys (§2.5)."""
973
+ with self._lock:
974
+ self._close_locked()
975
+
976
+ def _close_locked(self) -> None:
977
+ """Internal close implementation — called under self._lock."""
978
+ if self._closed:
979
+ return
980
+
981
+ # Shut down thread pool executor (§audit3: wait for in-flight tasks)
982
+ self._executor.shutdown(wait=True)
983
+
984
+ # ---------- Compliance imports (§7.14) ----------
985
+ from crp.security.audit_trail import ComplianceEventType
986
+
987
+ # ---------- Compliance audit: session closing (§7.14) ----------
988
+ self._compliance_audit.record(
989
+ ComplianceEventType.SESSION_CLOSED,
990
+ session_id=self._session.session_id,
991
+ data={
992
+ "windows_completed": self._windows_completed,
993
+ "total_input_tokens": self._total_input_tokens,
994
+ "total_output_tokens": self._total_output_tokens,
995
+ "facts_in_warm_store": self._warm_store.fact_count,
996
+ "audit_entries": self._compliance_audit.entry_count,
997
+ "processing_records": self._processing_records.activity_count,
998
+ "retention_tracked": self._retention_manager.tracked_count,
999
+ "lineage_tracked": len(self._lineage_tracker.to_dict().get("entries", {})),
1000
+ },
1001
+ )
1002
+
1003
+ # Flush warm state facts → CKF cold storage for cross-session retrieval
1004
+ try:
1005
+ # Validate session_id to prevent path traversal (§audit5 SEC-M1)
1006
+ sid = self._session.session_id
1007
+ if not _UUID_RE.fullmatch(sid):
1008
+ logger.error("Invalid session_id format in close: %r", sid[:40])
1009
+ raise ValueError(f"session_id is not a valid UUID: {sid!r:.40}")
1010
+ persist_dir = os.path.join(
1011
+ os.environ.get("CRP_DATA_DIR", "."),
1012
+ "crp_sessions",
1013
+ )
1014
+ os.makedirs(persist_dir, exist_ok=True)
1015
+ persist_path = os.path.join(
1016
+ persist_dir,
1017
+ f"{sid}.json",
1018
+ )
1019
+ self._ckf.persist(persist_path)
1020
+ logger.debug("CKF persisted to cold storage: %s", persist_path)
1021
+ except Exception as exc:
1022
+ logger.warning("CKF persistence failed: %s", exc)
1023
+
1024
+ # Persist encrypted event log (§7.3)
1025
+ if self._config.get("encrypt_cold_state", True):
1026
+ try:
1027
+ event_data = json.dumps({
1028
+ "session_id": self._session.session_id,
1029
+ "windows_completed": self._windows_completed,
1030
+ "integrity_chain": self._integrity_chain.to_dict(),
1031
+ }, separators=(",", ":")).encode("utf-8")
1032
+ persist_dir = os.path.join(
1033
+ os.environ.get("CRP_DATA_DIR", "."),
1034
+ "crp_sessions",
1035
+ )
1036
+ os.makedirs(persist_dir, exist_ok=True)
1037
+ event_blob = self._encryptor.encrypt_event_log(event_data)
1038
+ event_path = os.path.join(
1039
+ persist_dir,
1040
+ f"{self._session.session_id}.events.bin",
1041
+ )
1042
+ with open(event_path, "wb") as f:
1043
+ f.write(event_blob.ciphertext)
1044
+ logger.debug("Encrypted event log persisted: %s", event_path)
1045
+ except Exception as exc:
1046
+ logger.debug("Event log persistence skipped: %s", exc)
1047
+
1048
+ # ---------- Persist compliance audit trail (§7.14) ----------
1049
+ try:
1050
+ persist_dir = os.path.join(
1051
+ os.environ.get("CRP_DATA_DIR", "."),
1052
+ "crp_sessions",
1053
+ )
1054
+ os.makedirs(persist_dir, exist_ok=True)
1055
+ audit_path = os.path.join(
1056
+ persist_dir,
1057
+ f"{self._session.session_id}.audit.jsonl",
1058
+ )
1059
+ audit_jsonl = self._compliance_audit.export_jsonl()
1060
+ with open(audit_path, "w", encoding="utf-8") as f:
1061
+ f.write(audit_jsonl)
1062
+ logger.debug("Compliance audit trail persisted: %s (%d entries)",
1063
+ audit_path, self._compliance_audit.entry_count)
1064
+ except Exception as exc:
1065
+ logger.error("Compliance audit trail persistence failed: %s", exc)
1066
+
1067
+ # Zero out sensitive state from memory
1068
+ self._extraction_history.clear()
1069
+
1070
+ # Final resource GC
1071
+ mgr = getattr(self, "_resource_manager", None)
1072
+ if mgr is not None:
1073
+ mgr.run_gc()
1074
+
1075
+ self._closed = True
1076
+ logger.info(
1077
+ "Session %s closed — %d windows, %d input tokens, %d output tokens",
1078
+ self._session.session_id,
1079
+ self._windows_completed,
1080
+ self._total_input_tokens,
1081
+ self._total_output_tokens,
1082
+ )
1083
+
1084
+ # Emit session.closed event and stop emitter (§9)
1085
+ self._emitter.emit("session.closed", {
1086
+ "session_id": self._session.session_id,
1087
+ "windows_completed": self._windows_completed,
1088
+ "total_input_tokens": self._total_input_tokens,
1089
+ "total_output_tokens": self._total_output_tokens,
1090
+ })
1091
+ self._emitter.stop()
1092
+
1093
+ # Close telemetry writer if active (§9, D8 fix)
1094
+ if self._telemetry_writer is not None:
1095
+ self._telemetry_writer.close()
1096
+ self._telemetry_writer = None
1097
+
1098
+ # Opportunistic session file cleanup (§audit H11)
1099
+ try:
1100
+ from crp.state.session_cleanup import cleanup_expired_sessions
1101
+ cleanup_expired_sessions()
1102
+ except Exception: # noqa: BLE001
1103
+ pass # best-effort, don't fail close
1104
+
1105
+ # ------------------------------------------------------------------
1106
+ # Session resume — cross-session continuity (§2.5)
1107
+ # ------------------------------------------------------------------
1108
+
1109
+ @classmethod
1110
+ def resume(
1111
+ cls,
1112
+ session_id: str,
1113
+ provider: LLMProvider | None = None,
1114
+ *,
1115
+ llm: LLMProvider | None = None,
1116
+ data_dir: str | None = None,
1117
+ **kwargs: Any,
1118
+ ) -> "CRPOrchestrator":
1119
+ """Resume a previously closed session by restoring CKF state.
1120
+
1121
+ Loads persisted facts from cold storage into a new orchestrator
1122
+ instance, providing cross-session knowledge continuity.
1123
+
1124
+ Args:
1125
+ session_id: The session_id from the previous session.
1126
+ provider: LLM provider to use for the resumed session.
1127
+ data_dir: Override for CRP_DATA_DIR (default: env or '.').
1128
+
1129
+ Returns:
1130
+ A new CRPOrchestrator with the previous session's facts loaded.
1131
+ """
1132
+ base_dir = data_dir or os.environ.get("CRP_DATA_DIR", ".")
1133
+ # Validate session_id to prevent path traversal (§audit4 SEC-M2)
1134
+ if not _UUID_RE.fullmatch(session_id):
1135
+ raise ValidationError(
1136
+ f"Invalid session_id format: expected UUID, got {session_id!r:.40}"
1137
+ )
1138
+ persist_path = os.path.join(base_dir, "crp_sessions", f"{session_id}.json")
1139
+
1140
+ # Create new orchestrator
1141
+ orch = cls(provider=provider, llm=llm, **kwargs)
1142
+
1143
+ # Restore CKF state from cold storage
1144
+ if os.path.exists(persist_path):
1145
+ try:
1146
+ orch._ckf.restore(persist_path)
1147
+ logger.info(
1148
+ "Resumed session: loaded CKF from %s (%d facts)",
1149
+ persist_path,
1150
+ len(orch._ckf._warm._facts),
1151
+ )
1152
+ # Pre-populate WarmStore with restored CKF facts for
1153
+ # immediate availability in first envelope
1154
+ restored_facts = [sf.fact for sf in orch._ckf._warm._facts.values()]
1155
+ if restored_facts:
1156
+ orch._warm_store.add_facts(restored_facts)
1157
+ logger.info("Restored %d facts into WarmStore", len(restored_facts))
1158
+ except Exception as exc:
1159
+ logger.warning("CKF restore failed for session %s: %s", session_id, exc)
1160
+ else:
1161
+ logger.warning("No persisted session found at %s", persist_path)
1162
+
1163
+ return orch
1164
+
1165
+ # ------------------------------------------------------------------
1166
+ # Streaming dispatch (§6.10.5)
1167
+ def export_state(self, fmt: str | None = None) -> bytes:
1168
+ """Export session state as encrypted bytes (§2.5).
1169
+
1170
+ Includes warm store facts, critical state, structural state, and CKF health.
1171
+ Embeddings are NOT exported (§7.11) — recomputed on import.
1172
+
1173
+ Args:
1174
+ fmt: Export format (reserved, currently only 'json' supported).
1175
+
1176
+ Returns:
1177
+ Encrypted bytes (AES-256-GCM with session-bound key).
1178
+ """
1179
+ self._check_session()
1180
+
1181
+ # ---------- Compliance imports (§7.14) ----------
1182
+ from crp.security.audit_trail import ComplianceEventType
1183
+ from crp.security.consent import ProcessingPurpose
1184
+
1185
+ # ---------- Consent verification for export (§7.13) ----------
1186
+ self._consent_manager.check_required(ProcessingPurpose.EXPORT)
1187
+
1188
+ # RBAC permission check (§7.10)
1189
+ from crp.security.rbac import Permission
1190
+ perm_result = self._rbac.check_permission(Permission.EXPORT_STATE)
1191
+ if not perm_result.allowed:
1192
+ self._compliance_audit.record(
1193
+ ComplianceEventType.RBAC_DENIED,
1194
+ session_id=self._session.session_id,
1195
+ data={"operation": "export_state", "reason": perm_result.reason},
1196
+ )
1197
+ raise RateLimitExceededError(perm_result.reason)
1198
+
1199
+ # ---------- Compliance audit: export started (§7.14) ----------
1200
+ self._compliance_audit.record(
1201
+ ComplianceEventType.DATA_EXPORTED,
1202
+ session_id=self._session.session_id,
1203
+ data={
1204
+ "operation": "export_state",
1205
+ "phase": "started",
1206
+ "format": fmt or "json",
1207
+ "facts_count": self._warm_store.fact_count,
1208
+ },
1209
+ )
1210
+
1211
+ # ---------- Processing record for export (§7.13 — GDPR Art. 30) ----------
1212
+ self._processing_records.record(
1213
+ purpose=ProcessingPurpose.EXPORT,
1214
+ data_categories=["session_state", "warm_store_facts", "ckf_state"],
1215
+ legal_basis="legitimate_interest",
1216
+ input_size_bytes=0,
1217
+ output_size_bytes=0,
1218
+ automated_decision=True,
1219
+ human_oversight=False,
1220
+ retention_period="external",
1221
+ )
1222
+
1223
+ ckf_health = self._ckf.health()
1224
+ state = {
1225
+ "session_id": self._session.session_id,
1226
+ "protocol_version": self._session.protocol_version,
1227
+ "windows_completed": self._windows_completed,
1228
+ "total_input_tokens": self._total_input_tokens,
1229
+ "total_output_tokens": self._total_output_tokens,
1230
+ "facts_in_warm_state": self._warm_store.fact_count,
1231
+ "ckf_facts": ckf_health.fact_count,
1232
+ "ckf_edges": ckf_health.edge_count,
1233
+ "ckf_communities": ckf_health.community_count,
1234
+ "warm_store": self._warm_store.to_dict(),
1235
+ "format": fmt or "json",
1236
+ "exported_at": time.time(),
1237
+ }
1238
+
1239
+ # Strip embeddings from export (§7.11)
1240
+ from crp.security.embedding_defense import EmbeddingDefense
1241
+ state = EmbeddingDefense.strip_embeddings_for_export(state)
1242
+
1243
+ # Include integrity chain signature for verification on import
1244
+ if self._integrity_chain.size > 0:
1245
+ try:
1246
+ state["integrity_chain_signature"] = self._integrity_chain.chain_signature()
1247
+ except ValueError:
1248
+ pass # No session key — skip chain signature
1249
+
1250
+ plaintext = json.dumps(state, separators=(",", ":")).encode("utf-8")
1251
+
1252
+ # Encrypt with session-bound AES-256-GCM key (§7.3)
1253
+ blob = self._encryptor.encrypt_cold_state(plaintext)
1254
+
1255
+ # ---------- Compliance audit: export completed (§7.14) ----------
1256
+ self._compliance_audit.record(
1257
+ ComplianceEventType.DATA_EXPORTED,
1258
+ session_id=self._session.session_id,
1259
+ data={
1260
+ "operation": "export_state",
1261
+ "phase": "completed",
1262
+ "format": fmt or "json",
1263
+ "ciphertext_bytes": len(blob.ciphertext),
1264
+ "facts_exported": self._warm_store.fact_count,
1265
+ },
1266
+ )
1267
+
1268
+ return blob.ciphertext
1269
+
1270
+ # ------------------------------------------------------------------
1271
+ # Properties
1272
+ # ------------------------------------------------------------------
1273
+
1274
+ @property
1275
+ def session(self) -> SessionHandle:
1276
+ return self._session
1277
+
1278
+ @property
1279
+ def dag(self) -> WindowDAG:
1280
+ return self._dag
1281
+
1282
+ @property
1283
+ def config(self) -> CRPConfig:
1284
+ return self._config
1285
+
1286
+ @property
1287
+ def warm_store(self) -> WarmStateStore:
1288
+ """Access the WarmStateStore for direct fact operations."""
1289
+ return self._warm_store
1290
+
1291
+ @property
1292
+ def ckf(self) -> ContextualKnowledgeFabric:
1293
+ """Access the CKF for 4-mode retrieval queries."""
1294
+ return self._ckf
1295
+
1296
+ @property
1297
+ def compliance_audit(self) -> ComplianceAuditTrail:
1298
+ """Access the compliance audit trail for inspection (§7.14)."""
1299
+ return self._compliance_audit
1300
+
1301
+ @property
1302
+ def pii_scanner(self) -> PIIScanner:
1303
+ """Access the PII scanner (§7.12)."""
1304
+ return self._pii_scanner
1305
+
1306
+ @property
1307
+ def consent_manager(self) -> ConsentManager:
1308
+ """Access the consent manager (§7.13)."""
1309
+ return self._consent_manager
1310
+
1311
+ @property
1312
+ def processing_records(self) -> ProcessingRecordKeeper:
1313
+ """Access the processing record keeper — GDPR Art. 30 (§7.13)."""
1314
+ return self._processing_records
1315
+
1316
+ @property
1317
+ def retention_manager(self) -> RetentionManager:
1318
+ """Access the retention manager (§7.12)."""
1319
+ return self._retention_manager
1320
+
1321
+ @property
1322
+ def lineage_tracker(self) -> DataLineageTracker:
1323
+ """Access the data lineage tracker (§7.12)."""
1324
+ return self._lineage_tracker
1325
+
1326
+ @property
1327
+ def human_oversight(self) -> HumanOversightController:
1328
+ """Access the human oversight controller — EU AI Act Art. 14 (§7.13)."""
1329
+ return self._human_oversight
1330
+
1331
+ @property
1332
+ def compliance_reporter(self) -> ComplianceReporter:
1333
+ """Access the compliance reporter (§7.15)."""
1334
+ return self._compliance_reporter
1335
+
1336
+ @property
1337
+ def risk_classifier(self) -> RiskClassifier:
1338
+ """Access the risk classifier — EU AI Act Art. 6 (§7.15)."""
1339
+ return self._risk_classifier
1340
+
1341
+ @property
1342
+ def extraction_pipeline(self) -> ExtractionPipeline:
1343
+ """Access the extraction pipeline for configuration."""
1344
+ return self._extraction
1345
+
1346
+ # ------------------------------------------------------------------
1347
+ # Async API (§6.10) — asyncio.to_thread bridge
1348
+ # ------------------------------------------------------------------
1349
+
1350
+ async def async_dispatch(
1351
+ self,
1352
+ system_prompt: str,
1353
+ task_input: str,
1354
+ **kwargs: Any,
1355
+ ) -> tuple[str, QualityReport]:
1356
+ """Async version of dispatch() — runs in a configurable thread pool.
1357
+
1358
+ Uses the orchestrator's own ThreadPoolExecutor (sized via
1359
+ ``max_threads`` config) instead of the default asyncio executor,
1360
+ which prevents thread-pool saturation under concurrent load.
1361
+
1362
+ Use from async code (FastAPI, asyncio, etc.)::
1363
+
1364
+ output, report = await client.async_dispatch(
1365
+ "You are helpful.", "Explain CRP."
1366
+ )
1367
+ """
1368
+ import asyncio
1369
+
1370
+ loop = asyncio.get_running_loop()
1371
+ return await loop.run_in_executor(
1372
+ self._executor, lambda: self.dispatch(system_prompt, task_input, **kwargs)
1373
+ )
1374
+
1375
+ async def async_ingest(
1376
+ self,
1377
+ raw_text: str,
1378
+ *,
1379
+ source_label: str = "",
1380
+ ) -> ExtractionResult:
1381
+ """Async version of ingest() — runs in the orchestrator's thread pool."""
1382
+ import asyncio
1383
+
1384
+ loop = asyncio.get_running_loop()
1385
+ return await loop.run_in_executor(
1386
+ self._executor, lambda: self.ingest(raw_text, source_label=source_label)
1387
+ )
1388
+
1389
+ async def async_close(self) -> None:
1390
+ """Async version of close() — runs in the orchestrator's thread pool."""
1391
+ import asyncio
1392
+
1393
+ loop = asyncio.get_running_loop()
1394
+ await loop.run_in_executor(self._executor, self.close)
1395
+
1396
+ async def async_dispatch_stream(
1397
+ self,
1398
+ system_prompt: str,
1399
+ task_input: str,
1400
+ **kwargs: Any,
1401
+ ):
1402
+ """Async streaming dispatch — yields StreamEvent objects.
1403
+
1404
+ Usage::
1405
+
1406
+ async for event in client.async_dispatch_stream(
1407
+ "You are helpful.", "Explain CRP."
1408
+ ):
1409
+ if event.event_type == "token":
1410
+ print(event.data, end="")
1411
+ """
1412
+ import asyncio
1413
+ import queue
1414
+
1415
+ q: queue.Queue[StreamEvent | None] = queue.Queue(maxsize=1000) # Bounded queue (§audit M12)
1416
+
1417
+ def _produce():
1418
+ try:
1419
+ for event in self.dispatch_stream(
1420
+ system_prompt, task_input, **kwargs
1421
+ ):
1422
+ q.put(event)
1423
+ finally:
1424
+ q.put(None) # Sentinel
1425
+
1426
+ loop = asyncio.get_running_loop()
1427
+ fut = loop.run_in_executor(self._executor, _produce)
1428
+
1429
+ while True:
1430
+ event = await asyncio.to_thread(q.get)
1431
+ if event is None:
1432
+ break
1433
+ yield event
1434
+
1435
+ await fut # Propagate exceptions