crprotocol 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crp/__init__.py +126 -0
- crp/__main__.py +8 -0
- crp/_typing.py +27 -0
- crp/_version.py +5 -0
- crp/adapters.py +31 -0
- crp/advanced/__init__.py +40 -0
- crp/advanced/auto_ingest.py +400 -0
- crp/advanced/cqs.py +235 -0
- crp/advanced/cross_window.py +477 -0
- crp/advanced/curator.py +265 -0
- crp/advanced/feedback.py +146 -0
- crp/advanced/hierarchical.py +211 -0
- crp/advanced/meta_learning.py +401 -0
- crp/advanced/parallel.py +98 -0
- crp/advanced/review_cycle.py +329 -0
- crp/advanced/scale_mode.py +129 -0
- crp/advanced/source_grounding.py +207 -0
- crp/ckf/__init__.py +35 -0
- crp/ckf/community.py +377 -0
- crp/ckf/fabric.py +445 -0
- crp/ckf/gc.py +175 -0
- crp/ckf/graph_walk.py +87 -0
- crp/ckf/merge.py +133 -0
- crp/ckf/pattern_query.py +122 -0
- crp/ckf/pubsub.py +128 -0
- crp/ckf/semantic.py +207 -0
- crp/cli/__init__.py +7 -0
- crp/cli/main.py +329 -0
- crp/cli/sidecar.py +929 -0
- crp/cli/startup.py +272 -0
- crp/continuation/__init__.py +103 -0
- crp/continuation/completion.py +348 -0
- crp/continuation/degradation.py +157 -0
- crp/continuation/document_map.py +160 -0
- crp/continuation/flow.py +109 -0
- crp/continuation/gap.py +419 -0
- crp/continuation/manager.py +484 -0
- crp/continuation/quality_monitor.py +179 -0
- crp/continuation/stitch.py +419 -0
- crp/continuation/trigger.py +142 -0
- crp/continuation/voice.py +157 -0
- crp/core/__init__.py +69 -0
- crp/core/batch.py +77 -0
- crp/core/circuit_breaker.py +116 -0
- crp/core/config.py +377 -0
- crp/core/context_tools.py +540 -0
- crp/core/dispatch_router.py +3977 -0
- crp/core/errors.py +128 -0
- crp/core/extraction_facade.py +384 -0
- crp/core/facilitator.py +713 -0
- crp/core/idempotency.py +215 -0
- crp/core/orchestrator.py +1435 -0
- crp/core/relay_strategies.py +613 -0
- crp/core/security_manager.py +140 -0
- crp/core/session.py +134 -0
- crp/core/task_intent.py +36 -0
- crp/core/window.py +363 -0
- crp/envelope/__init__.py +30 -0
- crp/envelope/builder.py +288 -0
- crp/envelope/decomposer.py +236 -0
- crp/envelope/formatter.py +168 -0
- crp/envelope/packer.py +211 -0
- crp/envelope/reranker.py +209 -0
- crp/envelope/scoring.py +310 -0
- crp/extraction/__init__.py +45 -0
- crp/extraction/complexity.py +96 -0
- crp/extraction/contradiction.py +132 -0
- crp/extraction/pipeline.py +360 -0
- crp/extraction/quality_gate.py +237 -0
- crp/extraction/stage1_regex.py +173 -0
- crp/extraction/stage2_statistical.py +244 -0
- crp/extraction/stage3_gliner.py +210 -0
- crp/extraction/stage4_uie.py +183 -0
- crp/extraction/stage5_discourse.py +175 -0
- crp/extraction/stage6_llm.py +178 -0
- crp/extraction/structured_output.py +219 -0
- crp/extraction/types.py +299 -0
- crp/license_guard.py +722 -0
- crp/observability/__init__.py +30 -0
- crp/observability/audit.py +118 -0
- crp/observability/events.py +233 -0
- crp/observability/metrics.py +264 -0
- crp/observability/quality.py +135 -0
- crp/observability/structured_logging.py +81 -0
- crp/observability/telemetry.py +117 -0
- crp/provenance/__init__.py +314 -0
- crp/provenance/_embeddings.py +97 -0
- crp/provenance/_types.py +378 -0
- crp/provenance/attribution_scorer.py +252 -0
- crp/provenance/claim_detector.py +229 -0
- crp/provenance/contradiction_detector.py +243 -0
- crp/provenance/distortion_detector.py +397 -0
- crp/provenance/entailment_verifier.py +358 -0
- crp/provenance/fabrication_detector.py +203 -0
- crp/provenance/hallucination_scorer.py +320 -0
- crp/provenance/omission_analyzer.py +106 -0
- crp/provenance/provenance_chain.py +205 -0
- crp/provenance/report_generator.py +440 -0
- crp/providers/__init__.py +43 -0
- crp/providers/anthropic.py +270 -0
- crp/providers/base.py +135 -0
- crp/providers/custom.py +63 -0
- crp/providers/diagnostic.py +251 -0
- crp/providers/llamacpp.py +224 -0
- crp/providers/manager.py +139 -0
- crp/providers/ollama.py +243 -0
- crp/providers/openai.py +628 -0
- crp/providers/tokenizers.py +48 -0
- crp/py.typed +0 -0
- crp/resources/__init__.py +53 -0
- crp/resources/adaptive_allocator.py +525 -0
- crp/resources/cost_model.py +388 -0
- crp/resources/overhead_manager.py +217 -0
- crp/resources/resource_manager.py +262 -0
- crp/schemas/__init__.py +20 -0
- crp/schemas/cost-estimate.json +33 -0
- crp/schemas/crp-error.json +43 -0
- crp/schemas/envelope-preview.json +40 -0
- crp/schemas/persisted-state-header.json +27 -0
- crp/schemas/quality-report.json +94 -0
- crp/schemas/session-handle.json +33 -0
- crp/schemas/session-status.json +57 -0
- crp/schemas/stream-event.json +18 -0
- crp/schemas/task-intent.json +42 -0
- crp/security/__init__.py +93 -0
- crp/security/audit_trail.py +392 -0
- crp/security/binding.py +192 -0
- crp/security/compliance.py +813 -0
- crp/security/consent.py +593 -0
- crp/security/embedding_defense.py +161 -0
- crp/security/encryption.py +202 -0
- crp/security/injection.py +335 -0
- crp/security/integrity.py +267 -0
- crp/security/privacy.py +662 -0
- crp/security/quarantine.py +249 -0
- crp/security/rbac.py +221 -0
- crp/security/validation.py +164 -0
- crp/state/__init__.py +31 -0
- crp/state/cold_storage.py +258 -0
- crp/state/compaction.py +263 -0
- crp/state/critical_state.py +104 -0
- crp/state/event_log.py +313 -0
- crp/state/fact.py +189 -0
- crp/state/serialization.py +189 -0
- crp/state/session_cleanup.py +77 -0
- crp/state/snapshot.py +290 -0
- crp/state/warm_store.py +346 -0
- crprotocol-2.0.0.dist-info/METADATA +1295 -0
- crprotocol-2.0.0.dist-info/RECORD +153 -0
- crprotocol-2.0.0.dist-info/WHEEL +4 -0
- crprotocol-2.0.0.dist-info/entry_points.txt +2 -0
- crprotocol-2.0.0.dist-info/licenses/LICENSE.md +170 -0
- crprotocol-2.0.0.dist-info/licenses/NOTICE +18 -0
crp/core/orchestrator.py
ADDED
|
@@ -0,0 +1,1435 @@
|
|
|
1
|
+
# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
|
|
2
|
+
# Licensed under Elastic License 2.0 — see LICENSE.md for details.
|
|
3
|
+
"""CRP orchestrator — full-featured dispatch with all subsystems (§2.5, §6.5).
|
|
4
|
+
|
|
5
|
+
Capabilities:
|
|
6
|
+
- Envelope-based dispatch with 6-phase fact packing
|
|
7
|
+
- Graduated 6-stage extraction pipeline on outputs + ingestion
|
|
8
|
+
- WarmStateStore with CriticalState / StructuralState
|
|
9
|
+
- CKF (Contextual Knowledge Fabric) 4-mode retrieval
|
|
10
|
+
- Multi-window continuation with 3-way termination
|
|
11
|
+
- Token measurement & budget validation
|
|
12
|
+
- Message assembly per Axiom 4 (Model Ignorance)
|
|
13
|
+
- QualityReport with real extraction & security stats
|
|
14
|
+
- Session status & cost estimation
|
|
15
|
+
- Budget cap enforcement
|
|
16
|
+
- Streaming dispatch (§6.10.5)
|
|
17
|
+
- Zero-LLM ingestion with graduated extraction (§2.5)
|
|
18
|
+
- State export (§2.5)
|
|
19
|
+
- Injection detection (advisory, §7.5)
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import atexit
|
|
25
|
+
import hashlib
|
|
26
|
+
import json
|
|
27
|
+
import logging
|
|
28
|
+
import os
|
|
29
|
+
import re
|
|
30
|
+
import threading
|
|
31
|
+
import time
|
|
32
|
+
import uuid
|
|
33
|
+
from collections import deque
|
|
34
|
+
from collections.abc import Generator
|
|
35
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
36
|
+
from dataclasses import dataclass, field
|
|
37
|
+
from typing import Any
|
|
38
|
+
|
|
39
|
+
from crp.core.config import ConfigurationResolver, CRPConfig
|
|
40
|
+
from crp.core.errors import (
|
|
41
|
+
BudgetExhaustedError,
|
|
42
|
+
ProviderError,
|
|
43
|
+
RateLimitExceededError,
|
|
44
|
+
SessionClosedError,
|
|
45
|
+
SessionExpiredError,
|
|
46
|
+
ValidationError,
|
|
47
|
+
)
|
|
48
|
+
from crp.core.session import (
|
|
49
|
+
CostEstimate,
|
|
50
|
+
EnvelopePreview,
|
|
51
|
+
QualityReport,
|
|
52
|
+
RemainingBudget,
|
|
53
|
+
SecurityFlags,
|
|
54
|
+
SessionHandle,
|
|
55
|
+
SessionStatus,
|
|
56
|
+
)
|
|
57
|
+
from crp.core.task_intent import TaskIntent
|
|
58
|
+
from crp.core.window import (
|
|
59
|
+
WindowDAG,
|
|
60
|
+
WindowMetrics,
|
|
61
|
+
WindowNode,
|
|
62
|
+
WindowState,
|
|
63
|
+
compute_envelope_budget,
|
|
64
|
+
resolve_generation_reserve,
|
|
65
|
+
)
|
|
66
|
+
from crp.providers.base import LLMProvider
|
|
67
|
+
|
|
68
|
+
# Deferred imports to avoid circular dependency chains.
|
|
69
|
+
# These modules import from crp.core.task_intent (which is in crp.core),
|
|
70
|
+
# creating a cycle via crp.core.__init__ → crp.core.orchestrator.
|
|
71
|
+
# We import them lazily at class init or first use instead.
|
|
72
|
+
from typing import TYPE_CHECKING
|
|
73
|
+
|
|
74
|
+
if TYPE_CHECKING:
|
|
75
|
+
from crp.advanced.source_grounding import SourceGroundingEngine
|
|
76
|
+
from crp.continuation.manager import ContinuationConfig, ContinuationManager
|
|
77
|
+
from crp.envelope.builder import EnvelopeResult, EnvelopeState
|
|
78
|
+
from crp.extraction.pipeline import ExtractionPipeline
|
|
79
|
+
from crp.extraction.quality_gate import run_quality_gate as _run_quality_gate
|
|
80
|
+
from crp.extraction.types import ExtractionResult as PipelineExtractionResult, Fact
|
|
81
|
+
from crp.security.audit_trail import ComplianceAuditTrail
|
|
82
|
+
from crp.security.binding import SessionBindingManager
|
|
83
|
+
from crp.security.consent import ConsentManager, ProcessingRecordKeeper, HumanOversightController
|
|
84
|
+
from crp.security.compliance import ComplianceReporter, RiskClassifier
|
|
85
|
+
from crp.security.embedding_defense import EmbeddingDefense
|
|
86
|
+
from crp.security.encryption import StateEncryptor
|
|
87
|
+
from crp.security.injection import InjectionDetector
|
|
88
|
+
from crp.security.integrity import FactIntegrityChain
|
|
89
|
+
from crp.security.privacy import PIIScanner, RetentionManager, DataLineageTracker
|
|
90
|
+
from crp.security.quarantine import IngestQuarantine
|
|
91
|
+
from crp.security.rbac import RBACEnforcer
|
|
92
|
+
from crp.security.validation import InputValidator
|
|
93
|
+
from crp.state.warm_store import WarmStateStore, WarmStoreConfig
|
|
94
|
+
|
|
95
|
+
from crp.core.dispatch_router import (
|
|
96
|
+
DispatchMixin,
|
|
97
|
+
StreamEvent,
|
|
98
|
+
ExtractionProgress,
|
|
99
|
+
WindowSummary,
|
|
100
|
+
_classify_quality_tier,
|
|
101
|
+
_safe_provider_error,
|
|
102
|
+
assemble_messages,
|
|
103
|
+
)
|
|
104
|
+
from crp.core.extraction_facade import ExtractionMixin, ExtractionResult
|
|
105
|
+
|
|
106
|
+
logger = logging.getLogger("crp.orchestrator")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# ---------------------------------------------------------------------------
|
|
110
|
+
# ContinuationInfo (kept here — only used in orchestrator context)
|
|
111
|
+
# ---------------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclass
|
|
115
|
+
class ContinuationInfo:
|
|
116
|
+
"""Emitted when a continuation window is triggered."""
|
|
117
|
+
|
|
118
|
+
continuation_index: int = 0
|
|
119
|
+
reason: str = ""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ---------------------------------------------------------------------------
|
|
123
|
+
# Auto-detection: zero-config provider resolution
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
def _auto_detect_provider(model: str | None = None) -> LLMProvider:
|
|
127
|
+
"""Auto-detect the best available LLM provider.
|
|
128
|
+
|
|
129
|
+
Resolution order:
|
|
130
|
+
1. If ``model`` is given and matches a known provider pattern, use that.
|
|
131
|
+
2. If ``OPENAI_API_KEY`` is set → OpenAIAdapter.
|
|
132
|
+
3. If ``ANTHROPIC_API_KEY`` is set → AnthropicAdapter.
|
|
133
|
+
4. If Ollama is running locally (``OLLAMA_HOST`` or localhost:11434) → OllamaAdapter.
|
|
134
|
+
5. Raise a helpful error.
|
|
135
|
+
"""
|
|
136
|
+
import os
|
|
137
|
+
|
|
138
|
+
# Model-name heuristics
|
|
139
|
+
if model:
|
|
140
|
+
lower = model.lower()
|
|
141
|
+
if any(lower.startswith(p) for p in ("gpt-", "o1", "o3", "o4")):
|
|
142
|
+
from crp.providers.openai import OpenAIAdapter
|
|
143
|
+
return OpenAIAdapter(model=model)
|
|
144
|
+
if lower.startswith("claude"):
|
|
145
|
+
from crp.providers.anthropic import AnthropicAdapter
|
|
146
|
+
return AnthropicAdapter(model=model)
|
|
147
|
+
# Any other model name → try Ollama
|
|
148
|
+
from crp.providers.ollama import OllamaAdapter
|
|
149
|
+
return OllamaAdapter(model=model)
|
|
150
|
+
|
|
151
|
+
# Environment-based detection
|
|
152
|
+
if os.environ.get("OPENAI_API_KEY"):
|
|
153
|
+
from crp.providers.openai import OpenAIAdapter
|
|
154
|
+
return OpenAIAdapter()
|
|
155
|
+
|
|
156
|
+
if os.environ.get("ANTHROPIC_API_KEY"):
|
|
157
|
+
from crp.providers.anthropic import AnthropicAdapter
|
|
158
|
+
return AnthropicAdapter()
|
|
159
|
+
|
|
160
|
+
# Try Ollama on localhost
|
|
161
|
+
ollama_host = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
|
|
162
|
+
try:
|
|
163
|
+
import urllib.request
|
|
164
|
+
req = urllib.request.Request(f"{ollama_host}/api/tags", method="GET")
|
|
165
|
+
with urllib.request.urlopen(req, timeout=2):
|
|
166
|
+
from crp.providers.ollama import OllamaAdapter
|
|
167
|
+
return OllamaAdapter()
|
|
168
|
+
except Exception:
|
|
169
|
+
pass
|
|
170
|
+
|
|
171
|
+
raise ValueError(
|
|
172
|
+
"No LLM provider detected. Either:\n"
|
|
173
|
+
" 1. Pass provider=... explicitly:\n"
|
|
174
|
+
" from crp.providers import OpenAIAdapter\n"
|
|
175
|
+
" client = crp.Client(provider=OpenAIAdapter())\n"
|
|
176
|
+
" 2. Pass model=... for auto-detection:\n"
|
|
177
|
+
" client = crp.Client(model='gpt-4o')\n"
|
|
178
|
+
" 3. Set an API key environment variable:\n"
|
|
179
|
+
" OPENAI_API_KEY=sk-... or ANTHROPIC_API_KEY=...\n"
|
|
180
|
+
" 4. Start Ollama locally:\n"
|
|
181
|
+
" ollama serve && ollama pull llama3.1"
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------------------------
|
|
186
|
+
# Helpers
|
|
187
|
+
# ---------------------------------------------------------------------------
|
|
188
|
+
|
|
189
|
+
_UUID_RE = re.compile(r"^[0-9a-fA-F\-]{36}$")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
# ---------------------------------------------------------------------------
|
|
193
|
+
# Orchestrator
|
|
194
|
+
# ---------------------------------------------------------------------------
|
|
195
|
+
|
|
196
|
+
class CRPOrchestrator(DispatchMixin, ExtractionMixin):
|
|
197
|
+
"""Central dispatcher managing window lifecycle and protocol operations.
|
|
198
|
+
|
|
199
|
+
Integrates ALL CRP subsystems:
|
|
200
|
+
- ExtractionPipeline (graduated 6-stage extraction on outputs + ingestion)
|
|
201
|
+
- WarmStateStore (persistent fact accumulation, ranking, aging)
|
|
202
|
+
- CKF (ContextualKnowledgeFabric — 4-mode retrieval)
|
|
203
|
+
- Envelope builder (6-phase budget-aware fact packing)
|
|
204
|
+
- ContinuationManager (3-way termination, gap analysis, stitching)
|
|
205
|
+
- InjectionDetector (advisory security scanning)
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
def __init__(
|
|
209
|
+
self,
|
|
210
|
+
provider: LLMProvider | None = None,
|
|
211
|
+
config: CRPConfig | None = None,
|
|
212
|
+
*,
|
|
213
|
+
llm: LLMProvider | None = None,
|
|
214
|
+
model: str | None = None,
|
|
215
|
+
**init_kwargs: Any,
|
|
216
|
+
) -> None:
|
|
217
|
+
# Deferred imports (avoid circular: orchestrator → ckf → extraction → core)
|
|
218
|
+
from crp.ckf.fabric import CKFConfig, ContextualKnowledgeFabric
|
|
219
|
+
from crp.continuation.manager import ContinuationConfig
|
|
220
|
+
from crp.extraction.pipeline import ExtractionPipeline
|
|
221
|
+
from crp.state.warm_store import WarmStateStore, WarmStoreConfig
|
|
222
|
+
|
|
223
|
+
# Accept 'llm' as alias for 'provider' (backwards compat with docs)
|
|
224
|
+
resolved_provider = provider or llm
|
|
225
|
+
if resolved_provider is None:
|
|
226
|
+
resolved_provider = _auto_detect_provider(model=model)
|
|
227
|
+
|
|
228
|
+
self._provider = resolved_provider
|
|
229
|
+
|
|
230
|
+
# Resolve configuration
|
|
231
|
+
if config is not None:
|
|
232
|
+
self._config = config
|
|
233
|
+
else:
|
|
234
|
+
self._config = ConfigurationResolver().resolve(**init_kwargs)
|
|
235
|
+
|
|
236
|
+
# Session
|
|
237
|
+
timeout = self._config.session_timeout
|
|
238
|
+
now = time.time()
|
|
239
|
+
self._session = SessionHandle(
|
|
240
|
+
expires_at=now + timeout,
|
|
241
|
+
)
|
|
242
|
+
self._closed = False
|
|
243
|
+
|
|
244
|
+
# Wire structured logging (§audit M2)
|
|
245
|
+
from crp.observability.structured_logging import (
|
|
246
|
+
configure_structured_logging,
|
|
247
|
+
set_session_context,
|
|
248
|
+
)
|
|
249
|
+
configure_structured_logging()
|
|
250
|
+
set_session_context(self._session.session_id)
|
|
251
|
+
|
|
252
|
+
# Thread safety — RLock guards all mutable state (§audit C1)
|
|
253
|
+
self._lock = threading.RLock()
|
|
254
|
+
|
|
255
|
+
# Configurable thread pool for async dispatch (§audit C5)
|
|
256
|
+
max_workers = self._config.get("max_threads", 2)
|
|
257
|
+
self._executor = ThreadPoolExecutor(
|
|
258
|
+
max_workers=max(1, int(max_workers)),
|
|
259
|
+
thread_name_prefix="crp-dispatch",
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
# Circuit breaker — prevent cascading failures on provider errors (§audit H4)
|
|
263
|
+
from crp.core.circuit_breaker import CircuitBreaker, CircuitBreakerConfig
|
|
264
|
+
self._circuit_breaker = CircuitBreaker(CircuitBreakerConfig())
|
|
265
|
+
|
|
266
|
+
# Counters
|
|
267
|
+
self._windows_completed = 0
|
|
268
|
+
self._total_input_tokens = 0
|
|
269
|
+
self._total_output_tokens = 0
|
|
270
|
+
self._continuation_windows_total = 0
|
|
271
|
+
|
|
272
|
+
# DAG
|
|
273
|
+
self._dag = WindowDAG()
|
|
274
|
+
|
|
275
|
+
# ── Subsystems ─────────────────────────────────────────
|
|
276
|
+
# Resource manager — centralized tracking (§audit R1)
|
|
277
|
+
from crp.resources.resource_manager import MODEL_ESTIMATES, ResourceManager
|
|
278
|
+
self._resource_manager = ResourceManager(
|
|
279
|
+
budget_mb=self._config.get("memory_budget_mb", 512),
|
|
280
|
+
)
|
|
281
|
+
for model_name, est_mb in MODEL_ESTIMATES.items():
|
|
282
|
+
self._resource_manager.register_model(model_name, est_mb)
|
|
283
|
+
|
|
284
|
+
# Overhead manager — feature shedding cascade (§audit R2)
|
|
285
|
+
from crp.resources.overhead_manager import OverheadBudgetManager
|
|
286
|
+
self._overhead_manager = OverheadBudgetManager(
|
|
287
|
+
max_overhead_pct=self._config.get("overhead_cap_pct", 15.0),
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
# Adaptive resource allocator — dynamic pipeline tuning (§resource-alloc)
|
|
291
|
+
from crp.resources.adaptive_allocator import AdaptiveAllocator
|
|
292
|
+
self._adaptive_allocator = AdaptiveAllocator(
|
|
293
|
+
resource_manager=self._resource_manager,
|
|
294
|
+
overhead_manager=self._overhead_manager,
|
|
295
|
+
overhead_cap_pct=self._config.get("overhead_cap_pct", 15.0),
|
|
296
|
+
idle_model_timeout_s=self._config.get("idle_model_timeout_s", 300.0),
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
# WarmStateStore — Tier 2 in-memory fact storage
|
|
300
|
+
self._warm_store = WarmStateStore(WarmStoreConfig(
|
|
301
|
+
max_facts=10_000,
|
|
302
|
+
))
|
|
303
|
+
|
|
304
|
+
# Extraction pipeline — graduated 6-stage
|
|
305
|
+
# Extraction pipeline — graduated 6-stage
|
|
306
|
+
self._extraction = ExtractionPipeline(
|
|
307
|
+
enable_stage_3=self._config.get("enable_stage_3", True),
|
|
308
|
+
enable_stage_4=self._config.get("enable_stage_4", True),
|
|
309
|
+
enable_stage_5=self._config.get("enable_stage_5", True),
|
|
310
|
+
enable_stage_6=self._config.get("enable_stage_6", False),
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
# CKF — Contextual Knowledge Fabric (4-mode retrieval)
|
|
314
|
+
self._ckf = ContextualKnowledgeFabric(CKFConfig(
|
|
315
|
+
max_facts=10_000,
|
|
316
|
+
))
|
|
317
|
+
|
|
318
|
+
# Eagerly load cross-encoder model to avoid cold-start delay
|
|
319
|
+
from crp.envelope.reranker import preload_cross_encoder
|
|
320
|
+
preload_cross_encoder()
|
|
321
|
+
|
|
322
|
+
# Injection detector (advisory, never blocks)
|
|
323
|
+
# ── Security subsystems — delegated to SecurityManager (§audit4 CQ-C1) ──
|
|
324
|
+
from crp.core.security_manager import SecurityManager
|
|
325
|
+
self._security = SecurityManager(
|
|
326
|
+
session_id=self._session.session_id,
|
|
327
|
+
session_key=b"", # session_binding creates its own key
|
|
328
|
+
config=self._config,
|
|
329
|
+
)
|
|
330
|
+
# Backward-compatible attribute delegation
|
|
331
|
+
self._injection_detector = self._security.injection_detector
|
|
332
|
+
self._input_validator = self._security.input_validator
|
|
333
|
+
self._session_binding = self._security.session_binding
|
|
334
|
+
self._rbac = self._security.rbac
|
|
335
|
+
self._integrity_chain = self._security.integrity_chain
|
|
336
|
+
self._encryptor = self._security.encryptor
|
|
337
|
+
self._quarantine = self._security.quarantine
|
|
338
|
+
self._embedding_defense = self._security.embedding_defense
|
|
339
|
+
self._pii_scanner = self._security.pii_scanner
|
|
340
|
+
self._retention_manager = self._security.retention_manager
|
|
341
|
+
self._erasure_manager = self._security.erasure_manager
|
|
342
|
+
self._lineage_tracker = self._security.lineage_tracker
|
|
343
|
+
self._consent_manager = self._security.consent_manager
|
|
344
|
+
self._processing_records = self._security.processing_records
|
|
345
|
+
self._human_oversight = self._security.human_oversight
|
|
346
|
+
self._compliance_audit = self._security.compliance_audit
|
|
347
|
+
self._risk_classifier = self._security.risk_classifier
|
|
348
|
+
self._compliance_reporter = self._security.compliance_reporter
|
|
349
|
+
|
|
350
|
+
# Source grounding — store original passages alongside facts (§17)
|
|
351
|
+
from crp.advanced.source_grounding import SourceGroundingEngine
|
|
352
|
+
self._source_grounding = SourceGroundingEngine()
|
|
353
|
+
|
|
354
|
+
# Meta-learning scaffolds for small models (§19)
|
|
355
|
+
from crp.advanced.meta_learning import MetaLearningEngine, MetaLearningConfig
|
|
356
|
+
self._meta_learning = MetaLearningEngine(
|
|
357
|
+
dispatch_fn=self._make_meta_dispatch_fn(),
|
|
358
|
+
config=MetaLearningConfig(),
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
# Decision Provenance Engine — claim attribution & audit (§7.14.3)
|
|
362
|
+
from crp.provenance import DecisionProvenanceEngine, ProvenanceConfig
|
|
363
|
+
self._provenance_engine = DecisionProvenanceEngine(
|
|
364
|
+
config=ProvenanceConfig(),
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
# LLM context curator — progressive understanding synthesis (§18)
|
|
368
|
+
# V6 fix: Wire dispatch function at init time, not lazily during
|
|
369
|
+
# dispatch. This avoids race conditions and stale provider references.
|
|
370
|
+
from crp.advanced.curator import LLMContextCurator, CurationConfig
|
|
371
|
+
|
|
372
|
+
def _curator_dispatch(sys_prompt: str, task: str, **kw):
|
|
373
|
+
try:
|
|
374
|
+
output, _ = self._provider.generate_chat(
|
|
375
|
+
[
|
|
376
|
+
{"role": "system", "content": sys_prompt},
|
|
377
|
+
{"role": "user", "content": task},
|
|
378
|
+
],
|
|
379
|
+
**kw,
|
|
380
|
+
)
|
|
381
|
+
return (output, {})
|
|
382
|
+
except Exception as exc:
|
|
383
|
+
logger.debug("Curator dispatch failed: %s", exc)
|
|
384
|
+
return ("", {})
|
|
385
|
+
|
|
386
|
+
self._curator = LLMContextCurator(
|
|
387
|
+
dispatch_fn=_curator_dispatch,
|
|
388
|
+
config=CurationConfig(),
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
# ── Wire embedding function globally (§5B.3, GAP A fix) ──
|
|
392
|
+
# This enables: lazy fact embeddings, semantic gap analysis,
|
|
393
|
+
# CKF semantic mode, and cosine similarity for fulfillment scoring.
|
|
394
|
+
from crp.envelope.decomposer import get_embedding_fn
|
|
395
|
+
from crp.state.fact import set_embedding_function
|
|
396
|
+
self._embedding_fn = get_embedding_fn()
|
|
397
|
+
if self._embedding_fn is not None:
|
|
398
|
+
set_embedding_function(self._embedding_fn)
|
|
399
|
+
logger.info("Embedding function wired (all-MiniLM-L6-v2)")
|
|
400
|
+
|
|
401
|
+
# Continuation manager — created per dispatch (stateful per task)
|
|
402
|
+
self._continuation_config = ContinuationConfig(
|
|
403
|
+
max_continuations=self._config.max_continuations,
|
|
404
|
+
l3_extractor=self._make_l3_extractor(),
|
|
405
|
+
embedding_fn=self._embedding_fn,
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
# Extraction history for quality gate anomaly detection (§audit4 REL-M1)
|
|
409
|
+
self._extraction_history: deque[PipelineExtractionResult] = deque(maxlen=20)
|
|
410
|
+
|
|
411
|
+
# ── Observability subsystems (§9) ─────────────────────
|
|
412
|
+
# EventEmitter — protocol-wide event bus for all pipeline stages
|
|
413
|
+
from crp.observability.events import EventEmitter
|
|
414
|
+
self._emitter = EventEmitter()
|
|
415
|
+
self._emitter.start()
|
|
416
|
+
self._emitter.emit("session.created", {
|
|
417
|
+
"session_id": self._session.session_id,
|
|
418
|
+
"model": self._provider.model_name,
|
|
419
|
+
"context_window": self._provider.context_window_size(),
|
|
420
|
+
})
|
|
421
|
+
|
|
422
|
+
# TelemetryWriter — per-window JSONL telemetry (optional file sink)
|
|
423
|
+
from crp.observability.telemetry import TelemetryWriter, WindowTelemetry
|
|
424
|
+
telemetry_path = self._config.get("telemetry_path", "")
|
|
425
|
+
self._telemetry_writer: TelemetryWriter | None = (
|
|
426
|
+
TelemetryWriter(telemetry_path) if telemetry_path else None
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
# ── Advanced modules (previously disconnected) ────────
|
|
430
|
+
# CQSDetector — detect context hunger in LLM output (§CQS)
|
|
431
|
+
from crp.advanced.cqs import CQSDetector
|
|
432
|
+
self._cqs_detector = CQSDetector()
|
|
433
|
+
|
|
434
|
+
# CrossWindowValidator — consistency validation across windows (§cross-window)
|
|
435
|
+
from crp.advanced.cross_window import CrossWindowValidator
|
|
436
|
+
|
|
437
|
+
def _cross_window_dispatch(sys_prompt: str, task: str, **kw) -> tuple[str, Any]:
|
|
438
|
+
try:
|
|
439
|
+
output, _ = self._provider.generate_chat(
|
|
440
|
+
[{"role": "system", "content": sys_prompt},
|
|
441
|
+
{"role": "user", "content": task}],
|
|
442
|
+
**kw,
|
|
443
|
+
)
|
|
444
|
+
return output, {}
|
|
445
|
+
except Exception:
|
|
446
|
+
return "", {}
|
|
447
|
+
|
|
448
|
+
self._cross_window_validator = CrossWindowValidator(
|
|
449
|
+
dispatch_fn=_cross_window_dispatch,
|
|
450
|
+
embedding_fn=self._embedding_fn,
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
# FeedbackLoop — fact confidence adjustment from user/system feedback (§feedback)
|
|
454
|
+
from crp.advanced.feedback import FeedbackLoop
|
|
455
|
+
self._feedback_loop = FeedbackLoop()
|
|
456
|
+
|
|
457
|
+
# ParallelFanOut — parallel multi-task dispatch (§parallel)
|
|
458
|
+
from crp.advanced.parallel import ParallelFanOut
|
|
459
|
+
|
|
460
|
+
def _fanout_dispatch(sys_prompt: str, task: str, **kw) -> tuple[str, Any]:
|
|
461
|
+
try:
|
|
462
|
+
output, _ = self._provider.generate_chat(
|
|
463
|
+
[{"role": "system", "content": sys_prompt},
|
|
464
|
+
{"role": "user", "content": task}],
|
|
465
|
+
**kw,
|
|
466
|
+
)
|
|
467
|
+
return output, {}
|
|
468
|
+
except Exception:
|
|
469
|
+
return "", {}
|
|
470
|
+
|
|
471
|
+
def _fanout_extract(text) -> list[dict[str, Any]]:
|
|
472
|
+
result = self._extraction.extract(text, source_window_id="parallel-fanout")
|
|
473
|
+
if not result.facts:
|
|
474
|
+
return []
|
|
475
|
+
return [{"id": f.id, "text": f.text, "confidence": f.confidence} for f in result.facts]
|
|
476
|
+
|
|
477
|
+
self._parallel_fanout = ParallelFanOut(
|
|
478
|
+
dispatch_fn=_fanout_dispatch,
|
|
479
|
+
extract_fn=_fanout_extract,
|
|
480
|
+
max_concurrent=self._config.get("parallel_max_concurrent", 4),
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
# ReviewCycleManager — periodic quality review during long generation (§review)
|
|
484
|
+
from crp.advanced.review_cycle import ReviewCycleManager
|
|
485
|
+
self._review_cycle = ReviewCycleManager(
|
|
486
|
+
dispatch_fn=_cross_window_dispatch,
|
|
487
|
+
model_review_capability=self._meta_learning.assess_model_capability()
|
|
488
|
+
if hasattr(self._meta_learning, "assess_model_capability") else 1,
|
|
489
|
+
)
|
|
490
|
+
|
|
491
|
+
# ScaleModeSelector — processing mode selection based on task size (§scale)
|
|
492
|
+
from crp.advanced.scale_mode import ScaleModeSelector
|
|
493
|
+
self._scale_mode = ScaleModeSelector(
|
|
494
|
+
context_window=self._provider.context_window_size(),
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
# ── Provider manager with fallback (§05, §F3 fix) ────
|
|
498
|
+
from crp.providers.manager import LLMProviderManager
|
|
499
|
+
self._provider_manager = LLMProviderManager(self._provider)
|
|
500
|
+
|
|
501
|
+
# ---------- Compliance audit: session created (§7.14) ----------
|
|
502
|
+
from crp.security.audit_trail import ComplianceEventType
|
|
503
|
+
self._compliance_audit.record(
|
|
504
|
+
ComplianceEventType.SESSION_CREATED,
|
|
505
|
+
session_id=self._session.session_id,
|
|
506
|
+
data={
|
|
507
|
+
"protocol_version": self._session.protocol_version,
|
|
508
|
+
"model": self._provider.model_name,
|
|
509
|
+
"context_window": self._provider.context_window_size(),
|
|
510
|
+
},
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
# Lock immutable fields after session creation
|
|
514
|
+
self._config.lock()
|
|
515
|
+
|
|
516
|
+
# Register atexit handler for graceful shutdown (§audit H6)
|
|
517
|
+
atexit.register(self._atexit_close)
|
|
518
|
+
|
|
519
|
+
# ------------------------------------------------------------------
|
|
520
|
+
# L3 LLM-assisted requirement extraction (§5B.1)
|
|
521
|
+
# ------------------------------------------------------------------
|
|
522
|
+
|
|
523
|
+
def _make_l3_extractor(self):
|
|
524
|
+
"""Create L3 extractor callback that uses the LLM provider.
|
|
525
|
+
|
|
526
|
+
Returns a callable that takes task_intent (str) and returns
|
|
527
|
+
a list of Requirement objects. Used by ContinuationManager for
|
|
528
|
+
LLM-assisted requirement discovery beyond L1 regex / L2 semantic.
|
|
529
|
+
|
|
530
|
+
V4 fix: Budget-aware — caps input tokens, tracks overhead in
|
|
531
|
+
telemetry, and limits max_tokens to avoid unbounded LLM cost.
|
|
532
|
+
This is an analytical side-call (not content generation) so a
|
|
533
|
+
full envelope pipeline is unnecessary, but we enforce the budget
|
|
534
|
+
formula and log the interaction.
|
|
535
|
+
"""
|
|
536
|
+
def _extract_via_llm(task_intent: str):
|
|
537
|
+
from crp.continuation.gap import Requirement
|
|
538
|
+
|
|
539
|
+
if self._provider is None:
|
|
540
|
+
return []
|
|
541
|
+
try:
|
|
542
|
+
# Budget-aware: cap task intent to fit within a modest budget
|
|
543
|
+
context_window = self._provider.context_window_size()
|
|
544
|
+
# Reserve 512 for output, ~100 for system prompt overhead
|
|
545
|
+
max_input_chars = min(2000, int(context_window * 3.3 * 0.5))
|
|
546
|
+
truncated_intent = task_intent[:max_input_chars]
|
|
547
|
+
|
|
548
|
+
prompt = (
|
|
549
|
+
"Analyze the following task and list the distinct requirements "
|
|
550
|
+
"that must be fulfilled. Return each requirement on its own line "
|
|
551
|
+
"prefixed with '- '. Be specific and concise.\n\n"
|
|
552
|
+
f"Task: {truncated_intent}"
|
|
553
|
+
)
|
|
554
|
+
messages = [
|
|
555
|
+
{"role": "system", "content": "You are a task analyst. Extract requirements."},
|
|
556
|
+
{"role": "user", "content": prompt},
|
|
557
|
+
]
|
|
558
|
+
# Track token cost for telemetry
|
|
559
|
+
input_tokens = self._provider.count_tokens(prompt) + self._provider.count_tokens(messages[0]["content"])
|
|
560
|
+
logger.debug("L3 extractor: %d input tokens (analytical side-call)", input_tokens)
|
|
561
|
+
|
|
562
|
+
output, _ = self._provider.generate_chat(messages, max_tokens=512)
|
|
563
|
+
requirements: list[Requirement] = []
|
|
564
|
+
for line in output.splitlines():
|
|
565
|
+
line = line.strip()
|
|
566
|
+
if line.startswith("- ") and len(line) > 4:
|
|
567
|
+
requirements.append(Requirement(
|
|
568
|
+
text=line[2:].strip(),
|
|
569
|
+
level=3,
|
|
570
|
+
category="llm_extracted",
|
|
571
|
+
weight=1.0,
|
|
572
|
+
))
|
|
573
|
+
logger.debug("L3 extractor: %d requirements extracted", len(requirements))
|
|
574
|
+
return requirements
|
|
575
|
+
except Exception as exc:
|
|
576
|
+
logger.debug("L3 extractor failed (non-fatal): %s", exc)
|
|
577
|
+
return [] # L3 failure is non-fatal (§5B.1)
|
|
578
|
+
|
|
579
|
+
return _extract_via_llm
|
|
580
|
+
|
|
581
|
+
# ------------------------------------------------------------------
|
|
582
|
+
# Meta-learning dispatch function (§19 ORC support)
|
|
583
|
+
# ------------------------------------------------------------------
|
|
584
|
+
|
|
585
|
+
def _make_meta_dispatch_fn(self):
|
|
586
|
+
"""Create a dispatch callback for MetaLearningEngine.
|
|
587
|
+
|
|
588
|
+
MetaLearningEngine.dispatch_fn signature: (system_prompt, user_prompt) -> (output, meta).
|
|
589
|
+
Used for ORC (Orchestrated Reasoning Chains) LLM-assisted decomposition.
|
|
590
|
+
|
|
591
|
+
V5 fix: Budget-aware — caps prompt to half context window, logs
|
|
592
|
+
token cost, and stores extracted facts from meta-learning output
|
|
593
|
+
back into the warm store so they're not ephemeral.
|
|
594
|
+
"""
|
|
595
|
+
def _meta_dispatch(system_prompt: str, user_prompt: str):
|
|
596
|
+
if self._provider is None:
|
|
597
|
+
return ("", {})
|
|
598
|
+
try:
|
|
599
|
+
# Budget-aware: cap user prompt to fit within context window
|
|
600
|
+
context_window = self._provider.context_window_size()
|
|
601
|
+
sys_tokens = self._provider.count_tokens(system_prompt)
|
|
602
|
+
max_user_tokens = context_window - sys_tokens - 1024 # reserve for output
|
|
603
|
+
user_tokens = self._provider.count_tokens(user_prompt)
|
|
604
|
+
if user_tokens > max_user_tokens > 0:
|
|
605
|
+
# Truncate proportionally
|
|
606
|
+
ratio = max_user_tokens / user_tokens
|
|
607
|
+
user_prompt = user_prompt[:int(len(user_prompt) * ratio)]
|
|
608
|
+
|
|
609
|
+
messages = [
|
|
610
|
+
{"role": "system", "content": system_prompt},
|
|
611
|
+
{"role": "user", "content": user_prompt},
|
|
612
|
+
]
|
|
613
|
+
logger.debug(
|
|
614
|
+
"Meta-learning dispatch: %d+%d input tokens (ORC side-call)",
|
|
615
|
+
sys_tokens, self._provider.count_tokens(user_prompt),
|
|
616
|
+
)
|
|
617
|
+
output, finish_reason = self._provider.generate_chat(messages, max_tokens=1024)
|
|
618
|
+
|
|
619
|
+
# Store extracted facts from meta-learning output so they're
|
|
620
|
+
# not ephemeral (they feed back into CKF and warm store)
|
|
621
|
+
if output and len(output) > 50:
|
|
622
|
+
try:
|
|
623
|
+
meta_result = self._extraction.extract(
|
|
624
|
+
output, source_window_id="meta-learning",
|
|
625
|
+
)
|
|
626
|
+
if meta_result.facts:
|
|
627
|
+
self._warm_store.add_facts(meta_result.facts)
|
|
628
|
+
self._ckf.store(meta_result.facts, window_id="meta-learning")
|
|
629
|
+
logger.debug(
|
|
630
|
+
"Meta-learning: %d facts extracted and stored",
|
|
631
|
+
len(meta_result.facts),
|
|
632
|
+
)
|
|
633
|
+
except Exception as exc:
|
|
634
|
+
logger.debug("Meta-learning extraction failed: %s", exc)
|
|
635
|
+
|
|
636
|
+
return (output, {"finish_reason": finish_reason})
|
|
637
|
+
except Exception as exc:
|
|
638
|
+
logger.debug("Meta-learning dispatch failed: %s", exc)
|
|
639
|
+
return ("", {})
|
|
640
|
+
|
|
641
|
+
return _meta_dispatch
|
|
642
|
+
|
|
643
|
+
# ------------------------------------------------------------------
|
|
644
|
+
# Guards
|
|
645
|
+
# ------------------------------------------------------------------
|
|
646
|
+
|
|
647
|
+
def _check_session(self) -> None:
|
|
648
|
+
"""Raise if session is closed or expired."""
|
|
649
|
+
if self._closed:
|
|
650
|
+
raise SessionClosedError()
|
|
651
|
+
if self._session.is_expired:
|
|
652
|
+
raise SessionExpiredError()
|
|
653
|
+
|
|
654
|
+
def _check_budget(self, input_tokens: int) -> None:
|
|
655
|
+
"""Raise BudgetExhaustedError if caps would be exceeded."""
|
|
656
|
+
max_w = self._config.max_windows_per_session
|
|
657
|
+
if max_w and self._windows_completed >= max_w:
|
|
658
|
+
raise BudgetExhaustedError(
|
|
659
|
+
"Max windows per session exceeded",
|
|
660
|
+
limit=max_w,
|
|
661
|
+
used=self._windows_completed,
|
|
662
|
+
)
|
|
663
|
+
max_in = self._config.max_total_input_tokens
|
|
664
|
+
if max_in and (self._total_input_tokens + input_tokens) > max_in:
|
|
665
|
+
raise BudgetExhaustedError(
|
|
666
|
+
"Max total input tokens exceeded",
|
|
667
|
+
limit=max_in,
|
|
668
|
+
used=self._total_input_tokens,
|
|
669
|
+
requested=input_tokens,
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
# ------------------------------------------------------------------
|
|
673
|
+
# Internal: extraction + fact storage
|
|
674
|
+
def session_status(self) -> SessionStatus:
|
|
675
|
+
"""Get live session metrics."""
|
|
676
|
+
self._check_session()
|
|
677
|
+
|
|
678
|
+
# RBAC permission check (§7.10)
|
|
679
|
+
from crp.security.rbac import Permission
|
|
680
|
+
perm_result = self._rbac.check_permission(Permission.READ_STATE)
|
|
681
|
+
if not perm_result.allowed:
|
|
682
|
+
raise RateLimitExceededError(perm_result.reason)
|
|
683
|
+
|
|
684
|
+
remaining = None
|
|
685
|
+
max_w = self._config.max_windows_per_session
|
|
686
|
+
max_in = self._config.max_total_input_tokens
|
|
687
|
+
max_out = self._config.max_total_output_tokens
|
|
688
|
+
if max_w or max_in or max_out:
|
|
689
|
+
remaining = RemainingBudget(
|
|
690
|
+
windows_remaining=(max_w - self._windows_completed) if max_w else None,
|
|
691
|
+
input_tokens_remaining=(max_in - self._total_input_tokens) if max_in else None,
|
|
692
|
+
output_tokens_remaining=(max_out - self._total_output_tokens) if max_out else None,
|
|
693
|
+
)
|
|
694
|
+
|
|
695
|
+
# Overhead from continuation windows
|
|
696
|
+
total = self._windows_completed
|
|
697
|
+
overhead = self._continuation_windows_total / total if total > 0 else 0.0
|
|
698
|
+
|
|
699
|
+
# USD cost — compute from provider pricing if available
|
|
700
|
+
cost: float | None = None
|
|
701
|
+
in_cost, out_cost = self._provider.cost_per_1k_tokens()
|
|
702
|
+
if in_cost > 0 or out_cost > 0:
|
|
703
|
+
cost = (
|
|
704
|
+
(self._total_input_tokens / 1000 * in_cost)
|
|
705
|
+
+ (self._total_output_tokens / 1000 * out_cost)
|
|
706
|
+
)
|
|
707
|
+
|
|
708
|
+
return SessionStatus(
|
|
709
|
+
session_id=self._session.session_id,
|
|
710
|
+
windows_completed=self._windows_completed,
|
|
711
|
+
total_input_tokens=self._total_input_tokens,
|
|
712
|
+
total_output_tokens=self._total_output_tokens,
|
|
713
|
+
facts_in_warm_state=self._warm_store.fact_count,
|
|
714
|
+
overhead_ratio=overhead,
|
|
715
|
+
remaining_budget=remaining,
|
|
716
|
+
total_cost=cost,
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
def estimate_session(
|
|
720
|
+
self,
|
|
721
|
+
system_prompt: str = "",
|
|
722
|
+
task_input: str = "",
|
|
723
|
+
*,
|
|
724
|
+
planned_dispatches: int = 1,
|
|
725
|
+
avg_output_tokens: int | None = None,
|
|
726
|
+
) -> CostEstimate:
|
|
727
|
+
"""Pre-flight cost estimation WITHOUT executing LLM calls.
|
|
728
|
+
|
|
729
|
+
Args:
|
|
730
|
+
system_prompt: Representative system prompt for token estimation.
|
|
731
|
+
task_input: Representative task input for token estimation.
|
|
732
|
+
planned_dispatches: Number of dispatches planned (default: 1).
|
|
733
|
+
avg_output_tokens: Expected average output tokens per dispatch.
|
|
734
|
+
If None, assumes model uses full generation reserve.
|
|
735
|
+
|
|
736
|
+
Returns:
|
|
737
|
+
CostEstimate with token estimates and USD cost (if pricing known).
|
|
738
|
+
"""
|
|
739
|
+
self._check_session()
|
|
740
|
+
|
|
741
|
+
# RBAC permission check (§7.10)
|
|
742
|
+
from crp.security.rbac import Permission
|
|
743
|
+
perm_result = self._rbac.check_permission(Permission.READ_STATE)
|
|
744
|
+
if not perm_result.allowed:
|
|
745
|
+
raise RateLimitExceededError(perm_result.reason)
|
|
746
|
+
|
|
747
|
+
s_tokens = self._provider.count_tokens(system_prompt) if system_prompt else 0
|
|
748
|
+
t_tokens = self._provider.count_tokens(task_input) if task_input else 0
|
|
749
|
+
context_window = self._provider.context_window_size()
|
|
750
|
+
g = resolve_generation_reserve(
|
|
751
|
+
None, self._provider.max_output_tokens, context_window,
|
|
752
|
+
is_thinking_model=getattr(self._provider, "is_thinking_model", False),
|
|
753
|
+
)
|
|
754
|
+
|
|
755
|
+
estimated_input_per = s_tokens + t_tokens
|
|
756
|
+
estimated_output_per = avg_output_tokens or g
|
|
757
|
+
total_input = estimated_input_per * planned_dispatches
|
|
758
|
+
total_output = estimated_output_per * planned_dispatches
|
|
759
|
+
|
|
760
|
+
# USD cost estimation via provider pricing
|
|
761
|
+
cost_usd: float | None = None
|
|
762
|
+
in_cost, out_cost = self._provider.cost_per_1k_tokens()
|
|
763
|
+
if in_cost > 0 or out_cost > 0:
|
|
764
|
+
cost_usd = (total_input / 1000 * in_cost) + (total_output / 1000 * out_cost)
|
|
765
|
+
|
|
766
|
+
confidence = "medium"
|
|
767
|
+
if planned_dispatches > 10:
|
|
768
|
+
confidence = "low"
|
|
769
|
+
elif planned_dispatches > 1 and system_prompt and task_input:
|
|
770
|
+
confidence = "high"
|
|
771
|
+
|
|
772
|
+
return CostEstimate(
|
|
773
|
+
estimated_windows=planned_dispatches,
|
|
774
|
+
estimated_input_tokens=total_input,
|
|
775
|
+
estimated_output_tokens=total_output,
|
|
776
|
+
estimated_cost_usd=cost_usd,
|
|
777
|
+
confidence=confidence,
|
|
778
|
+
)
|
|
779
|
+
|
|
780
|
+
def preview_envelope(
|
|
781
|
+
self, system_prompt: str, task_input: str
|
|
782
|
+
) -> EnvelopePreview:
|
|
783
|
+
"""Inspect envelope contents WITHOUT dispatching."""
|
|
784
|
+
self._check_session()
|
|
785
|
+
|
|
786
|
+
# RBAC permission check (§7.10)
|
|
787
|
+
from crp.security.rbac import Permission
|
|
788
|
+
perm_result = self._rbac.check_permission(Permission.READ_ENVELOPE)
|
|
789
|
+
if not perm_result.allowed:
|
|
790
|
+
raise RateLimitExceededError(perm_result.reason)
|
|
791
|
+
|
|
792
|
+
s_tokens = self._provider.count_tokens(system_prompt)
|
|
793
|
+
t_tokens = self._provider.count_tokens(task_input)
|
|
794
|
+
context_window = self._provider.context_window_size()
|
|
795
|
+
g = resolve_generation_reserve(
|
|
796
|
+
None, self._provider.max_output_tokens, context_window,
|
|
797
|
+
is_thinking_model=getattr(self._provider, "is_thinking_model", False),
|
|
798
|
+
)
|
|
799
|
+
|
|
800
|
+
# Build a real envelope preview from accumulated facts
|
|
801
|
+
envelope_result = self._build_envelope(system_prompt, task_input, g)
|
|
802
|
+
|
|
803
|
+
return EnvelopePreview(
|
|
804
|
+
total_tokens=s_tokens + t_tokens + envelope_result.envelope_tokens,
|
|
805
|
+
envelope_tokens=envelope_result.envelope_tokens,
|
|
806
|
+
generation_reserve=g,
|
|
807
|
+
facts_included=envelope_result.facts_included,
|
|
808
|
+
facts_available=self._warm_store.fact_count,
|
|
809
|
+
saturation=envelope_result.saturation,
|
|
810
|
+
)
|
|
811
|
+
|
|
812
|
+
# ------------------------------------------------------------------
|
|
813
|
+
# Configuration
|
|
814
|
+
# ------------------------------------------------------------------
|
|
815
|
+
|
|
816
|
+
def configure(self, **kwargs: Any) -> None:
|
|
817
|
+
"""Runtime configuration changes (mutable fields only)."""
|
|
818
|
+
self._check_session()
|
|
819
|
+
|
|
820
|
+
# RBAC permission check (§7.10)
|
|
821
|
+
from crp.security.rbac import Permission
|
|
822
|
+
perm_result = self._rbac.check_permission(Permission.CONFIGURE)
|
|
823
|
+
if not perm_result.allowed:
|
|
824
|
+
raise RateLimitExceededError(perm_result.reason)
|
|
825
|
+
|
|
826
|
+
from crp.ckf.fabric import CKFConfig, ContextualKnowledgeFabric
|
|
827
|
+
from crp.state.warm_store import WarmStateStore, WarmStoreConfig
|
|
828
|
+
|
|
829
|
+
self._config.update(kwargs)
|
|
830
|
+
|
|
831
|
+
# ------------------------------------------------------------------
|
|
832
|
+
# State management
|
|
833
|
+
# ------------------------------------------------------------------
|
|
834
|
+
|
|
835
|
+
def reset_session(self) -> None:
|
|
836
|
+
"""Clear warm state, CKF, event log, DAG. Keeps session open."""
|
|
837
|
+
self._check_session()
|
|
838
|
+
|
|
839
|
+
# ---------- Compliance imports (§7.14) ----------
|
|
840
|
+
from crp.security.audit_trail import ComplianceEventType
|
|
841
|
+
|
|
842
|
+
# ---------- Compliance audit: session reset (§7.14) ----------
|
|
843
|
+
self._compliance_audit.record(
|
|
844
|
+
ComplianceEventType.DATA_DELETED,
|
|
845
|
+
session_id=self._session.session_id,
|
|
846
|
+
data={
|
|
847
|
+
"operation": "reset_session",
|
|
848
|
+
"facts_deleted": self._warm_store.fact_count,
|
|
849
|
+
"windows_reset": self._windows_completed,
|
|
850
|
+
},
|
|
851
|
+
)
|
|
852
|
+
|
|
853
|
+
# RBAC permission check (§7.10) — destructive, requires ADMIN
|
|
854
|
+
from crp.security.rbac import Permission
|
|
855
|
+
perm_result = self._rbac.check_permission(Permission.MANAGE_SESSIONS)
|
|
856
|
+
if not perm_result.allowed:
|
|
857
|
+
self._compliance_audit.record(
|
|
858
|
+
ComplianceEventType.RBAC_DENIED,
|
|
859
|
+
session_id=self._session.session_id,
|
|
860
|
+
data={"operation": "reset_session", "reason": perm_result.reason},
|
|
861
|
+
)
|
|
862
|
+
raise RateLimitExceededError(perm_result.reason)
|
|
863
|
+
from crp.ckf.fabric import CKFConfig, ContextualKnowledgeFabric
|
|
864
|
+
from crp.security.integrity import FactIntegrityChain
|
|
865
|
+
from crp.security.quarantine import IngestQuarantine
|
|
866
|
+
from crp.state.warm_store import WarmStateStore, WarmStoreConfig
|
|
867
|
+
|
|
868
|
+
self._windows_completed = 0
|
|
869
|
+
self._total_input_tokens = 0
|
|
870
|
+
self._total_output_tokens = 0
|
|
871
|
+
self._continuation_windows_total = 0
|
|
872
|
+
self._dag.clear()
|
|
873
|
+
# Reset all subsystems
|
|
874
|
+
self._warm_store = WarmStateStore(WarmStoreConfig(max_facts=10_000))
|
|
875
|
+
self._ckf = ContextualKnowledgeFabric(CKFConfig(max_facts=10_000))
|
|
876
|
+
self._extraction_history.clear()
|
|
877
|
+
# Reset security subsystems
|
|
878
|
+
self._integrity_chain = FactIntegrityChain(
|
|
879
|
+
session_key=self._session_binding.session_key,
|
|
880
|
+
)
|
|
881
|
+
self._quarantine = IngestQuarantine()
|
|
882
|
+
self._rbac.reset_limits()
|
|
883
|
+
logger.info("Session %s reset (all subsystems)", self._session.session_id)
|
|
884
|
+
|
|
885
|
+
# ------------------------------------------------------------------
|
|
886
|
+
# Public API: Observability (§9 — EventEmitter access)
|
|
887
|
+
# ------------------------------------------------------------------
|
|
888
|
+
|
|
889
|
+
@property
|
|
890
|
+
def emitter(self):
|
|
891
|
+
"""Access the protocol event bus for subscribing to events."""
|
|
892
|
+
return self._emitter
|
|
893
|
+
|
|
894
|
+
def on(self, event_type: str, listener) -> None:
|
|
895
|
+
"""Subscribe to a protocol event (convenience wrapper).
|
|
896
|
+
|
|
897
|
+
Usage::
|
|
898
|
+
|
|
899
|
+
def on_dispatch(event):
|
|
900
|
+
print(f"Dispatch completed: {event.data}")
|
|
901
|
+
|
|
902
|
+
orch.on("dispatch.completed", on_dispatch)
|
|
903
|
+
"""
|
|
904
|
+
self._emitter.on(event_type, listener)
|
|
905
|
+
|
|
906
|
+
# ------------------------------------------------------------------
|
|
907
|
+
# Public API: Feedback Loop (§feedback — fact confidence adjustment)
|
|
908
|
+
# ------------------------------------------------------------------
|
|
909
|
+
|
|
910
|
+
@property
|
|
911
|
+
def feedback(self):
|
|
912
|
+
"""Access the feedback loop for fact confidence adjustments."""
|
|
913
|
+
return self._feedback_loop
|
|
914
|
+
|
|
915
|
+
def boost_fact(self, fact_id: str, delta: float = 0.1, reason: str = "") -> None:
|
|
916
|
+
"""Boost a fact's confidence (positive feedback)."""
|
|
917
|
+
self._feedback_loop.boost_confidence(fact_id, delta, reason)
|
|
918
|
+
self._emitter.emit("fact.boosted", {"fact_id": fact_id, "delta": delta})
|
|
919
|
+
|
|
920
|
+
def penalize_fact(self, fact_id: str, delta: float = -0.2, reason: str = "") -> None:
|
|
921
|
+
"""Penalize a fact's confidence (negative feedback)."""
|
|
922
|
+
self._feedback_loop.penalize_confidence(fact_id, delta, reason)
|
|
923
|
+
self._emitter.emit("fact.penalized", {"fact_id": fact_id, "delta": delta})
|
|
924
|
+
|
|
925
|
+
def reject_fact(self, fact_id: str, reason: str = "") -> None:
|
|
926
|
+
"""Reject a fact entirely (user override)."""
|
|
927
|
+
self._feedback_loop.reject_fact(fact_id, reason)
|
|
928
|
+
self._emitter.emit("fact.rejected", {"fact_id": fact_id, "reason": reason})
|
|
929
|
+
|
|
930
|
+
# ------------------------------------------------------------------
|
|
931
|
+
# Public API: Provider manager (§05 — multi-provider routing)
|
|
932
|
+
# ------------------------------------------------------------------
|
|
933
|
+
|
|
934
|
+
def register_provider(self, provider: LLMProvider) -> None:
|
|
935
|
+
"""Register an additional LLM provider for fallback routing."""
|
|
936
|
+
self._provider_manager.register(provider)
|
|
937
|
+
self._emitter.emit("provider.connected", {
|
|
938
|
+
"model": provider.model_name,
|
|
939
|
+
})
|
|
940
|
+
|
|
941
|
+
# ------------------------------------------------------------------
|
|
942
|
+
# Public API: Parallel fan-out (§parallel)
|
|
943
|
+
# ------------------------------------------------------------------
|
|
944
|
+
|
|
945
|
+
@property
|
|
946
|
+
def parallel(self):
|
|
947
|
+
"""Access the parallel fan-out engine for multi-task dispatch."""
|
|
948
|
+
return self._parallel_fanout
|
|
949
|
+
|
|
950
|
+
# ------------------------------------------------------------------
|
|
951
|
+
# Session close
|
|
952
|
+
# ------------------------------------------------------------------
|
|
953
|
+
|
|
954
|
+
def _atexit_close(self) -> None:
|
|
955
|
+
"""Safety-net close called by atexit — never raises (§audit H6)."""
|
|
956
|
+
try:
|
|
957
|
+
# Suppress logging during shutdown to avoid I/O-on-closed-file errors
|
|
958
|
+
logging.disable(logging.CRITICAL)
|
|
959
|
+
# Use timeout to prevent deadlock if lock is held (§audit3 ORCH-C5)
|
|
960
|
+
acquired = self._lock.acquire(timeout=5.0)
|
|
961
|
+
if acquired:
|
|
962
|
+
try:
|
|
963
|
+
self._close_locked()
|
|
964
|
+
finally:
|
|
965
|
+
self._lock.release()
|
|
966
|
+
except Exception: # noqa: BLE001
|
|
967
|
+
pass
|
|
968
|
+
finally:
|
|
969
|
+
logging.disable(logging.NOTSET)
|
|
970
|
+
|
|
971
|
+
def close(self) -> None:
|
|
972
|
+
"""Close session — flush warm→cold, persist CKF, zero keys (§2.5)."""
|
|
973
|
+
with self._lock:
|
|
974
|
+
self._close_locked()
|
|
975
|
+
|
|
976
|
+
def _close_locked(self) -> None:
|
|
977
|
+
"""Internal close implementation — called under self._lock."""
|
|
978
|
+
if self._closed:
|
|
979
|
+
return
|
|
980
|
+
|
|
981
|
+
# Shut down thread pool executor (§audit3: wait for in-flight tasks)
|
|
982
|
+
self._executor.shutdown(wait=True)
|
|
983
|
+
|
|
984
|
+
# ---------- Compliance imports (§7.14) ----------
|
|
985
|
+
from crp.security.audit_trail import ComplianceEventType
|
|
986
|
+
|
|
987
|
+
# ---------- Compliance audit: session closing (§7.14) ----------
|
|
988
|
+
self._compliance_audit.record(
|
|
989
|
+
ComplianceEventType.SESSION_CLOSED,
|
|
990
|
+
session_id=self._session.session_id,
|
|
991
|
+
data={
|
|
992
|
+
"windows_completed": self._windows_completed,
|
|
993
|
+
"total_input_tokens": self._total_input_tokens,
|
|
994
|
+
"total_output_tokens": self._total_output_tokens,
|
|
995
|
+
"facts_in_warm_store": self._warm_store.fact_count,
|
|
996
|
+
"audit_entries": self._compliance_audit.entry_count,
|
|
997
|
+
"processing_records": self._processing_records.activity_count,
|
|
998
|
+
"retention_tracked": self._retention_manager.tracked_count,
|
|
999
|
+
"lineage_tracked": len(self._lineage_tracker.to_dict().get("entries", {})),
|
|
1000
|
+
},
|
|
1001
|
+
)
|
|
1002
|
+
|
|
1003
|
+
# Flush warm state facts → CKF cold storage for cross-session retrieval
|
|
1004
|
+
try:
|
|
1005
|
+
# Validate session_id to prevent path traversal (§audit5 SEC-M1)
|
|
1006
|
+
sid = self._session.session_id
|
|
1007
|
+
if not _UUID_RE.fullmatch(sid):
|
|
1008
|
+
logger.error("Invalid session_id format in close: %r", sid[:40])
|
|
1009
|
+
raise ValueError(f"session_id is not a valid UUID: {sid!r:.40}")
|
|
1010
|
+
persist_dir = os.path.join(
|
|
1011
|
+
os.environ.get("CRP_DATA_DIR", "."),
|
|
1012
|
+
"crp_sessions",
|
|
1013
|
+
)
|
|
1014
|
+
os.makedirs(persist_dir, exist_ok=True)
|
|
1015
|
+
persist_path = os.path.join(
|
|
1016
|
+
persist_dir,
|
|
1017
|
+
f"{sid}.json",
|
|
1018
|
+
)
|
|
1019
|
+
self._ckf.persist(persist_path)
|
|
1020
|
+
logger.debug("CKF persisted to cold storage: %s", persist_path)
|
|
1021
|
+
except Exception as exc:
|
|
1022
|
+
logger.warning("CKF persistence failed: %s", exc)
|
|
1023
|
+
|
|
1024
|
+
# Persist encrypted event log (§7.3)
|
|
1025
|
+
if self._config.get("encrypt_cold_state", True):
|
|
1026
|
+
try:
|
|
1027
|
+
event_data = json.dumps({
|
|
1028
|
+
"session_id": self._session.session_id,
|
|
1029
|
+
"windows_completed": self._windows_completed,
|
|
1030
|
+
"integrity_chain": self._integrity_chain.to_dict(),
|
|
1031
|
+
}, separators=(",", ":")).encode("utf-8")
|
|
1032
|
+
persist_dir = os.path.join(
|
|
1033
|
+
os.environ.get("CRP_DATA_DIR", "."),
|
|
1034
|
+
"crp_sessions",
|
|
1035
|
+
)
|
|
1036
|
+
os.makedirs(persist_dir, exist_ok=True)
|
|
1037
|
+
event_blob = self._encryptor.encrypt_event_log(event_data)
|
|
1038
|
+
event_path = os.path.join(
|
|
1039
|
+
persist_dir,
|
|
1040
|
+
f"{self._session.session_id}.events.bin",
|
|
1041
|
+
)
|
|
1042
|
+
with open(event_path, "wb") as f:
|
|
1043
|
+
f.write(event_blob.ciphertext)
|
|
1044
|
+
logger.debug("Encrypted event log persisted: %s", event_path)
|
|
1045
|
+
except Exception as exc:
|
|
1046
|
+
logger.debug("Event log persistence skipped: %s", exc)
|
|
1047
|
+
|
|
1048
|
+
# ---------- Persist compliance audit trail (§7.14) ----------
|
|
1049
|
+
try:
|
|
1050
|
+
persist_dir = os.path.join(
|
|
1051
|
+
os.environ.get("CRP_DATA_DIR", "."),
|
|
1052
|
+
"crp_sessions",
|
|
1053
|
+
)
|
|
1054
|
+
os.makedirs(persist_dir, exist_ok=True)
|
|
1055
|
+
audit_path = os.path.join(
|
|
1056
|
+
persist_dir,
|
|
1057
|
+
f"{self._session.session_id}.audit.jsonl",
|
|
1058
|
+
)
|
|
1059
|
+
audit_jsonl = self._compliance_audit.export_jsonl()
|
|
1060
|
+
with open(audit_path, "w", encoding="utf-8") as f:
|
|
1061
|
+
f.write(audit_jsonl)
|
|
1062
|
+
logger.debug("Compliance audit trail persisted: %s (%d entries)",
|
|
1063
|
+
audit_path, self._compliance_audit.entry_count)
|
|
1064
|
+
except Exception as exc:
|
|
1065
|
+
logger.error("Compliance audit trail persistence failed: %s", exc)
|
|
1066
|
+
|
|
1067
|
+
# Zero out sensitive state from memory
|
|
1068
|
+
self._extraction_history.clear()
|
|
1069
|
+
|
|
1070
|
+
# Final resource GC
|
|
1071
|
+
mgr = getattr(self, "_resource_manager", None)
|
|
1072
|
+
if mgr is not None:
|
|
1073
|
+
mgr.run_gc()
|
|
1074
|
+
|
|
1075
|
+
self._closed = True
|
|
1076
|
+
logger.info(
|
|
1077
|
+
"Session %s closed — %d windows, %d input tokens, %d output tokens",
|
|
1078
|
+
self._session.session_id,
|
|
1079
|
+
self._windows_completed,
|
|
1080
|
+
self._total_input_tokens,
|
|
1081
|
+
self._total_output_tokens,
|
|
1082
|
+
)
|
|
1083
|
+
|
|
1084
|
+
# Emit session.closed event and stop emitter (§9)
|
|
1085
|
+
self._emitter.emit("session.closed", {
|
|
1086
|
+
"session_id": self._session.session_id,
|
|
1087
|
+
"windows_completed": self._windows_completed,
|
|
1088
|
+
"total_input_tokens": self._total_input_tokens,
|
|
1089
|
+
"total_output_tokens": self._total_output_tokens,
|
|
1090
|
+
})
|
|
1091
|
+
self._emitter.stop()
|
|
1092
|
+
|
|
1093
|
+
# Close telemetry writer if active (§9, D8 fix)
|
|
1094
|
+
if self._telemetry_writer is not None:
|
|
1095
|
+
self._telemetry_writer.close()
|
|
1096
|
+
self._telemetry_writer = None
|
|
1097
|
+
|
|
1098
|
+
# Opportunistic session file cleanup (§audit H11)
|
|
1099
|
+
try:
|
|
1100
|
+
from crp.state.session_cleanup import cleanup_expired_sessions
|
|
1101
|
+
cleanup_expired_sessions()
|
|
1102
|
+
except Exception: # noqa: BLE001
|
|
1103
|
+
pass # best-effort, don't fail close
|
|
1104
|
+
|
|
1105
|
+
# ------------------------------------------------------------------
|
|
1106
|
+
# Session resume — cross-session continuity (§2.5)
|
|
1107
|
+
# ------------------------------------------------------------------
|
|
1108
|
+
|
|
1109
|
+
@classmethod
|
|
1110
|
+
def resume(
|
|
1111
|
+
cls,
|
|
1112
|
+
session_id: str,
|
|
1113
|
+
provider: LLMProvider | None = None,
|
|
1114
|
+
*,
|
|
1115
|
+
llm: LLMProvider | None = None,
|
|
1116
|
+
data_dir: str | None = None,
|
|
1117
|
+
**kwargs: Any,
|
|
1118
|
+
) -> "CRPOrchestrator":
|
|
1119
|
+
"""Resume a previously closed session by restoring CKF state.
|
|
1120
|
+
|
|
1121
|
+
Loads persisted facts from cold storage into a new orchestrator
|
|
1122
|
+
instance, providing cross-session knowledge continuity.
|
|
1123
|
+
|
|
1124
|
+
Args:
|
|
1125
|
+
session_id: The session_id from the previous session.
|
|
1126
|
+
provider: LLM provider to use for the resumed session.
|
|
1127
|
+
data_dir: Override for CRP_DATA_DIR (default: env or '.').
|
|
1128
|
+
|
|
1129
|
+
Returns:
|
|
1130
|
+
A new CRPOrchestrator with the previous session's facts loaded.
|
|
1131
|
+
"""
|
|
1132
|
+
base_dir = data_dir or os.environ.get("CRP_DATA_DIR", ".")
|
|
1133
|
+
# Validate session_id to prevent path traversal (§audit4 SEC-M2)
|
|
1134
|
+
if not _UUID_RE.fullmatch(session_id):
|
|
1135
|
+
raise ValidationError(
|
|
1136
|
+
f"Invalid session_id format: expected UUID, got {session_id!r:.40}"
|
|
1137
|
+
)
|
|
1138
|
+
persist_path = os.path.join(base_dir, "crp_sessions", f"{session_id}.json")
|
|
1139
|
+
|
|
1140
|
+
# Create new orchestrator
|
|
1141
|
+
orch = cls(provider=provider, llm=llm, **kwargs)
|
|
1142
|
+
|
|
1143
|
+
# Restore CKF state from cold storage
|
|
1144
|
+
if os.path.exists(persist_path):
|
|
1145
|
+
try:
|
|
1146
|
+
orch._ckf.restore(persist_path)
|
|
1147
|
+
logger.info(
|
|
1148
|
+
"Resumed session: loaded CKF from %s (%d facts)",
|
|
1149
|
+
persist_path,
|
|
1150
|
+
len(orch._ckf._warm._facts),
|
|
1151
|
+
)
|
|
1152
|
+
# Pre-populate WarmStore with restored CKF facts for
|
|
1153
|
+
# immediate availability in first envelope
|
|
1154
|
+
restored_facts = [sf.fact for sf in orch._ckf._warm._facts.values()]
|
|
1155
|
+
if restored_facts:
|
|
1156
|
+
orch._warm_store.add_facts(restored_facts)
|
|
1157
|
+
logger.info("Restored %d facts into WarmStore", len(restored_facts))
|
|
1158
|
+
except Exception as exc:
|
|
1159
|
+
logger.warning("CKF restore failed for session %s: %s", session_id, exc)
|
|
1160
|
+
else:
|
|
1161
|
+
logger.warning("No persisted session found at %s", persist_path)
|
|
1162
|
+
|
|
1163
|
+
return orch
|
|
1164
|
+
|
|
1165
|
+
# ------------------------------------------------------------------
|
|
1166
|
+
# Streaming dispatch (§6.10.5)
|
|
1167
|
+
def export_state(self, fmt: str | None = None) -> bytes:
|
|
1168
|
+
"""Export session state as encrypted bytes (§2.5).
|
|
1169
|
+
|
|
1170
|
+
Includes warm store facts, critical state, structural state, and CKF health.
|
|
1171
|
+
Embeddings are NOT exported (§7.11) — recomputed on import.
|
|
1172
|
+
|
|
1173
|
+
Args:
|
|
1174
|
+
fmt: Export format (reserved, currently only 'json' supported).
|
|
1175
|
+
|
|
1176
|
+
Returns:
|
|
1177
|
+
Encrypted bytes (AES-256-GCM with session-bound key).
|
|
1178
|
+
"""
|
|
1179
|
+
self._check_session()
|
|
1180
|
+
|
|
1181
|
+
# ---------- Compliance imports (§7.14) ----------
|
|
1182
|
+
from crp.security.audit_trail import ComplianceEventType
|
|
1183
|
+
from crp.security.consent import ProcessingPurpose
|
|
1184
|
+
|
|
1185
|
+
# ---------- Consent verification for export (§7.13) ----------
|
|
1186
|
+
self._consent_manager.check_required(ProcessingPurpose.EXPORT)
|
|
1187
|
+
|
|
1188
|
+
# RBAC permission check (§7.10)
|
|
1189
|
+
from crp.security.rbac import Permission
|
|
1190
|
+
perm_result = self._rbac.check_permission(Permission.EXPORT_STATE)
|
|
1191
|
+
if not perm_result.allowed:
|
|
1192
|
+
self._compliance_audit.record(
|
|
1193
|
+
ComplianceEventType.RBAC_DENIED,
|
|
1194
|
+
session_id=self._session.session_id,
|
|
1195
|
+
data={"operation": "export_state", "reason": perm_result.reason},
|
|
1196
|
+
)
|
|
1197
|
+
raise RateLimitExceededError(perm_result.reason)
|
|
1198
|
+
|
|
1199
|
+
# ---------- Compliance audit: export started (§7.14) ----------
|
|
1200
|
+
self._compliance_audit.record(
|
|
1201
|
+
ComplianceEventType.DATA_EXPORTED,
|
|
1202
|
+
session_id=self._session.session_id,
|
|
1203
|
+
data={
|
|
1204
|
+
"operation": "export_state",
|
|
1205
|
+
"phase": "started",
|
|
1206
|
+
"format": fmt or "json",
|
|
1207
|
+
"facts_count": self._warm_store.fact_count,
|
|
1208
|
+
},
|
|
1209
|
+
)
|
|
1210
|
+
|
|
1211
|
+
# ---------- Processing record for export (§7.13 — GDPR Art. 30) ----------
|
|
1212
|
+
self._processing_records.record(
|
|
1213
|
+
purpose=ProcessingPurpose.EXPORT,
|
|
1214
|
+
data_categories=["session_state", "warm_store_facts", "ckf_state"],
|
|
1215
|
+
legal_basis="legitimate_interest",
|
|
1216
|
+
input_size_bytes=0,
|
|
1217
|
+
output_size_bytes=0,
|
|
1218
|
+
automated_decision=True,
|
|
1219
|
+
human_oversight=False,
|
|
1220
|
+
retention_period="external",
|
|
1221
|
+
)
|
|
1222
|
+
|
|
1223
|
+
ckf_health = self._ckf.health()
|
|
1224
|
+
state = {
|
|
1225
|
+
"session_id": self._session.session_id,
|
|
1226
|
+
"protocol_version": self._session.protocol_version,
|
|
1227
|
+
"windows_completed": self._windows_completed,
|
|
1228
|
+
"total_input_tokens": self._total_input_tokens,
|
|
1229
|
+
"total_output_tokens": self._total_output_tokens,
|
|
1230
|
+
"facts_in_warm_state": self._warm_store.fact_count,
|
|
1231
|
+
"ckf_facts": ckf_health.fact_count,
|
|
1232
|
+
"ckf_edges": ckf_health.edge_count,
|
|
1233
|
+
"ckf_communities": ckf_health.community_count,
|
|
1234
|
+
"warm_store": self._warm_store.to_dict(),
|
|
1235
|
+
"format": fmt or "json",
|
|
1236
|
+
"exported_at": time.time(),
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
# Strip embeddings from export (§7.11)
|
|
1240
|
+
from crp.security.embedding_defense import EmbeddingDefense
|
|
1241
|
+
state = EmbeddingDefense.strip_embeddings_for_export(state)
|
|
1242
|
+
|
|
1243
|
+
# Include integrity chain signature for verification on import
|
|
1244
|
+
if self._integrity_chain.size > 0:
|
|
1245
|
+
try:
|
|
1246
|
+
state["integrity_chain_signature"] = self._integrity_chain.chain_signature()
|
|
1247
|
+
except ValueError:
|
|
1248
|
+
pass # No session key — skip chain signature
|
|
1249
|
+
|
|
1250
|
+
plaintext = json.dumps(state, separators=(",", ":")).encode("utf-8")
|
|
1251
|
+
|
|
1252
|
+
# Encrypt with session-bound AES-256-GCM key (§7.3)
|
|
1253
|
+
blob = self._encryptor.encrypt_cold_state(plaintext)
|
|
1254
|
+
|
|
1255
|
+
# ---------- Compliance audit: export completed (§7.14) ----------
|
|
1256
|
+
self._compliance_audit.record(
|
|
1257
|
+
ComplianceEventType.DATA_EXPORTED,
|
|
1258
|
+
session_id=self._session.session_id,
|
|
1259
|
+
data={
|
|
1260
|
+
"operation": "export_state",
|
|
1261
|
+
"phase": "completed",
|
|
1262
|
+
"format": fmt or "json",
|
|
1263
|
+
"ciphertext_bytes": len(blob.ciphertext),
|
|
1264
|
+
"facts_exported": self._warm_store.fact_count,
|
|
1265
|
+
},
|
|
1266
|
+
)
|
|
1267
|
+
|
|
1268
|
+
return blob.ciphertext
|
|
1269
|
+
|
|
1270
|
+
# ------------------------------------------------------------------
|
|
1271
|
+
# Properties
|
|
1272
|
+
# ------------------------------------------------------------------
|
|
1273
|
+
|
|
1274
|
+
@property
|
|
1275
|
+
def session(self) -> SessionHandle:
|
|
1276
|
+
return self._session
|
|
1277
|
+
|
|
1278
|
+
@property
|
|
1279
|
+
def dag(self) -> WindowDAG:
|
|
1280
|
+
return self._dag
|
|
1281
|
+
|
|
1282
|
+
@property
|
|
1283
|
+
def config(self) -> CRPConfig:
|
|
1284
|
+
return self._config
|
|
1285
|
+
|
|
1286
|
+
@property
|
|
1287
|
+
def warm_store(self) -> WarmStateStore:
|
|
1288
|
+
"""Access the WarmStateStore for direct fact operations."""
|
|
1289
|
+
return self._warm_store
|
|
1290
|
+
|
|
1291
|
+
@property
|
|
1292
|
+
def ckf(self) -> ContextualKnowledgeFabric:
|
|
1293
|
+
"""Access the CKF for 4-mode retrieval queries."""
|
|
1294
|
+
return self._ckf
|
|
1295
|
+
|
|
1296
|
+
@property
|
|
1297
|
+
def compliance_audit(self) -> ComplianceAuditTrail:
|
|
1298
|
+
"""Access the compliance audit trail for inspection (§7.14)."""
|
|
1299
|
+
return self._compliance_audit
|
|
1300
|
+
|
|
1301
|
+
@property
|
|
1302
|
+
def pii_scanner(self) -> PIIScanner:
|
|
1303
|
+
"""Access the PII scanner (§7.12)."""
|
|
1304
|
+
return self._pii_scanner
|
|
1305
|
+
|
|
1306
|
+
@property
|
|
1307
|
+
def consent_manager(self) -> ConsentManager:
|
|
1308
|
+
"""Access the consent manager (§7.13)."""
|
|
1309
|
+
return self._consent_manager
|
|
1310
|
+
|
|
1311
|
+
@property
|
|
1312
|
+
def processing_records(self) -> ProcessingRecordKeeper:
|
|
1313
|
+
"""Access the processing record keeper — GDPR Art. 30 (§7.13)."""
|
|
1314
|
+
return self._processing_records
|
|
1315
|
+
|
|
1316
|
+
@property
|
|
1317
|
+
def retention_manager(self) -> RetentionManager:
|
|
1318
|
+
"""Access the retention manager (§7.12)."""
|
|
1319
|
+
return self._retention_manager
|
|
1320
|
+
|
|
1321
|
+
@property
|
|
1322
|
+
def lineage_tracker(self) -> DataLineageTracker:
|
|
1323
|
+
"""Access the data lineage tracker (§7.12)."""
|
|
1324
|
+
return self._lineage_tracker
|
|
1325
|
+
|
|
1326
|
+
@property
|
|
1327
|
+
def human_oversight(self) -> HumanOversightController:
|
|
1328
|
+
"""Access the human oversight controller — EU AI Act Art. 14 (§7.13)."""
|
|
1329
|
+
return self._human_oversight
|
|
1330
|
+
|
|
1331
|
+
@property
|
|
1332
|
+
def compliance_reporter(self) -> ComplianceReporter:
|
|
1333
|
+
"""Access the compliance reporter (§7.15)."""
|
|
1334
|
+
return self._compliance_reporter
|
|
1335
|
+
|
|
1336
|
+
@property
|
|
1337
|
+
def risk_classifier(self) -> RiskClassifier:
|
|
1338
|
+
"""Access the risk classifier — EU AI Act Art. 6 (§7.15)."""
|
|
1339
|
+
return self._risk_classifier
|
|
1340
|
+
|
|
1341
|
+
@property
|
|
1342
|
+
def extraction_pipeline(self) -> ExtractionPipeline:
|
|
1343
|
+
"""Access the extraction pipeline for configuration."""
|
|
1344
|
+
return self._extraction
|
|
1345
|
+
|
|
1346
|
+
# ------------------------------------------------------------------
|
|
1347
|
+
# Async API (§6.10) — asyncio.to_thread bridge
|
|
1348
|
+
# ------------------------------------------------------------------
|
|
1349
|
+
|
|
1350
|
+
async def async_dispatch(
|
|
1351
|
+
self,
|
|
1352
|
+
system_prompt: str,
|
|
1353
|
+
task_input: str,
|
|
1354
|
+
**kwargs: Any,
|
|
1355
|
+
) -> tuple[str, QualityReport]:
|
|
1356
|
+
"""Async version of dispatch() — runs in a configurable thread pool.
|
|
1357
|
+
|
|
1358
|
+
Uses the orchestrator's own ThreadPoolExecutor (sized via
|
|
1359
|
+
``max_threads`` config) instead of the default asyncio executor,
|
|
1360
|
+
which prevents thread-pool saturation under concurrent load.
|
|
1361
|
+
|
|
1362
|
+
Use from async code (FastAPI, asyncio, etc.)::
|
|
1363
|
+
|
|
1364
|
+
output, report = await client.async_dispatch(
|
|
1365
|
+
"You are helpful.", "Explain CRP."
|
|
1366
|
+
)
|
|
1367
|
+
"""
|
|
1368
|
+
import asyncio
|
|
1369
|
+
|
|
1370
|
+
loop = asyncio.get_running_loop()
|
|
1371
|
+
return await loop.run_in_executor(
|
|
1372
|
+
self._executor, lambda: self.dispatch(system_prompt, task_input, **kwargs)
|
|
1373
|
+
)
|
|
1374
|
+
|
|
1375
|
+
async def async_ingest(
|
|
1376
|
+
self,
|
|
1377
|
+
raw_text: str,
|
|
1378
|
+
*,
|
|
1379
|
+
source_label: str = "",
|
|
1380
|
+
) -> ExtractionResult:
|
|
1381
|
+
"""Async version of ingest() — runs in the orchestrator's thread pool."""
|
|
1382
|
+
import asyncio
|
|
1383
|
+
|
|
1384
|
+
loop = asyncio.get_running_loop()
|
|
1385
|
+
return await loop.run_in_executor(
|
|
1386
|
+
self._executor, lambda: self.ingest(raw_text, source_label=source_label)
|
|
1387
|
+
)
|
|
1388
|
+
|
|
1389
|
+
async def async_close(self) -> None:
|
|
1390
|
+
"""Async version of close() — runs in the orchestrator's thread pool."""
|
|
1391
|
+
import asyncio
|
|
1392
|
+
|
|
1393
|
+
loop = asyncio.get_running_loop()
|
|
1394
|
+
await loop.run_in_executor(self._executor, self.close)
|
|
1395
|
+
|
|
1396
|
+
async def async_dispatch_stream(
|
|
1397
|
+
self,
|
|
1398
|
+
system_prompt: str,
|
|
1399
|
+
task_input: str,
|
|
1400
|
+
**kwargs: Any,
|
|
1401
|
+
):
|
|
1402
|
+
"""Async streaming dispatch — yields StreamEvent objects.
|
|
1403
|
+
|
|
1404
|
+
Usage::
|
|
1405
|
+
|
|
1406
|
+
async for event in client.async_dispatch_stream(
|
|
1407
|
+
"You are helpful.", "Explain CRP."
|
|
1408
|
+
):
|
|
1409
|
+
if event.event_type == "token":
|
|
1410
|
+
print(event.data, end="")
|
|
1411
|
+
"""
|
|
1412
|
+
import asyncio
|
|
1413
|
+
import queue
|
|
1414
|
+
|
|
1415
|
+
q: queue.Queue[StreamEvent | None] = queue.Queue(maxsize=1000) # Bounded queue (§audit M12)
|
|
1416
|
+
|
|
1417
|
+
def _produce():
|
|
1418
|
+
try:
|
|
1419
|
+
for event in self.dispatch_stream(
|
|
1420
|
+
system_prompt, task_input, **kwargs
|
|
1421
|
+
):
|
|
1422
|
+
q.put(event)
|
|
1423
|
+
finally:
|
|
1424
|
+
q.put(None) # Sentinel
|
|
1425
|
+
|
|
1426
|
+
loop = asyncio.get_running_loop()
|
|
1427
|
+
fut = loop.run_in_executor(self._executor, _produce)
|
|
1428
|
+
|
|
1429
|
+
while True:
|
|
1430
|
+
event = await asyncio.to_thread(q.get)
|
|
1431
|
+
if event is None:
|
|
1432
|
+
break
|
|
1433
|
+
yield event
|
|
1434
|
+
|
|
1435
|
+
await fut # Propagate exceptions
|