retrievalloom 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. retrievalloom/__init__.py +197 -0
  2. retrievalloom/_logging.py +68 -0
  3. retrievalloom/config.py +392 -0
  4. retrievalloom/contracts/__init__.py +1 -0
  5. retrievalloom/contracts/component.py +119 -0
  6. retrievalloom/contracts/evidence.py +94 -0
  7. retrievalloom/contracts/indexes.py +304 -0
  8. retrievalloom/contracts/infra.py +76 -0
  9. retrievalloom/contracts/models.py +188 -0
  10. retrievalloom/domain/__init__.py +1 -0
  11. retrievalloom/domain/base.py +92 -0
  12. retrievalloom/domain/filters.py +177 -0
  13. retrievalloom/domain/hashing.py +81 -0
  14. retrievalloom/domain/inputs.py +269 -0
  15. retrievalloom/domain/records.py +389 -0
  16. retrievalloom/domain/reports.py +139 -0
  17. retrievalloom/domain/retrieval.py +355 -0
  18. retrievalloom/engine.py +1035 -0
  19. retrievalloom/errors.py +229 -0
  20. retrievalloom/evidence/__init__.py +1 -0
  21. retrievalloom/evidence/consolidation.py +355 -0
  22. retrievalloom/evidence/dedup.py +113 -0
  23. retrievalloom/evidence/fusion.py +243 -0
  24. retrievalloom/evidence/normalizers.py +144 -0
  25. retrievalloom/evidence/rerank.py +201 -0
  26. retrievalloom/evidence/validation.py +102 -0
  27. retrievalloom/graph/__init__.py +1 -0
  28. retrievalloom/graph/ladybug_store.py +482 -0
  29. retrievalloom/graph/normalizer.py +152 -0
  30. retrievalloom/graph/retriever.py +125 -0
  31. retrievalloom/index/__init__.py +1 -0
  32. retrievalloom/index/embedding.py +98 -0
  33. retrievalloom/index/fuzzy.py +318 -0
  34. retrievalloom/index/keyword.py +159 -0
  35. retrievalloom/index/memory.py +223 -0
  36. retrievalloom/index/sparse.py +336 -0
  37. retrievalloom/index/text.py +285 -0
  38. retrievalloom/index/vector.py +565 -0
  39. retrievalloom/infra/__init__.py +14 -0
  40. retrievalloom/infra/access.py +41 -0
  41. retrievalloom/infra/cache.py +108 -0
  42. retrievalloom/infra/metrics.py +27 -0
  43. retrievalloom/infra/recorders.py +31 -0
  44. retrievalloom/integrations/__init__.py +1 -0
  45. retrievalloom/integrations/agxai.py +160 -0
  46. retrievalloom/lifecycle/__init__.py +17 -0
  47. retrievalloom/lifecycle/canonical.py +198 -0
  48. retrievalloom/lifecycle/coordinator.py +621 -0
  49. retrievalloom/lifecycle/derivation.py +475 -0
  50. retrievalloom/lifecycle/indexer.py +925 -0
  51. retrievalloom/lifecycle/ingestion.py +870 -0
  52. retrievalloom/lifecycle/purge.py +378 -0
  53. retrievalloom/lifecycle/units.py +194 -0
  54. retrievalloom/py.typed +0 -0
  55. retrievalloom/retrieval/__init__.py +20 -0
  56. retrievalloom/retrieval/candidates.py +282 -0
  57. retrievalloom/retrieval/pipeline.py +841 -0
  58. retrievalloom/retrieval/planning.py +278 -0
  59. retrievalloom/retrieval/snapshot.py +330 -0
  60. retrievalloom/runtime/__init__.py +1 -0
  61. retrievalloom/runtime/offloader.py +301 -0
  62. retrievalloom/storage/__init__.py +27 -0
  63. retrievalloom/storage/base.py +740 -0
  64. retrievalloom/storage/migrations/env.py +31 -0
  65. retrievalloom/storage/migrations/script.py.mako +26 -0
  66. retrievalloom/storage/migrations/versions/0001_initial.py +335 -0
  67. retrievalloom/storage/migrations/versions/0002_retention_indexes.py +35 -0
  68. retrievalloom/storage/migrations/versions/0003_purge_log.py +36 -0
  69. retrievalloom/storage/migrations/versions/0004_incremental_indexing.py +48 -0
  70. retrievalloom/storage/migrations/versions/0005_snapshot_token.py +25 -0
  71. retrievalloom/storage/migrations/versions/0006_objects_kind_order.py +32 -0
  72. retrievalloom/storage/postgres.py +318 -0
  73. retrievalloom/storage/refresh_state.py +83 -0
  74. retrievalloom/storage/schema.py +398 -0
  75. retrievalloom/storage/sql.py +2574 -0
  76. retrievalloom/storage/sqlite.py +182 -0
  77. retrievalloom/structure/__init__.py +1 -0
  78. retrievalloom/structure/treesearch_engine.py +438 -0
  79. retrievalloom/testing/__init__.py +51 -0
  80. retrievalloom/testing/_base.py +98 -0
  81. retrievalloom/testing/evidence.py +294 -0
  82. retrievalloom/testing/indexes.py +477 -0
  83. retrievalloom/testing/infra.py +39 -0
  84. retrievalloom/testing/models.py +160 -0
  85. retrievalloom/testing/storage.py +831 -0
  86. retrievalloom-0.1.0.dist-info/METADATA +378 -0
  87. retrievalloom-0.1.0.dist-info/RECORD +89 -0
  88. retrievalloom-0.1.0.dist-info/WHEEL +4 -0
  89. retrievalloom-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,197 @@
1
+ """RetrievalLoom: async-first, provider-agnostic, multi-channel retrieval and evidence engine.
2
+
3
+ Extension contracts (ABCs) live in :mod:`retrievalloom.contracts`; default adapters live in
4
+ :mod:`retrievalloom.index`, :mod:`retrievalloom.evidence`, :mod:`retrievalloom.infra`,
5
+ :mod:`retrievalloom.graph` (``graph`` extra) and :mod:`retrievalloom.structure`
6
+ (``structure`` extra).
7
+ """
8
+
9
+ from importlib.metadata import PackageNotFoundError, version
10
+ from typing import TYPE_CHECKING, Any
11
+
12
+ from retrievalloom.config import (
13
+ ChannelsConfig,
14
+ Components,
15
+ ConsolidationConfig,
16
+ EngineConfig,
17
+ EngineResources,
18
+ FailurePolicy,
19
+ FusionConfig,
20
+ FuzzyConfig,
21
+ GraphChannelConfig,
22
+ IndexChannelConfig,
23
+ IndexWait,
24
+ IngestionConfig,
25
+ IngestionLimits,
26
+ LifecycleConfig,
27
+ MethodConfig,
28
+ NLIConfig,
29
+ RerankingConfig,
30
+ RetrievalConfig,
31
+ StructureChannelConfig,
32
+ TenantPolicy,
33
+ )
34
+ from retrievalloom.domain.filters import And, Eq, Exists, FilterExpression, In, Not, Or, Range
35
+ from retrievalloom.domain.inputs import (
36
+ ChunkInput,
37
+ ChunksRepresentation,
38
+ DocumentInput,
39
+ GraphEntityInput,
40
+ GraphRelationshipInput,
41
+ GraphRepresentation,
42
+ Scope,
43
+ SourceDescriptor,
44
+ StructureNodeInput,
45
+ StructureRepresentation,
46
+ TextRepresentation,
47
+ TextSpan,
48
+ )
49
+ from retrievalloom.domain.records import Channel, MissingIndexPolicy
50
+ from retrievalloom.domain.reports import (
51
+ DeleteReport,
52
+ HealthReport,
53
+ HealthStatus,
54
+ IngestReport,
55
+ LifecycleTaskRef,
56
+ LifecycleTaskResult,
57
+ PurgeReport,
58
+ ScopeStatus,
59
+ )
60
+ from retrievalloom.domain.retrieval import (
61
+ Candidate,
62
+ Evidence,
63
+ ResultStatus,
64
+ RetrievalExecutionConfig,
65
+ RetrievalResult,
66
+ RetrievalWarning,
67
+ Severity,
68
+ WarningCode,
69
+ )
70
+ from retrievalloom.engine import EngineState, RetrievalEngine, create_retrieval_engine
71
+ from retrievalloom.errors import (
72
+ AccessDeniedError,
73
+ BackendError,
74
+ BackendTimeoutError,
75
+ BackendUnavailableError,
76
+ ComponentContractError,
77
+ ConfigurationError,
78
+ CoverageError,
79
+ EmbeddingValidationError,
80
+ IdentityError,
81
+ IngestionError,
82
+ LifecycleError,
83
+ MigrationError,
84
+ OperationCancelledError,
85
+ OperationTimeoutError,
86
+ ProvenanceError,
87
+ ResourceAdmissionError,
88
+ RetrievalError,
89
+ RetrievalLoomError,
90
+ TenantIsolationError,
91
+ ValidationError,
92
+ )
93
+ from retrievalloom.runtime.offloader import OffloaderConfig
94
+
95
+ if TYPE_CHECKING:
96
+ from retrievalloom.storage.postgres import PostgresMetadataStore
97
+
98
+ try:
99
+ __version__ = version("retrievalloom")
100
+ except PackageNotFoundError: # pragma: no cover - only when running from a raw checkout
101
+ __version__ = "0.0.0"
102
+
103
+ __all__ = [
104
+ "AccessDeniedError",
105
+ "And",
106
+ "BackendError",
107
+ "BackendTimeoutError",
108
+ "BackendUnavailableError",
109
+ "Candidate",
110
+ "Channel",
111
+ "ChannelsConfig",
112
+ "ChunkInput",
113
+ "ChunksRepresentation",
114
+ "ComponentContractError",
115
+ "Components",
116
+ "ConfigurationError",
117
+ "ConsolidationConfig",
118
+ "CoverageError",
119
+ "DeleteReport",
120
+ "DocumentInput",
121
+ "EmbeddingValidationError",
122
+ "EngineConfig",
123
+ "EngineResources",
124
+ "EngineState",
125
+ "Eq",
126
+ "Evidence",
127
+ "Exists",
128
+ "FailurePolicy",
129
+ "FilterExpression",
130
+ "FusionConfig",
131
+ "FuzzyConfig",
132
+ "GraphChannelConfig",
133
+ "GraphEntityInput",
134
+ "GraphRelationshipInput",
135
+ "GraphRepresentation",
136
+ "HealthReport",
137
+ "HealthStatus",
138
+ "IdentityError",
139
+ "In",
140
+ "IndexChannelConfig",
141
+ "IndexWait",
142
+ "IngestReport",
143
+ "IngestionConfig",
144
+ "IngestionError",
145
+ "IngestionLimits",
146
+ "LifecycleConfig",
147
+ "LifecycleError",
148
+ "LifecycleTaskRef",
149
+ "LifecycleTaskResult",
150
+ "MethodConfig",
151
+ "MigrationError",
152
+ "MissingIndexPolicy",
153
+ "NLIConfig",
154
+ "Not",
155
+ "OffloaderConfig",
156
+ "OperationCancelledError",
157
+ "OperationTimeoutError",
158
+ "Or",
159
+ "PostgresMetadataStore",
160
+ "ProvenanceError",
161
+ "PurgeReport",
162
+ "Range",
163
+ "RerankingConfig",
164
+ "ResourceAdmissionError",
165
+ "ResultStatus",
166
+ "RetrievalConfig",
167
+ "RetrievalEngine",
168
+ "RetrievalError",
169
+ "RetrievalExecutionConfig",
170
+ "RetrievalLoomError",
171
+ "RetrievalResult",
172
+ "RetrievalWarning",
173
+ "Scope",
174
+ "ScopeStatus",
175
+ "Severity",
176
+ "SourceDescriptor",
177
+ "StructureChannelConfig",
178
+ "StructureNodeInput",
179
+ "StructureRepresentation",
180
+ "TenantIsolationError",
181
+ "TenantPolicy",
182
+ "TextRepresentation",
183
+ "TextSpan",
184
+ "ValidationError",
185
+ "WarningCode",
186
+ "__version__",
187
+ "create_retrieval_engine",
188
+ ]
189
+
190
+
191
+ def __getattr__(name: str) -> Any:
192
+ # Optional backends are imported on first use so ``import retrievalloom`` stays light.
193
+ if name == "PostgresMetadataStore":
194
+ from retrievalloom.storage.postgres import PostgresMetadataStore
195
+
196
+ return PostgresMetadataStore
197
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,68 @@
1
+ """Library logging helpers built on structlog.
2
+
3
+ RetrievalLoom never configures structlog or the standard library logging system.
4
+
5
+ - When the host application has configured structlog, events go through that
6
+ configuration (``structlog.stdlib.get_logger``) under ``retrievalloom.*`` names.
7
+ - Otherwise structlog's default would print to stdout, so events are instead rendered
8
+ as ``key=value`` text into the ``retrievalloom.*`` standard library loggers. A
9
+ ``NullHandler`` on ``retrievalloom`` keeps them silent until the application adds
10
+ handlers or levels for them.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import logging
16
+ from typing import Any
17
+
18
+ import structlog
19
+
20
+ ROOT_LOGGER_NAME = "retrievalloom"
21
+
22
+ logging.getLogger(ROOT_LOGGER_NAME).addHandler(logging.NullHandler())
23
+
24
+ _FALLBACK_PROCESSORS: list[Any] = [
25
+ structlog.stdlib.filter_by_level,
26
+ structlog.processors.format_exc_info,
27
+ structlog.processors.KeyValueRenderer(key_order=["event"], sort_keys=True),
28
+ ]
29
+
30
+
31
+ class _LibraryLogger:
32
+ """Bound logger that picks structlog's or the stdlib route at every call."""
33
+
34
+ __slots__ = ("_context", "_name")
35
+
36
+ def __init__(self, name: str, context: dict[str, Any]) -> None:
37
+ self._name = name
38
+ self._context = context
39
+
40
+ def bind(self, **values: Any) -> _LibraryLogger:
41
+ return _LibraryLogger(self._name, {**self._context, **values})
42
+
43
+ def _target(self) -> Any:
44
+ if structlog.is_configured():
45
+ return structlog.stdlib.get_logger(self._name, **self._context)
46
+ return structlog.wrap_logger(
47
+ logging.getLogger(self._name),
48
+ processors=_FALLBACK_PROCESSORS,
49
+ wrapper_class=structlog.stdlib.BoundLogger,
50
+ **self._context,
51
+ )
52
+
53
+ def __getattr__(self, method: str) -> Any:
54
+ return getattr(self._target(), method)
55
+
56
+
57
+ def get_logger(name: str | None = None, **initial_values: Any) -> Any:
58
+ """Return a structlog-style logger bound to a ``retrievalloom`` logger name.
59
+
60
+ Args:
61
+ name: Dotted suffix under ``retrievalloom``; ``None`` returns the root logger.
62
+ **initial_values: Key-value context bound to the returned logger.
63
+
64
+ Returns:
65
+ A logger with ``bind`` and the usual level methods (``info``, ``warning``, ...).
66
+ """
67
+ full_name = ROOT_LOGGER_NAME if not name else f"{ROOT_LOGGER_NAME}.{name}"
68
+ return _LibraryLogger(full_name, dict(initial_values))
@@ -0,0 +1,392 @@
1
+ """Engine configuration (frozen Pydantic models) and component/resource wiring."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from enum import StrEnum
7
+ from pathlib import Path
8
+ from typing import TYPE_CHECKING, Any, Literal
9
+
10
+ from pydantic import Field, model_validator
11
+
12
+ from retrievalloom.domain.base import FrozenModel, Identifier
13
+ from retrievalloom.domain.hashing import HASH_PROFILE, SUPPORTED_HASH_PROFILES
14
+ from retrievalloom.domain.records import Channel, IndexMethod, MissingIndexPolicy
15
+ from retrievalloom.runtime.offloader import OffloaderConfig
16
+
17
+ if TYPE_CHECKING:
18
+ from concurrent.futures import Executor
19
+
20
+ from resourcera import ResourceManager
21
+
22
+ from retrievalloom.contracts.evidence import (
23
+ DeduplicatorABC,
24
+ EvidenceConsolidatorABC,
25
+ FusionABC,
26
+ )
27
+ from retrievalloom.contracts.indexes import (
28
+ FuzzyIndexABC,
29
+ GraphRetrieverABC,
30
+ GraphStoreABC,
31
+ KeywordIndexABC,
32
+ SparseIndexABC,
33
+ StructureEngineABC,
34
+ VectorStoreABC,
35
+ )
36
+ from retrievalloom.contracts.infra import AccessPolicyABC, CacheABC, ProvenanceRecorderABC
37
+ from retrievalloom.contracts.models import (
38
+ NLIABC,
39
+ EmbeddingModelABC,
40
+ GraphExtractorABC,
41
+ GraphNormalizerABC,
42
+ KeywordDictionaryABC,
43
+ RerankerABC,
44
+ TokenizerABC,
45
+ )
46
+ from retrievalloom.storage.base import DocumentMetadataStoreABC
47
+
48
+ FusionMethod = Literal["rrf", "weighted_rrf", "weighted_score", "combsum", "combmnz"]
49
+ NormalizerName = Literal["minmax", "zscore", "rank", "sigmoid"]
50
+ OnError = Literal["degrade", "fail"]
51
+
52
+
53
+ class FailurePolicy(StrEnum):
54
+ """How one invalid document affects its batch."""
55
+
56
+ PER_DOCUMENT = "per_document"
57
+ FAIL_BATCH = "fail_batch"
58
+
59
+
60
+ class IndexWait(StrEnum):
61
+ """What ``ingest`` waits for after the allocation commit.
62
+
63
+ ``none`` returns immediately and lets background lifecycle index the documents;
64
+ ``all`` runs one lifecycle round for the scope and reports the coverage reached.
65
+ """
66
+
67
+ NONE = "none"
68
+ ALL = "all"
69
+
70
+
71
+ class IngestionLimits(FrozenModel):
72
+ """Input size limits."""
73
+
74
+ max_documents_per_call: int = Field(default=1000, ge=1)
75
+ max_representations: int = Field(default=16, ge=1)
76
+ max_chunks: int = Field(default=10_000, ge=1)
77
+ max_text_chars: int = Field(default=10_000_000, ge=1)
78
+ max_tree_depth: int = Field(default=64, ge=1)
79
+ max_tree_nodes: int = Field(default=50_000, ge=1)
80
+ max_graph_items: int = Field(default=50_000, ge=1)
81
+
82
+
83
+ class IngestionConfig(FrozenModel):
84
+ """Ingestion behaviour.
85
+
86
+ Attributes:
87
+ hash_profile: Content hash profile.
88
+ transient_metadata_keys: Metadata keys that never create a new version (a change
89
+ only bumps ``metadata_revision``).
90
+ redelivery_window_seconds: Window of the advisory redelivery check (0 disables).
91
+ concurrency: Documents processed concurrently per ``ingest`` call.
92
+ failure_policy: ``per_document`` reports failures per document; ``fail_batch``
93
+ validates the whole batch first and rejects it entirely on any invalid input.
94
+ index_wait: Whether ``ingest`` waits for indexing.
95
+ index_wait_timeout: Seconds ``ingest`` waits when ``index_wait=all``.
96
+ limits: Input size limits.
97
+ """
98
+
99
+ hash_profile: str = HASH_PROFILE
100
+ transient_metadata_keys: tuple[str, ...] = ()
101
+ redelivery_window_seconds: float = Field(default=300.0, ge=0)
102
+ concurrency: int = Field(default=8, ge=1, le=256)
103
+ failure_policy: FailurePolicy = FailurePolicy.PER_DOCUMENT
104
+ index_wait: IndexWait = IndexWait.ALL
105
+ index_wait_timeout: float = Field(default=120.0, gt=0)
106
+ limits: IngestionLimits = Field(default_factory=IngestionLimits)
107
+
108
+ @model_validator(mode="after")
109
+ def _check(self) -> IngestionConfig:
110
+ if self.hash_profile not in SUPPORTED_HASH_PROFILES:
111
+ raise ValueError(f"unsupported hash profile {self.hash_profile!r}")
112
+ return self
113
+
114
+
115
+ class MethodConfig(FrozenModel):
116
+ """One retrieval method."""
117
+
118
+ enabled: bool = True
119
+ candidate_k: int = Field(default=50, ge=1, le=10_000)
120
+ weight: float = Field(default=1.0, ge=0.0, allow_inf_nan=False)
121
+ on_error: OnError = "degrade"
122
+
123
+
124
+ class FuzzyConfig(MethodConfig):
125
+ """Fuzzy method settings (RapidFuzz)."""
126
+
127
+ score_cutoff: float = Field(default=85.0, ge=0.0, le=100.0)
128
+ max_ngram: int = Field(default=3, ge=1, le=6)
129
+ corpus_vocabulary: bool = True
130
+ min_token_length: int = Field(default=3, ge=1)
131
+
132
+
133
+ class IndexChannelConfig(FrozenModel):
134
+ """Index channel: keyword, fuzzy, sparse and dense methods."""
135
+
136
+ enabled: bool = True
137
+ keyword: MethodConfig = Field(default_factory=MethodConfig)
138
+ fuzzy: FuzzyConfig = Field(default_factory=FuzzyConfig)
139
+ sparse: MethodConfig = Field(default_factory=MethodConfig)
140
+ dense: MethodConfig = Field(default_factory=MethodConfig)
141
+ weight: float = Field(default=1.0, ge=0.0, allow_inf_nan=False)
142
+
143
+ def method(self, name: str) -> MethodConfig:
144
+ """Return the configuration of an index method."""
145
+ return getattr(self, IndexMethod(name).value)
146
+
147
+
148
+ class GraphChannelConfig(MethodConfig):
149
+ """Graph channel (disabled by default)."""
150
+
151
+ enabled: bool = False
152
+ depth: int = Field(default=1, ge=1, le=4)
153
+
154
+
155
+ class StructureChannelConfig(MethodConfig):
156
+ """Structure channel (disabled by default)."""
157
+
158
+ enabled: bool = False
159
+
160
+
161
+ class ChannelsConfig(FrozenModel):
162
+ """All channels.
163
+
164
+ At least one channel must be enabled; an enabled index channel needs at least one
165
+ enabled method. An engine with nothing to index or query is rejected.
166
+ """
167
+
168
+ index: IndexChannelConfig = Field(default_factory=IndexChannelConfig)
169
+ graph: GraphChannelConfig = Field(default_factory=GraphChannelConfig)
170
+ structure: StructureChannelConfig = Field(default_factory=StructureChannelConfig)
171
+
172
+ @model_validator(mode="after")
173
+ def _check(self) -> ChannelsConfig:
174
+ index = self.index.enabled and any(self.index.method(name).enabled for name in IndexMethod)
175
+ if not (index or self.graph.enabled or self.structure.enabled):
176
+ raise ValueError("at least one channel (and index method) must be enabled")
177
+ return self
178
+
179
+
180
+ class FusionConfig(FrozenModel):
181
+ """Fusion settings of one level."""
182
+
183
+ method: FusionMethod = "rrf"
184
+ k: int = Field(default=60, ge=1)
185
+ normalizer: NormalizerName = "minmax"
186
+ sigmoid_center: float = 0.0
187
+ sigmoid_temperature: float = Field(default=1.0, gt=0.0)
188
+
189
+
190
+ class RerankingConfig(FrozenModel):
191
+ """Reranking of the fused (RRF by default) candidate order.
192
+
193
+ When no reranker component is supplied, the engine wires the built-in FlashRank
194
+ cross-encoder, loading ``model_name`` from ``model_dir`` (default ``<data_dir>/models``,
195
+ or ``~/.cache/retrievalloom/models`` for in-memory engines). Models are never downloaded
196
+ unless ``allow_download`` is set; when the model is missing the default reranker is
197
+ dropped at startup (one ``reranker_unavailable`` log) and results keep the fused order.
198
+ A supplied reranker always replaces the default.
199
+ """
200
+
201
+ enabled: bool = True
202
+ rerank_k: int = Field(default=50, ge=1, le=1000)
203
+ on_error: OnError = "degrade"
204
+ model_dir: Path | None = None
205
+ model_name: str = "ms-marco-TinyBERT-L-2-v2"
206
+ allow_download: bool = False
207
+
208
+
209
+ class NLIConfig(FrozenModel):
210
+ """NLI judgements (disabled by default)."""
211
+
212
+ enabled: bool = False
213
+ nli_k: int = Field(default=10, ge=1, le=200)
214
+ on_error: OnError = "degrade"
215
+
216
+
217
+ class ConsolidationConfig(FrozenModel):
218
+ """Evidence consolidation (always executed)."""
219
+
220
+ window_multiplier: int = Field(default=3, ge=1, le=20)
221
+ group_structural: bool = True
222
+ merge_adjacent: bool = True
223
+ span_iou_threshold: float = Field(default=0.5, gt=0.0, le=1.0)
224
+ semantic_relations: bool = False
225
+ semantic_threshold: float = Field(default=0.9, gt=0.0, le=1.0)
226
+ contradiction_overflow: int | None = Field(default=None, ge=0)
227
+
228
+
229
+ class TenantPolicy(FrozenModel):
230
+ """Per-tenant limits and capabilities."""
231
+
232
+ max_concurrent_requests: int = Field(default=16, ge=1)
233
+ max_concurrent_cpu_jobs: int = Field(default=4, ge=1)
234
+ max_top_k: int = Field(default=100, ge=1)
235
+ max_candidate_k: int = Field(default=500, ge=1)
236
+ max_ingest_batch: int = Field(default=1000, ge=1)
237
+ allowed_channels: tuple[Channel, ...] = (Channel.INDEX, Channel.GRAPH, Channel.STRUCTURE)
238
+
239
+
240
+ class RetrievalConfig(FrozenModel):
241
+ """Retrieval behaviour."""
242
+
243
+ channels: ChannelsConfig = Field(default_factory=ChannelsConfig)
244
+ method_fusion: FusionConfig = Field(default_factory=FusionConfig)
245
+ channel_fusion: FusionConfig = Field(default_factory=FusionConfig)
246
+ reranking: RerankingConfig = Field(default_factory=RerankingConfig)
247
+ nli: NLIConfig = Field(default_factory=NLIConfig)
248
+ consolidation: ConsolidationConfig = Field(default_factory=ConsolidationConfig)
249
+ missing_index_policy: MissingIndexPolicy = MissingIndexPolicy.SKIP
250
+ default_top_k: int = Field(default=10, ge=1)
251
+ max_query_chars: int = Field(default=4096, ge=1)
252
+ max_filter_nodes: int = Field(default=64, ge=1)
253
+ request_timeout: float = Field(default=30.0, gt=0)
254
+ method_timeout: float = Field(default=10.0, gt=0)
255
+ allow_partial_on_deadline: bool = True
256
+ allow_stale_versions: bool = False
257
+ fail_on_recording_error: bool = False
258
+ embedding_cache_size: int = Field(default=1024, ge=0)
259
+ embedding_cache_ttl: float = Field(default=600.0, gt=0)
260
+ default_tenant_policy: TenantPolicy = Field(default_factory=TenantPolicy)
261
+ tenant_policies: dict[Identifier, TenantPolicy] = Field(default_factory=dict)
262
+
263
+ def tenant_policy(self, tenant_id: str) -> TenantPolicy:
264
+ """Return the policy of ``tenant_id`` (or the default policy)."""
265
+ return self.tenant_policies.get(tenant_id, self.default_tenant_policy)
266
+
267
+
268
+ class LifecycleConfig(FrozenModel):
269
+ """Background lifecycle (refresh-engine) settings.
270
+
271
+ Attributes:
272
+ background: Run a periodic lifecycle scheduler while the engine is open.
273
+ poll_interval: Seconds between scheduled lifecycle rounds.
274
+ batch_size: Documents per backend write batch.
275
+ max_concurrency: Index units processed concurrently.
276
+ max_attempts: Attempts per unit round (refresh-engine retry), and attempts per
277
+ document before its coverage is marked ``failed``.
278
+ retry_failed_after: Seconds before coverage that failed with a retryable error
279
+ (for example a backend outage) is queued again by the background lifecycle;
280
+ the delay doubles with every further failure up to
281
+ ``retry_failed_max_delay``. Non-retryable failures stay ``failed`` until a
282
+ backfill. ``None`` disables automatic retries.
283
+ retry_failed_max_delay: Upper bound, in seconds, of the retry delay of failed
284
+ coverage.
285
+ unit_timeout: Seconds one unit round may take.
286
+ instance_ttl: Seconds an engine-instance registration lives without heartbeat.
287
+ lease_ttl: Seconds a durable reader lease lives.
288
+ generation_retention: Seconds a retired generation is kept before dropping.
289
+ auto_rebuild_ephemeral: Rebuild ephemeral backends from metadata at
290
+ startup. Without ``background`` (or with ``wait_for_rebuild``) the
291
+ rebuild completes before the engine opens; otherwise it runs in the
292
+ background and retrievals report ``index_missing`` until it finishes.
293
+ wait_for_rebuild: With ``background``, still finish the startup rebuild
294
+ before the engine opens (bounded by ``rebuild_timeout``).
295
+ rebuild_timeout: Seconds a startup rebuild may block ``initialize`` when
296
+ ``background`` is on; on timeout the rebuild continues in the
297
+ background.
298
+ purge_after: Seconds a deleted document stays tombstoned before it is
299
+ physically erased by the background lifecycle; ``None`` disables.
300
+ max_versions: Version-history entries kept per document; older
301
+ unreferenced versions and their representations are pruned.
302
+ ``None`` disables.
303
+ event_retention: Seconds committed or failed ingestion events are kept;
304
+ ``None`` disables.
305
+ purge_log_retention: Seconds purge-log entries are kept. Engines evict purged
306
+ objects from their in-memory indexes by reading this log; an engine that
307
+ missed pruned entries rebuilds its in-memory indexes instead. Keep it well
308
+ above ``instance_ttl``.
309
+ migration: Metadata schema policy (``auto`` or ``verify``).
310
+ shutdown_timeout: Seconds ``close`` waits for in-flight work.
311
+ """
312
+
313
+ background: bool = True
314
+ poll_interval: float = Field(default=2.0, gt=0)
315
+ batch_size: int = Field(default=128, ge=1, le=10_000)
316
+ max_concurrency: int = Field(default=4, ge=1, le=64)
317
+ max_attempts: int = Field(default=3, ge=1, le=20)
318
+ retry_failed_after: float | None = Field(default=30.0, gt=0)
319
+ retry_failed_max_delay: float = Field(default=900.0, gt=0)
320
+ unit_timeout: float = Field(default=600.0, gt=0)
321
+ instance_ttl: float = Field(default=120.0, gt=0)
322
+ lease_ttl: float = Field(default=120.0, gt=0)
323
+ generation_retention: float = Field(default=0.0, ge=0)
324
+ auto_rebuild_ephemeral: bool = True
325
+ wait_for_rebuild: bool = False
326
+ rebuild_timeout: float = Field(default=300.0, gt=0)
327
+ purge_after: float | None = Field(default=7 * 86_400.0, ge=0)
328
+ max_versions: int | None = Field(default=10, ge=1)
329
+ event_retention: float | None = Field(default=90 * 86_400.0, ge=0)
330
+ purge_log_retention: float = Field(default=86_400.0, gt=0)
331
+ migration: Literal["auto", "verify"] = "auto"
332
+ shutdown_timeout: float = Field(default=30.0, gt=0)
333
+
334
+
335
+ class EngineConfig(FrozenModel):
336
+ """Complete engine configuration."""
337
+
338
+ ingestion: IngestionConfig = Field(default_factory=IngestionConfig)
339
+ retrieval: RetrievalConfig = Field(default_factory=RetrievalConfig)
340
+ lifecycle: LifecycleConfig = Field(default_factory=LifecycleConfig)
341
+ offloader: OffloaderConfig = Field(default_factory=OffloaderConfig)
342
+ log_content: bool = False
343
+
344
+
345
+ @dataclass(frozen=True, slots=True, kw_only=True)
346
+ class Components:
347
+ """Pluggable components; ``None`` selects the default (or disables the capability).
348
+
349
+ Application-supplied components stay application-owned unless the engine created them.
350
+ """
351
+
352
+ metadata_store: DocumentMetadataStoreABC | None = None
353
+ embedding: EmbeddingModelABC | None = None
354
+ reranker: RerankerABC | None = None
355
+ nli: NLIABC | None = None
356
+ tokenizer: TokenizerABC | None = None
357
+ keyword_dictionary: KeywordDictionaryABC | None = None
358
+ keyword_index: KeywordIndexABC | None = None
359
+ fuzzy_index: FuzzyIndexABC | None = None
360
+ sparse_index: SparseIndexABC | None = None
361
+ vector_store: VectorStoreABC | None = None
362
+ graph_extractor: GraphExtractorABC | None = None
363
+ graph_normalizer: GraphNormalizerABC | None = None
364
+ graph_store: GraphStoreABC | None = None
365
+ graph_retriever: GraphRetrieverABC | None = None
366
+ structure_engine: StructureEngineABC | None = None
367
+ method_fusion: FusionABC | None = None
368
+ channel_fusion: FusionABC | None = None
369
+ deduplicator: DeduplicatorABC | None = None
370
+ consolidator: EvidenceConsolidatorABC | None = None
371
+ cache: CacheABC | None = None
372
+ access_policy: AccessPolicyABC | None = None
373
+ provenance_recorder: ProvenanceRecorderABC | None = None
374
+
375
+
376
+ @dataclass(frozen=True, slots=True, kw_only=True)
377
+ class EngineResources:
378
+ """Application-owned infrastructure the engine uses but never closes.
379
+
380
+ Attributes:
381
+ resource_manager: resourcera manager for admission control.
382
+ xai_runtime: agxai ``XAIRuntime``; enables the agxai provenance recorder.
383
+ executor: Executor for offloaded blocking work.
384
+ data_dir: Directory for durable component files (graph, structure, metadata).
385
+ metrics: refresh-engine ``MetricsSink``.
386
+ """
387
+
388
+ resource_manager: ResourceManager | None = None
389
+ xai_runtime: Any = None
390
+ executor: Executor | None = None
391
+ data_dir: Path | None = None
392
+ metrics: Any = None
@@ -0,0 +1 @@
1
+ """Abstract component contracts (ABCs). Concrete adapters must subclass them."""