retrievalloom 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retrievalloom/__init__.py +197 -0
- retrievalloom/_logging.py +68 -0
- retrievalloom/config.py +392 -0
- retrievalloom/contracts/__init__.py +1 -0
- retrievalloom/contracts/component.py +119 -0
- retrievalloom/contracts/evidence.py +94 -0
- retrievalloom/contracts/indexes.py +304 -0
- retrievalloom/contracts/infra.py +76 -0
- retrievalloom/contracts/models.py +188 -0
- retrievalloom/domain/__init__.py +1 -0
- retrievalloom/domain/base.py +92 -0
- retrievalloom/domain/filters.py +177 -0
- retrievalloom/domain/hashing.py +81 -0
- retrievalloom/domain/inputs.py +269 -0
- retrievalloom/domain/records.py +389 -0
- retrievalloom/domain/reports.py +139 -0
- retrievalloom/domain/retrieval.py +355 -0
- retrievalloom/engine.py +1035 -0
- retrievalloom/errors.py +229 -0
- retrievalloom/evidence/__init__.py +1 -0
- retrievalloom/evidence/consolidation.py +355 -0
- retrievalloom/evidence/dedup.py +113 -0
- retrievalloom/evidence/fusion.py +243 -0
- retrievalloom/evidence/normalizers.py +144 -0
- retrievalloom/evidence/rerank.py +201 -0
- retrievalloom/evidence/validation.py +102 -0
- retrievalloom/graph/__init__.py +1 -0
- retrievalloom/graph/ladybug_store.py +482 -0
- retrievalloom/graph/normalizer.py +152 -0
- retrievalloom/graph/retriever.py +125 -0
- retrievalloom/index/__init__.py +1 -0
- retrievalloom/index/embedding.py +98 -0
- retrievalloom/index/fuzzy.py +318 -0
- retrievalloom/index/keyword.py +159 -0
- retrievalloom/index/memory.py +223 -0
- retrievalloom/index/sparse.py +336 -0
- retrievalloom/index/text.py +285 -0
- retrievalloom/index/vector.py +565 -0
- retrievalloom/infra/__init__.py +14 -0
- retrievalloom/infra/access.py +41 -0
- retrievalloom/infra/cache.py +108 -0
- retrievalloom/infra/metrics.py +27 -0
- retrievalloom/infra/recorders.py +31 -0
- retrievalloom/integrations/__init__.py +1 -0
- retrievalloom/integrations/agxai.py +160 -0
- retrievalloom/lifecycle/__init__.py +17 -0
- retrievalloom/lifecycle/canonical.py +198 -0
- retrievalloom/lifecycle/coordinator.py +621 -0
- retrievalloom/lifecycle/derivation.py +475 -0
- retrievalloom/lifecycle/indexer.py +925 -0
- retrievalloom/lifecycle/ingestion.py +870 -0
- retrievalloom/lifecycle/purge.py +378 -0
- retrievalloom/lifecycle/units.py +194 -0
- retrievalloom/py.typed +0 -0
- retrievalloom/retrieval/__init__.py +20 -0
- retrievalloom/retrieval/candidates.py +282 -0
- retrievalloom/retrieval/pipeline.py +841 -0
- retrievalloom/retrieval/planning.py +278 -0
- retrievalloom/retrieval/snapshot.py +330 -0
- retrievalloom/runtime/__init__.py +1 -0
- retrievalloom/runtime/offloader.py +301 -0
- retrievalloom/storage/__init__.py +27 -0
- retrievalloom/storage/base.py +740 -0
- retrievalloom/storage/migrations/env.py +31 -0
- retrievalloom/storage/migrations/script.py.mako +26 -0
- retrievalloom/storage/migrations/versions/0001_initial.py +335 -0
- retrievalloom/storage/migrations/versions/0002_retention_indexes.py +35 -0
- retrievalloom/storage/migrations/versions/0003_purge_log.py +36 -0
- retrievalloom/storage/migrations/versions/0004_incremental_indexing.py +48 -0
- retrievalloom/storage/migrations/versions/0005_snapshot_token.py +25 -0
- retrievalloom/storage/migrations/versions/0006_objects_kind_order.py +32 -0
- retrievalloom/storage/postgres.py +318 -0
- retrievalloom/storage/refresh_state.py +83 -0
- retrievalloom/storage/schema.py +398 -0
- retrievalloom/storage/sql.py +2574 -0
- retrievalloom/storage/sqlite.py +182 -0
- retrievalloom/structure/__init__.py +1 -0
- retrievalloom/structure/treesearch_engine.py +438 -0
- retrievalloom/testing/__init__.py +51 -0
- retrievalloom/testing/_base.py +98 -0
- retrievalloom/testing/evidence.py +294 -0
- retrievalloom/testing/indexes.py +477 -0
- retrievalloom/testing/infra.py +39 -0
- retrievalloom/testing/models.py +160 -0
- retrievalloom/testing/storage.py +831 -0
- retrievalloom-0.1.0.dist-info/METADATA +378 -0
- retrievalloom-0.1.0.dist-info/RECORD +89 -0
- retrievalloom-0.1.0.dist-info/WHEEL +4 -0
- retrievalloom-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""RetrievalLoom: async-first, provider-agnostic, multi-channel retrieval and evidence engine.
|
|
2
|
+
|
|
3
|
+
Extension contracts (ABCs) live in :mod:`retrievalloom.contracts`; default adapters live in
|
|
4
|
+
:mod:`retrievalloom.index`, :mod:`retrievalloom.evidence`, :mod:`retrievalloom.infra`,
|
|
5
|
+
:mod:`retrievalloom.graph` (``graph`` extra) and :mod:`retrievalloom.structure`
|
|
6
|
+
(``structure`` extra).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
10
|
+
from typing import TYPE_CHECKING, Any
|
|
11
|
+
|
|
12
|
+
from retrievalloom.config import (
|
|
13
|
+
ChannelsConfig,
|
|
14
|
+
Components,
|
|
15
|
+
ConsolidationConfig,
|
|
16
|
+
EngineConfig,
|
|
17
|
+
EngineResources,
|
|
18
|
+
FailurePolicy,
|
|
19
|
+
FusionConfig,
|
|
20
|
+
FuzzyConfig,
|
|
21
|
+
GraphChannelConfig,
|
|
22
|
+
IndexChannelConfig,
|
|
23
|
+
IndexWait,
|
|
24
|
+
IngestionConfig,
|
|
25
|
+
IngestionLimits,
|
|
26
|
+
LifecycleConfig,
|
|
27
|
+
MethodConfig,
|
|
28
|
+
NLIConfig,
|
|
29
|
+
RerankingConfig,
|
|
30
|
+
RetrievalConfig,
|
|
31
|
+
StructureChannelConfig,
|
|
32
|
+
TenantPolicy,
|
|
33
|
+
)
|
|
34
|
+
from retrievalloom.domain.filters import And, Eq, Exists, FilterExpression, In, Not, Or, Range
|
|
35
|
+
from retrievalloom.domain.inputs import (
|
|
36
|
+
ChunkInput,
|
|
37
|
+
ChunksRepresentation,
|
|
38
|
+
DocumentInput,
|
|
39
|
+
GraphEntityInput,
|
|
40
|
+
GraphRelationshipInput,
|
|
41
|
+
GraphRepresentation,
|
|
42
|
+
Scope,
|
|
43
|
+
SourceDescriptor,
|
|
44
|
+
StructureNodeInput,
|
|
45
|
+
StructureRepresentation,
|
|
46
|
+
TextRepresentation,
|
|
47
|
+
TextSpan,
|
|
48
|
+
)
|
|
49
|
+
from retrievalloom.domain.records import Channel, MissingIndexPolicy
|
|
50
|
+
from retrievalloom.domain.reports import (
|
|
51
|
+
DeleteReport,
|
|
52
|
+
HealthReport,
|
|
53
|
+
HealthStatus,
|
|
54
|
+
IngestReport,
|
|
55
|
+
LifecycleTaskRef,
|
|
56
|
+
LifecycleTaskResult,
|
|
57
|
+
PurgeReport,
|
|
58
|
+
ScopeStatus,
|
|
59
|
+
)
|
|
60
|
+
from retrievalloom.domain.retrieval import (
|
|
61
|
+
Candidate,
|
|
62
|
+
Evidence,
|
|
63
|
+
ResultStatus,
|
|
64
|
+
RetrievalExecutionConfig,
|
|
65
|
+
RetrievalResult,
|
|
66
|
+
RetrievalWarning,
|
|
67
|
+
Severity,
|
|
68
|
+
WarningCode,
|
|
69
|
+
)
|
|
70
|
+
from retrievalloom.engine import EngineState, RetrievalEngine, create_retrieval_engine
|
|
71
|
+
from retrievalloom.errors import (
|
|
72
|
+
AccessDeniedError,
|
|
73
|
+
BackendError,
|
|
74
|
+
BackendTimeoutError,
|
|
75
|
+
BackendUnavailableError,
|
|
76
|
+
ComponentContractError,
|
|
77
|
+
ConfigurationError,
|
|
78
|
+
CoverageError,
|
|
79
|
+
EmbeddingValidationError,
|
|
80
|
+
IdentityError,
|
|
81
|
+
IngestionError,
|
|
82
|
+
LifecycleError,
|
|
83
|
+
MigrationError,
|
|
84
|
+
OperationCancelledError,
|
|
85
|
+
OperationTimeoutError,
|
|
86
|
+
ProvenanceError,
|
|
87
|
+
ResourceAdmissionError,
|
|
88
|
+
RetrievalError,
|
|
89
|
+
RetrievalLoomError,
|
|
90
|
+
TenantIsolationError,
|
|
91
|
+
ValidationError,
|
|
92
|
+
)
|
|
93
|
+
from retrievalloom.runtime.offloader import OffloaderConfig
|
|
94
|
+
|
|
95
|
+
if TYPE_CHECKING:
|
|
96
|
+
from retrievalloom.storage.postgres import PostgresMetadataStore
|
|
97
|
+
|
|
98
|
+
try:
|
|
99
|
+
__version__ = version("retrievalloom")
|
|
100
|
+
except PackageNotFoundError: # pragma: no cover - only when running from a raw checkout
|
|
101
|
+
__version__ = "0.0.0"
|
|
102
|
+
|
|
103
|
+
__all__ = [
|
|
104
|
+
"AccessDeniedError",
|
|
105
|
+
"And",
|
|
106
|
+
"BackendError",
|
|
107
|
+
"BackendTimeoutError",
|
|
108
|
+
"BackendUnavailableError",
|
|
109
|
+
"Candidate",
|
|
110
|
+
"Channel",
|
|
111
|
+
"ChannelsConfig",
|
|
112
|
+
"ChunkInput",
|
|
113
|
+
"ChunksRepresentation",
|
|
114
|
+
"ComponentContractError",
|
|
115
|
+
"Components",
|
|
116
|
+
"ConfigurationError",
|
|
117
|
+
"ConsolidationConfig",
|
|
118
|
+
"CoverageError",
|
|
119
|
+
"DeleteReport",
|
|
120
|
+
"DocumentInput",
|
|
121
|
+
"EmbeddingValidationError",
|
|
122
|
+
"EngineConfig",
|
|
123
|
+
"EngineResources",
|
|
124
|
+
"EngineState",
|
|
125
|
+
"Eq",
|
|
126
|
+
"Evidence",
|
|
127
|
+
"Exists",
|
|
128
|
+
"FailurePolicy",
|
|
129
|
+
"FilterExpression",
|
|
130
|
+
"FusionConfig",
|
|
131
|
+
"FuzzyConfig",
|
|
132
|
+
"GraphChannelConfig",
|
|
133
|
+
"GraphEntityInput",
|
|
134
|
+
"GraphRelationshipInput",
|
|
135
|
+
"GraphRepresentation",
|
|
136
|
+
"HealthReport",
|
|
137
|
+
"HealthStatus",
|
|
138
|
+
"IdentityError",
|
|
139
|
+
"In",
|
|
140
|
+
"IndexChannelConfig",
|
|
141
|
+
"IndexWait",
|
|
142
|
+
"IngestReport",
|
|
143
|
+
"IngestionConfig",
|
|
144
|
+
"IngestionError",
|
|
145
|
+
"IngestionLimits",
|
|
146
|
+
"LifecycleConfig",
|
|
147
|
+
"LifecycleError",
|
|
148
|
+
"LifecycleTaskRef",
|
|
149
|
+
"LifecycleTaskResult",
|
|
150
|
+
"MethodConfig",
|
|
151
|
+
"MigrationError",
|
|
152
|
+
"MissingIndexPolicy",
|
|
153
|
+
"NLIConfig",
|
|
154
|
+
"Not",
|
|
155
|
+
"OffloaderConfig",
|
|
156
|
+
"OperationCancelledError",
|
|
157
|
+
"OperationTimeoutError",
|
|
158
|
+
"Or",
|
|
159
|
+
"PostgresMetadataStore",
|
|
160
|
+
"ProvenanceError",
|
|
161
|
+
"PurgeReport",
|
|
162
|
+
"Range",
|
|
163
|
+
"RerankingConfig",
|
|
164
|
+
"ResourceAdmissionError",
|
|
165
|
+
"ResultStatus",
|
|
166
|
+
"RetrievalConfig",
|
|
167
|
+
"RetrievalEngine",
|
|
168
|
+
"RetrievalError",
|
|
169
|
+
"RetrievalExecutionConfig",
|
|
170
|
+
"RetrievalLoomError",
|
|
171
|
+
"RetrievalResult",
|
|
172
|
+
"RetrievalWarning",
|
|
173
|
+
"Scope",
|
|
174
|
+
"ScopeStatus",
|
|
175
|
+
"Severity",
|
|
176
|
+
"SourceDescriptor",
|
|
177
|
+
"StructureChannelConfig",
|
|
178
|
+
"StructureNodeInput",
|
|
179
|
+
"StructureRepresentation",
|
|
180
|
+
"TenantIsolationError",
|
|
181
|
+
"TenantPolicy",
|
|
182
|
+
"TextRepresentation",
|
|
183
|
+
"TextSpan",
|
|
184
|
+
"ValidationError",
|
|
185
|
+
"WarningCode",
|
|
186
|
+
"__version__",
|
|
187
|
+
"create_retrieval_engine",
|
|
188
|
+
]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def __getattr__(name: str) -> Any:
|
|
192
|
+
# Optional backends are imported on first use so ``import retrievalloom`` stays light.
|
|
193
|
+
if name == "PostgresMetadataStore":
|
|
194
|
+
from retrievalloom.storage.postgres import PostgresMetadataStore
|
|
195
|
+
|
|
196
|
+
return PostgresMetadataStore
|
|
197
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Library logging helpers built on structlog.
|
|
2
|
+
|
|
3
|
+
RetrievalLoom never configures structlog or the standard library logging system.
|
|
4
|
+
|
|
5
|
+
- When the host application has configured structlog, events go through that
|
|
6
|
+
configuration (``structlog.stdlib.get_logger``) under ``retrievalloom.*`` names.
|
|
7
|
+
- Otherwise structlog's default would print to stdout, so events are instead rendered
|
|
8
|
+
as ``key=value`` text into the ``retrievalloom.*`` standard library loggers. A
|
|
9
|
+
``NullHandler`` on ``retrievalloom`` keeps them silent until the application adds
|
|
10
|
+
handlers or levels for them.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import logging
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
import structlog
|
|
19
|
+
|
|
20
|
+
ROOT_LOGGER_NAME = "retrievalloom"
|
|
21
|
+
|
|
22
|
+
logging.getLogger(ROOT_LOGGER_NAME).addHandler(logging.NullHandler())
|
|
23
|
+
|
|
24
|
+
_FALLBACK_PROCESSORS: list[Any] = [
|
|
25
|
+
structlog.stdlib.filter_by_level,
|
|
26
|
+
structlog.processors.format_exc_info,
|
|
27
|
+
structlog.processors.KeyValueRenderer(key_order=["event"], sort_keys=True),
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _LibraryLogger:
|
|
32
|
+
"""Bound logger that picks structlog's or the stdlib route at every call."""
|
|
33
|
+
|
|
34
|
+
__slots__ = ("_context", "_name")
|
|
35
|
+
|
|
36
|
+
def __init__(self, name: str, context: dict[str, Any]) -> None:
|
|
37
|
+
self._name = name
|
|
38
|
+
self._context = context
|
|
39
|
+
|
|
40
|
+
def bind(self, **values: Any) -> _LibraryLogger:
|
|
41
|
+
return _LibraryLogger(self._name, {**self._context, **values})
|
|
42
|
+
|
|
43
|
+
def _target(self) -> Any:
|
|
44
|
+
if structlog.is_configured():
|
|
45
|
+
return structlog.stdlib.get_logger(self._name, **self._context)
|
|
46
|
+
return structlog.wrap_logger(
|
|
47
|
+
logging.getLogger(self._name),
|
|
48
|
+
processors=_FALLBACK_PROCESSORS,
|
|
49
|
+
wrapper_class=structlog.stdlib.BoundLogger,
|
|
50
|
+
**self._context,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
def __getattr__(self, method: str) -> Any:
|
|
54
|
+
return getattr(self._target(), method)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def get_logger(name: str | None = None, **initial_values: Any) -> Any:
|
|
58
|
+
"""Return a structlog-style logger bound to a ``retrievalloom`` logger name.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
name: Dotted suffix under ``retrievalloom``; ``None`` returns the root logger.
|
|
62
|
+
**initial_values: Key-value context bound to the returned logger.
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
A logger with ``bind`` and the usual level methods (``info``, ``warning``, ...).
|
|
66
|
+
"""
|
|
67
|
+
full_name = ROOT_LOGGER_NAME if not name else f"{ROOT_LOGGER_NAME}.{name}"
|
|
68
|
+
return _LibraryLogger(full_name, dict(initial_values))
|
retrievalloom/config.py
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
"""Engine configuration (frozen Pydantic models) and component/resource wiring."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from enum import StrEnum
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
9
|
+
|
|
10
|
+
from pydantic import Field, model_validator
|
|
11
|
+
|
|
12
|
+
from retrievalloom.domain.base import FrozenModel, Identifier
|
|
13
|
+
from retrievalloom.domain.hashing import HASH_PROFILE, SUPPORTED_HASH_PROFILES
|
|
14
|
+
from retrievalloom.domain.records import Channel, IndexMethod, MissingIndexPolicy
|
|
15
|
+
from retrievalloom.runtime.offloader import OffloaderConfig
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from concurrent.futures import Executor
|
|
19
|
+
|
|
20
|
+
from resourcera import ResourceManager
|
|
21
|
+
|
|
22
|
+
from retrievalloom.contracts.evidence import (
|
|
23
|
+
DeduplicatorABC,
|
|
24
|
+
EvidenceConsolidatorABC,
|
|
25
|
+
FusionABC,
|
|
26
|
+
)
|
|
27
|
+
from retrievalloom.contracts.indexes import (
|
|
28
|
+
FuzzyIndexABC,
|
|
29
|
+
GraphRetrieverABC,
|
|
30
|
+
GraphStoreABC,
|
|
31
|
+
KeywordIndexABC,
|
|
32
|
+
SparseIndexABC,
|
|
33
|
+
StructureEngineABC,
|
|
34
|
+
VectorStoreABC,
|
|
35
|
+
)
|
|
36
|
+
from retrievalloom.contracts.infra import AccessPolicyABC, CacheABC, ProvenanceRecorderABC
|
|
37
|
+
from retrievalloom.contracts.models import (
|
|
38
|
+
NLIABC,
|
|
39
|
+
EmbeddingModelABC,
|
|
40
|
+
GraphExtractorABC,
|
|
41
|
+
GraphNormalizerABC,
|
|
42
|
+
KeywordDictionaryABC,
|
|
43
|
+
RerankerABC,
|
|
44
|
+
TokenizerABC,
|
|
45
|
+
)
|
|
46
|
+
from retrievalloom.storage.base import DocumentMetadataStoreABC
|
|
47
|
+
|
|
48
|
+
FusionMethod = Literal["rrf", "weighted_rrf", "weighted_score", "combsum", "combmnz"]
|
|
49
|
+
NormalizerName = Literal["minmax", "zscore", "rank", "sigmoid"]
|
|
50
|
+
OnError = Literal["degrade", "fail"]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class FailurePolicy(StrEnum):
|
|
54
|
+
"""How one invalid document affects its batch."""
|
|
55
|
+
|
|
56
|
+
PER_DOCUMENT = "per_document"
|
|
57
|
+
FAIL_BATCH = "fail_batch"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class IndexWait(StrEnum):
|
|
61
|
+
"""What ``ingest`` waits for after the allocation commit.
|
|
62
|
+
|
|
63
|
+
``none`` returns immediately and lets background lifecycle index the documents;
|
|
64
|
+
``all`` runs one lifecycle round for the scope and reports the coverage reached.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
NONE = "none"
|
|
68
|
+
ALL = "all"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class IngestionLimits(FrozenModel):
|
|
72
|
+
"""Input size limits."""
|
|
73
|
+
|
|
74
|
+
max_documents_per_call: int = Field(default=1000, ge=1)
|
|
75
|
+
max_representations: int = Field(default=16, ge=1)
|
|
76
|
+
max_chunks: int = Field(default=10_000, ge=1)
|
|
77
|
+
max_text_chars: int = Field(default=10_000_000, ge=1)
|
|
78
|
+
max_tree_depth: int = Field(default=64, ge=1)
|
|
79
|
+
max_tree_nodes: int = Field(default=50_000, ge=1)
|
|
80
|
+
max_graph_items: int = Field(default=50_000, ge=1)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class IngestionConfig(FrozenModel):
|
|
84
|
+
"""Ingestion behaviour.
|
|
85
|
+
|
|
86
|
+
Attributes:
|
|
87
|
+
hash_profile: Content hash profile.
|
|
88
|
+
transient_metadata_keys: Metadata keys that never create a new version (a change
|
|
89
|
+
only bumps ``metadata_revision``).
|
|
90
|
+
redelivery_window_seconds: Window of the advisory redelivery check (0 disables).
|
|
91
|
+
concurrency: Documents processed concurrently per ``ingest`` call.
|
|
92
|
+
failure_policy: ``per_document`` reports failures per document; ``fail_batch``
|
|
93
|
+
validates the whole batch first and rejects it entirely on any invalid input.
|
|
94
|
+
index_wait: Whether ``ingest`` waits for indexing.
|
|
95
|
+
index_wait_timeout: Seconds ``ingest`` waits when ``index_wait=all``.
|
|
96
|
+
limits: Input size limits.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
hash_profile: str = HASH_PROFILE
|
|
100
|
+
transient_metadata_keys: tuple[str, ...] = ()
|
|
101
|
+
redelivery_window_seconds: float = Field(default=300.0, ge=0)
|
|
102
|
+
concurrency: int = Field(default=8, ge=1, le=256)
|
|
103
|
+
failure_policy: FailurePolicy = FailurePolicy.PER_DOCUMENT
|
|
104
|
+
index_wait: IndexWait = IndexWait.ALL
|
|
105
|
+
index_wait_timeout: float = Field(default=120.0, gt=0)
|
|
106
|
+
limits: IngestionLimits = Field(default_factory=IngestionLimits)
|
|
107
|
+
|
|
108
|
+
@model_validator(mode="after")
|
|
109
|
+
def _check(self) -> IngestionConfig:
|
|
110
|
+
if self.hash_profile not in SUPPORTED_HASH_PROFILES:
|
|
111
|
+
raise ValueError(f"unsupported hash profile {self.hash_profile!r}")
|
|
112
|
+
return self
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class MethodConfig(FrozenModel):
|
|
116
|
+
"""One retrieval method."""
|
|
117
|
+
|
|
118
|
+
enabled: bool = True
|
|
119
|
+
candidate_k: int = Field(default=50, ge=1, le=10_000)
|
|
120
|
+
weight: float = Field(default=1.0, ge=0.0, allow_inf_nan=False)
|
|
121
|
+
on_error: OnError = "degrade"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class FuzzyConfig(MethodConfig):
|
|
125
|
+
"""Fuzzy method settings (RapidFuzz)."""
|
|
126
|
+
|
|
127
|
+
score_cutoff: float = Field(default=85.0, ge=0.0, le=100.0)
|
|
128
|
+
max_ngram: int = Field(default=3, ge=1, le=6)
|
|
129
|
+
corpus_vocabulary: bool = True
|
|
130
|
+
min_token_length: int = Field(default=3, ge=1)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class IndexChannelConfig(FrozenModel):
|
|
134
|
+
"""Index channel: keyword, fuzzy, sparse and dense methods."""
|
|
135
|
+
|
|
136
|
+
enabled: bool = True
|
|
137
|
+
keyword: MethodConfig = Field(default_factory=MethodConfig)
|
|
138
|
+
fuzzy: FuzzyConfig = Field(default_factory=FuzzyConfig)
|
|
139
|
+
sparse: MethodConfig = Field(default_factory=MethodConfig)
|
|
140
|
+
dense: MethodConfig = Field(default_factory=MethodConfig)
|
|
141
|
+
weight: float = Field(default=1.0, ge=0.0, allow_inf_nan=False)
|
|
142
|
+
|
|
143
|
+
def method(self, name: str) -> MethodConfig:
|
|
144
|
+
"""Return the configuration of an index method."""
|
|
145
|
+
return getattr(self, IndexMethod(name).value)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class GraphChannelConfig(MethodConfig):
|
|
149
|
+
"""Graph channel (disabled by default)."""
|
|
150
|
+
|
|
151
|
+
enabled: bool = False
|
|
152
|
+
depth: int = Field(default=1, ge=1, le=4)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class StructureChannelConfig(MethodConfig):
|
|
156
|
+
"""Structure channel (disabled by default)."""
|
|
157
|
+
|
|
158
|
+
enabled: bool = False
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class ChannelsConfig(FrozenModel):
|
|
162
|
+
"""All channels.
|
|
163
|
+
|
|
164
|
+
At least one channel must be enabled; an enabled index channel needs at least one
|
|
165
|
+
enabled method. An engine with nothing to index or query is rejected.
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
index: IndexChannelConfig = Field(default_factory=IndexChannelConfig)
|
|
169
|
+
graph: GraphChannelConfig = Field(default_factory=GraphChannelConfig)
|
|
170
|
+
structure: StructureChannelConfig = Field(default_factory=StructureChannelConfig)
|
|
171
|
+
|
|
172
|
+
@model_validator(mode="after")
|
|
173
|
+
def _check(self) -> ChannelsConfig:
|
|
174
|
+
index = self.index.enabled and any(self.index.method(name).enabled for name in IndexMethod)
|
|
175
|
+
if not (index or self.graph.enabled or self.structure.enabled):
|
|
176
|
+
raise ValueError("at least one channel (and index method) must be enabled")
|
|
177
|
+
return self
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class FusionConfig(FrozenModel):
|
|
181
|
+
"""Fusion settings of one level."""
|
|
182
|
+
|
|
183
|
+
method: FusionMethod = "rrf"
|
|
184
|
+
k: int = Field(default=60, ge=1)
|
|
185
|
+
normalizer: NormalizerName = "minmax"
|
|
186
|
+
sigmoid_center: float = 0.0
|
|
187
|
+
sigmoid_temperature: float = Field(default=1.0, gt=0.0)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class RerankingConfig(FrozenModel):
|
|
191
|
+
"""Reranking of the fused (RRF by default) candidate order.
|
|
192
|
+
|
|
193
|
+
When no reranker component is supplied, the engine wires the built-in FlashRank
|
|
194
|
+
cross-encoder, loading ``model_name`` from ``model_dir`` (default ``<data_dir>/models``,
|
|
195
|
+
or ``~/.cache/retrievalloom/models`` for in-memory engines). Models are never downloaded
|
|
196
|
+
unless ``allow_download`` is set; when the model is missing the default reranker is
|
|
197
|
+
dropped at startup (one ``reranker_unavailable`` log) and results keep the fused order.
|
|
198
|
+
A supplied reranker always replaces the default.
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
enabled: bool = True
|
|
202
|
+
rerank_k: int = Field(default=50, ge=1, le=1000)
|
|
203
|
+
on_error: OnError = "degrade"
|
|
204
|
+
model_dir: Path | None = None
|
|
205
|
+
model_name: str = "ms-marco-TinyBERT-L-2-v2"
|
|
206
|
+
allow_download: bool = False
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class NLIConfig(FrozenModel):
|
|
210
|
+
"""NLI judgements (disabled by default)."""
|
|
211
|
+
|
|
212
|
+
enabled: bool = False
|
|
213
|
+
nli_k: int = Field(default=10, ge=1, le=200)
|
|
214
|
+
on_error: OnError = "degrade"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class ConsolidationConfig(FrozenModel):
|
|
218
|
+
"""Evidence consolidation (always executed)."""
|
|
219
|
+
|
|
220
|
+
window_multiplier: int = Field(default=3, ge=1, le=20)
|
|
221
|
+
group_structural: bool = True
|
|
222
|
+
merge_adjacent: bool = True
|
|
223
|
+
span_iou_threshold: float = Field(default=0.5, gt=0.0, le=1.0)
|
|
224
|
+
semantic_relations: bool = False
|
|
225
|
+
semantic_threshold: float = Field(default=0.9, gt=0.0, le=1.0)
|
|
226
|
+
contradiction_overflow: int | None = Field(default=None, ge=0)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class TenantPolicy(FrozenModel):
|
|
230
|
+
"""Per-tenant limits and capabilities."""
|
|
231
|
+
|
|
232
|
+
max_concurrent_requests: int = Field(default=16, ge=1)
|
|
233
|
+
max_concurrent_cpu_jobs: int = Field(default=4, ge=1)
|
|
234
|
+
max_top_k: int = Field(default=100, ge=1)
|
|
235
|
+
max_candidate_k: int = Field(default=500, ge=1)
|
|
236
|
+
max_ingest_batch: int = Field(default=1000, ge=1)
|
|
237
|
+
allowed_channels: tuple[Channel, ...] = (Channel.INDEX, Channel.GRAPH, Channel.STRUCTURE)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class RetrievalConfig(FrozenModel):
|
|
241
|
+
"""Retrieval behaviour."""
|
|
242
|
+
|
|
243
|
+
channels: ChannelsConfig = Field(default_factory=ChannelsConfig)
|
|
244
|
+
method_fusion: FusionConfig = Field(default_factory=FusionConfig)
|
|
245
|
+
channel_fusion: FusionConfig = Field(default_factory=FusionConfig)
|
|
246
|
+
reranking: RerankingConfig = Field(default_factory=RerankingConfig)
|
|
247
|
+
nli: NLIConfig = Field(default_factory=NLIConfig)
|
|
248
|
+
consolidation: ConsolidationConfig = Field(default_factory=ConsolidationConfig)
|
|
249
|
+
missing_index_policy: MissingIndexPolicy = MissingIndexPolicy.SKIP
|
|
250
|
+
default_top_k: int = Field(default=10, ge=1)
|
|
251
|
+
max_query_chars: int = Field(default=4096, ge=1)
|
|
252
|
+
max_filter_nodes: int = Field(default=64, ge=1)
|
|
253
|
+
request_timeout: float = Field(default=30.0, gt=0)
|
|
254
|
+
method_timeout: float = Field(default=10.0, gt=0)
|
|
255
|
+
allow_partial_on_deadline: bool = True
|
|
256
|
+
allow_stale_versions: bool = False
|
|
257
|
+
fail_on_recording_error: bool = False
|
|
258
|
+
embedding_cache_size: int = Field(default=1024, ge=0)
|
|
259
|
+
embedding_cache_ttl: float = Field(default=600.0, gt=0)
|
|
260
|
+
default_tenant_policy: TenantPolicy = Field(default_factory=TenantPolicy)
|
|
261
|
+
tenant_policies: dict[Identifier, TenantPolicy] = Field(default_factory=dict)
|
|
262
|
+
|
|
263
|
+
def tenant_policy(self, tenant_id: str) -> TenantPolicy:
|
|
264
|
+
"""Return the policy of ``tenant_id`` (or the default policy)."""
|
|
265
|
+
return self.tenant_policies.get(tenant_id, self.default_tenant_policy)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class LifecycleConfig(FrozenModel):
|
|
269
|
+
"""Background lifecycle (refresh-engine) settings.
|
|
270
|
+
|
|
271
|
+
Attributes:
|
|
272
|
+
background: Run a periodic lifecycle scheduler while the engine is open.
|
|
273
|
+
poll_interval: Seconds between scheduled lifecycle rounds.
|
|
274
|
+
batch_size: Documents per backend write batch.
|
|
275
|
+
max_concurrency: Index units processed concurrently.
|
|
276
|
+
max_attempts: Attempts per unit round (refresh-engine retry), and attempts per
|
|
277
|
+
document before its coverage is marked ``failed``.
|
|
278
|
+
retry_failed_after: Seconds before coverage that failed with a retryable error
|
|
279
|
+
(for example a backend outage) is queued again by the background lifecycle;
|
|
280
|
+
the delay doubles with every further failure up to
|
|
281
|
+
``retry_failed_max_delay``. Non-retryable failures stay ``failed`` until a
|
|
282
|
+
backfill. ``None`` disables automatic retries.
|
|
283
|
+
retry_failed_max_delay: Upper bound, in seconds, of the retry delay of failed
|
|
284
|
+
coverage.
|
|
285
|
+
unit_timeout: Seconds one unit round may take.
|
|
286
|
+
instance_ttl: Seconds an engine-instance registration lives without heartbeat.
|
|
287
|
+
lease_ttl: Seconds a durable reader lease lives.
|
|
288
|
+
generation_retention: Seconds a retired generation is kept before dropping.
|
|
289
|
+
auto_rebuild_ephemeral: Rebuild ephemeral backends from metadata at
|
|
290
|
+
startup. Without ``background`` (or with ``wait_for_rebuild``) the
|
|
291
|
+
rebuild completes before the engine opens; otherwise it runs in the
|
|
292
|
+
background and retrievals report ``index_missing`` until it finishes.
|
|
293
|
+
wait_for_rebuild: With ``background``, still finish the startup rebuild
|
|
294
|
+
before the engine opens (bounded by ``rebuild_timeout``).
|
|
295
|
+
rebuild_timeout: Seconds a startup rebuild may block ``initialize`` when
|
|
296
|
+
``background`` is on; on timeout the rebuild continues in the
|
|
297
|
+
background.
|
|
298
|
+
purge_after: Seconds a deleted document stays tombstoned before it is
|
|
299
|
+
physically erased by the background lifecycle; ``None`` disables.
|
|
300
|
+
max_versions: Version-history entries kept per document; older
|
|
301
|
+
unreferenced versions and their representations are pruned.
|
|
302
|
+
``None`` disables.
|
|
303
|
+
event_retention: Seconds committed or failed ingestion events are kept;
|
|
304
|
+
``None`` disables.
|
|
305
|
+
purge_log_retention: Seconds purge-log entries are kept. Engines evict purged
|
|
306
|
+
objects from their in-memory indexes by reading this log; an engine that
|
|
307
|
+
missed pruned entries rebuilds its in-memory indexes instead. Keep it well
|
|
308
|
+
above ``instance_ttl``.
|
|
309
|
+
migration: Metadata schema policy (``auto`` or ``verify``).
|
|
310
|
+
shutdown_timeout: Seconds ``close`` waits for in-flight work.
|
|
311
|
+
"""
|
|
312
|
+
|
|
313
|
+
background: bool = True
|
|
314
|
+
poll_interval: float = Field(default=2.0, gt=0)
|
|
315
|
+
batch_size: int = Field(default=128, ge=1, le=10_000)
|
|
316
|
+
max_concurrency: int = Field(default=4, ge=1, le=64)
|
|
317
|
+
max_attempts: int = Field(default=3, ge=1, le=20)
|
|
318
|
+
retry_failed_after: float | None = Field(default=30.0, gt=0)
|
|
319
|
+
retry_failed_max_delay: float = Field(default=900.0, gt=0)
|
|
320
|
+
unit_timeout: float = Field(default=600.0, gt=0)
|
|
321
|
+
instance_ttl: float = Field(default=120.0, gt=0)
|
|
322
|
+
lease_ttl: float = Field(default=120.0, gt=0)
|
|
323
|
+
generation_retention: float = Field(default=0.0, ge=0)
|
|
324
|
+
auto_rebuild_ephemeral: bool = True
|
|
325
|
+
wait_for_rebuild: bool = False
|
|
326
|
+
rebuild_timeout: float = Field(default=300.0, gt=0)
|
|
327
|
+
purge_after: float | None = Field(default=7 * 86_400.0, ge=0)
|
|
328
|
+
max_versions: int | None = Field(default=10, ge=1)
|
|
329
|
+
event_retention: float | None = Field(default=90 * 86_400.0, ge=0)
|
|
330
|
+
purge_log_retention: float = Field(default=86_400.0, gt=0)
|
|
331
|
+
migration: Literal["auto", "verify"] = "auto"
|
|
332
|
+
shutdown_timeout: float = Field(default=30.0, gt=0)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class EngineConfig(FrozenModel):
|
|
336
|
+
"""Complete engine configuration."""
|
|
337
|
+
|
|
338
|
+
ingestion: IngestionConfig = Field(default_factory=IngestionConfig)
|
|
339
|
+
retrieval: RetrievalConfig = Field(default_factory=RetrievalConfig)
|
|
340
|
+
lifecycle: LifecycleConfig = Field(default_factory=LifecycleConfig)
|
|
341
|
+
offloader: OffloaderConfig = Field(default_factory=OffloaderConfig)
|
|
342
|
+
log_content: bool = False
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
346
|
+
class Components:
|
|
347
|
+
"""Pluggable components; ``None`` selects the default (or disables the capability).
|
|
348
|
+
|
|
349
|
+
Application-supplied components stay application-owned unless the engine created them.
|
|
350
|
+
"""
|
|
351
|
+
|
|
352
|
+
metadata_store: DocumentMetadataStoreABC | None = None
|
|
353
|
+
embedding: EmbeddingModelABC | None = None
|
|
354
|
+
reranker: RerankerABC | None = None
|
|
355
|
+
nli: NLIABC | None = None
|
|
356
|
+
tokenizer: TokenizerABC | None = None
|
|
357
|
+
keyword_dictionary: KeywordDictionaryABC | None = None
|
|
358
|
+
keyword_index: KeywordIndexABC | None = None
|
|
359
|
+
fuzzy_index: FuzzyIndexABC | None = None
|
|
360
|
+
sparse_index: SparseIndexABC | None = None
|
|
361
|
+
vector_store: VectorStoreABC | None = None
|
|
362
|
+
graph_extractor: GraphExtractorABC | None = None
|
|
363
|
+
graph_normalizer: GraphNormalizerABC | None = None
|
|
364
|
+
graph_store: GraphStoreABC | None = None
|
|
365
|
+
graph_retriever: GraphRetrieverABC | None = None
|
|
366
|
+
structure_engine: StructureEngineABC | None = None
|
|
367
|
+
method_fusion: FusionABC | None = None
|
|
368
|
+
channel_fusion: FusionABC | None = None
|
|
369
|
+
deduplicator: DeduplicatorABC | None = None
|
|
370
|
+
consolidator: EvidenceConsolidatorABC | None = None
|
|
371
|
+
cache: CacheABC | None = None
|
|
372
|
+
access_policy: AccessPolicyABC | None = None
|
|
373
|
+
provenance_recorder: ProvenanceRecorderABC | None = None
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
377
|
+
class EngineResources:
|
|
378
|
+
"""Application-owned infrastructure the engine uses but never closes.
|
|
379
|
+
|
|
380
|
+
Attributes:
|
|
381
|
+
resource_manager: resourcera manager for admission control.
|
|
382
|
+
xai_runtime: agxai ``XAIRuntime``; enables the agxai provenance recorder.
|
|
383
|
+
executor: Executor for offloaded blocking work.
|
|
384
|
+
data_dir: Directory for durable component files (graph, structure, metadata).
|
|
385
|
+
metrics: refresh-engine ``MetricsSink``.
|
|
386
|
+
"""
|
|
387
|
+
|
|
388
|
+
resource_manager: ResourceManager | None = None
|
|
389
|
+
xai_runtime: Any = None
|
|
390
|
+
executor: Executor | None = None
|
|
391
|
+
data_dir: Path | None = None
|
|
392
|
+
metrics: Any = None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Abstract component contracts (ABCs). Concrete adapters must subclass them."""
|