modular-rag 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modular_rag-0.1.0.dist-info/METADATA +1073 -0
- modular_rag-0.1.0.dist-info/RECORD +58 -0
- modular_rag-0.1.0.dist-info/WHEEL +5 -0
- modular_rag-0.1.0.dist-info/licenses/LICENSE +21 -0
- modular_rag-0.1.0.dist-info/top_level.txt +1 -0
- ragkit/__init__.py +53 -0
- ragkit/callbacks/__init__.py +20 -0
- ragkit/callbacks/builtin.py +259 -0
- ragkit/callbacks/events.py +58 -0
- ragkit/callbacks/manager.py +89 -0
- ragkit/callbacks/progress.py +57 -0
- ragkit/catalog/__init__.py +16 -0
- ragkit/catalog/base.py +74 -0
- ragkit/catalog/chunk_store.py +109 -0
- ragkit/catalog/filtering.py +61 -0
- ragkit/catalog/models.py +236 -0
- ragkit/catalog/scalar.py +122 -0
- ragkit/catalog/store_catalog.py +66 -0
- ragkit/catalog/version.py +50 -0
- ragkit/core/__init__.py +11 -0
- ragkit/core/config.py +60 -0
- ragkit/core/embeddings.py +161 -0
- ragkit/core/filtering.py +44 -0
- ragkit/core/hashing.py +60 -0
- ragkit/core/schema.py +67 -0
- ragkit/middleware/__init__.py +20 -0
- ragkit/middleware/base.py +117 -0
- ragkit/middleware/builtin.py +256 -0
- ragkit/pipeline.py +391 -0
- ragkit/py.typed +0 -0
- ragkit/ragkit_version.py +1 -0
- ragkit/readers/__init__.py +61 -0
- ragkit/readers/_common.py +38 -0
- ragkit/readers/base.py +35 -0
- ragkit/readers/markdown_parser.py +221 -0
- ragkit/readers/office.py +291 -0
- ragkit/readers/pdf.py +162 -0
- ragkit/readers/text.py +62 -0
- ragkit/rerank/__init__.py +95 -0
- ragkit/rerank/base.py +63 -0
- ragkit/retrieval/__init__.py +7 -0
- ragkit/retrieval/fusion.py +61 -0
- ragkit/retrieval/hybrid.py +81 -0
- ragkit/retrieval/retrievers.py +133 -0
- ragkit/splitters/__init__.py +41 -0
- ragkit/splitters/base.py +83 -0
- ragkit/splitters/heading.py +76 -0
- ragkit/splitters/qa.py +26 -0
- ragkit/splitters/recursive.py +49 -0
- ragkit/splitters/table.py +54 -0
- ragkit/store/__init__.py +23 -0
- ragkit/store/base.py +40 -0
- ragkit/store/memory.py +55 -0
- ragkit/store/qdrant_store.py +203 -0
- ragkit/store/weaviate_store.py +249 -0
- ragkit/utils/__init__.py +5 -0
- ragkit/utils/registry.py +46 -0
- ragkit/utils/timeout.py +49 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
modular_rag-0.1.0.dist-info/licenses/LICENSE,sha256=6_7ITuxFnNMK5_IcGJ0E61TKRdHkm93FF5qD0AECRcA,1076
|
|
2
|
+
ragkit/__init__.py,sha256=LYmQ51ig9QmMhsdyJBO3gtvzTo4_QOErAHeTr8khODM,3044
|
|
3
|
+
ragkit/pipeline.py,sha256=xoNm0VLJwyPQwo07KS9E2FokCeujvCycP7WTXprpSiU,19085
|
|
4
|
+
ragkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
ragkit/ragkit_version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
|
|
6
|
+
ragkit/callbacks/__init__.py,sha256=Jsv4USqg0XDUalu2HtrrKzZ-V0FKtIAXZNtM-aXJdGg,951
|
|
7
|
+
ragkit/callbacks/builtin.py,sha256=e0buOgXg7zqSCK6Sxcc7C4z-k7ZpSsW82saT21FuiOg,9611
|
|
8
|
+
ragkit/callbacks/events.py,sha256=wGcAmDWLnnzSnSeyEkBnloM1SmXMbGdkIhfqlqUNimU,1961
|
|
9
|
+
ragkit/callbacks/manager.py,sha256=3IUjSFU4UGtXeVs2evdixtwklYTqKhag2p-53lkMYZc,3727
|
|
10
|
+
ragkit/callbacks/progress.py,sha256=56zBqeCglY75mXtOIZMIPpUi7ts-6sHqsFjtcM_woFw,2326
|
|
11
|
+
ragkit/catalog/__init__.py,sha256=gaDS09FemdovDr0zlPAsGnvCUUZDO3g6BjucJklUq18,792
|
|
12
|
+
ragkit/catalog/base.py,sha256=Jf9l_CIauWptdFPDBP8QqriZwiHPS3uQbkCAB6f-7sA,2595
|
|
13
|
+
ragkit/catalog/chunk_store.py,sha256=ak-mhSiq30YAVgahdruMqavW8Diw6EGFGw69YniAF1Y,4841
|
|
14
|
+
ragkit/catalog/filtering.py,sha256=r2Lue7s03VLv21wDN3yAHecCUSBr4NeoBhERvsAWiP8,2128
|
|
15
|
+
ragkit/catalog/models.py,sha256=2RJl2Yz7EjCNtMZBabv4INV1BAmuBN-JgRdA8rPDvXs,10034
|
|
16
|
+
ragkit/catalog/scalar.py,sha256=V6LWgE5N9mYtqBvdgGouBD-PlQ9ZTYZfGWdbnxE5gag,4293
|
|
17
|
+
ragkit/catalog/store_catalog.py,sha256=9WRZuDLSpAX-eiq77rw244Zw4pTbXaAI8K8JDX8wKTk,2469
|
|
18
|
+
ragkit/catalog/version.py,sha256=8kNSNzQd0KaMZcNBx9TUZYtdtmmfYdRfLJDHrFKukuY,1527
|
|
19
|
+
ragkit/core/__init__.py,sha256=LTgq4yGIC_6jHqoRXHpjVlK-Pt7YyMeW0b5A5l6RZPc,519
|
|
20
|
+
ragkit/core/config.py,sha256=PrXpNxVZj6INS4emu89pvdlYSd6f5OPJ9grfE_4OGyA,2433
|
|
21
|
+
ragkit/core/embeddings.py,sha256=yR1HZmTg8RO5mKMp1kMzeNkAP-O3pW0E6w4szUD_7p8,6550
|
|
22
|
+
ragkit/core/filtering.py,sha256=DzGnRQkVWrmAG2FQiea5KkhbLMTmm-b-xa9Qsrh6JMY,1493
|
|
23
|
+
ragkit/core/hashing.py,sha256=HPtoWbLn9_ZE2fp-n-WGNo5WgoXjKG6bkrpauC3WBNY,1838
|
|
24
|
+
ragkit/core/schema.py,sha256=gYSd3BWv8RDoSJ0cDl0G18Zj5e0VgD46r46SBXKlIEE,1911
|
|
25
|
+
ragkit/middleware/__init__.py,sha256=82ttFM8cG7-lDqcJAgLEXghAXj9d_UC6ZtyTtkC-o9w,1070
|
|
26
|
+
ragkit/middleware/base.py,sha256=LqPmTgh_DU3GofN-zgaps1miZwBY5-x-rxTHmk3Ss40,4166
|
|
27
|
+
ragkit/middleware/builtin.py,sha256=tgqBD1_68ft0Bwwu4ib1tzSDP7Z1E4Z1otnMp_gAHpc,8974
|
|
28
|
+
ragkit/readers/__init__.py,sha256=Ouq4mbpRyf-TLi-n9cbpmCpUoYSaarLozqE7g7GjYVQ,2154
|
|
29
|
+
ragkit/readers/_common.py,sha256=GljbwY5E1PR5bcwpPhlgV1DVvsbL6hMNaotVEDbVm0k,1438
|
|
30
|
+
ragkit/readers/base.py,sha256=SVNVvOKSpvuVZXhbHJPcXCRP-CeZyHxyt44YM-9SZHA,1241
|
|
31
|
+
ragkit/readers/markdown_parser.py,sha256=0RSIMRCHU5F5LeXDLBKFQc415DWzLAuLZCf_eKZKzcs,7297
|
|
32
|
+
ragkit/readers/office.py,sha256=6cbJu5edGLqEDStLbB2yqH3AL0R1YKPmIAC-UgvGV4M,10939
|
|
33
|
+
ragkit/readers/pdf.py,sha256=BZ0iUbXz002YvqcV28zK9-BD9m62wrhLBVmPsbpmhhA,6987
|
|
34
|
+
ragkit/readers/text.py,sha256=B531Yo2_98qWJIBSKlM3jugAVMrcgEVc73iOOKUN39I,2270
|
|
35
|
+
ragkit/rerank/__init__.py,sha256=yxZZi6LPYczDfOzKoXDNNaLhR5dKR_el73KQV2Di3yk,3659
|
|
36
|
+
ragkit/rerank/base.py,sha256=au3M73mB_w-TGcCwiGcPatzP_roG596xdeRgBGQqOso,2655
|
|
37
|
+
ragkit/retrieval/__init__.py,sha256=eBGsC4EWpPtZw_x4sUZp86tlV2VP0rjroXtDX-V4J-U,301
|
|
38
|
+
ragkit/retrieval/fusion.py,sha256=T9UT55aijPZMHDpY2Wf7sIUM6DhEbGbGmqVK3poz-aI,2096
|
|
39
|
+
ragkit/retrieval/hybrid.py,sha256=M4sME-uzYPF603kaR8B5wreKEMKoCMMmnTqkY82NMLg,3198
|
|
40
|
+
ragkit/retrieval/retrievers.py,sha256=6hUeRuVxFYYZ0o_885yGrVzeidiA6YUwAGn3He5rIDM,4833
|
|
41
|
+
ragkit/splitters/__init__.py,sha256=TR36WNSIf3zdvNJgwwotcZlKtd5RqlulLRn4lnbr_8k,1408
|
|
42
|
+
ragkit/splitters/base.py,sha256=2UUBVKDMDWOjNuyEiINI0St2017nFETh9QE2ianpWMY,3068
|
|
43
|
+
ragkit/splitters/heading.py,sha256=_sCXSszE4UwkxIX3OzwQaQjOIQQxYwofFeDOIPCoS9Y,3427
|
|
44
|
+
ragkit/splitters/qa.py,sha256=gxnCxoeUC-KcUG6A_irqcQOtC0bamKhhSdLjXndjP0g,913
|
|
45
|
+
ragkit/splitters/recursive.py,sha256=qUA_6CY7hXXB10QRAbLpptTXJJgJR1P5w6yTXDNI0e0,1883
|
|
46
|
+
ragkit/splitters/table.py,sha256=WbPGmrYDWAH6WHl2FYt_RDFWb4kSu3zF7luB52JLiBw,2128
|
|
47
|
+
ragkit/store/__init__.py,sha256=6NGaRymLHWfcEpxJPAJ_lpYGEvo9UosSThdkO-SV2I8,828
|
|
48
|
+
ragkit/store/base.py,sha256=qNoesyESJD91tA_v3iIx9TwbNvY-miOTgs1_VLbohcY,1316
|
|
49
|
+
ragkit/store/memory.py,sha256=Yu1MQ5e4i9zYPXN7sg9LQYG6YE_WpevVLVUv4zNpAMI,2067
|
|
50
|
+
ragkit/store/qdrant_store.py,sha256=NBYZ86hmK15nCtG86ZkmtEUVY8WX8iKphh573IAgXiE,8326
|
|
51
|
+
ragkit/store/weaviate_store.py,sha256=6_vIZ5cUJnrvEI-IEYktmu7mZKHuKvsFpZWtFcGEZYI,10891
|
|
52
|
+
ragkit/utils/__init__.py,sha256=sAq5qe9CSHrb-OMiIayQM7DPFudM-hSvJTnAWeBoNFE,200
|
|
53
|
+
ragkit/utils/registry.py,sha256=FdNV9wo3VGsqeD4Qv2d2QK4XMEsr-mK_FSjBifvwiLI,1467
|
|
54
|
+
ragkit/utils/timeout.py,sha256=Nq33B4356L_6sI5rXzdrHcAhsOh4zN7SeQSX7HIlt9g,1807
|
|
55
|
+
modular_rag-0.1.0.dist-info/METADATA,sha256=vnsBGaisDEgAayIW71pmlWWLqqHeqv-uFc0jsv8mCpI,39562
|
|
56
|
+
modular_rag-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
57
|
+
modular_rag-0.1.0.dist-info/top_level.txt,sha256=k_6E8qLkT4SfWT8hMp8Oqj4tKp_vcPATvqtaHEoE7WA,7
|
|
58
|
+
modular_rag-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ragkit contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ragkit
|
ragkit/__init__.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""ragkit — 可扩展、可替换抽象的 RAG 库。
|
|
2
|
+
|
|
3
|
+
from ragkit import RAGPipeline, PipelineConfig
|
|
4
|
+
|
|
5
|
+
pipe = RAGPipeline(PipelineConfig())
|
|
6
|
+
pipe.ingest("docs/guide.md")
|
|
7
|
+
results = pipe.retrieve("超时设置默认值是多少?")
|
|
8
|
+
|
|
9
|
+
启用主表+附表(同业务 id 多版本自动取最高版本):
|
|
10
|
+
|
|
11
|
+
from ragkit import InMemoryCatalog, InMemoryChunkStore, FileRecord
|
|
12
|
+
pipe = RAGPipeline(catalog=InMemoryCatalog(), chunk_store=InMemoryChunkStore())
|
|
13
|
+
pipe.ingest(doc, record=FileRecord(business_id="guide", version="v2",
|
|
14
|
+
status="active", owner="alice", md5="..."))
|
|
15
|
+
"""
|
|
16
|
+
from .callbacks import (CallbackManager, ErrorAlertCallbacks, ErrorInfo, Event,
|
|
17
|
+
JsonlAuditCallbacks, LoggingCallbacks, MetricsCallbacks,
|
|
18
|
+
ProgressBarCallbacks, ProgressReporter, SamplingCallbacks,
|
|
19
|
+
TimingCallbacks, TraceCollector)
|
|
20
|
+
from .catalog import (BaseCatalog, BaseChunkStore, ChunkRecord, FileRecord,
|
|
21
|
+
InMemoryCatalog, InMemoryChunkStore, StoreCatalog,
|
|
22
|
+
compare_version, make_version_prefilter)
|
|
23
|
+
from .core.config import (PipelineConfig, RerankConfig, RetrievalConfig,
|
|
24
|
+
SplitConfig)
|
|
25
|
+
from .core.embeddings import BaseEmbedder, HashEmbedder, OllamaEmbedder
|
|
26
|
+
from .core.filtering import make_prefilter
|
|
27
|
+
from .core.schema import Chunk, Document, ScoredChunk
|
|
28
|
+
from .middleware import (AuditMiddleware, BaseMiddleware, CacheMiddleware,
|
|
29
|
+
CircuitBreakerMiddleware, DedupMiddleware,
|
|
30
|
+
GuardMiddleware, LoggingMiddleware, MiddlewareChain,
|
|
31
|
+
MiddlewareError, QueryNormalizeMiddleware,
|
|
32
|
+
ResultFilterMiddleware, RetryMiddleware,
|
|
33
|
+
TopKClampMiddleware)
|
|
34
|
+
from .pipeline import RAGPipeline
|
|
35
|
+
from .ragkit_version import __version__
|
|
36
|
+
from .store import (InMemoryVectorStore, QdrantVectorStore, WeaviateVectorStore,
|
|
37
|
+
create_store)
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"RAGPipeline", "PipelineConfig", "SplitConfig", "RetrievalConfig", "RerankConfig",
|
|
41
|
+
"Document", "Chunk", "ScoredChunk", "BaseEmbedder", "HashEmbedder", "OllamaEmbedder",
|
|
42
|
+
"make_prefilter", "__version__",
|
|
43
|
+
"FileRecord", "ChunkRecord", "BaseCatalog", "InMemoryCatalog", "StoreCatalog",
|
|
44
|
+
"BaseChunkStore", "InMemoryChunkStore", "make_version_prefilter", "compare_version",
|
|
45
|
+
"InMemoryVectorStore", "QdrantVectorStore", "WeaviateVectorStore", "create_store",
|
|
46
|
+
"CallbackManager", "Event", "ErrorInfo", "LoggingCallbacks",
|
|
47
|
+
"TimingCallbacks", "TraceCollector", "ProgressReporter", "ProgressBarCallbacks",
|
|
48
|
+
"MetricsCallbacks", "JsonlAuditCallbacks", "ErrorAlertCallbacks", "SamplingCallbacks",
|
|
49
|
+
"BaseMiddleware", "MiddlewareChain", "MiddlewareError", "LoggingMiddleware",
|
|
50
|
+
"RetryMiddleware", "CacheMiddleware", "GuardMiddleware", "ResultFilterMiddleware",
|
|
51
|
+
"DedupMiddleware", "TopKClampMiddleware", "QueryNormalizeMiddleware",
|
|
52
|
+
"CircuitBreakerMiddleware", "AuditMiddleware",
|
|
53
|
+
]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""回调机制:生命周期事件、错误上报、进度、内置调试与运维实现。
|
|
2
|
+
|
|
3
|
+
- 事件与错误:Event / ErrorInfo
|
|
4
|
+
- 注册分发:CallbackManager / as_manager
|
|
5
|
+
- 进度:ProgressReporter
|
|
6
|
+
- 内置实现:Logging / Timing / Trace / ProgressBar / Metrics / JsonlAudit / ErrorAlert / Sampling
|
|
7
|
+
"""
|
|
8
|
+
from .builtin import (ErrorAlertCallbacks, JsonlAuditCallbacks, LoggingCallbacks,
|
|
9
|
+
MetricsCallbacks, ProgressBarCallbacks, SamplingCallbacks,
|
|
10
|
+
TimingCallbacks, TraceCollector)
|
|
11
|
+
from .events import ErrorInfo, Event
|
|
12
|
+
from .manager import CallbackManager, Handler, as_manager
|
|
13
|
+
from .progress import ProgressReporter
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"Event", "ErrorInfo", "CallbackManager", "Handler", "as_manager",
|
|
17
|
+
"ProgressReporter",
|
|
18
|
+
"LoggingCallbacks", "TimingCallbacks", "TraceCollector", "ProgressBarCallbacks",
|
|
19
|
+
"MetricsCallbacks", "JsonlAuditCallbacks", "ErrorAlertCallbacks", "SamplingCallbacks",
|
|
20
|
+
]
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""内置回调实现:日志 / 耗时 / 追踪 / 进度条 / 指标 / 审计 / 告警 / 采样。
|
|
2
|
+
|
|
3
|
+
全部只需 `cb.on(event, 实例)` 即可挂载,也可直接作为 callbacks= 传入。
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import random
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from typing import Any, Dict, List, Optional, TextIO
|
|
13
|
+
|
|
14
|
+
from .events import ErrorInfo, Event
|
|
15
|
+
|
|
16
|
+
_STAGE_LABEL = {"read": "读取", "split": "切分", "embed": "向量化", "store": "入库"}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _safe(payload: Dict[str, Any]) -> Dict[str, Any]:
|
|
20
|
+
"""只保留可 JSON 序列化的标量字段,避免日志/审计写崩。"""
|
|
21
|
+
skip = {"error_info", "documents", "chunks", "results", "detail"}
|
|
22
|
+
return {k: v for k, v in payload.items()
|
|
23
|
+
if k not in skip and isinstance(v, (str, int, float, bool, type(None), list, dict))}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class LoggingCallbacks:
|
|
27
|
+
"""把事件打到标准 logging,调试阶段问题最直接的方式。"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, logger: Optional[logging.Logger] = None,
|
|
30
|
+
level: int = logging.INFO, include: Optional[List[Event]] = None) -> None:
|
|
31
|
+
self.logger = logger or logging.getLogger("ragkit")
|
|
32
|
+
self.level = level
|
|
33
|
+
self.include = {e.value for e in include} if include else None
|
|
34
|
+
|
|
35
|
+
def __call__(self, payload: Dict[str, Any]) -> None:
|
|
36
|
+
event = payload.get("event", "")
|
|
37
|
+
if self.include is not None and event not in self.include:
|
|
38
|
+
return
|
|
39
|
+
if "error_info" in payload:
|
|
40
|
+
self.logger.log(self.level, "[ragkit] %s 失败: %s",
|
|
41
|
+
payload.get("stage", event), payload["error_info"].message)
|
|
42
|
+
return
|
|
43
|
+
extra = {k: v for k, v in payload.items()
|
|
44
|
+
if k in ("count", "top_k", "duration_ms", "source", "mode", "percent")}
|
|
45
|
+
self.logger.log(self.level, "[ragkit] %s %s", event, extra or "")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class TimingCallbacks:
|
|
49
|
+
"""记录各阶段耗时,summary() 汇总用于定位瓶颈。"""
|
|
50
|
+
|
|
51
|
+
def __init__(self) -> None:
|
|
52
|
+
self.records: Dict[str, List[float]] = {}
|
|
53
|
+
|
|
54
|
+
def __call__(self, payload: Dict[str, Any]) -> None:
|
|
55
|
+
duration = payload.get("duration_ms")
|
|
56
|
+
if duration is not None:
|
|
57
|
+
self.records.setdefault(payload.get("event", ""), []).append(float(duration))
|
|
58
|
+
|
|
59
|
+
def summary(self) -> Dict[str, Dict[str, float]]:
|
|
60
|
+
out: Dict[str, Dict[str, float]] = {}
|
|
61
|
+
for event, values in self.records.items():
|
|
62
|
+
out[event] = {
|
|
63
|
+
"count": len(values),
|
|
64
|
+
"total_ms": round(sum(values), 2),
|
|
65
|
+
"avg_ms": round(sum(values) / len(values), 2),
|
|
66
|
+
"max_ms": round(max(values), 2),
|
|
67
|
+
}
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class TraceCollector:
|
|
72
|
+
"""按顺序收集全部事件,dump() 打印完整调用链,便于复现问题。"""
|
|
73
|
+
|
|
74
|
+
def __init__(self, maxlen: int = 500) -> None:
|
|
75
|
+
self.events: List[Dict[str, Any]] = []
|
|
76
|
+
self.maxlen = maxlen
|
|
77
|
+
|
|
78
|
+
def __call__(self, payload: Dict[str, Any]) -> None:
|
|
79
|
+
self.events.append(payload)
|
|
80
|
+
if len(self.events) > self.maxlen:
|
|
81
|
+
self.events.pop(0)
|
|
82
|
+
|
|
83
|
+
def dump(self, only: Optional[List[Event]] = None) -> str:
|
|
84
|
+
keep = {e.value for e in only} if only else None
|
|
85
|
+
lines = []
|
|
86
|
+
for p in self.events:
|
|
87
|
+
if keep is not None and p.get("event") not in keep:
|
|
88
|
+
continue
|
|
89
|
+
detail = {k: v for k, v in p.items()
|
|
90
|
+
if k not in ("event", "ts") and isinstance(v, (str, int, float, bool))}
|
|
91
|
+
lines.append(f"{p.get('event')} {detail or ''}".rstrip())
|
|
92
|
+
return "\n".join(lines)
|
|
93
|
+
|
|
94
|
+
def attach_to(self, manager) -> "TraceCollector":
|
|
95
|
+
for event in Event:
|
|
96
|
+
manager.on(event, self)
|
|
97
|
+
return self
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class ProgressBarCallbacks:
|
|
101
|
+
"""终端进度条:挂在 PROGRESS 上即可原地刷新。
|
|
102
|
+
|
|
103
|
+
cb.on(Event.PROGRESS, ProgressBarCallbacks())
|
|
104
|
+
|
|
105
|
+
非终端环境(如 CI 日志)可设 `enabled=False` 或传入自己的 stream。
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
def __init__(self, stream: Optional[TextIO] = None, width: int = 28,
|
|
109
|
+
enabled: bool = True) -> None:
|
|
110
|
+
self.stream = stream or sys.stdout
|
|
111
|
+
self.width = width
|
|
112
|
+
self.enabled = enabled
|
|
113
|
+
self._last_stage: Optional[str] = None
|
|
114
|
+
|
|
115
|
+
def __call__(self, payload: Dict[str, Any]) -> None:
|
|
116
|
+
if not self.enabled:
|
|
117
|
+
return
|
|
118
|
+
if payload.get("event") != Event.PROGRESS.value and "percent" not in payload:
|
|
119
|
+
return
|
|
120
|
+
stage = payload.get("stage", "")
|
|
121
|
+
if self._last_stage and self._last_stage != stage:
|
|
122
|
+
self.stream.write("\n") # 阶段切换换行
|
|
123
|
+
self._last_stage = stage
|
|
124
|
+
|
|
125
|
+
percent = float(payload.get("percent", 0.0))
|
|
126
|
+
filled = int(self.width * percent / 100)
|
|
127
|
+
bar = "█" * filled + "░" * (self.width - filled)
|
|
128
|
+
eta = payload.get("eta_ms")
|
|
129
|
+
eta_text = f" 剩余 {eta / 1000:.1f}s" if eta and eta >= 100 else ""
|
|
130
|
+
msg = f" {payload['message']}" if payload.get("message") else ""
|
|
131
|
+
self.stream.write(
|
|
132
|
+
f"\r[{_STAGE_LABEL.get(stage, stage)}] |{bar}| "
|
|
133
|
+
f"{payload.get('current', 0)}/{payload.get('total', 0)} "
|
|
134
|
+
f"{percent:5.1f}%{eta_text}{msg} ")
|
|
135
|
+
self.stream.flush()
|
|
136
|
+
if percent >= 100:
|
|
137
|
+
self.stream.write("\n")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class MetricsCallbacks:
|
|
141
|
+
"""轻量指标收集,便于对接 Prometheus / 自建看板。
|
|
142
|
+
|
|
143
|
+
metrics = MetricsCallbacks()
|
|
144
|
+
cb.on(Event.ERROR, metrics) # 或 attach_to 全量
|
|
145
|
+
metrics.snapshot() # {"events": {...}, "errors": {...}, "durations": {...}}
|
|
146
|
+
"""
|
|
147
|
+
|
|
148
|
+
def __init__(self) -> None:
|
|
149
|
+
self.counts: Dict[str, int] = {}
|
|
150
|
+
self.errors: Dict[str, int] = {}
|
|
151
|
+
self.durations: Dict[str, List[float]] = {}
|
|
152
|
+
|
|
153
|
+
def __call__(self, payload: Dict[str, Any]) -> None:
|
|
154
|
+
event = payload.get("event", "")
|
|
155
|
+
self.counts[event] = self.counts.get(event, 0) + 1
|
|
156
|
+
if "error_info" in payload:
|
|
157
|
+
stage = payload.get("stage", event)
|
|
158
|
+
self.errors[stage] = self.errors.get(stage, 0) + 1
|
|
159
|
+
d = payload.get("duration_ms")
|
|
160
|
+
if d is not None:
|
|
161
|
+
self.durations.setdefault(event, []).append(float(d))
|
|
162
|
+
|
|
163
|
+
def snapshot(self) -> Dict[str, Any]:
|
|
164
|
+
durations = {
|
|
165
|
+
e: {"count": len(v), "avg": round(sum(v) / len(v), 2), "max": round(max(v), 2)}
|
|
166
|
+
for e, v in self.durations.items()
|
|
167
|
+
}
|
|
168
|
+
return {"events": dict(self.counts), "errors": dict(self.errors),
|
|
169
|
+
"durations": durations}
|
|
170
|
+
|
|
171
|
+
def attach_to(self, manager) -> "MetricsCallbacks":
|
|
172
|
+
for event in Event:
|
|
173
|
+
manager.on(event, self)
|
|
174
|
+
return self
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class JsonlAuditCallbacks:
|
|
178
|
+
"""把事件按行写成 JSONL,用于审计与离线分析。
|
|
179
|
+
|
|
180
|
+
audit = JsonlAuditCallbacks("audit.jsonl")
|
|
181
|
+
audit.attach_to(cb)
|
|
182
|
+
"""
|
|
183
|
+
|
|
184
|
+
def __init__(self, path: Optional[str] = None, stream: Optional[TextIO] = None,
|
|
185
|
+
flush_each: bool = True) -> None:
|
|
186
|
+
if stream is None and path is None:
|
|
187
|
+
raise ValueError("需要 path 或 stream 之一")
|
|
188
|
+
self.path = path
|
|
189
|
+
self.stream = stream or open(path, "a", encoding="utf-8") # noqa: SIM115
|
|
190
|
+
self.flush_each = flush_each
|
|
191
|
+
|
|
192
|
+
def __call__(self, payload: Dict[str, Any]) -> None:
|
|
193
|
+
record = _safe(payload)
|
|
194
|
+
if "error_info" in payload:
|
|
195
|
+
record["error"] = payload["error_info"].to_dict()
|
|
196
|
+
self.stream.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
|
|
197
|
+
if self.flush_each:
|
|
198
|
+
self.stream.flush()
|
|
199
|
+
|
|
200
|
+
def attach_to(self, manager) -> "JsonlAuditCallbacks":
|
|
201
|
+
for event in Event:
|
|
202
|
+
manager.on(event, self)
|
|
203
|
+
return self
|
|
204
|
+
|
|
205
|
+
def close(self) -> None:
|
|
206
|
+
try:
|
|
207
|
+
self.stream.close()
|
|
208
|
+
except Exception:
|
|
209
|
+
pass
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class ErrorAlertCallbacks:
|
|
213
|
+
"""错误告警:把 ErrorInfo 交给注入的 notifier(webhook / 邮件 / Sentry…)。
|
|
214
|
+
|
|
215
|
+
cb.on(Event.ERROR, ErrorAlertCallbacks(lambda info: send_webhook(info.to_dict())))
|
|
216
|
+
"""
|
|
217
|
+
|
|
218
|
+
def __init__(self, notifier, *, cooldown: float = 60.0) -> None:
|
|
219
|
+
if not callable(notifier):
|
|
220
|
+
raise ValueError("notifier 必须是 callable(ErrorInfo)")
|
|
221
|
+
self.notifier = notifier
|
|
222
|
+
self.cooldown = cooldown
|
|
223
|
+
self._last: Dict[str, float] = {}
|
|
224
|
+
self.fired = 0
|
|
225
|
+
|
|
226
|
+
def __call__(self, payload: Dict[str, Any]) -> None:
|
|
227
|
+
info: Optional[ErrorInfo] = payload.get("error_info")
|
|
228
|
+
if info is None:
|
|
229
|
+
return
|
|
230
|
+
now = time.time()
|
|
231
|
+
if now - self._last.get(info.stage, 0.0) < self.cooldown:
|
|
232
|
+
return # 同一阶段冷却,避免告警风暴
|
|
233
|
+
self._last[info.stage] = now
|
|
234
|
+
self.fired += 1
|
|
235
|
+
try:
|
|
236
|
+
self.notifier(info)
|
|
237
|
+
except Exception:
|
|
238
|
+
pass # 告警失败不能影响主流程
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class SamplingCallbacks:
|
|
242
|
+
"""按概率采样转发,用于高频场景降噪(如只记录 10% 的检索事件)。"""
|
|
243
|
+
|
|
244
|
+
def __init__(self, target, rate: float = 0.1, seed: Optional[int] = None) -> None:
|
|
245
|
+
if not callable(target):
|
|
246
|
+
raise ValueError("target 必须是 callable")
|
|
247
|
+
if not 0.0 <= rate <= 1.0:
|
|
248
|
+
raise ValueError("rate 必须在 [0, 1]")
|
|
249
|
+
self.target = target
|
|
250
|
+
self.rate = rate
|
|
251
|
+
self._rng = random.Random(seed)
|
|
252
|
+
self.seen = 0
|
|
253
|
+
self.forwarded = 0
|
|
254
|
+
|
|
255
|
+
def __call__(self, payload: Dict[str, Any]) -> None:
|
|
256
|
+
self.seen += 1
|
|
257
|
+
if self._rng.random() < self.rate:
|
|
258
|
+
self.forwarded += 1
|
|
259
|
+
self.target(payload)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""事件枚举与结构化错误上下文。"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
import traceback
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from enum import Enum
|
|
8
|
+
from typing import Any, Dict
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Event(str, Enum):
|
|
12
|
+
"""生命周期事件。值即事件名,可直接与字符串比较。"""
|
|
13
|
+
|
|
14
|
+
INGEST_START = "ingest:start"
|
|
15
|
+
PROGRESS = "ingest:progress" # 细粒度进度(防"看起来卡死")
|
|
16
|
+
READ_END = "read:end"
|
|
17
|
+
SPLIT_END = "split:end"
|
|
18
|
+
EMBED_END = "embed:end"
|
|
19
|
+
STORE_END = "store:end"
|
|
20
|
+
INGEST_END = "ingest:end"
|
|
21
|
+
INGEST_SKIPPED = "ingest:skipped" # 内容未变(MD5 相同)而跳过
|
|
22
|
+
INGEST_ERROR = "ingest:error"
|
|
23
|
+
|
|
24
|
+
RETRIEVE_START = "retrieve:start"
|
|
25
|
+
RECALL_END = "recall:end"
|
|
26
|
+
RERANK_END = "rerank:end"
|
|
27
|
+
RERANK_FALLBACK = "rerank:fallback"
|
|
28
|
+
RETRIEVE_END = "retrieve:end"
|
|
29
|
+
RETRIEVE_ERROR = "retrieve:error"
|
|
30
|
+
|
|
31
|
+
ERROR = "error" # 通用错误(所有阶段失败都会发)
|
|
32
|
+
CALLBACK_ERROR = "callback:error" # 回调自身出错(用于自检)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class ErrorInfo:
|
|
37
|
+
"""结构化错误上下文,便于日志与排查。"""
|
|
38
|
+
|
|
39
|
+
stage: str
|
|
40
|
+
error: BaseException
|
|
41
|
+
payload: Dict[str, Any] = field(default_factory=dict)
|
|
42
|
+
timestamp: float = field(default_factory=time.time)
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def message(self) -> str:
|
|
46
|
+
return f"{type(self.error).__name__}: {self.error}"
|
|
47
|
+
|
|
48
|
+
def format(self) -> str:
|
|
49
|
+
tb = "".join(traceback.format_exception(
|
|
50
|
+
type(self.error), self.error, self.error.__traceback__))
|
|
51
|
+
return f"[{self.stage}] {self.message}\n{tb}"
|
|
52
|
+
|
|
53
|
+
def to_dict(self) -> dict:
|
|
54
|
+
return {"stage": self.stage, "message": self.message,
|
|
55
|
+
"error_type": type(self.error).__name__,
|
|
56
|
+
"payload": {k: v for k, v in self.payload.items()
|
|
57
|
+
if isinstance(v, (str, int, float, bool, type(None)))},
|
|
58
|
+
"timestamp": self.timestamp}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""回调注册与分发(错误隔离)。"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
from typing import Any, Callable, Dict, List
|
|
6
|
+
|
|
7
|
+
from .events import ErrorInfo, Event
|
|
8
|
+
|
|
9
|
+
Handler = Callable[[Dict[str, Any]], None]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class CallbackManager:
|
|
13
|
+
"""回调注册与分发。所有 handler 签名为 fn(payload: dict)。"""
|
|
14
|
+
|
|
15
|
+
def __init__(self, *, max_errors: int = 100) -> None:
|
|
16
|
+
self._handlers: Dict[str, List[tuple[int, Handler]]] = {}
|
|
17
|
+
self.callback_errors: List[Dict[str, Any]] = []
|
|
18
|
+
self._max_errors = max_errors
|
|
19
|
+
|
|
20
|
+
# ---- 注册 / 注销 ----
|
|
21
|
+
def on(self, event: Event | str, fn: Handler, *, priority: int = 0) -> "CallbackManager":
|
|
22
|
+
key = event.value if isinstance(event, Event) else str(event)
|
|
23
|
+
self._handlers.setdefault(key, []).append((priority, fn))
|
|
24
|
+
self._handlers[key].sort(key=lambda x: -x[0]) # 高优先级先执行
|
|
25
|
+
return self
|
|
26
|
+
|
|
27
|
+
def off(self, event: Event | str, fn: Handler) -> "CallbackManager":
|
|
28
|
+
key = event.value if isinstance(event, Event) else str(event)
|
|
29
|
+
self._handlers[key] = [(p, f) for p, f in self._handlers.get(key, []) if f is not fn]
|
|
30
|
+
return self
|
|
31
|
+
|
|
32
|
+
def clear(self, event: Event | str | None = None) -> "CallbackManager":
|
|
33
|
+
if event is None:
|
|
34
|
+
self._handlers.clear()
|
|
35
|
+
else:
|
|
36
|
+
key = event.value if isinstance(event, Event) else str(event)
|
|
37
|
+
self._handlers.pop(key, None)
|
|
38
|
+
return self
|
|
39
|
+
|
|
40
|
+
def has(self, event: Event | str) -> bool:
|
|
41
|
+
key = event.value if isinstance(event, Event) else str(event)
|
|
42
|
+
return bool(self._handlers.get(key))
|
|
43
|
+
|
|
44
|
+
# ---- 分发(错误隔离)----
|
|
45
|
+
def emit(self, event: Event | str, **payload: Any) -> None:
|
|
46
|
+
key = event.value if isinstance(event, Event) else str(event)
|
|
47
|
+
handlers = self._handlers.get(key)
|
|
48
|
+
if not handlers:
|
|
49
|
+
return
|
|
50
|
+
data = {"event": key, "ts": time.time(), **payload}
|
|
51
|
+
for _, fn in list(handlers):
|
|
52
|
+
try:
|
|
53
|
+
fn(data)
|
|
54
|
+
except Exception as exc: # 回调错误绝不影响主流程
|
|
55
|
+
self._record_callback_error(key, exc)
|
|
56
|
+
|
|
57
|
+
def _record_callback_error(self, event: str, exc: BaseException) -> None:
|
|
58
|
+
self.callback_errors.append({"event": event, "error": repr(exc),
|
|
59
|
+
"ts": time.time()})
|
|
60
|
+
if len(self.callback_errors) > self._max_errors:
|
|
61
|
+
self.callback_errors.pop(0)
|
|
62
|
+
|
|
63
|
+
def emit_error(self, stage: str, error: BaseException, **payload: Any) -> None:
|
|
64
|
+
"""上报结构化错误:先发通用 ERROR,再发阶段专属错误事件。"""
|
|
65
|
+
info = ErrorInfo(stage=stage, error=error, payload=payload)
|
|
66
|
+
self.emit(Event.ERROR, stage=stage, error=error, error_info=info, **payload)
|
|
67
|
+
stage_event = {"ingest": Event.INGEST_ERROR,
|
|
68
|
+
"retrieve": Event.RETRIEVE_ERROR}.get(stage)
|
|
69
|
+
if stage_event is not None:
|
|
70
|
+
self.emit(stage_event, stage=stage, error=error, error_info=info, **payload)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def as_manager(callbacks: Any) -> CallbackManager:
|
|
74
|
+
"""把 None / CallbackManager / dict{event: [fn]} / callable 统一成 CallbackManager。"""
|
|
75
|
+
if isinstance(callbacks, CallbackManager):
|
|
76
|
+
return callbacks
|
|
77
|
+
mgr = CallbackManager()
|
|
78
|
+
if callbacks is None:
|
|
79
|
+
return mgr
|
|
80
|
+
if isinstance(callbacks, dict):
|
|
81
|
+
for event, fns in callbacks.items():
|
|
82
|
+
for fn in (fns if isinstance(fns, (list, tuple)) else [fns]):
|
|
83
|
+
mgr.on(event, fn)
|
|
84
|
+
return mgr
|
|
85
|
+
if callable(callbacks):
|
|
86
|
+
for event in Event:
|
|
87
|
+
mgr.on(event, callbacks)
|
|
88
|
+
return mgr
|
|
89
|
+
raise TypeError("callbacks 需要 CallbackManager / dict / callable / None")
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""带节流的进度上报器。"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
from typing import Dict
|
|
6
|
+
|
|
7
|
+
from .events import Event
|
|
8
|
+
from .manager import CallbackManager
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ProgressReporter:
|
|
12
|
+
"""专治"长耗时流程看起来卡死"。
|
|
13
|
+
|
|
14
|
+
- 默认 200ms 节流,避免高频 emit 拖慢主流程;
|
|
15
|
+
- 到 100% 时强制发一次,保证前端/终端能收尾;
|
|
16
|
+
- 未注册 PROGRESS 回调时直接短路,零额外开销;
|
|
17
|
+
- payload 含 current / total / percent / elapsed_ms / eta_ms,可直接驱动进度条。
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(self, manager: CallbackManager, *, interval: float = 0.2,
|
|
21
|
+
event: Event | str = Event.PROGRESS) -> None:
|
|
22
|
+
self.manager = manager
|
|
23
|
+
self.interval = interval
|
|
24
|
+
self.event = event
|
|
25
|
+
self._start: Dict[str, float] = {}
|
|
26
|
+
self._last: Dict[str, float] = {}
|
|
27
|
+
self._total: Dict[str, int] = {}
|
|
28
|
+
|
|
29
|
+
def begin(self, stage: str, total: int = 0) -> None:
|
|
30
|
+
now = time.perf_counter()
|
|
31
|
+
self._start[stage] = now
|
|
32
|
+
self._last[stage] = 0.0
|
|
33
|
+
self._total[stage] = total
|
|
34
|
+
self.update(stage, 0, total, force=True)
|
|
35
|
+
|
|
36
|
+
def update(self, stage: str, current: int, total: int,
|
|
37
|
+
message: str = "", force: bool = False) -> None:
|
|
38
|
+
if not self.manager.has(self.event):
|
|
39
|
+
return
|
|
40
|
+
now = time.perf_counter()
|
|
41
|
+
done = total > 0 and current >= total
|
|
42
|
+
if not force and not done and now - self._last.get(stage, 0.0) < self.interval:
|
|
43
|
+
return # 节流:非完成态且未到间隔,跳过
|
|
44
|
+
self._last[stage] = now
|
|
45
|
+
self._total[stage] = total
|
|
46
|
+
start = self._start.setdefault(stage, now)
|
|
47
|
+
elapsed = (now - start) * 1000
|
|
48
|
+
percent = round(current / total * 100, 1) if total else 0.0
|
|
49
|
+
eta = round(elapsed / current * (total - current), 1) if current > 0 else None
|
|
50
|
+
self.manager.emit(self.event, stage=stage, current=current, total=total,
|
|
51
|
+
percent=percent, elapsed_ms=round(elapsed, 2),
|
|
52
|
+
eta_ms=eta, message=message)
|
|
53
|
+
|
|
54
|
+
def finish(self, stage: str, total: int | None = None, message: str = "") -> None:
|
|
55
|
+
if total is None:
|
|
56
|
+
total = self._total.get(stage, 0)
|
|
57
|
+
self.update(stage, total, total, message=message, force=True)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""catalog 包:主表 + 附表 + 版本择优 + 字段映射。"""
|
|
2
|
+
from .base import BaseCatalog, InMemoryCatalog
|
|
3
|
+
from .chunk_store import BaseChunkStore, InMemoryChunkStore
|
|
4
|
+
from .filtering import make_version_prefilter
|
|
5
|
+
from .models import ChunkRecord, FileRecord
|
|
6
|
+
from .scalar import (from_scalar_props, property_columns, property_types,
|
|
7
|
+
to_scalar_props)
|
|
8
|
+
from .store_catalog import StoreCatalog
|
|
9
|
+
from .version import compare_version, highest_version, version_key
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"FileRecord", "ChunkRecord", "BaseCatalog", "InMemoryCatalog", "StoreCatalog",
|
|
13
|
+
"BaseChunkStore", "InMemoryChunkStore",
|
|
14
|
+
"make_version_prefilter", "compare_version", "highest_version", "version_key",
|
|
15
|
+
"to_scalar_props", "from_scalar_props", "property_columns", "property_types",
|
|
16
|
+
]
|