modular-rag 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. modular_rag-0.1.0.dist-info/METADATA +1073 -0
  2. modular_rag-0.1.0.dist-info/RECORD +58 -0
  3. modular_rag-0.1.0.dist-info/WHEEL +5 -0
  4. modular_rag-0.1.0.dist-info/licenses/LICENSE +21 -0
  5. modular_rag-0.1.0.dist-info/top_level.txt +1 -0
  6. ragkit/__init__.py +53 -0
  7. ragkit/callbacks/__init__.py +20 -0
  8. ragkit/callbacks/builtin.py +259 -0
  9. ragkit/callbacks/events.py +58 -0
  10. ragkit/callbacks/manager.py +89 -0
  11. ragkit/callbacks/progress.py +57 -0
  12. ragkit/catalog/__init__.py +16 -0
  13. ragkit/catalog/base.py +74 -0
  14. ragkit/catalog/chunk_store.py +109 -0
  15. ragkit/catalog/filtering.py +61 -0
  16. ragkit/catalog/models.py +236 -0
  17. ragkit/catalog/scalar.py +122 -0
  18. ragkit/catalog/store_catalog.py +66 -0
  19. ragkit/catalog/version.py +50 -0
  20. ragkit/core/__init__.py +11 -0
  21. ragkit/core/config.py +60 -0
  22. ragkit/core/embeddings.py +161 -0
  23. ragkit/core/filtering.py +44 -0
  24. ragkit/core/hashing.py +60 -0
  25. ragkit/core/schema.py +67 -0
  26. ragkit/middleware/__init__.py +20 -0
  27. ragkit/middleware/base.py +117 -0
  28. ragkit/middleware/builtin.py +256 -0
  29. ragkit/pipeline.py +391 -0
  30. ragkit/py.typed +0 -0
  31. ragkit/ragkit_version.py +1 -0
  32. ragkit/readers/__init__.py +61 -0
  33. ragkit/readers/_common.py +38 -0
  34. ragkit/readers/base.py +35 -0
  35. ragkit/readers/markdown_parser.py +221 -0
  36. ragkit/readers/office.py +291 -0
  37. ragkit/readers/pdf.py +162 -0
  38. ragkit/readers/text.py +62 -0
  39. ragkit/rerank/__init__.py +95 -0
  40. ragkit/rerank/base.py +63 -0
  41. ragkit/retrieval/__init__.py +7 -0
  42. ragkit/retrieval/fusion.py +61 -0
  43. ragkit/retrieval/hybrid.py +81 -0
  44. ragkit/retrieval/retrievers.py +133 -0
  45. ragkit/splitters/__init__.py +41 -0
  46. ragkit/splitters/base.py +83 -0
  47. ragkit/splitters/heading.py +76 -0
  48. ragkit/splitters/qa.py +26 -0
  49. ragkit/splitters/recursive.py +49 -0
  50. ragkit/splitters/table.py +54 -0
  51. ragkit/store/__init__.py +23 -0
  52. ragkit/store/base.py +40 -0
  53. ragkit/store/memory.py +55 -0
  54. ragkit/store/qdrant_store.py +203 -0
  55. ragkit/store/weaviate_store.py +249 -0
  56. ragkit/utils/__init__.py +5 -0
  57. ragkit/utils/registry.py +46 -0
  58. ragkit/utils/timeout.py +49 -0
@@ -0,0 +1,58 @@
1
+ modular_rag-0.1.0.dist-info/licenses/LICENSE,sha256=6_7ITuxFnNMK5_IcGJ0E61TKRdHkm93FF5qD0AECRcA,1076
2
+ ragkit/__init__.py,sha256=LYmQ51ig9QmMhsdyJBO3gtvzTo4_QOErAHeTr8khODM,3044
3
+ ragkit/pipeline.py,sha256=xoNm0VLJwyPQwo07KS9E2FokCeujvCycP7WTXprpSiU,19085
4
+ ragkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ ragkit/ragkit_version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
6
+ ragkit/callbacks/__init__.py,sha256=Jsv4USqg0XDUalu2HtrrKzZ-V0FKtIAXZNtM-aXJdGg,951
7
+ ragkit/callbacks/builtin.py,sha256=e0buOgXg7zqSCK6Sxcc7C4z-k7ZpSsW82saT21FuiOg,9611
8
+ ragkit/callbacks/events.py,sha256=wGcAmDWLnnzSnSeyEkBnloM1SmXMbGdkIhfqlqUNimU,1961
9
+ ragkit/callbacks/manager.py,sha256=3IUjSFU4UGtXeVs2evdixtwklYTqKhag2p-53lkMYZc,3727
10
+ ragkit/callbacks/progress.py,sha256=56zBqeCglY75mXtOIZMIPpUi7ts-6sHqsFjtcM_woFw,2326
11
+ ragkit/catalog/__init__.py,sha256=gaDS09FemdovDr0zlPAsGnvCUUZDO3g6BjucJklUq18,792
12
+ ragkit/catalog/base.py,sha256=Jf9l_CIauWptdFPDBP8QqriZwiHPS3uQbkCAB6f-7sA,2595
13
+ ragkit/catalog/chunk_store.py,sha256=ak-mhSiq30YAVgahdruMqavW8Diw6EGFGw69YniAF1Y,4841
14
+ ragkit/catalog/filtering.py,sha256=r2Lue7s03VLv21wDN3yAHecCUSBr4NeoBhERvsAWiP8,2128
15
+ ragkit/catalog/models.py,sha256=2RJl2Yz7EjCNtMZBabv4INV1BAmuBN-JgRdA8rPDvXs,10034
16
+ ragkit/catalog/scalar.py,sha256=V6LWgE5N9mYtqBvdgGouBD-PlQ9ZTYZfGWdbnxE5gag,4293
17
+ ragkit/catalog/store_catalog.py,sha256=9WRZuDLSpAX-eiq77rw244Zw4pTbXaAI8K8JDX8wKTk,2469
18
+ ragkit/catalog/version.py,sha256=8kNSNzQd0KaMZcNBx9TUZYtdtmmfYdRfLJDHrFKukuY,1527
19
+ ragkit/core/__init__.py,sha256=LTgq4yGIC_6jHqoRXHpjVlK-Pt7YyMeW0b5A5l6RZPc,519
20
+ ragkit/core/config.py,sha256=PrXpNxVZj6INS4emu89pvdlYSd6f5OPJ9grfE_4OGyA,2433
21
+ ragkit/core/embeddings.py,sha256=yR1HZmTg8RO5mKMp1kMzeNkAP-O3pW0E6w4szUD_7p8,6550
22
+ ragkit/core/filtering.py,sha256=DzGnRQkVWrmAG2FQiea5KkhbLMTmm-b-xa9Qsrh6JMY,1493
23
+ ragkit/core/hashing.py,sha256=HPtoWbLn9_ZE2fp-n-WGNo5WgoXjKG6bkrpauC3WBNY,1838
24
+ ragkit/core/schema.py,sha256=gYSd3BWv8RDoSJ0cDl0G18Zj5e0VgD46r46SBXKlIEE,1911
25
+ ragkit/middleware/__init__.py,sha256=82ttFM8cG7-lDqcJAgLEXghAXj9d_UC6ZtyTtkC-o9w,1070
26
+ ragkit/middleware/base.py,sha256=LqPmTgh_DU3GofN-zgaps1miZwBY5-x-rxTHmk3Ss40,4166
27
+ ragkit/middleware/builtin.py,sha256=tgqBD1_68ft0Bwwu4ib1tzSDP7Z1E4Z1otnMp_gAHpc,8974
28
+ ragkit/readers/__init__.py,sha256=Ouq4mbpRyf-TLi-n9cbpmCpUoYSaarLozqE7g7GjYVQ,2154
29
+ ragkit/readers/_common.py,sha256=GljbwY5E1PR5bcwpPhlgV1DVvsbL6hMNaotVEDbVm0k,1438
30
+ ragkit/readers/base.py,sha256=SVNVvOKSpvuVZXhbHJPcXCRP-CeZyHxyt44YM-9SZHA,1241
31
+ ragkit/readers/markdown_parser.py,sha256=0RSIMRCHU5F5LeXDLBKFQc415DWzLAuLZCf_eKZKzcs,7297
32
+ ragkit/readers/office.py,sha256=6cbJu5edGLqEDStLbB2yqH3AL0R1YKPmIAC-UgvGV4M,10939
33
+ ragkit/readers/pdf.py,sha256=BZ0iUbXz002YvqcV28zK9-BD9m62wrhLBVmPsbpmhhA,6987
34
+ ragkit/readers/text.py,sha256=B531Yo2_98qWJIBSKlM3jugAVMrcgEVc73iOOKUN39I,2270
35
+ ragkit/rerank/__init__.py,sha256=yxZZi6LPYczDfOzKoXDNNaLhR5dKR_el73KQV2Di3yk,3659
36
+ ragkit/rerank/base.py,sha256=au3M73mB_w-TGcCwiGcPatzP_roG596xdeRgBGQqOso,2655
37
+ ragkit/retrieval/__init__.py,sha256=eBGsC4EWpPtZw_x4sUZp86tlV2VP0rjroXtDX-V4J-U,301
38
+ ragkit/retrieval/fusion.py,sha256=T9UT55aijPZMHDpY2Wf7sIUM6DhEbGbGmqVK3poz-aI,2096
39
+ ragkit/retrieval/hybrid.py,sha256=M4sME-uzYPF603kaR8B5wreKEMKoCMMmnTqkY82NMLg,3198
40
+ ragkit/retrieval/retrievers.py,sha256=6hUeRuVxFYYZ0o_885yGrVzeidiA6YUwAGn3He5rIDM,4833
41
+ ragkit/splitters/__init__.py,sha256=TR36WNSIf3zdvNJgwwotcZlKtd5RqlulLRn4lnbr_8k,1408
42
+ ragkit/splitters/base.py,sha256=2UUBVKDMDWOjNuyEiINI0St2017nFETh9QE2ianpWMY,3068
43
+ ragkit/splitters/heading.py,sha256=_sCXSszE4UwkxIX3OzwQaQjOIQQxYwofFeDOIPCoS9Y,3427
44
+ ragkit/splitters/qa.py,sha256=gxnCxoeUC-KcUG6A_irqcQOtC0bamKhhSdLjXndjP0g,913
45
+ ragkit/splitters/recursive.py,sha256=qUA_6CY7hXXB10QRAbLpptTXJJgJR1P5w6yTXDNI0e0,1883
46
+ ragkit/splitters/table.py,sha256=WbPGmrYDWAH6WHl2FYt_RDFWb4kSu3zF7luB52JLiBw,2128
47
+ ragkit/store/__init__.py,sha256=6NGaRymLHWfcEpxJPAJ_lpYGEvo9UosSThdkO-SV2I8,828
48
+ ragkit/store/base.py,sha256=qNoesyESJD91tA_v3iIx9TwbNvY-miOTgs1_VLbohcY,1316
49
+ ragkit/store/memory.py,sha256=Yu1MQ5e4i9zYPXN7sg9LQYG6YE_WpevVLVUv4zNpAMI,2067
50
+ ragkit/store/qdrant_store.py,sha256=NBYZ86hmK15nCtG86ZkmtEUVY8WX8iKphh573IAgXiE,8326
51
+ ragkit/store/weaviate_store.py,sha256=6_vIZ5cUJnrvEI-IEYktmu7mZKHuKvsFpZWtFcGEZYI,10891
52
+ ragkit/utils/__init__.py,sha256=sAq5qe9CSHrb-OMiIayQM7DPFudM-hSvJTnAWeBoNFE,200
53
+ ragkit/utils/registry.py,sha256=FdNV9wo3VGsqeD4Qv2d2QK4XMEsr-mK_FSjBifvwiLI,1467
54
+ ragkit/utils/timeout.py,sha256=Nq33B4356L_6sI5rXzdrHcAhsOh4zN7SeQSX7HIlt9g,1807
55
+ modular_rag-0.1.0.dist-info/METADATA,sha256=vnsBGaisDEgAayIW71pmlWWLqqHeqv-uFc0jsv8mCpI,39562
56
+ modular_rag-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
57
+ modular_rag-0.1.0.dist-info/top_level.txt,sha256=k_6E8qLkT4SfWT8hMp8Oqj4tKp_vcPATvqtaHEoE7WA,7
58
+ modular_rag-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 ragkit contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ ragkit
ragkit/__init__.py ADDED
@@ -0,0 +1,53 @@
1
+ """ragkit — 可扩展、可替换抽象的 RAG 库。
2
+
3
+ from ragkit import RAGPipeline, PipelineConfig
4
+
5
+ pipe = RAGPipeline(PipelineConfig())
6
+ pipe.ingest("docs/guide.md")
7
+ results = pipe.retrieve("超时设置默认值是多少?")
8
+
9
+ 启用主表+附表(同业务 id 多版本自动取最高版本):
10
+
11
+ from ragkit import InMemoryCatalog, InMemoryChunkStore, FileRecord
12
+ pipe = RAGPipeline(catalog=InMemoryCatalog(), chunk_store=InMemoryChunkStore())
13
+ pipe.ingest(doc, record=FileRecord(business_id="guide", version="v2",
14
+ status="active", owner="alice", md5="..."))
15
+ """
16
+ from .callbacks import (CallbackManager, ErrorAlertCallbacks, ErrorInfo, Event,
17
+ JsonlAuditCallbacks, LoggingCallbacks, MetricsCallbacks,
18
+ ProgressBarCallbacks, ProgressReporter, SamplingCallbacks,
19
+ TimingCallbacks, TraceCollector)
20
+ from .catalog import (BaseCatalog, BaseChunkStore, ChunkRecord, FileRecord,
21
+ InMemoryCatalog, InMemoryChunkStore, StoreCatalog,
22
+ compare_version, make_version_prefilter)
23
+ from .core.config import (PipelineConfig, RerankConfig, RetrievalConfig,
24
+ SplitConfig)
25
+ from .core.embeddings import BaseEmbedder, HashEmbedder, OllamaEmbedder
26
+ from .core.filtering import make_prefilter
27
+ from .core.schema import Chunk, Document, ScoredChunk
28
+ from .middleware import (AuditMiddleware, BaseMiddleware, CacheMiddleware,
29
+ CircuitBreakerMiddleware, DedupMiddleware,
30
+ GuardMiddleware, LoggingMiddleware, MiddlewareChain,
31
+ MiddlewareError, QueryNormalizeMiddleware,
32
+ ResultFilterMiddleware, RetryMiddleware,
33
+ TopKClampMiddleware)
34
+ from .pipeline import RAGPipeline
35
+ from .ragkit_version import __version__
36
+ from .store import (InMemoryVectorStore, QdrantVectorStore, WeaviateVectorStore,
37
+ create_store)
38
+
39
+ __all__ = [
40
+ "RAGPipeline", "PipelineConfig", "SplitConfig", "RetrievalConfig", "RerankConfig",
41
+ "Document", "Chunk", "ScoredChunk", "BaseEmbedder", "HashEmbedder", "OllamaEmbedder",
42
+ "make_prefilter", "__version__",
43
+ "FileRecord", "ChunkRecord", "BaseCatalog", "InMemoryCatalog", "StoreCatalog",
44
+ "BaseChunkStore", "InMemoryChunkStore", "make_version_prefilter", "compare_version",
45
+ "InMemoryVectorStore", "QdrantVectorStore", "WeaviateVectorStore", "create_store",
46
+ "CallbackManager", "Event", "ErrorInfo", "LoggingCallbacks",
47
+ "TimingCallbacks", "TraceCollector", "ProgressReporter", "ProgressBarCallbacks",
48
+ "MetricsCallbacks", "JsonlAuditCallbacks", "ErrorAlertCallbacks", "SamplingCallbacks",
49
+ "BaseMiddleware", "MiddlewareChain", "MiddlewareError", "LoggingMiddleware",
50
+ "RetryMiddleware", "CacheMiddleware", "GuardMiddleware", "ResultFilterMiddleware",
51
+ "DedupMiddleware", "TopKClampMiddleware", "QueryNormalizeMiddleware",
52
+ "CircuitBreakerMiddleware", "AuditMiddleware",
53
+ ]
@@ -0,0 +1,20 @@
1
+ """回调机制:生命周期事件、错误上报、进度、内置调试与运维实现。
2
+
3
+ - 事件与错误:Event / ErrorInfo
4
+ - 注册分发:CallbackManager / as_manager
5
+ - 进度:ProgressReporter
6
+ - 内置实现:Logging / Timing / Trace / ProgressBar / Metrics / JsonlAudit / ErrorAlert / Sampling
7
+ """
8
+ from .builtin import (ErrorAlertCallbacks, JsonlAuditCallbacks, LoggingCallbacks,
9
+ MetricsCallbacks, ProgressBarCallbacks, SamplingCallbacks,
10
+ TimingCallbacks, TraceCollector)
11
+ from .events import ErrorInfo, Event
12
+ from .manager import CallbackManager, Handler, as_manager
13
+ from .progress import ProgressReporter
14
+
15
+ __all__ = [
16
+ "Event", "ErrorInfo", "CallbackManager", "Handler", "as_manager",
17
+ "ProgressReporter",
18
+ "LoggingCallbacks", "TimingCallbacks", "TraceCollector", "ProgressBarCallbacks",
19
+ "MetricsCallbacks", "JsonlAuditCallbacks", "ErrorAlertCallbacks", "SamplingCallbacks",
20
+ ]
@@ -0,0 +1,259 @@
1
+ """内置回调实现:日志 / 耗时 / 追踪 / 进度条 / 指标 / 审计 / 告警 / 采样。
2
+
3
+ 全部只需 `cb.on(event, 实例)` 即可挂载,也可直接作为 callbacks= 传入。
4
+ """
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ import logging
9
+ import random
10
+ import sys
11
+ import time
12
+ from typing import Any, Dict, List, Optional, TextIO
13
+
14
+ from .events import ErrorInfo, Event
15
+
16
+ _STAGE_LABEL = {"read": "读取", "split": "切分", "embed": "向量化", "store": "入库"}
17
+
18
+
19
+ def _safe(payload: Dict[str, Any]) -> Dict[str, Any]:
20
+ """只保留可 JSON 序列化的标量字段,避免日志/审计写崩。"""
21
+ skip = {"error_info", "documents", "chunks", "results", "detail"}
22
+ return {k: v for k, v in payload.items()
23
+ if k not in skip and isinstance(v, (str, int, float, bool, type(None), list, dict))}
24
+
25
+
26
+ class LoggingCallbacks:
27
+ """把事件打到标准 logging,调试阶段问题最直接的方式。"""
28
+
29
+ def __init__(self, logger: Optional[logging.Logger] = None,
30
+ level: int = logging.INFO, include: Optional[List[Event]] = None) -> None:
31
+ self.logger = logger or logging.getLogger("ragkit")
32
+ self.level = level
33
+ self.include = {e.value for e in include} if include else None
34
+
35
+ def __call__(self, payload: Dict[str, Any]) -> None:
36
+ event = payload.get("event", "")
37
+ if self.include is not None and event not in self.include:
38
+ return
39
+ if "error_info" in payload:
40
+ self.logger.log(self.level, "[ragkit] %s 失败: %s",
41
+ payload.get("stage", event), payload["error_info"].message)
42
+ return
43
+ extra = {k: v for k, v in payload.items()
44
+ if k in ("count", "top_k", "duration_ms", "source", "mode", "percent")}
45
+ self.logger.log(self.level, "[ragkit] %s %s", event, extra or "")
46
+
47
+
48
+ class TimingCallbacks:
49
+ """记录各阶段耗时,summary() 汇总用于定位瓶颈。"""
50
+
51
+ def __init__(self) -> None:
52
+ self.records: Dict[str, List[float]] = {}
53
+
54
+ def __call__(self, payload: Dict[str, Any]) -> None:
55
+ duration = payload.get("duration_ms")
56
+ if duration is not None:
57
+ self.records.setdefault(payload.get("event", ""), []).append(float(duration))
58
+
59
+ def summary(self) -> Dict[str, Dict[str, float]]:
60
+ out: Dict[str, Dict[str, float]] = {}
61
+ for event, values in self.records.items():
62
+ out[event] = {
63
+ "count": len(values),
64
+ "total_ms": round(sum(values), 2),
65
+ "avg_ms": round(sum(values) / len(values), 2),
66
+ "max_ms": round(max(values), 2),
67
+ }
68
+ return out
69
+
70
+
71
+ class TraceCollector:
72
+ """按顺序收集全部事件,dump() 打印完整调用链,便于复现问题。"""
73
+
74
+ def __init__(self, maxlen: int = 500) -> None:
75
+ self.events: List[Dict[str, Any]] = []
76
+ self.maxlen = maxlen
77
+
78
+ def __call__(self, payload: Dict[str, Any]) -> None:
79
+ self.events.append(payload)
80
+ if len(self.events) > self.maxlen:
81
+ self.events.pop(0)
82
+
83
+ def dump(self, only: Optional[List[Event]] = None) -> str:
84
+ keep = {e.value for e in only} if only else None
85
+ lines = []
86
+ for p in self.events:
87
+ if keep is not None and p.get("event") not in keep:
88
+ continue
89
+ detail = {k: v for k, v in p.items()
90
+ if k not in ("event", "ts") and isinstance(v, (str, int, float, bool))}
91
+ lines.append(f"{p.get('event')} {detail or ''}".rstrip())
92
+ return "\n".join(lines)
93
+
94
+ def attach_to(self, manager) -> "TraceCollector":
95
+ for event in Event:
96
+ manager.on(event, self)
97
+ return self
98
+
99
+
100
+ class ProgressBarCallbacks:
101
+ """终端进度条:挂在 PROGRESS 上即可原地刷新。
102
+
103
+ cb.on(Event.PROGRESS, ProgressBarCallbacks())
104
+
105
+ 非终端环境(如 CI 日志)可设 `enabled=False` 或传入自己的 stream。
106
+ """
107
+
108
+ def __init__(self, stream: Optional[TextIO] = None, width: int = 28,
109
+ enabled: bool = True) -> None:
110
+ self.stream = stream or sys.stdout
111
+ self.width = width
112
+ self.enabled = enabled
113
+ self._last_stage: Optional[str] = None
114
+
115
+ def __call__(self, payload: Dict[str, Any]) -> None:
116
+ if not self.enabled:
117
+ return
118
+ if payload.get("event") != Event.PROGRESS.value and "percent" not in payload:
119
+ return
120
+ stage = payload.get("stage", "")
121
+ if self._last_stage and self._last_stage != stage:
122
+ self.stream.write("\n") # 阶段切换换行
123
+ self._last_stage = stage
124
+
125
+ percent = float(payload.get("percent", 0.0))
126
+ filled = int(self.width * percent / 100)
127
+ bar = "█" * filled + "░" * (self.width - filled)
128
+ eta = payload.get("eta_ms")
129
+ eta_text = f" 剩余 {eta / 1000:.1f}s" if eta and eta >= 100 else ""
130
+ msg = f" {payload['message']}" if payload.get("message") else ""
131
+ self.stream.write(
132
+ f"\r[{_STAGE_LABEL.get(stage, stage)}] |{bar}| "
133
+ f"{payload.get('current', 0)}/{payload.get('total', 0)} "
134
+ f"{percent:5.1f}%{eta_text}{msg} ")
135
+ self.stream.flush()
136
+ if percent >= 100:
137
+ self.stream.write("\n")
138
+
139
+
140
+ class MetricsCallbacks:
141
+ """轻量指标收集,便于对接 Prometheus / 自建看板。
142
+
143
+ metrics = MetricsCallbacks()
144
+ cb.on(Event.ERROR, metrics) # 或 attach_to 全量
145
+ metrics.snapshot() # {"events": {...}, "errors": {...}, "durations": {...}}
146
+ """
147
+
148
+ def __init__(self) -> None:
149
+ self.counts: Dict[str, int] = {}
150
+ self.errors: Dict[str, int] = {}
151
+ self.durations: Dict[str, List[float]] = {}
152
+
153
+ def __call__(self, payload: Dict[str, Any]) -> None:
154
+ event = payload.get("event", "")
155
+ self.counts[event] = self.counts.get(event, 0) + 1
156
+ if "error_info" in payload:
157
+ stage = payload.get("stage", event)
158
+ self.errors[stage] = self.errors.get(stage, 0) + 1
159
+ d = payload.get("duration_ms")
160
+ if d is not None:
161
+ self.durations.setdefault(event, []).append(float(d))
162
+
163
+ def snapshot(self) -> Dict[str, Any]:
164
+ durations = {
165
+ e: {"count": len(v), "avg": round(sum(v) / len(v), 2), "max": round(max(v), 2)}
166
+ for e, v in self.durations.items()
167
+ }
168
+ return {"events": dict(self.counts), "errors": dict(self.errors),
169
+ "durations": durations}
170
+
171
+ def attach_to(self, manager) -> "MetricsCallbacks":
172
+ for event in Event:
173
+ manager.on(event, self)
174
+ return self
175
+
176
+
177
+ class JsonlAuditCallbacks:
178
+ """把事件按行写成 JSONL,用于审计与离线分析。
179
+
180
+ audit = JsonlAuditCallbacks("audit.jsonl")
181
+ audit.attach_to(cb)
182
+ """
183
+
184
+ def __init__(self, path: Optional[str] = None, stream: Optional[TextIO] = None,
185
+ flush_each: bool = True) -> None:
186
+ if stream is None and path is None:
187
+ raise ValueError("需要 path 或 stream 之一")
188
+ self.path = path
189
+ self.stream = stream or open(path, "a", encoding="utf-8") # noqa: SIM115
190
+ self.flush_each = flush_each
191
+
192
+ def __call__(self, payload: Dict[str, Any]) -> None:
193
+ record = _safe(payload)
194
+ if "error_info" in payload:
195
+ record["error"] = payload["error_info"].to_dict()
196
+ self.stream.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
197
+ if self.flush_each:
198
+ self.stream.flush()
199
+
200
+ def attach_to(self, manager) -> "JsonlAuditCallbacks":
201
+ for event in Event:
202
+ manager.on(event, self)
203
+ return self
204
+
205
+ def close(self) -> None:
206
+ try:
207
+ self.stream.close()
208
+ except Exception:
209
+ pass
210
+
211
+
212
+ class ErrorAlertCallbacks:
213
+ """错误告警:把 ErrorInfo 交给注入的 notifier(webhook / 邮件 / Sentry…)。
214
+
215
+ cb.on(Event.ERROR, ErrorAlertCallbacks(lambda info: send_webhook(info.to_dict())))
216
+ """
217
+
218
+ def __init__(self, notifier, *, cooldown: float = 60.0) -> None:
219
+ if not callable(notifier):
220
+ raise ValueError("notifier 必须是 callable(ErrorInfo)")
221
+ self.notifier = notifier
222
+ self.cooldown = cooldown
223
+ self._last: Dict[str, float] = {}
224
+ self.fired = 0
225
+
226
+ def __call__(self, payload: Dict[str, Any]) -> None:
227
+ info: Optional[ErrorInfo] = payload.get("error_info")
228
+ if info is None:
229
+ return
230
+ now = time.time()
231
+ if now - self._last.get(info.stage, 0.0) < self.cooldown:
232
+ return # 同一阶段冷却,避免告警风暴
233
+ self._last[info.stage] = now
234
+ self.fired += 1
235
+ try:
236
+ self.notifier(info)
237
+ except Exception:
238
+ pass # 告警失败不能影响主流程
239
+
240
+
241
+ class SamplingCallbacks:
242
+ """按概率采样转发,用于高频场景降噪(如只记录 10% 的检索事件)。"""
243
+
244
+ def __init__(self, target, rate: float = 0.1, seed: Optional[int] = None) -> None:
245
+ if not callable(target):
246
+ raise ValueError("target 必须是 callable")
247
+ if not 0.0 <= rate <= 1.0:
248
+ raise ValueError("rate 必须在 [0, 1]")
249
+ self.target = target
250
+ self.rate = rate
251
+ self._rng = random.Random(seed)
252
+ self.seen = 0
253
+ self.forwarded = 0
254
+
255
+ def __call__(self, payload: Dict[str, Any]) -> None:
256
+ self.seen += 1
257
+ if self._rng.random() < self.rate:
258
+ self.forwarded += 1
259
+ self.target(payload)
@@ -0,0 +1,58 @@
1
+ """事件枚举与结构化错误上下文。"""
2
+ from __future__ import annotations
3
+
4
+ import time
5
+ import traceback
6
+ from dataclasses import dataclass, field
7
+ from enum import Enum
8
+ from typing import Any, Dict
9
+
10
+
11
+ class Event(str, Enum):
12
+ """生命周期事件。值即事件名,可直接与字符串比较。"""
13
+
14
+ INGEST_START = "ingest:start"
15
+ PROGRESS = "ingest:progress" # 细粒度进度(防"看起来卡死")
16
+ READ_END = "read:end"
17
+ SPLIT_END = "split:end"
18
+ EMBED_END = "embed:end"
19
+ STORE_END = "store:end"
20
+ INGEST_END = "ingest:end"
21
+ INGEST_SKIPPED = "ingest:skipped" # 内容未变(MD5 相同)而跳过
22
+ INGEST_ERROR = "ingest:error"
23
+
24
+ RETRIEVE_START = "retrieve:start"
25
+ RECALL_END = "recall:end"
26
+ RERANK_END = "rerank:end"
27
+ RERANK_FALLBACK = "rerank:fallback"
28
+ RETRIEVE_END = "retrieve:end"
29
+ RETRIEVE_ERROR = "retrieve:error"
30
+
31
+ ERROR = "error" # 通用错误(所有阶段失败都会发)
32
+ CALLBACK_ERROR = "callback:error" # 回调自身出错(用于自检)
33
+
34
+
35
+ @dataclass
36
+ class ErrorInfo:
37
+ """结构化错误上下文,便于日志与排查。"""
38
+
39
+ stage: str
40
+ error: BaseException
41
+ payload: Dict[str, Any] = field(default_factory=dict)
42
+ timestamp: float = field(default_factory=time.time)
43
+
44
+ @property
45
+ def message(self) -> str:
46
+ return f"{type(self.error).__name__}: {self.error}"
47
+
48
+ def format(self) -> str:
49
+ tb = "".join(traceback.format_exception(
50
+ type(self.error), self.error, self.error.__traceback__))
51
+ return f"[{self.stage}] {self.message}\n{tb}"
52
+
53
+ def to_dict(self) -> dict:
54
+ return {"stage": self.stage, "message": self.message,
55
+ "error_type": type(self.error).__name__,
56
+ "payload": {k: v for k, v in self.payload.items()
57
+ if isinstance(v, (str, int, float, bool, type(None)))},
58
+ "timestamp": self.timestamp}
@@ -0,0 +1,89 @@
1
+ """回调注册与分发(错误隔离)。"""
2
+ from __future__ import annotations
3
+
4
+ import time
5
+ from typing import Any, Callable, Dict, List
6
+
7
+ from .events import ErrorInfo, Event
8
+
9
+ Handler = Callable[[Dict[str, Any]], None]
10
+
11
+
12
+ class CallbackManager:
13
+ """回调注册与分发。所有 handler 签名为 fn(payload: dict)。"""
14
+
15
+ def __init__(self, *, max_errors: int = 100) -> None:
16
+ self._handlers: Dict[str, List[tuple[int, Handler]]] = {}
17
+ self.callback_errors: List[Dict[str, Any]] = []
18
+ self._max_errors = max_errors
19
+
20
+ # ---- 注册 / 注销 ----
21
+ def on(self, event: Event | str, fn: Handler, *, priority: int = 0) -> "CallbackManager":
22
+ key = event.value if isinstance(event, Event) else str(event)
23
+ self._handlers.setdefault(key, []).append((priority, fn))
24
+ self._handlers[key].sort(key=lambda x: -x[0]) # 高优先级先执行
25
+ return self
26
+
27
+ def off(self, event: Event | str, fn: Handler) -> "CallbackManager":
28
+ key = event.value if isinstance(event, Event) else str(event)
29
+ self._handlers[key] = [(p, f) for p, f in self._handlers.get(key, []) if f is not fn]
30
+ return self
31
+
32
+ def clear(self, event: Event | str | None = None) -> "CallbackManager":
33
+ if event is None:
34
+ self._handlers.clear()
35
+ else:
36
+ key = event.value if isinstance(event, Event) else str(event)
37
+ self._handlers.pop(key, None)
38
+ return self
39
+
40
+ def has(self, event: Event | str) -> bool:
41
+ key = event.value if isinstance(event, Event) else str(event)
42
+ return bool(self._handlers.get(key))
43
+
44
+ # ---- 分发(错误隔离)----
45
+ def emit(self, event: Event | str, **payload: Any) -> None:
46
+ key = event.value if isinstance(event, Event) else str(event)
47
+ handlers = self._handlers.get(key)
48
+ if not handlers:
49
+ return
50
+ data = {"event": key, "ts": time.time(), **payload}
51
+ for _, fn in list(handlers):
52
+ try:
53
+ fn(data)
54
+ except Exception as exc: # 回调错误绝不影响主流程
55
+ self._record_callback_error(key, exc)
56
+
57
+ def _record_callback_error(self, event: str, exc: BaseException) -> None:
58
+ self.callback_errors.append({"event": event, "error": repr(exc),
59
+ "ts": time.time()})
60
+ if len(self.callback_errors) > self._max_errors:
61
+ self.callback_errors.pop(0)
62
+
63
+ def emit_error(self, stage: str, error: BaseException, **payload: Any) -> None:
64
+ """上报结构化错误:先发通用 ERROR,再发阶段专属错误事件。"""
65
+ info = ErrorInfo(stage=stage, error=error, payload=payload)
66
+ self.emit(Event.ERROR, stage=stage, error=error, error_info=info, **payload)
67
+ stage_event = {"ingest": Event.INGEST_ERROR,
68
+ "retrieve": Event.RETRIEVE_ERROR}.get(stage)
69
+ if stage_event is not None:
70
+ self.emit(stage_event, stage=stage, error=error, error_info=info, **payload)
71
+
72
+
73
+ def as_manager(callbacks: Any) -> CallbackManager:
74
+ """把 None / CallbackManager / dict{event: [fn]} / callable 统一成 CallbackManager。"""
75
+ if isinstance(callbacks, CallbackManager):
76
+ return callbacks
77
+ mgr = CallbackManager()
78
+ if callbacks is None:
79
+ return mgr
80
+ if isinstance(callbacks, dict):
81
+ for event, fns in callbacks.items():
82
+ for fn in (fns if isinstance(fns, (list, tuple)) else [fns]):
83
+ mgr.on(event, fn)
84
+ return mgr
85
+ if callable(callbacks):
86
+ for event in Event:
87
+ mgr.on(event, callbacks)
88
+ return mgr
89
+ raise TypeError("callbacks 需要 CallbackManager / dict / callable / None")
@@ -0,0 +1,57 @@
1
+ """带节流的进度上报器。"""
2
+ from __future__ import annotations
3
+
4
+ import time
5
+ from typing import Dict
6
+
7
+ from .events import Event
8
+ from .manager import CallbackManager
9
+
10
+
11
+ class ProgressReporter:
12
+ """专治"长耗时流程看起来卡死"。
13
+
14
+ - 默认 200ms 节流,避免高频 emit 拖慢主流程;
15
+ - 到 100% 时强制发一次,保证前端/终端能收尾;
16
+ - 未注册 PROGRESS 回调时直接短路,零额外开销;
17
+ - payload 含 current / total / percent / elapsed_ms / eta_ms,可直接驱动进度条。
18
+ """
19
+
20
+ def __init__(self, manager: CallbackManager, *, interval: float = 0.2,
21
+ event: Event | str = Event.PROGRESS) -> None:
22
+ self.manager = manager
23
+ self.interval = interval
24
+ self.event = event
25
+ self._start: Dict[str, float] = {}
26
+ self._last: Dict[str, float] = {}
27
+ self._total: Dict[str, int] = {}
28
+
29
+ def begin(self, stage: str, total: int = 0) -> None:
30
+ now = time.perf_counter()
31
+ self._start[stage] = now
32
+ self._last[stage] = 0.0
33
+ self._total[stage] = total
34
+ self.update(stage, 0, total, force=True)
35
+
36
+ def update(self, stage: str, current: int, total: int,
37
+ message: str = "", force: bool = False) -> None:
38
+ if not self.manager.has(self.event):
39
+ return
40
+ now = time.perf_counter()
41
+ done = total > 0 and current >= total
42
+ if not force and not done and now - self._last.get(stage, 0.0) < self.interval:
43
+ return # 节流:非完成态且未到间隔,跳过
44
+ self._last[stage] = now
45
+ self._total[stage] = total
46
+ start = self._start.setdefault(stage, now)
47
+ elapsed = (now - start) * 1000
48
+ percent = round(current / total * 100, 1) if total else 0.0
49
+ eta = round(elapsed / current * (total - current), 1) if current > 0 else None
50
+ self.manager.emit(self.event, stage=stage, current=current, total=total,
51
+ percent=percent, elapsed_ms=round(elapsed, 2),
52
+ eta_ms=eta, message=message)
53
+
54
+ def finish(self, stage: str, total: int | None = None, message: str = "") -> None:
55
+ if total is None:
56
+ total = self._total.get(stage, 0)
57
+ self.update(stage, total, total, message=message, force=True)
@@ -0,0 +1,16 @@
1
+ """catalog 包:主表 + 附表 + 版本择优 + 字段映射。"""
2
+ from .base import BaseCatalog, InMemoryCatalog
3
+ from .chunk_store import BaseChunkStore, InMemoryChunkStore
4
+ from .filtering import make_version_prefilter
5
+ from .models import ChunkRecord, FileRecord
6
+ from .scalar import (from_scalar_props, property_columns, property_types,
7
+ to_scalar_props)
8
+ from .store_catalog import StoreCatalog
9
+ from .version import compare_version, highest_version, version_key
10
+
11
+ __all__ = [
12
+ "FileRecord", "ChunkRecord", "BaseCatalog", "InMemoryCatalog", "StoreCatalog",
13
+ "BaseChunkStore", "InMemoryChunkStore",
14
+ "make_version_prefilter", "compare_version", "highest_version", "version_key",
15
+ "to_scalar_props", "from_scalar_props", "property_columns", "property_types",
16
+ ]