docpipe-maestro 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docpipe_maestro/__init__.py +153 -0
- docpipe_maestro/__main__.py +8 -0
- docpipe_maestro/_crossrefs.py +189 -0
- docpipe_maestro/_filters.py +635 -0
- docpipe_maestro/_pii.py +986 -0
- docpipe_maestro/_registry.py +124 -0
- docpipe_maestro/_resolve.py +44 -0
- docpipe_maestro/_stages.py +93 -0
- docpipe_maestro/cli.py +1032 -0
- docpipe_maestro/config.py +64 -0
- docpipe_maestro/embedder/__init__.py +71 -0
- docpipe_maestro/embedder/default.py +367 -0
- docpipe_maestro/eval/__init__.py +37 -0
- docpipe_maestro/eval/harness.py +415 -0
- docpipe_maestro/eval/metrics.py +95 -0
- docpipe_maestro/ingest/__init__.py +1 -0
- docpipe_maestro/ingest/chunker/__init__.py +41 -0
- docpipe_maestro/ingest/chunker/default.py +277 -0
- docpipe_maestro/ingest/clean/__init__.py +41 -0
- docpipe_maestro/ingest/clean/default.py +239 -0
- docpipe_maestro/ingest/crossrefs.py +19 -0
- docpipe_maestro/ingest/enricher/__init__.py +46 -0
- docpipe_maestro/ingest/enricher/crossrefs.py +103 -0
- docpipe_maestro/ingest/enricher/pii.py +154 -0
- docpipe_maestro/ingest/intake/__init__.py +70 -0
- docpipe_maestro/ingest/intake/_common.py +194 -0
- docpipe_maestro/ingest/intake/docx.py +96 -0
- docpipe_maestro/ingest/intake/html.py +194 -0
- docpipe_maestro/ingest/intake/markdown.py +31 -0
- docpipe_maestro/ingest/intake/pdf.py +64 -0
- docpipe_maestro/ingest/intake/text.py +31 -0
- docpipe_maestro/ingest/pipeline.py +1178 -0
- docpipe_maestro/ingest/stages.py +66 -0
- docpipe_maestro/log.py +215 -0
- docpipe_maestro/manifest.py +193 -0
- docpipe_maestro/py.typed +0 -0
- docpipe_maestro/query/__init__.py +1 -0
- docpipe_maestro/query/answer.py +403 -0
- docpipe_maestro/query/egress.py +153 -0
- docpipe_maestro/query/prompts.py +94 -0
- docpipe_maestro/query/reranker/__init__.py +66 -0
- docpipe_maestro/query/reranker/cross_encoder.py +120 -0
- docpipe_maestro/query/retriever.py +335 -0
- docpipe_maestro/query/searcher/__init__.py +44 -0
- docpipe_maestro/query/searcher/dense.py +68 -0
- docpipe_maestro/query/searcher/hybrid.py +123 -0
- docpipe_maestro/query/stages.py +44 -0
- docpipe_maestro/store/__init__.py +43 -0
- docpipe_maestro/store/chroma.py +302 -0
- docpipe_maestro/store/lexical.py +323 -0
- docpipe_maestro/types.py +558 -0
- docpipe_maestro-0.2.0.dist-info/METADATA +843 -0
- docpipe_maestro-0.2.0.dist-info/RECORD +56 -0
- docpipe_maestro-0.2.0.dist-info/WHEEL +4 -0
- docpipe_maestro-0.2.0.dist-info/entry_points.txt +2 -0
- docpipe_maestro-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""docpipe-maestro: a minimal, provider-agnostic, structure-aware RAG pipeline.
|
|
2
|
+
|
|
3
|
+
Public surface organized around two separated execution phases:
|
|
4
|
+
|
|
5
|
+
- Phase 1 (Offline Ingestion & Corpus Lifecycle): ``ingest``, ``sync``, ``remove``,
|
|
6
|
+
``status``, and ``reindex``.
|
|
7
|
+
- Phase 2 (Online Read-Only Retrieval & Optional Generation): ``retrieve``,
|
|
8
|
+
``build_prompt``, and lazily loaded ``answer``.
|
|
9
|
+
- Shared Data Shapes & Exceptions: ``Chunk``, ``EmbeddedChunk``, ``RetrievedChunk``,
|
|
10
|
+
``SourceRecord``, ``IngestResult``, ``SyncResult``, ``CollectionStatus``, and
|
|
11
|
+
``DocpipeError`` subclasses.
|
|
12
|
+
|
|
13
|
+
Static chunking profiles (``DEFAULT``, ``POLICY``) live in ``docpipe_maestro.config``.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import importlib
|
|
19
|
+
import sys
|
|
20
|
+
import types as _stdlib_types
|
|
21
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
22
|
+
from typing import TYPE_CHECKING, Any
|
|
23
|
+
|
|
24
|
+
from docpipe_maestro.query.prompts import build_chat_messages, build_prompt
|
|
25
|
+
from docpipe_maestro.types import (
|
|
26
|
+
Chunk,
|
|
27
|
+
CollectionStatus,
|
|
28
|
+
DocpipeError,
|
|
29
|
+
EmbeddedChunk,
|
|
30
|
+
GenerationResult,
|
|
31
|
+
IngestResult,
|
|
32
|
+
MissingCredentialError,
|
|
33
|
+
MissingExtraError,
|
|
34
|
+
ModelMismatchError,
|
|
35
|
+
PiiBlockedError,
|
|
36
|
+
PiiPolicy,
|
|
37
|
+
RegistryError,
|
|
38
|
+
RetrievedChunk,
|
|
39
|
+
SourceRecord,
|
|
40
|
+
SyncResult,
|
|
41
|
+
UnsupportedFormatError,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
if TYPE_CHECKING:
|
|
45
|
+
from docpipe_maestro._pii import register_pii_recognizer
|
|
46
|
+
from docpipe_maestro.embedder import register_embedder
|
|
47
|
+
from docpipe_maestro.ingest.chunker import register_chunker
|
|
48
|
+
from docpipe_maestro.ingest.clean import register_cleaner
|
|
49
|
+
from docpipe_maestro.ingest.enricher import register_enricher
|
|
50
|
+
from docpipe_maestro.ingest.intake import register_parser
|
|
51
|
+
from docpipe_maestro.ingest.pipeline import ingest, reindex, remove, status, sync
|
|
52
|
+
from docpipe_maestro.query.answer import answer, answer_detailed
|
|
53
|
+
from docpipe_maestro.query.reranker import register_reranker
|
|
54
|
+
from docpipe_maestro.query.retriever import retrieve
|
|
55
|
+
from docpipe_maestro.query.searcher import register_searcher
|
|
56
|
+
from docpipe_maestro.store import register_store
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
__version__ = version("docpipe-maestro")
|
|
60
|
+
except PackageNotFoundError:
|
|
61
|
+
__version__ = "0.2.0.dev0"
|
|
62
|
+
|
|
63
|
+
__all__ = [
|
|
64
|
+
"Chunk",
|
|
65
|
+
"CollectionStatus",
|
|
66
|
+
"DocpipeError",
|
|
67
|
+
"EmbeddedChunk",
|
|
68
|
+
"GenerationResult",
|
|
69
|
+
"IngestResult",
|
|
70
|
+
"MissingCredentialError",
|
|
71
|
+
"MissingExtraError",
|
|
72
|
+
"ModelMismatchError",
|
|
73
|
+
"PiiBlockedError",
|
|
74
|
+
"PiiPolicy",
|
|
75
|
+
"RegistryError",
|
|
76
|
+
"RetrievedChunk",
|
|
77
|
+
"SourceRecord",
|
|
78
|
+
"SyncResult",
|
|
79
|
+
"UnsupportedFormatError",
|
|
80
|
+
"__version__",
|
|
81
|
+
"answer",
|
|
82
|
+
"answer_detailed",
|
|
83
|
+
"build_chat_messages",
|
|
84
|
+
"build_prompt",
|
|
85
|
+
"ingest",
|
|
86
|
+
"register_chunker",
|
|
87
|
+
"register_cleaner",
|
|
88
|
+
"register_embedder",
|
|
89
|
+
"register_enricher",
|
|
90
|
+
"register_parser",
|
|
91
|
+
"register_pii_recognizer",
|
|
92
|
+
"register_reranker",
|
|
93
|
+
"register_searcher",
|
|
94
|
+
"register_store",
|
|
95
|
+
"reindex",
|
|
96
|
+
"remove",
|
|
97
|
+
"retrieve",
|
|
98
|
+
"status",
|
|
99
|
+
"sync",
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
_PIPELINE_EXPORTS = frozenset({"ingest", "reindex", "remove", "status", "sync"})
|
|
103
|
+
_REGISTER_EXPORTS: dict[str, tuple[str, str]] = {
|
|
104
|
+
"register_parser": ("docpipe_maestro.ingest.intake", "register_parser"),
|
|
105
|
+
"register_cleaner": ("docpipe_maestro.ingest.clean", "register_cleaner"),
|
|
106
|
+
"register_enricher": ("docpipe_maestro.ingest.enricher", "register_enricher"),
|
|
107
|
+
"register_chunker": ("docpipe_maestro.ingest.chunker", "register_chunker"),
|
|
108
|
+
"register_embedder": ("docpipe_maestro.embedder", "register_embedder"),
|
|
109
|
+
"register_store": ("docpipe_maestro.store", "register_store"),
|
|
110
|
+
"register_searcher": ("docpipe_maestro.query.searcher", "register_searcher"),
|
|
111
|
+
"register_reranker": ("docpipe_maestro.query.reranker", "register_reranker"),
|
|
112
|
+
"register_pii_recognizer": ("docpipe_maestro._pii", "register_pii_recognizer"),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def __getattr__(name: str) -> Any:
|
|
117
|
+
"""Lazily expose heavy vector/LLM pipeline helpers and stage registration entry points."""
|
|
118
|
+
if name in _REGISTER_EXPORTS:
|
|
119
|
+
mod_path, attr = _REGISTER_EXPORTS[name]
|
|
120
|
+
obj = getattr(importlib.import_module(mod_path), attr)
|
|
121
|
+
globals()[name] = obj
|
|
122
|
+
return obj
|
|
123
|
+
if name in {"answer", "answer_detailed"}:
|
|
124
|
+
from docpipe_maestro.query import answer as _answer_mod
|
|
125
|
+
|
|
126
|
+
obj = getattr(_answer_mod, name)
|
|
127
|
+
globals()[name] = obj
|
|
128
|
+
return obj
|
|
129
|
+
if name in _PIPELINE_EXPORTS:
|
|
130
|
+
from docpipe_maestro.ingest import pipeline as _pipeline_mod
|
|
131
|
+
|
|
132
|
+
obj = getattr(_pipeline_mod, name)
|
|
133
|
+
globals()[name] = obj
|
|
134
|
+
return obj
|
|
135
|
+
if name == "retrieve":
|
|
136
|
+
from docpipe_maestro.query.retriever import retrieve as _retrieve
|
|
137
|
+
|
|
138
|
+
globals()[name] = _retrieve
|
|
139
|
+
return _retrieve
|
|
140
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class _DocpipeModule(_stdlib_types.ModuleType):
|
|
144
|
+
"""Ensure lazy function exports (``ingest``) are never shadowed by subpackages."""
|
|
145
|
+
|
|
146
|
+
def __getattribute__(self, name: str) -> Any:
|
|
147
|
+
val = super().__getattribute__(name)
|
|
148
|
+
if name == "ingest" and isinstance(val, _stdlib_types.ModuleType):
|
|
149
|
+
return __getattr__(name)
|
|
150
|
+
return val
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
sys.modules[__name__].__class__ = _DocpipeModule
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Phase-neutral cross-reference matching, target indexing, and bounded excerpt utilities."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
|
|
8
|
+
CROSSREF_RE = re.compile(
|
|
9
|
+
r"\b(?:(Section|Article|Clause)\s+([0-9]+(?:\.[0-9]+)*))|"
|
|
10
|
+
r"(§\s*([0-9]+(?:\.[0-9]+)*))",
|
|
11
|
+
re.IGNORECASE,
|
|
12
|
+
)
|
|
13
|
+
_DOC_SCOPED_CROSSREF_RE = re.compile(
|
|
14
|
+
r"(?:\b(Section|Article|Clause)\s+([0-9]+(?:\.[0-9]+)*)|(§)\s*([0-9]+(?:\.[0-9]+)*))"
|
|
15
|
+
r"\s+(?:of|in)\s+(?:the\s+)?"
|
|
16
|
+
r"([A-Za-z0-9_.-]+\.(?:md|markdown|txt|pdf|docx|html)\b|"
|
|
17
|
+
r"[A-Z][A-Za-z0-9._-]*"
|
|
18
|
+
r"(?:\s+(?!\b(?:Section|Article|Clause)\s+[0-9])"
|
|
19
|
+
r"(?:(?:of|and|&|for)\s+(?!\b(?:Section|Article|Clause)\s+[0-9]))?[A-Z][A-Za-z0-9._-]*){0,5})",
|
|
20
|
+
)
|
|
21
|
+
_TOKEN_RE = re.compile(r"[a-z0-9]+")
|
|
22
|
+
_DOC_PHRASE_STOPWORDS = frozenset({"the", "of", "and", "for", "a", "an", "in", "under"})
|
|
23
|
+
|
|
24
|
+
_SECTION_NUM_RE = re.compile(
|
|
25
|
+
r"(?:\b(?:Section|Article|Clause)\s+([0-9]+(?:\.[0-9]+)*)\b"
|
|
26
|
+
r"|^([0-9]+(?:\.[0-9]+)+)(?=\.?(?:\s|$|[:\-—]))"
|
|
27
|
+
r"|^([0-9]+)\.(?=\s|$))",
|
|
28
|
+
re.IGNORECASE,
|
|
29
|
+
)
|
|
30
|
+
_CLAUSE_START_LINE_RE = re.compile(r"^([0-9]+(?:\.[0-9]+)+)\s+(.+)$")
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"CROSSREF_RE",
|
|
34
|
+
"extract_doc_crossrefs",
|
|
35
|
+
"extract_section_num",
|
|
36
|
+
"index_section_targets",
|
|
37
|
+
"iter_clause_blocks",
|
|
38
|
+
"match_source_key",
|
|
39
|
+
"parse_crossref_match",
|
|
40
|
+
"truncate_excerpt",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def extract_section_num(label: str) -> str | None:
|
|
45
|
+
"""Extract normalized section identifier from a heading label, if present.
|
|
46
|
+
|
|
47
|
+
Prefixed forms (``Section 7``, ``Article 4``) and dotted decimal headings
|
|
48
|
+
(``7.5 Public API``, ``20.2: Rules``) match directly, whereas bare integer
|
|
49
|
+
headings require a trailing period (``1. Title``) to avoid false-positive
|
|
50
|
+
matches on years or measurements (``2026 Roadmap``, ``3.5mm``).
|
|
51
|
+
"""
|
|
52
|
+
match = _SECTION_NUM_RE.search(label)
|
|
53
|
+
if not match:
|
|
54
|
+
return None
|
|
55
|
+
return next((group for group in match.groups() if group is not None), None)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def truncate_excerpt(text: str, max_chars: int = 400) -> str:
|
|
59
|
+
"""Truncate text cleanly at a word boundary without chopping words mid-token."""
|
|
60
|
+
clean = text.strip()
|
|
61
|
+
if len(clean) <= max_chars:
|
|
62
|
+
return clean
|
|
63
|
+
cutoff = clean.rfind(" ", 0, max_chars)
|
|
64
|
+
if cutoff <= 0:
|
|
65
|
+
cutoff = max_chars
|
|
66
|
+
return clean[:cutoff].rstrip(" ,;:") + "..."
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def parse_crossref_match(match: re.Match[str]) -> tuple[list[str], str, str]:
|
|
70
|
+
"""Extract normalized lookup keys, numeric target, and display label from a match."""
|
|
71
|
+
if match.group(1) and match.group(2):
|
|
72
|
+
kind = match.group(1).lower()
|
|
73
|
+
num = match.group(2)
|
|
74
|
+
return [f"{kind} {num}", num], num, f"{match.group(1)} {num}"
|
|
75
|
+
num = match.group(4)
|
|
76
|
+
return [num, f"clause {num}", f"section {num}"], num, f"§{num}"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _strip_crossref_annotation_lines(text: str) -> list[str]:
|
|
80
|
+
"""Return lines with single- or multi-line ``[Crossref ...]`` blocks blanked out."""
|
|
81
|
+
lines: list[str] = []
|
|
82
|
+
in_crossref_block = False
|
|
83
|
+
bracket_depth = 0
|
|
84
|
+
for raw_line in text.splitlines():
|
|
85
|
+
stripped = raw_line.strip()
|
|
86
|
+
if not in_crossref_block:
|
|
87
|
+
if stripped.startswith("[Crossref "):
|
|
88
|
+
bracket_depth = stripped.count("[") - stripped.count("]")
|
|
89
|
+
in_crossref_block = bracket_depth > 0
|
|
90
|
+
lines.append("")
|
|
91
|
+
else:
|
|
92
|
+
lines.append(raw_line)
|
|
93
|
+
else:
|
|
94
|
+
bracket_depth += stripped.count("[") - stripped.count("]")
|
|
95
|
+
if bracket_depth <= 0:
|
|
96
|
+
in_crossref_block = False
|
|
97
|
+
lines.append("")
|
|
98
|
+
return lines
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def iter_clause_blocks(raw_text: str) -> list[tuple[str, str]]:
|
|
102
|
+
"""Extract numbered clause numbers paired with their full wrapped paragraph text."""
|
|
103
|
+
blocks: list[tuple[str, str]] = []
|
|
104
|
+
cur_num: str | None = None
|
|
105
|
+
cur_lines: list[str] = []
|
|
106
|
+
|
|
107
|
+
def flush() -> None:
|
|
108
|
+
if cur_num is not None and cur_lines:
|
|
109
|
+
blocks.append((cur_num, " ".join(cur_lines)))
|
|
110
|
+
cur_lines.clear()
|
|
111
|
+
|
|
112
|
+
for raw_line in _strip_crossref_annotation_lines(raw_text):
|
|
113
|
+
line = raw_line.strip()
|
|
114
|
+
if not line:
|
|
115
|
+
flush()
|
|
116
|
+
cur_num = None
|
|
117
|
+
continue
|
|
118
|
+
match = _CLAUSE_START_LINE_RE.match(line)
|
|
119
|
+
if match:
|
|
120
|
+
flush()
|
|
121
|
+
cur_num = match.group(1)
|
|
122
|
+
cur_lines.append(line)
|
|
123
|
+
elif cur_num is not None:
|
|
124
|
+
cur_lines.append(line)
|
|
125
|
+
|
|
126
|
+
flush()
|
|
127
|
+
return blocks
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def index_section_targets(sections: Sequence[tuple[str, str]]) -> dict[str, tuple[str, int]]:
|
|
131
|
+
"""Index normalized section and clause numbers to ``(excerpt_text, section_idx)``."""
|
|
132
|
+
targets: dict[str, tuple[str, int]] = {}
|
|
133
|
+
for sec_idx, (section_label, raw_text) in enumerate(sections):
|
|
134
|
+
num = extract_section_num(section_label)
|
|
135
|
+
if num is not None:
|
|
136
|
+
full = f"{section_label}: {raw_text}"
|
|
137
|
+
targets.setdefault(num, (full, sec_idx))
|
|
138
|
+
targets.setdefault(f"section {num}".lower(), (full, sec_idx))
|
|
139
|
+
for c_num, c_text in iter_clause_blocks(raw_text):
|
|
140
|
+
targets.setdefault(c_num, (c_text, sec_idx))
|
|
141
|
+
targets.setdefault(f"clause {c_num}".lower(), (c_text, sec_idx))
|
|
142
|
+
return targets
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def extract_doc_crossrefs(text: str) -> list[tuple[str, list[str], str]]:
|
|
146
|
+
"""Extract explicit document-scoped cross-references from non-annotation lines."""
|
|
147
|
+
cleaned_body = "\n".join(_strip_crossref_annotation_lines(text))
|
|
148
|
+
refs: list[tuple[str, list[str], str]] = []
|
|
149
|
+
seen: set[tuple[str, str]] = set()
|
|
150
|
+
for match in _DOC_SCOPED_CROSSREF_RE.finditer(cleaned_body):
|
|
151
|
+
lookup_keys, _num, display = parse_crossref_match(match)
|
|
152
|
+
doc_phrase = match.group(5).strip().rstrip(".,;:")
|
|
153
|
+
dedup_key = (display.lower(), doc_phrase.lower())
|
|
154
|
+
if dedup_key in seen:
|
|
155
|
+
continue
|
|
156
|
+
seen.add(dedup_key)
|
|
157
|
+
refs.append((display, lookup_keys, doc_phrase))
|
|
158
|
+
return refs
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def match_source_key(
|
|
162
|
+
doc_phrase: str,
|
|
163
|
+
available_sources: Sequence[str],
|
|
164
|
+
*,
|
|
165
|
+
current_source: str | None = None,
|
|
166
|
+
) -> str | None:
|
|
167
|
+
"""Resolve a cited document name/filename unambiguously against collection sources."""
|
|
168
|
+
phrase_lower = doc_phrase.strip().lower()
|
|
169
|
+
candidates = [src for src in available_sources if src != current_source]
|
|
170
|
+
if not candidates or not phrase_lower:
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
for src in candidates:
|
|
174
|
+
stem = src.rsplit(".", 1)[0].lower()
|
|
175
|
+
if phrase_lower in {src.lower(), stem}:
|
|
176
|
+
return src
|
|
177
|
+
|
|
178
|
+
phrase_tokens = {
|
|
179
|
+
tok for tok in _TOKEN_RE.findall(phrase_lower) if tok not in _DOC_PHRASE_STOPWORDS
|
|
180
|
+
}
|
|
181
|
+
if len(phrase_tokens) < 2:
|
|
182
|
+
return None
|
|
183
|
+
|
|
184
|
+
matched: list[str] = []
|
|
185
|
+
for src in candidates:
|
|
186
|
+
src_tokens = set(_TOKEN_RE.findall(src.lower()))
|
|
187
|
+
if phrase_tokens.issubset(src_tokens):
|
|
188
|
+
matched.append(src)
|
|
189
|
+
return matched[0] if len(matched) == 1 else None
|