docpipe-maestro 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. docpipe_maestro/__init__.py +153 -0
  2. docpipe_maestro/__main__.py +8 -0
  3. docpipe_maestro/_crossrefs.py +189 -0
  4. docpipe_maestro/_filters.py +635 -0
  5. docpipe_maestro/_pii.py +986 -0
  6. docpipe_maestro/_registry.py +124 -0
  7. docpipe_maestro/_resolve.py +44 -0
  8. docpipe_maestro/_stages.py +93 -0
  9. docpipe_maestro/cli.py +1032 -0
  10. docpipe_maestro/config.py +64 -0
  11. docpipe_maestro/embedder/__init__.py +71 -0
  12. docpipe_maestro/embedder/default.py +367 -0
  13. docpipe_maestro/eval/__init__.py +37 -0
  14. docpipe_maestro/eval/harness.py +415 -0
  15. docpipe_maestro/eval/metrics.py +95 -0
  16. docpipe_maestro/ingest/__init__.py +1 -0
  17. docpipe_maestro/ingest/chunker/__init__.py +41 -0
  18. docpipe_maestro/ingest/chunker/default.py +277 -0
  19. docpipe_maestro/ingest/clean/__init__.py +41 -0
  20. docpipe_maestro/ingest/clean/default.py +239 -0
  21. docpipe_maestro/ingest/crossrefs.py +19 -0
  22. docpipe_maestro/ingest/enricher/__init__.py +46 -0
  23. docpipe_maestro/ingest/enricher/crossrefs.py +103 -0
  24. docpipe_maestro/ingest/enricher/pii.py +154 -0
  25. docpipe_maestro/ingest/intake/__init__.py +70 -0
  26. docpipe_maestro/ingest/intake/_common.py +194 -0
  27. docpipe_maestro/ingest/intake/docx.py +96 -0
  28. docpipe_maestro/ingest/intake/html.py +194 -0
  29. docpipe_maestro/ingest/intake/markdown.py +31 -0
  30. docpipe_maestro/ingest/intake/pdf.py +64 -0
  31. docpipe_maestro/ingest/intake/text.py +31 -0
  32. docpipe_maestro/ingest/pipeline.py +1178 -0
  33. docpipe_maestro/ingest/stages.py +66 -0
  34. docpipe_maestro/log.py +215 -0
  35. docpipe_maestro/manifest.py +193 -0
  36. docpipe_maestro/py.typed +0 -0
  37. docpipe_maestro/query/__init__.py +1 -0
  38. docpipe_maestro/query/answer.py +403 -0
  39. docpipe_maestro/query/egress.py +153 -0
  40. docpipe_maestro/query/prompts.py +94 -0
  41. docpipe_maestro/query/reranker/__init__.py +66 -0
  42. docpipe_maestro/query/reranker/cross_encoder.py +120 -0
  43. docpipe_maestro/query/retriever.py +335 -0
  44. docpipe_maestro/query/searcher/__init__.py +44 -0
  45. docpipe_maestro/query/searcher/dense.py +68 -0
  46. docpipe_maestro/query/searcher/hybrid.py +123 -0
  47. docpipe_maestro/query/stages.py +44 -0
  48. docpipe_maestro/store/__init__.py +43 -0
  49. docpipe_maestro/store/chroma.py +302 -0
  50. docpipe_maestro/store/lexical.py +323 -0
  51. docpipe_maestro/types.py +558 -0
  52. docpipe_maestro-0.2.0.dist-info/METADATA +843 -0
  53. docpipe_maestro-0.2.0.dist-info/RECORD +56 -0
  54. docpipe_maestro-0.2.0.dist-info/WHEEL +4 -0
  55. docpipe_maestro-0.2.0.dist-info/entry_points.txt +2 -0
  56. docpipe_maestro-0.2.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,153 @@
1
+ """docpipe-maestro: a minimal, provider-agnostic, structure-aware RAG pipeline.
2
+
3
+ Public surface organized around two separated execution phases:
4
+
5
+ - Phase 1 (Offline Ingestion & Corpus Lifecycle): ``ingest``, ``sync``, ``remove``,
6
+ ``status``, and ``reindex``.
7
+ - Phase 2 (Online Read-Only Retrieval & Optional Generation): ``retrieve``,
8
+ ``build_prompt``, and lazily loaded ``answer``.
9
+ - Shared Data Shapes & Exceptions: ``Chunk``, ``EmbeddedChunk``, ``RetrievedChunk``,
10
+ ``SourceRecord``, ``IngestResult``, ``SyncResult``, ``CollectionStatus``, and
11
+ ``DocpipeError`` subclasses.
12
+
13
+ Static chunking profiles (``DEFAULT``, ``POLICY``) live in ``docpipe_maestro.config``.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import importlib
19
+ import sys
20
+ import types as _stdlib_types
21
+ from importlib.metadata import PackageNotFoundError, version
22
+ from typing import TYPE_CHECKING, Any
23
+
24
+ from docpipe_maestro.query.prompts import build_chat_messages, build_prompt
25
+ from docpipe_maestro.types import (
26
+ Chunk,
27
+ CollectionStatus,
28
+ DocpipeError,
29
+ EmbeddedChunk,
30
+ GenerationResult,
31
+ IngestResult,
32
+ MissingCredentialError,
33
+ MissingExtraError,
34
+ ModelMismatchError,
35
+ PiiBlockedError,
36
+ PiiPolicy,
37
+ RegistryError,
38
+ RetrievedChunk,
39
+ SourceRecord,
40
+ SyncResult,
41
+ UnsupportedFormatError,
42
+ )
43
+
44
+ if TYPE_CHECKING:
45
+ from docpipe_maestro._pii import register_pii_recognizer
46
+ from docpipe_maestro.embedder import register_embedder
47
+ from docpipe_maestro.ingest.chunker import register_chunker
48
+ from docpipe_maestro.ingest.clean import register_cleaner
49
+ from docpipe_maestro.ingest.enricher import register_enricher
50
+ from docpipe_maestro.ingest.intake import register_parser
51
+ from docpipe_maestro.ingest.pipeline import ingest, reindex, remove, status, sync
52
+ from docpipe_maestro.query.answer import answer, answer_detailed
53
+ from docpipe_maestro.query.reranker import register_reranker
54
+ from docpipe_maestro.query.retriever import retrieve
55
+ from docpipe_maestro.query.searcher import register_searcher
56
+ from docpipe_maestro.store import register_store
57
+
58
+ try:
59
+ __version__ = version("docpipe-maestro")
60
+ except PackageNotFoundError:
61
+ __version__ = "0.2.0.dev0"
62
+
63
+ __all__ = [
64
+ "Chunk",
65
+ "CollectionStatus",
66
+ "DocpipeError",
67
+ "EmbeddedChunk",
68
+ "GenerationResult",
69
+ "IngestResult",
70
+ "MissingCredentialError",
71
+ "MissingExtraError",
72
+ "ModelMismatchError",
73
+ "PiiBlockedError",
74
+ "PiiPolicy",
75
+ "RegistryError",
76
+ "RetrievedChunk",
77
+ "SourceRecord",
78
+ "SyncResult",
79
+ "UnsupportedFormatError",
80
+ "__version__",
81
+ "answer",
82
+ "answer_detailed",
83
+ "build_chat_messages",
84
+ "build_prompt",
85
+ "ingest",
86
+ "register_chunker",
87
+ "register_cleaner",
88
+ "register_embedder",
89
+ "register_enricher",
90
+ "register_parser",
91
+ "register_pii_recognizer",
92
+ "register_reranker",
93
+ "register_searcher",
94
+ "register_store",
95
+ "reindex",
96
+ "remove",
97
+ "retrieve",
98
+ "status",
99
+ "sync",
100
+ ]
101
+
102
+ _PIPELINE_EXPORTS = frozenset({"ingest", "reindex", "remove", "status", "sync"})
103
+ _REGISTER_EXPORTS: dict[str, tuple[str, str]] = {
104
+ "register_parser": ("docpipe_maestro.ingest.intake", "register_parser"),
105
+ "register_cleaner": ("docpipe_maestro.ingest.clean", "register_cleaner"),
106
+ "register_enricher": ("docpipe_maestro.ingest.enricher", "register_enricher"),
107
+ "register_chunker": ("docpipe_maestro.ingest.chunker", "register_chunker"),
108
+ "register_embedder": ("docpipe_maestro.embedder", "register_embedder"),
109
+ "register_store": ("docpipe_maestro.store", "register_store"),
110
+ "register_searcher": ("docpipe_maestro.query.searcher", "register_searcher"),
111
+ "register_reranker": ("docpipe_maestro.query.reranker", "register_reranker"),
112
+ "register_pii_recognizer": ("docpipe_maestro._pii", "register_pii_recognizer"),
113
+ }
114
+
115
+
116
+ def __getattr__(name: str) -> Any:
117
+ """Lazily expose heavy vector/LLM pipeline helpers and stage registration entry points."""
118
+ if name in _REGISTER_EXPORTS:
119
+ mod_path, attr = _REGISTER_EXPORTS[name]
120
+ obj = getattr(importlib.import_module(mod_path), attr)
121
+ globals()[name] = obj
122
+ return obj
123
+ if name in {"answer", "answer_detailed"}:
124
+ from docpipe_maestro.query import answer as _answer_mod
125
+
126
+ obj = getattr(_answer_mod, name)
127
+ globals()[name] = obj
128
+ return obj
129
+ if name in _PIPELINE_EXPORTS:
130
+ from docpipe_maestro.ingest import pipeline as _pipeline_mod
131
+
132
+ obj = getattr(_pipeline_mod, name)
133
+ globals()[name] = obj
134
+ return obj
135
+ if name == "retrieve":
136
+ from docpipe_maestro.query.retriever import retrieve as _retrieve
137
+
138
+ globals()[name] = _retrieve
139
+ return _retrieve
140
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
141
+
142
+
143
+ class _DocpipeModule(_stdlib_types.ModuleType):
144
+ """Ensure lazy function exports (``ingest``) are never shadowed by subpackages."""
145
+
146
+ def __getattribute__(self, name: str) -> Any:
147
+ val = super().__getattribute__(name)
148
+ if name == "ingest" and isinstance(val, _stdlib_types.ModuleType):
149
+ return __getattr__(name)
150
+ return val
151
+
152
+
153
+ sys.modules[__name__].__class__ = _DocpipeModule
@@ -0,0 +1,8 @@
1
+ """Module entry point enabling ``python -m docpipe_maestro``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from docpipe_maestro.cli import main
6
+
7
+ if __name__ == "__main__": # pragma: no cover
8
+ raise SystemExit(main())
@@ -0,0 +1,189 @@
1
+ """Phase-neutral cross-reference matching, target indexing, and bounded excerpt utilities."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections.abc import Sequence
7
+
8
+ CROSSREF_RE = re.compile(
9
+ r"\b(?:(Section|Article|Clause)\s+([0-9]+(?:\.[0-9]+)*))|"
10
+ r"(§\s*([0-9]+(?:\.[0-9]+)*))",
11
+ re.IGNORECASE,
12
+ )
13
+ _DOC_SCOPED_CROSSREF_RE = re.compile(
14
+ r"(?:\b(Section|Article|Clause)\s+([0-9]+(?:\.[0-9]+)*)|(§)\s*([0-9]+(?:\.[0-9]+)*))"
15
+ r"\s+(?:of|in)\s+(?:the\s+)?"
16
+ r"([A-Za-z0-9_.-]+\.(?:md|markdown|txt|pdf|docx|html)\b|"
17
+ r"[A-Z][A-Za-z0-9._-]*"
18
+ r"(?:\s+(?!\b(?:Section|Article|Clause)\s+[0-9])"
19
+ r"(?:(?:of|and|&|for)\s+(?!\b(?:Section|Article|Clause)\s+[0-9]))?[A-Z][A-Za-z0-9._-]*){0,5})",
20
+ )
21
+ _TOKEN_RE = re.compile(r"[a-z0-9]+")
22
+ _DOC_PHRASE_STOPWORDS = frozenset({"the", "of", "and", "for", "a", "an", "in", "under"})
23
+
24
+ _SECTION_NUM_RE = re.compile(
25
+ r"(?:\b(?:Section|Article|Clause)\s+([0-9]+(?:\.[0-9]+)*)\b"
26
+ r"|^([0-9]+(?:\.[0-9]+)+)(?=\.?(?:\s|$|[:\-—]))"
27
+ r"|^([0-9]+)\.(?=\s|$))",
28
+ re.IGNORECASE,
29
+ )
30
+ _CLAUSE_START_LINE_RE = re.compile(r"^([0-9]+(?:\.[0-9]+)+)\s+(.+)$")
31
+
32
+ __all__ = [
33
+ "CROSSREF_RE",
34
+ "extract_doc_crossrefs",
35
+ "extract_section_num",
36
+ "index_section_targets",
37
+ "iter_clause_blocks",
38
+ "match_source_key",
39
+ "parse_crossref_match",
40
+ "truncate_excerpt",
41
+ ]
42
+
43
+
44
+ def extract_section_num(label: str) -> str | None:
45
+ """Extract normalized section identifier from a heading label, if present.
46
+
47
+ Prefixed forms (``Section 7``, ``Article 4``) and dotted decimal headings
48
+ (``7.5 Public API``, ``20.2: Rules``) match directly, whereas bare integer
49
+ headings require a trailing period (``1. Title``) to avoid false-positive
50
+ matches on years or measurements (``2026 Roadmap``, ``3.5mm``).
51
+ """
52
+ match = _SECTION_NUM_RE.search(label)
53
+ if not match:
54
+ return None
55
+ return next((group for group in match.groups() if group is not None), None)
56
+
57
+
58
+ def truncate_excerpt(text: str, max_chars: int = 400) -> str:
59
+ """Truncate text cleanly at a word boundary without chopping words mid-token."""
60
+ clean = text.strip()
61
+ if len(clean) <= max_chars:
62
+ return clean
63
+ cutoff = clean.rfind(" ", 0, max_chars)
64
+ if cutoff <= 0:
65
+ cutoff = max_chars
66
+ return clean[:cutoff].rstrip(" ,;:") + "..."
67
+
68
+
69
+ def parse_crossref_match(match: re.Match[str]) -> tuple[list[str], str, str]:
70
+ """Extract normalized lookup keys, numeric target, and display label from a match."""
71
+ if match.group(1) and match.group(2):
72
+ kind = match.group(1).lower()
73
+ num = match.group(2)
74
+ return [f"{kind} {num}", num], num, f"{match.group(1)} {num}"
75
+ num = match.group(4)
76
+ return [num, f"clause {num}", f"section {num}"], num, f"§{num}"
77
+
78
+
79
+ def _strip_crossref_annotation_lines(text: str) -> list[str]:
80
+ """Return lines with single- or multi-line ``[Crossref ...]`` blocks blanked out."""
81
+ lines: list[str] = []
82
+ in_crossref_block = False
83
+ bracket_depth = 0
84
+ for raw_line in text.splitlines():
85
+ stripped = raw_line.strip()
86
+ if not in_crossref_block:
87
+ if stripped.startswith("[Crossref "):
88
+ bracket_depth = stripped.count("[") - stripped.count("]")
89
+ in_crossref_block = bracket_depth > 0
90
+ lines.append("")
91
+ else:
92
+ lines.append(raw_line)
93
+ else:
94
+ bracket_depth += stripped.count("[") - stripped.count("]")
95
+ if bracket_depth <= 0:
96
+ in_crossref_block = False
97
+ lines.append("")
98
+ return lines
99
+
100
+
101
+ def iter_clause_blocks(raw_text: str) -> list[tuple[str, str]]:
102
+ """Extract numbered clause numbers paired with their full wrapped paragraph text."""
103
+ blocks: list[tuple[str, str]] = []
104
+ cur_num: str | None = None
105
+ cur_lines: list[str] = []
106
+
107
+ def flush() -> None:
108
+ if cur_num is not None and cur_lines:
109
+ blocks.append((cur_num, " ".join(cur_lines)))
110
+ cur_lines.clear()
111
+
112
+ for raw_line in _strip_crossref_annotation_lines(raw_text):
113
+ line = raw_line.strip()
114
+ if not line:
115
+ flush()
116
+ cur_num = None
117
+ continue
118
+ match = _CLAUSE_START_LINE_RE.match(line)
119
+ if match:
120
+ flush()
121
+ cur_num = match.group(1)
122
+ cur_lines.append(line)
123
+ elif cur_num is not None:
124
+ cur_lines.append(line)
125
+
126
+ flush()
127
+ return blocks
128
+
129
+
130
+ def index_section_targets(sections: Sequence[tuple[str, str]]) -> dict[str, tuple[str, int]]:
131
+ """Index normalized section and clause numbers to ``(excerpt_text, section_idx)``."""
132
+ targets: dict[str, tuple[str, int]] = {}
133
+ for sec_idx, (section_label, raw_text) in enumerate(sections):
134
+ num = extract_section_num(section_label)
135
+ if num is not None:
136
+ full = f"{section_label}: {raw_text}"
137
+ targets.setdefault(num, (full, sec_idx))
138
+ targets.setdefault(f"section {num}".lower(), (full, sec_idx))
139
+ for c_num, c_text in iter_clause_blocks(raw_text):
140
+ targets.setdefault(c_num, (c_text, sec_idx))
141
+ targets.setdefault(f"clause {c_num}".lower(), (c_text, sec_idx))
142
+ return targets
143
+
144
+
145
+ def extract_doc_crossrefs(text: str) -> list[tuple[str, list[str], str]]:
146
+ """Extract explicit document-scoped cross-references from non-annotation lines."""
147
+ cleaned_body = "\n".join(_strip_crossref_annotation_lines(text))
148
+ refs: list[tuple[str, list[str], str]] = []
149
+ seen: set[tuple[str, str]] = set()
150
+ for match in _DOC_SCOPED_CROSSREF_RE.finditer(cleaned_body):
151
+ lookup_keys, _num, display = parse_crossref_match(match)
152
+ doc_phrase = match.group(5).strip().rstrip(".,;:")
153
+ dedup_key = (display.lower(), doc_phrase.lower())
154
+ if dedup_key in seen:
155
+ continue
156
+ seen.add(dedup_key)
157
+ refs.append((display, lookup_keys, doc_phrase))
158
+ return refs
159
+
160
+
161
+ def match_source_key(
162
+ doc_phrase: str,
163
+ available_sources: Sequence[str],
164
+ *,
165
+ current_source: str | None = None,
166
+ ) -> str | None:
167
+ """Resolve a cited document name/filename unambiguously against collection sources."""
168
+ phrase_lower = doc_phrase.strip().lower()
169
+ candidates = [src for src in available_sources if src != current_source]
170
+ if not candidates or not phrase_lower:
171
+ return None
172
+
173
+ for src in candidates:
174
+ stem = src.rsplit(".", 1)[0].lower()
175
+ if phrase_lower in {src.lower(), stem}:
176
+ return src
177
+
178
+ phrase_tokens = {
179
+ tok for tok in _TOKEN_RE.findall(phrase_lower) if tok not in _DOC_PHRASE_STOPWORDS
180
+ }
181
+ if len(phrase_tokens) < 2:
182
+ return None
183
+
184
+ matched: list[str] = []
185
+ for src in candidates:
186
+ src_tokens = set(_TOKEN_RE.findall(src.lower()))
187
+ if phrase_tokens.issubset(src_tokens):
188
+ matched.append(src)
189
+ return matched[0] if len(matched) == 1 else None