docpipe-maestro 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docpipe_maestro-0.2.0/.gitignore +119 -0
- docpipe_maestro-0.2.0/.specify/memory/constitution.md +176 -0
- docpipe_maestro-0.2.0/CHANGELOG.md +43 -0
- docpipe_maestro-0.2.0/LICENSE +21 -0
- docpipe_maestro-0.2.0/PKG-INFO +843 -0
- docpipe_maestro-0.2.0/PRODUCT.md +702 -0
- docpipe_maestro-0.2.0/README.md +797 -0
- docpipe_maestro-0.2.0/docs/README.md +26 -0
- docpipe_maestro-0.2.0/docs/api-reference.md +524 -0
- docpipe_maestro-0.2.0/docs/architecture.md +227 -0
- docpipe_maestro-0.2.0/docs/cli-reference.md +361 -0
- docpipe_maestro-0.2.0/docs/development.md +134 -0
- docpipe_maestro-0.2.0/docs/eval-baseline.json +2207 -0
- docpipe_maestro-0.2.0/docs/eval-baseline.md +98 -0
- docpipe_maestro-0.2.0/docs/future-features.md +176 -0
- docpipe_maestro-0.2.0/pyproject.toml +162 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/__init__.py +153 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/__main__.py +8 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/_crossrefs.py +189 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/_filters.py +635 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/_pii.py +986 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/_registry.py +124 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/_resolve.py +44 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/_stages.py +93 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/cli.py +1032 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/config.py +64 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/embedder/__init__.py +71 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/embedder/default.py +367 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/eval/__init__.py +37 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/eval/harness.py +415 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/eval/metrics.py +95 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/__init__.py +1 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/chunker/__init__.py +41 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/chunker/default.py +277 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/clean/__init__.py +41 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/clean/default.py +239 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/crossrefs.py +19 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/enricher/__init__.py +46 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/enricher/crossrefs.py +103 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/enricher/pii.py +154 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/__init__.py +70 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/_common.py +194 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/docx.py +96 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/html.py +194 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/markdown.py +31 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/pdf.py +64 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/text.py +31 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/pipeline.py +1178 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/stages.py +66 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/log.py +215 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/manifest.py +193 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/py.typed +0 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/__init__.py +1 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/answer.py +403 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/egress.py +153 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/prompts.py +94 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/reranker/__init__.py +66 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/reranker/cross_encoder.py +120 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/retriever.py +335 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/searcher/__init__.py +44 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/searcher/dense.py +68 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/searcher/hybrid.py +123 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/query/stages.py +44 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/store/__init__.py +43 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/store/chroma.py +302 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/store/lexical.py +323 -0
- docpipe_maestro-0.2.0/src/docpipe_maestro/types.py +558 -0
- docpipe_maestro-0.2.0/tests/conftest.py +122 -0
- docpipe_maestro-0.2.0/tests/fixtures/docs/sample.docx +0 -0
- docpipe_maestro-0.2.0/tests/fixtures/docs/sample.html +15 -0
- docpipe_maestro-0.2.0/tests/fixtures/docs/sample.md +11 -0
- docpipe_maestro-0.2.0/tests/fixtures/docs/sample.pdf +32 -0
- docpipe_maestro-0.2.0/tests/fixtures/docs/sample.txt +5 -0
- docpipe_maestro-0.2.0/tests/fixtures/eval_corpus/incident_response_runbook.md +23 -0
- docpipe_maestro-0.2.0/tests/fixtures/eval_corpus/information_security_policy.md +27 -0
- docpipe_maestro-0.2.0/tests/fixtures/eval_corpus/procurement_vendor_policy.md +21 -0
- docpipe_maestro-0.2.0/tests/fixtures/eval_policy_corpus.json +126 -0
- docpipe_maestro-0.2.0/tests/fixtures/golden_query_stage1.json +455 -0
- docpipe_maestro-0.2.0/tests/fixtures/policy_corpus/hr_leave_policy.md +21 -0
- docpipe_maestro-0.2.0/tests/fixtures/policy_corpus/travel_expenses_policy.md +11 -0
- docpipe_maestro-0.2.0/tests/test_api_stability.py +288 -0
- docpipe_maestro-0.2.0/tests/test_chunker.py +116 -0
- docpipe_maestro-0.2.0/tests/test_cli.py +2097 -0
- docpipe_maestro-0.2.0/tests/test_crossrefs.py +458 -0
- docpipe_maestro-0.2.0/tests/test_eval.py +696 -0
- docpipe_maestro-0.2.0/tests/test_extended_complex.py +458 -0
- docpipe_maestro-0.2.0/tests/test_golden_refactor.py +60 -0
- docpipe_maestro-0.2.0/tests/test_hybrid_search.py +1008 -0
- docpipe_maestro-0.2.0/tests/test_intake_clean.py +187 -0
- docpipe_maestro-0.2.0/tests/test_invariants_stage1.py +705 -0
- docpipe_maestro-0.2.0/tests/test_library_purity.py +215 -0
- docpipe_maestro-0.2.0/tests/test_logging.py +468 -0
- docpipe_maestro-0.2.0/tests/test_manifest.py +128 -0
- docpipe_maestro-0.2.0/tests/test_pii_safety.py +1040 -0
- docpipe_maestro-0.2.0/tests/test_prompts_answer.py +452 -0
- docpipe_maestro-0.2.0/tests/test_registry.py +818 -0
- docpipe_maestro-0.2.0/tests/test_reranker.py +693 -0
- docpipe_maestro-0.2.0/tests/test_retriever.py +406 -0
- docpipe_maestro-0.2.0/tests/test_reverse_safety.py +142 -0
- docpipe_maestro-0.2.0/tests/test_security_hardening.py +333 -0
- docpipe_maestro-0.2.0/tests/test_store.py +208 -0
- docpipe_maestro-0.2.0/tests/test_sync_lifecycle.py +165 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller / binary distribution
|
|
30
|
+
*.manifest
|
|
31
|
+
*.spec
|
|
32
|
+
|
|
33
|
+
# Installer logs
|
|
34
|
+
pip-log.txt
|
|
35
|
+
pip-delete-this-directory.txt
|
|
36
|
+
|
|
37
|
+
# Unit test / coverage reports
|
|
38
|
+
htmlcov/
|
|
39
|
+
.tox/
|
|
40
|
+
.nox/
|
|
41
|
+
.coverage
|
|
42
|
+
.coverage.*
|
|
43
|
+
.cache
|
|
44
|
+
nosetests.xml
|
|
45
|
+
coverage.xml
|
|
46
|
+
*.cover
|
|
47
|
+
*.py,cover
|
|
48
|
+
.hypothesis/
|
|
49
|
+
.pytest_cache/
|
|
50
|
+
cover/
|
|
51
|
+
|
|
52
|
+
# Type checking
|
|
53
|
+
.mypy_cache/
|
|
54
|
+
.pyre/
|
|
55
|
+
.pytype/
|
|
56
|
+
.ruff_cache/
|
|
57
|
+
|
|
58
|
+
# Environments and Virtualenvs
|
|
59
|
+
.env
|
|
60
|
+
.venv
|
|
61
|
+
env/
|
|
62
|
+
venv/
|
|
63
|
+
ENV/
|
|
64
|
+
env.bak/
|
|
65
|
+
venv.bak/
|
|
66
|
+
|
|
67
|
+
# Environment variables & secrets (CRITICAL: Constitution Principle III & Data/Privacy)
|
|
68
|
+
*.env
|
|
69
|
+
*.env.*
|
|
70
|
+
!.env.example
|
|
71
|
+
*.env.local
|
|
72
|
+
.env.production
|
|
73
|
+
.env.development
|
|
74
|
+
|
|
75
|
+
# Local vector databases & transient storage (e.g., ChromaDB, local document stores)
|
|
76
|
+
.docpipe/
|
|
77
|
+
docpipe.toml
|
|
78
|
+
documents/test.pdf
|
|
79
|
+
.chroma/
|
|
80
|
+
chroma_db/
|
|
81
|
+
chroma_data/
|
|
82
|
+
*.chroma
|
|
83
|
+
*.sqlite
|
|
84
|
+
*.sqlite3
|
|
85
|
+
*.db
|
|
86
|
+
local_data/
|
|
87
|
+
data/
|
|
88
|
+
corpus/
|
|
89
|
+
test_corpus_tmp/
|
|
90
|
+
|
|
91
|
+
# Jupyter Notebook
|
|
92
|
+
.ipynb_checkpoints
|
|
93
|
+
|
|
94
|
+
# IDEs and Editors
|
|
95
|
+
.vscode/
|
|
96
|
+
.idea/
|
|
97
|
+
*.swp
|
|
98
|
+
*.swo
|
|
99
|
+
*~
|
|
100
|
+
.project
|
|
101
|
+
.pydevproject
|
|
102
|
+
.settings/
|
|
103
|
+
|
|
104
|
+
# macOS / OS artifacts
|
|
105
|
+
.DS_Store
|
|
106
|
+
.AppleDouble
|
|
107
|
+
.LSOverride
|
|
108
|
+
Icon
|
|
109
|
+
._*
|
|
110
|
+
.Spotlight-V100
|
|
111
|
+
.Trashes
|
|
112
|
+
ehthumbs.db
|
|
113
|
+
Thumbs.db
|
|
114
|
+
[Dd]esktop.ini
|
|
115
|
+
|
|
116
|
+
# Logs
|
|
117
|
+
logs/
|
|
118
|
+
*.log
|
|
119
|
+
agy-logs/
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
# Sync Impact Report
|
|
3
|
+
- Version change: 1.3.0 -> 1.3.1
|
|
4
|
+
- Rationale: Fulfill v1.3.0 follow-up TODO by syncing finalized Stage 4 (`009-pii-protection`) public root exports (`register_pii_recognizer`, `PiiPolicy`, `PiiBlockedError`) into Quality Gates.
|
|
5
|
+
- Migration Note: Strictly additive root exports (`__all__` count incremented to 36); zero breaking changes for existing callers or Phase 1/2 signatures.
|
|
6
|
+
- List of modified principles: None
|
|
7
|
+
- Modified sections:
|
|
8
|
+
- Additional Constraints -> Quality Gates (synced `register_pii_recognizer`, `PiiPolicy`, `PiiBlockedError` into the root public API surface list)
|
|
9
|
+
- Added sections: None
|
|
10
|
+
- Removed sections: None
|
|
11
|
+
- Follow-up TODOs: None
|
|
12
|
+
-->
|
|
13
|
+
|
|
14
|
+
# docpipe-maestro Constitution
|
|
15
|
+
|
|
16
|
+
## Core Principles
|
|
17
|
+
|
|
18
|
+
### I. Scope Discipline (NON-NEGOTIABLE)
|
|
19
|
+
|
|
20
|
+
The product is a minimal RAG pipeline library — not an application, not a framework,
|
|
21
|
+
not a platform. "Minimal" applies strictly to feature surface and scope promise,
|
|
22
|
+
never to engineering rigor, correctness guarantees, or safety discipline
|
|
23
|
+
(*minimal in promise, not in rigor*). Anything that assumes "a user is waiting" (UI, HTTP API,
|
|
24
|
+
sessions, auth, streaming, deployment) belongs to the consuming application. Anything
|
|
25
|
+
that cannot be reached through the public API surface does not belong in the core.
|
|
26
|
+
Every feature proposal must pass the one-sentence positioning test:
|
|
27
|
+
|
|
28
|
+
> "A minimal RAG pipeline: it is the engine — not the app, not the framework."
|
|
29
|
+
|
|
30
|
+
### II. Two-Phase Architecture
|
|
31
|
+
|
|
32
|
+
Every module belongs to exactly one phase: Phase 1 (ingest, offline, run once per
|
|
33
|
+
batch) or Phase 2 (query, online, run per question). Shared modules (types, config,
|
|
34
|
+
store, manifest) must serve both phases without blurring the boundary. Phase 1 work
|
|
35
|
+
must never degrade Phase 2 latency; Phase 2 must treat the store and manifest as
|
|
36
|
+
read-only. The file tree must make the phase of every module legible without opening it.
|
|
37
|
+
|
|
38
|
+
### III. Library Purity
|
|
39
|
+
|
|
40
|
+
No module in the library reads a user config or policy file. cli.py is the only file
|
|
41
|
+
that may; runtime configuration and PII policies MUST be passed programmatically to
|
|
42
|
+
library entry points. Chunking profiles are Python constants, not files. Provider
|
|
43
|
+
credentials come exclusively from environment variables — never from code, config, logs,
|
|
44
|
+
or error messages. There is no provider registry in the codebase; provider resolution is
|
|
45
|
+
delegated to the unified client via `provider/model` strings. Adding a provider must
|
|
46
|
+
require zero code changes.
|
|
47
|
+
|
|
48
|
+
### IV. Search Small, Return Big (NON-NEGOTIABLE)
|
|
49
|
+
|
|
50
|
+
Retrieval must match child chunks and return parent sections. Every child must
|
|
51
|
+
carry a `parent_id`. Parents must be stored with a `type` field that excludes them
|
|
52
|
+
from similarity search via an explicit filter. Parent sections must be deduplicated
|
|
53
|
+
before being returned. This principle exists because incomplete retrieval can
|
|
54
|
+
*invert the meaning* of a rule — not merely reduce precision. A system that returns
|
|
55
|
+
"20 days of leave" without "except for new hires" is not imprecise; it is wrong.
|
|
56
|
+
|
|
57
|
+
### V. Exception Preservation
|
|
58
|
+
|
|
59
|
+
Any retrieval context containing a rule with an exception marker (`except`,
|
|
60
|
+
`unless`, `not entitled`, `subject to`, `provided that`) must also contain that
|
|
61
|
+
exception. This is enforced by the reverse test against a fixture policy corpus,
|
|
62
|
+
which serves as mandatory **Release Gate #1** (`@pytest.mark.reverse`). PII ingestion
|
|
63
|
+
safeguards, outbound egress protection, and zero-raw-PII audit guarantees are enforced
|
|
64
|
+
by the PII safety suite as mandatory **Release Gate #2** (`@pytest.mark.pii`). A
|
|
65
|
+
release that fails Release Gate #1 or Release Gate #2 MUST NOT be published, regardless
|
|
66
|
+
of other passing tests; safety claims are only credible when verified by automated gates.
|
|
67
|
+
|
|
68
|
+
### VI. Provider Agnosticism and Local-First Default
|
|
69
|
+
|
|
70
|
+
Embeddings and generation are addressed by `provider/model` strings through a
|
|
71
|
+
single unified interface. The default generation model is local, so the
|
|
72
|
+
out-of-the-box configuration requires no third-party account and no network access.
|
|
73
|
+
The full pipeline must be runnable with zero external network calls when configured
|
|
74
|
+
with local providers. Privacy is a default, not an opt-in.
|
|
75
|
+
|
|
76
|
+
### VII. Structured Correctness Over Speed
|
|
77
|
+
|
|
78
|
+
Chunking must be structure-aware: split on headings, then sections, then clauses,
|
|
79
|
+
then sentences, and only as a last resort on characters. No clause may be split
|
|
80
|
+
across chunks in policy profiles. Overlap is scoped to a parent section and resets
|
|
81
|
+
at section boundaries. Crossrefs are resolved or logged — never silently dropped.
|
|
82
|
+
Data shapes are defined once in `types.py` and imported everywhere; schema drift
|
|
83
|
+
across modules is prohibited.
|
|
84
|
+
|
|
85
|
+
## Additional Constraints
|
|
86
|
+
|
|
87
|
+
### Data and Privacy
|
|
88
|
+
|
|
89
|
+
- Credentials are environment variables only. No credential may appear in source,
|
|
90
|
+
config, logs, or error output.
|
|
91
|
+
- Local execution must require no outbound network call. When strict local-only mode is
|
|
92
|
+
requested, any remote LLM provider or non-loopback `api_base` MUST be rejected with a
|
|
93
|
+
typed `DocpipeError` before any network request is made.
|
|
94
|
+
- Every retrieved section must carry metadata sufficient to cite source document,
|
|
95
|
+
section, and page.
|
|
96
|
+
- Ingestion must be deterministic (identical inputs → identical content-addressed
|
|
97
|
+
chunk IDs) and incremental:
|
|
98
|
+
- Changes are tracked via a database-collocated ingestion manifest using content
|
|
99
|
+
hashes (`sha256`), with `mtime` and `size` used strictly as fast-path hints.
|
|
100
|
+
- Updates use source-scoped **delete-then-reinsert** (never ID-only upsert) wrapped
|
|
101
|
+
in a staging/version-flip envelope so shrinking documents never leave orphaned
|
|
102
|
+
clauses and failed re-ingestions leave the prior version intact and queryable.
|
|
103
|
+
- Collections lock their embedding model identifier in the manifest; model
|
|
104
|
+
mismatches between ingest and query are refused unless explicitly reindexed.
|
|
105
|
+
- When a PII policy is configured during ingestion:
|
|
106
|
+
- Detection and transformation MUST execute pre-chunking across both section titles
|
|
107
|
+
and body text so redacted or tokenized values never leak into child context headers
|
|
108
|
+
or stored chunks.
|
|
109
|
+
- Block-mode matches MUST halt ingestion via a typed `DocpipeError` before any vector
|
|
110
|
+
store write or manifest commit, leaving prior collection state intact.
|
|
111
|
+
- Tokenization MUST use a keyed/salted cryptographic digest of the matched value
|
|
112
|
+
(never character-offset hashing) so repeated identifiers correlate deterministically
|
|
113
|
+
without exposing plaintext.
|
|
114
|
+
- Errors, telemetry, and PII audit logs (`.audit.jsonl`) MUST contain zero raw PII
|
|
115
|
+
values (counts, entity types, actions, source keys, and document version digests
|
|
116
|
+
only), and audit log storage MUST restrict directory permissions (`0o700` on POSIX
|
|
117
|
+
systems).
|
|
118
|
+
- When remote LLM completion is invoked outside local-only mode, outbound prompts and
|
|
119
|
+
role-separated chat messages MUST pass through the configured PII egress policy prior
|
|
120
|
+
to transmission. Documentation MUST NOT overclaim "100% accurate detection" or blanket
|
|
121
|
+
regulatory compliance (e.g., "GDPR compliant").
|
|
122
|
+
|
|
123
|
+
### Quality Gates
|
|
124
|
+
|
|
125
|
+
- Every behavioral change ships with tests. Running **all tests** across the entire
|
|
126
|
+
suite after implementation is mandatory, and **all tests must pass** (zero
|
|
127
|
+
failures). CI runs linting, static type checking, and the full suite; any
|
|
128
|
+
failing test or red CI blocks completion and release.
|
|
129
|
+
- **Release Gate #1 (mandatory)**: The exception-preservation reverse test
|
|
130
|
+
(`@pytest.mark.reverse`) must pass prior to every release.
|
|
131
|
+
- **Release Gate #2 (mandatory)**: The PII protection safety suite (`@pytest.mark.pii`)
|
|
132
|
+
must pass prior to every release, verifying ingestion block/redaction/tokenization
|
|
133
|
+
invariants, outbound PII egress guarding, local-only remote-call refusal, and
|
|
134
|
+
zero-raw-PII audit logs.
|
|
135
|
+
- The package must install from TestPyPI in a clean environment before any PyPI
|
|
136
|
+
release, and publishing uses trusted publishing rather than long-lived tokens.
|
|
137
|
+
- The public API surface is the contract: `ingest`, `sync`, `remove`, `status`,
|
|
138
|
+
`reindex`, `retrieve`, `build_prompt`, `build_chat_messages`, optional `answer`,
|
|
139
|
+
`answer_detailed`, extension decorators (`register_parser`, `register_cleaner`,
|
|
140
|
+
`register_enricher`, `register_chunker`, `register_embedder`, `register_store`,
|
|
141
|
+
`register_searcher`, `register_reranker`, `register_pii_recognizer`), shared data/result shapes (`Chunk`,
|
|
142
|
+
`EmbeddedChunk`, `RetrievedChunk`, `SourceRecord`, `IngestResult`, `SyncResult`,
|
|
143
|
+
`CollectionStatus`, `GenerationResult`, `PiiPolicy`), public exceptions (`DocpipeError`,
|
|
144
|
+
`MissingExtraError`, `UnsupportedFormatError`, `EmptyDocumentError`,
|
|
145
|
+
`ModelMismatchError`, `MissingCredentialError`, `RegistryError`, `IngestionError`,
|
|
146
|
+
`GenerationError`, `PiiBlockedError`), and `__version__`. If a feature cannot be reached through that
|
|
147
|
+
surface, it does not belong in the core.
|
|
148
|
+
|
|
149
|
+
### Footprint
|
|
150
|
+
|
|
151
|
+
- The default local configuration must run comfortably on 16 GB RAM with no GPU.
|
|
152
|
+
- The core install stays lean; generation is **optional to invoke** at runtime (Phase 1
|
|
153
|
+
ingestion and Phase 2 retrieval work offline with zero LLM completion calls), while
|
|
154
|
+
the unified client library handles both embeddings and completions in the core install.
|
|
155
|
+
- Built-in pattern/checksum PII recognizers MUST operate offline in the core install with
|
|
156
|
+
zero automatic model downloads; optional NER-based PII dependencies MUST stay isolated
|
|
157
|
+
behind an optional extra and load lazily only when requested.
|
|
158
|
+
- A new file format or storage backend must be additive — no existing module should
|
|
159
|
+
need modification beyond registration.
|
|
160
|
+
|
|
161
|
+
## Governance
|
|
162
|
+
|
|
163
|
+
- This constitution supersedes feature convenience. A feature that violates a
|
|
164
|
+
principle must be rejected, or the constitution amended first — never quietly
|
|
165
|
+
worked around.
|
|
166
|
+
- Amendments require a documented rationale, a version bump, and a migration note
|
|
167
|
+
describing what existing code or consumers must change.
|
|
168
|
+
- Every spec, plan, and task must include a "Constitution Check" confirming
|
|
169
|
+
compliance with Principles I–VII before implementation begins.
|
|
170
|
+
- The one-sentence positioning is the tiebreaker for any scope dispute:
|
|
171
|
+
|
|
172
|
+
> "docpipe-maestro is a minimal RAG pipeline: it ingests any document, chunks it
|
|
173
|
+
> correctly, embeds it, stores it, retrieves the right sections, and optionally
|
|
174
|
+
> generates a grounded answer. It is the engine — not the app, not the framework."
|
|
175
|
+
|
|
176
|
+
**Version**: 1.3.1 | **Ratified**: 2026-11-12 | **Last Amended**: 2026-11-15
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
## v0.2.0 (2026-11-17)
|
|
2
|
+
|
|
3
|
+
### Feat
|
|
4
|
+
|
|
5
|
+
- **log**: add opt-in structured JSON logging and `ContextVar`-backed `correlation_id()` tracing via submodule `docpipe_maestro.log` (`JsonFormatter`, `TextFormatter`, `get_logger()` with passive `NullHandler`-only defaults, `stage_event()` timing helper with `isEnabledFor()` short-circuiting), instrument Phase 1 (`ingest`, `sync`, `remove`, `status`, `reindex` with `INFO` batch summaries and `DEBUG` per-doc details across `intake`/`clean`/`enrich`/`chunk`/`embed`/`store`) and Phase 2 (`retrieve`, searchers, `rerank`, `guard_egress`, `answer_detailed`, `evaluate` across `embed`/`search`/`rerank`/`egress`/`answer`) with traceback-free `ERROR` `error_type` failure attribution and zero raw PII/content/secret emission, and add CLI-only global `--log-json` / `--log-level {DEBUG,INFO,WARNING,ERROR}` flags writing strictly to `stderr`
|
|
6
|
+
- **retrieve**: add structured metadata pre-filtering via `retrieve(..., where=...)`, `evaluate(..., where=...)`, and repeatable CLI `--where KEY=VALUE` across `query`, `ask`, and `eval`, supporting stored chunk fields (`source`, `section`, `clause`, `page`, `chunk_index`, `doc_version`, `embed_model`), top-level manifest attributes (`profile` case-insensitive, UTC ISO-8601 `ingested_after` / `ingested_before` inclusive bounds) with zero re-ingestion migration and zero-embedder empty short-circuiting, full 10-operator Chroma + FTS5 SQLite parity (`$eq`, `$ne`, `$in`, `$nin`, `$gt`, `$gte`, `$lt`, `$lte`, `$and`, `$or`), and upfront unknown-key / reserved `"type"` validation with store-level `{"$and": [{"type": "child"}, ...]}` defense-in-depth
|
|
7
|
+
- **pii**: add configurable pre-chunking `PiiEnricher` (`redact`, `tokenize`, `block`) across headings, body text, and clauses, 7 built-in deterministic recognizers + extensible `@register_pii_recognizer` (`docpipe_maestro.register_pii_recognizer`, `PiiRecognizer`), lazy optional `[pii]` multi-language Presidio NER (`en`, `es`, etc. with zero runtime downloads), salted value-based HMAC tokenization, owner-only (`0o700`/`0o600`) zero-raw-PII `<collection>.audit.jsonl` logs with transactional `PiiBlockedError` rollback, shared `--local-only` / `local_only` preflight across embedding and generation workflows, Phase 2 `guard_egress` for prompt strings and ChatML messages, CLI `--pii-policy` JSON/TOML support, and Release Gate #2 (`@pytest.mark.pii`)
|
|
8
|
+
- **enricher**: formalize Phase 1 post-clean, pre-chunk `Enricher` stage (`ChunkingProfile.enrichers`, ordered execution with duplicate support, upfront `ENRICHERS` resolution, atomic abort guarantees, `IngestContext`), migrate intra-document cross-reference resolution into lazy `CrossrefsEnricher` (`"crossrefs"`), extract pure shared cross-reference helpers into `docpipe_maestro._crossrefs`, and keep silent `docpipe_maestro.ingest.crossrefs` compatibility re-exports
|
|
9
|
+
- **rerank**: add opt-in child-first local cross-encoder reranking (`--rerank`, `retrieve(..., rerank=True)`, `evaluate(..., rerank=True)`), optional `[rerank]` extra (`sentence-transformers`), lazy `CrossEncoderReranker` (`BAAI/bge-reranker-base`) with process-local model caching, CLI/TOML configuration, and live `Q1` benchmark validation
|
|
10
|
+
- **eval**: add read-only retrieval evaluation harness (`docpipe_maestro.eval`, `docpipe-maestro eval`), curated policy benchmark corpus, and offline `default` vs `fts5_hybrid` baselines
|
|
11
|
+
- **searcher**: add hybrid SQLite FTS5 BM25 + dense vector RRF search (`--searcher fts5_hybrid`) and collocated `LexicalIndex` lifecycle sync
|
|
12
|
+
- **query**: add online LLM customization (`temperature`, `max_tokens`, `timeout`, `model_kwargs`), `build_chat_messages()` role separation, `answer_detailed()` + `GenerationResult`, CLI/TOML `ask` options (`--system-prompt`, `--system-prompt-file`, `--fail-on-truncation`, `--show-context`, `--json`), loopback-vs-HTTPS `api_base` transport validation, scoped CLI `.env` + `./docpipe.toml` auto-discovery (`--no-config`), direct Ollama `/api/embed` fast path, digit-normalized header/footer stripping, and cross-document `follow_crossrefs` retrieval
|
|
13
|
+
- **cli**: add `docpipe-maestro query --json` structured retrieval output envelope
|
|
14
|
+
|
|
15
|
+
### Refactor
|
|
16
|
+
|
|
17
|
+
- **pipeline**: introduce Strategy + Registry extension architecture (`register_*` entry points)
|
|
18
|
+
|
|
19
|
+
### Fix
|
|
20
|
+
|
|
21
|
+
- **query**: improve multi-section prompt synthesis and report explicit `finish_reason="length"` / `reasoning_tokens` diagnostics on empty model responses
|
|
22
|
+
- **eval**: use `math.fsum` in aggregate metrics for Python 3.12+ determinism
|
|
23
|
+
- **store**: skip unnecessary HNSW vector reads in `get_all()` and release test Chroma database handles cleanly on Windows
|
|
24
|
+
- **cross-os**: normalize LF line endings across fixtures via `.gitattributes`, handle `float32` / `os.PathLike` checks across macOS and Windows, and advance `mtime` explicitly in same-size manifest fast-path tests
|
|
25
|
+
|
|
26
|
+
### CI & Docs
|
|
27
|
+
|
|
28
|
+
- **workflow**: harden GitHub Actions CI matrix (`3.10`–`3.13`, `--no-extras` core smoke, `uv build` + `twine check --strict`, `pip-audit`, macOS/Windows smoke) and add OIDC Trusted Publishing release workflow (`.github/workflows/release.yml`)
|
|
29
|
+
- **docs**: enrich `README.md`, `docs/` guides, `CONTRIBUTING.md`, `SECURITY.md`, and `PRODUCT.md`, plus add Contributor Covenant Code of Conduct and `.env.example`
|
|
30
|
+
|
|
31
|
+
## v0.1.0 (2026-11-13)
|
|
32
|
+
|
|
33
|
+
### Feat
|
|
34
|
+
|
|
35
|
+
- **pipeline**: implement v0.1 core RAG pipeline, CLI, and quality gates
|
|
36
|
+
|
|
37
|
+
### Fix
|
|
38
|
+
|
|
39
|
+
- **security**: harden pipeline, add CI workflow, and update gitignore
|
|
40
|
+
|
|
41
|
+
### Refactor
|
|
42
|
+
|
|
43
|
+
- **cli**: rename CLI command to docpipe-maestro
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 docpipe-maestro contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|