docpipe-maestro 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. docpipe_maestro-0.2.0/.gitignore +119 -0
  2. docpipe_maestro-0.2.0/.specify/memory/constitution.md +176 -0
  3. docpipe_maestro-0.2.0/CHANGELOG.md +43 -0
  4. docpipe_maestro-0.2.0/LICENSE +21 -0
  5. docpipe_maestro-0.2.0/PKG-INFO +843 -0
  6. docpipe_maestro-0.2.0/PRODUCT.md +702 -0
  7. docpipe_maestro-0.2.0/README.md +797 -0
  8. docpipe_maestro-0.2.0/docs/README.md +26 -0
  9. docpipe_maestro-0.2.0/docs/api-reference.md +524 -0
  10. docpipe_maestro-0.2.0/docs/architecture.md +227 -0
  11. docpipe_maestro-0.2.0/docs/cli-reference.md +361 -0
  12. docpipe_maestro-0.2.0/docs/development.md +134 -0
  13. docpipe_maestro-0.2.0/docs/eval-baseline.json +2207 -0
  14. docpipe_maestro-0.2.0/docs/eval-baseline.md +98 -0
  15. docpipe_maestro-0.2.0/docs/future-features.md +176 -0
  16. docpipe_maestro-0.2.0/pyproject.toml +162 -0
  17. docpipe_maestro-0.2.0/src/docpipe_maestro/__init__.py +153 -0
  18. docpipe_maestro-0.2.0/src/docpipe_maestro/__main__.py +8 -0
  19. docpipe_maestro-0.2.0/src/docpipe_maestro/_crossrefs.py +189 -0
  20. docpipe_maestro-0.2.0/src/docpipe_maestro/_filters.py +635 -0
  21. docpipe_maestro-0.2.0/src/docpipe_maestro/_pii.py +986 -0
  22. docpipe_maestro-0.2.0/src/docpipe_maestro/_registry.py +124 -0
  23. docpipe_maestro-0.2.0/src/docpipe_maestro/_resolve.py +44 -0
  24. docpipe_maestro-0.2.0/src/docpipe_maestro/_stages.py +93 -0
  25. docpipe_maestro-0.2.0/src/docpipe_maestro/cli.py +1032 -0
  26. docpipe_maestro-0.2.0/src/docpipe_maestro/config.py +64 -0
  27. docpipe_maestro-0.2.0/src/docpipe_maestro/embedder/__init__.py +71 -0
  28. docpipe_maestro-0.2.0/src/docpipe_maestro/embedder/default.py +367 -0
  29. docpipe_maestro-0.2.0/src/docpipe_maestro/eval/__init__.py +37 -0
  30. docpipe_maestro-0.2.0/src/docpipe_maestro/eval/harness.py +415 -0
  31. docpipe_maestro-0.2.0/src/docpipe_maestro/eval/metrics.py +95 -0
  32. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/__init__.py +1 -0
  33. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/chunker/__init__.py +41 -0
  34. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/chunker/default.py +277 -0
  35. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/clean/__init__.py +41 -0
  36. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/clean/default.py +239 -0
  37. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/crossrefs.py +19 -0
  38. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/enricher/__init__.py +46 -0
  39. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/enricher/crossrefs.py +103 -0
  40. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/enricher/pii.py +154 -0
  41. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/__init__.py +70 -0
  42. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/_common.py +194 -0
  43. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/docx.py +96 -0
  44. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/html.py +194 -0
  45. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/markdown.py +31 -0
  46. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/pdf.py +64 -0
  47. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/intake/text.py +31 -0
  48. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/pipeline.py +1178 -0
  49. docpipe_maestro-0.2.0/src/docpipe_maestro/ingest/stages.py +66 -0
  50. docpipe_maestro-0.2.0/src/docpipe_maestro/log.py +215 -0
  51. docpipe_maestro-0.2.0/src/docpipe_maestro/manifest.py +193 -0
  52. docpipe_maestro-0.2.0/src/docpipe_maestro/py.typed +0 -0
  53. docpipe_maestro-0.2.0/src/docpipe_maestro/query/__init__.py +1 -0
  54. docpipe_maestro-0.2.0/src/docpipe_maestro/query/answer.py +403 -0
  55. docpipe_maestro-0.2.0/src/docpipe_maestro/query/egress.py +153 -0
  56. docpipe_maestro-0.2.0/src/docpipe_maestro/query/prompts.py +94 -0
  57. docpipe_maestro-0.2.0/src/docpipe_maestro/query/reranker/__init__.py +66 -0
  58. docpipe_maestro-0.2.0/src/docpipe_maestro/query/reranker/cross_encoder.py +120 -0
  59. docpipe_maestro-0.2.0/src/docpipe_maestro/query/retriever.py +335 -0
  60. docpipe_maestro-0.2.0/src/docpipe_maestro/query/searcher/__init__.py +44 -0
  61. docpipe_maestro-0.2.0/src/docpipe_maestro/query/searcher/dense.py +68 -0
  62. docpipe_maestro-0.2.0/src/docpipe_maestro/query/searcher/hybrid.py +123 -0
  63. docpipe_maestro-0.2.0/src/docpipe_maestro/query/stages.py +44 -0
  64. docpipe_maestro-0.2.0/src/docpipe_maestro/store/__init__.py +43 -0
  65. docpipe_maestro-0.2.0/src/docpipe_maestro/store/chroma.py +302 -0
  66. docpipe_maestro-0.2.0/src/docpipe_maestro/store/lexical.py +323 -0
  67. docpipe_maestro-0.2.0/src/docpipe_maestro/types.py +558 -0
  68. docpipe_maestro-0.2.0/tests/conftest.py +122 -0
  69. docpipe_maestro-0.2.0/tests/fixtures/docs/sample.docx +0 -0
  70. docpipe_maestro-0.2.0/tests/fixtures/docs/sample.html +15 -0
  71. docpipe_maestro-0.2.0/tests/fixtures/docs/sample.md +11 -0
  72. docpipe_maestro-0.2.0/tests/fixtures/docs/sample.pdf +32 -0
  73. docpipe_maestro-0.2.0/tests/fixtures/docs/sample.txt +5 -0
  74. docpipe_maestro-0.2.0/tests/fixtures/eval_corpus/incident_response_runbook.md +23 -0
  75. docpipe_maestro-0.2.0/tests/fixtures/eval_corpus/information_security_policy.md +27 -0
  76. docpipe_maestro-0.2.0/tests/fixtures/eval_corpus/procurement_vendor_policy.md +21 -0
  77. docpipe_maestro-0.2.0/tests/fixtures/eval_policy_corpus.json +126 -0
  78. docpipe_maestro-0.2.0/tests/fixtures/golden_query_stage1.json +455 -0
  79. docpipe_maestro-0.2.0/tests/fixtures/policy_corpus/hr_leave_policy.md +21 -0
  80. docpipe_maestro-0.2.0/tests/fixtures/policy_corpus/travel_expenses_policy.md +11 -0
  81. docpipe_maestro-0.2.0/tests/test_api_stability.py +288 -0
  82. docpipe_maestro-0.2.0/tests/test_chunker.py +116 -0
  83. docpipe_maestro-0.2.0/tests/test_cli.py +2097 -0
  84. docpipe_maestro-0.2.0/tests/test_crossrefs.py +458 -0
  85. docpipe_maestro-0.2.0/tests/test_eval.py +696 -0
  86. docpipe_maestro-0.2.0/tests/test_extended_complex.py +458 -0
  87. docpipe_maestro-0.2.0/tests/test_golden_refactor.py +60 -0
  88. docpipe_maestro-0.2.0/tests/test_hybrid_search.py +1008 -0
  89. docpipe_maestro-0.2.0/tests/test_intake_clean.py +187 -0
  90. docpipe_maestro-0.2.0/tests/test_invariants_stage1.py +705 -0
  91. docpipe_maestro-0.2.0/tests/test_library_purity.py +215 -0
  92. docpipe_maestro-0.2.0/tests/test_logging.py +468 -0
  93. docpipe_maestro-0.2.0/tests/test_manifest.py +128 -0
  94. docpipe_maestro-0.2.0/tests/test_pii_safety.py +1040 -0
  95. docpipe_maestro-0.2.0/tests/test_prompts_answer.py +452 -0
  96. docpipe_maestro-0.2.0/tests/test_registry.py +818 -0
  97. docpipe_maestro-0.2.0/tests/test_reranker.py +693 -0
  98. docpipe_maestro-0.2.0/tests/test_retriever.py +406 -0
  99. docpipe_maestro-0.2.0/tests/test_reverse_safety.py +142 -0
  100. docpipe_maestro-0.2.0/tests/test_security_hardening.py +333 -0
  101. docpipe_maestro-0.2.0/tests/test_store.py +208 -0
  102. docpipe_maestro-0.2.0/tests/test_sync_lifecycle.py +165 -0
@@ -0,0 +1,119 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller / binary distribution
30
+ *.manifest
31
+ *.spec
32
+
33
+ # Installer logs
34
+ pip-log.txt
35
+ pip-delete-this-directory.txt
36
+
37
+ # Unit test / coverage reports
38
+ htmlcov/
39
+ .tox/
40
+ .nox/
41
+ .coverage
42
+ .coverage.*
43
+ .cache
44
+ nosetests.xml
45
+ coverage.xml
46
+ *.cover
47
+ *.py,cover
48
+ .hypothesis/
49
+ .pytest_cache/
50
+ cover/
51
+
52
+ # Type checking
53
+ .mypy_cache/
54
+ .pyre/
55
+ .pytype/
56
+ .ruff_cache/
57
+
58
+ # Environments and Virtualenvs
59
+ .env
60
+ .venv
61
+ env/
62
+ venv/
63
+ ENV/
64
+ env.bak/
65
+ venv.bak/
66
+
67
+ # Environment variables & secrets (CRITICAL: Constitution Principle III & Data/Privacy)
68
+ *.env
69
+ *.env.*
70
+ !.env.example
71
+ *.env.local
72
+ .env.production
73
+ .env.development
74
+
75
+ # Local vector databases & transient storage (e.g., ChromaDB, local document stores)
76
+ .docpipe/
77
+ docpipe.toml
78
+ documents/test.pdf
79
+ .chroma/
80
+ chroma_db/
81
+ chroma_data/
82
+ *.chroma
83
+ *.sqlite
84
+ *.sqlite3
85
+ *.db
86
+ local_data/
87
+ data/
88
+ corpus/
89
+ test_corpus_tmp/
90
+
91
+ # Jupyter Notebook
92
+ .ipynb_checkpoints
93
+
94
+ # IDEs and Editors
95
+ .vscode/
96
+ .idea/
97
+ *.swp
98
+ *.swo
99
+ *~
100
+ .project
101
+ .pydevproject
102
+ .settings/
103
+
104
+ # macOS / OS artifacts
105
+ .DS_Store
106
+ .AppleDouble
107
+ .LSOverride
108
+ Icon
109
+ ._*
110
+ .Spotlight-V100
111
+ .Trashes
112
+ ehthumbs.db
113
+ Thumbs.db
114
+ [Dd]esktop.ini
115
+
116
+ # Logs
117
+ logs/
118
+ *.log
119
+ agy-logs/
@@ -0,0 +1,176 @@
1
+ <!--
2
+ # Sync Impact Report
3
+ - Version change: 1.3.0 -> 1.3.1
4
+ - Rationale: Fulfill v1.3.0 follow-up TODO by syncing finalized Stage 4 (`009-pii-protection`) public root exports (`register_pii_recognizer`, `PiiPolicy`, `PiiBlockedError`) into Quality Gates.
5
+ - Migration Note: Strictly additive root exports (`__all__` count incremented to 36); zero breaking changes for existing callers or Phase 1/2 signatures.
6
+ - List of modified principles: None
7
+ - Modified sections:
8
+ - Additional Constraints -> Quality Gates (synced `register_pii_recognizer`, `PiiPolicy`, `PiiBlockedError` into the root public API surface list)
9
+ - Added sections: None
10
+ - Removed sections: None
11
+ - Follow-up TODOs: None
12
+ -->
13
+
14
+ # docpipe-maestro Constitution
15
+
16
+ ## Core Principles
17
+
18
+ ### I. Scope Discipline (NON-NEGOTIABLE)
19
+
20
+ The product is a minimal RAG pipeline library — not an application, not a framework,
21
+ not a platform. "Minimal" applies strictly to feature surface and scope promise,
22
+ never to engineering rigor, correctness guarantees, or safety discipline
23
+ (*minimal in promise, not in rigor*). Anything that assumes "a user is waiting" (UI, HTTP API,
24
+ sessions, auth, streaming, deployment) belongs to the consuming application. Anything
25
+ that cannot be reached through the public API surface does not belong in the core.
26
+ Every feature proposal must pass the one-sentence positioning test:
27
+
28
+ > "A minimal RAG pipeline: it is the engine — not the app, not the framework."
29
+
30
+ ### II. Two-Phase Architecture
31
+
32
+ Every module belongs to exactly one phase: Phase 1 (ingest, offline, run once per
33
+ batch) or Phase 2 (query, online, run per question). Shared modules (types, config,
34
+ store, manifest) must serve both phases without blurring the boundary. Phase 1 work
35
+ must never degrade Phase 2 latency; Phase 2 must treat the store and manifest as
36
+ read-only. The file tree must make the phase of every module legible without opening it.
37
+
38
+ ### III. Library Purity
39
+
40
+ No module in the library reads a user config or policy file. cli.py is the only file
41
+ that may; runtime configuration and PII policies MUST be passed programmatically to
42
+ library entry points. Chunking profiles are Python constants, not files. Provider
43
+ credentials come exclusively from environment variables — never from code, config, logs,
44
+ or error messages. There is no provider registry in the codebase; provider resolution is
45
+ delegated to the unified client via `provider/model` strings. Adding a provider must
46
+ require zero code changes.
47
+
48
+ ### IV. Search Small, Return Big (NON-NEGOTIABLE)
49
+
50
+ Retrieval must match child chunks and return parent sections. Every child must
51
+ carry a `parent_id`. Parents must be stored with a `type` field that excludes them
52
+ from similarity search via an explicit filter. Parent sections must be deduplicated
53
+ before being returned. This principle exists because incomplete retrieval can
54
+ *invert the meaning* of a rule — not merely reduce precision. A system that returns
55
+ "20 days of leave" without "except for new hires" is not imprecise; it is wrong.
56
+
57
+ ### V. Exception Preservation
58
+
59
+ Any retrieval context containing a rule with an exception marker (`except`,
60
+ `unless`, `not entitled`, `subject to`, `provided that`) must also contain that
61
+ exception. This is enforced by the reverse test against a fixture policy corpus,
62
+ which serves as mandatory **Release Gate #1** (`@pytest.mark.reverse`). PII ingestion
63
+ safeguards, outbound egress protection, and zero-raw-PII audit guarantees are enforced
64
+ by the PII safety suite as mandatory **Release Gate #2** (`@pytest.mark.pii`). A
65
+ release that fails Release Gate #1 or Release Gate #2 MUST NOT be published, regardless
66
+ of other passing tests; safety claims are only credible when verified by automated gates.
67
+
68
+ ### VI. Provider Agnosticism and Local-First Default
69
+
70
+ Embeddings and generation are addressed by `provider/model` strings through a
71
+ single unified interface. The default generation model is local, so the
72
+ out-of-the-box configuration requires no third-party account and no network access.
73
+ The full pipeline must be runnable with zero external network calls when configured
74
+ with local providers. Privacy is a default, not an opt-in.
75
+
76
+ ### VII. Structured Correctness Over Speed
77
+
78
+ Chunking must be structure-aware: split on headings, then sections, then clauses,
79
+ then sentences, and only as a last resort on characters. No clause may be split
80
+ across chunks in policy profiles. Overlap is scoped to a parent section and resets
81
+ at section boundaries. Crossrefs are resolved or logged — never silently dropped.
82
+ Data shapes are defined once in `types.py` and imported everywhere; schema drift
83
+ across modules is prohibited.
84
+
85
+ ## Additional Constraints
86
+
87
+ ### Data and Privacy
88
+
89
+ - Credentials are environment variables only. No credential may appear in source,
90
+ config, logs, or error output.
91
+ - Local execution must require no outbound network call. When strict local-only mode is
92
+ requested, any remote LLM provider or non-loopback `api_base` MUST be rejected with a
93
+ typed `DocpipeError` before any network request is made.
94
+ - Every retrieved section must carry metadata sufficient to cite source document,
95
+ section, and page.
96
+ - Ingestion must be deterministic (identical inputs → identical content-addressed
97
+ chunk IDs) and incremental:
98
+ - Changes are tracked via a database-collocated ingestion manifest using content
99
+ hashes (`sha256`), with `mtime` and `size` used strictly as fast-path hints.
100
+ - Updates use source-scoped **delete-then-reinsert** (never ID-only upsert) wrapped
101
+ in a staging/version-flip envelope so shrinking documents never leave orphaned
102
+ clauses and failed re-ingestions leave the prior version intact and queryable.
103
+ - Collections lock their embedding model identifier in the manifest; model
104
+ mismatches between ingest and query are refused unless explicitly reindexed.
105
+ - When a PII policy is configured during ingestion:
106
+ - Detection and transformation MUST execute pre-chunking across both section titles
107
+ and body text so redacted or tokenized values never leak into child context headers
108
+ or stored chunks.
109
+ - Block-mode matches MUST halt ingestion via a typed `DocpipeError` before any vector
110
+ store write or manifest commit, leaving prior collection state intact.
111
+ - Tokenization MUST use a keyed/salted cryptographic digest of the matched value
112
+ (never character-offset hashing) so repeated identifiers correlate deterministically
113
+ without exposing plaintext.
114
+ - Errors, telemetry, and PII audit logs (`.audit.jsonl`) MUST contain zero raw PII
115
+ values (counts, entity types, actions, source keys, and document version digests
116
+ only), and audit log storage MUST restrict directory permissions (`0o700` on POSIX
117
+ systems).
118
+ - When remote LLM completion is invoked outside local-only mode, outbound prompts and
119
+ role-separated chat messages MUST pass through the configured PII egress policy prior
120
+ to transmission. Documentation MUST NOT overclaim "100% accurate detection" or blanket
121
+ regulatory compliance (e.g., "GDPR compliant").
122
+
123
+ ### Quality Gates
124
+
125
+ - Every behavioral change ships with tests. Running **all tests** across the entire
126
+ suite after implementation is mandatory, and **all tests must pass** (zero
127
+ failures). CI runs linting, static type checking, and the full suite; any
128
+ failing test or red CI blocks completion and release.
129
+ - **Release Gate #1 (mandatory)**: The exception-preservation reverse test
130
+ (`@pytest.mark.reverse`) must pass prior to every release.
131
+ - **Release Gate #2 (mandatory)**: The PII protection safety suite (`@pytest.mark.pii`)
132
+ must pass prior to every release, verifying ingestion block/redaction/tokenization
133
+ invariants, outbound PII egress guarding, local-only remote-call refusal, and
134
+ zero-raw-PII audit logs.
135
+ - The package must install from TestPyPI in a clean environment before any PyPI
136
+ release, and publishing uses trusted publishing rather than long-lived tokens.
137
+ - The public API surface is the contract: `ingest`, `sync`, `remove`, `status`,
138
+ `reindex`, `retrieve`, `build_prompt`, `build_chat_messages`, optional `answer`,
139
+ `answer_detailed`, extension decorators (`register_parser`, `register_cleaner`,
140
+ `register_enricher`, `register_chunker`, `register_embedder`, `register_store`,
141
+ `register_searcher`, `register_reranker`, `register_pii_recognizer`), shared data/result shapes (`Chunk`,
142
+ `EmbeddedChunk`, `RetrievedChunk`, `SourceRecord`, `IngestResult`, `SyncResult`,
143
+ `CollectionStatus`, `GenerationResult`, `PiiPolicy`), public exceptions (`DocpipeError`,
144
+ `MissingExtraError`, `UnsupportedFormatError`, `EmptyDocumentError`,
145
+ `ModelMismatchError`, `MissingCredentialError`, `RegistryError`, `IngestionError`,
146
+ `GenerationError`, `PiiBlockedError`), and `__version__`. If a feature cannot be reached through that
147
+ surface, it does not belong in the core.
148
+
149
+ ### Footprint
150
+
151
+ - The default local configuration must run comfortably on 16 GB RAM with no GPU.
152
+ - The core install stays lean; generation is **optional to invoke** at runtime (Phase 1
153
+ ingestion and Phase 2 retrieval work offline with zero LLM completion calls), while
154
+ the unified client library handles both embeddings and completions in the core install.
155
+ - Built-in pattern/checksum PII recognizers MUST operate offline in the core install with
156
+ zero automatic model downloads; optional NER-based PII dependencies MUST stay isolated
157
+ behind an optional extra and load lazily only when requested.
158
+ - A new file format or storage backend must be additive — no existing module should
159
+ need modification beyond registration.
160
+
161
+ ## Governance
162
+
163
+ - This constitution supersedes feature convenience. A feature that violates a
164
+ principle must be rejected, or the constitution amended first — never quietly
165
+ worked around.
166
+ - Amendments require a documented rationale, a version bump, and a migration note
167
+ describing what existing code or consumers must change.
168
+ - Every spec, plan, and task must include a "Constitution Check" confirming
169
+ compliance with Principles I–VII before implementation begins.
170
+ - The one-sentence positioning is the tiebreaker for any scope dispute:
171
+
172
+ > "docpipe-maestro is a minimal RAG pipeline: it ingests any document, chunks it
173
+ > correctly, embeds it, stores it, retrieves the right sections, and optionally
174
+ > generates a grounded answer. It is the engine — not the app, not the framework."
175
+
176
+ **Version**: 1.3.1 | **Ratified**: 2026-11-12 | **Last Amended**: 2026-11-15
@@ -0,0 +1,43 @@
1
+ ## v0.2.0 (2026-11-17)
2
+
3
+ ### Feat
4
+
5
+ - **log**: add opt-in structured JSON logging and `ContextVar`-backed `correlation_id()` tracing via submodule `docpipe_maestro.log` (`JsonFormatter`, `TextFormatter`, `get_logger()` with passive `NullHandler`-only defaults, `stage_event()` timing helper with `isEnabledFor()` short-circuiting), instrument Phase 1 (`ingest`, `sync`, `remove`, `status`, `reindex` with `INFO` batch summaries and `DEBUG` per-doc details across `intake`/`clean`/`enrich`/`chunk`/`embed`/`store`) and Phase 2 (`retrieve`, searchers, `rerank`, `guard_egress`, `answer_detailed`, `evaluate` across `embed`/`search`/`rerank`/`egress`/`answer`) with traceback-free `ERROR` `error_type` failure attribution and zero raw PII/content/secret emission, and add CLI-only global `--log-json` / `--log-level {DEBUG,INFO,WARNING,ERROR}` flags writing strictly to `stderr`
6
+ - **retrieve**: add structured metadata pre-filtering via `retrieve(..., where=...)`, `evaluate(..., where=...)`, and repeatable CLI `--where KEY=VALUE` across `query`, `ask`, and `eval`, supporting stored chunk fields (`source`, `section`, `clause`, `page`, `chunk_index`, `doc_version`, `embed_model`), top-level manifest attributes (`profile` case-insensitive, UTC ISO-8601 `ingested_after` / `ingested_before` inclusive bounds) with zero re-ingestion migration and zero-embedder empty short-circuiting, full 10-operator Chroma + FTS5 SQLite parity (`$eq`, `$ne`, `$in`, `$nin`, `$gt`, `$gte`, `$lt`, `$lte`, `$and`, `$or`), and upfront unknown-key / reserved `"type"` validation with store-level `{"$and": [{"type": "child"}, ...]}` defense-in-depth
7
+ - **pii**: add configurable pre-chunking `PiiEnricher` (`redact`, `tokenize`, `block`) across headings, body text, and clauses, 7 built-in deterministic recognizers + extensible `@register_pii_recognizer` (`docpipe_maestro.register_pii_recognizer`, `PiiRecognizer`), lazy optional `[pii]` multi-language Presidio NER (`en`, `es`, etc. with zero runtime downloads), salted value-based HMAC tokenization, owner-only (`0o700`/`0o600`) zero-raw-PII `<collection>.audit.jsonl` logs with transactional `PiiBlockedError` rollback, shared `--local-only` / `local_only` preflight across embedding and generation workflows, Phase 2 `guard_egress` for prompt strings and ChatML messages, CLI `--pii-policy` JSON/TOML support, and Release Gate #2 (`@pytest.mark.pii`)
8
+ - **enricher**: formalize Phase 1 post-clean, pre-chunk `Enricher` stage (`ChunkingProfile.enrichers`, ordered execution with duplicate support, upfront `ENRICHERS` resolution, atomic abort guarantees, `IngestContext`), migrate intra-document cross-reference resolution into lazy `CrossrefsEnricher` (`"crossrefs"`), extract pure shared cross-reference helpers into `docpipe_maestro._crossrefs`, and keep silent `docpipe_maestro.ingest.crossrefs` compatibility re-exports
9
+ - **rerank**: add opt-in child-first local cross-encoder reranking (`--rerank`, `retrieve(..., rerank=True)`, `evaluate(..., rerank=True)`), optional `[rerank]` extra (`sentence-transformers`), lazy `CrossEncoderReranker` (`BAAI/bge-reranker-base`) with process-local model caching, CLI/TOML configuration, and live `Q1` benchmark validation
10
+ - **eval**: add read-only retrieval evaluation harness (`docpipe_maestro.eval`, `docpipe-maestro eval`), curated policy benchmark corpus, and offline `default` vs `fts5_hybrid` baselines
11
+ - **searcher**: add hybrid SQLite FTS5 BM25 + dense vector RRF search (`--searcher fts5_hybrid`) and collocated `LexicalIndex` lifecycle sync
12
+ - **query**: add online LLM customization (`temperature`, `max_tokens`, `timeout`, `model_kwargs`), `build_chat_messages()` role separation, `answer_detailed()` + `GenerationResult`, CLI/TOML `ask` options (`--system-prompt`, `--system-prompt-file`, `--fail-on-truncation`, `--show-context`, `--json`), loopback-vs-HTTPS `api_base` transport validation, scoped CLI `.env` + `./docpipe.toml` auto-discovery (`--no-config`), direct Ollama `/api/embed` fast path, digit-normalized header/footer stripping, and cross-document `follow_crossrefs` retrieval
13
+ - **cli**: add `docpipe-maestro query --json` structured retrieval output envelope
14
+
15
+ ### Refactor
16
+
17
+ - **pipeline**: introduce Strategy + Registry extension architecture (`register_*` entry points)
18
+
19
+ ### Fix
20
+
21
+ - **query**: improve multi-section prompt synthesis and report explicit `finish_reason="length"` / `reasoning_tokens` diagnostics on empty model responses
22
+ - **eval**: use `math.fsum` in aggregate metrics for Python 3.12+ determinism
23
+ - **store**: skip unnecessary HNSW vector reads in `get_all()` and release test Chroma database handles cleanly on Windows
24
+ - **cross-os**: normalize LF line endings across fixtures via `.gitattributes`, handle `float32` / `os.PathLike` checks across macOS and Windows, and advance `mtime` explicitly in same-size manifest fast-path tests
25
+
26
+ ### CI & Docs
27
+
28
+ - **workflow**: harden GitHub Actions CI matrix (`3.10`–`3.13`, `--no-extras` core smoke, `uv build` + `twine check --strict`, `pip-audit`, macOS/Windows smoke) and add OIDC Trusted Publishing release workflow (`.github/workflows/release.yml`)
29
+ - **docs**: enrich `README.md`, `docs/` guides, `CONTRIBUTING.md`, `SECURITY.md`, and `PRODUCT.md`, plus add Contributor Covenant Code of Conduct and `.env.example`
30
+
31
+ ## v0.1.0 (2026-11-13)
32
+
33
+ ### Feat
34
+
35
+ - **pipeline**: implement v0.1 core RAG pipeline, CLI, and quality gates
36
+
37
+ ### Fix
38
+
39
+ - **security**: harden pipeline, add CI workflow, and update gitignore
40
+
41
+ ### Refactor
42
+
43
+ - **cli**: rename CLI command to docpipe-maestro
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 docpipe-maestro contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.