haiku.rag 0.7.5__tar.gz → 0.19.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. haiku_rag-0.19.6/.dockerignore +66 -0
  2. haiku_rag-0.19.6/.gitignore +31 -0
  3. {haiku_rag-0.7.5 → haiku_rag-0.19.6}/.pre-commit-config.yaml +2 -12
  4. haiku_rag-0.19.6/.python-version +1 -0
  5. haiku_rag-0.19.6/CHANGELOG.md +398 -0
  6. haiku_rag-0.19.6/PKG-INFO +212 -0
  7. haiku_rag-0.19.6/README.md +188 -0
  8. {haiku_rag-0.7.5 → haiku_rag-0.19.6}/mkdocs.yml +20 -5
  9. {haiku_rag-0.7.5 → haiku_rag-0.19.6}/pyproject.toml +55 -30
  10. haiku_rag-0.19.6/scripts/build-docker-images.sh +38 -0
  11. haiku_rag-0.19.6/scripts/bump_version.py +155 -0
  12. haiku_rag-0.19.6/server.json +42 -0
  13. haiku_rag-0.19.6/uv.lock +5509 -0
  14. haiku_rag-0.7.5/.github/FUNDING.yml +0 -3
  15. haiku_rag-0.7.5/.github/workflows/build-docs.yml +0 -28
  16. haiku_rag-0.7.5/.github/workflows/build-publish.yml +0 -18
  17. haiku_rag-0.7.5/.gitignore +0 -20
  18. haiku_rag-0.7.5/.python-version +0 -1
  19. haiku_rag-0.7.5/PKG-INFO +0 -131
  20. haiku_rag-0.7.5/README.md +0 -93
  21. haiku_rag-0.7.5/docs/benchmarks.md +0 -36
  22. haiku_rag-0.7.5/docs/cli.md +0 -142
  23. haiku_rag-0.7.5/docs/configuration.md +0 -225
  24. haiku_rag-0.7.5/docs/index.md +0 -62
  25. haiku_rag-0.7.5/docs/installation.md +0 -74
  26. haiku_rag-0.7.5/docs/mcp.md +0 -33
  27. haiku_rag-0.7.5/docs/python.md +0 -188
  28. haiku_rag-0.7.5/docs/server.md +0 -42
  29. haiku_rag-0.7.5/src/haiku/rag/__init__.py +0 -0
  30. haiku_rag-0.7.5/src/haiku/rag/app.py +0 -183
  31. haiku_rag-0.7.5/src/haiku/rag/chunker.py +0 -51
  32. haiku_rag-0.7.5/src/haiku/rag/cli.py +0 -311
  33. haiku_rag-0.7.5/src/haiku/rag/client.py +0 -555
  34. haiku_rag-0.7.5/src/haiku/rag/config.py +0 -67
  35. haiku_rag-0.7.5/src/haiku/rag/embeddings/__init__.py +0 -30
  36. haiku_rag-0.7.5/src/haiku/rag/embeddings/base.py +0 -15
  37. haiku_rag-0.7.5/src/haiku/rag/embeddings/ollama.py +0 -17
  38. haiku_rag-0.7.5/src/haiku/rag/embeddings/openai.py +0 -16
  39. haiku_rag-0.7.5/src/haiku/rag/embeddings/vllm.py +0 -19
  40. haiku_rag-0.7.5/src/haiku/rag/embeddings/voyageai.py +0 -17
  41. haiku_rag-0.7.5/src/haiku/rag/logging.py +0 -53
  42. haiku_rag-0.7.5/src/haiku/rag/mcp.py +0 -144
  43. haiku_rag-0.7.5/src/haiku/rag/migration.py +0 -316
  44. haiku_rag-0.7.5/src/haiku/rag/monitor.py +0 -73
  45. haiku_rag-0.7.5/src/haiku/rag/qa/__init__.py +0 -15
  46. haiku_rag-0.7.5/src/haiku/rag/qa/agent.py +0 -84
  47. haiku_rag-0.7.5/src/haiku/rag/qa/prompts.py +0 -60
  48. haiku_rag-0.7.5/src/haiku/rag/reader.py +0 -115
  49. haiku_rag-0.7.5/src/haiku/rag/reranking/__init__.py +0 -34
  50. haiku_rag-0.7.5/src/haiku/rag/reranking/base.py +0 -13
  51. haiku_rag-0.7.5/src/haiku/rag/reranking/cohere.py +0 -34
  52. haiku_rag-0.7.5/src/haiku/rag/reranking/mxbai.py +0 -28
  53. haiku_rag-0.7.5/src/haiku/rag/reranking/vllm.py +0 -44
  54. haiku_rag-0.7.5/src/haiku/rag/store/__init__.py +0 -4
  55. haiku_rag-0.7.5/src/haiku/rag/store/engine.py +0 -203
  56. haiku_rag-0.7.5/src/haiku/rag/store/models/__init__.py +0 -4
  57. haiku_rag-0.7.5/src/haiku/rag/store/models/chunk.py +0 -15
  58. haiku_rag-0.7.5/src/haiku/rag/store/models/document.py +0 -16
  59. haiku_rag-0.7.5/src/haiku/rag/store/repositories/__init__.py +0 -9
  60. haiku_rag-0.7.5/src/haiku/rag/store/repositories/chunk.py +0 -375
  61. haiku_rag-0.7.5/src/haiku/rag/store/repositories/document.py +0 -214
  62. haiku_rag-0.7.5/src/haiku/rag/store/repositories/settings.py +0 -143
  63. haiku_rag-0.7.5/src/haiku/rag/store/upgrades/__init__.py +0 -1
  64. haiku_rag-0.7.5/src/haiku/rag/utils.py +0 -108
  65. haiku_rag-0.7.5/tests/__init__.py +0 -0
  66. haiku_rag-0.7.5/tests/conftest.py +0 -26
  67. haiku_rag-0.7.5/tests/generate_benchmark_db.py +0 -166
  68. haiku_rag-0.7.5/tests/llm_judge.py +0 -76
  69. haiku_rag-0.7.5/tests/test_app.py +0 -241
  70. haiku_rag-0.7.5/tests/test_chunk.py +0 -197
  71. haiku_rag-0.7.5/tests/test_chunker.py +0 -39
  72. haiku_rag-0.7.5/tests/test_cli.py +0 -157
  73. haiku_rag-0.7.5/tests/test_client.py +0 -752
  74. haiku_rag-0.7.5/tests/test_document.py +0 -108
  75. haiku_rag-0.7.5/tests/test_embedder.py +0 -171
  76. haiku_rag-0.7.5/tests/test_lancedb_connection.py +0 -86
  77. haiku_rag-0.7.5/tests/test_monitor.py +0 -93
  78. haiku_rag-0.7.5/tests/test_qa.py +0 -106
  79. haiku_rag-0.7.5/tests/test_reader.py +0 -23
  80. haiku_rag-0.7.5/tests/test_rebuild.py +0 -49
  81. haiku_rag-0.7.5/tests/test_reranker.py +0 -89
  82. haiku_rag-0.7.5/tests/test_search.py +0 -176
  83. haiku_rag-0.7.5/tests/test_settings.py +0 -84
  84. haiku_rag-0.7.5/tests/test_utils.py +0 -115
  85. haiku_rag-0.7.5/uv.lock +0 -3830
  86. {haiku_rag-0.7.5 → haiku_rag-0.19.6}/LICENSE +0 -0
@@ -0,0 +1,66 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ build/
8
+ develop-eggs/
9
+ dist/
10
+ downloads/
11
+ eggs/
12
+ .eggs/
13
+ lib/
14
+ lib64/
15
+ parts/
16
+ sdist/
17
+ var/
18
+ wheels/
19
+ *.egg-info/
20
+ .installed.cfg
21
+ *.egg
22
+
23
+ # Virtual environments (uv best practice)
24
+ .venv/
25
+ venv/
26
+ env/
27
+
28
+ # Node.js
29
+ node_modules/
30
+ .next/
31
+ npm-debug.log*
32
+ yarn-debug.log*
33
+ yarn-error.log*
34
+
35
+ # Data
36
+ *.lancedb/
37
+ data/
38
+
39
+ # Docs
40
+ mkdocs.yml
41
+ docs/
42
+
43
+ # IDE
44
+ .vscode/
45
+ .idea/
46
+ *.swp
47
+ *.swo
48
+ *~
49
+
50
+ # OS
51
+ .DS_Store
52
+ Thumbs.db
53
+
54
+ # Git
55
+ .git/
56
+ .gitignore
57
+
58
+ # Development
59
+ tests/
60
+ .pytest_cache/
61
+ .coverage
62
+ htmlcov/
63
+ evaluations/evaluations/data/
64
+ server.json
65
+ # Examples
66
+ examples/
@@ -0,0 +1,31 @@
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+
12
+ # tests
13
+ .coverage*
14
+ evaluations/evaluations/data/
15
+ tests/data/
16
+ .pytest_cache/
17
+ .ruff_cache/
18
+
19
+ # environment variables and config files
20
+ .env
21
+ haiku.rag.yaml
22
+ TODO.md
23
+ PLAN.md
24
+ DEVNOTES.md
25
+
26
+ # mcp registry
27
+ .mcpregistry_github_token
28
+ .mcpregistry_registry_token
29
+
30
+ # MkDocs site directory when doing local docs builds
31
+ site/
@@ -9,7 +9,7 @@ repos:
9
9
  - id: debug-statements
10
10
  - repo: https://github.com/astral-sh/ruff-pre-commit
11
11
  # Ruff version.
12
- rev: v0.11.4
12
+ rev: v0.14.3
13
13
  hooks:
14
14
  # Run the linter.
15
15
  - id: ruff
@@ -17,16 +17,6 @@ repos:
17
17
  - id: ruff-format
18
18
 
19
19
  - repo: https://github.com/RobertCraigie/pyright-python
20
- rev: v1.1.399
20
+ rev: v1.1.407
21
21
  hooks:
22
22
  - id: pyright
23
-
24
- # - repo: https://github.com/RodrigoGonzalez/check-mkdocs
25
- # rev: v1.2.0
26
- # hooks:
27
- # - id: check-mkdocs
28
- # name: check-mkdocs
29
- # args: ["--config", "mkdocs.yml"] # Optional, mkdocs.yml is the default
30
- # # If you have additional plugins or libraries that are not included in
31
- # # check-mkdocs, add them here
32
- # additional_dependencies: ["mkdocs-material"]
@@ -0,0 +1 @@
1
+ 3.13
@@ -0,0 +1,398 @@
1
+ # Changelog
2
+ ## [Unreleased]
3
+
4
+ ## [0.19.6] - 2025-12-03
5
+
6
+ ## [0.19.6] - 2025-12-03
7
+
8
+ ### Changed
9
+
10
+ - **BREAKING: Explicit Database Creation**: Databases must now be explicitly created before use
11
+ - New `haiku-rag init` command creates a new empty database
12
+ - Python API: `HaikuRAG(path, create=True)` to create database programmatically
13
+ - Operations on non-existent databases raise `FileNotFoundError`
14
+ - **BREAKING: Embeddings Configuration**: Restructured to nested `EmbeddingModelConfig`
15
+ - Config path changed from `embeddings.{provider, model, vector_dim}` to `embeddings.model.{provider, name, vector_dim}`
16
+ - Automatic migration upgrades existing databases to new format
17
+ - **Database Migrations**: Always run when opening an existing database
18
+
19
+ ## [0.19.5] - 2025-12-01
20
+
21
+ ### Changed
22
+
23
+ - **Rebuild Performance**: Optimized `rebuild --embed-only` to use batch updates via LanceDB's `merge_insert` instead of individual chunk updates, and skip chunks with unchanged embeddings
24
+
25
+ ## [0.19.4] - 2025-11-28
26
+
27
+ ### Added
28
+
29
+ - **Rebuild Modes**: New options for `rebuild` command to control what gets rebuilt
30
+ - `--embed-only`: Only regenerate embeddings, keeping existing chunks (fastest option when changing embedding model)
31
+ - `--rechunk`: Re-chunk from existing document content without accessing source files
32
+ - Default (no flag): Full rebuild with source file re-conversion
33
+ - Python API: `rebuild_database(mode=RebuildMode.EMBED_ONLY | RECHUNK | FULL)`
34
+
35
+ ## [0.19.3] - 2025-11-27
36
+
37
+ ### Changed
38
+
39
+ - **Async Chunker**: `DoclingServeChunker` now uses `httpx.AsyncClient` instead of sync `requests`
40
+
41
+ ### Fixed
42
+
43
+ - **OCR Options**: Fixed `DoclingLocalConverter` using base `OcrOptions` class which docling's OCR factory doesn't recognize. Now uses `OcrAutoOptions` for automatic OCR engine selection.
44
+ - **Dependencies**: Added `opencv-python-headless` to the `docling` optional dependency for table structure detection.
45
+
46
+ ## [0.19.2] - 2025-11-27
47
+
48
+ ### Changed
49
+
50
+ - **Async Converters**: Made document converters fully async
51
+ - `BaseConverter.convert_file()` and `convert_text()` are now async methods
52
+ - `DoclingLocalConverter` wraps blocking Docling operations with `asyncio.to_thread()`
53
+ - `DoclingServeConverter` now uses `httpx.AsyncClient` instead of sync `requests`
54
+ - **Async Model Prefetch**: `prefetch_models()` is now async
55
+ - Uses `httpx.AsyncClient` for Ollama model pulls
56
+ - Wraps blocking Docling and HuggingFace downloads with `asyncio.to_thread()`
57
+
58
+ ## [0.19.1] - 2025-11-26
59
+
60
+ ### Added
61
+
62
+ - **LM Studio Provider**: Added support for LM Studio as a provider for embeddings and QA/research models
63
+ - Configure with `provider: lm_studio` in embeddings, QA, or research model settings
64
+ - Supports thinking control for reasoning models (gpt-oss, etc.)
65
+ - Default base URL: `http://localhost:1234`
66
+
67
+ ### Fixed
68
+
69
+ - **Configuration**: Fixed `init-config` command generating invalid configuration files (#165)
70
+ - Refactored `generate_default_config()` to use Pydantic model serialization instead of manual dict construction
71
+ - Updated `qa`, `research`, and `reranking` sections to use new `ModelConfig` structure
72
+
73
+ ## [0.19.0] - 2025-11-25
74
+
75
+ ### Added
76
+
77
+ - **Model Customization**: Added support for per-model configuration settings
78
+ - New `enable_thinking` parameter to control reasoning behavior (true/false/None)
79
+ - Support for `temperature` and `max_tokens` settings on QA and research models
80
+ - All settings apply to any provider that supports them
81
+ - **Database Inspector**: New `inspect` CLI command launches interactive TUI for browsing documents and chunks & searching
82
+ - **Evaluations**: Added `evaluations` CLI script for running benchmarks (replaces `python -m evaluations.benchmark`)
83
+ - **Evaluations**: Added `--db` option to override evaluation database path
84
+ - Default database location moved to haiku.rag data directory:
85
+ - macOS: `~/Library/Application Support/haiku.rag/evaluations/dbs/`
86
+ - Linux: `~/.local/share/haiku.rag/evaluations/dbs/`
87
+ - Windows: `C:/Users/<USER>/AppData/Roaming/haiku.rag/evaluations/dbs/`
88
+ - Previously stored in `evaluations/data/` within the repository
89
+ - **Evaluations**: Added comprehensive experiment metadata tracking for better reproducibility
90
+ - Records dataset name, test case count, and all model configurations
91
+ - Tracks embedder settings: provider, model, and vector dimensions
92
+ - Tracks QA model: provider and model name
93
+ - Tracks judge model: provider and model name for LLM evaluation
94
+ - Tracks processing parameters: `chunk_size` and `context_chunk_radius`
95
+ - Tracks retrieval configuration: `retrieval_limit` for number of chunks retrieved
96
+ - Tracks reranking configuration: `rerank_provider` and `rerank_model`
97
+ - Enables comparison of evaluation runs with different configurations in Logfire
98
+ - **Evaluations**: Refactored retrieval evaluation to use pydantic-ai experiment framework
99
+ - New `evaluators` module with `MRREvaluator` (Mean Reciprocal Rank) and `MAPEvaluator` (Mean Average Precision)
100
+ - Retrieval benchmarks now use `Dataset.evaluate()` with full Logfire experiment tracking
101
+ - Dataset specifications now declare their retrieval evaluator (MRR for RepliQA, MAP for Wix)
102
+ - Replaced Recall@K and Success@K with industry-standard MRR and MAP metrics
103
+ - Unified evaluation framework for both retrieval and QA benchmarks
104
+ - **AG-UI Events**: Enhanced ActivitySnapshot events with richer structured data
105
+ - Added `stepName` field to identify which graph node emitted each activity
106
+ - Added structured fields to activity content while preserving backward-compatible `message` field:
107
+ - **Planning**: `sub_questions` - list of sub-question strings
108
+ - **Searching**: `query` - the search query, `confidence` - answer confidence (on success), `error` - error message (on failure)
109
+ - **Analyzing** (research): `insights` - list of insight objects, `gaps` - list of gap objects, `resolved_gaps` - list of resolved gap strings
110
+ - **Evaluating** (research): `confidence` - confidence score, `is_sufficient` - sufficiency flag
111
+ - **Evaluating** (deep QA): `is_sufficient` - sufficiency flag, `iterations` - iteration count
112
+
113
+ ### Changed
114
+
115
+ - **Evaluations**: Renamed `--qa-limit` CLI parameter to `--limit`, now applies to both retrieval and QA benchmarks
116
+ - **Evaluations**: Retrieval evaluator selection moved from runtime logic to dataset configuration
117
+
118
+ ## [0.18.0] - 2025-11-21
119
+
120
+ ### Added
121
+
122
+ - **Manual Vector Indexing**: New `create-index` CLI command for explicit vector index creation
123
+ - Creates IVF_PQ indexes
124
+ - Requires minimum 256 chunks (LanceDB training data requirement)
125
+ - New `search.vector_index_metric` config option: `cosine` (default), `l2`, or `dot`
126
+ - New `search.vector_refine_factor` config option (default: 30) for accuracy/speed tradeoff
127
+ - Indexes not created automatically during ingestion to avoid performance degradation
128
+ - Manual rebuilding required after adding significant new data
129
+ - **Enhanced Info Command**: `haiku-rag info` now shows storage sizes and vector index statistics
130
+ - Displays storage size for documents and chunks tables in human-readable format
131
+ - Shows vector index status (exists/not created)
132
+ - Shows indexed and unindexed chunk counts for monitoring index staleness
133
+
134
+ ### Changed
135
+
136
+ - **BREAKING: Default Embedding Model**: Changed default embedding model from `qwen3-embedding` to `qwen3-embedding:4b` with vector dimension 2560 (previously 4096)
137
+ - New installations will use the smaller, more efficient 4B parameter model by default
138
+ - **Action required**: Existing databases created with the old default will be incompatible. Users must either:
139
+ - Explicitly set `embeddings.model: "qwen3-embedding"` and `embeddings.vector_dim: 4096` in their config to maintain compatibility with existing databases
140
+ - Or run `haiku-rag rebuild` to re-embed all documents with the new default
141
+ - This change provides better performance for most use cases while reducing resource requirements
142
+ - **Evaluations**: Improved evaluation dataset naming and simplified evaluator configuration
143
+ - `EvalDataset` now accepts dataset name for better organization in Logfire
144
+ - Added `--name` CLI parameter to override evaluation run names
145
+ - Removed `IsInstance` evaluator, using only `LLMJudge` for QA evaluation
146
+ - **Search Accuracy**: Applied `refine_factor` to vector and hybrid searches for improved accuracy
147
+ - Retrieves `refine_factor * limit` candidates and re-ranks in memory
148
+ - Higher values increase accuracy but slow down queries
149
+
150
+ ### Fixed
151
+
152
+ - **AG-UI Activity Events**: Activity events now correctly use structured dict content instead of strings
153
+ - **Graph Configuration**: Graph builder functions now properly accept and use non-global config (#149)
154
+ - `build_research_graph()` and `build_deep_qa_graph()` now pass config to all agents and model creation
155
+ - `get_model()` utility function accepts `config` parameter (defaults to global Config)
156
+ - Allows creating multiple graphs with different configurations in the same application
157
+
158
+
159
+ ## [0.17.2] - 2025-11-19
160
+
161
+ ### Added
162
+
163
+ - **Document Update API**: New `update_document_fields()` method for partial document updates
164
+ - Update individual fields (content, metadata, title, chunks) without fetching full document
165
+ - Support for custom chunks or auto-generation from content
166
+
167
+ ### Changed
168
+
169
+ - **Chunk Creation**: `ChunkRepository.create()` now accepts both single chunks and lists for batch insertion
170
+ - Batch insertion reduces LanceDB version creation when adding multiple chunks with custom chunks
171
+ - Batch embedding generation for improved performance with multiple chunks
172
+ - Updated core dependencies
173
+
174
+ ## [0.17.1] - 2025-11-18
175
+
176
+ ### Added
177
+
178
+ - **Conversion Options**: Fine-grained control over document conversion for both local and remote converters
179
+ - New `conversion_options` config section in `ProcessingConfig`
180
+ - OCR settings: `do_ocr`, `force_ocr`, `ocr_lang` for controlling OCR behavior
181
+ - Table extraction: `do_table_structure`, `table_mode` (fast/accurate), `table_cell_matching`
182
+ - Image settings: `images_scale` to control image resolution
183
+ - Options work identically with both `docling-local` and `docling-serve` converters
184
+
185
+ ### Changed
186
+
187
+ - Increase reranking candidate retrieval multiplier from 3x to 10x for improved result quality
188
+ - **Docker Images**: Main `haiku.rag` image no longer automatically built and published
189
+ - **Conversion Options**: Removed the legacy `pdf_backend` setting; docling now chooses the optimal backend automatically
190
+
191
+ ## [0.17.0] - 2025-11-17
192
+
193
+ ### Added
194
+
195
+ - **Remote Processing**: Support for docling-serve as remote document processing and chunking service
196
+ - New `converter` config option: `docling-local` (default) or `docling-serve`
197
+ - New `chunker` config option: `docling-local` (default) or `docling-serve`
198
+ - New `providers.docling_serve` config section with `base_url`, `api_key`, and `timeout`
199
+ - Comprehensive error handling for connection, timeout, and authentication issues
200
+ - **Chunking Strategies**: Support for both hybrid and hierarchical chunking
201
+ - New `chunker_type` config option: `hybrid` (default) or `hierarchical`
202
+ - Hybrid chunking: Structure-aware splitting that respects document boundaries
203
+ - Hierarchical chunking: Preserves document hierarchy for nested documents
204
+ - **Table Serialization Control**: Configurable table representation in chunks
205
+ - New `chunking_use_markdown_tables` config option (default: `false`)
206
+ - `false`: Tables serialized as narrative text ("Value A, Column 2 = Value B")
207
+ - `true`: Tables preserved as markdown format with structure
208
+ - **Chunking Configuration**: Additional chunking control options
209
+ - New `chunking_merge_peers` config option (default: `true`) to merge undersized successive chunks
210
+ - **Docker Images**: Two Docker images for different deployment scenarios
211
+ - `haiku.rag`: Full image with all dependencies for self-contained deployments
212
+ - `haiku.rag-slim`: Minimal image designed for use with external docling-serve
213
+ - Multi-platform support (linux/amd64, linux/arm64)
214
+ - Docker Compose examples with docling-serve integration
215
+ - Automated CI/CD workflows for both images
216
+ - Build script (`scripts/build-docker-images.sh`) for local multi-platform builds
217
+
218
+ ### Changed
219
+
220
+ - **BREAKING: Chunking Tokenizer**: Switched from tiktoken to HuggingFace tokenizers for consistency with docling-serve
221
+ - Default tokenizer changed from tiktoken "gpt-4o" to "Qwen/Qwen3-Embedding-0.6B"
222
+ - New `chunking_tokenizer` config option in `ProcessingConfig` for customization
223
+ - `download-models` CLI command now also downloads the configured HuggingFace tokenizer
224
+ - **Docker Examples**: Updated examples to demonstrate remote processing
225
+ - `examples/docker` now uses slim image with docling-serve
226
+ - `examples/ag-ui-research` backend uses slim image with docling-serve
227
+ - Configuration examples include remote processing setup
228
+
229
+ ## [0.16.1] - 2025-11-14
230
+
231
+ ### Changed
232
+
233
+ - **Evaluations**: Refactored QA benchmark to run entire dataset as single evaluation for better Logfire experiment tracking
234
+ - **Evaluations**: Added `.env` file loading support via `python-dotenv` dependency
235
+
236
+ ## [0.16.0] - 2025-11-13
237
+
238
+ ### Added
239
+
240
+ - **AG-UI Protocol Support**: Full AG-UI (Agent-UI) protocol implementation for graph execution with event streaming
241
+ - New `AGUIEmitter` class for emitting AG-UI events from graphs
242
+ - Support for all AG-UI event types: lifecycle events (`RUN_STARTED`, `RUN_FINISHED`, `RUN_ERROR`), step events (`STEP_STARTED`, `STEP_FINISHED`), state updates (`STATE_SNAPSHOT`, `STATE_DELTA`), activity narration (`ACTIVITY_SNAPSHOT`), and text messages (`TEXT_MESSAGE_CHUNK`)
243
+ - `AGUIConsoleRenderer` for rendering AG-UI event streams to terminal with Rich formatting
244
+ - `stream_graph()` utility function for executing graphs with AG-UI event emission
245
+ - State diff computation for efficient state synchronization
246
+ - **Delta State Updates**: AG-UI emitter now supports incremental state updates via JSON Patch operations (`STATE_DELTA` events) to reduce bandwidth, configurable via `use_deltas` parameter (enabled by default)
247
+ - **AG-UI Server**: Starlette-based HTTP server for serving graphs via AG-UI protocol
248
+ - Server-Sent Events (SSE) streaming endpoint at `/v1/agent/stream`
249
+ - Health check endpoint at `/health`
250
+ - Full CORS support configurable via `agui` config section
251
+ - `create_agui_server()` function for programmatic server creation
252
+ - **Deep QA AG-UI Support**: Deep QA graph now fully supports AG-UI event streaming
253
+ - Integration with `AGUIEmitter` for progress tracking
254
+ - Step-by-step execution visibility via AG-UI events
255
+ - **CLI AG-UI Flag**: New `--agui` flag for `serve` command to start AG-UI server
256
+ - **Graph Module**: New unified `haiku.rag.graph` module containing all graph-related functionality
257
+ - **Common Graph Nodes**: New factory functions (`create_plan_node`, `create_search_node`) in `haiku.rag.graph.common.nodes` for reusable graph components
258
+ - **AG-UI Research Example**: New full-stack example (`examples/ag-ui-research`) demonstrating agent+graph architecture with CopilotKit frontend
259
+ - Pydantic AI agent with research tool that invokes the research graph
260
+ - Custom AG-UI streaming endpoint with anyio memory streams
261
+ - React/Next.js frontend with split-pane UI showing live research state
262
+ - Real-time progress tracking of questions, answers, insights, and gaps
263
+ - Docker Compose setup for easy local development
264
+
265
+ ### Changed
266
+
267
+ - **Vacuum Retention**: Default `vacuum_retention_seconds` increased from 60 seconds to 86400 seconds (1 day) for better version retention in typical workflows
268
+ - **BREAKING**: Major refactoring of graph-related code into unified `haiku.rag.graph` module structure:
269
+ - `haiku.rag.research` → `haiku.rag.graph.research`
270
+ - `haiku.rag.qa.deep` → `haiku.rag.graph.deep_qa`
271
+ - `haiku.rag.agui` → `haiku.rag.graph.agui`
272
+ - `haiku.rag.graph_common` → `haiku.rag.graph.common`
273
+ - **BREAKING**: Research and Deep QA graphs now use AG-UI event protocol instead of direct console logging
274
+ - Removed `console` and `stream` parameters from graph dependencies
275
+ - All progress updates now emit through `AGUIEmitter`
276
+ - **BREAKING**: `ResearchState` converted from dataclass to Pydantic `BaseModel` for JSON serialization and AG-UI compatibility
277
+ - Research and Deep QA graphs now emit detailed execution events for better observability
278
+ - CLI research command now uses AG-UI event rendering for `--verbose` output
279
+ - Improved graph execution visibility with step-by-step progress tracking
280
+ - Updated all documentation to reflect new import paths and AG-UI usage
281
+ - Updated examples (ag-ui-research, a2a-server) to use new import paths
282
+
283
+ ### Fixed
284
+
285
+ - **Document Creation**: Optimized `create_document` to skip unnecessary DoclingDocument conversion when chunks are pre-provided
286
+ - **FileReader**: Error messages now include both original exception details and file path for easier debugging
287
+ - **Database Auto-creation**: Read operations (search, list, get, ask, research) no longer auto-create empty databases. Write operations (add, add-src, delete, rebuild) still create the database as needed. This prevents the confusing scenario where a search query creates an empty database. Fixes issue #137.
288
+
289
+ ### Removed
290
+
291
+ - **BREAKING**: Removed `disable_autocreate` config option - the behavior is now automatic based on operation type
292
+ - **BREAKING**: Removed legacy `ResearchStream` and `ResearchStreamEvent` classes (replaced by AG-UI event protocol)
293
+
294
+ ## [0.15.0] - 2025-11-07
295
+
296
+ ### Added
297
+
298
+ - **File Monitor**: Orphan deletion feature - automatically removes documents from database when source files are deleted (enabled via `monitor.delete_orphans` config option, default: false)
299
+
300
+ ### Changed
301
+
302
+ - **Configuration**: All CLI commands now properly support `--config` parameter for specifying custom configuration files
303
+ - Configuration loading consolidated across CLI, app, and client with consistent resolution order
304
+ - `HaikuRAGApp` and MCP server now accept `config` parameter for programmatic configuration
305
+ - Updated CLI documentation to clarify global vs per-command options
306
+ - **BREAKING**: Standardized configuration filename to `haiku.rag.yaml` in user directories (was incorrectly using `config.yaml`). Users with existing `config.yaml` in their user directory will need to rename it to `haiku.rag.yaml`
307
+
308
+ ### Fixed
309
+
310
+ - **File Monitor**: Fixed incorrect "Updated document" logging for unchanged files - monitor now properly skips files when MD5 hash hasn't changed
311
+
312
+ ### Removed
313
+
314
+ - **BREAKING**: A2A (Agent-to-Agent) protocol support has been moved to a separate self-contained package in `examples/a2a-server/`. The A2A server is no longer part of the main haiku.rag package. Users who need A2A functionality can install and run it from the examples directory with `cd examples/a2a-server && uv sync`.
315
+ - **BREAKING**: Removed deprecated `.env`-based configuration system. The `haiku-rag init-config --from-env` command and `load_config_from_env()` function have been removed. All configuration must now be done via YAML files. Environment variables for API keys (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) and service URLs (e.g., `OLLAMA_BASE_URL`) are still supported and can be set via `.env` files.
316
+
317
+ ## [0.14.1] - 2025-11-06
318
+
319
+ ### Added
320
+
321
+ - Migrated research and deep QA agents to use Pydantic Graph beta API for better graph execution
322
+ - Automatic semaphore-based concurrency control for parallel sub-question processing
323
+ - `max_concurrency` parameter for controlling parallel execution in research and deep QA (default: 1)
324
+
325
+ ### Changed
326
+
327
+ - **BREAKING**: Research and Deep QA graphs now use `pydantic_graph.beta` instead of the class-based graph implementation
328
+ - Refactored graph common patterns into `graph_common` module
329
+ - Sub-questions now process using `.map()` for true parallel execution
330
+ - Improved graph structure with cleaner node definitions and flow control
331
+ - Pinned critical dependencies: `docling-core`, `lancedb`, `docling`
332
+
333
+ ## [0.14.0] - 2024-11-05
334
+
335
+ ### Added
336
+
337
+ - New `haiku.rag-slim` package with minimal dependencies for users who want to install only what they need
338
+ - Evaluations package (`haiku.rag-evals`) for internal benchmarking and testing
339
+ - Improved search filtering performance by using pandas DataFrames for joins instead of SQL WHERE IN clauses
340
+
341
+ ### Changed
342
+
343
+ - **BREAKING**: Restructured project into UV workspace with three packages:
344
+ - `haiku.rag-slim` - Core package with minimal dependencies
345
+ - `haiku.rag` - Full package with all extras (recommended for most users)
346
+ - `haiku.rag-evals` - Internal benchmarking and evaluation tools
347
+ - Migrated from `pydantic-ai` to `pydantic-ai-slim` with extras system
348
+ - Docling is now an optional dependency (install with `haiku.rag-slim[docling]`)
349
+ - Package metadata checks now use `haiku.rag-slim` (always present) instead of `haiku.rag`
350
+ - Docker image optimized: removed evaluations package, reducing installed packages from 307 to 259
351
+ - Improved vector search performance through optimized score normalization
352
+
353
+ ### Fixed
354
+
355
+ - ImportError now properly raised when optional docling dependency is missing
356
+
357
+ ## [0.13.3] - 2024-11-04
358
+
359
+ ### Added
360
+
361
+ - Support for Zero Entropy reranker
362
+ - Filter parameter to `search()` for filtering documents before search
363
+ - Filter parameter to CLI `search` command
364
+ - Filter parameter to CLI `list` command for filtering document listings
365
+ - Config option to pass custom configuration files to evaluation commands
366
+ - Document filtering now respects configured include/exclude patterns when using `add-src` with directories
367
+ - Max retries to insight_agent when producing structured output
368
+
369
+ ### Fixed
370
+
371
+ - CLI now loads `.env` files at startup
372
+ - Info command no longer attempts to use deprecated `.env` settings
373
+ - Documentation typos
374
+
375
+ ## [0.13.2] - 2024-11-04
376
+
377
+ ### Added
378
+
379
+ - Gitignore-style pattern filtering for file monitoring using pathspec
380
+ - Include/exclude pattern documentation for FileMonitor
381
+
382
+ ### Changed
383
+
384
+ - Moved monitor configuration to its own section in config
385
+ - Improved configuration documentation
386
+ - Updated dependencies
387
+
388
+ ## [0.13.1] - 2024-11-03
389
+
390
+ ### Added
391
+
392
+ - Initial version tracking
393
+
394
+ [Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.14.0...HEAD
395
+ [0.14.0]: https://github.com/ggozad/haiku.rag/compare/0.13.3...0.14.0
396
+ [0.13.3]: https://github.com/ggozad/haiku.rag/compare/0.13.2...0.13.3
397
+ [0.13.2]: https://github.com/ggozad/haiku.rag/compare/0.13.1...0.13.2
398
+ [0.13.1]: https://github.com/ggozad/haiku.rag/releases/tag/0.13.1