haiku.rag 0.7.5__tar.gz → 0.19.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- haiku_rag-0.19.6/.dockerignore +66 -0
- haiku_rag-0.19.6/.gitignore +31 -0
- {haiku_rag-0.7.5 → haiku_rag-0.19.6}/.pre-commit-config.yaml +2 -12
- haiku_rag-0.19.6/.python-version +1 -0
- haiku_rag-0.19.6/CHANGELOG.md +398 -0
- haiku_rag-0.19.6/PKG-INFO +212 -0
- haiku_rag-0.19.6/README.md +188 -0
- {haiku_rag-0.7.5 → haiku_rag-0.19.6}/mkdocs.yml +20 -5
- {haiku_rag-0.7.5 → haiku_rag-0.19.6}/pyproject.toml +55 -30
- haiku_rag-0.19.6/scripts/build-docker-images.sh +38 -0
- haiku_rag-0.19.6/scripts/bump_version.py +155 -0
- haiku_rag-0.19.6/server.json +42 -0
- haiku_rag-0.19.6/uv.lock +5509 -0
- haiku_rag-0.7.5/.github/FUNDING.yml +0 -3
- haiku_rag-0.7.5/.github/workflows/build-docs.yml +0 -28
- haiku_rag-0.7.5/.github/workflows/build-publish.yml +0 -18
- haiku_rag-0.7.5/.gitignore +0 -20
- haiku_rag-0.7.5/.python-version +0 -1
- haiku_rag-0.7.5/PKG-INFO +0 -131
- haiku_rag-0.7.5/README.md +0 -93
- haiku_rag-0.7.5/docs/benchmarks.md +0 -36
- haiku_rag-0.7.5/docs/cli.md +0 -142
- haiku_rag-0.7.5/docs/configuration.md +0 -225
- haiku_rag-0.7.5/docs/index.md +0 -62
- haiku_rag-0.7.5/docs/installation.md +0 -74
- haiku_rag-0.7.5/docs/mcp.md +0 -33
- haiku_rag-0.7.5/docs/python.md +0 -188
- haiku_rag-0.7.5/docs/server.md +0 -42
- haiku_rag-0.7.5/src/haiku/rag/__init__.py +0 -0
- haiku_rag-0.7.5/src/haiku/rag/app.py +0 -183
- haiku_rag-0.7.5/src/haiku/rag/chunker.py +0 -51
- haiku_rag-0.7.5/src/haiku/rag/cli.py +0 -311
- haiku_rag-0.7.5/src/haiku/rag/client.py +0 -555
- haiku_rag-0.7.5/src/haiku/rag/config.py +0 -67
- haiku_rag-0.7.5/src/haiku/rag/embeddings/__init__.py +0 -30
- haiku_rag-0.7.5/src/haiku/rag/embeddings/base.py +0 -15
- haiku_rag-0.7.5/src/haiku/rag/embeddings/ollama.py +0 -17
- haiku_rag-0.7.5/src/haiku/rag/embeddings/openai.py +0 -16
- haiku_rag-0.7.5/src/haiku/rag/embeddings/vllm.py +0 -19
- haiku_rag-0.7.5/src/haiku/rag/embeddings/voyageai.py +0 -17
- haiku_rag-0.7.5/src/haiku/rag/logging.py +0 -53
- haiku_rag-0.7.5/src/haiku/rag/mcp.py +0 -144
- haiku_rag-0.7.5/src/haiku/rag/migration.py +0 -316
- haiku_rag-0.7.5/src/haiku/rag/monitor.py +0 -73
- haiku_rag-0.7.5/src/haiku/rag/qa/__init__.py +0 -15
- haiku_rag-0.7.5/src/haiku/rag/qa/agent.py +0 -84
- haiku_rag-0.7.5/src/haiku/rag/qa/prompts.py +0 -60
- haiku_rag-0.7.5/src/haiku/rag/reader.py +0 -115
- haiku_rag-0.7.5/src/haiku/rag/reranking/__init__.py +0 -34
- haiku_rag-0.7.5/src/haiku/rag/reranking/base.py +0 -13
- haiku_rag-0.7.5/src/haiku/rag/reranking/cohere.py +0 -34
- haiku_rag-0.7.5/src/haiku/rag/reranking/mxbai.py +0 -28
- haiku_rag-0.7.5/src/haiku/rag/reranking/vllm.py +0 -44
- haiku_rag-0.7.5/src/haiku/rag/store/__init__.py +0 -4
- haiku_rag-0.7.5/src/haiku/rag/store/engine.py +0 -203
- haiku_rag-0.7.5/src/haiku/rag/store/models/__init__.py +0 -4
- haiku_rag-0.7.5/src/haiku/rag/store/models/chunk.py +0 -15
- haiku_rag-0.7.5/src/haiku/rag/store/models/document.py +0 -16
- haiku_rag-0.7.5/src/haiku/rag/store/repositories/__init__.py +0 -9
- haiku_rag-0.7.5/src/haiku/rag/store/repositories/chunk.py +0 -375
- haiku_rag-0.7.5/src/haiku/rag/store/repositories/document.py +0 -214
- haiku_rag-0.7.5/src/haiku/rag/store/repositories/settings.py +0 -143
- haiku_rag-0.7.5/src/haiku/rag/store/upgrades/__init__.py +0 -1
- haiku_rag-0.7.5/src/haiku/rag/utils.py +0 -108
- haiku_rag-0.7.5/tests/__init__.py +0 -0
- haiku_rag-0.7.5/tests/conftest.py +0 -26
- haiku_rag-0.7.5/tests/generate_benchmark_db.py +0 -166
- haiku_rag-0.7.5/tests/llm_judge.py +0 -76
- haiku_rag-0.7.5/tests/test_app.py +0 -241
- haiku_rag-0.7.5/tests/test_chunk.py +0 -197
- haiku_rag-0.7.5/tests/test_chunker.py +0 -39
- haiku_rag-0.7.5/tests/test_cli.py +0 -157
- haiku_rag-0.7.5/tests/test_client.py +0 -752
- haiku_rag-0.7.5/tests/test_document.py +0 -108
- haiku_rag-0.7.5/tests/test_embedder.py +0 -171
- haiku_rag-0.7.5/tests/test_lancedb_connection.py +0 -86
- haiku_rag-0.7.5/tests/test_monitor.py +0 -93
- haiku_rag-0.7.5/tests/test_qa.py +0 -106
- haiku_rag-0.7.5/tests/test_reader.py +0 -23
- haiku_rag-0.7.5/tests/test_rebuild.py +0 -49
- haiku_rag-0.7.5/tests/test_reranker.py +0 -89
- haiku_rag-0.7.5/tests/test_search.py +0 -176
- haiku_rag-0.7.5/tests/test_settings.py +0 -84
- haiku_rag-0.7.5/tests/test_utils.py +0 -115
- haiku_rag-0.7.5/uv.lock +0 -3830
- {haiku_rag-0.7.5 → haiku_rag-0.19.6}/LICENSE +0 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
build/
|
|
8
|
+
develop-eggs/
|
|
9
|
+
dist/
|
|
10
|
+
downloads/
|
|
11
|
+
eggs/
|
|
12
|
+
.eggs/
|
|
13
|
+
lib/
|
|
14
|
+
lib64/
|
|
15
|
+
parts/
|
|
16
|
+
sdist/
|
|
17
|
+
var/
|
|
18
|
+
wheels/
|
|
19
|
+
*.egg-info/
|
|
20
|
+
.installed.cfg
|
|
21
|
+
*.egg
|
|
22
|
+
|
|
23
|
+
# Virtual environments (uv best practice)
|
|
24
|
+
.venv/
|
|
25
|
+
venv/
|
|
26
|
+
env/
|
|
27
|
+
|
|
28
|
+
# Node.js
|
|
29
|
+
node_modules/
|
|
30
|
+
.next/
|
|
31
|
+
npm-debug.log*
|
|
32
|
+
yarn-debug.log*
|
|
33
|
+
yarn-error.log*
|
|
34
|
+
|
|
35
|
+
# Data
|
|
36
|
+
*.lancedb/
|
|
37
|
+
data/
|
|
38
|
+
|
|
39
|
+
# Docs
|
|
40
|
+
mkdocs.yml
|
|
41
|
+
docs/
|
|
42
|
+
|
|
43
|
+
# IDE
|
|
44
|
+
.vscode/
|
|
45
|
+
.idea/
|
|
46
|
+
*.swp
|
|
47
|
+
*.swo
|
|
48
|
+
*~
|
|
49
|
+
|
|
50
|
+
# OS
|
|
51
|
+
.DS_Store
|
|
52
|
+
Thumbs.db
|
|
53
|
+
|
|
54
|
+
# Git
|
|
55
|
+
.git/
|
|
56
|
+
.gitignore
|
|
57
|
+
|
|
58
|
+
# Development
|
|
59
|
+
tests/
|
|
60
|
+
.pytest_cache/
|
|
61
|
+
.coverage
|
|
62
|
+
htmlcov/
|
|
63
|
+
evaluations/evaluations/data/
|
|
64
|
+
server.json
|
|
65
|
+
# Examples
|
|
66
|
+
examples/
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Python-generated files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info
|
|
8
|
+
|
|
9
|
+
# Virtual environments
|
|
10
|
+
.venv
|
|
11
|
+
|
|
12
|
+
# tests
|
|
13
|
+
.coverage*
|
|
14
|
+
evaluations/evaluations/data/
|
|
15
|
+
tests/data/
|
|
16
|
+
.pytest_cache/
|
|
17
|
+
.ruff_cache/
|
|
18
|
+
|
|
19
|
+
# environment variables and config files
|
|
20
|
+
.env
|
|
21
|
+
haiku.rag.yaml
|
|
22
|
+
TODO.md
|
|
23
|
+
PLAN.md
|
|
24
|
+
DEVNOTES.md
|
|
25
|
+
|
|
26
|
+
# mcp registry
|
|
27
|
+
.mcpregistry_github_token
|
|
28
|
+
.mcpregistry_registry_token
|
|
29
|
+
|
|
30
|
+
# MkDocs site directory when doing local docs builds
|
|
31
|
+
site/
|
|
@@ -9,7 +9,7 @@ repos:
|
|
|
9
9
|
- id: debug-statements
|
|
10
10
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
11
11
|
# Ruff version.
|
|
12
|
-
rev: v0.
|
|
12
|
+
rev: v0.14.3
|
|
13
13
|
hooks:
|
|
14
14
|
# Run the linter.
|
|
15
15
|
- id: ruff
|
|
@@ -17,16 +17,6 @@ repos:
|
|
|
17
17
|
- id: ruff-format
|
|
18
18
|
|
|
19
19
|
- repo: https://github.com/RobertCraigie/pyright-python
|
|
20
|
-
rev: v1.1.
|
|
20
|
+
rev: v1.1.407
|
|
21
21
|
hooks:
|
|
22
22
|
- id: pyright
|
|
23
|
-
|
|
24
|
-
# - repo: https://github.com/RodrigoGonzalez/check-mkdocs
|
|
25
|
-
# rev: v1.2.0
|
|
26
|
-
# hooks:
|
|
27
|
-
# - id: check-mkdocs
|
|
28
|
-
# name: check-mkdocs
|
|
29
|
-
# args: ["--config", "mkdocs.yml"] # Optional, mkdocs.yml is the default
|
|
30
|
-
# # If you have additional plugins or libraries that are not included in
|
|
31
|
-
# # check-mkdocs, add them here
|
|
32
|
-
# additional_dependencies: ["mkdocs-material"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
## [Unreleased]
|
|
3
|
+
|
|
4
|
+
## [0.19.6] - 2025-12-03
|
|
5
|
+
|
|
6
|
+
## [0.19.6] - 2025-12-03
|
|
7
|
+
|
|
8
|
+
### Changed
|
|
9
|
+
|
|
10
|
+
- **BREAKING: Explicit Database Creation**: Databases must now be explicitly created before use
|
|
11
|
+
- New `haiku-rag init` command creates a new empty database
|
|
12
|
+
- Python API: `HaikuRAG(path, create=True)` to create database programmatically
|
|
13
|
+
- Operations on non-existent databases raise `FileNotFoundError`
|
|
14
|
+
- **BREAKING: Embeddings Configuration**: Restructured to nested `EmbeddingModelConfig`
|
|
15
|
+
- Config path changed from `embeddings.{provider, model, vector_dim}` to `embeddings.model.{provider, name, vector_dim}`
|
|
16
|
+
- Automatic migration upgrades existing databases to new format
|
|
17
|
+
- **Database Migrations**: Always run when opening an existing database
|
|
18
|
+
|
|
19
|
+
## [0.19.5] - 2025-12-01
|
|
20
|
+
|
|
21
|
+
### Changed
|
|
22
|
+
|
|
23
|
+
- **Rebuild Performance**: Optimized `rebuild --embed-only` to use batch updates via LanceDB's `merge_insert` instead of individual chunk updates, and skip chunks with unchanged embeddings
|
|
24
|
+
|
|
25
|
+
## [0.19.4] - 2025-11-28
|
|
26
|
+
|
|
27
|
+
### Added
|
|
28
|
+
|
|
29
|
+
- **Rebuild Modes**: New options for `rebuild` command to control what gets rebuilt
|
|
30
|
+
- `--embed-only`: Only regenerate embeddings, keeping existing chunks (fastest option when changing embedding model)
|
|
31
|
+
- `--rechunk`: Re-chunk from existing document content without accessing source files
|
|
32
|
+
- Default (no flag): Full rebuild with source file re-conversion
|
|
33
|
+
- Python API: `rebuild_database(mode=RebuildMode.EMBED_ONLY | RECHUNK | FULL)`
|
|
34
|
+
|
|
35
|
+
## [0.19.3] - 2025-11-27
|
|
36
|
+
|
|
37
|
+
### Changed
|
|
38
|
+
|
|
39
|
+
- **Async Chunker**: `DoclingServeChunker` now uses `httpx.AsyncClient` instead of sync `requests`
|
|
40
|
+
|
|
41
|
+
### Fixed
|
|
42
|
+
|
|
43
|
+
- **OCR Options**: Fixed `DoclingLocalConverter` using base `OcrOptions` class which docling's OCR factory doesn't recognize. Now uses `OcrAutoOptions` for automatic OCR engine selection.
|
|
44
|
+
- **Dependencies**: Added `opencv-python-headless` to the `docling` optional dependency for table structure detection.
|
|
45
|
+
|
|
46
|
+
## [0.19.2] - 2025-11-27
|
|
47
|
+
|
|
48
|
+
### Changed
|
|
49
|
+
|
|
50
|
+
- **Async Converters**: Made document converters fully async
|
|
51
|
+
- `BaseConverter.convert_file()` and `convert_text()` are now async methods
|
|
52
|
+
- `DoclingLocalConverter` wraps blocking Docling operations with `asyncio.to_thread()`
|
|
53
|
+
- `DoclingServeConverter` now uses `httpx.AsyncClient` instead of sync `requests`
|
|
54
|
+
- **Async Model Prefetch**: `prefetch_models()` is now async
|
|
55
|
+
- Uses `httpx.AsyncClient` for Ollama model pulls
|
|
56
|
+
- Wraps blocking Docling and HuggingFace downloads with `asyncio.to_thread()`
|
|
57
|
+
|
|
58
|
+
## [0.19.1] - 2025-11-26
|
|
59
|
+
|
|
60
|
+
### Added
|
|
61
|
+
|
|
62
|
+
- **LM Studio Provider**: Added support for LM Studio as a provider for embeddings and QA/research models
|
|
63
|
+
- Configure with `provider: lm_studio` in embeddings, QA, or research model settings
|
|
64
|
+
- Supports thinking control for reasoning models (gpt-oss, etc.)
|
|
65
|
+
- Default base URL: `http://localhost:1234`
|
|
66
|
+
|
|
67
|
+
### Fixed
|
|
68
|
+
|
|
69
|
+
- **Configuration**: Fixed `init-config` command generating invalid configuration files (#165)
|
|
70
|
+
- Refactored `generate_default_config()` to use Pydantic model serialization instead of manual dict construction
|
|
71
|
+
- Updated `qa`, `research`, and `reranking` sections to use new `ModelConfig` structure
|
|
72
|
+
|
|
73
|
+
## [0.19.0] - 2025-11-25
|
|
74
|
+
|
|
75
|
+
### Added
|
|
76
|
+
|
|
77
|
+
- **Model Customization**: Added support for per-model configuration settings
|
|
78
|
+
- New `enable_thinking` parameter to control reasoning behavior (true/false/None)
|
|
79
|
+
- Support for `temperature` and `max_tokens` settings on QA and research models
|
|
80
|
+
- All settings apply to any provider that supports them
|
|
81
|
+
- **Database Inspector**: New `inspect` CLI command launches interactive TUI for browsing documents and chunks & searching
|
|
82
|
+
- **Evaluations**: Added `evaluations` CLI script for running benchmarks (replaces `python -m evaluations.benchmark`)
|
|
83
|
+
- **Evaluations**: Added `--db` option to override evaluation database path
|
|
84
|
+
- Default database location moved to haiku.rag data directory:
|
|
85
|
+
- macOS: `~/Library/Application Support/haiku.rag/evaluations/dbs/`
|
|
86
|
+
- Linux: `~/.local/share/haiku.rag/evaluations/dbs/`
|
|
87
|
+
- Windows: `C:/Users/<USER>/AppData/Roaming/haiku.rag/evaluations/dbs/`
|
|
88
|
+
- Previously stored in `evaluations/data/` within the repository
|
|
89
|
+
- **Evaluations**: Added comprehensive experiment metadata tracking for better reproducibility
|
|
90
|
+
- Records dataset name, test case count, and all model configurations
|
|
91
|
+
- Tracks embedder settings: provider, model, and vector dimensions
|
|
92
|
+
- Tracks QA model: provider and model name
|
|
93
|
+
- Tracks judge model: provider and model name for LLM evaluation
|
|
94
|
+
- Tracks processing parameters: `chunk_size` and `context_chunk_radius`
|
|
95
|
+
- Tracks retrieval configuration: `retrieval_limit` for number of chunks retrieved
|
|
96
|
+
- Tracks reranking configuration: `rerank_provider` and `rerank_model`
|
|
97
|
+
- Enables comparison of evaluation runs with different configurations in Logfire
|
|
98
|
+
- **Evaluations**: Refactored retrieval evaluation to use pydantic-ai experiment framework
|
|
99
|
+
- New `evaluators` module with `MRREvaluator` (Mean Reciprocal Rank) and `MAPEvaluator` (Mean Average Precision)
|
|
100
|
+
- Retrieval benchmarks now use `Dataset.evaluate()` with full Logfire experiment tracking
|
|
101
|
+
- Dataset specifications now declare their retrieval evaluator (MRR for RepliQA, MAP for Wix)
|
|
102
|
+
- Replaced Recall@K and Success@K with industry-standard MRR and MAP metrics
|
|
103
|
+
- Unified evaluation framework for both retrieval and QA benchmarks
|
|
104
|
+
- **AG-UI Events**: Enhanced ActivitySnapshot events with richer structured data
|
|
105
|
+
- Added `stepName` field to identify which graph node emitted each activity
|
|
106
|
+
- Added structured fields to activity content while preserving backward-compatible `message` field:
|
|
107
|
+
- **Planning**: `sub_questions` - list of sub-question strings
|
|
108
|
+
- **Searching**: `query` - the search query, `confidence` - answer confidence (on success), `error` - error message (on failure)
|
|
109
|
+
- **Analyzing** (research): `insights` - list of insight objects, `gaps` - list of gap objects, `resolved_gaps` - list of resolved gap strings
|
|
110
|
+
- **Evaluating** (research): `confidence` - confidence score, `is_sufficient` - sufficiency flag
|
|
111
|
+
- **Evaluating** (deep QA): `is_sufficient` - sufficiency flag, `iterations` - iteration count
|
|
112
|
+
|
|
113
|
+
### Changed
|
|
114
|
+
|
|
115
|
+
- **Evaluations**: Renamed `--qa-limit` CLI parameter to `--limit`, now applies to both retrieval and QA benchmarks
|
|
116
|
+
- **Evaluations**: Retrieval evaluator selection moved from runtime logic to dataset configuration
|
|
117
|
+
|
|
118
|
+
## [0.18.0] - 2025-11-21
|
|
119
|
+
|
|
120
|
+
### Added
|
|
121
|
+
|
|
122
|
+
- **Manual Vector Indexing**: New `create-index` CLI command for explicit vector index creation
|
|
123
|
+
- Creates IVF_PQ indexes
|
|
124
|
+
- Requires minimum 256 chunks (LanceDB training data requirement)
|
|
125
|
+
- New `search.vector_index_metric` config option: `cosine` (default), `l2`, or `dot`
|
|
126
|
+
- New `search.vector_refine_factor` config option (default: 30) for accuracy/speed tradeoff
|
|
127
|
+
- Indexes not created automatically during ingestion to avoid performance degradation
|
|
128
|
+
- Manual rebuilding required after adding significant new data
|
|
129
|
+
- **Enhanced Info Command**: `haiku-rag info` now shows storage sizes and vector index statistics
|
|
130
|
+
- Displays storage size for documents and chunks tables in human-readable format
|
|
131
|
+
- Shows vector index status (exists/not created)
|
|
132
|
+
- Shows indexed and unindexed chunk counts for monitoring index staleness
|
|
133
|
+
|
|
134
|
+
### Changed
|
|
135
|
+
|
|
136
|
+
- **BREAKING: Default Embedding Model**: Changed default embedding model from `qwen3-embedding` to `qwen3-embedding:4b` with vector dimension 2560 (previously 4096)
|
|
137
|
+
- New installations will use the smaller, more efficient 4B parameter model by default
|
|
138
|
+
- **Action required**: Existing databases created with the old default will be incompatible. Users must either:
|
|
139
|
+
- Explicitly set `embeddings.model: "qwen3-embedding"` and `embeddings.vector_dim: 4096` in their config to maintain compatibility with existing databases
|
|
140
|
+
- Or run `haiku-rag rebuild` to re-embed all documents with the new default
|
|
141
|
+
- This change provides better performance for most use cases while reducing resource requirements
|
|
142
|
+
- **Evaluations**: Improved evaluation dataset naming and simplified evaluator configuration
|
|
143
|
+
- `EvalDataset` now accepts dataset name for better organization in Logfire
|
|
144
|
+
- Added `--name` CLI parameter to override evaluation run names
|
|
145
|
+
- Removed `IsInstance` evaluator, using only `LLMJudge` for QA evaluation
|
|
146
|
+
- **Search Accuracy**: Applied `refine_factor` to vector and hybrid searches for improved accuracy
|
|
147
|
+
- Retrieves `refine_factor * limit` candidates and re-ranks in memory
|
|
148
|
+
- Higher values increase accuracy but slow down queries
|
|
149
|
+
|
|
150
|
+
### Fixed
|
|
151
|
+
|
|
152
|
+
- **AG-UI Activity Events**: Activity events now correctly use structured dict content instead of strings
|
|
153
|
+
- **Graph Configuration**: Graph builder functions now properly accept and use non-global config (#149)
|
|
154
|
+
- `build_research_graph()` and `build_deep_qa_graph()` now pass config to all agents and model creation
|
|
155
|
+
- `get_model()` utility function accepts `config` parameter (defaults to global Config)
|
|
156
|
+
- Allows creating multiple graphs with different configurations in the same application
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
## [0.17.2] - 2025-11-19
|
|
160
|
+
|
|
161
|
+
### Added
|
|
162
|
+
|
|
163
|
+
- **Document Update API**: New `update_document_fields()` method for partial document updates
|
|
164
|
+
- Update individual fields (content, metadata, title, chunks) without fetching full document
|
|
165
|
+
- Support for custom chunks or auto-generation from content
|
|
166
|
+
|
|
167
|
+
### Changed
|
|
168
|
+
|
|
169
|
+
- **Chunk Creation**: `ChunkRepository.create()` now accepts both single chunks and lists for batch insertion
|
|
170
|
+
- Batch insertion reduces LanceDB version creation when adding multiple chunks with custom chunks
|
|
171
|
+
- Batch embedding generation for improved performance with multiple chunks
|
|
172
|
+
- Updated core dependencies
|
|
173
|
+
|
|
174
|
+
## [0.17.1] - 2025-11-18
|
|
175
|
+
|
|
176
|
+
### Added
|
|
177
|
+
|
|
178
|
+
- **Conversion Options**: Fine-grained control over document conversion for both local and remote converters
|
|
179
|
+
- New `conversion_options` config section in `ProcessingConfig`
|
|
180
|
+
- OCR settings: `do_ocr`, `force_ocr`, `ocr_lang` for controlling OCR behavior
|
|
181
|
+
- Table extraction: `do_table_structure`, `table_mode` (fast/accurate), `table_cell_matching`
|
|
182
|
+
- Image settings: `images_scale` to control image resolution
|
|
183
|
+
- Options work identically with both `docling-local` and `docling-serve` converters
|
|
184
|
+
|
|
185
|
+
### Changed
|
|
186
|
+
|
|
187
|
+
- Increase reranking candidate retrieval multiplier from 3x to 10x for improved result quality
|
|
188
|
+
- **Docker Images**: Main `haiku.rag` image no longer automatically built and published
|
|
189
|
+
- **Conversion Options**: Removed the legacy `pdf_backend` setting; docling now chooses the optimal backend automatically
|
|
190
|
+
|
|
191
|
+
## [0.17.0] - 2025-11-17
|
|
192
|
+
|
|
193
|
+
### Added
|
|
194
|
+
|
|
195
|
+
- **Remote Processing**: Support for docling-serve as remote document processing and chunking service
|
|
196
|
+
- New `converter` config option: `docling-local` (default) or `docling-serve`
|
|
197
|
+
- New `chunker` config option: `docling-local` (default) or `docling-serve`
|
|
198
|
+
- New `providers.docling_serve` config section with `base_url`, `api_key`, and `timeout`
|
|
199
|
+
- Comprehensive error handling for connection, timeout, and authentication issues
|
|
200
|
+
- **Chunking Strategies**: Support for both hybrid and hierarchical chunking
|
|
201
|
+
- New `chunker_type` config option: `hybrid` (default) or `hierarchical`
|
|
202
|
+
- Hybrid chunking: Structure-aware splitting that respects document boundaries
|
|
203
|
+
- Hierarchical chunking: Preserves document hierarchy for nested documents
|
|
204
|
+
- **Table Serialization Control**: Configurable table representation in chunks
|
|
205
|
+
- New `chunking_use_markdown_tables` config option (default: `false`)
|
|
206
|
+
- `false`: Tables serialized as narrative text ("Value A, Column 2 = Value B")
|
|
207
|
+
- `true`: Tables preserved as markdown format with structure
|
|
208
|
+
- **Chunking Configuration**: Additional chunking control options
|
|
209
|
+
- New `chunking_merge_peers` config option (default: `true`) to merge undersized successive chunks
|
|
210
|
+
- **Docker Images**: Two Docker images for different deployment scenarios
|
|
211
|
+
- `haiku.rag`: Full image with all dependencies for self-contained deployments
|
|
212
|
+
- `haiku.rag-slim`: Minimal image designed for use with external docling-serve
|
|
213
|
+
- Multi-platform support (linux/amd64, linux/arm64)
|
|
214
|
+
- Docker Compose examples with docling-serve integration
|
|
215
|
+
- Automated CI/CD workflows for both images
|
|
216
|
+
- Build script (`scripts/build-docker-images.sh`) for local multi-platform builds
|
|
217
|
+
|
|
218
|
+
### Changed
|
|
219
|
+
|
|
220
|
+
- **BREAKING: Chunking Tokenizer**: Switched from tiktoken to HuggingFace tokenizers for consistency with docling-serve
|
|
221
|
+
- Default tokenizer changed from tiktoken "gpt-4o" to "Qwen/Qwen3-Embedding-0.6B"
|
|
222
|
+
- New `chunking_tokenizer` config option in `ProcessingConfig` for customization
|
|
223
|
+
- `download-models` CLI command now also downloads the configured HuggingFace tokenizer
|
|
224
|
+
- **Docker Examples**: Updated examples to demonstrate remote processing
|
|
225
|
+
- `examples/docker` now uses slim image with docling-serve
|
|
226
|
+
- `examples/ag-ui-research` backend uses slim image with docling-serve
|
|
227
|
+
- Configuration examples include remote processing setup
|
|
228
|
+
|
|
229
|
+
## [0.16.1] - 2025-11-14
|
|
230
|
+
|
|
231
|
+
### Changed
|
|
232
|
+
|
|
233
|
+
- **Evaluations**: Refactored QA benchmark to run entire dataset as single evaluation for better Logfire experiment tracking
|
|
234
|
+
- **Evaluations**: Added `.env` file loading support via `python-dotenv` dependency
|
|
235
|
+
|
|
236
|
+
## [0.16.0] - 2025-11-13
|
|
237
|
+
|
|
238
|
+
### Added
|
|
239
|
+
|
|
240
|
+
- **AG-UI Protocol Support**: Full AG-UI (Agent-UI) protocol implementation for graph execution with event streaming
|
|
241
|
+
- New `AGUIEmitter` class for emitting AG-UI events from graphs
|
|
242
|
+
- Support for all AG-UI event types: lifecycle events (`RUN_STARTED`, `RUN_FINISHED`, `RUN_ERROR`), step events (`STEP_STARTED`, `STEP_FINISHED`), state updates (`STATE_SNAPSHOT`, `STATE_DELTA`), activity narration (`ACTIVITY_SNAPSHOT`), and text messages (`TEXT_MESSAGE_CHUNK`)
|
|
243
|
+
- `AGUIConsoleRenderer` for rendering AG-UI event streams to terminal with Rich formatting
|
|
244
|
+
- `stream_graph()` utility function for executing graphs with AG-UI event emission
|
|
245
|
+
- State diff computation for efficient state synchronization
|
|
246
|
+
- **Delta State Updates**: AG-UI emitter now supports incremental state updates via JSON Patch operations (`STATE_DELTA` events) to reduce bandwidth, configurable via `use_deltas` parameter (enabled by default)
|
|
247
|
+
- **AG-UI Server**: Starlette-based HTTP server for serving graphs via AG-UI protocol
|
|
248
|
+
- Server-Sent Events (SSE) streaming endpoint at `/v1/agent/stream`
|
|
249
|
+
- Health check endpoint at `/health`
|
|
250
|
+
- Full CORS support configurable via `agui` config section
|
|
251
|
+
- `create_agui_server()` function for programmatic server creation
|
|
252
|
+
- **Deep QA AG-UI Support**: Deep QA graph now fully supports AG-UI event streaming
|
|
253
|
+
- Integration with `AGUIEmitter` for progress tracking
|
|
254
|
+
- Step-by-step execution visibility via AG-UI events
|
|
255
|
+
- **CLI AG-UI Flag**: New `--agui` flag for `serve` command to start AG-UI server
|
|
256
|
+
- **Graph Module**: New unified `haiku.rag.graph` module containing all graph-related functionality
|
|
257
|
+
- **Common Graph Nodes**: New factory functions (`create_plan_node`, `create_search_node`) in `haiku.rag.graph.common.nodes` for reusable graph components
|
|
258
|
+
- **AG-UI Research Example**: New full-stack example (`examples/ag-ui-research`) demonstrating agent+graph architecture with CopilotKit frontend
|
|
259
|
+
- Pydantic AI agent with research tool that invokes the research graph
|
|
260
|
+
- Custom AG-UI streaming endpoint with anyio memory streams
|
|
261
|
+
- React/Next.js frontend with split-pane UI showing live research state
|
|
262
|
+
- Real-time progress tracking of questions, answers, insights, and gaps
|
|
263
|
+
- Docker Compose setup for easy local development
|
|
264
|
+
|
|
265
|
+
### Changed
|
|
266
|
+
|
|
267
|
+
- **Vacuum Retention**: Default `vacuum_retention_seconds` increased from 60 seconds to 86400 seconds (1 day) for better version retention in typical workflows
|
|
268
|
+
- **BREAKING**: Major refactoring of graph-related code into unified `haiku.rag.graph` module structure:
|
|
269
|
+
- `haiku.rag.research` → `haiku.rag.graph.research`
|
|
270
|
+
- `haiku.rag.qa.deep` → `haiku.rag.graph.deep_qa`
|
|
271
|
+
- `haiku.rag.agui` → `haiku.rag.graph.agui`
|
|
272
|
+
- `haiku.rag.graph_common` → `haiku.rag.graph.common`
|
|
273
|
+
- **BREAKING**: Research and Deep QA graphs now use AG-UI event protocol instead of direct console logging
|
|
274
|
+
- Removed `console` and `stream` parameters from graph dependencies
|
|
275
|
+
- All progress updates now emit through `AGUIEmitter`
|
|
276
|
+
- **BREAKING**: `ResearchState` converted from dataclass to Pydantic `BaseModel` for JSON serialization and AG-UI compatibility
|
|
277
|
+
- Research and Deep QA graphs now emit detailed execution events for better observability
|
|
278
|
+
- CLI research command now uses AG-UI event rendering for `--verbose` output
|
|
279
|
+
- Improved graph execution visibility with step-by-step progress tracking
|
|
280
|
+
- Updated all documentation to reflect new import paths and AG-UI usage
|
|
281
|
+
- Updated examples (ag-ui-research, a2a-server) to use new import paths
|
|
282
|
+
|
|
283
|
+
### Fixed
|
|
284
|
+
|
|
285
|
+
- **Document Creation**: Optimized `create_document` to skip unnecessary DoclingDocument conversion when chunks are pre-provided
|
|
286
|
+
- **FileReader**: Error messages now include both original exception details and file path for easier debugging
|
|
287
|
+
- **Database Auto-creation**: Read operations (search, list, get, ask, research) no longer auto-create empty databases. Write operations (add, add-src, delete, rebuild) still create the database as needed. This prevents the confusing scenario where a search query creates an empty database. Fixes issue #137.
|
|
288
|
+
|
|
289
|
+
### Removed
|
|
290
|
+
|
|
291
|
+
- **BREAKING**: Removed `disable_autocreate` config option - the behavior is now automatic based on operation type
|
|
292
|
+
- **BREAKING**: Removed legacy `ResearchStream` and `ResearchStreamEvent` classes (replaced by AG-UI event protocol)
|
|
293
|
+
|
|
294
|
+
## [0.15.0] - 2025-11-07
|
|
295
|
+
|
|
296
|
+
### Added
|
|
297
|
+
|
|
298
|
+
- **File Monitor**: Orphan deletion feature - automatically removes documents from database when source files are deleted (enabled via `monitor.delete_orphans` config option, default: false)
|
|
299
|
+
|
|
300
|
+
### Changed
|
|
301
|
+
|
|
302
|
+
- **Configuration**: All CLI commands now properly support `--config` parameter for specifying custom configuration files
|
|
303
|
+
- Configuration loading consolidated across CLI, app, and client with consistent resolution order
|
|
304
|
+
- `HaikuRAGApp` and MCP server now accept `config` parameter for programmatic configuration
|
|
305
|
+
- Updated CLI documentation to clarify global vs per-command options
|
|
306
|
+
- **BREAKING**: Standardized configuration filename to `haiku.rag.yaml` in user directories (was incorrectly using `config.yaml`). Users with existing `config.yaml` in their user directory will need to rename it to `haiku.rag.yaml`
|
|
307
|
+
|
|
308
|
+
### Fixed
|
|
309
|
+
|
|
310
|
+
- **File Monitor**: Fixed incorrect "Updated document" logging for unchanged files - monitor now properly skips files when MD5 hash hasn't changed
|
|
311
|
+
|
|
312
|
+
### Removed
|
|
313
|
+
|
|
314
|
+
- **BREAKING**: A2A (Agent-to-Agent) protocol support has been moved to a separate self-contained package in `examples/a2a-server/`. The A2A server is no longer part of the main haiku.rag package. Users who need A2A functionality can install and run it from the examples directory with `cd examples/a2a-server && uv sync`.
|
|
315
|
+
- **BREAKING**: Removed deprecated `.env`-based configuration system. The `haiku-rag init-config --from-env` command and `load_config_from_env()` function have been removed. All configuration must now be done via YAML files. Environment variables for API keys (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) and service URLs (e.g., `OLLAMA_BASE_URL`) are still supported and can be set via `.env` files.
|
|
316
|
+
|
|
317
|
+
## [0.14.1] - 2025-11-06
|
|
318
|
+
|
|
319
|
+
### Added
|
|
320
|
+
|
|
321
|
+
- Migrated research and deep QA agents to use Pydantic Graph beta API for better graph execution
|
|
322
|
+
- Automatic semaphore-based concurrency control for parallel sub-question processing
|
|
323
|
+
- `max_concurrency` parameter for controlling parallel execution in research and deep QA (default: 1)
|
|
324
|
+
|
|
325
|
+
### Changed
|
|
326
|
+
|
|
327
|
+
- **BREAKING**: Research and Deep QA graphs now use `pydantic_graph.beta` instead of the class-based graph implementation
|
|
328
|
+
- Refactored graph common patterns into `graph_common` module
|
|
329
|
+
- Sub-questions now process using `.map()` for true parallel execution
|
|
330
|
+
- Improved graph structure with cleaner node definitions and flow control
|
|
331
|
+
- Pinned critical dependencies: `docling-core`, `lancedb`, `docling`
|
|
332
|
+
|
|
333
|
+
## [0.14.0] - 2024-11-05
|
|
334
|
+
|
|
335
|
+
### Added
|
|
336
|
+
|
|
337
|
+
- New `haiku.rag-slim` package with minimal dependencies for users who want to install only what they need
|
|
338
|
+
- Evaluations package (`haiku.rag-evals`) for internal benchmarking and testing
|
|
339
|
+
- Improved search filtering performance by using pandas DataFrames for joins instead of SQL WHERE IN clauses
|
|
340
|
+
|
|
341
|
+
### Changed
|
|
342
|
+
|
|
343
|
+
- **BREAKING**: Restructured project into UV workspace with three packages:
|
|
344
|
+
- `haiku.rag-slim` - Core package with minimal dependencies
|
|
345
|
+
- `haiku.rag` - Full package with all extras (recommended for most users)
|
|
346
|
+
- `haiku.rag-evals` - Internal benchmarking and evaluation tools
|
|
347
|
+
- Migrated from `pydantic-ai` to `pydantic-ai-slim` with extras system
|
|
348
|
+
- Docling is now an optional dependency (install with `haiku.rag-slim[docling]`)
|
|
349
|
+
- Package metadata checks now use `haiku.rag-slim` (always present) instead of `haiku.rag`
|
|
350
|
+
- Docker image optimized: removed evaluations package, reducing installed packages from 307 to 259
|
|
351
|
+
- Improved vector search performance through optimized score normalization
|
|
352
|
+
|
|
353
|
+
### Fixed
|
|
354
|
+
|
|
355
|
+
- ImportError now properly raised when optional docling dependency is missing
|
|
356
|
+
|
|
357
|
+
## [0.13.3] - 2024-11-04
|
|
358
|
+
|
|
359
|
+
### Added
|
|
360
|
+
|
|
361
|
+
- Support for Zero Entropy reranker
|
|
362
|
+
- Filter parameter to `search()` for filtering documents before search
|
|
363
|
+
- Filter parameter to CLI `search` command
|
|
364
|
+
- Filter parameter to CLI `list` command for filtering document listings
|
|
365
|
+
- Config option to pass custom configuration files to evaluation commands
|
|
366
|
+
- Document filtering now respects configured include/exclude patterns when using `add-src` with directories
|
|
367
|
+
- Max retries to insight_agent when producing structured output
|
|
368
|
+
|
|
369
|
+
### Fixed
|
|
370
|
+
|
|
371
|
+
- CLI now loads `.env` files at startup
|
|
372
|
+
- Info command no longer attempts to use deprecated `.env` settings
|
|
373
|
+
- Documentation typos
|
|
374
|
+
|
|
375
|
+
## [0.13.2] - 2024-11-04
|
|
376
|
+
|
|
377
|
+
### Added
|
|
378
|
+
|
|
379
|
+
- Gitignore-style pattern filtering for file monitoring using pathspec
|
|
380
|
+
- Include/exclude pattern documentation for FileMonitor
|
|
381
|
+
|
|
382
|
+
### Changed
|
|
383
|
+
|
|
384
|
+
- Moved monitor configuration to its own section in config
|
|
385
|
+
- Improved configuration documentation
|
|
386
|
+
- Updated dependencies
|
|
387
|
+
|
|
388
|
+
## [0.13.1] - 2024-11-03
|
|
389
|
+
|
|
390
|
+
### Added
|
|
391
|
+
|
|
392
|
+
- Initial version tracking
|
|
393
|
+
|
|
394
|
+
[Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.14.0...HEAD
|
|
395
|
+
[0.14.0]: https://github.com/ggozad/haiku.rag/compare/0.13.3...0.14.0
|
|
396
|
+
[0.13.3]: https://github.com/ggozad/haiku.rag/compare/0.13.2...0.13.3
|
|
397
|
+
[0.13.2]: https://github.com/ggozad/haiku.rag/compare/0.13.1...0.13.2
|
|
398
|
+
[0.13.1]: https://github.com/ggozad/haiku.rag/releases/tag/0.13.1
|