tunarag-python 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/.gitignore +4 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/CHANGELOG.md +36 -0
- tunarag_python-0.2.1/README.md → tunarag_python-0.3.0/PKG-INFO +214 -86
- tunarag_python-0.2.1/PKG-INFO → tunarag_python-0.3.0/README.md +144 -128
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/pyproject.toml +48 -4
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/__init__.py +3 -1
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/contracts.py +1 -9
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/engine.py +9 -4
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/integrations/__init__.py +2 -2
- tunarag_python-0.3.0/src/tunarag/integrations/google_adk.py +144 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/integrations/runnables.py +19 -19
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/search.py +10 -2
- tunarag_python-0.2.1/.gitattributes +0 -7
- tunarag_python-0.2.1/.github/workflows/ci.yml +0 -56
- tunarag_python-0.2.1/.github/workflows/publish.yml +0 -74
- tunarag_python-0.2.1/CONTRIBUTING.md +0 -23
- tunarag_python-0.2.1/DECISIONS.md +0 -88
- tunarag_python-0.2.1/FLOW.md +0 -66
- tunarag_python-0.2.1/PROJECT_PLAN.md +0 -111
- tunarag_python-0.2.1/RELEASE_CHECKLIST.md +0 -26
- tunarag_python-0.2.1/SECURITY.md +0 -24
- tunarag_python-0.2.1/docs/01_API_Specification.docx +0 -0
- tunarag_python-0.2.1/docs/02_Coding_Conventions.docx +0 -0
- tunarag_python-0.2.1/docs/03_Component_Map.docx +0 -0
- tunarag_python-0.2.1/docs/04_Data_Extraction_Specification.docx +0 -0
- tunarag_python-0.2.1/docs/05_Database_Schema.docx +0 -0
- tunarag_python-0.2.1/docs/06_Environment_Config_Reference.docx +0 -0
- tunarag_python-0.2.1/docs/07_File_Folder_Structure.docx +0 -0
- tunarag_python-0.2.1/docs/08_Product_Requirement_Document.docx +0 -0
- tunarag_python-0.2.1/docs/09_System_Architecture_Document.docx +0 -0
- tunarag_python-0.2.1/docs/10_Tech_Stack.docx +0 -0
- tunarag_python-0.2.1/docs/11_Technical_Design_Document.docx +0 -0
- tunarag_python-0.2.1/docs/12_Documentation_Index.docx +0 -0
- tunarag_python-0.2.1/docs/api-specification.md +0 -307
- tunarag_python-0.2.1/docs/coding-conventions.md +0 -42
- tunarag_python-0.2.1/docs/component-map.md +0 -27
- tunarag_python-0.2.1/docs/data-extraction-specification.md +0 -53
- tunarag_python-0.2.1/docs/database-schema.md +0 -57
- tunarag_python-0.2.1/docs/decision.md +0 -255
- tunarag_python-0.2.1/docs/environment-config-reference.md +0 -41
- tunarag_python-0.2.1/docs/file-folder-structure.md +0 -50
- tunarag_python-0.2.1/docs/flow.md +0 -148
- tunarag_python-0.2.1/docs/product-requirements-document.md +0 -55
- tunarag_python-0.2.1/docs/system-architecture-document.md +0 -59
- tunarag_python-0.2.1/docs/tech-stack.md +0 -25
- tunarag_python-0.2.1/docs/technical-design-document.md +0 -197
- tunarag_python-0.2.1/docs/ui-ux-wireframes.md +0 -37
- tunarag_python-0.2.1/examples/README.md +0 -41
- tunarag_python-0.2.1/examples/__init__.py +0 -1
- tunarag_python-0.2.1/examples/quickstart.py +0 -83
- tunarag_python-0.2.1/examples/reference_rag.py +0 -105
- tunarag_python-0.2.1/tests/test_cache.py +0 -133
- tunarag_python-0.2.1/tests/test_callbacks.py +0 -173
- tunarag_python-0.2.1/tests/test_concurrency.py +0 -116
- tunarag_python-0.2.1/tests/test_config.py +0 -87
- tunarag_python-0.2.1/tests/test_dataset_ingestion.py +0 -141
- tunarag_python-0.2.1/tests/test_domain.py +0 -35
- tunarag_python-0.2.1/tests/test_engine.py +0 -300
- tunarag_python-0.2.1/tests/test_errors.py +0 -59
- tunarag_python-0.2.1/tests/test_evaluators.py +0 -164
- tunarag_python-0.2.1/tests/test_framework_compatibility.py +0 -61
- tunarag_python-0.2.1/tests/test_llm_search.py +0 -113
- tunarag_python-0.2.1/tests/test_mlflow_integration.py +0 -96
- tunarag_python-0.2.1/tests/test_objective_and_stopping.py +0 -98
- tunarag_python-0.2.1/tests/test_package.py +0 -9
- tunarag_python-0.2.1/tests/test_properties.py +0 -52
- tunarag_python-0.2.1/tests/test_reference_example.py +0 -16
- tunarag_python-0.2.1/tests/test_resilience.py +0 -362
- tunarag_python-0.2.1/tests/test_result_reporting.py +0 -106
- tunarag_python-0.2.1/tests/test_resume.py +0 -301
- tunarag_python-0.2.1/tests/test_retry.py +0 -47
- tunarag_python-0.2.1/tests/test_runnable_integrations.py +0 -150
- tunarag_python-0.2.1/tests/test_search.py +0 -58
- tunarag_python-0.2.1/tests/test_store.py +0 -276
- tunarag_python-0.2.1/tests/test_synthetic.py +0 -91
- tunarag_python-0.2.1/tests/test_usage_documentation.py +0 -531
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/cache.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/config.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/dataset.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/domain.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/errors.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/evaluators.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/integrations/mlflow.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/objective.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/py.typed +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/result.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/retry.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/serialization.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/stopping.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/store.py +0 -0
- {tunarag_python-0.2.1 → tunarag_python-0.3.0}/src/tunarag/synthetic.py +0 -0
|
@@ -8,6 +8,7 @@ __pycache__/
|
|
|
8
8
|
.tmp/
|
|
9
9
|
.coverage
|
|
10
10
|
htmlcov/
|
|
11
|
+
site/
|
|
11
12
|
|
|
12
13
|
# Local environments
|
|
13
14
|
.venv/
|
|
@@ -22,5 +23,8 @@ dist/
|
|
|
22
23
|
# Local configuration and editor files
|
|
23
24
|
.env
|
|
24
25
|
.env.*
|
|
26
|
+
!.env.example
|
|
27
|
+
!tests/live/.env.example
|
|
28
|
+
live-test-output/
|
|
25
29
|
.vscode/
|
|
26
30
|
.idea/
|
|
@@ -2,6 +2,42 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to TunaRAG follow semantic versioning.
|
|
4
4
|
|
|
5
|
+
## 0.3.0 - 2026-10-09
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
|
|
9
|
+
- Tune framework pipelines exclusively through official adapters. `Optimizer` rejects custom
|
|
10
|
+
`run` objects; the public `RAGAdapter` and `AsyncRunnable` extension contracts are removed.
|
|
11
|
+
- Validate genuine LangChain `Runnable` and LangGraph `CompiledStateGraph` objects, including
|
|
12
|
+
per-candidate factory results. Handwritten `ainvoke` objects are no longer accepted.
|
|
13
|
+
- Replace the handwritten reference RAG with an offline LangChain retrieval-chain quickstart.
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- Lean source distributions containing runtime source, build configuration, README, changelog,
|
|
18
|
+
and generated metadata only. CI and publishing reject repository-only assets in archives.
|
|
19
|
+
|
|
20
|
+
- `GoogleADKAdapter` for Google ADK 2.x runners, with fresh evaluation sessions, session-state
|
|
21
|
+
candidate mapping, final-answer extraction, reported token usage, and guaranteed cleanup.
|
|
22
|
+
- Optional `google-adk` extra (Python 3.10+), ADK runner example, and real-framework tests.
|
|
23
|
+
- Opt-in OpenRouter, real RAGAS, and local MLflow tests, with a separate `live-test`
|
|
24
|
+
extra, explicit paid-execution gates, request caps, and a documented test plan.
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- Seeded RandomSearch resume now skips duplicate draws omitted from persisted trials.
|
|
29
|
+
- Callback dispatch no longer skips events committed while another callback awaits.
|
|
30
|
+
- Constrain the legacy LangChain community dependency used by RAGAS 0.4 to retain
|
|
31
|
+
the imports required by its evaluator wrappers.
|
|
32
|
+
|
|
33
|
+
### Migration
|
|
34
|
+
|
|
35
|
+
- Install the appropriate framework extra and wrap an existing framework pipeline with
|
|
36
|
+
`LangChainAdapter`, `LangGraphAdapter`, or `GoogleADKAdapter`.
|
|
37
|
+
- Custom evaluators, search strategies, objectives, callbacks, and synthetic providers remain
|
|
38
|
+
supported. Existing SQLite studies remain readable; changed pipeline identities require new
|
|
39
|
+
studies rather than silently resuming under different semantics.
|
|
40
|
+
|
|
5
41
|
## 0.2.1 - 2026-10-04
|
|
6
42
|
|
|
7
43
|
### Added
|
|
@@ -1,3 +1,73 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tunarag-python
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Tune LangChain, LangGraph, and Google ADK RAG pipelines
|
|
5
|
+
Project-URL: Repository, https://github.com/shivamshinde123/tunarag-python
|
|
6
|
+
Project-URL: Issues, https://github.com/shivamshinde123/tunarag-python/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/shivamshinde123/tunarag-python/blob/main/CHANGELOG.md
|
|
8
|
+
Author: Shivam Shinde
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
Keywords: evaluation,llm,optimization,rag,retrieval
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: tomli>=2.0; python_version < '3.11'
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
25
|
+
Requires-Dist: hypothesis>=6.100; extra == 'dev'
|
|
26
|
+
Requires-Dist: langchain-classic<2,>=1; extra == 'dev'
|
|
27
|
+
Requires-Dist: langchain-core<2,>=1; extra == 'dev'
|
|
28
|
+
Requires-Dist: langgraph<2,>=1; extra == 'dev'
|
|
29
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
30
|
+
Requires-Dist: numpy<3,>=1.26; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
34
|
+
Provides-Extra: docs
|
|
35
|
+
Requires-Dist: mkdocs-include-markdown-plugin<8,>=7; extra == 'docs'
|
|
36
|
+
Requires-Dist: mkdocs-material<10,>=9.5; extra == 'docs'
|
|
37
|
+
Requires-Dist: mkdocs<2,>=1.6; extra == 'docs'
|
|
38
|
+
Requires-Dist: mkdocstrings[python]<1,>=0.27; extra == 'docs'
|
|
39
|
+
Provides-Extra: frameworks
|
|
40
|
+
Requires-Dist: google-adk<3,>=2; extra == 'frameworks'
|
|
41
|
+
Requires-Dist: langchain<2,>=1; extra == 'frameworks'
|
|
42
|
+
Requires-Dist: langgraph<2,>=1; extra == 'frameworks'
|
|
43
|
+
Requires-Dist: numpy<3,>=1.26; extra == 'frameworks'
|
|
44
|
+
Provides-Extra: google-adk
|
|
45
|
+
Requires-Dist: google-adk<3,>=2; extra == 'google-adk'
|
|
46
|
+
Provides-Extra: langchain
|
|
47
|
+
Requires-Dist: langchain<2,>=1; extra == 'langchain'
|
|
48
|
+
Requires-Dist: numpy<3,>=1.26; extra == 'langchain'
|
|
49
|
+
Provides-Extra: langgraph
|
|
50
|
+
Requires-Dist: langgraph<2,>=1; extra == 'langgraph'
|
|
51
|
+
Provides-Extra: live-adk-test
|
|
52
|
+
Requires-Dist: google-adk<3,>=2.11; extra == 'live-adk-test'
|
|
53
|
+
Requires-Dist: litellm<2,>=1.104; extra == 'live-adk-test'
|
|
54
|
+
Provides-Extra: live-test
|
|
55
|
+
Requires-Dist: langchain-classic<2,>=1; extra == 'live-test'
|
|
56
|
+
Requires-Dist: langchain-community<0.4,>=0.3.31; extra == 'live-test'
|
|
57
|
+
Requires-Dist: langchain-openai<2,>=1; extra == 'live-test'
|
|
58
|
+
Requires-Dist: langgraph<2,>=1; extra == 'live-test'
|
|
59
|
+
Requires-Dist: mlflow<4,>=3; extra == 'live-test'
|
|
60
|
+
Requires-Dist: pypdf<7,>=5; extra == 'live-test'
|
|
61
|
+
Requires-Dist: pytest>=8; extra == 'live-test'
|
|
62
|
+
Requires-Dist: python-dotenv<2,>=1; extra == 'live-test'
|
|
63
|
+
Requires-Dist: ragas<0.5,>=0.4; extra == 'live-test'
|
|
64
|
+
Provides-Extra: mlflow
|
|
65
|
+
Requires-Dist: mlflow<4,>=3; extra == 'mlflow'
|
|
66
|
+
Provides-Extra: ragas
|
|
67
|
+
Requires-Dist: langchain-community<0.4,>=0.3.31; extra == 'ragas'
|
|
68
|
+
Requires-Dist: ragas<0.5,>=0.4; extra == 'ragas'
|
|
69
|
+
Description-Content-Type: text/markdown
|
|
70
|
+
|
|
1
71
|
# TunaRAG Python
|
|
2
72
|
|
|
3
73
|
**Find the best configuration for your existing RAG application.**
|
|
@@ -16,7 +86,7 @@ isolates failures, supports resumable execution, and produces results you can in
|
|
|
16
86
|
- Compare configurations with custom metrics or optional RAGAS evaluators.
|
|
17
87
|
- Optimize a weighted objective across quality, provider cost, and response latency.
|
|
18
88
|
- Search with deterministic random search or provider-neutral LLM-guided search.
|
|
19
|
-
- Tune
|
|
89
|
+
- Tune LangChain, LangGraph, or Google ADK RAG pipelines without rebuilding them.
|
|
20
90
|
- Preserve reproducible experiments with caching, retries, callbacks, reports, and SQLite resume.
|
|
21
91
|
|
|
22
92
|
### Who is it for?
|
|
@@ -28,9 +98,30 @@ indexes, provider clients, and credentials; TunaRAG owns the optimization workfl
|
|
|
28
98
|
> Install the distribution as `tunarag-python` and import it as `tunarag`. Python 3.10-3.13 is
|
|
29
99
|
> supported.
|
|
30
100
|
|
|
101
|
+
**Documentation:** [shivamshinde123.github.io/tunarag-python](https://shivamshinde123.github.io/tunarag-python/)
|
|
102
|
+
|
|
103
|
+
**Lightweight distribution:** pip installs runtime code and metadata, not tests, datasets,
|
|
104
|
+
or documentation images. Framework dependencies are optional extras. See
|
|
105
|
+
[package contents and release checks](docs/package-distribution.md).
|
|
106
|
+
|
|
107
|
+
**Live integration checks:** [OpenRouter, RAGAS, and MLflow test plan](docs/live-integration-test-plan.md).
|
|
108
|
+
|
|
109
|
+
**End-to-end verification:** [Full-PDF campaigns, held-out quality checks, crash recovery,
|
|
110
|
+
and clean-wheel testing](docs/comprehensive-testing.md). Paid tests are explicit opt-ins;
|
|
111
|
+
ordinary CI never makes paid provider calls.
|
|
112
|
+
These opt-in tests include spending safeguards; normal tests never make paid calls.
|
|
113
|
+
|
|
114
|
+
**Google ADK validation:** [Real OpenRouter RAG tuning, RAGAS scoring, SQLite resume,
|
|
115
|
+
and MLflow checks](docs/live-google-adk-testing.md).
|
|
116
|
+
|
|
117
|
+
**Quality and reliability:** [Ordered validation plan](docs/quality-validation-plan.md)
|
|
118
|
+
and [bounded repeated tuning, ADK tools, billing and recovery checks](docs/reliability-validation.md).
|
|
119
|
+
See [mixed-topic validation](docs/multicorpus-validation.md) for cross-model QA,
|
|
120
|
+
simulated faults/load, and the unresolved adversarial-prompt failure.
|
|
121
|
+
|
|
31
122
|
## Comprehensive Usage Guide
|
|
32
123
|
|
|
33
|
-
The sections below explain every public TunaRAG Python V0.
|
|
124
|
+
The sections below explain every public TunaRAG Python V0.3 feature with executable examples.
|
|
34
125
|
|
|
35
126
|
## Contents
|
|
36
127
|
|
|
@@ -82,6 +173,8 @@ A **study** contains ordered **trials**. Each trial evaluates one complete candi
|
|
|
82
173
|
against every dataset example. Same-named sample metrics are aggregated into trial metrics, then the
|
|
83
174
|
objective converts those metrics into one larger-is-better score.
|
|
84
175
|
|
|
176
|
+

|
|
177
|
+
|
|
85
178
|
TunaRAG does not create or host a RAG application, provision indexes, manage provider credentials,
|
|
86
179
|
or expose a hosted API. Your application keeps ownership of those concerns.
|
|
87
180
|
|
|
@@ -95,141 +188,140 @@ python -m pip install tunarag-python
|
|
|
95
188
|
|---|---|
|
|
96
189
|
| LangChain | `python -m pip install "tunarag-python[langchain]"` |
|
|
97
190
|
| LangGraph | `python -m pip install "tunarag-python[langgraph]"` |
|
|
98
|
-
|
|
|
191
|
+
| LangChain and LangGraph; ADK on Python 3.10+ | `python -m pip install "tunarag-python[frameworks]"` |
|
|
192
|
+
| Google ADK 2.x (Python 3.10+) | `python -m pip install "tunarag-python[google-adk]"` |
|
|
99
193
|
| RAGAS | `python -m pip install "tunarag-python[ragas]"` |
|
|
100
194
|
| MLflow | `python -m pip install "tunarag-python[mlflow]"` |
|
|
101
195
|
|
|
102
|
-
|
|
196
|
+
Framework dependencies load lazily. Install the extra for your pipeline before constructing its adapter. Google ADK requires Python 3.10 or newer; LangChain and LangGraph retain Python 3.10 support.
|
|
103
197
|
|
|
104
198
|
## End-to-end quickstart
|
|
105
199
|
|
|
106
|
-
|
|
200
|
+
Use an installed framework integration. The offline quickstart uses LangChain's vector store,
|
|
201
|
+
retriever, prompt, and fake model; the provider-backed example uses OpenRouter. Fake embeddings and
|
|
202
|
+
the fake model test wiring only; their scores do not demonstrate retrieval or answer quality.
|
|
107
203
|
|
|
108
204
|
```python
|
|
205
|
+
from pathlib import Path
|
|
206
|
+
|
|
207
|
+
from langchain_core.documents import Document
|
|
208
|
+
from langchain_core.embeddings import DeterministicFakeEmbedding
|
|
209
|
+
from langchain_core.language_models.fake import FakeListLLM
|
|
210
|
+
from langchain_core.prompts import ChatPromptTemplate
|
|
211
|
+
from langchain_core.runnables import RunnablePassthrough
|
|
212
|
+
from langchain_core.vectorstores import InMemoryVectorStore
|
|
109
213
|
from tunarag import (
|
|
110
214
|
EvaluationDataset,
|
|
111
215
|
IntegerRange,
|
|
216
|
+
LangChainAdapter,
|
|
112
217
|
MetricEvaluator,
|
|
113
218
|
Objective,
|
|
114
219
|
ObjectiveTerm,
|
|
115
220
|
Optimizer,
|
|
116
221
|
RandomSearch,
|
|
117
222
|
SearchSpace,
|
|
118
|
-
UsageRecord,
|
|
119
223
|
)
|
|
120
224
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
async def run(self, candidate, example):
|
|
130
|
-
query_words = set(example.query.lower().split())
|
|
131
|
-
ranked = sorted(
|
|
132
|
-
DOCUMENTS.values(),
|
|
133
|
-
key=lambda text: len(query_words & set(text.lower().split())),
|
|
134
|
-
reverse=True,
|
|
135
|
-
)
|
|
136
|
-
contexts = ranked[: candidate.parameters["top_k"]]
|
|
137
|
-
return {"answer": " ".join(contexts)}, (UsageRecord("small-rag", latency_seconds=0.001),)
|
|
225
|
+
vector_store = InMemoryVectorStore.from_documents(
|
|
226
|
+
[
|
|
227
|
+
Document(page_content="SQLite stores local experiments."),
|
|
228
|
+
Document(page_content="TunaRAG compares RAG configurations."),
|
|
229
|
+
],
|
|
230
|
+
DeterministicFakeEmbedding(size=32),
|
|
231
|
+
)
|
|
232
|
+
prompt = ChatPromptTemplate.from_template("Context: {context}\nQuestion: {question}")
|
|
138
233
|
|
|
139
234
|
|
|
140
|
-
def
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
235
|
+
def build_chain(candidate):
|
|
236
|
+
retriever = vector_store.as_retriever(search_kwargs={"k": candidate.parameters["top_k"]})
|
|
237
|
+
return {"context": retriever, "question": RunnablePassthrough()} | RunnablePassthrough.assign(
|
|
238
|
+
answer=prompt | FakeListLLM(responses=["SQLite stores local experiments."])
|
|
239
|
+
)
|
|
144
240
|
|
|
145
241
|
|
|
146
242
|
dataset = EvaluationDataset.from_records(
|
|
147
243
|
[
|
|
148
244
|
{
|
|
149
245
|
"id": "q1",
|
|
150
|
-
"query": "What
|
|
151
|
-
"reference_answer": "
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
},
|
|
246
|
+
"query": "What stores experiments and compares configurations?",
|
|
247
|
+
"reference_answer": "SQLite and TunaRAG",
|
|
248
|
+
"reference_contexts": [
|
|
249
|
+
"SQLite stores local experiments.",
|
|
250
|
+
"TunaRAG compares RAG configurations.",
|
|
251
|
+
],
|
|
252
|
+
}
|
|
158
253
|
]
|
|
159
254
|
)
|
|
160
255
|
optimizer = Optimizer(
|
|
161
|
-
adapter=
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
256
|
+
adapter=LangChainAdapter(
|
|
257
|
+
factory=build_chain, input_mapper=lambda example, candidate: example.query
|
|
258
|
+
),
|
|
259
|
+
strategy=RandomSearch(SearchSpace({"top_k": IntegerRange(1, 2)}), seed=7),
|
|
260
|
+
evaluators=(
|
|
261
|
+
MetricEvaluator(
|
|
262
|
+
"retrieval_recall",
|
|
263
|
+
lambda output, example: (
|
|
264
|
+
len(
|
|
265
|
+
{document.page_content for document in output["context"]}
|
|
266
|
+
& set(example.reference_contexts)
|
|
267
|
+
)
|
|
268
|
+
/ len(example.reference_contexts)
|
|
269
|
+
),
|
|
270
|
+
),
|
|
271
|
+
),
|
|
272
|
+
objective=Objective((ObjectiveTerm("retrieval_recall"),)),
|
|
165
273
|
)
|
|
166
274
|
try:
|
|
167
|
-
result = optimizer.optimize(dataset, study_name="
|
|
275
|
+
result = optimizer.optimize(dataset, study_name="langchain-rag")
|
|
168
276
|
print(result.best_candidate.parameters if result.best_candidate else "no winner")
|
|
169
277
|
finally:
|
|
170
278
|
optimizer.close()
|
|
171
279
|
```
|
|
172
280
|
|
|
173
281
|
The default store is in memory. Use `SQLiteStore(path)` for a durable study.
|
|
282
|
+
Here `top_k=1` retrieves one of two required contexts (recall `0.5`); `top_k=2` retrieves both
|
|
283
|
+
(recall `1.0`), so tuning selects `top_k=2`. This verifies parameter application and ranking with
|
|
284
|
+
offline framework components; it does not benchmark a real embedding model or generated answer.
|
|
174
285
|
|
|
175
286
|
## RAG adapters
|
|
176
287
|
|
|
177
|
-
|
|
288
|
+
Choose an official adapter: `LangChainAdapter`, `LangGraphAdapter`, or `GoogleADKAdapter`.
|
|
289
|
+
The optimizer rejects handwritten `run` adapters. LangChain accepts genuine framework `Runnable`
|
|
290
|
+
objects; LangGraph accepts `CompiledStateGraph`; Google ADK accepts `Runner`. Factory results are
|
|
291
|
+
validated on every sample, and failures enter the normal trial failure and retry boundary.
|
|
178
292
|
|
|
179
293
|
```python
|
|
180
|
-
from
|
|
181
|
-
|
|
182
|
-
from tunarag import UsageRecord
|
|
183
|
-
|
|
294
|
+
from langchain_core.runnables import RunnableLambda
|
|
295
|
+
from tunarag import LangChainAdapter
|
|
184
296
|
|
|
185
|
-
|
|
297
|
+
chain = RunnableLambda(lambda inputs: {"answer": inputs["question"]})
|
|
298
|
+
adapter = LangChainAdapter(chain)
|
|
186
299
|
```
|
|
187
300
|
|
|
188
|
-
|
|
189
|
-
by your evaluators. Usage may be empty. Adapter exceptions enter TunaRAG's retry and trial-failure
|
|
190
|
-
boundary.
|
|
301
|
+

|
|
191
302
|
|
|
192
|
-
|
|
193
|
-
|
|
303
|
+
Candidate parameters go into framework configuration or initial ADK session state. A parameter
|
|
304
|
+
only changes behavior if the framework consumes it. For model, prompt, retriever, or index choices
|
|
305
|
+
that require rebuilding the pipeline, use a per-candidate framework factory. Mapping callbacks
|
|
306
|
+
adapt framework inputs and outputs; they do not require implementing a RAG adapter.
|
|
194
307
|
|
|
308
|
+
```python
|
|
309
|
+
from langchain_core.runnables import RunnableLambda
|
|
310
|
+
from tunarag import LangChainAdapter
|
|
195
311
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
async def run(self, candidate, example):
|
|
201
|
-
response = await self.service.answer(
|
|
202
|
-
query=example.query,
|
|
203
|
-
top_k=candidate.parameters["top_k"],
|
|
204
|
-
model=candidate.parameters["model"],
|
|
205
|
-
)
|
|
206
|
-
return response, (
|
|
207
|
-
UsageRecord(
|
|
208
|
-
component="generator",
|
|
209
|
-
input_tokens=response.input_tokens,
|
|
210
|
-
output_tokens=response.output_tokens,
|
|
211
|
-
cost=response.cost,
|
|
212
|
-
latency_seconds=response.latency_seconds,
|
|
213
|
-
status=ValueStatus.EXACT,
|
|
214
|
-
pricing_version="provider-2026-10",
|
|
215
|
-
),
|
|
216
|
-
)
|
|
217
|
-
|
|
218
|
-
def fingerprint(self):
|
|
219
|
-
return {
|
|
220
|
-
"adapter_version": 2,
|
|
221
|
-
"index_revision": "docs-2026-10-04",
|
|
222
|
-
"prompt_revision": "qa-v3",
|
|
223
|
-
}
|
|
312
|
+
adapter = LangChainAdapter(
|
|
313
|
+
RunnableLambda(lambda inputs: {"answer": inputs["question"]}),
|
|
314
|
+
identity={"index_revision": "docs-v1", "prompt_revision": "qa-v1"},
|
|
315
|
+
)
|
|
224
316
|
```
|
|
225
317
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
318
|
+
Include stable corpus, index, model, prompt, and mapper revisions in `identity` for cache and resume
|
|
319
|
+
compatibility. Never include credentials. Framework factories and mapping callbacks remain normal
|
|
320
|
+
Python configuration code; retrieval and generation execute through the framework.
|
|
229
321
|
|
|
230
322
|
## LangChain integration
|
|
231
323
|
|
|
232
|
-
`LangChainAdapter` accepts
|
|
324
|
+
`LangChainAdapter` accepts a real LangChain `Runnable` with `ainvoke(input, config=...)`.
|
|
233
325
|
|
|
234
326
|
```python
|
|
235
327
|
from langchain_core.runnables import RunnableLambda
|
|
@@ -265,11 +357,16 @@ def build_chain(candidate):
|
|
|
265
357
|
|
|
266
358
|
adapter = LangChainAdapter(
|
|
267
359
|
factory=build_chain,
|
|
360
|
+
input_key="input",
|
|
268
361
|
output_mapper=lambda output: output["answer"],
|
|
269
362
|
identity={"factory_schema": 1},
|
|
270
363
|
)
|
|
271
364
|
```
|
|
272
365
|
|
|
366
|
+
This factory fragment assumes `vector_store` and `build_qa_chain(model)` already exist. The
|
|
367
|
+
standard `create_retrieval_chain` requires the `input` key and comes from `langchain-classic`;
|
|
368
|
+
install that package and import it before using this fragment.
|
|
369
|
+
|
|
273
370
|
Customize all boundaries when required:
|
|
274
371
|
|
|
275
372
|
```python
|
|
@@ -299,6 +396,12 @@ and message objects. The adapter always adds `langchain.runnable` latency. Set
|
|
|
299
396
|
`capture_message_usage=False` to disable automatic token extraction. A custom `usage_mapper` takes
|
|
300
397
|
precedence.
|
|
301
398
|
|
|
399
|
+
For a complete provider-backed pipeline—from documents and embeddings through optimization and
|
|
400
|
+
reports—run
|
|
401
|
+
[`examples/langchain_full_rag.py`](https://github.com/shivamshinde123/tunarag-python/blob/main/examples/langchain_full_rag.py).
|
|
402
|
+
The example loads `OPENROUTER_API_KEY` from a repository-root `.env` file and uses OpenRouter for
|
|
403
|
+
both chat generation and embeddings; copy `.env.example` to `.env` to get started.
|
|
404
|
+
|
|
302
405
|
## LangGraph integration
|
|
303
406
|
|
|
304
407
|
`LangGraphAdapter` has the same factory and mapping options.
|
|
@@ -350,8 +453,27 @@ adapter = LangGraphAdapter(
|
|
|
350
453
|
The adapter always adds `langgraph.runnable` latency. Framework errors remain visible to the engine's
|
|
351
454
|
retry and failure handling.
|
|
352
455
|
|
|
456
|
+
## Google ADK integration
|
|
457
|
+
|
|
458
|
+
Install `tunarag-python[google-adk]` on Python 3.10+. `GoogleADKAdapter(runner)` accepts a real ADK
|
|
459
|
+
2.x `Runner` or `InMemoryRunner`. Candidate parameters become initial session state. Each sample
|
|
460
|
+
and retry gets a fresh session, which is removed after success, failure, or cancellation. Providers,
|
|
461
|
+
agent tools, corpus retrieval, and runner ownership remain with the application.
|
|
462
|
+
|
|
463
|
+
Use `factory=lambda candidate: build_runner(candidate)` when tuning settings such as a model or
|
|
464
|
+
retrieval tool that require constructing an agent. `state_mapper` can map candidates to agent state;
|
|
465
|
+
`output_mapper` receives the final ADK event, allowing retrieval evidence to be exposed for RAGAS.
|
|
466
|
+
By default the output is `{"answer": final_text}`. Usage captures reported model token counts and
|
|
467
|
+
runner latency; dollar cost is unavailable unless your evaluator supplies a priced metric.
|
|
468
|
+
Missing final responses and ADK error events fail the trial rather than becoming empty answers.
|
|
469
|
+
|
|
470
|
+
See the executable [ADK example](https://github.com/shivamshinde123/tunarag-python/blob/main/examples/google_adk_tuning.py)
|
|
471
|
+
and [integration reference](https://github.com/shivamshinde123/tunarag-python/blob/main/docs/reference/integrations.md).
|
|
472
|
+
|
|
353
473
|
## Evaluation datasets
|
|
354
474
|
|
|
475
|
+

|
|
476
|
+
|
|
355
477
|
### Canonical model
|
|
356
478
|
|
|
357
479
|
```python
|
|
@@ -486,6 +608,8 @@ using them as gold data. Generation usage is separate from optimization usage.
|
|
|
486
608
|
|
|
487
609
|
## Search spaces and strategies
|
|
488
610
|
|
|
611
|
+

|
|
612
|
+
|
|
489
613
|
### Typed space
|
|
490
614
|
|
|
491
615
|
```python
|
|
@@ -861,6 +985,8 @@ labels are metadata, not encryption.
|
|
|
861
985
|
|
|
862
986
|
## SQLite persistence and resume
|
|
863
987
|
|
|
988
|
+

|
|
989
|
+
|
|
864
990
|
```python
|
|
865
991
|
from pathlib import Path
|
|
866
992
|
|
|
@@ -958,6 +1084,8 @@ reconciliation is not included in V0.2.
|
|
|
958
1084
|
|
|
959
1085
|
## Results and reports
|
|
960
1086
|
|
|
1087
|
+

|
|
1088
|
+
|
|
961
1089
|
`OptimizationResult` exposes:
|
|
962
1090
|
|
|
963
1091
|
```python
|
|
@@ -1040,7 +1168,7 @@ Protocols are structural; inheritance is not required.
|
|
|
1040
1168
|
|
|
1041
1169
|
| Contract | Required behavior |
|
|
1042
1170
|
|---|---|
|
|
1043
|
-
|
|
|
1171
|
+
| Official framework adapters | `LangChainAdapter`, `LangGraphAdapter`, `GoogleADKAdapter` |
|
|
1044
1172
|
| `Dataset` | `.version` and async iteration |
|
|
1045
1173
|
| `Evaluator` | `async evaluate(output, example) -> metrics` |
|
|
1046
1174
|
| `SearchStrategy` | `async suggest()` and `async observe(...)` |
|
|
@@ -1084,7 +1212,7 @@ Provider descriptions and component fingerprints must not expose secrets.
|
|
|
1084
1212
|
|
|
1085
1213
|
- Use a file-backed `SQLiteStore` and back it up.
|
|
1086
1214
|
- Keep a human-reviewed test split outside optimization.
|
|
1087
|
-
-
|
|
1215
|
+
- Supply framework pipeline revisions through adapter `identity` and fingerprint custom evaluators.
|
|
1088
1216
|
- Include a hard stopping bound and conservative concurrency.
|
|
1089
1217
|
- Track exact usage when available and label estimates.
|
|
1090
1218
|
- Configure sample and trial timeouts.
|
|
@@ -1100,7 +1228,7 @@ Provider descriptions and component fingerprints must not expose secrets.
|
|
|
1100
1228
|
V0.2 does not provide a hosted service/dashboard, automatic RAG construction, vector-database
|
|
1101
1229
|
provisioning, distributed workers, Pareto optimization, Bayesian/evolutionary search, encrypted
|
|
1102
1230
|
payloads, MLflow reconciliation, provider pricing management, or a TypeScript implementation.
|
|
1103
|
-
LangChain and
|
|
1231
|
+
LangChain, LangGraph, and Google ADK 2.x are official integrations. Additional frameworks require a future first-party integration; custom RAG adapters are unsupported.
|
|
1104
1232
|
|
|
1105
1233
|
| Term | Meaning |
|
|
1106
1234
|
|---|---|
|