tunarag-python 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tunarag/__init__.py +3 -1
- tunarag/contracts.py +1 -9
- tunarag/engine.py +9 -4
- tunarag/integrations/__init__.py +2 -2
- tunarag/integrations/google_adk.py +144 -0
- tunarag/integrations/runnables.py +19 -19
- tunarag/search.py +10 -2
- {tunarag_python-0.2.1.dist-info → tunarag_python-0.3.1.dist-info}/METADATA +207 -116
- {tunarag_python-0.2.1.dist-info → tunarag_python-0.3.1.dist-info}/RECORD +11 -9
- tunarag_python-0.3.1.dist-info/licenses/LICENSE +21 -0
- {tunarag_python-0.2.1.dist-info → tunarag_python-0.3.1.dist-info}/WHEEL +0 -0
tunarag/__init__.py
CHANGED
|
@@ -34,6 +34,7 @@ from .errors import (
|
|
|
34
34
|
TunaRAGError,
|
|
35
35
|
)
|
|
36
36
|
from .evaluators import MetricEvaluator, MetricScorer, RagasEvaluator, RagasSampleMapper
|
|
37
|
+
from .integrations.google_adk import GoogleADKAdapter
|
|
37
38
|
from .integrations.runnables import LangChainAdapter, LangGraphAdapter
|
|
38
39
|
from .objective import Objective, ObjectiveDirection, ObjectiveResult, ObjectiveTerm
|
|
39
40
|
from .result import OptimizationResult, TrialSummary, UsageSummary
|
|
@@ -103,6 +104,7 @@ __all__ = [
|
|
|
103
104
|
"EvaluationDataset",
|
|
104
105
|
"EvaluationExample",
|
|
105
106
|
"FloatRange",
|
|
107
|
+
"GoogleADKAdapter",
|
|
106
108
|
"InMemoryDataset",
|
|
107
109
|
"IntegerRange",
|
|
108
110
|
"LangChainAdapter",
|
|
@@ -163,4 +165,4 @@ __all__ = [
|
|
|
163
165
|
"__version__",
|
|
164
166
|
]
|
|
165
167
|
|
|
166
|
-
__version__ = "0.
|
|
168
|
+
__version__ = "0.3.1"
|
tunarag/contracts.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Internal pipeline boundary and public optimization extension protocols."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -12,14 +12,6 @@ if TYPE_CHECKING:
|
|
|
12
12
|
from .store import StudyEvent
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
class RAGAdapter(Protocol):
|
|
16
|
-
"""Execute a user's RAG system for one candidate and example."""
|
|
17
|
-
|
|
18
|
-
async def run(
|
|
19
|
-
self, candidate: Candidate, example: Any
|
|
20
|
-
) -> tuple[Any, Sequence[UsageRecord]]: ...
|
|
21
|
-
|
|
22
|
-
|
|
23
15
|
class Dataset(Protocol):
|
|
24
16
|
"""Provide versioned evaluation examples."""
|
|
25
17
|
|
tunarag/engine.py
CHANGED
|
@@ -18,12 +18,13 @@ from .contracts import (
|
|
|
18
18
|
Evaluator,
|
|
19
19
|
EventCallback,
|
|
20
20
|
Fingerprintable,
|
|
21
|
-
RAGAdapter,
|
|
22
21
|
ReplayableSearchStrategy,
|
|
23
22
|
SearchStrategy,
|
|
24
23
|
)
|
|
25
24
|
from .domain import Candidate, MetricValue, UsageRecord, ValueStatus
|
|
26
25
|
from .errors import ResumeError, SearchSpaceError, SyncInAsyncContextError
|
|
26
|
+
from .integrations.google_adk import GoogleADKAdapter
|
|
27
|
+
from .integrations.runnables import LangChainAdapter, LangGraphAdapter
|
|
27
28
|
from .objective import Objective, ObjectiveResult
|
|
28
29
|
from .result import OptimizationResult, TrialSummary, UsageSummary
|
|
29
30
|
from .retry import RetryPolicy
|
|
@@ -51,12 +52,12 @@ class _SampleResult:
|
|
|
51
52
|
|
|
52
53
|
|
|
53
54
|
class Optimizer:
|
|
54
|
-
"""
|
|
55
|
+
"""Tune pipelines through official LangChain, LangGraph, or Google ADK adapters."""
|
|
55
56
|
|
|
56
57
|
def __init__(
|
|
57
58
|
self,
|
|
58
59
|
*,
|
|
59
|
-
adapter:
|
|
60
|
+
adapter: LangChainAdapter | LangGraphAdapter | GoogleADKAdapter,
|
|
60
61
|
strategy: SearchStrategy,
|
|
61
62
|
evaluators: Sequence[Evaluator],
|
|
62
63
|
objective: Objective,
|
|
@@ -70,6 +71,11 @@ class Optimizer:
|
|
|
70
71
|
clock: Callable[[], float] = time.monotonic,
|
|
71
72
|
duplicate_suggestion_limit: int = 100,
|
|
72
73
|
) -> None:
|
|
74
|
+
if not isinstance(adapter, (LangChainAdapter, LangGraphAdapter, GoogleADKAdapter)):
|
|
75
|
+
raise TypeError(
|
|
76
|
+
"optimizer requires LangChainAdapter, LangGraphAdapter, or GoogleADKAdapter; "
|
|
77
|
+
"custom RAG adapters are unsupported"
|
|
78
|
+
)
|
|
73
79
|
if not evaluators:
|
|
74
80
|
raise ValueError("optimizer requires at least one evaluator")
|
|
75
81
|
if duplicate_suggestion_limit < 1:
|
|
@@ -300,7 +306,6 @@ class Optimizer:
|
|
|
300
306
|
"cause": type(error).__name__,
|
|
301
307
|
},
|
|
302
308
|
)
|
|
303
|
-
self._event_cursors[study_id] = self._latest_event_sequence(study_id)
|
|
304
309
|
|
|
305
310
|
def _batch_capacity(self, summaries: Sequence[TrialSummary]) -> int:
|
|
306
311
|
capacity = self._settings.concurrency
|
tunarag/integrations/__init__.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"""Optional integration adapters."""
|
|
2
2
|
|
|
3
|
+
from .google_adk import GoogleADKAdapter
|
|
3
4
|
from .mlflow import MLflowCallback
|
|
4
5
|
from .runnables import (
|
|
5
|
-
AsyncRunnable,
|
|
6
6
|
LangChainAdapter,
|
|
7
7
|
LangGraphAdapter,
|
|
8
8
|
RunnableConfigMapper,
|
|
@@ -13,7 +13,7 @@ from .runnables import (
|
|
|
13
13
|
)
|
|
14
14
|
|
|
15
15
|
__all__ = [
|
|
16
|
-
"
|
|
16
|
+
"GoogleADKAdapter",
|
|
17
17
|
"LangChainAdapter",
|
|
18
18
|
"LangGraphAdapter",
|
|
19
19
|
"MLflowCallback",
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Google ADK 2.x runner integration with isolated evaluation sessions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Callable, Mapping
|
|
7
|
+
from importlib import import_module
|
|
8
|
+
from typing import Any
|
|
9
|
+
from uuid import uuid4
|
|
10
|
+
|
|
11
|
+
from ..domain import Candidate, UsageRecord
|
|
12
|
+
from ..errors import AdapterError, MissingOptionalDependencyError
|
|
13
|
+
from .runnables import _callable_identity, _token_count, _type_name
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class GoogleADKAdapter:
|
|
17
|
+
"""Evaluate a Google ADK runner in a fresh session for every sample and retry.
|
|
18
|
+
|
|
19
|
+
Candidate parameters become initial session state. Use a factory when changing
|
|
20
|
+
an agent's model, tools, or graph requires constructing a new framework runner.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
runner: Any = None,
|
|
26
|
+
*,
|
|
27
|
+
factory: Callable[[Candidate], Any] | None = None,
|
|
28
|
+
state_mapper: Callable[[Candidate], Mapping[str, Any]] | None = None,
|
|
29
|
+
output_mapper: Callable[[Any], Any] | None = None,
|
|
30
|
+
identity: Any = None,
|
|
31
|
+
) -> None:
|
|
32
|
+
if (runner is None) == (factory is None):
|
|
33
|
+
raise ValueError("provide exactly one of runner or factory")
|
|
34
|
+
for name, callback in (
|
|
35
|
+
("factory", factory),
|
|
36
|
+
("state_mapper", state_mapper),
|
|
37
|
+
("output_mapper", output_mapper),
|
|
38
|
+
):
|
|
39
|
+
if callback is not None and not callable(callback):
|
|
40
|
+
raise TypeError(f"{name} must be callable")
|
|
41
|
+
if runner is not None:
|
|
42
|
+
_require_runner(runner)
|
|
43
|
+
self._runner = runner
|
|
44
|
+
self._factory = factory
|
|
45
|
+
self._state_mapper = state_mapper
|
|
46
|
+
self._output_mapper = output_mapper
|
|
47
|
+
self._identity = identity
|
|
48
|
+
|
|
49
|
+
async def run(self, candidate: Candidate, example: Any) -> tuple[Any, tuple[UsageRecord, ...]]:
|
|
50
|
+
"""Consume runner events, capture usage, and remove the temporary session."""
|
|
51
|
+
runner = self._runner if self._factory is None else self._factory(candidate)
|
|
52
|
+
_require_runner(runner)
|
|
53
|
+
types = import_module("google.genai.types")
|
|
54
|
+
query = getattr(example, "query", example)
|
|
55
|
+
if not isinstance(query, str) or not query.strip():
|
|
56
|
+
raise TypeError("Google ADK evaluation input must contain a nonempty query")
|
|
57
|
+
state = (
|
|
58
|
+
dict(candidate.parameters)
|
|
59
|
+
if self._state_mapper is None
|
|
60
|
+
else self._state_mapper(candidate)
|
|
61
|
+
)
|
|
62
|
+
if not isinstance(state, Mapping):
|
|
63
|
+
raise TypeError("state_mapper must return a mapping")
|
|
64
|
+
session_id = uuid4().hex
|
|
65
|
+
user_id = "tunarag-evaluation"
|
|
66
|
+
session = await runner.session_service.create_session(
|
|
67
|
+
app_name=runner.app_name,
|
|
68
|
+
user_id=user_id,
|
|
69
|
+
session_id=session_id,
|
|
70
|
+
state=dict(state),
|
|
71
|
+
)
|
|
72
|
+
usage: list[UsageRecord] = []
|
|
73
|
+
final_event = None
|
|
74
|
+
started_at = time.perf_counter()
|
|
75
|
+
events = runner.run_async(
|
|
76
|
+
user_id=user_id,
|
|
77
|
+
session_id=session.id,
|
|
78
|
+
new_message=types.Content(role="user", parts=[types.Part.from_text(text=query)]),
|
|
79
|
+
)
|
|
80
|
+
try:
|
|
81
|
+
async for event in events:
|
|
82
|
+
if event.error_code:
|
|
83
|
+
raise AdapterError("Google ADK runner returned an error event", stage="adapter")
|
|
84
|
+
metadata = event.usage_metadata
|
|
85
|
+
if metadata is not None:
|
|
86
|
+
usage.append(
|
|
87
|
+
UsageRecord(
|
|
88
|
+
"google_adk.model",
|
|
89
|
+
input_tokens=_token_count(metadata.prompt_token_count),
|
|
90
|
+
output_tokens=_token_count(metadata.candidates_token_count),
|
|
91
|
+
)
|
|
92
|
+
)
|
|
93
|
+
if event.is_final_response():
|
|
94
|
+
final_event = event
|
|
95
|
+
if final_event is None:
|
|
96
|
+
raise AdapterError("Google ADK runner produced no final response", stage="adapter")
|
|
97
|
+
if self._output_mapper is None and final_event.content is None:
|
|
98
|
+
raise AdapterError(
|
|
99
|
+
"ADK final event has no text; provide output_mapper for structured output",
|
|
100
|
+
stage="adapter",
|
|
101
|
+
)
|
|
102
|
+
output = (
|
|
103
|
+
self._output_mapper(final_event)
|
|
104
|
+
if self._output_mapper is not None
|
|
105
|
+
else {
|
|
106
|
+
"answer": "".join(
|
|
107
|
+
part.text or ""
|
|
108
|
+
for part in final_event.content.parts or ()
|
|
109
|
+
if not part.thought
|
|
110
|
+
)
|
|
111
|
+
}
|
|
112
|
+
)
|
|
113
|
+
usage.append(
|
|
114
|
+
UsageRecord("google_adk.runner", latency_seconds=time.perf_counter() - started_at)
|
|
115
|
+
)
|
|
116
|
+
return output, tuple(usage)
|
|
117
|
+
finally:
|
|
118
|
+
try:
|
|
119
|
+
await events.aclose()
|
|
120
|
+
finally:
|
|
121
|
+
await runner.session_service.delete_session(
|
|
122
|
+
app_name=runner.app_name, user_id=user_id, session_id=session.id
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
def fingerprint(self) -> dict[str, Any]:
|
|
126
|
+
"""Describe integration semantics without sessions or provider credentials."""
|
|
127
|
+
return {
|
|
128
|
+
"integration": "google_adk",
|
|
129
|
+
"schema_version": 1,
|
|
130
|
+
"runner": _type_name(self._runner) if self._runner is not None else None,
|
|
131
|
+
"factory": _callable_identity(self._factory),
|
|
132
|
+
"state_mapper": _callable_identity(self._state_mapper),
|
|
133
|
+
"output_mapper": _callable_identity(self._output_mapper),
|
|
134
|
+
"identity": self._identity,
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _require_runner(value: Any) -> None:
|
|
139
|
+
try:
|
|
140
|
+
runner_type = import_module("google.adk.runners").Runner
|
|
141
|
+
except ImportError as error:
|
|
142
|
+
raise MissingOptionalDependencyError("Google ADK", "google-adk") from error
|
|
143
|
+
if not isinstance(value, runner_type):
|
|
144
|
+
raise TypeError("GoogleADKAdapter requires a real Google ADK Runner")
|
|
@@ -5,22 +5,12 @@ from __future__ import annotations
|
|
|
5
5
|
import inspect
|
|
6
6
|
import time
|
|
7
7
|
from collections.abc import Callable, Mapping, Sequence
|
|
8
|
-
from
|
|
8
|
+
from importlib import import_module
|
|
9
|
+
from typing import Any
|
|
9
10
|
|
|
10
11
|
from ..dataset import EvaluationExample
|
|
11
12
|
from ..domain import Candidate, UsageRecord
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class AsyncRunnable(Protocol):
|
|
15
|
-
"""Shared asynchronous invocation surface implemented by both frameworks."""
|
|
16
|
-
|
|
17
|
-
async def ainvoke(
|
|
18
|
-
self,
|
|
19
|
-
input: Any,
|
|
20
|
-
config: Any = None,
|
|
21
|
-
**kwargs: Any,
|
|
22
|
-
) -> Any: ...
|
|
23
|
-
|
|
13
|
+
from ..errors import MissingOptionalDependencyError
|
|
24
14
|
|
|
25
15
|
RunnableFactory = Callable[[Candidate], Any]
|
|
26
16
|
RunnableInputMapper = Callable[[Any, Candidate], Any]
|
|
@@ -47,7 +37,7 @@ class _RunnableAdapter:
|
|
|
47
37
|
if (runnable is None) == (factory is None):
|
|
48
38
|
raise ValueError("provide exactly one of runnable or factory")
|
|
49
39
|
if runnable is not None:
|
|
50
|
-
_require_runnable(runnable)
|
|
40
|
+
_require_runnable(runnable, framework)
|
|
51
41
|
if factory is not None and not callable(factory):
|
|
52
42
|
raise TypeError("runnable factory must be callable")
|
|
53
43
|
if not input_key.strip():
|
|
@@ -116,7 +106,7 @@ class _RunnableAdapter:
|
|
|
116
106
|
if self._factory is None:
|
|
117
107
|
raise RuntimeError("runnable adapter is not configured")
|
|
118
108
|
runnable = self._factory(candidate)
|
|
119
|
-
_require_runnable(runnable)
|
|
109
|
+
_require_runnable(runnable, self._framework)
|
|
120
110
|
return runnable
|
|
121
111
|
|
|
122
112
|
def _map_input(self, example: Any, candidate: Candidate) -> Any:
|
|
@@ -251,10 +241,20 @@ def _token_count(value: Any) -> int | None:
|
|
|
251
241
|
return value
|
|
252
242
|
|
|
253
243
|
|
|
254
|
-
def _require_runnable(value: Any) -> None:
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
244
|
+
def _require_runnable(value: Any, framework: str) -> None:
|
|
245
|
+
module, name = (
|
|
246
|
+
("langchain_core.runnables", "Runnable")
|
|
247
|
+
if framework == "langchain"
|
|
248
|
+
else ("langgraph.graph.state", "CompiledStateGraph")
|
|
249
|
+
)
|
|
250
|
+
try:
|
|
251
|
+
framework_type = getattr(import_module(module), name)
|
|
252
|
+
except ImportError as error:
|
|
253
|
+
raise MissingOptionalDependencyError(framework, framework) from error
|
|
254
|
+
if not isinstance(value, framework_type):
|
|
255
|
+
raise TypeError(
|
|
256
|
+
f"{framework} integration requires a real {name}, not a custom ainvoke object"
|
|
257
|
+
)
|
|
258
258
|
|
|
259
259
|
|
|
260
260
|
def _callable_identity(value: Callable[..., Any] | None) -> str | None:
|
tunarag/search.py
CHANGED
|
@@ -147,6 +147,7 @@ class RandomSearch:
|
|
|
147
147
|
self._space = space
|
|
148
148
|
self._seed = seed
|
|
149
149
|
self._generator = random.Random(seed)
|
|
150
|
+
self._replayed_candidates: list[Candidate] = []
|
|
150
151
|
|
|
151
152
|
async def suggest(self) -> Candidate:
|
|
152
153
|
return self._space.sample(self._generator)
|
|
@@ -158,9 +159,16 @@ class RandomSearch:
|
|
|
158
159
|
"""Advance and verify deterministic generator state during resume."""
|
|
159
160
|
|
|
160
161
|
del metrics, usage
|
|
161
|
-
|
|
162
|
-
if expected != candidate:
|
|
162
|
+
if candidate in self._replayed_candidates:
|
|
163
163
|
raise ValueError("persisted candidate does not match seeded random search state")
|
|
164
|
+
for _ in range(10_000):
|
|
165
|
+
expected = self._space.sample(self._generator)
|
|
166
|
+
if expected == candidate:
|
|
167
|
+
self._replayed_candidates.append(candidate)
|
|
168
|
+
return
|
|
169
|
+
if expected not in self._replayed_candidates:
|
|
170
|
+
break
|
|
171
|
+
raise ValueError("persisted candidate does not match seeded random search state")
|
|
164
172
|
|
|
165
173
|
def fingerprint(self) -> Any:
|
|
166
174
|
"""Return stable search-space and seed semantics."""
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tunarag-python
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Tune LangChain, LangGraph, and Google ADK RAG pipelines
|
|
5
|
+
Project-URL: Documentation, https://shivamshinde123.github.io/tunarag-python/
|
|
5
6
|
Project-URL: Repository, https://github.com/shivamshinde123/tunarag-python
|
|
6
7
|
Project-URL: Issues, https://github.com/shivamshinde123/tunarag-python/issues
|
|
7
8
|
Project-URL: Changelog, https://github.com/shivamshinde123/tunarag-python/blob/main/CHANGELOG.md
|
|
8
9
|
Author: Shivam Shinde
|
|
9
10
|
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
10
12
|
Keywords: evaluation,llm,optimization,rag,retrieval
|
|
11
13
|
Classifier: Development Status :: 3 - Alpha
|
|
12
14
|
Classifier: Intended Audience :: Developers
|
|
@@ -23,20 +25,48 @@ Requires-Dist: tomli>=2.0; python_version < '3.11'
|
|
|
23
25
|
Provides-Extra: dev
|
|
24
26
|
Requires-Dist: build>=1.2; extra == 'dev'
|
|
25
27
|
Requires-Dist: hypothesis>=6.100; extra == 'dev'
|
|
28
|
+
Requires-Dist: langchain-classic<2,>=1; extra == 'dev'
|
|
29
|
+
Requires-Dist: langchain-core<2,>=1; extra == 'dev'
|
|
30
|
+
Requires-Dist: langgraph<2,>=1; extra == 'dev'
|
|
26
31
|
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
32
|
+
Requires-Dist: numpy<3,>=1.26; extra == 'dev'
|
|
27
33
|
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
28
34
|
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
29
35
|
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
36
|
+
Provides-Extra: docs
|
|
37
|
+
Requires-Dist: mkdocs-include-markdown-plugin<8,>=7; extra == 'docs'
|
|
38
|
+
Requires-Dist: mkdocs-material<10,>=9.5; extra == 'docs'
|
|
39
|
+
Requires-Dist: mkdocs<2,>=1.6; extra == 'docs'
|
|
40
|
+
Requires-Dist: mkdocstrings[python]<1,>=0.27; extra == 'docs'
|
|
30
41
|
Provides-Extra: frameworks
|
|
42
|
+
Requires-Dist: google-adk<3,>=2; extra == 'frameworks'
|
|
31
43
|
Requires-Dist: langchain<2,>=1; extra == 'frameworks'
|
|
32
44
|
Requires-Dist: langgraph<2,>=1; extra == 'frameworks'
|
|
45
|
+
Requires-Dist: numpy<3,>=1.26; extra == 'frameworks'
|
|
46
|
+
Provides-Extra: google-adk
|
|
47
|
+
Requires-Dist: google-adk<3,>=2; extra == 'google-adk'
|
|
33
48
|
Provides-Extra: langchain
|
|
34
49
|
Requires-Dist: langchain<2,>=1; extra == 'langchain'
|
|
50
|
+
Requires-Dist: numpy<3,>=1.26; extra == 'langchain'
|
|
35
51
|
Provides-Extra: langgraph
|
|
36
52
|
Requires-Dist: langgraph<2,>=1; extra == 'langgraph'
|
|
53
|
+
Provides-Extra: live-adk-test
|
|
54
|
+
Requires-Dist: google-adk<3,>=2.11; extra == 'live-adk-test'
|
|
55
|
+
Requires-Dist: litellm<2,>=1.104; extra == 'live-adk-test'
|
|
56
|
+
Provides-Extra: live-test
|
|
57
|
+
Requires-Dist: langchain-classic<2,>=1; extra == 'live-test'
|
|
58
|
+
Requires-Dist: langchain-community<0.4,>=0.3.31; extra == 'live-test'
|
|
59
|
+
Requires-Dist: langchain-openai<2,>=1; extra == 'live-test'
|
|
60
|
+
Requires-Dist: langgraph<2,>=1; extra == 'live-test'
|
|
61
|
+
Requires-Dist: mlflow<4,>=3; extra == 'live-test'
|
|
62
|
+
Requires-Dist: pypdf<7,>=5; extra == 'live-test'
|
|
63
|
+
Requires-Dist: pytest>=8; extra == 'live-test'
|
|
64
|
+
Requires-Dist: python-dotenv<2,>=1; extra == 'live-test'
|
|
65
|
+
Requires-Dist: ragas<0.5,>=0.4; extra == 'live-test'
|
|
37
66
|
Provides-Extra: mlflow
|
|
38
67
|
Requires-Dist: mlflow<4,>=3; extra == 'mlflow'
|
|
39
68
|
Provides-Extra: ragas
|
|
69
|
+
Requires-Dist: langchain-community<0.4,>=0.3.31; extra == 'ragas'
|
|
40
70
|
Requires-Dist: ragas<0.5,>=0.4; extra == 'ragas'
|
|
41
71
|
Description-Content-Type: text/markdown
|
|
42
72
|
|
|
@@ -58,7 +88,7 @@ isolates failures, supports resumable execution, and produces results you can in
|
|
|
58
88
|
- Compare configurations with custom metrics or optional RAGAS evaluators.
|
|
59
89
|
- Optimize a weighted objective across quality, provider cost, and response latency.
|
|
60
90
|
- Search with deterministic random search or provider-neutral LLM-guided search.
|
|
61
|
-
- Tune
|
|
91
|
+
- Tune LangChain, LangGraph, or Google ADK RAG pipelines without rebuilding them.
|
|
62
92
|
- Preserve reproducible experiments with caching, retries, callbacks, reports, and SQLite resume.
|
|
63
93
|
|
|
64
94
|
### Who is it for?
|
|
@@ -70,9 +100,30 @@ indexes, provider clients, and credentials; TunaRAG owns the optimization workfl
|
|
|
70
100
|
> Install the distribution as `tunarag-python` and import it as `tunarag`. Python 3.10-3.13 is
|
|
71
101
|
> supported.
|
|
72
102
|
|
|
103
|
+
**Documentation:** [shivamshinde123.github.io/tunarag-python](https://shivamshinde123.github.io/tunarag-python/)
|
|
104
|
+
|
|
105
|
+
**Lightweight distribution:** pip installs runtime code and metadata, not tests, datasets,
|
|
106
|
+
or documentation images. Framework dependencies are optional extras. See
|
|
107
|
+
[package contents and release checks](https://shivamshinde123.github.io/tunarag-python/package-distribution/).
|
|
108
|
+
|
|
109
|
+
**Live integration checks:** [OpenRouter, RAGAS, and MLflow test plan](https://shivamshinde123.github.io/tunarag-python/live-integration-test-plan/).
|
|
110
|
+
|
|
111
|
+
**End-to-end verification:** [Full-PDF campaigns, held-out quality checks, crash recovery,
|
|
112
|
+
and clean-wheel testing](https://shivamshinde123.github.io/tunarag-python/comprehensive-testing/). Paid tests are explicit opt-ins;
|
|
113
|
+
ordinary CI never makes paid provider calls.
|
|
114
|
+
These opt-in tests include spending safeguards; normal tests never make paid calls.
|
|
115
|
+
|
|
116
|
+
**Google ADK validation:** [Real OpenRouter RAG tuning, RAGAS scoring, SQLite resume,
|
|
117
|
+
and MLflow checks](https://shivamshinde123.github.io/tunarag-python/live-google-adk-testing/).
|
|
118
|
+
|
|
119
|
+
**Quality and reliability:** [Ordered validation plan](https://shivamshinde123.github.io/tunarag-python/quality-validation-plan/)
|
|
120
|
+
and [bounded repeated tuning, ADK tools, billing and recovery checks](https://shivamshinde123.github.io/tunarag-python/reliability-validation/).
|
|
121
|
+
See [mixed-topic validation](https://shivamshinde123.github.io/tunarag-python/multicorpus-validation/) for cross-model QA,
|
|
122
|
+
simulated faults/load, and the unresolved adversarial-prompt failure.
|
|
123
|
+
|
|
73
124
|
## Comprehensive Usage Guide
|
|
74
125
|
|
|
75
|
-
The sections below explain every public TunaRAG Python V0.
|
|
126
|
+
The sections below explain every public TunaRAG Python V0.3 feature with executable examples.
|
|
76
127
|
|
|
77
128
|
## Contents
|
|
78
129
|
|
|
@@ -105,20 +156,7 @@ TunaRAG compares configurations of an **existing** RAG pipeline. You provide the
|
|
|
105
156
|
evaluation dataset, a search space, evaluators, and an objective. TunaRAG schedules trials, isolates
|
|
106
157
|
failures, persists state, and returns an inspectable result.
|
|
107
158
|
|
|
108
|
-
|
|
109
|
-
flowchart LR
|
|
110
|
-
D[Versioned dataset] --> O[Optimizer]
|
|
111
|
-
S[Search strategy] -->|Candidate| O
|
|
112
|
-
O -->|Candidate + example| A[RAG adapter]
|
|
113
|
-
A -->|Output + usage| E[Evaluators]
|
|
114
|
-
E -->|Metrics| J[Objective]
|
|
115
|
-
J -->|Benefit score| O
|
|
116
|
-
O -->|Observe| S
|
|
117
|
-
O --> DB[(SQLite store)]
|
|
118
|
-
O <--> C[(Evaluation cache)]
|
|
119
|
-
DB --> CB[Post-commit callbacks]
|
|
120
|
-
O --> R[OptimizationResult]
|
|
121
|
-
```
|
|
159
|
+

|
|
122
160
|
|
|
123
161
|
A **study** contains ordered **trials**. Each trial evaluates one complete candidate configuration
|
|
124
162
|
against every dataset example. Same-named sample metrics are aggregated into trial metrics, then the
|
|
@@ -137,141 +175,143 @@ python -m pip install tunarag-python
|
|
|
137
175
|
|---|---|
|
|
138
176
|
| LangChain | `python -m pip install "tunarag-python[langchain]"` |
|
|
139
177
|
| LangGraph | `python -m pip install "tunarag-python[langgraph]"` |
|
|
140
|
-
|
|
|
178
|
+
| LangChain and LangGraph; ADK on Python 3.10+ | `python -m pip install "tunarag-python[frameworks]"` |
|
|
179
|
+
| Google ADK 2.x (Python 3.10+) | `python -m pip install "tunarag-python[google-adk]"` |
|
|
141
180
|
| RAGAS | `python -m pip install "tunarag-python[ragas]"` |
|
|
142
181
|
| MLflow | `python -m pip install "tunarag-python[mlflow]"` |
|
|
143
182
|
|
|
144
|
-
|
|
183
|
+
Framework dependencies load lazily. Install the extra for your pipeline before constructing its adapter. Google ADK requires Python 3.10 or newer; LangChain and LangGraph retain Python 3.10 support.
|
|
145
184
|
|
|
146
185
|
## End-to-end quickstart
|
|
147
186
|
|
|
148
|
-
|
|
187
|
+
Use an installed framework integration. The offline quickstart uses LangChain's vector store,
|
|
188
|
+
retriever, prompt, and fake model; the provider-backed example uses OpenRouter. Fake embeddings and
|
|
189
|
+
the fake model test wiring only; their scores do not demonstrate retrieval or answer quality.
|
|
149
190
|
|
|
150
191
|
```python
|
|
192
|
+
from pathlib import Path
|
|
193
|
+
|
|
194
|
+
from langchain_core.documents import Document
|
|
195
|
+
from langchain_core.embeddings import DeterministicFakeEmbedding
|
|
196
|
+
from langchain_core.language_models.fake import FakeListLLM
|
|
197
|
+
from langchain_core.prompts import ChatPromptTemplate
|
|
198
|
+
from langchain_core.runnables import RunnablePassthrough
|
|
199
|
+
from langchain_core.vectorstores import InMemoryVectorStore
|
|
151
200
|
from tunarag import (
|
|
152
201
|
EvaluationDataset,
|
|
153
202
|
IntegerRange,
|
|
203
|
+
LangChainAdapter,
|
|
154
204
|
MetricEvaluator,
|
|
155
205
|
Objective,
|
|
156
206
|
ObjectiveTerm,
|
|
157
207
|
Optimizer,
|
|
158
208
|
RandomSearch,
|
|
159
209
|
SearchSpace,
|
|
160
|
-
UsageRecord,
|
|
161
210
|
)
|
|
162
211
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
async def run(self, candidate, example):
|
|
172
|
-
query_words = set(example.query.lower().split())
|
|
173
|
-
ranked = sorted(
|
|
174
|
-
DOCUMENTS.values(),
|
|
175
|
-
key=lambda text: len(query_words & set(text.lower().split())),
|
|
176
|
-
reverse=True,
|
|
177
|
-
)
|
|
178
|
-
contexts = ranked[: candidate.parameters["top_k"]]
|
|
179
|
-
return {"answer": " ".join(contexts)}, (UsageRecord("small-rag", latency_seconds=0.001),)
|
|
212
|
+
vector_store = InMemoryVectorStore.from_documents(
|
|
213
|
+
[
|
|
214
|
+
Document(page_content="SQLite stores local experiments."),
|
|
215
|
+
Document(page_content="TunaRAG compares RAG configurations."),
|
|
216
|
+
],
|
|
217
|
+
DeterministicFakeEmbedding(size=32),
|
|
218
|
+
)
|
|
219
|
+
prompt = ChatPromptTemplate.from_template("Context: {context}\nQuestion: {question}")
|
|
180
220
|
|
|
181
221
|
|
|
182
|
-
def
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
222
|
+
def build_chain(candidate):
|
|
223
|
+
retriever = vector_store.as_retriever(search_kwargs={"k": candidate.parameters["top_k"]})
|
|
224
|
+
return {"context": retriever, "question": RunnablePassthrough()} | RunnablePassthrough.assign(
|
|
225
|
+
answer=prompt | FakeListLLM(responses=["SQLite stores local experiments."])
|
|
226
|
+
)
|
|
186
227
|
|
|
187
228
|
|
|
188
229
|
dataset = EvaluationDataset.from_records(
|
|
189
230
|
[
|
|
190
231
|
{
|
|
191
232
|
"id": "q1",
|
|
192
|
-
"query": "What
|
|
193
|
-
"reference_answer": "
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
},
|
|
233
|
+
"query": "What stores experiments and compares configurations?",
|
|
234
|
+
"reference_answer": "SQLite and TunaRAG",
|
|
235
|
+
"reference_contexts": [
|
|
236
|
+
"SQLite stores local experiments.",
|
|
237
|
+
"TunaRAG compares RAG configurations.",
|
|
238
|
+
],
|
|
239
|
+
}
|
|
200
240
|
]
|
|
201
241
|
)
|
|
202
242
|
optimizer = Optimizer(
|
|
203
|
-
adapter=
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
243
|
+
adapter=LangChainAdapter(
|
|
244
|
+
factory=build_chain, input_mapper=lambda example, candidate: example.query
|
|
245
|
+
),
|
|
246
|
+
strategy=RandomSearch(SearchSpace({"top_k": IntegerRange(1, 2)}), seed=7),
|
|
247
|
+
evaluators=(
|
|
248
|
+
MetricEvaluator(
|
|
249
|
+
"retrieval_recall",
|
|
250
|
+
lambda output, example: (
|
|
251
|
+
len(
|
|
252
|
+
{document.page_content for document in output["context"]}
|
|
253
|
+
& set(example.reference_contexts)
|
|
254
|
+
)
|
|
255
|
+
/ len(example.reference_contexts)
|
|
256
|
+
),
|
|
257
|
+
),
|
|
258
|
+
),
|
|
259
|
+
objective=Objective((ObjectiveTerm("retrieval_recall"),)),
|
|
207
260
|
)
|
|
208
261
|
try:
|
|
209
|
-
result = optimizer.optimize(dataset, study_name="
|
|
262
|
+
result = optimizer.optimize(dataset, study_name="langchain-rag")
|
|
210
263
|
print(result.best_candidate.parameters if result.best_candidate else "no winner")
|
|
211
264
|
finally:
|
|
212
265
|
optimizer.close()
|
|
213
266
|
```
|
|
214
267
|
|
|
215
268
|
The default store is in memory. Use `SQLiteStore(path)` for a durable study.
|
|
269
|
+
Here `top_k=1` retrieves one of two required contexts (recall `0.5`); `top_k=2` retrieves both
|
|
270
|
+
(recall `1.0`), so tuning selects `top_k=2`. This verifies parameter application and ranking with
|
|
271
|
+
offline framework components; it does not benchmark a real embedding model or generated answer.
|
|
216
272
|
|
|
217
273
|
## RAG adapters
|
|
218
274
|
|
|
219
|
-
|
|
275
|
+
Choose an official adapter: `LangChainAdapter`, `LangGraphAdapter`, or `GoogleADKAdapter`.
|
|
276
|
+
The optimizer rejects handwritten `run` adapters. LangChain accepts genuine framework `Runnable`
|
|
277
|
+
objects; LangGraph accepts `CompiledStateGraph`; Google ADK accepts `Runner`. Factory results are
|
|
278
|
+
validated on every sample, and failures enter the normal trial failure and retry boundary.
|
|
220
279
|
|
|
221
280
|
```python
|
|
222
|
-
from
|
|
223
|
-
|
|
224
|
-
from tunarag import UsageRecord
|
|
225
|
-
|
|
281
|
+
from langchain_core.runnables import RunnableLambda
|
|
282
|
+
from tunarag import LangChainAdapter
|
|
226
283
|
|
|
227
|
-
|
|
284
|
+
chain = RunnableLambda(lambda inputs: {"answer": inputs["question"]})
|
|
285
|
+
adapter = LangChainAdapter(chain)
|
|
228
286
|
```
|
|
229
287
|
|
|
230
|
-
|
|
231
|
-
by your evaluators. Usage may be empty. Adapter exceptions enter TunaRAG's retry and trial-failure
|
|
232
|
-
boundary.
|
|
288
|
+

|
|
233
289
|
|
|
234
|
-
|
|
235
|
-
|
|
290
|
+
The adapter connects two responsibilities; it does not move your RAG into TunaRAG.
|
|
291
|
+
Your framework must consume the candidate settings for tuning to change its behavior.
|
|
236
292
|
|
|
293
|
+
Candidate parameters go into framework configuration or initial ADK session state. A parameter
|
|
294
|
+
only changes behavior if the framework consumes it. For model, prompt, retriever, or index choices
|
|
295
|
+
that require rebuilding the pipeline, use a per-candidate framework factory. Mapping callbacks
|
|
296
|
+
adapt framework inputs and outputs; they do not require implementing a RAG adapter.
|
|
237
297
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
async def run(self, candidate, example):
|
|
243
|
-
response = await self.service.answer(
|
|
244
|
-
query=example.query,
|
|
245
|
-
top_k=candidate.parameters["top_k"],
|
|
246
|
-
model=candidate.parameters["model"],
|
|
247
|
-
)
|
|
248
|
-
return response, (
|
|
249
|
-
UsageRecord(
|
|
250
|
-
component="generator",
|
|
251
|
-
input_tokens=response.input_tokens,
|
|
252
|
-
output_tokens=response.output_tokens,
|
|
253
|
-
cost=response.cost,
|
|
254
|
-
latency_seconds=response.latency_seconds,
|
|
255
|
-
status=ValueStatus.EXACT,
|
|
256
|
-
pricing_version="provider-2026-10",
|
|
257
|
-
),
|
|
258
|
-
)
|
|
298
|
+
```python
|
|
299
|
+
from langchain_core.runnables import RunnableLambda
|
|
300
|
+
from tunarag import LangChainAdapter
|
|
259
301
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
"prompt_revision": "qa-v3",
|
|
265
|
-
}
|
|
302
|
+
adapter = LangChainAdapter(
|
|
303
|
+
RunnableLambda(lambda inputs: {"answer": inputs["question"]}),
|
|
304
|
+
identity={"index_revision": "docs-v1", "prompt_revision": "qa-v1"},
|
|
305
|
+
)
|
|
266
306
|
```
|
|
267
307
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
308
|
+
Include stable corpus, index, model, prompt, and mapper revisions in `identity` for cache and resume
|
|
309
|
+
compatibility. Never include credentials. Framework factories and mapping callbacks remain normal
|
|
310
|
+
Python configuration code; retrieval and generation execute through the framework.
|
|
271
311
|
|
|
272
312
|
## LangChain integration
|
|
273
313
|
|
|
274
|
-
`LangChainAdapter` accepts
|
|
314
|
+
`LangChainAdapter` accepts a real LangChain `Runnable` with `ainvoke(input, config=...)`.
|
|
275
315
|
|
|
276
316
|
```python
|
|
277
317
|
from langchain_core.runnables import RunnableLambda
|
|
@@ -307,11 +347,16 @@ def build_chain(candidate):
|
|
|
307
347
|
|
|
308
348
|
adapter = LangChainAdapter(
|
|
309
349
|
factory=build_chain,
|
|
350
|
+
input_key="input",
|
|
310
351
|
output_mapper=lambda output: output["answer"],
|
|
311
352
|
identity={"factory_schema": 1},
|
|
312
353
|
)
|
|
313
354
|
```
|
|
314
355
|
|
|
356
|
+
This factory fragment assumes `vector_store` and `build_qa_chain(model)` already exist. The
|
|
357
|
+
standard `create_retrieval_chain` requires the `input` key and comes from `langchain-classic`;
|
|
358
|
+
install that package and import it before using this fragment.
|
|
359
|
+
|
|
315
360
|
Customize all boundaries when required:
|
|
316
361
|
|
|
317
362
|
```python
|
|
@@ -341,6 +386,12 @@ and message objects. The adapter always adds `langchain.runnable` latency. Set
|
|
|
341
386
|
`capture_message_usage=False` to disable automatic token extraction. A custom `usage_mapper` takes
|
|
342
387
|
precedence.
|
|
343
388
|
|
|
389
|
+
For a complete provider-backed pipeline—from documents and embeddings through optimization and
|
|
390
|
+
reports—run
|
|
391
|
+
[`examples/langchain_full_rag.py`](https://github.com/shivamshinde123/tunarag-python/blob/main/examples/langchain_full_rag.py).
|
|
392
|
+
The example loads `OPENROUTER_API_KEY` from a repository-root `.env` file and uses OpenRouter for
|
|
393
|
+
both chat generation and embeddings; copy `.env.example` to `.env` to get started.
|
|
394
|
+
|
|
344
395
|
## LangGraph integration
|
|
345
396
|
|
|
346
397
|
`LangGraphAdapter` has the same factory and mapping options.
|
|
@@ -392,6 +443,23 @@ adapter = LangGraphAdapter(
|
|
|
392
443
|
The adapter always adds `langgraph.runnable` latency. Framework errors remain visible to the engine's
|
|
393
444
|
retry and failure handling.
|
|
394
445
|
|
|
446
|
+
## Google ADK integration
|
|
447
|
+
|
|
448
|
+
Install `tunarag-python[google-adk]` on Python 3.10+. `GoogleADKAdapter(runner)` accepts a real ADK
|
|
449
|
+
2.x `Runner` or `InMemoryRunner`. Candidate parameters become initial session state. Each sample
|
|
450
|
+
and retry gets a fresh session, which is removed after success, failure, or cancellation. Providers,
|
|
451
|
+
agent tools, corpus retrieval, and runner ownership remain with the application.
|
|
452
|
+
|
|
453
|
+
Use `factory=lambda candidate: build_runner(candidate)` when tuning settings such as a model or
|
|
454
|
+
retrieval tool that require constructing an agent. `state_mapper` can map candidates to agent state;
|
|
455
|
+
`output_mapper` receives the final ADK event, allowing retrieval evidence to be exposed for RAGAS.
|
|
456
|
+
By default the output is `{"answer": final_text}`. Usage captures reported model token counts and
|
|
457
|
+
runner latency; dollar cost is unavailable unless your evaluator supplies a priced metric.
|
|
458
|
+
Missing final responses and ADK error events fail the trial rather than becoming empty answers.
|
|
459
|
+
|
|
460
|
+
See the executable [ADK example](https://github.com/shivamshinde123/tunarag-python/blob/main/examples/google_adk_tuning.py)
|
|
461
|
+
and [integration reference](https://github.com/shivamshinde123/tunarag-python/blob/main/docs/reference/integrations.md).
|
|
462
|
+
|
|
395
463
|
## Evaluation datasets
|
|
396
464
|
|
|
397
465
|
### Canonical model
|
|
@@ -462,6 +530,13 @@ optimization_data = split_dataset.select(DatasetSplit.OPTIMIZE)
|
|
|
462
530
|
Splits are hash-based and stable across input order. Group splitting keeps related examples
|
|
463
531
|
together. `select()` rejects an empty split.
|
|
464
532
|
|
|
533
|
+

|
|
534
|
+
|
|
535
|
+
Pass only the selected optimization subset to the tuning study. Use validation data for
|
|
536
|
+
development decisions, and evaluate the frozen winner on the untouched test subset.
|
|
537
|
+
TunaRAG does not automatically enforce this separation or run the final held-out evaluation;
|
|
538
|
+
calling `optimize()` with the test set would use it for parameter selection and leak test data.
|
|
539
|
+
|
|
465
540
|
For custom examples, use a user-managed version:
|
|
466
541
|
|
|
467
542
|
```python
|
|
@@ -528,6 +603,8 @@ using them as gold data. Generation usage is separate from optimization usage.
|
|
|
528
603
|
|
|
529
604
|
## Search spaces and strategies
|
|
530
605
|
|
|
606
|
+

|
|
607
|
+
|
|
531
608
|
### Typed space
|
|
532
609
|
|
|
533
610
|
```python
|
|
@@ -666,6 +743,11 @@ configuration remain owned by your RAGAS components.
|
|
|
666
743
|
|
|
667
744
|
## Objectives
|
|
668
745
|
|
|
746
|
+

|
|
747
|
+
|
|
748
|
+
Your objective weights define the tradeoff; the winning configuration is best among the
|
|
749
|
+
evaluated trials under that objective, not a guarantee of the globally best RAG.
|
|
750
|
+
|
|
669
751
|
```python
|
|
670
752
|
from tunarag import Objective, ObjectiveDirection, ObjectiveTerm
|
|
671
753
|
|
|
@@ -903,6 +985,8 @@ labels are metadata, not encryption.
|
|
|
903
985
|
|
|
904
986
|
## SQLite persistence and resume
|
|
905
987
|
|
|
988
|
+

|
|
989
|
+
|
|
906
990
|
```python
|
|
907
991
|
from pathlib import Path
|
|
908
992
|
|
|
@@ -960,18 +1044,7 @@ Events cover study, trial, and attempt lifecycles plus callback failures. Handle
|
|
|
960
1044
|
types defensively. Callback failures never roll back committed work; TunaRAG records them as durable
|
|
961
1045
|
events. Make callbacks idempotent.
|
|
962
1046
|
|
|
963
|
-
|
|
964
|
-
sequenceDiagram
|
|
965
|
-
participant O as Optimizer
|
|
966
|
-
participant S as SQLite
|
|
967
|
-
participant C as Callback
|
|
968
|
-
O->>S: Commit terminal trial and event
|
|
969
|
-
S-->>O: Committed
|
|
970
|
-
O->>C: on_event(event)
|
|
971
|
-
alt callback fails
|
|
972
|
-
O->>S: Append callback.failed
|
|
973
|
-
end
|
|
974
|
-
```
|
|
1047
|
+

|
|
975
1048
|
|
|
976
1049
|
MLflow is an optional post-commit projection; SQLite remains authoritative:
|
|
977
1050
|
|
|
@@ -1082,7 +1155,7 @@ Protocols are structural; inheritance is not required.
|
|
|
1082
1155
|
|
|
1083
1156
|
| Contract | Required behavior |
|
|
1084
1157
|
|---|---|
|
|
1085
|
-
|
|
|
1158
|
+
| Official framework adapters | `LangChainAdapter`, `LangGraphAdapter`, `GoogleADKAdapter` |
|
|
1086
1159
|
| `Dataset` | `.version` and async iteration |
|
|
1087
1160
|
| `Evaluator` | `async evaluate(output, example) -> metrics` |
|
|
1088
1161
|
| `SearchStrategy` | `async suggest()` and `async observe(...)` |
|
|
@@ -1126,7 +1199,7 @@ Provider descriptions and component fingerprints must not expose secrets.
|
|
|
1126
1199
|
|
|
1127
1200
|
- Use a file-backed `SQLiteStore` and back it up.
|
|
1128
1201
|
- Keep a human-reviewed test split outside optimization.
|
|
1129
|
-
-
|
|
1202
|
+
- Supply framework pipeline revisions through adapter `identity` and fingerprint custom evaluators.
|
|
1130
1203
|
- Include a hard stopping bound and conservative concurrency.
|
|
1131
1204
|
- Track exact usage when available and label estimates.
|
|
1132
1205
|
- Configure sample and trial timeouts.
|
|
@@ -1142,7 +1215,7 @@ Provider descriptions and component fingerprints must not expose secrets.
|
|
|
1142
1215
|
V0.2 does not provide a hosted service/dashboard, automatic RAG construction, vector-database
|
|
1143
1216
|
provisioning, distributed workers, Pareto optimization, Bayesian/evolutionary search, encrypted
|
|
1144
1217
|
payloads, MLflow reconciliation, provider pricing management, or a TypeScript implementation.
|
|
1145
|
-
LangChain and
|
|
1218
|
+
LangChain, LangGraph, and Google ADK 2.x are official integrations. Additional frameworks require a future first-party integration; custom RAG adapters are unsupported.
|
|
1146
1219
|
|
|
1147
1220
|
| Term | Meaning |
|
|
1148
1221
|
|---|---|
|
|
@@ -1159,6 +1232,24 @@ LangChain and LangGraph are official integrations; other frameworks use `RAGAdap
|
|
|
1159
1232
|
| Trial | One candidate evaluated over the whole dataset |
|
|
1160
1233
|
| Usage | Tokens, cost, latency, confidence, and pricing provenance |
|
|
1161
1234
|
|
|
1162
|
-
For deeper contracts and architecture, see [
|
|
1163
|
-
[
|
|
1164
|
-
[
|
|
1235
|
+
For deeper contracts and architecture, see the [API specification](https://shivamshinde123.github.io/tunarag-python/api-specification/),
|
|
1236
|
+
[system architecture](https://shivamshinde123.github.io/tunarag-python/system-architecture-document/), and
|
|
1237
|
+
[technical design](https://shivamshinde123.github.io/tunarag-python/technical-design-document/).
|
|
1238
|
+
|
|
1239
|
+
## License and responsibility
|
|
1240
|
+
|
|
1241
|
+
TunaRAG is distributed under the [MIT License](https://opensource.org/license/mit).
|
|
1242
|
+
The full text and copyright notice are in the repository and distribution `LICENSE` file.
|
|
1243
|
+
It permits use, modification,
|
|
1244
|
+
and redistribution subject to retaining the copyright and permission notice. The software
|
|
1245
|
+
is provided **as is, without warranty**, and the license includes a limitation of authors'
|
|
1246
|
+
and copyright holders' liability. The complete license text, not this summary, governs.
|
|
1247
|
+
|
|
1248
|
+
You are responsible for testing fitness for your use case, validating generated answers,
|
|
1249
|
+
protecting data and credentials, setting provider-side spending limits, and monitoring costs.
|
|
1250
|
+
TunaRAG does not guarantee accuracy, security, optimization improvements, or provider bills.
|
|
1251
|
+
See the [Security Policy](https://github.com/shivamshinde123/tunarag-python/blob/main/SECURITY.md)
|
|
1252
|
+
for boundaries and known responsibilities.
|
|
1253
|
+
|
|
1254
|
+
No license can guarantee protection from every claim or override applicable mandatory law.
|
|
1255
|
+
Seek qualified legal advice for liability-sensitive use or distribution.
|
|
@@ -1,24 +1,26 @@
|
|
|
1
|
-
tunarag/__init__.py,sha256=
|
|
1
|
+
tunarag/__init__.py,sha256=1ugGAQZJbEWCvjzENV0mizMhgQKpAIIvfGt_gpc3ojk,3695
|
|
2
2
|
tunarag/cache.py,sha256=prra0uAjW1JYVez6nA9xXFnRwANODHo1rj6o1EiGLI8,11556
|
|
3
3
|
tunarag/config.py,sha256=EzoRWhViL4OQRMFadhiNwEwvne64p2XYMgigaV4tDqg,9325
|
|
4
|
-
tunarag/contracts.py,sha256=
|
|
4
|
+
tunarag/contracts.py,sha256=H3rpzKJjN9F34t6ukJj4q7iBrOjFa9PzOwLk-s8A_Pk,2073
|
|
5
5
|
tunarag/dataset.py,sha256=IPFepM9TOOMREZRUOr4XJdQCrSnXrA8PHVpmKiDa-8I,18111
|
|
6
6
|
tunarag/domain.py,sha256=7bGGb_lHiWQQEMF2bohwMJvrCvslhj-9h5CAKtLd2-A,2663
|
|
7
|
-
tunarag/engine.py,sha256=
|
|
7
|
+
tunarag/engine.py,sha256=ceotkoHaL0OX5MAuHHGpuFg92tVDeMx9C0stQA-aR2M,40931
|
|
8
8
|
tunarag/errors.py,sha256=cB6gZ6txDRPdKcwXmWvEL_7mUcooQdUIrW5qyxBQM9Q,5196
|
|
9
9
|
tunarag/evaluators.py,sha256=BRhreWBEUeu2WU2TTwHqZnhopGA76Ou16T83X9zxuhw,7306
|
|
10
10
|
tunarag/objective.py,sha256=VZ2M_km5bVbsMbzow7LcMl88dO6b_wB5-JIlk3pnMC8,2785
|
|
11
11
|
tunarag/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
12
12
|
tunarag/result.py,sha256=K-zI5Zr1uG_Q295YgE0buWSrWMeEMnQEAaVp5QUYnws,10468
|
|
13
13
|
tunarag/retry.py,sha256=buxAMa9AuYCFvg2GxhHDQbFco3UExVdyBYuLHQaj_ck,2682
|
|
14
|
-
tunarag/search.py,sha256=
|
|
14
|
+
tunarag/search.py,sha256=E0co3arc__ExCsB5q0NYPtiODjGiKdBgIPqju4fJpIs,10275
|
|
15
15
|
tunarag/serialization.py,sha256=c4KyCVc6z47OzTqy96FVeG1Jfkvcxxy0sdZ9X-I8-M0,2870
|
|
16
16
|
tunarag/stopping.py,sha256=0MkFD4rHFbwq_I8qq7bv770Aii5JLvYDuqXIXL8dKB4,6899
|
|
17
17
|
tunarag/store.py,sha256=_d2z315K-FwZt8obmdLmfS5O8zpHEEXyWDzwZrnVw_c,36281
|
|
18
18
|
tunarag/synthetic.py,sha256=TOSpOTTSEMvbtWcK0R1we5dXkJNxEGBuzkU6C5djaM0,9860
|
|
19
|
-
tunarag/integrations/__init__.py,sha256=
|
|
19
|
+
tunarag/integrations/__init__.py,sha256=7eE5ScMAhleHTFwqglB5ZXR4oyjblovIIosIYYcO7CY,550
|
|
20
|
+
tunarag/integrations/google_adk.py,sha256=P_fL0g60Ok7dirwQlGxBq6BBMxNrP8OSEroSk9ciB8Q,5942
|
|
20
21
|
tunarag/integrations/mlflow.py,sha256=Ig96h8CItgEnm-OX41Sp_MQvW9cbQureneZR2xCdikM,5807
|
|
21
|
-
tunarag/integrations/runnables.py,sha256=
|
|
22
|
-
tunarag_python-0.
|
|
23
|
-
tunarag_python-0.
|
|
24
|
-
tunarag_python-0.
|
|
22
|
+
tunarag/integrations/runnables.py,sha256=B9RLnrpn8nZnYnOqkbDQmMmul2SfxOG8oOE-fiPv948,10268
|
|
23
|
+
tunarag_python-0.3.1.dist-info/METADATA,sha256=oC8AqMNj6aJWy1hKa5kss90R4q6EYW7V93TlBYnxI3U,46536
|
|
24
|
+
tunarag_python-0.3.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
25
|
+
tunarag_python-0.3.1.dist-info/licenses/LICENSE,sha256=ve5VOCp_tHEW3KWjdX4JuIMa0TKdOyAXyaSIS9okbUA,1070
|
|
26
|
+
tunarag_python-0.3.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shivam Shinde
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
File without changes
|