tunarag-python 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tunarag/__init__.py +1 -1
- tunarag/dataset.py +19 -3
- tunarag/domain.py +5 -2
- tunarag/engine.py +134 -151
- tunarag/evaluators.py +18 -9
- tunarag/integrations/google_adk.py +114 -25
- tunarag/integrations/mlflow.py +24 -7
- tunarag/integrations/runnables.py +162 -31
- tunarag/objective.py +12 -2
- tunarag/result.py +23 -7
- tunarag/retry.py +50 -5
- tunarag/search.py +3 -1
- tunarag/serialization.py +100 -1
- tunarag/stopping.py +9 -2
- tunarag/store.py +3 -2
- tunarag/synthetic.py +36 -3
- {tunarag_python-0.3.0.dist-info → tunarag_python-0.4.0.dist-info}/METADATA +153 -88
- tunarag_python-0.4.0.dist-info/RECORD +26 -0
- {tunarag_python-0.3.0.dist-info → tunarag_python-0.4.0.dist-info}/WHEEL +1 -1
- tunarag_python-0.4.0.dist-info/licenses/LICENSE +21 -0
- tunarag_python-0.3.0.dist-info/RECORD +0 -25
tunarag/__init__.py
CHANGED
tunarag/dataset.py
CHANGED
|
@@ -136,12 +136,16 @@ class EvaluationDataset:
|
|
|
136
136
|
ids = [example.id for example in self.examples]
|
|
137
137
|
if len(ids) != len(set(ids)):
|
|
138
138
|
raise ValueError("dataset example ids must be unique")
|
|
139
|
+
# Order-independent: samples are aggregated, so reordering rows is the same dataset.
|
|
139
140
|
version = content_hash(
|
|
140
141
|
{
|
|
141
142
|
"schema_version": _SCHEMA_VERSION,
|
|
142
|
-
"examples": [
|
|
143
|
+
"examples": [
|
|
144
|
+
example.semantic_data()
|
|
145
|
+
for example in sorted(self.examples, key=lambda example: example.id)
|
|
146
|
+
],
|
|
143
147
|
},
|
|
144
|
-
namespace="tunarag:dataset:
|
|
148
|
+
namespace="tunarag:dataset:v2",
|
|
145
149
|
)
|
|
146
150
|
object.__setattr__(self, "version", version)
|
|
147
151
|
|
|
@@ -157,6 +161,7 @@ class EvaluationDataset:
|
|
|
157
161
|
|
|
158
162
|
mapping = field_map or DatasetFieldMap()
|
|
159
163
|
examples: list[EvaluationExample] = []
|
|
164
|
+
first_record_for_id: dict[str, int] = {}
|
|
160
165
|
for index, record in enumerate(records, start=1):
|
|
161
166
|
if not isinstance(record, Mapping):
|
|
162
167
|
raise DatasetError(
|
|
@@ -164,7 +169,7 @@ class EvaluationDataset:
|
|
|
164
169
|
details={"source": source, "record": index},
|
|
165
170
|
)
|
|
166
171
|
try:
|
|
167
|
-
|
|
172
|
+
example = _example_from_record(record, mapping)
|
|
168
173
|
except (TypeError, ValueError, json.JSONDecodeError) as error:
|
|
169
174
|
raise DatasetError(
|
|
170
175
|
"dataset record is invalid",
|
|
@@ -174,6 +179,17 @@ class EvaluationDataset:
|
|
|
174
179
|
"cause": type(error).__name__,
|
|
175
180
|
},
|
|
176
181
|
) from error
|
|
182
|
+
if example.id in first_record_for_id:
|
|
183
|
+
raise DatasetError(
|
|
184
|
+
"duplicate example: give each record a unique id (records without an id "
|
|
185
|
+
"get one derived from their content, so identical rows collide)",
|
|
186
|
+
details={
|
|
187
|
+
"source": source,
|
|
188
|
+
"records": [first_record_for_id[example.id], index],
|
|
189
|
+
},
|
|
190
|
+
)
|
|
191
|
+
first_record_for_id[example.id] = index
|
|
192
|
+
examples.append(example)
|
|
177
193
|
try:
|
|
178
194
|
return cls(tuple(examples))
|
|
179
195
|
except ValueError as error:
|
tunarag/domain.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Immutable domain values shared by the package contracts."""
|
|
2
2
|
|
|
3
|
+
import math
|
|
3
4
|
from collections.abc import Mapping
|
|
4
5
|
from dataclasses import dataclass
|
|
5
6
|
from enum import Enum
|
|
@@ -43,6 +44,8 @@ class MetricValue:
|
|
|
43
44
|
raise ValueError("metric coverage must be between 0 and 1")
|
|
44
45
|
if self.value is not None and not isinstance(self.value, (int, float)):
|
|
45
46
|
raise TypeError("metric value must be numeric or None")
|
|
47
|
+
if self.value is not None and not math.isfinite(self.value):
|
|
48
|
+
raise ValueError("metric value must be finite; use None for an unavailable value")
|
|
46
49
|
|
|
47
50
|
|
|
48
51
|
@dataclass(frozen=True, slots=True)
|
|
@@ -66,8 +69,8 @@ class UsageRecord:
|
|
|
66
69
|
raise ValueError(f"{name} must not be negative")
|
|
67
70
|
for name in ("cost", "latency_seconds"):
|
|
68
71
|
usage_value: Any = getattr(self, name)
|
|
69
|
-
if usage_value is not None and usage_value < 0:
|
|
70
|
-
raise ValueError(f"{name} must not
|
|
72
|
+
if usage_value is not None and (not math.isfinite(usage_value) or usage_value < 0):
|
|
73
|
+
raise ValueError(f"{name} must be finite and not negative")
|
|
71
74
|
|
|
72
75
|
|
|
73
76
|
@dataclass(frozen=True, slots=True)
|
tunarag/engine.py
CHANGED
|
@@ -2,11 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import json
|
|
5
|
+
import logging
|
|
6
|
+
import math
|
|
5
7
|
import sqlite3
|
|
6
8
|
import time
|
|
7
9
|
from collections import defaultdict
|
|
8
10
|
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
9
11
|
from dataclasses import dataclass
|
|
12
|
+
from datetime import datetime
|
|
10
13
|
from statistics import fmean
|
|
11
14
|
from typing import Any, TypeVar, cast
|
|
12
15
|
|
|
@@ -22,16 +25,25 @@ from .contracts import (
|
|
|
22
25
|
SearchStrategy,
|
|
23
26
|
)
|
|
24
27
|
from .domain import Candidate, MetricValue, UsageRecord, ValueStatus
|
|
25
|
-
from .errors import ResumeError, SearchSpaceError, SyncInAsyncContextError
|
|
28
|
+
from .errors import ResumeError, SearchSpaceError, SyncInAsyncContextError, TunaRAGError
|
|
26
29
|
from .integrations.google_adk import GoogleADKAdapter
|
|
27
30
|
from .integrations.runnables import LangChainAdapter, LangGraphAdapter
|
|
28
31
|
from .objective import Objective, ObjectiveResult
|
|
29
|
-
from .result import OptimizationResult, TrialSummary,
|
|
32
|
+
from .result import OptimizationResult, TrialSummary, summarize_usage
|
|
30
33
|
from .retry import RetryPolicy
|
|
31
34
|
from .serialization import content_hash
|
|
32
35
|
from .stopping import MaxTrials, StopCondition, StopContext
|
|
33
|
-
from .store import
|
|
36
|
+
from .store import (
|
|
37
|
+
AttemptStatus,
|
|
38
|
+
SQLiteStore,
|
|
39
|
+
StudyEvent,
|
|
40
|
+
StudyRecord,
|
|
41
|
+
StudyStatus,
|
|
42
|
+
TrialRecord,
|
|
43
|
+
TrialStatus,
|
|
44
|
+
)
|
|
34
45
|
|
|
46
|
+
_LOGGER = logging.getLogger("tunarag")
|
|
35
47
|
_OBJECTIVE_METRIC = "tunarag.objective"
|
|
36
48
|
_CACHE_PAYLOAD_VERSION = 1
|
|
37
49
|
_T = TypeVar("_T")
|
|
@@ -41,16 +53,9 @@ _T = TypeVar("_T")
|
|
|
41
53
|
class _TrialFailure(Exception):
|
|
42
54
|
stage: str
|
|
43
55
|
cause_type: str
|
|
44
|
-
usage: tuple[UsageRecord, ...]
|
|
45
56
|
retryable: bool = False
|
|
46
57
|
|
|
47
58
|
|
|
48
|
-
@dataclass(frozen=True, slots=True)
|
|
49
|
-
class _SampleResult:
|
|
50
|
-
metrics: tuple[MetricValue, ...]
|
|
51
|
-
usage: tuple[UsageRecord, ...]
|
|
52
|
-
|
|
53
|
-
|
|
54
59
|
class Optimizer:
|
|
55
60
|
"""Tune pipelines through official LangChain, LangGraph, or Google ADK adapters."""
|
|
56
61
|
|
|
@@ -143,13 +148,6 @@ class Optimizer:
|
|
|
143
148
|
stage="resume.validate",
|
|
144
149
|
details={"study_id": study_id},
|
|
145
150
|
)
|
|
146
|
-
if study.status in {StudyStatus.FAILED, StudyStatus.CANCELLED}:
|
|
147
|
-
raise ResumeError(
|
|
148
|
-
"failed or cancelled studies are terminal and cannot resume",
|
|
149
|
-
stage="resume.validate",
|
|
150
|
-
details={"study_id": study_id, "status": study.status.value},
|
|
151
|
-
)
|
|
152
|
-
|
|
153
151
|
self._event_cursors[study.id] = self._latest_event_sequence(study.id)
|
|
154
152
|
self._event_locks[study.id] = asyncio.Lock()
|
|
155
153
|
summaries = self._load_terminal_summaries(study.id)
|
|
@@ -158,7 +156,10 @@ class Optimizer:
|
|
|
158
156
|
return _build_result(
|
|
159
157
|
study.id, study.status, self._persisted_stop_reason(study.id), summaries
|
|
160
158
|
)
|
|
161
|
-
|
|
159
|
+
prior_elapsed = _active_seconds(self._store.list_events(study.id))
|
|
160
|
+
previous_status = study.status
|
|
161
|
+
if study.status is not StudyStatus.RUNNING:
|
|
162
|
+
# Created, interrupted (Ctrl-C/cancel) and crashed studies all continue from SQLite.
|
|
162
163
|
study = self._store.transition_study(study.id, StudyStatus.RUNNING)
|
|
163
164
|
|
|
164
165
|
abandoned = self._store.abandon_expired_attempts(study_id=study.id)
|
|
@@ -189,10 +190,13 @@ class Optimizer:
|
|
|
189
190
|
"terminal_trials": len(summaries),
|
|
190
191
|
"unfinished_trials": len(unfinished),
|
|
191
192
|
"abandoned_attempts": len(abandoned),
|
|
193
|
+
"previous_status": previous_status.value,
|
|
192
194
|
},
|
|
193
195
|
)
|
|
194
196
|
await self._dispatch_events(study.id)
|
|
195
|
-
return await self._continue_study(
|
|
197
|
+
return await self._continue_study(
|
|
198
|
+
dataset, study, summaries, unfinished, prior_elapsed=prior_elapsed
|
|
199
|
+
)
|
|
196
200
|
|
|
197
201
|
async def _continue_study(
|
|
198
202
|
self,
|
|
@@ -200,8 +204,11 @@ class Optimizer:
|
|
|
200
204
|
study: StudyRecord,
|
|
201
205
|
summaries: list[TrialSummary],
|
|
202
206
|
unfinished: Sequence[TrialRecord],
|
|
207
|
+
*,
|
|
208
|
+
prior_elapsed: float = 0.0,
|
|
203
209
|
) -> OptimizationResult:
|
|
204
|
-
|
|
210
|
+
# TimeBudget counts active run time across resumes, not downtime between them.
|
|
211
|
+
started_at = self._clock() - prior_elapsed
|
|
205
212
|
pending = list(unfinished)
|
|
206
213
|
stop_reason = "completed"
|
|
207
214
|
search_exhausted = False
|
|
@@ -436,16 +443,20 @@ class Optimizer:
|
|
|
436
443
|
trial.id, lease_seconds=self._retry_policy.lease_seconds
|
|
437
444
|
)
|
|
438
445
|
await self._dispatch_events(trial.study_id)
|
|
446
|
+
# Usage is appended as each sample finishes, so money spent before a trial
|
|
447
|
+
# timeout, failure, or cancellation is still recorded.
|
|
448
|
+
spent: list[UsageRecord] = []
|
|
449
|
+
committed = False
|
|
439
450
|
try:
|
|
440
451
|
if cached_metrics is None:
|
|
441
|
-
metrics
|
|
442
|
-
self._execute_candidate(dataset, candidate),
|
|
452
|
+
metrics = await _wait_for(
|
|
453
|
+
self._execute_candidate(dataset, candidate, spent),
|
|
443
454
|
self._retry_policy.trial_timeout_seconds,
|
|
444
455
|
)
|
|
445
456
|
else:
|
|
446
|
-
metrics
|
|
447
|
-
|
|
448
|
-
|
|
457
|
+
metrics = cached_metrics
|
|
458
|
+
usage = tuple(spent)
|
|
459
|
+
objective = self._evaluate_objective(metrics)
|
|
449
460
|
objective_metric = MetricValue(
|
|
450
461
|
_OBJECTIVE_METRIC, cast(float, objective.score), objective.status
|
|
451
462
|
)
|
|
@@ -456,6 +467,8 @@ class Optimizer:
|
|
|
456
467
|
metrics=(*metrics, objective_metric),
|
|
457
468
|
usage=usage,
|
|
458
469
|
)
|
|
470
|
+
committed = True
|
|
471
|
+
accumulated_usage.extend(usage)
|
|
459
472
|
cached = cached_metrics is not None
|
|
460
473
|
if cached:
|
|
461
474
|
self._store.append_event(
|
|
@@ -476,72 +489,37 @@ class Optimizer:
|
|
|
476
489
|
cached=cached,
|
|
477
490
|
)
|
|
478
491
|
except asyncio.CancelledError:
|
|
479
|
-
|
|
480
|
-
trial.
|
|
481
|
-
status=TrialStatus.CANCELLED,
|
|
482
|
-
attempt_id=attempt.id,
|
|
483
|
-
error="trial cancelled",
|
|
484
|
-
)
|
|
485
|
-
raise
|
|
486
|
-
except _TrialFailure as failure:
|
|
487
|
-
accumulated_usage.extend(failure.usage)
|
|
488
|
-
failure_message = f"{failure.stage} failed: {failure.cause_type}"
|
|
489
|
-
if failure.retryable and attempt.number < self._retry_policy.max_retries:
|
|
492
|
+
if not committed:
|
|
493
|
+
# Leave the trial running without an active attempt: resume re-runs it.
|
|
490
494
|
self._store.finish_attempt(
|
|
491
495
|
attempt.id,
|
|
492
|
-
AttemptStatus.
|
|
493
|
-
usage=
|
|
494
|
-
error=
|
|
496
|
+
AttemptStatus.CANCELLED,
|
|
497
|
+
usage=tuple(spent),
|
|
498
|
+
error="attempt cancelled",
|
|
495
499
|
)
|
|
496
|
-
|
|
497
|
-
self._store.append_event(
|
|
498
|
-
trial.study_id,
|
|
499
|
-
"trial.retry_scheduled",
|
|
500
|
-
{
|
|
501
|
-
"trial_id": trial.id,
|
|
502
|
-
"attempt": attempt.number,
|
|
503
|
-
"delay_seconds": delay,
|
|
504
|
-
},
|
|
505
|
-
)
|
|
506
|
-
await self._dispatch_events(trial.study_id)
|
|
507
|
-
try:
|
|
508
|
-
await self._sleeper(delay)
|
|
509
|
-
except asyncio.CancelledError:
|
|
510
|
-
self._store.transition_trial(trial.id, TrialStatus.CANCELLED)
|
|
511
|
-
raise
|
|
512
|
-
cached_metrics = None
|
|
513
|
-
continue
|
|
514
|
-
self._store.commit_trial_result(
|
|
515
|
-
trial.id,
|
|
516
|
-
status=TrialStatus.FAILED,
|
|
517
|
-
attempt_id=attempt.id,
|
|
518
|
-
usage=failure.usage,
|
|
519
|
-
error=failure_message,
|
|
520
|
-
)
|
|
521
|
-
await self._dispatch_events(trial.study_id)
|
|
522
|
-
return TrialSummary(
|
|
523
|
-
id=trial.id,
|
|
524
|
-
sequence=trial.sequence,
|
|
525
|
-
candidate=candidate,
|
|
526
|
-
status=TrialStatus.FAILED,
|
|
527
|
-
metrics=(),
|
|
528
|
-
usage=tuple(accumulated_usage),
|
|
529
|
-
objective=None,
|
|
530
|
-
attempts=attempt.number + 1,
|
|
531
|
-
cached=cached_metrics is not None,
|
|
532
|
-
error=failure_message,
|
|
533
|
-
)
|
|
500
|
+
raise
|
|
534
501
|
except Exception as error:
|
|
535
|
-
|
|
536
|
-
|
|
502
|
+
failure = (
|
|
503
|
+
error
|
|
504
|
+
if isinstance(error, _TrialFailure)
|
|
505
|
+
else _TrialFailure(
|
|
506
|
+
"trial", _describe(error), self._retry_policy.is_retryable(error)
|
|
507
|
+
)
|
|
537
508
|
)
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
509
|
+
usage = tuple(spent)
|
|
510
|
+
accumulated_usage.extend(usage)
|
|
511
|
+
failure_message = f"{failure.stage} failed: {failure.cause_type}"
|
|
512
|
+
# Persisted errors stay redacted; the full cause goes to the console log only.
|
|
513
|
+
_LOGGER.warning(
|
|
514
|
+
"trial %d attempt %d %s (cause: %s)",
|
|
515
|
+
trial.sequence,
|
|
516
|
+
attempt.number,
|
|
517
|
+
failure_message,
|
|
518
|
+
_cause_text(error),
|
|
541
519
|
)
|
|
542
|
-
if
|
|
520
|
+
if failure.retryable and attempt.number < self._retry_policy.max_retries:
|
|
543
521
|
self._store.finish_attempt(
|
|
544
|
-
attempt.id, AttemptStatus.FAILED, error=failure_message
|
|
522
|
+
attempt.id, AttemptStatus.FAILED, usage=usage, error=failure_message
|
|
545
523
|
)
|
|
546
524
|
delay = self._retry_policy.delay_before_retry(attempt.number)
|
|
547
525
|
self._store.append_event(
|
|
@@ -554,17 +532,15 @@ class Optimizer:
|
|
|
554
532
|
},
|
|
555
533
|
)
|
|
556
534
|
await self._dispatch_events(trial.study_id)
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
except asyncio.CancelledError:
|
|
560
|
-
self._store.transition_trial(trial.id, TrialStatus.CANCELLED)
|
|
561
|
-
raise
|
|
535
|
+
# A cancellation here leaves the trial recoverable for resume.
|
|
536
|
+
await self._sleeper(delay)
|
|
562
537
|
cached_metrics = None
|
|
563
538
|
continue
|
|
564
539
|
self._store.commit_trial_result(
|
|
565
540
|
trial.id,
|
|
566
541
|
status=TrialStatus.FAILED,
|
|
567
542
|
attempt_id=attempt.id,
|
|
543
|
+
usage=usage,
|
|
568
544
|
error=failure_message,
|
|
569
545
|
)
|
|
570
546
|
await self._dispatch_events(trial.study_id)
|
|
@@ -577,32 +553,28 @@ class Optimizer:
|
|
|
577
553
|
usage=tuple(accumulated_usage),
|
|
578
554
|
objective=None,
|
|
579
555
|
attempts=attempt.number + 1,
|
|
580
|
-
cached=
|
|
556
|
+
cached=cached_metrics is not None,
|
|
581
557
|
error=failure_message,
|
|
582
558
|
)
|
|
583
559
|
|
|
584
560
|
async def _execute_candidate(
|
|
585
|
-
self, dataset: Dataset, candidate: Candidate
|
|
586
|
-
) -> tuple[
|
|
561
|
+
self, dataset: Dataset, candidate: Candidate, spent: list[UsageRecord]
|
|
562
|
+
) -> tuple[MetricValue, ...]:
|
|
587
563
|
observations: dict[str, list[MetricValue]] = defaultdict(list)
|
|
588
|
-
usage: list[UsageRecord] = []
|
|
589
564
|
try:
|
|
590
565
|
examples = [example async for example in dataset]
|
|
591
566
|
except Exception as error:
|
|
592
567
|
raise _TrialFailure(
|
|
593
|
-
"dataset",
|
|
594
|
-
type(error).__name__,
|
|
595
|
-
(),
|
|
596
|
-
self._retry_policy.is_retryable(error),
|
|
568
|
+
"dataset", _describe(error), self._retry_policy.is_retryable(error)
|
|
597
569
|
) from error
|
|
598
570
|
if not examples:
|
|
599
|
-
raise _TrialFailure("dataset", "empty"
|
|
571
|
+
raise _TrialFailure("dataset", "empty")
|
|
600
572
|
|
|
601
573
|
semaphore = asyncio.Semaphore(self._settings.sample_concurrency)
|
|
602
574
|
|
|
603
|
-
async def bounded(example: Any) ->
|
|
575
|
+
async def bounded(example: Any) -> tuple[MetricValue, ...]:
|
|
604
576
|
async with semaphore:
|
|
605
|
-
return await self._execute_sample(candidate, example)
|
|
577
|
+
return await self._execute_sample(candidate, example, spent)
|
|
606
578
|
|
|
607
579
|
outcomes = await asyncio.gather(
|
|
608
580
|
*(bounded(example) for example in examples), return_exceptions=True
|
|
@@ -610,28 +582,21 @@ class Optimizer:
|
|
|
610
582
|
first_failure: _TrialFailure | None = None
|
|
611
583
|
for outcome in outcomes:
|
|
612
584
|
if isinstance(outcome, _TrialFailure):
|
|
613
|
-
|
|
614
|
-
if first_failure is None:
|
|
615
|
-
first_failure = outcome
|
|
585
|
+
first_failure = first_failure or outcome
|
|
616
586
|
continue
|
|
617
587
|
if isinstance(outcome, BaseException):
|
|
618
588
|
raise outcome
|
|
619
|
-
|
|
620
|
-
for metric in outcome.metrics:
|
|
589
|
+
for metric in outcome:
|
|
621
590
|
observations[metric.name].append(metric)
|
|
622
591
|
if first_failure is not None:
|
|
623
|
-
raise
|
|
624
|
-
first_failure.stage,
|
|
625
|
-
first_failure.cause_type,
|
|
626
|
-
tuple(usage),
|
|
627
|
-
first_failure.retryable,
|
|
628
|
-
) from first_failure
|
|
592
|
+
raise first_failure
|
|
629
593
|
if not observations:
|
|
630
|
-
raise _TrialFailure("evaluation", "no_metrics"
|
|
631
|
-
return _aggregate_metrics(observations, len(examples))
|
|
594
|
+
raise _TrialFailure("evaluation", "no_metrics")
|
|
595
|
+
return _aggregate_metrics(observations, len(examples))
|
|
632
596
|
|
|
633
|
-
async def _execute_sample(
|
|
634
|
-
|
|
597
|
+
async def _execute_sample(
|
|
598
|
+
self, candidate: Candidate, example: Any, spent: list[UsageRecord]
|
|
599
|
+
) -> tuple[MetricValue, ...]:
|
|
635
600
|
stage = "adapter"
|
|
636
601
|
try:
|
|
637
602
|
output, sample_usage = await _wait_for(
|
|
@@ -641,6 +606,7 @@ class Optimizer:
|
|
|
641
606
|
usage = tuple(sample_usage)
|
|
642
607
|
if any(not isinstance(item, UsageRecord) for item in usage):
|
|
643
608
|
raise TypeError("adapter usage must contain UsageRecord values")
|
|
609
|
+
spent.extend(usage)
|
|
644
610
|
sample_metrics: list[MetricValue] = []
|
|
645
611
|
sample_names: set[str] = set()
|
|
646
612
|
for evaluator in self._evaluators:
|
|
@@ -660,29 +626,23 @@ class Optimizer:
|
|
|
660
626
|
raise ValueError(f"duplicate metric for sample: {metric.name}")
|
|
661
627
|
sample_names.add(metric.name)
|
|
662
628
|
sample_metrics.append(metric)
|
|
663
|
-
return
|
|
629
|
+
return tuple(sample_metrics)
|
|
664
630
|
except Exception as error:
|
|
665
631
|
raise _TrialFailure(
|
|
666
|
-
stage,
|
|
667
|
-
type(error).__name__,
|
|
668
|
-
usage,
|
|
669
|
-
self._retry_policy.is_retryable(error),
|
|
632
|
+
stage, _describe(error), self._retry_policy.is_retryable(error)
|
|
670
633
|
) from error
|
|
671
634
|
|
|
672
|
-
def _evaluate_objective(
|
|
673
|
-
self, metrics: Sequence[MetricValue], usage: tuple[UsageRecord, ...]
|
|
674
|
-
) -> ObjectiveResult:
|
|
635
|
+
def _evaluate_objective(self, metrics: Sequence[MetricValue]) -> ObjectiveResult:
|
|
675
636
|
try:
|
|
676
637
|
objective = self._objective.evaluate(metrics)
|
|
677
638
|
except Exception as error:
|
|
678
639
|
raise _TrialFailure(
|
|
679
|
-
"objective",
|
|
680
|
-
type(error).__name__,
|
|
681
|
-
usage,
|
|
682
|
-
self._retry_policy.is_retryable(error),
|
|
640
|
+
"objective", _describe(error), self._retry_policy.is_retryable(error)
|
|
683
641
|
) from error
|
|
684
642
|
if objective.score is None:
|
|
685
|
-
raise _TrialFailure("objective", "unavailable"
|
|
643
|
+
raise _TrialFailure("objective", "unavailable")
|
|
644
|
+
if not math.isfinite(objective.score):
|
|
645
|
+
raise _TrialFailure("objective", "non_finite")
|
|
686
646
|
return objective
|
|
687
647
|
|
|
688
648
|
def _read_cache(self, trial: TrialRecord, key: CacheKey) -> tuple[MetricValue, ...] | None:
|
|
@@ -766,6 +726,7 @@ class Optimizer:
|
|
|
766
726
|
"direction": term.direction.value,
|
|
767
727
|
"minimum": term.minimum,
|
|
768
728
|
"maximum": term.maximum,
|
|
729
|
+
"min_coverage": term.min_coverage,
|
|
769
730
|
}
|
|
770
731
|
for term in self._objective.terms
|
|
771
732
|
],
|
|
@@ -913,6 +874,7 @@ def _stop_context(summaries: Sequence[TrialSummary], elapsed_seconds: float) ->
|
|
|
913
874
|
successful_trials=successful,
|
|
914
875
|
failed_trials=failed,
|
|
915
876
|
objective_scores=scores,
|
|
877
|
+
failure_history=tuple(summary.status is TrialStatus.FAILED for summary in summaries),
|
|
916
878
|
)
|
|
917
879
|
|
|
918
880
|
|
|
@@ -932,14 +894,14 @@ def _build_result(
|
|
|
932
894
|
best = (
|
|
933
895
|
max(successful, key=lambda item: _objective_score(item.objective)) if successful else None
|
|
934
896
|
)
|
|
935
|
-
all_usage =
|
|
897
|
+
all_usage = tuple(item for summary in summaries for item in summary.usage)
|
|
936
898
|
return OptimizationResult(
|
|
937
899
|
study_id=study_id,
|
|
938
900
|
study_status=study_status,
|
|
939
901
|
stop_reason=stop_reason,
|
|
940
902
|
trials=tuple(summaries),
|
|
941
903
|
best_trial=best,
|
|
942
|
-
usage=
|
|
904
|
+
usage=summarize_usage(all_usage),
|
|
943
905
|
)
|
|
944
906
|
|
|
945
907
|
|
|
@@ -949,27 +911,48 @@ def _objective_score(result: ObjectiveResult | None) -> float:
|
|
|
949
911
|
return result.score
|
|
950
912
|
|
|
951
913
|
|
|
952
|
-
def
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
if
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
914
|
+
def _describe(error: BaseException) -> str:
|
|
915
|
+
"""Name a failure without leaking third-party messages, which may contain user data.
|
|
916
|
+
|
|
917
|
+
TunaRAG's own error messages are authored diagnostics, so they are kept.
|
|
918
|
+
"""
|
|
919
|
+
|
|
920
|
+
name = type(error).__name__
|
|
921
|
+
if isinstance(error, TunaRAGError):
|
|
922
|
+
return f"{name}: {error.message}"
|
|
923
|
+
return name
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
def _cause_text(error: BaseException) -> str:
|
|
927
|
+
root = error.__cause__ if isinstance(error, _TrialFailure) and error.__cause__ else error
|
|
928
|
+
return f"{type(root).__name__}: {root}"[:500]
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
def _active_seconds(events: Sequence[StudyEvent]) -> float:
|
|
932
|
+
"""Sum the run segments (start/resume to last event) recorded in the event log."""
|
|
933
|
+
|
|
934
|
+
total = 0.0
|
|
935
|
+
segment_start: float | None = None
|
|
936
|
+
last: float | None = None
|
|
937
|
+
for event in events:
|
|
938
|
+
timestamp = _event_time(event)
|
|
939
|
+
if timestamp is None:
|
|
940
|
+
continue
|
|
941
|
+
if event.type in {"study.running", "study.resumed"}:
|
|
942
|
+
if segment_start is not None and last is not None:
|
|
943
|
+
total += max(0.0, last - segment_start)
|
|
944
|
+
segment_start = timestamp
|
|
945
|
+
last = timestamp
|
|
946
|
+
if segment_start is not None and last is not None:
|
|
947
|
+
total += max(0.0, last - segment_start)
|
|
948
|
+
return total
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def _event_time(event: StudyEvent) -> float | None:
|
|
952
|
+
try:
|
|
953
|
+
return datetime.fromisoformat(event.created_at).timestamp()
|
|
954
|
+
except (TypeError, ValueError):
|
|
955
|
+
return None
|
|
973
956
|
|
|
974
957
|
|
|
975
958
|
def _type_name(value: object) -> str:
|
tunarag/evaluators.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import copy
|
|
5
6
|
import inspect
|
|
6
7
|
import math
|
|
7
8
|
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
@@ -12,6 +13,7 @@ from typing import Any
|
|
|
12
13
|
from .contracts import Fingerprintable
|
|
13
14
|
from .domain import MetricValue, ValueStatus
|
|
14
15
|
from .errors import EvaluationError, MissingOptionalDependencyError
|
|
16
|
+
from .serialization import callable_identity, callable_name
|
|
15
17
|
|
|
16
18
|
MetricScorer = Callable[[Any, Any], float | None | Awaitable[float | None]]
|
|
17
19
|
RagasSampleMapper = Callable[[Any, Any], Mapping[str, Any]]
|
|
@@ -26,6 +28,7 @@ class MetricEvaluator:
|
|
|
26
28
|
scorer: MetricScorer,
|
|
27
29
|
*,
|
|
28
30
|
status: ValueStatus = ValueStatus.EXACT,
|
|
31
|
+
identity: Any = None,
|
|
29
32
|
) -> None:
|
|
30
33
|
if not name.strip():
|
|
31
34
|
raise ValueError("metric name must not be empty")
|
|
@@ -36,6 +39,7 @@ class MetricEvaluator:
|
|
|
36
39
|
self._name = name
|
|
37
40
|
self._scorer = scorer
|
|
38
41
|
self._status = status
|
|
42
|
+
self._identity = identity
|
|
39
43
|
|
|
40
44
|
async def evaluate(self, output: Any, example: Any) -> Sequence[MetricValue]:
|
|
41
45
|
"""Score one adapter output and evaluation example."""
|
|
@@ -53,7 +57,14 @@ class MetricEvaluator:
|
|
|
53
57
|
return {
|
|
54
58
|
"name": self._name,
|
|
55
59
|
"status": self._status.value,
|
|
56
|
-
|
|
60
|
+
# Without an explicit identity, a code digest makes different lambdas and edited
|
|
61
|
+
# functions differ; an explicit identity is the declared version instead.
|
|
62
|
+
"scorer": (
|
|
63
|
+
callable_identity(self._scorer)
|
|
64
|
+
if self._identity is None
|
|
65
|
+
else callable_name(self._scorer)
|
|
66
|
+
),
|
|
67
|
+
"identity": self._identity,
|
|
57
68
|
}
|
|
58
69
|
|
|
59
70
|
|
|
@@ -100,9 +111,13 @@ class RagasEvaluator:
|
|
|
100
111
|
try:
|
|
101
112
|
sample = dataset_schema.SingleTurnSample(**dict(sample_data))
|
|
102
113
|
dataset = ragas.EvaluationDataset(samples=[sample])
|
|
114
|
+
# RAGAS attaches the judge LLM/embeddings to metric objects and detaches them when
|
|
115
|
+
# each call finishes. Samples run concurrently, so each call gets its own copies;
|
|
116
|
+
# otherwise one sample's cleanup strips the judge from another mid-evaluation.
|
|
117
|
+
metrics = [copy.copy(metric) for metric in self._metrics]
|
|
103
118
|
result = await ragas.aevaluate(
|
|
104
119
|
dataset=dataset,
|
|
105
|
-
metrics=
|
|
120
|
+
metrics=metrics,
|
|
106
121
|
llm=self._llm,
|
|
107
122
|
embeddings=self._embeddings,
|
|
108
123
|
raise_exceptions=self._raise_exceptions,
|
|
@@ -141,7 +156,7 @@ class RagasEvaluator:
|
|
|
141
156
|
|
|
142
157
|
return {
|
|
143
158
|
"metrics": [_component_identity(metric) for metric in self._metrics],
|
|
144
|
-
"sample_mapper":
|
|
159
|
+
"sample_mapper": callable_identity(self._sample_mapper),
|
|
145
160
|
"metric_prefix": self._metric_prefix,
|
|
146
161
|
"llm": _component_identity(self._llm),
|
|
147
162
|
"embeddings": _component_identity(self._embeddings),
|
|
@@ -169,12 +184,6 @@ def _numeric_score(value: Any, *, metric: str) -> float | None:
|
|
|
169
184
|
return normalized if math.isfinite(normalized) else None
|
|
170
185
|
|
|
171
186
|
|
|
172
|
-
def _callable_identity(function: Callable[..., Any]) -> str:
|
|
173
|
-
module = getattr(function, "__module__", type(function).__module__)
|
|
174
|
-
name = getattr(function, "__qualname__", type(function).__qualname__)
|
|
175
|
-
return f"{module}.{name}"
|
|
176
|
-
|
|
177
|
-
|
|
178
187
|
def _component_identity(component: Any) -> Any:
|
|
179
188
|
if component is None:
|
|
180
189
|
return None
|