tunarag-python 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tunarag/__init__.py CHANGED
@@ -165,4 +165,4 @@ __all__ = [
165
165
  "__version__",
166
166
  ]
167
167
 
168
- __version__ = "0.3.0"
168
+ __version__ = "0.4.0"
tunarag/dataset.py CHANGED
@@ -136,12 +136,16 @@ class EvaluationDataset:
136
136
  ids = [example.id for example in self.examples]
137
137
  if len(ids) != len(set(ids)):
138
138
  raise ValueError("dataset example ids must be unique")
139
+ # Order-independent: samples are aggregated, so reordering rows is the same dataset.
139
140
  version = content_hash(
140
141
  {
141
142
  "schema_version": _SCHEMA_VERSION,
142
- "examples": [example.semantic_data() for example in self.examples],
143
+ "examples": [
144
+ example.semantic_data()
145
+ for example in sorted(self.examples, key=lambda example: example.id)
146
+ ],
143
147
  },
144
- namespace="tunarag:dataset:v1",
148
+ namespace="tunarag:dataset:v2",
145
149
  )
146
150
  object.__setattr__(self, "version", version)
147
151
 
@@ -157,6 +161,7 @@ class EvaluationDataset:
157
161
 
158
162
  mapping = field_map or DatasetFieldMap()
159
163
  examples: list[EvaluationExample] = []
164
+ first_record_for_id: dict[str, int] = {}
160
165
  for index, record in enumerate(records, start=1):
161
166
  if not isinstance(record, Mapping):
162
167
  raise DatasetError(
@@ -164,7 +169,7 @@ class EvaluationDataset:
164
169
  details={"source": source, "record": index},
165
170
  )
166
171
  try:
167
- examples.append(_example_from_record(record, mapping))
172
+ example = _example_from_record(record, mapping)
168
173
  except (TypeError, ValueError, json.JSONDecodeError) as error:
169
174
  raise DatasetError(
170
175
  "dataset record is invalid",
@@ -174,6 +179,17 @@ class EvaluationDataset:
174
179
  "cause": type(error).__name__,
175
180
  },
176
181
  ) from error
182
+ if example.id in first_record_for_id:
183
+ raise DatasetError(
184
+ "duplicate example: give each record a unique id (records without an id "
185
+ "get one derived from their content, so identical rows collide)",
186
+ details={
187
+ "source": source,
188
+ "records": [first_record_for_id[example.id], index],
189
+ },
190
+ )
191
+ first_record_for_id[example.id] = index
192
+ examples.append(example)
177
193
  try:
178
194
  return cls(tuple(examples))
179
195
  except ValueError as error:
tunarag/domain.py CHANGED
@@ -1,5 +1,6 @@
1
1
  """Immutable domain values shared by the package contracts."""
2
2
 
3
+ import math
3
4
  from collections.abc import Mapping
4
5
  from dataclasses import dataclass
5
6
  from enum import Enum
@@ -43,6 +44,8 @@ class MetricValue:
43
44
  raise ValueError("metric coverage must be between 0 and 1")
44
45
  if self.value is not None and not isinstance(self.value, (int, float)):
45
46
  raise TypeError("metric value must be numeric or None")
47
+ if self.value is not None and not math.isfinite(self.value):
48
+ raise ValueError("metric value must be finite; use None for an unavailable value")
46
49
 
47
50
 
48
51
  @dataclass(frozen=True, slots=True)
@@ -66,8 +69,8 @@ class UsageRecord:
66
69
  raise ValueError(f"{name} must not be negative")
67
70
  for name in ("cost", "latency_seconds"):
68
71
  usage_value: Any = getattr(self, name)
69
- if usage_value is not None and usage_value < 0:
70
- raise ValueError(f"{name} must not be negative")
72
+ if usage_value is not None and (not math.isfinite(usage_value) or usage_value < 0):
73
+ raise ValueError(f"{name} must be finite and not negative")
71
74
 
72
75
 
73
76
  @dataclass(frozen=True, slots=True)
tunarag/engine.py CHANGED
@@ -2,11 +2,14 @@
2
2
 
3
3
  import asyncio
4
4
  import json
5
+ import logging
6
+ import math
5
7
  import sqlite3
6
8
  import time
7
9
  from collections import defaultdict
8
10
  from collections.abc import Awaitable, Callable, Mapping, Sequence
9
11
  from dataclasses import dataclass
12
+ from datetime import datetime
10
13
  from statistics import fmean
11
14
  from typing import Any, TypeVar, cast
12
15
 
@@ -22,16 +25,25 @@ from .contracts import (
22
25
  SearchStrategy,
23
26
  )
24
27
  from .domain import Candidate, MetricValue, UsageRecord, ValueStatus
25
- from .errors import ResumeError, SearchSpaceError, SyncInAsyncContextError
28
+ from .errors import ResumeError, SearchSpaceError, SyncInAsyncContextError, TunaRAGError
26
29
  from .integrations.google_adk import GoogleADKAdapter
27
30
  from .integrations.runnables import LangChainAdapter, LangGraphAdapter
28
31
  from .objective import Objective, ObjectiveResult
29
- from .result import OptimizationResult, TrialSummary, UsageSummary
32
+ from .result import OptimizationResult, TrialSummary, summarize_usage
30
33
  from .retry import RetryPolicy
31
34
  from .serialization import content_hash
32
35
  from .stopping import MaxTrials, StopCondition, StopContext
33
- from .store import AttemptStatus, SQLiteStore, StudyRecord, StudyStatus, TrialRecord, TrialStatus
36
+ from .store import (
37
+ AttemptStatus,
38
+ SQLiteStore,
39
+ StudyEvent,
40
+ StudyRecord,
41
+ StudyStatus,
42
+ TrialRecord,
43
+ TrialStatus,
44
+ )
34
45
 
46
+ _LOGGER = logging.getLogger("tunarag")
35
47
  _OBJECTIVE_METRIC = "tunarag.objective"
36
48
  _CACHE_PAYLOAD_VERSION = 1
37
49
  _T = TypeVar("_T")
@@ -41,16 +53,9 @@ _T = TypeVar("_T")
41
53
  class _TrialFailure(Exception):
42
54
  stage: str
43
55
  cause_type: str
44
- usage: tuple[UsageRecord, ...]
45
56
  retryable: bool = False
46
57
 
47
58
 
48
- @dataclass(frozen=True, slots=True)
49
- class _SampleResult:
50
- metrics: tuple[MetricValue, ...]
51
- usage: tuple[UsageRecord, ...]
52
-
53
-
54
59
  class Optimizer:
55
60
  """Tune pipelines through official LangChain, LangGraph, or Google ADK adapters."""
56
61
 
@@ -143,13 +148,6 @@ class Optimizer:
143
148
  stage="resume.validate",
144
149
  details={"study_id": study_id},
145
150
  )
146
- if study.status in {StudyStatus.FAILED, StudyStatus.CANCELLED}:
147
- raise ResumeError(
148
- "failed or cancelled studies are terminal and cannot resume",
149
- stage="resume.validate",
150
- details={"study_id": study_id, "status": study.status.value},
151
- )
152
-
153
151
  self._event_cursors[study.id] = self._latest_event_sequence(study.id)
154
152
  self._event_locks[study.id] = asyncio.Lock()
155
153
  summaries = self._load_terminal_summaries(study.id)
@@ -158,7 +156,10 @@ class Optimizer:
158
156
  return _build_result(
159
157
  study.id, study.status, self._persisted_stop_reason(study.id), summaries
160
158
  )
161
- if study.status is StudyStatus.CREATED:
159
+ prior_elapsed = _active_seconds(self._store.list_events(study.id))
160
+ previous_status = study.status
161
+ if study.status is not StudyStatus.RUNNING:
162
+ # Created, interrupted (Ctrl-C/cancel) and crashed studies all continue from SQLite.
162
163
  study = self._store.transition_study(study.id, StudyStatus.RUNNING)
163
164
 
164
165
  abandoned = self._store.abandon_expired_attempts(study_id=study.id)
@@ -189,10 +190,13 @@ class Optimizer:
189
190
  "terminal_trials": len(summaries),
190
191
  "unfinished_trials": len(unfinished),
191
192
  "abandoned_attempts": len(abandoned),
193
+ "previous_status": previous_status.value,
192
194
  },
193
195
  )
194
196
  await self._dispatch_events(study.id)
195
- return await self._continue_study(dataset, study, summaries, unfinished)
197
+ return await self._continue_study(
198
+ dataset, study, summaries, unfinished, prior_elapsed=prior_elapsed
199
+ )
196
200
 
197
201
  async def _continue_study(
198
202
  self,
@@ -200,8 +204,11 @@ class Optimizer:
200
204
  study: StudyRecord,
201
205
  summaries: list[TrialSummary],
202
206
  unfinished: Sequence[TrialRecord],
207
+ *,
208
+ prior_elapsed: float = 0.0,
203
209
  ) -> OptimizationResult:
204
- started_at = self._clock()
210
+ # TimeBudget counts active run time across resumes, not downtime between them.
211
+ started_at = self._clock() - prior_elapsed
205
212
  pending = list(unfinished)
206
213
  stop_reason = "completed"
207
214
  search_exhausted = False
@@ -436,16 +443,20 @@ class Optimizer:
436
443
  trial.id, lease_seconds=self._retry_policy.lease_seconds
437
444
  )
438
445
  await self._dispatch_events(trial.study_id)
446
+ # Usage is appended as each sample finishes, so money spent before a trial
447
+ # timeout, failure, or cancellation is still recorded.
448
+ spent: list[UsageRecord] = []
449
+ committed = False
439
450
  try:
440
451
  if cached_metrics is None:
441
- metrics, usage = await _wait_for(
442
- self._execute_candidate(dataset, candidate),
452
+ metrics = await _wait_for(
453
+ self._execute_candidate(dataset, candidate, spent),
443
454
  self._retry_policy.trial_timeout_seconds,
444
455
  )
445
456
  else:
446
- metrics, usage = cached_metrics, ()
447
- objective = self._evaluate_objective(metrics, usage)
448
- accumulated_usage.extend(usage)
457
+ metrics = cached_metrics
458
+ usage = tuple(spent)
459
+ objective = self._evaluate_objective(metrics)
449
460
  objective_metric = MetricValue(
450
461
  _OBJECTIVE_METRIC, cast(float, objective.score), objective.status
451
462
  )
@@ -456,6 +467,8 @@ class Optimizer:
456
467
  metrics=(*metrics, objective_metric),
457
468
  usage=usage,
458
469
  )
470
+ committed = True
471
+ accumulated_usage.extend(usage)
459
472
  cached = cached_metrics is not None
460
473
  if cached:
461
474
  self._store.append_event(
@@ -476,72 +489,37 @@ class Optimizer:
476
489
  cached=cached,
477
490
  )
478
491
  except asyncio.CancelledError:
479
- self._store.commit_trial_result(
480
- trial.id,
481
- status=TrialStatus.CANCELLED,
482
- attempt_id=attempt.id,
483
- error="trial cancelled",
484
- )
485
- raise
486
- except _TrialFailure as failure:
487
- accumulated_usage.extend(failure.usage)
488
- failure_message = f"{failure.stage} failed: {failure.cause_type}"
489
- if failure.retryable and attempt.number < self._retry_policy.max_retries:
492
+ if not committed:
493
+ # Leave the trial running without an active attempt: resume re-runs it.
490
494
  self._store.finish_attempt(
491
495
  attempt.id,
492
- AttemptStatus.FAILED,
493
- usage=failure.usage,
494
- error=failure_message,
496
+ AttemptStatus.CANCELLED,
497
+ usage=tuple(spent),
498
+ error="attempt cancelled",
495
499
  )
496
- delay = self._retry_policy.delay_before_retry(attempt.number)
497
- self._store.append_event(
498
- trial.study_id,
499
- "trial.retry_scheduled",
500
- {
501
- "trial_id": trial.id,
502
- "attempt": attempt.number,
503
- "delay_seconds": delay,
504
- },
505
- )
506
- await self._dispatch_events(trial.study_id)
507
- try:
508
- await self._sleeper(delay)
509
- except asyncio.CancelledError:
510
- self._store.transition_trial(trial.id, TrialStatus.CANCELLED)
511
- raise
512
- cached_metrics = None
513
- continue
514
- self._store.commit_trial_result(
515
- trial.id,
516
- status=TrialStatus.FAILED,
517
- attempt_id=attempt.id,
518
- usage=failure.usage,
519
- error=failure_message,
520
- )
521
- await self._dispatch_events(trial.study_id)
522
- return TrialSummary(
523
- id=trial.id,
524
- sequence=trial.sequence,
525
- candidate=candidate,
526
- status=TrialStatus.FAILED,
527
- metrics=(),
528
- usage=tuple(accumulated_usage),
529
- objective=None,
530
- attempts=attempt.number + 1,
531
- cached=cached_metrics is not None,
532
- error=failure_message,
533
- )
500
+ raise
534
501
  except Exception as error:
535
- classified_failure = _TrialFailure(
536
- "trial", type(error).__name__, (), self._retry_policy.is_retryable(error)
502
+ failure = (
503
+ error
504
+ if isinstance(error, _TrialFailure)
505
+ else _TrialFailure(
506
+ "trial", _describe(error), self._retry_policy.is_retryable(error)
507
+ )
537
508
  )
538
- accumulated_usage.extend(classified_failure.usage)
539
- failure_message = (
540
- f"{classified_failure.stage} failed: {classified_failure.cause_type}"
509
+ usage = tuple(spent)
510
+ accumulated_usage.extend(usage)
511
+ failure_message = f"{failure.stage} failed: {failure.cause_type}"
512
+ # Persisted errors stay redacted; the full cause goes to the console log only.
513
+ _LOGGER.warning(
514
+ "trial %d attempt %d %s (cause: %s)",
515
+ trial.sequence,
516
+ attempt.number,
517
+ failure_message,
518
+ _cause_text(error),
541
519
  )
542
- if classified_failure.retryable and attempt.number < self._retry_policy.max_retries:
520
+ if failure.retryable and attempt.number < self._retry_policy.max_retries:
543
521
  self._store.finish_attempt(
544
- attempt.id, AttemptStatus.FAILED, error=failure_message
522
+ attempt.id, AttemptStatus.FAILED, usage=usage, error=failure_message
545
523
  )
546
524
  delay = self._retry_policy.delay_before_retry(attempt.number)
547
525
  self._store.append_event(
@@ -554,17 +532,15 @@ class Optimizer:
554
532
  },
555
533
  )
556
534
  await self._dispatch_events(trial.study_id)
557
- try:
558
- await self._sleeper(delay)
559
- except asyncio.CancelledError:
560
- self._store.transition_trial(trial.id, TrialStatus.CANCELLED)
561
- raise
535
+ # A cancellation here leaves the trial recoverable for resume.
536
+ await self._sleeper(delay)
562
537
  cached_metrics = None
563
538
  continue
564
539
  self._store.commit_trial_result(
565
540
  trial.id,
566
541
  status=TrialStatus.FAILED,
567
542
  attempt_id=attempt.id,
543
+ usage=usage,
568
544
  error=failure_message,
569
545
  )
570
546
  await self._dispatch_events(trial.study_id)
@@ -577,32 +553,28 @@ class Optimizer:
577
553
  usage=tuple(accumulated_usage),
578
554
  objective=None,
579
555
  attempts=attempt.number + 1,
580
- cached=False,
556
+ cached=cached_metrics is not None,
581
557
  error=failure_message,
582
558
  )
583
559
 
584
560
  async def _execute_candidate(
585
- self, dataset: Dataset, candidate: Candidate
586
- ) -> tuple[tuple[MetricValue, ...], tuple[UsageRecord, ...]]:
561
+ self, dataset: Dataset, candidate: Candidate, spent: list[UsageRecord]
562
+ ) -> tuple[MetricValue, ...]:
587
563
  observations: dict[str, list[MetricValue]] = defaultdict(list)
588
- usage: list[UsageRecord] = []
589
564
  try:
590
565
  examples = [example async for example in dataset]
591
566
  except Exception as error:
592
567
  raise _TrialFailure(
593
- "dataset",
594
- type(error).__name__,
595
- (),
596
- self._retry_policy.is_retryable(error),
568
+ "dataset", _describe(error), self._retry_policy.is_retryable(error)
597
569
  ) from error
598
570
  if not examples:
599
- raise _TrialFailure("dataset", "empty", tuple(usage))
571
+ raise _TrialFailure("dataset", "empty")
600
572
 
601
573
  semaphore = asyncio.Semaphore(self._settings.sample_concurrency)
602
574
 
603
- async def bounded(example: Any) -> _SampleResult:
575
+ async def bounded(example: Any) -> tuple[MetricValue, ...]:
604
576
  async with semaphore:
605
- return await self._execute_sample(candidate, example)
577
+ return await self._execute_sample(candidate, example, spent)
606
578
 
607
579
  outcomes = await asyncio.gather(
608
580
  *(bounded(example) for example in examples), return_exceptions=True
@@ -610,28 +582,21 @@ class Optimizer:
610
582
  first_failure: _TrialFailure | None = None
611
583
  for outcome in outcomes:
612
584
  if isinstance(outcome, _TrialFailure):
613
- usage.extend(outcome.usage)
614
- if first_failure is None:
615
- first_failure = outcome
585
+ first_failure = first_failure or outcome
616
586
  continue
617
587
  if isinstance(outcome, BaseException):
618
588
  raise outcome
619
- usage.extend(outcome.usage)
620
- for metric in outcome.metrics:
589
+ for metric in outcome:
621
590
  observations[metric.name].append(metric)
622
591
  if first_failure is not None:
623
- raise _TrialFailure(
624
- first_failure.stage,
625
- first_failure.cause_type,
626
- tuple(usage),
627
- first_failure.retryable,
628
- ) from first_failure
592
+ raise first_failure
629
593
  if not observations:
630
- raise _TrialFailure("evaluation", "no_metrics", tuple(usage))
631
- return _aggregate_metrics(observations, len(examples)), tuple(usage)
594
+ raise _TrialFailure("evaluation", "no_metrics")
595
+ return _aggregate_metrics(observations, len(examples))
632
596
 
633
- async def _execute_sample(self, candidate: Candidate, example: Any) -> _SampleResult:
634
- usage: tuple[UsageRecord, ...] = ()
597
+ async def _execute_sample(
598
+ self, candidate: Candidate, example: Any, spent: list[UsageRecord]
599
+ ) -> tuple[MetricValue, ...]:
635
600
  stage = "adapter"
636
601
  try:
637
602
  output, sample_usage = await _wait_for(
@@ -641,6 +606,7 @@ class Optimizer:
641
606
  usage = tuple(sample_usage)
642
607
  if any(not isinstance(item, UsageRecord) for item in usage):
643
608
  raise TypeError("adapter usage must contain UsageRecord values")
609
+ spent.extend(usage)
644
610
  sample_metrics: list[MetricValue] = []
645
611
  sample_names: set[str] = set()
646
612
  for evaluator in self._evaluators:
@@ -660,29 +626,23 @@ class Optimizer:
660
626
  raise ValueError(f"duplicate metric for sample: {metric.name}")
661
627
  sample_names.add(metric.name)
662
628
  sample_metrics.append(metric)
663
- return _SampleResult(tuple(sample_metrics), usage)
629
+ return tuple(sample_metrics)
664
630
  except Exception as error:
665
631
  raise _TrialFailure(
666
- stage,
667
- type(error).__name__,
668
- usage,
669
- self._retry_policy.is_retryable(error),
632
+ stage, _describe(error), self._retry_policy.is_retryable(error)
670
633
  ) from error
671
634
 
672
- def _evaluate_objective(
673
- self, metrics: Sequence[MetricValue], usage: tuple[UsageRecord, ...]
674
- ) -> ObjectiveResult:
635
+ def _evaluate_objective(self, metrics: Sequence[MetricValue]) -> ObjectiveResult:
675
636
  try:
676
637
  objective = self._objective.evaluate(metrics)
677
638
  except Exception as error:
678
639
  raise _TrialFailure(
679
- "objective",
680
- type(error).__name__,
681
- usage,
682
- self._retry_policy.is_retryable(error),
640
+ "objective", _describe(error), self._retry_policy.is_retryable(error)
683
641
  ) from error
684
642
  if objective.score is None:
685
- raise _TrialFailure("objective", "unavailable", usage)
643
+ raise _TrialFailure("objective", "unavailable")
644
+ if not math.isfinite(objective.score):
645
+ raise _TrialFailure("objective", "non_finite")
686
646
  return objective
687
647
 
688
648
  def _read_cache(self, trial: TrialRecord, key: CacheKey) -> tuple[MetricValue, ...] | None:
@@ -766,6 +726,7 @@ class Optimizer:
766
726
  "direction": term.direction.value,
767
727
  "minimum": term.minimum,
768
728
  "maximum": term.maximum,
729
+ "min_coverage": term.min_coverage,
769
730
  }
770
731
  for term in self._objective.terms
771
732
  ],
@@ -913,6 +874,7 @@ def _stop_context(summaries: Sequence[TrialSummary], elapsed_seconds: float) ->
913
874
  successful_trials=successful,
914
875
  failed_trials=failed,
915
876
  objective_scores=scores,
877
+ failure_history=tuple(summary.status is TrialStatus.FAILED for summary in summaries),
916
878
  )
917
879
 
918
880
 
@@ -932,14 +894,14 @@ def _build_result(
932
894
  best = (
933
895
  max(successful, key=lambda item: _objective_score(item.objective)) if successful else None
934
896
  )
935
- all_usage = [item for summary in summaries for item in summary.usage]
897
+ all_usage = tuple(item for summary in summaries for item in summary.usage)
936
898
  return OptimizationResult(
937
899
  study_id=study_id,
938
900
  study_status=study_status,
939
901
  stop_reason=stop_reason,
940
902
  trials=tuple(summaries),
941
903
  best_trial=best,
942
- usage=_usage_summary(all_usage),
904
+ usage=summarize_usage(all_usage),
943
905
  )
944
906
 
945
907
 
@@ -949,27 +911,48 @@ def _objective_score(result: ObjectiveResult | None) -> float:
949
911
  return result.score
950
912
 
951
913
 
952
- def _usage_summary(usage: Sequence[UsageRecord]) -> UsageSummary:
953
- input_tokens = sum(item.input_tokens or 0 for item in usage)
954
- output_tokens = sum(item.output_tokens or 0 for item in usage)
955
- exact_cost = sum(item.cost or 0.0 for item in usage if item.status is ValueStatus.EXACT)
956
- estimated_cost = sum(item.cost or 0.0 for item in usage if item.status is not ValueStatus.EXACT)
957
- latency = sum(item.latency_seconds or 0.0 for item in usage)
958
- statuses = {item.status for item in usage}
959
- if not usage or statuses == {ValueStatus.UNAVAILABLE}:
960
- status = ValueStatus.UNAVAILABLE
961
- elif statuses == {ValueStatus.EXACT}:
962
- status = ValueStatus.EXACT
963
- else:
964
- status = ValueStatus.ESTIMATED
965
- return UsageSummary(
966
- input_tokens,
967
- output_tokens,
968
- exact_cost,
969
- estimated_cost,
970
- latency,
971
- status,
972
- )
914
+ def _describe(error: BaseException) -> str:
915
+ """Name a failure without leaking third-party messages, which may contain user data.
916
+
917
+ TunaRAG's own error messages are authored diagnostics, so they are kept.
918
+ """
919
+
920
+ name = type(error).__name__
921
+ if isinstance(error, TunaRAGError):
922
+ return f"{name}: {error.message}"
923
+ return name
924
+
925
+
926
+ def _cause_text(error: BaseException) -> str:
927
+ root = error.__cause__ if isinstance(error, _TrialFailure) and error.__cause__ else error
928
+ return f"{type(root).__name__}: {root}"[:500]
929
+
930
+
931
+ def _active_seconds(events: Sequence[StudyEvent]) -> float:
932
+ """Sum the run segments (start/resume to last event) recorded in the event log."""
933
+
934
+ total = 0.0
935
+ segment_start: float | None = None
936
+ last: float | None = None
937
+ for event in events:
938
+ timestamp = _event_time(event)
939
+ if timestamp is None:
940
+ continue
941
+ if event.type in {"study.running", "study.resumed"}:
942
+ if segment_start is not None and last is not None:
943
+ total += max(0.0, last - segment_start)
944
+ segment_start = timestamp
945
+ last = timestamp
946
+ if segment_start is not None and last is not None:
947
+ total += max(0.0, last - segment_start)
948
+ return total
949
+
950
+
951
+ def _event_time(event: StudyEvent) -> float | None:
952
+ try:
953
+ return datetime.fromisoformat(event.created_at).timestamp()
954
+ except (TypeError, ValueError):
955
+ return None
973
956
 
974
957
 
975
958
  def _type_name(value: object) -> str:
tunarag/evaluators.py CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import copy
5
6
  import inspect
6
7
  import math
7
8
  from collections.abc import Awaitable, Callable, Mapping, Sequence
@@ -12,6 +13,7 @@ from typing import Any
12
13
  from .contracts import Fingerprintable
13
14
  from .domain import MetricValue, ValueStatus
14
15
  from .errors import EvaluationError, MissingOptionalDependencyError
16
+ from .serialization import callable_identity, callable_name
15
17
 
16
18
  MetricScorer = Callable[[Any, Any], float | None | Awaitable[float | None]]
17
19
  RagasSampleMapper = Callable[[Any, Any], Mapping[str, Any]]
@@ -26,6 +28,7 @@ class MetricEvaluator:
26
28
  scorer: MetricScorer,
27
29
  *,
28
30
  status: ValueStatus = ValueStatus.EXACT,
31
+ identity: Any = None,
29
32
  ) -> None:
30
33
  if not name.strip():
31
34
  raise ValueError("metric name must not be empty")
@@ -36,6 +39,7 @@ class MetricEvaluator:
36
39
  self._name = name
37
40
  self._scorer = scorer
38
41
  self._status = status
42
+ self._identity = identity
39
43
 
40
44
  async def evaluate(self, output: Any, example: Any) -> Sequence[MetricValue]:
41
45
  """Score one adapter output and evaluation example."""
@@ -53,7 +57,14 @@ class MetricEvaluator:
53
57
  return {
54
58
  "name": self._name,
55
59
  "status": self._status.value,
56
- "scorer": _callable_identity(self._scorer),
60
+ # Without an explicit identity, a code digest makes different lambdas and edited
61
+ # functions differ; an explicit identity is the declared version instead.
62
+ "scorer": (
63
+ callable_identity(self._scorer)
64
+ if self._identity is None
65
+ else callable_name(self._scorer)
66
+ ),
67
+ "identity": self._identity,
57
68
  }
58
69
 
59
70
 
@@ -100,9 +111,13 @@ class RagasEvaluator:
100
111
  try:
101
112
  sample = dataset_schema.SingleTurnSample(**dict(sample_data))
102
113
  dataset = ragas.EvaluationDataset(samples=[sample])
114
+ # RAGAS attaches the judge LLM/embeddings to metric objects and detaches them when
115
+ # each call finishes. Samples run concurrently, so each call gets its own copies;
116
+ # otherwise one sample's cleanup strips the judge from another mid-evaluation.
117
+ metrics = [copy.copy(metric) for metric in self._metrics]
103
118
  result = await ragas.aevaluate(
104
119
  dataset=dataset,
105
- metrics=list(self._metrics),
120
+ metrics=metrics,
106
121
  llm=self._llm,
107
122
  embeddings=self._embeddings,
108
123
  raise_exceptions=self._raise_exceptions,
@@ -141,7 +156,7 @@ class RagasEvaluator:
141
156
 
142
157
  return {
143
158
  "metrics": [_component_identity(metric) for metric in self._metrics],
144
- "sample_mapper": _callable_identity(self._sample_mapper),
159
+ "sample_mapper": callable_identity(self._sample_mapper),
145
160
  "metric_prefix": self._metric_prefix,
146
161
  "llm": _component_identity(self._llm),
147
162
  "embeddings": _component_identity(self._embeddings),
@@ -169,12 +184,6 @@ def _numeric_score(value: Any, *, metric: str) -> float | None:
169
184
  return normalized if math.isfinite(normalized) else None
170
185
 
171
186
 
172
- def _callable_identity(function: Callable[..., Any]) -> str:
173
- module = getattr(function, "__module__", type(function).__module__)
174
- name = getattr(function, "__qualname__", type(function).__qualname__)
175
- return f"{module}.{name}"
176
-
177
-
178
187
  def _component_identity(component: Any) -> Any:
179
188
  if component is None:
180
189
  return None