traigent 0.18.1.dev7__tar.gz → 0.18.1.dev9__tar.gz

{traigent-0.18.1.dev7/traigent.egg-info → traigent-0.18.1.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: traigent
-Version: 0.18.1.dev7
+Version: 0.18.1.dev9
 Summary: Enterprise-grade LLM optimization platform with advanced analytics and AI-powered insights
 Author-email: Traigent Team <opensource@traigent.ai>
 License-Expression: AGPL-3.0-only OR LicenseRef-Traigent-Commercial

{traigent-0.18.1.dev7 → traigent-0.18.1.dev9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "traigent"
-version = "0.18.1.dev7"
+version = "0.18.1.dev9"
 authors = [
     {name = "Traigent Team", email = "opensource@traigent.ai"},
 ]

{traigent-0.18.1.dev7 → traigent-0.18.1.dev9}/traigent/core/orchestrator.py

@@ -101,6 +101,7 @@ from traigent.core.workflow_trace_manager import WorkflowTraceManager
 from traigent.evaluators.base import BaseEvaluator, Dataset
 from traigent.metrics.registry import clone_registry
 from traigent.optimizers.base import BaseOptimizer
+from traigent.optimizers.interactive_optimizer import CloudBrainOptimizationComplete
 from traigent.tvl.promotion_gate import PromotionGate
 from traigent.utils.callbacks import CallbackManager, OptimizationCallback, ProgressInfo
 from traigent.utils.env_config import (  # noqa: F401
@@ -2588,6 +2589,10 @@ class OptimizationOrchestrator:
                 self._stop_reason = "vendor_error"
                 return trial_count, "break"
             return trial_count, "continue"  # User chose to resume
+        except CloudBrainOptimizationComplete as complete:
+            self._stop_reason = cast(StopReason, complete.stop_reason)
+            logger.info("Cloud brain completed optimization: %s", complete.reason)
+            return trial_count, "break"
 
     @staticmethod
     def _vendor_retry_settings() -> tuple[int, float]:

{traigent-0.18.1.dev7 → traigent-0.18.1.dev9}/traigent/core/trial_lifecycle.py

@@ -761,6 +761,11 @@ class TrialLifecycle:
             decision = report(payload) if callable(report) else {}
             if isinstance(decision, dict) and decision.get("prune") is True:
                 reason = decision.get("prune_reason") or "cloud smart pruning"
+                logger.info(
+                    "Cloud smart pruning requested trial pruning after %s examples: %s",
+                    state["evaluated"],
+                    reason,
+                )
                 raise TrialPrunedError(str(reason), step=int(state["evaluated"]))
 
         return _progress_callback, state

{traigent-0.18.1.dev7 → traigent-0.18.1.dev9}/traigent/evaluators/base.py

@@ -1610,7 +1610,10 @@ class BaseEvaluator(ABC):
 
         if progress_callback:
             progress_callback(
-                index, {"success": error is None, "output": output, "error": error}
+                index,
+                self._build_non_detailed_progress_payload(
+                    example, index, output, error
+                ),
             )
 
         return output, error
@@ -1643,6 +1646,7 @@ class BaseEvaluator(ABC):
                         config,
                         example,
                         i,
+                        dataset,
                         executor=None,
                         progress_callback=progress_callback,
                     )
@@ -1727,6 +1731,7 @@ class BaseEvaluator(ABC):
         detailed: bool,
         progress_callback: Callable[[int, dict[str, Any]], Any] | None,
         index: int,
+        example: EvaluationExample,
         outputs: list[Any],
         errors: list[str | None],
         example_results: list[ExampleResult | None],
@@ -1742,7 +1747,10 @@ class BaseEvaluator(ABC):
             errors.append(error)
             if progress_callback:
                 progress_callback(
-                    index, {"success": error is None, "output": output, "error": error}
+                    index,
+                    self._build_non_detailed_progress_payload(
+                        example, index, output, error
+                    ),
                 )
 
     async def _handle_task_result(
@@ -1813,6 +1821,7 @@ class BaseEvaluator(ABC):
             result,
             detailed,
             progress_callback,
+            examples[index],
             outputs_by_index,
             errors_by_index,
             example_results_by_index,
@@ -1845,6 +1854,7 @@ class BaseEvaluator(ABC):
         result: Any,
         detailed: bool,
         progress_callback: Callable[[int, dict[str, Any]], Any] | None,
+        example: EvaluationExample,
         outputs_by_index: dict[int, Any],
         errors_by_index: dict[int, str | None],
         example_results_by_index: dict[int, ExampleResult | None],
@@ -1864,7 +1874,9 @@ class BaseEvaluator(ABC):
                 # Note: This is sync in the original, we handle exceptions elsewhere
                 progress_callback(
                     index,
-                    {"success": error is None, "output": output, "error": error},
+                    self._build_non_detailed_progress_payload(
+                        example, index, output, error
+                    ),
                 )
 
     async def _safe_progress_callback(
@@ -1998,6 +2010,7 @@ class BaseEvaluator(ABC):
             consumed, exhausted = await self._run_concurrent_tasks(
                 func,
                 config,
+                dataset,
                 examples,
                 sample_lease,
                 detailed,
@@ -2032,6 +2045,7 @@ class BaseEvaluator(ABC):
         self,
         func: Callable[..., Any],
         config: dict[str, Any],
+        dataset: Dataset,
         examples: list[EvaluationExample],
         sample_lease: SampleBudgetLease | None,
         detailed: bool,
@@ -2055,6 +2069,7 @@ class BaseEvaluator(ABC):
                         config,
                         example,
                         idx,
+                        dataset,
                         executor=None,
                         progress_callback=progress_callback,
                     )
@@ -2227,37 +2242,239 @@ class BaseEvaluator(ABC):
         result: ExampleResult,
         config: dict[str, Any],
         error: str | None,
+        dataset: Dataset,
+        example_index: int,
     ) -> dict[str, Any]:
         """Build progress callback payload with metrics."""
-        metrics_payload: dict[str, float] = {}
-
-        if error is None and example.expected_output is not None:
-            try:
-                metrics_payload["accuracy"] = (
-                    1.0
-                    if _accuracy_values_match(
-                        result.actual_output, example.expected_output
-                    )
-                    else 0.0
-                )
-            except Exception as exc:
-                logger.warning(
-                    "Failed to compute accuracy metric for example %s: %s",
-                    example_id,
-                    exc,
-                )
-
         metrics_update, _ = self._extract_response_metrics(
             example_id, example, result, config
         )
+        llm_metrics = dict(result.metrics or metrics_update)
+
+        metrics_payload = self._build_progress_objective_metrics(
+            example=example,
+            result=result,
+            config=config,
+            error=error,
+            dataset=dataset,
+            example_index=example_index,
+            llm_metrics=llm_metrics or None,
+        )
+        if not metrics_payload and not self._has_configured_progress_objective():
+            metrics_payload = self._build_progress_accuracy_metrics(
+                output=result.actual_output,
+                expected_output=example.expected_output,
+                error=error,
+                example_id=example_id,
+            )
         metrics_payload.update(metrics_update)
 
-        return {
+        payload: dict[str, Any] = {
             "success": result.success,
             "error": result.error_message,
-            "metrics": metrics_payload,
             "output": result.actual_output,
         }
+        if metrics_payload:
+            payload["metrics"] = metrics_payload
+        return payload
+
+    def _has_configured_progress_objective(self) -> bool:
+        """Return True when callers configured a custom per-example objective."""
+        return bool(
+            getattr(self, "metric_functions", None)
+            or getattr(self, "scoring_function", None)
+        )
+
+    def _build_progress_objective_metrics(
+        self,
+        *,
+        example: EvaluationExample,
+        result: ExampleResult,
+        config: dict[str, Any],
+        error: str | None,
+        dataset: Dataset,
+        example_index: int,
+        llm_metrics: dict[str, Any] | None,
+    ) -> dict[str, float]:
+        """Return configured per-example objective metrics for detailed progress."""
+        if error is not None:
+            return {}
+
+        metric_functions = getattr(self, "metric_functions", None) or {}
+        if metric_functions:
+            metrics = self._call_progress_metric_functions(
+                metric_functions=metric_functions,
+                output=result.actual_output,
+                example=example,
+                config=config,
+                dataset=dataset,
+                example_index=example_index,
+                llm_metrics=llm_metrics,
+            )
+            if metrics:
+                return metrics
+
+        scoring_function = getattr(self, "scoring_function", None)
+        if scoring_function is not None:
+            metrics = self._call_progress_scoring_function(
+                output=result.actual_output,
+                example=example,
+                config=config,
+                dataset=dataset,
+                example_index=example_index,
+                llm_metrics=llm_metrics,
+            )
+            if metrics:
+                return metrics
+
+        return {}
+
+    def _call_progress_metric_functions(
+        self,
+        *,
+        metric_functions: Mapping[str, Callable[..., Any]],
+        output: Any,
+        example: EvaluationExample,
+        config: dict[str, Any],
+        dataset: Dataset,
+        example_index: int,
+        llm_metrics: dict[str, Any] | None,
+    ) -> dict[str, float]:
+        """Call configured metric functions using the evaluator's scoring helper."""
+        call_metric_functions = getattr(self, "_call_metric_functions", None)
+        if callable(call_metric_functions):
+            return self._coerce_progress_metrics(
+                call_metric_functions(
+                    output, example, config, dataset, example_index, llm_metrics
+                )
+            )
+
+        invoke_metric = getattr(self, "_invoke_metric_function", None)
+        if callable(invoke_metric):
+            metrics: dict[str, float] = {}
+            for metric_name, metric_func in metric_functions.items():
+                value = invoke_metric(
+                    metric_func,
+                    metric_name,
+                    output,
+                    example,
+                    config,
+                    llm_metrics or {},
+                    example_index,
+                )
+                if isinstance(value, Mapping):
+                    metrics.update(self._coerce_progress_metrics(value))
+                elif value is not None:
+                    try:
+                        metrics[metric_name] = float(value)
+                    except (TypeError, ValueError):
+                        logger.warning(
+                            "Metric function %s returned non-numeric progress value "
+                            "for example %s",
+                            metric_name,
+                            getattr(example, "example_id", None) or example_index,
+                        )
+            return metrics
+
+        return {}
+
+    def _call_progress_scoring_function(
+        self,
+        *,
+        output: Any,
+        example: EvaluationExample,
+        config: dict[str, Any],
+        dataset: Dataset,
+        example_index: int,
+        llm_metrics: dict[str, Any] | None,
+    ) -> dict[str, float]:
+        """Call configured scoring function using the evaluator's scoring helper."""
+        compute_example_metrics = getattr(self, "_compute_example_metrics", None)
+        if callable(compute_example_metrics):
+            try:
+                return self._coerce_progress_metrics(
+                    compute_example_metrics(
+                        output, example, config, dataset, example_index, llm_metrics
+                    )
+                )
+            except TypeError:
+                logger.debug(
+                    "Evaluator _compute_example_metrics signature does not support "
+                    "detailed progress scoring; trying scoring helper",
+                    exc_info=True,
+                )
+
+        call_scoring_function = getattr(self, "_call_scoring_function", None)
+        if callable(call_scoring_function):
+            return self._coerce_progress_metrics(
+                call_scoring_function(output, example, llm_metrics)
+            )
+
+        return {}
+
+    @staticmethod
+    def _coerce_progress_metrics(values: Mapping[str, Any] | None) -> dict[str, float]:
+        """Keep only finite numeric metrics for the progress payload."""
+        metrics: dict[str, float] = {}
+        if not values:
+            return metrics
+        for name, value in values.items():
+            if isinstance(value, bool) or value is None:
+                continue
+            try:
+                number = float(value)
+            except (TypeError, ValueError):
+                continue
+            if math.isfinite(number):
+                metrics[str(name)] = number
+        return metrics
+
+    def _build_progress_accuracy_metrics(
+        self,
+        *,
+        output: Any,
+        expected_output: Any,
+        error: str | None,
+        example_id: str,
+    ) -> dict[str, float]:
+        """Return per-example accuracy using the same comparator as aggregate accuracy."""
+        if error is not None or _is_empty_expected_output(expected_output):
+            return {}
+        try:
+            return {
+                "accuracy": (
+                    1.0 if _accuracy_values_match(output, expected_output) else 0.0
+                )
+            }
+        except Exception as exc:
+            logger.warning(
+                "Failed to compute accuracy metric for example %s: %s",
+                example_id,
+                exc,
+            )
+            return {}
+
+    def _build_non_detailed_progress_payload(
+        self,
+        example: EvaluationExample,
+        index: int,
+        output: Any,
+        error: str | None,
+    ) -> dict[str, Any]:
+        payload: dict[str, Any] = {
+            "success": error is None,
+            "output": output,
+            "error": error,
+        }
+        metrics_payload = self._build_progress_accuracy_metrics(
+            output=output,
+            expected_output=example.expected_output,
+            error=error,
+            example_id=_example_correlation_key(example, index),
+        )
+        if metrics_payload:
+            payload["metrics"] = metrics_payload
+        return payload
 
     def _record_example_trace(
         self,
@@ -2386,6 +2603,7 @@ class BaseEvaluator(ABC):
         config: dict[str, Any],
         example: EvaluationExample,
         example_index: int,
+        dataset: Dataset,
         executor: Any | None = None,
         progress_callback: Callable[[int, dict[str, Any]], Any] | None = None,
     ) -> ExampleResult:
@@ -2452,7 +2670,7 @@ class BaseEvaluator(ABC):
 
             if progress_callback:
                 progress_payload = self._build_progress_payload(
-                    example_id, example, result, config, error
+                    example_id, example, result, config, error, dataset, example_index
                 )
                 try:
                     progress_callback(example_index, progress_payload)

{traigent-0.18.1.dev7 → traigent-0.18.1.dev9}/traigent/optimizers/interactive_optimizer.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 
 import time
 from datetime import UTC, datetime
-from typing import TYPE_CHECKING, Any, Protocol, TypeAlias
+from typing import TYPE_CHECKING, Any, Protocol, TypeAlias, cast
 
 from traigent.api.types import TrialResult
 from traigent.api.types import TrialStatus as SDKTrialStatus
@@ -94,7 +94,7 @@ def _coerce_trial_status(status: TrialStatusLike) -> Any:
 
 def _is_completed_status(status: TrialStatusLike) -> bool:
     """Check completion across SDK and cloud status enums."""
-    return getattr(status, "value", status) == "completed"
+    return bool(getattr(status, "value", status) == "completed")
 
 
 def _string_sequence(value: Any) -> tuple[str, ...]:
@@ -104,6 +104,23 @@ def _string_sequence(value: Any) -> tuple[str, ...]:
     return tuple(str(item) for item in value if item)
 
 
+class CloudBrainOptimizationComplete(RuntimeError):
+    """Normal cloud-brain terminal signal from the next-trial API."""
+
+    def __init__(self, reason: str | None = None) -> None:
+        self.reason = reason or "cloud brain completed optimization"
+        super().__init__(self.reason)
+
+    @property
+    def stop_reason(self) -> str:
+        normalized = self.reason.lower().replace("-", "_").replace(" ", "_")
+        if "max" in normalized and "trial" in normalized:
+            return "max_trials_reached"
+        if "conver" in normalized:
+            return "convergence"
+        return "optimizer"
+
+
 class RemoteGuidanceService(Protocol):
     """Protocol for remote guidance services."""
 
@@ -185,6 +202,7 @@ class InteractiveOptimizer(BaseOptimizer):
         self._pending_trials: dict[str, TrialSuggestion] = {}
         self._completed_trials: list[TrialResultSubmission] = []
         self._start_time: float | None = None
+        self._completion_reason: str | None = None
 
     async def initialize_session(
         self,
@@ -271,42 +289,52 @@ class InteractiveOptimizer(BaseOptimizer):
         if not self.session_id:
             raise OptimizationError("No active session. Call initialize_session first.")
 
-        try:
-            request = NextTrialRequest(
-                session_id=self.session_id,
-                previous_results=previous_results or self._completed_trials[-5:],
-                request_metadata={
-                    "dataset_size": dataset_size,
-                    "completed_trials": len(self._completed_trials),
-                },
-            )
+        request = NextTrialRequest(
+            session_id=self.session_id,
+            previous_results=previous_results or self._completed_trials[-5:],
+            request_metadata={
+                "dataset_size": dataset_size,
+                "completed_trials": len(self._completed_trials),
+            },
+        )
 
+        try:
             response = await self.remote_service.get_next_trial(request)
+        except Exception as e:
+            logger.error(f"Failed to get next suggestion: {e}")
+            raise OptimizationError(f"Failed to get suggestion: {e}") from e
 
-            # Update local session status
-            if self.session:
-                self.session.status = response.session_status
+        # Update local session status
+        if self.session:
+            self.session.status = response.session_status
 
-            if not response.should_continue or not response.suggestion:
-                logger.info(f"Optimization complete: {response.reason}")
-                return None
+        if not response.should_continue:
+            self._completion_reason = (
+                response.stop_reason
+                or response.reason
+                or "cloud brain completed optimization"
+            )
+            logger.info(f"Optimization complete: {self._completion_reason}")
+            return None
 
-            suggestion = response.suggestion
+        if not response.suggestion:
+            raise OptimizationError(
+                "Cloud brain returned no suggestion while should_continue=True"
+            )
 
-            # Store pending trial
-            self._pending_trials[suggestion.trial_id] = suggestion
+        self._completion_reason = None
+        suggestion = response.suggestion
 
-            logger.info(
-                f"Got suggestion {suggestion.trial_id}: "
-                f"{suggestion.exploration_type} with "
-                f"{len(suggestion.dataset_subset.indices)} examples"
-            )
+        # Store pending trial
+        self._pending_trials[suggestion.trial_id] = suggestion
 
-            return suggestion
+        logger.info(
+            f"Got suggestion {suggestion.trial_id}: "
+            f"{suggestion.exploration_type} with "
+            f"{len(suggestion.dataset_subset.indices)} examples"
+        )
 
-        except Exception as e:
-            logger.error(f"Failed to get next suggestion: {e}")
-            raise OptimizationError(f"Failed to get suggestion: {e}") from e
+        return suggestion
 
     async def suggest_next_trial_async(
         self,
@@ -323,7 +351,7 @@ class InteractiveOptimizer(BaseOptimizer):
             dataset_size=max(dataset_size, 1),
         )
         if suggestion is None:
-            raise OptimizationError("Cloud brain returned no further suggestions")
+            raise CloudBrainOptimizationComplete(self._completion_reason)
 
         config = dict(suggestion.config or {})
         config["_traigent_backend_trial_id"] = suggestion.trial_id
@@ -509,8 +537,8 @@ class InteractiveOptimizer(BaseOptimizer):
         # Simple comparison on primary objective
         if self.objectives and self.objectives[0] in metrics:
             primary = self.objectives[0]
-            current = metrics.get(primary, 0)
-            best = self.session.best_metrics.get(primary, 0)
+            current = float(metrics.get(primary, 0.0))
+            best = float(self.session.best_metrics.get(primary, 0.0))
 
             # Assume higher is better for now
             return current > best
@@ -568,7 +596,10 @@ class InteractiveOptimizer(BaseOptimizer):
                 ),
                 effectuation_events=result_metadata.get("effectuation_events"),
             )
-            return merge_tvar_observation_metadata(result_metadata, observation)
+            return cast(
+                dict[str, Any],
+                merge_tvar_observation_metadata(result_metadata, observation),
+            )
         except Exception as exc:
             logger.debug(
                 "Skipping TVAR observation metadata for trial %s: %s",

{traigent-0.18.1.dev7 → traigent-0.18.1.dev9/traigent.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: traigent
-Version: 0.18.1.dev7
+Version: 0.18.1.dev9
 Summary: Enterprise-grade LLM optimization platform with advanced analytics and AI-powered insights
 Author-email: Traigent Team <opensource@traigent.ai>
 License-Expression: AGPL-3.0-only OR LicenseRef-Traigent-Commercial