traigent 0.18.1.dev8__tar.gz → 0.18.1.dev9__tar.gz

{traigent-0.18.1.dev8/traigent.egg-info → traigent-0.18.1.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: traigent
-Version: 0.18.1.dev8
+Version: 0.18.1.dev9
 Summary: Enterprise-grade LLM optimization platform with advanced analytics and AI-powered insights
 Author-email: Traigent Team <opensource@traigent.ai>
 License-Expression: AGPL-3.0-only OR LicenseRef-Traigent-Commercial

{traigent-0.18.1.dev8 → traigent-0.18.1.dev9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "traigent"
-version = "0.18.1.dev8"
+version = "0.18.1.dev9"
 authors = [
     {name = "Traigent Team", email = "opensource@traigent.ai"},
 ]

{traigent-0.18.1.dev8 → traigent-0.18.1.dev9}/traigent/core/orchestrator.py

@@ -101,6 +101,7 @@ from traigent.core.workflow_trace_manager import WorkflowTraceManager
 from traigent.evaluators.base import BaseEvaluator, Dataset
 from traigent.metrics.registry import clone_registry
 from traigent.optimizers.base import BaseOptimizer
+from traigent.optimizers.interactive_optimizer import CloudBrainOptimizationComplete
 from traigent.tvl.promotion_gate import PromotionGate
 from traigent.utils.callbacks import CallbackManager, OptimizationCallback, ProgressInfo
 from traigent.utils.env_config import (  # noqa: F401
@@ -2588,6 +2589,10 @@ class OptimizationOrchestrator:
                 self._stop_reason = "vendor_error"
                 return trial_count, "break"
             return trial_count, "continue"  # User chose to resume
+        except CloudBrainOptimizationComplete as complete:
+            self._stop_reason = cast(StopReason, complete.stop_reason)
+            logger.info("Cloud brain completed optimization: %s", complete.reason)
+            return trial_count, "break"
 
     @staticmethod
     def _vendor_retry_settings() -> tuple[int, float]:

{traigent-0.18.1.dev8 → traigent-0.18.1.dev9}/traigent/evaluators/base.py

@@ -1646,6 +1646,7 @@ class BaseEvaluator(ABC):
                         config,
                         example,
                         i,
+                        dataset,
                         executor=None,
                         progress_callback=progress_callback,
                     )
@@ -2009,6 +2010,7 @@ class BaseEvaluator(ABC):
             consumed, exhausted = await self._run_concurrent_tasks(
                 func,
                 config,
+                dataset,
                 examples,
                 sample_lease,
                 detailed,
@@ -2043,6 +2045,7 @@ class BaseEvaluator(ABC):
         self,
         func: Callable[..., Any],
         config: dict[str, Any],
+        dataset: Dataset,
         examples: list[EvaluationExample],
         sample_lease: SampleBudgetLease | None,
         detailed: bool,
@@ -2066,6 +2069,7 @@ class BaseEvaluator(ABC):
                         config,
                         example,
                         idx,
+                        dataset,
                         executor=None,
                         progress_callback=progress_callback,
                     )
@@ -2238,18 +2242,31 @@ class BaseEvaluator(ABC):
         result: ExampleResult,
         config: dict[str, Any],
         error: str | None,
+        dataset: Dataset,
+        example_index: int,
     ) -> dict[str, Any]:
         """Build progress callback payload with metrics."""
-        metrics_payload = self._build_progress_accuracy_metrics(
-            output=result.actual_output,
-            expected_output=example.expected_output,
-            error=error,
-            example_id=example_id,
-        )
-
         metrics_update, _ = self._extract_response_metrics(
             example_id, example, result, config
         )
+        llm_metrics = dict(result.metrics or metrics_update)
+
+        metrics_payload = self._build_progress_objective_metrics(
+            example=example,
+            result=result,
+            config=config,
+            error=error,
+            dataset=dataset,
+            example_index=example_index,
+            llm_metrics=llm_metrics or None,
+        )
+        if not metrics_payload and not self._has_configured_progress_objective():
+            metrics_payload = self._build_progress_accuracy_metrics(
+                output=result.actual_output,
+                expected_output=example.expected_output,
+                error=error,
+                example_id=example_id,
+            )
         metrics_payload.update(metrics_update)
 
         payload: dict[str, Any] = {
@@ -2261,6 +2278,157 @@ class BaseEvaluator(ABC):
             payload["metrics"] = metrics_payload
         return payload
 
+    def _has_configured_progress_objective(self) -> bool:
+        """Return True when callers configured a custom per-example objective."""
+        return bool(
+            getattr(self, "metric_functions", None)
+            or getattr(self, "scoring_function", None)
+        )
+
+    def _build_progress_objective_metrics(
+        self,
+        *,
+        example: EvaluationExample,
+        result: ExampleResult,
+        config: dict[str, Any],
+        error: str | None,
+        dataset: Dataset,
+        example_index: int,
+        llm_metrics: dict[str, Any] | None,
+    ) -> dict[str, float]:
+        """Return configured per-example objective metrics for detailed progress."""
+        if error is not None:
+            return {}
+
+        metric_functions = getattr(self, "metric_functions", None) or {}
+        if metric_functions:
+            metrics = self._call_progress_metric_functions(
+                metric_functions=metric_functions,
+                output=result.actual_output,
+                example=example,
+                config=config,
+                dataset=dataset,
+                example_index=example_index,
+                llm_metrics=llm_metrics,
+            )
+            if metrics:
+                return metrics
+
+        scoring_function = getattr(self, "scoring_function", None)
+        if scoring_function is not None:
+            metrics = self._call_progress_scoring_function(
+                output=result.actual_output,
+                example=example,
+                config=config,
+                dataset=dataset,
+                example_index=example_index,
+                llm_metrics=llm_metrics,
+            )
+            if metrics:
+                return metrics
+
+        return {}
+
+    def _call_progress_metric_functions(
+        self,
+        *,
+        metric_functions: Mapping[str, Callable[..., Any]],
+        output: Any,
+        example: EvaluationExample,
+        config: dict[str, Any],
+        dataset: Dataset,
+        example_index: int,
+        llm_metrics: dict[str, Any] | None,
+    ) -> dict[str, float]:
+        """Call configured metric functions using the evaluator's scoring helper."""
+        call_metric_functions = getattr(self, "_call_metric_functions", None)
+        if callable(call_metric_functions):
+            return self._coerce_progress_metrics(
+                call_metric_functions(
+                    output, example, config, dataset, example_index, llm_metrics
+                )
+            )
+
+        invoke_metric = getattr(self, "_invoke_metric_function", None)
+        if callable(invoke_metric):
+            metrics: dict[str, float] = {}
+            for metric_name, metric_func in metric_functions.items():
+                value = invoke_metric(
+                    metric_func,
+                    metric_name,
+                    output,
+                    example,
+                    config,
+                    llm_metrics or {},
+                    example_index,
+                )
+                if isinstance(value, Mapping):
+                    metrics.update(self._coerce_progress_metrics(value))
+                elif value is not None:
+                    try:
+                        metrics[metric_name] = float(value)
+                    except (TypeError, ValueError):
+                        logger.warning(
+                            "Metric function %s returned non-numeric progress value "
+                            "for example %s",
+                            metric_name,
+                            getattr(example, "example_id", None) or example_index,
+                        )
+            return metrics
+
+        return {}
+
+    def _call_progress_scoring_function(
+        self,
+        *,
+        output: Any,
+        example: EvaluationExample,
+        config: dict[str, Any],
+        dataset: Dataset,
+        example_index: int,
+        llm_metrics: dict[str, Any] | None,
+    ) -> dict[str, float]:
+        """Call configured scoring function using the evaluator's scoring helper."""
+        compute_example_metrics = getattr(self, "_compute_example_metrics", None)
+        if callable(compute_example_metrics):
+            try:
+                return self._coerce_progress_metrics(
+                    compute_example_metrics(
+                        output, example, config, dataset, example_index, llm_metrics
+                    )
+                )
+            except TypeError:
+                logger.debug(
+                    "Evaluator _compute_example_metrics signature does not support "
+                    "detailed progress scoring; trying scoring helper",
+                    exc_info=True,
+                )
+
+        call_scoring_function = getattr(self, "_call_scoring_function", None)
+        if callable(call_scoring_function):
+            return self._coerce_progress_metrics(
+                call_scoring_function(output, example, llm_metrics)
+            )
+
+        return {}
+
+    @staticmethod
+    def _coerce_progress_metrics(values: Mapping[str, Any] | None) -> dict[str, float]:
+        """Keep only finite numeric metrics for the progress payload."""
+        metrics: dict[str, float] = {}
+        if not values:
+            return metrics
+        for name, value in values.items():
+            if isinstance(value, bool) or value is None:
+                continue
+            try:
+                number = float(value)
+            except (TypeError, ValueError):
+                continue
+            if math.isfinite(number):
+                metrics[str(name)] = number
+        return metrics
+
     def _build_progress_accuracy_metrics(
         self,
         *,
@@ -2435,6 +2603,7 @@ class BaseEvaluator(ABC):
         config: dict[str, Any],
         example: EvaluationExample,
         example_index: int,
+        dataset: Dataset,
         executor: Any | None = None,
         progress_callback: Callable[[int, dict[str, Any]], Any] | None = None,
     ) -> ExampleResult:
@@ -2501,7 +2670,7 @@ class BaseEvaluator(ABC):
 
             if progress_callback:
                 progress_payload = self._build_progress_payload(
-                    example_id, example, result, config, error
+                    example_id, example, result, config, error, dataset, example_index
                 )
                 try:
                     progress_callback(example_index, progress_payload)

{traigent-0.18.1.dev8 → traigent-0.18.1.dev9}/traigent/optimizers/interactive_optimizer.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 
 import time
 from datetime import UTC, datetime
-from typing import TYPE_CHECKING, Any, Protocol, TypeAlias
+from typing import TYPE_CHECKING, Any, Protocol, TypeAlias, cast
 
 from traigent.api.types import TrialResult
 from traigent.api.types import TrialStatus as SDKTrialStatus
@@ -94,7 +94,7 @@ def _coerce_trial_status(status: TrialStatusLike) -> Any:
 
 def _is_completed_status(status: TrialStatusLike) -> bool:
     """Check completion across SDK and cloud status enums."""
-    return getattr(status, "value", status) == "completed"
+    return bool(getattr(status, "value", status) == "completed")
 
 
 def _string_sequence(value: Any) -> tuple[str, ...]:
@@ -104,6 +104,23 @@ def _string_sequence(value: Any) -> tuple[str, ...]:
     return tuple(str(item) for item in value if item)
 
 
+class CloudBrainOptimizationComplete(RuntimeError):
+    """Normal cloud-brain terminal signal from the next-trial API."""
+
+    def __init__(self, reason: str | None = None) -> None:
+        self.reason = reason or "cloud brain completed optimization"
+        super().__init__(self.reason)
+
+    @property
+    def stop_reason(self) -> str:
+        normalized = self.reason.lower().replace("-", "_").replace(" ", "_")
+        if "max" in normalized and "trial" in normalized:
+            return "max_trials_reached"
+        if "conver" in normalized:
+            return "convergence"
+        return "optimizer"
+
+
 class RemoteGuidanceService(Protocol):
     """Protocol for remote guidance services."""
 
@@ -185,6 +202,7 @@ class InteractiveOptimizer(BaseOptimizer):
         self._pending_trials: dict[str, TrialSuggestion] = {}
         self._completed_trials: list[TrialResultSubmission] = []
         self._start_time: float | None = None
+        self._completion_reason: str | None = None
 
     async def initialize_session(
         self,
@@ -271,42 +289,52 @@ class InteractiveOptimizer(BaseOptimizer):
         if not self.session_id:
             raise OptimizationError("No active session. Call initialize_session first.")
 
-        try:
-            request = NextTrialRequest(
-                session_id=self.session_id,
-                previous_results=previous_results or self._completed_trials[-5:],
-                request_metadata={
-                    "dataset_size": dataset_size,
-                    "completed_trials": len(self._completed_trials),
-                },
-            )
+        request = NextTrialRequest(
+            session_id=self.session_id,
+            previous_results=previous_results or self._completed_trials[-5:],
+            request_metadata={
+                "dataset_size": dataset_size,
+                "completed_trials": len(self._completed_trials),
+            },
+        )
 
+        try:
             response = await self.remote_service.get_next_trial(request)
+        except Exception as e:
+            logger.error(f"Failed to get next suggestion: {e}")
+            raise OptimizationError(f"Failed to get suggestion: {e}") from e
 
-            # Update local session status
-            if self.session:
-                self.session.status = response.session_status
+        # Update local session status
+        if self.session:
+            self.session.status = response.session_status
 
-            if not response.should_continue or not response.suggestion:
-                logger.info(f"Optimization complete: {response.reason}")
-                return None
+        if not response.should_continue:
+            self._completion_reason = (
+                response.stop_reason
+                or response.reason
+                or "cloud brain completed optimization"
+            )
+            logger.info(f"Optimization complete: {self._completion_reason}")
+            return None
 
-            suggestion = response.suggestion
+        if not response.suggestion:
+            raise OptimizationError(
+                "Cloud brain returned no suggestion while should_continue=True"
+            )
 
-            # Store pending trial
-            self._pending_trials[suggestion.trial_id] = suggestion
+        self._completion_reason = None
+        suggestion = response.suggestion
 
-            logger.info(
-                f"Got suggestion {suggestion.trial_id}: "
-                f"{suggestion.exploration_type} with "
-                f"{len(suggestion.dataset_subset.indices)} examples"
-            )
+        # Store pending trial
+        self._pending_trials[suggestion.trial_id] = suggestion
 
-            return suggestion
+        logger.info(
+            f"Got suggestion {suggestion.trial_id}: "
+            f"{suggestion.exploration_type} with "
+            f"{len(suggestion.dataset_subset.indices)} examples"
+        )
 
-        except Exception as e:
-            logger.error(f"Failed to get next suggestion: {e}")
-            raise OptimizationError(f"Failed to get suggestion: {e}") from e
+        return suggestion
 
     async def suggest_next_trial_async(
         self,
@@ -323,7 +351,7 @@ class InteractiveOptimizer(BaseOptimizer):
             dataset_size=max(dataset_size, 1),
         )
         if suggestion is None:
-            raise OptimizationError("Cloud brain returned no further suggestions")
+            raise CloudBrainOptimizationComplete(self._completion_reason)
 
         config = dict(suggestion.config or {})
         config["_traigent_backend_trial_id"] = suggestion.trial_id
@@ -509,8 +537,8 @@ class InteractiveOptimizer(BaseOptimizer):
         # Simple comparison on primary objective
         if self.objectives and self.objectives[0] in metrics:
             primary = self.objectives[0]
-            current = metrics.get(primary, 0)
-            best = self.session.best_metrics.get(primary, 0)
+            current = float(metrics.get(primary, 0.0))
+            best = float(self.session.best_metrics.get(primary, 0.0))
 
             # Assume higher is better for now
             return current > best
@@ -568,7 +596,10 @@ class InteractiveOptimizer(BaseOptimizer):
                 ),
                 effectuation_events=result_metadata.get("effectuation_events"),
             )
-            return merge_tvar_observation_metadata(result_metadata, observation)
+            return cast(
+                dict[str, Any],
+                merge_tvar_observation_metadata(result_metadata, observation),
+            )
         except Exception as exc:
             logger.debug(
                 "Skipping TVAR observation metadata for trial %s: %s",

{traigent-0.18.1.dev8 → traigent-0.18.1.dev9/traigent.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: traigent
-Version: 0.18.1.dev8
+Version: 0.18.1.dev9
 Summary: Enterprise-grade LLM optimization platform with advanced analytics and AI-powered insights
 Author-email: Traigent Team <opensource@traigent.ai>
 License-Expression: AGPL-3.0-only OR LicenseRef-Traigent-Commercial