google-cloud-agentplatform 2.7.0__tar.gz → 2.8.0__tar.gz

{google_cloud_agentplatform-2.7.0/google_cloud_agentplatform.egg-info → google_cloud_agentplatform-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-cloud-agentplatform
-Version: 2.7.0
+Version: 2.8.0
 Summary: Google Cloud Agent Platform Client Library
 Author-email: Google LLC <googleapis-packages@google.com>
 License: Apache-2.0
@@ -61,8 +61,7 @@ Provides-Extra: evaluation
 Requires-Dist: pandas>=1.0.0; extra == "evaluation"
 Requires-Dist: tqdm>=4.23.0; extra == "evaluation"
 Requires-Dist: pyyaml; extra == "evaluation"
-Requires-Dist: litellm<1.86.0,>=1.83.7; python_version < "3.14" and extra == "evaluation"
-Requires-Dist: litellm<1.97.0,>=1.93.0; python_version >= "3.14" and extra == "evaluation"
+Requires-Dist: litellm<1.107.0,>=1.106.5; python_version < "3.15" and extra == "evaluation"
 Requires-Dist: google-cloud-storage<4.0.0,>=1.32.0; extra == "evaluation"
 Provides-Extra: bigquery
 Requires-Dist: google-cloud-bigquery<4.0.0,>=1.15.0; extra == "bigquery"

{google_cloud_agentplatform-2.7.0 → google_cloud_agentplatform-2.8.0}/agentplatform/_genai/_evals_common.py

@@ -3238,6 +3238,31 @@ def _process_multi_turn_agent_response(
     ).model_dump(exclude_unset=True)
 
 
+def _extract_final_response_text(
+    events: list[dict[str, Any]],
+) -> tuple[Optional[int], str]:
+    """Returns the index and text of the last final response in the events.
+
+    As in ADK's `Event.is_final_response()`, events with function calls or
+    responses are not final responses, so they stay in the intermediate events.
+    Thought parts are not part of the response text.
+    """
+    for index in range(len(events) - 1, -1, -1):
+        parts = (events[index].get("content") or {}).get("parts") or []
+        if any(
+            part.get("function_call") or part.get("function_response") for part in parts
+        ):
+            continue
+        texts = [
+            part["text"]
+            for part in parts
+            if part.get("text") and not part.get("thought")
+        ]
+        if texts:
+            return index, "".join(texts)
+    return None, ""
+
+
 def _process_single_turn_agent_response(
     resp_item: Any,
     agent_data_agents: Optional[dict[str, Any]],
@@ -3253,8 +3278,10 @@ def _process_single_turn_agent_response(
 
     if isinstance(resp_item, list):
         try:
-            response_row = resp_item[-1]["content"]["parts"][0]["text"]
-            for intermediate_event in resp_item[:-1]:
+            response_index, response_row = _extract_final_response_text(resp_item)
+            for index, intermediate_event in enumerate(resp_item):
+                if index == response_index:
+                    continue
                 intermediate_events_row.append(
                     {
                         "event_id": intermediate_event.get("id"),
@@ -3263,7 +3290,7 @@ def _process_single_turn_agent_response(
                         "author": intermediate_event.get("author"),
                     }
                 )
-            # Construct AgentData natively for single-turn runs
+                # Construct AgentData natively for single-turn runs
             agent_events = []
             for event_dict in resp_item:
                 content_dict = event_dict.get("content")
@@ -3288,6 +3315,7 @@ def _process_single_turn_agent_response(
                 agents=agent_data_agents,
             ).model_dump(exclude_unset=True)
         except Exception as e:  # pylint: disable=broad-exception-caught
+            intermediate_events_row = []
             error_payload = {
                 "error": (
                     f"Failed to parse agent run response {str(resp_item)} to "

{google_cloud_agentplatform-2.7.0 → google_cloud_agentplatform-2.8.0}/agentplatform/_genai/_evals_metric_handlers.py

@@ -846,6 +846,66 @@ class LLMMetricHandler(MetricHandler[types.LLMMetric]):
             return _default_aggregate_scores(self.metric.name, eval_case_metric_results)
 
 
+class StepBasedLLMMetricHandler(LLMMetricHandler):
+    """Metric handler for step-based LLM metrics."""
+
+    @override
+    def get_metric_result(
+        self, eval_case: types.EvalCase, response_index: int
+    ) -> types.EvalCaseMetricResult:
+        """Evaluates one case with the step-based LLM metric."""
+        try:
+            response_content = _get_response_from_eval_case(
+                eval_case, response_index, self.metric_name
+            )
+            if not response_content and not getattr(eval_case, "agent_data", None):
+                raise ValueError(
+                    f"Response content missing for candidate {response_index}."
+                )
+
+            # Joined so dataset columns referenced by any step reach other_data.
+            prompt_templates = "\n".join(
+                step.prompt_template or "" for step in self.metric.steps or []
+            )
+            instance = _build_evaluation_instance(
+                eval_case, response_content, prompt_template=prompt_templates
+            )
+            api_response = _call_with_retry(
+                lambda: self.module._evaluate_instances(
+                    metrics=[self.metric],
+                    instance=instance,
+                    allow_cross_region_model=self.allow_cross_region_model,
+                ),
+                self.metric_name,
+            )
+
+            if api_response and api_response.metric_results:
+                result = api_response.metric_results[0]
+                error_msg = None
+                if result.error and getattr(result.error, "code"):
+                    error_msg = f"Error in metric result: {result.error}"
+                return types.EvalCaseMetricResult(
+                    metric_name=self.metric_name,
+                    score=result.score,
+                    explanation=result.explanation,
+                    error_message=error_msg,
+                )
+            return types.EvalCaseMetricResult(
+                metric_name=self.metric_name,
+                error_message="Metric results missing in API response.",
+            )
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.error(
+                "Error processing metric %s for case %s.",
+                self.metric_name,
+                eval_case.eval_case_id,
+                exc_info=True,
+            )
+            return types.EvalCaseMetricResult(
+                metric_name=self.metric_name, error_message=str(e)
+            )
+
+
 class CustomMetricHandler(MetricHandler[types.Metric]):
     """Metric handler for custom metrics."""
 
@@ -1375,6 +1435,7 @@ _METRIC_HANDLER_MAPPING = [
         lambda m: getattr(m, "metric_resource_name", None) is not None,
         RegisteredMetricHandler,
     ),
+    (lambda m: getattr(m, "steps", None), StepBasedLLMMetricHandler),
     (
         lambda m: m.name in ComputationMetricHandler.SUPPORTED_COMPUTATION_METRICS,
         ComputationMetricHandler,
@@ -1395,6 +1456,7 @@ MetricHandlerType = TypeVar(
     ComputationMetricHandler,
     TranslationMetricHandler,
     LLMMetricHandler,
+    StepBasedLLMMetricHandler,
     CustomMetricHandler,
     CustomCodeExecutionMetricHandler,
     PredefinedMetricHandler,

{google_cloud_agentplatform-2.7.0 → google_cloud_agentplatform-2.8.0}/agentplatform/_genai/_transformers.py

@@ -89,6 +89,11 @@ def t_metrics(
             metric_payload_item["custom_code_execution_spec"] = {
                 "evaluation_function": metric.custom_function
             }
+        # Step-based LLM metrics
+        elif getv(metric, ["steps"]):
+            metric_payload_item["step_based_llm_metric_spec"] = (
+                _step_based_llm_metric_spec(metric)
+            )
         # LLM-based metrics
         elif hasattr(metric, "prompt_template") and metric.prompt_template:
             llm_based_spec: dict[str, Any] = {
@@ -145,6 +150,47 @@ def t_metrics(
     return metrics_payload
 
 
+def _dump(value: Any) -> Any:
+    if hasattr(value, "model_dump"):
+        return value.model_dump(exclude_none=True)
+    return value
+
+
+def _step_based_llm_metric_spec(metric: Any) -> dict[str, Any]:
+    """Builds a StepBasedLlmMetricSpec payload.
+
+    autorater_config fields a step leaves unset fall back to the metric's judge
+    model settings.
+    """
+    default_autorater_config: dict[str, Any] = {}
+    if getv(metric, ["judge_model"]):
+        default_autorater_config["autorater_model"] = getv(metric, ["judge_model"])
+    if getv(metric, ["judge_model_generation_config"]):
+        default_autorater_config["generation_config"] = _dump(
+            getv(metric, ["judge_model_generation_config"])
+        )
+    if getv(metric, ["judge_model_sampling_count"]):
+        default_autorater_config["sampling_count"] = getv(
+            metric, ["judge_model_sampling_count"]
+        )
+
+    steps = []
+    for step in getv(metric, ["steps"]):
+        step_payload = _dump(step)
+        if default_autorater_config:
+            step_payload["autorater_config"] = {
+                **default_autorater_config,
+                **step_payload.get("autorater_config", {}),
+            }
+        steps.append(step_payload)
+
+    spec: dict[str, Any] = {"steps": steps}
+    preprocessor = getv(metric, ["preprocessor"])
+    if preprocessor:
+        spec["preprocessor"] = _dump(preprocessor)
+    return spec
+
+
 def t_metric_sources(metrics: list[Any]) -> list[dict[str, Any]]:
     """Prepares the MetricSource payload."""
     sources_payload = []
@@ -217,6 +263,12 @@ def t_metric_for_registry(
             "evaluation_function": metric.custom_function
         }
 
+    # Step-based LLM metric
+    elif getv(metric, ["steps"]):
+        metric_payload_item["step_based_llm_metric_spec"] = _step_based_llm_metric_spec(
+            metric
+        )
+
     # LLM-based metric
     elif (hasattr(metric, "prompt_template") and metric.prompt_template) or (
         hasattr(metric, "rubric_group_name") and metric.rubric_group_name

{google_cloud_agentplatform-2.7.0 → google_cloud_agentplatform-2.8.0}/agentplatform/_genai/evals.py

@@ -1172,6 +1172,13 @@ def _UnifiedMetric_from_vertex(
             getv(from_object, ["computationBasedMetricSpec"]),
         )
 
+    if getv(from_object, ["stepBasedLlmMetricSpec"]) is not None:
+        setv(
+            to_object,
+            ["step_based_llm_metric_spec"],
+            getv(from_object, ["stepBasedLlmMetricSpec"]),
+        )
+
     return to_object
 
 
@@ -1223,6 +1230,13 @@ def _UnifiedMetric_to_vertex(
             getv(from_object, ["computation_based_metric_spec"]),
         )
 
+    if getv(from_object, ["step_based_llm_metric_spec"]) is not None:
+        setv(
+            to_object,
+            ["stepBasedLlmMetricSpec"],
+            getv(from_object, ["step_based_llm_metric_spec"]),
+        )
+
     return to_object
 
 

{google_cloud_agentplatform-2.7.0 → google_cloud_agentplatform-2.8.0}/agentplatform/_genai/runtimes.py

@@ -1607,6 +1607,7 @@ class Runtimes(_api_module.BaseModule):
             agent
             or config.source_packages
             or config.developer_connect_source
+            or config.git_source
             or config.agent_config_source
             or config.container_spec
         ):
@@ -1860,6 +1861,9 @@ class Runtimes(_api_module.BaseModule):
             developer_connect_source = json.loads(
                 developer_connect_source.model_dump_json()
             )
+        git_source = config.git_source
+        if git_source is not None:
+            git_source = json.loads(git_source.model_dump_json(exclude_none=True))
         agent_config_source = config.agent_config_source
         if agent_config_source is not None:
             agent_config_source = json.loads(agent_config_source.model_dump_json())
@@ -1900,6 +1904,7 @@ class Runtimes(_api_module.BaseModule):
             class_methods=config.class_methods,
             source_packages=config.source_packages,
             developer_connect_source=developer_connect_source,
+            git_source=git_source,
             entrypoint_module=config.entrypoint_module,
             entrypoint_object=config.entrypoint_object,
             requirements_file=config.requirements_file,
@@ -1969,6 +1974,9 @@ class Runtimes(_api_module.BaseModule):
         developer_connect_source: Optional[
             types.ReasoningEngineSpecSourceCodeSpecDeveloperConnectConfigDict
         ] = None,
+        git_source: Optional[
+            types.ReasoningEngineSpecSourceCodeSpecGitSourceDict
+        ] = None,
         class_methods: Optional[Sequence[dict[str, Any]]] = None,
         entrypoint_module: Optional[str] = None,
         entrypoint_object: Optional[str] = None,
@@ -2000,10 +2008,13 @@ class Runtimes(_api_module.BaseModule):
             source_code_spec["developer_connect_source"] = {
                 "config": developer_connect_source
             }
+        elif git_source:
+            update_masks.append("spec.source_code_spec.git_source")
+            source_code_spec["git_source"] = git_source
         elif not agent_config_source:
             raise ValueError(
                 "Please specify one of `source_packages`, `developer_connect_source`, "
-                "or `agent_config_source`."
+                "`git_source`, or `agent_config_source`."
             )
         if class_methods is not None:
             update_masks.append("spec.class_methods")
@@ -2019,8 +2030,8 @@ class Runtimes(_api_module.BaseModule):
         elif image_spec is None:
             raise ValueError(
                 "`class_methods` must be specified if `source_packages`, "
-                "`developer_connect_source`, or `agent_config_source` is "
-                "specified without a Dockerfile or `image_spec`."
+                "`developer_connect_source`, `git_source`, or `agent_config_source` "
+                "is specified without a Dockerfile or `image_spec`."
             )
         if image_spec is not None:
             if entrypoint_module or entrypoint_object or requirements_file:
@@ -2077,13 +2088,13 @@ class Runtimes(_api_module.BaseModule):
 
         if not entrypoint_module:
             raise ValueError(
-                "`entrypoint_module` must be specified if `source_packages` or `developer_connect_source` is specified."
+                "`entrypoint_module` must be specified if `source_packages`, `developer_connect_source`, or `git_source` is specified."
             )
         update_masks.append("spec.source_code_spec.python_spec.entrypoint_module")
         python_spec["entrypoint_module"] = entrypoint_module
         if not entrypoint_object:
             raise ValueError(
-                "`entrypoint_object` must be specified if `source_packages` or `developer_connect_source` is specified."
+                "`entrypoint_object` must be specified if `source_packages`, `developer_connect_source`, or `git_source` is specified."
             )
         update_masks.append("spec.source_code_spec.python_spec.entrypoint_object")
         python_spec["entrypoint_object"] = entrypoint_object
@@ -2215,6 +2226,9 @@ class Runtimes(_api_module.BaseModule):
         developer_connect_source: Optional[
             types.ReasoningEngineSpecSourceCodeSpecDeveloperConnectConfigDict
         ] = None,
+        git_source: Optional[
+            types.ReasoningEngineSpecSourceCodeSpecGitSourceDict
+        ] = None,
         entrypoint_module: Optional[str] = None,
         entrypoint_object: Optional[str] = None,
         requirements_file: Optional[str] = None,
@@ -2284,9 +2298,18 @@ class Runtimes(_api_module.BaseModule):
                     "do not specify `agent` in `runtimes.create()` or "
                     "`runtimes.update()`."
                 )
-        elif source_packages and developer_connect_source:
+            if git_source:
+                raise ValueError(
+                    "If you have provided `git_source` in `config`, please "
+                    "do not specify `agent` in `runtimes.create()` or "
+                    "`runtimes.update()`."
+                )
+        elif (
+            sum(map(bool, (source_packages, developer_connect_source, git_source))) > 1
+        ):
             raise ValueError(
-                "Please specify only one of `source_packages` or `developer_connect_source` in `config`."
+                "Please specify only one of `source_packages`, "
+                "`developer_connect_source`, or `git_source` in `config`."
             )
 
         if container_spec:
@@ -2296,10 +2319,11 @@ class Runtimes(_api_module.BaseModule):
                     "do not specify `agent` in `runtimes.create()` or "
                     "`runtimes.update()`."
                 )
-            if source_packages or developer_connect_source:
+            if source_packages or developer_connect_source or git_source:
                 raise ValueError(
                     "If you have provided `container_spec` in `config`, please "
-                    "do not specify `source_packages` or `developer_connect_source` in `config`."
+                    "do not specify `source_packages`, `developer_connect_source`, or "
+                    "`git_source` in `config`."
                 )
 
         runtime_spec: Any = None
@@ -2323,6 +2347,7 @@ class Runtimes(_api_module.BaseModule):
         elif (
             source_packages
             or developer_connect_source
+            or git_source
             or image_spec
             or agent_config_source
         ):
@@ -2332,6 +2357,7 @@ class Runtimes(_api_module.BaseModule):
                 update_masks=update_masks,
                 source_packages=source_packages,
                 developer_connect_source=developer_connect_source,
+                git_source=git_source,
                 class_methods=class_methods,
                 entrypoint_module=entrypoint_module,
                 entrypoint_object=entrypoint_object,
@@ -2373,7 +2399,7 @@ class Runtimes(_api_module.BaseModule):
                 "`max_instances`, `resource_limits`, `container_concurrency`, or "
                 "`keep_alive_probe`, you must also provide the `agent` variable or "
                 "the source code options (`source_packages`, "
-                "`developer_connect_source` or `agent_config_source`)."
+                "`developer_connect_source`, `git_source` or `agent_config_source`)."
             )
 
         if runtime_spec is not None:
@@ -2672,6 +2698,9 @@ class Runtimes(_api_module.BaseModule):
             developer_connect_source = json.loads(
                 developer_connect_source.model_dump_json()
             )
+        git_source = config.git_source
+        if git_source is not None:
+            git_source = json.loads(git_source.model_dump_json(exclude_none=True))
         agent_config_source = config.agent_config_source
         if agent_config_source is not None:
             agent_config_source = json.loads(agent_config_source.model_dump_json())
@@ -2721,6 +2750,7 @@ class Runtimes(_api_module.BaseModule):
             class_methods=config.class_methods,
             source_packages=config.source_packages,
             developer_connect_source=developer_connect_source,
+            git_source=git_source,
             entrypoint_module=config.entrypoint_module,
             entrypoint_object=config.entrypoint_object,
             requirements_file=config.requirements_file,

{google_cloud_agentplatform-2.7.0 → google_cloud_agentplatform-2.8.0}/agentplatform/_genai/types/__init__.py

@@ -199,6 +199,9 @@ from .common import _UpdateSkillRequestParameters
 from .common import _UploadRagFileParameters
 from .common import _UpsertExamplesParameters
 from .common import AcceleratorType
+from .common import AgentDataPreprocessor
+from .common import AgentDataPreprocessorDict
+from .common import AgentDataPreprocessorOrDict
 from .common import AgentRunConfig
 from .common import AgentRunConfigDict
 from .common import AgentRunConfigOrDict
@@ -312,6 +315,9 @@ from .common import ClientConnectionConfig
 from .common import ClientConnectionConfigDict
 from .common import ClientConnectionConfigOrDict
 from .common import CodeExecutionMetric
+from .common import CodePreprocessorConfig
+from .common import CodePreprocessorConfigDict
+from .common import CodePreprocessorConfigOrDict
 from .common import ColorMap
 from .common import CometResult
 from .common import CometResultDict
@@ -409,6 +415,9 @@ from .common import CreateSkillConfigOrDict
 from .common import CustomCodeExecutionSpec
 from .common import CustomCodeExecutionSpecDict
 from .common import CustomCodeExecutionSpecOrDict
+from .common import CustomCodeParserConfig
+from .common import CustomCodeParserConfigDict
+from .common import CustomCodeParserConfigOrDict
 from .common import CustomJob
 from .common import CustomJobDict
 from .common import CustomJobOrDict
@@ -806,6 +815,9 @@ from .common import FetchExamplesConfigOrDict
 from .common import FetchExamplesResponse
 from .common import FetchExamplesResponseDict
 from .common import FetchExamplesResponseOrDict
+from .common import FilterPreprocessorConfig
+from .common import FilterPreprocessorConfigDict
+from .common import FilterPreprocessorConfigOrDict
 from .common import FlexStart
 from .common import FlexStartDict
 from .common import FlexStartOrDict
@@ -1795,6 +1807,9 @@ from .common import ReasoningEngineSpecSourceCodeSpecDeveloperConnectSource
 from .common import ReasoningEngineSpecSourceCodeSpecDeveloperConnectSourceDict
 from .common import ReasoningEngineSpecSourceCodeSpecDeveloperConnectSourceOrDict
 from .common import ReasoningEngineSpecSourceCodeSpecDict
+from .common import ReasoningEngineSpecSourceCodeSpecGitSource
+from .common import ReasoningEngineSpecSourceCodeSpecGitSourceDict
+from .common import ReasoningEngineSpecSourceCodeSpecGitSourceOrDict
 from .common import ReasoningEngineSpecSourceCodeSpecImageSpec
 from .common import ReasoningEngineSpecSourceCodeSpecImageSpecDict
 from .common import ReasoningEngineSpecSourceCodeSpecImageSpecOrDict
@@ -1851,6 +1866,9 @@ from .common import ResponseCandidateOrDict
 from .common import ResponseCandidateResult
 from .common import ResponseCandidateResultDict
 from .common import ResponseCandidateResultOrDict
+from .common import ResponseParserConfig
+from .common import ResponseParserConfigDict
+from .common import ResponseParserConfigOrDict
 from .common import RestoreVersionConfig
 from .common import RestoreVersionConfigDict
 from .common import RestoreVersionConfigOrDict
@@ -2075,6 +2093,9 @@ from .common import SchemaPromptSpecPartListOrDict
 from .common import SchemaPromptSpecPromptMessage
 from .common import SchemaPromptSpecPromptMessageDict
 from .common import SchemaPromptSpecPromptMessageOrDict
+from .common import SchemaPromptSpecPromptMessageOutputConfig
+from .common import SchemaPromptSpecPromptMessageOutputConfigDict
+from .common import SchemaPromptSpecPromptMessageOutputConfigOrDict
 from .common import SchemaPromptSpecReferenceSentencePair
 from .common import SchemaPromptSpecReferenceSentencePairDict
 from .common import SchemaPromptSpecReferenceSentencePairList
@@ -2185,6 +2206,13 @@ from .common import SpeculativeDecodingSpecNgramSpeculationDict
 from .common import SpeculativeDecodingSpecNgramSpeculationOrDict
 from .common import SpeculativeDecodingSpecOrDict
 from .common import State
+from .common import StepBasedLLMMetric
+from .common import StepBasedLLMMetricSpec
+from .common import StepBasedLLMMetricSpecDict
+from .common import StepBasedLLMMetricSpecOrDict
+from .common import StepBasedLLMMetricStep
+from .common import StepBasedLLMMetricStepDict
+from .common import StepBasedLLMMetricStepOrDict
 from .common import StoredContentsExample
 from .common import StoredContentsExampleDict
 from .common import StoredContentsExampleFilter
@@ -2273,6 +2301,9 @@ from .common import ToolParameterKVMatchResultsOrDict
 from .common import ToolParameterKVMatchSpec
 from .common import ToolParameterKVMatchSpecDict
 from .common import ToolParameterKVMatchSpecOrDict
+from .common import TraceCompressionConfig
+from .common import TraceCompressionConfigDict
+from .common import TraceCompressionConfigOrDict
 from .common import TuningResourceUsageAssessmentConfig
 from .common import TuningResourceUsageAssessmentConfigDict
 from .common import TuningResourceUsageAssessmentConfigOrDict
@@ -2406,6 +2437,27 @@ __all__ = [
     "EvaluationItem",
     "EvaluationItemDict",
     "EvaluationItemOrDict",
+    "TraceCompressionConfig",
+    "TraceCompressionConfigDict",
+    "TraceCompressionConfigOrDict",
+    "FilterPreprocessorConfig",
+    "FilterPreprocessorConfigDict",
+    "FilterPreprocessorConfigOrDict",
+    "CodePreprocessorConfig",
+    "CodePreprocessorConfigDict",
+    "CodePreprocessorConfigOrDict",
+    "AgentDataPreprocessor",
+    "AgentDataPreprocessorDict",
+    "AgentDataPreprocessorOrDict",
+    "CustomCodeParserConfig",
+    "CustomCodeParserConfigDict",
+    "CustomCodeParserConfigOrDict",
+    "ResponseParserConfig",
+    "ResponseParserConfigDict",
+    "ResponseParserConfigOrDict",
+    "StepBasedLLMMetricStep",
+    "StepBasedLLMMetricStepDict",
+    "StepBasedLLMMetricStepOrDict",
     "Metric",
     "MetricDict",
     "MetricOrDict",
@@ -2415,6 +2467,9 @@ __all__ = [
     "CustomCodeExecutionSpec",
     "CustomCodeExecutionSpecDict",
     "CustomCodeExecutionSpecOrDict",
+    "StepBasedLLMMetricSpec",
+    "StepBasedLLMMetricSpecDict",
+    "StepBasedLLMMetricSpecOrDict",
     "UnifiedMetric",
     "UnifiedMetricDict",
     "UnifiedMetricOrDict",
@@ -2946,6 +3001,9 @@ __all__ = [
     "ReasoningEngineSpecSourceCodeSpecPythonSpec",
     "ReasoningEngineSpecSourceCodeSpecPythonSpecDict",
     "ReasoningEngineSpecSourceCodeSpecPythonSpecOrDict",
+    "ReasoningEngineSpecSourceCodeSpecGitSource",
+    "ReasoningEngineSpecSourceCodeSpecGitSourceDict",
+    "ReasoningEngineSpecSourceCodeSpecGitSourceOrDict",
     "ReasoningEngineSpecSourceCodeSpec",
     "ReasoningEngineSpecSourceCodeSpecDict",
     "ReasoningEngineSpecSourceCodeSpecOrDict",
@@ -3723,6 +3781,9 @@ __all__ = [
     "SchemaPromptInstancePromptExecution",
     "SchemaPromptInstancePromptExecutionDict",
     "SchemaPromptInstancePromptExecutionOrDict",
+    "SchemaPromptSpecPromptMessageOutputConfig",
+    "SchemaPromptSpecPromptMessageOutputConfigDict",
+    "SchemaPromptSpecPromptMessageOutputConfigOrDict",
     "SchemaPromptSpecPromptMessage",
     "SchemaPromptSpecPromptMessageDict",
     "SchemaPromptSpecPromptMessageOrDict",
@@ -4504,6 +4565,7 @@ __all__ = [
     "PromptDataOrDict",
     "LLMMetric",
     "CodeExecutionMetric",
+    "StepBasedLLMMetric",
     "MetricPromptBuilder",
     "RubricContentProperty",
     "RubricContentPropertyDict",