arize 8.0.0b1__py3-none-any.whl → 8.0.0b4__py3-none-any.whl

arize/exceptions/parameters.py

@@ -3,159 +3,6 @@
 from arize.constants.ml import MAX_NUMBER_OF_EMBEDDINGS
 from arize.exceptions.base import ValidationError
 
-# class MissingPredictionIdColumnForDelayedRecords(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Prediction_Id_Column_For_Delayed_Records"
-#
-#     def __init__(self, has_actual_info, has_feature_importance_info) -> None:
-#         self.has_actual_info = has_actual_info
-#         self.has_feature_importance_info = has_feature_importance_info
-#
-#     def error_message(self) -> str:
-#         actual = "actual" if self.has_actual_info else ""
-#         feat_imp = (
-#             "feature importance" if self.has_feature_importance_info else ""
-#         )
-#         if self.has_actual_info and self.has_feature_importance_info:
-#             msg = " and ".join([actual, feat_imp])
-#         else:
-#             msg = "".join([actual, feat_imp])
-#
-#         return (
-#             "Missing 'prediction_id_column_name'. While prediction id is optional for most cases, "
-#             "it is required when sending delayed actuals, i.e. when sending actual or feature importances "
-#             f"without predictions. In this case, {msg} information was found (without predictions). "
-#             "To learn more about delayed joins, please see the docs at "
-#             "https://docs.arize.com/arize/sending-data-guides/how-to-send-delayed-actuals"
-#         )
-
-
-# class MissingColumns(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Columns"
-#
-#     def __init__(self, cols: Iterable) -> None:
-#         self.missing_cols = set(cols)
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The following columns are declared in the schema "
-#             "but are not found in the dataframe: "
-#             f"{', '.join(map(str, self.missing_cols))}."
-#         )
-
-
-# class MissingRequiredColumnsMetricsValidation(ValidationError):
-#     """
-#     This error is used only for model mapping validations.
-#     """
-#
-#     def __repr__(self) -> str:
-#         return "Missing_Columns_Required_By_Metrics_Validation"
-#
-#     def __init__(
-#         self, model_type: ModelTypes, metrics: List[Metrics], cols: Iterable
-#     ) -> None:
-#         self.model_type = model_type
-#         self.metrics = metrics
-#         self.missing_cols = cols
-#
-#     def error_message(self) -> str:
-#         return (
-#             f"For logging data for a {self.model_type.name} model with support for metrics "
-#             f"{', '.join(m.name for m in self.metrics)}, "
-#             f"schema must include: {', '.join(map(str, self.missing_cols))}."
-#         )
-
-
-# class ReservedColumns(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Reserved_Columns"
-#
-#     def __init__(self, cols: Iterable) -> None:
-#         self.reserved_columns = cols
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The following columns are reserved and can only be specified "
-#             "in the proper fields of the schema: "
-#             f"{', '.join(map(str, self.reserved_columns))}."
-#         )
-
-
-# class InvalidModelTypeAndMetricsCombination(ValidationError):
-#     """
-#     This error is used only for model mapping validations.
-#     """
-#
-#     def __repr__(self) -> str:
-#         return "Invalid_ModelType_And_Metrics_Combination"
-#
-#     def __init__(
-#         self,
-#         model_type: ModelTypes,
-#         metrics: List[Metrics],
-#         suggested_model_metric_combinations: List[List[str]],
-#     ) -> None:
-#         self.model_type = model_type
-#         self.metrics = metrics
-#         self.suggested_combinations = suggested_model_metric_combinations
-#
-#     def error_message(self) -> str:
-#         valid_combos = ", or \n".join(
-#             "[" + ", ".join(combo) + "]"
-#             for combo in self.suggested_combinations
-#         )
-#         return (
-#             f"Invalid combination of model type {self.model_type.name} and metrics: "
-#             f"{', '.join(m.name for m in self.metrics)}. "
-#             f"Valid Metric combinations for this model type:\n{valid_combos}.\n"
-#         )
-
-
-# class InvalidShapSuffix(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_SHAP_Suffix"
-#
-#     def __init__(self, cols: Iterable) -> None:
-#         self.invalid_column_names = cols
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The following features or tags must not be named with a `_shap` suffix: "
-#             f"{', '.join(map(str, self.invalid_column_names))}."
-#         )
-
-
-# class InvalidModelType(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Model_Type"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "Model type not valid. Choose one of the following: "
-#             f"{', '.join('ModelTypes.' + mt.name for mt in ModelTypes)}. "
-#         )
-
-
-# class InvalidEnvironment(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Environment"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "Environment not valid. Choose one of the following: "
-#             f"{', '.join('Environments.' + env.name for env in Environments)}. "
-#         )
-
-
-# class InvalidBatchId(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Batch_ID"
-#
-#     def error_message(self) -> str:
-#         return "Batch ID must be a nonempty string if logging to validation environment."
-
 
 class InvalidModelVersion(ValidationError):
     """Raised when model version is empty or invalid."""
@@ -169,14 +16,6 @@ class InvalidModelVersion(ValidationError):
         return "Model version must be a nonempty string."
 
 
-# class InvalidModelId(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Model_ID"
-#
-#     def error_message(self) -> str:
-#         return "Model ID must be a nonempty string."
-
-
 class InvalidProjectName(ValidationError):
     """Raised when project name is empty or invalid."""
 
@@ -193,174 +32,6 @@ class InvalidProjectName(ValidationError):
         )
 
 
-# class MissingPredActShap(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Pred_or_Act_or_SHAP"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The schema must specify at least one of the following: "
-#             "prediction label, actual label, or SHAP value column names"
-#         )
-
-
-# class MissingPreprodPredAct(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Preproduction_Pred_and_Act"
-#
-#     def error_message(self) -> str:
-#         return "For logging pre-production data, the schema must specify both "
-#         "prediction and actual label columns."
-
-
-# class MissingPreprodAct(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Preproduction_Act"
-#
-#     def error_message(self) -> str:
-#         return "For logging pre-production data, the schema must specify actual label column."
-
-
-# class MissingPreprodPredActNumericAndCategorical(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Preproduction_Pred_and_Act_Numeric_and_Categorical"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "For logging pre-production data for a numeric or a categorical model, "
-#             "the schema must specify both prediction and actual label or score columns."
-#         )
-
-
-# class MissingRequiredColumnsForRankingModel(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Required_Columns_For_Ranking_Model"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "For logging data for a ranking model, schema must specify: "
-#             "prediction_group_id_column_name and rank_column_name"
-#         )
-
-
-# class MissingCVPredAct(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_CV_Prediction_or_Actual"
-#
-#     def __init__(self, environment: Environments):
-#         self.environment = environment
-#
-#     def error_message(self) -> str:
-#         if self.environment in (Environments.TRAINING, Environments.VALIDATION):
-#             env = "pre-production"
-#             opt = "and"
-#         elif self.environment == Environments.PRODUCTION:
-#             env = "production"
-#             opt = "or"
-#         else:
-#             raise TypeError("Invalid environment")
-#         return (
-#             f"For logging {env} data for an Object Detection model,"
-#             "the schema must specify one of: "
-#             f"('object_detection_prediction_column_names' {opt} "
-#             f"'object_detection_actual_column_names') "
-#             f"or ('semantic_segmentation_prediction_column_names' {opt} "
-#             f"'semantic_segmentation_actual_column_names') "
-#             f"or ('instance_segmentation_prediction_column_names' {opt} "
-#             f"'instance_segmentation_actual_column_names')"
-#         )
-
-
-# class MultipleCVPredAct(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Multiple_CV_Prediction_or_Actual"
-#
-#     def __init__(self, environment: Environments):
-#         self.environment = environment
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The schema must only specify one of the following: "
-#             "'object_detection_prediction_column_names'/'object_detection_actual_column_names'"
-#             "'semantic_segmentation_prediction_column_names'/'semantic_segmentation_actual_column_names'"
-#             "'instance_segmentation_prediction_column_names'/'instance_segmentation_actual_column_names'"
-#         )
-
-
-# class InvalidPredActCVColumnNamesForModelType(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_CV_Prediction_or_Actual_Column_Names_for_Model_Type"
-#
-#     def __init__(
-#         self,
-#         invalid_model_type: ModelTypes,
-#     ) -> None:
-#         self.invalid_model_type = invalid_model_type
-#
-#     def error_message(self) -> str:
-#         return (
-#             f"Cannot use 'object_detection_prediction_column_names' or "
-#             f"'object_detection_actual_column_names' or "
-#             f"'semantic_segmentation_prediction_column_names' or "
-#             f"'semantic_segmentation_actual_column_names' or "
-#             f"'instance_segmentation_prediction_column_names' or "
-#             f"'instance_segmentation_actual_column_names' for {self.invalid_model_type} model "
-#             f"type. They are only allowed for ModelTypes.OBJECT_DETECTION models"
-#         )
-
-
-# class MissingReqPredActColumnNamesForMultiClass(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Required_Prediction_or_Actual_Column_Names_for_Multi_Class_Model_Type"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "For logging data for a multi class model, schema must specify: "
-#             "prediction_scores_column_name and/or actual_score_column_name. "
-#             "Optionally, you may include multi_class_threshold_scores_column_name"
-#             " (must include prediction_scores_column_name)"
-#         )
-
-
-# class InvalidPredActColumnNamesForModelType(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Prediction_or_Actual_Column_Names_for_Model_Type"
-#
-#     def __init__(
-#         self,
-#         invalid_model_type: ModelTypes,
-#         allowed_fields: List[str],
-#         wrong_columns: List[str],
-#     ) -> None:
-#         self.invalid_model_type = invalid_model_type
-#         self.allowed_fields = allowed_fields
-#         self.wrong_columns = wrong_columns
-#
-#     def error_message(self) -> str:
-#         allowed_col_msg = ""
-#         if self.allowed_fields is not None:
-#             allowed_col_msg = f" Allowed Schema fields are {log_a_list(self.allowed_fields, 'and')}"
-#         return (
-#             f"Invalid Schema fields for {self.invalid_model_type} model type. {allowed_col_msg}"
-#             "The following columns of your dataframe are sent as an invalid schema field: "
-#             f"{log_a_list(self.wrong_columns, 'and')}"
-#         )
-
-
-# class DuplicateColumnsInDataframe(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Duplicate_Columns_In_Dataframe"
-#
-#     def __init__(self, cols: Iterable) -> None:
-#         self.duplicate_cols = cols
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The following columns are present in the schema and have duplicates in the dataframe: "
-#             f"{self.duplicate_cols}. "
-#         )
-
-
 class InvalidNumberOfEmbeddings(ValidationError):
     """Raised when number of embedding features exceeds the maximum allowed."""
 
@@ -390,7 +61,7 @@ class InvalidValueType(Exception):
     def __init__(
         self,
         value_name: str,
-        value: bool | int | float | str,
+        value: object,
         correct_type: str,
     ) -> None:
         """Initialize the exception with value type validation context.

arize/exceptions/values.py

@@ -533,14 +533,15 @@ class InvalidMultiClassClassNameLength(ValidationError):
         err_msg = ""
         for col, class_names in self.invalid_col_class_name.items():
             # limit to 10
-            class_names = (
+            class_names_list = (
                 list(class_names)[:10]
                 if len(class_names) > 10
                 else list(class_names)
             )
             err_msg += (
-                f"Found some invalid class names: {log_a_list(class_names, 'and')} in the {col} column. Class"
-                f" names must have at least one character and less than {MAX_MULTI_CLASS_NAME_LENGTH}.\n"
+                f"Found some invalid class names: {log_a_list(class_names_list, 'and')} "
+                f"in the {col} column. Class names must have at least one character and "
+                f"less than {MAX_MULTI_CLASS_NAME_LENGTH}.\n"
             )
         return err_msg
 
@@ -565,9 +566,11 @@ class InvalidMultiClassPredScoreValue(ValidationError):
         err_msg = ""
         for col, scores in self.invalid_col_class_scores.items():
             # limit to 10
-            scores = list(scores)[:10] if len(scores) > 10 else list(scores)
+            scores_list = (
+                list(scores)[:10] if len(scores) > 10 else list(scores)
+            )
             err_msg += (
-                f"Found some invalid scores: {log_a_list(scores, 'and')} in the {col} column that was "
+                f"Found some invalid scores: {log_a_list(scores_list, 'and')} in the {col} column that was "
                 "invalid. All scores (values in dictionary) must be between 0 and 1, inclusive. \n"
             )
         return err_msg

arize/experiments/__init__.py

@@ -1,5 +1,8 @@
 """Experiment tracking and evaluation functionality for the Arize SDK."""
 
+from arize.experiments.evaluators.base import (
+    Evaluator,
+)
 from arize.experiments.evaluators.types import (
     EvaluationResult,
     EvaluationResultFieldNames,
@@ -9,5 +12,6 @@ from arize.experiments.types import ExperimentTaskFieldNames
 __all__ = [
     "EvaluationResult",
     "EvaluationResultFieldNames",
+    "Evaluator",
     "ExperimentTaskFieldNames",
 ]

arize/experiments/client.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 
 import opentelemetry.sdk.trace as trace_sdk
 import pandas as pd
@@ -36,8 +36,13 @@ from arize.utils.openinference_conversion import (
 from arize.utils.size import get_payload_size_mb
 
 if TYPE_CHECKING:
+    # builtins is needed to use builtins.list in type annotations because
+    # the class has a list() method that shadows the built-in list type
+    import builtins
+
     from opentelemetry.trace import Tracer
 
+    from arize._generated.api_client.api_client import ApiClient
     from arize.config import SDKConfiguration
     from arize.experiments.evaluators.base import Evaluators
     from arize.experiments.evaluators.types import EvaluationResultFieldNames
@@ -61,20 +66,22 @@ class ExperimentsClient:
     :class:`arize.config.SDKConfiguration`.
     """
 
-    def __init__(self, *, sdk_config: SDKConfiguration) -> None:
+    def __init__(
+        self, *, sdk_config: SDKConfiguration, generated_client: ApiClient
+    ) -> None:
         """
         Args:
             sdk_config: Resolved SDK configuration.
+            generated_client: Shared generated API client instance.
         """  # noqa: D205, D212
         self._sdk_config = sdk_config
         from arize._generated import api_client as gen
 
-        self._api = gen.ExperimentsApi(self._sdk_config.get_generated_client())
+        # Use the provided client directly for both APIs
+        self._api = gen.ExperimentsApi(generated_client)
         # TODO(Kiko): Space ID should not be needed,
         # should work on server tech debt to remove this
-        self._datasets_api = gen.DatasetsApi(
-            self._sdk_config.get_generated_client()
-        )
+        self._datasets_api = gen.DatasetsApi(generated_client)
 
     @prerelease_endpoint(key="experiments.list", stage=ReleaseStage.BETA)
     def list(
@@ -113,7 +120,7 @@ class ExperimentsClient:
         *,
         name: str,
         dataset_id: str,
-        experiment_runs: list[dict[str, object]] | pd.DataFrame,
+        experiment_runs: builtins.list[dict[str, object]] | pd.DataFrame,
         task_fields: ExperimentTaskFieldNames,
         evaluator_columns: dict[str, EvaluationResultFieldNames] | None = None,
         force_http: bool = False,
@@ -141,7 +148,7 @@ class ExperimentsClient:
             dataset_id: Dataset ID to attach the experiment to.
             experiment_runs: Experiment runs either as:
                 - a list of JSON-like dicts, or
-                - a pandas DataFrame.
+                - a :class:`pandas.DataFrame`.
             task_fields: Mapping that identifies the columns/fields containing the
                 task results (e.g. `example_id`, output fields).
             evaluator_columns: Optional mapping describing evaluator result columns.
@@ -178,7 +185,7 @@ class ExperimentsClient:
             body = gen.ExperimentsCreateRequest(
                 name=name,
                 dataset_id=dataset_id,
-                experiment_runs=data,  # type: ignore
+                experiment_runs=cast("list[gen.ExperimentRunCreate]", data),
             )
             return self._api.experiments_create(experiments_create_request=body)
 
@@ -229,7 +236,8 @@ class ExperimentsClient:
         Args:
             experiment_id: Experiment ID to delete.
 
-        Returns: This method returns None on success (common empty 204 response)
+        Returns:
+            This method returns None on success (common empty 204 response).
 
         Raises:
             arize._generated.api_client.exceptions.ApiException: If the REST API
@@ -299,7 +307,10 @@ class ExperimentsClient:
             )
         if experiment_df is not None:
             return models.ExperimentsRunsList200Response(
-                experimentRuns=experiment_df.to_dict(orient="records"),  # type: ignore
+                experiment_runs=cast(
+                    "list[models.ExperimentRun]",
+                    experiment_df.to_dict(orient="records"),
+                ),
                 pagination=models.PaginationMetadata(
                     has_more=False,  # Note that all=True
                 ),
@@ -339,7 +350,10 @@ class ExperimentsClient:
         )
 
         return models.ExperimentsRunsList200Response(
-            experimentRuns=experiment_df.to_dict(orient="records"),  # type: ignore
+            experiment_runs=cast(
+                "list[models.ExperimentRun]",
+                experiment_df.to_dict(orient="records"),
+            ),
             pagination=models.PaginationMetadata(
                 has_more=False,  # Note that all=True
             ),
@@ -357,6 +371,7 @@ class ExperimentsClient:
         concurrency: int = 3,
         set_global_tracer_provider: bool = False,
         exit_on_error: bool = False,
+        timeout: int = 120,
     ) -> tuple[models.Experiment | None, pd.DataFrame]:
         """Run an experiment on a dataset and optionally upload results.
 
@@ -387,6 +402,7 @@ class ExperimentsClient:
                 provider for the experiment run.
             exit_on_error: If True, stop on the first error encountered during
                 execution.
+            timeout: The timeout in seconds for each task execution. Defaults to 120.
 
         Returns:
             If `dry_run=True`, returns `(None, results_df)`.
@@ -505,6 +521,7 @@ class ExperimentsClient:
                 evaluators=evaluators,
                 concurrency=concurrency,
                 exit_on_error=exit_on_error,
+                timeout=timeout,
             )
             output_df = convert_default_columns_to_json_str(output_df)
             output_df = convert_boolean_columns_to_str(output_df)
@@ -546,9 +563,7 @@ class ExperimentsClient:
                 logger.error(msg)
                 raise RuntimeError(msg)
 
-            experiment = self.get(
-                experiment_id=str(post_resp.experiment_id)  # type: ignore
-            )
+            experiment = self.get(experiment_id=str(post_resp.experiment_id))
             return experiment, output_df
 
     def _create_experiment_via_flight(
@@ -629,9 +644,7 @@ class ExperimentsClient:
                 logger.error(msg)
                 raise RuntimeError(msg)
 
-        return self.get(
-            experiment_id=str(post_resp.experiment_id)  # type: ignore
-        )
+        return self.get(experiment_id=str(post_resp.experiment_id))
 
 
 def _get_tracer_resource(

arize/experiments/evaluators/base.py

@@ -7,7 +7,7 @@ import inspect
 from abc import ABC
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from types import MappingProxyType
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, cast
 
 from arize.experiments.evaluators.types import (
     AnnotatorKind,
@@ -79,10 +79,10 @@ class Evaluator(ABC):
         method and the asynchronous `async_evaluate` method, but it is not required.
 
         Args:
-            dataset_row (Optional[Mapping[str, JSONSerializable]]): A row from the dataset.
+            dataset_row (Mapping[str, JSONSerializable] | :obj:`None`): A row from the dataset.
             input (ExampleInput): The input provided for evaluation.
-            output (Optional[TaskOutput]): The output produced by the task.
-            experiment_output (Optional[TaskOutput]): The experiment output for comparison.
+            output (TaskOutput | :obj:`None`): The output produced by the task.
+            experiment_output (TaskOutput | :obj:`None`): The experiment output for comparison.
             dataset_output (ExampleOutput): The expected output from the dataset.
             metadata (ExampleMetadata): Metadata associated with the example.
             **kwargs (Any): Additional keyword arguments.
@@ -112,10 +112,10 @@ class Evaluator(ABC):
         method and the synchronous `evaluate` method, but it is not required.
 
         Args:
-            dataset_row (Optional[Mapping[str, JSONSerializable]]): A row from the dataset.
+            dataset_row (Mapping[str, JSONSerializable] | :obj:`None`): A row from the dataset.
             input (ExampleInput): The input provided for evaluation.
-            output (Optional[TaskOutput]): The output produced by the task.
-            experiment_output (Optional[TaskOutput]): The experiment output for comparison.
+            output (TaskOutput | :obj:`None`): The output produced by the task.
+            experiment_output (TaskOutput | :obj:`None`): The experiment output for comparison.
             dataset_output (ExampleOutput): The expected output from the dataset.
             metadata (ExampleMetadata): Metadata associated with the example.
             **kwargs (Any): Additional keyword arguments.
@@ -162,7 +162,9 @@ class Evaluator(ABC):
                         f"`evaluate()` method should be callable, got {type(evaluate)}"
                     )
                 # need to remove the first param, i.e. `self`
-                _validate_sig(functools.partial(evaluate, None), "evaluate")
+                _validate_sig(
+                    functools.partial(evaluate, cast("Any", None)), "evaluate"
+                )
                 return
             if async_evaluate := super_cls.__dict__.get(
                 Evaluator.async_evaluate.__name__
@@ -175,7 +177,8 @@ class Evaluator(ABC):
                     )
                 # need to remove the first param, i.e. `self`
                 _validate_sig(
-                    functools.partial(async_evaluate, None), "async_evaluate"
+                    functools.partial(async_evaluate, cast("Any", None)),
+                    "async_evaluate",
                 )
                 return
         raise ValueError(