arize 8.0.0b0__py3-none-any.whl → 8.0.0b2__py3-none-any.whl

arize/exceptions/parameters.py

@@ -3,159 +3,6 @@
 from arize.constants.ml import MAX_NUMBER_OF_EMBEDDINGS
 from arize.exceptions.base import ValidationError
 
-# class MissingPredictionIdColumnForDelayedRecords(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Prediction_Id_Column_For_Delayed_Records"
-#
-#     def __init__(self, has_actual_info, has_feature_importance_info) -> None:
-#         self.has_actual_info = has_actual_info
-#         self.has_feature_importance_info = has_feature_importance_info
-#
-#     def error_message(self) -> str:
-#         actual = "actual" if self.has_actual_info else ""
-#         feat_imp = (
-#             "feature importance" if self.has_feature_importance_info else ""
-#         )
-#         if self.has_actual_info and self.has_feature_importance_info:
-#             msg = " and ".join([actual, feat_imp])
-#         else:
-#             msg = "".join([actual, feat_imp])
-#
-#         return (
-#             "Missing 'prediction_id_column_name'. While prediction id is optional for most cases, "
-#             "it is required when sending delayed actuals, i.e. when sending actual or feature importances "
-#             f"without predictions. In this case, {msg} information was found (without predictions). "
-#             "To learn more about delayed joins, please see the docs at "
-#             "https://docs.arize.com/arize/sending-data-guides/how-to-send-delayed-actuals"
-#         )
-
-
-# class MissingColumns(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Columns"
-#
-#     def __init__(self, cols: Iterable) -> None:
-#         self.missing_cols = set(cols)
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The following columns are declared in the schema "
-#             "but are not found in the dataframe: "
-#             f"{', '.join(map(str, self.missing_cols))}."
-#         )
-
-
-# class MissingRequiredColumnsMetricsValidation(ValidationError):
-#     """
-#     This error is used only for model mapping validations.
-#     """
-#
-#     def __repr__(self) -> str:
-#         return "Missing_Columns_Required_By_Metrics_Validation"
-#
-#     def __init__(
-#         self, model_type: ModelTypes, metrics: List[Metrics], cols: Iterable
-#     ) -> None:
-#         self.model_type = model_type
-#         self.metrics = metrics
-#         self.missing_cols = cols
-#
-#     def error_message(self) -> str:
-#         return (
-#             f"For logging data for a {self.model_type.name} model with support for metrics "
-#             f"{', '.join(m.name for m in self.metrics)}, "
-#             f"schema must include: {', '.join(map(str, self.missing_cols))}."
-#         )
-
-
-# class ReservedColumns(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Reserved_Columns"
-#
-#     def __init__(self, cols: Iterable) -> None:
-#         self.reserved_columns = cols
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The following columns are reserved and can only be specified "
-#             "in the proper fields of the schema: "
-#             f"{', '.join(map(str, self.reserved_columns))}."
-#         )
-
-
-# class InvalidModelTypeAndMetricsCombination(ValidationError):
-#     """
-#     This error is used only for model mapping validations.
-#     """
-#
-#     def __repr__(self) -> str:
-#         return "Invalid_ModelType_And_Metrics_Combination"
-#
-#     def __init__(
-#         self,
-#         model_type: ModelTypes,
-#         metrics: List[Metrics],
-#         suggested_model_metric_combinations: List[List[str]],
-#     ) -> None:
-#         self.model_type = model_type
-#         self.metrics = metrics
-#         self.suggested_combinations = suggested_model_metric_combinations
-#
-#     def error_message(self) -> str:
-#         valid_combos = ", or \n".join(
-#             "[" + ", ".join(combo) + "]"
-#             for combo in self.suggested_combinations
-#         )
-#         return (
-#             f"Invalid combination of model type {self.model_type.name} and metrics: "
-#             f"{', '.join(m.name for m in self.metrics)}. "
-#             f"Valid Metric combinations for this model type:\n{valid_combos}.\n"
-#         )
-
-
-# class InvalidShapSuffix(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_SHAP_Suffix"
-#
-#     def __init__(self, cols: Iterable) -> None:
-#         self.invalid_column_names = cols
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The following features or tags must not be named with a `_shap` suffix: "
-#             f"{', '.join(map(str, self.invalid_column_names))}."
-#         )
-
-
-# class InvalidModelType(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Model_Type"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "Model type not valid. Choose one of the following: "
-#             f"{', '.join('ModelTypes.' + mt.name for mt in ModelTypes)}. "
-#         )
-
-
-# class InvalidEnvironment(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Environment"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "Environment not valid. Choose one of the following: "
-#             f"{', '.join('Environments.' + env.name for env in Environments)}. "
-#         )
-
-
-# class InvalidBatchId(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Batch_ID"
-#
-#     def error_message(self) -> str:
-#         return "Batch ID must be a nonempty string if logging to validation environment."
-
 
 class InvalidModelVersion(ValidationError):
     """Raised when model version is empty or invalid."""
@@ -169,14 +16,6 @@ class InvalidModelVersion(ValidationError):
         return "Model version must be a nonempty string."
 
 
-# class InvalidModelId(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Model_ID"
-#
-#     def error_message(self) -> str:
-#         return "Model ID must be a nonempty string."
-
-
 class InvalidProjectName(ValidationError):
     """Raised when project name is empty or invalid."""
 
@@ -193,174 +32,6 @@ class InvalidProjectName(ValidationError):
         )
 
 
-# class MissingPredActShap(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Pred_or_Act_or_SHAP"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The schema must specify at least one of the following: "
-#             "prediction label, actual label, or SHAP value column names"
-#         )
-
-
-# class MissingPreprodPredAct(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Preproduction_Pred_and_Act"
-#
-#     def error_message(self) -> str:
-#         return "For logging pre-production data, the schema must specify both "
-#         "prediction and actual label columns."
-
-
-# class MissingPreprodAct(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Preproduction_Act"
-#
-#     def error_message(self) -> str:
-#         return "For logging pre-production data, the schema must specify actual label column."
-
-
-# class MissingPreprodPredActNumericAndCategorical(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Preproduction_Pred_and_Act_Numeric_and_Categorical"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "For logging pre-production data for a numeric or a categorical model, "
-#             "the schema must specify both prediction and actual label or score columns."
-#         )
-
-
-# class MissingRequiredColumnsForRankingModel(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Required_Columns_For_Ranking_Model"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "For logging data for a ranking model, schema must specify: "
-#             "prediction_group_id_column_name and rank_column_name"
-#         )
-
-
-# class MissingCVPredAct(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_CV_Prediction_or_Actual"
-#
-#     def __init__(self, environment: Environments):
-#         self.environment = environment
-#
-#     def error_message(self) -> str:
-#         if self.environment in (Environments.TRAINING, Environments.VALIDATION):
-#             env = "pre-production"
-#             opt = "and"
-#         elif self.environment == Environments.PRODUCTION:
-#             env = "production"
-#             opt = "or"
-#         else:
-#             raise TypeError("Invalid environment")
-#         return (
-#             f"For logging {env} data for an Object Detection model,"
-#             "the schema must specify one of: "
-#             f"('object_detection_prediction_column_names' {opt} "
-#             f"'object_detection_actual_column_names') "
-#             f"or ('semantic_segmentation_prediction_column_names' {opt} "
-#             f"'semantic_segmentation_actual_column_names') "
-#             f"or ('instance_segmentation_prediction_column_names' {opt} "
-#             f"'instance_segmentation_actual_column_names')"
-#         )
-
-
-# class MultipleCVPredAct(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Multiple_CV_Prediction_or_Actual"
-#
-#     def __init__(self, environment: Environments):
-#         self.environment = environment
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The schema must only specify one of the following: "
-#             "'object_detection_prediction_column_names'/'object_detection_actual_column_names'"
-#             "'semantic_segmentation_prediction_column_names'/'semantic_segmentation_actual_column_names'"
-#             "'instance_segmentation_prediction_column_names'/'instance_segmentation_actual_column_names'"
-#         )
-
-
-# class InvalidPredActCVColumnNamesForModelType(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_CV_Prediction_or_Actual_Column_Names_for_Model_Type"
-#
-#     def __init__(
-#         self,
-#         invalid_model_type: ModelTypes,
-#     ) -> None:
-#         self.invalid_model_type = invalid_model_type
-#
-#     def error_message(self) -> str:
-#         return (
-#             f"Cannot use 'object_detection_prediction_column_names' or "
-#             f"'object_detection_actual_column_names' or "
-#             f"'semantic_segmentation_prediction_column_names' or "
-#             f"'semantic_segmentation_actual_column_names' or "
-#             f"'instance_segmentation_prediction_column_names' or "
-#             f"'instance_segmentation_actual_column_names' for {self.invalid_model_type} model "
-#             f"type. They are only allowed for ModelTypes.OBJECT_DETECTION models"
-#         )
-
-
-# class MissingReqPredActColumnNamesForMultiClass(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Missing_Required_Prediction_or_Actual_Column_Names_for_Multi_Class_Model_Type"
-#
-#     def error_message(self) -> str:
-#         return (
-#             "For logging data for a multi class model, schema must specify: "
-#             "prediction_scores_column_name and/or actual_score_column_name. "
-#             "Optionally, you may include multi_class_threshold_scores_column_name"
-#             " (must include prediction_scores_column_name)"
-#         )
-
-
-# class InvalidPredActColumnNamesForModelType(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Invalid_Prediction_or_Actual_Column_Names_for_Model_Type"
-#
-#     def __init__(
-#         self,
-#         invalid_model_type: ModelTypes,
-#         allowed_fields: List[str],
-#         wrong_columns: List[str],
-#     ) -> None:
-#         self.invalid_model_type = invalid_model_type
-#         self.allowed_fields = allowed_fields
-#         self.wrong_columns = wrong_columns
-#
-#     def error_message(self) -> str:
-#         allowed_col_msg = ""
-#         if self.allowed_fields is not None:
-#             allowed_col_msg = f" Allowed Schema fields are {log_a_list(self.allowed_fields, 'and')}"
-#         return (
-#             f"Invalid Schema fields for {self.invalid_model_type} model type. {allowed_col_msg}"
-#             "The following columns of your dataframe are sent as an invalid schema field: "
-#             f"{log_a_list(self.wrong_columns, 'and')}"
-#         )
-
-
-# class DuplicateColumnsInDataframe(ValidationError):
-#     def __repr__(self) -> str:
-#         return "Duplicate_Columns_In_Dataframe"
-#
-#     def __init__(self, cols: Iterable) -> None:
-#         self.duplicate_cols = cols
-#
-#     def error_message(self) -> str:
-#         return (
-#             "The following columns are present in the schema and have duplicates in the dataframe: "
-#             f"{self.duplicate_cols}. "
-#         )
-
-
 class InvalidNumberOfEmbeddings(ValidationError):
     """Raised when number of embedding features exceeds the maximum allowed."""
 

arize/experiments/__init__.py

@@ -4,10 +4,10 @@ from arize.experiments.evaluators.types import (
     EvaluationResult,
     EvaluationResultFieldNames,
 )
-from arize.experiments.types import ExperimentTaskResultFieldNames
+from arize.experiments.types import ExperimentTaskFieldNames
 
 __all__ = [
     "EvaluationResult",
     "EvaluationResultFieldNames",
-    "ExperimentTaskResultFieldNames",
+    "ExperimentTaskFieldNames",
 ]

arize/experiments/client.py

@@ -38,12 +38,13 @@ from arize.utils.size import get_payload_size_mb
 if TYPE_CHECKING:
     from opentelemetry.trace import Tracer
 
+    from arize._generated.api_client.api_client import ApiClient
     from arize.config import SDKConfiguration
     from arize.experiments.evaluators.base import Evaluators
     from arize.experiments.evaluators.types import EvaluationResultFieldNames
     from arize.experiments.types import (
         ExperimentTask,
-        ExperimentTaskResultFieldNames,
+        ExperimentTaskFieldNames,
     )
 
 logger = logging.getLogger(__name__)
@@ -61,20 +62,22 @@ class ExperimentsClient:
     :class:`arize.config.SDKConfiguration`.
     """
 
-    def __init__(self, *, sdk_config: SDKConfiguration) -> None:
+    def __init__(
+        self, *, sdk_config: SDKConfiguration, generated_client: ApiClient
+    ) -> None:
         """
         Args:
             sdk_config: Resolved SDK configuration.
+            generated_client: Shared generated API client instance.
         """  # noqa: D205, D212
         self._sdk_config = sdk_config
         from arize._generated import api_client as gen
 
-        self._api = gen.ExperimentsApi(self._sdk_config.get_generated_client())
+        # Use the provided client directly for both APIs
+        self._api = gen.ExperimentsApi(generated_client)
         # TODO(Kiko): Space ID should not be needed,
         # should work on server tech debt to remove this
-        self._datasets_api = gen.DatasetsApi(
-            self._sdk_config.get_generated_client()
-        )
+        self._datasets_api = gen.DatasetsApi(generated_client)
 
     @prerelease_endpoint(key="experiments.list", stage=ReleaseStage.BETA)
     def list(
@@ -114,7 +117,7 @@ class ExperimentsClient:
         name: str,
         dataset_id: str,
         experiment_runs: list[dict[str, object]] | pd.DataFrame,
-        task_fields: ExperimentTaskResultFieldNames,
+        task_fields: ExperimentTaskFieldNames,
         evaluator_columns: dict[str, EvaluationResultFieldNames] | None = None,
         force_http: bool = False,
     ) -> models.Experiment:
@@ -141,7 +144,7 @@ class ExperimentsClient:
             dataset_id: Dataset ID to attach the experiment to.
             experiment_runs: Experiment runs either as:
                 - a list of JSON-like dicts, or
-                - a pandas DataFrame.
+                - a :class:`pandas.DataFrame`.
             task_fields: Mapping that identifies the columns/fields containing the
                 task results (e.g. `example_id`, output fields).
             evaluator_columns: Optional mapping describing evaluator result columns.
@@ -175,7 +178,6 @@ class ExperimentsClient:
             from arize._generated import api_client as gen
 
             data = experiment_df.to_dict(orient="records")
-
             body = gen.ExperimentsCreateRequest(
                 name=name,
                 dataset_id=dataset_id,
@@ -230,7 +232,8 @@ class ExperimentsClient:
         Args:
             experiment_id: Experiment ID to delete.
 
-        Returns: This method returns None on success (common empty 204 response)
+        Returns:
+            This method returns None on success (common empty 204 response).
 
         Raises:
             arize._generated.api_client.exceptions.ApiException: If the REST API
@@ -358,6 +361,7 @@ class ExperimentsClient:
         concurrency: int = 3,
         set_global_tracer_provider: bool = False,
         exit_on_error: bool = False,
+        timeout: int = 120,
     ) -> tuple[models.Experiment | None, pd.DataFrame]:
         """Run an experiment on a dataset and optionally upload results.
 
@@ -388,6 +392,7 @@ class ExperimentsClient:
                 provider for the experiment run.
             exit_on_error: If True, stop on the first error encountered during
                 execution.
+            timeout: The timeout in seconds for each task execution. Defaults to 120.
 
         Returns:
             If `dry_run=True`, returns `(None, results_df)`.
@@ -506,6 +511,7 @@ class ExperimentsClient:
                 evaluators=evaluators,
                 concurrency=concurrency,
                 exit_on_error=exit_on_error,
+                timeout=timeout,
             )
             output_df = convert_default_columns_to_json_str(output_df)
             output_df = convert_boolean_columns_to_str(output_df)

arize/experiments/evaluators/base.py

@@ -79,10 +79,10 @@ class Evaluator(ABC):
         method and the asynchronous `async_evaluate` method, but it is not required.
 
         Args:
-            dataset_row (Optional[Mapping[str, JSONSerializable]]): A row from the dataset.
+            dataset_row (Mapping[str, JSONSerializable] | :obj:`None`): A row from the dataset.
             input (ExampleInput): The input provided for evaluation.
-            output (Optional[TaskOutput]): The output produced by the task.
-            experiment_output (Optional[TaskOutput]): The experiment output for comparison.
+            output (TaskOutput | :obj:`None`): The output produced by the task.
+            experiment_output (TaskOutput | :obj:`None`): The experiment output for comparison.
             dataset_output (ExampleOutput): The expected output from the dataset.
             metadata (ExampleMetadata): Metadata associated with the example.
             **kwargs (Any): Additional keyword arguments.
@@ -112,10 +112,10 @@ class Evaluator(ABC):
         method and the synchronous `evaluate` method, but it is not required.
 
         Args:
-            dataset_row (Optional[Mapping[str, JSONSerializable]]): A row from the dataset.
+            dataset_row (Mapping[str, JSONSerializable] | :obj:`None`): A row from the dataset.
             input (ExampleInput): The input provided for evaluation.
-            output (Optional[TaskOutput]): The output produced by the task.
-            experiment_output (Optional[TaskOutput]): The experiment output for comparison.
+            output (TaskOutput | :obj:`None`): The output produced by the task.
+            experiment_output (TaskOutput | :obj:`None`): The experiment output for comparison.
             dataset_output (ExampleOutput): The expected output from the dataset.
             metadata (ExampleMetadata): Metadata associated with the example.
             **kwargs (Any): Additional keyword arguments.

arize/experiments/evaluators/executors.py

@@ -94,7 +94,7 @@ class AsyncExecutor(Executor):
 
         concurrency (int, optional): The number of concurrent consumers. Defaults to 3.
 
-        tqdm_bar_format (Optional[str], optional): The format string for the progress bar.
+        tqdm_bar_format (str | :obj:`None`, optional): The format string for the progress bar.
             Defaults to None.
 
         max_retries (int, optional): The maximum number of times to retry on exceptions.
@@ -119,6 +119,7 @@ class AsyncExecutor(Executor):
         exit_on_error: bool = True,
         fallback_return_value: Unset | object = _unset,
         termination_signal: signal.Signals = signal.SIGINT,
+        timeout: int = 120,
     ) -> None:
         """Initialize the async executor with configuration parameters.
 
@@ -130,6 +131,7 @@ class AsyncExecutor(Executor):
             exit_on_error: Whether to exit on first error.
             fallback_return_value: Value to return when execution fails.
             termination_signal: Signal to handle for graceful termination.
+            timeout: Timeout for each task in seconds.
         """
         self.generate = generation_fn
         self.fallback_return_value = fallback_return_value
@@ -139,6 +141,7 @@ class AsyncExecutor(Executor):
         self.exit_on_error = exit_on_error
         self.base_priority = 0
         self.termination_signal = termination_signal
+        self.timeout = timeout
 
     async def producer(
         self,
@@ -195,7 +198,7 @@ class AsyncExecutor(Executor):
                 )
                 done, _pending = await asyncio.wait(
                     [generate_task, termination_event_watcher],
-                    timeout=120,
+                    timeout=self.timeout,
                     return_when=asyncio.FIRST_COMPLETED,
                 )
 
@@ -341,7 +344,7 @@ class SyncExecutor(Executor):
         generation_fn (Callable[[object], Any]): The generation function that takes an input and
             returns an output.
 
-        tqdm_bar_format (Optional[str], optional): The format string for the progress bar. Defaults
+        tqdm_bar_format (str | :obj:`None`, optional): The format string for the progress bar. Defaults
             to None.
 
         max_retries (int, optional): The maximum number of times to retry on exceptions. Defaults to
@@ -460,6 +463,7 @@ def get_executor_on_sync_context(
     max_retries: int = 10,
     exit_on_error: bool = True,
     fallback_return_value: Unset | object = _unset,
+    timeout: int = 120,
 ) -> Executor:
     """Get an appropriate executor based on the current threading context.
 
@@ -475,6 +479,7 @@ def get_executor_on_sync_context(
         max_retries: Maximum number of retry attempts. Defaults to 10.
         exit_on_error: Whether to exit on first error. Defaults to True.
         fallback_return_value: Value to return on failure. Defaults to unset.
+        timeout: Timeout for each task in seconds. Defaults to 120.
 
     Returns:
         An Executor instance configured for the current context.
@@ -513,6 +518,7 @@ def get_executor_on_sync_context(
                 max_retries=max_retries,
                 exit_on_error=exit_on_error,
                 fallback_return_value=fallback_return_value,
+                timeout=timeout,
             )
         logger.warning(
             "🐌!! If running inside a notebook, patching the event loop with "
@@ -533,6 +539,7 @@ def get_executor_on_sync_context(
         max_retries=max_retries,
         exit_on_error=exit_on_error,
         fallback_return_value=fallback_return_value,
+        timeout=timeout,
     )
 
 

arize/experiments/evaluators/types.py

@@ -94,14 +94,14 @@ EvaluatorOutput = (
 
 @dataclass
 class EvaluationResultFieldNames:
-    """Column names for mapping evaluation results in a DataFrame.
+    """Column names for mapping evaluation results in a :class:`pandas.DataFrame`.
 
     Args:
         score: Optional name of column containing evaluation scores
         label: Optional name of column containing evaluation labels
         explanation: Optional name of column containing evaluation explanations
         metadata: Optional mapping of metadata keys to column names. If a column name
-            is None or empty string, the metadata key will be used as the column name.
+            is :obj:`None` or empty string, the metadata key will be used as the column name.
 
     Examples:
         >>> # Basic usage with score and label columns