arize 8.0.0b0__py3-none-any.whl → 8.0.0b2__py3-none-any.whl

arize/experiments/functions.py

@@ -56,7 +56,7 @@ from arize.experiments.types import (
     ExperimentEvaluationRun,
     ExperimentRun,
     ExperimentTask,
-    ExperimentTaskResultFieldNames,
+    ExperimentTaskFieldNames,
     _TaskSummary,
 )
 
@@ -76,23 +76,25 @@ def run_experiment(
     evaluators: Evaluators | None = None,
     concurrency: int = 3,
     exit_on_error: bool = False,
+    timeout: int = 120,
 ) -> pd.DataFrame:
     """Run an experiment on a dataset.
 
     Args:
         experiment_name (str): The name for the experiment.
         experiment_id (str): The ID for the experiment.
-        dataset (pd.DataFrame): The dataset to run the experiment on.
+        dataset (:class:`pandas.DataFrame`): The dataset to run the experiment on.
         task (ExperimentTask): The task to be executed on the dataset.
         tracer (Tracer): Tracer for tracing the experiment.
         resource (Resource): The resource for tracing the experiment.
-        rate_limit_errors (Optional[RateLimitErrors]): Optional rate limit errors.
-        evaluators (Optional[Evaluators]): Optional evaluators to assess the task.
+        rate_limit_errors (RateLimitErrors | :obj:`None`): Optional rate limit errors.
+        evaluators (Evaluators | :obj:`None`): Optional evaluators to assess the task.
         concurrency (int): The number of concurrent tasks to run. Default is 3.
         exit_on_error (bool): Whether to exit on error. Default is False.
+        timeout (int): The timeout for each task execution in seconds. Default is 120.
 
     Returns:
-        pd.DataFrame: The results of the experiment.
+        :class:`pandas.DataFrame`: The results of the experiment.
     """
     task_signature = inspect.signature(task)
     _validate_task_signature(task_signature)
@@ -289,6 +291,7 @@ def run_experiment(
         fallback_return_value=None,
         tqdm_bar_format=get_tqdm_progress_bar_formatter("running tasks"),
         concurrency=concurrency,
+        timeout=timeout,
     )
 
     runs, _ = executor.run(examples)
@@ -321,6 +324,7 @@ def run_experiment(
             tracer=tracer,
             resource=resource,
             exit_on_error=exit_on_error,
+            timeout=timeout,
         )
 
         if exit_on_error and (None in eval_results):
@@ -371,6 +375,7 @@ def evaluate_experiment(
     tracer: Tracer | None = None,
     resource: Resource | None = None,
     exit_on_error: bool = False,
+    timeout: int = 120,
 ) -> list[ExperimentEvaluationRun]:
     """Evaluate the results of an experiment using the provided evaluators.
 
@@ -379,11 +384,12 @@ def evaluate_experiment(
         examples (Sequence[Example]): The examples to evaluate.
         experiment_results (Sequence[ExperimentRun]): The results of the experiment.
         evaluators (Evaluators): The evaluators to use for assessment.
-        rate_limit_errors (Optional[RateLimitErrors]): Optional rate limit errors.
+        rate_limit_errors (RateLimitErrors | :obj:`None`): Optional rate limit errors.
         concurrency (int): The number of concurrent tasks to run. Default is 3.
-        tracer (Optional[Tracer]): Optional tracer for tracing the evaluation.
-        resource (Optional[Resource]): Optional resource for the evaluation.
+        tracer (Tracer | :obj:`None`): Optional tracer for tracing the evaluation.
+        resource (Resource | :obj:`None`): Optional resource for the evaluation.
         exit_on_error (bool): Whether to exit on error. Default is False.
+        timeout (int): The timeout for each evaluation in seconds. Default is 120.
 
     Returns:
         List[ExperimentEvaluationRun]: The evaluation results.
@@ -556,6 +562,7 @@ def evaluate_experiment(
             "running experiment evaluations"
         ),
         concurrency=concurrency,
+        timeout=timeout,
     )
     eval_runs, _ = executor.run(evaluation_input)
     return eval_runs
@@ -768,19 +775,19 @@ def get_result_attr(r: object, attr: str, default: object = None) -> object:
 
 def transform_to_experiment_format(
     experiment_runs: list[dict[str, object]] | pd.DataFrame,
-    task_fields: ExperimentTaskResultFieldNames,
+    task_fields: ExperimentTaskFieldNames,
     evaluator_fields: dict[str, EvaluationResultFieldNames] | None = None,
 ) -> pd.DataFrame:
-    """Transform a DataFrame to match the format returned by run_experiment().
+    """Transform a :class:`pandas.DataFrame` to match the format returned by run_experiment().
 
     Args:
-        experiment_runs: Input list of dictionaries or DataFrame containing experiment results
+        experiment_runs: Input list of dictionaries or :class:`pandas.DataFrame` containing experiment results
         task_fields: Field name mapping for task results
         evaluator_fields: Dictionary mapping evaluator names (str)
             to their field name mappings (EvaluationResultFieldNames)
 
     Returns:
-        DataFrame in the format matching run_experiment() output
+        :class:`pandas.DataFrame` in the format matching run_experiment() output
     """
     data = (
         experiment_runs
@@ -788,7 +795,7 @@ def transform_to_experiment_format(
         else pd.DataFrame(experiment_runs)
     )
     # Validate required columns
-    required_cols = {task_fields.example_id, task_fields.result}
+    required_cols = {task_fields.example_id, task_fields.output}
     missing_cols = required_cols - set(data.columns)
     if missing_cols:
         raise ValueError(f"Missing required columns: {missing_cols}")
@@ -799,11 +806,11 @@ def transform_to_experiment_format(
     out_df["example_id"] = data[task_fields.example_id]
     if task_fields.example_id != "example_id":
         out_df.drop(task_fields.example_id, axis=1, inplace=True)
-    out_df["result"] = data[task_fields.result].apply(
+    out_df["output"] = data[task_fields.output].apply(
         lambda x: json.dumps(x) if isinstance(x, dict) else x
     )
-    if task_fields.result != "result":
-        out_df.drop(task_fields.result, axis=1, inplace=True)
+    if task_fields.output != "output":
+        out_df.drop(task_fields.output, axis=1, inplace=True)
 
     # Process evaluator results
     if evaluator_fields:
@@ -822,7 +829,7 @@ def _add_evaluator_columns(
     evaluator_name: str,
     column_names: EvaluationResultFieldNames,
 ) -> None:
-    """Helper function to add evaluator columns to output DataFrame."""
+    """Helper function to add evaluator columns to output :class:`pandas.DataFrame`."""
     # Add score if specified
     if column_names.score and column_names.score in input_df.columns:
         output_df[f"eval.{evaluator_name}.score"] = input_df[column_names.score]

arize/experiments/types.py

@@ -23,8 +23,6 @@ from arize.experiments.evaluators.types import (
 )
 
 ExperimentId = str
-# DatasetId= str
-# DatasetVersionId= str
 ExampleId = str
 RepetitionNumber = int
 ExperimentRunId = str
@@ -263,9 +261,9 @@ class ExperimentEvaluationRun:
         name: The name of the evaluation run.
         annotator_kind: The kind of annotator used in the evaluation run.
         error: The error message if the evaluation run failed.
-        result (Optional[EvaluationResult]): The result of the evaluation run.
+        result (EvaluationResult | :obj:`None`): The result of the evaluation run.
         id (str): The unique identifier for the evaluation run.
-        trace_id (Optional[TraceId]): The trace identifier for the evaluation run.
+        trace_id (TraceId | :obj:`None`): The trace identifier for the evaluation run.
     """
 
     experiment_run_id: ExperimentRunId
@@ -397,17 +395,17 @@ def _top_string(s: pd.Series, length: int = 100) -> str | None:
 
 
 @dataclass
-class ExperimentTaskResultFieldNames:
-    """Column names for mapping experiment task results in a DataFrame.
+class ExperimentTaskFieldNames:
+    """Column names for mapping experiment task results in a :class:`pandas.DataFrame`.
 
     Args:
         example_id: Name of column containing example IDs.
             The ID values must match the id of the dataset rows.
-        result: Name of column containing task results
+        output: Name of column containing task results
     """
 
     example_id: str
-    result: str
+    output: str
 
 
 TaskOutput = JSONSerializable

arize/logging.py

@@ -259,7 +259,7 @@ def log_a_list(values: Iterable[Any] | None, join_word: str) -> str:
     """Format a list of values into a human-readable string with a joining word.
 
     Args:
-        values: An iterable of values to format, or None.
+        values: An iterable of values to format, or :obj:`None`.
         join_word: The word to use before the last item (e.g., "and", "or").
 
     Returns: