langwatch 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

langwatch/experiment/__init__.py

@@ -0,0 +1,108 @@
+"""
+langwatch.experiment - Run experiments on LangWatch platform or via SDK.
+
+This module provides two ways to run experiments:
+
+1. Platform experiments (CI/CD):
+   Run experiments configured in the LangWatch platform UI.
+
+   ```python
+   import langwatch
+
+   result = langwatch.experiment.run("my-experiment-slug")
+   result.print_summary()
+   ```
+
+2. SDK-defined experiments:
+   Define and run experiments programmatically.
+
+   ```python
+   import langwatch
+
+   experiment = langwatch.experiment.init("my-experiment")
+
+   for index, row in experiment.loop(df.iterrows(), threads=4):
+       async def task(index, row):
+           result = await my_agent(row["input"])
+           experiment.evaluate(
+               "langevals/exact_match",
+               index=index,
+               data={"output": result, "expected_output": row["expected"]},
+               settings={},
+           )
+       experiment.submit(task, index, row)
+   ```
+"""
+from typing import Optional
+
+# Re-export the Experiment class for SDK-defined experiments
+from langwatch.experiment.experiment import Experiment
+
+# Re-export the platform run function and related types
+from langwatch.experiment.platform_run import (
+    run,
+    ExperimentRunResult,
+    ExperimentRunSummary,
+    ExperimentNotFoundError,
+    ExperimentTimeoutError,
+    ExperimentRunFailedError,
+    ExperimentsApiError,
+    TargetStats,
+    EvaluatorStats,
+)
+
+
+def init(name: str, *, run_id: Optional[str] = None) -> Experiment:
+    """
+    Initialize an SDK-defined experiment.
+
+    This creates an Experiment instance that you can use to run evaluators
+    programmatically using datasets and custom logic.
+
+    Args:
+        name: Name for this experiment run
+        run_id: Optional custom run ID (auto-generated if not provided)
+
+    Returns:
+        Experiment instance with methods:
+        - loop(): Iterate over dataset rows with parallel execution
+        - evaluate(): Run an evaluator on the current row
+        - log(): Log custom metrics
+        - submit(): Submit async tasks
+
+    Example:
+        ```python
+        import langwatch
+
+        experiment = langwatch.experiment.init("my-experiment")
+
+        for index, row in experiment.loop(df.iterrows(), threads=4):
+            async def task(index, row):
+                result = await my_agent(row["input"])
+                experiment.evaluate(
+                    "langevals/exact_match",
+                    index=index,
+                    data={"output": result, "expected_output": row["expected"]},
+                    settings={},
+                )
+            experiment.submit(task, index, row)
+        ```
+    """
+    experiment = Experiment(name, run_id=run_id)
+    experiment.init()
+    return experiment
+
+
+__all__ = [
+    "init",
+    "run",
+    "Experiment",
+    "ExperimentRunResult",
+    "ExperimentRunSummary",
+    "ExperimentNotFoundError",
+    "ExperimentTimeoutError",
+    "ExperimentRunFailedError",
+    "ExperimentsApiError",
+    "TargetStats",
+    "EvaluatorStats",
+]

langwatch/{evaluation/evaluation.py → experiment/experiment.py}

@@ -135,7 +135,7 @@ class IterationInfo(TypedDict):
     error: Optional[Exception]
 
 
-class Evaluation:
+class Experiment:
     _executor: ThreadPoolExecutor
     _futures: List[Future[Any]]
     _current_index: int
@@ -255,7 +255,7 @@ class Evaluation:
                 progress_bar.close()
 
         except Exception as e:
-            Evaluation._log_results(
+            Experiment._log_results(
                 langwatch.get_api_key() or "",
                 {
                     "experiment_slug": self.experiment_slug,
@@ -456,7 +456,7 @@ class Evaluation:
 
             # Start a new thread to send the batch
             thread = threading.Thread(
-                target=Evaluation._log_results,
+                target=Experiment._log_results,
                 args=(langwatch.get_api_key(), body),
             )
             thread.start()
@@ -485,7 +485,7 @@ class Evaluation:
         better_raise_for_status(response)
 
     def _wait_for_completion(self):
-        async def wait_for_completion(self: Evaluation):
+        async def wait_for_completion(self: Experiment):
             # Send any remaining batch
             self._send_batch(finished=True)
 
@@ -837,7 +837,7 @@ class Evaluation:
         with self.lock:
             self.batch["evaluations"].append(eval)
 
-    def run(
+    def evaluate(
         self,
         evaluator_id: str,
         index: Union[int, Hashable],
@@ -846,6 +846,17 @@ class Evaluation:
         name: Optional[str] = None,
         as_guardrail: bool = False,
     ):
+        """
+        Run an evaluator on the current row.
+
+        Args:
+            evaluator_id: The evaluator type/slug (e.g., "langevals/exact_match", "ragas/faithfulness")
+            index: The row index for this evaluation
+            data: Data to pass to the evaluator (e.g., {"input": ..., "output": ..., "expected_output": ...})
+            settings: Evaluator-specific settings
+            name: Optional display name for the evaluation (defaults to evaluator_id)
+            as_guardrail: Whether to run as a guardrail (stricter pass/fail)
+        """
         duration: Optional[int] = None
 
         start_time = time.time()
@@ -871,3 +882,31 @@ class Evaluation:
             duration=duration,
             cost=result.cost,
         )
+
+    def run(
+        self,
+        evaluator_id: str,
+        index: Union[int, Hashable],
+        data: Dict[str, Any],
+        settings: Dict[str, Any],
+        name: Optional[str] = None,
+        as_guardrail: bool = False,
+    ):
+        """
+        Deprecated: Use `evaluate()` instead.
+        """
+        import warnings
+
+        warnings.warn(
+            "evaluation.run() is deprecated, use evaluation.evaluate() instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.evaluate(
+            evaluator_id=evaluator_id,
+            index=index,
+            data=data,
+            settings=settings,
+            name=name,
+            as_guardrail=as_guardrail,
+        )

langwatch/{evaluation → experiment}/platform_run.py

@@ -1,5 +1,5 @@
 """
-Runner for platform-configured evaluations (Evaluations V3).
+Runner for platform-configured experiments (Experiments Workbench).
 
 This module provides the `run()` function to execute evaluations that are
 configured in the LangWatch platform from CI/CD pipelines or scripts.
@@ -35,16 +35,16 @@ def _replace_url_domain(url: str, new_base: str) -> str:
     ))
 
 
-class EvaluationNotFoundError(Exception):
-    """Raised when evaluation slug doesn't exist."""
+class ExperimentNotFoundError(Exception):
+    """Raised when experiment slug doesn't exist."""
 
     def __init__(self, slug: str):
         self.slug = slug
         super().__init__(f"Evaluation not found: {slug}")
 
 
-class EvaluationTimeoutError(Exception):
-    """Raised when evaluation run times out."""
+class ExperimentTimeoutError(Exception):
+    """Raised when experiment run times out."""
 
     def __init__(self, run_id: str, progress: int, total: int):
         self.run_id = run_id
@@ -55,8 +55,8 @@ class EvaluationTimeoutError(Exception):
         )
 
 
-class EvaluationRunFailedError(Exception):
-    """Raised when evaluation run fails."""
+class ExperimentRunFailedError(Exception):
+    """Raised when experiment run fails."""
 
     def __init__(self, run_id: str, error: str):
         self.run_id = run_id
@@ -64,7 +64,7 @@ class EvaluationRunFailedError(Exception):
         super().__init__(f"Evaluation run failed: {error}")
 
 
-class EvaluationsApiError(Exception):
+class ExperimentsApiError(Exception):
     """Raised for other API errors."""
 
     def __init__(self, message: str, status_code: int):
@@ -97,8 +97,8 @@ class EvaluatorStats:
 
 
 @dataclass
-class EvaluationRunSummary:
-    """Summary of a completed evaluation run."""
+class ExperimentRunSummary:
+    """Summary of a completed experiment run."""
 
     run_id: str
     total_cells: int
@@ -115,7 +115,7 @@ class EvaluationRunSummary:
 
 
 @dataclass
-class EvaluationRunResult:
+class ExperimentRunResult:
     """Result of running a platform evaluation."""
 
     run_id: str
@@ -125,7 +125,7 @@ class EvaluationRunResult:
     pass_rate: float
     duration: int
     run_url: str
-    summary: EvaluationRunSummary
+    summary: ExperimentRunSummary
 
     def print_summary(self, exit_on_failure: Optional[bool] = None) -> None:
         """
@@ -161,42 +161,42 @@ def _is_notebook() -> bool:
         return False
 
 
-def evaluate(
+def run(
     slug: str,
     *,
     poll_interval: float = 2.0,
     timeout: float = 600.0,
     on_progress: Optional[Callable[[int, int], None]] = None,
     api_key: Optional[str] = None,
-) -> EvaluationRunResult:
+) -> ExperimentRunResult:
     """
-    Run a platform-configured evaluation and wait for completion.
+    Run a platform-configured experiment and wait for completion.
 
-    This runs an Evaluation that you have configured in the LangWatch platform.
-    The evaluation will execute all targets and evaluators defined in the configuration.
+    This runs an Experiment that you have configured in the LangWatch platform.
+    The experiment will execute all targets and evaluators defined in the configuration.
 
     Args:
-        slug: The slug of the evaluation to run (found in the evaluation URL)
+        slug: The slug of the experiment to run (found in the experiment URL)
         poll_interval: Seconds between status checks (default: 2.0)
         timeout: Maximum seconds to wait for completion (default: 600.0 = 10 minutes)
         on_progress: Optional callback for progress updates (completed, total)
         api_key: Optional API key override (uses LANGWATCH_API_KEY env var by default)
 
     Returns:
-        EvaluationRunResult with pass rate and summary. Call result.print_summary()
+        ExperimentRunResult with pass rate and summary. Call result.print_summary()
         to display results and exit with code 1 on failure.
 
     Raises:
-        EvaluationNotFoundError: If the evaluation slug doesn't exist
-        EvaluationTimeoutError: If the evaluation doesn't complete within timeout
-        EvaluationRunFailedError: If the evaluation fails
-        EvaluationsApiError: For other API errors
+        ExperimentNotFoundError: If the experiment slug doesn't exist
+        ExperimentTimeoutError: If the experiment doesn't complete within timeout
+        ExperimentRunFailedError: If the experiment fails
+        ExperimentsApiError: For other API errors
 
     Example:
         ```python
         import langwatch
 
-        result = langwatch.evaluation.evaluate("my-evaluation-slug")
+        result = langwatch.experiment.run("my-experiment-slug")
         result.print_summary()
         ```
     """
@@ -219,7 +219,7 @@ def evaluate(
     api_run_url = start_response.get("runUrl", "")
     run_url = _replace_url_domain(api_run_url, endpoint) if api_run_url else ""
 
-    print(f"Started evaluation run: {run_id}")
+    print(f"Started experiment run: {run_id}")
     if run_url:
         print(f"Follow live: {run_url}")
 
@@ -238,7 +238,7 @@ def evaluate(
         if time.time() - start_time > timeout:
             print()  # Newline after progress
             status = _get_run_status(run_id, endpoint, effective_api_key)
-            raise EvaluationTimeoutError(
+            raise ExperimentTimeoutError(
                 run_id, status.get("progress", 0), status.get("total", 0)
             )
 
@@ -267,7 +267,7 @@ def evaluate(
 
         if run_status == "failed":
             print()  # Newline after progress
-            raise EvaluationRunFailedError(
+            raise ExperimentRunFailedError(
                 run_id, status.get("error", "Unknown error")
             )
 
@@ -278,7 +278,7 @@ def evaluate(
 
 
 def _start_run(slug: str, endpoint: str, api_key: str) -> dict:
-    """Start an evaluation run."""
+    """Start an experiment run."""
     with httpx.Client(timeout=60) as client:
         response = client.post(
             f"{endpoint}/api/evaluations/v3/{slug}/run",
@@ -286,12 +286,12 @@ def _start_run(slug: str, endpoint: str, api_key: str) -> dict:
         )
 
     if response.status_code == 404:
-        raise EvaluationNotFoundError(slug)
+        raise ExperimentNotFoundError(slug)
     if response.status_code == 401:
-        raise EvaluationsApiError("Unauthorized - check your API key", 401)
+        raise ExperimentsApiError("Unauthorized - check your API key", 401)
     if not response.is_success:
         error_body = response.json() if response.content else {}
-        raise EvaluationsApiError(
+        raise ExperimentsApiError(
             error_body.get("error", f"Failed to start evaluation: {response.status_code}"),
             response.status_code,
         )
@@ -308,12 +308,12 @@ def _get_run_status(run_id: str, endpoint: str, api_key: str) -> dict:
         )
 
     if response.status_code == 404:
-        raise EvaluationsApiError(f"Run not found: {run_id}", 404)
+        raise ExperimentsApiError(f"Run not found: {run_id}", 404)
     if response.status_code == 401:
-        raise EvaluationsApiError("Unauthorized - check your API key", 401)
+        raise ExperimentsApiError("Unauthorized - check your API key", 401)
     if not response.is_success:
         error_body = response.json() if response.content else {}
-        raise EvaluationsApiError(
+        raise ExperimentsApiError(
             error_body.get("error", f"Failed to get run status: {response.status_code}"),
             response.status_code,
         )
@@ -326,7 +326,7 @@ def _build_result(
     status: Literal["completed", "failed", "stopped"],
     summary_data: dict,
     run_url: str,
-) -> EvaluationRunResult:
+) -> ExperimentRunResult:
     """Build the result object from API response."""
     total_cells = summary_data.get("totalCells", 0)
     completed_cells = summary_data.get("completedCells", 0)
@@ -368,7 +368,7 @@ def _build_result(
             )
         )
 
-    summary = EvaluationRunSummary(
+    summary = ExperimentRunSummary(
         run_id=run_id,
         total_cells=total_cells,
         completed_cells=completed_cells,
@@ -383,7 +383,7 @@ def _build_result(
         total_cost=summary_data.get("totalCost", 0),
     )
 
-    return EvaluationRunResult(
+    return ExperimentRunResult(
         run_id=run_id,
         status=status,
         passed=total_passed,
@@ -395,12 +395,12 @@ def _build_result(
     )
 
 
-def _print_summary(result: EvaluationRunResult) -> None:
-    """Print a CI-friendly summary of the evaluation results."""
+def _print_summary(result: ExperimentRunResult) -> None:
+    """Print a CI-friendly summary of the experiment results."""
     summary = result.summary
 
     print("\n" + "═" * 60)
-    print("  EVALUATION RESULTS")
+    print("  EXPERIMENT RESULTS")
     print("═" * 60)
     print(f"  Run ID:     {result.run_id}")
     print(f"  Status:     {result.status.upper()}")
@@ -433,30 +433,3 @@ def _print_summary(result: EvaluationRunResult) -> None:
     print("═" * 60 + "\n")
 
 
-def run(
-    slug: str,
-    *,
-    poll_interval: float = 2.0,
-    timeout: float = 600.0,
-    on_progress: Optional[Callable[[int, int], None]] = None,
-    api_key: Optional[str] = None,
-) -> EvaluationRunResult:
-    """
-    Deprecated: Use `evaluate()` instead.
-
-    Run a platform-configured evaluation and wait for completion.
-    """
-    import warnings
-
-    warnings.warn(
-        "langwatch.evaluation.run() is deprecated, use langwatch.evaluation.evaluate() instead",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return evaluate(
-        slug,
-        poll_interval=poll_interval,
-        timeout=timeout,
-        on_progress=on_progress,
-        api_key=api_key,
-    )

{langwatch-0.9.0.dist-info → langwatch-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch
-Version: 0.9.0
+Version: 0.10.0
 Summary: LangWatch Python SDK, for monitoring your LLMs
 Author-email: Langwatch Engineers <engineering@langwatch.ai>
 License: MIT

{langwatch-0.9.0.dist-info → langwatch-0.10.0.dist-info}/RECORD

@@ -1,9 +1,9 @@
-langwatch/__init__.py,sha256=GMq4SV2Tz2i0JD05shqnw2lBW5cgMx4Zzo141hp106k,4266
-langwatch/__version__.py,sha256=sympc_lD0EH1ffjgsP80P8i4Sqm2XBcIgblEeQTq6bs,91
+langwatch/__init__.py,sha256=VGkLDw3h8hOtzyaTMObWupGTQosn4E17Dk5zcfDmy7g,4462
+langwatch/__version__.py,sha256=o5DKyCSbKMRVAiPvoV2UuvLgN1NaTqfqaz2GumN3K3A,92
 langwatch/attributes.py,sha256=nXdI_G85wQQCAdAcwjCiLYdEYj3wATmfgCmhlf6dVIk,3910
 langwatch/batch_evaluation.py,sha256=Y_S3teXpHV07U-vvJYyV1PB6d0CgyFM_rTzPp6GnEBo,16165
 langwatch/client.py,sha256=WTNcYSik7kZ2kH-qGDnhbMTosc8e_Xhab_lZlfh5TC8,25559
-langwatch/evaluations.py,sha256=W-nxhcbQ4Niyx949LjHjTehw74pj31dUVs5rjkQUBLo,16951
+langwatch/evaluations.py,sha256=-rvlpw8J3-0lMn9tdnte1Z3qHpuE60DGB3gmI8VMexQ,8983
 langwatch/guardrails.py,sha256=4d320HyklXPUVszF34aWsDKGzuvPggcDM_f45_eJTnc,1352
 langwatch/langchain.py,sha256=HjbBBIDwwt688g0W4K0EsZGuGBbo1Mv5LQ-7Mkn56iQ,18726
 langwatch/litellm.py,sha256=mPcw5cLykt0SQf9bTNSoT7elMx4gj-wZ_K2PC14Bw50,11998
@@ -16,9 +16,10 @@ langwatch/types.py,sha256=h6r3tNTzWqENx-9j_JPmOMZfFoKq9SNpEtxpAACk2G0,3114
 langwatch/dataset/__init__.py,sha256=hZBcbjXuBO2qE5osJtd9wIE9f45F6-jpNTrne5nk4eE,2606
 langwatch/domain/__init__.py,sha256=gSCOV3WkRhp_--9D1vxw7BYpnMRbpGh-2NbsXd4KZC0,6074
 langwatch/dspy/__init__.py,sha256=wp8AmobV8XGVWOI8MQFmXPHu-8Wq3wvjB6YiHQm9Fdg,33007
-langwatch/evaluation/__init__.py,sha256=dctG-Ec0N_Or2Ta0XW6liYtdpMZN3ZtRXqUoeG5ksnk,870
-langwatch/evaluation/evaluation.py,sha256=MqMiGlsPIS5zqN1wKfhEs6mIGLRwB452iqDTSQFbtYo,31735
-langwatch/evaluation/platform_run.py,sha256=cwuRNtG99nhvqGL-YoOwdvEH3x-hDaVUzl7Vx9orjPo,14546
+langwatch/evaluation/__init__.py,sha256=8SOSZZbSzXa1jL-9Zlyt0f9u5sOA_TrO1J61ueASBLI,16980
+langwatch/experiment/__init__.py,sha256=nv2OfoNMMZwUA9KfozW2ZNaR1-J1LCmU4NykjGfe9is,3001
+langwatch/experiment/experiment.py,sha256=5xj58FKVC0y_LxgfwjJZP9lDp7tZ9FUUbERBtui_nC8,33026
+langwatch/experiment/platform_run.py,sha256=qiy_bwp786TbkH4HIlZVlJPmCtQlStAq9vUdG4-3VdU,13850
 langwatch/exporters/filterable_batch_span_exporter.py,sha256=MlhZjui56XD6p2sa8kEGyr-Hb3wqudknngmemnB4Twg,2142
 langwatch/generated/langwatch_rest_api_client/__init__.py,sha256=8r-9pAj7fK7vnVX3mT0y_zS4B9ZRqD6RZiBo5fPra60,156
 langwatch/generated/langwatch_rest_api_client/client.py,sha256=o_mdLqyBCQstu5tS1WZFwqIEbGwkvWQ7eQjuCJw_5VY,12419
@@ -416,6 +417,6 @@ langwatch/utils/initialization.py,sha256=1KoZmkHOvGEVF0j-4t4xRQdA_2C_SPiF7qFXqEG
 langwatch/utils/module.py,sha256=KLBNOK3mA9gCSifCcQX_lOtU48BJQDWvFKtF6NMvwVA,688
 langwatch/utils/transformation.py,sha256=76MGXyrYTxM0Yri36NJqLK-XxL4BBYdmKWAXXlw3D4Q,7690
 langwatch/utils/utils.py,sha256=ZCOSie4o9LdJ7odshNfCNjmgwgQ27ojc5ENqt1rXuSs,596
-langwatch-0.9.0.dist-info/METADATA,sha256=JtLLtVbyy0iau3ySelLpMO4RpjrQAEyhd72J9NkxHl8,13192
-langwatch-0.9.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-langwatch-0.9.0.dist-info/RECORD,,
+langwatch-0.10.0.dist-info/METADATA,sha256=ZiqO5NpMuwsb60yAVPverSp8GXlMyIctxaDgO3DSRoA,13193
+langwatch-0.10.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+langwatch-0.10.0.dist-info/RECORD,,