arize-phoenix 4.4.4rc5__py3-none-any.whl → 4.4.4rc6__py3-none-any.whl

phoenix/{datasets/experiments.py → experiments/functions.py}

@@ -3,13 +3,13 @@ import json
 from binascii import hexlify
 from contextlib import ExitStack
 from copy import deepcopy
+from dataclasses import replace
 from datetime import datetime, timezone
 from itertools import product
 from typing import (
     Any,
     Awaitable,
     Dict,
-    Iterable,
     Mapping,
     Optional,
     Sequence,
@@ -22,6 +22,7 @@ from urllib.parse import urljoin
 
 import httpx
 import opentelemetry.sdk.trace as trace_sdk
+import pandas as pd
 from openinference.semconv.resource import ResourceAttributes
 from openinference.semconv.trace import (
     OpenInferenceMimeTypeValues,
@@ -33,76 +34,51 @@ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExport
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import Span
 from opentelemetry.sdk.trace.export import SimpleSpanProcessor
-from opentelemetry.trace import Status, StatusCode
+from opentelemetry.trace import Status, StatusCode, Tracer
 from typing_extensions import TypeAlias
 
-from phoenix.config import (
-    get_env_client_headers,
-    get_env_collector_endpoint,
-    get_env_host,
-    get_env_port,
-)
-from phoenix.datasets.evaluators.utils import (
+from phoenix.config import get_base_url, get_env_client_headers
+from phoenix.evals.executors import get_executor_on_sync_context
+from phoenix.evals.models.rate_limiters import RateLimiter
+from phoenix.evals.utils import get_tqdm_progress_bar_formatter
+from phoenix.experiments.evaluators import create_evaluator
+from phoenix.experiments.evaluators.base import (
     Evaluator,
-    EvaluatorName,
     ExperimentEvaluator,
-    create_evaluator,
 )
-from phoenix.datasets.tracing import capture_spans
-from phoenix.datasets.types import (
+from phoenix.experiments.tracing import capture_spans
+from phoenix.experiments.types import (
+    DRY_RUN,
     Dataset,
+    EvaluationParameters,
     EvaluationResult,
+    EvaluationSummary,
+    EvaluatorName,
     Example,
     Experiment,
     ExperimentEvaluationRun,
+    ExperimentParameters,
     ExperimentResult,
     ExperimentRun,
-    ExperimentRunId,
     ExperimentTask,
+    RanExperiment,
+    TaskSummary,
     TestCase,
+    _asdict,
+    _replace,
 )
-from phoenix.evals.executors import get_executor_on_sync_context
-from phoenix.evals.models.rate_limiters import RateLimiter
-from phoenix.evals.utils import get_tqdm_progress_bar_formatter
-from phoenix.session.session import active_session
+from phoenix.experiments.utils import get_dataset_experiments_url, get_experiment_url
 from phoenix.trace.attributes import flatten
 from phoenix.utilities.json import jsonify
 
 
-def _get_base_url() -> str:
-    host = get_env_host()
-    if host == "0.0.0.0":
-        host = "127.0.0.1"
-    base_url = get_env_collector_endpoint() or f"http://{host}:{get_env_port()}"
-    return base_url if base_url.endswith("/") else base_url + "/"
-
-
-def _get_web_base_url() -> str:
-    """Return the web UI base URL.
-
-    Returns:
-        str: the web UI base URL
-    """
-    if session := active_session():
-        return session.url
-    return _get_base_url()
-
-
-def _get_experiment_url(*, dataset_id: str, experiment_id: str) -> str:
-    return f"{_get_web_base_url()}datasets/{dataset_id}/compare?experimentId={experiment_id}"
-
-
-def _get_dataset_experiments_url(*, dataset_id: str) -> str:
-    return f"{_get_web_base_url()}datasets/{dataset_id}/experiments"
-
-
 def _phoenix_clients() -> Tuple[httpx.Client, httpx.AsyncClient]:
     headers = get_env_client_headers()
     return httpx.Client(
-        base_url=_get_base_url(),
+        base_url=get_base_url(),
         headers=headers,
     ), httpx.AsyncClient(
-        base_url=_get_base_url(),
+        base_url=get_base_url(),
         headers=headers,
     )
 
@@ -114,16 +90,23 @@ Evaluators: TypeAlias = Union[
 ]
 
 
+RateLimitErrors: TypeAlias = Union[Type[BaseException], Sequence[Type[BaseException]]]
+
+
 def run_experiment(
     dataset: Dataset,
     task: ExperimentTask,
+    evaluators: Optional[Evaluators] = None,
     *,
     experiment_name: Optional[str] = None,
     experiment_description: Optional[str] = None,
     experiment_metadata: Optional[Mapping[str, Any]] = None,
-    evaluators: Optional[Evaluators] = None,
-    rate_limit_errors: Optional[Union[Type[BaseException], Tuple[Type[BaseException], ...]]] = None,
-) -> Experiment:
+    rate_limit_errors: Optional[RateLimitErrors] = None,
+    dry_run: Union[bool, int] = False,
+    print_summary: bool = True,
+) -> RanExperiment:
+    if not dataset.examples:
+        raise ValueError(f"Dataset has no examples: {dataset.id=}, {dataset.version_id=}")
     # Add this to the params once supported in the UI
     repetitions = 1
     assert repetitions > 0, "Must run the experiment at least once."
@@ -131,44 +114,60 @@ def run_experiment(
 
     sync_client, async_client = _phoenix_clients()
 
-    experiment_response = sync_client.post(
-        f"/v1/datasets/{dataset.id}/experiments",
-        json={
-            "version-id": dataset.version_id,
-            "name": experiment_name,
-            "description": experiment_description,
-            "metadata": experiment_metadata,
-            "repetitions": repetitions,
-        },
-    )
-    experiment_response.raise_for_status()
-    exp_json = experiment_response.json()
-    experiment_id = exp_json["id"]
-    project_name = exp_json["project_name"]
+    payload = {
+        "version_id": dataset.version_id,
+        "name": experiment_name,
+        "description": experiment_description,
+        "metadata": experiment_metadata,
+        "repetitions": repetitions,
+    }
+    if not dry_run:
+        experiment_response = sync_client.post(
+            f"/v1/datasets/{dataset.id}/experiments",
+            json=payload,
+        )
+        experiment_response.raise_for_status()
+        exp_json = experiment_response.json()["data"]
+        project_name = exp_json["project_name"]
+        experiment = Experiment(
+            dataset_id=dataset.id,
+            dataset_version_id=dataset.version_id,
+            repetitions=repetitions,
+            id=exp_json["id"],
+            project_name=project_name,
+        )
+    else:
+        experiment = Experiment(
+            dataset_id=dataset.id,
+            dataset_version_id=dataset.version_id,
+            repetitions=repetitions,
+            id=DRY_RUN,
+            project_name="",
+        )
 
-    resource = Resource({ResourceAttributes.PROJECT_NAME: project_name} if project_name else {})
-    tracer_provider = trace_sdk.TracerProvider(resource=resource)
-    tracer_provider.add_span_processor(
-        SimpleSpanProcessor(OTLPSpanExporter(urljoin(f"{_get_base_url()}", "v1/traces")))
-    )
-    tracer = tracer_provider.get_tracer(__name__)
+    tracer, resource = _get_tracer(experiment.project_name)
     root_span_name = f"Task: {_get_task_name(task)}"
     root_span_kind = CHAIN
 
-    dataset_experiments_url = _get_dataset_experiments_url(dataset_id=dataset.id)
-    experiment_compare_url = _get_experiment_url(dataset_id=dataset.id, experiment_id=experiment_id)
     print("🧪 Experiment started.")
-    print(f"📺 View dataset experiments: {dataset_experiments_url}")
-    print(f"🔗 View this experiment: {experiment_compare_url}")
-
-    errors: Tuple[Optional[Type[BaseException]], ...]
-    if not hasattr(rate_limit_errors, "__iter__"):
-        errors = (rate_limit_errors,)
+    if dry_run:
+        examples = {
+            (ex := dataset[i]).id: ex
+            for i in pd.Series(range(len(dataset)))
+            .sample(min(len(dataset), int(dry_run)), random_state=42)
+            .sort_values()
+        }
+        id_selection = "\n".join(examples)
+        print(f"🌵️ This is a dry-run for these example IDs:\n{id_selection}")
+        dataset = replace(dataset, examples=examples)
     else:
-        rate_limit_errors = cast(Tuple[Type[BaseException], ...], rate_limit_errors)
-        errors = rate_limit_errors
-
-    rate_limiters = [RateLimiter(rate_limit_error=rate_limit_error) for rate_limit_error in errors]
+        dataset_experiments_url = get_dataset_experiments_url(dataset_id=dataset.id)
+        experiment_compare_url = get_experiment_url(
+            dataset_id=dataset.id,
+            experiment_id=experiment.id,
+        )
+        print(f"📺 View dataset experiments: {dataset_experiments_url}")
+        print(f"🔗 View this experiment: {experiment_compare_url}")
 
     def sync_run_experiment(test_case: TestCase) -> ExperimentRun:
         example, repetition_number = test_case.example, test_case.repetition_number
@@ -193,6 +192,7 @@ def run_experiment(
                 span.record_exception(exc)
                 status = Status(StatusCode.ERROR, f"{type(exc).__name__}: {exc}")
                 error = exc
+            output = jsonify(output)
             span.set_attribute(INPUT_VALUE, json.dumps(example.input, ensure_ascii=False))
             span.set_attribute(INPUT_MIME_TYPE, JSON.value)
             if result := ExperimentResult(result=output) if output is not None else None:
@@ -208,21 +208,21 @@ def run_experiment(
         assert isinstance(
             output, (dict, list, str, int, float, bool, type(None))
         ), "Output must be JSON serializable"
-        experiment_run = ExperimentRun(
+        exp_run = ExperimentRun(
             start_time=_decode_unix_nano(cast(int, span.start_time)),
             end_time=_decode_unix_nano(cast(int, span.end_time)),
-            experiment_id=experiment_id,
+            experiment_id=experiment.id,
             dataset_example_id=example.id,
             repetition_number=repetition_number,
             output=result,
             error=repr(error) if error else None,
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
-        resp = sync_client.post(
-            f"/v1/experiments/{experiment_id}/runs", json=jsonify(experiment_run)
-        )
-        resp.raise_for_status()
-        return experiment_run
+        if not dry_run:
+            resp = sync_client.post(f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run))
+            resp.raise_for_status()
+            exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+        return exp_run
 
     async def async_run_experiment(test_case: TestCase) -> ExperimentRun:
         example, repetition_number = test_case.example, test_case.repetition_number
@@ -247,6 +247,7 @@ def run_experiment(
                 span.record_exception(exc)
                 status = Status(StatusCode.ERROR, f"{type(exc).__name__}: {exc}")
                 error = exc
+            output = jsonify(output)
             span.set_attribute(INPUT_VALUE, json.dumps(example.input, ensure_ascii=False))
             span.set_attribute(INPUT_MIME_TYPE, JSON.value)
             if result := ExperimentResult(result=output) if output is not None else None:
@@ -262,21 +263,31 @@ def run_experiment(
         assert isinstance(
             output, (dict, list, str, int, float, bool, type(None))
         ), "Output must be JSON serializable"
-        experiment_run = ExperimentRun(
+        exp_run = ExperimentRun(
             start_time=_decode_unix_nano(cast(int, span.start_time)),
             end_time=_decode_unix_nano(cast(int, span.end_time)),
-            experiment_id=experiment_id,
+            experiment_id=experiment.id,
             dataset_example_id=example.id,
             repetition_number=repetition_number,
             output=result,
             error=repr(error) if error else None,
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
-        resp = await async_client.post(
-            f"/v1/experiments/{experiment_id}/runs", json=jsonify(experiment_run)
-        )
-        resp.raise_for_status()
-        return experiment_run
+        if not dry_run:
+            resp = await async_client.post(
+                f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run)
+            )
+            resp.raise_for_status()
+            exp_run = replace(exp_run, id=resp.json()["data"]["id"])
+        return exp_run
+
+    _errors: Tuple[Type[BaseException], ...]
+    if not hasattr(rate_limit_errors, "__iter__"):
+        _errors = (rate_limit_errors,) if rate_limit_errors is not None else ()
+    else:
+        rate_limit_errors = cast(Sequence[Type[BaseException]], rate_limit_errors)
+        _errors = tuple(filter(None, rate_limit_errors))
+    rate_limiters = [RateLimiter(rate_limit_error=rate_limit_error) for rate_limit_error in _errors]
 
     rate_limited_sync_run_experiment = functools.reduce(
         lambda fn, limiter: limiter.limit(fn), rate_limiters, sync_run_experiment
@@ -295,84 +306,97 @@ def run_experiment(
     )
 
     test_cases = [
-        TestCase(example=ex, repetition_number=rep)
-        for ex, rep in product(dataset.examples, range(1, repetitions + 1))
+        TestCase(example=deepcopy(ex), repetition_number=rep)
+        for ex, rep in product(dataset.examples.values(), range(1, repetitions + 1))
     ]
-    _, _execution_details = executor.run(test_cases)
-    experiment = Experiment(
-        id=experiment_id,
-        dataset_id=dataset.id,
-        dataset_version_id=dataset.version_id,
-        project_name=project_name,
-    )
-
+    task_runs, _execution_details = executor.run(test_cases)
     print("✅ Task runs completed.")
-
+    params = ExperimentParameters(n_examples=len(dataset.examples), n_repetitions=repetitions)
+    task_summary = TaskSummary.from_task_runs(params, task_runs)
+    ran_experiment: RanExperiment = object.__new__(RanExperiment)
+    ran_experiment.__init__(  # type: ignore[misc]
+        params=params,
+        dataset=dataset,
+        runs={r.id: r for r in task_runs},
+        task_summary=task_summary,
+        **_asdict(experiment),
+    )
     if evaluators_by_name:
-        _evaluate_experiment(
-            experiment,
+        return evaluate_experiment(
+            ran_experiment,
             evaluators=evaluators_by_name,
-            dataset_examples=dataset.examples,
-            clients=(sync_client, async_client),
+            dry_run=dry_run,
+            print_summary=print_summary,
+            rate_limit_errors=rate_limit_errors,
         )
-
-    return experiment
+    if print_summary:
+        print(ran_experiment)
+    return ran_experiment
 
 
 def evaluate_experiment(
     experiment: Experiment,
-    evaluators: Union[
-        ExperimentEvaluator,
-        Sequence[ExperimentEvaluator],
-        Mapping[EvaluatorName, ExperimentEvaluator],
-    ],
-) -> None:
+    evaluators: Evaluators,
+    *,
+    dry_run: Union[bool, int] = False,
+    print_summary: bool = True,
+    rate_limit_errors: Optional[RateLimitErrors] = None,
+) -> RanExperiment:
+    if not dry_run and _is_dry_run(experiment):
+        dry_run = True
+    evaluators_by_name = _evaluators_by_name(evaluators)
+    if not evaluators_by_name:
+        raise ValueError("Must specify at least one Evaluator")
     sync_client, async_client = _phoenix_clients()
     dataset_id = experiment.dataset_id
     dataset_version_id = experiment.dataset_version_id
-
-    dataset_examples = [
-        Example.from_dict(ex)
-        for ex in (
+    if isinstance(experiment, RanExperiment):
+        ran_experiment: RanExperiment = experiment
+    else:
+        dataset = Dataset.from_dict(
             sync_client.get(
                 f"/v1/datasets/{dataset_id}/examples",
-                params={"version-id": str(dataset_version_id)},
-            )
-            .json()
-            .get("data", {})
-            .get("examples", [])
+                params={"version_id": str(dataset_version_id)},
+            ).json()["data"]
         )
-    ]
-    _evaluate_experiment(
-        experiment,
-        evaluators=evaluators,
-        dataset_examples=dataset_examples,
-        clients=(sync_client, async_client),
-    )
-
-
-def _evaluate_experiment(
-    experiment: Experiment,
-    *,
-    evaluators: Evaluators,
-    dataset_examples: Iterable[Example],
-    clients: Tuple[httpx.Client, httpx.AsyncClient],
-) -> None:
-    evaluators_by_name = _evaluators_by_name(evaluators)
-    if not evaluators_by_name:
-        raise ValueError("Must specify at least one Evaluator")
-    experiment_id = experiment.id
-    sync_client, async_client = clients
-    experiment_runs = [
-        ExperimentRun.from_dict(exp_run)
-        for exp_run in sync_client.get(f"/v1/experiments/{experiment_id}/runs").json()
-    ]
-
+        if not dataset.examples:
+            raise ValueError(f"Dataset has no examples: {dataset_id=}, {dataset_version_id=}")
+        experiment_runs = tuple(
+            ExperimentRun.from_dict(exp_run)
+            for exp_run in sync_client.get(f"/v1/experiments/{experiment.id}/runs").json()["data"]
+        )
+        if not experiment_runs:
+            raise ValueError("Experiment has not been run")
+        params = ExperimentParameters(n_examples=len(dataset.examples))
+        task_summary = TaskSummary.from_task_runs(params, experiment_runs)
+        ran_experiment = object.__new__(RanExperiment)
+        ran_experiment.__init__(  # type: ignore[misc]
+            dataset=dataset,
+            params=params,
+            runs=experiment_runs,
+            task_summary=task_summary,
+            **_asdict(experiment),
+        )
+    print("🧠 Evaluation started.")
+    examples = ran_experiment.dataset.examples
+    if dry_run:
+        if not _is_dry_run(ran_experiment):
+            dataset = ran_experiment.dataset
+            examples = {
+                (ex := dataset[i]).id: ex
+                for i in pd.Series(range(len(dataset)))
+                .sample(min(len(dataset), int(dry_run)), random_state=42)
+                .sort_values()
+            }
+            dataset = replace(ran_experiment.dataset, examples=examples)
+            ran_experiment = _replace(ran_experiment, id=DRY_RUN, dataset=dataset)
+        id_selection = "\n".join(examples)
+        print(f"🌵️ This is a dry-run for these example IDs:\n{id_selection}")
     # not all dataset examples have associated experiment runs, so we need to pair them up
     example_run_pairs = []
-    examples_by_id = {example.id: example for example in dataset_examples}
-    for exp_run in experiment_runs:
-        example = examples_by_id.get(exp_run.dataset_example_id)
+    examples = ran_experiment.dataset.examples
+    for exp_run in ran_experiment.runs.values():
+        example = examples.get(exp_run.dataset_example_id)
         if example:
             example_run_pairs.append((deepcopy(example), exp_run))
     evaluation_input = [
@@ -380,13 +404,7 @@ def _evaluate_experiment(
         for (example, run), evaluator in product(example_run_pairs, evaluators_by_name.values())
     ]
 
-    project_name = "evaluators"
-    resource = Resource({ResourceAttributes.PROJECT_NAME: project_name} if project_name else {})
-    tracer_provider = trace_sdk.TracerProvider(resource=resource)
-    tracer_provider.add_span_processor(
-        SimpleSpanProcessor(OTLPSpanExporter(urljoin(f"{_get_base_url()}", "v1/traces")))
-    )
-    tracer = tracer_provider.get_tracer(__name__)
+    tracer, resource = _get_tracer(None if dry_run else "evaluators")
     root_span_kind = EVALUATOR
 
     def sync_evaluate_run(
@@ -404,7 +422,7 @@ def _evaluate_experiment(
             stack.enter_context(capture_spans(resource))
             try:
                 result = evaluator.evaluate(
-                    output=None if experiment_run.output is None else experiment_run.output.result,
+                    output=experiment_run.task_output,
                     expected=example.output,
                     input=example.input,
                     metadata=example.metadata,
@@ -418,8 +436,8 @@ def _evaluate_experiment(
             span.set_attribute(OPENINFERENCE_SPAN_KIND, root_span_kind)
             span.set_status(status)
 
-        evaluator_payload = ExperimentEvaluationRun(
-            experiment_run_id=cast(ExperimentRunId, experiment_run.id),
+        eval_run = ExperimentEvaluationRun(
+            experiment_run_id=experiment_run.id,
             start_time=_decode_unix_nano(cast(int, span.start_time)),
             end_time=_decode_unix_nano(cast(int, span.end_time)),
             name=evaluator.name,
@@ -428,9 +446,11 @@ def _evaluate_experiment(
             result=result,
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
-        resp = sync_client.post("/v1/experiment_evaluations", json=jsonify(evaluator_payload))
-        resp.raise_for_status()
-        return evaluator_payload
+        if not dry_run:
+            resp = sync_client.post("/v1/experiment_evaluations", json=jsonify(eval_run))
+            resp.raise_for_status()
+            eval_run = replace(eval_run, id=resp.json()["data"]["id"])
+        return eval_run
 
     async def async_evaluate_run(
         obj: Tuple[Example, ExperimentRun, Evaluator],
@@ -447,7 +467,7 @@ def _evaluate_experiment(
             stack.enter_context(capture_spans(resource))
             try:
                 result = await evaluator.async_evaluate(
-                    output=None if experiment_run.output is None else experiment_run.output.result,
+                    output=experiment_run.task_output,
                     expected=example.output,
                     input=example.input,
                     metadata=example.metadata,
@@ -461,8 +481,8 @@ def _evaluate_experiment(
             span.set_attribute(OPENINFERENCE_SPAN_KIND, root_span_kind)
             span.set_status(status)
 
-        evaluator_payload = ExperimentEvaluationRun(
-            experiment_run_id=cast(ExperimentRunId, experiment_run.id),
+        eval_run = ExperimentEvaluationRun(
+            experiment_run_id=experiment_run.id,
             start_time=_decode_unix_nano(cast(int, span.start_time)),
             end_time=_decode_unix_nano(cast(int, span.end_time)),
             name=evaluator.name,
@@ -471,22 +491,47 @@ def _evaluate_experiment(
             result=result,
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
-        resp = await async_client.post(
-            "/v1/experiment_evaluations", json=jsonify(evaluator_payload)
-        )
-        resp.raise_for_status()
-        return evaluator_payload
+        if not dry_run:
+            resp = await async_client.post("/v1/experiment_evaluations", json=jsonify(eval_run))
+            resp.raise_for_status()
+            eval_run = replace(eval_run, id=resp.json()["data"]["id"])
+        return eval_run
+
+    _errors: Tuple[Type[BaseException], ...]
+    if not hasattr(rate_limit_errors, "__iter__"):
+        _errors = (rate_limit_errors,) if rate_limit_errors is not None else ()
+    else:
+        rate_limit_errors = cast(Sequence[Type[BaseException]], rate_limit_errors)
+        _errors = tuple(filter(None, rate_limit_errors))
+    rate_limiters = [RateLimiter(rate_limit_error=rate_limit_error) for rate_limit_error in _errors]
+
+    rate_limited_sync_evaluate_run = functools.reduce(
+        lambda fn, limiter: limiter.limit(fn), rate_limiters, sync_evaluate_run
+    )
+    rate_limited_async_evaluate_run = functools.reduce(
+        lambda fn, limiter: limiter.alimit(fn), rate_limiters, async_evaluate_run
+    )
 
     executor = get_executor_on_sync_context(
-        sync_evaluate_run,
-        async_evaluate_run,
+        rate_limited_sync_evaluate_run,
+        rate_limited_async_evaluate_run,
         max_retries=0,
         exit_on_error=False,
         fallback_return_value=None,
         tqdm_bar_format=get_tqdm_progress_bar_formatter("running experiment evaluations"),
     )
-    print("🧠 Evaluation started.")
-    _, _execution_details = executor.run(evaluation_input)
+    eval_runs, _execution_details = executor.run(evaluation_input)
+    eval_summary = EvaluationSummary.from_eval_runs(
+        EvaluationParameters(
+            eval_names=frozenset(evaluators_by_name),
+            exp_params=ran_experiment.params,
+        ),
+        *eval_runs,
+    )
+    ran_experiment = ran_experiment.add(eval_summary, *eval_runs)
+    if print_summary:
+        print(ran_experiment)
+    return ran_experiment
 
 
 def _evaluators_by_name(obj: Optional[Evaluators]) -> Mapping[EvaluatorName, Evaluator]:
@@ -519,6 +564,18 @@ def _evaluators_by_name(obj: Optional[Evaluators]) -> Mapping[EvaluatorName, Eva
     return evaluators_by_name
 
 
+def _get_tracer(project_name: Optional[str] = None) -> Tuple[Tracer, Resource]:
+    resource = Resource({ResourceAttributes.PROJECT_NAME: project_name} if project_name else {})
+    tracer_provider = trace_sdk.TracerProvider(resource=resource)
+    span_processor = (
+        SimpleSpanProcessor(OTLPSpanExporter(urljoin(f"{get_base_url()}", "v1/traces")))
+        if project_name
+        else _NoOpProcessor()
+    )
+    tracer_provider.add_span_processor(span_processor)
+    return tracer_provider.get_tracer(__name__), resource
+
+
 def _str_trace_id(id_: int) -> str:
     return hexlify(id_.to_bytes(16, "big")).decode()
 
@@ -539,6 +596,15 @@ def _get_task_name(task: ExperimentTask) -> str:
     return str(task)
 
 
+def _is_dry_run(obj: Any) -> bool:
+    return hasattr(obj, "id") and isinstance(obj.id, str) and obj.id.startswith(DRY_RUN)
+
+
+class _NoOpProcessor(trace_sdk.SpanProcessor):
+    def force_flush(self, *_: Any) -> bool:
+        return True
+
+
 INPUT_VALUE = SpanAttributes.INPUT_VALUE
 OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE
 INPUT_MIME_TYPE = SpanAttributes.INPUT_MIME_TYPE