arize-phoenix 4.4.4rc5__py3-none-any.whl → 4.5.0__py3-none-any.whl

phoenix/session/session.py

@@ -37,16 +37,10 @@ from phoenix.config import (
     get_exported_files,
     get_working_dir,
 )
-from phoenix.core.model_schema_adapter import create_model_from_inferences
+from phoenix.core.model_schema_adapter import create_model_from_datasets
 from phoenix.inferences.inferences import EMPTY_INFERENCES, Inferences
 from phoenix.pointcloud.umap_parameters import get_umap_parameters
-from phoenix.server.app import (
-    SessionFactory,
-    _db,
-    create_app,
-    create_engine_and_run_migrations,
-    instrument_engine_if_enabled,
-)
+from phoenix.server.app import create_app
 from phoenix.server.thread_server import ThreadServer
 from phoenix.services import AppService
 from phoenix.session.client import Client
@@ -114,9 +108,9 @@ class Session(TraceDataExtractor, ABC):
     def __init__(
         self,
         database_url: str,
-        primary_inferences: Inferences,
-        reference_inferences: Optional[Inferences] = None,
-        corpus_inferences: Optional[Inferences] = None,
+        primary_dataset: Inferences,
+        reference_dataset: Optional[Inferences] = None,
+        corpus_dataset: Optional[Inferences] = None,
         trace_dataset: Optional[TraceDataset] = None,
         default_umap_parameters: Optional[Mapping[str, Any]] = None,
         host: Optional[str] = None,
@@ -124,9 +118,9 @@ class Session(TraceDataExtractor, ABC):
         notebook_env: Optional[NotebookEnvironment] = None,
     ):
         self._database_url = database_url
-        self.primary_inferences = primary_inferences
-        self.reference_inferences = reference_inferences
-        self.corpus_inferences = corpus_inferences
+        self.primary_dataset = primary_dataset
+        self.reference_dataset = reference_dataset
+        self.corpus_dataset = corpus_dataset
         self.trace_dataset = trace_dataset
         self.umap_parameters = get_umap_parameters(default_umap_parameters)
         self.host = host or get_env_host()
@@ -270,9 +264,9 @@ class ProcessSession(Session):
     def __init__(
         self,
         database_url: str,
-        primary_inferences: Inferences,
-        reference_inferences: Optional[Inferences] = None,
-        corpus_inferences: Optional[Inferences] = None,
+        primary_dataset: Inferences,
+        reference_dataset: Optional[Inferences] = None,
+        corpus_dataset: Optional[Inferences] = None,
         trace_dataset: Optional[TraceDataset] = None,
         default_umap_parameters: Optional[Mapping[str, Any]] = None,
         host: Optional[str] = None,
@@ -282,20 +276,20 @@ class ProcessSession(Session):
     ) -> None:
         super().__init__(
             database_url=database_url,
-            primary_inferences=primary_inferences,
-            reference_inferences=reference_inferences,
-            corpus_inferences=corpus_inferences,
+            primary_dataset=primary_dataset,
+            reference_dataset=reference_dataset,
+            corpus_dataset=corpus_dataset,
             trace_dataset=trace_dataset,
             default_umap_parameters=default_umap_parameters,
             host=host,
             port=port,
             notebook_env=notebook_env,
         )
-        primary_inferences.to_disc()
-        if isinstance(reference_inferences, Inferences):
-            reference_inferences.to_disc()
-        if isinstance(corpus_inferences, Inferences):
-            corpus_inferences.to_disc()
+        primary_dataset.to_disc()
+        if isinstance(reference_dataset, Inferences):
+            reference_dataset.to_disc()
+        if isinstance(corpus_dataset, Inferences):
+            corpus_dataset.to_disc()
         if isinstance(trace_dataset, TraceDataset):
             trace_dataset.to_disc()
         umap_params_str = (
@@ -310,13 +304,13 @@ class ProcessSession(Session):
             host=self.host,
             port=self.port,
             root_path=self.root_path,
-            primary_inferences_name=self.primary_inferences.name,
+            primary_dataset_name=self.primary_dataset.name,
             umap_params=umap_params_str,
-            reference_inferences_name=(
-                self.reference_inferences.name if self.reference_inferences is not None else None
+            reference_dataset_name=(
+                self.reference_dataset.name if self.reference_dataset is not None else None
             ),
-            corpus_inferences_name=(
-                self.corpus_inferences.name if self.corpus_inferences is not None else None
+            corpus_dataset_name=(
+                self.corpus_dataset.name if self.corpus_dataset is not None else None
             ),
             trace_dataset_name=(
                 self.trace_dataset.name if self.trace_dataset is not None else None
@@ -336,9 +330,9 @@ class ThreadSession(Session):
     def __init__(
         self,
         database_url: str,
-        primary_inferences: Inferences,
-        reference_inferences: Optional[Inferences] = None,
-        corpus_inferences: Optional[Inferences] = None,
+        primary_dataset: Inferences,
+        reference_dataset: Optional[Inferences] = None,
+        corpus_dataset: Optional[Inferences] = None,
         trace_dataset: Optional[TraceDataset] = None,
         default_umap_parameters: Optional[Mapping[str, Any]] = None,
         host: Optional[str] = None,
@@ -348,32 +342,29 @@ class ThreadSession(Session):
     ):
         super().__init__(
             database_url=database_url,
-            primary_inferences=primary_inferences,
-            reference_inferences=reference_inferences,
-            corpus_inferences=corpus_inferences,
+            primary_dataset=primary_dataset,
+            reference_dataset=reference_dataset,
+            corpus_dataset=corpus_dataset,
             trace_dataset=trace_dataset,
             default_umap_parameters=default_umap_parameters,
             host=host,
             port=port,
             notebook_env=notebook_env,
         )
-        self.model = create_model_from_inferences(
-            primary_inferences,
-            reference_inferences,
+        self.model = create_model_from_datasets(
+            primary_dataset,
+            reference_dataset,
         )
         self.corpus = (
-            create_model_from_inferences(
-                corpus_inferences,
+            create_model_from_datasets(
+                corpus_dataset,
             )
-            if corpus_inferences is not None
+            if corpus_dataset is not None
             else None
         )
         # Initialize an app service that keeps the server running
-        engine = create_engine_and_run_migrations(database_url)
-        instrumentation_cleanups = instrument_engine_if_enabled(engine)
-        factory = SessionFactory(session_factory=_db(engine), dialect=engine.dialect.name)
         self.app = create_app(
-            db=factory,
+            database_url=database_url,
             export_path=self.export_path,
             model=self.model,
             corpus=self.corpus,
@@ -384,7 +375,6 @@ class ThreadSession(Session):
                 if (trace_dataset and (initial_evaluations := trace_dataset.evaluations))
                 else None
             ),
-            clean_up_callbacks=instrumentation_cleanups,
         )
         self.server = ThreadServer(
             app=self.app,

phoenix/trace/exporter.py

@@ -7,17 +7,13 @@ from types import MethodType
 from typing import Any, Optional
 from urllib.parse import urljoin
 
-import httpx
+import requests
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from requests import Session
 from typing_extensions import TypeAlias, assert_never
 
 import phoenix.trace.v1 as pb
-from phoenix.config import (
-    get_env_client_headers,
-    get_env_collector_endpoint,
-    get_env_host,
-    get_env_port,
-)
+from phoenix.config import get_env_collector_endpoint, get_env_host, get_env_port
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.NullHandler())
@@ -79,10 +75,9 @@ class HttpExporter:
         )
         self._base_url = base_url if base_url.endswith("/") else base_url + "/"
         _warn_if_phoenix_is_not_running(self._base_url)
-        headers = get_env_client_headers()
-        self._client = httpx.Client(headers=headers)
-        weakref.finalize(self, self._client.close)
-        self._client.headers.update(
+        self._session = Session()
+        weakref.finalize(self, self._session.close)
+        self._session.headers.update(
             {
                 "content-type": "application/x-protobuf",
                 "content-encoding": "gzip",
@@ -115,9 +110,9 @@ class HttpExporter:
 
     def _send(self, message: Message) -> None:
         serialized = message.SerializeToString()
-        content = gzip.compress(serialized)
+        data = gzip.compress(serialized)
         try:
-            self._client.post(self._url(message), content=content).raise_for_status()
+            self._session.post(self._url(message), data=data).raise_for_status()
         except Exception as e:
             logger.exception(e)
 
@@ -130,7 +125,7 @@ class HttpExporter:
 
 def _warn_if_phoenix_is_not_running(base_url: str) -> None:
     try:
-        httpx.get(urljoin(base_url, "arize_phoenix_version")).raise_for_status()
+        requests.get(urljoin(base_url, "arize_phoenix_version")).raise_for_status()
     except Exception:
         logger.warning(
             f"Arize Phoenix is not running on {base_url}. Launch Phoenix "

phoenix/trace/fixtures.py

@@ -1,29 +1,18 @@
-import logging
-import shutil
 from binascii import hexlify
 from dataclasses import dataclass, field, replace
 from datetime import datetime, timezone
-from io import StringIO
 from random import getrandbits
-from tempfile import NamedTemporaryFile
-from time import sleep, time
-from typing import Dict, Iterable, Iterator, List, NamedTuple, Optional, Sequence, Tuple, cast
+from typing import Dict, Iterable, Iterator, List, NamedTuple, Optional, Tuple, cast
 from urllib import request
-from urllib.parse import urljoin
 
-import httpx
 import pandas as pd
 from google.protobuf.wrappers_pb2 import DoubleValue, StringValue
-from httpx import ConnectError, HTTPStatusError
 
 import phoenix.trace.v1 as pb
-from phoenix import Client
 from phoenix.trace.schemas import Span
 from phoenix.trace.trace_dataset import TraceDataset
 from phoenix.trace.utils import json_lines_to_df
 
-logger = logging.getLogger(__name__)
-
 
 class EvaluationResultSchema(NamedTuple):
     label: Optional[str] = "label"
@@ -43,44 +32,12 @@ class DocumentEvaluationFixture(EvaluationFixture):
     document_position: str = "document_position"
 
 
-@dataclass(frozen=True)
-class DatasetFixture:
-    file_name: str
-    name: str
-    input_keys: Sequence[str]
-    output_keys: Sequence[str]
-    metadata_keys: Sequence[str] = ()
-    description: Optional[str] = field(default=None)
-    _df: Optional[pd.DataFrame] = field(default=None, init=False, repr=False)
-    _csv: Optional[str] = field(default=None, init=False, repr=False)
-
-    def load(self) -> "DatasetFixture":
-        if self._df is None:
-            df = pd.read_csv(_url(self.file_name))
-            object.__setattr__(self, "_df", df)
-        return self
-
-    @property
-    def dataframe(self) -> pd.DataFrame:
-        self.load()
-        return cast(pd.DataFrame, self._df).copy(deep=False)
-
-    @property
-    def csv(self) -> StringIO:
-        if self._csv is None:
-            with StringIO() as buffer:
-                self.dataframe.to_csv(buffer, index=False)
-                object.__setattr__(self, "_csv", buffer.getvalue())
-        return StringIO(self._csv)
-
-
 @dataclass(frozen=True)
 class TracesFixture:
     name: str
     description: str
     file_name: str
     evaluation_fixtures: Iterable[EvaluationFixture] = ()
-    dataset_fixtures: Iterable[DatasetFixture] = ()
 
 
 llama_index_rag_fixture = TracesFixture(
@@ -101,36 +58,6 @@ llama_index_rag_fixture = TracesFixture(
             file_name="llama_index_rag_v8.retrieved_documents_eval.parquet",
         ),
     ),
-    dataset_fixtures=(
-        DatasetFixture(
-            file_name="hybridial_samples.csv.gz",
-            input_keys=("messages", "ctxs"),
-            output_keys=("answers",),
-            name="ChatRAG-Bench: Hybrid Dialogue (samples)",
-            description="https://huggingface.co/datasets/nvidia/ChatRAG-Bench/viewer/hybridial",
-        ),
-        DatasetFixture(
-            file_name="sqa_samples.csv.gz",
-            input_keys=("messages", "ctxs"),
-            output_keys=("answers",),
-            name="ChatRAG-Bench: SQA (samples)",
-            description="https://huggingface.co/datasets/nvidia/ChatRAG-Bench/viewer/sqa",
-        ),
-        DatasetFixture(
-            file_name="doqa_cooking_samples.csv.gz",
-            input_keys=("messages", "ctxs"),
-            output_keys=("answers",),
-            name="ChatRAG-Bench: DoQA Cooking (samples)",
-            description="https://huggingface.co/datasets/nvidia/ChatRAG-Bench/viewer/doqa_cooking",
-        ),
-        DatasetFixture(
-            file_name="synthetic_convqa_samples.csv.gz",
-            input_keys=("messages", "document"),
-            output_keys=("answers",),
-            name="ChatQA-Train: Synthetic ConvQA (samples)",
-            description="https://huggingface.co/datasets/nvidia/ChatQA-Training-Data/viewer/synthetic_convqa",
-        ),
-    ),
 )
 
 llama_index_calculator_agent_fixture = TracesFixture(
@@ -211,69 +138,16 @@ def download_traces_fixture(
         return cast(List[str], f.readlines())
 
 
-def load_example_traces(fixture_name: str) -> TraceDataset:
+def load_example_traces(use_case: str) -> TraceDataset:
     """
     Loads a trace dataframe by name.
     """
-    fixture = get_trace_fixture_by_name(fixture_name)
+    fixture = get_trace_fixture_by_name(use_case)
     return TraceDataset(json_lines_to_df(download_traces_fixture(fixture)))
 
 
-def get_dataset_fixtures(fixture_name: str) -> Iterable[DatasetFixture]:
-    return (fixture.load() for fixture in get_trace_fixture_by_name(fixture_name).dataset_fixtures)
-
-
-def send_dataset_fixtures(
-    endpoint: str,
-    fixtures: Iterable[DatasetFixture],
-) -> None:
-    expiration = time() + 5
-    while time() < expiration:
-        try:
-            url = urljoin(endpoint, "/healthz")
-            httpx.get(url=url).raise_for_status()
-        except ConnectError:
-            sleep(0.1)
-            continue
-        except Exception as e:
-            print(str(e))
-            raise
-        break
-    client = Client(endpoint=endpoint)
-    for i, fixture in enumerate(fixtures):
-        try:
-            if i % 2:
-                client.upload_dataset(
-                    fixture.dataframe,
-                    name=fixture.name,
-                    input_keys=fixture.input_keys,
-                    output_keys=fixture.output_keys,
-                    metadata_keys=fixture.metadata_keys,
-                    description=fixture.description,
-                )
-            else:
-                with NamedTemporaryFile() as tf:
-                    with open(tf.name, "w") as f:
-                        shutil.copyfileobj(fixture.csv, f)
-                        f.flush()
-                    client.upload_dataset(
-                        tf.name,
-                        name=fixture.name,
-                        input_keys=fixture.input_keys,
-                        output_keys=fixture.output_keys,
-                        metadata_keys=fixture.metadata_keys,
-                        description=fixture.description,
-                    )
-        except HTTPStatusError as e:
-            print(e.response.content.decode())
-            pass
-        else:
-            name, df = fixture.name, fixture.dataframe
-            print(f"Dataset sent: {name=}, {len(df)=}")
-
-
-def get_evals_from_fixture(fixture_name: str) -> Iterator[pb.Evaluation]:
-    fixture = get_trace_fixture_by_name(fixture_name)
+def get_evals_from_fixture(use_case: str) -> Iterator[pb.Evaluation]:
+    fixture = get_trace_fixture_by_name(use_case)
     for eval_fixture in fixture.evaluation_fixtures:
         yield from _read_eval_fixture(eval_fixture)
 
@@ -321,8 +195,8 @@ def _read_eval_fixture(eval_fixture: EvaluationFixture) -> Iterator[pb.Evaluatio
 def _url(
     file_name: str,
     host: Optional[str] = "https://storage.googleapis.com/",
-    bucket: Optional[str] = "arize-phoenix-assets",
-    prefix: Optional[str] = "traces/",
+    bucket: Optional[str] = "arize-assets",
+    prefix: Optional[str] = "phoenix/traces/",
 ) -> str:
     return f"{host}{bucket}/{prefix}{file_name}"
 

phoenix/trace/schemas.py

@@ -29,6 +29,8 @@ class SpanKind(Enum):
     """
     SpanKind is loosely inspired by OpenTelemetry's SpanKind
     It captures the type of work that a Span encapsulates.
+
+    NB: this is actively under construction
     """
 
     TOOL = "TOOL"
@@ -38,7 +40,6 @@ class SpanKind(Enum):
     EMBEDDING = "EMBEDDING"
     AGENT = "AGENT"
     RERANKER = "RERANKER"
-    EVALUATOR = "EVALUATOR"
     UNKNOWN = "UNKNOWN"
 
     def __str__(self) -> str:

phoenix/trace/span_evaluations.py

@@ -11,7 +11,7 @@ import pandas as pd
 from pandas.api.types import is_integer_dtype, is_numeric_dtype, is_string_dtype
 from pyarrow import RecordBatchStreamReader, Schema, Table, parquet
 
-from phoenix.config import TRACE_DATASETS_DIR
+from phoenix.config import TRACE_DATASET_DIR
 from phoenix.exceptions import PhoenixEvaluationNameIsMissing
 from phoenix.trace.errors import InvalidParquetMetadataError
 
@@ -201,7 +201,7 @@ class Evaluations(NeedsNamedIndex, NeedsResultColumns, ABC):
             UUID: The ID of the evaluations, which can be used as a key to load
             the evaluations from disk using `load`.
         """
-        directory = Path(directory) if directory else TRACE_DATASETS_DIR
+        directory = Path(directory) if directory else TRACE_DATASET_DIR
         path = directory / EVAL_PARQUET_FILE_NAME.format(id=self.id)
         table = self.to_pyarrow_table()
         parquet.write_table(table, path)
@@ -229,7 +229,7 @@ class Evaluations(NeedsNamedIndex, NeedsResultColumns, ABC):
         """
         if not isinstance(id, UUID):
             id = UUID(id)
-        path = Path(directory or TRACE_DATASETS_DIR) / EVAL_PARQUET_FILE_NAME.format(id=id)
+        path = Path(directory or TRACE_DATASET_DIR) / EVAL_PARQUET_FILE_NAME.format(id=id)
         schema = parquet.read_schema(path)
         eval_id, eval_name, evaluations_cls = _parse_schema_metadata(schema)
         if id != eval_id:

phoenix/trace/trace_dataset.py

@@ -14,7 +14,7 @@ from openinference.semconv.trace import (
 from pandas import DataFrame, read_parquet
 from pyarrow import Schema, Table, parquet
 
-from phoenix.config import GENERATED_INFERENCES_NAME_PREFIX, INFERENCES_DIR, TRACE_DATASETS_DIR
+from phoenix.config import DATASET_DIR, GENERATED_DATASET_NAME_PREFIX, TRACE_DATASET_DIR
 from phoenix.datetime_utils import normalize_timestamps
 from phoenix.trace.attributes import unflatten
 from phoenix.trace.errors import InvalidParquetMetadataError
@@ -138,7 +138,7 @@ class TraceDataset:
         self._id = uuid4()
         self.dataframe = normalize_dataframe(dataframe)
         # TODO: This is not used in any meaningful way. Should remove
-        self.name = name or f"{GENERATED_INFERENCES_NAME_PREFIX}{str(self._id)}"
+        self.name = name or f"{GENERATED_DATASET_NAME_PREFIX}{str(self._id)}"
         self.evaluations = list(evaluations)
 
     @classmethod
@@ -201,13 +201,13 @@ class TraceDataset:
     @classmethod
     def from_name(cls, name: str) -> "TraceDataset":
         """Retrieves a dataset by name from the file system"""
-        directory = INFERENCES_DIR / name
+        directory = DATASET_DIR / name
         df = read_parquet(directory / cls._data_file_name)
         return cls(df, name)
 
     def to_disc(self) -> None:
         """writes the data to disc"""
-        directory = INFERENCES_DIR / self.name
+        directory = DATASET_DIR / self.name
         directory.mkdir(parents=True, exist_ok=True)
         get_serializable_spans_dataframe(self.dataframe).to_parquet(
             directory / self._data_file_name,
@@ -230,7 +230,7 @@ class TraceDataset:
             UUID: The id of the trace dataset, which can be used as key to load
             the dataset from disk using `load`.
         """
-        directory = Path(directory or TRACE_DATASETS_DIR)
+        directory = Path(directory or TRACE_DATASET_DIR)
         for evals in self.evaluations:
             evals.save(directory)
         path = directory / TRACE_DATASET_PARQUET_FILE_NAME.format(id=self._id)
@@ -280,7 +280,7 @@ class TraceDataset:
         """
         if not isinstance(id, UUID):
             id = UUID(id)
-        path = Path(directory or TRACE_DATASETS_DIR) / TRACE_DATASET_PARQUET_FILE_NAME.format(id=id)
+        path = Path(directory or TRACE_DATASET_DIR) / TRACE_DATASET_PARQUET_FILE_NAME.format(id=id)
         schema = parquet.read_schema(path)
         dataset_id, dataset_name, eval_ids = _parse_schema_metadata(schema)
         if id != dataset_id:

phoenix/version.py

@@ -1 +1 @@
-__version__ = "4.4.4rc5"
+__version__ = "4.5.0"

phoenix/datasets/evaluators/__init__.py

@@ -1,18 +0,0 @@
-from phoenix.datasets.evaluators.code_evaluators import ContainsKeyword, JSONParsable
-from phoenix.datasets.evaluators.llm_evaluators import (
-    CoherenceEvaluator,
-    ConcisenessEvaluator,
-    HelpfulnessEvaluator,
-    LLMCriteriaEvaluator,
-    RelevanceEvaluator,
-)
-
-__all__ = [
-    "ContainsKeyword",
-    "JSONParsable",
-    "CoherenceEvaluator",
-    "ConcisenessEvaluator",
-    "LLMCriteriaEvaluator",
-    "HelpfulnessEvaluator",
-    "RelevanceEvaluator",
-]

phoenix/datasets/evaluators/code_evaluators.py

@@ -1,99 +0,0 @@
-from __future__ import annotations
-
-import json
-import re
-from typing import Any, List, Optional, Union
-
-from phoenix.datasets.evaluators.utils import Evaluator
-from phoenix.datasets.types import EvaluationResult, TaskOutput
-
-
-class JSONParsable(Evaluator):
-    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
-        assert isinstance(output, str), "Experiment run output must be a string"
-        try:
-            json.loads(output)
-            json_parsable = True
-        except BaseException:
-            json_parsable = False
-        return EvaluationResult(
-            score=int(json_parsable),
-        )
-
-
-class ContainsKeyword(Evaluator):
-    def __init__(self, keyword: str, name: Optional[str] = None) -> None:
-        self.keyword = keyword
-        self._name = name or f"Contains({repr(keyword)})"
-
-    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
-        assert isinstance(output, str), "Experiment run output must be a string"
-        found = self.keyword in output
-        return EvaluationResult(
-            score=float(found),
-            explanation=(
-                f"the string {repr(self.keyword)} was "
-                f"{'found' if found else 'not found'} in the output"
-            ),
-        )
-
-
-class ContainsAnyKeyword(Evaluator):
-    def __init__(self, keywords: List[str], name: Optional[str] = None) -> None:
-        self.keywords = keywords
-        self._name = name or f"ContainsAny({keywords})"
-
-    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
-        assert isinstance(output, str), "Experiment run output must be a string"
-        found = [keyword for keyword in self.keywords if keyword in output]
-        if found:
-            explanation = f"the keywords {found} were found in the output"
-        else:
-            explanation = f"none of the keywords {self.keywords} were found in the output"
-        return EvaluationResult(
-            score=float(bool(found)),
-            explanation=explanation,
-        )
-
-
-class ContainsAllKeywords(Evaluator):
-    def __init__(self, keywords: List[str], name: Optional[str] = None) -> None:
-        self.keywords = keywords
-        self._name = name or f"ContainsAll({keywords})"
-
-    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
-        assert isinstance(output, str), "Experiment run output must be a string"
-        not_found = [keyword for keyword in self.keywords if keyword not in output]
-        if not_found:
-            contains_all = False
-            explanation = f"the keywords {not_found} were not found in the output"
-        else:
-            contains_all = True
-            explanation = f"all of the keywords {self.keywords} were found in the output"
-        return EvaluationResult(
-            score=float(contains_all),
-            explanation=explanation,
-        )
-
-
-class MatchesRegex(Evaluator):
-    def __init__(self, pattern: Union[str, re.Pattern[str]], name: Optional[str] = None) -> None:
-        if isinstance(pattern, str):
-            pattern = re.compile(pattern)
-        self.pattern = pattern
-        assert isinstance(pattern, re.Pattern)
-        self._name = name or f"matches_({pattern})"
-
-    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
-        assert isinstance(output, str), "Experiment run output must be a string"
-        matches = self.pattern.findall(output)
-        if matches:
-            explanation = (
-                f"the substrings {matches} matched the regex pattern {self.pattern.pattern}"
-            )
-        else:
-            explanation = f"no substrings matched the regex pattern {self.pattern.pattern}"
-        return EvaluationResult(
-            score=float(bool(matches)),
-            explanation=explanation,
-        )