arize-phoenix 4.4.4rc4__tar.gz → 4.4.4rc6__tar.gz

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.4.4rc4
+Version: 4.4.4rc6
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -20,7 +20,7 @@ Requires-Python: <3.13,>=3.8
 Requires-Dist: aioitertools
 Requires-Dist: aiosqlite
 Requires-Dist: alembic<2,>=1.3.0
-Requires-Dist: arize-phoenix-evals>=0.3.0
+Requires-Dist: arize-phoenix-evals>=0.13.1
 Requires-Dist: cachetools
 Requires-Dist: grpcio
 Requires-Dist: hdbscan>=0.8.33
@@ -31,12 +31,12 @@ Requires-Dist: openinference-instrumentation
 Requires-Dist: openinference-instrumentation-langchain>=0.1.12
 Requires-Dist: openinference-instrumentation-llama-index>=1.2.0
 Requires-Dist: openinference-instrumentation-openai>=0.1.4
-Requires-Dist: openinference-semantic-conventions>=0.1.5
+Requires-Dist: openinference-semantic-conventions>=0.1.9
 Requires-Dist: opentelemetry-exporter-otlp
 Requires-Dist: opentelemetry-proto>=1.12.0
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: opentelemetry-semantic-conventions
-Requires-Dist: pandas
+Requires-Dist: pandas>=1.0
 Requires-Dist: protobuf<6.0,>=3.20
 Requires-Dist: psutil
 Requires-Dist: pyarrow
@@ -79,6 +79,7 @@ Requires-Dist: llama-index>=0.10.3; extra == 'dev'
 Requires-Dist: nbqa; extra == 'dev'
 Requires-Dist: pandas-stubs==2.0.3.230814; (python_version < '3.9') and extra == 'dev'
 Requires-Dist: pandas-stubs==2.2.2.240603; (python_version >= '3.9') and extra == 'dev'
+Requires-Dist: pandas>=1.0; extra == 'dev'
 Requires-Dist: pre-commit; extra == 'dev'
 Requires-Dist: prometheus-client; extra == 'dev'
 Requires-Dist: psycopg[binary]; extra == 'dev'
@@ -88,10 +89,15 @@ Requires-Dist: pytest-postgresql; extra == 'dev'
 Requires-Dist: pytest==8.2.2; extra == 'dev'
 Requires-Dist: ruff==0.4.9; extra == 'dev'
 Requires-Dist: strawberry-graphql[debug-server,opentelemetry]==0.235.0; extra == 'dev'
+Requires-Dist: tabulate; extra == 'dev'
+Requires-Dist: types-tabulate; extra == 'dev'
 Provides-Extra: evals
 Provides-Extra: experimental
 Provides-Extra: llama-index
-Requires-Dist: llama-index==0.10.44; extra == 'llama-index'
+Requires-Dist: llama-index-embeddings-openai; extra == 'llama-index'
+Requires-Dist: llama-index-llms-openai; extra == 'llama-index'
+Requires-Dist: llama-index-readers-file; extra == 'llama-index'
+Requires-Dist: llama-index==0.10.51; extra == 'llama-index'
 Provides-Extra: pg
 Requires-Dist: asyncpg; extra == 'pg'
 Description-Content-Type: text/markdown
@@ -127,7 +133,7 @@ Description-Content-Type: text/markdown
 
 Phoenix is an open-source AI observability platform designed for experimentation, evaluation, and troubleshooting. It provides:
 
--   **_Tracing_** - Trace your LLM application's runtime using using OpenTelemetry-based instrumentation.
+-   **_Tracing_** - Trace your LLM application's runtime using OpenTelemetry-based instrumentation.
 -   **_Evaluation_** - Leverage LLMs to benchmark your application's performance using response and retrieval evals.
 -   **_Inference Analysis_** - Visualize inferences and embeddings using dimensionality reduction and clustering to identify drift and performance degradation.
 

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/README.md

@@ -29,7 +29,7 @@
 
 Phoenix is an open-source AI observability platform designed for experimentation, evaluation, and troubleshooting. It provides:
 
--   **_Tracing_** - Trace your LLM application's runtime using using OpenTelemetry-based instrumentation.
+-   **_Tracing_** - Trace your LLM application's runtime using OpenTelemetry-based instrumentation.
 -   **_Evaluation_** - Leverage LLMs to benchmark your application's performance using response and retrieval evals.
 -   **_Inference Analysis_** - Visualize inferences and embeddings using dimensionality reduction and clustering to identify drift and performance degradation.
 

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/examples/manually-instrumented-chatbot/frontend/requirements.txt

@@ -87,7 +87,7 @@ referencing==0.34.0
     # via
     #   jsonschema
     #   jsonschema-specifications
-requests==2.31.0
+requests==2.32.2
     # via streamlit
 rich==13.7.1
     # via streamlit
@@ -110,7 +110,7 @@ toml==0.10.2
     # via streamlit
 toolz==0.12.1
     # via altair
-tornado==6.4
+tornado==6.4.1
     # via streamlit
 typing-extensions==4.11.0
     # via

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
 dependencies = [
   "scikit-learn",
   "numpy<2",  # https://github.com/scikit-learn-contrib/hdbscan/issues/642
-  "pandas",
+  "pandas>=1.0",
   "jinja2",
   "umap-learn",
   "hdbscan>=0.8.33",
@@ -46,7 +46,7 @@ dependencies = [
   "opentelemetry-proto>=1.12.0",  # needed to avoid this issue: https://github.com/Arize-ai/phoenix/issues/2695
   "opentelemetry-exporter-otlp",
   "opentelemetry-semantic-conventions",
-  "openinference-semantic-conventions>=0.1.5",
+  "openinference-semantic-conventions>=0.1.9",
   "openinference-instrumentation",
   "openinference-instrumentation-langchain>=0.1.12",
   "openinference-instrumentation-llama-index>=1.2.0",
@@ -58,7 +58,7 @@ dependencies = [
   "sqlean.py>=3.45.1",
   "cachetools",
   "python-multipart",  # see https://www.starlette.io/#dependencies
-  "arize-phoenix-evals>=0.3.0",
+  "arize-phoenix-evals>=0.13.1",
 ]
 dynamic = ["version"]
 
@@ -69,6 +69,9 @@ dev = [
   "jupyter",
   "nbqa",
   "ruff==0.4.9",
+  "pandas>=1.0",
+  "tabulate",  # used by DataFrame.to_markdown()
+  "types-tabulate",
   "pandas-stubs==2.2.2.240603; python_version>='3.9'",
   "pandas-stubs==2.0.3.230814; python_version<'3.9'",
   "pytest==8.2.2",
@@ -90,7 +93,10 @@ dev = [
 evals = []
 experimental = []
 llama-index = [
-  "llama-index==0.10.44",  # always pin to a version that keeps our notebooks working
+  "llama-index==0.10.51",  # always pin to a version that keeps our notebooks working
+  "llama-index-readers-file",
+  "llama-index-llms-openai",
+  "llama-index-embeddings-openai",
 ]
 pg = [
   "asyncpg",
@@ -163,7 +169,9 @@ dependencies = [
 dependencies = [
   "mypy==1.10.0",
   "tenacity",
+  "pandas>=1.0",
   "pandas-stubs==2.0.3.230814",
+  "types-tabulate",
   "types-psutil",
   "types-tqdm",
   "types-protobuf",
@@ -206,6 +214,7 @@ dependencies = [
 [tool.hatch.envs.docs]
 detached = true
 dependencies = [
+  "pyment",
   "interrogate",
 ]
 
@@ -336,6 +345,7 @@ disallow_untyped_defs = true
 disallow_incomplete_defs  = true
 strict = true
 exclude = [
+  "api_reference",
   "packages/",
   "src/phoenix/evals/",
   "dist/",
@@ -357,6 +367,7 @@ module = [
   "wrapt",
   "langchain.*",
   "litellm",
+  "litellm.*",
   "nest_asyncio",
   "opentelemetry.*",
   "pyarrow",

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/src/phoenix/config.py

@@ -233,4 +233,25 @@ def get_env_client_headers() -> Optional[Dict[str, str]]:
     return None
 
 
+def get_base_url() -> str:
+    host = get_env_host()
+    if host == "0.0.0.0":
+        host = "127.0.0.1"
+    base_url = get_env_collector_endpoint() or f"http://{host}:{get_env_port()}"
+    return base_url if base_url.endswith("/") else base_url + "/"
+
+
+def get_web_base_url() -> str:
+    """Return the web UI base URL.
+
+    Returns:
+        str: the web UI base URL
+    """
+    from phoenix.session.session import active_session
+
+    if session := active_session():
+        return session.url
+    return get_base_url()
+
+
 DEFAULT_PROJECT_NAME = "default"

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/src/phoenix/datetime_utils.py

@@ -14,6 +14,10 @@ from pandas.core.dtypes.common import (
 _LOCAL_TIMEZONE = datetime.now(timezone.utc).astimezone().tzinfo
 
 
+def local_now() -> datetime:
+    return datetime.now(timezone.utc).astimezone(tz=_LOCAL_TIMEZONE)
+
+
 def normalize_datetime(
     dt: Optional[datetime],
     tz: Optional[tzinfo] = None,

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/src/phoenix/db/insertion/dataset.py

@@ -1,17 +1,17 @@
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from enum import Enum
 from itertools import chain
 from typing import (
     Any,
     Awaitable,
+    Dict,
     FrozenSet,
     Iterable,
     Iterator,
     Mapping,
     Optional,
-    Sequence,
     Union,
     cast,
 )
@@ -30,7 +30,16 @@ DatasetVersionId: TypeAlias = int
 DatasetExampleId: TypeAlias = int
 DatasetExampleRevisionId: TypeAlias = int
 SpanRowId: TypeAlias = int
-Examples: TypeAlias = Iterable[Mapping[str, Any]]
+
+
+@dataclass(frozen=True)
+class ExampleContent:
+    input: Dict[str, Any] = field(default_factory=dict)
+    output: Dict[str, Any] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+Examples: TypeAlias = Iterable[ExampleContent]
 
 
 @dataclass(frozen=True)
@@ -149,14 +158,10 @@ async def add_dataset_examples(
     session: AsyncSession,
     name: str,
     examples: Union[Examples, Awaitable[Examples]],
-    input_keys: Sequence[str],
-    output_keys: Sequence[str],
-    metadata_keys: Sequence[str] = (),
     description: Optional[str] = None,
     metadata: Optional[Mapping[str, Any]] = None,
     action: DatasetAction = DatasetAction.CREATE,
 ) -> Optional[DatasetExampleAdditionEvent]:
-    keys = DatasetKeys(frozenset(input_keys), frozenset(output_keys), frozenset(metadata_keys))
     created_at = datetime.now(timezone.utc)
     dataset_id: Optional[DatasetId] = None
     if action is DatasetAction.APPEND and name:
@@ -173,9 +178,7 @@ async def add_dataset_examples(
                 created_at=created_at,
             )
         except Exception:
-            logger.exception(
-                f"Fail to insert dataset: {input_keys=}, {output_keys=}, {metadata_keys=}"
-            )
+            logger.exception(f"Failed to insert dataset: {name=}")
             raise
     try:
         dataset_version_id = await insert_dataset_version(
@@ -184,7 +187,7 @@ async def add_dataset_examples(
             created_at=created_at,
         )
     except Exception:
-        logger.exception(f"Fail to insert dataset version for {dataset_id=}")
+        logger.exception(f"Failed to insert dataset version for {dataset_id=}")
         raise
     for example in (await examples) if isinstance(examples, Awaitable) else examples:
         try:
@@ -194,21 +197,21 @@ async def add_dataset_examples(
                 created_at=created_at,
             )
         except Exception:
-            logger.exception(f"Fail to insert dataset example for {dataset_id=}")
+            logger.exception(f"Failed to insert dataset example for {dataset_id=}")
             raise
         try:
             await insert_dataset_example_revision(
                 session=session,
                 dataset_version_id=dataset_version_id,
                 dataset_example_id=dataset_example_id,
-                input={key: example.get(key) for key in keys.input},
-                output={key: example.get(key) for key in keys.output},
-                metadata={key: example.get(key) for key in keys.metadata},
+                input=example.input,
+                output=example.output,
+                metadata=example.metadata,
                 created_at=created_at,
             )
         except Exception:
             logger.exception(
-                f"Fail to insert dataset example revision for {dataset_version_id=}, "
+                f"Failed to insert dataset example revision for {dataset_version_id=}, "
                 f"{dataset_example_id=}"
             )
             raise

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/src/phoenix/db/insertion/evaluation.py

@@ -6,7 +6,7 @@ from typing_extensions import assert_never
 
 from phoenix.db import models
 from phoenix.db.helpers import SupportedSQLDialect, num_docs_col
-from phoenix.db.insertion.helpers import OnConflict, insert_stmt
+from phoenix.db.insertion.helpers import OnConflict, insert_on_conflict
 from phoenix.exceptions import PhoenixException
 from phoenix.trace import v1 as pb
 
@@ -91,7 +91,7 @@ async def _insert_trace_evaluation(
     set_.pop("metadata_")
     set_["metadata"] = values["metadata_"]  # `metadata` must match database
     await session.execute(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.TraceAnnotation,
             values=values,
@@ -139,7 +139,7 @@ async def _insert_span_evaluation(
     set_.pop("metadata_")
     set_["metadata"] = values["metadata_"]  # `metadata` must match database
     await session.execute(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.SpanAnnotation,
             values=values,
@@ -196,7 +196,7 @@ async def _insert_document_evaluation(
     set_.pop("metadata_")
     set_["metadata"] = values["metadata_"]  # `metadata` must match database
     await session.execute(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.DocumentAnnotation,
             values=values,

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/src/phoenix/db/insertion/helpers.py

@@ -2,7 +2,7 @@ from abc import ABC
 from enum import Enum, auto
 from typing import Any, Awaitable, Callable, Mapping, Optional, Sequence
 
-from sqlalchemy import Insert, insert
+from sqlalchemy import Insert
 from sqlalchemy.dialects.postgresql import insert as insert_postgresql
 from sqlalchemy.dialects.sqlite import insert as insert_sqlite
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -25,26 +25,18 @@ class OnConflict(Enum):
     DO_UPDATE = auto()
 
 
-def insert_stmt(
+def insert_on_conflict(
     dialect: SupportedSQLDialect,
     table: Any,
     values: Mapping[str, Any],
-    constraint: Optional[str] = None,
-    column_names: Sequence[str] = (),
+    constraint: str,
+    column_names: Sequence[str],
     on_conflict: OnConflict = OnConflict.DO_NOTHING,
     set_: Optional[Mapping[str, Any]] = None,
 ) -> Insert:
     """
     Dialect specific insertion statement using ON CONFLICT DO syntax.
     """
-    if bool(constraint) != bool(column_names):
-        raise ValueError(
-            "Both `constraint` and `column_names` must be provided or omitted at the same time."
-        )
-    if (dialect is SupportedSQLDialect.POSTGRESQL and constraint is None) or (
-        dialect is SupportedSQLDialect.SQLITE and not column_names
-    ):
-        return insert(table).values(values)
     if dialect is SupportedSQLDialect.POSTGRESQL:
         stmt_postgresql = insert_postgresql(table).values(values)
         if on_conflict is OnConflict.DO_NOTHING or not set_:

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/src/phoenix/db/insertion/span.py

@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from phoenix.db import models
 from phoenix.db.helpers import SupportedSQLDialect
-from phoenix.db.insertion.helpers import OnConflict, insert_stmt
+from phoenix.db.insertion.helpers import OnConflict, insert_on_conflict
 from phoenix.trace.attributes import get_attribute_value
 from phoenix.trace.schemas import Span, SpanStatusCode
 
@@ -27,7 +27,7 @@ async def insert_span(
 ) -> Optional[SpanInsertionEvent]:
     dialect = SupportedSQLDialect(session.bind.dialect.name)
     project_rowid = await session.scalar(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.Project,
             constraint="uq_projects_name",
@@ -87,7 +87,7 @@ async def insert_span(
         cumulative_llm_token_count_prompt += cast(int, accumulation[1] or 0)
         cumulative_llm_token_count_completion += cast(int, accumulation[2] or 0)
     span_rowid = await session.scalar(
-        insert_stmt(
+        insert_on_conflict(
             dialect=dialect,
             table=models.Span,
             constraint="uq_spans_span_id",

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/src/phoenix/db/migrations/versions/10460e46d750_datasets.py

@@ -72,7 +72,7 @@ def upgrade() -> None:
         sa.Column(
             "span_rowid",
             sa.Integer,
-            sa.ForeignKey("spans.id"),
+            sa.ForeignKey("spans.id", ondelete="SET NULL"),
             nullable=True,
             index=True,
         ),
@@ -198,7 +198,7 @@ def upgrade() -> None:
             sa.String,
             nullable=True,
         ),
-        sa.Column("output", JSON_, nullable=True),
+        sa.Column("output", JSON_, nullable=False),
         sa.Column("start_time", sa.TIMESTAMP(timezone=True), nullable=False),
         sa.Column("end_time", sa.TIMESTAMP(timezone=True), nullable=False),
         sa.Column(

{arize_phoenix-4.4.4rc4 → arize_phoenix-4.4.4rc6}/src/phoenix/db/models.py

@@ -1,5 +1,5 @@
 from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, TypedDict
 
 from sqlalchemy import (
     JSON,
@@ -91,6 +91,10 @@ class UtcTimeStamp(TypeDecorator[datetime]):
         return normalize_datetime(value, timezone.utc)
 
 
+class ExperimentResult(TypedDict, total=False):
+    result: Any
+
+
 class Base(DeclarativeBase):
     # Enforce best practices for naming constraints
     # https://alembic.sqlalchemy.org/en/latest/naming.html#integration-of-naming-conventions-into-operations-autogenerate
@@ -106,6 +110,7 @@ class Base(DeclarativeBase):
     type_annotation_map = {
         Dict[str, Any]: JsonDict,
         List[Dict[str, Any]]: JsonList,
+        ExperimentResult: JsonDict,
     }
 
 
@@ -483,7 +488,7 @@ class DatasetExample(Base):
         index=True,
     )
     span_rowid: Mapped[Optional[int]] = mapped_column(
-        ForeignKey("spans.id"),
+        ForeignKey("spans.id", ondelete="SET NULL"),
         index=True,
         nullable=True,
     )
@@ -556,7 +561,7 @@ class ExperimentRun(Base):
     )
     repetition_number: Mapped[int]
     trace_id: Mapped[Optional[str]]
-    output: Mapped[Optional[Dict[str, Any]]]
+    output: Mapped[ExperimentResult]
     start_time: Mapped[datetime] = mapped_column(UtcTimeStamp)
     end_time: Mapped[datetime] = mapped_column(UtcTimeStamp)
     prompt_token_count: Mapped[Optional[int]]

arize_phoenix-4.4.4rc6/src/phoenix/experiments/__init__.py

@@ -0,0 +1,6 @@
+from .functions import evaluate_experiment, run_experiment
+
+__all__ = [
+    "evaluate_experiment",
+    "run_experiment",
+]

arize_phoenix-4.4.4rc6/src/phoenix/experiments/evaluators/__init__.py

@@ -0,0 +1,29 @@
+from phoenix.experiments.evaluators.code_evaluators import (
+    ContainsAllKeywords,
+    ContainsAnyKeyword,
+    ContainsKeyword,
+    JSONParsable,
+    MatchesRegex,
+)
+from phoenix.experiments.evaluators.llm_evaluators import (
+    CoherenceEvaluator,
+    ConcisenessEvaluator,
+    HelpfulnessEvaluator,
+    LLMCriteriaEvaluator,
+    RelevanceEvaluator,
+)
+from phoenix.experiments.evaluators.utils import create_evaluator
+
+__all__ = [
+    "create_evaluator",
+    "ContainsAllKeywords",
+    "ContainsAnyKeyword",
+    "ContainsKeyword",
+    "JSONParsable",
+    "MatchesRegex",
+    "CoherenceEvaluator",
+    "ConcisenessEvaluator",
+    "LLMCriteriaEvaluator",
+    "HelpfulnessEvaluator",
+    "RelevanceEvaluator",
+]

arize_phoenix-4.4.4rc6/src/phoenix/experiments/evaluators/base.py

@@ -0,0 +1,153 @@
+import functools
+import inspect
+from abc import ABC
+from types import MappingProxyType
+from typing import Any, Awaitable, Callable, Optional, Union
+
+from typing_extensions import TypeAlias
+
+from phoenix.experiments.evaluators.utils import validate_signature
+from phoenix.experiments.types import (
+    AnnotatorKind,
+    EvaluationResult,
+    EvaluatorKind,
+    EvaluatorName,
+    EvaluatorOutput,
+    ExampleInput,
+    ExampleMetadata,
+    ExampleOutput,
+    TaskOutput,
+)
+
+
+class Evaluator(ABC):
+    """
+    A helper super class to guide the implementation of an `Evaluator` object.
+    Subclasses must implement either the `evaluate` or `async_evaluate` method.
+    Implementing both methods is recommended, but not required.
+
+    This Class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind: AnnotatorKind
+    _name: EvaluatorName
+
+    @functools.cached_property
+    def name(self) -> EvaluatorName:
+        if hasattr(self, "_name"):
+            return self._name
+        return self.__class__.__name__
+
+    @functools.cached_property
+    def kind(self) -> EvaluatorKind:
+        if hasattr(self, "_kind"):
+            return self._kind.value
+        return AnnotatorKind.CODE.value
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "Evaluator":
+        if cls is Evaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+    def evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        expected: Optional[ExampleOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **kwargs: Any,
+    ) -> EvaluationResult:
+        # For subclassing, one should implement either this sync method or the
+        # async version. Implementing both is recommended but not required.
+        raise NotImplementedError
+
+    async def async_evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        expected: Optional[ExampleOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **kwargs: Any,
+    ) -> EvaluationResult:
+        # For subclassing, one should implement either this async method or the
+        # sync version. Implementing both is recommended but not required.
+        return self.evaluate(
+            output=output,
+            expected=expected,
+            metadata=metadata,
+            input=input,
+            **kwargs,
+        )
+
+    def __init_subclass__(cls, is_abstract: bool = False, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+        if is_abstract:
+            return
+        evaluate_fn_signature = inspect.signature(Evaluator.evaluate)
+        for super_cls in inspect.getmro(cls):
+            if super_cls in (LLMEvaluator, Evaluator):
+                break
+            if evaluate := super_cls.__dict__.get(Evaluator.evaluate.__name__):
+                assert callable(evaluate), "`evaluate()` method should be callable"
+                # need to remove the first param, i.e. `self`
+                _validate_sig(functools.partial(evaluate, None), "evaluate")
+                return
+            if async_evaluate := super_cls.__dict__.get(Evaluator.async_evaluate.__name__):
+                assert callable(async_evaluate), "`async_evaluate()` method should be callable"
+                # need to remove the first param, i.e. `self`
+                _validate_sig(functools.partial(async_evaluate, None), "async_evaluate")
+                return
+        raise ValueError(
+            f"Evaluator must implement either "
+            f"`def evaluate{evaluate_fn_signature}` or "
+            f"`async def async_evaluate{evaluate_fn_signature}`"
+        )
+
+
+def _validate_sig(fn: Callable[..., Any], fn_name: str) -> None:
+    sig = inspect.signature(fn)
+    validate_signature(sig)
+    for param in sig.parameters.values():
+        if param.kind is inspect.Parameter.VAR_KEYWORD:
+            return
+    else:
+        raise ValueError(f"`{fn_name}` should allow variadic keyword arguments `**kwargs`")
+
+
+class CodeEvaluator(Evaluator, ABC, is_abstract=True):
+    """
+    A convenience super class for defining code evaluators.
+
+    This class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind = AnnotatorKind.CODE
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "CodeEvaluator":
+        if cls is CodeEvaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+
+class LLMEvaluator(Evaluator, ABC, is_abstract=True):
+    """
+    A convenience super class for defining LLM evaluators.
+
+    This class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind = AnnotatorKind.LLM
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "LLMEvaluator":
+        if cls is LLMEvaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+
+ExperimentEvaluator: TypeAlias = Union[
+    Evaluator,
+    Callable[..., EvaluatorOutput],
+    Callable[..., Awaitable[EvaluatorOutput]],
+]