arize-phoenix 4.4.4rc6__py3-none-any.whl → 4.5.0__py3-none-any.whl

phoenix/db/insertion/dataset.py

@@ -1,237 +0,0 @@
-import logging
-from dataclasses import dataclass, field
-from datetime import datetime, timezone
-from enum import Enum
-from itertools import chain
-from typing import (
-    Any,
-    Awaitable,
-    Dict,
-    FrozenSet,
-    Iterable,
-    Iterator,
-    Mapping,
-    Optional,
-    Union,
-    cast,
-)
-
-from sqlalchemy import insert, select
-from sqlalchemy.ext.asyncio import AsyncSession
-from typing_extensions import TypeAlias
-
-from phoenix.db import models
-from phoenix.db.insertion.helpers import DataManipulationEvent
-
-logger = logging.getLogger(__name__)
-
-DatasetId: TypeAlias = int
-DatasetVersionId: TypeAlias = int
-DatasetExampleId: TypeAlias = int
-DatasetExampleRevisionId: TypeAlias = int
-SpanRowId: TypeAlias = int
-
-
-@dataclass(frozen=True)
-class ExampleContent:
-    input: Dict[str, Any] = field(default_factory=dict)
-    output: Dict[str, Any] = field(default_factory=dict)
-    metadata: Dict[str, Any] = field(default_factory=dict)
-
-
-Examples: TypeAlias = Iterable[ExampleContent]
-
-
-@dataclass(frozen=True)
-class DatasetExampleAdditionEvent(DataManipulationEvent):
-    dataset_id: DatasetId
-
-
-async def insert_dataset(
-    session: AsyncSession,
-    name: str,
-    description: Optional[str] = None,
-    metadata: Optional[Mapping[str, Any]] = None,
-    created_at: Optional[datetime] = None,
-) -> DatasetId:
-    id_ = await session.scalar(
-        insert(models.Dataset)
-        .values(
-            name=name,
-            description=description,
-            metadata_=metadata,
-            created_at=created_at,
-        )
-        .returning(models.Dataset.id)
-    )
-    return cast(DatasetId, id_)
-
-
-async def insert_dataset_version(
-    session: AsyncSession,
-    dataset_id: DatasetId,
-    description: Optional[str] = None,
-    metadata: Optional[Mapping[str, Any]] = None,
-    created_at: Optional[datetime] = None,
-) -> DatasetVersionId:
-    id_ = await session.scalar(
-        insert(models.DatasetVersion)
-        .values(
-            dataset_id=dataset_id,
-            description=description,
-            metadata_=metadata,
-            created_at=created_at,
-        )
-        .returning(models.DatasetVersion.id)
-    )
-    return cast(DatasetVersionId, id_)
-
-
-async def insert_dataset_example(
-    session: AsyncSession,
-    dataset_id: DatasetId,
-    span_rowid: Optional[SpanRowId] = None,
-    created_at: Optional[datetime] = None,
-) -> DatasetExampleId:
-    id_ = await session.scalar(
-        insert(models.DatasetExample)
-        .values(
-            dataset_id=dataset_id,
-            span_rowid=span_rowid,
-            created_at=created_at,
-        )
-        .returning(models.DatasetExample.id)
-    )
-    return cast(DatasetExampleId, id_)
-
-
-class RevisionKind(Enum):
-    CREATE = "CREATE"
-    PATCH = "PATCH"
-    DELETE = "DELETE"
-
-    @classmethod
-    def _missing_(cls, v: Any) -> "RevisionKind":
-        if isinstance(v, str) and v and v.isascii() and not v.isupper():
-            return cls(v.upper())
-        raise ValueError(f"Invalid revision kind: {v}")
-
-
-async def insert_dataset_example_revision(
-    session: AsyncSession,
-    dataset_version_id: DatasetVersionId,
-    dataset_example_id: DatasetExampleId,
-    input: Mapping[str, Any],
-    output: Mapping[str, Any],
-    metadata: Optional[Mapping[str, Any]] = None,
-    revision_kind: RevisionKind = RevisionKind.CREATE,
-    created_at: Optional[datetime] = None,
-) -> DatasetExampleRevisionId:
-    id_ = await session.scalar(
-        insert(models.DatasetExampleRevision)
-        .values(
-            dataset_version_id=dataset_version_id,
-            dataset_example_id=dataset_example_id,
-            input=input,
-            output=output,
-            metadata_=metadata,
-            revision_kind=revision_kind.value,
-            created_at=created_at,
-        )
-        .returning(models.DatasetExampleRevision.id)
-    )
-    return cast(DatasetExampleRevisionId, id_)
-
-
-class DatasetAction(Enum):
-    CREATE = "create"
-    APPEND = "append"
-
-    @classmethod
-    def _missing_(cls, v: Any) -> "DatasetAction":
-        if isinstance(v, str) and v and v.isascii() and not v.islower():
-            return cls(v.lower())
-        raise ValueError(f"Invalid dateset action: {v}")
-
-
-async def add_dataset_examples(
-    session: AsyncSession,
-    name: str,
-    examples: Union[Examples, Awaitable[Examples]],
-    description: Optional[str] = None,
-    metadata: Optional[Mapping[str, Any]] = None,
-    action: DatasetAction = DatasetAction.CREATE,
-) -> Optional[DatasetExampleAdditionEvent]:
-    created_at = datetime.now(timezone.utc)
-    dataset_id: Optional[DatasetId] = None
-    if action is DatasetAction.APPEND and name:
-        dataset_id = await session.scalar(
-            select(models.Dataset.id).where(models.Dataset.name == name)
-        )
-    if action is DatasetAction.CREATE or dataset_id is None:
-        try:
-            dataset_id = await insert_dataset(
-                session=session,
-                name=name,
-                description=description,
-                metadata=metadata,
-                created_at=created_at,
-            )
-        except Exception:
-            logger.exception(f"Failed to insert dataset: {name=}")
-            raise
-    try:
-        dataset_version_id = await insert_dataset_version(
-            session=session,
-            dataset_id=dataset_id,
-            created_at=created_at,
-        )
-    except Exception:
-        logger.exception(f"Failed to insert dataset version for {dataset_id=}")
-        raise
-    for example in (await examples) if isinstance(examples, Awaitable) else examples:
-        try:
-            dataset_example_id = await insert_dataset_example(
-                session=session,
-                dataset_id=dataset_id,
-                created_at=created_at,
-            )
-        except Exception:
-            logger.exception(f"Failed to insert dataset example for {dataset_id=}")
-            raise
-        try:
-            await insert_dataset_example_revision(
-                session=session,
-                dataset_version_id=dataset_version_id,
-                dataset_example_id=dataset_example_id,
-                input=example.input,
-                output=example.output,
-                metadata=example.metadata,
-                created_at=created_at,
-            )
-        except Exception:
-            logger.exception(
-                f"Failed to insert dataset example revision for {dataset_version_id=}, "
-                f"{dataset_example_id=}"
-            )
-            raise
-    return DatasetExampleAdditionEvent(dataset_id=dataset_id)
-
-
-@dataclass(frozen=True)
-class DatasetKeys:
-    input: FrozenSet[str]
-    output: FrozenSet[str]
-    metadata: FrozenSet[str]
-
-    def __iter__(self) -> Iterator[str]:
-        yield from sorted(set(chain(self.input, self.output, self.metadata)))
-
-    def check_differences(self, column_headers_set: FrozenSet[str]) -> None:
-        for category, keys in (
-            ("input", self.input),
-            ("output", self.output),
-            ("metadata", self.metadata),
-        ):
-            if diff := keys.difference(column_headers_set):
-                raise ValueError(f"{category} keys not found in table column headers: {diff}")

phoenix/db/migrations/types.py

@@ -1,29 +0,0 @@
-from typing import Any
-
-from sqlalchemy import JSON
-from sqlalchemy.dialects import postgresql
-from sqlalchemy.ext.compiler import compiles
-
-
-class JSONB(JSON):
-    # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
-    __visit_name__ = "JSONB"
-
-
-@compiles(JSONB, "sqlite")  # type: ignore
-def _(*args: Any, **kwargs: Any) -> str:
-    # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
-    return "JSONB"
-
-
-JSON_ = (
-    JSON()
-    .with_variant(
-        postgresql.JSONB(),  # type: ignore
-        "postgresql",
-    )
-    .with_variant(
-        JSONB(),
-        "sqlite",
-    )
-)

phoenix/db/migrations/versions/10460e46d750_datasets.py

@@ -1,291 +0,0 @@
-"""datasets
-
-Revision ID: 10460e46d750
-Revises: cf03bd6bae1d
-Create Date: 2024-05-10 11:24:23.985834
-
-"""
-
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from phoenix.db.migrations.types import JSON_
-
-# revision identifiers, used by Alembic.
-revision: str = "10460e46d750"
-down_revision: Union[str, None] = "cf03bd6bae1d"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "datasets",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column("name", sa.String, nullable=False, unique=True),
-        sa.Column("description", sa.String, nullable=True),
-        sa.Column("metadata", JSON_, nullable=False),
-        sa.Column(
-            "created_at",
-            sa.TIMESTAMP(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column(
-            "updated_at",
-            sa.TIMESTAMP(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-            onupdate=sa.func.now(),
-        ),
-    )
-    op.create_table(
-        "dataset_versions",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "dataset_id",
-            sa.Integer,
-            sa.ForeignKey("datasets.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("description", sa.String, nullable=True),
-        sa.Column("metadata", JSON_, nullable=False),
-        sa.Column(
-            "created_at",
-            sa.TIMESTAMP(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-    op.create_table(
-        "dataset_examples",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "dataset_id",
-            sa.Integer,
-            sa.ForeignKey("datasets.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column(
-            "span_rowid",
-            sa.Integer,
-            sa.ForeignKey("spans.id", ondelete="SET NULL"),
-            nullable=True,
-            index=True,
-        ),
-        sa.Column(
-            "created_at",
-            sa.TIMESTAMP(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-    op.create_table(
-        "dataset_example_revisions",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "dataset_example_id",
-            sa.Integer,
-            sa.ForeignKey("dataset_examples.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column(
-            "dataset_version_id",
-            sa.Integer,
-            sa.ForeignKey("dataset_versions.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("input", JSON_, nullable=False),
-        sa.Column("output", JSON_, nullable=False),
-        sa.Column("metadata", JSON_, nullable=False),
-        sa.Column(
-            "revision_kind",
-            sa.String,
-            sa.CheckConstraint(
-                "revision_kind IN ('CREATE', 'PATCH', 'DELETE')",
-                name="valid_revision_kind",
-            ),
-            nullable=False,
-        ),
-        sa.Column(
-            "created_at",
-            sa.TIMESTAMP(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.UniqueConstraint(
-            "dataset_example_id",
-            "dataset_version_id",
-        ),
-    )
-    op.create_table(
-        "experiments",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "dataset_id",
-            sa.Integer,
-            sa.ForeignKey("datasets.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column(
-            "dataset_version_id",
-            sa.Integer,
-            sa.ForeignKey("dataset_versions.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column(
-            "name",
-            sa.String,
-            nullable=False,
-        ),
-        sa.Column(
-            "description",
-            sa.String,
-            nullable=True,
-        ),
-        sa.Column(
-            "repetitions",
-            sa.Integer,
-            nullable=False,
-        ),
-        sa.Column("metadata", JSON_, nullable=False),
-        sa.Column("project_name", sa.String, index=True),
-        sa.Column(
-            "created_at",
-            sa.TIMESTAMP(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column(
-            "updated_at",
-            sa.TIMESTAMP(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-            onupdate=sa.func.now(),
-        ),
-    )
-    op.create_table(
-        "experiment_runs",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "experiment_id",
-            sa.Integer,
-            sa.ForeignKey("experiments.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column(
-            "dataset_example_id",
-            sa.Integer,
-            sa.ForeignKey("dataset_examples.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column(
-            "repetition_number",
-            sa.Integer,
-            nullable=False,
-        ),
-        sa.Column(
-            "trace_id",
-            sa.String,
-            nullable=True,
-        ),
-        sa.Column("output", JSON_, nullable=False),
-        sa.Column("start_time", sa.TIMESTAMP(timezone=True), nullable=False),
-        sa.Column("end_time", sa.TIMESTAMP(timezone=True), nullable=False),
-        sa.Column(
-            "prompt_token_count",
-            sa.Integer,
-            nullable=True,
-        ),
-        sa.Column(
-            "completion_token_count",
-            sa.Integer,
-            nullable=True,
-        ),
-        sa.Column(
-            "error",
-            sa.String,
-            nullable=True,
-        ),
-        sa.UniqueConstraint(
-            "experiment_id",
-            "dataset_example_id",
-            "repetition_number",
-        ),
-    )
-    op.create_table(
-        "experiment_run_annotations",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "experiment_run_id",
-            sa.Integer,
-            sa.ForeignKey("experiment_runs.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column(
-            "name",
-            sa.String,
-            nullable=False,
-        ),
-        sa.Column(
-            "annotator_kind",
-            sa.String,
-            sa.CheckConstraint(
-                "annotator_kind IN ('LLM', 'CODE', 'HUMAN')",
-                name="valid_annotator_kind",
-            ),
-            nullable=False,
-        ),
-        sa.Column(
-            "label",
-            sa.String,
-            nullable=True,
-        ),
-        sa.Column(
-            "score",
-            sa.Float,
-            nullable=True,
-        ),
-        sa.Column(
-            "explanation",
-            sa.String,
-            nullable=True,
-        ),
-        sa.Column(
-            "trace_id",
-            sa.String,
-            nullable=True,
-        ),
-        sa.Column(
-            "error",
-            sa.String,
-            nullable=True,
-        ),
-        sa.Column("metadata", JSON_, nullable=False),
-        sa.Column("start_time", sa.TIMESTAMP(timezone=True), nullable=False),
-        sa.Column("end_time", sa.TIMESTAMP(timezone=True), nullable=False),
-        sa.UniqueConstraint(
-            "experiment_run_id",
-            "name",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("experiment_run_annotations")
-    op.drop_table("experiment_runs")
-    op.drop_table("experiments")
-    op.drop_table("dataset_example_revisions")
-    op.drop_table("dataset_examples")
-    op.drop_table("dataset_versions")
-    op.drop_table("datasets")

phoenix/experiments/__init__.py

@@ -1,6 +0,0 @@
-from .functions import evaluate_experiment, run_experiment
-
-__all__ = [
-    "evaluate_experiment",
-    "run_experiment",
-]

phoenix/experiments/evaluators/__init__.py

@@ -1,29 +0,0 @@
-from phoenix.experiments.evaluators.code_evaluators import (
-    ContainsAllKeywords,
-    ContainsAnyKeyword,
-    ContainsKeyword,
-    JSONParsable,
-    MatchesRegex,
-)
-from phoenix.experiments.evaluators.llm_evaluators import (
-    CoherenceEvaluator,
-    ConcisenessEvaluator,
-    HelpfulnessEvaluator,
-    LLMCriteriaEvaluator,
-    RelevanceEvaluator,
-)
-from phoenix.experiments.evaluators.utils import create_evaluator
-
-__all__ = [
-    "create_evaluator",
-    "ContainsAllKeywords",
-    "ContainsAnyKeyword",
-    "ContainsKeyword",
-    "JSONParsable",
-    "MatchesRegex",
-    "CoherenceEvaluator",
-    "ConcisenessEvaluator",
-    "LLMCriteriaEvaluator",
-    "HelpfulnessEvaluator",
-    "RelevanceEvaluator",
-]