arize-phoenix 4.4.4rc6__py3-none-any.whl → 4.5.0__py3-none-any.whl

phoenix/server/api/routers/v1/experiment_evaluations.py

@@ -1,65 +0,0 @@
-from datetime import datetime
-
-from starlette.requests import Request
-from starlette.responses import JSONResponse, Response
-from starlette.status import HTTP_404_NOT_FOUND
-from strawberry.relay import GlobalID
-
-from phoenix.db import models
-from phoenix.db.helpers import SupportedSQLDialect
-from phoenix.db.insertion.helpers import OnConflict, insert_on_conflict
-from phoenix.server.api.types.node import from_global_id_with_expected_type
-
-
-async def upsert_experiment_evaluation(request: Request) -> Response:
-    payload = await request.json()
-    experiment_run_gid = GlobalID.from_id(payload["experiment_run_id"])
-    try:
-        experiment_run_id = from_global_id_with_expected_type(experiment_run_gid, "ExperimentRun")
-    except ValueError:
-        return Response(
-            content=f"ExperimentRun with ID {experiment_run_gid} does not exist",
-            status_code=HTTP_404_NOT_FOUND,
-        )
-    name = payload["name"]
-    annotator_kind = payload["annotator_kind"]
-    result = payload.get("result")
-    label = result.get("label") if result else None
-    score = result.get("score") if result else None
-    explanation = result.get("explanation") if result else None
-    error = payload.get("error")
-    metadata = payload.get("metadata") or {}
-    start_time = payload["start_time"]
-    end_time = payload["end_time"]
-    async with request.app.state.db() as session:
-        values = dict(
-            experiment_run_id=experiment_run_id,
-            name=name,
-            annotator_kind=annotator_kind,
-            label=label,
-            score=score,
-            explanation=explanation,
-            error=error,
-            metadata_=metadata,  # `metadata_` must match database
-            start_time=datetime.fromisoformat(start_time),
-            end_time=datetime.fromisoformat(end_time),
-            trace_id=payload.get("trace_id"),
-        )
-        set_ = {
-            **{k: v for k, v in values.items() if k != "metadata_"},
-            "metadata": values["metadata_"],  # `metadata` must match database
-        }
-        dialect = SupportedSQLDialect(session.bind.dialect.name)
-        exp_eval_run = await session.scalar(
-            insert_on_conflict(
-                dialect=dialect,
-                table=models.ExperimentRunAnnotation,
-                values=values,
-                constraint="uq_experiment_run_annotations_experiment_run_id_name",
-                column_names=("experiment_run_id", "name"),
-                on_conflict=OnConflict.DO_UPDATE,
-                set_=set_,
-            ).returning(models.ExperimentRunAnnotation)
-        )
-    evaluation_gid = GlobalID("ExperimentEvaluation", str(exp_eval_run.id))
-    return JSONResponse(content={"data": {"id": str(evaluation_gid)}})

phoenix/server/api/routers/v1/experiment_runs.py

@@ -1,96 +0,0 @@
-from datetime import datetime
-
-from sqlalchemy import select
-from starlette.requests import Request
-from starlette.responses import JSONResponse, Response
-from starlette.status import HTTP_404_NOT_FOUND
-from strawberry.relay import GlobalID
-
-from phoenix.db import models
-from phoenix.experiments.types import ExperimentResult, ExperimentRun
-from phoenix.server.api.types.node import from_global_id_with_expected_type
-from phoenix.utilities.json import jsonify
-
-
-async def create_experiment_run(request: Request) -> Response:
-    experiment_gid = GlobalID.from_id(request.path_params["experiment_id"])
-    try:
-        experiment_id = from_global_id_with_expected_type(experiment_gid, "Experiment")
-    except ValueError:
-        return Response(
-            content=f"Experiment with ID {experiment_gid} does not exist",
-            status_code=HTTP_404_NOT_FOUND,
-        )
-
-    payload = await request.json()
-
-    example_gid = GlobalID.from_id(payload["dataset_example_id"])
-    try:
-        dataset_example_id = from_global_id_with_expected_type(example_gid, "DatasetExample")
-    except ValueError:
-        return Response(
-            content=f"DatasetExample with ID {example_gid} does not exist",
-            status_code=HTTP_404_NOT_FOUND,
-        )
-
-    trace_id = payload.get("trace_id", None)
-    output = payload["output"]
-    repetition_number = payload["repetition_number"]
-    start_time = payload["start_time"]
-    end_time = payload["end_time"]
-    error = payload.get("error")
-
-    async with request.app.state.db() as session:
-        exp_run = models.ExperimentRun(
-            experiment_id=experiment_id,
-            dataset_example_id=dataset_example_id,
-            trace_id=trace_id,
-            output=output,
-            repetition_number=repetition_number,
-            start_time=datetime.fromisoformat(start_time),
-            end_time=datetime.fromisoformat(end_time),
-            error=error,
-        )
-        session.add(exp_run)
-        await session.flush()
-    run_gid = GlobalID("ExperimentRun", str(exp_run.id))
-    return JSONResponse(content={"data": {"id": str(run_gid)}})
-
-
-async def list_experiment_runs(request: Request) -> Response:
-    experiment_gid = GlobalID.from_id(request.path_params["experiment_id"])
-    try:
-        experiment_id = from_global_id_with_expected_type(experiment_gid, "Experiment")
-    except ValueError:
-        return Response(
-            content=f"Experiment with ID {experiment_gid} does not exist",
-            status_code=HTTP_404_NOT_FOUND,
-        )
-
-    async with request.app.state.db() as session:
-        experiment_runs = await session.execute(
-            select(models.ExperimentRun)
-            .where(models.ExperimentRun.experiment_id == experiment_id)
-            # order by dataset_example_id to be consistent with `list_dataset_examples`
-            .order_by(models.ExperimentRun.dataset_example_id.asc())
-        )
-        experiment_runs = experiment_runs.scalars().all()
-        runs = []
-        for exp_run in experiment_runs:
-            run_gid = GlobalID("ExperimentRun", str(exp_run.id))
-            experiment_gid = GlobalID("Experiment", str(exp_run.experiment_id))
-            example_gid = GlobalID("DatasetExample", str(exp_run.dataset_example_id))
-            runs.append(
-                ExperimentRun(
-                    start_time=exp_run.start_time,
-                    end_time=exp_run.end_time,
-                    experiment_id=str(experiment_gid),
-                    dataset_example_id=str(example_gid),
-                    repetition_number=exp_run.repetition_number,
-                    output=ExperimentResult.from_dict(exp_run.output) if exp_run.output else None,
-                    error=exp_run.error,
-                    id=str(run_gid),
-                    trace_id=exp_run.trace_id,
-                )
-            )
-    return JSONResponse(content={"data": jsonify(runs)}, status_code=200)

phoenix/server/api/routers/v1/experiments.py

@@ -1,174 +0,0 @@
-from random import getrandbits
-
-from sqlalchemy import select
-from starlette.requests import Request
-from starlette.responses import JSONResponse, Response
-from starlette.status import HTTP_404_NOT_FOUND
-from strawberry.relay import GlobalID
-
-from phoenix.db import models
-from phoenix.db.helpers import SupportedSQLDialect
-from phoenix.db.insertion.helpers import insert_on_conflict
-from phoenix.server.api.types.node import from_global_id_with_expected_type
-
-
-def _short_uuid() -> str:
-    return str(getrandbits(32).to_bytes(4, "big").hex())
-
-
-def _generate_experiment_name(dataset_name: str) -> str:
-    """
-    Generate a semi-unique name for the experiment.
-    """
-    short_ds_name = dataset_name[:8].replace(" ", "-")
-    return f"{short_ds_name}-{_short_uuid()}"
-
-
-async def create_experiment(request: Request) -> Response:
-    dataset_globalid = GlobalID.from_id(request.path_params["dataset_id"])
-    try:
-        dataset_id = from_global_id_with_expected_type(dataset_globalid, "Dataset")
-    except ValueError:
-        return Response(
-            content="Dataset with ID {dataset_globalid} does not exist",
-            status_code=HTTP_404_NOT_FOUND,
-        )
-
-    payload = await request.json()
-    repetitions = payload.get("repetitions", 1)
-    metadata = payload.get("metadata") or {}
-    dataset_version_globalid_str = payload.get("version_id")
-    if dataset_version_globalid_str is not None:
-        try:
-            dataset_version_globalid = GlobalID.from_id(dataset_version_globalid_str)
-            dataset_version_id = from_global_id_with_expected_type(
-                dataset_version_globalid, "DatasetVersion"
-            )
-        except ValueError:
-            return Response(
-                content="DatasetVersion with ID {dataset_version_globalid} does not exist",
-                status_code=HTTP_404_NOT_FOUND,
-            )
-
-    async with request.app.state.db() as session:
-        result = (
-            await session.execute(select(models.Dataset).where(models.Dataset.id == dataset_id))
-        ).scalar()
-        if result is None:
-            return Response(
-                content=f"Dataset with ID {dataset_globalid} does not exist",
-                status_code=HTTP_404_NOT_FOUND,
-            )
-        dataset_name = result.name
-        if dataset_version_globalid_str is None:
-            dataset_version_result = await session.execute(
-                select(models.DatasetVersion)
-                .where(models.DatasetVersion.dataset_id == dataset_id)
-                .order_by(models.DatasetVersion.id.desc())
-            )
-            dataset_version = dataset_version_result.scalar()
-            if not dataset_version:
-                return Response(
-                    content=f"Dataset {dataset_globalid} does not have any versions",
-                    status_code=HTTP_404_NOT_FOUND,
-                )
-            dataset_version_id = dataset_version.id
-            dataset_version_globalid = GlobalID("DatasetVersion", str(dataset_version_id))
-        else:
-            dataset_version = await session.execute(
-                select(models.DatasetVersion).where(models.DatasetVersion.id == dataset_version_id)
-            )
-            dataset_version = dataset_version.scalar()
-            if not dataset_version:
-                return Response(
-                    content=f"DatasetVersion with ID {dataset_version_globalid} does not exist",
-                    status_code=HTTP_404_NOT_FOUND,
-                )
-
-        # generate a semi-unique name for the experiment
-        experiment_name = payload.get("name") or _generate_experiment_name(dataset_name)
-        project_name = f"Experiment-{getrandbits(96).to_bytes(12, 'big').hex()}"
-        project_description = (
-            f"dataset_id: {dataset_globalid}\ndataset_version_id: {dataset_version_globalid}"
-        )
-        experiment = models.Experiment(
-            dataset_id=int(dataset_id),
-            dataset_version_id=int(dataset_version_id),
-            name=experiment_name,
-            description=payload.get("description"),
-            repetitions=repetitions,
-            metadata_=metadata,
-            project_name=project_name,
-        )
-        session.add(experiment)
-        await session.flush()
-
-        dialect = SupportedSQLDialect(session.bind.dialect.name)
-        project_rowid = await session.scalar(
-            insert_on_conflict(
-                dialect=dialect,
-                table=models.Project,
-                constraint="uq_projects_name",
-                column_names=("name",),
-                values=dict(
-                    name=project_name,
-                    description=project_description,
-                    created_at=experiment.created_at,
-                    updated_at=experiment.updated_at,
-                ),
-            ).returning(models.Project.id)
-        )
-        assert project_rowid is not None
-
-        experiment_globalid = GlobalID("Experiment", str(experiment.id))
-        if dataset_version_globalid_str is None:
-            dataset_version_globalid = GlobalID(
-                "DatasetVersion", str(experiment.dataset_version_id)
-            )
-        experiment_payload = {
-            "id": str(experiment_globalid),
-            "dataset_id": str(dataset_globalid),
-            "dataset_version_id": str(dataset_version_globalid),
-            "repetitions": experiment.repetitions,
-            "metadata": experiment.metadata_,
-            "project_name": experiment.project_name,
-            "created_at": experiment.created_at.isoformat(),
-            "updated_at": experiment.updated_at.isoformat(),
-        }
-    return JSONResponse(content={"data": experiment_payload})
-
-
-async def read_experiment(request: Request) -> Response:
-    experiment_globalid = GlobalID.from_id(request.path_params["experiment_id"])
-    try:
-        experiment_id = from_global_id_with_expected_type(experiment_globalid, "Experiment")
-    except ValueError:
-        return Response(
-            content="Experiment with ID {experiment_globalid} does not exist",
-            status_code=HTTP_404_NOT_FOUND,
-        )
-
-    async with request.app.state.db() as session:
-        experiment = await session.execute(
-            select(models.Experiment).where(models.Experiment.id == experiment_id)
-        )
-        experiment = experiment.scalar()
-        if not experiment:
-            return Response(
-                content=f"Experiment with ID {experiment_globalid} does not exist",
-                status_code=HTTP_404_NOT_FOUND,
-            )
-
-        dataset_globalid = GlobalID("Dataset", str(experiment.dataset_id))
-        dataset_version_globalid = GlobalID("DatasetVersion", str(experiment.dataset_version_id))
-        experiment_payload = {
-            "id": str(experiment_globalid),
-            "dataset_id": str(dataset_globalid),
-            "dataset_version_id": str(dataset_version_globalid),
-            "repetitions": experiment.repetitions,
-            "metadata": experiment.metadata_,
-            "project_name": experiment.project_name,
-            "created_at": experiment.created_at.isoformat(),
-            "updated_at": experiment.updated_at.isoformat(),
-        }
-    return JSONResponse(content={"data": experiment_payload})

phoenix/server/api/types/AnnotatorKind.py

@@ -1,10 +0,0 @@
-from enum import Enum
-
-import strawberry
-
-
-@strawberry.enum
-class AnnotatorKind(Enum):
-    LLM = "LLM"
-    HUMAN = "HUMAN"
-    CODE = "CODE"

phoenix/server/api/types/CreateDatasetPayload.py

@@ -1,8 +0,0 @@
-import strawberry
-
-from phoenix.server.api.types.Dataset import Dataset
-
-
-@strawberry.type
-class CreateDatasetPayload:
-    dataset: Dataset

phoenix/server/api/types/DatasetExample.py

@@ -1,85 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-import strawberry
-from sqlalchemy import select
-from sqlalchemy.orm import joinedload
-from strawberry import UNSET
-from strawberry.relay.types import Connection, GlobalID, Node, NodeID
-from strawberry.types import Info
-
-from phoenix.db import models
-from phoenix.server.api.context import Context
-from phoenix.server.api.types.DatasetExampleRevision import DatasetExampleRevision
-from phoenix.server.api.types.DatasetVersion import DatasetVersion
-from phoenix.server.api.types.ExperimentRun import ExperimentRun, to_gql_experiment_run
-from phoenix.server.api.types.node import from_global_id_with_expected_type
-from phoenix.server.api.types.pagination import (
-    ConnectionArgs,
-    CursorString,
-    connection_from_list,
-)
-from phoenix.server.api.types.Span import Span, to_gql_span
-
-
-@strawberry.type
-class DatasetExample(Node):
-    id_attr: NodeID[int]
-    created_at: datetime
-    version_id: strawberry.Private[Optional[int]] = None
-
-    @strawberry.field
-    async def revision(
-        self,
-        info: Info[Context, None],
-        dataset_version_id: Optional[GlobalID] = UNSET,
-    ) -> DatasetExampleRevision:
-        example_id = self.id_attr
-        version_id: Optional[int] = None
-        if dataset_version_id:
-            version_id = from_global_id_with_expected_type(
-                global_id=dataset_version_id, expected_type_name=DatasetVersion.__name__
-            )
-        elif self.version_id is not None:
-            version_id = self.version_id
-        return await info.context.data_loaders.dataset_example_revisions.load(
-            (example_id, version_id)
-        )
-
-    @strawberry.field
-    async def span(
-        self,
-        info: Info[Context, None],
-    ) -> Optional[Span]:
-        return (
-            to_gql_span(span)
-            if (span := await info.context.data_loaders.dataset_example_spans.load(self.id_attr))
-            else None
-        )
-
-    @strawberry.field
-    async def experiment_runs(
-        self,
-        info: Info[Context, None],
-        first: Optional[int] = 50,
-        last: Optional[int] = UNSET,
-        after: Optional[CursorString] = UNSET,
-        before: Optional[CursorString] = UNSET,
-    ) -> Connection[ExperimentRun]:
-        args = ConnectionArgs(
-            first=first,
-            after=after if isinstance(after, CursorString) else None,
-            last=last,
-            before=before if isinstance(before, CursorString) else None,
-        )
-        example_id = self.id_attr
-        query = (
-            select(models.ExperimentRun)
-            .options(joinedload(models.ExperimentRun.trace).load_only(models.Trace.trace_id))
-            .join(models.Experiment, models.Experiment.id == models.ExperimentRun.experiment_id)
-            .where(models.ExperimentRun.dataset_example_id == example_id)
-            .order_by(models.Experiment.id.desc())
-        )
-        async with info.context.db() as session:
-            runs = (await session.scalars(query)).all()
-        return connection_from_list([to_gql_experiment_run(run) for run in runs], args)

phoenix/server/api/types/DatasetExampleRevision.py

@@ -1,34 +0,0 @@
-from datetime import datetime
-from enum import Enum
-
-import strawberry
-
-from phoenix.db import models
-from phoenix.server.api.types.ExampleRevisionInterface import ExampleRevision
-
-
-@strawberry.enum
-class RevisionKind(Enum):
-    CREATE = "CREATE"
-    PATCH = "PATCH"
-    DELETE = "DELETE"
-
-
-@strawberry.type
-class DatasetExampleRevision(ExampleRevision):
-    """
-    Represents a revision (i.e., update or alteration) of a dataset example.
-    """
-
-    revision_kind: RevisionKind
-    created_at: datetime
-
-    @classmethod
-    def from_orm_revision(cls, revision: models.DatasetExampleRevision) -> "DatasetExampleRevision":
-        return cls(
-            input=revision.input,
-            output=revision.output,
-            metadata=revision.metadata_,
-            revision_kind=RevisionKind(revision.revision_kind),
-            created_at=revision.created_at,
-        )

phoenix/server/api/types/DatasetVersion.py

@@ -1,14 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-import strawberry
-from strawberry.relay import Node, NodeID
-from strawberry.scalars import JSON
-
-
-@strawberry.type
-class DatasetVersion(Node):
-    id_attr: NodeID[int]
-    description: Optional[str]
-    metadata: JSON
-    created_at: datetime

phoenix/server/api/types/ExampleRevisionInterface.py

@@ -1,14 +0,0 @@
-import strawberry
-from strawberry.scalars import JSON
-
-
-@strawberry.interface
-class ExampleRevision:
-    """
-    Represents an example revision for generative tasks.
-    For example, you might have text -> text, text -> labels, etc.
-    """
-
-    input: JSON
-    output: JSON
-    metadata: JSON

phoenix/server/api/types/Experiment.py

@@ -1,147 +0,0 @@
-from datetime import datetime
-from typing import List, Optional
-
-import strawberry
-from sqlalchemy import select
-from sqlalchemy.orm import joinedload
-from strawberry import UNSET, Private
-from strawberry.relay import Connection, Node, NodeID
-from strawberry.scalars import JSON
-from strawberry.types import Info
-
-from phoenix.db import models
-from phoenix.server.api.context import Context
-from phoenix.server.api.types.ExperimentAnnotationSummary import ExperimentAnnotationSummary
-from phoenix.server.api.types.ExperimentRun import ExperimentRun, to_gql_experiment_run
-from phoenix.server.api.types.pagination import (
-    ConnectionArgs,
-    CursorString,
-    connection_from_list,
-)
-from phoenix.server.api.types.Project import Project
-
-
-@strawberry.type
-class Experiment(Node):
-    cached_sequence_number: Private[Optional[int]] = None
-    id_attr: NodeID[int]
-    name: str
-    project_name: Optional[str]
-    description: Optional[str]
-    metadata: JSON
-    created_at: datetime
-    updated_at: datetime
-
-    @strawberry.field(
-        description="Sequence number (1-based) of experiments belonging to the same dataset"
-    )  # type: ignore
-    async def sequence_number(
-        self,
-        info: Info[Context, None],
-    ) -> int:
-        if self.cached_sequence_number is None:
-            seq_num = await info.context.data_loaders.experiment_sequence_number.load(self.id_attr)
-            if seq_num is None:
-                raise ValueError(f"invalid experiment: id={self.id_attr}")
-            self.cached_sequence_number = seq_num
-        return self.cached_sequence_number
-
-    @strawberry.field
-    async def runs(
-        self,
-        info: Info[Context, None],
-        first: Optional[int] = 50,
-        last: Optional[int] = UNSET,
-        after: Optional[CursorString] = UNSET,
-        before: Optional[CursorString] = UNSET,
-    ) -> Connection[ExperimentRun]:
-        args = ConnectionArgs(
-            first=first,
-            after=after if isinstance(after, CursorString) else None,
-            last=last,
-            before=before if isinstance(before, CursorString) else None,
-        )
-        experiment_id = self.id_attr
-        async with info.context.db() as session:
-            runs = (
-                await session.scalars(
-                    select(models.ExperimentRun)
-                    .where(models.ExperimentRun.experiment_id == experiment_id)
-                    .order_by(models.ExperimentRun.id.desc())
-                    .options(
-                        joinedload(models.ExperimentRun.trace).load_only(models.Trace.trace_id)
-                    )
-                )
-            ).all()
-        return connection_from_list([to_gql_experiment_run(run) for run in runs], args)
-
-    @strawberry.field
-    async def run_count(self, info: Info[Context, None]) -> int:
-        experiment_id = self.id_attr
-        return await info.context.data_loaders.experiment_run_counts.load(experiment_id)
-
-    @strawberry.field
-    async def annotation_summaries(
-        self, info: Info[Context, None]
-    ) -> List[ExperimentAnnotationSummary]:
-        experiment_id = self.id_attr
-        return [
-            ExperimentAnnotationSummary(
-                annotation_name=summary.annotation_name,
-                min_score=summary.min_score,
-                max_score=summary.max_score,
-                mean_score=summary.mean_score,
-                count=summary.count,
-                error_count=summary.error_count,
-            )
-            for summary in await info.context.data_loaders.experiment_annotation_summaries.load(
-                experiment_id
-            )
-        ]
-
-    @strawberry.field
-    async def error_rate(self, info: Info[Context, None]) -> Optional[float]:
-        return await info.context.data_loaders.experiment_error_rates.load(self.id_attr)
-
-    @strawberry.field
-    async def average_run_latency_ms(self, info: Info[Context, None]) -> float:
-        latency_seconds = await info.context.data_loaders.average_experiment_run_latency.load(
-            self.id_attr
-        )
-        return latency_seconds * 1000
-
-    @strawberry.field
-    async def project(self, info: Info[Context, None]) -> Optional[Project]:
-        if self.project_name is None:
-            return None
-
-        db_project = await info.context.data_loaders.project_by_name.load(self.project_name)
-
-        if db_project is None:
-            return None
-
-        return Project(
-            id_attr=db_project.id,
-            name=db_project.name,
-            gradient_start_color=db_project.gradient_start_color,
-            gradient_end_color=db_project.gradient_end_color,
-        )
-
-
-def to_gql_experiment(
-    experiment: models.Experiment,
-    sequence_number: Optional[int] = None,
-) -> Experiment:
-    """
-    Converts an ORM experiment to a GraphQL Experiment.
-    """
-    return Experiment(
-        cached_sequence_number=sequence_number,
-        id_attr=experiment.id,
-        name=experiment.name,
-        project_name=experiment.project_name,
-        description=experiment.description,
-        metadata=experiment.metadata_,
-        created_at=experiment.created_at,
-        updated_at=experiment.updated_at,
-    )

phoenix/server/api/types/ExperimentAnnotationSummary.py

@@ -1,13 +0,0 @@
-from typing import Optional
-
-import strawberry
-
-
-@strawberry.type
-class ExperimentAnnotationSummary:
-    annotation_name: str
-    min_score: Optional[float]
-    max_score: Optional[float]
-    mean_score: Optional[float]
-    count: int
-    error_count: int

phoenix/server/api/types/ExperimentComparison.py

@@ -1,19 +0,0 @@
-from typing import List
-
-import strawberry
-from strawberry.relay import GlobalID
-
-from phoenix.server.api.types.DatasetExample import DatasetExample
-from phoenix.server.api.types.ExperimentRun import ExperimentRun
-
-
-@strawberry.type
-class RunComparisonItem:
-    experiment_id: GlobalID
-    runs: List[ExperimentRun]
-
-
-@strawberry.type
-class ExperimentComparison:
-    example: DatasetExample
-    run_comparison_items: List[RunComparisonItem]