arize-phoenix 4.5.0__py3-none-any.whl → 4.6.2__py3-none-any.whl

phoenix/server/api/types/ExperimentRun.py

@@ -0,0 +1,91 @@
+from datetime import datetime
+from typing import Optional
+
+import strawberry
+from sqlalchemy import select
+from strawberry import UNSET
+from strawberry.relay import Connection, GlobalID, Node, NodeID
+from strawberry.scalars import JSON
+from strawberry.types import Info
+
+from phoenix.db import models
+from phoenix.server.api.context import Context
+from phoenix.server.api.types.ExperimentRunAnnotation import (
+    ExperimentRunAnnotation,
+    to_gql_experiment_run_annotation,
+)
+from phoenix.server.api.types.pagination import (
+    ConnectionArgs,
+    CursorString,
+    connection_from_list,
+)
+from phoenix.server.api.types.Trace import Trace
+
+
+@strawberry.type
+class ExperimentRun(Node):
+    id_attr: NodeID[int]
+    experiment_id: GlobalID
+    trace_id: Optional[str]
+    output: Optional[JSON]
+    start_time: datetime
+    end_time: datetime
+    error: Optional[str]
+
+    @strawberry.field
+    async def annotations(
+        self,
+        info: Info[Context, None],
+        first: Optional[int] = 50,
+        last: Optional[int] = UNSET,
+        after: Optional[CursorString] = UNSET,
+        before: Optional[CursorString] = UNSET,
+    ) -> Connection[ExperimentRunAnnotation]:
+        args = ConnectionArgs(
+            first=first,
+            after=after if isinstance(after, CursorString) else None,
+            last=last,
+            before=before if isinstance(before, CursorString) else None,
+        )
+        run_id = self.id_attr
+        async with info.context.db() as session:
+            annotations = (
+                await session.scalars(
+                    select(models.ExperimentRunAnnotation)
+                    .where(models.ExperimentRunAnnotation.experiment_run_id == run_id)
+                    .order_by(models.ExperimentRunAnnotation.name.desc())
+                )
+            ).all()
+        return connection_from_list(
+            [to_gql_experiment_run_annotation(annotation) for annotation in annotations], args
+        )
+
+    @strawberry.field
+    async def trace(self, info: Info) -> Optional[Trace]:
+        if not self.trace_id:
+            return None
+        dataloader = info.context.data_loaders.trace_row_ids
+        if (trace := await dataloader.load(self.trace_id)) is None:
+            return None
+        trace_rowid, project_rowid = trace
+        return Trace(id_attr=trace_rowid, trace_id=self.trace_id, project_rowid=project_rowid)
+
+
+def to_gql_experiment_run(run: models.ExperimentRun) -> ExperimentRun:
+    """
+    Converts an ORM experiment run to a GraphQL ExperimentRun.
+    """
+
+    from phoenix.server.api.types.Experiment import Experiment
+
+    return ExperimentRun(
+        id_attr=run.id,
+        experiment_id=GlobalID(Experiment.__name__, str(run.experiment_id)),
+        trace_id=trace_id
+        if (trace := run.trace) and (trace_id := trace.trace_id) is not None
+        else None,
+        output=run.output.get("task_output"),
+        start_time=run.start_time,
+        end_time=run.end_time,
+        error=run.error,
+    )

phoenix/server/api/types/ExperimentRunAnnotation.py

@@ -0,0 +1,57 @@
+from datetime import datetime
+from typing import Optional
+
+import strawberry
+from strawberry import Info
+from strawberry.relay import Node, NodeID
+from strawberry.scalars import JSON
+
+from phoenix.db import models
+from phoenix.server.api.types.AnnotatorKind import AnnotatorKind
+from phoenix.server.api.types.Trace import Trace
+
+
+@strawberry.type
+class ExperimentRunAnnotation(Node):
+    id_attr: NodeID[int]
+    name: str
+    annotator_kind: AnnotatorKind
+    label: Optional[str]
+    score: Optional[float]
+    explanation: Optional[str]
+    error: Optional[str]
+    metadata: JSON
+    start_time: datetime
+    end_time: datetime
+    trace_id: Optional[str]
+
+    @strawberry.field
+    async def trace(self, info: Info) -> Optional[Trace]:
+        if not self.trace_id:
+            return None
+        dataloader = info.context.data_loaders.trace_row_ids
+        if (trace := await dataloader.load(self.trace_id)) is None:
+            return None
+        trace_row_id, project_row_id = trace
+        return Trace(id_attr=trace_row_id, trace_id=self.trace_id, project_rowid=project_row_id)
+
+
+def to_gql_experiment_run_annotation(
+    annotation: models.ExperimentRunAnnotation,
+) -> ExperimentRunAnnotation:
+    """
+    Converts an ORM experiment run annotation to a GraphQL ExperimentRunAnnotation.
+    """
+    return ExperimentRunAnnotation(
+        id_attr=annotation.id,
+        name=annotation.name,
+        annotator_kind=AnnotatorKind(annotation.annotator_kind),
+        label=annotation.label,
+        score=annotation.score,
+        explanation=annotation.explanation,
+        error=annotation.error,
+        metadata=annotation.metadata_,
+        start_time=annotation.start_time,
+        end_time=annotation.end_time,
+        trace_id=annotation.trace_id,
+    )

phoenix/server/api/types/Inferences.py

@@ -0,0 +1,80 @@
+from datetime import datetime
+from typing import Iterable, List, Optional, Set, Union
+
+import strawberry
+from strawberry.scalars import ID
+from strawberry.unset import UNSET
+
+import phoenix.core.model_schema as ms
+from phoenix.core.model_schema import FEATURE, TAG, ScalarDimension
+
+from ..input_types.DimensionInput import DimensionInput
+from .Dimension import Dimension, to_gql_dimension
+from .Event import Event, create_event, create_event_id, parse_event_ids_by_inferences_role
+from .InferencesRole import AncillaryInferencesRole, InferencesRole
+
+
+@strawberry.type
+class Inferences:
+    start_time: datetime = strawberry.field(description="The start bookend of the data")
+    end_time: datetime = strawberry.field(description="The end bookend of the data")
+    record_count: int = strawberry.field(description="The record count of the data")
+    inferences: strawberry.Private[ms.Inferences]
+    inferences_role: strawberry.Private[Union[InferencesRole, AncillaryInferencesRole]]
+    model: strawberry.Private[ms.Model]
+
+    # type ignored here to get around the following: https://github.com/strawberry-graphql/strawberry/issues/1929
+    @strawberry.field(description="Returns a human friendly name for the inferences.")  # type: ignore
+    def name(self) -> str:
+        return self.inferences.display_name
+
+    @strawberry.field
+    def events(
+        self,
+        event_ids: List[ID],
+        dimensions: Optional[List[DimensionInput]] = UNSET,
+    ) -> List[Event]:
+        """
+        Returns events for specific event IDs and dimensions. If no input
+        dimensions are provided, returns all features and tags.
+        """
+        if not event_ids:
+            return []
+        row_ids = parse_event_ids_by_inferences_role(event_ids)
+        if len(row_ids) > 1 or self.inferences_role not in row_ids:
+            raise ValueError("eventIds contains IDs from incorrect inferences.")
+        events = self.inferences[row_ids[self.inferences_role]]
+        requested_gql_dimensions = _get_requested_features_and_tags(
+            core_dimensions=self.model.scalar_dimensions,
+            requested_dimension_names=set(dim.name for dim in dimensions)
+            if isinstance(dimensions, list)
+            else None,
+        )
+        return [
+            create_event(
+                event_id=create_event_id(event.id.row_id, self.inferences_role),
+                event=event,
+                dimensions=requested_gql_dimensions,
+                is_document_record=self.inferences_role is AncillaryInferencesRole.corpus,
+            )
+            for event in events
+        ]
+
+
+def _get_requested_features_and_tags(
+    core_dimensions: Iterable[ScalarDimension],
+    requested_dimension_names: Optional[Set[str]] = UNSET,
+) -> List[Dimension]:
+    """
+    Returns requested features and tags as a list of strawberry Inferences. If no
+    dimensions are explicitly requested, returns all features and tags.
+    """
+    requested_features_and_tags: List[Dimension] = []
+    for id, dim in enumerate(core_dimensions):
+        is_requested = (
+            not isinstance(requested_dimension_names, Set)
+        ) or dim.name in requested_dimension_names
+        is_feature_or_tag = dim.role in (FEATURE, TAG)
+        if is_requested and is_feature_or_tag:
+            requested_features_and_tags.append(to_gql_dimension(id_attr=id, dimension=dim))
+    return requested_features_and_tags

phoenix/server/api/types/InferencesRole.py

@@ -0,0 +1,23 @@
+from enum import Enum
+from typing import Dict, Union
+
+import strawberry
+
+from phoenix.core.model_schema import PRIMARY, REFERENCE
+
+
+@strawberry.enum
+class InferencesRole(Enum):
+    primary = PRIMARY
+    reference = REFERENCE
+
+
+class AncillaryInferencesRole(Enum):
+    corpus = "InferencesRole.CORPUS"
+
+
+STR_TO_INFEREENCES_ROLE: Dict[str, Union[InferencesRole, AncillaryInferencesRole]] = {
+    str(InferencesRole.primary.value): InferencesRole.primary,
+    str(InferencesRole.reference.value): InferencesRole.reference,
+    str(AncillaryInferencesRole.corpus.value): AncillaryInferencesRole.corpus,
+}

phoenix/server/api/types/Model.py

@@ -2,6 +2,7 @@ import asyncio
 from typing import List, Optional
 
 import strawberry
+from strawberry.relay import Connection
 from strawberry.types import Info
 from strawberry.unset import UNSET
 from typing_extensions import Annotated
@@ -14,12 +15,12 @@ from ..input_types.DimensionFilter import DimensionFilter
 from ..input_types.Granularity import Granularity
 from ..input_types.PerformanceMetricInput import PerformanceMetricInput
 from ..input_types.TimeRange import TimeRange
-from .Dataset import Dataset
-from .DatasetRole import AncillaryDatasetRole, DatasetRole
 from .Dimension import Dimension, to_gql_dimension
 from .EmbeddingDimension import EmbeddingDimension, to_gql_embedding_dimension
 from .ExportedFile import ExportedFile
-from .pagination import Connection, ConnectionArgs, CursorString, connection_from_list
+from .Inferences import Inferences
+from .InferencesRole import AncillaryInferencesRole, InferencesRole
+from .pagination import ConnectionArgs, CursorString, connection_from_list
 from .TimeSeries import (
     PerformanceTimeSeries,
     ensure_timeseries_parameters,
@@ -57,45 +58,45 @@ class Model:
         )
 
     @strawberry.field
-    def primary_dataset(self, info: Info[Context, None]) -> Dataset:
-        dataset = info.context.model[PRIMARY]
-        start, stop = dataset.time_range
-        return Dataset(
+    def primary_inferences(self, info: Info[Context, None]) -> Inferences:
+        inferences = info.context.model[PRIMARY]
+        start, stop = inferences.time_range
+        return Inferences(
             start_time=start,
             end_time=stop,
-            record_count=len(dataset),
-            dataset=dataset,
-            dataset_role=DatasetRole.primary,
+            record_count=len(inferences),
+            inferences=inferences,
+            inferences_role=InferencesRole.primary,
             model=info.context.model,
         )
 
     @strawberry.field
-    def reference_dataset(self, info: Info[Context, None]) -> Optional[Dataset]:
-        if (dataset := info.context.model[REFERENCE]).empty:
+    def reference_inferences(self, info: Info[Context, None]) -> Optional[Inferences]:
+        if (inferences := info.context.model[REFERENCE]).empty:
             return None
-        start, stop = dataset.time_range
-        return Dataset(
+        start, stop = inferences.time_range
+        return Inferences(
             start_time=start,
             end_time=stop,
-            record_count=len(dataset),
-            dataset=dataset,
-            dataset_role=DatasetRole.reference,
+            record_count=len(inferences),
+            inferences=inferences,
+            inferences_role=InferencesRole.reference,
             model=info.context.model,
         )
 
     @strawberry.field
-    def corpus_dataset(self, info: Info[Context, None]) -> Optional[Dataset]:
+    def corpus_inferences(self, info: Info[Context, None]) -> Optional[Inferences]:
         if info.context.corpus is None:
             return None
-        if (dataset := info.context.corpus[PRIMARY]).empty:
+        if (inferences := info.context.corpus[PRIMARY]).empty:
             return None
-        start, stop = dataset.time_range
-        return Dataset(
+        start, stop = inferences.time_range
+        return Inferences(
             start_time=start,
             end_time=stop,
-            record_count=len(dataset),
-            dataset=dataset,
-            dataset_role=AncillaryDatasetRole.corpus,
+            record_count=len(inferences),
+            inferences=inferences,
+            inferences_role=AncillaryInferencesRole.corpus,
             model=info.context.corpus,
         )
 
@@ -156,24 +157,24 @@ class Model:
         info: Info[Context, None],
         metric: PerformanceMetricInput,
         time_range: Optional[TimeRange] = UNSET,
-        dataset_role: Annotated[
-            Optional[DatasetRole],
+        inferences_role: Annotated[
+            Optional[InferencesRole],
             strawberry.argument(
-                description="The dataset (primary or reference) to query",
+                description="The inferences (primary or reference) to query",
             ),
-        ] = DatasetRole.primary,
+        ] = InferencesRole.primary,
     ) -> Optional[float]:
-        if not isinstance(dataset_role, DatasetRole):
-            dataset_role = DatasetRole.primary
+        if not isinstance(inferences_role, InferencesRole):
+            inferences_role = InferencesRole.primary
         model = info.context.model
-        dataset = model[dataset_role.value]
+        inferences = model[inferences_role.value]
         time_range, granularity = ensure_timeseries_parameters(
-            dataset,
+            inferences,
             time_range,
         )
         metric_instance = metric.metric_instance(model)
         data = get_timeseries_data(
-            dataset,
+            inferences,
             metric_instance,
             time_range,
             granularity,
@@ -194,26 +195,26 @@ class Model:
         metric: PerformanceMetricInput,
         time_range: TimeRange,
         granularity: Granularity,
-        dataset_role: Annotated[
-            Optional[DatasetRole],
+        inferences_role: Annotated[
+            Optional[InferencesRole],
             strawberry.argument(
-                description="The dataset (primary or reference) to query",
+                description="The inferences (primary or reference) to query",
             ),
-        ] = DatasetRole.primary,
+        ] = InferencesRole.primary,
     ) -> PerformanceTimeSeries:
-        if not isinstance(dataset_role, DatasetRole):
-            dataset_role = DatasetRole.primary
+        if not isinstance(inferences_role, InferencesRole):
+            inferences_role = InferencesRole.primary
         model = info.context.model
-        dataset = model[dataset_role.value]
+        inferences = model[inferences_role.value]
         time_range, granularity = ensure_timeseries_parameters(
-            dataset,
+            inferences,
             time_range,
             granularity,
         )
         metric_instance = metric.metric_instance(model)
         return PerformanceTimeSeries(
             data=get_timeseries_data(
-                dataset,
+                inferences,
                 metric_instance,
                 time_range,
                 granularity,

phoenix/server/api/types/Project.py

@@ -1,6 +1,10 @@
 import operator
 from datetime import datetime
-from typing import Any, List, Optional
+from typing import (
+    Any,
+    List,
+    Optional,
+)
 
 import strawberry
 from aioitertools.itertools import islice
@@ -8,6 +12,7 @@ from sqlalchemy import and_, desc, distinct, select
 from sqlalchemy.orm import contains_eager
 from sqlalchemy.sql.expression import tuple_
 from strawberry import ID, UNSET
+from strawberry.relay import Connection, Node, NodeID
 from strawberry.types import Info
 
 from phoenix.datetime_utils import right_open_time_range
@@ -17,13 +22,11 @@ from phoenix.server.api.input_types.SpanSort import SpanSort, SpanSortConfig
 from phoenix.server.api.input_types.TimeRange import TimeRange
 from phoenix.server.api.types.DocumentEvaluationSummary import DocumentEvaluationSummary
 from phoenix.server.api.types.EvaluationSummary import EvaluationSummary
-from phoenix.server.api.types.node import Node
 from phoenix.server.api.types.pagination import (
-    Connection,
     Cursor,
     CursorSortColumn,
     CursorString,
-    connections,
+    connection_from_cursors_and_nodes,
 )
 from phoenix.server.api.types.SortDir import SortDir
 from phoenix.server.api.types.Span import Span, to_gql_span
@@ -31,11 +34,10 @@ from phoenix.server.api.types.Trace import Trace
 from phoenix.server.api.types.ValidationResult import ValidationResult
 from phoenix.trace.dsl import SpanFilter
 
-SPANS_LIMIT = 1000
-
 
 @strawberry.type
 class Project(Node):
+    id_attr: NodeID[int]
     name: str
     gradient_start_color: str
     gradient_end_color: str
@@ -149,7 +151,7 @@ class Project(Node):
         async with info.context.db() as session:
             if (id_attr := await session.scalar(stmt)) is None:
                 return None
-        return Trace(id_attr=id_attr)
+        return Trace(id_attr=id_attr, trace_id=trace_id, project_rowid=self.id_attr)
 
     @strawberry.field
     async def spans(
@@ -168,7 +170,7 @@ class Project(Node):
             select(models.Span)
             .join(models.Trace)
             .where(models.Trace.project_rowid == self.id_attr)
-            .options(contains_eager(models.Span.trace))
+            .options(contains_eager(models.Span.trace).load_only(models.Trace.trace_id))
         )
         if time_range:
             stmt = stmt.where(
@@ -213,7 +215,7 @@ class Project(Node):
                 first + 1  # overfetch by one to determine whether there's a next page
             )
         stmt = stmt.order_by(cursor_rowid_column)
-        data = []
+        cursors_and_nodes = []
         async with info.context.db() as session:
             span_records = await session.execute(stmt)
             async for span_record in islice(span_records, first):
@@ -230,15 +232,15 @@ class Project(Node):
                         else None
                     ),
                 )
-                data.append((cursor, to_gql_span(span)))
+                cursors_and_nodes.append((cursor, to_gql_span(span)))
             has_next_page = True
             try:
                 next(span_records)
             except StopIteration:
                 has_next_page = False
 
-        return connections(
-            data,
+        return connection_from_cursors_and_nodes(
+            cursors_and_nodes,
             has_previous_page=False,
             has_next_page=has_next_page,
         )
@@ -355,3 +357,15 @@ class Project(Node):
                 is_valid=False,
                 error_message=e.msg,
             )
+
+
+def to_gql_project(project: models.Project) -> Project:
+    """
+    Converts an ORM project to a GraphQL Project.
+    """
+    return Project(
+        id_attr=project.id,
+        name=project.name,
+        gradient_start_color=project.gradient_start_color,
+        gradient_end_color=project.gradient_end_color,
+    )

phoenix/server/api/types/Span.py

@@ -1,23 +1,33 @@
 import json
+from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
-from typing import Any, List, Mapping, Optional, Sized, cast
+from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Sized, cast
 
 import numpy as np
 import strawberry
 from openinference.semconv.trace import EmbeddingAttributes, SpanAttributes
 from strawberry import ID, UNSET
+from strawberry.relay import Node, NodeID
 from strawberry.types import Info
+from typing_extensions import Annotated
 
 import phoenix.trace.schemas as trace_schema
 from phoenix.db import models
 from phoenix.server.api.context import Context
+from phoenix.server.api.helpers.dataset_helpers import (
+    get_dataset_example_input,
+    get_dataset_example_output,
+)
 from phoenix.server.api.types.DocumentRetrievalMetrics import DocumentRetrievalMetrics
 from phoenix.server.api.types.Evaluation import DocumentEvaluation, SpanEvaluation
+from phoenix.server.api.types.ExampleRevisionInterface import ExampleRevision
 from phoenix.server.api.types.MimeType import MimeType
-from phoenix.server.api.types.node import Node
 from phoenix.trace.attributes import get_attribute_value
 
+if TYPE_CHECKING:
+    from phoenix.server.api.types.Project import Project
+
 EMBEDDING_EMBEDDINGS = SpanAttributes.EMBEDDING_EMBEDDINGS
 EMBEDDING_VECTOR = EmbeddingAttributes.EMBEDDING_VECTOR
 INPUT_MIME_TYPE = SpanAttributes.INPUT_MIME_TYPE
@@ -25,6 +35,9 @@ INPUT_VALUE = SpanAttributes.INPUT_VALUE
 LLM_TOKEN_COUNT_COMPLETION = SpanAttributes.LLM_TOKEN_COUNT_COMPLETION
 LLM_TOKEN_COUNT_PROMPT = SpanAttributes.LLM_TOKEN_COUNT_PROMPT
 LLM_TOKEN_COUNT_TOTAL = SpanAttributes.LLM_TOKEN_COUNT_TOTAL
+LLM_PROMPT_TEMPLATE_VARIABLES = SpanAttributes.LLM_PROMPT_TEMPLATE_VARIABLES
+LLM_INPUT_MESSAGES = SpanAttributes.LLM_INPUT_MESSAGES
+LLM_OUTPUT_MESSAGES = SpanAttributes.LLM_OUTPUT_MESSAGES
 METADATA = SpanAttributes.METADATA
 OUTPUT_MIME_TYPE = SpanAttributes.OUTPUT_MIME_TYPE
 OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE
@@ -46,6 +59,7 @@ class SpanKind(Enum):
     embedding = "EMBEDDING"
     agent = "AGENT"
     reranker = "RERANKER"
+    evaluator = "EVALUATOR"
     unknown = "UNKNOWN"
 
     @classmethod
@@ -101,8 +115,14 @@ class SpanEvent:
         )
 
 
+@strawberry.type
+class SpanAsExampleRevision(ExampleRevision): ...
+
+
 @strawberry.type
 class Span(Node):
+    id_attr: NodeID[int]
+    db_span: strawberry.Private[models.Span]
     name: str
     status_code: SpanStatusCode
     status_message: str
@@ -188,6 +208,44 @@ class Span(Node):
         spans = await info.context.data_loaders.span_descendants.load(span_id)
         return [to_gql_span(span) for span in spans]
 
+    @strawberry.field(
+        description="The span's attributes translated into an example revision for a dataset",
+    )  # type: ignore
+    def as_example_revision(self) -> SpanAsExampleRevision:
+        db_span = self.db_span
+        attributes = db_span.attributes
+        span_io = _SpanIO(
+            span_kind=db_span.span_kind,
+            input_value=get_attribute_value(attributes, INPUT_VALUE),
+            input_mime_type=get_attribute_value(attributes, INPUT_MIME_TYPE),
+            output_value=get_attribute_value(attributes, OUTPUT_VALUE),
+            output_mime_type=get_attribute_value(attributes, OUTPUT_MIME_TYPE),
+            llm_prompt_template_variables=get_attribute_value(
+                attributes, LLM_PROMPT_TEMPLATE_VARIABLES
+            ),
+            llm_input_messages=get_attribute_value(attributes, LLM_INPUT_MESSAGES),
+            llm_output_messages=get_attribute_value(attributes, LLM_OUTPUT_MESSAGES),
+            retrieval_documents=get_attribute_value(attributes, RETRIEVAL_DOCUMENTS),
+        )
+        return SpanAsExampleRevision(
+            input=get_dataset_example_input(span_io),
+            output=get_dataset_example_output(span_io),
+            metadata=attributes,
+        )
+
+    @strawberry.field(description="The project that this span belongs to.")  # type: ignore
+    async def project(
+        self,
+        info: Info[Context, None],
+    ) -> Annotated[
+        "Project", strawberry.lazy("phoenix.server.api.types.Project")
+    ]:  # use lazy types to avoid circular import: https://strawberry.rocks/docs/types/lazy
+        from phoenix.server.api.types.Project import to_gql_project
+
+        span_id = self.id_attr
+        project = await info.context.data_loaders.span_projects.load(span_id)
+        return to_gql_project(project)
+
 
 def to_gql_span(span: models.Span) -> Span:
     events: List[SpanEvent] = list(map(SpanEvent.from_dict, span.events))
@@ -197,6 +255,7 @@ def to_gql_span(span: models.Span) -> Span:
     num_documents = len(retrieval_documents) if isinstance(retrieval_documents, Sized) else None
     return Span(
         id_attr=span.id,
+        db_span=span,
         name=span.name,
         status_code=SpanStatusCode(span.status_code),
         status_message=span.status_message,
@@ -302,3 +361,21 @@ def _convert_metadata_to_string(metadata: Any) -> Optional[str]:
         return json.dumps(metadata)
     except Exception:
         return str(metadata)
+
+
+@dataclass
+class _SpanIO:
+    """
+    An class that contains the information needed to extract dataset example
+    input and output values from a span.
+    """
+
+    span_kind: Optional[str]
+    input_value: Any
+    input_mime_type: Optional[str]
+    output_value: Any
+    output_mime_type: Optional[str]
+    llm_prompt_template_variables: Any
+    llm_input_messages: Any
+    llm_output_messages: Any
+    retrieval_documents: Any