arize-phoenix 8.32.0__py3-none-any.whl → 9.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of arize-phoenix might be problematic. Click here for more details.

Files changed (81) hide show
  1. {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/METADATA +3 -2
  2. {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/RECORD +78 -58
  3. phoenix/db/constants.py +1 -0
  4. phoenix/db/facilitator.py +55 -0
  5. phoenix/db/insertion/document_annotation.py +31 -13
  6. phoenix/db/insertion/evaluation.py +15 -3
  7. phoenix/db/insertion/helpers.py +2 -1
  8. phoenix/db/insertion/span_annotation.py +26 -9
  9. phoenix/db/insertion/trace_annotation.py +25 -9
  10. phoenix/db/insertion/types.py +7 -0
  11. phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py +322 -0
  12. phoenix/db/migrations/versions/8a3764fe7f1a_change_jsonb_to_json_for_prompts.py +76 -0
  13. phoenix/db/migrations/versions/bb8139330879_create_project_trace_retention_policies_table.py +77 -0
  14. phoenix/db/models.py +151 -10
  15. phoenix/db/types/annotation_configs.py +97 -0
  16. phoenix/db/types/db_models.py +41 -0
  17. phoenix/db/types/trace_retention.py +267 -0
  18. phoenix/experiments/functions.py +5 -1
  19. phoenix/server/api/auth.py +9 -0
  20. phoenix/server/api/context.py +5 -0
  21. phoenix/server/api/dataloaders/__init__.py +4 -0
  22. phoenix/server/api/dataloaders/annotation_summaries.py +203 -24
  23. phoenix/server/api/dataloaders/project_ids_by_trace_retention_policy_id.py +42 -0
  24. phoenix/server/api/dataloaders/trace_retention_policy_id_by_project_id.py +34 -0
  25. phoenix/server/api/helpers/annotations.py +9 -0
  26. phoenix/server/api/helpers/prompts/models.py +34 -67
  27. phoenix/server/api/input_types/CreateSpanAnnotationInput.py +9 -0
  28. phoenix/server/api/input_types/CreateTraceAnnotationInput.py +3 -0
  29. phoenix/server/api/input_types/PatchAnnotationInput.py +3 -0
  30. phoenix/server/api/input_types/SpanAnnotationFilter.py +67 -0
  31. phoenix/server/api/mutations/__init__.py +6 -0
  32. phoenix/server/api/mutations/annotation_config_mutations.py +413 -0
  33. phoenix/server/api/mutations/dataset_mutations.py +62 -39
  34. phoenix/server/api/mutations/project_trace_retention_policy_mutations.py +245 -0
  35. phoenix/server/api/mutations/span_annotations_mutations.py +272 -70
  36. phoenix/server/api/mutations/trace_annotations_mutations.py +203 -74
  37. phoenix/server/api/queries.py +86 -0
  38. phoenix/server/api/routers/v1/__init__.py +4 -0
  39. phoenix/server/api/routers/v1/annotation_configs.py +449 -0
  40. phoenix/server/api/routers/v1/annotations.py +161 -0
  41. phoenix/server/api/routers/v1/evaluations.py +6 -0
  42. phoenix/server/api/routers/v1/projects.py +1 -50
  43. phoenix/server/api/routers/v1/spans.py +37 -8
  44. phoenix/server/api/routers/v1/traces.py +22 -13
  45. phoenix/server/api/routers/v1/utils.py +60 -0
  46. phoenix/server/api/types/Annotation.py +7 -0
  47. phoenix/server/api/types/AnnotationConfig.py +124 -0
  48. phoenix/server/api/types/AnnotationSource.py +9 -0
  49. phoenix/server/api/types/AnnotationSummary.py +28 -14
  50. phoenix/server/api/types/AnnotatorKind.py +1 -0
  51. phoenix/server/api/types/CronExpression.py +15 -0
  52. phoenix/server/api/types/Evaluation.py +4 -30
  53. phoenix/server/api/types/Project.py +50 -2
  54. phoenix/server/api/types/ProjectTraceRetentionPolicy.py +110 -0
  55. phoenix/server/api/types/Span.py +78 -0
  56. phoenix/server/api/types/SpanAnnotation.py +24 -0
  57. phoenix/server/api/types/Trace.py +2 -2
  58. phoenix/server/api/types/TraceAnnotation.py +23 -0
  59. phoenix/server/app.py +20 -0
  60. phoenix/server/retention.py +76 -0
  61. phoenix/server/static/.vite/manifest.json +36 -36
  62. phoenix/server/static/assets/components-B2MWTXnm.js +4326 -0
  63. phoenix/server/static/assets/{index-B0CbpsxD.js → index-Bfvpea_-.js} +10 -10
  64. phoenix/server/static/assets/pages-CZ2vKu8H.js +7268 -0
  65. phoenix/server/static/assets/vendor-BRDkBC5J.js +903 -0
  66. phoenix/server/static/assets/{vendor-arizeai-CxXYQNUl.js → vendor-arizeai-BvTqp_W8.js} +3 -3
  67. phoenix/server/static/assets/{vendor-codemirror-B0NIFPOL.js → vendor-codemirror-COt9UfW7.js} +1 -1
  68. phoenix/server/static/assets/{vendor-recharts-CrrDFWK1.js → vendor-recharts-BoHX9Hvs.js} +2 -2
  69. phoenix/server/static/assets/{vendor-shiki-C5bJ-RPf.js → vendor-shiki-Cw1dsDAz.js} +1 -1
  70. phoenix/session/client.py +6 -1
  71. phoenix/trace/dsl/filter.py +25 -5
  72. phoenix/trace/dsl/query.py +93 -13
  73. phoenix/utilities/__init__.py +18 -0
  74. phoenix/version.py +1 -1
  75. phoenix/server/static/assets/components-x-gKFJ8C.js +0 -3414
  76. phoenix/server/static/assets/pages-BU4VdyeH.js +0 -5867
  77. phoenix/server/static/assets/vendor-BfhM_F1u.js +0 -902
  78. {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/WHEEL +0 -0
  79. {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/entry_points.txt +0 -0
  80. {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/licenses/IP_NOTICE +0 -0
  81. {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,3 +1,4 @@
1
+ import warnings
1
2
  from asyncio import get_running_loop
2
3
  from collections.abc import AsyncIterator
3
4
  from datetime import datetime, timezone
@@ -20,6 +21,7 @@ from phoenix.db.helpers import SupportedSQLDialect
20
21
  from phoenix.db.insertion.helpers import as_kv, insert_on_conflict
21
22
  from phoenix.db.insertion.types import Precursors
22
23
  from phoenix.server.api.routers.utils import df_to_bytes
24
+ from phoenix.server.bearer_auth import PhoenixUser
23
25
  from phoenix.server.dml_event import SpanAnnotationInsertEvent
24
26
  from phoenix.trace.dsl import SpanQuery as SpanQuery_
25
27
  from phoenix.utilities.json import encode_df_as_json_string
@@ -47,6 +49,7 @@ class QuerySpansRequestBody(V1RoutesBaseModel):
47
49
  end_time: Optional[datetime] = None
48
50
  limit: int = DEFAULT_SPAN_LIMIT
49
51
  root_spans_only: Optional[bool] = None
52
+ orphan_span_as_root_span: bool = True
50
53
  project_name: Optional[str] = Field(
51
54
  default=None,
52
55
  description=(
@@ -116,6 +119,7 @@ async def query_spans_handler(
116
119
  ),
117
120
  limit=request_body.limit,
118
121
  root_spans_only=request_body.root_spans_only,
122
+ orphan_span_as_root_span=request_body.orphan_span_as_root_span,
119
123
  )
120
124
  )
121
125
  if not results:
@@ -169,10 +173,10 @@ class SpanAnnotationResult(V1RoutesBaseModel):
169
173
  )
170
174
 
171
175
 
172
- class SpanAnnotation(V1RoutesBaseModel):
176
+ class SpanAnnotationData(V1RoutesBaseModel):
173
177
  span_id: str = Field(description="OpenTelemetry Span ID (hex format w/o 0x prefix)")
174
178
  name: str = Field(description="The name of the annotation")
175
- annotator_kind: Literal["LLM", "HUMAN"] = Field(
179
+ annotator_kind: Literal["LLM", "CODE", "HUMAN"] = Field(
176
180
  description="The kind of annotator used for the annotation"
177
181
  )
178
182
  result: Optional[SpanAnnotationResult] = Field(
@@ -181,8 +185,15 @@ class SpanAnnotation(V1RoutesBaseModel):
181
185
  metadata: Optional[dict[str, Any]] = Field(
182
186
  default=None, description="Metadata for the annotation"
183
187
  )
188
+ identifier: str = Field(
189
+ default="",
190
+ description=(
191
+ "The identifier of the annotation. "
192
+ "If provided, the annotation will be updated if it already exists."
193
+ ),
194
+ )
184
195
 
185
- def as_precursor(self) -> Precursors.SpanAnnotation:
196
+ def as_precursor(self, *, user_id: Optional[int] = None) -> Precursors.SpanAnnotation:
186
197
  return Precursors.SpanAnnotation(
187
198
  self.span_id,
188
199
  models.SpanAnnotation(
@@ -192,12 +203,15 @@ class SpanAnnotation(V1RoutesBaseModel):
192
203
  label=self.result.label if self.result else None,
193
204
  explanation=self.result.explanation if self.result else None,
194
205
  metadata_=self.metadata or {},
206
+ identifier=self.identifier,
207
+ source="API",
208
+ user_id=user_id,
195
209
  ),
196
210
  )
197
211
 
198
212
 
199
- class AnnotateSpansRequestBody(RequestBody[list[SpanAnnotation]]):
200
- data: list[SpanAnnotation]
213
+ class AnnotateSpansRequestBody(RequestBody[list[SpanAnnotationData]]):
214
+ data: list[SpanAnnotationData]
201
215
 
202
216
 
203
217
  class InsertedSpanAnnotation(V1RoutesBaseModel):
@@ -211,7 +225,7 @@ class AnnotateSpansResponseBody(ResponseBody[list[InsertedSpanAnnotation]]):
211
225
  @router.post(
212
226
  "/span_annotations",
213
227
  operation_id="annotateSpans",
214
- summary="Create or update span annotations",
228
+ summary="Create span annotations",
215
229
  responses=add_errors_to_responses(
216
230
  [{"status_code": HTTP_404_NOT_FOUND, "description": "Span not found"}]
217
231
  ),
@@ -225,7 +239,22 @@ async def annotate_spans(
225
239
  ) -> AnnotateSpansResponseBody:
226
240
  if not request_body.data:
227
241
  return AnnotateSpansResponseBody(data=[])
228
- precursors = [d.as_precursor() for d in request_body.data]
242
+
243
+ user_id: Optional[int] = None
244
+ if request.app.state.authentication_enabled and isinstance(request.user, PhoenixUser):
245
+ user_id = int(request.user.identity)
246
+
247
+ span_annotations = request_body.data
248
+ filtered_span_annotations = list(filter(lambda d: d.name != "note", span_annotations))
249
+ if len(filtered_span_annotations) != len(span_annotations):
250
+ warnings.warn(
251
+ (
252
+ "Span annotations with the name 'note' are not supported in this endpoint. "
253
+ "They will be ignored."
254
+ ),
255
+ UserWarning,
256
+ )
257
+ precursors = [d.as_precursor(user_id=user_id) for d in filtered_span_annotations]
229
258
  if not sync:
230
259
  await request.state.enqueue(*precursors)
231
260
  return AnnotateSpansResponseBody(data=[])
@@ -254,7 +283,7 @@ async def annotate_spans(
254
283
  values,
255
284
  dialect=dialect,
256
285
  table=models.SpanAnnotation,
257
- unique_by=("name", "span_rowid"),
286
+ unique_by=("name", "span_rowid", "identifier"),
258
287
  ).returning(models.SpanAnnotation.id)
259
288
  )
260
289
  inserted_ids.append(span_annotation_id)
@@ -10,7 +10,7 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
10
10
  ExportTraceServiceResponse,
11
11
  )
12
12
  from pydantic import Field
13
- from sqlalchemy import select
13
+ from sqlalchemy import insert, select
14
14
  from starlette.concurrency import run_in_threadpool
15
15
  from starlette.datastructures import State
16
16
  from starlette.requests import Request
@@ -23,9 +23,9 @@ from starlette.status import (
23
23
  from strawberry.relay import GlobalID
24
24
 
25
25
  from phoenix.db import models
26
- from phoenix.db.helpers import SupportedSQLDialect
27
- from phoenix.db.insertion.helpers import as_kv, insert_on_conflict
26
+ from phoenix.db.insertion.helpers import as_kv
28
27
  from phoenix.db.insertion.types import Precursors
28
+ from phoenix.server.bearer_auth import PhoenixUser
29
29
  from phoenix.server.dml_event import TraceAnnotationInsertEvent
30
30
  from phoenix.trace.otel import decode_otlp_span
31
31
  from phoenix.utilities.project import get_project_name
@@ -114,8 +114,15 @@ class TraceAnnotation(V1RoutesBaseModel):
114
114
  metadata: Optional[dict[str, Any]] = Field(
115
115
  default=None, description="Metadata for the annotation"
116
116
  )
117
+ identifier: str = Field(
118
+ default="",
119
+ description=(
120
+ "The identifier of the annotation. "
121
+ "If provided, the annotation will be updated if it already exists."
122
+ ),
123
+ )
117
124
 
118
- def as_precursor(self) -> Precursors.TraceAnnotation:
125
+ def as_precursor(self, *, user_id: Optional[int] = None) -> Precursors.TraceAnnotation:
119
126
  return Precursors.TraceAnnotation(
120
127
  self.trace_id,
121
128
  models.TraceAnnotation(
@@ -125,6 +132,9 @@ class TraceAnnotation(V1RoutesBaseModel):
125
132
  label=self.result.label if self.result else None,
126
133
  explanation=self.result.explanation if self.result else None,
127
134
  metadata_=self.metadata or {},
135
+ identifier=self.identifier,
136
+ source="APP",
137
+ user_id=user_id,
128
138
  ),
129
139
  )
130
140
 
@@ -144,7 +154,7 @@ class AnnotateTracesResponseBody(ResponseBody[list[InsertedTraceAnnotation]]):
144
154
  @router.post(
145
155
  "/trace_annotations",
146
156
  operation_id="annotateTraces",
147
- summary="Create or update trace annotations",
157
+ summary="Create trace annotations",
148
158
  responses=add_errors_to_responses(
149
159
  [{"status_code": HTTP_404_NOT_FOUND, "description": "Trace not found"}]
150
160
  ),
@@ -157,7 +167,12 @@ async def annotate_traces(
157
167
  ) -> AnnotateTracesResponseBody:
158
168
  if not request_body.data:
159
169
  return AnnotateTracesResponseBody(data=[])
160
- precursors = [d.as_precursor() for d in request_body.data]
170
+
171
+ user_id: Optional[int] = None
172
+ if request.app.state.authentication_enabled and isinstance(request.user, PhoenixUser):
173
+ user_id = int(request.user.identity)
174
+
175
+ precursors = [d.as_precursor(user_id=user_id) for d in request_body.data]
161
176
  if not sync:
162
177
  await request.state.enqueue(*precursors)
163
178
  return AnnotateTracesResponseBody(data=[])
@@ -178,16 +193,10 @@ async def annotate_traces(
178
193
  status_code=HTTP_404_NOT_FOUND,
179
194
  )
180
195
  inserted_ids = []
181
- dialect = SupportedSQLDialect(session.bind.dialect.name)
182
196
  for p in precursors:
183
197
  values = dict(as_kv(p.as_insertable(existing_traces[p.trace_id]).row))
184
198
  trace_annotation_id = await session.scalar(
185
- insert_on_conflict(
186
- values,
187
- dialect=dialect,
188
- table=models.TraceAnnotation,
189
- unique_by=("name", "trace_rowid"),
190
- ).returning(models.TraceAnnotation.id)
199
+ insert(models.TraceAnnotation).values(**values).returning(models.TraceAnnotation.id)
191
200
  )
192
201
  inserted_ids.append(trace_annotation_id)
193
202
  request.state.event_queue.put(TraceAnnotationInsertEvent(tuple(inserted_ids)))
@@ -1,7 +1,19 @@
1
1
  from typing import Any, Generic, Optional, TypedDict, TypeVar, Union
2
2
 
3
+ from fastapi import HTTPException
4
+ from sqlalchemy import select
5
+ from sqlalchemy.ext.asyncio import AsyncSession
6
+ from starlette.status import (
7
+ HTTP_404_NOT_FOUND,
8
+ HTTP_422_UNPROCESSABLE_ENTITY,
9
+ )
10
+ from strawberry.relay import GlobalID
3
11
  from typing_extensions import TypeAlias, assert_never
4
12
 
13
+ from phoenix.db import models
14
+ from phoenix.server.api.types.node import from_global_id_with_expected_type
15
+ from phoenix.server.api.types.Project import Project as ProjectNodeType
16
+
5
17
  from .models import V1RoutesBaseModel
6
18
 
7
19
  StatusCode: TypeAlias = int
@@ -93,3 +105,51 @@ def add_text_csv_content_to_responses(
93
105
  "text/csv": {"schema": {"type": "string", "contentMediaType": "text/csv"}}
94
106
  }
95
107
  return output_responses
108
+
109
+
110
+ async def _get_project_by_identifier(
111
+ session: AsyncSession,
112
+ project_identifier: str,
113
+ ) -> models.Project:
114
+ """
115
+ Get a project by its ID or name.
116
+
117
+ Args:
118
+ session: The database session.
119
+ project_identifier: The project ID or name.
120
+
121
+ Returns:
122
+ The project object.
123
+
124
+ Raises:
125
+ HTTPException: If the identifier format is invalid or the project is not found.
126
+ """
127
+ # Try to parse as a GlobalID first
128
+ try:
129
+ id_ = from_global_id_with_expected_type(
130
+ GlobalID.from_id(project_identifier),
131
+ ProjectNodeType.__name__,
132
+ )
133
+ except Exception:
134
+ try:
135
+ name = project_identifier
136
+ except HTTPException:
137
+ raise HTTPException(
138
+ status_code=HTTP_422_UNPROCESSABLE_ENTITY,
139
+ detail=f"Invalid project identifier format: {project_identifier}",
140
+ )
141
+ stmt = select(models.Project).filter_by(name=name)
142
+ project = await session.scalar(stmt)
143
+ if project is None:
144
+ raise HTTPException(
145
+ status_code=HTTP_404_NOT_FOUND,
146
+ detail=f"Project with name {name} not found",
147
+ )
148
+ else:
149
+ project = await session.get(models.Project, id_)
150
+ if project is None:
151
+ raise HTTPException(
152
+ status_code=HTTP_404_NOT_FOUND,
153
+ detail=f"Project with ID {project_identifier} not found",
154
+ )
155
+ return project
@@ -1,3 +1,4 @@
1
+ from datetime import datetime
1
2
  from typing import Optional
2
3
 
3
4
  import strawberry
@@ -22,3 +23,9 @@ class Annotation:
22
23
  description="The annotator's explanation for the annotation result (i.e. "
23
24
  "score or label, or both) given to the subject."
24
25
  )
26
+ created_at: datetime = strawberry.field(
27
+ description="The date and time when the annotation was created."
28
+ )
29
+ updated_at: datetime = strawberry.field(
30
+ description="The date and time when the annotation was last updated."
31
+ )
@@ -0,0 +1,124 @@
1
+ from typing import Annotated, Optional, Union
2
+
3
+ import strawberry
4
+ from strawberry.relay import Node, NodeID
5
+ from typing_extensions import TypeAlias, assert_never
6
+
7
+ from phoenix.db import models
8
+ from phoenix.db.types.annotation_configs import (
9
+ AnnotationType,
10
+ OptimizationDirection,
11
+ )
12
+ from phoenix.db.types.annotation_configs import (
13
+ CategoricalAnnotationConfig as CategoricalAnnotationConfigModel,
14
+ )
15
+ from phoenix.db.types.annotation_configs import (
16
+ ContinuousAnnotationConfig as ContinuousAnnotationConfigModel,
17
+ )
18
+ from phoenix.db.types.annotation_configs import (
19
+ FreeformAnnotationConfig as FreeformAnnotationConfigModel,
20
+ )
21
+
22
+
23
+ @strawberry.interface
24
+ class AnnotationConfigBase:
25
+ name: str
26
+ description: Optional[str]
27
+ annotation_type: AnnotationType
28
+
29
+
30
+ @strawberry.type
31
+ class CategoricalAnnotationValue:
32
+ label: str
33
+ score: Optional[float]
34
+
35
+
36
+ @strawberry.type
37
+ class CategoricalAnnotationConfig(Node, AnnotationConfigBase):
38
+ id_attr: NodeID[int]
39
+ optimization_direction: OptimizationDirection
40
+ values: list[CategoricalAnnotationValue]
41
+
42
+
43
+ @strawberry.type
44
+ class ContinuousAnnotationConfig(Node, AnnotationConfigBase):
45
+ id_attr: NodeID[int]
46
+ optimization_direction: OptimizationDirection
47
+ lower_bound: Optional[float]
48
+ upper_bound: Optional[float]
49
+
50
+
51
+ @strawberry.type
52
+ class FreeformAnnotationConfig(Node, AnnotationConfigBase):
53
+ id_attr: NodeID[int]
54
+
55
+
56
+ AnnotationConfig: TypeAlias = Annotated[
57
+ Union[CategoricalAnnotationConfig, ContinuousAnnotationConfig, FreeformAnnotationConfig],
58
+ strawberry.union("AnnotationConfig"),
59
+ ]
60
+
61
+
62
+ def _to_gql_categorical_annotation_config(
63
+ annotation_config: models.AnnotationConfig,
64
+ ) -> CategoricalAnnotationConfig:
65
+ config = annotation_config.config
66
+ assert isinstance(config, CategoricalAnnotationConfigModel)
67
+ values = [
68
+ CategoricalAnnotationValue(
69
+ label=val.label,
70
+ score=val.score,
71
+ )
72
+ for val in config.values
73
+ ]
74
+ return CategoricalAnnotationConfig(
75
+ id_attr=annotation_config.id,
76
+ name=annotation_config.name,
77
+ annotation_type=config.type,
78
+ optimization_direction=config.optimization_direction,
79
+ description=config.description,
80
+ values=values,
81
+ )
82
+
83
+
84
+ def _to_gql_continuous_annotation_config(
85
+ annotation_config: models.AnnotationConfig,
86
+ ) -> ContinuousAnnotationConfig:
87
+ config = annotation_config.config
88
+ assert isinstance(config, ContinuousAnnotationConfigModel)
89
+ return ContinuousAnnotationConfig(
90
+ id_attr=annotation_config.id,
91
+ name=annotation_config.name,
92
+ annotation_type=config.type,
93
+ optimization_direction=config.optimization_direction,
94
+ description=config.description,
95
+ lower_bound=config.lower_bound,
96
+ upper_bound=config.upper_bound,
97
+ )
98
+
99
+
100
+ def _to_gql_freeform_annotation_config(
101
+ annotation_config: models.AnnotationConfig,
102
+ ) -> FreeformAnnotationConfig:
103
+ config = annotation_config.config
104
+ assert isinstance(config, FreeformAnnotationConfigModel)
105
+ return FreeformAnnotationConfig(
106
+ id_attr=annotation_config.id,
107
+ name=annotation_config.name,
108
+ annotation_type=config.type,
109
+ description=config.description,
110
+ )
111
+
112
+
113
+ def to_gql_annotation_config(annotation_config: models.AnnotationConfig) -> AnnotationConfig:
114
+ """
115
+ Convert an SQLAlchemy AnnotationConfig instance to one of the GraphQL types.
116
+ """
117
+ config = annotation_config.config
118
+ if isinstance(config, ContinuousAnnotationConfigModel):
119
+ return _to_gql_continuous_annotation_config(annotation_config)
120
+ if isinstance(config, CategoricalAnnotationConfigModel):
121
+ return _to_gql_categorical_annotation_config(annotation_config)
122
+ if isinstance(config, FreeformAnnotationConfigModel):
123
+ return _to_gql_freeform_annotation_config(annotation_config)
124
+ assert_never(annotation_config)
@@ -0,0 +1,9 @@
1
+ from enum import Enum
2
+
3
+ import strawberry
4
+
5
+
6
+ @strawberry.enum
7
+ class AnnotationSource(Enum):
8
+ API = "API"
9
+ APP = "APP"
@@ -12,10 +12,9 @@ AnnotationType = Union[models.SpanAnnotation, models.TraceAnnotation]
12
12
 
13
13
  @strawberry.type
14
14
  class AnnotationSummary:
15
+ name: str
15
16
  df: Private[pd.DataFrame]
16
-
17
- def __init__(self, dataframe: pd.DataFrame) -> None:
18
- self.df = dataframe
17
+ simple_avg: Private[bool] = False
19
18
 
20
19
  @strawberry.field
21
20
  def count(self) -> int:
@@ -23,28 +22,43 @@ class AnnotationSummary:
23
22
 
24
23
  @strawberry.field
25
24
  def labels(self) -> list[str]:
26
- return self.df.label.dropna().tolist()
25
+ unique_labels = self.df["label"].dropna().unique()
26
+ return [str(label) for label in unique_labels]
27
27
 
28
28
  @strawberry.field
29
29
  def label_fractions(self) -> list[LabelFraction]:
30
- if not (n := self.df.label_count.sum()):
31
- return []
30
+ if self.simple_avg:
31
+ if not (n := self.df.label_count.sum()):
32
+ return []
33
+ return [
34
+ LabelFraction(
35
+ label=cast(str, row.label),
36
+ fraction=row.label_count / n,
37
+ )
38
+ for row in self.df.loc[
39
+ self.df.label.notna(),
40
+ ["label", "label_count"],
41
+ ].itertuples()
42
+ ]
32
43
  return [
33
44
  LabelFraction(
34
- label=cast(str, row.label),
35
- fraction=row.label_count / n,
45
+ label=row.label,
46
+ fraction=float(row.avg_label_fraction),
36
47
  )
37
- for row in self.df.loc[
38
- self.df.label.notna(),
39
- ["label", "label_count"],
40
- ].itertuples()
48
+ for row in self.df.itertuples()
49
+ if row.label is not None
41
50
  ]
42
51
 
43
52
  @strawberry.field
44
53
  def mean_score(self) -> Optional[float]:
45
- if not (n := self.df.score_count.sum()):
54
+ if self.simple_avg:
55
+ if not (n := self.df.score_count.sum()):
56
+ return None
57
+ return cast(float, self.df.score_sum.sum() / n)
58
+ avg_scores = self.df["avg_score"].dropna()
59
+ if avg_scores.empty:
46
60
  return None
47
- return cast(float, self.df.score_sum.sum() / n)
61
+ return float(avg_scores.mean()) # all avg_scores should be the same
48
62
 
49
63
  @strawberry.field
50
64
  def score_count(self) -> int:
@@ -14,3 +14,4 @@ class ExperimentRunAnnotatorKind(Enum):
14
14
  class AnnotatorKind(Enum):
15
15
  LLM = "LLM"
16
16
  HUMAN = "HUMAN"
17
+ CODE = "CODE"
@@ -0,0 +1,15 @@
1
+ from typing import NewType
2
+
3
+ import strawberry
4
+
5
+ from phoenix.db.types.trace_retention import TraceRetentionCronExpression
6
+
7
+
8
+ def parse_value(value: str) -> str:
9
+ return TraceRetentionCronExpression.model_validate(value).root
10
+
11
+
12
+ CronExpression = strawberry.scalar(
13
+ NewType("CronExpression", str),
14
+ parse_value=parse_value,
15
+ )
@@ -1,6 +1,5 @@
1
1
  import strawberry
2
2
 
3
- import phoenix.trace.v1 as pb
4
3
  from phoenix.db.models import DocumentAnnotation, TraceAnnotation
5
4
 
6
5
  from .Annotation import Annotation
@@ -8,19 +7,6 @@ from .Annotation import Annotation
8
7
 
9
8
  @strawberry.type
10
9
  class TraceEvaluation(Annotation):
11
- @staticmethod
12
- def from_pb_evaluation(evaluation: pb.Evaluation) -> "TraceEvaluation":
13
- result = evaluation.result
14
- score = result.score.value if result.HasField("score") else None
15
- label = result.label.value if result.HasField("label") else None
16
- explanation = result.explanation.value if result.HasField("explanation") else None
17
- return TraceEvaluation(
18
- name=evaluation.name,
19
- score=score,
20
- label=label,
21
- explanation=explanation,
22
- )
23
-
24
10
  @staticmethod
25
11
  def from_sql_trace_annotation(annotation: TraceAnnotation) -> "TraceEvaluation":
26
12
  return TraceEvaluation(
@@ -28,6 +14,8 @@ class TraceEvaluation(Annotation):
28
14
  score=annotation.score,
29
15
  label=annotation.label,
30
16
  explanation=annotation.explanation,
17
+ created_at=annotation.created_at,
18
+ updated_at=annotation.updated_at,
31
19
  )
32
20
 
33
21
 
@@ -38,22 +26,6 @@ class DocumentEvaluation(Annotation):
38
26
  "is collected as a list (even when ordering is not inherently meaningful)."
39
27
  )
40
28
 
41
- @staticmethod
42
- def from_pb_evaluation(evaluation: pb.Evaluation) -> "DocumentEvaluation":
43
- result = evaluation.result
44
- score = result.score.value if result.HasField("score") else None
45
- label = result.label.value if result.HasField("label") else None
46
- explanation = result.explanation.value if result.HasField("explanation") else None
47
- document_retrieval_id = evaluation.subject_id.document_retrieval_id
48
- document_position = document_retrieval_id.document_position
49
- return DocumentEvaluation(
50
- name=evaluation.name,
51
- score=score,
52
- label=label,
53
- explanation=explanation,
54
- document_position=document_position,
55
- )
56
-
57
29
  @staticmethod
58
30
  def from_sql_document_annotation(annotation: DocumentAnnotation) -> "DocumentEvaluation":
59
31
  return DocumentEvaluation(
@@ -62,4 +34,6 @@ class DocumentEvaluation(Annotation):
62
34
  label=annotation.label,
63
35
  explanation=annotation.explanation,
64
36
  document_position=annotation.document_position,
37
+ created_at=annotation.created_at,
38
+ updated_at=annotation.updated_at,
65
39
  )
@@ -1,6 +1,8 @@
1
+ from __future__ import annotations
2
+
1
3
  import operator
2
4
  from datetime import datetime
3
- from typing import Any, ClassVar, Optional
5
+ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Optional
4
6
 
5
7
  import strawberry
6
8
  from aioitertools.itertools import islice
@@ -8,7 +10,7 @@ from openinference.semconv.trace import SpanAttributes
8
10
  from sqlalchemy import desc, distinct, func, or_, select
9
11
  from sqlalchemy.sql.elements import ColumnElement
10
12
  from sqlalchemy.sql.expression import tuple_
11
- from strawberry import ID, UNSET, Private
13
+ from strawberry import ID, UNSET, Private, lazy
12
14
  from strawberry.relay import Connection, Node, NodeID
13
15
  from strawberry.types import Info
14
16
  from typing_extensions import assert_never
@@ -22,13 +24,16 @@ from phoenix.server.api.input_types.ProjectSessionSort import (
22
24
  )
23
25
  from phoenix.server.api.input_types.SpanSort import SpanSort, SpanSortConfig
24
26
  from phoenix.server.api.input_types.TimeRange import TimeRange
27
+ from phoenix.server.api.types.AnnotationConfig import AnnotationConfig, to_gql_annotation_config
25
28
  from phoenix.server.api.types.AnnotationSummary import AnnotationSummary
26
29
  from phoenix.server.api.types.DocumentEvaluationSummary import DocumentEvaluationSummary
27
30
  from phoenix.server.api.types.pagination import (
31
+ ConnectionArgs,
28
32
  Cursor,
29
33
  CursorSortColumn,
30
34
  CursorString,
31
35
  connection_from_cursors_and_nodes,
36
+ connection_from_list,
32
37
  )
33
38
  from phoenix.server.api.types.ProjectSession import ProjectSession, to_gql_project_session
34
39
  from phoenix.server.api.types.SortDir import SortDir
@@ -38,6 +43,8 @@ from phoenix.server.api.types.ValidationResult import ValidationResult
38
43
  from phoenix.trace.dsl import SpanFilter
39
44
 
40
45
  DEFAULT_PAGE_SIZE = 30
46
+ if TYPE_CHECKING:
47
+ from phoenix.server.api.types.ProjectTraceRetentionPolicy import ProjectTraceRetentionPolicy
41
48
 
42
49
 
43
50
  @strawberry.type
@@ -536,6 +543,47 @@ class Project(Node):
536
543
  error_message=e.msg,
537
544
  )
538
545
 
546
+ @strawberry.field
547
+ async def annotation_configs(
548
+ self,
549
+ info: Info[Context, None],
550
+ first: Optional[int] = 50,
551
+ last: Optional[int] = None,
552
+ after: Optional[str] = None,
553
+ before: Optional[str] = None,
554
+ ) -> Connection[AnnotationConfig]:
555
+ args = ConnectionArgs(
556
+ first=first,
557
+ after=after if isinstance(after, CursorString) else None,
558
+ last=last,
559
+ before=before if isinstance(before, CursorString) else None,
560
+ )
561
+ async with info.context.db() as session:
562
+ annotation_configs = await session.stream_scalars(
563
+ select(models.AnnotationConfig)
564
+ .join(
565
+ models.ProjectAnnotationConfig,
566
+ models.AnnotationConfig.id
567
+ == models.ProjectAnnotationConfig.annotation_config_id,
568
+ )
569
+ .where(models.ProjectAnnotationConfig.project_id == self.project_rowid)
570
+ .order_by(models.AnnotationConfig.name)
571
+ )
572
+ data = [to_gql_annotation_config(config) async for config in annotation_configs]
573
+ return connection_from_list(data=data, args=args)
574
+
575
+ @strawberry.field
576
+ async def trace_retention_policy(
577
+ self,
578
+ info: Info[Context, None],
579
+ ) -> Annotated[ProjectTraceRetentionPolicy, lazy(".ProjectTraceRetentionPolicy")]:
580
+ from .ProjectTraceRetentionPolicy import ProjectTraceRetentionPolicy
581
+
582
+ id_ = await info.context.data_loaders.trace_retention_policy_id_by_project_id.load(
583
+ self.project_rowid
584
+ )
585
+ return ProjectTraceRetentionPolicy(id=id_)
586
+
539
587
 
540
588
  INPUT_VALUE = SpanAttributes.INPUT_VALUE.split(".")
541
589
  OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE.split(".")