arize-phoenix 8.32.0__py3-none-any.whl → 9.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of arize-phoenix might be problematic. Click here for more details.
- {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/METADATA +3 -2
- {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/RECORD +78 -58
- phoenix/db/constants.py +1 -0
- phoenix/db/facilitator.py +55 -0
- phoenix/db/insertion/document_annotation.py +31 -13
- phoenix/db/insertion/evaluation.py +15 -3
- phoenix/db/insertion/helpers.py +2 -1
- phoenix/db/insertion/span_annotation.py +26 -9
- phoenix/db/insertion/trace_annotation.py +25 -9
- phoenix/db/insertion/types.py +7 -0
- phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py +322 -0
- phoenix/db/migrations/versions/8a3764fe7f1a_change_jsonb_to_json_for_prompts.py +76 -0
- phoenix/db/migrations/versions/bb8139330879_create_project_trace_retention_policies_table.py +77 -0
- phoenix/db/models.py +151 -10
- phoenix/db/types/annotation_configs.py +97 -0
- phoenix/db/types/db_models.py +41 -0
- phoenix/db/types/trace_retention.py +267 -0
- phoenix/experiments/functions.py +5 -1
- phoenix/server/api/auth.py +9 -0
- phoenix/server/api/context.py +5 -0
- phoenix/server/api/dataloaders/__init__.py +4 -0
- phoenix/server/api/dataloaders/annotation_summaries.py +203 -24
- phoenix/server/api/dataloaders/project_ids_by_trace_retention_policy_id.py +42 -0
- phoenix/server/api/dataloaders/trace_retention_policy_id_by_project_id.py +34 -0
- phoenix/server/api/helpers/annotations.py +9 -0
- phoenix/server/api/helpers/prompts/models.py +34 -67
- phoenix/server/api/input_types/CreateSpanAnnotationInput.py +9 -0
- phoenix/server/api/input_types/CreateTraceAnnotationInput.py +3 -0
- phoenix/server/api/input_types/PatchAnnotationInput.py +3 -0
- phoenix/server/api/input_types/SpanAnnotationFilter.py +67 -0
- phoenix/server/api/mutations/__init__.py +6 -0
- phoenix/server/api/mutations/annotation_config_mutations.py +413 -0
- phoenix/server/api/mutations/dataset_mutations.py +62 -39
- phoenix/server/api/mutations/project_trace_retention_policy_mutations.py +245 -0
- phoenix/server/api/mutations/span_annotations_mutations.py +272 -70
- phoenix/server/api/mutations/trace_annotations_mutations.py +203 -74
- phoenix/server/api/queries.py +86 -0
- phoenix/server/api/routers/v1/__init__.py +4 -0
- phoenix/server/api/routers/v1/annotation_configs.py +449 -0
- phoenix/server/api/routers/v1/annotations.py +161 -0
- phoenix/server/api/routers/v1/evaluations.py +6 -0
- phoenix/server/api/routers/v1/projects.py +1 -50
- phoenix/server/api/routers/v1/spans.py +37 -8
- phoenix/server/api/routers/v1/traces.py +22 -13
- phoenix/server/api/routers/v1/utils.py +60 -0
- phoenix/server/api/types/Annotation.py +7 -0
- phoenix/server/api/types/AnnotationConfig.py +124 -0
- phoenix/server/api/types/AnnotationSource.py +9 -0
- phoenix/server/api/types/AnnotationSummary.py +28 -14
- phoenix/server/api/types/AnnotatorKind.py +1 -0
- phoenix/server/api/types/CronExpression.py +15 -0
- phoenix/server/api/types/Evaluation.py +4 -30
- phoenix/server/api/types/Project.py +50 -2
- phoenix/server/api/types/ProjectTraceRetentionPolicy.py +110 -0
- phoenix/server/api/types/Span.py +78 -0
- phoenix/server/api/types/SpanAnnotation.py +24 -0
- phoenix/server/api/types/Trace.py +2 -2
- phoenix/server/api/types/TraceAnnotation.py +23 -0
- phoenix/server/app.py +20 -0
- phoenix/server/retention.py +76 -0
- phoenix/server/static/.vite/manifest.json +36 -36
- phoenix/server/static/assets/components-B2MWTXnm.js +4326 -0
- phoenix/server/static/assets/{index-B0CbpsxD.js → index-Bfvpea_-.js} +10 -10
- phoenix/server/static/assets/pages-CZ2vKu8H.js +7268 -0
- phoenix/server/static/assets/vendor-BRDkBC5J.js +903 -0
- phoenix/server/static/assets/{vendor-arizeai-CxXYQNUl.js → vendor-arizeai-BvTqp_W8.js} +3 -3
- phoenix/server/static/assets/{vendor-codemirror-B0NIFPOL.js → vendor-codemirror-COt9UfW7.js} +1 -1
- phoenix/server/static/assets/{vendor-recharts-CrrDFWK1.js → vendor-recharts-BoHX9Hvs.js} +2 -2
- phoenix/server/static/assets/{vendor-shiki-C5bJ-RPf.js → vendor-shiki-Cw1dsDAz.js} +1 -1
- phoenix/session/client.py +6 -1
- phoenix/trace/dsl/filter.py +25 -5
- phoenix/trace/dsl/query.py +93 -13
- phoenix/utilities/__init__.py +18 -0
- phoenix/version.py +1 -1
- phoenix/server/static/assets/components-x-gKFJ8C.js +0 -3414
- phoenix/server/static/assets/pages-BU4VdyeH.js +0 -5867
- phoenix/server/static/assets/vendor-BfhM_F1u.js +0 -902
- {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/WHEEL +0 -0
- {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/entry_points.txt +0 -0
- {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/licenses/IP_NOTICE +0 -0
- {arize_phoenix-8.32.0.dist-info → arize_phoenix-9.0.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Annotated, List, Literal, Optional, Union
|
|
3
|
+
|
|
4
|
+
from fastapi import APIRouter, HTTPException, Path, Query
|
|
5
|
+
from pydantic import Field, RootModel
|
|
6
|
+
from sqlalchemy import delete, select
|
|
7
|
+
from sqlalchemy.exc import IntegrityError as PostgreSQLIntegrityError
|
|
8
|
+
from sqlean.dbapi2 import IntegrityError as SQLiteIntegrityError # type: ignore[import-untyped]
|
|
9
|
+
from starlette.requests import Request
|
|
10
|
+
from starlette.status import (
|
|
11
|
+
HTTP_400_BAD_REQUEST,
|
|
12
|
+
HTTP_404_NOT_FOUND,
|
|
13
|
+
HTTP_409_CONFLICT,
|
|
14
|
+
)
|
|
15
|
+
from strawberry.relay import GlobalID
|
|
16
|
+
from typing_extensions import TypeAlias, assert_never
|
|
17
|
+
|
|
18
|
+
from phoenix.db import models
|
|
19
|
+
from phoenix.db.types.annotation_configs import (
|
|
20
|
+
AnnotationConfigType,
|
|
21
|
+
AnnotationType,
|
|
22
|
+
OptimizationDirection,
|
|
23
|
+
)
|
|
24
|
+
from phoenix.db.types.annotation_configs import (
|
|
25
|
+
CategoricalAnnotationConfig as CategoricalAnnotationConfigModel,
|
|
26
|
+
)
|
|
27
|
+
from phoenix.db.types.annotation_configs import (
|
|
28
|
+
CategoricalAnnotationValue as CategoricalAnnotationValueModel,
|
|
29
|
+
)
|
|
30
|
+
from phoenix.db.types.annotation_configs import (
|
|
31
|
+
ContinuousAnnotationConfig as ContinuousAnnotationConfigModel,
|
|
32
|
+
)
|
|
33
|
+
from phoenix.db.types.annotation_configs import (
|
|
34
|
+
FreeformAnnotationConfig as FreeformAnnotationConfigModel,
|
|
35
|
+
)
|
|
36
|
+
from phoenix.server.api.routers.v1.models import V1RoutesBaseModel
|
|
37
|
+
from phoenix.server.api.routers.v1.utils import PaginatedResponseBody, ResponseBody
|
|
38
|
+
from phoenix.server.api.types.AnnotationConfig import (
|
|
39
|
+
CategoricalAnnotationConfig as CategoricalAnnotationConfigType,
|
|
40
|
+
)
|
|
41
|
+
from phoenix.server.api.types.AnnotationConfig import (
|
|
42
|
+
ContinuousAnnotationConfig as ContinuousAnnotationConfigType,
|
|
43
|
+
)
|
|
44
|
+
from phoenix.server.api.types.AnnotationConfig import (
|
|
45
|
+
FreeformAnnotationConfig as FreeformAnnotationConfigType,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
logger = logging.getLogger(__name__)
|
|
49
|
+
|
|
50
|
+
router = APIRouter(tags=["annotation_configs"])
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class CategoricalAnnotationValue(V1RoutesBaseModel):
|
|
54
|
+
label: str
|
|
55
|
+
score: Optional[float] = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class CategoricalAnnotationConfigData(V1RoutesBaseModel):
|
|
59
|
+
name: str
|
|
60
|
+
type: Literal[AnnotationType.CATEGORICAL.value] # type: ignore[name-defined]
|
|
61
|
+
description: Optional[str] = None
|
|
62
|
+
optimization_direction: OptimizationDirection
|
|
63
|
+
values: List[CategoricalAnnotationValue]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class ContinuousAnnotationConfigData(V1RoutesBaseModel):
|
|
67
|
+
name: str
|
|
68
|
+
type: Literal[AnnotationType.CONTINUOUS.value] # type: ignore[name-defined]
|
|
69
|
+
description: Optional[str] = None
|
|
70
|
+
optimization_direction: OptimizationDirection
|
|
71
|
+
lower_bound: Optional[float] = None
|
|
72
|
+
upper_bound: Optional[float] = None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class FreeformAnnotationConfigData(V1RoutesBaseModel):
|
|
76
|
+
name: str
|
|
77
|
+
type: Literal[AnnotationType.FREEFORM.value] # type: ignore[name-defined]
|
|
78
|
+
description: Optional[str] = None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
AnnotationConfigData: TypeAlias = Annotated[
|
|
82
|
+
Union[
|
|
83
|
+
CategoricalAnnotationConfigData,
|
|
84
|
+
ContinuousAnnotationConfigData,
|
|
85
|
+
FreeformAnnotationConfigData,
|
|
86
|
+
],
|
|
87
|
+
Field(..., discriminator="type"),
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class CategoricalAnnotationConfig(CategoricalAnnotationConfigData):
|
|
92
|
+
id: str
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class ContinuousAnnotationConfig(ContinuousAnnotationConfigData):
|
|
96
|
+
id: str
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class FreeformAnnotationConfig(FreeformAnnotationConfigData):
|
|
100
|
+
id: str
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
AnnotationConfig: TypeAlias = Annotated[
|
|
104
|
+
Union[
|
|
105
|
+
CategoricalAnnotationConfig,
|
|
106
|
+
ContinuousAnnotationConfig,
|
|
107
|
+
FreeformAnnotationConfig,
|
|
108
|
+
],
|
|
109
|
+
Field(..., discriminator="type"),
|
|
110
|
+
]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def db_to_api_annotation_config(
|
|
114
|
+
annotation_config: models.AnnotationConfig,
|
|
115
|
+
) -> AnnotationConfig:
|
|
116
|
+
config = annotation_config.config
|
|
117
|
+
name = annotation_config.name
|
|
118
|
+
type_ = config.type
|
|
119
|
+
description = config.description
|
|
120
|
+
if isinstance(config, ContinuousAnnotationConfigModel):
|
|
121
|
+
return ContinuousAnnotationConfig(
|
|
122
|
+
id=str(GlobalID(ContinuousAnnotationConfigType.__name__, str(annotation_config.id))),
|
|
123
|
+
name=name,
|
|
124
|
+
type=type_,
|
|
125
|
+
description=description,
|
|
126
|
+
optimization_direction=config.optimization_direction,
|
|
127
|
+
lower_bound=config.lower_bound,
|
|
128
|
+
upper_bound=config.upper_bound,
|
|
129
|
+
)
|
|
130
|
+
if isinstance(config, CategoricalAnnotationConfigModel):
|
|
131
|
+
return CategoricalAnnotationConfig(
|
|
132
|
+
id=str(GlobalID(CategoricalAnnotationConfigType.__name__, str(annotation_config.id))),
|
|
133
|
+
name=name,
|
|
134
|
+
type=type_,
|
|
135
|
+
description=description,
|
|
136
|
+
optimization_direction=config.optimization_direction,
|
|
137
|
+
values=[
|
|
138
|
+
CategoricalAnnotationValue(label=val.label, score=val.score)
|
|
139
|
+
for val in config.values
|
|
140
|
+
],
|
|
141
|
+
)
|
|
142
|
+
if isinstance(config, FreeformAnnotationConfigModel):
|
|
143
|
+
return FreeformAnnotationConfig(
|
|
144
|
+
id=str(GlobalID(FreeformAnnotationConfigType.__name__, str(annotation_config.id))),
|
|
145
|
+
name=name,
|
|
146
|
+
type=type_,
|
|
147
|
+
description=description,
|
|
148
|
+
)
|
|
149
|
+
assert_never(config)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _get_annotation_global_id(annotation_config: models.AnnotationConfig) -> GlobalID:
|
|
153
|
+
config = annotation_config.config
|
|
154
|
+
if isinstance(config, ContinuousAnnotationConfigModel):
|
|
155
|
+
return GlobalID(ContinuousAnnotationConfigType.__name__, str(annotation_config.id))
|
|
156
|
+
if isinstance(config, CategoricalAnnotationConfigModel):
|
|
157
|
+
return GlobalID(CategoricalAnnotationConfigType.__name__, str(annotation_config.id))
|
|
158
|
+
if isinstance(config, FreeformAnnotationConfigModel):
|
|
159
|
+
return GlobalID(FreeformAnnotationConfigType.__name__, str(annotation_config.id))
|
|
160
|
+
assert_never(config)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class CreateAnnotationConfigData(RootModel[AnnotationConfigData]):
|
|
164
|
+
root: AnnotationConfigData
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class GetAnnotationConfigsResponseBody(PaginatedResponseBody[AnnotationConfig]):
|
|
168
|
+
pass
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class GetAnnotationConfigResponseBody(ResponseBody[AnnotationConfig]):
|
|
172
|
+
pass
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class CreateAnnotationConfigResponseBody(ResponseBody[AnnotationConfig]):
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class UpdateAnnotationConfigResponseBody(ResponseBody[AnnotationConfig]):
|
|
180
|
+
pass
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class DeleteAnnotationConfigResponseBody(ResponseBody[AnnotationConfig]):
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@router.get(
|
|
188
|
+
"/annotation_configs",
|
|
189
|
+
summary="List annotation configurations",
|
|
190
|
+
description="Retrieve a paginated list of all annotation configurations in the system.",
|
|
191
|
+
response_description="A list of annotation configurations with pagination information",
|
|
192
|
+
)
|
|
193
|
+
async def list_annotation_configs(
|
|
194
|
+
request: Request,
|
|
195
|
+
cursor: Optional[str] = Query(
|
|
196
|
+
default=None,
|
|
197
|
+
description="Cursor for pagination (base64-encoded annotation config ID)",
|
|
198
|
+
),
|
|
199
|
+
limit: int = Query(100, gt=0, description="Maximum number of configs to return"),
|
|
200
|
+
) -> GetAnnotationConfigsResponseBody:
|
|
201
|
+
cursor_id: Optional[int] = None
|
|
202
|
+
if cursor:
|
|
203
|
+
try:
|
|
204
|
+
cursor_gid = GlobalID.from_id(cursor)
|
|
205
|
+
except ValueError:
|
|
206
|
+
raise HTTPException(
|
|
207
|
+
detail=f"Invalid cursor: {cursor}",
|
|
208
|
+
status_code=HTTP_400_BAD_REQUEST,
|
|
209
|
+
)
|
|
210
|
+
if cursor_gid.type_name not in (
|
|
211
|
+
CategoricalAnnotationConfigType.__name__,
|
|
212
|
+
ContinuousAnnotationConfigType.__name__,
|
|
213
|
+
FreeformAnnotationConfigType.__name__,
|
|
214
|
+
):
|
|
215
|
+
raise HTTPException(
|
|
216
|
+
detail=f"Invalid cursor: {cursor}",
|
|
217
|
+
status_code=HTTP_400_BAD_REQUEST,
|
|
218
|
+
)
|
|
219
|
+
cursor_id = int(cursor_gid.node_id)
|
|
220
|
+
|
|
221
|
+
async with request.app.state.db() as session:
|
|
222
|
+
query = (
|
|
223
|
+
select(models.AnnotationConfig)
|
|
224
|
+
.order_by(models.AnnotationConfig.id.desc())
|
|
225
|
+
.limit(limit + 1) # overfetch by 1 to check if there are more results
|
|
226
|
+
)
|
|
227
|
+
if cursor_id is not None:
|
|
228
|
+
query = query.where(models.AnnotationConfig.id <= cursor_id)
|
|
229
|
+
|
|
230
|
+
result = await session.scalars(query)
|
|
231
|
+
configs = result.all()
|
|
232
|
+
|
|
233
|
+
next_cursor = None
|
|
234
|
+
if len(configs) == limit + 1:
|
|
235
|
+
last_config = configs[-1]
|
|
236
|
+
next_cursor = str(_get_annotation_global_id(last_config))
|
|
237
|
+
configs = configs[:-1]
|
|
238
|
+
|
|
239
|
+
return GetAnnotationConfigsResponseBody(
|
|
240
|
+
next_cursor=next_cursor,
|
|
241
|
+
data=[db_to_api_annotation_config(config) for config in configs],
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@router.get(
|
|
246
|
+
"/annotation_configs/{config_identifier}",
|
|
247
|
+
summary="Get an annotation configuration by ID or name",
|
|
248
|
+
)
|
|
249
|
+
async def get_annotation_config_by_name_or_id(
|
|
250
|
+
request: Request,
|
|
251
|
+
config_identifier: str = Path(..., description="ID or name of the annotation configuration"),
|
|
252
|
+
) -> GetAnnotationConfigResponseBody:
|
|
253
|
+
async with request.app.state.db() as session:
|
|
254
|
+
query = select(models.AnnotationConfig)
|
|
255
|
+
# Try to interpret the identifier as an integer ID; if not, use it as a name.
|
|
256
|
+
try:
|
|
257
|
+
db_id = _get_annotation_config_db_id(config_identifier)
|
|
258
|
+
query = query.where(models.AnnotationConfig.id == db_id)
|
|
259
|
+
except ValueError:
|
|
260
|
+
query = query.where(models.AnnotationConfig.name == config_identifier)
|
|
261
|
+
config = await session.scalar(query)
|
|
262
|
+
if not config:
|
|
263
|
+
raise HTTPException(
|
|
264
|
+
status_code=HTTP_404_NOT_FOUND, detail="Annotation configuration not found"
|
|
265
|
+
)
|
|
266
|
+
return GetAnnotationConfigResponseBody(data=db_to_api_annotation_config(config))
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
@router.post(
|
|
270
|
+
"/annotation_configs",
|
|
271
|
+
summary="Create an annotation configuration",
|
|
272
|
+
)
|
|
273
|
+
async def create_annotation_config(
|
|
274
|
+
request: Request,
|
|
275
|
+
data: CreateAnnotationConfigData,
|
|
276
|
+
) -> CreateAnnotationConfigResponseBody:
|
|
277
|
+
input_config = data.root
|
|
278
|
+
_reserve_note_annotation_name(input_config)
|
|
279
|
+
|
|
280
|
+
try:
|
|
281
|
+
db_config = _to_db_annotation_config(input_config)
|
|
282
|
+
except ValueError as error:
|
|
283
|
+
raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=str(error))
|
|
284
|
+
|
|
285
|
+
async with request.app.state.db() as session:
|
|
286
|
+
annotation_config = models.AnnotationConfig(
|
|
287
|
+
name=input_config.name,
|
|
288
|
+
config=db_config,
|
|
289
|
+
)
|
|
290
|
+
session.add(annotation_config)
|
|
291
|
+
try:
|
|
292
|
+
await session.commit()
|
|
293
|
+
except (PostgreSQLIntegrityError, SQLiteIntegrityError):
|
|
294
|
+
raise HTTPException(
|
|
295
|
+
status_code=HTTP_409_CONFLICT,
|
|
296
|
+
detail="The name of the annotation configuration is already taken",
|
|
297
|
+
)
|
|
298
|
+
return CreateAnnotationConfigResponseBody(
|
|
299
|
+
data=db_to_api_annotation_config(annotation_config)
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
@router.put(
|
|
304
|
+
"/annotation_configs/{config_id}",
|
|
305
|
+
summary="Update an annotation configuration",
|
|
306
|
+
)
|
|
307
|
+
async def update_annotation_config(
|
|
308
|
+
request: Request,
|
|
309
|
+
data: CreateAnnotationConfigData,
|
|
310
|
+
config_id: str = Path(..., description="ID of the annotation configuration"),
|
|
311
|
+
) -> UpdateAnnotationConfigResponseBody:
|
|
312
|
+
input_config = data.root
|
|
313
|
+
_reserve_note_annotation_name(input_config)
|
|
314
|
+
|
|
315
|
+
config_gid = GlobalID.from_id(config_id)
|
|
316
|
+
if config_gid.type_name not in (
|
|
317
|
+
CategoricalAnnotationConfigType.__name__,
|
|
318
|
+
ContinuousAnnotationConfigType.__name__,
|
|
319
|
+
FreeformAnnotationConfigType.__name__,
|
|
320
|
+
):
|
|
321
|
+
raise HTTPException(
|
|
322
|
+
status_code=HTTP_400_BAD_REQUEST, detail="Invalid annotation configuration ID"
|
|
323
|
+
)
|
|
324
|
+
config_rowid = int(config_gid.node_id)
|
|
325
|
+
|
|
326
|
+
try:
|
|
327
|
+
db_config = _to_db_annotation_config(input_config)
|
|
328
|
+
except ValueError as error:
|
|
329
|
+
raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=str(error))
|
|
330
|
+
|
|
331
|
+
async with request.app.state.db() as session:
|
|
332
|
+
existing_config = await session.get(models.AnnotationConfig, config_rowid)
|
|
333
|
+
if not existing_config:
|
|
334
|
+
raise HTTPException(
|
|
335
|
+
status_code=HTTP_404_NOT_FOUND, detail="Annotation configuration not found"
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
existing_config.name = input_config.name
|
|
339
|
+
existing_config.config = db_config
|
|
340
|
+
|
|
341
|
+
try:
|
|
342
|
+
await session.commit()
|
|
343
|
+
except (PostgreSQLIntegrityError, SQLiteIntegrityError):
|
|
344
|
+
raise HTTPException(
|
|
345
|
+
status_code=HTTP_409_CONFLICT,
|
|
346
|
+
detail="The name of the annotation configuration is already taken",
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
return UpdateAnnotationConfigResponseBody(data=db_to_api_annotation_config(existing_config))
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
@router.delete(
|
|
353
|
+
"/annotation_configs/{config_id}",
|
|
354
|
+
summary="Delete an annotation configuration",
|
|
355
|
+
)
|
|
356
|
+
async def delete_annotation_config(
|
|
357
|
+
request: Request,
|
|
358
|
+
config_id: str = Path(..., description="ID of the annotation configuration"),
|
|
359
|
+
) -> DeleteAnnotationConfigResponseBody:
|
|
360
|
+
config_gid = GlobalID.from_id(config_id)
|
|
361
|
+
if config_gid.type_name not in (
|
|
362
|
+
CategoricalAnnotationConfigType.__name__,
|
|
363
|
+
ContinuousAnnotationConfigType.__name__,
|
|
364
|
+
FreeformAnnotationConfigType.__name__,
|
|
365
|
+
):
|
|
366
|
+
raise HTTPException(
|
|
367
|
+
status_code=HTTP_400_BAD_REQUEST, detail="Invalid annotation configuration ID"
|
|
368
|
+
)
|
|
369
|
+
config_rowid = int(config_gid.node_id)
|
|
370
|
+
async with request.app.state.db() as session:
|
|
371
|
+
stmt = (
|
|
372
|
+
delete(models.AnnotationConfig)
|
|
373
|
+
.where(models.AnnotationConfig.id == config_rowid)
|
|
374
|
+
.returning(models.AnnotationConfig)
|
|
375
|
+
)
|
|
376
|
+
annotation_config = await session.scalar(stmt)
|
|
377
|
+
if annotation_config is None:
|
|
378
|
+
raise HTTPException(
|
|
379
|
+
status_code=HTTP_404_NOT_FOUND, detail="Annotation configuration not found"
|
|
380
|
+
)
|
|
381
|
+
await session.commit()
|
|
382
|
+
return DeleteAnnotationConfigResponseBody(data=db_to_api_annotation_config(annotation_config))
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _get_annotation_config_db_id(config_gid: str) -> int:
|
|
386
|
+
gid = GlobalID.from_id(config_gid)
|
|
387
|
+
type_name, node_id = gid.type_name, int(gid.node_id)
|
|
388
|
+
if type_name not in (
|
|
389
|
+
CategoricalAnnotationConfigType.__name__,
|
|
390
|
+
ContinuousAnnotationConfigType.__name__,
|
|
391
|
+
FreeformAnnotationConfigType.__name__,
|
|
392
|
+
):
|
|
393
|
+
raise ValueError(f"Invalid annotation configuration ID: {config_gid}")
|
|
394
|
+
return node_id
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _reserve_note_annotation_name(data: AnnotationConfigData) -> str:
|
|
398
|
+
name = data.name
|
|
399
|
+
if name == "note":
|
|
400
|
+
raise HTTPException(
|
|
401
|
+
status_code=HTTP_409_CONFLICT, detail="The name 'note' is reserved for span notes"
|
|
402
|
+
)
|
|
403
|
+
return name
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _to_db_annotation_config(input_config: AnnotationConfigData) -> AnnotationConfigType:
|
|
407
|
+
if isinstance(input_config, ContinuousAnnotationConfigData):
|
|
408
|
+
return _to_db_continuous_annotation_config(input_config)
|
|
409
|
+
if isinstance(input_config, CategoricalAnnotationConfigData):
|
|
410
|
+
return _to_db_categorical_annotation_config(input_config)
|
|
411
|
+
if isinstance(input_config, FreeformAnnotationConfigData):
|
|
412
|
+
return _to_db_freeform_annotation_config(input_config)
|
|
413
|
+
assert_never(input_config)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _to_db_continuous_annotation_config(
|
|
417
|
+
input_config: ContinuousAnnotationConfigData,
|
|
418
|
+
) -> ContinuousAnnotationConfigModel:
|
|
419
|
+
return ContinuousAnnotationConfigModel(
|
|
420
|
+
type=AnnotationType.CONTINUOUS.value,
|
|
421
|
+
description=input_config.description,
|
|
422
|
+
optimization_direction=input_config.optimization_direction,
|
|
423
|
+
lower_bound=input_config.lower_bound,
|
|
424
|
+
upper_bound=input_config.upper_bound,
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _to_db_categorical_annotation_config(
|
|
429
|
+
input_config: CategoricalAnnotationConfigData,
|
|
430
|
+
) -> CategoricalAnnotationConfigModel:
|
|
431
|
+
values = [
|
|
432
|
+
CategoricalAnnotationValueModel(label=value.label, score=value.score)
|
|
433
|
+
for value in input_config.values
|
|
434
|
+
]
|
|
435
|
+
return CategoricalAnnotationConfigModel(
|
|
436
|
+
type=AnnotationType.CATEGORICAL.value,
|
|
437
|
+
description=input_config.description,
|
|
438
|
+
optimization_direction=input_config.optimization_direction,
|
|
439
|
+
values=values,
|
|
440
|
+
)
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _to_db_freeform_annotation_config(
|
|
444
|
+
input_config: FreeformAnnotationConfigData,
|
|
445
|
+
) -> FreeformAnnotationConfigModel:
|
|
446
|
+
return FreeformAnnotationConfigModel(
|
|
447
|
+
type=AnnotationType.FREEFORM.value,
|
|
448
|
+
description=input_config.description,
|
|
449
|
+
)
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from typing import Literal, Optional
|
|
6
|
+
|
|
7
|
+
from fastapi import APIRouter, HTTPException, Path, Query
|
|
8
|
+
from sqlalchemy import exists, select
|
|
9
|
+
from starlette.requests import Request
|
|
10
|
+
from starlette.status import HTTP_200_OK, HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY
|
|
11
|
+
from strawberry.relay import GlobalID
|
|
12
|
+
|
|
13
|
+
from phoenix.db import models
|
|
14
|
+
from phoenix.server.api.types.SpanAnnotation import SpanAnnotation as SpanAnnotationNodeType
|
|
15
|
+
from phoenix.server.api.types.User import User as UserNodeType
|
|
16
|
+
|
|
17
|
+
from .spans import SpanAnnotationData, SpanAnnotationResult
|
|
18
|
+
from .utils import PaginatedResponseBody, _get_project_by_identifier, add_errors_to_responses
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
SPAN_ANNOTATION_NODE_NAME = SpanAnnotationNodeType.__name__
|
|
23
|
+
USER_NODE_NAME = UserNodeType.__name__
|
|
24
|
+
MAX_SPAN_IDS = 1_000
|
|
25
|
+
|
|
26
|
+
router = APIRouter(tags=["annotations"])
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SpanAnnotation(SpanAnnotationData):
|
|
30
|
+
id: str
|
|
31
|
+
created_at: datetime
|
|
32
|
+
updated_at: datetime
|
|
33
|
+
source: Literal["API", "APP"]
|
|
34
|
+
user_id: Optional[str]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SpanAnnotationsResponseBody(PaginatedResponseBody[SpanAnnotation]):
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@router.get(
|
|
42
|
+
"/projects/{project_identifier}/span_annotations",
|
|
43
|
+
operation_id="listSpanAnnotationsBySpanIds",
|
|
44
|
+
summary="Get span annotations for a list of span_ids.",
|
|
45
|
+
status_code=HTTP_200_OK,
|
|
46
|
+
responses=add_errors_to_responses(
|
|
47
|
+
[
|
|
48
|
+
{"status_code": HTTP_404_NOT_FOUND, "description": "Project or spans not found"},
|
|
49
|
+
{"status_code": HTTP_422_UNPROCESSABLE_ENTITY, "description": "Invalid parameters"},
|
|
50
|
+
]
|
|
51
|
+
),
|
|
52
|
+
)
|
|
53
|
+
async def list_span_annotations(
|
|
54
|
+
request: Request,
|
|
55
|
+
project_identifier: str = Path(
|
|
56
|
+
description=(
|
|
57
|
+
"The project identifier: either project ID or project name. If using a project name as "
|
|
58
|
+
"the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) "
|
|
59
|
+
"characters."
|
|
60
|
+
)
|
|
61
|
+
),
|
|
62
|
+
span_ids: list[str] = Query(
|
|
63
|
+
..., min_length=1, description="One or more span id to fetch annotations for"
|
|
64
|
+
),
|
|
65
|
+
cursor: Optional[str] = Query(default=None, description="A cursor for pagination"),
|
|
66
|
+
limit: int = Query(
|
|
67
|
+
default=10,
|
|
68
|
+
gt=0,
|
|
69
|
+
le=10000,
|
|
70
|
+
description="The maximum number of annotations to return in a single request",
|
|
71
|
+
),
|
|
72
|
+
) -> SpanAnnotationsResponseBody:
|
|
73
|
+
span_ids = list({*span_ids})
|
|
74
|
+
if len(span_ids) > MAX_SPAN_IDS:
|
|
75
|
+
raise HTTPException(
|
|
76
|
+
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
|
77
|
+
detail=f"Too many span_ids supplied: {len(span_ids)} (max {MAX_SPAN_IDS})",
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
async with request.app.state.db() as session:
|
|
81
|
+
project = await _get_project_by_identifier(session, project_identifier)
|
|
82
|
+
if not project:
|
|
83
|
+
raise HTTPException(
|
|
84
|
+
status_code=HTTP_404_NOT_FOUND,
|
|
85
|
+
detail=f"Project with identifier {project_identifier} not found",
|
|
86
|
+
)
|
|
87
|
+
stmt = (
|
|
88
|
+
select(models.Span.span_id, models.SpanAnnotation)
|
|
89
|
+
.join(models.Trace, models.Span.trace_rowid == models.Trace.id)
|
|
90
|
+
.join(models.Project, models.Trace.project_rowid == models.Project.id)
|
|
91
|
+
.join(models.SpanAnnotation, models.SpanAnnotation.span_rowid == models.Span.id)
|
|
92
|
+
.where(
|
|
93
|
+
models.Project.id == project.id,
|
|
94
|
+
models.Span.span_id.in_(span_ids),
|
|
95
|
+
)
|
|
96
|
+
.order_by(models.SpanAnnotation.id.desc())
|
|
97
|
+
.limit(limit + 1)
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if cursor:
|
|
101
|
+
try:
|
|
102
|
+
cursor_id = int(GlobalID.from_id(cursor).node_id)
|
|
103
|
+
except ValueError:
|
|
104
|
+
raise HTTPException(
|
|
105
|
+
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
|
106
|
+
detail="Invalid cursor value",
|
|
107
|
+
)
|
|
108
|
+
stmt = stmt.where(models.SpanAnnotation.id <= cursor_id)
|
|
109
|
+
|
|
110
|
+
rows: list[tuple[str, models.SpanAnnotation]] = [
|
|
111
|
+
r async for r in (await session.stream(stmt))
|
|
112
|
+
]
|
|
113
|
+
|
|
114
|
+
next_cursor: Optional[str] = None
|
|
115
|
+
if len(rows) == limit + 1:
|
|
116
|
+
*rows, extra = rows
|
|
117
|
+
next_cursor = str(GlobalID(SPAN_ANNOTATION_NODE_NAME, str(extra[1].id)))
|
|
118
|
+
|
|
119
|
+
if not rows:
|
|
120
|
+
spans_exist = await session.scalar(
|
|
121
|
+
select(
|
|
122
|
+
exists().where(
|
|
123
|
+
models.Span.span_id.in_(span_ids),
|
|
124
|
+
models.Span.trace_rowid.in_(
|
|
125
|
+
select(models.Trace.id)
|
|
126
|
+
.join(models.Project)
|
|
127
|
+
.where(models.Project.id == project.id)
|
|
128
|
+
),
|
|
129
|
+
)
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
if not spans_exist:
|
|
133
|
+
raise HTTPException(
|
|
134
|
+
detail="None of the supplied span_ids exist in this project",
|
|
135
|
+
status_code=HTTP_404_NOT_FOUND,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
return SpanAnnotationsResponseBody(data=[], next_cursor=None)
|
|
139
|
+
|
|
140
|
+
data = [
|
|
141
|
+
SpanAnnotation(
|
|
142
|
+
id=str(GlobalID(SPAN_ANNOTATION_NODE_NAME, str(anno.id))),
|
|
143
|
+
span_id=span_id,
|
|
144
|
+
name=anno.name,
|
|
145
|
+
result=SpanAnnotationResult(
|
|
146
|
+
label=anno.label,
|
|
147
|
+
score=anno.score,
|
|
148
|
+
explanation=anno.explanation,
|
|
149
|
+
),
|
|
150
|
+
metadata=anno.metadata_,
|
|
151
|
+
annotator_kind=anno.annotator_kind,
|
|
152
|
+
created_at=anno.created_at,
|
|
153
|
+
updated_at=anno.updated_at,
|
|
154
|
+
identifier=anno.identifier,
|
|
155
|
+
source=anno.source,
|
|
156
|
+
user_id=str(GlobalID(USER_NODE_NAME, str(anno.user_id))) if anno.user_id else None,
|
|
157
|
+
)
|
|
158
|
+
for span_id, anno in rows
|
|
159
|
+
]
|
|
160
|
+
|
|
161
|
+
return SpanAnnotationsResponseBody(data=data, next_cursor=next_cursor)
|
|
@@ -211,6 +211,8 @@ async def _add_evaluations(state: State, evaluations: Evaluations) -> None:
|
|
|
211
211
|
score, label, explanation = _get_annotation_result(row)
|
|
212
212
|
document_annotation = cls(cast(Union[tuple[str, int], tuple[int, str]], index))(
|
|
213
213
|
name=eval_name,
|
|
214
|
+
identifier="",
|
|
215
|
+
source="API",
|
|
214
216
|
annotator_kind="LLM",
|
|
215
217
|
score=score,
|
|
216
218
|
label=label,
|
|
@@ -223,6 +225,8 @@ async def _add_evaluations(state: State, evaluations: Evaluations) -> None:
|
|
|
223
225
|
score, label, explanation = _get_annotation_result(row)
|
|
224
226
|
span_annotation = _span_annotation_factory(cast(str, index))(
|
|
225
227
|
name=eval_name,
|
|
228
|
+
identifier="",
|
|
229
|
+
source="API",
|
|
226
230
|
annotator_kind="LLM",
|
|
227
231
|
score=score,
|
|
228
232
|
label=label,
|
|
@@ -235,6 +239,8 @@ async def _add_evaluations(state: State, evaluations: Evaluations) -> None:
|
|
|
235
239
|
score, label, explanation = _get_annotation_result(row)
|
|
236
240
|
trace_annotation = _trace_annotation_factory(cast(str, index))(
|
|
237
241
|
name=eval_name,
|
|
242
|
+
identifier="",
|
|
243
|
+
source="API",
|
|
238
244
|
annotator_kind="LLM",
|
|
239
245
|
score=score,
|
|
240
246
|
label=label,
|
|
@@ -3,7 +3,6 @@ from typing import Optional
|
|
|
3
3
|
from fastapi import APIRouter, HTTPException, Path, Query
|
|
4
4
|
from pydantic import Field
|
|
5
5
|
from sqlalchemy import select
|
|
6
|
-
from sqlalchemy.ext.asyncio import AsyncSession
|
|
7
6
|
from starlette.requests import Request
|
|
8
7
|
from starlette.status import (
|
|
9
8
|
HTTP_204_NO_CONTENT,
|
|
@@ -21,9 +20,9 @@ from phoenix.server.api.routers.v1.models import V1RoutesBaseModel
|
|
|
21
20
|
from phoenix.server.api.routers.v1.utils import (
|
|
22
21
|
PaginatedResponseBody,
|
|
23
22
|
ResponseBody,
|
|
23
|
+
_get_project_by_identifier,
|
|
24
24
|
add_errors_to_responses,
|
|
25
25
|
)
|
|
26
|
-
from phoenix.server.api.types.node import from_global_id_with_expected_type
|
|
27
26
|
from phoenix.server.api.types.Project import Project as ProjectNodeType
|
|
28
27
|
|
|
29
28
|
router = APIRouter(tags=["projects"])
|
|
@@ -343,51 +342,3 @@ def _to_project_response(project: models.Project) -> Project:
|
|
|
343
342
|
name=project.name,
|
|
344
343
|
description=project.description,
|
|
345
344
|
)
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
async def _get_project_by_identifier(
|
|
349
|
-
session: AsyncSession,
|
|
350
|
-
project_identifier: str,
|
|
351
|
-
) -> models.Project:
|
|
352
|
-
"""
|
|
353
|
-
Get a project by its ID or name.
|
|
354
|
-
|
|
355
|
-
Args:
|
|
356
|
-
session: The database session.
|
|
357
|
-
project_identifier: The project ID or name.
|
|
358
|
-
|
|
359
|
-
Returns:
|
|
360
|
-
The project object.
|
|
361
|
-
|
|
362
|
-
Raises:
|
|
363
|
-
HTTPException: If the identifier format is invalid or the project is not found.
|
|
364
|
-
"""
|
|
365
|
-
# Try to parse as a GlobalID first
|
|
366
|
-
try:
|
|
367
|
-
id_ = from_global_id_with_expected_type(
|
|
368
|
-
GlobalID.from_id(project_identifier),
|
|
369
|
-
ProjectNodeType.__name__,
|
|
370
|
-
)
|
|
371
|
-
except Exception:
|
|
372
|
-
try:
|
|
373
|
-
name = project_identifier
|
|
374
|
-
except HTTPException:
|
|
375
|
-
raise HTTPException(
|
|
376
|
-
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
|
377
|
-
detail=f"Invalid project identifier format: {project_identifier}",
|
|
378
|
-
)
|
|
379
|
-
stmt = select(models.Project).filter_by(name=name)
|
|
380
|
-
project = await session.scalar(stmt)
|
|
381
|
-
if project is None:
|
|
382
|
-
raise HTTPException(
|
|
383
|
-
status_code=HTTP_404_NOT_FOUND,
|
|
384
|
-
detail=f"Project with name {name} not found",
|
|
385
|
-
)
|
|
386
|
-
else:
|
|
387
|
-
project = await session.get(models.Project, id_)
|
|
388
|
-
if project is None:
|
|
389
|
-
raise HTTPException(
|
|
390
|
-
status_code=HTTP_404_NOT_FOUND,
|
|
391
|
-
detail=f"Project with ID {project_identifier} not found",
|
|
392
|
-
)
|
|
393
|
-
return project
|