arize-phoenix 3.16.0__py3-none-any.whl → 7.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of arize-phoenix might be problematic. Click here for more details.
- arize_phoenix-7.7.0.dist-info/METADATA +261 -0
- arize_phoenix-7.7.0.dist-info/RECORD +345 -0
- {arize_phoenix-3.16.0.dist-info → arize_phoenix-7.7.0.dist-info}/WHEEL +1 -1
- arize_phoenix-7.7.0.dist-info/entry_points.txt +3 -0
- phoenix/__init__.py +86 -14
- phoenix/auth.py +309 -0
- phoenix/config.py +675 -45
- phoenix/core/model.py +32 -30
- phoenix/core/model_schema.py +102 -109
- phoenix/core/model_schema_adapter.py +48 -45
- phoenix/datetime_utils.py +24 -3
- phoenix/db/README.md +54 -0
- phoenix/db/__init__.py +4 -0
- phoenix/db/alembic.ini +85 -0
- phoenix/db/bulk_inserter.py +294 -0
- phoenix/db/engines.py +208 -0
- phoenix/db/enums.py +20 -0
- phoenix/db/facilitator.py +113 -0
- phoenix/db/helpers.py +159 -0
- phoenix/db/insertion/constants.py +2 -0
- phoenix/db/insertion/dataset.py +227 -0
- phoenix/db/insertion/document_annotation.py +171 -0
- phoenix/db/insertion/evaluation.py +191 -0
- phoenix/db/insertion/helpers.py +98 -0
- phoenix/db/insertion/span.py +193 -0
- phoenix/db/insertion/span_annotation.py +158 -0
- phoenix/db/insertion/trace_annotation.py +158 -0
- phoenix/db/insertion/types.py +256 -0
- phoenix/db/migrate.py +86 -0
- phoenix/db/migrations/data_migration_scripts/populate_project_sessions.py +199 -0
- phoenix/db/migrations/env.py +114 -0
- phoenix/db/migrations/script.py.mako +26 -0
- phoenix/db/migrations/versions/10460e46d750_datasets.py +317 -0
- phoenix/db/migrations/versions/3be8647b87d8_add_token_columns_to_spans_table.py +126 -0
- phoenix/db/migrations/versions/4ded9e43755f_create_project_sessions_table.py +66 -0
- phoenix/db/migrations/versions/cd164e83824f_users_and_tokens.py +157 -0
- phoenix/db/migrations/versions/cf03bd6bae1d_init.py +280 -0
- phoenix/db/models.py +807 -0
- phoenix/exceptions.py +5 -1
- phoenix/experiments/__init__.py +6 -0
- phoenix/experiments/evaluators/__init__.py +29 -0
- phoenix/experiments/evaluators/base.py +158 -0
- phoenix/experiments/evaluators/code_evaluators.py +184 -0
- phoenix/experiments/evaluators/llm_evaluators.py +473 -0
- phoenix/experiments/evaluators/utils.py +236 -0
- phoenix/experiments/functions.py +772 -0
- phoenix/experiments/tracing.py +86 -0
- phoenix/experiments/types.py +726 -0
- phoenix/experiments/utils.py +25 -0
- phoenix/inferences/__init__.py +0 -0
- phoenix/{datasets → inferences}/errors.py +6 -5
- phoenix/{datasets → inferences}/fixtures.py +49 -42
- phoenix/{datasets/dataset.py → inferences/inferences.py} +121 -105
- phoenix/{datasets → inferences}/schema.py +11 -11
- phoenix/{datasets → inferences}/validation.py +13 -14
- phoenix/logging/__init__.py +3 -0
- phoenix/logging/_config.py +90 -0
- phoenix/logging/_filter.py +6 -0
- phoenix/logging/_formatter.py +69 -0
- phoenix/metrics/__init__.py +5 -4
- phoenix/metrics/binning.py +4 -3
- phoenix/metrics/metrics.py +2 -1
- phoenix/metrics/mixins.py +7 -6
- phoenix/metrics/retrieval_metrics.py +2 -1
- phoenix/metrics/timeseries.py +5 -4
- phoenix/metrics/wrappers.py +9 -3
- phoenix/pointcloud/clustering.py +5 -5
- phoenix/pointcloud/pointcloud.py +7 -5
- phoenix/pointcloud/projectors.py +5 -6
- phoenix/pointcloud/umap_parameters.py +53 -52
- phoenix/server/api/README.md +28 -0
- phoenix/server/api/auth.py +44 -0
- phoenix/server/api/context.py +152 -9
- phoenix/server/api/dataloaders/__init__.py +91 -0
- phoenix/server/api/dataloaders/annotation_summaries.py +139 -0
- phoenix/server/api/dataloaders/average_experiment_run_latency.py +54 -0
- phoenix/server/api/dataloaders/cache/__init__.py +3 -0
- phoenix/server/api/dataloaders/cache/two_tier_cache.py +68 -0
- phoenix/server/api/dataloaders/dataset_example_revisions.py +131 -0
- phoenix/server/api/dataloaders/dataset_example_spans.py +38 -0
- phoenix/server/api/dataloaders/document_evaluation_summaries.py +144 -0
- phoenix/server/api/dataloaders/document_evaluations.py +31 -0
- phoenix/server/api/dataloaders/document_retrieval_metrics.py +89 -0
- phoenix/server/api/dataloaders/experiment_annotation_summaries.py +79 -0
- phoenix/server/api/dataloaders/experiment_error_rates.py +58 -0
- phoenix/server/api/dataloaders/experiment_run_annotations.py +36 -0
- phoenix/server/api/dataloaders/experiment_run_counts.py +49 -0
- phoenix/server/api/dataloaders/experiment_sequence_number.py +44 -0
- phoenix/server/api/dataloaders/latency_ms_quantile.py +188 -0
- phoenix/server/api/dataloaders/min_start_or_max_end_times.py +85 -0
- phoenix/server/api/dataloaders/project_by_name.py +31 -0
- phoenix/server/api/dataloaders/record_counts.py +116 -0
- phoenix/server/api/dataloaders/session_io.py +79 -0
- phoenix/server/api/dataloaders/session_num_traces.py +30 -0
- phoenix/server/api/dataloaders/session_num_traces_with_error.py +32 -0
- phoenix/server/api/dataloaders/session_token_usages.py +41 -0
- phoenix/server/api/dataloaders/session_trace_latency_ms_quantile.py +55 -0
- phoenix/server/api/dataloaders/span_annotations.py +26 -0
- phoenix/server/api/dataloaders/span_dataset_examples.py +31 -0
- phoenix/server/api/dataloaders/span_descendants.py +57 -0
- phoenix/server/api/dataloaders/span_projects.py +33 -0
- phoenix/server/api/dataloaders/token_counts.py +124 -0
- phoenix/server/api/dataloaders/trace_by_trace_ids.py +25 -0
- phoenix/server/api/dataloaders/trace_root_spans.py +32 -0
- phoenix/server/api/dataloaders/user_roles.py +30 -0
- phoenix/server/api/dataloaders/users.py +33 -0
- phoenix/server/api/exceptions.py +48 -0
- phoenix/server/api/helpers/__init__.py +12 -0
- phoenix/server/api/helpers/dataset_helpers.py +217 -0
- phoenix/server/api/helpers/experiment_run_filters.py +763 -0
- phoenix/server/api/helpers/playground_clients.py +948 -0
- phoenix/server/api/helpers/playground_registry.py +70 -0
- phoenix/server/api/helpers/playground_spans.py +455 -0
- phoenix/server/api/input_types/AddExamplesToDatasetInput.py +16 -0
- phoenix/server/api/input_types/AddSpansToDatasetInput.py +14 -0
- phoenix/server/api/input_types/ChatCompletionInput.py +38 -0
- phoenix/server/api/input_types/ChatCompletionMessageInput.py +24 -0
- phoenix/server/api/input_types/ClearProjectInput.py +15 -0
- phoenix/server/api/input_types/ClusterInput.py +2 -2
- phoenix/server/api/input_types/CreateDatasetInput.py +12 -0
- phoenix/server/api/input_types/CreateSpanAnnotationInput.py +18 -0
- phoenix/server/api/input_types/CreateTraceAnnotationInput.py +18 -0
- phoenix/server/api/input_types/DataQualityMetricInput.py +5 -2
- phoenix/server/api/input_types/DatasetExampleInput.py +14 -0
- phoenix/server/api/input_types/DatasetSort.py +17 -0
- phoenix/server/api/input_types/DatasetVersionSort.py +16 -0
- phoenix/server/api/input_types/DeleteAnnotationsInput.py +7 -0
- phoenix/server/api/input_types/DeleteDatasetExamplesInput.py +13 -0
- phoenix/server/api/input_types/DeleteDatasetInput.py +7 -0
- phoenix/server/api/input_types/DeleteExperimentsInput.py +7 -0
- phoenix/server/api/input_types/DimensionFilter.py +4 -4
- phoenix/server/api/input_types/GenerativeModelInput.py +17 -0
- phoenix/server/api/input_types/Granularity.py +1 -1
- phoenix/server/api/input_types/InvocationParameters.py +162 -0
- phoenix/server/api/input_types/PatchAnnotationInput.py +19 -0
- phoenix/server/api/input_types/PatchDatasetExamplesInput.py +35 -0
- phoenix/server/api/input_types/PatchDatasetInput.py +14 -0
- phoenix/server/api/input_types/PerformanceMetricInput.py +5 -2
- phoenix/server/api/input_types/ProjectSessionSort.py +29 -0
- phoenix/server/api/input_types/SpanAnnotationSort.py +17 -0
- phoenix/server/api/input_types/SpanSort.py +134 -69
- phoenix/server/api/input_types/TemplateOptions.py +10 -0
- phoenix/server/api/input_types/TraceAnnotationSort.py +17 -0
- phoenix/server/api/input_types/UserRoleInput.py +9 -0
- phoenix/server/api/mutations/__init__.py +28 -0
- phoenix/server/api/mutations/api_key_mutations.py +167 -0
- phoenix/server/api/mutations/chat_mutations.py +593 -0
- phoenix/server/api/mutations/dataset_mutations.py +591 -0
- phoenix/server/api/mutations/experiment_mutations.py +75 -0
- phoenix/server/api/{types/ExportEventsMutation.py → mutations/export_events_mutations.py} +21 -18
- phoenix/server/api/mutations/project_mutations.py +57 -0
- phoenix/server/api/mutations/span_annotations_mutations.py +128 -0
- phoenix/server/api/mutations/trace_annotations_mutations.py +127 -0
- phoenix/server/api/mutations/user_mutations.py +329 -0
- phoenix/server/api/openapi/__init__.py +0 -0
- phoenix/server/api/openapi/main.py +17 -0
- phoenix/server/api/openapi/schema.py +16 -0
- phoenix/server/api/queries.py +738 -0
- phoenix/server/api/routers/__init__.py +11 -0
- phoenix/server/api/routers/auth.py +284 -0
- phoenix/server/api/routers/embeddings.py +26 -0
- phoenix/server/api/routers/oauth2.py +488 -0
- phoenix/server/api/routers/v1/__init__.py +64 -0
- phoenix/server/api/routers/v1/datasets.py +1017 -0
- phoenix/server/api/routers/v1/evaluations.py +362 -0
- phoenix/server/api/routers/v1/experiment_evaluations.py +115 -0
- phoenix/server/api/routers/v1/experiment_runs.py +167 -0
- phoenix/server/api/routers/v1/experiments.py +308 -0
- phoenix/server/api/routers/v1/pydantic_compat.py +78 -0
- phoenix/server/api/routers/v1/spans.py +267 -0
- phoenix/server/api/routers/v1/traces.py +208 -0
- phoenix/server/api/routers/v1/utils.py +95 -0
- phoenix/server/api/schema.py +44 -247
- phoenix/server/api/subscriptions.py +597 -0
- phoenix/server/api/types/Annotation.py +21 -0
- phoenix/server/api/types/AnnotationSummary.py +55 -0
- phoenix/server/api/types/AnnotatorKind.py +16 -0
- phoenix/server/api/types/ApiKey.py +27 -0
- phoenix/server/api/types/AuthMethod.py +9 -0
- phoenix/server/api/types/ChatCompletionMessageRole.py +11 -0
- phoenix/server/api/types/ChatCompletionSubscriptionPayload.py +46 -0
- phoenix/server/api/types/Cluster.py +25 -24
- phoenix/server/api/types/CreateDatasetPayload.py +8 -0
- phoenix/server/api/types/DataQualityMetric.py +31 -13
- phoenix/server/api/types/Dataset.py +288 -63
- phoenix/server/api/types/DatasetExample.py +85 -0
- phoenix/server/api/types/DatasetExampleRevision.py +34 -0
- phoenix/server/api/types/DatasetVersion.py +14 -0
- phoenix/server/api/types/Dimension.py +32 -31
- phoenix/server/api/types/DocumentEvaluationSummary.py +9 -8
- phoenix/server/api/types/EmbeddingDimension.py +56 -49
- phoenix/server/api/types/Evaluation.py +25 -31
- phoenix/server/api/types/EvaluationSummary.py +30 -50
- phoenix/server/api/types/Event.py +20 -20
- phoenix/server/api/types/ExampleRevisionInterface.py +14 -0
- phoenix/server/api/types/Experiment.py +152 -0
- phoenix/server/api/types/ExperimentAnnotationSummary.py +13 -0
- phoenix/server/api/types/ExperimentComparison.py +17 -0
- phoenix/server/api/types/ExperimentRun.py +119 -0
- phoenix/server/api/types/ExperimentRunAnnotation.py +56 -0
- phoenix/server/api/types/GenerativeModel.py +9 -0
- phoenix/server/api/types/GenerativeProvider.py +85 -0
- phoenix/server/api/types/Inferences.py +80 -0
- phoenix/server/api/types/InferencesRole.py +23 -0
- phoenix/server/api/types/LabelFraction.py +7 -0
- phoenix/server/api/types/MimeType.py +2 -2
- phoenix/server/api/types/Model.py +54 -54
- phoenix/server/api/types/PerformanceMetric.py +8 -5
- phoenix/server/api/types/Project.py +407 -142
- phoenix/server/api/types/ProjectSession.py +139 -0
- phoenix/server/api/types/Segments.py +4 -4
- phoenix/server/api/types/Span.py +221 -176
- phoenix/server/api/types/SpanAnnotation.py +43 -0
- phoenix/server/api/types/SpanIOValue.py +15 -0
- phoenix/server/api/types/SystemApiKey.py +9 -0
- phoenix/server/api/types/TemplateLanguage.py +10 -0
- phoenix/server/api/types/TimeSeries.py +19 -15
- phoenix/server/api/types/TokenUsage.py +11 -0
- phoenix/server/api/types/Trace.py +154 -0
- phoenix/server/api/types/TraceAnnotation.py +45 -0
- phoenix/server/api/types/UMAPPoints.py +7 -7
- phoenix/server/api/types/User.py +60 -0
- phoenix/server/api/types/UserApiKey.py +45 -0
- phoenix/server/api/types/UserRole.py +15 -0
- phoenix/server/api/types/node.py +13 -107
- phoenix/server/api/types/pagination.py +156 -57
- phoenix/server/api/utils.py +34 -0
- phoenix/server/app.py +864 -115
- phoenix/server/bearer_auth.py +163 -0
- phoenix/server/dml_event.py +136 -0
- phoenix/server/dml_event_handler.py +256 -0
- phoenix/server/email/__init__.py +0 -0
- phoenix/server/email/sender.py +97 -0
- phoenix/server/email/templates/__init__.py +0 -0
- phoenix/server/email/templates/password_reset.html +19 -0
- phoenix/server/email/types.py +11 -0
- phoenix/server/grpc_server.py +102 -0
- phoenix/server/jwt_store.py +505 -0
- phoenix/server/main.py +305 -116
- phoenix/server/oauth2.py +52 -0
- phoenix/server/openapi/__init__.py +0 -0
- phoenix/server/prometheus.py +111 -0
- phoenix/server/rate_limiters.py +188 -0
- phoenix/server/static/.vite/manifest.json +87 -0
- phoenix/server/static/assets/components-Cy9nwIvF.js +2125 -0
- phoenix/server/static/assets/index-BKvHIxkk.js +113 -0
- phoenix/server/static/assets/pages-CUi2xCVQ.js +4449 -0
- phoenix/server/static/assets/vendor-DvC8cT4X.js +894 -0
- phoenix/server/static/assets/vendor-DxkFTwjz.css +1 -0
- phoenix/server/static/assets/vendor-arizeai-Do1793cv.js +662 -0
- phoenix/server/static/assets/vendor-codemirror-BzwZPyJM.js +24 -0
- phoenix/server/static/assets/vendor-recharts-_Jb7JjhG.js +59 -0
- phoenix/server/static/assets/vendor-shiki-Cl9QBraO.js +5 -0
- phoenix/server/static/assets/vendor-three-DwGkEfCM.js +2998 -0
- phoenix/server/telemetry.py +68 -0
- phoenix/server/templates/index.html +82 -23
- phoenix/server/thread_server.py +3 -3
- phoenix/server/types.py +275 -0
- phoenix/services.py +27 -18
- phoenix/session/client.py +743 -68
- phoenix/session/data_extractor.py +31 -7
- phoenix/session/evaluation.py +3 -9
- phoenix/session/session.py +263 -219
- phoenix/settings.py +22 -0
- phoenix/trace/__init__.py +2 -22
- phoenix/trace/attributes.py +338 -0
- phoenix/trace/dsl/README.md +116 -0
- phoenix/trace/dsl/filter.py +663 -213
- phoenix/trace/dsl/helpers.py +73 -21
- phoenix/trace/dsl/query.py +574 -201
- phoenix/trace/exporter.py +24 -19
- phoenix/trace/fixtures.py +368 -32
- phoenix/trace/otel.py +71 -219
- phoenix/trace/projects.py +3 -2
- phoenix/trace/schemas.py +33 -11
- phoenix/trace/span_evaluations.py +21 -16
- phoenix/trace/span_json_decoder.py +6 -4
- phoenix/trace/span_json_encoder.py +2 -2
- phoenix/trace/trace_dataset.py +47 -32
- phoenix/trace/utils.py +21 -4
- phoenix/utilities/__init__.py +0 -26
- phoenix/utilities/client.py +132 -0
- phoenix/utilities/deprecation.py +31 -0
- phoenix/utilities/error_handling.py +3 -2
- phoenix/utilities/json.py +109 -0
- phoenix/utilities/logging.py +8 -0
- phoenix/utilities/project.py +2 -2
- phoenix/utilities/re.py +49 -0
- phoenix/utilities/span_store.py +0 -23
- phoenix/utilities/template_formatters.py +99 -0
- phoenix/version.py +1 -1
- arize_phoenix-3.16.0.dist-info/METADATA +0 -495
- arize_phoenix-3.16.0.dist-info/RECORD +0 -178
- phoenix/core/project.py +0 -617
- phoenix/core/traces.py +0 -100
- phoenix/experimental/evals/__init__.py +0 -73
- phoenix/experimental/evals/evaluators.py +0 -413
- phoenix/experimental/evals/functions/__init__.py +0 -4
- phoenix/experimental/evals/functions/classify.py +0 -453
- phoenix/experimental/evals/functions/executor.py +0 -353
- phoenix/experimental/evals/functions/generate.py +0 -138
- phoenix/experimental/evals/functions/processing.py +0 -76
- phoenix/experimental/evals/models/__init__.py +0 -14
- phoenix/experimental/evals/models/anthropic.py +0 -175
- phoenix/experimental/evals/models/base.py +0 -170
- phoenix/experimental/evals/models/bedrock.py +0 -221
- phoenix/experimental/evals/models/litellm.py +0 -134
- phoenix/experimental/evals/models/openai.py +0 -448
- phoenix/experimental/evals/models/rate_limiters.py +0 -246
- phoenix/experimental/evals/models/vertex.py +0 -173
- phoenix/experimental/evals/models/vertexai.py +0 -186
- phoenix/experimental/evals/retrievals.py +0 -96
- phoenix/experimental/evals/templates/__init__.py +0 -50
- phoenix/experimental/evals/templates/default_templates.py +0 -472
- phoenix/experimental/evals/templates/template.py +0 -195
- phoenix/experimental/evals/utils/__init__.py +0 -172
- phoenix/experimental/evals/utils/threads.py +0 -27
- phoenix/server/api/helpers.py +0 -11
- phoenix/server/api/routers/evaluation_handler.py +0 -109
- phoenix/server/api/routers/span_handler.py +0 -70
- phoenix/server/api/routers/trace_handler.py +0 -60
- phoenix/server/api/types/DatasetRole.py +0 -23
- phoenix/server/static/index.css +0 -6
- phoenix/server/static/index.js +0 -7447
- phoenix/storage/span_store/__init__.py +0 -23
- phoenix/storage/span_store/text_file.py +0 -85
- phoenix/trace/dsl/missing.py +0 -60
- phoenix/trace/langchain/__init__.py +0 -3
- phoenix/trace/langchain/instrumentor.py +0 -35
- phoenix/trace/llama_index/__init__.py +0 -3
- phoenix/trace/llama_index/callback.py +0 -102
- phoenix/trace/openai/__init__.py +0 -3
- phoenix/trace/openai/instrumentor.py +0 -30
- {arize_phoenix-3.16.0.dist-info → arize_phoenix-7.7.0.dist-info}/licenses/IP_NOTICE +0 -0
- {arize_phoenix-3.16.0.dist-info → arize_phoenix-7.7.0.dist-info}/licenses/LICENSE +0 -0
- /phoenix/{datasets → db/insertion}/__init__.py +0 -0
- /phoenix/{experimental → db/migrations}/__init__.py +0 -0
- /phoenix/{storage → db/migrations/data_migration_scripts}/__init__.py +0 -0
|
@@ -1,195 +0,0 @@
|
|
|
1
|
-
import re
|
|
2
|
-
from dataclasses import dataclass
|
|
3
|
-
from typing import Callable, List, Mapping, Optional, Tuple, Union
|
|
4
|
-
|
|
5
|
-
import pandas as pd
|
|
6
|
-
|
|
7
|
-
from phoenix.exceptions import PhoenixException
|
|
8
|
-
from phoenix.experimental.evals.utils import NOT_PARSABLE
|
|
9
|
-
|
|
10
|
-
DEFAULT_START_DELIM = "{"
|
|
11
|
-
DEFAULT_END_DELIM = "}"
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
@dataclass
|
|
15
|
-
class PromptOptions:
|
|
16
|
-
provide_explanation: bool = False
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
class InvalidClassificationTemplateError(PhoenixException):
|
|
20
|
-
pass
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
class PromptTemplate:
|
|
24
|
-
template: str
|
|
25
|
-
variables: List[str]
|
|
26
|
-
|
|
27
|
-
def __init__(
|
|
28
|
-
self,
|
|
29
|
-
template: str,
|
|
30
|
-
delimiters: Tuple[str, str] = (DEFAULT_START_DELIM, DEFAULT_END_DELIM),
|
|
31
|
-
):
|
|
32
|
-
self.template = template
|
|
33
|
-
self._start_delim, self._end_delim = delimiters
|
|
34
|
-
self.variables = self._parse_variables(self.template)
|
|
35
|
-
|
|
36
|
-
def prompt(self, options: Optional[PromptOptions] = None) -> str:
|
|
37
|
-
return self.template
|
|
38
|
-
|
|
39
|
-
def format(
|
|
40
|
-
self,
|
|
41
|
-
variable_values: Mapping[str, Union[bool, int, float, str]],
|
|
42
|
-
options: Optional[PromptOptions] = None,
|
|
43
|
-
) -> str:
|
|
44
|
-
prompt = self.prompt(options)
|
|
45
|
-
for variable_name in self.variables:
|
|
46
|
-
prompt = prompt.replace(
|
|
47
|
-
self._start_delim + variable_name + self._end_delim,
|
|
48
|
-
str(variable_values[variable_name]),
|
|
49
|
-
)
|
|
50
|
-
return prompt
|
|
51
|
-
|
|
52
|
-
def _parse_variables(self, text: str) -> List[str]:
|
|
53
|
-
pattern = re.escape(self._start_delim) + "(.*?)" + re.escape(self._end_delim)
|
|
54
|
-
variables = re.findall(pattern, text)
|
|
55
|
-
return variables
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
class ClassificationTemplate(PromptTemplate):
|
|
59
|
-
def __init__(
|
|
60
|
-
self,
|
|
61
|
-
rails: List[str],
|
|
62
|
-
template: str,
|
|
63
|
-
explanation_template: Optional[str] = None,
|
|
64
|
-
explanation_label_parser: Optional[Callable[[str], str]] = None,
|
|
65
|
-
delimiters: Tuple[str, str] = (DEFAULT_START_DELIM, DEFAULT_END_DELIM),
|
|
66
|
-
scores: Optional[List[float]] = None,
|
|
67
|
-
):
|
|
68
|
-
if scores is not None and len(rails) != len(scores):
|
|
69
|
-
raise InvalidClassificationTemplateError(
|
|
70
|
-
"If scores are provided, each rail must have one and only one score "
|
|
71
|
-
"(i.e., the length of both lists must be the same)."
|
|
72
|
-
)
|
|
73
|
-
self.rails = rails
|
|
74
|
-
self.template = template
|
|
75
|
-
self.explanation_template = explanation_template
|
|
76
|
-
self.explanation_label_parser = explanation_label_parser
|
|
77
|
-
self._start_delim, self._end_delim = delimiters
|
|
78
|
-
self.variables: List[str] = []
|
|
79
|
-
for text in [template, explanation_template]:
|
|
80
|
-
if text is not None:
|
|
81
|
-
self.variables += self._parse_variables(text)
|
|
82
|
-
self._scores = scores
|
|
83
|
-
|
|
84
|
-
def __repr__(self) -> str:
|
|
85
|
-
return self.template
|
|
86
|
-
|
|
87
|
-
def prompt(self, options: Optional[PromptOptions] = None) -> str:
|
|
88
|
-
if options is None:
|
|
89
|
-
return self.template
|
|
90
|
-
|
|
91
|
-
if options.provide_explanation and self.explanation_template:
|
|
92
|
-
return self.explanation_template
|
|
93
|
-
else:
|
|
94
|
-
return self.template
|
|
95
|
-
|
|
96
|
-
def extract_label_from_explanation(self, raw_string: str) -> str:
|
|
97
|
-
if parser := self.explanation_label_parser:
|
|
98
|
-
return parser(raw_string)
|
|
99
|
-
return parse_label_from_chain_of_thought_response(raw_string)
|
|
100
|
-
|
|
101
|
-
def score(self, rail: str) -> float:
|
|
102
|
-
if self._scores is None:
|
|
103
|
-
return 0.0
|
|
104
|
-
try:
|
|
105
|
-
return self._scores[self.rails.index(rail)]
|
|
106
|
-
except (IndexError, ValueError):
|
|
107
|
-
return 0.0
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
def parse_label_from_chain_of_thought_response(raw_string: str) -> str:
|
|
111
|
-
label_delimiter = r"\W*label\W*"
|
|
112
|
-
parts = re.split(label_delimiter, raw_string, maxsplit=1, flags=re.IGNORECASE)
|
|
113
|
-
if len(parts) == 2:
|
|
114
|
-
return parts[1]
|
|
115
|
-
return NOT_PARSABLE
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def normalize_classification_template(
|
|
119
|
-
rails: List[str], template: Union[PromptTemplate, ClassificationTemplate, str]
|
|
120
|
-
) -> ClassificationTemplate:
|
|
121
|
-
"""
|
|
122
|
-
Normalizes a template to a ClassificationTemplate object.
|
|
123
|
-
Args:
|
|
124
|
-
template (Union[ClassificationTemplate, str]): The template to be normalized.
|
|
125
|
-
Returns:
|
|
126
|
-
ClassificationTemplate: The normalized template.
|
|
127
|
-
"""
|
|
128
|
-
if isinstance(template, ClassificationTemplate):
|
|
129
|
-
return template
|
|
130
|
-
|
|
131
|
-
if isinstance(template, PromptTemplate):
|
|
132
|
-
return ClassificationTemplate(rails=rails, template=template.template)
|
|
133
|
-
|
|
134
|
-
if isinstance(template, str):
|
|
135
|
-
return ClassificationTemplate(rails=rails, template=template)
|
|
136
|
-
|
|
137
|
-
raise TypeError(
|
|
138
|
-
"Invalid type for argument `template`. Expected a string or ClassificationTemplate "
|
|
139
|
-
f"but found {type(template)}."
|
|
140
|
-
)
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
def normalize_prompt_template(template: Union[PromptTemplate, str]) -> PromptTemplate:
|
|
144
|
-
"""
|
|
145
|
-
Normalizes a template to a PromptTemplate object.
|
|
146
|
-
Args:
|
|
147
|
-
template (Union[PromptTemplate, str]): The template to be normalized.
|
|
148
|
-
Returns:
|
|
149
|
-
PromptTemplate: The normalized template.
|
|
150
|
-
"""
|
|
151
|
-
if isinstance(template, PromptTemplate):
|
|
152
|
-
return template
|
|
153
|
-
|
|
154
|
-
if isinstance(template, str):
|
|
155
|
-
return PromptTemplate(template=template)
|
|
156
|
-
|
|
157
|
-
raise TypeError(
|
|
158
|
-
"Invalid type for argument `template`. Expected a string or PromptTemplate "
|
|
159
|
-
f"but found {type(template)}."
|
|
160
|
-
)
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
def map_template(
|
|
164
|
-
dataframe: pd.DataFrame,
|
|
165
|
-
template: PromptTemplate,
|
|
166
|
-
options: Optional[PromptOptions] = None,
|
|
167
|
-
) -> "pd.Series[str]":
|
|
168
|
-
"""
|
|
169
|
-
Maps over a dataframe to construct a list of prompts from a template and a dataframe.
|
|
170
|
-
"""
|
|
171
|
-
# Was considering to construct the prompts and generate answers concurrently. However,
|
|
172
|
-
# if there's errors in the prompt construction it could interrupt the process and we
|
|
173
|
-
# would've used API credits for nothing. We could solve this problem by streaming the
|
|
174
|
-
# answers so that, if there is an error, we keep the answers obtained up to that point.
|
|
175
|
-
# These are out of scope for M0, but good to keep in mind and consider for the future.
|
|
176
|
-
prompt_options: PromptOptions = PromptOptions() if options is None else options
|
|
177
|
-
|
|
178
|
-
try:
|
|
179
|
-
prompts = dataframe.apply(
|
|
180
|
-
lambda row: template.format(
|
|
181
|
-
variable_values={var_name: row[var_name] for var_name in template.variables},
|
|
182
|
-
options=prompt_options,
|
|
183
|
-
),
|
|
184
|
-
axis=1,
|
|
185
|
-
)
|
|
186
|
-
return prompts
|
|
187
|
-
except KeyError as e:
|
|
188
|
-
raise RuntimeError(
|
|
189
|
-
f"Error while constructing the prompts from the template and dataframe. "
|
|
190
|
-
f"The template variable {e} is not found as a column in the dataframe."
|
|
191
|
-
)
|
|
192
|
-
except Exception as e:
|
|
193
|
-
raise RuntimeError(
|
|
194
|
-
f"Error while constructing the prompts from the template and dataframe variables: {e}."
|
|
195
|
-
)
|
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from io import BytesIO
|
|
3
|
-
from typing import Any, Dict, List, Optional, Tuple
|
|
4
|
-
from urllib.error import HTTPError
|
|
5
|
-
from urllib.request import urlopen
|
|
6
|
-
from zipfile import ZipFile
|
|
7
|
-
|
|
8
|
-
import pandas as pd
|
|
9
|
-
|
|
10
|
-
from phoenix.utilities.logging import printif
|
|
11
|
-
|
|
12
|
-
# Rather than returning None, we return this string to indicate that the LLM output could not be
|
|
13
|
-
# parsed.
|
|
14
|
-
# This is useful for debugging as well as to just treat the output as a non-parsable category
|
|
15
|
-
NOT_PARSABLE = "NOT_PARSABLE"
|
|
16
|
-
|
|
17
|
-
# values in the default openai function call,
|
|
18
|
-
# defined here only to prevent typos
|
|
19
|
-
_RESPONSE = "response"
|
|
20
|
-
_EXPLANATION = "explanation"
|
|
21
|
-
_FUNCTION_NAME = "record_response"
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def download_benchmark_dataset(task: str, dataset_name: str) -> pd.DataFrame:
|
|
25
|
-
"""Downloads an Arize evals benchmark dataset as a pandas dataframe.
|
|
26
|
-
|
|
27
|
-
Args:
|
|
28
|
-
task (str): Task to be performed.
|
|
29
|
-
dataset_name (str): Name of the dataset.
|
|
30
|
-
|
|
31
|
-
Returns:
|
|
32
|
-
pandas.DataFrame: A pandas dataframe containing the data.
|
|
33
|
-
"""
|
|
34
|
-
jsonl_file_name = f"{dataset_name}.jsonl"
|
|
35
|
-
url = f"http://storage.googleapis.com/arize-phoenix-assets/evals/{task}/{jsonl_file_name}.zip"
|
|
36
|
-
try:
|
|
37
|
-
with urlopen(url) as response:
|
|
38
|
-
zip_byte_stream = BytesIO(response.read())
|
|
39
|
-
with ZipFile(zip_byte_stream) as zip_file:
|
|
40
|
-
with zip_file.open(jsonl_file_name) as jsonl_file:
|
|
41
|
-
return pd.DataFrame(map(json.loads, jsonl_file.readlines()))
|
|
42
|
-
except HTTPError:
|
|
43
|
-
raise ValueError(f'Dataset "{dataset_name}" for "{task}" task does not exist.')
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def get_tqdm_progress_bar_formatter(title: str) -> str:
|
|
47
|
-
"""Returns a progress bar formatter for use with tqdm.
|
|
48
|
-
|
|
49
|
-
Args:
|
|
50
|
-
title (str): The title of the progress bar, displayed as a prefix.
|
|
51
|
-
|
|
52
|
-
Returns:
|
|
53
|
-
str: A formatter to be passed to the bar_format argument of tqdm.
|
|
54
|
-
"""
|
|
55
|
-
return (
|
|
56
|
-
title + " |{bar}| {n_fmt}/{total_fmt} ({percentage:3.1f}%) "
|
|
57
|
-
"| ⏳ {elapsed}<{remaining} | {rate_fmt}{postfix}"
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def snap_to_rail(raw_string: Optional[str], rails: List[str], verbose: bool = False) -> str:
|
|
62
|
-
"""
|
|
63
|
-
Snaps a string to the nearest rail, or returns None if the string cannot be
|
|
64
|
-
snapped to a rail.
|
|
65
|
-
|
|
66
|
-
Args:
|
|
67
|
-
raw_string (str): An input to be snapped to a rail.
|
|
68
|
-
|
|
69
|
-
rails (List[str]): The target set of strings to snap to.
|
|
70
|
-
|
|
71
|
-
Returns:
|
|
72
|
-
str: A string from the rails argument or "UNPARSABLE" if the input
|
|
73
|
-
string could not be snapped.
|
|
74
|
-
"""
|
|
75
|
-
if not raw_string:
|
|
76
|
-
return NOT_PARSABLE
|
|
77
|
-
snap_string = raw_string.lower()
|
|
78
|
-
rails = list(set(rail.lower() for rail in rails))
|
|
79
|
-
rails.sort(key=len, reverse=True)
|
|
80
|
-
found_rails = set()
|
|
81
|
-
for rail in rails:
|
|
82
|
-
if rail in snap_string:
|
|
83
|
-
found_rails.add(rail)
|
|
84
|
-
snap_string = snap_string.replace(rail, "")
|
|
85
|
-
if len(found_rails) != 1:
|
|
86
|
-
printif(verbose, f"- Cannot snap {repr(raw_string)} to rails")
|
|
87
|
-
return NOT_PARSABLE
|
|
88
|
-
rail = list(found_rails)[0]
|
|
89
|
-
printif(verbose, f"- Snapped {repr(raw_string)} to rail: {rail}")
|
|
90
|
-
return rail
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def parse_openai_function_call(raw_output: str) -> Tuple[str, Optional[str]]:
|
|
94
|
-
"""
|
|
95
|
-
Parses the output of an OpenAI function call.
|
|
96
|
-
|
|
97
|
-
Args:
|
|
98
|
-
raw_output (str): The raw output of an OpenAI function call.
|
|
99
|
-
|
|
100
|
-
Returns:
|
|
101
|
-
Tuple[str, Optional[str]]: A tuple of the unrailed label and an optional
|
|
102
|
-
explanation.
|
|
103
|
-
"""
|
|
104
|
-
try:
|
|
105
|
-
function_arguments = json.loads(raw_output, strict=False)
|
|
106
|
-
unrailed_label = function_arguments.get(_RESPONSE, "")
|
|
107
|
-
explanation = function_arguments.get(_EXPLANATION)
|
|
108
|
-
except json.JSONDecodeError:
|
|
109
|
-
unrailed_label = raw_output
|
|
110
|
-
explanation = None
|
|
111
|
-
return unrailed_label, explanation
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
def openai_function_call_kwargs(rails: List[str], provide_explanation: bool) -> Dict[str, Any]:
|
|
115
|
-
"""
|
|
116
|
-
Returns keyword arguments needed to invoke an OpenAI model with function
|
|
117
|
-
calling for classification.
|
|
118
|
-
|
|
119
|
-
Args:
|
|
120
|
-
rails (List[str]): The rails to snap the output to.
|
|
121
|
-
|
|
122
|
-
provide_explanation (bool): Whether to provide an explanation.
|
|
123
|
-
|
|
124
|
-
Returns:
|
|
125
|
-
Dict[str, Any]: A dictionary containing function call arguments.
|
|
126
|
-
"""
|
|
127
|
-
openai_function = _default_openai_function(rails, provide_explanation)
|
|
128
|
-
return {
|
|
129
|
-
"functions": [openai_function],
|
|
130
|
-
"function_call": {"name": openai_function["name"]},
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
def _default_openai_function(
|
|
135
|
-
rails: List[str],
|
|
136
|
-
with_explanation: bool = False,
|
|
137
|
-
) -> Dict[str, Any]:
|
|
138
|
-
"""
|
|
139
|
-
Returns a default OpenAI function call for classification.
|
|
140
|
-
|
|
141
|
-
Args:
|
|
142
|
-
rails (List[str]): A list of rails to snap the output to.
|
|
143
|
-
|
|
144
|
-
with_explanation (bool, optional): Whether to include an explanation.
|
|
145
|
-
|
|
146
|
-
Returns:
|
|
147
|
-
Dict[str, Any]: A JSON schema object advertising a function to record
|
|
148
|
-
the result of the LLM's classification.
|
|
149
|
-
"""
|
|
150
|
-
properties = {
|
|
151
|
-
**(
|
|
152
|
-
{
|
|
153
|
-
_EXPLANATION: {
|
|
154
|
-
"type": "string",
|
|
155
|
-
"description": "Explanation of the reasoning for your response.",
|
|
156
|
-
},
|
|
157
|
-
}
|
|
158
|
-
if with_explanation
|
|
159
|
-
else {}
|
|
160
|
-
),
|
|
161
|
-
_RESPONSE: {"type": "string", "description": "Your response.", "enum": rails},
|
|
162
|
-
}
|
|
163
|
-
required = [*([_EXPLANATION] if with_explanation else []), _RESPONSE]
|
|
164
|
-
return {
|
|
165
|
-
"name": _FUNCTION_NAME,
|
|
166
|
-
"description": "A function to record your response.",
|
|
167
|
-
"parameters": {
|
|
168
|
-
"type": "object",
|
|
169
|
-
"properties": properties,
|
|
170
|
-
"required": required,
|
|
171
|
-
},
|
|
172
|
-
}
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
"""High-level support for working with threads in asyncio
|
|
2
|
-
Directly copied from: https://github.com/python/cpython/blob/main/Lib/asyncio/threads.py#L12
|
|
3
|
-
since this helper function 'to_thread' is not available in python<3.9
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
import contextvars
|
|
7
|
-
import functools
|
|
8
|
-
from asyncio import events
|
|
9
|
-
from typing import Any
|
|
10
|
-
|
|
11
|
-
__all__ = ("to_thread",)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
async def to_thread(func, /, *args, **kwargs) -> Any: # type:ignore
|
|
15
|
-
"""Asynchronously run function *func* in a separate thread.
|
|
16
|
-
|
|
17
|
-
Any *args and **kwargs supplied for this function are directly passed
|
|
18
|
-
to *func*. Also, the current :class:`contextvars.Context` is propagated,
|
|
19
|
-
allowing context variables from the main thread to be accessed in the
|
|
20
|
-
separate thread.
|
|
21
|
-
|
|
22
|
-
Return a coroutine that can be awaited to get the eventual result of *func*.
|
|
23
|
-
"""
|
|
24
|
-
loop = events.get_running_loop()
|
|
25
|
-
ctx = contextvars.copy_context()
|
|
26
|
-
func_call = functools.partial(ctx.run, func, *args, **kwargs)
|
|
27
|
-
return await loop.run_in_executor(None, func_call)
|
phoenix/server/api/helpers.py
DELETED
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import gzip
|
|
3
|
-
from typing import AsyncIterator
|
|
4
|
-
|
|
5
|
-
import pyarrow as pa
|
|
6
|
-
from google.protobuf.message import DecodeError
|
|
7
|
-
from starlette.background import BackgroundTask
|
|
8
|
-
from starlette.endpoints import HTTPEndpoint
|
|
9
|
-
from starlette.requests import Request
|
|
10
|
-
from starlette.responses import Response, StreamingResponse
|
|
11
|
-
from starlette.status import (
|
|
12
|
-
HTTP_404_NOT_FOUND,
|
|
13
|
-
HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
|
14
|
-
HTTP_422_UNPROCESSABLE_ENTITY,
|
|
15
|
-
)
|
|
16
|
-
|
|
17
|
-
import phoenix.trace.v1 as pb
|
|
18
|
-
from phoenix.config import DEFAULT_PROJECT_NAME
|
|
19
|
-
from phoenix.core.traces import Traces
|
|
20
|
-
from phoenix.server.api.routers.utils import table_to_bytes
|
|
21
|
-
from phoenix.session.evaluation import encode_evaluations
|
|
22
|
-
from phoenix.trace.span_evaluations import Evaluations
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class EvaluationHandler(HTTPEndpoint):
|
|
26
|
-
traces: Traces
|
|
27
|
-
|
|
28
|
-
async def post(self, request: Request) -> Response:
|
|
29
|
-
content_type = request.headers.get("content-type")
|
|
30
|
-
project_name = request.headers.get("project-name", DEFAULT_PROJECT_NAME)
|
|
31
|
-
if content_type == "application/x-pandas-arrow":
|
|
32
|
-
return await self._process_pyarrow(request)
|
|
33
|
-
if content_type != "application/x-protobuf":
|
|
34
|
-
return Response(
|
|
35
|
-
content="Unsupported content type",
|
|
36
|
-
status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
|
37
|
-
)
|
|
38
|
-
body = await request.body()
|
|
39
|
-
content_encoding = request.headers.get("content-encoding")
|
|
40
|
-
if content_encoding == "gzip":
|
|
41
|
-
body = gzip.decompress(body)
|
|
42
|
-
elif content_encoding:
|
|
43
|
-
return Response(
|
|
44
|
-
content="Unsupported content encoding",
|
|
45
|
-
status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
|
46
|
-
)
|
|
47
|
-
evaluation = pb.Evaluation()
|
|
48
|
-
try:
|
|
49
|
-
evaluation.ParseFromString(body)
|
|
50
|
-
except DecodeError:
|
|
51
|
-
return Response(
|
|
52
|
-
content="Request body is invalid",
|
|
53
|
-
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
|
54
|
-
)
|
|
55
|
-
self.traces.put(evaluation, project_name=project_name)
|
|
56
|
-
return Response()
|
|
57
|
-
|
|
58
|
-
async def get(self, request: Request) -> Response:
|
|
59
|
-
payload = await request.json()
|
|
60
|
-
project_name = payload.pop("project_name", None) or DEFAULT_PROJECT_NAME
|
|
61
|
-
project = self.traces.get_project(project_name)
|
|
62
|
-
if not project:
|
|
63
|
-
return Response(status_code=HTTP_404_NOT_FOUND)
|
|
64
|
-
loop = asyncio.get_running_loop()
|
|
65
|
-
results = await loop.run_in_executor(
|
|
66
|
-
None,
|
|
67
|
-
project.export_evaluations,
|
|
68
|
-
)
|
|
69
|
-
if not results:
|
|
70
|
-
return Response(status_code=HTTP_404_NOT_FOUND)
|
|
71
|
-
|
|
72
|
-
async def content() -> AsyncIterator[bytes]:
|
|
73
|
-
for result in results:
|
|
74
|
-
yield await loop.run_in_executor(
|
|
75
|
-
None,
|
|
76
|
-
lambda: table_to_bytes(result.to_pyarrow_table()),
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
return StreamingResponse(
|
|
80
|
-
content=content(),
|
|
81
|
-
media_type="application/x-pandas-arrow",
|
|
82
|
-
)
|
|
83
|
-
|
|
84
|
-
async def _process_pyarrow(self, request: Request) -> Response:
|
|
85
|
-
body = await request.body()
|
|
86
|
-
try:
|
|
87
|
-
reader = pa.ipc.open_stream(body)
|
|
88
|
-
except pa.ArrowInvalid:
|
|
89
|
-
return Response(
|
|
90
|
-
content="Request body is not valid pyarrow",
|
|
91
|
-
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
|
92
|
-
)
|
|
93
|
-
try:
|
|
94
|
-
evaluations = Evaluations.from_pyarrow_reader(reader)
|
|
95
|
-
except Exception:
|
|
96
|
-
return Response(
|
|
97
|
-
content="Invalid data in request body",
|
|
98
|
-
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
|
99
|
-
)
|
|
100
|
-
return Response(
|
|
101
|
-
background=BackgroundTask(
|
|
102
|
-
self._add_evaluations,
|
|
103
|
-
evaluations,
|
|
104
|
-
)
|
|
105
|
-
)
|
|
106
|
-
|
|
107
|
-
async def _add_evaluations(self, evaluations: Evaluations) -> None:
|
|
108
|
-
for evaluation in encode_evaluations(evaluations):
|
|
109
|
-
self.traces.put(evaluation)
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
from functools import partial
|
|
3
|
-
from typing import AsyncIterator
|
|
4
|
-
|
|
5
|
-
from starlette.endpoints import HTTPEndpoint
|
|
6
|
-
from starlette.requests import Request
|
|
7
|
-
from starlette.responses import Response, StreamingResponse
|
|
8
|
-
from starlette.status import HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY
|
|
9
|
-
|
|
10
|
-
from phoenix.config import DEFAULT_PROJECT_NAME
|
|
11
|
-
from phoenix.core.traces import Traces
|
|
12
|
-
from phoenix.server.api.routers.utils import df_to_bytes, from_iso_format
|
|
13
|
-
from phoenix.trace.dsl import SpanQuery
|
|
14
|
-
from phoenix.utilities import query_spans
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class SpanHandler(HTTPEndpoint):
|
|
18
|
-
traces: Traces
|
|
19
|
-
|
|
20
|
-
async def get(self, request: Request) -> Response:
|
|
21
|
-
payload = await request.json()
|
|
22
|
-
queries = payload.pop("queries", [])
|
|
23
|
-
project_name = payload.pop("project_name", None) or DEFAULT_PROJECT_NAME
|
|
24
|
-
if not (project := self.traces.get_project(project_name)):
|
|
25
|
-
return Response(status_code=HTTP_404_NOT_FOUND)
|
|
26
|
-
loop = asyncio.get_running_loop()
|
|
27
|
-
valid_eval_names = (
|
|
28
|
-
await loop.run_in_executor(
|
|
29
|
-
None,
|
|
30
|
-
project.get_span_evaluation_names,
|
|
31
|
-
)
|
|
32
|
-
if project
|
|
33
|
-
else ()
|
|
34
|
-
)
|
|
35
|
-
try:
|
|
36
|
-
span_queries = [
|
|
37
|
-
SpanQuery.from_dict(
|
|
38
|
-
query,
|
|
39
|
-
evals=project,
|
|
40
|
-
valid_eval_names=valid_eval_names,
|
|
41
|
-
)
|
|
42
|
-
for query in queries
|
|
43
|
-
]
|
|
44
|
-
except Exception as e:
|
|
45
|
-
return Response(
|
|
46
|
-
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
|
47
|
-
content=f"Invalid query: {e}",
|
|
48
|
-
)
|
|
49
|
-
results = await loop.run_in_executor(
|
|
50
|
-
None,
|
|
51
|
-
partial(
|
|
52
|
-
query_spans,
|
|
53
|
-
project,
|
|
54
|
-
*span_queries,
|
|
55
|
-
start_time=from_iso_format(payload.get("start_time")),
|
|
56
|
-
stop_time=from_iso_format(payload.get("stop_time")),
|
|
57
|
-
root_spans_only=payload.get("root_spans_only"),
|
|
58
|
-
),
|
|
59
|
-
)
|
|
60
|
-
if not results:
|
|
61
|
-
return Response(status_code=HTTP_404_NOT_FOUND)
|
|
62
|
-
|
|
63
|
-
async def content() -> AsyncIterator[bytes]:
|
|
64
|
-
for result in results:
|
|
65
|
-
yield df_to_bytes(result)
|
|
66
|
-
|
|
67
|
-
return StreamingResponse(
|
|
68
|
-
content=content(),
|
|
69
|
-
media_type="application/x-pandas-arrow",
|
|
70
|
-
)
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import gzip
|
|
3
|
-
import zlib
|
|
4
|
-
from typing import Optional
|
|
5
|
-
|
|
6
|
-
from google.protobuf.message import DecodeError
|
|
7
|
-
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
|
|
8
|
-
ExportTraceServiceRequest,
|
|
9
|
-
)
|
|
10
|
-
from opentelemetry.proto.trace.v1.trace_pb2 import TracesData
|
|
11
|
-
from starlette.endpoints import HTTPEndpoint
|
|
12
|
-
from starlette.requests import Request
|
|
13
|
-
from starlette.responses import Response
|
|
14
|
-
from starlette.status import HTTP_415_UNSUPPORTED_MEDIA_TYPE, HTTP_422_UNPROCESSABLE_ENTITY
|
|
15
|
-
|
|
16
|
-
from phoenix.core.traces import Traces
|
|
17
|
-
from phoenix.storage.span_store import SpanStore
|
|
18
|
-
from phoenix.trace.otel import decode
|
|
19
|
-
from phoenix.utilities.project import get_project_name
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class TraceHandler(HTTPEndpoint):
|
|
23
|
-
traces: Traces
|
|
24
|
-
store: Optional[SpanStore]
|
|
25
|
-
|
|
26
|
-
async def post(self, request: Request) -> Response:
|
|
27
|
-
content_type = request.headers.get("content-type")
|
|
28
|
-
if content_type != "application/x-protobuf":
|
|
29
|
-
return Response(
|
|
30
|
-
content=f"Unsupported content type: {content_type}",
|
|
31
|
-
status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
|
32
|
-
)
|
|
33
|
-
content_encoding = request.headers.get("content-encoding")
|
|
34
|
-
if content_encoding and content_encoding not in ("gzip", "deflate"):
|
|
35
|
-
return Response(
|
|
36
|
-
content=f"Unsupported content encoding: {content_encoding}",
|
|
37
|
-
status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
|
38
|
-
)
|
|
39
|
-
body = await request.body()
|
|
40
|
-
if content_encoding == "gzip":
|
|
41
|
-
body = gzip.decompress(body)
|
|
42
|
-
elif content_encoding == "deflate":
|
|
43
|
-
body = zlib.decompress(body)
|
|
44
|
-
req = ExportTraceServiceRequest()
|
|
45
|
-
try:
|
|
46
|
-
req.ParseFromString(body)
|
|
47
|
-
except DecodeError:
|
|
48
|
-
return Response(
|
|
49
|
-
content="Request body is invalid ExportTraceServiceRequest",
|
|
50
|
-
status_code=HTTP_422_UNPROCESSABLE_ENTITY,
|
|
51
|
-
)
|
|
52
|
-
if self.store:
|
|
53
|
-
self.store.save(TracesData(resource_spans=req.resource_spans))
|
|
54
|
-
for resource_spans in req.resource_spans:
|
|
55
|
-
project_name = get_project_name(resource_spans.resource.attributes)
|
|
56
|
-
for scope_span in resource_spans.scope_spans:
|
|
57
|
-
for span in scope_span.spans:
|
|
58
|
-
self.traces.put(decode(span), project_name=project_name)
|
|
59
|
-
await asyncio.sleep(0)
|
|
60
|
-
return Response()
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
|
-
from typing import Dict, Union
|
|
3
|
-
|
|
4
|
-
import strawberry
|
|
5
|
-
|
|
6
|
-
from phoenix.core.model_schema import PRIMARY, REFERENCE
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
@strawberry.enum
|
|
10
|
-
class DatasetRole(Enum):
|
|
11
|
-
primary = PRIMARY
|
|
12
|
-
reference = REFERENCE
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class AncillaryDatasetRole(Enum):
|
|
16
|
-
corpus = "DatasetRole.CORPUS"
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
STR_TO_DATASET_ROLE: Dict[str, Union[DatasetRole, AncillaryDatasetRole]] = {
|
|
20
|
-
str(DatasetRole.primary.value): DatasetRole.primary,
|
|
21
|
-
str(DatasetRole.reference.value): DatasetRole.reference,
|
|
22
|
-
str(AncillaryDatasetRole.corpus.value): AncillaryDatasetRole.corpus,
|
|
23
|
-
}
|
phoenix/server/static/index.css
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}main{display:block}h1{font-size:2em;margin:.67em 0}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}img{border-style:none}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button::-moz-focus-inner,[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner{border-style:none;padding:0}button:-moz-focusring,[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details{display:block}summary{display:list-item}template{display:none}[hidden]{display:none}
|
|
2
|
-
/*! Bundled license information:
|
|
3
|
-
|
|
4
|
-
normalize.css/normalize.css:
|
|
5
|
-
(*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css *)
|
|
6
|
-
*/
|