arize-phoenix 3.16.1__py3-none-any.whl → 7.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of arize-phoenix might be problematic. Click here for more details.
- arize_phoenix-7.7.0.dist-info/METADATA +261 -0
- arize_phoenix-7.7.0.dist-info/RECORD +345 -0
- {arize_phoenix-3.16.1.dist-info → arize_phoenix-7.7.0.dist-info}/WHEEL +1 -1
- arize_phoenix-7.7.0.dist-info/entry_points.txt +3 -0
- phoenix/__init__.py +86 -14
- phoenix/auth.py +309 -0
- phoenix/config.py +675 -45
- phoenix/core/model.py +32 -30
- phoenix/core/model_schema.py +102 -109
- phoenix/core/model_schema_adapter.py +48 -45
- phoenix/datetime_utils.py +24 -3
- phoenix/db/README.md +54 -0
- phoenix/db/__init__.py +4 -0
- phoenix/db/alembic.ini +85 -0
- phoenix/db/bulk_inserter.py +294 -0
- phoenix/db/engines.py +208 -0
- phoenix/db/enums.py +20 -0
- phoenix/db/facilitator.py +113 -0
- phoenix/db/helpers.py +159 -0
- phoenix/db/insertion/constants.py +2 -0
- phoenix/db/insertion/dataset.py +227 -0
- phoenix/db/insertion/document_annotation.py +171 -0
- phoenix/db/insertion/evaluation.py +191 -0
- phoenix/db/insertion/helpers.py +98 -0
- phoenix/db/insertion/span.py +193 -0
- phoenix/db/insertion/span_annotation.py +158 -0
- phoenix/db/insertion/trace_annotation.py +158 -0
- phoenix/db/insertion/types.py +256 -0
- phoenix/db/migrate.py +86 -0
- phoenix/db/migrations/data_migration_scripts/populate_project_sessions.py +199 -0
- phoenix/db/migrations/env.py +114 -0
- phoenix/db/migrations/script.py.mako +26 -0
- phoenix/db/migrations/versions/10460e46d750_datasets.py +317 -0
- phoenix/db/migrations/versions/3be8647b87d8_add_token_columns_to_spans_table.py +126 -0
- phoenix/db/migrations/versions/4ded9e43755f_create_project_sessions_table.py +66 -0
- phoenix/db/migrations/versions/cd164e83824f_users_and_tokens.py +157 -0
- phoenix/db/migrations/versions/cf03bd6bae1d_init.py +280 -0
- phoenix/db/models.py +807 -0
- phoenix/exceptions.py +5 -1
- phoenix/experiments/__init__.py +6 -0
- phoenix/experiments/evaluators/__init__.py +29 -0
- phoenix/experiments/evaluators/base.py +158 -0
- phoenix/experiments/evaluators/code_evaluators.py +184 -0
- phoenix/experiments/evaluators/llm_evaluators.py +473 -0
- phoenix/experiments/evaluators/utils.py +236 -0
- phoenix/experiments/functions.py +772 -0
- phoenix/experiments/tracing.py +86 -0
- phoenix/experiments/types.py +726 -0
- phoenix/experiments/utils.py +25 -0
- phoenix/inferences/__init__.py +0 -0
- phoenix/{datasets → inferences}/errors.py +6 -5
- phoenix/{datasets → inferences}/fixtures.py +49 -42
- phoenix/{datasets/dataset.py → inferences/inferences.py} +121 -105
- phoenix/{datasets → inferences}/schema.py +11 -11
- phoenix/{datasets → inferences}/validation.py +13 -14
- phoenix/logging/__init__.py +3 -0
- phoenix/logging/_config.py +90 -0
- phoenix/logging/_filter.py +6 -0
- phoenix/logging/_formatter.py +69 -0
- phoenix/metrics/__init__.py +5 -4
- phoenix/metrics/binning.py +4 -3
- phoenix/metrics/metrics.py +2 -1
- phoenix/metrics/mixins.py +7 -6
- phoenix/metrics/retrieval_metrics.py +2 -1
- phoenix/metrics/timeseries.py +5 -4
- phoenix/metrics/wrappers.py +9 -3
- phoenix/pointcloud/clustering.py +5 -5
- phoenix/pointcloud/pointcloud.py +7 -5
- phoenix/pointcloud/projectors.py +5 -6
- phoenix/pointcloud/umap_parameters.py +53 -52
- phoenix/server/api/README.md +28 -0
- phoenix/server/api/auth.py +44 -0
- phoenix/server/api/context.py +152 -9
- phoenix/server/api/dataloaders/__init__.py +91 -0
- phoenix/server/api/dataloaders/annotation_summaries.py +139 -0
- phoenix/server/api/dataloaders/average_experiment_run_latency.py +54 -0
- phoenix/server/api/dataloaders/cache/__init__.py +3 -0
- phoenix/server/api/dataloaders/cache/two_tier_cache.py +68 -0
- phoenix/server/api/dataloaders/dataset_example_revisions.py +131 -0
- phoenix/server/api/dataloaders/dataset_example_spans.py +38 -0
- phoenix/server/api/dataloaders/document_evaluation_summaries.py +144 -0
- phoenix/server/api/dataloaders/document_evaluations.py +31 -0
- phoenix/server/api/dataloaders/document_retrieval_metrics.py +89 -0
- phoenix/server/api/dataloaders/experiment_annotation_summaries.py +79 -0
- phoenix/server/api/dataloaders/experiment_error_rates.py +58 -0
- phoenix/server/api/dataloaders/experiment_run_annotations.py +36 -0
- phoenix/server/api/dataloaders/experiment_run_counts.py +49 -0
- phoenix/server/api/dataloaders/experiment_sequence_number.py +44 -0
- phoenix/server/api/dataloaders/latency_ms_quantile.py +188 -0
- phoenix/server/api/dataloaders/min_start_or_max_end_times.py +85 -0
- phoenix/server/api/dataloaders/project_by_name.py +31 -0
- phoenix/server/api/dataloaders/record_counts.py +116 -0
- phoenix/server/api/dataloaders/session_io.py +79 -0
- phoenix/server/api/dataloaders/session_num_traces.py +30 -0
- phoenix/server/api/dataloaders/session_num_traces_with_error.py +32 -0
- phoenix/server/api/dataloaders/session_token_usages.py +41 -0
- phoenix/server/api/dataloaders/session_trace_latency_ms_quantile.py +55 -0
- phoenix/server/api/dataloaders/span_annotations.py +26 -0
- phoenix/server/api/dataloaders/span_dataset_examples.py +31 -0
- phoenix/server/api/dataloaders/span_descendants.py +57 -0
- phoenix/server/api/dataloaders/span_projects.py +33 -0
- phoenix/server/api/dataloaders/token_counts.py +124 -0
- phoenix/server/api/dataloaders/trace_by_trace_ids.py +25 -0
- phoenix/server/api/dataloaders/trace_root_spans.py +32 -0
- phoenix/server/api/dataloaders/user_roles.py +30 -0
- phoenix/server/api/dataloaders/users.py +33 -0
- phoenix/server/api/exceptions.py +48 -0
- phoenix/server/api/helpers/__init__.py +12 -0
- phoenix/server/api/helpers/dataset_helpers.py +217 -0
- phoenix/server/api/helpers/experiment_run_filters.py +763 -0
- phoenix/server/api/helpers/playground_clients.py +948 -0
- phoenix/server/api/helpers/playground_registry.py +70 -0
- phoenix/server/api/helpers/playground_spans.py +455 -0
- phoenix/server/api/input_types/AddExamplesToDatasetInput.py +16 -0
- phoenix/server/api/input_types/AddSpansToDatasetInput.py +14 -0
- phoenix/server/api/input_types/ChatCompletionInput.py +38 -0
- phoenix/server/api/input_types/ChatCompletionMessageInput.py +24 -0
- phoenix/server/api/input_types/ClearProjectInput.py +15 -0
- phoenix/server/api/input_types/ClusterInput.py +2 -2
- phoenix/server/api/input_types/CreateDatasetInput.py +12 -0
- phoenix/server/api/input_types/CreateSpanAnnotationInput.py +18 -0
- phoenix/server/api/input_types/CreateTraceAnnotationInput.py +18 -0
- phoenix/server/api/input_types/DataQualityMetricInput.py +5 -2
- phoenix/server/api/input_types/DatasetExampleInput.py +14 -0
- phoenix/server/api/input_types/DatasetSort.py +17 -0
- phoenix/server/api/input_types/DatasetVersionSort.py +16 -0
- phoenix/server/api/input_types/DeleteAnnotationsInput.py +7 -0
- phoenix/server/api/input_types/DeleteDatasetExamplesInput.py +13 -0
- phoenix/server/api/input_types/DeleteDatasetInput.py +7 -0
- phoenix/server/api/input_types/DeleteExperimentsInput.py +7 -0
- phoenix/server/api/input_types/DimensionFilter.py +4 -4
- phoenix/server/api/input_types/GenerativeModelInput.py +17 -0
- phoenix/server/api/input_types/Granularity.py +1 -1
- phoenix/server/api/input_types/InvocationParameters.py +162 -0
- phoenix/server/api/input_types/PatchAnnotationInput.py +19 -0
- phoenix/server/api/input_types/PatchDatasetExamplesInput.py +35 -0
- phoenix/server/api/input_types/PatchDatasetInput.py +14 -0
- phoenix/server/api/input_types/PerformanceMetricInput.py +5 -2
- phoenix/server/api/input_types/ProjectSessionSort.py +29 -0
- phoenix/server/api/input_types/SpanAnnotationSort.py +17 -0
- phoenix/server/api/input_types/SpanSort.py +134 -69
- phoenix/server/api/input_types/TemplateOptions.py +10 -0
- phoenix/server/api/input_types/TraceAnnotationSort.py +17 -0
- phoenix/server/api/input_types/UserRoleInput.py +9 -0
- phoenix/server/api/mutations/__init__.py +28 -0
- phoenix/server/api/mutations/api_key_mutations.py +167 -0
- phoenix/server/api/mutations/chat_mutations.py +593 -0
- phoenix/server/api/mutations/dataset_mutations.py +591 -0
- phoenix/server/api/mutations/experiment_mutations.py +75 -0
- phoenix/server/api/{types/ExportEventsMutation.py → mutations/export_events_mutations.py} +21 -18
- phoenix/server/api/mutations/project_mutations.py +57 -0
- phoenix/server/api/mutations/span_annotations_mutations.py +128 -0
- phoenix/server/api/mutations/trace_annotations_mutations.py +127 -0
- phoenix/server/api/mutations/user_mutations.py +329 -0
- phoenix/server/api/openapi/__init__.py +0 -0
- phoenix/server/api/openapi/main.py +17 -0
- phoenix/server/api/openapi/schema.py +16 -0
- phoenix/server/api/queries.py +738 -0
- phoenix/server/api/routers/__init__.py +11 -0
- phoenix/server/api/routers/auth.py +284 -0
- phoenix/server/api/routers/embeddings.py +26 -0
- phoenix/server/api/routers/oauth2.py +488 -0
- phoenix/server/api/routers/v1/__init__.py +64 -0
- phoenix/server/api/routers/v1/datasets.py +1017 -0
- phoenix/server/api/routers/v1/evaluations.py +362 -0
- phoenix/server/api/routers/v1/experiment_evaluations.py +115 -0
- phoenix/server/api/routers/v1/experiment_runs.py +167 -0
- phoenix/server/api/routers/v1/experiments.py +308 -0
- phoenix/server/api/routers/v1/pydantic_compat.py +78 -0
- phoenix/server/api/routers/v1/spans.py +267 -0
- phoenix/server/api/routers/v1/traces.py +208 -0
- phoenix/server/api/routers/v1/utils.py +95 -0
- phoenix/server/api/schema.py +44 -241
- phoenix/server/api/subscriptions.py +597 -0
- phoenix/server/api/types/Annotation.py +21 -0
- phoenix/server/api/types/AnnotationSummary.py +55 -0
- phoenix/server/api/types/AnnotatorKind.py +16 -0
- phoenix/server/api/types/ApiKey.py +27 -0
- phoenix/server/api/types/AuthMethod.py +9 -0
- phoenix/server/api/types/ChatCompletionMessageRole.py +11 -0
- phoenix/server/api/types/ChatCompletionSubscriptionPayload.py +46 -0
- phoenix/server/api/types/Cluster.py +25 -24
- phoenix/server/api/types/CreateDatasetPayload.py +8 -0
- phoenix/server/api/types/DataQualityMetric.py +31 -13
- phoenix/server/api/types/Dataset.py +288 -63
- phoenix/server/api/types/DatasetExample.py +85 -0
- phoenix/server/api/types/DatasetExampleRevision.py +34 -0
- phoenix/server/api/types/DatasetVersion.py +14 -0
- phoenix/server/api/types/Dimension.py +32 -31
- phoenix/server/api/types/DocumentEvaluationSummary.py +9 -8
- phoenix/server/api/types/EmbeddingDimension.py +56 -49
- phoenix/server/api/types/Evaluation.py +25 -31
- phoenix/server/api/types/EvaluationSummary.py +30 -50
- phoenix/server/api/types/Event.py +20 -20
- phoenix/server/api/types/ExampleRevisionInterface.py +14 -0
- phoenix/server/api/types/Experiment.py +152 -0
- phoenix/server/api/types/ExperimentAnnotationSummary.py +13 -0
- phoenix/server/api/types/ExperimentComparison.py +17 -0
- phoenix/server/api/types/ExperimentRun.py +119 -0
- phoenix/server/api/types/ExperimentRunAnnotation.py +56 -0
- phoenix/server/api/types/GenerativeModel.py +9 -0
- phoenix/server/api/types/GenerativeProvider.py +85 -0
- phoenix/server/api/types/Inferences.py +80 -0
- phoenix/server/api/types/InferencesRole.py +23 -0
- phoenix/server/api/types/LabelFraction.py +7 -0
- phoenix/server/api/types/MimeType.py +2 -2
- phoenix/server/api/types/Model.py +54 -54
- phoenix/server/api/types/PerformanceMetric.py +8 -5
- phoenix/server/api/types/Project.py +407 -142
- phoenix/server/api/types/ProjectSession.py +139 -0
- phoenix/server/api/types/Segments.py +4 -4
- phoenix/server/api/types/Span.py +221 -176
- phoenix/server/api/types/SpanAnnotation.py +43 -0
- phoenix/server/api/types/SpanIOValue.py +15 -0
- phoenix/server/api/types/SystemApiKey.py +9 -0
- phoenix/server/api/types/TemplateLanguage.py +10 -0
- phoenix/server/api/types/TimeSeries.py +19 -15
- phoenix/server/api/types/TokenUsage.py +11 -0
- phoenix/server/api/types/Trace.py +154 -0
- phoenix/server/api/types/TraceAnnotation.py +45 -0
- phoenix/server/api/types/UMAPPoints.py +7 -7
- phoenix/server/api/types/User.py +60 -0
- phoenix/server/api/types/UserApiKey.py +45 -0
- phoenix/server/api/types/UserRole.py +15 -0
- phoenix/server/api/types/node.py +4 -112
- phoenix/server/api/types/pagination.py +156 -57
- phoenix/server/api/utils.py +34 -0
- phoenix/server/app.py +864 -115
- phoenix/server/bearer_auth.py +163 -0
- phoenix/server/dml_event.py +136 -0
- phoenix/server/dml_event_handler.py +256 -0
- phoenix/server/email/__init__.py +0 -0
- phoenix/server/email/sender.py +97 -0
- phoenix/server/email/templates/__init__.py +0 -0
- phoenix/server/email/templates/password_reset.html +19 -0
- phoenix/server/email/types.py +11 -0
- phoenix/server/grpc_server.py +102 -0
- phoenix/server/jwt_store.py +505 -0
- phoenix/server/main.py +305 -116
- phoenix/server/oauth2.py +52 -0
- phoenix/server/openapi/__init__.py +0 -0
- phoenix/server/prometheus.py +111 -0
- phoenix/server/rate_limiters.py +188 -0
- phoenix/server/static/.vite/manifest.json +87 -0
- phoenix/server/static/assets/components-Cy9nwIvF.js +2125 -0
- phoenix/server/static/assets/index-BKvHIxkk.js +113 -0
- phoenix/server/static/assets/pages-CUi2xCVQ.js +4449 -0
- phoenix/server/static/assets/vendor-DvC8cT4X.js +894 -0
- phoenix/server/static/assets/vendor-DxkFTwjz.css +1 -0
- phoenix/server/static/assets/vendor-arizeai-Do1793cv.js +662 -0
- phoenix/server/static/assets/vendor-codemirror-BzwZPyJM.js +24 -0
- phoenix/server/static/assets/vendor-recharts-_Jb7JjhG.js +59 -0
- phoenix/server/static/assets/vendor-shiki-Cl9QBraO.js +5 -0
- phoenix/server/static/assets/vendor-three-DwGkEfCM.js +2998 -0
- phoenix/server/telemetry.py +68 -0
- phoenix/server/templates/index.html +82 -23
- phoenix/server/thread_server.py +3 -3
- phoenix/server/types.py +275 -0
- phoenix/services.py +27 -18
- phoenix/session/client.py +743 -68
- phoenix/session/data_extractor.py +31 -7
- phoenix/session/evaluation.py +3 -9
- phoenix/session/session.py +263 -219
- phoenix/settings.py +22 -0
- phoenix/trace/__init__.py +2 -22
- phoenix/trace/attributes.py +338 -0
- phoenix/trace/dsl/README.md +116 -0
- phoenix/trace/dsl/filter.py +663 -213
- phoenix/trace/dsl/helpers.py +73 -21
- phoenix/trace/dsl/query.py +574 -201
- phoenix/trace/exporter.py +24 -19
- phoenix/trace/fixtures.py +368 -32
- phoenix/trace/otel.py +71 -219
- phoenix/trace/projects.py +3 -2
- phoenix/trace/schemas.py +33 -11
- phoenix/trace/span_evaluations.py +21 -16
- phoenix/trace/span_json_decoder.py +6 -4
- phoenix/trace/span_json_encoder.py +2 -2
- phoenix/trace/trace_dataset.py +47 -32
- phoenix/trace/utils.py +21 -4
- phoenix/utilities/__init__.py +0 -26
- phoenix/utilities/client.py +132 -0
- phoenix/utilities/deprecation.py +31 -0
- phoenix/utilities/error_handling.py +3 -2
- phoenix/utilities/json.py +109 -0
- phoenix/utilities/logging.py +8 -0
- phoenix/utilities/project.py +2 -2
- phoenix/utilities/re.py +49 -0
- phoenix/utilities/span_store.py +0 -23
- phoenix/utilities/template_formatters.py +99 -0
- phoenix/version.py +1 -1
- arize_phoenix-3.16.1.dist-info/METADATA +0 -495
- arize_phoenix-3.16.1.dist-info/RECORD +0 -178
- phoenix/core/project.py +0 -619
- phoenix/core/traces.py +0 -96
- phoenix/experimental/evals/__init__.py +0 -73
- phoenix/experimental/evals/evaluators.py +0 -413
- phoenix/experimental/evals/functions/__init__.py +0 -4
- phoenix/experimental/evals/functions/classify.py +0 -453
- phoenix/experimental/evals/functions/executor.py +0 -353
- phoenix/experimental/evals/functions/generate.py +0 -138
- phoenix/experimental/evals/functions/processing.py +0 -76
- phoenix/experimental/evals/models/__init__.py +0 -14
- phoenix/experimental/evals/models/anthropic.py +0 -175
- phoenix/experimental/evals/models/base.py +0 -170
- phoenix/experimental/evals/models/bedrock.py +0 -221
- phoenix/experimental/evals/models/litellm.py +0 -134
- phoenix/experimental/evals/models/openai.py +0 -448
- phoenix/experimental/evals/models/rate_limiters.py +0 -246
- phoenix/experimental/evals/models/vertex.py +0 -173
- phoenix/experimental/evals/models/vertexai.py +0 -186
- phoenix/experimental/evals/retrievals.py +0 -96
- phoenix/experimental/evals/templates/__init__.py +0 -50
- phoenix/experimental/evals/templates/default_templates.py +0 -472
- phoenix/experimental/evals/templates/template.py +0 -195
- phoenix/experimental/evals/utils/__init__.py +0 -172
- phoenix/experimental/evals/utils/threads.py +0 -27
- phoenix/server/api/helpers.py +0 -11
- phoenix/server/api/routers/evaluation_handler.py +0 -109
- phoenix/server/api/routers/span_handler.py +0 -70
- phoenix/server/api/routers/trace_handler.py +0 -60
- phoenix/server/api/types/DatasetRole.py +0 -23
- phoenix/server/static/index.css +0 -6
- phoenix/server/static/index.js +0 -7447
- phoenix/storage/span_store/__init__.py +0 -23
- phoenix/storage/span_store/text_file.py +0 -85
- phoenix/trace/dsl/missing.py +0 -60
- phoenix/trace/langchain/__init__.py +0 -3
- phoenix/trace/langchain/instrumentor.py +0 -35
- phoenix/trace/llama_index/__init__.py +0 -3
- phoenix/trace/llama_index/callback.py +0 -102
- phoenix/trace/openai/__init__.py +0 -3
- phoenix/trace/openai/instrumentor.py +0 -30
- {arize_phoenix-3.16.1.dist-info → arize_phoenix-7.7.0.dist-info}/licenses/IP_NOTICE +0 -0
- {arize_phoenix-3.16.1.dist-info → arize_phoenix-7.7.0.dist-info}/licenses/LICENSE +0 -0
- /phoenix/{datasets → db/insertion}/__init__.py +0 -0
- /phoenix/{experimental → db/migrations}/__init__.py +0 -0
- /phoenix/{storage → db/migrations/data_migration_scripts}/__init__.py +0 -0
phoenix/session/client.py
CHANGED
|
@@ -1,117 +1,214 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import gzip
|
|
1
3
|
import logging
|
|
4
|
+
import re
|
|
2
5
|
import weakref
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
3
8
|
from datetime import datetime
|
|
4
9
|
from io import BytesIO
|
|
5
|
-
from
|
|
6
|
-
from
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, BinaryIO, Literal, Optional, Union, cast
|
|
12
|
+
from urllib.parse import quote, urljoin
|
|
7
13
|
|
|
14
|
+
import httpx
|
|
8
15
|
import pandas as pd
|
|
9
16
|
import pyarrow as pa
|
|
10
|
-
from
|
|
11
|
-
from
|
|
17
|
+
from httpx import HTTPStatusError, Response
|
|
18
|
+
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
|
|
19
|
+
from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
|
|
20
|
+
from opentelemetry.proto.resource.v1.resource_pb2 import Resource
|
|
21
|
+
from opentelemetry.proto.trace.v1.trace_pb2 import ResourceSpans, ScopeSpans
|
|
22
|
+
from pyarrow import ArrowInvalid, Table
|
|
23
|
+
from typing_extensions import TypeAlias, assert_never
|
|
12
24
|
|
|
13
|
-
import phoenix as px
|
|
14
25
|
from phoenix.config import (
|
|
15
26
|
get_env_collector_endpoint,
|
|
16
27
|
get_env_host,
|
|
17
28
|
get_env_port,
|
|
18
29
|
get_env_project_name,
|
|
19
30
|
)
|
|
20
|
-
from phoenix.
|
|
21
|
-
from phoenix.
|
|
31
|
+
from phoenix.datetime_utils import normalize_datetime
|
|
32
|
+
from phoenix.db.insertion.dataset import DatasetKeys
|
|
33
|
+
from phoenix.experiments.types import Dataset, Example, Experiment
|
|
34
|
+
from phoenix.session.data_extractor import DEFAULT_SPAN_LIMIT, TraceDataExtractor
|
|
35
|
+
from phoenix.trace import Evaluations, TraceDataset
|
|
22
36
|
from phoenix.trace.dsl import SpanQuery
|
|
37
|
+
from phoenix.trace.otel import encode_span_to_otlp
|
|
38
|
+
from phoenix.utilities.client import VersionedClient
|
|
39
|
+
from phoenix.utilities.json import decode_df_from_json_string
|
|
23
40
|
|
|
24
41
|
logger = logging.getLogger(__name__)
|
|
25
42
|
|
|
43
|
+
DEFAULT_TIMEOUT_IN_SECONDS = 5
|
|
44
|
+
|
|
45
|
+
DatasetAction: TypeAlias = Literal["create", "append"]
|
|
46
|
+
|
|
26
47
|
|
|
27
48
|
class Client(TraceDataExtractor):
|
|
28
49
|
def __init__(
|
|
29
50
|
self,
|
|
30
51
|
*,
|
|
31
52
|
endpoint: Optional[str] = None,
|
|
32
|
-
|
|
53
|
+
warn_if_server_not_running: bool = True,
|
|
54
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
55
|
+
api_key: Optional[str] = None,
|
|
56
|
+
**kwargs: Any, # for backward-compatibility
|
|
33
57
|
):
|
|
34
58
|
"""
|
|
35
59
|
Client for connecting to a Phoenix server.
|
|
36
60
|
|
|
37
61
|
Args:
|
|
38
|
-
endpoint (str, optional): Phoenix server endpoint, e.g.
|
|
39
|
-
provided, the endpoint will be
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
62
|
+
endpoint (str, optional): Phoenix server endpoint, e.g.
|
|
63
|
+
http://localhost:6006. If not provided, the endpoint will be
|
|
64
|
+
inferred from the environment variables.
|
|
65
|
+
|
|
66
|
+
headers (Mapping[str, str], optional): Headers to include in each
|
|
67
|
+
network request. If not provided, the headers will be inferred from
|
|
68
|
+
the environment variables (if present).
|
|
69
|
+
"""
|
|
70
|
+
if kwargs.pop("use_active_session_if_available", None) is not None:
|
|
71
|
+
print(
|
|
72
|
+
"`use_active_session_if_available` is deprecated "
|
|
73
|
+
"and will be removed in the future."
|
|
74
|
+
)
|
|
75
|
+
if kwargs:
|
|
76
|
+
raise TypeError(f"Unexpected keyword arguments: {', '.join(kwargs)}")
|
|
77
|
+
headers = dict(headers or {})
|
|
78
|
+
if api_key:
|
|
79
|
+
headers = {
|
|
80
|
+
**{k: v for k, v in (headers or {}).items() if k.lower() != "authorization"},
|
|
81
|
+
"Authorization": f"Bearer {api_key}",
|
|
82
|
+
}
|
|
46
83
|
host = get_env_host()
|
|
47
84
|
if host == "0.0.0.0":
|
|
48
85
|
host = "127.0.0.1"
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
)
|
|
52
|
-
self.
|
|
53
|
-
|
|
54
|
-
if not (self._use_active_session_if_available and px.active_session()):
|
|
86
|
+
base_url = endpoint or get_env_collector_endpoint() or f"http://{host}:{get_env_port()}"
|
|
87
|
+
self._base_url = base_url if base_url.endswith("/") else base_url + "/"
|
|
88
|
+
self._client = VersionedClient(headers=headers)
|
|
89
|
+
weakref.finalize(self, self._client.close)
|
|
90
|
+
if warn_if_server_not_running:
|
|
55
91
|
self._warn_if_phoenix_is_not_running()
|
|
56
92
|
|
|
93
|
+
@property
|
|
94
|
+
def web_url(self) -> str:
|
|
95
|
+
"""
|
|
96
|
+
Return the web URL of the Phoenix UI. This is different from the base
|
|
97
|
+
URL in the cases where there is a proxy like colab
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
str: A fully qualified URL to the Phoenix UI.
|
|
102
|
+
"""
|
|
103
|
+
# Avoid circular import
|
|
104
|
+
from phoenix.session.session import active_session
|
|
105
|
+
|
|
106
|
+
if session := active_session():
|
|
107
|
+
return session.url
|
|
108
|
+
return self._base_url
|
|
109
|
+
|
|
57
110
|
def query_spans(
|
|
58
111
|
self,
|
|
59
112
|
*queries: SpanQuery,
|
|
60
113
|
start_time: Optional[datetime] = None,
|
|
61
|
-
|
|
114
|
+
end_time: Optional[datetime] = None,
|
|
115
|
+
limit: Optional[int] = DEFAULT_SPAN_LIMIT,
|
|
62
116
|
root_spans_only: Optional[bool] = None,
|
|
63
117
|
project_name: Optional[str] = None,
|
|
64
|
-
|
|
118
|
+
# Deprecated
|
|
119
|
+
stop_time: Optional[datetime] = None,
|
|
120
|
+
timeout: Optional[int] = DEFAULT_TIMEOUT_IN_SECONDS,
|
|
121
|
+
) -> Optional[Union[pd.DataFrame, list[pd.DataFrame]]]:
|
|
65
122
|
"""
|
|
66
123
|
Queries spans from the Phoenix server or active session based on specified criteria.
|
|
67
124
|
|
|
68
125
|
Args:
|
|
69
126
|
queries (SpanQuery): One or more SpanQuery objects defining the query criteria.
|
|
70
127
|
start_time (datetime, optional): The start time for the query range. Default None.
|
|
71
|
-
|
|
128
|
+
end_time (datetime, optional): The end time for the query range. Default None.
|
|
72
129
|
root_spans_only (bool, optional): If True, only root spans are returned. Default None.
|
|
73
130
|
project_name (str, optional): The project name to query spans for. This can be set
|
|
74
131
|
using environment variables. If not provided, falls back to the default project.
|
|
132
|
+
timeout (int, optional): The number of seconds to wait for the server to respond.
|
|
75
133
|
|
|
76
134
|
Returns:
|
|
77
|
-
Union[pd.DataFrame,
|
|
135
|
+
Union[pd.DataFrame, list[pd.DataFrame]]:
|
|
136
|
+
A pandas DataFrame or a list of pandas.
|
|
78
137
|
DataFrames containing the queried span data, or None if no spans are found.
|
|
79
138
|
"""
|
|
80
139
|
project_name = project_name or get_env_project_name()
|
|
81
140
|
if not queries:
|
|
82
141
|
queries = (SpanQuery(),)
|
|
83
|
-
if
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
stop_time=stop_time,
|
|
88
|
-
root_spans_only=root_spans_only,
|
|
89
|
-
project_name=project_name,
|
|
142
|
+
if stop_time is not None:
|
|
143
|
+
# Deprecated. Raise a warning
|
|
144
|
+
logger.warning(
|
|
145
|
+
"stop_time is deprecated. Use end_time instead.",
|
|
90
146
|
)
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
"
|
|
95
|
-
"
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
147
|
+
end_time = end_time or stop_time
|
|
148
|
+
try:
|
|
149
|
+
response = self._client.post(
|
|
150
|
+
headers={"accept": "application/json"},
|
|
151
|
+
url=urljoin(self._base_url, "v1/spans"),
|
|
152
|
+
params={
|
|
153
|
+
"project_name": project_name,
|
|
154
|
+
"project-name": project_name, # for backward-compatibility
|
|
155
|
+
},
|
|
156
|
+
json={
|
|
157
|
+
"queries": [q.to_dict() for q in queries],
|
|
158
|
+
"start_time": _to_iso_format(normalize_datetime(start_time)),
|
|
159
|
+
"end_time": _to_iso_format(normalize_datetime(end_time)),
|
|
160
|
+
"limit": limit,
|
|
161
|
+
"root_spans_only": root_spans_only,
|
|
162
|
+
},
|
|
163
|
+
timeout=timeout,
|
|
164
|
+
)
|
|
165
|
+
except httpx.TimeoutException as error:
|
|
166
|
+
error_message = (
|
|
167
|
+
(
|
|
168
|
+
f"The request timed out after {timeout} seconds. The timeout can be increased "
|
|
169
|
+
"by passing a larger value to the `timeout` parameter "
|
|
170
|
+
"and can be disabled altogether by passing `None`."
|
|
171
|
+
)
|
|
172
|
+
if timeout is not None
|
|
173
|
+
else (
|
|
174
|
+
"The request timed out. The timeout can be adjusted by "
|
|
175
|
+
"passing a number of seconds to the `timeout` parameter "
|
|
176
|
+
"and can be disabled altogether by passing `None`."
|
|
177
|
+
)
|
|
178
|
+
)
|
|
179
|
+
raise TimeoutError(error_message) from error
|
|
101
180
|
if response.status_code == 404:
|
|
102
181
|
logger.info("No spans found.")
|
|
103
182
|
return None
|
|
104
183
|
elif response.status_code == 422:
|
|
105
184
|
raise ValueError(response.content.decode())
|
|
106
185
|
response.raise_for_status()
|
|
107
|
-
source = BytesIO(response.content)
|
|
108
186
|
results = []
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
187
|
+
content_type = response.headers.get("Content-Type")
|
|
188
|
+
if isinstance(content_type, str) and "multipart/mixed" in content_type:
|
|
189
|
+
if "boundary=" in content_type:
|
|
190
|
+
boundary_token = content_type.split("boundary=")[1].split(";", 1)[0]
|
|
191
|
+
else:
|
|
192
|
+
raise ValueError(
|
|
193
|
+
"Boundary not found in Content-Type header for multipart/mixed response"
|
|
194
|
+
)
|
|
195
|
+
boundary = f"--{boundary_token}"
|
|
196
|
+
text = response.text
|
|
197
|
+
while boundary in text:
|
|
198
|
+
part, text = text.split(boundary, 1)
|
|
199
|
+
if "Content-Type: application/json" in part:
|
|
200
|
+
json_string = part.split("\r\n\r\n", 1)[1].strip()
|
|
201
|
+
df = decode_df_from_json_string(json_string)
|
|
202
|
+
results.append(df)
|
|
203
|
+
else:
|
|
204
|
+
# For backward compatibility
|
|
205
|
+
source = BytesIO(response.content)
|
|
206
|
+
while True:
|
|
207
|
+
try:
|
|
208
|
+
with pa.ipc.open_stream(source) as reader:
|
|
209
|
+
results.append(reader.read_pandas())
|
|
210
|
+
except ArrowInvalid:
|
|
211
|
+
break
|
|
115
212
|
if len(results) == 1:
|
|
116
213
|
df = results[0]
|
|
117
214
|
return None if df.shape == (0, 0) else df
|
|
@@ -120,7 +217,9 @@ class Client(TraceDataExtractor):
|
|
|
120
217
|
def get_evaluations(
|
|
121
218
|
self,
|
|
122
219
|
project_name: Optional[str] = None,
|
|
123
|
-
|
|
220
|
+
*, # Only support kwargs from now on
|
|
221
|
+
timeout: Optional[int] = DEFAULT_TIMEOUT_IN_SECONDS,
|
|
222
|
+
) -> list[Evaluations]:
|
|
124
223
|
"""
|
|
125
224
|
Retrieves evaluations for a given project from the Phoenix server or active session.
|
|
126
225
|
|
|
@@ -128,17 +227,21 @@ class Client(TraceDataExtractor):
|
|
|
128
227
|
project_name (str, optional): The name of the project to retrieve evaluations for.
|
|
129
228
|
This can be set using environment variables. If not provided, falls back to the
|
|
130
229
|
default project.
|
|
230
|
+
timeout (int, optional): The number of seconds to wait for the server to respond.
|
|
131
231
|
|
|
132
232
|
Returns:
|
|
133
|
-
|
|
233
|
+
list[Evaluations]:
|
|
234
|
+
A list of Evaluations objects containing evaluation data. Returns an
|
|
134
235
|
empty list if no evaluations are found.
|
|
135
236
|
"""
|
|
136
237
|
project_name = project_name or get_env_project_name()
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
238
|
+
response = self._client.get(
|
|
239
|
+
url=urljoin(self._base_url, "v1/evaluations"),
|
|
240
|
+
params={
|
|
241
|
+
"project_name": project_name,
|
|
242
|
+
"project-name": project_name, # for backward-compatibility
|
|
243
|
+
},
|
|
244
|
+
timeout=timeout,
|
|
142
245
|
)
|
|
143
246
|
if response.status_code == 404:
|
|
144
247
|
logger.info("No evaluations found.")
|
|
@@ -158,41 +261,613 @@ class Client(TraceDataExtractor):
|
|
|
158
261
|
|
|
159
262
|
def _warn_if_phoenix_is_not_running(self) -> None:
|
|
160
263
|
try:
|
|
161
|
-
self.
|
|
264
|
+
self._client.get(urljoin(self._base_url, "arize_phoenix_version")).raise_for_status()
|
|
162
265
|
except Exception:
|
|
163
266
|
logger.warning(
|
|
164
267
|
f"Arize Phoenix is not running on {self._base_url}. Launch Phoenix "
|
|
165
268
|
f"with `import phoenix as px; px.launch_app()`"
|
|
166
269
|
)
|
|
167
270
|
|
|
168
|
-
def log_evaluations(
|
|
271
|
+
def log_evaluations(
|
|
272
|
+
self,
|
|
273
|
+
*evals: Evaluations,
|
|
274
|
+
timeout: Optional[int] = DEFAULT_TIMEOUT_IN_SECONDS,
|
|
275
|
+
**kwargs: Any,
|
|
276
|
+
) -> None:
|
|
169
277
|
"""
|
|
170
278
|
Logs evaluation data to the Phoenix server.
|
|
171
279
|
|
|
172
280
|
Args:
|
|
173
281
|
evals (Evaluations): One or more Evaluations objects containing the data to log.
|
|
174
|
-
|
|
175
|
-
This can be set using environment variables. If not provided, falls back to the
|
|
176
|
-
default project.
|
|
282
|
+
timeout (int, optional): The number of seconds to wait for the server to respond.
|
|
177
283
|
|
|
178
284
|
Returns:
|
|
179
285
|
None
|
|
180
286
|
"""
|
|
181
|
-
project_name
|
|
287
|
+
if kwargs.pop("project_name", None) is not None:
|
|
288
|
+
print("Keyword argument `project_name` is no longer necessary and is ignored.")
|
|
289
|
+
if kwargs:
|
|
290
|
+
raise TypeError(f"Unexpected keyword arguments: {', '.join(kwargs)}")
|
|
182
291
|
for evaluation in evals:
|
|
183
292
|
table = evaluation.to_pyarrow_table()
|
|
184
293
|
sink = pa.BufferOutputStream()
|
|
185
294
|
headers = {"content-type": "application/x-pandas-arrow"}
|
|
186
|
-
if project_name:
|
|
187
|
-
headers["project-name"] = project_name
|
|
188
295
|
with pa.ipc.new_stream(sink, table.schema) as writer:
|
|
189
296
|
writer.write_table(table)
|
|
190
|
-
self.
|
|
191
|
-
urljoin(self._base_url, "
|
|
192
|
-
|
|
297
|
+
self._client.post(
|
|
298
|
+
url=urljoin(self._base_url, "v1/evaluations"),
|
|
299
|
+
content=cast(bytes, sink.getvalue().to_pybytes()),
|
|
193
300
|
headers=headers,
|
|
301
|
+
timeout=timeout,
|
|
194
302
|
).raise_for_status()
|
|
195
303
|
|
|
304
|
+
def log_traces(self, trace_dataset: TraceDataset, project_name: Optional[str] = None) -> None:
|
|
305
|
+
"""
|
|
306
|
+
Logs traces from a TraceDataset to the Phoenix server.
|
|
307
|
+
|
|
308
|
+
Args:
|
|
309
|
+
trace_dataset (TraceDataset): A TraceDataset instance with the traces to log to
|
|
310
|
+
the Phoenix server.
|
|
311
|
+
project_name (str, optional): The project name under which to log the evaluations.
|
|
312
|
+
This can be set using environment variables. If not provided, falls back to the
|
|
313
|
+
default project.
|
|
314
|
+
|
|
315
|
+
Returns:
|
|
316
|
+
None
|
|
317
|
+
"""
|
|
318
|
+
project_name = project_name or get_env_project_name()
|
|
319
|
+
spans = trace_dataset.to_spans()
|
|
320
|
+
otlp_spans = [
|
|
321
|
+
ExportTraceServiceRequest(
|
|
322
|
+
resource_spans=[
|
|
323
|
+
ResourceSpans(
|
|
324
|
+
resource=Resource(
|
|
325
|
+
attributes=[
|
|
326
|
+
KeyValue(
|
|
327
|
+
key="openinference.project.name",
|
|
328
|
+
value=AnyValue(string_value=project_name),
|
|
329
|
+
)
|
|
330
|
+
]
|
|
331
|
+
),
|
|
332
|
+
scope_spans=[ScopeSpans(spans=[encode_span_to_otlp(span)])],
|
|
333
|
+
)
|
|
334
|
+
],
|
|
335
|
+
)
|
|
336
|
+
for span in spans
|
|
337
|
+
]
|
|
338
|
+
for otlp_span in otlp_spans:
|
|
339
|
+
serialized = otlp_span.SerializeToString()
|
|
340
|
+
content = gzip.compress(serialized)
|
|
341
|
+
response = self._client.post(
|
|
342
|
+
url=urljoin(self._base_url, "v1/traces"),
|
|
343
|
+
content=content,
|
|
344
|
+
headers={
|
|
345
|
+
"content-type": "application/x-protobuf",
|
|
346
|
+
"content-encoding": "gzip",
|
|
347
|
+
},
|
|
348
|
+
)
|
|
349
|
+
response.raise_for_status()
|
|
350
|
+
|
|
351
|
+
def _get_dataset_id_by_name(self, name: str) -> str:
|
|
352
|
+
"""
|
|
353
|
+
Gets a dataset by name.
|
|
354
|
+
|
|
355
|
+
Args:
|
|
356
|
+
name (str): The name of the dataset.
|
|
357
|
+
version_id (Optional[str]): The version ID of the dataset. Default None.
|
|
358
|
+
|
|
359
|
+
Returns:
|
|
360
|
+
Dataset: The dataset object.
|
|
361
|
+
"""
|
|
362
|
+
response = self._client.get(
|
|
363
|
+
urljoin(self._base_url, "v1/datasets"),
|
|
364
|
+
params={"name": name},
|
|
365
|
+
)
|
|
366
|
+
response.raise_for_status()
|
|
367
|
+
if not (records := response.json()["data"]):
|
|
368
|
+
raise ValueError(f"Failed to query dataset by name: {name}")
|
|
369
|
+
if len(records) > 1 or not records[0]:
|
|
370
|
+
raise ValueError(f"Failed to find a single dataset with the given name: {name}")
|
|
371
|
+
dataset = records[0]
|
|
372
|
+
return str(dataset["id"])
|
|
373
|
+
|
|
374
|
+
def get_dataset(
|
|
375
|
+
self,
|
|
376
|
+
*,
|
|
377
|
+
id: Optional[str] = None,
|
|
378
|
+
name: Optional[str] = None,
|
|
379
|
+
version_id: Optional[str] = None,
|
|
380
|
+
) -> Dataset:
|
|
381
|
+
"""
|
|
382
|
+
Gets the dataset for a specific version, or gets the latest version of
|
|
383
|
+
the dataset if no version is specified.
|
|
384
|
+
|
|
385
|
+
Args:
|
|
386
|
+
|
|
387
|
+
id (Optional[str]): An ID for the dataset.
|
|
388
|
+
|
|
389
|
+
name (Optional[str]): the name for the dataset. If provided, the ID
|
|
390
|
+
is ignored and the dataset is retrieved by name.
|
|
391
|
+
|
|
392
|
+
version_id (Optional[str]): An ID for the version of the dataset, or
|
|
393
|
+
None.
|
|
394
|
+
|
|
395
|
+
Returns:
|
|
396
|
+
A dataset object.
|
|
397
|
+
"""
|
|
398
|
+
if name:
|
|
399
|
+
id = self._get_dataset_id_by_name(name)
|
|
400
|
+
|
|
401
|
+
if not id:
|
|
402
|
+
raise ValueError("Dataset id or name must be provided.")
|
|
403
|
+
|
|
404
|
+
response = self._client.get(
|
|
405
|
+
urljoin(self._base_url, f"v1/datasets/{quote(id)}/examples"),
|
|
406
|
+
params={"version_id": version_id} if version_id else None,
|
|
407
|
+
)
|
|
408
|
+
response.raise_for_status()
|
|
409
|
+
data = response.json()["data"]
|
|
410
|
+
examples = {
|
|
411
|
+
example["id"]: Example(
|
|
412
|
+
id=example["id"],
|
|
413
|
+
input=example["input"],
|
|
414
|
+
output=example["output"],
|
|
415
|
+
metadata=example["metadata"],
|
|
416
|
+
updated_at=datetime.fromisoformat(example["updated_at"]),
|
|
417
|
+
)
|
|
418
|
+
for example in data["examples"]
|
|
419
|
+
}
|
|
420
|
+
resolved_dataset_id = data["dataset_id"]
|
|
421
|
+
resolved_version_id = data["version_id"]
|
|
422
|
+
return Dataset(
|
|
423
|
+
id=resolved_dataset_id,
|
|
424
|
+
version_id=resolved_version_id,
|
|
425
|
+
examples=examples,
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
def get_dataset_versions(
|
|
429
|
+
self,
|
|
430
|
+
dataset_id: str,
|
|
431
|
+
*,
|
|
432
|
+
limit: Optional[int] = 100,
|
|
433
|
+
) -> pd.DataFrame:
|
|
434
|
+
"""
|
|
435
|
+
Get dataset versions as pandas DataFrame.
|
|
436
|
+
|
|
437
|
+
Args:
|
|
438
|
+
dataset_id (str): dataset ID
|
|
439
|
+
limit (Optional[int]): maximum number of versions to return,
|
|
440
|
+
starting from the most recent version
|
|
441
|
+
|
|
442
|
+
Returns:
|
|
443
|
+
pandas DataFrame
|
|
444
|
+
"""
|
|
445
|
+
url = urljoin(self._base_url, f"v1/datasets/{dataset_id}/versions")
|
|
446
|
+
response = self._client.get(url=url, params={"limit": limit})
|
|
447
|
+
response.raise_for_status()
|
|
448
|
+
if not (records := response.json()["data"]):
|
|
449
|
+
return pd.DataFrame()
|
|
450
|
+
df = pd.DataFrame.from_records(records, index="version_id")
|
|
451
|
+
df["created_at"] = df["created_at"].apply(datetime.fromisoformat)
|
|
452
|
+
return df
|
|
453
|
+
|
|
454
|
+
def upload_dataset(
|
|
455
|
+
self,
|
|
456
|
+
*,
|
|
457
|
+
dataset_name: str,
|
|
458
|
+
dataframe: Optional[pd.DataFrame] = None,
|
|
459
|
+
csv_file_path: Optional[Union[str, Path]] = None,
|
|
460
|
+
input_keys: Iterable[str] = (),
|
|
461
|
+
output_keys: Iterable[str] = (),
|
|
462
|
+
metadata_keys: Iterable[str] = (),
|
|
463
|
+
inputs: Iterable[Mapping[str, Any]] = (),
|
|
464
|
+
outputs: Iterable[Mapping[str, Any]] = (),
|
|
465
|
+
metadata: Iterable[Mapping[str, Any]] = (),
|
|
466
|
+
dataset_description: Optional[str] = None,
|
|
467
|
+
) -> Dataset:
|
|
468
|
+
"""
|
|
469
|
+
Upload examples as dataset to the Phoenix server. If `dataframe` or
|
|
470
|
+
`csv_file_path` are provided, must also provide `input_keys` (and
|
|
471
|
+
optionally with `output_keys` or `metadata_keys` or both), which is a
|
|
472
|
+
list of strings denoting the column names in the dataframe or the csv
|
|
473
|
+
file. On the other hand, a sequence of dictionaries can also be provided
|
|
474
|
+
via `inputs` (and optionally with `outputs` or `metadat` or both), each
|
|
475
|
+
item of which represents a separate example in the dataset.
|
|
476
|
+
|
|
477
|
+
Args:
|
|
478
|
+
dataset_name: (str): Name of the dataset.
|
|
479
|
+
dataframe (pd.DataFrame): pandas DataFrame.
|
|
480
|
+
csv_file_path (str | Path): Location of a CSV text file
|
|
481
|
+
input_keys (Iterable[str]): List of column names used as input keys.
|
|
482
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
483
|
+
exist in CSV column headers.
|
|
484
|
+
output_keys (Iterable[str]): List of column names used as output keys.
|
|
485
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
486
|
+
exist in CSV column headers.
|
|
487
|
+
metadata_keys (Iterable[str]): List of column names used as metadata keys.
|
|
488
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
489
|
+
exist in CSV column headers.
|
|
490
|
+
inputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
491
|
+
corresponding to an example in the dataset.
|
|
492
|
+
outputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
493
|
+
corresponding to an example in the dataset.
|
|
494
|
+
metadata (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
495
|
+
corresponding to an example in the dataset.
|
|
496
|
+
dataset_description: (Optional[str]): Description of the dataset.
|
|
497
|
+
|
|
498
|
+
Returns:
|
|
499
|
+
A Dataset object with the uploaded examples.
|
|
500
|
+
"""
|
|
501
|
+
if dataframe is not None or csv_file_path is not None:
|
|
502
|
+
if dataframe is not None and csv_file_path is not None:
|
|
503
|
+
raise ValueError(
|
|
504
|
+
"Please provide either `dataframe` or `csv_file_path`, but not both"
|
|
505
|
+
)
|
|
506
|
+
if list(inputs) or list(outputs) or list(metadata):
|
|
507
|
+
option = "dataframe" if dataframe is not None else "csv_file_path"
|
|
508
|
+
raise ValueError(
|
|
509
|
+
f"Please provide only either `{option}` or list of dictionaries "
|
|
510
|
+
f"via `inputs` (with `outputs` and `metadata`) but not both."
|
|
511
|
+
)
|
|
512
|
+
table = dataframe if dataframe is not None else csv_file_path
|
|
513
|
+
assert table is not None # for type-checker
|
|
514
|
+
return self._upload_tabular_dataset(
|
|
515
|
+
table,
|
|
516
|
+
dataset_name=dataset_name,
|
|
517
|
+
input_keys=input_keys,
|
|
518
|
+
output_keys=output_keys,
|
|
519
|
+
metadata_keys=metadata_keys,
|
|
520
|
+
dataset_description=dataset_description,
|
|
521
|
+
)
|
|
522
|
+
return self._upload_json_dataset(
|
|
523
|
+
dataset_name=dataset_name,
|
|
524
|
+
inputs=inputs,
|
|
525
|
+
outputs=outputs,
|
|
526
|
+
metadata=metadata,
|
|
527
|
+
dataset_description=dataset_description,
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
def append_to_dataset(
|
|
531
|
+
self,
|
|
532
|
+
*,
|
|
533
|
+
dataset_name: str,
|
|
534
|
+
dataframe: Optional[pd.DataFrame] = None,
|
|
535
|
+
csv_file_path: Optional[Union[str, Path]] = None,
|
|
536
|
+
input_keys: Iterable[str] = (),
|
|
537
|
+
output_keys: Iterable[str] = (),
|
|
538
|
+
metadata_keys: Iterable[str] = (),
|
|
539
|
+
inputs: Iterable[Mapping[str, Any]] = (),
|
|
540
|
+
outputs: Iterable[Mapping[str, Any]] = (),
|
|
541
|
+
metadata: Iterable[Mapping[str, Any]] = (),
|
|
542
|
+
) -> Dataset:
|
|
543
|
+
"""
|
|
544
|
+
Append examples to dataset on the Phoenix server. If `dataframe` or
|
|
545
|
+
`csv_file_path` are provided, must also provide `input_keys` (and
|
|
546
|
+
optionally with `output_keys` or `metadata_keys` or both), which is a
|
|
547
|
+
list of strings denoting the column names in the dataframe or the csv
|
|
548
|
+
file. On the other hand, a sequence of dictionaries can also be provided
|
|
549
|
+
via `inputs` (and optionally with `outputs` or `metadat` or both), each
|
|
550
|
+
item of which represents a separate example in the dataset.
|
|
551
|
+
|
|
552
|
+
Args:
|
|
553
|
+
dataset_name: (str): Name of the dataset.
|
|
554
|
+
dataframe (pd.DataFrame): pandas DataFrame.
|
|
555
|
+
csv_file_path (str | Path): Location of a CSV text file
|
|
556
|
+
input_keys (Iterable[str]): List of column names used as input keys.
|
|
557
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
558
|
+
exist in CSV column headers.
|
|
559
|
+
output_keys (Iterable[str]): List of column names used as output keys.
|
|
560
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
561
|
+
exist in CSV column headers.
|
|
562
|
+
metadata_keys (Iterable[str]): List of column names used as metadata keys.
|
|
563
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
564
|
+
exist in CSV column headers.
|
|
565
|
+
inputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
566
|
+
corresponding to an example in the dataset.
|
|
567
|
+
outputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
568
|
+
corresponding to an example in the dataset.
|
|
569
|
+
metadata (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
570
|
+
corresponding to an example in the dataset.
|
|
571
|
+
|
|
572
|
+
Returns:
|
|
573
|
+
A Dataset object with its examples.
|
|
574
|
+
"""
|
|
575
|
+
if dataframe is not None or csv_file_path is not None:
|
|
576
|
+
if dataframe is not None and csv_file_path is not None:
|
|
577
|
+
raise ValueError(
|
|
578
|
+
"Please provide either `dataframe` or `csv_file_path`, but not both"
|
|
579
|
+
)
|
|
580
|
+
if list(inputs) or list(outputs) or list(metadata):
|
|
581
|
+
option = "dataframe" if dataframe is not None else "csv_file_path"
|
|
582
|
+
raise ValueError(
|
|
583
|
+
f"Please provide only either `{option}` or list of dictionaries "
|
|
584
|
+
f"via `inputs` (with `outputs` and `metadata`) but not both."
|
|
585
|
+
)
|
|
586
|
+
table = dataframe if dataframe is not None else csv_file_path
|
|
587
|
+
assert table is not None # for type-checker
|
|
588
|
+
return self._upload_tabular_dataset(
|
|
589
|
+
table,
|
|
590
|
+
dataset_name=dataset_name,
|
|
591
|
+
input_keys=input_keys,
|
|
592
|
+
output_keys=output_keys,
|
|
593
|
+
metadata_keys=metadata_keys,
|
|
594
|
+
action="append",
|
|
595
|
+
)
|
|
596
|
+
return self._upload_json_dataset(
|
|
597
|
+
dataset_name=dataset_name,
|
|
598
|
+
inputs=inputs,
|
|
599
|
+
outputs=outputs,
|
|
600
|
+
metadata=metadata,
|
|
601
|
+
action="append",
|
|
602
|
+
)
|
|
603
|
+
|
|
604
|
+
def get_experiment(self, *, experiment_id: str) -> Experiment:
|
|
605
|
+
"""
|
|
606
|
+
Get an experiment by ID.
|
|
607
|
+
|
|
608
|
+
Retrieve an Experiment object by ID, enables running `evaluate_experiment` after finishing
|
|
609
|
+
the initial experiment run.
|
|
610
|
+
|
|
611
|
+
Args:
|
|
612
|
+
experiment_id (str): ID of the experiment. This can be found in the UI.
|
|
613
|
+
"""
|
|
614
|
+
response = self._client.get(
|
|
615
|
+
url=urljoin(self._base_url, f"v1/experiments/{experiment_id}"),
|
|
616
|
+
)
|
|
617
|
+
experiment = response.json()["data"]
|
|
618
|
+
return Experiment.from_dict(experiment)
|
|
619
|
+
|
|
620
|
+
def _upload_tabular_dataset(
|
|
621
|
+
self,
|
|
622
|
+
table: Union[str, Path, pd.DataFrame],
|
|
623
|
+
/,
|
|
624
|
+
*,
|
|
625
|
+
dataset_name: str,
|
|
626
|
+
input_keys: Iterable[str],
|
|
627
|
+
output_keys: Iterable[str] = (),
|
|
628
|
+
metadata_keys: Iterable[str] = (),
|
|
629
|
+
dataset_description: Optional[str] = None,
|
|
630
|
+
action: DatasetAction = "create",
|
|
631
|
+
) -> Dataset:
|
|
632
|
+
"""
|
|
633
|
+
Upload examples as dataset to the Phoenix server.
|
|
634
|
+
|
|
635
|
+
Args:
|
|
636
|
+
table (str | Path | pd.DataFrame): Location of a CSV text file, or
|
|
637
|
+
pandas DataFrame.
|
|
638
|
+
dataset_name: (str): Name of the dataset. Required if action=append.
|
|
639
|
+
input_keys (Iterable[str]): List of column names used as input keys.
|
|
640
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
641
|
+
exist in CSV column headers.
|
|
642
|
+
output_keys (Iterable[str]): List of column names used as output keys.
|
|
643
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
644
|
+
exist in CSV column headers.
|
|
645
|
+
metadata_keys (Iterable[str]): List of column names used as metadata keys.
|
|
646
|
+
input_keys, output_keys, metadata_keys must be disjoint, and must
|
|
647
|
+
exist in CSV column headers.
|
|
648
|
+
dataset_description: (Optional[str]): Description of the dataset.
|
|
649
|
+
action: (Literal["create", "append"]): Create new dataset or append to an
|
|
650
|
+
existing one. If action="append" and dataset does not exist, it'll
|
|
651
|
+
be created.
|
|
652
|
+
|
|
653
|
+
Returns:
|
|
654
|
+
A Dataset object with the uploaded examples.
|
|
655
|
+
"""
|
|
656
|
+
if action not in ("create", "append"):
|
|
657
|
+
raise ValueError(f"Invalid action: {action}")
|
|
658
|
+
if not dataset_name:
|
|
659
|
+
raise ValueError("Dataset name must not be blank")
|
|
660
|
+
input_keys, output_keys, metadata_keys = (
|
|
661
|
+
(keys,) if isinstance(keys, str) else (keys or ())
|
|
662
|
+
for keys in (input_keys, output_keys, metadata_keys)
|
|
663
|
+
)
|
|
664
|
+
if not any(map(bool, (input_keys, output_keys, metadata_keys))):
|
|
665
|
+
input_keys, output_keys, metadata_keys = _infer_keys(table)
|
|
666
|
+
keys = DatasetKeys(
|
|
667
|
+
frozenset(input_keys),
|
|
668
|
+
frozenset(output_keys),
|
|
669
|
+
frozenset(metadata_keys),
|
|
670
|
+
)
|
|
671
|
+
if isinstance(table, pd.DataFrame):
|
|
672
|
+
file = _prepare_pyarrow(table, keys)
|
|
673
|
+
elif isinstance(table, (str, Path)):
|
|
674
|
+
file = _prepare_csv(Path(table), keys)
|
|
675
|
+
else:
|
|
676
|
+
assert_never(table)
|
|
677
|
+
print("📤 Uploading dataset...")
|
|
678
|
+
response = self._client.post(
|
|
679
|
+
url=urljoin(self._base_url, "v1/datasets/upload"),
|
|
680
|
+
files={"file": file},
|
|
681
|
+
data={
|
|
682
|
+
"action": action,
|
|
683
|
+
"name": dataset_name,
|
|
684
|
+
"description": dataset_description,
|
|
685
|
+
"input_keys[]": sorted(keys.input),
|
|
686
|
+
"output_keys[]": sorted(keys.output),
|
|
687
|
+
"metadata_keys[]": sorted(keys.metadata),
|
|
688
|
+
},
|
|
689
|
+
params={"sync": True},
|
|
690
|
+
)
|
|
691
|
+
return self._process_dataset_upload_response(response)
|
|
692
|
+
|
|
693
|
+
def _upload_json_dataset(
|
|
694
|
+
self,
|
|
695
|
+
*,
|
|
696
|
+
dataset_name: str,
|
|
697
|
+
inputs: Iterable[Mapping[str, Any]],
|
|
698
|
+
outputs: Iterable[Mapping[str, Any]] = (),
|
|
699
|
+
metadata: Iterable[Mapping[str, Any]] = (),
|
|
700
|
+
dataset_description: Optional[str] = None,
|
|
701
|
+
action: DatasetAction = "create",
|
|
702
|
+
) -> Dataset:
|
|
703
|
+
"""
|
|
704
|
+
Upload examples as dataset to the Phoenix server.
|
|
705
|
+
|
|
706
|
+
Args:
|
|
707
|
+
dataset_name: (str): Name of the dataset
|
|
708
|
+
inputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
709
|
+
corresponding to an example in the dataset.
|
|
710
|
+
outputs (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
711
|
+
corresponding to an example in the dataset.
|
|
712
|
+
metadata (Iterable[Mapping[str, Any]]): List of dictionaries object each
|
|
713
|
+
corresponding to an example in the dataset.
|
|
714
|
+
dataset_description: (Optional[str]): Description of the dataset.
|
|
715
|
+
action: (Literal["create", "append"]): Create new dataset or append to an
|
|
716
|
+
existing one. If action="append" and dataset does not exist, it'll
|
|
717
|
+
be created.
|
|
718
|
+
|
|
719
|
+
Returns:
|
|
720
|
+
A Dataset object with the uploaded examples.
|
|
721
|
+
"""
|
|
722
|
+
# convert to list to avoid issues with pandas Series
|
|
723
|
+
inputs, outputs, metadata = list(inputs), list(outputs), list(metadata)
|
|
724
|
+
if not inputs or not _is_all_dict(inputs):
|
|
725
|
+
raise ValueError(
|
|
726
|
+
"`inputs` should be a non-empty sequence containing only dictionary objects"
|
|
727
|
+
)
|
|
728
|
+
for name, seq in {"outputs": outputs, "metadata": metadata}.items():
|
|
729
|
+
if seq and not (len(seq) == len(inputs) and _is_all_dict(seq)):
|
|
730
|
+
raise ValueError(
|
|
731
|
+
f"`{name}` should be a sequence of the same length as `inputs` "
|
|
732
|
+
"containing only dictionary objects"
|
|
733
|
+
)
|
|
734
|
+
print("📤 Uploading dataset...")
|
|
735
|
+
response = self._client.post(
|
|
736
|
+
url=urljoin(self._base_url, "v1/datasets/upload"),
|
|
737
|
+
headers={"Content-Encoding": "gzip"},
|
|
738
|
+
json={
|
|
739
|
+
"action": action,
|
|
740
|
+
"name": dataset_name,
|
|
741
|
+
"description": dataset_description,
|
|
742
|
+
"inputs": inputs,
|
|
743
|
+
"outputs": outputs,
|
|
744
|
+
"metadata": metadata,
|
|
745
|
+
},
|
|
746
|
+
params={"sync": True},
|
|
747
|
+
)
|
|
748
|
+
return self._process_dataset_upload_response(response)
|
|
749
|
+
|
|
750
|
+
def _process_dataset_upload_response(self, response: Response) -> Dataset:
|
|
751
|
+
try:
|
|
752
|
+
response.raise_for_status()
|
|
753
|
+
except HTTPStatusError as e:
|
|
754
|
+
if msg := response.text:
|
|
755
|
+
raise DatasetUploadError(msg) from e
|
|
756
|
+
raise
|
|
757
|
+
data = response.json()["data"]
|
|
758
|
+
dataset_id = data["dataset_id"]
|
|
759
|
+
response = self._client.get(
|
|
760
|
+
url=urljoin(self._base_url, f"v1/datasets/{dataset_id}/examples")
|
|
761
|
+
)
|
|
762
|
+
response.raise_for_status()
|
|
763
|
+
data = response.json()["data"]
|
|
764
|
+
version_id = data["version_id"]
|
|
765
|
+
examples = data["examples"]
|
|
766
|
+
print(f"💾 Examples uploaded: {self.web_url}datasets/{dataset_id}/examples")
|
|
767
|
+
print(f"🗄️ Dataset version ID: {version_id}")
|
|
768
|
+
|
|
769
|
+
return Dataset(
|
|
770
|
+
id=dataset_id,
|
|
771
|
+
version_id=version_id,
|
|
772
|
+
examples={
|
|
773
|
+
example["id"]: Example(
|
|
774
|
+
id=example["id"],
|
|
775
|
+
input=example["input"],
|
|
776
|
+
output=example["output"],
|
|
777
|
+
metadata=example["metadata"],
|
|
778
|
+
updated_at=datetime.fromisoformat(example["updated_at"]),
|
|
779
|
+
)
|
|
780
|
+
for example in examples
|
|
781
|
+
},
|
|
782
|
+
)
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
FileName: TypeAlias = str
|
|
786
|
+
FilePointer: TypeAlias = BinaryIO
|
|
787
|
+
FileType: TypeAlias = str
|
|
788
|
+
FileHeaders: TypeAlias = dict[str, str]
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def _get_csv_column_headers(path: Path) -> tuple[str, ...]:
|
|
792
|
+
path = path.resolve()
|
|
793
|
+
if not path.is_file():
|
|
794
|
+
raise FileNotFoundError(f"File does not exist: {path}")
|
|
795
|
+
with open(path, "r") as f:
|
|
796
|
+
rows = csv.reader(f)
|
|
797
|
+
try:
|
|
798
|
+
column_headers = tuple(next(rows))
|
|
799
|
+
_ = next(rows)
|
|
800
|
+
except StopIteration:
|
|
801
|
+
raise ValueError("csv file has no data")
|
|
802
|
+
return column_headers
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _prepare_csv(
|
|
806
|
+
path: Path,
|
|
807
|
+
keys: DatasetKeys,
|
|
808
|
+
) -> tuple[FileName, FilePointer, FileType, FileHeaders]:
|
|
809
|
+
column_headers = _get_csv_column_headers(path)
|
|
810
|
+
(header, freq), *_ = Counter(column_headers).most_common(1)
|
|
811
|
+
if freq > 1:
|
|
812
|
+
raise ValueError(f"Duplicated column header in CSV file: {header}")
|
|
813
|
+
keys.check_differences(frozenset(column_headers))
|
|
814
|
+
file = BytesIO()
|
|
815
|
+
with open(path, "rb") as f:
|
|
816
|
+
file.write(gzip.compress(f.read()))
|
|
817
|
+
return path.name, file, "text/csv", {"Content-Encoding": "gzip"}
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _prepare_pyarrow(
|
|
821
|
+
df: pd.DataFrame,
|
|
822
|
+
keys: DatasetKeys,
|
|
823
|
+
) -> tuple[FileName, FilePointer, FileType, FileHeaders]:
|
|
824
|
+
if df.empty:
|
|
825
|
+
raise ValueError("dataframe has no data")
|
|
826
|
+
(header, freq), *_ = Counter(df.columns).most_common(1)
|
|
827
|
+
if freq > 1:
|
|
828
|
+
raise ValueError(f"Duplicated column header in file: {header}")
|
|
829
|
+
keys.check_differences(frozenset(df.columns))
|
|
830
|
+
table = Table.from_pandas(df.loc[:, list(keys)])
|
|
831
|
+
sink = pa.BufferOutputStream()
|
|
832
|
+
options = pa.ipc.IpcWriteOptions(compression="lz4")
|
|
833
|
+
with pa.ipc.new_stream(sink, table.schema, options=options) as writer:
|
|
834
|
+
writer.write_table(table)
|
|
835
|
+
file = BytesIO(sink.getvalue().to_pybytes())
|
|
836
|
+
return "pandas", file, "application/x-pandas-pyarrow", {}
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
_response_header = re.compile(r"(?i)(response|answer|output)s*$")
|
|
840
|
+
|
|
841
|
+
|
|
842
|
+
def _infer_keys(
|
|
843
|
+
table: Union[str, Path, pd.DataFrame],
|
|
844
|
+
) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]:
|
|
845
|
+
column_headers = (
|
|
846
|
+
tuple(table.columns)
|
|
847
|
+
if isinstance(table, pd.DataFrame)
|
|
848
|
+
else _get_csv_column_headers(Path(table))
|
|
849
|
+
)
|
|
850
|
+
for i, header in enumerate(column_headers):
|
|
851
|
+
if _response_header.search(header):
|
|
852
|
+
break
|
|
853
|
+
else:
|
|
854
|
+
i = len(column_headers)
|
|
855
|
+
return (
|
|
856
|
+
column_headers[:i],
|
|
857
|
+
column_headers[i : i + 1],
|
|
858
|
+
column_headers[i + 1 :],
|
|
859
|
+
)
|
|
860
|
+
|
|
196
861
|
|
|
197
862
|
def _to_iso_format(value: Optional[datetime]) -> Optional[str]:
|
|
198
863
|
return value.isoformat() if value else None
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
def _is_all_dict(seq: Sequence[Any]) -> bool:
|
|
867
|
+
return all(map(lambda obj: isinstance(obj, dict), seq))
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
class DatasetUploadError(Exception): ...
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
class TimeoutError(Exception): ...
|