arize-phoenix 3.16.0__py3-none-any.whl → 7.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of arize-phoenix might be problematic. Click here for more details.

Files changed (338) hide show
  1. arize_phoenix-7.7.0.dist-info/METADATA +261 -0
  2. arize_phoenix-7.7.0.dist-info/RECORD +345 -0
  3. {arize_phoenix-3.16.0.dist-info → arize_phoenix-7.7.0.dist-info}/WHEEL +1 -1
  4. arize_phoenix-7.7.0.dist-info/entry_points.txt +3 -0
  5. phoenix/__init__.py +86 -14
  6. phoenix/auth.py +309 -0
  7. phoenix/config.py +675 -45
  8. phoenix/core/model.py +32 -30
  9. phoenix/core/model_schema.py +102 -109
  10. phoenix/core/model_schema_adapter.py +48 -45
  11. phoenix/datetime_utils.py +24 -3
  12. phoenix/db/README.md +54 -0
  13. phoenix/db/__init__.py +4 -0
  14. phoenix/db/alembic.ini +85 -0
  15. phoenix/db/bulk_inserter.py +294 -0
  16. phoenix/db/engines.py +208 -0
  17. phoenix/db/enums.py +20 -0
  18. phoenix/db/facilitator.py +113 -0
  19. phoenix/db/helpers.py +159 -0
  20. phoenix/db/insertion/constants.py +2 -0
  21. phoenix/db/insertion/dataset.py +227 -0
  22. phoenix/db/insertion/document_annotation.py +171 -0
  23. phoenix/db/insertion/evaluation.py +191 -0
  24. phoenix/db/insertion/helpers.py +98 -0
  25. phoenix/db/insertion/span.py +193 -0
  26. phoenix/db/insertion/span_annotation.py +158 -0
  27. phoenix/db/insertion/trace_annotation.py +158 -0
  28. phoenix/db/insertion/types.py +256 -0
  29. phoenix/db/migrate.py +86 -0
  30. phoenix/db/migrations/data_migration_scripts/populate_project_sessions.py +199 -0
  31. phoenix/db/migrations/env.py +114 -0
  32. phoenix/db/migrations/script.py.mako +26 -0
  33. phoenix/db/migrations/versions/10460e46d750_datasets.py +317 -0
  34. phoenix/db/migrations/versions/3be8647b87d8_add_token_columns_to_spans_table.py +126 -0
  35. phoenix/db/migrations/versions/4ded9e43755f_create_project_sessions_table.py +66 -0
  36. phoenix/db/migrations/versions/cd164e83824f_users_and_tokens.py +157 -0
  37. phoenix/db/migrations/versions/cf03bd6bae1d_init.py +280 -0
  38. phoenix/db/models.py +807 -0
  39. phoenix/exceptions.py +5 -1
  40. phoenix/experiments/__init__.py +6 -0
  41. phoenix/experiments/evaluators/__init__.py +29 -0
  42. phoenix/experiments/evaluators/base.py +158 -0
  43. phoenix/experiments/evaluators/code_evaluators.py +184 -0
  44. phoenix/experiments/evaluators/llm_evaluators.py +473 -0
  45. phoenix/experiments/evaluators/utils.py +236 -0
  46. phoenix/experiments/functions.py +772 -0
  47. phoenix/experiments/tracing.py +86 -0
  48. phoenix/experiments/types.py +726 -0
  49. phoenix/experiments/utils.py +25 -0
  50. phoenix/inferences/__init__.py +0 -0
  51. phoenix/{datasets → inferences}/errors.py +6 -5
  52. phoenix/{datasets → inferences}/fixtures.py +49 -42
  53. phoenix/{datasets/dataset.py → inferences/inferences.py} +121 -105
  54. phoenix/{datasets → inferences}/schema.py +11 -11
  55. phoenix/{datasets → inferences}/validation.py +13 -14
  56. phoenix/logging/__init__.py +3 -0
  57. phoenix/logging/_config.py +90 -0
  58. phoenix/logging/_filter.py +6 -0
  59. phoenix/logging/_formatter.py +69 -0
  60. phoenix/metrics/__init__.py +5 -4
  61. phoenix/metrics/binning.py +4 -3
  62. phoenix/metrics/metrics.py +2 -1
  63. phoenix/metrics/mixins.py +7 -6
  64. phoenix/metrics/retrieval_metrics.py +2 -1
  65. phoenix/metrics/timeseries.py +5 -4
  66. phoenix/metrics/wrappers.py +9 -3
  67. phoenix/pointcloud/clustering.py +5 -5
  68. phoenix/pointcloud/pointcloud.py +7 -5
  69. phoenix/pointcloud/projectors.py +5 -6
  70. phoenix/pointcloud/umap_parameters.py +53 -52
  71. phoenix/server/api/README.md +28 -0
  72. phoenix/server/api/auth.py +44 -0
  73. phoenix/server/api/context.py +152 -9
  74. phoenix/server/api/dataloaders/__init__.py +91 -0
  75. phoenix/server/api/dataloaders/annotation_summaries.py +139 -0
  76. phoenix/server/api/dataloaders/average_experiment_run_latency.py +54 -0
  77. phoenix/server/api/dataloaders/cache/__init__.py +3 -0
  78. phoenix/server/api/dataloaders/cache/two_tier_cache.py +68 -0
  79. phoenix/server/api/dataloaders/dataset_example_revisions.py +131 -0
  80. phoenix/server/api/dataloaders/dataset_example_spans.py +38 -0
  81. phoenix/server/api/dataloaders/document_evaluation_summaries.py +144 -0
  82. phoenix/server/api/dataloaders/document_evaluations.py +31 -0
  83. phoenix/server/api/dataloaders/document_retrieval_metrics.py +89 -0
  84. phoenix/server/api/dataloaders/experiment_annotation_summaries.py +79 -0
  85. phoenix/server/api/dataloaders/experiment_error_rates.py +58 -0
  86. phoenix/server/api/dataloaders/experiment_run_annotations.py +36 -0
  87. phoenix/server/api/dataloaders/experiment_run_counts.py +49 -0
  88. phoenix/server/api/dataloaders/experiment_sequence_number.py +44 -0
  89. phoenix/server/api/dataloaders/latency_ms_quantile.py +188 -0
  90. phoenix/server/api/dataloaders/min_start_or_max_end_times.py +85 -0
  91. phoenix/server/api/dataloaders/project_by_name.py +31 -0
  92. phoenix/server/api/dataloaders/record_counts.py +116 -0
  93. phoenix/server/api/dataloaders/session_io.py +79 -0
  94. phoenix/server/api/dataloaders/session_num_traces.py +30 -0
  95. phoenix/server/api/dataloaders/session_num_traces_with_error.py +32 -0
  96. phoenix/server/api/dataloaders/session_token_usages.py +41 -0
  97. phoenix/server/api/dataloaders/session_trace_latency_ms_quantile.py +55 -0
  98. phoenix/server/api/dataloaders/span_annotations.py +26 -0
  99. phoenix/server/api/dataloaders/span_dataset_examples.py +31 -0
  100. phoenix/server/api/dataloaders/span_descendants.py +57 -0
  101. phoenix/server/api/dataloaders/span_projects.py +33 -0
  102. phoenix/server/api/dataloaders/token_counts.py +124 -0
  103. phoenix/server/api/dataloaders/trace_by_trace_ids.py +25 -0
  104. phoenix/server/api/dataloaders/trace_root_spans.py +32 -0
  105. phoenix/server/api/dataloaders/user_roles.py +30 -0
  106. phoenix/server/api/dataloaders/users.py +33 -0
  107. phoenix/server/api/exceptions.py +48 -0
  108. phoenix/server/api/helpers/__init__.py +12 -0
  109. phoenix/server/api/helpers/dataset_helpers.py +217 -0
  110. phoenix/server/api/helpers/experiment_run_filters.py +763 -0
  111. phoenix/server/api/helpers/playground_clients.py +948 -0
  112. phoenix/server/api/helpers/playground_registry.py +70 -0
  113. phoenix/server/api/helpers/playground_spans.py +455 -0
  114. phoenix/server/api/input_types/AddExamplesToDatasetInput.py +16 -0
  115. phoenix/server/api/input_types/AddSpansToDatasetInput.py +14 -0
  116. phoenix/server/api/input_types/ChatCompletionInput.py +38 -0
  117. phoenix/server/api/input_types/ChatCompletionMessageInput.py +24 -0
  118. phoenix/server/api/input_types/ClearProjectInput.py +15 -0
  119. phoenix/server/api/input_types/ClusterInput.py +2 -2
  120. phoenix/server/api/input_types/CreateDatasetInput.py +12 -0
  121. phoenix/server/api/input_types/CreateSpanAnnotationInput.py +18 -0
  122. phoenix/server/api/input_types/CreateTraceAnnotationInput.py +18 -0
  123. phoenix/server/api/input_types/DataQualityMetricInput.py +5 -2
  124. phoenix/server/api/input_types/DatasetExampleInput.py +14 -0
  125. phoenix/server/api/input_types/DatasetSort.py +17 -0
  126. phoenix/server/api/input_types/DatasetVersionSort.py +16 -0
  127. phoenix/server/api/input_types/DeleteAnnotationsInput.py +7 -0
  128. phoenix/server/api/input_types/DeleteDatasetExamplesInput.py +13 -0
  129. phoenix/server/api/input_types/DeleteDatasetInput.py +7 -0
  130. phoenix/server/api/input_types/DeleteExperimentsInput.py +7 -0
  131. phoenix/server/api/input_types/DimensionFilter.py +4 -4
  132. phoenix/server/api/input_types/GenerativeModelInput.py +17 -0
  133. phoenix/server/api/input_types/Granularity.py +1 -1
  134. phoenix/server/api/input_types/InvocationParameters.py +162 -0
  135. phoenix/server/api/input_types/PatchAnnotationInput.py +19 -0
  136. phoenix/server/api/input_types/PatchDatasetExamplesInput.py +35 -0
  137. phoenix/server/api/input_types/PatchDatasetInput.py +14 -0
  138. phoenix/server/api/input_types/PerformanceMetricInput.py +5 -2
  139. phoenix/server/api/input_types/ProjectSessionSort.py +29 -0
  140. phoenix/server/api/input_types/SpanAnnotationSort.py +17 -0
  141. phoenix/server/api/input_types/SpanSort.py +134 -69
  142. phoenix/server/api/input_types/TemplateOptions.py +10 -0
  143. phoenix/server/api/input_types/TraceAnnotationSort.py +17 -0
  144. phoenix/server/api/input_types/UserRoleInput.py +9 -0
  145. phoenix/server/api/mutations/__init__.py +28 -0
  146. phoenix/server/api/mutations/api_key_mutations.py +167 -0
  147. phoenix/server/api/mutations/chat_mutations.py +593 -0
  148. phoenix/server/api/mutations/dataset_mutations.py +591 -0
  149. phoenix/server/api/mutations/experiment_mutations.py +75 -0
  150. phoenix/server/api/{types/ExportEventsMutation.py → mutations/export_events_mutations.py} +21 -18
  151. phoenix/server/api/mutations/project_mutations.py +57 -0
  152. phoenix/server/api/mutations/span_annotations_mutations.py +128 -0
  153. phoenix/server/api/mutations/trace_annotations_mutations.py +127 -0
  154. phoenix/server/api/mutations/user_mutations.py +329 -0
  155. phoenix/server/api/openapi/__init__.py +0 -0
  156. phoenix/server/api/openapi/main.py +17 -0
  157. phoenix/server/api/openapi/schema.py +16 -0
  158. phoenix/server/api/queries.py +738 -0
  159. phoenix/server/api/routers/__init__.py +11 -0
  160. phoenix/server/api/routers/auth.py +284 -0
  161. phoenix/server/api/routers/embeddings.py +26 -0
  162. phoenix/server/api/routers/oauth2.py +488 -0
  163. phoenix/server/api/routers/v1/__init__.py +64 -0
  164. phoenix/server/api/routers/v1/datasets.py +1017 -0
  165. phoenix/server/api/routers/v1/evaluations.py +362 -0
  166. phoenix/server/api/routers/v1/experiment_evaluations.py +115 -0
  167. phoenix/server/api/routers/v1/experiment_runs.py +167 -0
  168. phoenix/server/api/routers/v1/experiments.py +308 -0
  169. phoenix/server/api/routers/v1/pydantic_compat.py +78 -0
  170. phoenix/server/api/routers/v1/spans.py +267 -0
  171. phoenix/server/api/routers/v1/traces.py +208 -0
  172. phoenix/server/api/routers/v1/utils.py +95 -0
  173. phoenix/server/api/schema.py +44 -247
  174. phoenix/server/api/subscriptions.py +597 -0
  175. phoenix/server/api/types/Annotation.py +21 -0
  176. phoenix/server/api/types/AnnotationSummary.py +55 -0
  177. phoenix/server/api/types/AnnotatorKind.py +16 -0
  178. phoenix/server/api/types/ApiKey.py +27 -0
  179. phoenix/server/api/types/AuthMethod.py +9 -0
  180. phoenix/server/api/types/ChatCompletionMessageRole.py +11 -0
  181. phoenix/server/api/types/ChatCompletionSubscriptionPayload.py +46 -0
  182. phoenix/server/api/types/Cluster.py +25 -24
  183. phoenix/server/api/types/CreateDatasetPayload.py +8 -0
  184. phoenix/server/api/types/DataQualityMetric.py +31 -13
  185. phoenix/server/api/types/Dataset.py +288 -63
  186. phoenix/server/api/types/DatasetExample.py +85 -0
  187. phoenix/server/api/types/DatasetExampleRevision.py +34 -0
  188. phoenix/server/api/types/DatasetVersion.py +14 -0
  189. phoenix/server/api/types/Dimension.py +32 -31
  190. phoenix/server/api/types/DocumentEvaluationSummary.py +9 -8
  191. phoenix/server/api/types/EmbeddingDimension.py +56 -49
  192. phoenix/server/api/types/Evaluation.py +25 -31
  193. phoenix/server/api/types/EvaluationSummary.py +30 -50
  194. phoenix/server/api/types/Event.py +20 -20
  195. phoenix/server/api/types/ExampleRevisionInterface.py +14 -0
  196. phoenix/server/api/types/Experiment.py +152 -0
  197. phoenix/server/api/types/ExperimentAnnotationSummary.py +13 -0
  198. phoenix/server/api/types/ExperimentComparison.py +17 -0
  199. phoenix/server/api/types/ExperimentRun.py +119 -0
  200. phoenix/server/api/types/ExperimentRunAnnotation.py +56 -0
  201. phoenix/server/api/types/GenerativeModel.py +9 -0
  202. phoenix/server/api/types/GenerativeProvider.py +85 -0
  203. phoenix/server/api/types/Inferences.py +80 -0
  204. phoenix/server/api/types/InferencesRole.py +23 -0
  205. phoenix/server/api/types/LabelFraction.py +7 -0
  206. phoenix/server/api/types/MimeType.py +2 -2
  207. phoenix/server/api/types/Model.py +54 -54
  208. phoenix/server/api/types/PerformanceMetric.py +8 -5
  209. phoenix/server/api/types/Project.py +407 -142
  210. phoenix/server/api/types/ProjectSession.py +139 -0
  211. phoenix/server/api/types/Segments.py +4 -4
  212. phoenix/server/api/types/Span.py +221 -176
  213. phoenix/server/api/types/SpanAnnotation.py +43 -0
  214. phoenix/server/api/types/SpanIOValue.py +15 -0
  215. phoenix/server/api/types/SystemApiKey.py +9 -0
  216. phoenix/server/api/types/TemplateLanguage.py +10 -0
  217. phoenix/server/api/types/TimeSeries.py +19 -15
  218. phoenix/server/api/types/TokenUsage.py +11 -0
  219. phoenix/server/api/types/Trace.py +154 -0
  220. phoenix/server/api/types/TraceAnnotation.py +45 -0
  221. phoenix/server/api/types/UMAPPoints.py +7 -7
  222. phoenix/server/api/types/User.py +60 -0
  223. phoenix/server/api/types/UserApiKey.py +45 -0
  224. phoenix/server/api/types/UserRole.py +15 -0
  225. phoenix/server/api/types/node.py +13 -107
  226. phoenix/server/api/types/pagination.py +156 -57
  227. phoenix/server/api/utils.py +34 -0
  228. phoenix/server/app.py +864 -115
  229. phoenix/server/bearer_auth.py +163 -0
  230. phoenix/server/dml_event.py +136 -0
  231. phoenix/server/dml_event_handler.py +256 -0
  232. phoenix/server/email/__init__.py +0 -0
  233. phoenix/server/email/sender.py +97 -0
  234. phoenix/server/email/templates/__init__.py +0 -0
  235. phoenix/server/email/templates/password_reset.html +19 -0
  236. phoenix/server/email/types.py +11 -0
  237. phoenix/server/grpc_server.py +102 -0
  238. phoenix/server/jwt_store.py +505 -0
  239. phoenix/server/main.py +305 -116
  240. phoenix/server/oauth2.py +52 -0
  241. phoenix/server/openapi/__init__.py +0 -0
  242. phoenix/server/prometheus.py +111 -0
  243. phoenix/server/rate_limiters.py +188 -0
  244. phoenix/server/static/.vite/manifest.json +87 -0
  245. phoenix/server/static/assets/components-Cy9nwIvF.js +2125 -0
  246. phoenix/server/static/assets/index-BKvHIxkk.js +113 -0
  247. phoenix/server/static/assets/pages-CUi2xCVQ.js +4449 -0
  248. phoenix/server/static/assets/vendor-DvC8cT4X.js +894 -0
  249. phoenix/server/static/assets/vendor-DxkFTwjz.css +1 -0
  250. phoenix/server/static/assets/vendor-arizeai-Do1793cv.js +662 -0
  251. phoenix/server/static/assets/vendor-codemirror-BzwZPyJM.js +24 -0
  252. phoenix/server/static/assets/vendor-recharts-_Jb7JjhG.js +59 -0
  253. phoenix/server/static/assets/vendor-shiki-Cl9QBraO.js +5 -0
  254. phoenix/server/static/assets/vendor-three-DwGkEfCM.js +2998 -0
  255. phoenix/server/telemetry.py +68 -0
  256. phoenix/server/templates/index.html +82 -23
  257. phoenix/server/thread_server.py +3 -3
  258. phoenix/server/types.py +275 -0
  259. phoenix/services.py +27 -18
  260. phoenix/session/client.py +743 -68
  261. phoenix/session/data_extractor.py +31 -7
  262. phoenix/session/evaluation.py +3 -9
  263. phoenix/session/session.py +263 -219
  264. phoenix/settings.py +22 -0
  265. phoenix/trace/__init__.py +2 -22
  266. phoenix/trace/attributes.py +338 -0
  267. phoenix/trace/dsl/README.md +116 -0
  268. phoenix/trace/dsl/filter.py +663 -213
  269. phoenix/trace/dsl/helpers.py +73 -21
  270. phoenix/trace/dsl/query.py +574 -201
  271. phoenix/trace/exporter.py +24 -19
  272. phoenix/trace/fixtures.py +368 -32
  273. phoenix/trace/otel.py +71 -219
  274. phoenix/trace/projects.py +3 -2
  275. phoenix/trace/schemas.py +33 -11
  276. phoenix/trace/span_evaluations.py +21 -16
  277. phoenix/trace/span_json_decoder.py +6 -4
  278. phoenix/trace/span_json_encoder.py +2 -2
  279. phoenix/trace/trace_dataset.py +47 -32
  280. phoenix/trace/utils.py +21 -4
  281. phoenix/utilities/__init__.py +0 -26
  282. phoenix/utilities/client.py +132 -0
  283. phoenix/utilities/deprecation.py +31 -0
  284. phoenix/utilities/error_handling.py +3 -2
  285. phoenix/utilities/json.py +109 -0
  286. phoenix/utilities/logging.py +8 -0
  287. phoenix/utilities/project.py +2 -2
  288. phoenix/utilities/re.py +49 -0
  289. phoenix/utilities/span_store.py +0 -23
  290. phoenix/utilities/template_formatters.py +99 -0
  291. phoenix/version.py +1 -1
  292. arize_phoenix-3.16.0.dist-info/METADATA +0 -495
  293. arize_phoenix-3.16.0.dist-info/RECORD +0 -178
  294. phoenix/core/project.py +0 -617
  295. phoenix/core/traces.py +0 -100
  296. phoenix/experimental/evals/__init__.py +0 -73
  297. phoenix/experimental/evals/evaluators.py +0 -413
  298. phoenix/experimental/evals/functions/__init__.py +0 -4
  299. phoenix/experimental/evals/functions/classify.py +0 -453
  300. phoenix/experimental/evals/functions/executor.py +0 -353
  301. phoenix/experimental/evals/functions/generate.py +0 -138
  302. phoenix/experimental/evals/functions/processing.py +0 -76
  303. phoenix/experimental/evals/models/__init__.py +0 -14
  304. phoenix/experimental/evals/models/anthropic.py +0 -175
  305. phoenix/experimental/evals/models/base.py +0 -170
  306. phoenix/experimental/evals/models/bedrock.py +0 -221
  307. phoenix/experimental/evals/models/litellm.py +0 -134
  308. phoenix/experimental/evals/models/openai.py +0 -448
  309. phoenix/experimental/evals/models/rate_limiters.py +0 -246
  310. phoenix/experimental/evals/models/vertex.py +0 -173
  311. phoenix/experimental/evals/models/vertexai.py +0 -186
  312. phoenix/experimental/evals/retrievals.py +0 -96
  313. phoenix/experimental/evals/templates/__init__.py +0 -50
  314. phoenix/experimental/evals/templates/default_templates.py +0 -472
  315. phoenix/experimental/evals/templates/template.py +0 -195
  316. phoenix/experimental/evals/utils/__init__.py +0 -172
  317. phoenix/experimental/evals/utils/threads.py +0 -27
  318. phoenix/server/api/helpers.py +0 -11
  319. phoenix/server/api/routers/evaluation_handler.py +0 -109
  320. phoenix/server/api/routers/span_handler.py +0 -70
  321. phoenix/server/api/routers/trace_handler.py +0 -60
  322. phoenix/server/api/types/DatasetRole.py +0 -23
  323. phoenix/server/static/index.css +0 -6
  324. phoenix/server/static/index.js +0 -7447
  325. phoenix/storage/span_store/__init__.py +0 -23
  326. phoenix/storage/span_store/text_file.py +0 -85
  327. phoenix/trace/dsl/missing.py +0 -60
  328. phoenix/trace/langchain/__init__.py +0 -3
  329. phoenix/trace/langchain/instrumentor.py +0 -35
  330. phoenix/trace/llama_index/__init__.py +0 -3
  331. phoenix/trace/llama_index/callback.py +0 -102
  332. phoenix/trace/openai/__init__.py +0 -3
  333. phoenix/trace/openai/instrumentor.py +0 -30
  334. {arize_phoenix-3.16.0.dist-info → arize_phoenix-7.7.0.dist-info}/licenses/IP_NOTICE +0 -0
  335. {arize_phoenix-3.16.0.dist-info → arize_phoenix-7.7.0.dist-info}/licenses/LICENSE +0 -0
  336. /phoenix/{datasets → db/insertion}/__init__.py +0 -0
  337. /phoenix/{experimental → db/migrations}/__init__.py +0 -0
  338. /phoenix/{storage → db/migrations/data_migration_scripts}/__init__.py +0 -0
@@ -1,195 +0,0 @@
1
- import re
2
- from dataclasses import dataclass
3
- from typing import Callable, List, Mapping, Optional, Tuple, Union
4
-
5
- import pandas as pd
6
-
7
- from phoenix.exceptions import PhoenixException
8
- from phoenix.experimental.evals.utils import NOT_PARSABLE
9
-
10
- DEFAULT_START_DELIM = "{"
11
- DEFAULT_END_DELIM = "}"
12
-
13
-
14
- @dataclass
15
- class PromptOptions:
16
- provide_explanation: bool = False
17
-
18
-
19
- class InvalidClassificationTemplateError(PhoenixException):
20
- pass
21
-
22
-
23
- class PromptTemplate:
24
- template: str
25
- variables: List[str]
26
-
27
- def __init__(
28
- self,
29
- template: str,
30
- delimiters: Tuple[str, str] = (DEFAULT_START_DELIM, DEFAULT_END_DELIM),
31
- ):
32
- self.template = template
33
- self._start_delim, self._end_delim = delimiters
34
- self.variables = self._parse_variables(self.template)
35
-
36
- def prompt(self, options: Optional[PromptOptions] = None) -> str:
37
- return self.template
38
-
39
- def format(
40
- self,
41
- variable_values: Mapping[str, Union[bool, int, float, str]],
42
- options: Optional[PromptOptions] = None,
43
- ) -> str:
44
- prompt = self.prompt(options)
45
- for variable_name in self.variables:
46
- prompt = prompt.replace(
47
- self._start_delim + variable_name + self._end_delim,
48
- str(variable_values[variable_name]),
49
- )
50
- return prompt
51
-
52
- def _parse_variables(self, text: str) -> List[str]:
53
- pattern = re.escape(self._start_delim) + "(.*?)" + re.escape(self._end_delim)
54
- variables = re.findall(pattern, text)
55
- return variables
56
-
57
-
58
- class ClassificationTemplate(PromptTemplate):
59
- def __init__(
60
- self,
61
- rails: List[str],
62
- template: str,
63
- explanation_template: Optional[str] = None,
64
- explanation_label_parser: Optional[Callable[[str], str]] = None,
65
- delimiters: Tuple[str, str] = (DEFAULT_START_DELIM, DEFAULT_END_DELIM),
66
- scores: Optional[List[float]] = None,
67
- ):
68
- if scores is not None and len(rails) != len(scores):
69
- raise InvalidClassificationTemplateError(
70
- "If scores are provided, each rail must have one and only one score "
71
- "(i.e., the length of both lists must be the same)."
72
- )
73
- self.rails = rails
74
- self.template = template
75
- self.explanation_template = explanation_template
76
- self.explanation_label_parser = explanation_label_parser
77
- self._start_delim, self._end_delim = delimiters
78
- self.variables: List[str] = []
79
- for text in [template, explanation_template]:
80
- if text is not None:
81
- self.variables += self._parse_variables(text)
82
- self._scores = scores
83
-
84
- def __repr__(self) -> str:
85
- return self.template
86
-
87
- def prompt(self, options: Optional[PromptOptions] = None) -> str:
88
- if options is None:
89
- return self.template
90
-
91
- if options.provide_explanation and self.explanation_template:
92
- return self.explanation_template
93
- else:
94
- return self.template
95
-
96
- def extract_label_from_explanation(self, raw_string: str) -> str:
97
- if parser := self.explanation_label_parser:
98
- return parser(raw_string)
99
- return parse_label_from_chain_of_thought_response(raw_string)
100
-
101
- def score(self, rail: str) -> float:
102
- if self._scores is None:
103
- return 0.0
104
- try:
105
- return self._scores[self.rails.index(rail)]
106
- except (IndexError, ValueError):
107
- return 0.0
108
-
109
-
110
- def parse_label_from_chain_of_thought_response(raw_string: str) -> str:
111
- label_delimiter = r"\W*label\W*"
112
- parts = re.split(label_delimiter, raw_string, maxsplit=1, flags=re.IGNORECASE)
113
- if len(parts) == 2:
114
- return parts[1]
115
- return NOT_PARSABLE
116
-
117
-
118
- def normalize_classification_template(
119
- rails: List[str], template: Union[PromptTemplate, ClassificationTemplate, str]
120
- ) -> ClassificationTemplate:
121
- """
122
- Normalizes a template to a ClassificationTemplate object.
123
- Args:
124
- template (Union[ClassificationTemplate, str]): The template to be normalized.
125
- Returns:
126
- ClassificationTemplate: The normalized template.
127
- """
128
- if isinstance(template, ClassificationTemplate):
129
- return template
130
-
131
- if isinstance(template, PromptTemplate):
132
- return ClassificationTemplate(rails=rails, template=template.template)
133
-
134
- if isinstance(template, str):
135
- return ClassificationTemplate(rails=rails, template=template)
136
-
137
- raise TypeError(
138
- "Invalid type for argument `template`. Expected a string or ClassificationTemplate "
139
- f"but found {type(template)}."
140
- )
141
-
142
-
143
- def normalize_prompt_template(template: Union[PromptTemplate, str]) -> PromptTemplate:
144
- """
145
- Normalizes a template to a PromptTemplate object.
146
- Args:
147
- template (Union[PromptTemplate, str]): The template to be normalized.
148
- Returns:
149
- PromptTemplate: The normalized template.
150
- """
151
- if isinstance(template, PromptTemplate):
152
- return template
153
-
154
- if isinstance(template, str):
155
- return PromptTemplate(template=template)
156
-
157
- raise TypeError(
158
- "Invalid type for argument `template`. Expected a string or PromptTemplate "
159
- f"but found {type(template)}."
160
- )
161
-
162
-
163
- def map_template(
164
- dataframe: pd.DataFrame,
165
- template: PromptTemplate,
166
- options: Optional[PromptOptions] = None,
167
- ) -> "pd.Series[str]":
168
- """
169
- Maps over a dataframe to construct a list of prompts from a template and a dataframe.
170
- """
171
- # Was considering to construct the prompts and generate answers concurrently. However,
172
- # if there's errors in the prompt construction it could interrupt the process and we
173
- # would've used API credits for nothing. We could solve this problem by streaming the
174
- # answers so that, if there is an error, we keep the answers obtained up to that point.
175
- # These are out of scope for M0, but good to keep in mind and consider for the future.
176
- prompt_options: PromptOptions = PromptOptions() if options is None else options
177
-
178
- try:
179
- prompts = dataframe.apply(
180
- lambda row: template.format(
181
- variable_values={var_name: row[var_name] for var_name in template.variables},
182
- options=prompt_options,
183
- ),
184
- axis=1,
185
- )
186
- return prompts
187
- except KeyError as e:
188
- raise RuntimeError(
189
- f"Error while constructing the prompts from the template and dataframe. "
190
- f"The template variable {e} is not found as a column in the dataframe."
191
- )
192
- except Exception as e:
193
- raise RuntimeError(
194
- f"Error while constructing the prompts from the template and dataframe variables: {e}."
195
- )
@@ -1,172 +0,0 @@
1
- import json
2
- from io import BytesIO
3
- from typing import Any, Dict, List, Optional, Tuple
4
- from urllib.error import HTTPError
5
- from urllib.request import urlopen
6
- from zipfile import ZipFile
7
-
8
- import pandas as pd
9
-
10
- from phoenix.utilities.logging import printif
11
-
12
- # Rather than returning None, we return this string to indicate that the LLM output could not be
13
- # parsed.
14
- # This is useful for debugging as well as to just treat the output as a non-parsable category
15
- NOT_PARSABLE = "NOT_PARSABLE"
16
-
17
- # values in the default openai function call,
18
- # defined here only to prevent typos
19
- _RESPONSE = "response"
20
- _EXPLANATION = "explanation"
21
- _FUNCTION_NAME = "record_response"
22
-
23
-
24
- def download_benchmark_dataset(task: str, dataset_name: str) -> pd.DataFrame:
25
- """Downloads an Arize evals benchmark dataset as a pandas dataframe.
26
-
27
- Args:
28
- task (str): Task to be performed.
29
- dataset_name (str): Name of the dataset.
30
-
31
- Returns:
32
- pandas.DataFrame: A pandas dataframe containing the data.
33
- """
34
- jsonl_file_name = f"{dataset_name}.jsonl"
35
- url = f"http://storage.googleapis.com/arize-phoenix-assets/evals/{task}/{jsonl_file_name}.zip"
36
- try:
37
- with urlopen(url) as response:
38
- zip_byte_stream = BytesIO(response.read())
39
- with ZipFile(zip_byte_stream) as zip_file:
40
- with zip_file.open(jsonl_file_name) as jsonl_file:
41
- return pd.DataFrame(map(json.loads, jsonl_file.readlines()))
42
- except HTTPError:
43
- raise ValueError(f'Dataset "{dataset_name}" for "{task}" task does not exist.')
44
-
45
-
46
- def get_tqdm_progress_bar_formatter(title: str) -> str:
47
- """Returns a progress bar formatter for use with tqdm.
48
-
49
- Args:
50
- title (str): The title of the progress bar, displayed as a prefix.
51
-
52
- Returns:
53
- str: A formatter to be passed to the bar_format argument of tqdm.
54
- """
55
- return (
56
- title + " |{bar}| {n_fmt}/{total_fmt} ({percentage:3.1f}%) "
57
- "| ⏳ {elapsed}<{remaining} | {rate_fmt}{postfix}"
58
- )
59
-
60
-
61
- def snap_to_rail(raw_string: Optional[str], rails: List[str], verbose: bool = False) -> str:
62
- """
63
- Snaps a string to the nearest rail, or returns None if the string cannot be
64
- snapped to a rail.
65
-
66
- Args:
67
- raw_string (str): An input to be snapped to a rail.
68
-
69
- rails (List[str]): The target set of strings to snap to.
70
-
71
- Returns:
72
- str: A string from the rails argument or "UNPARSABLE" if the input
73
- string could not be snapped.
74
- """
75
- if not raw_string:
76
- return NOT_PARSABLE
77
- snap_string = raw_string.lower()
78
- rails = list(set(rail.lower() for rail in rails))
79
- rails.sort(key=len, reverse=True)
80
- found_rails = set()
81
- for rail in rails:
82
- if rail in snap_string:
83
- found_rails.add(rail)
84
- snap_string = snap_string.replace(rail, "")
85
- if len(found_rails) != 1:
86
- printif(verbose, f"- Cannot snap {repr(raw_string)} to rails")
87
- return NOT_PARSABLE
88
- rail = list(found_rails)[0]
89
- printif(verbose, f"- Snapped {repr(raw_string)} to rail: {rail}")
90
- return rail
91
-
92
-
93
- def parse_openai_function_call(raw_output: str) -> Tuple[str, Optional[str]]:
94
- """
95
- Parses the output of an OpenAI function call.
96
-
97
- Args:
98
- raw_output (str): The raw output of an OpenAI function call.
99
-
100
- Returns:
101
- Tuple[str, Optional[str]]: A tuple of the unrailed label and an optional
102
- explanation.
103
- """
104
- try:
105
- function_arguments = json.loads(raw_output, strict=False)
106
- unrailed_label = function_arguments.get(_RESPONSE, "")
107
- explanation = function_arguments.get(_EXPLANATION)
108
- except json.JSONDecodeError:
109
- unrailed_label = raw_output
110
- explanation = None
111
- return unrailed_label, explanation
112
-
113
-
114
- def openai_function_call_kwargs(rails: List[str], provide_explanation: bool) -> Dict[str, Any]:
115
- """
116
- Returns keyword arguments needed to invoke an OpenAI model with function
117
- calling for classification.
118
-
119
- Args:
120
- rails (List[str]): The rails to snap the output to.
121
-
122
- provide_explanation (bool): Whether to provide an explanation.
123
-
124
- Returns:
125
- Dict[str, Any]: A dictionary containing function call arguments.
126
- """
127
- openai_function = _default_openai_function(rails, provide_explanation)
128
- return {
129
- "functions": [openai_function],
130
- "function_call": {"name": openai_function["name"]},
131
- }
132
-
133
-
134
- def _default_openai_function(
135
- rails: List[str],
136
- with_explanation: bool = False,
137
- ) -> Dict[str, Any]:
138
- """
139
- Returns a default OpenAI function call for classification.
140
-
141
- Args:
142
- rails (List[str]): A list of rails to snap the output to.
143
-
144
- with_explanation (bool, optional): Whether to include an explanation.
145
-
146
- Returns:
147
- Dict[str, Any]: A JSON schema object advertising a function to record
148
- the result of the LLM's classification.
149
- """
150
- properties = {
151
- **(
152
- {
153
- _EXPLANATION: {
154
- "type": "string",
155
- "description": "Explanation of the reasoning for your response.",
156
- },
157
- }
158
- if with_explanation
159
- else {}
160
- ),
161
- _RESPONSE: {"type": "string", "description": "Your response.", "enum": rails},
162
- }
163
- required = [*([_EXPLANATION] if with_explanation else []), _RESPONSE]
164
- return {
165
- "name": _FUNCTION_NAME,
166
- "description": "A function to record your response.",
167
- "parameters": {
168
- "type": "object",
169
- "properties": properties,
170
- "required": required,
171
- },
172
- }
@@ -1,27 +0,0 @@
1
- """High-level support for working with threads in asyncio
2
- Directly copied from: https://github.com/python/cpython/blob/main/Lib/asyncio/threads.py#L12
3
- since this helper function 'to_thread' is not available in python<3.9
4
- """
5
-
6
- import contextvars
7
- import functools
8
- from asyncio import events
9
- from typing import Any
10
-
11
- __all__ = ("to_thread",)
12
-
13
-
14
- async def to_thread(func, /, *args, **kwargs) -> Any: # type:ignore
15
- """Asynchronously run function *func* in a separate thread.
16
-
17
- Any *args and **kwargs supplied for this function are directly passed
18
- to *func*. Also, the current :class:`contextvars.Context` is propagated,
19
- allowing context variables from the main thread to be accessed in the
20
- separate thread.
21
-
22
- Return a coroutine that can be awaited to get the eventual result of *func*.
23
- """
24
- loop = events.get_running_loop()
25
- ctx = contextvars.copy_context()
26
- func_call = functools.partial(ctx.run, func, *args, **kwargs)
27
- return await loop.run_in_executor(None, func_call)
@@ -1,11 +0,0 @@
1
- from typing import Iterable, List, Optional, TypeVar
2
-
3
- T = TypeVar("T")
4
-
5
-
6
- def ensure_list(obj: Optional[Iterable[T]]) -> List[T]:
7
- if isinstance(obj, List):
8
- return obj
9
- if isinstance(obj, Iterable):
10
- return list(obj)
11
- return []
@@ -1,109 +0,0 @@
1
- import asyncio
2
- import gzip
3
- from typing import AsyncIterator
4
-
5
- import pyarrow as pa
6
- from google.protobuf.message import DecodeError
7
- from starlette.background import BackgroundTask
8
- from starlette.endpoints import HTTPEndpoint
9
- from starlette.requests import Request
10
- from starlette.responses import Response, StreamingResponse
11
- from starlette.status import (
12
- HTTP_404_NOT_FOUND,
13
- HTTP_415_UNSUPPORTED_MEDIA_TYPE,
14
- HTTP_422_UNPROCESSABLE_ENTITY,
15
- )
16
-
17
- import phoenix.trace.v1 as pb
18
- from phoenix.config import DEFAULT_PROJECT_NAME
19
- from phoenix.core.traces import Traces
20
- from phoenix.server.api.routers.utils import table_to_bytes
21
- from phoenix.session.evaluation import encode_evaluations
22
- from phoenix.trace.span_evaluations import Evaluations
23
-
24
-
25
- class EvaluationHandler(HTTPEndpoint):
26
- traces: Traces
27
-
28
- async def post(self, request: Request) -> Response:
29
- content_type = request.headers.get("content-type")
30
- project_name = request.headers.get("project-name", DEFAULT_PROJECT_NAME)
31
- if content_type == "application/x-pandas-arrow":
32
- return await self._process_pyarrow(request)
33
- if content_type != "application/x-protobuf":
34
- return Response(
35
- content="Unsupported content type",
36
- status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE,
37
- )
38
- body = await request.body()
39
- content_encoding = request.headers.get("content-encoding")
40
- if content_encoding == "gzip":
41
- body = gzip.decompress(body)
42
- elif content_encoding:
43
- return Response(
44
- content="Unsupported content encoding",
45
- status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE,
46
- )
47
- evaluation = pb.Evaluation()
48
- try:
49
- evaluation.ParseFromString(body)
50
- except DecodeError:
51
- return Response(
52
- content="Request body is invalid",
53
- status_code=HTTP_422_UNPROCESSABLE_ENTITY,
54
- )
55
- self.traces.put(evaluation, project_name=project_name)
56
- return Response()
57
-
58
- async def get(self, request: Request) -> Response:
59
- payload = await request.json()
60
- project_name = payload.pop("project_name", None) or DEFAULT_PROJECT_NAME
61
- project = self.traces.get_project(project_name)
62
- if not project:
63
- return Response(status_code=HTTP_404_NOT_FOUND)
64
- loop = asyncio.get_running_loop()
65
- results = await loop.run_in_executor(
66
- None,
67
- project.export_evaluations,
68
- )
69
- if not results:
70
- return Response(status_code=HTTP_404_NOT_FOUND)
71
-
72
- async def content() -> AsyncIterator[bytes]:
73
- for result in results:
74
- yield await loop.run_in_executor(
75
- None,
76
- lambda: table_to_bytes(result.to_pyarrow_table()),
77
- )
78
-
79
- return StreamingResponse(
80
- content=content(),
81
- media_type="application/x-pandas-arrow",
82
- )
83
-
84
- async def _process_pyarrow(self, request: Request) -> Response:
85
- body = await request.body()
86
- try:
87
- reader = pa.ipc.open_stream(body)
88
- except pa.ArrowInvalid:
89
- return Response(
90
- content="Request body is not valid pyarrow",
91
- status_code=HTTP_422_UNPROCESSABLE_ENTITY,
92
- )
93
- try:
94
- evaluations = Evaluations.from_pyarrow_reader(reader)
95
- except Exception:
96
- return Response(
97
- content="Invalid data in request body",
98
- status_code=HTTP_422_UNPROCESSABLE_ENTITY,
99
- )
100
- return Response(
101
- background=BackgroundTask(
102
- self._add_evaluations,
103
- evaluations,
104
- )
105
- )
106
-
107
- async def _add_evaluations(self, evaluations: Evaluations) -> None:
108
- for evaluation in encode_evaluations(evaluations):
109
- self.traces.put(evaluation)
@@ -1,70 +0,0 @@
1
- import asyncio
2
- from functools import partial
3
- from typing import AsyncIterator
4
-
5
- from starlette.endpoints import HTTPEndpoint
6
- from starlette.requests import Request
7
- from starlette.responses import Response, StreamingResponse
8
- from starlette.status import HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY
9
-
10
- from phoenix.config import DEFAULT_PROJECT_NAME
11
- from phoenix.core.traces import Traces
12
- from phoenix.server.api.routers.utils import df_to_bytes, from_iso_format
13
- from phoenix.trace.dsl import SpanQuery
14
- from phoenix.utilities import query_spans
15
-
16
-
17
- class SpanHandler(HTTPEndpoint):
18
- traces: Traces
19
-
20
- async def get(self, request: Request) -> Response:
21
- payload = await request.json()
22
- queries = payload.pop("queries", [])
23
- project_name = payload.pop("project_name", None) or DEFAULT_PROJECT_NAME
24
- if not (project := self.traces.get_project(project_name)):
25
- return Response(status_code=HTTP_404_NOT_FOUND)
26
- loop = asyncio.get_running_loop()
27
- valid_eval_names = (
28
- await loop.run_in_executor(
29
- None,
30
- project.get_span_evaluation_names,
31
- )
32
- if project
33
- else ()
34
- )
35
- try:
36
- span_queries = [
37
- SpanQuery.from_dict(
38
- query,
39
- evals=project,
40
- valid_eval_names=valid_eval_names,
41
- )
42
- for query in queries
43
- ]
44
- except Exception as e:
45
- return Response(
46
- status_code=HTTP_422_UNPROCESSABLE_ENTITY,
47
- content=f"Invalid query: {e}",
48
- )
49
- results = await loop.run_in_executor(
50
- None,
51
- partial(
52
- query_spans,
53
- project,
54
- *span_queries,
55
- start_time=from_iso_format(payload.get("start_time")),
56
- stop_time=from_iso_format(payload.get("stop_time")),
57
- root_spans_only=payload.get("root_spans_only"),
58
- ),
59
- )
60
- if not results:
61
- return Response(status_code=HTTP_404_NOT_FOUND)
62
-
63
- async def content() -> AsyncIterator[bytes]:
64
- for result in results:
65
- yield df_to_bytes(result)
66
-
67
- return StreamingResponse(
68
- content=content(),
69
- media_type="application/x-pandas-arrow",
70
- )
@@ -1,60 +0,0 @@
1
- import asyncio
2
- import gzip
3
- import zlib
4
- from typing import Optional
5
-
6
- from google.protobuf.message import DecodeError
7
- from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
8
- ExportTraceServiceRequest,
9
- )
10
- from opentelemetry.proto.trace.v1.trace_pb2 import TracesData
11
- from starlette.endpoints import HTTPEndpoint
12
- from starlette.requests import Request
13
- from starlette.responses import Response
14
- from starlette.status import HTTP_415_UNSUPPORTED_MEDIA_TYPE, HTTP_422_UNPROCESSABLE_ENTITY
15
-
16
- from phoenix.core.traces import Traces
17
- from phoenix.storage.span_store import SpanStore
18
- from phoenix.trace.otel import decode
19
- from phoenix.utilities.project import get_project_name
20
-
21
-
22
- class TraceHandler(HTTPEndpoint):
23
- traces: Traces
24
- store: Optional[SpanStore]
25
-
26
- async def post(self, request: Request) -> Response:
27
- content_type = request.headers.get("content-type")
28
- if content_type != "application/x-protobuf":
29
- return Response(
30
- content=f"Unsupported content type: {content_type}",
31
- status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE,
32
- )
33
- content_encoding = request.headers.get("content-encoding")
34
- if content_encoding and content_encoding not in ("gzip", "deflate"):
35
- return Response(
36
- content=f"Unsupported content encoding: {content_encoding}",
37
- status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE,
38
- )
39
- body = await request.body()
40
- if content_encoding == "gzip":
41
- body = gzip.decompress(body)
42
- elif content_encoding == "deflate":
43
- body = zlib.decompress(body)
44
- req = ExportTraceServiceRequest()
45
- try:
46
- req.ParseFromString(body)
47
- except DecodeError:
48
- return Response(
49
- content="Request body is invalid ExportTraceServiceRequest",
50
- status_code=HTTP_422_UNPROCESSABLE_ENTITY,
51
- )
52
- if self.store:
53
- self.store.save(TracesData(resource_spans=req.resource_spans))
54
- for resource_spans in req.resource_spans:
55
- project_name = get_project_name(resource_spans.resource.attributes)
56
- for scope_span in resource_spans.scope_spans:
57
- for span in scope_span.spans:
58
- self.traces.put(decode(span), project_name=project_name)
59
- await asyncio.sleep(0)
60
- return Response()
@@ -1,23 +0,0 @@
1
- from enum import Enum
2
- from typing import Dict, Union
3
-
4
- import strawberry
5
-
6
- from phoenix.core.model_schema import PRIMARY, REFERENCE
7
-
8
-
9
- @strawberry.enum
10
- class DatasetRole(Enum):
11
- primary = PRIMARY
12
- reference = REFERENCE
13
-
14
-
15
- class AncillaryDatasetRole(Enum):
16
- corpus = "DatasetRole.CORPUS"
17
-
18
-
19
- STR_TO_DATASET_ROLE: Dict[str, Union[DatasetRole, AncillaryDatasetRole]] = {
20
- str(DatasetRole.primary.value): DatasetRole.primary,
21
- str(DatasetRole.reference.value): DatasetRole.reference,
22
- str(AncillaryDatasetRole.corpus.value): AncillaryDatasetRole.corpus,
23
- }
@@ -1,6 +0,0 @@
1
- html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}main{display:block}h1{font-size:2em;margin:.67em 0}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}img{border-style:none}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button::-moz-focus-inner,[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner{border-style:none;padding:0}button:-moz-focusring,[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details{display:block}summary{display:list-item}template{display:none}[hidden]{display:none}
2
- /*! Bundled license information:
3
-
4
- normalize.css/normalize.css:
5
- (*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css *)
6
- */