rapidata 2.35.3__py3-none-any.whl → 2.36.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rapidata might be problematic. Click here for more details.
- rapidata/__init__.py +1 -1
- rapidata/api_client/__init__.py +21 -3
- rapidata/api_client/api/__init__.py +1 -0
- rapidata/api_client/api/benchmark_api.py +294 -0
- rapidata/api_client/api/campaign_api.py +268 -0
- rapidata/api_client/api/customer_rapid_api.py +247 -0
- rapidata/api_client/api/pipeline_api.py +0 -873
- rapidata/api_client/api/sample_api.py +299 -0
- rapidata/api_client/models/__init__.py +20 -3
- rapidata/api_client/models/and_filter.py +121 -0
- rapidata/api_client/models/and_filter_filters_inner.py +268 -0
- rapidata/api_client/models/boost_mode.py +37 -0
- rapidata/api_client/models/boost_query_result.py +10 -1
- rapidata/api_client/models/campaign_filter.py +98 -0
- rapidata/api_client/models/change_boost_model.py +89 -0
- rapidata/api_client/models/compare_rapid_blueprint.py +5 -3
- rapidata/api_client/models/compare_rapid_blueprint1.py +96 -0
- rapidata/api_client/models/country_filter.py +98 -0
- rapidata/api_client/models/create_leaderboard_model.py +32 -2
- rapidata/api_client/models/demographic_filter.py +100 -0
- rapidata/api_client/models/feature_flag_model.py +4 -4
- rapidata/api_client/models/free_text_payload.py +10 -3
- rapidata/api_client/models/free_text_rapid_blueprint.py +10 -3
- rapidata/api_client/models/get_compare_ab_summary_result.py +4 -2
- rapidata/api_client/models/get_leaderboard_by_id_result.py +29 -2
- rapidata/api_client/models/get_public_responses_result.py +95 -0
- rapidata/api_client/models/get_sample_by_id_result.py +126 -0
- rapidata/api_client/models/language_filter.py +98 -0
- rapidata/api_client/models/leaderboard_query_result.py +29 -2
- rapidata/api_client/models/new_user_filter.py +96 -0
- rapidata/api_client/models/not_filter.py +117 -0
- rapidata/api_client/models/or_filter.py +121 -0
- rapidata/api_client/models/public_rapid_response.py +112 -0
- rapidata/api_client/models/response_count_filter.py +109 -0
- rapidata/api_client/models/sample_by_identifier.py +126 -0
- rapidata/api_client/models/sample_by_identifier_paged_result.py +105 -0
- rapidata/api_client/models/simple_workflow_config_blueprint.py +37 -23
- rapidata/api_client/models/user_score_filter.py +102 -0
- rapidata/api_client/models/user_state.py +38 -0
- rapidata/api_client/models/user_state_filter.py +101 -0
- rapidata/api_client_README.md +24 -6
- rapidata/rapidata_client/benchmark/rapidata_benchmark.py +26 -2
- rapidata/rapidata_client/order/rapidata_order_manager.py +298 -219
- rapidata/rapidata_client/workflow/_compare_workflow.py +7 -2
- {rapidata-2.35.3.dist-info → rapidata-2.36.0.dist-info}/METADATA +1 -1
- {rapidata-2.35.3.dist-info → rapidata-2.36.0.dist-info}/RECORD +48 -26
- {rapidata-2.35.3.dist-info → rapidata-2.36.0.dist-info}/LICENSE +0 -0
- {rapidata-2.35.3.dist-info → rapidata-2.36.0.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Rapidata.Dataset
|
|
5
|
+
|
|
6
|
+
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
|
|
7
|
+
|
|
8
|
+
The version of the OpenAPI document: v1
|
|
9
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
10
|
+
|
|
11
|
+
Do not edit the class manually.
|
|
12
|
+
""" # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
import pprint
|
|
17
|
+
import re # noqa: F401
|
|
18
|
+
import json
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator
|
|
21
|
+
from typing import Any, ClassVar, Dict, List, Optional
|
|
22
|
+
from rapidata.api_client.models.user_state import UserState
|
|
23
|
+
from typing import Optional, Set
|
|
24
|
+
from typing_extensions import Self
|
|
25
|
+
|
|
26
|
+
class UserStateFilter(BaseModel):
|
|
27
|
+
"""
|
|
28
|
+
UserStateFilter
|
|
29
|
+
""" # noqa: E501
|
|
30
|
+
t: StrictStr = Field(description="Discriminator value for UserStateFilter", alias="_t")
|
|
31
|
+
states: List[UserState]
|
|
32
|
+
execution_order: Optional[StrictInt] = Field(default=None, alias="executionOrder")
|
|
33
|
+
__properties: ClassVar[List[str]] = ["_t", "states", "executionOrder"]
|
|
34
|
+
|
|
35
|
+
@field_validator('t')
|
|
36
|
+
def t_validate_enum(cls, value):
|
|
37
|
+
"""Validates the enum"""
|
|
38
|
+
if value not in set(['UserStateFilter']):
|
|
39
|
+
raise ValueError("must be one of enum values ('UserStateFilter')")
|
|
40
|
+
return value
|
|
41
|
+
|
|
42
|
+
model_config = ConfigDict(
|
|
43
|
+
populate_by_name=True,
|
|
44
|
+
validate_assignment=True,
|
|
45
|
+
protected_namespaces=(),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def to_str(self) -> str:
|
|
50
|
+
"""Returns the string representation of the model using alias"""
|
|
51
|
+
return pprint.pformat(self.model_dump(by_alias=True))
|
|
52
|
+
|
|
53
|
+
def to_json(self) -> str:
|
|
54
|
+
"""Returns the JSON representation of the model using alias"""
|
|
55
|
+
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
|
|
56
|
+
return json.dumps(self.to_dict())
|
|
57
|
+
|
|
58
|
+
@classmethod
|
|
59
|
+
def from_json(cls, json_str: str) -> Optional[Self]:
|
|
60
|
+
"""Create an instance of UserStateFilter from a JSON string"""
|
|
61
|
+
return cls.from_dict(json.loads(json_str))
|
|
62
|
+
|
|
63
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
64
|
+
"""Return the dictionary representation of the model using alias.
|
|
65
|
+
|
|
66
|
+
This has the following differences from calling pydantic's
|
|
67
|
+
`self.model_dump(by_alias=True)`:
|
|
68
|
+
|
|
69
|
+
* `None` is only added to the output dict for nullable fields that
|
|
70
|
+
were set at model initialization. Other fields with value `None`
|
|
71
|
+
are ignored.
|
|
72
|
+
* OpenAPI `readOnly` fields are excluded.
|
|
73
|
+
"""
|
|
74
|
+
excluded_fields: Set[str] = set([
|
|
75
|
+
"execution_order",
|
|
76
|
+
])
|
|
77
|
+
|
|
78
|
+
_dict = self.model_dump(
|
|
79
|
+
by_alias=True,
|
|
80
|
+
exclude=excluded_fields,
|
|
81
|
+
exclude_none=True,
|
|
82
|
+
)
|
|
83
|
+
return _dict
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
|
|
87
|
+
"""Create an instance of UserStateFilter from a dict"""
|
|
88
|
+
if obj is None:
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
if not isinstance(obj, dict):
|
|
92
|
+
return cls.model_validate(obj)
|
|
93
|
+
|
|
94
|
+
_obj = cls.model_validate({
|
|
95
|
+
"_t": obj.get("_t") if obj.get("_t") is not None else 'UserStateFilter',
|
|
96
|
+
"states": obj.get("states"),
|
|
97
|
+
"executionOrder": obj.get("executionOrder")
|
|
98
|
+
})
|
|
99
|
+
return _obj
|
|
100
|
+
|
|
101
|
+
|
rapidata/api_client_README.md
CHANGED
|
@@ -81,9 +81,11 @@ Class | Method | HTTP request | Description
|
|
|
81
81
|
*BenchmarkApi* | [**benchmark_benchmark_id_participants_post**](rapidata/api_client/docs/BenchmarkApi.md#benchmark_benchmark_id_participants_post) | **POST** /benchmark/{benchmarkId}/participants | Creates a participant in a benchmark.
|
|
82
82
|
*BenchmarkApi* | [**benchmark_benchmark_id_prompt_post**](rapidata/api_client/docs/BenchmarkApi.md#benchmark_benchmark_id_prompt_post) | **POST** /benchmark/{benchmarkId}/prompt | Adds a new prompt to a benchmark.
|
|
83
83
|
*BenchmarkApi* | [**benchmark_benchmark_id_prompts_get**](rapidata/api_client/docs/BenchmarkApi.md#benchmark_benchmark_id_prompts_get) | **GET** /benchmark/{benchmarkId}/prompts | Returns the paged prompts of a benchmark by its ID.
|
|
84
|
+
*BenchmarkApi* | [**benchmark_benchmark_id_samples_identifier_get**](rapidata/api_client/docs/BenchmarkApi.md#benchmark_benchmark_id_samples_identifier_get) | **GET** /benchmark/{benchmarkId}/samples/{identifier} | Returns the paged prompts of a benchmark by its ID.
|
|
84
85
|
*BenchmarkApi* | [**benchmark_benchmark_id_tags_get**](rapidata/api_client/docs/BenchmarkApi.md#benchmark_benchmark_id_tags_get) | **GET** /benchmark/{benchmarkId}/tags | Query all tags within a benchmark
|
|
85
86
|
*BenchmarkApi* | [**benchmark_post**](rapidata/api_client/docs/BenchmarkApi.md#benchmark_post) | **POST** /benchmark | Creates a benchmark
|
|
86
87
|
*BenchmarkApi* | [**benchmarks_get**](rapidata/api_client/docs/BenchmarkApi.md#benchmarks_get) | **GET** /benchmarks | Queries all benchmarks of the user.
|
|
88
|
+
*CampaignApi* | [**campaign_boost_put**](rapidata/api_client/docs/CampaignApi.md#campaign_boost_put) | **PUT** /campaign/boost | Updates the boost with manual boosts.
|
|
87
89
|
*CampaignApi* | [**campaign_boost_status_get**](rapidata/api_client/docs/CampaignApi.md#campaign_boost_status_get) | **GET** /campaign/boost/status | Gets the status of the boost.
|
|
88
90
|
*CampaignApi* | [**campaign_campaign_id_pause_post**](rapidata/api_client/docs/CampaignApi.md#campaign_campaign_id_pause_post) | **POST** /campaign/{campaignId}/pause | Pauses a campaign.
|
|
89
91
|
*CampaignApi* | [**campaign_campaign_id_resume_post**](rapidata/api_client/docs/CampaignApi.md#campaign_campaign_id_resume_post) | **POST** /campaign/{campaignId}/resume | Resumes a campaign.
|
|
@@ -99,6 +101,7 @@ Class | Method | HTTP request | Description
|
|
|
99
101
|
*CompareWorkflowApi* | [**workflow_compare_workflow_id_results_get**](rapidata/api_client/docs/CompareWorkflowApi.md#workflow_compare_workflow_id_results_get) | **GET** /workflow/compare/{workflowId}/results | Get the result overview for a compare workflow.
|
|
100
102
|
*CustomerRapidApi* | [**rapid_correlation_id_validation_potential_get**](rapidata/api_client/docs/CustomerRapidApi.md#rapid_correlation_id_validation_potential_get) | **GET** /rapid/{correlationId}/validation-potential | Queries rapids that are potentially eligible for validation set creation.
|
|
101
103
|
*CustomerRapidApi* | [**rapid_demographic_post**](rapidata/api_client/docs/CustomerRapidApi.md#rapid_demographic_post) | **POST** /rapid/demographic | Creates a new Demographic Rapid.
|
|
104
|
+
*CustomerRapidApi* | [**rapid_global_responses_get**](rapidata/api_client/docs/CustomerRapidApi.md#rapid_global_responses_get) | **GET** /rapid/global-responses | A public endpoint to query the most recent responses globally
|
|
102
105
|
*CustomerRapidApi* | [**rapid_rapid_id_delete**](rapidata/api_client/docs/CustomerRapidApi.md#rapid_rapid_id_delete) | **DELETE** /rapid/{rapidId} | Deletes a rapid.
|
|
103
106
|
*CustomerRapidApi* | [**rapid_rapid_id_responses_get**](rapidata/api_client/docs/CustomerRapidApi.md#rapid_rapid_id_responses_get) | **GET** /rapid/{rapidId}/responses | Gets all responses for a given rapid.
|
|
104
107
|
*CustomerRapidApi* | [**rapid_rapid_id_unflag_post**](rapidata/api_client/docs/CustomerRapidApi.md#rapid_rapid_id_unflag_post) | **POST** /rapid/{rapidId}/unflag | Unflags a flagged rapid. This will add the rapid back to the active labeling pool and prevent it from being flagged again.
|
|
@@ -172,14 +175,12 @@ Class | Method | HTTP request | Description
|
|
|
172
175
|
*ParticipantApi* | [**participants_participant_id_submit_post**](rapidata/api_client/docs/ParticipantApi.md#participants_participant_id_submit_post) | **POST** /participants/{participantId}/submit | Submits a participant to a benchmark.
|
|
173
176
|
*PipelineApi* | [**pipeline_id_workflow_config_artifact_id_put**](rapidata/api_client/docs/PipelineApi.md#pipeline_id_workflow_config_artifact_id_put) | **PUT** /pipeline/{id}/workflow-config/{artifactId} | Updates the workflow configuration for a pipeline.
|
|
174
177
|
*PipelineApi* | [**pipeline_id_workflow_config_put**](rapidata/api_client/docs/PipelineApi.md#pipeline_id_workflow_config_put) | **PUT** /pipeline/{id}/workflow-config | Updates the workflow configuration for a pipeline.
|
|
175
|
-
*PipelineApi* | [**pipeline_pipeline_id_campaign_artifact_id_put**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_campaign_artifact_id_put) | **PUT** /pipeline/{pipelineId}/campaign/{artifactId} | Updates a specific campaign for a pipeline.
|
|
176
|
-
*PipelineApi* | [**pipeline_pipeline_id_campaign_post**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_campaign_post) | **POST** /pipeline/{pipelineId}/campaign | Adds a campaign to a pipeline.
|
|
177
|
-
*PipelineApi* | [**pipeline_pipeline_id_campaign_put**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_campaign_put) | **PUT** /pipeline/{pipelineId}/campaign | Updates a specific campaign for a pipeline.
|
|
178
178
|
*PipelineApi* | [**pipeline_pipeline_id_get**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_get) | **GET** /pipeline/{pipelineId} | Gets a pipeline by its id.
|
|
179
179
|
*PipelineApi* | [**pipeline_pipeline_id_preliminary_download_post**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_preliminary_download_post) | **POST** /pipeline/{pipelineId}/preliminary-download | Initiates a preliminary download of the pipeline.
|
|
180
180
|
*PipelineApi* | [**pipeline_preliminary_download_preliminary_download_id_get**](rapidata/api_client/docs/PipelineApi.md#pipeline_preliminary_download_preliminary_download_id_get) | **GET** /pipeline/preliminary-download/{preliminaryDownloadId} | Gets the preliminary download.
|
|
181
181
|
*PromptApi* | [**benchmark_prompt_prompt_id_tags_put**](rapidata/api_client/docs/PromptApi.md#benchmark_prompt_prompt_id_tags_put) | **PUT** /benchmark-prompt/{promptId}/tags | Updates the tags associated with a prompt.
|
|
182
182
|
*RapidataIdentityAPIApi* | [**root_get**](rapidata/api_client/docs/RapidataIdentityAPIApi.md#root_get) | **GET** / |
|
|
183
|
+
*SampleApi* | [**benchmark_sample_sample_id_get**](rapidata/api_client/docs/SampleApi.md#benchmark_sample_sample_id_get) | **GET** /benchmark-sample/{sampleId} | Gets a sample by its Id.
|
|
183
184
|
*SimpleWorkflowApi* | [**workflow_simple_workflow_id_results_get**](rapidata/api_client/docs/SimpleWorkflowApi.md#workflow_simple_workflow_id_results_get) | **GET** /workflow/simple/{workflowId}/results | Get the result overview for a simple workflow.
|
|
184
185
|
*UserInfoApi* | [**connect_userinfo_get**](rapidata/api_client/docs/UserInfoApi.md#connect_userinfo_get) | **GET** /connect/userinfo | Retrieves information about the authenticated user.
|
|
185
186
|
*UserRapidApi* | [**rapid_rapid_bag_is_valid_get**](rapidata/api_client/docs/UserRapidApi.md#rapid_rapid_bag_is_valid_get) | **GET** /rapid/rapid-bag/is-valid | Validates that the rapids associated with the current user are active.
|
|
@@ -215,8 +216,6 @@ Class | Method | HTTP request | Description
|
|
|
215
216
|
|
|
216
217
|
- [AbTestSelection](rapidata/api_client/docs/AbTestSelection.md)
|
|
217
218
|
- [AbTestSelectionAInner](rapidata/api_client/docs/AbTestSelectionAInner.md)
|
|
218
|
-
- [AddCampaignArtifactResult](rapidata/api_client/docs/AddCampaignArtifactResult.md)
|
|
219
|
-
- [AddCampaignModel](rapidata/api_client/docs/AddCampaignModel.md)
|
|
220
219
|
- [AddUserResponseResult](rapidata/api_client/docs/AddUserResponseResult.md)
|
|
221
220
|
- [AddValidationRapidModel](rapidata/api_client/docs/AddValidationRapidModel.md)
|
|
222
221
|
- [AddValidationRapidModelPayload](rapidata/api_client/docs/AddValidationRapidModelPayload.md)
|
|
@@ -226,6 +225,8 @@ Class | Method | HTTP request | Description
|
|
|
226
225
|
- [AgeGroup](rapidata/api_client/docs/AgeGroup.md)
|
|
227
226
|
- [AgeUserFilterModel](rapidata/api_client/docs/AgeUserFilterModel.md)
|
|
228
227
|
- [AggregatorType](rapidata/api_client/docs/AggregatorType.md)
|
|
228
|
+
- [AndFilter](rapidata/api_client/docs/AndFilter.md)
|
|
229
|
+
- [AndFilterFiltersInner](rapidata/api_client/docs/AndFilterFiltersInner.md)
|
|
229
230
|
- [AndUserFilterModel](rapidata/api_client/docs/AndUserFilterModel.md)
|
|
230
231
|
- [AndUserFilterModelFiltersInner](rapidata/api_client/docs/AndUserFilterModelFiltersInner.md)
|
|
231
232
|
- [AreRapidsActiveResult](rapidata/api_client/docs/AreRapidsActiveResult.md)
|
|
@@ -238,6 +239,7 @@ Class | Method | HTTP request | Description
|
|
|
238
239
|
- [BenchmarkQueryResult](rapidata/api_client/docs/BenchmarkQueryResult.md)
|
|
239
240
|
- [BenchmarkQueryResultPagedResult](rapidata/api_client/docs/BenchmarkQueryResultPagedResult.md)
|
|
240
241
|
- [BoostLeaderboardModel](rapidata/api_client/docs/BoostLeaderboardModel.md)
|
|
242
|
+
- [BoostMode](rapidata/api_client/docs/BoostMode.md)
|
|
241
243
|
- [BoostQueryResult](rapidata/api_client/docs/BoostQueryResult.md)
|
|
242
244
|
- [BoostStatus](rapidata/api_client/docs/BoostStatus.md)
|
|
243
245
|
- [BoundingBoxPayload](rapidata/api_client/docs/BoundingBoxPayload.md)
|
|
@@ -246,11 +248,13 @@ Class | Method | HTTP request | Description
|
|
|
246
248
|
- [BoundingBoxTruth](rapidata/api_client/docs/BoundingBoxTruth.md)
|
|
247
249
|
- [BoxShape](rapidata/api_client/docs/BoxShape.md)
|
|
248
250
|
- [CampaignArtifactModel](rapidata/api_client/docs/CampaignArtifactModel.md)
|
|
251
|
+
- [CampaignFilter](rapidata/api_client/docs/CampaignFilter.md)
|
|
249
252
|
- [CampaignQueryResult](rapidata/api_client/docs/CampaignQueryResult.md)
|
|
250
253
|
- [CampaignQueryResultPagedResult](rapidata/api_client/docs/CampaignQueryResultPagedResult.md)
|
|
251
254
|
- [CampaignStatus](rapidata/api_client/docs/CampaignStatus.md)
|
|
252
255
|
- [CampaignUserFilterModel](rapidata/api_client/docs/CampaignUserFilterModel.md)
|
|
253
256
|
- [CappedSelection](rapidata/api_client/docs/CappedSelection.md)
|
|
257
|
+
- [ChangeBoostModel](rapidata/api_client/docs/ChangeBoostModel.md)
|
|
254
258
|
- [ClassificationMetadata](rapidata/api_client/docs/ClassificationMetadata.md)
|
|
255
259
|
- [ClassificationMetadataFilterConfig](rapidata/api_client/docs/ClassificationMetadataFilterConfig.md)
|
|
256
260
|
- [ClassificationMetadataModel](rapidata/api_client/docs/ClassificationMetadataModel.md)
|
|
@@ -281,6 +285,7 @@ Class | Method | HTTP request | Description
|
|
|
281
285
|
- [CountClassificationMetadataFilterConfig](rapidata/api_client/docs/CountClassificationMetadataFilterConfig.md)
|
|
282
286
|
- [CountMetadata](rapidata/api_client/docs/CountMetadata.md)
|
|
283
287
|
- [CountMetadataModel](rapidata/api_client/docs/CountMetadataModel.md)
|
|
288
|
+
- [CountryFilter](rapidata/api_client/docs/CountryFilter.md)
|
|
284
289
|
- [CountryUserFilterModel](rapidata/api_client/docs/CountryUserFilterModel.md)
|
|
285
290
|
- [CreateBenchmarkModel](rapidata/api_client/docs/CreateBenchmarkModel.md)
|
|
286
291
|
- [CreateBenchmarkParticipantModel](rapidata/api_client/docs/CreateBenchmarkParticipantModel.md)
|
|
@@ -324,6 +329,7 @@ Class | Method | HTTP request | Description
|
|
|
324
329
|
- [DatasetDatasetIdDatapointsPostRequestMetadataInner](rapidata/api_client/docs/DatasetDatasetIdDatapointsPostRequestMetadataInner.md)
|
|
325
330
|
- [DatasetEvaluationStepModel](rapidata/api_client/docs/DatasetEvaluationStepModel.md)
|
|
326
331
|
- [Demographic](rapidata/api_client/docs/Demographic.md)
|
|
332
|
+
- [DemographicFilter](rapidata/api_client/docs/DemographicFilter.md)
|
|
327
333
|
- [DemographicMetadataModel](rapidata/api_client/docs/DemographicMetadataModel.md)
|
|
328
334
|
- [DemographicSelection](rapidata/api_client/docs/DemographicSelection.md)
|
|
329
335
|
- [DynamicClientRegistrationRequest](rapidata/api_client/docs/DynamicClientRegistrationRequest.md)
|
|
@@ -375,8 +381,10 @@ Class | Method | HTTP request | Description
|
|
|
375
381
|
- [GetPipelineByIdResult](rapidata/api_client/docs/GetPipelineByIdResult.md)
|
|
376
382
|
- [GetPipelineByIdResultArtifactsValue](rapidata/api_client/docs/GetPipelineByIdResultArtifactsValue.md)
|
|
377
383
|
- [GetPublicOrdersResult](rapidata/api_client/docs/GetPublicOrdersResult.md)
|
|
384
|
+
- [GetPublicResponsesResult](rapidata/api_client/docs/GetPublicResponsesResult.md)
|
|
378
385
|
- [GetRapidResponsesResult](rapidata/api_client/docs/GetRapidResponsesResult.md)
|
|
379
386
|
- [GetResponsesResult](rapidata/api_client/docs/GetResponsesResult.md)
|
|
387
|
+
- [GetSampleByIdResult](rapidata/api_client/docs/GetSampleByIdResult.md)
|
|
380
388
|
- [GetSimpleWorkflowResultsModel](rapidata/api_client/docs/GetSimpleWorkflowResultsModel.md)
|
|
381
389
|
- [GetStandingByIdResult](rapidata/api_client/docs/GetStandingByIdResult.md)
|
|
382
390
|
- [GetValidationRapidsResult](rapidata/api_client/docs/GetValidationRapidsResult.md)
|
|
@@ -400,6 +408,7 @@ Class | Method | HTTP request | Description
|
|
|
400
408
|
- [JsonWebKey](rapidata/api_client/docs/JsonWebKey.md)
|
|
401
409
|
- [JsonWebKeySet](rapidata/api_client/docs/JsonWebKeySet.md)
|
|
402
410
|
- [LabelingSelection](rapidata/api_client/docs/LabelingSelection.md)
|
|
411
|
+
- [LanguageFilter](rapidata/api_client/docs/LanguageFilter.md)
|
|
403
412
|
- [LanguageUserFilterModel](rapidata/api_client/docs/LanguageUserFilterModel.md)
|
|
404
413
|
- [LeaderboardQueryResult](rapidata/api_client/docs/LeaderboardQueryResult.md)
|
|
405
414
|
- [LeaderboardQueryResultPagedResult](rapidata/api_client/docs/LeaderboardQueryResultPagedResult.md)
|
|
@@ -440,15 +449,18 @@ Class | Method | HTTP request | Description
|
|
|
440
449
|
- [NamedEntityTruth](rapidata/api_client/docs/NamedEntityTruth.md)
|
|
441
450
|
- [NeverEndingRefereeConfig](rapidata/api_client/docs/NeverEndingRefereeConfig.md)
|
|
442
451
|
- [NeverEndingRefereeInfo](rapidata/api_client/docs/NeverEndingRefereeInfo.md)
|
|
452
|
+
- [NewUserFilter](rapidata/api_client/docs/NewUserFilter.md)
|
|
443
453
|
- [NewUserFilterModel](rapidata/api_client/docs/NewUserFilterModel.md)
|
|
444
454
|
- [NewsletterModel](rapidata/api_client/docs/NewsletterModel.md)
|
|
445
455
|
- [NotAvailableYetResult](rapidata/api_client/docs/NotAvailableYetResult.md)
|
|
456
|
+
- [NotFilter](rapidata/api_client/docs/NotFilter.md)
|
|
446
457
|
- [NotUserFilterModel](rapidata/api_client/docs/NotUserFilterModel.md)
|
|
447
458
|
- [NullAsset](rapidata/api_client/docs/NullAsset.md)
|
|
448
459
|
- [NullAssetModel](rapidata/api_client/docs/NullAssetModel.md)
|
|
449
460
|
- [OnlinePairMakerConfig](rapidata/api_client/docs/OnlinePairMakerConfig.md)
|
|
450
461
|
- [OnlinePairMakerConfigModel](rapidata/api_client/docs/OnlinePairMakerConfigModel.md)
|
|
451
462
|
- [OnlinePairMakerInformation](rapidata/api_client/docs/OnlinePairMakerInformation.md)
|
|
463
|
+
- [OrFilter](rapidata/api_client/docs/OrFilter.md)
|
|
452
464
|
- [OrUserFilterModel](rapidata/api_client/docs/OrUserFilterModel.md)
|
|
453
465
|
- [OrderModel](rapidata/api_client/docs/OrderModel.md)
|
|
454
466
|
- [OrderModelPagedResult](rapidata/api_client/docs/OrderModelPagedResult.md)
|
|
@@ -482,6 +494,7 @@ Class | Method | HTTP request | Description
|
|
|
482
494
|
- [PromptMetadataModel](rapidata/api_client/docs/PromptMetadataModel.md)
|
|
483
495
|
- [ProxyFileWrapper](rapidata/api_client/docs/ProxyFileWrapper.md)
|
|
484
496
|
- [PublicOrderModel](rapidata/api_client/docs/PublicOrderModel.md)
|
|
497
|
+
- [PublicRapidResponse](rapidata/api_client/docs/PublicRapidResponse.md)
|
|
485
498
|
- [PublicTextMetadataInput](rapidata/api_client/docs/PublicTextMetadataInput.md)
|
|
486
499
|
- [QueryModel](rapidata/api_client/docs/QueryModel.md)
|
|
487
500
|
- [QueryValidationModel](rapidata/api_client/docs/QueryValidationModel.md)
|
|
@@ -499,12 +512,15 @@ Class | Method | HTTP request | Description
|
|
|
499
512
|
- [RegisterTemporaryCustomerModel](rapidata/api_client/docs/RegisterTemporaryCustomerModel.md)
|
|
500
513
|
- [RegisterTemporaryCustomerResult](rapidata/api_client/docs/RegisterTemporaryCustomerResult.md)
|
|
501
514
|
- [ReportModel](rapidata/api_client/docs/ReportModel.md)
|
|
515
|
+
- [ResponseCountFilter](rapidata/api_client/docs/ResponseCountFilter.md)
|
|
502
516
|
- [ResponseCountUserFilterModel](rapidata/api_client/docs/ResponseCountUserFilterModel.md)
|
|
503
517
|
- [RetrievalMode](rapidata/api_client/docs/RetrievalMode.md)
|
|
504
518
|
- [RootFilter](rapidata/api_client/docs/RootFilter.md)
|
|
505
519
|
- [RunStatus](rapidata/api_client/docs/RunStatus.md)
|
|
506
520
|
- [RunsByLeaderboardResult](rapidata/api_client/docs/RunsByLeaderboardResult.md)
|
|
507
521
|
- [RunsByLeaderboardResultPagedResult](rapidata/api_client/docs/RunsByLeaderboardResultPagedResult.md)
|
|
522
|
+
- [SampleByIdentifier](rapidata/api_client/docs/SampleByIdentifier.md)
|
|
523
|
+
- [SampleByIdentifierPagedResult](rapidata/api_client/docs/SampleByIdentifierPagedResult.md)
|
|
508
524
|
- [SampleByParticipant](rapidata/api_client/docs/SampleByParticipant.md)
|
|
509
525
|
- [SampleByParticipantPagedResult](rapidata/api_client/docs/SampleByParticipantPagedResult.md)
|
|
510
526
|
- [ScrubPayload](rapidata/api_client/docs/ScrubPayload.md)
|
|
@@ -560,7 +576,6 @@ Class | Method | HTTP request | Description
|
|
|
560
576
|
- [TranslatedString](rapidata/api_client/docs/TranslatedString.md)
|
|
561
577
|
- [UnlockOrderResult](rapidata/api_client/docs/UnlockOrderResult.md)
|
|
562
578
|
- [UpdateBenchmarkNameModel](rapidata/api_client/docs/UpdateBenchmarkNameModel.md)
|
|
563
|
-
- [UpdateCampaignModel](rapidata/api_client/docs/UpdateCampaignModel.md)
|
|
564
579
|
- [UpdateDatasetNameModel](rapidata/api_client/docs/UpdateDatasetNameModel.md)
|
|
565
580
|
- [UpdateDimensionsModel](rapidata/api_client/docs/UpdateDimensionsModel.md)
|
|
566
581
|
- [UpdateLeaderboardNameModel](rapidata/api_client/docs/UpdateLeaderboardNameModel.md)
|
|
@@ -574,7 +589,10 @@ Class | Method | HTTP request | Description
|
|
|
574
589
|
- [UploadCocoResult](rapidata/api_client/docs/UploadCocoResult.md)
|
|
575
590
|
- [UploadFromS3Result](rapidata/api_client/docs/UploadFromS3Result.md)
|
|
576
591
|
- [UrlAssetInput](rapidata/api_client/docs/UrlAssetInput.md)
|
|
592
|
+
- [UserScoreFilter](rapidata/api_client/docs/UserScoreFilter.md)
|
|
577
593
|
- [UserScoreUserFilterModel](rapidata/api_client/docs/UserScoreUserFilterModel.md)
|
|
594
|
+
- [UserState](rapidata/api_client/docs/UserState.md)
|
|
595
|
+
- [UserStateFilter](rapidata/api_client/docs/UserStateFilter.md)
|
|
578
596
|
- [ValidationChance](rapidata/api_client/docs/ValidationChance.md)
|
|
579
597
|
- [ValidationSelection](rapidata/api_client/docs/ValidationSelection.md)
|
|
580
598
|
- [ValidationSetModel](rapidata/api_client/docs/ValidationSetModel.md)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import re
|
|
2
|
-
from typing import Literal, Optional
|
|
2
|
+
from typing import Literal, Optional, Sequence
|
|
3
3
|
from rapidata.api_client.models.root_filter import RootFilter
|
|
4
4
|
from rapidata.api_client.models.filter import Filter
|
|
5
5
|
from rapidata.api_client.models.query_model import QueryModel
|
|
@@ -15,7 +15,9 @@ from rapidata.api_client.models.submit_prompt_model_prompt_asset import (
|
|
|
15
15
|
from rapidata.api_client.models.url_asset_input import UrlAssetInput
|
|
16
16
|
from rapidata.api_client.models.file_asset_model import FileAssetModel
|
|
17
17
|
from rapidata.api_client.models.source_url_metadata_model import SourceUrlMetadataModel
|
|
18
|
-
|
|
18
|
+
from rapidata.api_client.models.and_user_filter_model_filters_inner import (
|
|
19
|
+
AndUserFilterModelFiltersInner,
|
|
20
|
+
)
|
|
19
21
|
|
|
20
22
|
from rapidata.rapidata_client.benchmark.participant._participant import (
|
|
21
23
|
BenchmarkParticipant,
|
|
@@ -28,6 +30,8 @@ from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import
|
|
|
28
30
|
)
|
|
29
31
|
from rapidata.rapidata_client.datapoints.assets import MediaAsset
|
|
30
32
|
from rapidata.rapidata_client.benchmark._detail_mapper import DetailMapper
|
|
33
|
+
from rapidata.rapidata_client.filter import RapidataFilter
|
|
34
|
+
from rapidata.rapidata_client.settings import RapidataSetting
|
|
31
35
|
|
|
32
36
|
|
|
33
37
|
class RapidataBenchmark:
|
|
@@ -260,6 +264,9 @@ class RapidataBenchmark:
|
|
|
260
264
|
inverse_ranking: bool = False,
|
|
261
265
|
level_of_detail: Literal["low", "medium", "high", "very high"] = "low",
|
|
262
266
|
min_responses_per_matchup: int = 3,
|
|
267
|
+
validation_set_id: str | None = None,
|
|
268
|
+
filters: Sequence[RapidataFilter] = [],
|
|
269
|
+
settings: Sequence[RapidataSetting] = [],
|
|
263
270
|
) -> RapidataLeaderboard:
|
|
264
271
|
"""
|
|
265
272
|
Creates a new leaderboard for the benchmark.
|
|
@@ -272,6 +279,9 @@ class RapidataBenchmark:
|
|
|
272
279
|
inverse_ranking: Whether to inverse the ranking of the leaderboard. (if the question is inversed, e.g. "Which video is worse?")
|
|
273
280
|
level_of_detail: The level of detail of the leaderboard. This will effect how many comparisons are done per model evaluation. (default: "low")
|
|
274
281
|
min_responses_per_matchup: The minimum number of responses required to be considered for the leaderboard. (default: 3)
|
|
282
|
+
validation_set_id: The id of the validation set that should be attached to the leaderboard. (default: None)
|
|
283
|
+
filters: The filters that should be applied to the leaderboard. Will determine who can solve answer in the leaderboard. (default: [])
|
|
284
|
+
settings: The settings that should be applied to the leaderboard. Will determine the behavior of the tasks on the leaderboard. (default: [])
|
|
275
285
|
"""
|
|
276
286
|
if not isinstance(min_responses_per_matchup, int):
|
|
277
287
|
raise ValueError("Min responses per matchup must be an integer")
|
|
@@ -289,6 +299,20 @@ class RapidataBenchmark:
|
|
|
289
299
|
isInversed=inverse_ranking,
|
|
290
300
|
minResponses=min_responses_per_matchup,
|
|
291
301
|
responseBudget=DetailMapper.get_budget(level_of_detail),
|
|
302
|
+
validationSetId=validation_set_id,
|
|
303
|
+
filters=(
|
|
304
|
+
[
|
|
305
|
+
AndUserFilterModelFiltersInner(filter._to_model())
|
|
306
|
+
for filter in filters
|
|
307
|
+
]
|
|
308
|
+
if filters
|
|
309
|
+
else None
|
|
310
|
+
),
|
|
311
|
+
featureFlags=(
|
|
312
|
+
[setting._to_feature_flag() for setting in settings]
|
|
313
|
+
if settings
|
|
314
|
+
else None
|
|
315
|
+
),
|
|
292
316
|
)
|
|
293
317
|
)
|
|
294
318
|
|