rapidata 1.10.1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rapidata might be problematic. Click here for more details.
- rapidata/__init__.py +22 -17
- rapidata/api_client/__init__.py +16 -5
- rapidata/api_client/api/coco_api.py +14 -29
- rapidata/api_client/api/dataset_api.py +6 -6
- rapidata/api_client/api/identity_api.py +3 -3
- rapidata/api_client/api/pipeline_api.py +1008 -95
- rapidata/api_client/api/rapid_api.py +6 -6
- rapidata/api_client/api/validation_api.py +12 -42
- rapidata/api_client/models/__init__.py +16 -5
- rapidata/api_client/models/add_campaign_model.py +5 -5
- rapidata/api_client/models/add_validation_text_rapid_model.py +1 -1
- rapidata/api_client/models/age_group.py +5 -4
- rapidata/api_client/models/base_error.py +1 -4
- rapidata/api_client/models/compare_workflow_config.py +9 -24
- rapidata/api_client/models/compare_workflow_config_model.py +9 -29
- rapidata/api_client/models/compare_workflow_config_model_pair_maker_config.py +140 -0
- rapidata/api_client/models/compare_workflow_config_pair_maker_config.py +140 -0
- rapidata/api_client/models/compare_workflow_model.py +7 -3
- rapidata/api_client/models/compare_workflow_model1.py +7 -3
- rapidata/api_client/models/compare_workflow_model1_pair_maker_information.py +140 -0
- rapidata/api_client/models/compare_workflow_model_pair_maker_config.py +140 -0
- rapidata/api_client/models/create_order_model.py +4 -2
- rapidata/api_client/models/create_order_model_user_filters_inner.py +25 -11
- rapidata/api_client/models/custom_user_filter_model.py +98 -0
- rapidata/api_client/models/file_asset_model_metadata_inner.py +8 -22
- rapidata/api_client/models/get_classify_workflow_result_overview_result.py +144 -0
- rapidata/api_client/models/get_pipeline_by_id_result.py +13 -3
- rapidata/api_client/models/identity_read_bridge_token_get202_response.py +140 -0
- rapidata/api_client/models/not_available_yet_result.py +96 -0
- rapidata/api_client/models/online_pair_maker_config.py +98 -0
- rapidata/api_client/models/online_pair_maker_config_model.py +98 -0
- rapidata/api_client/models/online_pair_maker_information.py +100 -0
- rapidata/api_client/models/pipeline_id_workflow_put_request.py +140 -0
- rapidata/api_client/models/pre_arranged_pair_maker_config.py +100 -0
- rapidata/api_client/models/pre_arranged_pair_maker_config_model.py +96 -0
- rapidata/api_client/models/pre_arranged_pair_maker_information.py +102 -0
- rapidata/api_client/models/read_bridge_token_keys_result.py +11 -2
- rapidata/api_client/models/simple_workflow_config.py +7 -26
- rapidata/api_client/models/simple_workflow_config_model.py +4 -28
- rapidata/api_client/models/simple_workflow_get_result_overview_get200_response.py +16 -16
- rapidata/api_client/models/simple_workflow_model1.py +3 -3
- rapidata/api_client/models/update_campaign_model.py +99 -0
- rapidata/api_client/models/validation_import_post_request_blueprint.py +1 -1
- rapidata/api_client_README.md +21 -7
- rapidata/rapidata_client/__init__.py +20 -10
- rapidata/rapidata_client/assets/__init__.py +5 -4
- rapidata/rapidata_client/assets/{media_asset.py → _media_asset.py} +32 -11
- rapidata/rapidata_client/assets/{multi_asset.py → _multi_asset.py} +1 -1
- rapidata/rapidata_client/assets/{text_asset.py → _text_asset.py} +1 -1
- rapidata/rapidata_client/assets/data_type_enum.py +7 -0
- rapidata/rapidata_client/filter/__init__.py +2 -1
- rapidata/rapidata_client/filter/_base_filter.py +10 -0
- rapidata/rapidata_client/filter/age_filter.py +12 -5
- rapidata/rapidata_client/filter/campaign_filter.py +12 -3
- rapidata/rapidata_client/filter/country_filter.py +10 -3
- rapidata/rapidata_client/filter/custom_filter.py +29 -0
- rapidata/rapidata_client/filter/gender_filter.py +12 -5
- rapidata/rapidata_client/filter/language_filter.py +14 -3
- rapidata/rapidata_client/filter/models/age_group.py +26 -0
- rapidata/rapidata_client/filter/models/gender.py +19 -0
- rapidata/rapidata_client/filter/rapidata_filters.py +35 -0
- rapidata/rapidata_client/filter/user_score_filter.py +20 -4
- rapidata/rapidata_client/metadata/__init__.py +5 -5
- rapidata/rapidata_client/metadata/{base_metadata.py → _base_metadata.py} +2 -1
- rapidata/rapidata_client/metadata/{private_text_metadata.py → _private_text_metadata.py} +2 -2
- rapidata/rapidata_client/metadata/{prompt_metadata.py → _prompt_metadata.py} +3 -2
- rapidata/rapidata_client/metadata/{public_text_metadata.py → _public_text_metadata.py} +2 -2
- rapidata/rapidata_client/metadata/{select_words_metadata.py → _select_words_metadata.py} +3 -2
- rapidata/rapidata_client/{dataset/rapidata_dataset.py → order/_rapidata_dataset.py} +7 -8
- rapidata/rapidata_client/order/_rapidata_order_builder.py +365 -0
- rapidata/rapidata_client/order/rapidata_order.py +50 -32
- rapidata/rapidata_client/order/rapidata_order_manager.py +461 -0
- rapidata/rapidata_client/rapidata_client.py +12 -201
- rapidata/rapidata_client/referee/__init__.py +3 -3
- rapidata/rapidata_client/referee/{base_referee.py → _base_referee.py} +3 -3
- rapidata/rapidata_client/referee/{early_stopping_referee.py → _early_stopping_referee.py} +14 -11
- rapidata/rapidata_client/referee/{naive_referee.py → _naive_referee.py} +9 -9
- rapidata/rapidata_client/selection/__init__.py +1 -1
- rapidata/rapidata_client/{filter/base_filter.py → selection/_base_selection.py} +2 -2
- rapidata/rapidata_client/selection/capped_selection.py +15 -5
- rapidata/rapidata_client/selection/conditional_validation_selection.py +17 -4
- rapidata/rapidata_client/selection/demographic_selection.py +18 -7
- rapidata/rapidata_client/selection/labeling_selection.py +10 -3
- rapidata/rapidata_client/selection/rapidata_selections.py +21 -0
- rapidata/rapidata_client/selection/validation_selection.py +11 -4
- rapidata/rapidata_client/settings/__init__.py +9 -2
- rapidata/rapidata_client/settings/_rapidata_setting.py +11 -0
- rapidata/rapidata_client/settings/alert_on_fast_response.py +21 -0
- rapidata/rapidata_client/settings/custom_setting.py +16 -0
- rapidata/rapidata_client/settings/free_text_minimum_characters.py +16 -0
- rapidata/rapidata_client/settings/models/__init__.py +1 -0
- rapidata/rapidata_client/settings/models/translation_behaviour_options.py +14 -0
- rapidata/rapidata_client/settings/no_shuffle.py +16 -0
- rapidata/rapidata_client/settings/play_video_until_the_end.py +16 -0
- rapidata/rapidata_client/settings/rapidata_settings.py +31 -0
- rapidata/rapidata_client/settings/translation_behaviour.py +18 -0
- rapidata/rapidata_client/validation/__init__.py +1 -0
- rapidata/rapidata_client/{dataset/validation_rapid_parts.py → validation/_validation_rapid_parts.py} +7 -6
- rapidata/rapidata_client/validation/_validation_set_builder.py +371 -0
- rapidata/rapidata_client/{dataset → validation}/rapidata_validation_set.py +54 -50
- rapidata/rapidata_client/validation/rapids/__init__.py +1 -0
- rapidata/rapidata_client/validation/rapids/box.py +17 -0
- rapidata/rapidata_client/validation/rapids/rapids.py +94 -0
- rapidata/rapidata_client/validation/rapids/rapids_manager.py +163 -0
- rapidata/rapidata_client/validation/validation_set_manager.py +338 -0
- rapidata/rapidata_client/workflow/__init__.py +8 -6
- rapidata/rapidata_client/workflow/_base_workflow.py +25 -0
- rapidata/rapidata_client/workflow/{classify_workflow.py → _classify_workflow.py} +6 -6
- rapidata/rapidata_client/workflow/{compare_workflow.py → _compare_workflow.py} +10 -16
- rapidata/rapidata_client/workflow/_draw_workflow.py +22 -0
- rapidata/rapidata_client/workflow/_evaluation_workflow.py +26 -0
- rapidata/rapidata_client/workflow/{free_text_workflow.py → _free_text_workflow.py} +10 -16
- rapidata/rapidata_client/workflow/_locate_workflow.py +22 -0
- rapidata/rapidata_client/workflow/{select_words_workflow.py → _select_words_workflow.py} +2 -8
- rapidata/service/credential_manager.py +11 -1
- rapidata/service/openapi_service.py +23 -4
- {rapidata-1.10.1.dist-info → rapidata-2.1.0.dist-info}/METADATA +2 -1
- {rapidata-1.10.1.dist-info → rapidata-2.1.0.dist-info}/RECORD +122 -96
- rapidata/constants.py +0 -1
- rapidata/rapidata_client/dataset/rapid_builders/__init__.py +0 -4
- rapidata/rapidata_client/dataset/rapid_builders/base_rapid_builder.py +0 -33
- rapidata/rapidata_client/dataset/rapid_builders/classify_rapid_builders.py +0 -166
- rapidata/rapidata_client/dataset/rapid_builders/compare_rapid_builders.py +0 -145
- rapidata/rapidata_client/dataset/rapid_builders/rapids.py +0 -33
- rapidata/rapidata_client/dataset/rapid_builders/select_words_rapid_builders.py +0 -124
- rapidata/rapidata_client/dataset/validation_set_builder.py +0 -336
- rapidata/rapidata_client/order/order_builder.py +0 -25
- rapidata/rapidata_client/order/rapidata_order_builder.py +0 -463
- rapidata/rapidata_client/selection/base_selection.py +0 -9
- rapidata/rapidata_client/settings/feature_flags.py +0 -125
- rapidata/rapidata_client/settings/settings.py +0 -124
- rapidata/rapidata_client/simple_builders/__init__.py +0 -0
- rapidata/rapidata_client/simple_builders/simple_classification_builders.py +0 -271
- rapidata/rapidata_client/simple_builders/simple_compare_builders.py +0 -267
- rapidata/rapidata_client/simple_builders/simple_free_text_builders.py +0 -192
- rapidata/rapidata_client/simple_builders/simple_select_words_builders.py +0 -196
- rapidata/rapidata_client/workflow/base_workflow.py +0 -42
- rapidata/rapidata_client/workflow/evaluation_workflow.py +0 -15
- /rapidata/rapidata_client/assets/{base_asset.py → _base_asset.py} +0 -0
- /rapidata/rapidata_client/{dataset → filter/models}/__init__.py +0 -0
- {rapidata-1.10.1.dist-info → rapidata-2.1.0.dist-info}/LICENSE +0 -0
- {rapidata-1.10.1.dist-info → rapidata-2.1.0.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
from rapidata.rapidata_client.validation._validation_set_builder import ValidationSetBuilder
|
|
2
|
+
from rapidata.rapidata_client.validation.rapidata_validation_set import RapidataValidationSet
|
|
3
|
+
from rapidata.service.openapi_service import OpenAPIService
|
|
4
|
+
from rapidata.rapidata_client.assets.data_type_enum import RapidataDataTypes
|
|
5
|
+
from rapidata.rapidata_client.validation.rapids.rapids_manager import RapidsManager
|
|
6
|
+
from rapidata.rapidata_client.validation.rapids.rapids import Rapid
|
|
7
|
+
from rapidata.rapidata_client.metadata import PromptMetadata
|
|
8
|
+
|
|
9
|
+
from rapidata.api_client.models.page_info import PageInfo
|
|
10
|
+
from rapidata.api_client.models.root_filter import RootFilter
|
|
11
|
+
from rapidata.api_client.models.filter import Filter
|
|
12
|
+
from rapidata.api_client.models.sort_criterion import SortCriterion
|
|
13
|
+
from rapidata.api_client.exceptions import BadRequestException
|
|
14
|
+
from urllib3._collections import HTTPHeaderDict
|
|
15
|
+
|
|
16
|
+
from rapidata.rapidata_client.validation.rapids.box import Box
|
|
17
|
+
|
|
18
|
+
from rapidata.api_client.models.query_validation_set_model import QueryValidationSetModel
|
|
19
|
+
|
|
20
|
+
from typing import Sequence
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ValidationSetManager:
|
|
24
|
+
"""
|
|
25
|
+
Responsible for everything related to validation sets. From creation to retrieval.
|
|
26
|
+
|
|
27
|
+
Attributes:
|
|
28
|
+
rapid (RapidsManager): The RapidsManager instance.
|
|
29
|
+
"""
|
|
30
|
+
def __init__(self, openapi_service: OpenAPIService) -> None:
|
|
31
|
+
self.__openapi_service = openapi_service
|
|
32
|
+
self.rapid = RapidsManager()
|
|
33
|
+
|
|
34
|
+
def create_classification_set(self,
|
|
35
|
+
name: str,
|
|
36
|
+
instruction: str,
|
|
37
|
+
answer_options: list[str],
|
|
38
|
+
datapoints: list[str],
|
|
39
|
+
truths: list[list[str]],
|
|
40
|
+
data_type: str = RapidataDataTypes.MEDIA,
|
|
41
|
+
contexts: list[str] | None = None,
|
|
42
|
+
print_confirmation: bool = True
|
|
43
|
+
) -> RapidataValidationSet:
|
|
44
|
+
"""Create a classification validation set.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
name (str): The name of the validation set. (will not be shown to the labeler)
|
|
48
|
+
instruction (str): The instruction by which the labeler will answer.
|
|
49
|
+
answer_options (list[str]): The options to choose from when answering.
|
|
50
|
+
datapoints (list[str]): The datapoints that will be used for validation.
|
|
51
|
+
truths (list[list[str]]): The truths for each datapoint. Outher list is for each datapoint, inner list is for each truth.\n
|
|
52
|
+
example:
|
|
53
|
+
options: ["yes", "no", "maybe"]
|
|
54
|
+
datapoints: ["datapoint1", "datapoint2"]
|
|
55
|
+
truths: [["yes"], ["no", "maybe"]] -> first datapoint correct answer is "yes", second datapoint is "no" or "maybe"
|
|
56
|
+
data_type (str, optional): The type of data. Defaults to RapidataDataTypes.MEDIA. Other option: RapidataDataTypes.TEXT ("text").
|
|
57
|
+
contexts (list[str], optional): The contexts for each datapoint. Defaults to None.\n
|
|
58
|
+
If provided has to be the same length as datapoints and will be shown in addition to the instruction and answer options. (Therefore will be different for each datapoint)
|
|
59
|
+
Will be match up with the datapoints using the list index.
|
|
60
|
+
print_confirmation (bool, optional): Whether to print a confirmation message that validation set has been created. Defaults to True.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
if len(datapoints) != len(truths):
|
|
64
|
+
raise ValueError("The number of datapoints and truths must be equal")
|
|
65
|
+
if contexts and len(contexts) != len(datapoints):
|
|
66
|
+
raise ValueError("The number of contexts and datapoints must be equal")
|
|
67
|
+
|
|
68
|
+
rapids = []
|
|
69
|
+
for i in range(len(datapoints)):
|
|
70
|
+
rapids.append(
|
|
71
|
+
self.rapid.classification_rapid(
|
|
72
|
+
instruction=instruction,
|
|
73
|
+
answer_options=answer_options,
|
|
74
|
+
datapoint=datapoints[i],
|
|
75
|
+
truths=truths[i],
|
|
76
|
+
data_type=data_type,
|
|
77
|
+
metadata=[PromptMetadata(contexts[i])] if contexts else []
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
validation_set_builder = ValidationSetBuilder(name, self.__openapi_service)
|
|
82
|
+
for rapid in rapids:
|
|
83
|
+
validation_set_builder._add_rapid(rapid)
|
|
84
|
+
|
|
85
|
+
return validation_set_builder._submit(print_confirmation)
|
|
86
|
+
|
|
87
|
+
def create_compare_set(self,
|
|
88
|
+
name: str,
|
|
89
|
+
instruction: str,
|
|
90
|
+
datapoints: list[list[str]],
|
|
91
|
+
truths: list[str],
|
|
92
|
+
data_type: str = RapidataDataTypes.MEDIA,
|
|
93
|
+
contexts: list[str] | None = None,
|
|
94
|
+
print_confirmation: bool = True
|
|
95
|
+
) -> RapidataValidationSet:
|
|
96
|
+
"""Create a comparison validation set.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
name (str): The name of the validation set. (will not be shown to the labeler)
|
|
100
|
+
instruction (str): The instruction to compare against.
|
|
101
|
+
truths (list[str]): The truths for each comparison. List is for each comparison.\n
|
|
102
|
+
example:
|
|
103
|
+
instruction: "Which image has a cat?"
|
|
104
|
+
datapoints = [["image1.jpg", "image2.jpg"], ["image3.jpg", "image4.jpg"]]
|
|
105
|
+
truths: ["image1.jpg", "image4.jpg"] -> first comparison image1.jpg has a cat, second comparison image4.jpg has a cat
|
|
106
|
+
datapoints (list[list[str]]): The compare datapoints to create the validation set with.
|
|
107
|
+
Outer list is for each comparison, inner list the two images/texts that will be compared.
|
|
108
|
+
data_type (str, optional): The type of data. Defaults to RapidataDataTypes.MEDIA. Other option: RapidataDataTypes.TEXT ("text").
|
|
109
|
+
contexts (list[str], optional): The contexts for each datapoint. Defaults to None.\n
|
|
110
|
+
If provided has to be the same length as datapoints and will be shown in addition to the instruction and truth. (Therefore will be different for each datapoint)
|
|
111
|
+
Will be match up with the datapoints using the list index.
|
|
112
|
+
print_confirmation (bool, optional): Whether to print a confirmation message that validation set has been created. Defaults to True.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
if len(datapoints) != len(truths):
|
|
116
|
+
raise ValueError("The number of datapoints and truths must be equal")
|
|
117
|
+
|
|
118
|
+
if contexts and len(contexts) != len(datapoints):
|
|
119
|
+
raise ValueError("The number of contexts and datapoints must be equal")
|
|
120
|
+
|
|
121
|
+
rapids = []
|
|
122
|
+
for i in range(len(datapoints)):
|
|
123
|
+
rapids.append(
|
|
124
|
+
self.rapid.compare_rapid(
|
|
125
|
+
instruction=instruction,
|
|
126
|
+
truth=truths[i],
|
|
127
|
+
datapoint=datapoints[i],
|
|
128
|
+
data_type=data_type,
|
|
129
|
+
metadata=[PromptMetadata(contexts[i])] if contexts else []
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
validation_set_builder = ValidationSetBuilder(name, self.__openapi_service)
|
|
134
|
+
for rapid in rapids:
|
|
135
|
+
validation_set_builder._add_rapid(rapid)
|
|
136
|
+
|
|
137
|
+
return validation_set_builder._submit(print_confirmation)
|
|
138
|
+
|
|
139
|
+
def create_select_words_set(self,
|
|
140
|
+
name: str,
|
|
141
|
+
instruction: str,
|
|
142
|
+
truths: list[list[int]],
|
|
143
|
+
datapoints: list[str],
|
|
144
|
+
sentences: list[str],
|
|
145
|
+
strict_grading: bool = True,
|
|
146
|
+
print_confirmation: bool = True
|
|
147
|
+
) -> RapidataValidationSet:
|
|
148
|
+
"""Create a select words validation set.
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
name (str): The name of the validation set. (will not be shown to the labeler)
|
|
152
|
+
instruction (str): The instruction to show to the labeler.
|
|
153
|
+
truths (list[list[int]]): The truths for each datapoint. Outher list is for each datapoint, inner list is for each truth.\n
|
|
154
|
+
example:
|
|
155
|
+
datapoints: ["datapoint1", "datapoint2"]
|
|
156
|
+
sentences: ["this example 1", "this example 2"]
|
|
157
|
+
truths: [[0, 1], [2]] -> first datapoint correct words are "this" and "example", second datapoint is "2"
|
|
158
|
+
datapoints (list[str]): The datapoints that will be used for validation.
|
|
159
|
+
sentences (list[str]): The sentences that will be used for validation. The sentece will be split up by spaces to be selected by the labeler.
|
|
160
|
+
Must be the same length as datapoints.
|
|
161
|
+
strict_grading (bool, optional): Whether to grade strictly. Defaults to True.\n
|
|
162
|
+
If True, the labeler must select all correct words to be graded as correct. (and no wrong words)
|
|
163
|
+
If False, the labeler must select at least one correct word to be graded as correct. (and no wrong words)
|
|
164
|
+
print_confirmation (bool, optional): Whether to print a confirmation message that validation set has been created. Defaults to True.
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
if len(datapoints) != len(truths) or len(datapoints) != len(sentences):
|
|
168
|
+
raise ValueError("The number of datapoints, truths, and sentences must be equal")
|
|
169
|
+
|
|
170
|
+
rapids = []
|
|
171
|
+
for i in range(len(datapoints)):
|
|
172
|
+
rapids.append(
|
|
173
|
+
self.rapid.select_words_rapid(
|
|
174
|
+
instruction=instruction,
|
|
175
|
+
truths=truths[i],
|
|
176
|
+
datapoint=datapoints[i],
|
|
177
|
+
sentence=sentences[i],
|
|
178
|
+
strict_grading=strict_grading
|
|
179
|
+
)
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
validation_set_builder = ValidationSetBuilder(name, self.__openapi_service)
|
|
183
|
+
for rapid in rapids:
|
|
184
|
+
validation_set_builder._add_rapid(rapid)
|
|
185
|
+
|
|
186
|
+
return validation_set_builder._submit(print_confirmation)
|
|
187
|
+
|
|
188
|
+
def create_locate_set(self,
|
|
189
|
+
name: str,
|
|
190
|
+
instruction: str,
|
|
191
|
+
truths: list[list[Box]],
|
|
192
|
+
datapoints: list[str],
|
|
193
|
+
contexts: list[str] | None = None,
|
|
194
|
+
print_confirmation: bool = True
|
|
195
|
+
) -> RapidataValidationSet:
|
|
196
|
+
"""Create a locate validation set.
|
|
197
|
+
|
|
198
|
+
Args:
|
|
199
|
+
name (str): The name of the validation set. (will not be shown to the labeler)
|
|
200
|
+
instruction (str): The instruction to show to the labeler.
|
|
201
|
+
truths (list[list[Box]]): The truths for each datapoint. Outher list is for each datapoint, inner list is for each truth.\n
|
|
202
|
+
example:
|
|
203
|
+
datapoints: ["datapoint1", "datapoint2"]
|
|
204
|
+
truths: [[Box(0, 0, 100, 100)], [Box(50, 50, 150, 150)]] -> first datapoint the object is in the top left corner, second datapoint the object is in the center
|
|
205
|
+
datapoints (list[str]): The datapoints that will be used for validation.
|
|
206
|
+
contexts (list[str], optional): The contexts for each datapoint. Defaults to None.
|
|
207
|
+
print_confirmation (bool, optional): Whether to print a confirmation message that validation set has been created. Defaults to True.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
if len(datapoints) != len(truths):
|
|
211
|
+
raise ValueError("The number of datapoints and truths must be equal")
|
|
212
|
+
|
|
213
|
+
if contexts and len(contexts) != len(datapoints):
|
|
214
|
+
raise ValueError("The number of contexts and datapoints must be equal")
|
|
215
|
+
|
|
216
|
+
rapids = []
|
|
217
|
+
for i in range(len(datapoints)):
|
|
218
|
+
rapids.append(
|
|
219
|
+
self.rapid.locate_rapid(
|
|
220
|
+
instruction=instruction,
|
|
221
|
+
truths=truths[i],
|
|
222
|
+
datapoint=datapoints[i],
|
|
223
|
+
metadata=[PromptMetadata(contexts[i])] if contexts else []
|
|
224
|
+
)
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
validation_set_builder = ValidationSetBuilder(name, self.__openapi_service)
|
|
228
|
+
for rapid in rapids:
|
|
229
|
+
validation_set_builder._add_rapid(rapid)
|
|
230
|
+
|
|
231
|
+
return validation_set_builder._submit(print_confirmation)
|
|
232
|
+
|
|
233
|
+
def create_draw_set(self,
|
|
234
|
+
name: str,
|
|
235
|
+
instruction: str,
|
|
236
|
+
truths: list[list[Box]],
|
|
237
|
+
datapoints: list[str],
|
|
238
|
+
contexts: list[str] | None = None,
|
|
239
|
+
print_confirmation: bool = True
|
|
240
|
+
) -> RapidataValidationSet:
|
|
241
|
+
"""Create a draw validation set.
|
|
242
|
+
|
|
243
|
+
Args:
|
|
244
|
+
name (str): The name of the validation set. (will not be shown to the labeler)
|
|
245
|
+
instruction (str): The instruction to show to the labeler.
|
|
246
|
+
truths (list[list[Box]]): The truths for each datapoint. Outher list is for each datapoint, inner list is for each truth.\n
|
|
247
|
+
example:
|
|
248
|
+
datapoints: ["datapoint1", "datapoint2"]
|
|
249
|
+
truths: [[Box(0, 0, 100, 100)], [Box(50, 50, 150, 150)]] -> first datapoint the object is in the top left corner, second datapoint the object is in the center
|
|
250
|
+
datapoints (list[str]): The datapoints that will be used for validation.
|
|
251
|
+
contexts (list[str], optional): The contexts for each datapoint. Defaults to None.
|
|
252
|
+
print_confirmation (bool, optional): Whether to print a confirmation message that validation set has been created. Defaults to True.
|
|
253
|
+
"""
|
|
254
|
+
|
|
255
|
+
if len(datapoints) != len(truths):
|
|
256
|
+
raise ValueError("The number of datapoints and truths must be equal")
|
|
257
|
+
|
|
258
|
+
if contexts and len(contexts) != len(datapoints):
|
|
259
|
+
raise ValueError("The number of contexts and datapoints must be equal")
|
|
260
|
+
|
|
261
|
+
rapids = []
|
|
262
|
+
for i in range(len(datapoints)):
|
|
263
|
+
rapids.append(
|
|
264
|
+
self.rapid.draw_rapid(
|
|
265
|
+
instruction=instruction,
|
|
266
|
+
truths=truths[i],
|
|
267
|
+
datapoint=datapoints[i],
|
|
268
|
+
metadata=[PromptMetadata(contexts[i])] if contexts else []
|
|
269
|
+
)
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
validation_set_builder = ValidationSetBuilder(name, self.__openapi_service)
|
|
273
|
+
for rapid in rapids:
|
|
274
|
+
validation_set_builder._add_rapid(rapid)
|
|
275
|
+
|
|
276
|
+
return validation_set_builder._submit(print_confirmation)
|
|
277
|
+
|
|
278
|
+
def create_mixed_set(self,
|
|
279
|
+
name: str,
|
|
280
|
+
rapids: Sequence[Rapid],
|
|
281
|
+
print_confirmation: bool = True
|
|
282
|
+
) -> RapidataValidationSet:
|
|
283
|
+
"""Create a validation set with a list of rapids.
|
|
284
|
+
|
|
285
|
+
Args:
|
|
286
|
+
name (str): The name of the validation set. (will not be shown to the labeler)
|
|
287
|
+
rapids (Sequence[Rapid]): The list of rapids to add to the validation set.
|
|
288
|
+
print_confirmation (bool, optional): Whether to print a confirmation message that validation set has been created. Defaults to True.
|
|
289
|
+
"""
|
|
290
|
+
|
|
291
|
+
validation_set_builder = ValidationSetBuilder(name, self.__openapi_service)
|
|
292
|
+
for rapid in rapids:
|
|
293
|
+
validation_set_builder._add_rapid(rapid)
|
|
294
|
+
|
|
295
|
+
return validation_set_builder._submit(print_confirmation)
|
|
296
|
+
|
|
297
|
+
def get_validation_set_by_id(self, validation_set_id: str) -> RapidataValidationSet:
|
|
298
|
+
"""Get a validation set by ID.
|
|
299
|
+
|
|
300
|
+
Args:
|
|
301
|
+
validation_set_id (str): The ID of the validation set.
|
|
302
|
+
|
|
303
|
+
Returns:
|
|
304
|
+
RapidataValidationSet: The ValidationSet instance.
|
|
305
|
+
"""
|
|
306
|
+
try:
|
|
307
|
+
validation_set = self.__openapi_service.validation_api.validation_get_by_id_get(id=validation_set_id)
|
|
308
|
+
except Exception:
|
|
309
|
+
raise ValueError(f"ValidationSet with ID {validation_set_id} not found.")
|
|
310
|
+
|
|
311
|
+
return RapidataValidationSet(validation_set_id, self.__openapi_service, validation_set.name)
|
|
312
|
+
|
|
313
|
+
def find_validation_sets(self, name: str = "", amount: int = 1) -> list[RapidataValidationSet]:
|
|
314
|
+
"""Find validation sets by name.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
name (str, optional): The name to search for. Defaults to "" to match with any set.
|
|
318
|
+
amount (int, optional): The amount of validation sets to return. Defaults to 1.
|
|
319
|
+
|
|
320
|
+
Returns:
|
|
321
|
+
list[RapidataValidationSet]: The list of validation sets.
|
|
322
|
+
"""
|
|
323
|
+
try:
|
|
324
|
+
validation_page_result = self.__openapi_service.validation_api.validation_query_validation_sets_get(QueryValidationSetModel(
|
|
325
|
+
pageInfo=PageInfo(index=1, size=amount),
|
|
326
|
+
filter=RootFilter(filters=[Filter(field="Name", operator="Contains", value=name)]),
|
|
327
|
+
sortCriteria=[SortCriterion(direction="Desc", propertyName="CreatedAt")]
|
|
328
|
+
))
|
|
329
|
+
|
|
330
|
+
except BadRequestException as e:
|
|
331
|
+
raise ValueError(f"Error occured during request. \nError: {e.body} \nTraceid: {e.headers.get('X-Trace-Id') if isinstance(e.headers, HTTPHeaderDict) else 'Unknown'}")
|
|
332
|
+
|
|
333
|
+
except Exception as e:
|
|
334
|
+
raise ValueError(f"Unknown error occured: {e}")
|
|
335
|
+
|
|
336
|
+
validation_sets = [self.get_validation_set_by_id(validation_set.id) for validation_set in validation_page_result.items]
|
|
337
|
+
return validation_sets
|
|
338
|
+
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
from .
|
|
2
|
-
from .
|
|
3
|
-
from .
|
|
4
|
-
from .
|
|
5
|
-
from .
|
|
6
|
-
from .
|
|
1
|
+
from ._locate_workflow import LocateWorkflow
|
|
2
|
+
from ._draw_workflow import DrawWorkflow
|
|
3
|
+
from ._base_workflow import Workflow
|
|
4
|
+
from ._classify_workflow import ClassifyWorkflow
|
|
5
|
+
from ._compare_workflow import CompareWorkflow
|
|
6
|
+
from ._free_text_workflow import FreeTextWorkflow
|
|
7
|
+
from ._select_words_workflow import SelectWordsWorkflow
|
|
8
|
+
from ._evaluation_workflow import EvaluationWorkflow
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from rapidata.api_client.models.simple_workflow_model import SimpleWorkflowModel
|
|
5
|
+
from rapidata.api_client.models.evaluation_workflow_model import EvaluationWorkflowModel
|
|
6
|
+
from rapidata.api_client.models.compare_workflow_model import CompareWorkflowModel
|
|
7
|
+
from rapidata.rapidata_client.referee._base_referee import Referee
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Workflow(ABC):
|
|
11
|
+
|
|
12
|
+
def __init__(self, type: str):
|
|
13
|
+
self._type = type
|
|
14
|
+
self._target_country_codes: list[str] = []
|
|
15
|
+
|
|
16
|
+
def _to_dict(self) -> dict[str, Any]:
|
|
17
|
+
return {
|
|
18
|
+
"_t": self._type,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
@abstractmethod
|
|
22
|
+
def _to_model(
|
|
23
|
+
self,
|
|
24
|
+
) -> SimpleWorkflowModel | CompareWorkflowModel | EvaluationWorkflowModel:
|
|
25
|
+
pass
|
|
@@ -21,14 +21,14 @@ class ClassifyWorkflow(Workflow):
|
|
|
21
21
|
_options (list[str]): The list of classification options.
|
|
22
22
|
"""
|
|
23
23
|
|
|
24
|
-
def __init__(self,
|
|
24
|
+
def __init__(self, instruction: str, answer_options: list[str]):
|
|
25
25
|
super().__init__(type="SimpleWorkflowConfig")
|
|
26
|
-
self._question =
|
|
27
|
-
self._options =
|
|
26
|
+
self._question = instruction
|
|
27
|
+
self._options = answer_options
|
|
28
28
|
|
|
29
|
-
def
|
|
29
|
+
def _to_dict(self) -> dict[str, Any]:
|
|
30
30
|
return {
|
|
31
|
-
**super().
|
|
31
|
+
**super()._to_dict(),
|
|
32
32
|
"blueprint": {
|
|
33
33
|
"_t": "ClassifyBlueprint",
|
|
34
34
|
"title": self._question,
|
|
@@ -36,7 +36,7 @@ class ClassifyWorkflow(Workflow):
|
|
|
36
36
|
}
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
def
|
|
39
|
+
def _to_model(self) -> SimpleWorkflowModel:
|
|
40
40
|
blueprint = AttachCategoryRapidBlueprint(
|
|
41
41
|
_t="ClassifyBlueprint",
|
|
42
42
|
title=self._question,
|
|
@@ -10,35 +10,29 @@ class CompareWorkflow(Workflow):
|
|
|
10
10
|
A workflow for comparison tasks.
|
|
11
11
|
|
|
12
12
|
This class represents a comparison workflow where items are compared based on
|
|
13
|
-
specified
|
|
13
|
+
specified instruction.
|
|
14
14
|
|
|
15
15
|
Attributes:
|
|
16
|
-
|
|
16
|
+
_instruction (str): The instruction used for comparison.
|
|
17
17
|
|
|
18
18
|
Args:
|
|
19
|
-
|
|
19
|
+
instruction (str): The instruction to be used for comparison.
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
|
-
def __init__(self,
|
|
23
|
-
"""
|
|
24
|
-
Initialize a CompareWorkflow instance.
|
|
25
|
-
|
|
26
|
-
Args:
|
|
27
|
-
criteria (str): The criteria to be used for comparison.
|
|
28
|
-
"""
|
|
22
|
+
def __init__(self, instruction: str):
|
|
29
23
|
super().__init__(type="CompareWorkflowConfig")
|
|
30
|
-
self.
|
|
24
|
+
self._instruction = instruction
|
|
31
25
|
|
|
32
|
-
def
|
|
26
|
+
def _to_dict(self) -> dict[str, Any]:
|
|
33
27
|
return {
|
|
34
|
-
**super().
|
|
35
|
-
"criteria": self.
|
|
28
|
+
**super()._to_dict(),
|
|
29
|
+
"criteria": self._instruction,
|
|
36
30
|
}
|
|
37
31
|
|
|
38
|
-
def
|
|
32
|
+
def _to_model(self) -> SimpleWorkflowModel:
|
|
39
33
|
blueprint = CompareRapidBlueprint(
|
|
40
34
|
_t="CompareBlueprint",
|
|
41
|
-
criteria=self.
|
|
35
|
+
criteria=self._instruction,
|
|
42
36
|
)
|
|
43
37
|
|
|
44
38
|
return SimpleWorkflowModel(
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from rapidata.api_client.models.simple_workflow_model import SimpleWorkflowModel
|
|
2
|
+
from rapidata.api_client.models.simple_workflow_model_blueprint import SimpleWorkflowModelBlueprint
|
|
3
|
+
from rapidata.api_client.models.line_rapid_blueprint import LineRapidBlueprint
|
|
4
|
+
from rapidata.rapidata_client.workflow._base_workflow import Workflow
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DrawWorkflow(Workflow):
|
|
8
|
+
|
|
9
|
+
def __init__(self, target: str):
|
|
10
|
+
super().__init__(type="SimpleWorkflowConfig")
|
|
11
|
+
self._target = target
|
|
12
|
+
|
|
13
|
+
def _to_model(self) -> SimpleWorkflowModel:
|
|
14
|
+
blueprint = LineRapidBlueprint(
|
|
15
|
+
_t="LineBlueprint",
|
|
16
|
+
target=self._target
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
return SimpleWorkflowModel(
|
|
20
|
+
_t="SimpleWorkflow",
|
|
21
|
+
blueprint=SimpleWorkflowModelBlueprint(blueprint)
|
|
22
|
+
)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from rapidata.api_client.models.evaluation_workflow_model import EvaluationWorkflowModel
|
|
2
|
+
from rapidata.rapidata_client.workflow._base_workflow import Workflow
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class EvaluationWorkflow(Workflow):
|
|
6
|
+
"""
|
|
7
|
+
A workflow to run evaluation orders.
|
|
8
|
+
|
|
9
|
+
This is used internally only and should not be necessary to be used by clients.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
validation_set_id (str): a source for the tasks that will be sent to the user
|
|
13
|
+
should_accept_incorrect (bool): indicates if the user should get feedback on their answers if they answer wrong. If set to true the user will not notice that he was tested.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(self, validation_set_id: str, should_accept_incorrect: bool):
|
|
17
|
+
super().__init__("EvaluationWorkflow")
|
|
18
|
+
self.validation_set_id = validation_set_id
|
|
19
|
+
self.should_accept_incorrect = should_accept_incorrect
|
|
20
|
+
|
|
21
|
+
def _to_model(self):
|
|
22
|
+
return EvaluationWorkflowModel(
|
|
23
|
+
_t="EvaluationWorkflow",
|
|
24
|
+
validationSetId=self.validation_set_id,
|
|
25
|
+
shouldAcceptIncorrect=self.should_accept_incorrect,
|
|
26
|
+
)
|
|
@@ -10,38 +10,32 @@ class FreeTextWorkflow(Workflow):
|
|
|
10
10
|
A workflow for free text input tasks.
|
|
11
11
|
|
|
12
12
|
This class represents a workflow where users can provide free-form text responses
|
|
13
|
-
to a given
|
|
13
|
+
to a given instruction.
|
|
14
14
|
|
|
15
15
|
Attributes:
|
|
16
|
-
|
|
16
|
+
_instruction (str): The instruction to be answered with free text.
|
|
17
17
|
|
|
18
18
|
Args:
|
|
19
|
-
|
|
19
|
+
instruction (str): The instruction to be presented for free text input.
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
|
-
def __init__(self,
|
|
23
|
-
"""
|
|
24
|
-
Initialize a FreeTextWorkflow instance.
|
|
25
|
-
|
|
26
|
-
Args:
|
|
27
|
-
question (str): The question to be presented for free text input.
|
|
28
|
-
"""
|
|
22
|
+
def __init__(self, instruction: str):
|
|
29
23
|
super().__init__(type="SimpleWorkflowConfig")
|
|
30
|
-
self.
|
|
24
|
+
self._instruction = instruction
|
|
31
25
|
|
|
32
|
-
def
|
|
26
|
+
def _to_dict(self) -> dict[str, Any]:
|
|
33
27
|
return {
|
|
34
|
-
**super().
|
|
28
|
+
**super()._to_dict(),
|
|
35
29
|
"blueprint": {
|
|
36
30
|
"_t": "FreeTextBlueprint",
|
|
37
|
-
"question": self.
|
|
31
|
+
"question": self._instruction,
|
|
38
32
|
},
|
|
39
33
|
}
|
|
40
34
|
|
|
41
|
-
def
|
|
35
|
+
def _to_model(self) -> SimpleWorkflowModel:
|
|
42
36
|
blueprint = FreeTextRapidBlueprint(
|
|
43
37
|
_t="FreeTextBlueprint",
|
|
44
|
-
question=self.
|
|
38
|
+
question=self._instruction,
|
|
45
39
|
)
|
|
46
40
|
|
|
47
41
|
return SimpleWorkflowModel(
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from rapidata.api_client.models.simple_workflow_model import SimpleWorkflowModel
|
|
2
|
+
from rapidata.api_client.models.simple_workflow_model_blueprint import SimpleWorkflowModelBlueprint
|
|
3
|
+
from rapidata.api_client.models.locate_rapid_blueprint import LocateRapidBlueprint
|
|
4
|
+
from rapidata.rapidata_client.workflow._base_workflow import Workflow
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LocateWorkflow(Workflow):
|
|
8
|
+
|
|
9
|
+
def __init__(self, target: str):
|
|
10
|
+
super().__init__(type="SimpleWorkflowConfig")
|
|
11
|
+
self._target = target
|
|
12
|
+
|
|
13
|
+
def _to_model(self) -> SimpleWorkflowModel:
|
|
14
|
+
blueprint = LocateRapidBlueprint(
|
|
15
|
+
_t="LocateBlueprint",
|
|
16
|
+
target=self._target
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
return SimpleWorkflowModel(
|
|
20
|
+
_t="SimpleWorkflow",
|
|
21
|
+
blueprint=SimpleWorkflowModelBlueprint(blueprint)
|
|
22
|
+
)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from rapidata.api_client.models.simple_workflow_model import SimpleWorkflowModel
|
|
2
2
|
from rapidata.api_client.models.simple_workflow_model_blueprint import SimpleWorkflowModelBlueprint
|
|
3
3
|
from rapidata.api_client.models.transcription_rapid_blueprint import TranscriptionRapidBlueprint
|
|
4
|
-
from rapidata.rapidata_client.workflow.
|
|
4
|
+
from rapidata.rapidata_client.workflow._base_workflow import Workflow
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class SelectWordsWorkflow(Workflow):
|
|
@@ -19,16 +19,10 @@ class SelectWordsWorkflow(Workflow):
|
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
def __init__(self, instruction: str):
|
|
22
|
-
"""
|
|
23
|
-
Initialize a SelectWordsWorkflow instance.
|
|
24
|
-
|
|
25
|
-
Args:
|
|
26
|
-
instruction (str): The instruction to be provided for the select words task.
|
|
27
|
-
"""
|
|
28
22
|
super().__init__(type="SimpleWorkflowConfig")
|
|
29
23
|
self._instruction = instruction
|
|
30
24
|
|
|
31
|
-
def
|
|
25
|
+
def _to_model(self) -> SimpleWorkflowModel:
|
|
32
26
|
blueprint = TranscriptionRapidBlueprint(
|
|
33
27
|
_t="TranscriptionBlueprint",
|
|
34
28
|
title=self._instruction
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
3
|
import time
|
|
4
|
+
import urllib.parse
|
|
4
5
|
import webbrowser
|
|
5
6
|
from datetime import datetime, timezone
|
|
6
7
|
from pathlib import Path
|
|
@@ -8,6 +9,7 @@ from socket import gethostname
|
|
|
8
9
|
from typing import Dict, List, Optional, Tuple
|
|
9
10
|
|
|
10
11
|
import requests
|
|
12
|
+
from colorama import Fore
|
|
11
13
|
from pydantic import BaseModel
|
|
12
14
|
|
|
13
15
|
|
|
@@ -206,7 +208,15 @@ class CredentialManager:
|
|
|
206
208
|
return None
|
|
207
209
|
|
|
208
210
|
auth_url = f"{self.endpoint}/connect/authorize/external?clientId=rapidata-cli&scope=openid profile email roles&writeKey={bridge_endpoint.write_key}"
|
|
209
|
-
webbrowser.open(auth_url)
|
|
211
|
+
could_open_browser = webbrowser.open(auth_url)
|
|
212
|
+
|
|
213
|
+
if not could_open_browser:
|
|
214
|
+
encoded_url = urllib.parse.quote(auth_url, safe="%/:=&?~#+!$,;'@()*[]")
|
|
215
|
+
print(
|
|
216
|
+
Fore.RED
|
|
217
|
+
+ f'Please open the following URL in your browser to log in: "{encoded_url}"'
|
|
218
|
+
+ Fore.RESET
|
|
219
|
+
)
|
|
210
220
|
|
|
211
221
|
access_token = self._poll_read_key(bridge_endpoint.read_key)
|
|
212
222
|
if not access_token:
|