rapidata 2.21.4__py3-none-any.whl → 2.22.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rapidata might be problematic. Click here for more details.
- rapidata/__init__.py +5 -0
- rapidata/api_client/__init__.py +8 -4
- rapidata/api_client/api/__init__.py +1 -0
- rapidata/api_client/api/evaluation_workflow_api.py +372 -0
- rapidata/api_client/api/identity_api.py +268 -0
- rapidata/api_client/api/rapid_api.py +353 -1987
- rapidata/api_client/api/simple_workflow_api.py +6 -6
- rapidata/api_client/models/__init__.py +7 -4
- rapidata/api_client/models/add_campaign_model.py +25 -1
- rapidata/api_client/models/add_validation_rapid_model_truth.py +24 -10
- rapidata/api_client/models/compare_result.py +2 -0
- rapidata/api_client/models/create_order_model.py +43 -2
- rapidata/api_client/models/evaluation_workflow_model1.py +115 -0
- rapidata/api_client/models/filter.py +2 -2
- rapidata/api_client/models/get_validation_rapids_result.py +11 -4
- rapidata/api_client/models/get_validation_rapids_result_truth.py +24 -10
- rapidata/api_client/models/get_workflow_by_id_result_workflow.py +23 -9
- rapidata/api_client/models/get_workflow_results_result.py +118 -0
- rapidata/api_client/models/get_workflow_results_result_paged_result.py +105 -0
- rapidata/api_client/models/google_one_tap_login_model.py +87 -0
- rapidata/api_client/models/labeling_selection.py +22 -3
- rapidata/api_client/models/logic_operator.py +1 -0
- rapidata/api_client/models/rapid_response.py +3 -1
- rapidata/api_client/models/retrieval_mode.py +38 -0
- rapidata/api_client/models/root_filter.py +2 -2
- rapidata/api_client/models/skip_truth.py +94 -0
- rapidata/api_client/models/sticky_state.py +38 -0
- rapidata/api_client/models/update_validation_rapid_model.py +11 -4
- rapidata/api_client/models/update_validation_rapid_model_truth.py +24 -10
- rapidata/api_client/rest.py +1 -0
- rapidata/api_client_README.md +10 -11
- rapidata/rapidata_client/__init__.py +7 -0
- rapidata/rapidata_client/api/rapidata_exception.py +5 -3
- rapidata/rapidata_client/assets/__init__.py +1 -0
- rapidata/rapidata_client/assets/_media_asset.py +16 -10
- rapidata/rapidata_client/assets/_multi_asset.py +6 -0
- rapidata/rapidata_client/assets/_sessions.py +35 -0
- rapidata/rapidata_client/assets/_text_asset.py +6 -0
- rapidata/rapidata_client/demographic/demographic_manager.py +2 -35
- rapidata/rapidata_client/logging/__init__.py +2 -0
- rapidata/rapidata_client/logging/logger.py +47 -0
- rapidata/rapidata_client/logging/output_manager.py +16 -0
- rapidata/rapidata_client/order/_rapidata_dataset.py +11 -15
- rapidata/rapidata_client/order/_rapidata_order_builder.py +15 -2
- rapidata/rapidata_client/order/rapidata_order.py +23 -14
- rapidata/rapidata_client/order/rapidata_order_manager.py +4 -2
- rapidata/rapidata_client/order/rapidata_results.py +2 -1
- rapidata/rapidata_client/rapidata_client.py +6 -1
- rapidata/rapidata_client/selection/__init__.py +1 -0
- rapidata/rapidata_client/selection/labeling_selection.py +8 -2
- rapidata/rapidata_client/selection/retrieval_modes.py +9 -0
- rapidata/rapidata_client/settings/alert_on_fast_response.py +2 -1
- rapidata/rapidata_client/settings/free_text_minimum_characters.py +2 -1
- rapidata/rapidata_client/validation/rapidata_validation_set.py +4 -34
- rapidata/rapidata_client/validation/rapids/rapids.py +6 -7
- rapidata/rapidata_client/validation/validation_set_manager.py +39 -36
- rapidata/service/credential_manager.py +22 -30
- rapidata/service/openapi_service.py +11 -0
- {rapidata-2.21.4.dist-info → rapidata-2.22.0.dist-info}/METADATA +2 -1
- {rapidata-2.21.4.dist-info → rapidata-2.22.0.dist-info}/RECORD +62 -49
- {rapidata-2.21.4.dist-info → rapidata-2.22.0.dist-info}/WHEEL +1 -1
- {rapidata-2.21.4.dist-info → rapidata-2.22.0.dist-info}/LICENSE +0 -0
|
@@ -17,8 +17,8 @@ import pprint
|
|
|
17
17
|
import re # noqa: F401
|
|
18
18
|
import json
|
|
19
19
|
|
|
20
|
-
from pydantic import BaseModel, ConfigDict, StrictStr
|
|
21
|
-
from typing import Any, ClassVar, Dict, List, Optional
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr
|
|
21
|
+
from typing import Any, ClassVar, Dict, List, Optional, Union
|
|
22
22
|
from rapidata.api_client.models.update_validation_rapid_model_truth import UpdateValidationRapidModelTruth
|
|
23
23
|
from typing import Optional, Set
|
|
24
24
|
from typing_extensions import Self
|
|
@@ -30,7 +30,8 @@ class UpdateValidationRapidModel(BaseModel):
|
|
|
30
30
|
truth: UpdateValidationRapidModelTruth
|
|
31
31
|
explanation: Optional[StrictStr] = None
|
|
32
32
|
prompt: Optional[StrictStr] = None
|
|
33
|
-
|
|
33
|
+
random_correct_probability: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, alias="randomCorrectProbability")
|
|
34
|
+
__properties: ClassVar[List[str]] = ["truth", "explanation", "prompt", "randomCorrectProbability"]
|
|
34
35
|
|
|
35
36
|
model_config = ConfigDict(
|
|
36
37
|
populate_by_name=True,
|
|
@@ -84,6 +85,11 @@ class UpdateValidationRapidModel(BaseModel):
|
|
|
84
85
|
if self.prompt is None and "prompt" in self.model_fields_set:
|
|
85
86
|
_dict['prompt'] = None
|
|
86
87
|
|
|
88
|
+
# set to None if random_correct_probability (nullable) is None
|
|
89
|
+
# and model_fields_set contains the field
|
|
90
|
+
if self.random_correct_probability is None and "random_correct_probability" in self.model_fields_set:
|
|
91
|
+
_dict['randomCorrectProbability'] = None
|
|
92
|
+
|
|
87
93
|
return _dict
|
|
88
94
|
|
|
89
95
|
@classmethod
|
|
@@ -98,7 +104,8 @@ class UpdateValidationRapidModel(BaseModel):
|
|
|
98
104
|
_obj = cls.model_validate({
|
|
99
105
|
"truth": UpdateValidationRapidModelTruth.from_dict(obj["truth"]) if obj.get("truth") is not None else None,
|
|
100
106
|
"explanation": obj.get("explanation"),
|
|
101
|
-
"prompt": obj.get("prompt")
|
|
107
|
+
"prompt": obj.get("prompt"),
|
|
108
|
+
"randomCorrectProbability": obj.get("randomCorrectProbability")
|
|
102
109
|
})
|
|
103
110
|
return _obj
|
|
104
111
|
|
|
@@ -26,12 +26,13 @@ from rapidata.api_client.models.locate_box_truth import LocateBoxTruth
|
|
|
26
26
|
from rapidata.api_client.models.named_entity_truth import NamedEntityTruth
|
|
27
27
|
from rapidata.api_client.models.polygon_truth import PolygonTruth
|
|
28
28
|
from rapidata.api_client.models.scrub_truth import ScrubTruth
|
|
29
|
+
from rapidata.api_client.models.skip_truth import SkipTruth
|
|
29
30
|
from rapidata.api_client.models.transcription_truth import TranscriptionTruth
|
|
30
31
|
from pydantic import StrictStr, Field
|
|
31
32
|
from typing import Union, List, Set, Optional, Dict
|
|
32
33
|
from typing_extensions import Literal, Self
|
|
33
34
|
|
|
34
|
-
UPDATEVALIDATIONRAPIDMODELTRUTH_ONE_OF_SCHEMAS = ["AttachCategoryTruth", "BoundingBoxTruth", "CompareTruth", "EmptyValidationTruth", "LineTruth", "LocateBoxTruth", "NamedEntityTruth", "PolygonTruth", "ScrubTruth", "TranscriptionTruth"]
|
|
35
|
+
UPDATEVALIDATIONRAPIDMODELTRUTH_ONE_OF_SCHEMAS = ["AttachCategoryTruth", "BoundingBoxTruth", "CompareTruth", "EmptyValidationTruth", "LineTruth", "LocateBoxTruth", "NamedEntityTruth", "PolygonTruth", "ScrubTruth", "SkipTruth", "TranscriptionTruth"]
|
|
35
36
|
|
|
36
37
|
class UpdateValidationRapidModelTruth(BaseModel):
|
|
37
38
|
"""
|
|
@@ -53,12 +54,14 @@ class UpdateValidationRapidModelTruth(BaseModel):
|
|
|
53
54
|
oneof_schema_7_validator: Optional[EmptyValidationTruth] = None
|
|
54
55
|
# data type: CompareTruth
|
|
55
56
|
oneof_schema_8_validator: Optional[CompareTruth] = None
|
|
57
|
+
# data type: SkipTruth
|
|
58
|
+
oneof_schema_9_validator: Optional[SkipTruth] = None
|
|
56
59
|
# data type: AttachCategoryTruth
|
|
57
|
-
|
|
60
|
+
oneof_schema_10_validator: Optional[AttachCategoryTruth] = None
|
|
58
61
|
# data type: BoundingBoxTruth
|
|
59
|
-
|
|
60
|
-
actual_instance: Optional[Union[AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, TranscriptionTruth]] = None
|
|
61
|
-
one_of_schemas: Set[str] = { "AttachCategoryTruth", "BoundingBoxTruth", "CompareTruth", "EmptyValidationTruth", "LineTruth", "LocateBoxTruth", "NamedEntityTruth", "PolygonTruth", "ScrubTruth", "TranscriptionTruth" }
|
|
62
|
+
oneof_schema_11_validator: Optional[BoundingBoxTruth] = None
|
|
63
|
+
actual_instance: Optional[Union[AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, SkipTruth, TranscriptionTruth]] = None
|
|
64
|
+
one_of_schemas: Set[str] = { "AttachCategoryTruth", "BoundingBoxTruth", "CompareTruth", "EmptyValidationTruth", "LineTruth", "LocateBoxTruth", "NamedEntityTruth", "PolygonTruth", "ScrubTruth", "SkipTruth", "TranscriptionTruth" }
|
|
62
65
|
|
|
63
66
|
model_config = ConfigDict(
|
|
64
67
|
validate_assignment=True,
|
|
@@ -124,6 +127,11 @@ class UpdateValidationRapidModelTruth(BaseModel):
|
|
|
124
127
|
error_messages.append(f"Error! Input type `{type(v)}` is not `CompareTruth`")
|
|
125
128
|
else:
|
|
126
129
|
match += 1
|
|
130
|
+
# validate data type: SkipTruth
|
|
131
|
+
if not isinstance(v, SkipTruth):
|
|
132
|
+
error_messages.append(f"Error! Input type `{type(v)}` is not `SkipTruth`")
|
|
133
|
+
else:
|
|
134
|
+
match += 1
|
|
127
135
|
# validate data type: AttachCategoryTruth
|
|
128
136
|
if not isinstance(v, AttachCategoryTruth):
|
|
129
137
|
error_messages.append(f"Error! Input type `{type(v)}` is not `AttachCategoryTruth`")
|
|
@@ -136,10 +144,10 @@ class UpdateValidationRapidModelTruth(BaseModel):
|
|
|
136
144
|
match += 1
|
|
137
145
|
if match > 1:
|
|
138
146
|
# more than 1 match
|
|
139
|
-
raise ValueError("Multiple matches found when setting `actual_instance` in UpdateValidationRapidModelTruth with oneOf schemas: AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, TranscriptionTruth. Details: " + ", ".join(error_messages))
|
|
147
|
+
raise ValueError("Multiple matches found when setting `actual_instance` in UpdateValidationRapidModelTruth with oneOf schemas: AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, SkipTruth, TranscriptionTruth. Details: " + ", ".join(error_messages))
|
|
140
148
|
elif match == 0:
|
|
141
149
|
# no match
|
|
142
|
-
raise ValueError("No match found when setting `actual_instance` in UpdateValidationRapidModelTruth with oneOf schemas: AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, TranscriptionTruth. Details: " + ", ".join(error_messages))
|
|
150
|
+
raise ValueError("No match found when setting `actual_instance` in UpdateValidationRapidModelTruth with oneOf schemas: AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, SkipTruth, TranscriptionTruth. Details: " + ", ".join(error_messages))
|
|
143
151
|
else:
|
|
144
152
|
return v
|
|
145
153
|
|
|
@@ -202,6 +210,12 @@ class UpdateValidationRapidModelTruth(BaseModel):
|
|
|
202
210
|
match += 1
|
|
203
211
|
except (ValidationError, ValueError) as e:
|
|
204
212
|
error_messages.append(str(e))
|
|
213
|
+
# deserialize data into SkipTruth
|
|
214
|
+
try:
|
|
215
|
+
instance.actual_instance = SkipTruth.from_json(json_str)
|
|
216
|
+
match += 1
|
|
217
|
+
except (ValidationError, ValueError) as e:
|
|
218
|
+
error_messages.append(str(e))
|
|
205
219
|
# deserialize data into AttachCategoryTruth
|
|
206
220
|
try:
|
|
207
221
|
instance.actual_instance = AttachCategoryTruth.from_json(json_str)
|
|
@@ -217,10 +231,10 @@ class UpdateValidationRapidModelTruth(BaseModel):
|
|
|
217
231
|
|
|
218
232
|
if match > 1:
|
|
219
233
|
# more than 1 match
|
|
220
|
-
raise ValueError("Multiple matches found when deserializing the JSON string into UpdateValidationRapidModelTruth with oneOf schemas: AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, TranscriptionTruth. Details: " + ", ".join(error_messages))
|
|
234
|
+
raise ValueError("Multiple matches found when deserializing the JSON string into UpdateValidationRapidModelTruth with oneOf schemas: AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, SkipTruth, TranscriptionTruth. Details: " + ", ".join(error_messages))
|
|
221
235
|
elif match == 0:
|
|
222
236
|
# no match
|
|
223
|
-
raise ValueError("No match found when deserializing the JSON string into UpdateValidationRapidModelTruth with oneOf schemas: AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, TranscriptionTruth. Details: " + ", ".join(error_messages))
|
|
237
|
+
raise ValueError("No match found when deserializing the JSON string into UpdateValidationRapidModelTruth with oneOf schemas: AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, SkipTruth, TranscriptionTruth. Details: " + ", ".join(error_messages))
|
|
224
238
|
else:
|
|
225
239
|
return instance
|
|
226
240
|
|
|
@@ -234,7 +248,7 @@ class UpdateValidationRapidModelTruth(BaseModel):
|
|
|
234
248
|
else:
|
|
235
249
|
return json.dumps(self.actual_instance)
|
|
236
250
|
|
|
237
|
-
def to_dict(self) -> Optional[Union[Dict[str, Any], AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, TranscriptionTruth]]:
|
|
251
|
+
def to_dict(self) -> Optional[Union[Dict[str, Any], AttachCategoryTruth, BoundingBoxTruth, CompareTruth, EmptyValidationTruth, LineTruth, LocateBoxTruth, NamedEntityTruth, PolygonTruth, ScrubTruth, SkipTruth, TranscriptionTruth]]:
|
|
238
252
|
"""Returns the dict representation of the actual instance"""
|
|
239
253
|
if self.actual_instance is None:
|
|
240
254
|
return None
|
rapidata/api_client/rest.py
CHANGED
rapidata/api_client_README.md
CHANGED
|
@@ -110,11 +110,13 @@ Class | Method | HTTP request | Description
|
|
|
110
110
|
*DatasetApi* | [**dataset_import_post**](rapidata/api_client/docs/DatasetApi.md#dataset_import_post) | **POST** /dataset/import | Imports datapoints from a csv file.
|
|
111
111
|
*DatasetApi* | [**dataset_updatename_post**](rapidata/api_client/docs/DatasetApi.md#dataset_updatename_post) | **POST** /dataset/updatename | Updates the name of a dataset.
|
|
112
112
|
*DatasetApi* | [**dataset_uploadfilesfroms3_post**](rapidata/api_client/docs/DatasetApi.md#dataset_uploadfilesfroms3_post) | **POST** /dataset/uploadfilesfroms3 | Uploads files from an S3 bucket to a dataset.
|
|
113
|
+
*EvaluationWorkflowApi* | [**workflow_evaluation_workflow_id_results_get**](rapidata/api_client/docs/EvaluationWorkflowApi.md#workflow_evaluation_workflow_id_results_get) | **GET** /workflow/evaluation/{workflowId}/results | Get the results for an evaluation workflow.
|
|
113
114
|
*FeedbackApi* | [**feedback_feedback_post**](rapidata/api_client/docs/FeedbackApi.md#feedback_feedback_post) | **POST** /feedback/feedback | Submits feedback about our services.
|
|
114
115
|
*FeedbackApi* | [**feedback_post**](rapidata/api_client/docs/FeedbackApi.md#feedback_post) | **POST** /feedback | Submits feedback about our services.
|
|
115
116
|
*IdentityApi* | [**identity_bridge_token_get**](rapidata/api_client/docs/IdentityApi.md#identity_bridge_token_get) | **GET** /identity/bridge-token | Tries to read the bridge token keys for a given read key. The read key is used to retrieve the authentication result written by the write key.
|
|
116
117
|
*IdentityApi* | [**identity_bridge_token_post**](rapidata/api_client/docs/IdentityApi.md#identity_bridge_token_post) | **POST** /identity/bridge-token | Creates a pair of read and write keys for a client. The write key is used to store the authentication result. The read key is used to retrieve the authentication result.
|
|
117
118
|
*IdentityApi* | [**identity_createbridgetoken_post**](rapidata/api_client/docs/IdentityApi.md#identity_createbridgetoken_post) | **POST** /identity/createbridgetoken | Creates a pair of read and write keys for a client. The write key is used to store the authentication result. The read key is used to retrieve the authentication result.
|
|
119
|
+
*IdentityApi* | [**identity_google_one_tap_post**](rapidata/api_client/docs/IdentityApi.md#identity_google_one_tap_post) | **POST** /identity/google-one-tap | Signs in a user using a token received from Google One Tap.
|
|
118
120
|
*IdentityApi* | [**identity_readbridgetoken_get**](rapidata/api_client/docs/IdentityApi.md#identity_readbridgetoken_get) | **GET** /identity/readbridgetoken | Tries to read the bridge token keys for a given read key. The read key is used to retrieve the authentication result written by the write key.
|
|
119
121
|
*IdentityApi* | [**identity_referrer_post**](rapidata/api_client/docs/IdentityApi.md#identity_referrer_post) | **POST** /identity/referrer | Sets the referrer for the current customer.
|
|
120
122
|
*IdentityApi* | [**identity_registertemporary_post**](rapidata/api_client/docs/IdentityApi.md#identity_registertemporary_post) | **POST** /identity/registertemporary | Registers and logs in a temporary customer.
|
|
@@ -170,21 +172,15 @@ Class | Method | HTTP request | Description
|
|
|
170
172
|
*PipelineApi* | [**pipeline_pipeline_id_get**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_get) | **GET** /pipeline/{pipelineId} | Gets a pipeline by its id.
|
|
171
173
|
*PipelineApi* | [**pipeline_pipeline_id_preliminary_download_post**](rapidata/api_client/docs/PipelineApi.md#pipeline_pipeline_id_preliminary_download_post) | **POST** /pipeline/{pipelineId}/preliminary-download | Initiates a preliminary download of the pipeline.
|
|
172
174
|
*PipelineApi* | [**pipeline_preliminary_download_preliminary_download_id_get**](rapidata/api_client/docs/PipelineApi.md#pipeline_preliminary_download_preliminary_download_id_get) | **GET** /pipeline/preliminary-download/{preliminaryDownloadId} | Gets the preliminary download.
|
|
173
|
-
*RapidApi* | [**rapid_adduserguess_post**](rapidata/api_client/docs/RapidApi.md#rapid_adduserguess_post) | **POST** /rapid/adduserguess | Submits a user guess for a Rapid.
|
|
174
|
-
*RapidApi* | [**rapid_createdemographicrapid_post**](rapidata/api_client/docs/RapidApi.md#rapid_createdemographicrapid_post) | **POST** /rapid/createdemographicrapid | Creates a new Demographic Rapid.
|
|
175
175
|
*RapidApi* | [**rapid_demographic_post**](rapidata/api_client/docs/RapidApi.md#rapid_demographic_post) | **POST** /rapid/demographic | Creates a new Demographic Rapid.
|
|
176
|
-
*RapidApi* | [**rapid_queryvalidationrapids_get**](rapidata/api_client/docs/RapidApi.md#rapid_queryvalidationrapids_get) | **GET** /rapid/queryvalidationrapids | Queries the validation rapids for a specific validation set.
|
|
177
176
|
*RapidApi* | [**rapid_rapid_bag_is_valid_get**](rapidata/api_client/docs/RapidApi.md#rapid_rapid_bag_is_valid_get) | **GET** /rapid/rapid-bag/is-valid | Validates that the rapids associated with the current user are active.
|
|
178
177
|
*RapidApi* | [**rapid_rapid_id_delete**](rapidata/api_client/docs/RapidApi.md#rapid_rapid_id_delete) | **DELETE** /rapid/{rapidId} | Deletes a rapid.
|
|
179
178
|
*RapidApi* | [**rapid_rapid_id_report_post**](rapidata/api_client/docs/RapidApi.md#rapid_rapid_id_report_post) | **POST** /rapid/{rapidId}/report | Used to report an issue with a rapid.
|
|
180
179
|
*RapidApi* | [**rapid_rapid_id_responses_get**](rapidata/api_client/docs/RapidApi.md#rapid_rapid_id_responses_get) | **GET** /rapid/{rapidId}/responses | Gets all responses for a given rapid.
|
|
181
|
-
*RapidApi* | [**rapid_report_report_id_get**](rapidata/api_client/docs/RapidApi.md#rapid_report_report_id_get) | **GET** /rapid/report/{reportId} | Inspects a report's dump.
|
|
180
|
+
*RapidApi* | [**rapid_report_report_id_get**](rapidata/api_client/docs/RapidApi.md#rapid_report_report_id_get) | **GET** /rapid/report/{reportId} | Inspects a report's dump. Can be used to restore zustand state or anything alike.
|
|
182
181
|
*RapidApi* | [**rapid_response_post**](rapidata/api_client/docs/RapidApi.md#rapid_response_post) | **POST** /rapid/response | Submits a response for a Rapid.
|
|
183
182
|
*RapidApi* | [**rapid_skip_post**](rapidata/api_client/docs/RapidApi.md#rapid_skip_post) | **POST** /rapid/skip | Skips a Rapid for the user.
|
|
184
|
-
*RapidApi* | [**rapid_skipuserguess_post**](rapidata/api_client/docs/RapidApi.md#rapid_skipuserguess_post) | **POST** /rapid/skipuserguess | Skips a Rapid for the user.
|
|
185
|
-
*RapidApi* | [**rapid_validatecurrentrapidbag_get**](rapidata/api_client/docs/RapidApi.md#rapid_validatecurrentrapidbag_get) | **GET** /rapid/validatecurrentrapidbag | Validates that the rapids associated with the current user are active.
|
|
186
183
|
*RapidApi* | [**rapid_validation_rapid_id_patch**](rapidata/api_client/docs/RapidApi.md#rapid_validation_rapid_id_patch) | **PATCH** /rapid/validation/{rapidId} | Updates the validation information of a rapid.
|
|
187
|
-
*RapidApi* | [**rapid_validation_rapid_id_put**](rapidata/api_client/docs/RapidApi.md#rapid_validation_rapid_id_put) | **PUT** /rapid/validation/{rapidId} | Updates the validation information of a rapid.
|
|
188
184
|
*RapidataIdentityAPIApi* | [**root_get**](rapidata/api_client/docs/RapidataIdentityAPIApi.md#root_get) | **GET** / |
|
|
189
185
|
*SimpleWorkflowApi* | [**workflow_simple_workflow_id_results_get**](rapidata/api_client/docs/SimpleWorkflowApi.md#workflow_simple_workflow_id_results_get) | **GET** /workflow/simple/{workflowId}/results | Get the result overview for a simple workflow.
|
|
190
186
|
*UserInfoApi* | [**connect_userinfo_get**](rapidata/api_client/docs/UserInfoApi.md#connect_userinfo_get) | **GET** /connect/userinfo | Retrieves information about the authenticated user.
|
|
@@ -325,6 +321,7 @@ Class | Method | HTTP request | Description
|
|
|
325
321
|
- [EmptyValidationTruth](rapidata/api_client/docs/EmptyValidationTruth.md)
|
|
326
322
|
- [EvaluationWorkflowConfig](rapidata/api_client/docs/EvaluationWorkflowConfig.md)
|
|
327
323
|
- [EvaluationWorkflowModel](rapidata/api_client/docs/EvaluationWorkflowModel.md)
|
|
324
|
+
- [EvaluationWorkflowModel1](rapidata/api_client/docs/EvaluationWorkflowModel1.md)
|
|
328
325
|
- [FeatureFlag](rapidata/api_client/docs/FeatureFlag.md)
|
|
329
326
|
- [FeatureFlagModel](rapidata/api_client/docs/FeatureFlagModel.md)
|
|
330
327
|
- [FeedbackModel](rapidata/api_client/docs/FeedbackModel.md)
|
|
@@ -355,8 +352,6 @@ Class | Method | HTTP request | Description
|
|
|
355
352
|
- [GetRapidResponsesResult](rapidata/api_client/docs/GetRapidResponsesResult.md)
|
|
356
353
|
- [GetResponsesResult](rapidata/api_client/docs/GetResponsesResult.md)
|
|
357
354
|
- [GetSimpleWorkflowResultsModel](rapidata/api_client/docs/GetSimpleWorkflowResultsModel.md)
|
|
358
|
-
- [GetSimpleWorkflowResultsResult](rapidata/api_client/docs/GetSimpleWorkflowResultsResult.md)
|
|
359
|
-
- [GetSimpleWorkflowResultsResultPagedResult](rapidata/api_client/docs/GetSimpleWorkflowResultsResultPagedResult.md)
|
|
360
355
|
- [GetValidationRapidsResult](rapidata/api_client/docs/GetValidationRapidsResult.md)
|
|
361
356
|
- [GetValidationRapidsResultAsset](rapidata/api_client/docs/GetValidationRapidsResultAsset.md)
|
|
362
357
|
- [GetValidationRapidsResultPagedResult](rapidata/api_client/docs/GetValidationRapidsResultPagedResult.md)
|
|
@@ -366,6 +361,9 @@ Class | Method | HTTP request | Description
|
|
|
366
361
|
- [GetWorkflowByIdResult](rapidata/api_client/docs/GetWorkflowByIdResult.md)
|
|
367
362
|
- [GetWorkflowByIdResultWorkflow](rapidata/api_client/docs/GetWorkflowByIdResultWorkflow.md)
|
|
368
363
|
- [GetWorkflowProgressResult](rapidata/api_client/docs/GetWorkflowProgressResult.md)
|
|
364
|
+
- [GetWorkflowResultsResult](rapidata/api_client/docs/GetWorkflowResultsResult.md)
|
|
365
|
+
- [GetWorkflowResultsResultPagedResult](rapidata/api_client/docs/GetWorkflowResultsResultPagedResult.md)
|
|
366
|
+
- [GoogleOneTapLoginModel](rapidata/api_client/docs/GoogleOneTapLoginModel.md)
|
|
369
367
|
- [IWorkflowModelPagedResult](rapidata/api_client/docs/IWorkflowModelPagedResult.md)
|
|
370
368
|
- [ImageDimensionMetadata](rapidata/api_client/docs/ImageDimensionMetadata.md)
|
|
371
369
|
- [ImageDimensionMetadataModel](rapidata/api_client/docs/ImageDimensionMetadataModel.md)
|
|
@@ -429,8 +427,6 @@ Class | Method | HTTP request | Description
|
|
|
429
427
|
- [PublicOrderModel](rapidata/api_client/docs/PublicOrderModel.md)
|
|
430
428
|
- [PublicTextMetadataInput](rapidata/api_client/docs/PublicTextMetadataInput.md)
|
|
431
429
|
- [QueryModel](rapidata/api_client/docs/QueryModel.md)
|
|
432
|
-
- [QueryValidationRapidsResult](rapidata/api_client/docs/QueryValidationRapidsResult.md)
|
|
433
|
-
- [QueryValidationRapidsResultPagedResult](rapidata/api_client/docs/QueryValidationRapidsResultPagedResult.md)
|
|
434
430
|
- [QueryWorkflowsModel](rapidata/api_client/docs/QueryWorkflowsModel.md)
|
|
435
431
|
- [RapidIssue](rapidata/api_client/docs/RapidIssue.md)
|
|
436
432
|
- [RapidResponse](rapidata/api_client/docs/RapidResponse.md)
|
|
@@ -444,6 +440,7 @@ Class | Method | HTTP request | Description
|
|
|
444
440
|
- [RegisterTemporaryCustomerResult](rapidata/api_client/docs/RegisterTemporaryCustomerResult.md)
|
|
445
441
|
- [ReportModel](rapidata/api_client/docs/ReportModel.md)
|
|
446
442
|
- [ResponseCountUserFilterModel](rapidata/api_client/docs/ResponseCountUserFilterModel.md)
|
|
443
|
+
- [RetrievalMode](rapidata/api_client/docs/RetrievalMode.md)
|
|
447
444
|
- [RootFilter](rapidata/api_client/docs/RootFilter.md)
|
|
448
445
|
- [ScrubPayload](rapidata/api_client/docs/ScrubPayload.md)
|
|
449
446
|
- [ScrubRange](rapidata/api_client/docs/ScrubRange.md)
|
|
@@ -460,11 +457,13 @@ Class | Method | HTTP request | Description
|
|
|
460
457
|
- [SimpleWorkflowModel1](rapidata/api_client/docs/SimpleWorkflowModel1.md)
|
|
461
458
|
- [SimpleWorkflowModelBlueprint](rapidata/api_client/docs/SimpleWorkflowModelBlueprint.md)
|
|
462
459
|
- [SkipResult](rapidata/api_client/docs/SkipResult.md)
|
|
460
|
+
- [SkipTruth](rapidata/api_client/docs/SkipTruth.md)
|
|
463
461
|
- [SortCriterion](rapidata/api_client/docs/SortCriterion.md)
|
|
464
462
|
- [SortDirection](rapidata/api_client/docs/SortDirection.md)
|
|
465
463
|
- [SourceUrlMetadata](rapidata/api_client/docs/SourceUrlMetadata.md)
|
|
466
464
|
- [SourceUrlMetadataModel](rapidata/api_client/docs/SourceUrlMetadataModel.md)
|
|
467
465
|
- [StaticSelection](rapidata/api_client/docs/StaticSelection.md)
|
|
466
|
+
- [StickyState](rapidata/api_client/docs/StickyState.md)
|
|
468
467
|
- [SubmitCocoModel](rapidata/api_client/docs/SubmitCocoModel.md)
|
|
469
468
|
- [SubmitCocoResult](rapidata/api_client/docs/SubmitCocoResult.md)
|
|
470
469
|
- [TextAssetModel](rapidata/api_client/docs/TextAssetModel.md)
|
|
@@ -6,6 +6,7 @@ from .selection import (
|
|
|
6
6
|
ConditionalValidationSelection,
|
|
7
7
|
CappedSelection,
|
|
8
8
|
ShufflingSelection,
|
|
9
|
+
RetrievalMode,
|
|
9
10
|
)
|
|
10
11
|
from .metadata import (
|
|
11
12
|
PrivateTextMetadata,
|
|
@@ -45,4 +46,10 @@ from .filter import (
|
|
|
45
46
|
ResponseCountFilter,
|
|
46
47
|
)
|
|
47
48
|
|
|
49
|
+
from .logging import (
|
|
50
|
+
configure_logger,
|
|
51
|
+
logger,
|
|
52
|
+
OutputManger
|
|
53
|
+
)
|
|
54
|
+
|
|
48
55
|
from .validation import Box
|
|
@@ -2,7 +2,7 @@ from typing import Optional, Any
|
|
|
2
2
|
from rapidata.api_client.api_client import ApiClient, rest, ApiResponse, ApiResponseT
|
|
3
3
|
from rapidata.api_client.exceptions import ApiException
|
|
4
4
|
import json
|
|
5
|
-
|
|
5
|
+
from rapidata.rapidata_client.logging import logger
|
|
6
6
|
|
|
7
7
|
class RapidataError(Exception):
|
|
8
8
|
"""Custom error class for Rapidata API errors."""
|
|
@@ -97,9 +97,11 @@ class RapidataApiClient(ApiClient):
|
|
|
97
97
|
# If we can't parse the body as JSON, use the original message
|
|
98
98
|
pass
|
|
99
99
|
|
|
100
|
-
|
|
100
|
+
error_formatted = RapidataError(
|
|
101
101
|
status_code=status_code,
|
|
102
102
|
message=message,
|
|
103
103
|
original_exception=e,
|
|
104
104
|
details=details
|
|
105
|
-
)
|
|
105
|
+
)
|
|
106
|
+
logger.error(f"Error: {error_formatted}")
|
|
107
|
+
raise error_formatted from None
|
|
@@ -4,7 +4,7 @@ Defines the MediaAsset class for handling media file paths within assets.
|
|
|
4
4
|
Implements lazy loading for URL-based media to prevent unnecessary downloads.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
from typing import Optional, cast
|
|
7
|
+
from typing import Optional, cast
|
|
8
8
|
import os
|
|
9
9
|
from io import BytesIO
|
|
10
10
|
from rapidata.rapidata_client.assets._base_asset import BaseAsset
|
|
@@ -16,8 +16,8 @@ import tempfile
|
|
|
16
16
|
from pydantic import StrictStr, StrictBytes
|
|
17
17
|
import logging
|
|
18
18
|
from functools import cached_property
|
|
19
|
-
from
|
|
20
|
-
from
|
|
19
|
+
from rapidata.rapidata_client.assets._sessions import SessionManager
|
|
20
|
+
from rapidata.rapidata_client.logging import logger
|
|
21
21
|
|
|
22
22
|
class MediaAsset(BaseAsset):
|
|
23
23
|
"""MediaAsset Class with Lazy Loading
|
|
@@ -78,7 +78,7 @@ class MediaAsset(BaseAsset):
|
|
|
78
78
|
|
|
79
79
|
self._url = None
|
|
80
80
|
self._content = None
|
|
81
|
-
self.session:
|
|
81
|
+
self.session: requests.Session = SessionManager.get_session()
|
|
82
82
|
|
|
83
83
|
if re.match(r'^https?://', path):
|
|
84
84
|
self._url = path
|
|
@@ -130,11 +130,11 @@ class MediaAsset(BaseAsset):
|
|
|
130
130
|
tmp.flush()
|
|
131
131
|
tmp_path = tmp.name
|
|
132
132
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
133
|
+
try:
|
|
134
|
+
tag = TinyTag.get(tmp_path)
|
|
135
|
+
finally:
|
|
136
|
+
# Clean up the temporary file
|
|
137
|
+
os.unlink(tmp_path)
|
|
138
138
|
|
|
139
139
|
if tag.duration is None:
|
|
140
140
|
raise ValueError("Could not read duration from file")
|
|
@@ -169,7 +169,7 @@ class MediaAsset(BaseAsset):
|
|
|
169
169
|
def __check_name_ending(self, name: str) -> str:
|
|
170
170
|
"""Check if the media path is valid."""
|
|
171
171
|
if not name.endswith(('.jpg', '.jpeg', '.png', '.gif', '.mp3', '.mp4', '.webp')):
|
|
172
|
-
|
|
172
|
+
logger.warning("Warning: Supported file types: jpg, jpeg, png, gif, mp3, mp4. Image might not be displayed correctly.")
|
|
173
173
|
name = name + '.jpg'
|
|
174
174
|
return name
|
|
175
175
|
|
|
@@ -298,3 +298,9 @@ class MediaAsset(BaseAsset):
|
|
|
298
298
|
return self.path
|
|
299
299
|
else:
|
|
300
300
|
return (self.name, self.content)
|
|
301
|
+
|
|
302
|
+
def __str__(self) -> str:
|
|
303
|
+
return f"MediaAsset(path={self.path})"
|
|
304
|
+
|
|
305
|
+
def __repr__(self) -> str:
|
|
306
|
+
return f"MediaAsset(path={self.path})"
|
|
@@ -58,3 +58,9 @@ class MultiAsset(BaseAsset):
|
|
|
58
58
|
Iterator[BaseAsset]: An iterator over the assets.
|
|
59
59
|
"""
|
|
60
60
|
return iter(self.assets)
|
|
61
|
+
|
|
62
|
+
def __str__(self) -> str:
|
|
63
|
+
return f"MultiAsset(assets={self.assets})"
|
|
64
|
+
|
|
65
|
+
def __repr__(self) -> str:
|
|
66
|
+
return f"MultiAsset(assets={self.assets})"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from requests.adapters import HTTPAdapter
|
|
3
|
+
from urllib3.util.retry import Retry
|
|
4
|
+
|
|
5
|
+
class SessionManager:
|
|
6
|
+
_session = None
|
|
7
|
+
|
|
8
|
+
@classmethod
|
|
9
|
+
def get_session(cls, ) -> requests.Session:
|
|
10
|
+
"""Get a singleton requests session with retry logic.
|
|
11
|
+
|
|
12
|
+
Returns:
|
|
13
|
+
requests.Session: A singleton requests session with retry logic.
|
|
14
|
+
"""
|
|
15
|
+
if cls._session is None:
|
|
16
|
+
max_retries: int = 5
|
|
17
|
+
max_workers: int = 10
|
|
18
|
+
cls._session = requests.Session()
|
|
19
|
+
retries = Retry(
|
|
20
|
+
total=max_retries,
|
|
21
|
+
backoff_factor=1,
|
|
22
|
+
status_forcelist=[500, 502, 503, 504],
|
|
23
|
+
allowed_methods=["GET"],
|
|
24
|
+
respect_retry_after_header=True
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
adapter = HTTPAdapter(
|
|
28
|
+
pool_connections=max_workers * 2,
|
|
29
|
+
pool_maxsize=max_workers * 4,
|
|
30
|
+
max_retries=retries
|
|
31
|
+
)
|
|
32
|
+
cls._session.mount('http://', adapter)
|
|
33
|
+
cls._session.mount('https://', adapter)
|
|
34
|
+
|
|
35
|
+
return cls._session
|
|
@@ -2,13 +2,12 @@ from rapidata.service.openapi_service import OpenAPIService
|
|
|
2
2
|
from rapidata.rapidata_client.assets import MediaAsset
|
|
3
3
|
from rapidata.api_client.models.create_demographic_rapid_model import CreateDemographicRapidModel
|
|
4
4
|
from rapidata.api_client.models.classify_payload import ClassifyPayload
|
|
5
|
-
import
|
|
6
|
-
from requests.adapters import HTTPAdapter, Retry
|
|
7
|
-
|
|
5
|
+
from rapidata.rapidata_client.logging import logger
|
|
8
6
|
|
|
9
7
|
class DemographicManager:
|
|
10
8
|
def __init__(self, openapi_service: OpenAPIService):
|
|
11
9
|
self._openapi_service = openapi_service
|
|
10
|
+
logger.debug("DemographicManager initialized")
|
|
12
11
|
|
|
13
12
|
def create_demographic_rapid(self,
|
|
14
13
|
instruction: str,
|
|
@@ -25,38 +24,6 @@ class DemographicManager:
|
|
|
25
24
|
title=instruction
|
|
26
25
|
)
|
|
27
26
|
)
|
|
28
|
-
session = self._get_session()
|
|
29
|
-
media.session = session
|
|
30
27
|
|
|
31
28
|
self._openapi_service.rapid_api.rapid_demographic_post(model=model, file=[media.to_file()])
|
|
32
|
-
|
|
33
|
-
def _get_session(self, max_retries: int = 5, max_workers: int = 10) -> requests.Session:
|
|
34
|
-
"""Get a requests session with retry logic.
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
Args:
|
|
38
|
-
max_retries (int): The maximum number of retries.
|
|
39
|
-
max_workers (int): The maximum number of workers.
|
|
40
|
-
|
|
41
|
-
Returns:
|
|
42
|
-
requests.Session: A requests session with retry logic.
|
|
43
|
-
"""
|
|
44
|
-
session = requests.Session()
|
|
45
|
-
retries = Retry(
|
|
46
|
-
total=max_retries,
|
|
47
|
-
backoff_factor=1,
|
|
48
|
-
status_forcelist=[500, 502, 503, 504],
|
|
49
|
-
allowed_methods=["GET"],
|
|
50
|
-
respect_retry_after_header=True
|
|
51
|
-
)
|
|
52
|
-
|
|
53
|
-
adapter = HTTPAdapter(
|
|
54
|
-
pool_connections=max_workers * 2,
|
|
55
|
-
pool_maxsize=max_workers * 4,
|
|
56
|
-
max_retries=retries
|
|
57
|
-
)
|
|
58
|
-
session.mount('http://', adapter)
|
|
59
|
-
session.mount('https://', adapter)
|
|
60
|
-
|
|
61
|
-
return session
|
|
62
29
|
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
# Create module-level logger
|
|
6
|
+
logger = logging.getLogger("rapidata")
|
|
7
|
+
|
|
8
|
+
def configure_logger(
|
|
9
|
+
level: str = "WARNING",
|
|
10
|
+
log_file: Optional[str] = None,
|
|
11
|
+
format_string: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
12
|
+
):
|
|
13
|
+
"""Configure the Rapidata logger with custom settings."""
|
|
14
|
+
# Convert string level to logging level
|
|
15
|
+
level_map = {
|
|
16
|
+
"DEBUG": logging.DEBUG,
|
|
17
|
+
"INFO": logging.INFO,
|
|
18
|
+
"WARNING": logging.WARNING,
|
|
19
|
+
"ERROR": logging.ERROR,
|
|
20
|
+
"CRITICAL": logging.CRITICAL
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
numeric_level = level_map.get(level.upper(), logging.INFO)
|
|
24
|
+
logger.setLevel(numeric_level)
|
|
25
|
+
|
|
26
|
+
# Remove existing handlers if any
|
|
27
|
+
for handler in logger.handlers[:]:
|
|
28
|
+
logger.removeHandler(handler)
|
|
29
|
+
|
|
30
|
+
# Create formatter
|
|
31
|
+
formatter = logging.Formatter(format_string)
|
|
32
|
+
|
|
33
|
+
# Console handler
|
|
34
|
+
console_handler = logging.StreamHandler()
|
|
35
|
+
console_handler.setFormatter(formatter)
|
|
36
|
+
logger.addHandler(console_handler)
|
|
37
|
+
|
|
38
|
+
# File handler (optional)
|
|
39
|
+
if log_file:
|
|
40
|
+
log_dir = os.path.dirname(log_file)
|
|
41
|
+
if log_dir:
|
|
42
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
43
|
+
file_handler = logging.FileHandler(log_file)
|
|
44
|
+
file_handler.setFormatter(formatter)
|
|
45
|
+
logger.addHandler(file_handler)
|
|
46
|
+
|
|
47
|
+
configure_logger()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
class OutputManger:
|
|
2
|
+
silent_mode = False
|
|
3
|
+
|
|
4
|
+
@classmethod
|
|
5
|
+
def enable_silent_mode(cls):
|
|
6
|
+
cls.silent_mode = True
|
|
7
|
+
|
|
8
|
+
@classmethod
|
|
9
|
+
def disable_silent_mode(cls):
|
|
10
|
+
cls.silent_mode = False
|
|
11
|
+
|
|
12
|
+
def managed_print(*args, **kwargs):
|
|
13
|
+
if not OutputManger.silent_mode:
|
|
14
|
+
print(*args, **kwargs)
|
|
15
|
+
|
|
16
|
+
|
|
@@ -17,7 +17,7 @@ from tqdm import tqdm
|
|
|
17
17
|
|
|
18
18
|
from pydantic import StrictStr
|
|
19
19
|
from typing import cast, Sequence, Generator
|
|
20
|
-
from logging import
|
|
20
|
+
from rapidata.rapidata_client.logging import logger
|
|
21
21
|
import time
|
|
22
22
|
import threading
|
|
23
23
|
|
|
@@ -33,7 +33,6 @@ class RapidataDataset:
|
|
|
33
33
|
self.dataset_id = dataset_id
|
|
34
34
|
self.openapi_service = openapi_service
|
|
35
35
|
self.local_file_service = LocalFileService()
|
|
36
|
-
self._logger = Logger(__name__)
|
|
37
36
|
|
|
38
37
|
def _add_texts(
|
|
39
38
|
self,
|
|
@@ -89,7 +88,6 @@ class RapidataDataset:
|
|
|
89
88
|
media_asset: MediaAsset or MultiAsset to upload
|
|
90
89
|
meta: Optional metadata for the asset
|
|
91
90
|
index: Sort index for the upload
|
|
92
|
-
session: Requests session for HTTP requests
|
|
93
91
|
|
|
94
92
|
Returns:
|
|
95
93
|
tuple[list[str], list[str]]: Lists of successful and failed identifiers
|
|
@@ -145,14 +143,14 @@ class RapidataDataset:
|
|
|
145
143
|
|
|
146
144
|
if upload_response.errors:
|
|
147
145
|
error_msg = f"Error uploading datapoint: {upload_response.errors}"
|
|
148
|
-
|
|
146
|
+
logger.error(error_msg)
|
|
149
147
|
local_failed.extend(identifiers_to_track)
|
|
150
148
|
raise ValueError(error_msg)
|
|
151
149
|
|
|
152
150
|
local_successful.extend(identifiers_to_track)
|
|
153
151
|
|
|
154
152
|
except Exception as e:
|
|
155
|
-
|
|
153
|
+
logger.error(f"\nUpload failed for {identifiers_to_track}: {str(e)}") # \n to avoid same line as tqdm
|
|
156
154
|
local_failed.extend(identifiers_to_track)
|
|
157
155
|
|
|
158
156
|
return local_successful, local_failed
|
|
@@ -234,11 +232,11 @@ class RapidataDataset:
|
|
|
234
232
|
# If we're not at 100% but it's been a while with no progress
|
|
235
233
|
if stall_count > 5:
|
|
236
234
|
# We've polled several times with no progress, assume we're done
|
|
237
|
-
|
|
235
|
+
logger.warning(f"\nProgress seems stalled at {total_completed}/{total_uploads}. Please try again.")
|
|
238
236
|
break
|
|
239
237
|
|
|
240
238
|
except Exception as e:
|
|
241
|
-
|
|
239
|
+
logger.error(f"\nError checking progress: {str(e)}")
|
|
242
240
|
stall_count += 1
|
|
243
241
|
|
|
244
242
|
if stall_count > 10: # Too many consecutive errors
|
|
@@ -249,7 +247,7 @@ class RapidataDataset:
|
|
|
249
247
|
time.sleep(progress_poll_interval)
|
|
250
248
|
|
|
251
249
|
except Exception as e:
|
|
252
|
-
|
|
250
|
+
logger.error(f"Progress tracking thread error: {str(e)}")
|
|
253
251
|
progress_error_event.set()
|
|
254
252
|
|
|
255
253
|
# Create and return the thread
|
|
@@ -272,7 +270,6 @@ class RapidataDataset:
|
|
|
272
270
|
Args:
|
|
273
271
|
media_paths: List of assets to upload
|
|
274
272
|
metadata: Optional sequence of metadata
|
|
275
|
-
session: Requests session for HTTP requests
|
|
276
273
|
max_workers: Maximum number of concurrent workers
|
|
277
274
|
chunk_size: Number of items to process in each batch
|
|
278
275
|
stop_progress_tracking: Event to signal progress tracking to stop
|
|
@@ -310,7 +307,7 @@ class RapidataDataset:
|
|
|
310
307
|
successful_uploads.extend(chunk_successful)
|
|
311
308
|
failed_uploads.extend(chunk_failed)
|
|
312
309
|
except Exception as e:
|
|
313
|
-
|
|
310
|
+
logger.error(f"Future execution failed: {str(e)}")
|
|
314
311
|
finally:
|
|
315
312
|
# Signal to the progress tracking thread that all uploads have been submitted
|
|
316
313
|
stop_progress_tracking.set()
|
|
@@ -351,14 +348,13 @@ class RapidataDataset:
|
|
|
351
348
|
|
|
352
349
|
success_rate = (total_ready / total_uploads * 100) if total_uploads > 0 else 0
|
|
353
350
|
|
|
354
|
-
|
|
355
|
-
print(f"Upload complete, {total_ready} ready, {total_uploads-total_ready} failed ({success_rate:.1f}% success rate)")
|
|
351
|
+
logger.info(f"Upload complete: {total_ready} ready, {total_uploads-total_ready} failed ({success_rate:.1f}% success rate)")
|
|
356
352
|
except Exception as e:
|
|
357
|
-
|
|
358
|
-
|
|
353
|
+
logger.error(f"Error getting final progress: {str(e)}")
|
|
354
|
+
logger.info(f"Upload summary from local tracking: {len(successful_uploads)} succeeded, {len(failed_uploads)} failed")
|
|
359
355
|
|
|
360
356
|
if failed_uploads:
|
|
361
|
-
|
|
357
|
+
logger.error(f"Failed uploads: {failed_uploads}")
|
|
362
358
|
|
|
363
359
|
def _add_media_from_paths(
|
|
364
360
|
self,
|