cognite-toolkit 0.7.56__py3-none-any.whl → 0.7.57__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/apps/_auth_app.py +2 -2
- cognite_toolkit/_cdf_tk/apps/_core_app.py +4 -4
- cognite_toolkit/_cdf_tk/apps/_dev_app.py +10 -1
- cognite_toolkit/_cdf_tk/apps/_download_app.py +13 -12
- cognite_toolkit/_cdf_tk/apps/_dump_app.py +13 -13
- cognite_toolkit/_cdf_tk/apps/_landing_app.py +10 -1
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +13 -13
- cognite_toolkit/_cdf_tk/apps/_modules_app.py +29 -5
- cognite_toolkit/_cdf_tk/apps/_profile_app.py +4 -4
- cognite_toolkit/_cdf_tk/apps/_purge.py +4 -5
- cognite_toolkit/_cdf_tk/apps/_repo_app.py +9 -2
- cognite_toolkit/_cdf_tk/apps/_run.py +5 -4
- cognite_toolkit/_cdf_tk/apps/_upload_app.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/agents.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/annotations.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/assets.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/containers.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/data_models.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/datasets.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/events.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/extraction_pipelines.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/filemetadata.py +6 -6
- cognite_toolkit/_cdf_tk/client/api/function_schedules.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/functions.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/graphql_data_models.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/groups.py +5 -7
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_destinations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_jobs.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_mappings.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_sources.py +4 -4
- cognite_toolkit/_cdf_tk/client/api/infield.py +8 -8
- cognite_toolkit/_cdf_tk/client/api/instances.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/labels.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/legacy/extended_functions.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/location_filters.py +8 -8
- cognite_toolkit/_cdf_tk/client/api/project.py +14 -2
- cognite_toolkit/_cdf_tk/client/api/raw.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/relationships.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/robotics_capabilities.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_data_postprocessing.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_frames.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_locations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_maps.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_robots.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/search_config.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/security_categories.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/sequences.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/simulator_models.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/spaces.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/streams.py +6 -6
- cognite_toolkit/_cdf_tk/client/api/three_d.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/timeseries.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/transformations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/views.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/workflow_triggers.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/workflow_versions.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/workflows.py +3 -3
- cognite_toolkit/_cdf_tk/client/cdf_client/api.py +11 -11
- cognite_toolkit/_cdf_tk/client/http_client/__init__.py +23 -23
- cognite_toolkit/_cdf_tk/client/http_client/_client.py +46 -50
- cognite_toolkit/_cdf_tk/client/http_client/{_data_classes2.py → _data_classes.py} +25 -25
- cognite_toolkit/_cdf_tk/client/http_client/_item_classes.py +16 -16
- cognite_toolkit/_cdf_tk/client/resource_classes/project.py +30 -0
- cognite_toolkit/_cdf_tk/commands/_base.py +18 -1
- cognite_toolkit/_cdf_tk/commands/_import_cmd.py +3 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/command.py +7 -7
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +11 -11
- cognite_toolkit/_cdf_tk/commands/_profile.py +10 -5
- cognite_toolkit/_cdf_tk/commands/_purge.py +17 -21
- cognite_toolkit/_cdf_tk/commands/_upload.py +3 -3
- cognite_toolkit/_cdf_tk/commands/build_cmd.py +2 -1
- cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py +8 -2
- cognite_toolkit/_cdf_tk/commands/deploy.py +8 -2
- cognite_toolkit/_cdf_tk/commands/init.py +9 -2
- cognite_toolkit/_cdf_tk/commands/modules.py +3 -1
- cognite_toolkit/_cdf_tk/commands/pull.py +8 -2
- cognite_toolkit/_cdf_tk/commands/repo.py +3 -1
- cognite_toolkit/_cdf_tk/commands/resources.py +0 -3
- cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py +1 -0
- cognite_toolkit/_cdf_tk/storageio/_applications.py +6 -6
- cognite_toolkit/_cdf_tk/storageio/_base.py +2 -2
- cognite_toolkit/_cdf_tk/storageio/_datapoints.py +7 -7
- cognite_toolkit/_cdf_tk/storageio/_file_content.py +22 -22
- cognite_toolkit/_cdf_tk/storageio/_raw.py +2 -2
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.56.dist-info → cognite_toolkit-0.7.57.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.7.56.dist-info → cognite_toolkit-0.7.57.dist-info}/RECORD +92 -91
- {cognite_toolkit-0.7.56.dist-info → cognite_toolkit-0.7.57.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.7.56.dist-info → cognite_toolkit-0.7.57.dist-info}/entry_points.txt +0 -0
|
@@ -22,10 +22,10 @@ from cognite_toolkit._cdf_tk.client._resource_base import (
|
|
|
22
22
|
)
|
|
23
23
|
from cognite_toolkit._cdf_tk.client.http_client import (
|
|
24
24
|
HTTPClient,
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
25
|
+
ItemsRequest,
|
|
26
|
+
ItemsSuccessResponse,
|
|
27
|
+
RequestMessage,
|
|
28
|
+
SuccessResponse,
|
|
29
29
|
)
|
|
30
30
|
from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
|
|
31
31
|
|
|
@@ -79,7 +79,7 @@ class CDFResourceAPI(Generic[T_Identifier, T_RequestResource, T_ResponseResource
|
|
|
79
79
|
|
|
80
80
|
@abstractmethod
|
|
81
81
|
def _validate_page_response(
|
|
82
|
-
self, response:
|
|
82
|
+
self, response: SuccessResponse | ItemsSuccessResponse
|
|
83
83
|
) -> PagedResponse[T_ResponseResource]:
|
|
84
84
|
"""Parse a single item response."""
|
|
85
85
|
raise NotImplementedError()
|
|
@@ -131,7 +131,7 @@ class CDFResourceAPI(Generic[T_Identifier, T_RequestResource, T_ResponseResource
|
|
|
131
131
|
params: dict[str, Any] | None = None,
|
|
132
132
|
extra_body: dict[str, Any] | None = None,
|
|
133
133
|
endpoint_path: str | None = None,
|
|
134
|
-
) -> Iterable[
|
|
134
|
+
) -> Iterable[SuccessResponse]:
|
|
135
135
|
"""Send requests in chunks and yield responses.
|
|
136
136
|
|
|
137
137
|
Args:
|
|
@@ -150,7 +150,7 @@ class CDFResourceAPI(Generic[T_Identifier, T_RequestResource, T_ResponseResource
|
|
|
150
150
|
endpoint = self._method_endpoint_map[method]
|
|
151
151
|
|
|
152
152
|
for chunk in chunker_sequence(items, endpoint.item_limit):
|
|
153
|
-
request =
|
|
153
|
+
request = RequestMessage(
|
|
154
154
|
endpoint_url=f"{self._make_url(endpoint_path or endpoint.path)}",
|
|
155
155
|
method=endpoint.method,
|
|
156
156
|
body_content={"items": serialization(chunk), **(extra_body or {})}, # type: ignore[dict-item]
|
|
@@ -212,7 +212,7 @@ class CDFResourceAPI(Generic[T_Identifier, T_RequestResource, T_ResponseResource
|
|
|
212
212
|
method: APIMethod,
|
|
213
213
|
params: dict[str, Any] | None = None,
|
|
214
214
|
extra_body: dict[str, Any] | None = None,
|
|
215
|
-
) -> Iterable[
|
|
215
|
+
) -> Iterable[ItemsSuccessResponse]:
|
|
216
216
|
"""Request items with retries and splitting on failures.
|
|
217
217
|
|
|
218
218
|
This method handles large batches of items by chunking them according to the endpoint's item limit.
|
|
@@ -235,7 +235,7 @@ class CDFResourceAPI(Generic[T_Identifier, T_RequestResource, T_ResponseResource
|
|
|
235
235
|
endpoint = self._method_endpoint_map[method]
|
|
236
236
|
|
|
237
237
|
for chunk in chunker_sequence(items, endpoint.item_limit):
|
|
238
|
-
request =
|
|
238
|
+
request = ItemsRequest(
|
|
239
239
|
endpoint_url=f"{self._make_url(endpoint.path)}",
|
|
240
240
|
method=endpoint.method,
|
|
241
241
|
parameters=request_params,
|
|
@@ -245,7 +245,7 @@ class CDFResourceAPI(Generic[T_Identifier, T_RequestResource, T_ResponseResource
|
|
|
245
245
|
)
|
|
246
246
|
responses = self._http_client.request_items_retries(request)
|
|
247
247
|
for response in responses:
|
|
248
|
-
if isinstance(response,
|
|
248
|
+
if isinstance(response, ItemsSuccessResponse):
|
|
249
249
|
yield response
|
|
250
250
|
|
|
251
251
|
@classmethod
|
|
@@ -309,7 +309,7 @@ class CDFResourceAPI(Generic[T_Identifier, T_RequestResource, T_ResponseResource
|
|
|
309
309
|
else:
|
|
310
310
|
raise NotImplementedError(f"Unsupported method {endpoint.method} for pagination.")
|
|
311
311
|
|
|
312
|
-
request =
|
|
312
|
+
request = RequestMessage(
|
|
313
313
|
endpoint_url=self._make_url(endpoint_path or endpoint.path),
|
|
314
314
|
method=endpoint.method,
|
|
315
315
|
parameters=request_params,
|
|
@@ -1,33 +1,33 @@
|
|
|
1
1
|
from ._client import HTTPClient
|
|
2
|
-
from .
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
2
|
+
from ._data_classes import (
|
|
3
|
+
ErrorDetails,
|
|
4
|
+
FailedRequest,
|
|
5
|
+
FailedResponse,
|
|
6
|
+
HTTPResult,
|
|
7
|
+
RequestMessage,
|
|
8
|
+
SuccessResponse,
|
|
9
9
|
)
|
|
10
10
|
from ._exception import ToolkitAPIError
|
|
11
11
|
from ._item_classes import (
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
ItemsFailedRequest,
|
|
13
|
+
ItemsFailedResponse,
|
|
14
|
+
ItemsRequest,
|
|
15
|
+
ItemsResultMessage,
|
|
16
|
+
ItemsSuccessResponse,
|
|
17
17
|
)
|
|
18
18
|
|
|
19
19
|
__all__ = [
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
20
|
+
"ErrorDetails",
|
|
21
|
+
"FailedRequest",
|
|
22
|
+
"FailedResponse",
|
|
23
23
|
"HTTPClient",
|
|
24
|
-
"
|
|
25
|
-
"
|
|
26
|
-
"
|
|
27
|
-
"
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
24
|
+
"HTTPResult",
|
|
25
|
+
"ItemsFailedRequest",
|
|
26
|
+
"ItemsFailedResponse",
|
|
27
|
+
"ItemsRequest",
|
|
28
|
+
"ItemsResultMessage",
|
|
29
|
+
"ItemsSuccessResponse",
|
|
30
|
+
"RequestMessage",
|
|
31
|
+
"SuccessResponse",
|
|
32
32
|
"ToolkitAPIError",
|
|
33
33
|
]
|
|
@@ -9,22 +9,22 @@ import httpx
|
|
|
9
9
|
from cognite.client import global_config
|
|
10
10
|
from rich.console import Console
|
|
11
11
|
|
|
12
|
-
from cognite_toolkit._cdf_tk.client.http_client.
|
|
12
|
+
from cognite_toolkit._cdf_tk.client.http_client._data_classes import (
|
|
13
13
|
BaseRequestMessage,
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
14
|
+
ErrorDetails,
|
|
15
|
+
FailedRequest,
|
|
16
|
+
FailedResponse,
|
|
17
|
+
HTTPResult,
|
|
18
|
+
RequestMessage,
|
|
19
|
+
SuccessResponse,
|
|
20
20
|
)
|
|
21
21
|
from cognite_toolkit._cdf_tk.client.http_client._item_classes import (
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
ItemsFailedRequest,
|
|
23
|
+
ItemsFailedResponse,
|
|
24
|
+
ItemsRequest,
|
|
25
25
|
ItemsResultList,
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
ItemsResultMessage,
|
|
27
|
+
ItemsSuccessResponse,
|
|
28
28
|
)
|
|
29
29
|
from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning
|
|
30
30
|
from cognite_toolkit._cdf_tk.utils.auxiliary import get_current_toolkit_version, get_user_agent
|
|
@@ -136,13 +136,13 @@ class HTTPClient:
|
|
|
136
136
|
backoff_time = 0.5 * (2**attempts)
|
|
137
137
|
return min(backoff_time, global_config.max_retry_backoff) * random.uniform(0, 1.0)
|
|
138
138
|
|
|
139
|
-
def request_single(self, message:
|
|
139
|
+
def request_single(self, message: RequestMessage) -> RequestMessage | HTTPResult:
|
|
140
140
|
"""Send an HTTP request and return the response.
|
|
141
141
|
|
|
142
142
|
Args:
|
|
143
|
-
message (
|
|
143
|
+
message (RequestMessage): The request message to send.
|
|
144
144
|
Returns:
|
|
145
|
-
RequestMessage2 |
|
|
145
|
+
RequestMessage2 | HTTPResult: The response message.
|
|
146
146
|
"""
|
|
147
147
|
try:
|
|
148
148
|
response = self._make_request(message)
|
|
@@ -151,7 +151,7 @@ class HTTPClient:
|
|
|
151
151
|
result = self._handle_error_single(e, message)
|
|
152
152
|
return result
|
|
153
153
|
|
|
154
|
-
def request_single_retries(self, message:
|
|
154
|
+
def request_single_retries(self, message: RequestMessage) -> HTTPResult:
|
|
155
155
|
"""Send an HTTP request and handle retries.
|
|
156
156
|
|
|
157
157
|
This method will keep retrying the request until it either succeeds or
|
|
@@ -161,7 +161,7 @@ class HTTPClient:
|
|
|
161
161
|
it is blocking.
|
|
162
162
|
|
|
163
163
|
Args:
|
|
164
|
-
message (
|
|
164
|
+
message (RequestMessage): The request message to send.
|
|
165
165
|
Returns:
|
|
166
166
|
HTTPMessage2: The final response message, which can be either successful response or failed request.
|
|
167
167
|
"""
|
|
@@ -170,9 +170,9 @@ class HTTPClient:
|
|
|
170
170
|
current_request = message
|
|
171
171
|
while True:
|
|
172
172
|
result = self.request_single(current_request)
|
|
173
|
-
if isinstance(result,
|
|
173
|
+
if isinstance(result, RequestMessage):
|
|
174
174
|
current_request = result
|
|
175
|
-
elif isinstance(result,
|
|
175
|
+
elif isinstance(result, HTTPResult):
|
|
176
176
|
return result
|
|
177
177
|
else:
|
|
178
178
|
raise TypeError(f"Unexpected result type: {type(result)}")
|
|
@@ -195,11 +195,9 @@ class HTTPClient:
|
|
|
195
195
|
follow_redirects=False,
|
|
196
196
|
)
|
|
197
197
|
|
|
198
|
-
def _handle_response_single(
|
|
199
|
-
self, response: httpx.Response, request: RequestMessage2
|
|
200
|
-
) -> RequestMessage2 | HTTPResult2:
|
|
198
|
+
def _handle_response_single(self, response: httpx.Response, request: RequestMessage) -> RequestMessage | HTTPResult:
|
|
201
199
|
if 200 <= response.status_code < 300:
|
|
202
|
-
return
|
|
200
|
+
return SuccessResponse(
|
|
203
201
|
status_code=response.status_code,
|
|
204
202
|
body=response.text,
|
|
205
203
|
content=response.content,
|
|
@@ -208,10 +206,10 @@ class HTTPClient:
|
|
|
208
206
|
return retry_request
|
|
209
207
|
else:
|
|
210
208
|
# Permanent failure
|
|
211
|
-
return
|
|
209
|
+
return FailedResponse(
|
|
212
210
|
status_code=response.status_code,
|
|
213
211
|
body=response.text,
|
|
214
|
-
error=
|
|
212
|
+
error=ErrorDetails.from_response(response),
|
|
215
213
|
)
|
|
216
214
|
|
|
217
215
|
def _retry_request(self, response: httpx.Response, request: _T_Request_Message) -> _T_Request_Message | None:
|
|
@@ -232,7 +230,7 @@ class HTTPClient:
|
|
|
232
230
|
return request
|
|
233
231
|
return None
|
|
234
232
|
|
|
235
|
-
def _handle_error_single(self, e: Exception, request:
|
|
233
|
+
def _handle_error_single(self, e: Exception, request: RequestMessage) -> RequestMessage | HTTPResult:
|
|
236
234
|
if isinstance(e, httpx.ReadTimeout | httpx.TimeoutException):
|
|
237
235
|
error_type = "read"
|
|
238
236
|
request.read_attempt += 1
|
|
@@ -243,7 +241,7 @@ class HTTPClient:
|
|
|
243
241
|
attempts = request.connect_attempt
|
|
244
242
|
else:
|
|
245
243
|
error_msg = f"Unexpected exception: {e!s}"
|
|
246
|
-
return
|
|
244
|
+
return FailedRequest(error=error_msg)
|
|
247
245
|
|
|
248
246
|
if attempts <= self._max_retries:
|
|
249
247
|
time.sleep(self._backoff_time(request.total_attempts))
|
|
@@ -252,20 +250,20 @@ class HTTPClient:
|
|
|
252
250
|
# We have already incremented the attempt count, so we subtract 1 here
|
|
253
251
|
error_msg = f"RequestException after {request.total_attempts - 1} attempts ({error_type} error): {e!s}"
|
|
254
252
|
|
|
255
|
-
return
|
|
253
|
+
return FailedRequest(error=error_msg)
|
|
256
254
|
|
|
257
|
-
def request_items(self, message:
|
|
255
|
+
def request_items(self, message: ItemsRequest) -> Sequence[ItemsRequest | ItemsResultMessage]:
|
|
258
256
|
"""Send an HTTP request with multiple items and return the response.
|
|
259
257
|
|
|
260
258
|
Args:
|
|
261
|
-
message (
|
|
259
|
+
message (ItemsRequest): The request message to send.
|
|
262
260
|
Returns:
|
|
263
|
-
Sequence[ItemsRequest2 |
|
|
261
|
+
Sequence[ItemsRequest2 | ItemsResultMessage]: The response message(s). This can also
|
|
264
262
|
include ItemsRequest2(s) to be retried or split.
|
|
265
263
|
"""
|
|
266
264
|
if message.tracker and message.tracker.limit_reached():
|
|
267
265
|
return [
|
|
268
|
-
|
|
266
|
+
ItemsFailedRequest(
|
|
269
267
|
ids=[str(item) for item in message.items],
|
|
270
268
|
error_message=f"Aborting further splitting of requests after {message.tracker.failed_split_count} failed attempts.",
|
|
271
269
|
)
|
|
@@ -277,7 +275,7 @@ class HTTPClient:
|
|
|
277
275
|
results = self._handle_items_error(e, message)
|
|
278
276
|
return results
|
|
279
277
|
|
|
280
|
-
def request_items_retries(self, message:
|
|
278
|
+
def request_items_retries(self, message: ItemsRequest) -> ItemsResultList:
|
|
281
279
|
"""Send an HTTP request with multiple items and handle retries.
|
|
282
280
|
|
|
283
281
|
This method will keep retrying the request until it either succeeds or
|
|
@@ -287,13 +285,13 @@ class HTTPClient:
|
|
|
287
285
|
it is blocking.
|
|
288
286
|
|
|
289
287
|
Args:
|
|
290
|
-
message (
|
|
288
|
+
message (ItemsRequest): The request message to send.
|
|
291
289
|
Returns:
|
|
292
|
-
Sequence[
|
|
290
|
+
Sequence[ItemsResultMessage]: The final response message, which can be either successful response or failed request.
|
|
293
291
|
"""
|
|
294
292
|
if message.total_attempts > 0:
|
|
295
293
|
raise RuntimeError(f"ItemsRequest2 has already been attempted {message.total_attempts} times.")
|
|
296
|
-
pending_requests: deque[
|
|
294
|
+
pending_requests: deque[ItemsRequest] = deque()
|
|
297
295
|
pending_requests.append(message)
|
|
298
296
|
final_responses = ItemsResultList([])
|
|
299
297
|
while pending_requests:
|
|
@@ -301,9 +299,9 @@ class HTTPClient:
|
|
|
301
299
|
results = self.request_items(current_request)
|
|
302
300
|
|
|
303
301
|
for result in results:
|
|
304
|
-
if isinstance(result,
|
|
302
|
+
if isinstance(result, ItemsRequest):
|
|
305
303
|
pending_requests.append(result)
|
|
306
|
-
elif isinstance(result,
|
|
304
|
+
elif isinstance(result, ItemsResultMessage):
|
|
307
305
|
final_responses.append(result)
|
|
308
306
|
else:
|
|
309
307
|
raise TypeError(f"Unexpected result type: {type(result)}")
|
|
@@ -311,11 +309,11 @@ class HTTPClient:
|
|
|
311
309
|
return final_responses
|
|
312
310
|
|
|
313
311
|
def _handle_items_response(
|
|
314
|
-
self, response: httpx.Response, request:
|
|
315
|
-
) -> Sequence[
|
|
312
|
+
self, response: httpx.Response, request: ItemsRequest
|
|
313
|
+
) -> Sequence[ItemsRequest | ItemsResultMessage]:
|
|
316
314
|
if 200 <= response.status_code < 300:
|
|
317
315
|
return [
|
|
318
|
-
|
|
316
|
+
ItemsSuccessResponse(
|
|
319
317
|
ids=[str(item) for item in request.items],
|
|
320
318
|
status_code=response.status_code,
|
|
321
319
|
body=response.text,
|
|
@@ -331,11 +329,11 @@ class HTTPClient:
|
|
|
331
329
|
splits = request.split(status_attempts=status_attempts)
|
|
332
330
|
if splits[0].tracker and splits[0].tracker.limit_reached():
|
|
333
331
|
return [
|
|
334
|
-
|
|
332
|
+
ItemsFailedResponse(
|
|
335
333
|
ids=[str(item) for item in request.items],
|
|
336
334
|
status_code=response.status_code,
|
|
337
335
|
body=response.text,
|
|
338
|
-
error=
|
|
336
|
+
error=ErrorDetails.from_response(response),
|
|
339
337
|
)
|
|
340
338
|
]
|
|
341
339
|
return splits
|
|
@@ -345,17 +343,15 @@ class HTTPClient:
|
|
|
345
343
|
else:
|
|
346
344
|
# Permanent failure
|
|
347
345
|
return [
|
|
348
|
-
|
|
346
|
+
ItemsFailedResponse(
|
|
349
347
|
ids=[str(item) for item in request.items],
|
|
350
348
|
status_code=response.status_code,
|
|
351
349
|
body=response.text,
|
|
352
|
-
error=
|
|
350
|
+
error=ErrorDetails.from_response(response),
|
|
353
351
|
)
|
|
354
352
|
]
|
|
355
353
|
|
|
356
|
-
def _handle_items_error(
|
|
357
|
-
self, e: Exception, request: ItemsRequest2
|
|
358
|
-
) -> Sequence[ItemsRequest2 | ItemsResultMessage2]:
|
|
354
|
+
def _handle_items_error(self, e: Exception, request: ItemsRequest) -> Sequence[ItemsRequest | ItemsResultMessage]:
|
|
359
355
|
if isinstance(e, httpx.ReadTimeout | httpx.TimeoutException):
|
|
360
356
|
error_type = "read"
|
|
361
357
|
request.read_attempt += 1
|
|
@@ -367,7 +363,7 @@ class HTTPClient:
|
|
|
367
363
|
else:
|
|
368
364
|
error_msg = f"Unexpected exception: {e!s}"
|
|
369
365
|
return [
|
|
370
|
-
|
|
366
|
+
ItemsFailedRequest(
|
|
371
367
|
ids=[str(item) for item in request.items],
|
|
372
368
|
error_message=error_msg,
|
|
373
369
|
)
|
|
@@ -381,7 +377,7 @@ class HTTPClient:
|
|
|
381
377
|
error_msg = f"RequestException after {request.total_attempts - 1} attempts ({error_type} error): {e!s}"
|
|
382
378
|
|
|
383
379
|
return [
|
|
384
|
-
|
|
380
|
+
ItemsFailedRequest(
|
|
385
381
|
ids=[str(item) for item in request.items],
|
|
386
382
|
error_message=error_msg,
|
|
387
383
|
)
|
|
@@ -10,60 +10,60 @@ from cognite_toolkit._cdf_tk.client.http_client._exception import ToolkitAPIErro
|
|
|
10
10
|
from cognite_toolkit._cdf_tk.utils.useful_types import PrimitiveType
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
|
-
from cognite_toolkit._cdf_tk.client.http_client._item_classes import
|
|
13
|
+
from cognite_toolkit._cdf_tk.client.http_client._item_classes import ItemsResultMessage
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
class
|
|
17
|
-
def get_success_or_raise(self) -> "
|
|
16
|
+
class HTTPResult(BaseModel):
|
|
17
|
+
def get_success_or_raise(self) -> "SuccessResponse":
|
|
18
18
|
"""Raises an exception if any response in the list indicates a failure."""
|
|
19
|
-
if isinstance(self,
|
|
19
|
+
if isinstance(self, SuccessResponse):
|
|
20
20
|
return self
|
|
21
|
-
elif isinstance(self,
|
|
21
|
+
elif isinstance(self, FailedResponse):
|
|
22
22
|
raise ToolkitAPIError(
|
|
23
23
|
f"Request failed with status code {self.status_code}: {self.error.message}",
|
|
24
24
|
missing=self.error.missing, # type: ignore[arg-type]
|
|
25
25
|
duplicated=self.error.duplicated, # type: ignore[arg-type]
|
|
26
26
|
)
|
|
27
|
-
elif isinstance(self,
|
|
27
|
+
elif isinstance(self, FailedRequest):
|
|
28
28
|
raise ToolkitAPIError(f"Request failed with error: {self.error}")
|
|
29
29
|
else:
|
|
30
30
|
raise ToolkitAPIError("Unknown HTTPResult2 type")
|
|
31
31
|
|
|
32
|
-
def as_item_response(self, item_id: str) -> "
|
|
32
|
+
def as_item_response(self, item_id: str) -> "ItemsResultMessage":
|
|
33
33
|
# Avoid circular import
|
|
34
34
|
from cognite_toolkit._cdf_tk.client.http_client._item_classes import (
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
ItemsFailedRequest,
|
|
36
|
+
ItemsFailedResponse,
|
|
37
|
+
ItemsSuccessResponse,
|
|
38
38
|
)
|
|
39
39
|
|
|
40
|
-
if isinstance(self,
|
|
41
|
-
return
|
|
40
|
+
if isinstance(self, SuccessResponse):
|
|
41
|
+
return ItemsSuccessResponse(
|
|
42
42
|
status_code=self.status_code, content=self.content, ids=[item_id], body=self.body
|
|
43
43
|
)
|
|
44
|
-
elif isinstance(self,
|
|
45
|
-
return
|
|
44
|
+
elif isinstance(self, FailedResponse):
|
|
45
|
+
return ItemsFailedResponse(
|
|
46
46
|
status_code=self.status_code,
|
|
47
47
|
ids=[item_id],
|
|
48
48
|
body=self.body,
|
|
49
|
-
error=
|
|
49
|
+
error=ErrorDetails(
|
|
50
50
|
code=self.error.code,
|
|
51
51
|
message=self.error.message,
|
|
52
52
|
missing=self.error.missing,
|
|
53
53
|
duplicated=self.error.duplicated,
|
|
54
54
|
),
|
|
55
55
|
)
|
|
56
|
-
elif isinstance(self,
|
|
57
|
-
return
|
|
56
|
+
elif isinstance(self, FailedRequest):
|
|
57
|
+
return ItemsFailedRequest(ids=[item_id], error_message=self.error)
|
|
58
58
|
else:
|
|
59
59
|
raise ToolkitAPIError(f"Unknown {type(self).__name__} type")
|
|
60
60
|
|
|
61
61
|
|
|
62
|
-
class
|
|
62
|
+
class FailedRequest(HTTPResult):
|
|
63
63
|
error: str
|
|
64
64
|
|
|
65
65
|
|
|
66
|
-
class
|
|
66
|
+
class SuccessResponse(HTTPResult):
|
|
67
67
|
status_code: int
|
|
68
68
|
body: str
|
|
69
69
|
content: bytes
|
|
@@ -74,7 +74,7 @@ class SuccessResponse2(HTTPResult2):
|
|
|
74
74
|
return TypeAdapter(dict[str, JsonValue]).validate_json(self.body)
|
|
75
75
|
|
|
76
76
|
|
|
77
|
-
class
|
|
77
|
+
class ErrorDetails(BaseModel):
|
|
78
78
|
"""This is the expected structure of error details in the CDF API"""
|
|
79
79
|
|
|
80
80
|
code: int
|
|
@@ -84,19 +84,19 @@ class ErrorDetails2(BaseModel):
|
|
|
84
84
|
is_auto_retryable: bool | None = None
|
|
85
85
|
|
|
86
86
|
@classmethod
|
|
87
|
-
def from_response(cls, response: httpx.Response) -> "
|
|
87
|
+
def from_response(cls, response: httpx.Response) -> "ErrorDetails":
|
|
88
88
|
"""Populate the error details from a httpx response."""
|
|
89
89
|
try:
|
|
90
|
-
res = TypeAdapter(dict[Literal["error"],
|
|
90
|
+
res = TypeAdapter(dict[Literal["error"], ErrorDetails]).validate_json(response.text)
|
|
91
91
|
except ValueError:
|
|
92
92
|
return cls(code=response.status_code, message=response.text)
|
|
93
93
|
return res["error"]
|
|
94
94
|
|
|
95
95
|
|
|
96
|
-
class
|
|
96
|
+
class FailedResponse(HTTPResult):
|
|
97
97
|
status_code: int
|
|
98
98
|
body: str
|
|
99
|
-
error:
|
|
99
|
+
error: ErrorDetails
|
|
100
100
|
|
|
101
101
|
|
|
102
102
|
class BaseRequestMessage(BaseModel, ABC):
|
|
@@ -122,7 +122,7 @@ class BaseRequestMessage(BaseModel, ABC):
|
|
|
122
122
|
def content(self) -> str | bytes | None: ...
|
|
123
123
|
|
|
124
124
|
|
|
125
|
-
class
|
|
125
|
+
class RequestMessage(BaseRequestMessage):
|
|
126
126
|
data_content: bytes | None = None
|
|
127
127
|
body_content: dict[str, JsonValue] | None = None
|
|
128
128
|
|
|
@@ -9,32 +9,32 @@ from cognite.client import global_config
|
|
|
9
9
|
from pydantic import BaseModel, ConfigDict, Field, JsonValue
|
|
10
10
|
|
|
11
11
|
from cognite_toolkit._cdf_tk.client._resource_base import RequestItem
|
|
12
|
-
from cognite_toolkit._cdf_tk.client.http_client.
|
|
12
|
+
from cognite_toolkit._cdf_tk.client.http_client._data_classes import (
|
|
13
13
|
_BODY_SERIALIZER,
|
|
14
14
|
BaseRequestMessage,
|
|
15
|
-
|
|
15
|
+
ErrorDetails,
|
|
16
16
|
)
|
|
17
17
|
from cognite_toolkit._cdf_tk.client.http_client._exception import ToolkitAPIError
|
|
18
18
|
from cognite_toolkit._cdf_tk.client.http_client._tracker import ItemsRequestTracker
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
class
|
|
21
|
+
class ItemsResultMessage(BaseModel):
|
|
22
22
|
ids: list[str]
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
class
|
|
25
|
+
class ItemsFailedRequest(ItemsResultMessage):
|
|
26
26
|
error_message: str
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
class
|
|
29
|
+
class ItemsSuccessResponse(ItemsResultMessage):
|
|
30
30
|
status_code: int
|
|
31
31
|
body: str
|
|
32
32
|
content: bytes
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
class
|
|
35
|
+
class ItemsFailedResponse(ItemsResultMessage):
|
|
36
36
|
status_code: int
|
|
37
|
-
error:
|
|
37
|
+
error: ErrorDetails
|
|
38
38
|
body: str
|
|
39
39
|
|
|
40
40
|
|
|
@@ -44,7 +44,7 @@ def _set_default_tracker(data: dict[str, Any]) -> ItemsRequestTracker:
|
|
|
44
44
|
return data["tracker"]
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
class
|
|
47
|
+
class ItemsRequest(BaseRequestMessage):
|
|
48
48
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
49
49
|
items: Sequence[RequestItem]
|
|
50
50
|
extra_body_fields: dict[str, JsonValue] | None = None
|
|
@@ -61,13 +61,13 @@ class ItemsRequest2(BaseRequestMessage):
|
|
|
61
61
|
return gzip.compress(res)
|
|
62
62
|
return res
|
|
63
63
|
|
|
64
|
-
def split(self, status_attempts: int) -> list["
|
|
64
|
+
def split(self, status_attempts: int) -> list["ItemsRequest"]:
|
|
65
65
|
"""Split the request into multiple requests with a single item each."""
|
|
66
66
|
mid = len(self.items) // 2
|
|
67
67
|
if mid == 0:
|
|
68
68
|
return [self]
|
|
69
69
|
self.tracker.register_failure()
|
|
70
|
-
messages: list[
|
|
70
|
+
messages: list[ItemsRequest] = []
|
|
71
71
|
for part in (self.items[:mid], self.items[mid:]):
|
|
72
72
|
new_request = self.model_copy(update={"items": part, "status_attempt": status_attempts})
|
|
73
73
|
new_request.tracker = self.tracker
|
|
@@ -79,14 +79,14 @@ class ItemResponse(BaseModel):
|
|
|
79
79
|
items: list[dict[str, JsonValue]]
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
class ItemsResultList(UserList[
|
|
83
|
-
def __init__(self, collection: Sequence[
|
|
82
|
+
class ItemsResultList(UserList[ItemsResultMessage]):
|
|
83
|
+
def __init__(self, collection: Sequence[ItemsResultMessage] | None = None) -> None:
|
|
84
84
|
super().__init__(collection or [])
|
|
85
85
|
|
|
86
86
|
def raise_for_status(self) -> None:
|
|
87
87
|
"""Raises an exception if any response in the list indicates a failure."""
|
|
88
|
-
failed_responses = [resp for resp in self.data if isinstance(resp,
|
|
89
|
-
failed_requests = [resp for resp in self.data if isinstance(resp,
|
|
88
|
+
failed_responses = [resp for resp in self.data if isinstance(resp, ItemsFailedResponse)]
|
|
89
|
+
failed_requests = [resp for resp in self.data if isinstance(resp, ItemsFailedRequest)]
|
|
90
90
|
if not failed_responses and not failed_requests:
|
|
91
91
|
return
|
|
92
92
|
error_messages = "; ".join(f"Status {err.status_code}: {err.error.message}" for err in failed_responses)
|
|
@@ -104,7 +104,7 @@ class ItemsResultList(UserList[ItemsResultMessage2]):
|
|
|
104
104
|
bool: True if there are any failed responses or requests, False otherwise.
|
|
105
105
|
"""
|
|
106
106
|
for resp in self.data:
|
|
107
|
-
if isinstance(resp,
|
|
107
|
+
if isinstance(resp, ItemsFailedResponse | ItemsFailedRequest):
|
|
108
108
|
return True
|
|
109
109
|
return False
|
|
110
110
|
|
|
@@ -112,7 +112,7 @@ class ItemsResultList(UserList[ItemsResultMessage2]):
|
|
|
112
112
|
"""Get the items from all successful responses."""
|
|
113
113
|
items: list[dict[str, JsonValue]] = []
|
|
114
114
|
for resp in self.data:
|
|
115
|
-
if isinstance(resp,
|
|
115
|
+
if isinstance(resp, ItemsSuccessResponse):
|
|
116
116
|
body_json = ItemResponse.model_validate_json(resp.body)
|
|
117
117
|
items.extend(body_json.items)
|
|
118
118
|
return items
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from cognite_toolkit._cdf_tk.client._resource_base import BaseModelObject
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class UserProfilesConfiguration(BaseModelObject):
|
|
5
|
+
enabled: bool
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Claim(BaseModelObject):
|
|
9
|
+
claim_name: str
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class OidcConfiguration(BaseModelObject):
|
|
13
|
+
jwks_url: str
|
|
14
|
+
token_url: str | None = None
|
|
15
|
+
issuer: str
|
|
16
|
+
audience: str
|
|
17
|
+
skew_ms: int | None = None
|
|
18
|
+
access_claims: list[Claim]
|
|
19
|
+
scope_claims: list[Claim]
|
|
20
|
+
log_claims: list[Claim]
|
|
21
|
+
is_group_callback_enabled: bool | None = None
|
|
22
|
+
identity_provider_scope: str | None = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class OrganizationResponse(BaseModelObject):
|
|
26
|
+
name: str
|
|
27
|
+
url_name: str
|
|
28
|
+
organization: str
|
|
29
|
+
user_profiles_configuration: UserProfilesConfiguration
|
|
30
|
+
oidc_configuration: OidcConfiguration | None = None
|