cognite-toolkit 0.7.55__py3-none-any.whl → 0.7.57__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/apps/_auth_app.py +2 -2
- cognite_toolkit/_cdf_tk/apps/_core_app.py +4 -4
- cognite_toolkit/_cdf_tk/apps/_dev_app.py +10 -1
- cognite_toolkit/_cdf_tk/apps/_download_app.py +13 -12
- cognite_toolkit/_cdf_tk/apps/_dump_app.py +13 -13
- cognite_toolkit/_cdf_tk/apps/_landing_app.py +10 -1
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +13 -13
- cognite_toolkit/_cdf_tk/apps/_modules_app.py +29 -5
- cognite_toolkit/_cdf_tk/apps/_profile_app.py +4 -4
- cognite_toolkit/_cdf_tk/apps/_purge.py +4 -5
- cognite_toolkit/_cdf_tk/apps/_repo_app.py +9 -2
- cognite_toolkit/_cdf_tk/apps/_run.py +5 -4
- cognite_toolkit/_cdf_tk/apps/_upload_app.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/agents.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/annotations.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/assets.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/containers.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/data_models.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/datasets.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/events.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/extraction_pipelines.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/filemetadata.py +6 -6
- cognite_toolkit/_cdf_tk/client/api/function_schedules.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/functions.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/graphql_data_models.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/groups.py +5 -7
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_destinations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_jobs.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_mappings.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_sources.py +4 -4
- cognite_toolkit/_cdf_tk/client/api/infield.py +8 -8
- cognite_toolkit/_cdf_tk/client/api/instances.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/labels.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/legacy/extended_functions.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/location_filters.py +8 -8
- cognite_toolkit/_cdf_tk/client/api/project.py +14 -2
- cognite_toolkit/_cdf_tk/client/api/raw.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/relationships.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/robotics_capabilities.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_data_postprocessing.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_frames.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_locations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_maps.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_robots.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/search_config.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/security_categories.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/sequences.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/simulator_models.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/spaces.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/streams.py +6 -6
- cognite_toolkit/_cdf_tk/client/api/three_d.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/timeseries.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/transformations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/views.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/workflow_triggers.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/workflow_versions.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/workflows.py +3 -3
- cognite_toolkit/_cdf_tk/client/cdf_client/api.py +11 -11
- cognite_toolkit/_cdf_tk/client/http_client/__init__.py +13 -51
- cognite_toolkit/_cdf_tk/client/http_client/_client.py +48 -209
- cognite_toolkit/_cdf_tk/client/http_client/_data_classes.py +106 -383
- cognite_toolkit/_cdf_tk/client/http_client/_item_classes.py +16 -16
- cognite_toolkit/_cdf_tk/client/resource_classes/filemetadata.py +7 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/project.py +30 -0
- cognite_toolkit/_cdf_tk/commands/_base.py +18 -1
- cognite_toolkit/_cdf_tk/commands/_import_cmd.py +3 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/command.py +8 -8
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +25 -24
- cognite_toolkit/_cdf_tk/commands/_profile.py +10 -5
- cognite_toolkit/_cdf_tk/commands/_purge.py +30 -35
- cognite_toolkit/_cdf_tk/commands/_upload.py +4 -6
- cognite_toolkit/_cdf_tk/commands/build_cmd.py +2 -1
- cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py +8 -2
- cognite_toolkit/_cdf_tk/commands/deploy.py +8 -2
- cognite_toolkit/_cdf_tk/commands/init.py +9 -2
- cognite_toolkit/_cdf_tk/commands/modules.py +3 -1
- cognite_toolkit/_cdf_tk/commands/pull.py +8 -2
- cognite_toolkit/_cdf_tk/commands/repo.py +3 -1
- cognite_toolkit/_cdf_tk/commands/resources.py +0 -3
- cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py +1 -0
- cognite_toolkit/_cdf_tk/protocols.py +3 -1
- cognite_toolkit/_cdf_tk/storageio/_applications.py +9 -9
- cognite_toolkit/_cdf_tk/storageio/_base.py +15 -10
- cognite_toolkit/_cdf_tk/storageio/_datapoints.py +36 -24
- cognite_toolkit/_cdf_tk/storageio/_file_content.py +47 -43
- cognite_toolkit/_cdf_tk/storageio/_raw.py +5 -4
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.55.dist-info → cognite_toolkit-0.7.57.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.7.55.dist-info → cognite_toolkit-0.7.57.dist-info}/RECORD +94 -94
- cognite_toolkit/_cdf_tk/client/http_client/_data_classes2.py +0 -151
- {cognite_toolkit-0.7.55.dist-info → cognite_toolkit-0.7.57.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.7.55.dist-info → cognite_toolkit-0.7.57.dist-info}/entry_points.txt +0 -0
|
@@ -1,428 +1,151 @@
|
|
|
1
|
+
import gzip
|
|
1
2
|
from abc import ABC, abstractmethod
|
|
2
|
-
from
|
|
3
|
-
from collections.abc import Hashable, Sequence
|
|
4
|
-
from dataclasses import dataclass, field
|
|
5
|
-
from typing import Generic, Literal, Protocol, TypeAlias, TypeVar
|
|
3
|
+
from typing import Any, Literal
|
|
6
4
|
|
|
7
5
|
import httpx
|
|
8
|
-
from cognite.client
|
|
6
|
+
from cognite.client import global_config
|
|
7
|
+
from pydantic import TYPE_CHECKING, BaseModel, JsonValue, TypeAdapter, model_validator
|
|
9
8
|
|
|
10
9
|
from cognite_toolkit._cdf_tk.client.http_client._exception import ToolkitAPIError
|
|
11
|
-
from cognite_toolkit._cdf_tk.
|
|
12
|
-
from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal, PrimitiveType
|
|
10
|
+
from cognite_toolkit._cdf_tk.utils.useful_types import PrimitiveType
|
|
13
11
|
|
|
14
|
-
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from cognite_toolkit._cdf_tk.client.http_client._item_classes import ItemsResultMessage
|
|
15
14
|
|
|
16
15
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
class HTTPResult(BaseModel):
|
|
17
|
+
def get_success_or_raise(self) -> "SuccessResponse":
|
|
18
|
+
"""Raises an exception if any response in the list indicates a failure."""
|
|
19
|
+
if isinstance(self, SuccessResponse):
|
|
20
|
+
return self
|
|
21
|
+
elif isinstance(self, FailedResponse):
|
|
22
|
+
raise ToolkitAPIError(
|
|
23
|
+
f"Request failed with status code {self.status_code}: {self.error.message}",
|
|
24
|
+
missing=self.error.missing, # type: ignore[arg-type]
|
|
25
|
+
duplicated=self.error.duplicated, # type: ignore[arg-type]
|
|
26
|
+
)
|
|
27
|
+
elif isinstance(self, FailedRequest):
|
|
28
|
+
raise ToolkitAPIError(f"Request failed with error: {self.error}")
|
|
29
|
+
else:
|
|
30
|
+
raise ToolkitAPIError("Unknown HTTPResult2 type")
|
|
31
|
+
|
|
32
|
+
def as_item_response(self, item_id: str) -> "ItemsResultMessage":
|
|
33
|
+
# Avoid circular import
|
|
34
|
+
from cognite_toolkit._cdf_tk.client.http_client._item_classes import (
|
|
35
|
+
ItemsFailedRequest,
|
|
36
|
+
ItemsFailedResponse,
|
|
37
|
+
ItemsSuccessResponse,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
if isinstance(self, SuccessResponse):
|
|
41
|
+
return ItemsSuccessResponse(
|
|
42
|
+
status_code=self.status_code, content=self.content, ids=[item_id], body=self.body
|
|
43
|
+
)
|
|
44
|
+
elif isinstance(self, FailedResponse):
|
|
45
|
+
return ItemsFailedResponse(
|
|
46
|
+
status_code=self.status_code,
|
|
47
|
+
ids=[item_id],
|
|
48
|
+
body=self.body,
|
|
49
|
+
error=ErrorDetails(
|
|
50
|
+
code=self.error.code,
|
|
51
|
+
message=self.error.message,
|
|
52
|
+
missing=self.error.missing,
|
|
53
|
+
duplicated=self.error.duplicated,
|
|
54
|
+
),
|
|
55
|
+
)
|
|
56
|
+
elif isinstance(self, FailedRequest):
|
|
57
|
+
return ItemsFailedRequest(ids=[item_id], error_message=self.error)
|
|
58
|
+
else:
|
|
59
|
+
raise ToolkitAPIError(f"Unknown {type(self).__name__} type")
|
|
60
|
+
|
|
20
61
|
|
|
21
|
-
|
|
22
|
-
|
|
62
|
+
class FailedRequest(HTTPResult):
|
|
63
|
+
error: str
|
|
23
64
|
|
|
24
|
-
Returns:
|
|
25
|
-
dict[str, JsonVal]: The message as a dictionary.
|
|
26
|
-
"""
|
|
27
|
-
# We avoid using the asdict function as we know we have a shallow structure,
|
|
28
|
-
# and this roughly ~10x faster.
|
|
29
|
-
output = self.__dict__.copy()
|
|
30
|
-
output["type"] = type(self).__name__
|
|
31
|
-
return output
|
|
32
65
|
|
|
66
|
+
class SuccessResponse(HTTPResult):
|
|
67
|
+
status_code: int
|
|
68
|
+
body: str
|
|
69
|
+
content: bytes
|
|
33
70
|
|
|
34
|
-
@
|
|
35
|
-
|
|
71
|
+
@property
|
|
72
|
+
def body_json(self) -> dict[str, Any]:
|
|
73
|
+
"""Parse the response body as JSON."""
|
|
74
|
+
return TypeAdapter(dict[str, JsonValue]).validate_json(self.body)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class ErrorDetails(BaseModel):
|
|
36
78
|
"""This is the expected structure of error details in the CDF API"""
|
|
37
79
|
|
|
38
80
|
code: int
|
|
39
81
|
message: str
|
|
40
|
-
missing: list[
|
|
41
|
-
duplicated: list[
|
|
82
|
+
missing: list[JsonValue] | None = None
|
|
83
|
+
duplicated: list[JsonValue] | None = None
|
|
42
84
|
is_auto_retryable: bool | None = None
|
|
43
85
|
|
|
44
86
|
@classmethod
|
|
45
87
|
def from_response(cls, response: httpx.Response) -> "ErrorDetails":
|
|
88
|
+
"""Populate the error details from a httpx response."""
|
|
46
89
|
try:
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
return cls(code=response.status_code, message=str(error_data))
|
|
50
|
-
return cls(
|
|
51
|
-
code=error_data["code"],
|
|
52
|
-
message=error_data["message"],
|
|
53
|
-
missing=error_data.get("missing"),
|
|
54
|
-
duplicated=error_data.get("duplicated"),
|
|
55
|
-
is_auto_retryable=error_data.get("isAutoRetryable"),
|
|
56
|
-
)
|
|
57
|
-
except (ValueError, KeyError):
|
|
58
|
-
# Fallback if response is not JSON or does not have expected structure
|
|
90
|
+
res = TypeAdapter(dict[Literal["error"], ErrorDetails]).validate_json(response.text)
|
|
91
|
+
except ValueError:
|
|
59
92
|
return cls(code=response.status_code, message=response.text)
|
|
93
|
+
return res["error"]
|
|
60
94
|
|
|
61
|
-
def dump(self) -> dict[str, JsonVal]:
|
|
62
|
-
output: dict[str, JsonVal] = {
|
|
63
|
-
"code": self.code,
|
|
64
|
-
"message": self.message,
|
|
65
|
-
}
|
|
66
|
-
if self.missing is not None:
|
|
67
|
-
output["missing"] = self.missing
|
|
68
|
-
if self.duplicated is not None:
|
|
69
|
-
output["duplicated"] = self.duplicated
|
|
70
|
-
if self.is_auto_retryable is not None:
|
|
71
|
-
output["isAutoRetryable"] = self.is_auto_retryable
|
|
72
|
-
return output
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
@dataclass
|
|
76
|
-
class FailedRequestMessage(HTTPMessage):
|
|
77
|
-
error: str
|
|
78
|
-
|
|
79
|
-
def __str__(self) -> str:
|
|
80
|
-
return self.error
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
@dataclass
|
|
84
|
-
class ResponseMessage(HTTPMessage):
|
|
85
|
-
status_code: StatusCode
|
|
86
95
|
|
|
96
|
+
class FailedResponse(HTTPResult):
|
|
97
|
+
status_code: int
|
|
98
|
+
body: str
|
|
99
|
+
error: ErrorDetails
|
|
87
100
|
|
|
88
|
-
@dataclass
|
|
89
|
-
class RequestMessage(HTTPMessage):
|
|
90
|
-
"""Base class for HTTP request messages"""
|
|
91
101
|
|
|
102
|
+
class BaseRequestMessage(BaseModel, ABC):
|
|
92
103
|
endpoint_url: str
|
|
93
104
|
method: Literal["GET", "POST", "PATCH", "DELETE", "PUT"]
|
|
94
105
|
connect_attempt: int = 0
|
|
95
106
|
read_attempt: int = 0
|
|
96
107
|
status_attempt: int = 0
|
|
97
108
|
api_version: str | None = None
|
|
109
|
+
disable_gzip: bool = False
|
|
110
|
+
content_length: int | None = None
|
|
98
111
|
content_type: str = "application/json"
|
|
99
112
|
accept: str = "application/json"
|
|
100
|
-
|
|
113
|
+
|
|
114
|
+
parameters: dict[str, PrimitiveType] | None = None
|
|
101
115
|
|
|
102
116
|
@property
|
|
103
117
|
def total_attempts(self) -> int:
|
|
104
118
|
return self.connect_attempt + self.read_attempt + self.status_attempt
|
|
105
119
|
|
|
120
|
+
@property
|
|
106
121
|
@abstractmethod
|
|
107
|
-
def
|
|
108
|
-
raise NotImplementedError()
|
|
109
|
-
|
|
110
|
-
@abstractmethod
|
|
111
|
-
def create_failure_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
|
|
112
|
-
raise NotImplementedError()
|
|
113
|
-
|
|
114
|
-
@abstractmethod
|
|
115
|
-
def create_failed_request(self, error_message: str) -> Sequence[HTTPMessage]:
|
|
116
|
-
raise NotImplementedError()
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
@dataclass
|
|
120
|
-
class SuccessResponse(ResponseMessage):
|
|
121
|
-
body: str
|
|
122
|
-
content: bytes
|
|
123
|
-
|
|
124
|
-
def dump(self) -> dict[str, JsonVal]:
|
|
125
|
-
output = super().dump()
|
|
126
|
-
# We cannot serialize bytes, so we indicate its presence instead
|
|
127
|
-
output["content"] = "<bytes>" if self.content else None
|
|
128
|
-
return output
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
@dataclass
|
|
132
|
-
class FailedResponse(ResponseMessage):
|
|
133
|
-
body: str
|
|
134
|
-
error: ErrorDetails
|
|
135
|
-
|
|
136
|
-
def dump(self) -> dict[str, JsonVal]:
|
|
137
|
-
output = super().dump()
|
|
138
|
-
output["error"] = self.error.dump()
|
|
139
|
-
return output
|
|
140
|
-
|
|
141
|
-
def __str__(self) -> str:
|
|
142
|
-
return f"{self.error.code} | {self.error.message}"
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
@dataclass
|
|
146
|
-
class SimpleRequest(RequestMessage):
|
|
147
|
-
"""Base class for requests with a simple success/fail response structure"""
|
|
148
|
-
|
|
149
|
-
@classmethod
|
|
150
|
-
def create_success_response(cls, response: httpx.Response) -> Sequence[ResponseMessage]:
|
|
151
|
-
return [SuccessResponse(status_code=response.status_code, body=response.text, content=response.content)]
|
|
152
|
-
|
|
153
|
-
@classmethod
|
|
154
|
-
def create_failure_response(cls, response: httpx.Response) -> Sequence[HTTPMessage]:
|
|
155
|
-
return [
|
|
156
|
-
FailedResponse(
|
|
157
|
-
status_code=response.status_code, error=ErrorDetails.from_response(response), body=response.text
|
|
158
|
-
)
|
|
159
|
-
]
|
|
160
|
-
|
|
161
|
-
@classmethod
|
|
162
|
-
def create_failed_request(cls, error_message: str) -> Sequence[HTTPMessage]:
|
|
163
|
-
return [FailedRequestMessage(error=error_message)]
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
@dataclass
|
|
167
|
-
class BodyRequest(RequestMessage, ABC):
|
|
168
|
-
"""Base class for HTTP request messages with a body"""
|
|
169
|
-
|
|
170
|
-
@abstractmethod
|
|
171
|
-
def data(self) -> str:
|
|
172
|
-
raise NotImplementedError()
|
|
173
|
-
|
|
122
|
+
def content(self) -> str | bytes | None: ...
|
|
174
123
|
|
|
175
|
-
@dataclass
|
|
176
|
-
class ParamRequest(SimpleRequest):
|
|
177
|
-
"""Base class for HTTP request messages with query parameters"""
|
|
178
124
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
@dataclass
|
|
183
|
-
class SimpleBodyRequest(SimpleRequest, BodyRequest):
|
|
184
|
-
body_content: dict[str, JsonVal] = field(default_factory=dict)
|
|
185
|
-
|
|
186
|
-
def data(self) -> str:
|
|
187
|
-
return _dump_body(self.body_content)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
@dataclass
|
|
191
|
-
class DataBodyRequest(SimpleRequest):
|
|
192
|
-
data_content: bytes = b""
|
|
193
|
-
|
|
194
|
-
def data(self) -> bytes:
|
|
195
|
-
return self.data_content
|
|
196
|
-
|
|
197
|
-
def dump(self) -> dict[str, JsonVal]:
|
|
198
|
-
output = super().dump()
|
|
199
|
-
# We cannot serialize bytes, so we indicate its presence instead
|
|
200
|
-
output["data_content"] = "<bytes>"
|
|
201
|
-
return output
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
T_COVARIANT_ID = TypeVar("T_COVARIANT_ID", covariant=True)
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
@dataclass
|
|
208
|
-
class ItemMessage(Generic[T_COVARIANT_ID], ABC):
|
|
209
|
-
"""Base class for message related to a specific item identified by an ID"""
|
|
210
|
-
|
|
211
|
-
ids: list[T_COVARIANT_ID] = field(default_factory=list)
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
@dataclass
|
|
215
|
-
class SuccessResponseItems(ItemMessage[T_COVARIANT_ID], SuccessResponse): ...
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
@dataclass
|
|
219
|
-
class FailedResponseItems(ItemMessage[T_COVARIANT_ID], FailedResponse): ...
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
@dataclass
|
|
223
|
-
class FailedRequestItems(ItemMessage[T_COVARIANT_ID], FailedRequestMessage): ...
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
class RequestItem(Generic[T_COVARIANT_ID], Protocol):
|
|
227
|
-
def dump(self) -> JsonVal: ...
|
|
228
|
-
|
|
229
|
-
def as_id(self) -> T_COVARIANT_ID: ...
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
@dataclass
|
|
233
|
-
class ItemsRequest(Generic[T_COVARIANT_ID], BodyRequest):
|
|
234
|
-
"""Requests message for endpoints that accept multiple items in a single request.
|
|
235
|
-
|
|
236
|
-
This class provides functionality to split large requests into smaller ones, handle responses for each item,
|
|
237
|
-
and manage errors effectively.
|
|
238
|
-
|
|
239
|
-
Attributes:
|
|
240
|
-
items (list[T_RequestItem]): The list of items to be sent in the request body.
|
|
241
|
-
extra_body_fields (dict[str, JsonVal]): Additional fields to include in the request body
|
|
242
|
-
max_failures_before_abort (int): The maximum number of failed split requests before aborting further splits.
|
|
243
|
-
|
|
244
|
-
"""
|
|
245
|
-
|
|
246
|
-
items: list[RequestItem[T_COVARIANT_ID]] = field(default_factory=list)
|
|
247
|
-
extra_body_fields: dict[str, JsonVal] = field(default_factory=dict)
|
|
248
|
-
max_failures_before_abort: int = 50
|
|
249
|
-
tracker: ItemsRequestTracker | None = field(default=None, init=False)
|
|
125
|
+
class RequestMessage(BaseRequestMessage):
|
|
126
|
+
data_content: bytes | None = None
|
|
127
|
+
body_content: dict[str, JsonValue] | None = None
|
|
250
128
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
Returns:
|
|
257
|
-
dict[str, JsonVal]: The message as a dictionary.
|
|
258
|
-
"""
|
|
259
|
-
output = super().dump()
|
|
260
|
-
output["items"] = self.dump_items()
|
|
261
|
-
if self.tracker is not None:
|
|
262
|
-
# We cannot serialize the tracker
|
|
263
|
-
del output["tracker"]
|
|
264
|
-
return output
|
|
265
|
-
|
|
266
|
-
def dump_items(self) -> list[JsonVal]:
|
|
267
|
-
"""Dumps the items to a list of JSON serializable dictionaries.
|
|
268
|
-
|
|
269
|
-
Returns:
|
|
270
|
-
list[JsonVal]: The items as a list of dictionaries.
|
|
271
|
-
"""
|
|
272
|
-
return [item.dump() for item in self.items]
|
|
273
|
-
|
|
274
|
-
def body(self) -> dict[str, JsonVal]:
|
|
275
|
-
if self.extra_body_fields:
|
|
276
|
-
return {"items": self.dump_items(), **self.extra_body_fields}
|
|
277
|
-
return {"items": self.dump_items()}
|
|
278
|
-
|
|
279
|
-
def data(self) -> str:
|
|
280
|
-
return _dump_body(self.body())
|
|
281
|
-
|
|
282
|
-
def split(self, status_attempts: int) -> "list[ItemsRequest]":
|
|
283
|
-
"""Splits the request into two smaller requests.
|
|
284
|
-
|
|
285
|
-
This is useful for retrying requests that fail due to size limits or timeouts.
|
|
286
|
-
|
|
287
|
-
Args:
|
|
288
|
-
status_attempts: The number of status attempts to set for the new requests. This is used when the
|
|
289
|
-
request failed with a 5xx status code and we want to track the number of attempts. For 4xx errors,
|
|
290
|
-
there is at least one item causing the error, so we do not increment the status attempts, but
|
|
291
|
-
instead essentially do a binary search to find the problematic item(s).
|
|
292
|
-
|
|
293
|
-
Returns:
|
|
294
|
-
A list containing two new ItemsRequest instances, each with half of the original items.
|
|
295
|
-
|
|
296
|
-
"""
|
|
297
|
-
mid = len(self.items) // 2
|
|
298
|
-
if mid == 0:
|
|
299
|
-
return [self]
|
|
300
|
-
tracker = self.tracker or ItemsRequestTracker(self.max_failures_before_abort)
|
|
301
|
-
tracker.register_failure()
|
|
302
|
-
first_half = ItemsRequest[T_COVARIANT_ID](
|
|
303
|
-
endpoint_url=self.endpoint_url,
|
|
304
|
-
method=self.method,
|
|
305
|
-
items=self.items[:mid],
|
|
306
|
-
extra_body_fields=self.extra_body_fields,
|
|
307
|
-
connect_attempt=self.connect_attempt,
|
|
308
|
-
read_attempt=self.read_attempt,
|
|
309
|
-
status_attempt=status_attempts,
|
|
310
|
-
api_version=self.api_version,
|
|
311
|
-
content_type=self.content_type,
|
|
312
|
-
accept=self.accept,
|
|
313
|
-
content_length=self.content_length,
|
|
314
|
-
max_failures_before_abort=self.max_failures_before_abort,
|
|
315
|
-
)
|
|
316
|
-
first_half.tracker = tracker
|
|
317
|
-
second_half = ItemsRequest[T_COVARIANT_ID](
|
|
318
|
-
endpoint_url=self.endpoint_url,
|
|
319
|
-
method=self.method,
|
|
320
|
-
items=self.items[mid:],
|
|
321
|
-
extra_body_fields=self.extra_body_fields,
|
|
322
|
-
connect_attempt=self.connect_attempt,
|
|
323
|
-
read_attempt=self.read_attempt,
|
|
324
|
-
status_attempt=status_attempts,
|
|
325
|
-
api_version=self.api_version,
|
|
326
|
-
content_type=self.content_type,
|
|
327
|
-
accept=self.accept,
|
|
328
|
-
content_length=self.content_length,
|
|
329
|
-
max_failures_before_abort=self.max_failures_before_abort,
|
|
330
|
-
)
|
|
331
|
-
second_half.tracker = tracker
|
|
332
|
-
return [first_half, second_half]
|
|
333
|
-
|
|
334
|
-
def create_success_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
|
|
335
|
-
ids = [item.as_id() for item in self.items]
|
|
336
|
-
return [
|
|
337
|
-
SuccessResponseItems(
|
|
338
|
-
status_code=response.status_code, ids=ids, body=response.text, content=response.content
|
|
339
|
-
)
|
|
340
|
-
]
|
|
341
|
-
|
|
342
|
-
def create_failure_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
|
|
343
|
-
error = ErrorDetails.from_response(response)
|
|
344
|
-
ids = [item.as_id() for item in self.items]
|
|
345
|
-
return [FailedResponseItems(status_code=response.status_code, ids=ids, error=error, body=response.text)]
|
|
346
|
-
|
|
347
|
-
def create_failed_request(self, error_message: str) -> Sequence[HTTPMessage]:
|
|
348
|
-
ids = [item.as_id() for item in self.items]
|
|
349
|
-
return [FailedRequestItems(ids=ids, error=error_message)]
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
class ResponseList(UserList[ResponseMessage | FailedRequestMessage]):
|
|
353
|
-
def __init__(self, collection: Sequence[ResponseMessage | FailedRequestMessage] | None = None) -> None:
|
|
354
|
-
super().__init__(collection or [])
|
|
355
|
-
|
|
356
|
-
def raise_for_status(self) -> None:
|
|
357
|
-
"""Raises an exception if any response in the list indicates a failure."""
|
|
358
|
-
failed_responses = [resp for resp in self.data if isinstance(resp, FailedResponse)]
|
|
359
|
-
failed_requests = [resp for resp in self.data if isinstance(resp, FailedRequestMessage)]
|
|
360
|
-
if not failed_responses and not failed_requests:
|
|
361
|
-
return
|
|
362
|
-
error_messages = "; ".join(f"Status {err.status_code}: {err.error}" for err in failed_responses)
|
|
363
|
-
if failed_requests:
|
|
364
|
-
if error_messages:
|
|
365
|
-
error_messages += "; "
|
|
366
|
-
error_messages += "; ".join(f"Request error: {err.error}" for err in failed_requests)
|
|
367
|
-
raise ToolkitAPIError(f"One or more requests failed: {error_messages}")
|
|
129
|
+
@model_validator(mode="before")
|
|
130
|
+
def check_data_or_body(cls, values: dict[str, Any]) -> dict[str, Any]:
|
|
131
|
+
if values.get("data_content") is not None and values.get("body_content") is not None:
|
|
132
|
+
raise ValueError("Only one of data_content or body_content can be set.")
|
|
133
|
+
return values
|
|
368
134
|
|
|
369
135
|
@property
|
|
370
|
-
def
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
Returns:
|
|
388
|
-
dict[str, JsonVal]: The body of the first successful response.
|
|
389
|
-
"""
|
|
390
|
-
for resp in self.data:
|
|
391
|
-
if isinstance(resp, SuccessResponse) and resp.body is not None:
|
|
392
|
-
return _json.loads(resp.body)
|
|
393
|
-
raise ValueError("No successful responses with a body found.")
|
|
394
|
-
|
|
395
|
-
def as_item_responses(self, item_id: Hashable) -> list[ResponseMessage | FailedRequestMessage]:
|
|
396
|
-
# Convert the responses to per-item responses
|
|
397
|
-
results: list[ResponseMessage | FailedRequestMessage] = []
|
|
398
|
-
for message in self.data:
|
|
399
|
-
if isinstance(message, SuccessResponse):
|
|
400
|
-
results.append(
|
|
401
|
-
SuccessResponseItems(
|
|
402
|
-
status_code=message.status_code, content=message.content, ids=[item_id], body=message.body
|
|
403
|
-
)
|
|
404
|
-
)
|
|
405
|
-
elif isinstance(message, FailedResponse):
|
|
406
|
-
results.append(
|
|
407
|
-
FailedResponseItems(
|
|
408
|
-
status_code=message.status_code, ids=[item_id], body=message.body, error=message.error
|
|
409
|
-
)
|
|
410
|
-
)
|
|
411
|
-
elif isinstance(message, FailedRequestMessage):
|
|
412
|
-
results.append(FailedRequestItems(ids=[item_id], error=message.error))
|
|
413
|
-
else:
|
|
414
|
-
results.append(message)
|
|
415
|
-
return results
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
def _dump_body(body: dict[str, JsonVal]) -> str:
|
|
419
|
-
try:
|
|
420
|
-
return _json.dumps(body, allow_nan=False)
|
|
421
|
-
except ValueError as e:
|
|
422
|
-
# A lot of work to give a more human friendly error message when nans and infs are present:
|
|
423
|
-
msg = "Out of range float values are not JSON compliant"
|
|
424
|
-
if msg in str(e): # exc. might e.g. contain an extra ": nan", depending on build (_json.make_encoder)
|
|
425
|
-
raise ValueError(f"{msg}. Make sure your data does not contain NaN(s) or +/- Inf!").with_traceback(
|
|
426
|
-
e.__traceback__
|
|
427
|
-
) from None
|
|
428
|
-
raise
|
|
136
|
+
def content(self) -> str | bytes | None:
|
|
137
|
+
data: str | bytes | None = None
|
|
138
|
+
if self.data_content is not None:
|
|
139
|
+
data = self.data_content
|
|
140
|
+
if not global_config.disable_gzip and not self.disable_gzip:
|
|
141
|
+
data = gzip.compress(data)
|
|
142
|
+
elif self.body_content is not None:
|
|
143
|
+
# We serialize using pydantic instead of json.dumps. This is because pydantic is faster
|
|
144
|
+
# and handles more complex types such as datetime, float('nan'), etc.
|
|
145
|
+
data = _BODY_SERIALIZER.dump_json(self.body_content)
|
|
146
|
+
if not global_config.disable_gzip and not self.disable_gzip and isinstance(data, bytes):
|
|
147
|
+
data = gzip.compress(data)
|
|
148
|
+
return data
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
_BODY_SERIALIZER = TypeAdapter(dict[str, JsonValue])
|
|
@@ -9,32 +9,32 @@ from cognite.client import global_config
|
|
|
9
9
|
from pydantic import BaseModel, ConfigDict, Field, JsonValue
|
|
10
10
|
|
|
11
11
|
from cognite_toolkit._cdf_tk.client._resource_base import RequestItem
|
|
12
|
-
from cognite_toolkit._cdf_tk.client.http_client.
|
|
12
|
+
from cognite_toolkit._cdf_tk.client.http_client._data_classes import (
|
|
13
13
|
_BODY_SERIALIZER,
|
|
14
14
|
BaseRequestMessage,
|
|
15
|
-
|
|
15
|
+
ErrorDetails,
|
|
16
16
|
)
|
|
17
17
|
from cognite_toolkit._cdf_tk.client.http_client._exception import ToolkitAPIError
|
|
18
18
|
from cognite_toolkit._cdf_tk.client.http_client._tracker import ItemsRequestTracker
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
class
|
|
21
|
+
class ItemsResultMessage(BaseModel):
|
|
22
22
|
ids: list[str]
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
class
|
|
25
|
+
class ItemsFailedRequest(ItemsResultMessage):
|
|
26
26
|
error_message: str
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
class
|
|
29
|
+
class ItemsSuccessResponse(ItemsResultMessage):
|
|
30
30
|
status_code: int
|
|
31
31
|
body: str
|
|
32
32
|
content: bytes
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
class
|
|
35
|
+
class ItemsFailedResponse(ItemsResultMessage):
|
|
36
36
|
status_code: int
|
|
37
|
-
error:
|
|
37
|
+
error: ErrorDetails
|
|
38
38
|
body: str
|
|
39
39
|
|
|
40
40
|
|
|
@@ -44,7 +44,7 @@ def _set_default_tracker(data: dict[str, Any]) -> ItemsRequestTracker:
|
|
|
44
44
|
return data["tracker"]
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
class
|
|
47
|
+
class ItemsRequest(BaseRequestMessage):
|
|
48
48
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
49
49
|
items: Sequence[RequestItem]
|
|
50
50
|
extra_body_fields: dict[str, JsonValue] | None = None
|
|
@@ -61,13 +61,13 @@ class ItemsRequest2(BaseRequestMessage):
|
|
|
61
61
|
return gzip.compress(res)
|
|
62
62
|
return res
|
|
63
63
|
|
|
64
|
-
def split(self, status_attempts: int) -> list["
|
|
64
|
+
def split(self, status_attempts: int) -> list["ItemsRequest"]:
|
|
65
65
|
"""Split the request into multiple requests with a single item each."""
|
|
66
66
|
mid = len(self.items) // 2
|
|
67
67
|
if mid == 0:
|
|
68
68
|
return [self]
|
|
69
69
|
self.tracker.register_failure()
|
|
70
|
-
messages: list[
|
|
70
|
+
messages: list[ItemsRequest] = []
|
|
71
71
|
for part in (self.items[:mid], self.items[mid:]):
|
|
72
72
|
new_request = self.model_copy(update={"items": part, "status_attempt": status_attempts})
|
|
73
73
|
new_request.tracker = self.tracker
|
|
@@ -79,14 +79,14 @@ class ItemResponse(BaseModel):
|
|
|
79
79
|
items: list[dict[str, JsonValue]]
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
class ItemsResultList(UserList[
|
|
83
|
-
def __init__(self, collection: Sequence[
|
|
82
|
+
class ItemsResultList(UserList[ItemsResultMessage]):
|
|
83
|
+
def __init__(self, collection: Sequence[ItemsResultMessage] | None = None) -> None:
|
|
84
84
|
super().__init__(collection or [])
|
|
85
85
|
|
|
86
86
|
def raise_for_status(self) -> None:
|
|
87
87
|
"""Raises an exception if any response in the list indicates a failure."""
|
|
88
|
-
failed_responses = [resp for resp in self.data if isinstance(resp,
|
|
89
|
-
failed_requests = [resp for resp in self.data if isinstance(resp,
|
|
88
|
+
failed_responses = [resp for resp in self.data if isinstance(resp, ItemsFailedResponse)]
|
|
89
|
+
failed_requests = [resp for resp in self.data if isinstance(resp, ItemsFailedRequest)]
|
|
90
90
|
if not failed_responses and not failed_requests:
|
|
91
91
|
return
|
|
92
92
|
error_messages = "; ".join(f"Status {err.status_code}: {err.error.message}" for err in failed_responses)
|
|
@@ -104,7 +104,7 @@ class ItemsResultList(UserList[ItemsResultMessage2]):
|
|
|
104
104
|
bool: True if there are any failed responses or requests, False otherwise.
|
|
105
105
|
"""
|
|
106
106
|
for resp in self.data:
|
|
107
|
-
if isinstance(resp,
|
|
107
|
+
if isinstance(resp, ItemsFailedResponse | ItemsFailedRequest):
|
|
108
108
|
return True
|
|
109
109
|
return False
|
|
110
110
|
|
|
@@ -112,7 +112,7 @@ class ItemsResultList(UserList[ItemsResultMessage2]):
|
|
|
112
112
|
"""Get the items from all successful responses."""
|
|
113
113
|
items: list[dict[str, JsonValue]] = []
|
|
114
114
|
for resp in self.data:
|
|
115
|
-
if isinstance(resp,
|
|
115
|
+
if isinstance(resp, ItemsSuccessResponse):
|
|
116
116
|
body_json = ItemResponse.model_validate_json(resp.body)
|
|
117
117
|
items.extend(body_json.items)
|
|
118
118
|
return items
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import ClassVar, Literal
|
|
1
|
+
from typing import Any, ClassVar, Literal
|
|
2
2
|
|
|
3
3
|
from pydantic import Field, JsonValue
|
|
4
4
|
|
|
@@ -37,6 +37,12 @@ class FileMetadataRequest(FileMetadata, UpdatableRequestResource):
|
|
|
37
37
|
# from response to request.
|
|
38
38
|
instance_id: NodeReference | None = Field(default=None, exclude=True)
|
|
39
39
|
|
|
40
|
+
def as_update(self, mode: Literal["patch", "replace"]) -> dict[str, Any]:
|
|
41
|
+
update = super().as_update(mode)
|
|
42
|
+
# Name cannot be updated.
|
|
43
|
+
update["update"].pop("name", None)
|
|
44
|
+
return update
|
|
45
|
+
|
|
40
46
|
|
|
41
47
|
class FileMetadataResponse(FileMetadata, ResponseResource[FileMetadataRequest]):
|
|
42
48
|
created_time: int
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from cognite_toolkit._cdf_tk.client._resource_base import BaseModelObject
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class UserProfilesConfiguration(BaseModelObject):
|
|
5
|
+
enabled: bool
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Claim(BaseModelObject):
|
|
9
|
+
claim_name: str
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class OidcConfiguration(BaseModelObject):
|
|
13
|
+
jwks_url: str
|
|
14
|
+
token_url: str | None = None
|
|
15
|
+
issuer: str
|
|
16
|
+
audience: str
|
|
17
|
+
skew_ms: int | None = None
|
|
18
|
+
access_claims: list[Claim]
|
|
19
|
+
scope_claims: list[Claim]
|
|
20
|
+
log_claims: list[Claim]
|
|
21
|
+
is_group_callback_enabled: bool | None = None
|
|
22
|
+
identity_provider_scope: str | None = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class OrganizationResponse(BaseModelObject):
|
|
26
|
+
name: str
|
|
27
|
+
url_name: str
|
|
28
|
+
organization: str
|
|
29
|
+
user_profiles_configuration: UserProfilesConfiguration
|
|
30
|
+
oidc_configuration: OidcConfiguration | None = None
|