cognite-toolkit 0.7.67__py3-none-any.whl → 0.7.69__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/client/api/instances.py +7 -3
- cognite_toolkit/_cdf_tk/client/api/security_categories.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/transformations.py +4 -21
- cognite_toolkit/_cdf_tk/client/api/workflow_versions.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/workflows.py +2 -2
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_instance.py +19 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/extraction_pipeline.py +7 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_job.py +19 -5
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_mapping.py +2 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_auth.py +1 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_base.py +2 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_eventhub.py +3 -2
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_kafka.py +2 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_mqtt.py +2 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_rest.py +2 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/identifiers.py +23 -2
- cognite_toolkit/_cdf_tk/client/resource_classes/securitycategory.py +13 -4
- cognite_toolkit/_cdf_tk/client/resource_classes/sequence.py +44 -2
- cognite_toolkit/_cdf_tk/client/resource_classes/transformation.py +4 -3
- cognite_toolkit/_cdf_tk/client/testing.py +10 -0
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py +15 -4
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/hosted_extractors.py +160 -125
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.67.dist-info → cognite_toolkit-0.7.69.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.7.67.dist-info → cognite_toolkit-0.7.69.dist-info}/RECORD +30 -30
- {cognite_toolkit-0.7.67.dist-info → cognite_toolkit-0.7.69.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.7.67.dist-info → cognite_toolkit-0.7.69.dist-info}/entry_points.txt +0 -0
|
@@ -4,33 +4,36 @@ from typing import Any
|
|
|
4
4
|
|
|
5
5
|
from cognite.client.data_classes import ClientCredentials
|
|
6
6
|
from cognite.client.data_classes.capabilities import Capability, HostedExtractorsAcl
|
|
7
|
-
from cognite.client.data_classes.hosted_extractors import (
|
|
8
|
-
Destination,
|
|
9
|
-
DestinationList,
|
|
10
|
-
DestinationWrite,
|
|
11
|
-
EventHubSourceWrite,
|
|
12
|
-
Job,
|
|
13
|
-
JobList,
|
|
14
|
-
JobWrite,
|
|
15
|
-
KafkaSourceWrite,
|
|
16
|
-
Mapping,
|
|
17
|
-
MappingList,
|
|
18
|
-
MappingWrite,
|
|
19
|
-
Source,
|
|
20
|
-
SourceList,
|
|
21
|
-
SourceWrite,
|
|
22
|
-
)
|
|
23
|
-
from cognite.client.data_classes.hosted_extractors.sources import (
|
|
24
|
-
AuthenticationWrite,
|
|
25
|
-
BasicAuthenticationWrite,
|
|
26
|
-
RESTClientCredentialsAuthenticationWrite,
|
|
27
|
-
RestSourceWrite,
|
|
28
|
-
_MQTTSourceWrite,
|
|
29
|
-
)
|
|
30
7
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
31
8
|
from rich.console import Console
|
|
32
9
|
|
|
33
10
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
11
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.hosted_extractor_destination import (
|
|
12
|
+
HostedExtractorDestinationRequest,
|
|
13
|
+
HostedExtractorDestinationResponse,
|
|
14
|
+
)
|
|
15
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.hosted_extractor_job import (
|
|
16
|
+
HostedExtractorJobRequest,
|
|
17
|
+
HostedExtractorJobResponse,
|
|
18
|
+
)
|
|
19
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.hosted_extractor_mapping import (
|
|
20
|
+
HostedExtractorMappingRequest,
|
|
21
|
+
HostedExtractorMappingResponse,
|
|
22
|
+
)
|
|
23
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.hosted_extractor_source import (
|
|
24
|
+
BasicAuthenticationRequest,
|
|
25
|
+
ClientCredentialAuthenticationRequest,
|
|
26
|
+
EventHubSourceRequest,
|
|
27
|
+
HostedExtractorSourceRequest,
|
|
28
|
+
HostedExtractorSourceRequestUnion,
|
|
29
|
+
HostedExtractorSourceResponseUnion,
|
|
30
|
+
HTTPBasicAuthenticationRequest,
|
|
31
|
+
KafkaSourceRequest,
|
|
32
|
+
MQTTSourceRequest,
|
|
33
|
+
RESTSourceRequest,
|
|
34
|
+
ScramShaAuthenticationRequest,
|
|
35
|
+
)
|
|
36
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId
|
|
34
37
|
from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
|
|
35
38
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitNotSupported
|
|
36
39
|
from cognite_toolkit._cdf_tk.resource_classes import (
|
|
@@ -44,33 +47,35 @@ from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning
|
|
|
44
47
|
from .data_organization import DataSetsCRUD
|
|
45
48
|
|
|
46
49
|
|
|
47
|
-
class HostedExtractorSourceCRUD(
|
|
50
|
+
class HostedExtractorSourceCRUD(
|
|
51
|
+
ResourceCRUD[ExternalId, HostedExtractorSourceRequestUnion, HostedExtractorSourceResponseUnion]
|
|
52
|
+
):
|
|
48
53
|
folder_name = "hosted_extractors"
|
|
49
|
-
resource_cls =
|
|
50
|
-
resource_write_cls =
|
|
54
|
+
resource_cls = HostedExtractorSourceResponseUnion # type: ignore[assignment]
|
|
55
|
+
resource_write_cls = HostedExtractorSourceRequestUnion # type: ignore[assignment]
|
|
51
56
|
kind = "Source"
|
|
52
57
|
yaml_cls = HostedExtractorSourceYAML
|
|
53
58
|
_doc_base_url = "https://api-docs.cognite.com/20230101-alpha/tag/"
|
|
54
59
|
_doc_url = "Sources/operation/create_sources"
|
|
55
|
-
_SupportedSources = (
|
|
60
|
+
_SupportedSources = (MQTTSourceRequest, KafkaSourceRequest, RESTSourceRequest, EventHubSourceRequest)
|
|
56
61
|
|
|
57
62
|
@property
|
|
58
63
|
def display_name(self) -> str:
|
|
59
64
|
return "hosted extractor sources"
|
|
60
65
|
|
|
61
66
|
@classmethod
|
|
62
|
-
def get_id(cls, item:
|
|
67
|
+
def get_id(cls, item: HostedExtractorSourceRequestUnion | HostedExtractorSourceResponseUnion | dict) -> ExternalId:
|
|
63
68
|
if isinstance(item, dict):
|
|
64
|
-
return item["externalId"]
|
|
65
|
-
return item.external_id
|
|
69
|
+
return ExternalId(external_id=item["externalId"])
|
|
70
|
+
return ExternalId(external_id=item.external_id)
|
|
66
71
|
|
|
67
72
|
@classmethod
|
|
68
|
-
def dump_id(cls, id:
|
|
69
|
-
return
|
|
73
|
+
def dump_id(cls, id: ExternalId) -> dict[str, Any]:
|
|
74
|
+
return id.dump()
|
|
70
75
|
|
|
71
76
|
@classmethod
|
|
72
77
|
def get_required_capability(
|
|
73
|
-
cls, items: Sequence[
|
|
78
|
+
cls, items: Sequence[HostedExtractorSourceRequestUnion] | None, read_only: bool
|
|
74
79
|
) -> Capability | list[Capability]:
|
|
75
80
|
if not items and items is not None:
|
|
76
81
|
return []
|
|
@@ -86,17 +91,19 @@ class HostedExtractorSourceCRUD(ResourceCRUD[str, SourceWrite, Source]):
|
|
|
86
91
|
HostedExtractorsAcl.Scope.All(),
|
|
87
92
|
)
|
|
88
93
|
|
|
89
|
-
def create(self, items: Sequence[
|
|
90
|
-
return self.client.hosted_extractors.sources.create(items)
|
|
94
|
+
def create(self, items: Sequence[HostedExtractorSourceRequestUnion]) -> list[HostedExtractorSourceResponseUnion]:
|
|
95
|
+
return self.client.tool.hosted_extractors.sources.create(list(items))
|
|
91
96
|
|
|
92
|
-
def retrieve(self, ids: SequenceNotStr[
|
|
93
|
-
return self.client.hosted_extractors.sources.retrieve(
|
|
97
|
+
def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[HostedExtractorSourceResponseUnion]:
|
|
98
|
+
return self.client.tool.hosted_extractors.sources.retrieve(list(ids), ignore_unknown_ids=True)
|
|
94
99
|
|
|
95
|
-
def update(self, items: Sequence[
|
|
96
|
-
return self.client.hosted_extractors.sources.update(items, mode="replace")
|
|
100
|
+
def update(self, items: Sequence[HostedExtractorSourceRequestUnion]) -> list[HostedExtractorSourceResponseUnion]:
|
|
101
|
+
return self.client.tool.hosted_extractors.sources.update(list(items), mode="replace")
|
|
97
102
|
|
|
98
|
-
def delete(self, ids: SequenceNotStr[
|
|
99
|
-
|
|
103
|
+
def delete(self, ids: SequenceNotStr[ExternalId]) -> int:
|
|
104
|
+
if not ids:
|
|
105
|
+
return 0
|
|
106
|
+
self.client.tool.hosted_extractors.sources.delete(list(ids), ignore_unknown_ids=True)
|
|
100
107
|
return len(ids)
|
|
101
108
|
|
|
102
109
|
def _iterate(
|
|
@@ -104,50 +111,62 @@ class HostedExtractorSourceCRUD(ResourceCRUD[str, SourceWrite, Source]):
|
|
|
104
111
|
data_set_external_id: str | None = None,
|
|
105
112
|
space: str | None = None,
|
|
106
113
|
parent_ids: list[Hashable] | None = None,
|
|
107
|
-
) -> Iterable[
|
|
108
|
-
|
|
114
|
+
) -> Iterable[HostedExtractorSourceResponseUnion]:
|
|
115
|
+
for sources in self.client.tool.hosted_extractors.sources.iterate():
|
|
116
|
+
yield from sources
|
|
109
117
|
|
|
110
|
-
def dump_resource(
|
|
118
|
+
def dump_resource(
|
|
119
|
+
self, resource: HostedExtractorSourceResponseUnion, local: dict[str, Any] | None = None
|
|
120
|
+
) -> dict[str, Any]:
|
|
111
121
|
HighSeverityWarning(
|
|
112
122
|
"Sources will always be considered different, and thus will always be redeployed."
|
|
113
123
|
).print_warning()
|
|
114
|
-
return self.dump_id(resource
|
|
124
|
+
return self.dump_id(self.get_id(resource))
|
|
115
125
|
|
|
116
|
-
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) ->
|
|
117
|
-
loaded =
|
|
126
|
+
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> HostedExtractorSourceRequestUnion:
|
|
127
|
+
loaded = HostedExtractorSourceRequest.validate_python(resource)
|
|
118
128
|
if not isinstance(loaded, self._SupportedSources):
|
|
119
129
|
# We need to explicitly check for the supported types as we need to ensure that we know
|
|
120
130
|
# what the sensitive strings are for each type of source.
|
|
121
|
-
# Technically, we could support any new source type
|
|
131
|
+
# Technically, we could support any new source type, but
|
|
122
132
|
# we would risk printing sensitive strings in the terminal.
|
|
123
133
|
raise ToolkitNotSupported(
|
|
124
|
-
f"Hosted extractor source type {type(loaded).__name__} is not supported."
|
|
134
|
+
f"Hosted extractor source type {type(loaded).__name__} is not supported. "
|
|
125
135
|
f"Please contact support to request support for this source type."
|
|
126
136
|
)
|
|
127
137
|
return loaded
|
|
128
138
|
|
|
129
|
-
def sensitive_strings(self, item:
|
|
130
|
-
if isinstance(item,
|
|
139
|
+
def sensitive_strings(self, item: HostedExtractorSourceRequestUnion) -> Iterable[str]:
|
|
140
|
+
if isinstance(item, MQTTSourceRequest | KafkaSourceRequest | RESTSourceRequest) and item.authentication:
|
|
131
141
|
yield from self._sensitive_auth_strings(item.authentication)
|
|
132
142
|
if (
|
|
133
|
-
isinstance(item,
|
|
143
|
+
isinstance(item, MQTTSourceRequest | KafkaSourceRequest)
|
|
134
144
|
and item.auth_certificate
|
|
135
145
|
and item.auth_certificate.key_password
|
|
136
146
|
):
|
|
137
147
|
yield item.auth_certificate.key_password
|
|
138
148
|
|
|
139
149
|
@staticmethod
|
|
140
|
-
def _sensitive_auth_strings(
|
|
141
|
-
|
|
150
|
+
def _sensitive_auth_strings(
|
|
151
|
+
auth: BasicAuthenticationRequest
|
|
152
|
+
| ClientCredentialAuthenticationRequest
|
|
153
|
+
| ScramShaAuthenticationRequest
|
|
154
|
+
| HTTPBasicAuthenticationRequest,
|
|
155
|
+
) -> Iterable[str]:
|
|
156
|
+
if isinstance(auth, BasicAuthenticationRequest | ScramShaAuthenticationRequest) and auth.password:
|
|
142
157
|
yield auth.password
|
|
143
|
-
elif isinstance(auth,
|
|
158
|
+
elif isinstance(auth, ClientCredentialAuthenticationRequest):
|
|
144
159
|
yield auth.client_secret
|
|
160
|
+
elif isinstance(auth, HTTPBasicAuthenticationRequest):
|
|
161
|
+
yield auth.value
|
|
145
162
|
|
|
146
163
|
|
|
147
|
-
class HostedExtractorDestinationCRUD(
|
|
164
|
+
class HostedExtractorDestinationCRUD(
|
|
165
|
+
ResourceCRUD[ExternalId, HostedExtractorDestinationRequest, HostedExtractorDestinationResponse]
|
|
166
|
+
):
|
|
148
167
|
folder_name = "hosted_extractors"
|
|
149
|
-
resource_cls =
|
|
150
|
-
resource_write_cls =
|
|
168
|
+
resource_cls = HostedExtractorDestinationResponse
|
|
169
|
+
resource_write_cls = HostedExtractorDestinationRequest
|
|
151
170
|
dependencies = frozenset({DataSetsCRUD})
|
|
152
171
|
kind = "Destination"
|
|
153
172
|
_doc_base_url = "https://api-docs.cognite.com/20230101-alpha/tag/"
|
|
@@ -163,18 +182,18 @@ class HostedExtractorDestinationCRUD(ResourceCRUD[str, DestinationWrite, Destina
|
|
|
163
182
|
return "hosted extractor destinations"
|
|
164
183
|
|
|
165
184
|
@classmethod
|
|
166
|
-
def get_id(cls, item:
|
|
185
|
+
def get_id(cls, item: HostedExtractorDestinationRequest | HostedExtractorDestinationResponse | dict) -> ExternalId:
|
|
167
186
|
if isinstance(item, dict):
|
|
168
|
-
return item["externalId"]
|
|
169
|
-
return item.external_id
|
|
187
|
+
return ExternalId(external_id=item["externalId"])
|
|
188
|
+
return ExternalId(external_id=item.external_id)
|
|
170
189
|
|
|
171
190
|
@classmethod
|
|
172
|
-
def dump_id(cls, id:
|
|
173
|
-
return
|
|
191
|
+
def dump_id(cls, id: ExternalId) -> dict[str, Any]:
|
|
192
|
+
return id.dump()
|
|
174
193
|
|
|
175
194
|
@classmethod
|
|
176
195
|
def get_required_capability(
|
|
177
|
-
cls, items: Sequence[
|
|
196
|
+
cls, items: Sequence[HostedExtractorDestinationRequest] | None, read_only: bool
|
|
178
197
|
) -> Capability | list[Capability]:
|
|
179
198
|
if not items and items is not None:
|
|
180
199
|
return []
|
|
@@ -190,17 +209,19 @@ class HostedExtractorDestinationCRUD(ResourceCRUD[str, DestinationWrite, Destina
|
|
|
190
209
|
HostedExtractorsAcl.Scope.All(),
|
|
191
210
|
)
|
|
192
211
|
|
|
193
|
-
def create(self, items: Sequence[
|
|
194
|
-
return self.client.hosted_extractors.destinations.create(items)
|
|
212
|
+
def create(self, items: Sequence[HostedExtractorDestinationRequest]) -> list[HostedExtractorDestinationResponse]:
|
|
213
|
+
return self.client.tool.hosted_extractors.destinations.create(list(items))
|
|
195
214
|
|
|
196
|
-
def retrieve(self, ids: SequenceNotStr[
|
|
197
|
-
return self.client.hosted_extractors.destinations.retrieve(
|
|
215
|
+
def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[HostedExtractorDestinationResponse]:
|
|
216
|
+
return self.client.tool.hosted_extractors.destinations.retrieve(list(ids), ignore_unknown_ids=True)
|
|
198
217
|
|
|
199
|
-
def update(self, items: Sequence[
|
|
200
|
-
return self.client.hosted_extractors.destinations.update(items, mode="replace")
|
|
218
|
+
def update(self, items: Sequence[HostedExtractorDestinationRequest]) -> list[HostedExtractorDestinationResponse]:
|
|
219
|
+
return self.client.tool.hosted_extractors.destinations.update(list(items), mode="replace")
|
|
201
220
|
|
|
202
|
-
def delete(self, ids: SequenceNotStr[
|
|
203
|
-
|
|
221
|
+
def delete(self, ids: SequenceNotStr[ExternalId]) -> int:
|
|
222
|
+
if not ids:
|
|
223
|
+
return 0
|
|
224
|
+
self.client.tool.hosted_extractors.destinations.delete(list(ids), ignore_unknown_ids=True)
|
|
204
225
|
return len(ids)
|
|
205
226
|
|
|
206
227
|
def _iterate(
|
|
@@ -208,13 +229,14 @@ class HostedExtractorDestinationCRUD(ResourceCRUD[str, DestinationWrite, Destina
|
|
|
208
229
|
data_set_external_id: str | None = None,
|
|
209
230
|
space: str | None = None,
|
|
210
231
|
parent_ids: list[Hashable] | None = None,
|
|
211
|
-
) -> Iterable[
|
|
212
|
-
|
|
232
|
+
) -> Iterable[HostedExtractorDestinationResponse]:
|
|
233
|
+
for destinations in self.client.tool.hosted_extractors.destinations.iterate():
|
|
234
|
+
yield from destinations
|
|
213
235
|
|
|
214
|
-
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) ->
|
|
236
|
+
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> HostedExtractorDestinationRequest:
|
|
215
237
|
if raw_auth := resource.pop("credentials", None):
|
|
216
238
|
credentials = ClientCredentials._load(raw_auth)
|
|
217
|
-
self._authentication_by_id[self.get_id(resource)] = credentials
|
|
239
|
+
self._authentication_by_id[self.get_id(resource).external_id] = credentials
|
|
218
240
|
if is_dry_run:
|
|
219
241
|
resource["credentials"] = {"nonce": "dummy_nonce"}
|
|
220
242
|
else:
|
|
@@ -222,30 +244,33 @@ class HostedExtractorDestinationCRUD(ResourceCRUD[str, DestinationWrite, Destina
|
|
|
222
244
|
resource["credentials"] = {"nonce": session.nonce}
|
|
223
245
|
if ds_external_id := resource.pop("targetDataSetExternalId", None):
|
|
224
246
|
resource["targetDataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
|
|
225
|
-
return
|
|
247
|
+
return HostedExtractorDestinationRequest.model_validate(resource)
|
|
226
248
|
|
|
227
|
-
def dump_resource(
|
|
249
|
+
def dump_resource(
|
|
250
|
+
self, resource: HostedExtractorDestinationResponse, local: dict[str, Any] | None = None
|
|
251
|
+
) -> dict[str, Any]:
|
|
228
252
|
HighSeverityWarning(
|
|
229
253
|
"Destinations will always be considered different, and thus will always be redeployed."
|
|
230
254
|
).print_warning()
|
|
231
|
-
return self.dump_id(resource
|
|
255
|
+
return self.dump_id(self.get_id(resource))
|
|
232
256
|
|
|
233
257
|
@classmethod
|
|
234
258
|
def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
|
|
235
259
|
if "targetDataSetId" in item:
|
|
236
260
|
yield DataSetsCRUD, item["targetDataSetId"]
|
|
237
261
|
|
|
238
|
-
def sensitive_strings(self, item:
|
|
239
|
-
|
|
262
|
+
def sensitive_strings(self, item: HostedExtractorDestinationRequest) -> Iterable[str]:
|
|
263
|
+
if item.credentials:
|
|
264
|
+
yield item.credentials.nonce
|
|
240
265
|
id_ = self.get_id(item)
|
|
241
|
-
if id_ in self._authentication_by_id:
|
|
242
|
-
yield self._authentication_by_id[id_].client_secret
|
|
266
|
+
if id_.external_id in self._authentication_by_id:
|
|
267
|
+
yield self._authentication_by_id[id_.external_id].client_secret
|
|
243
268
|
|
|
244
269
|
|
|
245
|
-
class HostedExtractorJobCRUD(ResourceCRUD[
|
|
270
|
+
class HostedExtractorJobCRUD(ResourceCRUD[ExternalId, HostedExtractorJobRequest, HostedExtractorJobResponse]):
|
|
246
271
|
folder_name = "hosted_extractors"
|
|
247
|
-
resource_cls =
|
|
248
|
-
resource_write_cls =
|
|
272
|
+
resource_cls = HostedExtractorJobResponse
|
|
273
|
+
resource_write_cls = HostedExtractorJobRequest
|
|
249
274
|
dependencies = frozenset({HostedExtractorSourceCRUD, HostedExtractorDestinationCRUD})
|
|
250
275
|
kind = "Job"
|
|
251
276
|
yaml_cls = HostedExtractorJobYAML
|
|
@@ -257,18 +282,18 @@ class HostedExtractorJobCRUD(ResourceCRUD[str, JobWrite, Job]):
|
|
|
257
282
|
return "hosted extractor jobs"
|
|
258
283
|
|
|
259
284
|
@classmethod
|
|
260
|
-
def get_id(cls, item:
|
|
285
|
+
def get_id(cls, item: HostedExtractorJobRequest | HostedExtractorJobResponse | dict) -> ExternalId:
|
|
261
286
|
if isinstance(item, dict):
|
|
262
|
-
return item["externalId"]
|
|
263
|
-
return item.external_id
|
|
287
|
+
return ExternalId(external_id=item["externalId"])
|
|
288
|
+
return ExternalId(external_id=item.external_id)
|
|
264
289
|
|
|
265
290
|
@classmethod
|
|
266
|
-
def dump_id(cls, id:
|
|
267
|
-
return
|
|
291
|
+
def dump_id(cls, id: ExternalId) -> dict[str, Any]:
|
|
292
|
+
return id.dump()
|
|
268
293
|
|
|
269
294
|
@classmethod
|
|
270
295
|
def get_required_capability(
|
|
271
|
-
cls, items: Sequence[
|
|
296
|
+
cls, items: Sequence[HostedExtractorJobRequest] | None, read_only: bool
|
|
272
297
|
) -> Capability | list[Capability]:
|
|
273
298
|
if not items and items is not None:
|
|
274
299
|
return []
|
|
@@ -284,24 +309,28 @@ class HostedExtractorJobCRUD(ResourceCRUD[str, JobWrite, Job]):
|
|
|
284
309
|
HostedExtractorsAcl.Scope.All(),
|
|
285
310
|
)
|
|
286
311
|
|
|
287
|
-
def dump_resource(
|
|
288
|
-
|
|
312
|
+
def dump_resource(
|
|
313
|
+
self, resource: HostedExtractorJobResponse, local: dict[str, Any] | None = None
|
|
314
|
+
) -> dict[str, Any]:
|
|
315
|
+
dumped = resource.as_request_resource().dump()
|
|
289
316
|
local = local or {}
|
|
290
317
|
if not dumped.get("config") and "config" not in local:
|
|
291
318
|
dumped.pop("config", None)
|
|
292
319
|
return dumped
|
|
293
320
|
|
|
294
|
-
def create(self, items: Sequence[
|
|
295
|
-
return self.client.hosted_extractors.jobs.create(items)
|
|
321
|
+
def create(self, items: Sequence[HostedExtractorJobRequest]) -> list[HostedExtractorJobResponse]:
|
|
322
|
+
return self.client.tool.hosted_extractors.jobs.create(list(items))
|
|
296
323
|
|
|
297
|
-
def retrieve(self, ids: SequenceNotStr[
|
|
298
|
-
return self.client.hosted_extractors.jobs.retrieve(
|
|
324
|
+
def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[HostedExtractorJobResponse]:
|
|
325
|
+
return self.client.tool.hosted_extractors.jobs.retrieve(list(ids), ignore_unknown_ids=True)
|
|
299
326
|
|
|
300
|
-
def update(self, items: Sequence[
|
|
301
|
-
return self.client.hosted_extractors.jobs.update(items, mode="replace")
|
|
327
|
+
def update(self, items: Sequence[HostedExtractorJobRequest]) -> list[HostedExtractorJobResponse]:
|
|
328
|
+
return self.client.tool.hosted_extractors.jobs.update(list(items), mode="replace")
|
|
302
329
|
|
|
303
|
-
def delete(self, ids: SequenceNotStr[
|
|
304
|
-
|
|
330
|
+
def delete(self, ids: SequenceNotStr[ExternalId]) -> int:
|
|
331
|
+
if not ids:
|
|
332
|
+
return 0
|
|
333
|
+
self.client.tool.hosted_extractors.jobs.delete(list(ids), ignore_unknown_ids=True)
|
|
305
334
|
return len(ids)
|
|
306
335
|
|
|
307
336
|
def _iterate(
|
|
@@ -309,21 +338,24 @@ class HostedExtractorJobCRUD(ResourceCRUD[str, JobWrite, Job]):
|
|
|
309
338
|
data_set_external_id: str | None = None,
|
|
310
339
|
space: str | None = None,
|
|
311
340
|
parent_ids: list[Hashable] | None = None,
|
|
312
|
-
) -> Iterable[
|
|
313
|
-
|
|
341
|
+
) -> Iterable[HostedExtractorJobResponse]:
|
|
342
|
+
for jobs in self.client.tool.hosted_extractors.jobs.iterate():
|
|
343
|
+
yield from jobs
|
|
314
344
|
|
|
315
345
|
@classmethod
|
|
316
346
|
def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
|
|
317
347
|
if "sourceId" in item:
|
|
318
|
-
yield HostedExtractorSourceCRUD, item["sourceId"]
|
|
348
|
+
yield HostedExtractorSourceCRUD, ExternalId(external_id=item["sourceId"])
|
|
319
349
|
if "destinationId" in item:
|
|
320
|
-
yield HostedExtractorDestinationCRUD, item["destinationId"]
|
|
350
|
+
yield HostedExtractorDestinationCRUD, ExternalId(external_id=item["destinationId"])
|
|
321
351
|
|
|
322
352
|
|
|
323
|
-
class HostedExtractorMappingCRUD(
|
|
353
|
+
class HostedExtractorMappingCRUD(
|
|
354
|
+
ResourceCRUD[ExternalId, HostedExtractorMappingRequest, HostedExtractorMappingResponse]
|
|
355
|
+
):
|
|
324
356
|
folder_name = "hosted_extractors"
|
|
325
|
-
resource_cls =
|
|
326
|
-
resource_write_cls =
|
|
357
|
+
resource_cls = HostedExtractorMappingResponse
|
|
358
|
+
resource_write_cls = HostedExtractorMappingRequest
|
|
327
359
|
# This is not an explicit dependency, however, adding it here as mapping will should be deployed after source.
|
|
328
360
|
dependencies = frozenset({HostedExtractorSourceCRUD})
|
|
329
361
|
kind = "Mapping"
|
|
@@ -336,18 +368,18 @@ class HostedExtractorMappingCRUD(ResourceCRUD[str, MappingWrite, Mapping]):
|
|
|
336
368
|
return "hosted extractor mappings"
|
|
337
369
|
|
|
338
370
|
@classmethod
|
|
339
|
-
def get_id(cls, item:
|
|
371
|
+
def get_id(cls, item: HostedExtractorMappingRequest | HostedExtractorMappingResponse | dict) -> ExternalId:
|
|
340
372
|
if isinstance(item, dict):
|
|
341
|
-
return item["externalId"]
|
|
342
|
-
return item.external_id
|
|
373
|
+
return ExternalId(external_id=item["externalId"])
|
|
374
|
+
return ExternalId(external_id=item.external_id)
|
|
343
375
|
|
|
344
376
|
@classmethod
|
|
345
|
-
def dump_id(cls, id:
|
|
346
|
-
return
|
|
377
|
+
def dump_id(cls, id: ExternalId) -> dict[str, Any]:
|
|
378
|
+
return id.dump()
|
|
347
379
|
|
|
348
380
|
@classmethod
|
|
349
381
|
def get_required_capability(
|
|
350
|
-
cls, items: Sequence[
|
|
382
|
+
cls, items: Sequence[HostedExtractorMappingRequest] | None, read_only: bool
|
|
351
383
|
) -> Capability | list[Capability]:
|
|
352
384
|
if not items and items is not None:
|
|
353
385
|
return []
|
|
@@ -363,17 +395,19 @@ class HostedExtractorMappingCRUD(ResourceCRUD[str, MappingWrite, Mapping]):
|
|
|
363
395
|
HostedExtractorsAcl.Scope.All(),
|
|
364
396
|
)
|
|
365
397
|
|
|
366
|
-
def create(self, items: Sequence[
|
|
367
|
-
return self.client.hosted_extractors.mappings.create(items)
|
|
398
|
+
def create(self, items: Sequence[HostedExtractorMappingRequest]) -> list[HostedExtractorMappingResponse]:
|
|
399
|
+
return self.client.tool.hosted_extractors.mappings.create(list(items))
|
|
368
400
|
|
|
369
|
-
def retrieve(self, ids: SequenceNotStr[
|
|
370
|
-
return self.client.hosted_extractors.mappings.retrieve(
|
|
401
|
+
def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[HostedExtractorMappingResponse]:
|
|
402
|
+
return self.client.tool.hosted_extractors.mappings.retrieve(list(ids), ignore_unknown_ids=True)
|
|
371
403
|
|
|
372
|
-
def update(self, items: Sequence[
|
|
373
|
-
return self.client.hosted_extractors.mappings.update(items)
|
|
404
|
+
def update(self, items: Sequence[HostedExtractorMappingRequest]) -> list[HostedExtractorMappingResponse]:
|
|
405
|
+
return self.client.tool.hosted_extractors.mappings.update(list(items), mode="replace")
|
|
374
406
|
|
|
375
|
-
def delete(self, ids: SequenceNotStr[
|
|
376
|
-
|
|
407
|
+
def delete(self, ids: SequenceNotStr[ExternalId]) -> int:
|
|
408
|
+
if not ids:
|
|
409
|
+
return 0
|
|
410
|
+
self.client.tool.hosted_extractors.mappings.delete(list(ids), ignore_unknown_ids=True)
|
|
377
411
|
return len(ids)
|
|
378
412
|
|
|
379
413
|
def _iterate(
|
|
@@ -381,5 +415,6 @@ class HostedExtractorMappingCRUD(ResourceCRUD[str, MappingWrite, Mapping]):
|
|
|
381
415
|
data_set_external_id: str | None = None,
|
|
382
416
|
space: str | None = None,
|
|
383
417
|
parent_ids: list[Hashable] | None = None,
|
|
384
|
-
) -> Iterable[
|
|
385
|
-
|
|
418
|
+
) -> Iterable[HostedExtractorMappingResponse]:
|
|
419
|
+
for mappings in self.client.tool.hosted_extractors.mappings.iterate():
|
|
420
|
+
yield from mappings
|
cognite_toolkit/_version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.7.
|
|
1
|
+
__version__ = "0.7.69"
|