cognite-toolkit 0.6.90__py3-none-any.whl → 0.6.92__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/client/_toolkit_client.py +5 -0
- cognite_toolkit/_cdf_tk/client/api/infield.py +156 -0
- cognite_toolkit/_cdf_tk/client/data_classes/api_classes.py +17 -0
- cognite_toolkit/_cdf_tk/client/data_classes/base.py +63 -0
- cognite_toolkit/_cdf_tk/client/data_classes/infield.py +102 -0
- cognite_toolkit/_cdf_tk/client/data_classes/instance_api.py +157 -0
- cognite_toolkit/_cdf_tk/client/testing.py +3 -0
- cognite_toolkit/_cdf_tk/commands/_utils.py +1 -24
- cognite_toolkit/_cdf_tk/commands/clean.py +11 -5
- cognite_toolkit/_cdf_tk/commands/deploy.py +14 -10
- cognite_toolkit/_cdf_tk/commands/pull.py +13 -8
- cognite_toolkit/_cdf_tk/cruds/__init__.py +3 -0
- cognite_toolkit/_cdf_tk/cruds/_base_cruds.py +28 -25
- cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +10 -7
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py +2 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py +15 -0
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py +111 -2
- cognite_toolkit/_cdf_tk/cruds/_worker.py +24 -20
- cognite_toolkit/_cdf_tk/protocols.py +97 -0
- cognite_toolkit/_cdf_tk/resource_classes/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/resource_classes/infield_cdmv1.py +3 -1
- cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +13 -0
- cognite_toolkit/_cdf_tk/utils/text.py +23 -0
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.6.90.dist-info → cognite_toolkit-0.6.92.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.6.90.dist-info → cognite_toolkit-0.6.92.dist-info}/RECORD +32 -26
- {cognite_toolkit-0.6.90.dist-info → cognite_toolkit-0.6.92.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.6.90.dist-info → cognite_toolkit-0.6.92.dist-info}/entry_points.txt +0 -0
- {cognite_toolkit-0.6.90.dist-info → cognite_toolkit-0.6.92.dist-info}/licenses/LICENSE +0 -0
|
@@ -3,7 +3,6 @@ from graphlib import TopologicalSorter
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import overload
|
|
5
5
|
|
|
6
|
-
from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
|
|
7
6
|
from cognite.client.exceptions import CogniteAPIError, CogniteDuplicatedError
|
|
8
7
|
from cognite.client.utils._identifier import T_ID
|
|
9
8
|
from rich import print
|
|
@@ -26,7 +25,6 @@ from cognite_toolkit._cdf_tk.cruds import (
|
|
|
26
25
|
ResourceCRUD,
|
|
27
26
|
ResourceWorker,
|
|
28
27
|
)
|
|
29
|
-
from cognite_toolkit._cdf_tk.cruds._base_cruds import T_WritableCogniteResourceList
|
|
30
28
|
from cognite_toolkit._cdf_tk.cruds._worker import CategorizedResources
|
|
31
29
|
from cognite_toolkit._cdf_tk.data_classes import (
|
|
32
30
|
BuildEnvironment,
|
|
@@ -45,6 +43,12 @@ from cognite_toolkit._cdf_tk.exceptions import (
|
|
|
45
43
|
ToolkitFileNotFoundError,
|
|
46
44
|
ToolkitNotADirectoryError,
|
|
47
45
|
)
|
|
46
|
+
from cognite_toolkit._cdf_tk.protocols import (
|
|
47
|
+
T_ResourceRequest,
|
|
48
|
+
T_ResourceRequestList,
|
|
49
|
+
T_ResourceResponse,
|
|
50
|
+
T_ResourceResponseList,
|
|
51
|
+
)
|
|
48
52
|
from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning
|
|
49
53
|
from cognite_toolkit._cdf_tk.tk_warnings.base import WarningList, catch_warnings
|
|
50
54
|
from cognite_toolkit._cdf_tk.tk_warnings.other import (
|
|
@@ -324,7 +328,7 @@ class DeployCommand(ToolkitCommand):
|
|
|
324
328
|
def deploy_resource_type(
|
|
325
329
|
self,
|
|
326
330
|
loader: ResourceCRUD[
|
|
327
|
-
T_ID,
|
|
331
|
+
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
328
332
|
],
|
|
329
333
|
env_vars: EnvironmentVariables,
|
|
330
334
|
read_modules: list[ReadModule] | None = None,
|
|
@@ -387,9 +391,9 @@ class DeployCommand(ToolkitCommand):
|
|
|
387
391
|
|
|
388
392
|
def actual_deploy(
|
|
389
393
|
self,
|
|
390
|
-
resources: CategorizedResources[T_ID,
|
|
394
|
+
resources: CategorizedResources[T_ID, T_ResourceResponseList],
|
|
391
395
|
loader: ResourceCRUD[
|
|
392
|
-
T_ID,
|
|
396
|
+
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
393
397
|
],
|
|
394
398
|
env_var_warnings: WarningList | None = None,
|
|
395
399
|
) -> ResourceDeployResult:
|
|
@@ -421,9 +425,9 @@ class DeployCommand(ToolkitCommand):
|
|
|
421
425
|
|
|
422
426
|
@staticmethod
|
|
423
427
|
def dry_run_deploy(
|
|
424
|
-
resources: CategorizedResources[T_ID,
|
|
428
|
+
resources: CategorizedResources[T_ID, T_ResourceResponseList],
|
|
425
429
|
loader: ResourceCRUD[
|
|
426
|
-
T_ID,
|
|
430
|
+
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
427
431
|
],
|
|
428
432
|
has_done_drop: bool,
|
|
429
433
|
has_dropped_data: bool,
|
|
@@ -455,7 +459,7 @@ class DeployCommand(ToolkitCommand):
|
|
|
455
459
|
|
|
456
460
|
@staticmethod
|
|
457
461
|
def _verbose_print(
|
|
458
|
-
resources: CategorizedResources[T_ID,
|
|
462
|
+
resources: CategorizedResources[T_ID, T_ResourceResponseList],
|
|
459
463
|
loader: ResourceCRUD,
|
|
460
464
|
dry_run: bool,
|
|
461
465
|
) -> None:
|
|
@@ -479,7 +483,7 @@ class DeployCommand(ToolkitCommand):
|
|
|
479
483
|
|
|
480
484
|
def _create_resources(
|
|
481
485
|
self,
|
|
482
|
-
resources:
|
|
486
|
+
resources: T_ResourceResponseList,
|
|
483
487
|
loader: ResourceCRUD,
|
|
484
488
|
environment_variable_warning_by_id: dict[Hashable, EnvironmentVariableMissingWarning],
|
|
485
489
|
) -> int:
|
|
@@ -502,7 +506,7 @@ class DeployCommand(ToolkitCommand):
|
|
|
502
506
|
|
|
503
507
|
def _update_resources(
|
|
504
508
|
self,
|
|
505
|
-
resources:
|
|
509
|
+
resources: T_ResourceResponseList,
|
|
506
510
|
loader: ResourceCRUD,
|
|
507
511
|
environment_variable_warning_by_id: dict[Hashable, EnvironmentVariableMissingWarning],
|
|
508
512
|
) -> int:
|
|
@@ -11,7 +11,6 @@ from typing import Any, Union
|
|
|
11
11
|
|
|
12
12
|
import questionary
|
|
13
13
|
import yaml
|
|
14
|
-
from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
|
|
15
14
|
from questionary import Choice
|
|
16
15
|
from rich import print
|
|
17
16
|
from rich.markdown import Markdown
|
|
@@ -30,7 +29,13 @@ from cognite_toolkit._cdf_tk.cruds import (
|
|
|
30
29
|
ResourceCRUD,
|
|
31
30
|
StreamlitCRUD,
|
|
32
31
|
)
|
|
33
|
-
from cognite_toolkit._cdf_tk.cruds._base_cruds import
|
|
32
|
+
from cognite_toolkit._cdf_tk.cruds._base_cruds import (
|
|
33
|
+
T_ID,
|
|
34
|
+
T_ResourceRequest,
|
|
35
|
+
T_ResourceRequestList,
|
|
36
|
+
T_ResourceResponse,
|
|
37
|
+
T_ResourceResponseList,
|
|
38
|
+
)
|
|
34
39
|
from cognite_toolkit._cdf_tk.data_classes import (
|
|
35
40
|
BuildEnvironment,
|
|
36
41
|
BuildVariable,
|
|
@@ -543,14 +548,14 @@ class PullCommand(ToolkitCommand):
|
|
|
543
548
|
def _pull_resources(
|
|
544
549
|
self,
|
|
545
550
|
loader: ResourceCRUD[
|
|
546
|
-
T_ID,
|
|
551
|
+
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
547
552
|
],
|
|
548
553
|
resources: BuiltFullResourceList[T_ID],
|
|
549
554
|
dry_run: bool,
|
|
550
555
|
environment_variables: dict[str, str | None],
|
|
551
556
|
) -> ResourceDeployResult:
|
|
552
557
|
cdf_resources = loader.retrieve(resources.identifiers)
|
|
553
|
-
cdf_resource_by_id: dict[T_ID,
|
|
558
|
+
cdf_resource_by_id: dict[T_ID, T_ResourceResponse] = {loader.get_id(r): r for r in cdf_resources}
|
|
554
559
|
|
|
555
560
|
resources_by_file = resources.by_file()
|
|
556
561
|
file_results = ResourceDeployResult(loader.display_name)
|
|
@@ -574,10 +579,10 @@ class PullCommand(ToolkitCommand):
|
|
|
574
579
|
def _get_to_write(
|
|
575
580
|
self,
|
|
576
581
|
local_resource_by_id: dict[T_ID, dict[str, Any]],
|
|
577
|
-
cdf_resource_by_id: dict[T_ID,
|
|
582
|
+
cdf_resource_by_id: dict[T_ID, T_ResourceResponse],
|
|
578
583
|
file_results: ResourceDeployResult,
|
|
579
584
|
loader: ResourceCRUD[
|
|
580
|
-
T_ID,
|
|
585
|
+
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
581
586
|
],
|
|
582
587
|
) -> tuple[bool, dict[T_ID, dict[str, Any]]]:
|
|
583
588
|
to_write: dict[T_ID, dict[str, Any]] = {}
|
|
@@ -608,7 +613,7 @@ class PullCommand(ToolkitCommand):
|
|
|
608
613
|
def _get_local_resource_dict_by_id(
|
|
609
614
|
resources: BuiltFullResourceList[T_ID],
|
|
610
615
|
loader: ResourceCRUD[
|
|
611
|
-
T_ID,
|
|
616
|
+
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
612
617
|
],
|
|
613
618
|
environment_variables: dict[str, str | None],
|
|
614
619
|
) -> dict[T_ID, dict[str, Any]]:
|
|
@@ -647,7 +652,7 @@ class PullCommand(ToolkitCommand):
|
|
|
647
652
|
resources: BuiltFullResourceList[T_ID],
|
|
648
653
|
environment_variables: dict[str, str | None],
|
|
649
654
|
loader: ResourceCRUD[
|
|
650
|
-
T_ID,
|
|
655
|
+
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
651
656
|
],
|
|
652
657
|
source_file: Path,
|
|
653
658
|
) -> tuple[str, dict[Path, str]]:
|
|
@@ -41,6 +41,7 @@ from ._resource_cruds import (
|
|
|
41
41
|
HostedExtractorJobCRUD,
|
|
42
42
|
HostedExtractorMappingCRUD,
|
|
43
43
|
HostedExtractorSourceCRUD,
|
|
44
|
+
InFieldLocationConfigCRUD,
|
|
44
45
|
InfieldV1CRUD,
|
|
45
46
|
LabelCRUD,
|
|
46
47
|
LocationFilterCRUD,
|
|
@@ -79,6 +80,7 @@ if not FeatureFlag.is_enabled(Flags.AGENTS):
|
|
|
79
80
|
_EXCLUDED_CRUDS.add(AgentCRUD)
|
|
80
81
|
if not FeatureFlag.is_enabled(Flags.INFIELD):
|
|
81
82
|
_EXCLUDED_CRUDS.add(InfieldV1CRUD)
|
|
83
|
+
_EXCLUDED_CRUDS.add(InFieldLocationConfigCRUD)
|
|
82
84
|
if not FeatureFlag.is_enabled(Flags.MIGRATE):
|
|
83
85
|
_EXCLUDED_CRUDS.add(ResourceViewMappingCRUD)
|
|
84
86
|
if not FeatureFlag.is_enabled(Flags.SEARCH_CONFIG):
|
|
@@ -172,6 +174,7 @@ __all__ = [
|
|
|
172
174
|
"HostedExtractorJobCRUD",
|
|
173
175
|
"HostedExtractorMappingCRUD",
|
|
174
176
|
"HostedExtractorSourceCRUD",
|
|
177
|
+
"InFieldLocationConfigCRUD",
|
|
175
178
|
"LabelCRUD",
|
|
176
179
|
"LocationFilterCRUD",
|
|
177
180
|
"NodeCRUD",
|
|
@@ -5,21 +5,22 @@ from collections.abc import Hashable, Iterable, Sequence, Set, Sized
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from typing import TYPE_CHECKING, Any, Generic, TypeVar
|
|
7
7
|
|
|
8
|
-
from cognite.client.data_classes._base import (
|
|
9
|
-
T_CogniteResourceList,
|
|
10
|
-
T_WritableCogniteResource,
|
|
11
|
-
T_WriteClass,
|
|
12
|
-
)
|
|
13
8
|
from cognite.client.data_classes.capabilities import Capability
|
|
14
9
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
15
10
|
from rich.console import Console
|
|
16
11
|
|
|
17
12
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
18
13
|
from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING, EXCL_FILES
|
|
14
|
+
from cognite_toolkit._cdf_tk.protocols import (
|
|
15
|
+
T_ResourceRequest,
|
|
16
|
+
T_ResourceRequestList,
|
|
17
|
+
T_ResourceResponse,
|
|
18
|
+
T_ResourceResponseList,
|
|
19
|
+
)
|
|
19
20
|
from cognite_toolkit._cdf_tk.resource_classes import ToolkitResource
|
|
20
21
|
from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
|
|
21
22
|
from cognite_toolkit._cdf_tk.utils import load_yaml_inject_variables, safe_read, sanitize_filename
|
|
22
|
-
from cognite_toolkit._cdf_tk.utils.useful_types import T_ID
|
|
23
|
+
from cognite_toolkit._cdf_tk.utils.useful_types import T_ID
|
|
23
24
|
|
|
24
25
|
if TYPE_CHECKING:
|
|
25
26
|
from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment
|
|
@@ -151,7 +152,7 @@ T_Loader = TypeVar("T_Loader", bound=Loader)
|
|
|
151
152
|
class ResourceCRUD(
|
|
152
153
|
Loader,
|
|
153
154
|
ABC,
|
|
154
|
-
Generic[T_ID,
|
|
155
|
+
Generic[T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList],
|
|
155
156
|
):
|
|
156
157
|
"""This is the base class for all resource CRUD.
|
|
157
158
|
|
|
@@ -178,10 +179,10 @@ class ResourceCRUD(
|
|
|
178
179
|
"""
|
|
179
180
|
|
|
180
181
|
# Must be set in the subclass
|
|
181
|
-
resource_write_cls: type[
|
|
182
|
-
resource_cls: type[
|
|
183
|
-
list_cls: type[
|
|
184
|
-
list_write_cls: type[
|
|
182
|
+
resource_write_cls: type[T_ResourceRequest]
|
|
183
|
+
resource_cls: type[T_ResourceResponse]
|
|
184
|
+
list_cls: type[T_ResourceResponseList]
|
|
185
|
+
list_write_cls: type[T_ResourceRequestList]
|
|
185
186
|
yaml_cls: type[ToolkitResource]
|
|
186
187
|
# Optional to set in the subclass
|
|
187
188
|
support_drop = True
|
|
@@ -196,7 +197,7 @@ class ResourceCRUD(
|
|
|
196
197
|
# The methods that must be implemented in the subclass
|
|
197
198
|
@classmethod
|
|
198
199
|
@abstractmethod
|
|
199
|
-
def get_id(cls, item:
|
|
200
|
+
def get_id(cls, item: T_ResourceRequest | T_ResourceResponse | dict) -> T_ID:
|
|
200
201
|
raise NotImplementedError
|
|
201
202
|
|
|
202
203
|
@classmethod
|
|
@@ -207,19 +208,19 @@ class ResourceCRUD(
|
|
|
207
208
|
@classmethod
|
|
208
209
|
@abstractmethod
|
|
209
210
|
def get_required_capability(
|
|
210
|
-
cls, items: Sequence[
|
|
211
|
+
cls, items: Sequence[T_ResourceRequest] | None, read_only: bool
|
|
211
212
|
) -> Capability | list[Capability]:
|
|
212
213
|
raise NotImplementedError(f"get_required_capability must be implemented for {cls.__name__}.")
|
|
213
214
|
|
|
214
215
|
@abstractmethod
|
|
215
|
-
def create(self, items:
|
|
216
|
+
def create(self, items: T_ResourceRequestList) -> Sized:
|
|
216
217
|
raise NotImplementedError
|
|
217
218
|
|
|
218
219
|
@abstractmethod
|
|
219
|
-
def retrieve(self, ids: SequenceNotStr[T_ID]) ->
|
|
220
|
+
def retrieve(self, ids: SequenceNotStr[T_ID]) -> T_ResourceResponseList:
|
|
220
221
|
raise NotImplementedError
|
|
221
222
|
|
|
222
|
-
def update(self, items:
|
|
223
|
+
def update(self, items: T_ResourceRequestList) -> Sized:
|
|
223
224
|
raise NotImplementedError(f"Update is not supported for {type(self).__name__}.")
|
|
224
225
|
|
|
225
226
|
@abstractmethod
|
|
@@ -231,7 +232,7 @@ class ResourceCRUD(
|
|
|
231
232
|
data_set_external_id: str | None = None,
|
|
232
233
|
space: str | None = None,
|
|
233
234
|
parent_ids: list[Hashable] | None = None,
|
|
234
|
-
) -> Iterable[
|
|
235
|
+
) -> Iterable[T_ResourceResponse]:
|
|
235
236
|
if sum([1 for x in [data_set_external_id, space, parent_ids] if x is not None]) > 1:
|
|
236
237
|
raise ValueError("At most one of data_set_external_id, space, or parent_ids must be set.")
|
|
237
238
|
if parent_ids is not None and not self.parent_resource:
|
|
@@ -254,7 +255,7 @@ class ResourceCRUD(
|
|
|
254
255
|
data_set_external_id: str | None = None,
|
|
255
256
|
space: str | None = None,
|
|
256
257
|
parent_ids: list[Hashable] | None = None,
|
|
257
|
-
) -> Iterable[
|
|
258
|
+
) -> Iterable[T_ResourceResponse]:
|
|
258
259
|
raise NotImplementedError
|
|
259
260
|
|
|
260
261
|
### These methods can be optionally overwritten in the subclass ###
|
|
@@ -297,7 +298,7 @@ class ResourceCRUD(
|
|
|
297
298
|
return []
|
|
298
299
|
|
|
299
300
|
@classmethod
|
|
300
|
-
def get_internal_id(cls, item:
|
|
301
|
+
def get_internal_id(cls, item: T_ResourceResponse | dict) -> int:
|
|
301
302
|
raise NotImplementedError(f"{cls.__name__} does not have an internal id.")
|
|
302
303
|
|
|
303
304
|
@classmethod
|
|
@@ -334,18 +335,18 @@ class ResourceCRUD(
|
|
|
334
335
|
)
|
|
335
336
|
return raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
|
|
336
337
|
|
|
337
|
-
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) ->
|
|
338
|
+
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> T_ResourceRequest:
|
|
338
339
|
"""Loads the resource from a dictionary. Can be overwritten in subclasses."""
|
|
339
340
|
return self.resource_write_cls._load(resource)
|
|
340
341
|
|
|
341
|
-
def dump_resource(self, resource:
|
|
342
|
+
def dump_resource(self, resource: T_ResourceResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
342
343
|
"""Dumps the resource to a dictionary that matches the write format.
|
|
343
344
|
|
|
344
345
|
This is intended to be overwritten in subclasses that require special dumping logic, for example,
|
|
345
346
|
replacing dataSetId with dataSetExternalId.
|
|
346
347
|
|
|
347
348
|
Args:
|
|
348
|
-
resource (
|
|
349
|
+
resource (T_ResourceResponse): The resource to dump (typically comes from CDF).
|
|
349
350
|
local (dict[str, Any] | None): The local resource. When used in a dump/import command, there is no local
|
|
350
351
|
resource.
|
|
351
352
|
"""
|
|
@@ -394,7 +395,7 @@ class ResourceCRUD(
|
|
|
394
395
|
f"Missing implementation for {type(self).__name__} for {'.'.join(map(str, json_path))}."
|
|
395
396
|
)
|
|
396
397
|
|
|
397
|
-
def sensitive_strings(self, item:
|
|
398
|
+
def sensitive_strings(self, item: T_ResourceRequest) -> Iterable[str]:
|
|
398
399
|
"""Returns a list of strings that should be masked when printing.
|
|
399
400
|
|
|
400
401
|
This is used by the loaders with credentials to mask the credentials secrets. For example, the
|
|
@@ -405,7 +406,9 @@ class ResourceCRUD(
|
|
|
405
406
|
|
|
406
407
|
# Helper methods
|
|
407
408
|
@classmethod
|
|
408
|
-
def get_ids(
|
|
409
|
+
def get_ids(
|
|
410
|
+
cls, items: Sequence[T_ResourceRequest | T_ResourceResponse | dict] | T_ResourceResponseList
|
|
411
|
+
) -> list[T_ID]:
|
|
409
412
|
return [cls.get_id(item) for item in items]
|
|
410
413
|
|
|
411
414
|
@classmethod
|
|
@@ -418,7 +421,7 @@ class ResourceCRUD(
|
|
|
418
421
|
|
|
419
422
|
|
|
420
423
|
class ResourceContainerCRUD(
|
|
421
|
-
ResourceCRUD[T_ID,
|
|
424
|
+
ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList],
|
|
422
425
|
ABC,
|
|
423
426
|
):
|
|
424
427
|
"""This is the base class for all resource CRUD' containers.
|
|
@@ -5,15 +5,20 @@ from typing import TYPE_CHECKING, cast, final
|
|
|
5
5
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
from cognite.client.data_classes import FileMetadataWrite
|
|
8
|
-
from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
|
|
9
8
|
|
|
10
9
|
from cognite_toolkit._cdf_tk.client.data_classes.extendable_cognite_file import ExtendableCogniteFileApply
|
|
11
10
|
from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
|
|
12
11
|
from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
|
|
12
|
+
from cognite_toolkit._cdf_tk.protocols import (
|
|
13
|
+
T_ResourceRequest,
|
|
14
|
+
T_ResourceRequestList,
|
|
15
|
+
T_ResourceResponse,
|
|
16
|
+
T_ResourceResponseList,
|
|
17
|
+
)
|
|
13
18
|
from cognite_toolkit._cdf_tk.utils import read_yaml_content, safe_read
|
|
14
19
|
from cognite_toolkit._cdf_tk.utils.file import read_csv
|
|
15
20
|
|
|
16
|
-
from ._base_cruds import T_ID, DataCRUD, ResourceCRUD
|
|
21
|
+
from ._base_cruds import T_ID, DataCRUD, ResourceCRUD
|
|
17
22
|
from ._resource_cruds import CogniteFileCRUD, FileMetadataCRUD, RawTableCRUD, TimeSeriesCRUD
|
|
18
23
|
|
|
19
24
|
if TYPE_CHECKING:
|
|
@@ -131,15 +136,13 @@ class FileCRUD(DataCRUD):
|
|
|
131
136
|
def _read_metadata(
|
|
132
137
|
destination: Path,
|
|
133
138
|
loader: type[
|
|
134
|
-
ResourceCRUD[
|
|
135
|
-
T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
|
|
136
|
-
]
|
|
139
|
+
ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList]
|
|
137
140
|
],
|
|
138
141
|
identifier: T_ID,
|
|
139
|
-
) ->
|
|
142
|
+
) -> T_ResourceRequest:
|
|
140
143
|
built_content = read_yaml_content(safe_read(destination, encoding=BUILD_FOLDER_ENCODING))
|
|
141
144
|
if isinstance(built_content, dict):
|
|
142
|
-
return loader.resource_write_cls.
|
|
145
|
+
return loader.resource_write_cls._load(built_content)
|
|
143
146
|
elif isinstance(built_content, list):
|
|
144
147
|
try:
|
|
145
148
|
return next(m for m in loader.list_write_cls.load(built_content) if loader.get_id(m) == identifier)
|
|
@@ -13,7 +13,7 @@ from .datamodel import (
|
|
|
13
13
|
ViewCRUD,
|
|
14
14
|
)
|
|
15
15
|
from .extraction_pipeline import ExtractionPipelineConfigCRUD, ExtractionPipelineCRUD
|
|
16
|
-
from .fieldops import InfieldV1CRUD
|
|
16
|
+
from .fieldops import InFieldLocationConfigCRUD, InfieldV1CRUD
|
|
17
17
|
from .file import CogniteFileCRUD, FileMetadataCRUD
|
|
18
18
|
from .function import FunctionCRUD, FunctionScheduleCRUD
|
|
19
19
|
from .group_scoped import GroupResourceScopedCRUD
|
|
@@ -63,6 +63,7 @@ __all__ = [
|
|
|
63
63
|
"HostedExtractorJobCRUD",
|
|
64
64
|
"HostedExtractorMappingCRUD",
|
|
65
65
|
"HostedExtractorSourceCRUD",
|
|
66
|
+
"InFieldLocationConfigCRUD",
|
|
66
67
|
"InfieldV1CRUD",
|
|
67
68
|
"LabelCRUD",
|
|
68
69
|
"LocationFilterCRUD",
|
|
@@ -6,6 +6,7 @@ from typing import Any, final
|
|
|
6
6
|
|
|
7
7
|
import pandas as pd
|
|
8
8
|
from cognite.client.data_classes import (
|
|
9
|
+
AggregateResultItem,
|
|
9
10
|
Asset,
|
|
10
11
|
AssetList,
|
|
11
12
|
AssetWrite,
|
|
@@ -213,6 +214,20 @@ class AssetCRUD(ResourceCRUD[str, AssetWrite, Asset, AssetWriteList, AssetList])
|
|
|
213
214
|
dumped.pop("metadata", None)
|
|
214
215
|
if "parentId" in dumped and "parentId" not in local:
|
|
215
216
|
dumped.pop("parentId")
|
|
217
|
+
if resource.aggregates:
|
|
218
|
+
# This is only included when the asset is downloaded/migrated with aggregated properties
|
|
219
|
+
aggregates = (
|
|
220
|
+
resource.aggregates.dump()
|
|
221
|
+
if isinstance(resource.aggregates, AggregateResultItem)
|
|
222
|
+
else resource.aggregates
|
|
223
|
+
)
|
|
224
|
+
if "path" in aggregates:
|
|
225
|
+
path = aggregates.pop("path", [])
|
|
226
|
+
if path:
|
|
227
|
+
aggregates["path"] = self.client.lookup.assets.external_id(
|
|
228
|
+
[segment["id"] for segment in path if "id" in segment]
|
|
229
|
+
)
|
|
230
|
+
dumped.update(aggregates)
|
|
216
231
|
return dumped
|
|
217
232
|
|
|
218
233
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import collections.abc
|
|
2
|
-
from collections.abc import Hashable, Iterable
|
|
2
|
+
from collections.abc import Hashable, Iterable, Sequence, Sized
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import Any, final
|
|
5
5
|
|
|
@@ -14,9 +14,11 @@ from cognite_toolkit._cdf_tk.client.data_classes.apm_config_v1 import (
|
|
|
14
14
|
APMConfigWrite,
|
|
15
15
|
APMConfigWriteList,
|
|
16
16
|
)
|
|
17
|
+
from cognite_toolkit._cdf_tk.client.data_classes.infield import InfieldLocationConfig, InfieldLocationConfigList
|
|
18
|
+
from cognite_toolkit._cdf_tk.client.data_classes.instance_api import InstanceResult, NodeIdentifier
|
|
17
19
|
from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
|
|
18
20
|
from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
|
|
19
|
-
from cognite_toolkit._cdf_tk.resource_classes import InfieldV1YAML
|
|
21
|
+
from cognite_toolkit._cdf_tk.resource_classes import InfieldLocationConfigYAML, InfieldV1YAML
|
|
20
22
|
from cognite_toolkit._cdf_tk.utils import quote_int_value_by_key_in_yaml, safe_read
|
|
21
23
|
from cognite_toolkit._cdf_tk.utils.cdf import iterate_instances
|
|
22
24
|
from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable, hash_dict
|
|
@@ -238,3 +240,110 @@ class InfieldV1CRUD(ResourceCRUD[str, APMConfigWrite, APMConfig, APMConfigWriteL
|
|
|
238
240
|
if not isinstance(feature_configuration, dict):
|
|
239
241
|
return None
|
|
240
242
|
return feature_configuration.get("rootLocationConfigurations")
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@final
|
|
246
|
+
class InFieldLocationConfigCRUD(
|
|
247
|
+
ResourceCRUD[
|
|
248
|
+
NodeIdentifier,
|
|
249
|
+
InfieldLocationConfig,
|
|
250
|
+
InfieldLocationConfig,
|
|
251
|
+
InfieldLocationConfigList,
|
|
252
|
+
InfieldLocationConfigList,
|
|
253
|
+
]
|
|
254
|
+
):
|
|
255
|
+
folder_name = "cdf_applications"
|
|
256
|
+
filename_pattern = r"^.*\.InFieldLocationConfig$"
|
|
257
|
+
filetypes = frozenset({"yaml", "yml"})
|
|
258
|
+
resource_cls = InfieldLocationConfig
|
|
259
|
+
resource_write_cls = InfieldLocationConfig
|
|
260
|
+
list_cls = InfieldLocationConfigList
|
|
261
|
+
list_write_cls = InfieldLocationConfigList
|
|
262
|
+
kind = "InFieldLocationConfig"
|
|
263
|
+
yaml_cls = InfieldLocationConfigYAML
|
|
264
|
+
dependencies = frozenset({SpaceCRUD, GroupAllScopedCRUD, GroupResourceScopedCRUD})
|
|
265
|
+
_doc_url = "Instances/operation/applyNodeAndEdges"
|
|
266
|
+
|
|
267
|
+
@classmethod
|
|
268
|
+
def get_id(cls, item: InfieldLocationConfig | dict) -> NodeIdentifier:
|
|
269
|
+
if isinstance(item, dict):
|
|
270
|
+
return NodeIdentifier(space=item["space"], external_id=item["externalId"])
|
|
271
|
+
return NodeIdentifier(space=item.space, external_id=item.external_id)
|
|
272
|
+
|
|
273
|
+
@classmethod
|
|
274
|
+
def dump_id(cls, id: NodeIdentifier) -> dict[str, Any]:
|
|
275
|
+
return id.dump(include_type=False)
|
|
276
|
+
|
|
277
|
+
@classmethod
|
|
278
|
+
def get_required_capability(
|
|
279
|
+
cls, items: Sequence[InfieldLocationConfig] | None, read_only: bool
|
|
280
|
+
) -> Capability | list[Capability]:
|
|
281
|
+
if not items or items is None:
|
|
282
|
+
return []
|
|
283
|
+
|
|
284
|
+
actions = (
|
|
285
|
+
[DataModelInstancesAcl.Action.Read]
|
|
286
|
+
if read_only
|
|
287
|
+
else [DataModelInstancesAcl.Action.Read, DataModelInstancesAcl.Action.Write]
|
|
288
|
+
)
|
|
289
|
+
instance_spaces = sorted({item.space for item in items})
|
|
290
|
+
|
|
291
|
+
return DataModelInstancesAcl(actions, DataModelInstancesAcl.Scope.SpaceID(instance_spaces))
|
|
292
|
+
|
|
293
|
+
def dump_resource(self, resource: InfieldLocationConfig, local: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
294
|
+
dumped = resource.dump()
|
|
295
|
+
local = local or {}
|
|
296
|
+
dumped.pop("instanceType", None)
|
|
297
|
+
if isinstance(cdf_dec := dumped.get("dataExplorationConfig"), dict):
|
|
298
|
+
cdf_dec.pop("instanceType", None)
|
|
299
|
+
if isinstance(local_dec := local.get("dataExplorationConfig"), dict):
|
|
300
|
+
if "space" in cdf_dec and "space" not in local_dec:
|
|
301
|
+
# Default space is used for the data exploration config if not specified locally.
|
|
302
|
+
cdf_dec.pop("space")
|
|
303
|
+
if "externalId" in cdf_dec and "externalId" not in local_dec:
|
|
304
|
+
# Default externalId is used for the data exploration config if not specified locally.
|
|
305
|
+
cdf_dec.pop("externalId")
|
|
306
|
+
|
|
307
|
+
return dumped
|
|
308
|
+
|
|
309
|
+
def create(self, items: InfieldLocationConfigList) -> list[InstanceResult]:
|
|
310
|
+
created = self.client.infield.config.apply(items)
|
|
311
|
+
config_ids = {config.as_id() for config in items}
|
|
312
|
+
# We filter out all the data exploration configs that were created along with the infield location configs
|
|
313
|
+
# as we only want to count the infield location configs here.
|
|
314
|
+
return [res for res in created if res.as_id() in config_ids]
|
|
315
|
+
|
|
316
|
+
def retrieve(self, ids: SequenceNotStr[NodeIdentifier]) -> InfieldLocationConfigList:
|
|
317
|
+
return InfieldLocationConfigList(self.client.infield.config.retrieve(list(ids)))
|
|
318
|
+
|
|
319
|
+
def update(self, items: InfieldLocationConfigList) -> Sized:
|
|
320
|
+
return self.create(items)
|
|
321
|
+
|
|
322
|
+
def delete(self, ids: SequenceNotStr[NodeIdentifier]) -> int:
|
|
323
|
+
# We must retrieve the full resource to get hte DataExplorationConfig linked resource deleted as well.
|
|
324
|
+
retrieved = self.retrieve(list(ids))
|
|
325
|
+
# Then, we pass the entire resource to the delete method, which will delete both the InfieldLocationConfig
|
|
326
|
+
# and the linked DataExplorationConfig.
|
|
327
|
+
_ = self.client.infield.config.delete(retrieved)
|
|
328
|
+
return len(retrieved)
|
|
329
|
+
|
|
330
|
+
def _iterate(
|
|
331
|
+
self,
|
|
332
|
+
data_set_external_id: str | None = None,
|
|
333
|
+
space: str | None = None,
|
|
334
|
+
parent_ids: list[Hashable] | None = None,
|
|
335
|
+
) -> Iterable[InfieldLocationConfig]:
|
|
336
|
+
raise NotImplementedError(f"Iteration over {self.display_name} is not supported.")
|
|
337
|
+
|
|
338
|
+
def diff_list(
|
|
339
|
+
self, local: list[Any], cdf: list[Any], json_path: tuple[str | int, ...]
|
|
340
|
+
) -> tuple[dict[int, int], list[int]]:
|
|
341
|
+
if json_path == ("accessManagement", "templateAdmins"):
|
|
342
|
+
return diff_list_hashable(local, cdf)
|
|
343
|
+
elif json_path == ("accessManagement", "checklistAdmins"):
|
|
344
|
+
return diff_list_hashable(local, cdf)
|
|
345
|
+
elif json_path == ("dataFilters", "general", "spaces"):
|
|
346
|
+
return diff_list_hashable(local, cdf)
|
|
347
|
+
elif json_path == ("dataExplorationConfig", "documents", "supportedFormats"):
|
|
348
|
+
return diff_list_hashable(local, cdf)
|
|
349
|
+
return super().diff_list(local, cdf, json_path)
|
|
@@ -7,11 +7,6 @@ from pathlib import Path
|
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Generic, cast
|
|
8
8
|
|
|
9
9
|
from cognite.client.data_classes import FunctionWrite
|
|
10
|
-
from cognite.client.data_classes._base import (
|
|
11
|
-
T_CogniteResourceList,
|
|
12
|
-
T_WritableCogniteResource,
|
|
13
|
-
T_WriteClass,
|
|
14
|
-
)
|
|
15
10
|
from cognite.client.data_classes.capabilities import Capability
|
|
16
11
|
from rich import print
|
|
17
12
|
from rich.panel import Panel
|
|
@@ -23,27 +18,34 @@ from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarnin
|
|
|
23
18
|
from cognite_toolkit._cdf_tk.utils import to_diff
|
|
24
19
|
|
|
25
20
|
from . import FunctionCRUD
|
|
26
|
-
from ._base_cruds import
|
|
21
|
+
from ._base_cruds import (
|
|
22
|
+
T_ID,
|
|
23
|
+
ResourceCRUD,
|
|
24
|
+
T_ResourceRequest,
|
|
25
|
+
T_ResourceRequestList,
|
|
26
|
+
T_ResourceResponse,
|
|
27
|
+
T_ResourceResponseList,
|
|
28
|
+
)
|
|
27
29
|
|
|
28
30
|
if TYPE_CHECKING:
|
|
29
31
|
from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
|
|
30
32
|
|
|
31
33
|
|
|
32
34
|
@dataclass
|
|
33
|
-
class CategorizedResources(Generic[T_ID,
|
|
34
|
-
to_create:
|
|
35
|
-
to_update:
|
|
35
|
+
class CategorizedResources(Generic[T_ID, T_ResourceRequestList]):
|
|
36
|
+
to_create: T_ResourceRequestList
|
|
37
|
+
to_update: T_ResourceRequestList
|
|
36
38
|
to_delete: list[T_ID]
|
|
37
|
-
unchanged:
|
|
39
|
+
unchanged: T_ResourceRequestList
|
|
38
40
|
|
|
39
41
|
|
|
40
42
|
class ResourceWorker(
|
|
41
|
-
Generic[T_ID,
|
|
43
|
+
Generic[T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList]
|
|
42
44
|
):
|
|
43
45
|
def __init__(
|
|
44
46
|
self,
|
|
45
47
|
loader: ResourceCRUD[
|
|
46
|
-
T_ID,
|
|
48
|
+
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
47
49
|
],
|
|
48
50
|
action: str,
|
|
49
51
|
):
|
|
@@ -102,14 +104,14 @@ class ResourceWorker(
|
|
|
102
104
|
self.validate_access(local_by_id, is_dry_run)
|
|
103
105
|
|
|
104
106
|
# Lookup the existing resources in CDF
|
|
105
|
-
cdf_resources:
|
|
107
|
+
cdf_resources: T_ResourceResponseList
|
|
106
108
|
cdf_resources = self.loader.retrieve(list(local_by_id.keys()))
|
|
107
109
|
return self.categorize_resources(local_by_id, cdf_resources, force_update, verbose)
|
|
108
110
|
|
|
109
111
|
def load_resources(
|
|
110
112
|
self, filepaths: list[Path], environment_variables: dict[str, str | None] | None, is_dry_run: bool
|
|
111
|
-
) -> dict[T_ID, tuple[dict[str, Any],
|
|
112
|
-
local_by_id: dict[T_ID, tuple[dict[str, Any],
|
|
113
|
+
) -> dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]]:
|
|
114
|
+
local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]] = {}
|
|
113
115
|
# Load all resources from files, get ids, and remove duplicates.
|
|
114
116
|
environment_variables = environment_variables or {}
|
|
115
117
|
for filepath in filepaths:
|
|
@@ -145,7 +147,9 @@ class ResourceWorker(
|
|
|
145
147
|
warning.print_warning()
|
|
146
148
|
return local_by_id
|
|
147
149
|
|
|
148
|
-
def validate_access(
|
|
150
|
+
def validate_access(
|
|
151
|
+
self, local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]], is_dry_run: bool
|
|
152
|
+
) -> None:
|
|
149
153
|
capabilities: Capability | list[Capability]
|
|
150
154
|
if isinstance(self.loader, FunctionCRUD):
|
|
151
155
|
function_loader: FunctionCRUD = self.loader
|
|
@@ -160,12 +164,12 @@ class ResourceWorker(
|
|
|
160
164
|
|
|
161
165
|
def categorize_resources(
|
|
162
166
|
self,
|
|
163
|
-
local_by_id: dict[T_ID, tuple[dict[str, Any],
|
|
164
|
-
cdf_resources:
|
|
167
|
+
local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]],
|
|
168
|
+
cdf_resources: T_ResourceResponseList,
|
|
165
169
|
force_update: bool,
|
|
166
170
|
verbose: bool,
|
|
167
|
-
) -> CategorizedResources:
|
|
168
|
-
resources: CategorizedResources[T_ID,
|
|
171
|
+
) -> CategorizedResources[T_ID, T_ResourceRequestList]:
|
|
172
|
+
resources: CategorizedResources[T_ID, T_ResourceRequestList] = CategorizedResources(
|
|
169
173
|
to_create=self.loader.list_write_cls([]),
|
|
170
174
|
to_update=self.loader.list_write_cls([]),
|
|
171
175
|
to_delete=[],
|