cognite-toolkit 0.6.99__py3-none-any.whl → 0.6.101__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/commands/_upload.py +47 -0
- cognite_toolkit/_cdf_tk/constants.py +11 -0
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py +173 -7
- cognite_toolkit/_cdf_tk/storageio/_instances.py +34 -2
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.6.99.dist-info → cognite_toolkit-0.6.101.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.6.99.dist-info → cognite_toolkit-0.6.101.dist-info}/RECORD +13 -13
- {cognite_toolkit-0.6.99.dist-info → cognite_toolkit-0.6.101.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.6.99.dist-info → cognite_toolkit-0.6.101.dist-info}/entry_points.txt +0 -0
- {cognite_toolkit-0.6.99.dist-info → cognite_toolkit-0.6.101.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,13 +1,18 @@
|
|
|
1
|
+
from collections import Counter
|
|
1
2
|
from collections.abc import Sequence
|
|
2
3
|
from functools import partial
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
from cognite.client.data_classes._base import T_CogniteResource
|
|
7
|
+
from cognite.client.data_classes.data_modeling import (
|
|
8
|
+
ViewId,
|
|
9
|
+
)
|
|
6
10
|
from pydantic import ValidationError
|
|
7
11
|
from rich.console import Console
|
|
8
12
|
|
|
9
13
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
10
14
|
from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
|
|
15
|
+
from cognite_toolkit._cdf_tk.cruds import ViewCRUD
|
|
11
16
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
|
12
17
|
from cognite_toolkit._cdf_tk.storageio import (
|
|
13
18
|
T_Selector,
|
|
@@ -17,6 +22,7 @@ from cognite_toolkit._cdf_tk.storageio import (
|
|
|
17
22
|
)
|
|
18
23
|
from cognite_toolkit._cdf_tk.storageio._base import T_WriteCogniteResource, TableUploadableStorageIO, UploadItem
|
|
19
24
|
from cognite_toolkit._cdf_tk.storageio.selectors import Selector, SelectorAdapter
|
|
25
|
+
from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpaceSelector
|
|
20
26
|
from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning
|
|
21
27
|
from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning
|
|
22
28
|
from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
|
|
@@ -86,8 +92,49 @@ class UploadCommand(ToolkitCommand):
|
|
|
86
92
|
|
|
87
93
|
self._deploy_resource_folder(input_dir / DATA_RESOURCE_DIR, deploy_resources, client, console, dry_run, verbose)
|
|
88
94
|
|
|
95
|
+
data_files_by_selector = self._topological_sort_if_instance_selector(data_files_by_selector, client)
|
|
96
|
+
|
|
89
97
|
self._upload_data(data_files_by_selector, client, dry_run, input_dir, console, verbose)
|
|
90
98
|
|
|
99
|
+
def _topological_sort_if_instance_selector(
|
|
100
|
+
self, data_files_by_selector: dict[Selector, list[Path]], client: ToolkitClient
|
|
101
|
+
) -> dict[Selector, list[Path]]:
|
|
102
|
+
"""Topologically sorts InstanceSpaceSelectors (if they are present) to determine the order of upload based on container dependencies from the views.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
data_files_by_selector: A dictionary mapping selectors to their data files.
|
|
106
|
+
client: The cognite client to use for the upload.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
A dictionary mapping selectors to their data files with necessary preprocessing.
|
|
110
|
+
"""
|
|
111
|
+
counts = Counter(type(selector) for selector in data_files_by_selector.keys())
|
|
112
|
+
if counts[InstanceSpaceSelector] <= 1:
|
|
113
|
+
return data_files_by_selector
|
|
114
|
+
|
|
115
|
+
selector_by_view_id: dict[ViewId, Selector] = {}
|
|
116
|
+
for selector in data_files_by_selector:
|
|
117
|
+
if isinstance(selector, InstanceSpaceSelector) and selector.view is not None:
|
|
118
|
+
selector_by_view_id[selector.view.as_id()] = selector
|
|
119
|
+
|
|
120
|
+
view_dependencies = ViewCRUD.create_loader(client).topological_sort_container_constraints(
|
|
121
|
+
list(selector_by_view_id.keys())
|
|
122
|
+
)
|
|
123
|
+
prepared_selectors: dict[Selector, list[Path]] = {}
|
|
124
|
+
|
|
125
|
+
# Reorder selectors according to the dependency-sorted view list
|
|
126
|
+
for view_id in view_dependencies:
|
|
127
|
+
selector = selector_by_view_id[view_id]
|
|
128
|
+
prepared_selectors[selector] = data_files_by_selector[selector]
|
|
129
|
+
|
|
130
|
+
# Preserve selectors that aren't affected by view dependencies
|
|
131
|
+
# (e.g., raw tables, time series, non-view instance data)
|
|
132
|
+
for selector in data_files_by_selector.keys():
|
|
133
|
+
if selector not in prepared_selectors:
|
|
134
|
+
prepared_selectors[selector] = data_files_by_selector[selector]
|
|
135
|
+
|
|
136
|
+
return prepared_selectors
|
|
137
|
+
|
|
91
138
|
def _find_data_files(
|
|
92
139
|
self,
|
|
93
140
|
input_dir: Path,
|
|
@@ -161,6 +161,17 @@ COGNITE_TIME_SERIES_CONTAINER = ContainerId("cdf_cdm", "CogniteTimeSeries")
|
|
|
161
161
|
COGNITE_FILE_CONTAINER = ContainerId("cdf_cdm", "CogniteFile")
|
|
162
162
|
CDF_UNIT_SPACE = "cdf_cdm_units"
|
|
163
163
|
|
|
164
|
+
|
|
165
|
+
# Container properties that are read-only in DMS needs to be handled with extra care, as this aspect is not currently exposed by the API.
|
|
166
|
+
READONLY_CONTAINER_PROPERTIES = {
|
|
167
|
+
ContainerId(space="cdf_cdm", external_id="CogniteAsset"): {
|
|
168
|
+
"assetHierarchy_path_last_updated_time",
|
|
169
|
+
"assetHierarchy_path",
|
|
170
|
+
"assetHierarchy_root",
|
|
171
|
+
},
|
|
172
|
+
ContainerId(space="cdf_cdm", external_id="CogniteFile"): {"isUploaded", "uploadedTime"},
|
|
173
|
+
}
|
|
174
|
+
|
|
164
175
|
# Data Plugin Constants
|
|
165
176
|
DATA_DEFAULT_DIR = "data"
|
|
166
177
|
DATA_RESOURCE_DIR = "resources"
|
|
@@ -40,16 +40,19 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
40
40
|
DataModelApply,
|
|
41
41
|
DataModelApplyList,
|
|
42
42
|
DataModelList,
|
|
43
|
+
DirectRelation,
|
|
43
44
|
Edge,
|
|
44
45
|
EdgeApply,
|
|
45
46
|
EdgeApplyList,
|
|
46
47
|
EdgeApplyResultList,
|
|
47
48
|
EdgeList,
|
|
49
|
+
MappedProperty,
|
|
48
50
|
Node,
|
|
49
51
|
NodeApply,
|
|
50
52
|
NodeApplyList,
|
|
51
53
|
NodeApplyResultList,
|
|
52
54
|
NodeList,
|
|
55
|
+
RequiresConstraint,
|
|
53
56
|
Space,
|
|
54
57
|
SpaceApply,
|
|
55
58
|
SpaceApplyList,
|
|
@@ -74,6 +77,7 @@ from rich.console import Console
|
|
|
74
77
|
from rich.markup import escape
|
|
75
78
|
from rich.panel import Panel
|
|
76
79
|
|
|
80
|
+
from cognite_toolkit._cdf_tk import constants
|
|
77
81
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
78
82
|
from cognite_toolkit._cdf_tk.client.data_classes.graphql_data_models import (
|
|
79
83
|
GraphQLDataModel,
|
|
@@ -259,6 +263,16 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
|
|
|
259
263
|
yaml_cls = ContainerYAML
|
|
260
264
|
_doc_url = "Containers/operation/ApplyContainers"
|
|
261
265
|
|
|
266
|
+
def __init__(
|
|
267
|
+
self,
|
|
268
|
+
client: ToolkitClient,
|
|
269
|
+
build_dir: Path | None,
|
|
270
|
+
console: Console | None = None,
|
|
271
|
+
topological_sort_implements: bool = False,
|
|
272
|
+
) -> None:
|
|
273
|
+
super().__init__(client, build_dir, console)
|
|
274
|
+
self._container_by_id: dict[ContainerId, Container] = {}
|
|
275
|
+
|
|
262
276
|
@property
|
|
263
277
|
def display_name(self) -> str:
|
|
264
278
|
return "containers"
|
|
@@ -425,6 +439,71 @@ class ContainerCRUD(ResourceContainerCRUD[ContainerId, ContainerApply, Container
|
|
|
425
439
|
):
|
|
426
440
|
yield instances.as_ids()
|
|
427
441
|
|
|
442
|
+
def _lookup_containers(self, container_ids: Sequence[ContainerId]) -> dict[ContainerId, Container]:
|
|
443
|
+
ids_to_lookup = [container_id for container_id in container_ids if container_id not in self._container_by_id]
|
|
444
|
+
if ids_to_lookup:
|
|
445
|
+
retrieved_containers = self.client.data_modeling.containers.retrieve(ids_to_lookup)
|
|
446
|
+
for container in retrieved_containers:
|
|
447
|
+
self._container_by_id[container.as_id()] = container
|
|
448
|
+
if missing_container_ids := set(container_ids) - set(self._container_by_id.keys()):
|
|
449
|
+
MediumSeverityWarning(
|
|
450
|
+
f"Containers {missing_container_ids} not found or you don't have permission to access them."
|
|
451
|
+
).print_warning(console=self.console)
|
|
452
|
+
return {
|
|
453
|
+
container_id: self._container_by_id[container_id]
|
|
454
|
+
for container_id in container_ids
|
|
455
|
+
if container_id in self._container_by_id
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
def _find_direct_container_dependencies(
|
|
459
|
+
self, container_ids: Sequence[ContainerId]
|
|
460
|
+
) -> dict[ContainerId, set[ContainerId]]:
|
|
461
|
+
containers_by_id = self._lookup_containers(container_ids)
|
|
462
|
+
container_dependencies: dict[ContainerId, set[ContainerId]] = defaultdict(set)
|
|
463
|
+
for container_id, container in containers_by_id.items():
|
|
464
|
+
for constraint in container.constraints.values():
|
|
465
|
+
if not isinstance(constraint, RequiresConstraint):
|
|
466
|
+
continue
|
|
467
|
+
container_dependencies[container_id].add(constraint.require)
|
|
468
|
+
for property in container.properties.values():
|
|
469
|
+
if not isinstance(property.type, DirectRelation) or property.type.container is None:
|
|
470
|
+
continue
|
|
471
|
+
container_dependencies[container_id].add(property.type.container)
|
|
472
|
+
return container_dependencies
|
|
473
|
+
|
|
474
|
+
def _propagate_indirect_container_dependencies(
|
|
475
|
+
self, container_dependencies_by_id: dict[ContainerId, set[ContainerId]], dependants: Sequence[ContainerId]
|
|
476
|
+
) -> dict[ContainerId, set[ContainerId]]:
|
|
477
|
+
"""Propagate indirect container dependencies using a recursive approach.
|
|
478
|
+
|
|
479
|
+
Args:
|
|
480
|
+
container_dependencies_by_id: Mapping of container IDs to their direct dependencies
|
|
481
|
+
dependants: Chain of dependant containers to propagate dependencies to
|
|
482
|
+
|
|
483
|
+
Returns:
|
|
484
|
+
Updated dictionary mapping each container ID to all its direct and indirect dependencies
|
|
485
|
+
"""
|
|
486
|
+
current_container_id = dependants[0]
|
|
487
|
+
dependencies_to_propagate: set[ContainerId] = set()
|
|
488
|
+
for container_dependency in container_dependencies_by_id[current_container_id]:
|
|
489
|
+
if container_dependency in container_dependencies_by_id:
|
|
490
|
+
# If already processed, propagate its dependencies to current container instead of revisiting it
|
|
491
|
+
dependencies_to_propagate.update(container_dependencies_by_id[container_dependency])
|
|
492
|
+
continue
|
|
493
|
+
self._propagate_indirect_container_dependencies(
|
|
494
|
+
container_dependencies_by_id, [container_dependency, *dependants]
|
|
495
|
+
)
|
|
496
|
+
container_dependencies_by_id[current_container_id].update(dependencies_to_propagate)
|
|
497
|
+
return container_dependencies_by_id
|
|
498
|
+
|
|
499
|
+
def _find_direct_and_indirect_container_dependencies(
|
|
500
|
+
self, container_ids: Sequence[ContainerId]
|
|
501
|
+
) -> dict[ContainerId, set[ContainerId]]:
|
|
502
|
+
container_dependencies_by_id = self._find_direct_container_dependencies(container_ids)
|
|
503
|
+
for container_id in list(container_dependencies_by_id.keys()):
|
|
504
|
+
self._propagate_indirect_container_dependencies(container_dependencies_by_id, [container_id])
|
|
505
|
+
return container_dependencies_by_id
|
|
506
|
+
|
|
428
507
|
@staticmethod
|
|
429
508
|
def _chunker(seq: Sequence, size: int) -> Iterable[Sequence]:
|
|
430
509
|
return (seq[pos : pos + size] for pos in range(0, len(seq), size))
|
|
@@ -556,7 +635,9 @@ class ViewCRUD(ResourceCRUD[ViewId, ViewApply, View, ViewApplyList, ViewList]):
|
|
|
556
635
|
# We sort the implements in topological order to ensure that the child view get the order grandparent,
|
|
557
636
|
# parent, such that the parent's source is used.
|
|
558
637
|
try:
|
|
559
|
-
dumped["implements"] = [
|
|
638
|
+
dumped["implements"] = [
|
|
639
|
+
view_id.dump() for view_id in self.topological_sort_implements(resource.implements)
|
|
640
|
+
]
|
|
560
641
|
except ToolkitCycleError as e:
|
|
561
642
|
warning = MediumSeverityWarning(f"Failed to sort implements for view {resource.as_id()}: {e}")
|
|
562
643
|
warning.print_warning(console=self.console)
|
|
@@ -669,14 +750,51 @@ class ViewCRUD(ResourceCRUD[ViewId, ViewApply, View, ViewApplyList, ViewList]):
|
|
|
669
750
|
self._view_by_id[view.as_id()] = view
|
|
670
751
|
return {view_id: self._view_by_id[view_id] for view_id in view_ids if view_id in self._view_by_id}
|
|
671
752
|
|
|
672
|
-
def
|
|
753
|
+
def get_readonly_properties(self, view_id: ViewId) -> set[str]:
|
|
754
|
+
"""Retrieve the set of read-only properties for a given view."""
|
|
755
|
+
|
|
756
|
+
readonly_properties: set[str] = set()
|
|
757
|
+
|
|
758
|
+
# Retrieve the view to check its properties
|
|
759
|
+
view = self._lookup_views([view_id]).get(view_id)
|
|
760
|
+
if view is None:
|
|
761
|
+
return readonly_properties
|
|
762
|
+
|
|
763
|
+
# Check each property in the view
|
|
764
|
+
for property_identifier, property in view.properties.items():
|
|
765
|
+
if isinstance(
|
|
766
|
+
property, MappedProperty
|
|
767
|
+
) and property.container_property_identifier in constants.READONLY_CONTAINER_PROPERTIES.get(
|
|
768
|
+
property.container, set()
|
|
769
|
+
):
|
|
770
|
+
readonly_properties.add(property_identifier)
|
|
771
|
+
return readonly_properties
|
|
772
|
+
|
|
773
|
+
def _build_view_implements_dependencies(
|
|
774
|
+
self, view_by_ids: dict[ViewId, View], include: set[ViewId] | None = None
|
|
775
|
+
) -> dict[ViewId, set[ViewId]]:
|
|
776
|
+
"""Build a dependency graph based on view implements relationships.
|
|
777
|
+
|
|
778
|
+
Args:
|
|
779
|
+
view_by_ids: Mapping of view IDs to View objects
|
|
780
|
+
include: Optional set of view IDs to include in the dependencies, if None, all views are included.
|
|
781
|
+
|
|
782
|
+
Returns:
|
|
783
|
+
Dictionary mapping each view ID to the set of view IDs it depends on (implements)
|
|
784
|
+
"""
|
|
785
|
+
dependencies: dict[ViewId, set[ViewId]] = {}
|
|
786
|
+
for view_id, view in view_by_ids.items():
|
|
787
|
+
dependencies[view_id] = set()
|
|
788
|
+
for implemented_view_id in view.implements or []:
|
|
789
|
+
if include is None or implemented_view_id in include:
|
|
790
|
+
dependencies[view_id].add(implemented_view_id)
|
|
791
|
+
return dependencies
|
|
792
|
+
|
|
793
|
+
def topological_sort_implements(self, view_ids: list[ViewId]) -> list[ViewId]:
|
|
673
794
|
"""Sorts the views in topological order based on their implements and through properties."""
|
|
674
795
|
view_by_ids = self._lookup_views(view_ids)
|
|
675
|
-
parents_by_child
|
|
676
|
-
|
|
677
|
-
parents_by_child[child] = set()
|
|
678
|
-
for parent in view.implements or []:
|
|
679
|
-
parents_by_child[child].add(parent)
|
|
796
|
+
parents_by_child = self._build_view_implements_dependencies(view_by_ids)
|
|
797
|
+
|
|
680
798
|
try:
|
|
681
799
|
sorted_views = list(TopologicalSorter(parents_by_child).static_order())
|
|
682
800
|
except CycleError as e:
|
|
@@ -686,6 +804,54 @@ class ViewCRUD(ResourceCRUD[ViewId, ViewApply, View, ViewApplyList, ViewList]):
|
|
|
686
804
|
|
|
687
805
|
return sorted_views
|
|
688
806
|
|
|
807
|
+
def topological_sort_container_constraints(self, view_ids: list[ViewId]) -> list[ViewId]:
|
|
808
|
+
"""Sorts the views in topological order based on their container constraints."""
|
|
809
|
+
|
|
810
|
+
view_by_ids = self._lookup_views(view_ids)
|
|
811
|
+
if missing_view_ids := set(view_ids) - set(view_by_ids.keys()):
|
|
812
|
+
MediumSeverityWarning(
|
|
813
|
+
f"Views {missing_view_ids} not found or you don't have permission to access them, skipping dependency check."
|
|
814
|
+
).print_warning(console=self.console)
|
|
815
|
+
return view_ids
|
|
816
|
+
|
|
817
|
+
view_to_containers: dict[ViewId, set[ContainerId]] = {}
|
|
818
|
+
container_to_views: defaultdict[ContainerId, set[ViewId]] = defaultdict(set)
|
|
819
|
+
for view_id, view in view_by_ids.items():
|
|
820
|
+
view_to_containers[view_id] = view.referenced_containers()
|
|
821
|
+
for container_id in view_to_containers[view_id]:
|
|
822
|
+
container_to_views[container_id].add(view_id)
|
|
823
|
+
|
|
824
|
+
container_crud = ContainerCRUD.create_loader(self.client)
|
|
825
|
+
container_dependencies_by_id = container_crud._find_direct_and_indirect_container_dependencies(
|
|
826
|
+
list(container_to_views.keys())
|
|
827
|
+
)
|
|
828
|
+
|
|
829
|
+
# First, add view dependencies based on implements relationships
|
|
830
|
+
view_dependencies = self._build_view_implements_dependencies(view_by_ids, set(view_to_containers.keys()))
|
|
831
|
+
|
|
832
|
+
# Then, add view dependencies based on mapped container constraints
|
|
833
|
+
for view_id, mapped_containers in view_to_containers.items():
|
|
834
|
+
for container_id in mapped_containers:
|
|
835
|
+
# Get all containers this container depends on
|
|
836
|
+
if container_id not in container_dependencies_by_id:
|
|
837
|
+
continue
|
|
838
|
+
for required_container in container_dependencies_by_id[container_id]:
|
|
839
|
+
if required_container not in container_to_views:
|
|
840
|
+
continue
|
|
841
|
+
# If this view already implements the required container, the requirement is self-satisfied
|
|
842
|
+
# and we don't need to depend on other views that also implement it (they are peers).
|
|
843
|
+
if required_container in mapped_containers:
|
|
844
|
+
continue
|
|
845
|
+
# This view doesn't implement the required container, so depend on all views that do
|
|
846
|
+
view_dependencies[view_id].update(container_to_views[required_container])
|
|
847
|
+
try:
|
|
848
|
+
sorted_views = list(TopologicalSorter(view_dependencies).static_order())
|
|
849
|
+
except CycleError as e:
|
|
850
|
+
raise ToolkitCycleError(
|
|
851
|
+
f"Failed to sort views topologically. This likely due to a cycle in implements. {e.args[1]}"
|
|
852
|
+
)
|
|
853
|
+
return sorted_views
|
|
854
|
+
|
|
689
855
|
|
|
690
856
|
@final
|
|
691
857
|
class DataModelCRUD(ResourceCRUD[DataModelId, DataModelApply, DataModel, DataModelApplyList, DataModelList]):
|
|
@@ -4,15 +4,18 @@ from typing import ClassVar
|
|
|
4
4
|
|
|
5
5
|
from cognite.client.data_classes.aggregations import Count
|
|
6
6
|
from cognite.client.data_classes.data_modeling import (
|
|
7
|
+
ContainerId,
|
|
7
8
|
ContainerList,
|
|
8
9
|
EdgeApply,
|
|
9
10
|
NodeApply,
|
|
10
11
|
SpaceList,
|
|
12
|
+
ViewId,
|
|
11
13
|
ViewList,
|
|
12
14
|
)
|
|
13
15
|
from cognite.client.data_classes.data_modeling.instances import Instance, InstanceApply
|
|
14
16
|
from cognite.client.utils._identifier import InstanceId
|
|
15
17
|
|
|
18
|
+
from cognite_toolkit._cdf_tk import constants
|
|
16
19
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
17
20
|
from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceList
|
|
18
21
|
from cognite_toolkit._cdf_tk.cruds import ContainerCRUD, SpaceCRUD, ViewCRUD
|
|
@@ -54,10 +57,35 @@ class InstanceIO(
|
|
|
54
57
|
def __init__(self, client: ToolkitClient, remove_existing_version: bool = True) -> None:
|
|
55
58
|
super().__init__(client)
|
|
56
59
|
self._remove_existing_version = remove_existing_version
|
|
60
|
+
# Cache for view to read-only properties mapping
|
|
61
|
+
self._view_readonly_properties_cache: dict[ViewId, set[str]] = {}
|
|
62
|
+
self._view_crud = ViewCRUD.create_loader(self.client)
|
|
57
63
|
|
|
58
64
|
def as_id(self, item: Instance) -> str:
|
|
59
65
|
return f"{item.space}:{item.external_id}"
|
|
60
66
|
|
|
67
|
+
def _filter_readonly_properties(self, instance: InstanceApply) -> None:
|
|
68
|
+
"""
|
|
69
|
+
Filter out read-only properties from the instance.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
instance: The instance to filter readonly properties from
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
for source in instance.sources:
|
|
76
|
+
readonly_properties = set()
|
|
77
|
+
if isinstance(source.source, ViewId):
|
|
78
|
+
if source.source not in self._view_readonly_properties_cache:
|
|
79
|
+
self._view_readonly_properties_cache[source.source] = self._view_crud.get_readonly_properties(
|
|
80
|
+
source.source
|
|
81
|
+
)
|
|
82
|
+
readonly_properties = self._view_readonly_properties_cache[source.source]
|
|
83
|
+
elif isinstance(source.source, ContainerId):
|
|
84
|
+
if source.source in constants.READONLY_CONTAINER_PROPERTIES:
|
|
85
|
+
readonly_properties = constants.READONLY_CONTAINER_PROPERTIES[source.source]
|
|
86
|
+
|
|
87
|
+
source.properties = {k: v for k, v in source.properties.items() if k not in readonly_properties}
|
|
88
|
+
|
|
61
89
|
def stream_data(self, selector: InstanceSelector, limit: int | None = None) -> Iterable[Page]:
|
|
62
90
|
if isinstance(selector, InstanceViewSelector | InstanceSpaceSelector):
|
|
63
91
|
chunk = InstanceList([])
|
|
@@ -133,12 +161,16 @@ class InstanceIO(
|
|
|
133
161
|
item_to_load = dict(item_json)
|
|
134
162
|
if self._remove_existing_version and "existingVersion" in item_to_load:
|
|
135
163
|
del item_to_load["existingVersion"]
|
|
164
|
+
instance: InstanceApply
|
|
136
165
|
if instance_type == "node":
|
|
137
|
-
|
|
166
|
+
instance = NodeApply._load(item_to_load, cognite_client=self.client)
|
|
138
167
|
elif instance_type == "edge":
|
|
139
|
-
|
|
168
|
+
instance = EdgeApply._load(item_to_load, cognite_client=self.client)
|
|
140
169
|
else:
|
|
141
170
|
raise ValueError(f"Unknown instance type {instance_type!r}")
|
|
171
|
+
# Filter out read-only properties if applicable
|
|
172
|
+
self._filter_readonly_properties(instance)
|
|
173
|
+
return instance
|
|
142
174
|
|
|
143
175
|
def configurations(self, selector: InstanceSelector) -> Iterable[StorageIOConfig]:
|
|
144
176
|
if not isinstance(selector, InstanceViewSelector | InstanceSpaceSelector):
|
|
@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
|
|
|
4
4
|
[modules]
|
|
5
5
|
# This is the version of the modules. It should not be changed manually.
|
|
6
6
|
# It will be updated by the 'cdf modules upgrade' command.
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.101"
|
|
8
8
|
|
|
9
9
|
[alpha_flags]
|
|
10
10
|
external-libraries = true
|
cognite_toolkit/_version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.6.
|
|
1
|
+
__version__ = "0.6.101"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cognite_toolkit
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.101
|
|
4
4
|
Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
|
|
5
5
|
Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
|
|
6
6
|
Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
2
|
cognite_toolkit/_cdf.py,sha256=0abeQr1Tfk4lkGaoXyrnFC28wDSlR_8UGrh10noGduQ,6085
|
|
3
|
-
cognite_toolkit/_version.py,sha256=
|
|
3
|
+
cognite_toolkit/_version.py,sha256=icqdH700ON8_fhaYsz0dp8JcAVtxK87DxpAV54EAoUw,24
|
|
4
4
|
cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=IjmzNVLxsOV6tsMDgmJmXsy-LQru-8IEQdFzGW5DxVk,8117
|
|
6
|
-
cognite_toolkit/_cdf_tk/constants.py,sha256=
|
|
6
|
+
cognite_toolkit/_cdf_tk/constants.py,sha256=Z0nsRXBrqVUgalMa7MZFiwPp26KUfaHqX2h89Daox-M,6936
|
|
7
7
|
cognite_toolkit/_cdf_tk/exceptions.py,sha256=xG0jMwi5A20nvPvyo6sCyz_cyKycynPyIzpYiGR4gcU,6064
|
|
8
8
|
cognite_toolkit/_cdf_tk/feature_flags.py,sha256=XMIZIOjPCdnH42SupUOETf_0d11TgKPZSfjCKnLoDJo,3017
|
|
9
9
|
cognite_toolkit/_cdf_tk/hints.py,sha256=UI1ymi2T5wCcYOpEbKbVaDnlyFReFy8TDtMVt-5E1h8,6493
|
|
@@ -107,7 +107,7 @@ cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=TK6U_rm6VZT_V941kTyHMou
|
|
|
107
107
|
cognite_toolkit/_cdf_tk/commands/_download.py,sha256=OBKPM_HGGA1i32th1SAgkQM_81CUFvm39kGqBuOeeTs,6816
|
|
108
108
|
cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=_4iX3AHAI6eLmRVUlWXCSvVHx1BZW2yDr_i2i9ECg6U,43120
|
|
109
109
|
cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=RadQHsmkPez3fZ5HCP9b82o2_fBx8P_-bTo7prkvWXU,32525
|
|
110
|
-
cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=
|
|
110
|
+
cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=ZK9h6RlA-p348x64HMOY-Dnff4rdxDgCNQFDIV_fthY,14427
|
|
111
111
|
cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=UxMJW5QYKts4om5n6x2Tq2ihvfO9gWjhQKeqZNFTlKg,402
|
|
112
112
|
cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=GFAid4hplixmj9_HkcXqU5yCLj-fTXm4cloGD6U2swY,2180
|
|
113
113
|
cognite_toolkit/_cdf_tk/commands/auth.py,sha256=N6JgtF0_Qoh-xM8VlBb_IK1n0Lo5I7bIkIHmXm1l7ug,31638
|
|
@@ -146,7 +146,7 @@ cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py,sha256=iGG2_btpEqip3o6OKpc
|
|
|
146
146
|
cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py,sha256=7RdiWvh6MLI1lLmt3gcqDQj61xbwREhsvoyjFuJn2F0,26402
|
|
147
147
|
cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py,sha256=KrL7bj8q5q18mGB2V-NDkW5U5nfseZOyorXiUbp2uLw,6100
|
|
148
148
|
cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py,sha256=iXn9iAtwA8mhH-7j9GF-MlLomTcaw3GhEbFY28Wx0iA,9927
|
|
149
|
-
cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py,sha256=
|
|
149
|
+
cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py,sha256=kA9urLWU65fqskvdSpjJEluvuWzUcCVhWCWoho68OJY,65642
|
|
150
150
|
cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py,sha256=zv36HPO9goRmU3NM_i1wOvWQEdsgpQTI4bcAl-eis1g,18232
|
|
151
151
|
cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py,sha256=SnQMbxiZ3SSYkTLXQ_vIu2HVf_WyD1jplNRJuoeOUfA,16723
|
|
152
152
|
cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py,sha256=F3n2FOWAPder4z3OTYs81VB-6C6r3oUzJsHvigdhaD0,15500
|
|
@@ -243,7 +243,7 @@ cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=DbTvIneN8Hw3ByhdH1kXk
|
|
|
243
243
|
cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=6f4akYlaOp8t22ecb8ZEt1rcBU58oD-iPm8UdwLdPuE,11716
|
|
244
244
|
cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
|
|
245
245
|
cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=nV8jaF5YLvMKhDPU3euf554GvSmfNYkzC9ZvEF7kbP8,8660
|
|
246
|
-
cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=
|
|
246
|
+
cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=t9fNpHnT6kCk8LDoPj3qZXmHpyDbPF5BZ6pI8ziTyFw,10810
|
|
247
247
|
cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=5WjAFiVR0KKRhMqCy1IRy1TQFWj86D7nGu5WSFNLp6U,3869
|
|
248
248
|
cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=KqbQDKKyjD8hndVjpIYSUWbEsvovxnGnvX_84bwZKfI,1629
|
|
249
249
|
cognite_toolkit/_cdf_tk/storageio/selectors/_asset_centric.py,sha256=7Iv_ccVX6Vzt3ZLFZ0Er3hN92iEsFTm9wgF-yermOWE,1467
|
|
@@ -298,13 +298,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
|
|
|
298
298
|
cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
|
|
299
299
|
cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
|
|
300
300
|
cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
|
|
301
|
-
cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=
|
|
302
|
-
cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=
|
|
303
|
-
cognite_toolkit/_resources/cdf.toml,sha256=
|
|
301
|
+
cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=GGjNjIB2WTlqzE8nsKwmKKGGk16Atrof0lQStSq5Bqk,668
|
|
302
|
+
cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=uidSAJl3b4jsjAkTstDQeZJvUiwEHEYPEYNYKwp9nK4,2431
|
|
303
|
+
cognite_toolkit/_resources/cdf.toml,sha256=KACiugkqkO1J-LZTTVpeWsrWFacvjhk59KlnmV0qi3Y,488
|
|
304
304
|
cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
|
|
305
305
|
cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
|
|
306
|
-
cognite_toolkit-0.6.
|
|
307
|
-
cognite_toolkit-0.6.
|
|
308
|
-
cognite_toolkit-0.6.
|
|
309
|
-
cognite_toolkit-0.6.
|
|
310
|
-
cognite_toolkit-0.6.
|
|
306
|
+
cognite_toolkit-0.6.101.dist-info/METADATA,sha256=BmHFC1AeT2tJ3TQR-e15Tg9urTrDUOaU9RF8reZFEY8,4502
|
|
307
|
+
cognite_toolkit-0.6.101.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
308
|
+
cognite_toolkit-0.6.101.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
|
|
309
|
+
cognite_toolkit-0.6.101.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
|
|
310
|
+
cognite_toolkit-0.6.101.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|