cognite-toolkit 0.6.100__py3-none-any.whl → 0.6.102__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/apps/_landing_app.py +32 -4
- cognite_toolkit/_cdf_tk/cdf_toml.py +20 -1
- cognite_toolkit/_cdf_tk/client/api/migration.py +109 -1
- cognite_toolkit/_cdf_tk/client/data_classes/migration.py +2 -2
- cognite_toolkit/_cdf_tk/commands/_changes.py +3 -6
- cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +18 -39
- cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py +56 -21
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +102 -8
- cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py +9 -4
- cognite_toolkit/_cdf_tk/commands/_questionary_style.py +16 -0
- cognite_toolkit/_cdf_tk/commands/_upload.py +47 -0
- cognite_toolkit/_cdf_tk/commands/auth.py +2 -1
- cognite_toolkit/_cdf_tk/commands/init.py +225 -3
- cognite_toolkit/_cdf_tk/commands/modules.py +18 -42
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py +151 -7
- cognite_toolkit/_cdf_tk/storageio/__init__.py +2 -2
- cognite_toolkit/_cdf_tk/storageio/_annotations.py +2 -2
- cognite_toolkit/_cdf_tk/utils/dtype_conversion.py +9 -3
- cognite_toolkit/_cdf_tk/utils/useful_types.py +6 -2
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.6.100.dist-info → cognite_toolkit-0.6.102.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.6.100.dist-info → cognite_toolkit-0.6.102.dist-info}/RECORD +28 -27
- {cognite_toolkit-0.6.100.dist-info → cognite_toolkit-0.6.102.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.6.100.dist-info → cognite_toolkit-0.6.102.dist-info}/entry_points.txt +0 -0
- {cognite_toolkit-0.6.100.dist-info → cognite_toolkit-0.6.102.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,14 +1,42 @@
|
|
|
1
|
+
from typing import Annotated
|
|
2
|
+
|
|
1
3
|
import typer
|
|
2
4
|
|
|
5
|
+
from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
|
|
3
6
|
from cognite_toolkit._cdf_tk.commands import InitCommand
|
|
7
|
+
from cognite_toolkit._cdf_tk.feature_flags import Flags
|
|
4
8
|
|
|
5
9
|
|
|
6
10
|
class LandingApp(typer.Typer):
|
|
7
11
|
def __init__(self, *args, **kwargs) -> None: # type: ignore
|
|
8
12
|
super().__init__(*args, **kwargs)
|
|
9
|
-
self.command()(self.main_init)
|
|
10
13
|
|
|
11
|
-
def main_init(
|
|
12
|
-
|
|
14
|
+
def main_init(
|
|
15
|
+
self,
|
|
16
|
+
dry_run: Annotated[
|
|
17
|
+
bool,
|
|
18
|
+
typer.Option(
|
|
19
|
+
"--dry-run",
|
|
20
|
+
"-r",
|
|
21
|
+
help="Whether to do a dry-run, do dry-run if present.",
|
|
22
|
+
),
|
|
23
|
+
] = False,
|
|
24
|
+
# TODO: this is a temporary solution to be able to test the functionality
|
|
25
|
+
# in a new environment, assuming that the toml file doesn't exist yet.
|
|
26
|
+
# remove this once v.07 is released
|
|
27
|
+
v7: Annotated[
|
|
28
|
+
bool,
|
|
29
|
+
typer.Option(
|
|
30
|
+
"--seven",
|
|
31
|
+
"-s",
|
|
32
|
+
help="Emulate v0.7",
|
|
33
|
+
hidden=(Flags.v07.is_enabled() or not CDFToml.load().is_loaded_from_file),
|
|
34
|
+
),
|
|
35
|
+
] = False,
|
|
36
|
+
) -> None:
|
|
37
|
+
"""Getting started checklist"""
|
|
13
38
|
cmd = InitCommand()
|
|
14
|
-
|
|
39
|
+
# Tracking command with the usual lambda run construct
|
|
40
|
+
# is intentionally left out because we don't want to expose the user to the warning
|
|
41
|
+
# before they've had the chance to opt in (which is something they'll do later using this command).
|
|
42
|
+
cmd.execute(dry_run=dry_run, emulate_dot_seven=v7)
|
|
@@ -9,7 +9,7 @@ from typing import Any, ClassVar
|
|
|
9
9
|
from rich import print
|
|
10
10
|
|
|
11
11
|
from cognite_toolkit import _version
|
|
12
|
-
from cognite_toolkit._cdf_tk.constants import clean_name
|
|
12
|
+
from cognite_toolkit._cdf_tk.constants import RESOURCES_PATH, EnvType, clean_name
|
|
13
13
|
from cognite_toolkit._cdf_tk.exceptions import (
|
|
14
14
|
ToolkitRequiredValueError,
|
|
15
15
|
ToolkitTOMLFormatError,
|
|
@@ -176,6 +176,25 @@ class CDFToml:
|
|
|
176
176
|
is_loaded_from_file=False,
|
|
177
177
|
)
|
|
178
178
|
|
|
179
|
+
@classmethod
|
|
180
|
+
def write(cls, organization_dir: Path, env: EnvType = "dev", version: str = _version.__version__) -> None:
|
|
181
|
+
destination = Path.cwd() / CDFToml.file_name
|
|
182
|
+
if destination.exists():
|
|
183
|
+
print("cdf.toml file already exists. Skipping creation.")
|
|
184
|
+
return
|
|
185
|
+
cdf_toml_content = (RESOURCES_PATH / CDFToml.file_name).read_text(encoding="utf-8")
|
|
186
|
+
cdf_toml_content = cdf_toml_content.replace("0.0.0", version)
|
|
187
|
+
if organization_dir != Path.cwd():
|
|
188
|
+
cdf_toml_content = cdf_toml_content.replace(
|
|
189
|
+
"#<PLACEHOLDER>",
|
|
190
|
+
f'''
|
|
191
|
+
default_organization_dir = "{organization_dir.name}"''',
|
|
192
|
+
)
|
|
193
|
+
else:
|
|
194
|
+
cdf_toml_content = cdf_toml_content.replace("#<PLACEHOLDER>", "")
|
|
195
|
+
cdf_toml_content = cdf_toml_content.replace("<DEFAULT_ENV_PLACEHOLDER>", env)
|
|
196
|
+
destination.write_text(cdf_toml_content, encoding="utf-8")
|
|
197
|
+
|
|
179
198
|
|
|
180
199
|
def _read_toml(file_path: Path) -> dict[str, Any]:
|
|
181
200
|
# TOML files are required to be UTF-8 encoded
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import warnings
|
|
2
2
|
from collections.abc import Sequence
|
|
3
3
|
from itertools import groupby
|
|
4
|
-
from typing import TypeVar, overload
|
|
4
|
+
from typing import Literal, TypeVar, cast, overload
|
|
5
5
|
|
|
6
6
|
from cognite.client._constants import DEFAULT_LIMIT_READ
|
|
7
7
|
from cognite.client.data_classes.data_modeling import (
|
|
@@ -347,9 +347,117 @@ class SpaceSourceAPI:
|
|
|
347
347
|
return results
|
|
348
348
|
|
|
349
349
|
|
|
350
|
+
class LookupAPI:
|
|
351
|
+
def __init__(self, instance_api: ExtendedInstancesAPI, resource_type: AssetCentricType) -> None:
|
|
352
|
+
self._instance_api = instance_api
|
|
353
|
+
self._resource_type = resource_type
|
|
354
|
+
self._view_id = InstanceSource.get_source()
|
|
355
|
+
self._node_id_by_id: dict[int, NodeId | None] = {}
|
|
356
|
+
self._node_id_by_external_id: dict[str, NodeId | None] = {}
|
|
357
|
+
self._RETRIEVE_LIMIT = 1000
|
|
358
|
+
|
|
359
|
+
@overload
|
|
360
|
+
def __call__(self, id: int, external_id: None = None) -> NodeId | None: ...
|
|
361
|
+
|
|
362
|
+
@overload
|
|
363
|
+
def __call__(self, id: Sequence[int], external_id: None = None) -> dict[int, NodeId]: ...
|
|
364
|
+
|
|
365
|
+
@overload
|
|
366
|
+
def __call__(self, *, external_id: str) -> NodeId | None: ...
|
|
367
|
+
|
|
368
|
+
@overload
|
|
369
|
+
def __call__(self, *, external_id: SequenceNotStr[str]) -> dict[str, NodeId]: ...
|
|
370
|
+
|
|
371
|
+
def __call__(
|
|
372
|
+
self, id: int | Sequence[int] | None = None, external_id: str | SequenceNotStr[str] | None = None
|
|
373
|
+
) -> dict[int, NodeId] | dict[str, NodeId] | NodeId | None:
|
|
374
|
+
"""Lookup NodeId by either internal ID or external ID.
|
|
375
|
+
|
|
376
|
+
Args:
|
|
377
|
+
id (int | Sequence[int] | None): The internal ID(s) to lookup.
|
|
378
|
+
external_id (str | SequenceNotStr[str] | None): The external ID(s) to lookup.
|
|
379
|
+
|
|
380
|
+
Returns:
|
|
381
|
+
NodeId | dict[int, NodeId] | dict[str, NodeId] | None: The corresponding NodeId(s) if found, otherwise None.
|
|
382
|
+
|
|
383
|
+
"""
|
|
384
|
+
if id is not None and external_id is None:
|
|
385
|
+
return self._lookup_by_id(id)
|
|
386
|
+
elif external_id is not None and id is None:
|
|
387
|
+
return self._lookup_by_external_id(external_id)
|
|
388
|
+
else:
|
|
389
|
+
raise ValueError("Either id or external_id must be provided, but not both.")
|
|
390
|
+
|
|
391
|
+
def _lookup_by_id(self, id: int | Sequence[int]) -> dict[int, NodeId] | NodeId | None:
|
|
392
|
+
ids: list[int] = [id] if isinstance(id, int) else list(id)
|
|
393
|
+
|
|
394
|
+
missing = [id_ for id_ in ids if id_ not in self._node_id_by_id]
|
|
395
|
+
if missing:
|
|
396
|
+
self._fetch_and_cache(missing, by="id")
|
|
397
|
+
if isinstance(id, int):
|
|
398
|
+
return self._node_id_by_id.get(id)
|
|
399
|
+
return {id_: node_id for id_ in ids if isinstance(node_id := self._node_id_by_id.get(id_), NodeId)}
|
|
400
|
+
|
|
401
|
+
def _lookup_by_external_id(self, external_id: str | SequenceNotStr[str]) -> dict[str, NodeId] | NodeId | None:
|
|
402
|
+
external_ids: list[str] = [external_id] if isinstance(external_id, str) else list(external_id)
|
|
403
|
+
|
|
404
|
+
missing = [ext_id for ext_id in external_ids if ext_id not in self._node_id_by_external_id]
|
|
405
|
+
if missing:
|
|
406
|
+
self._fetch_and_cache(missing, by="classicExternalId")
|
|
407
|
+
if isinstance(external_id, str):
|
|
408
|
+
return self._node_id_by_external_id.get(external_id)
|
|
409
|
+
return {
|
|
410
|
+
ext_id: node_id
|
|
411
|
+
for ext_id in external_ids
|
|
412
|
+
if isinstance(node_id := self._node_id_by_external_id.get(ext_id), NodeId)
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
def _fetch_and_cache(self, identifiers: Sequence[int | str], by: Literal["id", "classicExternalId"]) -> None:
|
|
416
|
+
for chunk in chunker_sequence(identifiers, self._RETRIEVE_LIMIT):
|
|
417
|
+
retrieve_query = query.Query(
|
|
418
|
+
with_={
|
|
419
|
+
"instanceSource": query.NodeResultSetExpression(
|
|
420
|
+
filter=filters.And(
|
|
421
|
+
filters.HasData(views=[self._view_id]),
|
|
422
|
+
filters.Equals(self._view_id.as_property_ref("resourceType"), self._resource_type),
|
|
423
|
+
filters.In(self._view_id.as_property_ref(by), list(chunk)),
|
|
424
|
+
),
|
|
425
|
+
limit=len(chunk),
|
|
426
|
+
),
|
|
427
|
+
},
|
|
428
|
+
select={"instanceSource": query.Select([query.SourceSelector(self._view_id, ["*"])])},
|
|
429
|
+
)
|
|
430
|
+
chunk_response = self._instance_api.query(retrieve_query)
|
|
431
|
+
items = chunk_response.get("instanceSource", [])
|
|
432
|
+
for item in items:
|
|
433
|
+
instance_source = InstanceSource._load(item.dump())
|
|
434
|
+
node_id = instance_source.as_id()
|
|
435
|
+
self._node_id_by_id[instance_source.id_] = node_id
|
|
436
|
+
if instance_source.classic_external_id:
|
|
437
|
+
self._node_id_by_external_id[instance_source.classic_external_id] = node_id
|
|
438
|
+
missing = set(chunk) - set(self._node_id_by_id.keys()) - set(self._node_id_by_external_id.keys())
|
|
439
|
+
if by == "id":
|
|
440
|
+
for missing_id in cast(set[int], missing):
|
|
441
|
+
if missing_id not in self._node_id_by_id:
|
|
442
|
+
self._node_id_by_id[missing_id] = None
|
|
443
|
+
elif by == "classicExternalId":
|
|
444
|
+
for missing_ext_id in cast(set[str], missing):
|
|
445
|
+
if missing_ext_id not in self._node_id_by_external_id:
|
|
446
|
+
self._node_id_by_external_id[missing_ext_id] = None
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
class MigrationLookupAPI:
|
|
450
|
+
def __init__(self, instance_api: ExtendedInstancesAPI) -> None:
|
|
451
|
+
self.assets = LookupAPI(instance_api, "asset")
|
|
452
|
+
self.events = LookupAPI(instance_api, "event")
|
|
453
|
+
self.files = LookupAPI(instance_api, "file")
|
|
454
|
+
self.time_series = LookupAPI(instance_api, "timeseries")
|
|
455
|
+
|
|
456
|
+
|
|
350
457
|
class MigrationAPI:
|
|
351
458
|
def __init__(self, instance_api: ExtendedInstancesAPI) -> None:
|
|
352
459
|
self.instance_source = InstanceSourceAPI(instance_api)
|
|
353
460
|
self.resource_view_mapping = ResourceViewMappingAPI(instance_api)
|
|
354
461
|
self.created_source_system = CreatedSourceSystemAPI(instance_api)
|
|
355
462
|
self.space_source = SpaceSourceAPI(instance_api)
|
|
463
|
+
self.lookup = MigrationLookupAPI(instance_api)
|
|
@@ -16,7 +16,7 @@ from cognite.client.data_classes.data_modeling.instances import (
|
|
|
16
16
|
|
|
17
17
|
from cognite_toolkit._cdf_tk.constants import COGNITE_MIGRATION_SPACE
|
|
18
18
|
from cognite_toolkit._cdf_tk.tk_warnings import IgnoredValueWarning
|
|
19
|
-
from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricType
|
|
19
|
+
from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricType, AssetCentricTypeExtended
|
|
20
20
|
|
|
21
21
|
if sys.version_info >= (3, 11):
|
|
22
22
|
from typing import Self
|
|
@@ -26,7 +26,7 @@ else:
|
|
|
26
26
|
|
|
27
27
|
@dataclass(frozen=True)
|
|
28
28
|
class AssetCentricId(CogniteObject):
|
|
29
|
-
resource_type:
|
|
29
|
+
resource_type: AssetCentricTypeExtended
|
|
30
30
|
id_: int
|
|
31
31
|
|
|
32
32
|
@classmethod
|
|
@@ -12,6 +12,7 @@ from packaging.version import parse as parse_version
|
|
|
12
12
|
from rich import print
|
|
13
13
|
|
|
14
14
|
from cognite_toolkit._cdf_tk.builders import get_loader
|
|
15
|
+
from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
|
|
15
16
|
from cognite_toolkit._cdf_tk.constants import DOCKER_IMAGE_NAME
|
|
16
17
|
from cognite_toolkit._cdf_tk.data_classes import ModuleDirectories
|
|
17
18
|
from cognite_toolkit._cdf_tk.utils import iterate_modules, read_yaml_file, safe_read, safe_write
|
|
@@ -362,7 +363,6 @@ After:
|
|
|
362
363
|
|
|
363
364
|
def do(self) -> set[Path]:
|
|
364
365
|
# Avoid circular import
|
|
365
|
-
from .modules import ModulesCommand
|
|
366
366
|
|
|
367
367
|
system_yaml = self._organization_dir / "_system.yaml"
|
|
368
368
|
if not system_yaml.exists():
|
|
@@ -372,11 +372,8 @@ After:
|
|
|
372
372
|
content = read_yaml_file(system_yaml)
|
|
373
373
|
current_version = content.get("cdf_toolkit_version", __version__)
|
|
374
374
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
cdf_toml_path = Path.cwd() / "cdf.toml"
|
|
379
|
-
cdf_toml_path.write_text(cdf_toml_content, encoding="utf-8")
|
|
375
|
+
CDFToml.write(self._organization_dir, version=current_version)
|
|
376
|
+
cdf_toml_path = Path.cwd() / CDFToml.file_name
|
|
380
377
|
system_yaml.unlink()
|
|
381
378
|
return {cdf_toml_path, system_yaml}
|
|
382
379
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from collections.abc import Mapping, Set
|
|
2
2
|
from dataclasses import dataclass
|
|
3
|
-
from typing import Any, ClassVar
|
|
3
|
+
from typing import Any, ClassVar
|
|
4
4
|
|
|
5
5
|
from cognite.client.data_classes import Annotation, Asset, Event, FileMetadata, TimeSeries
|
|
6
6
|
from cognite.client.data_classes.data_modeling import (
|
|
@@ -13,6 +13,7 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
13
13
|
)
|
|
14
14
|
from cognite.client.data_classes.data_modeling.instances import EdgeApply, NodeOrEdgeData, PropertyValueWrite
|
|
15
15
|
from cognite.client.data_classes.data_modeling.views import ViewProperty
|
|
16
|
+
from cognite.client.utils._identifier import InstanceId
|
|
16
17
|
|
|
17
18
|
from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId, ResourceViewMapping
|
|
18
19
|
from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
|
|
@@ -22,7 +23,7 @@ from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
|
|
|
22
23
|
)
|
|
23
24
|
from cognite_toolkit._cdf_tk.utils.useful_types import (
|
|
24
25
|
AssetCentricResourceExtended,
|
|
25
|
-
|
|
26
|
+
AssetCentricTypeExtended,
|
|
26
27
|
)
|
|
27
28
|
|
|
28
29
|
from .data_model import INSTANCE_SOURCE_VIEW_ID
|
|
@@ -51,29 +52,31 @@ class DirectRelationCache:
|
|
|
51
52
|
|
|
52
53
|
"""
|
|
53
54
|
|
|
54
|
-
ASSET_REFERENCE_PROPERTIES: ClassVar[Set[tuple[
|
|
55
|
+
ASSET_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
|
|
55
56
|
("timeseries", "assetId"),
|
|
56
57
|
("file", "assetIds"),
|
|
57
58
|
("event", "assetIds"),
|
|
58
59
|
("sequence", "assetId"),
|
|
59
60
|
("asset", "parentId"),
|
|
60
|
-
("
|
|
61
|
+
("annotation", "data.assetRef.id"),
|
|
61
62
|
}
|
|
62
|
-
SOURCE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[
|
|
63
|
+
SOURCE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
|
|
63
64
|
("asset", "source"),
|
|
64
65
|
("event", "source"),
|
|
65
66
|
("file", "source"),
|
|
66
67
|
}
|
|
67
|
-
FILE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[
|
|
68
|
-
("
|
|
69
|
-
("
|
|
68
|
+
FILE_REFERENCE_PROPERTIES: ClassVar[Set[tuple[AssetCentricTypeExtended, str]]] = {
|
|
69
|
+
("annotation", "data.fileRef.id"),
|
|
70
|
+
("annotation", "annotatedResourceId"),
|
|
70
71
|
}
|
|
71
72
|
|
|
72
73
|
asset: Mapping[int, DirectRelationReference]
|
|
73
74
|
source: Mapping[str, DirectRelationReference]
|
|
74
75
|
file: Mapping[int, DirectRelationReference]
|
|
75
76
|
|
|
76
|
-
def get(
|
|
77
|
+
def get(
|
|
78
|
+
self, resource_type: AssetCentricTypeExtended, property_id: str
|
|
79
|
+
) -> Mapping[str | int, DirectRelationReference]:
|
|
77
80
|
key = resource_type, property_id
|
|
78
81
|
if key in self.ASSET_REFERENCE_PROPERTIES:
|
|
79
82
|
return self.asset # type: ignore[return-value]
|
|
@@ -84,33 +87,9 @@ class DirectRelationCache:
|
|
|
84
87
|
return {}
|
|
85
88
|
|
|
86
89
|
|
|
87
|
-
@overload
|
|
88
90
|
def asset_centric_to_dm(
|
|
89
91
|
resource: AssetCentricResourceExtended,
|
|
90
|
-
instance_id:
|
|
91
|
-
view_source: ResourceViewMapping,
|
|
92
|
-
view_properties: dict[str, ViewProperty],
|
|
93
|
-
asset_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
94
|
-
source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
|
|
95
|
-
file_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
96
|
-
) -> tuple[NodeApply | None, ConversionIssue]: ...
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
@overload
|
|
100
|
-
def asset_centric_to_dm(
|
|
101
|
-
resource: AssetCentricResourceExtended,
|
|
102
|
-
instance_id: EdgeId,
|
|
103
|
-
view_source: ResourceViewMapping,
|
|
104
|
-
view_properties: dict[str, ViewProperty],
|
|
105
|
-
asset_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
106
|
-
source_instance_id_by_external_id: Mapping[str, DirectRelationReference],
|
|
107
|
-
file_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
108
|
-
) -> tuple[EdgeApply | None, ConversionIssue]: ...
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
def asset_centric_to_dm(
|
|
112
|
-
resource: AssetCentricResourceExtended,
|
|
113
|
-
instance_id: NodeId | EdgeId,
|
|
92
|
+
instance_id: InstanceId,
|
|
114
93
|
view_source: ResourceViewMapping,
|
|
115
94
|
view_properties: dict[str, ViewProperty],
|
|
116
95
|
asset_instance_id_by_id: Mapping[int, DirectRelationReference],
|
|
@@ -165,7 +144,7 @@ def asset_centric_to_dm(
|
|
|
165
144
|
if properties:
|
|
166
145
|
sources.append(NodeOrEdgeData(source=view_source.view_id, properties=properties))
|
|
167
146
|
|
|
168
|
-
if resource_type != "
|
|
147
|
+
if resource_type != "annotation":
|
|
169
148
|
instance_source_properties = {
|
|
170
149
|
"resourceType": resource_type,
|
|
171
150
|
"id": id_,
|
|
@@ -196,7 +175,7 @@ def asset_centric_to_dm(
|
|
|
196
175
|
return instance, issue
|
|
197
176
|
|
|
198
177
|
|
|
199
|
-
def _lookup_resource_type(resource_type: AssetCentricResourceExtended) ->
|
|
178
|
+
def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetCentricTypeExtended:
|
|
200
179
|
if isinstance(resource_type, Asset):
|
|
201
180
|
return "asset"
|
|
202
181
|
elif isinstance(resource_type, FileMetadata):
|
|
@@ -210,7 +189,7 @@ def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetC
|
|
|
210
189
|
"diagrams.AssetLink",
|
|
211
190
|
"diagrams.FileLink",
|
|
212
191
|
):
|
|
213
|
-
return "
|
|
192
|
+
return "annotation"
|
|
214
193
|
raise ValueError(f"Unsupported resource type: {resource_type}")
|
|
215
194
|
|
|
216
195
|
|
|
@@ -218,7 +197,7 @@ def create_properties(
|
|
|
218
197
|
dumped: dict[str, Any],
|
|
219
198
|
view_properties: dict[str, ViewProperty],
|
|
220
199
|
property_mapping: dict[str, str],
|
|
221
|
-
resource_type:
|
|
200
|
+
resource_type: AssetCentricTypeExtended,
|
|
222
201
|
issue: ConversionIssue,
|
|
223
202
|
direct_relation_cache: DirectRelationCache,
|
|
224
203
|
) -> dict[str, PropertyValueWrite]:
|
|
@@ -289,7 +268,7 @@ def create_properties(
|
|
|
289
268
|
def create_edge_properties(
|
|
290
269
|
dumped: dict[str, Any],
|
|
291
270
|
property_mapping: dict[str, str],
|
|
292
|
-
resource_type:
|
|
271
|
+
resource_type: AssetCentricTypeExtended,
|
|
293
272
|
issue: ConversionIssue,
|
|
294
273
|
direct_relation_cache: DirectRelationCache,
|
|
295
274
|
default_instance_space: str,
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from typing import Any, Generic, Literal
|
|
3
|
+
from typing import Annotated, Any, Generic, Literal
|
|
4
4
|
|
|
5
5
|
from cognite.client.data_classes._base import (
|
|
6
6
|
WriteableCogniteResource,
|
|
7
7
|
WriteableCogniteResourceList,
|
|
8
8
|
)
|
|
9
|
-
from cognite.client.data_classes.data_modeling import InstanceApply, NodeId, ViewId
|
|
9
|
+
from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, NodeId, ViewId
|
|
10
|
+
from cognite.client.utils._identifier import InstanceId
|
|
10
11
|
from cognite.client.utils._text import to_camel_case
|
|
11
|
-
from pydantic import BaseModel, field_validator, model_validator
|
|
12
|
+
from pydantic import BaseModel, BeforeValidator, field_validator, model_validator
|
|
12
13
|
|
|
13
14
|
from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
|
|
14
15
|
from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
|
|
@@ -17,10 +18,9 @@ from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_de
|
|
|
17
18
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
|
18
19
|
from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
|
|
19
20
|
from cognite_toolkit._cdf_tk.utils.useful_types import (
|
|
20
|
-
|
|
21
|
-
AssetCentricType,
|
|
21
|
+
AssetCentricKindExtended,
|
|
22
22
|
JsonVal,
|
|
23
|
-
|
|
23
|
+
T_AssetCentricResourceExtended,
|
|
24
24
|
)
|
|
25
25
|
|
|
26
26
|
|
|
@@ -37,8 +37,8 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
|
|
|
37
37
|
for example, the Canvas migration to determine which view to use for the resource.
|
|
38
38
|
"""
|
|
39
39
|
|
|
40
|
-
resource_type:
|
|
41
|
-
instance_id:
|
|
40
|
+
resource_type: str
|
|
41
|
+
instance_id: InstanceId
|
|
42
42
|
id: int
|
|
43
43
|
data_set_id: int | None = None
|
|
44
44
|
ingestion_view: str | None = None
|
|
@@ -56,7 +56,8 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
|
|
|
56
56
|
raise ToolkitValueError(f"No default ingestion view specified for resource type '{self.resource_type}'")
|
|
57
57
|
|
|
58
58
|
def as_asset_centric_id(self) -> AssetCentricId:
|
|
59
|
-
|
|
59
|
+
# MyPy fails to understand that resource_type is AssetCentricKindExtended in subclasses
|
|
60
|
+
return AssetCentricId(resource_type=self.resource_type, id_=self.id) # type: ignore[arg-type]
|
|
60
61
|
|
|
61
62
|
@model_validator(mode="before")
|
|
62
63
|
def _handle_flat_dict(cls, values: Any) -> Any:
|
|
@@ -87,12 +88,6 @@ class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore",
|
|
|
87
88
|
return ViewId.load(v)
|
|
88
89
|
return v
|
|
89
90
|
|
|
90
|
-
@field_validator("instance_id", mode="before")
|
|
91
|
-
def _validate_instance_id(cls, v: Any) -> Any:
|
|
92
|
-
if isinstance(v, dict):
|
|
93
|
-
return NodeId.load(v)
|
|
94
|
-
return v
|
|
95
|
-
|
|
96
91
|
|
|
97
92
|
class MigrationMappingList(ModelList[MigrationMapping]):
|
|
98
93
|
@classmethod
|
|
@@ -113,14 +108,22 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
113
108
|
|
|
114
109
|
def as_node_ids(self) -> list[NodeId]:
|
|
115
110
|
"""Return a list of NodeIds from the migration mappings."""
|
|
116
|
-
return [mapping.instance_id for mapping in self]
|
|
111
|
+
return [mapping.instance_id for mapping in self if isinstance(mapping.instance_id, NodeId)]
|
|
112
|
+
|
|
113
|
+
def as_edge_ids(self) -> list[EdgeId]:
|
|
114
|
+
"""Return a list of EdgeIds from the migration mappings."""
|
|
115
|
+
return [mapping.instance_id for mapping in self if isinstance(mapping.instance_id, EdgeId)]
|
|
117
116
|
|
|
118
117
|
def spaces(self) -> set[str]:
|
|
119
118
|
"""Return a set of spaces from the migration mappings."""
|
|
120
119
|
return {mapping.instance_id.space for mapping in self}
|
|
121
120
|
|
|
122
121
|
def as_pending_ids(self) -> list[PendingInstanceId]:
|
|
123
|
-
return [
|
|
122
|
+
return [
|
|
123
|
+
PendingInstanceId(pending_instance_id=mapping.instance_id, id=mapping.id)
|
|
124
|
+
for mapping in self
|
|
125
|
+
if isinstance(mapping.instance_id, NodeId)
|
|
126
|
+
]
|
|
124
127
|
|
|
125
128
|
def get_data_set_ids(self) -> set[int]:
|
|
126
129
|
"""Return a list of data set IDs from the migration mappings."""
|
|
@@ -131,7 +134,9 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
131
134
|
return {mapping.id: mapping for mapping in self}
|
|
132
135
|
|
|
133
136
|
@classmethod
|
|
134
|
-
def read_csv_file(
|
|
137
|
+
def read_csv_file(
|
|
138
|
+
cls, filepath: Path, resource_type: AssetCentricKindExtended | None = None
|
|
139
|
+
) -> "MigrationMappingList":
|
|
135
140
|
if cls is not MigrationMappingList or resource_type is None:
|
|
136
141
|
return super().read_csv_file(filepath)
|
|
137
142
|
cls_by_resource_type: dict[str, type[MigrationMappingList]] = {
|
|
@@ -139,6 +144,7 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
139
144
|
"TimeSeries": TimeSeriesMigrationMappingList,
|
|
140
145
|
"FileMetadata": FileMigrationMappingList,
|
|
141
146
|
"Events": EventMigrationMappingList,
|
|
147
|
+
"Annotations": AnnotationMigrationMappingList,
|
|
142
148
|
}
|
|
143
149
|
if resource_type not in cls_by_resource_type:
|
|
144
150
|
raise ToolkitValueError(
|
|
@@ -147,20 +153,41 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
147
153
|
return cls_by_resource_type[resource_type].read_csv_file(filepath, resource_type=None)
|
|
148
154
|
|
|
149
155
|
|
|
156
|
+
def _validate_node_id(value: Any) -> Any:
|
|
157
|
+
if isinstance(value, dict):
|
|
158
|
+
return NodeId.load(value)
|
|
159
|
+
return value
|
|
160
|
+
|
|
161
|
+
|
|
150
162
|
class AssetMapping(MigrationMapping):
|
|
151
163
|
resource_type: Literal["asset"] = "asset"
|
|
164
|
+
instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
|
|
152
165
|
|
|
153
166
|
|
|
154
167
|
class EventMapping(MigrationMapping):
|
|
155
168
|
resource_type: Literal["event"] = "event"
|
|
169
|
+
instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
|
|
156
170
|
|
|
157
171
|
|
|
158
172
|
class TimeSeriesMapping(MigrationMapping):
|
|
159
173
|
resource_type: Literal["timeseries"] = "timeseries"
|
|
174
|
+
instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
|
|
160
175
|
|
|
161
176
|
|
|
162
177
|
class FileMapping(MigrationMapping):
|
|
163
178
|
resource_type: Literal["file"] = "file"
|
|
179
|
+
instance_id: Annotated[NodeId, BeforeValidator(_validate_node_id)]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class AnnotationMapping(MigrationMapping):
|
|
183
|
+
resource_type: Literal["annotation"] = "annotation"
|
|
184
|
+
instance_id: EdgeId
|
|
185
|
+
|
|
186
|
+
@field_validator("instance_id", mode="before")
|
|
187
|
+
def _validate_instance_id(cls, v: Any) -> Any:
|
|
188
|
+
if isinstance(v, dict):
|
|
189
|
+
return EdgeId.load(v)
|
|
190
|
+
return v
|
|
164
191
|
|
|
165
192
|
|
|
166
193
|
class AssetMigrationMappingList(MigrationMappingList):
|
|
@@ -187,10 +214,16 @@ class TimeSeriesMigrationMappingList(MigrationMappingList):
|
|
|
187
214
|
return TimeSeriesMapping
|
|
188
215
|
|
|
189
216
|
|
|
217
|
+
class AnnotationMigrationMappingList(MigrationMappingList):
|
|
218
|
+
@classmethod
|
|
219
|
+
def _get_base_model_cls(cls) -> type[AnnotationMapping]:
|
|
220
|
+
return AnnotationMapping
|
|
221
|
+
|
|
222
|
+
|
|
190
223
|
@dataclass
|
|
191
|
-
class AssetCentricMapping(Generic[
|
|
224
|
+
class AssetCentricMapping(Generic[T_AssetCentricResourceExtended], WriteableCogniteResource[InstanceApply]):
|
|
192
225
|
mapping: MigrationMapping
|
|
193
|
-
resource:
|
|
226
|
+
resource: T_AssetCentricResourceExtended
|
|
194
227
|
|
|
195
228
|
def as_write(self) -> InstanceApply:
|
|
196
229
|
raise NotImplementedError()
|
|
@@ -205,7 +238,9 @@ class AssetCentricMapping(Generic[T_AssetCentricResource], WriteableCogniteResou
|
|
|
205
238
|
}
|
|
206
239
|
|
|
207
240
|
|
|
208
|
-
class AssetCentricMappingList(
|
|
241
|
+
class AssetCentricMappingList(
|
|
242
|
+
WriteableCogniteResourceList[InstanceApply, AssetCentricMapping[T_AssetCentricResourceExtended]]
|
|
243
|
+
):
|
|
209
244
|
_RESOURCE: type = AssetCentricMapping
|
|
210
245
|
|
|
211
246
|
def as_write(self) -> InstanceApplyList:
|