cognite-toolkit 0.6.78__py3-none-any.whl → 0.6.80__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-toolkit might be problematic. Click here for more details.
- cognite_toolkit/_builtin_modules/cdf.toml +1 -1
- cognite_toolkit/_cdf.py +0 -4
- cognite_toolkit/_cdf_tk/apps/__init__.py +0 -2
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +134 -21
- cognite_toolkit/_cdf_tk/apps/_modules_app.py +27 -0
- cognite_toolkit/_cdf_tk/commands/__init__.py +0 -6
- cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py +0 -4
- cognite_toolkit/_cdf_tk/commands/_migrate/command.py +6 -8
- cognite_toolkit/_cdf_tk/commands/_migrate/creators.py +26 -2
- cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py +42 -8
- cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +4 -6
- cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py +1 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +203 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py +66 -0
- cognite_toolkit/_cdf_tk/commands/modules.py +59 -14
- cognite_toolkit/_cdf_tk/constants.py +3 -0
- cognite_toolkit/_cdf_tk/feature_flags.py +0 -4
- cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +6 -6
- cognite_toolkit/_cdf_tk/storageio/_base.py +2 -5
- cognite_toolkit/_cdf_tk/utils/useful_types.py +3 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +13 -0
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.6.78.dist-info → cognite_toolkit-0.6.80.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.6.78.dist-info → cognite_toolkit-0.6.80.dist-info}/RECORD +29 -32
- cognite_toolkit/_cdf_tk/apps/_populate_app.py +0 -80
- cognite_toolkit/_cdf_tk/commands/_migrate/adapter.py +0 -368
- cognite_toolkit/_cdf_tk/commands/_migrate/assets.py +0 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/files.py +0 -165
- cognite_toolkit/_cdf_tk/commands/_migrate/timeseries.py +0 -165
- cognite_toolkit/_cdf_tk/commands/_populate.py +0 -306
- {cognite_toolkit-0.6.78.dist-info → cognite_toolkit-0.6.80.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.6.78.dist-info → cognite_toolkit-0.6.80.dist-info}/entry_points.txt +0 -0
- {cognite_toolkit-0.6.78.dist-info → cognite_toolkit-0.6.80.dist-info}/licenses/LICENSE +0 -0
cognite_toolkit/_cdf.py
CHANGED
|
@@ -29,7 +29,6 @@ from cognite_toolkit._cdf_tk.apps import (
|
|
|
29
29
|
LandingApp,
|
|
30
30
|
MigrateApp,
|
|
31
31
|
ModulesApp,
|
|
32
|
-
PopulateApp,
|
|
33
32
|
ProfileApp,
|
|
34
33
|
PurgeApp,
|
|
35
34
|
RepoApp,
|
|
@@ -100,9 +99,6 @@ if Plugins.dump.value.is_enabled():
|
|
|
100
99
|
if Plugins.purge.value.is_enabled() and not Flags.v07.is_enabled():
|
|
101
100
|
_app.add_typer(PurgeApp(**default_typer_kws), name="purge")
|
|
102
101
|
|
|
103
|
-
if Flags.POPULATE.is_enabled():
|
|
104
|
-
_app.add_typer(PopulateApp(**default_typer_kws), name="populate")
|
|
105
|
-
|
|
106
102
|
if Flags.PROFILE.is_enabled():
|
|
107
103
|
_app.add_typer(ProfileApp(**default_typer_kws), name="profile")
|
|
108
104
|
|
|
@@ -6,7 +6,6 @@ from ._dump_app import DumpApp
|
|
|
6
6
|
from ._landing_app import LandingApp
|
|
7
7
|
from ._migrate_app import MigrateApp
|
|
8
8
|
from ._modules_app import ModulesApp
|
|
9
|
-
from ._populate_app import PopulateApp
|
|
10
9
|
from ._profile_app import ProfileApp
|
|
11
10
|
from ._purge import PurgeApp
|
|
12
11
|
from ._repo_app import RepoApp
|
|
@@ -22,7 +21,6 @@ __all__ = [
|
|
|
22
21
|
"LandingApp",
|
|
23
22
|
"MigrateApp",
|
|
24
23
|
"ModulesApp",
|
|
25
|
-
"PopulateApp",
|
|
26
24
|
"ProfileApp",
|
|
27
25
|
"PurgeApp",
|
|
28
26
|
"RepoApp",
|
|
@@ -8,21 +8,24 @@ from cognite.client.data_classes.data_modeling import ContainerId
|
|
|
8
8
|
|
|
9
9
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
10
10
|
from cognite_toolkit._cdf_tk.commands import (
|
|
11
|
-
MigrateFilesCommand,
|
|
12
11
|
MigrationCanvasCommand,
|
|
13
12
|
MigrationPrepareCommand,
|
|
14
13
|
)
|
|
15
14
|
from cognite_toolkit._cdf_tk.commands._migrate import MigrationCommand
|
|
16
|
-
from cognite_toolkit._cdf_tk.commands._migrate.
|
|
17
|
-
|
|
15
|
+
from cognite_toolkit._cdf_tk.commands._migrate.creators import (
|
|
16
|
+
InfieldV2ConfigCreator,
|
|
17
|
+
InstanceSpaceCreator,
|
|
18
|
+
SourceSystemCreator,
|
|
19
|
+
)
|
|
20
|
+
from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper
|
|
21
|
+
from cognite_toolkit._cdf_tk.commands._migrate.migration_io import (
|
|
22
|
+
AssetCentricMigrationIO,
|
|
23
|
+
)
|
|
24
|
+
from cognite_toolkit._cdf_tk.commands._migrate.selectors import (
|
|
25
|
+
AssetCentricMigrationSelector,
|
|
18
26
|
MigrateDataSetSelector,
|
|
19
27
|
MigrationCSVFileSelector,
|
|
20
|
-
MigrationSelector,
|
|
21
|
-
TimeSeriesIOAdapter,
|
|
22
28
|
)
|
|
23
|
-
from cognite_toolkit._cdf_tk.commands._migrate.creators import InstanceSpaceCreator, SourceSystemCreator
|
|
24
|
-
from cognite_toolkit._cdf_tk.commands._migrate.data_mapper import AssetCentricMapper
|
|
25
|
-
from cognite_toolkit._cdf_tk.storageio import AssetIO, EventIO
|
|
26
29
|
from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
|
|
27
30
|
from cognite_toolkit._cdf_tk.utils.cli_args import parse_view_str
|
|
28
31
|
from cognite_toolkit._cdf_tk.utils.interactive_select import (
|
|
@@ -47,6 +50,8 @@ class MigrateApp(typer.Typer):
|
|
|
47
50
|
self.command("timeseries")(self.timeseries)
|
|
48
51
|
self.command("files")(self.files)
|
|
49
52
|
self.command("canvas")(self.canvas)
|
|
53
|
+
# Uncomment when infield v2 config migration is ready
|
|
54
|
+
# self.command("infield-configs")(self.infield_configs)
|
|
50
55
|
|
|
51
56
|
def main(self, ctx: typer.Context) -> None:
|
|
52
57
|
"""Migrate resources from Asset-Centric to data modeling in CDF."""
|
|
@@ -317,7 +322,7 @@ class MigrateApp(typer.Typer):
|
|
|
317
322
|
cmd.run(
|
|
318
323
|
lambda: cmd.migrate(
|
|
319
324
|
selected=selected,
|
|
320
|
-
data=
|
|
325
|
+
data=AssetCentricMigrationIO(client),
|
|
321
326
|
mapper=AssetCentricMapper(client),
|
|
322
327
|
log_dir=log_dir,
|
|
323
328
|
dry_run=dry_run,
|
|
@@ -337,11 +342,11 @@ class MigrateApp(typer.Typer):
|
|
|
337
342
|
kind: AssetCentricKind,
|
|
338
343
|
resource_type: str,
|
|
339
344
|
container_id: ContainerId,
|
|
340
|
-
) -> tuple[
|
|
345
|
+
) -> tuple[AssetCentricMigrationSelector, bool, bool]:
|
|
341
346
|
if data_set_id is not None and mapping_file is not None:
|
|
342
347
|
raise typer.BadParameter("Cannot specify both data_set_id and mapping_file")
|
|
343
348
|
elif mapping_file is not None:
|
|
344
|
-
selected:
|
|
349
|
+
selected: AssetCentricMigrationSelector = MigrationCSVFileSelector(datafile=mapping_file, kind=kind)
|
|
345
350
|
elif data_set_id is not None:
|
|
346
351
|
parsed_view = parse_view_str(consumption_view) if consumption_view is not None else None
|
|
347
352
|
selected = MigrateDataSetSelector(
|
|
@@ -466,7 +471,7 @@ class MigrateApp(typer.Typer):
|
|
|
466
471
|
cmd.run(
|
|
467
472
|
lambda: cmd.migrate(
|
|
468
473
|
selected=selected,
|
|
469
|
-
data=
|
|
474
|
+
data=AssetCentricMigrationIO(client),
|
|
470
475
|
mapper=AssetCentricMapper(client),
|
|
471
476
|
log_dir=log_dir,
|
|
472
477
|
dry_run=dry_run,
|
|
@@ -573,7 +578,7 @@ class MigrateApp(typer.Typer):
|
|
|
573
578
|
cmd.run(
|
|
574
579
|
lambda: cmd.migrate(
|
|
575
580
|
selected=selected,
|
|
576
|
-
data=
|
|
581
|
+
data=AssetCentricMigrationIO(client, skip_linking=skip_linking),
|
|
577
582
|
mapper=AssetCentricMapper(client),
|
|
578
583
|
log_dir=log_dir,
|
|
579
584
|
dry_run=dry_run,
|
|
@@ -581,11 +586,12 @@ class MigrateApp(typer.Typer):
|
|
|
581
586
|
)
|
|
582
587
|
)
|
|
583
588
|
|
|
584
|
-
@
|
|
589
|
+
@classmethod
|
|
585
590
|
def files(
|
|
591
|
+
cls,
|
|
586
592
|
ctx: typer.Context,
|
|
587
593
|
mapping_file: Annotated[
|
|
588
|
-
Path,
|
|
594
|
+
Path | None,
|
|
589
595
|
typer.Option(
|
|
590
596
|
"--mapping-file",
|
|
591
597
|
"-m",
|
|
@@ -593,7 +599,51 @@ class MigrateApp(typer.Typer):
|
|
|
593
599
|
"This file is expected to have the following columns: [id, dataSetId, space, externalId]."
|
|
594
600
|
"The dataSetId is optional, and can be skipped. If it is set, it is used to check the access to the dataset.",
|
|
595
601
|
),
|
|
596
|
-
],
|
|
602
|
+
] = None,
|
|
603
|
+
data_set_id: Annotated[
|
|
604
|
+
str | None,
|
|
605
|
+
typer.Option(
|
|
606
|
+
"--data-set-id",
|
|
607
|
+
"-s",
|
|
608
|
+
help="The data set ID to select for the files to migrate. If not provided, the dataSetId from the mapping file is used. "
|
|
609
|
+
"If neither is provided, the default data set for the project is used.",
|
|
610
|
+
),
|
|
611
|
+
] = None,
|
|
612
|
+
ingestion_mapping: Annotated[
|
|
613
|
+
str | None,
|
|
614
|
+
typer.Option(
|
|
615
|
+
"--ingestion-mapping",
|
|
616
|
+
"-i",
|
|
617
|
+
help="The ingestion mapping to use for the migrated files. If not provided, "
|
|
618
|
+
"the default mapping to CogniteFile in CogniteCore will be used.",
|
|
619
|
+
),
|
|
620
|
+
] = None,
|
|
621
|
+
consumption_view: Annotated[
|
|
622
|
+
str | None,
|
|
623
|
+
typer.Option(
|
|
624
|
+
"--consumption-view",
|
|
625
|
+
"-c",
|
|
626
|
+
help="The consumption view to assign to the migrated files Given as space:externalId/version. "
|
|
627
|
+
"This will be used in Canvas to select which view to use when migrating timeseries. If not provided, "
|
|
628
|
+
"CogniteFile in CogniteCore will be used.",
|
|
629
|
+
),
|
|
630
|
+
] = None,
|
|
631
|
+
log_dir: Annotated[
|
|
632
|
+
Path,
|
|
633
|
+
typer.Option(
|
|
634
|
+
"--log-dir",
|
|
635
|
+
"-l",
|
|
636
|
+
help="Path to the directory where logs will be stored. If the directory does not exist, it will be created.",
|
|
637
|
+
),
|
|
638
|
+
] = Path(f"migration_logs_{TODAY!s}"),
|
|
639
|
+
skip_linking: Annotated[
|
|
640
|
+
bool,
|
|
641
|
+
typer.Option(
|
|
642
|
+
"--skip-linking",
|
|
643
|
+
"-x",
|
|
644
|
+
help="If set, the migration will not create links between the old FileMetadata and the new CogniteFile.",
|
|
645
|
+
),
|
|
646
|
+
] = False,
|
|
597
647
|
dry_run: Annotated[
|
|
598
648
|
bool,
|
|
599
649
|
typer.Option(
|
|
@@ -612,13 +662,33 @@ class MigrateApp(typer.Typer):
|
|
|
612
662
|
] = False,
|
|
613
663
|
) -> None:
|
|
614
664
|
"""Migrate Files to CogniteFiles."""
|
|
615
|
-
|
|
616
665
|
client = EnvironmentVariables.create_from_environment().get_client(enable_set_pending_ids=True)
|
|
617
|
-
|
|
666
|
+
|
|
667
|
+
selected, dry_run, verbose = cls._prepare_asset_centric_arguments(
|
|
668
|
+
client=client,
|
|
669
|
+
mapping_file=mapping_file,
|
|
670
|
+
data_set_id=data_set_id,
|
|
671
|
+
consumption_view=consumption_view,
|
|
672
|
+
ingestion_mapping=ingestion_mapping,
|
|
673
|
+
dry_run=dry_run,
|
|
674
|
+
verbose=verbose,
|
|
675
|
+
kind="FileMetadata",
|
|
676
|
+
resource_type="file",
|
|
677
|
+
container_id=ContainerId("cdf_cdm", "CogniteFile"),
|
|
678
|
+
)
|
|
679
|
+
cmd = MigrationCommand()
|
|
680
|
+
|
|
681
|
+
if data_set_id is None:
|
|
682
|
+
skip_linking = not questionary.confirm(
|
|
683
|
+
"Do you want to link old and new Files?", default=not skip_linking
|
|
684
|
+
).ask()
|
|
685
|
+
|
|
618
686
|
cmd.run(
|
|
619
|
-
lambda: cmd.
|
|
620
|
-
|
|
621
|
-
|
|
687
|
+
lambda: cmd.migrate(
|
|
688
|
+
selected=selected,
|
|
689
|
+
data=AssetCentricMigrationIO(client, skip_linking=skip_linking),
|
|
690
|
+
mapper=AssetCentricMapper(client),
|
|
691
|
+
log_dir=log_dir,
|
|
622
692
|
dry_run=dry_run,
|
|
623
693
|
verbose=verbose,
|
|
624
694
|
)
|
|
@@ -669,3 +739,46 @@ class MigrateApp(typer.Typer):
|
|
|
669
739
|
verbose=verbose,
|
|
670
740
|
)
|
|
671
741
|
)
|
|
742
|
+
|
|
743
|
+
@staticmethod
|
|
744
|
+
def infield_configs(
|
|
745
|
+
ctx: typer.Context,
|
|
746
|
+
output_dir: Annotated[
|
|
747
|
+
Path,
|
|
748
|
+
typer.Option(
|
|
749
|
+
"--output-dir",
|
|
750
|
+
"-o",
|
|
751
|
+
help="Path to the directory where the Infield V2 configuration definitions will be dumped. It is recommended "
|
|
752
|
+
"to govern these configurations in a git repository.",
|
|
753
|
+
),
|
|
754
|
+
] = Path("tmp"),
|
|
755
|
+
dry_run: Annotated[
|
|
756
|
+
bool,
|
|
757
|
+
typer.Option(
|
|
758
|
+
"--dry-run",
|
|
759
|
+
"-d",
|
|
760
|
+
help="If set, the migration will not be executed, but only a report of what would be done is printed.",
|
|
761
|
+
),
|
|
762
|
+
] = False,
|
|
763
|
+
verbose: Annotated[
|
|
764
|
+
bool,
|
|
765
|
+
typer.Option(
|
|
766
|
+
"--verbose",
|
|
767
|
+
"-v",
|
|
768
|
+
help="Turn on to get more verbose output when running the command",
|
|
769
|
+
),
|
|
770
|
+
] = False,
|
|
771
|
+
) -> None:
|
|
772
|
+
"""Creates Infield V2 configurations from existing APM Configurations in CDF."""
|
|
773
|
+
client = EnvironmentVariables.create_from_environment().get_client()
|
|
774
|
+
|
|
775
|
+
cmd = MigrationCommand()
|
|
776
|
+
cmd.run(
|
|
777
|
+
lambda: cmd.create(
|
|
778
|
+
client,
|
|
779
|
+
creator=InfieldV2ConfigCreator(client),
|
|
780
|
+
output_dir=output_dir,
|
|
781
|
+
dry_run=dry_run,
|
|
782
|
+
verbose=verbose,
|
|
783
|
+
)
|
|
784
|
+
)
|
|
@@ -6,6 +6,7 @@ from rich import print
|
|
|
6
6
|
|
|
7
7
|
from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
|
|
8
8
|
from cognite_toolkit._cdf_tk.commands import ModulesCommand, PullCommand
|
|
9
|
+
from cognite_toolkit._cdf_tk.feature_flags import Flags
|
|
9
10
|
from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
|
|
10
11
|
from cognite_toolkit._version import __version__
|
|
11
12
|
|
|
@@ -51,6 +52,24 @@ class ModulesApp(typer.Typer):
|
|
|
51
52
|
help="Clean target directory if it exists",
|
|
52
53
|
),
|
|
53
54
|
] = False,
|
|
55
|
+
library_url: Annotated[
|
|
56
|
+
str | None,
|
|
57
|
+
typer.Option(
|
|
58
|
+
"--library-url",
|
|
59
|
+
"-u",
|
|
60
|
+
help="URL of the library to add to the project.",
|
|
61
|
+
hidden=not Flags.EXTERNAL_LIBRARIES.is_enabled(),
|
|
62
|
+
),
|
|
63
|
+
] = None,
|
|
64
|
+
library_checksum: Annotated[
|
|
65
|
+
str | None,
|
|
66
|
+
typer.Option(
|
|
67
|
+
"--library-checksum",
|
|
68
|
+
"-c",
|
|
69
|
+
help="Checksum of the library to add to the project.",
|
|
70
|
+
hidden=not Flags.EXTERNAL_LIBRARIES.is_enabled(),
|
|
71
|
+
),
|
|
72
|
+
] = None,
|
|
54
73
|
verbose: Annotated[
|
|
55
74
|
bool,
|
|
56
75
|
typer.Option(
|
|
@@ -62,12 +81,20 @@ class ModulesApp(typer.Typer):
|
|
|
62
81
|
) -> None:
|
|
63
82
|
"""Initialize or upgrade a new CDF project with templates interactively."""
|
|
64
83
|
|
|
84
|
+
if library_url and not library_checksum:
|
|
85
|
+
raise typer.BadParameter(
|
|
86
|
+
"--library-checksum must be provided when --library-url is specified.",
|
|
87
|
+
param_hint="--library-checksum",
|
|
88
|
+
)
|
|
89
|
+
|
|
65
90
|
with ModulesCommand() as cmd:
|
|
66
91
|
cmd.run(
|
|
67
92
|
lambda: cmd.init(
|
|
68
93
|
organization_dir=organization_dir,
|
|
69
94
|
select_all=all,
|
|
70
95
|
clean=clean,
|
|
96
|
+
library_url=library_url,
|
|
97
|
+
library_checksum=library_checksum,
|
|
71
98
|
)
|
|
72
99
|
)
|
|
73
100
|
|
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
from ._download import DownloadCommand
|
|
2
2
|
from ._migrate import (
|
|
3
|
-
MigrateFilesCommand,
|
|
4
|
-
MigrateTimeseriesCommand,
|
|
5
3
|
MigrationCanvasCommand,
|
|
6
4
|
MigrationPrepareCommand,
|
|
7
5
|
)
|
|
8
|
-
from ._populate import PopulateCommand
|
|
9
6
|
from ._profile import ProfileAssetCentricCommand, ProfileAssetCommand, ProfileRawCommand, ProfileTransformationCommand
|
|
10
7
|
from ._purge import PurgeCommand
|
|
11
8
|
from ._upload import UploadCommand
|
|
@@ -34,12 +31,9 @@ __all__ = [
|
|
|
34
31
|
"DumpResourceCommand",
|
|
35
32
|
"FeatureFlagCommand",
|
|
36
33
|
"InitCommand",
|
|
37
|
-
"MigrateFilesCommand",
|
|
38
|
-
"MigrateTimeseriesCommand",
|
|
39
34
|
"MigrationCanvasCommand",
|
|
40
35
|
"MigrationPrepareCommand",
|
|
41
36
|
"ModulesCommand",
|
|
42
|
-
"PopulateCommand",
|
|
43
37
|
"ProfileAssetCentricCommand",
|
|
44
38
|
"ProfileAssetCommand",
|
|
45
39
|
"ProfileRawCommand",
|
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
from .canvas import MigrationCanvasCommand
|
|
2
2
|
from .command import MigrationCommand
|
|
3
|
-
from .files import MigrateFilesCommand
|
|
4
3
|
from .prepare import MigrationPrepareCommand
|
|
5
|
-
from .timeseries import MigrateTimeseriesCommand
|
|
6
4
|
|
|
7
5
|
__all__ = [
|
|
8
|
-
"MigrateFilesCommand",
|
|
9
|
-
"MigrateTimeseriesCommand",
|
|
10
6
|
"MigrationCanvasCommand",
|
|
11
7
|
"MigrationCommand",
|
|
12
8
|
"MigrationPrepareCommand",
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
from collections.abc import Callable, Iterable, Sequence
|
|
2
2
|
from enum import Enum
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from typing import TypeVar
|
|
5
4
|
|
|
6
|
-
from cognite.client.data_classes._base import
|
|
5
|
+
from cognite.client.data_classes._base import T_CogniteResource
|
|
7
6
|
from rich import print
|
|
8
7
|
from rich.console import Console
|
|
9
8
|
from rich.table import Table
|
|
@@ -28,12 +27,10 @@ from cognite_toolkit._cdf_tk.utils.fileio import Chunk, CSVWriter, NDJsonWriter,
|
|
|
28
27
|
from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, ItemMessage, SuccessResponseItems
|
|
29
28
|
from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
|
|
30
29
|
from cognite_toolkit._cdf_tk.utils.progress_tracker import AVAILABLE_STATUS, ProgressTracker, Status
|
|
30
|
+
from cognite_toolkit._cdf_tk.utils.useful_types import T_WriteCogniteResource
|
|
31
31
|
|
|
32
32
|
from .data_model import INSTANCE_SOURCE_VIEW_ID, MODEL_ID, RESOURCE_VIEW_MAPPING_VIEW_ID
|
|
33
33
|
|
|
34
|
-
T_CogniteResource = TypeVar("T_CogniteResource", bound=CogniteResource)
|
|
35
|
-
T_WriteCogniteResource = TypeVar("T_WriteCogniteResource", bound=CogniteResource)
|
|
36
|
-
|
|
37
34
|
|
|
38
35
|
class MigrationCommand(ToolkitCommand):
|
|
39
36
|
class Steps(str, Enum):
|
|
@@ -79,7 +76,7 @@ class MigrationCommand(ToolkitCommand):
|
|
|
79
76
|
](
|
|
80
77
|
download_iterable=self._download_iterable(selected, data, tracker),
|
|
81
78
|
process=self._convert(mapper, data, tracker, log_file),
|
|
82
|
-
write=self._upload(write_client, data, tracker, log_file, dry_run),
|
|
79
|
+
write=self._upload(selected, write_client, data, tracker, log_file, dry_run),
|
|
83
80
|
iteration_count=iteration_count,
|
|
84
81
|
max_queue_size=10,
|
|
85
82
|
download_description=f"Downloading {selected.display_name}",
|
|
@@ -179,6 +176,7 @@ class MigrationCommand(ToolkitCommand):
|
|
|
179
176
|
|
|
180
177
|
def _upload(
|
|
181
178
|
self,
|
|
179
|
+
selected: T_Selector,
|
|
182
180
|
write_client: HTTPClient,
|
|
183
181
|
target: UploadableStorageIO[T_Selector, T_CogniteResource, T_WriteCogniteResource],
|
|
184
182
|
tracker: ProgressTracker[str],
|
|
@@ -192,7 +190,7 @@ class MigrationCommand(ToolkitCommand):
|
|
|
192
190
|
if dry_run:
|
|
193
191
|
responses = [SuccessResponseItems(200, "", [item.source_id for item in data_item])]
|
|
194
192
|
else:
|
|
195
|
-
responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=
|
|
193
|
+
responses = target.upload_items(data_chunk=data_item, http_client=write_client, selector=selected)
|
|
196
194
|
|
|
197
195
|
issues: list[Chunk] = []
|
|
198
196
|
for item in responses:
|
|
@@ -281,7 +279,7 @@ class MigrationCommand(ToolkitCommand):
|
|
|
281
279
|
result = deploy_cmd.dry_run_deploy(resources, crud, False, False)
|
|
282
280
|
else:
|
|
283
281
|
result = deploy_cmd.actual_deploy(resources, crud)
|
|
284
|
-
if result.calculated_total > 0:
|
|
282
|
+
if result.calculated_total > 0 and creator.HAS_LINEAGE:
|
|
285
283
|
store_count = creator.store_lineage(resource_list)
|
|
286
284
|
self.console(f"Stored lineage for {store_count:,} {creator.DISPLAY_NAME}.")
|
|
287
285
|
|
|
@@ -20,6 +20,7 @@ from cognite.client.data_classes.documents import SourceFileProperty
|
|
|
20
20
|
from cognite.client.data_classes.events import EventProperty
|
|
21
21
|
|
|
22
22
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
23
|
+
from cognite_toolkit._cdf_tk.client.data_classes.apm_config_v1 import APMConfig, APMConfigList
|
|
23
24
|
from cognite_toolkit._cdf_tk.cruds import NodeCRUD, ResourceCRUD, SpaceCRUD
|
|
24
25
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitRequiredValueError
|
|
25
26
|
from cognite_toolkit._cdf_tk.utils import humanize_collection
|
|
@@ -34,10 +35,11 @@ class ResourceConfig:
|
|
|
34
35
|
|
|
35
36
|
|
|
36
37
|
class MigrationCreator(ABC, Generic[T_CogniteResourceList]):
|
|
37
|
-
"""Base class for migration resources configurations that are created
|
|
38
|
+
"""Base class for migration resources configurations that are created resources."""
|
|
38
39
|
|
|
39
40
|
CRUD: type[ResourceCRUD]
|
|
40
41
|
DISPLAY_NAME: str
|
|
42
|
+
HAS_LINEAGE: bool = True
|
|
41
43
|
|
|
42
44
|
def __init__(self, client: ToolkitClient) -> None:
|
|
43
45
|
self.client = client
|
|
@@ -50,7 +52,6 @@ class MigrationCreator(ABC, Generic[T_CogniteResourceList]):
|
|
|
50
52
|
def resource_configs(self, resources: T_CogniteResourceList) -> list[ResourceConfig]:
|
|
51
53
|
raise NotImplementedError("Subclasses should implement this method")
|
|
52
54
|
|
|
53
|
-
@abstractmethod
|
|
54
55
|
def store_lineage(self, resources: T_CogniteResourceList) -> int:
|
|
55
56
|
"""Store lineage information for the created resources.
|
|
56
57
|
|
|
@@ -65,6 +66,7 @@ class InstanceSpaceCreator(MigrationCreator[SpaceApplyList]):
|
|
|
65
66
|
|
|
66
67
|
CRUD = SpaceCRUD
|
|
67
68
|
DISPLAY_NAME = "Instance Space"
|
|
69
|
+
HAS_LINEAGE = True
|
|
68
70
|
|
|
69
71
|
def __init__(
|
|
70
72
|
self, client: ToolkitClient, datasets: DataSetList | None = None, data_set_external_ids: list[str] | None = None
|
|
@@ -205,3 +207,25 @@ class SourceSystemCreator(MigrationCreator[NodeApplyList]):
|
|
|
205
207
|
def store_lineage(self, resources: NodeApplyList) -> int:
|
|
206
208
|
# We already store lineage when creating the resources.
|
|
207
209
|
return len(resources)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class InfieldV2ConfigCreator(MigrationCreator[NodeApplyList]):
|
|
213
|
+
CRUD = NodeCRUD
|
|
214
|
+
DISPLAY_NAME = "Infield V2 Configuration"
|
|
215
|
+
HAS_LINEAGE = False
|
|
216
|
+
|
|
217
|
+
def create_resources(self) -> NodeApplyList:
|
|
218
|
+
apm_config_nodes = self.client.data_modeling.instances.list(instance_type="node", sources=APMConfig.view_id)
|
|
219
|
+
apm_config = APMConfigList.from_nodes(apm_config_nodes)
|
|
220
|
+
|
|
221
|
+
new_config_nodes = NodeApplyList([])
|
|
222
|
+
for config in apm_config:
|
|
223
|
+
new_config = self._create_infield_v2_config(config)
|
|
224
|
+
new_config_nodes.append(new_config)
|
|
225
|
+
return new_config_nodes
|
|
226
|
+
|
|
227
|
+
def resource_configs(self, resources: NodeApplyList) -> list[ResourceConfig]:
|
|
228
|
+
return [ResourceConfig(filestem=node.external_id, data=node.dump()) for node in resources]
|
|
229
|
+
|
|
230
|
+
def _create_infield_v2_config(self, config: APMConfig) -> NodeApply:
|
|
231
|
+
raise NotImplementedError("To be implemented")
|
|
@@ -1,10 +1,17 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
1
2
|
from pathlib import Path
|
|
2
|
-
from typing import Any, Literal
|
|
3
|
+
from typing import Any, Generic, Literal
|
|
3
4
|
|
|
4
|
-
from cognite.client.data_classes.
|
|
5
|
+
from cognite.client.data_classes._base import (
|
|
6
|
+
T_WritableCogniteResource,
|
|
7
|
+
WriteableCogniteResource,
|
|
8
|
+
WriteableCogniteResourceList,
|
|
9
|
+
)
|
|
10
|
+
from cognite.client.data_classes.data_modeling import InstanceApply, NodeId, ViewId
|
|
5
11
|
from cognite.client.utils._text import to_camel_case
|
|
6
12
|
from pydantic import BaseModel, field_validator, model_validator
|
|
7
13
|
|
|
14
|
+
from cognite_toolkit._cdf_tk.client.data_classes.instances import InstanceApplyList
|
|
8
15
|
from cognite_toolkit._cdf_tk.client.data_classes.migration import AssetCentricId
|
|
9
16
|
from cognite_toolkit._cdf_tk.client.data_classes.pending_instances_ids import PendingInstanceId
|
|
10
17
|
from cognite_toolkit._cdf_tk.commands._migrate.default_mappings import create_default_mappings
|
|
@@ -12,7 +19,7 @@ from cognite_toolkit._cdf_tk.exceptions import (
|
|
|
12
19
|
ToolkitValueError,
|
|
13
20
|
)
|
|
14
21
|
from cognite_toolkit._cdf_tk.storageio._data_classes import ModelList
|
|
15
|
-
from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricType
|
|
22
|
+
from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind, AssetCentricType, JsonVal
|
|
16
23
|
|
|
17
24
|
|
|
18
25
|
class MigrationMapping(BaseModel, alias_generator=to_camel_case, extra="ignore", populate_by_name=True):
|
|
@@ -122,14 +129,14 @@ class MigrationMappingList(ModelList[MigrationMapping]):
|
|
|
122
129
|
return {mapping.id: mapping for mapping in self}
|
|
123
130
|
|
|
124
131
|
@classmethod
|
|
125
|
-
def read_csv_file(cls, filepath: Path, resource_type:
|
|
132
|
+
def read_csv_file(cls, filepath: Path, resource_type: AssetCentricKind | None = None) -> "MigrationMappingList":
|
|
126
133
|
if cls is not MigrationMappingList or resource_type is None:
|
|
127
134
|
return super().read_csv_file(filepath)
|
|
128
135
|
cls_by_resource_type: dict[str, type[MigrationMappingList]] = {
|
|
129
|
-
"
|
|
130
|
-
"
|
|
131
|
-
"
|
|
132
|
-
"
|
|
136
|
+
"Assets": AssetMigrationMappingList,
|
|
137
|
+
"TimeSeries": TimeSeriesMigrationMappingList,
|
|
138
|
+
"FileMetadata": FileMigrationMappingList,
|
|
139
|
+
"Events": EventMigrationMappingList,
|
|
133
140
|
}
|
|
134
141
|
if resource_type not in cls_by_resource_type:
|
|
135
142
|
raise ToolkitValueError(
|
|
@@ -176,3 +183,30 @@ class TimeSeriesMigrationMappingList(MigrationMappingList):
|
|
|
176
183
|
@classmethod
|
|
177
184
|
def _get_base_model_cls(cls) -> type[TimeSeriesMapping]:
|
|
178
185
|
return TimeSeriesMapping
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@dataclass
|
|
189
|
+
class AssetCentricMapping(Generic[T_WritableCogniteResource], WriteableCogniteResource[InstanceApply]):
|
|
190
|
+
mapping: MigrationMapping
|
|
191
|
+
resource: T_WritableCogniteResource
|
|
192
|
+
|
|
193
|
+
def as_write(self) -> InstanceApply:
|
|
194
|
+
raise NotImplementedError()
|
|
195
|
+
|
|
196
|
+
def dump(self, camel_case: bool = True) -> dict[str, JsonVal]:
|
|
197
|
+
mapping = self.mapping.model_dump(exclude_unset=True, by_alias=camel_case)
|
|
198
|
+
# Ensure that resource type is always included, even if unset.
|
|
199
|
+
mapping["resourceType" if camel_case else "resource_type"] = self.mapping.resource_type
|
|
200
|
+
return {
|
|
201
|
+
"mapping": mapping,
|
|
202
|
+
"resource": self.resource.dump(camel_case=camel_case),
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
class AssetCentricMappingList(
|
|
207
|
+
WriteableCogniteResourceList[InstanceApply, AssetCentricMapping[T_WritableCogniteResource]]
|
|
208
|
+
):
|
|
209
|
+
_RESOURCE: type = AssetCentricMapping
|
|
210
|
+
|
|
211
|
+
def as_write(self) -> InstanceApplyList:
|
|
212
|
+
return InstanceApplyList([item.as_write() for item in self])
|
|
@@ -8,12 +8,10 @@ from cognite.client.data_classes.data_modeling import DirectRelationReference, I
|
|
|
8
8
|
|
|
9
9
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
10
10
|
from cognite_toolkit._cdf_tk.client.data_classes.migration import ResourceViewMapping
|
|
11
|
-
from cognite_toolkit._cdf_tk.commands._migrate.adapter import (
|
|
12
|
-
AssetCentricMapping,
|
|
13
|
-
MigrationSelector,
|
|
14
|
-
)
|
|
15
11
|
from cognite_toolkit._cdf_tk.commands._migrate.conversion import asset_centric_to_dm
|
|
12
|
+
from cognite_toolkit._cdf_tk.commands._migrate.data_classes import AssetCentricMapping
|
|
16
13
|
from cognite_toolkit._cdf_tk.commands._migrate.issues import ConversionIssue, MigrationIssue
|
|
14
|
+
from cognite_toolkit._cdf_tk.commands._migrate.selectors import AssetCentricMigrationSelector
|
|
17
15
|
from cognite_toolkit._cdf_tk.constants import MISSING_INSTANCE_SPACE
|
|
18
16
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
|
19
17
|
from cognite_toolkit._cdf_tk.storageio._base import T_Selector, T_WriteCogniteResource
|
|
@@ -45,7 +43,7 @@ class DataMapper(Generic[T_Selector, T_CogniteResource, T_WriteCogniteResource],
|
|
|
45
43
|
raise NotImplementedError("Subclasses must implement this method.")
|
|
46
44
|
|
|
47
45
|
|
|
48
|
-
class AssetCentricMapper(DataMapper[
|
|
46
|
+
class AssetCentricMapper(DataMapper[AssetCentricMigrationSelector, AssetCentricMapping, InstanceApply]):
|
|
49
47
|
def __init__(self, client: ToolkitClient) -> None:
|
|
50
48
|
self.client = client
|
|
51
49
|
self._ingestion_view_by_id: dict[ViewId, View] = {}
|
|
@@ -55,7 +53,7 @@ class AssetCentricMapper(DataMapper[MigrationSelector, AssetCentricMapping, Inst
|
|
|
55
53
|
self._asset_mapping_by_id: dict[int, DirectRelationReference] = {}
|
|
56
54
|
self._source_system_mapping_by_id: dict[str, DirectRelationReference] = {}
|
|
57
55
|
|
|
58
|
-
def prepare(self, source_selector:
|
|
56
|
+
def prepare(self, source_selector: AssetCentricMigrationSelector) -> None:
|
|
59
57
|
ingestion_view_ids = source_selector.get_ingestion_mappings()
|
|
60
58
|
ingestion_views = self.client.migration.resource_view_mapping.retrieve(ingestion_view_ids)
|
|
61
59
|
self._view_mapping_by_id = {view.external_id: view for view in ingestion_views}
|
|
@@ -32,6 +32,7 @@ def create_default_mappings() -> list[ResourceViewMappingApply]:
|
|
|
32
32
|
"labels": "tags",
|
|
33
33
|
"mimeType": "mimeType",
|
|
34
34
|
"directory": "directory",
|
|
35
|
+
"assetIds": "assets",
|
|
35
36
|
"sourceCreatedTime": "sourceCreatedTime",
|
|
36
37
|
"sourceModifiedTime": "sourceUpdatedTime",
|
|
37
38
|
}
|