cognite-toolkit 0.7.30__py3-none-any.whl → 0.7.51__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf.py +5 -6
- cognite_toolkit/_cdf_tk/apps/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/apps/_core_app.py +7 -1
- cognite_toolkit/_cdf_tk/apps/_download_app.py +2 -2
- cognite_toolkit/_cdf_tk/apps/_dump_app.py +1 -1
- cognite_toolkit/_cdf_tk/apps/_import_app.py +41 -0
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +183 -8
- cognite_toolkit/_cdf_tk/builders/_function.py +81 -9
- cognite_toolkit/_cdf_tk/builders/_raw.py +1 -1
- cognite_toolkit/_cdf_tk/client/_resource_base.py +187 -0
- cognite_toolkit/_cdf_tk/client/_toolkit_client.py +44 -12
- cognite_toolkit/_cdf_tk/client/api/agents.py +107 -0
- cognite_toolkit/_cdf_tk/client/api/annotations.py +129 -0
- cognite_toolkit/_cdf_tk/client/api/assets.py +130 -0
- cognite_toolkit/_cdf_tk/client/api/containers.py +132 -0
- cognite_toolkit/_cdf_tk/client/api/data_models.py +137 -0
- cognite_toolkit/_cdf_tk/client/api/datasets.py +141 -0
- cognite_toolkit/_cdf_tk/client/api/events.py +129 -0
- cognite_toolkit/_cdf_tk/client/api/extraction_pipelines.py +148 -0
- cognite_toolkit/_cdf_tk/client/api/filemetadata.py +176 -0
- cognite_toolkit/_cdf_tk/client/api/function_schedules.py +115 -0
- cognite_toolkit/_cdf_tk/client/api/functions.py +113 -0
- cognite_toolkit/_cdf_tk/client/api/graphql_data_models.py +167 -0
- cognite_toolkit/_cdf_tk/client/api/groups.py +121 -0
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_destinations.py +131 -0
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_jobs.py +122 -0
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_mappings.py +129 -0
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_sources.py +136 -0
- cognite_toolkit/_cdf_tk/client/api/hosted_extractors.py +23 -0
- cognite_toolkit/_cdf_tk/client/api/infield.py +40 -35
- cognite_toolkit/_cdf_tk/client/api/instances.py +139 -0
- cognite_toolkit/_cdf_tk/client/api/labels.py +125 -0
- cognite_toolkit/_cdf_tk/client/api/{canvas.py → legacy/canvas.py} +16 -8
- cognite_toolkit/_cdf_tk/client/api/{charts.py → legacy/charts.py} +1 -1
- cognite_toolkit/_cdf_tk/client/api/{extended_data_modeling.py → legacy/extended_data_modeling.py} +1 -1
- cognite_toolkit/_cdf_tk/client/api/{extended_files.py → legacy/extended_files.py} +2 -2
- cognite_toolkit/_cdf_tk/client/api/{extended_functions.py → legacy/extended_functions.py} +9 -9
- cognite_toolkit/_cdf_tk/client/api/{extended_raw.py → legacy/extended_raw.py} +1 -1
- cognite_toolkit/_cdf_tk/client/api/{extended_timeseries.py → legacy/extended_timeseries.py} +5 -2
- cognite_toolkit/_cdf_tk/client/api/legacy/location_filters.py +112 -0
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/__init__.py +8 -0
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/capabilities.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/data_postprocessing.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/frames.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/locations.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/maps.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/robots.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/legacy/search_config.py +53 -0
- cognite_toolkit/_cdf_tk/client/api/location_filters.py +140 -75
- cognite_toolkit/_cdf_tk/client/api/migration.py +2 -3
- cognite_toolkit/_cdf_tk/client/api/project.py +9 -8
- cognite_toolkit/_cdf_tk/client/api/raw.py +204 -0
- cognite_toolkit/_cdf_tk/client/api/relationships.py +133 -0
- cognite_toolkit/_cdf_tk/client/api/robotics.py +19 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_capabilities.py +127 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_data_postprocessing.py +138 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_frames.py +122 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_locations.py +127 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_maps.py +122 -0
- cognite_toolkit/_cdf_tk/client/api/robotics_robots.py +122 -0
- cognite_toolkit/_cdf_tk/client/api/search.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/search_config.py +83 -31
- cognite_toolkit/_cdf_tk/client/api/security_categories.py +94 -0
- cognite_toolkit/_cdf_tk/client/api/sequences.py +133 -0
- cognite_toolkit/_cdf_tk/client/api/simulator_models.py +154 -0
- cognite_toolkit/_cdf_tk/client/api/simulators.py +8 -0
- cognite_toolkit/_cdf_tk/client/api/spaces.py +117 -0
- cognite_toolkit/_cdf_tk/client/api/streams.py +68 -56
- cognite_toolkit/_cdf_tk/client/api/three_d.py +385 -34
- cognite_toolkit/_cdf_tk/client/api/timeseries.py +137 -0
- cognite_toolkit/_cdf_tk/client/api/token.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/transformations.py +146 -0
- cognite_toolkit/_cdf_tk/client/api/views.py +139 -0
- cognite_toolkit/_cdf_tk/client/api/workflow_triggers.py +128 -0
- cognite_toolkit/_cdf_tk/client/api/workflow_versions.py +138 -0
- cognite_toolkit/_cdf_tk/client/api/workflows.py +119 -0
- cognite_toolkit/_cdf_tk/client/cdf_client/__init__.py +10 -0
- cognite_toolkit/_cdf_tk/client/cdf_client/api.py +358 -0
- cognite_toolkit/_cdf_tk/client/cdf_client/responses.py +38 -0
- cognite_toolkit/_cdf_tk/{utils → client}/http_client/__init__.py +30 -0
- cognite_toolkit/_cdf_tk/{utils → client}/http_client/_client.py +26 -16
- cognite_toolkit/_cdf_tk/{utils → client}/http_client/_data_classes.py +18 -2
- cognite_toolkit/_cdf_tk/client/http_client/_data_classes2.py +151 -0
- cognite_toolkit/_cdf_tk/client/http_client/_exception.py +13 -0
- cognite_toolkit/_cdf_tk/client/http_client/_item_classes.py +118 -0
- cognite_toolkit/_cdf_tk/{utils → client}/http_client/_tracker.py +5 -2
- cognite_toolkit/_cdf_tk/client/request_classes/base.py +19 -0
- cognite_toolkit/_cdf_tk/client/request_classes/filters.py +113 -0
- cognite_toolkit/_cdf_tk/client/request_classes/graphql.py +28 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/__init__.py +0 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/agent.py +130 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/annotation.py +79 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/apm_config.py +128 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/asset.py +47 -0
- cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes}/charts_data.py +1 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/cognite_file.py +53 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/__init__.py +168 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_constraints.py +37 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_container.py +50 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_data_model.py +73 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_data_types.py +116 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_indexes.py +26 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_instance.py +154 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_references.py +86 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_space.py +26 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_view.py +143 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/data_modeling/_view_property.py +152 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/dataset.py +35 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/event.py +40 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/extraction_pipeline.py +59 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/filemetadata.py +52 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/function.py +53 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/function_schedule.py +65 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/graphql_data_model.py +40 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/group/__init__.py +187 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/group/_constants.py +2 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/group/acls.py +653 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/group/capability.py +56 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/group/group.py +63 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/group/scopes.py +166 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_destination.py +34 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_job.py +134 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_mapping.py +72 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/__init__.py +63 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_auth.py +63 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_base.py +26 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_certificate.py +20 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_eventhub.py +31 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_kafka.py +53 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_mqtt.py +36 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/hosted_extractor_source/_rest.py +49 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/identifiers.py +59 -0
- cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes}/infield.py +1 -1
- cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes}/instance_api.py +53 -14
- cognite_toolkit/_cdf_tk/client/resource_classes/label.py +27 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/legacy/__init__.py +0 -0
- cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/canvas.py +48 -15
- cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/charts.py +1 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/location_filter.py +84 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/raw.py +44 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/relationship.py +49 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/resource_view_mapping.py +38 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/__init__.py +37 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_capability.py +53 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_common.py +34 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_data_post_processing.py +49 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_frame.py +46 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_location.py +36 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_map.py +65 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/robotics/_robot.py +58 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/search_config.py +54 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/securitycategory.py +24 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/sequence.py +45 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/sequence_rows.py +56 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/simulator_model.py +50 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/streamlit_.py +71 -0
- cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes}/streams.py +9 -26
- cognite_toolkit/_cdf_tk/client/resource_classes/three_d.py +135 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/timeseries.py +52 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/transformation.py +140 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/workflow.py +27 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/workflow_trigger.py +63 -0
- cognite_toolkit/_cdf_tk/client/resource_classes/workflow_version.py +155 -0
- cognite_toolkit/_cdf_tk/client/testing.py +54 -17
- cognite_toolkit/_cdf_tk/commands/__init__.py +1 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/command.py +103 -108
- cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +29 -16
- cognite_toolkit/_cdf_tk/commands/_migrate/creators.py +1 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py +38 -6
- cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +262 -36
- cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py +1 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py +1 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/issues.py +39 -38
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +227 -8
- cognite_toolkit/_cdf_tk/commands/_profile.py +1 -1
- cognite_toolkit/_cdf_tk/commands/_purge.py +14 -16
- cognite_toolkit/_cdf_tk/commands/_upload.py +1 -1
- cognite_toolkit/_cdf_tk/commands/build_cmd.py +12 -2
- cognite_toolkit/_cdf_tk/commands/build_v2/__init__.py +0 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/_module_parser.py +138 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/_modules_parser.py +163 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py +228 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/build_parameters.py +71 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_modules.py +27 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_resource.py +22 -0
- cognite_toolkit/_cdf_tk/commands/dump_resource.py +4 -4
- cognite_toolkit/_cdf_tk/commands/pull.py +97 -2
- cognite_toolkit/_cdf_tk/commands/run.py +1 -1
- cognite_toolkit/_cdf_tk/cruds/__init__.py +16 -6
- cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +9 -5
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py +6 -2
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py +70 -89
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py +1 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py +1 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py +1 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py +34 -25
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py +50 -59
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py +14 -2
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/industrial_tool.py +1 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/location.py +4 -3
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py +1 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/raw.py +6 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py +5 -4
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/robotics.py +1 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/simulators.py +122 -0
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py +15 -31
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py +42 -47
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py +49 -14
- cognite_toolkit/_cdf_tk/data_classes/__init__.py +3 -0
- cognite_toolkit/_cdf_tk/data_classes/_issues.py +36 -0
- cognite_toolkit/_cdf_tk/data_classes/_module_directories.py +2 -1
- cognite_toolkit/_cdf_tk/data_classes/_module_toml.py +1 -0
- cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py +4 -0
- cognite_toolkit/_cdf_tk/feature_flags.py +8 -0
- cognite_toolkit/_cdf_tk/resource_classes/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/resource_classes/capabilities.py +6 -0
- cognite_toolkit/_cdf_tk/resource_classes/search_config.py +1 -1
- cognite_toolkit/_cdf_tk/resource_classes/simulator_model.py +17 -0
- cognite_toolkit/_cdf_tk/resource_classes/workflow_version.py +173 -5
- cognite_toolkit/_cdf_tk/storageio/_applications.py +53 -15
- cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +117 -107
- cognite_toolkit/_cdf_tk/storageio/_base.py +3 -1
- cognite_toolkit/_cdf_tk/storageio/_datapoints.py +7 -7
- cognite_toolkit/_cdf_tk/storageio/_file_content.py +65 -56
- cognite_toolkit/_cdf_tk/storageio/_instances.py +1 -1
- cognite_toolkit/_cdf_tk/storageio/_raw.py +1 -1
- cognite_toolkit/_cdf_tk/storageio/logger.py +162 -0
- cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py +10 -1
- cognite_toolkit/_cdf_tk/storageio/selectors/_three_d.py +34 -0
- cognite_toolkit/_cdf_tk/tk_warnings/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/tk_warnings/fileread.py +20 -0
- cognite_toolkit/_cdf_tk/utils/__init__.py +11 -1
- cognite_toolkit/_cdf_tk/utils/cdf.py +1 -1
- cognite_toolkit/_cdf_tk/utils/interactive_select.py +53 -4
- cognite_toolkit/_cdf_tk/utils/modules.py +7 -0
- cognite_toolkit/_cdf_tk/utils/pip_validator.py +96 -0
- cognite_toolkit/_cdf_tk/utils/useful_types.py +4 -7
- cognite_toolkit/_cdf_tk/utils/useful_types2.py +14 -0
- cognite_toolkit/_cdf_tk/validation.py +4 -0
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.30.dist-info → cognite_toolkit-0.7.51.dist-info}/METADATA +13 -3
- cognite_toolkit-0.7.51.dist-info/RECORD +445 -0
- {cognite_toolkit-0.7.30.dist-info → cognite_toolkit-0.7.51.dist-info}/WHEEL +2 -2
- cognite_toolkit/_cdf_tk/client/api/robotics/__init__.py +0 -3
- cognite_toolkit/_cdf_tk/client/data_classes/api_classes.py +0 -17
- cognite_toolkit/_cdf_tk/client/data_classes/base.py +0 -87
- cognite_toolkit/_cdf_tk/client/data_classes/three_d.py +0 -47
- cognite_toolkit/_cdf_tk/prototypes/import_app.py +0 -41
- cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py +0 -187
- cognite_toolkit/_cdf_tk/utils/http_client/_exception.py +0 -4
- cognite_toolkit-0.7.30.dist-info/RECORD +0 -316
- /cognite_toolkit/_cdf_tk/client/{data_classes → api/legacy}/__init__.py +0 -0
- /cognite_toolkit/_cdf_tk/client/api/{dml.py → legacy/dml.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/api/{fixed_transformations.py → legacy/fixed_transformations.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/api.py +0 -0
- /cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/utlis.py +0 -0
- /cognite_toolkit/_cdf_tk/{prototypes/commands → client/request_classes}/__init__.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes}/capabilities.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/apm_config_v1.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/extendable_cognite_file.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/extended_filemetadata.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/extended_filemetdata.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/extended_timeseries.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/functions.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/graphql_data_models.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/instances.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/location_filters.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/migration.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/pending_instances_ids.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/project.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/raw.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/robotics.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/search_config.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/sequences.py +0 -0
- /cognite_toolkit/_cdf_tk/client/{data_classes → resource_classes/legacy}/streamlit_.py +0 -0
- /cognite_toolkit/_cdf_tk/{prototypes/commands/import_.py → commands/_import_cmd.py} +0 -0
- {cognite_toolkit-0.7.30.dist-info → cognite_toolkit-0.7.51.dist-info}/entry_points.txt +0 -0
|
@@ -1,34 +1,45 @@
|
|
|
1
|
-
import
|
|
2
|
-
from typing import Any, ClassVar
|
|
1
|
+
from typing import Any, Literal
|
|
3
2
|
|
|
4
|
-
from cognite.client.data_classes.data_modeling import NodeId
|
|
5
|
-
from cognite.client.utils._identifier import InstanceId
|
|
6
|
-
from cognite.client.utils._text import to_camel_case
|
|
7
3
|
from pydantic import BaseModel, Field, field_serializer
|
|
4
|
+
from pydantic.alias_generators import to_camel
|
|
8
5
|
|
|
9
|
-
from cognite_toolkit._cdf_tk.client.
|
|
10
|
-
from cognite_toolkit._cdf_tk.
|
|
6
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeReference
|
|
7
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.migration import AssetCentricId
|
|
8
|
+
from cognite_toolkit._cdf_tk.storageio.logger import LogEntry
|
|
11
9
|
|
|
12
10
|
|
|
13
|
-
class
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class MigrationIssue(MigrationObject):
|
|
11
|
+
class MigrationIssue(LogEntry):
|
|
17
12
|
"""Represents an issue encountered during migration."""
|
|
18
13
|
|
|
19
|
-
type:
|
|
20
|
-
|
|
21
|
-
def dump(self) -> dict[str, JsonVal]:
|
|
22
|
-
# Dump json to ensure it is serializable
|
|
23
|
-
dumped = json.loads(self.model_dump_json(by_alias=True))
|
|
24
|
-
dumped["type"] = self.type
|
|
25
|
-
return dumped
|
|
14
|
+
type: str
|
|
26
15
|
|
|
27
16
|
@property
|
|
28
17
|
def has_issues(self) -> bool:
|
|
29
18
|
"""Check if there are any issues recorded in this MigrationIssue."""
|
|
30
19
|
return True
|
|
31
20
|
|
|
21
|
+
def dump(self) -> dict[str, Any]:
|
|
22
|
+
"""Serialize the MigrationIssue to a dictionary."""
|
|
23
|
+
return self.model_dump(by_alias=True)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ThreeDModelMigrationIssue(MigrationIssue):
|
|
27
|
+
"""Represents a 3D model migration issue encountered during migration.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
model_external_id (str): The external ID of the 3D model that could not be migrated.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
type: Literal["threeDModelMigration"] = "threeDModelMigration"
|
|
34
|
+
model_name: str
|
|
35
|
+
model_id: int
|
|
36
|
+
error_message: list[str] = Field(default_factory=list)
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def has_issues(self) -> bool:
|
|
40
|
+
"""Check if there are any issues recorded in this ThreeDModelMigrationIssue."""
|
|
41
|
+
return bool(self.error_message)
|
|
42
|
+
|
|
32
43
|
|
|
33
44
|
class ChartMigrationIssue(MigrationIssue):
|
|
34
45
|
"""Represents a chart migration issue encountered during migration.
|
|
@@ -37,7 +48,7 @@ class ChartMigrationIssue(MigrationIssue):
|
|
|
37
48
|
chart_external_id (str): The external ID of the chart that could not be migrated.
|
|
38
49
|
"""
|
|
39
50
|
|
|
40
|
-
type:
|
|
51
|
+
type: Literal["chartMigration"] = "chartMigration"
|
|
41
52
|
chart_external_id: str
|
|
42
53
|
missing_timeseries_ids: list[int] = Field(default_factory=list)
|
|
43
54
|
missing_timeseries_external_ids: list[str] = Field(default_factory=list)
|
|
@@ -52,7 +63,7 @@ class ChartMigrationIssue(MigrationIssue):
|
|
|
52
63
|
|
|
53
64
|
|
|
54
65
|
class CanvasMigrationIssue(MigrationIssue):
|
|
55
|
-
type:
|
|
66
|
+
type: Literal["canvasMigration"] = "canvasMigration"
|
|
56
67
|
canvas_external_id: str
|
|
57
68
|
canvas_name: str
|
|
58
69
|
missing_reference_ids: list[AssetCentricId] = Field(default_factory=list)
|
|
@@ -66,7 +77,7 @@ class CanvasMigrationIssue(MigrationIssue):
|
|
|
66
77
|
class ReadIssue(MigrationIssue):
|
|
67
78
|
"""Represents a read issue encountered during migration."""
|
|
68
79
|
|
|
69
|
-
|
|
80
|
+
...
|
|
70
81
|
|
|
71
82
|
|
|
72
83
|
class ReadFileIssue(ReadIssue):
|
|
@@ -77,7 +88,7 @@ class ReadFileIssue(ReadIssue):
|
|
|
77
88
|
error (str | None): An optional error message providing additional details about the read issue.
|
|
78
89
|
"""
|
|
79
90
|
|
|
80
|
-
type:
|
|
91
|
+
type: Literal["fileRead"] = "fileRead"
|
|
81
92
|
|
|
82
93
|
row_no: int
|
|
83
94
|
error: str | None = None
|
|
@@ -91,7 +102,7 @@ class ReadAPIIssue(ReadIssue):
|
|
|
91
102
|
error (str | None): An optional error message providing additional details about the read issue.
|
|
92
103
|
"""
|
|
93
104
|
|
|
94
|
-
type:
|
|
105
|
+
type: Literal["apiRead"] = "apiRead"
|
|
95
106
|
asset_centric_id: AssetCentricId
|
|
96
107
|
error: str | None = None
|
|
97
108
|
|
|
@@ -103,7 +114,7 @@ class ReadAPIIssue(ReadIssue):
|
|
|
103
114
|
}
|
|
104
115
|
|
|
105
116
|
|
|
106
|
-
class FailedConversion(
|
|
117
|
+
class FailedConversion(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
|
|
107
118
|
"""Represents a property that failed to convert during migration.
|
|
108
119
|
|
|
109
120
|
Attributes:
|
|
@@ -117,7 +128,7 @@ class FailedConversion(MigrationObject):
|
|
|
117
128
|
error: str
|
|
118
129
|
|
|
119
130
|
|
|
120
|
-
class InvalidPropertyDataType(
|
|
131
|
+
class InvalidPropertyDataType(BaseModel, alias_generator=to_camel, extra="ignore", populate_by_name=True):
|
|
121
132
|
"""Represents a property with an invalid type during migration.
|
|
122
133
|
|
|
123
134
|
Attributes:
|
|
@@ -142,9 +153,9 @@ class ConversionIssue(MigrationIssue):
|
|
|
142
153
|
failed_conversions (list[FailedConversion]): List of properties that failed to convert with reasons.
|
|
143
154
|
"""
|
|
144
155
|
|
|
145
|
-
type:
|
|
156
|
+
type: Literal["conversion"] = "conversion"
|
|
146
157
|
asset_centric_id: AssetCentricId
|
|
147
|
-
instance_id:
|
|
158
|
+
instance_id: NodeReference
|
|
148
159
|
missing_asset_centric_properties: list[str] = Field(default_factory=list)
|
|
149
160
|
missing_instance_properties: list[str] = Field(default_factory=list)
|
|
150
161
|
invalid_instance_property_types: list[InvalidPropertyDataType] = Field(default_factory=list)
|
|
@@ -163,10 +174,6 @@ class ConversionIssue(MigrationIssue):
|
|
|
163
174
|
or self.missing_instance_space
|
|
164
175
|
)
|
|
165
176
|
|
|
166
|
-
@field_serializer("instance_id")
|
|
167
|
-
def serialize_instance_id(self, instance_id: NodeId) -> dict[str, str]:
|
|
168
|
-
return instance_id.dump(include_instance_type=True)
|
|
169
|
-
|
|
170
177
|
@field_serializer("asset_centric_id")
|
|
171
178
|
def serialize_asset_centric_id(self, asset_centric_id: AssetCentricId) -> dict[str, Any]:
|
|
172
179
|
return {
|
|
@@ -179,16 +186,10 @@ class WriteIssue(MigrationIssue):
|
|
|
179
186
|
"""Represents a write issue encountered during migration.
|
|
180
187
|
|
|
181
188
|
Attributes:
|
|
182
|
-
instance_id (InstanceId): The InstanceId of the data model instance that could not be written.
|
|
183
189
|
status_code (int): The HTTP status code returned during the write operation.
|
|
184
190
|
message (str | None): An optional message providing additional details about the write issue.
|
|
185
191
|
"""
|
|
186
192
|
|
|
187
|
-
type:
|
|
188
|
-
instance_id: InstanceId
|
|
193
|
+
type: Literal["write"] = "write"
|
|
189
194
|
status_code: int
|
|
190
195
|
message: str | None = None
|
|
191
|
-
|
|
192
|
-
@field_serializer("instance_id")
|
|
193
|
-
def serialize_instance_id(self, instance_id: NodeId) -> dict[str, str]:
|
|
194
|
-
return instance_id.dump(include_instance_type=True)
|
|
@@ -1,29 +1,49 @@
|
|
|
1
1
|
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
2
|
-
from typing import ClassVar, cast
|
|
2
|
+
from typing import ClassVar, Literal, cast
|
|
3
3
|
|
|
4
4
|
from cognite.client.data_classes import Annotation
|
|
5
5
|
from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, NodeId
|
|
6
6
|
|
|
7
7
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
8
|
-
from cognite_toolkit._cdf_tk.client.
|
|
8
|
+
from cognite_toolkit._cdf_tk.client.http_client import (
|
|
9
|
+
FailedResponse,
|
|
10
|
+
HTTPClient,
|
|
11
|
+
HTTPMessage,
|
|
12
|
+
ItemsRequest,
|
|
13
|
+
RequestMessage2,
|
|
14
|
+
SuccessResponseItems,
|
|
15
|
+
ToolkitAPIError,
|
|
16
|
+
)
|
|
17
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.pending_instances_ids import PendingInstanceId
|
|
18
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.three_d import (
|
|
19
|
+
AssetMappingClassicResponse,
|
|
20
|
+
AssetMappingDMRequest,
|
|
21
|
+
ThreeDModelResponse,
|
|
22
|
+
)
|
|
23
|
+
from cognite_toolkit._cdf_tk.commands._migrate.data_classes import ThreeDMigrationRequest
|
|
9
24
|
from cognite_toolkit._cdf_tk.constants import MISSING_EXTERNAL_ID, MISSING_INSTANCE_SPACE
|
|
10
25
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError, ToolkitValueError
|
|
11
26
|
from cognite_toolkit._cdf_tk.storageio import (
|
|
12
27
|
AnnotationIO,
|
|
13
28
|
HierarchyIO,
|
|
14
29
|
InstanceIO,
|
|
30
|
+
T_Selector,
|
|
15
31
|
UploadableStorageIO,
|
|
16
32
|
)
|
|
17
33
|
from cognite_toolkit._cdf_tk.storageio._base import Page, UploadItem
|
|
34
|
+
from cognite_toolkit._cdf_tk.storageio.selectors import (
|
|
35
|
+
ThreeDModelFilteredSelector,
|
|
36
|
+
ThreeDModelIdSelector,
|
|
37
|
+
ThreeDSelector,
|
|
38
|
+
)
|
|
18
39
|
from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
|
|
19
40
|
from cognite_toolkit._cdf_tk.utils.collection import chunker_sequence
|
|
20
|
-
from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, HTTPMessage, ItemsRequest, SuccessResponseItems
|
|
21
41
|
from cognite_toolkit._cdf_tk.utils.useful_types import (
|
|
22
42
|
AssetCentricKindExtended,
|
|
23
43
|
AssetCentricType,
|
|
24
44
|
JsonVal,
|
|
25
|
-
T_AssetCentricResource,
|
|
26
45
|
)
|
|
46
|
+
from cognite_toolkit._cdf_tk.utils.useful_types2 import T_AssetCentricResource
|
|
27
47
|
|
|
28
48
|
from .data_classes import (
|
|
29
49
|
AnnotationMapping,
|
|
@@ -58,7 +78,7 @@ class AssetCentricMigrationIO(
|
|
|
58
78
|
self.skip_linking = skip_linking
|
|
59
79
|
|
|
60
80
|
def as_id(self, item: AssetCentricMapping) -> str:
|
|
61
|
-
return
|
|
81
|
+
return str(item.mapping.as_asset_centric_id())
|
|
62
82
|
|
|
63
83
|
def stream_data(self, selector: AssetCentricMigrationSelector, limit: int | None = None) -> Iterator[Page]:
|
|
64
84
|
if isinstance(selector, MigrationCSVFileSelector):
|
|
@@ -161,7 +181,7 @@ class AssetCentricMigrationIO(
|
|
|
161
181
|
|
|
162
182
|
pending_instance_id_endpoint = self.PENDING_INSTANCE_ID_ENDPOINT_BY_KIND[selector.kind]
|
|
163
183
|
results: list[HTTPMessage] = []
|
|
164
|
-
to_upload = self.link_asset_centric(data_chunk, http_client,
|
|
184
|
+
to_upload = self.link_asset_centric(data_chunk, http_client, pending_instance_id_endpoint)
|
|
165
185
|
if to_upload:
|
|
166
186
|
results.extend(list(super().upload_items(to_upload, http_client, None)))
|
|
167
187
|
return results
|
|
@@ -171,7 +191,6 @@ class AssetCentricMigrationIO(
|
|
|
171
191
|
cls,
|
|
172
192
|
data_chunk: Sequence[UploadItem[InstanceApply]],
|
|
173
193
|
http_client: HTTPClient,
|
|
174
|
-
results: list[HTTPMessage],
|
|
175
194
|
pending_instance_id_endpoint: str,
|
|
176
195
|
) -> Sequence[UploadItem[InstanceApply]]:
|
|
177
196
|
"""Links asset-centric resources to their (uncreated) instances using the pending-instance-ids endpoint."""
|
|
@@ -192,7 +211,6 @@ class AssetCentricMigrationIO(
|
|
|
192
211
|
for res in batch_results:
|
|
193
212
|
if isinstance(res, SuccessResponseItems):
|
|
194
213
|
successful_linked.update(res.ids)
|
|
195
|
-
results.extend(batch_results)
|
|
196
214
|
to_upload = [item for item in data_chunk if item.source_id in successful_linked]
|
|
197
215
|
return to_upload
|
|
198
216
|
|
|
@@ -348,3 +366,204 @@ class AnnotationMigrationIO(
|
|
|
348
366
|
selector: AssetCentricMigrationSelector | None = None,
|
|
349
367
|
) -> list[dict[str, JsonVal]]:
|
|
350
368
|
raise NotImplementedError("Serializing Annotation Migrations to JSON is not supported.")
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
class ThreeDMigrationIO(UploadableStorageIO[ThreeDSelector, ThreeDModelResponse, ThreeDMigrationRequest]):
|
|
372
|
+
"""IO class for downloading and migrating 3D models.
|
|
373
|
+
|
|
374
|
+
Args:
|
|
375
|
+
client: The ToolkitClient to use for CDF interactions.
|
|
376
|
+
data_model_type: The type of 3D data model to download. Either "classic" or "DM".
|
|
377
|
+
|
|
378
|
+
"""
|
|
379
|
+
|
|
380
|
+
KIND = "3DMigration"
|
|
381
|
+
SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
|
|
382
|
+
SUPPORTED_COMPRESSIONS = frozenset({".gz"})
|
|
383
|
+
SUPPORTED_READ_FORMATS = frozenset({".ndjson"})
|
|
384
|
+
DOWNLOAD_LIMIT = 1000
|
|
385
|
+
CHUNK_SIZE = 1
|
|
386
|
+
UPLOAD_ENDPOINT = "/3d/migrate/models"
|
|
387
|
+
REVISION_ENDPOINT = "/3d/migrate/revisions"
|
|
388
|
+
|
|
389
|
+
def __init__(self, client: ToolkitClient, data_model_type: Literal["classic", "data modeling"] = "classic") -> None:
|
|
390
|
+
super().__init__(client)
|
|
391
|
+
self.data_model_type = data_model_type
|
|
392
|
+
|
|
393
|
+
def as_id(self, item: ThreeDModelResponse) -> str:
|
|
394
|
+
return item.name
|
|
395
|
+
|
|
396
|
+
def _is_selected(self, item: ThreeDModelResponse, included_models: set[int] | None) -> bool:
|
|
397
|
+
return self._is_correct_type(item) and (included_models is None or item.id in included_models)
|
|
398
|
+
|
|
399
|
+
def _is_correct_type(self, item: ThreeDModelResponse) -> bool:
|
|
400
|
+
if self.data_model_type == "classic":
|
|
401
|
+
return item.space is None
|
|
402
|
+
else:
|
|
403
|
+
return item.space is not None
|
|
404
|
+
|
|
405
|
+
def stream_data(self, selector: ThreeDSelector, limit: int | None = None) -> Iterable[Page[ThreeDModelResponse]]:
|
|
406
|
+
published: bool | None = None
|
|
407
|
+
if isinstance(selector, ThreeDModelFilteredSelector):
|
|
408
|
+
published = selector.published
|
|
409
|
+
included_models: set[int] | None = None
|
|
410
|
+
if isinstance(selector, ThreeDModelIdSelector):
|
|
411
|
+
included_models = set(selector.ids)
|
|
412
|
+
cursor: str | None = None
|
|
413
|
+
total = 0
|
|
414
|
+
while True:
|
|
415
|
+
request_limit = min(self.DOWNLOAD_LIMIT, limit - total) if limit is not None else self.DOWNLOAD_LIMIT
|
|
416
|
+
response = self.client.tool.three_d.models_classic.paginate(
|
|
417
|
+
published=published, include_revision_info=True, limit=request_limit, cursor=cursor
|
|
418
|
+
)
|
|
419
|
+
items = [item for item in response.items if self._is_selected(item, included_models)]
|
|
420
|
+
total += len(items)
|
|
421
|
+
if items:
|
|
422
|
+
yield Page(worker_id="main", items=items, next_cursor=response.next_cursor)
|
|
423
|
+
if response.next_cursor is None:
|
|
424
|
+
break
|
|
425
|
+
cursor = response.next_cursor
|
|
426
|
+
|
|
427
|
+
def count(self, selector: ThreeDSelector) -> int | None:
|
|
428
|
+
# There is no efficient way to count 3D models in CDF.
|
|
429
|
+
return None
|
|
430
|
+
|
|
431
|
+
def data_to_json_chunk(
|
|
432
|
+
self, data_chunk: Sequence[ThreeDModelResponse], selector: ThreeDSelector | None = None
|
|
433
|
+
) -> list[dict[str, JsonVal]]:
|
|
434
|
+
raise NotImplementedError("Deserializing Annotation Migrations from JSON is not supported.")
|
|
435
|
+
|
|
436
|
+
def json_to_resource(self, item_json: dict[str, JsonVal]) -> ThreeDMigrationRequest:
|
|
437
|
+
raise NotImplementedError("Deserializing ThreeD Migrations from JSON is not supported.")
|
|
438
|
+
|
|
439
|
+
def upload_items(
|
|
440
|
+
self,
|
|
441
|
+
data_chunk: Sequence[UploadItem[ThreeDMigrationRequest]],
|
|
442
|
+
http_client: HTTPClient,
|
|
443
|
+
selector: ThreeDSelector | None = None,
|
|
444
|
+
) -> Sequence[HTTPMessage]:
|
|
445
|
+
"""Migrate 3D models by uploading them to the migrate/models endpoint."""
|
|
446
|
+
if len(data_chunk) > self.CHUNK_SIZE:
|
|
447
|
+
raise RuntimeError(f"Uploading more than {self.CHUNK_SIZE} 3D models at a time is not supported.")
|
|
448
|
+
|
|
449
|
+
results: list[HTTPMessage] = []
|
|
450
|
+
responses = http_client.request_with_retries(
|
|
451
|
+
message=ItemsRequest(
|
|
452
|
+
endpoint_url=self.client.config.create_api_url(self.UPLOAD_ENDPOINT),
|
|
453
|
+
method="POST",
|
|
454
|
+
items=list(data_chunk),
|
|
455
|
+
)
|
|
456
|
+
)
|
|
457
|
+
if (
|
|
458
|
+
failed_response := next((res for res in responses if isinstance(res, FailedResponse)), None)
|
|
459
|
+
) and failed_response.status_code == 400:
|
|
460
|
+
raise ToolkitAPIError("3D model migration failed. You need to enable the 3D migration alpha feature flag.")
|
|
461
|
+
|
|
462
|
+
results.extend(responses)
|
|
463
|
+
success_ids = {id for res in responses if isinstance(res, SuccessResponseItems) for id in res.ids}
|
|
464
|
+
for data in data_chunk:
|
|
465
|
+
if data.source_id not in success_ids:
|
|
466
|
+
continue
|
|
467
|
+
revision = http_client.request_single_retries(
|
|
468
|
+
message=RequestMessage2(
|
|
469
|
+
endpoint_url=self.client.config.create_api_url(self.REVISION_ENDPOINT),
|
|
470
|
+
method="POST",
|
|
471
|
+
body_content={"items": [data.item.revision.dump(camel_case=True)]},
|
|
472
|
+
)
|
|
473
|
+
)
|
|
474
|
+
results.append(revision.as_item_response(data.source_id))
|
|
475
|
+
return results
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
class ThreeDAssetMappingMigrationIO(
|
|
479
|
+
UploadableStorageIO[ThreeDSelector, AssetMappingClassicResponse, AssetMappingDMRequest]
|
|
480
|
+
):
|
|
481
|
+
KIND = "3DMigrationAssetMapping"
|
|
482
|
+
SUPPORTED_DOWNLOAD_FORMATS = frozenset({".ndjson"})
|
|
483
|
+
SUPPORTED_COMPRESSIONS = frozenset({".gz"})
|
|
484
|
+
SUPPORTED_READ_FORMATS = frozenset({".ndjson"})
|
|
485
|
+
DOWNLOAD_LIMIT = 1000
|
|
486
|
+
CHUNK_SIZE = 100
|
|
487
|
+
UPLOAD_ENDPOINT = "/3d/models/{modelId}/revisions/{revisionId}/mappings"
|
|
488
|
+
|
|
489
|
+
def __init__(self, client: ToolkitClient, object_3D_space: str, cad_node_space: str) -> None:
|
|
490
|
+
super().__init__(client)
|
|
491
|
+
self.object_3D_space = object_3D_space
|
|
492
|
+
self.cad_node_space = cad_node_space
|
|
493
|
+
# We can only migrate asset mappings for 3D models that are already migrated to data modeling.
|
|
494
|
+
self._3D_io = ThreeDMigrationIO(client, data_model_type="data modeling")
|
|
495
|
+
|
|
496
|
+
def as_id(self, item: AssetMappingClassicResponse) -> str:
|
|
497
|
+
return f"AssetMapping_{item.model_id!s}_{item.revision_id!s}_{item.asset_id!s}"
|
|
498
|
+
|
|
499
|
+
def stream_data(
|
|
500
|
+
self, selector: ThreeDSelector, limit: int | None = None
|
|
501
|
+
) -> Iterable[Page[AssetMappingClassicResponse]]:
|
|
502
|
+
total = 0
|
|
503
|
+
for three_d_page in self._3D_io.stream_data(selector, None):
|
|
504
|
+
for model in three_d_page.items:
|
|
505
|
+
if model.last_revision_info is None or model.last_revision_info.revision_id is None:
|
|
506
|
+
# No revisions, so no asset mappings to
|
|
507
|
+
continue
|
|
508
|
+
cursor: str | None = None
|
|
509
|
+
while True:
|
|
510
|
+
request_limit = (
|
|
511
|
+
min(self.DOWNLOAD_LIMIT, limit - total) if limit is not None else self.DOWNLOAD_LIMIT
|
|
512
|
+
)
|
|
513
|
+
if limit is not None and total >= limit:
|
|
514
|
+
return
|
|
515
|
+
response = self.client.tool.three_d.asset_mappings_classic.paginate(
|
|
516
|
+
model_id=model.id,
|
|
517
|
+
revision_id=model.last_revision_info.revision_id,
|
|
518
|
+
cursor=cursor,
|
|
519
|
+
limit=request_limit,
|
|
520
|
+
)
|
|
521
|
+
items = response.items
|
|
522
|
+
total += len(items)
|
|
523
|
+
if items:
|
|
524
|
+
yield Page(worker_id="main", items=items, next_cursor=response.next_cursor)
|
|
525
|
+
if response.next_cursor is None:
|
|
526
|
+
break
|
|
527
|
+
cursor = response.next_cursor
|
|
528
|
+
|
|
529
|
+
def count(self, selector: ThreeDSelector) -> int | None:
|
|
530
|
+
# There is no efficient way to count 3D asset mappings in CDF.
|
|
531
|
+
return None
|
|
532
|
+
|
|
533
|
+
def upload_items(
|
|
534
|
+
self,
|
|
535
|
+
data_chunk: Sequence[UploadItem[AssetMappingDMRequest]],
|
|
536
|
+
http_client: HTTPClient,
|
|
537
|
+
selector: T_Selector | None = None,
|
|
538
|
+
) -> Sequence[HTTPMessage]:
|
|
539
|
+
"""Migrate 3D asset mappings by uploading them to the migrate/asset-mappings endpoint."""
|
|
540
|
+
if not data_chunk:
|
|
541
|
+
return []
|
|
542
|
+
# Assume all items in the chunk belong to the same model and revision, they should
|
|
543
|
+
# if the .stream_data method is used for downloading.
|
|
544
|
+
first = data_chunk[0]
|
|
545
|
+
model_id = first.item.model_id
|
|
546
|
+
revision_id = first.item.revision_id
|
|
547
|
+
endpoint = self.UPLOAD_ENDPOINT.format(modelId=model_id, revisionId=revision_id)
|
|
548
|
+
responses = http_client.request_with_retries(
|
|
549
|
+
ItemsRequest(
|
|
550
|
+
endpoint_url=self.client.config.create_api_url(endpoint),
|
|
551
|
+
method="POST",
|
|
552
|
+
items=list(data_chunk),
|
|
553
|
+
extra_body_fields={
|
|
554
|
+
"dmsContextualizationConfig": {
|
|
555
|
+
"object3DSpace": self.object_3D_space,
|
|
556
|
+
"cadNodeSpace": self.cad_node_space,
|
|
557
|
+
}
|
|
558
|
+
},
|
|
559
|
+
)
|
|
560
|
+
)
|
|
561
|
+
return responses
|
|
562
|
+
|
|
563
|
+
def json_to_resource(self, item_json: dict[str, JsonVal]) -> AssetMappingDMRequest:
|
|
564
|
+
raise NotImplementedError("Deserializing 3D Asset Mappings from JSON is not supported.")
|
|
565
|
+
|
|
566
|
+
def data_to_json_chunk(
|
|
567
|
+
self, data_chunk: Sequence[AssetMappingClassicResponse], selector: ThreeDSelector | None = None
|
|
568
|
+
) -> list[dict[str, JsonVal]]:
|
|
569
|
+
raise NotImplementedError("Serializing 3D Asset Mappings to JSON is not supported.")
|
|
@@ -21,7 +21,7 @@ from rich.spinner import Spinner
|
|
|
21
21
|
from rich.table import Table
|
|
22
22
|
|
|
23
23
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
24
|
-
from cognite_toolkit._cdf_tk.client.
|
|
24
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawProfileResults, RawTable
|
|
25
25
|
from cognite_toolkit._cdf_tk.constants import MAX_ROW_ITERATION_RUN_QUERY
|
|
26
26
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingDependencyError, ToolkitThrottledError, ToolkitValueError
|
|
27
27
|
from cognite_toolkit._cdf_tk.utils.aggregators import (
|
|
@@ -16,6 +16,14 @@ from rich.console import Console
|
|
|
16
16
|
from rich.panel import Panel
|
|
17
17
|
|
|
18
18
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
19
|
+
from cognite_toolkit._cdf_tk.client.http_client import (
|
|
20
|
+
HTTPClient,
|
|
21
|
+
ItemsRequest,
|
|
22
|
+
ItemsRequest2,
|
|
23
|
+
ItemsSuccessResponse2,
|
|
24
|
+
SuccessResponseItems,
|
|
25
|
+
)
|
|
26
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.instance_api import TypedInstanceIdentifier
|
|
19
27
|
from cognite_toolkit._cdf_tk.cruds import (
|
|
20
28
|
AssetCRUD,
|
|
21
29
|
ContainerCRUD,
|
|
@@ -58,13 +66,6 @@ from cognite_toolkit._cdf_tk.utils.aggregators import (
|
|
|
58
66
|
SequenceAggregator,
|
|
59
67
|
TimeSeriesAggregator,
|
|
60
68
|
)
|
|
61
|
-
from cognite_toolkit._cdf_tk.utils.http_client import (
|
|
62
|
-
FailedRequestItems,
|
|
63
|
-
FailedResponseItems,
|
|
64
|
-
HTTPClient,
|
|
65
|
-
ItemsRequest,
|
|
66
|
-
SuccessResponseItems,
|
|
67
|
-
)
|
|
68
69
|
from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
|
|
69
70
|
from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
|
|
70
71
|
from cognite_toolkit._cdf_tk.utils.validate_access import ValidateAccess
|
|
@@ -714,21 +715,18 @@ class PurgeCommand(ToolkitCommand):
|
|
|
714
715
|
results.deleted += len(items)
|
|
715
716
|
return
|
|
716
717
|
|
|
717
|
-
responses = delete_client.
|
|
718
|
-
|
|
719
|
-
delete_client.config.create_api_url("/models/instances/delete"),
|
|
718
|
+
responses = delete_client.request_items_retries(
|
|
719
|
+
ItemsRequest2(
|
|
720
|
+
endpoint_url=delete_client.config.create_api_url("/models/instances/delete"),
|
|
720
721
|
method="POST",
|
|
721
|
-
|
|
722
|
-
items=[DeleteItem(item=item, as_id_fun=InstanceId.load) for item in items], # type: ignore[arg-type]
|
|
722
|
+
items=[TypedInstanceIdentifier._load(item) for item in items],
|
|
723
723
|
)
|
|
724
724
|
)
|
|
725
725
|
for response in responses:
|
|
726
|
-
if isinstance(response,
|
|
726
|
+
if isinstance(response, ItemsSuccessResponse2):
|
|
727
727
|
results.deleted += len(response.ids)
|
|
728
|
-
elif isinstance(response, FailedResponseItems | FailedRequestItems):
|
|
729
|
-
results.failed += len(response.ids)
|
|
730
728
|
else:
|
|
731
|
-
results.failed += len(
|
|
729
|
+
results.failed += len(response.ids)
|
|
732
730
|
|
|
733
731
|
@staticmethod
|
|
734
732
|
def _unlink_timeseries(
|
|
@@ -9,6 +9,7 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
9
9
|
from rich.console import Console
|
|
10
10
|
|
|
11
11
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
12
|
+
from cognite_toolkit._cdf_tk.client.http_client import HTTPClient, ItemMessage, SuccessResponseItems
|
|
12
13
|
from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_SUFFIX, DATA_RESOURCE_DIR
|
|
13
14
|
from cognite_toolkit._cdf_tk.cruds import ViewCRUD
|
|
14
15
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
|
@@ -24,7 +25,6 @@ from cognite_toolkit._cdf_tk.storageio.selectors._instances import InstanceSpace
|
|
|
24
25
|
from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, MediumSeverityWarning, ToolkitWarning
|
|
25
26
|
from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
|
|
26
27
|
from cognite_toolkit._cdf_tk.utils.fileio import MultiFileReader
|
|
27
|
-
from cognite_toolkit._cdf_tk.utils.http_client import HTTPClient, ItemMessage, SuccessResponseItems
|
|
28
28
|
from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
|
|
29
29
|
from cognite_toolkit._cdf_tk.utils.progress_tracker import ProgressTracker
|
|
30
30
|
from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
|
|
@@ -11,10 +11,10 @@ from rich import print
|
|
|
11
11
|
from rich.panel import Panel
|
|
12
12
|
from rich.progress import track
|
|
13
13
|
|
|
14
|
-
from cognite_toolkit._cdf_tk.builders import Builder, create_builder
|
|
14
|
+
from cognite_toolkit._cdf_tk.builders import Builder, FunctionBuilder, create_builder
|
|
15
15
|
from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
|
|
16
16
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
17
|
-
from cognite_toolkit._cdf_tk.client.
|
|
17
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawDatabase
|
|
18
18
|
from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
|
|
19
19
|
from cognite_toolkit._cdf_tk.constants import (
|
|
20
20
|
_RUNNING_IN_BROWSER,
|
|
@@ -33,6 +33,7 @@ from cognite_toolkit._cdf_tk.cruds import (
|
|
|
33
33
|
DataSetsCRUD,
|
|
34
34
|
ExtractionPipelineConfigCRUD,
|
|
35
35
|
FileCRUD,
|
|
36
|
+
FunctionCRUD,
|
|
36
37
|
LocationFilterCRUD,
|
|
37
38
|
NodeCRUD,
|
|
38
39
|
RawDatabaseCRUD,
|
|
@@ -415,6 +416,15 @@ class BuildCommand(ToolkitCommand):
|
|
|
415
416
|
|
|
416
417
|
build_resources_by_folder[resource_name].extend(built_resources)
|
|
417
418
|
|
|
419
|
+
# Collect validation metrics from FunctionBuilder
|
|
420
|
+
if resource_name == FunctionCRUD.folder_name and isinstance(builder, FunctionBuilder):
|
|
421
|
+
self._additional_tracking_info.function_validation_count += builder.validation_count
|
|
422
|
+
self._additional_tracking_info.function_validation_failures += builder.validation_failures
|
|
423
|
+
self._additional_tracking_info.function_validation_credential_errors += (
|
|
424
|
+
builder.validation_credential_errors
|
|
425
|
+
)
|
|
426
|
+
self._additional_tracking_info.function_validation_time_ms += builder.validation_time_ms
|
|
427
|
+
|
|
418
428
|
return build_resources_by_folder
|
|
419
429
|
|
|
420
430
|
def _get_builder(self, build_dir: Path, resource_name: str) -> Builder:
|
|
File without changes
|