cognite-toolkit 0.7.55__py3-none-any.whl → 0.7.57__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/apps/_auth_app.py +2 -2
- cognite_toolkit/_cdf_tk/apps/_core_app.py +4 -4
- cognite_toolkit/_cdf_tk/apps/_dev_app.py +10 -1
- cognite_toolkit/_cdf_tk/apps/_download_app.py +13 -12
- cognite_toolkit/_cdf_tk/apps/_dump_app.py +13 -13
- cognite_toolkit/_cdf_tk/apps/_landing_app.py +10 -1
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +13 -13
- cognite_toolkit/_cdf_tk/apps/_modules_app.py +29 -5
- cognite_toolkit/_cdf_tk/apps/_profile_app.py +4 -4
- cognite_toolkit/_cdf_tk/apps/_purge.py +4 -5
- cognite_toolkit/_cdf_tk/apps/_repo_app.py +9 -2
- cognite_toolkit/_cdf_tk/apps/_run.py +5 -4
- cognite_toolkit/_cdf_tk/apps/_upload_app.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/agents.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/annotations.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/assets.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/containers.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/data_models.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/datasets.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/events.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/extraction_pipelines.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/filemetadata.py +6 -6
- cognite_toolkit/_cdf_tk/client/api/function_schedules.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/functions.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/graphql_data_models.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/groups.py +5 -7
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_destinations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_jobs.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_mappings.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/hosted_extractor_sources.py +4 -4
- cognite_toolkit/_cdf_tk/client/api/infield.py +8 -8
- cognite_toolkit/_cdf_tk/client/api/instances.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/labels.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/legacy/extended_functions.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/location_filters.py +8 -8
- cognite_toolkit/_cdf_tk/client/api/project.py +14 -2
- cognite_toolkit/_cdf_tk/client/api/raw.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/relationships.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/robotics_capabilities.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_data_postprocessing.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_frames.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_locations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_maps.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_robots.py +3 -5
- cognite_toolkit/_cdf_tk/client/api/search_config.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/security_categories.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/sequences.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/simulator_models.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/spaces.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/streams.py +6 -6
- cognite_toolkit/_cdf_tk/client/api/three_d.py +5 -5
- cognite_toolkit/_cdf_tk/client/api/timeseries.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/transformations.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/views.py +2 -4
- cognite_toolkit/_cdf_tk/client/api/workflow_triggers.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/workflow_versions.py +3 -3
- cognite_toolkit/_cdf_tk/client/api/workflows.py +3 -3
- cognite_toolkit/_cdf_tk/client/cdf_client/api.py +11 -11
- cognite_toolkit/_cdf_tk/client/http_client/__init__.py +13 -51
- cognite_toolkit/_cdf_tk/client/http_client/_client.py +48 -209
- cognite_toolkit/_cdf_tk/client/http_client/_data_classes.py +106 -383
- cognite_toolkit/_cdf_tk/client/http_client/_item_classes.py +16 -16
- cognite_toolkit/_cdf_tk/client/resource_classes/filemetadata.py +7 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/project.py +30 -0
- cognite_toolkit/_cdf_tk/commands/_base.py +18 -1
- cognite_toolkit/_cdf_tk/commands/_import_cmd.py +3 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/command.py +8 -8
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +25 -24
- cognite_toolkit/_cdf_tk/commands/_profile.py +10 -5
- cognite_toolkit/_cdf_tk/commands/_purge.py +30 -35
- cognite_toolkit/_cdf_tk/commands/_upload.py +4 -6
- cognite_toolkit/_cdf_tk/commands/build_cmd.py +2 -1
- cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py +8 -2
- cognite_toolkit/_cdf_tk/commands/deploy.py +8 -2
- cognite_toolkit/_cdf_tk/commands/init.py +9 -2
- cognite_toolkit/_cdf_tk/commands/modules.py +3 -1
- cognite_toolkit/_cdf_tk/commands/pull.py +8 -2
- cognite_toolkit/_cdf_tk/commands/repo.py +3 -1
- cognite_toolkit/_cdf_tk/commands/resources.py +0 -3
- cognite_toolkit/_cdf_tk/data_classes/_tracking_info.py +1 -0
- cognite_toolkit/_cdf_tk/protocols.py +3 -1
- cognite_toolkit/_cdf_tk/storageio/_applications.py +9 -9
- cognite_toolkit/_cdf_tk/storageio/_base.py +15 -10
- cognite_toolkit/_cdf_tk/storageio/_datapoints.py +36 -24
- cognite_toolkit/_cdf_tk/storageio/_file_content.py +47 -43
- cognite_toolkit/_cdf_tk/storageio/_raw.py +5 -4
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.55.dist-info → cognite_toolkit-0.7.57.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.7.55.dist-info → cognite_toolkit-0.7.57.dist-info}/RECORD +94 -94
- cognite_toolkit/_cdf_tk/client/http_client/_data_classes2.py +0 -151
- {cognite_toolkit-0.7.55.dist-info → cognite_toolkit-0.7.57.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.7.55.dist-info → cognite_toolkit-0.7.57.dist-info}/entry_points.txt +0 -0
|
@@ -5,6 +5,8 @@ from typing import Any
|
|
|
5
5
|
from rich import print
|
|
6
6
|
from rich.console import Console
|
|
7
7
|
|
|
8
|
+
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
9
|
+
from cognite_toolkit._cdf_tk.client.http_client import ToolkitAPIError
|
|
8
10
|
from cognite_toolkit._cdf_tk.data_classes import CommandTrackingInfo
|
|
9
11
|
from cognite_toolkit._cdf_tk.tk_warnings import (
|
|
10
12
|
ToolkitWarning,
|
|
@@ -16,12 +18,27 @@ _HAS_PRINTED_COLLECT_MESSAGE = False
|
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
class ToolkitCommand:
|
|
19
|
-
def __init__(
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
print_warning: bool = True,
|
|
24
|
+
skip_tracking: bool = False,
|
|
25
|
+
silent: bool = False,
|
|
26
|
+
client: ToolkitClient | None = None,
|
|
27
|
+
):
|
|
20
28
|
self._print_warning = print_warning
|
|
21
29
|
self.silent = silent
|
|
22
30
|
self.warning_list = WarningList[ToolkitWarning]()
|
|
23
31
|
self.tracker = Tracker(skip_tracking)
|
|
24
32
|
self._additional_tracking_info = CommandTrackingInfo()
|
|
33
|
+
if client is not None:
|
|
34
|
+
self._additional_tracking_info.cluster = client.config.cdf_cluster
|
|
35
|
+
try:
|
|
36
|
+
result = client.project.organization()
|
|
37
|
+
except (ToolkitAPIError, ValueError):
|
|
38
|
+
self._additional_tracking_info.project = client.config.project
|
|
39
|
+
else:
|
|
40
|
+
self._additional_tracking_info.organization = result.organization
|
|
41
|
+
self._additional_tracking_info.project = result.name
|
|
25
42
|
|
|
26
43
|
@property
|
|
27
44
|
def print_warning(self) -> bool:
|
|
@@ -21,8 +21,10 @@ class ImportTransformationCLI(ToolkitCommand):
|
|
|
21
21
|
get_client: Callable[[], ToolkitClient] | None = None,
|
|
22
22
|
print_warning: bool = True,
|
|
23
23
|
skip_tracking: bool = False,
|
|
24
|
+
silent: bool = False,
|
|
25
|
+
client: ToolkitClient | None = None,
|
|
24
26
|
):
|
|
25
|
-
super().__init__(print_warning, skip_tracking)
|
|
27
|
+
super().__init__(print_warning, skip_tracking, silent, get_client() if get_client is not None else client)
|
|
26
28
|
self._dataset_external_id_by_id: dict[int, str] = {}
|
|
27
29
|
# We only initialize the client if we need to look up dataset ids.
|
|
28
30
|
self._client: ToolkitClient | None = None
|
|
@@ -9,10 +9,10 @@ from rich.table import Table
|
|
|
9
9
|
|
|
10
10
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
11
11
|
from cognite_toolkit._cdf_tk.client.http_client import (
|
|
12
|
-
FailedRequestItems,
|
|
13
|
-
FailedResponseItems,
|
|
14
12
|
HTTPClient,
|
|
15
|
-
|
|
13
|
+
ItemsFailedRequest,
|
|
14
|
+
ItemsFailedResponse,
|
|
15
|
+
ItemsSuccessResponse,
|
|
16
16
|
)
|
|
17
17
|
from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
|
|
18
18
|
from cognite_toolkit._cdf_tk.commands._migrate.creators import MigrationCreator
|
|
@@ -187,20 +187,20 @@ class MigrationCommand(ToolkitCommand):
|
|
|
187
187
|
# Todo: Move logging into the UploadableStorageIO class
|
|
188
188
|
issues: list[WriteIssue] = []
|
|
189
189
|
for item in responses:
|
|
190
|
-
if isinstance(item,
|
|
190
|
+
if isinstance(item, ItemsSuccessResponse):
|
|
191
191
|
target.logger.tracker.finalize_item(item.ids, "success")
|
|
192
192
|
continue
|
|
193
|
-
if isinstance(item,
|
|
193
|
+
if isinstance(item, ItemsFailedResponse):
|
|
194
194
|
error = item.error
|
|
195
195
|
for id_ in item.ids:
|
|
196
196
|
issue = WriteIssue(id=str(id_), status_code=error.code, message=error.message)
|
|
197
197
|
issues.append(issue)
|
|
198
|
-
elif isinstance(item,
|
|
198
|
+
elif isinstance(item, ItemsFailedRequest):
|
|
199
199
|
for id_ in item.ids:
|
|
200
|
-
issue = WriteIssue(id=str(id_), status_code=0, message=item.
|
|
200
|
+
issue = WriteIssue(id=str(id_), status_code=0, message=item.error_message)
|
|
201
201
|
issues.append(issue)
|
|
202
202
|
|
|
203
|
-
if isinstance(item,
|
|
203
|
+
if isinstance(item, ItemsFailedResponse | ItemsFailedRequest):
|
|
204
204
|
target.logger.tracker.finalize_item(item.ids, "failure")
|
|
205
205
|
if issues:
|
|
206
206
|
target.logger.log(issues)
|
|
@@ -6,14 +6,16 @@ from cognite.client.data_classes.data_modeling import EdgeId, InstanceApply, Nod
|
|
|
6
6
|
|
|
7
7
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
8
8
|
from cognite_toolkit._cdf_tk.client.http_client import (
|
|
9
|
-
FailedResponse,
|
|
10
9
|
HTTPClient,
|
|
11
|
-
|
|
12
|
-
ItemsRequest,
|
|
13
|
-
RequestMessage2,
|
|
14
|
-
SuccessResponseItems,
|
|
10
|
+
RequestMessage,
|
|
15
11
|
ToolkitAPIError,
|
|
16
12
|
)
|
|
13
|
+
from cognite_toolkit._cdf_tk.client.http_client._item_classes import (
|
|
14
|
+
ItemsFailedResponse,
|
|
15
|
+
ItemsRequest,
|
|
16
|
+
ItemsResultList,
|
|
17
|
+
ItemsSuccessResponse,
|
|
18
|
+
)
|
|
17
19
|
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.pending_instances_ids import PendingInstanceId
|
|
18
20
|
from cognite_toolkit._cdf_tk.client.resource_classes.three_d import (
|
|
19
21
|
AssetMappingClassicResponse,
|
|
@@ -170,20 +172,20 @@ class AssetCentricMigrationIO(
|
|
|
170
172
|
data_chunk: Sequence[UploadItem[InstanceApply]],
|
|
171
173
|
http_client: HTTPClient,
|
|
172
174
|
selector: AssetCentricMigrationSelector | None = None,
|
|
173
|
-
) ->
|
|
175
|
+
) -> ItemsResultList:
|
|
174
176
|
"""Upload items by first linking them using files/set-pending-instance-ids and then uploading the instances."""
|
|
175
177
|
if self.skip_linking:
|
|
176
|
-
return
|
|
178
|
+
return super().upload_items(data_chunk, http_client, None)
|
|
177
179
|
elif selector is None:
|
|
178
180
|
raise ToolkitNotImplementedError(f"Selector must be provided for uploading {self.KIND} items.")
|
|
179
181
|
elif selector.kind not in self.PENDING_INSTANCE_ID_ENDPOINT_BY_KIND:
|
|
180
|
-
return
|
|
182
|
+
return super().upload_items(data_chunk, http_client, None)
|
|
181
183
|
|
|
182
184
|
pending_instance_id_endpoint = self.PENDING_INSTANCE_ID_ENDPOINT_BY_KIND[selector.kind]
|
|
183
|
-
results
|
|
185
|
+
results = ItemsResultList()
|
|
184
186
|
to_upload = self.link_asset_centric(data_chunk, http_client, pending_instance_id_endpoint)
|
|
185
187
|
if to_upload:
|
|
186
|
-
results.extend(
|
|
188
|
+
results.extend(super().upload_items(to_upload, http_client, None))
|
|
187
189
|
return results
|
|
188
190
|
|
|
189
191
|
@classmethod
|
|
@@ -197,7 +199,7 @@ class AssetCentricMigrationIO(
|
|
|
197
199
|
config = http_client.config
|
|
198
200
|
successful_linked: set[str] = set()
|
|
199
201
|
for batch in chunker_sequence(data_chunk, cls.CHUNK_SIZE):
|
|
200
|
-
batch_results = http_client.
|
|
202
|
+
batch_results = http_client.request_items_retries(
|
|
201
203
|
message=ItemsRequest(
|
|
202
204
|
endpoint_url=config.create_api_url(pending_instance_id_endpoint),
|
|
203
205
|
method="POST",
|
|
@@ -209,7 +211,7 @@ class AssetCentricMigrationIO(
|
|
|
209
211
|
)
|
|
210
212
|
)
|
|
211
213
|
for res in batch_results:
|
|
212
|
-
if isinstance(res,
|
|
214
|
+
if isinstance(res, ItemsSuccessResponse):
|
|
213
215
|
successful_linked.update(res.ids)
|
|
214
216
|
to_upload = [item for item in data_chunk if item.source_id in successful_linked]
|
|
215
217
|
return to_upload
|
|
@@ -441,31 +443,31 @@ class ThreeDMigrationIO(UploadableStorageIO[ThreeDSelector, ThreeDModelResponse,
|
|
|
441
443
|
data_chunk: Sequence[UploadItem[ThreeDMigrationRequest]],
|
|
442
444
|
http_client: HTTPClient,
|
|
443
445
|
selector: ThreeDSelector | None = None,
|
|
444
|
-
) ->
|
|
446
|
+
) -> ItemsResultList:
|
|
445
447
|
"""Migrate 3D models by uploading them to the migrate/models endpoint."""
|
|
446
448
|
if len(data_chunk) > self.CHUNK_SIZE:
|
|
447
449
|
raise RuntimeError(f"Uploading more than {self.CHUNK_SIZE} 3D models at a time is not supported.")
|
|
448
450
|
|
|
449
|
-
results
|
|
450
|
-
responses = http_client.
|
|
451
|
+
results = ItemsResultList()
|
|
452
|
+
responses = http_client.request_items_retries(
|
|
451
453
|
message=ItemsRequest(
|
|
452
454
|
endpoint_url=self.client.config.create_api_url(self.UPLOAD_ENDPOINT),
|
|
453
455
|
method="POST",
|
|
454
|
-
items=
|
|
456
|
+
items=data_chunk,
|
|
455
457
|
)
|
|
456
458
|
)
|
|
457
459
|
if (
|
|
458
|
-
failed_response := next((res for res in responses if isinstance(res,
|
|
460
|
+
failed_response := next((res for res in responses if isinstance(res, ItemsFailedResponse)), None)
|
|
459
461
|
) and failed_response.status_code == 400:
|
|
460
462
|
raise ToolkitAPIError("3D model migration failed. You need to enable the 3D migration alpha feature flag.")
|
|
461
463
|
|
|
462
464
|
results.extend(responses)
|
|
463
|
-
success_ids = {id for res in responses if isinstance(res,
|
|
465
|
+
success_ids = {id for res in responses if isinstance(res, ItemsSuccessResponse) for id in res.ids}
|
|
464
466
|
for data in data_chunk:
|
|
465
467
|
if data.source_id not in success_ids:
|
|
466
468
|
continue
|
|
467
469
|
revision = http_client.request_single_retries(
|
|
468
|
-
message=
|
|
470
|
+
message=RequestMessage(
|
|
469
471
|
endpoint_url=self.client.config.create_api_url(self.REVISION_ENDPOINT),
|
|
470
472
|
method="POST",
|
|
471
473
|
body_content={"items": [data.item.revision.dump(camel_case=True)]},
|
|
@@ -535,21 +537,21 @@ class ThreeDAssetMappingMigrationIO(
|
|
|
535
537
|
data_chunk: Sequence[UploadItem[AssetMappingDMRequest]],
|
|
536
538
|
http_client: HTTPClient,
|
|
537
539
|
selector: T_Selector | None = None,
|
|
538
|
-
) ->
|
|
540
|
+
) -> ItemsResultList:
|
|
539
541
|
"""Migrate 3D asset mappings by uploading them to the migrate/asset-mappings endpoint."""
|
|
540
542
|
if not data_chunk:
|
|
541
|
-
return
|
|
543
|
+
return ItemsResultList()
|
|
542
544
|
# Assume all items in the chunk belong to the same model and revision, they should
|
|
543
545
|
# if the .stream_data method is used for downloading.
|
|
544
546
|
first = data_chunk[0]
|
|
545
547
|
model_id = first.item.model_id
|
|
546
548
|
revision_id = first.item.revision_id
|
|
547
549
|
endpoint = self.UPLOAD_ENDPOINT.format(modelId=model_id, revisionId=revision_id)
|
|
548
|
-
|
|
550
|
+
return http_client.request_items_retries(
|
|
549
551
|
ItemsRequest(
|
|
550
552
|
endpoint_url=self.client.config.create_api_url(endpoint),
|
|
551
553
|
method="POST",
|
|
552
|
-
items=
|
|
554
|
+
items=data_chunk,
|
|
553
555
|
extra_body_fields={
|
|
554
556
|
"dmsContextualizationConfig": {
|
|
555
557
|
"object3DSpace": self.object_3D_space,
|
|
@@ -558,7 +560,6 @@ class ThreeDAssetMappingMigrationIO(
|
|
|
558
560
|
},
|
|
559
561
|
)
|
|
560
562
|
)
|
|
561
|
-
return responses
|
|
562
563
|
|
|
563
564
|
def json_to_resource(self, item_json: dict[str, JsonVal]) -> AssetMappingDMRequest:
|
|
564
565
|
raise NotImplementedError("Deserializing 3D Asset Mappings from JSON is not supported.")
|
|
@@ -71,8 +71,9 @@ class ProfileCommand(ToolkitCommand, ABC, Generic[T_Index]):
|
|
|
71
71
|
print_warning: bool = True,
|
|
72
72
|
skip_tracking: bool = False,
|
|
73
73
|
silent: bool = False,
|
|
74
|
+
client: ToolkitClient | None = None,
|
|
74
75
|
) -> None:
|
|
75
|
-
super().__init__(print_warning, skip_tracking, silent)
|
|
76
|
+
super().__init__(print_warning, skip_tracking, silent, client)
|
|
76
77
|
self.table_title = self.__class__.__name__.removesuffix("Command")
|
|
77
78
|
self.output_spreadsheet: Path | None = output_spreadsheet
|
|
78
79
|
if output_spreadsheet is not None:
|
|
@@ -347,8 +348,9 @@ class ProfileAssetCommand(ProfileCommand[AssetIndex]):
|
|
|
347
348
|
print_warning: bool = True,
|
|
348
349
|
skip_tracking: bool = False,
|
|
349
350
|
silent: bool = False,
|
|
351
|
+
client: ToolkitClient | None = None,
|
|
350
352
|
) -> None:
|
|
351
|
-
super().__init__(output_spreadsheet, print_warning, skip_tracking, silent)
|
|
353
|
+
super().__init__(output_spreadsheet, print_warning, skip_tracking, silent, client)
|
|
352
354
|
self.table_title = "Asset Profile for Hierarchy"
|
|
353
355
|
self.hierarchy: str | None = None
|
|
354
356
|
self.aggregators: dict[str, MetadataAggregator] = {}
|
|
@@ -641,8 +643,9 @@ class ProfileAssetCentricCommand(ProfileCommand[str]):
|
|
|
641
643
|
print_warning: bool = True,
|
|
642
644
|
skip_tracking: bool = False,
|
|
643
645
|
silent: bool = False,
|
|
646
|
+
client: ToolkitClient | None = None,
|
|
644
647
|
) -> None:
|
|
645
|
-
super().__init__(output_spreadsheet, print_warning, skip_tracking, silent)
|
|
648
|
+
super().__init__(output_spreadsheet, print_warning, skip_tracking, silent, client)
|
|
646
649
|
self.hierarchy: str | None = None
|
|
647
650
|
self.table_title = "Asset Centric Profile"
|
|
648
651
|
self.aggregators: dict[str, AssetCentricAggregator] = {}
|
|
@@ -742,8 +745,9 @@ class ProfileTransformationCommand(ProfileCommand[str]):
|
|
|
742
745
|
print_warning: bool = True,
|
|
743
746
|
skip_tracking: bool = False,
|
|
744
747
|
silent: bool = False,
|
|
748
|
+
client: ToolkitClient | None = None,
|
|
745
749
|
) -> None:
|
|
746
|
-
super().__init__(output_spreadsheet, print_warning, skip_tracking, silent)
|
|
750
|
+
super().__init__(output_spreadsheet, print_warning, skip_tracking, silent, client)
|
|
747
751
|
self.table_title = "Transformation Profile"
|
|
748
752
|
self.destination_type: AssetCentricDestinationType | None = None
|
|
749
753
|
|
|
@@ -826,8 +830,9 @@ class ProfileRawCommand(ProfileCommand[RawProfileIndex]):
|
|
|
826
830
|
print_warning: bool = True,
|
|
827
831
|
skip_tracking: bool = False,
|
|
828
832
|
silent: bool = False,
|
|
833
|
+
client: ToolkitClient | None = None,
|
|
829
834
|
) -> None:
|
|
830
|
-
super().__init__(output_spreadsheet, print_warning, skip_tracking, silent)
|
|
835
|
+
super().__init__(output_spreadsheet, print_warning, skip_tracking, silent, client)
|
|
831
836
|
self.table_title = "RAW Profile"
|
|
832
837
|
self.destination_type: AssetCentricDestinationType | None = None
|
|
833
838
|
self.client: ToolkitClient | None = None
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import uuid
|
|
2
2
|
from abc import ABC, abstractmethod
|
|
3
|
-
from collections.abc import Callable,
|
|
3
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
4
4
|
from dataclasses import dataclass
|
|
5
5
|
from functools import partial
|
|
6
|
-
from typing import Literal, cast
|
|
6
|
+
from typing import Any, Literal, cast
|
|
7
7
|
|
|
8
8
|
import questionary
|
|
9
9
|
from cognite.client.data_classes import DataSetUpdate
|
|
@@ -11,17 +11,17 @@ from cognite.client.data_classes.data_modeling import Edge, NodeId
|
|
|
11
11
|
from cognite.client.data_classes.data_modeling.statistics import SpaceStatistics
|
|
12
12
|
from cognite.client.exceptions import CogniteAPIError
|
|
13
13
|
from cognite.client.utils._identifier import InstanceId
|
|
14
|
+
from pydantic import JsonValue
|
|
14
15
|
from rich import print
|
|
15
16
|
from rich.console import Console
|
|
16
17
|
from rich.panel import Panel
|
|
17
18
|
|
|
18
19
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
20
|
+
from cognite_toolkit._cdf_tk.client._resource_base import RequestItem
|
|
19
21
|
from cognite_toolkit._cdf_tk.client.http_client import (
|
|
20
22
|
HTTPClient,
|
|
21
23
|
ItemsRequest,
|
|
22
|
-
|
|
23
|
-
ItemsSuccessResponse2,
|
|
24
|
-
SuccessResponseItems,
|
|
24
|
+
ItemsSuccessResponse,
|
|
25
25
|
)
|
|
26
26
|
from cognite_toolkit._cdf_tk.client.resource_classes.instance_api import TypedInstanceIdentifier
|
|
27
27
|
from cognite_toolkit._cdf_tk.cruds import (
|
|
@@ -79,15 +79,14 @@ class DeleteResults:
|
|
|
79
79
|
failed: int = 0
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
as_id_fun: Callable[[JsonVal], Hashable]
|
|
82
|
+
class DeleteItem(RequestItem):
|
|
83
|
+
item: JsonValue
|
|
84
|
+
as_id_fun: Callable[[JsonValue], str]
|
|
86
85
|
|
|
87
|
-
def dump(self) ->
|
|
88
|
-
return self.item
|
|
86
|
+
def dump(self, camel_case: bool = True, exclude_extra: bool = False) -> dict[str, Any]:
|
|
87
|
+
return self.item # type: ignore[return-value]
|
|
89
88
|
|
|
90
|
-
def
|
|
89
|
+
def __str__(self) -> str:
|
|
91
90
|
return self.as_id_fun(self.item)
|
|
92
91
|
|
|
93
92
|
|
|
@@ -222,7 +221,7 @@ class PurgeCommand(ToolkitCommand):
|
|
|
222
221
|
if not dry_run and not auto_yes:
|
|
223
222
|
confirm = questionary.confirm(
|
|
224
223
|
f"Are you really sure you want to purge the {selected_space!r} space?", default=False
|
|
225
|
-
).
|
|
224
|
+
).ask()
|
|
226
225
|
if not confirm:
|
|
227
226
|
return DeployResults([], "purge", dry_run=dry_run)
|
|
228
227
|
|
|
@@ -364,15 +363,15 @@ class PurgeCommand(ToolkitCommand):
|
|
|
364
363
|
) -> Callable[[list[JsonVal]], None]:
|
|
365
364
|
crud = delete_item.crud
|
|
366
365
|
|
|
367
|
-
def as_id(item: JsonVal) ->
|
|
366
|
+
def as_id(item: JsonVal) -> str:
|
|
368
367
|
try:
|
|
369
|
-
return crud.get_id(item)
|
|
368
|
+
return str(crud.get_id(item))
|
|
370
369
|
except KeyError:
|
|
371
370
|
# Fallback to internal ID
|
|
372
|
-
return crud.get_internal_id(item)
|
|
371
|
+
return str(crud.get_internal_id(item))
|
|
373
372
|
|
|
374
373
|
def process(items: list[JsonVal]) -> None:
|
|
375
|
-
responses = delete_client.
|
|
374
|
+
responses = delete_client.request_items_retries(
|
|
376
375
|
ItemsRequest(
|
|
377
376
|
endpoint_url=delete_url,
|
|
378
377
|
method="POST",
|
|
@@ -381,10 +380,10 @@ class PurgeCommand(ToolkitCommand):
|
|
|
381
380
|
)
|
|
382
381
|
)
|
|
383
382
|
for response in responses:
|
|
384
|
-
if isinstance(response,
|
|
383
|
+
if isinstance(response, ItemsSuccessResponse):
|
|
385
384
|
result.deleted += len(response.ids)
|
|
386
385
|
else:
|
|
387
|
-
result.unchanged += len(
|
|
386
|
+
result.unchanged += len(response.ids)
|
|
388
387
|
|
|
389
388
|
return process
|
|
390
389
|
|
|
@@ -423,7 +422,7 @@ class PurgeCommand(ToolkitCommand):
|
|
|
423
422
|
if not dry_run and not auto_yes:
|
|
424
423
|
confirm = questionary.confirm(
|
|
425
424
|
f"Are you really sure you want to purge the {selected_data_set_external_id!r} dataSet?", default=False
|
|
426
|
-
).
|
|
425
|
+
).ask()
|
|
427
426
|
if not confirm:
|
|
428
427
|
return DeployResults([], "purge", dry_run=dry_run)
|
|
429
428
|
|
|
@@ -595,7 +594,7 @@ class PurgeCommand(ToolkitCommand):
|
|
|
595
594
|
confirm = questionary.confirm(
|
|
596
595
|
f"Are you really sure you want to purge all {total:,} instances in {selector!s}?",
|
|
597
596
|
default=False,
|
|
598
|
-
).
|
|
597
|
+
).ask()
|
|
599
598
|
if not confirm:
|
|
600
599
|
return DeleteResults()
|
|
601
600
|
|
|
@@ -716,14 +715,14 @@ class PurgeCommand(ToolkitCommand):
|
|
|
716
715
|
return
|
|
717
716
|
|
|
718
717
|
responses = delete_client.request_items_retries(
|
|
719
|
-
|
|
718
|
+
ItemsRequest(
|
|
720
719
|
endpoint_url=delete_client.config.create_api_url("/models/instances/delete"),
|
|
721
720
|
method="POST",
|
|
722
721
|
items=[TypedInstanceIdentifier._load(item) for item in items],
|
|
723
722
|
)
|
|
724
723
|
)
|
|
725
724
|
for response in responses:
|
|
726
|
-
if isinstance(response,
|
|
725
|
+
if isinstance(response, ItemsSuccessResponse):
|
|
727
726
|
results.deleted += len(response.ids)
|
|
728
727
|
else:
|
|
729
728
|
results.failed += len(response.ids)
|
|
@@ -735,17 +734,13 @@ class PurgeCommand(ToolkitCommand):
|
|
|
735
734
|
node_ids = [instance for instance in instances if isinstance(instance, NodeId)]
|
|
736
735
|
if node_ids:
|
|
737
736
|
timeseries = client.time_series.retrieve_multiple(instance_ids=node_ids, ignore_unknown_ids=True)
|
|
738
|
-
migrated_timeseries_ids = [
|
|
739
|
-
ts.id
|
|
740
|
-
for ts in timeseries
|
|
741
|
-
if ts.instance_id and ts.pending_instance_id # type: ignore[attr-defined]
|
|
742
|
-
]
|
|
743
737
|
if not dry_run and timeseries:
|
|
738
|
+
migrated_timeseries_ids = [ts.id for ts in timeseries if ts.instance_id and ts.pending_instance_id] # type: ignore[attr-defined]
|
|
744
739
|
client.time_series.unlink_instance_ids(id=migrated_timeseries_ids)
|
|
745
740
|
if verbose:
|
|
746
741
|
console.print(f"Unlinked {len(migrated_timeseries_ids)} timeseries from datapoints.")
|
|
747
742
|
elif verbose and timeseries:
|
|
748
|
-
console.print(f"Would have unlinked {len(
|
|
743
|
+
console.print(f"Would have unlinked {len(timeseries)} timeseries from datapoints.")
|
|
749
744
|
return instances
|
|
750
745
|
|
|
751
746
|
@staticmethod
|
|
@@ -755,15 +750,15 @@ class PurgeCommand(ToolkitCommand):
|
|
|
755
750
|
file_ids = [instance for instance in instances if isinstance(instance, NodeId)]
|
|
756
751
|
if file_ids:
|
|
757
752
|
files = client.files.retrieve_multiple(instance_ids=file_ids, ignore_unknown_ids=True)
|
|
758
|
-
migrated_file_ids = [
|
|
759
|
-
file.id
|
|
760
|
-
for file in files
|
|
761
|
-
if file.instance_id and file.pending_instance_id and file.id is not None # type: ignore[attr-defined]
|
|
762
|
-
]
|
|
763
753
|
if not dry_run and files:
|
|
754
|
+
migrated_file_ids = [
|
|
755
|
+
file.id
|
|
756
|
+
for file in files
|
|
757
|
+
if file.instance_id and file.pending_instance_id and file.id is not None # type: ignore[attr-defined]
|
|
758
|
+
]
|
|
764
759
|
client.files.unlink_instance_ids(id=migrated_file_ids)
|
|
765
760
|
if verbose:
|
|
766
761
|
console.print(f"Unlinked {len(migrated_file_ids)} files from nodes.")
|
|
767
762
|
elif verbose and files:
|
|
768
|
-
console.print(f"Would have unlinked {len(
|
|
763
|
+
console.print(f"Would have unlinked {len(files)} files from their blob content.")
|
|
769
764
|
return instances
|
|
@@ -3,13 +3,11 @@ from collections.abc import Sequence
|
|
|
3
3
|
from functools import partial
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
|
|
6
|
-
from cognite.client.data_classes.data_modeling import
|
|
7
|
-
ViewId,
|
|
8
|
-
)
|
|
6
|
+
from cognite.client.data_classes.data_modeling import ViewId
|
|
9
7
|
from rich.console import Console
|
|
10
8
|
|
|
11
9
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
12
|
-
from cognite_toolkit._cdf_tk.client.http_client import HTTPClient,
|
|
10
|
+
from cognite_toolkit._cdf_tk.client.http_client import HTTPClient, ItemsResultMessage, ItemsSuccessResponse
|
|
13
11
|
from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_SUFFIX, DATA_RESOURCE_DIR
|
|
14
12
|
from cognite_toolkit._cdf_tk.cruds import ViewCRUD
|
|
15
13
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
|
@@ -279,10 +277,10 @@ class UploadCommand(ToolkitCommand):
|
|
|
279
277
|
return
|
|
280
278
|
results = io.upload_items(data_chunk, upload_client, selector)
|
|
281
279
|
for message in results:
|
|
282
|
-
if isinstance(message,
|
|
280
|
+
if isinstance(message, ItemsSuccessResponse):
|
|
283
281
|
for id_ in message.ids:
|
|
284
282
|
tracker.set_progress(id_, step=cls._UPLOAD, status="success")
|
|
285
|
-
elif isinstance(message,
|
|
283
|
+
elif isinstance(message, ItemsResultMessage):
|
|
286
284
|
for id_ in message.ids:
|
|
287
285
|
tracker.set_progress(id_, step=cls._UPLOAD, status="failed")
|
|
288
286
|
else:
|
|
@@ -99,8 +99,9 @@ class BuildCommand(ToolkitCommand):
|
|
|
99
99
|
print_warning: bool = True,
|
|
100
100
|
skip_tracking: bool = False,
|
|
101
101
|
silent: bool = False,
|
|
102
|
+
client: ToolkitClient | None = None,
|
|
102
103
|
) -> None:
|
|
103
|
-
super().__init__(print_warning, skip_tracking, silent)
|
|
104
|
+
super().__init__(print_warning, skip_tracking, silent, client)
|
|
104
105
|
self.existing_resources_by_loader: dict[type[ResourceCRUD], set[Hashable]] = defaultdict(set)
|
|
105
106
|
self.instantiated_loaders: dict[type[ResourceCRUD], ResourceCRUD] = {}
|
|
106
107
|
|
|
@@ -21,8 +21,14 @@ from cognite_toolkit._version import __version__
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class BuildCommand(ToolkitCommand):
|
|
24
|
-
def __init__(
|
|
25
|
-
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
print_warning: bool = True,
|
|
27
|
+
skip_tracking: bool = False,
|
|
28
|
+
silent: bool = False,
|
|
29
|
+
client: ToolkitClient | None = None,
|
|
30
|
+
) -> None:
|
|
31
|
+
super().__init__(print_warning, skip_tracking, silent, client)
|
|
26
32
|
self.issues = IssueList()
|
|
27
33
|
|
|
28
34
|
def execute(
|
|
@@ -62,8 +62,14 @@ from ._utils import _print_ids_or_length
|
|
|
62
62
|
|
|
63
63
|
|
|
64
64
|
class DeployCommand(ToolkitCommand):
|
|
65
|
-
def __init__(
|
|
66
|
-
|
|
65
|
+
def __init__(
|
|
66
|
+
self,
|
|
67
|
+
print_warning: bool = True,
|
|
68
|
+
skip_tracking: bool = False,
|
|
69
|
+
silent: bool = False,
|
|
70
|
+
client: ToolkitClient | None = None,
|
|
71
|
+
) -> None:
|
|
72
|
+
super().__init__(print_warning, skip_tracking, silent, client)
|
|
67
73
|
self._clean_command = CleanCommand(print_warning, skip_tracking=True)
|
|
68
74
|
|
|
69
75
|
def deploy_build_directory(
|
|
@@ -13,6 +13,7 @@ from rich import print
|
|
|
13
13
|
from rich.panel import Panel
|
|
14
14
|
|
|
15
15
|
from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
|
|
16
|
+
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
16
17
|
from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
|
|
17
18
|
from cognite_toolkit._cdf_tk.commands.auth import AuthCommand
|
|
18
19
|
from cognite_toolkit._cdf_tk.commands.collect import CollectCommand
|
|
@@ -54,8 +55,14 @@ class InitChecklistItem:
|
|
|
54
55
|
class InitCommand(ToolkitCommand):
|
|
55
56
|
organization_dir: Path | None
|
|
56
57
|
|
|
57
|
-
def __init__(
|
|
58
|
-
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
print_warning: bool = True,
|
|
61
|
+
skip_tracking: bool = False,
|
|
62
|
+
silent: bool = False,
|
|
63
|
+
client: ToolkitClient | None = None,
|
|
64
|
+
) -> None:
|
|
65
|
+
super().__init__(print_warning, skip_tracking, silent, client)
|
|
59
66
|
self.organization_dir = None
|
|
60
67
|
|
|
61
68
|
def execute(self, dry_run: bool = False) -> None:
|
|
@@ -23,6 +23,7 @@ from rich.table import Table
|
|
|
23
23
|
from rich.tree import Tree
|
|
24
24
|
|
|
25
25
|
from cognite_toolkit._cdf_tk.cdf_toml import CDFToml, Library
|
|
26
|
+
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
26
27
|
from cognite_toolkit._cdf_tk.commands import _cli_commands as CLICommands
|
|
27
28
|
from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
|
|
28
29
|
from cognite_toolkit._cdf_tk.commands._changes import (
|
|
@@ -81,10 +82,11 @@ class ModulesCommand(ToolkitCommand):
|
|
|
81
82
|
print_warning: bool = True,
|
|
82
83
|
skip_tracking: bool = False,
|
|
83
84
|
silent: bool = False,
|
|
85
|
+
client: ToolkitClient | None = None,
|
|
84
86
|
temp_dir_suffix: str | None = None,
|
|
85
87
|
module_source_dir: Path | None = None,
|
|
86
88
|
):
|
|
87
|
-
super().__init__(print_warning, skip_tracking, silent)
|
|
89
|
+
super().__init__(print_warning, skip_tracking, silent, client)
|
|
88
90
|
self._module_source_dir: Path | None = module_source_dir
|
|
89
91
|
# Use suffix to make temp directory unique (useful for parallel test execution)
|
|
90
92
|
modules_dir_name = f"{MODULES}.{temp_dir_suffix}" if temp_dir_suffix else MODULES
|
|
@@ -411,8 +411,14 @@ class TextFileDifference(UserList):
|
|
|
411
411
|
|
|
412
412
|
|
|
413
413
|
class PullCommand(ToolkitCommand):
|
|
414
|
-
def __init__(
|
|
415
|
-
|
|
414
|
+
def __init__(
|
|
415
|
+
self,
|
|
416
|
+
print_warning: bool = True,
|
|
417
|
+
skip_tracking: bool = False,
|
|
418
|
+
silent: bool = False,
|
|
419
|
+
client: ToolkitClient | None = None,
|
|
420
|
+
) -> None:
|
|
421
|
+
super().__init__(print_warning, skip_tracking, silent, client)
|
|
416
422
|
self._clean_command = CleanCommand(print_warning, skip_tracking=True)
|
|
417
423
|
|
|
418
424
|
def pull_module(
|
|
@@ -5,6 +5,7 @@ from pathlib import Path
|
|
|
5
5
|
import questionary
|
|
6
6
|
|
|
7
7
|
import cognite_toolkit
|
|
8
|
+
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
8
9
|
from cognite_toolkit._cdf_tk.constants import REPO_FILES_DIR
|
|
9
10
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
|
10
11
|
from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, MediumSeverityWarning
|
|
@@ -27,8 +28,9 @@ class RepoCommand(ToolkitCommand):
|
|
|
27
28
|
skip_tracking: bool = False,
|
|
28
29
|
silent: bool = False,
|
|
29
30
|
skip_git_verify: bool = False,
|
|
31
|
+
client: ToolkitClient | None = None,
|
|
30
32
|
) -> None:
|
|
31
|
-
super().__init__(print_warning=print_warning, skip_tracking=skip_tracking, silent=silent)
|
|
33
|
+
super().__init__(print_warning=print_warning, skip_tracking=skip_tracking, silent=silent, client=client)
|
|
32
34
|
self._repo_files = Path(resources.files(cognite_toolkit.__name__)) / REPO_FILES_DIR # type: ignore [arg-type]
|
|
33
35
|
self.skip_git_verify = skip_git_verify
|
|
34
36
|
|
|
@@ -17,9 +17,6 @@ from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class ResourcesCommand(ToolkitCommand):
|
|
20
|
-
def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
|
|
21
|
-
super().__init__(print_warning, skip_tracking, silent)
|
|
22
|
-
|
|
23
20
|
def _get_or_prompt_module_path(self, module: str | None, organization_dir: Path, verbose: bool) -> Path:
|
|
24
21
|
"""
|
|
25
22
|
Check if the module exists in the organization directory and return the module path.
|
|
@@ -25,6 +25,7 @@ class CommandTrackingInfo(BaseModel):
|
|
|
25
25
|
|
|
26
26
|
project: str | None = Field(default=None)
|
|
27
27
|
cluster: str | None = Field(default=None)
|
|
28
|
+
organization: str | None = Field(default=None)
|
|
28
29
|
module_ids: set[str] = Field(default_factory=set, alias="moduleIds")
|
|
29
30
|
package_ids: set[str] = Field(default_factory=set, alias="packageIds")
|
|
30
31
|
installed_module_ids: set[str] = Field(default_factory=set, alias="installedModuleIds")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
from collections.abc import Collection, Iterator
|
|
3
|
-
from typing import Any, Generic, Protocol, TypeVar
|
|
3
|
+
from typing import Any, Generic, Protocol, TypeVar, runtime_checkable
|
|
4
4
|
|
|
5
5
|
from cognite.client import CogniteClient
|
|
6
6
|
|
|
@@ -10,6 +10,7 @@ else:
|
|
|
10
10
|
from typing_extensions import Self
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
@runtime_checkable
|
|
13
14
|
class ResourceRequestProtocol(Protocol):
|
|
14
15
|
@classmethod
|
|
15
16
|
def _load(cls, data: dict[str, Any]) -> Self: ...
|
|
@@ -17,6 +18,7 @@ class ResourceRequestProtocol(Protocol):
|
|
|
17
18
|
def dump(self, camel_case: bool = True) -> dict[str, Any]: ...
|
|
18
19
|
|
|
19
20
|
|
|
21
|
+
@runtime_checkable
|
|
20
22
|
class ResourceResponseProtocol(Protocol):
|
|
21
23
|
def as_write(self) -> ResourceRequestProtocol: ...
|
|
22
24
|
|