cognite-toolkit 0.7.42__py3-none-any.whl → 0.7.44__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/client/_toolkit_client.py +7 -1
- cognite_toolkit/_cdf_tk/client/api/events.py +20 -2
- cognite_toolkit/_cdf_tk/client/api/filemetadata.py +145 -0
- cognite_toolkit/_cdf_tk/client/api/raw.py +174 -0
- cognite_toolkit/_cdf_tk/client/api/simulator_models.py +118 -0
- cognite_toolkit/_cdf_tk/client/api/simulators.py +8 -0
- cognite_toolkit/_cdf_tk/client/api/timeseries.py +20 -2
- cognite_toolkit/_cdf_tk/client/cdf_client/__init__.py +2 -1
- cognite_toolkit/_cdf_tk/client/cdf_client/api.py +40 -6
- cognite_toolkit/_cdf_tk/client/data_classes/agent.py +6 -9
- cognite_toolkit/_cdf_tk/client/data_classes/annotation.py +79 -0
- cognite_toolkit/_cdf_tk/client/data_classes/asset.py +7 -14
- cognite_toolkit/_cdf_tk/client/data_classes/base.py +15 -5
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/__init__.py +164 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_constraints.py +37 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_container.py +50 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_data_model.py +73 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_data_types.py +116 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_indexes.py +26 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_instance.py +143 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_references.py +86 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_space.py +26 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_view.py +143 -0
- cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_view_property.py +152 -0
- cognite_toolkit/_cdf_tk/client/data_classes/dataset.py +35 -0
- cognite_toolkit/_cdf_tk/client/data_classes/event.py +12 -15
- cognite_toolkit/_cdf_tk/client/data_classes/extraction_pipeline.py +59 -0
- cognite_toolkit/_cdf_tk/client/data_classes/filemetadata.py +15 -19
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_destination.py +34 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_job.py +134 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_mapping.py +72 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/__init__.py +63 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_auth.py +63 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_base.py +26 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_certificate.py +20 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_eventhub.py +31 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_kafka.py +53 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_mqtt.py +36 -0
- cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_rest.py +49 -0
- cognite_toolkit/_cdf_tk/client/data_classes/identifiers.py +8 -0
- cognite_toolkit/_cdf_tk/client/data_classes/label.py +27 -0
- cognite_toolkit/_cdf_tk/client/data_classes/raw.py +3 -2
- cognite_toolkit/_cdf_tk/client/data_classes/securitycategory.py +24 -0
- cognite_toolkit/_cdf_tk/client/data_classes/sequence.py +45 -0
- cognite_toolkit/_cdf_tk/client/data_classes/simulator_model.py +50 -0
- cognite_toolkit/_cdf_tk/client/data_classes/timeseries.py +15 -18
- cognite_toolkit/_cdf_tk/client/data_classes/transformation.py +140 -0
- cognite_toolkit/_cdf_tk/client/data_classes/workflow.py +27 -0
- cognite_toolkit/_cdf_tk/client/data_classes/workflow_trigger.py +63 -0
- cognite_toolkit/_cdf_tk/client/data_classes/workflow_version.py +155 -0
- cognite_toolkit/_cdf_tk/client/testing.py +6 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +10 -7
- cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +4 -4
- cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +7 -3
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py +5 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py +40 -39
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py +56 -59
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py +3 -3
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py +48 -47
- cognite_toolkit/_cdf_tk/resource_classes/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/resource_classes/simulator_model.py +17 -0
- cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +84 -71
- cognite_toolkit/_cdf_tk/storageio/_file_content.py +22 -19
- cognite_toolkit/_cdf_tk/utils/useful_types2.py +5 -3
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.42.dist-info → cognite_toolkit-0.7.44.dist-info}/METADATA +11 -1
- {cognite_toolkit-0.7.42.dist-info → cognite_toolkit-0.7.44.dist-info}/RECORD +72 -34
- {cognite_toolkit-0.7.42.dist-info → cognite_toolkit-0.7.44.dist-info}/WHEEL +1 -1
- {cognite_toolkit-0.7.42.dist-info → cognite_toolkit-0.7.44.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
from typing import Annotated, Any, Literal
|
|
2
|
+
|
|
3
|
+
from pydantic import Field, JsonValue, field_validator
|
|
4
|
+
from pydantic_core.core_schema import ValidationInfo
|
|
5
|
+
|
|
6
|
+
from cognite_toolkit._cdf_tk.client.data_classes.base import (
|
|
7
|
+
BaseModelObject,
|
|
8
|
+
RequestResource,
|
|
9
|
+
ResponseResource,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
from .identifiers import Identifier, WorkflowVersionId
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TaskId(Identifier):
|
|
16
|
+
external_id: str
|
|
17
|
+
|
|
18
|
+
def __str__(self) -> str:
|
|
19
|
+
return f"externalId='{self.external_id}'"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TaskParameterDefinition(BaseModelObject):
|
|
23
|
+
type: str
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class CogniteFunctionRef(BaseModelObject):
|
|
27
|
+
external_id: str
|
|
28
|
+
data: JsonValue | str | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class FunctionTaskParameters(TaskParameterDefinition):
|
|
32
|
+
type: Literal["function"] = Field("function", exclude=True)
|
|
33
|
+
function: CogniteFunctionRef
|
|
34
|
+
is_async_complete: bool | None = None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class TransformationRef(BaseModelObject):
|
|
38
|
+
external_id: str
|
|
39
|
+
concurrency_policy: Literal["fail", "waitForCurrent", "restartAfterCurrent"] | None = None
|
|
40
|
+
use_transformation_credentials: bool | None = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class TransformationTaskParameters(TaskParameterDefinition):
|
|
44
|
+
type: Literal["transformation"] = Field("transformation", exclude=True)
|
|
45
|
+
transformation: TransformationRef
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class CDFRequest(BaseModelObject):
|
|
49
|
+
resource_path: str
|
|
50
|
+
method: str
|
|
51
|
+
query_parameters: JsonValue | str | None = None
|
|
52
|
+
body: JsonValue | str | None = None
|
|
53
|
+
request_timeout_in_millis: float | str | None = None
|
|
54
|
+
cdf_version_header: str | None = None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class CDFTaskParameters(TaskParameterDefinition):
|
|
58
|
+
type: Literal["cdf"] = Field("cdf", exclude=True)
|
|
59
|
+
cdf_request: CDFRequest
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class DynamicRef(BaseModelObject):
|
|
63
|
+
tasks: str
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class DynamicTaskParameters(TaskParameterDefinition):
|
|
67
|
+
type: Literal["dynamic"] = Field("dynamic", exclude=True)
|
|
68
|
+
dynamic: DynamicRef
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class SubworkflowRef(BaseModelObject):
|
|
72
|
+
tasks: "list[Task]"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class SubworkflowTaskParameters(TaskParameterDefinition):
|
|
76
|
+
type: Literal["subworkflow"] = Field("subworkflow", exclude=True)
|
|
77
|
+
subworkflow: WorkflowVersionId | SubworkflowRef
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class SimulatorInputUnit(BaseModelObject):
|
|
81
|
+
name: str
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class SimulatorInput(BaseModelObject):
|
|
85
|
+
reference_id: str
|
|
86
|
+
value: str | int | float | list[str] | list[int] | list[float]
|
|
87
|
+
unit: SimulatorInputUnit | None = None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class SimulationRef(BaseModelObject):
|
|
91
|
+
routine_external_id: str
|
|
92
|
+
run_time: int | None = None
|
|
93
|
+
inputs: list[SimulatorInput] | None = None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class SimulationTaskParameters(TaskParameterDefinition):
|
|
97
|
+
type: Literal["simulation"] = Field("simulation", exclude=True)
|
|
98
|
+
simulation: SimulationRef
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
Parameter = Annotated[
|
|
102
|
+
FunctionTaskParameters
|
|
103
|
+
| TransformationTaskParameters
|
|
104
|
+
| CDFTaskParameters
|
|
105
|
+
| DynamicTaskParameters
|
|
106
|
+
| SubworkflowTaskParameters
|
|
107
|
+
| SimulationTaskParameters,
|
|
108
|
+
Field(discriminator="type"),
|
|
109
|
+
]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class Task(BaseModelObject):
|
|
113
|
+
external_id: str
|
|
114
|
+
type: str
|
|
115
|
+
name: str | None = None
|
|
116
|
+
description: str | None = None
|
|
117
|
+
retries: int | None = None
|
|
118
|
+
timeout: int | None = None
|
|
119
|
+
on_failure: Literal["abortWorkflow", "skipTask"] = "abortWorkflow"
|
|
120
|
+
depends_on: list[TaskId] | None = None
|
|
121
|
+
parameters: Parameter | None = None
|
|
122
|
+
|
|
123
|
+
@field_validator("parameters", mode="before")
|
|
124
|
+
@classmethod
|
|
125
|
+
def move_type_to_field(cls, value: Any, info: ValidationInfo) -> Any:
|
|
126
|
+
if not isinstance(value, dict) or "type" not in info.data:
|
|
127
|
+
return value
|
|
128
|
+
value = dict(value)
|
|
129
|
+
value["type"] = info.data["type"]
|
|
130
|
+
return value
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class WorkflowDefinition(BaseModelObject):
|
|
134
|
+
description: str | None = None
|
|
135
|
+
tasks: list[Task]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class WorkflowVersion(BaseModelObject):
|
|
139
|
+
workflow_external_id: str
|
|
140
|
+
version: str
|
|
141
|
+
workflow_definition: WorkflowDefinition
|
|
142
|
+
|
|
143
|
+
def as_id(self) -> WorkflowVersionId:
|
|
144
|
+
return WorkflowVersionId(workflow_external_id=self.workflow_external_id, version=self.version)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class WorkflowVersionRequest(WorkflowVersion, RequestResource): ...
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class WorkflowVersionResponse(WorkflowVersion, ResponseResource[WorkflowVersionRequest]):
|
|
151
|
+
created_time: int
|
|
152
|
+
last_updated_time: int
|
|
153
|
+
|
|
154
|
+
def as_request_resource(self) -> WorkflowVersionRequest:
|
|
155
|
+
return WorkflowVersionRequest.model_validate(self.dump(), extra="ignore")
|
|
@@ -7,12 +7,14 @@ from cognite.client._api.datapoints import DatapointsAPI
|
|
|
7
7
|
from cognite.client._api.datapoints_subscriptions import DatapointsSubscriptionAPI
|
|
8
8
|
from cognite.client._api.functions import FunctionCallsAPI, FunctionSchedulesAPI
|
|
9
9
|
from cognite.client._api.raw import RawDatabasesAPI, RawRowsAPI, RawTablesAPI
|
|
10
|
+
from cognite.client._api.simulators import SimulatorModelsAPI, SimulatorsAPI
|
|
10
11
|
from cognite.client._api.synthetic_time_series import SyntheticDatapointsAPI
|
|
11
12
|
from cognite.client.testing import CogniteClientMock
|
|
12
13
|
from rich.console import Console
|
|
13
14
|
|
|
14
15
|
from cognite_toolkit._cdf_tk.client._toolkit_client import ToolkitClient
|
|
15
16
|
from cognite_toolkit._cdf_tk.client.api.assets import AssetsAPI
|
|
17
|
+
from cognite_toolkit._cdf_tk.client.api.filemetadata import FileMetadataAPI
|
|
16
18
|
from cognite_toolkit._cdf_tk.client.api.legacy.canvas import CanvasAPI, IndustrialCanvasAPI
|
|
17
19
|
from cognite_toolkit._cdf_tk.client.api.legacy.charts import ChartsAPI
|
|
18
20
|
from cognite_toolkit._cdf_tk.client.api.legacy.dml import DMLAPI
|
|
@@ -142,8 +144,11 @@ class ToolkitClientMock(CogniteClientMock):
|
|
|
142
144
|
self.tool.three_d = MagicMock(spec=ThreeDAPI)
|
|
143
145
|
self.tool.three_d.models = MagicMock(spec_set=ThreeDModelAPI)
|
|
144
146
|
self.tool.assets = MagicMock(spec_set=AssetsAPI)
|
|
145
|
-
self.tool.
|
|
147
|
+
self.tool.timeseries = MagicMock(spec_set=TimeSeriesAPI)
|
|
148
|
+
self.tool.filemetadata = MagicMock(spec_set=FileMetadataAPI)
|
|
146
149
|
self.tool.events = MagicMock(spec_set=EventsAPI)
|
|
150
|
+
self.tool.simulators = MagicMock(spec=SimulatorsAPI)
|
|
151
|
+
self.tool.simulators.models = MagicMock(spec_set=SimulatorModelsAPI)
|
|
147
152
|
|
|
148
153
|
self.streams = MagicMock(spec=StreamsAPI)
|
|
149
154
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from collections.abc import Iterable, Mapping, Set
|
|
2
2
|
from typing import Any, ClassVar, cast
|
|
3
3
|
|
|
4
|
-
from cognite.client.data_classes import Annotation
|
|
4
|
+
from cognite.client.data_classes import Annotation
|
|
5
5
|
from cognite.client.data_classes.data_modeling import (
|
|
6
6
|
DirectRelation,
|
|
7
7
|
DirectRelationReference,
|
|
@@ -17,10 +17,13 @@ from cognite.client.utils._identifier import InstanceId
|
|
|
17
17
|
|
|
18
18
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
19
19
|
from cognite_toolkit._cdf_tk.client.data_classes.asset import AssetResponse
|
|
20
|
+
from cognite_toolkit._cdf_tk.client.data_classes.event import EventResponse
|
|
21
|
+
from cognite_toolkit._cdf_tk.client.data_classes.filemetadata import FileMetadataResponse
|
|
20
22
|
from cognite_toolkit._cdf_tk.client.data_classes.legacy.migration import (
|
|
21
23
|
AssetCentricId,
|
|
22
24
|
ResourceViewMappingApply,
|
|
23
25
|
)
|
|
26
|
+
from cognite_toolkit._cdf_tk.client.data_classes.timeseries import TimeSeriesResponse
|
|
24
27
|
from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
|
|
25
28
|
from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
|
|
26
29
|
asset_centric_convert_to_primary_property,
|
|
@@ -115,17 +118,17 @@ class DirectRelationCache:
|
|
|
115
118
|
source_ids.add(resource.source)
|
|
116
119
|
if resource.parent_id is not None:
|
|
117
120
|
asset_ids.add(resource.parent_id)
|
|
118
|
-
elif isinstance(resource,
|
|
121
|
+
elif isinstance(resource, FileMetadataResponse):
|
|
119
122
|
if resource.source:
|
|
120
123
|
source_ids.add(resource.source)
|
|
121
124
|
if resource.asset_ids:
|
|
122
125
|
asset_ids.update(resource.asset_ids)
|
|
123
|
-
elif isinstance(resource,
|
|
126
|
+
elif isinstance(resource, EventResponse):
|
|
124
127
|
if resource.source:
|
|
125
128
|
source_ids.add(resource.source)
|
|
126
129
|
if resource.asset_ids:
|
|
127
130
|
asset_ids.update(resource.asset_ids)
|
|
128
|
-
elif isinstance(resource,
|
|
131
|
+
elif isinstance(resource, TimeSeriesResponse):
|
|
129
132
|
if resource.asset_id is not None:
|
|
130
133
|
asset_ids.add(resource.asset_id)
|
|
131
134
|
if asset_ids:
|
|
@@ -246,11 +249,11 @@ def asset_centric_to_dm(
|
|
|
246
249
|
def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetCentricTypeExtended:
|
|
247
250
|
if isinstance(resource_type, AssetResponse):
|
|
248
251
|
return "asset"
|
|
249
|
-
elif isinstance(resource_type,
|
|
252
|
+
elif isinstance(resource_type, FileMetadataResponse):
|
|
250
253
|
return "file"
|
|
251
|
-
elif isinstance(resource_type,
|
|
254
|
+
elif isinstance(resource_type, EventResponse):
|
|
252
255
|
return "event"
|
|
253
|
-
elif isinstance(resource_type,
|
|
256
|
+
elif isinstance(resource_type, TimeSeriesResponse):
|
|
254
257
|
return "timeseries"
|
|
255
258
|
elif isinstance(resource_type, Annotation):
|
|
256
259
|
if resource_type.annotated_resource_type == "file" and resource_type.annotation_type in (
|
|
@@ -449,15 +449,15 @@ class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrati
|
|
|
449
449
|
return None, issue
|
|
450
450
|
|
|
451
451
|
mapped_request = ThreeDMigrationRequest(
|
|
452
|
-
|
|
452
|
+
model_id=item.id,
|
|
453
453
|
type=model_type,
|
|
454
454
|
space=instance_space,
|
|
455
455
|
revision=ThreeDRevisionMigrationRequest(
|
|
456
456
|
space=instance_space,
|
|
457
457
|
type=model_type,
|
|
458
|
-
|
|
458
|
+
revision_id=last_revision_id,
|
|
459
459
|
model=Model(
|
|
460
|
-
|
|
460
|
+
instance_id=InstanceIdentifier(
|
|
461
461
|
space=instance_space,
|
|
462
462
|
external_id=f"cog_3d_model_{item.id!s}",
|
|
463
463
|
)
|
|
@@ -508,7 +508,7 @@ class ThreeDAssetMapper(DataMapper[ThreeDSelector, AssetMappingResponse, AssetMa
|
|
|
508
508
|
if asset_node_id is None:
|
|
509
509
|
issue.error_message.append(f"Missing asset instance for asset ID {item.asset_id!r}")
|
|
510
510
|
return None, issue
|
|
511
|
-
asset_instance_id = NodeReference(space=asset_node_id.space,
|
|
511
|
+
asset_instance_id = NodeReference(space=asset_node_id.space, external_id=asset_node_id.external_id)
|
|
512
512
|
|
|
513
513
|
if asset_instance_id is None:
|
|
514
514
|
issue.error_message.append("Neither assetInstanceId nor assetId provided for mapping.")
|
|
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast, final
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
from cognite.client.data_classes import FileMetadataWrite
|
|
8
8
|
|
|
9
|
+
from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId
|
|
9
10
|
from cognite_toolkit._cdf_tk.client.data_classes.legacy.extendable_cognite_file import ExtendableCogniteFileApply
|
|
10
11
|
from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawTable
|
|
11
12
|
from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
|
|
@@ -119,11 +120,14 @@ class FileCRUD(DataCRUD):
|
|
|
119
120
|
if not datafile.exists():
|
|
120
121
|
continue
|
|
121
122
|
|
|
123
|
+
identifier = resource.identifier
|
|
124
|
+
if isinstance(identifier, ExternalId):
|
|
125
|
+
identifier = identifier.external_id
|
|
122
126
|
if dry_run:
|
|
123
|
-
yield f" Would upload file '{datafile!s}' to file with {id_name}={
|
|
127
|
+
yield f" Would upload file '{datafile!s}' to file with {id_name}={identifier!r}", 1
|
|
124
128
|
else:
|
|
125
|
-
self.client.files.upload_content(path=str(datafile), **{id_name:
|
|
126
|
-
yield f" Uploaded file '{datafile!s}' to file with {id_name}={
|
|
129
|
+
self.client.files.upload_content(path=str(datafile), **{id_name: identifier})
|
|
130
|
+
yield f" Uploaded file '{datafile!s}' to file with {id_name}={identifier!r}", 1
|
|
127
131
|
|
|
128
132
|
@staticmethod
|
|
129
133
|
def _read_metadata(
|
|
@@ -41,6 +41,7 @@ from rich.console import Console
|
|
|
41
41
|
from rich.markup import escape
|
|
42
42
|
|
|
43
43
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
44
|
+
from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId
|
|
44
45
|
from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawDatabase, RawTable
|
|
45
46
|
from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
|
|
46
47
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError
|
|
@@ -190,7 +191,10 @@ class GroupCRUD(ResourceCRUD[str, GroupWrite, Group]):
|
|
|
190
191
|
loader = LocationFilterCRUD
|
|
191
192
|
if loader is not None and isinstance(ids, dict) and "ids" in ids:
|
|
192
193
|
for id_ in ids["ids"]:
|
|
193
|
-
|
|
194
|
+
if loader is TimeSeriesCRUD:
|
|
195
|
+
yield TimeSeriesCRUD, ExternalId(external_id=id_)
|
|
196
|
+
else:
|
|
197
|
+
yield loader, id_
|
|
194
198
|
|
|
195
199
|
def _substitute_scope_ids(self, group: dict[str, Any], is_dry_run: bool, reverse: bool = False) -> dict[str, Any]:
|
|
196
200
|
replace_method_by_acl = self._create_replace_method_by_acl_and_scope()
|
|
@@ -5,15 +5,12 @@ from pathlib import Path
|
|
|
5
5
|
from typing import Any, final
|
|
6
6
|
|
|
7
7
|
import pandas as pd
|
|
8
|
+
from cognite.client.data_classes import Sequence as CDFSequence
|
|
8
9
|
from cognite.client.data_classes import (
|
|
9
|
-
Event,
|
|
10
|
-
EventList,
|
|
11
|
-
EventWrite,
|
|
12
10
|
SequenceList,
|
|
13
11
|
SequenceWrite,
|
|
14
12
|
capabilities,
|
|
15
13
|
)
|
|
16
|
-
from cognite.client.data_classes import Sequence as CDFSequence
|
|
17
14
|
from cognite.client.data_classes.capabilities import Capability
|
|
18
15
|
from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
|
|
19
16
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
@@ -25,6 +22,7 @@ from cognite_toolkit._cdf_tk.client.data_classes.asset import (
|
|
|
25
22
|
AssetRequest,
|
|
26
23
|
AssetResponse,
|
|
27
24
|
)
|
|
25
|
+
from cognite_toolkit._cdf_tk.client.data_classes.event import EventRequest, EventResponse
|
|
28
26
|
from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId, InternalOrExternalId
|
|
29
27
|
from cognite_toolkit._cdf_tk.client.data_classes.legacy.sequences import (
|
|
30
28
|
ToolkitSequenceRows,
|
|
@@ -496,10 +494,10 @@ class SequenceRowCRUD(ResourceCRUD[str, ToolkitSequenceRowsWrite, ToolkitSequenc
|
|
|
496
494
|
|
|
497
495
|
|
|
498
496
|
@final
|
|
499
|
-
class EventCRUD(ResourceCRUD[
|
|
497
|
+
class EventCRUD(ResourceCRUD[ExternalId, EventRequest, EventResponse]):
|
|
500
498
|
folder_name = "classic"
|
|
501
|
-
resource_cls =
|
|
502
|
-
resource_write_cls =
|
|
499
|
+
resource_cls = EventResponse
|
|
500
|
+
resource_write_cls = EventRequest
|
|
503
501
|
yaml_cls = EventYAML
|
|
504
502
|
kind = "Event"
|
|
505
503
|
dependencies = frozenset({DataSetsCRUD, AssetCRUD})
|
|
@@ -510,15 +508,15 @@ class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
|
|
|
510
508
|
return "events"
|
|
511
509
|
|
|
512
510
|
@classmethod
|
|
513
|
-
def get_id(cls, item:
|
|
511
|
+
def get_id(cls, item: EventRequest | EventResponse | dict) -> ExternalId:
|
|
514
512
|
if isinstance(item, dict):
|
|
515
|
-
return item["externalId"]
|
|
513
|
+
return ExternalId(external_id=item["externalId"])
|
|
516
514
|
if not item.external_id:
|
|
517
515
|
raise KeyError("Event must have external_id")
|
|
518
|
-
return item.external_id
|
|
516
|
+
return ExternalId(external_id=item.external_id)
|
|
519
517
|
|
|
520
518
|
@classmethod
|
|
521
|
-
def get_internal_id(cls, item:
|
|
519
|
+
def get_internal_id(cls, item: EventResponse | dict) -> int:
|
|
522
520
|
if isinstance(item, dict):
|
|
523
521
|
return item["id"]
|
|
524
522
|
if not item.id:
|
|
@@ -526,12 +524,12 @@ class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
|
|
|
526
524
|
return item.id
|
|
527
525
|
|
|
528
526
|
@classmethod
|
|
529
|
-
def dump_id(cls, id:
|
|
530
|
-
return
|
|
527
|
+
def dump_id(cls, id: ExternalId) -> dict[str, Any]:
|
|
528
|
+
return id.dump()
|
|
531
529
|
|
|
532
530
|
@classmethod
|
|
533
531
|
def get_required_capability(
|
|
534
|
-
cls, items: collections.abc.Sequence[
|
|
532
|
+
cls, items: collections.abc.Sequence[EventRequest] | None, read_only: bool
|
|
535
533
|
) -> Capability | list[Capability]:
|
|
536
534
|
if not items and items is not None:
|
|
537
535
|
return []
|
|
@@ -551,35 +549,38 @@ class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
|
|
|
551
549
|
|
|
552
550
|
return capabilities.EventsAcl(actions, scope)
|
|
553
551
|
|
|
554
|
-
def create(self, items: Sequence[
|
|
555
|
-
return self.client.events.create(items)
|
|
552
|
+
def create(self, items: collections.abc.Sequence[EventRequest]) -> list[EventResponse]:
|
|
553
|
+
return self.client.tool.events.create(items)
|
|
556
554
|
|
|
557
|
-
def retrieve(self, ids: SequenceNotStr[
|
|
558
|
-
return self.client.events.
|
|
555
|
+
def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[EventResponse]:
|
|
556
|
+
return self.client.tool.events.retrieve(list(ids), ignore_unknown_ids=True)
|
|
559
557
|
|
|
560
|
-
def update(self, items: Sequence[
|
|
561
|
-
return self.client.events.update(items, mode="replace")
|
|
558
|
+
def update(self, items: collections.abc.Sequence[EventRequest]) -> list[EventResponse]:
|
|
559
|
+
return self.client.tool.events.update(items, mode="replace")
|
|
562
560
|
|
|
563
|
-
def delete(self, ids: SequenceNotStr[
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
non_existing = set(e.failed or [])
|
|
569
|
-
if existing := [id_ for id_ in ids if id_ not in non_existing]:
|
|
570
|
-
internal_ids, external_ids = self._split_ids(existing)
|
|
571
|
-
self.client.events.delete(id=internal_ids, external_id=external_ids)
|
|
572
|
-
return len(existing)
|
|
573
|
-
else:
|
|
574
|
-
return len(ids)
|
|
561
|
+
def delete(self, ids: SequenceNotStr[InternalOrExternalId]) -> int:
|
|
562
|
+
if not ids:
|
|
563
|
+
return 0
|
|
564
|
+
self.client.tool.events.delete(list(ids), ignore_unknown_ids=True)
|
|
565
|
+
return len(ids)
|
|
575
566
|
|
|
576
567
|
def _iterate(
|
|
577
568
|
self,
|
|
578
569
|
data_set_external_id: str | None = None,
|
|
579
570
|
space: str | None = None,
|
|
580
571
|
parent_ids: list[Hashable] | None = None,
|
|
581
|
-
) -> Iterable[
|
|
582
|
-
|
|
572
|
+
) -> Iterable[EventResponse]:
|
|
573
|
+
cursor: str | None = None
|
|
574
|
+
while True:
|
|
575
|
+
page = self.client.tool.events.iterate(
|
|
576
|
+
data_set_external_ids=[data_set_external_id] if data_set_external_id else None,
|
|
577
|
+
limit=1000,
|
|
578
|
+
cursor=cursor,
|
|
579
|
+
)
|
|
580
|
+
yield from page.items
|
|
581
|
+
if not page.next_cursor or not page.items:
|
|
582
|
+
break
|
|
583
|
+
cursor = page.next_cursor
|
|
583
584
|
|
|
584
585
|
@classmethod
|
|
585
586
|
def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
|
|
@@ -594,15 +595,15 @@ class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
|
|
|
594
595
|
if isinstance(asset_id, str):
|
|
595
596
|
yield AssetCRUD, ExternalId(external_id=asset_id)
|
|
596
597
|
|
|
597
|
-
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) ->
|
|
598
|
-
if ds_external_id := resource.
|
|
598
|
+
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> EventRequest:
|
|
599
|
+
if ds_external_id := resource.pop("dataSetExternalId", None):
|
|
599
600
|
resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
|
|
600
601
|
if asset_external_ids := resource.pop("assetExternalIds", []):
|
|
601
602
|
resource["assetIds"] = self.client.lookup.assets.id(asset_external_ids, is_dry_run)
|
|
602
|
-
return
|
|
603
|
+
return EventRequest.model_validate(resource)
|
|
603
604
|
|
|
604
|
-
def dump_resource(self, resource:
|
|
605
|
-
dumped = resource.
|
|
605
|
+
def dump_resource(self, resource: EventResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
606
|
+
dumped = resource.as_request_resource().dump()
|
|
606
607
|
local = local or {}
|
|
607
608
|
if data_set_id := dumped.pop("dataSetId", None):
|
|
608
609
|
dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
|
|
@@ -15,13 +15,8 @@
|
|
|
15
15
|
|
|
16
16
|
from collections.abc import Hashable, Iterable, Sequence
|
|
17
17
|
from datetime import date, datetime
|
|
18
|
-
from typing import Any,
|
|
18
|
+
from typing import Any, final
|
|
19
19
|
|
|
20
|
-
from cognite.client.data_classes import (
|
|
21
|
-
FileMetadata,
|
|
22
|
-
FileMetadataList,
|
|
23
|
-
FileMetadataWrite,
|
|
24
|
-
)
|
|
25
20
|
from cognite.client.data_classes.capabilities import (
|
|
26
21
|
Capability,
|
|
27
22
|
DataModelInstancesAcl,
|
|
@@ -31,9 +26,9 @@ from cognite.client.data_classes.data_modeling import NodeApplyResultList, NodeI
|
|
|
31
26
|
from cognite.client.exceptions import CogniteAPIError
|
|
32
27
|
from cognite.client.utils._time import convert_data_modelling_timestamp
|
|
33
28
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
34
|
-
from rich import print
|
|
35
29
|
|
|
36
|
-
from cognite_toolkit._cdf_tk.client.data_classes.
|
|
30
|
+
from cognite_toolkit._cdf_tk.client.data_classes.filemetadata import FileMetadataRequest, FileMetadataResponse
|
|
31
|
+
from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId, InternalOrExternalId
|
|
37
32
|
from cognite_toolkit._cdf_tk.client.data_classes.legacy.extendable_cognite_file import (
|
|
38
33
|
ExtendableCogniteFile,
|
|
39
34
|
ExtendableCogniteFileApply,
|
|
@@ -56,11 +51,11 @@ from .datamodel import SpaceCRUD, ViewCRUD
|
|
|
56
51
|
|
|
57
52
|
|
|
58
53
|
@final
|
|
59
|
-
class FileMetadataCRUD(ResourceContainerCRUD[
|
|
54
|
+
class FileMetadataCRUD(ResourceContainerCRUD[ExternalId, FileMetadataRequest, FileMetadataResponse]):
|
|
60
55
|
item_name = "file contents"
|
|
61
56
|
folder_name = "files"
|
|
62
|
-
resource_cls =
|
|
63
|
-
resource_write_cls =
|
|
57
|
+
resource_cls = FileMetadataResponse
|
|
58
|
+
resource_write_cls = FileMetadataRequest
|
|
64
59
|
yaml_cls = FileMetadataYAML
|
|
65
60
|
kind = "FileMetadata"
|
|
66
61
|
dependencies = frozenset({DataSetsCRUD, GroupAllScopedCRUD, LabelCRUD, AssetCRUD})
|
|
@@ -73,7 +68,7 @@ class FileMetadataCRUD(ResourceContainerCRUD[str, FileMetadataWrite, FileMetadat
|
|
|
73
68
|
|
|
74
69
|
@classmethod
|
|
75
70
|
def get_required_capability(
|
|
76
|
-
cls, items: Sequence[
|
|
71
|
+
cls, items: Sequence[FileMetadataRequest] | None, read_only: bool
|
|
77
72
|
) -> Capability | list[Capability]:
|
|
78
73
|
if not items and items is not None:
|
|
79
74
|
return []
|
|
@@ -88,22 +83,22 @@ class FileMetadataCRUD(ResourceContainerCRUD[str, FileMetadataWrite, FileMetadat
|
|
|
88
83
|
return FilesAcl(actions, scope)
|
|
89
84
|
|
|
90
85
|
@classmethod
|
|
91
|
-
def get_id(cls, item:
|
|
86
|
+
def get_id(cls, item: FileMetadataRequest | FileMetadataResponse | dict) -> ExternalId:
|
|
92
87
|
if isinstance(item, dict):
|
|
93
|
-
return item["externalId"]
|
|
88
|
+
return ExternalId(external_id=item["externalId"])
|
|
94
89
|
if item.external_id is None:
|
|
95
90
|
raise ToolkitRequiredValueError("FileMetadata must have external_id set.")
|
|
96
|
-
return item.external_id
|
|
91
|
+
return ExternalId(external_id=item.external_id)
|
|
97
92
|
|
|
98
93
|
@classmethod
|
|
99
|
-
def get_internal_id(cls, item:
|
|
94
|
+
def get_internal_id(cls, item: FileMetadataResponse | dict) -> int:
|
|
100
95
|
if isinstance(item, dict):
|
|
101
96
|
return item["id"]
|
|
102
97
|
return item.id
|
|
103
98
|
|
|
104
99
|
@classmethod
|
|
105
|
-
def dump_id(cls, id:
|
|
106
|
-
return
|
|
100
|
+
def dump_id(cls, id: ExternalId) -> dict[str, Any]:
|
|
101
|
+
return id.dump()
|
|
107
102
|
|
|
108
103
|
@classmethod
|
|
109
104
|
def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
|
|
@@ -121,69 +116,71 @@ class FileMetadataCRUD(ResourceContainerCRUD[str, FileMetadataWrite, FileMetadat
|
|
|
121
116
|
for asset_external_id in item.get("assetExternalIds", []):
|
|
122
117
|
yield AssetCRUD, ExternalId(external_id=asset_external_id)
|
|
123
118
|
|
|
124
|
-
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) ->
|
|
125
|
-
if resource.
|
|
126
|
-
ds_external_id = resource.pop("dataSetExternalId")
|
|
119
|
+
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> FileMetadataRequest:
|
|
120
|
+
if ds_external_id := resource.pop("dataSetExternalId", None):
|
|
127
121
|
resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
|
|
128
122
|
if security_categories_names := resource.pop("securityCategoryNames", []):
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
123
|
+
resource["securityCategories"] = self.client.lookup.security_categories.id(
|
|
124
|
+
security_categories_names, is_dry_run
|
|
125
|
+
)
|
|
126
|
+
if asset_external_ids := resource.pop("assetExternalIds", None):
|
|
127
|
+
resource["assetIds"] = self.client.lookup.assets.id(asset_external_ids, is_dry_run)
|
|
128
|
+
return FileMetadataRequest.model_validate(resource)
|
|
129
|
+
|
|
130
|
+
def dump_resource(self, resource: FileMetadataResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
131
|
+
dumped = resource.as_request_resource().dump()
|
|
132
|
+
if data_set_id := dumped.pop("dataSetId", None):
|
|
133
|
+
dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
|
|
139
134
|
if security_categories := dumped.pop("securityCategories", []):
|
|
140
135
|
dumped["securityCategoryNames"] = self.client.lookup.security_categories.external_id(security_categories)
|
|
141
136
|
if asset_ids := dumped.pop("assetIds", []):
|
|
142
137
|
dumped["assetExternalIds"] = self.client.lookup.assets.external_id(asset_ids)
|
|
143
138
|
return dumped
|
|
144
139
|
|
|
145
|
-
def create(self, items: Sequence[
|
|
146
|
-
|
|
147
|
-
for meta in items:
|
|
148
|
-
try:
|
|
149
|
-
created.append(self.client.files.create(meta))
|
|
150
|
-
except CogniteAPIError as e:
|
|
151
|
-
if e.code == 409:
|
|
152
|
-
print(f" [bold yellow]WARNING:[/] File {meta.external_id} already exists, skipping upload.")
|
|
153
|
-
return created
|
|
140
|
+
def create(self, items: Sequence[FileMetadataRequest]) -> list[FileMetadataResponse]:
|
|
141
|
+
return self.client.tool.filemetadata.create(items, overwrite=True)
|
|
154
142
|
|
|
155
|
-
def retrieve(self, ids: SequenceNotStr[
|
|
156
|
-
return self.client.
|
|
143
|
+
def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[FileMetadataResponse]:
|
|
144
|
+
return self.client.tool.filemetadata.retrieve(list(ids), ignore_unknown_ids=True)
|
|
157
145
|
|
|
158
|
-
def update(self, items: Sequence[
|
|
159
|
-
return self.client.
|
|
146
|
+
def update(self, items: Sequence[FileMetadataRequest]) -> list[FileMetadataResponse]:
|
|
147
|
+
return self.client.tool.filemetadata.update(items, mode="replace")
|
|
160
148
|
|
|
161
|
-
def delete(self, ids:
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
149
|
+
def delete(self, ids: SequenceNotStr[InternalOrExternalId]) -> int:
|
|
150
|
+
if not ids:
|
|
151
|
+
return 0
|
|
152
|
+
self.client.tool.filemetadata.delete(list(ids), ignore_unknown_ids=True)
|
|
153
|
+
return len(ids)
|
|
165
154
|
|
|
166
155
|
def _iterate(
|
|
167
156
|
self,
|
|
168
157
|
data_set_external_id: str | None = None,
|
|
169
158
|
space: str | None = None,
|
|
170
159
|
parent_ids: list[Hashable] | None = None,
|
|
171
|
-
) -> Iterable[
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
160
|
+
) -> Iterable[FileMetadataResponse]:
|
|
161
|
+
cursor: str | None = None
|
|
162
|
+
while True:
|
|
163
|
+
page = self.client.tool.filemetadata.iterate(
|
|
164
|
+
data_set_external_ids=[data_set_external_id] if data_set_external_id else None,
|
|
165
|
+
limit=1000,
|
|
166
|
+
cursor=cursor,
|
|
167
|
+
)
|
|
168
|
+
yield from page.items
|
|
169
|
+
if not page.next_cursor or not page.items:
|
|
170
|
+
break
|
|
171
|
+
cursor = page.next_cursor
|
|
172
|
+
|
|
173
|
+
def count(self, ids: SequenceNotStr[ExternalId]) -> int:
|
|
175
174
|
return sum(
|
|
176
|
-
1
|
|
177
|
-
for meta in self.client.files.retrieve_multiple(external_ids=list(ids), ignore_unknown_ids=True)
|
|
178
|
-
if meta.uploaded
|
|
175
|
+
1 for meta in self.client.tool.filemetadata.retrieve(list(ids), ignore_unknown_ids=True) if meta.uploaded
|
|
179
176
|
)
|
|
180
177
|
|
|
181
|
-
def drop_data(self, ids: SequenceNotStr[
|
|
182
|
-
existing = self.client.
|
|
178
|
+
def drop_data(self, ids: SequenceNotStr[ExternalId]) -> int:
|
|
179
|
+
existing = self.client.tool.filemetadata.retrieve(list(ids), ignore_unknown_ids=True)
|
|
183
180
|
# File and FileMetadata is tightly coupled, so we need to delete the metadata and recreate it
|
|
184
181
|
# without the source set to delete the file.
|
|
185
|
-
deleted_files = self.delete(
|
|
186
|
-
self.create(
|
|
182
|
+
deleted_files = self.delete([meta.as_id() for meta in existing])
|
|
183
|
+
self.create([meta.as_request_resource() for meta in existing])
|
|
187
184
|
return deleted_files
|
|
188
185
|
|
|
189
186
|
|