cognite-toolkit 0.7.42__py3-none-any.whl → 0.7.44__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. cognite_toolkit/_cdf_tk/client/_toolkit_client.py +7 -1
  2. cognite_toolkit/_cdf_tk/client/api/events.py +20 -2
  3. cognite_toolkit/_cdf_tk/client/api/filemetadata.py +145 -0
  4. cognite_toolkit/_cdf_tk/client/api/raw.py +174 -0
  5. cognite_toolkit/_cdf_tk/client/api/simulator_models.py +118 -0
  6. cognite_toolkit/_cdf_tk/client/api/simulators.py +8 -0
  7. cognite_toolkit/_cdf_tk/client/api/timeseries.py +20 -2
  8. cognite_toolkit/_cdf_tk/client/cdf_client/__init__.py +2 -1
  9. cognite_toolkit/_cdf_tk/client/cdf_client/api.py +40 -6
  10. cognite_toolkit/_cdf_tk/client/data_classes/agent.py +6 -9
  11. cognite_toolkit/_cdf_tk/client/data_classes/annotation.py +79 -0
  12. cognite_toolkit/_cdf_tk/client/data_classes/asset.py +7 -14
  13. cognite_toolkit/_cdf_tk/client/data_classes/base.py +15 -5
  14. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/__init__.py +164 -0
  15. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_constraints.py +37 -0
  16. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_container.py +50 -0
  17. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_data_model.py +73 -0
  18. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_data_types.py +116 -0
  19. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_indexes.py +26 -0
  20. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_instance.py +143 -0
  21. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_references.py +86 -0
  22. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_space.py +26 -0
  23. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_view.py +143 -0
  24. cognite_toolkit/_cdf_tk/client/data_classes/data_modeling/_view_property.py +152 -0
  25. cognite_toolkit/_cdf_tk/client/data_classes/dataset.py +35 -0
  26. cognite_toolkit/_cdf_tk/client/data_classes/event.py +12 -15
  27. cognite_toolkit/_cdf_tk/client/data_classes/extraction_pipeline.py +59 -0
  28. cognite_toolkit/_cdf_tk/client/data_classes/filemetadata.py +15 -19
  29. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_destination.py +34 -0
  30. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_job.py +134 -0
  31. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_mapping.py +72 -0
  32. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/__init__.py +63 -0
  33. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_auth.py +63 -0
  34. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_base.py +26 -0
  35. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_certificate.py +20 -0
  36. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_eventhub.py +31 -0
  37. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_kafka.py +53 -0
  38. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_mqtt.py +36 -0
  39. cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_rest.py +49 -0
  40. cognite_toolkit/_cdf_tk/client/data_classes/identifiers.py +8 -0
  41. cognite_toolkit/_cdf_tk/client/data_classes/label.py +27 -0
  42. cognite_toolkit/_cdf_tk/client/data_classes/raw.py +3 -2
  43. cognite_toolkit/_cdf_tk/client/data_classes/securitycategory.py +24 -0
  44. cognite_toolkit/_cdf_tk/client/data_classes/sequence.py +45 -0
  45. cognite_toolkit/_cdf_tk/client/data_classes/simulator_model.py +50 -0
  46. cognite_toolkit/_cdf_tk/client/data_classes/timeseries.py +15 -18
  47. cognite_toolkit/_cdf_tk/client/data_classes/transformation.py +140 -0
  48. cognite_toolkit/_cdf_tk/client/data_classes/workflow.py +27 -0
  49. cognite_toolkit/_cdf_tk/client/data_classes/workflow_trigger.py +63 -0
  50. cognite_toolkit/_cdf_tk/client/data_classes/workflow_version.py +155 -0
  51. cognite_toolkit/_cdf_tk/client/testing.py +6 -1
  52. cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +10 -7
  53. cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +4 -4
  54. cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +7 -3
  55. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py +5 -1
  56. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py +40 -39
  57. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py +56 -59
  58. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py +3 -3
  59. cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py +48 -47
  60. cognite_toolkit/_cdf_tk/resource_classes/__init__.py +2 -0
  61. cognite_toolkit/_cdf_tk/resource_classes/simulator_model.py +17 -0
  62. cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +84 -71
  63. cognite_toolkit/_cdf_tk/storageio/_file_content.py +22 -19
  64. cognite_toolkit/_cdf_tk/utils/useful_types2.py +5 -3
  65. cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
  66. cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
  67. cognite_toolkit/_resources/cdf.toml +1 -1
  68. cognite_toolkit/_version.py +1 -1
  69. {cognite_toolkit-0.7.42.dist-info → cognite_toolkit-0.7.44.dist-info}/METADATA +11 -1
  70. {cognite_toolkit-0.7.42.dist-info → cognite_toolkit-0.7.44.dist-info}/RECORD +72 -34
  71. {cognite_toolkit-0.7.42.dist-info → cognite_toolkit-0.7.44.dist-info}/WHEEL +1 -1
  72. {cognite_toolkit-0.7.42.dist-info → cognite_toolkit-0.7.44.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,155 @@
1
+ from typing import Annotated, Any, Literal
2
+
3
+ from pydantic import Field, JsonValue, field_validator
4
+ from pydantic_core.core_schema import ValidationInfo
5
+
6
+ from cognite_toolkit._cdf_tk.client.data_classes.base import (
7
+ BaseModelObject,
8
+ RequestResource,
9
+ ResponseResource,
10
+ )
11
+
12
+ from .identifiers import Identifier, WorkflowVersionId
13
+
14
+
15
+ class TaskId(Identifier):
16
+ external_id: str
17
+
18
+ def __str__(self) -> str:
19
+ return f"externalId='{self.external_id}'"
20
+
21
+
22
+ class TaskParameterDefinition(BaseModelObject):
23
+ type: str
24
+
25
+
26
+ class CogniteFunctionRef(BaseModelObject):
27
+ external_id: str
28
+ data: JsonValue | str | None = None
29
+
30
+
31
+ class FunctionTaskParameters(TaskParameterDefinition):
32
+ type: Literal["function"] = Field("function", exclude=True)
33
+ function: CogniteFunctionRef
34
+ is_async_complete: bool | None = None
35
+
36
+
37
+ class TransformationRef(BaseModelObject):
38
+ external_id: str
39
+ concurrency_policy: Literal["fail", "waitForCurrent", "restartAfterCurrent"] | None = None
40
+ use_transformation_credentials: bool | None = None
41
+
42
+
43
+ class TransformationTaskParameters(TaskParameterDefinition):
44
+ type: Literal["transformation"] = Field("transformation", exclude=True)
45
+ transformation: TransformationRef
46
+
47
+
48
+ class CDFRequest(BaseModelObject):
49
+ resource_path: str
50
+ method: str
51
+ query_parameters: JsonValue | str | None = None
52
+ body: JsonValue | str | None = None
53
+ request_timeout_in_millis: float | str | None = None
54
+ cdf_version_header: str | None = None
55
+
56
+
57
+ class CDFTaskParameters(TaskParameterDefinition):
58
+ type: Literal["cdf"] = Field("cdf", exclude=True)
59
+ cdf_request: CDFRequest
60
+
61
+
62
+ class DynamicRef(BaseModelObject):
63
+ tasks: str
64
+
65
+
66
+ class DynamicTaskParameters(TaskParameterDefinition):
67
+ type: Literal["dynamic"] = Field("dynamic", exclude=True)
68
+ dynamic: DynamicRef
69
+
70
+
71
+ class SubworkflowRef(BaseModelObject):
72
+ tasks: "list[Task]"
73
+
74
+
75
+ class SubworkflowTaskParameters(TaskParameterDefinition):
76
+ type: Literal["subworkflow"] = Field("subworkflow", exclude=True)
77
+ subworkflow: WorkflowVersionId | SubworkflowRef
78
+
79
+
80
+ class SimulatorInputUnit(BaseModelObject):
81
+ name: str
82
+
83
+
84
+ class SimulatorInput(BaseModelObject):
85
+ reference_id: str
86
+ value: str | int | float | list[str] | list[int] | list[float]
87
+ unit: SimulatorInputUnit | None = None
88
+
89
+
90
+ class SimulationRef(BaseModelObject):
91
+ routine_external_id: str
92
+ run_time: int | None = None
93
+ inputs: list[SimulatorInput] | None = None
94
+
95
+
96
+ class SimulationTaskParameters(TaskParameterDefinition):
97
+ type: Literal["simulation"] = Field("simulation", exclude=True)
98
+ simulation: SimulationRef
99
+
100
+
101
+ Parameter = Annotated[
102
+ FunctionTaskParameters
103
+ | TransformationTaskParameters
104
+ | CDFTaskParameters
105
+ | DynamicTaskParameters
106
+ | SubworkflowTaskParameters
107
+ | SimulationTaskParameters,
108
+ Field(discriminator="type"),
109
+ ]
110
+
111
+
112
+ class Task(BaseModelObject):
113
+ external_id: str
114
+ type: str
115
+ name: str | None = None
116
+ description: str | None = None
117
+ retries: int | None = None
118
+ timeout: int | None = None
119
+ on_failure: Literal["abortWorkflow", "skipTask"] = "abortWorkflow"
120
+ depends_on: list[TaskId] | None = None
121
+ parameters: Parameter | None = None
122
+
123
+ @field_validator("parameters", mode="before")
124
+ @classmethod
125
+ def move_type_to_field(cls, value: Any, info: ValidationInfo) -> Any:
126
+ if not isinstance(value, dict) or "type" not in info.data:
127
+ return value
128
+ value = dict(value)
129
+ value["type"] = info.data["type"]
130
+ return value
131
+
132
+
133
+ class WorkflowDefinition(BaseModelObject):
134
+ description: str | None = None
135
+ tasks: list[Task]
136
+
137
+
138
+ class WorkflowVersion(BaseModelObject):
139
+ workflow_external_id: str
140
+ version: str
141
+ workflow_definition: WorkflowDefinition
142
+
143
+ def as_id(self) -> WorkflowVersionId:
144
+ return WorkflowVersionId(workflow_external_id=self.workflow_external_id, version=self.version)
145
+
146
+
147
+ class WorkflowVersionRequest(WorkflowVersion, RequestResource): ...
148
+
149
+
150
+ class WorkflowVersionResponse(WorkflowVersion, ResponseResource[WorkflowVersionRequest]):
151
+ created_time: int
152
+ last_updated_time: int
153
+
154
+ def as_request_resource(self) -> WorkflowVersionRequest:
155
+ return WorkflowVersionRequest.model_validate(self.dump(), extra="ignore")
@@ -7,12 +7,14 @@ from cognite.client._api.datapoints import DatapointsAPI
7
7
  from cognite.client._api.datapoints_subscriptions import DatapointsSubscriptionAPI
8
8
  from cognite.client._api.functions import FunctionCallsAPI, FunctionSchedulesAPI
9
9
  from cognite.client._api.raw import RawDatabasesAPI, RawRowsAPI, RawTablesAPI
10
+ from cognite.client._api.simulators import SimulatorModelsAPI, SimulatorsAPI
10
11
  from cognite.client._api.synthetic_time_series import SyntheticDatapointsAPI
11
12
  from cognite.client.testing import CogniteClientMock
12
13
  from rich.console import Console
13
14
 
14
15
  from cognite_toolkit._cdf_tk.client._toolkit_client import ToolkitClient
15
16
  from cognite_toolkit._cdf_tk.client.api.assets import AssetsAPI
17
+ from cognite_toolkit._cdf_tk.client.api.filemetadata import FileMetadataAPI
16
18
  from cognite_toolkit._cdf_tk.client.api.legacy.canvas import CanvasAPI, IndustrialCanvasAPI
17
19
  from cognite_toolkit._cdf_tk.client.api.legacy.charts import ChartsAPI
18
20
  from cognite_toolkit._cdf_tk.client.api.legacy.dml import DMLAPI
@@ -142,8 +144,11 @@ class ToolkitClientMock(CogniteClientMock):
142
144
  self.tool.three_d = MagicMock(spec=ThreeDAPI)
143
145
  self.tool.three_d.models = MagicMock(spec_set=ThreeDModelAPI)
144
146
  self.tool.assets = MagicMock(spec_set=AssetsAPI)
145
- self.tool.time_series = MagicMock(spec_set=TimeSeriesAPI)
147
+ self.tool.timeseries = MagicMock(spec_set=TimeSeriesAPI)
148
+ self.tool.filemetadata = MagicMock(spec_set=FileMetadataAPI)
146
149
  self.tool.events = MagicMock(spec_set=EventsAPI)
150
+ self.tool.simulators = MagicMock(spec=SimulatorsAPI)
151
+ self.tool.simulators.models = MagicMock(spec_set=SimulatorModelsAPI)
147
152
 
148
153
  self.streams = MagicMock(spec=StreamsAPI)
149
154
 
@@ -1,7 +1,7 @@
1
1
  from collections.abc import Iterable, Mapping, Set
2
2
  from typing import Any, ClassVar, cast
3
3
 
4
- from cognite.client.data_classes import Annotation, Event, FileMetadata, TimeSeries
4
+ from cognite.client.data_classes import Annotation
5
5
  from cognite.client.data_classes.data_modeling import (
6
6
  DirectRelation,
7
7
  DirectRelationReference,
@@ -17,10 +17,13 @@ from cognite.client.utils._identifier import InstanceId
17
17
 
18
18
  from cognite_toolkit._cdf_tk.client import ToolkitClient
19
19
  from cognite_toolkit._cdf_tk.client.data_classes.asset import AssetResponse
20
+ from cognite_toolkit._cdf_tk.client.data_classes.event import EventResponse
21
+ from cognite_toolkit._cdf_tk.client.data_classes.filemetadata import FileMetadataResponse
20
22
  from cognite_toolkit._cdf_tk.client.data_classes.legacy.migration import (
21
23
  AssetCentricId,
22
24
  ResourceViewMappingApply,
23
25
  )
26
+ from cognite_toolkit._cdf_tk.client.data_classes.timeseries import TimeSeriesResponse
24
27
  from cognite_toolkit._cdf_tk.utils.collection import flatten_dict_json_path
25
28
  from cognite_toolkit._cdf_tk.utils.dtype_conversion import (
26
29
  asset_centric_convert_to_primary_property,
@@ -115,17 +118,17 @@ class DirectRelationCache:
115
118
  source_ids.add(resource.source)
116
119
  if resource.parent_id is not None:
117
120
  asset_ids.add(resource.parent_id)
118
- elif isinstance(resource, FileMetadata):
121
+ elif isinstance(resource, FileMetadataResponse):
119
122
  if resource.source:
120
123
  source_ids.add(resource.source)
121
124
  if resource.asset_ids:
122
125
  asset_ids.update(resource.asset_ids)
123
- elif isinstance(resource, Event):
126
+ elif isinstance(resource, EventResponse):
124
127
  if resource.source:
125
128
  source_ids.add(resource.source)
126
129
  if resource.asset_ids:
127
130
  asset_ids.update(resource.asset_ids)
128
- elif isinstance(resource, TimeSeries):
131
+ elif isinstance(resource, TimeSeriesResponse):
129
132
  if resource.asset_id is not None:
130
133
  asset_ids.add(resource.asset_id)
131
134
  if asset_ids:
@@ -246,11 +249,11 @@ def asset_centric_to_dm(
246
249
  def _lookup_resource_type(resource_type: AssetCentricResourceExtended) -> AssetCentricTypeExtended:
247
250
  if isinstance(resource_type, AssetResponse):
248
251
  return "asset"
249
- elif isinstance(resource_type, FileMetadata):
252
+ elif isinstance(resource_type, FileMetadataResponse):
250
253
  return "file"
251
- elif isinstance(resource_type, Event):
254
+ elif isinstance(resource_type, EventResponse):
252
255
  return "event"
253
- elif isinstance(resource_type, TimeSeries):
256
+ elif isinstance(resource_type, TimeSeriesResponse):
254
257
  return "timeseries"
255
258
  elif isinstance(resource_type, Annotation):
256
259
  if resource_type.annotated_resource_type == "file" and resource_type.annotation_type in (
@@ -449,15 +449,15 @@ class ThreeDMapper(DataMapper[ThreeDSelector, ThreeDModelResponse, ThreeDMigrati
449
449
  return None, issue
450
450
 
451
451
  mapped_request = ThreeDMigrationRequest(
452
- modelId=item.id,
452
+ model_id=item.id,
453
453
  type=model_type,
454
454
  space=instance_space,
455
455
  revision=ThreeDRevisionMigrationRequest(
456
456
  space=instance_space,
457
457
  type=model_type,
458
- revisionId=last_revision_id,
458
+ revision_id=last_revision_id,
459
459
  model=Model(
460
- instanceId=InstanceIdentifier(
460
+ instance_id=InstanceIdentifier(
461
461
  space=instance_space,
462
462
  external_id=f"cog_3d_model_{item.id!s}",
463
463
  )
@@ -508,7 +508,7 @@ class ThreeDAssetMapper(DataMapper[ThreeDSelector, AssetMappingResponse, AssetMa
508
508
  if asset_node_id is None:
509
509
  issue.error_message.append(f"Missing asset instance for asset ID {item.asset_id!r}")
510
510
  return None, issue
511
- asset_instance_id = NodeReference(space=asset_node_id.space, externalId=asset_node_id.external_id)
511
+ asset_instance_id = NodeReference(space=asset_node_id.space, external_id=asset_node_id.external_id)
512
512
 
513
513
  if asset_instance_id is None:
514
514
  issue.error_message.append("Neither assetInstanceId nor assetId provided for mapping.")
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast, final
6
6
  import pandas as pd
7
7
  from cognite.client.data_classes import FileMetadataWrite
8
8
 
9
+ from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId
9
10
  from cognite_toolkit._cdf_tk.client.data_classes.legacy.extendable_cognite_file import ExtendableCogniteFileApply
10
11
  from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawTable
11
12
  from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
@@ -119,11 +120,14 @@ class FileCRUD(DataCRUD):
119
120
  if not datafile.exists():
120
121
  continue
121
122
 
123
+ identifier = resource.identifier
124
+ if isinstance(identifier, ExternalId):
125
+ identifier = identifier.external_id
122
126
  if dry_run:
123
- yield f" Would upload file '{datafile!s}' to file with {id_name}={resource.identifier!r}", 1
127
+ yield f" Would upload file '{datafile!s}' to file with {id_name}={identifier!r}", 1
124
128
  else:
125
- self.client.files.upload_content(path=str(datafile), **{id_name: resource.identifier})
126
- yield f" Uploaded file '{datafile!s}' to file with {id_name}={resource.identifier!r}", 1
129
+ self.client.files.upload_content(path=str(datafile), **{id_name: identifier})
130
+ yield f" Uploaded file '{datafile!s}' to file with {id_name}={identifier!r}", 1
127
131
 
128
132
  @staticmethod
129
133
  def _read_metadata(
@@ -41,6 +41,7 @@ from rich.console import Console
41
41
  from rich.markup import escape
42
42
 
43
43
  from cognite_toolkit._cdf_tk.client import ToolkitClient
44
+ from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId
44
45
  from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawDatabase, RawTable
45
46
  from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
46
47
  from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError
@@ -190,7 +191,10 @@ class GroupCRUD(ResourceCRUD[str, GroupWrite, Group]):
190
191
  loader = LocationFilterCRUD
191
192
  if loader is not None and isinstance(ids, dict) and "ids" in ids:
192
193
  for id_ in ids["ids"]:
193
- yield loader, id_
194
+ if loader is TimeSeriesCRUD:
195
+ yield TimeSeriesCRUD, ExternalId(external_id=id_)
196
+ else:
197
+ yield loader, id_
194
198
 
195
199
  def _substitute_scope_ids(self, group: dict[str, Any], is_dry_run: bool, reverse: bool = False) -> dict[str, Any]:
196
200
  replace_method_by_acl = self._create_replace_method_by_acl_and_scope()
@@ -5,15 +5,12 @@ from pathlib import Path
5
5
  from typing import Any, final
6
6
 
7
7
  import pandas as pd
8
+ from cognite.client.data_classes import Sequence as CDFSequence
8
9
  from cognite.client.data_classes import (
9
- Event,
10
- EventList,
11
- EventWrite,
12
10
  SequenceList,
13
11
  SequenceWrite,
14
12
  capabilities,
15
13
  )
16
- from cognite.client.data_classes import Sequence as CDFSequence
17
14
  from cognite.client.data_classes.capabilities import Capability
18
15
  from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
19
16
  from cognite.client.utils.useful_types import SequenceNotStr
@@ -25,6 +22,7 @@ from cognite_toolkit._cdf_tk.client.data_classes.asset import (
25
22
  AssetRequest,
26
23
  AssetResponse,
27
24
  )
25
+ from cognite_toolkit._cdf_tk.client.data_classes.event import EventRequest, EventResponse
28
26
  from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId, InternalOrExternalId
29
27
  from cognite_toolkit._cdf_tk.client.data_classes.legacy.sequences import (
30
28
  ToolkitSequenceRows,
@@ -496,10 +494,10 @@ class SequenceRowCRUD(ResourceCRUD[str, ToolkitSequenceRowsWrite, ToolkitSequenc
496
494
 
497
495
 
498
496
  @final
499
- class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
497
+ class EventCRUD(ResourceCRUD[ExternalId, EventRequest, EventResponse]):
500
498
  folder_name = "classic"
501
- resource_cls = Event
502
- resource_write_cls = EventWrite
499
+ resource_cls = EventResponse
500
+ resource_write_cls = EventRequest
503
501
  yaml_cls = EventYAML
504
502
  kind = "Event"
505
503
  dependencies = frozenset({DataSetsCRUD, AssetCRUD})
@@ -510,15 +508,15 @@ class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
510
508
  return "events"
511
509
 
512
510
  @classmethod
513
- def get_id(cls, item: Event | EventWrite | dict) -> str:
511
+ def get_id(cls, item: EventRequest | EventResponse | dict) -> ExternalId:
514
512
  if isinstance(item, dict):
515
- return item["externalId"]
513
+ return ExternalId(external_id=item["externalId"])
516
514
  if not item.external_id:
517
515
  raise KeyError("Event must have external_id")
518
- return item.external_id
516
+ return ExternalId(external_id=item.external_id)
519
517
 
520
518
  @classmethod
521
- def get_internal_id(cls, item: Event | dict) -> int:
519
+ def get_internal_id(cls, item: EventResponse | dict) -> int:
522
520
  if isinstance(item, dict):
523
521
  return item["id"]
524
522
  if not item.id:
@@ -526,12 +524,12 @@ class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
526
524
  return item.id
527
525
 
528
526
  @classmethod
529
- def dump_id(cls, id: str) -> dict[str, Any]:
530
- return {"externalId": id}
527
+ def dump_id(cls, id: ExternalId) -> dict[str, Any]:
528
+ return id.dump()
531
529
 
532
530
  @classmethod
533
531
  def get_required_capability(
534
- cls, items: collections.abc.Sequence[EventWrite] | None, read_only: bool
532
+ cls, items: collections.abc.Sequence[EventRequest] | None, read_only: bool
535
533
  ) -> Capability | list[Capability]:
536
534
  if not items and items is not None:
537
535
  return []
@@ -551,35 +549,38 @@ class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
551
549
 
552
550
  return capabilities.EventsAcl(actions, scope)
553
551
 
554
- def create(self, items: Sequence[EventWrite]) -> EventList:
555
- return self.client.events.create(items)
552
+ def create(self, items: collections.abc.Sequence[EventRequest]) -> list[EventResponse]:
553
+ return self.client.tool.events.create(items)
556
554
 
557
- def retrieve(self, ids: SequenceNotStr[str]) -> EventList:
558
- return self.client.events.retrieve_multiple(external_ids=ids, ignore_unknown_ids=True)
555
+ def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[EventResponse]:
556
+ return self.client.tool.events.retrieve(list(ids), ignore_unknown_ids=True)
559
557
 
560
- def update(self, items: Sequence[EventWrite]) -> EventList:
561
- return self.client.events.update(items, mode="replace")
558
+ def update(self, items: collections.abc.Sequence[EventRequest]) -> list[EventResponse]:
559
+ return self.client.tool.events.update(items, mode="replace")
562
560
 
563
- def delete(self, ids: SequenceNotStr[str | int]) -> int:
564
- internal_ids, external_ids = self._split_ids(ids)
565
- try:
566
- self.client.events.delete(id=internal_ids, external_id=external_ids)
567
- except (CogniteAPIError, CogniteNotFoundError) as e:
568
- non_existing = set(e.failed or [])
569
- if existing := [id_ for id_ in ids if id_ not in non_existing]:
570
- internal_ids, external_ids = self._split_ids(existing)
571
- self.client.events.delete(id=internal_ids, external_id=external_ids)
572
- return len(existing)
573
- else:
574
- return len(ids)
561
+ def delete(self, ids: SequenceNotStr[InternalOrExternalId]) -> int:
562
+ if not ids:
563
+ return 0
564
+ self.client.tool.events.delete(list(ids), ignore_unknown_ids=True)
565
+ return len(ids)
575
566
 
576
567
  def _iterate(
577
568
  self,
578
569
  data_set_external_id: str | None = None,
579
570
  space: str | None = None,
580
571
  parent_ids: list[Hashable] | None = None,
581
- ) -> Iterable[Event]:
582
- return iter(self.client.events(data_set_external_ids=[data_set_external_id] if data_set_external_id else None))
572
+ ) -> Iterable[EventResponse]:
573
+ cursor: str | None = None
574
+ while True:
575
+ page = self.client.tool.events.iterate(
576
+ data_set_external_ids=[data_set_external_id] if data_set_external_id else None,
577
+ limit=1000,
578
+ cursor=cursor,
579
+ )
580
+ yield from page.items
581
+ if not page.next_cursor or not page.items:
582
+ break
583
+ cursor = page.next_cursor
583
584
 
584
585
  @classmethod
585
586
  def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
@@ -594,15 +595,15 @@ class EventCRUD(ResourceCRUD[str, EventWrite, Event]):
594
595
  if isinstance(asset_id, str):
595
596
  yield AssetCRUD, ExternalId(external_id=asset_id)
596
597
 
597
- def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> EventWrite:
598
- if ds_external_id := resource.get("dataSetExternalId", None):
598
+ def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> EventRequest:
599
+ if ds_external_id := resource.pop("dataSetExternalId", None):
599
600
  resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
600
601
  if asset_external_ids := resource.pop("assetExternalIds", []):
601
602
  resource["assetIds"] = self.client.lookup.assets.id(asset_external_ids, is_dry_run)
602
- return EventWrite._load(resource)
603
+ return EventRequest.model_validate(resource)
603
604
 
604
- def dump_resource(self, resource: Event, local: dict[str, Any] | None = None) -> dict[str, Any]:
605
- dumped = resource.as_write().dump()
605
+ def dump_resource(self, resource: EventResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
606
+ dumped = resource.as_request_resource().dump()
606
607
  local = local or {}
607
608
  if data_set_id := dumped.pop("dataSetId", None):
608
609
  dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
@@ -15,13 +15,8 @@
15
15
 
16
16
  from collections.abc import Hashable, Iterable, Sequence
17
17
  from datetime import date, datetime
18
- from typing import Any, cast, final
18
+ from typing import Any, final
19
19
 
20
- from cognite.client.data_classes import (
21
- FileMetadata,
22
- FileMetadataList,
23
- FileMetadataWrite,
24
- )
25
20
  from cognite.client.data_classes.capabilities import (
26
21
  Capability,
27
22
  DataModelInstancesAcl,
@@ -31,9 +26,9 @@ from cognite.client.data_classes.data_modeling import NodeApplyResultList, NodeI
31
26
  from cognite.client.exceptions import CogniteAPIError
32
27
  from cognite.client.utils._time import convert_data_modelling_timestamp
33
28
  from cognite.client.utils.useful_types import SequenceNotStr
34
- from rich import print
35
29
 
36
- from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId
30
+ from cognite_toolkit._cdf_tk.client.data_classes.filemetadata import FileMetadataRequest, FileMetadataResponse
31
+ from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId, InternalOrExternalId
37
32
  from cognite_toolkit._cdf_tk.client.data_classes.legacy.extendable_cognite_file import (
38
33
  ExtendableCogniteFile,
39
34
  ExtendableCogniteFileApply,
@@ -56,11 +51,11 @@ from .datamodel import SpaceCRUD, ViewCRUD
56
51
 
57
52
 
58
53
  @final
59
- class FileMetadataCRUD(ResourceContainerCRUD[str, FileMetadataWrite, FileMetadata]):
54
+ class FileMetadataCRUD(ResourceContainerCRUD[ExternalId, FileMetadataRequest, FileMetadataResponse]):
60
55
  item_name = "file contents"
61
56
  folder_name = "files"
62
- resource_cls = FileMetadata
63
- resource_write_cls = FileMetadataWrite
57
+ resource_cls = FileMetadataResponse
58
+ resource_write_cls = FileMetadataRequest
64
59
  yaml_cls = FileMetadataYAML
65
60
  kind = "FileMetadata"
66
61
  dependencies = frozenset({DataSetsCRUD, GroupAllScopedCRUD, LabelCRUD, AssetCRUD})
@@ -73,7 +68,7 @@ class FileMetadataCRUD(ResourceContainerCRUD[str, FileMetadataWrite, FileMetadat
73
68
 
74
69
  @classmethod
75
70
  def get_required_capability(
76
- cls, items: Sequence[FileMetadataWrite] | None, read_only: bool
71
+ cls, items: Sequence[FileMetadataRequest] | None, read_only: bool
77
72
  ) -> Capability | list[Capability]:
78
73
  if not items and items is not None:
79
74
  return []
@@ -88,22 +83,22 @@ class FileMetadataCRUD(ResourceContainerCRUD[str, FileMetadataWrite, FileMetadat
88
83
  return FilesAcl(actions, scope)
89
84
 
90
85
  @classmethod
91
- def get_id(cls, item: FileMetadata | FileMetadataWrite | dict) -> str:
86
+ def get_id(cls, item: FileMetadataRequest | FileMetadataResponse | dict) -> ExternalId:
92
87
  if isinstance(item, dict):
93
- return item["externalId"]
88
+ return ExternalId(external_id=item["externalId"])
94
89
  if item.external_id is None:
95
90
  raise ToolkitRequiredValueError("FileMetadata must have external_id set.")
96
- return item.external_id
91
+ return ExternalId(external_id=item.external_id)
97
92
 
98
93
  @classmethod
99
- def get_internal_id(cls, item: FileMetadata | dict) -> int:
94
+ def get_internal_id(cls, item: FileMetadataResponse | dict) -> int:
100
95
  if isinstance(item, dict):
101
96
  return item["id"]
102
97
  return item.id
103
98
 
104
99
  @classmethod
105
- def dump_id(cls, id: str) -> dict[str, Any]:
106
- return {"externalId": id}
100
+ def dump_id(cls, id: ExternalId) -> dict[str, Any]:
101
+ return id.dump()
107
102
 
108
103
  @classmethod
109
104
  def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
@@ -121,69 +116,71 @@ class FileMetadataCRUD(ResourceContainerCRUD[str, FileMetadataWrite, FileMetadat
121
116
  for asset_external_id in item.get("assetExternalIds", []):
122
117
  yield AssetCRUD, ExternalId(external_id=asset_external_id)
123
118
 
124
- def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> FileMetadataWrite:
125
- if resource.get("dataSetExternalId") is not None:
126
- ds_external_id = resource.pop("dataSetExternalId")
119
+ def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> FileMetadataRequest:
120
+ if ds_external_id := resource.pop("dataSetExternalId", None):
127
121
  resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
128
122
  if security_categories_names := resource.pop("securityCategoryNames", []):
129
- security_categories = self.client.lookup.security_categories.id(security_categories_names, is_dry_run)
130
- resource["securityCategories"] = security_categories
131
- if "assetExternalIds" in resource:
132
- resource["assetIds"] = self.client.lookup.assets.id(resource["assetExternalIds"], is_dry_run)
133
- return FileMetadataWrite._load(resource)
134
-
135
- def dump_resource(self, resource: FileMetadata, local: dict[str, Any] | None = None) -> dict[str, Any]:
136
- dumped = resource.as_write().dump()
137
- if ds_id := dumped.pop("dataSetId", None):
138
- dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(ds_id)
123
+ resource["securityCategories"] = self.client.lookup.security_categories.id(
124
+ security_categories_names, is_dry_run
125
+ )
126
+ if asset_external_ids := resource.pop("assetExternalIds", None):
127
+ resource["assetIds"] = self.client.lookup.assets.id(asset_external_ids, is_dry_run)
128
+ return FileMetadataRequest.model_validate(resource)
129
+
130
+ def dump_resource(self, resource: FileMetadataResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
131
+ dumped = resource.as_request_resource().dump()
132
+ if data_set_id := dumped.pop("dataSetId", None):
133
+ dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
139
134
  if security_categories := dumped.pop("securityCategories", []):
140
135
  dumped["securityCategoryNames"] = self.client.lookup.security_categories.external_id(security_categories)
141
136
  if asset_ids := dumped.pop("assetIds", []):
142
137
  dumped["assetExternalIds"] = self.client.lookup.assets.external_id(asset_ids)
143
138
  return dumped
144
139
 
145
- def create(self, items: Sequence[FileMetadataWrite]) -> FileMetadataList:
146
- created = FileMetadataList([])
147
- for meta in items:
148
- try:
149
- created.append(self.client.files.create(meta))
150
- except CogniteAPIError as e:
151
- if e.code == 409:
152
- print(f" [bold yellow]WARNING:[/] File {meta.external_id} already exists, skipping upload.")
153
- return created
140
+ def create(self, items: Sequence[FileMetadataRequest]) -> list[FileMetadataResponse]:
141
+ return self.client.tool.filemetadata.create(items, overwrite=True)
154
142
 
155
- def retrieve(self, ids: SequenceNotStr[str]) -> FileMetadataList:
156
- return self.client.files.retrieve_multiple(external_ids=ids, ignore_unknown_ids=True)
143
+ def retrieve(self, ids: SequenceNotStr[ExternalId]) -> list[FileMetadataResponse]:
144
+ return self.client.tool.filemetadata.retrieve(list(ids), ignore_unknown_ids=True)
157
145
 
158
- def update(self, items: Sequence[FileMetadataWrite]) -> FileMetadataList:
159
- return self.client.files.update(items, mode="replace")
146
+ def update(self, items: Sequence[FileMetadataRequest]) -> list[FileMetadataResponse]:
147
+ return self.client.tool.filemetadata.update(items, mode="replace")
160
148
 
161
- def delete(self, ids: str | int | SequenceNotStr[str | int] | None) -> int:
162
- internal_ids, external_ids = self._split_ids(ids)
163
- self.client.files.delete(id=internal_ids, external_id=external_ids)
164
- return len(cast(SequenceNotStr[str], ids))
149
+ def delete(self, ids: SequenceNotStr[InternalOrExternalId]) -> int:
150
+ if not ids:
151
+ return 0
152
+ self.client.tool.filemetadata.delete(list(ids), ignore_unknown_ids=True)
153
+ return len(ids)
165
154
 
166
155
  def _iterate(
167
156
  self,
168
157
  data_set_external_id: str | None = None,
169
158
  space: str | None = None,
170
159
  parent_ids: list[Hashable] | None = None,
171
- ) -> Iterable[FileMetadata]:
172
- return iter(self.client.files(data_set_external_ids=[data_set_external_id] if data_set_external_id else None))
173
-
174
- def count(self, ids: SequenceNotStr[str]) -> int:
160
+ ) -> Iterable[FileMetadataResponse]:
161
+ cursor: str | None = None
162
+ while True:
163
+ page = self.client.tool.filemetadata.iterate(
164
+ data_set_external_ids=[data_set_external_id] if data_set_external_id else None,
165
+ limit=1000,
166
+ cursor=cursor,
167
+ )
168
+ yield from page.items
169
+ if not page.next_cursor or not page.items:
170
+ break
171
+ cursor = page.next_cursor
172
+
173
+ def count(self, ids: SequenceNotStr[ExternalId]) -> int:
175
174
  return sum(
176
- 1
177
- for meta in self.client.files.retrieve_multiple(external_ids=list(ids), ignore_unknown_ids=True)
178
- if meta.uploaded
175
+ 1 for meta in self.client.tool.filemetadata.retrieve(list(ids), ignore_unknown_ids=True) if meta.uploaded
179
176
  )
180
177
 
181
- def drop_data(self, ids: SequenceNotStr[str]) -> int:
182
- existing = self.client.files.retrieve_multiple(external_ids=list(ids), ignore_unknown_ids=True)
178
+ def drop_data(self, ids: SequenceNotStr[ExternalId]) -> int:
179
+ existing = self.client.tool.filemetadata.retrieve(list(ids), ignore_unknown_ids=True)
183
180
  # File and FileMetadata is tightly coupled, so we need to delete the metadata and recreate it
184
181
  # without the source set to delete the file.
185
- deleted_files = self.delete(existing.as_external_ids())
186
- self.create(existing.as_write())
182
+ deleted_files = self.delete([meta.as_id() for meta in existing])
183
+ self.create([meta.as_request_resource() for meta in existing])
187
184
  return deleted_files
188
185
 
189
186