cognite-toolkit 0.7.10__py3-none-any.whl → 0.7.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/commands/_purge.py +27 -28
- cognite_toolkit/_cdf_tk/commands/_upload.py +5 -1
- cognite_toolkit/_cdf_tk/commands/clean.py +11 -13
- cognite_toolkit/_cdf_tk/commands/deploy.py +9 -17
- cognite_toolkit/_cdf_tk/commands/dump_resource.py +6 -4
- cognite_toolkit/_cdf_tk/commands/pull.py +6 -19
- cognite_toolkit/_cdf_tk/cruds/_base_cruds.py +7 -25
- cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +3 -6
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py +4 -6
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py +4 -12
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py +19 -36
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py +4 -10
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py +4 -12
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py +16 -41
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py +5 -15
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py +8 -21
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py +6 -22
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py +5 -15
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/hosted_extractors.py +12 -26
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/industrial_tool.py +3 -6
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/location.py +3 -14
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py +4 -8
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/raw.py +4 -8
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py +3 -6
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/robotics.py +15 -34
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py +2 -5
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/three_d_model.py +3 -6
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py +5 -13
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py +4 -19
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/workflow.py +20 -37
- cognite_toolkit/_cdf_tk/cruds/_worker.py +13 -30
- cognite_toolkit/_cdf_tk/storageio/_base.py +23 -0
- cognite_toolkit/_cdf_tk/storageio/_file_content.py +4 -0
- cognite_toolkit/_cdf_tk/utils/fileio/_readers.py +42 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.10.dist-info → cognite_toolkit-0.7.11.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.7.10.dist-info → cognite_toolkit-0.7.11.dist-info}/RECORD +43 -43
- {cognite_toolkit-0.7.10.dist-info → cognite_toolkit-0.7.11.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.7.10.dist-info → cognite_toolkit-0.7.11.dist-info}/entry_points.txt +0 -0
- {cognite_toolkit-0.7.10.dist-info → cognite_toolkit-0.7.11.dist-info}/licenses/LICENSE +0 -0
|
@@ -13,23 +13,18 @@ from cognite_toolkit._cdf_tk.client.data_classes.robotics import (
|
|
|
13
13
|
DataPostProcessing,
|
|
14
14
|
DataPostProcessingList,
|
|
15
15
|
DataPostProcessingWrite,
|
|
16
|
-
DataPostProcessingWriteList,
|
|
17
16
|
Frame,
|
|
18
17
|
FrameList,
|
|
19
18
|
FrameWrite,
|
|
20
|
-
FrameWriteList,
|
|
21
19
|
Location,
|
|
22
20
|
LocationList,
|
|
23
21
|
LocationWrite,
|
|
24
|
-
LocationWriteList,
|
|
25
22
|
Map,
|
|
26
23
|
MapList,
|
|
27
24
|
MapWrite,
|
|
28
|
-
MapWriteList,
|
|
29
25
|
RobotCapability,
|
|
30
26
|
RobotCapabilityList,
|
|
31
27
|
RobotCapabilityWrite,
|
|
32
|
-
RobotCapabilityWriteList,
|
|
33
28
|
)
|
|
34
29
|
from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
|
|
35
30
|
from cognite_toolkit._cdf_tk.resource_classes import (
|
|
@@ -42,12 +37,10 @@ from cognite_toolkit._cdf_tk.resource_classes import (
|
|
|
42
37
|
from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable
|
|
43
38
|
|
|
44
39
|
|
|
45
|
-
class RoboticFrameCRUD(ResourceCRUD[str, FrameWrite, Frame
|
|
40
|
+
class RoboticFrameCRUD(ResourceCRUD[str, FrameWrite, Frame]):
|
|
46
41
|
folder_name = "robotics"
|
|
47
42
|
resource_cls = Frame
|
|
48
43
|
resource_write_cls = FrameWrite
|
|
49
|
-
list_cls = FrameList
|
|
50
|
-
list_write_cls = FrameWriteList
|
|
51
44
|
kind = "Frame"
|
|
52
45
|
yaml_cls = RobotFrameYAML
|
|
53
46
|
_doc_url = "Frames/operation/createFrames"
|
|
@@ -84,13 +77,13 @@ class RoboticFrameCRUD(ResourceCRUD[str, FrameWrite, Frame, FrameWriteList, Fram
|
|
|
84
77
|
capabilities.RoboticsAcl.Scope.All(),
|
|
85
78
|
)
|
|
86
79
|
|
|
87
|
-
def create(self, items:
|
|
80
|
+
def create(self, items: Sequence[FrameWrite]) -> FrameList:
|
|
88
81
|
return self.client.robotics.frames.create(items)
|
|
89
82
|
|
|
90
83
|
def retrieve(self, ids: SequenceNotStr[str]) -> FrameList:
|
|
91
84
|
return _fallback_to_one_by_one(self.client.robotics.frames.retrieve, ids, FrameList)
|
|
92
85
|
|
|
93
|
-
def update(self, items:
|
|
86
|
+
def update(self, items: Sequence[FrameWrite]) -> FrameList:
|
|
94
87
|
return self.client.robotics.frames.update(items)
|
|
95
88
|
|
|
96
89
|
def delete(self, ids: SequenceNotStr[str]) -> int:
|
|
@@ -109,12 +102,10 @@ class RoboticFrameCRUD(ResourceCRUD[str, FrameWrite, Frame, FrameWriteList, Fram
|
|
|
109
102
|
return iter(self.client.robotics.frames)
|
|
110
103
|
|
|
111
104
|
|
|
112
|
-
class RoboticLocationCRUD(ResourceCRUD[str, LocationWrite, Location
|
|
105
|
+
class RoboticLocationCRUD(ResourceCRUD[str, LocationWrite, Location]):
|
|
113
106
|
folder_name = "robotics"
|
|
114
107
|
resource_cls = Location
|
|
115
108
|
resource_write_cls = LocationWrite
|
|
116
|
-
list_cls = LocationList
|
|
117
|
-
list_write_cls = LocationWriteList
|
|
118
109
|
kind = "Location"
|
|
119
110
|
yaml_cls = RobotLocationYAML
|
|
120
111
|
_doc_url = "Locations/operation/createLocations"
|
|
@@ -156,13 +147,13 @@ class RoboticLocationCRUD(ResourceCRUD[str, LocationWrite, Location, LocationWri
|
|
|
156
147
|
|
|
157
148
|
return capabilities.RoboticsAcl(actions, capabilities.RoboticsAcl.Scope.All())
|
|
158
149
|
|
|
159
|
-
def create(self, items:
|
|
150
|
+
def create(self, items: Sequence[LocationWrite]) -> LocationList:
|
|
160
151
|
return self.client.robotics.locations.create(items)
|
|
161
152
|
|
|
162
153
|
def retrieve(self, ids: SequenceNotStr[str]) -> LocationList:
|
|
163
154
|
return _fallback_to_one_by_one(self.client.robotics.locations.retrieve, ids, LocationList)
|
|
164
155
|
|
|
165
|
-
def update(self, items:
|
|
156
|
+
def update(self, items: Sequence[LocationWrite]) -> LocationList:
|
|
166
157
|
return self.client.robotics.locations.update(items)
|
|
167
158
|
|
|
168
159
|
def delete(self, ids: SequenceNotStr[str]) -> int:
|
|
@@ -181,14 +172,10 @@ class RoboticLocationCRUD(ResourceCRUD[str, LocationWrite, Location, LocationWri
|
|
|
181
172
|
return iter(self.client.robotics.locations)
|
|
182
173
|
|
|
183
174
|
|
|
184
|
-
class RoboticsDataPostProcessingCRUD(
|
|
185
|
-
ResourceCRUD[str, DataPostProcessingWrite, DataPostProcessing, DataPostProcessingWriteList, DataPostProcessingList]
|
|
186
|
-
):
|
|
175
|
+
class RoboticsDataPostProcessingCRUD(ResourceCRUD[str, DataPostProcessingWrite, DataPostProcessing]):
|
|
187
176
|
folder_name = "robotics"
|
|
188
177
|
resource_cls = DataPostProcessing
|
|
189
178
|
resource_write_cls = DataPostProcessingWrite
|
|
190
|
-
list_cls = DataPostProcessingList
|
|
191
|
-
list_write_cls = DataPostProcessingWriteList
|
|
192
179
|
kind = "DataPostProcessing"
|
|
193
180
|
yaml_cls = RobotDataPostProcessingYAML
|
|
194
181
|
_doc_url = "DataPostProcessing/operation/createDataPostProcessing"
|
|
@@ -230,13 +217,13 @@ class RoboticsDataPostProcessingCRUD(
|
|
|
230
217
|
|
|
231
218
|
return capabilities.RoboticsAcl(actions, capabilities.RoboticsAcl.Scope.All())
|
|
232
219
|
|
|
233
|
-
def create(self, items:
|
|
220
|
+
def create(self, items: Sequence[DataPostProcessingWrite]) -> DataPostProcessingList:
|
|
234
221
|
return self.client.robotics.data_postprocessing.create(items)
|
|
235
222
|
|
|
236
223
|
def retrieve(self, ids: SequenceNotStr[str]) -> DataPostProcessingList:
|
|
237
224
|
return _fallback_to_one_by_one(self.client.robotics.data_postprocessing.retrieve, ids, DataPostProcessingList)
|
|
238
225
|
|
|
239
|
-
def update(self, items:
|
|
226
|
+
def update(self, items: Sequence[DataPostProcessingWrite]) -> DataPostProcessingList:
|
|
240
227
|
# There is a bug in the /update endpoint that requires the input_schema to be a string
|
|
241
228
|
# and not an object. This is a workaround until the bug is fixed.
|
|
242
229
|
# We do the serialization to avoid modifying the original object.
|
|
@@ -274,14 +261,10 @@ class RoboticsDataPostProcessingCRUD(
|
|
|
274
261
|
return super().diff_list(local, cdf, json_path)
|
|
275
262
|
|
|
276
263
|
|
|
277
|
-
class RobotCapabilityCRUD(
|
|
278
|
-
ResourceCRUD[str, RobotCapabilityWrite, RobotCapability, RobotCapabilityWriteList, RobotCapabilityList]
|
|
279
|
-
):
|
|
264
|
+
class RobotCapabilityCRUD(ResourceCRUD[str, RobotCapabilityWrite, RobotCapability]):
|
|
280
265
|
folder_name = "robotics"
|
|
281
266
|
resource_cls = RobotCapability
|
|
282
267
|
resource_write_cls = RobotCapabilityWrite
|
|
283
|
-
list_cls = RobotCapabilityList
|
|
284
|
-
list_write_cls = RobotCapabilityWriteList
|
|
285
268
|
kind = "RobotCapability"
|
|
286
269
|
yaml_cls = RobotCapabilityYAML
|
|
287
270
|
_doc_url = "RobotCapabilities/operation/createRobotCapabilities"
|
|
@@ -323,13 +306,13 @@ class RobotCapabilityCRUD(
|
|
|
323
306
|
|
|
324
307
|
return capabilities.RoboticsAcl(actions, capabilities.RoboticsAcl.Scope.All())
|
|
325
308
|
|
|
326
|
-
def create(self, items:
|
|
309
|
+
def create(self, items: Sequence[RobotCapabilityWrite]) -> RobotCapabilityList:
|
|
327
310
|
return self.client.robotics.capabilities.create(items)
|
|
328
311
|
|
|
329
312
|
def retrieve(self, ids: SequenceNotStr[str]) -> RobotCapabilityList:
|
|
330
313
|
return _fallback_to_one_by_one(self.client.robotics.capabilities.retrieve, ids, RobotCapabilityList)
|
|
331
314
|
|
|
332
|
-
def update(self, items:
|
|
315
|
+
def update(self, items: Sequence[RobotCapabilityWrite]) -> RobotCapabilityList:
|
|
333
316
|
# There is a bug in the /update endpoint that requires the input_schema to be a string
|
|
334
317
|
# and not an object. This is a workaround until the bug is fixed.
|
|
335
318
|
# We do the serialization to avoid modifying the original object.
|
|
@@ -370,12 +353,10 @@ class RobotCapabilityCRUD(
|
|
|
370
353
|
return super().diff_list(local, cdf, json_path)
|
|
371
354
|
|
|
372
355
|
|
|
373
|
-
class RoboticMapCRUD(ResourceCRUD[str, MapWrite, Map
|
|
356
|
+
class RoboticMapCRUD(ResourceCRUD[str, MapWrite, Map]):
|
|
374
357
|
folder_name = "robotics"
|
|
375
358
|
resource_cls = Map
|
|
376
359
|
resource_write_cls = MapWrite
|
|
377
|
-
list_cls = MapList
|
|
378
|
-
list_write_cls = MapWriteList
|
|
379
360
|
kind = "Map"
|
|
380
361
|
dependencies = frozenset({RoboticFrameCRUD, RoboticLocationCRUD})
|
|
381
362
|
yaml_cls = RobotMapYAML
|
|
@@ -427,13 +408,13 @@ class RoboticMapCRUD(ResourceCRUD[str, MapWrite, Map, MapWriteList, MapList]):
|
|
|
427
408
|
del dump["scale"]
|
|
428
409
|
return dump
|
|
429
410
|
|
|
430
|
-
def create(self, items:
|
|
411
|
+
def create(self, items: Sequence[MapWrite]) -> MapList:
|
|
431
412
|
return self.client.robotics.maps.create(items)
|
|
432
413
|
|
|
433
414
|
def retrieve(self, ids: SequenceNotStr[str]) -> MapList:
|
|
434
415
|
return _fallback_to_one_by_one(self.client.robotics.maps.retrieve, ids, MapList)
|
|
435
416
|
|
|
436
|
-
def update(self, items:
|
|
417
|
+
def update(self, items: Sequence[MapWrite]) -> MapList:
|
|
437
418
|
return self.client.robotics.maps.update(items)
|
|
438
419
|
|
|
439
420
|
def delete(self, ids: SequenceNotStr[str]) -> int:
|
|
@@ -6,7 +6,6 @@ from cognite.client.utils.useful_types import SequenceNotStr
|
|
|
6
6
|
|
|
7
7
|
from cognite_toolkit._cdf_tk.client.data_classes.streams import (
|
|
8
8
|
StreamRequest,
|
|
9
|
-
StreamRequestList,
|
|
10
9
|
StreamResponse,
|
|
11
10
|
StreamResponseList,
|
|
12
11
|
)
|
|
@@ -18,13 +17,11 @@ from .datamodel import ContainerCRUD
|
|
|
18
17
|
|
|
19
18
|
|
|
20
19
|
@final
|
|
21
|
-
class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse
|
|
20
|
+
class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse]):
|
|
22
21
|
folder_name = "streams"
|
|
23
22
|
filetypes = frozenset({"yaml", "yml"})
|
|
24
23
|
resource_cls = StreamResponse
|
|
25
24
|
resource_write_cls = StreamRequest
|
|
26
|
-
list_cls = StreamResponseList
|
|
27
|
-
list_write_cls = StreamRequestList
|
|
28
25
|
kind = "Streams"
|
|
29
26
|
yaml_cls = StreamYAML
|
|
30
27
|
dependencies = frozenset({ContainerCRUD})
|
|
@@ -59,7 +56,7 @@ class StreamCRUD(ResourceCRUD[str, StreamRequest, StreamResponse, StreamRequestL
|
|
|
59
56
|
)
|
|
60
57
|
return StreamsAcl(actions, StreamsAcl.Scope.All())
|
|
61
58
|
|
|
62
|
-
def create(self, items:
|
|
59
|
+
def create(self, items: Sequence[StreamRequest]) -> StreamResponseList:
|
|
63
60
|
created = self.client.streams.create(list(items))
|
|
64
61
|
return StreamResponseList(created)
|
|
65
62
|
|
|
@@ -6,7 +6,6 @@ from cognite.client.data_classes import (
|
|
|
6
6
|
ThreeDModelList,
|
|
7
7
|
ThreeDModelUpdate,
|
|
8
8
|
ThreeDModelWrite,
|
|
9
|
-
ThreeDModelWriteList,
|
|
10
9
|
capabilities,
|
|
11
10
|
)
|
|
12
11
|
from cognite.client.data_classes.capabilities import Capability
|
|
@@ -20,12 +19,10 @@ from .data_organization import DataSetsCRUD
|
|
|
20
19
|
|
|
21
20
|
|
|
22
21
|
@final
|
|
23
|
-
class ThreeDModelCRUD(ResourceContainerCRUD[str, ThreeDModelWrite, ThreeDModel
|
|
22
|
+
class ThreeDModelCRUD(ResourceContainerCRUD[str, ThreeDModelWrite, ThreeDModel]):
|
|
24
23
|
folder_name = "3dmodels"
|
|
25
24
|
resource_cls = ThreeDModel
|
|
26
25
|
resource_write_cls = ThreeDModelWrite
|
|
27
|
-
list_cls = ThreeDModelList
|
|
28
|
-
list_write_cls = ThreeDModelWriteList
|
|
29
26
|
kind = "3DModel"
|
|
30
27
|
yaml_cls = ThreeDModelYAML
|
|
31
28
|
dependencies = frozenset({DataSetsCRUD})
|
|
@@ -82,7 +79,7 @@ class ThreeDModelCRUD(ResourceContainerCRUD[str, ThreeDModelWrite, ThreeDModel,
|
|
|
82
79
|
|
|
83
80
|
return capabilities.ThreeDAcl(actions, scope)
|
|
84
81
|
|
|
85
|
-
def create(self, items:
|
|
82
|
+
def create(self, items: Sequence[ThreeDModelWrite]) -> ThreeDModelList:
|
|
86
83
|
created = ThreeDModelList([])
|
|
87
84
|
for item in items:
|
|
88
85
|
new_item = self.client.three_d.models.create(**item.dump(camel_case=False))
|
|
@@ -104,7 +101,7 @@ class ThreeDModelCRUD(ResourceContainerCRUD[str, ThreeDModelWrite, ThreeDModel,
|
|
|
104
101
|
break
|
|
105
102
|
return output
|
|
106
103
|
|
|
107
|
-
def update(self, items:
|
|
104
|
+
def update(self, items: Sequence[ThreeDModelWrite]) -> ThreeDModelList:
|
|
108
105
|
found = self.retrieve([item.name for item in items])
|
|
109
106
|
id_by_name = {model.name: model.id for model in found}
|
|
110
107
|
# 3D Model does not have an external identifier, only internal.
|
|
@@ -9,11 +9,9 @@ from cognite.client.data_classes import (
|
|
|
9
9
|
DatapointSubscriptionList,
|
|
10
10
|
DataPointSubscriptionUpdate,
|
|
11
11
|
DataPointSubscriptionWrite,
|
|
12
|
-
DatapointSubscriptionWriteList,
|
|
13
12
|
TimeSeries,
|
|
14
13
|
TimeSeriesList,
|
|
15
14
|
TimeSeriesWrite,
|
|
16
|
-
TimeSeriesWriteList,
|
|
17
15
|
)
|
|
18
16
|
from cognite.client.data_classes.capabilities import (
|
|
19
17
|
Capability,
|
|
@@ -43,13 +41,11 @@ from .data_organization import DataSetsCRUD
|
|
|
43
41
|
|
|
44
42
|
|
|
45
43
|
@final
|
|
46
|
-
class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries
|
|
44
|
+
class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries]):
|
|
47
45
|
item_name = "datapoints"
|
|
48
46
|
folder_name = "timeseries"
|
|
49
47
|
resource_cls = TimeSeries
|
|
50
48
|
resource_write_cls = TimeSeriesWrite
|
|
51
|
-
list_cls = TimeSeriesList
|
|
52
|
-
list_write_cls = TimeSeriesWriteList
|
|
53
49
|
yaml_cls = TimeSeriesYAML
|
|
54
50
|
kind = "TimeSeries"
|
|
55
51
|
dependencies = frozenset({DataSetsCRUD, GroupAllScopedCRUD, AssetCRUD})
|
|
@@ -124,7 +120,7 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries, Tim
|
|
|
124
120
|
dumped["assetExternalId"] = self.client.lookup.assets.external_id(asset_id)
|
|
125
121
|
return dumped
|
|
126
122
|
|
|
127
|
-
def create(self, items:
|
|
123
|
+
def create(self, items: Sequence[TimeSeriesWrite]) -> TimeSeriesList:
|
|
128
124
|
return self.client.time_series.create(items)
|
|
129
125
|
|
|
130
126
|
def retrieve(self, ids: SequenceNotStr[str | int]) -> TimeSeriesList:
|
|
@@ -133,7 +129,7 @@ class TimeSeriesCRUD(ResourceContainerCRUD[str, TimeSeriesWrite, TimeSeries, Tim
|
|
|
133
129
|
ids=internal_ids, external_ids=external_ids, ignore_unknown_ids=True
|
|
134
130
|
)
|
|
135
131
|
|
|
136
|
-
def update(self, items:
|
|
132
|
+
def update(self, items: Sequence[TimeSeriesWrite]) -> TimeSeriesList:
|
|
137
133
|
return self.client.time_series.update(items, mode="replace")
|
|
138
134
|
|
|
139
135
|
def delete(self, ids: SequenceNotStr[str | int]) -> int:
|
|
@@ -181,15 +177,11 @@ class DatapointSubscriptionCRUD(
|
|
|
181
177
|
str,
|
|
182
178
|
DataPointSubscriptionWrite,
|
|
183
179
|
DatapointSubscription,
|
|
184
|
-
DatapointSubscriptionWriteList,
|
|
185
|
-
DatapointSubscriptionList,
|
|
186
180
|
]
|
|
187
181
|
):
|
|
188
182
|
folder_name = "timeseries"
|
|
189
183
|
resource_cls = DatapointSubscription
|
|
190
184
|
resource_write_cls = DataPointSubscriptionWrite
|
|
191
|
-
list_cls = DatapointSubscriptionList
|
|
192
|
-
list_write_cls = DatapointSubscriptionWriteList
|
|
193
185
|
kind = "DatapointSubscription"
|
|
194
186
|
_doc_url = "Data-point-subscriptions/operation/postSubscriptions"
|
|
195
187
|
dependencies = frozenset(
|
|
@@ -251,7 +243,7 @@ class DatapointSubscriptionCRUD(
|
|
|
251
243
|
|
|
252
244
|
return TimeSeriesSubscriptionsAcl(actions, scope)
|
|
253
245
|
|
|
254
|
-
def create(self, items:
|
|
246
|
+
def create(self, items: Sequence[DataPointSubscriptionWrite]) -> DatapointSubscriptionList:
|
|
255
247
|
created_list = DatapointSubscriptionList([])
|
|
256
248
|
for item in items:
|
|
257
249
|
to_create, batches = self.create_split_timeseries_ids(item)
|
|
@@ -269,7 +261,7 @@ class DatapointSubscriptionCRUD(
|
|
|
269
261
|
items.append(retrieved)
|
|
270
262
|
return items
|
|
271
263
|
|
|
272
|
-
def update(self, items:
|
|
264
|
+
def update(self, items: Sequence[DataPointSubscriptionWrite]) -> DatapointSubscriptionList:
|
|
273
265
|
updated_list = DatapointSubscriptionList([])
|
|
274
266
|
for item in items:
|
|
275
267
|
current = self.client.time_series.subscriptions.list_member_time_series(item.external_id, limit=-1)
|
|
@@ -43,9 +43,7 @@ from cognite.client.data_classes import (
|
|
|
43
43
|
TransformationSchedule,
|
|
44
44
|
TransformationScheduleList,
|
|
45
45
|
TransformationScheduleWrite,
|
|
46
|
-
TransformationScheduleWriteList,
|
|
47
46
|
TransformationWrite,
|
|
48
|
-
TransformationWriteList,
|
|
49
47
|
)
|
|
50
48
|
from cognite.client.data_classes.capabilities import (
|
|
51
49
|
Capability,
|
|
@@ -58,7 +56,6 @@ from cognite.client.data_classes.data_modeling.ids import (
|
|
|
58
56
|
from cognite.client.data_classes.transformations import NonceCredentials
|
|
59
57
|
from cognite.client.data_classes.transformations.notifications import (
|
|
60
58
|
TransformationNotificationWrite,
|
|
61
|
-
TransformationNotificationWriteList,
|
|
62
59
|
)
|
|
63
60
|
from cognite.client.exceptions import CogniteAPIError, CogniteAuthError, CogniteDuplicatedError, CogniteNotFoundError
|
|
64
61
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
@@ -103,14 +100,10 @@ from .raw import RawDatabaseCRUD, RawTableCRUD
|
|
|
103
100
|
|
|
104
101
|
|
|
105
102
|
@final
|
|
106
|
-
class TransformationCRUD(
|
|
107
|
-
ResourceCRUD[str, TransformationWrite, Transformation, TransformationWriteList, TransformationList]
|
|
108
|
-
):
|
|
103
|
+
class TransformationCRUD(ResourceCRUD[str, TransformationWrite, Transformation]):
|
|
109
104
|
folder_name = "transformations"
|
|
110
105
|
resource_cls = Transformation
|
|
111
106
|
resource_write_cls = TransformationWrite
|
|
112
|
-
list_cls = TransformationList
|
|
113
|
-
list_write_cls = TransformationWriteList
|
|
114
107
|
kind = "Transformation"
|
|
115
108
|
yaml_cls = TransformationYAML
|
|
116
109
|
dependencies = frozenset(
|
|
@@ -510,15 +503,11 @@ class TransformationScheduleCRUD(
|
|
|
510
503
|
str,
|
|
511
504
|
TransformationScheduleWrite,
|
|
512
505
|
TransformationSchedule,
|
|
513
|
-
TransformationScheduleWriteList,
|
|
514
|
-
TransformationScheduleList,
|
|
515
506
|
]
|
|
516
507
|
):
|
|
517
508
|
folder_name = "transformations"
|
|
518
509
|
resource_cls = TransformationSchedule
|
|
519
510
|
resource_write_cls = TransformationScheduleWrite
|
|
520
|
-
list_cls = TransformationScheduleList
|
|
521
|
-
list_write_cls = TransformationScheduleWriteList
|
|
522
511
|
kind = "Schedule"
|
|
523
512
|
yaml_cls = TransformationScheduleYAML
|
|
524
513
|
dependencies = frozenset({TransformationCRUD})
|
|
@@ -568,7 +557,7 @@ class TransformationScheduleCRUD(
|
|
|
568
557
|
def retrieve(self, ids: SequenceNotStr[str]) -> TransformationScheduleList:
|
|
569
558
|
return self.client.transformations.schedules.retrieve_multiple(external_ids=ids, ignore_unknown_ids=True)
|
|
570
559
|
|
|
571
|
-
def update(self, items:
|
|
560
|
+
def update(self, items: Sequence[TransformationScheduleWrite]) -> TransformationScheduleList:
|
|
572
561
|
return self.client.transformations.schedules.update(items, mode="replace")
|
|
573
562
|
|
|
574
563
|
def delete(self, ids: str | SequenceNotStr[str] | None) -> int:
|
|
@@ -606,15 +595,11 @@ class TransformationNotificationCRUD(
|
|
|
606
595
|
str,
|
|
607
596
|
TransformationNotificationWrite,
|
|
608
597
|
TransformationNotification,
|
|
609
|
-
TransformationNotificationWriteList,
|
|
610
|
-
TransformationNotificationList,
|
|
611
598
|
]
|
|
612
599
|
):
|
|
613
600
|
folder_name = "transformations"
|
|
614
601
|
resource_cls = TransformationNotification
|
|
615
602
|
resource_write_cls = TransformationNotificationWrite
|
|
616
|
-
list_cls = TransformationNotificationList
|
|
617
|
-
list_write_cls = TransformationNotificationWriteList
|
|
618
603
|
kind = "Notification"
|
|
619
604
|
dependencies = frozenset({TransformationCRUD})
|
|
620
605
|
_doc_url = "Transformation-Notifications/operation/createTransformationNotifications"
|
|
@@ -661,7 +646,7 @@ class TransformationNotificationCRUD(
|
|
|
661
646
|
dumped["transformationExternalId"] = local["transformationExternalId"]
|
|
662
647
|
return dumped
|
|
663
648
|
|
|
664
|
-
def create(self, items:
|
|
649
|
+
def create(self, items: Sequence[TransformationNotificationWrite]) -> TransformationNotificationList:
|
|
665
650
|
return self.client.transformations.notifications.create(items)
|
|
666
651
|
|
|
667
652
|
def retrieve(self, ids: SequenceNotStr[str]) -> TransformationNotificationList:
|
|
@@ -692,7 +677,7 @@ class TransformationNotificationCRUD(
|
|
|
692
677
|
|
|
693
678
|
return retrieved
|
|
694
679
|
|
|
695
|
-
def update(self, items:
|
|
680
|
+
def update(self, items: Sequence[TransformationNotificationWrite]) -> TransformationNotificationList:
|
|
696
681
|
# Note that since a notification is identified by the combination of transformationExternalId and destination,
|
|
697
682
|
# which is the entire object, an update should never happen. However, implementing just in case.
|
|
698
683
|
item_by_id = {self.get_id(item): item for item in items}
|
|
@@ -22,24 +22,23 @@ from typing import Any, final
|
|
|
22
22
|
from cognite.client.data_classes import (
|
|
23
23
|
ClientCredentials,
|
|
24
24
|
Workflow,
|
|
25
|
-
WorkflowList,
|
|
26
25
|
WorkflowTrigger,
|
|
27
|
-
WorkflowTriggerList,
|
|
28
26
|
WorkflowTriggerUpsert,
|
|
29
|
-
WorkflowTriggerUpsertList,
|
|
30
27
|
WorkflowUpsert,
|
|
31
|
-
WorkflowUpsertList,
|
|
32
28
|
WorkflowVersion,
|
|
33
29
|
WorkflowVersionId,
|
|
34
|
-
WorkflowVersionList,
|
|
35
30
|
WorkflowVersionUpsert,
|
|
36
|
-
WorkflowVersionUpsertList,
|
|
37
31
|
)
|
|
38
32
|
from cognite.client.data_classes.capabilities import (
|
|
39
33
|
Capability,
|
|
40
34
|
WorkflowOrchestrationAcl,
|
|
41
35
|
)
|
|
42
|
-
from cognite.client.data_classes.workflows import
|
|
36
|
+
from cognite.client.data_classes.workflows import (
|
|
37
|
+
SubworkflowReferenceParameters,
|
|
38
|
+
WorkflowList,
|
|
39
|
+
WorkflowTriggerList,
|
|
40
|
+
WorkflowVersionList,
|
|
41
|
+
)
|
|
43
42
|
from cognite.client.exceptions import CogniteAuthError, CogniteNotFoundError
|
|
44
43
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
45
44
|
from rich import print
|
|
@@ -75,12 +74,10 @@ from .transformation import TransformationCRUD
|
|
|
75
74
|
|
|
76
75
|
|
|
77
76
|
@final
|
|
78
|
-
class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow
|
|
77
|
+
class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow]):
|
|
79
78
|
folder_name = "workflows"
|
|
80
79
|
resource_cls = Workflow
|
|
81
80
|
resource_write_cls = WorkflowUpsert
|
|
82
|
-
list_cls = WorkflowList
|
|
83
|
-
list_write_cls = WorkflowUpsertList
|
|
84
81
|
kind = "Workflow"
|
|
85
82
|
dependencies = frozenset(
|
|
86
83
|
{
|
|
@@ -139,18 +136,14 @@ class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertLis
|
|
|
139
136
|
dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
|
|
140
137
|
return dumped
|
|
141
138
|
|
|
142
|
-
def retrieve(self, ids: SequenceNotStr[str]) ->
|
|
139
|
+
def retrieve(self, ids: SequenceNotStr[str]) -> Sequence[Workflow]:
|
|
143
140
|
return self.client.workflows.retrieve(external_id=ids, ignore_unknown_ids=True)
|
|
144
141
|
|
|
145
|
-
def
|
|
146
|
-
|
|
147
|
-
return self.client.workflows.upsert(upserts)
|
|
142
|
+
def create(self, items: Sequence[WorkflowUpsert]) -> WorkflowList:
|
|
143
|
+
return self.client.workflows.upsert(items)
|
|
148
144
|
|
|
149
|
-
def
|
|
150
|
-
return self.
|
|
151
|
-
|
|
152
|
-
def update(self, items: WorkflowUpsertList) -> WorkflowList:
|
|
153
|
-
return self._upsert(items)
|
|
145
|
+
def update(self, items: Sequence[WorkflowUpsert]) -> WorkflowList:
|
|
146
|
+
return self.client.workflows.upsert(items)
|
|
154
147
|
|
|
155
148
|
def delete(self, ids: SequenceNotStr[str]) -> int:
|
|
156
149
|
successes = 0
|
|
@@ -191,16 +184,10 @@ class WorkflowCRUD(ResourceCRUD[str, WorkflowUpsert, Workflow, WorkflowUpsertLis
|
|
|
191
184
|
|
|
192
185
|
|
|
193
186
|
@final
|
|
194
|
-
class WorkflowVersionCRUD(
|
|
195
|
-
ResourceCRUD[
|
|
196
|
-
WorkflowVersionId, WorkflowVersionUpsert, WorkflowVersion, WorkflowVersionUpsertList, WorkflowVersionList
|
|
197
|
-
]
|
|
198
|
-
):
|
|
187
|
+
class WorkflowVersionCRUD(ResourceCRUD[WorkflowVersionId, WorkflowVersionUpsert, WorkflowVersion]):
|
|
199
188
|
folder_name = "workflows"
|
|
200
189
|
resource_cls = WorkflowVersion
|
|
201
190
|
resource_write_cls = WorkflowVersionUpsert
|
|
202
|
-
list_cls = WorkflowVersionList
|
|
203
|
-
list_write_cls = WorkflowVersionUpsertList
|
|
204
191
|
kind = "WorkflowVersion"
|
|
205
192
|
dependencies = frozenset({WorkflowCRUD})
|
|
206
193
|
parent_resource = frozenset({WorkflowCRUD})
|
|
@@ -396,16 +383,16 @@ class WorkflowVersionCRUD(
|
|
|
396
383
|
return WorkflowVersionList([])
|
|
397
384
|
return self.client.workflows.versions.retrieve(workflow_external_id=list(ids), ignore_unknown_ids=True)
|
|
398
385
|
|
|
399
|
-
def _upsert(self, items:
|
|
386
|
+
def _upsert(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
|
|
400
387
|
return self.client.workflows.versions.upsert(items)
|
|
401
388
|
|
|
402
|
-
def create(self, items:
|
|
389
|
+
def create(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
|
|
403
390
|
upserted: list[WorkflowVersion] = []
|
|
404
391
|
for item in self.topological_sort(items):
|
|
405
392
|
upserted.extend(self.client.workflows.versions.upsert([item]))
|
|
406
393
|
return WorkflowVersionList(upserted)
|
|
407
394
|
|
|
408
|
-
def update(self, items:
|
|
395
|
+
def update(self, items: Sequence[WorkflowVersionUpsert]) -> WorkflowVersionList:
|
|
409
396
|
return self._upsert(items)
|
|
410
397
|
|
|
411
398
|
def delete(self, ids: SequenceNotStr[WorkflowVersionId]) -> int:
|
|
@@ -465,14 +452,10 @@ class WorkflowVersionCRUD(
|
|
|
465
452
|
|
|
466
453
|
|
|
467
454
|
@final
|
|
468
|
-
class WorkflowTriggerCRUD(
|
|
469
|
-
ResourceCRUD[str, WorkflowTriggerUpsert, WorkflowTrigger, WorkflowTriggerUpsertList, WorkflowTriggerList]
|
|
470
|
-
):
|
|
455
|
+
class WorkflowTriggerCRUD(ResourceCRUD[str, WorkflowTriggerUpsert, WorkflowTrigger]):
|
|
471
456
|
folder_name = "workflows"
|
|
472
457
|
resource_cls = WorkflowTrigger
|
|
473
458
|
resource_write_cls = WorkflowTriggerUpsert
|
|
474
|
-
list_cls = WorkflowTriggerList
|
|
475
|
-
list_write_cls = WorkflowTriggerUpsertList
|
|
476
459
|
kind = "WorkflowTrigger"
|
|
477
460
|
dependencies = frozenset({WorkflowCRUD, WorkflowVersionCRUD, GroupResourceScopedCRUD, GroupAllScopedCRUD})
|
|
478
461
|
parent_resource = frozenset({WorkflowCRUD})
|
|
@@ -519,10 +502,10 @@ class WorkflowTriggerCRUD(
|
|
|
519
502
|
WorkflowOrchestrationAcl.Scope.All(),
|
|
520
503
|
)
|
|
521
504
|
|
|
522
|
-
def create(self, items:
|
|
505
|
+
def create(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
|
|
523
506
|
return self._upsert(items)
|
|
524
507
|
|
|
525
|
-
def _upsert(self, items:
|
|
508
|
+
def _upsert(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
|
|
526
509
|
created = WorkflowTriggerList([])
|
|
527
510
|
for item in items:
|
|
528
511
|
created.append(self._upsert_item(item))
|
|
@@ -542,7 +525,7 @@ class WorkflowTriggerCRUD(
|
|
|
542
525
|
lookup = set(ids)
|
|
543
526
|
return WorkflowTriggerList([trigger for trigger in all_triggers if trigger.external_id in lookup])
|
|
544
527
|
|
|
545
|
-
def update(self, items:
|
|
528
|
+
def update(self, items: Sequence[WorkflowTriggerUpsert]) -> WorkflowTriggerList:
|
|
546
529
|
return self._upsert(items)
|
|
547
530
|
|
|
548
531
|
def delete(self, ids: SequenceNotStr[str]) -> int:
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import re
|
|
2
2
|
import warnings
|
|
3
|
-
from collections.abc import Hashable
|
|
3
|
+
from collections.abc import Hashable, Sequence
|
|
4
4
|
from copy import deepcopy
|
|
5
|
-
from dataclasses import dataclass
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Generic, cast
|
|
8
8
|
|
|
@@ -18,35 +18,24 @@ from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarnin
|
|
|
18
18
|
from cognite_toolkit._cdf_tk.utils import to_diff
|
|
19
19
|
|
|
20
20
|
from . import FunctionCRUD
|
|
21
|
-
from ._base_cruds import
|
|
22
|
-
T_ID,
|
|
23
|
-
ResourceCRUD,
|
|
24
|
-
T_ResourceRequest,
|
|
25
|
-
T_ResourceRequestList,
|
|
26
|
-
T_ResourceResponse,
|
|
27
|
-
T_ResourceResponseList,
|
|
28
|
-
)
|
|
21
|
+
from ._base_cruds import T_ID, ResourceCRUD, T_ResourceRequest, T_ResourceResponse
|
|
29
22
|
|
|
30
23
|
if TYPE_CHECKING:
|
|
31
24
|
from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
|
|
32
25
|
|
|
33
26
|
|
|
34
27
|
@dataclass
|
|
35
|
-
class CategorizedResources(Generic[T_ID,
|
|
36
|
-
to_create:
|
|
37
|
-
to_update:
|
|
38
|
-
to_delete: list[T_ID]
|
|
39
|
-
unchanged:
|
|
28
|
+
class CategorizedResources(Generic[T_ID, T_ResourceRequest]):
|
|
29
|
+
to_create: list[T_ResourceRequest] = field(default_factory=list)
|
|
30
|
+
to_update: list[T_ResourceRequest] = field(default_factory=list)
|
|
31
|
+
to_delete: list[T_ID] = field(default_factory=list)
|
|
32
|
+
unchanged: list[T_ResourceRequest] = field(default_factory=list)
|
|
40
33
|
|
|
41
34
|
|
|
42
|
-
class ResourceWorker(
|
|
43
|
-
Generic[T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList]
|
|
44
|
-
):
|
|
35
|
+
class ResourceWorker(Generic[T_ID, T_ResourceRequest, T_ResourceResponse]):
|
|
45
36
|
def __init__(
|
|
46
37
|
self,
|
|
47
|
-
loader: ResourceCRUD[
|
|
48
|
-
T_ID, T_ResourceRequest, T_ResourceResponse, T_ResourceRequestList, T_ResourceResponseList
|
|
49
|
-
],
|
|
38
|
+
loader: ResourceCRUD[T_ID, T_ResourceRequest, T_ResourceResponse],
|
|
50
39
|
action: str,
|
|
51
40
|
):
|
|
52
41
|
self.loader = loader
|
|
@@ -104,7 +93,6 @@ class ResourceWorker(
|
|
|
104
93
|
self.validate_access(local_by_id, is_dry_run)
|
|
105
94
|
|
|
106
95
|
# Lookup the existing resources in CDF
|
|
107
|
-
cdf_resources: T_ResourceResponseList
|
|
108
96
|
cdf_resources = self.loader.retrieve(list(local_by_id.keys()))
|
|
109
97
|
return self.categorize_resources(local_by_id, cdf_resources, force_update, verbose)
|
|
110
98
|
|
|
@@ -165,16 +153,11 @@ class ResourceWorker(
|
|
|
165
153
|
def categorize_resources(
|
|
166
154
|
self,
|
|
167
155
|
local_by_id: dict[T_ID, tuple[dict[str, Any], T_ResourceRequest]],
|
|
168
|
-
cdf_resources:
|
|
156
|
+
cdf_resources: Sequence[T_ResourceResponse],
|
|
169
157
|
force_update: bool,
|
|
170
158
|
verbose: bool,
|
|
171
|
-
) -> CategorizedResources[T_ID,
|
|
172
|
-
resources: CategorizedResources[T_ID,
|
|
173
|
-
to_create=self.loader.list_write_cls([]),
|
|
174
|
-
to_update=self.loader.list_write_cls([]),
|
|
175
|
-
to_delete=[],
|
|
176
|
-
unchanged=self.loader.list_write_cls([]),
|
|
177
|
-
)
|
|
159
|
+
) -> CategorizedResources[T_ID, T_ResourceRequest]:
|
|
160
|
+
resources: CategorizedResources[T_ID, T_ResourceRequest] = CategorizedResources()
|
|
178
161
|
cdf_resource_by_id = {self.loader.get_id(resource): resource for resource in cdf_resources}
|
|
179
162
|
for identifier, (local_dict, local_resource) in local_by_id.items():
|
|
180
163
|
cdf_resource = cdf_resource_by_id.get(identifier)
|
|
@@ -218,12 +218,35 @@ class UploadableStorageIO(
|
|
|
218
218
|
def read_chunks(
|
|
219
219
|
cls, reader: MultiFileReader, selector: T_Selector
|
|
220
220
|
) -> Iterable[list[tuple[str, dict[str, JsonVal]]]]:
|
|
221
|
+
"""Read data from a MultiFileReader in chunks.
|
|
222
|
+
|
|
223
|
+
This method yields chunks of data, where each chunk is a list of tuples. Each tuple contains a source ID
|
|
224
|
+
(e.g., line number or row identifier) and a dictionary representing the data in a JSON-compatible format.
|
|
225
|
+
|
|
226
|
+
This method can be overridden by subclasses to customize how data is read and chunked.
|
|
227
|
+
Args:
|
|
228
|
+
reader: An instance of MultiFileReader to read data from.
|
|
229
|
+
selector: The selection criteria to identify the data.
|
|
230
|
+
"""
|
|
221
231
|
data_name = "row" if reader.is_table else "line"
|
|
222
232
|
# Include name of line for better error messages
|
|
223
233
|
iterable = ((f"{data_name} {line_no}", item) for line_no, item in reader.read_chunks_with_line_numbers())
|
|
224
234
|
|
|
225
235
|
yield from chunker(iterable, cls.CHUNK_SIZE)
|
|
226
236
|
|
|
237
|
+
@classmethod
|
|
238
|
+
def count_chunks(cls, reader: MultiFileReader) -> int:
|
|
239
|
+
"""Count the number of items in a MultiFileReader.
|
|
240
|
+
|
|
241
|
+
This method can be overridden by subclasses to customize how items are counted.
|
|
242
|
+
|
|
243
|
+
Args:
|
|
244
|
+
reader: An instance of MultiFileReader to count items from.
|
|
245
|
+
Returns:
|
|
246
|
+
The number of items in the reader.
|
|
247
|
+
"""
|
|
248
|
+
return reader.count()
|
|
249
|
+
|
|
227
250
|
|
|
228
251
|
class TableUploadableStorageIO(UploadableStorageIO[T_Selector, T_ResourceResponse, T_ResourceRequest], ABC):
|
|
229
252
|
"""A base class for storage items that support uploading data with table schemas."""
|