cognite-toolkit 0.7.71__py3-none-any.whl → 0.7.73__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf_tk/builders/_raw.py +2 -2
- cognite_toolkit/_cdf_tk/client/_toolkit_client.py +0 -2
- cognite_toolkit/_cdf_tk/client/api/raw.py +47 -27
- cognite_toolkit/_cdf_tk/client/api/robotics_capabilities.py +9 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_data_postprocessing.py +12 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_frames.py +10 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_locations.py +10 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_maps.py +10 -3
- cognite_toolkit/_cdf_tk/client/api/robotics_robots.py +10 -3
- cognite_toolkit/_cdf_tk/client/http_client/_client.py +1 -1
- cognite_toolkit/_cdf_tk/client/http_client/_data_classes.py +1 -0
- cognite_toolkit/_cdf_tk/client/http_client/_exception.py +6 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/identifiers.py +26 -1
- cognite_toolkit/_cdf_tk/client/resource_classes/raw.py +70 -16
- cognite_toolkit/_cdf_tk/client/testing.py +23 -19
- cognite_toolkit/_cdf_tk/commands/build_cmd.py +2 -2
- cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +3 -3
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py +4 -5
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py +3 -3
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/raw.py +99 -91
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/robotics.py +154 -160
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py +3 -3
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.7.71.dist-info → cognite_toolkit-0.7.73.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.7.71.dist-info → cognite_toolkit-0.7.73.dist-info}/RECORD +30 -40
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/__init__.py +0 -8
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/api.py +0 -20
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/capabilities.py +0 -142
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/data_postprocessing.py +0 -144
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/frames.py +0 -136
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/locations.py +0 -136
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/maps.py +0 -136
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/robots.py +0 -132
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/utlis.py +0 -13
- cognite_toolkit/_cdf_tk/client/resource_classes/legacy/robotics.py +0 -970
- {cognite_toolkit-0.7.71.dist-info → cognite_toolkit-0.7.73.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.7.71.dist-info → cognite_toolkit-0.7.73.dist-info}/entry_points.txt +0 -0
|
@@ -6,7 +6,9 @@ from unittest.mock import MagicMock
|
|
|
6
6
|
from cognite.client._api.datapoints import DatapointsAPI
|
|
7
7
|
from cognite.client._api.datapoints_subscriptions import DatapointsSubscriptionAPI
|
|
8
8
|
from cognite.client._api.functions import FunctionCallsAPI, FunctionSchedulesAPI
|
|
9
|
-
from cognite.client._api.raw import RawDatabasesAPI
|
|
9
|
+
from cognite.client._api.raw import RawDatabasesAPI as LegacyRawDatabasesAPI
|
|
10
|
+
from cognite.client._api.raw import RawRowsAPI
|
|
11
|
+
from cognite.client._api.raw import RawTablesAPI as LegacyRawTablesAPI
|
|
10
12
|
from cognite.client._api.simulators import SimulatorModelsAPI, SimulatorsAPI
|
|
11
13
|
from cognite.client._api.synthetic_time_series import SyntheticDatapointsAPI
|
|
12
14
|
from cognite.client.testing import CogniteClientMock
|
|
@@ -23,15 +25,15 @@ from cognite_toolkit._cdf_tk.client.api.legacy.extended_functions import Extende
|
|
|
23
25
|
from cognite_toolkit._cdf_tk.client.api.legacy.extended_raw import ExtendedRawAPI
|
|
24
26
|
from cognite_toolkit._cdf_tk.client.api.legacy.extended_timeseries import ExtendedTimeSeriesAPI
|
|
25
27
|
from cognite_toolkit._cdf_tk.client.api.legacy.location_filters import LocationFiltersAPI
|
|
26
|
-
from cognite_toolkit._cdf_tk.client.api.legacy.robotics import (
|
|
27
|
-
CapabilitiesAPI,
|
|
28
|
-
DataPostProcessingAPI,
|
|
29
|
-
FramesAPI,
|
|
30
|
-
MapsAPI,
|
|
31
|
-
RoboticsAPI,
|
|
32
|
-
)
|
|
33
|
-
from cognite_toolkit._cdf_tk.client.api.legacy.robotics import LocationsAPI as RoboticsLocationsAPI
|
|
34
28
|
from cognite_toolkit._cdf_tk.client.api.legacy.search_config import SearchConfigurationsAPI
|
|
29
|
+
from cognite_toolkit._cdf_tk.client.api.raw import RawAPI, RawDatabasesAPI, RawTablesAPI
|
|
30
|
+
from cognite_toolkit._cdf_tk.client.api.robotics import RoboticsAPI
|
|
31
|
+
from cognite_toolkit._cdf_tk.client.api.robotics_capabilities import CapabilitiesAPI
|
|
32
|
+
from cognite_toolkit._cdf_tk.client.api.robotics_data_postprocessing import DataPostProcessingAPI
|
|
33
|
+
from cognite_toolkit._cdf_tk.client.api.robotics_frames import FramesAPI
|
|
34
|
+
from cognite_toolkit._cdf_tk.client.api.robotics_locations import LocationsAPI
|
|
35
|
+
from cognite_toolkit._cdf_tk.client.api.robotics_maps import MapsAPI
|
|
36
|
+
from cognite_toolkit._cdf_tk.client.api.robotics_robots import RobotsAPI
|
|
35
37
|
|
|
36
38
|
from ._toolkit_client import ToolAPI
|
|
37
39
|
from .api.assets import AssetsAPI
|
|
@@ -135,17 +137,9 @@ class ToolkitClientMock(CogniteClientMock):
|
|
|
135
137
|
self.migration.resource_view_mapping = MagicMock(spec_set=ResourceViewMappingAPI)
|
|
136
138
|
self.migration.created_source_system = MagicMock(spec_set=CreatedSourceSystemAPI)
|
|
137
139
|
self.raw = MagicMock(spec=ExtendedRawAPI)
|
|
138
|
-
self.raw.databases = MagicMock(spec_set=
|
|
140
|
+
self.raw.databases = MagicMock(spec_set=LegacyRawDatabasesAPI)
|
|
139
141
|
self.raw.rows = MagicMock(spec_set=RawRowsAPI)
|
|
140
|
-
self.raw.tables = MagicMock(spec_set=
|
|
141
|
-
|
|
142
|
-
self.robotics = MagicMock()
|
|
143
|
-
self.robotics.robots = MagicMock(spec=RoboticsAPI)
|
|
144
|
-
self.robotics.data_postprocessing = MagicMock(spec_set=DataPostProcessingAPI)
|
|
145
|
-
self.robotics.locations = MagicMock(spec_set=RoboticsLocationsAPI)
|
|
146
|
-
self.robotics.frames = MagicMock(spec_set=FramesAPI)
|
|
147
|
-
self.robotics.maps = MagicMock(spec_set=MapsAPI)
|
|
148
|
-
self.robotics.capabilities = MagicMock(spec_set=CapabilitiesAPI)
|
|
142
|
+
self.raw.tables = MagicMock(spec_set=LegacyRawTablesAPI)
|
|
149
143
|
|
|
150
144
|
self.data_modeling.instances = MagicMock(spec_set=ExtendedInstancesAPI)
|
|
151
145
|
|
|
@@ -171,6 +165,16 @@ class ToolkitClientMock(CogniteClientMock):
|
|
|
171
165
|
self.tool.hosted_extractors.destinations = MagicMock(spec_set=HostedExtractorDestinationsAPI)
|
|
172
166
|
self.tool.hosted_extractors.mappings = MagicMock(spec_set=HostedExtractorMappingsAPI)
|
|
173
167
|
self.tool.labels = MagicMock(spec_set=LabelsAPI)
|
|
168
|
+
self.tool.raw = MagicMock(spec=RawAPI)
|
|
169
|
+
self.tool.raw.databases = MagicMock(spec_set=RawDatabasesAPI)
|
|
170
|
+
self.tool.raw.tables = MagicMock(spec_set=RawTablesAPI)
|
|
171
|
+
self.tool.robotics = MagicMock(spec=RoboticsAPI)
|
|
172
|
+
self.tool.robotics.capabilities = MagicMock(spec_set=CapabilitiesAPI)
|
|
173
|
+
self.tool.robotics.data_postprocessing = MagicMock(spec_set=DataPostProcessingAPI)
|
|
174
|
+
self.tool.robotics.frames = MagicMock(spec_set=FramesAPI)
|
|
175
|
+
self.tool.robotics.locations = MagicMock(spec_set=LocationsAPI)
|
|
176
|
+
self.tool.robotics.maps = MagicMock(spec_set=MapsAPI)
|
|
177
|
+
self.tool.robotics.robots = MagicMock(spec_set=RobotsAPI)
|
|
174
178
|
self.tool.security_categories = MagicMock(spec_set=SecurityCategoriesAPI)
|
|
175
179
|
self.tool.sequences = MagicMock(spec_set=SequencesAPI)
|
|
176
180
|
self.tool.transformations = MagicMock(spec_set=TransformationsAPI)
|
|
@@ -14,7 +14,7 @@ from rich.progress import track
|
|
|
14
14
|
from cognite_toolkit._cdf_tk.builders import Builder, FunctionBuilder, create_builder
|
|
15
15
|
from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
|
|
16
16
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
17
|
-
from cognite_toolkit._cdf_tk.client.resource_classes.
|
|
17
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import RawDatabaseId
|
|
18
18
|
from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
|
|
19
19
|
from cognite_toolkit._cdf_tk.constants import (
|
|
20
20
|
_RUNNING_IN_BROWSER,
|
|
@@ -606,7 +606,7 @@ class BuildCommand(ToolkitCommand):
|
|
|
606
606
|
if identifier:
|
|
607
607
|
identifier_kind_pairs.append((identifier, item_loader.kind))
|
|
608
608
|
if first_seen := self._ids_by_resource_type[item_loader].get(identifier):
|
|
609
|
-
if isinstance(identifier,
|
|
609
|
+
if isinstance(identifier, RawDatabaseId):
|
|
610
610
|
# RawDatabases are picked up from both RawTables and RawDatabases files. Note it is not possible
|
|
611
611
|
# to define a raw table without also defining the raw database. Thus, it is impossible to
|
|
612
612
|
# avoid duplicated RawDatabase warnings if you have multiple RawTables files.
|
|
@@ -8,7 +8,7 @@ from cognite.client.data_classes import FileMetadataWrite
|
|
|
8
8
|
|
|
9
9
|
from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId
|
|
10
10
|
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.extendable_cognite_file import ExtendableCogniteFileApply
|
|
11
|
-
from cognite_toolkit._cdf_tk.client.resource_classes.
|
|
11
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.raw import RAWTableResponse
|
|
12
12
|
from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
|
|
13
13
|
from cognite_toolkit._cdf_tk.protocols import (
|
|
14
14
|
T_ResourceRequest,
|
|
@@ -167,7 +167,7 @@ class RawFileCRUD(DataCRUD):
|
|
|
167
167
|
for resource in state.built_resources[self.folder_name]:
|
|
168
168
|
if resource.kind != RawTableCRUD.kind:
|
|
169
169
|
continue
|
|
170
|
-
table = cast(
|
|
170
|
+
table = cast(RAWTableResponse, resource.identifier)
|
|
171
171
|
datafile = next(
|
|
172
172
|
(
|
|
173
173
|
resource.source.path.with_suffix(f".{file_type}")
|
|
@@ -210,7 +210,7 @@ class RawFileCRUD(DataCRUD):
|
|
|
210
210
|
continue
|
|
211
211
|
|
|
212
212
|
self.client.raw.rows.insert_dataframe(
|
|
213
|
-
db_name=table.db_name, table_name=table.
|
|
213
|
+
db_name=table.db_name, table_name=table.name, dataframe=data, ensure_parent=False
|
|
214
214
|
)
|
|
215
215
|
yield (
|
|
216
216
|
(f" Inserted {len(data):,} rows of {len(data.columns):,} columns from '{datafile!s}' into {table!r}."),
|
|
@@ -41,8 +41,7 @@ from rich.console import Console
|
|
|
41
41
|
from rich.markup import escape
|
|
42
42
|
|
|
43
43
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
44
|
-
from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId
|
|
45
|
-
from cognite_toolkit._cdf_tk.client.resource_classes.legacy.raw import RawDatabase, RawTable
|
|
44
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import ExternalId, RawDatabaseId, RawTableId
|
|
46
45
|
from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
|
|
47
46
|
from cognite_toolkit._cdf_tk.exceptions import ToolkitWrongResourceError
|
|
48
47
|
from cognite_toolkit._cdf_tk.resource_classes import GroupYAML, SecurityCategoriesYAML
|
|
@@ -161,12 +160,12 @@ class GroupCRUD(ResourceCRUD[str, GroupWrite, Group]):
|
|
|
161
160
|
yield DataSetsCRUD, data_set_id
|
|
162
161
|
if table_ids := scope.get(cap.TableScope._scope_name, []):
|
|
163
162
|
for db_name, tables in table_ids.get("dbsToTables", {}).items():
|
|
164
|
-
yield RawDatabaseCRUD,
|
|
163
|
+
yield RawDatabaseCRUD, RawDatabaseId(name=db_name)
|
|
165
164
|
if isinstance(tables, list):
|
|
166
|
-
yield from ((RawTableCRUD,
|
|
165
|
+
yield from ((RawTableCRUD, RawTableId(db_name=db_name, name=table)) for table in tables)
|
|
167
166
|
elif isinstance(tables, dict) and "tables" in tables:
|
|
168
167
|
for table in tables["tables"]:
|
|
169
|
-
yield RawTableCRUD,
|
|
168
|
+
yield RawTableCRUD, RawTableId(db_name=db_name, name=table)
|
|
170
169
|
if extraction_pipeline_ids := scope.get(cap.ExtractionPipelineScope._scope_name, []):
|
|
171
170
|
if isinstance(extraction_pipeline_ids, dict) and "ids" in extraction_pipeline_ids:
|
|
172
171
|
for extraction_pipeline_id in extraction_pipeline_ids["ids"]:
|
|
@@ -35,7 +35,7 @@ from cognite.client.data_classes.extractionpipelines import (
|
|
|
35
35
|
from cognite.client.exceptions import CogniteAPIError, CogniteDuplicatedError, CogniteNotFoundError
|
|
36
36
|
from cognite.client.utils.useful_types import SequenceNotStr
|
|
37
37
|
|
|
38
|
-
from cognite_toolkit._cdf_tk.client.resource_classes.
|
|
38
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import RawDatabaseId, RawTableId
|
|
39
39
|
from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING
|
|
40
40
|
from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceCRUD
|
|
41
41
|
from cognite_toolkit._cdf_tk.exceptions import (
|
|
@@ -124,9 +124,9 @@ class ExtractionPipelineCRUD(ResourceCRUD[str, ExtractionPipelineWrite, Extracti
|
|
|
124
124
|
if db := entry.get("dbName"):
|
|
125
125
|
if db not in seen_databases:
|
|
126
126
|
seen_databases.add(db)
|
|
127
|
-
yield RawDatabaseCRUD,
|
|
127
|
+
yield RawDatabaseCRUD, RawDatabaseId(name=db)
|
|
128
128
|
if "tableName" in entry:
|
|
129
|
-
yield RawTableCRUD,
|
|
129
|
+
yield RawTableCRUD, RawTableId(db_name=db, name=entry["tableName"])
|
|
130
130
|
|
|
131
131
|
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> ExtractionPipelineWrite:
|
|
132
132
|
if ds_external_id := resource.pop("dataSetExternalId", None):
|
|
@@ -17,7 +17,7 @@ import itertools
|
|
|
17
17
|
from collections import defaultdict
|
|
18
18
|
from collections.abc import Hashable, Iterable, Sequence
|
|
19
19
|
from pathlib import Path
|
|
20
|
-
from typing import Any,
|
|
20
|
+
from typing import Any, final
|
|
21
21
|
|
|
22
22
|
from cognite.client.data_classes.capabilities import (
|
|
23
23
|
Capability,
|
|
@@ -29,11 +29,13 @@ from rich import print
|
|
|
29
29
|
from rich.console import Console
|
|
30
30
|
|
|
31
31
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
|
32
|
-
from cognite_toolkit._cdf_tk.client.
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
32
|
+
from cognite_toolkit._cdf_tk.client.http_client import ToolkitAPIError
|
|
33
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.identifiers import NameId, RawDatabaseId, RawTableId
|
|
34
|
+
from cognite_toolkit._cdf_tk.client.resource_classes.raw import (
|
|
35
|
+
RAWDatabaseRequest,
|
|
36
|
+
RAWDatabaseResponse,
|
|
37
|
+
RAWTableRequest,
|
|
38
|
+
RAWTableResponse,
|
|
37
39
|
)
|
|
38
40
|
from cognite_toolkit._cdf_tk.cruds._base_cruds import ResourceContainerCRUD, ResourceCRUD
|
|
39
41
|
from cognite_toolkit._cdf_tk.resource_classes import DatabaseYAML, TableYAML
|
|
@@ -42,11 +44,11 @@ from .auth import GroupAllScopedCRUD
|
|
|
42
44
|
|
|
43
45
|
|
|
44
46
|
@final
|
|
45
|
-
class RawDatabaseCRUD(ResourceContainerCRUD[
|
|
47
|
+
class RawDatabaseCRUD(ResourceContainerCRUD[RawDatabaseId, RAWDatabaseRequest, RAWDatabaseResponse]):
|
|
46
48
|
item_name = "raw tables"
|
|
47
49
|
folder_name = "raw"
|
|
48
|
-
resource_cls =
|
|
49
|
-
resource_write_cls =
|
|
50
|
+
resource_cls = RAWDatabaseResponse
|
|
51
|
+
resource_write_cls = RAWDatabaseRequest
|
|
50
52
|
kind = "Database"
|
|
51
53
|
yaml_cls = DatabaseYAML
|
|
52
54
|
dependencies = frozenset({GroupAllScopedCRUD})
|
|
@@ -63,7 +65,7 @@ class RawDatabaseCRUD(ResourceContainerCRUD[RawDatabase, RawDatabase, RawDatabas
|
|
|
63
65
|
|
|
64
66
|
@classmethod
|
|
65
67
|
def get_required_capability(
|
|
66
|
-
cls, items: Sequence[
|
|
68
|
+
cls, items: Sequence[RAWDatabaseRequest] | None, read_only: bool
|
|
67
69
|
) -> Capability | list[Capability]:
|
|
68
70
|
if not items and items is not None:
|
|
69
71
|
return []
|
|
@@ -78,86 +80,89 @@ class RawDatabaseCRUD(ResourceContainerCRUD[RawDatabase, RawDatabase, RawDatabas
|
|
|
78
80
|
if items:
|
|
79
81
|
tables_by_database: dict[str, list[str]] = {}
|
|
80
82
|
for item in items:
|
|
81
|
-
tables_by_database[item.
|
|
83
|
+
tables_by_database[item.name] = []
|
|
82
84
|
|
|
83
85
|
scope = RawAcl.Scope.Table(dict(tables_by_database)) if tables_by_database else RawAcl.Scope.All()
|
|
84
86
|
|
|
85
87
|
return RawAcl(actions, scope)
|
|
86
88
|
|
|
87
89
|
@classmethod
|
|
88
|
-
def get_id(cls, item:
|
|
90
|
+
def get_id(cls, item: RAWDatabaseResponse | RAWDatabaseRequest | dict) -> RawDatabaseId:
|
|
89
91
|
if isinstance(item, dict):
|
|
90
|
-
return
|
|
91
|
-
return item
|
|
92
|
+
return RawDatabaseId.model_validate(item)
|
|
93
|
+
return RawDatabaseId(name=item.name)
|
|
92
94
|
|
|
93
95
|
@classmethod
|
|
94
|
-
def dump_id(cls, id:
|
|
95
|
-
return {"dbName": id.
|
|
96
|
+
def dump_id(cls, id: RawDatabaseId) -> dict[str, Any]:
|
|
97
|
+
return {"dbName": id.name}
|
|
96
98
|
|
|
97
|
-
def
|
|
98
|
-
|
|
99
|
-
return RawDatabaseList([RawDatabase(db_name=db.name) for db in database_list if db.name])
|
|
99
|
+
def dump_resource(self, resource: RAWDatabaseResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
100
|
+
return {"dbName": resource.name}
|
|
100
101
|
|
|
101
|
-
def
|
|
102
|
-
|
|
103
|
-
target_dbs = {db.db_name for db in ids}
|
|
104
|
-
return RawDatabaseList([RawDatabase(db_name=db.name) for db in database_list if db.name in target_dbs])
|
|
102
|
+
def create(self, items: Sequence[RAWDatabaseRequest]) -> list[RAWDatabaseResponse]:
|
|
103
|
+
return self.client.tool.raw.databases.create(items)
|
|
105
104
|
|
|
106
|
-
def
|
|
107
|
-
|
|
105
|
+
def retrieve(self, ids: SequenceNotStr[RawDatabaseId]) -> list[RAWDatabaseResponse]:
|
|
106
|
+
database_list = self.client.tool.raw.databases.list(limit=None)
|
|
107
|
+
target_dbs = {db.name for db in ids}
|
|
108
|
+
return [db for db in database_list if db.name in target_dbs]
|
|
109
|
+
|
|
110
|
+
def delete(self, ids: SequenceNotStr[RawDatabaseId]) -> int:
|
|
111
|
+
ids_list = list(ids)
|
|
108
112
|
try:
|
|
109
|
-
self.client.raw.databases.delete(
|
|
110
|
-
except
|
|
113
|
+
self.client.tool.raw.databases.delete(ids_list)
|
|
114
|
+
except ToolkitAPIError as e:
|
|
111
115
|
# Bug in API, missing is returned as failed
|
|
112
|
-
if e.
|
|
113
|
-
self.client.raw.databases.delete(
|
|
116
|
+
if e.missing and (remaining := [db for db in ids_list if db.name not in e.missing]):
|
|
117
|
+
self.client.tool.raw.databases.delete(remaining)
|
|
114
118
|
elif e.code == 404 and "not found" in e.message and "database" in e.message:
|
|
115
119
|
return 0
|
|
116
120
|
else:
|
|
117
121
|
raise e
|
|
118
|
-
return len(
|
|
122
|
+
return len(ids_list)
|
|
119
123
|
|
|
120
124
|
def _iterate(
|
|
121
125
|
self,
|
|
122
126
|
data_set_external_id: str | None = None,
|
|
123
127
|
space: str | None = None,
|
|
124
128
|
parent_ids: list[Hashable] | None = None,
|
|
125
|
-
) -> Iterable[
|
|
126
|
-
|
|
129
|
+
) -> Iterable[RAWDatabaseResponse]:
|
|
130
|
+
for databases in self.client.tool.raw.databases.iterate():
|
|
131
|
+
yield from databases
|
|
127
132
|
|
|
128
|
-
def count(self, ids: SequenceNotStr[
|
|
133
|
+
def count(self, ids: SequenceNotStr[RawDatabaseId]) -> int:
|
|
129
134
|
nr_of_tables = 0
|
|
130
|
-
for db_name, raw_tables in itertools.groupby(sorted(ids, key=lambda x: x.
|
|
135
|
+
for db_name, raw_tables in itertools.groupby(sorted(ids, key=lambda x: x.name), key=lambda x: x.name):
|
|
131
136
|
try:
|
|
132
|
-
tables = self.client.raw.tables.list(db_name=db_name, limit
|
|
137
|
+
tables = self.client.tool.raw.tables.list(db_name=db_name, limit=None)
|
|
133
138
|
except CogniteAPIError as e:
|
|
134
139
|
if db_name in {item.get("name") for item in e.missing or []}:
|
|
135
140
|
continue
|
|
136
141
|
raise e
|
|
137
|
-
nr_of_tables += len(tables
|
|
142
|
+
nr_of_tables += len(tables)
|
|
138
143
|
return nr_of_tables
|
|
139
144
|
|
|
140
|
-
def drop_data(self, ids: SequenceNotStr[
|
|
145
|
+
def drop_data(self, ids: SequenceNotStr[RawDatabaseId]) -> int:
|
|
141
146
|
nr_of_tables = 0
|
|
142
|
-
for db_name, raw_tables in itertools.groupby(sorted(ids, key=lambda x: x.
|
|
147
|
+
for db_name, raw_tables in itertools.groupby(sorted(ids, key=lambda x: x.name), key=lambda x: x.name):
|
|
143
148
|
try:
|
|
144
|
-
existing = self.client.raw.tables.list(db_name=db_name, limit
|
|
145
|
-
except
|
|
149
|
+
existing = self.client.tool.raw.tables.list(db_name=db_name, limit=None)
|
|
150
|
+
except ToolkitAPIError as e:
|
|
146
151
|
if db_name in {item.get("name") for item in e.missing or []}:
|
|
147
152
|
continue
|
|
148
153
|
raise e
|
|
149
154
|
if existing:
|
|
150
|
-
self.client.raw.tables.delete(
|
|
155
|
+
self.client.tool.raw.tables.delete([table.as_id() for table in existing])
|
|
151
156
|
nr_of_tables += len(existing)
|
|
152
157
|
return nr_of_tables
|
|
153
158
|
|
|
154
159
|
|
|
155
160
|
@final
|
|
156
|
-
class RawTableCRUD(ResourceContainerCRUD[
|
|
161
|
+
class RawTableCRUD(ResourceContainerCRUD[RawTableId, RAWTableRequest, RAWTableResponse]):
|
|
157
162
|
item_name = "raw rows"
|
|
158
163
|
folder_name = "raw"
|
|
159
|
-
resource_cls =
|
|
160
|
-
resource_write_cls =
|
|
164
|
+
resource_cls = RAWTableResponse
|
|
165
|
+
resource_write_cls = RAWTableRequest
|
|
161
166
|
kind = "Table"
|
|
162
167
|
yaml_cls = TableYAML
|
|
163
168
|
support_update = False
|
|
@@ -175,7 +180,7 @@ class RawTableCRUD(ResourceContainerCRUD[RawTable, RawTable, RawTable]):
|
|
|
175
180
|
|
|
176
181
|
@classmethod
|
|
177
182
|
def get_required_capability(
|
|
178
|
-
cls, items: Sequence[
|
|
183
|
+
cls, items: Sequence[RAWTableRequest] | None, read_only: bool
|
|
179
184
|
) -> Capability | list[Capability]:
|
|
180
185
|
if not items and items is not None:
|
|
181
186
|
return []
|
|
@@ -190,75 +195,70 @@ class RawTableCRUD(ResourceContainerCRUD[RawTable, RawTable, RawTable]):
|
|
|
190
195
|
if items:
|
|
191
196
|
tables_by_database = defaultdict(list)
|
|
192
197
|
for item in items:
|
|
193
|
-
tables_by_database[item.db_name].append(item.
|
|
198
|
+
tables_by_database[item.db_name].append(item.name)
|
|
194
199
|
|
|
195
200
|
scope = RawAcl.Scope.Table(dict(tables_by_database)) if tables_by_database else RawAcl.Scope.All()
|
|
196
201
|
|
|
197
202
|
return RawAcl(actions, scope)
|
|
198
203
|
|
|
199
204
|
@classmethod
|
|
200
|
-
def get_id(cls, item:
|
|
205
|
+
def get_id(cls, item: RAWTableResponse | RAWTableRequest | dict) -> RawTableId:
|
|
201
206
|
if isinstance(item, dict):
|
|
202
207
|
if missing := tuple(k for k in {"dbName", "tableName"} if k not in item):
|
|
203
208
|
# We need to raise a KeyError with all missing keys to get the correct error message.
|
|
204
209
|
raise KeyError(*missing)
|
|
205
|
-
return
|
|
206
|
-
return item
|
|
210
|
+
return RawTableId(db_name=item["dbName"], name=item["tableName"])
|
|
211
|
+
return RawTableId(db_name=item.db_name, name=item.name)
|
|
207
212
|
|
|
208
213
|
@classmethod
|
|
209
|
-
def dump_id(cls, id:
|
|
210
|
-
return {"dbName": id.db_name, "tableName": id.
|
|
214
|
+
def dump_id(cls, id: RawTableId) -> dict[str, Any]:
|
|
215
|
+
return {"dbName": id.db_name, "tableName": id.name}
|
|
211
216
|
|
|
212
217
|
@classmethod
|
|
213
218
|
def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceCRUD], Hashable]]:
|
|
214
219
|
if "dbName" in item:
|
|
215
|
-
yield RawDatabaseCRUD,
|
|
216
|
-
|
|
217
|
-
def
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
def retrieve(self, ids: SequenceNotStr[RawTable]) -> RawTableList:
|
|
226
|
-
retrieved = RawTableList([])
|
|
220
|
+
yield RawDatabaseCRUD, RawDatabaseId(name=item["dbName"])
|
|
221
|
+
|
|
222
|
+
def dump_resource(self, resource: RAWTableResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
223
|
+
return {"dbName": resource.db_name, "tableName": resource.name}
|
|
224
|
+
|
|
225
|
+
def create(self, items: Sequence[RAWTableRequest]) -> list[RAWTableResponse]:
|
|
226
|
+
return self.client.tool.raw.tables.create(items)
|
|
227
|
+
|
|
228
|
+
def retrieve(self, ids: SequenceNotStr[RawTableId]) -> list[RAWTableResponse]:
|
|
229
|
+
retrieved: list[RAWTableResponse] = []
|
|
227
230
|
for db_name, raw_tables in itertools.groupby(sorted(ids, key=lambda x: x.db_name), key=lambda x: x.db_name):
|
|
228
|
-
expected_tables = {table.
|
|
231
|
+
expected_tables = {table.name for table in raw_tables}
|
|
229
232
|
try:
|
|
230
|
-
tables = self.client.raw.tables.list(db_name=db_name, limit
|
|
231
|
-
except
|
|
233
|
+
tables = self.client.tool.raw.tables.list(db_name=db_name, limit=None)
|
|
234
|
+
except ToolkitAPIError as e:
|
|
232
235
|
if db_name in {item.get("name") for item in e.missing or []}:
|
|
233
236
|
continue
|
|
234
237
|
raise e
|
|
235
|
-
retrieved.extend(
|
|
236
|
-
[RawTable(db_name=db_name, table_name=table.name) for table in tables if table.name in expected_tables]
|
|
237
|
-
)
|
|
238
|
+
retrieved.extend(table for table in tables if table.name in expected_tables)
|
|
238
239
|
return retrieved
|
|
239
240
|
|
|
240
|
-
def delete(self, ids: SequenceNotStr[
|
|
241
|
+
def delete(self, ids: SequenceNotStr[RawTableId]) -> int:
|
|
241
242
|
count = 0
|
|
242
243
|
for db_name, raw_tables in itertools.groupby(sorted(ids, key=lambda x: x.db_name), key=lambda x: x.db_name):
|
|
243
|
-
|
|
244
|
-
if
|
|
244
|
+
tables_to_delete = [table for table in raw_tables if table.name]
|
|
245
|
+
if tables_to_delete:
|
|
245
246
|
try:
|
|
246
|
-
self.client.raw.tables.delete(
|
|
247
|
-
except
|
|
247
|
+
self.client.tool.raw.tables.delete(tables_to_delete)
|
|
248
|
+
except ToolkitAPIError as e:
|
|
248
249
|
if e.code != 404:
|
|
249
250
|
raise e
|
|
250
|
-
|
|
251
|
-
missing = {item.get("name") for item in (e.missing or [])}.union(set(e.failed or []))
|
|
251
|
+
missing = {item.get("name") for item in (e.missing or [])}
|
|
252
252
|
if "not found" in e.message and "database" in e.message:
|
|
253
253
|
continue
|
|
254
|
-
elif
|
|
255
|
-
self.client.raw.tables.delete(
|
|
256
|
-
elif not
|
|
254
|
+
elif remaining := [t for t in tables_to_delete if t.name not in missing]:
|
|
255
|
+
self.client.tool.raw.tables.delete(remaining)
|
|
256
|
+
elif not remaining:
|
|
257
257
|
# Table does not exist.
|
|
258
258
|
continue
|
|
259
259
|
else:
|
|
260
260
|
raise e
|
|
261
|
-
count += len(
|
|
261
|
+
count += len(tables_to_delete)
|
|
262
262
|
return count
|
|
263
263
|
|
|
264
264
|
def _iterate(
|
|
@@ -266,28 +266,36 @@ class RawTableCRUD(ResourceContainerCRUD[RawTable, RawTable, RawTable]):
|
|
|
266
266
|
data_set_external_id: str | None = None,
|
|
267
267
|
space: str | None = None,
|
|
268
268
|
parent_ids: list[Hashable] | None = None,
|
|
269
|
-
) -> Iterable[
|
|
270
|
-
|
|
271
|
-
|
|
269
|
+
) -> Iterable[RAWTableResponse]:
|
|
270
|
+
if parent_ids is None:
|
|
271
|
+
dbs = self.client.tool.raw.databases.list(limit=None)
|
|
272
|
+
parent_ids = [RawDatabaseId(name=db.name) for db in dbs]
|
|
273
|
+
for parent_id in parent_ids:
|
|
274
|
+
if not isinstance(parent_id, NameId):
|
|
272
275
|
continue
|
|
273
|
-
for
|
|
274
|
-
yield
|
|
276
|
+
for tables in self.client.tool.raw.tables.iterate(db_name=parent_id.name):
|
|
277
|
+
yield from tables
|
|
275
278
|
|
|
276
|
-
def count(self, ids: SequenceNotStr[
|
|
279
|
+
def count(self, ids: SequenceNotStr[RawTableId]) -> int:
|
|
277
280
|
if not self._printed_warning:
|
|
278
281
|
print(" [bold green]INFO:[/] Raw rows do not support count (there is no aggregation method).")
|
|
279
282
|
self._printed_warning = True
|
|
280
283
|
return -1
|
|
281
284
|
|
|
282
|
-
def drop_data(self, ids: SequenceNotStr[
|
|
285
|
+
def drop_data(self, ids: SequenceNotStr[RawTableId]) -> int:
|
|
283
286
|
for db_name, raw_tables in itertools.groupby(sorted(ids, key=lambda x: x.db_name), key=lambda x: x.db_name):
|
|
284
287
|
try:
|
|
285
|
-
|
|
288
|
+
existing_tables = self.client.tool.raw.tables.list(db_name=db_name, limit=None)
|
|
289
|
+
existing_names = {table.name for table in existing_tables}
|
|
286
290
|
except CogniteAPIError as e:
|
|
287
291
|
if db_name in {item.get("name") for item in e.missing or []}:
|
|
288
292
|
continue
|
|
289
293
|
raise e
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
294
|
+
tables_to_delete = [
|
|
295
|
+
RAWTableResponse(db_name=db_name, name=table.name)
|
|
296
|
+
for table in raw_tables
|
|
297
|
+
if table.name in existing_names
|
|
298
|
+
]
|
|
299
|
+
if tables_to_delete:
|
|
300
|
+
self.client.tool.raw.tables.delete([table.as_id() for table in tables_to_delete])
|
|
293
301
|
return -1
|