cognite-toolkit 0.5.62__py3-none-any.whl → 0.5.64__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_builtin_modules/cdf.toml +1 -1
- cognite_toolkit/_cdf_tk/apps/_profile_app.py +32 -2
- cognite_toolkit/_cdf_tk/client/data_classes/search_config.py +198 -0
- cognite_toolkit/_cdf_tk/commands/__init__.py +2 -1
- cognite_toolkit/_cdf_tk/commands/_profile.py +74 -1
- cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py +10 -14
- cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py +50 -10
- cognite_toolkit/_cdf_tk/utils/sql_parser.py +4 -0
- cognite_toolkit/_cdf_tk/utils/text.py +42 -0
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_version.py +1 -1
- {cognite_toolkit-0.5.62.dist-info → cognite_toolkit-0.5.64.dist-info}/METADATA +1 -1
- {cognite_toolkit-0.5.62.dist-info → cognite_toolkit-0.5.64.dist-info}/RECORD +17 -15
- {cognite_toolkit-0.5.62.dist-info → cognite_toolkit-0.5.64.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.5.62.dist-info → cognite_toolkit-0.5.64.dist-info}/entry_points.txt +0 -0
- {cognite_toolkit-0.5.62.dist-info → cognite_toolkit-0.5.64.dist-info}/licenses/LICENSE +0 -0
@@ -1,9 +1,9 @@
|
|
1
|
-
from typing import Any
|
1
|
+
from typing import Annotated, Any
|
2
2
|
|
3
3
|
import typer
|
4
4
|
from rich import print
|
5
5
|
|
6
|
-
from cognite_toolkit._cdf_tk.commands import ProfileAssetCentricCommand
|
6
|
+
from cognite_toolkit._cdf_tk.commands import ProfileAssetCentricCommand, ProfileTransformationCommand
|
7
7
|
from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
|
8
8
|
|
9
9
|
|
@@ -12,6 +12,7 @@ class ProfileApp(typer.Typer):
|
|
12
12
|
super().__init__(*args, **kwargs)
|
13
13
|
self.callback(invoke_without_command=True)(self.main)
|
14
14
|
self.command("asset-centric")(self.asset_centric)
|
15
|
+
self.command("transformations")(self.transformations)
|
15
16
|
|
16
17
|
def main(self, ctx: typer.Context) -> None:
|
17
18
|
"""Commands profile functionality"""
|
@@ -34,3 +35,32 @@ class ProfileApp(typer.Typer):
|
|
34
35
|
verbose,
|
35
36
|
)
|
36
37
|
)
|
38
|
+
|
39
|
+
@staticmethod
|
40
|
+
def transformations(
|
41
|
+
destination: Annotated[
|
42
|
+
str,
|
43
|
+
typer.Option(
|
44
|
+
"--destination",
|
45
|
+
"-d",
|
46
|
+
help="Destination type the transformations data should be written to. This can be 'assets', 'events', 'files',"
|
47
|
+
"'timeseries', or 'sequences'.",
|
48
|
+
),
|
49
|
+
],
|
50
|
+
verbose: bool = False,
|
51
|
+
) -> None:
|
52
|
+
"""This command gives an overview over the transformations that write to the given destination.
|
53
|
+
It works by checking all transformations that writes to the given destination, lists the sources of the data,
|
54
|
+
and the target columns.
|
55
|
+
This is intended to show the flow of data from raw into CDF. This can, for example, be used to determine the
|
56
|
+
source of the data in a specific CDF resource.
|
57
|
+
"""
|
58
|
+
client = EnvironmentVariables.create_from_environment().get_client()
|
59
|
+
cmd = ProfileTransformationCommand()
|
60
|
+
cmd.run(
|
61
|
+
lambda: cmd.transformation(
|
62
|
+
client,
|
63
|
+
destination,
|
64
|
+
verbose,
|
65
|
+
)
|
66
|
+
)
|
@@ -0,0 +1,198 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
from abc import ABC
|
4
|
+
from dataclasses import dataclass
|
5
|
+
from typing import Any
|
6
|
+
|
7
|
+
from cognite.client import CogniteClient
|
8
|
+
from cognite.client.data_classes._base import (
|
9
|
+
CogniteObject,
|
10
|
+
CogniteResourceList,
|
11
|
+
WriteableCogniteResource,
|
12
|
+
WriteableCogniteResourceList,
|
13
|
+
)
|
14
|
+
from typing_extensions import Self
|
15
|
+
|
16
|
+
|
17
|
+
@dataclass
|
18
|
+
class SearchConfigView(CogniteObject):
|
19
|
+
external_id: str
|
20
|
+
space: str
|
21
|
+
|
22
|
+
@classmethod
|
23
|
+
def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
|
24
|
+
return cls(
|
25
|
+
external_id=resource["externalId"],
|
26
|
+
space=resource["space"],
|
27
|
+
)
|
28
|
+
|
29
|
+
|
30
|
+
@dataclass
|
31
|
+
class SearchConfigViewProperty(CogniteObject):
|
32
|
+
property: str
|
33
|
+
disabled: bool | None = None
|
34
|
+
selected: bool | None = None
|
35
|
+
hidden: bool | None = None
|
36
|
+
|
37
|
+
@classmethod
|
38
|
+
def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
|
39
|
+
return cls(
|
40
|
+
property=resource["property"],
|
41
|
+
disabled=resource.get("disabled"),
|
42
|
+
selected=resource.get("selected"),
|
43
|
+
hidden=resource.get("hidden"),
|
44
|
+
)
|
45
|
+
|
46
|
+
|
47
|
+
class SearchConfigCore(WriteableCogniteResource["SearchConfigWrite"], ABC):
|
48
|
+
"""
|
49
|
+
Core model for a single Configuration.
|
50
|
+
|
51
|
+
Args:
|
52
|
+
view: The configuration for one specific view.
|
53
|
+
id: A server-generated ID for the object.
|
54
|
+
use_as_name: The name of property to use for the name column in the UI.
|
55
|
+
use_as_description: The name of property to use for the description column in the UI.
|
56
|
+
column_layout: Array of column configurations per property.
|
57
|
+
filter_layout: Array of filter configurations per property.
|
58
|
+
properties_layout: Array of property configurations per property.
|
59
|
+
"""
|
60
|
+
|
61
|
+
def __init__(
|
62
|
+
self,
|
63
|
+
view: SearchConfigView,
|
64
|
+
id: int | None = None,
|
65
|
+
use_as_name: str | None = None,
|
66
|
+
use_as_description: str | None = None,
|
67
|
+
column_layout: list[SearchConfigViewProperty] | None = None,
|
68
|
+
filter_layout: list[SearchConfigViewProperty] | None = None,
|
69
|
+
properties_layout: list[SearchConfigViewProperty] | None = None,
|
70
|
+
) -> None:
|
71
|
+
self.view = view
|
72
|
+
self.id = id
|
73
|
+
self.use_as_name = use_as_name
|
74
|
+
self.use_as_description = use_as_description
|
75
|
+
self.column_layout = column_layout
|
76
|
+
self.filter_layout = filter_layout
|
77
|
+
self.properties_layout = properties_layout
|
78
|
+
|
79
|
+
def as_write(self) -> SearchConfigWrite:
|
80
|
+
return SearchConfigWrite(
|
81
|
+
view=self.view,
|
82
|
+
id=self.id,
|
83
|
+
use_as_name=self.use_as_name,
|
84
|
+
use_as_description=self.use_as_description,
|
85
|
+
column_layout=self.column_layout,
|
86
|
+
filter_layout=self.filter_layout,
|
87
|
+
properties_layout=self.properties_layout,
|
88
|
+
)
|
89
|
+
|
90
|
+
def dump(self, camel_case: bool = True) -> dict[str, Any]:
|
91
|
+
output = super().dump(camel_case)
|
92
|
+
if self.column_layout:
|
93
|
+
output["columLayout" if camel_case else "column_layout"] = [
|
94
|
+
_data.dump(camel_case) for _data in self.column_layout
|
95
|
+
]
|
96
|
+
if self.filter_layout:
|
97
|
+
output["filterLayout" if camel_case else "filter_layout"] = [
|
98
|
+
_data.dump(camel_case) for _data in self.filter_layout
|
99
|
+
]
|
100
|
+
if self.properties_layout:
|
101
|
+
output["propertiesLayout" if camel_case else "properties_layout"] = [
|
102
|
+
_data.dump(camel_case) for _data in self.properties_layout
|
103
|
+
]
|
104
|
+
if self.view:
|
105
|
+
output["view"] = self.view.dump(camel_case)
|
106
|
+
return output
|
107
|
+
|
108
|
+
|
109
|
+
class SearchConfigWrite(SearchConfigCore):
|
110
|
+
@classmethod
|
111
|
+
def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
|
112
|
+
return cls(
|
113
|
+
id=resource.get("id"),
|
114
|
+
view=SearchConfigView.load(resource["view"]),
|
115
|
+
use_as_name=resource.get("useAsName"),
|
116
|
+
use_as_description=resource.get("useAsDescription"),
|
117
|
+
column_layout=[SearchConfigViewProperty.load(item) for item in resource.get("columnLayout", [])]
|
118
|
+
if resource.get("columnLayout")
|
119
|
+
else None,
|
120
|
+
filter_layout=[SearchConfigViewProperty.load(item) for item in resource.get("filterLayout", [])]
|
121
|
+
if resource.get("filterLayout")
|
122
|
+
else None,
|
123
|
+
properties_layout=[SearchConfigViewProperty.load(item) for item in resource.get("propertiesLayout", [])]
|
124
|
+
if resource.get("propertiesLayout")
|
125
|
+
else None,
|
126
|
+
)
|
127
|
+
|
128
|
+
|
129
|
+
class SearchConfig(SearchConfigCore):
|
130
|
+
"""
|
131
|
+
Response model for a single Configuration.
|
132
|
+
|
133
|
+
Args:
|
134
|
+
view: The configuration for one specific view.
|
135
|
+
id: A server-generated ID for the object.
|
136
|
+
created_time: The time when the search config was created.
|
137
|
+
updated_time: The time when the search config was last updated.
|
138
|
+
use_as_name: The name of property to use for the name column in the UI.
|
139
|
+
use_as_description: The name of property to use for the description column in the UI.
|
140
|
+
column_layout: Array of column configurations per property.
|
141
|
+
filter_layout: Array of filter configurations per property.
|
142
|
+
properties_layout: Array of property configurations per property.
|
143
|
+
"""
|
144
|
+
|
145
|
+
def __init__(
|
146
|
+
self,
|
147
|
+
view: SearchConfigView,
|
148
|
+
id: int,
|
149
|
+
created_time: int,
|
150
|
+
updated_time: int,
|
151
|
+
use_as_name: str | None = None,
|
152
|
+
use_as_description: str | None = None,
|
153
|
+
column_layout: list[SearchConfigViewProperty] | None = None,
|
154
|
+
filter_layout: list[SearchConfigViewProperty] | None = None,
|
155
|
+
properties_layout: list[SearchConfigViewProperty] | None = None,
|
156
|
+
) -> None:
|
157
|
+
super().__init__(
|
158
|
+
view,
|
159
|
+
id,
|
160
|
+
use_as_name,
|
161
|
+
use_as_description,
|
162
|
+
column_layout,
|
163
|
+
filter_layout,
|
164
|
+
properties_layout,
|
165
|
+
)
|
166
|
+
self.created_time = created_time
|
167
|
+
self.updated_time = updated_time
|
168
|
+
|
169
|
+
@classmethod
|
170
|
+
def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
|
171
|
+
return cls(
|
172
|
+
view=SearchConfigView.load(resource["view"]),
|
173
|
+
id=resource["id"],
|
174
|
+
created_time=resource["createdTime"],
|
175
|
+
updated_time=resource["lastUpdatedTime"],
|
176
|
+
use_as_name=resource.get("useAsName"),
|
177
|
+
use_as_description=resource.get("useAsDescription"),
|
178
|
+
column_layout=[SearchConfigViewProperty.load(item) for item in resource.get("columnLayout", [])]
|
179
|
+
if resource.get("columnLayout")
|
180
|
+
else None,
|
181
|
+
filter_layout=[SearchConfigViewProperty.load(item) for item in resource.get("filterLayout", [])]
|
182
|
+
if resource.get("filterLayout")
|
183
|
+
else None,
|
184
|
+
properties_layout=[SearchConfigViewProperty.load(item) for item in resource.get("propertiesLayout", [])]
|
185
|
+
if resource.get("propertiesLayout")
|
186
|
+
else None,
|
187
|
+
)
|
188
|
+
|
189
|
+
|
190
|
+
class SearchConfigWriteList(CogniteResourceList):
|
191
|
+
_RESOURCE = SearchConfigWrite
|
192
|
+
|
193
|
+
|
194
|
+
class SearchConfigList(WriteableCogniteResourceList[SearchConfigWrite, SearchConfig]):
|
195
|
+
_RESOURCE = SearchConfig
|
196
|
+
|
197
|
+
def as_write(self) -> SearchConfigWriteList:
|
198
|
+
return SearchConfigWriteList([searchConfig.as_write() for searchConfig in self])
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from ._migrate import MigrateTimeseriesCommand, MigrationPrepareCommand
|
2
2
|
from ._populate import PopulateCommand
|
3
|
-
from ._profile import ProfileAssetCentricCommand
|
3
|
+
from ._profile import ProfileAssetCentricCommand, ProfileTransformationCommand
|
4
4
|
from ._purge import PurgeCommand
|
5
5
|
from .auth import AuthCommand
|
6
6
|
from .build_cmd import BuildCommand
|
@@ -31,6 +31,7 @@ __all__ = [
|
|
31
31
|
"ModulesCommand",
|
32
32
|
"PopulateCommand",
|
33
33
|
"ProfileAssetCentricCommand",
|
34
|
+
"ProfileTransformationCommand",
|
34
35
|
"PullCommand",
|
35
36
|
"PurgeCommand",
|
36
37
|
"RepoCommand",
|
@@ -1,9 +1,11 @@
|
|
1
|
+
import itertools
|
1
2
|
from abc import ABC, abstractmethod
|
2
|
-
from collections.abc import Callable, Mapping
|
3
|
+
from collections.abc import Callable, Iterable, Mapping
|
3
4
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
4
5
|
from functools import cached_property
|
5
6
|
from typing import ClassVar, Literal, TypeAlias, overload
|
6
7
|
|
8
|
+
from cognite.client.data_classes import Transformation
|
7
9
|
from cognite.client.exceptions import CogniteException
|
8
10
|
from rich import box
|
9
11
|
from rich.console import Console
|
@@ -12,6 +14,8 @@ from rich.spinner import Spinner
|
|
12
14
|
from rich.table import Table
|
13
15
|
|
14
16
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
17
|
+
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
18
|
+
from cognite_toolkit._cdf_tk.utils import humanize_collection
|
15
19
|
from cognite_toolkit._cdf_tk.utils.aggregators import (
|
16
20
|
AssetAggregator,
|
17
21
|
AssetCentricAggregator,
|
@@ -24,6 +28,7 @@ from cognite_toolkit._cdf_tk.utils.aggregators import (
|
|
24
28
|
SequenceAggregator,
|
25
29
|
TimeSeriesAggregator,
|
26
30
|
)
|
31
|
+
from cognite_toolkit._cdf_tk.utils.sql_parser import SQLParser, SQLTable
|
27
32
|
|
28
33
|
from ._base import ToolkitCommand
|
29
34
|
|
@@ -237,3 +242,71 @@ class ProfileAssetCentricCommand(ProfileCommand):
|
|
237
242
|
elif col == self.Columns.Transformation:
|
238
243
|
return aggregator.transformation_count
|
239
244
|
raise ValueError(f"Unknown column: {col} for row: {row}")
|
245
|
+
|
246
|
+
|
247
|
+
class ProfileTransformationCommand(ProfileCommand):
|
248
|
+
valid_destinations: frozenset[str] = frozenset({"assets", "files", "events", "timeseries", "sequences"})
|
249
|
+
|
250
|
+
def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
|
251
|
+
super().__init__(print_warning, skip_tracking, silent)
|
252
|
+
self.table_title = "Transformation Profile"
|
253
|
+
self.destination_type: Literal["assets", "files", "events", "timeseries", "sequences"] | None = None
|
254
|
+
|
255
|
+
class Columns:
|
256
|
+
Transformation = "Transformation"
|
257
|
+
Source = "Sources"
|
258
|
+
DestinationColumns = "Destination Columns"
|
259
|
+
Destination = "Destination"
|
260
|
+
ConflictMode = "Conflict Mode"
|
261
|
+
IsPaused = "Is Paused"
|
262
|
+
|
263
|
+
def transformation(
|
264
|
+
self, client: ToolkitClient, destination_type: str | None = None, verbose: bool = False
|
265
|
+
) -> list[dict[str, CellValue]]:
|
266
|
+
self.destination_type = self._validate_destination_type(destination_type)
|
267
|
+
return self.create_profile_table(client)
|
268
|
+
|
269
|
+
@classmethod
|
270
|
+
def _validate_destination_type(
|
271
|
+
cls, destination_type: str | None
|
272
|
+
) -> Literal["assets", "files", "events", "timeseries", "sequences"]:
|
273
|
+
if destination_type is None or destination_type not in cls.valid_destinations:
|
274
|
+
raise ToolkitValueError(
|
275
|
+
f"Invalid destination type: {destination_type}. Must be one of {humanize_collection(cls.valid_destinations)}."
|
276
|
+
)
|
277
|
+
# We validated the destination type above
|
278
|
+
return destination_type # type: ignore[return-value]
|
279
|
+
|
280
|
+
def create_initial_table(self, client: ToolkitClient) -> dict[tuple[str, str], PendingCellValue]:
|
281
|
+
if self.valid_destinations is None:
|
282
|
+
raise ToolkitValueError("Destination type must be set before calling create_initial_table.")
|
283
|
+
iterable: Iterable[Transformation] = client.transformations.list(
|
284
|
+
destination_type=self.destination_type, limit=-1
|
285
|
+
)
|
286
|
+
if self.destination_type == "assets":
|
287
|
+
iterable = itertools.chain(iterable, client.transformations(destination_type="asset_hierarchy", limit=-1))
|
288
|
+
table: dict[tuple[str, str], PendingCellValue] = {}
|
289
|
+
for transformation in iterable:
|
290
|
+
sources: list[SQLTable] = []
|
291
|
+
destination_columns: list[str] = []
|
292
|
+
if transformation.query:
|
293
|
+
parser = SQLParser(transformation.query, operation="Profile transformations")
|
294
|
+
sources = parser.sources
|
295
|
+
destination_columns = parser.destination_columns
|
296
|
+
index = str(transformation.id)
|
297
|
+
table[(index, self.Columns.Transformation)] = transformation.name or transformation.external_id or "Unknown"
|
298
|
+
table[(index, self.Columns.Source)] = ", ".join(map(str, sources))
|
299
|
+
table[(index, self.Columns.DestinationColumns)] = (
|
300
|
+
", ".join(destination_columns) or None if destination_columns else None
|
301
|
+
)
|
302
|
+
table[(index, self.Columns.Destination)] = (
|
303
|
+
transformation.destination.type or "Unknown" if transformation.destination else "Unknown"
|
304
|
+
)
|
305
|
+
table[(index, self.Columns.ConflictMode)] = transformation.conflict_mode or "Unknown"
|
306
|
+
table[(index, self.Columns.IsPaused)] = (
|
307
|
+
str(transformation.schedule.is_paused) if transformation.schedule else "No schedule"
|
308
|
+
)
|
309
|
+
return table
|
310
|
+
|
311
|
+
def call_api(self, row: str, col: str, client: ToolkitClient) -> Callable:
|
312
|
+
raise NotImplementedError(f"{type(self).__name__} does not support API calls for {col} in row {row}.")
|
@@ -46,6 +46,7 @@ from cognite_toolkit._cdf_tk.utils import (
|
|
46
46
|
calculate_secure_hash,
|
47
47
|
)
|
48
48
|
from cognite_toolkit._cdf_tk.utils.cdf import read_auth, try_find_error
|
49
|
+
from cognite_toolkit._cdf_tk.utils.text import suffix_description
|
49
50
|
|
50
51
|
from .auth_loaders import GroupAllScopedLoader
|
51
52
|
from .data_organization_loaders import DataSetsLoader
|
@@ -464,20 +465,15 @@ class FunctionScheduleLoader(
|
|
464
465
|
)
|
465
466
|
self.authentication_by_id[identifier] = credentials
|
466
467
|
auth_hash = calculate_secure_hash(credentials.dump(camel_case=True), shorten=True)
|
467
|
-
extra_str = f"
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
LowSeverityWarning(f"Description is too long for schedule {identifier!r}. Truncating...").print_warning(
|
477
|
-
console=self.console
|
478
|
-
)
|
479
|
-
truncation = self._description_character_limit - len(extra_str) - 3
|
480
|
-
resource["description"] = f"{resource['description'][:truncation]}...{extra_str}"
|
468
|
+
extra_str = f"{self._hash_key}: {auth_hash}"
|
469
|
+
resource["description"] = suffix_description(
|
470
|
+
extra_str,
|
471
|
+
resource.get("description"),
|
472
|
+
self._description_character_limit,
|
473
|
+
self.get_id(resource),
|
474
|
+
self.display_name,
|
475
|
+
self.console,
|
476
|
+
)
|
481
477
|
return resources
|
482
478
|
|
483
479
|
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> FunctionScheduleWrite:
|
@@ -1,5 +1,7 @@
|
|
1
|
+
import json
|
1
2
|
from collections.abc import Hashable, Iterable, Sequence
|
2
3
|
from functools import lru_cache
|
4
|
+
from pathlib import Path
|
3
5
|
from typing import Any, cast, final
|
4
6
|
|
5
7
|
from cognite.client.data_classes import (
|
@@ -28,7 +30,9 @@ from cognite_toolkit._cdf_tk.exceptions import (
|
|
28
30
|
)
|
29
31
|
from cognite_toolkit._cdf_tk.loaders._base_loaders import ResourceContainerLoader, ResourceLoader
|
30
32
|
from cognite_toolkit._cdf_tk.resource_classes import TimeSeriesYAML
|
33
|
+
from cognite_toolkit._cdf_tk.utils import calculate_hash
|
31
34
|
from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable, dm_identifier
|
35
|
+
from cognite_toolkit._cdf_tk.utils.text import suffix_description
|
32
36
|
|
33
37
|
from .auth_loaders import GroupAllScopedLoader, SecurityCategoryLoader
|
34
38
|
from .classic_loaders import AssetLoader
|
@@ -216,6 +220,9 @@ class DatapointSubscriptionLoader(
|
|
216
220
|
}
|
217
221
|
)
|
218
222
|
|
223
|
+
_hash_key = "cdf-hash"
|
224
|
+
_description_character_limit = 1000
|
225
|
+
|
219
226
|
@property
|
220
227
|
def display_name(self) -> str:
|
221
228
|
return "timeseries subscriptions"
|
@@ -325,25 +332,58 @@ class DatapointSubscriptionLoader(
|
|
325
332
|
) -> Iterable[DatapointSubscription]:
|
326
333
|
return iter(self.client.time_series.subscriptions)
|
327
334
|
|
335
|
+
def load_resource_file(
|
336
|
+
self, filepath: Path, environment_variables: dict[str, str | None] | None = None
|
337
|
+
) -> list[dict[str, Any]]:
|
338
|
+
resources = super().load_resource_file(filepath, environment_variables)
|
339
|
+
for resource in resources:
|
340
|
+
if "timeSeriesIds" not in resource and "instanceIds" not in resource:
|
341
|
+
continue
|
342
|
+
# If the timeSeriesIds or instanceIds is set, we need to add the auth hash to the description.
|
343
|
+
# such that we can detect if the subscription has changed.
|
344
|
+
content: dict[str, object] = {}
|
345
|
+
if "timeSeriesIds" in resource:
|
346
|
+
content["timeSeriesIds"] = resource["timeSeriesIds"]
|
347
|
+
if "instanceIds" in resource:
|
348
|
+
content["instanceIds"] = resource["instanceIds"]
|
349
|
+
timeseries_hash = calculate_hash(json.dumps(content), shorten=True)
|
350
|
+
extra_str = f"{self._hash_key}: {timeseries_hash}"
|
351
|
+
resource["description"] = suffix_description(
|
352
|
+
extra_str,
|
353
|
+
resource.get("description"),
|
354
|
+
self._description_character_limit,
|
355
|
+
self.get_id(resource),
|
356
|
+
self.display_name,
|
357
|
+
self.console,
|
358
|
+
)
|
359
|
+
|
360
|
+
return resources
|
361
|
+
|
328
362
|
def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> DataPointSubscriptionWrite:
|
329
363
|
if ds_external_id := resource.pop("dataSetExternalId", None):
|
330
364
|
resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run)
|
331
365
|
return DataPointSubscriptionWrite._load(resource)
|
332
366
|
|
333
367
|
def dump_resource(self, resource: DatapointSubscription, local: dict[str, Any] | None = None) -> dict[str, Any]:
|
334
|
-
|
368
|
+
if resource.filter is not None:
|
369
|
+
dumped = resource.as_write().dump()
|
370
|
+
else:
|
371
|
+
# If filter is not set, the subscription uses explicit timeSeriesIds, which are not returned in the
|
372
|
+
# response. Calling .as_write() in this case raises ValueError because either filter or
|
373
|
+
# timeSeriesIds must be set.
|
374
|
+
dumped = resource.dump()
|
375
|
+
for server_prop in ("createdTime", "lastUpdatedTime", "timeSeriesCount"):
|
376
|
+
dumped.pop(server_prop, None)
|
335
377
|
local = local or {}
|
336
378
|
if data_set_id := dumped.pop("dataSetId", None):
|
337
379
|
dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id)
|
338
|
-
|
339
|
-
|
340
|
-
#
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
dumped["timeSeriesIds"], key=lambda ts_id: ts_order_by_id.get(ts_id, end_of_list)
|
346
|
-
)
|
380
|
+
# timeSeriesIds and instanceIds are not returned in the response, so we need to add them
|
381
|
+
# to the dumped resource if they are set in the local resource. If there is a discrepancy between
|
382
|
+
# the local and dumped resource, th hash added to the description will change.
|
383
|
+
if "timeSeriesIds" in local:
|
384
|
+
dumped["timeSeriesIds"] = local["timeSeriesIds"]
|
385
|
+
if "instanceIds" in local:
|
386
|
+
dumped["instanceIds"] = local["instanceIds"]
|
347
387
|
return dumped
|
348
388
|
|
349
389
|
def diff_list(
|
@@ -14,6 +14,10 @@ class SQLTable:
|
|
14
14
|
schema: str
|
15
15
|
name: str
|
16
16
|
|
17
|
+
def __str__(self) -> str:
|
18
|
+
"""Return the table name in the format 'schema.table'."""
|
19
|
+
return f"{self.schema}.{self.name}"
|
20
|
+
|
17
21
|
|
18
22
|
class SQLParser:
|
19
23
|
def __init__(self, query: str, operation: str) -> None:
|
@@ -0,0 +1,42 @@
|
|
1
|
+
from collections.abc import Hashable
|
2
|
+
|
3
|
+
from rich.console import Console
|
4
|
+
|
5
|
+
from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
|
6
|
+
|
7
|
+
|
8
|
+
def suffix_description(
|
9
|
+
suffix: str,
|
10
|
+
description: str | None,
|
11
|
+
description_character_limit: int,
|
12
|
+
identifier: Hashable,
|
13
|
+
resource_type: str,
|
14
|
+
console: Console | None = None,
|
15
|
+
) -> str:
|
16
|
+
"""Appends a suffix to a description if it is not already present.
|
17
|
+
If the description is too long after appending the suffix, it will be truncated to fit within the character limit.
|
18
|
+
|
19
|
+
Args:
|
20
|
+
suffix: The suffix to append to the description.
|
21
|
+
description: The original description to which the suffix will be appended.
|
22
|
+
description_character_limit: The maximum number of characters allowed in the description after appending the suffix.
|
23
|
+
identifier: Hashable identifier for the resource, used in warnings.
|
24
|
+
resource_type: Type of the resource, used in warnings.
|
25
|
+
console: Console object for printing warnings.
|
26
|
+
|
27
|
+
Returns:
|
28
|
+
str: The modified description with the suffix appended, or truncated if necessary.
|
29
|
+
"""
|
30
|
+
if description is None or description == "":
|
31
|
+
return suffix
|
32
|
+
elif description.endswith(suffix):
|
33
|
+
# The suffix is already in the description
|
34
|
+
return description
|
35
|
+
elif len(description) + len(suffix) + 1 < description_character_limit:
|
36
|
+
return f"{description} {suffix}"
|
37
|
+
else:
|
38
|
+
LowSeverityWarning(f"Description is too long for {resource_type} {identifier!r}. Truncating...").print_warning(
|
39
|
+
console=console
|
40
|
+
)
|
41
|
+
truncation = description_character_limit - len(suffix) - 3
|
42
|
+
return f"{description[:truncation]}...{suffix}"
|
cognite_toolkit/_version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "0.5.
|
1
|
+
__version__ = "0.5.64"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: cognite_toolkit
|
3
|
-
Version: 0.5.
|
3
|
+
Version: 0.5.64
|
4
4
|
Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
|
5
5
|
Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
|
6
6
|
Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
|
@@ -1,10 +1,10 @@
|
|
1
1
|
cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
2
|
cognite_toolkit/_cdf.py,sha256=WWMslI-y2VbIYDMH19wnINebGwlOvAeYr-qkPRC1f68,5834
|
3
|
-
cognite_toolkit/_version.py,sha256=
|
3
|
+
cognite_toolkit/_version.py,sha256=YmQRiPOhelComYapMsMGUkwpYKgzkePjx8U_IXrWTZ4,23
|
4
4
|
cognite_toolkit/config.dev.yaml,sha256=CIDmi1OGNOJ-70h2BNCozZRmhvU5BfpZoh6Q04b8iMs,109
|
5
5
|
cognite_toolkit/_builtin_modules/README.md,sha256=roU3G05E6ogP5yhw4hdIvVDKV831zCh2pzt9BVddtBg,307
|
6
6
|
cognite_toolkit/_builtin_modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
7
|
-
cognite_toolkit/_builtin_modules/cdf.toml,sha256=
|
7
|
+
cognite_toolkit/_builtin_modules/cdf.toml,sha256=cNxARlcrmEJ3CEAaRbEy-M9jxKukb_uZ6slnSAO2hI4,273
|
8
8
|
cognite_toolkit/_builtin_modules/packages.toml,sha256=RdY44Sxvh6sUtAkgp1dHID1mtqkOTzP_rbZL2Q27fYw,1147
|
9
9
|
cognite_toolkit/_builtin_modules/bootcamp/README.md,sha256=iTVqoy3PLpC-xPi5pbuMIAEHILBSfWTGLexwa1AltpY,211
|
10
10
|
cognite_toolkit/_builtin_modules/bootcamp/default.config.yaml,sha256=MqYTcRiz03bow4LT8E3jumnd_BsqC5SvjgYOVVkHGE0,93
|
@@ -502,7 +502,7 @@ cognite_toolkit/_cdf_tk/apps/_landing_app.py,sha256=v4t2ryxzFre7y9IkEPIDwmyJDO8V
|
|
502
502
|
cognite_toolkit/_cdf_tk/apps/_migrate_app.py,sha256=GRsOlqYAWB0rsZsdTJTGfjPm1OkbUq7xBrM4pzQRKoY,3708
|
503
503
|
cognite_toolkit/_cdf_tk/apps/_modules_app.py,sha256=tjCP-QbuPYd7iw6dkxnhrrWf514Lr25_oVgSJyJcaL8,6642
|
504
504
|
cognite_toolkit/_cdf_tk/apps/_populate_app.py,sha256=PGUqK_USOqdPCDvUJI-4ne9TN6EssC33pUbEeCmiLPg,2805
|
505
|
-
cognite_toolkit/_cdf_tk/apps/_profile_app.py,sha256=
|
505
|
+
cognite_toolkit/_cdf_tk/apps/_profile_app.py,sha256=TaKTOgkd538QyIWBRdAILJ-TotBxYreZgWBqK4yrebQ,2562
|
506
506
|
cognite_toolkit/_cdf_tk/apps/_purge.py,sha256=RxlUx2vzOuxETBszARUazK8azDpZsf-Y_HHuG9PBVd4,4089
|
507
507
|
cognite_toolkit/_cdf_tk/apps/_repo_app.py,sha256=jOf_s7oUWJqnRyz89JFiSzT2l8GlyQ7wqidHUQavGo0,1455
|
508
508
|
cognite_toolkit/_cdf_tk/apps/_run.py,sha256=vAuPzYBYfAAFJ_0myn5AxFXG3BJWq8A0HKrhMZ7PaHI,8539
|
@@ -555,18 +555,19 @@ cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py,sha256=WDZRthWF8
|
|
555
555
|
cognite_toolkit/_cdf_tk/client/data_classes/pending_instances_ids.py,sha256=W99jhHMLzW_0TvZoaeeaeWXljN9GjuXPoFO-SRjsd-s,1888
|
556
556
|
cognite_toolkit/_cdf_tk/client/data_classes/raw.py,sha256=FRu6MPxGmpl7_6eigsckkmnOeivBWBHlALRaz9c6VhQ,14828
|
557
557
|
cognite_toolkit/_cdf_tk/client/data_classes/robotics.py,sha256=eORgVu4fbXoreyInimBECszgsxzP7VBIIItKXRgsxvU,36143
|
558
|
+
cognite_toolkit/_cdf_tk/client/data_classes/search_config.py,sha256=uUQes8l2LyIxg5SUeLMdXs6xXSp6EUSC1nXAgYfJAV4,7432
|
558
559
|
cognite_toolkit/_cdf_tk/client/data_classes/sequences.py,sha256=02d34fPcJ1H7U5ZnCCfOi36z5WJ4WnRfCWwkp99mW2E,6234
|
559
560
|
cognite_toolkit/_cdf_tk/client/data_classes/statistics.py,sha256=LIYufCSFVLXBuAUVYGaPcjjXzI9BoslxLo6oNBybvE8,4569
|
560
561
|
cognite_toolkit/_cdf_tk/client/data_classes/streamlit_.py,sha256=OGoMQ_K88F9vSZuUbSXcdLBy0X6AdiPB04odxv72UeQ,6712
|
561
562
|
cognite_toolkit/_cdf_tk/client/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
562
563
|
cognite_toolkit/_cdf_tk/client/utils/_concurrency.py,sha256=z6gqFv-kw80DsEpbaR7sI0-_WvZdOdAsR4VoFvTqvyU,1309
|
563
564
|
cognite_toolkit/_cdf_tk/client/utils/_http_client.py,sha256=oXNKrIaizG4WiSAhL_kSCHAuL4aaaEhCU4pOJGxh6Xs,483
|
564
|
-
cognite_toolkit/_cdf_tk/commands/__init__.py,sha256
|
565
|
+
cognite_toolkit/_cdf_tk/commands/__init__.py,sha256=6rUv97s6CB5Fje6eg2X3cd9Za9rYJY51xUcPk_RoJT8,1261
|
565
566
|
cognite_toolkit/_cdf_tk/commands/_base.py,sha256=3Zc3ffR8mjZ1eV7WrC-Y1sYmyMzdbbJDDmsiKEMEJwo,2480
|
566
567
|
cognite_toolkit/_cdf_tk/commands/_changes.py,sha256=3bR_C8p02IW6apexwAAoXuneBM4RcUGdX6Hw_Rtx7Kg,24775
|
567
568
|
cognite_toolkit/_cdf_tk/commands/_cli_commands.py,sha256=6nezoDrw3AkF8hANHjUILgTj_nbdzgT0siweaKI35Fk,1047
|
568
569
|
cognite_toolkit/_cdf_tk/commands/_populate.py,sha256=59VXEFRc4521xhTmCuQnjgWNYE3z4TUkUq8YbFREDGc,12280
|
569
|
-
cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=
|
570
|
+
cognite_toolkit/_cdf_tk/commands/_profile.py,sha256=IQtojZXLMQIPVFuVhLVq5bFEZzuqmXB3tGUB8O-slZA,13098
|
570
571
|
cognite_toolkit/_cdf_tk/commands/_purge.py,sha256=bE2ytMMlMuZc5xGyktKayvZ25x0kdzoKspjwgfab1Qs,26483
|
571
572
|
cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=_IfPBLyfOUc7ubbABiHPpg1MzNGNCxElQ-hmV-vfFDc,1271
|
572
573
|
cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=45_aEPZJeyfGmS2Ph_lucaO7ujY7AF5L5N1K3UH3F0o,2216
|
@@ -614,7 +615,7 @@ cognite_toolkit/_cdf_tk/loaders/_resource_loaders/datamodel_loaders.py,sha256=xK
|
|
614
615
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/extraction_pipeline_loaders.py,sha256=zqNPiIX1xvYV8alpxPKMqyy4QlH6oqDYNtrITC7ZKdo,18188
|
615
616
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/fieldops_loaders.py,sha256=pzjOjnxaJRXId-b0GZ31a6Mab8BdNnKXx3zUiBaxB0E,12203
|
616
617
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/file_loader.py,sha256=49uHmkYA5dzL7fyXYRIeZCkYnB5laclO-6Pis_qhVes,17633
|
617
|
-
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py,sha256=
|
618
|
+
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py,sha256=5_p3G0Kuz08saJs_4T6hD26BwHsVvvFG20IhAPZlZbw,25937
|
618
619
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/group_scoped_loader.py,sha256=Rerw0Y6tY6Nle3vmyl4nhX5lRsVkUVcnp39qZ3R_tDs,1830
|
619
620
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/hosted_extractors.py,sha256=lZuq_dj7_r0O3BcM6xnAenAXAtcJ3ke_LO1IguRKCiE,17527
|
620
621
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/industrial_tool_loaders.py,sha256=j1FsynrFk_iPIAsHuqjXmtbMM7ZM6RVlpQkxCH9ATO0,8493
|
@@ -623,7 +624,7 @@ cognite_toolkit/_cdf_tk/loaders/_resource_loaders/raw_loaders.py,sha256=RM-zxDX6
|
|
623
624
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/relationship_loader.py,sha256=KRWGHk11vRWhFiQUF0vChv95ElGY-E_VGQNCHkuY5Y4,7219
|
624
625
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/robotics_loaders.py,sha256=tgPdVA4B8gNfHeHlEMzDgBaWs2jj-FSZAm1O1r5Uzrc,17224
|
625
626
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/three_d_model_loaders.py,sha256=IcnZ9wyW6Dpl83nlOdAM44UyrMfgum2hLdqi1u2AEuw,8288
|
626
|
-
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py,sha256=
|
627
|
+
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/timeseries_loaders.py,sha256=PU2gRi-9EQKooZ8BLpKnQOtg3lNQ0lRBW0C26BU8r_0,17288
|
627
628
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/transformation_loaders.py,sha256=D0iD7jWPetiUM35JKQ1M_By086k4n5co9RaA8-VfF_A,35304
|
628
629
|
cognite_toolkit/_cdf_tk/loaders/_resource_loaders/workflow_loaders.py,sha256=KmqDOIg6ttVqL_7widJgW5rNjs4eipd28AcLDs_bwDc,28427
|
629
630
|
cognite_toolkit/_cdf_tk/prototypes/import_app.py,sha256=7dy852cBlHI2RQF1MidSmxl0jPBxekGWXnd2VtI7QFI,1899
|
@@ -689,20 +690,21 @@ cognite_toolkit/_cdf_tk/utils/modules.py,sha256=9LLjMtowJWn8KRO3OU12VXGb5q2psrob
|
|
689
690
|
cognite_toolkit/_cdf_tk/utils/producer_worker.py,sha256=v5G1GR2Q7LhkxVTxW8mxe5YQC5o0KBJ-p8nFueyh_ro,8014
|
690
691
|
cognite_toolkit/_cdf_tk/utils/repository.py,sha256=voQLZ6NiNvdAFxqeWHbvzDLsLHl6spjQBihiLyCsGW8,4104
|
691
692
|
cognite_toolkit/_cdf_tk/utils/sentry_utils.py,sha256=YWQdsePeFpT214-T-tZ8kEsUyC84gj8pgks42_BDJuU,575
|
692
|
-
cognite_toolkit/_cdf_tk/utils/sql_parser.py,sha256=
|
693
|
+
cognite_toolkit/_cdf_tk/utils/sql_parser.py,sha256=RhUPWjVjwb9RBv1fixmG7bKvAb4JT_CC0O7Aqnx5Pgg,6196
|
693
694
|
cognite_toolkit/_cdf_tk/utils/table_writers.py,sha256=wEBVlfCFv5bLLy836UiXQubwSxo8kUlSFZeQxnHrTX4,17932
|
694
695
|
cognite_toolkit/_cdf_tk/utils/tarjan.py,sha256=mr3gMzlrkDadn1v7u7-Uzao81KKiM3xfXlZ185HL__A,1359
|
696
|
+
cognite_toolkit/_cdf_tk/utils/text.py,sha256=gBl3o60dXRlEBsg8izdnOmuLo86jr35pQFZcxnKdNSY,1715
|
695
697
|
cognite_toolkit/_repo_files/.env.tmpl,sha256=UmgKZVvIp-OzD8oOcYuwb_6c7vSJsqkLhuFaiVgK7RI,972
|
696
698
|
cognite_toolkit/_repo_files/.gitignore,sha256=3exydcQPCJTldGFJoZy1RPHc1horbAprAoaShU8sYnM,5262
|
697
699
|
cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
|
698
700
|
cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=KVBxW8urCRDtVlJ6HN-kYmw0NCpW6c4lD-nlxz9tZsQ,692
|
699
701
|
cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=Cp4KYraeWPjP8SnnEIbJoJnjmrRUwc982DPjOOzy2iM,722
|
700
|
-
cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=
|
701
|
-
cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=
|
702
|
+
cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=lxYJqdd3aR0ZHU-Xwf4r0tk8vh_yDZABoLHYozuJatA,667
|
703
|
+
cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=pJjrtZvQ_1iIuKom7FAAGa4QxiwRwTSifrdNON5oLMs,2430
|
702
704
|
cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
|
703
705
|
cognite_toolkit/demo/_base.py,sha256=63nWYI_MHU5EuPwEX_inEAQxxiD5P6k8IAmlgl4CxpE,8082
|
704
|
-
cognite_toolkit-0.5.
|
705
|
-
cognite_toolkit-0.5.
|
706
|
-
cognite_toolkit-0.5.
|
707
|
-
cognite_toolkit-0.5.
|
708
|
-
cognite_toolkit-0.5.
|
706
|
+
cognite_toolkit-0.5.64.dist-info/METADATA,sha256=PulESZBXZQdjGl-QJlNBp0VFkJ6HgMlHjxKykzoREu8,4410
|
707
|
+
cognite_toolkit-0.5.64.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
708
|
+
cognite_toolkit-0.5.64.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
|
709
|
+
cognite_toolkit-0.5.64.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
|
710
|
+
cognite_toolkit-0.5.64.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|