cognite-toolkit 0.5.61__py3-none-any.whl → 0.5.63__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_builtin_modules/cdf.toml +1 -1
- cognite_toolkit/_cdf_tk/apps/_modules_app.py +9 -9
- cognite_toolkit/_cdf_tk/apps/_profile_app.py +33 -3
- cognite_toolkit/_cdf_tk/cdf_toml.py +46 -3
- cognite_toolkit/_cdf_tk/client/data_classes/canvas.py +395 -0
- cognite_toolkit/_cdf_tk/client/data_classes/search_config.py +198 -0
- cognite_toolkit/_cdf_tk/commands/__init__.py +3 -2
- cognite_toolkit/_cdf_tk/commands/_profile.py +271 -89
- cognite_toolkit/_cdf_tk/commands/modules.py +128 -5
- cognite_toolkit/_cdf_tk/data_classes/_packages.py +2 -2
- cognite_toolkit/_cdf_tk/feature_flags.py +4 -0
- cognite_toolkit/_cdf_tk/utils/sql_parser.py +4 -0
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_version.py +1 -1
- cognite_toolkit/config.dev.yaml +7 -0
- {cognite_toolkit-0.5.61.dist-info → cognite_toolkit-0.5.63.dist-info}/METADATA +2 -2
- {cognite_toolkit-0.5.61.dist-info → cognite_toolkit-0.5.63.dist-info}/RECORD +22 -19
- /cognite_toolkit/_builtin_modules/{package.toml → packages.toml} +0 -0
- {cognite_toolkit-0.5.61.dist-info → cognite_toolkit-0.5.63.dist-info}/WHEEL +0 -0
- {cognite_toolkit-0.5.61.dist-info → cognite_toolkit-0.5.63.dist-info}/entry_points.txt +0 -0
- {cognite_toolkit-0.5.61.dist-info → cognite_toolkit-0.5.63.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,198 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
from abc import ABC
|
4
|
+
from dataclasses import dataclass
|
5
|
+
from typing import Any
|
6
|
+
|
7
|
+
from cognite.client import CogniteClient
|
8
|
+
from cognite.client.data_classes._base import (
|
9
|
+
CogniteObject,
|
10
|
+
CogniteResourceList,
|
11
|
+
WriteableCogniteResource,
|
12
|
+
WriteableCogniteResourceList,
|
13
|
+
)
|
14
|
+
from typing_extensions import Self
|
15
|
+
|
16
|
+
|
17
|
+
@dataclass
|
18
|
+
class SearchConfigView(CogniteObject):
|
19
|
+
external_id: str
|
20
|
+
space: str
|
21
|
+
|
22
|
+
@classmethod
|
23
|
+
def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
|
24
|
+
return cls(
|
25
|
+
external_id=resource["externalId"],
|
26
|
+
space=resource["space"],
|
27
|
+
)
|
28
|
+
|
29
|
+
|
30
|
+
@dataclass
|
31
|
+
class SearchConfigViewProperty(CogniteObject):
|
32
|
+
property: str
|
33
|
+
disabled: bool | None = None
|
34
|
+
selected: bool | None = None
|
35
|
+
hidden: bool | None = None
|
36
|
+
|
37
|
+
@classmethod
|
38
|
+
def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
|
39
|
+
return cls(
|
40
|
+
property=resource["property"],
|
41
|
+
disabled=resource.get("disabled"),
|
42
|
+
selected=resource.get("selected"),
|
43
|
+
hidden=resource.get("hidden"),
|
44
|
+
)
|
45
|
+
|
46
|
+
|
47
|
+
class SearchConfigCore(WriteableCogniteResource["SearchConfigWrite"], ABC):
|
48
|
+
"""
|
49
|
+
Core model for a single Configuration.
|
50
|
+
|
51
|
+
Args:
|
52
|
+
view: The configuration for one specific view.
|
53
|
+
id: A server-generated ID for the object.
|
54
|
+
use_as_name: The name of property to use for the name column in the UI.
|
55
|
+
use_as_description: The name of property to use for the description column in the UI.
|
56
|
+
column_layout: Array of column configurations per property.
|
57
|
+
filter_layout: Array of filter configurations per property.
|
58
|
+
properties_layout: Array of property configurations per property.
|
59
|
+
"""
|
60
|
+
|
61
|
+
def __init__(
|
62
|
+
self,
|
63
|
+
view: SearchConfigView,
|
64
|
+
id: int | None = None,
|
65
|
+
use_as_name: str | None = None,
|
66
|
+
use_as_description: str | None = None,
|
67
|
+
column_layout: list[SearchConfigViewProperty] | None = None,
|
68
|
+
filter_layout: list[SearchConfigViewProperty] | None = None,
|
69
|
+
properties_layout: list[SearchConfigViewProperty] | None = None,
|
70
|
+
) -> None:
|
71
|
+
self.view = view
|
72
|
+
self.id = id
|
73
|
+
self.use_as_name = use_as_name
|
74
|
+
self.use_as_description = use_as_description
|
75
|
+
self.column_layout = column_layout
|
76
|
+
self.filter_layout = filter_layout
|
77
|
+
self.properties_layout = properties_layout
|
78
|
+
|
79
|
+
def as_write(self) -> SearchConfigWrite:
|
80
|
+
return SearchConfigWrite(
|
81
|
+
view=self.view,
|
82
|
+
id=self.id,
|
83
|
+
use_as_name=self.use_as_name,
|
84
|
+
use_as_description=self.use_as_description,
|
85
|
+
column_layout=self.column_layout,
|
86
|
+
filter_layout=self.filter_layout,
|
87
|
+
properties_layout=self.properties_layout,
|
88
|
+
)
|
89
|
+
|
90
|
+
def dump(self, camel_case: bool = True) -> dict[str, Any]:
|
91
|
+
output = super().dump(camel_case)
|
92
|
+
if self.column_layout:
|
93
|
+
output["columLayout" if camel_case else "column_layout"] = [
|
94
|
+
_data.dump(camel_case) for _data in self.column_layout
|
95
|
+
]
|
96
|
+
if self.filter_layout:
|
97
|
+
output["filterLayout" if camel_case else "filter_layout"] = [
|
98
|
+
_data.dump(camel_case) for _data in self.filter_layout
|
99
|
+
]
|
100
|
+
if self.properties_layout:
|
101
|
+
output["propertiesLayout" if camel_case else "properties_layout"] = [
|
102
|
+
_data.dump(camel_case) for _data in self.properties_layout
|
103
|
+
]
|
104
|
+
if self.view:
|
105
|
+
output["view"] = self.view.dump(camel_case)
|
106
|
+
return output
|
107
|
+
|
108
|
+
|
109
|
+
class SearchConfigWrite(SearchConfigCore):
|
110
|
+
@classmethod
|
111
|
+
def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
|
112
|
+
return cls(
|
113
|
+
id=resource.get("id"),
|
114
|
+
view=SearchConfigView.load(resource["view"]),
|
115
|
+
use_as_name=resource.get("useAsName"),
|
116
|
+
use_as_description=resource.get("useAsDescription"),
|
117
|
+
column_layout=[SearchConfigViewProperty.load(item) for item in resource.get("columnLayout", [])]
|
118
|
+
if resource.get("columnLayout")
|
119
|
+
else None,
|
120
|
+
filter_layout=[SearchConfigViewProperty.load(item) for item in resource.get("filterLayout", [])]
|
121
|
+
if resource.get("filterLayout")
|
122
|
+
else None,
|
123
|
+
properties_layout=[SearchConfigViewProperty.load(item) for item in resource.get("propertiesLayout", [])]
|
124
|
+
if resource.get("propertiesLayout")
|
125
|
+
else None,
|
126
|
+
)
|
127
|
+
|
128
|
+
|
129
|
+
class SearchConfig(SearchConfigCore):
|
130
|
+
"""
|
131
|
+
Response model for a single Configuration.
|
132
|
+
|
133
|
+
Args:
|
134
|
+
view: The configuration for one specific view.
|
135
|
+
id: A server-generated ID for the object.
|
136
|
+
created_time: The time when the search config was created.
|
137
|
+
updated_time: The time when the search config was last updated.
|
138
|
+
use_as_name: The name of property to use for the name column in the UI.
|
139
|
+
use_as_description: The name of property to use for the description column in the UI.
|
140
|
+
column_layout: Array of column configurations per property.
|
141
|
+
filter_layout: Array of filter configurations per property.
|
142
|
+
properties_layout: Array of property configurations per property.
|
143
|
+
"""
|
144
|
+
|
145
|
+
def __init__(
|
146
|
+
self,
|
147
|
+
view: SearchConfigView,
|
148
|
+
id: int,
|
149
|
+
created_time: int,
|
150
|
+
updated_time: int,
|
151
|
+
use_as_name: str | None = None,
|
152
|
+
use_as_description: str | None = None,
|
153
|
+
column_layout: list[SearchConfigViewProperty] | None = None,
|
154
|
+
filter_layout: list[SearchConfigViewProperty] | None = None,
|
155
|
+
properties_layout: list[SearchConfigViewProperty] | None = None,
|
156
|
+
) -> None:
|
157
|
+
super().__init__(
|
158
|
+
view,
|
159
|
+
id,
|
160
|
+
use_as_name,
|
161
|
+
use_as_description,
|
162
|
+
column_layout,
|
163
|
+
filter_layout,
|
164
|
+
properties_layout,
|
165
|
+
)
|
166
|
+
self.created_time = created_time
|
167
|
+
self.updated_time = updated_time
|
168
|
+
|
169
|
+
@classmethod
|
170
|
+
def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self:
|
171
|
+
return cls(
|
172
|
+
view=SearchConfigView.load(resource["view"]),
|
173
|
+
id=resource["id"],
|
174
|
+
created_time=resource["createdTime"],
|
175
|
+
updated_time=resource["lastUpdatedTime"],
|
176
|
+
use_as_name=resource.get("useAsName"),
|
177
|
+
use_as_description=resource.get("useAsDescription"),
|
178
|
+
column_layout=[SearchConfigViewProperty.load(item) for item in resource.get("columnLayout", [])]
|
179
|
+
if resource.get("columnLayout")
|
180
|
+
else None,
|
181
|
+
filter_layout=[SearchConfigViewProperty.load(item) for item in resource.get("filterLayout", [])]
|
182
|
+
if resource.get("filterLayout")
|
183
|
+
else None,
|
184
|
+
properties_layout=[SearchConfigViewProperty.load(item) for item in resource.get("propertiesLayout", [])]
|
185
|
+
if resource.get("propertiesLayout")
|
186
|
+
else None,
|
187
|
+
)
|
188
|
+
|
189
|
+
|
190
|
+
class SearchConfigWriteList(CogniteResourceList):
|
191
|
+
_RESOURCE = SearchConfigWrite
|
192
|
+
|
193
|
+
|
194
|
+
class SearchConfigList(WriteableCogniteResourceList[SearchConfigWrite, SearchConfig]):
|
195
|
+
_RESOURCE = SearchConfig
|
196
|
+
|
197
|
+
def as_write(self) -> SearchConfigWriteList:
|
198
|
+
return SearchConfigWriteList([searchConfig.as_write() for searchConfig in self])
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from ._migrate import MigrateTimeseriesCommand, MigrationPrepareCommand
|
2
2
|
from ._populate import PopulateCommand
|
3
|
-
from ._profile import
|
3
|
+
from ._profile import ProfileAssetCentricCommand, ProfileTransformationCommand
|
4
4
|
from ._purge import PurgeCommand
|
5
5
|
from .auth import AuthCommand
|
6
6
|
from .build_cmd import BuildCommand
|
@@ -30,7 +30,8 @@ __all__ = [
|
|
30
30
|
"MigrationPrepareCommand",
|
31
31
|
"ModulesCommand",
|
32
32
|
"PopulateCommand",
|
33
|
-
"
|
33
|
+
"ProfileAssetCentricCommand",
|
34
|
+
"ProfileTransformationCommand",
|
34
35
|
"PullCommand",
|
35
36
|
"PurgeCommand",
|
36
37
|
"RepoCommand",
|
@@ -1,12 +1,21 @@
|
|
1
|
-
|
1
|
+
import itertools
|
2
|
+
from abc import ABC, abstractmethod
|
3
|
+
from collections.abc import Callable, Iterable, Mapping
|
2
4
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
5
|
+
from functools import cached_property
|
6
|
+
from typing import ClassVar, Literal, TypeAlias, overload
|
3
7
|
|
8
|
+
from cognite.client.data_classes import Transformation
|
4
9
|
from cognite.client.exceptions import CogniteException
|
10
|
+
from rich import box
|
11
|
+
from rich.console import Console
|
5
12
|
from rich.live import Live
|
6
13
|
from rich.spinner import Spinner
|
7
14
|
from rich.table import Table
|
8
15
|
|
9
16
|
from cognite_toolkit._cdf_tk.client import ToolkitClient
|
17
|
+
from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
|
18
|
+
from cognite_toolkit._cdf_tk.utils import humanize_collection
|
10
19
|
from cognite_toolkit._cdf_tk.utils.aggregators import (
|
11
20
|
AssetAggregator,
|
12
21
|
AssetCentricAggregator,
|
@@ -19,112 +28,285 @@ from cognite_toolkit._cdf_tk.utils.aggregators import (
|
|
19
28
|
SequenceAggregator,
|
20
29
|
TimeSeriesAggregator,
|
21
30
|
)
|
31
|
+
from cognite_toolkit._cdf_tk.utils.sql_parser import SQLParser, SQLTable
|
22
32
|
|
23
33
|
from ._base import ToolkitCommand
|
24
34
|
|
25
35
|
|
26
|
-
class
|
27
|
-
|
28
|
-
|
29
|
-
Count = "Count"
|
30
|
-
MetadataKeyCount = "Metadata Key Count"
|
31
|
-
LabelCount = "Label Count"
|
32
|
-
Transformation = "Transformations"
|
36
|
+
class WaitingAPICallClass:
|
37
|
+
def __bool__(self) -> bool:
|
38
|
+
return False
|
33
39
|
|
34
|
-
columns = (
|
35
|
-
Columns.Resource,
|
36
|
-
Columns.Count,
|
37
|
-
Columns.MetadataKeyCount,
|
38
|
-
Columns.LabelCount,
|
39
|
-
Columns.Transformation,
|
40
|
-
)
|
41
|
-
spinner_speed = 1.0
|
42
40
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
41
|
+
WaitingAPICall = WaitingAPICallClass()
|
42
|
+
|
43
|
+
PendingCellValue: TypeAlias = int | float | str | bool | None | WaitingAPICallClass
|
44
|
+
CellValue: TypeAlias = int | float | str | bool | None
|
45
|
+
PendingTable: TypeAlias = dict[tuple[str, str], PendingCellValue]
|
46
|
+
|
47
|
+
|
48
|
+
class ProfileCommand(ToolkitCommand, ABC):
|
49
|
+
def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
|
50
|
+
super().__init__(print_warning, skip_tracking, silent)
|
51
|
+
self.table_title = self.__class__.__name__.removesuffix("Command")
|
52
|
+
|
53
|
+
class Columns: # Placeholder for columns, subclasses should define their own Columns class
|
54
|
+
...
|
55
|
+
|
56
|
+
spinner_args: ClassVar[Mapping] = dict(name="arc", text="loading...", style="bold green", speed=1.0)
|
57
|
+
|
58
|
+
max_workers = 8
|
59
|
+
is_dynamic_table = False
|
60
|
+
|
61
|
+
@cached_property
|
62
|
+
def columns(self) -> tuple[str, ...]:
|
63
|
+
return (
|
64
|
+
tuple([attr for attr in self.Columns.__dict__.keys() if not attr.startswith("_")])
|
65
|
+
if hasattr(self, "Columns")
|
66
|
+
else tuple()
|
67
|
+
)
|
68
|
+
|
69
|
+
def create_profile_table(self, client: ToolkitClient) -> list[dict[str, CellValue]]:
|
70
|
+
console = Console()
|
71
|
+
with console.status("Setting up", spinner="aesthetic", speed=0.4) as _:
|
72
|
+
table = self.create_initial_table(client)
|
73
|
+
with (
|
74
|
+
Live(self.draw_table(table), refresh_per_second=4, console=console) as live,
|
75
|
+
ThreadPoolExecutor(max_workers=self.max_workers) as executor,
|
76
|
+
):
|
77
|
+
while True:
|
78
|
+
current_calls = {
|
79
|
+
executor.submit(self.call_api(row, col, client)): (row, col)
|
80
|
+
for (row, col), cell in table.items()
|
81
|
+
if cell is WaitingAPICall
|
66
82
|
}
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
83
|
+
if not current_calls:
|
84
|
+
break
|
85
|
+
for future in as_completed(current_calls):
|
86
|
+
row, col = current_calls[future]
|
87
|
+
try:
|
88
|
+
result = future.result()
|
89
|
+
except CogniteException as e:
|
90
|
+
result = type(e).__name__
|
91
|
+
table[(row, col)] = self.format_result(result, row, col)
|
92
|
+
if self.is_dynamic_table:
|
93
|
+
table = self.update_table(table, result, row, col)
|
94
|
+
live.update(self.draw_table(table))
|
95
|
+
return self.as_record_format(table, allow_waiting_api_call=False)
|
72
96
|
|
73
|
-
@
|
74
|
-
def
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
row: dict[str, str | Spinner] = {
|
81
|
-
cls.Columns.Resource: aggregator.display_name,
|
82
|
-
cls.Columns.Count: Spinner("arc", text="loading...", style="bold green", speed=cls.spinner_speed),
|
83
|
-
}
|
84
|
-
api_calls[(index, cls.Columns.Count)] = cls._call_api(aggregator.count)
|
85
|
-
count: str | Spinner = "-"
|
86
|
-
if isinstance(aggregator, MetadataAggregator):
|
87
|
-
count = Spinner("arc", text="loading...", style="bold green", speed=cls.spinner_speed)
|
88
|
-
api_calls[(index, cls.Columns.MetadataKeyCount)] = cls._call_api(aggregator.metadata_key_count)
|
89
|
-
row[cls.Columns.MetadataKeyCount] = count
|
97
|
+
@abstractmethod
|
98
|
+
def create_initial_table(self, client: ToolkitClient) -> PendingTable:
|
99
|
+
"""
|
100
|
+
Create the initial table with placeholders for API calls.
|
101
|
+
Each cell that requires an API call should be initialized with WaitingAPICall.
|
102
|
+
"""
|
103
|
+
raise NotImplementedError("Subclasses must implement create_initial_table.")
|
90
104
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
api_calls[(index, cls.Columns.LabelCount)] = cls._call_api(aggregator.label_count)
|
95
|
-
row[cls.Columns.LabelCount] = count
|
105
|
+
@abstractmethod
|
106
|
+
def call_api(self, row: str, col: str, client: ToolkitClient) -> Callable:
|
107
|
+
raise NotImplementedError("Subclasses must implement call_api.")
|
96
108
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
109
|
+
def format_result(self, result: object, row: str, col: str) -> CellValue:
|
110
|
+
"""
|
111
|
+
Format the result of an API call for display in the table.
|
112
|
+
This can be overridden by subclasses to customize formatting.
|
113
|
+
"""
|
114
|
+
if isinstance(result, int | float | bool | str):
|
115
|
+
return result
|
116
|
+
raise NotImplementedError("Subclasses must implement format_result.")
|
101
117
|
|
102
|
-
|
103
|
-
|
118
|
+
def update_table(
|
119
|
+
self,
|
120
|
+
current_table: PendingTable,
|
121
|
+
result: object,
|
122
|
+
row: str,
|
123
|
+
col: str,
|
124
|
+
) -> PendingTable:
|
125
|
+
raise NotImplementedError("Subclasses must implement update_table.")
|
104
126
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
title="Asset Centric Profile",
|
127
|
+
def draw_table(self, table: PendingTable) -> Table:
|
128
|
+
rich_table = Table(
|
129
|
+
title=self.table_title,
|
109
130
|
title_justify="left",
|
110
131
|
show_header=True,
|
111
132
|
header_style="bold magenta",
|
133
|
+
box=box.MINIMAL,
|
112
134
|
)
|
113
|
-
for col in
|
114
|
-
|
135
|
+
for col in self.columns:
|
136
|
+
rich_table.add_column(col)
|
137
|
+
|
138
|
+
rows = self.as_record_format(table)
|
115
139
|
|
116
140
|
for row in rows:
|
117
|
-
|
118
|
-
return
|
141
|
+
rich_table.add_row(*[self._as_cell(value) for value in row.values()])
|
142
|
+
return rich_table
|
119
143
|
|
120
|
-
@
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
144
|
+
@classmethod
|
145
|
+
@overload
|
146
|
+
def as_record_format(
|
147
|
+
cls, table: PendingTable, allow_waiting_api_call: Literal[True] = True
|
148
|
+
) -> list[dict[str, PendingCellValue]]: ...
|
149
|
+
|
150
|
+
@classmethod
|
151
|
+
@overload
|
152
|
+
def as_record_format(
|
153
|
+
cls,
|
154
|
+
table: PendingTable,
|
155
|
+
allow_waiting_api_call: Literal[False],
|
156
|
+
) -> list[dict[str, CellValue]]: ...
|
157
|
+
|
158
|
+
@classmethod
|
159
|
+
def as_record_format(
|
160
|
+
cls,
|
161
|
+
table: PendingTable,
|
162
|
+
allow_waiting_api_call: bool = True,
|
163
|
+
) -> list[dict[str, PendingCellValue]] | list[dict[str, CellValue]]:
|
164
|
+
rows: list[dict[str, PendingCellValue]] = []
|
165
|
+
row_indices: dict[str, int] = {}
|
166
|
+
for (row, col), value in table.items():
|
167
|
+
if value is WaitingAPICall and not allow_waiting_api_call:
|
168
|
+
value = None
|
169
|
+
if row not in row_indices:
|
170
|
+
row_indices[row] = len(rows)
|
171
|
+
rows.append({col: value})
|
127
172
|
else:
|
128
|
-
|
173
|
+
rows[row_indices[row]][col] = value
|
174
|
+
return rows
|
175
|
+
|
176
|
+
def _as_cell(self, value: PendingCellValue) -> str | Spinner:
|
177
|
+
if isinstance(value, WaitingAPICallClass):
|
178
|
+
return Spinner(**self.spinner_args)
|
179
|
+
elif isinstance(value, int):
|
180
|
+
return f"{value:,}"
|
181
|
+
elif isinstance(value, float):
|
182
|
+
return f"{value:.2f}"
|
183
|
+
elif value is None:
|
184
|
+
return "-"
|
185
|
+
return str(value)
|
186
|
+
|
187
|
+
|
188
|
+
class ProfileAssetCentricCommand(ProfileCommand):
|
189
|
+
def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
|
190
|
+
super().__init__(print_warning, skip_tracking, silent)
|
191
|
+
self.table_title = "Asset Centric Profile"
|
192
|
+
self.aggregators: dict[str, AssetCentricAggregator] = {}
|
193
|
+
|
194
|
+
class Columns:
|
195
|
+
Resource = "Resource"
|
196
|
+
Count = "Count"
|
197
|
+
MetadataKeyCount = "Metadata Key Count"
|
198
|
+
LabelCount = "Label Count"
|
199
|
+
Transformation = "Transformations"
|
200
|
+
|
201
|
+
def asset_centric(self, client: ToolkitClient, verbose: bool = False) -> list[dict[str, CellValue]]:
|
202
|
+
self.aggregators.update(
|
203
|
+
{
|
204
|
+
agg.display_name: agg
|
205
|
+
for agg in [
|
206
|
+
AssetAggregator(client),
|
207
|
+
EventAggregator(client),
|
208
|
+
FileAggregator(client),
|
209
|
+
TimeSeriesAggregator(client),
|
210
|
+
SequenceAggregator(client),
|
211
|
+
RelationshipAggregator(client),
|
212
|
+
LabelCountAggregator(client),
|
213
|
+
]
|
214
|
+
}
|
215
|
+
)
|
216
|
+
return self.create_profile_table(client)
|
217
|
+
|
218
|
+
def create_initial_table(self, client: ToolkitClient) -> PendingTable:
|
219
|
+
table: dict[tuple[str, str], str | int | float | bool | None | WaitingAPICallClass] = {}
|
220
|
+
for index, aggregator in self.aggregators.items():
|
221
|
+
table[(index, self.Columns.Resource)] = aggregator.display_name
|
222
|
+
table[(index, self.Columns.Count)] = WaitingAPICall
|
223
|
+
if isinstance(aggregator, MetadataAggregator):
|
224
|
+
table[(index, self.Columns.MetadataKeyCount)] = WaitingAPICall
|
225
|
+
else:
|
226
|
+
table[(index, self.Columns.MetadataKeyCount)] = None
|
227
|
+
if isinstance(aggregator, LabelAggregator):
|
228
|
+
table[(index, self.Columns.LabelCount)] = WaitingAPICall
|
229
|
+
else:
|
230
|
+
table[(index, self.Columns.LabelCount)] = None
|
231
|
+
table[(index, self.Columns.Transformation)] = WaitingAPICall
|
232
|
+
return table
|
233
|
+
|
234
|
+
def call_api(self, row: str, col: str, client: ToolkitClient) -> Callable:
|
235
|
+
aggregator = self.aggregators[row]
|
236
|
+
if col == self.Columns.Count:
|
237
|
+
return aggregator.count
|
238
|
+
elif col == self.Columns.MetadataKeyCount and isinstance(aggregator, MetadataAggregator):
|
239
|
+
return aggregator.metadata_key_count
|
240
|
+
elif col == self.Columns.LabelCount and isinstance(aggregator, LabelAggregator):
|
241
|
+
return aggregator.label_count
|
242
|
+
elif col == self.Columns.Transformation:
|
243
|
+
return aggregator.transformation_count
|
244
|
+
raise ValueError(f"Unknown column: {col} for row: {row}")
|
245
|
+
|
246
|
+
|
247
|
+
class ProfileTransformationCommand(ProfileCommand):
|
248
|
+
valid_destinations: frozenset[str] = frozenset({"assets", "files", "events", "timeseries", "sequences"})
|
249
|
+
|
250
|
+
def __init__(self, print_warning: bool = True, skip_tracking: bool = False, silent: bool = False) -> None:
|
251
|
+
super().__init__(print_warning, skip_tracking, silent)
|
252
|
+
self.table_title = "Transformation Profile"
|
253
|
+
self.destination_type: Literal["assets", "files", "events", "timeseries", "sequences"] | None = None
|
254
|
+
|
255
|
+
class Columns:
|
256
|
+
Transformation = "Transformation"
|
257
|
+
Source = "Sources"
|
258
|
+
DestinationColumns = "Destination Columns"
|
259
|
+
Destination = "Destination"
|
260
|
+
ConflictMode = "Conflict Mode"
|
261
|
+
IsPaused = "Is Paused"
|
262
|
+
|
263
|
+
def transformation(
|
264
|
+
self, client: ToolkitClient, destination_type: str | None = None, verbose: bool = False
|
265
|
+
) -> list[dict[str, CellValue]]:
|
266
|
+
self.destination_type = self._validate_destination_type(destination_type)
|
267
|
+
return self.create_profile_table(client)
|
268
|
+
|
269
|
+
@classmethod
|
270
|
+
def _validate_destination_type(
|
271
|
+
cls, destination_type: str | None
|
272
|
+
) -> Literal["assets", "files", "events", "timeseries", "sequences"]:
|
273
|
+
if destination_type is None or destination_type not in cls.valid_destinations:
|
274
|
+
raise ToolkitValueError(
|
275
|
+
f"Invalid destination type: {destination_type}. Must be one of {humanize_collection(cls.valid_destinations)}."
|
276
|
+
)
|
277
|
+
# We validated the destination type above
|
278
|
+
return destination_type # type: ignore[return-value]
|
279
|
+
|
280
|
+
def create_initial_table(self, client: ToolkitClient) -> dict[tuple[str, str], PendingCellValue]:
|
281
|
+
if self.valid_destinations is None:
|
282
|
+
raise ToolkitValueError("Destination type must be set before calling create_initial_table.")
|
283
|
+
iterable: Iterable[Transformation] = client.transformations.list(
|
284
|
+
destination_type=self.destination_type, limit=-1
|
285
|
+
)
|
286
|
+
if self.destination_type == "assets":
|
287
|
+
iterable = itertools.chain(iterable, client.transformations(destination_type="asset_hierarchy", limit=-1))
|
288
|
+
table: dict[tuple[str, str], PendingCellValue] = {}
|
289
|
+
for transformation in iterable:
|
290
|
+
sources: list[SQLTable] = []
|
291
|
+
destination_columns: list[str] = []
|
292
|
+
if transformation.query:
|
293
|
+
parser = SQLParser(transformation.query, operation="Profile transformations")
|
294
|
+
sources = parser.sources
|
295
|
+
destination_columns = parser.destination_columns
|
296
|
+
index = str(transformation.id)
|
297
|
+
table[(index, self.Columns.Transformation)] = transformation.name or transformation.external_id or "Unknown"
|
298
|
+
table[(index, self.Columns.Source)] = ", ".join(map(str, sources))
|
299
|
+
table[(index, self.Columns.DestinationColumns)] = (
|
300
|
+
", ".join(destination_columns) or None if destination_columns else None
|
301
|
+
)
|
302
|
+
table[(index, self.Columns.Destination)] = (
|
303
|
+
transformation.destination.type or "Unknown" if transformation.destination else "Unknown"
|
304
|
+
)
|
305
|
+
table[(index, self.Columns.ConflictMode)] = transformation.conflict_mode or "Unknown"
|
306
|
+
table[(index, self.Columns.IsPaused)] = (
|
307
|
+
str(transformation.schedule.is_paused) if transformation.schedule else "No schedule"
|
308
|
+
)
|
309
|
+
return table
|
129
310
|
|
130
|
-
|
311
|
+
def call_api(self, row: str, col: str, client: ToolkitClient) -> Callable:
|
312
|
+
raise NotImplementedError(f"{type(self).__name__} does not support API calls for {col} in row {row}.")
|