castor-extractor 0.24.25__py3-none-any.whl → 0.24.29__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of castor-extractor might be problematic. Click here for more details.
- CHANGELOG.md +16 -0
- castor_extractor/transformation/coalesce/client/client.py +1 -1
- castor_extractor/visualization/strategy/assets.py +7 -1
- castor_extractor/visualization/strategy/client/client.py +89 -117
- castor_extractor/visualization/strategy/client/properties.py +145 -0
- castor_extractor/warehouse/snowflake/queries/database.sql +1 -0
- {castor_extractor-0.24.25.dist-info → castor_extractor-0.24.29.dist-info}/METADATA +17 -1
- {castor_extractor-0.24.25.dist-info → castor_extractor-0.24.29.dist-info}/RECORD +11 -10
- {castor_extractor-0.24.25.dist-info → castor_extractor-0.24.29.dist-info}/LICENCE +0 -0
- {castor_extractor-0.24.25.dist-info → castor_extractor-0.24.29.dist-info}/WHEEL +0 -0
- {castor_extractor-0.24.25.dist-info → castor_extractor-0.24.29.dist-info}/entry_points.txt +0 -0
CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.24.29 - 2025-06-24
|
|
4
|
+
|
|
5
|
+
* Strategy: skip descriptions on ValueErrors
|
|
6
|
+
|
|
7
|
+
## 0.24.28 - 2025-06-20
|
|
8
|
+
|
|
9
|
+
* Snowflake: ignore private notebooks
|
|
10
|
+
|
|
11
|
+
## 0.24.27 - 2025-06-20
|
|
12
|
+
|
|
13
|
+
* Strategy: extract logical tables
|
|
14
|
+
|
|
15
|
+
## 0.24.26 - 2025-06-16
|
|
16
|
+
|
|
17
|
+
* Coalesce: increase _MAX_ERRORS client parameter
|
|
18
|
+
|
|
3
19
|
## 0.24.25 - 2025-06-12
|
|
4
20
|
|
|
5
21
|
* DBT: Fix API base url
|
|
@@ -1,14 +1,20 @@
|
|
|
1
|
-
from ...types import ExternalAsset
|
|
1
|
+
from ...types import ExternalAsset, classproperty
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class StrategyAsset(ExternalAsset):
|
|
5
5
|
"""Strategy assets that can be extracted"""
|
|
6
6
|
|
|
7
7
|
ATTRIBUTE = "attribute"
|
|
8
|
+
COLUMN = "column"
|
|
8
9
|
CUBE = "cube"
|
|
9
10
|
DASHBOARD = "dashboard"
|
|
10
11
|
DOCUMENT = "document"
|
|
11
12
|
FACT = "fact"
|
|
13
|
+
LOGICAL_TABLE = "logical_table"
|
|
12
14
|
METRIC = "metric"
|
|
13
15
|
REPORT = "report"
|
|
14
16
|
USER = "user"
|
|
17
|
+
|
|
18
|
+
@classproperty
|
|
19
|
+
def optional(cls) -> set["StrategyAsset"]:
|
|
20
|
+
return {StrategyAsset.COLUMN}
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from collections.abc import Iterator
|
|
3
|
-
from enum import Enum
|
|
4
3
|
from typing import Any, Callable, Optional
|
|
5
4
|
from urllib.parse import urlparse
|
|
6
5
|
|
|
7
6
|
from mstrio.connection import Connection # type: ignore
|
|
8
7
|
from mstrio.helpers import IServerError # type: ignore
|
|
9
8
|
from mstrio.modeling import ( # type: ignore
|
|
9
|
+
Attribute,
|
|
10
|
+
LogicalTable,
|
|
11
|
+
PhysicalTable,
|
|
12
|
+
PhysicalTableType,
|
|
10
13
|
list_attributes,
|
|
11
14
|
list_facts,
|
|
15
|
+
list_logical_tables,
|
|
12
16
|
list_metrics,
|
|
13
17
|
)
|
|
14
18
|
from mstrio.project_objects import ( # type: ignore
|
|
@@ -19,93 +23,21 @@ from mstrio.project_objects import ( # type: ignore
|
|
|
19
23
|
list_reports,
|
|
20
24
|
)
|
|
21
25
|
from mstrio.server import Environment # type: ignore
|
|
22
|
-
from mstrio.types import ObjectSubTypes, ObjectTypes # type: ignore
|
|
23
26
|
from mstrio.users_and_groups import User, list_users # type: ignore
|
|
24
27
|
from mstrio.utils.entity import Entity # type: ignore
|
|
25
|
-
from mstrio.utils.helper import is_dashboard # type: ignore
|
|
26
|
-
from pydantic import BaseModel, ConfigDict
|
|
27
28
|
|
|
28
29
|
from ..assets import StrategyAsset
|
|
29
30
|
from .credentials import StrategyCredentials
|
|
31
|
+
from .properties import (
|
|
32
|
+
column_properties,
|
|
33
|
+
format_url,
|
|
34
|
+
list_dependencies,
|
|
35
|
+
lookup_table_id,
|
|
36
|
+
safe_get_property,
|
|
37
|
+
)
|
|
30
38
|
|
|
31
39
|
logger = logging.getLogger(__name__)
|
|
32
40
|
|
|
33
|
-
_BATCH_SIZE: int = 100
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class URLTemplates(Enum):
|
|
37
|
-
DASHBOARD = "https://{hostname}/MicroStrategyLibrary/app/{project_id}/{id_}"
|
|
38
|
-
DOCUMENT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?documentID={id_}&projectID={project_id}"
|
|
39
|
-
REPORT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?reportID={id_}&projectID={project_id}"
|
|
40
|
-
FOLDER = "https://{hostname}/MicroStrategy/servlet/mstrWeb?folderID={id_}&projectID={project_id}"
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def _is_dashboard(entity: Entity) -> bool:
|
|
44
|
-
"""
|
|
45
|
-
Returns True if the entity is a Dashboard. They can only be distinguished
|
|
46
|
-
from Documents by checking the `view_media` property.
|
|
47
|
-
"""
|
|
48
|
-
is_type_document = entity.type == ObjectTypes.DOCUMENT_DEFINITION
|
|
49
|
-
return is_type_document and is_dashboard(entity.view_media)
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def _is_report(entity: Entity) -> bool:
|
|
53
|
-
"""
|
|
54
|
-
Returns True if the entity is a Report. Cubes share the same type as Reports,
|
|
55
|
-
so the subtype must be checked.
|
|
56
|
-
"""
|
|
57
|
-
is_type_report = entity.type == ObjectTypes.REPORT_DEFINITION
|
|
58
|
-
is_subtype_cube = entity.subtype == ObjectSubTypes.OLAP_CUBE.value
|
|
59
|
-
return is_type_report and not is_subtype_cube
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def _safe_get_attribute(entity: Entity, attribute: str) -> Optional[str]:
|
|
63
|
-
"""
|
|
64
|
-
Some properties may raise an error. Example: retrieving a Report's `sql` fails if the Report has not been published.
|
|
65
|
-
This safely returns the attribute value, or None if the retrieval fails.
|
|
66
|
-
"""
|
|
67
|
-
try:
|
|
68
|
-
value = getattr(entity, attribute)
|
|
69
|
-
except IServerError as e:
|
|
70
|
-
logger.error(f"Could not get {attribute} for entity {entity.id}: {e}")
|
|
71
|
-
value = None
|
|
72
|
-
return value
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
class Dependency(BaseModel):
|
|
76
|
-
id: str
|
|
77
|
-
name: str
|
|
78
|
-
subtype: int
|
|
79
|
-
type: int
|
|
80
|
-
|
|
81
|
-
model_config = ConfigDict(extra="ignore")
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
def _list_dependencies(entity: Entity) -> list[dict]:
|
|
85
|
-
"""Lists the entity's dependencies, keeping only relevant fields."""
|
|
86
|
-
dependencies: list[dict] = []
|
|
87
|
-
|
|
88
|
-
offset = 0
|
|
89
|
-
while True:
|
|
90
|
-
batch = entity.list_dependencies(offset=offset, limit=_BATCH_SIZE)
|
|
91
|
-
dependencies.extend(batch)
|
|
92
|
-
if len(batch) < _BATCH_SIZE:
|
|
93
|
-
break
|
|
94
|
-
offset += _BATCH_SIZE
|
|
95
|
-
|
|
96
|
-
return [
|
|
97
|
-
Dependency(**dependency).model_dump() for dependency in dependencies
|
|
98
|
-
]
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
def _level_1_folder_id(folders: list[dict]) -> str:
|
|
102
|
-
"""Searches for the first enclosing folder and returns its ID."""
|
|
103
|
-
for folder in folders:
|
|
104
|
-
if folder["level"] == 1:
|
|
105
|
-
return folder["id"]
|
|
106
|
-
|
|
107
|
-
raise ValueError("No level 1 folder found")
|
|
108
|
-
|
|
109
41
|
|
|
110
42
|
class StrategyClient:
|
|
111
43
|
"""Connect to Strategy through mstrio-py and fetch main assets."""
|
|
@@ -116,6 +48,7 @@ class StrategyClient:
|
|
|
116
48
|
base_url=self.base_url,
|
|
117
49
|
username=credentials.username,
|
|
118
50
|
password=credentials.password,
|
|
51
|
+
verbose=False,
|
|
119
52
|
)
|
|
120
53
|
|
|
121
54
|
self.hostname = urlparse(self.base_url).hostname
|
|
@@ -129,36 +62,6 @@ class StrategyClient:
|
|
|
129
62
|
def close(self):
|
|
130
63
|
self.connection.close()
|
|
131
64
|
|
|
132
|
-
def _url(self, entity: Entity) -> str:
|
|
133
|
-
"""
|
|
134
|
-
Formats the right URL.
|
|
135
|
-
* Dashboards : viewed in MicroStrategy
|
|
136
|
-
* Reports and Documents : viewed in MicroStrategy Web
|
|
137
|
-
* other (i.e. Cubes): the URL leads to the folder in MicroStrategy Web
|
|
138
|
-
"""
|
|
139
|
-
if _is_dashboard(entity):
|
|
140
|
-
id_ = entity.id
|
|
141
|
-
template = URLTemplates.DASHBOARD
|
|
142
|
-
|
|
143
|
-
elif entity.type == ObjectTypes.DOCUMENT_DEFINITION:
|
|
144
|
-
id_ = entity.id
|
|
145
|
-
template = URLTemplates.DOCUMENT
|
|
146
|
-
|
|
147
|
-
elif _is_report(entity):
|
|
148
|
-
id_ = entity.id
|
|
149
|
-
template = URLTemplates.REPORT
|
|
150
|
-
|
|
151
|
-
else:
|
|
152
|
-
# default to folder URL
|
|
153
|
-
id_ = _level_1_folder_id(entity.ancestors)
|
|
154
|
-
template = URLTemplates.FOLDER
|
|
155
|
-
|
|
156
|
-
return template.value.format(
|
|
157
|
-
hostname=self.hostname,
|
|
158
|
-
id_=id_,
|
|
159
|
-
project_id=entity.project_id,
|
|
160
|
-
)
|
|
161
|
-
|
|
162
65
|
def _common_entity_properties(
|
|
163
66
|
self,
|
|
164
67
|
entity: Entity,
|
|
@@ -169,7 +72,7 @@ class StrategyClient:
|
|
|
169
72
|
Returns the entity's properties, including its dependencies
|
|
170
73
|
and optional URL and/or description.
|
|
171
74
|
"""
|
|
172
|
-
dependencies =
|
|
75
|
+
dependencies = list_dependencies(entity)
|
|
173
76
|
owner_id = entity.owner.id if isinstance(entity.owner, User) else None
|
|
174
77
|
properties = {
|
|
175
78
|
"dependencies": dependencies,
|
|
@@ -182,23 +85,80 @@ class StrategyClient:
|
|
|
182
85
|
}
|
|
183
86
|
|
|
184
87
|
if with_url:
|
|
185
|
-
|
|
88
|
+
assert self.hostname
|
|
89
|
+
properties["url"] = format_url(
|
|
90
|
+
entity=entity, hostname=self.hostname
|
|
91
|
+
)
|
|
186
92
|
|
|
187
93
|
if with_description:
|
|
188
|
-
properties["description"] =
|
|
189
|
-
|
|
94
|
+
properties["description"] = safe_get_property(entity, "description")
|
|
95
|
+
|
|
96
|
+
return properties
|
|
97
|
+
|
|
98
|
+
def _attributes_properties(self, attribute: Attribute) -> dict[str, Any]:
|
|
99
|
+
"""
|
|
100
|
+
Attributes have a lookup table, which we need to compute the table lineage.
|
|
101
|
+
"""
|
|
102
|
+
return {
|
|
103
|
+
**self._common_entity_properties(attribute, with_url=False),
|
|
104
|
+
"lookup_table_id": lookup_table_id(attribute),
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
def _physical_table_properties(
|
|
108
|
+
self, table: Optional[PhysicalTable]
|
|
109
|
+
) -> Optional[dict[str, Any]]:
|
|
110
|
+
"""
|
|
111
|
+
Returns the properties of the physical table, including its columns.
|
|
112
|
+
A physical table can have 1 of these types:
|
|
113
|
+
* "normal": meaning it matches 1 warehouse table
|
|
114
|
+
* "sql": it is based on an SQL statement
|
|
115
|
+
Other types are not supported (and they technically shouldn't be possible.)
|
|
116
|
+
"""
|
|
117
|
+
if not table:
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
properties = {
|
|
121
|
+
"id": table.id,
|
|
122
|
+
"name": table.name,
|
|
123
|
+
"type": table.table_type.value,
|
|
124
|
+
"columns": column_properties(table.columns),
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if table.table_type == PhysicalTableType.SQL:
|
|
128
|
+
physical_table = PhysicalTable(
|
|
129
|
+
connection=self.connection,
|
|
130
|
+
id=table.id,
|
|
190
131
|
)
|
|
132
|
+
properties["sql_statement"] = physical_table.sql_statement
|
|
133
|
+
|
|
134
|
+
elif table.table_type == PhysicalTableType.NORMAL:
|
|
135
|
+
properties["table_prefix"] = table.table_prefix
|
|
136
|
+
properties["table_name"] = table.table_name
|
|
191
137
|
|
|
192
138
|
return properties
|
|
193
139
|
|
|
140
|
+
def _logical_table_properties(self, table: LogicalTable) -> dict[str, Any]:
|
|
141
|
+
"""
|
|
142
|
+
Returns properties for:
|
|
143
|
+
* the logical table itself
|
|
144
|
+
* its physical table (though it may not be accessible)
|
|
145
|
+
* the columns of the physical table
|
|
146
|
+
"""
|
|
147
|
+
physical_table = safe_get_property(table, "physical_table")
|
|
148
|
+
return {
|
|
149
|
+
"id": table.id,
|
|
150
|
+
"name": table.name,
|
|
151
|
+
"physical_table": self._physical_table_properties(physical_table),
|
|
152
|
+
}
|
|
153
|
+
|
|
194
154
|
def _report_properties(self, report: Report) -> dict[str, Any]:
|
|
195
155
|
"""
|
|
196
156
|
Report properties contain an optional SQL source query. Due to a typing
|
|
197
157
|
bug in the mstrio package, the typing must be ignored.
|
|
198
158
|
"""
|
|
199
159
|
properties = self._common_entity_properties(report) # type: ignore
|
|
200
|
-
properties["url"] = self.
|
|
201
|
-
properties["sql"] =
|
|
160
|
+
properties["url"] = format_url(entity=report, hostname=self.hostname) # type: ignore
|
|
161
|
+
properties["sql"] = safe_get_property(report, "sql") # type: ignore
|
|
202
162
|
return properties
|
|
203
163
|
|
|
204
164
|
@staticmethod
|
|
@@ -243,7 +203,7 @@ class StrategyClient:
|
|
|
243
203
|
def _fetch_attributes(self) -> Iterator[dict[str, Any]]:
|
|
244
204
|
return self._fetch_entities(
|
|
245
205
|
list_attributes,
|
|
246
|
-
|
|
206
|
+
custom_property_extractor=self._attributes_properties,
|
|
247
207
|
)
|
|
248
208
|
|
|
249
209
|
def _fetch_cubes(self) -> Iterator[dict[str, Any]]:
|
|
@@ -263,6 +223,15 @@ class StrategyClient:
|
|
|
263
223
|
with_description=False,
|
|
264
224
|
)
|
|
265
225
|
|
|
226
|
+
def _fetch_logical_tables(self) -> Iterator[dict[str, Any]]:
|
|
227
|
+
"""
|
|
228
|
+
Yields all logical tables, including their physical tables and their columns.
|
|
229
|
+
"""
|
|
230
|
+
return self._fetch_entities(
|
|
231
|
+
list_logical_tables,
|
|
232
|
+
custom_property_extractor=self._logical_table_properties,
|
|
233
|
+
)
|
|
234
|
+
|
|
266
235
|
def _fetch_metrics(self) -> Iterator[dict[str, Any]]:
|
|
267
236
|
return self._fetch_entities(
|
|
268
237
|
list_metrics,
|
|
@@ -298,6 +267,9 @@ class StrategyClient:
|
|
|
298
267
|
elif asset == StrategyAsset.FACT:
|
|
299
268
|
yield from self._fetch_facts()
|
|
300
269
|
|
|
270
|
+
elif asset == StrategyAsset.LOGICAL_TABLE:
|
|
271
|
+
yield from self._fetch_logical_tables()
|
|
272
|
+
|
|
301
273
|
elif asset == StrategyAsset.METRIC:
|
|
302
274
|
yield from self._fetch_metrics()
|
|
303
275
|
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from typing import Any, Optional
|
|
4
|
+
|
|
5
|
+
from mstrio.helpers import IServerError # type: ignore
|
|
6
|
+
from mstrio.modeling import ( # type: ignore
|
|
7
|
+
Attribute,
|
|
8
|
+
TableColumn,
|
|
9
|
+
)
|
|
10
|
+
from mstrio.types import ObjectSubTypes, ObjectTypes # type: ignore
|
|
11
|
+
from mstrio.utils.entity import Entity # type: ignore
|
|
12
|
+
from mstrio.utils.helper import is_dashboard # type: ignore
|
|
13
|
+
from pydantic import BaseModel, ConfigDict
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
_BATCH_SIZE: int = 100
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class URLTemplates(Enum):
|
|
21
|
+
DASHBOARD = "https://{hostname}/MicroStrategyLibrary/app/{project_id}/{id_}"
|
|
22
|
+
DOCUMENT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?documentID={id_}&projectID={project_id}"
|
|
23
|
+
REPORT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?reportID={id_}&projectID={project_id}"
|
|
24
|
+
FOLDER = "https://{hostname}/MicroStrategy/servlet/mstrWeb?folderID={id_}&projectID={project_id}"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Dependency(BaseModel):
|
|
28
|
+
id: str
|
|
29
|
+
name: str
|
|
30
|
+
subtype: int
|
|
31
|
+
type: int
|
|
32
|
+
|
|
33
|
+
model_config = ConfigDict(extra="ignore")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def list_dependencies(entity: Entity) -> list[dict]:
|
|
37
|
+
"""Lists the entity's dependencies, keeping only relevant fields."""
|
|
38
|
+
dependencies: list[dict] = []
|
|
39
|
+
|
|
40
|
+
offset = 0
|
|
41
|
+
while True:
|
|
42
|
+
batch = entity.list_dependencies(offset=offset, limit=_BATCH_SIZE)
|
|
43
|
+
dependencies.extend(batch)
|
|
44
|
+
if len(batch) < _BATCH_SIZE:
|
|
45
|
+
break
|
|
46
|
+
offset += _BATCH_SIZE
|
|
47
|
+
|
|
48
|
+
return [
|
|
49
|
+
Dependency(**dependency).model_dump() for dependency in dependencies
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _is_dashboard(entity: Entity) -> bool:
|
|
54
|
+
"""
|
|
55
|
+
Returns True if the entity is a Dashboard. They can only be distinguished
|
|
56
|
+
from Documents by checking the `view_media` property.
|
|
57
|
+
"""
|
|
58
|
+
is_type_document = entity.type == ObjectTypes.DOCUMENT_DEFINITION
|
|
59
|
+
return is_type_document and is_dashboard(entity.view_media)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _is_report(entity: Entity) -> bool:
|
|
63
|
+
"""
|
|
64
|
+
Returns True if the entity is a Report. Cubes share the same type as Reports,
|
|
65
|
+
so the subtype must be checked.
|
|
66
|
+
"""
|
|
67
|
+
is_type_report = entity.type == ObjectTypes.REPORT_DEFINITION
|
|
68
|
+
is_subtype_cube = entity.subtype == ObjectSubTypes.OLAP_CUBE.value
|
|
69
|
+
return is_type_report and not is_subtype_cube
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def format_url(entity: Entity, hostname: str) -> str:
|
|
73
|
+
"""
|
|
74
|
+
Formats the right URL.
|
|
75
|
+
* Dashboards : viewed in MicroStrategy
|
|
76
|
+
* Reports and Documents : viewed in MicroStrategy Web
|
|
77
|
+
* other (i.e. Cubes): the URL leads to the folder in MicroStrategy Web
|
|
78
|
+
"""
|
|
79
|
+
if _is_dashboard(entity):
|
|
80
|
+
id_ = entity.id
|
|
81
|
+
template = URLTemplates.DASHBOARD
|
|
82
|
+
|
|
83
|
+
elif entity.type == ObjectTypes.DOCUMENT_DEFINITION:
|
|
84
|
+
id_ = entity.id
|
|
85
|
+
template = URLTemplates.DOCUMENT
|
|
86
|
+
|
|
87
|
+
elif _is_report(entity):
|
|
88
|
+
id_ = entity.id
|
|
89
|
+
template = URLTemplates.REPORT
|
|
90
|
+
|
|
91
|
+
else:
|
|
92
|
+
# default to folder URL
|
|
93
|
+
id_ = level_1_folder_id(entity.ancestors)
|
|
94
|
+
template = URLTemplates.FOLDER
|
|
95
|
+
|
|
96
|
+
return template.value.format(
|
|
97
|
+
hostname=hostname,
|
|
98
|
+
id_=id_,
|
|
99
|
+
project_id=entity.project_id,
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def safe_get_property(entity: Entity, attribute: str) -> Optional[str]:
|
|
104
|
+
"""
|
|
105
|
+
Some properties may raise an error. Example: retrieving a Report's `sql` fails if the Report has not been published.
|
|
106
|
+
This safely returns the attribute value, or None if the retrieval fails.
|
|
107
|
+
"""
|
|
108
|
+
try:
|
|
109
|
+
value = getattr(entity, attribute)
|
|
110
|
+
except (IServerError, ValueError) as e:
|
|
111
|
+
logger.error(f"Could not get {attribute} for entity {entity.id}: {e}")
|
|
112
|
+
value = None
|
|
113
|
+
return value
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def column_properties(columns: list[TableColumn]) -> list[dict[str, Any]]:
|
|
117
|
+
"""Returns the properties of a physical table's columns."""
|
|
118
|
+
properties: list[dict[str, Any]] = []
|
|
119
|
+
|
|
120
|
+
for column in columns:
|
|
121
|
+
column_properties = {
|
|
122
|
+
"id": column.id,
|
|
123
|
+
"name": column.name,
|
|
124
|
+
"column_name": column.column_name,
|
|
125
|
+
}
|
|
126
|
+
properties.append(column_properties)
|
|
127
|
+
|
|
128
|
+
return properties
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def level_1_folder_id(folders: list[dict]) -> str:
|
|
132
|
+
"""Searches for the first enclosing folder and returns its ID."""
|
|
133
|
+
for folder in folders:
|
|
134
|
+
if folder["level"] == 1:
|
|
135
|
+
return folder["id"]
|
|
136
|
+
|
|
137
|
+
raise ValueError("No level 1 folder found")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def lookup_table_id(attribute: Attribute):
|
|
141
|
+
"""Returns the lookup table's ID, if there is one."""
|
|
142
|
+
lookup_table = attribute.attribute_lookup_table
|
|
143
|
+
if not lookup_table:
|
|
144
|
+
return None
|
|
145
|
+
return lookup_table.object_id
|
|
@@ -10,6 +10,7 @@ SELECT
|
|
|
10
10
|
FROM snowflake.account_usage.databases
|
|
11
11
|
WHERE TRUE
|
|
12
12
|
AND UPPER(database_name) NOT IN ('SNOWFLAKE', 'UTIL_DB')
|
|
13
|
+
AND UPPER(database_name) NOT LIKE 'USER$%' -- ignore private notebooks
|
|
13
14
|
AND (
|
|
14
15
|
deleted IS NULL
|
|
15
16
|
OR deleted > CURRENT_TIMESTAMP - INTERVAL '1 day'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: castor-extractor
|
|
3
|
-
Version: 0.24.
|
|
3
|
+
Version: 0.24.29
|
|
4
4
|
Summary: Extract your metadata assets.
|
|
5
5
|
Home-page: https://www.castordoc.com/
|
|
6
6
|
License: EULA
|
|
@@ -215,6 +215,22 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
|
|
|
215
215
|
|
|
216
216
|
# Changelog
|
|
217
217
|
|
|
218
|
+
## 0.24.29 - 2025-06-24
|
|
219
|
+
|
|
220
|
+
* Strategy: skip descriptions on ValueErrors
|
|
221
|
+
|
|
222
|
+
## 0.24.28 - 2025-06-20
|
|
223
|
+
|
|
224
|
+
* Snowflake: ignore private notebooks
|
|
225
|
+
|
|
226
|
+
## 0.24.27 - 2025-06-20
|
|
227
|
+
|
|
228
|
+
* Strategy: extract logical tables
|
|
229
|
+
|
|
230
|
+
## 0.24.26 - 2025-06-16
|
|
231
|
+
|
|
232
|
+
* Coalesce: increase _MAX_ERRORS client parameter
|
|
233
|
+
|
|
218
234
|
## 0.24.25 - 2025-06-12
|
|
219
235
|
|
|
220
236
|
* DBT: Fix API base url
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
CHANGELOG.md,sha256=
|
|
1
|
+
CHANGELOG.md,sha256=GanHBv0C_fiKSfPSLzumCuBYUIQy4ZRLbA9BAXZYFAg,18440
|
|
2
2
|
Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
|
|
3
3
|
DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
|
|
4
4
|
LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
@@ -76,7 +76,7 @@ castor_extractor/transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
|
|
|
76
76
|
castor_extractor/transformation/coalesce/__init__.py,sha256=CW_qdtEfwgJRsCyBlk5hNlxwEO-VV6mBXZvkRbND_J8,112
|
|
77
77
|
castor_extractor/transformation/coalesce/assets.py,sha256=pzccYPP66c9PAnVroemx7-6MeRHw7Ft1OlTC6jIamAA,363
|
|
78
78
|
castor_extractor/transformation/coalesce/client/__init__.py,sha256=VRmVpH29rOghtDQnCN7dAdA0dI0Lxseu4BC8rnwM9dU,80
|
|
79
|
-
castor_extractor/transformation/coalesce/client/client.py,sha256=
|
|
79
|
+
castor_extractor/transformation/coalesce/client/client.py,sha256=7EVJDDxnIm5_uMHLFZ2PD6JzfebVglKST9IiURwn4vs,6524
|
|
80
80
|
castor_extractor/transformation/coalesce/client/credentials.py,sha256=jbJxjbdPspf-dzYKfeb7oqL_8TXd1nvkJrjAcdAnLPc,548
|
|
81
81
|
castor_extractor/transformation/coalesce/client/endpoint.py,sha256=0uLh7dpA1vsR9qr_50SEYV_-heQE4BwED9oNMgYsL-w,1272
|
|
82
82
|
castor_extractor/transformation/coalesce/client/type.py,sha256=oiiVP9NL0ijTXyQmaB8aJVYckc7m-m8ZgMyNIAduUKE,43
|
|
@@ -278,10 +278,11 @@ castor_extractor/visualization/sigma/client/endpoints.py,sha256=DBFphbgoH78_MZUG
|
|
|
278
278
|
castor_extractor/visualization/sigma/client/pagination.py,sha256=kNEhNq08tTGbypyMjxs0w4uvDtQc_iaWpOZweaa_FsU,690
|
|
279
279
|
castor_extractor/visualization/sigma/extract.py,sha256=XIT1qsj6g6dgBWP8HPfj_medZexu48EaY9tUwi14gzM,2298
|
|
280
280
|
castor_extractor/visualization/strategy/__init__.py,sha256=HOMv4JxqF5ZmViWi-pDE-PSXJRLTdXal_jtpHG_rlR8,123
|
|
281
|
-
castor_extractor/visualization/strategy/assets.py,sha256=
|
|
281
|
+
castor_extractor/visualization/strategy/assets.py,sha256=yFXF_dX01patC0HQ1eU7Jo_4DZ4m6IJEg0uCB71tMoI,480
|
|
282
282
|
castor_extractor/visualization/strategy/client/__init__.py,sha256=XWP0yF5j6JefDJkDfX-RSJn3HF2ceQ0Yx1PLCfB3BBo,80
|
|
283
|
-
castor_extractor/visualization/strategy/client/client.py,sha256=
|
|
283
|
+
castor_extractor/visualization/strategy/client/client.py,sha256=6DJO0Fh67FXxmwY5h_X9cu5sEq3GhM19b9hwn_fvhSE,9460
|
|
284
284
|
castor_extractor/visualization/strategy/client/credentials.py,sha256=urFfNxWX1JG6wwFMYImufQzHa5g-sgjdlVGzi63owwg,1113
|
|
285
|
+
castor_extractor/visualization/strategy/client/properties.py,sha256=66oBm8Kz6HEQW_jNR5_fAI_O921R2F5yH2Ff3zjtJOk,4500
|
|
285
286
|
castor_extractor/visualization/strategy/extract.py,sha256=2fBuvS2xiOGXRpxXnZsE_C3en6t1-BlM5TbusjHyEkg,1166
|
|
286
287
|
castor_extractor/visualization/tableau/__init__.py,sha256=eFI_1hjdkxyUiAYiy3szwyuwn3yJ5C_KbpBU0ySJDcQ,138
|
|
287
288
|
castor_extractor/visualization/tableau/assets.py,sha256=HbCRd8VCj1WBEeqg9jwnygnT7xOFJ6PQD7Lq7sV-XR0,635
|
|
@@ -405,7 +406,7 @@ castor_extractor/warehouse/snowflake/extract.py,sha256=3yc9kcVtt2c1uWJOJJgeZchV4
|
|
|
405
406
|
castor_extractor/warehouse/snowflake/queries/.sqlfluff,sha256=vttrwcr64JVIuvc7WIg9C54cbOkjg_VjXNR7YnTGOPE,31
|
|
406
407
|
castor_extractor/warehouse/snowflake/queries/column.sql,sha256=Ru-yC0s76I9LehOA4aCZ--xz6D9H1Hyr3OZdILOBHAw,1882
|
|
407
408
|
castor_extractor/warehouse/snowflake/queries/column_lineage.sql,sha256=YKBiZ6zySSNcXLDXwm31EjGIIkkkZc0-S6hI1SRM80o,1179
|
|
408
|
-
castor_extractor/warehouse/snowflake/queries/database.sql,sha256=
|
|
409
|
+
castor_extractor/warehouse/snowflake/queries/database.sql,sha256=yxHzV2oQfW0bo2dJVqJkyZy-GruudTi6ObmLaJ505hE,559
|
|
409
410
|
castor_extractor/warehouse/snowflake/queries/function.sql,sha256=8LRh0ybhd-RldJ8UZspWUm3yv52evq11O2uqIO4KqeQ,372
|
|
410
411
|
castor_extractor/warehouse/snowflake/queries/grant_to_role.sql,sha256=O7AJ1LzoXGDFmiVvQ8EMJ5x8FSAnaxRPdmRyAlEmkUM,272
|
|
411
412
|
castor_extractor/warehouse/snowflake/queries/grant_to_user.sql,sha256=7AalVajU5vRRpIiys1igSwmDXirbwpMTvJr2ihSz2NE,143
|
|
@@ -427,8 +428,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
|
|
|
427
428
|
castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
|
|
428
429
|
castor_extractor/warehouse/sqlserver/query.py,sha256=g0hPT-RmeGi2DyenAi3o72cTlQsLToXIFYojqc8E5fQ,533
|
|
429
430
|
castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
|
|
430
|
-
castor_extractor-0.24.
|
|
431
|
-
castor_extractor-0.24.
|
|
432
|
-
castor_extractor-0.24.
|
|
433
|
-
castor_extractor-0.24.
|
|
434
|
-
castor_extractor-0.24.
|
|
431
|
+
castor_extractor-0.24.29.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
432
|
+
castor_extractor-0.24.29.dist-info/METADATA,sha256=0aRiXGPqFGxNnPoSmsuE1BkgRCalVbEotI17_Vv2VDo,25893
|
|
433
|
+
castor_extractor-0.24.29.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
434
|
+
castor_extractor-0.24.29.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
|
|
435
|
+
castor_extractor-0.24.29.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|