castor-extractor 0.24.42__py3-none-any.whl → 0.24.48__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of castor-extractor might be problematic. Click here for more details.
- CHANGELOG.md +26 -0
- castor_extractor/commands/extract_sqlserver.py +8 -0
- castor_extractor/transformation/coalesce/client/client.py +1 -9
- castor_extractor/visualization/sigma/client/client.py +1 -0
- castor_extractor/visualization/sigma/client/pagination.py +1 -1
- castor_extractor/warehouse/abstract/asset.py +5 -0
- castor_extractor/warehouse/databricks/api_client.py +7 -1
- castor_extractor/warehouse/sqlserver/client.py +91 -0
- castor_extractor/warehouse/sqlserver/extract.py +10 -1
- castor_extractor/warehouse/sqlserver/queries/column.sql +8 -8
- castor_extractor/warehouse/sqlserver/queries/query.sql +25 -0
- castor_extractor/warehouse/sqlserver/queries/schema.sql +4 -4
- castor_extractor/warehouse/sqlserver/queries/table.sql +9 -9
- castor_extractor/warehouse/sqlserver/queries/view_ddl.sql +13 -0
- castor_extractor/warehouse/sqlserver/query.py +7 -0
- {castor_extractor-0.24.42.dist-info → castor_extractor-0.24.48.dist-info}/METADATA +27 -1
- {castor_extractor-0.24.42.dist-info → castor_extractor-0.24.48.dist-info}/RECORD +20 -18
- {castor_extractor-0.24.42.dist-info → castor_extractor-0.24.48.dist-info}/LICENCE +0 -0
- {castor_extractor-0.24.42.dist-info → castor_extractor-0.24.48.dist-info}/WHEEL +0 -0
- {castor_extractor-0.24.42.dist-info → castor_extractor-0.24.48.dist-info}/entry_points.txt +0 -0
CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.24.48 - 2025-09-09
|
|
4
|
+
|
|
5
|
+
* SqlServer: handle hyphens in database name
|
|
6
|
+
|
|
7
|
+
## 0.24.47 - 2025-09-08
|
|
8
|
+
|
|
9
|
+
* SqlServer: extract SQL queries and lineage
|
|
10
|
+
|
|
11
|
+
## 0.24.46 - 2025-09-03
|
|
12
|
+
|
|
13
|
+
* Sigma: Added `HTTPStatus.FORBIDDEN` to the list of ignored errors
|
|
14
|
+
|
|
15
|
+
## 0.24.45 - 2025-08-27
|
|
16
|
+
|
|
17
|
+
* Sigma: Increasing pagination limit for Sigma extraction
|
|
18
|
+
|
|
19
|
+
## 0.24.44 - 2025-08-22
|
|
20
|
+
|
|
21
|
+
* Coalesce: do not skip nodes raising a 500 Server Error
|
|
22
|
+
|
|
23
|
+
## 0.24.43 - 2025-08-20
|
|
24
|
+
|
|
25
|
+
* SQLServer:
|
|
26
|
+
* raise error when no database is left after filtering
|
|
27
|
+
* use uppercase INFORMATION_SCHEMA for case-sensitive database compatibility
|
|
28
|
+
|
|
3
29
|
## 0.24.42 - 2025-08-19
|
|
4
30
|
|
|
5
31
|
* Strategy: exclude COLUMNS from extraction
|
|
@@ -15,6 +15,14 @@ def main():
|
|
|
15
15
|
parser.add_argument("-u", "--user", help="MSSQL User")
|
|
16
16
|
parser.add_argument("-p", "--password", help="MSSQL Password")
|
|
17
17
|
|
|
18
|
+
parser.add_argument(
|
|
19
|
+
"-s",
|
|
20
|
+
"--skip-queries",
|
|
21
|
+
dest="skip_queries",
|
|
22
|
+
action="store_true",
|
|
23
|
+
help="Skip the extraction of SQL queries",
|
|
24
|
+
)
|
|
25
|
+
|
|
18
26
|
parser.add_argument("-o", "--output", help="Directory to write to")
|
|
19
27
|
|
|
20
28
|
parser.add_argument(
|
|
@@ -3,8 +3,6 @@ from functools import partial
|
|
|
3
3
|
from http import HTTPStatus
|
|
4
4
|
from typing import Callable, Optional
|
|
5
5
|
|
|
6
|
-
from pydantic import ValidationError
|
|
7
|
-
|
|
8
6
|
from ....utils import (
|
|
9
7
|
APIClient,
|
|
10
8
|
BearerAuth,
|
|
@@ -123,13 +121,7 @@ class CoalesceClient(APIClient):
|
|
|
123
121
|
for index, env in enumerate(environments):
|
|
124
122
|
env_id = env["id"]
|
|
125
123
|
logger.info(f"Fetching nodes for env #{env_id} - {index}/{total}")
|
|
126
|
-
|
|
127
|
-
nodes.extend(self._fetch_env_nodes(env_id))
|
|
128
|
-
except ValidationError as e:
|
|
129
|
-
# 500 Server Error: Internal Server Error on Coalesce API
|
|
130
|
-
logger.warning(
|
|
131
|
-
f"Skipping nodes for {env_id} due to the following Error: {e}"
|
|
132
|
-
)
|
|
124
|
+
nodes.extend(self._fetch_env_nodes(env_id))
|
|
133
125
|
logger.info(f"{len(nodes)} nodes extracted so far")
|
|
134
126
|
return nodes
|
|
135
127
|
|
|
@@ -64,6 +64,11 @@ FUNCTIONS_ASSETS = (WarehouseAsset.FUNCTION,)
|
|
|
64
64
|
QUERIES_ASSETS = (WarehouseAsset.QUERY,)
|
|
65
65
|
VIEWS_ASSETS = (WarehouseAsset.VIEW_DDL,)
|
|
66
66
|
|
|
67
|
+
QUERIES_ASSET_GROUPS = (
|
|
68
|
+
WarehouseAssetGroup.QUERY,
|
|
69
|
+
WarehouseAssetGroup.VIEW_DDL,
|
|
70
|
+
)
|
|
71
|
+
|
|
67
72
|
EXTERNAL_LINEAGE_ASSETS = (
|
|
68
73
|
WarehouseAsset.EXTERNAL_COLUMN_LINEAGE,
|
|
69
74
|
WarehouseAsset.EXTERNAL_TABLE_LINEAGE,
|
|
@@ -81,7 +81,13 @@ class DatabricksAPIClient(APIClient):
|
|
|
81
81
|
def databases(self) -> list[dict]:
|
|
82
82
|
content = self._get(DatabricksEndpointFactory.databases())
|
|
83
83
|
_databases = self.formatter.format_database(content.get("catalogs", []))
|
|
84
|
-
|
|
84
|
+
filtered_databases = [
|
|
85
|
+
d for d in _databases if self._keep_catalog(d["database_name"])
|
|
86
|
+
]
|
|
87
|
+
logger.info(
|
|
88
|
+
f"Available databases: {[d['database_name'] for d in filtered_databases]}"
|
|
89
|
+
)
|
|
90
|
+
return filtered_databases
|
|
85
91
|
|
|
86
92
|
def _schemas_of_database(self, database: dict) -> list[dict]:
|
|
87
93
|
payload = {"catalog_name": database["database_name"]}
|
|
@@ -66,3 +66,94 @@ class MSSQLClient(SqlalchemyClient):
|
|
|
66
66
|
for row in result
|
|
67
67
|
if row["name"] not in _SYSTEM_DATABASES
|
|
68
68
|
]
|
|
69
|
+
|
|
70
|
+
def _current_database(self) -> str:
|
|
71
|
+
result = self.execute(
|
|
72
|
+
ExtractionQuery("SELECT DB_NAME() AS database_name", {})
|
|
73
|
+
)
|
|
74
|
+
return next(result)["database_name"]
|
|
75
|
+
|
|
76
|
+
def _has_access(self, name: str, object_type: str, permission: str) -> bool:
|
|
77
|
+
query_text = f"""
|
|
78
|
+
SELECT
|
|
79
|
+
HAS_PERMS_BY_NAME('{name}', '{object_type}', '{permission}')
|
|
80
|
+
AS has_permission
|
|
81
|
+
"""
|
|
82
|
+
query = ExtractionQuery(query_text, dict())
|
|
83
|
+
result = next(self.execute(query))
|
|
84
|
+
return result["has_permission"] == 1
|
|
85
|
+
|
|
86
|
+
def _has_table_read_access(self, table_name: str) -> bool:
|
|
87
|
+
"""
|
|
88
|
+
Check whether we have READ access to the given table
|
|
89
|
+
"""
|
|
90
|
+
return self._has_access(
|
|
91
|
+
name=table_name,
|
|
92
|
+
object_type="OBJECT",
|
|
93
|
+
permission="SELECT",
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def _has_view_database_state(self) -> bool:
|
|
97
|
+
"""
|
|
98
|
+
Check whether we have VIEW DATABASE STATE permissions, which
|
|
99
|
+
is necessary to fetch data from the Query Store
|
|
100
|
+
"""
|
|
101
|
+
return self._has_access(
|
|
102
|
+
name=self._current_database(),
|
|
103
|
+
object_type="DATABASE",
|
|
104
|
+
permission="VIEW DATABASE STATE",
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def _has_query_store(self) -> bool:
|
|
108
|
+
"""
|
|
109
|
+
Checks whether the Query Store is activated on this instance.
|
|
110
|
+
This is required to extract the SQL queries history.
|
|
111
|
+
https://learn.microsoft.com/en-us/sql/relational-databases/performance/monitoring-performance-by-using-the-query-store?view=sql-server-ver17"""
|
|
112
|
+
sql = """
|
|
113
|
+
SELECT
|
|
114
|
+
desired_state
|
|
115
|
+
FROM
|
|
116
|
+
sys.database_query_store_options
|
|
117
|
+
"""
|
|
118
|
+
query = ExtractionQuery(sql, {})
|
|
119
|
+
# 2 = READ_WRITE, which means the Query Store is activated
|
|
120
|
+
return next(self.execute(query))["desired_state"] == 2
|
|
121
|
+
|
|
122
|
+
def has_queries_permissions(self) -> bool:
|
|
123
|
+
"""
|
|
124
|
+
Verify that we habe the required permissions to extract
|
|
125
|
+
query history and view object definitions (DDL).
|
|
126
|
+
|
|
127
|
+
This check ensures:
|
|
128
|
+
- Query Store is enabled on the database.
|
|
129
|
+
- We have the VIEW DATABASE STATE permissions
|
|
130
|
+
- We have read access to the relevant system tables.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
tables = (
|
|
134
|
+
# SQL queries
|
|
135
|
+
"sys.query_store_plan",
|
|
136
|
+
"sys.query_store_query",
|
|
137
|
+
"sys.query_store_query_text",
|
|
138
|
+
"sys.query_store_runtime_stats",
|
|
139
|
+
# views DDL
|
|
140
|
+
"sys.schemas",
|
|
141
|
+
"sys.sql_modules",
|
|
142
|
+
"sys.views",
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
has_permissions = True
|
|
146
|
+
for table in tables:
|
|
147
|
+
if not self._has_table_read_access(table):
|
|
148
|
+
logger.info(f"Missing READ permissions for table {table}")
|
|
149
|
+
has_permissions = False
|
|
150
|
+
|
|
151
|
+
if not self._has_view_database_state():
|
|
152
|
+
logger.info("Missing permissions: VIEW DATABASE STATE")
|
|
153
|
+
has_permissions = False
|
|
154
|
+
|
|
155
|
+
if not self._has_query_store():
|
|
156
|
+
logger.info("Missing permissions: Query Store is not activated")
|
|
157
|
+
has_permissions = False
|
|
158
|
+
|
|
159
|
+
return has_permissions
|
|
@@ -4,6 +4,8 @@ from ...utils import LocalStorage, filter_items, from_env, write_summary
|
|
|
4
4
|
from ..abstract import (
|
|
5
5
|
CATALOG_ASSETS,
|
|
6
6
|
EXTERNAL_LINEAGE_ASSETS,
|
|
7
|
+
QUERIES_ASSETS,
|
|
8
|
+
VIEWS_ASSETS,
|
|
7
9
|
SQLExtractionProcessor,
|
|
8
10
|
SupportedAssets,
|
|
9
11
|
WarehouseAsset,
|
|
@@ -19,8 +21,10 @@ logger = logging.getLogger(__name__)
|
|
|
19
21
|
|
|
20
22
|
MSSQL_ASSETS: SupportedAssets = {
|
|
21
23
|
WarehouseAssetGroup.CATALOG: CATALOG_ASSETS,
|
|
22
|
-
WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
|
|
23
24
|
WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
|
|
25
|
+
WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
|
|
26
|
+
WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
|
|
27
|
+
WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
|
|
@@ -57,6 +61,8 @@ def extract_all(**kwargs) -> None:
|
|
|
57
61
|
blocked=kwargs.get("db_blocked"),
|
|
58
62
|
)
|
|
59
63
|
|
|
64
|
+
logger.info(f"Available databases: {databases}\n")
|
|
65
|
+
|
|
60
66
|
query_builder = MSSQLQueryBuilder(
|
|
61
67
|
databases=databases,
|
|
62
68
|
)
|
|
@@ -69,7 +75,10 @@ def extract_all(**kwargs) -> None:
|
|
|
69
75
|
storage=storage,
|
|
70
76
|
)
|
|
71
77
|
|
|
78
|
+
skip_queries = kwargs.get("skip_queries") or False
|
|
72
79
|
for group in extractable_asset_groups(MSSQL_ASSETS):
|
|
80
|
+
if group == WarehouseAssetGroup.QUERY and skip_queries:
|
|
81
|
+
continue
|
|
73
82
|
for asset in group:
|
|
74
83
|
logger.info(f"Extracting `{asset.value.upper()}` ...")
|
|
75
84
|
location = extractor.extract(asset, skip_existing)
|
|
@@ -11,7 +11,7 @@ WITH extended_tables AS (
|
|
|
11
11
|
table_owner_id = principal_id,
|
|
12
12
|
schema_id
|
|
13
13
|
FROM
|
|
14
|
-
{database}.sys.tables
|
|
14
|
+
[{database}].sys.tables
|
|
15
15
|
|
|
16
16
|
UNION
|
|
17
17
|
|
|
@@ -21,7 +21,7 @@ WITH extended_tables AS (
|
|
|
21
21
|
table_owner_id = principal_id,
|
|
22
22
|
schema_id
|
|
23
23
|
FROM
|
|
24
|
-
{database}.sys.views
|
|
24
|
+
[{database}].sys.views
|
|
25
25
|
|
|
26
26
|
UNION
|
|
27
27
|
|
|
@@ -31,7 +31,7 @@ WITH extended_tables AS (
|
|
|
31
31
|
table_owner_id = principal_id,
|
|
32
32
|
schema_id
|
|
33
33
|
FROM
|
|
34
|
-
{database}.sys.external_tables
|
|
34
|
+
[{database}].sys.external_tables
|
|
35
35
|
),
|
|
36
36
|
/*
|
|
37
37
|
`sys.columns` contains, among others:
|
|
@@ -54,11 +54,11 @@ column_ids AS (
|
|
|
54
54
|
schema_name = ss.name,
|
|
55
55
|
schema_id = ss.schema_id,
|
|
56
56
|
comment = CONVERT(varchar(1024), ep.value)
|
|
57
|
-
FROM {database}.sys.columns AS sc
|
|
57
|
+
FROM [{database}].sys.columns AS sc
|
|
58
58
|
LEFT JOIN extended_tables AS et ON sc.object_id = et.table_id
|
|
59
|
-
LEFT JOIN {database}.sys.schemas AS ss ON et.schema_id = ss.schema_id
|
|
60
|
-
LEFT JOIN {database}.sys.databases AS sd ON sd.name = '{database}'
|
|
61
|
-
LEFT JOIN {database}.sys.extended_properties AS ep
|
|
59
|
+
LEFT JOIN [{database}].sys.schemas AS ss ON et.schema_id = ss.schema_id
|
|
60
|
+
LEFT JOIN [{database}].sys.databases AS sd ON sd.name = '{database}'
|
|
61
|
+
LEFT JOIN [{database}].sys.extended_properties AS ep
|
|
62
62
|
ON
|
|
63
63
|
sc.object_id = ep.major_id
|
|
64
64
|
AND sc.column_id = ep.minor_id
|
|
@@ -87,7 +87,7 @@ columns AS (
|
|
|
87
87
|
i.comment,
|
|
88
88
|
column_id = CONCAT(i.table_id, '.', c.column_name)
|
|
89
89
|
FROM
|
|
90
|
-
{database}.
|
|
90
|
+
[{database}].INFORMATION_SCHEMA.COLUMNS AS c
|
|
91
91
|
LEFT JOIN column_ids AS i
|
|
92
92
|
ON
|
|
93
93
|
(
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
SELECT
|
|
2
|
+
q.query_id,
|
|
3
|
+
qt.query_sql_text as query_text,
|
|
4
|
+
rs.count_executions,
|
|
5
|
+
rs.last_duration as duration,
|
|
6
|
+
rs.last_execution_time as start_time,
|
|
7
|
+
'unknown-user' as user_name,
|
|
8
|
+
'unknown-user' as user_id,
|
|
9
|
+
DATEADD(SECOND, last_duration / 1000000,
|
|
10
|
+
DATEADD(MICROSECOND, last_duration % 1000000, rs.last_execution_time)
|
|
11
|
+
) AS end_time
|
|
12
|
+
FROM
|
|
13
|
+
sys.query_store_runtime_stats AS rs
|
|
14
|
+
INNER JOIN
|
|
15
|
+
sys.query_store_plan p
|
|
16
|
+
ON rs.plan_id = p.plan_id
|
|
17
|
+
INNER JOIN
|
|
18
|
+
sys.query_store_query q
|
|
19
|
+
ON p.query_id = q.query_id
|
|
20
|
+
INNER JOIN
|
|
21
|
+
sys.query_store_query_text qt
|
|
22
|
+
ON q.query_text_id = qt.query_text_id
|
|
23
|
+
WHERE
|
|
24
|
+
CAST(rs.last_execution_time AS DATE) = :day
|
|
25
|
+
AND DATEPART(HOUR, rs.last_execution_time) BETWEEN :hour_min AND :hour_max
|
|
@@ -8,7 +8,7 @@ WITH ids AS (
|
|
|
8
8
|
SELECT DISTINCT
|
|
9
9
|
table_catalog,
|
|
10
10
|
table_schema
|
|
11
|
-
FROM {database}.
|
|
11
|
+
FROM [{database}].INFORMATION_SCHEMA.TABLES
|
|
12
12
|
)
|
|
13
13
|
|
|
14
14
|
SELECT
|
|
@@ -18,10 +18,10 @@ SELECT
|
|
|
18
18
|
schema_id = CAST(d.database_id AS VARCHAR(10)) + '_' + CAST(s.schema_id AS VARCHAR(10)),
|
|
19
19
|
schema_owner = u.name,
|
|
20
20
|
schema_owner_id = u.uid
|
|
21
|
-
FROM {database}.sys.schemas AS s
|
|
21
|
+
FROM [{database}].sys.schemas AS s
|
|
22
22
|
INNER JOIN ids AS i
|
|
23
23
|
ON s.name = i.table_schema
|
|
24
|
-
LEFT JOIN {database}.sys.sysusers AS u
|
|
24
|
+
LEFT JOIN [{database}].sys.sysusers AS u
|
|
25
25
|
ON s.principal_id = u.uid
|
|
26
|
-
LEFT JOIN {database}.sys.databases AS d
|
|
26
|
+
LEFT JOIN [{database}].sys.databases AS d
|
|
27
27
|
ON i.table_catalog COLLATE DATABASE_DEFAULT = d.name COLLATE DATABASE_DEFAULT
|
|
@@ -11,7 +11,7 @@ WITH extended_tables AS (
|
|
|
11
11
|
table_owner_id = principal_id,
|
|
12
12
|
schema_id
|
|
13
13
|
FROM
|
|
14
|
-
{database}.sys.tables
|
|
14
|
+
[{database}].sys.tables
|
|
15
15
|
|
|
16
16
|
UNION
|
|
17
17
|
|
|
@@ -21,7 +21,7 @@ WITH extended_tables AS (
|
|
|
21
21
|
table_owner_id = principal_id,
|
|
22
22
|
schema_id
|
|
23
23
|
FROM
|
|
24
|
-
{database}.sys.views
|
|
24
|
+
[{database}].sys.views
|
|
25
25
|
|
|
26
26
|
UNION
|
|
27
27
|
|
|
@@ -31,14 +31,14 @@ WITH extended_tables AS (
|
|
|
31
31
|
table_owner_id = principal_id,
|
|
32
32
|
schema_id
|
|
33
33
|
FROM
|
|
34
|
-
{database}.sys.external_tables
|
|
34
|
+
[{database}].sys.external_tables
|
|
35
35
|
),
|
|
36
36
|
-- Get the row count per table
|
|
37
37
|
partitions AS (
|
|
38
38
|
SELECT
|
|
39
39
|
object_id,
|
|
40
40
|
row_count = SUM(rows)
|
|
41
|
-
FROM {database}.sys.partitions
|
|
41
|
+
FROM [{database}].sys.partitions
|
|
42
42
|
GROUP BY object_id
|
|
43
43
|
),
|
|
44
44
|
-- Append row count to table properties
|
|
@@ -70,11 +70,11 @@ table_ids AS (
|
|
|
70
70
|
row_count,
|
|
71
71
|
comment = CONVERT(varchar(1024), ep.value)
|
|
72
72
|
FROM extended_tables_with_row_count AS et
|
|
73
|
-
LEFT JOIN {database}.sys.schemas AS ss
|
|
73
|
+
LEFT JOIN [{database}].sys.schemas AS ss
|
|
74
74
|
ON et.schema_id = ss.schema_id
|
|
75
|
-
LEFT JOIN {database}.sys.sysusers AS u
|
|
75
|
+
LEFT JOIN [{database}].sys.sysusers AS u
|
|
76
76
|
ON et.table_owner_id = u.uid
|
|
77
|
-
LEFT JOIN {database}.sys.extended_properties AS ep
|
|
77
|
+
LEFT JOIN [{database}].sys.extended_properties AS ep
|
|
78
78
|
ON (
|
|
79
79
|
et.table_id = ep.major_id
|
|
80
80
|
AND ep.minor_id = 0
|
|
@@ -90,8 +90,8 @@ meta AS (
|
|
|
90
90
|
t.table_name,
|
|
91
91
|
t.table_type
|
|
92
92
|
FROM
|
|
93
|
-
{database}.
|
|
94
|
-
LEFT JOIN {database}.sys.databases AS db
|
|
93
|
+
[{database}].INFORMATION_SCHEMA.TABLES AS t
|
|
94
|
+
LEFT JOIN [{database}].sys.databases AS db
|
|
95
95
|
ON t.table_catalog COLLATE DATABASE_DEFAULT = db.name COLLATE DATABASE_DEFAULT
|
|
96
96
|
)
|
|
97
97
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
SELECT
|
|
2
|
+
v.name AS view_name,
|
|
3
|
+
m.definition AS view_definition,
|
|
4
|
+
s.name AS schema_name,
|
|
5
|
+
DB_NAME() AS database_name
|
|
6
|
+
FROM
|
|
7
|
+
sys.views v
|
|
8
|
+
INNER JOIN
|
|
9
|
+
sys.schemas s
|
|
10
|
+
ON v.schema_id = s.schema_id
|
|
11
|
+
INNER JOIN
|
|
12
|
+
sys.sql_modules m
|
|
13
|
+
ON v.object_id = m.object_id
|
|
@@ -11,6 +11,11 @@ from ..abstract import (
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
12
12
|
|
|
13
13
|
|
|
14
|
+
_NO_DATABASE_ERROR_MSG = (
|
|
15
|
+
"No databases eligible for extraction. "
|
|
16
|
+
"If you are using the db_allow/db_block options, please make sure to use the correct case."
|
|
17
|
+
)
|
|
18
|
+
|
|
14
19
|
_DATABASE_REQUIRED = (
|
|
15
20
|
WarehouseAsset.SCHEMA,
|
|
16
21
|
WarehouseAsset.TABLE,
|
|
@@ -29,6 +34,8 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
|
|
|
29
34
|
time_filter: Optional[TimeFilter] = None,
|
|
30
35
|
):
|
|
31
36
|
super().__init__(time_filter=time_filter)
|
|
37
|
+
if not databases:
|
|
38
|
+
raise ValueError(_NO_DATABASE_ERROR_MSG)
|
|
32
39
|
self._databases = databases
|
|
33
40
|
|
|
34
41
|
@staticmethod
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: castor-extractor
|
|
3
|
-
Version: 0.24.
|
|
3
|
+
Version: 0.24.48
|
|
4
4
|
Summary: Extract your metadata assets.
|
|
5
5
|
Home-page: https://www.castordoc.com/
|
|
6
6
|
License: EULA
|
|
@@ -215,6 +215,32 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
|
|
|
215
215
|
|
|
216
216
|
# Changelog
|
|
217
217
|
|
|
218
|
+
## 0.24.48 - 2025-09-09
|
|
219
|
+
|
|
220
|
+
* SqlServer: handle hyphens in database name
|
|
221
|
+
|
|
222
|
+
## 0.24.47 - 2025-09-08
|
|
223
|
+
|
|
224
|
+
* SqlServer: extract SQL queries and lineage
|
|
225
|
+
|
|
226
|
+
## 0.24.46 - 2025-09-03
|
|
227
|
+
|
|
228
|
+
* Sigma: Added `HTTPStatus.FORBIDDEN` to the list of ignored errors
|
|
229
|
+
|
|
230
|
+
## 0.24.45 - 2025-08-27
|
|
231
|
+
|
|
232
|
+
* Sigma: Increasing pagination limit for Sigma extraction
|
|
233
|
+
|
|
234
|
+
## 0.24.44 - 2025-08-22
|
|
235
|
+
|
|
236
|
+
* Coalesce: do not skip nodes raising a 500 Server Error
|
|
237
|
+
|
|
238
|
+
## 0.24.43 - 2025-08-20
|
|
239
|
+
|
|
240
|
+
* SQLServer:
|
|
241
|
+
* raise error when no database is left after filtering
|
|
242
|
+
* use uppercase INFORMATION_SCHEMA for case-sensitive database compatibility
|
|
243
|
+
|
|
218
244
|
## 0.24.42 - 2025-08-19
|
|
219
245
|
|
|
220
246
|
* Strategy: exclude COLUMNS from extraction
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
CHANGELOG.md,sha256=
|
|
1
|
+
CHANGELOG.md,sha256=DOYbtl6KMcBPSjj_bVV8lyt8Q6IjnIDXecsidYxdWyw,20140
|
|
2
2
|
Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
|
|
3
3
|
DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
|
|
4
4
|
LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
@@ -24,7 +24,7 @@ castor_extractor/commands/extract_salesforce.py,sha256=3j3YTmMkPAwocR-B1ozJQai0U
|
|
|
24
24
|
castor_extractor/commands/extract_salesforce_reporting.py,sha256=FdANTNiLkIPdm80XMYxWReHjdycLsIa61pyeCD-sUDk,962
|
|
25
25
|
castor_extractor/commands/extract_sigma.py,sha256=sxewHcZ1Doq35V2qnpX_zCKKXkrb1_9bYjUMg7BOW-k,643
|
|
26
26
|
castor_extractor/commands/extract_snowflake.py,sha256=GwlrRxwEBjHqGs_3bs5vM9fzmv61_iwvBr1KcIgFgWM,2161
|
|
27
|
-
castor_extractor/commands/extract_sqlserver.py,sha256
|
|
27
|
+
castor_extractor/commands/extract_sqlserver.py,sha256=zwPITImztIIyIbZJyE1SioGraVrlRfxgLq_vHX6l5l0,1664
|
|
28
28
|
castor_extractor/commands/extract_strategy.py,sha256=Q-pUymatPrBFGXobhyUPzFph0-t774-XOpjdCFF1dYo,821
|
|
29
29
|
castor_extractor/commands/extract_tableau.py,sha256=LNtI29LbVk1vp4RNrn89GmdW6R_7QBYunRmkowDhbco,1982
|
|
30
30
|
castor_extractor/commands/extract_thoughtspot.py,sha256=caAYJlH-vK7u5IUB6OKXxcaWfLgc7d_XqnFDWK6YNS4,639
|
|
@@ -76,7 +76,7 @@ castor_extractor/transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
|
|
|
76
76
|
castor_extractor/transformation/coalesce/__init__.py,sha256=CW_qdtEfwgJRsCyBlk5hNlxwEO-VV6mBXZvkRbND_J8,112
|
|
77
77
|
castor_extractor/transformation/coalesce/assets.py,sha256=pzccYPP66c9PAnVroemx7-6MeRHw7Ft1OlTC6jIamAA,363
|
|
78
78
|
castor_extractor/transformation/coalesce/client/__init__.py,sha256=VRmVpH29rOghtDQnCN7dAdA0dI0Lxseu4BC8rnwM9dU,80
|
|
79
|
-
castor_extractor/transformation/coalesce/client/client.py,sha256=
|
|
79
|
+
castor_extractor/transformation/coalesce/client/client.py,sha256=Fm4ozdjosV77AvLVUvOkdMuH2GhepH-AU4xyu7WEgMM,5826
|
|
80
80
|
castor_extractor/transformation/coalesce/client/credentials.py,sha256=jbJxjbdPspf-dzYKfeb7oqL_8TXd1nvkJrjAcdAnLPc,548
|
|
81
81
|
castor_extractor/transformation/coalesce/client/endpoint.py,sha256=0uLh7dpA1vsR9qr_50SEYV_-heQE4BwED9oNMgYsL-w,1272
|
|
82
82
|
castor_extractor/transformation/coalesce/client/pagination.py,sha256=zynyWCMEzUQ7HA1Q5AP4BAOmxRQI6NA5jCPEo0lHn44,705
|
|
@@ -272,11 +272,11 @@ castor_extractor/visualization/salesforce_reporting/extract.py,sha256=ScStilebLG
|
|
|
272
272
|
castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13cMtnKNytiFRomwav27Q,114
|
|
273
273
|
castor_extractor/visualization/sigma/assets.py,sha256=uKGKDaeY1ejc7XGh4eFaNp2ygG7hgca132xsX4eCwKQ,380
|
|
274
274
|
castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
|
|
275
|
-
castor_extractor/visualization/sigma/client/client.py,sha256=
|
|
275
|
+
castor_extractor/visualization/sigma/client/client.py,sha256=Hg9IACQyRkYiH19S-LqClArQDOxilt8-l2s7u9o717k,11448
|
|
276
276
|
castor_extractor/visualization/sigma/client/client_test.py,sha256=ae0ZOvKutCm44jnrJ-0_A5Y6ZGyDkMf9Ml3eEP8dNkY,581
|
|
277
277
|
castor_extractor/visualization/sigma/client/credentials.py,sha256=XddAuQSmCKpxJ70TQgRnOj0vMPYVtiStk_lMMQ1AiNM,693
|
|
278
278
|
castor_extractor/visualization/sigma/client/endpoints.py,sha256=i7KTKnl2Os6752CdtJl0vPSC_Z6JxmacodV_saOnce0,1662
|
|
279
|
-
castor_extractor/visualization/sigma/client/pagination.py,sha256=
|
|
279
|
+
castor_extractor/visualization/sigma/client/pagination.py,sha256=1yLpCNps5FnDiPcXCcgHu23cxg15Gfc6FvE3AJleb2c,728
|
|
280
280
|
castor_extractor/visualization/sigma/client/sources_transformer.py,sha256=n-5mZWSvzfTwpM5VP_bwlcxcaAwCKEEbpMCG_1KRVP4,3748
|
|
281
281
|
castor_extractor/visualization/sigma/client/sources_transformer_test.py,sha256=06yUHXyv65amXLKXhix6K3kkVc1kpBqSjIYcxbyMI4Y,2766
|
|
282
282
|
castor_extractor/visualization/sigma/extract.py,sha256=poTh70Xm2D6BwbdGApLkjXy6-t4iZnOoMB5DPfaTLEI,2929
|
|
@@ -311,7 +311,7 @@ castor_extractor/visualization/thoughtspot/client/pagination.py,sha256=iosYUJ7ZM
|
|
|
311
311
|
castor_extractor/visualization/thoughtspot/extract.py,sha256=mcXS0jGFpa50td98AVbbTqxchyI5wDCpB-v1o5iRc3g,1354
|
|
312
312
|
castor_extractor/warehouse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
313
313
|
castor_extractor/warehouse/abstract/__init__.py,sha256=Fdfa026tgOo64MvzVRLHM_F2G-JmcehrF0mh3dHgb7s,419
|
|
314
|
-
castor_extractor/warehouse/abstract/asset.py,sha256=
|
|
314
|
+
castor_extractor/warehouse/abstract/asset.py,sha256=wR5mJxAHBcqJ86HRb_Y8x3mDN4uUgSg8jMToLNu0jTM,2740
|
|
315
315
|
castor_extractor/warehouse/abstract/asset_test.py,sha256=_kd4ybNlWSAdSdEgJKC-jhJTa1nMRa9i8RO3YbqKLM4,758
|
|
316
316
|
castor_extractor/warehouse/abstract/extract.py,sha256=9Y2fUn3y2-2WjiHnrabjvAvOA8UETJeTYr18zcM7bdI,2924
|
|
317
317
|
castor_extractor/warehouse/abstract/query.py,sha256=20vGhKKX0Kdprj0pbyt0A7L82hwVPezovx38h8seEfA,2614
|
|
@@ -335,7 +335,7 @@ castor_extractor/warehouse/bigquery/queries/view_ddl.sql,sha256=obCm-IN9V8_YSZTw
|
|
|
335
335
|
castor_extractor/warehouse/bigquery/query.py,sha256=FEekxlkrfAXzsT8Kj1AIqYd5mURB5MlZIkbFVXVqEhU,4762
|
|
336
336
|
castor_extractor/warehouse/bigquery/types.py,sha256=rfKkKA13Et7TM4I0uVaXkLfuaBXkv51bNTp4AO0QSdw,57
|
|
337
337
|
castor_extractor/warehouse/databricks/__init__.py,sha256=YG3YSIJgCFRjjI8eExy9T7qGnfnjWhMFh8c15KTs_BA,184
|
|
338
|
-
castor_extractor/warehouse/databricks/api_client.py,sha256=
|
|
338
|
+
castor_extractor/warehouse/databricks/api_client.py,sha256=qBf2T5R1l5a1_kmTbrTYW8VNsz4h_L3zCpUuboNEunQ,6448
|
|
339
339
|
castor_extractor/warehouse/databricks/api_client_test.py,sha256=YTWC-X7L-XAfK5b39TUgTmR1ifv0QrY5tvLNoSbpmjg,466
|
|
340
340
|
castor_extractor/warehouse/databricks/client.py,sha256=LzpeVQIOYi_QTfdOHbK6SB4SgxhZ7p9TNxh0Iwfz850,3307
|
|
341
341
|
castor_extractor/warehouse/databricks/client_test.py,sha256=dqEdEAt-6e8CtQ7M2L5vDYkn4JvOjqyqZSFEpQ55WRc,1432
|
|
@@ -421,18 +421,20 @@ castor_extractor/warehouse/snowflake/queries/user.sql,sha256=88V8eRj1NDaD_ufclsK
|
|
|
421
421
|
castor_extractor/warehouse/snowflake/queries/view_ddl.sql,sha256=eWsci_50cxiYIv3N7BKkbXVM3RoIzqSDtohqRnE5kg4,673
|
|
422
422
|
castor_extractor/warehouse/snowflake/query.py,sha256=C2LTdPwBzMQ_zMncg0Kq4_WkoY7K9as5tvxBDrIOlwI,1763
|
|
423
423
|
castor_extractor/warehouse/sqlserver/__init__.py,sha256=PdOuYznmvKAbfWAm8UdN47MfEsd9jqPi_dDi3WEo1KY,116
|
|
424
|
-
castor_extractor/warehouse/sqlserver/client.py,sha256=
|
|
425
|
-
castor_extractor/warehouse/sqlserver/extract.py,sha256=
|
|
424
|
+
castor_extractor/warehouse/sqlserver/client.py,sha256=oyHYxp-KPJAJnSnYFwXce5zDH7NO1KbD2thT7bjYHa8,5155
|
|
425
|
+
castor_extractor/warehouse/sqlserver/extract.py,sha256=HEJFDM1a4OeQH7OWhYhCOjhkHfGW6qf_qvjFAeMGPYg,2623
|
|
426
426
|
castor_extractor/warehouse/sqlserver/queries/.sqlfluff,sha256=yy0KQdz8I_67vnXyX8eeWwOWkxTXvHyVKSVwhURktd8,48
|
|
427
|
-
castor_extractor/warehouse/sqlserver/queries/column.sql,sha256=
|
|
427
|
+
castor_extractor/warehouse/sqlserver/queries/column.sql,sha256=ojiUQQnHXdWMbgaYOcxKBiwfi7rtu_tyamK6r4t4IBM,2929
|
|
428
428
|
castor_extractor/warehouse/sqlserver/queries/database.sql,sha256=4dPeBCn85MEOXr1f-DPXxiI3RvvoE_1n8lsbTs26E0I,150
|
|
429
|
-
castor_extractor/warehouse/sqlserver/queries/
|
|
430
|
-
castor_extractor/warehouse/sqlserver/queries/
|
|
429
|
+
castor_extractor/warehouse/sqlserver/queries/query.sql,sha256=T0eleIygrFvg77vuT5RxTXrMMHXjxqaKwGV4icPsmEs,769
|
|
430
|
+
castor_extractor/warehouse/sqlserver/queries/schema.sql,sha256=6oJ3S-Ql4JMenDNE2hRHUqZ1d3tDGbzjPvR-dDzA5xs,889
|
|
431
|
+
castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=hJOlUCRVoZzRAcNQgPeM0ZFs5SA3_h0rZqC5S3xEOWs,2855
|
|
431
432
|
castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
|
|
432
|
-
castor_extractor/warehouse/sqlserver/
|
|
433
|
+
castor_extractor/warehouse/sqlserver/queries/view_ddl.sql,sha256=5XkVBmJR2zDeutDSEL0h-uCqwuyXuiRQ75qUbUSK8kw,276
|
|
434
|
+
castor_extractor/warehouse/sqlserver/query.py,sha256=QcGJ3lYbWpGCx-qMytTiEnTqGeEsUZrBLIdg_9jSm8A,1526
|
|
433
435
|
castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
|
|
434
|
-
castor_extractor-0.24.
|
|
435
|
-
castor_extractor-0.24.
|
|
436
|
-
castor_extractor-0.24.
|
|
437
|
-
castor_extractor-0.24.
|
|
438
|
-
castor_extractor-0.24.
|
|
436
|
+
castor_extractor-0.24.48.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
437
|
+
castor_extractor-0.24.48.dist-info/METADATA,sha256=pmX7WeHhQiNrhkAiKEnz9XXMuzvqjRh2c8PENHRDNM0,27593
|
|
438
|
+
castor_extractor-0.24.48.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
439
|
+
castor_extractor-0.24.48.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
|
|
440
|
+
castor_extractor-0.24.48.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|