castor-extractor 0.19.6__py3-none-any.whl → 0.19.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of castor-extractor might be problematic. Click here for more details.
- CHANGELOG.md +10 -0
- castor_extractor/knowledge/notion/client/client.py +3 -0
- castor_extractor/knowledge/notion/client/pagination.py +5 -3
- castor_extractor/quality/soda/client/client.py +15 -5
- castor_extractor/quality/soda/client/credentials.py +3 -0
- castor_extractor/quality/soda/client/pagination.py +1 -2
- castor_extractor/visualization/domo/client/client.py +3 -1
- castor_extractor/visualization/metabase/client/api/client.py +13 -2
- castor_extractor/warehouse/snowflake/queries/table.sql +2 -0
- {castor_extractor-0.19.6.dist-info → castor_extractor-0.19.8.dist-info}/METADATA +13 -3
- {castor_extractor-0.19.6.dist-info → castor_extractor-0.19.8.dist-info}/RECORD +14 -23
- castor_extractor/warehouse/synapse/__init__.py +0 -1
- castor_extractor/warehouse/synapse/extract.py +0 -21
- castor_extractor/warehouse/synapse/queries/.sqlfluff +0 -2
- castor_extractor/warehouse/synapse/queries/database.sql +0 -6
- castor_extractor/warehouse/synapse/queries/query.sql +0 -33
- castor_extractor/warehouse/synapse/queries/schema.sql +0 -7
- castor_extractor/warehouse/synapse/queries/table.sql +0 -36
- castor_extractor/warehouse/synapse/queries/user.sql +0 -4
- castor_extractor/warehouse/synapse/queries/view_ddl.sql +0 -8
- {castor_extractor-0.19.6.dist-info → castor_extractor-0.19.8.dist-info}/LICENCE +0 -0
- {castor_extractor-0.19.6.dist-info → castor_extractor-0.19.8.dist-info}/WHEEL +0 -0
- {castor_extractor-0.19.6.dist-info → castor_extractor-0.19.8.dist-info}/entry_points.txt +0 -0
CHANGELOG.md
CHANGED
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
|
|
2
2
|
# Changelog
|
|
3
3
|
|
|
4
|
+
## 0.19.8 - 2024-09-18
|
|
5
|
+
|
|
6
|
+
* Metabase: Handle duplicate dashboards
|
|
7
|
+
* Snowflake: Exclude unnamed tables from extraction
|
|
8
|
+
* Bump dependencies: cryptography, setuptools
|
|
9
|
+
|
|
10
|
+
## 0.19.7 - 2024-09-05
|
|
11
|
+
|
|
12
|
+
* Metabase: Handle compatibility with older version
|
|
13
|
+
|
|
4
14
|
## 0.19.6 - 2024-09-03
|
|
5
15
|
|
|
6
16
|
* Metabase: Adding error handler on API call
|
|
@@ -20,6 +20,8 @@ NOTION_BASE_HEADERS = {
|
|
|
20
20
|
"User-Agent": CASTOR_NOTION_USER_AGENT,
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
NOTION_DEFAULT_TIMEOUT_S = 180
|
|
24
|
+
|
|
23
25
|
|
|
24
26
|
def _search_filter(asset: str) -> Dict[str, Dict[str, str]]:
|
|
25
27
|
return {"filter": {"value": asset, "property": "object"}}
|
|
@@ -45,6 +47,7 @@ class NotionClient(APIClient):
|
|
|
45
47
|
auth=auth,
|
|
46
48
|
headers=NOTION_BASE_HEADERS,
|
|
47
49
|
safe_mode=safe_mode or NOTION_SAFE_MODE,
|
|
50
|
+
timeout=NOTION_DEFAULT_TIMEOUT_S,
|
|
48
51
|
)
|
|
49
52
|
|
|
50
53
|
def users(self) -> Iterator[dict]:
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
+
from pydantic import Field
|
|
4
|
+
|
|
3
5
|
from ....utils import PaginationModel
|
|
4
6
|
|
|
5
7
|
|
|
6
8
|
class NotionPagination(PaginationModel):
|
|
7
9
|
"""Class to handle paginated results"""
|
|
8
10
|
|
|
9
|
-
results: list
|
|
10
|
-
next_cursor: Optional[str]
|
|
11
|
-
has_more: bool
|
|
11
|
+
results: list = Field(default_factory=list)
|
|
12
|
+
next_cursor: Optional[str] = None
|
|
13
|
+
has_more: bool = False
|
|
12
14
|
|
|
13
15
|
def is_last(self) -> bool:
|
|
14
16
|
return not self.has_more
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from functools import partial
|
|
2
|
-
from typing import Iterator
|
|
2
|
+
from typing import Callable, Iterator
|
|
3
3
|
|
|
4
4
|
from ....utils import (
|
|
5
5
|
APIClient,
|
|
@@ -11,10 +11,10 @@ from .credentials import SodaCredentials
|
|
|
11
11
|
from .endpoints import SodaEndpointFactory
|
|
12
12
|
from .pagination import SodaCloudPagination
|
|
13
13
|
|
|
14
|
-
_CLOUD_API = "https://cloud.soda.io/api/v1/"
|
|
15
14
|
_REQUESTS_PER_MINUTE = 10
|
|
16
15
|
_SECONDS_PER_MINUTE = 60
|
|
17
16
|
_RATE_LIMIT_MS = (_SECONDS_PER_MINUTE // _REQUESTS_PER_MINUTE) + 1
|
|
17
|
+
_CLOUD_PAGE_SIZE = 100
|
|
18
18
|
|
|
19
19
|
HEADERS = {"Content-Type": "application/json"}
|
|
20
20
|
|
|
@@ -24,16 +24,26 @@ class SodaClient(APIClient):
|
|
|
24
24
|
cloud_auth = BasicAuth(
|
|
25
25
|
username=credentials.api_key, password=credentials.secret
|
|
26
26
|
)
|
|
27
|
-
super().__init__(
|
|
27
|
+
super().__init__(
|
|
28
|
+
host=credentials.host, auth=cloud_auth, headers=HEADERS
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def _base_request(self) -> Callable:
|
|
33
|
+
return partial(self._get, params={"size": _CLOUD_PAGE_SIZE})
|
|
28
34
|
|
|
29
35
|
def datasets(self) -> Iterator[dict]:
|
|
30
|
-
request = partial(
|
|
36
|
+
request = partial(
|
|
37
|
+
self._base_request, endpoint=SodaEndpointFactory.datasets()
|
|
38
|
+
)
|
|
31
39
|
yield from fetch_all_pages(
|
|
32
40
|
request, SodaCloudPagination, rate_limit=_RATE_LIMIT_MS
|
|
33
41
|
)
|
|
34
42
|
|
|
35
43
|
def checks(self) -> Iterator[dict]:
|
|
36
|
-
request = partial(
|
|
44
|
+
request = partial(
|
|
45
|
+
self._base_request, endpoint=SodaEndpointFactory.checks()
|
|
46
|
+
)
|
|
37
47
|
yield from fetch_all_pages(
|
|
38
48
|
request, SodaCloudPagination, rate_limit=_RATE_LIMIT_MS
|
|
39
49
|
)
|
|
@@ -3,6 +3,8 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
3
3
|
|
|
4
4
|
SODA_ENV_PREFIX = "CASTOR_SODA_"
|
|
5
5
|
|
|
6
|
+
_CLOUD_API = "https://cloud.soda.io/api/v1/"
|
|
7
|
+
|
|
6
8
|
|
|
7
9
|
class SodaCredentials(BaseSettings):
|
|
8
10
|
"""Class to handle Soda rest API permissions"""
|
|
@@ -15,3 +17,4 @@ class SodaCredentials(BaseSettings):
|
|
|
15
17
|
|
|
16
18
|
api_key: str = Field(repr=False)
|
|
17
19
|
secret: str = Field(repr=False)
|
|
20
|
+
host: str = Field(default=_CLOUD_API)
|
|
@@ -2,7 +2,6 @@ from typing import List
|
|
|
2
2
|
|
|
3
3
|
from ....utils import PaginationModel
|
|
4
4
|
|
|
5
|
-
_CLOUD_PAGE_SIZE = 100
|
|
6
5
|
_CLOUD_FIRST_PAGE = 0
|
|
7
6
|
|
|
8
7
|
|
|
@@ -19,7 +18,7 @@ class SodaCloudPagination(PaginationModel):
|
|
|
19
18
|
if self.current_page_payload
|
|
20
19
|
else _CLOUD_FIRST_PAGE
|
|
21
20
|
)
|
|
22
|
-
return {"page": current_page + 1
|
|
21
|
+
return {"page": current_page + 1}
|
|
23
22
|
|
|
24
23
|
def page_results(self) -> list:
|
|
25
24
|
return self.content
|
|
@@ -45,6 +45,8 @@ _RETRY_EXCEPTIONS = [
|
|
|
45
45
|
_RETRY_COUNT = 2
|
|
46
46
|
_RETRY_BASE_MS = 10 * 60 * 1000 # 10 minutes
|
|
47
47
|
|
|
48
|
+
_PARENT_FOLDER = "/Dashboards"
|
|
49
|
+
|
|
48
50
|
logger = logging.getLogger(__name__)
|
|
49
51
|
|
|
50
52
|
|
|
@@ -177,7 +179,7 @@ class DomoClient:
|
|
|
177
179
|
def _process_pages(
|
|
178
180
|
self,
|
|
179
181
|
page_tree: List[dict],
|
|
180
|
-
parent_path: str =
|
|
182
|
+
parent_path: str = _PARENT_FOLDER,
|
|
181
183
|
) -> Iterator[dict]:
|
|
182
184
|
"""Recursively fetch pages while building the folder architecture"""
|
|
183
185
|
if not page_tree:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from http import HTTPStatus
|
|
3
|
-
from typing import Any, Dict, Iterator, List, Optional, cast
|
|
3
|
+
from typing import Any, Dict, Iterator, List, Optional, Set, cast
|
|
4
4
|
|
|
5
5
|
import requests
|
|
6
6
|
from requests import HTTPError
|
|
@@ -134,11 +134,22 @@ class ApiClient:
|
|
|
134
134
|
collection = self._call(f"collection/{_id}/items?models=dashboard")
|
|
135
135
|
if not collection:
|
|
136
136
|
continue
|
|
137
|
+
|
|
138
|
+
seen_dashboard_ids: Set[int] = set()
|
|
139
|
+
|
|
137
140
|
for dashboard in cast(SerializedAsset, collection):
|
|
141
|
+
if dashboard.get("model") != "dashboard":
|
|
142
|
+
# This is to maintain compatibility with older versions
|
|
143
|
+
# where ?models=dashboard has no effects
|
|
144
|
+
continue
|
|
145
|
+
|
|
138
146
|
dashboard_id = dashboard.get("id")
|
|
139
147
|
if not dashboard_id:
|
|
140
148
|
continue
|
|
141
|
-
|
|
149
|
+
|
|
150
|
+
if dashboard_id not in seen_dashboard_ids:
|
|
151
|
+
seen_dashboard_ids.add(dashboard_id)
|
|
152
|
+
yield cast(Dict, self._call(f"dashboard/{dashboard_id}"))
|
|
142
153
|
|
|
143
154
|
@staticmethod
|
|
144
155
|
def _collection_specifics(collections: SerializedAsset) -> SerializedAsset:
|
|
@@ -41,6 +41,8 @@ FROM snowflake.account_usage.tables AS t
|
|
|
41
41
|
JOIN snowflake.account_usage.schemata AS s ON s.schema_id = t.table_schema_id
|
|
42
42
|
JOIN tags_agg_tables ta ON t.table_id = ta.table_id
|
|
43
43
|
WHERE TRUE
|
|
44
|
+
AND t.table_name IS NOT NULL
|
|
45
|
+
AND t.table_name != ''
|
|
44
46
|
AND UPPER(t.table_catalog) NOT IN ('SNOWFLAKE', 'UTIL_DB')
|
|
45
47
|
AND (
|
|
46
48
|
t.deleted IS NULL
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: castor-extractor
|
|
3
|
-
Version: 0.19.
|
|
3
|
+
Version: 0.19.8
|
|
4
4
|
Summary: Extract your metadata assets.
|
|
5
5
|
Home-page: https://www.castordoc.com/
|
|
6
6
|
License: EULA
|
|
@@ -29,7 +29,7 @@ Provides-Extra: redshift
|
|
|
29
29
|
Provides-Extra: snowflake
|
|
30
30
|
Provides-Extra: sqlserver
|
|
31
31
|
Provides-Extra: tableau
|
|
32
|
-
Requires-Dist: cryptography (>=
|
|
32
|
+
Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
|
|
33
33
|
Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
|
|
34
34
|
Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
|
|
35
35
|
Requires-Dist: google-auth (>=2,<3)
|
|
@@ -52,7 +52,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
|
|
|
52
52
|
Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
|
|
53
53
|
Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
|
|
54
54
|
Requires-Dist: requests (>=2.0.0,<3.0.0)
|
|
55
|
-
Requires-Dist: setuptools (>=
|
|
55
|
+
Requires-Dist: setuptools (>=74,<75)
|
|
56
56
|
Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
|
|
57
57
|
Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
|
|
58
58
|
Requires-Dist: sqlalchemy (>=1.4,<1.5)
|
|
@@ -208,6 +208,16 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
|
|
|
208
208
|
|
|
209
209
|
# Changelog
|
|
210
210
|
|
|
211
|
+
## 0.19.8 - 2024-09-18
|
|
212
|
+
|
|
213
|
+
* Metabase: Handle duplicate dashboards
|
|
214
|
+
* Snowflake: Exclude unnamed tables from extraction
|
|
215
|
+
* Bump dependencies: cryptography, setuptools
|
|
216
|
+
|
|
217
|
+
## 0.19.7 - 2024-09-05
|
|
218
|
+
|
|
219
|
+
* Metabase: Handle compatibility with older version
|
|
220
|
+
|
|
211
221
|
## 0.19.6 - 2024-09-03
|
|
212
222
|
|
|
213
223
|
* Metabase: Adding error handler on API call
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
CHANGELOG.md,sha256=
|
|
1
|
+
CHANGELOG.md,sha256=5uXftRzGolo1ItaqsT8BA9ItULJemWj5yjHHygl-ePg,13456
|
|
2
2
|
Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
|
|
3
3
|
DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
|
|
4
4
|
LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
@@ -41,22 +41,22 @@ castor_extractor/knowledge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
|
|
|
41
41
|
castor_extractor/knowledge/notion/__init__.py,sha256=ZDmh0eNSxHf1zVPm0aYlKPci-vzOXhAgdsWjS2hdjh4,117
|
|
42
42
|
castor_extractor/knowledge/notion/assets.py,sha256=QHv1-pomt5UeN_prP2L6t_zJ-tDSqB8LgopkGAODYPQ,164
|
|
43
43
|
castor_extractor/knowledge/notion/client/__init__.py,sha256=CDPorBCethuNTEtpjvHGcWnWeVfqkEq-IbakWjDKATw,76
|
|
44
|
-
castor_extractor/knowledge/notion/client/client.py,sha256=
|
|
44
|
+
castor_extractor/knowledge/notion/client/client.py,sha256=UJ2fHfwjCiNEpkTm2inVkPZkmSgfXtBxUgS0fChULy0,3685
|
|
45
45
|
castor_extractor/knowledge/notion/client/client_test.py,sha256=fo3_WgCIUfeF3nMfHTSB6wjxIW_dHp1XZL41tNzBisQ,1976
|
|
46
46
|
castor_extractor/knowledge/notion/client/constants.py,sha256=dZCpSrxFlLbR_cFPJKz4M5wcMVcY4UfWFG0N5S22Fhw,147
|
|
47
47
|
castor_extractor/knowledge/notion/client/credentials.py,sha256=-mBY6IhAb9z2RP1MilzIyS3BW39J__u_M5n9uIuUlxA,398
|
|
48
48
|
castor_extractor/knowledge/notion/client/endpoints.py,sha256=5wUodq7les28CpZkMm5drhry6FFWbBU3V4oeUAn0G-w,405
|
|
49
|
-
castor_extractor/knowledge/notion/client/pagination.py,sha256=
|
|
49
|
+
castor_extractor/knowledge/notion/client/pagination.py,sha256=yIHov3eoQsqRMYZxHrkBxJBl2uKHDLLn3A8PEBz8J8U,518
|
|
50
50
|
castor_extractor/knowledge/notion/extract.py,sha256=ExG354_5VJKLUp_7QQvIl6HnZqBmOypAquY3tA9rFXc,1717
|
|
51
51
|
castor_extractor/logger.py,sha256=ovf1mBEKwbJBskBXoqHbcAomBrp58mUwSrCWtEMlYPM,1197
|
|
52
52
|
castor_extractor/quality/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
53
53
|
castor_extractor/quality/soda/__init__.py,sha256=y2H6f_UqLrqlssKlTVqPwa4ITvRh7na4P6HywrUoTb8,78
|
|
54
54
|
castor_extractor/quality/soda/assets.py,sha256=1mMBFoVOEelmX4dtY0oxzMzHzfwFpGWq6t9UBykJ-80,140
|
|
55
55
|
castor_extractor/quality/soda/client/__init__.py,sha256=LUvHHyyL2t3mk5GORshnef4FgAu7VMqrIQWQx-7kO4I,48
|
|
56
|
-
castor_extractor/quality/soda/client/client.py,sha256=
|
|
57
|
-
castor_extractor/quality/soda/client/credentials.py,sha256=
|
|
56
|
+
castor_extractor/quality/soda/client/client.py,sha256=UEhtN3fpCrVYUGyaO3GXPMqzk8uGwBLyHaaU0hh4zs8,1712
|
|
57
|
+
castor_extractor/quality/soda/client/credentials.py,sha256=R1g7nHpJlQ5hBjtUFN06QjjWAouQtb_V-je7cAXXIA4,514
|
|
58
58
|
castor_extractor/quality/soda/client/endpoints.py,sha256=x3B-XlnDF8NJMuk-81N72_6HA-YZEzA895khLyj0j54,228
|
|
59
|
-
castor_extractor/quality/soda/client/pagination.py,sha256=
|
|
59
|
+
castor_extractor/quality/soda/client/pagination.py,sha256=IrAnju5CFipHarTLZRWZflVZ3KHoRVMvaCaNAePHVug,555
|
|
60
60
|
castor_extractor/types.py,sha256=Hd_shbsGAknJLTrAk3SBxZeFPOlbWBXXjIscC9C7CW8,1281
|
|
61
61
|
castor_extractor/uploader/__init__.py,sha256=SSRtwjg-dNoxME-RJy9G1flASiUKAC5bH1htq3CURQg,75
|
|
62
62
|
castor_extractor/uploader/constant.py,sha256=yTigLHDlYwoRr6CpFIl7ReElFsQd4H-qkluMZJPWSx0,865
|
|
@@ -134,7 +134,7 @@ castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
|
|
|
134
134
|
castor_extractor/visualization/domo/__init__.py,sha256=1axOCPm4RpdIyUt9LQEvlMvbOPllW8rk63h6EjVgJ0Y,111
|
|
135
135
|
castor_extractor/visualization/domo/assets.py,sha256=bK1urFR2tnlWkVkkhR32mAKMoKbESNlop-CNGx-65PY,206
|
|
136
136
|
castor_extractor/visualization/domo/client/__init__.py,sha256=Do0fU4B8Hhlhahcv734gnJl_ryCztfTBDea7XNCKfB8,72
|
|
137
|
-
castor_extractor/visualization/domo/client/client.py,sha256=
|
|
137
|
+
castor_extractor/visualization/domo/client/client.py,sha256=qzOzloGIUHyePsMTB_y7s388a0XU2OTd9WQnt387YfY,9646
|
|
138
138
|
castor_extractor/visualization/domo/client/credentials.py,sha256=fyJ4wOmCqMQ1g3jDoOjtChtUES0sHfp3eLAfb4JxZP4,879
|
|
139
139
|
castor_extractor/visualization/domo/client/endpoints.py,sha256=jl5JvE6HZ36g9kpbm4TU501uxLa0mMZyitJ7JTJP-HI,2781
|
|
140
140
|
castor_extractor/visualization/domo/client/pagination.py,sha256=ukVkHVzoH4mfZ29H9YcnC2YrdVolP10wv25J6Q3ehRw,821
|
|
@@ -161,7 +161,7 @@ castor_extractor/visualization/metabase/__init__.py,sha256=3E36cmkMyEgBB6Ot5rWk-
|
|
|
161
161
|
castor_extractor/visualization/metabase/assets.py,sha256=RrrE7H0ezqD-YpNLjDOo4u5fFPUK1PIRxLXNERFqFn8,2906
|
|
162
162
|
castor_extractor/visualization/metabase/client/__init__.py,sha256=KBvaPMofBRV3m_sZAnKNCrJGr-Z88EbpdzEzWPQ_uBk,99
|
|
163
163
|
castor_extractor/visualization/metabase/client/api/__init__.py,sha256=BYSPWHY4KbT-LvenNI0pRonxolzZ5HSl6v3-PbJr-7M,78
|
|
164
|
-
castor_extractor/visualization/metabase/client/api/client.py,sha256=
|
|
164
|
+
castor_extractor/visualization/metabase/client/api/client.py,sha256=wKtoRAzp5rqpQ6S7QRz8MLdskjc5-dzBUB6u65zanzY,6638
|
|
165
165
|
castor_extractor/visualization/metabase/client/api/client_test.py,sha256=7Lb5yvrvHQmCIOnFzS2D00oR2Zps1SVxGIAukzRVeKg,559
|
|
166
166
|
castor_extractor/visualization/metabase/client/api/credentials.py,sha256=SiHWyWWEDy_Ak4ne0hgXPZCSP6mEuHH26wDhsed_9v8,519
|
|
167
167
|
castor_extractor/visualization/metabase/client/db/__init__.py,sha256=nawDhJ-JGlpM6VMzZZRjf066QXk9kWzZr6l9n6OHTZ0,76
|
|
@@ -387,7 +387,7 @@ castor_extractor/warehouse/snowflake/queries/grant_to_user.sql,sha256=7AalVajU5v
|
|
|
387
387
|
castor_extractor/warehouse/snowflake/queries/query.sql,sha256=-OYcWUvdPBkpOfezkZaW7hrOdDz3JyoqjNdRm_88Rsk,1779
|
|
388
388
|
castor_extractor/warehouse/snowflake/queries/role.sql,sha256=D0VvGxLZMwug2SvefhAsNR9YIun0fZvcDWkz891xSYM,96
|
|
389
389
|
castor_extractor/warehouse/snowflake/queries/schema.sql,sha256=HCDEw0Nj_GPHBNH3Ik_5BF4rkD5yBfSyeN9UaiFGrI4,730
|
|
390
|
-
castor_extractor/warehouse/snowflake/queries/table.sql,sha256=
|
|
390
|
+
castor_extractor/warehouse/snowflake/queries/table.sql,sha256=qTwkAJ7-kM8vX03RP16U_5_euWW5ZTQAKuiLPsbj2hs,1438
|
|
391
391
|
castor_extractor/warehouse/snowflake/queries/user.sql,sha256=88V8eRj1NDaD_ufclsKOHHlqCtBMQHOV54yy6RKJaXk,570
|
|
392
392
|
castor_extractor/warehouse/snowflake/queries/view_ddl.sql,sha256=eWsci_50cxiYIv3N7BKkbXVM3RoIzqSDtohqRnE5kg4,673
|
|
393
393
|
castor_extractor/warehouse/snowflake/query.py,sha256=yDpG4e23xtjEfAKNSAgL9wx17ChFSlvAbig2mJ5ZEC0,1769
|
|
@@ -401,18 +401,9 @@ castor_extractor/warehouse/sqlserver/queries/schema.sql,sha256=elM9s02I9d9F5E4MH
|
|
|
401
401
|
castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx_LGkIf7LX6ojTjI8wgJDxm3f0,2542
|
|
402
402
|
castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
|
|
403
403
|
castor_extractor/warehouse/sqlserver/query.py,sha256=j_d5-HMnzBouwGfywVZMRSSwbXzPvzDWlFCZmvxcoGQ,539
|
|
404
|
-
castor_extractor/warehouse/synapse/__init__.py,sha256=-VyWS6O9gptZ-FTDnUVDi87r_PUA59dPXPjLG1lwzpY,36
|
|
405
|
-
castor_extractor/warehouse/synapse/extract.py,sha256=Tvo6BOXsw2T5Suf4b-aCwJf4GzssijjbpVhulBjazAs,538
|
|
406
|
-
castor_extractor/warehouse/synapse/queries/.sqlfluff,sha256=7eDQrhRQHKUc5zmmNCNG55acRNPMX4mzxylOLf2pM_g,26
|
|
407
404
|
castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
|
|
408
|
-
castor_extractor/
|
|
409
|
-
castor_extractor/
|
|
410
|
-
castor_extractor/
|
|
411
|
-
castor_extractor/
|
|
412
|
-
castor_extractor/
|
|
413
|
-
castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
|
|
414
|
-
castor_extractor-0.19.6.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
415
|
-
castor_extractor-0.19.6.dist-info/METADATA,sha256=WwVX_mbh9I8rK33bnNoCeljtxLXNwQQKgCl-pdyJe58,20434
|
|
416
|
-
castor_extractor-0.19.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
417
|
-
castor_extractor-0.19.6.dist-info/entry_points.txt,sha256=X_pDYOmhUUMbiAD9h2GZveuGdT8UgL38KJqP44xkvqo,1495
|
|
418
|
-
castor_extractor-0.19.6.dist-info/RECORD,,
|
|
405
|
+
castor_extractor-0.19.8.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
406
|
+
castor_extractor-0.19.8.dist-info/METADATA,sha256=aTfG24vormzYihCb2vjGUX2_EfOr9rDvmCyRABWhd10,20674
|
|
407
|
+
castor_extractor-0.19.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
408
|
+
castor_extractor-0.19.8.dist-info/entry_points.txt,sha256=X_pDYOmhUUMbiAD9h2GZveuGdT8UgL38KJqP44xkvqo,1495
|
|
409
|
+
castor_extractor-0.19.8.dist-info/RECORD,,
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
from .extract import SYNAPSE_ASSETS
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
|
|
3
|
-
from ..abstract import (
|
|
4
|
-
CATALOG_ASSETS,
|
|
5
|
-
EXTERNAL_LINEAGE_ASSETS,
|
|
6
|
-
QUERIES_ASSETS,
|
|
7
|
-
VIEWS_ASSETS,
|
|
8
|
-
SupportedAssets,
|
|
9
|
-
WarehouseAsset,
|
|
10
|
-
WarehouseAssetGroup,
|
|
11
|
-
)
|
|
12
|
-
|
|
13
|
-
logger = logging.getLogger(__name__)
|
|
14
|
-
|
|
15
|
-
SYNAPSE_ASSETS: SupportedAssets = {
|
|
16
|
-
WarehouseAssetGroup.CATALOG: CATALOG_ASSETS,
|
|
17
|
-
WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
|
|
18
|
-
WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
|
|
19
|
-
WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
|
|
20
|
-
WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
|
|
21
|
-
}
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
WITH query_execution AS (
|
|
2
|
-
SELECT
|
|
3
|
-
plan_id,
|
|
4
|
-
CASE execution_type_desc
|
|
5
|
-
WHEN 'Aborted' THEN 1
|
|
6
|
-
WHEN 'Exception' THEN 1
|
|
7
|
-
ELSE 0
|
|
8
|
-
END AS aborted,
|
|
9
|
-
MIN(CAST(first_execution_time AS DATETIME)) AS start_time,
|
|
10
|
-
MAX(CAST(last_execution_time AS DATETIME)) AS end_time
|
|
11
|
-
FROM sys.query_store_runtime_stats
|
|
12
|
-
WHERE CAST(first_execution_time AS DATE) = CAST(DATEADD(day, -1, getdate()) AS DATE)
|
|
13
|
-
-- Filtering on hour will be needed when running that query out of notebooks
|
|
14
|
-
GROUP BY plan_id, execution_type_desc
|
|
15
|
-
)
|
|
16
|
-
|
|
17
|
-
SELECT
|
|
18
|
-
q.query_id,
|
|
19
|
-
qt.query_sql_text AS query_text,
|
|
20
|
-
DB_NAME() AS database_id,
|
|
21
|
-
DB_NAME() AS database_name,
|
|
22
|
-
'SYNAPSE' AS user_name,
|
|
23
|
-
qp.plan_id AS process_id,
|
|
24
|
-
qe.aborted AS aborted,
|
|
25
|
-
qe.start_time AS start_time,
|
|
26
|
-
qe.end_time AS end_time
|
|
27
|
-
FROM sys.query_store_query AS q
|
|
28
|
-
JOIN sys.query_store_query_text AS qt
|
|
29
|
-
ON q.query_text_id = qt.query_text_id
|
|
30
|
-
JOIN sys.query_store_plan AS qp
|
|
31
|
-
ON q.query_id = qp.query_id
|
|
32
|
-
JOIN query_execution AS qe
|
|
33
|
-
ON qp.plan_id = qe.plan_id
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
WITH users AS (
|
|
2
|
-
SELECT DISTINCT
|
|
3
|
-
uid,
|
|
4
|
-
name
|
|
5
|
-
FROM sys.sysusers
|
|
6
|
-
)
|
|
7
|
-
|
|
8
|
-
SELECT
|
|
9
|
-
DB_NAME() as database_id,
|
|
10
|
-
DB_NAME() AS database_name,
|
|
11
|
-
DB_NAME() + '.' + s.name AS schema_id,
|
|
12
|
-
s.name AS schema_name,
|
|
13
|
-
DB_NAME() + '.' + s.name + '.' + o.name AS table_id,
|
|
14
|
-
o.name AS table_name,
|
|
15
|
-
CASE UPPER(REPLACE(o.type, ' ', ''))
|
|
16
|
-
WHEN 'U' THEN 'TABLE'
|
|
17
|
-
WHEN 'V' THEN 'VIEW'
|
|
18
|
-
ELSE 'UNKNOWN'
|
|
19
|
-
END AS table_type,
|
|
20
|
-
-- Normally schema owner is table owner, but we can
|
|
21
|
-
-- change table owner with ALTER AUTHORIZATION command
|
|
22
|
-
-- in this case we have to look up for the owner
|
|
23
|
-
-- in sys.objects table
|
|
24
|
-
COALESCE(su.name, pu.name) AS table_owner,
|
|
25
|
-
COALESCE(su.uid, pu.uid) AS table_owner_id,
|
|
26
|
-
NULL AS comment
|
|
27
|
-
FROM sys.objects AS o
|
|
28
|
-
JOIN sys.schemas AS s
|
|
29
|
-
ON o.schema_id = s.schema_id
|
|
30
|
-
LEFT JOIN users AS pu
|
|
31
|
-
ON s.principal_id = pu.uid
|
|
32
|
-
LEFT JOIN users AS su
|
|
33
|
-
ON o.principal_id = su.uid
|
|
34
|
-
WHERE
|
|
35
|
-
-- SYNAPSE NOTATION: U for TABLE, V for VIEW
|
|
36
|
-
o.type IN ('U', 'V')
|
|
File without changes
|
|
File without changes
|
|
File without changes
|