castor-extractor 0.24.57__py3-none-any.whl → 0.25.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of castor-extractor might be problematic. Click here for more details.
- CHANGELOG.md +28 -0
- castor_extractor/commands/extract_count.py +22 -0
- castor_extractor/commands/extract_powerbi.py +16 -1
- castor_extractor/utils/client/abstract.py +11 -0
- castor_extractor/utils/client/api/pagination.py +1 -1
- castor_extractor/visualization/count/__init__.py +3 -0
- castor_extractor/visualization/count/assets.py +11 -0
- castor_extractor/visualization/count/client/__init__.py +2 -0
- castor_extractor/visualization/count/client/client.py +50 -0
- castor_extractor/visualization/count/client/credentials.py +10 -0
- castor_extractor/visualization/count/client/queries/canvas_permissions.sql +6 -0
- castor_extractor/visualization/count/client/queries/canvases.sql +6 -0
- castor_extractor/visualization/count/client/queries/cells.sql +8 -0
- castor_extractor/visualization/count/client/queries/projects.sql +5 -0
- castor_extractor/visualization/count/client/queries/users.sql +8 -0
- castor_extractor/visualization/count/extract.py +54 -0
- castor_extractor/visualization/powerbi/client/__init__.py +1 -0
- castor_extractor/visualization/powerbi/client/authentication.py +20 -2
- castor_extractor/visualization/powerbi/client/credentials.py +17 -2
- castor_extractor/visualization/powerbi/extract.py +23 -3
- castor_extractor/visualization/sigma/client/client.py +28 -8
- castor_extractor/visualization/sigma/client/sources_transformer.py +29 -10
- castor_extractor/warehouse/abstract/extract.py +1 -0
- castor_extractor/warehouse/abstract/query.py +13 -2
- castor_extractor/warehouse/snowflake/extract.py +1 -6
- castor_extractor/warehouse/sqlserver/client.py +14 -1
- castor_extractor/warehouse/sqlserver/query.py +7 -6
- {castor_extractor-0.24.57.dist-info → castor_extractor-0.25.6.dist-info}/METADATA +31 -2
- {castor_extractor-0.24.57.dist-info → castor_extractor-0.25.6.dist-info}/RECORD +32 -23
- {castor_extractor-0.24.57.dist-info → castor_extractor-0.25.6.dist-info}/entry_points.txt +1 -0
- castor_extractor/warehouse/snowflake/queries/grant_to_role.sql +0 -14
- castor_extractor/warehouse/snowflake/queries/grant_to_user.sql +0 -8
- castor_extractor/warehouse/snowflake/queries/role.sql +0 -6
- {castor_extractor-0.24.57.dist-info → castor_extractor-0.25.6.dist-info}/LICENCE +0 -0
- {castor_extractor-0.24.57.dist-info → castor_extractor-0.25.6.dist-info}/WHEEL +0 -0
CHANGELOG.md
CHANGED
|
@@ -1,5 +1,33 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.25.6 - 2025-10-06
|
|
4
|
+
|
|
5
|
+
* PowerBi: Support additional authentication methods
|
|
6
|
+
|
|
7
|
+
## 0.25.5 - 2025-10-06
|
|
8
|
+
|
|
9
|
+
* Snowflake: remove extraction of roles and grants
|
|
10
|
+
|
|
11
|
+
## 0.25.4 - 2025-10-02
|
|
12
|
+
|
|
13
|
+
* SqlServer: switch database to support clients running their instance on Azure
|
|
14
|
+
|
|
15
|
+
## 0.25.3 - 2025-10-01
|
|
16
|
+
|
|
17
|
+
* Sigma: sleep between sources requests to avoid rate limit errors
|
|
18
|
+
|
|
19
|
+
## 0.25.2 - 2025-09-30
|
|
20
|
+
|
|
21
|
+
* PowerBi: Support auth with private_key
|
|
22
|
+
|
|
23
|
+
## 0.25.1 - 2025-09-29
|
|
24
|
+
|
|
25
|
+
* Sigma: catch ReadTimeouts during elements extraction
|
|
26
|
+
|
|
27
|
+
## 0.25.0 - 2025-09-15
|
|
28
|
+
|
|
29
|
+
* Count: adding connector
|
|
30
|
+
|
|
3
31
|
## 0.24.57 - 2025-09-24
|
|
4
32
|
|
|
5
33
|
* Sigma:
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from argparse import ArgumentParser
|
|
2
|
+
|
|
3
|
+
from castor_extractor.utils import parse_filled_arguments # type: ignore
|
|
4
|
+
from castor_extractor.visualization import count # type: ignore
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def main():
|
|
8
|
+
parser = ArgumentParser()
|
|
9
|
+
|
|
10
|
+
parser.add_argument(
|
|
11
|
+
"-c",
|
|
12
|
+
"--credentials",
|
|
13
|
+
help="GCP credentials as string",
|
|
14
|
+
)
|
|
15
|
+
parser.add_argument("-o", "--output", help="Directory to write to")
|
|
16
|
+
parser.add_argument(
|
|
17
|
+
"-d",
|
|
18
|
+
"--dataset_id",
|
|
19
|
+
help="dataset id, where count info is stored for the current customer",
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
count.extract_all(**parse_filled_arguments(parser))
|
|
@@ -9,10 +9,25 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
|
|
|
9
9
|
|
|
10
10
|
def main():
|
|
11
11
|
parser = ArgumentParser()
|
|
12
|
+
auth_group = parser.add_mutually_exclusive_group(required=True)
|
|
12
13
|
|
|
13
14
|
parser.add_argument("-t", "--tenant_id", help="PowerBi tenant ID")
|
|
14
15
|
parser.add_argument("-c", "--client_id", help="PowerBi client ID")
|
|
15
|
-
|
|
16
|
+
auth_group.add_argument(
|
|
17
|
+
"-s",
|
|
18
|
+
"--secret",
|
|
19
|
+
help="PowerBi password as a string",
|
|
20
|
+
)
|
|
21
|
+
auth_group.add_argument(
|
|
22
|
+
"-cert",
|
|
23
|
+
"--certificate",
|
|
24
|
+
help=(
|
|
25
|
+
"Path to certificate file for authentication. "
|
|
26
|
+
"Accepts: X.509 certificates (.pem, .crt), "
|
|
27
|
+
"PKCS#12 files (.p12, .pfx), or JSON files containing "
|
|
28
|
+
"certificate data or custom authentication secrets."
|
|
29
|
+
),
|
|
30
|
+
)
|
|
16
31
|
parser.add_argument(
|
|
17
32
|
"-sc",
|
|
18
33
|
"--scopes",
|
|
@@ -26,6 +26,17 @@ class AbstractSourceClient(ABC):
|
|
|
26
26
|
def execute(self, query: ExtractionQuery) -> Iterator[dict]:
|
|
27
27
|
pass
|
|
28
28
|
|
|
29
|
+
def set_database(self, database: Optional[str]) -> None:
|
|
30
|
+
"""
|
|
31
|
+
Set the active database for this client.
|
|
32
|
+
|
|
33
|
+
Some source technologies require establishing a new connection
|
|
34
|
+
or using a different connection URI when switching databases.
|
|
35
|
+
|
|
36
|
+
Default behaviour is to do nothing.
|
|
37
|
+
"""
|
|
38
|
+
pass
|
|
39
|
+
|
|
29
40
|
|
|
30
41
|
class SqlalchemyClient(AbstractSourceClient, ABC):
|
|
31
42
|
def __init__(self, credentials: dict):
|
|
@@ -58,7 +58,7 @@ class PaginationModel(BaseModel):
|
|
|
58
58
|
def fetch_all_pages(
|
|
59
59
|
request: Callable,
|
|
60
60
|
pagination_model: type[PaginationModel],
|
|
61
|
-
rate_limit: Optional[
|
|
61
|
+
rate_limit: Optional[float] = None,
|
|
62
62
|
) -> Iterator:
|
|
63
63
|
"""
|
|
64
64
|
Method to return all results of a Paginated API based on the
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from dataclasses import asdict
|
|
3
|
+
from typing import Any, Iterator
|
|
4
|
+
|
|
5
|
+
from ....utils import load_file
|
|
6
|
+
from ....warehouse.bigquery import BigQueryClient
|
|
7
|
+
from ..assets import (
|
|
8
|
+
CountAsset,
|
|
9
|
+
)
|
|
10
|
+
from .credentials import CountCredentials
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
_QUERIES_FOLDER = "queries"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class CountClient(BigQueryClient):
|
|
18
|
+
"""
|
|
19
|
+
Count.co does not currently provide an official API.
|
|
20
|
+
Instead, metadata such as dashboards, users, and queries is made available through
|
|
21
|
+
special metadata tables stored in BigQuery.
|
|
22
|
+
|
|
23
|
+
This client extends `BigQueryClient` to access and interact with those metadata tables.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, credentials: CountCredentials):
|
|
27
|
+
super().__init__(asdict(credentials))
|
|
28
|
+
self.project_id = credentials.project_id
|
|
29
|
+
self.dataset_id = credentials.dataset_id
|
|
30
|
+
|
|
31
|
+
def _load_query(self, asset: CountAsset) -> str:
|
|
32
|
+
query = load_file(
|
|
33
|
+
f"{_QUERIES_FOLDER}/{asset.name.lower()}.sql", __file__
|
|
34
|
+
)
|
|
35
|
+
return query.format(
|
|
36
|
+
project_id=self.project_id, dataset_id=self.dataset_id
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
def fetch(self, asset: CountAsset) -> Iterator[dict[str, Any]]:
|
|
40
|
+
"""
|
|
41
|
+
Fetch the asset given as param, by running a BigQuery query.
|
|
42
|
+
"""
|
|
43
|
+
logger.info(f"Running BigQuery query to fetch: {asset.name}")
|
|
44
|
+
|
|
45
|
+
query_str = self._load_query(asset)
|
|
46
|
+
job = self.client.query(query_str)
|
|
47
|
+
results = job.result()
|
|
48
|
+
|
|
49
|
+
for row in results:
|
|
50
|
+
yield dict(row)
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from pydantic.dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from ....warehouse.bigquery import BigQueryCredentials
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class CountCredentials(BigQueryCredentials):
|
|
8
|
+
"""Count credentials extending BigQuery credentials with additional dataset information"""
|
|
9
|
+
|
|
10
|
+
dataset_id: str
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Iterable, Iterator, Union
|
|
3
|
+
|
|
4
|
+
from ...utils import (
|
|
5
|
+
OUTPUT_DIR,
|
|
6
|
+
current_timestamp,
|
|
7
|
+
deep_serialize,
|
|
8
|
+
from_env,
|
|
9
|
+
get_output_filename,
|
|
10
|
+
write_json,
|
|
11
|
+
write_summary,
|
|
12
|
+
)
|
|
13
|
+
from .assets import (
|
|
14
|
+
CountAsset,
|
|
15
|
+
)
|
|
16
|
+
from .client import (
|
|
17
|
+
CountClient,
|
|
18
|
+
CountCredentials,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def iterate_all_data(
|
|
25
|
+
client: CountClient,
|
|
26
|
+
) -> Iterable[tuple[CountAsset, Union[list, Iterator, dict]]]:
|
|
27
|
+
"""Iterate over the extracted data from count"""
|
|
28
|
+
|
|
29
|
+
for asset in CountAsset:
|
|
30
|
+
logger.info(f"Extracting {asset.value} from API")
|
|
31
|
+
data = client.fetch(asset)
|
|
32
|
+
yield asset, deep_serialize(data)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def extract_all(**kwargs) -> None:
|
|
36
|
+
"""
|
|
37
|
+
Extract data from count BigQuery project
|
|
38
|
+
Store the output files locally under the given output_directory
|
|
39
|
+
"""
|
|
40
|
+
_output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
|
|
41
|
+
dataset_id = kwargs.get("dataset_id")
|
|
42
|
+
if not dataset_id:
|
|
43
|
+
raise ValueError("dataset_id is required")
|
|
44
|
+
|
|
45
|
+
credentials = CountCredentials(**kwargs)
|
|
46
|
+
client = CountClient(credentials=credentials)
|
|
47
|
+
|
|
48
|
+
ts = current_timestamp()
|
|
49
|
+
|
|
50
|
+
for key, data in iterate_all_data(client):
|
|
51
|
+
filename = get_output_filename(key.name.lower(), _output_directory, ts)
|
|
52
|
+
write_json(filename, list(data))
|
|
53
|
+
|
|
54
|
+
write_summary(_output_directory, ts)
|
|
@@ -1,11 +1,24 @@
|
|
|
1
|
+
from typing import Optional, Union
|
|
2
|
+
|
|
1
3
|
import msal # type: ignore
|
|
2
4
|
|
|
3
5
|
from ....utils import BearerAuth
|
|
4
6
|
from .constants import Keys
|
|
5
|
-
from .credentials import PowerbiCredentials
|
|
7
|
+
from .credentials import PowerbiCertificate, PowerbiCredentials
|
|
6
8
|
from .endpoints import PowerBiEndpointFactory
|
|
7
9
|
|
|
8
10
|
|
|
11
|
+
def _get_client_credential(
|
|
12
|
+
secret: Optional[str], certificate: Optional[PowerbiCertificate]
|
|
13
|
+
) -> Union[str, dict]:
|
|
14
|
+
if secret:
|
|
15
|
+
return secret
|
|
16
|
+
if certificate:
|
|
17
|
+
return certificate.model_dump()
|
|
18
|
+
|
|
19
|
+
raise ValueError("Either certificate or secret must be provided.")
|
|
20
|
+
|
|
21
|
+
|
|
9
22
|
class PowerBiBearerAuth(BearerAuth):
|
|
10
23
|
def __init__(self, credentials: PowerbiCredentials):
|
|
11
24
|
self.credentials = credentials
|
|
@@ -14,10 +27,15 @@ class PowerBiBearerAuth(BearerAuth):
|
|
|
14
27
|
api_base=self.credentials.api_base,
|
|
15
28
|
)
|
|
16
29
|
authority = endpoint_factory.authority(self.credentials.tenant_id)
|
|
30
|
+
|
|
31
|
+
client_credential = _get_client_credential(
|
|
32
|
+
self.credentials.secret, self.credentials.certificate
|
|
33
|
+
)
|
|
34
|
+
|
|
17
35
|
self.app = msal.ConfidentialClientApplication(
|
|
18
36
|
client_id=self.credentials.client_id,
|
|
19
37
|
authority=authority,
|
|
20
|
-
client_credential=
|
|
38
|
+
client_credential=client_credential,
|
|
21
39
|
)
|
|
22
40
|
|
|
23
41
|
def fetch_token(self):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
-
from pydantic import
|
|
3
|
+
from pydantic import BaseModel, field_validator
|
|
4
4
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
5
5
|
|
|
6
6
|
DEFAULT_SCOPE = "https://analysis.windows.net/powerbi/api/.default"
|
|
@@ -10,6 +10,20 @@ CLIENT_APP_BASE = "https://login.microsoftonline.com"
|
|
|
10
10
|
REST_API_BASE_PATH = "https://api.powerbi.com/v1.0/myorg"
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
class PowerbiCertificate(BaseModel):
|
|
14
|
+
"""
|
|
15
|
+
Supports all dict credentials formats supported by PowerBI
|
|
16
|
+
https://learn.microsoft.com/en-us/python/api/msal/msal.application.confidentialclientapplication
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
client_assertion: Optional[str] = None
|
|
20
|
+
passphrase: Optional[str] = None
|
|
21
|
+
private_key: Optional[str] = None
|
|
22
|
+
private_key_pfx_path: Optional[str] = None
|
|
23
|
+
public_certificate: Optional[str] = None
|
|
24
|
+
thumbprint: Optional[str] = None
|
|
25
|
+
|
|
26
|
+
|
|
13
27
|
class PowerbiCredentials(BaseSettings):
|
|
14
28
|
"""Class to handle PowerBI rest API permissions"""
|
|
15
29
|
|
|
@@ -21,7 +35,8 @@ class PowerbiCredentials(BaseSettings):
|
|
|
21
35
|
|
|
22
36
|
client_id: str
|
|
23
37
|
tenant_id: str
|
|
24
|
-
secret: str =
|
|
38
|
+
secret: Optional[str] = None
|
|
39
|
+
certificate: Optional[PowerbiCertificate] = None
|
|
25
40
|
api_base: str = REST_API_BASE_PATH
|
|
26
41
|
login_url: str = CLIENT_APP_BASE
|
|
27
42
|
scopes: list[str] = [DEFAULT_SCOPE]
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import logging
|
|
2
3
|
from collections.abc import Iterable
|
|
3
|
-
from typing import Union
|
|
4
|
+
from typing import Optional, Union
|
|
4
5
|
|
|
5
6
|
from ...utils import (
|
|
6
7
|
OUTPUT_DIR,
|
|
@@ -12,11 +13,22 @@ from ...utils import (
|
|
|
12
13
|
write_summary,
|
|
13
14
|
)
|
|
14
15
|
from .assets import PowerBiAsset
|
|
15
|
-
from .client import PowerbiClient, PowerbiCredentials
|
|
16
|
+
from .client import PowerbiCertificate, PowerbiClient, PowerbiCredentials
|
|
16
17
|
|
|
17
18
|
logger = logging.getLogger(__name__)
|
|
18
19
|
|
|
19
20
|
|
|
21
|
+
def _load_certificate(
|
|
22
|
+
certificate: Optional[str],
|
|
23
|
+
) -> Optional[PowerbiCertificate]:
|
|
24
|
+
if not certificate:
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
with open(certificate) as file:
|
|
28
|
+
cert = json.load(file)
|
|
29
|
+
return PowerbiCertificate(**cert)
|
|
30
|
+
|
|
31
|
+
|
|
20
32
|
def iterate_all_data(
|
|
21
33
|
client: PowerbiClient,
|
|
22
34
|
) -> Iterable[tuple[PowerBiAsset, Union[list, dict]]]:
|
|
@@ -36,7 +48,15 @@ def extract_all(**kwargs) -> None:
|
|
|
36
48
|
Store the output files locally under the given output_directory
|
|
37
49
|
"""
|
|
38
50
|
_output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
|
|
39
|
-
creds = PowerbiCredentials(
|
|
51
|
+
creds = PowerbiCredentials(
|
|
52
|
+
client_id=kwargs.get("client_id"),
|
|
53
|
+
tenant_id=kwargs.get("tenant_id"),
|
|
54
|
+
secret=kwargs.get("secret"),
|
|
55
|
+
certificate=_load_certificate(kwargs.get("certificate")),
|
|
56
|
+
api_base=kwargs.get("api_base"),
|
|
57
|
+
login_url=kwargs.get("login_url"),
|
|
58
|
+
scopes=kwargs.get("scopes"),
|
|
59
|
+
)
|
|
40
60
|
client = PowerbiClient(creds)
|
|
41
61
|
ts = current_timestamp()
|
|
42
62
|
|
|
@@ -4,6 +4,8 @@ from functools import partial
|
|
|
4
4
|
from http import HTTPStatus
|
|
5
5
|
from typing import Callable, Iterable, Optional
|
|
6
6
|
|
|
7
|
+
from requests import ReadTimeout
|
|
8
|
+
|
|
7
9
|
from ....utils import (
|
|
8
10
|
APIClient,
|
|
9
11
|
RequestSafeMode,
|
|
@@ -114,6 +116,31 @@ class SigmaClient(APIClient):
|
|
|
114
116
|
request = self._get_paginated(endpoint=SigmaEndpointFactory.workbooks())
|
|
115
117
|
yield from fetch_all_pages(request, SigmaPagination)
|
|
116
118
|
|
|
119
|
+
@staticmethod
|
|
120
|
+
def _safe_fetch_elements(
|
|
121
|
+
elements: Iterator[dict],
|
|
122
|
+
workbook_id: str,
|
|
123
|
+
page_id: str,
|
|
124
|
+
) -> Iterator[dict]:
|
|
125
|
+
"""
|
|
126
|
+
Safely iterates over elements with ReadTimeout handling. In case of
|
|
127
|
+
said error, it skips the entire rest of the page.
|
|
128
|
+
"""
|
|
129
|
+
try:
|
|
130
|
+
for element in elements:
|
|
131
|
+
if element.get("type") not in _DATA_ELEMENTS:
|
|
132
|
+
continue
|
|
133
|
+
yield {
|
|
134
|
+
**element,
|
|
135
|
+
"workbook_id": workbook_id,
|
|
136
|
+
"page_id": page_id,
|
|
137
|
+
}
|
|
138
|
+
except ReadTimeout:
|
|
139
|
+
logger.warning(
|
|
140
|
+
f"ReadTimeout for page {page_id} in workbook {workbook_id}"
|
|
141
|
+
)
|
|
142
|
+
return
|
|
143
|
+
|
|
117
144
|
def _get_elements_per_page(
|
|
118
145
|
self, page: dict, workbook_id: str
|
|
119
146
|
) -> Iterator[dict]:
|
|
@@ -122,14 +149,7 @@ class SigmaClient(APIClient):
|
|
|
122
149
|
SigmaEndpointFactory.elements(workbook_id, page_id)
|
|
123
150
|
)
|
|
124
151
|
elements = fetch_all_pages(request, SigmaPagination)
|
|
125
|
-
|
|
126
|
-
if element.get("type") not in _DATA_ELEMENTS:
|
|
127
|
-
continue
|
|
128
|
-
yield {
|
|
129
|
-
**element,
|
|
130
|
-
"workbook_id": workbook_id,
|
|
131
|
-
"page_id": page_id,
|
|
132
|
-
}
|
|
152
|
+
yield from self._safe_fetch_elements(elements, workbook_id, page_id)
|
|
133
153
|
|
|
134
154
|
def _get_all_elements(self, workbooks: list[dict]) -> Iterator[dict]:
|
|
135
155
|
for workbook in workbooks:
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from http import HTTPStatus
|
|
3
|
+
from time import sleep
|
|
3
4
|
from typing import TYPE_CHECKING, Callable, Iterator
|
|
4
5
|
|
|
5
6
|
from ....utils import fetch_all_pages, retry_request
|
|
@@ -13,6 +14,7 @@ logger = logging.getLogger(__name__)
|
|
|
13
14
|
|
|
14
15
|
SIGMA_CONNECTION_PATH_MAX_RETRY = 1
|
|
15
16
|
SIGMA_CONNECTION_PATH_SLEEP_MS = 30_000 # 30 seconds
|
|
17
|
+
SIGMA_SOURCES_SLEEP_MS = 2 / 1000 # 0.2 seconds
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
class SigmaSourcesTransformer:
|
|
@@ -84,6 +86,29 @@ class SigmaSourcesTransformer:
|
|
|
84
86
|
"sources": enhanced_sources,
|
|
85
87
|
}
|
|
86
88
|
|
|
89
|
+
def _fetch_asset_sources(
|
|
90
|
+
self,
|
|
91
|
+
asset_id: str,
|
|
92
|
+
endpoint: Callable[[str], str],
|
|
93
|
+
with_pagination: bool,
|
|
94
|
+
) -> dict:
|
|
95
|
+
"""Fetches sources for a single asset."""
|
|
96
|
+
endpoint_url = endpoint(asset_id)
|
|
97
|
+
|
|
98
|
+
if with_pagination:
|
|
99
|
+
request = self.api_client._get_paginated(endpoint=endpoint_url)
|
|
100
|
+
pages_generator = fetch_all_pages(
|
|
101
|
+
request=request,
|
|
102
|
+
pagination_model=SigmaTokenPagination,
|
|
103
|
+
rate_limit=SIGMA_SOURCES_SLEEP_MS,
|
|
104
|
+
)
|
|
105
|
+
sources = list(pages_generator)
|
|
106
|
+
else:
|
|
107
|
+
sources = self.api_client._get(endpoint=endpoint_url)
|
|
108
|
+
sleep(SIGMA_SOURCES_SLEEP_MS)
|
|
109
|
+
|
|
110
|
+
return {"asset_id": asset_id, "sources": sources}
|
|
111
|
+
|
|
87
112
|
def _get_all_sources(
|
|
88
113
|
self,
|
|
89
114
|
endpoint: Callable[[str], str],
|
|
@@ -91,16 +116,10 @@ class SigmaSourcesTransformer:
|
|
|
91
116
|
with_pagination: bool = False,
|
|
92
117
|
) -> Iterator[dict]:
|
|
93
118
|
"""Returns transformed sources for the given assets"""
|
|
94
|
-
all_sources = [
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
if with_pagination:
|
|
99
|
-
request = self.api_client._get_paginated(endpoint=endpoint_url)
|
|
100
|
-
sources = list(fetch_all_pages(request, SigmaTokenPagination))
|
|
101
|
-
else:
|
|
102
|
-
sources = self.api_client._get(endpoint=endpoint_url)
|
|
103
|
-
all_sources.append({"asset_id": asset_id, "sources": sources})
|
|
119
|
+
all_sources = [
|
|
120
|
+
self._fetch_asset_sources(asset_id, endpoint, with_pagination)
|
|
121
|
+
for asset_id in asset_ids
|
|
122
|
+
]
|
|
104
123
|
|
|
105
124
|
table_to_path = self._map_table_id_to_connection_path(all_sources)
|
|
106
125
|
|
|
@@ -51,6 +51,7 @@ class SQLExtractionProcessor:
|
|
|
51
51
|
def _fetch(self, query: ExtractionQuery) -> Iterator[dict]:
|
|
52
52
|
default: Callable[[], Iterator] = lambda: iter(()) # type: ignore
|
|
53
53
|
decorator = safe_mode(self._safe_mode, default)
|
|
54
|
+
self._client.set_database(query.database)
|
|
54
55
|
decorated_execute = decorator(self._client.execute)
|
|
55
56
|
return decorated_execute(query)
|
|
56
57
|
|
|
@@ -21,11 +21,22 @@ class AssetNotSupportedError(NotImplementedError):
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class ExtractionQuery:
|
|
24
|
-
"""
|
|
24
|
+
"""
|
|
25
|
+
Contains useful context to run the query:
|
|
26
|
+
- the sql statement itself
|
|
27
|
+
- parameters { ... }
|
|
28
|
+
- optionally, the target database (can be used to change the engine's URI)
|
|
29
|
+
"""
|
|
25
30
|
|
|
26
|
-
def __init__(
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
statement: str,
|
|
34
|
+
params: dict,
|
|
35
|
+
database: Optional[str] = None,
|
|
36
|
+
):
|
|
27
37
|
self.statement = statement
|
|
28
38
|
self.params = params
|
|
39
|
+
self.database = database
|
|
29
40
|
|
|
30
41
|
|
|
31
42
|
class AbstractQueryBuilder(ABC):
|
|
@@ -24,12 +24,7 @@ SNOWFLAKE_ASSETS: SupportedAssets = {
|
|
|
24
24
|
WarehouseAssetGroup.FUNCTION: FUNCTIONS_ASSETS,
|
|
25
25
|
WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
|
|
26
26
|
WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
|
|
27
|
-
WarehouseAssetGroup.ROLE: (
|
|
28
|
-
WarehouseAsset.GRANT_TO_ROLE,
|
|
29
|
-
WarehouseAsset.GRANT_TO_USER,
|
|
30
|
-
WarehouseAsset.ROLE,
|
|
31
|
-
WarehouseAsset.USER,
|
|
32
|
-
),
|
|
27
|
+
WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
|
|
33
28
|
WarehouseAssetGroup.SNOWFLAKE_LINEAGE: (WarehouseAsset.COLUMN_LINEAGE,),
|
|
34
29
|
WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
|
|
35
30
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from collections.abc import Iterator
|
|
3
|
+
from typing import Optional
|
|
3
4
|
|
|
4
|
-
from sqlalchemy import text
|
|
5
|
+
from sqlalchemy import create_engine, text
|
|
5
6
|
|
|
6
7
|
from ...utils import ExtractionQuery, SqlalchemyClient, uri_encode
|
|
7
8
|
|
|
@@ -63,6 +64,18 @@ class MSSQLClient(SqlalchemyClient):
|
|
|
63
64
|
finally:
|
|
64
65
|
self.close()
|
|
65
66
|
|
|
67
|
+
def set_database(self, database: Optional[str]) -> None:
|
|
68
|
+
"""
|
|
69
|
+
To support SQL Server running on Azure, we must change the engine's
|
|
70
|
+
URI for each database-scoped query
|
|
71
|
+
https://chatgpt.com/share/68de93c3-9550-8001-9d54-c5da86faa43c
|
|
72
|
+
"""
|
|
73
|
+
if not database:
|
|
74
|
+
return
|
|
75
|
+
self.close()
|
|
76
|
+
database_uri = f"{self._uri}/{database}"
|
|
77
|
+
self._engine = create_engine(database_uri, **self._options)
|
|
78
|
+
|
|
66
79
|
def get_databases(self) -> list[str]:
|
|
67
80
|
result = self.execute(
|
|
68
81
|
ExtractionQuery("SELECT name FROM sys.databases", {})
|
|
@@ -40,10 +40,14 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
|
|
|
40
40
|
self._databases = databases
|
|
41
41
|
|
|
42
42
|
@staticmethod
|
|
43
|
-
def _format(
|
|
43
|
+
def _format(
|
|
44
|
+
query: ExtractionQuery,
|
|
45
|
+
database: str,
|
|
46
|
+
) -> ExtractionQuery:
|
|
44
47
|
return ExtractionQuery(
|
|
45
|
-
statement=query.statement.format(
|
|
48
|
+
statement=query.statement.format(database=database),
|
|
46
49
|
params=query.params,
|
|
50
|
+
database=database,
|
|
47
51
|
)
|
|
48
52
|
|
|
49
53
|
def build(self, asset: WarehouseAsset) -> list[ExtractionQuery]:
|
|
@@ -58,7 +62,4 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
|
|
|
58
62
|
logger.info(
|
|
59
63
|
f"\tWill run queries with following database params: {self._databases}",
|
|
60
64
|
)
|
|
61
|
-
return [
|
|
62
|
-
self._format(query, {"database": database})
|
|
63
|
-
for database in self._databases
|
|
64
|
-
]
|
|
65
|
+
return [self._format(query, database) for database in self._databases]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: castor-extractor
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.25.6
|
|
4
4
|
Summary: Extract your metadata assets.
|
|
5
5
|
Home-page: https://www.castordoc.com/
|
|
6
6
|
License: EULA
|
|
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
17
|
Provides-Extra: all
|
|
18
18
|
Provides-Extra: bigquery
|
|
19
|
+
Provides-Extra: count
|
|
19
20
|
Provides-Extra: databricks
|
|
20
21
|
Provides-Extra: dbt
|
|
21
22
|
Provides-Extra: looker
|
|
@@ -57,7 +58,7 @@ Requires-Dist: setuptools (>=78.1)
|
|
|
57
58
|
Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
|
|
58
59
|
Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
|
|
59
60
|
Requires-Dist: sqlalchemy (>=1.4,<1.5)
|
|
60
|
-
Requires-Dist: sqlalchemy-bigquery[bqstorage] (>=1.0.0,<=2.0.0) ; extra == "bigquery" or extra == "all"
|
|
61
|
+
Requires-Dist: sqlalchemy-bigquery[bqstorage] (>=1.0.0,<=2.0.0) ; extra == "bigquery" or extra == "count" or extra == "all"
|
|
61
62
|
Requires-Dist: sqlalchemy-redshift (>=0.8.14,<0.9.0) ; extra == "redshift" or extra == "all"
|
|
62
63
|
Requires-Dist: tableauserverclient (>=0.25.0,<0.26.0) ; extra == "tableau" or extra == "all"
|
|
63
64
|
Requires-Dist: tqdm (>=4.0.0,<5.0.0)
|
|
@@ -215,6 +216,34 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
|
|
|
215
216
|
|
|
216
217
|
# Changelog
|
|
217
218
|
|
|
219
|
+
## 0.25.6 - 2025-10-06
|
|
220
|
+
|
|
221
|
+
* PowerBi: Support additional authentication methods
|
|
222
|
+
|
|
223
|
+
## 0.25.5 - 2025-10-06
|
|
224
|
+
|
|
225
|
+
* Snowflake: remove extraction of roles and grants
|
|
226
|
+
|
|
227
|
+
## 0.25.4 - 2025-10-02
|
|
228
|
+
|
|
229
|
+
* SqlServer: switch database to support clients running their instance on Azure
|
|
230
|
+
|
|
231
|
+
## 0.25.3 - 2025-10-01
|
|
232
|
+
|
|
233
|
+
* Sigma: sleep between sources requests to avoid rate limit errors
|
|
234
|
+
|
|
235
|
+
## 0.25.2 - 2025-09-30
|
|
236
|
+
|
|
237
|
+
* PowerBi: Support auth with private_key
|
|
238
|
+
|
|
239
|
+
## 0.25.1 - 2025-09-29
|
|
240
|
+
|
|
241
|
+
* Sigma: catch ReadTimeouts during elements extraction
|
|
242
|
+
|
|
243
|
+
## 0.25.0 - 2025-09-15
|
|
244
|
+
|
|
245
|
+
* Count: adding connector
|
|
246
|
+
|
|
218
247
|
## 0.24.57 - 2025-09-24
|
|
219
248
|
|
|
220
249
|
* Sigma:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
CHANGELOG.md,sha256
|
|
1
|
+
CHANGELOG.md,sha256=_zH2Mm-Bn_xdVUaseScAbBos3i0Ae_MspceXsG-5vjc,21437
|
|
2
2
|
Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
|
|
3
3
|
DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
|
|
4
4
|
LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
@@ -7,6 +7,7 @@ castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
|
|
|
7
7
|
castor_extractor/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
castor_extractor/commands/extract_bigquery.py,sha256=dU4OiYO1V0n32orvZnMh1_xtFKF_VxHNXcVsH3otY-g,1269
|
|
9
9
|
castor_extractor/commands/extract_confluence.py,sha256=blYcnDqywXNKRQ1aZAD9FclhLlO7x8Y_tb0lgl85v0w,1641
|
|
10
|
+
castor_extractor/commands/extract_count.py,sha256=cITp-2UmPYjbcICvYZzxE9oWieI8NbTH1DcWxLAZxJ4,611
|
|
10
11
|
castor_extractor/commands/extract_databricks.py,sha256=SVKyoa-BBUQAM6HRHf1Wdg9-tpICic2yyvXQwHcNBhA,1264
|
|
11
12
|
castor_extractor/commands/extract_domo.py,sha256=jvAawUsUTHrwCn_koK6StmQr4n_b5GyvJi6uu6WS0SM,1061
|
|
12
13
|
castor_extractor/commands/extract_looker.py,sha256=cySLiolLCgrREJ9d0kMrJ7P8K3efHTBTzShalWVfI3A,1214
|
|
@@ -17,7 +18,7 @@ castor_extractor/commands/extract_mode.py,sha256=Q4iO-VAKMg4zFPejhAO-foZibL5Ht3j
|
|
|
17
18
|
castor_extractor/commands/extract_mysql.py,sha256=7AH5qMzeLTsENCOeJwtesrWg8Vo8MCEq8fx2YT74Mcw,1034
|
|
18
19
|
castor_extractor/commands/extract_notion.py,sha256=uaxcF3_bT7D_-JxnIW0F7VVDphI_ZgOfQQxZzoLXo_M,504
|
|
19
20
|
castor_extractor/commands/extract_postgres.py,sha256=pX0RnCPi4nw6QQ6wiAuZ_Xt3ZbDuMUG9aQKuqFgJtAU,1154
|
|
20
|
-
castor_extractor/commands/extract_powerbi.py,sha256=
|
|
21
|
+
castor_extractor/commands/extract_powerbi.py,sha256=4q_brV5hEG8RINCquvZCtQ9G2v4nzqQClIUlP0Pdef4,1402
|
|
21
22
|
castor_extractor/commands/extract_qlik.py,sha256=VBe_xFKh_nR0QSFFIncAaC8yDqBeMa6VunBAga7AeGg,891
|
|
22
23
|
castor_extractor/commands/extract_redshift.py,sha256=zRBg2D_ft4GLdPSdmetRcgQVAA80DXtdRSYsQhAWIik,1334
|
|
23
24
|
castor_extractor/commands/extract_salesforce.py,sha256=3j3YTmMkPAwocR-B1ozJQai0UIZPtpmAyWj-hHvdWn4,1226
|
|
@@ -101,13 +102,13 @@ castor_extractor/utils/argument_parser_test.py,sha256=wnyLFJ74iEiPxxLSbwFtckR7FI
|
|
|
101
102
|
castor_extractor/utils/batch.py,sha256=SFlLmJgVjV2nVhIrjVIEp8wJ9du4dKKHq8YVYubnwQQ,448
|
|
102
103
|
castor_extractor/utils/batch_test.py,sha256=84JYXOxiTkZFAceVh0mzN6VtKxcqoFPbxkZfIDyLGlg,606
|
|
103
104
|
castor_extractor/utils/client/__init__.py,sha256=h5gm8UNNCCkAqhjYK5f6BY7k0cHFOyAvkmlktqwpir0,392
|
|
104
|
-
castor_extractor/utils/client/abstract.py,sha256=
|
|
105
|
+
castor_extractor/utils/client/abstract.py,sha256=OyTpb_SBgxaZd-4QFUV3n7yTzszAbyxY89fHsdAGKS8,2410
|
|
105
106
|
castor_extractor/utils/client/api/__init__.py,sha256=vlG7WXznYgLTn3XyMGsyUkgRkup8FbKM14EXJ8mv-b0,264
|
|
106
107
|
castor_extractor/utils/client/api/auth.py,sha256=lq0K3UEl1vwIIa_vKTdlpIQPdE5K1-5DXmCwO4dKzng,1890
|
|
107
108
|
castor_extractor/utils/client/api/auth_test.py,sha256=LlyXytnatg6ZzR4Zkvzk0BH99FYhHX7qn_nyr2MSnDI,1305
|
|
108
109
|
castor_extractor/utils/client/api/client.py,sha256=qmj7KoNqt6F-cmpdaMiz_aVxzwMCgbDNcgzXSbCdu1Y,5183
|
|
109
110
|
castor_extractor/utils/client/api/client_test.py,sha256=FM3ZxsLLfMOBn44cXX6FIgnA31-5TTNIyp9D4LBwtXE,1222
|
|
110
|
-
castor_extractor/utils/client/api/pagination.py,sha256=
|
|
111
|
+
castor_extractor/utils/client/api/pagination.py,sha256=SFO841ItRxPrQzhusx9JZiX5V394VujaB69YQJN1z2M,2501
|
|
111
112
|
castor_extractor/utils/client/api/pagination_test.py,sha256=jCOgXFXrH-jrCxe2dfk80ZksJF-EtmpJPU11BGabsqk,1385
|
|
112
113
|
castor_extractor/utils/client/api/safe_request.py,sha256=5pvI2WPRDtitX9F1aYcXTIMPNmDikRK9dKTD3ctoeoQ,1774
|
|
113
114
|
castor_extractor/utils/client/api/safe_request_test.py,sha256=LqS5FBxs6lLLcTkcgxIoLb6OinxShHXR5y4CWZpwmwg,2005
|
|
@@ -160,6 +161,17 @@ castor_extractor/utils/validation.py,sha256=dRvC9SoFVecVZuLQNN3URq37yX2sBSW3-NxI
|
|
|
160
161
|
castor_extractor/utils/validation_test.py,sha256=A7P6VmI0kYX2aGIeEN12y7LsY7Kpm8pE4bdVFhbBAMw,1184
|
|
161
162
|
castor_extractor/utils/write.py,sha256=KQVWF29N766avzmSb129IUWrId5c_8BtnYhVLmU6YIs,2133
|
|
162
163
|
castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
164
|
+
castor_extractor/visualization/count/__init__.py,sha256=lvxGtSe3erjTYK0aPnkOyJibcsC6Q1AFchnK-hZt558,114
|
|
165
|
+
castor_extractor/visualization/count/assets.py,sha256=VZCRVDKWSu6l2lVGJS4JKOOmfCUkbS8MnJiLcAY9vqw,232
|
|
166
|
+
castor_extractor/visualization/count/client/__init__.py,sha256=YawYDutDI0sprp72jN9tKi8bbXCoc0Ij0Ev582tKjqk,74
|
|
167
|
+
castor_extractor/visualization/count/client/client.py,sha256=WgljCj8G7D0Brxa0llaeOQ2Ipd7FvtDWFoLWoPyqT9A,1523
|
|
168
|
+
castor_extractor/visualization/count/client/credentials.py,sha256=LZWvcz7p5lrgdgoIQLcxFyv4gqUBW4Jj4qDKN-VW31I,273
|
|
169
|
+
castor_extractor/visualization/count/client/queries/canvas_permissions.sql,sha256=iFmMfR0zusjxTxmYUS6p0kibZCsnHOQMbAlxaNjx-H4,108
|
|
170
|
+
castor_extractor/visualization/count/client/queries/canvases.sql,sha256=Ur5HBD9JJH0r14xIj_rwoctnds082_F931vlfcnwi_I,86
|
|
171
|
+
castor_extractor/visualization/count/client/queries/cells.sql,sha256=Kkk0jyU337PD6RPshSo_ucLl5PS7kIvJZlUnVnmJUkM,111
|
|
172
|
+
castor_extractor/visualization/count/client/queries/projects.sql,sha256=3Jem3QCVwk4wHiWRJL7cN6Vl2Yc5RZ8yC8ndvPAkaFM,68
|
|
173
|
+
castor_extractor/visualization/count/client/queries/users.sql,sha256=H0n7S7P5cCAWbgPxU32psIc1epXySzsAaQ7MQ9JrkfM,102
|
|
174
|
+
castor_extractor/visualization/count/extract.py,sha256=ZBsJ9tMxxaq1jG8qJp_OGVK3yPDNkVUsP1_3rcUMtYg,1378
|
|
163
175
|
castor_extractor/visualization/domo/__init__.py,sha256=1axOCPm4RpdIyUt9LQEvlMvbOPllW8rk63h6EjVgJ0Y,111
|
|
164
176
|
castor_extractor/visualization/domo/assets.py,sha256=bK1urFR2tnlWkVkkhR32mAKMoKbESNlop-CNGx-65PY,206
|
|
165
177
|
castor_extractor/visualization/domo/client/__init__.py,sha256=Do0fU4B8Hhlhahcv734gnJl_ryCztfTBDea7XNCKfB8,72
|
|
@@ -236,16 +248,16 @@ castor_extractor/visualization/mode/errors.py,sha256=SKpFT2AiLOuWx2VRLyO7jbAiKcG
|
|
|
236
248
|
castor_extractor/visualization/mode/extract.py,sha256=PmLWWjUwplQh3TNMemiGwyFdxMcKVMvumZPxSMLJAwk,1625
|
|
237
249
|
castor_extractor/visualization/powerbi/__init__.py,sha256=hoZ73ngLhMc9edqxO9PUIE3FABQlvcfY2W8fuc6DEjY,197
|
|
238
250
|
castor_extractor/visualization/powerbi/assets.py,sha256=IB_XKwgdN1pZYGZ4RfeHrLjflianTzWf_6tg-4CIwu0,742
|
|
239
|
-
castor_extractor/visualization/powerbi/client/__init__.py,sha256=
|
|
240
|
-
castor_extractor/visualization/powerbi/client/authentication.py,sha256=
|
|
251
|
+
castor_extractor/visualization/powerbi/client/__init__.py,sha256=rxWeAtmGsy1XYn2oIrGz5rIlxcTrzh2rl1V-MGxFOY4,175
|
|
252
|
+
castor_extractor/visualization/powerbi/client/authentication.py,sha256=1pST-w7ceqrcKSccQSJBxT4lAsLU8keceSVJro1dg8k,1516
|
|
241
253
|
castor_extractor/visualization/powerbi/client/client.py,sha256=Q_WHYGFpHT4wJ6nZvJa96nBVcpUGv7E2WnyZHBftsJM,8340
|
|
242
254
|
castor_extractor/visualization/powerbi/client/client_test.py,sha256=zWgfc8fOHSRn3hxiX8ujJysmNHeypIoKin9h8_h178k,6668
|
|
243
255
|
castor_extractor/visualization/powerbi/client/constants.py,sha256=88R_aGachNNUZh6OSH2fkDwZtY4KTStzKm_g7HNCqqo,387
|
|
244
|
-
castor_extractor/visualization/powerbi/client/credentials.py,sha256=
|
|
256
|
+
castor_extractor/visualization/powerbi/client/credentials.py,sha256=Z6wJkevG3MvqTK54p3lxKnw7aUZ1mxPZqwuG44cOB_Q,1907
|
|
245
257
|
castor_extractor/visualization/powerbi/client/credentials_test.py,sha256=TzFqxsWVQ3sXR_n0bJsexK9Uz7ceXCEPVqDGWTJzW60,993
|
|
246
258
|
castor_extractor/visualization/powerbi/client/endpoints.py,sha256=38ZETzSSnNq3vA9O6nLZQ8T1BVE01R9CjMC03-PRXsM,1911
|
|
247
259
|
castor_extractor/visualization/powerbi/client/pagination.py,sha256=OZMjoDQPRGMoWd9QcKKrPh3aErJR20SHlrTqY_siLkk,755
|
|
248
|
-
castor_extractor/visualization/powerbi/extract.py,sha256=
|
|
260
|
+
castor_extractor/visualization/powerbi/extract.py,sha256=bZOUbciWGPNRRrtcMezSdoeClHB2yiBATBC8UqoXz5M,1904
|
|
249
261
|
castor_extractor/visualization/qlik/__init__.py,sha256=u6lIfm_WOykBwt6SlaB7C0Dtx37XBliUbM5oWv26gC8,177
|
|
250
262
|
castor_extractor/visualization/qlik/assets.py,sha256=Ab_kG61mHcK8GoGZbfQW7RSWyd7D9bVga9DOqnm0iSE,1625
|
|
251
263
|
castor_extractor/visualization/qlik/client/__init__.py,sha256=5O5N9Jrt3d99agFEJ28lKWs2KkDaXK-lZ07IUtLj56M,130
|
|
@@ -273,12 +285,12 @@ castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13
|
|
|
273
285
|
castor_extractor/visualization/sigma/assets.py,sha256=iVZqi7XtNgSOVXy0jgeHZonVOeXi7jyikor8ztbECBc,398
|
|
274
286
|
castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
|
|
275
287
|
castor_extractor/visualization/sigma/client/authentication.py,sha256=gHukrpfboIjZc_O9CcuDtrl6U-StH0J73VY2J74Bm9o,2279
|
|
276
|
-
castor_extractor/visualization/sigma/client/client.py,sha256=
|
|
288
|
+
castor_extractor/visualization/sigma/client/client.py,sha256=SxSf5OjdDr8x-WZDezm8YNOw01R6CCoYIgW0od0ZgN8,8907
|
|
277
289
|
castor_extractor/visualization/sigma/client/client_test.py,sha256=ae0ZOvKutCm44jnrJ-0_A5Y6ZGyDkMf9Ml3eEP8dNkY,581
|
|
278
290
|
castor_extractor/visualization/sigma/client/credentials.py,sha256=XddAuQSmCKpxJ70TQgRnOj0vMPYVtiStk_lMMQ1AiNM,693
|
|
279
291
|
castor_extractor/visualization/sigma/client/endpoints.py,sha256=by9VIFml2whlzQT66f2m56RYBsqPrWdAmIP4JkTaBV4,1799
|
|
280
292
|
castor_extractor/visualization/sigma/client/pagination.py,sha256=9kCYQpO7hAH2qvYmnVjnGVUDLkpkEM6BgYlv-JTY8AE,1241
|
|
281
|
-
castor_extractor/visualization/sigma/client/sources_transformer.py,sha256=
|
|
293
|
+
castor_extractor/visualization/sigma/client/sources_transformer.py,sha256=ofjBVxeaTDnawB_PoblLQ2Q_b5FdbqPEHk8arVEyx_o,5086
|
|
282
294
|
castor_extractor/visualization/sigma/client/sources_transformer_test.py,sha256=06yUHXyv65amXLKXhix6K3kkVc1kpBqSjIYcxbyMI4Y,2766
|
|
283
295
|
castor_extractor/visualization/sigma/extract.py,sha256=iRmRUzSnq_ObG9fxpOI5Rs07EKKT-VRLcyiti5-8D4c,2986
|
|
284
296
|
castor_extractor/visualization/strategy/__init__.py,sha256=HOMv4JxqF5ZmViWi-pDE-PSXJRLTdXal_jtpHG_rlR8,123
|
|
@@ -314,8 +326,8 @@ castor_extractor/warehouse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
|
|
|
314
326
|
castor_extractor/warehouse/abstract/__init__.py,sha256=Fdfa026tgOo64MvzVRLHM_F2G-JmcehrF0mh3dHgb7s,419
|
|
315
327
|
castor_extractor/warehouse/abstract/asset.py,sha256=wR5mJxAHBcqJ86HRb_Y8x3mDN4uUgSg8jMToLNu0jTM,2740
|
|
316
328
|
castor_extractor/warehouse/abstract/asset_test.py,sha256=_kd4ybNlWSAdSdEgJKC-jhJTa1nMRa9i8RO3YbqKLM4,758
|
|
317
|
-
castor_extractor/warehouse/abstract/extract.py,sha256=
|
|
318
|
-
castor_extractor/warehouse/abstract/query.py,sha256=
|
|
329
|
+
castor_extractor/warehouse/abstract/extract.py,sha256=T_GwXO4_IFSVdBuMmnKvsftl549y43nzE5uN4He4G80,2974
|
|
330
|
+
castor_extractor/warehouse/abstract/query.py,sha256=MSOou0F5BlTths1TsAxv9tcyOB013aJc1TegN4q7jfM,2852
|
|
319
331
|
castor_extractor/warehouse/abstract/time_filter.py,sha256=bggIONfMmUxffkA6TwM3BsjfS2l9WFxPq8krfsau5pw,935
|
|
320
332
|
castor_extractor/warehouse/abstract/time_filter_test.py,sha256=PIkegB7KOKBdpc6zIvmyl_CeQyADeFDplyQ8HTNU5LA,448
|
|
321
333
|
castor_extractor/warehouse/bigquery/__init__.py,sha256=PCGNYdi7dHv-SyanUWzRuBp-ypuQ01PkDaQjVnaNhbM,170
|
|
@@ -406,23 +418,20 @@ castor_extractor/warehouse/snowflake/client.py,sha256=RB72bbl_k91wDU76yrggPK6oeE
|
|
|
406
418
|
castor_extractor/warehouse/snowflake/client_test.py,sha256=ihWtOOAQfh8pu5JTr_EWfqefKOVIaJXznACURzaU1Qs,1432
|
|
407
419
|
castor_extractor/warehouse/snowflake/credentials.py,sha256=u0sZ6xPtcZmmvnUsAejJk-YxGl8BTzX_BlRjRk92BYU,932
|
|
408
420
|
castor_extractor/warehouse/snowflake/credentials_test.py,sha256=Lkc-DHXOvr50KrqAW4nt_x0IA0Mu_CsBVu6ATnzQB6I,673
|
|
409
|
-
castor_extractor/warehouse/snowflake/extract.py,sha256=
|
|
421
|
+
castor_extractor/warehouse/snowflake/extract.py,sha256=eGtIqW5kKJl-e36viqYQzkXn39CJkmMBR4oSt-B0ud4,3082
|
|
410
422
|
castor_extractor/warehouse/snowflake/queries/.sqlfluff,sha256=vttrwcr64JVIuvc7WIg9C54cbOkjg_VjXNR7YnTGOPE,31
|
|
411
423
|
castor_extractor/warehouse/snowflake/queries/column.sql,sha256=Ru-yC0s76I9LehOA4aCZ--xz6D9H1Hyr3OZdILOBHAw,1882
|
|
412
424
|
castor_extractor/warehouse/snowflake/queries/column_lineage.sql,sha256=YKBiZ6zySSNcXLDXwm31EjGIIkkkZc0-S6hI1SRM80o,1179
|
|
413
425
|
castor_extractor/warehouse/snowflake/queries/database.sql,sha256=yxHzV2oQfW0bo2dJVqJkyZy-GruudTi6ObmLaJ505hE,559
|
|
414
426
|
castor_extractor/warehouse/snowflake/queries/function.sql,sha256=8LRh0ybhd-RldJ8UZspWUm3yv52evq11O2uqIO4KqeQ,372
|
|
415
|
-
castor_extractor/warehouse/snowflake/queries/grant_to_role.sql,sha256=O7AJ1LzoXGDFmiVvQ8EMJ5x8FSAnaxRPdmRyAlEmkUM,272
|
|
416
|
-
castor_extractor/warehouse/snowflake/queries/grant_to_user.sql,sha256=7AalVajU5vRRpIiys1igSwmDXirbwpMTvJr2ihSz2NE,143
|
|
417
427
|
castor_extractor/warehouse/snowflake/queries/query.sql,sha256=w4T6-TgwUozDgaF3Fk-qex7bDdEIHLkkB5XEe2VJXZQ,1992
|
|
418
|
-
castor_extractor/warehouse/snowflake/queries/role.sql,sha256=D0VvGxLZMwug2SvefhAsNR9YIun0fZvcDWkz891xSYM,96
|
|
419
428
|
castor_extractor/warehouse/snowflake/queries/schema.sql,sha256=iLn6_y5rn63KigjE4GEAMp8ZuZZofhMXYGb8saPDGUc,776
|
|
420
429
|
castor_extractor/warehouse/snowflake/queries/table.sql,sha256=CbSLfJAylyyyD3mkGPSLLE7BHrGjlY499kzO9RN0e4Y,1473
|
|
421
430
|
castor_extractor/warehouse/snowflake/queries/user.sql,sha256=88V8eRj1NDaD_ufclsKOHHlqCtBMQHOV54yy6RKJaXk,570
|
|
422
431
|
castor_extractor/warehouse/snowflake/queries/view_ddl.sql,sha256=eWsci_50cxiYIv3N7BKkbXVM3RoIzqSDtohqRnE5kg4,673
|
|
423
432
|
castor_extractor/warehouse/snowflake/query.py,sha256=C2LTdPwBzMQ_zMncg0Kq4_WkoY7K9as5tvxBDrIOlwI,1763
|
|
424
433
|
castor_extractor/warehouse/sqlserver/__init__.py,sha256=PdOuYznmvKAbfWAm8UdN47MfEsd9jqPi_dDi3WEo1KY,116
|
|
425
|
-
castor_extractor/warehouse/sqlserver/client.py,sha256=
|
|
434
|
+
castor_extractor/warehouse/sqlserver/client.py,sha256=umxTNZ92S2ImtxkPwzXdTzGvym-bjv5xSNK29vawL-8,5549
|
|
426
435
|
castor_extractor/warehouse/sqlserver/extract.py,sha256=HEJFDM1a4OeQH7OWhYhCOjhkHfGW6qf_qvjFAeMGPYg,2623
|
|
427
436
|
castor_extractor/warehouse/sqlserver/queries/.sqlfluff,sha256=yy0KQdz8I_67vnXyX8eeWwOWkxTXvHyVKSVwhURktd8,48
|
|
428
437
|
castor_extractor/warehouse/sqlserver/queries/column.sql,sha256=ojiUQQnHXdWMbgaYOcxKBiwfi7rtu_tyamK6r4t4IBM,2929
|
|
@@ -432,10 +441,10 @@ castor_extractor/warehouse/sqlserver/queries/schema.sql,sha256=Fq_8-tCnArayON3fj
|
|
|
432
441
|
castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=ggzatJOlOfGkMG1NS-hD-n1-3WLbV9Yh8IsQrEFO5X4,2831
|
|
433
442
|
castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=MAlnTis43E3Amu1e1Oz_qhaX8Bz-iN0Lrbf9RiohX7Y,99
|
|
434
443
|
castor_extractor/warehouse/sqlserver/queries/view_ddl.sql,sha256=9rynvx6MWg3iZzrWPB7haZfVKEPkxulzryE2g19x804,315
|
|
435
|
-
castor_extractor/warehouse/sqlserver/query.py,sha256=
|
|
444
|
+
castor_extractor/warehouse/sqlserver/query.py,sha256=gr5lnZSUm-wSYuVnJlg6fc7jXWirbL-sCiQN9RnAiPQ,1789
|
|
436
445
|
castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
|
|
437
|
-
castor_extractor-0.
|
|
438
|
-
castor_extractor-0.
|
|
439
|
-
castor_extractor-0.
|
|
440
|
-
castor_extractor-0.
|
|
441
|
-
castor_extractor-0.
|
|
446
|
+
castor_extractor-0.25.6.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
447
|
+
castor_extractor-0.25.6.dist-info/METADATA,sha256=ayaVswDaw-D2Zele6ewtVK5inUm9fBOuDik-AGY7yAY,28939
|
|
448
|
+
castor_extractor-0.25.6.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
449
|
+
castor_extractor-0.25.6.dist-info/entry_points.txt,sha256=qyTrKNByoq2HYi1xbA79OU7qxg-OWPvle8VwDqt-KnE,1869
|
|
450
|
+
castor_extractor-0.25.6.dist-info/RECORD,,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
[console_scripts]
|
|
2
2
|
castor-extract-bigquery=castor_extractor.commands.extract_bigquery:main
|
|
3
3
|
castor-extract-confluence=castor_extractor.commands.extract_confluence:main
|
|
4
|
+
castor-extract-count=castor_extractor.commands.extract_count:main
|
|
4
5
|
castor-extract-databricks=castor_extractor.commands.extract_databricks:main
|
|
5
6
|
castor-extract-domo=castor_extractor.commands.extract_domo:main
|
|
6
7
|
castor-extract-looker=castor_extractor.commands.extract_looker:main
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
SELECT
|
|
2
|
-
created_on,
|
|
3
|
-
modified_on,
|
|
4
|
-
privilege,
|
|
5
|
-
granted_on,
|
|
6
|
-
name,
|
|
7
|
-
table_catalog AS "database",
|
|
8
|
-
table_schema AS "schema",
|
|
9
|
-
granted_to,
|
|
10
|
-
grantee_name,
|
|
11
|
-
grant_option,
|
|
12
|
-
granted_by,
|
|
13
|
-
deleted_on
|
|
14
|
-
FROM snowflake.account_usage.grants_to_roles
|
|
File without changes
|
|
File without changes
|