castor-extractor 0.5.3__py3-none-any.whl → 0.5.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of castor-extractor might be problematic. Click here for more details.
- CHANGELOG.md +15 -0
- castor_extractor/commands/extract_bigquery.py +3 -1
- castor_extractor/commands/extract_looker.py +4 -1
- castor_extractor/commands/extract_metabase_api.py +3 -1
- castor_extractor/commands/extract_metabase_db.py +6 -2
- castor_extractor/commands/extract_mode.py +6 -2
- castor_extractor/commands/extract_powerbi.py +4 -1
- castor_extractor/commands/extract_snowflake.py +4 -2
- castor_extractor/commands/extract_tableau.py +4 -1
- castor_extractor/commands/file_check.py +3 -1
- castor_extractor/commands/upload.py +5 -2
- castor_extractor/file_checker/file_test.py +6 -3
- castor_extractor/file_checker/templates/generic_warehouse.py +4 -2
- castor_extractor/transformation/dbt/client/credentials.py +1 -1
- castor_extractor/types.py +2 -1
- castor_extractor/uploader/upload.py +4 -2
- castor_extractor/uploader/upload_test.py +0 -1
- castor_extractor/utils/deprecate.py +1 -1
- castor_extractor/utils/files_test.py +2 -2
- castor_extractor/utils/formatter_test.py +0 -1
- castor_extractor/utils/pager.py +4 -2
- castor_extractor/utils/pager_test.py +1 -1
- castor_extractor/utils/retry.py +1 -1
- castor_extractor/utils/safe.py +1 -1
- castor_extractor/utils/string_test.py +0 -1
- castor_extractor/utils/validation.py +4 -3
- castor_extractor/visualization/looker/api/client.py +26 -9
- castor_extractor/visualization/looker/api/client_test.py +3 -2
- castor_extractor/visualization/looker/api/constants.py +3 -1
- castor_extractor/visualization/looker/api/utils.py +3 -2
- castor_extractor/visualization/looker/assets.py +1 -0
- castor_extractor/visualization/looker/constant.py +1 -1
- castor_extractor/visualization/looker/extract.py +6 -1
- castor_extractor/visualization/metabase/client/api/client.py +2 -1
- castor_extractor/visualization/metabase/client/api/credentials.py +1 -1
- castor_extractor/visualization/metabase/client/db/client.py +4 -3
- castor_extractor/visualization/metabase/client/db/credentials.py +2 -2
- castor_extractor/visualization/metabase/client/decryption_test.py +0 -1
- castor_extractor/visualization/metabase/extract.py +4 -4
- castor_extractor/visualization/mode/client/client.py +2 -1
- castor_extractor/visualization/mode/client/client_test.py +4 -3
- castor_extractor/visualization/mode/client/credentials.py +2 -2
- castor_extractor/visualization/powerbi/client/constants.py +1 -1
- castor_extractor/visualization/powerbi/client/credentials.py +0 -1
- castor_extractor/visualization/powerbi/client/credentials_test.py +11 -3
- castor_extractor/visualization/powerbi/client/rest.py +15 -5
- castor_extractor/visualization/powerbi/client/rest_test.py +40 -13
- castor_extractor/visualization/powerbi/extract.py +4 -3
- castor_extractor/visualization/qlik/client/engine/client.py +3 -1
- castor_extractor/visualization/qlik/client/engine/json_rpc.py +4 -1
- castor_extractor/visualization/qlik/client/engine/json_rpc_test.py +0 -1
- castor_extractor/visualization/qlik/client/master.py +11 -4
- castor_extractor/visualization/qlik/client/rest_test.py +3 -2
- castor_extractor/visualization/sigma/client/client.py +7 -3
- castor_extractor/visualization/sigma/client/client_test.py +4 -2
- castor_extractor/visualization/sigma/client/credentials.py +2 -2
- castor_extractor/visualization/sigma/constants.py +1 -1
- castor_extractor/visualization/sigma/extract.py +3 -1
- castor_extractor/visualization/tableau/client/client.py +7 -5
- castor_extractor/visualization/tableau/client/client_utils.py +6 -3
- castor_extractor/visualization/tableau/client/credentials.py +6 -4
- castor_extractor/visualization/tableau/client/project.py +3 -1
- castor_extractor/visualization/tableau/client/safe_mode.py +2 -1
- castor_extractor/visualization/tableau/extract.py +7 -7
- castor_extractor/visualization/tableau/gql_fields.py +4 -4
- castor_extractor/visualization/tableau/tests/unit/graphql/paginated_object_test.py +2 -1
- castor_extractor/visualization/tableau/tests/unit/rest_api/auth_test.py +6 -3
- castor_extractor/visualization/tableau/tests/unit/rest_api/credentials_test.py +1 -1
- castor_extractor/visualization/tableau/tests/unit/rest_api/usages_test.py +2 -1
- castor_extractor/warehouse/abstract/extract.py +3 -2
- castor_extractor/warehouse/abstract/time_filter_test.py +0 -1
- castor_extractor/warehouse/bigquery/client_test.py +1 -1
- castor_extractor/warehouse/bigquery/extract.py +3 -2
- castor_extractor/warehouse/bigquery/query.py +4 -3
- castor_extractor/warehouse/postgres/extract.py +5 -3
- castor_extractor/warehouse/redshift/client_test.py +0 -1
- castor_extractor/warehouse/redshift/extract.py +5 -3
- castor_extractor/warehouse/snowflake/client.py +1 -1
- castor_extractor/warehouse/snowflake/client_test.py +1 -1
- castor_extractor/warehouse/snowflake/extract.py +5 -3
- castor_extractor/warehouse/synapse/extract.py +1 -1
- {castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/METADATA +2 -2
- {castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/RECORD +85 -85
- {castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/WHEEL +0 -0
- {castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/entry_points.txt +0 -0
CHANGELOG.md
CHANGED
|
@@ -1,12 +1,27 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.5.6 - 2023-08-10
|
|
4
|
+
|
|
5
|
+
* Use latest version of certifi (2023.7.22)
|
|
6
|
+
|
|
7
|
+
## 0.5.5 - 2023-08-07
|
|
8
|
+
|
|
9
|
+
* Linting with flakeheaven
|
|
10
|
+
|
|
11
|
+
## 0.5.4 - 2023-08-01
|
|
12
|
+
|
|
13
|
+
* Add support for Looker's `Users Attributes`
|
|
14
|
+
|
|
3
15
|
## 0.5.3 - 2023-07-27
|
|
16
|
+
|
|
4
17
|
* Add support for PowerBI's `Activity Events`
|
|
5
18
|
|
|
6
19
|
## 0.5.2 - 2023-07-12
|
|
20
|
+
|
|
7
21
|
* Fix Metabase DbClient url
|
|
8
22
|
|
|
9
23
|
## 0.5.1 - 2023-07-03
|
|
24
|
+
|
|
10
25
|
* Add support for Looker's `ContentViews`
|
|
11
26
|
|
|
12
27
|
## 0.5.0 - 2023-06-28
|
|
@@ -7,7 +7,9 @@ def main():
|
|
|
7
7
|
parser = ArgumentParser()
|
|
8
8
|
|
|
9
9
|
parser.add_argument(
|
|
10
|
-
"-c",
|
|
10
|
+
"-c",
|
|
11
|
+
"--credentials",
|
|
12
|
+
help="File path to google credentials",
|
|
11
13
|
)
|
|
12
14
|
parser.add_argument("-o", "--output", help="Directory to write to")
|
|
13
15
|
parser.add_argument(
|
|
@@ -18,7 +18,9 @@ def main():
|
|
|
18
18
|
args = parser.parse_args()
|
|
19
19
|
|
|
20
20
|
client = metabase.ApiClient(
|
|
21
|
-
base_url=args.base_url,
|
|
21
|
+
base_url=args.base_url,
|
|
22
|
+
user=args.username,
|
|
23
|
+
password=args.password,
|
|
22
24
|
)
|
|
23
25
|
|
|
24
26
|
metabase.extract_all(
|
|
@@ -11,7 +11,9 @@ def main():
|
|
|
11
11
|
|
|
12
12
|
# mandatory
|
|
13
13
|
parser.add_argument(
|
|
14
|
-
"-H",
|
|
14
|
+
"-H",
|
|
15
|
+
"--host",
|
|
16
|
+
help="Host name where the server is running",
|
|
15
17
|
)
|
|
16
18
|
parser.add_argument("-P", "--port", help="TCP/IP port number")
|
|
17
19
|
parser.add_argument("-d", "--database", help="Database name")
|
|
@@ -19,7 +21,9 @@ def main():
|
|
|
19
21
|
parser.add_argument("-u", "--username", help="Username")
|
|
20
22
|
parser.add_argument("-p", "--password", help="Password")
|
|
21
23
|
parser.add_argument(
|
|
22
|
-
"-k",
|
|
24
|
+
"-k",
|
|
25
|
+
"--encryption_secret_key",
|
|
26
|
+
help="Encryption secret key",
|
|
23
27
|
)
|
|
24
28
|
|
|
25
29
|
parser.add_argument("-o", "--output", help="Directory to write to")
|
|
@@ -12,10 +12,14 @@ def main():
|
|
|
12
12
|
parser.add_argument("-H", "--host", help="Mode Analytics host")
|
|
13
13
|
parser.add_argument("-w", "--workspace", help="Mode Analytics workspace")
|
|
14
14
|
parser.add_argument(
|
|
15
|
-
"-t",
|
|
15
|
+
"-t",
|
|
16
|
+
"--token",
|
|
17
|
+
help="The Token value from the API token",
|
|
16
18
|
)
|
|
17
19
|
parser.add_argument(
|
|
18
|
-
"-s",
|
|
20
|
+
"-s",
|
|
21
|
+
"--secret",
|
|
22
|
+
help="The Password value from the API token",
|
|
19
23
|
)
|
|
20
24
|
|
|
21
25
|
parser.add_argument("-o", "--output", help="Directory to write to")
|
|
@@ -13,7 +13,10 @@ def main():
|
|
|
13
13
|
parser.add_argument("-c", "--client_id", help="PowerBi client ID")
|
|
14
14
|
parser.add_argument("-s", "--secret", help="PowerBi password")
|
|
15
15
|
parser.add_argument(
|
|
16
|
-
"-sc",
|
|
16
|
+
"-sc",
|
|
17
|
+
"--scopes",
|
|
18
|
+
help="PowerBi scopes, optional",
|
|
19
|
+
nargs="*",
|
|
17
20
|
)
|
|
18
21
|
parser.add_argument("-o", "--output", help="Directory to write to")
|
|
19
22
|
|
|
@@ -14,10 +14,12 @@ def main():
|
|
|
14
14
|
parser.add_argument("-p", "--password", help="Snowflake password")
|
|
15
15
|
|
|
16
16
|
parser.add_argument(
|
|
17
|
-
"--warehouse",
|
|
17
|
+
"--warehouse",
|
|
18
|
+
help="Use a specific WAREHOUSE to run extraction queries",
|
|
18
19
|
)
|
|
19
20
|
parser.add_argument(
|
|
20
|
-
"--role",
|
|
21
|
+
"--role",
|
|
22
|
+
help="Use a specific ROLE to run extraction queries",
|
|
21
23
|
)
|
|
22
24
|
|
|
23
25
|
parser.add_argument(
|
|
@@ -20,7 +20,10 @@ def main():
|
|
|
20
20
|
parser.add_argument("-b", "--server-url", help="Tableau server url")
|
|
21
21
|
parser.add_argument("-i", "--site-id", help="Tableau site ID")
|
|
22
22
|
parser.add_argument(
|
|
23
|
-
"-s",
|
|
23
|
+
"-s",
|
|
24
|
+
"--safe-mode",
|
|
25
|
+
help="Tableau safe mode",
|
|
26
|
+
action="store_true",
|
|
24
27
|
)
|
|
25
28
|
parser.add_argument("-o", "--output", help="Directory to write to")
|
|
26
29
|
|
|
@@ -78,7 +78,9 @@ def process(directory: str, verbose: bool):
|
|
|
78
78
|
def main():
|
|
79
79
|
parser = ArgumentParser()
|
|
80
80
|
parser.add_argument(
|
|
81
|
-
"-d",
|
|
81
|
+
"-d",
|
|
82
|
+
"--directory",
|
|
83
|
+
help="Directory containing the files to be checked",
|
|
82
84
|
)
|
|
83
85
|
parser.add_argument(
|
|
84
86
|
"--verbose",
|
|
@@ -19,7 +19,10 @@ def _args():
|
|
|
19
19
|
help="""Path to credentials or credentials as string""",
|
|
20
20
|
)
|
|
21
21
|
parser.add_argument(
|
|
22
|
-
"-s",
|
|
22
|
+
"-s",
|
|
23
|
+
"--source_id",
|
|
24
|
+
required=True,
|
|
25
|
+
help="source id provided by castor",
|
|
23
26
|
)
|
|
24
27
|
group = parser.add_mutually_exclusive_group(required=True)
|
|
25
28
|
group.add_argument("-f", "--file_path", help="path to file to upload")
|
|
@@ -35,7 +38,7 @@ def _args():
|
|
|
35
38
|
"-t",
|
|
36
39
|
"--file_type",
|
|
37
40
|
help="type of file to upload, currently supported are {}".format(
|
|
38
|
-
supported_file_type
|
|
41
|
+
supported_file_type,
|
|
39
42
|
),
|
|
40
43
|
choices=supported_file_type,
|
|
41
44
|
)
|
|
@@ -23,15 +23,18 @@ def _user_template() -> FileTemplate:
|
|
|
23
23
|
"name": ColumnChecker(),
|
|
24
24
|
"gender": ColumnChecker(enum_values={"MALE", "FEMALE"}),
|
|
25
25
|
"birth_date": ColumnChecker(
|
|
26
|
-
data_type=DataType.DATETIME,
|
|
26
|
+
data_type=DataType.DATETIME,
|
|
27
|
+
is_mandatory=False,
|
|
27
28
|
),
|
|
28
29
|
"description": ColumnChecker(is_mandatory=False),
|
|
29
30
|
"siblings_count": ColumnChecker(
|
|
30
|
-
data_type=DataType.INTEGER,
|
|
31
|
+
data_type=DataType.INTEGER,
|
|
32
|
+
is_mandatory=False,
|
|
31
33
|
),
|
|
32
34
|
"height": ColumnChecker(data_type=DataType.FLOAT),
|
|
33
35
|
"folder_id": ColumnChecker(
|
|
34
|
-
data_type=DataType.INTEGER,
|
|
36
|
+
data_type=DataType.INTEGER,
|
|
37
|
+
foreign=folder_ids,
|
|
35
38
|
),
|
|
36
39
|
}
|
|
37
40
|
|
|
@@ -52,7 +52,8 @@ class GenericWarehouseFileTemplate:
|
|
|
52
52
|
"description": ColumnChecker(is_mandatory=False),
|
|
53
53
|
"data_type": ColumnChecker(enum_values=COLUMN_TYPES),
|
|
54
54
|
"ordinal_position": ColumnChecker(
|
|
55
|
-
is_mandatory=False,
|
|
55
|
+
is_mandatory=False,
|
|
56
|
+
data_type=DataType.INTEGER,
|
|
56
57
|
),
|
|
57
58
|
}
|
|
58
59
|
|
|
@@ -64,7 +65,8 @@ class GenericWarehouseFileTemplate:
|
|
|
64
65
|
"user_id": ColumnChecker(foreign=user_ids),
|
|
65
66
|
"start_time": ColumnChecker(data_type=DataType.DATETIME),
|
|
66
67
|
"end_time": ColumnChecker(
|
|
67
|
-
data_type=DataType.DATETIME,
|
|
68
|
+
data_type=DataType.DATETIME,
|
|
69
|
+
is_mandatory=False,
|
|
68
70
|
),
|
|
69
71
|
}
|
|
70
72
|
|
castor_extractor/types.py
CHANGED
|
@@ -72,12 +72,14 @@ def _upload(
|
|
|
72
72
|
with open(file_path, "rb") as f:
|
|
73
73
|
blob.upload_from_file(f, timeout=timeout, num_retries=retries)
|
|
74
74
|
logger.info(
|
|
75
|
-
f"uploaded {file_path} as {file_type.value} to {blob.public_url}"
|
|
75
|
+
f"uploaded {file_path} as {file_type.value} to {blob.public_url}",
|
|
76
76
|
)
|
|
77
77
|
|
|
78
78
|
|
|
79
79
|
def upload_manifest(
|
|
80
|
-
credentials: Union[str, dict],
|
|
80
|
+
credentials: Union[str, dict],
|
|
81
|
+
source_id: UUID,
|
|
82
|
+
file_path: str,
|
|
81
83
|
) -> None:
|
|
82
84
|
"""
|
|
83
85
|
credentials: path to file or dict
|
|
@@ -17,7 +17,7 @@ def deprecate_python(min_version_supported: Tuple[int]):
|
|
|
17
17
|
min_supported_str = ".".join(map(str, min_version_supported))
|
|
18
18
|
|
|
19
19
|
message = f"You are using python version {python_version_str}, please upgrade to version {min_supported_str} or higher."
|
|
20
|
-
" Your version will be soon deprecated"
|
|
20
|
+
" Your version will be soon deprecated"
|
|
21
21
|
|
|
22
22
|
if python_version < min_version_supported:
|
|
23
23
|
warnings.warn(message, DeprecationWarning)
|
|
@@ -9,7 +9,7 @@ def test_explode():
|
|
|
9
9
|
"/file.json": ("/", "file", "json"),
|
|
10
10
|
"/without/extension/file": ("/without/extension", "file", ""),
|
|
11
11
|
"file.txt": ("", "file", "txt"),
|
|
12
|
-
"/tmp/.bashrc": ("/tmp", ".bashrc", ""),
|
|
12
|
+
"/tmp/.bashrc": ("/tmp", ".bashrc", ""), # noqa: S108
|
|
13
13
|
}
|
|
14
14
|
for path, expected in checks.items():
|
|
15
15
|
assert explode(path) == expected
|
|
@@ -28,7 +28,7 @@ def test_search_files():
|
|
|
28
28
|
|
|
29
29
|
with patch("glob.glob") as mocked:
|
|
30
30
|
mocked.return_value = file_list
|
|
31
|
-
directory = "/tmp"
|
|
31
|
+
directory = "/tmp" # noqa: S108
|
|
32
32
|
|
|
33
33
|
# no filters
|
|
34
34
|
files = search_files(directory)
|
|
@@ -7,7 +7,6 @@ from .formatter import CsvFormatter, Formatter, JsonFormatter, to_string_array
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
def test__to_string_array():
|
|
10
|
-
|
|
11
10
|
assert to_string_array('["foo"]') == ["foo"]
|
|
12
11
|
assert to_string_array('["foo", "bar"]') == ["foo", "bar"]
|
|
13
12
|
assert to_string_array('["1", "2"]') == ["1", "2"]
|
castor_extractor/utils/pager.py
CHANGED
|
@@ -66,7 +66,8 @@ class Pager(AbstractPager):
|
|
|
66
66
|
self._stop_strategy = stop_strategy
|
|
67
67
|
|
|
68
68
|
def iterator(
|
|
69
|
-
self,
|
|
69
|
+
self,
|
|
70
|
+
per_page: int = _DEFAULT_PER_PAGE,
|
|
70
71
|
) -> Iterator[Sequence[T]]:
|
|
71
72
|
"""Yields data provided by the callback as a list page by page"""
|
|
72
73
|
page = self._start_page
|
|
@@ -107,7 +108,8 @@ class PagerOnId(AbstractPager):
|
|
|
107
108
|
self._stop_strategy = stop_strategy
|
|
108
109
|
|
|
109
110
|
def iterator(
|
|
110
|
-
self,
|
|
111
|
+
self,
|
|
112
|
+
per_page: int = _DEFAULT_PER_PAGE,
|
|
111
113
|
) -> Iterator[Sequence[T]]:
|
|
112
114
|
"""Yields data provided by the callback as a list using the greatest UUID as a reference point"""
|
|
113
115
|
greater_than_id = _DEFAULT_MIN_UUID
|
|
@@ -69,7 +69,7 @@ ITEMS_WITH_IDS = [
|
|
|
69
69
|
|
|
70
70
|
|
|
71
71
|
def _make_callback_with_ids(
|
|
72
|
-
elements: List[Dict[str, str]]
|
|
72
|
+
elements: List[Dict[str, str]],
|
|
73
73
|
) -> Callable[[UUID, int], List[Dict[str, str]]]:
|
|
74
74
|
def _callback(max_id: UUID, per: int) -> List[Dict[str, str]]:
|
|
75
75
|
"""assumes the elements are sorted by id"""
|
castor_extractor/utils/retry.py
CHANGED
|
@@ -53,7 +53,7 @@ class Retry:
|
|
|
53
53
|
return base * self.count
|
|
54
54
|
# exponential
|
|
55
55
|
scaled = float(base) / MS_IN_SEC
|
|
56
|
-
return int(scaled
|
|
56
|
+
return int(scaled**self.count * MS_IN_SEC)
|
|
57
57
|
|
|
58
58
|
def check(self) -> bool:
|
|
59
59
|
"""check wheter retry should happen or not"""
|
castor_extractor/utils/safe.py
CHANGED
|
@@ -55,7 +55,7 @@ def safe_mode(
|
|
|
55
55
|
if safe_mode_params.should_raise:
|
|
56
56
|
raise e
|
|
57
57
|
logger.error(
|
|
58
|
-
f"Safe mode : skip error {e} in function {function.__name__} with args {args} kwargs {kwargs}"
|
|
58
|
+
f"Safe mode : skip error {e} in function {function.__name__} with args {args} kwargs {kwargs}",
|
|
59
59
|
)
|
|
60
60
|
logger.debug(e, exc_info=True)
|
|
61
61
|
safe_mode_params.errors_caught.append(e)
|
|
@@ -9,7 +9,6 @@ class InvalidBaseUrl(ValueError):
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def _preprocess_url(base_url: str) -> str:
|
|
12
|
-
|
|
13
12
|
if "://" not in base_url:
|
|
14
13
|
return f"{BASE_URL_SCHEME}://{base_url}"
|
|
15
14
|
|
|
@@ -52,14 +51,16 @@ def _urlsplit(base_url: str) -> Tuple[str, str, str, str, str, str]:
|
|
|
52
51
|
|
|
53
52
|
|
|
54
53
|
def _expect(
|
|
55
|
-
attr: str,
|
|
54
|
+
attr: str,
|
|
55
|
+
expected: Optional[List[str]],
|
|
56
|
+
actual: Optional[str],
|
|
56
57
|
) -> None:
|
|
57
58
|
if not expected and not actual:
|
|
58
59
|
return
|
|
59
60
|
if expected and actual in expected:
|
|
60
61
|
return
|
|
61
62
|
raise InvalidBaseUrl(
|
|
62
|
-
f"Invalid base url {attr} | expected: {expected}, got: {actual}"
|
|
63
|
+
f"Invalid base url {attr} | expected: {expected}, got: {actual}",
|
|
63
64
|
)
|
|
64
65
|
|
|
65
66
|
|
|
@@ -4,7 +4,7 @@ from datetime import date, timedelta
|
|
|
4
4
|
from typing import Callable, Iterator, List, Optional, Sequence, Tuple
|
|
5
5
|
|
|
6
6
|
from dateutil.utils import today
|
|
7
|
-
from looker_sdk.sdk.api40.models import ContentView
|
|
7
|
+
from looker_sdk.sdk.api40.models import ContentView, UserAttribute
|
|
8
8
|
|
|
9
9
|
from ....utils import Pager, PagerLogger, SafeMode, past_date, safe_mode
|
|
10
10
|
from ..env import page_size
|
|
@@ -22,6 +22,7 @@ from .constants import (
|
|
|
22
22
|
LOOKML_PROJECT_NAME_BLOCKLIST,
|
|
23
23
|
PROJECT_FIELDS,
|
|
24
24
|
USER_FIELDS,
|
|
25
|
+
USERS_ATTRIBUTES_FIELDS,
|
|
25
26
|
)
|
|
26
27
|
from .sdk import (
|
|
27
28
|
Credentials,
|
|
@@ -122,7 +123,9 @@ class ApiClient:
|
|
|
122
123
|
|
|
123
124
|
def _search(page: int, per_page: int) -> Sequence[Look]:
|
|
124
125
|
return self._sdk.search_looks(
|
|
125
|
-
fields=format_fields(LOOK_FIELDS),
|
|
126
|
+
fields=format_fields(LOOK_FIELDS),
|
|
127
|
+
per_page=per_page,
|
|
128
|
+
page=page,
|
|
126
129
|
)
|
|
127
130
|
|
|
128
131
|
logger.info("Use search_looks endpoint to retrieve Looks")
|
|
@@ -169,7 +172,7 @@ class ApiClient:
|
|
|
169
172
|
"""Iterates LookML models of the given Looker account"""
|
|
170
173
|
|
|
171
174
|
models = self._sdk.all_lookml_models(
|
|
172
|
-
fields=format_fields(LOOKML_FIELDS)
|
|
175
|
+
fields=format_fields(LOOKML_FIELDS),
|
|
173
176
|
)
|
|
174
177
|
|
|
175
178
|
logger.info("All LookML models fetched")
|
|
@@ -182,13 +185,13 @@ class ApiClient:
|
|
|
182
185
|
]
|
|
183
186
|
|
|
184
187
|
def explores(
|
|
185
|
-
self,
|
|
188
|
+
self,
|
|
189
|
+
explore_names=Iterator[Tuple[str, str]],
|
|
186
190
|
) -> List[LookmlModelExplore]:
|
|
187
191
|
"""Iterates explores of the given Looker account for the provided model/explore names"""
|
|
188
192
|
|
|
189
193
|
@safe_mode(self._safe_mode)
|
|
190
194
|
def _call(model_name: str, explore_name: str) -> LookmlModelExplore:
|
|
191
|
-
|
|
192
195
|
explore = self._sdk.lookml_model_explore(model_name, explore_name)
|
|
193
196
|
|
|
194
197
|
logger.info(f"Explore {model_name}/{explore_name} fetched")
|
|
@@ -205,7 +208,7 @@ class ApiClient:
|
|
|
205
208
|
"""Lists databases connections of the given Looker account"""
|
|
206
209
|
|
|
207
210
|
connections = self._sdk.all_connections(
|
|
208
|
-
fields=format_fields(CONNECTION_FIELDS)
|
|
211
|
+
fields=format_fields(CONNECTION_FIELDS),
|
|
209
212
|
)
|
|
210
213
|
|
|
211
214
|
logger.info("All looker connections fetched")
|
|
@@ -226,7 +229,7 @@ class ApiClient:
|
|
|
226
229
|
def groups_hierarchy(self) -> List[GroupHierarchy]:
|
|
227
230
|
"""Lists groups with hierarchy of the given Looker account"""
|
|
228
231
|
groups_hierarchy = self._sdk.search_groups_with_hierarchy(
|
|
229
|
-
fields=format_fields(GROUPS_HIERARCHY_FIELDS)
|
|
232
|
+
fields=format_fields(GROUPS_HIERARCHY_FIELDS),
|
|
230
233
|
)
|
|
231
234
|
logger.info("All looker groups_hierarchy fetched")
|
|
232
235
|
return list(groups_hierarchy)
|
|
@@ -234,7 +237,7 @@ class ApiClient:
|
|
|
234
237
|
def groups_roles(self) -> List[GroupSearch]:
|
|
235
238
|
"""Lists groups with roles of the given Looker account"""
|
|
236
239
|
groups_roles = self._sdk.search_groups_with_roles(
|
|
237
|
-
fields=format_fields(GROUPS_ROLES_FIELDS)
|
|
240
|
+
fields=format_fields(GROUPS_ROLES_FIELDS),
|
|
238
241
|
)
|
|
239
242
|
logger.info("All looker groups_roles fetched")
|
|
240
243
|
return list(groups_roles)
|
|
@@ -262,6 +265,20 @@ class ApiClient:
|
|
|
262
265
|
content_views.extend(look_views + dashboard_views)
|
|
263
266
|
|
|
264
267
|
logger.info(
|
|
265
|
-
f"All looker content views fetched - {len(content_views)} rows"
|
|
268
|
+
f"All looker content views fetched - {len(content_views)} rows",
|
|
266
269
|
)
|
|
267
270
|
return content_views
|
|
271
|
+
|
|
272
|
+
def users_attributes(self) -> List[UserAttribute]:
|
|
273
|
+
"""Lists user attributes of the given Looker account"""
|
|
274
|
+
user_attributes = list(
|
|
275
|
+
self._sdk.all_user_attributes(
|
|
276
|
+
fields=format_fields(USERS_ATTRIBUTES_FIELDS),
|
|
277
|
+
),
|
|
278
|
+
)
|
|
279
|
+
logger.info(
|
|
280
|
+
f"All looker user_attributes fetched - {len(user_attributes)} rows",
|
|
281
|
+
)
|
|
282
|
+
self._on_api_call()
|
|
283
|
+
|
|
284
|
+
return user_attributes
|
|
@@ -14,7 +14,7 @@ from source.packages.extractor.castor_extractor.visualization.looker.api.client
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
def _credentials():
|
|
17
|
-
return Credentials(
|
|
17
|
+
return Credentials( # noqa: S106
|
|
18
18
|
base_url="base_url",
|
|
19
19
|
client_id="client_id",
|
|
20
20
|
client_secret="secret",
|
|
@@ -24,7 +24,8 @@ def _credentials():
|
|
|
24
24
|
@patch("castor_extractor.visualization.looker.api.client.init40")
|
|
25
25
|
@patch("castor_extractor.visualization.looker.api.client.has_admin_permissions")
|
|
26
26
|
def test_api_client_has_admin_permissions(
|
|
27
|
-
mock_has_admin_permission,
|
|
27
|
+
mock_has_admin_permission,
|
|
28
|
+
mock_init40,
|
|
28
29
|
):
|
|
29
30
|
mock_has_admin_permission.return_value = False
|
|
30
31
|
with pytest.raises(PermissionError):
|
|
@@ -204,7 +204,7 @@ GROUPS_ROLES_FIELDS = (
|
|
|
204
204
|
"name",
|
|
205
205
|
{"permission_set": ("id", "name", "permission")},
|
|
206
206
|
{"model_set": ("id", "models", "name")},
|
|
207
|
-
)
|
|
207
|
+
),
|
|
208
208
|
},
|
|
209
209
|
)
|
|
210
210
|
|
|
@@ -217,6 +217,8 @@ CONTENT_VIEWS_FIELDS = (
|
|
|
217
217
|
)
|
|
218
218
|
CONTENT_VIEWS_HISTORY_DAYS = 30
|
|
219
219
|
|
|
220
|
+
USERS_ATTRIBUTES_FIELDS = ("default_value", "id", "label", "name", "type")
|
|
221
|
+
|
|
220
222
|
|
|
221
223
|
# Model from looker
|
|
222
224
|
LOOKML_PROJECT_NAME_BLOCKLIST = ("looker-data", "system__activity")
|
|
@@ -42,9 +42,10 @@ def dashboard_explore_names(
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
def explore_names_associated_to_dashboards(
|
|
45
|
-
lookmls: Iterable[LookmlModel],
|
|
45
|
+
lookmls: Iterable[LookmlModel],
|
|
46
|
+
dashboards: Iterable[Dashboard],
|
|
46
47
|
):
|
|
47
48
|
"""Retrieve only explores that are associated to a looker dashboard"""
|
|
48
49
|
return lookml_explore_names(lookmls).intersection(
|
|
49
|
-
dashboard_explore_names(dashboards)
|
|
50
|
+
dashboard_explore_names(dashboards),
|
|
50
51
|
)
|
|
@@ -13,5 +13,5 @@ CASTOR_LOOKER_PAGE_SIZE = "CASTOR_LOOKER_PAGE_SIZE"
|
|
|
13
13
|
# env variables
|
|
14
14
|
BASE_URL = "CASTOR_LOOKER_BASE_URL"
|
|
15
15
|
CLIENT_ID = "CASTOR_LOOKER_CLIENT_ID"
|
|
16
|
-
CLIENT_SECRET = "CASTOR_LOOKER_CLIENT_SECRET"
|
|
16
|
+
CLIENT_SECRET = "CASTOR_LOOKER_CLIENT_SECRET" # noqa: S105
|
|
17
17
|
ALL_LOOKS = "CASTOR_LOOKER_ALL_LOOKS"
|
|
@@ -39,7 +39,8 @@ def _client(
|
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
def iterate_all_data(
|
|
42
|
-
client: ApiClient,
|
|
42
|
+
client: ApiClient,
|
|
43
|
+
all_looks: bool,
|
|
43
44
|
) -> Iterable[Tuple[LookerAsset, list]]:
|
|
44
45
|
"""Iterate over the extracted Data From looker"""
|
|
45
46
|
|
|
@@ -84,6 +85,10 @@ def iterate_all_data(
|
|
|
84
85
|
content_views = client.content_views()
|
|
85
86
|
yield LookerAsset.CONTENT_VIEWS, deep_serialize(content_views)
|
|
86
87
|
|
|
88
|
+
logger.info("Extracting users attributes from Looker API")
|
|
89
|
+
users_attributes = client.users_attributes()
|
|
90
|
+
yield LookerAsset.USERS_ATTRIBUTES, deep_serialize(users_attributes)
|
|
91
|
+
|
|
87
92
|
|
|
88
93
|
def extract_all(**kwargs) -> None:
|
|
89
94
|
"""
|
|
@@ -20,7 +20,7 @@ CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
|
20
20
|
PG_URL = "postgresql://{user}:{password}@{host}:{port}/{database}"
|
|
21
21
|
SQL_FILE_PATH = "queries/{name}.sql"
|
|
22
22
|
|
|
23
|
-
ENCRYPTION_SECRET_KEY = "CASTOR_METABASE_ENCRYPTION_SECRET_KEY"
|
|
23
|
+
ENCRYPTION_SECRET_KEY = "CASTOR_METABASE_ENCRYPTION_SECRET_KEY" # noqa: S105
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class DbClient:
|
|
@@ -41,7 +41,7 @@ class DbClient:
|
|
|
41
41
|
password=get_value(CredentialsDbKey.PASSWORD, kwargs),
|
|
42
42
|
)
|
|
43
43
|
self.encryption_secret_key = kwargs.get(
|
|
44
|
-
"encryption_secret_key"
|
|
44
|
+
"encryption_secret_key",
|
|
45
45
|
) or from_env(ENCRYPTION_SECRET_KEY, allow_missing=True)
|
|
46
46
|
self._engine = self._login()
|
|
47
47
|
|
|
@@ -66,7 +66,8 @@ class DbClient:
|
|
|
66
66
|
return content.format(schema=self._credentials.schema)
|
|
67
67
|
|
|
68
68
|
def _database_specifics(
|
|
69
|
-
self,
|
|
69
|
+
self,
|
|
70
|
+
databases: SerializedAsset,
|
|
70
71
|
) -> SerializedAsset:
|
|
71
72
|
for db in databases:
|
|
72
73
|
assert DETAILS_KEY in db # this field is expected in database table
|
|
@@ -10,7 +10,7 @@ class CredentialsDbKey(Enum):
|
|
|
10
10
|
DATABASE = "database"
|
|
11
11
|
SCHEMA = "schema"
|
|
12
12
|
USER = "user"
|
|
13
|
-
PASSWORD = "password"
|
|
13
|
+
PASSWORD = "password" # noqa: S105
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
CREDENTIALS_ENV: Dict[CredentialsDbKey, str] = {
|
|
@@ -19,7 +19,7 @@ CREDENTIALS_ENV: Dict[CredentialsDbKey, str] = {
|
|
|
19
19
|
CredentialsDbKey.DATABASE: "CASTOR_METABASE_DB_DATABASE",
|
|
20
20
|
CredentialsDbKey.SCHEMA: "CASTOR_METABASE_DB_SCHEMA",
|
|
21
21
|
CredentialsDbKey.USER: "CASTOR_METABASE_DB_USERNAME",
|
|
22
|
-
CredentialsDbKey.PASSWORD: "CASTOR_METABASE_DB_PASSWORD",
|
|
22
|
+
CredentialsDbKey.PASSWORD: "CASTOR_METABASE_DB_PASSWORD", # noqa: S105
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
|