castor-extractor 0.5.3__py3-none-any.whl → 0.5.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of castor-extractor might be problematic. Click here for more details.

Files changed (85) hide show
  1. CHANGELOG.md +15 -0
  2. castor_extractor/commands/extract_bigquery.py +3 -1
  3. castor_extractor/commands/extract_looker.py +4 -1
  4. castor_extractor/commands/extract_metabase_api.py +3 -1
  5. castor_extractor/commands/extract_metabase_db.py +6 -2
  6. castor_extractor/commands/extract_mode.py +6 -2
  7. castor_extractor/commands/extract_powerbi.py +4 -1
  8. castor_extractor/commands/extract_snowflake.py +4 -2
  9. castor_extractor/commands/extract_tableau.py +4 -1
  10. castor_extractor/commands/file_check.py +3 -1
  11. castor_extractor/commands/upload.py +5 -2
  12. castor_extractor/file_checker/file_test.py +6 -3
  13. castor_extractor/file_checker/templates/generic_warehouse.py +4 -2
  14. castor_extractor/transformation/dbt/client/credentials.py +1 -1
  15. castor_extractor/types.py +2 -1
  16. castor_extractor/uploader/upload.py +4 -2
  17. castor_extractor/uploader/upload_test.py +0 -1
  18. castor_extractor/utils/deprecate.py +1 -1
  19. castor_extractor/utils/files_test.py +2 -2
  20. castor_extractor/utils/formatter_test.py +0 -1
  21. castor_extractor/utils/pager.py +4 -2
  22. castor_extractor/utils/pager_test.py +1 -1
  23. castor_extractor/utils/retry.py +1 -1
  24. castor_extractor/utils/safe.py +1 -1
  25. castor_extractor/utils/string_test.py +0 -1
  26. castor_extractor/utils/validation.py +4 -3
  27. castor_extractor/visualization/looker/api/client.py +26 -9
  28. castor_extractor/visualization/looker/api/client_test.py +3 -2
  29. castor_extractor/visualization/looker/api/constants.py +3 -1
  30. castor_extractor/visualization/looker/api/utils.py +3 -2
  31. castor_extractor/visualization/looker/assets.py +1 -0
  32. castor_extractor/visualization/looker/constant.py +1 -1
  33. castor_extractor/visualization/looker/extract.py +6 -1
  34. castor_extractor/visualization/metabase/client/api/client.py +2 -1
  35. castor_extractor/visualization/metabase/client/api/credentials.py +1 -1
  36. castor_extractor/visualization/metabase/client/db/client.py +4 -3
  37. castor_extractor/visualization/metabase/client/db/credentials.py +2 -2
  38. castor_extractor/visualization/metabase/client/decryption_test.py +0 -1
  39. castor_extractor/visualization/metabase/extract.py +4 -4
  40. castor_extractor/visualization/mode/client/client.py +2 -1
  41. castor_extractor/visualization/mode/client/client_test.py +4 -3
  42. castor_extractor/visualization/mode/client/credentials.py +2 -2
  43. castor_extractor/visualization/powerbi/client/constants.py +1 -1
  44. castor_extractor/visualization/powerbi/client/credentials.py +0 -1
  45. castor_extractor/visualization/powerbi/client/credentials_test.py +11 -3
  46. castor_extractor/visualization/powerbi/client/rest.py +15 -5
  47. castor_extractor/visualization/powerbi/client/rest_test.py +40 -13
  48. castor_extractor/visualization/powerbi/extract.py +4 -3
  49. castor_extractor/visualization/qlik/client/engine/client.py +3 -1
  50. castor_extractor/visualization/qlik/client/engine/json_rpc.py +4 -1
  51. castor_extractor/visualization/qlik/client/engine/json_rpc_test.py +0 -1
  52. castor_extractor/visualization/qlik/client/master.py +11 -4
  53. castor_extractor/visualization/qlik/client/rest_test.py +3 -2
  54. castor_extractor/visualization/sigma/client/client.py +7 -3
  55. castor_extractor/visualization/sigma/client/client_test.py +4 -2
  56. castor_extractor/visualization/sigma/client/credentials.py +2 -2
  57. castor_extractor/visualization/sigma/constants.py +1 -1
  58. castor_extractor/visualization/sigma/extract.py +3 -1
  59. castor_extractor/visualization/tableau/client/client.py +7 -5
  60. castor_extractor/visualization/tableau/client/client_utils.py +6 -3
  61. castor_extractor/visualization/tableau/client/credentials.py +6 -4
  62. castor_extractor/visualization/tableau/client/project.py +3 -1
  63. castor_extractor/visualization/tableau/client/safe_mode.py +2 -1
  64. castor_extractor/visualization/tableau/extract.py +7 -7
  65. castor_extractor/visualization/tableau/gql_fields.py +4 -4
  66. castor_extractor/visualization/tableau/tests/unit/graphql/paginated_object_test.py +2 -1
  67. castor_extractor/visualization/tableau/tests/unit/rest_api/auth_test.py +6 -3
  68. castor_extractor/visualization/tableau/tests/unit/rest_api/credentials_test.py +1 -1
  69. castor_extractor/visualization/tableau/tests/unit/rest_api/usages_test.py +2 -1
  70. castor_extractor/warehouse/abstract/extract.py +3 -2
  71. castor_extractor/warehouse/abstract/time_filter_test.py +0 -1
  72. castor_extractor/warehouse/bigquery/client_test.py +1 -1
  73. castor_extractor/warehouse/bigquery/extract.py +3 -2
  74. castor_extractor/warehouse/bigquery/query.py +4 -3
  75. castor_extractor/warehouse/postgres/extract.py +5 -3
  76. castor_extractor/warehouse/redshift/client_test.py +0 -1
  77. castor_extractor/warehouse/redshift/extract.py +5 -3
  78. castor_extractor/warehouse/snowflake/client.py +1 -1
  79. castor_extractor/warehouse/snowflake/client_test.py +1 -1
  80. castor_extractor/warehouse/snowflake/extract.py +5 -3
  81. castor_extractor/warehouse/synapse/extract.py +1 -1
  82. {castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/METADATA +2 -2
  83. {castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/RECORD +85 -85
  84. {castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/WHEEL +0 -0
  85. {castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/entry_points.txt +0 -0
CHANGELOG.md CHANGED
@@ -1,12 +1,27 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.5.6 - 2023-08-10
4
+
5
+ * Use latest version of certifi (2023.7.22)
6
+
7
+ ## 0.5.5 - 2023-08-07
8
+
9
+ * Linting with flakeheaven
10
+
11
+ ## 0.5.4 - 2023-08-01
12
+
13
+ * Add support for Looker's `Users Attributes`
14
+
3
15
  ## 0.5.3 - 2023-07-27
16
+
4
17
  * Add support for PowerBI's `Activity Events`
5
18
 
6
19
  ## 0.5.2 - 2023-07-12
20
+
7
21
  * Fix Metabase DbClient url
8
22
 
9
23
  ## 0.5.1 - 2023-07-03
24
+
10
25
  * Add support for Looker's `ContentViews`
11
26
 
12
27
  ## 0.5.0 - 2023-06-28
@@ -7,7 +7,9 @@ def main():
7
7
  parser = ArgumentParser()
8
8
 
9
9
  parser.add_argument(
10
- "-c", "--credentials", help="File path to google credentials"
10
+ "-c",
11
+ "--credentials",
12
+ help="File path to google credentials",
11
13
  )
12
14
  parser.add_argument("-o", "--output", help="Directory to write to")
13
15
  parser.add_argument(
@@ -16,7 +16,10 @@ def main():
16
16
  action="store_true",
17
17
  )
18
18
  parser.add_argument(
19
- "--safe-mode", "-s", help="Looker safe mode", action="store_true"
19
+ "--safe-mode",
20
+ "-s",
21
+ help="Looker safe mode",
22
+ action="store_true",
20
23
  )
21
24
 
22
25
  args = parser.parse_args()
@@ -18,7 +18,9 @@ def main():
18
18
  args = parser.parse_args()
19
19
 
20
20
  client = metabase.ApiClient(
21
- base_url=args.base_url, user=args.username, password=args.password
21
+ base_url=args.base_url,
22
+ user=args.username,
23
+ password=args.password,
22
24
  )
23
25
 
24
26
  metabase.extract_all(
@@ -11,7 +11,9 @@ def main():
11
11
 
12
12
  # mandatory
13
13
  parser.add_argument(
14
- "-H", "--host", help="Host name where the server is running"
14
+ "-H",
15
+ "--host",
16
+ help="Host name where the server is running",
15
17
  )
16
18
  parser.add_argument("-P", "--port", help="TCP/IP port number")
17
19
  parser.add_argument("-d", "--database", help="Database name")
@@ -19,7 +21,9 @@ def main():
19
21
  parser.add_argument("-u", "--username", help="Username")
20
22
  parser.add_argument("-p", "--password", help="Password")
21
23
  parser.add_argument(
22
- "-k", "--encryption_secret_key", help="Encryption secret key"
24
+ "-k",
25
+ "--encryption_secret_key",
26
+ help="Encryption secret key",
23
27
  )
24
28
 
25
29
  parser.add_argument("-o", "--output", help="Directory to write to")
@@ -12,10 +12,14 @@ def main():
12
12
  parser.add_argument("-H", "--host", help="Mode Analytics host")
13
13
  parser.add_argument("-w", "--workspace", help="Mode Analytics workspace")
14
14
  parser.add_argument(
15
- "-t", "--token", help="The Token value from the API token"
15
+ "-t",
16
+ "--token",
17
+ help="The Token value from the API token",
16
18
  )
17
19
  parser.add_argument(
18
- "-s", "--secret", help="The Password value from the API token"
20
+ "-s",
21
+ "--secret",
22
+ help="The Password value from the API token",
19
23
  )
20
24
 
21
25
  parser.add_argument("-o", "--output", help="Directory to write to")
@@ -13,7 +13,10 @@ def main():
13
13
  parser.add_argument("-c", "--client_id", help="PowerBi client ID")
14
14
  parser.add_argument("-s", "--secret", help="PowerBi password")
15
15
  parser.add_argument(
16
- "-sc", "--scopes", help="PowerBi scopes, optional", nargs="*"
16
+ "-sc",
17
+ "--scopes",
18
+ help="PowerBi scopes, optional",
19
+ nargs="*",
17
20
  )
18
21
  parser.add_argument("-o", "--output", help="Directory to write to")
19
22
 
@@ -14,10 +14,12 @@ def main():
14
14
  parser.add_argument("-p", "--password", help="Snowflake password")
15
15
 
16
16
  parser.add_argument(
17
- "--warehouse", help="Use a specific WAREHOUSE to run extraction queries"
17
+ "--warehouse",
18
+ help="Use a specific WAREHOUSE to run extraction queries",
18
19
  )
19
20
  parser.add_argument(
20
- "--role", help="Use a specific ROLE to run extraction queries"
21
+ "--role",
22
+ help="Use a specific ROLE to run extraction queries",
21
23
  )
22
24
 
23
25
  parser.add_argument(
@@ -20,7 +20,10 @@ def main():
20
20
  parser.add_argument("-b", "--server-url", help="Tableau server url")
21
21
  parser.add_argument("-i", "--site-id", help="Tableau site ID")
22
22
  parser.add_argument(
23
- "-s", "--safe-mode", help="Tableau safe mode", action="store_true"
23
+ "-s",
24
+ "--safe-mode",
25
+ help="Tableau safe mode",
26
+ action="store_true",
24
27
  )
25
28
  parser.add_argument("-o", "--output", help="Directory to write to")
26
29
 
@@ -78,7 +78,9 @@ def process(directory: str, verbose: bool):
78
78
  def main():
79
79
  parser = ArgumentParser()
80
80
  parser.add_argument(
81
- "-d", "--directory", help="Directory containing the files to be checked"
81
+ "-d",
82
+ "--directory",
83
+ help="Directory containing the files to be checked",
82
84
  )
83
85
  parser.add_argument(
84
86
  "--verbose",
@@ -19,7 +19,10 @@ def _args():
19
19
  help="""Path to credentials or credentials as string""",
20
20
  )
21
21
  parser.add_argument(
22
- "-s", "--source_id", required=True, help="source id provided by castor"
22
+ "-s",
23
+ "--source_id",
24
+ required=True,
25
+ help="source id provided by castor",
23
26
  )
24
27
  group = parser.add_mutually_exclusive_group(required=True)
25
28
  group.add_argument("-f", "--file_path", help="path to file to upload")
@@ -35,7 +38,7 @@ def _args():
35
38
  "-t",
36
39
  "--file_type",
37
40
  help="type of file to upload, currently supported are {}".format(
38
- supported_file_type
41
+ supported_file_type,
39
42
  ),
40
43
  choices=supported_file_type,
41
44
  )
@@ -23,15 +23,18 @@ def _user_template() -> FileTemplate:
23
23
  "name": ColumnChecker(),
24
24
  "gender": ColumnChecker(enum_values={"MALE", "FEMALE"}),
25
25
  "birth_date": ColumnChecker(
26
- data_type=DataType.DATETIME, is_mandatory=False
26
+ data_type=DataType.DATETIME,
27
+ is_mandatory=False,
27
28
  ),
28
29
  "description": ColumnChecker(is_mandatory=False),
29
30
  "siblings_count": ColumnChecker(
30
- data_type=DataType.INTEGER, is_mandatory=False
31
+ data_type=DataType.INTEGER,
32
+ is_mandatory=False,
31
33
  ),
32
34
  "height": ColumnChecker(data_type=DataType.FLOAT),
33
35
  "folder_id": ColumnChecker(
34
- data_type=DataType.INTEGER, foreign=folder_ids
36
+ data_type=DataType.INTEGER,
37
+ foreign=folder_ids,
35
38
  ),
36
39
  }
37
40
 
@@ -52,7 +52,8 @@ class GenericWarehouseFileTemplate:
52
52
  "description": ColumnChecker(is_mandatory=False),
53
53
  "data_type": ColumnChecker(enum_values=COLUMN_TYPES),
54
54
  "ordinal_position": ColumnChecker(
55
- is_mandatory=False, data_type=DataType.INTEGER
55
+ is_mandatory=False,
56
+ data_type=DataType.INTEGER,
56
57
  ),
57
58
  }
58
59
 
@@ -64,7 +65,8 @@ class GenericWarehouseFileTemplate:
64
65
  "user_id": ColumnChecker(foreign=user_ids),
65
66
  "start_time": ColumnChecker(data_type=DataType.DATETIME),
66
67
  "end_time": ColumnChecker(
67
- data_type=DataType.DATETIME, is_mandatory=False
68
+ data_type=DataType.DATETIME,
69
+ is_mandatory=False,
68
70
  ),
69
71
  }
70
72
 
@@ -8,7 +8,7 @@ from ....utils import from_env
8
8
  class CredentialsKey(Enum):
9
9
  """keys present in dbt credentials"""
10
10
 
11
- TOKEN = "token"
11
+ TOKEN = "token" # noqa: S105
12
12
  JOB_ID = "job_id"
13
13
 
14
14
 
castor_extractor/types.py CHANGED
@@ -2,5 +2,6 @@
2
2
  from typing import Literal, TypedDict
3
3
 
4
4
  CsvOptions = TypedDict(
5
- "CsvOptions", {"delimiter": str, "quoting": Literal[1], "quotechar": str}
5
+ "CsvOptions",
6
+ {"delimiter": str, "quoting": Literal[1], "quotechar": str},
6
7
  )
@@ -72,12 +72,14 @@ def _upload(
72
72
  with open(file_path, "rb") as f:
73
73
  blob.upload_from_file(f, timeout=timeout, num_retries=retries)
74
74
  logger.info(
75
- f"uploaded {file_path} as {file_type.value} to {blob.public_url}"
75
+ f"uploaded {file_path} as {file_type.value} to {blob.public_url}",
76
76
  )
77
77
 
78
78
 
79
79
  def upload_manifest(
80
- credentials: Union[str, dict], source_id: UUID, file_path: str
80
+ credentials: Union[str, dict],
81
+ source_id: UUID,
82
+ file_path: str,
81
83
  ) -> None:
82
84
  """
83
85
  credentials: path to file or dict
@@ -5,7 +5,6 @@ from .upload import _path
5
5
 
6
6
 
7
7
  def test__path():
8
-
9
8
  source_id = UUID("399a8b22-3187-11ec-8d3d-0242ac130003")
10
9
  file_type = FileType.VIZ
11
10
  file_path = "filename"
@@ -17,7 +17,7 @@ def deprecate_python(min_version_supported: Tuple[int]):
17
17
  min_supported_str = ".".join(map(str, min_version_supported))
18
18
 
19
19
  message = f"You are using python version {python_version_str}, please upgrade to version {min_supported_str} or higher."
20
- " Your version will be soon deprecated",
20
+ " Your version will be soon deprecated"
21
21
 
22
22
  if python_version < min_version_supported:
23
23
  warnings.warn(message, DeprecationWarning)
@@ -9,7 +9,7 @@ def test_explode():
9
9
  "/file.json": ("/", "file", "json"),
10
10
  "/without/extension/file": ("/without/extension", "file", ""),
11
11
  "file.txt": ("", "file", "txt"),
12
- "/tmp/.bashrc": ("/tmp", ".bashrc", ""),
12
+ "/tmp/.bashrc": ("/tmp", ".bashrc", ""), # noqa: S108
13
13
  }
14
14
  for path, expected in checks.items():
15
15
  assert explode(path) == expected
@@ -28,7 +28,7 @@ def test_search_files():
28
28
 
29
29
  with patch("glob.glob") as mocked:
30
30
  mocked.return_value = file_list
31
- directory = "/tmp"
31
+ directory = "/tmp" # noqa: S108
32
32
 
33
33
  # no filters
34
34
  files = search_files(directory)
@@ -7,7 +7,6 @@ from .formatter import CsvFormatter, Formatter, JsonFormatter, to_string_array
7
7
 
8
8
 
9
9
  def test__to_string_array():
10
-
11
10
  assert to_string_array('["foo"]') == ["foo"]
12
11
  assert to_string_array('["foo", "bar"]') == ["foo", "bar"]
13
12
  assert to_string_array('["1", "2"]') == ["1", "2"]
@@ -66,7 +66,8 @@ class Pager(AbstractPager):
66
66
  self._stop_strategy = stop_strategy
67
67
 
68
68
  def iterator(
69
- self, per_page: int = _DEFAULT_PER_PAGE
69
+ self,
70
+ per_page: int = _DEFAULT_PER_PAGE,
70
71
  ) -> Iterator[Sequence[T]]:
71
72
  """Yields data provided by the callback as a list page by page"""
72
73
  page = self._start_page
@@ -107,7 +108,8 @@ class PagerOnId(AbstractPager):
107
108
  self._stop_strategy = stop_strategy
108
109
 
109
110
  def iterator(
110
- self, per_page: int = _DEFAULT_PER_PAGE
111
+ self,
112
+ per_page: int = _DEFAULT_PER_PAGE,
111
113
  ) -> Iterator[Sequence[T]]:
112
114
  """Yields data provided by the callback as a list using the greatest UUID as a reference point"""
113
115
  greater_than_id = _DEFAULT_MIN_UUID
@@ -69,7 +69,7 @@ ITEMS_WITH_IDS = [
69
69
 
70
70
 
71
71
  def _make_callback_with_ids(
72
- elements: List[Dict[str, str]]
72
+ elements: List[Dict[str, str]],
73
73
  ) -> Callable[[UUID, int], List[Dict[str, str]]]:
74
74
  def _callback(max_id: UUID, per: int) -> List[Dict[str, str]]:
75
75
  """assumes the elements are sorted by id"""
@@ -53,7 +53,7 @@ class Retry:
53
53
  return base * self.count
54
54
  # exponential
55
55
  scaled = float(base) / MS_IN_SEC
56
- return int(scaled ** self.count * MS_IN_SEC)
56
+ return int(scaled**self.count * MS_IN_SEC)
57
57
 
58
58
  def check(self) -> bool:
59
59
  """check wheter retry should happen or not"""
@@ -55,7 +55,7 @@ def safe_mode(
55
55
  if safe_mode_params.should_raise:
56
56
  raise e
57
57
  logger.error(
58
- f"Safe mode : skip error {e} in function {function.__name__} with args {args} kwargs {kwargs}"
58
+ f"Safe mode : skip error {e} in function {function.__name__} with args {args} kwargs {kwargs}",
59
59
  )
60
60
  logger.debug(e, exc_info=True)
61
61
  safe_mode_params.errors_caught.append(e)
@@ -14,7 +14,6 @@ def test__string_to_tuple():
14
14
 
15
15
  # loop on supported symbols surrounding elements
16
16
  for symbols in ("[]", "{}", "()", " "):
17
-
18
17
  # empty list, set, tuple and empty string
19
18
  # [], {}, (), ''
20
19
  assert _test(symbols, "") == tuple()
@@ -9,7 +9,6 @@ class InvalidBaseUrl(ValueError):
9
9
 
10
10
 
11
11
  def _preprocess_url(base_url: str) -> str:
12
-
13
12
  if "://" not in base_url:
14
13
  return f"{BASE_URL_SCHEME}://{base_url}"
15
14
 
@@ -52,14 +51,16 @@ def _urlsplit(base_url: str) -> Tuple[str, str, str, str, str, str]:
52
51
 
53
52
 
54
53
  def _expect(
55
- attr: str, expected: Optional[List[str]], actual: Optional[str]
54
+ attr: str,
55
+ expected: Optional[List[str]],
56
+ actual: Optional[str],
56
57
  ) -> None:
57
58
  if not expected and not actual:
58
59
  return
59
60
  if expected and actual in expected:
60
61
  return
61
62
  raise InvalidBaseUrl(
62
- f"Invalid base url {attr} | expected: {expected}, got: {actual}"
63
+ f"Invalid base url {attr} | expected: {expected}, got: {actual}",
63
64
  )
64
65
 
65
66
 
@@ -4,7 +4,7 @@ from datetime import date, timedelta
4
4
  from typing import Callable, Iterator, List, Optional, Sequence, Tuple
5
5
 
6
6
  from dateutil.utils import today
7
- from looker_sdk.sdk.api40.models import ContentView
7
+ from looker_sdk.sdk.api40.models import ContentView, UserAttribute
8
8
 
9
9
  from ....utils import Pager, PagerLogger, SafeMode, past_date, safe_mode
10
10
  from ..env import page_size
@@ -22,6 +22,7 @@ from .constants import (
22
22
  LOOKML_PROJECT_NAME_BLOCKLIST,
23
23
  PROJECT_FIELDS,
24
24
  USER_FIELDS,
25
+ USERS_ATTRIBUTES_FIELDS,
25
26
  )
26
27
  from .sdk import (
27
28
  Credentials,
@@ -122,7 +123,9 @@ class ApiClient:
122
123
 
123
124
  def _search(page: int, per_page: int) -> Sequence[Look]:
124
125
  return self._sdk.search_looks(
125
- fields=format_fields(LOOK_FIELDS), per_page=per_page, page=page
126
+ fields=format_fields(LOOK_FIELDS),
127
+ per_page=per_page,
128
+ page=page,
126
129
  )
127
130
 
128
131
  logger.info("Use search_looks endpoint to retrieve Looks")
@@ -169,7 +172,7 @@ class ApiClient:
169
172
  """Iterates LookML models of the given Looker account"""
170
173
 
171
174
  models = self._sdk.all_lookml_models(
172
- fields=format_fields(LOOKML_FIELDS)
175
+ fields=format_fields(LOOKML_FIELDS),
173
176
  )
174
177
 
175
178
  logger.info("All LookML models fetched")
@@ -182,13 +185,13 @@ class ApiClient:
182
185
  ]
183
186
 
184
187
  def explores(
185
- self, explore_names=Iterator[Tuple[str, str]]
188
+ self,
189
+ explore_names=Iterator[Tuple[str, str]],
186
190
  ) -> List[LookmlModelExplore]:
187
191
  """Iterates explores of the given Looker account for the provided model/explore names"""
188
192
 
189
193
  @safe_mode(self._safe_mode)
190
194
  def _call(model_name: str, explore_name: str) -> LookmlModelExplore:
191
-
192
195
  explore = self._sdk.lookml_model_explore(model_name, explore_name)
193
196
 
194
197
  logger.info(f"Explore {model_name}/{explore_name} fetched")
@@ -205,7 +208,7 @@ class ApiClient:
205
208
  """Lists databases connections of the given Looker account"""
206
209
 
207
210
  connections = self._sdk.all_connections(
208
- fields=format_fields(CONNECTION_FIELDS)
211
+ fields=format_fields(CONNECTION_FIELDS),
209
212
  )
210
213
 
211
214
  logger.info("All looker connections fetched")
@@ -226,7 +229,7 @@ class ApiClient:
226
229
  def groups_hierarchy(self) -> List[GroupHierarchy]:
227
230
  """Lists groups with hierarchy of the given Looker account"""
228
231
  groups_hierarchy = self._sdk.search_groups_with_hierarchy(
229
- fields=format_fields(GROUPS_HIERARCHY_FIELDS)
232
+ fields=format_fields(GROUPS_HIERARCHY_FIELDS),
230
233
  )
231
234
  logger.info("All looker groups_hierarchy fetched")
232
235
  return list(groups_hierarchy)
@@ -234,7 +237,7 @@ class ApiClient:
234
237
  def groups_roles(self) -> List[GroupSearch]:
235
238
  """Lists groups with roles of the given Looker account"""
236
239
  groups_roles = self._sdk.search_groups_with_roles(
237
- fields=format_fields(GROUPS_ROLES_FIELDS)
240
+ fields=format_fields(GROUPS_ROLES_FIELDS),
238
241
  )
239
242
  logger.info("All looker groups_roles fetched")
240
243
  return list(groups_roles)
@@ -262,6 +265,20 @@ class ApiClient:
262
265
  content_views.extend(look_views + dashboard_views)
263
266
 
264
267
  logger.info(
265
- f"All looker content views fetched - {len(content_views)} rows"
268
+ f"All looker content views fetched - {len(content_views)} rows",
266
269
  )
267
270
  return content_views
271
+
272
+ def users_attributes(self) -> List[UserAttribute]:
273
+ """Lists user attributes of the given Looker account"""
274
+ user_attributes = list(
275
+ self._sdk.all_user_attributes(
276
+ fields=format_fields(USERS_ATTRIBUTES_FIELDS),
277
+ ),
278
+ )
279
+ logger.info(
280
+ f"All looker user_attributes fetched - {len(user_attributes)} rows",
281
+ )
282
+ self._on_api_call()
283
+
284
+ return user_attributes
@@ -14,7 +14,7 @@ from source.packages.extractor.castor_extractor.visualization.looker.api.client
14
14
 
15
15
 
16
16
  def _credentials():
17
- return Credentials(
17
+ return Credentials( # noqa: S106
18
18
  base_url="base_url",
19
19
  client_id="client_id",
20
20
  client_secret="secret",
@@ -24,7 +24,8 @@ def _credentials():
24
24
  @patch("castor_extractor.visualization.looker.api.client.init40")
25
25
  @patch("castor_extractor.visualization.looker.api.client.has_admin_permissions")
26
26
  def test_api_client_has_admin_permissions(
27
- mock_has_admin_permission, mock_init40
27
+ mock_has_admin_permission,
28
+ mock_init40,
28
29
  ):
29
30
  mock_has_admin_permission.return_value = False
30
31
  with pytest.raises(PermissionError):
@@ -204,7 +204,7 @@ GROUPS_ROLES_FIELDS = (
204
204
  "name",
205
205
  {"permission_set": ("id", "name", "permission")},
206
206
  {"model_set": ("id", "models", "name")},
207
- )
207
+ ),
208
208
  },
209
209
  )
210
210
 
@@ -217,6 +217,8 @@ CONTENT_VIEWS_FIELDS = (
217
217
  )
218
218
  CONTENT_VIEWS_HISTORY_DAYS = 30
219
219
 
220
+ USERS_ATTRIBUTES_FIELDS = ("default_value", "id", "label", "name", "type")
221
+
220
222
 
221
223
  # Model from looker
222
224
  LOOKML_PROJECT_NAME_BLOCKLIST = ("looker-data", "system__activity")
@@ -42,9 +42,10 @@ def dashboard_explore_names(
42
42
 
43
43
 
44
44
  def explore_names_associated_to_dashboards(
45
- lookmls: Iterable[LookmlModel], dashboards: Iterable[Dashboard]
45
+ lookmls: Iterable[LookmlModel],
46
+ dashboards: Iterable[Dashboard],
46
47
  ):
47
48
  """Retrieve only explores that are associated to a looker dashboard"""
48
49
  return lookml_explore_names(lookmls).intersection(
49
- dashboard_explore_names(dashboards)
50
+ dashboard_explore_names(dashboards),
50
51
  )
@@ -15,3 +15,4 @@ class LookerAsset(Enum):
15
15
  LOOKS = "looks"
16
16
  PROJECTS = "projects"
17
17
  USERS = "users"
18
+ USERS_ATTRIBUTES = "users_attributes"
@@ -13,5 +13,5 @@ CASTOR_LOOKER_PAGE_SIZE = "CASTOR_LOOKER_PAGE_SIZE"
13
13
  # env variables
14
14
  BASE_URL = "CASTOR_LOOKER_BASE_URL"
15
15
  CLIENT_ID = "CASTOR_LOOKER_CLIENT_ID"
16
- CLIENT_SECRET = "CASTOR_LOOKER_CLIENT_SECRET"
16
+ CLIENT_SECRET = "CASTOR_LOOKER_CLIENT_SECRET" # noqa: S105
17
17
  ALL_LOOKS = "CASTOR_LOOKER_ALL_LOOKS"
@@ -39,7 +39,8 @@ def _client(
39
39
 
40
40
 
41
41
  def iterate_all_data(
42
- client: ApiClient, all_looks: bool
42
+ client: ApiClient,
43
+ all_looks: bool,
43
44
  ) -> Iterable[Tuple[LookerAsset, list]]:
44
45
  """Iterate over the extracted Data From looker"""
45
46
 
@@ -84,6 +85,10 @@ def iterate_all_data(
84
85
  content_views = client.content_views()
85
86
  yield LookerAsset.CONTENT_VIEWS, deep_serialize(content_views)
86
87
 
88
+ logger.info("Extracting users attributes from Looker API")
89
+ users_attributes = client.users_attributes()
90
+ yield LookerAsset.USERS_ATTRIBUTES, deep_serialize(users_attributes)
91
+
87
92
 
88
93
  def extract_all(**kwargs) -> None:
89
94
  """
@@ -51,7 +51,8 @@ class ApiClient:
51
51
 
52
52
  def _url(self, endpoint: str) -> str:
53
53
  return URL_TEMPLATE.format(
54
- base_url=self._credentials.base_url, endpoint=endpoint
54
+ base_url=self._credentials.base_url,
55
+ endpoint=endpoint,
55
56
  )
56
57
 
57
58
  def _headers(self) -> dict:
@@ -7,7 +7,7 @@ from .....utils import from_env
7
7
  class CredentialsApiKey(Enum):
8
8
  BASE_URL = "base_url"
9
9
  USER = "user"
10
- PASSWORD = "password"
10
+ PASSWORD = "password" # noqa: S105
11
11
 
12
12
 
13
13
  CREDENTIALS_ENV: Dict[CredentialsApiKey, str] = {
@@ -20,7 +20,7 @@ CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
20
20
  PG_URL = "postgresql://{user}:{password}@{host}:{port}/{database}"
21
21
  SQL_FILE_PATH = "queries/{name}.sql"
22
22
 
23
- ENCRYPTION_SECRET_KEY = "CASTOR_METABASE_ENCRYPTION_SECRET_KEY"
23
+ ENCRYPTION_SECRET_KEY = "CASTOR_METABASE_ENCRYPTION_SECRET_KEY" # noqa: S105
24
24
 
25
25
 
26
26
  class DbClient:
@@ -41,7 +41,7 @@ class DbClient:
41
41
  password=get_value(CredentialsDbKey.PASSWORD, kwargs),
42
42
  )
43
43
  self.encryption_secret_key = kwargs.get(
44
- "encryption_secret_key"
44
+ "encryption_secret_key",
45
45
  ) or from_env(ENCRYPTION_SECRET_KEY, allow_missing=True)
46
46
  self._engine = self._login()
47
47
 
@@ -66,7 +66,8 @@ class DbClient:
66
66
  return content.format(schema=self._credentials.schema)
67
67
 
68
68
  def _database_specifics(
69
- self, databases: SerializedAsset
69
+ self,
70
+ databases: SerializedAsset,
70
71
  ) -> SerializedAsset:
71
72
  for db in databases:
72
73
  assert DETAILS_KEY in db # this field is expected in database table
@@ -10,7 +10,7 @@ class CredentialsDbKey(Enum):
10
10
  DATABASE = "database"
11
11
  SCHEMA = "schema"
12
12
  USER = "user"
13
- PASSWORD = "password"
13
+ PASSWORD = "password" # noqa: S105
14
14
 
15
15
 
16
16
  CREDENTIALS_ENV: Dict[CredentialsDbKey, str] = {
@@ -19,7 +19,7 @@ CREDENTIALS_ENV: Dict[CredentialsDbKey, str] = {
19
19
  CredentialsDbKey.DATABASE: "CASTOR_METABASE_DB_DATABASE",
20
20
  CredentialsDbKey.SCHEMA: "CASTOR_METABASE_DB_SCHEMA",
21
21
  CredentialsDbKey.USER: "CASTOR_METABASE_DB_USERNAME",
22
- CredentialsDbKey.PASSWORD: "CASTOR_METABASE_DB_PASSWORD",
22
+ CredentialsDbKey.PASSWORD: "CASTOR_METABASE_DB_PASSWORD", # noqa: S105
23
23
  }
24
24
 
25
25