castor-extractor 0.17.4__py3-none-any.whl → 0.18.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of castor-extractor might be problematic. Click here for more details.

Files changed (92) hide show
  1. CHANGELOG.md +28 -0
  2. DockerfileUsage.md +21 -0
  3. castor_extractor/commands/extract_domo.py +2 -10
  4. castor_extractor/commands/extract_looker.py +2 -13
  5. castor_extractor/commands/extract_metabase_api.py +5 -10
  6. castor_extractor/commands/extract_metabase_db.py +6 -16
  7. castor_extractor/commands/extract_mode.py +2 -13
  8. castor_extractor/commands/extract_powerbi.py +2 -8
  9. castor_extractor/commands/extract_qlik.py +2 -7
  10. castor_extractor/commands/extract_salesforce.py +3 -12
  11. castor_extractor/commands/extract_salesforce_reporting.py +2 -10
  12. castor_extractor/commands/extract_sigma.py +2 -7
  13. castor_extractor/utils/__init__.py +3 -1
  14. castor_extractor/utils/argument_parser.py +7 -0
  15. castor_extractor/utils/argument_parser_test.py +25 -0
  16. castor_extractor/utils/collection.py +8 -0
  17. castor_extractor/utils/safe_request.py +57 -0
  18. castor_extractor/utils/safe_request_test.py +77 -0
  19. castor_extractor/utils/salesforce/__init__.py +1 -2
  20. castor_extractor/utils/salesforce/constants.py +0 -11
  21. castor_extractor/utils/salesforce/credentials.py +22 -45
  22. castor_extractor/visualization/domo/__init__.py +1 -1
  23. castor_extractor/visualization/domo/client/__init__.py +1 -1
  24. castor_extractor/visualization/domo/client/client.py +37 -52
  25. castor_extractor/visualization/domo/client/credentials.py +14 -27
  26. castor_extractor/visualization/domo/extract.py +5 -26
  27. castor_extractor/visualization/looker/__init__.py +6 -1
  28. castor_extractor/visualization/looker/api/__init__.py +2 -1
  29. castor_extractor/visualization/looker/api/client.py +6 -4
  30. castor_extractor/visualization/looker/api/client_test.py +5 -3
  31. castor_extractor/visualization/looker/api/credentials.py +33 -0
  32. castor_extractor/visualization/looker/api/extraction_parameters.py +38 -0
  33. castor_extractor/visualization/looker/api/sdk.py +2 -28
  34. castor_extractor/visualization/looker/constant.py +2 -27
  35. castor_extractor/visualization/looker/constants.py +17 -0
  36. castor_extractor/visualization/looker/extract.py +29 -29
  37. castor_extractor/visualization/metabase/__init__.py +6 -1
  38. castor_extractor/visualization/metabase/client/__init__.py +2 -2
  39. castor_extractor/visualization/metabase/client/api/__init__.py +1 -0
  40. castor_extractor/visualization/metabase/client/api/client.py +8 -14
  41. castor_extractor/visualization/metabase/client/api/credentials.py +13 -40
  42. castor_extractor/visualization/metabase/client/db/__init__.py +1 -0
  43. castor_extractor/visualization/metabase/client/db/client.py +13 -34
  44. castor_extractor/visualization/metabase/client/db/credentials.py +19 -73
  45. castor_extractor/visualization/metabase/errors.py +5 -3
  46. castor_extractor/visualization/metabase/extract.py +3 -3
  47. castor_extractor/visualization/mode/__init__.py +1 -1
  48. castor_extractor/visualization/mode/client/__init__.py +1 -0
  49. castor_extractor/visualization/mode/client/client.py +9 -12
  50. castor_extractor/visualization/mode/client/client_test.py +3 -3
  51. castor_extractor/visualization/mode/client/credentials.py +18 -51
  52. castor_extractor/visualization/mode/extract.py +6 -3
  53. castor_extractor/visualization/powerbi/__init__.py +1 -1
  54. castor_extractor/visualization/powerbi/client/__init__.py +2 -1
  55. castor_extractor/visualization/powerbi/client/credentials.py +17 -9
  56. castor_extractor/visualization/powerbi/client/credentials_test.py +12 -4
  57. castor_extractor/visualization/powerbi/client/rest.py +2 -2
  58. castor_extractor/visualization/powerbi/client/rest_test.py +2 -2
  59. castor_extractor/visualization/powerbi/extract.py +5 -16
  60. castor_extractor/visualization/qlik/__init__.py +5 -1
  61. castor_extractor/visualization/qlik/client/__init__.py +1 -0
  62. castor_extractor/visualization/qlik/client/engine/__init__.py +1 -0
  63. castor_extractor/visualization/qlik/client/engine/client.py +5 -6
  64. castor_extractor/visualization/qlik/client/engine/credentials.py +26 -0
  65. castor_extractor/visualization/qlik/client/master.py +5 -11
  66. castor_extractor/visualization/qlik/client/rest.py +4 -4
  67. castor_extractor/visualization/qlik/client/rest_test.py +6 -2
  68. castor_extractor/visualization/qlik/extract.py +6 -13
  69. castor_extractor/visualization/salesforce_reporting/extract.py +6 -20
  70. castor_extractor/visualization/sigma/__init__.py +1 -1
  71. castor_extractor/visualization/sigma/client/__init__.py +1 -1
  72. castor_extractor/visualization/sigma/client/client.py +5 -4
  73. castor_extractor/visualization/sigma/client/credentials.py +12 -28
  74. castor_extractor/visualization/sigma/extract.py +5 -18
  75. castor_extractor/visualization/tableau_revamp/client/credentials.py +40 -87
  76. castor_extractor/warehouse/databricks/client.py +3 -0
  77. castor_extractor/warehouse/redshift/queries/column.sql +0 -5
  78. castor_extractor/warehouse/salesforce/extract.py +2 -2
  79. castor_extractor/warehouse/salesforce/format.py +5 -3
  80. castor_extractor/warehouse/snowflake/queries/column.sql +0 -1
  81. castor_extractor/warehouse/synapse/queries/column.sql +0 -1
  82. {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/METADATA +9 -9
  83. {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/RECORD +86 -83
  84. castor_extractor/visualization/domo/client/client_test.py +0 -60
  85. castor_extractor/visualization/domo/constants.py +0 -6
  86. castor_extractor/visualization/looker/env.py +0 -48
  87. castor_extractor/visualization/looker/parameters.py +0 -78
  88. castor_extractor/visualization/qlik/constants.py +0 -3
  89. castor_extractor/visualization/sigma/constants.py +0 -4
  90. {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/LICENCE +0 -0
  91. {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/WHEEL +0 -0
  92. {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/entry_points.txt +0 -0
@@ -9,7 +9,7 @@ from ...assets import EXPORTED_FIELDS, MetabaseAsset
9
9
  from ...errors import MetabaseLoginError, SuperuserCredentialsRequired
10
10
  from ...types import IdsType
11
11
  from ..shared import DETAILS_KEY, get_dbname_from_details
12
- from .credentials import CredentialsApi, CredentialsApiKey, get_value
12
+ from .credentials import MetabaseApiCredentials
13
13
 
14
14
  logger = logging.getLogger(__name__)
15
15
 
@@ -30,13 +30,11 @@ class ApiClient:
30
30
 
31
31
  def __init__(
32
32
  self,
33
- **kwargs,
33
+ credentials: MetabaseApiCredentials,
34
34
  ):
35
- self._credentials = CredentialsApi(
36
- base_url=get_value(CredentialsApiKey.BASE_URL, kwargs),
37
- user=get_value(CredentialsApiKey.USER, kwargs),
38
- password=get_value(CredentialsApiKey.PASSWORD, kwargs),
39
- )
35
+ self.base_url = credentials.base_url
36
+
37
+ self._credentials = credentials
40
38
  self._session = requests.Session()
41
39
  self._session_id = self._login()
42
40
  self._check_permissions() # verify that the given user is superuser
@@ -46,13 +44,9 @@ class ApiClient:
46
44
  """return the name of the client"""
47
45
  return "Metabase/API"
48
46
 
49
- def base_url(self) -> str:
50
- """Return base_url from credentials"""
51
- return self._credentials.base_url
52
-
53
47
  def _url(self, endpoint: str) -> str:
54
48
  return URL_TEMPLATE.format(
55
- base_url=self._credentials.base_url,
49
+ base_url=self.base_url,
56
50
  endpoint=endpoint,
57
51
  )
58
52
 
@@ -85,7 +79,7 @@ class ApiClient:
85
79
  except HTTPError as err:
86
80
  if err.response.status_code == 403: # forbidden
87
81
  raise SuperuserCredentialsRequired(
88
- credentials_info=self._credentials.to_dict(hide=True),
82
+ credentials_info=self._credentials,
89
83
  error_details=err.args,
90
84
  )
91
85
  raise
@@ -101,7 +95,7 @@ class ApiClient:
101
95
 
102
96
  if not response.json().get("id"):
103
97
  raise MetabaseLoginError(
104
- credentials_info=self._credentials.to_dict(hide=True),
98
+ credentials_info=self._credentials,
105
99
  error_details=response.json(),
106
100
  )
107
101
 
@@ -1,45 +1,18 @@
1
- from enum import Enum
2
- from typing import Dict
1
+ from pydantic import Field, SecretStr
2
+ from pydantic_settings import BaseSettings, SettingsConfigDict
3
3
 
4
- from .....utils import from_env
4
+ METABASE_API_ENV_PREFIX = "CASTOR_METABASE_API_"
5
5
 
6
6
 
7
- class CredentialsApiKey(Enum):
8
- BASE_URL = "base_url"
9
- USER = "user"
10
- PASSWORD = "password" # noqa: S105
7
+ class MetabaseApiCredentials(BaseSettings):
8
+ """Metabase's credentials to connect to Metabase API"""
11
9
 
10
+ model_config = SettingsConfigDict(
11
+ env_prefix=METABASE_API_ENV_PREFIX,
12
+ extra="ignore",
13
+ populate_by_name=True,
14
+ )
12
15
 
13
- CREDENTIALS_ENV: Dict[CredentialsApiKey, str] = {
14
- CredentialsApiKey.BASE_URL: "CASTOR_METABASE_API_BASE_URL",
15
- CredentialsApiKey.USER: "CASTOR_METABASE_API_USERNAME",
16
- CredentialsApiKey.PASSWORD: "CASTOR_METABASE_API_PASSWORD",
17
- }
18
-
19
-
20
- def get_value(key: CredentialsApiKey, kwargs: dict) -> str:
21
- """
22
- Returns the value of the given key:
23
- - from kwargs in priority
24
- - from ENV if not provided (raises an error if not found in ENV)
25
- """
26
- env_key = CREDENTIALS_ENV[key]
27
- return str(kwargs.get(key.value) or from_env(env_key))
28
-
29
-
30
- class CredentialsApi:
31
- """ValueObject for the credentials"""
32
-
33
- def __init__(self, base_url: str, user: str, password: str):
34
- self.base_url = base_url
35
- self.user = user
36
- self.password = password
37
-
38
- def to_dict(self, hide: bool = False) -> Dict[str, str]:
39
- safe = (CredentialsApiKey.BASE_URL, CredentialsApiKey.USER)
40
- unsafe = (CredentialsApiKey.PASSWORD,)
41
-
42
- def val(k: CredentialsApiKey, v: str) -> str:
43
- return "*" + v[-4:] if hide and k in unsafe else v
44
-
45
- return {a.value: val(a, getattr(self, a.value)) for a in safe + unsafe}
16
+ base_url: str
17
+ user: str = Field(validation_alias="CASTOR_METABASE_API_USERNAME")
18
+ password: str = Field(repr=False)
@@ -1 +1,2 @@
1
1
  from .client import DbClient
2
+ from .credentials import MetabaseDbCredentials
@@ -1,23 +1,15 @@
1
1
  import json
2
2
  import logging
3
3
  import os
4
- from typing import Optional
5
4
 
6
- from sqlalchemy import create_engine
7
- from sqlalchemy.engine import Engine
8
5
  from sqlalchemy.exc import OperationalError
9
6
 
10
- from .....utils import (
11
- ExtractionQuery,
12
- PostgresClient,
13
- SerializedAsset,
14
- from_env,
15
- )
7
+ from .....utils import ExtractionQuery, PostgresClient, SerializedAsset
16
8
  from ...assets import EXPORTED_FIELDS, MetabaseAsset
17
9
  from ...errors import EncryptionSecretKeyRequired, MetabaseLoginError
18
10
  from ..decryption import decrypt
19
11
  from ..shared import DETAILS_KEY, get_dbname_from_details
20
- from .credentials import CredentialsDb, CredentialsDbKey, get_value
12
+ from .credentials import MetabaseDbCredentials
21
13
 
22
14
  logger = logging.getLogger(__name__)
23
15
 
@@ -25,16 +17,6 @@ CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
25
17
 
26
18
  SQL_FILE_PATH = "queries/{name}.sql"
27
19
 
28
- ENCRYPTION_SECRET_KEY = "CASTOR_METABASE_ENCRYPTION_SECRET_KEY" # noqa: S105
29
- REQUIRE_SSL_KEY = "CASTOR_METABASE_REQUIRE_SSL_KEY" # noqa: S105
30
-
31
-
32
- def _optional_arg(args: dict, client_key: str, env_key: str) -> Optional[str]:
33
- arg_value = args.get(client_key)
34
- if arg_value is not None:
35
- return arg_value
36
- return from_env(env_key, allow_missing=True)
37
-
38
20
 
39
21
  class DbClient(PostgresClient):
40
22
  """
@@ -52,20 +34,15 @@ class DbClient(PostgresClient):
52
34
 
53
35
  def __init__(
54
36
  self,
55
- **kwargs,
37
+ credentials: MetabaseDbCredentials,
56
38
  ):
57
- self._credentials = CredentialsDb.from_env(kwargs)
58
- self.encryption_secret_key = _optional_arg(
59
- kwargs, "encryption_secret_key", ENCRYPTION_SECRET_KEY
60
- )
61
- self.require_ssl = bool(
62
- _optional_arg(kwargs, "require_ssl", REQUIRE_SSL_KEY)
63
- )
39
+ self._credentials = credentials
40
+ self.require_ssl = self._credentials.require_ssl
64
41
  try:
65
- super().__init__(self._credentials.to_dict())
42
+ super().__init__(self._credentials.dict())
66
43
  except OperationalError as err:
67
44
  raise MetabaseLoginError(
68
- credentials_info=self._credentials.to_dict(hide=True),
45
+ credentials_info=self._credentials,
69
46
  error_details=err.args,
70
47
  )
71
48
 
@@ -75,9 +52,10 @@ class DbClient(PostgresClient):
75
52
  path = os.path.join(CURRENT_DIR, filename)
76
53
  with open(path, "r") as f:
77
54
  content = f.read()
78
- statement = content.format(schema=self._credentials.schema)
55
+ statement = content.format(schema=self._credentials.schema_)
79
56
  return ExtractionQuery(statement=statement, params={})
80
57
 
58
+ @property
81
59
  def base_url(self) -> str:
82
60
  """Fetches the `base_url` of the Metabase instance"""
83
61
  query = self._load_query(name="base_url")
@@ -94,12 +72,13 @@ class DbClient(PostgresClient):
94
72
  try:
95
73
  details = json.loads(db[DETAILS_KEY])
96
74
  except json.decoder.JSONDecodeError as err:
97
- if not self.encryption_secret_key:
75
+ encryption_key = self._credentials.encryption_secret_key
76
+ if not encryption_key:
98
77
  raise EncryptionSecretKeyRequired(
99
- credentials_info=self._credentials.to_dict(hide=True),
78
+ credentials_info=self._credentials,
100
79
  error_details=err.args,
101
80
  )
102
- decrypted = decrypt(db[DETAILS_KEY], self.encryption_secret_key)
81
+ decrypted = decrypt(db[DETAILS_KEY], encryption_key)
103
82
  details = json.loads(decrypted)
104
83
 
105
84
  db["dbname"] = get_dbname_from_details(details)
@@ -1,80 +1,26 @@
1
- from enum import Enum
2
- from typing import Dict
1
+ from typing import Optional
3
2
 
4
- from .....utils import from_env
3
+ from pydantic import Field, SecretStr
4
+ from pydantic_settings import BaseSettings, SettingsConfigDict
5
5
 
6
+ METABASE_DB_ENV_PREFIX = "CASTOR_METABASE_DB_"
6
7
 
7
- class CredentialsDbKey(Enum):
8
- HOST = "host"
9
- PORT = "port"
10
- DATABASE = "database"
11
- SCHEMA = "schema"
12
- USER = "user"
13
- PASSWORD = "password" # noqa: S105
14
8
 
9
+ class MetabaseDbCredentials(BaseSettings):
10
+ """Metabase's credentials to connect to Metabase DB"""
15
11
 
16
- CREDENTIALS_ENV: Dict[CredentialsDbKey, str] = {
17
- CredentialsDbKey.HOST: "CASTOR_METABASE_DB_HOST",
18
- CredentialsDbKey.PORT: "CASTOR_METABASE_DB_PORT",
19
- CredentialsDbKey.DATABASE: "CASTOR_METABASE_DB_DATABASE",
20
- CredentialsDbKey.SCHEMA: "CASTOR_METABASE_DB_SCHEMA",
21
- CredentialsDbKey.USER: "CASTOR_METABASE_DB_USERNAME",
22
- CredentialsDbKey.PASSWORD: "CASTOR_METABASE_DB_PASSWORD", # noqa: S105
23
- }
12
+ model_config = SettingsConfigDict(
13
+ env_prefix=METABASE_DB_ENV_PREFIX,
14
+ extra="ignore",
15
+ populate_by_name=True,
16
+ )
24
17
 
18
+ host: str
19
+ port: str
20
+ database: str
21
+ schema_: str = Field(validation_alias=f"{METABASE_DB_ENV_PREFIX}SCHEMA")
22
+ user: str = Field(validation_alias=f"{METABASE_DB_ENV_PREFIX}USERNAME")
23
+ password: str = Field(repr=False)
25
24
 
26
- def get_value(key: CredentialsDbKey, kwargs: dict) -> str:
27
- """
28
- Returns the value of the given key:
29
- - from kwargs in priority
30
- - from ENV if not provided (raises an error if not found in ENV)
31
- """
32
- env_key = CREDENTIALS_ENV[key]
33
- return str(kwargs.get(key.value) or from_env(env_key))
34
-
35
-
36
- class CredentialsDb:
37
- """ValueObject for the credentials"""
38
-
39
- def __init__(
40
- self,
41
- host: str,
42
- port: str,
43
- database: str,
44
- schema: str,
45
- user: str,
46
- password: str,
47
- ):
48
- self.host = host
49
- self.port = port
50
- self.database = database
51
- self.schema = schema
52
- self.user = user
53
- self.password = password
54
-
55
- @classmethod
56
- def from_env(cls, env: dict) -> "CredentialsDb":
57
- """returns a new CredentialsDb with values from ENV"""
58
- return CredentialsDb(
59
- host=get_value(CredentialsDbKey.HOST, env),
60
- port=get_value(CredentialsDbKey.PORT, env),
61
- database=get_value(CredentialsDbKey.DATABASE, env),
62
- schema=get_value(CredentialsDbKey.SCHEMA, env),
63
- user=get_value(CredentialsDbKey.USER, env),
64
- password=get_value(CredentialsDbKey.PASSWORD, env),
65
- )
66
-
67
- def to_dict(self, hide: bool = False) -> Dict[str, str]:
68
- safe = (
69
- CredentialsDbKey.HOST,
70
- CredentialsDbKey.PORT,
71
- CredentialsDbKey.DATABASE,
72
- CredentialsDbKey.SCHEMA,
73
- CredentialsDbKey.USER,
74
- )
75
- unsafe = (CredentialsDbKey.PASSWORD,)
76
-
77
- def val(k: CredentialsDbKey, v: str) -> str:
78
- return "*" + v[-4:] if hide and k in unsafe else v
79
-
80
- return {a.value: val(a, getattr(self, a.value)) for a in safe + unsafe}
25
+ encryption_secret_key: Optional[str] = Field(repr=False)
26
+ require_ssl: Optional[str]
@@ -1,8 +1,10 @@
1
1
  from typing import Any
2
2
 
3
+ from pydantic_settings import BaseSettings
4
+
3
5
 
4
6
  class MetabaseLoginError(ConnectionError):
5
- def __init__(self, credentials_info: dict, error_details: Any):
7
+ def __init__(self, credentials_info: BaseSettings, error_details: Any):
6
8
  msg = "Connection to Metabase failed!\n"
7
9
  msg += f"Credentials: {credentials_info}\n"
8
10
  msg += f"Details: {error_details}\n"
@@ -10,7 +12,7 @@ class MetabaseLoginError(ConnectionError):
10
12
 
11
13
 
12
14
  class SuperuserCredentialsRequired(AssertionError):
13
- def __init__(self, credentials_info: dict, error_details: Any):
15
+ def __init__(self, credentials_info: BaseSettings, error_details: Any):
14
16
  msg = "Superuser credentials are required!\n"
15
17
  msg += f"Credentials: {credentials_info}\n"
16
18
  msg += f"Details: {error_details}\n"
@@ -22,7 +24,7 @@ class EncryptionSecretKeyRequired(AssertionError):
22
24
  Raised when missing the encryption secret key while it was required.
23
25
  """
24
26
 
25
- def __init__(self, credentials_info: dict, error_details: Any):
27
+ def __init__(self, credentials_info: BaseSettings, error_details: Any):
26
28
  msg = "Encryption secret key is required!\n"
27
29
  msg += f"Credentials: {credentials_info}\n"
28
30
  msg += f"Details: {error_details}\n"
@@ -40,12 +40,12 @@ def iterate_all_data(
40
40
  )
41
41
 
42
42
 
43
- def extract_all(client: ClientMetabase, **kwargs: str) -> None:
43
+ def extract_all(client: ClientMetabase, **kwargs) -> None:
44
44
  """
45
45
  Extract Data From metabase
46
46
  Store the output files locally under the given output_directory
47
47
  """
48
- output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
48
+ output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
49
49
  ts = current_timestamp()
50
50
 
51
51
  for key, data in iterate_all_data(client):
@@ -55,6 +55,6 @@ def extract_all(client: ClientMetabase, **kwargs: str) -> None:
55
55
  write_summary(
56
56
  output_directory,
57
57
  ts,
58
- base_url=client.base_url(),
58
+ base_url=client.base_url,
59
59
  client_name=client.name(),
60
60
  )
@@ -1,3 +1,3 @@
1
1
  from .assets import ModeAnalyticsAsset
2
- from .client import Client
2
+ from .client import Client, ModeCredentials as Credentials
3
3
  from .extract import extract_all, iterate_all_data
@@ -1 +1,2 @@
1
1
  from .client import Client
2
+ from .credentials import ModeCredentials
@@ -2,6 +2,7 @@ import logging
2
2
  from typing import Dict, List, Optional, cast
3
3
 
4
4
  import requests
5
+ from requests.auth import HTTPBasicAuth
5
6
 
6
7
  from ....utils import retry
7
8
  from ..assets import (
@@ -22,7 +23,7 @@ from .constants import (
22
23
  RETRY_JITTER_MS,
23
24
  RETRY_STRATEGY,
24
25
  )
25
- from .credentials import Credentials, CredentialsKey, get_value
26
+ from .credentials import ModeCredentials
26
27
 
27
28
  logger = logging.getLogger(__name__)
28
29
 
@@ -40,20 +41,16 @@ class Client:
40
41
 
41
42
  def __init__(
42
43
  self,
43
- **kwargs,
44
+ credentials: ModeCredentials,
44
45
  ):
45
- self._credentials = Credentials(
46
- host=get_value(CredentialsKey.HOST, kwargs).rstrip("/"),
47
- workspace=get_value(CredentialsKey.WORKSPACE, kwargs),
48
- token=get_value(CredentialsKey.TOKEN, kwargs),
49
- secret=get_value(CredentialsKey.SECRET, kwargs),
50
- )
46
+ self._credentials = credentials
51
47
  self._session = requests.Session()
52
- if not kwargs.get("no_checks"):
53
- self._check_connection()
48
+
49
+ def authenticate(self) -> HTTPBasicAuth:
50
+ return HTTPBasicAuth(self._credentials.token, self._credentials.secret)
54
51
 
55
52
  def _check_connection(self):
56
- authentication = self._credentials.authentication()
53
+ authentication = self.authenticate()
57
54
  url = self._url(with_workspace=False) + "/account"
58
55
  response = self._session.get(url, auth=authentication)
59
56
  self._handle_response(response)
@@ -121,7 +118,7 @@ class Client:
121
118
  report: Optional[str] = None,
122
119
  resource_name: Optional[str] = None,
123
120
  ) -> RawData:
124
- authentication = self._credentials.authentication()
121
+ authentication = self.authenticate()
125
122
  url = self._url(with_workspace, space, report, resource_name)
126
123
  logger.info(f"Calling {url}")
127
124
  response = self._session.get(url, auth=authentication)
@@ -2,20 +2,20 @@ import json
2
2
 
3
3
  from ....utils import load_file
4
4
  from ..assets import EXPORTED_FIELDS, ModeAnalyticsAsset
5
- from .client import Client
5
+ from .client import Client, ModeCredentials
6
6
 
7
7
  _HOST = "https://mode.com"
8
8
  _WORKSPACE = "castor"
9
9
 
10
10
 
11
11
  def _dummy_client() -> Client:
12
- return Client( # noqa: S106
12
+ credentials = ModeCredentials(
13
13
  host=_HOST,
14
14
  workspace=_WORKSPACE,
15
15
  token="dummy-token",
16
16
  secret="******",
17
- no_checks=True,
18
17
  )
18
+ return Client(credentials=credentials)
19
19
 
20
20
 
21
21
  def test__url():
@@ -1,57 +1,24 @@
1
- from enum import Enum
2
- from typing import Dict
1
+ from pydantic import Field, SecretStr, field_validator
2
+ from pydantic_settings import BaseSettings, SettingsConfigDict
3
3
 
4
- from requests.auth import HTTPBasicAuth
4
+ MODE_ENV_PREFIX = "CASTOR_MODE_ANALYTICS_"
5
5
 
6
- from ....utils import from_env
7
6
 
7
+ class ModeCredentials(BaseSettings):
8
+ """Class holding Mode credentials attributes"""
8
9
 
9
- class CredentialsKey(Enum):
10
- HOST = "host"
11
- WORKSPACE = "workspace"
12
- TOKEN = "token" # noqa: S105
13
- SECRET = "secret" # noqa: S105
10
+ model_config = SettingsConfigDict(
11
+ env_prefix=MODE_ENV_PREFIX,
12
+ extra="ignore",
13
+ populate_by_name=True,
14
+ )
14
15
 
16
+ host: str
17
+ secret: str = Field(repr=False)
18
+ token: str
19
+ workspace: str
15
20
 
16
- CREDENTIALS_ENV: Dict[CredentialsKey, str] = {
17
- CredentialsKey.HOST: "CASTOR_MODE_ANALYTICS_HOST",
18
- CredentialsKey.WORKSPACE: "CASTOR_MODE_ANALYTICS_WORKSPACE",
19
- CredentialsKey.TOKEN: "CASTOR_MODE_ANALYTICS_TOKEN",
20
- CredentialsKey.SECRET: "CASTOR_MODE_ANALYTICS_SECRET",
21
- }
22
-
23
-
24
- def get_value(key: CredentialsKey, kwargs: dict) -> str:
25
- """
26
- Returns the value of the given key:
27
- - from kwargs in priority
28
- - from ENV if not provided (raises an error if not found in ENV)
29
- """
30
- env_key = CREDENTIALS_ENV[key]
31
- return str(kwargs.get(key.value) or from_env(env_key))
32
-
33
-
34
- class Credentials:
35
- """ValueObject for the credentials"""
36
-
37
- def __init__(self, host: str, workspace: str, token: str, secret: str):
38
- self.host = host
39
- self.workspace = workspace
40
- self.token = token
41
- self.secret = secret
42
-
43
- def to_dict(self, hide: bool = False) -> Dict[str, str]:
44
- safe = (
45
- CredentialsKey.HOST,
46
- CredentialsKey.WORKSPACE,
47
- CredentialsKey.TOKEN,
48
- )
49
- unsafe = (CredentialsKey.SECRET,)
50
-
51
- def val(k: CredentialsKey, v: str) -> str:
52
- return "*" + v[-4:] if hide and k in unsafe else v
53
-
54
- return {a.value: val(a, getattr(self, a.value)) for a in safe + unsafe}
55
-
56
- def authentication(self) -> HTTPBasicAuth:
57
- return HTTPBasicAuth(self.token, self.secret)
21
+ @field_validator("host", mode="before")
22
+ @classmethod
23
+ def _check_base_url(cls, host: str) -> str:
24
+ return host.rstrip("/")
@@ -11,7 +11,7 @@ from ...utils import (
11
11
  write_summary,
12
12
  )
13
13
  from .assets import ModeAnalyticsAsset as Asset
14
- from .client import Client
14
+ from .client import Client, ModeCredentials
15
15
 
16
16
  logger = logging.getLogger(__name__)
17
17
 
@@ -37,9 +37,12 @@ def iterate_all_data(
37
37
  yield Asset.MEMBER, deep_serialize(members)
38
38
 
39
39
 
40
- def extract_all(client: Client, **kwargs: str) -> None:
40
+ def extract_all(**kwargs) -> None:
41
41
  """Extract Data From Mode Analytics and store it locally in files under the output_directory"""
42
- output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
42
+ output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
43
+ credentials = ModeCredentials(**kwargs)
44
+ client = Client(credentials)
45
+
43
46
  ts = current_timestamp()
44
47
 
45
48
  for key, data in iterate_all_data(client):
@@ -1,3 +1,3 @@
1
1
  from .assets import PowerBiAsset
2
- from .client import Client, Credentials
2
+ from .client import Client, PowerbiCredentials, Urls
3
3
  from .extract import extract_all
@@ -1,2 +1,3 @@
1
- from .credentials import Credentials
1
+ from .constants import Urls
2
+ from .credentials import PowerbiCredentials
2
3
  from .rest import Client
@@ -1,20 +1,28 @@
1
- from dataclasses import field
2
1
  from typing import List, Optional
3
2
 
4
- from pydantic.dataclasses import dataclass
3
+ from pydantic import Field, field_validator
4
+ from pydantic_settings import BaseSettings, SettingsConfigDict
5
5
 
6
6
  from .constants import Urls
7
7
 
8
+ POWERBI_ENV_PREFIX = "CASTOR_POWERBI_"
8
9
 
9
- @dataclass
10
- class Credentials:
10
+
11
+ class PowerbiCredentials(BaseSettings):
11
12
  """Class to handle PowerBI rest API permissions"""
12
13
 
14
+ model_config = SettingsConfigDict(
15
+ env_prefix=POWERBI_ENV_PREFIX,
16
+ extra="ignore",
17
+ populate_by_name=True,
18
+ )
19
+
13
20
  client_id: str
14
21
  tenant_id: str
15
- secret: str = field(metadata={"sensitive": True})
16
- scopes: Optional[List[str]] = None
22
+ secret: str = Field(repr=False)
23
+ scopes: List[str] = [Urls.DEFAULT_SCOPE]
17
24
 
18
- def __post_init__(self):
19
- if self.scopes is None:
20
- self.scopes = [Urls.DEFAULT_SCOPE]
25
+ @field_validator("scopes", mode="before")
26
+ @classmethod
27
+ def _check_scopes(cls, scopes: Optional[List[str]]) -> List[str]:
28
+ return scopes if scopes is not None else [Urls.DEFAULT_SCOPE]
@@ -1,5 +1,5 @@
1
1
  from .constants import Urls
2
- from .credentials import Credentials
2
+ from .credentials import PowerbiCredentials
3
3
 
4
4
 
5
5
  def test_credentials():
@@ -8,15 +8,23 @@ def test_credentials():
8
8
  secret = "🤫"
9
9
 
10
10
  # no scopes provided
11
- credentials = Credentials(
11
+ credentials = PowerbiCredentials(
12
12
  tenant_id=tenant_id,
13
13
  client_id=client_id,
14
14
  secret=secret,
15
15
  )
16
16
  assert credentials.scopes == [Urls.DEFAULT_SCOPE]
17
17
 
18
+ credentials = PowerbiCredentials(
19
+ tenant_id=tenant_id,
20
+ client_id=client_id,
21
+ secret=secret,
22
+ scopes=None,
23
+ )
24
+ assert credentials.scopes == [Urls.DEFAULT_SCOPE]
25
+
18
26
  # empty scopes
19
- credentials = Credentials(
27
+ credentials = PowerbiCredentials(
20
28
  tenant_id=tenant_id,
21
29
  client_id=client_id,
22
30
  secret=secret,
@@ -26,7 +34,7 @@ def test_credentials():
26
34
 
27
35
  # with scopes
28
36
  scopes = ["foo"]
29
- credentials = Credentials(
37
+ credentials = PowerbiCredentials(
30
38
  tenant_id=tenant_id,
31
39
  client_id=client_id,
32
40
  secret=secret,
@@ -18,7 +18,7 @@ from .constants import (
18
18
  QueryParams,
19
19
  Urls,
20
20
  )
21
- from .credentials import Credentials
21
+ from .credentials import PowerbiCredentials
22
22
  from .utils import batch_size_is_valid_or_assert, datetime_is_recent_or_assert
23
23
 
24
24
  logger = logging.getLogger(__name__)
@@ -62,7 +62,7 @@ class Client:
62
62
  https://learn.microsoft.com/en-us/rest/api/power-bi/admin
63
63
  """
64
64
 
65
- def __init__(self, credentials: Credentials):
65
+ def __init__(self, credentials: PowerbiCredentials):
66
66
  self.creds = credentials
67
67
  client_app = f"{Urls.CLIENT_APP_BASE}{self.creds.tenant_id}"
68
68
  self.app = msal.ConfidentialClientApplication(