castor-extractor 0.17.4__py3-none-any.whl → 0.18.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of castor-extractor might be problematic. Click here for more details.
- CHANGELOG.md +28 -0
- DockerfileUsage.md +21 -0
- castor_extractor/commands/extract_domo.py +2 -10
- castor_extractor/commands/extract_looker.py +2 -13
- castor_extractor/commands/extract_metabase_api.py +5 -10
- castor_extractor/commands/extract_metabase_db.py +6 -16
- castor_extractor/commands/extract_mode.py +2 -13
- castor_extractor/commands/extract_powerbi.py +2 -8
- castor_extractor/commands/extract_qlik.py +2 -7
- castor_extractor/commands/extract_salesforce.py +3 -12
- castor_extractor/commands/extract_salesforce_reporting.py +2 -10
- castor_extractor/commands/extract_sigma.py +2 -7
- castor_extractor/utils/__init__.py +3 -1
- castor_extractor/utils/argument_parser.py +7 -0
- castor_extractor/utils/argument_parser_test.py +25 -0
- castor_extractor/utils/collection.py +8 -0
- castor_extractor/utils/safe_request.py +57 -0
- castor_extractor/utils/safe_request_test.py +77 -0
- castor_extractor/utils/salesforce/__init__.py +1 -2
- castor_extractor/utils/salesforce/constants.py +0 -11
- castor_extractor/utils/salesforce/credentials.py +22 -45
- castor_extractor/visualization/domo/__init__.py +1 -1
- castor_extractor/visualization/domo/client/__init__.py +1 -1
- castor_extractor/visualization/domo/client/client.py +37 -52
- castor_extractor/visualization/domo/client/credentials.py +14 -27
- castor_extractor/visualization/domo/extract.py +5 -26
- castor_extractor/visualization/looker/__init__.py +6 -1
- castor_extractor/visualization/looker/api/__init__.py +2 -1
- castor_extractor/visualization/looker/api/client.py +6 -4
- castor_extractor/visualization/looker/api/client_test.py +5 -3
- castor_extractor/visualization/looker/api/credentials.py +33 -0
- castor_extractor/visualization/looker/api/extraction_parameters.py +38 -0
- castor_extractor/visualization/looker/api/sdk.py +2 -28
- castor_extractor/visualization/looker/constant.py +2 -27
- castor_extractor/visualization/looker/constants.py +17 -0
- castor_extractor/visualization/looker/extract.py +29 -29
- castor_extractor/visualization/metabase/__init__.py +6 -1
- castor_extractor/visualization/metabase/client/__init__.py +2 -2
- castor_extractor/visualization/metabase/client/api/__init__.py +1 -0
- castor_extractor/visualization/metabase/client/api/client.py +8 -14
- castor_extractor/visualization/metabase/client/api/credentials.py +13 -40
- castor_extractor/visualization/metabase/client/db/__init__.py +1 -0
- castor_extractor/visualization/metabase/client/db/client.py +13 -34
- castor_extractor/visualization/metabase/client/db/credentials.py +19 -73
- castor_extractor/visualization/metabase/errors.py +5 -3
- castor_extractor/visualization/metabase/extract.py +3 -3
- castor_extractor/visualization/mode/__init__.py +1 -1
- castor_extractor/visualization/mode/client/__init__.py +1 -0
- castor_extractor/visualization/mode/client/client.py +9 -12
- castor_extractor/visualization/mode/client/client_test.py +3 -3
- castor_extractor/visualization/mode/client/credentials.py +18 -51
- castor_extractor/visualization/mode/extract.py +6 -3
- castor_extractor/visualization/powerbi/__init__.py +1 -1
- castor_extractor/visualization/powerbi/client/__init__.py +2 -1
- castor_extractor/visualization/powerbi/client/credentials.py +17 -9
- castor_extractor/visualization/powerbi/client/credentials_test.py +12 -4
- castor_extractor/visualization/powerbi/client/rest.py +2 -2
- castor_extractor/visualization/powerbi/client/rest_test.py +2 -2
- castor_extractor/visualization/powerbi/extract.py +5 -16
- castor_extractor/visualization/qlik/__init__.py +5 -1
- castor_extractor/visualization/qlik/client/__init__.py +1 -0
- castor_extractor/visualization/qlik/client/engine/__init__.py +1 -0
- castor_extractor/visualization/qlik/client/engine/client.py +5 -6
- castor_extractor/visualization/qlik/client/engine/credentials.py +26 -0
- castor_extractor/visualization/qlik/client/master.py +5 -11
- castor_extractor/visualization/qlik/client/rest.py +4 -4
- castor_extractor/visualization/qlik/client/rest_test.py +6 -2
- castor_extractor/visualization/qlik/extract.py +6 -13
- castor_extractor/visualization/salesforce_reporting/extract.py +6 -20
- castor_extractor/visualization/sigma/__init__.py +1 -1
- castor_extractor/visualization/sigma/client/__init__.py +1 -1
- castor_extractor/visualization/sigma/client/client.py +5 -4
- castor_extractor/visualization/sigma/client/credentials.py +12 -28
- castor_extractor/visualization/sigma/extract.py +5 -18
- castor_extractor/visualization/tableau_revamp/client/credentials.py +40 -87
- castor_extractor/warehouse/databricks/client.py +3 -0
- castor_extractor/warehouse/redshift/queries/column.sql +0 -5
- castor_extractor/warehouse/salesforce/extract.py +2 -2
- castor_extractor/warehouse/salesforce/format.py +5 -3
- castor_extractor/warehouse/snowflake/queries/column.sql +0 -1
- castor_extractor/warehouse/synapse/queries/column.sql +0 -1
- {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/METADATA +9 -9
- {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/RECORD +86 -83
- castor_extractor/visualization/domo/client/client_test.py +0 -60
- castor_extractor/visualization/domo/constants.py +0 -6
- castor_extractor/visualization/looker/env.py +0 -48
- castor_extractor/visualization/looker/parameters.py +0 -78
- castor_extractor/visualization/qlik/constants.py +0 -3
- castor_extractor/visualization/sigma/constants.py +0 -4
- {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/LICENCE +0 -0
- {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/WHEEL +0 -0
- {castor_extractor-0.17.4.dist-info → castor_extractor-0.18.5.dist-info}/entry_points.txt +0 -0
|
@@ -9,7 +9,7 @@ from ...assets import EXPORTED_FIELDS, MetabaseAsset
|
|
|
9
9
|
from ...errors import MetabaseLoginError, SuperuserCredentialsRequired
|
|
10
10
|
from ...types import IdsType
|
|
11
11
|
from ..shared import DETAILS_KEY, get_dbname_from_details
|
|
12
|
-
from .credentials import
|
|
12
|
+
from .credentials import MetabaseApiCredentials
|
|
13
13
|
|
|
14
14
|
logger = logging.getLogger(__name__)
|
|
15
15
|
|
|
@@ -30,13 +30,11 @@ class ApiClient:
|
|
|
30
30
|
|
|
31
31
|
def __init__(
|
|
32
32
|
self,
|
|
33
|
-
|
|
33
|
+
credentials: MetabaseApiCredentials,
|
|
34
34
|
):
|
|
35
|
-
self.
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
password=get_value(CredentialsApiKey.PASSWORD, kwargs),
|
|
39
|
-
)
|
|
35
|
+
self.base_url = credentials.base_url
|
|
36
|
+
|
|
37
|
+
self._credentials = credentials
|
|
40
38
|
self._session = requests.Session()
|
|
41
39
|
self._session_id = self._login()
|
|
42
40
|
self._check_permissions() # verify that the given user is superuser
|
|
@@ -46,13 +44,9 @@ class ApiClient:
|
|
|
46
44
|
"""return the name of the client"""
|
|
47
45
|
return "Metabase/API"
|
|
48
46
|
|
|
49
|
-
def base_url(self) -> str:
|
|
50
|
-
"""Return base_url from credentials"""
|
|
51
|
-
return self._credentials.base_url
|
|
52
|
-
|
|
53
47
|
def _url(self, endpoint: str) -> str:
|
|
54
48
|
return URL_TEMPLATE.format(
|
|
55
|
-
base_url=self.
|
|
49
|
+
base_url=self.base_url,
|
|
56
50
|
endpoint=endpoint,
|
|
57
51
|
)
|
|
58
52
|
|
|
@@ -85,7 +79,7 @@ class ApiClient:
|
|
|
85
79
|
except HTTPError as err:
|
|
86
80
|
if err.response.status_code == 403: # forbidden
|
|
87
81
|
raise SuperuserCredentialsRequired(
|
|
88
|
-
credentials_info=self._credentials
|
|
82
|
+
credentials_info=self._credentials,
|
|
89
83
|
error_details=err.args,
|
|
90
84
|
)
|
|
91
85
|
raise
|
|
@@ -101,7 +95,7 @@ class ApiClient:
|
|
|
101
95
|
|
|
102
96
|
if not response.json().get("id"):
|
|
103
97
|
raise MetabaseLoginError(
|
|
104
|
-
credentials_info=self._credentials
|
|
98
|
+
credentials_info=self._credentials,
|
|
105
99
|
error_details=response.json(),
|
|
106
100
|
)
|
|
107
101
|
|
|
@@ -1,45 +1,18 @@
|
|
|
1
|
-
from
|
|
2
|
-
from
|
|
1
|
+
from pydantic import Field, SecretStr
|
|
2
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
METABASE_API_ENV_PREFIX = "CASTOR_METABASE_API_"
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
class
|
|
8
|
-
|
|
9
|
-
USER = "user"
|
|
10
|
-
PASSWORD = "password" # noqa: S105
|
|
7
|
+
class MetabaseApiCredentials(BaseSettings):
|
|
8
|
+
"""Metabase's credentials to connect to Metabase API"""
|
|
11
9
|
|
|
10
|
+
model_config = SettingsConfigDict(
|
|
11
|
+
env_prefix=METABASE_API_ENV_PREFIX,
|
|
12
|
+
extra="ignore",
|
|
13
|
+
populate_by_name=True,
|
|
14
|
+
)
|
|
12
15
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
CredentialsApiKey.PASSWORD: "CASTOR_METABASE_API_PASSWORD",
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def get_value(key: CredentialsApiKey, kwargs: dict) -> str:
|
|
21
|
-
"""
|
|
22
|
-
Returns the value of the given key:
|
|
23
|
-
- from kwargs in priority
|
|
24
|
-
- from ENV if not provided (raises an error if not found in ENV)
|
|
25
|
-
"""
|
|
26
|
-
env_key = CREDENTIALS_ENV[key]
|
|
27
|
-
return str(kwargs.get(key.value) or from_env(env_key))
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
class CredentialsApi:
|
|
31
|
-
"""ValueObject for the credentials"""
|
|
32
|
-
|
|
33
|
-
def __init__(self, base_url: str, user: str, password: str):
|
|
34
|
-
self.base_url = base_url
|
|
35
|
-
self.user = user
|
|
36
|
-
self.password = password
|
|
37
|
-
|
|
38
|
-
def to_dict(self, hide: bool = False) -> Dict[str, str]:
|
|
39
|
-
safe = (CredentialsApiKey.BASE_URL, CredentialsApiKey.USER)
|
|
40
|
-
unsafe = (CredentialsApiKey.PASSWORD,)
|
|
41
|
-
|
|
42
|
-
def val(k: CredentialsApiKey, v: str) -> str:
|
|
43
|
-
return "*" + v[-4:] if hide and k in unsafe else v
|
|
44
|
-
|
|
45
|
-
return {a.value: val(a, getattr(self, a.value)) for a in safe + unsafe}
|
|
16
|
+
base_url: str
|
|
17
|
+
user: str = Field(validation_alias="CASTOR_METABASE_API_USERNAME")
|
|
18
|
+
password: str = Field(repr=False)
|
|
@@ -1,23 +1,15 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
3
|
import os
|
|
4
|
-
from typing import Optional
|
|
5
4
|
|
|
6
|
-
from sqlalchemy import create_engine
|
|
7
|
-
from sqlalchemy.engine import Engine
|
|
8
5
|
from sqlalchemy.exc import OperationalError
|
|
9
6
|
|
|
10
|
-
from .....utils import
|
|
11
|
-
ExtractionQuery,
|
|
12
|
-
PostgresClient,
|
|
13
|
-
SerializedAsset,
|
|
14
|
-
from_env,
|
|
15
|
-
)
|
|
7
|
+
from .....utils import ExtractionQuery, PostgresClient, SerializedAsset
|
|
16
8
|
from ...assets import EXPORTED_FIELDS, MetabaseAsset
|
|
17
9
|
from ...errors import EncryptionSecretKeyRequired, MetabaseLoginError
|
|
18
10
|
from ..decryption import decrypt
|
|
19
11
|
from ..shared import DETAILS_KEY, get_dbname_from_details
|
|
20
|
-
from .credentials import
|
|
12
|
+
from .credentials import MetabaseDbCredentials
|
|
21
13
|
|
|
22
14
|
logger = logging.getLogger(__name__)
|
|
23
15
|
|
|
@@ -25,16 +17,6 @@ CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
|
25
17
|
|
|
26
18
|
SQL_FILE_PATH = "queries/{name}.sql"
|
|
27
19
|
|
|
28
|
-
ENCRYPTION_SECRET_KEY = "CASTOR_METABASE_ENCRYPTION_SECRET_KEY" # noqa: S105
|
|
29
|
-
REQUIRE_SSL_KEY = "CASTOR_METABASE_REQUIRE_SSL_KEY" # noqa: S105
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def _optional_arg(args: dict, client_key: str, env_key: str) -> Optional[str]:
|
|
33
|
-
arg_value = args.get(client_key)
|
|
34
|
-
if arg_value is not None:
|
|
35
|
-
return arg_value
|
|
36
|
-
return from_env(env_key, allow_missing=True)
|
|
37
|
-
|
|
38
20
|
|
|
39
21
|
class DbClient(PostgresClient):
|
|
40
22
|
"""
|
|
@@ -52,20 +34,15 @@ class DbClient(PostgresClient):
|
|
|
52
34
|
|
|
53
35
|
def __init__(
|
|
54
36
|
self,
|
|
55
|
-
|
|
37
|
+
credentials: MetabaseDbCredentials,
|
|
56
38
|
):
|
|
57
|
-
self._credentials =
|
|
58
|
-
self.
|
|
59
|
-
kwargs, "encryption_secret_key", ENCRYPTION_SECRET_KEY
|
|
60
|
-
)
|
|
61
|
-
self.require_ssl = bool(
|
|
62
|
-
_optional_arg(kwargs, "require_ssl", REQUIRE_SSL_KEY)
|
|
63
|
-
)
|
|
39
|
+
self._credentials = credentials
|
|
40
|
+
self.require_ssl = self._credentials.require_ssl
|
|
64
41
|
try:
|
|
65
|
-
super().__init__(self._credentials.
|
|
42
|
+
super().__init__(self._credentials.dict())
|
|
66
43
|
except OperationalError as err:
|
|
67
44
|
raise MetabaseLoginError(
|
|
68
|
-
credentials_info=self._credentials
|
|
45
|
+
credentials_info=self._credentials,
|
|
69
46
|
error_details=err.args,
|
|
70
47
|
)
|
|
71
48
|
|
|
@@ -75,9 +52,10 @@ class DbClient(PostgresClient):
|
|
|
75
52
|
path = os.path.join(CURRENT_DIR, filename)
|
|
76
53
|
with open(path, "r") as f:
|
|
77
54
|
content = f.read()
|
|
78
|
-
statement = content.format(schema=self._credentials.
|
|
55
|
+
statement = content.format(schema=self._credentials.schema_)
|
|
79
56
|
return ExtractionQuery(statement=statement, params={})
|
|
80
57
|
|
|
58
|
+
@property
|
|
81
59
|
def base_url(self) -> str:
|
|
82
60
|
"""Fetches the `base_url` of the Metabase instance"""
|
|
83
61
|
query = self._load_query(name="base_url")
|
|
@@ -94,12 +72,13 @@ class DbClient(PostgresClient):
|
|
|
94
72
|
try:
|
|
95
73
|
details = json.loads(db[DETAILS_KEY])
|
|
96
74
|
except json.decoder.JSONDecodeError as err:
|
|
97
|
-
|
|
75
|
+
encryption_key = self._credentials.encryption_secret_key
|
|
76
|
+
if not encryption_key:
|
|
98
77
|
raise EncryptionSecretKeyRequired(
|
|
99
|
-
credentials_info=self._credentials
|
|
78
|
+
credentials_info=self._credentials,
|
|
100
79
|
error_details=err.args,
|
|
101
80
|
)
|
|
102
|
-
decrypted = decrypt(db[DETAILS_KEY],
|
|
81
|
+
decrypted = decrypt(db[DETAILS_KEY], encryption_key)
|
|
103
82
|
details = json.loads(decrypted)
|
|
104
83
|
|
|
105
84
|
db["dbname"] = get_dbname_from_details(details)
|
|
@@ -1,80 +1,26 @@
|
|
|
1
|
-
from
|
|
2
|
-
from typing import Dict
|
|
1
|
+
from typing import Optional
|
|
3
2
|
|
|
4
|
-
from
|
|
3
|
+
from pydantic import Field, SecretStr
|
|
4
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
5
5
|
|
|
6
|
+
METABASE_DB_ENV_PREFIX = "CASTOR_METABASE_DB_"
|
|
6
7
|
|
|
7
|
-
class CredentialsDbKey(Enum):
|
|
8
|
-
HOST = "host"
|
|
9
|
-
PORT = "port"
|
|
10
|
-
DATABASE = "database"
|
|
11
|
-
SCHEMA = "schema"
|
|
12
|
-
USER = "user"
|
|
13
|
-
PASSWORD = "password" # noqa: S105
|
|
14
8
|
|
|
9
|
+
class MetabaseDbCredentials(BaseSettings):
|
|
10
|
+
"""Metabase's credentials to connect to Metabase DB"""
|
|
15
11
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
CredentialsDbKey.USER: "CASTOR_METABASE_DB_USERNAME",
|
|
22
|
-
CredentialsDbKey.PASSWORD: "CASTOR_METABASE_DB_PASSWORD", # noqa: S105
|
|
23
|
-
}
|
|
12
|
+
model_config = SettingsConfigDict(
|
|
13
|
+
env_prefix=METABASE_DB_ENV_PREFIX,
|
|
14
|
+
extra="ignore",
|
|
15
|
+
populate_by_name=True,
|
|
16
|
+
)
|
|
24
17
|
|
|
18
|
+
host: str
|
|
19
|
+
port: str
|
|
20
|
+
database: str
|
|
21
|
+
schema_: str = Field(validation_alias=f"{METABASE_DB_ENV_PREFIX}SCHEMA")
|
|
22
|
+
user: str = Field(validation_alias=f"{METABASE_DB_ENV_PREFIX}USERNAME")
|
|
23
|
+
password: str = Field(repr=False)
|
|
25
24
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
Returns the value of the given key:
|
|
29
|
-
- from kwargs in priority
|
|
30
|
-
- from ENV if not provided (raises an error if not found in ENV)
|
|
31
|
-
"""
|
|
32
|
-
env_key = CREDENTIALS_ENV[key]
|
|
33
|
-
return str(kwargs.get(key.value) or from_env(env_key))
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class CredentialsDb:
|
|
37
|
-
"""ValueObject for the credentials"""
|
|
38
|
-
|
|
39
|
-
def __init__(
|
|
40
|
-
self,
|
|
41
|
-
host: str,
|
|
42
|
-
port: str,
|
|
43
|
-
database: str,
|
|
44
|
-
schema: str,
|
|
45
|
-
user: str,
|
|
46
|
-
password: str,
|
|
47
|
-
):
|
|
48
|
-
self.host = host
|
|
49
|
-
self.port = port
|
|
50
|
-
self.database = database
|
|
51
|
-
self.schema = schema
|
|
52
|
-
self.user = user
|
|
53
|
-
self.password = password
|
|
54
|
-
|
|
55
|
-
@classmethod
|
|
56
|
-
def from_env(cls, env: dict) -> "CredentialsDb":
|
|
57
|
-
"""returns a new CredentialsDb with values from ENV"""
|
|
58
|
-
return CredentialsDb(
|
|
59
|
-
host=get_value(CredentialsDbKey.HOST, env),
|
|
60
|
-
port=get_value(CredentialsDbKey.PORT, env),
|
|
61
|
-
database=get_value(CredentialsDbKey.DATABASE, env),
|
|
62
|
-
schema=get_value(CredentialsDbKey.SCHEMA, env),
|
|
63
|
-
user=get_value(CredentialsDbKey.USER, env),
|
|
64
|
-
password=get_value(CredentialsDbKey.PASSWORD, env),
|
|
65
|
-
)
|
|
66
|
-
|
|
67
|
-
def to_dict(self, hide: bool = False) -> Dict[str, str]:
|
|
68
|
-
safe = (
|
|
69
|
-
CredentialsDbKey.HOST,
|
|
70
|
-
CredentialsDbKey.PORT,
|
|
71
|
-
CredentialsDbKey.DATABASE,
|
|
72
|
-
CredentialsDbKey.SCHEMA,
|
|
73
|
-
CredentialsDbKey.USER,
|
|
74
|
-
)
|
|
75
|
-
unsafe = (CredentialsDbKey.PASSWORD,)
|
|
76
|
-
|
|
77
|
-
def val(k: CredentialsDbKey, v: str) -> str:
|
|
78
|
-
return "*" + v[-4:] if hide and k in unsafe else v
|
|
79
|
-
|
|
80
|
-
return {a.value: val(a, getattr(self, a.value)) for a in safe + unsafe}
|
|
25
|
+
encryption_secret_key: Optional[str] = Field(repr=False)
|
|
26
|
+
require_ssl: Optional[str]
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
from typing import Any
|
|
2
2
|
|
|
3
|
+
from pydantic_settings import BaseSettings
|
|
4
|
+
|
|
3
5
|
|
|
4
6
|
class MetabaseLoginError(ConnectionError):
|
|
5
|
-
def __init__(self, credentials_info:
|
|
7
|
+
def __init__(self, credentials_info: BaseSettings, error_details: Any):
|
|
6
8
|
msg = "Connection to Metabase failed!\n"
|
|
7
9
|
msg += f"Credentials: {credentials_info}\n"
|
|
8
10
|
msg += f"Details: {error_details}\n"
|
|
@@ -10,7 +12,7 @@ class MetabaseLoginError(ConnectionError):
|
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
class SuperuserCredentialsRequired(AssertionError):
|
|
13
|
-
def __init__(self, credentials_info:
|
|
15
|
+
def __init__(self, credentials_info: BaseSettings, error_details: Any):
|
|
14
16
|
msg = "Superuser credentials are required!\n"
|
|
15
17
|
msg += f"Credentials: {credentials_info}\n"
|
|
16
18
|
msg += f"Details: {error_details}\n"
|
|
@@ -22,7 +24,7 @@ class EncryptionSecretKeyRequired(AssertionError):
|
|
|
22
24
|
Raised when missing the encryption secret key while it was required.
|
|
23
25
|
"""
|
|
24
26
|
|
|
25
|
-
def __init__(self, credentials_info:
|
|
27
|
+
def __init__(self, credentials_info: BaseSettings, error_details: Any):
|
|
26
28
|
msg = "Encryption secret key is required!\n"
|
|
27
29
|
msg += f"Credentials: {credentials_info}\n"
|
|
28
30
|
msg += f"Details: {error_details}\n"
|
|
@@ -40,12 +40,12 @@ def iterate_all_data(
|
|
|
40
40
|
)
|
|
41
41
|
|
|
42
42
|
|
|
43
|
-
def extract_all(client: ClientMetabase, **kwargs
|
|
43
|
+
def extract_all(client: ClientMetabase, **kwargs) -> None:
|
|
44
44
|
"""
|
|
45
45
|
Extract Data From metabase
|
|
46
46
|
Store the output files locally under the given output_directory
|
|
47
47
|
"""
|
|
48
|
-
output_directory = kwargs.get("
|
|
48
|
+
output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
|
|
49
49
|
ts = current_timestamp()
|
|
50
50
|
|
|
51
51
|
for key, data in iterate_all_data(client):
|
|
@@ -55,6 +55,6 @@ def extract_all(client: ClientMetabase, **kwargs: str) -> None:
|
|
|
55
55
|
write_summary(
|
|
56
56
|
output_directory,
|
|
57
57
|
ts,
|
|
58
|
-
base_url=client.base_url
|
|
58
|
+
base_url=client.base_url,
|
|
59
59
|
client_name=client.name(),
|
|
60
60
|
)
|
|
@@ -2,6 +2,7 @@ import logging
|
|
|
2
2
|
from typing import Dict, List, Optional, cast
|
|
3
3
|
|
|
4
4
|
import requests
|
|
5
|
+
from requests.auth import HTTPBasicAuth
|
|
5
6
|
|
|
6
7
|
from ....utils import retry
|
|
7
8
|
from ..assets import (
|
|
@@ -22,7 +23,7 @@ from .constants import (
|
|
|
22
23
|
RETRY_JITTER_MS,
|
|
23
24
|
RETRY_STRATEGY,
|
|
24
25
|
)
|
|
25
|
-
from .credentials import
|
|
26
|
+
from .credentials import ModeCredentials
|
|
26
27
|
|
|
27
28
|
logger = logging.getLogger(__name__)
|
|
28
29
|
|
|
@@ -40,20 +41,16 @@ class Client:
|
|
|
40
41
|
|
|
41
42
|
def __init__(
|
|
42
43
|
self,
|
|
43
|
-
|
|
44
|
+
credentials: ModeCredentials,
|
|
44
45
|
):
|
|
45
|
-
self._credentials =
|
|
46
|
-
host=get_value(CredentialsKey.HOST, kwargs).rstrip("/"),
|
|
47
|
-
workspace=get_value(CredentialsKey.WORKSPACE, kwargs),
|
|
48
|
-
token=get_value(CredentialsKey.TOKEN, kwargs),
|
|
49
|
-
secret=get_value(CredentialsKey.SECRET, kwargs),
|
|
50
|
-
)
|
|
46
|
+
self._credentials = credentials
|
|
51
47
|
self._session = requests.Session()
|
|
52
|
-
|
|
53
|
-
|
|
48
|
+
|
|
49
|
+
def authenticate(self) -> HTTPBasicAuth:
|
|
50
|
+
return HTTPBasicAuth(self._credentials.token, self._credentials.secret)
|
|
54
51
|
|
|
55
52
|
def _check_connection(self):
|
|
56
|
-
authentication = self.
|
|
53
|
+
authentication = self.authenticate()
|
|
57
54
|
url = self._url(with_workspace=False) + "/account"
|
|
58
55
|
response = self._session.get(url, auth=authentication)
|
|
59
56
|
self._handle_response(response)
|
|
@@ -121,7 +118,7 @@ class Client:
|
|
|
121
118
|
report: Optional[str] = None,
|
|
122
119
|
resource_name: Optional[str] = None,
|
|
123
120
|
) -> RawData:
|
|
124
|
-
authentication = self.
|
|
121
|
+
authentication = self.authenticate()
|
|
125
122
|
url = self._url(with_workspace, space, report, resource_name)
|
|
126
123
|
logger.info(f"Calling {url}")
|
|
127
124
|
response = self._session.get(url, auth=authentication)
|
|
@@ -2,20 +2,20 @@ import json
|
|
|
2
2
|
|
|
3
3
|
from ....utils import load_file
|
|
4
4
|
from ..assets import EXPORTED_FIELDS, ModeAnalyticsAsset
|
|
5
|
-
from .client import Client
|
|
5
|
+
from .client import Client, ModeCredentials
|
|
6
6
|
|
|
7
7
|
_HOST = "https://mode.com"
|
|
8
8
|
_WORKSPACE = "castor"
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def _dummy_client() -> Client:
|
|
12
|
-
|
|
12
|
+
credentials = ModeCredentials(
|
|
13
13
|
host=_HOST,
|
|
14
14
|
workspace=_WORKSPACE,
|
|
15
15
|
token="dummy-token",
|
|
16
16
|
secret="******",
|
|
17
|
-
no_checks=True,
|
|
18
17
|
)
|
|
18
|
+
return Client(credentials=credentials)
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
def test__url():
|
|
@@ -1,57 +1,24 @@
|
|
|
1
|
-
from
|
|
2
|
-
from
|
|
1
|
+
from pydantic import Field, SecretStr, field_validator
|
|
2
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
MODE_ENV_PREFIX = "CASTOR_MODE_ANALYTICS_"
|
|
5
5
|
|
|
6
|
-
from ....utils import from_env
|
|
7
6
|
|
|
7
|
+
class ModeCredentials(BaseSettings):
|
|
8
|
+
"""Class holding Mode credentials attributes"""
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
model_config = SettingsConfigDict(
|
|
11
|
+
env_prefix=MODE_ENV_PREFIX,
|
|
12
|
+
extra="ignore",
|
|
13
|
+
populate_by_name=True,
|
|
14
|
+
)
|
|
14
15
|
|
|
16
|
+
host: str
|
|
17
|
+
secret: str = Field(repr=False)
|
|
18
|
+
token: str
|
|
19
|
+
workspace: str
|
|
15
20
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
CredentialsKey.SECRET: "CASTOR_MODE_ANALYTICS_SECRET",
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def get_value(key: CredentialsKey, kwargs: dict) -> str:
|
|
25
|
-
"""
|
|
26
|
-
Returns the value of the given key:
|
|
27
|
-
- from kwargs in priority
|
|
28
|
-
- from ENV if not provided (raises an error if not found in ENV)
|
|
29
|
-
"""
|
|
30
|
-
env_key = CREDENTIALS_ENV[key]
|
|
31
|
-
return str(kwargs.get(key.value) or from_env(env_key))
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
class Credentials:
|
|
35
|
-
"""ValueObject for the credentials"""
|
|
36
|
-
|
|
37
|
-
def __init__(self, host: str, workspace: str, token: str, secret: str):
|
|
38
|
-
self.host = host
|
|
39
|
-
self.workspace = workspace
|
|
40
|
-
self.token = token
|
|
41
|
-
self.secret = secret
|
|
42
|
-
|
|
43
|
-
def to_dict(self, hide: bool = False) -> Dict[str, str]:
|
|
44
|
-
safe = (
|
|
45
|
-
CredentialsKey.HOST,
|
|
46
|
-
CredentialsKey.WORKSPACE,
|
|
47
|
-
CredentialsKey.TOKEN,
|
|
48
|
-
)
|
|
49
|
-
unsafe = (CredentialsKey.SECRET,)
|
|
50
|
-
|
|
51
|
-
def val(k: CredentialsKey, v: str) -> str:
|
|
52
|
-
return "*" + v[-4:] if hide and k in unsafe else v
|
|
53
|
-
|
|
54
|
-
return {a.value: val(a, getattr(self, a.value)) for a in safe + unsafe}
|
|
55
|
-
|
|
56
|
-
def authentication(self) -> HTTPBasicAuth:
|
|
57
|
-
return HTTPBasicAuth(self.token, self.secret)
|
|
21
|
+
@field_validator("host", mode="before")
|
|
22
|
+
@classmethod
|
|
23
|
+
def _check_base_url(cls, host: str) -> str:
|
|
24
|
+
return host.rstrip("/")
|
|
@@ -11,7 +11,7 @@ from ...utils import (
|
|
|
11
11
|
write_summary,
|
|
12
12
|
)
|
|
13
13
|
from .assets import ModeAnalyticsAsset as Asset
|
|
14
|
-
from .client import Client
|
|
14
|
+
from .client import Client, ModeCredentials
|
|
15
15
|
|
|
16
16
|
logger = logging.getLogger(__name__)
|
|
17
17
|
|
|
@@ -37,9 +37,12 @@ def iterate_all_data(
|
|
|
37
37
|
yield Asset.MEMBER, deep_serialize(members)
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
def extract_all(
|
|
40
|
+
def extract_all(**kwargs) -> None:
|
|
41
41
|
"""Extract Data From Mode Analytics and store it locally in files under the output_directory"""
|
|
42
|
-
output_directory = kwargs.get("
|
|
42
|
+
output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
|
|
43
|
+
credentials = ModeCredentials(**kwargs)
|
|
44
|
+
client = Client(credentials)
|
|
45
|
+
|
|
43
46
|
ts = current_timestamp()
|
|
44
47
|
|
|
45
48
|
for key, data in iterate_all_data(client):
|
|
@@ -1,20 +1,28 @@
|
|
|
1
|
-
from dataclasses import field
|
|
2
1
|
from typing import List, Optional
|
|
3
2
|
|
|
4
|
-
from pydantic
|
|
3
|
+
from pydantic import Field, field_validator
|
|
4
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
5
5
|
|
|
6
6
|
from .constants import Urls
|
|
7
7
|
|
|
8
|
+
POWERBI_ENV_PREFIX = "CASTOR_POWERBI_"
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
class
|
|
10
|
+
|
|
11
|
+
class PowerbiCredentials(BaseSettings):
|
|
11
12
|
"""Class to handle PowerBI rest API permissions"""
|
|
12
13
|
|
|
14
|
+
model_config = SettingsConfigDict(
|
|
15
|
+
env_prefix=POWERBI_ENV_PREFIX,
|
|
16
|
+
extra="ignore",
|
|
17
|
+
populate_by_name=True,
|
|
18
|
+
)
|
|
19
|
+
|
|
13
20
|
client_id: str
|
|
14
21
|
tenant_id: str
|
|
15
|
-
secret: str =
|
|
16
|
-
scopes:
|
|
22
|
+
secret: str = Field(repr=False)
|
|
23
|
+
scopes: List[str] = [Urls.DEFAULT_SCOPE]
|
|
17
24
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
25
|
+
@field_validator("scopes", mode="before")
|
|
26
|
+
@classmethod
|
|
27
|
+
def _check_scopes(cls, scopes: Optional[List[str]]) -> List[str]:
|
|
28
|
+
return scopes if scopes is not None else [Urls.DEFAULT_SCOPE]
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from .constants import Urls
|
|
2
|
-
from .credentials import
|
|
2
|
+
from .credentials import PowerbiCredentials
|
|
3
3
|
|
|
4
4
|
|
|
5
5
|
def test_credentials():
|
|
@@ -8,15 +8,23 @@ def test_credentials():
|
|
|
8
8
|
secret = "🤫"
|
|
9
9
|
|
|
10
10
|
# no scopes provided
|
|
11
|
-
credentials =
|
|
11
|
+
credentials = PowerbiCredentials(
|
|
12
12
|
tenant_id=tenant_id,
|
|
13
13
|
client_id=client_id,
|
|
14
14
|
secret=secret,
|
|
15
15
|
)
|
|
16
16
|
assert credentials.scopes == [Urls.DEFAULT_SCOPE]
|
|
17
17
|
|
|
18
|
+
credentials = PowerbiCredentials(
|
|
19
|
+
tenant_id=tenant_id,
|
|
20
|
+
client_id=client_id,
|
|
21
|
+
secret=secret,
|
|
22
|
+
scopes=None,
|
|
23
|
+
)
|
|
24
|
+
assert credentials.scopes == [Urls.DEFAULT_SCOPE]
|
|
25
|
+
|
|
18
26
|
# empty scopes
|
|
19
|
-
credentials =
|
|
27
|
+
credentials = PowerbiCredentials(
|
|
20
28
|
tenant_id=tenant_id,
|
|
21
29
|
client_id=client_id,
|
|
22
30
|
secret=secret,
|
|
@@ -26,7 +34,7 @@ def test_credentials():
|
|
|
26
34
|
|
|
27
35
|
# with scopes
|
|
28
36
|
scopes = ["foo"]
|
|
29
|
-
credentials =
|
|
37
|
+
credentials = PowerbiCredentials(
|
|
30
38
|
tenant_id=tenant_id,
|
|
31
39
|
client_id=client_id,
|
|
32
40
|
secret=secret,
|
|
@@ -18,7 +18,7 @@ from .constants import (
|
|
|
18
18
|
QueryParams,
|
|
19
19
|
Urls,
|
|
20
20
|
)
|
|
21
|
-
from .credentials import
|
|
21
|
+
from .credentials import PowerbiCredentials
|
|
22
22
|
from .utils import batch_size_is_valid_or_assert, datetime_is_recent_or_assert
|
|
23
23
|
|
|
24
24
|
logger = logging.getLogger(__name__)
|
|
@@ -62,7 +62,7 @@ class Client:
|
|
|
62
62
|
https://learn.microsoft.com/en-us/rest/api/power-bi/admin
|
|
63
63
|
"""
|
|
64
64
|
|
|
65
|
-
def __init__(self, credentials:
|
|
65
|
+
def __init__(self, credentials: PowerbiCredentials):
|
|
66
66
|
self.creds = credentials
|
|
67
67
|
client_app = f"{Urls.CLIENT_APP_BASE}{self.creds.tenant_id}"
|
|
68
68
|
self.app = msal.ConfidentialClientApplication(
|