castor-extractor 0.5.3__py3-none-any.whl → 0.5.6__py3-none-any.whl

castor_extractor/visualization/metabase/client/decryption_test.py

@@ -2,7 +2,6 @@ from .decryption import decrypt
 
 
 def test_decrypt():
-
     key = "8iY5gyHxPH0YYyBgOd2AvwT1pcHl3EGtvN5jAi9JwoA"
     enc_message = "sIqpGK6UgzKOkThtlvBGqkb046EtB+HxcBsO3nDiKZAJcszfqqxTgSyH+SXAALznfuMSnZjdX9yzpGe77+ByYuCVlXHkMilkUe6tkFsFkBPW5CPirp0kqLdyp1yHXrv3NmXCtGZcef2fC0v89huRMSgFcm8M6Zf3JjSDEludLUo="
     dec_message = '{"db":"zip:/app/metabase.jar!/sample-dataset.db;USER=GUEST;PASSWORD=guest"}'

castor_extractor/visualization/metabase/extract.py

@@ -25,18 +25,18 @@ def iterate_all_data(
 
     yield MetabaseAsset.USER, deep_serialize(client.fetch(MetabaseAsset.USER))
     yield MetabaseAsset.COLLECTION, deep_serialize(
-        client.fetch(MetabaseAsset.COLLECTION)
+        client.fetch(MetabaseAsset.COLLECTION),
     )
     yield MetabaseAsset.DATABASE, deep_serialize(
-        client.fetch(MetabaseAsset.DATABASE)
+        client.fetch(MetabaseAsset.DATABASE),
     )
     yield MetabaseAsset.TABLE, deep_serialize(client.fetch(MetabaseAsset.TABLE))
     yield MetabaseAsset.CARD, deep_serialize(client.fetch(MetabaseAsset.CARD))
     yield MetabaseAsset.DASHBOARD, deep_serialize(
-        client.fetch(MetabaseAsset.DASHBOARD)
+        client.fetch(MetabaseAsset.DASHBOARD),
     )
     yield MetabaseAsset.DASHBOARD_CARDS, deep_serialize(
-        client.fetch(MetabaseAsset.DASHBOARD_CARDS)
+        client.fetch(MetabaseAsset.DASHBOARD_CARDS),
     )
 
 

castor_extractor/visualization/mode/client/client.py

@@ -168,7 +168,8 @@ class Client:
             # why without workspace? because users can belong to several companies
             # example: https://modeanalytics.com/api/john_doe
             result = self._call(
-                resource_name=mb["member_username"], with_workspace=False
+                resource_name=mb["member_username"],
+                with_workspace=False,
             )
             members.append(cast(Dict, result))
         return members

castor_extractor/visualization/mode/client/client_test.py

@@ -9,7 +9,7 @@ _WORKSPACE = "castor"
 
 
 def _dummy_client() -> Client:
-    return Client(
+    return Client(  # noqa: S106
         host=_HOST,
         workspace=_WORKSPACE,
         token="dummy-token",
@@ -41,10 +41,11 @@ def test__post_processing():
     client = _dummy_client()
     raw = load_file("client_test.json", __file__)
     result = client._post_processing(
-        asset=ModeAnalyticsAsset.COLLECTION, data=[json.loads(raw)]
+        asset=ModeAnalyticsAsset.COLLECTION,
+        data=[json.loads(raw)],
     )
     collection = result[0]
     assert set(collection.keys()) == set(
-        EXPORTED_FIELDS[ModeAnalyticsAsset.COLLECTION]
+        EXPORTED_FIELDS[ModeAnalyticsAsset.COLLECTION],
     )
     assert collection["creator"] == "john_doe"

castor_extractor/visualization/mode/client/credentials.py

@@ -9,8 +9,8 @@ from ....utils import from_env
 class CredentialsKey(Enum):
     HOST = "host"
     WORKSPACE = "workspace"
-    TOKEN = "token"
-    SECRET = "secret"
+    TOKEN = "token"  # noqa: S105
+    SECRET = "secret"  # noqa: S105
 
 
 CREDENTIALS_ENV: Dict[CredentialsKey, str] = {

castor_extractor/visualization/powerbi/client/constants.py

@@ -58,7 +58,7 @@ class QueryParams:
 
 
 class Keys:
-    ACCESS_TOKEN = "access_token"
+    ACCESS_TOKEN = "access_token"  # noqa: S105
     ACTIVITY_EVENT_ENTITIES = "activityEventEntities"
     CONTINUATION_URI = "continuationUri"
     ID = "id"

castor_extractor/visualization/powerbi/client/credentials.py

@@ -14,7 +14,6 @@ class Credentials:
         secret: str,
         scopes: Optional[List[str]] = None,
     ):
-
         if scopes is None:
             scopes = [Urls.DEFAULT_SCOPE]
         self.tenant_id = tenant_id

castor_extractor/visualization/powerbi/client/credentials_test.py

@@ -9,19 +9,27 @@ def test_credentials():
 
     # no scopes provided
     credentials = Credentials(
-        tenant_id=tenant_id, client_id=client_id, secret=secret
+        tenant_id=tenant_id,
+        client_id=client_id,
+        secret=secret,
     )
     assert credentials.scopes == [Urls.DEFAULT_SCOPE]
 
     # empty scopes
     credentials = Credentials(
-        tenant_id=tenant_id, client_id=client_id, secret=secret, scopes=[]
+        tenant_id=tenant_id,
+        client_id=client_id,
+        secret=secret,
+        scopes=[],
     )
     assert credentials.scopes == []
 
     # with scopes
     scopes = ["foo"]
     credentials = Credentials(
-        tenant_id=tenant_id, client_id=client_id, secret=secret, scopes=scopes
+        tenant_id=tenant_id,
+        client_id=client_id,
+        secret=secret,
+        scopes=scopes,
     )
     assert credentials.scopes == scopes

castor_extractor/visualization/powerbi/client/rest.py

@@ -104,7 +104,11 @@ class Client:
         """
         logger.debug(f"Calling {method} on {url}")
         result = requests.request(
-            method, url, headers=self._header(), params=params, data=data
+            method,
+            url,
+            headers=self._header(),
+            params=params,
+            data=data,
         )
         result.raise_for_status()
 
@@ -131,11 +135,16 @@ class Client:
         processor: Optional[Callable] = None,
     ) -> Any:
         return self._call(
-            url, POST, params=params, data=data, processor=processor
+            url,
+            POST,
+            params=params,
+            data=data,
+            processor=processor,
         )
 
     def _workspace_ids(
-        self, modified_since: Optional[datetime] = None
+        self,
+        modified_since: Optional[datetime] = None,
     ) -> List[str]:
         """
         Get workspaces ids from powerBI admin API.
@@ -191,7 +200,7 @@ class Client:
                 break
             waiting_seconds += sleep_seconds
             logger.info(
-                f"Waiting {sleep_seconds} sec for scan {scan_id} to be ready…"
+                f"Waiting {sleep_seconds} sec for scan {scan_id} to be ready…",
             )
             sleep(sleep_seconds)
         return False
@@ -249,7 +258,8 @@ class Client:
         return self._get(Urls.DASHBOARD)[Keys.VALUE]
 
     def _metadata(
-        self, modified_since: Optional[datetime] = None
+        self,
+        modified_since: Optional[datetime] = None,
     ) -> Iterator[List[Dict]]:
         """
         Fetch metadata by workspace.

castor_extractor/visualization/powerbi/client/rest_test.py

@@ -15,7 +15,9 @@ FAKE_SECRET = "MeThree"
 
 def _client() -> Client:
     creds = Credentials(
-        tenant_id=FAKE_TENANT_ID, client_id=FAKE_CLIENT_ID, secret=FAKE_SECRET
+        tenant_id=FAKE_TENANT_ID,
+        client_id=FAKE_CLIENT_ID,
+        secret=FAKE_SECRET,
     )
     return Client(creds)
 
@@ -26,7 +28,6 @@ def _raise_http_error() -> None:
 
 @patch.object(msal, "ConfidentialClientApplication")
 def test__access_token(mock_app):
-
     # init mocks
     valid_response = {"access_token": "mock_token"}
     returning_valid_token = Mock(return_value=valid_response)
@@ -85,7 +86,7 @@ def test__get(mocked_access_token, mocked_request):
 def test__workspace_ids(_, mocked_request):
     client = _client()
     mocked_request.return_value = Mock(
-        json=lambda: [{"id": 1000}, {"id": 1001}, {"id": 1003}]
+        json=lambda: [{"id": 1000}, {"id": 1001}, {"id": 1003}],
     )
     ids = client._workspace_ids()
     assert ids == [1000, 1001, 1003]
@@ -103,7 +104,11 @@ def test__workspace_ids(_, mocked_request):
     }
 
     mocked_request.assert_called_with(
-        GET, Urls.WORKSPACE_IDS, data=None, headers=ANY, params=params
+        GET,
+        Urls.WORKSPACE_IDS,
+        data=None,
+        headers=ANY,
+        params=params,
     )
 
 
@@ -116,7 +121,11 @@ def test__post_default(_, mocked_request):
     data = {"bonjour": "hello"}
     client._post(url, params=params, data=data)
     mocked_request.assert_called_with(
-        POST, url, headers=ANY, params=QueryParams.METADATA_SCAN, data=data
+        POST,
+        url,
+        headers=ANY,
+        params=QueryParams.METADATA_SCAN,
+        data=data,
     )
 
 
@@ -129,7 +138,10 @@ def test__post_with_processor(_, mocked_request):
     data = {"bonjour": "hello"}
     mocked_request.return_value = Mock(json=lambda: {"id": 1000})
     result = client._post(
-        url, params=params, data=data, processor=lambda x: x.json()["id"]
+        url,
+        params=params,
+        data=data,
+        processor=lambda x: x.json()["id"],
     )
     assert result == 1000
 
@@ -139,11 +151,15 @@ def test__post_with_processor(_, mocked_request):
 def test__datasets(_, mocked_request):
     client = _client()
     mocked_request.return_value = Mock(
-        json=lambda: {"value": [{"id": 1, "type": "dataset"}]}
+        json=lambda: {"value": [{"id": 1, "type": "dataset"}]},
     )
     datasets = client._datasets()
     mocked_request.assert_called_with(
-        GET, Urls.DATASETS, data=None, headers=ANY, params=None
+        GET,
+        Urls.DATASETS,
+        data=None,
+        headers=ANY,
+        params=None,
     )
     assert datasets == [{"id": 1, "type": "dataset"}]
 
@@ -153,11 +169,15 @@ def test__datasets(_, mocked_request):
 def test__reports(_, mocked_request):
     client = _client()
     mocked_request.return_value = Mock(
-        json=lambda: {"value": [{"id": 1, "type": "report"}]}
+        json=lambda: {"value": [{"id": 1, "type": "report"}]},
     )
     reports = client._reports()
     mocked_request.assert_called_with(
-        GET, Urls.REPORTS, data=None, headers=ANY, params=None
+        GET,
+        Urls.REPORTS,
+        data=None,
+        headers=ANY,
+        params=None,
     )
     assert reports == [{"id": 1, "type": "report"}]
 
@@ -167,11 +187,15 @@ def test__reports(_, mocked_request):
 def test__dashboards(_, mocked_request):
     client = _client()
     mocked_request.return_value = Mock(
-        json=lambda: {"value": [{"id": 1, "type": "dashboard"}]}
+        json=lambda: {"value": [{"id": 1, "type": "dashboard"}]},
     )
     dashboards = client._dashboards()
     mocked_request.assert_called_with(
-        GET, Urls.DASHBOARD, data=None, headers=ANY, params=None
+        GET,
+        Urls.DASHBOARD,
+        data=None,
+        headers=ANY,
+        params=None,
     )
     assert dashboards == [{"id": 1, "type": "dashboard"}]
 
@@ -181,7 +205,10 @@ def test__dashboards(_, mocked_request):
 @patch.object(Client, "_wait_for_scan_result")
 @patch.object(Client, "_get_scan")
 def test__metadata(
-    mocked_get_scan, mocked_wait, mocked_create_scan, mocked_workspace_ids
+    mocked_get_scan,
+    mocked_wait,
+    mocked_create_scan,
+    mocked_workspace_ids,
 ):
     mocked_workspace_ids.return_value = list(range(200))
     mocked_create_scan.return_value = 314

castor_extractor/visualization/powerbi/extract.py

@@ -16,9 +16,7 @@ from .client import Client, Credentials
 def iterate_all_data(
     client: Client,
 ) -> Iterable[Tuple[PowerBiAsset, Union[List, dict]]]:
-
     for asset in PowerBiAsset:
-
         if asset in METADATA_ASSETS:
             continue
 
@@ -39,7 +37,10 @@ def extract_all(
     """
     _output_directory = output_directory or from_env(OUTPUT_DIR)
     creds = Credentials(
-        tenant_id=tenant_id, client_id=client_id, secret=secret, scopes=scopes
+        tenant_id=tenant_id,
+        client_id=client_id,
+        secret=secret,
+        scopes=scopes,
     )
     client = Client(creds)
     ts = current_timestamp()

castor_extractor/visualization/qlik/client/engine/client.py

@@ -69,7 +69,9 @@ class EngineApiClient:
             return _list_measures(client, app_id_)
 
         with open_websocket(
-            app_id=app_id, server_url=self.server_url, api_key=self.api_key
+            app_id=app_id,
+            server_url=self.server_url,
+            api_key=self.api_key,
         ) as websocket:
             json_rpc_client = JsonRpcClient(websocket=websocket)
             return _call(json_rpc_client, app_id)

castor_extractor/visualization/qlik/client/engine/json_rpc.py

@@ -25,7 +25,10 @@ class JsonRpcClient:
         self.call_id += 1
 
     def _format_message(
-        self, method: JsonRpcMethod, handle: int, params: Optional[list] = None
+        self,
+        method: JsonRpcMethod,
+        handle: int,
+        params: Optional[list] = None,
     ) -> dict:
         params = params or list()
         message = {

castor_extractor/visualization/qlik/client/engine/json_rpc_test.py

@@ -6,7 +6,6 @@ from .websocket import WebsocketConnection
 
 
 def test_json_rpc_client__send_message():
-
     dummy_server_url = "toto.fr"
     dummy_api_key = "IAmNotATrueSecretKey"
     ws = WebsocketConnection(server_url=dummy_server_url, api_key=dummy_api_key)

castor_extractor/visualization/qlik/client/master.py

@@ -21,7 +21,8 @@ class MissingAppsScopeError(Exception):
 
 
 def _include_app_external_id(
-    data: ListedData, app_external_id: str
+    data: ListedData,
+    app_external_id: str,
 ) -> ListedData:
     _data = data.copy()
     for element in _data:
@@ -30,7 +31,9 @@ def _include_app_external_id(
 
 
 def _fetch_children_on_apps(
-    apps: ListedData, fetch_callback: Callable, display_progress: bool
+    apps: ListedData,
+    fetch_callback: Callable,
+    display_progress: bool,
 ) -> ListedData:
     all_apps_data: ListedData = list()
     apps_iterator = apps if not display_progress else tqdm(apps)
@@ -66,7 +69,8 @@ class QlikMasterClient:
         )
 
         self.engine_api_client = EngineApiClient(
-            server_url=self._server_url, api_key=self._api_key
+            server_url=self._server_url,
+            api_key=self._api_key,
         )
 
     def _fetch_lineage(self, apps: ListedData) -> ListedData:
@@ -78,7 +82,10 @@ class QlikMasterClient:
         return _fetch_children_on_apps(apps, callback, self.display_progress)
 
     def fetch(
-        self, asset: QlikAsset, *, apps: Optional[ListedData] = None
+        self,
+        asset: QlikAsset,
+        *,
+        apps: Optional[ListedData] = None,
     ) -> ListedData:
         """
         Given a QlikAsset, returns the corresponding data using the

castor_extractor/visualization/qlik/client/rest_test.py

@@ -5,7 +5,9 @@ from .rest import RestApiClient
 
 
 def _check_called_once(
-    client: RestApiClient, first_page_url: str, return_value: Optional[dict]
+    client: RestApiClient,
+    first_page_url: str,
+    return_value: Optional[dict],
 ):
     with patch.object(RestApiClient, "_call") as mock_call:
         mock_call.return_value = return_value
@@ -18,7 +20,6 @@ def _check_called_once(
 
 
 def test_rest_api_client_pager():
-
     dummy_server_url = "https://clic.kom"
     dummy_api_key = "i-am-the-key-dont-let-others-know-about"
 

castor_extractor/visualization/sigma/client/client.py

@@ -25,7 +25,8 @@ class SigmaClient:
         self.headers: Optional[Dict[str, str]] = None
 
     def _get_token(self, token_api_path: str) -> Dict[str, str]:
-        token_response = requests.post(
+        # TODO: ADD TIMEOUT
+        token_response = requests.post(  # noqa: S113
             token_api_path,
             data={
                 CredentialsKey.GRANT_TYPE.value: "client_credentials",
@@ -52,7 +53,8 @@ class SigmaClient:
 
     def _get(self, endpoint_url: str) -> dict:
         url = urljoin(self.host, endpoint_url)
-        result = requests.get(url, headers=self._get_headers())
+        # TODO: add timeout
+        result = requests.get(url, headers=self._get_headers())  # noqa: S113
         try:
             return result.json()
         except:
@@ -77,7 +79,9 @@ class SigmaClient:
         yield from self._get_with_pagination(endpoint)
 
     def _per_page_get_elements(
-        self, workbook_id: str, page_id: str
+        self,
+        workbook_id: str,
+        page_id: str,
     ) -> Iterator[dict]:
         endpoint = EndpointFactory.elements(workbook_id, page_id)
         yield from self._get_with_pagination(endpoint)

castor_extractor/visualization/sigma/client/client_test.py

@@ -2,8 +2,10 @@ from unittest.mock import Mock, patch
 
 from .client import SigmaClient, SigmaCredentials
 
-FAKE_CREDENTIALS = SigmaCredentials(
-    host="IamFake", client_id="MeTwo", api_token="MeThree"
+FAKE_CREDENTIALS = SigmaCredentials(  # noqa: S106
+    host="IamFake",
+    client_id="MeTwo",
+    api_token="MeThree",
 )
 
 

castor_extractor/visualization/sigma/client/credentials.py

@@ -5,11 +5,11 @@ from enum import Enum
 class CredentialsKey(Enum):
     """Value enum object for the credentials"""
 
-    CLIENT_SECRET = "client_secret"
+    CLIENT_SECRET = "client_secret"  # noqa: S105
     CLIENT_ID = "client_id"
     HOST = "host"
     GRANT_TYPE = "grant_type"
-    API_TOKEN = "api_token"
+    API_TOKEN = "api_token"  # noqa: S105
 
 
 CLIENT_ALLOWED_KEYS = (

castor_extractor/visualization/sigma/constants.py

@@ -1,4 +1,4 @@
 # environment variable names
 HOST = "CASTOR_SIGMA_HOST"
-API_TOKEN = "CASTOR_SIGMA_API_TOKEN"
+API_TOKEN = "CASTOR_SIGMA_API_TOKEN"  # noqa: S105
 CLIENT_ID = "CASTOR_SIGMA_CLIENT_ID"

castor_extractor/visualization/sigma/extract.py

@@ -68,7 +68,9 @@ def extract_all(
     _api_token = api_token or from_env(API_TOKEN)
 
     credentials = SigmaCredentials(
-        host=_host, client_id=_client_id, api_token=_api_token
+        host=_host,
+        client_id=_client_id,
+        api_token=_api_token,
     )
     client = SigmaClient(credentials=credentials)
 

castor_extractor/visualization/tableau/client/client.py

@@ -30,7 +30,9 @@ class ApiClient:
             user=get_value(CredentialsKey.TABLEAU_USER, kwargs, True),
             password=get_value(CredentialsKey.TABLEAU_PASSWORD, kwargs, True),
             token_name=get_value(
-                CredentialsKey.TABLEAU_TOKEN_NAME, kwargs, True
+                CredentialsKey.TABLEAU_TOKEN_NAME,
+                kwargs,
+                True,
             ),
             token=get_value(CredentialsKey.TABLEAU_TOKEN, kwargs, True),
             server_url=get_value(CredentialsKey.TABLEAU_SERVER_URL, kwargs),
@@ -54,7 +56,7 @@ class ApiClient:
                 self._credentials.user,
                 self._credentials.password,
                 site_id=self._credentials.site_id,
-            )
+            ),
         )
 
     def _pat_login(self) -> None:
@@ -64,7 +66,7 @@ class ApiClient:
                 self._credentials.token_name,
                 self._credentials.token,
                 site_id=self._credentials.site_id,
-            )
+            ),
         )
 
     def login(self) -> None:
@@ -80,7 +82,7 @@ class ApiClient:
 
         raise ValueError(
             """Wrong authentication: you should provide either user and password
-             or personal access token"""
+             or personal access token""",
         )
 
     def base_url(self) -> str:
@@ -119,7 +121,7 @@ class ApiClient:
             [
                 extract_asset(project, TableauAsset.PROJECT)
                 for project in TSC.Pager(self._server.projects)
-            ]
+            ],
         )
 
     def _fetch_workbooks_to_datasource(self) -> SerializedAsset:

castor_extractor/visualization/tableau/client/client_utils.py

@@ -23,9 +23,10 @@ RESOURCE_TEMPLATE = "{resource}Connection"
 
 
 def get_paginated_objects(
-    server, asset: TableauAsset, page_size: int
+    server,
+    asset: TableauAsset,
+    page_size: int,
 ) -> SerializedAsset:
-
     assets: SerializedAsset = []
     for query in QUERY_FIELDS[asset]:
         fields = query["fields"].value
@@ -43,7 +44,9 @@ def get_paginated_objects(
 
 
 def query_scroll(
-    server, query: str, resource: str
+    server,
+    query: str,
+    resource: str,
 ) -> Iterator[SerializedAsset]:
     """build a tableau query iterator handling pagination and cursor"""
 

castor_extractor/visualization/tableau/client/credentials.py

@@ -13,9 +13,9 @@ class CredentialsKey(Enum):
     """Value enum object for the credentials"""
 
     TABLEAU_USER = "user"
-    TABLEAU_PASSWORD = "password"
-    TABLEAU_TOKEN_NAME = "token_name"
-    TABLEAU_TOKEN = "token"
+    TABLEAU_PASSWORD = "password"  # noqa: S105
+    TABLEAU_TOKEN_NAME = "token_name"  # noqa: S105
+    TABLEAU_TOKEN = "token"  # noqa: S105
     TABLEAU_SITE_ID = "site_id"
     TABLEAU_SERVER_URL = "server_url"
 
@@ -31,7 +31,9 @@ CREDENTIALS_ENV: Dict[CredentialsKey, str] = {
 
 
 def get_value(
-    key: CredentialsKey, kwargs: dict, optional: bool = False
+    key: CredentialsKey,
+    kwargs: dict,
+    optional: bool = False,
 ) -> Optional[str]:
     """
     Returns the value of the given key:

castor_extractor/visualization/tableau/client/project.py

@@ -4,7 +4,9 @@ from ....utils import SerializedAsset
 
 
 def _folder_path(
-    projects: SerializedAsset, project: dict, root: Optional[str] = ""
+    projects: SerializedAsset,
+    project: dict,
+    root: Optional[str] = "",
 ) -> str:
     """Recursive function to compute folder path with list of projects"""
     path = "/" + str(project["name"]) + (root or "")

castor_extractor/visualization/tableau/client/safe_mode.py

@@ -17,7 +17,8 @@ logger = logging.getLogger(__name__)
 def _paginated_option(page_number: int) -> TSC.RequestOptions:
     """Set up the Paginated option for TSC.RequestOptions"""
     return TSC.RequestOptions(
-        pagesize=SAFE_MODE_PAGE_SIZE, pagenumber=page_number
+        pagesize=SAFE_MODE_PAGE_SIZE,
+        pagenumber=page_number,
     )