castor-extractor 0.18.2__tar.gz → 0.18.7__tar.gz

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.18.7 - 2024-08-01
+
+* Salesforce: extract table descriptions
+
+## 0.18.6 - 2024-07-30
+
+* BigQuery: introduce extended regions to extract missing queries
+
+## 0.18.5 - 2024-07-17
+
+* Salesforce: extract DeveloperName and tooling url
+
+## 0.18.4 - 2024-07-16
+
+* Fix environment variables assignments for credentials
+
+## 0.18.3 - 2024-07-16
+
+* bump dependencies (minor and patches)
+
 ## 0.18.2 - 2024-07-08
 
 * Added StatusCode handling to SafeMode

castor_extractor-0.18.7/DockerfileUsage.md

@@ -0,0 +1,21 @@
+# Dockerfile usage for CastorDoc package
+
+## How To
+
+- The Dockerfile is present on the pypi package
+- For building it you should use this command `docker build -t castor-extractor-looker --build-arg EXTRA=looker .` with replacing looker one or several of: [bigquery,looker,metabase,powerbi,qlik,redshift,snowflake,tableau]
+- For running it you should do `docker run -v <local-path>:/data --env-file <castor-extract-looker.env> castor-extractor-looker` where `</local-path>` have to be replaced and `<castor-extract-looker.env>` have to be set.
+- Extracted datas would be available on `<local-path>`. The path should exists
+- `<castor-extract-looker.env>` would contain env vars for credentials, url...
+
+#### example
+
+```bash
+docker run -v /logs:/data --env-file /config/castor-extract-looker.env castor-extractor-looker
+```
+
+## Limitation
+
+- This docker image is for a specific techno
+- This docker image is based on python 3.11
+- This docker image use the latest castor-extractor package version

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.18.2
+Version: 0.18.7
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_domo.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import domo  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -38,13 +39,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    domo.extract_all(
-        api_token=args.api_token,
-        base_url=args.base_url,
-        client_id=args.client_id,
-        cloud_id=args.cloud_id,
-        developer_token=args.developer_token,
-        output_directory=args.output,
-    )
+    domo.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_looker.py

@@ -1,5 +1,6 @@
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import looker  # type: ignore
 
 
@@ -33,16 +34,4 @@ def main():
         action="store_true",
     )
 
-    args = parser.parse_args()
-
-    looker.extract_all(
-        base_url=args.base_url,
-        client_id=args.username,
-        client_secret=args.password,
-        log_to_stdout=args.log_to_stdout,
-        output_directory=args.output,
-        safe_mode=args.safe_mode,
-        search_per_folder=args.search_per_folder,
-        thread_pool_size=args.thread_pool_size,
-        timeout=args.timeout,
-    )
+    looker.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_metabase_api.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -15,16 +16,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
+    args = parse_filled_arguments(parser)
 
-    credentials = metabase.MetabaseApiCredentials(
-        base_url=args.base_url,
-        user=args.username,
-        password=args.password,
-    )
+    credentials = metabase.MetabaseApiCredentials(**args)
     client = metabase.ApiClient(credentials)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    metabase.extract_all(client, **args)

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_metabase_db.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -33,22 +34,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    credentials = metabase.MetabaseDbCredentials(
-        host=args.host,
-        port=args.port,
-        database=args.database,
-        schema=args.schema,
-        user=args.username,
-        password=args.password,
-        encryption_secret_key=args.encryption_secret_key,
-        require_ssl=args.require_ssl,
-    )
+    args = parse_filled_arguments(parser)
+    credentials = metabase.MetabaseDbCredentials(**args)
 
     client = metabase.DbClient(credentials)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    metabase.extract_all(client, **args)

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_mode.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import mode  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -24,15 +25,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    credentials = mode.Credentials(
-        host=args.host,
-        workspace=args.workspace,
-        token=args.token,
-        secret=args.secret,
-    )
-
-    mode.extract_all(
-        credentials,
-        output_directory=args.output,
-    )
+    mode.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_powerbi.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import powerbi  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -20,11 +21,4 @@ def main():
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    powerbi.extract_all(
-        tenant_id=args.tenant_id,
-        client_id=args.client_id,
-        secret=args.secret,
-        scopes=args.scopes,
-        output_directory=args.output,
-    )
+    powerbi.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_qlik.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import qlik  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -22,10 +23,4 @@ def main():
         "missing rights on some assets.",
     )
 
-    args = parser.parse_args()
-    qlik.extract_all(
-        base_url=args.base_url,
-        api_key=args.api_key,
-        output_directory=args.output,
-        except_http_error_statuses=args.except_http_error_statuses,
-    )
+    qlik.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_salesforce.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.warehouse import salesforce  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -29,15 +30,5 @@ def main():
     )
     parser.set_defaults(skip_existing=False)
 
-    args = parser.parse_args()
-
-    salesforce.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-        skip_existing=args.skip_existing,
-    )
+    args = parse_filled_arguments(parser)
+    salesforce.extract_all(output_directory=args.get("output"), **args)

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_salesforce_reporting.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import salesforce_reporting  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -21,13 +22,4 @@ def main():
     parser.add_argument("-b", "--base-url", help="Salesforce instance URL")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    salesforce_reporting.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-    )
+    salesforce_reporting.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/commands/extract_sigma.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import sigma  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -14,10 +15,4 @@ def main():
     parser.add_argument("-a", "--api-token", help="Generated API key")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    sigma.extract_all(
-        host=args.host,
-        client_id=args.client_id,
-        api_token=args.api_token,
-        output_directory=args.output,
-    )
+    sigma.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/utils/__init__.py

@@ -1,3 +1,4 @@
+from .argument_parser import parse_filled_arguments
 from .client import (
     AbstractSourceClient,
     ExtractionQuery,

castor_extractor-0.18.7/castor_extractor/utils/argument_parser.py

@@ -0,0 +1,7 @@
+from argparse import ArgumentParser
+
+
+def parse_filled_arguments(parser: ArgumentParser) -> dict:
+    """Parse arguments and remove all those with None values"""
+    parsed_arguments = vars(parser.parse_args())
+    return {k: v for k, v in parsed_arguments.items() if v is not None}

castor_extractor-0.18.7/castor_extractor/utils/argument_parser_test.py

@@ -0,0 +1,25 @@
+from argparse import Namespace
+
+from .argument_parser import parse_filled_arguments
+
+
+class MockArgumentParser:
+
+    def __init__(self):
+        self.attributes = {}
+
+    def add_argument(self, name, value):
+        self.attributes[name] = value
+
+    def parse_args(self) -> Namespace:
+        return Namespace(**self.attributes)
+
+
+def test_parse_filled_arguments():
+    parser = MockArgumentParser()
+    parser.add_argument("filled", "value")
+    parser.add_argument("unfilled", None)
+    parser.add_argument("empty_str", "")
+
+    expected = {"filled": "value", "empty_str": ""}
+    assert parse_filled_arguments(parser) == expected

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/visualization/domo/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Iterator, Optional, Tuple, Union
+from typing import Iterable, Iterator, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -42,28 +42,13 @@ def iterate_all_data(
     yield DomoAsset.DATAFLOWS, list(deep_serialize(dataflows))
 
 
-def extract_all(
-    api_token: Optional[str] = None,
-    base_url: Optional[str] = None,
-    client_id: Optional[str] = None,
-    cloud_id: Optional[str] = None,
-    developer_token: Optional[str] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Domo API
     Store the output files locally under the given output_directory
     """
-
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-
-    credentials = DomoCredentials(
-        base_url=base_url,
-        client_id=client_id,
-        api_token=api_token,
-        developer_token=developer_token,
-        cloud_id=cloud_id,
-    )
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = DomoCredentials(**kwargs)
     client = DomoClient(credentials=credentials)
 
     ts = current_timestamp()

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/visualization/looker/extract.py

@@ -126,11 +126,10 @@ def extract_all(**kwargs) -> None:
     Extract Data From looker and store it locally in files under the
     output_directory
     """
-    args = {arg: value for arg, value in kwargs.items() if value is not None}
-    extraction_parameters = ExtractionParameters(**args)
+    extraction_parameters = ExtractionParameters(**kwargs)
     output_directory = extraction_parameters.output_directory
 
-    credentials = LookerCredentials(**args)
+    credentials = LookerCredentials(**kwargs)
 
     if extraction_parameters.log_to_stdout:
         set_stream_handler_to_stdout()

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/visualization/metabase/extract.py

@@ -40,12 +40,12 @@ def iterate_all_data(
     )
 
 
-def extract_all(client: ClientMetabase, **kwargs: str) -> None:
+def extract_all(client: ClientMetabase, **kwargs) -> None:
     """
     Extract Data From metabase
     Store the output files locally under the given output_directory
     """
-    output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/visualization/mode/extract.py

@@ -37,11 +37,12 @@ def iterate_all_data(
     yield Asset.MEMBER, deep_serialize(members)
 
 
-def extract_all(credentials: ModeCredentials, output_directory: str) -> None:
+def extract_all(**kwargs) -> None:
     """Extract Data From Mode Analytics and store it locally in files under the output_directory"""
-    output_directory = output_directory or from_env(OUTPUT_DIR)
-
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = ModeCredentials(**kwargs)
     client = Client(credentials)
+
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/visualization/powerbi/extract.py

@@ -1,4 +1,4 @@
-from typing import Iterable, List, Optional, Tuple, Union
+from typing import Iterable, List, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -24,24 +24,13 @@ def iterate_all_data(
         yield asset, deep_serialize(data)
 
 
-def extract_all(
-    tenant_id: str,
-    client_id: str,
-    secret: str,
-    scopes: Optional[List[str]] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from PowerBI REST API
     Store the output files locally under the given output_directory
     """
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-    creds = PowerbiCredentials(
-        tenant_id=tenant_id,
-        client_id=client_id,
-        secret=secret,
-        scopes=scopes,
-    )
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    creds = PowerbiCredentials(**kwargs)
     client = Client(creds)
     ts = current_timestamp()
 

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/visualization/qlik/extract.py

@@ -47,18 +47,15 @@ def iterate_all_data(
 
 
 def extract_all(
-    base_url: Optional[str] = None,
-    api_key: Optional[str] = None,
-    output_directory: Optional[str] = None,
-    except_http_error_statuses: Optional[List[int]] = None,
+    except_http_error_statuses: Optional[List[int]] = None, **kwargs
 ) -> None:
     """
     Extract data from Qlik REST API
     Store the output files locally under the given output_directory
     """
 
-    credentials = QlikCredentials(base_url=base_url, api_key=api_key)
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
+    credentials = QlikCredentials(**kwargs)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
     client = QlikClient(
         credentials=credentials,

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/visualization/salesforce_reporting/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Optional, Tuple, Union
+from typing import Iterable, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -28,29 +28,15 @@ def iterate_all_data(
         yield asset.name.lower(), deep_serialize(data)
 
 
-def extract_all(
-    username: str,
-    password: str,
-    client_id: str,
-    client_secret: str,
-    security_token: str,
-    base_url: str,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Salesforce REST API
     Store the output files locally under the given output_directory
     """
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-    creds = SalesforceCredentials(
-        username=username,
-        password=password,
-        client_id=client_id,
-        client_secret=client_secret,
-        security_token=security_token,
-        base_url=base_url,
-    )
-    client = SalesforceReportingClient(credentials=creds)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = SalesforceCredentials(**kwargs)
+
+    client = SalesforceReportingClient(credentials=credentials)
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/visualization/sigma/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Iterator, Optional, Tuple, Union
+from typing import Iterable, Iterator, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -50,24 +50,15 @@ def iterate_all_data(
     yield SigmaAsset.LINEAGES, list(deep_serialize(lineages))
 
 
-def extract_all(
-    host: Optional[str] = None,
-    client_id: Optional[str] = None,
-    api_token: Optional[str] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Sigma API
     Store the output files locally under the given output_directory
     """
 
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
-    credentials = SigmaCredentials(
-        host=host,
-        client_id=client_id,
-        api_token=api_token,
-    )
+    credentials = SigmaCredentials(**kwargs)
     client = SigmaClient(credentials=credentials)
 
     ts = current_timestamp()

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/warehouse/bigquery/client.py

@@ -1,13 +1,14 @@
+import itertools
 import logging
-from typing import List, Optional, Set, Tuple
+from typing import List, Optional, Set
 
-from google.api_core.exceptions import Forbidden
-from google.api_core.page_iterator import Iterator as PageIterator
+from google.api_core.exceptions import Forbidden  # type: ignore
 from google.cloud.bigquery import Client as GoogleCloudClient  # type: ignore
 from google.cloud.bigquery.dataset import Dataset  # type: ignore
 from google.oauth2.service_account import Credentials  # type: ignore
 
 from ...utils import SqlalchemyClient, retry
+from .types import SetTwoString
 
 logger = logging.getLogger(__name__)
 
@@ -117,16 +118,50 @@ class BigQueryClient(SqlalchemyClient):
             ]
         return self._projects
 
-    def get_regions(self) -> Set[Tuple[str, str]]:
+    def get_regions(self) -> SetTwoString:
         """
-        Returns distinct (project_id, region) available for the given GCP client
+        Returns (project_id, region) available for the given GCP client
+        - Loops trough projects -> datasets -> region
+        - Returns distinct values
+        Example:
+            project_A
+                -> dataset_1:US
+            project_B
+                -> empty
+            project_C
+                -> dataset_2:EU
+                -> dataset_3:EU
+        Will return:
+            { (p_A, US), (p_C, EU) }
         """
         return {
             (ds.project, ds._properties["location"])
             for ds in self._list_datasets()
         }
 
-    def get_datasets(self) -> Set[Tuple[str, str]]:
+    def get_extended_regions(self) -> SetTwoString:
+        """
+        Returns all combinations of (project_id, region) for the given client
+        - Fetch all projects
+        - Fetch all regions (cross projects)
+        - Returns a combination of the two lists
+        Example:
+            project_A
+                -> dataset_1:US
+            project_B
+                -> empty
+            project_C
+                -> dataset_2:EU
+                -> dataset_3:EU
+        Will return:
+            { (p_A, EU), (p_A, US), (p_B, EU), (p_B, US), (p_C, EU), (p_C, US) }
+        """
+        projects = self.get_projects()
+        regions = {ds._properties["location"] for ds in self._list_datasets()}
+        combinations = itertools.product(projects, regions)
+        return set(combinations)
+
+    def get_datasets(self) -> SetTwoString:
         """
         Returns distinct (project_id, dataset_id) available for the given GCP client
         """

{castor_extractor-0.18.2 → castor_extractor-0.18.7}/castor_extractor/warehouse/bigquery/extract.py

@@ -68,6 +68,7 @@ def extract_all(**kwargs) -> None:
     query_builder = BigQueryQueryBuilder(
         regions=client.get_regions(),
         datasets=client.get_datasets(),
+        extended_regions=client.get_extended_regions(),
     )
 
     storage = LocalStorage(directory=output_directory)