castor-extractor 0.18.2__tar.gz → 0.18.5__tar.gz

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.18.5 - 2024-07-17
+
+* Salesforce: extract DeveloperName and tooling url
+
+## 0.18.4 - 2024-07-16
+
+* Fix environment variables assignments for credentials
+
+## 0.18.3 - 2024-07-16
+
+* bump dependencies (minor and patches)
+
 ## 0.18.2 - 2024-07-08
 
 * Added StatusCode handling to SafeMode

castor_extractor-0.18.5/DockerfileUsage.md

@@ -0,0 +1,21 @@
+# Dockerfile usage for CastorDoc package
+
+## How To
+
+- The Dockerfile is present on the pypi package
+- For building it you should use this command `docker build -t castor-extractor-looker --build-arg EXTRA=looker .` with replacing looker one or several of: [bigquery,looker,metabase,powerbi,qlik,redshift,snowflake,tableau]
+- For running it you should do `docker run -v <local-path>:/data --env-file <castor-extract-looker.env> castor-extractor-looker` where `</local-path>` have to be replaced and `<castor-extract-looker.env>` have to be set.
+- Extracted datas would be available on `<local-path>`. The path should exists
+- `<castor-extract-looker.env>` would contain env vars for credentials, url...
+
+#### example
+
+```bash
+docker run -v /logs:/data --env-file /config/castor-extract-looker.env castor-extractor-looker
+```
+
+## Limitation
+
+- This docker image is for a specific techno
+- This docker image is based on python 3.11
+- This docker image use the latest castor-extractor package version

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.18.2
+Version: 0.18.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_domo.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import domo  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -38,13 +39,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    domo.extract_all(
-        api_token=args.api_token,
-        base_url=args.base_url,
-        client_id=args.client_id,
-        cloud_id=args.cloud_id,
-        developer_token=args.developer_token,
-        output_directory=args.output,
-    )
+    domo.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_looker.py

@@ -1,5 +1,6 @@
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import looker  # type: ignore
 
 
@@ -33,16 +34,4 @@ def main():
         action="store_true",
     )
 
-    args = parser.parse_args()
-
-    looker.extract_all(
-        base_url=args.base_url,
-        client_id=args.username,
-        client_secret=args.password,
-        log_to_stdout=args.log_to_stdout,
-        output_directory=args.output,
-        safe_mode=args.safe_mode,
-        search_per_folder=args.search_per_folder,
-        thread_pool_size=args.thread_pool_size,
-        timeout=args.timeout,
-    )
+    looker.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_metabase_api.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -15,16 +16,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
+    args = parse_filled_arguments(parser)
 
-    credentials = metabase.MetabaseApiCredentials(
-        base_url=args.base_url,
-        user=args.username,
-        password=args.password,
-    )
+    credentials = metabase.MetabaseApiCredentials(**args)
     client = metabase.ApiClient(credentials)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    metabase.extract_all(client, **args)

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_metabase_db.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -33,22 +34,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    credentials = metabase.MetabaseDbCredentials(
-        host=args.host,
-        port=args.port,
-        database=args.database,
-        schema=args.schema,
-        user=args.username,
-        password=args.password,
-        encryption_secret_key=args.encryption_secret_key,
-        require_ssl=args.require_ssl,
-    )
+    args = parse_filled_arguments(parser)
+    credentials = metabase.MetabaseDbCredentials(**args)
 
     client = metabase.DbClient(credentials)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    metabase.extract_all(client, **args)

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_mode.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import mode  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -24,15 +25,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    credentials = mode.Credentials(
-        host=args.host,
-        workspace=args.workspace,
-        token=args.token,
-        secret=args.secret,
-    )
-
-    mode.extract_all(
-        credentials,
-        output_directory=args.output,
-    )
+    mode.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_powerbi.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import powerbi  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -20,11 +21,4 @@ def main():
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    powerbi.extract_all(
-        tenant_id=args.tenant_id,
-        client_id=args.client_id,
-        secret=args.secret,
-        scopes=args.scopes,
-        output_directory=args.output,
-    )
+    powerbi.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_qlik.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import qlik  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -22,10 +23,4 @@ def main():
         "missing rights on some assets.",
     )
 
-    args = parser.parse_args()
-    qlik.extract_all(
-        base_url=args.base_url,
-        api_key=args.api_key,
-        output_directory=args.output,
-        except_http_error_statuses=args.except_http_error_statuses,
-    )
+    qlik.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_salesforce.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.warehouse import salesforce  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -29,15 +30,5 @@ def main():
     )
     parser.set_defaults(skip_existing=False)
 
-    args = parser.parse_args()
-
-    salesforce.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-        skip_existing=args.skip_existing,
-    )
+    args = parse_filled_arguments(parser)
+    salesforce.extract_all(output_directory=args.get("output"), **args)

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_salesforce_reporting.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import salesforce_reporting  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -21,13 +22,4 @@ def main():
     parser.add_argument("-b", "--base-url", help="Salesforce instance URL")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    salesforce_reporting.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-    )
+    salesforce_reporting.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/commands/extract_sigma.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import sigma  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -14,10 +15,4 @@ def main():
     parser.add_argument("-a", "--api-token", help="Generated API key")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    sigma.extract_all(
-        host=args.host,
-        client_id=args.client_id,
-        api_token=args.api_token,
-        output_directory=args.output,
-    )
+    sigma.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/utils/__init__.py

@@ -1,3 +1,4 @@
+from .argument_parser import parse_filled_arguments
 from .client import (
     AbstractSourceClient,
     ExtractionQuery,

castor_extractor-0.18.5/castor_extractor/utils/argument_parser.py

@@ -0,0 +1,7 @@
+from argparse import ArgumentParser
+
+
+def parse_filled_arguments(parser: ArgumentParser) -> dict:
+    """Parse arguments and remove all those with None values"""
+    parsed_arguments = vars(parser.parse_args())
+    return {k: v for k, v in parsed_arguments.items() if v is not None}

castor_extractor-0.18.5/castor_extractor/utils/argument_parser_test.py

@@ -0,0 +1,25 @@
+from argparse import Namespace
+
+from .argument_parser import parse_filled_arguments
+
+
+class MockArgumentParser:
+
+    def __init__(self):
+        self.attributes = {}
+
+    def add_argument(self, name, value):
+        self.attributes[name] = value
+
+    def parse_args(self) -> Namespace:
+        return Namespace(**self.attributes)
+
+
+def test_parse_filled_arguments():
+    parser = MockArgumentParser()
+    parser.add_argument("filled", "value")
+    parser.add_argument("unfilled", None)
+    parser.add_argument("empty_str", "")
+
+    expected = {"filled": "value", "empty_str": ""}
+    assert parse_filled_arguments(parser) == expected

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/visualization/domo/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Iterator, Optional, Tuple, Union
+from typing import Iterable, Iterator, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -42,28 +42,13 @@ def iterate_all_data(
     yield DomoAsset.DATAFLOWS, list(deep_serialize(dataflows))
 
 
-def extract_all(
-    api_token: Optional[str] = None,
-    base_url: Optional[str] = None,
-    client_id: Optional[str] = None,
-    cloud_id: Optional[str] = None,
-    developer_token: Optional[str] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Domo API
     Store the output files locally under the given output_directory
     """
-
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-
-    credentials = DomoCredentials(
-        base_url=base_url,
-        client_id=client_id,
-        api_token=api_token,
-        developer_token=developer_token,
-        cloud_id=cloud_id,
-    )
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = DomoCredentials(**kwargs)
     client = DomoClient(credentials=credentials)
 
     ts = current_timestamp()

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/visualization/looker/extract.py

@@ -126,11 +126,10 @@ def extract_all(**kwargs) -> None:
     Extract Data From looker and store it locally in files under the
     output_directory
     """
-    args = {arg: value for arg, value in kwargs.items() if value is not None}
-    extraction_parameters = ExtractionParameters(**args)
+    extraction_parameters = ExtractionParameters(**kwargs)
     output_directory = extraction_parameters.output_directory
 
-    credentials = LookerCredentials(**args)
+    credentials = LookerCredentials(**kwargs)
 
     if extraction_parameters.log_to_stdout:
         set_stream_handler_to_stdout()

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/visualization/metabase/extract.py

@@ -40,12 +40,12 @@ def iterate_all_data(
     )
 
 
-def extract_all(client: ClientMetabase, **kwargs: str) -> None:
+def extract_all(client: ClientMetabase, **kwargs) -> None:
     """
     Extract Data From metabase
     Store the output files locally under the given output_directory
     """
-    output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/visualization/mode/extract.py

@@ -37,11 +37,12 @@ def iterate_all_data(
     yield Asset.MEMBER, deep_serialize(members)
 
 
-def extract_all(credentials: ModeCredentials, output_directory: str) -> None:
+def extract_all(**kwargs) -> None:
     """Extract Data From Mode Analytics and store it locally in files under the output_directory"""
-    output_directory = output_directory or from_env(OUTPUT_DIR)
-
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = ModeCredentials(**kwargs)
     client = Client(credentials)
+
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/visualization/powerbi/extract.py

@@ -1,4 +1,4 @@
-from typing import Iterable, List, Optional, Tuple, Union
+from typing import Iterable, List, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -24,24 +24,13 @@ def iterate_all_data(
         yield asset, deep_serialize(data)
 
 
-def extract_all(
-    tenant_id: str,
-    client_id: str,
-    secret: str,
-    scopes: Optional[List[str]] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from PowerBI REST API
     Store the output files locally under the given output_directory
     """
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-    creds = PowerbiCredentials(
-        tenant_id=tenant_id,
-        client_id=client_id,
-        secret=secret,
-        scopes=scopes,
-    )
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    creds = PowerbiCredentials(**kwargs)
     client = Client(creds)
     ts = current_timestamp()
 

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/visualization/qlik/extract.py

@@ -47,18 +47,15 @@ def iterate_all_data(
 
 
 def extract_all(
-    base_url: Optional[str] = None,
-    api_key: Optional[str] = None,
-    output_directory: Optional[str] = None,
-    except_http_error_statuses: Optional[List[int]] = None,
+    except_http_error_statuses: Optional[List[int]] = None, **kwargs
 ) -> None:
     """
     Extract data from Qlik REST API
     Store the output files locally under the given output_directory
     """
 
-    credentials = QlikCredentials(base_url=base_url, api_key=api_key)
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
+    credentials = QlikCredentials(**kwargs)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
     client = QlikClient(
         credentials=credentials,

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/visualization/salesforce_reporting/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Optional, Tuple, Union
+from typing import Iterable, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -28,29 +28,15 @@ def iterate_all_data(
         yield asset.name.lower(), deep_serialize(data)
 
 
-def extract_all(
-    username: str,
-    password: str,
-    client_id: str,
-    client_secret: str,
-    security_token: str,
-    base_url: str,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Salesforce REST API
     Store the output files locally under the given output_directory
     """
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-    creds = SalesforceCredentials(
-        username=username,
-        password=password,
-        client_id=client_id,
-        client_secret=client_secret,
-        security_token=security_token,
-        base_url=base_url,
-    )
-    client = SalesforceReportingClient(credentials=creds)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = SalesforceCredentials(**kwargs)
+
+    client = SalesforceReportingClient(credentials=credentials)
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/visualization/sigma/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Iterator, Optional, Tuple, Union
+from typing import Iterable, Iterator, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -50,24 +50,15 @@ def iterate_all_data(
     yield SigmaAsset.LINEAGES, list(deep_serialize(lineages))
 
 
-def extract_all(
-    host: Optional[str] = None,
-    client_id: Optional[str] = None,
-    api_token: Optional[str] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Sigma API
     Store the output files locally under the given output_directory
     """
 
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
-    credentials = SigmaCredentials(
-        host=host,
-        client_id=client_id,
-        api_token=api_token,
-    )
+    credentials = SigmaCredentials(**kwargs)
     client = SigmaClient(credentials=credentials)
 
     ts = current_timestamp()

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/warehouse/databricks/client.py

@@ -28,6 +28,7 @@ from .utils import build_path, tag_label
 
 logger = logging.getLogger(__name__)
 
+_DATABRICKS_CLIENT_TIMEOUT = 60
 _MAX_NUMBER_OF_LINEAGE_ERRORS = 1000
 _MAX_THREADS = 10
 _RETRY_ATTEMPTS = 3
@@ -98,6 +99,8 @@ class DatabricksClient(APIClient):
         self._db_blocked = db_blocked
         self._has_table_tags = has_table_tags
         self._has_column_tags = has_column_tags
+
+        self._timeout = _DATABRICKS_CLIENT_TIMEOUT
         self.formatter = DatabricksFormatter()
 
     def execute_sql(

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/castor_extractor/warehouse/salesforce/format.py

@@ -26,12 +26,14 @@ def _field_description(field: Dict[str, Any]) -> str:
 def _to_column_payload(field: dict, position: int, table_name: str) -> dict:
     field_name = field["QualifiedApiName"]
     return {
-        "id": f"{table_name}.{field_name}",
-        "table_id": table_name,
         "column_name": field_name,
-        "description": _field_description(field),
         "data_type": field.get("DataType"),
+        "description": _field_description(field),
+        "id": f"{table_name}.{field_name}",
         "ordinal_position": position,
+        "salesforce_developer_name": field.get("DeveloperName"),
+        "salesforce_tooling_url": field.get("attributes", {}).get("url"),
+        "table_id": table_name,
     }
 
 

{castor_extractor-0.18.2 → castor_extractor-0.18.5}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.18.2"
+version = "0.18.5"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"