castor-extractor 0.18.2__py3-none-any.whl → 0.18.7__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.18.7 - 2024-08-01
+
+* Salesforce: extract table descriptions
+
+## 0.18.6 - 2024-07-30
+
+* BigQuery: introduce extended regions to extract missing queries
+
+## 0.18.5 - 2024-07-17
+
+* Salesforce: extract DeveloperName and tooling url
+
+## 0.18.4 - 2024-07-16
+
+* Fix environment variables assignments for credentials
+
+## 0.18.3 - 2024-07-16
+
+* bump dependencies (minor and patches)
+
 ## 0.18.2 - 2024-07-08
 
 * Added StatusCode handling to SafeMode

DockerfileUsage.md

@@ -0,0 +1,21 @@
+# Dockerfile usage for CastorDoc package
+
+## How To
+
+- The Dockerfile is present on the pypi package
+- For building it you should use this command `docker build -t castor-extractor-looker --build-arg EXTRA=looker .` with replacing looker one or several of: [bigquery,looker,metabase,powerbi,qlik,redshift,snowflake,tableau]
+- For running it you should do `docker run -v <local-path>:/data --env-file <castor-extract-looker.env> castor-extractor-looker` where `</local-path>` have to be replaced and `<castor-extract-looker.env>` have to be set.
+- Extracted datas would be available on `<local-path>`. The path should exists
+- `<castor-extract-looker.env>` would contain env vars for credentials, url...
+
+#### example
+
+```bash
+docker run -v /logs:/data --env-file /config/castor-extract-looker.env castor-extractor-looker
+```
+
+## Limitation
+
+- This docker image is for a specific techno
+- This docker image is based on python 3.11
+- This docker image use the latest castor-extractor package version

castor_extractor/commands/extract_domo.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import domo  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -38,13 +39,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    domo.extract_all(
-        api_token=args.api_token,
-        base_url=args.base_url,
-        client_id=args.client_id,
-        cloud_id=args.cloud_id,
-        developer_token=args.developer_token,
-        output_directory=args.output,
-    )
+    domo.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_looker.py

@@ -1,5 +1,6 @@
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import looker  # type: ignore
 
 
@@ -33,16 +34,4 @@ def main():
         action="store_true",
     )
 
-    args = parser.parse_args()
-
-    looker.extract_all(
-        base_url=args.base_url,
-        client_id=args.username,
-        client_secret=args.password,
-        log_to_stdout=args.log_to_stdout,
-        output_directory=args.output,
-        safe_mode=args.safe_mode,
-        search_per_folder=args.search_per_folder,
-        thread_pool_size=args.thread_pool_size,
-        timeout=args.timeout,
-    )
+    looker.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_metabase_api.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -15,16 +16,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
+    args = parse_filled_arguments(parser)
 
-    credentials = metabase.MetabaseApiCredentials(
-        base_url=args.base_url,
-        user=args.username,
-        password=args.password,
-    )
+    credentials = metabase.MetabaseApiCredentials(**args)
     client = metabase.ApiClient(credentials)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    metabase.extract_all(client, **args)

castor_extractor/commands/extract_metabase_db.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import metabase  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -33,22 +34,9 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    credentials = metabase.MetabaseDbCredentials(
-        host=args.host,
-        port=args.port,
-        database=args.database,
-        schema=args.schema,
-        user=args.username,
-        password=args.password,
-        encryption_secret_key=args.encryption_secret_key,
-        require_ssl=args.require_ssl,
-    )
+    args = parse_filled_arguments(parser)
+    credentials = metabase.MetabaseDbCredentials(**args)
 
     client = metabase.DbClient(credentials)
 
-    metabase.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    metabase.extract_all(client, **args)

castor_extractor/commands/extract_mode.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import mode  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -24,15 +25,4 @@ def main():
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    credentials = mode.Credentials(
-        host=args.host,
-        workspace=args.workspace,
-        token=args.token,
-        secret=args.secret,
-    )
-
-    mode.extract_all(
-        credentials,
-        output_directory=args.output,
-    )
+    mode.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_powerbi.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import powerbi  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -20,11 +21,4 @@ def main():
     )
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    powerbi.extract_all(
-        tenant_id=args.tenant_id,
-        client_id=args.client_id,
-        secret=args.secret,
-        scopes=args.scopes,
-        output_directory=args.output,
-    )
+    powerbi.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_qlik.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import qlik  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -22,10 +23,4 @@ def main():
         "missing rights on some assets.",
     )
 
-    args = parser.parse_args()
-    qlik.extract_all(
-        base_url=args.base_url,
-        api_key=args.api_key,
-        output_directory=args.output,
-        except_http_error_statuses=args.except_http_error_statuses,
-    )
+    qlik.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_salesforce.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.warehouse import salesforce  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -29,15 +30,5 @@ def main():
     )
     parser.set_defaults(skip_existing=False)
 
-    args = parser.parse_args()
-
-    salesforce.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-        skip_existing=args.skip_existing,
-    )
+    args = parse_filled_arguments(parser)
+    salesforce.extract_all(output_directory=args.get("output"), **args)

castor_extractor/commands/extract_salesforce_reporting.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import salesforce_reporting  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -21,13 +22,4 @@ def main():
     parser.add_argument("-b", "--base-url", help="Salesforce instance URL")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    salesforce_reporting.extract_all(
-        username=args.username,
-        password=args.password,
-        client_id=args.client_id,
-        client_secret=args.client_secret,
-        security_token=args.security_token,
-        base_url=args.base_url,
-        output_directory=args.output,
-    )
+    salesforce_reporting.extract_all(**parse_filled_arguments(parser))

castor_extractor/commands/extract_sigma.py

@@ -1,6 +1,7 @@
 import logging
 from argparse import ArgumentParser
 
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 from castor_extractor.visualization import sigma  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
@@ -14,10 +15,4 @@ def main():
     parser.add_argument("-a", "--api-token", help="Generated API key")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-    sigma.extract_all(
-        host=args.host,
-        client_id=args.client_id,
-        api_token=args.api_token,
-        output_directory=args.output,
-    )
+    sigma.extract_all(**parse_filled_arguments(parser))

castor_extractor/utils/__init__.py

@@ -1,3 +1,4 @@
+from .argument_parser import parse_filled_arguments
 from .client import (
     AbstractSourceClient,
     ExtractionQuery,

castor_extractor/utils/argument_parser.py

@@ -0,0 +1,7 @@
+from argparse import ArgumentParser
+
+
+def parse_filled_arguments(parser: ArgumentParser) -> dict:
+    """Parse arguments and remove all those with None values"""
+    parsed_arguments = vars(parser.parse_args())
+    return {k: v for k, v in parsed_arguments.items() if v is not None}

castor_extractor/utils/argument_parser_test.py

@@ -0,0 +1,25 @@
+from argparse import Namespace
+
+from .argument_parser import parse_filled_arguments
+
+
+class MockArgumentParser:
+
+    def __init__(self):
+        self.attributes = {}
+
+    def add_argument(self, name, value):
+        self.attributes[name] = value
+
+    def parse_args(self) -> Namespace:
+        return Namespace(**self.attributes)
+
+
+def test_parse_filled_arguments():
+    parser = MockArgumentParser()
+    parser.add_argument("filled", "value")
+    parser.add_argument("unfilled", None)
+    parser.add_argument("empty_str", "")
+
+    expected = {"filled": "value", "empty_str": ""}
+    assert parse_filled_arguments(parser) == expected

castor_extractor/visualization/domo/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Iterator, Optional, Tuple, Union
+from typing import Iterable, Iterator, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -42,28 +42,13 @@ def iterate_all_data(
     yield DomoAsset.DATAFLOWS, list(deep_serialize(dataflows))
 
 
-def extract_all(
-    api_token: Optional[str] = None,
-    base_url: Optional[str] = None,
-    client_id: Optional[str] = None,
-    cloud_id: Optional[str] = None,
-    developer_token: Optional[str] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Domo API
     Store the output files locally under the given output_directory
     """
-
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-
-    credentials = DomoCredentials(
-        base_url=base_url,
-        client_id=client_id,
-        api_token=api_token,
-        developer_token=developer_token,
-        cloud_id=cloud_id,
-    )
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = DomoCredentials(**kwargs)
     client = DomoClient(credentials=credentials)
 
     ts = current_timestamp()

castor_extractor/visualization/looker/extract.py

@@ -126,11 +126,10 @@ def extract_all(**kwargs) -> None:
     Extract Data From looker and store it locally in files under the
     output_directory
     """
-    args = {arg: value for arg, value in kwargs.items() if value is not None}
-    extraction_parameters = ExtractionParameters(**args)
+    extraction_parameters = ExtractionParameters(**kwargs)
     output_directory = extraction_parameters.output_directory
 
-    credentials = LookerCredentials(**args)
+    credentials = LookerCredentials(**kwargs)
 
     if extraction_parameters.log_to_stdout:
         set_stream_handler_to_stdout()

castor_extractor/visualization/metabase/extract.py

@@ -40,12 +40,12 @@ def iterate_all_data(
     )
 
 
-def extract_all(client: ClientMetabase, **kwargs: str) -> None:
+def extract_all(client: ClientMetabase, **kwargs) -> None:
     """
     Extract Data From metabase
     Store the output files locally under the given output_directory
     """
-    output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

castor_extractor/visualization/mode/extract.py

@@ -37,11 +37,12 @@ def iterate_all_data(
     yield Asset.MEMBER, deep_serialize(members)
 
 
-def extract_all(credentials: ModeCredentials, output_directory: str) -> None:
+def extract_all(**kwargs) -> None:
     """Extract Data From Mode Analytics and store it locally in files under the output_directory"""
-    output_directory = output_directory or from_env(OUTPUT_DIR)
-
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = ModeCredentials(**kwargs)
     client = Client(credentials)
+
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

castor_extractor/visualization/powerbi/extract.py

@@ -1,4 +1,4 @@
-from typing import Iterable, List, Optional, Tuple, Union
+from typing import Iterable, List, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -24,24 +24,13 @@ def iterate_all_data(
         yield asset, deep_serialize(data)
 
 
-def extract_all(
-    tenant_id: str,
-    client_id: str,
-    secret: str,
-    scopes: Optional[List[str]] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from PowerBI REST API
     Store the output files locally under the given output_directory
     """
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-    creds = PowerbiCredentials(
-        tenant_id=tenant_id,
-        client_id=client_id,
-        secret=secret,
-        scopes=scopes,
-    )
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    creds = PowerbiCredentials(**kwargs)
     client = Client(creds)
     ts = current_timestamp()
 

castor_extractor/visualization/qlik/extract.py

@@ -47,18 +47,15 @@ def iterate_all_data(
 
 
 def extract_all(
-    base_url: Optional[str] = None,
-    api_key: Optional[str] = None,
-    output_directory: Optional[str] = None,
-    except_http_error_statuses: Optional[List[int]] = None,
+    except_http_error_statuses: Optional[List[int]] = None, **kwargs
 ) -> None:
     """
     Extract data from Qlik REST API
     Store the output files locally under the given output_directory
     """
 
-    credentials = QlikCredentials(base_url=base_url, api_key=api_key)
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
+    credentials = QlikCredentials(**kwargs)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
     client = QlikClient(
         credentials=credentials,

castor_extractor/visualization/salesforce_reporting/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Optional, Tuple, Union
+from typing import Iterable, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -28,29 +28,15 @@ def iterate_all_data(
         yield asset.name.lower(), deep_serialize(data)
 
 
-def extract_all(
-    username: str,
-    password: str,
-    client_id: str,
-    client_secret: str,
-    security_token: str,
-    base_url: str,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Salesforce REST API
     Store the output files locally under the given output_directory
     """
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
-    creds = SalesforceCredentials(
-        username=username,
-        password=password,
-        client_id=client_id,
-        client_secret=client_secret,
-        security_token=security_token,
-        base_url=base_url,
-    )
-    client = SalesforceReportingClient(credentials=creds)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = SalesforceCredentials(**kwargs)
+
+    client = SalesforceReportingClient(credentials=credentials)
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

castor_extractor/visualization/sigma/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Iterator, Optional, Tuple, Union
+from typing import Iterable, Iterator, Tuple, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -50,24 +50,15 @@ def iterate_all_data(
     yield SigmaAsset.LINEAGES, list(deep_serialize(lineages))
 
 
-def extract_all(
-    host: Optional[str] = None,
-    client_id: Optional[str] = None,
-    api_token: Optional[str] = None,
-    output_directory: Optional[str] = None,
-) -> None:
+def extract_all(**kwargs) -> None:
     """
     Extract data from Sigma API
     Store the output files locally under the given output_directory
     """
 
-    _output_directory = output_directory or from_env(OUTPUT_DIR)
+    _output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
 
-    credentials = SigmaCredentials(
-        host=host,
-        client_id=client_id,
-        api_token=api_token,
-    )
+    credentials = SigmaCredentials(**kwargs)
     client = SigmaClient(credentials=credentials)
 
     ts = current_timestamp()

castor_extractor/warehouse/bigquery/client.py

@@ -1,13 +1,14 @@
+import itertools
 import logging
-from typing import List, Optional, Set, Tuple
+from typing import List, Optional, Set
 
-from google.api_core.exceptions import Forbidden
-from google.api_core.page_iterator import Iterator as PageIterator
+from google.api_core.exceptions import Forbidden  # type: ignore
 from google.cloud.bigquery import Client as GoogleCloudClient  # type: ignore
 from google.cloud.bigquery.dataset import Dataset  # type: ignore
 from google.oauth2.service_account import Credentials  # type: ignore
 
 from ...utils import SqlalchemyClient, retry
+from .types import SetTwoString
 
 logger = logging.getLogger(__name__)
 
@@ -117,16 +118,50 @@ class BigQueryClient(SqlalchemyClient):
             ]
         return self._projects
 
-    def get_regions(self) -> Set[Tuple[str, str]]:
+    def get_regions(self) -> SetTwoString:
         """
-        Returns distinct (project_id, region) available for the given GCP client
+        Returns (project_id, region) available for the given GCP client
+        - Loops trough projects -> datasets -> region
+        - Returns distinct values
+        Example:
+            project_A
+                -> dataset_1:US
+            project_B
+                -> empty
+            project_C
+                -> dataset_2:EU
+                -> dataset_3:EU
+        Will return:
+            { (p_A, US), (p_C, EU) }
         """
         return {
             (ds.project, ds._properties["location"])
             for ds in self._list_datasets()
         }
 
-    def get_datasets(self) -> Set[Tuple[str, str]]:
+    def get_extended_regions(self) -> SetTwoString:
+        """
+        Returns all combinations of (project_id, region) for the given client
+        - Fetch all projects
+        - Fetch all regions (cross projects)
+        - Returns a combination of the two lists
+        Example:
+            project_A
+                -> dataset_1:US
+            project_B
+                -> empty
+            project_C
+                -> dataset_2:EU
+                -> dataset_3:EU
+        Will return:
+            { (p_A, EU), (p_A, US), (p_B, EU), (p_B, US), (p_C, EU), (p_C, US) }
+        """
+        projects = self.get_projects()
+        regions = {ds._properties["location"] for ds in self._list_datasets()}
+        combinations = itertools.product(projects, regions)
+        return set(combinations)
+
+    def get_datasets(self) -> SetTwoString:
         """
         Returns distinct (project_id, dataset_id) available for the given GCP client
         """

castor_extractor/warehouse/bigquery/extract.py

@@ -68,6 +68,7 @@ def extract_all(**kwargs) -> None:
     query_builder = BigQueryQueryBuilder(
         regions=client.get_regions(),
         datasets=client.get_datasets(),
+        extended_regions=client.get_extended_regions(),
     )
 
     storage = LocalStorage(directory=output_directory)

castor_extractor/warehouse/bigquery/query.py

@@ -2,18 +2,16 @@ import logging
 from typing import List, Optional
 
 from ..abstract import (
-    QUERIES_DIR,
     AbstractQueryBuilder,
     ExtractionQuery,
     TimeFilter,
     WarehouseAsset,
 )
-
-# Those queries must be formatted with {region}
-from .types import IterTwoString
+from .types import SetTwoString
 
 logger = logging.getLogger(__name__)
 
+# Those queries must be formatted with {region}
 REGION_REQUIRED = (
     WarehouseAsset.COLUMN,
     WarehouseAsset.DATABASE,
@@ -23,6 +21,11 @@ REGION_REQUIRED = (
     WarehouseAsset.USER,
 )
 
+# Some clients use empty projects (no datasets) to run their SQL queries
+# The extended regions is a combination of all regions with all projects
+# It allows to extract those queries which were left apart before
+EXTENDED_REGION_REQUIRED = (WarehouseAsset.QUERY,)
+
 # Those queries must be formatted with {dataset}
 DATASET_REQUIRED = (WarehouseAsset.VIEW_DDL,)
 
@@ -38,7 +41,7 @@ SHARDED_ASSETS = (WarehouseAsset.TABLE, WarehouseAsset.COLUMN)
 SHARDED_FILE_PATH = "cte/sharded.sql"
 
 
-def _database_formatted(datasets: IterTwoString) -> str:
+def _database_formatted(datasets: SetTwoString) -> str:
     databases = {db for _, db in datasets}
     if not databases:
         # when no datasets are provided condition should pass
@@ -55,10 +58,11 @@ class BigQueryQueryBuilder(AbstractQueryBuilder):
 
     def __init__(
         self,
-        regions: IterTwoString,
-        datasets: IterTwoString,
+        regions: SetTwoString,
+        datasets: SetTwoString,
         time_filter: Optional[TimeFilter] = None,
         sync_tags: Optional[bool] = False,
+        extended_regions: Optional[SetTwoString] = None,
     ):
         super().__init__(
             time_filter=time_filter,
@@ -67,6 +71,7 @@ class BigQueryQueryBuilder(AbstractQueryBuilder):
         self._regions = regions
         self._datasets = datasets
         self._sync_tags = sync_tags
+        self._extended_regions = extended_regions or regions
 
     @staticmethod
     def _format(query: ExtractionQuery, values: dict) -> ExtractionQuery:
@@ -97,6 +102,13 @@ class BigQueryQueryBuilder(AbstractQueryBuilder):
         sharded_statement = self._load_from_file(SHARDED_FILE_PATH)
         return statement.format(sharded_statement=sharded_statement)
 
+    def _get_regions(self, asset: WarehouseAsset) -> SetTwoString:
+        return (
+            self._extended_regions
+            if asset in EXTENDED_REGION_REQUIRED
+            else self._regions
+        )
+
     def build(self, asset: WarehouseAsset) -> List[ExtractionQuery]:
         """
         It would be easier to stitch data directly in the query statement (UNION ALL).
@@ -110,12 +122,14 @@ class BigQueryQueryBuilder(AbstractQueryBuilder):
         query = super().build_default(asset)
 
         if asset in REGION_REQUIRED:
+            regions = self._get_regions(asset)
+
             logger.info(
-                f"\tWill run queries with following region params: {self._regions}",
+                f"\tWill run queries with following region params: {regions}",
             )
             return [
                 self._format(query, {"project": project, "region": region})
-                for project, region in self._regions
+                for project, region in regions
             ]
 
         if asset in DATASET_REQUIRED:

castor_extractor/warehouse/bigquery/types.py

@@ -1,5 +1,4 @@
-from typing import Iterable, Set, Tuple
+from typing import Set, Tuple
 
 SetString = Set[str]
 SetTwoString = Set[Tuple[str, str]]
-IterTwoString = Iterable[Tuple[str, str]]

castor_extractor/warehouse/databricks/client.py

@@ -28,6 +28,7 @@ from .utils import build_path, tag_label
 
 logger = logging.getLogger(__name__)
 
+_DATABRICKS_CLIENT_TIMEOUT = 60
 _MAX_NUMBER_OF_LINEAGE_ERRORS = 1000
 _MAX_THREADS = 10
 _RETRY_ATTEMPTS = 3
@@ -98,6 +99,8 @@ class DatabricksClient(APIClient):
         self._db_blocked = db_blocked
         self._has_table_tags = has_table_tags
         self._has_column_tags = has_column_tags
+
+        self._timeout = _DATABRICKS_CLIENT_TIMEOUT
         self.formatter = DatabricksFormatter()
 
     def execute_sql(

castor_extractor/warehouse/salesforce/client.py

@@ -1,11 +1,15 @@
 import logging
-from typing import Dict, Iterator, List, Tuple
+from typing import Dict, Iterator, List, Optional, Tuple
 
 from tqdm import tqdm  # type: ignore
 
 from ...utils.salesforce import SalesforceBaseClient, SalesforceCredentials
 from .format import SalesforceFormatter
-from .soql import SOBJECT_FIELDS_QUERY_TPL, SOBJECTS_QUERY_TPL
+from .soql import (
+    DESCRIPTION_QUERY_TPL,
+    SOBJECT_FIELDS_QUERY_TPL,
+    SOBJECTS_QUERY_TPL,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -90,13 +94,34 @@ class SalesforceClient(SalesforceBaseClient):
         response = self._call(self.tooling_url, params={"q": query})
         return response["records"]
 
+    def fetch_description(self, table_name: str) -> Optional[str]:
+        """Retrieve description of a table"""
+        query = DESCRIPTION_QUERY_TPL.format(table_name=table_name)
+        response = self._call(self.tooling_url, params={"q": query})
+        if not response["records"]:
+            return None
+        return response["records"][0]["Description"]
+
+    def add_table_descriptions(self, sobjects: List[dict]) -> List[dict]:
+        """
+        Add table descriptions.
+        We use the tooling API which does not handle well the LIMIT in SOQL
+        so we have to retrieve descriptions individually
+        """
+        described_sobjects = []
+        for sobject in sobjects:
+            description = self.fetch_description(sobject["QualifiedApiName"])
+            described_sobjects.append({**sobject, "Description": description})
+        return described_sobjects
+
     def tables(self) -> List[dict]:
         """
         Get Salesforce sobjects as tables
         """
         sobjects = self.fetch_sobjects()
         logger.info(f"Extracted {len(sobjects)} sobjects")
-        return list(self.formatter.tables(sobjects))
+        described_sobjects = self.add_table_descriptions(sobjects)
+        return list(self.formatter.tables(described_sobjects))
 
     def columns(
         self, sobject_names: List[Tuple[str, str]], show_progress: bool = True

castor_extractor/warehouse/salesforce/format.py

@@ -26,12 +26,14 @@ def _field_description(field: Dict[str, Any]) -> str:
 def _to_column_payload(field: dict, position: int, table_name: str) -> dict:
     field_name = field["QualifiedApiName"]
     return {
-        "id": f"{table_name}.{field_name}",
-        "table_id": table_name,
         "column_name": field_name,
-        "description": _field_description(field),
         "data_type": field.get("DataType"),
+        "description": _field_description(field),
+        "id": f"{table_name}.{field_name}",
         "ordinal_position": position,
+        "salesforce_developer_name": field.get("DeveloperName"),
+        "salesforce_tooling_url": field.get("attributes", {}).get("url"),
+        "table_id": table_name,
     }
 
 
@@ -42,7 +44,7 @@ def _to_table_payload(sobject: dict, table_name: str) -> dict:
         "label": sobject["Label"],
         "schema_id": SCHEMA_NAME,
         "table_name": table_name,
-        "description": "",
+        "description": sobject.get("Description"),
         "tags": [],
         "type": "TABLE",
     }

castor_extractor/warehouse/salesforce/format_test.py

@@ -59,7 +59,7 @@ def test__merge_label_and_api_name():
         "label": "foo",
         "schema_id": SCHEMA_NAME,
         "table_name": expected_name,
-        "description": "",
+        "description": None,
         "tags": [],
         "type": "TABLE",
     }

castor_extractor/warehouse/salesforce/soql.py

@@ -1,3 +1,9 @@
+DESCRIPTION_QUERY_TPL = """
+    SELECT Description
+    FROM EntityDefinition
+    WHERE QualifiedApiName = '{table_name}'
+"""
+
 SOBJECTS_QUERY_TPL = """
     SELECT
         DeveloperName,
@@ -13,7 +19,6 @@ SOBJECTS_QUERY_TPL = """
     LIMIT {limit}
 """
 
-
 SOBJECT_FIELDS_QUERY_TPL = """
     SELECT
         DataType,

{castor_extractor-0.18.2.dist-info → castor_extractor-0.18.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.18.2
+Version: 0.18.7
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.18.2.dist-info → castor_extractor-0.18.7.dist-info}/RECORD

@@ -1,24 +1,25 @@
-CHANGELOG.md,sha256=qBflkUup535wc0FM_zDwfGLnLUKsgCuufANAPF7SA8w,11612
+CHANGELOG.md,sha256=uL2xlPDomxLQhjD55aaCwT6ItdW_ziWGTIjjOAF0IXo,11992
 Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
+DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=uF6PXm9ocPITlKVSh9afTakHmpLx3TvawLf-CbMP3wM,3578
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/commands/extract_bigquery.py,sha256=dU4OiYO1V0n32orvZnMh1_xtFKF_VxHNXcVsH3otY-g,1269
 castor_extractor/commands/extract_databricks.py,sha256=SVKyoa-BBUQAM6HRHf1Wdg9-tpICic2yyvXQwHcNBhA,1264
-castor_extractor/commands/extract_domo.py,sha256=lwJ7XeYOeLMF2plf5PK3cL56N9n2yjcDsyRM6UFwKTM,1208
-castor_extractor/commands/extract_looker.py,sha256=gwjIQPOHrXevgU_o2l8vDHHQT8Sb-mGdwcceb6wJJbg,1483
-castor_extractor/commands/extract_metabase_api.py,sha256=5le3lXz91jMYkQO-Hwv1IJF2zrZ9aNWeDbQ9k1NR-zw,841
-castor_extractor/commands/extract_metabase_db.py,sha256=qYHMk5Y6JjNV7KFfX4jjCwwq73vOat8Xmj1zvsRmcZ0,1483
-castor_extractor/commands/extract_mode.py,sha256=f8-iS8uCyjGm0Az-h-4w39DQotaT11tnScyk6vX1Jn8,967
+castor_extractor/commands/extract_domo.py,sha256=jvAawUsUTHrwCn_koK6StmQr4n_b5GyvJi6uu6WS0SM,1061
+castor_extractor/commands/extract_looker.py,sha256=pmz2_ZiC9R0_I64WLfNRcfitTSoWTAoeo2rgZ2lRE-Q,1208
+castor_extractor/commands/extract_metabase_api.py,sha256=5Oqf70wb0M_9QiDVElzZThhedXFy7MmzMVcg7gMhYs4,790
+castor_extractor/commands/extract_metabase_db.py,sha256=oexaEkWpkayV3NnXv00tJkwSix-fiufQd4Ln5qSN2EQ,1259
+castor_extractor/commands/extract_mode.py,sha256=Q4iO-VAKMg4zFPejhAO-foZibL5Ht3jsnhWKwJ0oqUU,823
 castor_extractor/commands/extract_mysql.py,sha256=7AH5qMzeLTsENCOeJwtesrWg8Vo8MCEq8fx2YT74Mcw,1034
 castor_extractor/commands/extract_postgres.py,sha256=pX0RnCPi4nw6QQ6wiAuZ_Xt3ZbDuMUG9aQKuqFgJtAU,1154
-castor_extractor/commands/extract_powerbi.py,sha256=e6MXDNOafdp0w4ZtOnE5z5o_CxvaodUbbQFk__pDiM4,875
-castor_extractor/commands/extract_qlik.py,sha256=mSeyGOprTyBExes-lzp___7tgBS1KeyTVpwKAqMpkiw,989
+castor_extractor/commands/extract_powerbi.py,sha256=f0G5w61KXExJ6Sw39_mJIwqQNpLorE5-LKmZXlUqvKI,783
+castor_extractor/commands/extract_qlik.py,sha256=VBe_xFKh_nR0QSFFIncAaC8yDqBeMa6VunBAga7AeGg,891
 castor_extractor/commands/extract_redshift.py,sha256=bdLp7d7ImZoKCkWc3f3NXF1imIzMVT43_KPI-x4UVac,1155
-castor_extractor/commands/extract_salesforce.py,sha256=s2o799ePjQFYsVwZbrGEflzOIwJKtxUMb9pcF4-K90Y,1400
-castor_extractor/commands/extract_salesforce_reporting.py,sha256=rmAo--dl_m2x7TtJ29w1PfsKt9tQDZocTdvwTqj-fnI,1146
-castor_extractor/commands/extract_sigma.py,sha256=agwfKj55C81-kect3K6xSJVBv3TDuPT6fGWSTytkQ2o,703
+castor_extractor/commands/extract_salesforce.py,sha256=3j3YTmMkPAwocR-B1ozJQai0UIZPtpmAyWj-hHvdWn4,1226
+castor_extractor/commands/extract_salesforce_reporting.py,sha256=FdANTNiLkIPdm80XMYxWReHjdycLsIa61pyeCD-sUDk,962
+castor_extractor/commands/extract_sigma.py,sha256=sxewHcZ1Doq35V2qnpX_zCKKXkrb1_9bYjUMg7BOW-k,643
 castor_extractor/commands/extract_snowflake.py,sha256=vYiruxRoo--GeMemOGsSE1w9kcKTh_y4E165HtMVzkM,1982
 castor_extractor/commands/extract_sqlserver.py,sha256=lwhbcNChaXHZgMgSOch3faVr7WJw-sDU6GHl3lzBt_0,1141
 castor_extractor/commands/extract_tableau.py,sha256=u-6UCd-kfXwyhNWYxZusqtgTTYkf4gAJS1vRIYWsAVU,1415
@@ -44,7 +45,9 @@ castor_extractor/uploader/env_test.py,sha256=ClCWWtwd2N-5ClIDUxVMeKkWfhhOTxpppsX
 castor_extractor/uploader/upload.py,sha256=W1TGqO8_PtFdR661qNlh6v-LOIRvoJoda65-5OujFXs,3239
 castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
 castor_extractor/uploader/utils.py,sha256=Tx_i875L2vJ8btOLV3-L0UMEFiyhH8E5n0XXRyLjO0Y,793
-castor_extractor/utils/__init__.py,sha256=oP5yVp7Vw7PqixkumDydEBAOncaFXmwf1IHqtDwLXGs,1275
+castor_extractor/utils/__init__.py,sha256=H7VtWoAqT1OWxhvTaylVG_4BrAMI_YD09WyI6vv6PYE,1327
+castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
+castor_extractor/utils/argument_parser_test.py,sha256=CvqXxX71V6nzKrsdGpUoU4EB_2XDtw7NSvz0DjpZt-s,634
 castor_extractor/utils/client/__init__.py,sha256=CRE-xJKm6fVV9dB8ljzB5YoOxX4I1sCD1KSgqs3Y8_Y,161
 castor_extractor/utils/client/abstract.py,sha256=aA5Qcb9TwWDSMq8WpXbGkOB20hehwX2VTpqQAwV76wk,2048
 castor_extractor/utils/client/api.py,sha256=AGDj2JH__Q_x7RQdodoVazGvjGQJ9TzNqs-XEX6Hrms,1840
@@ -110,7 +113,7 @@ castor_extractor/visualization/domo/client/credentials.py,sha256=_dcoJCQa5dKz2EG
 castor_extractor/visualization/domo/client/endpoints.py,sha256=6UI5psMYaIa1Pq_Gulb4cNna7NZ16xMotScX7yg5TRQ,2367
 castor_extractor/visualization/domo/client/pagination.py,sha256=ukVkHVzoH4mfZ29H9YcnC2YrdVolP10wv25J6Q3ehRw,821
 castor_extractor/visualization/domo/client/pagination_test.py,sha256=nV4yZWfus13QFCr-tlBUgwva21VqfpF6P-0ks_Awwis,581
-castor_extractor/visualization/domo/extract.py,sha256=RfaXHbjMKzcNsg27Avc-vpIH654H-g9boakWVJrCxfE,2230
+castor_extractor/visualization/domo/extract.py,sha256=W_HRz7GGShlU4JVAR017q96wM8fgR8dhUzA-o8icWkc,1846
 castor_extractor/visualization/looker/__init__.py,sha256=mem0020YeP4_5zDnqRXOW3gWfiU1_vl_oKj0E8qA88g,194
 castor_extractor/visualization/looker/api/__init__.py,sha256=HDLsLy3kDWHIplAzLl1_u_bvGlgY6cuplf8myJTdfTg,169
 castor_extractor/visualization/looker/api/client.py,sha256=1iXFpwSL39K5Nimh6pFFg-PJ45nGrrdWjEUMH7cp8r8,9896
@@ -124,7 +127,7 @@ castor_extractor/visualization/looker/api/utils.py,sha256=TJqq9UBVFtS33VB1zHzT6k
 castor_extractor/visualization/looker/assets.py,sha256=K08nV6MMIpfF9r91TmCO7_62smHzGRv3gR4aIOootMQ,827
 castor_extractor/visualization/looker/constant.py,sha256=UgfOdu6Xz3stXTeh3pzWus6I3tiPK51NL8EWi0C3G2c,129
 castor_extractor/visualization/looker/constants.py,sha256=522UNbOxx-VDJn4iPUVCpSKCQwrvEOjugsuS1yzyecI,361
-castor_extractor/visualization/looker/extract.py,sha256=8D0AsgvhS-19I-a8CiZ-iC2m-1AtHEgDLAWh4KwW6sI,5164
+castor_extractor/visualization/looker/extract.py,sha256=Rt_XJIiqfNcBt6hSZMyJc32h4ZeRPRWvrRNrX5Fx13M,5090
 castor_extractor/visualization/looker/fields.py,sha256=WmiSehmczWTufCLg4r2Ozq2grUpzxDNvIAHyGuOoGs4,636
 castor_extractor/visualization/looker/fields_test.py,sha256=7Cwq8Qky6aTZg8nCHp1gmPJtd9pGNB4QeMIRRWdHo5w,782
 castor_extractor/visualization/looker/multithreading.py,sha256=6CrMOy9kMBfhHnZI7XrpNNyBiYYCO3CE4AuIjQVlLH0,2610
@@ -151,7 +154,7 @@ castor_extractor/visualization/metabase/client/decryption.py,sha256=q1LyFlHUx1xU
 castor_extractor/visualization/metabase/client/decryption_test.py,sha256=qZAIksj1gj5MKjapDPVWz3BrkzZ3Mhd1V0EPzG2vb0w,590
 castor_extractor/visualization/metabase/client/shared.py,sha256=0uh-jwdELjZDFo5ZDyHPa7EeNCMTCMgz2k-pPQq99VI,389
 castor_extractor/visualization/metabase/errors.py,sha256=A6uykA5IO-TwcnyoJsqzNFjPHIokmSRawL1Nkiv6750,1074
-castor_extractor/visualization/metabase/extract.py,sha256=EEAgveLy_JUnQpaFTDiQnKRuPFALfWIpyHO7txEO9m8,1775
+castor_extractor/visualization/metabase/extract.py,sha256=IK6Ri5AstlGr_2VgvemZcwnWv4G__IrZbrdkaHmbCKA,1760
 castor_extractor/visualization/metabase/types.py,sha256=2dBZCUs5_x_y2rWbUVo6IGDWbgRZQm-c-tNetbkBzg4,45
 castor_extractor/visualization/mode/__init__.py,sha256=AoXzz7dGzyjDm0mjUNmkCiT9CpkA-t_z_YEXwpxpOwI,149
 castor_extractor/visualization/mode/assets.py,sha256=Gc8Z2oi5Jn8yyh9tj6OqYYYi2-gJ4zgos6XWLxirAzM,1574
@@ -162,7 +165,7 @@ castor_extractor/visualization/mode/client/client_test.py,sha256=zljcREpnIm3ZQtj
 castor_extractor/visualization/mode/client/constants.py,sha256=_Si5AF6VnpoSfnNNgb5afV1mby1E-_1GGE2WLSR_fRI,453
 castor_extractor/visualization/mode/client/credentials.py,sha256=fE12wiTeDvyKzDJ3XwXLk7bOW00uyESJpvIVmA9Xekk,617
 castor_extractor/visualization/mode/errors.py,sha256=SKpFT2AiLOuWx2VRLyO7jbAiKcGDFXXrsebpNEKtr0E,1495
-castor_extractor/visualization/mode/extract.py,sha256=mR9AloDqxuxcpoYsgrl-mqDts5MvWWY4dh-wwvUHveQ,1618
+castor_extractor/visualization/mode/extract.py,sha256=g_X7k8L8MldFPbuwOrnyNMF3BEH1r-IAAgNmi3KLF-U,1623
 castor_extractor/visualization/powerbi/__init__.py,sha256=VylJP6kw4yd2zGj31V-U9UXdhnPS9MK2Fz7Sd9KTkKI,119
 castor_extractor/visualization/powerbi/assets.py,sha256=SASUjxtoOMag3NAlZfhpCy0sLap7WfENEMaEZuBrw6o,801
 castor_extractor/visualization/powerbi/client/__init__.py,sha256=ewaEKS_shQlBbCpf-12J-bx3aUIpxbFZRJUL4eNOOno,97
@@ -173,7 +176,7 @@ castor_extractor/visualization/powerbi/client/rest.py,sha256=oBPc7n58We2TAcL7IuX
 castor_extractor/visualization/powerbi/client/rest_test.py,sha256=WMd8042r0nbUZECCVVC9JpJuNica2qlQBBbM8QuYfcQ,8528
 castor_extractor/visualization/powerbi/client/utils.py,sha256=0RcoWcKOdvIGH4f3lYDvufmiMo4tr_ABFlITSrvXjTs,541
 castor_extractor/visualization/powerbi/client/utils_test.py,sha256=ULHL2JLrcv0xjW2r7QF_ce2OaGeeSzajkMDywJ8ZdVA,719
-castor_extractor/visualization/powerbi/extract.py,sha256=PuWR4Q25k3NYNkMSrDVH5SJ6tTvBlxfdkeeP0FoSCiQ,1342
+castor_extractor/visualization/powerbi/extract.py,sha256=OPF2QxP44iruQWARmpAx1HSDj7NLadPApIWVl1yRVZI,1101
 castor_extractor/visualization/qlik/__init__.py,sha256=u6lIfm_WOykBwt6SlaB7C0Dtx37XBliUbM5oWv26gC8,177
 castor_extractor/visualization/qlik/assets.py,sha256=cG3Cqrj2s4inAqfW6dOaxRape2RPiCeGSYjKsRJRLLo,1657
 castor_extractor/visualization/qlik/client/__init__.py,sha256=5O5N9Jrt3d99agFEJ28lKWs2KkDaXK-lZ07IUtLj56M,130
@@ -190,13 +193,13 @@ castor_extractor/visualization/qlik/client/engine/websocket.py,sha256=KVgymI51s7
 castor_extractor/visualization/qlik/client/master.py,sha256=MQI62W9zG5wsIE2RZ9p-xuOF2kN1H90JU7H3qXh7CYo,3146
 castor_extractor/visualization/qlik/client/rest.py,sha256=HCAzWkTNiscu_6i-9EAC65-uPqWNORevx4bMKKI-wPs,6015
 castor_extractor/visualization/qlik/client/rest_test.py,sha256=yfiUht6BcpBYS2uGdaKCH-tYe0fQ-joM4MbitKwOf24,1799
-castor_extractor/visualization/qlik/extract.py,sha256=1S5KPnAg7aynh_xd_gMQxRpMTyYAGyzO3-JT0VNDIs0,2300
+castor_extractor/visualization/qlik/extract.py,sha256=2ACqmWheVdkIXmDtmhdunf8oyNPvTb_2LaMIF9SBZvU,2172
 castor_extractor/visualization/salesforce_reporting/__init__.py,sha256=MvArD0GKNIpCDvLIYcpKrjMjFLhMyDETK6i3k0Fb6Tk,124
 castor_extractor/visualization/salesforce_reporting/assets.py,sha256=2J-iAmJGGDufOcJUgE47M3-dEcjYXcVyVUNcmHrj79w,271
 castor_extractor/visualization/salesforce_reporting/client/__init__.py,sha256=DIA6f_vNJZqT89qVYxg98Le7QeDn2y0Qew03V3J9t9o,44
 castor_extractor/visualization/salesforce_reporting/client/rest.py,sha256=hzaXWLcYt0aAHXK46DbsLmzocjRY1llwrNj8_3TObKs,1849
 castor_extractor/visualization/salesforce_reporting/client/soql.py,sha256=DHTi058UEaZKZnxJVmsCouPfA9Lgr3gFY6vY7NiqyMM,1584
-castor_extractor/visualization/salesforce_reporting/extract.py,sha256=5QwZwP27uXrFJSf00El7Ku592-9fhmCtTdiUGpNkHZM,1678
+castor_extractor/visualization/salesforce_reporting/extract.py,sha256=RMhlf7NeYiAO_8hkFk4-JxTghNbC0vtX3P2M80qt9U8,1339
 castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13cMtnKNytiFRomwav27Q,114
 castor_extractor/visualization/sigma/assets.py,sha256=JZ1Cpxnml8P3mIJoTUM57hvylB18ErECQXaP5FF63O4,268
 castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
@@ -205,7 +208,7 @@ castor_extractor/visualization/sigma/client/client_test.py,sha256=BdxJsmTyhslup4
 castor_extractor/visualization/sigma/client/credentials.py,sha256=vXFBUeU9XpfmCGAdm8b2dC0tLQ6ITQnGOZcYxKFn_2c,486
 castor_extractor/visualization/sigma/client/endpoints.py,sha256=7yNDL0p9wZDE7CcVOj9Z1eP2tdJE1CUT9vbfSEV2kOY,1157
 castor_extractor/visualization/sigma/client/pagination.py,sha256=EZGMaONTzZ15VINUAtH0sOmJae0JlDjOQPKBXK_V8sg,667
-castor_extractor/visualization/sigma/extract.py,sha256=Qh0Dt1-IXg379Q4UWYhGul_zlj1LEowy25AcuwgYXV0,2501
+castor_extractor/visualization/sigma/extract.py,sha256=pnArK5-F6DZcO0f3wp3_km_Od0f18Qmw9yTmynJ-2TU,2278
 castor_extractor/visualization/tableau/__init__.py,sha256=hDohrWjkorrX01JMc154aa9vi3ZqBKmA1lkfQtMFfYE,114
 castor_extractor/visualization/tableau/assets.py,sha256=mfBUzcBCLyiU9gnTB_6rvtiB5yXSDU99nezhGC__HQo,1270
 castor_extractor/visualization/tableau/client/__init__.py,sha256=FQX1MdxS8Opn3Oyq8eby7suk3ANbLlpzzCPQ3zqvk0I,78
@@ -260,10 +263,10 @@ castor_extractor/warehouse/abstract/query.py,sha256=GAgeISCmAdrkTKzFGO79hQDf6SA6
 castor_extractor/warehouse/abstract/time_filter.py,sha256=bggIONfMmUxffkA6TwM3BsjfS2l9WFxPq8krfsau5pw,935
 castor_extractor/warehouse/abstract/time_filter_test.py,sha256=PIkegB7KOKBdpc6zIvmyl_CeQyADeFDplyQ8HTNU5LA,448
 castor_extractor/warehouse/bigquery/__init__.py,sha256=PCGNYdi7dHv-SyanUWzRuBp-ypuQ01PkDaQjVnaNhbM,170
-castor_extractor/warehouse/bigquery/client.py,sha256=ypLKXvvfR0RtKex4T2mNvoef4T-jRF1T_RZGCZ6qbOM,4495
+castor_extractor/warehouse/bigquery/client.py,sha256=UefTefQp0S9kpRQzIzJhlm6VcH5uoCRAHHCgfRB6I58,5606
 castor_extractor/warehouse/bigquery/client_test.py,sha256=Ym8e4d--0YQwiVcNUnXLx0X-X6ZznwNMBMbMaDS5oEA,1514
 castor_extractor/warehouse/bigquery/credentials.py,sha256=oCZ8H7qpudKzwM7PRMpVAmWXt7bjIRa8Harmp-ysQJ4,425
-castor_extractor/warehouse/bigquery/extract.py,sha256=vZFxJC1LtUMph5UhfhYdJLnsEto18IOERKzrt71jqJg,2883
+castor_extractor/warehouse/bigquery/extract.py,sha256=lwiam_9YsaUumoY4OwgRMMV4OwLhPwAwnQNCkt-JIeo,2939
 castor_extractor/warehouse/bigquery/queries/.sqlfluff,sha256=ce8UDW2k39v6RBVxgKqjOHHYMoGN9S9f7BCZNHHhox8,30
 castor_extractor/warehouse/bigquery/queries/column.sql,sha256=NxdTnHwomHTEGSc-UoXFKUwg59I9XAOwrSau7JUqGQE,1815
 castor_extractor/warehouse/bigquery/queries/cte/sharded.sql,sha256=-G7_4lxV7UPe72mYlp4HDGeM_fJjZWuXJ7Q0vxvj5_U,1454
@@ -274,10 +277,10 @@ castor_extractor/warehouse/bigquery/queries/table.sql,sha256=D15UNem03Bfcy0JYvKT
 castor_extractor/warehouse/bigquery/queries/table_with_tags.sql,sha256=mhWQHaLgyumtdJX5XyEbdrn_Qtt-RCu4cH1WLM6TN9o,2660
 castor_extractor/warehouse/bigquery/queries/user.sql,sha256=l-fkNGWJVdZwVhbFZL23B8tve-UKXAI6HRlnQq0gIwM,192
 castor_extractor/warehouse/bigquery/queries/view_ddl.sql,sha256=obCm-IN9V8_YSZTwcgNSBDD0ZXPgRjlxJjrZDSEH2MU,326
-castor_extractor/warehouse/bigquery/query.py,sha256=hrFfjd5jW2oQnZ6ozlkn-gDe6sCIzu5zSX19T9W6fIk,4162
-castor_extractor/warehouse/bigquery/types.py,sha256=LZVWSmE57lOemNbB5hBRyYmDk9bFAU4nbRaJWALl6N8,140
+castor_extractor/warehouse/bigquery/query.py,sha256=5Qc8PEa-kQKpTzguj4RNCAwKyvzWt20vAESYNB0lueo,4768
+castor_extractor/warehouse/bigquery/types.py,sha256=DHK3wUaaLyLMp7LP-7QkXTDYpYTZiPtvptAOkpxgp4g,88
 castor_extractor/warehouse/databricks/__init__.py,sha256=YG3YSIJgCFRjjI8eExy9T7qGnfnjWhMFh8c15KTs_BA,184
-castor_extractor/warehouse/databricks/client.py,sha256=pnYb6nl9U33nH6UukYP5piWGYF-m1SP2TYiWiUozM_4,20552
+castor_extractor/warehouse/databricks/client.py,sha256=ohqsCyLdUJCJGTBK-IBHCV2BUujPG2VsOkc8qAapaPk,20636
 castor_extractor/warehouse/databricks/client_test.py,sha256=KNp4Hi_CC6GwiW2QDJQQwqALfUebuT9D_qL6FuP_8tY,5246
 castor_extractor/warehouse/databricks/credentials.py,sha256=iphbVynVTQXMEbJy4QaT5fer-GpOi7QtbAlg8R7-Lj4,598
 castor_extractor/warehouse/databricks/extract.py,sha256=VYygE06f7ngYWVlRa48O6drLIZF-_4IBJdyXTYfxZQU,7395
@@ -326,12 +329,12 @@ castor_extractor/warehouse/redshift/queries/user.sql,sha256=sEXveJAuNvZacvpI6Wfw
 castor_extractor/warehouse/redshift/queries/view_ddl.sql,sha256=Pkyh_QT6d4rhTeyiVcqw6O8CRl7NEhk2p7eM5YIn5kg,719
 castor_extractor/warehouse/redshift/query.py,sha256=0C81rkt2cpkWrJIxxwALDyqr-49vlqQM04y_N6wwStc,540
 castor_extractor/warehouse/salesforce/__init__.py,sha256=NR4aNea5jeE1xYqeZ_29deeN84CkN0_D_Z7CLQdJvFY,137
-castor_extractor/warehouse/salesforce/client.py,sha256=ETnZ3n-GFFH0XohDB2ft74wI1HMspvTefR3k7ne-pmI,3891
+castor_extractor/warehouse/salesforce/client.py,sha256=F3UdD5-9umEU-u_c7uVtaksg81VZeXRW83BVsFvYDkE,4902
 castor_extractor/warehouse/salesforce/constants.py,sha256=GusduVBCPvwpk_Im6F3bDvXeNQ7hRnCMdIAjIg65RnE,52
 castor_extractor/warehouse/salesforce/extract.py,sha256=GaxkGWhdksDT_rlT24KX8DMpWnhKlhDMAUvBPGalli0,3454
-castor_extractor/warehouse/salesforce/format.py,sha256=f5mMJyPsVU1ZSLe5WGCUOpj2SyW7_DFfzNVNu_m2aV0,3126
-castor_extractor/warehouse/salesforce/format_test.py,sha256=HBlAYBoCOHaq_QOFudZlpcZb5TyZWV9v-cxK4tklg50,2253
-castor_extractor/warehouse/salesforce/soql.py,sha256=pAEaJE8ZUcyN3ptBsZGzNcGRhCcU81X6RMlnF1HRMw4,1063
+castor_extractor/warehouse/salesforce/format.py,sha256=eiPM_4i_m3FEg_2jkMYlhaRBg3gTvV-9xQuk8ghJZiM,3289
+castor_extractor/warehouse/salesforce/format_test.py,sha256=aWUUYDAX-hN1XQJHlv6ZtI2noXWjRobV-9zdjiXR5n4,2255
+castor_extractor/warehouse/salesforce/soql.py,sha256=XB8ohKwHFfC4Xger7Y84DXLW17IJDye_bZ3FL6DCcOI,1188
 castor_extractor/warehouse/snowflake/__init__.py,sha256=TEGXTyxWp4Tr9gIHb-UFVTRKj6YWmrRtqHruiKSZGiY,174
 castor_extractor/warehouse/snowflake/client.py,sha256=XT0QLVNff_586SDuMe40iu8FCwPDh2uBV5aKc1Ql914,5555
 castor_extractor/warehouse/snowflake/client_test.py,sha256=ihWtOOAQfh8pu5JTr_EWfqefKOVIaJXznACURzaU1Qs,1432
@@ -372,8 +375,8 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.18.2.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.18.2.dist-info/METADATA,sha256=5Fjm4WkyShhmoOMNzgj8QB8tpOq_jw91ufKOVMIVeU0,7209
-castor_extractor-0.18.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.18.2.dist-info/entry_points.txt,sha256=SbyPk58Gh-FRztfCNnUZQ6w7SatzNJFZ6GIJLNsy7tI,1427
-castor_extractor-0.18.2.dist-info/RECORD,,
+castor_extractor-0.18.7.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.18.7.dist-info/METADATA,sha256=QCkJjnGnFsOfbZ808-jK6dthnNFKqzC_YLydG1sUSqM,7209
+castor_extractor-0.18.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+castor_extractor-0.18.7.dist-info/entry_points.txt,sha256=SbyPk58Gh-FRztfCNnUZQ6w7SatzNJFZ6GIJLNsy7tI,1427
+castor_extractor-0.18.7.dist-info/RECORD,,