castor-extractor 0.16.3__py3-none-any.whl → 0.16.4__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.16.4 - 2024-04-25
+
+* Salesforce: extract sobjects and fields
+
 ## 0.16.3 - 2024-04-24
 
 * Databricks: Extract table owners

castor_extractor/commands/extract_databricks.py

@@ -1,7 +1,10 @@
+import logging
 from argparse import ArgumentParser
 
 from castor_extractor.warehouse import databricks  # type: ignore
 
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
 
 def main():
     parser = ArgumentParser()

castor_extractor/commands/extract_salesforce.py

@@ -0,0 +1,43 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.warehouse import salesforce  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def main():
+    parser = ArgumentParser()
+
+    parser.add_argument("-u", "--username", help="Salesforce username")
+    parser.add_argument("-p", "--password", help="Salesforce password")
+    parser.add_argument("-c", "--client-id", help="Salesforce client id")
+    parser.add_argument(
+        "-s", "--client-secret", help="Salesforce client secret"
+    )
+    parser.add_argument(
+        "-t", "--security-token", help="Salesforce security token"
+    )
+    parser.add_argument("-b", "--base-url", help="Salesforce instance URL")
+    parser.add_argument("-o", "--output", help="Directory to write to")
+
+    parser.add_argument(
+        "--skip-existing",
+        dest="skip_existing",
+        action="store_true",
+        help="Skips files already extracted instead of replacing them",
+    )
+    parser.set_defaults(skip_existing=False)
+
+    args = parser.parse_args()
+
+    salesforce.extract_all(
+        username=args.username,
+        password=args.password,
+        client_id=args.client_id,
+        client_secret=args.client_secret,
+        security_token=args.security_token,
+        base_url=args.base_url,
+        output_directory=args.output,
+        skip_existing=args.skip_existing,
+    )

castor_extractor/commands/extract_salesforce_reporting.py

@@ -11,23 +11,23 @@ def main():
 
     parser.add_argument("-u", "--username", help="Salesforce username")
     parser.add_argument("-p", "--password", help="Salesforce password")
-    parser.add_argument("-k", "--consumer-key", help="Salesforce consumer key")
+    parser.add_argument("-c", "--client-id", help="Salesforce client id")
     parser.add_argument(
-        "-s", "--consumer-secret", help="Salesforce consumer secret"
+        "-s", "--client-secret", help="Salesforce client secret"
     )
     parser.add_argument(
         "-t", "--security-token", help="Salesforce security token"
     )
-    parser.add_argument("-l", "--url", help="Salesforce instance URL")
+    parser.add_argument("-b", "--base-url", help="Salesforce instance URL")
     parser.add_argument("-o", "--output", help="Directory to write to")
 
     args = parser.parse_args()
     salesforce_reporting.extract_all(
         username=args.username,
         password=args.password,
-        consumer_key=args.consumer_key,
-        consumer_secret=args.consumer_secret,
+        client_id=args.client_id,
+        client_secret=args.client_secret,
         security_token=args.security_token,
-        instance_url=args.url,
+        base_url=args.base_url,
         output_directory=args.output,
     )

castor_extractor/utils/client/api.py

@@ -1,23 +1,25 @@
-import json
-from typing import Optional
+import logging
+from typing import Any, Callable, Dict, Literal, Optional
 
 import requests
 
-from ...warehouse.databricks.credentials import DatabricksCredentials
+logger = logging.getLogger(__name__)
 
 DEFAULT_TIMEOUT_MS = 30_000
-APICredentials = DatabricksCredentials
+
+# https://requests.readthedocs.io/en/latest/api/#requests.request
+HttpMethod = Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]
 
 
 class APIClient:
     """
     API client
-    - used for Databricks Unity Catalog for now
-    - authentication via access token for now
+    - authentication via access token
     """
 
-    def __init__(self, credentials: APICredentials):
-        self.credentials = credentials
+    def __init__(self, host: str, token: Optional[str] = None):
+        self._host = host
+        self._token = token or ""
         self._timeout = DEFAULT_TIMEOUT_MS
 
     @staticmethod
@@ -26,25 +28,32 @@ class APIClient:
             host = "https://" + host
         return f"{host.strip('/')}/{path}"
 
-    def _headers(self):
-        return {
-            "Content-type": "application/json",
-            "Authorization": f"Bearer {self.credentials.token}",
-        }
-
-    def get(self, path: str, payload: Optional[dict] = None) -> dict:
-        """
-        path: REST API operation path, such as /api/2.0/clusters/get
-        """
-        url = self.build_url(self.credentials.host, path)
-        response = requests.get(
-            url,
-            data=json.dumps(payload or dict()),
-            headers=self._headers(),
-            timeout=self._timeout,
+    def _headers(self) -> Dict[str, str]:
+        if self._token:
+            return {"Authorization": f"Bearer {self._token}"}
+        return dict()
+
+    def _call(
+        self,
+        url: str,
+        method: HttpMethod = "GET",
+        *,
+        params: Optional[dict] = None,
+        data: Optional[dict] = None,
+        processor: Optional[Callable] = None,
+    ) -> Any:
+        logger.debug(f"Calling {method} on {url}")
+        result = requests.request(
+            method, url, headers=self._headers(), params=params, json=data
         )
+        result.raise_for_status()
 
-        if response.content:
-            return json.loads(response.content)
+        if processor:
+            return processor(result)
 
-        return {}
+        return result.json()
+
+    def get(self, path: str, payload: Optional[dict] = None) -> dict:
+        """path: REST API operation path, such as /api/2.0/clusters/get"""
+        url = self.build_url(self._host, path)
+        return self._call(url=url, data=payload)

castor_extractor/utils/salesforce/__init__.py

@@ -0,0 +1,3 @@
+from .client import SalesforceBaseClient
+from .constants import Keys
+from .credentials import SalesforceCredentials, to_credentials

castor_extractor/utils/salesforce/client.py

@@ -0,0 +1,84 @@
+import logging
+from typing import Iterator, Optional, Tuple
+
+from requests import Response
+
+from ...utils.client.api import APIClient
+from .constants import DEFAULT_API_VERSION, DEFAULT_PAGINATION_LIMIT
+from .credentials import SalesforceCredentials
+
+logger = logging.getLogger(__name__)
+
+
+class SalesforceBaseClient(APIClient):
+    """
+    Salesforce API client.
+    https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_rest.htm
+    """
+
+    api_version = DEFAULT_API_VERSION
+    pagination_limit = DEFAULT_PAGINATION_LIMIT
+
+    PATH_TPL = "services/data/v{version}/{suffix}"
+
+    def __init__(self, credentials: SalesforceCredentials):
+        super().__init__(host=credentials.base_url)
+        self._token = self._access_token(credentials)
+
+    def _access_token(self, credentials: SalesforceCredentials) -> str:
+        url = self.build_url(self._host, "services/oauth2/token")
+        response = self._call(
+            url, "POST", params=credentials.token_request_payload()
+        )
+        return response["access_token"]
+
+    def _full_url(self, suffix: str) -> str:
+        path = self.PATH_TPL.format(version=self.api_version, suffix=suffix)
+        return self.build_url(self._host, path)
+
+    @property
+    def query_url(self) -> str:
+        """Returns the query API url"""
+        return self._full_url("query")
+
+    @property
+    def tooling_url(self) -> str:
+        """Returns the tooling API url"""
+        return self._full_url("tooling/query")
+
+    @staticmethod
+    def _query_processor(response: Response) -> Tuple[dict, Optional[str]]:
+        results = response.json()
+        return results["records"], results.get("nextRecordsUrl")
+
+    def _has_reached_pagination_limit(self, page_number: int) -> bool:
+        return page_number > self.pagination_limit
+
+    def _query_first_page(self, query: str) -> Tuple[Iterator[dict], str]:
+        url = self.query_url
+        logger.info("querying page 0")
+        records, next_page_url = self._call(
+            url, params={"q": query}, processor=self._query_processor
+        )
+        return records, next_page_url
+
+    def _query_all(self, query: str) -> Iterator[dict]:
+        """
+        Run a SOQL query over salesforce API.
+
+        more: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/dome_query.htm
+        """
+        records, next_page_path = self._query_first_page(query)
+        yield from records
+
+        page_count = 1
+        while next_page_path and not self._has_reached_pagination_limit(
+            page_count
+        ):
+            logger.info(f"querying page {page_count}")
+            url = self.build_url(self._host, next_page_path)
+            records, next_page = self._call(
+                url, processor=self._query_processor
+            )
+            yield from records
+            page_count += 1

castor_extractor/utils/salesforce/client_test.py

@@ -0,0 +1,21 @@
+from unittest.mock import patch
+
+from .client import SalesforceBaseClient
+from .credentials import SalesforceCredentials
+
+
+@patch.object(SalesforceBaseClient, "_call")
+def test_SalesforceBaseClient__urls(mock_call):
+    mock_call.return_value = {"access_token": "the_token"}
+    credentials = SalesforceCredentials(
+        username="usr",
+        password="pw",
+        client_id="key",
+        client_secret="secret",
+        security_token="token",
+        base_url="url",
+    )
+    client = SalesforceBaseClient(credentials)
+
+    assert client.query_url == "https://url/services/data/v59.0/query"
+    assert client.tooling_url == "https://url/services/data/v59.0/tooling/query"

castor_extractor/utils/salesforce/constants.py

@@ -0,0 +1,13 @@
+DEFAULT_API_VERSION = 59.0
+DEFAULT_PAGINATION_LIMIT = 100
+
+
+class Keys:
+    """Salesforce's credentials keys"""
+
+    USERNAME = "username"
+    PASSWORD = "password"  # noqa: S105
+    CLIENT_ID = "client_id"
+    CLIENT_SECRET = "client_secret"  # noqa: S105
+    SECURITY_TOKEN = "security_token"  # noqa: S105
+    BASE_URL = "base_url"

castor_extractor/utils/salesforce/credentials.py

@@ -0,0 +1,65 @@
+from typing import Dict
+
+from ...utils import from_env
+from .constants import Keys
+
+_USERNAME = "CASTOR_SALESFORCE_USERNAME"
+_PASSWORD = "CASTOR_SALESFORCE_PASSWORD"  # noqa: S105
+_SECURITY_TOKEN = "CASTOR_SALESFORCE_SECURITY_TOKEN"  # noqa: S105
+_CLIENT_ID = "CASTOR_SALESFORCE_CLIENT_ID"
+_CLIENT_SECRET = "CASTOR_SALESFORCE_CLIENT_SECRET"  # noqa: S105
+_BASE_URL = "CASTOR_SALESFORCE_BASE_URL"
+
+
+class SalesforceCredentials:
+    """
+    Class to handle Salesforce rest API permissions
+    """
+
+    def __init__(
+        self,
+        *,
+        username: str,
+        password: str,
+        security_token: str,
+        client_id: str,
+        client_secret: str,
+        base_url: str,
+    ):
+        self.username = username
+        self.password = password + security_token
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.base_url = base_url
+
+    def token_request_payload(self) -> Dict[str, str]:
+        """
+        Params to post to the API in order to retrieve the authentication token
+        """
+        return {
+            "grant_type": "password",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+            "username": self.username,
+            "password": self.password,
+        }
+
+
+def to_credentials(params: dict) -> SalesforceCredentials:
+    """extract Salesforce credentials"""
+    username = params.get(Keys.USERNAME) or from_env(_USERNAME)
+    password = params.get(Keys.PASSWORD) or from_env(_PASSWORD)
+    security_token = params.get(Keys.SECURITY_TOKEN) or from_env(
+        _SECURITY_TOKEN
+    )
+    client_id = params.get(Keys.CLIENT_ID) or from_env(_CLIENT_ID)
+    client_secret = params.get(Keys.CLIENT_SECRET) or from_env(_CLIENT_SECRET)
+    base_url = params.get(Keys.BASE_URL) or from_env(_BASE_URL)
+    return SalesforceCredentials(
+        username=username,
+        password=password,
+        client_id=client_id,
+        client_secret=client_secret,
+        security_token=security_token,
+        base_url=base_url,
+    )

castor_extractor/{visualization/salesforce_reporting/client → utils/salesforce}/credentials_test.py

@@ -5,9 +5,10 @@ def test_Credentials_token_request_payload():
     creds = SalesforceCredentials(
         username="giphy",
         password="1312",
-        consumer_key="degenie",
-        consumer_secret="fautpasledire",
+        client_id="degenie",
+        client_secret="fautpasledire",
         security_token="yo",
+        base_url="man",
     )
 
     payload = creds.token_request_payload()

castor_extractor/visualization/salesforce_reporting/__init__.py

@@ -1,4 +1,3 @@
 from .assets import SalesforceReportingAsset
-from .client import SalesforceClient
-from .client.credentials import SalesforceCredentials
+from .client import SalesforceReportingClient
 from .extract import extract_all

castor_extractor/visualization/salesforce_reporting/client/__init__.py

@@ -1,2 +1 @@
-from .credentials import SalesforceCredentials
-from .rest import SalesforceClient
+from .rest import SalesforceReportingClient

castor_extractor/visualization/salesforce_reporting/client/rest.py

@@ -1,13 +1,8 @@
 import logging
-import os
-from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
-
-import requests
-from requests import Response
+from typing import Dict, Iterator, List, Optional
 
+from ....utils.salesforce import SalesforceBaseClient
 from ..assets import SalesforceReportingAsset
-from .constants import DEFAULT_API_VERSION, DEFAULT_PAGINATION_LIMIT
-from .credentials import SalesforceCredentials
 from .soql import queries
 
 logger = logging.getLogger(__name__)
@@ -19,89 +14,11 @@ REQUIRING_URL_ASSETS = (
 )
 
 
-class SalesforceClient:
+class SalesforceReportingClient(SalesforceBaseClient):
     """
-    Salesforce API client.
-    https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_rest.htm
+    Salesforce Reporting API client
     """
 
-    api_version = DEFAULT_API_VERSION
-    pagination_limit = DEFAULT_PAGINATION_LIMIT
-
-    def __init__(
-        self,
-        credentials: SalesforceCredentials,
-        instance_url: str,
-    ):
-        self.credentials = credentials
-        self.instance_url = instance_url
-        self._token = self._access_token()
-
-    def _access_token(self) -> Tuple[str, str]:
-        url = f"{self.instance_url}/services/oauth2/token"
-        response = self._call(
-            url, "POST", data=self.credentials.token_request_payload()
-        )
-        return response["access_token"]
-
-    def _header(self) -> Dict:
-        return {"Authorization": f"Bearer {self._token}"}
-
-    @staticmethod
-    def _call(
-        url: str,
-        method: str = "GET",
-        *,
-        header: Optional[Dict] = None,
-        params: Optional[Dict] = None,
-        data: Optional[Dict] = None,
-        processor: Optional[Callable] = None,
-    ) -> Any:
-        logger.debug(f"Calling {method} on {url}")
-        result = requests.request(
-            method,
-            url,
-            headers=header,
-            params=params,
-            data=data,
-        )
-        result.raise_for_status()
-
-        if processor:
-            return processor(result)
-
-        return result.json()
-
-    @staticmethod
-    def _query_processor(response: Response) -> Tuple[dict, Optional[str]]:
-        results = response.json()
-        return results["records"], results.get("nextRecordsUrl")
-
-    def _query_all(self, query: str) -> Iterator[Dict]:
-        """
-        Run a SOQL query over salesforce API.
-
-        more: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/dome_query.htm
-        """
-        url = f"{self.instance_url}/services/data/v{self.api_version}/query"
-        records, next_page = self._call(
-            url,
-            params={"q": query},
-            processor=self._query_processor,
-            header=self._header(),
-        )
-        yield from records
-
-        page_count = 0
-        while next_page and page_count <= self.pagination_limit:
-            logger.info(f"querying page {page_count}")
-            url = f"{self.instance_url}{next_page}"
-            records, next_page = self._call(
-                url, processor=self._query_processor, header=self._header()
-            )
-            yield from records
-            page_count += 1
-
     def _get_asset_url(
         self, asset_type: SalesforceReportingAsset, asset: dict
     ) -> Optional[str]:
@@ -111,15 +28,15 @@ class SalesforceClient:
 
         if asset_type == SalesforceReportingAsset.DASHBOARDS:
             path = f"lightning/r/Dashboard/{asset['Id']}/view"
-            return os.path.join(self.instance_url, path)
+            return self.build_url(self._host, path)
 
         if asset_type == SalesforceReportingAsset.FOLDERS:
             path = asset["attributes"]["url"].lstrip("/")
-            return os.path.join(self.instance_url, path)
+            return self.build_url(self._host, path)
 
         if asset_type == SalesforceReportingAsset.REPORTS:
             path = f"lightning/r/Report/{asset['Id']}/view"
-            return os.path.join(self.instance_url, path)
+            return self.build_url(self._host, path)
 
         return None
 

castor_extractor/visualization/salesforce_reporting/extract.py

@@ -10,14 +10,15 @@ from ...utils import (
     write_json,
     write_summary,
 )
+from ...utils.salesforce import SalesforceCredentials
 from .assets import SalesforceReportingAsset
-from .client import SalesforceClient, SalesforceCredentials
+from .client import SalesforceReportingClient
 
 logger = logging.getLogger(__name__)
 
 
 def iterate_all_data(
-    client: SalesforceClient,
+    client: SalesforceReportingClient,
 ) -> Iterable[Tuple[str, Union[list, dict]]]:
     """Iterate over the extracted data from Salesforce"""
 
@@ -30,10 +31,10 @@ def iterate_all_data(
 def extract_all(
     username: str,
     password: str,
-    consumer_key: str,
-    consumer_secret: str,
+    client_id: str,
+    client_secret: str,
     security_token: str,
-    instance_url: str,
+    base_url: str,
     output_directory: Optional[str] = None,
 ) -> None:
     """
@@ -44,11 +45,12 @@ def extract_all(
     creds = SalesforceCredentials(
         username=username,
         password=password,
-        consumer_key=consumer_key,
-        consumer_secret=consumer_secret,
+        client_id=client_id,
+        client_secret=client_secret,
         security_token=security_token,
+        base_url=base_url,
     )
-    client = SalesforceClient(credentials=creds, instance_url=instance_url)
+    client = SalesforceReportingClient(credentials=creds)
     ts = current_timestamp()
 
     for key, data in iterate_all_data(client):

castor_extractor/warehouse/databricks/client.py

@@ -31,7 +31,7 @@ class DatabricksClient(APIClient):
         db_allowed: Optional[Set[str]] = None,
         db_blocked: Optional[Set[str]] = None,
     ):
-        super().__init__(credentials)
+        super().__init__(host=credentials.host, token=credentials.token)
         self._db_allowed = db_allowed
         self._db_blocked = db_blocked
         self.formatter = DatabricksFormatter()

castor_extractor/warehouse/databricks/credentials.py

@@ -25,7 +25,4 @@ def to_credentials(params: dict) -> DatabricksCredentials:
     """extract Databricks credentials"""
     host = params.get("host") or from_env(_HOST)
     token = params.get("token") or from_env(_TOKEN)
-    return DatabricksCredentials(
-        host=host,
-        token=token,
-    )
+    return DatabricksCredentials(host=host, token=token)

castor_extractor/warehouse/databricks/extract.py

@@ -43,7 +43,7 @@ class DatabricksExtractionProcessor:
         self._storage = storage
         self._skip_existing = skip_existing
 
-    def _should_not_reextract(self, asset_group) -> bool:
+    def _should_not_reextract(self, asset_group: WarehouseAssetGroup) -> bool:
         """helper function to determine whether we need to extract"""
         if not self._skip_existing:
             return False

castor_extractor/warehouse/salesforce/__init__.py

@@ -0,0 +1,6 @@
+from .client import SalesforceClient
+from .extract import (
+    SALESFORCE_ASSETS,
+    SalesforceExtractionProcessor,
+    extract_all,
+)

castor_extractor/warehouse/salesforce/client.py

@@ -0,0 +1,112 @@
+import logging
+from typing import Dict, Iterator, List
+
+from tqdm import tqdm  # type: ignore
+
+from ...utils.salesforce import SalesforceBaseClient, SalesforceCredentials
+from .format import SalesforceFormatter
+from .soql import SOBJECT_FIELDS_QUERY_TPL, SOBJECTS_QUERY_TPL
+
+logger = logging.getLogger(__name__)
+
+
+class SalesforceClient(SalesforceBaseClient):
+    """
+    Salesforce API client to extract sobjects
+    """
+
+    # Implicit (hard-coded in Salesforce) limitation when using SOQL of 2,000 rows
+    LIMIT_RECORDS_PER_PAGE = 2000
+
+    def __init__(self, credentials: SalesforceCredentials):
+        super().__init__(credentials)
+        self.formatter = SalesforceFormatter()
+
+    @staticmethod
+    def name() -> str:
+        return "Salesforce"
+
+    def _format_query(self, query_template: str, start_durable_id: str) -> str:
+        return query_template.format(
+            start_durable_id=start_durable_id,
+            limit=self.LIMIT_RECORDS_PER_PAGE,
+        )
+
+    def _next_records(
+        self, url: str, query_template: str, start_durable_id: str = "0000"
+    ) -> List[dict]:
+        query = self._format_query(
+            query_template, start_durable_id=start_durable_id
+        )
+        records, _ = self._call(
+            url, params={"q": query}, processor=self._query_processor
+        )
+        return records
+
+    def _is_last_page(self, records: List[dict]) -> bool:
+        return len(records) < self.LIMIT_RECORDS_PER_PAGE
+
+    def _should_query_next_page(
+        self, records: List[dict], page_number: int
+    ) -> bool:
+        return not (
+            self._is_last_page(records)
+            or self._has_reached_pagination_limit(page_number)
+        )
+
+    def _query_all(self, query_template: str) -> Iterator[dict]:
+        """
+        Run a SOQL query over salesforce API
+
+        Note, pagination is performed via a LIMIT in the SOQL query and requires
+        that ids are sorted. The SOQL query must support `limit` and
+        `start_durable_id` as parameters.
+        """
+        url = self.query_url
+        logger.info("querying page 0")
+        records = self._next_records(url, query_template)
+        yield from records
+
+        page_count = 1
+        while self._should_query_next_page(records, page_count):
+            logger.info(f"querying page {page_count}")
+            last_durable_id = records[-1]["DurableId"]
+            records = self._next_records(
+                url, query_template, start_durable_id=last_durable_id
+            )
+            yield from records
+            page_count += 1
+
+    def fetch_sobjects(self) -> List[dict]:
+        """Fetch all sobjects"""
+        logger.info("Extracting sobjects")
+        return list(self._query_all(SOBJECTS_QUERY_TPL))
+
+    def fetch_fields(self, sobject_name: str) -> List[dict]:
+        """Fetches fields of a given sobject"""
+        query = SOBJECT_FIELDS_QUERY_TPL.format(
+            entity_definition_id=sobject_name
+        )
+        response = self._call(self.tooling_url, params={"q": query})
+        return response["records"]
+
+    def tables(self) -> List[dict]:
+        """
+        Get Salesforce sobjects as tables
+        """
+        sobjects = self.fetch_sobjects()
+        logger.info(f"Extracted {len(sobjects)} sobjects")
+        return self.formatter.tables(sobjects)
+
+    def columns(
+        self, sobject_names: List[str], show_progress: bool = True
+    ) -> List[dict]:
+        """
+        Get salesforce sobject fields as columns
+        show_progress: optionally deactivate the tqdm progress bar
+        """
+        sobject_fields: Dict[str, List[dict]] = dict()
+        for sobject_name in tqdm(sobject_names, disable=not show_progress):
+            fields = self.fetch_fields(sobject_name)
+            sobject_fields[sobject_name] = fields
+        return self.formatter.columns(sobject_fields)

castor_extractor/warehouse/salesforce/constants.py

@@ -0,0 +1,2 @@
+DATABASE_NAME = "salesforce"
+SCHEMA_NAME = "schema"

castor_extractor/warehouse/salesforce/extract.py

@@ -0,0 +1,111 @@
+import logging
+from typing import Dict, List, Tuple
+
+from ...utils import AbstractStorage, LocalStorage, write_summary
+from ...utils.salesforce import to_credentials
+from ..abstract import (
+    SupportedAssets,
+    WarehouseAsset,
+    WarehouseAssetGroup,
+    common_args,
+)
+from .client import SalesforceClient
+
+logger = logging.getLogger(__name__)
+
+
+Paths = Dict[str, str]
+
+SALESFORCE_CATALOG_ASSETS: Tuple[WarehouseAsset, ...] = (
+    WarehouseAsset.TABLE,
+    WarehouseAsset.COLUMN,
+)
+
+SALESFORCE_ASSETS: SupportedAssets = {
+    WarehouseAssetGroup.CATALOG: SALESFORCE_CATALOG_ASSETS
+}
+
+
+class SalesforceExtractionProcessor:
+    """Salesforce API-based extraction management - warehouse part"""
+
+    def __init__(
+        self,
+        client: SalesforceClient,
+        storage: AbstractStorage,
+        skip_existing: bool = False,
+    ):
+        self._client = client
+        self._storage = storage
+        self._skip_existing = skip_existing
+
+    def _should_extract(self) -> bool:
+        """helper function to determine whether we need to extract"""
+        if not self._skip_existing:
+            return True
+
+        for asset in SALESFORCE_CATALOG_ASSETS:
+            if not self._storage.exists(asset.value):
+                return True
+
+        logger.info("Skipped, files for catalog already exist")
+        return False
+
+    def _existing_group_paths(self) -> Paths:
+        return {
+            a.value: self._storage.path(a.value)
+            for a in SALESFORCE_CATALOG_ASSETS
+        }
+
+    def extract_catalog(self, show_progress: bool = True) -> Paths:
+        """
+        Extract the following catalog assets: tables and columns
+        and return the locations of the extracted data
+        """
+        if not self._should_extract():
+            return self._existing_group_paths()
+
+        catalog_locations: Paths = dict()
+
+        tables = self._client.tables()
+        location = self._storage.put(WarehouseAsset.TABLE.value, tables)
+        catalog_locations[WarehouseAsset.TABLE.value] = location
+        logger.info(f"Extracted {len(tables)} tables to {location}")
+
+        table_names = [t["table_name"] for t in tables]
+        columns = self._client.columns(table_names, show_progress)
+        location = self._storage.put(WarehouseAsset.COLUMN.value, columns)
+        catalog_locations[WarehouseAsset.COLUMN.value] = location
+        logger.info(f"Extracted {len(columns)} columns to {location}")
+        return catalog_locations
+
+    def extract_role(self) -> Paths:
+        """extract no users and return the empty file location"""
+        users: List[dict] = []
+        location = self._storage.put(WarehouseAsset.USER.value, users)
+        logger.info(f"Extracted {len(users)} users to {location}")
+        return {WarehouseAsset.USER.value: location}
+
+
+def extract_all(**kwargs) -> None:
+    """
+    Extract all assets from Salesforce and store the results in CSV files
+    """
+    output_directory, skip_existing = common_args(kwargs)
+
+    client = SalesforceClient(credentials=to_credentials(kwargs))
+    storage = LocalStorage(directory=output_directory)
+    extractor = SalesforceExtractionProcessor(
+        client=client,
+        storage=storage,
+        skip_existing=skip_existing,
+    )
+
+    extractor.extract_catalog()
+    extractor.extract_role()
+
+    write_summary(
+        output_directory,
+        storage.stored_at_ts,
+        client_name=client.name(),
+    )

castor_extractor/warehouse/salesforce/format.py

@@ -0,0 +1,67 @@
+from typing import Any, Dict, List
+
+from .constants import SCHEMA_NAME
+
+
+def _clean(raw: str) -> str:
+    return raw.strip('"')
+
+
+def _field_description(field: Dict[str, Any]) -> str:
+    context: Dict[str, str] = {}
+
+    field_definition: Dict[str, str] = field.get("FieldDefinition") or {}
+    if description := field_definition.get("Description"):
+        context["Description"] = _clean(description)
+    if help_text := field.get("InlineHelpText"):
+        context["Help Text"] = _clean(help_text)
+    if compliance_group := field_definition.get("ComplianceGroup"):
+        context["Compliance Categorization"] = _clean(compliance_group)
+    if security_level := field_definition.get("SecurityClassification"):
+        context["Data Sensitivity Level"] = _clean(security_level)
+
+    return "\n".join([f"- {k}: {v}" for k, v in context.items()])
+
+
+def _to_column_payload(field: dict, position: int, table_name: str) -> dict:
+    field_name = field["QualifiedApiName"]
+    return {
+        "id": f"{table_name}.{field_name}",
+        "table_id": table_name,
+        "column_name": field_name,
+        "description": _field_description(field),
+        "data_type": field.get("DataType"),
+        "ordinal_position": position,
+    }
+
+
+def _to_table_payload(table: dict) -> dict:
+    return {
+        "id": table["QualifiedApiName"],
+        "schema_id": SCHEMA_NAME,
+        "table_name": table["QualifiedApiName"],
+        "description": "",
+        "tags": [],
+        "type": "TABLE",
+    }
+
+
+class SalesforceFormatter:
+    """
+    Helper functions that format the response in the format to be exported as
+    csv.
+    """
+
+    @staticmethod
+    def tables(sobjects: List[dict]) -> List[dict]:
+        """formats the raw list of sobjects to tables"""
+        return [_to_table_payload(s) for s in sobjects]
+
+    @staticmethod
+    def columns(sobject_fields: Dict[str, List[dict]]) -> List[dict]:
+        """formats the raw list of sobject fields to columns"""
+        return [
+            _to_column_payload(field, idx, table_name)
+            for table_name, fields in sobject_fields.items()
+            for idx, field in enumerate(fields)
+        ]

castor_extractor/warehouse/salesforce/format_test.py

@@ -0,0 +1,32 @@
+from .format import _field_description
+
+
+def test__field_description():
+
+    field = {}
+    assert _field_description(field) == ""
+
+    definition = {}
+    field = {"FieldDefinition": definition}
+    assert _field_description(field) == ""
+
+    definition.update({"Description": "foo"})
+    assert "foo" in _field_description(field)
+
+    field.update({"InlineHelpText": "bar"})
+    assert "bar" in _field_description(field)
+
+    definition.update({"ComplianceGroup": "bim"})
+    assert "bim" in _field_description(field)
+
+    definition.update({"SecurityClassification": "bam"})
+    description = _field_description(field)
+
+    assert "bam" in description
+    expected = (
+        "- Description: foo\n"
+        "- Help Text: bar\n"
+        "- Compliance Categorization: bim\n"
+        "- Data Sensitivity Level: bam"
+    )
+    assert description == expected

castor_extractor/warehouse/salesforce/soql.py

@@ -0,0 +1,45 @@
+SOBJECTS_QUERY_TPL = """
+    SELECT
+        DeploymentStatus,
+        DeveloperName,
+        DurableId,
+        ExternalSharingModel,
+        InternalSharingModel,
+        Label,
+        PluralLabel,
+        QualifiedApiName
+    FROM EntityDefinition
+    WHERE DurableId > '{start_durable_id}'
+    ORDER BY DurableId
+    LIMIT {limit}
+"""
+
+
+SOBJECT_FIELDS_QUERY_TPL = """
+    SELECT
+        DataType,
+        DeveloperName,
+        Digits,
+        FieldDefinition.BusinessOwnerId,
+        FieldDefinition.ComplianceGroup,
+        FieldDefinition.DataType,
+        FieldDefinition.Description,
+        FieldDefinition.IsIndexed,
+        FieldDefinition.LastModifiedBy.Username,
+        FieldDefinition.LastModifiedDate,
+        FieldDefinition.SecurityClassification,
+        InlineHelpText,
+        IsComponent,
+        IsCompound,
+        IsNillable,
+        IsUnique,
+        Label,
+        Length,
+        Precision,
+        QualifiedApiName,
+        ReferenceTo,
+        RelationshipName,
+        Scale
+    FROM EntityParticle
+    WHERE EntityDefinitionId='{entity_definition_id}'
+"""

{castor_extractor-0.16.3.dist-info → castor_extractor-0.16.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.16.3
+Version: 0.16.4
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.16.3.dist-info → castor_extractor-0.16.4.dist-info}/RECORD

@@ -1,11 +1,11 @@
-CHANGELOG.md,sha256=eRvmcZqJY1G4yDR2CzrA5wzf6xpeZM80HzVBw1tUynw,9959
+CHANGELOG.md,sha256=6ApRuFb6ZxwvpMFyBRJAU6L7teZ01KK2tD0pXFvsYdw,10026
 Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=uF6PXm9ocPITlKVSh9afTakHmpLx3TvawLf-CbMP3wM,3578
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/commands/extract_bigquery.py,sha256=dU4OiYO1V0n32orvZnMh1_xtFKF_VxHNXcVsH3otY-g,1269
-castor_extractor/commands/extract_databricks.py,sha256=gun8VK7bKMv5T198Z0wDihY83J2Y7xzKFEoxtXJ5O6s,1170
+castor_extractor/commands/extract_databricks.py,sha256=SVKyoa-BBUQAM6HRHf1Wdg9-tpICic2yyvXQwHcNBhA,1264
 castor_extractor/commands/extract_domo.py,sha256=lwJ7XeYOeLMF2plf5PK3cL56N9n2yjcDsyRM6UFwKTM,1208
 castor_extractor/commands/extract_looker.py,sha256=gwjIQPOHrXevgU_o2l8vDHHQT8Sb-mGdwcceb6wJJbg,1483
 castor_extractor/commands/extract_metabase_api.py,sha256=VPyEKO2VFXzk_OsbQnDhObE9siuBfoegechCZYPZi2k,778
@@ -16,7 +16,8 @@ castor_extractor/commands/extract_postgres.py,sha256=pX0RnCPi4nw6QQ6wiAuZ_Xt3ZbD
 castor_extractor/commands/extract_powerbi.py,sha256=e6MXDNOafdp0w4ZtOnE5z5o_CxvaodUbbQFk__pDiM4,875
 castor_extractor/commands/extract_qlik.py,sha256=mSeyGOprTyBExes-lzp___7tgBS1KeyTVpwKAqMpkiw,989
 castor_extractor/commands/extract_redshift.py,sha256=bdLp7d7ImZoKCkWc3f3NXF1imIzMVT43_KPI-x4UVac,1155
-castor_extractor/commands/extract_salesforce_reporting.py,sha256=Nzb0cn9HlPxvZDQNTFk7r3rBpVpGO9a2WhwbPpKGa2M,1160
+castor_extractor/commands/extract_salesforce.py,sha256=s2o799ePjQFYsVwZbrGEflzOIwJKtxUMb9pcF4-K90Y,1400
+castor_extractor/commands/extract_salesforce_reporting.py,sha256=rmAo--dl_m2x7TtJ29w1PfsKt9tQDZocTdvwTqj-fnI,1146
 castor_extractor/commands/extract_sigma.py,sha256=agwfKj55C81-kect3K6xSJVBv3TDuPT6fGWSTytkQ2o,703
 castor_extractor/commands/extract_snowflake.py,sha256=vYiruxRoo--GeMemOGsSE1w9kcKTh_y4E165HtMVzkM,1982
 castor_extractor/commands/extract_sqlserver.py,sha256=lwhbcNChaXHZgMgSOch3faVr7WJw-sDU6GHl3lzBt_0,1141
@@ -46,7 +47,7 @@ castor_extractor/uploader/utils.py,sha256=NCe0tkB28BVhqzOaDhDjaSfODjjcPWB17X6chn
 castor_extractor/utils/__init__.py,sha256=cZbvEJ4G2IcJR2BzHwi3oOwDLqJsBx0J9gD71lWE1BQ,1149
 castor_extractor/utils/client/__init__.py,sha256=CRE-xJKm6fVV9dB8ljzB5YoOxX4I1sCD1KSgqs3Y8_Y,161
 castor_extractor/utils/client/abstract.py,sha256=aA5Qcb9TwWDSMq8WpXbGkOB20hehwX2VTpqQAwV76wk,2048
-castor_extractor/utils/client/api.py,sha256=rRTp7brx-Wv-ky604kITdmLXcNQNZYpyedjaDicXM7Y,1324
+castor_extractor/utils/client/api.py,sha256=tHa7eC11sS_eOCXhlnvUa2haRfOLENmjKgjB09Ijt0s,1664
 castor_extractor/utils/client/api_test.py,sha256=NSMdXg1FLc37erqHp2FZsIsogWVv6lFSs7rDXHikr-E,542
 castor_extractor/utils/client/postgres.py,sha256=n6ulaT222WWPY0_6qAZ0MHF0m91HtI9mMqL71nyygo0,866
 castor_extractor/utils/client/query.py,sha256=O6D5EjD1KmBlwa786Uw4D4kzxx97_HH50xIIeSWt0B8,205
@@ -82,6 +83,12 @@ castor_extractor/utils/retry.py,sha256=vYdJMiM-Nr82H1MuD7_KZdqbFz98ffQGqJ4Owbr6m
 castor_extractor/utils/retry_test.py,sha256=nsMttlmyKygVcffX3Hay8U2S1BspkGPiCmzIXPpLKyk,2230
 castor_extractor/utils/safe.py,sha256=jpfIimwdBSVUvU2DPFrhqpKC_DSYwxQqd08MlIkSODY,1967
 castor_extractor/utils/safe_test.py,sha256=IHN1Z761tYMFslYC-2HAfkXmFPh4LYSqNLs4QZwykjk,2160
+castor_extractor/utils/salesforce/__init__.py,sha256=VGD4vd1Se79z2PAaVCvCSL3yhgWlhQFaVDLZ5aERug0,132
+castor_extractor/utils/salesforce/client.py,sha256=Mt9yykAPROFgme5eDqoZQv4u85hxcUoG-tmKFPwLibo,2856
+castor_extractor/utils/salesforce/client_test.py,sha256=s6UTogjC36jrJOnYA-gFuyTQsvROCt9y_eoD2O41xCg,682
+castor_extractor/utils/salesforce/constants.py,sha256=5sph6dbTCp0mAGWP24WTpC1wsIqeG8yI8-BsKrmV_wA,335
+castor_extractor/utils/salesforce/credentials.py,sha256=Wwb-_BlbFBJUl3dhXz72IIqcCfj1F3Zj3JoYr3FYk0A,2045
+castor_extractor/utils/salesforce/credentials_test.py,sha256=FQRyNk2Jsh6KtYiW20oL43CVnGjXLcAjdFATkE7jK0s,586
 castor_extractor/utils/store.py,sha256=D_pVaPsu1MKAJC0K47O_vYTs-Afl6oejravAJdvjmGc,2040
 castor_extractor/utils/string.py,sha256=aW6bbjqEGnh9kT5KZBnMlV6fhdgOJ0ENCkCTDon1xA0,2377
 castor_extractor/utils/string_test.py,sha256=OmRVCJUXMcCTwY-QJDhUViYpxkvQQgNRJLCaXY0iUnk,2535
@@ -182,15 +189,12 @@ castor_extractor/visualization/qlik/client/rest.py,sha256=EkHEs3_Vrmy0Ex5b9M_klm
 castor_extractor/visualization/qlik/client/rest_test.py,sha256=Z2gBTokUVv-JapBtrY2nZDJzBtOusRq6_lJutVvzqG8,1684
 castor_extractor/visualization/qlik/constants.py,sha256=Pbd1SH3_VI_yEhoDx4PIXBUup-MqXUFjxDkDRr2V4J8,95
 castor_extractor/visualization/qlik/extract.py,sha256=1ulrirDzoKEdsWxztR6MHcUAE8CiEMx75esdUE7PAmY,2397
-castor_extractor/visualization/salesforce_reporting/__init__.py,sha256=8Y_gnyPj-oYaz9Wz1fTf6jmDhofPnNPHlGoKd47KaeM,169
+castor_extractor/visualization/salesforce_reporting/__init__.py,sha256=MvArD0GKNIpCDvLIYcpKrjMjFLhMyDETK6i3k0Fb6Tk,124
 castor_extractor/visualization/salesforce_reporting/assets.py,sha256=2J-iAmJGGDufOcJUgE47M3-dEcjYXcVyVUNcmHrj79w,271
-castor_extractor/visualization/salesforce_reporting/client/__init__.py,sha256=RNQTeNLxVVR6MScvHfRekhOPsQjHKH1y4WogemVu4tc,82
-castor_extractor/visualization/salesforce_reporting/client/constants.py,sha256=7yPmUeyn4IHQiHLDutXE0L_OBd41E5080vFxqA_s4Dc,58
-castor_extractor/visualization/salesforce_reporting/client/credentials.py,sha256=gJapeUKs8gZSY_YdzX-j0Iv4vcaBzCTAlXMecO8Kk5k,875
-castor_extractor/visualization/salesforce_reporting/client/credentials_test.py,sha256=2qIYZ8QuuarHz5EZ9bU0sGEOvoDLnN0eiwsvnbtgXXY,567
-castor_extractor/visualization/salesforce_reporting/client/rest.py,sha256=_3wa5-bxKGwaNEwh-KLke3O6lbFWzOUAe1bL8n9hH04,4429
+castor_extractor/visualization/salesforce_reporting/client/__init__.py,sha256=DIA6f_vNJZqT89qVYxg98Le7QeDn2y0Qew03V3J9t9o,44
+castor_extractor/visualization/salesforce_reporting/client/rest.py,sha256=hzaXWLcYt0aAHXK46DbsLmzocjRY1llwrNj8_3TObKs,1849
 castor_extractor/visualization/salesforce_reporting/client/soql.py,sha256=DHTi058UEaZKZnxJVmsCouPfA9Lgr3gFY6vY7NiqyMM,1584
-castor_extractor/visualization/salesforce_reporting/extract.py,sha256=6cUMNrCz46DUS7xyF6eqKRKGP5VB0y_qwqhvVYfKSPE,1639
+castor_extractor/visualization/salesforce_reporting/extract.py,sha256=5QwZwP27uXrFJSf00El7Ku592-9fhmCtTdiUGpNkHZM,1678
 castor_extractor/visualization/sigma/__init__.py,sha256=m98AEysUsVHQAWT6m5nvrtLMs22SDQH9G78-IcUwBoY,130
 castor_extractor/visualization/sigma/assets.py,sha256=JZ1Cpxnml8P3mIJoTUM57hvylB18ErECQXaP5FF63O4,268
 castor_extractor/visualization/sigma/client/__init__.py,sha256=sFqsbcwilIxu75njtSbnAIsNlPdRgB39SAInNUf-nbQ,90
@@ -262,10 +266,10 @@ castor_extractor/warehouse/bigquery/queries/view_ddl.sql,sha256=obCm-IN9V8_YSZTw
 castor_extractor/warehouse/bigquery/query.py,sha256=hrFfjd5jW2oQnZ6ozlkn-gDe6sCIzu5zSX19T9W6fIk,4162
 castor_extractor/warehouse/bigquery/types.py,sha256=LZVWSmE57lOemNbB5hBRyYmDk9bFAU4nbRaJWALl6N8,140
 castor_extractor/warehouse/databricks/__init__.py,sha256=bTvDxjGQGM2J3hOnVhfNmFP1y8DK0tySiD_EXe5_xWE,200
-castor_extractor/warehouse/databricks/client.py,sha256=u1KpiG16IlFbaEVAIzBlxnzTk_bARGh-D0sZBXtgF4c,8043
+castor_extractor/warehouse/databricks/client.py,sha256=FIqHjlGN5EN2dvcZD2941zPAomOye91JmkgPlxGDk0g,8078
 castor_extractor/warehouse/databricks/client_test.py,sha256=ctOQnUXosuuFjWGJKgkxjUcV4vQUBWt2BQ_f0Tyzqe4,2717
-castor_extractor/warehouse/databricks/credentials.py,sha256=sMpOAKhBklcmTpcr3mi3o8qLud__8PTZbQUT3K_TRY8,678
-castor_extractor/warehouse/databricks/extract.py,sha256=mgl1_b9Mlir9ZU3R5HV689YlhzzhlyVN8IaBHaNwY54,5752
+castor_extractor/warehouse/databricks/credentials.py,sha256=PpGv5_GP320UQjV_gvaxSpOw58AmqSznmjGhGfe6bdU,655
+castor_extractor/warehouse/databricks/extract.py,sha256=-vJhAIxSu1lD_xGl-GXZYTmc5BGu0aXM3l-U0UghREM,5773
 castor_extractor/warehouse/databricks/format.py,sha256=LiPGCTPzL3gQQMMl1v6DvpcTk7BWxZFq03jnHdoYnuU,4968
 castor_extractor/warehouse/databricks/format_test.py,sha256=iPmdJof43fBYL1Sa_fBrCWDQHCHgm7IWCZag1kWkj9E,1970
 castor_extractor/warehouse/databricks/types.py,sha256=T2SyLy9pY_olLtstdC77moPxIiikVsuQLMxh92YMJQo,78
@@ -307,6 +311,13 @@ castor_extractor/warehouse/redshift/queries/table_freshness.sql,sha256=l61_ysmTE
 castor_extractor/warehouse/redshift/queries/user.sql,sha256=sEXveJAuNvZacvpI6WfwsX6VavoMb2VqYA32f6Dt-_Y,170
 castor_extractor/warehouse/redshift/queries/view_ddl.sql,sha256=Pkyh_QT6d4rhTeyiVcqw6O8CRl7NEhk2p7eM5YIn5kg,719
 castor_extractor/warehouse/redshift/query.py,sha256=0C81rkt2cpkWrJIxxwALDyqr-49vlqQM04y_N6wwStc,540
+castor_extractor/warehouse/salesforce/__init__.py,sha256=NR4aNea5jeE1xYqeZ_29deeN84CkN0_D_Z7CLQdJvFY,137
+castor_extractor/warehouse/salesforce/client.py,sha256=_XiQJJJfELKGmzuBv8Mr_C0FJ-oLg71KbvpehrGvJ_k,3842
+castor_extractor/warehouse/salesforce/constants.py,sha256=GusduVBCPvwpk_Im6F3bDvXeNQ7hRnCMdIAjIg65RnE,52
+castor_extractor/warehouse/salesforce/extract.py,sha256=ZTb58t7mqhavNvErrnw8M0L4Uu3qJpQEIldymurbgl0,3417
+castor_extractor/warehouse/salesforce/format.py,sha256=_BSj_G6C-kPwRubxSx1WuHg-_nYVQVNgAANqNfXL5RM,2154
+castor_extractor/warehouse/salesforce/format_test.py,sha256=6hy0USZH7-PDQt3oZ9_3Nwlr3eHLkqNEchqIM3bIDrU,858
+castor_extractor/warehouse/salesforce/soql.py,sha256=81lAtPpq7ccmi6o1zkwqLKC1esOfSsfNObdizkfgiSM,1089
 castor_extractor/warehouse/snowflake/__init__.py,sha256=TEGXTyxWp4Tr9gIHb-UFVTRKj6YWmrRtqHruiKSZGiY,174
 castor_extractor/warehouse/snowflake/client.py,sha256=XT0QLVNff_586SDuMe40iu8FCwPDh2uBV5aKc1Ql914,5555
 castor_extractor/warehouse/snowflake/client_test.py,sha256=ihWtOOAQfh8pu5JTr_EWfqefKOVIaJXznACURzaU1Qs,1432
@@ -346,8 +357,8 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.16.3.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.16.3.dist-info/METADATA,sha256=CaGi5itpnLSjjCv5PpKayJ2Oi859ewvcyrPFzHNIdYM,6370
-castor_extractor-0.16.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.16.3.dist-info/entry_points.txt,sha256=EQUCoNjSHevxmY5ZathX_fLZPcuBHng23rj0SSUrLtI,1345
-castor_extractor-0.16.3.dist-info/RECORD,,
+castor_extractor-0.16.4.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.16.4.dist-info/METADATA,sha256=-D39Tmu_LFDHRe3HrZ542JjZxl0puzZr0n8wMkW52P0,6370
+castor_extractor-0.16.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+castor_extractor-0.16.4.dist-info/entry_points.txt,sha256=SbyPk58Gh-FRztfCNnUZQ6w7SatzNJFZ6GIJLNsy7tI,1427
+castor_extractor-0.16.4.dist-info/RECORD,,

{castor_extractor-0.16.3.dist-info → castor_extractor-0.16.4.dist-info}/entry_points.txt

@@ -11,7 +11,8 @@ castor-extract-postgres=castor_extractor.commands.extract_postgres:main
 castor-extract-powerbi=castor_extractor.commands.extract_powerbi:main
 castor-extract-qlik=castor_extractor.commands.extract_qlik:main
 castor-extract-redshift=castor_extractor.commands.extract_redshift:main
-castor-extract-salesforce-viz=castor_extractor.commands.extract_salesforce_viz:main
+castor-extract-salesforce=castor_extractor.commands.extract_salesforce:main
+castor-extract-salesforce-viz=castor_extractor.commands.extract_salesforce_reporting:main
 castor-extract-sigma=castor_extractor.commands.extract_sigma:main
 castor-extract-snowflake=castor_extractor.commands.extract_snowflake:main
 castor-extract-sqlserver=castor_extractor.commands.extract_sqlserver:main

castor_extractor/visualization/salesforce_reporting/client/constants.py

@@ -1,2 +0,0 @@
-DEFAULT_API_VERSION = 59.0
-DEFAULT_PAGINATION_LIMIT = 100

castor_extractor/visualization/salesforce_reporting/client/credentials.py

@@ -1,33 +0,0 @@
-from typing import Dict
-
-
-class SalesforceCredentials:
-    """
-    Class to handle Salesforce rest API permissions
-    """
-
-    def __init__(
-        self,
-        *,
-        username: str,
-        password: str,
-        security_token: str,
-        consumer_key: str,
-        consumer_secret: str,
-    ):
-        self.username = username
-        self.password = password + security_token
-        self.consumer_key = consumer_key
-        self.consumer_secret = consumer_secret
-
-    def token_request_payload(self) -> Dict[str, str]:
-        """
-        Params to post to the API in order to retrieve the authentication token
-        """
-        return {
-            "grant_type": "password",
-            "client_id": self.consumer_key,
-            "client_secret": self.consumer_secret,
-            "username": self.username,
-            "password": self.password,
-        }