castor-extractor 0.16.9__tar.gz → 0.16.11__tar.gz

{castor_extractor-0.16.9 → castor_extractor-0.16.11}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.16.11 - 2024-06-03
+
+* Tableau: add extra fields to optimise storage
+
+## 0.16.10 - 2024-05-30
+
+* Salesforce: extract sobjects Label as table name
+
 ## 0.16.9 - 2024-05-28
 
 * Tableau: extract only fields that are necessary

{castor_extractor-0.16.9 → castor_extractor-0.16.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.16.9
+Version: 0.16.11
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.16.9 → castor_extractor-0.16.11}/castor_extractor/visualization/tableau_revamp/client/client.py

@@ -28,10 +28,13 @@ _TSC_ASSETS = (
     TableauRevampAsset.USAGE,
 )
 
-# speed up extraction: fields and columns are smaller but volumes are bigger
+# increase the value when extraction is too slow
+# decrease the value when timeouts arise
 _CUSTOM_PAGE_SIZE: Dict[TableauRevampAsset, int] = {
+    # fields and columns are light but volumes are bigger
+    TableauRevampAsset.COLUMN: 200,
     TableauRevampAsset.FIELD: 1000,
-    TableauRevampAsset.COLUMN: 1000,
+    TableauRevampAsset.TABLE: 50,
 }
 
 

{castor_extractor-0.16.9 → castor_extractor-0.16.11}/castor_extractor/visualization/tableau_revamp/client/gql_queries.py

@@ -18,7 +18,11 @@ QUERY_TEMPLATE = """
 
 _COLUMNS_QUERY = """
 downstreamDashboards { id }
-downstreamFields { id }
+downstreamFields {
+    id
+    __typename
+    datasource { id }
+}
 downstreamWorkbooks { id }
 id
 name

{castor_extractor-0.16.9 → castor_extractor-0.16.11}/castor_extractor/warehouse/salesforce/client.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, Iterator, List
+from typing import Dict, Iterator, List, Tuple
 
 from tqdm import tqdm  # type: ignore
 
@@ -96,17 +96,19 @@ class SalesforceClient(SalesforceBaseClient):
         """
         sobjects = self.fetch_sobjects()
         logger.info(f"Extracted {len(sobjects)} sobjects")
-        return self.formatter.tables(sobjects)
+        return list(self.formatter.tables(sobjects))
 
     def columns(
-        self, sobject_names: List[str], show_progress: bool = True
+        self, sobject_names: List[Tuple[str, str]], show_progress: bool = True
     ) -> List[dict]:
         """
         Get salesforce sobject fields as columns
         show_progress: optionally deactivate the tqdm progress bar
         """
         sobject_fields: Dict[str, List[dict]] = dict()
-        for sobject_name in tqdm(sobject_names, disable=not show_progress):
-            fields = self.fetch_fields(sobject_name)
-            sobject_fields[sobject_name] = fields
+        for api_name, table_name in tqdm(
+            sobject_names, disable=not show_progress
+        ):
+            fields = self.fetch_fields(api_name)
+            sobject_fields[table_name] = fields
         return self.formatter.columns(sobject_fields)

{castor_extractor-0.16.9 → castor_extractor-0.16.11}/castor_extractor/warehouse/salesforce/extract.py

@@ -72,8 +72,8 @@ class SalesforceExtractionProcessor:
         catalog_locations[WarehouseAsset.TABLE.value] = location
         logger.info(f"Extracted {len(tables)} tables to {location}")
 
-        table_names = [t["table_name"] for t in tables]
-        columns = self._client.columns(table_names, show_progress)
+        sobject_names = [(t["api_name"], t["table_name"]) for t in tables]
+        columns = self._client.columns(sobject_names, show_progress)
         location = self._storage.put(WarehouseAsset.COLUMN.value, columns)
         catalog_locations[WarehouseAsset.COLUMN.value] = location
         logger.info(f"Extracted {len(columns)} columns to {location}")

{castor_extractor-0.16.9 → castor_extractor-0.16.11}/castor_extractor/warehouse/salesforce/format.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, Iterator, List
 
 from .constants import SCHEMA_NAME
 
@@ -35,17 +35,35 @@ def _to_column_payload(field: dict, position: int, table_name: str) -> dict:
     }
 
 
-def _to_table_payload(table: dict) -> dict:
+def _to_table_payload(sobject: dict, table_name: str) -> dict:
     return {
-        "id": table["QualifiedApiName"],
+        "id": table_name,
+        "api_name": sobject["QualifiedApiName"],
+        "label": sobject["Label"],
         "schema_id": SCHEMA_NAME,
-        "table_name": table["QualifiedApiName"],
+        "table_name": table_name,
         "description": "",
         "tags": [],
         "type": "TABLE",
     }
 
 
+def _merge_label_and_api_name(sobject: dict) -> dict:
+    label = sobject["Label"]
+    api_name = sobject["QualifiedApiName"]
+    table_name = f"{label} ({api_name})"
+    return _to_table_payload(sobject, table_name)
+
+
+def _by_label(sobjects: List[dict]) -> Dict[str, List[dict]]:
+    by_label: Dict[str, List[dict]] = dict()
+    for sobject in sobjects:
+        label = sobject["Label"]
+        similar_sobjects = by_label.setdefault(label, [])
+        similar_sobjects.append(sobject)
+    return by_label
+
+
 class SalesforceFormatter:
     """
     Helper functions that format the response in the format to be exported as
@@ -53,9 +71,18 @@ class SalesforceFormatter:
     """
 
     @staticmethod
-    def tables(sobjects: List[dict]) -> List[dict]:
-        """formats the raw list of sobjects to tables"""
-        return [_to_table_payload(s) for s in sobjects]
+    def tables(sobjects: List[dict]) -> Iterator[dict]:
+        """
+        formats the raw list of sobjects to tables
+        if two tables share the same label, then we add the api name as well
+        """
+        by_label = _by_label(sobjects)
+        for label, similars in by_label.items():
+            if len(similars) > 1:
+                yield from [_merge_label_and_api_name(s) for s in similars]
+            else:
+                sobject = similars[0]  # unique sobject on label
+                yield _to_table_payload(sobject, label)
 
     @staticmethod
     def columns(sobject_fields: Dict[str, List[dict]]) -> List[dict]:

castor_extractor-0.16.11/castor_extractor/warehouse/salesforce/format_test.py

@@ -0,0 +1,80 @@
+from typing import Dict, Tuple
+
+from .format import (
+    SCHEMA_NAME,
+    SalesforceFormatter,
+    _by_label,
+    _field_description,
+    _merge_label_and_api_name,
+)
+
+
+def _example_sobjects() -> Tuple[Dict[str, str], ...]:
+    """Returns 4 sobjects with 2 sharing the same label"""
+    a = {"Label": "a", "QualifiedApiName": "a_one"}
+    b = {"Label": "b", "QualifiedApiName": "b"}
+    c = {"Label": "c", "QualifiedApiName": "c"}
+    a_prime = {"Label": "a", "QualifiedApiName": "a_two"}
+    return a, b, c, a_prime
+
+
+def test__field_description():
+
+    field = {}
+    assert _field_description(field) == ""
+
+    definition = {}
+    field = {"FieldDefinition": definition}
+    assert _field_description(field) == ""
+
+    definition.update({"Description": "foo"})
+    assert "foo" in _field_description(field)
+
+    field.update({"InlineHelpText": "bar"})
+    assert "bar" in _field_description(field)
+
+    definition.update({"ComplianceGroup": "bim"})
+    assert "bim" in _field_description(field)
+
+    definition.update({"SecurityClassification": "bam"})
+    description = _field_description(field)
+
+    assert "bam" in description
+    expected = (
+        "- Description: foo\n"
+        "- Help Text: bar\n"
+        "- Compliance Categorization: bim\n"
+        "- Data Sensitivity Level: bam"
+    )
+    assert description == expected
+
+
+def test__merge_label_and_api_name():
+    sobject = {"Label": "foo", "QualifiedApiName": "bar"}
+    payload = _merge_label_and_api_name(sobject)
+    expected_name = "foo (bar)"
+    assert payload == {
+        "id": expected_name,
+        "api_name": "bar",
+        "label": "foo",
+        "schema_id": SCHEMA_NAME,
+        "table_name": expected_name,
+        "description": "",
+        "tags": [],
+        "type": "TABLE",
+    }
+
+
+def test__by_label():
+    a, b, c, a_prime = _example_sobjects()
+    sobjects = [a, b, c, a_prime]
+    by_label = _by_label(sobjects)
+    assert by_label == {"a": [a, a_prime], "b": [b], "c": [c]}
+
+
+def test_salesforce_formatter_tables():
+    sobjects = [*_example_sobjects()]
+    tables = SalesforceFormatter.tables(sobjects)
+    expected_names = {"a (a_one)", "a (a_two)", "b", "c"}
+    payload_names = {t["table_name"] for t in tables}
+    assert payload_names == expected_names

{castor_extractor-0.16.9 → castor_extractor-0.16.11}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.16.9"
+version = "0.16.11"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.16.9/castor_extractor/warehouse/salesforce/format_test.py

@@ -1,32 +0,0 @@
-from .format import _field_description
-
-
-def test__field_description():
-
-    field = {}
-    assert _field_description(field) == ""
-
-    definition = {}
-    field = {"FieldDefinition": definition}
-    assert _field_description(field) == ""
-
-    definition.update({"Description": "foo"})
-    assert "foo" in _field_description(field)
-
-    field.update({"InlineHelpText": "bar"})
-    assert "bar" in _field_description(field)
-
-    definition.update({"ComplianceGroup": "bim"})
-    assert "bim" in _field_description(field)
-
-    definition.update({"SecurityClassification": "bam"})
-    description = _field_description(field)
-
-    assert "bam" in description
-    expected = (
-        "- Description: foo\n"
-        "- Help Text: bar\n"
-        "- Compliance Categorization: bim\n"
-        "- Data Sensitivity Level: bam"
-    )
-    assert description == expected