castor-extractor 0.16.9__py3-none-any.whl → 0.16.11__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.16.11 - 2024-06-03
+
+* Tableau: add extra fields to optimise storage
+
+## 0.16.10 - 2024-05-30
+
+* Salesforce: extract sobjects Label as table name
+
 ## 0.16.9 - 2024-05-28
 
 * Tableau: extract only fields that are necessary

castor_extractor/visualization/tableau_revamp/client/client.py

@@ -28,10 +28,13 @@ _TSC_ASSETS = (
     TableauRevampAsset.USAGE,
 )
 
-# speed up extraction: fields and columns are smaller but volumes are bigger
+# increase the value when extraction is too slow
+# decrease the value when timeouts arise
 _CUSTOM_PAGE_SIZE: Dict[TableauRevampAsset, int] = {
+    # fields and columns are light but volumes are bigger
+    TableauRevampAsset.COLUMN: 200,
     TableauRevampAsset.FIELD: 1000,
-    TableauRevampAsset.COLUMN: 1000,
+    TableauRevampAsset.TABLE: 50,
 }
 
 

castor_extractor/visualization/tableau_revamp/client/gql_queries.py

@@ -18,7 +18,11 @@ QUERY_TEMPLATE = """
 
 _COLUMNS_QUERY = """
 downstreamDashboards { id }
-downstreamFields { id }
+downstreamFields {
+    id
+    __typename
+    datasource { id }
+}
 downstreamWorkbooks { id }
 id
 name

castor_extractor/warehouse/salesforce/client.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, Iterator, List
+from typing import Dict, Iterator, List, Tuple
 
 from tqdm import tqdm  # type: ignore
 
@@ -96,17 +96,19 @@ class SalesforceClient(SalesforceBaseClient):
         """
         sobjects = self.fetch_sobjects()
         logger.info(f"Extracted {len(sobjects)} sobjects")
-        return self.formatter.tables(sobjects)
+        return list(self.formatter.tables(sobjects))
 
     def columns(
-        self, sobject_names: List[str], show_progress: bool = True
+        self, sobject_names: List[Tuple[str, str]], show_progress: bool = True
     ) -> List[dict]:
         """
         Get salesforce sobject fields as columns
         show_progress: optionally deactivate the tqdm progress bar
         """
         sobject_fields: Dict[str, List[dict]] = dict()
-        for sobject_name in tqdm(sobject_names, disable=not show_progress):
-            fields = self.fetch_fields(sobject_name)
-            sobject_fields[sobject_name] = fields
+        for api_name, table_name in tqdm(
+            sobject_names, disable=not show_progress
+        ):
+            fields = self.fetch_fields(api_name)
+            sobject_fields[table_name] = fields
         return self.formatter.columns(sobject_fields)

castor_extractor/warehouse/salesforce/extract.py

@@ -72,8 +72,8 @@ class SalesforceExtractionProcessor:
         catalog_locations[WarehouseAsset.TABLE.value] = location
         logger.info(f"Extracted {len(tables)} tables to {location}")
 
-        table_names = [t["table_name"] for t in tables]
-        columns = self._client.columns(table_names, show_progress)
+        sobject_names = [(t["api_name"], t["table_name"]) for t in tables]
+        columns = self._client.columns(sobject_names, show_progress)
         location = self._storage.put(WarehouseAsset.COLUMN.value, columns)
         catalog_locations[WarehouseAsset.COLUMN.value] = location
         logger.info(f"Extracted {len(columns)} columns to {location}")

castor_extractor/warehouse/salesforce/format.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, Iterator, List
 
 from .constants import SCHEMA_NAME
 
@@ -35,17 +35,35 @@ def _to_column_payload(field: dict, position: int, table_name: str) -> dict:
     }
 
 
-def _to_table_payload(table: dict) -> dict:
+def _to_table_payload(sobject: dict, table_name: str) -> dict:
     return {
-        "id": table["QualifiedApiName"],
+        "id": table_name,
+        "api_name": sobject["QualifiedApiName"],
+        "label": sobject["Label"],
         "schema_id": SCHEMA_NAME,
-        "table_name": table["QualifiedApiName"],
+        "table_name": table_name,
         "description": "",
         "tags": [],
         "type": "TABLE",
     }
 
 
+def _merge_label_and_api_name(sobject: dict) -> dict:
+    label = sobject["Label"]
+    api_name = sobject["QualifiedApiName"]
+    table_name = f"{label} ({api_name})"
+    return _to_table_payload(sobject, table_name)
+
+
+def _by_label(sobjects: List[dict]) -> Dict[str, List[dict]]:
+    by_label: Dict[str, List[dict]] = dict()
+    for sobject in sobjects:
+        label = sobject["Label"]
+        similar_sobjects = by_label.setdefault(label, [])
+        similar_sobjects.append(sobject)
+    return by_label
+
+
 class SalesforceFormatter:
     """
     Helper functions that format the response in the format to be exported as
@@ -53,9 +71,18 @@ class SalesforceFormatter:
     """
 
     @staticmethod
-    def tables(sobjects: List[dict]) -> List[dict]:
-        """formats the raw list of sobjects to tables"""
-        return [_to_table_payload(s) for s in sobjects]
+    def tables(sobjects: List[dict]) -> Iterator[dict]:
+        """
+        formats the raw list of sobjects to tables
+        if two tables share the same label, then we add the api name as well
+        """
+        by_label = _by_label(sobjects)
+        for label, similars in by_label.items():
+            if len(similars) > 1:
+                yield from [_merge_label_and_api_name(s) for s in similars]
+            else:
+                sobject = similars[0]  # unique sobject on label
+                yield _to_table_payload(sobject, label)
 
     @staticmethod
     def columns(sobject_fields: Dict[str, List[dict]]) -> List[dict]:

castor_extractor/warehouse/salesforce/format_test.py

@@ -1,4 +1,21 @@
-from .format import _field_description
+from typing import Dict, Tuple
+
+from .format import (
+    SCHEMA_NAME,
+    SalesforceFormatter,
+    _by_label,
+    _field_description,
+    _merge_label_and_api_name,
+)
+
+
+def _example_sobjects() -> Tuple[Dict[str, str], ...]:
+    """Returns 4 sobjects with 2 sharing the same label"""
+    a = {"Label": "a", "QualifiedApiName": "a_one"}
+    b = {"Label": "b", "QualifiedApiName": "b"}
+    c = {"Label": "c", "QualifiedApiName": "c"}
+    a_prime = {"Label": "a", "QualifiedApiName": "a_two"}
+    return a, b, c, a_prime
 
 
 def test__field_description():
@@ -30,3 +47,34 @@ def test__field_description():
         "- Data Sensitivity Level: bam"
     )
     assert description == expected
+
+
+def test__merge_label_and_api_name():
+    sobject = {"Label": "foo", "QualifiedApiName": "bar"}
+    payload = _merge_label_and_api_name(sobject)
+    expected_name = "foo (bar)"
+    assert payload == {
+        "id": expected_name,
+        "api_name": "bar",
+        "label": "foo",
+        "schema_id": SCHEMA_NAME,
+        "table_name": expected_name,
+        "description": "",
+        "tags": [],
+        "type": "TABLE",
+    }
+
+
+def test__by_label():
+    a, b, c, a_prime = _example_sobjects()
+    sobjects = [a, b, c, a_prime]
+    by_label = _by_label(sobjects)
+    assert by_label == {"a": [a, a_prime], "b": [b], "c": [c]}
+
+
+def test_salesforce_formatter_tables():
+    sobjects = [*_example_sobjects()]
+    tables = SalesforceFormatter.tables(sobjects)
+    expected_names = {"a (a_one)", "a (a_two)", "b", "c"}
+    payload_names = {t["table_name"] for t in tables}
+    assert payload_names == expected_names

{castor_extractor-0.16.9.dist-info → castor_extractor-0.16.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.16.9
+Version: 0.16.11
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.16.9.dist-info → castor_extractor-0.16.11.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=WwEWPQQuGqVnWLhPtEh3SuOlBrNgHyHcLsYuvahpN7E,10437
+CHANGELOG.md,sha256=CuRENmJ6p4IM6b8vrmt6QI8uN8mX4a-FI_hJ4cQkPps,10588
 Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=uF6PXm9ocPITlKVSh9afTakHmpLx3TvawLf-CbMP3wM,3578
@@ -244,10 +244,10 @@ castor_extractor/visualization/tableau/usage.py,sha256=LlFwlbEr-EnYUJjKZha99CRCR
 castor_extractor/visualization/tableau_revamp/__init__.py,sha256=a3DGjQhaz17gBqW-E84TAgupKbqLC40y5Ajo1yn-ot4,156
 castor_extractor/visualization/tableau_revamp/assets.py,sha256=owlwaI2E4UKk1YhkaHgaAXx6gu3Op6EqZ7bjp0tHI6s,351
 castor_extractor/visualization/tableau_revamp/client/__init__.py,sha256=wmS9uLtUiqNYVloi0-DgD8d2qzu3RVZEAtWiaDp6G_M,90
-castor_extractor/visualization/tableau_revamp/client/client.py,sha256=8BO7J-HFM2j6_f-Hjj3uSWip11eKeZ0cjhxGEqMTPRA,9428
+castor_extractor/visualization/tableau_revamp/client/client.py,sha256=T7v84dnT97sFqVdzJdk1aOZ7S6U9u6d-j3KBqVj91eY,9532
 castor_extractor/visualization/tableau_revamp/client/credentials.py,sha256=fHG32egq6ll2U4BNazalMof_plzfCMQjrN9WOs6kezk,3014
 castor_extractor/visualization/tableau_revamp/client/errors.py,sha256=dTe1shqmWmAXpDpCz-E24m8dGYjt6rvIGV9qQb4jnvI,150
-castor_extractor/visualization/tableau_revamp/client/gql_queries.py,sha256=jBxvjQnOIWfFjMJpr7S_ZPnQhdzabxoO3jyEKi8A8ns,2112
+castor_extractor/visualization/tableau_revamp/client/gql_queries.py,sha256=VP6xXi1mWKDGVnkWPLstLHqc3T4GVSnywyyoT6BJkFY,2153
 castor_extractor/visualization/tableau_revamp/client/tsc_fields.py,sha256=WsDliPCo-XsQ7wN-j0gpW9bdxCHvgH-aePywiltzfbU,688
 castor_extractor/visualization/tableau_revamp/constants.py,sha256=PcdudAogQhi3e-knalhgliMKjy5ahN0em_-7XSLrnxM,87
 castor_extractor/visualization/tableau_revamp/extract.py,sha256=2SLUxp5okM4AcEJJ61ZgcC2ikfZZl9MH17CEXMXmgl0,1450
@@ -323,11 +323,11 @@ castor_extractor/warehouse/redshift/queries/user.sql,sha256=sEXveJAuNvZacvpI6Wfw
 castor_extractor/warehouse/redshift/queries/view_ddl.sql,sha256=Pkyh_QT6d4rhTeyiVcqw6O8CRl7NEhk2p7eM5YIn5kg,719
 castor_extractor/warehouse/redshift/query.py,sha256=0C81rkt2cpkWrJIxxwALDyqr-49vlqQM04y_N6wwStc,540
 castor_extractor/warehouse/salesforce/__init__.py,sha256=NR4aNea5jeE1xYqeZ_29deeN84CkN0_D_Z7CLQdJvFY,137
-castor_extractor/warehouse/salesforce/client.py,sha256=_XiQJJJfELKGmzuBv8Mr_C0FJ-oLg71KbvpehrGvJ_k,3842
+castor_extractor/warehouse/salesforce/client.py,sha256=ETnZ3n-GFFH0XohDB2ft74wI1HMspvTefR3k7ne-pmI,3891
 castor_extractor/warehouse/salesforce/constants.py,sha256=GusduVBCPvwpk_Im6F3bDvXeNQ7hRnCMdIAjIg65RnE,52
-castor_extractor/warehouse/salesforce/extract.py,sha256=ZTb58t7mqhavNvErrnw8M0L4Uu3qJpQEIldymurbgl0,3417
-castor_extractor/warehouse/salesforce/format.py,sha256=_BSj_G6C-kPwRubxSx1WuHg-_nYVQVNgAANqNfXL5RM,2154
-castor_extractor/warehouse/salesforce/format_test.py,sha256=6hy0USZH7-PDQt3oZ9_3Nwlr3eHLkqNEchqIM3bIDrU,858
+castor_extractor/warehouse/salesforce/extract.py,sha256=IbhkCli8bSn7tjhRNlaD_HhfmZmv-5E5ajZfEUh68Hs,3438
+castor_extractor/warehouse/salesforce/format.py,sha256=f5mMJyPsVU1ZSLe5WGCUOpj2SyW7_DFfzNVNu_m2aV0,3126
+castor_extractor/warehouse/salesforce/format_test.py,sha256=HBlAYBoCOHaq_QOFudZlpcZb5TyZWV9v-cxK4tklg50,2253
 castor_extractor/warehouse/salesforce/soql.py,sha256=pAEaJE8ZUcyN3ptBsZGzNcGRhCcU81X6RMlnF1HRMw4,1063
 castor_extractor/warehouse/snowflake/__init__.py,sha256=TEGXTyxWp4Tr9gIHb-UFVTRKj6YWmrRtqHruiKSZGiY,174
 castor_extractor/warehouse/snowflake/client.py,sha256=XT0QLVNff_586SDuMe40iu8FCwPDh2uBV5aKc1Ql914,5555
@@ -368,8 +368,8 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.16.9.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.16.9.dist-info/METADATA,sha256=qRP78w8BztI4N8IyOLoESkFdhKWByXf7PQQjFLTvu6A,6582
-castor_extractor-0.16.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.16.9.dist-info/entry_points.txt,sha256=SbyPk58Gh-FRztfCNnUZQ6w7SatzNJFZ6GIJLNsy7tI,1427
-castor_extractor-0.16.9.dist-info/RECORD,,
+castor_extractor-0.16.11.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.16.11.dist-info/METADATA,sha256=sVbdD6MsgGVPxckw8tREx_xeajevgThiIkuU2IFYBaM,6583
+castor_extractor-0.16.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+castor_extractor-0.16.11.dist-info/entry_points.txt,sha256=SbyPk58Gh-FRztfCNnUZQ6w7SatzNJFZ6GIJLNsy7tI,1427
+castor_extractor-0.16.11.dist-info/RECORD,,