castor-extractor 0.21.9__py3-none-any.whl → 0.22.1__py3-none-any.whl

castor_extractor/warehouse/salesforce/client.py

@@ -1,6 +1,6 @@
 import logging
 from functools import partial
-from typing import Dict, List, Optional, Tuple
+from typing import Optional
 
 from tqdm import tqdm  # type: ignore
 
@@ -29,7 +29,7 @@ class SalesforceClient(SalesforceBaseClient):
     def name() -> str:
         return "Salesforce"
 
-    def fetch_sobjects(self) -> List[dict]:
+    def fetch_sobjects(self) -> list[dict]:
         """Fetch all sobjects"""
         logger.info("Extracting sobjects")
         query = format_sobject_query()
@@ -39,7 +39,7 @@ class SalesforceClient(SalesforceBaseClient):
         results = fetch_all_pages(request_, SalesforceSQLPagination)
         return list(results)
 
-    def fetch_fields(self, sobject_name: str) -> List[dict]:
+    def fetch_fields(self, sobject_name: str) -> list[dict]:
         """Fetches fields of a given sobject"""
         query = SOBJECT_FIELDS_QUERY_TPL.format(
             entity_definition_id=sobject_name
@@ -55,7 +55,7 @@ class SalesforceClient(SalesforceBaseClient):
             return None
         return response["records"][0]["Description"]
 
-    def add_table_descriptions(self, sobjects: List[dict]) -> List[dict]:
+    def add_table_descriptions(self, sobjects: list[dict]) -> list[dict]:
         """
         Add table descriptions.
         We use the tooling API which does not handle well the LIMIT in SOQL
@@ -67,7 +67,7 @@ class SalesforceClient(SalesforceBaseClient):
             described_sobjects.append({**sobject, "Description": description})
         return described_sobjects
 
-    def tables(self) -> List[dict]:
+    def tables(self) -> list[dict]:
         """
         Get Salesforce sobjects as tables
         """
@@ -77,13 +77,13 @@ class SalesforceClient(SalesforceBaseClient):
         return list(self.formatter.tables(described_sobjects))
 
     def columns(
-        self, sobject_names: List[Tuple[str, str]], show_progress: bool = True
-    ) -> List[dict]:
+        self, sobject_names: list[tuple[str, str]], show_progress: bool = True
+    ) -> list[dict]:
         """
         Get salesforce sobject fields as columns
         show_progress: optionally deactivate the tqdm progress bar
         """
-        sobject_fields: Dict[str, List[dict]] = dict()
+        sobject_fields: dict[str, list[dict]] = dict()
         for api_name, table_name in tqdm(
             sobject_names, disable=not show_progress
         ):

castor_extractor/warehouse/salesforce/extract.py

@@ -1,5 +1,4 @@
 import logging
-from typing import Dict, List, Tuple
 
 from ...utils import AbstractStorage, LocalStorage, write_summary
 from ...utils.salesforce import SalesforceCredentials
@@ -14,9 +13,9 @@ from .client import SalesforceClient
 logger = logging.getLogger(__name__)
 
 
-Paths = Dict[str, str]
+Paths = dict[str, str]
 
-SALESFORCE_CATALOG_ASSETS: Tuple[WarehouseAsset, ...] = (
+SALESFORCE_CATALOG_ASSETS: tuple[WarehouseAsset, ...] = (
     WarehouseAsset.TABLE,
     WarehouseAsset.COLUMN,
 )
@@ -81,7 +80,7 @@ class SalesforceExtractionProcessor:
 
     def extract_role(self) -> Paths:
         """extract no users and return the empty file location"""
-        users: List[dict] = []
+        users: list[dict] = []
         location = self._storage.put(WarehouseAsset.USER.value, users)
         logger.info(f"Extracted {len(users)} users to {location}")
         return {WarehouseAsset.USER.value: location}

castor_extractor/warehouse/salesforce/format.py

@@ -1,9 +1,10 @@
-from typing import Any, Dict, Iterator, List
+from collections.abc import Iterator
+from typing import Any
 
 from ...utils import group_by
 from .constants import SCHEMA_NAME
 
-_HAS_DUPLICATE_KEY = "#has_duplicate"
+_HAS_DUPLICATE_KEY = "#has_duplicate_label"
 
 
 def _clean(raw: str) -> str:
@@ -25,10 +26,10 @@ def _name(sobject: dict) -> str:
     return f"{label} ({api_name})"
 
 
-def _field_description(field: Dict[str, Any]) -> str:
-    context: Dict[str, str] = {}
+def _field_description(field: dict[str, Any]) -> str:
+    context: dict[str, str] = {}
 
-    field_definition: Dict[str, str] = field.get("FieldDefinition") or {}
+    field_definition: dict[str, str] = field.get("FieldDefinition") or {}
     if description := field_definition.get("Description"):
         context["Description"] = _clean(description)
     if help_text := field.get("InlineHelpText"):
@@ -69,9 +70,15 @@ def _to_table_payload(sobject: dict) -> dict:
     }
 
 
-def _detect_duplicates(sobjects: List[dict]) -> List[dict]:
+def _remove_duplicates(sobjects: list[dict]) -> list[dict]:
+    """only keep one object per QualifiedApiName"""
+    by_name = group_by("QualifiedApiName", sobjects)
+    return [objects[0] for _, objects in by_name.items()]
+
+
+def _detect_duplicate_labels(sobjects: list[dict]) -> list[dict]:
     """
-    enrich the given data with "has_duplicate" flag:
+    enrich the given data with "has_duplicate_label" flag:
     - True when another asset has the same Label in the list
     - False otherwise
     """
@@ -89,18 +96,19 @@ class SalesforceFormatter:
     """
 
     @staticmethod
-    def tables(sobjects: List[dict]) -> Iterator[dict]:
+    def tables(sobjects: list[dict]) -> Iterator[dict]:
         """
         formats the raw list of sobjects to tables
         """
-        sobjects = _detect_duplicates(sobjects)
+        sobjects = _remove_duplicates(sobjects)
+        sobjects = _detect_duplicate_labels(sobjects)
         for sobject in sobjects:
             yield _to_table_payload(sobject)
 
     @staticmethod
-    def columns(sobject_fields: Dict[str, List[dict]]) -> Iterator[dict]:
+    def columns(sobject_fields: dict[str, list[dict]]) -> Iterator[dict]:
         """formats the raw list of sobject fields to columns"""
         for table_name, fields in sobject_fields.items():
-            fields = _detect_duplicates(fields)
+            fields = _detect_duplicate_labels(fields)
             for index, field in enumerate(fields):
                 yield _to_column_payload(field, index, table_name)

castor_extractor/warehouse/salesforce/format_test.py

@@ -1,24 +1,24 @@
-from typing import Dict, List, Tuple
-
 from .format import (
     _HAS_DUPLICATE_KEY,
     SalesforceFormatter,
-    _detect_duplicates,
+    _detect_duplicate_labels,
     _field_description,
     _name,
+    _remove_duplicates,
 )
 
 
-def _tables_sobjects() -> Tuple[Dict[str, str], ...]:
+def _tables_sobjects() -> tuple[dict[str, str], ...]:
     """Returns 4 sobjects with 2 sharing the same label"""
     a = {"Label": "a", "QualifiedApiName": "a_one"}
     b = {"Label": "b", "QualifiedApiName": "b"}
-    c = {"Label": "c", "QualifiedApiName": "c"}
+    c = {"Label": "c", "QualifiedApiName": "c_unique_so_doesnt_matter"}
     a_prime = {"Label": "a", "QualifiedApiName": "a_two"}
-    return a, b, c, a_prime
+    b_exact_duplicate = {"Label": "b", "QualifiedApiName": "b"}
+    return a, b, c, a_prime, b_exact_duplicate
 
 
-def _columns_sobjects() -> Dict[str, List[dict]]:
+def _columns_sobjects() -> dict[str, list[dict]]:
     a = {"Label": "First Name", "QualifiedApiName": "owner_name"}
     b = {"Label": "First Name", "QualifiedApiName": "editor_name"}
     c = {"Label": "Foo Bar", "QualifiedApiName": "foo_bar"}
@@ -81,14 +81,14 @@ def test__name():
     assert _name(empty_label_sobject) == "empty_label"
 
 
-def test__detect_duplicates():
+def test__detect_duplicate_labels():
     objects = [
         {"Label": "Foo"},
         {"Label": "Bar"},
         {"Label": "Foo"},
     ]
 
-    objects = _detect_duplicates(objects)
+    objects = _detect_duplicate_labels(objects)
     assert objects == [
         {"Label": "Foo", _HAS_DUPLICATE_KEY: True},
         {"Label": "Bar", _HAS_DUPLICATE_KEY: False},
@@ -96,11 +96,25 @@ def test__detect_duplicates():
     ]
 
 
+def test__remove_duplicates():
+    objects = [
+        {"QualifiedApiName": "Foo"},
+        {"QualifiedApiName": "Bar"},
+        {"QualifiedApiName": "Foo"},
+    ]
+
+    objects = _remove_duplicates(objects)
+    assert len(objects) == 2
+    names = {sobject["QualifiedApiName"] for sobject in objects}
+    assert names == {"Foo", "Bar"}
+
+
 def test_salesforce_formatter_tables():
     sobjects = [*_tables_sobjects()]
-    tables = SalesforceFormatter.tables(sobjects)
+    tables = [t for t in SalesforceFormatter.tables(sobjects)]
     expected_names = {"a (a_one)", "a (a_two)", "b", "c"}
     payload_names = {t["table_name"] for t in tables}
+    assert len(tables) == 4  # we only keep one "b"
     assert payload_names == expected_names
 
 

castor_extractor/warehouse/snowflake/query.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import Optional
 
 from ..abstract import (
     CATALOG_ASSETS,
@@ -14,7 +14,7 @@ DB_FILTERED_ASSETS = (
 )
 
 
-def _database_filter(db_list: Optional[List[str]], allow: bool) -> str:
+def _database_filter(db_list: Optional[list[str]], allow: bool) -> str:
     if not db_list:
         return ""
     keyword = "IN" if allow else "NOT IN"
@@ -34,8 +34,8 @@ class SnowflakeQueryBuilder(AbstractQueryBuilder):
     def __init__(
         self,
         time_filter: Optional[TimeFilter] = None,
-        db_allowed: Optional[List[str]] = None,
-        db_blocked: Optional[List[str]] = None,
+        db_allowed: Optional[list[str]] = None,
+        db_blocked: Optional[list[str]] = None,
         fetch_transient: Optional[bool] = False,
     ):
         super().__init__(time_filter=time_filter)
@@ -52,7 +52,7 @@ class SnowflakeQueryBuilder(AbstractQueryBuilder):
 
         return statement
 
-    def build(self, asset: WarehouseAsset) -> List[ExtractionQuery]:
+    def build(self, asset: WarehouseAsset) -> list[ExtractionQuery]:
         query = self.build_default(asset)
 
         if asset in DB_FILTERED_ASSETS:

castor_extractor/warehouse/sqlserver/client.py

@@ -1,4 +1,4 @@
-from typing import Iterator
+from collections.abc import Iterator
 
 from sqlalchemy import text
 

castor_extractor/warehouse/sqlserver/query.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import Optional
 
 from ..abstract import (
     AbstractQueryBuilder,
@@ -19,6 +19,6 @@ class MSSQLQueryBuilder(AbstractQueryBuilder):
     ):
         super().__init__(time_filter=time_filter)
 
-    def build(self, asset: WarehouseAsset) -> List[ExtractionQuery]:
+    def build(self, asset: WarehouseAsset) -> list[ExtractionQuery]:
         query = self.build_default(asset)
         return [query]

{castor_extractor-0.21.9.dist-info → castor_extractor-0.22.1.dist-info}/METADATA

@@ -1,16 +1,15 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.21.9
+Version: 0.22.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
 Author: Castor
 Author-email: support@castordoc.com
-Requires-Python: >=3.8,<3.13
+Requires-Python: >=3.9,<3.13
 Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -20,6 +19,7 @@ Provides-Extra: bigquery
 Provides-Extra: databricks
 Provides-Extra: dbt
 Provides-Extra: looker
+Provides-Extra: lookerstudio
 Provides-Extra: metabase
 Provides-Extra: mysql
 Provides-Extra: postgres
@@ -32,6 +32,7 @@ Provides-Extra: tableau
 Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
 Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
+Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=2,<3)
@@ -39,10 +40,8 @@ Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=24.16.0,<24.17.0) ; extra == "looker" or extra == "all"
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
-Requires-Dist: numpy (<1.25) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "bigquery" or extra == "databricks" or extra == "all")
 Requires-Dist: numpy (<2) ; extra == "bigquery" or extra == "databricks" or extra == "all"
 Requires-Dist: numpy (>=1.26) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "bigquery" or extra == "databricks" or extra == "all")
-Requires-Dist: pandas (<2.1) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "databricks" or extra == "all")
 Requires-Dist: pandas (>=2.1) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "databricks" or extra == "all")
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0) ; extra == "metabase" or extra == "postgres" or extra == "redshift" or extra == "all"
 Requires-Dist: pycryptodome (>=3.0.0,<4.0.0) ; extra == "metabase" or extra == "all"
@@ -52,7 +51,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=75.3.0,<75.4.0)
+Requires-Dist: setuptools (>=75.6)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
@@ -208,6 +207,14 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.22.1 - 2024-12-05
+
+* Salesforce: deduplicate tables
+
+## 0.22.0 - 2024-12-04
+
+* Stop supporting python3.8
+
 ## 0.21.9 - 2024-12-04
 
 * Tableau: fix handling of timeout retry