castor-extractor 0.22.6__tar.gz → 0.23.2__tar.gz

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/CHANGELOG.md

@@ -1,11 +1,17 @@
-
 # Changelog
 
+## 0.23.2 - 2025-02-17
+
+* support page_size in Tableau extraction command
+
+## 0.23.1 - 2025-02-10
+
+* change command for Confluence from password to token to reflect better their documentation
+
 ## 0.22.6 - 2025-01-21
 
 * bump dependencies: looker, databricks, deptry, ...
 
-
 ## 0.22.5 - 2025-01-09
 
 * Databricks: validate and deduplicate lineage links

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.22.6
+Version: 0.23.2
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -30,7 +30,7 @@ Provides-Extra: snowflake
 Provides-Extra: sqlserver
 Provides-Extra: tableau
 Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
-Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
+Requires-Dist: databricks-sql-connector (==3.6.0) ; extra == "databricks" or extra == "all"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
@@ -204,14 +204,20 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 [Castor](https://castordoc.com) helps you find, understand, use your data assets
 
-
 # Changelog
 
+## 0.23.2 - 2025-02-17
+
+* support page_size in Tableau extraction command
+
+## 0.23.1 - 2025-02-10
+
+* change command for Confluence from password to token to reflect better their documentation
+
 ## 0.22.6 - 2025-01-21
 
 * bump dependencies: looker, databricks, deptry, ...
 
-
 ## 0.22.5 - 2025-01-09
 
 * Databricks: validate and deduplicate lineage links

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/commands/extract_confluence.py

@@ -13,7 +13,7 @@ def main():
     parser.add_argument("-a", "--account_id", help="Confluence account id")
     parser.add_argument("-b", "--base_url", help="Confluence account base url")
     parser.add_argument("-o", "--output", help="Directory to write to")
-    parser.add_argument("-p", "--password", help="Confluence password")
+    parser.add_argument("-t", "--token", help="Confluence API token")
     parser.add_argument("-u", "--username", help="Confluence username")
 
     confluence.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/commands/extract_tableau.py

@@ -28,6 +28,12 @@ def main():
         help="Extract Tableau Pulse assets: Metrics and Subscriptions",
     )
 
+    parser.add_argument(
+        "--page-size",
+        help="Lower the pagination when request exceeds the nodes size limit",
+        required=False,
+    )
+
     parser.add_argument("-o", "--output", help="Directory to write to")
 
     tableau_revamp.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/knowledge/confluence/client/client.py

@@ -37,7 +37,7 @@ class ConfluenceClient(APIClient):
     ):
         self.account_id = credentials.account_id
         auth = BasicAuth(
-            username=credentials.username, password=credentials.password
+            username=credentials.username, password=credentials.token
         )
         super().__init__(
             auth=auth,

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/knowledge/confluence/client/credentials.py

@@ -14,5 +14,5 @@ class ConfluenceCredentials(BaseSettings):
 
     account_id: str
     base_url: str
-    password: str
+    token: str
     username: str

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/knowledge/notion/client/client.py

@@ -11,7 +11,7 @@ from .endpoints import NotionEndpointFactory
 from .pagination import NotionPagination
 
 VOLUME_IGNORED = 10
-IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY,)
+IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY, HTTPStatus.NOT_FOUND)
 NOTION_SAFE_MODE = RequestSafeMode(
     max_errors=VOLUME_IGNORED,
     status_codes=IGNORED_ERROR_CODES,

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/utils/validation_test.py

@@ -37,10 +37,10 @@ def test_validate_baseurl():
         validate_baseurl("")
 
     with pytest.raises(ValueError):
-        validate_baseurl("ftp://tesla.looker.eu")
+        validate_baseurl("ftp://timber.looker.eu")
 
     with pytest.raises(ValueError):
-        validate_baseurl("https://tesla.looker.eu:19999?query=params")
+        validate_baseurl("https://timber.looker.eu:19999?query=params")
 
     with pytest.raises(ValueError):
-        validate_baseurl("https://tesla.looker.eu:19999#fragment")
+        validate_baseurl("https://timber.looker.eu:19999#fragment")

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/visualization/looker_studio/client/looker_studio_api_client.py

@@ -1,4 +1,6 @@
+from contextlib import contextmanager
 from functools import partial
+from http import HTTPStatus
 from typing import Iterator, Optional
 
 from google.auth.transport.requests import Request
@@ -7,6 +9,7 @@ from google.oauth2.service_account import Credentials
 from ....utils import (
     APIClient,
     BearerAuth,
+    RequestSafeMode,
     fetch_all_pages,
 )
 from .credentials import LookerStudioCredentials
@@ -16,6 +19,22 @@ from .pagination import LookerStudioPagination
 from .scopes import SCOPES
 
 
+@contextmanager
+def temporary_safe_request(client: APIClient):
+    """
+    Allows applying the safe mode only to specific requests instead of all
+    API calls.
+    """
+    client._safe_mode = RequestSafeMode(
+        max_errors=float("inf"),
+        status_codes=(HTTPStatus.UNAUTHORIZED,),
+    )
+    try:
+        yield client
+    finally:
+        client._safe_mode = RequestSafeMode()
+
+
 class LookerStudioAPIAuth(BearerAuth):
     def __init__(
         self,
@@ -56,9 +75,14 @@ class LookerStudioAPIClient(APIClient):
 
         See also https://developers.google.com/looker-studio/integrate/api/reference/types#Permissions
         """
-        data = self._get(LookerStudioAPIEndpoint.permissions(asset_name))
-        permissions = data["permissions"]
-        return len(permissions.keys()) == 1
+        with temporary_safe_request(self):
+            data = self._get(LookerStudioAPIEndpoint.permissions(asset_name))
+            permissions = data.get("permissions")
+
+            if not permissions:
+                return True
+
+            return len(permissions.keys()) == 1
 
     def _user_assets(
         self, asset_type: LookerStudioAssetType, user_email: str
@@ -76,7 +100,7 @@ class LookerStudioAPIClient(APIClient):
 
         for asset in assets:
             asset_name = asset["name"]
-            owner = asset["owner"]
+            owner = asset.get("owner", "")
             if owner == user_email and not self._is_private_asset(asset_name):
                 yield asset
 

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py

@@ -99,7 +99,7 @@ class TableauClientMetadataApi:
         override_page_size: Optional[int] = None,
     ):
         self._server = server
-        self._forced_page_size = override_page_size
+        self._override_page_size = override_page_size
 
     def _call(
         self,
@@ -117,7 +117,7 @@ class TableauClientMetadataApi:
 
     def _page_size(self, asset: TableauRevampAsset) -> int:
         return (
-            self._forced_page_size
+            self._override_page_size
             or _CUSTOM_PAGE_SIZE.get(asset)
             or DEFAULT_PAGE_SIZE
         )

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/visualization/tableau_revamp/extract.py

@@ -33,10 +33,15 @@ def extract_all(**kwargs) -> None:
     """
     output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
     with_pulse = kwargs.get("with_pulse") or False
+    page_size = kwargs.get("page_size")
     timestamp = current_timestamp()
 
     credentials = TableauRevampCredentials(**kwargs)
-    client = TableauRevampClient(credentials, with_pulse=with_pulse)
+    client = TableauRevampClient(
+        credentials,
+        with_pulse=with_pulse,
+        override_page_size=page_size,
+    )
     client.login()
 
     for key, data in iterate_all_data(client):

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/castor_extractor/warehouse/salesforce/extract.py

@@ -3,6 +3,7 @@ import logging
 from ...utils import AbstractStorage, LocalStorage, write_summary
 from ...utils.salesforce import SalesforceCredentials
 from ..abstract import (
+    EXTERNAL_LINEAGE_ASSETS,
     SupportedAssets,
     WarehouseAsset,
     WarehouseAssetGroup,
@@ -21,7 +22,8 @@ SALESFORCE_CATALOG_ASSETS: tuple[WarehouseAsset, ...] = (
 )
 
 SALESFORCE_ASSETS: SupportedAssets = {
-    WarehouseAssetGroup.CATALOG: SALESFORCE_CATALOG_ASSETS
+    WarehouseAssetGroup.CATALOG: SALESFORCE_CATALOG_ASSETS,
+    WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
 }
 
 

{castor_extractor-0.22.6 → castor_extractor-0.23.2}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.22.6"
+version = "0.23.2"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -64,8 +64,10 @@ pydantic = "^2.6"
 pydantic-settings = "^2.2"
 pymssql = { version = "^2.2.11", optional = true }
 pymysql = { extras = ["rsa"], version = "^1.1.0", optional = true }
-databricks-sql-connector = { version = "^3.2.0", optional = true }
 
+# 3.7 has issue with urrllib dependency
+# https://castorglobal.slack.com/archives/C049PRU1GK1/p1738328811319309
+databricks-sql-connector = { version = "3.6.0", optional = true }
 
 [tool.poetry.extras]
 bigquery = ["sqlalchemy-bigquery", "numpy"]