castor-extractor 0.22.5__tar.gz → 0.23.1__tar.gz

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/CHANGELOG.md

@@ -1,6 +1,13 @@
-
 # Changelog
 
+## 0.23.1 - 2025-02-10
+
+* change command for Confluence from password to token to reflect better their documentation
+
+## 0.22.6 - 2025-01-21
+
+* bump dependencies: looker, databricks, deptry, ...
+
 ## 0.22.5 - 2025-01-09
 
 * Databricks: validate and deduplicate lineage links

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.22.5
+Version: 0.23.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -30,7 +30,7 @@ Provides-Extra: snowflake
 Provides-Extra: sqlserver
 Provides-Extra: tableau
 Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
-Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
+Requires-Dist: databricks-sql-connector (==3.6.0) ; extra == "databricks" or extra == "all"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
@@ -38,7 +38,7 @@ Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=2,<3)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
-Requires-Dist: looker-sdk (>=24.16.0,<24.17.0) ; extra == "looker" or extra == "all"
+Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
 Requires-Dist: numpy (<2) ; extra == "bigquery" or extra == "databricks" or extra == "all"
 Requires-Dist: numpy (>=1.26) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "bigquery" or extra == "databricks" or extra == "all")
@@ -57,7 +57,7 @@ Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or e
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
 Requires-Dist: sqlalchemy-bigquery[bqstorage] (>=1.0.0,<=2.0.0) ; extra == "bigquery" or extra == "all"
 Requires-Dist: sqlalchemy-redshift (>=0.8.14,<0.9.0) ; extra == "redshift" or extra == "all"
-Requires-Dist: tableauserverclient (==0.25.0) ; extra == "tableau" or extra == "all"
+Requires-Dist: tableauserverclient (>=0.25.0,<0.26.0) ; extra == "tableau" or extra == "all"
 Requires-Dist: tqdm (>=4.0.0,<5.0.0)
 Requires-Dist: typing-extensions (>=4,<5)
 Requires-Dist: websocket-client (>=1,<2) ; extra == "qlik" or extra == "all"
@@ -204,9 +204,16 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 [Castor](https://castordoc.com) helps you find, understand, use your data assets
 
-
 # Changelog
 
+## 0.23.1 - 2025-02-10
+
+* change command for Confluence from password to token to reflect better their documentation
+
+## 0.22.6 - 2025-01-21
+
+* bump dependencies: looker, databricks, deptry, ...
+
 ## 0.22.5 - 2025-01-09
 
 * Databricks: validate and deduplicate lineage links

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/commands/extract_confluence.py

@@ -13,7 +13,7 @@ def main():
     parser.add_argument("-a", "--account_id", help="Confluence account id")
     parser.add_argument("-b", "--base_url", help="Confluence account base url")
     parser.add_argument("-o", "--output", help="Directory to write to")
-    parser.add_argument("-p", "--password", help="Confluence password")
+    parser.add_argument("-t", "--token", help="Confluence API token")
     parser.add_argument("-u", "--username", help="Confluence username")
 
     confluence.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/file_checker/file.py

@@ -123,7 +123,7 @@ class FileCheckerRun:
         """
         if not self.verbose:
             return
-        header = f"Issues detected on Row #{index+1}\n"
+        header = f"Issues detected on Row #{index + 1}\n"
         for k, v in row.items():
             header += f"{str(k):<20} {str(v):<100}\n"
         self.logger.info(header + _SEPARATOR + issue_log + _SEPARATOR)

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/knowledge/confluence/client/client.py

@@ -37,7 +37,7 @@ class ConfluenceClient(APIClient):
     ):
         self.account_id = credentials.account_id
         auth = BasicAuth(
-            username=credentials.username, password=credentials.password
+            username=credentials.username, password=credentials.token
         )
         super().__init__(
             auth=auth,

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/knowledge/confluence/client/credentials.py

@@ -14,5 +14,5 @@ class ConfluenceCredentials(BaseSettings):
 
     account_id: str
     base_url: str
-    password: str
+    token: str
     username: str

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/knowledge/notion/client/client.py

@@ -11,7 +11,7 @@ from .endpoints import NotionEndpointFactory
 from .pagination import NotionPagination
 
 VOLUME_IGNORED = 10
-IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY,)
+IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY, HTTPStatus.NOT_FOUND)
 NOTION_SAFE_MODE = RequestSafeMode(
     max_errors=VOLUME_IGNORED,
     status_codes=IGNORED_ERROR_CODES,

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/utils/validation_test.py

@@ -37,10 +37,10 @@ def test_validate_baseurl():
         validate_baseurl("")
 
     with pytest.raises(ValueError):
-        validate_baseurl("ftp://tesla.looker.eu")
+        validate_baseurl("ftp://timber.looker.eu")
 
     with pytest.raises(ValueError):
-        validate_baseurl("https://tesla.looker.eu:19999?query=params")
+        validate_baseurl("https://timber.looker.eu:19999?query=params")
 
     with pytest.raises(ValueError):
-        validate_baseurl("https://tesla.looker.eu:19999#fragment")
+        validate_baseurl("https://timber.looker.eu:19999#fragment")

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/visualization/looker_studio/client/looker_studio_api_client.py

@@ -1,4 +1,6 @@
+from contextlib import contextmanager
 from functools import partial
+from http import HTTPStatus
 from typing import Iterator, Optional
 
 from google.auth.transport.requests import Request
@@ -7,6 +9,7 @@ from google.oauth2.service_account import Credentials
 from ....utils import (
     APIClient,
     BearerAuth,
+    RequestSafeMode,
     fetch_all_pages,
 )
 from .credentials import LookerStudioCredentials
@@ -16,6 +19,22 @@ from .pagination import LookerStudioPagination
 from .scopes import SCOPES
 
 
+@contextmanager
+def temporary_safe_request(client: APIClient):
+    """
+    Allows applying the safe mode only to specific requests instead of all
+    API calls.
+    """
+    client._safe_mode = RequestSafeMode(
+        max_errors=float("inf"),
+        status_codes=(HTTPStatus.UNAUTHORIZED,),
+    )
+    try:
+        yield client
+    finally:
+        client._safe_mode = RequestSafeMode()
+
+
 class LookerStudioAPIAuth(BearerAuth):
     def __init__(
         self,
@@ -56,9 +75,14 @@ class LookerStudioAPIClient(APIClient):
 
         See also https://developers.google.com/looker-studio/integrate/api/reference/types#Permissions
         """
-        data = self._get(LookerStudioAPIEndpoint.permissions(asset_name))
-        permissions = data["permissions"]
-        return len(permissions.keys()) == 1
+        with temporary_safe_request(self):
+            data = self._get(LookerStudioAPIEndpoint.permissions(asset_name))
+            permissions = data.get("permissions")
+
+            if not permissions:
+                return True
+
+            return len(permissions.keys()) == 1
 
     def _user_assets(
         self, asset_type: LookerStudioAssetType, user_email: str
@@ -76,7 +100,7 @@ class LookerStudioAPIClient(APIClient):
 
         for asset in assets:
             asset_name = asset["name"]
-            owner = asset["owner"]
+            owner = asset.get("owner", "")
             if owner == user_email and not self._is_private_asset(asset_name):
                 yield asset
 

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/visualization/powerbi/assets.py

@@ -8,6 +8,7 @@ class PowerBiAsset(ExternalAsset):
     DASHBOARDS = "dashboards"
     DATASETS = "datasets"
     DATASET_FIELDS = "dataset_fields"
+    DATASET_RELATIONSHIPS = "dataset_relationships"
     METADATA = "metadata"
     PAGES = "pages"
     REPORTS = "reports"
@@ -19,20 +20,9 @@ class PowerBiAsset(ExternalAsset):
     def optional(cls) -> set["PowerBiAsset"]:
         return {
             PowerBiAsset.DATASET_FIELDS,
+            PowerBiAsset.DATASET_RELATIONSHIPS,
             PowerBiAsset.PAGES,
             PowerBiAsset.TABLES,
             PowerBiAsset.TILES,
             PowerBiAsset.USERS,
         }
-
-
-# Assets extracted from the Metadata file
-# They are not directly fetched from the PowerBi api.
-METADATA_ASSETS = (
-    PowerBiAsset.DATASET_FIELDS,
-    PowerBiAsset.TABLES,
-    PowerBiAsset.TILES,
-    PowerBiAsset.USERS,
-)
-
-REPORTS_ASSETS = (PowerBiAsset.PAGES,)

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/visualization/powerbi/extract.py

@@ -11,7 +11,7 @@ from ...utils import (
     write_json,
     write_summary,
 )
-from .assets import METADATA_ASSETS, REPORTS_ASSETS, PowerBiAsset
+from .assets import PowerBiAsset
 from .client import PowerbiClient, PowerbiCredentials
 
 logger = logging.getLogger(__name__)
@@ -21,7 +21,7 @@ def iterate_all_data(
     client: PowerbiClient,
 ) -> Iterable[tuple[PowerBiAsset, Union[list, dict]]]:
     for asset in PowerBiAsset:
-        if asset in METADATA_ASSETS + REPORTS_ASSETS:
+        if asset in PowerBiAsset.optional:
             continue
 
         logger.info(f"Extracting {asset.name} from API")

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/warehouse/abstract/extract.py

@@ -60,7 +60,7 @@ class SQLExtractionProcessor:
         total = len(queries)
 
         for i, query in enumerate(queries):
-            logger.info(f"Extracting {asset.value}: query {i+1}/{total}")
+            logger.info(f"Extracting {asset.value}: query {i + 1}/{total}")
             # concatenate results of all queries
             data = chain(data, self._fetch(query))
 

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/castor_extractor/warehouse/bigquery/client.py

@@ -66,9 +66,9 @@ class BigQueryClient(SqlalchemyClient):
         return BIGQUERY_URI
 
     def _credentials(self) -> Credentials:
-        assert (
-            CREDENTIALS_INFO_KEY in self._options
-        ), "Missing BigQuery credentials in engine's options"
+        assert CREDENTIALS_INFO_KEY in self._options, (
+            "Missing BigQuery credentials in engine's options"
+        )
         credentials = self._options[CREDENTIALS_INFO_KEY]
         return Credentials.from_service_account_info(credentials)
 

{castor_extractor-0.22.5 → castor_extractor-0.23.1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.22.5"
+version = "0.23.1"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -27,7 +27,7 @@ include = [
 [tool.poetry.dependencies]
 python = ">=3.9, <3.13"
 
-cryptography = { version = "^43.0.0", optional = true }
+cryptography = { version = "^43.0.0", optional = true } # cryptography (44.0.0) requires Python !=3.9.0,!=3.9.1,>=3.7
 google-api-core = ">=2.1.1, <3.0.0"
 google-api-python-client = {version = "^2.121.0", optional = true}
 google-auth = "^2"
@@ -35,7 +35,7 @@ google-cloud-core = ">=2.1.0, <3.0.0"
 google-cloud-storage = "^2"
 google-resumable-media = "^2.0.3"
 googleapis-common-protos = "^1.53.0"
-looker-sdk = { version = "~24.16.0", optional = true } # https://github.com/looker-open-source/sdk-codegen/issues/1518 -> lock until issue is resolved
+looker-sdk = { version = "^25.0.0", optional = true }
 msal = { version = "^1.20.0", optional = true }
 numpy = [ # Subdependency of pyarrow. As numpy add 3.12 in 1.26. And poetry doesn't manage it well.
   { version = ">=1.26", python = "~3.12", optional = true },
@@ -56,7 +56,7 @@ sqlalchemy-bigquery = { version = ">=1.0.0, <= 2.0.0", optional = true, extras =
   "bqstorage",
 ] }
 sqlalchemy-redshift = { version = "~0.8.14", optional = true }
-tableauserverclient = { version = "==0.25.0", optional = true }
+tableauserverclient = { version = "^0.25.0", optional = true } # https://gitlab.com/castordoc/extractor/-/merge_requests/2270#note_1982885120
 tqdm = ">=4.0.0, <5.0.0"
 typing-extensions = "^4"
 websocket-client = { version = "^1", optional = true }
@@ -64,8 +64,10 @@ pydantic = "^2.6"
 pydantic-settings = "^2.2"
 pymssql = { version = "^2.2.11", optional = true }
 pymysql = { extras = ["rsa"], version = "^1.1.0", optional = true }
-databricks-sql-connector = { version = "^3.2.0", optional = true }
 
+# 3.7 has issue with urrllib dependency
+# https://castorglobal.slack.com/archives/C049PRU1GK1/p1738328811319309
+databricks-sql-connector = { version = "3.6.0", optional = true } 
 
 [tool.poetry.extras]
 bigquery = ["sqlalchemy-bigquery", "numpy"]
@@ -109,7 +111,7 @@ all = [
 pytest = "^8"
 pytest-freezegun = "^0.4"
 requests-mock = "^1.11"
-ruff = "^0.7.4"
+ruff = "^0.9"
 
 [tool.poetry.group.dev.dependencies]
 deptry = ">=0.21"