castor-extractor 0.21.5__tar.gz → 0.21.9__tar.gz

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/CHANGELOG.md

@@ -1,6 +1,22 @@
 
 # Changelog
 
+## 0.21.9 - 2024-12-04
+
+* Tableau: fix handling of timeout retry
+
+## 0.21.8 - 2024-11-26
+
+* Redshift: improve deduplication of columns
+
+## 0.21.7 - 2024-11-26
+
+* Metabase: stop using deprecated table `view_log`
+
+## 0.21.6 - 2024-11-22
+
+* bump dependencies: ruff, setuptools
+
 ## 0.21.5 - 2024-11-20
 
 * PostgreSQL: Fix schema extraction when owner is a role without login privilege

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.21.5
+Version: 0.21.9
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -52,7 +52,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=74,<75)
+Requires-Dist: setuptools (>=75.3.0,<75.4.0)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
@@ -208,6 +208,22 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.21.9 - 2024-12-04
+
+* Tableau: fix handling of timeout retry
+
+## 0.21.8 - 2024-11-26
+
+* Redshift: improve deduplication of columns
+
+## 0.21.7 - 2024-11-26
+
+* Metabase: stop using deprecated table `view_log`
+
+## 0.21.6 - 2024-11-22
+
+* bump dependencies: ruff, setuptools
+
 ## 0.21.5 - 2024-11-20
 
 * PostgreSQL: Fix schema extraction when owner is a role without login privilege

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/castor_extractor/utils/client/api/client.py

@@ -19,6 +19,7 @@ HttpMethod = Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]
 
 DEFAULT_TIMEOUT = 60
 RETRY_ON_EXPIRED_TOKEN = 1
+RETRY_ON_GATEWAY_TIMEOUT = 3
 
 
 def _generate_payloads(
@@ -102,6 +103,10 @@ class APIClient:
             timeout=self._timeout,
         )
 
+    @retry_request(
+        status_codes=(HTTPStatus.GATEWAY_TIMEOUT,),
+        max_retries=RETRY_ON_GATEWAY_TIMEOUT,
+    )
     @retry_request(
         status_codes=(HTTPStatus.UNAUTHORIZED,),
         max_retries=RETRY_ON_EXPIRED_TOKEN,

castor_extractor-0.21.9/castor_extractor/visualization/metabase/client/db/queries/card.sql

@@ -0,0 +1,4 @@
+SELECT
+    *
+FROM
+    {schema}.report_card

castor_extractor-0.21.9/castor_extractor/visualization/metabase/client/db/queries/dashboard.sql

@@ -0,0 +1,4 @@
+SELECT
+    *
+FROM
+    {schema}.report_dashboard

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/castor_extractor/visualization/sigma/client/client.py

@@ -28,6 +28,7 @@ _DATA_ELEMENTS: Tuple[str, ...] = (
 )
 
 _AUTH_TIMEOUT_S = 60
+_SIGMA_TIMEOUT = 120
 
 _SIGMA_HEADERS = {
     "Content-Type": _CONTENT_TYPE,
@@ -75,6 +76,7 @@ class SigmaClient(APIClient):
             host=credentials.host,
             auth=auth,
             headers=_SIGMA_HEADERS,
+            timeout=_SIGMA_TIMEOUT,
             safe_mode=safe_mode or SIGMA_SAFE_MODE,
         )
 

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/castor_extractor/visualization/sigma/client/pagination.py

@@ -5,7 +5,7 @@ from pydantic.alias_generators import to_camel
 
 from ....utils import PaginationModel
 
-SIGMA_API_LIMIT = 500  # default number of records per page
+SIGMA_API_LIMIT = 200  # default number of records per page
 
 
 class SigmaPagination(PaginationModel):

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/castor_extractor/visualization/tableau_revamp/client/client.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 import tableauserverclient as TSC  # type: ignore
 
@@ -121,12 +122,16 @@ class TableauRevampClient:
         credentials: TableauRevampCredentials,
         timeout_sec: int = DEFAULT_TIMEOUT_SECONDS,
         with_pulse: bool = False,
+        override_page_size: Optional[int] = None,
     ):
         self._credentials = credentials
         self._server = _server(credentials.server_url, timeout_sec)
         self._with_pulse = with_pulse
 
-        self._client_metadata = TableauClientMetadataApi(server=self._server)
+        self._client_metadata = TableauClientMetadataApi(
+            server=self._server,
+            override_page_size=override_page_size,
+        )
         self._client_rest = TableauClientRestApi(server=self._server)
         self._client_tsc = TableauClientTSC(server=self._server)
 

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py

@@ -2,10 +2,10 @@ from typing import Dict, Iterator, Optional
 
 import tableauserverclient as TSC  # type: ignore
 
-from ....utils import SerializedAsset
+from ....utils import SerializedAsset, retry
 from ..assets import TableauRevampAsset
 from ..constants import DEFAULT_PAGE_SIZE
-from .errors import TableauApiError
+from .errors import TableauApiError, TableauApiTimeout
 from .gql_queries import FIELDS_QUERIES, GQL_QUERIES, QUERY_TEMPLATE
 
 # increase the value when extraction is too slow
@@ -20,21 +20,58 @@ _CUSTOM_PAGE_SIZE: Dict[TableauRevampAsset, int] = {
     TableauRevampAsset.TABLE: 50,
 }
 
+_TIMEOUT_MESSAGE = (
+    "Execution canceled because timeout of 30000 millis was reached"
+)
+
+_RETRY_BASE_MS = 10_000
+_RETRY_COUNT = 4
+
+
+def _check_errors(answer: dict) -> None:
+    """
+    handle errors in graphql response:
+    - return None when there's no errors in the answer
+    - TableauApiTimeout if any of the errors is a timeout
+    - TableauApiError (generic) otherwise
+    """
+    if "errors" not in answer:
+        return
+
+    errors = answer["errors"]
+
+    for error in errors:
+        if error.get("message") == _TIMEOUT_MESSAGE:
+            # we need specific handling for timeout issues (retry strategy)
+            raise TableauApiTimeout(errors)
+
+    raise TableauApiError(answer["errors"])
+
 
 def gql_query_scroll(
     server,
     query: str,
     resource: str,
 ) -> Iterator[SerializedAsset]:
-    """Iterate over GQL query results, handling pagination and cursor"""
+    """
+    Iterate over GQL query results, handling pagination and cursor
 
+    We have a retry strategy when timeout issues arise.
+    It's a known issue on Tableau side, still waiting for their fix:
+    https://issues.salesforce.com/issue/a028c00000zKahoAAC/undefined
+    """
+
+    @retry(
+        exceptions=(TableauApiTimeout,),
+        max_retries=_RETRY_COUNT,
+        base_ms=_RETRY_BASE_MS,
+    )
     def _call(cursor: Optional[str]) -> dict:
         # If cursor is defined it must be quoted else use null token
         token = "null" if cursor is None else f'"{cursor}"'
         query_ = query.replace("AFTER_TOKEN_SIGNAL", token)
         answer = server.metadata.query(query_)
-        if "errors" in answer:
-            raise TableauApiError(answer["errors"])
+        _check_errors(answer)
         return answer["data"][f"{resource}Connection"]
 
     cursor = None
@@ -58,8 +95,10 @@ class TableauClientMetadataApi:
     def __init__(
         self,
         server: TSC.Server,
+        override_page_size: Optional[int] = None,
     ):
         self._server = server
+        self._forced_page_size = override_page_size
 
     def _call(
         self,
@@ -75,9 +114,16 @@ class TableauClientMetadataApi:
         result_pages = gql_query_scroll(self._server, query, resource)
         return [asset for page in result_pages for asset in page]
 
+    def _page_size(self, asset: TableauRevampAsset) -> int:
+        return (
+            self._forced_page_size
+            or _CUSTOM_PAGE_SIZE.get(asset)
+            or DEFAULT_PAGE_SIZE
+        )
+
     def _fetch_fields(self) -> SerializedAsset:
         result: SerializedAsset = []
-        page_size = _CUSTOM_PAGE_SIZE[TableauRevampAsset.FIELD]
+        page_size = self._page_size(TableauRevampAsset.FIELD)
         for resource, fields in FIELDS_QUERIES:
             current = self._call(resource, fields, page_size)
             result.extend(current)
@@ -90,6 +136,6 @@ class TableauClientMetadataApi:
         if asset == TableauRevampAsset.FIELD:
             return self._fetch_fields()
 
-        page_size = _CUSTOM_PAGE_SIZE.get(asset) or DEFAULT_PAGE_SIZE
+        page_size = self._page_size(asset)
         resource, fields = GQL_QUERIES[asset]
         return self._call(resource, fields, page_size)

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/castor_extractor/visualization/tableau_revamp/client/errors.py

@@ -1,3 +1,8 @@
 class TableauApiError(ValueError):
     def __init__(self, error: str):
         super().__init__(f"Tableau API returned the following error: {error}")
+
+
+class TableauApiTimeout(ValueError):
+    def __init__(self, error: str):
+        super().__init__(f"Tableau API returned a timeout error: {error}")

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/castor_extractor/warehouse/redshift/queries/column.sql

@@ -155,9 +155,9 @@ external_columns AS (
         db.datname || '.' || s.schemaname || '.' || c.tablename AS table_id,
         c.columnname AS column_name,
         db.datname || '.' || s.schemaname || '.' || c.tablename || '.' || c.columnname AS column_id,
-        c.external_type AS data_type,
+        MIN(c.external_type) AS data_type,
         MIN(c.columnnum) AS ordinal_position,
-        CASE c.is_nullable WHEN 'false' THEN 'NO' ELSE 'YES' END AS is_nullable,
+        MIN(CASE c.is_nullable WHEN 'false' THEN 'NO' ELSE 'YES' END) AS is_nullable,
         NULL AS character_maximum_length,
         NULL AS character_octet_length,
         NULL AS numeric_precision,
@@ -172,7 +172,7 @@ external_columns AS (
         JOIN pg_catalog.pg_database AS db ON CURRENT_DATABASE() = db.datname
 
     -- To remove duplicate column names that can occur in external tables (no check on CSVs)
-    GROUP BY database_name, database_id, schema_name, schema_id, table_name, table_id, column_name, column_id, data_type, is_nullable
+    GROUP BY database_name, database_id, schema_name, schema_id, table_name, table_id, column_name, column_id
 )
 
 SELECT * FROM tables

{castor_extractor-0.21.5 → castor_extractor-0.21.9}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.21.5"
+version = "0.21.9"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -51,7 +51,7 @@ python-dateutil = ">=2.0.0, <=3.0.0"
 requests = ">=2.0.0, <3.0.0"
 snowflake-connector-python = { version = "^3.4.0", optional = true }
 snowflake-sqlalchemy = { version = "!=1.2.5, <2.0.0", optional = true }
-setuptools = "^74"
+setuptools = "~75.3.0"  # locked, requires python >= 3.9
 sqlalchemy = "~1.4"
 sqlalchemy-bigquery = { version = ">=1.0.0, <= 2.0.0", optional = true, extras = [
   "bqstorage",
@@ -108,10 +108,10 @@ all = [
 pytest = "^8"
 pytest-freezegun = "^0.4"
 requests-mock = "^1.11"
-ruff = "^0.6.4"
+ruff = "^0.7.4"
 
 [tool.poetry.group.dev.dependencies]
-deptry = "^0.20"
+deptry = "^0.20"  # locked, requires python >= 3.9
 
 [tool.poetry.scripts]
 castor-extract-bigquery = "castor_extractor.commands.extract_bigquery:main"

castor_extractor-0.21.5/castor_extractor/visualization/metabase/client/db/queries/card.sql

@@ -1,21 +0,0 @@
-WITH card_view_log AS
-(
-    SELECT
-        model_id,
-        COUNT(id) AS view_count,
-        MAX("timestamp") AS last_viewed_at
-    FROM
-        {schema}.view_log
-    WHERE
-        model = 'card'
-    GROUP BY
-        1
-)
-
-SELECT
-    rc.*,
-    vl.last_viewed_at AS last_viewed_at,
-    COALESCE(vl.view_count, 0) AS view_count
-FROM
-    {schema}.report_card AS rc
-    LEFT JOIN card_view_log AS vl ON rc.id = vl.model_id

castor_extractor-0.21.5/castor_extractor/visualization/metabase/client/db/queries/dashboard.sql

@@ -1,21 +0,0 @@
-WITH dashboard_view_log AS
-(
-    SELECT
-        model_id,
-        COUNT(id) AS view_count,
-        MAX("timestamp") AS last_viewed_at
-    FROM
-        {schema}.view_log
-    WHERE
-        model = 'dashboard'
-    GROUP BY
-        1
-)
-
-SELECT
-    rd.*,
-    vl.last_viewed_at AS last_viewed_at,
-    COALESCE(vl.view_count, 0) AS view_count
-FROM
-    {schema}.report_dashboard AS rd
-    LEFT JOIN dashboard_view_log AS vl ON rd.id = vl.model_id