castor-extractor 0.19.7__tar.gz → 0.19.8__tar.gz

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/CHANGELOG.md

@@ -1,6 +1,12 @@
 
 # Changelog
 
+## 0.19.8 - 2024-09-18
+
+* Metabase: Handle duplicate dashboards
+* Snowflake: Exclude unnamed tables from extraction
+* Bump dependencies: cryptography, setuptools
+
 ## 0.19.7 - 2024-09-05
 
 * Metabase: Handle compatibility with older version

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.19.7
+Version: 0.19.8
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -29,7 +29,7 @@ Provides-Extra: redshift
 Provides-Extra: snowflake
 Provides-Extra: sqlserver
 Provides-Extra: tableau
-Requires-Dist: cryptography (>=42.0.0,<43.0.0) ; extra == "snowflake"
+Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
 Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-auth (>=2,<3)
@@ -52,7 +52,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=72,<73)
+Requires-Dist: setuptools (>=74,<75)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
@@ -208,6 +208,12 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.19.8 - 2024-09-18
+
+* Metabase: Handle duplicate dashboards
+* Snowflake: Exclude unnamed tables from extraction
+* Bump dependencies: cryptography, setuptools
+
 ## 0.19.7 - 2024-09-05
 
 * Metabase: Handle compatibility with older version

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/castor_extractor/knowledge/notion/client/client.py

@@ -20,6 +20,8 @@ NOTION_BASE_HEADERS = {
     "User-Agent": CASTOR_NOTION_USER_AGENT,
 }
 
+NOTION_DEFAULT_TIMEOUT_S = 180
+
 
 def _search_filter(asset: str) -> Dict[str, Dict[str, str]]:
     return {"filter": {"value": asset, "property": "object"}}
@@ -45,6 +47,7 @@ class NotionClient(APIClient):
             auth=auth,
             headers=NOTION_BASE_HEADERS,
             safe_mode=safe_mode or NOTION_SAFE_MODE,
+            timeout=NOTION_DEFAULT_TIMEOUT_S,
         )
 
     def users(self) -> Iterator[dict]:

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/castor_extractor/knowledge/notion/client/pagination.py

@@ -1,14 +1,16 @@
 from typing import Optional
 
+from pydantic import Field
+
 from ....utils import PaginationModel
 
 
 class NotionPagination(PaginationModel):
     """Class to handle paginated results"""
 
-    results: list
-    next_cursor: Optional[str]
-    has_more: bool
+    results: list = Field(default_factory=list)
+    next_cursor: Optional[str] = None
+    has_more: bool = False
 
     def is_last(self) -> bool:
         return not self.has_more

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/castor_extractor/quality/soda/client/client.py

@@ -1,5 +1,5 @@
 from functools import partial
-from typing import Iterator
+from typing import Callable, Iterator
 
 from ....utils import (
     APIClient,
@@ -11,10 +11,10 @@ from .credentials import SodaCredentials
 from .endpoints import SodaEndpointFactory
 from .pagination import SodaCloudPagination
 
-_CLOUD_API = "https://cloud.soda.io/api/v1/"
 _REQUESTS_PER_MINUTE = 10
 _SECONDS_PER_MINUTE = 60
 _RATE_LIMIT_MS = (_SECONDS_PER_MINUTE // _REQUESTS_PER_MINUTE) + 1
+_CLOUD_PAGE_SIZE = 100
 
 HEADERS = {"Content-Type": "application/json"}
 
@@ -24,16 +24,26 @@ class SodaClient(APIClient):
         cloud_auth = BasicAuth(
             username=credentials.api_key, password=credentials.secret
         )
-        super().__init__(host=_CLOUD_API, auth=cloud_auth, headers=HEADERS)
+        super().__init__(
+            host=credentials.host, auth=cloud_auth, headers=HEADERS
+        )
+
+    @property
+    def _base_request(self) -> Callable:
+        return partial(self._get, params={"size": _CLOUD_PAGE_SIZE})
 
     def datasets(self) -> Iterator[dict]:
-        request = partial(self._get, endpoint=SodaEndpointFactory.datasets())
+        request = partial(
+            self._base_request, endpoint=SodaEndpointFactory.datasets()
+        )
         yield from fetch_all_pages(
             request, SodaCloudPagination, rate_limit=_RATE_LIMIT_MS
         )
 
     def checks(self) -> Iterator[dict]:
-        request = partial(self._get, endpoint=SodaEndpointFactory.checks())
+        request = partial(
+            self._base_request, endpoint=SodaEndpointFactory.checks()
+        )
         yield from fetch_all_pages(
             request, SodaCloudPagination, rate_limit=_RATE_LIMIT_MS
         )

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/castor_extractor/quality/soda/client/credentials.py

@@ -3,6 +3,8 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 
 SODA_ENV_PREFIX = "CASTOR_SODA_"
 
+_CLOUD_API = "https://cloud.soda.io/api/v1/"
+
 
 class SodaCredentials(BaseSettings):
     """Class to handle Soda rest API permissions"""
@@ -15,3 +17,4 @@ class SodaCredentials(BaseSettings):
 
     api_key: str = Field(repr=False)
     secret: str = Field(repr=False)
+    host: str = Field(default=_CLOUD_API)

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/castor_extractor/quality/soda/client/pagination.py

@@ -2,7 +2,6 @@ from typing import List
 
 from ....utils import PaginationModel
 
-_CLOUD_PAGE_SIZE = 100
 _CLOUD_FIRST_PAGE = 0
 
 
@@ -19,7 +18,7 @@ class SodaCloudPagination(PaginationModel):
             if self.current_page_payload
             else _CLOUD_FIRST_PAGE
         )
-        return {"page": current_page + 1, "size": _CLOUD_PAGE_SIZE}
+        return {"page": current_page + 1}
 
     def page_results(self) -> list:
         return self.content

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/castor_extractor/visualization/domo/client/client.py

@@ -45,6 +45,8 @@ _RETRY_EXCEPTIONS = [
 _RETRY_COUNT = 2
 _RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
 
+_PARENT_FOLDER = "/Dashboards"
+
 logger = logging.getLogger(__name__)
 
 
@@ -177,7 +179,7 @@ class DomoClient:
     def _process_pages(
         self,
         page_tree: List[dict],
-        parent_path: str = "/root",
+        parent_path: str = _PARENT_FOLDER,
     ) -> Iterator[dict]:
         """Recursively fetch pages while building the folder architecture"""
         if not page_tree:

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/castor_extractor/visualization/metabase/client/api/client.py

@@ -1,6 +1,6 @@
 import logging
 from http import HTTPStatus
-from typing import Any, Dict, Iterator, List, Optional, cast
+from typing import Any, Dict, Iterator, List, Optional, Set, cast
 
 import requests
 from requests import HTTPError
@@ -134,15 +134,22 @@ class ApiClient:
             collection = self._call(f"collection/{_id}/items?models=dashboard")
             if not collection:
                 continue
+
+            seen_dashboard_ids: Set[int] = set()
+
             for dashboard in cast(SerializedAsset, collection):
                 if dashboard.get("model") != "dashboard":
                     # This is to maintain compatibility with older versions
                     # where ?models=dashboard has no effects
                     continue
+
                 dashboard_id = dashboard.get("id")
                 if not dashboard_id:
                     continue
-                yield cast(Dict, self._call(f"dashboard/{dashboard_id}"))
+
+                if dashboard_id not in seen_dashboard_ids:
+                    seen_dashboard_ids.add(dashboard_id)
+                    yield cast(Dict, self._call(f"dashboard/{dashboard_id}"))
 
     @staticmethod
     def _collection_specifics(collections: SerializedAsset) -> SerializedAsset:

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/castor_extractor/warehouse/snowflake/queries/table.sql

@@ -41,6 +41,8 @@ FROM snowflake.account_usage.tables AS t
     JOIN snowflake.account_usage.schemata AS s ON s.schema_id = t.table_schema_id
     JOIN tags_agg_tables ta ON t.table_id = ta.table_id
 WHERE TRUE
+    AND t.table_name IS NOT NULL
+    AND t.table_name != ''
     AND UPPER(t.table_catalog) NOT IN ('SNOWFLAKE', 'UTIL_DB')
     AND (
         t.deleted IS NULL

{castor_extractor-0.19.7 → castor_extractor-0.19.8}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.19.7"
+version = "0.19.8"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -27,7 +27,7 @@ include = [
 [tool.poetry.dependencies]
 python = ">=3.8, <3.13"
 
-cryptography = { version = "^42.0.0", optional = true }
+cryptography = { version = "^43.0.0", optional = true }
 google-api-core = ">=2.1.1, <3.0.0"
 google-auth = "^2"
 google-cloud-core = ">=2.1.0, <3.0.0"
@@ -51,7 +51,7 @@ python-dateutil = ">=2.0.0, <=3.0.0"
 requests = ">=2.0.0, <3.0.0"
 snowflake-connector-python = { version = "^3.4.0", optional = true }
 snowflake-sqlalchemy = { version = "!=1.2.5, <2.0.0", optional = true }
-setuptools = "^72"
+setuptools = "^74"
 sqlalchemy = "~1.4"
 sqlalchemy-bigquery = { version = ">=1.0.0, <= 2.0.0", optional = true, extras = [
   "bqstorage",
@@ -108,10 +108,10 @@ all = [
 pytest = "^8"
 pytest-freezegun = "^0.4"
 requests-mock = "^1.11"
-ruff = "^0.5.7"
+ruff = "^0.6.4"
 
 [tool.poetry.group.dev.dependencies]
-deptry = "^0.19"
+deptry = "^0.20"
 
 [tool.poetry.scripts]
 castor-extract-bigquery = "castor_extractor.commands.extract_bigquery:main"

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/__init__.py

@@ -1 +0,0 @@
-from .extract import SYNAPSE_ASSETS

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/extract.py

@@ -1,21 +0,0 @@
-import logging
-
-from ..abstract import (
-    CATALOG_ASSETS,
-    EXTERNAL_LINEAGE_ASSETS,
-    QUERIES_ASSETS,
-    VIEWS_ASSETS,
-    SupportedAssets,
-    WarehouseAsset,
-    WarehouseAssetGroup,
-)
-
-logger = logging.getLogger(__name__)
-
-SYNAPSE_ASSETS: SupportedAssets = {
-    WarehouseAssetGroup.CATALOG: CATALOG_ASSETS,
-    WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
-    WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
-    WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
-    WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
-}

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/.sqlfluff

@@ -1,2 +0,0 @@
-[sqlfluff]
-dialect = tsql

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/database.sql

@@ -1,6 +0,0 @@
-SELECT
-    d.name AS database_id,
-    d.name AS database_name,
-    NULL AS "comment"
-FROM sys.databases AS d
-    WHERE d.name != 'master'

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/query.sql

@@ -1,33 +0,0 @@
-WITH query_execution AS (
-    SELECT
-        plan_id,
-        CASE execution_type_desc
-            WHEN 'Aborted' THEN 1
-            WHEN 'Exception' THEN 1
-            ELSE 0
-        END AS aborted,
-        MIN(CAST(first_execution_time AS DATETIME)) AS start_time,
-        MAX(CAST(last_execution_time AS DATETIME)) AS end_time
-    FROM sys.query_store_runtime_stats
-    WHERE CAST(first_execution_time AS DATE) = CAST(DATEADD(day, -1, getdate()) AS DATE)
-    -- Filtering on hour will be needed when running that query out of notebooks
-    GROUP BY plan_id, execution_type_desc
-)
-
-SELECT
-    q.query_id,
-    qt.query_sql_text AS query_text,
-    DB_NAME() AS database_id,
-    DB_NAME() AS database_name,
-    'SYNAPSE' AS user_name,
-    qp.plan_id AS process_id,
-    qe.aborted AS aborted,
-    qe.start_time AS start_time,
-    qe.end_time AS end_time
-FROM sys.query_store_query AS q
-    JOIN sys.query_store_query_text AS qt
-        ON q.query_text_id = qt.query_text_id
-    JOIN sys.query_store_plan AS qp
-        ON q.query_id = qp.query_id
-    JOIN query_execution AS qe
-        ON qp.plan_id = qe.plan_id

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/schema.sql

@@ -1,7 +0,0 @@
-SELECT DISTINCT
-    s.catalog_name AS database_id,
-    s.catalog_name AS database_name,
-    s.schema_name AS schema_name,
-    s.schema_owner AS schema_owner,
-    s.catalog_name + '.' + s.schema_name AS schema_id
-FROM INFORMATION_SCHEMA.SCHEMATA AS s

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/table.sql

@@ -1,36 +0,0 @@
-WITH users AS (
-    SELECT DISTINCT
-        uid,
-        name
-    FROM sys.sysusers
-)
-
-SELECT
-    DB_NAME() as database_id,
-    DB_NAME() AS database_name,
-    DB_NAME() + '.' + s.name AS schema_id,
-    s.name AS schema_name,
-    DB_NAME() + '.' + s.name + '.' + o.name AS table_id,
-    o.name AS table_name,
-    CASE UPPER(REPLACE(o.type, ' ', ''))
-        WHEN 'U' THEN 'TABLE'
-        WHEN 'V' THEN 'VIEW'
-        ELSE 'UNKNOWN'
-    END AS table_type,
-    -- Normally schema owner is table owner, but we can
-    -- change table owner with ALTER AUTHORIZATION command
-    -- in this case we have to look up for the owner
-    -- in sys.objects table
-    COALESCE(su.name, pu.name) AS table_owner,
-    COALESCE(su.uid, pu.uid) AS table_owner_id,
-    NULL AS comment
-FROM sys.objects AS o
-    JOIN sys.schemas AS s
-        ON o.schema_id = s.schema_id
-    LEFT JOIN users AS pu
-        ON s.principal_id = pu.uid
-    LEFT JOIN users AS su
-        ON o.principal_id = su.uid
-WHERE
--- SYNAPSE NOTATION: U for TABLE, V for VIEW
-    o.type IN ('U', 'V')

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/user.sql

@@ -1,4 +0,0 @@
-SELECT
-    name AS user_name,
-    uid AS user_id
-FROM sys.sysusers

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/view_ddl.sql

@@ -1,8 +0,0 @@
-SELECT
-    DB_NAME() AS database_id,
-    DB_NAME() AS database_name,
-    DB_NAME() + '.' + table_schema AS schema_id,
-    table_schema AS schema_name,
-    table_name AS view_name,
-    view_definition AS view_definition
-FROM INFORMATION_SCHEMA.VIEWS