castor-extractor 0.25.7__py3-none-any.whl → 0.25.10__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 0.25.10 - 2025-10-09
+
+* Fix import
+
+## 0.25.9 - 2025-10-09
+
+* Snowflake: raise an exception when no database is available for extraction
+* Databricks: raise an exception when no database is available for extraction
+* BigQuery: raise an exception when no project is available for extraction
+
+## 0.25.8 - 2025-10-09
+
+* Count: extracting queries and canvas_loads
+
 ## 0.25.7 - 2025-10-07
 
 * SqlServer: Ensure database consistency between query and engine

castor_extractor/exceptions.py

@@ -0,0 +1,5 @@
+class NoDatabaseProvidedException(Exception):
+    def __init__(self):
+        super().__init__("""No database eligible for extraction.
+If you are using the db_allow/db_block options, please make sure to use the correct case.
+        """)

castor_extractor/visualization/count/assets.py

@@ -5,7 +5,9 @@ class CountAsset(ExternalAsset):
     """Count assets"""
 
     CANVASES = "canvases"
+    CANVAS_LOADS = "canvas_loads"
     CANVAS_PERMISSIONS = "canvas_permissions"
     CELLS = "cells"
     PROJECTS = "projects"
+    QUERIES = "queries"
     USERS = "users"

castor_extractor/visualization/count/client/client.py

@@ -3,6 +3,7 @@ from dataclasses import asdict
 from typing import Any, Iterator
 
 from ....utils import load_file
+from ....warehouse.abstract import TimeFilter
 from ....warehouse.bigquery import BigQueryClient
 from ..assets import (
     CountAsset,
@@ -27,18 +28,21 @@ class CountClient(BigQueryClient):
         super().__init__(asdict(credentials))
         self.project_id = credentials.project_id
         self.dataset_id = credentials.dataset_id
+        self.time_filter = TimeFilter.default()  # setting current date - 1
 
     def _load_query(self, asset: CountAsset) -> str:
         query = load_file(
             f"{_QUERIES_FOLDER}/{asset.name.lower()}.sql", __file__
         )
         return query.format(
-            project_id=self.project_id, dataset_id=self.dataset_id
+            project_id=self.project_id,
+            dataset_id=self.dataset_id,
+            extract_date=self.time_filter.day,
         )
 
     def fetch(self, asset: CountAsset) -> Iterator[dict[str, Any]]:
         """
-        Fetch the asset given as param, by running a BigQuery query.
+        Fetch the asset given as a param by running a BigQuery query.
         """
         logger.info(f"Running BigQuery query to fetch: {asset.name}")
 

castor_extractor/visualization/count/client/queries/canvas_loads.sql

@@ -0,0 +1,14 @@
+WITH view_count AS(
+    SELECT
+        canvas_key,
+        DATE(loaded_at) AS load_day
+    FROM `{project_id}.{dataset_id}.canvas_loads`
+    WHERE TRUE
+        AND date(loaded_at) = '{extract_date}'
+)
+
+SELECT
+    canvas_key,
+    COUNT(*) AS view_count
+FROM view_count
+GROUP BY canvas_key

castor_extractor/visualization/count/client/queries/queries.sql

@@ -0,0 +1,23 @@
+WITH ranked_queries AS (
+    SELECT
+        cell_key,
+        connection_key,
+        query,
+        started_at,
+        ROW_NUMBER() OVER (
+            PARTITION BY cell_key
+            ORDER BY started_at DESC
+        ) AS rank
+    FROM `{project_id}.{dataset_id}.queries`
+    WHERE TRUE
+        AND DATE(started_at) = '{extract_date}'
+)
+
+SELECT
+    cell_key,
+    connection_key,
+    query,
+    started_at
+FROM ranked_queries
+WHERE TRUE
+    AND rank = 1

castor_extractor/warehouse/abstract/extract.py

@@ -48,7 +48,7 @@ class SQLExtractionProcessor:
         # dict > set > dict
         return [dict(t) for t in {tuple(d.items()) for d in data}]
 
-    def _fetch(self, query: ExtractionQuery) -> Iterator[dict]:
+    def fetch(self, query: ExtractionQuery) -> Iterator[dict]:
         default: Callable[[], Iterator] = lambda: iter(())  # type: ignore
         decorator = safe_mode(self._safe_mode, default)
         decorated_execute = decorator(self._client.execute)
@@ -62,7 +62,7 @@ class SQLExtractionProcessor:
         for i, query in enumerate(queries):
             logger.info(f"Extracting {asset.value}: query {i + 1}/{total}")
             # concatenate results of all queries
-            data = chain(data, self._fetch(query))
+            data = chain(data, self.fetch(query))
 
         if self._query_builder.needs_deduplication(asset):
             # cast the list to iterator, but the streaming pipeline is broken in that case

castor_extractor/warehouse/bigquery/extract.py

@@ -2,6 +2,9 @@ import json
 import logging
 from typing import cast
 
+from ...exceptions import (
+    NoDatabaseProvidedException,
+)
 from ...logger import add_logging_file_handler
 from ...utils import LocalStorage, SafeMode, from_env, write_summary
 from ..abstract import (
@@ -62,6 +65,9 @@ def extract_all(**kwargs) -> None:
         db_blocked=kwargs.get("db_blocked"),
         dataset_blocked=kwargs.get("dataset_blocked"),
     )
+    projects = client.get_projects()
+    if not projects:
+        raise NoDatabaseProvidedException
 
     logger.info(f"Available projects: {client.get_projects()}\n")
 

castor_extractor/warehouse/databricks/extract.py

@@ -2,6 +2,9 @@ import logging
 from datetime import date
 from typing import Optional
 
+from ...exceptions import (
+    NoDatabaseProvidedException,
+)
 from ...utils import AbstractStorage, LocalStorage, write_summary
 from ..abstract import (
     ADDITIONAL_LINEAGE_ASSETS,
@@ -177,6 +180,9 @@ def extract_all(**kwargs) -> None:
         db_blocked=kwargs.get("db_blocked"),
     )
 
+    if not client.databases():
+        raise NoDatabaseProvidedException
+
     storage = LocalStorage(directory=output_directory)
 
     extractor = DatabricksExtractionProcessor(

castor_extractor/warehouse/snowflake/extract.py

@@ -1,5 +1,8 @@
 import logging
 
+from ...exceptions import (
+    NoDatabaseProvidedException,
+)
 from ...utils import LocalStorage, from_env, write_summary
 from ..abstract import (
     CATALOG_ASSETS,
@@ -60,6 +63,14 @@ def _credentials(params: dict) -> dict:
     raise ValueError("missing password or private key")
 
 
+def _get_database_names(
+    extractor: SQLExtractionProcessor, query_builder: SnowflakeQueryBuilder
+) -> set[str]:
+    db_query = query_builder.build(WarehouseAsset.DATABASE)
+    databases = list(extractor.fetch(db_query[0]))
+    return {db["database_name"] for db in databases}
+
+
 def extract_all(**kwargs) -> None:
     """
     Extract all assets from Snowflake and store the results in CSV files
@@ -85,6 +96,12 @@ def extract_all(**kwargs) -> None:
         storage=storage,
     )
 
+    database_names = _get_database_names(extractor, query_builder)
+    if not database_names:
+        raise NoDatabaseProvidedException
+
+    logger.info(f"Available databases: {database_names}\n")
+
     for group in extractable_asset_groups(SNOWFLAKE_ASSETS):
         for asset in group:
             logger.info(f"Extracting `{asset.value.upper()}` ...")

{castor_extractor-0.25.7.dist-info → castor_extractor-0.25.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.25.7
+Version: 0.25.10
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -216,6 +216,20 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.25.10 - 2025-10-09
+
+* Fix import
+
+## 0.25.9 - 2025-10-09
+
+* Snowflake: raise an exception when no database is available for extraction
+* Databricks: raise an exception when no database is available for extraction
+* BigQuery: raise an exception when no project is available for extraction
+
+## 0.25.8 - 2025-10-09
+
+* Count: extracting queries and canvas_loads
+
 ## 0.25.7 - 2025-10-07
 
 * SqlServer: Ensure database consistency between query and engine

{castor_extractor-0.25.7.dist-info → castor_extractor-0.25.10.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=59CuAENUDPUzehj5T9VIKCbS3bG-HGUQ-7U_QtegrvA,21528
+CHANGELOG.md,sha256=-GLL2qrB_fkho3FplO7T7Fd6WtPbwEaA-9nfEhXWjtg,21892
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -31,6 +31,7 @@ castor_extractor/commands/extract_tableau.py,sha256=cfH-b0Hq9LGrQSJv02Yr_4d6oNqh
 castor_extractor/commands/extract_thoughtspot.py,sha256=caAYJlH-vK7u5IUB6OKXxcaWfLgc7d_XqnFDWK6YNS4,639
 castor_extractor/commands/file_check.py,sha256=TJx76Ymd0QCECmq35zRJMkPE8DJtSInB28MuSXWk8Ao,2644
 castor_extractor/commands/upload.py,sha256=sqpEF_qqCNvT_niIrM6jPhzLaFVjtYwpc2iZw540F20,1633
+castor_extractor/exceptions.py,sha256=WypgqL8AxYL1viYUCSY4528pk5TRcqVNPvyLMMMDWGw,238
 castor_extractor/file_checker/__init__.py,sha256=OSt6YLhUT42U_Cp3LCLHMVruwDkksL75Ij13X2UPnVk,119
 castor_extractor/file_checker/column.py,sha256=6bJhcW1snYwgHKkqlS0Ak7XLHZr4YBwO46JCIlnQNKg,3086
 castor_extractor/file_checker/column_test.py,sha256=1j8PxvmvmJgpd-mk30iMYOme32ovPSIn4yCXywFoXrg,1935
@@ -162,14 +163,16 @@ castor_extractor/utils/validation_test.py,sha256=A7P6VmI0kYX2aGIeEN12y7LsY7Kpm8p
 castor_extractor/utils/write.py,sha256=KQVWF29N766avzmSb129IUWrId5c_8BtnYhVLmU6YIs,2133
 castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/visualization/count/__init__.py,sha256=lvxGtSe3erjTYK0aPnkOyJibcsC6Q1AFchnK-hZt558,114
-castor_extractor/visualization/count/assets.py,sha256=VZCRVDKWSu6l2lVGJS4JKOOmfCUkbS8MnJiLcAY9vqw,232
+castor_extractor/visualization/count/assets.py,sha256=zvj2FuCrZlyKPAd8lx5rgWPKsEQ0biQJCk7AEYj6qd0,290
 castor_extractor/visualization/count/client/__init__.py,sha256=YawYDutDI0sprp72jN9tKi8bbXCoc0Ij0Ev582tKjqk,74
-castor_extractor/visualization/count/client/client.py,sha256=WgljCj8G7D0Brxa0llaeOQ2Ipd7FvtDWFoLWoPyqT9A,1523
+castor_extractor/visualization/count/client/client.py,sha256=vSrV8V3acQAz9Hc2_DKjeU500bx2FT-DHrzUwTdjroo,1706
 castor_extractor/visualization/count/client/credentials.py,sha256=LZWvcz7p5lrgdgoIQLcxFyv4gqUBW4Jj4qDKN-VW31I,273
+castor_extractor/visualization/count/client/queries/canvas_loads.sql,sha256=suR5_X2SU9UIGwDi7mSxZrJvoh_-4WJwgtfBYnKMV1E,288
 castor_extractor/visualization/count/client/queries/canvas_permissions.sql,sha256=iFmMfR0zusjxTxmYUS6p0kibZCsnHOQMbAlxaNjx-H4,108
 castor_extractor/visualization/count/client/queries/canvases.sql,sha256=Ur5HBD9JJH0r14xIj_rwoctnds082_F931vlfcnwi_I,86
 castor_extractor/visualization/count/client/queries/cells.sql,sha256=Kkk0jyU337PD6RPshSo_ucLl5PS7kIvJZlUnVnmJUkM,111
 castor_extractor/visualization/count/client/queries/projects.sql,sha256=3Jem3QCVwk4wHiWRJL7cN6Vl2Yc5RZ8yC8ndvPAkaFM,68
+castor_extractor/visualization/count/client/queries/queries.sql,sha256=ffnlRwMedTVoMzuXkQaTWw5oOP-Ties9vGCfQOXdhQ0,456
 castor_extractor/visualization/count/client/queries/users.sql,sha256=H0n7S7P5cCAWbgPxU32psIc1epXySzsAaQ7MQ9JrkfM,102
 castor_extractor/visualization/count/extract.py,sha256=ZBsJ9tMxxaq1jG8qJp_OGVK3yPDNkVUsP1_3rcUMtYg,1378
 castor_extractor/visualization/domo/__init__.py,sha256=1axOCPm4RpdIyUt9LQEvlMvbOPllW8rk63h6EjVgJ0Y,111
@@ -326,7 +329,7 @@ castor_extractor/warehouse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 castor_extractor/warehouse/abstract/__init__.py,sha256=Fdfa026tgOo64MvzVRLHM_F2G-JmcehrF0mh3dHgb7s,419
 castor_extractor/warehouse/abstract/asset.py,sha256=wR5mJxAHBcqJ86HRb_Y8x3mDN4uUgSg8jMToLNu0jTM,2740
 castor_extractor/warehouse/abstract/asset_test.py,sha256=_kd4ybNlWSAdSdEgJKC-jhJTa1nMRa9i8RO3YbqKLM4,758
-castor_extractor/warehouse/abstract/extract.py,sha256=9Y2fUn3y2-2WjiHnrabjvAvOA8UETJeTYr18zcM7bdI,2924
+castor_extractor/warehouse/abstract/extract.py,sha256=a3Poqm1hF6NQQNAWQVib9_uj41LcvXzoX8IQCPCT66c,2922
 castor_extractor/warehouse/abstract/query.py,sha256=h1VvSo6TpyS1myRqmPtoIFY1fVgbthsHOkkajUz-PKA,2444
 castor_extractor/warehouse/abstract/time_filter.py,sha256=bggIONfMmUxffkA6TwM3BsjfS2l9WFxPq8krfsau5pw,935
 castor_extractor/warehouse/abstract/time_filter_test.py,sha256=PIkegB7KOKBdpc6zIvmyl_CeQyADeFDplyQ8HTNU5LA,448
@@ -334,7 +337,7 @@ castor_extractor/warehouse/bigquery/__init__.py,sha256=PCGNYdi7dHv-SyanUWzRuBp-y
 castor_extractor/warehouse/bigquery/client.py,sha256=lPAn6WUwDq0rIPNaMOcabet8C4TYJ93EWZUxX72XrZc,5595
 castor_extractor/warehouse/bigquery/client_test.py,sha256=Ym8e4d--0YQwiVcNUnXLx0X-X6ZznwNMBMbMaDS5oEA,1514
 castor_extractor/warehouse/bigquery/credentials.py,sha256=oCZ8H7qpudKzwM7PRMpVAmWXt7bjIRa8Harmp-ysQJ4,425
-castor_extractor/warehouse/bigquery/extract.py,sha256=TAxpdwZ6kfSe7xv22vcFwAE3-Rr1VF9UnX0DM_IPdF8,2934
+castor_extractor/warehouse/bigquery/extract.py,sha256=N2LZz6UU0CRRhOw66mrnTmImiIxVn-_sGsCxOCW4wcY,3097
 castor_extractor/warehouse/bigquery/queries/.sqlfluff,sha256=ce8UDW2k39v6RBVxgKqjOHHYMoGN9S9f7BCZNHHhox8,30
 castor_extractor/warehouse/bigquery/queries/column.sql,sha256=NxdTnHwomHTEGSc-UoXFKUwg59I9XAOwrSau7JUqGQE,1815
 castor_extractor/warehouse/bigquery/queries/cte/sharded.sql,sha256=-G7_4lxV7UPe72mYlp4HDGeM_fJjZWuXJ7Q0vxvj5_U,1454
@@ -355,7 +358,7 @@ castor_extractor/warehouse/databricks/client_test.py,sha256=dqEdEAt-6e8CtQ7M2L5v
 castor_extractor/warehouse/databricks/credentials.py,sha256=ExtVcl2NpMXTx1Lg8vHQdzQtSEm2aqpg3D1BJrNAUjI,528
 castor_extractor/warehouse/databricks/endpoints.py,sha256=qPoL9CtPFJdwVuW9rJ37nmeMd-nChOBouEVYb4SlaUE,670
 castor_extractor/warehouse/databricks/enums.py,sha256=3T6BbVvbWvfWkD23krsYT1x0kKh1qRzNPl6WpcXe300,274
-castor_extractor/warehouse/databricks/extract.py,sha256=Z4VTEIf0QMiua0QGAlJdQ86kxmGAXekQ304aCKme6IY,7358
+castor_extractor/warehouse/databricks/extract.py,sha256=mv780vnwJQ9vHhBRBR5SNcc7Z8PbGCLOrq-XSV0uziA,7495
 castor_extractor/warehouse/databricks/format.py,sha256=S3BOcwJubc1pyKr-li26uftUUfsjfrm5Qf4LqmElXVk,6736
 castor_extractor/warehouse/databricks/format_test.py,sha256=ls0IcOElqp_qecAzNbK0zdca7Pms4seCHimbw8NAoAI,3322
 castor_extractor/warehouse/databricks/pagination.py,sha256=sM1G0sN1pf1TPpI0Y3Oew378UGEKVkMRc2Mlu9tDjLo,545
@@ -418,7 +421,7 @@ castor_extractor/warehouse/snowflake/client.py,sha256=RB72bbl_k91wDU76yrggPK6oeE
 castor_extractor/warehouse/snowflake/client_test.py,sha256=ihWtOOAQfh8pu5JTr_EWfqefKOVIaJXznACURzaU1Qs,1432
 castor_extractor/warehouse/snowflake/credentials.py,sha256=u0sZ6xPtcZmmvnUsAejJk-YxGl8BTzX_BlRjRk92BYU,932
 castor_extractor/warehouse/snowflake/credentials_test.py,sha256=Lkc-DHXOvr50KrqAW4nt_x0IA0Mu_CsBVu6ATnzQB6I,673
-castor_extractor/warehouse/snowflake/extract.py,sha256=eGtIqW5kKJl-e36viqYQzkXn39CJkmMBR4oSt-B0ud4,3082
+castor_extractor/warehouse/snowflake/extract.py,sha256=N-lZHOTW0bQyEeKA9EuhPB9YkAd1UjyHKkEsl8hat6g,3625
 castor_extractor/warehouse/snowflake/queries/.sqlfluff,sha256=vttrwcr64JVIuvc7WIg9C54cbOkjg_VjXNR7YnTGOPE,31
 castor_extractor/warehouse/snowflake/queries/column.sql,sha256=Ru-yC0s76I9LehOA4aCZ--xz6D9H1Hyr3OZdILOBHAw,1882
 castor_extractor/warehouse/snowflake/queries/column_lineage.sql,sha256=YKBiZ6zySSNcXLDXwm31EjGIIkkkZc0-S6hI1SRM80o,1179
@@ -443,8 +446,8 @@ castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=MAlnTis43E3Amu1e1Oz
 castor_extractor/warehouse/sqlserver/queries/view_ddl.sql,sha256=9rynvx6MWg3iZzrWPB7haZfVKEPkxulzryE2g19x804,315
 castor_extractor/warehouse/sqlserver/query.py,sha256=gr5lnZSUm-wSYuVnJlg6fc7jXWirbL-sCiQN9RnAiPQ,1789
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.25.7.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.25.7.dist-info/METADATA,sha256=omciSlMhL8CWEtzosDWL1hj0SP2C8DiI4z0CeMb-nTs,29030
-castor_extractor-0.25.7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.25.7.dist-info/entry_points.txt,sha256=qyTrKNByoq2HYi1xbA79OU7qxg-OWPvle8VwDqt-KnE,1869
-castor_extractor-0.25.7.dist-info/RECORD,,
+castor_extractor-0.25.10.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.25.10.dist-info/METADATA,sha256=G36bUjs5FkXG0aihDl5OmmTf_bFN01Js5Xlbp9T7crA,29395
+castor_extractor-0.25.10.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.25.10.dist-info/entry_points.txt,sha256=qyTrKNByoq2HYi1xbA79OU7qxg-OWPvle8VwDqt-KnE,1869
+castor_extractor-0.25.10.dist-info/RECORD,,