castor-extractor 0.3.1__py3-none-any.whl → 0.3.5__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.3.5 - 2023-04-07
+
+* Extract metadata from successful dbt runs only
+
+## 0.3.4 - 2023-04-05
+
+* Enhance uploader to support `QUALITY` files
+
+## 0.3.3 - 2023-04-04
+
+* Tableau : Improve Table <> Datasource lineage
+
+## 0.3.2 - 2023-04-04
+
+* Allow COPY statements from Snowflake
+
 ## 0.3.1 - 2023-03-30
 
 * Improved Field extraction in Tableau

castor_extractor/commands/upload.py

@@ -5,6 +5,8 @@ from castor_extractor.uploader import FileType  # type: ignore
 from castor_extractor.uploader import upload  # type: ignore
 from castor_extractor.uploader import upload_manifest  # type: ignore
 
+FILE_TYPES = {FileType.QUALITY, FileType.VIZ, FileType.WAREHOUSE}
+
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
 
 
@@ -51,7 +53,7 @@ def main():
     params = _args()
 
     file_type = params.get("file_type")
-    if file_type in [FileType.VIZ, FileType.WAREHOUSE]:
+    if file_type in FILE_TYPES:
         upload(**params)
 
     if file_type == FileType.DBT:

castor_extractor/transformation/dbt/client/client.py

@@ -1,3 +1,4 @@
+import logging
 from enum import Enum
 from typing import Optional
 
@@ -5,9 +6,12 @@ import requests
 
 from .credentials import CredentialsKey, DbtCredentials, get_value
 
+logger = logging.getLogger(__name__)
+
 _CLOUD_URL = "https://cloud.getdbt.com/api/v2/accounts"
 _DATA_KEY = "data"
 _MANIFEST_PATH = "manifest.json"
+_SUCCESSFUL_RUN_STATUS = 10
 
 
 class ContentType(Enum):
@@ -47,20 +51,32 @@ class DbtClient:
             headers=headers,
             params=params,
         )
-        result = response.json()
-        if content_type == ContentType.JSON:
-            return result[_DATA_KEY]
-        return result
+        try:
+            result = response.json()
+        except:
+            context = f"{url}, status {response.status_code}"
+            raise ValueError(f"Couldn't extract data from {context}")
+        else:
+            if content_type == ContentType.JSON:
+                return result[_DATA_KEY]
+            return result
 
     def _infer_account_id(self) -> int:
         result = self._call(url=_CLOUD_URL)
         return result[0]["id"]
 
     def _last_run_id(self) -> int:
+        """
+        Extract last successful run id
+        https://docs.getdbt.com/dbt-cloud/api-v2#tag/Runs/operation/listRunsForAccount
+        """
+
         url = f"{_CLOUD_URL}/{self._account_id}/runs/"
+
         params = {
             "job_definition_id": self._credentials.job_id,
             "order_by": "-finished_at",
+            "status": _SUCCESSFUL_RUN_STATUS,
             "limit": 1,
         }
         return self._call(url, params)[0]["id"]
@@ -72,6 +88,8 @@ class DbtClient:
         """
         run_id = self._last_run_id()
         url = f"{_CLOUD_URL}/{self._account_id}/runs/{run_id}/artifacts/{_MANIFEST_PATH}"
+        logger.info(f"Extracting manifest from run id {run_id} with url {url}")
+
         # setting text content as a workaround to this issue
         # https://stackoverflow.com/questions/68201659/dbt-cloud-api-to-extract-run-artifacts
         return self._call(url, content_type=ContentType.TEXT)

castor_extractor/uploader/constant.py

@@ -7,12 +7,14 @@ class FileType(Enum):
     """type of file to load"""
 
     DBT = "DBT"
+    QUALITY = "QUALITY"
     VIZ = "VIZ"
     WAREHOUSE = "WAREHOUSE"
 
 
 PATH_TEMPLATES = {
     FileType.DBT: "transformation-{source_id}/{timestamp}-manifest.json",
+    FileType.QUALITY: "quality-{source_id}/{timestamp}-{filename}",
     FileType.VIZ: "visualization-{source_id}/{filename}",
     FileType.WAREHOUSE: "warehouse-{source_id}/{filename}",
 }

castor_extractor/visualization/tableau/gql_fields.py

@@ -111,6 +111,9 @@ class GQLQueryFields(Enum):
     """
 
     DATASOURCE: str = """
+           ... on PublishedDatasource {
+               luid
+           }
            id
            name
            hasExtracts

castor_extractor/warehouse/snowflake/queries/query.sql

@@ -57,7 +57,6 @@ WHERE TRUE
         'ROLLBACK',
         'DESCRIBE',
         'ALTER_SESSION',
-        'COPY', -- https://docs.snowflake.com/en/sql-reference/sql/copy-into-location.html#syntax
         'PUT_FILES',
         'CREATE', -- create objects: stage|function|schema|procedure|file|storage|pipe|notification integration
         'SET',

{castor_extractor-0.3.1.dist-info → castor_extractor-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.3.1
+Version: 0.3.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.3.1.dist-info → castor_extractor-0.3.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=YlFaXjr9jWeYbuHXrxqdwltOPkwLbmxmfsHnDe2We-4,4518
+CHANGELOG.md,sha256=DIidPXNKp9icL0ZTvm7mnNSGno2190ncEFar6BoR0Is,4796
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=_H5aVcA57LLStpQaYm0yYndJRuw7BqQKt46D1xogRJo,3390
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -15,7 +15,7 @@ castor_extractor/commands/extract_redshift.py,sha256=bdLp7d7ImZoKCkWc3f3NXF1imIz
 castor_extractor/commands/extract_snowflake.py,sha256=KxUbbuBsKYy8hJxv_98YktoS-gemDp4hbWjGzj9sR1c,1799
 castor_extractor/commands/extract_tableau.py,sha256=ysEE0DHPJk79MKwFqo7mIZMBGSY4Ad-4vLr4OMxRB5k,1390
 castor_extractor/commands/file_check.py,sha256=HmPiuXWlwVWCgQ5lZMoYpuRAH9UJgaW2eIkdT-tHy_c,2667
-castor_extractor/commands/upload.py,sha256=xk_IWmHH-ouD8w0UU7pq7XOAxq8cKgye5s8TVVUdd9w,1887
+castor_extractor/commands/upload.py,sha256=xutRBSzg5nxoMA-fmj7m5Y_96P7xFs2b4MehKfqvzSo,1930
 castor_extractor/file_checker/__init__.py,sha256=OSt6YLhUT42U_Cp3LCLHMVruwDkksL75Ij13X2UPnVk,119
 castor_extractor/file_checker/column.py,sha256=fMchy5v-Sd-0xuYS0V9mob7wnljslzWLhQGqrKGybdk,3097
 castor_extractor/file_checker/column_test.py,sha256=1j8PxvmvmJgpd-mk30iMYOme32ovPSIn4yCXywFoXrg,1935
@@ -32,10 +32,10 @@ castor_extractor/transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 castor_extractor/transformation/dbt/__init__.py,sha256=fynkZOuDwD4AFcAYFdUBUbBPC5HoaQjVu-Z2BMcSnoI,91
 castor_extractor/transformation/dbt/assets.py,sha256=TqjktG7NeK_FUNAoOQJTtgCGVh8Cm9cTnoozNhXErmM,94
 castor_extractor/transformation/dbt/client/__init__.py,sha256=-9TvcGyeANOnoqceq3MnRrM2i6YzK906SZqeJqfLOpg,86
-castor_extractor/transformation/dbt/client/client.py,sha256=k8sxyg34JbZHV05zjJvoZhDeJEY3FAXrHQdJQBd-txU,2387
+castor_extractor/transformation/dbt/client/client.py,sha256=8SdcB5edeRHUUSz7HuTq67B_6RLo9O8z5lIOjyOOrpg,2938
 castor_extractor/transformation/dbt/client/credentials.py,sha256=67mWV2AML8vt--o9-2bvBzlcigj-5ZdocUcUxdc7J30,746
 castor_extractor/uploader/__init__.py,sha256=SSRtwjg-dNoxME-RJy9G1flASiUKAC5bH1htq3CURQg,75
-castor_extractor/uploader/constant.py,sha256=IYg97sA9axd-YLPc9kUSci0J1ucGMIKhOz7eIjta-6w,626
+castor_extractor/uploader/constant.py,sha256=hEJlWYx0dyBzgo59XUBKCYIKEODpIc2DyzwAZIiNO8g,718
 castor_extractor/uploader/env.py,sha256=5HSniVSOYVg4u38O4k8TB_qaJq9s8yJ1hjedkq_gdVg,878
 castor_extractor/uploader/env_test.py,sha256=ClCWWtwd2N-5ClIDUxVMeKkWfhhOTxpppsXUDmdjxSg,472
 castor_extractor/uploader/upload.py,sha256=NTXF4BlVEsRCA6S-SunyNHkcXN9JaRpK1W65ZeMWAcw,3353
@@ -158,7 +158,7 @@ castor_extractor/visualization/tableau/client/safe_mode.py,sha256=12pVLUns4BdK6o
 castor_extractor/visualization/tableau/constants.py,sha256=O2CqeviFz122BumNHoJ1N-e1lzyqIHF9OYnGQttg4hg,126
 castor_extractor/visualization/tableau/errors.py,sha256=WWvmnp5pdxFJqanPKeDRADZc0URSPxkJqxDI6bwoifQ,91
 castor_extractor/visualization/tableau/extract.py,sha256=75Z21qeOyinJqnbWFo74Mk4Q0ja0yorss9ZKPc1eQFg,2873
-castor_extractor/visualization/tableau/gql_fields.py,sha256=UTXBaPsE9meAmPue64LxtNZBv3eOS3SC5fRhRDFUjWs,4655
+castor_extractor/visualization/tableau/gql_fields.py,sha256=m03qXuJzP-4UfUnAVmV8RKBSNoeNl2wVWz8n9tMyGmI,4728
 castor_extractor/visualization/tableau/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/visualization/tableau/tests/unit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/visualization/tableau/tests/unit/assets/graphql/metadata/metadata_1_get.json,sha256=4iMvJ_VakDa67xN2ROraAccaz_DDxX6Y5Y1XnTU5F5Y,446
@@ -242,7 +242,7 @@ castor_extractor/warehouse/snowflake/queries/column_lineage.sql,sha256=YKBiZ6zyS
 castor_extractor/warehouse/snowflake/queries/database.sql,sha256=ifZXoKUXtsrGOxml6AcNhA4yybIyatH5va7bcp-lgCU,483
 castor_extractor/warehouse/snowflake/queries/grant_to_role.sql,sha256=O7AJ1LzoXGDFmiVvQ8EMJ5x8FSAnaxRPdmRyAlEmkUM,272
 castor_extractor/warehouse/snowflake/queries/grant_to_user.sql,sha256=7AalVajU5vRRpIiys1igSwmDXirbwpMTvJr2ihSz2NE,143
-castor_extractor/warehouse/snowflake/queries/query.sql,sha256=2ccND9WFHFZghi5SL8InhS-Bi0B6FnS4ds77J58FT40,1877
+castor_extractor/warehouse/snowflake/queries/query.sql,sha256=-OYcWUvdPBkpOfezkZaW7hrOdDz3JyoqjNdRm_88Rsk,1779
 castor_extractor/warehouse/snowflake/queries/role.sql,sha256=D0VvGxLZMwug2SvefhAsNR9YIun0fZvcDWkz891xSYM,96
 castor_extractor/warehouse/snowflake/queries/schema.sql,sha256=HCDEw0Nj_GPHBNH3Ik_5BF4rkD5yBfSyeN9UaiFGrI4,730
 castor_extractor/warehouse/snowflake/queries/table.sql,sha256=sSfJ2sNsn-pTVDiHQVJnwzcfq_MVZvddwpVgKuwP_n4,1378
@@ -259,7 +259,7 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.3.1.dist-info/entry_points.txt,sha256=kwmxu6k71-dgEuW0G5zD7LUcOsBhPCjp0cGtukkH3OA,915
-castor_extractor-0.3.1.dist-info/WHEEL,sha256=vxFmldFsRN_Hx10GDvsdv1wroKq8r5Lzvjp6GZ4OO8c,88
-castor_extractor-0.3.1.dist-info/METADATA,sha256=6g1MnW3rE9eb_xozy0O4PqtVxSdcEkvXsJlADPo6jsk,6365
-castor_extractor-0.3.1.dist-info/RECORD,,
+castor_extractor-0.3.5.dist-info/entry_points.txt,sha256=kwmxu6k71-dgEuW0G5zD7LUcOsBhPCjp0cGtukkH3OA,915
+castor_extractor-0.3.5.dist-info/WHEEL,sha256=vxFmldFsRN_Hx10GDvsdv1wroKq8r5Lzvjp6GZ4OO8c,88
+castor_extractor-0.3.5.dist-info/METADATA,sha256=UjUUD5Ule0Pd-WganeadVraVcdTqtlLnWSt5TsbN8PU,6365
+castor_extractor-0.3.5.dist-info/RECORD,,