castor-extractor 0.3.4__py3-none-any.whl → 0.3.5__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.3.5 - 2023-04-07
+
+* Extract metadata from successful dbt runs only
+
 ## 0.3.4 - 2023-04-05
 
 * Enhance uploader to support `QUALITY` files

castor_extractor/transformation/dbt/client/client.py

@@ -1,3 +1,4 @@
+import logging
 from enum import Enum
 from typing import Optional
 
@@ -5,9 +6,12 @@ import requests
 
 from .credentials import CredentialsKey, DbtCredentials, get_value
 
+logger = logging.getLogger(__name__)
+
 _CLOUD_URL = "https://cloud.getdbt.com/api/v2/accounts"
 _DATA_KEY = "data"
 _MANIFEST_PATH = "manifest.json"
+_SUCCESSFUL_RUN_STATUS = 10
 
 
 class ContentType(Enum):
@@ -47,20 +51,32 @@ class DbtClient:
             headers=headers,
             params=params,
         )
-        result = response.json()
-        if content_type == ContentType.JSON:
-            return result[_DATA_KEY]
-        return result
+        try:
+            result = response.json()
+        except:
+            context = f"{url}, status {response.status_code}"
+            raise ValueError(f"Couldn't extract data from {context}")
+        else:
+            if content_type == ContentType.JSON:
+                return result[_DATA_KEY]
+            return result
 
     def _infer_account_id(self) -> int:
         result = self._call(url=_CLOUD_URL)
         return result[0]["id"]
 
     def _last_run_id(self) -> int:
+        """
+        Extract last successful run id
+        https://docs.getdbt.com/dbt-cloud/api-v2#tag/Runs/operation/listRunsForAccount
+        """
+
         url = f"{_CLOUD_URL}/{self._account_id}/runs/"
+
         params = {
             "job_definition_id": self._credentials.job_id,
             "order_by": "-finished_at",
+            "status": _SUCCESSFUL_RUN_STATUS,
             "limit": 1,
         }
         return self._call(url, params)[0]["id"]
@@ -72,6 +88,8 @@ class DbtClient:
         """
         run_id = self._last_run_id()
         url = f"{_CLOUD_URL}/{self._account_id}/runs/{run_id}/artifacts/{_MANIFEST_PATH}"
+        logger.info(f"Extracting manifest from run id {run_id} with url {url}")
+
         # setting text content as a workaround to this issue
         # https://stackoverflow.com/questions/68201659/dbt-cloud-api-to-extract-run-artifacts
         return self._call(url, content_type=ContentType.TEXT)

{castor_extractor-0.3.4.dist-info → castor_extractor-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.3.4
+Version: 0.3.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.3.4.dist-info → castor_extractor-0.3.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=ODuUyWYuKprHJfhaCgKYfd63mRmxTILFkWtYqwoznYw,4723
+CHANGELOG.md,sha256=DIidPXNKp9icL0ZTvm7mnNSGno2190ncEFar6BoR0Is,4796
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=_H5aVcA57LLStpQaYm0yYndJRuw7BqQKt46D1xogRJo,3390
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -32,7 +32,7 @@ castor_extractor/transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 castor_extractor/transformation/dbt/__init__.py,sha256=fynkZOuDwD4AFcAYFdUBUbBPC5HoaQjVu-Z2BMcSnoI,91
 castor_extractor/transformation/dbt/assets.py,sha256=TqjktG7NeK_FUNAoOQJTtgCGVh8Cm9cTnoozNhXErmM,94
 castor_extractor/transformation/dbt/client/__init__.py,sha256=-9TvcGyeANOnoqceq3MnRrM2i6YzK906SZqeJqfLOpg,86
-castor_extractor/transformation/dbt/client/client.py,sha256=k8sxyg34JbZHV05zjJvoZhDeJEY3FAXrHQdJQBd-txU,2387
+castor_extractor/transformation/dbt/client/client.py,sha256=8SdcB5edeRHUUSz7HuTq67B_6RLo9O8z5lIOjyOOrpg,2938
 castor_extractor/transformation/dbt/client/credentials.py,sha256=67mWV2AML8vt--o9-2bvBzlcigj-5ZdocUcUxdc7J30,746
 castor_extractor/uploader/__init__.py,sha256=SSRtwjg-dNoxME-RJy9G1flASiUKAC5bH1htq3CURQg,75
 castor_extractor/uploader/constant.py,sha256=hEJlWYx0dyBzgo59XUBKCYIKEODpIc2DyzwAZIiNO8g,718
@@ -259,7 +259,7 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.3.4.dist-info/entry_points.txt,sha256=kwmxu6k71-dgEuW0G5zD7LUcOsBhPCjp0cGtukkH3OA,915
-castor_extractor-0.3.4.dist-info/WHEEL,sha256=vxFmldFsRN_Hx10GDvsdv1wroKq8r5Lzvjp6GZ4OO8c,88
-castor_extractor-0.3.4.dist-info/METADATA,sha256=Hp3hAsuqbXFxUgGA6pyKVRnjDZD0ShZ859o1VVFYb6c,6365
-castor_extractor-0.3.4.dist-info/RECORD,,
+castor_extractor-0.3.5.dist-info/entry_points.txt,sha256=kwmxu6k71-dgEuW0G5zD7LUcOsBhPCjp0cGtukkH3OA,915
+castor_extractor-0.3.5.dist-info/WHEEL,sha256=vxFmldFsRN_Hx10GDvsdv1wroKq8r5Lzvjp6GZ4OO8c,88
+castor_extractor-0.3.5.dist-info/METADATA,sha256=UjUUD5Ule0Pd-WganeadVraVcdTqtlLnWSt5TsbN8PU,6365
+castor_extractor-0.3.5.dist-info/RECORD,,