castor-extractor 0.3.4__tar.gz → 0.3.5__tar.gz

{castor_extractor-0.3.4 → castor_extractor-0.3.5}/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.3.5 - 2023-04-07
+
+* Extract metadata from successful dbt runs only
+
 ## 0.3.4 - 2023-04-05
 
 * Enhance uploader to support `QUALITY` files

{castor_extractor-0.3.4 → castor_extractor-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.3.4
+Version: 0.3.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.3.4 → castor_extractor-0.3.5}/castor_extractor/transformation/dbt/client/client.py

@@ -1,3 +1,4 @@
+import logging
 from enum import Enum
 from typing import Optional
 
@@ -5,9 +6,12 @@ import requests
 
 from .credentials import CredentialsKey, DbtCredentials, get_value
 
+logger = logging.getLogger(__name__)
+
 _CLOUD_URL = "https://cloud.getdbt.com/api/v2/accounts"
 _DATA_KEY = "data"
 _MANIFEST_PATH = "manifest.json"
+_SUCCESSFUL_RUN_STATUS = 10
 
 
 class ContentType(Enum):
@@ -47,20 +51,32 @@ class DbtClient:
             headers=headers,
             params=params,
         )
-        result = response.json()
-        if content_type == ContentType.JSON:
-            return result[_DATA_KEY]
-        return result
+        try:
+            result = response.json()
+        except:
+            context = f"{url}, status {response.status_code}"
+            raise ValueError(f"Couldn't extract data from {context}")
+        else:
+            if content_type == ContentType.JSON:
+                return result[_DATA_KEY]
+            return result
 
     def _infer_account_id(self) -> int:
         result = self._call(url=_CLOUD_URL)
         return result[0]["id"]
 
     def _last_run_id(self) -> int:
+        """
+        Extract last successful run id
+        https://docs.getdbt.com/dbt-cloud/api-v2#tag/Runs/operation/listRunsForAccount
+        """
+
         url = f"{_CLOUD_URL}/{self._account_id}/runs/"
+
         params = {
             "job_definition_id": self._credentials.job_id,
             "order_by": "-finished_at",
+            "status": _SUCCESSFUL_RUN_STATUS,
             "limit": 1,
         }
         return self._call(url, params)[0]["id"]
@@ -72,6 +88,8 @@ class DbtClient:
         """
         run_id = self._last_run_id()
         url = f"{_CLOUD_URL}/{self._account_id}/runs/{run_id}/artifacts/{_MANIFEST_PATH}"
+        logger.info(f"Extracting manifest from run id {run_id} with url {url}")
+
         # setting text content as a workaround to this issue
         # https://stackoverflow.com/questions/68201659/dbt-cloud-api-to-extract-run-artifacts
         return self._call(url, content_type=ContentType.TEXT)

{castor_extractor-0.3.4 → castor_extractor-0.3.5}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.3.4"
+version = "0.3.5"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

{castor_extractor-0.3.4 → castor_extractor-0.3.5}/setup.py

@@ -130,7 +130,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'castor-extractor',
-    'version': '0.3.4',
+    'version': '0.3.5',
     'description': 'Extract your metadata assets.',
     'long_description': '# Castor Extractor <img src="https://app.castordoc.com/images/castor_icon_dark.svg" width=30 />\n\nThis library contains utilities to extract your metadata assets into `JSON` or `CSV` files, on your local machine.\nAfter extraction, those files can be pushed to Castor for ingestion.\n\n- Visualization assets are typically:\n  - `dashboards`\n  - `users`\n  - `folders`\n  - ...\n\n- Warehouse assets are typically:\n  - `databases`\n  - `schemas`\n  - `tables`\n  - `columns`\n  - `queries`\n  - ...\n\nIt also embeds utilities to help you push your metadata to Castor:\n\n- `File Checker` to validate your [generic](https://docs.castordoc.com/integrations/data-warehouses/generic-warehouse) CSV files before pushing to Castor\n- `Uploader` to push extracted files to our Google-Cloud-Storage (GCS)\n\n# Table of contents\n\n- [Castor Extractor ](#castor-extractor-)\n- [Table of contents](#table-of-contents)\n- [Installation](#installation)\n  - [Create castor-env](#create-castor-env)\n  - [PIP install](#pip-install)\n  - [Create the output directory](#create-the-output-directory)\n- [Contact](#contact)\n\n# Installation\n\nRequirements: **python3.8+**\n<img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />\n\n## Create castor-env\n\nWe advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).\n\nHere\'s an example using `Pyenv` and Python `3.8.12`:\n\n- Install Pyenv\n\n```bash\nbrew install pyenv\nbrew install pyenv-virtualenv\n```\n\n- [optional] Update your `.bashrc` if you encounter this [issue](https://stackoverflow.com/questions/45577194/failed-to-activate-virtualenv-with-pyenv/45578839)\n\n```bash\neval "$(pyenv init -)"\neval "$(pyenv init --path)"\neval "$(pyenv virtualenv-init -)"\n```\n\n- [optional] Install python 3.8+\n\n```bash\npyenv versions # check your local python installations\n\npyenv install -v 3.8.12 # if none of the installed versions satisfy requirements 8+\n```\n\n- Create your virtual env\n\n```bash\npyenv virtualenv 3.8.12 castor-env # create a dedicated env\npyenv shell castor-env # activate the environment\n\n# optional checks\npython --version # should be `3.8.12`\npyenv version # should be `castor-env`\n```\n\n## PIP install\n\n⚠️ `castor-env` must be created AND activated first.\n\n```bash\npyenv shell castor-env\n(castor-env) $ # this means the environment is now active\n```\n\nℹ️ please upgrade `PIP` before installing Castor.\n\n```\npip install --upgrade pip\n```\n\nRun the following command to install `castor-extractor`:\n\n```\npip install castor-extractor\n```\n\nDepending on your use-case, you can also install one of the following `extras`:\n\n```\npip install castor-extractor[looker]\npip install castor-extractor[tableau]\npip install castor-extractor[metabase]\npip install castor-extractor[qlik]\npip install castor-extractor[bigquery]\npip install castor-extractor[redshift]\npip install castor-extractor[snowflake]\n```\n\n## Create the output directory\n\n```bash\nmkdir /tmp/castor\n```\n\nYou will provide this path in `extraction` scripts as following:\n\n```\ncastor-extract-bigquery --output=/tmp/castor\n```\n\nAlternatively, you can also set the following `ENV` in your `bashrc`:\n\n```bash\nexport CASTOR_OUTPUT_DIRECTORY="/tmp/castor"\n````\n\n# Contact\n\nFor any questions or bug report, contact us at [support@castordoc.com](mailto:support@castordoc.com)\n\n[Castor](https://castordoc.com) helps you find, understand, use your data assets\n',
     'author': 'Castor',