castor-extractor 0.3.4__tar.gz → 0.3.7__tar.gz

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.3.7 - 2023-04-28
+
+* Warning message to deprecate python < 3.8
+
+## 0.3.6 - 2023-04-24
+
+* Update enum keys for Metabase credentials
+
+## 0.3.5 - 2023-04-07
+
+* Extract metadata from successful dbt runs only
+
 ## 0.3.4 - 2023-04-05
 
 * Enhance uploader to support `QUALITY` files

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.3.4
+Version: 0.3.7
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

castor_extractor-0.3.7/castor_extractor/commands/__init__.py

@@ -0,0 +1,3 @@
+from castor_extractor.utils import deprecate_python  # type: ignore
+
+deprecate_python(min_version_supported=(3, 8))

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/commands/extract_metabase_api.py

@@ -18,7 +18,7 @@ def main():
     args = parser.parse_args()
 
     client = metabase.ApiClient(
-        base_url=args.base_url, username=args.username, password=args.password
+        base_url=args.base_url, user=args.username, password=args.password
     )
 
     metabase.extract_all(

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/commands/extract_metabase_db.py

@@ -31,7 +31,7 @@ def main():
         port=args.port,
         database=args.database,
         schema=args.schema,
-        username=args.username,
+        user=args.username,
         password=args.password,
         encryption_secret_key=args.encryption_secret_key,
     )

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/file_checker/file.py

@@ -125,7 +125,7 @@ class FileCheckerRun:
             return
         header = f"Issues detected on Row #{index+1}\n"
         for k, v in row.items():
-            header += "{:<20} {:<100}\n".format(str(k), str(v))
+            header += f"{str(k):<20} {str(v):<100}\n"
         self.logger.info(header + _SEPARATOR + issue_log + _SEPARATOR)
 
     def occurrences(self, name: str) -> Set[str]:

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/file_checker/file_test.py

@@ -13,8 +13,7 @@ _TEST_FILE_VALID = "file_test_users_valid.csv"
 def _content(path: str) -> Iterator[Dict]:
     absolute_path = os.path.join(os.path.dirname(__file__), path)
     with open(absolute_path) as csvfile:
-        for row in csv.DictReader(csvfile):
-            yield row
+        yield from csv.DictReader(csvfile)
 
 
 def _user_template() -> FileTemplate:

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/transformation/dbt/client/client.py

@@ -1,3 +1,4 @@
+import logging
 from enum import Enum
 from typing import Optional
 
@@ -5,9 +6,12 @@ import requests
 
 from .credentials import CredentialsKey, DbtCredentials, get_value
 
+logger = logging.getLogger(__name__)
+
 _CLOUD_URL = "https://cloud.getdbt.com/api/v2/accounts"
 _DATA_KEY = "data"
 _MANIFEST_PATH = "manifest.json"
+_SUCCESSFUL_RUN_STATUS = 10
 
 
 class ContentType(Enum):
@@ -47,20 +51,32 @@ class DbtClient:
             headers=headers,
             params=params,
         )
-        result = response.json()
-        if content_type == ContentType.JSON:
-            return result[_DATA_KEY]
-        return result
+        try:
+            result = response.json()
+        except:
+            context = f"{url}, status {response.status_code}"
+            raise ValueError(f"Couldn't extract data from {context}")
+        else:
+            if content_type == ContentType.JSON:
+                return result[_DATA_KEY]
+            return result
 
     def _infer_account_id(self) -> int:
         result = self._call(url=_CLOUD_URL)
         return result[0]["id"]
 
     def _last_run_id(self) -> int:
+        """
+        Extract last successful run id
+        https://docs.getdbt.com/dbt-cloud/api-v2#tag/Runs/operation/listRunsForAccount
+        """
+
         url = f"{_CLOUD_URL}/{self._account_id}/runs/"
+
         params = {
             "job_definition_id": self._credentials.job_id,
             "order_by": "-finished_at",
+            "status": _SUCCESSFUL_RUN_STATUS,
             "limit": 1,
         }
         return self._call(url, params)[0]["id"]
@@ -72,6 +88,8 @@ class DbtClient:
         """
         run_id = self._last_run_id()
         url = f"{_CLOUD_URL}/{self._account_id}/runs/{run_id}/artifacts/{_MANIFEST_PATH}"
+        logger.info(f"Extracting manifest from run id {run_id} with url {url}")
+
         # setting text content as a workaround to this issue
         # https://stackoverflow.com/questions/68201659/dbt-cloud-api-to-extract-run-artifacts
         return self._call(url, content_type=ContentType.TEXT)

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/uploader/upload_test.py

@@ -11,4 +11,4 @@ def test__path():
     file_path = "filename"
 
     path = _path(source_id, file_type, file_path)
-    assert path == "visualization-{}/{}".format(source_id, file_path)
+    assert path == f"visualization-{source_id}/{file_path}"

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/utils/__init__.py

@@ -1,5 +1,6 @@
 from .collection import group_by
 from .constants import OUTPUT_DIR
+from .deprecate import deprecate_python
 from .env import from_env
 from .files import explode, search_files
 from .formatter import to_string_array

castor_extractor-0.3.7/castor_extractor/utils/deprecate.py

@@ -0,0 +1,27 @@
+import logging
+import sys
+import warnings
+from typing import Tuple
+
+# DeprecationWarning is filtered by default
+warnings.filterwarnings("once", category=DeprecationWarning)
+
+
+def deprecate_python(min_version_supported: Tuple[int]):
+    """raises a warning if python version < min_version_supported"""
+
+    python_version = (
+        sys.version_info.major,
+        sys.version_info.minor,
+        sys.version_info.micro,
+    )
+
+    python_version_str = ".".join(map(str, python_version))
+    min_supported_str = ".".join(map(str, min_version_supported))
+
+    if python_version < min_version_supported:
+        warnings.warn(
+            f"You are using python version {python_version_str}, please upgrade to version {min_supported_str} or higher."
+            " Your version will be soon deprecated",
+            DeprecationWarning,
+        )

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/utils/formatter_test.py

@@ -37,7 +37,7 @@ SAMPLE_FILE = "formatter_test"
 def _test(formatter: Formatter) -> None:
     root = os.path.dirname(__file__)
     path = os.path.join(root, f"{SAMPLE_FILE}.{formatter.extension()}")
-    with open(path) as file:
+    with open(path, "r") as file:
         rows = list(formatter.deserialize(file))
         first = rows[0]
 

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/utils/store.py

@@ -67,5 +67,5 @@ class LocalStorage(AbstractStorage):
 
     def get(self, name: str) -> Iterator[dict]:
         path = self.path(name)
-        with open(path) as file:
+        with open(path, "r") as file:
             return self._formatter.deserialize(StringIO(file.read()))

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/utils/write.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import sys
 from datetime import datetime
 from typing import Any, List
 
@@ -51,6 +52,7 @@ def write_summary(output_directory: str, ts: int, **kwargs):
     write_json(
         summary,
         {
+            "python_version": sys.version,
             "version": _current_version(),
             "executed_at": datetime.fromtimestamp(ts).isoformat(),
             **kwargs,

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/visualization/metabase/client/api/client.py

@@ -33,7 +33,7 @@ class ApiClient:
     ):
         self._credentials = CredentialsApi(
             base_url=get_value(CredentialsApiKey.BASE_URL, kwargs),
-            username=get_value(CredentialsApiKey.USERNAME, kwargs),
+            user=get_value(CredentialsApiKey.USER, kwargs),
             password=get_value(CredentialsApiKey.PASSWORD, kwargs),
         )
         self._session = requests.Session()
@@ -91,7 +91,7 @@ class ApiClient:
     def _login(self) -> str:
         url = self._url("session")
         payload = {
-            "username": self._credentials.username,
+            "username": self._credentials.user,
             "password": self._credentials.password,
         }
         response = self._session.post(url, json=payload)

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/visualization/metabase/client/api/credentials.py

@@ -6,13 +6,13 @@ from .....utils import from_env
 
 class CredentialsApiKey(Enum):
     BASE_URL = "base_url"
-    USERNAME = "username"
+    USER = "user"
     PASSWORD = "password"
 
 
 CREDENTIALS_ENV: Dict[CredentialsApiKey, str] = {
     CredentialsApiKey.BASE_URL: "CASTOR_METABASE_API_BASE_URL",
-    CredentialsApiKey.USERNAME: "CASTOR_METABASE_API_USERNAME",
+    CredentialsApiKey.USER: "CASTOR_METABASE_API_USERNAME",
     CredentialsApiKey.PASSWORD: "CASTOR_METABASE_API_PASSWORD",
 }
 
@@ -30,13 +30,13 @@ def get_value(key: CredentialsApiKey, kwargs: dict) -> str:
 class CredentialsApi:
     """ValueObject for the credentials"""
 
-    def __init__(self, base_url: str, username: str, password: str):
+    def __init__(self, base_url: str, user: str, password: str):
         self.base_url = base_url
-        self.username = username
+        self.user = user
         self.password = password
 
     def to_dict(self, hide: bool = False) -> Dict[str, str]:
-        safe = (CredentialsApiKey.BASE_URL, CredentialsApiKey.USERNAME)
+        safe = (CredentialsApiKey.BASE_URL, CredentialsApiKey.USER)
         unsafe = (CredentialsApiKey.PASSWORD,)
 
         def val(k: CredentialsApiKey, v: str) -> str:

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/visualization/metabase/client/db/client.py

@@ -37,7 +37,7 @@ class DbClient:
             port=get_value(CredentialsDbKey.PORT, kwargs),
             database=get_value(CredentialsDbKey.DATABASE, kwargs),
             schema=get_value(CredentialsDbKey.SCHEMA, kwargs),
-            username=get_value(CredentialsDbKey.USERNAME, kwargs),
+            user=get_value(CredentialsDbKey.USER, kwargs),
             password=get_value(CredentialsDbKey.PASSWORD, kwargs),
         )
         self.encryption_secret_key = kwargs.get(

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/visualization/metabase/client/db/credentials.py

@@ -9,7 +9,7 @@ class CredentialsDbKey(Enum):
     PORT = "port"
     DATABASE = "database"
     SCHEMA = "schema"
-    USERNAME = "username"
+    USER = "user"
     PASSWORD = "password"
 
 
@@ -18,7 +18,7 @@ CREDENTIALS_ENV: Dict[CredentialsDbKey, str] = {
     CredentialsDbKey.PORT: "CASTOR_METABASE_DB_PORT",
     CredentialsDbKey.DATABASE: "CASTOR_METABASE_DB_DATABASE",
     CredentialsDbKey.SCHEMA: "CASTOR_METABASE_DB_SCHEMA",
-    CredentialsDbKey.USERNAME: "CASTOR_METABASE_DB_USERNAME",
+    CredentialsDbKey.USER: "CASTOR_METABASE_DB_USERNAME",
     CredentialsDbKey.PASSWORD: "CASTOR_METABASE_DB_PASSWORD",
 }
 
@@ -42,14 +42,14 @@ class CredentialsDb:
         port: str,
         database: str,
         schema: str,
-        username: str,
+        user: str,
         password: str,
     ):
         self.host = host
         self.port = port
         self.database = database
         self.schema = schema
-        self.username = username
+        self.user = user
         self.password = password
 
     def to_dict(self, hide: bool = False) -> Dict[str, str]:
@@ -58,7 +58,7 @@ class CredentialsDb:
             CredentialsDbKey.PORT,
             CredentialsDbKey.DATABASE,
             CredentialsDbKey.SCHEMA,
-            CredentialsDbKey.USERNAME,
+            CredentialsDbKey.USER,
         )
         unsafe = (CredentialsDbKey.PASSWORD,)
 

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/warehouse/bigquery/client.py

@@ -86,13 +86,13 @@ class BigQueryClient(SqlalchemyClient):
         """
         Returns distinct (project_id, region) available for the given GCP client
         """
-        return set(
+        return {
             (ds.project, ds._properties["location"])
             for ds in self._list_datasets()
-        )
+        }
 
     def get_datasets(self) -> Set[Tuple[str, str]]:
         """
         Returns distinct (project_id, dataset_id) available for the given GCP client
         """
-        return set((ds.project, ds.dataset_id) for ds in self._list_datasets())
+        return {(ds.project, ds.dataset_id) for ds in self._list_datasets()}

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/warehouse/bigquery/extract.py

@@ -38,7 +38,7 @@ def _credentials(params: dict) -> dict:
     """extract GCP credentials"""
     path = params.get("credentials") or from_env(BIGQUERY_CREDENTIALS)
     logger.info(f"Credentials fetched from {path}")
-    with open(path) as file:
+    with open(path, "r") as file:
         return cast(dict, json.load(file))
 
 

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/castor_extractor/warehouse/snowflake/client.py

@@ -34,7 +34,7 @@ def _use(connection: Connection, resource: UseResource, name: str):
     DON'T COPY THIS CODE FOR ANOTHER PURPOSE
     """
     target = resource.value
-    if not re.fullmatch("[\w_-]+", name):
+    if not re.fullmatch(r"[\w_-]+", name):
         raise NameError(f"{target} name invalid {name}")
     command = f'USE {target} "{name}"'
     logger.info(f"executing {command}")

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.3.4"
+version = "0.3.7"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

{castor_extractor-0.3.4 → castor_extractor-0.3.7}/setup.py

@@ -130,7 +130,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'castor-extractor',
-    'version': '0.3.4',
+    'version': '0.3.7',
     'description': 'Extract your metadata assets.',
     'long_description': '# Castor Extractor <img src="https://app.castordoc.com/images/castor_icon_dark.svg" width=30 />\n\nThis library contains utilities to extract your metadata assets into `JSON` or `CSV` files, on your local machine.\nAfter extraction, those files can be pushed to Castor for ingestion.\n\n- Visualization assets are typically:\n  - `dashboards`\n  - `users`\n  - `folders`\n  - ...\n\n- Warehouse assets are typically:\n  - `databases`\n  - `schemas`\n  - `tables`\n  - `columns`\n  - `queries`\n  - ...\n\nIt also embeds utilities to help you push your metadata to Castor:\n\n- `File Checker` to validate your [generic](https://docs.castordoc.com/integrations/data-warehouses/generic-warehouse) CSV files before pushing to Castor\n- `Uploader` to push extracted files to our Google-Cloud-Storage (GCS)\n\n# Table of contents\n\n- [Castor Extractor ](#castor-extractor-)\n- [Table of contents](#table-of-contents)\n- [Installation](#installation)\n  - [Create castor-env](#create-castor-env)\n  - [PIP install](#pip-install)\n  - [Create the output directory](#create-the-output-directory)\n- [Contact](#contact)\n\n# Installation\n\nRequirements: **python3.8+**\n<img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />\n\n## Create castor-env\n\nWe advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).\n\nHere\'s an example using `Pyenv` and Python `3.8.12`:\n\n- Install Pyenv\n\n```bash\nbrew install pyenv\nbrew install pyenv-virtualenv\n```\n\n- [optional] Update your `.bashrc` if you encounter this [issue](https://stackoverflow.com/questions/45577194/failed-to-activate-virtualenv-with-pyenv/45578839)\n\n```bash\neval "$(pyenv init -)"\neval "$(pyenv init --path)"\neval "$(pyenv virtualenv-init -)"\n```\n\n- [optional] Install python 3.8+\n\n```bash\npyenv versions # check your local python installations\n\npyenv install -v 3.8.12 # if none of the installed versions satisfy requirements 8+\n```\n\n- Create your virtual env\n\n```bash\npyenv virtualenv 3.8.12 castor-env # create a dedicated env\npyenv shell castor-env # activate the environment\n\n# optional checks\npython --version # should be `3.8.12`\npyenv version # should be `castor-env`\n```\n\n## PIP install\n\n⚠️ `castor-env` must be created AND activated first.\n\n```bash\npyenv shell castor-env\n(castor-env) $ # this means the environment is now active\n```\n\nℹ️ please upgrade `PIP` before installing Castor.\n\n```\npip install --upgrade pip\n```\n\nRun the following command to install `castor-extractor`:\n\n```\npip install castor-extractor\n```\n\nDepending on your use-case, you can also install one of the following `extras`:\n\n```\npip install castor-extractor[looker]\npip install castor-extractor[tableau]\npip install castor-extractor[metabase]\npip install castor-extractor[qlik]\npip install castor-extractor[bigquery]\npip install castor-extractor[redshift]\npip install castor-extractor[snowflake]\n```\n\n## Create the output directory\n\n```bash\nmkdir /tmp/castor\n```\n\nYou will provide this path in `extraction` scripts as following:\n\n```\ncastor-extract-bigquery --output=/tmp/castor\n```\n\nAlternatively, you can also set the following `ENV` in your `bashrc`:\n\n```bash\nexport CASTOR_OUTPUT_DIRECTORY="/tmp/castor"\n````\n\n# Contact\n\nFor any questions or bug report, contact us at [support@castordoc.com](mailto:support@castordoc.com)\n\n[Castor](https://castordoc.com) helps you find, understand, use your data assets\n',
     'author': 'Castor',

castor_extractor-0.3.4/castor_extractor/warehouse/__init__.py

@@ -1 +0,0 @@
-