castor-extractor 0.15.4__tar.gz → 0.16.1__tar.gz

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.16.1 - 2024-04-02
+
+* Systematically escape nul bytes on CSV write
+
+## 0.16.0 - 2024-03-26
+
+* Use pydantic v2
 
 ## 0.15.4 - 2024-03-25
 

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.5
 
-FROM --platform=linux/amd64 python:3.10-slim
+FROM --platform=linux/amd64 python:3.11-slim
 
 ARG EXTRA
 ENV EXTRA=${EXTRA}

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.15.4
+Version: 0.16.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: all
 Provides-Extra: bigquery
+Provides-Extra: dbt
 Provides-Extra: looker
 Provides-Extra: metabase
 Provides-Extra: mysql
@@ -38,7 +39,8 @@ Requires-Dist: looker-sdk (>=22.4.0,<=23.0.0) ; extra == "looker" or extra == "a
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0) ; extra == "metabase" or extra == "postgres" or extra == "redshift" or extra == "all"
 Requires-Dist: pycryptodome (>=3.0.0,<4.0.0) ; extra == "metabase" or extra == "all"
-Requires-Dist: pydantic (>=1.10,<2.0)
+Requires-Dist: pydantic (>=2.6,<3.0)
+Requires-Dist: pydantic-settings (>=2.2,<3.0)
 Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all"
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/castor_extractor/utils/dbt/client_test.py

@@ -89,7 +89,7 @@ def test_DbtClient_list_job_identifiers():
 
     with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
         mocked_call.return_value = jobs
-        credentials = DbtCredentials(token="some-token", job_id=1)
+        credentials = DbtCredentials(token="some-token", job_id="1")
         dbt_client = DbtClient(credentials=credentials)
 
         jobs_ids = dbt_client.list_job_identifiers()
@@ -103,7 +103,7 @@ def test_DbtClient_fetch_artifacts():
     url = "https://cloud.getdbt.com/api/v2/accounts/40/runs/{}/artifacts/{}"
 
     with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
-        credentials = DbtCredentials(token="some-token", job_id=1)
+        credentials = DbtCredentials(token="some-token", job_id="1")
         dbt_client = DbtClient(credentials=credentials)
 
         dbt_client.fetch_run_results(run_id)

castor_extractor-0.16.1/castor_extractor/utils/dbt/credentials.py

@@ -0,0 +1,15 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+DEFAULT_DBT_CLOUD_URL = "https://cloud.getdbt.com"
+
+
+class DbtCredentials(BaseSettings):
+    """dbt credentials: host has default value"""
+
+    host: str = Field(
+        default=DEFAULT_DBT_CLOUD_URL, validation_alias="CASTOR_DBT_HOST"
+    )
+    job_id: str = Field(..., validation_alias="CASTOR_DBT_JOB_ID")
+    token: str = Field(..., validation_alias="CASTOR_DBT_TOKEN")
+    model_config = SettingsConfigDict(extra="ignore", populate_by_name=True)

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/castor_extractor/utils/formatter.py

@@ -30,7 +30,14 @@ def _header(row: dict) -> Sequence[str]:
 
 
 def _scalar(value: Any) -> ScalarValue:
-    if isinstance(value, (int, float, str)):
+    if isinstance(value, str):
+        if "\x00" in value:  # infrequent error caused by bad encoding
+            value = remove_unsupported_byte(value)
+            logger.warning("Removed unsupported byte to write to csv")
+            return value
+
+        return value
+    if isinstance(value, (int, float)):
         return value
     if isinstance(value, (date, datetime)):
         return value.isoformat()
@@ -46,11 +53,11 @@ def _row(header: Sequence[str], row: dict) -> List[ScalarValue]:
     return [_scalar(row.get(h)) for h in header]
 
 
-def remove_unsupported_byte(row: List[ScalarValue]) -> List[ScalarValue]:
-    return [
-        re.sub("\x00", "", element) if isinstance(element, str) else element
-        for element in row
-    ]
+def remove_unsupported_byte(element: ScalarValue) -> ScalarValue:
+    if not isinstance(element, str):
+        return element
+
+    return re.sub("\x00", "", element)
 
 
 def to_string_array(arr_json: str) -> List[str]:
@@ -85,11 +92,7 @@ def to_csv(buffer: IO[str], data: Iterable[dict]) -> bool:
             header = _header(row)
             writer.writerow(header)
         converted = _row(header, row)
-        try:
-            writer.writerow(converted)
-        except csv.Error:  # infrequent error caused by bad encoding
-            writer.writerow(remove_unsupported_byte(converted))
-            logger.warning("Removed unsupported byte to write to csv")
+        writer.writerow(converted)
     return True
 
 

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/castor_extractor/utils/formatter_test.py

@@ -68,7 +68,10 @@ def test__json_formatter():
     _test(formatter)
 
 
-def test__remove_unsupported_byte():
-    row = [1, "foo", "bar\x00bie"]
-    cleaned = remove_unsupported_byte(row)
-    assert cleaned == [1, "foo", "barbie"]
+@pytest.mark.parametrize(
+    "element, expected_output",
+    [(1, 1), ("foo", "foo"), ("bar\x00bie", "barbie")],
+)
+def test__remove_unsupported_byte(element, expected_output):
+    cleaned = remove_unsupported_byte(element)
+    assert cleaned == expected_output

castor_extractor-0.16.1/castor_extractor/visualization/powerbi/client/credentials.py

@@ -0,0 +1,20 @@
+from dataclasses import field
+from typing import List, Optional
+
+from pydantic.dataclasses import dataclass
+
+from .constants import Urls
+
+
+@dataclass
+class Credentials:
+    """Class to handle PowerBI rest API permissions"""
+
+    client_id: str
+    tenant_id: str
+    secret: str = field(metadata={"sensitive": True})
+    scopes: Optional[List[str]] = None
+
+    def __post_init__(self):
+        if self.scopes is None:
+            self.scopes = [Urls.DEFAULT_SCOPE]

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/castor_extractor/visualization/powerbi/client/rest.py

@@ -273,6 +273,10 @@ class Client:
             self._wait_for_scan_result(scan_id)
             yield self._get_scan(scan_id)
 
+    def test_connection(self) -> None:
+        """Use credentials & verify requesting the API doesn't raise an error"""
+        self._header()
+
     def fetch(
         self,
         asset: PowerBiAsset,

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/castor_extractor/warehouse/databricks/extract.py

@@ -4,6 +4,7 @@ from typing import Dict, Optional
 from ...utils import AbstractStorage, LocalStorage, write_summary
 from ..abstract import (
     CATALOG_ASSETS,
+    EXTERNAL_LINEAGE_ASSETS,
     QUERIES_ASSETS,
     VIEWS_ASSETS,
     SupportedAssets,
@@ -20,6 +21,7 @@ DATABRICKS_ASSETS: SupportedAssets = {
     WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
     WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
     WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
+    WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
 }
 
 logger = logging.getLogger(__name__)

{castor_extractor-0.15.4 → castor_extractor-0.16.1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.15.4"
+version = "0.16.1"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -46,13 +46,15 @@ tableauserverclient = { version = "==0.17.0", optional = true }
 tqdm = ">=4.0.0, <5.0.0"
 typing-extensions = "^4"
 websocket-client = { version = "^1", optional = true }
-pydantic = "^1.10"
+pydantic = "^2.6"
+pydantic-settings = "^2.2"
 pymssql = { version = "^2.2.11", optional = true }
 pymysql = {extras = ["rsa"], version = "^1.1.0", optional = true}
 
 
 [tool.poetry.extras]
 bigquery = ["sqlalchemy-bigquery"]
+dbt = []
 looker = ["looker-sdk"]
 metabase = ["psycopg2-binary", "pycryptodome"]
 mysql = ["pymysql"]
@@ -82,12 +84,12 @@ all = [
 black = "^24"
 flakeheaven = "^3.3"
 isort = "^5"
-pytest = "^7"
+pytest = "^8"
 pytest-freezegun = "^0.4"
 requests-mock = "^1.11"
 
 [tool.poetry.group.dev.dependencies]
-deptry = "^0.12.0"
+deptry = "^0.15.0"
 
 [tool.poetry.scripts]
 castor-extract-bigquery = "castor_extractor.commands.extract_bigquery:main"

castor_extractor-0.15.4/castor_extractor/utils/dbt/credentials.py

@@ -1,16 +0,0 @@
-from pydantic import BaseSettings, Extra, Field
-
-_DEFAULT_CLOUD_URL = "https://cloud.getdbt.com"
-
-
-class DbtCredentials(BaseSettings):
-    """dbt credentials: host has default value"""
-
-    host: str = Field(default=_DEFAULT_CLOUD_URL, env="CASTOR_DBT_HOST")
-    job_id: str = Field(..., env="CASTOR_DBT_JOB_ID")
-    token: str = Field(..., env="CASTOR_DBT_TOKEN")
-
-    class Config:
-        """constructor settings: ignore extra kwargs provided"""
-
-        extra = Extra.ignore

castor_extractor-0.15.4/castor_extractor/visualization/powerbi/client/credentials.py

@@ -1,22 +0,0 @@
-from typing import List, Optional
-
-from .constants import Urls
-
-
-class Credentials:
-    """Class to handle PowerBI rest API permissions"""
-
-    def __init__(
-        self,
-        *,
-        tenant_id: str,
-        client_id: str,
-        secret: str,
-        scopes: Optional[List[str]] = None,
-    ):
-        if scopes is None:
-            scopes = [Urls.DEFAULT_SCOPE]
-        self.tenant_id = tenant_id
-        self.client_id = client_id
-        self.secret = secret
-        self.scopes = scopes