castor-extractor 0.15.4__py3-none-any.whl → 0.16.1__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.16.1 - 2024-04-02
+
+* Systematically escape nul bytes on CSV write
+
+## 0.16.0 - 2024-03-26
+
+* Use pydantic v2
 
 ## 0.15.4 - 2024-03-25
 

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.5
 
-FROM --platform=linux/amd64 python:3.10-slim
+FROM --platform=linux/amd64 python:3.11-slim
 
 ARG EXTRA
 ENV EXTRA=${EXTRA}

castor_extractor/utils/dbt/client_test.py

@@ -89,7 +89,7 @@ def test_DbtClient_list_job_identifiers():
 
     with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
         mocked_call.return_value = jobs
-        credentials = DbtCredentials(token="some-token", job_id=1)
+        credentials = DbtCredentials(token="some-token", job_id="1")
         dbt_client = DbtClient(credentials=credentials)
 
         jobs_ids = dbt_client.list_job_identifiers()
@@ -103,7 +103,7 @@ def test_DbtClient_fetch_artifacts():
     url = "https://cloud.getdbt.com/api/v2/accounts/40/runs/{}/artifacts/{}"
 
     with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
-        credentials = DbtCredentials(token="some-token", job_id=1)
+        credentials = DbtCredentials(token="some-token", job_id="1")
         dbt_client = DbtClient(credentials=credentials)
 
         dbt_client.fetch_run_results(run_id)

castor_extractor/utils/dbt/credentials.py

@@ -1,16 +1,15 @@
-from pydantic import BaseSettings, Extra, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
-_DEFAULT_CLOUD_URL = "https://cloud.getdbt.com"
+DEFAULT_DBT_CLOUD_URL = "https://cloud.getdbt.com"
 
 
 class DbtCredentials(BaseSettings):
     """dbt credentials: host has default value"""
 
-    host: str = Field(default=_DEFAULT_CLOUD_URL, env="CASTOR_DBT_HOST")
-    job_id: str = Field(..., env="CASTOR_DBT_JOB_ID")
-    token: str = Field(..., env="CASTOR_DBT_TOKEN")
-
-    class Config:
-        """constructor settings: ignore extra kwargs provided"""
-
-        extra = Extra.ignore
+    host: str = Field(
+        default=DEFAULT_DBT_CLOUD_URL, validation_alias="CASTOR_DBT_HOST"
+    )
+    job_id: str = Field(..., validation_alias="CASTOR_DBT_JOB_ID")
+    token: str = Field(..., validation_alias="CASTOR_DBT_TOKEN")
+    model_config = SettingsConfigDict(extra="ignore", populate_by_name=True)

castor_extractor/utils/formatter.py

@@ -30,7 +30,14 @@ def _header(row: dict) -> Sequence[str]:
 
 
 def _scalar(value: Any) -> ScalarValue:
-    if isinstance(value, (int, float, str)):
+    if isinstance(value, str):
+        if "\x00" in value:  # infrequent error caused by bad encoding
+            value = remove_unsupported_byte(value)
+            logger.warning("Removed unsupported byte to write to csv")
+            return value
+
+        return value
+    if isinstance(value, (int, float)):
         return value
     if isinstance(value, (date, datetime)):
         return value.isoformat()
@@ -46,11 +53,11 @@ def _row(header: Sequence[str], row: dict) -> List[ScalarValue]:
     return [_scalar(row.get(h)) for h in header]
 
 
-def remove_unsupported_byte(row: List[ScalarValue]) -> List[ScalarValue]:
-    return [
-        re.sub("\x00", "", element) if isinstance(element, str) else element
-        for element in row
-    ]
+def remove_unsupported_byte(element: ScalarValue) -> ScalarValue:
+    if not isinstance(element, str):
+        return element
+
+    return re.sub("\x00", "", element)
 
 
 def to_string_array(arr_json: str) -> List[str]:
@@ -85,11 +92,7 @@ def to_csv(buffer: IO[str], data: Iterable[dict]) -> bool:
             header = _header(row)
             writer.writerow(header)
         converted = _row(header, row)
-        try:
-            writer.writerow(converted)
-        except csv.Error:  # infrequent error caused by bad encoding
-            writer.writerow(remove_unsupported_byte(converted))
-            logger.warning("Removed unsupported byte to write to csv")
+        writer.writerow(converted)
     return True
 
 

castor_extractor/utils/formatter_test.py

@@ -68,7 +68,10 @@ def test__json_formatter():
     _test(formatter)
 
 
-def test__remove_unsupported_byte():
-    row = [1, "foo", "bar\x00bie"]
-    cleaned = remove_unsupported_byte(row)
-    assert cleaned == [1, "foo", "barbie"]
+@pytest.mark.parametrize(
+    "element, expected_output",
+    [(1, 1), ("foo", "foo"), ("bar\x00bie", "barbie")],
+)
+def test__remove_unsupported_byte(element, expected_output):
+    cleaned = remove_unsupported_byte(element)
+    assert cleaned == expected_output

castor_extractor/visualization/powerbi/client/credentials.py

@@ -1,22 +1,20 @@
+from dataclasses import field
 from typing import List, Optional
 
+from pydantic.dataclasses import dataclass
+
 from .constants import Urls
 
 
+@dataclass
 class Credentials:
     """Class to handle PowerBI rest API permissions"""
 
-    def __init__(
-        self,
-        *,
-        tenant_id: str,
-        client_id: str,
-        secret: str,
-        scopes: Optional[List[str]] = None,
-    ):
-        if scopes is None:
-            scopes = [Urls.DEFAULT_SCOPE]
-        self.tenant_id = tenant_id
-        self.client_id = client_id
-        self.secret = secret
-        self.scopes = scopes
+    client_id: str
+    tenant_id: str
+    secret: str = field(metadata={"sensitive": True})
+    scopes: Optional[List[str]] = None
+
+    def __post_init__(self):
+        if self.scopes is None:
+            self.scopes = [Urls.DEFAULT_SCOPE]

castor_extractor/visualization/powerbi/client/rest.py

@@ -273,6 +273,10 @@ class Client:
             self._wait_for_scan_result(scan_id)
             yield self._get_scan(scan_id)
 
+    def test_connection(self) -> None:
+        """Use credentials & verify requesting the API doesn't raise an error"""
+        self._header()
+
     def fetch(
         self,
         asset: PowerBiAsset,

castor_extractor/warehouse/databricks/extract.py

@@ -4,6 +4,7 @@ from typing import Dict, Optional
 from ...utils import AbstractStorage, LocalStorage, write_summary
 from ..abstract import (
     CATALOG_ASSETS,
+    EXTERNAL_LINEAGE_ASSETS,
     QUERIES_ASSETS,
     VIEWS_ASSETS,
     SupportedAssets,
@@ -20,6 +21,7 @@ DATABRICKS_ASSETS: SupportedAssets = {
     WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
     WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
     WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
+    WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
 }
 
 logger = logging.getLogger(__name__)

{castor_extractor-0.15.4.dist-info → castor_extractor-0.16.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.15.4
+Version: 0.16.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: all
 Provides-Extra: bigquery
+Provides-Extra: dbt
 Provides-Extra: looker
 Provides-Extra: metabase
 Provides-Extra: mysql
@@ -38,7 +39,8 @@ Requires-Dist: looker-sdk (>=22.4.0,<=23.0.0) ; extra == "looker" or extra == "a
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0) ; extra == "metabase" or extra == "postgres" or extra == "redshift" or extra == "all"
 Requires-Dist: pycryptodome (>=3.0.0,<4.0.0) ; extra == "metabase" or extra == "all"
-Requires-Dist: pydantic (>=1.10,<2.0)
+Requires-Dist: pydantic (>=2.6,<3.0)
+Requires-Dist: pydantic-settings (>=2.2,<3.0)
 Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all"
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)

{castor_extractor-0.15.4.dist-info → castor_extractor-0.16.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
-CHANGELOG.md,sha256=8mmULip1bn-hnzxQ7MGWp8MXCp-O2RlUrtt1tg-qefg,9723
-Dockerfile,sha256=TC6hFjG3mvnt1nkw2EpaS42hRYaGA2YIPKgWhVSKTWc,303
+CHANGELOG.md,sha256=t1xfX_GaaTJcrNGAJtvhbOZ-4fAeRdFKRH8eKil6xWM,9837
+Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=uF6PXm9ocPITlKVSh9afTakHmpLx3TvawLf-CbMP3wM,3578
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -57,16 +57,16 @@ castor_extractor/utils/constants.py,sha256=qBQprS9U66mS-RIBXiLujdTSV3WvGv40Bc0kh
 castor_extractor/utils/dbt/__init__.py,sha256=LHQROlMqYWCc7tcmhdjXtROFpJqUvCg9jPC8avHgD4I,107
 castor_extractor/utils/dbt/assets.py,sha256=JY1nKEGySZ84wNoe7dnizwAYw2q0t8NVaIfqhB2rSw0,148
 castor_extractor/utils/dbt/client.py,sha256=xBjbT-p99TXY850ooEAgjNp33yfGDwjWJRbXzeJoVaI,5538
-castor_extractor/utils/dbt/client_test.py,sha256=h-IIrE2Fy0oD0uemjvJzof9xMO3AmkHciOVopKCNjJs,4529
-castor_extractor/utils/dbt/credentials.py,sha256=qkfs43_8pNqPGmKc7xv7sb1IhbheXzgwt6yYdjyomAo,479
+castor_extractor/utils/dbt/client_test.py,sha256=FO_vpnECE-hoK0rZHbqDv17oaJj3-uPhFEqTrMPzUf4,4533
+castor_extractor/utils/dbt/credentials.py,sha256=pGq7GqFQTw9TwN1DXSHC-0yJ2H6B_wMAbHyQTLqJVh0,543
 castor_extractor/utils/deprecate.py,sha256=_uzQiwHiz2yEqQeNMmzvVmBY46IgBhhEbGPhTrVjZU4,817
 castor_extractor/utils/env.py,sha256=TqdtB50U8LE0993WhhEhpy89TJrHbjtIKjvg6KQ-5q0,596
 castor_extractor/utils/files.py,sha256=3C_u7P-kSZoOABVaKsuaf8lEhldRRxyxD27-K18_dEU,1545
 castor_extractor/utils/files_test.py,sha256=omRT3XSjaSAywYUoLh1SGWqYzl4UwBYKSYA9_7mXd_E,1542
-castor_extractor/utils/formatter.py,sha256=v50OyVWPDn-QdBm-XxSNGQibPPePeJIrcDcK3TATLRM,4934
+castor_extractor/utils/formatter.py,sha256=EJpwl5ff9zahbWpCcoHcrDTVzBSqwyZQffuvh1JmPbk,4948
 castor_extractor/utils/formatter_test.csv,sha256=UCNqPs8-xrY1AdMSpuctVFXInQe3Z_EABP4rF-Jw5ks,3802
 castor_extractor/utils/formatter_test.json,sha256=yPP_z1ZEavaUskC-Hx33uGlwKoInHYOFKqsJ9NgwIFo,12527
-castor_extractor/utils/formatter_test.py,sha256=sFuiKmU5WYQN4ocg1tEKLN2bMpuuPWVbNqCn0x-jn8I,1752
+castor_extractor/utils/formatter_test.py,sha256=VPlRTPQOaAeCySNs1wU1jd3bMppqxkVpD1dyCLt6p94,1856
 castor_extractor/utils/json_stream_write.py,sha256=OUdg4-47I47pgbpN9_a6y-lmxuod7aY6PObxjvB-wXI,2082
 castor_extractor/utils/load.py,sha256=MXwGVB_Dp_VitGwo71sNB_xDmGzQ4oQ13MhaXXyYkS0,265
 castor_extractor/utils/object.py,sha256=xCcQtoj9313TCcoyRXkLpDcMxmDeQMFMseDNx95oGc0,1959
@@ -158,9 +158,9 @@ castor_extractor/visualization/powerbi/__init__.py,sha256=XSr_fNSsR-EPuGOFo7Ai1r
 castor_extractor/visualization/powerbi/assets.py,sha256=SASUjxtoOMag3NAlZfhpCy0sLap7WfENEMaEZuBrw6o,801
 castor_extractor/visualization/powerbi/client/__init__.py,sha256=hU8LE1gV9RttTGJiwVpEa9xDLR4IMkUdshQGthg4zzE,62
 castor_extractor/visualization/powerbi/client/constants.py,sha256=Cx4pbgyAFc7t_aRQyWj7q-qfkltJl-JgKdMzeKmC9AI,2356
-castor_extractor/visualization/powerbi/client/credentials.py,sha256=_YYuHWzL6QbXeGVVmEF8vSZKYnWaQk5Mx-7-13StUMo,500
+castor_extractor/visualization/powerbi/client/credentials.py,sha256=iiYaCa2FM1PBHv4YA0Z1LgdX9gnaQhvHGD0LQb7Tcxw,465
 castor_extractor/visualization/powerbi/client/credentials_test.py,sha256=23ZlLCvsPB_fmqntnzULkv0mMRE8NCzBXtWS6wupJn4,787
-castor_extractor/visualization/powerbi/client/rest.py,sha256=qhmA49tayNUzZsX53VsdqL7cOhWfbryVLwN5bH1zGfA,9500
+castor_extractor/visualization/powerbi/client/rest.py,sha256=0gwqqmmzX76MzNRGfmcNkXw_jxVRAdMTViQExTBQy2Y,9644
 castor_extractor/visualization/powerbi/client/rest_test.py,sha256=r5rS_1FMwHCDWbYdco11-zvDJ5jYk9l8-VVJcpCtbwM,7343
 castor_extractor/visualization/powerbi/client/utils.py,sha256=0RcoWcKOdvIGH4f3lYDvufmiMo4tr_ABFlITSrvXjTs,541
 castor_extractor/visualization/powerbi/client/utils_test.py,sha256=ULHL2JLrcv0xjW2r7QF_ce2OaGeeSzajkMDywJ8ZdVA,719
@@ -265,7 +265,7 @@ castor_extractor/warehouse/databricks/__init__.py,sha256=bTvDxjGQGM2J3hOnVhfNmFP
 castor_extractor/warehouse/databricks/client.py,sha256=iojSVTARx5JmGy2Tm8D2H5wHO5hqGigVG9Ql2vHNdz8,7375
 castor_extractor/warehouse/databricks/client_test.py,sha256=rsqHWmVOgvqQ3VmYKJrpWpcGATD_C9FD1sG4CJsin2E,2201
 castor_extractor/warehouse/databricks/credentials.py,sha256=sMpOAKhBklcmTpcr3mi3o8qLud__8PTZbQUT3K_TRY8,678
-castor_extractor/warehouse/databricks/extract.py,sha256=7Tyr20nVlbDex-IjDSdX5VHj4NWvc3sWyfyiKBqPDwI,5612
+castor_extractor/warehouse/databricks/extract.py,sha256=eyt9LihZ9GfHEh8Z2c9PXAHqK6hibPsEIUOKGYfMwg8,5708
 castor_extractor/warehouse/databricks/format.py,sha256=tCBCApW5iZMBx04p-oCUs36d4JqNqJsBDHe6f-A7eiU,4925
 castor_extractor/warehouse/databricks/format_test.py,sha256=iPmdJof43fBYL1Sa_fBrCWDQHCHgm7IWCZag1kWkj9E,1970
 castor_extractor/warehouse/databricks/types.py,sha256=T2SyLy9pY_olLtstdC77moPxIiikVsuQLMxh92YMJQo,78
@@ -346,7 +346,7 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.15.4.dist-info/METADATA,sha256=sLC8V6FB8j-hGW4iuODYF5W_jpgIkXpJBd2WHa5f-0k,6347
-castor_extractor-0.15.4.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-castor_extractor-0.15.4.dist-info/entry_points.txt,sha256=EQUCoNjSHevxmY5ZathX_fLZPcuBHng23rj0SSUrLtI,1345
-castor_extractor-0.15.4.dist-info/RECORD,,
+castor_extractor-0.16.1.dist-info/METADATA,sha256=tjFndsdmxa0NO7qy3Ddcz1pFZGnDsfPmTQ3x3o-UEeA,6412
+castor_extractor-0.16.1.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+castor_extractor-0.16.1.dist-info/entry_points.txt,sha256=EQUCoNjSHevxmY5ZathX_fLZPcuBHng23rj0SSUrLtI,1345
+castor_extractor-0.16.1.dist-info/RECORD,,