castor-extractor 0.15.4__tar.gz → 0.16.3__tar.gz

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## 0.16.3 - 2024-04-24
+
+* Databricks: Extract table owners
+
+## 0.16.2 - 2024-04-09
+
+* PowerBI: Extract pages from report
+
+## 0.16.1 - 2024-04-02
+
+* Systematically escape nul bytes on CSV write
+
+## 0.16.0 - 2024-03-26
+
+* Use pydantic v2
 
 ## 0.15.4 - 2024-03-25
 

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.5
 
-FROM --platform=linux/amd64 python:3.10-slim
+FROM --platform=linux/amd64 python:3.11-slim
 
 ARG EXTRA
 ENV EXTRA=${EXTRA}

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.15.4
+Version: 0.16.3
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: all
 Provides-Extra: bigquery
+Provides-Extra: dbt
 Provides-Extra: looker
 Provides-Extra: metabase
 Provides-Extra: mysql
@@ -26,10 +27,9 @@ Provides-Extra: redshift
 Provides-Extra: snowflake
 Provides-Extra: sqlserver
 Provides-Extra: tableau
-Requires-Dist: click (>=8.1,<8.2)
 Requires-Dist: cryptography (>=41.0.5) ; extra == "snowflake"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
-Requires-Dist: google-auth (>=1.6.3,<3.0.0)
+Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=2,<3)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
@@ -38,7 +38,8 @@ Requires-Dist: looker-sdk (>=22.4.0,<=23.0.0) ; extra == "looker" or extra == "a
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0) ; extra == "metabase" or extra == "postgres" or extra == "redshift" or extra == "all"
 Requires-Dist: pycryptodome (>=3.0.0,<4.0.0) ; extra == "metabase" or extra == "all"
-Requires-Dist: pydantic (>=1.10,<2.0)
+Requires-Dist: pydantic (>=2.6,<3.0)
+Requires-Dist: pydantic-settings (>=2.2,<3.0)
 Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all"
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/utils/dbt/client_test.py

@@ -89,7 +89,7 @@ def test_DbtClient_list_job_identifiers():
 
     with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
         mocked_call.return_value = jobs
-        credentials = DbtCredentials(token="some-token", job_id=1)
+        credentials = DbtCredentials(token="some-token", job_id="1")
         dbt_client = DbtClient(credentials=credentials)
 
         jobs_ids = dbt_client.list_job_identifiers()
@@ -103,7 +103,7 @@ def test_DbtClient_fetch_artifacts():
     url = "https://cloud.getdbt.com/api/v2/accounts/40/runs/{}/artifacts/{}"
 
     with patch(infer_path, return_value=40), patch(call_path) as mocked_call:
-        credentials = DbtCredentials(token="some-token", job_id=1)
+        credentials = DbtCredentials(token="some-token", job_id="1")
         dbt_client = DbtClient(credentials=credentials)
 
         dbt_client.fetch_run_results(run_id)

castor_extractor-0.16.3/castor_extractor/utils/dbt/credentials.py

@@ -0,0 +1,15 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+DEFAULT_DBT_CLOUD_URL = "https://cloud.getdbt.com"
+
+
+class DbtCredentials(BaseSettings):
+    """dbt credentials: host has default value"""
+
+    host: str = Field(
+        default=DEFAULT_DBT_CLOUD_URL, validation_alias="CASTOR_DBT_HOST"
+    )
+    job_id: str = Field(..., validation_alias="CASTOR_DBT_JOB_ID")
+    token: str = Field(..., validation_alias="CASTOR_DBT_TOKEN")
+    model_config = SettingsConfigDict(extra="ignore", populate_by_name=True)

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/utils/formatter.py

@@ -30,7 +30,14 @@ def _header(row: dict) -> Sequence[str]:
 
 
 def _scalar(value: Any) -> ScalarValue:
-    if isinstance(value, (int, float, str)):
+    if isinstance(value, str):
+        if "\x00" in value:  # infrequent error caused by bad encoding
+            value = remove_unsupported_byte(value)
+            logger.warning("Removed unsupported byte to write to csv")
+            return value
+
+        return value
+    if isinstance(value, (int, float)):
         return value
     if isinstance(value, (date, datetime)):
         return value.isoformat()
@@ -46,11 +53,11 @@ def _row(header: Sequence[str], row: dict) -> List[ScalarValue]:
     return [_scalar(row.get(h)) for h in header]
 
 
-def remove_unsupported_byte(row: List[ScalarValue]) -> List[ScalarValue]:
-    return [
-        re.sub("\x00", "", element) if isinstance(element, str) else element
-        for element in row
-    ]
+def remove_unsupported_byte(element: ScalarValue) -> ScalarValue:
+    if not isinstance(element, str):
+        return element
+
+    return re.sub("\x00", "", element)
 
 
 def to_string_array(arr_json: str) -> List[str]:
@@ -85,11 +92,7 @@ def to_csv(buffer: IO[str], data: Iterable[dict]) -> bool:
             header = _header(row)
             writer.writerow(header)
         converted = _row(header, row)
-        try:
-            writer.writerow(converted)
-        except csv.Error:  # infrequent error caused by bad encoding
-            writer.writerow(remove_unsupported_byte(converted))
-            logger.warning("Removed unsupported byte to write to csv")
+        writer.writerow(converted)
     return True
 
 

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/utils/formatter_test.py

@@ -68,7 +68,10 @@ def test__json_formatter():
     _test(formatter)
 
 
-def test__remove_unsupported_byte():
-    row = [1, "foo", "bar\x00bie"]
-    cleaned = remove_unsupported_byte(row)
-    assert cleaned == [1, "foo", "barbie"]
+@pytest.mark.parametrize(
+    "element, expected_output",
+    [(1, 1), ("foo", "foo"), ("bar\x00bie", "barbie")],
+)
+def test__remove_unsupported_byte(element, expected_output):
+    cleaned = remove_unsupported_byte(element)
+    assert cleaned == expected_output

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/visualization/domo/client/client.py

@@ -14,7 +14,7 @@ RawData = Iterator[dict]
 
 DOMO_PUBLIC_URL = "https://api.domo.com"
 FORMAT = "%Y-%m-%d %I:%M:%S %p"
-DEFAULT_TIMEOUT = 300
+DEFAULT_TIMEOUT = 500
 TOKEN_EXPIRATION_SECONDS = timedelta(seconds=3000)  # auth token lasts 1 hour
 
 IGNORED_ERROR_CODES = (

castor_extractor-0.16.3/castor_extractor/visualization/powerbi/client/credentials.py

@@ -0,0 +1,20 @@
+from dataclasses import field
+from typing import List, Optional
+
+from pydantic.dataclasses import dataclass
+
+from .constants import Urls
+
+
+@dataclass
+class Credentials:
+    """Class to handle PowerBI rest API permissions"""
+
+    client_id: str
+    tenant_id: str
+    secret: str = field(metadata={"sensitive": True})
+    scopes: Optional[List[str]] = None
+
+    def __post_init__(self):
+        if self.scopes is None:
+            self.scopes = [Urls.DEFAULT_SCOPE]

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/visualization/powerbi/client/rest.py

@@ -64,19 +64,15 @@ class Client:
 
     def __init__(self, credentials: Credentials):
         self.creds = credentials
-
-    def _access_token(self) -> dict:
         client_app = f"{Urls.CLIENT_APP_BASE}{self.creds.tenant_id}"
-        app = msal.ConfidentialClientApplication(
+        self.app = msal.ConfidentialClientApplication(
             client_id=self.creds.client_id,
             authority=client_app,
             client_credential=self.creds.secret,
         )
 
-        token = app.acquire_token_silent(self.creds.scopes, account=None)
-
-        if not token:
-            token = app.acquire_token_for_client(scopes=self.creds.scopes)
+    def _access_token(self) -> dict:
+        token = self.app.acquire_token_for_client(scopes=self.creds.scopes)
 
         if Keys.ACCESS_TOKEN not in token:
             raise ValueError(f"No access token in token response: {token}")
@@ -248,7 +244,17 @@ class Client:
         Returns a list of reports for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/reports-get-reports-as-admin
         """
-        return self._get(Urls.REPORTS)[Keys.VALUE]
+        reports = self._get(Urls.REPORTS)[Keys.VALUE]
+        for report in reports:
+            report_id = report.get("id")
+            try:
+                url = Urls.REPORTS + f"/{report_id}/pages"
+                pages = self._get(url)[Keys.VALUE]
+                report["pages"] = pages
+            except (requests.HTTPError, requests.exceptions.Timeout) as e:
+                logger.debug(e)
+                continue
+        return reports
 
     def _dashboards(self) -> List[Dict]:
         """
@@ -273,6 +279,10 @@ class Client:
             self._wait_for_scan_result(scan_id)
             yield self._get_scan(scan_id)
 
+    def test_connection(self) -> None:
+        """Use credentials & verify requesting the API doesn't raise an error"""
+        self._header()
+
     def fetch(
         self,
         asset: PowerBiAsset,

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/visualization/powerbi/client/rest_test.py

@@ -31,7 +31,6 @@ def test__access_token(mock_app):
     # init mocks
     valid_response = {"access_token": "mock_token"}
     returning_valid_token = Mock(return_value=valid_response)
-    mock_app.return_value.acquire_token_silent = Mock(return_value=None)
     mock_app.return_value.acquire_token_for_client = returning_valid_token
 
     # init client
@@ -40,30 +39,29 @@ def test__access_token(mock_app):
     # generated token
     assert client._access_token() == valid_response
 
-    # via silent endpoint token
-    mock_app.return_value.acquire_token_silent = returning_valid_token
-    mock_app.return_value.acquire_token_for_client = None
-    assert client._access_token() == valid_response
-
     # token missing in response
     invalid_response = {"not_access_token": "666"}
     returning_invalid_token = Mock(return_value=invalid_response)
-    mock_app.return_value.acquire_token_silent = returning_invalid_token
+    mock_app.return_value.acquire_token_for_client = returning_invalid_token
 
     with pytest.raises(ValueError):
         client._access_token()
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch.object(Client, "_access_token")
-def test__headers(mock_access_token):
+def test__headers(mock_access_token, mock_app):
+    mock_app.return_value = None
     client = _client()
     mock_access_token.return_value = {Keys.ACCESS_TOKEN: "666"}
     assert client._header() == {"Authorization": "Bearer 666"}
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch("requests.request")
 @patch.object(Client, "_access_token")
-def test__get(mocked_access_token, mocked_request):
+def test__get(mocked_access_token, mocked_request, mock_app):
+    mock_app.return_value = None
     client = _client()
     mocked_access_token.return_value = {Keys.ACCESS_TOKEN: "666"}
     fact = {"fact": "Approximately 24 cat skins can make a coat.", "length": 43}
@@ -81,9 +79,11 @@ def test__get(mocked_access_token, mocked_request):
         result = client._get("https/whatev.er")
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch("requests.request")
 @patch.object(Client, "_access_token")
-def test__workspace_ids(_, mocked_request):
+def test__workspace_ids(_, mocked_request, mock_app):
+    mock_app.return_value = None
     client = _client()
     mocked_request.return_value = Mock(
         json=lambda: [{"id": 1000}, {"id": 1001}, {"id": 1003}],
@@ -112,9 +112,11 @@ def test__workspace_ids(_, mocked_request):
     )
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch("requests.request")
 @patch.object(Client, "_access_token")
-def test__post_default(_, mocked_request):
+def test__post_default(_, mocked_request, mock_app):
+    mock_app.return_value = None
     client = _client()
     url = "https://estcequecestbientotleweekend.fr/"
     params = QueryParams.METADATA_SCAN
@@ -129,9 +131,11 @@ def test__post_default(_, mocked_request):
     )
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch("requests.request")
 @patch.object(Client, "_access_token")
-def test__post_with_processor(_, mocked_request):
+def test__post_with_processor(_, mocked_request, mock_app):
+    mock_app.return_value = None
     client = _client()
     url = "https://estcequecestbientotleweekend.fr/"
     params = QueryParams.METADATA_SCAN
@@ -146,9 +150,11 @@ def test__post_with_processor(_, mocked_request):
     assert result == 1000
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch("requests.request")
 @patch.object(Client, "_access_token")
-def test__datasets(_, mocked_request):
+def test__datasets(_, mocked_request, mock_app):
+    mock_app.return_value = None
     client = _client()
     mocked_request.return_value = Mock(
         json=lambda: {"value": [{"id": 1, "type": "dataset"}]},
@@ -164,27 +170,50 @@ def test__datasets(_, mocked_request):
     assert datasets == [{"id": 1, "type": "dataset"}]
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch("requests.request")
 @patch.object(Client, "_access_token")
-def test__reports(_, mocked_request):
+def test__reports(_, mocked_request, mock_app):
+    mock_app.return_value = None
     client = _client()
-    mocked_request.return_value = Mock(
-        json=lambda: {"value": [{"id": 1, "type": "report"}]},
-    )
+    page_url = f"{Urls.REPORTS}/1/pages"
+    calls = [
+        call(GET, Urls.REPORTS, data=None, headers=ANY, params=None),
+        call(
+            GET,
+            page_url,
+            data=None,
+            headers=ANY,
+            params=None,
+        ),
+    ]
+    mocked_request.side_effect = [
+        Mock(json=lambda: {"value": [{"id": 1, "type": "report"}]}),
+        Mock(
+            json=lambda: {
+                "value": [
+                    {"name": "page_name", "displayName": "page", "order": 0}
+                ]
+            }
+        ),
+    ]
     reports = client._reports()
-    mocked_request.assert_called_with(
-        GET,
-        Urls.REPORTS,
-        data=None,
-        headers=ANY,
-        params=None,
-    )
-    assert reports == [{"id": 1, "type": "report"}]
+    mocked_request.assert_has_calls(calls)
+
+    assert reports == [
+        {
+            "id": 1,
+            "type": "report",
+            "pages": [{"name": "page_name", "displayName": "page", "order": 0}],
+        }
+    ]
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch("requests.request")
 @patch.object(Client, "_access_token")
-def test__dashboards(_, mocked_request):
+def test__dashboards(_, mocked_request, mock_app):
+    mock_app.return_value = None
     client = _client()
     mocked_request.return_value = Mock(
         json=lambda: {"value": [{"id": 1, "type": "dashboard"}]},
@@ -200,6 +229,7 @@ def test__dashboards(_, mocked_request):
     assert dashboards == [{"id": 1, "type": "dashboard"}]
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch.object(Client, "_workspace_ids")
 @patch.object(Client, "_create_scan")
 @patch.object(Client, "_wait_for_scan_result")
@@ -209,7 +239,9 @@ def test__metadata(
     mocked_wait,
     mocked_create_scan,
     mocked_workspace_ids,
+    mock_app,
 ):
+    mock_app.return_value = None
     mocked_workspace_ids.return_value = list(range(200))
     mocked_create_scan.return_value = 314
     mocked_wait.return_value = True
@@ -240,8 +272,10 @@ _CALLS = [
 ]
 
 
+@patch.object(msal, "ConfidentialClientApplication")
 @patch.object(Client, "_call")
-def test__activity_events(mocked):
+def test__activity_events(mocked, mock_app):
+    mock_app.return_value = None
     client = _client()
     mocked.side_effect = _CALLS
 

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/visualization/tableau/assets.py

@@ -12,10 +12,12 @@ class TableauAsset(ExternalAsset):
     CUSTOM_SQL_TABLE = "custom_sql_tables"
     CUSTOM_SQL_QUERY = "custom_sql_queries"
     DASHBOARD = "dashboards"
+    DASHBOARD_SHEET = "dashboards_sheets"
     DATASOURCE = "datasources"
     FIELD = "fields"
     PROJECT = "projects"
     PUBLISHED_DATASOURCE = "published_datasources"
+    SHEET = "sheets"
     USAGE = "views"
     USER = "users"
     WORKBOOK = "workbooks"
@@ -25,7 +27,9 @@ class TableauAsset(ExternalAsset):
     def optional(cls) -> Set["TableauAsset"]:
         return {
             TableauAsset.DASHBOARD,
+            TableauAsset.DASHBOARD_SHEET,
             TableauAsset.FIELD,
+            TableauAsset.SHEET,
             TableauAsset.PUBLISHED_DATASOURCE,
         }
 
@@ -42,4 +46,5 @@ class TableauGraphqlAsset(Enum):
     DASHBOARD = "dashboards"
     DATASOURCE = "datasources"
     GROUP_FIELD = "groupFields"
+    SHEETS = "sheets"
     WORKBOOK_TO_DATASOURCE = "workbooks"

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/visualization/tableau/client/client.py

@@ -173,6 +173,13 @@ class ApiClient:
             TableauAsset.DASHBOARD,
         )
 
+    def _fetch_sheets(self) -> SerializedAsset:
+        """Fetches sheets"""
+
+        return self._fetch_paginated_objects(
+            TableauAsset.SHEET,
+        )
+
     def _fetch_paginated_objects(self, asset: TableauAsset) -> SerializedAsset:
         """Fetches paginated objects"""
 
@@ -203,6 +210,9 @@ class ApiClient:
         if asset == TableauAsset.PUBLISHED_DATASOURCE:
             assets = self._fetch_published_datasources()
 
+        if asset == TableauAsset.SHEET:
+            assets = self._fetch_sheets()
+
         if asset == TableauAsset.USAGE:
             assets = self._fetch_usages(self._safe_mode)
 

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/visualization/tableau/gql_fields.py

@@ -111,15 +111,15 @@ class GQLQueryFields(Enum):
     """
 
     DASHBOARDS: str = """
-    id
-    name
-    path
-    tags {
-        name
-    }
-    workbook {
-       luid # to retrieve the parent
-    }
+            id
+            name
+            path
+            tags {
+                name
+            }
+            workbook {
+               luid # to retrieve the parent
+            }
     """
 
     DATASOURCE: str = """
@@ -160,6 +160,21 @@ class GQLQueryFields(Enum):
            role
     """
 
+    SHEET: str = """
+            containedInDashboards {
+                id
+            }
+            id
+            index
+            name
+            upstreamFields{
+                name
+            }
+            workbook {
+                luid
+            }
+    """
+
     WORKBOOK_TO_DATASOURCE: str = """
            luid
            id
@@ -219,6 +234,12 @@ QUERY_FIELDS: Dict[TableauAsset, QueryInfo] = {
             OBJECT_TYPE: TableauGraphqlAsset.GROUP_FIELD,
         },
     ],
+    TableauAsset.SHEET: [
+        {
+            FIELDS: GQLQueryFields.SHEET,
+            OBJECT_TYPE: TableauGraphqlAsset.SHEETS,
+        },
+    ],
     TableauAsset.WORKBOOK_TO_DATASOURCE: [
         {
             FIELDS: GQLQueryFields.WORKBOOK_TO_DATASOURCE,

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/warehouse/databricks/client.py

@@ -87,15 +87,32 @@ class DatabricksClient(APIClient):
             content.get("tables", []), schema
         )
 
-    def tables_and_columns(self, schemas: List[dict]) -> TablesColumns:
+    @staticmethod
+    def _match_table_with_user(table: dict, user_id_by_email: dict) -> dict:
+        table_owner_email = table.get("owner_email")
+        if not table_owner_email:
+            return table
+        owner_external_id = user_id_by_email.get(table_owner_email)
+        if not owner_external_id:
+            return table
+        return {**table, "owner_external_id": owner_external_id}
+
+    def tables_and_columns(
+        self, schemas: List[dict], users: List[dict]
+    ) -> TablesColumns:
         """
         Get the databricks tables & columns leveraging the unity catalog API
         """
         tables: List[dict] = []
         columns: List[dict] = []
+        user_id_by_email = {user.get("email"): user.get("id") for user in users}
         for schema in schemas:
             t_to_add, c_to_add = self._tables_columns_of_schema(schema)
-            tables.extend(t_to_add)
+            t_with_owner = [
+                self._match_table_with_user(table, user_id_by_email)
+                for table in t_to_add
+            ]
+            tables.extend(t_with_owner)
             columns.extend(c_to_add)
         return tables, columns
 

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/warehouse/databricks/client_test.py

@@ -64,3 +64,17 @@ def test_DatabricksClient__keep_catalog():
     assert client._keep_catalog("staging")
     assert not client._keep_catalog("dev")
     assert not client._keep_catalog("something_unknown")
+
+
+def test_DatabricksClient__match_table_with_user():
+    client = MockDatabricksClient()
+    users_by_email = {"bob@castordoc.com": 3}
+
+    table = {"id": 1, "owner_email": "bob@castordoc.com"}
+    table_with_owner = client._match_table_with_user(table, users_by_email)
+
+    assert table_with_owner == {**table, "owner_external_id": 3}
+
+    table_without_owner = {"id": 1, "owner_email": None}
+    actual = client._match_table_with_user(table_without_owner, users_by_email)
+    assert actual == table_without_owner

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/warehouse/databricks/extract.py

@@ -4,6 +4,7 @@ from typing import Dict, Optional
 from ...utils import AbstractStorage, LocalStorage, write_summary
 from ..abstract import (
     CATALOG_ASSETS,
+    EXTERNAL_LINEAGE_ASSETS,
     QUERIES_ASSETS,
     VIEWS_ASSETS,
     SupportedAssets,
@@ -20,6 +21,7 @@ DATABRICKS_ASSETS: SupportedAssets = {
     WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
     WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
     WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
+    WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
 }
 
 logger = logging.getLogger(__name__)
@@ -80,7 +82,8 @@ class DatabricksExtractionProcessor:
 
         del databases
 
-        tables, columns = self._client.tables_and_columns(schemas)
+        users = self._client.users()
+        tables, columns = self._client.tables_and_columns(schemas, users)
 
         location = self._storage.put(WarehouseAsset.TABLE.value, tables)
         catalog_locations[WarehouseAsset.TABLE.value] = location

{castor_extractor-0.15.4 → castor_extractor-0.16.3}/castor_extractor/warehouse/databricks/format.py

@@ -19,10 +19,11 @@ def _to_datetime_or_none(time_ms: Optional[int]) -> Optional[datetime]:
 
 def _table_payload(schema: dict, table: dict) -> dict:
     return {
+        "description": table.get("comment"),
         "id": table["table_id"],
+        "owner_email": table.get("owner"),
         "schema_id": f"{schema['id']}",
         "table_name": table["name"],
-        "description": table.get("comment"),
         "tags": [],
         "type": table.get("table_type"),
     }
@@ -30,12 +31,12 @@ def _table_payload(schema: dict, table: dict) -> dict:
 
 def _column_payload(table: dict, column: dict) -> dict:
     return {
-        "id": f"`{table['id']}`.`{column['name']}`",
         "column_name": column["name"],
-        "table_id": table["id"],
-        "description": column.get("comment"),
         "data_type": column["type_name"],
+        "description": column.get("comment"),
+        "id": f"`{table['id']}`.`{column['name']}`",
         "ordinal_position": column["position"],
+        "table_id": table["id"],
     }