castor-extractor 0.19.7__tar.gz → 0.20.0__tar.gz

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/CHANGELOG.md

@@ -1,6 +1,20 @@
 
 # Changelog
 
+## 0.20.0 - 2024-09-23
+
+* Switch to Tableau revamped connector
+
+## 0.19.9 - 2024-09-19
+
+* Databricks: multithreading to retrieve column lineage
+
+## 0.19.8 - 2024-09-18
+
+* Metabase: Handle duplicate dashboards
+* Snowflake: Exclude unnamed tables from extraction
+* Bump dependencies: cryptography, setuptools
+
 ## 0.19.7 - 2024-09-05
 
 * Metabase: Handle compatibility with older version

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.19.7
+Version: 0.20.0
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -29,7 +29,7 @@ Provides-Extra: redshift
 Provides-Extra: snowflake
 Provides-Extra: sqlserver
 Provides-Extra: tableau
-Requires-Dist: cryptography (>=42.0.0,<43.0.0) ; extra == "snowflake"
+Requires-Dist: cryptography (>=43.0.0,<44.0.0) ; extra == "snowflake"
 Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-auth (>=2,<3)
@@ -52,7 +52,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=72,<73)
+Requires-Dist: setuptools (>=74,<75)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
@@ -208,6 +208,20 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.20.0 - 2024-09-23
+
+* Switch to Tableau revamped connector
+
+## 0.19.9 - 2024-09-19
+
+* Databricks: multithreading to retrieve column lineage
+
+## 0.19.8 - 2024-09-18
+
+* Metabase: Handle duplicate dashboards
+* Snowflake: Exclude unnamed tables from extraction
+* Bump dependencies: cryptography, setuptools
+
 ## 0.19.7 - 2024-09-05
 
 * Metabase: Handle compatibility with older version

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/commands/extract_tableau.py

@@ -1,7 +1,8 @@
 import logging
 from argparse import ArgumentParser
 
-from castor_extractor.visualization import tableau  # type: ignore
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+from castor_extractor.visualization import tableau_revamp  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
 
@@ -19,29 +20,14 @@ def main():
 
     parser.add_argument("-b", "--server-url", help="Tableau server url")
     parser.add_argument("-i", "--site-id", help="Tableau site ID")
+
     parser.add_argument(
-        "-s",
-        "--safe-mode",
-        help="Tableau safe mode",
+        "--with-pulse",
+        dest="with_pulse",
         action="store_true",
+        help="Extract Tableau Pulse assets: Metrics and Subscriptions",
     )
-    parser.add_argument("-o", "--output", help="Directory to write to")
 
-    args = parser.parse_args()
-
-    client = tableau.ApiClient(
-        user=args.user,
-        password=args.password,
-        token_name=args.token_name,
-        token=args.token,
-        server_url=args.server_url,
-        site_id=args.site_id,
-        safe_mode=args.safe_mode,
-    )
-
-    client.login()
+    parser.add_argument("-o", "--output", help="Directory to write to")
 
-    tableau.extract_all(
-        client,
-        output_directory=args.output,
-    )
+    tableau_revamp.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/knowledge/notion/client/client.py

@@ -20,6 +20,8 @@ NOTION_BASE_HEADERS = {
     "User-Agent": CASTOR_NOTION_USER_AGENT,
 }
 
+NOTION_DEFAULT_TIMEOUT_S = 180
+
 
 def _search_filter(asset: str) -> Dict[str, Dict[str, str]]:
     return {"filter": {"value": asset, "property": "object"}}
@@ -45,6 +47,7 @@ class NotionClient(APIClient):
             auth=auth,
             headers=NOTION_BASE_HEADERS,
             safe_mode=safe_mode or NOTION_SAFE_MODE,
+            timeout=NOTION_DEFAULT_TIMEOUT_S,
         )
 
     def users(self) -> Iterator[dict]:

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/knowledge/notion/client/pagination.py

@@ -1,14 +1,16 @@
 from typing import Optional
 
+from pydantic import Field
+
 from ....utils import PaginationModel
 
 
 class NotionPagination(PaginationModel):
     """Class to handle paginated results"""
 
-    results: list
-    next_cursor: Optional[str]
-    has_more: bool
+    results: list = Field(default_factory=list)
+    next_cursor: Optional[str] = None
+    has_more: bool = False
 
     def is_last(self) -> bool:
         return not self.has_more

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/quality/soda/client/client.py

@@ -1,5 +1,5 @@
 from functools import partial
-from typing import Iterator
+from typing import Callable, Iterator
 
 from ....utils import (
     APIClient,
@@ -11,10 +11,10 @@ from .credentials import SodaCredentials
 from .endpoints import SodaEndpointFactory
 from .pagination import SodaCloudPagination
 
-_CLOUD_API = "https://cloud.soda.io/api/v1/"
 _REQUESTS_PER_MINUTE = 10
 _SECONDS_PER_MINUTE = 60
 _RATE_LIMIT_MS = (_SECONDS_PER_MINUTE // _REQUESTS_PER_MINUTE) + 1
+_CLOUD_PAGE_SIZE = 100
 
 HEADERS = {"Content-Type": "application/json"}
 
@@ -24,16 +24,26 @@ class SodaClient(APIClient):
         cloud_auth = BasicAuth(
             username=credentials.api_key, password=credentials.secret
         )
-        super().__init__(host=_CLOUD_API, auth=cloud_auth, headers=HEADERS)
+        super().__init__(
+            host=credentials.host, auth=cloud_auth, headers=HEADERS
+        )
+
+    @property
+    def _base_request(self) -> Callable:
+        return partial(self._get, params={"size": _CLOUD_PAGE_SIZE})
 
     def datasets(self) -> Iterator[dict]:
-        request = partial(self._get, endpoint=SodaEndpointFactory.datasets())
+        request = partial(
+            self._base_request, endpoint=SodaEndpointFactory.datasets()
+        )
         yield from fetch_all_pages(
             request, SodaCloudPagination, rate_limit=_RATE_LIMIT_MS
         )
 
     def checks(self) -> Iterator[dict]:
-        request = partial(self._get, endpoint=SodaEndpointFactory.checks())
+        request = partial(
+            self._base_request, endpoint=SodaEndpointFactory.checks()
+        )
         yield from fetch_all_pages(
             request, SodaCloudPagination, rate_limit=_RATE_LIMIT_MS
         )

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/quality/soda/client/credentials.py

@@ -3,6 +3,8 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 
 SODA_ENV_PREFIX = "CASTOR_SODA_"
 
+_CLOUD_API = "https://cloud.soda.io/api/v1/"
+
 
 class SodaCredentials(BaseSettings):
     """Class to handle Soda rest API permissions"""
@@ -15,3 +17,4 @@ class SodaCredentials(BaseSettings):
 
     api_key: str = Field(repr=False)
     secret: str = Field(repr=False)
+    host: str = Field(default=_CLOUD_API)

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/quality/soda/client/pagination.py

@@ -2,7 +2,6 @@ from typing import List
 
 from ....utils import PaginationModel
 
-_CLOUD_PAGE_SIZE = 100
 _CLOUD_FIRST_PAGE = 0
 
 
@@ -19,7 +18,7 @@ class SodaCloudPagination(PaginationModel):
             if self.current_page_payload
             else _CLOUD_FIRST_PAGE
         )
-        return {"page": current_page + 1, "size": _CLOUD_PAGE_SIZE}
+        return {"page": current_page + 1}
 
     def page_results(self) -> list:
         return self.content

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/utils/__init__.py

@@ -34,7 +34,7 @@ from .pager import (
     PagerOnIdLogger,
     PagerStopStrategy,
 )
-from .retry import RetryStrategy, retry
+from .retry import RetryStrategy, retry, retry_request
 from .safe import SafeMode, safe_mode
 from .store import AbstractStorage, LocalStorage
 from .string import decode_when_bytes, string_to_tuple

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/utils/client/api/client.py

@@ -26,18 +26,18 @@ def _generate_payloads(
     params: Optional[dict],
     data: Optional[dict],
     pagination_params: Optional[dict],
-) -> Tuple[dict, dict]:
+) -> Tuple[Optional[dict], Optional[dict]]:
     _pagination_params = pagination_params or {}
-    params = params or {}
-    data = data or {}
 
     if method == "GET":
+        params = params or {}
         params = {**params, **_pagination_params}
-    elif method == "POST":
+        return data, params
+    if method == "POST":
+        data = data or {}
         data = {**data, **_pagination_params}
-    else:
-        raise ValueError(f"Method {method} is not yet supported")
-    return data, params
+        return data, params
+    raise ValueError(f"Method {method} is not yet supported")
 
 
 class APIClient:

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/utils/client/api/pagination.py

@@ -67,7 +67,7 @@ def fetch_all_pages(
     response_payload = request()
     paginated_response = pagination_model(**response_payload)
     while not paginated_response.is_last():
-        logger.info(f"Fetching page number {page_number}")
+        logger.debug(f"Fetching page number {page_number}")
         yield from paginated_response.page_results()
         next_page_parameters = paginated_response.next_page_parameters()
         new_request = partial(request, **next_page_parameters)
@@ -79,5 +79,5 @@ def fetch_all_pages(
         page_number += 1
 
     # send last page's results
-    logger.info(f"Fetching page number {page_number}")
+    logger.debug(f"Fetching page number {page_number}")
     yield from paginated_response.page_results()

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/utils/write.py

@@ -35,7 +35,7 @@ def write_json(filename: str, data: Any):
     """
     with open(filename, "w", encoding=ENCODING) as f:
         json.dump(data, f)
-        logger.info(f"Wrote output file: {filename}")
+        logger.info(f"Wrote output file: {filename} ({f.tell()} bytes)")
 
 
 def _current_version() -> str:

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/visualization/domo/client/client.py

@@ -45,6 +45,8 @@ _RETRY_EXCEPTIONS = [
 _RETRY_COUNT = 2
 _RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
 
+_PARENT_FOLDER = "/Dashboards"
+
 logger = logging.getLogger(__name__)
 
 
@@ -177,7 +179,7 @@ class DomoClient:
     def _process_pages(
         self,
         page_tree: List[dict],
-        parent_path: str = "/root",
+        parent_path: str = _PARENT_FOLDER,
     ) -> Iterator[dict]:
         """Recursively fetch pages while building the folder architecture"""
         if not page_tree:

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/visualization/metabase/client/api/client.py

@@ -1,6 +1,6 @@
 import logging
 from http import HTTPStatus
-from typing import Any, Dict, Iterator, List, Optional, cast
+from typing import Any, Dict, Iterator, List, Optional, Set, cast
 
 import requests
 from requests import HTTPError
@@ -134,15 +134,22 @@ class ApiClient:
             collection = self._call(f"collection/{_id}/items?models=dashboard")
             if not collection:
                 continue
+
+            seen_dashboard_ids: Set[int] = set()
+
             for dashboard in cast(SerializedAsset, collection):
                 if dashboard.get("model") != "dashboard":
                     # This is to maintain compatibility with older versions
                     # where ?models=dashboard has no effects
                     continue
+
                 dashboard_id = dashboard.get("id")
                 if not dashboard_id:
                     continue
-                yield cast(Dict, self._call(f"dashboard/{dashboard_id}"))
+
+                if dashboard_id not in seen_dashboard_ids:
+                    seen_dashboard_ids.add(dashboard_id)
+                    yield cast(Dict, self._call(f"dashboard/{dashboard_id}"))
 
     @staticmethod
     def _collection_specifics(collections: SerializedAsset) -> SerializedAsset:

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/visualization/tableau_revamp/extract.py

@@ -11,7 +11,7 @@ from ...utils import (
     write_summary,
 )
 from .assets import TableauRevampAsset
-from .client import TableauRevampClient
+from .client import TableauRevampClient, TableauRevampCredentials
 
 logger = logging.getLogger(__name__)
 
@@ -26,16 +26,19 @@ def iterate_all_data(
         yield asset, deep_serialize(data)
 
 
-def extract_all(client: TableauRevampClient, **kwargs: str) -> None:
+def extract_all(**kwargs) -> None:
     """
-    Extract Data from tableau
-    Store data locally in files under the output_directory
-    If errors from Tableau's API are catch store them locally in file under the output_directory
+    Extract Data From tableau and store it locally in files under the
+    output_directory
     """
-    output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
-
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    with_pulse = kwargs.get("with_pulse") or False
     timestamp = current_timestamp()
 
+    credentials = TableauRevampCredentials(**kwargs)
+    client = TableauRevampClient(credentials, with_pulse=with_pulse)
+    client.login()
+
     for key, data in iterate_all_data(client):
         filename = get_output_filename(key.value, output_directory, timestamp)
         write_json(filename, data)

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/warehouse/databricks/api_client.py

@@ -1,5 +1,6 @@
 import logging
 from functools import partial
+from http import HTTPStatus
 from typing import Iterator, List, Optional, Set, Tuple
 
 import requests
@@ -12,6 +13,7 @@ from ...utils import (
     fetch_all_pages,
     handle_response,
     retry,
+    retry_request,
     safe_mode,
 )
 from ..abstract import TimeFilter
@@ -135,6 +137,10 @@ class DatabricksAPIClient(APIClient):
         max_retries=_RETRY_ATTEMPTS,
         base_ms=_RETRY_BASE_MS,
     )
+    @retry_request(
+        status_codes=(HTTPStatus.TOO_MANY_REQUESTS,),
+        max_retries=_RETRY_ATTEMPTS,
+    )
     def get_single_column_lineage(
         self,
         names: Tuple[str, str],

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/warehouse/databricks/client.py

@@ -15,7 +15,8 @@ from .types import TablesColumns, TimestampedLink
 
 logger = logging.getLogger(__name__)
 
-_MAX_THREADS = 10
+_THREADS_COLUMN_LINEAGE = 2
+_THREADS_TABLE_LINEAGE = 10
 
 
 class DatabricksClient:
@@ -99,7 +100,7 @@ class DatabricksClient:
         Wrapper function that retrieves all table lineage
         """
         # retrieve table lineage
-        with ThreadPoolExecutor(max_workers=_MAX_THREADS) as executor:
+        with ThreadPoolExecutor(max_workers=_THREADS_TABLE_LINEAGE) as executor:
             table_paths = [
                 ".".join([table["schema_id"], table["table_name"]])
                 for table in tables
@@ -121,10 +122,15 @@ class DatabricksClient:
         candidate_paths = paths_for_column_lineage(
             tables, columns, table_lineage
         )
+        # retrieve column lineage
+        with ThreadPoolExecutor(
+            max_workers=_THREADS_COLUMN_LINEAGE
+        ) as executor:
+            results = executor.map(
+                self.api_client.get_single_column_lineage, candidate_paths
+            )
         lineages: List[TimestampedLink] = [
-            link
-            for paths in candidate_paths
-            for link in self.api_client.get_single_column_lineage(paths)
+            link for links in results for link in links
         ]
         deduplicated = deduplicate_lineage(lineages)
         return self.formatter.format_lineage(deduplicated)

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/warehouse/snowflake/queries/table.sql

@@ -41,6 +41,8 @@ FROM snowflake.account_usage.tables AS t
     JOIN snowflake.account_usage.schemata AS s ON s.schema_id = t.table_schema_id
     JOIN tags_agg_tables ta ON t.table_id = ta.table_id
 WHERE TRUE
+    AND t.table_name IS NOT NULL
+    AND t.table_name != ''
     AND UPPER(t.table_catalog) NOT IN ('SNOWFLAKE', 'UTIL_DB')
     AND (
         t.deleted IS NULL

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/castor_extractor/warehouse/sqlserver/client.py

@@ -12,8 +12,8 @@ _KEYS = ("user", "password", "host", "port", "database")
 
 
 def _check_key(credentials: dict) -> None:
-    for key in credentials:
-        if key not in _KEYS:
+    for key in _KEYS:
+        if key not in credentials:
             raise KeyError(f"Missing {key} in credentials")
 
 

{castor_extractor-0.19.7 → castor_extractor-0.20.0}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.19.7"
+version = "0.20.0"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -27,7 +27,7 @@ include = [
 [tool.poetry.dependencies]
 python = ">=3.8, <3.13"
 
-cryptography = { version = "^42.0.0", optional = true }
+cryptography = { version = "^43.0.0", optional = true }
 google-api-core = ">=2.1.1, <3.0.0"
 google-auth = "^2"
 google-cloud-core = ">=2.1.0, <3.0.0"
@@ -51,7 +51,7 @@ python-dateutil = ">=2.0.0, <=3.0.0"
 requests = ">=2.0.0, <3.0.0"
 snowflake-connector-python = { version = "^3.4.0", optional = true }
 snowflake-sqlalchemy = { version = "!=1.2.5, <2.0.0", optional = true }
-setuptools = "^72"
+setuptools = "^74"
 sqlalchemy = "~1.4"
 sqlalchemy-bigquery = { version = ">=1.0.0, <= 2.0.0", optional = true, extras = [
   "bqstorage",
@@ -108,10 +108,10 @@ all = [
 pytest = "^8"
 pytest-freezegun = "^0.4"
 requests-mock = "^1.11"
-ruff = "^0.5.7"
+ruff = "^0.6.4"
 
 [tool.poetry.group.dev.dependencies]
-deptry = "^0.19"
+deptry = "^0.20"
 
 [tool.poetry.scripts]
 castor-extract-bigquery = "castor_extractor.commands.extract_bigquery:main"

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/__init__.py

@@ -1 +0,0 @@
-from .extract import SYNAPSE_ASSETS

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/extract.py

@@ -1,21 +0,0 @@
-import logging
-
-from ..abstract import (
-    CATALOG_ASSETS,
-    EXTERNAL_LINEAGE_ASSETS,
-    QUERIES_ASSETS,
-    VIEWS_ASSETS,
-    SupportedAssets,
-    WarehouseAsset,
-    WarehouseAssetGroup,
-)
-
-logger = logging.getLogger(__name__)
-
-SYNAPSE_ASSETS: SupportedAssets = {
-    WarehouseAssetGroup.CATALOG: CATALOG_ASSETS,
-    WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
-    WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
-    WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
-    WarehouseAssetGroup.EXTERNAL_LINEAGE: EXTERNAL_LINEAGE_ASSETS,
-}

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/.sqlfluff

@@ -1,2 +0,0 @@
-[sqlfluff]
-dialect = tsql

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/database.sql

@@ -1,6 +0,0 @@
-SELECT
-    d.name AS database_id,
-    d.name AS database_name,
-    NULL AS "comment"
-FROM sys.databases AS d
-    WHERE d.name != 'master'

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/query.sql

@@ -1,33 +0,0 @@
-WITH query_execution AS (
-    SELECT
-        plan_id,
-        CASE execution_type_desc
-            WHEN 'Aborted' THEN 1
-            WHEN 'Exception' THEN 1
-            ELSE 0
-        END AS aborted,
-        MIN(CAST(first_execution_time AS DATETIME)) AS start_time,
-        MAX(CAST(last_execution_time AS DATETIME)) AS end_time
-    FROM sys.query_store_runtime_stats
-    WHERE CAST(first_execution_time AS DATE) = CAST(DATEADD(day, -1, getdate()) AS DATE)
-    -- Filtering on hour will be needed when running that query out of notebooks
-    GROUP BY plan_id, execution_type_desc
-)
-
-SELECT
-    q.query_id,
-    qt.query_sql_text AS query_text,
-    DB_NAME() AS database_id,
-    DB_NAME() AS database_name,
-    'SYNAPSE' AS user_name,
-    qp.plan_id AS process_id,
-    qe.aborted AS aborted,
-    qe.start_time AS start_time,
-    qe.end_time AS end_time
-FROM sys.query_store_query AS q
-    JOIN sys.query_store_query_text AS qt
-        ON q.query_text_id = qt.query_text_id
-    JOIN sys.query_store_plan AS qp
-        ON q.query_id = qp.query_id
-    JOIN query_execution AS qe
-        ON qp.plan_id = qe.plan_id

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/schema.sql

@@ -1,7 +0,0 @@
-SELECT DISTINCT
-    s.catalog_name AS database_id,
-    s.catalog_name AS database_name,
-    s.schema_name AS schema_name,
-    s.schema_owner AS schema_owner,
-    s.catalog_name + '.' + s.schema_name AS schema_id
-FROM INFORMATION_SCHEMA.SCHEMATA AS s

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/table.sql

@@ -1,36 +0,0 @@
-WITH users AS (
-    SELECT DISTINCT
-        uid,
-        name
-    FROM sys.sysusers
-)
-
-SELECT
-    DB_NAME() as database_id,
-    DB_NAME() AS database_name,
-    DB_NAME() + '.' + s.name AS schema_id,
-    s.name AS schema_name,
-    DB_NAME() + '.' + s.name + '.' + o.name AS table_id,
-    o.name AS table_name,
-    CASE UPPER(REPLACE(o.type, ' ', ''))
-        WHEN 'U' THEN 'TABLE'
-        WHEN 'V' THEN 'VIEW'
-        ELSE 'UNKNOWN'
-    END AS table_type,
-    -- Normally schema owner is table owner, but we can
-    -- change table owner with ALTER AUTHORIZATION command
-    -- in this case we have to look up for the owner
-    -- in sys.objects table
-    COALESCE(su.name, pu.name) AS table_owner,
-    COALESCE(su.uid, pu.uid) AS table_owner_id,
-    NULL AS comment
-FROM sys.objects AS o
-    JOIN sys.schemas AS s
-        ON o.schema_id = s.schema_id
-    LEFT JOIN users AS pu
-        ON s.principal_id = pu.uid
-    LEFT JOIN users AS su
-        ON o.principal_id = su.uid
-WHERE
--- SYNAPSE NOTATION: U for TABLE, V for VIEW
-    o.type IN ('U', 'V')

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/user.sql

@@ -1,4 +0,0 @@
-SELECT
-    name AS user_name,
-    uid AS user_id
-FROM sys.sysusers

castor_extractor-0.19.7/castor_extractor/warehouse/synapse/queries/view_ddl.sql

@@ -1,8 +0,0 @@
-SELECT
-    DB_NAME() AS database_id,
-    DB_NAME() AS database_name,
-    DB_NAME() + '.' + table_schema AS schema_id,
-    table_schema AS schema_name,
-    table_name AS view_name,
-    view_definition AS view_definition
-FROM INFORMATION_SCHEMA.VIEWS