castor-extractor 0.23.2__tar.gz → 0.24.0__tar.gz

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.24.0 - 2025-03-10
+
+* Remove legacy Tableau Connector
+
+## 0.23.3 - 2025-02-19
+
+* Snowflake : add --insecure-mode option to turn off OCSP checking
+
 ## 0.23.2 - 2025-02-17
 
 * support page_size in Tableau extraction command

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.23.2
+Version: 0.24.0
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -100,14 +100,14 @@ It also embeds utilities to help you push your metadata to Castor:
 
 ## Installation
 
-Requirements: **python3.8+**
+Requirements: **python3.9+**
 <img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />
 
 ### Create castor-env
 
 We advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).
 
-Here's an example using `Pyenv` and Python `3.8.12`:
+Here's an example using `Pyenv`:
 
 - Install Pyenv
 
@@ -124,22 +124,24 @@ eval "$(pyenv init --path)"
 eval "$(pyenv virtualenv-init -)"
 ```
 
-- [optional] Install python 3.8+
+- [optional] Install python 3.9+ if needed
 
 ```bash
 pyenv versions # check your local python installations
 
-pyenv install -v 3.8.12 # if none of the installed versions satisfy requirements 8+
+# install Python if none of the installed versions satisfy requirements 3.9+
+# in this example, we will install Python 3.11.9
+pyenv install -v 3.11.9
 ```
 
 - Create your virtual env
 
 ```bash
-pyenv virtualenv 3.8.12 castor-env # create a dedicated env
+pyenv virtualenv 3.11.9 castor-env # create a dedicated env
 pyenv shell castor-env # activate the environment
 
 # optional checks
-python --version # should be `3.8.12`
+python --version # should be `3.11.9`
 pyenv version # should be `castor-env`
 ```
 
@@ -206,6 +208,14 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.24.0 - 2025-03-10
+
+* Remove legacy Tableau Connector
+
+## 0.23.3 - 2025-02-19
+
+* Snowflake : add --insecure-mode option to turn off OCSP checking
+
 ## 0.23.2 - 2025-02-17
 
 * support page_size in Tableau extraction command

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/README.md

@@ -34,14 +34,14 @@ It also embeds utilities to help you push your metadata to Castor:
 
 ## Installation
 
-Requirements: **python3.8+**
+Requirements: **python3.9+**
 <img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />
 
 ### Create castor-env
 
 We advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).
 
-Here's an example using `Pyenv` and Python `3.8.12`:
+Here's an example using `Pyenv`:
 
 - Install Pyenv
 
@@ -58,22 +58,24 @@ eval "$(pyenv init --path)"
 eval "$(pyenv virtualenv-init -)"
 ```
 
-- [optional] Install python 3.8+
+- [optional] Install python 3.9+ if needed
 
 ```bash
 pyenv versions # check your local python installations
 
-pyenv install -v 3.8.12 # if none of the installed versions satisfy requirements 8+
+# install Python if none of the installed versions satisfy requirements 3.9+
+# in this example, we will install Python 3.11.9
+pyenv install -v 3.11.9
 ```
 
 - Create your virtual env
 
 ```bash
-pyenv virtualenv 3.8.12 castor-env # create a dedicated env
+pyenv virtualenv 3.11.9 castor-env # create a dedicated env
 pyenv shell castor-env # activate the environment
 
 # optional checks
-python --version # should be `3.8.12`
+python --version # should be `3.11.9`
 pyenv version # should be `castor-env`
 ```
 

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/castor_extractor/commands/extract_snowflake.py

@@ -40,6 +40,11 @@ def main():
         action="store_true",
         help="Optional: will fetch transients tables if added",
     )
+    parser.add_argument(
+        "--insecure-mode",
+        action="store_true",
+        help="Optional: turns off OCSP checking",
+    )
 
     parser.add_argument("-o", "--output", help="Directory to write to")
 
@@ -65,4 +70,5 @@ def main():
         output_directory=args.output,
         skip_existing=args.skip_existing,
         fetch_transient=args.fetch_transient,
+        insecure_mode=args.insecure_mode,
     )

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/castor_extractor/knowledge/confluence/client/client.py

@@ -1,12 +1,9 @@
 from collections.abc import Iterator
 from functools import partial
-from http import HTTPStatus
-from typing import Optional
 
 from ....utils import (
     APIClient,
     BasicAuth,
-    RequestSafeMode,
     fetch_all_pages,
 )
 from ..assets import (
@@ -21,19 +18,11 @@ _HEADERS = {
     "Content-Type": "application/json",
 }
 
-_MAX_ERROR_IGNORED_COUNT = 10
-_IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY,)
-_SAFE_MODE = RequestSafeMode(
-    max_errors=_MAX_ERROR_IGNORED_COUNT,
-    status_codes=_IGNORED_ERROR_CODES,
-)
-
 
 class ConfluenceClient(APIClient):
     def __init__(
         self,
         credentials: ConfluenceCredentials,
-        safe_mode: Optional[RequestSafeMode] = None,
     ):
         self.account_id = credentials.account_id
         auth = BasicAuth(
@@ -43,14 +32,12 @@ class ConfluenceClient(APIClient):
             auth=auth,
             host=credentials.base_url,
             headers=_HEADERS,
-            safe_mode=safe_mode or _SAFE_MODE,
         )
 
     def pages(self):
         request = partial(
             self._get,
             endpoint=ConfluenceEndpointFactory.pages(),
-            params={"body-format": "atlas_doc_format"},
         )
         yield from fetch_all_pages(request, ConfluencePagination)
 

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/castor_extractor/knowledge/confluence/client/endpoints.py

@@ -14,7 +14,7 @@ class ConfluenceEndpointFactory:
         Endpoint to fetch all pages.
         More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-pages-get
         """
-        return f"{cls.API}{cls.PAGES}"
+        return f"{cls.API}{cls.PAGES}?body-format=atlas_doc_format"
 
     @classmethod
     def users(cls) -> str:

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/castor_extractor/warehouse/snowflake/client.py

@@ -112,6 +112,7 @@ class SnowflakeClient(SqlalchemyClient):
         role: Optional[str] = None,
     ):
         self.private_key = _private_key(credentials.get("private_key"))
+        self.insecure_mode = credentials.get("insecure_mode", False)
         super().__init__(credentials)
         self._role = role
         self._warehouse = warehouse
@@ -121,13 +122,22 @@ class SnowflakeClient(SqlalchemyClient):
         return "Snowflake"
 
     def _engine_options(self, credentials: dict) -> dict:
-        if not self.private_key:
-            return {}
-        return {
-            "connect_args": {
-                "private_key": self.private_key,
-            }
-        }
+        """
+        Sets the engine options.
+        * `insecure_mode` = True turns off OCSP checking.
+            See https://community.snowflake.com/s/article/How-to-turn-off-OCSP-checking-in-Snowflake-client-drivers
+        * `private_key` = Snowflake private key for key pair authentication.
+            See https://docs.snowflake.com/en/user-guide/key-pair-auth
+        """
+        connection_args: dict = dict()
+
+        if self.insecure_mode:
+            connection_args["insecure_mode"] = True
+
+        if self.private_key:
+            connection_args["private_key"] = self.private_key
+
+        return {"connect_args": connection_args}
 
     def _build_uri(self, credentials: dict) -> str:
         if self.private_key:

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/castor_extractor/warehouse/snowflake/credentials.py

@@ -17,6 +17,7 @@ class SnowflakeCredentials:
     account: str
     user: str
 
+    insecure_mode: Optional[bool] = field(default=False)
     password: Optional[str] = field(metadata={"sensitive": True}, default=None)
     private_key: Optional[str] = field(
         metadata={"sensitive": True}, default=None

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/castor_extractor/warehouse/snowflake/extract.py

@@ -38,6 +38,7 @@ SNOWFLAKE_ACCOUNT = "CASTOR_SNOWFLAKE_ACCOUNT"
 SNOWFLAKE_USER = "CASTOR_SNOWFLAKE_USER"
 SNOWFLAKE_PASSWORD = "CASTOR_SNOWFLAKE_PASSWORD"  # noqa: S105
 SNOWFLAKE_PRIVATE_KEY = "CASTOR_SNOWFLAKE_PRIVATE_KEY"
+SNOWFLAKE_INSECURE_MODE = "CASTOR_SNOWFLAKE_INSECURE_MODE"
 
 
 def _credentials(params: dict) -> dict:
@@ -46,9 +47,15 @@ def _credentials(params: dict) -> dict:
     private_key = params.get("private_key") or from_env(
         SNOWFLAKE_PRIVATE_KEY, True
     )
+    insecure_mode = (
+        params.get("insecure_mode")
+        or from_env(SNOWFLAKE_INSECURE_MODE, allow_missing=True)
+        or False
+    )
     common = {
         "account": params.get("account") or from_env(SNOWFLAKE_ACCOUNT),
         "user": params.get("user") or from_env(SNOWFLAKE_USER),
+        "insecure_mode": insecure_mode,
     }
     if password:
         return {**common, "password": password}

{castor_extractor-0.23.2 → castor_extractor-0.24.0}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.23.2"
+version = "0.24.0"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.23.2/castor_extractor/visualization/tableau/__init__.py

@@ -1,3 +0,0 @@
-from .assets import TableauAsset
-from .client import ApiClient
-from .extract import extract_all, iterate_all_data

castor_extractor-0.23.2/castor_extractor/visualization/tableau/assets.py

@@ -1,49 +0,0 @@
-from enum import Enum
-
-from ...types import ExternalAsset, classproperty
-
-
-class TableauAsset(ExternalAsset):
-    """
-    Tableau assets
-    """
-
-    CUSTOM_SQL_TABLE = "custom_sql_tables"
-    CUSTOM_SQL_QUERY = "custom_sql_queries"
-    DASHBOARD = "dashboards"
-    DASHBOARD_SHEET = "dashboards_sheets"
-    DATASOURCE = "datasources"
-    FIELD = "fields"
-    PROJECT = "projects"
-    PUBLISHED_DATASOURCE = "published_datasources"
-    SHEET = "sheets"
-    USAGE = "views"
-    USER = "users"
-    WORKBOOK = "workbooks"
-    WORKBOOK_TO_DATASOURCE = "workbooks_to_datasource"
-
-    @classproperty
-    def optional(cls) -> set["TableauAsset"]:
-        return {
-            TableauAsset.DASHBOARD,
-            TableauAsset.DASHBOARD_SHEET,
-            TableauAsset.FIELD,
-            TableauAsset.SHEET,
-            TableauAsset.PUBLISHED_DATASOURCE,
-        }
-
-
-class TableauGraphqlAsset(Enum):
-    """
-    Assets which can be fetched from Tableau
-    """
-
-    BIN_FIELD = "binFields"
-    CALCULATED_FIELD = "calculatedFields"
-    COLUMN_FIELD = "columnFields"
-    CUSTOM_SQL = "customSQLTables"
-    DASHBOARD = "dashboards"
-    DATASOURCE = "datasources"
-    GROUP_FIELD = "groupFields"
-    SHEETS = "sheets"
-    WORKBOOK_TO_DATASOURCE = "workbooks"

castor_extractor-0.23.2/castor_extractor/visualization/tableau/client/__init__.py

@@ -1,2 +0,0 @@
-from .client import ApiClient
-from .client_utils import get_paginated_objects

castor_extractor-0.23.2/castor_extractor/visualization/tableau/client/client.py

@@ -1,229 +0,0 @@
-import logging
-
-import tableauserverclient as TSC  # type: ignore
-
-from ....utils import SerializedAsset
-from ..assets import TableauAsset
-from ..constants import PAGE_SIZE, TABLEAU_SERVER_VERSION
-from ..usage import compute_usage_views
-from .client_utils import extract_asset, get_paginated_objects
-from .credentials import CredentialsApi, CredentialsKey, get_value
-from .project import compute_project_path
-from .safe_mode import safe_mode_fetch_usage
-
-logger = logging.getLogger(__name__)
-
-
-class ApiClient:
-    """
-    Connect to Tableau REST API and fetch main assets.
-    Superuser credentials are required.
-    https://tableau.github.io/server-client-python/docs/
-    """
-
-    def __init__(
-        self,
-        **kwargs,
-    ):
-        self._credentials = CredentialsApi(
-            user=get_value(CredentialsKey.TABLEAU_USER, kwargs, True),
-            password=get_value(CredentialsKey.TABLEAU_PASSWORD, kwargs, True),
-            token_name=get_value(
-                CredentialsKey.TABLEAU_TOKEN_NAME,
-                kwargs,
-                True,
-            ),
-            token=get_value(CredentialsKey.TABLEAU_TOKEN, kwargs, True),
-            server_url=get_value(CredentialsKey.TABLEAU_SERVER_URL, kwargs),
-            site_id=get_value(CredentialsKey.TABLEAU_SITE_ID, kwargs, True),
-        )
-        self._server = TSC.Server(self._credentials.server_url)
-        self._server.add_http_options({"verify": True})
-        self._page_size = PAGE_SIZE
-        self._server.version = TABLEAU_SERVER_VERSION
-        self._safe_mode = bool(kwargs.get("safe_mode"))
-        self.errors: list[str] = []
-
-    @staticmethod
-    def name() -> str:
-        return "Tableau/API"
-
-    def _user_password_login(self) -> None:
-        """Login into Tableau using user and password"""
-        self._server.auth.sign_in(
-            TSC.TableauAuth(
-                self._credentials.user,
-                self._credentials.password,
-                site_id=self._credentials.site_id,
-            ),
-        )
-
-    def _pat_login(self) -> None:
-        """Login into Tableau using personal authentication token"""
-        self._server.auth.sign_in(
-            TSC.PersonalAccessTokenAuth(
-                self._credentials.token_name,
-                self._credentials.token,
-                site_id=self._credentials.site_id,
-            ),
-        )
-
-    def login(self) -> None:
-        """Login into Tableau"""
-
-        if self._credentials.user and self._credentials.password:
-            logger.info("Logging in using user and password authentication")
-            return self._user_password_login()
-
-        if self._credentials.token_name and self._credentials.token:
-            logger.info("Logging in using token authentication")
-            return self._pat_login()
-
-        raise ValueError(
-            """Wrong authentication: you should provide either user and password
-             or personal access token""",
-        )
-
-    def base_url(self) -> str:
-        return self._credentials.server_url
-
-    def _fetch_users(self) -> SerializedAsset:
-        """Fetches list of User"""
-        return [
-            extract_asset(user, TableauAsset.USER)
-            for user in TSC.Pager(self._server.users)
-        ]
-
-    def _fetch_workbooks(self) -> SerializedAsset:
-        """Fetches list of Workbooks"""
-
-        return [
-            extract_asset(workbook, TableauAsset.WORKBOOK)
-            for workbook in TSC.Pager(self._server.workbooks)
-        ]
-
-    def _fetch_usages(self, safe_mode: bool) -> SerializedAsset:
-        """Fetches list of Usages"""
-        if not safe_mode:
-            usages = [
-                extract_asset(usage, TableauAsset.USAGE)
-                for usage in TSC.Pager(self._server.views, usage=True)
-            ]
-
-            return compute_usage_views(usages)
-
-        return safe_mode_fetch_usage(self)
-
-    def _fetch_projects(self) -> SerializedAsset:
-        """Fetches list of Projects"""
-        return compute_project_path(
-            [
-                extract_asset(project, TableauAsset.PROJECT)
-                for project in TSC.Pager(self._server.projects)
-            ],
-        )
-
-    def _fetch_workbooks_to_datasource(self) -> SerializedAsset:
-        """Fetches workbooks to datasource"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.WORKBOOK_TO_DATASOURCE,
-        )
-
-    def _fetch_published_datasources(self) -> SerializedAsset:
-        """Fetches list of published datasources"""
-
-        return [
-            extract_asset(datasource, TableauAsset.PUBLISHED_DATASOURCE)
-            for datasource in TSC.Pager(self._server.datasources)
-        ]
-
-    def _fetch_datasources(self) -> SerializedAsset:
-        """Fetches both embedded and published datasource"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.DATASOURCE,
-        )
-
-    def _fetch_fields(self) -> SerializedAsset:
-        """Fetches fields"""
-        return self._fetch_paginated_objects(
-            TableauAsset.FIELD,
-        )
-
-    def _fetch_custom_sql_queries(self) -> SerializedAsset:
-        """Fetches custom sql queries"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.CUSTOM_SQL_QUERY,
-        )
-
-    def _fetch_custom_sql_tables(self) -> SerializedAsset:
-        """Fetches custom sql tables"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.CUSTOM_SQL_TABLE,
-        )
-
-    def _fetch_dashboards(self) -> SerializedAsset:
-        """Fetches dashboards"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.DASHBOARD,
-        )
-
-    def _fetch_sheets(self) -> SerializedAsset:
-        """Fetches sheets"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.SHEET,
-        )
-
-    def _fetch_paginated_objects(self, asset: TableauAsset) -> SerializedAsset:
-        """Fetches paginated objects"""
-
-        return get_paginated_objects(self._server, asset, self._page_size)
-
-    def fetch(self, asset: TableauAsset) -> SerializedAsset:
-        """Fetches the given asset"""
-        logger.info(f"Fetching {asset.name}")
-
-        if asset == TableauAsset.CUSTOM_SQL_QUERY:
-            assets = self._fetch_custom_sql_queries()
-
-        if asset == TableauAsset.CUSTOM_SQL_TABLE:
-            assets = self._fetch_custom_sql_tables()
-
-        if asset == TableauAsset.DASHBOARD:
-            assets = self._fetch_dashboards()
-
-        if asset == TableauAsset.DATASOURCE:
-            assets = self._fetch_datasources()
-
-        if asset == TableauAsset.FIELD:
-            assets = self._fetch_fields()
-
-        if asset == TableauAsset.PROJECT:
-            assets = self._fetch_projects()
-
-        if asset == TableauAsset.PUBLISHED_DATASOURCE:
-            assets = self._fetch_published_datasources()
-
-        if asset == TableauAsset.SHEET:
-            assets = self._fetch_sheets()
-
-        if asset == TableauAsset.USAGE:
-            assets = self._fetch_usages(self._safe_mode)
-
-        if asset == TableauAsset.USER:
-            assets = self._fetch_users()
-
-        if asset == TableauAsset.WORKBOOK:
-            assets = self._fetch_workbooks()
-
-        if asset == TableauAsset.WORKBOOK_TO_DATASOURCE:
-            assets = self._fetch_workbooks_to_datasource()
-
-        logger.info(f"Fetched {asset.name} ({len(assets)} results)")
-
-        return assets

castor_extractor-0.23.2/castor_extractor/visualization/tableau/client/client_utils.py

@@ -1,75 +0,0 @@
-from collections.abc import Iterator
-from typing import Optional
-
-from ....utils import SerializedAsset
-from ..assets import TableauAsset
-from ..gql_fields import QUERY_FIELDS
-from ..tsc_fields import TSC_FIELDS
-
-QUERY_TEMPLATE = """
-{{
-  {object_type}Connection(first: {page_size}, after: AFTER_TOKEN_SIGNAL) {{
-    nodes {{ {query_fields}
-    }}
-    pageInfo {{
-      hasNextPage
-      endCursor
-    }}
-    totalCount
-  }}
-}}
-"""
-
-RESOURCE_TEMPLATE = "{resource}Connection"
-
-
-def get_paginated_objects(
-    server,
-    asset: TableauAsset,
-    page_size: int,
-) -> SerializedAsset:
-    assets: SerializedAsset = []
-    for query in QUERY_FIELDS[asset]:
-        fields = query["fields"].value
-        object_type = query["object_type"].value
-        query_formatted = QUERY_TEMPLATE.format(
-            object_type=object_type,
-            page_size=page_size,
-            query_fields=fields,
-        )
-        resource = RESOURCE_TEMPLATE.format(resource=object_type)
-        result_pages = query_scroll(server, query_formatted, resource)
-        queried_assets = [asset for page in result_pages for asset in page]
-        assets.extend(queried_assets)
-    return assets
-
-
-def query_scroll(
-    server,
-    query: str,
-    resource: str,
-) -> Iterator[SerializedAsset]:
-    """build a tableau query iterator handling pagination and cursor"""
-
-    def _call(cursor: Optional[str]) -> dict:
-        # If cursor is defined it must be quoted else use null token
-        token = "null" if cursor is None else f'"{cursor}"'
-        query_ = query.replace("AFTER_TOKEN_SIGNAL", token)
-
-        return server.metadata.query(query_)["data"][resource]
-
-    cursor = None
-    while True:
-        payload = _call(cursor)
-        yield payload["nodes"]
-
-        page_info = payload["pageInfo"]
-        if page_info["hasNextPage"]:
-            cursor = page_info["endCursor"]
-        else:
-            break
-
-
-def extract_asset(asset: dict, asset_type: TableauAsset) -> dict:
-    """Agnostic function extracting dedicated attributes with define asset"""
-    return {key: getattr(asset, key) for key in TSC_FIELDS[asset_type]}