castor-extractor 0.23.2__py3-none-any.whl → 0.24.0__py3-none-any.whl

castor_extractor/visualization/tableau/client/client.py

@@ -1,229 +0,0 @@
-import logging
-
-import tableauserverclient as TSC  # type: ignore
-
-from ....utils import SerializedAsset
-from ..assets import TableauAsset
-from ..constants import PAGE_SIZE, TABLEAU_SERVER_VERSION
-from ..usage import compute_usage_views
-from .client_utils import extract_asset, get_paginated_objects
-from .credentials import CredentialsApi, CredentialsKey, get_value
-from .project import compute_project_path
-from .safe_mode import safe_mode_fetch_usage
-
-logger = logging.getLogger(__name__)
-
-
-class ApiClient:
-    """
-    Connect to Tableau REST API and fetch main assets.
-    Superuser credentials are required.
-    https://tableau.github.io/server-client-python/docs/
-    """
-
-    def __init__(
-        self,
-        **kwargs,
-    ):
-        self._credentials = CredentialsApi(
-            user=get_value(CredentialsKey.TABLEAU_USER, kwargs, True),
-            password=get_value(CredentialsKey.TABLEAU_PASSWORD, kwargs, True),
-            token_name=get_value(
-                CredentialsKey.TABLEAU_TOKEN_NAME,
-                kwargs,
-                True,
-            ),
-            token=get_value(CredentialsKey.TABLEAU_TOKEN, kwargs, True),
-            server_url=get_value(CredentialsKey.TABLEAU_SERVER_URL, kwargs),
-            site_id=get_value(CredentialsKey.TABLEAU_SITE_ID, kwargs, True),
-        )
-        self._server = TSC.Server(self._credentials.server_url)
-        self._server.add_http_options({"verify": True})
-        self._page_size = PAGE_SIZE
-        self._server.version = TABLEAU_SERVER_VERSION
-        self._safe_mode = bool(kwargs.get("safe_mode"))
-        self.errors: list[str] = []
-
-    @staticmethod
-    def name() -> str:
-        return "Tableau/API"
-
-    def _user_password_login(self) -> None:
-        """Login into Tableau using user and password"""
-        self._server.auth.sign_in(
-            TSC.TableauAuth(
-                self._credentials.user,
-                self._credentials.password,
-                site_id=self._credentials.site_id,
-            ),
-        )
-
-    def _pat_login(self) -> None:
-        """Login into Tableau using personal authentication token"""
-        self._server.auth.sign_in(
-            TSC.PersonalAccessTokenAuth(
-                self._credentials.token_name,
-                self._credentials.token,
-                site_id=self._credentials.site_id,
-            ),
-        )
-
-    def login(self) -> None:
-        """Login into Tableau"""
-
-        if self._credentials.user and self._credentials.password:
-            logger.info("Logging in using user and password authentication")
-            return self._user_password_login()
-
-        if self._credentials.token_name and self._credentials.token:
-            logger.info("Logging in using token authentication")
-            return self._pat_login()
-
-        raise ValueError(
-            """Wrong authentication: you should provide either user and password
-             or personal access token""",
-        )
-
-    def base_url(self) -> str:
-        return self._credentials.server_url
-
-    def _fetch_users(self) -> SerializedAsset:
-        """Fetches list of User"""
-        return [
-            extract_asset(user, TableauAsset.USER)
-            for user in TSC.Pager(self._server.users)
-        ]
-
-    def _fetch_workbooks(self) -> SerializedAsset:
-        """Fetches list of Workbooks"""
-
-        return [
-            extract_asset(workbook, TableauAsset.WORKBOOK)
-            for workbook in TSC.Pager(self._server.workbooks)
-        ]
-
-    def _fetch_usages(self, safe_mode: bool) -> SerializedAsset:
-        """Fetches list of Usages"""
-        if not safe_mode:
-            usages = [
-                extract_asset(usage, TableauAsset.USAGE)
-                for usage in TSC.Pager(self._server.views, usage=True)
-            ]
-
-            return compute_usage_views(usages)
-
-        return safe_mode_fetch_usage(self)
-
-    def _fetch_projects(self) -> SerializedAsset:
-        """Fetches list of Projects"""
-        return compute_project_path(
-            [
-                extract_asset(project, TableauAsset.PROJECT)
-                for project in TSC.Pager(self._server.projects)
-            ],
-        )
-
-    def _fetch_workbooks_to_datasource(self) -> SerializedAsset:
-        """Fetches workbooks to datasource"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.WORKBOOK_TO_DATASOURCE,
-        )
-
-    def _fetch_published_datasources(self) -> SerializedAsset:
-        """Fetches list of published datasources"""
-
-        return [
-            extract_asset(datasource, TableauAsset.PUBLISHED_DATASOURCE)
-            for datasource in TSC.Pager(self._server.datasources)
-        ]
-
-    def _fetch_datasources(self) -> SerializedAsset:
-        """Fetches both embedded and published datasource"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.DATASOURCE,
-        )
-
-    def _fetch_fields(self) -> SerializedAsset:
-        """Fetches fields"""
-        return self._fetch_paginated_objects(
-            TableauAsset.FIELD,
-        )
-
-    def _fetch_custom_sql_queries(self) -> SerializedAsset:
-        """Fetches custom sql queries"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.CUSTOM_SQL_QUERY,
-        )
-
-    def _fetch_custom_sql_tables(self) -> SerializedAsset:
-        """Fetches custom sql tables"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.CUSTOM_SQL_TABLE,
-        )
-
-    def _fetch_dashboards(self) -> SerializedAsset:
-        """Fetches dashboards"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.DASHBOARD,
-        )
-
-    def _fetch_sheets(self) -> SerializedAsset:
-        """Fetches sheets"""
-
-        return self._fetch_paginated_objects(
-            TableauAsset.SHEET,
-        )
-
-    def _fetch_paginated_objects(self, asset: TableauAsset) -> SerializedAsset:
-        """Fetches paginated objects"""
-
-        return get_paginated_objects(self._server, asset, self._page_size)
-
-    def fetch(self, asset: TableauAsset) -> SerializedAsset:
-        """Fetches the given asset"""
-        logger.info(f"Fetching {asset.name}")
-
-        if asset == TableauAsset.CUSTOM_SQL_QUERY:
-            assets = self._fetch_custom_sql_queries()
-
-        if asset == TableauAsset.CUSTOM_SQL_TABLE:
-            assets = self._fetch_custom_sql_tables()
-
-        if asset == TableauAsset.DASHBOARD:
-            assets = self._fetch_dashboards()
-
-        if asset == TableauAsset.DATASOURCE:
-            assets = self._fetch_datasources()
-
-        if asset == TableauAsset.FIELD:
-            assets = self._fetch_fields()
-
-        if asset == TableauAsset.PROJECT:
-            assets = self._fetch_projects()
-
-        if asset == TableauAsset.PUBLISHED_DATASOURCE:
-            assets = self._fetch_published_datasources()
-
-        if asset == TableauAsset.SHEET:
-            assets = self._fetch_sheets()
-
-        if asset == TableauAsset.USAGE:
-            assets = self._fetch_usages(self._safe_mode)
-
-        if asset == TableauAsset.USER:
-            assets = self._fetch_users()
-
-        if asset == TableauAsset.WORKBOOK:
-            assets = self._fetch_workbooks()
-
-        if asset == TableauAsset.WORKBOOK_TO_DATASOURCE:
-            assets = self._fetch_workbooks_to_datasource()
-
-        logger.info(f"Fetched {asset.name} ({len(assets)} results)")
-
-        return assets

castor_extractor/visualization/tableau/client/client_utils.py

@@ -1,75 +0,0 @@
-from collections.abc import Iterator
-from typing import Optional
-
-from ....utils import SerializedAsset
-from ..assets import TableauAsset
-from ..gql_fields import QUERY_FIELDS
-from ..tsc_fields import TSC_FIELDS
-
-QUERY_TEMPLATE = """
-{{
-  {object_type}Connection(first: {page_size}, after: AFTER_TOKEN_SIGNAL) {{
-    nodes {{ {query_fields}
-    }}
-    pageInfo {{
-      hasNextPage
-      endCursor
-    }}
-    totalCount
-  }}
-}}
-"""
-
-RESOURCE_TEMPLATE = "{resource}Connection"
-
-
-def get_paginated_objects(
-    server,
-    asset: TableauAsset,
-    page_size: int,
-) -> SerializedAsset:
-    assets: SerializedAsset = []
-    for query in QUERY_FIELDS[asset]:
-        fields = query["fields"].value
-        object_type = query["object_type"].value
-        query_formatted = QUERY_TEMPLATE.format(
-            object_type=object_type,
-            page_size=page_size,
-            query_fields=fields,
-        )
-        resource = RESOURCE_TEMPLATE.format(resource=object_type)
-        result_pages = query_scroll(server, query_formatted, resource)
-        queried_assets = [asset for page in result_pages for asset in page]
-        assets.extend(queried_assets)
-    return assets
-
-
-def query_scroll(
-    server,
-    query: str,
-    resource: str,
-) -> Iterator[SerializedAsset]:
-    """build a tableau query iterator handling pagination and cursor"""
-
-    def _call(cursor: Optional[str]) -> dict:
-        # If cursor is defined it must be quoted else use null token
-        token = "null" if cursor is None else f'"{cursor}"'
-        query_ = query.replace("AFTER_TOKEN_SIGNAL", token)
-
-        return server.metadata.query(query_)["data"][resource]
-
-    cursor = None
-    while True:
-        payload = _call(cursor)
-        yield payload["nodes"]
-
-        page_info = payload["pageInfo"]
-        if page_info["hasNextPage"]:
-            cursor = page_info["endCursor"]
-        else:
-            break
-
-
-def extract_asset(asset: dict, asset_type: TableauAsset) -> dict:
-    """Agnostic function extracting dedicated attributes with define asset"""
-    return {key: getattr(asset, key) for key in TSC_FIELDS[asset_type]}

castor_extractor/visualization/tableau/client/credentials.py

@@ -1,104 +0,0 @@
-from enum import Enum
-from typing import Optional
-
-from ....utils import from_env
-
-AUTH_ERROR_MSG = "Need either user and password or token_name and token"
-
-# https://tableau.github.io/server-client-python/docs/api-ref#authentication
-DEFAULT_SERVER_SITE_ID = ""
-
-
-class CredentialsKey(Enum):
-    """Value enum object for the credentials"""
-
-    TABLEAU_USER = "user"
-    TABLEAU_PASSWORD = "password"  # noqa: S105
-    TABLEAU_TOKEN_NAME = "token_name"  # noqa: S105
-    TABLEAU_TOKEN = "token"  # noqa: S105
-    TABLEAU_SITE_ID = "site_id"
-    TABLEAU_SERVER_URL = "server_url"
-
-
-CREDENTIALS_ENV: dict[CredentialsKey, str] = {
-    CredentialsKey.TABLEAU_USER: "CASTOR_TABLEAU_USER",
-    CredentialsKey.TABLEAU_PASSWORD: "CASTOR_TABLEAU_PASSWORD",
-    CredentialsKey.TABLEAU_TOKEN_NAME: "CASTOR_TABLEAU_TOKEN_NAME",
-    CredentialsKey.TABLEAU_TOKEN: "CASTOR_TABLEAU_TOKEN",
-    CredentialsKey.TABLEAU_SITE_ID: "CASTOR_TABLEAU_SITE_ID",
-    CredentialsKey.TABLEAU_SERVER_URL: "CASTOR_TABLEAU_SERVER_URL",
-}
-
-
-def get_value(
-    key: CredentialsKey,
-    kwargs: dict,
-    optional: bool = False,
-) -> Optional[str]:
-    """
-    Returns the value of the given key:
-    - from kwargs in priority
-    - from ENV if not provided (raises an error if not found in ENV)
-    """
-    env_key = CREDENTIALS_ENV[key]
-
-    return kwargs.get(key.value) or from_env(env_key, optional)
-
-
-class CredentialsApi:
-    """ValueObject for the credentials"""
-
-    def __init__(
-        self,
-        *,
-        server_url: str,
-        site_id: Optional[str],
-        user: Optional[str],
-        password: Optional[str],
-        token_name: Optional[str],
-        token: Optional[str],
-    ):
-        credentials = self._get_credentials(user, password, token_name, token)
-
-        self.user = credentials.get(CredentialsKey.TABLEAU_USER)
-        self.site_id = site_id if site_id else DEFAULT_SERVER_SITE_ID
-        self.server_url = server_url
-        self.password = credentials.get(CredentialsKey.TABLEAU_PASSWORD)
-        self.token_name = credentials.get(CredentialsKey.TABLEAU_TOKEN_NAME)
-        self.token = credentials.get(CredentialsKey.TABLEAU_TOKEN)
-
-    @staticmethod
-    def _get_credentials(
-        user: Optional[str],
-        password: Optional[str],
-        token_name: Optional[str],
-        token: Optional[str],
-    ) -> dict:
-        """Helpers to retrieve credentials,
-        if both are given choose user and password authentication method"""
-        assert (user and password) or (token_name and token), AUTH_ERROR_MSG
-
-        if user and password:
-            return {
-                CredentialsKey.TABLEAU_USER: user,
-                CredentialsKey.TABLEAU_PASSWORD: password,
-            }
-
-        return {
-            CredentialsKey.TABLEAU_TOKEN_NAME: token_name,
-            CredentialsKey.TABLEAU_TOKEN: token,
-        }
-
-    def to_dict(self, hide: bool = False) -> dict[str, str]:
-        safe = (
-            CredentialsKey.TABLEAU_USER,
-            CredentialsKey.TABLEAU_SITE_ID,
-            CredentialsKey.TABLEAU_SERVER_URL,
-            CredentialsKey.TABLEAU_TOKEN_NAME,
-        )
-        unsafe = (CredentialsKey.TABLEAU_PASSWORD, CredentialsKey.TABLEAU_TOKEN)
-
-        def val(k: CredentialsKey, v: str) -> str:
-            return "*" + v[-4:] if hide and k in unsafe else v
-
-        return {a.value: val(a, getattr(self, a.value)) for a in safe + unsafe}

castor_extractor/visualization/tableau/client/project.py

@@ -1,28 +0,0 @@
-from typing import Optional
-
-from ....utils import SerializedAsset
-
-
-def _folder_path(
-    projects: SerializedAsset,
-    project: dict,
-    root: Optional[str] = "",
-) -> str:
-    """Recursive function to compute folder path with list of projects"""
-    path = "/" + str(project["name"]) + (root or "")
-    if project["parent_id"] is None:
-        return path
-
-    parent_project = next(
-        parent_project
-        for parent_project in projects
-        if parent_project["id"] == project["parent_id"]
-    )
-    return _folder_path(projects, parent_project, path)
-
-
-def compute_project_path(projects: SerializedAsset) -> SerializedAsset:
-    """Compute folder path with parent project name"""
-    for project in projects:
-        project["folder_path"] = _folder_path(projects, project)
-    return projects

castor_extractor/visualization/tableau/client/safe_mode.py

@@ -1,70 +0,0 @@
-import logging
-
-import tableauserverclient as TSC  # type: ignore
-
-from ....utils import SerializedAsset
-from ..assets import TableauAsset
-from ..constants import SAFE_MODE_PAGE_SIZE
-from ..errors import TableauErrorCode
-from ..types import PageReturn, ServerResponseError
-from ..usage import compute_usage_views
-from .client_utils import extract_asset
-
-logger = logging.getLogger(__name__)
-
-
-def _paginated_option(page_number: int) -> TSC.RequestOptions:
-    """Set up the Paginated option for TSC.RequestOptions"""
-    return TSC.RequestOptions(
-        pagesize=SAFE_MODE_PAGE_SIZE,
-        pagenumber=page_number,
-    )
-
-
-def _next_usage_page(client, page_number: int) -> PageReturn:
-    """
-    Request views per page
-    return Usages | ServerResponseError | TableauErrorCode
-    """
-    options = _paginated_option(page_number)
-    try:
-        all_usages_items, _ = client._server.views.get(options, usage=True)
-        return all_usages_items, None
-
-    except ServerResponseError as error:
-        expected = TableauErrorCode.PAGE_NUMBER_NOT_FOUND
-        if error.code == expected.value:
-            return None, expected
-        raise error
-
-    except ServerResponseError as error:
-        return None, error
-
-
-def safe_mode_fetch_usage(client) -> SerializedAsset:
-    """
-    Iterate throught each page
-    Returns computed usages when page number is not found
-    Log errors if ServerResponseError is return
-    """
-    list_usages: list[dict] = []
-    page_number: int = 0
-
-    while True:
-        page_number += 1
-        usages, error = _next_usage_page(client, page_number)
-        if error == TableauErrorCode.PAGE_NUMBER_NOT_FOUND:
-            return compute_usage_views(list_usages)
-
-        if error:
-            logger.warning(error)
-            client.errors.append(str(error))
-            continue
-
-        if not usages:
-            continue
-
-        new_usages = [
-            extract_asset(usage, TableauAsset.USAGE) for usage in usages
-        ]
-        list_usages.extend(new_usages)

castor_extractor/visualization/tableau/constants.py

@@ -1,9 +0,0 @@
-# tableau
-TABLEAU_SERVER_VERSION = "3.5"
-
-# pagination
-PAGE_SIZE = 100
-
-# safe mode
-SAFE_MODE_PAGE_SIZE = 1
-SAFE_MODE = False

castor_extractor/visualization/tableau/errors.py

@@ -1,5 +0,0 @@
-from enum import Enum
-
-
-class TableauErrorCode(Enum):
-    PAGE_NUMBER_NOT_FOUND = "400006"

castor_extractor/visualization/tableau/extract.py

@@ -1,121 +0,0 @@
-import logging
-from collections.abc import Iterable
-
-from ...utils import (
-    OUTPUT_DIR,
-    current_timestamp,
-    deep_serialize,
-    from_env,
-    get_output_filename,
-    write_errors_logs,
-    write_json,
-    write_summary,
-)
-from .assets import TableauAsset
-from .client import ApiClient as Client
-
-logger = logging.getLogger(__name__)
-
-
-def iterate_all_data(
-    client: Client,
-) -> Iterable[tuple[TableauAsset, list]]:
-    """Iterate over the extracted Data from Tableau"""
-
-    logger.info("Extracting USER from Tableau API")
-    yield TableauAsset.USER, deep_serialize(client.fetch(TableauAsset.USER))
-
-    logger.info("Extracting WORKBOOK from Tableau API")
-    yield (
-        TableauAsset.WORKBOOK,
-        deep_serialize(
-            client.fetch(TableauAsset.WORKBOOK),
-        ),
-    )
-
-    logger.info("Extracting DASHBOARD from Tableau API")
-    yield (
-        TableauAsset.DASHBOARD,
-        deep_serialize(
-            client.fetch(TableauAsset.DASHBOARD),
-        ),
-    )
-
-    logger.info("Extracting PUBLISHED DATASOURCE from Tableau API")
-    yield (
-        TableauAsset.PUBLISHED_DATASOURCE,
-        deep_serialize(
-            client.fetch(TableauAsset.PUBLISHED_DATASOURCE),
-        ),
-    )
-
-    logger.info("Extracting PROJECT from Tableau API")
-    yield (
-        TableauAsset.PROJECT,
-        deep_serialize(
-            client.fetch(TableauAsset.PROJECT),
-        ),
-    )
-
-    logger.info("Extracting USAGE from Tableau API")
-    yield TableauAsset.USAGE, deep_serialize(client.fetch(TableauAsset.USAGE))
-
-    logger.info("Extracting WORKBOOK_TO_DATASOURCE from Tableau API")
-    yield (
-        TableauAsset.WORKBOOK_TO_DATASOURCE,
-        deep_serialize(
-            client.fetch(TableauAsset.WORKBOOK_TO_DATASOURCE),
-        ),
-    )
-
-    logger.info("Extracting DATASOURCE from Tableau API")
-    yield (
-        TableauAsset.DATASOURCE,
-        deep_serialize(
-            client.fetch(TableauAsset.DATASOURCE),
-        ),
-    )
-
-    logger.info("Extracting CUSTOM_SQL_TABLE from Tableau API")
-    yield (
-        TableauAsset.CUSTOM_SQL_TABLE,
-        deep_serialize(
-            client.fetch(TableauAsset.CUSTOM_SQL_TABLE),
-        ),
-    )
-
-    logger.info("Extracting CUSTOM_SQL_QUERY from Tableau API")
-    yield (
-        TableauAsset.CUSTOM_SQL_QUERY,
-        deep_serialize(
-            client.fetch(TableauAsset.CUSTOM_SQL_QUERY),
-        ),
-    )
-
-    logger.info("Extracting FIELD from Tableau API")
-    yield TableauAsset.FIELD, deep_serialize(client.fetch(TableauAsset.FIELD))
-
-
-def extract_all(client: Client, **kwargs: str) -> None:
-    """
-    Extract Data from tableau
-    Store data locally in files under the output_directory
-    If errors from Tableau's API are catch store them locally in file under the output_directory
-    """
-    output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
-
-    timestamp = current_timestamp()
-
-    for key, data in iterate_all_data(client):
-        filename = get_output_filename(key.value, output_directory, timestamp)
-        write_json(filename, data)
-
-    write_summary(
-        output_directory,
-        timestamp,
-        base_url=client.base_url(),
-        client_name=client.name(),
-    )
-
-    if client.errors:
-        write_errors_logs(output_directory, timestamp, client.errors)