castor-extractor 0.23.3__py3-none-any.whl → 0.24.0__py3-none-any.whl

castor_extractor/visualization/tableau/client/safe_mode.py

@@ -1,70 +0,0 @@
-import logging
-
-import tableauserverclient as TSC  # type: ignore
-
-from ....utils import SerializedAsset
-from ..assets import TableauAsset
-from ..constants import SAFE_MODE_PAGE_SIZE
-from ..errors import TableauErrorCode
-from ..types import PageReturn, ServerResponseError
-from ..usage import compute_usage_views
-from .client_utils import extract_asset
-
-logger = logging.getLogger(__name__)
-
-
-def _paginated_option(page_number: int) -> TSC.RequestOptions:
-    """Set up the Paginated option for TSC.RequestOptions"""
-    return TSC.RequestOptions(
-        pagesize=SAFE_MODE_PAGE_SIZE,
-        pagenumber=page_number,
-    )
-
-
-def _next_usage_page(client, page_number: int) -> PageReturn:
-    """
-    Request views per page
-    return Usages | ServerResponseError | TableauErrorCode
-    """
-    options = _paginated_option(page_number)
-    try:
-        all_usages_items, _ = client._server.views.get(options, usage=True)
-        return all_usages_items, None
-
-    except ServerResponseError as error:
-        expected = TableauErrorCode.PAGE_NUMBER_NOT_FOUND
-        if error.code == expected.value:
-            return None, expected
-        raise error
-
-    except ServerResponseError as error:
-        return None, error
-
-
-def safe_mode_fetch_usage(client) -> SerializedAsset:
-    """
-    Iterate throught each page
-    Returns computed usages when page number is not found
-    Log errors if ServerResponseError is return
-    """
-    list_usages: list[dict] = []
-    page_number: int = 0
-
-    while True:
-        page_number += 1
-        usages, error = _next_usage_page(client, page_number)
-        if error == TableauErrorCode.PAGE_NUMBER_NOT_FOUND:
-            return compute_usage_views(list_usages)
-
-        if error:
-            logger.warning(error)
-            client.errors.append(str(error))
-            continue
-
-        if not usages:
-            continue
-
-        new_usages = [
-            extract_asset(usage, TableauAsset.USAGE) for usage in usages
-        ]
-        list_usages.extend(new_usages)

castor_extractor/visualization/tableau/constants.py

@@ -1,9 +0,0 @@
-# tableau
-TABLEAU_SERVER_VERSION = "3.5"
-
-# pagination
-PAGE_SIZE = 100
-
-# safe mode
-SAFE_MODE_PAGE_SIZE = 1
-SAFE_MODE = False

castor_extractor/visualization/tableau/errors.py

@@ -1,5 +0,0 @@
-from enum import Enum
-
-
-class TableauErrorCode(Enum):
-    PAGE_NUMBER_NOT_FOUND = "400006"

castor_extractor/visualization/tableau/extract.py

@@ -1,121 +0,0 @@
-import logging
-from collections.abc import Iterable
-
-from ...utils import (
-    OUTPUT_DIR,
-    current_timestamp,
-    deep_serialize,
-    from_env,
-    get_output_filename,
-    write_errors_logs,
-    write_json,
-    write_summary,
-)
-from .assets import TableauAsset
-from .client import ApiClient as Client
-
-logger = logging.getLogger(__name__)
-
-
-def iterate_all_data(
-    client: Client,
-) -> Iterable[tuple[TableauAsset, list]]:
-    """Iterate over the extracted Data from Tableau"""
-
-    logger.info("Extracting USER from Tableau API")
-    yield TableauAsset.USER, deep_serialize(client.fetch(TableauAsset.USER))
-
-    logger.info("Extracting WORKBOOK from Tableau API")
-    yield (
-        TableauAsset.WORKBOOK,
-        deep_serialize(
-            client.fetch(TableauAsset.WORKBOOK),
-        ),
-    )
-
-    logger.info("Extracting DASHBOARD from Tableau API")
-    yield (
-        TableauAsset.DASHBOARD,
-        deep_serialize(
-            client.fetch(TableauAsset.DASHBOARD),
-        ),
-    )
-
-    logger.info("Extracting PUBLISHED DATASOURCE from Tableau API")
-    yield (
-        TableauAsset.PUBLISHED_DATASOURCE,
-        deep_serialize(
-            client.fetch(TableauAsset.PUBLISHED_DATASOURCE),
-        ),
-    )
-
-    logger.info("Extracting PROJECT from Tableau API")
-    yield (
-        TableauAsset.PROJECT,
-        deep_serialize(
-            client.fetch(TableauAsset.PROJECT),
-        ),
-    )
-
-    logger.info("Extracting USAGE from Tableau API")
-    yield TableauAsset.USAGE, deep_serialize(client.fetch(TableauAsset.USAGE))
-
-    logger.info("Extracting WORKBOOK_TO_DATASOURCE from Tableau API")
-    yield (
-        TableauAsset.WORKBOOK_TO_DATASOURCE,
-        deep_serialize(
-            client.fetch(TableauAsset.WORKBOOK_TO_DATASOURCE),
-        ),
-    )
-
-    logger.info("Extracting DATASOURCE from Tableau API")
-    yield (
-        TableauAsset.DATASOURCE,
-        deep_serialize(
-            client.fetch(TableauAsset.DATASOURCE),
-        ),
-    )
-
-    logger.info("Extracting CUSTOM_SQL_TABLE from Tableau API")
-    yield (
-        TableauAsset.CUSTOM_SQL_TABLE,
-        deep_serialize(
-            client.fetch(TableauAsset.CUSTOM_SQL_TABLE),
-        ),
-    )
-
-    logger.info("Extracting CUSTOM_SQL_QUERY from Tableau API")
-    yield (
-        TableauAsset.CUSTOM_SQL_QUERY,
-        deep_serialize(
-            client.fetch(TableauAsset.CUSTOM_SQL_QUERY),
-        ),
-    )
-
-    logger.info("Extracting FIELD from Tableau API")
-    yield TableauAsset.FIELD, deep_serialize(client.fetch(TableauAsset.FIELD))
-
-
-def extract_all(client: Client, **kwargs: str) -> None:
-    """
-    Extract Data from tableau
-    Store data locally in files under the output_directory
-    If errors from Tableau's API are catch store them locally in file under the output_directory
-    """
-    output_directory = kwargs.get("output_directory") or from_env(OUTPUT_DIR)
-
-    timestamp = current_timestamp()
-
-    for key, data in iterate_all_data(client):
-        filename = get_output_filename(key.value, output_directory, timestamp)
-        write_json(filename, data)
-
-    write_summary(
-        output_directory,
-        timestamp,
-        base_url=client.base_url(),
-        client_name=client.name(),
-    )
-
-    if client.errors:
-        write_errors_logs(output_directory, timestamp, client.errors)

castor_extractor/visualization/tableau/gql_fields.py

@@ -1,249 +0,0 @@
-# Fields which will be use for Tableau GraphQL API
-from enum import Enum
-from typing import Union
-
-from .assets import TableauAsset, TableauGraphqlAsset
-
-FIELDS = "fields"
-OBJECT_TYPE = "object_type"
-
-
-class GQLQueryFields(Enum):
-    BIN_FIELDS: str = """
-           datasource {
-               ... on PublishedDatasource {
-                    name
-                    luid
-               }
-           }
-           dataType
-           description
-           fields {
-               id
-           }
-           folderName
-           id
-           name
-           role
-    """
-
-    CALCULATED_FIELDS: str = """
-           datasource {
-               ... on PublishedDatasource {
-                    name
-                    luid
-               }
-           }
-           fields {
-               id
-           }
-           dataType
-           description
-           folderName
-           id
-           name
-           role
-
-    """
-
-    COLUMN_FIELDS: str = """
-           columns {
-               id
-               name
-               table {
-                   ... on DatabaseTable {
-                       id
-                       description
-                       name
-                       fullName
-                       schema
-                       database {
-                           name
-                           luid
-                       }
-                   }
-               }
-
-           }
-           datasource {
-               ... on PublishedDatasource {
-                    name
-                    luid
-               }
-           }
-           dataType
-           description
-           folderName
-           id
-           name
-           role
-    """
-
-    CUSTOM_SQL_TABLE: str = """
-           id
-           name
-           columns {
-               referencedByFields {
-                   datasource {
-                       ... on PublishedDatasource {
-                           luid
-                       }
-
-                       ... on EmbeddedDatasource {
-                           id
-                       }
-                   }
-               }
-           }
-    """
-
-    CUSTOM_SQL_QUERY: str = """
-           id
-           name
-           query
-           database {
-               name
-               connectionType
-           }
-           tables {
-               name
-           }
-    """
-
-    DASHBOARDS: str = """
-            id
-            name
-            path
-            tags {
-                name
-            }
-            workbook {
-               luid # to retrieve the parent
-            }
-    """
-
-    DATASOURCE: str = """
-           ... on PublishedDatasource {
-               luid
-           }
-           id
-           name
-           hasExtracts
-           upstreamTables {
-               id
-               schema
-               name
-               fullName
-               database {
-                   id
-                   name
-                   connectionType
-               }
-           }
-    """
-
-    GROUP_FIELDS: str = """
-           datasource {
-               ... on PublishedDatasource {
-                    name
-                    luid
-               }
-           }
-           dataType
-           description
-           fields {
-               id
-           }
-           folderName
-           id
-           name
-           role
-    """
-
-    SHEET: str = """
-            containedInDashboards {
-                id
-            }
-            id
-            index
-            name
-            upstreamFields{
-                name
-            }
-            workbook {
-                luid
-            }
-    """
-
-    WORKBOOK_TO_DATASOURCE: str = """
-           luid
-           id
-           embeddedDatasources {
-               id
-               name
-           }
-           upstreamDatasources {
-               luid
-               name
-           }
-    """
-
-
-QueryInfo = list[dict[str, Union[GQLQueryFields, TableauGraphqlAsset]]]
-
-QUERY_FIELDS: dict[TableauAsset, QueryInfo] = {
-    TableauAsset.CUSTOM_SQL_TABLE: [
-        {
-            FIELDS: GQLQueryFields.CUSTOM_SQL_TABLE,
-            OBJECT_TYPE: TableauGraphqlAsset.CUSTOM_SQL,
-        },
-    ],
-    TableauAsset.CUSTOM_SQL_QUERY: [
-        {
-            FIELDS: GQLQueryFields.CUSTOM_SQL_QUERY,
-            OBJECT_TYPE: TableauGraphqlAsset.CUSTOM_SQL,
-        },
-    ],
-    TableauAsset.DASHBOARD: [
-        {
-            FIELDS: GQLQueryFields.DASHBOARDS,
-            OBJECT_TYPE: TableauGraphqlAsset.DASHBOARD,
-        },
-    ],
-    TableauAsset.DATASOURCE: [
-        {
-            FIELDS: GQLQueryFields.DATASOURCE,
-            OBJECT_TYPE: TableauGraphqlAsset.DATASOURCE,
-        },
-    ],
-    TableauAsset.FIELD: [
-        {
-            FIELDS: GQLQueryFields.BIN_FIELDS,
-            OBJECT_TYPE: TableauGraphqlAsset.BIN_FIELD,
-        },
-        {
-            FIELDS: GQLQueryFields.CALCULATED_FIELDS,
-            OBJECT_TYPE: TableauGraphqlAsset.CALCULATED_FIELD,
-        },
-        {
-            FIELDS: GQLQueryFields.COLUMN_FIELDS,
-            OBJECT_TYPE: TableauGraphqlAsset.COLUMN_FIELD,
-        },
-        {
-            FIELDS: GQLQueryFields.GROUP_FIELDS,
-            OBJECT_TYPE: TableauGraphqlAsset.GROUP_FIELD,
-        },
-    ],
-    TableauAsset.SHEET: [
-        {
-            FIELDS: GQLQueryFields.SHEET,
-            OBJECT_TYPE: TableauGraphqlAsset.SHEETS,
-        },
-    ],
-    TableauAsset.WORKBOOK_TO_DATASOURCE: [
-        {
-            FIELDS: GQLQueryFields.WORKBOOK_TO_DATASOURCE,
-            OBJECT_TYPE: TableauGraphqlAsset.WORKBOOK_TO_DATASOURCE,
-        },
-    ],
-}

castor_extractor/visualization/tableau/tests/unit/assets/graphql/metadata/metadata_1_get.json

@@ -1,15 +0,0 @@
-{
-  "data": {
-      "datasourcesConnection": {
-          "pageInfo": {
-              "hasNextPage": true,
-              "endCursor": "eyJ0eXBlIjoiUHVibGlzaGVkRGF0YXNvdXJjZSIsInNjb3BlIjoic2l0ZXMvMSIsInNvcnRPcmRlclZhbHVlIjp7Imxhc3RJZCI6IjAwMzllNWQ1LTI1ZmEtMTk2Yi1jNjZlLWMwNjc1ODM5ZTBiMCJ9fQ=="
-          },
-          "nodes": [
-              {
-                  "id": "0039e5d5-25fa-196b-c66e-c0675839e0b0"
-              }
-          ]
-      }
-  }
-}

castor_extractor/visualization/tableau/tests/unit/assets/graphql/metadata/metadata_2_get.json

@@ -1,15 +0,0 @@
-{
-  "data": {
-      "datasourcesConnection": {
-          "pageInfo": {
-              "hasNextPage": false,
-              "endCursor": "eyJ0eXBlIjoiUHVibGlzaGVkRGF0YXNvdXJjZSIsInNjb3BlIjoic2l0ZXMvMSIsInNvcnRPcmRlclZhbHVlIjp7Imxhc3RJZCI6IjAwYjE5MWNlLTYwNTUtYWZmNS1lMjc1LWMyNjYxMGM4YzRkNiJ9fQ=="
-          },
-          "nodes": [
-              {
-                  "id": "00b191ce-6055-aff5-e275-c26610c8c4d6"
-              }
-          ]
-      }
-  }
-}

castor_extractor/visualization/tableau/tests/unit/assets/rest_api/auth.xml

@@ -1,7 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
-    <credentials token="eIX6mvFsqyansa4KqEI1UwOpS8ggRs2l">
-        <site id="6b7179ba-b82b-4f0f-91ed-812074ac5da6" contentUrl="Samples" />
-        <user id="1a96d216-e9b8-497b-a82a-0b899a965e01" />
-    </credentials>
-</tsResponse>

castor_extractor/visualization/tableau/tests/unit/assets/rest_api/project_get.xml

@@ -1,9 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
-    <pagination pageNumber="1" pageSize="100" totalAvailable="3" />
-    <projects>
-        <project id="ee8c6e70-43b6-11e6-af4f-f7b0d8e20760" name="default" description="The default project that was automatically created by Tableau." contentPermissions="ManagedByOwner"><owner id="dd2239f6-ddf1-4107-981a-4cf94e415794" /></project>
-        <project id="1d0304cd-3796-429f-b815-7258370b9b74" name="Tableau" description="" contentPermissions="ManagedByOwner"><owner id="2a47bbf8-8900-4ebb-b0a4-2723bd7c46c3" /></project>
-        <project id="4cc52973-5e3a-4d1f-a4fb-5b5f73796edf" name="Tableau/Child_1" description="" contentPermissions="ManagedByOwner" parentProjectId="1d0304cd-3796-429f-b815-7258370b9b74"><owner id="dd2239f6-ddf1-4107-981a-4cf94e415794" /></project>
-    </projects>
-</tsResponse>

castor_extractor/visualization/tableau/tests/unit/assets/rest_api/user_get.xml

@@ -1,8 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
-    <pagination pageNumber="1" pageSize="100" totalAvailable="2" />
-    <users>
-        <user id="dd2239f6-ddf1-4107-981a-4cf94e415794" name="alice" siteRole="Publisher" lastLogin="2016-08-16T23:17:06Z" externalAuthUserId="" fullName="alice cook" email="alicecook@test.com" />
-        <user id="2a47bbf8-8900-4ebb-b0a4-2723bd7c46c3" name="Bob" siteRole="Interactor" externalAuthUserId="" fullName="Bob Smith" email="bob@test.com" />
-    </users>
-</tsResponse>

castor_extractor/visualization/tableau/tests/unit/assets/rest_api/view_get_usage.xml

@@ -1,24 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
-    <pagination pageNumber="1" pageSize="100" totalAvailable="3" />
-    <views>
-        <view id="d79634e1-6063-4ec9-95ff-50acbf609ff5" name="ENDANGERED SAFARI" contentUrl="SafariSample/sheets/ENDANGEREDSAFARI">
-            <workbook id="3cc6cd06-89ce-4fdc-b935-5294135d6d42" />
-            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
-            <tags />
-            <usage totalViewCount="7" />
-        </view>
-        <view id="f79634e1-6063-4ec9-95ff-50acbf609ff5" name="ENDANGERED SAFARI" contentUrl="SafariSample/sheets/ENDANGEREDSAFARI">
-            <workbook id="3cc6cd06-89ce-4fdc-b935-5294135d6d42" />
-            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
-            <tags />
-            <usage totalViewCount="4" />
-        </view>
-        <view id="fd252f73-593c-4c4e-8584-c032b8022adc" name="Overview" contentUrl="Superstore/sheets/Overview" createdAt="2002-05-30T09:00:00Z" updatedAt="2002-06-05T08:00:59Z" sheetType="story">
-            <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
-            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
-            <tags />
-            <usage totalViewCount="13" />
-        </view>
-    </views>
-</tsResponse>

castor_extractor/visualization/tableau/tests/unit/assets/rest_api/workbook_get.xml

@@ -1,19 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
-    <pagination pageNumber="1" pageSize="100" totalAvailable="2" />
-    <workbooks>
-        <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" name="Superstore" description="description for Superstore" contentUrl="Superstore" webpageUrl="http://tableauserver/#/workbooks/1/views" showTabs="false" size="1" createdAt="1" updatedAt="1">
-            <project id="ee8c6e70-43b6-11e6-af4f-f7b0d8e20760" name="default" />
-            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
-            <tags />
-        </workbook>
-        <workbook id="3cc6cd06-89ce-4fdc-b935-5294135d6d42" name="SafariSample" description="description for SafariSample" contentUrl="SafariSample" webpageUrl="http://tableauserver/#/workbooks/2/views" showTabs="false" size="26" createdAt="1" updatedAt="1">            
-            <project id="ee8c6e70-43b6-11e6-af4f-f7b0d8e20760" name="default" />
-            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
-            <tags>
-                <tag label="Safari" />
-                <tag label="Sample" />
-            </tags>
-        </workbook>
-    </workbooks>
-</tsResponse>

castor_extractor/visualization/tableau/tests/unit/graphql/paginated_object_test.py

@@ -1,63 +0,0 @@
-import os
-import unittest
-from unittest import mock
-
-import requests_mock  # type: ignore
-import tableauserverclient as TSC  # type: ignore
-
-from .....tableau import TableauAsset
-from ....client import ApiClient, get_paginated_objects
-from ....constants import TABLEAU_SERVER_VERSION
-from ..utils import KEYS
-
-TEST_ASSET_DIR = os.path.join(
-    os.path.dirname(__file__),
-    "../assets/graphql/metadata",
-)
-
-METADATA_1 = os.path.join(TEST_ASSET_DIR, "metadata_1_get.json")
-METADATA_2 = os.path.join(TEST_ASSET_DIR, "metadata_2_get.json")
-
-
-EXPECTED_PAGINATED = [
-    {"id": "0039e5d5-25fa-196b-c66e-c0675839e0b0"},
-    {"id": "00b191ce-6055-aff5-e275-c26610c8c4d6"},
-]
-
-
-class PaginatedObjectTests(unittest.TestCase):
-    @mock.patch.dict(os.environ, KEYS)
-    def setUp(self):
-        self._client = ApiClient()
-        self._client._server = TSC.Server("http://test")
-
-        # Fake signin
-        self._client._server._site_id = "dad65087-b08b-4603-af4e-2887b8aafc67"
-        self._client._server._auth_token = "j80k54ll2lfMZ0tv97mlPvvSCRyD0DOM"
-        self._client._server.version = TABLEAU_SERVER_VERSION
-
-        self.baseurl = self._client._server.metadata.baseurl
-
-    def test_paginated_object_with_datasources(self):
-        with open(METADATA_1, "rb") as f:
-            response_1_json = f.read().decode()
-
-        with open(METADATA_2, "rb") as f:
-            response_2_json = f.read().decode()
-
-        with requests_mock.mock() as m:
-            m.post(
-                self.baseurl,
-                [
-                    {"text": response_1_json, "status_code": 200},
-                    {"text": response_2_json, "status_code": 200},
-                ],
-            )
-
-            results = get_paginated_objects(
-                self._client._server,
-                TableauAsset.DATASOURCE,
-                self._client._page_size,
-            )
-
-        self.assertEqual(results, EXPECTED_PAGINATED)

castor_extractor/visualization/tableau/tests/unit/rest_api/auth_test.py

@@ -1,39 +0,0 @@
-import os.path
-import unittest
-from unittest import mock
-
-import requests_mock  # type: ignore
-
-from ....client import ApiClient
-from ..utils import KEYS
-
-TEST_ASSET_DIR = os.path.join(os.path.dirname(__file__), "../assets/rest_api/")
-
-AUTH_XML = os.path.join(TEST_ASSET_DIR, "auth.xml")
-
-
-class AuthTests(unittest.TestCase):
-    @mock.patch.dict(os.environ, KEYS)
-    def setUp(self):
-        self._client = ApiClient()
-        self.baseurl = self._client._server.auth.baseurl
-
-    def test_auth(self):
-        with open(AUTH_XML, "rb") as f:
-            response_xml = f.read().decode("utf-8")
-        with requests_mock.mock() as m:
-            m.post(self.baseurl + "/signin", text=response_xml)
-            self._client.login()
-
-        self.assertEqual(
-            "eIX6mvFsqyansa4KqEI1UwOpS8ggRs2l",
-            self._client._server.auth_token,
-        )
-        self.assertEqual(
-            "6b7179ba-b82b-4f0f-91ed-812074ac5da6",
-            self._client._server.site_id,
-        )
-        self.assertEqual(
-            "1a96d216-e9b8-497b-a82a-0b899a965e01",
-            self._client._server.user_id,
-        )