castor-extractor 0.23.3__tar.gz → 0.24.1__tar.gz

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.24.1 - 2025-03-14
+
+* Added support for Looker Studio
+
+## 0.24.0 - 2025-03-10
+
+* Remove legacy Tableau Connector
+
 ## 0.23.3 - 2025-02-19
 
 * Snowflake : add --insecure-mode option to turn off OCSP checking

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.23.3
+Version: 0.24.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -170,7 +170,9 @@ Depending on your use case, you can also install one of the following `extras`:
 
 ```bash
 pip install castor-extractor[bigquery]
+pip install castor-extractor[databricks]
 pip install castor-extractor[looker]
+pip install castor-extractor[lookerstudio]
 pip install castor-extractor[metabase]
 pip install castor-extractor[mysql]
 pip install castor-extractor[powerbi]
@@ -208,6 +210,14 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.24.1 - 2025-03-14
+
+* Added support for Looker Studio
+
+## 0.24.0 - 2025-03-10
+
+* Remove legacy Tableau Connector
+
 ## 0.23.3 - 2025-02-19
 
 * Snowflake : add --insecure-mode option to turn off OCSP checking

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/README.md

@@ -104,7 +104,9 @@ Depending on your use case, you can also install one of the following `extras`:
 
 ```bash
 pip install castor-extractor[bigquery]
+pip install castor-extractor[databricks]
 pip install castor-extractor[looker]
+pip install castor-extractor[lookerstudio]
 pip install castor-extractor[metabase]
 pip install castor-extractor[mysql]
 pip install castor-extractor[powerbi]

castor_extractor-0.24.1/castor_extractor/commands/extract_looker_studio.py

@@ -0,0 +1,30 @@
+from argparse import ArgumentParser
+
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+from castor_extractor.visualization import looker_studio  # type: ignore
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--credentials",
+        help="File path to Service Account credentials with Looker Studio access",
+    )
+    parser.add_argument(
+        "-a",
+        "--admin-email",
+        help="Email of a Google Workspace user with admin access",
+    )
+    parser.add_argument(
+        "-b",
+        "--bigquery-credentials",
+        help=(
+            "Optional: file path to Service Account credentials with BigQuery access. "
+            "This can be the same file path as for Looker Studio."
+        ),
+    )
+
+    parser.add_argument("-o", "--output", help="Directory to write to")
+
+    looker_studio.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/castor_extractor/knowledge/confluence/client/client.py

@@ -1,12 +1,9 @@
 from collections.abc import Iterator
 from functools import partial
-from http import HTTPStatus
-from typing import Optional
 
 from ....utils import (
     APIClient,
     BasicAuth,
-    RequestSafeMode,
     fetch_all_pages,
 )
 from ..assets import (
@@ -21,19 +18,11 @@ _HEADERS = {
     "Content-Type": "application/json",
 }
 
-_MAX_ERROR_IGNORED_COUNT = 10
-_IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY,)
-_SAFE_MODE = RequestSafeMode(
-    max_errors=_MAX_ERROR_IGNORED_COUNT,
-    status_codes=_IGNORED_ERROR_CODES,
-)
-
 
 class ConfluenceClient(APIClient):
     def __init__(
         self,
         credentials: ConfluenceCredentials,
-        safe_mode: Optional[RequestSafeMode] = None,
     ):
         self.account_id = credentials.account_id
         auth = BasicAuth(
@@ -43,14 +32,12 @@ class ConfluenceClient(APIClient):
             auth=auth,
             host=credentials.base_url,
             headers=_HEADERS,
-            safe_mode=safe_mode or _SAFE_MODE,
         )
 
     def pages(self):
         request = partial(
             self._get,
             endpoint=ConfluenceEndpointFactory.pages(),
-            params={"body-format": "atlas_doc_format"},
         )
         yield from fetch_all_pages(request, ConfluencePagination)
 

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/castor_extractor/knowledge/confluence/client/endpoints.py

@@ -14,7 +14,7 @@ class ConfluenceEndpointFactory:
         Endpoint to fetch all pages.
         More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-pages-get
         """
-        return f"{cls.API}{cls.PAGES}"
+        return f"{cls.API}{cls.PAGES}?body-format=atlas_doc_format"
 
     @classmethod
     def users(cls) -> str:

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/castor_extractor/visualization/looker_studio/__init__.py

@@ -4,3 +4,4 @@ from .client import (
     LookerStudioClient,
     LookerStudioCredentials,
 )
+from .extract import extract_all

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/castor_extractor/visualization/looker_studio/assets.py

@@ -3,4 +3,5 @@ from ...types import ExternalAsset
 
 class LookerStudioAsset(ExternalAsset):
     ASSETS = "assets"
+    SOURCE_QUERIES = "source_queries"
     VIEW_ACTIVITY = "view_activity"

castor_extractor-0.24.1/castor_extractor/visualization/looker_studio/client/client.py

@@ -0,0 +1,87 @@
+from typing import Iterator, Optional
+
+from ....utils import empty_iterator
+from ....warehouse.abstract import WarehouseAsset
+from ....warehouse.bigquery import BigQueryClient, BigQueryQueryBuilder
+from .. import LookerStudioAsset
+from .admin_sdk_client import USER_EMAIL_FIELD, AdminSDKClient
+from .credentials import LookerStudioCredentials
+from .looker_studio_api_client import LookerStudioAPIClient
+
+
+class LookerStudioQueryBuilder(BigQueryQueryBuilder):
+    def job_history_queries(self) -> list:
+        """
+        This class and method are a convenient workaround to build the
+        ExtractionQueries which retrieve BigQuery's job history, but filtered on
+        Looker Studio only.
+
+        Compared to the generic BigQuery query history, only the SQL "template"
+        changes. By defining this class here, this will pick the SQL file
+        `queries/query.sql` located in the same directory as this file.
+        """
+        return super().build(WarehouseAsset.QUERY)  # type: ignore
+
+
+class LookerStudioClient:
+    """
+    Acts as a wrapper class to fetch Looker Studio assets, which requires
+    coordinating calls between the Admin SDK API and the Looker Studio API.
+
+    If the BigQuery credentials are provided, it can also fetch the source queries
+    of BigQuery data sources.
+    """
+
+    def __init__(
+        self,
+        credentials: LookerStudioCredentials,
+        bigquery_credentials: Optional[dict] = None,
+    ):
+        self.admin_sdk_client = AdminSDKClient(credentials)
+        self.looker_studio_client = LookerStudioAPIClient(credentials)
+
+        self.bigquery_client: Optional[BigQueryClient] = None
+        if bigquery_credentials:
+            self.bigquery_client = BigQueryClient(bigquery_credentials)
+
+    def _get_assets(self) -> Iterator[dict]:
+        """
+        Extracts reports and data sources user by user.
+        """
+        users = self.admin_sdk_client.list_users()
+
+        for user in users:
+            email = user[USER_EMAIL_FIELD]
+            yield from self.looker_studio_client.fetch_user_assets(email)
+
+    def _get_source_queries(self) -> Iterator[dict]:
+        """
+        Extracts the BigQuery jobs triggered by Looker Studio. The last job
+        per data source is returned.
+        """
+        if not self.bigquery_client:
+            return empty_iterator()
+
+        query_builder = LookerStudioQueryBuilder(
+            regions=self.bigquery_client.get_regions(),
+            datasets=self.bigquery_client.get_datasets(),
+            extended_regions=self.bigquery_client.get_extended_regions(),
+        )
+
+        queries = query_builder.job_history_queries()
+
+        for query in queries:
+            yield from self.bigquery_client.execute(query)
+
+    def fetch(self, asset: LookerStudioAsset) -> Iterator[dict]:
+        if asset == LookerStudioAsset.ASSETS:
+            yield from self._get_assets()
+
+        elif asset == LookerStudioAsset.SOURCE_QUERIES:
+            yield from self._get_source_queries()
+
+        elif asset == LookerStudioAsset.VIEW_ACTIVITY:
+            yield from self.admin_sdk_client.list_view_events()
+
+        else:
+            raise ValueError(f"The asset {asset}, is not supported")

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/castor_extractor/visualization/looker_studio/client/credentials.py

@@ -2,6 +2,11 @@ from pydantic import BaseModel, SecretStr, field_serializer
 
 
 class LookerStudioCredentials(BaseModel):
+    """
+    Looker Studio Credentials match the Service Account credentials JSON
+    but with an additional admin_email field.
+    """
+
     admin_email: str
     auth_provider_x509_cert_url: str
     auth_uri: str

castor_extractor-0.24.1/castor_extractor/visualization/looker_studio/client/queries/query.sql

@@ -0,0 +1,52 @@
+/*
+Gets the query jobs triggered by Looker Studio when refreshing a BigQuery data source. Only the latest query per
+data source is selected.
+
+The `labels` column should indicate the `looker_studio_datasource_id` that triggered the job. In some cases, it also
+contains a `looker_studio_report_id` value, which gives us a link between the data source and a report.
+*/
+WITH ranked_by_datasource AS (
+    SELECT
+        creation_time,
+        project_id AS database_name,
+        user_email,
+        query AS query_text,
+        referenced_tables,
+        labels,
+        ROW_NUMBER() OVER (
+            PARTITION BY (
+                SELECT
+                    label.value
+                FROM
+                    UNNEST(labels) AS label
+                WHERE
+                    label.key = 'looker_studio_datasource_id'
+            )
+            ORDER BY
+                creation_time DESC
+        ) AS row_num
+    FROM
+        `{project}.region-{region}.INFORMATION_SCHEMA.JOBS_BY_PROJECT`
+    WHERE
+        job_type = 'QUERY'
+        AND EXISTS (
+            SELECT
+                1
+            FROM
+                UNNEST(labels) AS label
+            WHERE
+                label.key = 'requestor'
+                AND label.value = 'looker_studio'
+        )
+)
+SELECT
+    creation_time,
+    database_name,
+    user_email,
+    query_text,
+    referenced_tables,
+    labels
+FROM
+    ranked_by_datasource
+WHERE
+    row_num = 1;

castor_extractor-0.24.1/castor_extractor/visualization/looker_studio/extract.py

@@ -0,0 +1,84 @@
+import json
+import logging
+from collections.abc import Iterable
+from typing import Optional, Union, cast
+
+from ...utils import (
+    OUTPUT_DIR,
+    current_timestamp,
+    deep_serialize,
+    from_env,
+    get_output_filename,
+    write_json,
+    write_summary,
+)
+from .assets import LookerStudioAsset
+from .client import LookerStudioClient, LookerStudioCredentials
+
+logger = logging.getLogger(__name__)
+
+APPLICATION_CREDENTIALS = "GOOGLE_APPLICATION_CREDENTIALS"
+LOOKER_STUDIO_ADMIN_EMAIL = "CASTOR_LOOKER_STUDIO_ADMIN_EMAIL"
+
+
+def iterate_all_data(
+    client: LookerStudioClient,
+) -> Iterable[tuple[LookerStudioAsset, Union[list, dict]]]:
+    for asset in LookerStudioAsset:
+        logger.info(f"Extracting {asset.name} from API")
+        data = list(deep_serialize(client.fetch(asset)))
+        yield asset, data
+        logger.info(f"Extracted {len(data)} {asset.name} from API")
+
+
+def _credentials(params: dict) -> LookerStudioCredentials:
+    """
+    Builds the Looker Studio credentials by combining the Service Account
+    credentials with the admin email.
+    """
+    path = params.get("credentials") or from_env(APPLICATION_CREDENTIALS)
+    logger.info(f"Looker Studio credentials loaded from {path}")
+    with open(path) as file:
+        credentials = cast(dict, json.load(file))
+
+    admin_email = params.get("admin_email") or from_env(
+        LOOKER_STUDIO_ADMIN_EMAIL
+    )
+    credentials["admin_email"] = admin_email
+    return LookerStudioCredentials(**credentials)
+
+
+def _bigquery_credentials_or_none(params: dict) -> Optional[dict]:
+    """Extracts optional GCP credentials to access BigQuery"""
+    path = params.get("bigquery_credentials") or from_env(
+        APPLICATION_CREDENTIALS,
+        allow_missing=True,
+    )
+    if not path:
+        return None
+
+    logger.info(f"BigQuery credentials loaded from {path}")
+    with open(path) as file:
+        return cast(dict, json.load(file))
+
+
+def extract_all(**kwargs) -> None:
+    """
+    Extracts data from Looker Studio and stores the output files locally under
+    the given output_directory.
+    """
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = _credentials(kwargs)
+    bigquery_credentials = _bigquery_credentials_or_none(kwargs)
+
+    client = LookerStudioClient(
+        credentials=credentials,
+        bigquery_credentials=bigquery_credentials,
+    )
+    ts = current_timestamp()
+
+    for key, data in iterate_all_data(client):
+        filename = get_output_filename(key.name.lower(), output_directory, ts)
+        write_json(filename, data)
+
+    write_summary(output_directory, ts)

{castor_extractor-0.23.3 → castor_extractor-0.24.1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.23.3"
+version = "0.24.1"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -122,6 +122,7 @@ castor-extract-confluence = "castor_extractor.commands.extract_confluence:main"
 castor-extract-databricks = "castor_extractor.commands.extract_databricks:main"
 castor-extract-domo = "castor_extractor.commands.extract_domo:main"
 castor-extract-looker = "castor_extractor.commands.extract_looker:main"
+castor-extract-looker-studio = "castor_extractor.commands.extract_looker_studio:main"
 castor-extract-metabase-api = "castor_extractor.commands.extract_metabase_api:main"
 castor-extract-metabase-db = "castor_extractor.commands.extract_metabase_db:main"
 castor-extract-mode = "castor_extractor.commands.extract_mode:main"
@@ -206,7 +207,6 @@ DEP002 = [
 
 [tool.ruff]
 line-length = 80
-target-version = "py38"
 src = ["extractor"]
 include = ["**/*.py", "**/*.pyi"]
 

castor_extractor-0.23.3/castor_extractor/visualization/looker_studio/client/client.py

@@ -1,37 +0,0 @@
-from typing import Iterator
-
-from .. import LookerStudioAsset
-from .admin_sdk_client import USER_EMAIL_FIELD, AdminSDKClient
-from .credentials import LookerStudioCredentials
-from .looker_studio_api_client import LookerStudioAPIClient
-
-
-class LookerStudioClient:
-    """
-    Acts as a wrapper class to fetch Looker Studio assets, which requires
-    coordinating calls between the Admin SDK API and the Looker Studio API.
-    """
-
-    def __init__(self, credentials: LookerStudioCredentials):
-        self.admin_sdk_client = AdminSDKClient(credentials)
-        self.looker_studio_client = LookerStudioAPIClient(credentials)
-
-    def _get_assets(self) -> Iterator[dict]:
-        """
-        Extracts reports and data sources user by user.
-        """
-        users = self.admin_sdk_client.list_users()
-
-        for user in users:
-            email = user[USER_EMAIL_FIELD]
-            yield from self.looker_studio_client.fetch_user_assets(email)
-
-    def fetch(self, asset: LookerStudioAsset) -> Iterator[dict]:
-        if asset == LookerStudioAsset.VIEW_ACTIVITY:
-            yield from self.admin_sdk_client.list_view_events()
-
-        elif asset == LookerStudioAsset.ASSETS:
-            yield from self._get_assets()
-
-        else:
-            raise ValueError(f"The asset {asset}, is not supported")

castor_extractor-0.23.3/castor_extractor/visualization/tableau/__init__.py

@@ -1,3 +0,0 @@
-from .assets import TableauAsset
-from .client import ApiClient
-from .extract import extract_all, iterate_all_data

castor_extractor-0.23.3/castor_extractor/visualization/tableau/assets.py

@@ -1,49 +0,0 @@
-from enum import Enum
-
-from ...types import ExternalAsset, classproperty
-
-
-class TableauAsset(ExternalAsset):
-    """
-    Tableau assets
-    """
-
-    CUSTOM_SQL_TABLE = "custom_sql_tables"
-    CUSTOM_SQL_QUERY = "custom_sql_queries"
-    DASHBOARD = "dashboards"
-    DASHBOARD_SHEET = "dashboards_sheets"
-    DATASOURCE = "datasources"
-    FIELD = "fields"
-    PROJECT = "projects"
-    PUBLISHED_DATASOURCE = "published_datasources"
-    SHEET = "sheets"
-    USAGE = "views"
-    USER = "users"
-    WORKBOOK = "workbooks"
-    WORKBOOK_TO_DATASOURCE = "workbooks_to_datasource"
-
-    @classproperty
-    def optional(cls) -> set["TableauAsset"]:
-        return {
-            TableauAsset.DASHBOARD,
-            TableauAsset.DASHBOARD_SHEET,
-            TableauAsset.FIELD,
-            TableauAsset.SHEET,
-            TableauAsset.PUBLISHED_DATASOURCE,
-        }
-
-
-class TableauGraphqlAsset(Enum):
-    """
-    Assets which can be fetched from Tableau
-    """
-
-    BIN_FIELD = "binFields"
-    CALCULATED_FIELD = "calculatedFields"
-    COLUMN_FIELD = "columnFields"
-    CUSTOM_SQL = "customSQLTables"
-    DASHBOARD = "dashboards"
-    DATASOURCE = "datasources"
-    GROUP_FIELD = "groupFields"
-    SHEETS = "sheets"
-    WORKBOOK_TO_DATASOURCE = "workbooks"

castor_extractor-0.23.3/castor_extractor/visualization/tableau/client/__init__.py

@@ -1,2 +0,0 @@
-from .client import ApiClient
-from .client_utils import get_paginated_objects