castor-extractor 0.24.0__tar.gz → 0.24.1__tar.gz

{castor_extractor-0.24.0 → castor_extractor-0.24.1}/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.24.1 - 2025-03-14
+
+* Added support for Looker Studio
+
 ## 0.24.0 - 2025-03-10
 
 * Remove legacy Tableau Connector

{castor_extractor-0.24.0 → castor_extractor-0.24.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.0
+Version: 0.24.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -170,7 +170,9 @@ Depending on your use case, you can also install one of the following `extras`:
 
 ```bash
 pip install castor-extractor[bigquery]
+pip install castor-extractor[databricks]
 pip install castor-extractor[looker]
+pip install castor-extractor[lookerstudio]
 pip install castor-extractor[metabase]
 pip install castor-extractor[mysql]
 pip install castor-extractor[powerbi]
@@ -208,6 +210,10 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.24.1 - 2025-03-14
+
+* Added support for Looker Studio
+
 ## 0.24.0 - 2025-03-10
 
 * Remove legacy Tableau Connector

{castor_extractor-0.24.0 → castor_extractor-0.24.1}/README.md

@@ -104,7 +104,9 @@ Depending on your use case, you can also install one of the following `extras`:
 
 ```bash
 pip install castor-extractor[bigquery]
+pip install castor-extractor[databricks]
 pip install castor-extractor[looker]
+pip install castor-extractor[lookerstudio]
 pip install castor-extractor[metabase]
 pip install castor-extractor[mysql]
 pip install castor-extractor[powerbi]

castor_extractor-0.24.1/castor_extractor/commands/extract_looker_studio.py

@@ -0,0 +1,30 @@
+from argparse import ArgumentParser
+
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+from castor_extractor.visualization import looker_studio  # type: ignore
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--credentials",
+        help="File path to Service Account credentials with Looker Studio access",
+    )
+    parser.add_argument(
+        "-a",
+        "--admin-email",
+        help="Email of a Google Workspace user with admin access",
+    )
+    parser.add_argument(
+        "-b",
+        "--bigquery-credentials",
+        help=(
+            "Optional: file path to Service Account credentials with BigQuery access. "
+            "This can be the same file path as for Looker Studio."
+        ),
+    )
+
+    parser.add_argument("-o", "--output", help="Directory to write to")
+
+    looker_studio.extract_all(**parse_filled_arguments(parser))

{castor_extractor-0.24.0 → castor_extractor-0.24.1}/castor_extractor/visualization/looker_studio/__init__.py

@@ -4,3 +4,4 @@ from .client import (
     LookerStudioClient,
     LookerStudioCredentials,
 )
+from .extract import extract_all

{castor_extractor-0.24.0 → castor_extractor-0.24.1}/castor_extractor/visualization/looker_studio/assets.py

@@ -3,4 +3,5 @@ from ...types import ExternalAsset
 
 class LookerStudioAsset(ExternalAsset):
     ASSETS = "assets"
+    SOURCE_QUERIES = "source_queries"
     VIEW_ACTIVITY = "view_activity"

castor_extractor-0.24.1/castor_extractor/visualization/looker_studio/client/client.py

@@ -0,0 +1,87 @@
+from typing import Iterator, Optional
+
+from ....utils import empty_iterator
+from ....warehouse.abstract import WarehouseAsset
+from ....warehouse.bigquery import BigQueryClient, BigQueryQueryBuilder
+from .. import LookerStudioAsset
+from .admin_sdk_client import USER_EMAIL_FIELD, AdminSDKClient
+from .credentials import LookerStudioCredentials
+from .looker_studio_api_client import LookerStudioAPIClient
+
+
+class LookerStudioQueryBuilder(BigQueryQueryBuilder):
+    def job_history_queries(self) -> list:
+        """
+        This class and method are a convenient workaround to build the
+        ExtractionQueries which retrieve BigQuery's job history, but filtered on
+        Looker Studio only.
+
+        Compared to the generic BigQuery query history, only the SQL "template"
+        changes. By defining this class here, this will pick the SQL file
+        `queries/query.sql` located in the same directory as this file.
+        """
+        return super().build(WarehouseAsset.QUERY)  # type: ignore
+
+
+class LookerStudioClient:
+    """
+    Acts as a wrapper class to fetch Looker Studio assets, which requires
+    coordinating calls between the Admin SDK API and the Looker Studio API.
+
+    If the BigQuery credentials are provided, it can also fetch the source queries
+    of BigQuery data sources.
+    """
+
+    def __init__(
+        self,
+        credentials: LookerStudioCredentials,
+        bigquery_credentials: Optional[dict] = None,
+    ):
+        self.admin_sdk_client = AdminSDKClient(credentials)
+        self.looker_studio_client = LookerStudioAPIClient(credentials)
+
+        self.bigquery_client: Optional[BigQueryClient] = None
+        if bigquery_credentials:
+            self.bigquery_client = BigQueryClient(bigquery_credentials)
+
+    def _get_assets(self) -> Iterator[dict]:
+        """
+        Extracts reports and data sources user by user.
+        """
+        users = self.admin_sdk_client.list_users()
+
+        for user in users:
+            email = user[USER_EMAIL_FIELD]
+            yield from self.looker_studio_client.fetch_user_assets(email)
+
+    def _get_source_queries(self) -> Iterator[dict]:
+        """
+        Extracts the BigQuery jobs triggered by Looker Studio. The last job
+        per data source is returned.
+        """
+        if not self.bigquery_client:
+            return empty_iterator()
+
+        query_builder = LookerStudioQueryBuilder(
+            regions=self.bigquery_client.get_regions(),
+            datasets=self.bigquery_client.get_datasets(),
+            extended_regions=self.bigquery_client.get_extended_regions(),
+        )
+
+        queries = query_builder.job_history_queries()
+
+        for query in queries:
+            yield from self.bigquery_client.execute(query)
+
+    def fetch(self, asset: LookerStudioAsset) -> Iterator[dict]:
+        if asset == LookerStudioAsset.ASSETS:
+            yield from self._get_assets()
+
+        elif asset == LookerStudioAsset.SOURCE_QUERIES:
+            yield from self._get_source_queries()
+
+        elif asset == LookerStudioAsset.VIEW_ACTIVITY:
+            yield from self.admin_sdk_client.list_view_events()
+
+        else:
+            raise ValueError(f"The asset {asset}, is not supported")

{castor_extractor-0.24.0 → castor_extractor-0.24.1}/castor_extractor/visualization/looker_studio/client/credentials.py

@@ -2,6 +2,11 @@ from pydantic import BaseModel, SecretStr, field_serializer
 
 
 class LookerStudioCredentials(BaseModel):
+    """
+    Looker Studio Credentials match the Service Account credentials JSON
+    but with an additional admin_email field.
+    """
+
     admin_email: str
     auth_provider_x509_cert_url: str
     auth_uri: str

castor_extractor-0.24.1/castor_extractor/visualization/looker_studio/client/queries/query.sql

@@ -0,0 +1,52 @@
+/*
+Gets the query jobs triggered by Looker Studio when refreshing a BigQuery data source. Only the latest query per
+data source is selected.
+
+The `labels` column should indicate the `looker_studio_datasource_id` that triggered the job. In some cases, it also
+contains a `looker_studio_report_id` value, which gives us a link between the data source and a report.
+*/
+WITH ranked_by_datasource AS (
+    SELECT
+        creation_time,
+        project_id AS database_name,
+        user_email,
+        query AS query_text,
+        referenced_tables,
+        labels,
+        ROW_NUMBER() OVER (
+            PARTITION BY (
+                SELECT
+                    label.value
+                FROM
+                    UNNEST(labels) AS label
+                WHERE
+                    label.key = 'looker_studio_datasource_id'
+            )
+            ORDER BY
+                creation_time DESC
+        ) AS row_num
+    FROM
+        `{project}.region-{region}.INFORMATION_SCHEMA.JOBS_BY_PROJECT`
+    WHERE
+        job_type = 'QUERY'
+        AND EXISTS (
+            SELECT
+                1
+            FROM
+                UNNEST(labels) AS label
+            WHERE
+                label.key = 'requestor'
+                AND label.value = 'looker_studio'
+        )
+)
+SELECT
+    creation_time,
+    database_name,
+    user_email,
+    query_text,
+    referenced_tables,
+    labels
+FROM
+    ranked_by_datasource
+WHERE
+    row_num = 1;

castor_extractor-0.24.1/castor_extractor/visualization/looker_studio/extract.py

@@ -0,0 +1,84 @@
+import json
+import logging
+from collections.abc import Iterable
+from typing import Optional, Union, cast
+
+from ...utils import (
+    OUTPUT_DIR,
+    current_timestamp,
+    deep_serialize,
+    from_env,
+    get_output_filename,
+    write_json,
+    write_summary,
+)
+from .assets import LookerStudioAsset
+from .client import LookerStudioClient, LookerStudioCredentials
+
+logger = logging.getLogger(__name__)
+
+APPLICATION_CREDENTIALS = "GOOGLE_APPLICATION_CREDENTIALS"
+LOOKER_STUDIO_ADMIN_EMAIL = "CASTOR_LOOKER_STUDIO_ADMIN_EMAIL"
+
+
+def iterate_all_data(
+    client: LookerStudioClient,
+) -> Iterable[tuple[LookerStudioAsset, Union[list, dict]]]:
+    for asset in LookerStudioAsset:
+        logger.info(f"Extracting {asset.name} from API")
+        data = list(deep_serialize(client.fetch(asset)))
+        yield asset, data
+        logger.info(f"Extracted {len(data)} {asset.name} from API")
+
+
+def _credentials(params: dict) -> LookerStudioCredentials:
+    """
+    Builds the Looker Studio credentials by combining the Service Account
+    credentials with the admin email.
+    """
+    path = params.get("credentials") or from_env(APPLICATION_CREDENTIALS)
+    logger.info(f"Looker Studio credentials loaded from {path}")
+    with open(path) as file:
+        credentials = cast(dict, json.load(file))
+
+    admin_email = params.get("admin_email") or from_env(
+        LOOKER_STUDIO_ADMIN_EMAIL
+    )
+    credentials["admin_email"] = admin_email
+    return LookerStudioCredentials(**credentials)
+
+
+def _bigquery_credentials_or_none(params: dict) -> Optional[dict]:
+    """Extracts optional GCP credentials to access BigQuery"""
+    path = params.get("bigquery_credentials") or from_env(
+        APPLICATION_CREDENTIALS,
+        allow_missing=True,
+    )
+    if not path:
+        return None
+
+    logger.info(f"BigQuery credentials loaded from {path}")
+    with open(path) as file:
+        return cast(dict, json.load(file))
+
+
+def extract_all(**kwargs) -> None:
+    """
+    Extracts data from Looker Studio and stores the output files locally under
+    the given output_directory.
+    """
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+    credentials = _credentials(kwargs)
+    bigquery_credentials = _bigquery_credentials_or_none(kwargs)
+
+    client = LookerStudioClient(
+        credentials=credentials,
+        bigquery_credentials=bigquery_credentials,
+    )
+    ts = current_timestamp()
+
+    for key, data in iterate_all_data(client):
+        filename = get_output_filename(key.name.lower(), output_directory, ts)
+        write_json(filename, data)
+
+    write_summary(output_directory, ts)

{castor_extractor-0.24.0 → castor_extractor-0.24.1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.0"
+version = "0.24.1"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -122,6 +122,7 @@ castor-extract-confluence = "castor_extractor.commands.extract_confluence:main"
 castor-extract-databricks = "castor_extractor.commands.extract_databricks:main"
 castor-extract-domo = "castor_extractor.commands.extract_domo:main"
 castor-extract-looker = "castor_extractor.commands.extract_looker:main"
+castor-extract-looker-studio = "castor_extractor.commands.extract_looker_studio:main"
 castor-extract-metabase-api = "castor_extractor.commands.extract_metabase_api:main"
 castor-extract-metabase-db = "castor_extractor.commands.extract_metabase_db:main"
 castor-extract-mode = "castor_extractor.commands.extract_mode:main"
@@ -206,7 +207,6 @@ DEP002 = [
 
 [tool.ruff]
 line-length = 80
-target-version = "py38"
 src = ["extractor"]
 include = ["**/*.py", "**/*.pyi"]
 

castor_extractor-0.24.0/castor_extractor/visualization/looker_studio/client/client.py

@@ -1,37 +0,0 @@
-from typing import Iterator
-
-from .. import LookerStudioAsset
-from .admin_sdk_client import USER_EMAIL_FIELD, AdminSDKClient
-from .credentials import LookerStudioCredentials
-from .looker_studio_api_client import LookerStudioAPIClient
-
-
-class LookerStudioClient:
-    """
-    Acts as a wrapper class to fetch Looker Studio assets, which requires
-    coordinating calls between the Admin SDK API and the Looker Studio API.
-    """
-
-    def __init__(self, credentials: LookerStudioCredentials):
-        self.admin_sdk_client = AdminSDKClient(credentials)
-        self.looker_studio_client = LookerStudioAPIClient(credentials)
-
-    def _get_assets(self) -> Iterator[dict]:
-        """
-        Extracts reports and data sources user by user.
-        """
-        users = self.admin_sdk_client.list_users()
-
-        for user in users:
-            email = user[USER_EMAIL_FIELD]
-            yield from self.looker_studio_client.fetch_user_assets(email)
-
-    def fetch(self, asset: LookerStudioAsset) -> Iterator[dict]:
-        if asset == LookerStudioAsset.VIEW_ACTIVITY:
-            yield from self.admin_sdk_client.list_view_events()
-
-        elif asset == LookerStudioAsset.ASSETS:
-            yield from self._get_assets()
-
-        else:
-            raise ValueError(f"The asset {asset}, is not supported")