castor-extractor 0.24.2__tar.gz → 0.24.7__tar.gz

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.24.7 - 2025-04-07
+
+* Tableau - switch from `cursor` to `offset` pagination to mitigate timeout issues
+
+## 0.24.6 - 2025-04-03
+
+* Domo - extract cards metadata by batch to prevent from hitting URL max length
+
+## 0.24.5 - 2025-04-02
+
+* bump dependencies: google-cloud-storage
+
+## 0.24.4 - 2025-03-19
+
+* Snowflake:
+  * improve the list of ignored queries in the query history extraction
+    * ignore the following query types : CALL, COMMENT, EXPLAIN, REFRESH_DYNAMIC_TABLE_AT_REFRESH_VERSION, REVOKE, TRUNCATE_TABLE, UNDROP
+    * ignore queries with empty text
+  * filter out schemas with empty names
+
+## 0.24.3 - 2025-03-18
+
+* Replace ThoughtSpot endpoint `/api/rest/2.0/report/liveboard` with `/api/rest/2.0/metadata/liveboard/data` following the deprecation of the CSV option
+
 ## 0.24.2 - 2025-03-17
 
 * Rename Revamped Tableau Connector classes

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.2
+Version: 0.24.7
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -35,7 +35,7 @@ Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=2,<3)
+Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
@@ -51,7 +51,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=75.6)
+Requires-Dist: setuptools (>=78.1)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
@@ -210,6 +210,30 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.24.7 - 2025-04-07
+
+* Tableau - switch from `cursor` to `offset` pagination to mitigate timeout issues
+
+## 0.24.6 - 2025-04-03
+
+* Domo - extract cards metadata by batch to prevent from hitting URL max length
+
+## 0.24.5 - 2025-04-02
+
+* bump dependencies: google-cloud-storage
+
+## 0.24.4 - 2025-03-19
+
+* Snowflake:
+  * improve the list of ignored queries in the query history extraction
+    * ignore the following query types : CALL, COMMENT, EXPLAIN, REFRESH_DYNAMIC_TABLE_AT_REFRESH_VERSION, REVOKE, TRUNCATE_TABLE, UNDROP
+    * ignore queries with empty text
+  * filter out schemas with empty names
+
+## 0.24.3 - 2025-03-18
+
+* Replace ThoughtSpot endpoint `/api/rest/2.0/report/liveboard` with `/api/rest/2.0/metadata/liveboard/data` following the deprecation of the CSV option
+
 ## 0.24.2 - 2025-03-17
 
 * Rename Revamped Tableau Connector classes

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/utils/__init__.py

@@ -1,4 +1,5 @@
 from .argument_parser import parse_filled_arguments
+from .batch import batch_of_length
 from .client import (
     AbstractSourceClient,
     APIClient,

castor_extractor-0.24.7/castor_extractor/utils/batch.py

@@ -0,0 +1,16 @@
+from typing import Iterator, List, TypeVar
+
+T = TypeVar("T")
+
+
+def batch_of_length(
+    elements: List[T],
+    batch_size: int,
+) -> Iterator[List[T]]:
+    """
+    Split the given elements into smaller chunks
+    """
+    assert batch_size > 1, "batch size must be greater or equal to 1"
+    element_count = len(elements)
+    for index in range(0, element_count, batch_size):
+        yield elements[index : min((index + batch_size), element_count)]

castor_extractor-0.24.7/castor_extractor/utils/batch_test.py

@@ -0,0 +1,27 @@
+import pytest
+
+from .batch import batch_of_length
+
+
+def test_batch_of_length():
+    elements = ["a", "b", "c", "d", "e", "f", "g", "h"]
+    result = list(batch_of_length(elements, 3))
+    assert result == [
+        ["a", "b", "c"],
+        ["d", "e", "f"],
+        ["g", "h"],
+    ]
+
+    result = list(batch_of_length(elements, 1000))
+    assert result == [
+        elements,
+    ]
+
+    result = list(batch_of_length(elements, 7))
+    assert result == [
+        ["a", "b", "c", "d", "e", "f", "g"],
+        ["h"],
+    ]
+
+    with pytest.raises(AssertionError):
+        list(batch_of_length(elements, -12))

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/visualization/domo/client/client.py

@@ -9,6 +9,7 @@ import requests
 from ....utils import (
     RequestSafeMode,
     at_midnight,
+    batch_of_length,
     current_date,
     empty_iterator,
     handle_response,
@@ -48,6 +49,8 @@ _RETRY_BASE_MS = 10 * 60 * 1000  # 10 minutes
 
 _PARENT_FOLDER = "/Dashboards"
 
+_CARDS_BATCH_SIZE = 100
+
 logger = logging.getLogger(__name__)
 
 
@@ -156,16 +159,19 @@ class DomoClient:
 
         return all_results
 
+    def _cards_metadata(self, card_ids: list[int]) -> Iterator[dict]:
+        # batch to avoid hitting the URL max length
+        for batch_card_ids in batch_of_length(card_ids, _CARDS_BATCH_SIZE):
+            endpoint = self._endpoint_factory.cards_metadata(batch_card_ids)
+            yield from self._get_element(endpoint)
+
     def _datasources(self, card_ids: list[int]) -> RawData:
         """Yields all distinct datasources associated to the given cards"""
         if not card_ids:
             return empty_iterator()
 
-        endpoint = self._endpoint_factory.cards_metadata(card_ids)
-        cards_metadata = self._get_element(endpoint)
-
         processed: set[str] = set()
-        for card in cards_metadata:
+        for card in self._cards_metadata(card_ids):
             for datasource in card["datasources"]:
                 id_ = datasource["dataSourceId"]
                 if id_ in processed:

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/visualization/tableau/client/client_metadata_api.py

@@ -1,3 +1,4 @@
+import logging
 from collections.abc import Iterator
 from typing import Optional
 
@@ -9,15 +10,14 @@ from ..constants import DEFAULT_PAGE_SIZE
 from .errors import TableauApiError, TableauApiTimeout
 from .gql_queries import FIELDS_QUERIES, GQL_QUERIES, QUERY_TEMPLATE
 
+logger = logging.getLogger(__name__)
+
 # increase the value when extraction is too slow
 # decrease the value when timeouts arise
 _CUSTOM_PAGE_SIZE: dict[TableauAsset, int] = {
-    # for some clients, extraction of columns tend to hit the node limit
-    # https://community.tableau.com/s/question/0D54T00000YuK60SAF/metadata-query-nodelimitexceeded-error
-    # the workaround is to reduce pagination
-    TableauAsset.COLUMN: 50,
     # fields are light but volumes are bigger
     TableauAsset.FIELD: 1000,
+    # tables are sometimes heavy
     TableauAsset.TABLE: 50,
 }
 
@@ -51,8 +51,9 @@ def _check_errors(answer: dict) -> None:
 
 def gql_query_scroll(
     server,
-    query: str,
     resource: str,
+    fields: str,
+    page_size: int,
 ) -> Iterator[SerializedAsset]:
     """
     Iterate over GQL query results, handling pagination and cursor
@@ -67,23 +68,27 @@ def gql_query_scroll(
         max_retries=_RETRY_COUNT,
         base_ms=_RETRY_BASE_MS,
     )
-    def _call(cursor: Optional[str]) -> dict:
-        # If cursor is defined it must be quoted else use null token
-        token = "null" if cursor is None else f'"{cursor}"'
-        query_ = query.replace("AFTER_TOKEN_SIGNAL", token)
-        answer = server.metadata.query(query_)
+    def _call(first: int, offset: int) -> dict:
+        query = QUERY_TEMPLATE.format(
+            resource=resource,
+            fields=fields,
+            first=first,
+            offset=offset,
+        )
+        answer = server.metadata.query(query)
         _check_errors(answer)
         return answer["data"][f"{resource}Connection"]
 
-    cursor = None
+    current_offset = 0
     while True:
-        payload = _call(cursor)
+        payload = _call(first=page_size, offset=current_offset)
         yield payload["nodes"]
 
-        page_info = payload["pageInfo"]
-        if page_info["hasNextPage"]:
-            cursor = page_info["endCursor"]
-        else:
+        current_offset += len(payload["nodes"])
+        total = payload["totalCount"]
+        logger.info(f"Extracted {current_offset}/{total} {resource}")
+
+        if not payload["pageInfo"]["hasNextPage"]:
             break
 
 
@@ -107,12 +112,12 @@ class TableauClientMetadataApi:
         fields: str,
         page_size: int = DEFAULT_PAGE_SIZE,
     ) -> SerializedAsset:
-        query = QUERY_TEMPLATE.format(
+        result_pages = gql_query_scroll(
+            self._server,
             resource=resource,
             fields=fields,
             page_size=page_size,
         )
-        result_pages = gql_query_scroll(self._server, query, resource)
         return [asset for page in result_pages for asset in page]
 
     def _page_size(self, asset: TableauAsset) -> int:

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/visualization/tableau/client/gql_queries.py

@@ -2,7 +2,7 @@ from ..assets import TableauAsset
 
 QUERY_TEMPLATE = """
 {{
-  {resource}Connection(first: {page_size}, after: AFTER_TOKEN_SIGNAL) {{
+  {resource}Connection(first: {first}, offset: {offset}) {{
     nodes {{ {fields}
     }}
     pageInfo {{

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/visualization/thoughtspot/client/client.py

@@ -1,13 +1,17 @@
+import logging
 from collections.abc import Iterator
-from typing import Optional
+from functools import partial
+from typing import Iterable, Optional
 
 import requests
+from requests import Response
 
 from ....utils import (
     APIClient,
     BearerAuth,
     RequestSafeMode,
     build_url,
+    fetch_all_pages,
     handle_response,
 )
 from ..assets import (
@@ -19,9 +23,7 @@ from .credentials import (
 from .endpoints import (
     ThoughtspotEndpointFactory,
 )
-from .utils import (
-    usage_liveboard_reader,
-)
+from .pagination import METADATA_BATCH_SIZE, ThoughtSpotPagination
 
 _AUTH_TIMEOUT_S = 60
 _THOUGHTSPOT_HEADERS = {
@@ -29,7 +31,6 @@ _THOUGHTSPOT_HEADERS = {
     "Accept": "application/json",
     "Content-Type": "application/json",
 }
-_METADATA_BATCH_SIZE = 100
 # https://docs.thoughtspot.com/cloud/latest/object-usage-liveboard
 _OBJECT_USAGE_LIVEBOARD = "Object Usage"
 _ANSWER_USAGE_VIZ = "Answer Usage, by User"
@@ -40,6 +41,9 @@ _LIVEBOARD_USAGE_VIZ = "Popular Liveboards Last 30 Days"
 THOUGHTSPOT_SAFE_MODE = RequestSafeMode()
 
 
+logger = logging.getLogger(__name__)
+
+
 class ThoughtspotBearerAuth(BearerAuth):
     def __init__(self, host: str, token_payload: dict[str, str]):
         auth_endpoint = ThoughtspotEndpointFactory.authentication()
@@ -86,7 +90,7 @@ class ThoughtspotClient(APIClient):
             search_filters = {
                 "metadata": [{"type": metadata_type}],
                 "include_details": True,
-                "record_size": _METADATA_BATCH_SIZE,
+                "record_size": METADATA_BATCH_SIZE,
                 "record_offset": offset,
             }
             if identifier:
@@ -100,9 +104,9 @@ class ThoughtspotClient(APIClient):
                 data=search_filters,
             )
             yield from metadata
-            if len(metadata) < _METADATA_BATCH_SIZE:
+            if len(metadata) < METADATA_BATCH_SIZE:
                 break
-            offset = offset + _METADATA_BATCH_SIZE
+            offset = offset + METADATA_BATCH_SIZE
 
     def _get_all_answers(self) -> Iterator[dict]:
         yield from self._metadata_search(metadata_type="ANSWER")
@@ -120,7 +124,7 @@ class ThoughtspotClient(APIClient):
         self,
         liveboard_name: str,
         visualization_name: str,
-    ) -> Iterator[dict]:
+    ) -> Iterator[list[list]]:
         """
         Yields the data of a given visualization in the given liveboard.
         ThoughtSpot maintains two system liveboards with stats about data usage,
@@ -133,29 +137,62 @@ class ThoughtspotClient(APIClient):
         )
         liveboard_id = usage_liveboard["metadata_id"]
 
-        data = self._post(
-            endpoint=ThoughtspotEndpointFactory.liveboard(),
-            headers={"Accept": "application/octet-stream"},
+        def handler(response: Response) -> dict:
+            response_dict = response.json()
+            contents = response_dict.get("contents", [])
+            if not contents:
+                logger.warning("No data found in response")
+                return dict()
+            return contents[0]
+
+        request = partial(
+            self._post,
+            endpoint=ThoughtspotEndpointFactory.liveboard_data(),
             data={
                 "metadata_identifier": liveboard_id,
-                "file_format": "CSV",
                 "visualization_identifiers": [visualization_name],
+                "record_offset": 0,
+                "record_size": METADATA_BATCH_SIZE,
             },
-            handler=lambda x: x.text,
+            handler=handler,
         )
-        yield from usage_liveboard_reader(data)
+        yield from fetch_all_pages(request, ThoughtSpotPagination)
 
     def _get_answer_usages(self) -> Iterator[dict]:
-        return self._get_usages(
+        """
+        Returns the usage data of saved Answers, which is found in a visualization
+        of the "Object Usage" liveboard.
+        Each data row returned by the API is transformed from a list into a dictionary.
+        The columns are explicitly listed here because in the API response,
+        there is a mismatch between the number of column names and the number
+        of values per data row.
+        """
+        data: Iterable[list[list]] = self._get_usages(
             liveboard_name=_OBJECT_USAGE_LIVEBOARD,
             visualization_name=_ANSWER_USAGE_VIZ,
         )
+        columns = (
+            "Answer name",
+            "Number of unique users",
+            "Count of object interactions",
+        )
+        for row in data:
+            yield dict(zip(columns, row))
 
     def _get_liveboards_usages(self) -> Iterator[dict]:
-        return self._get_usages(
+        """
+        Returns the usage data of Liveboards, which is found in a visualization
+        of the "User Adoption" liveboard.
+        Each data row returned by the API is transformed from a list into a dictionary.
+        See `_get_answer_usages` regarding the columns list.
+        """
+        data: Iterable[list[list]] = self._get_usages(
             liveboard_name=_USER_ADOPTION_LIVEBOARD,
             visualization_name=_LIVEBOARD_USAGE_VIZ,
         )
+        columns = ("Pinboard", "Unique Number of User", "Pinboard Views")
+        for row in data:
+            yield dict(zip(columns, row))
 
     def fetch(self, asset: ThoughtspotAsset) -> Iterator[dict]:
         if asset == ThoughtspotAsset.ANSWERS:

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/visualization/thoughtspot/client/endpoints.py

@@ -8,5 +8,5 @@ class ThoughtspotEndpointFactory:
         return "api/rest/2.0/metadata/search"
 
     @classmethod
-    def liveboard(cls) -> str:
-        return "api/rest/2.0/report/liveboard"
+    def liveboard_data(cls) -> str:
+        return "api/rest/2.0/metadata/liveboard/data"

castor_extractor-0.24.7/castor_extractor/visualization/thoughtspot/client/pagination.py

@@ -0,0 +1,25 @@
+from pydantic import ConfigDict, Field
+
+from ....utils import PaginationModel
+
+METADATA_BATCH_SIZE = 100
+
+
+class ThoughtSpotPagination(PaginationModel):
+    data_rows: list = Field(default_factory=list)
+    record_offset: int
+    record_size: int
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        from_attributes=True,
+    )
+
+    def is_last(self) -> bool:
+        return len(self.data_rows) < METADATA_BATCH_SIZE
+
+    def next_page_payload(self) -> dict:
+        return {"record_offset": self.record_offset + METADATA_BATCH_SIZE}
+
+    def page_results(self) -> list:
+        return self.data_rows

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/warehouse/snowflake/queries/column.sql

@@ -47,7 +47,9 @@ FROM snowflake.account_usage.columns AS c
     JOIN snowflake.account_usage.tables AS t ON t.table_id = c.table_id
     JOIN tags_agg_columns ta ON c.column_id = ta.column_id
 WHERE TRUE
-    AND COALESCE(c.column_name, '') != ''
+    AND TRIM(COALESCE(c.column_name, '')) != ''
+    AND TRIM(COALESCE(t.table_name, '')) != ''
+    AND TRIM(COALESCE(s.schema_name, '')) != ''
     AND UPPER(c.table_catalog) NOT IN ('SNOWFLAKE', 'UTIL_DB')
     AND (
         c.deleted IS NULL

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/warehouse/snowflake/queries/query.sql

@@ -51,20 +51,28 @@ WHERE TRUE
     AND HOUR(CONVERT_TIMEZONE('UTC', start_time)) BETWEEN :hour_min AND :hour_max
     AND execution_status = 'SUCCESS'
     AND query_text != 'SELECT 1'
+    AND TRIM(COALESCE(query_text, '')) != ''
     AND query_type NOT IN (
-        'SHOW',
-        'USE',
-        'ROLLBACK',
-        'DESCRIBE',
         'ALTER_SESSION',
-        'PUT_FILES',
+        'BEGIN_TRANSACTION',
+        'CALL',
+        'COMMENT',
+        'COMMIT',
         'CREATE', -- create objects: stage|function|schema|procedure|file|storage|pipe|notification integration
-        'SET',
+        'DESCRIBE',
+        'DROP',
+        'EXPLAIN',
+        'GET_FILES',
         'GRANT',
-        'COMMIT',
+        'PUT_FILES',
+        'REFRESH_DYNAMIC_TABLE_AT_REFRESH_VERSION',
+        'REMOVE_FILES',
+        'REVOKE',
+        'ROLLBACK',
+        'SET',
+        'SHOW',
+        'TRUNCATE_TABLE',
+        'UNDROP',
         'UNLOAD',
-        'GET_FILES',
-        'DROP',
-        'BEGIN_TRANSACTION',
-        'REMOVE_FILES'
+        'USE'
     )

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/warehouse/snowflake/queries/schema.sql

@@ -16,6 +16,7 @@ WHERE TRUE
         deleted IS NULL
         OR deleted > CURRENT_TIMESTAMP - INTERVAL '1 day'
     )
+    AND TRIM(COALESCE(schema_name, '')) != ''
     {database_allowed}
     {database_blocked}
     AND CASE {has_fetch_transient} WHEN FALSE THEN NOT s.is_transient::BOOLEAN ELSE TRUE END

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/castor_extractor/warehouse/snowflake/queries/table.sql

@@ -41,8 +41,8 @@ FROM snowflake.account_usage.tables AS t
     JOIN snowflake.account_usage.schemata AS s ON s.schema_id = t.table_schema_id
     JOIN tags_agg_tables ta ON t.table_id = ta.table_id
 WHERE TRUE
-    AND t.table_name IS NOT NULL
-    AND t.table_name != ''
+    AND TRIM(COALESCE(t.table_name, '')) != ''
+    AND TRIM(COALESCE(s.schema_name, '')) != ''
     AND UPPER(t.table_catalog) NOT IN ('SNOWFLAKE', 'UTIL_DB')
     AND (
         t.deleted IS NULL

{castor_extractor-0.24.2 → castor_extractor-0.24.7}/pyproject.toml

@@ -1,9 +1,9 @@
-[build-system]
+    [build-system]
 requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.2"
+version = "0.24.7"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -32,7 +32,7 @@ google-api-core = ">=2.1.1, <3.0.0"
 google-api-python-client = {version = "^2.121.0", optional = true}
 google-auth = "^2"
 google-cloud-core = ">=2.1.0, <3.0.0"
-google-cloud-storage = "^2"
+google-cloud-storage = "^3.1.0"
 google-resumable-media = "^2.0.3"
 googleapis-common-protos = "^1.53.0"
 looker-sdk = { version = "^25.0.0", optional = true }
@@ -50,7 +50,7 @@ python-dateutil = ">=2.0.0, <=3.0.0"
 requests = ">=2.0.0, <3.0.0"
 snowflake-connector-python = { version = "^3.4.0", optional = true }
 snowflake-sqlalchemy = { version = "!=1.2.5, <2.0.0", optional = true }
-setuptools = ">=75.6"
+setuptools = ">=78.1"
 sqlalchemy = "~1.4"
 sqlalchemy-bigquery = { version = ">=1.0.0, <= 2.0.0", optional = true, extras = [
   "bqstorage",
@@ -66,7 +66,7 @@ pymssql = { version = "^2.2.11", optional = true }
 pymysql = { extras = ["rsa"], version = "^1.1.0", optional = true }
 
 # 3.7 has issue with urrllib dependency
-# https://castorglobal.slack.com/archives/C049PRU1GK1/p1738328811319309
+# https://coalescesoftware.slack.com/archives/C08HEQHS487/p1738328811319309
 databricks-sql-connector = { version = "3.6.0", optional = true }
 
 [tool.poetry.extras]

castor_extractor-0.24.2/castor_extractor/visualization/thoughtspot/client/utils.py

@@ -1,31 +0,0 @@
-import csv
-import re
-from collections.abc import Iterator
-from io import StringIO
-
-_END_OF_GENERATED_TEXT = r'^""$'
-
-
-def usage_liveboard_reader(usage_liveboard_csv: str) -> Iterator[dict]:
-    """
-    Converts a CSV string into an iterator of dictionaries after
-    ignoring the generated text that preceeds the actual CSV header row.
-    The generated block ends with a row containing only two double quotes.
-    Here is an example:
-
-        "Data extract produced by Castor on 09/19/2024 06:54"
-        "Filters applied on data :"
-        "User Action IN [pinboard_embed_view,pinboard_tspublic_no_runtime_filter,pinboard_tspublic_runtime_filter,pinboard_view]"
-        "Pinboard NOT IN [mlm - availability pinboard,null]"
-        "Timestamp >= 20240820 00:00:00 < 20240919 00:00:00"
-        "Timestamp >= 20240919 00:00:00 < 20240920 00:00:00"
-        ""
-
-    """
-    csv_file = StringIO(usage_liveboard_csv)
-
-    line = next(csv_file)
-    while not re.match(_END_OF_GENERATED_TEXT, line.strip()):
-        line = next(csv_file)
-
-    yield from csv.DictReader(csv_file)