castor-extractor 0.21.7__py3-none-any.whl → 0.22.0__py3-none-any.whl

castor_extractor/utils/object.py

@@ -1,6 +1,7 @@
+from collections.abc import Iterator
 from datetime import date, datetime
 from enum import Enum
-from typing import Any, Iterator, Union, overload
+from typing import Any, Union, overload
 from uuid import UUID
 
 from .type import Getter

castor_extractor/utils/pager/pager.py

@@ -1,13 +1,11 @@
 from abc import abstractmethod
+from collections.abc import Iterator, Sequence
 from enum import Enum
 from itertools import chain
 from typing import (
     Callable,
     Generic,
-    Iterator,
-    List,
     Optional,
-    Sequence,
     TypeVar,
 )
 
@@ -32,7 +30,7 @@ class PagerStopStrategy(Enum):
 
 
 class AbstractPager(Generic[T]):
-    def all(self, per_page: int = DEFAULT_PER_PAGE) -> List[T]:
+    def all(self, per_page: int = DEFAULT_PER_PAGE) -> list[T]:
         """Returns all data provided by the callback as a list"""
         return list(chain.from_iterable(self.iterator(per_page=per_page)))
 

castor_extractor/utils/pager/pager_on_id.py

@@ -1,4 +1,5 @@
-from typing import Callable, Iterator, Optional, Protocol, Sequence, TypeVar
+from collections.abc import Iterator, Sequence
+from typing import Callable, Optional, Protocol, TypeVar
 from uuid import UUID
 
 from .pager import DEFAULT_PER_PAGE, AbstractPager, PagerStopStrategy

castor_extractor/utils/pager/pager_on_id_test.py

@@ -1,4 +1,4 @@
-from typing import Callable, Dict, List
+from typing import Callable
 from uuid import UUID
 
 from .pager_on_id import PagerOnId
@@ -9,11 +9,11 @@ ITEMS_WITH_IDS = [
 
 
 def _make_callback_with_ids(
-    elements: List[Dict[str, str]],
-) -> Callable[[UUID, int], List[Dict[str, str]]]:
-    def _callback(max_id: UUID, per: int) -> List[Dict[str, str]]:
+    elements: list[dict[str, str]],
+) -> Callable[[UUID, int], list[dict[str, str]]]:
+    def _callback(max_id: UUID, per: int) -> list[dict[str, str]]:
         """assumes the elements are sorted by id"""
-        to_return: List[Dict[str, str]] = []
+        to_return: list[dict[str, str]] = []
         for element in elements:
             if element["id"] > str(max_id):
                 to_return.append(element)

castor_extractor/utils/pager/pager_test.py

@@ -1,10 +1,10 @@
-from typing import Callable, List
+from typing import Callable
 
 from .pager import Pager, PagerLogger
 
 
-def _make_callback(elements: List[int]) -> Callable[[int, int], List[int]]:
-    def _callback(page: int, per: int) -> List[int]:
+def _make_callback(elements: list[int]) -> Callable[[int, int], list[int]]:
+    def _callback(page: int, per: int) -> list[int]:
         _start = (page - 1) * per
         _end = _start + per
         return elements[_start:_end]

castor_extractor/utils/retry.py

@@ -1,8 +1,9 @@
 import logging
 import random
 import time
+from collections.abc import Sequence
 from enum import Enum
-from typing import Any, Callable, Sequence, Tuple, Type, Union
+from typing import Any, Callable, Union
 
 from pydantic import BaseModel, PositiveInt, PrivateAttr
 from pydantic.fields import Field
@@ -75,11 +76,11 @@ class Retry(BaseModel):
         return True
 
 
-WrapperReturnType = Union[Tuple[BaseException, None], Tuple[None, Any]]
+WrapperReturnType = Union[tuple[BaseException, None], tuple[None, Any]]
 
 
 def retry(
-    exceptions: Sequence[Type[BaseException]],
+    exceptions: Sequence[type[BaseException]],
     max_retries: int = 1,
     base_ms: int = 0,
     jitter_ms: int = 1,

castor_extractor/utils/retry_test.py

@@ -1,7 +1,6 @@
 from http import HTTPStatus
 from statistics import variance
 from time import time
-from typing import List
 from unittest.mock import patch
 
 import pytest
@@ -43,8 +42,8 @@ def test_retry_strategy__jitter():
     assert variance(jitters) > 0
 
 
-def _iterate_base(retry: Retry, count: int) -> List[int]:
-    bases: List[int] = []
+def _iterate_base(retry: Retry, count: int) -> list[int]:
+    bases: list[int] = []
     for _ in range(count):
         retry._retry_attempts += 1
         bases.append(retry.base())

castor_extractor/utils/safe.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Callable, List, Optional, Tuple, Type, Union
+from typing import Callable, Optional, Union
 
 logger = logging.getLogger(__name__)
 
@@ -16,12 +16,12 @@ class SafeMode:
 
     def __init__(
         self,
-        exceptions: Tuple[Type[BaseException], ...],
+        exceptions: tuple[type[BaseException], ...],
         max_errors: Union[int, float],
     ):
         self.exceptions = exceptions
         self.max_errors = max_errors
-        self.errors_caught: List[Type[BaseException]] = []
+        self.errors_caught: list[type[BaseException]] = []
 
     @property
     def should_raise(self) -> bool:

castor_extractor/utils/salesforce/client.py

@@ -1,6 +1,7 @@
 import logging
+from collections.abc import Iterator
 from functools import partial
-from typing import Iterator, Optional
+from typing import Optional
 
 import requests
 

castor_extractor/utils/salesforce/credentials.py

@@ -1,5 +1,3 @@
-from typing import Dict
-
 from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -29,7 +27,7 @@ class SalesforceCredentials(BaseSettings):
         """Generates the password for authentication"""
         return self.password + self.security_token
 
-    def token_request_payload(self) -> Dict[str, str]:
+    def token_request_payload(self) -> dict[str, str]:
         """
         Params to post to the API in order to retrieve the authentication token
         """

castor_extractor/utils/store.py

@@ -1,7 +1,8 @@
 import os
 from abc import ABC, abstractmethod
+from collections.abc import Iterable, Iterator
 from io import StringIO
-from typing import Iterable, Iterator, Optional
+from typing import Optional
 
 from .formatter import CsvFormatter, Formatter
 from .time import current_timestamp

castor_extractor/utils/string.py

@@ -1,7 +1,7 @@
 import ast
 import re
 import string
-from typing import Tuple, TypeVar, Union
+from typing import TypeVar, Union
 
 _ALPHANUMERIC_REGEX = r"^\w+$"
 _SYMBOLS = "[]{}()"
@@ -31,7 +31,7 @@ def _clean(input_: str) -> str:
     return cleaned
 
 
-def string_to_tuple(input_: str) -> Tuple[str, ...]:
+def string_to_tuple(input_: str) -> tuple[str, ...]:
     """
     Parse the given string and returns the corresponding Tuple of strings
 

castor_extractor/utils/string_test.py

@@ -1,11 +1,9 @@
-from typing import Tuple
-
 import pytest
 
 from .string import decode_when_bytes, string_to_tuple
 
 
-def _test(symbols: str, input_: str) -> Tuple[str, ...]:
+def _test(symbols: str, input_: str) -> tuple[str, ...]:
     return string_to_tuple(symbols[0] + input_ + symbols[1])
 
 

castor_extractor/utils/type.py

@@ -1,6 +1,7 @@
-from typing import Any, Callable, Dict, List, Mapping, Sequence, Union
+from collections.abc import Mapping, Sequence
+from typing import Any, Callable, Union
 
-SerializedAsset = List[Dict]
+SerializedAsset = list[dict]
 
 # https://stackoverflow.com/questions/51291722/define-a-jsonable-type-using-mypy-pep-526
 JsonType = Union[Sequence, Mapping, str, int, float, bool]

castor_extractor/utils/validation.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Tuple
+from typing import Optional
 from urllib.parse import urlsplit, urlunsplit
 
 BASE_URL_SCHEME = "https"
@@ -18,13 +18,13 @@ def _clean_path(path: str) -> str:
     return path.rstrip("/")
 
 
-def _get_hostname_port(netloc: str) -> Tuple[str, str]:
+def _get_hostname_port(netloc: str) -> tuple[str, str]:
     hostname, *rest = netloc.split(":")
     port = ":".join(rest)
     return hostname, port
 
 
-def _urlsplit(base_url: str) -> Tuple[str, str, str, str, str, str]:
+def _urlsplit(base_url: str) -> tuple[str, str, str, str, str, str]:
     """
     Returns URL split parts
 
@@ -51,7 +51,7 @@ def _urlsplit(base_url: str) -> Tuple[str, str, str, str, str, str]:
 
 def _expect(
     attr: str,
-    expected: Optional[List[str]],
+    expected: Optional[list[str]],
     actual: Optional[str],
 ) -> None:
     if not expected and not actual:

castor_extractor/utils/write.py

@@ -3,7 +3,7 @@ import logging
 import os
 import sys
 from datetime import datetime
-from typing import Any, List
+from typing import Any
 
 import pkg_resources
 
@@ -70,7 +70,7 @@ def get_summary_payload(client_info: dict, dt: datetime) -> dict:
     }
 
 
-def write_errors_logs(output_directory: str, ts: int, errors: List[str]):
+def write_errors_logs(output_directory: str, ts: int, errors: list[str]):
     """
     write a json file logs from code execution
     """

castor_extractor/visualization/domo/client/client.py

@@ -1,7 +1,8 @@
 import logging
+from collections.abc import Iterator
 from datetime import datetime, timedelta
 from http import HTTPStatus
-from typing import Any, Iterator, List, Optional, Set
+from typing import Any, Optional
 
 import requests
 
@@ -137,14 +138,14 @@ class DomoClient:
         endpoint: Endpoint,
         params: Optional[dict] = None,
         asset_id: Optional[str] = None,
-    ) -> List[dict]:
+    ) -> list[dict]:
         """Used when the response contains multiple elements"""
         return self._get(endpoint, params, asset_id)
 
-    def _get_paginated(self, endpoint: Endpoint) -> List[dict]:
+    def _get_paginated(self, endpoint: Endpoint) -> list[dict]:
         """Used when the response is paginated and need iterations"""
         pagination = Pagination()
-        all_results: List[dict] = []
+        all_results: list[dict] = []
 
         while pagination.needs_increment:
             params = {**pagination.params, **endpoint.params}
@@ -155,7 +156,7 @@ class DomoClient:
 
         return all_results
 
-    def _datasources(self, card_ids: List[int]) -> RawData:
+    def _datasources(self, card_ids: list[int]) -> RawData:
         """Yields all distinct datasources associated to the given cards"""
         if not card_ids:
             return empty_iterator()
@@ -178,7 +179,7 @@ class DomoClient:
 
     def _process_pages(
         self,
-        page_tree: List[dict],
+        page_tree: list[dict],
         parent_path: str = _PARENT_FOLDER,
     ) -> Iterator[dict]:
         """Recursively fetch pages while building the folder architecture"""
@@ -252,7 +253,7 @@ class DomoClient:
             }
 
     def _unique_datasets(self) -> RawData:
-        ids_encountered: Set[str] = set()
+        ids_encountered: set[str] = set()
         for dataset in self._datasets():
             dataset_id = dataset.get("id")
             if not dataset_id or dataset_id in ids_encountered:

castor_extractor/visualization/domo/client/credentials.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional
+from typing import Optional
 
 from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -27,5 +27,5 @@ class DomoCredentials(BaseSettings):
         return HTTPBasicAuth(self.client_id, self.api_token)
 
     @property
-    def private_headers(self) -> Dict[str, str]:
+    def private_headers(self) -> dict[str, str]:
         return {"X-DOMO-Developer-Token": self.developer_token}

castor_extractor/visualization/domo/client/endpoints.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import List, Optional
+from typing import Optional
 
 _AUTH_URL = (
     "grant_type=client_credentials&scope=data%20dashboard%20audit%20user"
@@ -79,7 +79,7 @@ class EndpointFactory:
             is_private=True,
         )
 
-    def cards_metadata(self, card_ids: List[int]) -> Endpoint:
+    def cards_metadata(self, card_ids: list[int]) -> Endpoint:
         urns = ",".join(map(str, card_ids))
         url = f"{self.base_url}/api/content/v1/cards?urns={urns}&parts=datasources"
 

castor_extractor/visualization/domo/extract.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Iterable, Iterator, Tuple, Union
+from collections.abc import Iterable, Iterator
+from typing import Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -18,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 def iterate_all_data(
     client: DomoClient,
-) -> Iterable[Tuple[DomoAsset, Union[list, Iterator, dict]]]:
+) -> Iterable[tuple[DomoAsset, Union[list, Iterator, dict]]]:
     """Iterate over the extracted data from Domo"""
 
     logger.info("Extracting PAGES from API")

castor_extractor/visualization/looker/api/client.py

@@ -1,7 +1,8 @@
 import functools
 import logging
+from collections.abc import Iterator, Sequence
 from datetime import date, timedelta
-from typing import Callable, Iterator, List, Optional, Sequence, Tuple
+from typing import Callable, Optional
 
 from dateutil.utils import today
 from looker_sdk import init40
@@ -98,7 +99,7 @@ class ApiClient:
         self.per_page = page_size
         self._safe_mode = safe_mode
 
-    def folders(self) -> List[Folder]:
+    def folders(self) -> list[Folder]:
         """Lists folders of the given Looker account"""
 
         def _search(page: int, per_page: int) -> Sequence[Folder]:
@@ -110,7 +111,7 @@ class ApiClient:
 
         return Pager(_search, logger=self._logger).all(per_page=self.per_page)
 
-    def dashboards(self, folder_id: Optional[str] = None) -> List[Dashboard]:
+    def dashboards(self, folder_id: Optional[str] = None) -> list[Dashboard]:
         """
         Lists dashboards of the given Looker account using pagination.
         The optional folder_id allows restricting the search to the given folder.
@@ -132,7 +133,7 @@ class ApiClient:
 
         return Pager(_search, logger=self._logger).all(per_page=self.per_page)
 
-    def looks(self, folder_id: Optional[str] = None) -> List[Look]:
+    def looks(self, folder_id: Optional[str] = None) -> list[Look]:
         """
         Fetch looks via `search_looks` using pagination. The optional folder_id
         allows restricting the search to the given folder.
@@ -155,7 +156,7 @@ class ApiClient:
 
         return Pager(_search, logger=self._logger).all(per_page=self.per_page)
 
-    def _all_looks(self) -> List[Look]:
+    def _all_looks(self) -> list[Look]:
         """
         fetch looks via `all_looks`
         https://castor.cloud.looker.com/extensions/marketplace_extension_api_explorer::api-explorer/4.0/methods/Look/all_looks
@@ -163,7 +164,7 @@ class ApiClient:
         # No pagination : see https://community.looker.com/looker-api-77/api-paging-limits-14598
         return list(self._sdk.all_looks(fields=format_fields(LOOK_FIELDS)))
 
-    def users(self) -> List[User]:
+    def users(self) -> list[User]:
         """Lists users of the given Looker account"""
 
         def _search(page: int, per_page: int) -> Sequence[User]:
@@ -179,7 +180,7 @@ class ApiClient:
 
         return Pager(_search, logger=self._logger).all(per_page=self.per_page)
 
-    def lookml_models(self) -> List[LookmlModel]:
+    def lookml_models(self) -> list[LookmlModel]:
         """Iterates LookML models of the given Looker account"""
 
         models = self._sdk.all_lookml_models(
@@ -197,7 +198,7 @@ class ApiClient:
 
     def explores(
         self,
-        explore_names=Iterator[Tuple[str, str]],
+        explore_names=Iterator[tuple[str, str]],
     ) -> Iterator[LookmlModelExplore]:
         """Iterates explores of the given Looker account for the provided model/explore names"""
 
@@ -214,7 +215,7 @@ class ApiClient:
             if explore_ is not None:
                 yield explore_
 
-    def connections(self) -> List[DBConnection]:
+    def connections(self) -> list[DBConnection]:
         """Lists databases connections of the given Looker account"""
 
         connections = self._sdk.all_connections(
@@ -226,7 +227,7 @@ class ApiClient:
 
         return list(connections)
 
-    def projects(self) -> List[Project]:
+    def projects(self) -> list[Project]:
         """Lists projects of the given Looker account"""
 
         projects = self._sdk.all_projects(fields=format_fields(PROJECT_FIELDS))
@@ -236,7 +237,7 @@ class ApiClient:
 
         return list(projects)
 
-    def groups_hierarchy(self) -> List[GroupHierarchy]:
+    def groups_hierarchy(self) -> list[GroupHierarchy]:
         """Lists groups with hierarchy of the given Looker account"""
         groups_hierarchy = self._sdk.search_groups_with_hierarchy(
             fields=format_fields(GROUPS_HIERARCHY_FIELDS),
@@ -244,7 +245,7 @@ class ApiClient:
         logger.info("All looker groups_hierarchy fetched")
         return list(groups_hierarchy)
 
-    def groups_roles(self) -> List[GroupSearch]:
+    def groups_roles(self) -> list[GroupSearch]:
         """Lists groups with roles of the given Looker account"""
         groups_roles = self._sdk.search_groups_with_roles(
             fields=format_fields(GROUPS_ROLES_FIELDS),
@@ -252,12 +253,12 @@ class ApiClient:
         logger.info("All looker groups_roles fetched")
         return list(groups_roles)
 
-    def content_views(self) -> List[ContentView]:
+    def content_views(self) -> list[ContentView]:
         """
         List the number of views per {user x week x dashboard|look}
         https://cloud.google.com/looker/docs/reference/looker-api/latest/types/ContentView
         """
-        content_views: List[ContentView] = []
+        content_views: list[ContentView] = []
 
         for day in _mondays(history_depth_in_days=CONTENT_VIEWS_HISTORY_DAYS):
             formatted_day = day.strftime("%Y-%m-%d")
@@ -279,7 +280,7 @@ class ApiClient:
         )
         return content_views
 
-    def users_attributes(self) -> List[UserAttribute]:
+    def users_attributes(self) -> list[UserAttribute]:
         """Lists user attributes of the given Looker account"""
         user_attributes = list(
             self._sdk.all_user_attributes(
@@ -298,7 +299,7 @@ class ApiClient:
         asset: LookerAsset,
         *,
         folder_id: Optional[str] = None,
-        explore_names: Optional[Iterator[Tuple[str, str]]] = None,
+        explore_names: Optional[Iterator[tuple[str, str]]] = None,
     ) -> list:
         if asset == LookerAsset.USERS:
             return self.users()

castor_extractor/visualization/looker/api/utils.py

@@ -1,11 +1,11 @@
-from typing import Iterable, Set, Tuple
+from collections.abc import Iterable
 
 from looker_sdk.sdk.api40.models import LookmlModel
 
 
 def lookml_explore_names(
     lookmls: Iterable[LookmlModel],
-) -> Set[Tuple[str, str]]:
+) -> set[tuple[str, str]]:
     """
     Explores from the lookml models
     Only valid explores are yielded: with all infos

castor_extractor/visualization/looker/assets.py

@@ -1,5 +1,3 @@
-from typing import Set
-
 from ...types import ExternalAsset, classproperty
 
 
@@ -20,7 +18,7 @@ class LookerAsset(ExternalAsset):
     USERS_ATTRIBUTES = "users_attributes"
 
     @classproperty
-    def optional(cls) -> Set["LookerAsset"]:
+    def optional(cls) -> set["LookerAsset"]:
         return {
             LookerAsset.CONNECTIONS,
             LookerAsset.CONTENT_VIEWS,

castor_extractor/visualization/looker/extract.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Iterable, Optional, Set, Tuple, Union
+from collections.abc import Iterable
+from typing import Optional, Union
 
 from looker_sdk.sdk.api40.models import LookmlModel
 
@@ -58,7 +59,7 @@ def iterate_all_data(
     search_per_folder: bool,
     thread_pool_size: int,
     log_to_stdout: bool,
-) -> Iterable[Union[StreamableList, Tuple[LookerAsset, list]]]:
+) -> Iterable[Union[StreamableList, tuple[LookerAsset, list]]]:
     """Iterate over the extracted Data From looker"""
 
     logger.info("Extracting users from Looker API")
@@ -67,7 +68,7 @@ def iterate_all_data(
 
     logger.info("Extracting folders from Looker API")
     folders = client.folders()
-    folder_ids: Set[str] = {folder.id for folder in folders if folder.id}
+    folder_ids: set[str] = {folder.id for folder in folders if folder.id}
     yield LookerAsset.FOLDERS, deep_serialize(folders)
 
     logger.info("Extracting looks from Looker API")

castor_extractor/visualization/looker/fields.py

@@ -1,7 +1,7 @@
-from typing import Any, Dict, Tuple, Union, cast
+from typing import Any, Union, cast
 
-Field = Union[str, Dict[str, Any]]
-Fields = Union[Field, Tuple[Field, ...]]
+Field = Union[str, dict[str, Any]]
+Fields = Union[Field, tuple[Field, ...]]
 
 
 def format_fields(fields: Fields) -> str:

castor_extractor/visualization/looker/multithreading.py

@@ -1,8 +1,8 @@
 import logging
 import sys
+from collections.abc import Iterable
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from typing import Iterable, List, Set
 
 from looker_sdk.error import SDKError
 from tqdm import tqdm  # type: ignore
@@ -31,7 +31,7 @@ def _make_api_request(
     client: ApiClient,
     asset: LookerAsset,
     folder_id: str,
-) -> List:
+) -> list:
     """
     Calls the appropriate Looker API endpoint to retrieve either Looks or
     Dashboards withered by the given folder ID.
@@ -44,7 +44,7 @@ def _make_api_request(
 class MultithreadingFetcher:
     def __init__(
         self,
-        folder_ids: Set[str],
+        folder_ids: set[str],
         client: ApiClient,
         thread_pool_size: int,
         log_to_stdout: bool,

castor_extractor/visualization/metabase/assets.py

@@ -1,5 +1,3 @@
-from typing import Dict, Tuple
-
 from ...types import ExternalAsset
 
 
@@ -15,7 +13,7 @@ class MetabaseAsset(ExternalAsset):
     DASHBOARD_CARDS = "dashboard_cards"
 
 
-EXPORTED_FIELDS: Dict[MetabaseAsset, Tuple[str, ...]] = {
+EXPORTED_FIELDS: dict[MetabaseAsset, tuple[str, ...]] = {
     MetabaseAsset.COLLECTION: (
         "id",
         "name",