castor-extractor 0.21.7__py3-none-any.whl → 0.22.0__py3-none-any.whl

castor_extractor/visualization/metabase/client/api/client.py

@@ -1,6 +1,7 @@
 import logging
+from collections.abc import Iterator
 from http import HTTPStatus
-from typing import Any, Dict, Iterator, List, Optional, Set, cast
+from typing import Any, Optional, cast
 
 import requests
 from requests import HTTPError
@@ -78,7 +79,7 @@ class ApiClient:
     @staticmethod
     def _answer(response: Any):
         answer = response
-        if isinstance(answer, Dict) and DATA_KEY in answer:
+        if isinstance(answer, dict) and DATA_KEY in answer:
             # v0.41 of Metabase introduced embedded data for certain calls
             # {'data': [{ }, ...] , 'total': 15, 'limit': None, 'offset': None}"
             return answer[DATA_KEY]
@@ -123,7 +124,7 @@ class ApiClient:
     def _fetch_ids(self, asset: MetabaseAsset) -> IdsType:
         ids: IdsType = []
         results = self._call(endpoint=asset.name.lower())
-        for res in cast(List, results):
+        for res in cast(list, results):
             assert isinstance(res, dict)
             ids.append(res["id"])
         return ids
@@ -135,7 +136,7 @@ class ApiClient:
             if not collection:
                 continue
 
-            seen_dashboard_ids: Set[int] = set()
+            seen_dashboard_ids: set[int] = set()
 
             for dashboard in cast(SerializedAsset, collection):
                 if dashboard.get("model") != "dashboard":
@@ -149,7 +150,7 @@ class ApiClient:
 
                 if dashboard_id not in seen_dashboard_ids:
                     seen_dashboard_ids.add(dashboard_id)
-                    yield cast(Dict, self._call(f"dashboard/{dashboard_id}"))
+                    yield cast(dict, self._call(f"dashboard/{dashboard_id}"))
 
     @staticmethod
     def _collection_specifics(collections: SerializedAsset) -> SerializedAsset:
@@ -170,7 +171,7 @@ class ApiClient:
         return databases
 
     @staticmethod
-    def _dashboard_cards(dashboards: SerializedAsset) -> Iterator[Dict]:
+    def _dashboard_cards(dashboards: SerializedAsset) -> Iterator[dict]:
         for d in dashboards:
             d_cards = d.get(CARDS_KEY) or d.get(CARDS_KEY_DEPRECATED) or []
             yield from d_cards
@@ -186,7 +187,7 @@ class ApiClient:
 
         else:
             answer = self._call(asset.name.lower())
-            assets = cast(List, answer)
+            assets = cast(list, answer)
 
         if asset == MetabaseAsset.DATABASE:
             assets = self._database_specifics(assets)

castor_extractor/visualization/metabase/extract.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Iterable, Tuple, Union
+from collections.abc import Iterable
+from typing import Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -20,7 +21,7 @@ ClientMetabase = Union[DbClient, ApiClient]
 
 def iterate_all_data(
     client: ClientMetabase,
-) -> Iterable[Tuple[MetabaseAsset, list]]:
+) -> Iterable[tuple[MetabaseAsset, list]]:
     """Iterate over the extracted Data From metabase"""
 
     yield MetabaseAsset.USER, deep_serialize(client.fetch(MetabaseAsset.USER))

castor_extractor/visualization/metabase/types.py

@@ -1,3 +1 @@
-from typing import List
-
-IdsType = List[str]
+IdsType = list[str]

castor_extractor/visualization/mode/client/client.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, List, Optional, cast
+from typing import Optional, cast
 
 import requests
 from requests.auth import HTTPBasicAuth
@@ -29,8 +29,8 @@ logger = logging.getLogger(__name__)
 
 URL_TEMPLATE = "{host}/api"
 
-RawData = List[Dict]
-Tokens = Optional[List[str]]
+RawData = list[dict]
+Tokens = Optional[list[str]]
 
 
 class Client:
@@ -76,8 +76,8 @@ class Client:
 
         # most of calls return data in ["_embedded"]["resource_name"] node
         try:
-            embedded = cast(Dict, result["_embedded"])
-            return cast(List, embedded[resource_name])
+            embedded = cast(dict, result["_embedded"])
+            return cast(list, embedded[resource_name])
         except (ValueError, KeyError):
             raise UnexpectedApiResponseError(resource_name, result)
 
@@ -168,7 +168,7 @@ class Client:
                 resource_name=mb["member_username"],
                 with_workspace=False,
             )
-            members.append(cast(Dict, result))
+            members.append(cast(dict, result))
         return members
 
     @staticmethod

castor_extractor/visualization/mode/extract.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Iterable, Tuple
+from collections.abc import Iterable
 
 from ...utils import (
     OUTPUT_DIR,
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 
 def iterate_all_data(
     client: Client,
-) -> Iterable[Tuple[Asset, list]]:
+) -> Iterable[tuple[Asset, list]]:
     """Iterate over the extracted Data From Mode Analytics"""
 
     datasources = client.fetch(Asset.DATASOURCE)

castor_extractor/visualization/powerbi/assets.py

@@ -1,5 +1,3 @@
-from typing import Set
-
 from ...types import ExternalAsset, classproperty
 
 
@@ -18,7 +16,7 @@ class PowerBiAsset(ExternalAsset):
     USERS = "users"
 
     @classproperty
-    def optional(cls) -> Set["PowerBiAsset"]:
+    def optional(cls) -> set["PowerBiAsset"]:
         return {
             PowerBiAsset.DATASET_FIELDS,
             PowerBiAsset.PAGES,

castor_extractor/visualization/powerbi/client/client.py

@@ -1,8 +1,9 @@
 import logging
+from collections.abc import Iterator
 from datetime import date
 from functools import partial
 from time import sleep
-from typing import Dict, Iterator, List, Optional, Union
+from typing import Optional, Union
 
 import requests
 from requests import HTTPError
@@ -40,7 +41,7 @@ class PowerbiClient(APIClient):
             timeout=POWERBI_DEFAULT_TIMEOUT_S,
         )
 
-    def _activity_events(self, day: Optional[date] = None) -> Iterator[Dict]:
+    def _activity_events(self, day: Optional[date] = None) -> Iterator[dict]:
         """
         Returns a list of activity events for the organization.
         https://learn.microsoft.com/en-us/power-bi/admin/service-admin-auditing#activityevents-rest-api
@@ -52,21 +53,21 @@ class PowerbiClient(APIClient):
         )
         yield from fetch_all_pages(request, PowerBiPagination)
 
-    def _datasets(self) -> Iterator[Dict]:
+    def _datasets(self) -> Iterator[dict]:
         """
         Returns a list of datasets for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/datasets-get-datasets-as-admin
         """
         yield from self._get(PowerBiEndpointFactory.datasets())[Keys.VALUE]
 
-    def _dashboards(self) -> Iterator[Dict]:
+    def _dashboards(self) -> Iterator[dict]:
         """
         Returns a list of dashboards for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/dashboards-get-dashboards-as-admin
         """
         yield from self._get(PowerBiEndpointFactory.dashboards())[Keys.VALUE]
 
-    def _reports(self) -> Iterator[Dict]:
+    def _reports(self) -> Iterator[dict]:
         """
         Returns a list of reports for the organization.
         https://learn.microsoft.com/en-us/rest/api/power-bi/admin/reports-get-reports-as-admin
@@ -87,12 +88,12 @@ class PowerbiClient(APIClient):
 
         return reports
 
-    def _workspace_ids(self) -> List[str]:
+    def _workspace_ids(self) -> list[str]:
         """
         Get workspaces ids from powerBI admin API.
         more: https://learn.microsoft.com/en-us/rest/api/power-bi/admin/workspace-info-get-modified-workspaces
         """
-        params: Dict[str, Union[bool, str]] = {
+        params: dict[str, Union[bool, str]] = {
             Keys.INACTIVE_WORKSPACES: True,
             Keys.PERSONAL_WORKSPACES: True,
         }
@@ -104,7 +105,7 @@ class PowerbiClient(APIClient):
 
         return [x[Keys.ID] for x in response]
 
-    def _get_scan_result(self, scan_id: int) -> Iterator[Dict]:
+    def _get_scan_result(self, scan_id: int) -> Iterator[dict]:
         endpoint = PowerBiEndpointFactory.metadata_scan_result(scan_id)
         yield from self._get(endpoint)[Keys.WORKSPACES]
 
@@ -136,7 +137,7 @@ class PowerbiClient(APIClient):
         logger.warning(f"Scan {scan_id} timed out")
         return False
 
-    def _create_scan(self, workspaces_ids: List[str]) -> int:
+    def _create_scan(self, workspaces_ids: list[str]) -> int:
         """
         Tells the Power BI API to start an asynchronous metadata scan.
         Returns the scan's ID.
@@ -157,7 +158,7 @@ class PowerbiClient(APIClient):
         )
         return scan_id[Keys.ID]
 
-    def _metadata(self) -> Iterator[Dict]:
+    def _metadata(self) -> Iterator[dict]:
         """
         Fetch metadata by workspace. The metadata scanning is asynchronous and
         requires the following steps:
@@ -183,7 +184,7 @@ class PowerbiClient(APIClient):
         asset: PowerBiAsset,
         *,
         day: Optional[date] = None,
-    ) -> Iterator[Dict]:
+    ) -> Iterator[dict]:
         """
         Given a PowerBi asset, returns the corresponding data using the
         appropriate client.

castor_extractor/visualization/powerbi/client/credentials.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import Optional
 
 from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -19,9 +19,9 @@ class PowerbiCredentials(BaseSettings):
     client_id: str
     tenant_id: str
     secret: str = Field(repr=False)
-    scopes: List[str] = [DEFAULT_SCOPE]
+    scopes: list[str] = [DEFAULT_SCOPE]
 
     @field_validator("scopes", mode="before")
     @classmethod
-    def _check_scopes(cls, scopes: Optional[List[str]]) -> List[str]:
+    def _check_scopes(cls, scopes: Optional[list[str]]) -> list[str]:
         return scopes if scopes is not None else [DEFAULT_SCOPE]

castor_extractor/visualization/powerbi/client/endpoints.py

@@ -1,5 +1,5 @@
 from datetime import date, datetime
-from typing import Optional, Tuple
+from typing import Optional
 
 from ....utils import at_midnight, format_date, yesterday
 
@@ -7,7 +7,7 @@ _CLIENT_APP_BASE = "https://login.microsoftonline.com"
 _REST_API_BASE_PATH = "https://api.powerbi.com/v1.0/myorg"
 
 
-def _time_filter(day: Optional[date]) -> Tuple[datetime, datetime]:
+def _time_filter(day: Optional[date]) -> tuple[datetime, datetime]:
     target_day = day or yesterday()
     start = at_midnight(target_day)
     end = datetime.combine(target_day, datetime.max.time())

castor_extractor/visualization/powerbi/extract.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Iterable, Tuple, Union
+from collections.abc import Iterable
+from typing import Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -18,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 def iterate_all_data(
     client: PowerbiClient,
-) -> Iterable[Tuple[PowerBiAsset, Union[list, dict]]]:
+) -> Iterable[tuple[PowerBiAsset, Union[list, dict]]]:
     for asset in PowerBiAsset:
         if asset in METADATA_ASSETS + REPORTS_ASSETS:
             continue

castor_extractor/visualization/qlik/assets.py

@@ -1,5 +1,3 @@
-from typing import Dict, Tuple
-
 from ...types import ExternalAsset
 
 
@@ -14,7 +12,7 @@ class QlikAsset(ExternalAsset):
     CONNECTIONS = "connections"
 
 
-EXPORTED_FIELDS: Dict[QlikAsset, Tuple[str, ...]] = {
+EXPORTED_FIELDS: dict[QlikAsset, tuple[str, ...]] = {
     QlikAsset.SPACES: (
         "id",
         "type",

castor_extractor/visualization/qlik/client/constants.py

@@ -1,5 +1,3 @@
-from typing import Dict
-
 from ..assets import QlikAsset
 
 APP_EXTERNAL_ID_KEY = "resourceId"
@@ -17,7 +15,7 @@ RESPONSE_DICT_EXPECTED_MSG = _RESPONSE_BASE_EXPECTED_MSG.format(type="dict")
 RESPONSE_LIST_EXPECTED_MSG = _RESPONSE_BASE_EXPECTED_MSG.format(type="list")
 
 
-ASSET_PATHS: Dict[QlikAsset, str] = {
+ASSET_PATHS: dict[QlikAsset, str] = {
     QlikAsset.CONNECTIONS: "data-connections",
     QlikAsset.SPACES: "spaces",
     QlikAsset.USERS: "users",

castor_extractor/visualization/qlik/client/engine/error.py

@@ -1,5 +1,3 @@
-from typing import Dict, Type
-
 from .constants import (
     ACCESS_DENIED_ERROR_CODE,
     APP_SIZE_EXCEEDED_ERROR_CODE,
@@ -48,7 +46,7 @@ class QlikResponseKeyError(Exception):
     """
 
 
-ERROR_CODE_MAPPING: Dict[int, Type[JsonRpcError]] = {
+ERROR_CODE_MAPPING: dict[int, type[JsonRpcError]] = {
     ACCESS_DENIED_ERROR_CODE: AccessDeniedError,
     APP_SIZE_EXCEEDED_ERROR_CODE: AppSizeExceededError,
     PERSISTENCE_READ_FAILED_ERROR_CODE: PersistenceReadFailedError,

castor_extractor/visualization/qlik/client/master.py

@@ -1,4 +1,4 @@
-from typing import Callable, List, Optional
+from typing import Callable, Optional
 
 from tqdm import tqdm  # type: ignore
 
@@ -7,7 +7,7 @@ from .constants import APP_EXTERNAL_ID_KEY, SCOPED_ASSETS
 from .engine import EngineApiClient, QlikCredentials
 from .rest import RestApiClient
 
-ListedData = List[dict]
+ListedData = list[dict]
 
 
 class MissingAppsScopeError(Exception):
@@ -54,7 +54,7 @@ class QlikMasterClient:
     def __init__(
         self,
         credentials: QlikCredentials,
-        except_http_error_statuses: Optional[List[int]] = None,
+        except_http_error_statuses: Optional[list[int]] = None,
         display_progress: bool = True,
     ):
         self._server_url = credentials.base_url

castor_extractor/visualization/qlik/client/rest.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List, Optional, Union
+from typing import Optional, Union
 from urllib.parse import urljoin
 
 import requests
@@ -20,7 +20,7 @@ from .engine import QlikCredentials
 
 logger = logging.getLogger(__name__)
 
-Response = Union[dict, List[dict]]
+Response = Union[dict, list[dict]]
 
 
 def _session() -> requests.Session:
@@ -62,7 +62,7 @@ class RestApiClient:
     def __init__(
         self,
         credentials: QlikCredentials,
-        except_http_error_statuses: Optional[List[int]] = None,
+        except_http_error_statuses: Optional[list[int]] = None,
     ):
         self._server_url = credentials.base_url
         self._api_key = credentials.api_key
@@ -111,10 +111,10 @@ class RestApiClient:
         except HTTPError as error:
             return self._handle_http_error(error)
 
-    def _pager(self, first_page_url: str) -> List[dict]:
+    def _pager(self, first_page_url: str) -> list[dict]:
         current_page_url = first_page_url
 
-        data: List[dict] = []
+        data: list[dict] = []
         while current_page_url:
             response = self._call(current_page_url)
             if not response:
@@ -130,7 +130,7 @@ class RestApiClient:
             current_page_url = next_page_url
         return data
 
-    def get(self, asset: QlikAsset) -> List[dict]:
+    def get(self, asset: QlikAsset) -> list[dict]:
         """
         Calls the route corresponding to the asset and returns the list of
         corresponding data
@@ -144,7 +144,7 @@ class RestApiClient:
 
         return [_filter_fields(row) for row in data]
 
-    def get_with_scope(self, asset: QlikAsset, app_id: str) -> List[dict]:
+    def get_with_scope(self, asset: QlikAsset, app_id: str) -> list[dict]:
         """
         Calls the route corresponding to the asset scoped on an app_id and
         returns the corresponding data
@@ -157,7 +157,7 @@ class RestApiClient:
         assert isinstance(response, list), RESPONSE_LIST_EXPECTED_MSG
         return response
 
-    def data_connections(self) -> List[dict]:
+    def data_connections(self) -> list[dict]:
         """
         Returns the list of data Connections
 
@@ -165,7 +165,7 @@ class RestApiClient:
         """
         return self.get(QlikAsset.CONNECTIONS)
 
-    def spaces(self) -> List[dict]:
+    def spaces(self) -> list[dict]:
         """
         Returns the list of Spaces
 
@@ -173,7 +173,7 @@ class RestApiClient:
         """
         return self.get(QlikAsset.SPACES)
 
-    def users(self) -> List[dict]:
+    def users(self) -> list[dict]:
         """
         Returns the list of Users
 
@@ -181,7 +181,7 @@ class RestApiClient:
         """
         return self.get(QlikAsset.USERS)
 
-    def apps(self) -> List[dict]:
+    def apps(self) -> list[dict]:
         """
         Returns the list of Apps
 
@@ -189,7 +189,7 @@ class RestApiClient:
         """
         return self.get(QlikAsset.APPS)
 
-    def data_lineage(self, app_id: str) -> List[dict]:
+    def data_lineage(self, app_id: str) -> list[dict]:
         """
         Returns the data lineage for a given source
 

castor_extractor/visualization/qlik/extract.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Iterable, List, Optional, Tuple, Union
+from collections.abc import Iterable
+from typing import Optional, Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -18,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 def iterate_all_data(
     client: QlikClient,
-) -> Iterable[Tuple[QlikAsset, Union[list, dict]]]:
+) -> Iterable[tuple[QlikAsset, Union[list, dict]]]:
     """Iterate over the extracted data from Qlik"""
 
     logger.info("Extracting CONNECTIONS from REST API")
@@ -47,7 +48,7 @@ def iterate_all_data(
 
 
 def extract_all(
-    except_http_error_statuses: Optional[List[int]] = None, **kwargs
+    except_http_error_statuses: Optional[list[int]] = None, **kwargs
 ) -> None:
     """
     Extract data from Qlik REST API

castor_extractor/visualization/salesforce_reporting/client/rest.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Dict, Iterator, List, Optional
+from collections.abc import Iterator
+from typing import Optional
 
 from ....utils import build_url
 from ....utils.salesforce import SalesforceBaseClient
@@ -49,7 +50,7 @@ class SalesforceReportingClient(SalesforceBaseClient):
             url = self._get_asset_url(asset_type, asset)
             yield {**asset, "Url": url}
 
-    def fetch(self, asset: SalesforceReportingAsset) -> List[Dict]:
+    def fetch(self, asset: SalesforceReportingAsset) -> list[dict]:
         """
         Fetch Salesforce Reporting assets
         """

castor_extractor/visualization/salesforce_reporting/client/soql.py

@@ -1,8 +1,6 @@
-from typing import Dict
-
 from ..assets import SalesforceReportingAsset
 
-queries: Dict[SalesforceReportingAsset, str] = {
+queries: dict[SalesforceReportingAsset, str] = {
     SalesforceReportingAsset.DASHBOARDS: """
         SELECT
             CreatedBy.Id,

castor_extractor/visualization/salesforce_reporting/extract.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Iterable, Tuple, Union
+from collections.abc import Iterable
+from typing import Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -19,7 +20,7 @@ logger = logging.getLogger(__name__)
 
 def iterate_all_data(
     client: SalesforceReportingClient,
-) -> Iterable[Tuple[str, Union[list, dict]]]:
+) -> Iterable[tuple[str, Union[list, dict]]]:
     """Iterate over the extracted data from Salesforce"""
 
     for asset in SalesforceReportingAsset:

castor_extractor/visualization/sigma/client/client.py

@@ -1,6 +1,7 @@
+from collections.abc import Iterator
 from functools import partial
 from http import HTTPStatus
-from typing import Callable, Dict, Iterator, List, Optional, Tuple
+from typing import Callable, Optional
 
 import requests
 
@@ -19,7 +20,7 @@ from .pagination import SIGMA_API_LIMIT, SigmaPagination
 
 _CONTENT_TYPE = "application/x-www-form-urlencoded"
 
-_DATA_ELEMENTS: Tuple[str, ...] = (
+_DATA_ELEMENTS: tuple[str, ...] = (
     "input-table",
     "pivot-table",
     "table",
@@ -28,6 +29,7 @@ _DATA_ELEMENTS: Tuple[str, ...] = (
 )
 
 _AUTH_TIMEOUT_S = 60
+_SIGMA_TIMEOUT = 120
 
 _SIGMA_HEADERS = {
     "Content-Type": _CONTENT_TYPE,
@@ -48,7 +50,7 @@ SIGMA_SAFE_MODE = RequestSafeMode(
 
 
 class SigmaBearerAuth(BearerAuth):
-    def __init__(self, host: str, token_payload: Dict[str, str]):
+    def __init__(self, host: str, token_payload: dict[str, str]):
         auth_endpoint = SigmaEndpointFactory.authentication()
         self.authentication_url = build_url(host, auth_endpoint)
         self.token_payload = token_payload
@@ -75,6 +77,7 @@ class SigmaClient(APIClient):
             host=credentials.host,
             auth=auth,
             headers=_SIGMA_HEADERS,
+            timeout=_SIGMA_TIMEOUT,
             safe_mode=safe_mode or SIGMA_SAFE_MODE,
         )
 
@@ -116,7 +119,7 @@ class SigmaClient(APIClient):
                 "page_id": page_id,
             }
 
-    def _get_all_elements(self, workbooks: List[dict]) -> Iterator[dict]:
+    def _get_all_elements(self, workbooks: list[dict]) -> Iterator[dict]:
         for workbook in workbooks:
             workbook_id = workbook["workbookId"]
 
@@ -130,7 +133,7 @@ class SigmaClient(APIClient):
                     page=page, workbook_id=workbook_id
                 )
 
-    def _get_all_lineages(self, elements: List[dict]) -> Iterator[dict]:
+    def _get_all_lineages(self, elements: list[dict]) -> Iterator[dict]:
         for element in elements:
             workbook_id = element["workbook_id"]
             element_id = element["elementId"]
@@ -143,7 +146,7 @@ class SigmaClient(APIClient):
                 "element_id": element_id,
             }
 
-    def _get_all_queries(self, workbooks: List[dict]) -> Iterator[dict]:
+    def _get_all_queries(self, workbooks: list[dict]) -> Iterator[dict]:
         for workbook in workbooks:
             workbook_id = workbook["workbookId"]
             request = self._get_paginated(
@@ -157,8 +160,8 @@ class SigmaClient(APIClient):
     def fetch(
         self,
         asset: SigmaAsset,
-        workbooks: Optional[List[dict]] = None,
-        elements: Optional[List[dict]] = None,
+        workbooks: Optional[list[dict]] = None,
+        elements: Optional[list[dict]] = None,
     ) -> Iterator[dict]:
         """Returns the needed metadata for the queried asset"""
         if asset == SigmaAsset.DATASETS:

castor_extractor/visualization/sigma/client/credentials.py

@@ -1,5 +1,3 @@
-from typing import Dict
-
 from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -21,7 +19,7 @@ class SigmaCredentials(BaseSettings):
     grant_type: str = "client_credentials"
 
     @property
-    def token_payload(self) -> Dict[str, str]:
+    def token_payload(self) -> dict[str, str]:
         return {
             "grant_type": self.grant_type,
             "client_id": self.client_id,

castor_extractor/visualization/sigma/client/pagination.py

@@ -5,7 +5,7 @@ from pydantic.alias_generators import to_camel
 
 from ....utils import PaginationModel
 
-SIGMA_API_LIMIT = 500  # default number of records per page
+SIGMA_API_LIMIT = 200  # default number of records per page
 
 
 class SigmaPagination(PaginationModel):

castor_extractor/visualization/sigma/extract.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Iterable, Iterator, Tuple, Union
+from collections.abc import Iterable, Iterator
+from typing import Union
 
 from ...utils import (
     OUTPUT_DIR,
@@ -18,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 def iterate_all_data(
     client: SigmaClient,
-) -> Iterable[Tuple[SigmaAsset, Union[list, Iterator, dict]]]:
+) -> Iterable[tuple[SigmaAsset, Union[list, Iterator, dict]]]:
     """Iterate over the extracted data from Sigma"""
 
     logger.info("Extracting DATASETS from API")