cognite-extractor-utils 7.5.4__py3-none-any.whl → 7.5.5__py3-none-any.whl

cognite/extractorutils/uploader/files.py

@@ -18,7 +18,16 @@ from io import BytesIO, RawIOBase
 from math import ceil
 from os import PathLike
 from types import TracebackType
-from typing import Any, BinaryIO, Callable, Dict, Iterator, List, Optional, Tuple, Type, Union
+from typing import (
+    Any,
+    BinaryIO,
+    Callable,
+    Iterator,
+    List,
+    Optional,
+    Type,
+    Union,
+)
 from urllib.parse import ParseResult, urlparse
 
 from httpx import URL, Client, Headers, Request, StreamConsumed, SyncByteStream
@@ -27,7 +36,9 @@ from requests.utils import super_len
 from cognite.client import CogniteClient
 from cognite.client.data_classes import FileMetadata, FileMetadataUpdate
 from cognite.client.data_classes.data_modeling import NodeId
-from cognite.client.data_classes.data_modeling.extractor_extensions.v1 import CogniteExtractorFileApply
+from cognite.client.data_classes.data_modeling.extractor_extensions.v1 import (
+    CogniteExtractorFileApply,
+)
 from cognite.client.utils._identifier import IdentifierSequence
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader._base import (
@@ -42,6 +53,7 @@ from cognite.extractorutils.uploader._metrics import (
     FILES_UPLOADER_QUEUED,
     FILES_UPLOADER_WRITTEN,
 )
+from cognite.extractorutils.uploader.upload_failure_handler import FileFailureManager
 from cognite.extractorutils.util import cognite_exceptions, retry
 
 _QUEUES: int = 0
@@ -54,6 +66,7 @@ _MAX_FILE_CHUNK_SIZE = 4 * 1024 * 1024 * 1000
 
 _CDF_ALPHA_VERSION_HEADER = {"cdf-version": "alpha"}
 
+
 FileMetadataOrCogniteExtractorFile = Union[FileMetadata, CogniteExtractorFileApply]
 
 
@@ -97,7 +110,10 @@ class ChunkedStream(RawIOBase, BinaryIO):
         return super().__enter__()
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+        self,
+        exc_type: Type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
     ) -> None:
         return super().__exit__(exc_type, exc_val, exc_tb)
 
@@ -186,13 +202,14 @@ class IOFileUploadQueue(AbstractUploadQueue):
     def __init__(
         self,
         cdf_client: CogniteClient,
-        post_upload_function: Optional[Callable[[List[FileMetadataOrCogniteExtractorFile]], None]] = None,
-        max_queue_size: Optional[int] = None,
+        post_upload_function: Callable[[list[FileMetadataOrCogniteExtractorFile]], None] | None = None,
+        max_queue_size: int | None = None,
         trigger_log_level: str = "DEBUG",
-        thread_name: Optional[str] = None,
+        thread_name: str | None = None,
         overwrite_existing: bool = False,
         cancellation_token: Optional[CancellationToken] = None,
         max_parallelism: Optional[int] = None,
+        failure_logging_path: None | str = None,
     ):
         # Super sets post_upload and threshold
         super().__init__(
@@ -208,6 +225,9 @@ class IOFileUploadQueue(AbstractUploadQueue):
         if self.threshold <= 0:
             raise ValueError("Max queue size must be positive for file upload queues")
 
+        self.failure_logging_path = failure_logging_path or None
+        self.initialize_failure_logging()
+
         self.upload_queue: List[Future] = []
         self.errors: List[Exception] = []
 
@@ -235,10 +255,31 @@ class IOFileUploadQueue(AbstractUploadQueue):
         global _QUEUES, _QUEUES_LOCK
         with _QUEUES_LOCK:
             self._pool = ThreadPoolExecutor(
-                max_workers=self.parallelism, thread_name_prefix=f"FileUploadQueue-{_QUEUES}"
+                max_workers=self.parallelism,
+                thread_name_prefix=f"FileUploadQueue-{_QUEUES}",
             )
             _QUEUES += 1
 
+    def initialize_failure_logging(self) -> None:
+        self._file_failure_manager: FileFailureManager | None = (
+            FileFailureManager(path_to_file=self.failure_logging_path)
+            if self.failure_logging_path is not None
+            else None
+        )
+
+    def get_failure_logger(self) -> FileFailureManager | None:
+        return self._file_failure_manager
+
+    def add_entry_failure_logger(self, file_name: str, error: Exception) -> None:
+        if self._file_failure_manager is not None:
+            error_reason = str(error)
+            self._file_failure_manager.add(file_name=file_name, error_reason=error_reason)
+
+    def flush_failure_logger(self) -> None:
+        if self._file_failure_manager is not None:
+            self.logger.info("Flushing failure logs")
+            self._file_failure_manager.write_to_file()
+
     def _remove_done_from_queue(self) -> None:
         while not self.cancellation_token.is_cancelled:
             with self.lock:
@@ -251,7 +292,7 @@ class IOFileUploadQueue(AbstractUploadQueue):
         node = instance_result.nodes[0]
         return node.as_id()
 
-    def _upload_empty(
+    def _upload_only_metadata(
         self, file_meta: FileMetadataOrCogniteExtractorFile
     ) -> tuple[FileMetadataOrCogniteExtractorFile, str]:
         if isinstance(file_meta, CogniteExtractorFileApply):
@@ -281,11 +322,51 @@ class IOFileUploadQueue(AbstractUploadQueue):
 
         return file_meta_response, url
 
+    def _upload_empty_file(
+        self,
+        file_meta: FileMetadataOrCogniteExtractorFile,
+    ) -> None:
+        file_meta_response, url = self._upload_only_metadata(file_meta)
+
+        self._upload_only_file_reference(file_meta, url)
+
     def _upload_bytes(self, size: int, file: BinaryIO, file_meta: FileMetadataOrCogniteExtractorFile) -> None:
-        file_meta, url = self._upload_empty(file_meta)
+        file_meta, url = self._upload_only_metadata(file_meta)
         resp = self._httpx_client.send(self._get_file_upload_request(url, file, size, file_meta.mime_type))
         resp.raise_for_status()
 
+    def _prepare_request_data_for_empty_file(self, url_str: str) -> Request:
+        FILE_SIZE = 0  # this path is only entered for an empty file
+        EMPTY_CONTENT = ""
+
+        url = URL(url_str)
+        base_url = URL(self.cdf_client.config.base_url)
+
+        if url.host == base_url.host:
+            upload_url = url
+        else:
+            parsed_url: ParseResult = urlparse(url_str)
+            parsed_base_url: ParseResult = urlparse(self.cdf_client.config.base_url)
+            replaced_upload_url = parsed_url._replace(netloc=parsed_base_url.netloc).geturl()
+            upload_url = URL(replaced_upload_url)
+
+        headers = Headers(self._httpx_client.headers)
+        headers.update(
+            {
+                "Accept": "*/*",
+                "Content-Length": str(FILE_SIZE),
+                "Host": upload_url.netloc.decode("ascii"),
+                "x-cdp-app": self.cdf_client._config.client_name,
+            }
+        )
+
+        return Request(method="PUT", url=upload_url, headers=headers, content=EMPTY_CONTENT)
+
+    def _upload_only_file_reference(self, file_meta: FileMetadataOrCogniteExtractorFile, url_str: str) -> None:
+        request_data = self._prepare_request_data_for_empty_file(url_str)
+        resp = self._httpx_client.send(request_data)
+        resp.raise_for_status()
+
     def _upload_multipart(self, size: int, file: BinaryIO, file_meta: FileMetadataOrCogniteExtractorFile) -> None:
         chunks = ChunkedStream(file, self.max_file_chunk_size, size)
         self.logger.debug(
@@ -329,7 +410,10 @@ class IOFileUploadQueue(AbstractUploadQueue):
             res = self.cdf_client.files._post(
                 url_path="/files/initmultipartupload",
                 json=file_meta.dump(camel_case=True),
-                params={"overwrite": self.overwrite_existing, "parts": chunks.chunk_count},
+                params={
+                    "overwrite": self.overwrite_existing,
+                    "parts": chunks.chunk_count,
+                },
             )
             res.raise_for_status()
             return res.json()
@@ -338,9 +422,7 @@ class IOFileUploadQueue(AbstractUploadQueue):
         self,
         file_meta: FileMetadataOrCogniteExtractorFile,
         read_file: Callable[[], BinaryIO],
-        extra_retries: Optional[
-            Union[Tuple[Type[Exception], ...], Dict[Type[Exception], Callable[[Any], bool]]]
-        ] = None,
+        extra_retries: tuple[Type[Exception], ...] | dict[Type[Exception], Callable[[Any], bool]] | None = None,
     ) -> None:
         """
         Add file to upload queue. The file will start uploading immedeately. If the size of the queue is larger than
@@ -366,12 +448,14 @@ class IOFileUploadQueue(AbstractUploadQueue):
             max_delay=RETRY_MAX_DELAY,
             backoff=RETRY_BACKOFF_FACTOR,
         )
-        def upload_file(read_file: Callable[[], BinaryIO], file_meta: FileMetadataOrCogniteExtractorFile) -> None:
+        def upload_file(
+            read_file: Callable[[], BinaryIO],
+            file_meta: FileMetadataOrCogniteExtractorFile,
+        ) -> None:
             with read_file() as file:
                 size = super_len(file)
                 if size == 0:
-                    # upload just the file metadata witout data
-                    file_meta, _ = self._upload_empty(file_meta)
+                    self._upload_empty_file(file_meta)
                 elif size >= self.max_single_chunk_file_size:
                     # The minimum chunk size is 4000MiB.
                     self._upload_multipart(size, file, file_meta)
@@ -388,12 +472,18 @@ class IOFileUploadQueue(AbstractUploadQueue):
                 except Exception as e:
                     self.logger.error("Error in upload callback: %s", str(e))
 
-        def wrapped_upload(read_file: Callable[[], BinaryIO], file_meta: FileMetadataOrCogniteExtractorFile) -> None:
+        def wrapped_upload(
+            read_file: Callable[[], BinaryIO],
+            file_meta: FileMetadataOrCogniteExtractorFile,
+        ) -> None:
             try:
                 upload_file(read_file, file_meta)
 
             except Exception as e:
-                self.logger.exception(f"Unexpected error while uploading file: {file_meta.external_id}")
+                self.logger.exception(
+                    f"Unexpected error while uploading file: {file_meta.external_id} {file_meta.name}"
+                )
+                self.add_entry_failure_logger(file_name=str(file_meta.name), error=e)
                 self.errors.append(e)
 
             finally:
@@ -416,7 +506,7 @@ class IOFileUploadQueue(AbstractUploadQueue):
             self.queue_size.set(self.upload_queue_size)
 
     def _get_file_upload_request(
-        self, url_str: str, stream: BinaryIO, size: int, mime_type: Optional[str] = None
+        self, url_str: str, stream: BinaryIO, size: int, mime_type: str | None = None
     ) -> Request:
         url = URL(url_str)
         base_url = URL(self.cdf_client.config.base_url)
@@ -460,7 +550,7 @@ class IOFileUploadQueue(AbstractUploadQueue):
         resp_json = res.json()["items"][0]
         return FileMetadata.load(resp_json), resp_json["uploadUrl"]
 
-    def upload(self, fail_on_errors: bool = True, timeout: Optional[float] = None) -> None:
+    def upload(self, fail_on_errors: bool = True, timeout: float | None = None) -> None:
         """
         Wait for all uploads to finish
         """
@@ -470,6 +560,7 @@ class IOFileUploadQueue(AbstractUploadQueue):
             self.queue_size.set(self.upload_queue_size)
         if fail_on_errors and self.errors:
             # There might be more errors, but we can only have one as the cause, so pick the first
+            self.flush_failure_logger()
             raise RuntimeError(f"{len(self.errors)} upload(s) finished with errors") from self.errors[0]
 
     def __enter__(self) -> "IOFileUploadQueue":
@@ -485,7 +576,10 @@ class IOFileUploadQueue(AbstractUploadQueue):
         return self
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
     ) -> None:
         """
         Wraps around stop method, for use as context manager
@@ -524,13 +618,13 @@ class FileUploadQueue(IOFileUploadQueue):
     def __init__(
         self,
         cdf_client: CogniteClient,
-        post_upload_function: Optional[Callable[[List[FileMetadataOrCogniteExtractorFile]], None]] = None,
-        max_queue_size: Optional[int] = None,
-        max_upload_interval: Optional[int] = None,
+        post_upload_function: Callable[[list[FileMetadataOrCogniteExtractorFile]], None] | None = None,
+        max_queue_size: int | None = None,
+        max_upload_interval: int | None = None,
         trigger_log_level: str = "DEBUG",
-        thread_name: Optional[str] = None,
+        thread_name: str | None = None,
         overwrite_existing: bool = False,
-        cancellation_token: Optional[CancellationToken] = None,
+        cancellation_token: CancellationToken | None = None,
     ):
         # Super sets post_upload and threshold
         super().__init__(
@@ -544,7 +638,9 @@ class FileUploadQueue(IOFileUploadQueue):
         )
 
     def add_to_upload_queue(
-        self, file_meta: FileMetadataOrCogniteExtractorFile, file_name: Union[str, PathLike]
+        self,
+        file_meta: FileMetadataOrCogniteExtractorFile,
+        file_name: Union[str, PathLike],
     ) -> None:
         """
         Add file to upload queue. The queue will be uploaded if the queue size is larger than the threshold
@@ -579,12 +675,12 @@ class BytesUploadQueue(IOFileUploadQueue):
     def __init__(
         self,
         cdf_client: CogniteClient,
-        post_upload_function: Optional[Callable[[List[FileMetadataOrCogniteExtractorFile]], None]] = None,
-        max_queue_size: Optional[int] = None,
+        post_upload_function: Callable[[list[FileMetadataOrCogniteExtractorFile]], None] | None = None,
+        max_queue_size: int | None = None,
         trigger_log_level: str = "DEBUG",
-        thread_name: Optional[str] = None,
+        thread_name: str | None = None,
         overwrite_existing: bool = False,
-        cancellation_token: Optional[CancellationToken] = None,
+        cancellation_token: CancellationToken | None = None,
     ) -> None:
         super().__init__(
             cdf_client,

cognite/extractorutils/uploader/raw.py

@@ -13,7 +13,7 @@
 #  limitations under the License.
 
 from types import TracebackType
-from typing import Any, Callable, Dict, List, Optional, Type
+from typing import Any, Callable, Type
 
 import arrow
 from arrow import Arrow
@@ -56,12 +56,12 @@ class RawUploadQueue(AbstractUploadQueue):
     def __init__(
         self,
         cdf_client: CogniteClient,
-        post_upload_function: Optional[Callable[[List[Any]], None]] = None,
-        max_queue_size: Optional[int] = None,
-        max_upload_interval: Optional[int] = None,
+        post_upload_function: Callable[[list[Any]], None] | None = None,
+        max_queue_size: int | None = None,
+        max_upload_interval: int | None = None,
         trigger_log_level: str = "DEBUG",
-        thread_name: Optional[str] = None,
-        cancellation_token: Optional[CancellationToken] = None,
+        thread_name: str | None = None,
+        cancellation_token: CancellationToken | None = None,
     ):
         # Super sets post_upload and thresholds
         super().__init__(
@@ -73,7 +73,7 @@ class RawUploadQueue(AbstractUploadQueue):
             thread_name,
             cancellation_token,
         )
-        self.upload_queue: Dict[str, Dict[str, List[TimestampedObject]]] = {}
+        self.upload_queue: dict[str, dict[str, list[TimestampedObject]]] = {}
 
         # It is a hack since Prometheus client registers metrics on object creation, so object has to be created once
         self.rows_queued = RAW_UPLOADER_ROWS_QUEUED
@@ -119,7 +119,7 @@ class RawUploadQueue(AbstractUploadQueue):
             max_delay=RETRY_MAX_DELAY,
             backoff=RETRY_BACKOFF_FACTOR,
         )
-        def _upload_batch(database: str, table: str, patch: List[Row]) -> None:
+        def _upload_batch(database: str, table: str, patch: list[Row]) -> None:
             # Upload
             self.cdf_client.raw.rows.insert(db_name=database, table_name=table, row=patch, ensure_parent=True)
 
@@ -133,7 +133,7 @@ class RawUploadQueue(AbstractUploadQueue):
 
                     # Deduplicate
                     # In case of duplicate keys, the first key is preserved, and the last value is preserved.
-                    patch: Dict[str, Row] = {r.payload.key: r.payload for r in rows}
+                    patch: dict[str, Row] = {r.payload.key: r.payload for r in rows}
                     self.rows_duplicates.labels(_labels).inc(len(rows) - len(patch))
 
                     _upload_batch(database=database, table=table, patch=list(patch.values()))
@@ -162,7 +162,7 @@ class RawUploadQueue(AbstractUploadQueue):
         return self
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+        self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
     ) -> None:
         """
         Wraps around stop method, for use as context manager

cognite/extractorutils/uploader/time_series.py

@@ -15,7 +15,7 @@
 import math
 from datetime import datetime
 from types import TracebackType
-from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
+from typing import Any, Callable, Type
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import (
@@ -50,13 +50,13 @@ MAX_DATAPOINT_STRING_LENGTH = 255
 MAX_DATAPOINT_VALUE = 1e100
 MIN_DATAPOINT_VALUE = -1e100
 
-TimeStamp = Union[int, datetime]
+TimeStamp = int | datetime
 
-DataPointWithoutStatus = Union[Tuple[TimeStamp, float], Tuple[TimeStamp, str], Tuple[TimeStamp, int]]
-FullStatusCode = Union[StatusCode, int]
-DataPointWithStatus = Union[Tuple[TimeStamp, float, FullStatusCode], Tuple[TimeStamp, str, FullStatusCode]]
-DataPoint = Union[DataPointWithoutStatus, DataPointWithStatus]
-DataPointList = List[DataPoint]
+DataPointWithoutStatus = tuple[TimeStamp, float] | tuple[TimeStamp, str] | tuple[TimeStamp, int]
+FullStatusCode = StatusCode | int
+DataPointWithStatus = tuple[TimeStamp, float, FullStatusCode] | tuple[TimeStamp, str, FullStatusCode]
+DataPoint = DataPointWithoutStatus | DataPointWithStatus
+DataPointList = list[DataPoint]
 
 
 def default_time_series_factory(external_id: str, datapoints: DataPointList) -> TimeSeries:
@@ -103,14 +103,14 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
     def __init__(
         self,
         cdf_client: CogniteClient,
-        post_upload_function: Optional[Callable[[List[Dict[str, Union[str, DataPointList]]]], None]] = None,
-        max_queue_size: Optional[int] = None,
-        max_upload_interval: Optional[int] = None,
+        post_upload_function: Callable[[list[dict[str, str | DataPointList]]], None] | None = None,
+        max_queue_size: int | None = None,
+        max_upload_interval: int | None = None,
         trigger_log_level: str = "DEBUG",
-        thread_name: Optional[str] = None,
-        create_missing: Union[Callable[[str, DataPointList], TimeSeries], bool] = False,
-        data_set_id: Optional[int] = None,
-        cancellation_token: Optional[CancellationToken] = None,
+        thread_name: str | None = None,
+        create_missing: Callable[[str, DataPointList], TimeSeries] | bool = False,
+        data_set_id: int | None = None,
+        cancellation_token: CancellationToken | None = None,
     ):
         # Super sets post_upload and threshold
         super().__init__(
@@ -132,14 +132,14 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             self.create_missing = True
             self.missing_factory = create_missing
 
-        self.upload_queue: Dict[EitherId, DataPointList] = {}
+        self.upload_queue: dict[EitherId, DataPointList] = {}
 
         self.points_queued = TIMESERIES_UPLOADER_POINTS_QUEUED
         self.points_written = TIMESERIES_UPLOADER_POINTS_WRITTEN
         self.queue_size = TIMESERIES_UPLOADER_QUEUE_SIZE
         self.data_set_id = data_set_id
 
-    def _verify_datapoint_time(self, time: Union[int, float, datetime, str]) -> bool:
+    def _verify_datapoint_time(self, time: int | float | datetime | str) -> bool:
         if isinstance(time, int) or isinstance(time, float):
             return not math.isnan(time) and time >= MIN_DATAPOINT_TIMESTAMP
         elif isinstance(time, str):
@@ -147,7 +147,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
         else:
             return time.timestamp() * 1000.0 >= MIN_DATAPOINT_TIMESTAMP
 
-    def _verify_datapoint_value(self, value: Union[int, float, datetime, str]) -> bool:
+    def _verify_datapoint_value(self, value: int | float | datetime | str) -> bool:
         if isinstance(value, float):
             return not (
                 math.isnan(value) or math.isinf(value) or value > MAX_DATAPOINT_VALUE or value < MIN_DATAPOINT_VALUE
@@ -171,7 +171,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             return True
 
     def add_to_upload_queue(
-        self, *, id: Optional[int] = None, external_id: Optional[str] = None, datapoints: Optional[DataPointList] = None
+        self, *, id: int | None = None, external_id: str | None = None, datapoints: DataPointList | None = None
     ) -> None:
         """
         Add data points to upload queue. The queue will be uploaded if the queue size is larger than the threshold
@@ -180,7 +180,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
         Args:
             id: Internal ID of time series. Either this or external_id must be set.
             external_id: External ID of time series. Either this or external_id must be set.
-            datapoints: List of data points to add
+            datapoints: list of data points to add
         """
         datapoints = datapoints or []
         old_len = len(datapoints)
@@ -219,7 +219,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             max_delay=RETRY_MAX_DELAY,
             backoff=RETRY_BACKOFF_FACTOR,
         )
-        def _upload_batch(upload_this: List[Dict], retries: int = 5) -> List[Dict]:
+        def _upload_batch(upload_this: list[dict], retries: int = 5) -> list[dict]:
             if len(upload_this) == 0:
                 return upload_this
 
@@ -241,14 +241,14 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
                     create_these_ids = set(
                         [id_dict["externalId"] for id_dict in ex.not_found if "externalId" in id_dict]
                     )
-                    datapoints_lists: Dict[str, DataPointList] = {
+                    datapoints_lists: dict[str, DataPointList] = {
                         ts_dict["externalId"]: ts_dict["datapoints"]
                         for ts_dict in upload_this
                         if ts_dict["externalId"] in create_these_ids
                     }
 
                     self.logger.info(f"Creating {len(create_these_ids)} time series")
-                    to_create: List[TimeSeries] = [
+                    to_create: list[TimeSeries] = [
                         self.missing_factory(external_id, datapoints_lists[external_id])
                         for external_id in create_these_ids
                     ]
@@ -317,7 +317,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
         return self
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+        self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
     ) -> None:
         """
         Wraps around stop method, for use as context manager
@@ -343,13 +343,13 @@ class SequenceUploadQueue(AbstractUploadQueue):
     def __init__(
         self,
         cdf_client: CogniteClient,
-        post_upload_function: Optional[Callable[[List[Any]], None]] = None,
-        max_queue_size: Optional[int] = None,
-        max_upload_interval: Optional[int] = None,
+        post_upload_function: Callable[[list[Any]], None] | None = None,
+        max_queue_size: int | None = None,
+        max_upload_interval: int | None = None,
         trigger_log_level: str = "DEBUG",
-        thread_name: Optional[str] = None,
+        thread_name: str | None = None,
         create_missing: bool = False,
-        cancellation_token: Optional[CancellationToken] = None,
+        cancellation_token: CancellationToken | None = None,
     ):
         """
         Args:
@@ -374,15 +374,15 @@ class SequenceUploadQueue(AbstractUploadQueue):
             thread_name,
             cancellation_token,
         )
-        self.upload_queue: Dict[EitherId, SequenceRows] = {}
-        self.sequence_metadata: Dict[EitherId, Dict[str, Union[str, int, float]]] = {}
-        self.sequence_asset_external_ids: Dict[EitherId, str] = {}
-        self.sequence_dataset_external_ids: Dict[EitherId, str] = {}
-        self.sequence_names: Dict[EitherId, str] = {}
-        self.sequence_descriptions: Dict[EitherId, str] = {}
-        self.column_definitions: Dict[EitherId, List[Dict[str, str]]] = {}
-        self.asset_ids: Dict[str, int] = {}
-        self.dataset_ids: Dict[str, int] = {}
+        self.upload_queue: dict[EitherId, SequenceRows] = {}
+        self.sequence_metadata: dict[EitherId, dict[str, str | int | float]] = {}
+        self.sequence_asset_external_ids: dict[EitherId, str] = {}
+        self.sequence_dataset_external_ids: dict[EitherId, str] = {}
+        self.sequence_names: dict[EitherId, str] = {}
+        self.sequence_descriptions: dict[EitherId, str] = {}
+        self.column_definitions: dict[EitherId, list[dict[str, str]]] = {}
+        self.asset_ids: dict[str, int] = {}
+        self.dataset_ids: dict[str, int] = {}
         self.create_missing = create_missing
 
         self.points_queued = SEQUENCES_UPLOADER_POINTS_QUEUED
@@ -391,13 +391,13 @@ class SequenceUploadQueue(AbstractUploadQueue):
 
     def set_sequence_metadata(
         self,
-        metadata: Dict[str, Union[str, int, float]],
-        id: Optional[int] = None,
-        external_id: Optional[str] = None,
-        asset_external_id: Optional[str] = None,
-        dataset_external_id: Optional[str] = None,
-        name: Optional[str] = None,
-        description: Optional[str] = None,
+        metadata: dict[str, str | int | float],
+        id: int | None = None,
+        external_id: str | None = None,
+        asset_external_id: str | None = None,
+        dataset_external_id: str | None = None,
+        name: str | None = None,
+        description: str | None = None,
     ) -> None:
         """
         Set sequence metadata. Metadata will be cached until the sequence is created. The metadata will be updated
@@ -426,7 +426,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
             self.sequence_descriptions[either_id] = description
 
     def set_sequence_column_definition(
-        self, col_def: List[Dict[str, str]], id: Optional[int] = None, external_id: Optional[str] = None
+        self, col_def: list[dict[str, str]], id: int | None = None, external_id: str | None = None
     ) -> None:
         """
         Set sequence column definition
@@ -443,16 +443,14 @@ class SequenceUploadQueue(AbstractUploadQueue):
 
     def add_to_upload_queue(
         self,
-        rows: Union[
-            Dict[int, List[Union[int, float, str]]],
-            List[Tuple[int, Union[int, float, str]]],
-            List[Dict[str, Any]],
-            SequenceData,
-            SequenceRows,
-        ],
-        column_external_ids: Optional[List[dict]] = None,
-        id: Optional[int] = None,
-        external_id: Optional[str] = None,
+        rows: dict[int, list[int | float | str]]
+        | list[tuple[int, int | float | str]]
+        | list[dict[str, Any]]
+        | SequenceData
+        | SequenceRows,
+        column_external_ids: list[dict] | None = None,
+        id: int | None = None,
+        external_id: str | None = None,
     ) -> None:
         """
         Add sequence rows to upload queue. Mirrors implementation of SequenceApi.insert. Inserted rows will be
@@ -461,7 +459,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
         Args:
             rows: The rows to be inserted. Can either be a list of tuples, a list of ["rownumber": ..., "values": ...]
                 objects, a dictionary of rowNumber: data, or a SequenceData object.
-            column_external_ids: List of external id for the columns of the sequence
+            column_external_ids: list of external id for the columns of the sequence
             id: Sequence internal ID
                 Use if external_id is None
             external_id: Sequence external ID
@@ -477,7 +475,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
             # Already in the desired format
             pass
         elif isinstance(rows, (dict, list)):
-            rows_raw: List[Dict[str, Any]]
+            rows_raw: list[dict[str, Any]]
             if isinstance(rows, dict):
                 rows_raw = [{"rowNumber": row_number, "values": values} for row_number, values in rows.items()]
             elif isinstance(rows, list) and rows and isinstance(rows[0], (tuple, list)):
@@ -658,7 +656,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
         return self
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+        self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
     ) -> None:
         """
         Wraps around stop method, for use as context manager