cognite-extractor-utils 7.5.4__py3-none-any.whl → 7.5.6__py3-none-any.whl

cognite/extractorutils/uploader/time_series.py

@@ -15,7 +15,7 @@
 import math
 from datetime import datetime
 from types import TracebackType
-from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
+from typing import Any, Callable, Type
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import (
@@ -50,13 +50,13 @@ MAX_DATAPOINT_STRING_LENGTH = 255
 MAX_DATAPOINT_VALUE = 1e100
 MIN_DATAPOINT_VALUE = -1e100
 
-TimeStamp = Union[int, datetime]
+TimeStamp = int | datetime
 
-DataPointWithoutStatus = Union[Tuple[TimeStamp, float], Tuple[TimeStamp, str], Tuple[TimeStamp, int]]
-FullStatusCode = Union[StatusCode, int]
-DataPointWithStatus = Union[Tuple[TimeStamp, float, FullStatusCode], Tuple[TimeStamp, str, FullStatusCode]]
-DataPoint = Union[DataPointWithoutStatus, DataPointWithStatus]
-DataPointList = List[DataPoint]
+DataPointWithoutStatus = tuple[TimeStamp, float] | tuple[TimeStamp, str] | tuple[TimeStamp, int]
+FullStatusCode = StatusCode | int
+DataPointWithStatus = tuple[TimeStamp, float, FullStatusCode] | tuple[TimeStamp, str, FullStatusCode]
+DataPoint = DataPointWithoutStatus | DataPointWithStatus
+DataPointList = list[DataPoint]
 
 
 def default_time_series_factory(external_id: str, datapoints: DataPointList) -> TimeSeries:
@@ -103,14 +103,14 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
     def __init__(
         self,
         cdf_client: CogniteClient,
-        post_upload_function: Optional[Callable[[List[Dict[str, Union[str, DataPointList]]]], None]] = None,
-        max_queue_size: Optional[int] = None,
-        max_upload_interval: Optional[int] = None,
+        post_upload_function: Callable[[list[dict[str, str | DataPointList]]], None] | None = None,
+        max_queue_size: int | None = None,
+        max_upload_interval: int | None = None,
         trigger_log_level: str = "DEBUG",
-        thread_name: Optional[str] = None,
-        create_missing: Union[Callable[[str, DataPointList], TimeSeries], bool] = False,
-        data_set_id: Optional[int] = None,
-        cancellation_token: Optional[CancellationToken] = None,
+        thread_name: str | None = None,
+        create_missing: Callable[[str, DataPointList], TimeSeries] | bool = False,
+        data_set_id: int | None = None,
+        cancellation_token: CancellationToken | None = None,
     ):
         # Super sets post_upload and threshold
         super().__init__(
@@ -132,14 +132,14 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             self.create_missing = True
             self.missing_factory = create_missing
 
-        self.upload_queue: Dict[EitherId, DataPointList] = {}
+        self.upload_queue: dict[EitherId, DataPointList] = {}
 
         self.points_queued = TIMESERIES_UPLOADER_POINTS_QUEUED
         self.points_written = TIMESERIES_UPLOADER_POINTS_WRITTEN
         self.queue_size = TIMESERIES_UPLOADER_QUEUE_SIZE
         self.data_set_id = data_set_id
 
-    def _verify_datapoint_time(self, time: Union[int, float, datetime, str]) -> bool:
+    def _verify_datapoint_time(self, time: int | float | datetime | str) -> bool:
         if isinstance(time, int) or isinstance(time, float):
             return not math.isnan(time) and time >= MIN_DATAPOINT_TIMESTAMP
         elif isinstance(time, str):
@@ -147,7 +147,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
         else:
             return time.timestamp() * 1000.0 >= MIN_DATAPOINT_TIMESTAMP
 
-    def _verify_datapoint_value(self, value: Union[int, float, datetime, str]) -> bool:
+    def _verify_datapoint_value(self, value: int | float | datetime | str) -> bool:
         if isinstance(value, float):
             return not (
                 math.isnan(value) or math.isinf(value) or value > MAX_DATAPOINT_VALUE or value < MIN_DATAPOINT_VALUE
@@ -171,7 +171,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             return True
 
     def add_to_upload_queue(
-        self, *, id: Optional[int] = None, external_id: Optional[str] = None, datapoints: Optional[DataPointList] = None
+        self, *, id: int | None = None, external_id: str | None = None, datapoints: DataPointList | None = None
     ) -> None:
         """
         Add data points to upload queue. The queue will be uploaded if the queue size is larger than the threshold
@@ -180,7 +180,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
         Args:
             id: Internal ID of time series. Either this or external_id must be set.
             external_id: External ID of time series. Either this or external_id must be set.
-            datapoints: List of data points to add
+            datapoints: list of data points to add
         """
         datapoints = datapoints or []
         old_len = len(datapoints)
@@ -219,7 +219,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             max_delay=RETRY_MAX_DELAY,
             backoff=RETRY_BACKOFF_FACTOR,
         )
-        def _upload_batch(upload_this: List[Dict], retries: int = 5) -> List[Dict]:
+        def _upload_batch(upload_this: list[dict], retries: int = 5) -> list[dict]:
             if len(upload_this) == 0:
                 return upload_this
 
@@ -241,14 +241,14 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
                     create_these_ids = set(
                         [id_dict["externalId"] for id_dict in ex.not_found if "externalId" in id_dict]
                     )
-                    datapoints_lists: Dict[str, DataPointList] = {
+                    datapoints_lists: dict[str, DataPointList] = {
                         ts_dict["externalId"]: ts_dict["datapoints"]
                         for ts_dict in upload_this
                         if ts_dict["externalId"] in create_these_ids
                     }
 
                     self.logger.info(f"Creating {len(create_these_ids)} time series")
-                    to_create: List[TimeSeries] = [
+                    to_create: list[TimeSeries] = [
                         self.missing_factory(external_id, datapoints_lists[external_id])
                         for external_id in create_these_ids
                     ]
@@ -317,7 +317,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
         return self
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+        self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
     ) -> None:
         """
         Wraps around stop method, for use as context manager
@@ -343,13 +343,13 @@ class SequenceUploadQueue(AbstractUploadQueue):
     def __init__(
         self,
         cdf_client: CogniteClient,
-        post_upload_function: Optional[Callable[[List[Any]], None]] = None,
-        max_queue_size: Optional[int] = None,
-        max_upload_interval: Optional[int] = None,
+        post_upload_function: Callable[[list[Any]], None] | None = None,
+        max_queue_size: int | None = None,
+        max_upload_interval: int | None = None,
         trigger_log_level: str = "DEBUG",
-        thread_name: Optional[str] = None,
+        thread_name: str | None = None,
         create_missing: bool = False,
-        cancellation_token: Optional[CancellationToken] = None,
+        cancellation_token: CancellationToken | None = None,
     ):
         """
         Args:
@@ -374,15 +374,15 @@ class SequenceUploadQueue(AbstractUploadQueue):
             thread_name,
             cancellation_token,
         )
-        self.upload_queue: Dict[EitherId, SequenceRows] = {}
-        self.sequence_metadata: Dict[EitherId, Dict[str, Union[str, int, float]]] = {}
-        self.sequence_asset_external_ids: Dict[EitherId, str] = {}
-        self.sequence_dataset_external_ids: Dict[EitherId, str] = {}
-        self.sequence_names: Dict[EitherId, str] = {}
-        self.sequence_descriptions: Dict[EitherId, str] = {}
-        self.column_definitions: Dict[EitherId, List[Dict[str, str]]] = {}
-        self.asset_ids: Dict[str, int] = {}
-        self.dataset_ids: Dict[str, int] = {}
+        self.upload_queue: dict[EitherId, SequenceRows] = {}
+        self.sequence_metadata: dict[EitherId, dict[str, str | int | float]] = {}
+        self.sequence_asset_external_ids: dict[EitherId, str] = {}
+        self.sequence_dataset_external_ids: dict[EitherId, str] = {}
+        self.sequence_names: dict[EitherId, str] = {}
+        self.sequence_descriptions: dict[EitherId, str] = {}
+        self.column_definitions: dict[EitherId, list[dict[str, str]]] = {}
+        self.asset_ids: dict[str, int] = {}
+        self.dataset_ids: dict[str, int] = {}
         self.create_missing = create_missing
 
         self.points_queued = SEQUENCES_UPLOADER_POINTS_QUEUED
@@ -391,13 +391,13 @@ class SequenceUploadQueue(AbstractUploadQueue):
 
     def set_sequence_metadata(
         self,
-        metadata: Dict[str, Union[str, int, float]],
-        id: Optional[int] = None,
-        external_id: Optional[str] = None,
-        asset_external_id: Optional[str] = None,
-        dataset_external_id: Optional[str] = None,
-        name: Optional[str] = None,
-        description: Optional[str] = None,
+        metadata: dict[str, str | int | float],
+        id: int | None = None,
+        external_id: str | None = None,
+        asset_external_id: str | None = None,
+        dataset_external_id: str | None = None,
+        name: str | None = None,
+        description: str | None = None,
     ) -> None:
         """
         Set sequence metadata. Metadata will be cached until the sequence is created. The metadata will be updated
@@ -426,7 +426,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
             self.sequence_descriptions[either_id] = description
 
     def set_sequence_column_definition(
-        self, col_def: List[Dict[str, str]], id: Optional[int] = None, external_id: Optional[str] = None
+        self, col_def: list[dict[str, str]], id: int | None = None, external_id: str | None = None
     ) -> None:
         """
         Set sequence column definition
@@ -443,16 +443,14 @@ class SequenceUploadQueue(AbstractUploadQueue):
 
     def add_to_upload_queue(
         self,
-        rows: Union[
-            Dict[int, List[Union[int, float, str]]],
-            List[Tuple[int, Union[int, float, str]]],
-            List[Dict[str, Any]],
-            SequenceData,
-            SequenceRows,
-        ],
-        column_external_ids: Optional[List[dict]] = None,
-        id: Optional[int] = None,
-        external_id: Optional[str] = None,
+        rows: dict[int, list[int | float | str]]
+        | list[tuple[int, int | float | str]]
+        | list[dict[str, Any]]
+        | SequenceData
+        | SequenceRows,
+        column_external_ids: list[dict] | None = None,
+        id: int | None = None,
+        external_id: str | None = None,
     ) -> None:
         """
         Add sequence rows to upload queue. Mirrors implementation of SequenceApi.insert. Inserted rows will be
@@ -461,7 +459,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
         Args:
             rows: The rows to be inserted. Can either be a list of tuples, a list of ["rownumber": ..., "values": ...]
                 objects, a dictionary of rowNumber: data, or a SequenceData object.
-            column_external_ids: List of external id for the columns of the sequence
+            column_external_ids: list of external id for the columns of the sequence
             id: Sequence internal ID
                 Use if external_id is None
             external_id: Sequence external ID
@@ -477,7 +475,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
             # Already in the desired format
             pass
         elif isinstance(rows, (dict, list)):
-            rows_raw: List[Dict[str, Any]]
+            rows_raw: list[dict[str, Any]]
             if isinstance(rows, dict):
                 rows_raw = [{"rowNumber": row_number, "values": values} for row_number, values in rows.items()]
             elif isinstance(rows, list) and rows and isinstance(rows[0], (tuple, list)):
@@ -658,7 +656,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
         return self
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+        self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
     ) -> None:
         """
         Wraps around stop method, for use as context manager

cognite/extractorutils/uploader/upload_failure_handler.py

@@ -0,0 +1,64 @@
+from datetime import datetime
+from typing import Iterator, List
+
+import jsonlines
+
+
+class FileErrorMapping:
+    def __init__(self, file_name: str, error_reason: str) -> None:
+        self.file_name = file_name
+        self.error_reason = error_reason
+
+    def __iter__(self) -> Iterator[List[str]]:
+        return iter([[self.file_name, self.error_reason]])
+
+
+class FileFailureManager:
+    MAX_QUEUE_SIZE = 500
+    START_TIME_KEY = "start_time"
+    FILE_REASON_MAP_KEY = "file_error_reason_map"
+
+    def __init__(self, start_time: str | None = None, path_to_file: str | None = None) -> None:
+        self.failure_logs: dict[str, str] = {}
+
+        self.path_to_failure_log: str = self._pre_process_file_extension(path_to_file)
+        self.start_time = start_time or str(datetime.now())
+        self._initialize_failure_logs()
+
+    def _pre_process_file_extension(self, path_to_file: str | None) -> str:
+        if path_to_file and not path_to_file.endswith(".jsonl"):
+            return path_to_file + ".jsonl"
+        return str(path_to_file)
+
+    def _initialize_failure_logs(self) -> None:
+        self.failure_logs = {}
+
+    def __len__(self) -> int:
+        return len(self.failure_logs)
+
+    def clear(self) -> None:
+        self.failure_logs.clear()
+        self._initialize_failure_logs()
+
+    def add(self, file_name: str, error_reason: str) -> None:
+        error_file_object = FileErrorMapping(file_name=file_name, error_reason=error_reason)
+        error_file_dict = dict(error_file_object)
+
+        self.failure_logs.update(error_file_dict)
+
+        if len(self) >= self.MAX_QUEUE_SIZE:
+            self.write_to_file()
+
+    def write_to_file(self) -> None:
+        if len(self) == 0:
+            return
+
+        dict_to_write = {
+            self.START_TIME_KEY: self.start_time,
+            self.FILE_REASON_MAP_KEY: self.failure_logs,
+        }
+
+        with jsonlines.open(self.path_to_failure_log, mode="a") as writer:
+            writer.write(dict_to_write)
+
+        self.clear()

cognite/extractorutils/uploader_extractor.py

@@ -15,9 +15,10 @@
 """
 A module containing a slightly more advanced base extractor class, sorting a generic output into upload queues.
 """
+
 from dataclasses import dataclass
 from types import TracebackType
-from typing import Any, Callable, Iterable, List, Optional, Type, TypeVar
+from typing import Any, Callable, Iterable, Type, TypeVar
 
 from more_itertools import peekable
 
@@ -41,10 +42,11 @@ class QueueConfigClass:
 
 @dataclass
 class UploaderExtractorConfig(BaseConfig):
-    queues: Optional[QueueConfigClass]
+    queues: QueueConfigClass | None
 
 
 UploaderExtractorConfigClass = TypeVar("UploaderExtractorConfigClass", bound=UploaderExtractorConfig)
+RunHandle = Callable[[CogniteClient, AbstractStateStore, UploaderExtractorConfigClass, CancellationToken], None]
 
 
 class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
@@ -76,19 +78,17 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
         *,
         name: str,
         description: str,
-        version: Optional[str] = None,
-        run_handle: Optional[
-            Callable[[CogniteClient, AbstractStateStore, UploaderExtractorConfigClass, CancellationToken], None]
-        ] = None,
+        version: str | None = None,
+        run_handle: RunHandle | None = None,
         config_class: Type[UploaderExtractorConfigClass],
-        metrics: Optional[BaseMetrics] = None,
+        metrics: BaseMetrics | None = None,
         use_default_state_store: bool = True,
-        cancellation_token: Optional[CancellationToken] = None,
-        config_file_path: Optional[str] = None,
+        cancellation_token: CancellationToken | None = None,
+        config_file_path: str | None = None,
         continuous_extractor: bool = False,
         heartbeat_waiting_time: int = 600,
         handle_interrupts: bool = True,
-        middleware: Optional[List[Callable[[dict], dict]]] = None,
+        middleware: list[Callable[[dict], dict]] | None = None,
     ):
         super(UploaderExtractor, self).__init__(
             name=name,
@@ -170,7 +170,7 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
         return self
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+        self, exc_type: Type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
     ) -> bool:
         self.event_queue.__exit__(exc_type, exc_val, exc_tb)
         self.raw_queue.__exit__(exc_type, exc_val, exc_tb)

cognite/extractorutils/uploader_types.py

@@ -1,27 +1,19 @@
-import sys
-from typing import Iterable, List, Optional, Union
+from typing import Iterable, TypeAlias
 
 from cognite.client.data_classes import Event as _Event
 from cognite.client.data_classes import Row as _Row
-
-if sys.version_info >= (3, 10):
-    from typing import TypeAlias
-else:
-    from typing_extensions import TypeAlias
-
-
 from cognite.extractorutils.uploader.time_series import DataPoint
 
 
 class InsertDatapoints:
-    def __init__(self, *, id: Optional[int] = None, external_id: Optional[str] = None, datapoints: List[DataPoint]):
+    def __init__(self, *, id: int | None = None, external_id: str | None = None, datapoints: list[DataPoint]):
         self.id = id
         self.external_id = external_id
         self.datapoints = datapoints
 
 
 class RawRow:
-    def __init__(self, db_name: str, table_name: str, row: Union[_Row, Iterable[_Row]]):
+    def __init__(self, db_name: str, table_name: str, row: _Row | Iterable[_Row]):
         self.db_name = db_name
         self.table_name = table_name
         if isinstance(row, Iterable):
@@ -32,4 +24,4 @@ class RawRow:
 
 Event: TypeAlias = _Event
 
-CdfTypes = Union[Event, Iterable[Event], RawRow, Iterable[RawRow], InsertDatapoints, Iterable[InsertDatapoints]]
+CdfTypes = Event | Iterable[Event] | RawRow | Iterable[RawRow] | InsertDatapoints | Iterable[InsertDatapoints]

cognite/extractorutils/util.py

@@ -25,7 +25,7 @@ from functools import partial, wraps
 from io import RawIOBase
 from threading import Thread
 from time import time
-from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple, Type, TypeVar, Union
+from typing import Any, Callable, Generator, Iterable, Type, TypeVar
 
 from decorator import decorator
 
@@ -89,7 +89,7 @@ class EitherId:
         TypeError: If none of both of id types are set.
     """
 
-    def __init__(self, **kwargs: Union[int, str, None]):
+    def __init__(self, **kwargs: int | str | None):
         internal_id = kwargs.get("id")
         external_id = kwargs.get("externalId") or kwargs.get("external_id")
 
@@ -105,8 +105,8 @@ class EitherId:
         if external_id is not None and not isinstance(external_id, str):
             raise TypeError("External IDs must be strings")
 
-        self.internal_id: Optional[int] = internal_id
-        self.external_id: Optional[str] = external_id
+        self.internal_id: int | None = internal_id
+        self.external_id: str | None = external_id
 
     def type(self) -> str:
         """
@@ -117,7 +117,7 @@ class EitherId:
         """
         return "id" if self.internal_id is not None else "externalId"
 
-    def content(self) -> Union[int, str]:
+    def content(self) -> int | str:
         """
         Get the value of the ID
 
@@ -249,7 +249,7 @@ def add_extraction_pipeline(
             ##############################
             _logger.info(f"Starting to run function: {input_function.__name__}")
 
-            heartbeat_thread: Optional[Thread] = None
+            heartbeat_thread: Thread | None = None
             try:
                 heartbeat_thread = Thread(target=heartbeat_loop, name="HeartbeatLoop", daemon=True)
                 heartbeat_thread.start()
@@ -313,12 +313,12 @@ _T2 = TypeVar("_T2")
 def _retry_internal(
     f: Callable[..., _T2],
     cancellation_token: CancellationToken,
-    exceptions: Union[Tuple[Type[Exception], ...], Dict[Type[Exception], Callable[[Exception], bool]]],
+    exceptions: tuple[Type[Exception], ...] | dict[Type[Exception], Callable[[Exception], bool]],
     tries: int,
     delay: float,
-    max_delay: Optional[float],
+    max_delay: float | None,
     backoff: float,
-    jitter: Union[float, Tuple[float, float]],
+    jitter: float | tuple[float, float],
 ) -> _T2:
     logger = logging.getLogger(__name__)
 
@@ -366,13 +366,13 @@ def _retry_internal(
 
 
 def retry(
-    cancellation_token: Optional[CancellationToken] = None,
-    exceptions: Union[Tuple[Type[Exception], ...], Dict[Type[Exception], Callable[[Any], bool]]] = (Exception,),
+    cancellation_token: CancellationToken | None = None,
+    exceptions: tuple[Type[Exception], ...] | dict[Type[Exception], Callable[[Any], bool]] = (Exception,),
     tries: int = 10,
     delay: float = 1,
-    max_delay: Optional[float] = 60,
+    max_delay: float | None = 60,
     backoff: float = 2,
-    jitter: Union[float, Tuple[float, float]] = (0, 2),
+    jitter: float | tuple[float, float] = (0, 2),
 ) -> Callable[[Callable[..., _T2]], Callable[..., _T2]]:
     """
     Returns a retry decorator.
@@ -414,8 +414,8 @@ def retry(
 
 
 def requests_exceptions(
-    status_codes: Optional[List[int]] = None,
-) -> Dict[Type[Exception], Callable[[Any], bool]]:
+    status_codes: list[int] | None = None,
+) -> dict[Type[Exception], Callable[[Any], bool]]:
     """
     Retry exceptions from using the ``requests`` library. This will retry all connection and HTTP errors matching
     the given status codes.
@@ -448,8 +448,8 @@ def requests_exceptions(
 
 
 def httpx_exceptions(
-    status_codes: Optional[List[int]] = None,
-) -> Dict[Type[Exception], Callable[[Any], bool]]:
+    status_codes: list[int] | None = None,
+) -> dict[Type[Exception], Callable[[Any], bool]]:
     """
     Retry exceptions from using the ``httpx`` library. This will retry all connection and HTTP errors matching
     the given status codes.
@@ -482,8 +482,8 @@ def httpx_exceptions(
 
 
 def cognite_exceptions(
-    status_codes: Optional[List[int]] = None,
-) -> Dict[Type[Exception], Callable[[Any], bool]]:
+    status_codes: list[int] | None = None,
+) -> dict[Type[Exception], Callable[[Any], bool]]:
     """
     Retry exceptions from using the Cognite SDK. This will retry all connection and HTTP errors matching
     the given status codes.
@@ -569,9 +569,7 @@ def truncate_byte_len(item: str, ln: int) -> str:
 
 
 class BufferedReadWithLength(io.BufferedReader):
-    def __init__(
-        self, raw: RawIOBase, buffer_size: int, len: int, on_close: Optional[Callable[[], None]] = None
-    ) -> None:
+    def __init__(self, raw: RawIOBase, buffer_size: int, len: int, on_close: Callable[[], None] | None = None) -> None:
         super().__init__(raw, buffer_size)
         # Do not remove even if it appears to be unused. :P
         # Requests uses this to add the content-length header, which is necessary for writing to files in azure clusters
@@ -588,7 +586,7 @@ def iterable_to_stream(
     iterator: Iterable[bytes],
     file_size_bytes: int,
     buffer_size: int = io.DEFAULT_BUFFER_SIZE,
-    on_close: Optional[Callable[[], None]] = None,
+    on_close: Callable[[], None] | None = None,
 ) -> BufferedReadWithLength:
     class ChunkIteratorStream(io.RawIOBase):
         def __init__(self) -> None:

{cognite_extractor_utils-7.5.4.dist-info → cognite_extractor_utils-7.5.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: cognite-extractor-utils
-Version: 7.5.4
+Version: 7.5.6
 Summary: Utilities for easier development of extractors for CDF
 Home-page: https://github.com/cognitedata/python-extractor-utils
 License: Apache-2.0
@@ -19,10 +19,11 @@ Requires-Dist: arrow (>=1.0.0,<2.0.0)
 Requires-Dist: azure-identity (>=1.14.0,<2.0.0)
 Requires-Dist: azure-keyvault-secrets (>=4.7.0,<5.0.0)
 Requires-Dist: cognite-sdk (>=7.59.0,<8.0.0)
-Requires-Dist: croniter (>=5.0.0,<6.0.0)
+Requires-Dist: croniter (>=6.0.0,<7.0.0)
 Requires-Dist: dacite (>=1.6.0,<2.0.0)
 Requires-Dist: decorator (>=5.1.1,<6.0.0)
 Requires-Dist: httpx (>=0.27.0,<0.28.0)
+Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
 Requires-Dist: more-itertools (>=10.0.0,<11.0.0)
 Requires-Dist: orjson (>=3.10.3,<4.0.0)
 Requires-Dist: prometheus-client (>0.7.0,<=1.0.0)

cognite_extractor_utils-7.5.6.dist-info/RECORD

@@ -0,0 +1,49 @@
+cognite/extractorutils/__init__.py,sha256=uWu9rc8gh485FrNT-ocKmSQwkxPzFo7L9ac103j7E-s,764
+cognite/extractorutils/_inner_util.py,sha256=cdoz9Sl3Wt1IsxiCZlcd913_hKrTCxDRrM_L-Zn1_F8,1800
+cognite/extractorutils/base.py,sha256=pV3xy0Dzt8q9I5DvI-TvmRZXMmSTk8Kk-d0jZWa_ua8,16333
+cognite/extractorutils/configtools/__init__.py,sha256=llNMzHu4yCWx5Kjm8G9IN5Pij8OUaVT_VZuZ2r3JtAA,3616
+cognite/extractorutils/configtools/_util.py,sha256=uXpR8YnEkfeZOuaZGjRRk_wgC5AGOEKNWMYfV50atsc,4746
+cognite/extractorutils/configtools/elements.py,sha256=ti3PFmwHyiFJFXNEzObRY6IxQo18LABSsYafPxuoYSU,26590
+cognite/extractorutils/configtools/loaders.py,sha256=w8NoZcZJZbEctvkTq8aG_UH2x2gct_fpb2KenksmVaQ,18294
+cognite/extractorutils/configtools/validators.py,sha256=xug3GOMIO4NOdyyvXtYlpKyq9wuDtGf7-xqIefD5bIo,1016
+cognite/extractorutils/exceptions.py,sha256=NDmiElg1cmGMwIl82kpCDF37UcAFNnfDK9NxUn_u2rk,1149
+cognite/extractorutils/metrics.py,sha256=-sUBaZ7lNrcdxuQcsh7rU-CwMNTqlT3DiMRyn5CxPTQ,15422
+cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cognite/extractorutils/statestore/__init__.py,sha256=hV3r11FUXkH6-60Ct6zLSROMNVrEeiE3Shmkf28Q-co,359
+cognite/extractorutils/statestore/_base.py,sha256=mWdFk4EZl886V6uXRj4O2sv2_ANJ3Sigmgeql-XEsmc,2675
+cognite/extractorutils/statestore/hashing.py,sha256=Le6PUpLYV7kTKgO2nc5BKCEf-3LTXoGzEVzLtw8tkn0,8011
+cognite/extractorutils/statestore/watermark.py,sha256=U_cA0XlqkgMML-ZeEl13KE8KjQHsId5t7mMHibRhUyA,16713
+cognite/extractorutils/threading.py,sha256=RN9oEXO6N2RqYKThFoDqzSeo593hkzTVePK1KSVOu3A,3586
+cognite/extractorutils/unstable/__init__.py,sha256=L6nqJHjylpk67CE-PbXJyb_TBI4yjhEYEz9J9WShDfM,341
+cognite/extractorutils/unstable/configuration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cognite/extractorutils/unstable/configuration/exceptions.py,sha256=-cziC11IbUP308ldbAYoQn4x2SNCIxYanN2eIV1n9To,654
+cognite/extractorutils/unstable/configuration/loaders.py,sha256=iMlCx6abKaDHx5-nOQSRtf-creqJPv1QrnbapCaIZkA,3689
+cognite/extractorutils/unstable/configuration/models.py,sha256=jFlA5eEeNRq39KEwAjZV9UkbV2juVUHANNeXq0VtqL4,8210
+cognite/extractorutils/unstable/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cognite/extractorutils/unstable/core/_dto.py,sha256=tvvy39cvf-QT28GWz5FpqxQ5vAVk0t69JoPPhpWlweY,1293
+cognite/extractorutils/unstable/core/_messaging.py,sha256=D9rOW8fijryXffbm90d8VTf2vy5FmwVGU-H0O-cn-EI,68
+cognite/extractorutils/unstable/core/base.py,sha256=QljO7Zpn5RSTEI9PHIavhKWdr4Hp-Ni5tdmsQ_ocOLk,12190
+cognite/extractorutils/unstable/core/errors.py,sha256=D8QAaqwJec62ZbhBNC0flmKjw_EdHLKGn8npqtPQhZE,1706
+cognite/extractorutils/unstable/core/restart_policy.py,sha256=SodG2Gs9Es05yk3EbAAWY_sbSoBUmhTRrUMBR4BSQbQ,622
+cognite/extractorutils/unstable/core/runtime.py,sha256=sb8ouTCZqvzpns_8UpVwPd4nGnfinf7vsVvOk23jksQ,11834
+cognite/extractorutils/unstable/core/tasks.py,sha256=K3R40sNSqYJ1Oc0UMTUDF4lY_WaZ7HokvZ5kctDsjGQ,585
+cognite/extractorutils/unstable/scheduling/__init__.py,sha256=L90_rCZNHvti-PInne0r7W9edIkifctELjiaxEoQiSc,67
+cognite/extractorutils/unstable/scheduling/_scheduler.py,sha256=tzu3-olhBU8uFDYj-Q6mEJUVBVin8wSGJONJVrNP3NE,3694
+cognite/extractorutils/unstable/scheduling/_schedules.py,sha256=y0NVeXYZOFcAyzBgAe8jqK0W-SZL5m99UwXAacGzqIw,677
+cognite/extractorutils/uploader/__init__.py,sha256=MgyvZojwLE-oUCZ0VALISd2rUCqShlyozxhzAKX5uj4,3396
+cognite/extractorutils/uploader/_base.py,sha256=JPr5Dp25XYzwN4MJ2ddd-xhPg5kVV3jASNecD8sAaKs,5273
+cognite/extractorutils/uploader/_metrics.py,sha256=J2LJXb19L_SLSJ_voNIQHYLp0pjxUKevpH1q_xKX6Hk,3247
+cognite/extractorutils/uploader/assets.py,sha256=SDX48xjqIT4tbQ9HtaIgQT8bw61XHJGic5ofZJeK7UE,5692
+cognite/extractorutils/uploader/data_modeling.py,sha256=Vd9eDWE-KPICChtxcKZdFcH3mSbavD8s1627wXxF_SI,3593
+cognite/extractorutils/uploader/events.py,sha256=qo1rVhk3eUfcbNLauZfvBohQ2aFRazbyGuMFcU-UyQ8,5640
+cognite/extractorutils/uploader/files.py,sha256=3VH8lsZmPL4TI3r_mIzTf8T2YmYc3kAtyBeo_4g9zP0,26610
+cognite/extractorutils/uploader/raw.py,sha256=VMYfeZN8XAHfZ77AuGcL85bIWvhaO7-Whx_marnGAmQ,6692
+cognite/extractorutils/uploader/time_series.py,sha256=yBN7ppD5hg0CgUIw7WvhhAPyOj0gbIWG4_-ifPaAuOE,26575
+cognite/extractorutils/uploader/upload_failure_handler.py,sha256=Oj3xDK_qlGQdEOzswE-6ti7tDAQXR0Rvee3lg6KBg3s,2000
+cognite/extractorutils/uploader_extractor.py,sha256=X71M_7JcGMwC3kHMETmTF8cdjSQwZaNmIGlT-mBs3Pk,7687
+cognite/extractorutils/uploader_types.py,sha256=eLKFQJT53zpn9_3-SDUtgHUMASGdK7c85HWrLWEF-JE,865
+cognite/extractorutils/util.py,sha256=TL3fkHlvPqWjdyr4yorq5LNJbPxJSom69HKyeQM92xE,21042
+cognite_extractor_utils-7.5.6.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+cognite_extractor_utils-7.5.6.dist-info/METADATA,sha256=A1Sc24JpE_1afcPP8Dor7_f6KJb6NQ5dSY6_12Zswfk,5691
+cognite_extractor_utils-7.5.6.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+cognite_extractor_utils-7.5.6.dist-info/RECORD,,

{cognite_extractor_utils-7.5.4.dist-info → cognite_extractor_utils-7.5.6.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.0.0
 Root-Is-Purelib: true
 Tag: py3-none-any

cognite/extractorutils/unstable/core/__main__.py

@@ -1,31 +0,0 @@
-"""
-Example of how you would build an extractor with the new base class
-"""
-
-from cognite.extractorutils.unstable.configuration.models import ExtractorConfig
-
-from .base import Extractor
-from .runtime import Runtime
-
-
-class MyConfig(ExtractorConfig):
-    parameter_one: int
-    parameter_two: str
-
-
-class MyExtractor(Extractor[MyConfig]):
-    NAME = "Test extractor"
-    EXTERNAL_ID = "test-extractor"
-    DESCRIPTION = "Test of the new runtime"
-    VERSION = "1.0.0"
-    CONFIG_TYPE = MyConfig
-
-    def run(self) -> None:
-        self.logger.info("Started!")
-        if not self.cancellation_token.wait(10):
-            raise ValueError("Oops")
-
-
-if __name__ == "__main__":
-    runtime = Runtime(MyExtractor)
-    runtime.run()