cognite-extractor-utils 6.4.0__py3-none-any.whl → 7.0.0__py3-none-any.whl

cognite/extractorutils/uploader/time_series.py

@@ -13,13 +13,10 @@
 #  limitations under the License.
 
 import math
-import threading
 from datetime import datetime
 from types import TracebackType
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
 
-from requests import ConnectionError
-
 from cognite.client import CogniteClient
 from cognite.client.data_classes import (
     Sequence,
@@ -27,7 +24,8 @@ from cognite.client.data_classes import (
     SequenceRows,
     TimeSeries,
 )
-from cognite.client.exceptions import CogniteAPIError, CogniteDuplicatedError, CogniteNotFoundError
+from cognite.client.exceptions import CogniteDuplicatedError, CogniteNotFoundError
+from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader._base import (
     RETRIES,
     RETRY_BACKOFF_FACTOR,
@@ -44,7 +42,7 @@ from cognite.extractorutils.uploader._metrics import (
     TIMESERIES_UPLOADER_POINTS_WRITTEN,
     TIMESERIES_UPLOADER_QUEUE_SIZE,
 )
-from cognite.extractorutils.util import EitherId, retry
+from cognite.extractorutils.util import EitherId, cognite_exceptions, retry
 
 MIN_DATAPOINT_TIMESTAMP = -2208988800000
 MAX_DATAPOINT_STRING_LENGTH = 255
@@ -108,7 +106,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
         thread_name: Optional[str] = None,
         create_missing: Union[Callable[[str, DataPointList], TimeSeries], bool] = False,
         data_set_id: Optional[int] = None,
-        cancellation_token: threading.Event = threading.Event(),
+        cancellation_token: Optional[CancellationToken] = None,
     ):
         # Super sets post_upload and threshold
         super().__init__(
@@ -169,7 +167,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             return True
 
     def add_to_upload_queue(
-        self, *, id: Optional[int] = None, external_id: Optional[str] = None, datapoints: DataPointList = []
+        self, *, id: Optional[int] = None, external_id: Optional[str] = None, datapoints: Optional[DataPointList] = None
     ) -> None:
         """
         Add data points to upload queue. The queue will be uploaded if the queue size is larger than the threshold
@@ -180,6 +178,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             external_id: External ID of time series. Either this or external_id must be set.
             datapoints: List of data points to add
         """
+        datapoints = datapoints or []
         old_len = len(datapoints)
         datapoints = list(filter(self._is_datapoint_valid, datapoints))
 
@@ -207,11 +206,82 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
         """
         Trigger an upload of the queue, clears queue afterwards
         """
+
+        @retry(
+            exceptions=cognite_exceptions(),
+            cancellation_token=self.cancellation_token,
+            tries=RETRIES,
+            delay=RETRY_DELAY,
+            max_delay=RETRY_MAX_DELAY,
+            backoff=RETRY_BACKOFF_FACTOR,
+        )
+        def _upload_batch(upload_this: List[Dict], retries: int = 5) -> List[Dict]:
+            if len(upload_this) == 0:
+                return upload_this
+
+            try:
+                self.cdf_client.time_series.data.insert_multiple(upload_this)
+
+            except CogniteNotFoundError as ex:
+                if not retries:
+                    raise ex
+
+                if not self.create_missing:
+                    self.logger.error("Could not upload data points to %s: %s", str(ex.not_found), str(ex))
+
+                # Get IDs of time series that exists, but failed because of the non-existing time series
+                retry_these = [EitherId(**id_dict) for id_dict in ex.failed if id_dict not in ex.not_found]
+
+                if self.create_missing:
+                    # Get the time series that can be created
+                    create_these_ids = set(
+                        [id_dict["externalId"] for id_dict in ex.not_found if "externalId" in id_dict]
+                    )
+                    datapoints_lists: Dict[str, DataPointList] = {
+                        ts_dict["externalId"]: ts_dict["datapoints"]
+                        for ts_dict in upload_this
+                        if ts_dict["externalId"] in create_these_ids
+                    }
+
+                    self.logger.info(f"Creating {len(create_these_ids)} time series")
+                    to_create: List[TimeSeries] = [
+                        self.missing_factory(external_id, datapoints_lists[external_id])
+                        for external_id in create_these_ids
+                    ]
+                    if self.data_set_id is not None:
+                        for ts in to_create:
+                            if ts.data_set_id is None:
+                                ts.data_set_id = self.data_set_id
+                    self.cdf_client.time_series.create(to_create)
+
+                    retry_these.extend([EitherId(external_id=i) for i in create_these_ids])
+
+                    if len(ex.not_found) != len(create_these_ids):
+                        missing = [id_dict for id_dict in ex.not_found if id_dict.get("externalId") not in retry_these]
+                        missing_num = len(ex.not_found) - len(create_these_ids)
+                        self.logger.error(
+                            f"{missing_num} time series not found, and could not be created automatically:\n"
+                            + str(missing)
+                            + "\nData will be dropped"
+                        )
+
+                # Remove entries with non-existing time series from upload queue
+                upload_this = [
+                    entry
+                    for entry in upload_this
+                    if EitherId(id=entry.get("id"), external_id=entry.get("externalId")) in retry_these
+                ]
+
+                # Upload remaining
+                _upload_batch(upload_this, retries - 1)
+
+            return upload_this
+
         if len(self.upload_queue) == 0:
             return
 
         with self.lock:
-            upload_this = self._upload_batch(
+            upload_this = _upload_batch(
                 [
                     {either_id.type(): either_id.content(), "datapoints": datapoints}
                     for either_id, datapoints in self.upload_queue.items()
@@ -219,7 +289,7 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
                 ]
             )
 
-            for either_id, datapoints in self.upload_queue.items():
+            for _either_id, datapoints in self.upload_queue.items():
                 self.points_written.inc(len(datapoints))
 
             try:
@@ -232,72 +302,6 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
             self.upload_queue_size = 0
             self.queue_size.set(self.upload_queue_size)
 
-    @retry(
-        exceptions=(CogniteAPIError, ConnectionError),
-        tries=RETRIES,
-        delay=RETRY_DELAY,
-        max_delay=RETRY_MAX_DELAY,
-        backoff=RETRY_BACKOFF_FACTOR,
-    )
-    def _upload_batch(self, upload_this: List[Dict], retries: int = 5) -> List[Dict]:
-        if len(upload_this) == 0:
-            return upload_this
-
-        try:
-            self.cdf_client.time_series.data.insert_multiple(upload_this)
-
-        except CogniteNotFoundError as ex:
-            if not retries:
-                raise ex
-
-            if not self.create_missing:
-                self.logger.error("Could not upload data points to %s: %s", str(ex.not_found), str(ex))
-
-            # Get IDs of time series that exists, but failed because of the non-existing time series
-            retry_these = [EitherId(**id_dict) for id_dict in ex.failed if id_dict not in ex.not_found]
-
-            if self.create_missing:
-                # Get the time series that can be created
-                create_these_ids = set([id_dict["externalId"] for id_dict in ex.not_found if "externalId" in id_dict])
-                datapoints_lists: Dict[str, DataPointList] = {
-                    ts_dict["externalId"]: ts_dict["datapoints"]
-                    for ts_dict in upload_this
-                    if ts_dict["externalId"] in create_these_ids
-                }
-
-                self.logger.info(f"Creating {len(create_these_ids)} time series")
-                to_create: List[TimeSeries] = [
-                    self.missing_factory(external_id, datapoints_lists[external_id]) for external_id in create_these_ids
-                ]
-                if self.data_set_id is not None:
-                    for ts in to_create:
-                        if ts.data_set_id is None:
-                            ts.data_set_id = self.data_set_id
-                self.cdf_client.time_series.create(to_create)
-
-                retry_these.extend([EitherId(external_id=i) for i in create_these_ids])
-
-                if len(ex.not_found) != len(create_these_ids):
-                    missing = [id_dict for id_dict in ex.not_found if id_dict.get("externalId") not in retry_these]
-                    missing_num = len(ex.not_found) - len(create_these_ids)
-                    self.logger.error(
-                        f"{missing_num} time series not found, and could not be created automatically:\n"
-                        + str(missing)
-                        + "\nData will be dropped"
-                    )
-
-            # Remove entries with non-existing time series from upload queue
-            upload_this = [
-                entry
-                for entry in upload_this
-                if EitherId(id=entry.get("id"), external_id=entry.get("externalId")) in retry_these
-            ]
-
-            # Upload remaining
-            self._upload_batch(upload_this, retries - 1)
-
-        return upload_this
-
     def __enter__(self) -> "TimeSeriesUploadQueue":
         """
         Wraps around start method, for use as context manager
@@ -341,7 +345,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
         trigger_log_level: str = "DEBUG",
         thread_name: Optional[str] = None,
         create_missing: bool = False,
-        cancellation_token: threading.Event = threading.Event(),
+        cancellation_token: Optional[CancellationToken] = None,
     ):
         """
         Args:
@@ -504,6 +508,42 @@ class SequenceUploadQueue(AbstractUploadQueue):
         """
         Trigger an upload of the queue, clears queue afterwards
         """
+
+        @retry(
+            exceptions=cognite_exceptions(),
+            cancellation_token=self.cancellation_token,
+            tries=RETRIES,
+            delay=RETRY_DELAY,
+            max_delay=RETRY_MAX_DELAY,
+            backoff=RETRY_BACKOFF_FACTOR,
+        )
+        def _upload_single(either_id: EitherId, upload_this: SequenceData) -> SequenceData:
+            self.logger.debug("Writing {} rows to sequence {}".format(len(upload_this.values), either_id))
+
+            try:
+                self.cdf_client.sequences.data.insert(
+                    id=either_id.internal_id,  # type: ignore
+                    external_id=either_id.external_id,  # type: ignore
+                    rows=upload_this,
+                    column_external_ids=None,
+                )
+            except CogniteNotFoundError as ex:
+                if self.create_missing:
+                    # Create missing sequence
+                    self._create_or_update(either_id)
+
+                    # Retry
+                    self.cdf_client.sequences.data.insert(
+                        id=either_id.internal_id,  # type: ignore
+                        external_id=either_id.external_id,  # type: ignore
+                        rows=upload_this,
+                        column_external_ids=None,
+                    )
+                else:
+                    raise ex
+
+            return upload_this
+
         if len(self.upload_queue) == 0:
             return
 
@@ -514,7 +554,7 @@ class SequenceUploadQueue(AbstractUploadQueue):
 
             for either_id, upload_this in self.upload_queue.items():
                 _labels = str(either_id.content())
-                self._upload_single(either_id, upload_this)
+                _upload_single(either_id, upload_this)
                 self.points_written.inc()
 
             try:
@@ -527,40 +567,6 @@ class SequenceUploadQueue(AbstractUploadQueue):
             self.upload_queue_size = 0
             self.queue_size.set(self.upload_queue_size)
 
-    @retry(
-        exceptions=(CogniteAPIError, ConnectionError),
-        tries=RETRIES,
-        delay=RETRY_DELAY,
-        max_delay=RETRY_MAX_DELAY,
-        backoff=RETRY_BACKOFF_FACTOR,
-    )
-    def _upload_single(self, either_id: EitherId, upload_this: SequenceData) -> SequenceData:
-        self.logger.debug("Writing {} rows to sequence {}".format(len(upload_this.values), either_id))
-
-        try:
-            self.cdf_client.sequences.data.insert(
-                id=either_id.internal_id,  # type: ignore
-                external_id=either_id.external_id,  # type: ignore
-                rows=upload_this,
-                column_external_ids=None,
-            )
-        except CogniteNotFoundError as ex:
-            if self.create_missing:
-                # Create missing sequence
-                self._create_or_update(either_id)
-
-                # Retry
-                self.cdf_client.sequences.data.insert(
-                    id=either_id.internal_id,  # type: ignore
-                    external_id=either_id.external_id,  # type: ignore
-                    rows=upload_this,
-                    column_external_ids=None,
-                )
-            else:
-                raise ex
-
-        return upload_this
-
     def _create_or_update(self, either_id: EitherId) -> None:
         """
         Create or update sequence, based on provided metadata and column definitions

cognite/extractorutils/uploader_extractor.py

@@ -15,7 +15,6 @@
 """
 A module containing a slightly more advanced base extractor class, sorting a generic output into upload queues.
 """
-import threading
 from dataclasses import dataclass
 from types import TracebackType
 from typing import Any, Callable, Iterable, List, Optional, Type, TypeVar
@@ -27,6 +26,7 @@ from cognite.extractorutils.base import Extractor
 from cognite.extractorutils.configtools import BaseConfig, TimeIntervalConfig
 from cognite.extractorutils.metrics import BaseMetrics
 from cognite.extractorutils.statestore import AbstractStateStore
+from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader import EventUploadQueue, RawUploadQueue, TimeSeriesUploadQueue
 from cognite.extractorutils.uploader_types import CdfTypes, Event, InsertDatapoints, RawRow
 
@@ -78,17 +78,17 @@ class UploaderExtractor(Extractor[UploaderExtractorConfigClass]):
         description: str,
         version: Optional[str] = None,
         run_handle: Optional[
-            Callable[[CogniteClient, AbstractStateStore, UploaderExtractorConfigClass, threading.Event], None]
+            Callable[[CogniteClient, AbstractStateStore, UploaderExtractorConfigClass, CancellationToken], None]
         ] = None,
         config_class: Type[UploaderExtractorConfigClass],
         metrics: Optional[BaseMetrics] = None,
         use_default_state_store: bool = True,
-        cancellation_token: threading.Event = threading.Event(),
+        cancellation_token: Optional[CancellationToken] = None,
         config_file_path: Optional[str] = None,
         continuous_extractor: bool = False,
         heartbeat_waiting_time: int = 600,
         handle_interrupts: bool = True,
-        middleware: List[Callable[[dict], dict]] = [],
+        middleware: Optional[List[Callable[[dict], dict]]] = None,
     ):
         super(UploaderExtractor, self).__init__(
             name=name,

cognite/extractorutils/util.py

@@ -18,10 +18,8 @@ extractors.
 """
 import logging
 import random
-import signal
-import threading
 from functools import partial, wraps
-from threading import Event, Thread
+from threading import Thread
 from time import time
 from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple, Type, TypeVar, Union
 
@@ -29,7 +27,8 @@ from decorator import decorator
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import Asset, ExtractionPipelineRun, TimeSeries
-from cognite.client.exceptions import CogniteNotFoundError
+from cognite.client.exceptions import CogniteAPIError, CogniteException, CogniteNotFoundError
+from cognite.extractorutils.threading import CancellationToken
 
 
 def _ensure(endpoint: Any, items: Iterable[Any]) -> None:
@@ -73,27 +72,6 @@ def ensure_assets(cdf_client: CogniteClient, assets: Iterable[Asset]) -> None:
     _ensure(cdf_client.assets, assets)
 
 
-def set_event_on_interrupt(stop_event: Event) -> None:
-    """
-    Set given event on SIGINT (Ctrl-C) instead of throwing a KeyboardInterrupt exception.
-
-    Args:
-        stop_event: Event to set
-    """
-
-    def sigint_handler(sig_num: int, frame: Any) -> None:
-        logger = logging.getLogger(__name__)
-        logger.warning("Interrupt signal received, stopping extractor gracefully")
-        stop_event.set()
-        logger.info("Waiting for threads to complete. Send another interrupt to force quit.")
-        signal.signal(signal.SIGINT, signal.default_int_handler)
-
-    try:
-        signal.signal(signal.SIGINT, sigint_handler)
-    except ValueError as e:
-        logging.getLogger(__name__).warning(f"Could not register handler for interrupt signals: {str(e)}")
-
-
 class EitherId:
     """
     Class representing an ID in CDF, which can either be an external or internal ID. An EitherId can only hold one ID
@@ -220,7 +198,7 @@ def add_extraction_pipeline(
     # TODO 1. Consider refactoring this decorator to share methods with the Extractor context manager in .base.py
     # as they serve a similar purpose
 
-    cancellation_token: Event = Event()
+    cancellation_token: CancellationToken = CancellationToken()
 
     _logger = logging.getLogger(__name__)
 
@@ -254,7 +232,7 @@ def add_extraction_pipeline(
                 )
 
             def heartbeat_loop() -> None:
-                while not cancellation_token.is_set():
+                while not cancellation_token.is_cancelled:
                     cognite_client.extraction_pipelines.runs.create(
                         ExtractionPipelineRun(extpipe_external_id=extraction_pipeline_ext_id, status="seen")
                     )
@@ -279,7 +257,7 @@ def add_extraction_pipeline(
                 _report_success()
                 _logger.info("Extraction ran successfully")
             finally:
-                cancellation_token.set()
+                cancellation_token.cancel()
                 if heartbeat_thread:
                     heartbeat_thread.join()
 
@@ -290,7 +268,7 @@ def add_extraction_pipeline(
     return decorator_ext_pip
 
 
-def throttled_loop(target_time: int, cancellation_token: Event) -> Generator[None, None, None]:
+def throttled_loop(target_time: int, cancellation_token: CancellationToken) -> Generator[None, None, None]:
     """
     A loop generator that automatically sleeps until each iteration has taken the desired amount of time. Useful for
     when you want to avoid overloading a source system with requests.
@@ -312,7 +290,7 @@ def throttled_loop(target_time: int, cancellation_token: Event) -> Generator[Non
     """
     logger = logging.getLogger(__name__)
 
-    while not cancellation_token.is_set():
+    while not cancellation_token.is_cancelled:
         start_time = time()
         yield
         iteration_time = time() - start_time
@@ -329,7 +307,7 @@ _T2 = TypeVar("_T2")
 
 def _retry_internal(
     f: Callable[..., _T2],
-    cancellation_token: threading.Event,
+    cancellation_token: CancellationToken,
     exceptions: Union[Tuple[Type[Exception], ...], Dict[Type[Exception], Callable[[Exception], bool]]],
     tries: int,
     delay: float,
@@ -339,7 +317,7 @@ def _retry_internal(
 ) -> _T2:
     logger = logging.getLogger(__name__)
 
-    while tries and not cancellation_token.is_set():
+    while tries and not cancellation_token.is_cancelled:
         try:
             return f()
 
@@ -380,7 +358,7 @@ def _retry_internal(
 
 
 def retry(
-    cancellation_token: threading.Event = threading.Event(),
+    cancellation_token: Optional[CancellationToken] = None,
     exceptions: Union[Tuple[Type[Exception], ...], Dict[Type[Exception], Callable[[Any], bool]]] = (Exception,),
     tries: int = 10,
     delay: float = 1,
@@ -415,7 +393,7 @@ def retry(
 
         return _retry_internal(
             partial(f, *args, **kwargs),
-            cancellation_token,
+            cancellation_token or CancellationToken(),
             exceptions,
             tries,
             delay,
@@ -428,7 +406,7 @@ def retry(
 
 
 def requests_exceptions(
-    status_codes: List[int] = [408, 425, 429, 500, 502, 503, 504],
+    status_codes: Optional[List[int]] = None,
 ) -> Dict[Type[Exception], Callable[[Any], bool]]:
     """
     Retry exceptions from using the ``requests`` library. This will retry all connection and HTTP errors matching
@@ -443,6 +421,7 @@ def requests_exceptions(
             ...
 
     """
+    status_codes = status_codes or [408, 425, 429, 500, 502, 503, 504]
     # types ignored, since they are not installed as we don't depend on the package
     from requests.exceptions import HTTPError, RequestException  # type: ignore
 
@@ -461,7 +440,7 @@ def requests_exceptions(
 
 
 def httpx_exceptions(
-    status_codes: List[int] = [408, 425, 429, 500, 502, 503, 504],
+    status_codes: Optional[List[int]] = None,
 ) -> Dict[Type[Exception], Callable[[Any], bool]]:
     """
     Retry exceptions from using the ``httpx`` library. This will retry all connection and HTTP errors matching
@@ -476,6 +455,7 @@ def httpx_exceptions(
             ...
 
     """
+    status_codes = status_codes or [408, 425, 429, 500, 502, 503, 504]
     # types ignored, since they are not installed as we don't depend on the package
     from httpx import HTTPError, HTTPStatusError  # type: ignore
 
@@ -491,3 +471,28 @@ def httpx_exceptions(
             return True
 
     return {HTTPError: handle_http_errors}
+
+
+def cognite_exceptions(
+    status_codes: Optional[List[int]] = None,
+) -> Dict[Type[Exception], Callable[[Any], bool]]:
+    """
+    Retry exceptions from using the Cognite SDK. This will retry all connection and HTTP errors matching
+    the given status codes.
+
+    Example:
+
+    .. code-block:: python
+
+        @retry(exceptions = cognite_exceptions())
+        def my_function() -> None:
+            ...
+    """
+    status_codes = status_codes or [408, 425, 429, 500, 502, 503, 504]
+
+    def handle_cognite_errors(exception: CogniteException) -> bool:
+        if isinstance(exception, CogniteAPIError):
+            return exception.code in status_codes
+        return True
+
+    return {CogniteException: handle_cognite_errors}

{cognite_extractor_utils-6.4.0.dist-info → cognite_extractor_utils-7.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-extractor-utils
-Version: 6.4.0
+Version: 7.0.0
 Summary: Utilities for easier development of extractors for CDF
 Home-page: https://github.com/cognitedata/python-extractor-utils
 License: Apache-2.0
@@ -21,8 +21,6 @@ Requires-Dist: azure-keyvault-secrets (>=4.7.0,<5.0.0)
 Requires-Dist: cognite-sdk (>=7,<8)
 Requires-Dist: dacite (>=1.6.0,<2.0.0)
 Requires-Dist: decorator (>=5.1.1,<6.0.0)
-Requires-Dist: jq (>=1.3.0,<2.0.0) ; sys_platform == "darwin"
-Requires-Dist: jq (>=1.3.0,<2.0.0) ; sys_platform == "linux"
 Requires-Dist: more-itertools (>=10.0.0,<11.0.0)
 Requires-Dist: prometheus-client (>0.7.0,<=1.0.0)
 Requires-Dist: psutil (>=5.7.0,<6.0.0)

cognite_extractor_utils-7.0.0.dist-info/RECORD

@@ -0,0 +1,27 @@
+cognite/extractorutils/__init__.py,sha256=7RFWajf36G6iOaLnzGuuL3rDlIkt5L1GnsrgApS5-_8,739
+cognite/extractorutils/_inner_util.py,sha256=gmz6aqS7jDNsg8z4RHgJjMFohDLOMiaU4gMWBhg3xcE,1558
+cognite/extractorutils/base.py,sha256=o5oFzGBaszv7-HWMVG2ATmZwZfzi8enyrYIKwUagyCQ,16148
+cognite/extractorutils/configtools/__init__.py,sha256=fj9kH8DdisNi9mI8cKm2sz50vnzeOkJQErIGB3mTYRo,2861
+cognite/extractorutils/configtools/_util.py,sha256=SZycZm_py9v9WZbDiDQbgS6_PiLtu-TtwuuH7tG2YCI,4739
+cognite/extractorutils/configtools/elements.py,sha256=CM9ZVAUgy7DnbTGI_kgnNclioEkaUqv0Lt_a2zoS8Yc,20680
+cognite/extractorutils/configtools/loaders.py,sha256=Q062K-AFJLn9MjJaURar_Le79edGUZvvpgQdg9wm51I,15082
+cognite/extractorutils/exceptions.py,sha256=PERRmySUfJRM2Ta8cFvADTe-KUdXsoMLKdk4140AOHI,1061
+cognite/extractorutils/metrics.py,sha256=IzYevOH19N_Nhi2XfhcuGgsGfmF7SexfcHcTQyZmH1o,15635
+cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cognite/extractorutils/statestore.py,sha256=iJdVJQihjU2ljfS46FNEKdD5PSurTzaAH09iGbDYJro,18598
+cognite/extractorutils/threading.py,sha256=C77KgNZolI8E_xbv-G-BkJwLEALkkvZs5UoNx4hJiFg,3249
+cognite/extractorutils/uploader/__init__.py,sha256=W22u6QHA4cR0j78LN5LTL5YGbfC-uTApagTyP5ab7uQ,3110
+cognite/extractorutils/uploader/_base.py,sha256=-aFfoMSBGd9YUUMHL3ZQpLIuNMA7TNklWCEjPA18ER8,5282
+cognite/extractorutils/uploader/_metrics.py,sha256=J2LJXb19L_SLSJ_voNIQHYLp0pjxUKevpH1q_xKX6Hk,3247
+cognite/extractorutils/uploader/assets.py,sha256=2E90N1kxsaA6Ah4h0_r_dTVhDYY_68ItRWrHYkkltJw,5628
+cognite/extractorutils/uploader/events.py,sha256=NZP2tMoU_rh_rb-EZiUBsOT5KdNABHN4c9Oddk0OsdE,5680
+cognite/extractorutils/uploader/files.py,sha256=2BKwdgj2rlJqJWU-13ktCINdktI6IoKkeU2nAWUKOus,13176
+cognite/extractorutils/uploader/raw.py,sha256=wFjF90PFTjmByOWx_Y4_YfDJ2w2jl0EQJ2Tjx2MP2PM,6738
+cognite/extractorutils/uploader/time_series.py,sha256=VptUq129MY0t8yw4rxeL0kOhz2dMibz4XdyvfhfYGj8,26840
+cognite/extractorutils/uploader_extractor.py,sha256=E-mpVvbPg_Tk90U4S9JybV0duptJ2SXE88HB6npE3zI,7732
+cognite/extractorutils/uploader_types.py,sha256=5MKT14DUnTFVD5Nx4Zvnfp2SfaICuTKLWAFAYaZishk,1045
+cognite/extractorutils/util.py,sha256=PFxVkDfpAEXTONYC1U-iO5DiaE3snn81qVUUT2O8xAM,17025
+cognite_extractor_utils-7.0.0.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+cognite_extractor_utils-7.0.0.dist-info/METADATA,sha256=lSHI8N78-cWlmraTSSiHovCTL2P-4olyjpWLlwTwTD0,5437
+cognite_extractor_utils-7.0.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+cognite_extractor_utils-7.0.0.dist-info/RECORD,,

cognite/extractorutils/middleware.py

@@ -1,36 +0,0 @@
-"""
-This module is deprecated and will be removed in a future version
-"""
-
-
-from sys import platform
-from typing import Any, Union
-
-from cognite.client.data_classes import Row
-
-
-class JQMiddleware:
-    def __init__(self, jq_rules: str) -> None:
-        if platform == "win32":
-            raise Exception("Windows platform doesn't support jq bindings for Python yet")
-        import jq  # type: ignore
-
-        self._jq = jq.compile(jq_rules)
-
-    def __call__(self, data: Union[Row, dict]) -> Union[Row, dict]:
-        if not isinstance(data, (Row, dict)):
-            raise ValueError(f"type {type(data).__name__} is not currently supported")
-
-        if isinstance(data, Row):
-            data.columns = self._jq.input(data.columns).first()
-            self._raise_for_non_dict(data.columns)
-
-        if isinstance(data, dict):
-            data = self._jq.input(data).first()
-            self._raise_for_non_dict(data)
-
-        return data
-
-    def _raise_for_non_dict(self, data: Any) -> None:
-        if not isinstance(data, dict):
-            raise ValueError("output of jq middleware must be a dict")

cognite_extractor_utils-6.4.0.dist-info/RECORD

@@ -1,27 +0,0 @@
-cognite/extractorutils/__init__.py,sha256=kPofZ4EKmTEmGYHabSV5nihFfcJguswS0gTKtpaFGQs,739
-cognite/extractorutils/_inner_util.py,sha256=jCPLg-FfTpyLAHkhoKaKObNwbqm19d2Z11to_DYk5EU,1558
-cognite/extractorutils/base.py,sha256=8kzePjVlCIQVftDQ1t_4join9qhs-VOlB6eTpeX9zGk,16018
-cognite/extractorutils/configtools/__init__.py,sha256=fj9kH8DdisNi9mI8cKm2sz50vnzeOkJQErIGB3mTYRo,2861
-cognite/extractorutils/configtools/_util.py,sha256=SZycZm_py9v9WZbDiDQbgS6_PiLtu-TtwuuH7tG2YCI,4739
-cognite/extractorutils/configtools/elements.py,sha256=OgNuLF6iOBnPsANwMsSRrJ_0KwBngD_ZKIgE4NZfOOc,20626
-cognite/extractorutils/configtools/loaders.py,sha256=s4tHuNgBhk6TnrDDifIYQX8CickXVSqVCol9E62DPIs,14969
-cognite/extractorutils/exceptions.py,sha256=PERRmySUfJRM2Ta8cFvADTe-KUdXsoMLKdk4140AOHI,1061
-cognite/extractorutils/metrics.py,sha256=PFrOO0yDRGaFiJmNLXModKY7CW1gvHDBtPaoK-ETCxw,15508
-cognite/extractorutils/middleware.py,sha256=d5bnKEOmC49QmoBK7OyM9krOGDYX8sXxM00wRWKhmHg,1108
-cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cognite/extractorutils/statestore.py,sha256=wC619be0L0EKmn_Fn-ccT3-D2_wt4Mb9eAr_oqrDdis,18272
-cognite/extractorutils/uploader/__init__.py,sha256=W22u6QHA4cR0j78LN5LTL5YGbfC-uTApagTyP5ab7uQ,3110
-cognite/extractorutils/uploader/_base.py,sha256=ed9F-qFmlIylTg8UtuxfykYUfEsG_R0PrfufXYEPcjI,5169
-cognite/extractorutils/uploader/_metrics.py,sha256=J2LJXb19L_SLSJ_voNIQHYLp0pjxUKevpH1q_xKX6Hk,3247
-cognite/extractorutils/uploader/assets.py,sha256=Gqu1VJqfFcIQphsfqciDFghL_zlNhBmHU8hPfaKSyFw,5441
-cognite/extractorutils/uploader/events.py,sha256=dzqdDHRKsSUiBWqpcwJX9BpTiUGLb6abneqk1k_QmLA,5502
-cognite/extractorutils/uploader/files.py,sha256=u1tJDT8cCD9W9EegQWqOYevsj3BfOdFfyiO3DSWHjsc,11784
-cognite/extractorutils/uploader/raw.py,sha256=9kvHtJoZVLD-V9xcnNEkucwvaa8DPMqsXjvCWMGiw_g,6732
-cognite/extractorutils/uploader/time_series.py,sha256=ZPkz9NVCmz2vE7SkRm0-LjYi-ZKCRZ_Qe-YR1EVtGms,26315
-cognite/extractorutils/uploader_extractor.py,sha256=K8cH8CIDiFNB0fXNdUymE_MKh1GYfqxHHcRoH07CFew,7673
-cognite/extractorutils/uploader_types.py,sha256=5MKT14DUnTFVD5Nx4Zvnfp2SfaICuTKLWAFAYaZishk,1045
-cognite/extractorutils/util.py,sha256=N63Mb_hxfZOcGg7oZOGKnunjj-Lkjib1EK5azEKDeT0,16835
-cognite_extractor_utils-6.4.0.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-cognite_extractor_utils-6.4.0.dist-info/METADATA,sha256=Z5UZiVA4dM4Nk_L1cJPvY2oQDC-Kn0yt0IZPLxCZwUM,5560
-cognite_extractor_utils-6.4.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-cognite_extractor_utils-6.4.0.dist-info/RECORD,,