cognite-extractor-utils 7.0.0__py3-none-any.whl → 7.0.2__py3-none-any.whl

cognite/extractorutils/__init__.py

@@ -16,5 +16,5 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.0.0"
+__version__ = "7.0.2"
 from .base import Extractor

cognite/extractorutils/configtools/__init__.py

@@ -85,10 +85,15 @@ Get a state store object as configured:
 However, all of these things will be automatically done for you if you are using the base Extractor class.
 """
 
+from cognite.extractorutils.exceptions import InvalidConfigError
+
 from .elements import (
+    AuthenticatorConfig,
     BaseConfig,
+    CertificateConfig,
     CogniteConfig,
     ConfigType,
+    ConnectionConfig,
     EitherIdConfig,
     FileSizeConfig,
     LocalStateStoreConfig,
@@ -99,4 +104,4 @@ from .elements import (
     StateStoreConfig,
     TimeIntervalConfig,
 )
-from .loaders import ConfigResolver, load_yaml
+from .loaders import ConfigResolver, KeyVaultAuthenticationMethod, KeyVaultLoader, load_yaml, load_yaml_dict

cognite/extractorutils/configtools/elements.py

@@ -53,6 +53,10 @@ _logger = logging.getLogger(__name__)
 
 @dataclass
 class CertificateConfig:
+    """
+    Configuration parameters for certificates
+    """
+
     path: str
     password: Optional[str]
     authority_url: Optional[str] = None
@@ -94,6 +98,11 @@ class ConnectionConfig:
 
 @dataclass
 class EitherIdConfig:
+    """
+    Configuration parameter representing an ID in CDF, which can either be an external or internal ID.
+    An EitherId can only hold one ID type, not both.
+    """
+
     id: Optional[int]
     external_id: Optional[str]
 
@@ -103,6 +112,10 @@ class EitherIdConfig:
 
 
 class TimeIntervalConfig(yaml.YAMLObject):
+    """
+    Configuration parameter for setting a time interval
+    """
+
     def __init__(self, expression: str) -> None:
         self._interval, self._expression = TimeIntervalConfig._parse_expression(expression)
 
@@ -167,6 +180,10 @@ class TimeIntervalConfig(yaml.YAMLObject):
 
 
 class FileSizeConfig(yaml.YAMLObject):
+    """
+    Configuration parameter for setting a file size
+    """
+
     def __init__(self, expression: str) -> None:
         self._bytes, self._expression = FileSizeConfig._parse_expression(expression)
 
@@ -569,23 +586,39 @@ class BaseConfig(_BaseConfig):
 
 @dataclass
 class RawDestinationConfig:
+    """
+    Configuration parameters for using Raw
+    """
+
     database: str
     table: str
 
 
 @dataclass
 class RawStateStoreConfig(RawDestinationConfig):
+    """
+    Configuration of a state store based on CDF RAW
+    """
+
     upload_interval: TimeIntervalConfig = TimeIntervalConfig("30s")
 
 
 @dataclass
 class LocalStateStoreConfig:
+    """
+    Configuration of a state store using a local JSON file
+    """
+
     path: str
     save_interval: TimeIntervalConfig = TimeIntervalConfig("30s")
 
 
 @dataclass
 class StateStoreConfig:
+    """
+    Configuration of the State Store, containing ``LocalStateStoreConfig`` or ``RawStateStoreConfig``
+    """
+
     raw: Optional[RawStateStoreConfig] = None
     local: Optional[LocalStateStoreConfig] = None
 

cognite/extractorutils/configtools/loaders.py

@@ -47,6 +47,10 @@ class KeyVaultAuthenticationMethod(Enum):
 
 
 class KeyVaultLoader:
+    """
+    Class responsible for configuring keyvault for clients using Azure
+    """
+
     def __init__(self, config: Optional[dict]):
         self.config = config
 

cognite/extractorutils/exceptions.py

@@ -15,7 +15,7 @@
 
 class InvalidConfigError(Exception):
     """
-    Exception thrown from ``load_yaml`` if config file is invalid. This can be due to
+    Exception thrown from ``load_yaml`` and ``load_yaml_dict`` if config file is invalid. This can be due to
 
       * Missing fields
       * Incompatible types

cognite/extractorutils/threading.py

@@ -1,6 +1,7 @@
 import logging
 import signal
 from threading import Condition
+from time import time
 from typing import Any, Optional
 
 
@@ -59,11 +60,21 @@ class CancellationToken:
         self.cancel()
 
     def wait(self, timeout: Optional[float] = None) -> bool:
+        endtime = None
+        if timeout is not None:
+            endtime = time() + timeout
+
         while not self.is_cancelled:
             with self._cv:
-                did_not_time_out = self._cv.wait(timeout)
-                if not did_not_time_out:
-                    return False
+                if endtime is not None:
+                    remaining_time = endtime - time()
+                    if remaining_time <= 0.0:
+                        return True
+                    timed_out = not self._cv.wait(remaining_time)
+                    if timed_out:
+                        return False
+                else:
+                    self._cv.wait()
         return True
 
     def create_child_token(self) -> "CancellationToken":

cognite/extractorutils/uploader/files.py

@@ -19,6 +19,8 @@ from os import PathLike
 from types import TracebackType
 from typing import Any, BinaryIO, Callable, Dict, List, Optional, Tuple, Type, Union
 
+from requests.utils import super_len
+
 from cognite.client import CogniteClient
 from cognite.client.data_classes import FileMetadata
 from cognite.extractorutils.threading import CancellationToken
@@ -154,23 +156,34 @@ class IOFileUploadQueue(AbstractUploadQueue):
             try:
                 # Upload file
                 with read_file() as file:
-                    file_meta = self.cdf_client.files.upload_bytes(
-                        file,
-                        file_meta.name if file_meta.name is not None else "",
-                        overwrite=self.overwrite_existing,
-                        external_id=file_meta.external_id,
-                        source=file_meta.source,
-                        mime_type=file_meta.mime_type,
-                        metadata=file_meta.metadata,
-                        directory=file_meta.directory,
-                        asset_ids=file_meta.asset_ids,
-                        data_set_id=file_meta.data_set_id,
-                        labels=file_meta.labels,
-                        geo_location=file_meta.geo_location,
-                        source_created_time=file_meta.source_created_time,
-                        source_modified_time=file_meta.source_modified_time,
-                        security_categories=file_meta.security_categories,
-                    )
+                    size = super_len(file)
+                    if size == 0:
+                        # upload just the file metadata witout data
+                        file_meta, _url = self.cdf_client.files.create(
+                            file_metadata=file_meta, overwrite=self.overwrite_existing
+                        )
+                    elif size > pow(5, 9):
+                        # File bigger than 5Gb
+                        self.logger.warning(f"File {file_meta.source} has more than 5Gb")
+                        file_meta = FileMetadata()
+                    else:
+                        file_meta = self.cdf_client.files.upload_bytes(
+                            file,
+                            file_meta.name if file_meta.name is not None else "",
+                            overwrite=self.overwrite_existing,
+                            external_id=file_meta.external_id,
+                            source=file_meta.source,
+                            mime_type=file_meta.mime_type,
+                            metadata=file_meta.metadata,
+                            directory=file_meta.directory,
+                            asset_ids=file_meta.asset_ids,
+                            data_set_id=file_meta.data_set_id,
+                            labels=file_meta.labels,
+                            geo_location=file_meta.geo_location,
+                            source_created_time=file_meta.source_created_time,
+                            source_modified_time=file_meta.source_modified_time,
+                            security_categories=file_meta.security_categories,
+                        )
 
                 if self.post_upload_function:
                     try:

cognite/extractorutils/util.py

@@ -79,7 +79,7 @@ class EitherId:
 
     Args:
         id: Internal ID
-        externalId or external_id: external ID
+        external_id: external ID. It can be `external_id` or `externalId`
 
     Raises:
         TypeError: If none of both of id types are set.
@@ -175,24 +175,25 @@ def add_extraction_pipeline(
     added_message: str = "",
 ) -> Callable[[Callable[..., _T1]], Callable[..., _T1]]:
     """
-    This is to be used as a decorator for extractor functions to add extraction pipeline information
+    This is to be used as a decorator for extractor functions to add extraction pipeline information.
 
     Args:
-        extraction_pipeline_ext_id:
-        cognite_client:
-        heartbeat_waiting_time:
-        added_message:
+        extraction_pipeline_ext_id: External ID of the extraction pipeline
+        cognite_client: Client to use when communicating with CDF
+        heartbeat_waiting_time: Target interval between heartbeats, in seconds
 
     Usage:
         If you have a function named "extract_data(*args, **kwargs)" and want to connect it to an extraction
         pipeline, you can use this decorator function as:
-        @add_extraction_pipeline(
-            extraction_pipeline_ext_id=<INSERT EXTERNAL ID>,
-            cognite_client=<INSERT COGNITE CLIENT OBJECT>,
-            logger=<INSERT LOGGER>,
-        )
-        def extract_data(*args, **kwargs):
-            <INSERT FUNCTION BODY>
+
+        .. code-block:: python
+
+            @add_extraction_pipeline(
+                extraction_pipeline_ext_id=<INSERT EXTERNAL ID>,
+                cognite_client=<INSERT COGNITE CLIENT OBJECT>,
+            )
+            def extract_data(*args, **kwargs):
+                <INSERT FUNCTION BODY>
     """
 
     # TODO 1. Consider refactoring this decorator to share methods with the Extractor context manager in .base.py

{cognite_extractor_utils-7.0.0.dist-info → cognite_extractor_utils-7.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-extractor-utils
-Version: 7.0.0
+Version: 7.0.2
 Summary: Utilities for easier development of extractors for CDF
 Home-page: https://github.com/cognitedata/python-extractor-utils
 License: Apache-2.0

{cognite_extractor_utils-7.0.0.dist-info → cognite_extractor_utils-7.0.2.dist-info}/RECORD

@@ -1,27 +1,27 @@
-cognite/extractorutils/__init__.py,sha256=7RFWajf36G6iOaLnzGuuL3rDlIkt5L1GnsrgApS5-_8,739
+cognite/extractorutils/__init__.py,sha256=2lG10jlln2GBVFsgvhjHFvBLyPUQgeESNNfejOuuRuU,739
 cognite/extractorutils/_inner_util.py,sha256=gmz6aqS7jDNsg8z4RHgJjMFohDLOMiaU4gMWBhg3xcE,1558
 cognite/extractorutils/base.py,sha256=o5oFzGBaszv7-HWMVG2ATmZwZfzi8enyrYIKwUagyCQ,16148
-cognite/extractorutils/configtools/__init__.py,sha256=fj9kH8DdisNi9mI8cKm2sz50vnzeOkJQErIGB3mTYRo,2861
+cognite/extractorutils/configtools/__init__.py,sha256=L-daaqInIsmHcjb2forJeY0fW8tz1mlteOUo7IsWnrU,3059
 cognite/extractorutils/configtools/_util.py,sha256=SZycZm_py9v9WZbDiDQbgS6_PiLtu-TtwuuH7tG2YCI,4739
-cognite/extractorutils/configtools/elements.py,sha256=CM9ZVAUgy7DnbTGI_kgnNclioEkaUqv0Lt_a2zoS8Yc,20680
-cognite/extractorutils/configtools/loaders.py,sha256=Q062K-AFJLn9MjJaURar_Le79edGUZvvpgQdg9wm51I,15082
-cognite/extractorutils/exceptions.py,sha256=PERRmySUfJRM2Ta8cFvADTe-KUdXsoMLKdk4140AOHI,1061
+cognite/extractorutils/configtools/elements.py,sha256=G9-2J-tzao4z5PfOpWifJCMgP7X7ABHK257NtrWsweI,21382
+cognite/extractorutils/configtools/loaders.py,sha256=Y5UjJBXrYFcvjeGKvjWxc_7FW6PzKSIVVO3q8eNxPkA,15170
+cognite/extractorutils/exceptions.py,sha256=XiwyNPSN0YxFYaPw7tfA63B94PL48xDK3EfdGdhgQgc,1084
 cognite/extractorutils/metrics.py,sha256=IzYevOH19N_Nhi2XfhcuGgsGfmF7SexfcHcTQyZmH1o,15635
 cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/extractorutils/statestore.py,sha256=iJdVJQihjU2ljfS46FNEKdD5PSurTzaAH09iGbDYJro,18598
-cognite/extractorutils/threading.py,sha256=C77KgNZolI8E_xbv-G-BkJwLEALkkvZs5UoNx4hJiFg,3249
+cognite/extractorutils/threading.py,sha256=2Hke5cFvP-wA45Crvh58JahoKXB64P3tr7R4y_BhBqM,3605
 cognite/extractorutils/uploader/__init__.py,sha256=W22u6QHA4cR0j78LN5LTL5YGbfC-uTApagTyP5ab7uQ,3110
 cognite/extractorutils/uploader/_base.py,sha256=-aFfoMSBGd9YUUMHL3ZQpLIuNMA7TNklWCEjPA18ER8,5282
 cognite/extractorutils/uploader/_metrics.py,sha256=J2LJXb19L_SLSJ_voNIQHYLp0pjxUKevpH1q_xKX6Hk,3247
 cognite/extractorutils/uploader/assets.py,sha256=2E90N1kxsaA6Ah4h0_r_dTVhDYY_68ItRWrHYkkltJw,5628
 cognite/extractorutils/uploader/events.py,sha256=NZP2tMoU_rh_rb-EZiUBsOT5KdNABHN4c9Oddk0OsdE,5680
-cognite/extractorutils/uploader/files.py,sha256=2BKwdgj2rlJqJWU-13ktCINdktI6IoKkeU2nAWUKOus,13176
+cognite/extractorutils/uploader/files.py,sha256=A0HxmXLT6lK2QEZf-55Ma0FiisUl1a-VWxKF3SB07GM,13869
 cognite/extractorutils/uploader/raw.py,sha256=wFjF90PFTjmByOWx_Y4_YfDJ2w2jl0EQJ2Tjx2MP2PM,6738
 cognite/extractorutils/uploader/time_series.py,sha256=VptUq129MY0t8yw4rxeL0kOhz2dMibz4XdyvfhfYGj8,26840
 cognite/extractorutils/uploader_extractor.py,sha256=E-mpVvbPg_Tk90U4S9JybV0duptJ2SXE88HB6npE3zI,7732
 cognite/extractorutils/uploader_types.py,sha256=5MKT14DUnTFVD5Nx4Zvnfp2SfaICuTKLWAFAYaZishk,1045
-cognite/extractorutils/util.py,sha256=PFxVkDfpAEXTONYC1U-iO5DiaE3snn81qVUUT2O8xAM,17025
-cognite_extractor_utils-7.0.0.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-cognite_extractor_utils-7.0.0.dist-info/METADATA,sha256=lSHI8N78-cWlmraTSSiHovCTL2P-4olyjpWLlwTwTD0,5437
-cognite_extractor_utils-7.0.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-cognite_extractor_utils-7.0.0.dist-info/RECORD,,
+cognite/extractorutils/util.py,sha256=1Atq4-N2Pyi6GrxMa1PEfMCTqhPiLb2YoUXTzd60Roc,17179
+cognite_extractor_utils-7.0.2.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+cognite_extractor_utils-7.0.2.dist-info/METADATA,sha256=pE7JlOYLjX4ry0sfTGY-1uPgQ5c556g04Lqd8bXiCKg,5437
+cognite_extractor_utils-7.0.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_extractor_utils-7.0.2.dist-info/RECORD,,

{cognite_extractor_utils-7.0.0.dist-info → cognite_extractor_utils-7.0.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.8.1
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any