cognite-extractor-utils 7.5.12__py3-none-any.whl → 7.5.14__py3-none-any.whl

cognite/extractorutils/__init__.py

@@ -16,7 +16,7 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.5.12"
+__version__ = "7.5.14"
 from .base import Extractor
 
 __all__ = ["Extractor"]

cognite/extractorutils/base.py

@@ -22,7 +22,7 @@ from threading import Thread
 from types import TracebackType
 from typing import Any, Generic, TypeVar
 
-from dotenv import find_dotenv, load_dotenv
+from dotenv import load_dotenv
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes import ExtractionPipeline, ExtractionPipelineRun
@@ -252,12 +252,11 @@ class Extractor(Generic[CustomConfigClass]):
 
         if str(os.getenv("COGNITE_FUNCTION_RUNTIME", False)).lower() != "true":
             # Environment Variables
-            env_file_path = find_dotenv(usecwd=True)
-            if env_file_path:
-                load_dotenv(dotenv_path=env_file_path, override=True)
-                dotenv_message = f"Successfully ingested environment variables from {env_file_path}"
+            env_file_found = load_dotenv(dotenv_path="./.env", override=True)
+            if env_file_found:
+                dotenv_message = "Successfully ingested environment variables from './.env'"
             else:
-                dotenv_message = "No .env file found"
+                dotenv_message = "No .env file found at {Path.cwd() / '.env'}"
         else:
             dotenv_message = "No .env file imported when using Cognite Functions"
 

cognite/extractorutils/configtools/elements.py

@@ -18,6 +18,7 @@ from dataclasses import dataclass, field
 from datetime import timedelta
 from enum import Enum
 from logging.handlers import TimedRotatingFileHandler
+from pathlib import Path
 from time import sleep
 from typing import Any
 from urllib.parse import urljoin, urlparse
@@ -512,6 +513,7 @@ class _PushGatewayConfig:
     push_interval: TimeIntervalConfig = TimeIntervalConfig("30s")
 
 
+@dataclass
 class _PromServerConfig:
     port: int = 9000
     host: str = "0.0.0.0"
@@ -649,7 +651,7 @@ class LocalStateStoreConfig:
     Configuration of a state store using a local JSON file
     """
 
-    path: str
+    path: Path
     save_interval: TimeIntervalConfig = TimeIntervalConfig("30s")
 
 
@@ -695,8 +697,11 @@ class StateStoreConfig:
             )
 
         if self.local:
+            if self.local.path.is_dir():
+                raise ValueError(f"{self.local.path} is a directory, and not a file")
+
             return LocalStateStore(
-                file_path=self.local.path,
+                file_path=str(self.local.path),
                 save_interval=self.local.save_interval.seconds,
                 cancellation_token=cancellation_token,
             )

cognite/extractorutils/configtools/loaders.py

@@ -69,7 +69,7 @@ class KeyVaultLoader:
         self.client: SecretClient | None = None
 
     def _init_client(self) -> None:
-        from dotenv import find_dotenv, load_dotenv
+        from dotenv import load_dotenv
 
         if not self.config:
             raise InvalidConfigError(
@@ -98,8 +98,10 @@ class KeyVaultLoader:
 
             _logger.info("Using Azure ClientSecret credentials to access KeyVault")
 
-            dotenv_path = find_dotenv(usecwd=True)
-            load_dotenv(dotenv_path=dotenv_path, override=True)
+            env_file_found = load_dotenv("./.env", override=True)
+
+            if not env_file_found:
+                _logger.info(f"Local environment file not found at {Path.cwd() / '.env'}")
 
             if all(param in self.config for param in auth_parameters):
                 tenant_id = os.path.expandvars(self.config.get("tenant-id", None))

cognite/extractorutils/unstable/configuration/loaders.py

@@ -7,6 +7,7 @@ from typing import TextIO, TypeVar
 from pydantic import ValidationError
 
 from cognite.client import CogniteClient
+from cognite.client.exceptions import CogniteAPIError
 from cognite.extractorutils.configtools.loaders import _load_yaml_dict_raw
 from cognite.extractorutils.exceptions import InvalidConfigError as OldInvalidConfigError
 from cognite.extractorutils.unstable.configuration.exceptions import InvalidConfigError
@@ -41,12 +42,17 @@ def load_from_cdf(
     params: dict[str, str | int] = {"integration": external_id}
     if revision:
         params["revision"] = revision
-    response = cognite_client.get(
-        f"/api/v1/projects/{cognite_client.config.project}/odin/config",
-        params=params,
-        headers={"cdf-version": "alpha"},
-    )
-    response.raise_for_status()
+    try:
+        response = cognite_client.get(
+            f"/api/v1/projects/{cognite_client.config.project}/odin/config",
+            params=params,
+            headers={"cdf-version": "alpha"},
+        )
+    except CogniteAPIError as e:
+        if e.code == 404:
+            raise InvalidConfigError("No configuration found for the given integration") from e
+        raise e
+
     data = response.json()
 
     try:

cognite/extractorutils/unstable/core/runtime.py

@@ -5,17 +5,29 @@ import time
 from argparse import ArgumentParser, Namespace
 from multiprocessing import Process, Queue
 from pathlib import Path
+from random import randint
 from typing import Any, Generic, TypeVar
+from uuid import uuid4
 
 from requests.exceptions import ConnectionError
 from typing_extensions import assert_never
 
-from cognite.client.exceptions import CogniteAPIError, CogniteAuthError, CogniteConnectionError
+from cognite.client import CogniteClient
+from cognite.client.exceptions import (
+    CogniteAPIError,
+    CogniteAuthError,
+    CogniteConnectionError,
+)
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.unstable.configuration.exceptions import InvalidConfigError
-from cognite.extractorutils.unstable.configuration.loaders import load_file, load_from_cdf
+from cognite.extractorutils.unstable.configuration.loaders import (
+    load_file,
+    load_from_cdf,
+)
 from cognite.extractorutils.unstable.configuration.models import ConnectionConfig
 from cognite.extractorutils.unstable.core._dto import Error
+from cognite.extractorutils.unstable.core.errors import ErrorLevel
+from cognite.extractorutils.util import now
 
 from ._messaging import RuntimeMessage
 from .base import ConfigRevision, ConfigType, Extractor, FullConfig
@@ -26,6 +38,8 @@ ExtractorType = TypeVar("ExtractorType", bound=Extractor)
 
 
 class Runtime(Generic[ExtractorType]):
+    RETRY_CONFIG_INTERVAL = 30
+
     def __init__(
         self,
         extractor: type[ExtractorType],
@@ -37,6 +51,8 @@ class Runtime(Generic[ExtractorType]):
         self.logger = logging.getLogger(f"{self._extractor_class.EXTERNAL_ID}.runtime")
         self._setup_logging()
 
+        self._cognite_client: CogniteClient
+
     def _create_argparser(self) -> ArgumentParser:
         argparser = ArgumentParser(
             prog=sys.argv[0],
@@ -121,7 +137,7 @@ class Runtime(Generic[ExtractorType]):
         self.logger.info(f"Started extractor with PID {process.pid}")
         return process
 
-    def _get_application_config(
+    def _try_get_application_config(
         self,
         args: Namespace,
         connection_config: ConnectionConfig,
@@ -143,39 +159,65 @@ class Runtime(Generic[ExtractorType]):
 
         else:
             self.logger.info("Loading application config from CDF")
-            client = connection_config.get_cognite_client(
-                f"{self._extractor_class.EXTERNAL_ID}-{self._extractor_class.VERSION}"
+
+            application_config, current_config_revision = load_from_cdf(
+                self._cognite_client,
+                connection_config.integration,
+                self._extractor_class.CONFIG_TYPE,
             )
 
-            errors: list[Error] = []
+        return application_config, current_config_revision
+
+    def _safe_get_application_config(
+        self,
+        args: Namespace,
+        connection_config: ConnectionConfig,
+    ) -> tuple[ConfigType, ConfigRevision] | None:
+        prev_error: str | None = None
 
+        while not self._cancellation_token.is_cancelled:
             try:
-                application_config, current_config_revision = load_from_cdf(
-                    client,
-                    connection_config.integration,
-                    self._extractor_class.CONFIG_TYPE,
+                return self._try_get_application_config(args, connection_config)
+
+            except Exception as e:
+                error_message = str(e)
+                if error_message == prev_error:
+                    # Same error as before, no need to log it again
+                    self._cancellation_token.wait(randint(1, self.RETRY_CONFIG_INTERVAL))
+                    continue
+                prev_error = error_message
+
+                ts = now()
+                error = Error(
+                    external_id=str(uuid4()),
+                    level=ErrorLevel.fatal.value,
+                    start_time=ts,
+                    end_time=ts,
+                    description=error_message,
+                    details=None,
+                    task=None,
                 )
 
-            finally:
-                if errors:
-                    client.post(
-                        f"/api/v1/projects/{client.config.project}/odin/checkin",
-                        json={
-                            "externalId": connection_config.integration,
-                            "errors": [e.model_dump() for e in errors],
-                        },
-                        headers={"cdf-version": "alpha"},
-                    )
+                self._cognite_client.post(
+                    f"/api/v1/projects/{self._cognite_client.config.project}/odin/checkin",
+                    json={
+                        "externalId": connection_config.integration,
+                        "errors": [error.model_dump()],
+                    },
+                    headers={"cdf-version": "alpha"},
+                )
 
-        return application_config, current_config_revision
+                self._cancellation_token.wait(randint(1, self.RETRY_CONFIG_INTERVAL))
+
+        return None
 
     def _verify_connection_config(self, connection_config: ConnectionConfig) -> bool:
-        client = connection_config.get_cognite_client(
+        self._cognite_client = connection_config.get_cognite_client(
             f"{self._extractor_class.EXTERNAL_ID}-{self._extractor_class.VERSION}"
         )
         try:
-            client.post(
-                f"/api/v1/projects/{client.config.project}/odin/checkin",
+            self._cognite_client.post(
+                f"/api/v1/projects/{self._cognite_client.config.project}/odin/checkin",
                 json={
                     "externalId": connection_config.integration,
                 },
@@ -234,16 +276,19 @@ class Runtime(Generic[ExtractorType]):
         if not args.skip_init_checks and not self._verify_connection_config(connection_config):
             sys.exit(1)
 
-        # This has to be Any. We don't know the type of the extractors' config at type checking since the sel doesn't
+        # This has to be Any. We don't know the type of the extractors' config at type checking since the self doesn't
         # exist yet, and I have not found a way to represent it in a generic way that isn't just an Any in disguise.
         application_config: Any
+        config: tuple[Any, ConfigRevision] | None
+
         while not self._cancellation_token.is_cancelled:
-            try:
-                application_config, current_config_revision = self._get_application_config(args, connection_config)
+            config = self._safe_get_application_config(args, connection_config)
+            if config is None:
+                if self._cancellation_token.is_cancelled:
+                    break
+                continue
 
-            except InvalidConfigError:
-                self.logger.critical("Could not get a valid application config file. Shutting down")
-                sys.exit(1)
+            application_config, current_config_revision = config
 
             # Start extractor in separate process, and wait for it to end
             process = self._spawn_extractor(

{cognite_extractor_utils-7.5.12.dist-info → cognite_extractor_utils-7.5.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-extractor-utils
-Version: 7.5.12
+Version: 7.5.14
 Summary: Utilities for easier development of extractors for CDF
 Project-URL: repository, https://github.com/cognitedata/python-extractor-utils
 Author-email: Mathias Lohne <mathias.lohne@cognite.com>

{cognite_extractor_utils-7.5.12.dist-info → cognite_extractor_utils-7.5.14.dist-info}/RECORD

@@ -1,6 +1,6 @@
-cognite/extractorutils/__init__.py,sha256=_uW4et3n6Sh-nX8s8o8Okl7SK5Iicg6ZZOKBCWdEnhw,765
+cognite/extractorutils/__init__.py,sha256=CoUfXp9Vt0lIxThUC3H7MjFYpg5gS32akEvDoPirSV8,765
 cognite/extractorutils/_inner_util.py,sha256=ZMZIBwFpSD39RzLZq_OJlrFtVt8NFOtU4ObzAG8vTB4,1779
-cognite/extractorutils/base.py,sha256=QeJJfufp13waxlUikptMNHeQLHpBY1TZ3lYsA_rO__E,16454
+cognite/extractorutils/base.py,sha256=eGz3sHHsLJWgPYTVw8xJ-7PJUK7GCK8K61CMpU5R3WU,16414
 cognite/extractorutils/exceptions.py,sha256=4qreRiTwZH9lyLIKR67TP02MUxN9oYhCd2vFTb5bRME,1125
 cognite/extractorutils/metrics.py,sha256=dBxtfJkTWcVbVpqM5VYmP4U3cO7iultz7k036Um4Tx0,15406
 cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -10,8 +10,8 @@ cognite/extractorutils/uploader_types.py,sha256=lePcXPrXVcQYtvnF9Uv12gHaeOIGfYKx
 cognite/extractorutils/util.py,sha256=ZAx688yXV85xmWuAASlZ3e9yY5pvyR8H0r4rhnPdgF4,21052
 cognite/extractorutils/configtools/__init__.py,sha256=LSa7gVws_mrLCMRTKGs1zfiG6-IM0HzyOF6x0EfllSo,3616
 cognite/extractorutils/configtools/_util.py,sha256=VMxXXmPvNPf3Jjwknqm7i-Bp-z_ORN0DFKjBHgBsWA0,4773
-cognite/extractorutils/configtools/elements.py,sha256=chgcv7WSOb1zbHl1XU5dlsCSB7SDTY4NeKREYNeF9kE,26590
-cognite/extractorutils/configtools/loaders.py,sha256=a3LMwcuPCvuegxGzOrS3-mcnErd9R-oa0JRFEKhE5fs,18309
+cognite/extractorutils/configtools/elements.py,sha256=n2Nbl-fZnuMmyduWXh0KfOYibUVZQnCwUeT6OQTnJ6U,26760
+cognite/extractorutils/configtools/loaders.py,sha256=Iqyn1i050j8vDa2Bb7yJgW2I-riXgdbdFpGX9TRLU84,18374
 cognite/extractorutils/configtools/validators.py,sha256=xug3GOMIO4NOdyyvXtYlpKyq9wuDtGf7-xqIefD5bIo,1016
 cognite/extractorutils/statestore/__init__.py,sha256=hV3r11FUXkH6-60Ct6zLSROMNVrEeiE3Shmkf28Q-co,359
 cognite/extractorutils/statestore/_base.py,sha256=mWdFk4EZl886V6uXRj4O2sv2_ANJ3Sigmgeql-XEsmc,2675
@@ -20,7 +20,7 @@ cognite/extractorutils/statestore/watermark.py,sha256=CDQW0QkdBSp_dYFVcOEQCqKNQZ
 cognite/extractorutils/unstable/__init__.py,sha256=L6nqJHjylpk67CE-PbXJyb_TBI4yjhEYEz9J9WShDfM,341
 cognite/extractorutils/unstable/configuration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/extractorutils/unstable/configuration/exceptions.py,sha256=2-0jUp9IFwzk2QTQzWLoGgW1KOApk9UmPmLwtwRUBaE,591
-cognite/extractorutils/unstable/configuration/loaders.py,sha256=guBSVz7nd3dfsNEfPEHUUmU5WoYdPQmBBaJk2FuKXVw,3640
+cognite/extractorutils/unstable/configuration/loaders.py,sha256=cb7dvEvIypSv5k81qSFhs95bHROTPhGx5quHOviIb7g,3863
 cognite/extractorutils/unstable/configuration/models.py,sha256=YYGXxSwct04J9A6NDlW0kpKHvlvtPxuLeMHgq5KwH-g,9567
 cognite/extractorutils/unstable/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/extractorutils/unstable/core/_dto.py,sha256=tvvy39cvf-QT28GWz5FpqxQ5vAVk0t69JoPPhpWlweY,1293
@@ -29,7 +29,7 @@ cognite/extractorutils/unstable/core/base.py,sha256=eOtz4nNR3bv-vgQ-ma5yaUGpNN4n
 cognite/extractorutils/unstable/core/errors.py,sha256=oTRB5Alt-rM90wHfQQM0idjATpv0BrKtoAIOvtu7P-k,2115
 cognite/extractorutils/unstable/core/logger.py,sha256=MLkOZ6ofFKBpm7UAVy7l_RoUEm9ipLdkkay4mxzJJH0,3765
 cognite/extractorutils/unstable/core/restart_policy.py,sha256=4FUohTXeC74mnq36Q16PQ2i9jPI-WuUZm8XwClFSnYk,631
-cognite/extractorutils/unstable/core/runtime.py,sha256=vz9gE05u4UMn8QkGhc6ooPLo7ZTkGZGBzWY9kl0L2gk,9779
+cognite/extractorutils/unstable/core/runtime.py,sha256=RdRRBplPyHxyM6xujAl1e3phjbUA3Fk18x-pPFQB6e0,11067
 cognite/extractorutils/unstable/core/tasks.py,sha256=XvakIwp_zCV7XaWUgYT2z3LCGoQTljY0xdVJWFfm-ng,3158
 cognite/extractorutils/unstable/scheduling/__init__.py,sha256=L90_rCZNHvti-PInne0r7W9edIkifctELjiaxEoQiSc,67
 cognite/extractorutils/unstable/scheduling/_scheduler.py,sha256=jxr5gICz0nrLFr7PfplEWi1nrO5uW1NS-F9CH9v3eHs,3721
@@ -44,7 +44,7 @@ cognite/extractorutils/uploader/files.py,sha256=ag3TUrImJdAUiOSXugagD03-9WGlFpIy
 cognite/extractorutils/uploader/raw.py,sha256=8duMk9uVJPK3O6C0nzwnvrl0UzBtJs_T_L1la0Vo89k,6713
 cognite/extractorutils/uploader/time_series.py,sha256=3HaKfWcSOTKFmB6VwpRwioOykkRa6qmruSSsZu8Njdk,26507
 cognite/extractorutils/uploader/upload_failure_handler.py,sha256=wbUSXUP26rbmK2NGdQMzcWQwNOh2sXYOnlj0A_sk1n8,2003
-cognite_extractor_utils-7.5.12.dist-info/METADATA,sha256=R1sEQCVzrdz5z2MvinBnCI2ycWs_4FtsGofEEg-KC8g,4889
-cognite_extractor_utils-7.5.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_extractor_utils-7.5.12.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-cognite_extractor_utils-7.5.12.dist-info/RECORD,,
+cognite_extractor_utils-7.5.14.dist-info/METADATA,sha256=fOEVjTfL3MwM-AYOc64C0pVx4yaay5mvyXLcqLiTEFI,4889
+cognite_extractor_utils-7.5.14.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_extractor_utils-7.5.14.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+cognite_extractor_utils-7.5.14.dist-info/RECORD,,