cognite-extractor-utils 7.7.0__tar.gz → 7.8.1__tar.gz

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-extractor-utils
-Version: 7.7.0
+Version: 7.8.1
 Summary: Utilities for easier development of extractors for CDF
 Project-URL: repository, https://github.com/cognitedata/python-extractor-utils
 Author-email: Mathias Lohne <mathias.lohne@cognite.com>
@@ -12,9 +12,9 @@ Requires-Python: >=3.10
 Requires-Dist: arrow>=1.0.0
 Requires-Dist: azure-identity>=1.14.0
 Requires-Dist: azure-keyvault-secrets>=4.7.0
-Requires-Dist: cognite-sdk>=7.59.0
+Requires-Dist: cognite-sdk>=7.75.2
 Requires-Dist: croniter>=6.0.0
-Requires-Dist: dacite<1.9.0,>=1.6.0
+Requires-Dist: dacite<1.10.0,>=1.9.2
 Requires-Dist: decorator>=5.1.1
 Requires-Dist: httpx<1,>=0.27.0
 Requires-Dist: jsonlines>=4.0.0
@@ -26,6 +26,7 @@ Requires-Dist: pydantic>=2.8.2
 Requires-Dist: pyhumps>=3.8.0
 Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: pyyaml<7,>=5.3.0
+Requires-Dist: simple-winservice>=0.1.0; sys_platform == 'win32'
 Requires-Dist: typing-extensions<5,>=3.7.4
 Provides-Extra: experimental
 Requires-Dist: cognite-sdk-experimental; extra == 'experimental'

cognite_extractor_utils-7.8.1/cognite/examples/unstable/extractors/simple_extractor/config/config.yaml

@@ -0,0 +1,3 @@
+log-handlers:
+  - type: console
+    level: INFO

cognite_extractor_utils-7.8.1/cognite/examples/unstable/extractors/simple_extractor/config/connection_config.yaml

@@ -0,0 +1,10 @@
+project: ${COGNITE_PROJECT}
+base_url: ${COGNITE_BASE_URL}
+integration:
+  external_id: ${COGNITE_INTEGRATION_ID}
+authentication:
+  type: "client-credentials"
+  client_id: ${COGNITE_CLIENT_ID}
+  client_secret: ${COGNITE_CLIENT_SECRET}
+  token_url: ${COGNITE_TOKEN_URL}
+  scopes: ${COGNITE_SCOPES}

cognite_extractor_utils-7.8.1/cognite/examples/unstable/extractors/simple_extractor/main.py

@@ -0,0 +1,81 @@
+"""
+An example extractor that logs messages at various levels.
+"""
+
+from cognite.extractorutils.unstable.configuration.models import ExtractorConfig, IntervalConfig, TimeIntervalConfig
+from cognite.extractorutils.unstable.core.base import Extractor, StartupTask, TaskContext
+from cognite.extractorutils.unstable.core.runtime import Runtime
+from cognite.extractorutils.unstable.core.tasks import ScheduledTask
+
+
+class SimpleConfig(ExtractorConfig):
+    """
+    Defines the configuration for the SimpleExtractor.
+    """
+
+    pass
+
+
+class SimpleExtractor(Extractor[SimpleConfig]):
+    """
+    An example extractor that logs messages at various levels.
+    """
+
+    NAME = "SimpleTestExtractor"
+    EXTERNAL_ID = "test-extractor"
+    DESCRIPTION = "An extractor for testing log levels"
+    VERSION = "1.0.0"
+    CONFIG_TYPE = SimpleConfig
+    SUPPORTS_DRY_RUN = True
+
+    def __init_tasks__(self) -> None:
+        """
+        Initializes and adds tasks to the extractor.
+        """
+        self.add_task(StartupTask(name="main_task", target=self.run_my_task))
+        self.add_task(
+            ScheduledTask(
+                name="scheduled_task",
+                target=self.scheduled_task,
+                schedule=IntervalConfig(type="interval", expression=TimeIntervalConfig("3s")),
+            )
+        )
+
+    # example task that logs messages at different levels
+    def run_my_task(self, ctx: TaskContext) -> None:
+        """
+        An example task that logs messages at different levels.
+
+        Args:
+            ctx: The context for the task execution, used for logging.
+        """
+        ctx.debug("This is a detailed debug message.")
+        ctx.info("This is an informational message.")
+        ctx.warning("This is a warning message.")
+        ctx.info("Test finished.")
+
+    def scheduled_task(self, ctx: TaskContext) -> None:
+        """
+        An example scheduled task that logs a message.
+
+        Args:
+            ctx: The context for the task execution, used for logging.
+        """
+        ctx.info("This is a scheduled task running.")
+        ctx.warning("This is a warning from the scheduled task.")
+        ctx.debug("Debugging the scheduled task execution.")
+        ctx.error("This is an error message from the scheduled task.")
+
+    # add more tasks as needed
+
+
+def main() -> None:
+    """
+    Main function to run the SimpleExtractor.
+    """
+    runtime = Runtime(SimpleExtractor)
+    runtime.run()
+
+
+if __name__ == "__main__":
+    main()

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/__init__.py

@@ -16,7 +16,7 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.7.0"
+__version__ = "7.8.1"
 from .base import Extractor
 
 __all__ = ["Extractor"]

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/_inner_util.py

@@ -37,14 +37,14 @@ def resolve_log_level_for_httpx(level: str) -> str:
 
 
 class _DecimalEncoder(json.JSONEncoder):
-    def default(self, obj: Any) -> dict[str, str]:
+    def default(self, obj: Any) -> dict[str, str]:  # noqa: ANN401
         if isinstance(obj, Decimal):
             return {"type": "decimal_encoded", "value": str(obj)}
         return super().default(obj)
 
 
 class _DecimalDecoder(json.JSONDecoder):
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
+    def __init__(self, *args: Any, **kwargs: Any) -> None:  # noqa: ANN401
         json.JSONDecoder.__init__(self, *args, object_hook=self.object_hook, **kwargs)
 
     def object_hook(self, obj_dict: dict[str, str]) -> dict[str, str] | Decimal:

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/base.py

@@ -112,7 +112,7 @@ class Extractor(Generic[CustomConfigClass]):
         reload_config_interval: int | None = 300,
         reload_config_action: ReloadConfigAction = ReloadConfigAction.DO_NOTHING,
         success_message: str = "Successful shutdown",
-    ):
+    ) -> None:
         self.name = name
         self.description = description
         self.run_handle = run_handle

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/configtools/elements.py

@@ -29,6 +29,7 @@ from urllib.parse import urljoin, urlparse
 
 import yaml
 from prometheus_client import REGISTRY, start_http_server
+from typing_extensions import Self
 
 from cognite.client import ClientConfig, CogniteClient
 from cognite.client.credentials import (
@@ -604,6 +605,7 @@ class LoggingConfig:
                 when="midnight",
                 utc=True,
                 backupCount=self.file.retention,
+                encoding="utf-8",
             )
             file_handler.setLevel(self.file.level)
             file_handler.setFormatter(fmt)
@@ -926,7 +928,7 @@ class CastableInt(int):
     file.
     """
 
-    def __new__(cls, value: Any) -> "CastableInt":
+    def __new__(cls, value: int | str | bytes) -> Self:
         """
         Returns value as is if it's int.
 
@@ -955,7 +957,7 @@ class PortNumber(CastableInt):
     not a valid port number raises a ValueError at instantiation.
     """
 
-    def __new__(cls, value: Any) -> "PortNumber":
+    def __new__(cls, value: int | str | bytes) -> Self:
         """
         Try to cast the value to an integer and validate it as a port number.
 

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/configtools/loaders.py

@@ -72,7 +72,7 @@ class KeyVaultLoader:
         config: A dictionary containing the configuration for the keyvault.
     """
 
-    def __init__(self, config: dict | None):
+    def __init__(self, config: dict | None) -> None:
         self.config = config
 
         self.client: SecretClient | None = None
@@ -374,7 +374,7 @@ class ConfigResolver(Generic[CustomConfigClass]):
     Automatically reloads the configuration file if it has changed
     """
 
-    def __init__(self, config_path: str, config_type: type[CustomConfigClass]):
+    def __init__(self, config_path: str, config_type: type[CustomConfigClass]) -> None:
         self.config_path = config_path
         self.config_type = config_type
 
@@ -384,8 +384,22 @@ class ConfigResolver(Generic[CustomConfigClass]):
         self._cognite_client: CogniteClient | None = None
 
     def _reload_file(self) -> None:
-        with open(self.config_path) as stream:
-            self._config_text = stream.read()
+        try:
+            with open(self.config_path, encoding="utf-8") as stream:
+                self._config_text = stream.read()
+        except UnicodeDecodeError:
+            _logger.warning(
+                f"Config file '{self.config_path}' is not valid UTF-8. Falling back to system default encoding."
+            )
+            try:
+                with open(self.config_path) as stream:
+                    self._config_text = stream.read()
+            except Exception as e:
+                _logger.error(
+                    f"Failed to read '{self.config_path}' with both UTF-8 and system default encoding. "
+                    f"The file may be corrupt or in an unsupported format. Final error: {e}"
+                )
+                raise RuntimeError("Unable to read configuration file.") from e
 
     @property
     def cognite_client(self) -> CogniteClient | None:

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/exceptions.py

@@ -25,7 +25,7 @@ class InvalidConfigError(Exception):
       * Unknown fields
     """
 
-    def __init__(self, message: str, details: list[str] | None = None):
+    def __init__(self, message: str, details: list[str] | None = None) -> None:
         super().__init__()
         self.message = message
         self.details = details

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/metrics.py

@@ -67,7 +67,7 @@ _metrics_singularities = {}
 T = TypeVar("T")
 
 
-def safe_get(cls: type[T], *args: Any, **kwargs: Any) -> T:
+def safe_get(cls: type[T], *args: Any, **kwargs: Any) -> T:  # noqa: ANN401
     """
     A factory for instances of metrics collections.
 
@@ -122,7 +122,7 @@ class BaseMetrics:
         process_scrape_interval: Interval (in seconds) between each fetch of data for the ``process_*`` gauges
     """
 
-    def __init__(self, extractor_name: str, extractor_version: str, process_scrape_interval: float = 15):
+    def __init__(self, extractor_name: str, extractor_version: str, process_scrape_interval: float = 15) -> None:
         extractor_name = extractor_name.strip().replace(" ", "_")
 
         self.startup = Gauge(f"{extractor_name}_start_time", "Timestamp (seconds) of when the extractor last started")
@@ -187,7 +187,7 @@ class AbstractMetricsPusher(ABC):
         push_interval: int | None = None,
         thread_name: str | None = None,
         cancellation_token: CancellationToken | None = None,
-    ):
+    ) -> None:
         self.push_interval = push_interval
         self.thread_name = thread_name
 
@@ -274,7 +274,7 @@ class PrometheusPusher(AbstractMetricsPusher):
         password: str | None = None,
         thread_name: str | None = None,
         cancellation_token: CancellationToken | None = None,
-    ):
+    ) -> None:
         super().__init__(push_interval, thread_name, cancellation_token)
 
         self.username = username
@@ -283,7 +283,9 @@ class PrometheusPusher(AbstractMetricsPusher):
 
         self.url = url
 
-    def _auth_handler(self, url: str, method: str, timeout: int, headers: list[tuple[str, str]], data: Any) -> Callable:
+    def _auth_handler(
+        self, url: str, method: str, timeout: int, headers: list[tuple[str, str]], data: bytes
+    ) -> Callable[[], None]:
         """
         Returns a authentication handler against the Prometheus Pushgateway to use in the pushadd_to_gateway method.
 
@@ -350,7 +352,7 @@ class CognitePusher(AbstractMetricsPusher):
         data_set: EitherId | None = None,
         thread_name: str | None = None,
         cancellation_token: CancellationToken | None = None,
-    ):
+    ) -> None:
         super().__init__(push_interval, thread_name, cancellation_token)
 
         self.cdf_client = cdf_client

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/statestore/watermark.py

@@ -1,3 +1,6 @@
+# ruff: noqa: ANN401
+# TODO: the state stores should be generic over the type of state, not just Any.
+
 #  Copyright 2020 Cognite AS
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -66,7 +69,7 @@ class AbstractStateStore(_BaseStateStore, ABC):
         trigger_log_level: str = "DEBUG",
         thread_name: str | None = None,
         cancellation_token: CancellationToken | None = None,
-    ):
+    ) -> None:
         super().__init__(
             save_interval=save_interval,
             trigger_log_level=trigger_log_level,
@@ -249,7 +252,7 @@ class RawStateStore(AbstractStateStore):
         trigger_log_level: str = "DEBUG",
         thread_name: str | None = None,
         cancellation_token: CancellationToken | None = None,
-    ):
+    ) -> None:
         super().__init__(save_interval, trigger_log_level, thread_name, cancellation_token)
 
         self._cdf_client = cdf_client
@@ -395,7 +398,7 @@ class LocalStateStore(AbstractStateStore):
         trigger_log_level: str = "DEBUG",
         thread_name: str | None = None,
         cancellation_token: CancellationToken | None = None,
-    ):
+    ) -> None:
         super().__init__(save_interval, trigger_log_level, thread_name, cancellation_token)
 
         self._file_path = file_path

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/threading.py

@@ -6,7 +6,7 @@ import logging
 import signal
 from threading import Condition
 from time import time
-from typing import Any
+from types import FrameType
 
 
 class CancellationToken:
@@ -114,7 +114,7 @@ class CancellationToken:
         This will set the cancellation token instead of throwing a KeyboardInterrupt exception.
         """
 
-        def sigint_handler(sig_num: int, frame: Any) -> None:
+        def sigint_handler(sig_num: int, frame: FrameType | None) -> None:
             logger = logging.getLogger(__name__)
             logger.warning("Interrupt signal received, stopping extractor gracefully")
             self.cancel()

{cognite_extractor_utils-7.7.0 → cognite_extractor_utils-7.8.1}/cognite/extractorutils/unstable/configuration/exceptions.py

@@ -12,7 +12,7 @@ class InvalidConfigError(Exception):
       * Unknown fields
     """
 
-    def __init__(self, message: str, details: list[str] | None = None):
+    def __init__(self, message: str, details: list[str] | None = None) -> None:
         super().__init__()
         self.message = message
         self.details = details
@@ -30,3 +30,30 @@ class InvalidConfigError(Exception):
         Underlying message prefixed with 'Invalid config:'.
         """
         return self.__str__()
+
+
+class InvalidArgumentError(Exception):
+    """
+    Exception thrown when an invalid argument is passed to the extractor.
+
+    This can be due to:
+      * Missing required arguments
+      * Invalid argument types
+      * Unsupported argument values
+    """
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+    def __str__(self) -> str:
+        """
+        Underlying message prefixed with 'Invalid argument:'.
+        """
+        return f"Invalid argument: {self.message}"
+
+    def __repr__(self) -> str:
+        """
+        Underlying message prefixed with 'Invalid argument:'.
+        """
+        return self.__str__()