cognite-extractor-utils 5.0.1__py3-none-any.whl → 5.2.0__py3-none-any.whl

cognite/extractorutils/__init__.py

@@ -16,5 +16,5 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "5.0.1"
+__version__ = "5.2.0"
 from .base import Extractor

cognite/extractorutils/_inner_util.py

@@ -18,6 +18,7 @@ A module containing utilities meant for use inside the extractor-utils package
 
 import json
 from decimal import Decimal
+from typing import Any, Dict, Union
 
 
 def _resolve_log_level(level: str) -> int:
@@ -25,17 +26,17 @@ def _resolve_log_level(level: str) -> int:
 
 
 class _DecimalEncoder(json.JSONEncoder):
-    def default(self, obj):
+    def default(self, obj: Any) -> Dict[str, str]:
         if isinstance(obj, Decimal):
             return {"type": "decimal_encoded", "value": str(obj)}
         return super(_DecimalEncoder, self).default(obj)
 
 
 class _DecimalDecoder(json.JSONDecoder):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
 
-    def object_hook(self, obj_dict):
+    def object_hook(self, obj_dict: Dict[str, str]) -> Union[Dict[str, str], Decimal]:
         if obj_dict.get("type") == "decimal_encoded":
             return Decimal(obj_dict["value"])
         return obj_dict

cognite/extractorutils/base.py

@@ -21,10 +21,10 @@ from threading import Event, Thread
 from types import TracebackType
 from typing import Any, Callable, Dict, Generic, Optional, Type, TypeVar
 
-from cognite.client import CogniteClient
-from cognite.client.data_classes import ExtractionPipeline, ExtractionPipelineRun
 from dotenv import find_dotenv, load_dotenv
 
+from cognite.client import CogniteClient
+from cognite.client.data_classes import ExtractionPipeline, ExtractionPipelineRun
 from cognite.extractorutils.configtools import BaseConfig, ConfigResolver, StateStoreConfig
 from cognite.extractorutils.exceptions import InvalidConfigError
 from cognite.extractorutils.metrics import BaseMetrics
@@ -94,7 +94,7 @@ class Extractor(Generic[CustomConfigClass]):
         self.run_handle = run_handle
         self.config_class = config_class
         self.use_default_state_store = use_default_state_store
-        self.version = version
+        self.version = version or "unknown"
         self.cancellation_token = cancellation_token
         self.config_file_path = config_file_path
         self.continuous_extractor = continuous_extractor
@@ -117,7 +117,7 @@ class Extractor(Generic[CustomConfigClass]):
         if metrics:
             self.metrics = metrics
         else:
-            self.metrics = BaseMetrics(extractor_name=name, extractor_version=version)
+            self.metrics = BaseMetrics(extractor_name=name, extractor_version=self.version)
 
     def _initial_load_config(self, override_path: Optional[str] = None) -> None:
         """
@@ -133,9 +133,9 @@ class Extractor(Generic[CustomConfigClass]):
             self.config_resolver = ConfigResolver.from_cli(self.name, self.description, self.version, self.config_class)
 
         self.config = self.config_resolver.config
-        Extractor._config_singleton = self.config
+        Extractor._config_singleton = self.config  # type: ignore
 
-        def config_refresher():
+        def config_refresher() -> None:
             while not self.cancellation_token.is_set():
                 self.cancellation_token.wait(self.reload_config_interval)
                 if self.config_resolver.has_changed:
@@ -144,7 +144,7 @@ class Extractor(Generic[CustomConfigClass]):
         if self.reload_config_interval and self.reload_config_action != ReloadConfigAction.DO_NOTHING:
             Thread(target=config_refresher, name="ConfigReloader", daemon=True).start()
 
-    def reload_config_callback(self):
+    def reload_config_callback(self) -> None:
         self.logger.error("Method for reloading configs has not been overridden in subclass")
 
     def _reload_config(self) -> None:
@@ -154,7 +154,7 @@ class Extractor(Generic[CustomConfigClass]):
             self.logger.info("Loading in new config file")
             self.config_resolver.accept_new_config()
             self.config = self.config_resolver.config
-            Extractor._config_singleton = self.config
+            Extractor._config_singleton = self.config  # type: ignore
 
         elif self.reload_config_action == ReloadConfigAction.SHUTDOWN:
             self.logger.info("Shutting down, expecting to be restarted")
@@ -193,7 +193,7 @@ class Extractor(Generic[CustomConfigClass]):
 
         try:
             self.state_store.initialize()
-        except ValueError as e:
+        except ValueError:
             self.logger.exception("Could not load state store, using an empty state store as default")
 
         Extractor._statestore_singleton = self.state_store
@@ -212,18 +212,16 @@ class Extractor(Generic[CustomConfigClass]):
                 )
             )
 
-    def _report_error(self, exc_type: Type[BaseException], exc_val: BaseException, exc_tb: TracebackType) -> None:
+    def _report_error(self, exception: BaseException) -> None:
         """
         Called on an unsuccessful exit of the extractor
 
         Args:
-            exc_type: Type of exception that caused the extractor to fail
-            exc_val: Exception object that caused the extractor to fail
-            exc_tb: Stack trace of where the extractor failed
+            exception: Exception object that caused the extractor to fail
         """
-        self.logger.error("Unexpected error during extraction", exc_info=exc_val)
+        self.logger.error("Unexpected error during extraction", exc_info=exception)
         if self.extraction_pipeline:
-            message = f"{exc_type.__name__}: {str(exc_val)}"[:1000]
+            message = f"{type(exception).__name__}: {str(exception)}"[:1000]
 
             self.logger.info(f"Reporting new failed run: {message}")
             self.cognite_client.extraction_pipelines.runs.create(
@@ -254,8 +252,8 @@ class Extractor(Generic[CustomConfigClass]):
         try:
             self._initial_load_config(override_path=self.config_file_path)
         except InvalidConfigError as e:
-            print("Critical error: Could not read config file", file=sys.stderr)
-            print(str(e), file=sys.stderr)
+            print("Critical error: Could not read config file", file=sys.stderr)  # noqa: T201
+            print(str(e), file=sys.stderr)  # noqa: T201
             sys.exit(1)
 
         if not self.configured_logger:
@@ -276,11 +274,11 @@ class Extractor(Generic[CustomConfigClass]):
         self.extraction_pipeline = self.config.cognite.get_extraction_pipeline(self.cognite_client)
 
         try:
-            self.config.metrics.start_pushers(self.cognite_client)
+            self.config.metrics.start_pushers(self.cognite_client)  # type: ignore
         except AttributeError:
             pass
 
-        def heartbeat_loop():
+        def heartbeat_loop() -> None:
             while not self.cancellation_token.is_set():
                 self.cancellation_token.wait(self.heartbeat_waiting_time)
 
@@ -289,7 +287,7 @@ class Extractor(Generic[CustomConfigClass]):
                     try:
                         self.cognite_client.extraction_pipelines.runs.create(
                             ExtractionPipelineRun(
-                                extpipe_external_id=self.extraction_pipeline.external_id, status="seen"
+                                extpipe_external_id=self.extraction_pipeline.external_id, status="seen"  # type: ignore
                             )
                         )
                     except Exception:
@@ -336,12 +334,12 @@ class Extractor(Generic[CustomConfigClass]):
             self.state_store.synchronize()
 
         try:
-            self.config.metrics.stop_pushers()
+            self.config.metrics.stop_pushers()  # type: ignore
         except AttributeError:
             pass
 
         if exc_val:
-            self._report_error(exc_type, exc_val, exc_tb)
+            self._report_error(exc_val)
         else:
             self._report_success()
 
@@ -364,9 +362,9 @@ class Extractor(Generic[CustomConfigClass]):
 
     @classmethod
     def get_current_config(cls) -> CustomConfigClass:
-        if Extractor._config_singleton is None:
+        if Extractor._config_singleton is None:  # type: ignore
             raise ValueError("No config singleton created. Have a config file been loaded?")
-        return Extractor._config_singleton
+        return Extractor._config_singleton  # type: ignore
 
     @classmethod
     def get_current_statestore(cls) -> AbstractStateStore:

cognite/extractorutils/configtools/__init__.py

@@ -16,9 +16,9 @@
 Module containing tools for loading and verifying config files, and a YAML loader to automatically serialize these
 dataclasses from a config file.
 
-Configs are described as ``dataclass``\es, and use the ``BaseConfig`` class as a superclass to get a few things built-in:
-config version, Cognite project and logging. Use type hints to specify types, use the ``Optional`` type to specify that
-a config parameter is optional, and give the attribute a value to give it a default.
+Configs are described as ``dataclass``\es, and use the ``BaseConfig`` class as a superclass to get a few things
+built-in: config version, Cognite project and logging. Use type hints to specify types, use the ``Optional`` type to
+specify that a config parameter is optional, and give the attribute a value to give it a default.
 
 For example, a config class for an extractor may look like the following:
 

cognite/extractorutils/configtools/_util.py

@@ -14,7 +14,7 @@
 import base64
 import re
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives import serialization as serialization
@@ -37,11 +37,11 @@ def _to_snake_case(dictionary: Dict[str, Any], case_style: str) -> Dict[str, Any
         An updated dictionary with keys in the given convention.
     """
 
-    def fix_list(list_, key_translator):
+    def fix_list(list_: List[Any], key_translator: Callable[[str], str]) -> List[Any]:
         if list_ is None:
             return []
 
-        new_list = [None] * len(list_)
+        new_list: List[Any] = [None] * len(list_)
         for i, element in enumerate(list_):
             if isinstance(element, dict):
                 new_list[i] = fix_dict(element, key_translator)
@@ -51,11 +51,11 @@ def _to_snake_case(dictionary: Dict[str, Any], case_style: str) -> Dict[str, Any
                 new_list[i] = element
         return new_list
 
-    def fix_dict(dict_, key_translator):
+    def fix_dict(dict_: Dict[str, Any], key_translator: Callable[[str], str]) -> Dict[str, Any]:
         if dict_ is None:
             return {}
 
-        new_dict = {}
+        new_dict: Dict[str, Any] = {}
         for key in dict_:
             if isinstance(dict_[key], dict):
                 new_dict[key_translator(key)] = fix_dict(dict_[key], key_translator)
@@ -65,10 +65,10 @@ def _to_snake_case(dictionary: Dict[str, Any], case_style: str) -> Dict[str, Any
                 new_dict[key_translator(key)] = dict_[key]
         return new_dict
 
-    def translate_hyphen(key):
+    def translate_hyphen(key: str) -> str:
         return key.replace("-", "_")
 
-    def translate_camel(key):
+    def translate_camel(key: str) -> str:
         return re.sub(r"([A-Z]+)", r"_\1", key).strip("_").lower()
 
     if case_style == "snake" or case_style == "underscore":
@@ -81,7 +81,7 @@ def _to_snake_case(dictionary: Dict[str, Any], case_style: str) -> Dict[str, Any
         raise ValueError(f"Invalid case style: {case_style}")
 
 
-def _load_certificate_data(cert_path: str, password: Optional[str]) -> Tuple[str, str]:
+def _load_certificate_data(cert_path: str, password: Optional[str]) -> Union[Tuple[str, str], Tuple[bytes, bytes]]:
     path = Path(cert_path)
     cert_data = Path(path).read_bytes()
 
@@ -93,16 +93,22 @@ def _load_certificate_data(cert_path: str, password: Optional[str]) -> Tuple[str
             format=serialization.PrivateFormat.TraditionalOpenSSL,
             encryption_algorithm=serialization.NoEncryption(),
         )
-        return (base64.b16encode(cert.fingerprint(hashes.SHA1())), private_key_str)
+        return base64.b16encode(cert.fingerprint(hashes.SHA1())), private_key_str
     elif path.suffix == ".pfx":
-        (private_key, cert, _) = pkcs12.load_key_and_certificates(
+        (private_key_pfx, cert_pfx, _) = pkcs12.load_key_and_certificates(
             cert_data, password=password.encode() if password else None
         )
-        private_key_str = private_key.private_bytes(
+
+        if private_key_pfx is None:
+            raise InvalidConfigError(f"Can't load private key from {cert_path}")
+        if cert_pfx is None:
+            raise InvalidConfigError(f"Can't load certificate from {cert_path}")
+
+        private_key_str = private_key_pfx.private_bytes(
             encoding=serialization.Encoding.PEM,
             format=serialization.PrivateFormat.TraditionalOpenSSL,
             encryption_algorithm=serialization.NoEncryption(),
         )
-        return (base64.b16encode(cert.fingerprint(hashes.SHA1())), private_key_str)
+        return base64.b16encode(cert_pfx.fingerprint(hashes.SHA1())), private_key_str
     else:
         raise InvalidConfigError(f"Unknown certificate format '{path.suffix}'. Allowed formats are 'pem' and 'pfx'")

cognite/extractorutils/configtools/elements.py

@@ -20,15 +20,15 @@ from enum import Enum
 from logging.handlers import TimedRotatingFileHandler
 from threading import Event
 from time import sleep
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 from urllib.parse import urljoin
 
 import yaml
-from cognite.client import ClientConfig, CogniteClient
-from cognite.client.credentials import OAuthClientCertificate, OAuthClientCredentials
-from cognite.client.data_classes import Asset, DataSet, ExtractionPipeline
 from prometheus_client import REGISTRY, start_http_server
 
+from cognite.client import ClientConfig, CogniteClient
+from cognite.client.credentials import CredentialProvider, OAuthClientCertificate, OAuthClientCredentials
+from cognite.client.data_classes import Asset, DataSet, ExtractionPipeline
 from cognite.extractorutils.configtools._util import _load_certificate_data
 from cognite.extractorutils.exceptions import InvalidConfigError
 from cognite.extractorutils.metrics import AbstractMetricsPusher, CognitePusher, PrometheusPusher
@@ -63,6 +63,22 @@ class AuthenticatorConfig:
     certificate: Optional[CertificateConfig] = None
 
 
+@dataclass
+class ConnectionConfig:
+    """
+    Configuration parameters for the global_config python SDK settings
+    """
+
+    disable_gzip: bool = False
+    status_forcelist: List[int] = field(default_factory=lambda: [429, 502, 503, 504])
+    max_retries: int = 10
+    max_retries_connect: int = 3
+    max_retry_backoff: int = 30
+    max_connection_pool_size: int = 50
+    disable_ssl: bool = False
+    proxies: Dict[str, str] = field(default_factory=dict)
+
+
 @dataclass
 class EitherIdConfig:
     id: Optional[int]
@@ -74,7 +90,7 @@ class EitherIdConfig:
 
 
 class TimeIntervalConfig(yaml.YAMLObject):
-    def __init__(self, expression):
+    def __init__(self, expression: str) -> None:
         self._interval, self._expression = TimeIntervalConfig._parse_expression(expression)
 
     @classmethod
@@ -130,7 +146,7 @@ class TimeIntervalConfig(yaml.YAMLObject):
 
 
 class FileSizeConfig(yaml.YAMLObject):
-    def __init__(self, expression):
+    def __init__(self, expression: str) -> None:
         self._bytes, self._expression = FileSizeConfig._parse_expression(expression)
 
     @classmethod
@@ -220,16 +236,26 @@ class CogniteConfig:
     data_set_external_id: Optional[str]
     extraction_pipeline: Optional[EitherIdConfig]
     timeout: TimeIntervalConfig = TimeIntervalConfig("30s")
+    connection: ConnectionConfig = field(default_factory=ConnectionConfig)
     external_id_prefix: str = ""
     host: str = "https://api.cognitedata.com"
 
     def get_cognite_client(
-        self, client_name: str, token_custom_args: Optional[Dict[str, str]] = None, use_experimental_sdk=False
+        self, client_name: str, token_custom_args: Optional[Dict[str, str]] = None, use_experimental_sdk: bool = False
     ) -> CogniteClient:
         from cognite.client.config import global_config
 
         global_config.disable_pypi_version_check = True
-
+        global_config.disable_gzip = self.connection.disable_gzip
+        global_config.status_forcelist = set(self.connection.status_forcelist)
+        global_config.max_retries = self.connection.max_retries
+        global_config.max_retries_connect = self.connection.max_retries_connect
+        global_config.max_retry_backoff = self.connection.max_retry_backoff
+        global_config.max_connection_pool_size = self.connection.max_connection_pool_size
+        global_config.disable_ssl = self.connection.disable_ssl
+        global_config.proxies = self.connection.proxies
+
+        credential_provider: CredentialProvider
         if self.idp_authentication.certificate:
             if self.idp_authentication.certificate.authority_url:
                 authority_url = self.idp_authentication.certificate.authority_url
@@ -243,13 +269,13 @@ class CogniteConfig:
             credential_provider = OAuthClientCertificate(
                 authority_url=authority_url,
                 client_id=self.idp_authentication.client_id,
-                cert_thumbprint=thumprint,
-                certificate=key,
+                cert_thumbprint=str(thumprint),
+                certificate=str(key),
                 scopes=self.idp_authentication.scopes,
             )
 
         elif self.idp_authentication.secret:
-            kwargs = {}
+            kwargs: Dict[str, Any] = {}
             if self.idp_authentication.token_url:
                 kwargs["token_url"] = self.idp_authentication.token_url
             elif self.idp_authentication.tenant:
@@ -264,7 +290,7 @@ class CogniteConfig:
                 token_custom_args["resource"] = self.idp_authentication.resource
             if self.idp_authentication.audience:
                 token_custom_args["audience"] = self.idp_authentication.audience
-            credential_provider = OAuthClientCredentials(**kwargs, **token_custom_args)
+            credential_provider = OAuthClientCredentials(**kwargs, **token_custom_args)  # type: ignore
 
         else:
             raise InvalidConfigError("No client certificate or secret provided")
@@ -278,7 +304,7 @@ class CogniteConfig:
         )
 
         if use_experimental_sdk:
-            from cognite.experimental import CogniteClient as ExperimentalCogniteClient
+            from cognite.experimental import CogniteClient as ExperimentalCogniteClient  # type: ignore
 
             return ExperimentalCogniteClient(client_config)
 
@@ -299,7 +325,7 @@ class CogniteConfig:
             return None
 
         return cdf_client.data_sets.retrieve(
-            id=self.data_set.either_id.internal_id, external_id=self.data_set.either_id.external_id
+            id=self.data_set.either_id.internal_id, external_id=self.data_set.either_id.external_id  # type: ignore
         )
 
     def get_extraction_pipeline(self, cdf_client: CogniteClient) -> Optional[ExtractionPipeline]:
@@ -308,8 +334,8 @@ class CogniteConfig:
 
         either_id = self.extraction_pipeline.either_id
         extraction_pipeline = cdf_client.extraction_pipelines.retrieve(
-            id=either_id.internal_id,
-            external_id=either_id.external_id,
+            id=either_id.internal_id,  # type: ignore
+            external_id=either_id.external_id,  # type: ignore
         )
         if extraction_pipeline is None:
             raise ValueError(f"Extraction pipeline with {either_id.type()} {either_id.content()} not found")
@@ -341,7 +367,7 @@ class LoggingConfig:
     # Prometheus and/or Cognite
     metrics: Optional[bool] = False
 
-    def setup_logging(self, suppress_console=False) -> None:
+    def setup_logging(self, suppress_console: bool = False) -> None:
         """
         Sets up the default logger in the logging package to be configured as defined in this config object
 
@@ -430,6 +456,7 @@ class MetricsConfig:
 
         push_gateways = self.push_gateways or []
 
+        pusher: AbstractMetricsPusher
         for counter, push_gateway in enumerate(push_gateways):
             pusher = PrometheusPusher(
                 job_name=push_gateway.job_name,
@@ -449,7 +476,7 @@ class MetricsConfig:
         if self.cognite:
             asset = None
 
-            if self.cognite.asset_name is not None:
+            if self.cognite.asset_name is not None and self.cognite.asset_external_id:
                 asset = Asset(name=self.cognite.asset_name, external_id=self.cognite.asset_external_id)
 
             pusher = CognitePusher(

cognite/extractorutils/configtools/loaders.py

@@ -40,10 +40,10 @@ def _load_yaml(
     source: Union[TextIO, str],
     config_type: Type[CustomConfigClass],
     case_style: str = "hyphen",
-    expand_envvars=True,
+    expand_envvars: bool = True,
     dict_manipulator: Callable[[Dict[str, Any]], Dict[str, Any]] = lambda x: x,
 ) -> CustomConfigClass:
-    def env_constructor(_: yaml.SafeLoader, node):
+    def env_constructor(_: yaml.SafeLoader, node: yaml.Node) -> bool:
         bool_values = {
             "true": True,
             "false": False,
@@ -61,7 +61,7 @@ def _load_yaml(
 
     # Safe to use load instead of safe_load since both loader classes are based on SafeLoader
     try:
-        config_dict = yaml.load(source, Loader=loader)
+        config_dict = yaml.load(source, Loader=loader)  # noqa: S506
     except ScannerError as e:
         location = e.problem_mark or e.context_mark
         formatted_location = f" at line {location.line+1}, column {location.column+1}" if location is not None else ""
@@ -80,7 +80,10 @@ def _load_yaml(
         raise InvalidConfigError(f"Unknown config parameter{'s' if len(unknowns) > 1 else ''} {', '.join(unknowns)}")
 
     except (dacite.WrongTypeError, dacite.MissingValueError, dacite.UnionMatchError) as e:
-        path = e.field_path.replace("_", "-") if case_style == "hyphen" else e.field_path
+        if e.field_path:
+            path = e.field_path.replace("_", "-") if case_style == "hyphen" else e.field_path
+        else:
+            path = None
 
         def name(type_: Type) -> str:
             return type_.__name__ if hasattr(type_, "__name__") else str(type_)
@@ -106,7 +109,10 @@ def _load_yaml(
 
 
 def load_yaml(
-    source: Union[TextIO, str], config_type: Type[CustomConfigClass], case_style: str = "hyphen", expand_envvars=True
+    source: Union[TextIO, str],
+    config_type: Type[CustomConfigClass],
+    case_style: str = "hyphen",
+    expand_envvars: bool = True,
 ) -> CustomConfigClass:
     """
     Read a YAML file, and create a config object based on its contents.
@@ -135,7 +141,7 @@ class ConfigResolver(Generic[CustomConfigClass]):
         self._config: Optional[CustomConfigClass] = None
         self._next_config: Optional[CustomConfigClass] = None
 
-    def _reload_file(self):
+    def _reload_file(self) -> None:
         with open(self.config_path, "r") as stream:
             self._config_text = stream.read()
 
@@ -152,17 +158,17 @@ class ConfigResolver(Generic[CustomConfigClass]):
     def has_changed(self) -> bool:
         try:
             self._resolve_config()
-        except Exception as e:
+        except Exception:
             _logger.exception("Failed to reload configuration file")
             return False
-        return self._config._file_hash != self._next_config._file_hash
+        return self._config._file_hash != self._next_config._file_hash if self._config else True  # type: ignore
 
     @property
     def config(self) -> CustomConfigClass:
         if self._config is None:
             self._resolve_config()
             self.accept_new_config()
-        return self._config
+        return self._config  # type: ignore
 
     def accept_new_config(self) -> None:
         self._config = self._next_config
@@ -196,11 +202,13 @@ class ConfigResolver(Generic[CustomConfigClass]):
         if local_part.cognite.host is not None:
             remote_part["cognite"]["host"] = local_part.cognite.host
         remote_part["cognite"]["project"] = local_part.cognite.project
+
+        # Ignoring None type, extraction pipelines is required at this point
         remote_part["cognite"]["extraction-pipeline"] = {}
-        remote_part["cognite"]["extraction-pipeline"]["id"] = local_part.cognite.extraction_pipeline.id
+        remote_part["cognite"]["extraction-pipeline"]["id"] = local_part.cognite.extraction_pipeline.id  # type: ignore
         remote_part["cognite"]["extraction-pipeline"][
             "external_id"
-        ] = local_part.cognite.extraction_pipeline.external_id
+        ] = local_part.cognite.extraction_pipeline.external_id  # type: ignore
 
         return remote_part
 
@@ -209,10 +217,10 @@ class ConfigResolver(Generic[CustomConfigClass]):
 
         if self.is_remote:
             _logger.debug("Loading remote config file")
-            tmp_config: _BaseConfig = load_yaml(self._config_text, _BaseConfig)
+            tmp_config: _BaseConfig = load_yaml(self._config_text, _BaseConfig)  # type: ignore
             client = tmp_config.cognite.get_cognite_client("config_resolver")
             response = client.extraction_pipelines.config.retrieve(
-                tmp_config.cognite.get_extraction_pipeline(client).external_id
+                tmp_config.cognite.get_extraction_pipeline(client).external_id  # type: ignore  # ignoring extpipe None
             )
 
             self._next_config = _load_yaml(

cognite/extractorutils/metrics.py

@@ -43,23 +43,28 @@ import threading
 from abc import ABC, abstractmethod
 from threading import Event
 from time import sleep
-from typing import Any, Callable, Dict, List, Optional, T, Tuple, Type, Union
+from types import TracebackType
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
 
 import arrow
 import psutil
-from cognite.client import CogniteClient
-from cognite.client.data_classes import Asset, TimeSeries
-from cognite.client.exceptions import CogniteDuplicatedError
 from prometheus_client import Gauge, Info, Metric
 from prometheus_client.core import REGISTRY
 from prometheus_client.exposition import basic_auth_handler, delete_from_gateway, pushadd_to_gateway
 
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Asset, Datapoints, DatapointsArray, TimeSeries
+from cognite.client.exceptions import CogniteDuplicatedError
+
 from .util import ensure_time_series
 
 _metrics_singularities = {}
 
 
-def safe_get(cls: Type[T], *args, **kwargs) -> T:
+T = TypeVar("T")
+
+
+def safe_get(cls: Type[T], *args: Any, **kwargs: Any) -> T:
     """
     A factory for instances of metrics collections.
 
@@ -226,7 +231,9 @@ class AbstractMetricsPusher(ABC):
         self.start()
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+    def __exit__(
+        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+    ) -> None:
         """
         Wraps around stop method, for use as context manager
 
@@ -270,7 +277,7 @@ class PrometheusPusher(AbstractMetricsPusher):
 
         self.url = url
 
-    def _auth_handler(self, url: str, method: str, timeout: int, headers: Dict[str, str], data: Any) -> Callable:
+    def _auth_handler(self, url: str, method: str, timeout: int, headers: List[Tuple[str, str]], data: Any) -> Callable:
         """
         Returns a authentication handler against the Prometheus Pushgateway to use in the pushadd_to_gateway method.
 
@@ -298,7 +305,7 @@ class PrometheusPusher(AbstractMetricsPusher):
 
         except OSError as exp:
             self.logger.warning("Failed to push metrics to %s: %s", self.url, str(exp))
-        except:
+        except Exception:
             self.logger.exception("Failed to push metrics to %s", self.url)
 
         self.logger.debug("Pushed metrics to %s", self.url)
@@ -354,6 +361,7 @@ class CognitePusher(AbstractMetricsPusher):
 
         if self.asset is not None:
             # Ensure that asset exist, and retrieve internal ID
+            asset: Optional[Asset]
             try:
                 asset = self.cdf_client.assets.create(self.asset)
             except CogniteDuplicatedError:
@@ -374,7 +382,7 @@ class CognitePusher(AbstractMetricsPusher):
                         name=metric.name,
                         legacy_name=external_id,
                         description=metric.documentation,
-                        asset_id=asset_id,
+                        asset_id=asset_id,  # type: ignore  # this is optional. Type hint in SDK is wrong
                     )
                 )
 
@@ -386,7 +394,7 @@ class CognitePusher(AbstractMetricsPusher):
         """
         timestamp = int(arrow.get().float_timestamp * 1000)
 
-        datapoints: List[Dict[str, Union[str, List[Tuple[float, float]]]]] = []
+        datapoints: List[Dict[str, Union[str, int, List[Any], Datapoints, DatapointsArray]]] = []
 
         for metric in REGISTRY.collect():
             if type(metric) == Metric and metric.type in ["gauge", "counter"]: