cognite-extractor-utils 7.2.0__py3-none-any.whl → 7.2.2__py3-none-any.whl

cognite/extractorutils/__init__.py

@@ -16,5 +16,5 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.2.0"
+__version__ = "7.2.2"
 from .base import Extractor

cognite/extractorutils/configtools/elements.py

@@ -327,6 +327,8 @@ class CogniteConfig:
             elif self.idp_authentication.tenant:
                 base_url = urljoin(self.idp_authentication.authority, self.idp_authentication.tenant)
                 kwargs["token_url"] = f"{base_url}/oauth2/v2.0/token"
+            else:
+                raise InvalidConfigError("Either token-url or tenant is required for client credentials authentication")
             kwargs["client_id"] = self.idp_authentication.client_id
             kwargs["client_secret"] = self.idp_authentication.secret
             kwargs["scopes"] = self.idp_authentication.scopes

cognite/extractorutils/configtools/loaders.py

@@ -139,11 +139,10 @@ def _env_constructor(_: yaml.SafeLoader, node: yaml.Node) -> bool:
     return bool_values.get(expanded_value.lower(), expanded_value)
 
 
-def _load_yaml_dict(
+def _load_yaml_dict_raw(
     source: Union[TextIO, str],
-    case_style: str = "hyphen",
     expand_envvars: bool = True,
-    dict_manipulator: Callable[[Dict[str, Any]], Dict[str, Any]] = lambda x: x,
+    keyvault_loader: Optional[KeyVaultLoader] = None,
 ) -> Dict[str, Any]:
     loader = _EnvLoader if expand_envvars else yaml.SafeLoader
 
@@ -159,11 +158,14 @@ def _load_yaml_dict(
     if not isinstance(source, str):
         source.seek(0)
 
-    keyvault_config = initial_load.get("azure-keyvault", initial_load.get("key-vault"))
+    if keyvault_loader:
+        _EnvLoader.add_constructor("!keyvault", keyvault_loader)
+    else:
+        keyvault_config = initial_load.get("azure-keyvault", initial_load.get("key-vault"))
+        _EnvLoader.add_constructor("!keyvault", KeyVaultLoader(keyvault_config))
 
     _EnvLoader.add_implicit_resolver("!env", re.compile(r"\$\{([^}^{]+)\}"), None)
     _EnvLoader.add_constructor("!env", _env_constructor)
-    _EnvLoader.add_constructor("!keyvault", KeyVaultLoader(keyvault_config))
 
     try:
         config_dict = yaml.load(source, Loader=loader)  # noqa: S506
@@ -173,12 +175,26 @@ def _load_yaml_dict(
         cause = e.problem or e.context
         raise InvalidConfigError(f"Invalid YAML{formatted_location}: {cause or ''}") from e
 
-    if "azure-keyvault" in config_dict:
-        config_dict.pop("azure-keyvault")
+    return config_dict
+
+
+def _load_yaml_dict(
+    source: Union[TextIO, str],
+    case_style: str = "hyphen",
+    expand_envvars: bool = True,
+    dict_manipulator: Callable[[Dict[str, Any]], Dict[str, Any]] = lambda x: x,
+    keyvault_loader: Optional[KeyVaultLoader] = None,
+) -> Dict[str, Any]:
+    config_dict = _load_yaml_dict_raw(source, expand_envvars, keyvault_loader)
 
     config_dict = dict_manipulator(config_dict)
     config_dict = _to_snake_case(config_dict, case_style)
 
+    if "azure_keyvault" in config_dict:
+        config_dict.pop("azure_keyvault")
+    if "key_vault" in config_dict:
+        config_dict.pop("key_vault")
+
     return config_dict
 
 
@@ -188,9 +204,14 @@ def _load_yaml(
     case_style: str = "hyphen",
     expand_envvars: bool = True,
     dict_manipulator: Callable[[Dict[str, Any]], Dict[str, Any]] = lambda x: x,
+    keyvault_loader: Optional[KeyVaultLoader] = None,
 ) -> CustomConfigClass:
     config_dict = _load_yaml_dict(
-        source, case_style=case_style, expand_envvars=expand_envvars, dict_manipulator=dict_manipulator
+        source,
+        case_style=case_style,
+        expand_envvars=expand_envvars,
+        dict_manipulator=dict_manipulator,
+        keyvault_loader=keyvault_loader,
     )
 
     try:
@@ -239,6 +260,7 @@ def load_yaml(
     config_type: Type[CustomConfigClass],
     case_style: str = "hyphen",
     expand_envvars: bool = True,
+    keyvault_loader: Optional[KeyVaultLoader] = None,
 ) -> CustomConfigClass:
     """
     Read a YAML file, and create a config object based on its contents.
@@ -249,6 +271,7 @@ def load_yaml(
         case_style: Casing convention of config file. Valid options are 'snake', 'hyphen' or 'camel'. Should be
             'hyphen'.
         expand_envvars: Substitute values with the pattern ${VAR} with the content of the environment variable VAR
+        keyvault_loader: Pre-built loader for keyvault tags. Will be loaded from config if not set.
 
     Returns:
         An initialized config object.
@@ -256,13 +279,20 @@ def load_yaml(
     Raises:
         InvalidConfigError: If any config field is given as an invalid type, is missing or is unknown
     """
-    return _load_yaml(source=source, config_type=config_type, case_style=case_style, expand_envvars=expand_envvars)
+    return _load_yaml(
+        source=source,
+        config_type=config_type,
+        case_style=case_style,
+        expand_envvars=expand_envvars,
+        keyvault_loader=keyvault_loader,
+    )
 
 
 def load_yaml_dict(
     source: Union[TextIO, str],
     case_style: str = "hyphen",
     expand_envvars: bool = True,
+    keyvault_loader: Optional[KeyVaultLoader] = None,
 ) -> Dict[str, Any]:
     """
     Read a YAML file and return a dictionary from its contents
@@ -272,6 +302,7 @@ def load_yaml_dict(
         case_style: Casing convention of config file. Valid options are 'snake', 'hyphen' or 'camel'. Should be
             'hyphen'.
         expand_envvars: Substitute values with the pattern ${VAR} with the content of the environment variable VAR
+        keyvault_loader: Pre-built loader for keyvault tags. Will be loaded from config if not set.
 
     Returns:
         A raw dict with the contents of the config file.
@@ -279,7 +310,9 @@ def load_yaml_dict(
     Raises:
         InvalidConfigError: If any config field is given as an invalid type, is missing or is unknown
     """
-    return _load_yaml_dict(source=source, case_style=case_style, expand_envvars=expand_envvars)
+    return _load_yaml_dict(
+        source=source, case_style=case_style, expand_envvars=expand_envvars, keyvault_loader=keyvault_loader
+    )
 
 
 class ConfigResolver(Generic[CustomConfigClass]):
@@ -374,6 +407,10 @@ class ConfigResolver(Generic[CustomConfigClass]):
             and tmp_config.cognite.idp_authentication == self._config.cognite.idp_authentication
         )
 
+    def _get_keyvault_loader(self) -> KeyVaultLoader:
+        temp_config = _load_yaml_dict_raw(self._config_text)
+        return KeyVaultLoader(temp_config.get("azure-keyvault", temp_config.get("key-vault")))
+
     def _resolve_config(self) -> None:
         self._reload_file()
 
@@ -400,6 +437,7 @@ class ConfigResolver(Generic[CustomConfigClass]):
                 source=response.config,
                 config_type=self.config_type,
                 dict_manipulator=lambda d: self._inject_cognite(tmp_config, d),
+                keyvault_loader=self._get_keyvault_loader(),
             )
 
         else:

cognite/extractorutils/uploader/time_series.py

@@ -264,9 +264,9 @@ class TimeSeriesUploadQueue(AbstractUploadQueue):
                         missing = [id_dict for id_dict in ex.not_found if id_dict.get("externalId") not in retry_these]
                         missing_num = len(ex.not_found) - len(create_these_ids)
                         self.logger.error(
-                            f"{missing_num} time series not found, and could not be created automatically:\n"
+                            f"{missing_num} time series not found, and could not be created automatically: "
                             + str(missing)
-                            + "\nData will be dropped"
+                            + " Data will be dropped"
                         )
 
                 # Remove entries with non-existing time series from upload queue

{cognite_extractor_utils-7.2.0.dist-info → cognite_extractor_utils-7.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-extractor-utils
-Version: 7.2.0
+Version: 7.2.2
 Summary: Utilities for easier development of extractors for CDF
 Home-page: https://github.com/cognitedata/python-extractor-utils
 License: Apache-2.0
@@ -24,7 +24,7 @@ Requires-Dist: decorator (>=5.1.1,<6.0.0)
 Requires-Dist: more-itertools (>=10.0.0,<11.0.0)
 Requires-Dist: orjson (>=3.10.3,<4.0.0)
 Requires-Dist: prometheus-client (>0.7.0,<=1.0.0)
-Requires-Dist: psutil (>=5.7.0,<6.0.0)
+Requires-Dist: psutil (>=6.0.0,<7.0.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: pyyaml (>=5.3.0,<7)
 Requires-Dist: typing-extensions (>=3.7.4,<5)

{cognite_extractor_utils-7.2.0.dist-info → cognite_extractor_utils-7.2.2.dist-info}/RECORD

@@ -1,10 +1,10 @@
-cognite/extractorutils/__init__.py,sha256=FSo7g_qtWL_Dtb6sUYbzlhG16HbyQ1tNJWzPc8B4tEI,739
+cognite/extractorutils/__init__.py,sha256=G-4E2RTr6EDXP2zRRAR3D2oyZLbluCvTFmL3cvy4IoE,739
 cognite/extractorutils/_inner_util.py,sha256=gmz6aqS7jDNsg8z4RHgJjMFohDLOMiaU4gMWBhg3xcE,1558
 cognite/extractorutils/base.py,sha256=q6NU2bPec3WOasVnnIFoh-aUJudVZWZ2R6emz3IRj8Q,16391
 cognite/extractorutils/configtools/__init__.py,sha256=L-daaqInIsmHcjb2forJeY0fW8tz1mlteOUo7IsWnrU,3059
 cognite/extractorutils/configtools/_util.py,sha256=SZycZm_py9v9WZbDiDQbgS6_PiLtu-TtwuuH7tG2YCI,4739
-cognite/extractorutils/configtools/elements.py,sha256=FbkKg1YQOpCs4Bwjs4RacRswttM08TJHnkA9HNweYFs,21564
-cognite/extractorutils/configtools/loaders.py,sha256=VmKNfGqwdHycwZB91i-BHarjW-2Mw_Shv31R6uozm88,16317
+cognite/extractorutils/configtools/elements.py,sha256=k6q8NXZrV3efD_8BcTQvhb7lvTvsHbX30nshQgqeIQk,21703
+cognite/extractorutils/configtools/loaders.py,sha256=_bMEb2_WaBL_G9w5IXU5UKwSu1VGge0PPAWgfSribYI,17647
 cognite/extractorutils/exceptions.py,sha256=XiwyNPSN0YxFYaPw7tfA63B94PL48xDK3EfdGdhgQgc,1084
 cognite/extractorutils/metrics.py,sha256=01ZMRbDisXPxrfCSyTSEkXMsslzmZwEqw18fuu9okdc,15509
 cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -21,11 +21,11 @@ cognite/extractorutils/uploader/data_modeling.py,sha256=w35Ix5mu0Cgfn4ywnDyif4VV
 cognite/extractorutils/uploader/events.py,sha256=NZP2tMoU_rh_rb-EZiUBsOT5KdNABHN4c9Oddk0OsdE,5680
 cognite/extractorutils/uploader/files.py,sha256=31kPS4fwz8ZSXWss-CKmYTM6ZLVx9LtsDe7LHT7Wy98,18329
 cognite/extractorutils/uploader/raw.py,sha256=wFjF90PFTjmByOWx_Y4_YfDJ2w2jl0EQJ2Tjx2MP2PM,6738
-cognite/extractorutils/uploader/time_series.py,sha256=WAtEQy7k5IjG-sw1oWwCujIM6PjHZYl4LKa4wy2tBPw,26817
+cognite/extractorutils/uploader/time_series.py,sha256=HBtQdsQoIOaL-EG5lMsaY-ORwVb0kGiXG86VjE5-_Bg,26815
 cognite/extractorutils/uploader_extractor.py,sha256=E-mpVvbPg_Tk90U4S9JybV0duptJ2SXE88HB6npE3zI,7732
 cognite/extractorutils/uploader_types.py,sha256=wxfrsiKPTzG5lmoYtQsxt8Xyj-s5HnaLl8WDzJNrazg,1020
 cognite/extractorutils/util.py,sha256=UA6mUZ1caHd6vtA45gZXrk6cxo5cSB2PZ32bMwfEU0M,17229
-cognite_extractor_utils-7.2.0.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-cognite_extractor_utils-7.2.0.dist-info/METADATA,sha256=KoWttPiWPXifkWzMkDefy1Hz2pNNiZZrli7W7EoSAFM,5486
-cognite_extractor_utils-7.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_extractor_utils-7.2.0.dist-info/RECORD,,
+cognite_extractor_utils-7.2.2.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+cognite_extractor_utils-7.2.2.dist-info/METADATA,sha256=8cw9XRHtl8RAjaiaGQFwX4i0rBFgkRidxuitf9Cx00U,5486
+cognite_extractor_utils-7.2.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_extractor_utils-7.2.2.dist-info/RECORD,,