cognite-extractor-utils 7.2.2__py3-none-any.whl → 7.2.3__py3-none-any.whl

cognite/extractorutils/__init__.py

@@ -16,5 +16,5 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.2.2"
+__version__ = "7.2.3"
 from .base import Extractor

cognite/extractorutils/configtools/elements.py

@@ -20,7 +20,7 @@ from enum import Enum
 from logging.handlers import TimedRotatingFileHandler
 from time import sleep
 from typing import Any, Dict, List, Optional, Tuple, Union
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
 
 import yaml
 from prometheus_client import REGISTRY, start_http_server
@@ -70,7 +70,7 @@ class AuthenticatorConfig:
 
     client_id: str
     scopes: List[str]
-    secret: Optional[str]
+    secret: Optional[str] = None
     tenant: Optional[str] = None
     token_url: Optional[str] = None
     resource: Optional[str] = None
@@ -264,6 +264,18 @@ class FileSizeConfig(yaml.YAMLObject):
         return self._expression
 
 
+path_elem_regex = re.compile(r"^([a-zA-Z0-9.\-_~!$&'()*+,;=:@]|%[A-F0-9]{2})*$")
+
+
+def _validate_https_url(value: str, name: str) -> None:
+    try:
+        url = urlparse(value)
+    except Exception as e:
+        raise InvalidConfigError(f"{name} ({value}) is not a valid URL") from e
+    if url.scheme != "https":
+        raise InvalidConfigError(f"{name} ({value}) must be HTTPS")
+
+
 @dataclass
 class CogniteConfig:
     """
@@ -272,10 +284,10 @@ class CogniteConfig:
 
     project: str
     idp_authentication: AuthenticatorConfig
-    data_set: Optional[EitherIdConfig]
-    data_set_id: Optional[int]
-    data_set_external_id: Optional[str]
-    extraction_pipeline: Optional[EitherIdConfig]
+    data_set: Optional[EitherIdConfig] = None
+    data_set_id: Optional[int] = None
+    data_set_external_id: Optional[str] = None
+    extraction_pipeline: Optional[EitherIdConfig] = None
     timeout: TimeIntervalConfig = TimeIntervalConfig("30s")
     connection: ConnectionConfig = field(default_factory=ConnectionConfig)
     security_categories: Optional[List[int]] = None
@@ -300,11 +312,20 @@ class CogniteConfig:
         global_config.disable_ssl = self.connection.disable_ssl
         global_config.proxies = self.connection.proxies
 
+        if not self.project:
+            raise InvalidConfigError("Project is not set")
+        if not path_elem_regex.match(self.project):
+            raise InvalidConfigError(f"Project ({self.project}) is not valid")
+
         credential_provider: CredentialProvider
         if self.idp_authentication.certificate:
             if self.idp_authentication.certificate.authority_url:
                 authority_url = self.idp_authentication.certificate.authority_url
+                _validate_https_url(self.idp_authentication.certificate.authority_url, "Authority URL")
             elif self.idp_authentication.tenant:
+                _validate_https_url(self.idp_authentication.authority, "Authority")
+                if not path_elem_regex.match(self.idp_authentication.tenant):
+                    raise InvalidConfigError(f"Tenant {self.idp_authentication.tenant} is not valid")
                 authority_url = urljoin(self.idp_authentication.authority, self.idp_authentication.tenant)
             else:
                 raise InvalidConfigError("Either authority-url or tenant is required for certificate authentication")
@@ -312,6 +333,8 @@ class CogniteConfig:
                 self.idp_authentication.certificate.path,
                 self.idp_authentication.certificate.password,
             )
+            if not self.idp_authentication.scopes:
+                _logger.warn("No scopes configured. Authenticating with CDF is unlikely to work correctly")
             credential_provider = OAuthClientCertificate(
                 authority_url=authority_url,
                 client_id=self.idp_authentication.client_id,
@@ -323,14 +346,20 @@ class CogniteConfig:
         elif self.idp_authentication.secret:
             kwargs: Dict[str, Any] = {}
             if self.idp_authentication.token_url:
+                _validate_https_url(self.idp_authentication.token_url, "Token URL")
                 kwargs["token_url"] = self.idp_authentication.token_url
             elif self.idp_authentication.tenant:
+                _validate_https_url(self.idp_authentication.authority, "Authority")
+                if not path_elem_regex.match(self.idp_authentication.tenant):
+                    raise InvalidConfigError(f"Tenant ({self.idp_authentication.tenant}) is not valid")
                 base_url = urljoin(self.idp_authentication.authority, self.idp_authentication.tenant)
                 kwargs["token_url"] = f"{base_url}/oauth2/v2.0/token"
             else:
                 raise InvalidConfigError("Either token-url or tenant is required for client credentials authentication")
             kwargs["client_id"] = self.idp_authentication.client_id
             kwargs["client_secret"] = self.idp_authentication.secret
+            if not self.idp_authentication.scopes:
+                _logger.warning("No scopes configured. Authenticating with CDF is unlikely to work correctly")
             kwargs["scopes"] = self.idp_authentication.scopes
             if token_custom_args is None:
                 token_custom_args = {}

{cognite_extractor_utils-7.2.2.dist-info → cognite_extractor_utils-7.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-extractor-utils
-Version: 7.2.2
+Version: 7.2.3
 Summary: Utilities for easier development of extractors for CDF
 Home-page: https://github.com/cognitedata/python-extractor-utils
 License: Apache-2.0

{cognite_extractor_utils-7.2.2.dist-info → cognite_extractor_utils-7.2.3.dist-info}/RECORD

@@ -1,9 +1,9 @@
-cognite/extractorutils/__init__.py,sha256=G-4E2RTr6EDXP2zRRAR3D2oyZLbluCvTFmL3cvy4IoE,739
+cognite/extractorutils/__init__.py,sha256=No8QzFZvAX7DY63jX70dTCPGuk7k0Rd0k99nU_sqeTo,739
 cognite/extractorutils/_inner_util.py,sha256=gmz6aqS7jDNsg8z4RHgJjMFohDLOMiaU4gMWBhg3xcE,1558
 cognite/extractorutils/base.py,sha256=q6NU2bPec3WOasVnnIFoh-aUJudVZWZ2R6emz3IRj8Q,16391
 cognite/extractorutils/configtools/__init__.py,sha256=L-daaqInIsmHcjb2forJeY0fW8tz1mlteOUo7IsWnrU,3059
 cognite/extractorutils/configtools/_util.py,sha256=SZycZm_py9v9WZbDiDQbgS6_PiLtu-TtwuuH7tG2YCI,4739
-cognite/extractorutils/configtools/elements.py,sha256=k6q8NXZrV3efD_8BcTQvhb7lvTvsHbX30nshQgqeIQk,21703
+cognite/extractorutils/configtools/elements.py,sha256=0q1DaQFeKpP8tQvgJ7sN4Tx8ryY0eR8kFolKvzoqSoM,23392
 cognite/extractorutils/configtools/loaders.py,sha256=_bMEb2_WaBL_G9w5IXU5UKwSu1VGge0PPAWgfSribYI,17647
 cognite/extractorutils/exceptions.py,sha256=XiwyNPSN0YxFYaPw7tfA63B94PL48xDK3EfdGdhgQgc,1084
 cognite/extractorutils/metrics.py,sha256=01ZMRbDisXPxrfCSyTSEkXMsslzmZwEqw18fuu9okdc,15509
@@ -25,7 +25,7 @@ cognite/extractorutils/uploader/time_series.py,sha256=HBtQdsQoIOaL-EG5lMsaY-ORwV
 cognite/extractorutils/uploader_extractor.py,sha256=E-mpVvbPg_Tk90U4S9JybV0duptJ2SXE88HB6npE3zI,7732
 cognite/extractorutils/uploader_types.py,sha256=wxfrsiKPTzG5lmoYtQsxt8Xyj-s5HnaLl8WDzJNrazg,1020
 cognite/extractorutils/util.py,sha256=UA6mUZ1caHd6vtA45gZXrk6cxo5cSB2PZ32bMwfEU0M,17229
-cognite_extractor_utils-7.2.2.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-cognite_extractor_utils-7.2.2.dist-info/METADATA,sha256=8cw9XRHtl8RAjaiaGQFwX4i0rBFgkRidxuitf9Cx00U,5486
-cognite_extractor_utils-7.2.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_extractor_utils-7.2.2.dist-info/RECORD,,
+cognite_extractor_utils-7.2.3.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+cognite_extractor_utils-7.2.3.dist-info/METADATA,sha256=u-YGjhv1yLXR0t8OYCx07D7vp1_8Vg_yFwMke0HThZ8,5486
+cognite_extractor_utils-7.2.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_extractor_utils-7.2.3.dist-info/RECORD,,