cognite-extractor-utils 7.4.3__py3-none-any.whl → 7.4.4__py3-none-any.whl

cognite/extractorutils/__init__.py

@@ -16,5 +16,5 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.4.3"
+__version__ = "7.4.4"
 from .base import Extractor

cognite/extractorutils/exceptions.py

@@ -13,6 +13,9 @@
 #  limitations under the License.
 
 
+from typing import List, Optional
+
+
 class InvalidConfigError(Exception):
     """
     Exception thrown from ``load_yaml`` and ``load_yaml_dict`` if config file is invalid. This can be due to
@@ -22,9 +25,10 @@ class InvalidConfigError(Exception):
       * Unkown fields
     """
 
-    def __init__(self, message: str):
+    def __init__(self, message: str, details: Optional[List[str]] = None):
         super(InvalidConfigError, self).__init__()
         self.message = message
+        self.details = details
 
     def __str__(self) -> str:
         return f"Invalid config: {self.message}"

cognite/extractorutils/unstable/configuration/loaders.py

@@ -0,0 +1,111 @@
+import json
+from enum import Enum
+from io import StringIO
+from pathlib import Path
+from typing import Dict, Optional, TextIO, Type, TypeVar, Union
+
+from pydantic import ValidationError
+
+from cognite.client import CogniteClient
+from cognite.extractorutils.configtools.loaders import _load_yaml_dict_raw
+from cognite.extractorutils.exceptions import InvalidConfigError
+from cognite.extractorutils.unstable.configuration.models import ConfigModel
+
+_T = TypeVar("_T", bound=ConfigModel)
+
+
+class ConfigFormat(Enum):
+    JSON = "json"
+    YAML = "yaml"
+
+
+def load_file(path: Path, schema: Type[_T]) -> _T:
+    if path.suffix in [".yaml", ".yml"]:
+        format = ConfigFormat.YAML
+    elif path.suffix == ".json":
+        format = ConfigFormat.JSON
+    else:
+        raise InvalidConfigError(f"Unknown file type {path.suffix}")
+
+    with open(path, "r") as stream:
+        return load_io(stream, format, schema)
+
+
+def load_from_cdf(
+    cognite_client: CogniteClient, external_id: str, schema: Type[_T], revision: Optional[int] = None
+) -> _T:
+    params: Dict[str, Union[str, int]] = {"externalId": external_id}
+    if revision:
+        params["revision"] = revision
+    response = cognite_client.get(
+        f"/api/v1/projects/{cognite_client.config.project}/odin/config",
+        params=params,
+        headers={"cdf-version": "alpha"},
+    )
+    response.raise_for_status()
+    data = response.json()
+    return load_io(StringIO(data["config"]), ConfigFormat.YAML, schema)
+
+
+def load_io(stream: TextIO, format: ConfigFormat, schema: Type[_T]) -> _T:
+    if format == ConfigFormat.JSON:
+        data = json.load(stream)
+
+    elif format == ConfigFormat.YAML:
+        data = _load_yaml_dict_raw(stream)
+
+        if "azure-keyvault" in data:
+            data.pop("azure-keyvault")
+        if "key-vault" in data:
+            data.pop("key-vault")
+
+    return load_dict(data, schema)
+
+
+def _make_loc_str(loc: tuple) -> str:
+    # Remove the body parameter if it is present
+    if loc[0] == "body":
+        loc = loc[1:]
+
+    # Create a string from the loc parameter
+    loc_str = ""
+    needs_sep = False
+    for lo in loc:
+        if not needs_sep:
+            loc_str = f"{loc_str}{lo}"
+            needs_sep = True
+        else:
+            if isinstance(lo, int):
+                loc_str = f"{loc_str}[{lo}]"
+            else:
+                loc_str = f"{loc_str}.{lo}"
+
+    return loc_str
+
+
+def load_dict(data: dict, schema: Type[_T]) -> _T:
+    try:
+        return schema.model_validate(data)
+
+    except ValidationError as e:
+        messages = []
+        for err in e.errors():
+            loc = err.get("loc")
+            if loc is None:
+                continue
+
+            # Create a string from the loc parameter
+            loc_str = _make_loc_str(loc)
+
+            if "ctx" in err and "error" in err["ctx"]:
+                exc = err["ctx"]["error"]
+                if isinstance(exc, ValueError) or isinstance(exc, AssertionError):
+                    messages.append(f"{loc_str}: {str(exc)}")
+                    continue
+
+            if err.get("type") == "json_invalid":
+                messages.append(f"{err.get('msg')}: {loc_str}")
+            else:
+                messages.append(f"{loc_str}: {err.get('msg')}")
+
+        raise InvalidConfigError(", ".join(messages), details=messages) from e

cognite/extractorutils/unstable/configuration/models.py

@@ -0,0 +1,159 @@
+import re
+from datetime import timedelta
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
+
+from humps import kebabize
+from pydantic import BaseModel, ConfigDict, Field, GetCoreSchemaHandler
+from pydantic_core import CoreSchema, core_schema
+
+from cognite.extractorutils.exceptions import InvalidConfigError
+
+
+class ConfigModel(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=kebabize,
+        populate_by_name=True,
+        extra="forbid",
+        # arbitrary_types_allowed=True,
+    )
+
+
+class _ClientCredentialsConfig(ConfigModel):
+    type: Literal["client-credentials"]
+    client_id: str
+    client_secret: str
+    token_url: str
+    scopes: List[str]
+    resource: Optional[str] = None
+    audience: Optional[str] = None
+
+
+class _ClientCertificateConfig(ConfigModel):
+    type: Literal["client-certificate"]
+    client_id: str
+    certificate_path: Path
+    scopes: List[str]
+
+
+AuthenticationConfig = Annotated[Union[_ClientCredentialsConfig, _ClientCertificateConfig], Field(discriminator="type")]
+
+
+class TimeIntervalConfig:
+    """
+    Configuration parameter for setting a time interval
+    """
+
+    def __init__(self, expression: str) -> None:
+        self._interval, self._expression = TimeIntervalConfig._parse_expression(expression)
+
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> CoreSchema:
+        return core_schema.no_info_after_validator_function(cls, handler(Union[str, int]))
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, TimeIntervalConfig):
+            return NotImplemented
+        return self._interval == other._interval
+
+    def __hash__(self) -> int:
+        return hash(self._interval)
+
+    @classmethod
+    def _parse_expression(cls, expression: str) -> tuple[int, str]:
+        # First, try to parse pure number and assume seconds (for backwards compatibility)
+        try:
+            return int(expression), f"{expression}s"
+        except ValueError:
+            pass
+
+        match = re.match(r"(\d+)[ \t]*(s|m|h|d)", expression)
+        if not match:
+            raise InvalidConfigError("Invalid interval pattern")
+
+        number, unit = match.groups()
+        numeric_unit = {"s": 1, "m": 60, "h": 60 * 60, "d": 60 * 60 * 24}[unit]
+
+        return int(number) * numeric_unit, expression
+
+    @property
+    def seconds(self) -> int:
+        return self._interval
+
+    @property
+    def minutes(self) -> float:
+        return self._interval / 60
+
+    @property
+    def hours(self) -> float:
+        return self._interval / (60 * 60)
+
+    @property
+    def days(self) -> float:
+        return self._interval / (60 * 60 * 24)
+
+    @property
+    def timedelta(self) -> timedelta:
+        days = self._interval // (60 * 60 * 24)
+        seconds = self._interval % (60 * 60 * 24)
+        return timedelta(days=days, seconds=seconds)
+
+    def __int__(self) -> int:
+        return int(self._interval)
+
+    def __float__(self) -> float:
+        return float(self._interval)
+
+    def __str__(self) -> str:
+        return self._expression
+
+    def __repr__(self) -> str:
+        return self._expression
+
+
+class _ConnectionParameters(ConfigModel):
+    gzip_compression: bool = False
+    status_forcelist: List[int] = Field(default_factory=lambda: [429, 502, 503, 504])
+    max_retries: int = 10
+    max_retries_connect: int = 3
+    max_retry_backoff: TimeIntervalConfig = Field(default_factory=lambda: TimeIntervalConfig("30s"))
+    max_connection_pool_size: int = 50
+    ssl_verify: bool = True
+    proxies: Dict[str, str] = Field(default_factory=dict)
+
+
+class ConnectionConfig(ConfigModel):
+    project: str
+    base_url: str
+
+    extraction_pipeline: str
+
+    authentication: AuthenticationConfig
+
+    connection: _ConnectionParameters = Field(default_factory=_ConnectionParameters)
+
+
+class LogLevel(Enum):
+    CRITICAL = "CRITICAL"
+    ERROR = "ERROR"
+    WARNING = "WARNING"
+    INFO = "INFO"
+    DEBUG = "DEBUG"
+
+
+class LogFileHandlerConfig(ConfigModel):
+    path: Path
+    level: LogLevel
+    retention: int = 7
+
+
+class LogConsoleHandlerConfig(ConfigModel):
+    level: LogLevel
+
+
+LogHandlerConfig = Union[LogFileHandlerConfig, LogConsoleHandlerConfig]
+
+
+class ExtractorConfig(ConfigModel):
+    log_handlers: List[LogHandlerConfig] = Field(default_factory=lambda: [LogConsoleHandlerConfig(level=LogLevel.INFO)])

{cognite_extractor_utils-7.4.3.dist-info → cognite_extractor_utils-7.4.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-extractor-utils
-Version: 7.4.3
+Version: 7.4.4
 Summary: Utilities for easier development of extractors for CDF
 Home-page: https://github.com/cognitedata/python-extractor-utils
 License: Apache-2.0
@@ -17,7 +17,7 @@ Provides-Extra: experimental
 Requires-Dist: arrow (>=1.0.0,<2.0.0)
 Requires-Dist: azure-identity (>=1.14.0,<2.0.0)
 Requires-Dist: azure-keyvault-secrets (>=4.7.0,<5.0.0)
-Requires-Dist: cognite-sdk (>=7.58.4,<8.0.0)
+Requires-Dist: cognite-sdk (>=7.59.0,<8.0.0)
 Requires-Dist: dacite (>=1.6.0,<2.0.0)
 Requires-Dist: decorator (>=5.1.1,<6.0.0)
 Requires-Dist: httpx (>=0.27.0,<0.28.0)
@@ -25,6 +25,8 @@ Requires-Dist: more-itertools (>=10.0.0,<11.0.0)
 Requires-Dist: orjson (>=3.10.3,<4.0.0)
 Requires-Dist: prometheus-client (>0.7.0,<=1.0.0)
 Requires-Dist: psutil (>=6.0.0,<7.0.0)
+Requires-Dist: pydantic (>=2.8.2,<3.0.0)
+Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: pyyaml (>=5.3.0,<7)
 Requires-Dist: typing-extensions (>=3.7.4,<5)

{cognite_extractor_utils-7.4.3.dist-info → cognite_extractor_utils-7.4.4.dist-info}/RECORD

@@ -1,11 +1,11 @@
-cognite/extractorutils/__init__.py,sha256=piRKiXLS1XurR0Hwyz9mKGaicqhmzSvu3q1yPTE87ro,739
+cognite/extractorutils/__init__.py,sha256=31r2uuSwG5TA05s9mUYwn8tfl1ShWccfkVVf7uVOcYs,739
 cognite/extractorutils/_inner_util.py,sha256=gmz6aqS7jDNsg8z4RHgJjMFohDLOMiaU4gMWBhg3xcE,1558
 cognite/extractorutils/base.py,sha256=q6NU2bPec3WOasVnnIFoh-aUJudVZWZ2R6emz3IRj8Q,16391
 cognite/extractorutils/configtools/__init__.py,sha256=L-daaqInIsmHcjb2forJeY0fW8tz1mlteOUo7IsWnrU,3059
 cognite/extractorutils/configtools/_util.py,sha256=SZycZm_py9v9WZbDiDQbgS6_PiLtu-TtwuuH7tG2YCI,4739
 cognite/extractorutils/configtools/elements.py,sha256=0q1DaQFeKpP8tQvgJ7sN4Tx8ryY0eR8kFolKvzoqSoM,23392
 cognite/extractorutils/configtools/loaders.py,sha256=_bMEb2_WaBL_G9w5IXU5UKwSu1VGge0PPAWgfSribYI,17647
-cognite/extractorutils/exceptions.py,sha256=XiwyNPSN0YxFYaPw7tfA63B94PL48xDK3EfdGdhgQgc,1084
+cognite/extractorutils/exceptions.py,sha256=1PgvW1FrgVnuNtkwC0RTvG1-FZp1qmBuYrY1AWW-BJc,1188
 cognite/extractorutils/metrics.py,sha256=01ZMRbDisXPxrfCSyTSEkXMsslzmZwEqw18fuu9okdc,15509
 cognite/extractorutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/extractorutils/statestore/__init__.py,sha256=hV3r11FUXkH6-60Ct6zLSROMNVrEeiE3Shmkf28Q-co,359
@@ -14,6 +14,9 @@ cognite/extractorutils/statestore/hashing.py,sha256=o-efTv21_ATQnyxYmple3MF7r5Af
 cognite/extractorutils/statestore/watermark.py,sha256=c_lcmJfo8bOvWyCJ9iRbbE4BlqRVulom4TpHb2pOnkE,16755
 cognite/extractorutils/threading.py,sha256=2Hke5cFvP-wA45Crvh58JahoKXB64P3tr7R4y_BhBqM,3605
 cognite/extractorutils/unstable/__init__.py,sha256=L6nqJHjylpk67CE-PbXJyb_TBI4yjhEYEz9J9WShDfM,341
+cognite/extractorutils/unstable/configuration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cognite/extractorutils/unstable/configuration/loaders.py,sha256=LBFgPjJrlz2ZHGKxuIbebOMLx1LU7DsGbyqYqd8Aq4w,3313
+cognite/extractorutils/unstable/configuration/models.py,sha256=HG_7MfCKtMdVpW6VmCqGug4Y8QrEek_ATpRBgVc4_5k,4429
 cognite/extractorutils/uploader/__init__.py,sha256=W22u6QHA4cR0j78LN5LTL5YGbfC-uTApagTyP5ab7uQ,3110
 cognite/extractorutils/uploader/_base.py,sha256=wktbV8dpb8zBOsNaECZkBNoJSpOz437NlNMER3-a3xQ,5304
 cognite/extractorutils/uploader/_metrics.py,sha256=J2LJXb19L_SLSJ_voNIQHYLp0pjxUKevpH1q_xKX6Hk,3247
@@ -26,7 +29,7 @@ cognite/extractorutils/uploader/time_series.py,sha256=HBtQdsQoIOaL-EG5lMsaY-ORwV
 cognite/extractorutils/uploader_extractor.py,sha256=E-mpVvbPg_Tk90U4S9JybV0duptJ2SXE88HB6npE3zI,7732
 cognite/extractorutils/uploader_types.py,sha256=wxfrsiKPTzG5lmoYtQsxt8Xyj-s5HnaLl8WDzJNrazg,1020
 cognite/extractorutils/util.py,sha256=T6ef5b7aYJ8yq9swQwybYaLe3YGr3hElsJQy8E-d5Rs,17469
-cognite_extractor_utils-7.4.3.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-cognite_extractor_utils-7.4.3.dist-info/METADATA,sha256=XjJEfPowEj6H8hx77yT9OljKGfREA8ilNPaWVTAAPlI,5476
-cognite_extractor_utils-7.4.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cognite_extractor_utils-7.4.3.dist-info/RECORD,,
+cognite_extractor_utils-7.4.4.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+cognite_extractor_utils-7.4.4.dist-info/METADATA,sha256=CtumhyB72ieH7_rSizELwMfG7zVFk087hAwfjnN7jVk,5557
+cognite_extractor_utils-7.4.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cognite_extractor_utils-7.4.4.dist-info/RECORD,,