cognite-neat 0.123.2__py3-none-any.whl → 0.127.30__py3-none-any.whl

cognite/neat/_utils/http_client/_config.py

@@ -0,0 +1,19 @@
+import functools
+import platform
+
+from cognite.neat._utils.auxiliary import get_current_neat_version
+
+
+@functools.lru_cache(maxsize=1)
+def get_user_agent() -> str:
+    neat_version = f"CogniteNeat/{get_current_neat_version()}"
+    python_version = (
+        f"{platform.python_implementation()}/{platform.python_version()} "
+        f"({platform.python_build()};{platform.python_compiler()})"
+    )
+    os_version_info = [platform.release(), platform.machine(), platform.architecture()[0]]
+    os_version_info = [s for s in os_version_info if s]  # Ignore empty strings
+    os_version_info_str = "-".join(os_version_info)
+    operating_system = f"{platform.system()}/{os_version_info_str}"
+
+    return f"{neat_version} {python_version} {operating_system}"

cognite/neat/_utils/http_client/_data_classes.py

@@ -0,0 +1,294 @@
+import sys
+from abc import ABC, abstractmethod
+from collections import UserList
+from collections.abc import MutableSequence, Sequence
+from typing import Generic, Literal, TypeAlias, TypeVar
+
+import httpx
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_serializer
+
+from cognite.neat._exceptions import CDFAPIException
+from cognite.neat._utils.http_client._tracker import ItemsRequestTracker
+from cognite.neat._utils.useful_types import PrimaryTypes, ReferenceObject, T_Reference
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+StatusCode: TypeAlias = int
+
+
+class HTTPMessage(BaseModel):
+    """Base class for HTTP messages (requests and responses)"""
+
+
+class FailedRequestMessage(HTTPMessage):
+    message: str
+
+    def __str__(self) -> str:
+        return self.message
+
+
+class ResponseMessage(HTTPMessage):
+    code: StatusCode
+    body: str
+
+
+class SuccessResponse(ResponseMessage): ...
+
+
+class ErrorDetails(BaseModel):
+    """This is the structure of failure responses from CDF APIs"""
+
+    code: StatusCode
+    message: str
+    missing: list[JsonValue] | None = None
+    duplicated: list[JsonValue] | None = None
+    is_auto_retryable: bool | None = Field(None, alias="isAutoRetryable")
+
+    @classmethod
+    def from_response(cls, response: httpx.Response) -> "ErrorDetails":
+        try:
+            return _ErrorResponse.model_validate_json(response.text).error
+        except ValidationError:
+            return cls(code=response.status_code, message=response.text)
+
+
+class _ErrorResponse(BaseModel):
+    error: ErrorDetails
+
+
+class FailedResponse(ResponseMessage):
+    error: ErrorDetails
+
+    def __str__(self) -> str:
+        return f"HTTP {self.code} | {self.error.message}"
+
+
+class RequestMessage(HTTPMessage, ABC):
+    """Base class for HTTP request messages"""
+
+    endpoint_url: str
+    method: Literal["GET", "POST", "PATCH", "DELETE"]
+    connect_attempt: int = 0
+    read_attempt: int = 0
+    status_attempt: int = 0
+    api_version: str | None = None
+
+    @property
+    def total_attempts(self) -> int:
+        return self.connect_attempt + self.read_attempt + self.status_attempt
+
+    @abstractmethod
+    def create_success_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def create_failure_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def create_failed_request(self, error_message: str) -> Sequence[HTTPMessage]:
+        raise NotImplementedError()
+
+
+class SimpleRequest(RequestMessage):
+    """Base class for requests with a simple success/fail response structure"""
+
+    def create_success_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
+        return [SuccessResponse(code=response.status_code, body=response.text)]
+
+    def create_failure_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
+        return [
+            FailedResponse(code=response.status_code, body=response.text, error=ErrorDetails.from_response(response))
+        ]
+
+    def create_failed_request(self, error_message: str) -> Sequence[HTTPMessage]:
+        return [FailedRequestMessage(message=error_message)]
+
+
+class ParametersRequest(SimpleRequest):
+    """Base class for HTTP request messages with query parameters"""
+
+    parameters: dict[str, PrimaryTypes] | None = None
+
+
+class BodyRequest(ParametersRequest, ABC):
+    """Base class for HTTP request messages with a body"""
+
+    @abstractmethod
+    def data(self) -> str:
+        raise NotImplementedError()
+
+
+class SimpleBodyRequest(BodyRequest):
+    body: str
+
+    def data(self) -> str:
+        return self.body
+
+
+class ItemMessage(BaseModel, Generic[T_Reference], ABC):
+    """Base class for message related to a specific item"""
+
+    ids: Sequence[T_Reference]
+
+
+class SuccessResponseItems(ItemMessage[T_Reference], SuccessResponse): ...
+
+
+class FailedResponseItems(ItemMessage[T_Reference], FailedResponse): ...
+
+
+class FailedRequestItems(ItemMessage[T_Reference], FailedRequestMessage): ...
+
+
+T_BaseModel = TypeVar("T_BaseModel", bound=BaseModel)
+
+
+class ItemBody(BaseModel, Generic[T_Reference, T_BaseModel], ABC):
+    items: Sequence[T_BaseModel]
+    extra_args: dict[str, JsonValue] | None = None
+
+    @model_serializer(mode="plain", return_type=dict)
+    def serialize(self) -> dict[str, JsonValue]:
+        data: dict[str, JsonValue] = {
+            "items": [item.model_dump(exclude_unset=False, by_alias=True, exclude_none=False) for item in self.items]
+        }
+        if isinstance(self.extra_args, dict):
+            data.update(self.extra_args)
+        return data
+
+    @abstractmethod
+    def as_ids(self) -> list[T_Reference]:
+        """Returns the list of item identifiers for the items in the body."""
+        raise NotImplementedError()
+
+    def split(self, mid: int) -> tuple[Self, Self]:
+        """Splits the body into two smaller bodies.
+
+        This is useful for retrying requests that fail due to size limits or timeouts.
+
+        Args:
+            mid: The index at which to split the items.
+        Returns:
+            A tuple containing two new ItemBody instances, each with half of the original items.
+
+        """
+
+        type_ = type(self)
+        return type_(items=self.items[:mid], extra_args=self.extra_args), type_(
+            items=self.items[mid:], extra_args=self.extra_args
+        )
+
+
+class ItemIDBody(ItemBody[ReferenceObject, ReferenceObject]):
+    def as_ids(self) -> list[ReferenceObject]:
+        return list(self.items)
+
+
+class ItemsRequest(BodyRequest, Generic[T_Reference, T_BaseModel]):
+    """Requests message for endpoints that accept multiple items in a single request.
+
+    This class provides functionality to split large requests into smaller ones, handle responses for each item,
+    and manage errors effectively.
+
+    Attributes:
+        body (ItemBody): The body of the request containing the items to be processed.
+        max_failures_before_abort (int): The maximum number of failed split requests before aborting further splits.
+
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    body: ItemBody[T_Reference, T_BaseModel]
+    max_failures_before_abort: int = 50
+    tracker: ItemsRequestTracker | None = None
+
+    def data(self) -> str:
+        return self.body.model_dump_json(exclude_unset=True, by_alias=True)
+
+    def split(self, status_attempts: int) -> "list[ItemsRequest]":
+        """Splits the request into two smaller requests.
+
+        This is useful for retrying requests that fail due to size limits or timeouts.
+
+        Args:
+            status_attempts: The number of status attempts to set for the new requests. This is used when the
+                request failed with a 5xx status code and we want to track the number of attempts. For 4xx errors,
+                there is at least one item causing the error, so we do not increment the status attempts, but
+                instead essentially do a binary search to find the problematic item(s).
+
+        Returns:
+            A list containing two new ItemsRequest instances, each with half of the original items.
+
+        """
+        mid = len(self.body.items) // 2
+        if mid == 0:
+            return [self]
+        tracker = self.tracker or ItemsRequestTracker(self.max_failures_before_abort)
+        tracker.register_failure()
+        messages: list[ItemsRequest] = []
+        for body in self.body.split(mid):
+            item_request = ItemsRequest(
+                endpoint_url=self.endpoint_url,
+                method=self.method,
+                body=body,
+                connect_attempt=self.connect_attempt,
+                read_attempt=self.read_attempt,
+                status_attempt=status_attempts,
+            )
+            item_request.tracker = tracker
+            messages.append(item_request)
+        return messages
+
+    def create_success_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
+        return [SuccessResponseItems(code=response.status_code, body=response.text, ids=self.body.as_ids())]
+
+    def create_failure_response(self, response: httpx.Response) -> Sequence[HTTPMessage]:
+        """Creates response messages based on the HTTP response and the original request.
+
+        Args:
+            response: The HTTP response received from the server.
+        Returns:
+            A sequence of HTTPMessage instances representing the outcome for each item in the request.
+        """
+        return [
+            FailedResponseItems(
+                code=response.status_code,
+                body=response.text,
+                error=ErrorDetails.from_response(response),
+                ids=self.body.as_ids(),
+            )
+        ]
+
+    def create_failed_request(self, error_message: str) -> Sequence[HTTPMessage]:
+        """Creates failed request messages for each item in the request.
+
+        Args:
+            error_message: The error message to include in the failed request messages.
+
+        Returns:
+            A sequence of HTTPMessage instances representing the failed request for each item.
+        """
+        return [FailedRequestItems(message=error_message, ids=self.body.as_ids())]
+
+
+class APIResponse(UserList, MutableSequence[ResponseMessage | FailedRequestMessage]):
+    def __init__(self, collection: Sequence[ResponseMessage | FailedRequestMessage] | None = None):
+        super().__init__(collection or [])
+
+    def raise_for_status(self) -> None:
+        error_messages = [message for message in self.data if not isinstance(message, SuccessResponse)]
+        if error_messages:
+            raise CDFAPIException(error_messages)
+
+    @property
+    def success_response(self) -> SuccessResponse:
+        success = [msg for msg in self.data if isinstance(msg, SuccessResponse)]
+        if len(success) == 1:
+            return success[0]
+        elif success:
+            raise ValueError("Multiple successful HTTP responses found in the messages.")
+        else:
+            raise ValueError("No successful HTTP response found in the messages.")

cognite/neat/_utils/http_client/_tracker.py

@@ -0,0 +1,31 @@
+import threading
+from dataclasses import dataclass, field
+
+
+@dataclass
+class ItemsRequestTracker:
+    """Tracks the state of requests split from an original request.
+
+    Attributes:
+        max_failures_before_abort (int): Maximum number of allowed failed split requests before aborting
+            the entire operation. A value of -1 indicates no early abort.
+        lock (threading.Lock): A lock to ensure thread-safe updates to the failure count.
+        failed_split_count (int): The current count of failed split requests.
+
+    """
+
+    max_failures_before_abort: int = -1  # -1 means no early abort
+    lock: threading.Lock = field(default_factory=threading.Lock, init=False)
+    failed_split_count: int = field(default=0, init=False)
+
+    def register_failure(self) -> None:
+        """Register a failed split request and return whether to continue splitting."""
+        with self.lock:
+            self.failed_split_count += 1
+
+    def limit_reached(self) -> bool:
+        """Check if the failure limit has been reached."""
+        with self.lock:
+            if self.max_failures_before_abort < 0:
+                return False
+            return self.failed_split_count >= self.max_failures_before_abort

cognite/neat/_utils/text.py

@@ -0,0 +1,71 @@
+import re
+from collections.abc import Collection
+from typing import Any
+
+NEWLINE = "\n"
+TAB = "\t"
+
+
+def humanize_collection(collection: Collection[Any], /, *, sort: bool = True, bind_word: str = "and") -> str:
+    """Convert a collection of items to a human-readable string.
+
+    Args:
+        collection: The collection of items to convert.
+        sort: Whether to sort the collection before converting. Default is True.
+        bind_word: The word to use to bind the last two items. Default is "and".
+
+    Returns:
+        A human-readable string of the collection.
+
+    Examples:
+        >>> humanize_collection(["b", "c", "a"])
+        'a, b and c'
+        >>> humanize_collection(["b", "c", "a"], sort=False)
+        'b, c and a'
+        >>> humanize_collection(["a", "b"])
+        'a and b'
+        >>> humanize_collection(["a"])
+        'a'
+        >>> humanize_collection([])
+        ''
+
+    """
+    if not collection:
+        return ""
+    elif len(collection) == 1:
+        return str(next(iter(collection)))
+
+    strings = (str(item) for item in collection)
+    if sort:
+        sequence = sorted(strings)
+    else:
+        sequence = list(strings)
+
+    return f"{', '.join(sequence[:-1])} {bind_word} {sequence[-1]}"
+
+
+def title_case(s: str) -> str:
+    """Convert a string to title case, handling underscores and hyphens.
+
+    Args:
+        s: The string to convert.
+    Returns:
+        The title-cased string.
+    Examples:
+        >>> title_case("hello world")
+        'Hello World'
+        >>> title_case("hello_world")
+        'Hello World'
+        >>> title_case("hello-world")
+        'Hello World'
+        >>> title_case("hello_world-and-universe")
+        'Hello World And Universe'
+        >>> title_case("HELLO WORLD")
+        'Hello World'
+    """
+    return " ".join(word.capitalize() for word in s.replace("_", " ").replace("-", " ").split())
+
+
+def split_on_capitals(text: str) -> list[str]:
+    """Split a string at capital letters."""
+    return re.findall(r"[A-Z][a-z]*", text)

cognite/neat/_utils/useful_types.py

@@ -0,0 +1,37 @@
+from collections.abc import Hashable
+from datetime import date, datetime, time, timedelta
+from typing import Literal, TypeAlias, TypeVar
+
+from pydantic import BaseModel
+from pydantic.alias_generators import to_camel
+
+JsonVal: TypeAlias = None | str | int | float | bool | dict[str, "JsonVal"] | list["JsonVal"]
+PrimaryTypes: TypeAlias = str | int | float | bool
+
+T_ID = TypeVar("T_ID", bound=Hashable)
+# These are the types that openpyxl supports in cells
+CellValueType: TypeAlias = str | int | float | bool | datetime | date | time | timedelta | None
+
+# The format expected for excel sheets representing a data model
+DataModelTableType: TypeAlias = dict[str, list[dict[str, CellValueType]]]
+PrimitiveType: TypeAlias = str | int | float | bool
+
+
+class BaseModelObject(BaseModel, alias_generator=to_camel, extra="ignore"):
+    """Base class for all object. This includes resources and nested objects."""
+
+    ...
+
+
+T_Item = TypeVar("T_Item", bound=BaseModelObject)
+
+
+class ReferenceObject(BaseModelObject, frozen=True, populate_by_name=True):
+    """Base class for all reference objects - these are identifiers."""
+
+    ...
+
+
+T_Reference = TypeVar("T_Reference", bound=ReferenceObject, covariant=True)
+
+ModusOperandi: TypeAlias = Literal["rebuild", "additive"]

cognite/neat/_utils/validation.py

@@ -0,0 +1,154 @@
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Literal
+
+from pydantic_core import ErrorDetails
+
+
+def as_json_path(loc: tuple[str | int, ...]) -> str:
+    """Converts a location tuple to a JSON path.
+
+    Args:
+        loc: The location tuple to convert.
+
+    Returns:
+        A JSON path string.
+    """
+    if not loc:
+        return ""
+    # +1 to convert from 0-based to 1-based indexing
+    prefix = ""
+    if isinstance(loc[0], int):
+        prefix = "item"
+
+    suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[")
+    return f"{prefix}{suffix}"
+
+
+@dataclass
+class ValidationContext:
+    """
+    Context for validation errors providing configuration for error message formatting.
+
+    This class configures how validation errors are reported, including location formatting,
+    field naming conventions, and how to present missing required fields.
+
+    Attributes:
+        parent_loc: Optional location tuple to prepend to each error location.
+            This is useful when the error is for a nested model and you want to include the location
+            of the parent model.
+        humanize_location: A function that converts a location tuple to a human-readable string.
+            The default is `as_json_path`, which converts the location to a JSON path.
+            This can for example be replaced when the location comes from an Excel table.
+        field_name: The name use for "field" in error messages. Default is "field". This can be changed to
+            "column" or "value" to better fit the context.
+        field_renaming: Optional mapping of field names to source names.
+            This is useful when the field names in the model are different from the names in the source.
+            For example, if the model field is "asset_id" but the source column is "Asset ID",
+            you can provide a mapping {"asset_id": "Asset ID"} to have the error messages use the source names.
+        missing_required_descriptor: How to describe missing required fields. Default is "missing".
+            Other option is "empty" which can be more suitable for table data.
+    """
+
+    parent_loc: tuple[int | str, ...] = field(default_factory=tuple)
+    humanize_location: Callable[[tuple[int | str, ...]], str] = as_json_path
+    field_name: Literal["field", "column", "value"] = "field"
+    field_renaming: Mapping[str, str] = field(default_factory=dict)
+    missing_required_descriptor: Literal["empty", "missing"] = "missing"
+
+
+def humanize_validation_error(
+    error: ErrorDetails,
+    context: ValidationContext | None = None,
+) -> str:
+    """Converts a pydantic ErrorDetails object to a human-readable format.
+    This overwrites the default error messages from Pydantic to be better suited for NEAT users.
+    Args:
+        error: The ErrorDetails object to convert.
+        context: The context for humanizing the error.
+    Returns:
+        A human-readable error message.
+    """
+
+    context = context or ValidationContext()
+
+    loc = (*context.parent_loc, *error["loc"])
+    type_ = error["type"]
+
+    if type_ == "missing":
+        msg = f"Missing required {context.field_name}: {loc[-1]!r}"
+    elif type_ == "extra_forbidden":
+        msg = f"Unused {context.field_name}: {loc[-1]!r}"
+    elif type_ == "value_error":
+        msg = str(error["ctx"]["error"])
+    elif type_ == "literal_error":
+        msg = f"{error['msg']}. Got {error['input']!r}."
+    elif type_ == "string_type":
+        msg = f"{error['msg']}. Got {error['input']!r} of type {type(error['input']).__name__}. "
+    elif type_ == "model_type":
+        model_name = error["ctx"].get("class_name", "unknown")
+        msg = (
+            f"Input must be an object of type {model_name}. Got {error['input']!r} of "
+            f"type {type(error['input']).__name__}."
+        )
+    elif type_ == "union_tag_invalid":
+        msg = error["msg"].replace(", 'direct'", "").replace("found using 'type' ", "").replace("tag", "value")
+    elif type_ == "string_pattern_mismatch":
+        msg = f"string '{error['input']}' does not match the required pattern: '{error['ctx']['pattern']}'."
+
+    elif type_.endswith("_type"):
+        msg = f"{error['msg']}. Got {error['input']!r} of type {type(error['input']).__name__}."
+    else:
+        # Default to the Pydantic error message
+        msg = error["msg"]
+
+    if type_.endswith("dict_type") and len(loc) > 1:
+        # If this is a dict_type error for a JSON field, the location will be:
+        #  dict[str,json-or-python[json=any,python=tagged-union[list[...],dict[str,...],str,bool,int,float,none]]]
+        #  This is hard to read, so we simplify it to just the field name.
+        loc = tuple(["dict" if isinstance(x, str) and "json-or-python" in x else x for x in loc])
+
+    error_suffix = f"{msg[:1].casefold()}{msg[1:]}"
+
+    if len(loc) >= 3 and context.field_name == "column" and loc[-3:] == ("type", "enum", "values"):
+        # Special handling for enum errors in table columns
+        msg = _enum_message(type_, loc, context)
+    elif len(loc) > 1 and type_ in {"extra_forbidden", "missing"}:
+        if context.missing_required_descriptor == "empty" and type_ == "missing":
+            # This is a table so we modify the error message.
+            msg = (
+                f"In {context.humanize_location(loc[:-1])} the {context.field_name}"
+                f" {context.field_renaming.get(str(loc[-1]), loc[-1])!r} "
+                "cannot be empty."
+            )
+        else:
+            # We skip the last element as this is in the message already
+            msg = f"In {context.humanize_location(loc[:-1])} {error_suffix.replace('field', context.field_name)}"
+    elif len(loc) > 1:
+        if context.parent_loc == ("Metadata",) and len(loc) == 2:
+            msg = f"In table '{loc[0]}' '{loc[1]}' {error_suffix}"
+        else:
+            msg = f"In {context.humanize_location(loc)} {error_suffix}"
+    elif len(loc) == 1 and isinstance(loc[0], str) and type_ not in {"extra_forbidden", "missing"}:
+        msg = f"In {context.field_name} {loc[0]!r}, {error_suffix}"
+
+    msg = msg.strip()
+    if not msg.endswith("."):
+        msg += "."
+    return msg
+
+
+def _enum_message(type_: str, loc: tuple[int | str, ...], context: ValidationContext) -> str:
+    """Special handling of enum errors in table columns."""
+
+    if loc[-1] != "values":
+        raise RuntimeError("This is a neat bug, report to the team.")
+    if type_ == "missing":
+        return (
+            f"In {context.humanize_location(loc[:-1])} definition should include "
+            "a reference to a collection in the 'Enum' sheet (e.g., collection='MyEnumCollection')."
+        )
+    elif type_ == "too_short":
+        return f"In {context.humanize_location(loc[:-1])} collection is not defined in the 'Enum' sheet"
+    else:
+        raise RuntimeError("This is a neat bug, report to the team.")

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.123.2"
+__version__ = "0.127.30"
 __engine__ = "^2.0.4"