cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl

cognite/neat/issues/errors/_properties.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+from typing import Generic
+
+from cognite.neat.issues._base import ResourceType
+
+from ._resources import ResourceError, T_Identifier, T_ReferenceIdentifier
+
+
+@dataclass(frozen=True)
+class PropertyError(ResourceError[T_Identifier]):
+    """Base class for property errors {resource_type} with identifier {identifier}.{property_name}"""
+
+    property_name: str
+
+
+@dataclass(frozen=True)
+class PropertyNotFoundError(PropertyError, Generic[T_Identifier, T_ReferenceIdentifier]):
+    """The {resource_type} with identifier {identifier} does not have a property {property_name}"""
+
+    extra = "referred to by {referred_type} {referred_by} does not exist"
+    fix = "Ensure the {resource_type} {identifier} has a {property_name} property"
+
+    referred_by: T_ReferenceIdentifier | None = None
+    referred_type: ResourceType | None = None
+
+
+@dataclass(frozen=True)
+class PropertyTypeNotSupportedError(PropertyError[T_Identifier]):
+    """The {resource_type} with identifier {identifier} has a property {property_name}
+    of unsupported type {property_type}"""
+
+    property_type: str
+
+
+# This is a generic error that should be used sparingly
+@dataclass(frozen=True)
+class PropertyDefinitionError(PropertyError[T_Identifier]):
+    """Invalid property definition for {resource_type} {identifier}.{property_name}: {reason}"""
+
+    reason: str
+
+
+@dataclass(frozen=True)
+class PropertyDefinitionDuplicatedError(PropertyError[T_Identifier]):
+    """The {resource_type} with identifier {identifier} has multiple definitions for the property {property_name}
+    with values {property_values}
+    """
+
+    extra = "in locations {locations} with name {location_name}"
+
+    property_values: frozenset[str | int | float | bool | None | tuple[str | int | float | bool | None, ...]]
+    locations: tuple[str | int, ...] | None = None
+    location_name: str | None = None
+
+
+@dataclass(frozen=True)
+class PropertyMappingDuplicatedError(PropertyError[T_Identifier], Generic[T_Identifier, T_ReferenceIdentifier]):
+    """The {resource_type} with identifier {identifier}.{property_name} is mapped to by: {mappings}. Ensure
+    that only one {mapping_type} maps to {resource_type} {identifier}.{property_name}"""
+
+    mappings: frozenset[T_ReferenceIdentifier]
+    mapping_type: ResourceType

cognite/neat/issues/errors/_resources.py

@@ -0,0 +1,111 @@
+from dataclasses import dataclass
+from typing import Generic
+
+from cognite.neat.issues._base import NeatError, ResourceType, T_Identifier, T_ReferenceIdentifier
+from cognite.neat.utils.text import humanize_collection
+
+
+@dataclass(frozen=True)
+class ResourceError(NeatError, Generic[T_Identifier], RuntimeError):
+    """Base class for resource errors {resource_type} with identifier {identifier}"""
+
+    identifier: T_Identifier
+    resource_type: ResourceType
+
+
+@dataclass(frozen=True)
+class ResourceDuplicatedError(ResourceError[T_Identifier]):
+    """The {resource_type} with identifier {identifier} is duplicated in {location}"""
+
+    fix = "Remove the duplicate {resource_type} {identifier}."
+    location: str
+
+
+@dataclass(frozen=True)
+class ResourceRetrievalError(ResourceError[T_Identifier]):
+    """Failed to retrieve {resource_type} with identifier {identifier}. The error was: {error}"""
+
+    error: str
+
+
+@dataclass(frozen=True)
+class ResourceNotFoundError(ResourceError, Generic[T_Identifier, T_ReferenceIdentifier]):
+    """The {resource_type} with identifier {identifier} does not exist"""
+
+    extra = " This is expected by {referred_type} {referred_by}."
+
+    fix = "Create the {resource_type} {identifier}"
+
+    referred_by: T_ReferenceIdentifier | None = None
+    referred_type: ResourceType | None = None
+    more: str | None = None
+
+    def as_message(self) -> str:
+        msg = (self.__doc__ or "").format(resource_type=self.resource_type, identifier=self.identifier)
+        if self.referred_by and self.referred_type:
+            msg += self.extra.format(referred_type=self.referred_type, referred_by=self.referred_by)
+        if self.more:
+            msg += f" {self.more}"
+        msg += f" Fix {self.fix.format(resource_type=self.resource_type, identifier=self.identifier)}"
+        return msg
+
+
+@dataclass(frozen=True)
+class ResourceNotDefinedError(ResourceError[T_Identifier]):
+    """The {resource_type} {identifier} is not defined in the {location}"""
+
+    extra = "{column_name} {row_number} in {sheet_name}"
+    fix = "Define the {resource_type} {identifier} in {location}."
+
+    location: str
+    column_name: str | None = None
+    row_number: int | None = None
+    sheet_name: str | None = None
+
+
+@dataclass(frozen=True)
+class ResourceConvertionError(ResourceError, ValueError):
+    """Failed to convert the {resource_type} {identifier} to {target_format}: {reason}"""
+
+    fix = "Check the error message and correct the rules."
+    target_format: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class ResourceCreationError(ResourceError[T_Identifier], ValueError):
+    """Failed to create {resource_type} with identifier {identifier}. The error was: {error}"""
+
+    error: str
+
+
+@dataclass(frozen=True)
+class ResourceMissingIdentifierError(NeatError, ValueError):
+    """The {resource_type} with name {name} is missing an identifier."""
+
+    resource_type: ResourceType
+    name: str | None = None
+
+
+@dataclass(frozen=True)
+class ResourceChangedError(ResourceError[T_Identifier]):
+    """The {resource_type} with identifier {identifier} has changed{changed}"""
+
+    fix = (
+        "When extending model with extension set to addition or reshape, "
+        "the {resource_type} properties must remain the same"
+    )
+
+    changed_properties: frozenset[str]
+    changed_attributes: frozenset[str]
+
+    def as_message(self) -> str:
+        if self.changed_properties:
+            changed = f" properties {humanize_collection(self.changed_properties)}."
+        elif self.changed_attributes:
+            changed = f" attributes {humanize_collection(self.changed_attributes)}."
+        else:
+            changed = "."
+        msg = (self.__doc__ or "").format(resource_type=self.resource_type, identifier=self.identifier, changed=changed)
+        msg += f"Fix {self.fix.format(resource_type=self.resource_type)}"
+        return msg

cognite/neat/issues/errors/_workflow.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+
+from cognite.neat.issues import NeatError
+
+
+@dataclass(frozen=True)
+class WorkFlowMissingDataError(NeatError, ValueError):
+    """In the workflow step {step_name} the following data is missing: {missing_data}."""
+
+    step_name: str
+    missing_data: frozenset[str]
+
+
+@dataclass(frozen=True)
+class WorkflowStepNotInitializedError(NeatError, RuntimeError):
+    """Step {step_name} has not been initialized."""
+
+    step_name: str
+
+
+@dataclass(frozen=True)
+class WorkflowConfigurationNotSetError(NeatError, RuntimeError):
+    """The configuration variable '{config_variable}' is not set. Please set the configuration
+    before running the workflow."""
+
+    config_variable: str
+
+
+@dataclass(frozen=True)
+class WorkflowStepOutputError(NeatError, RuntimeError):
+    """Object type {step_type} is not supported as step output.
+
+    Step output must be of type DataContract or a FlowMessage.
+    """
+
+    step_type: str

cognite/neat/issues/formatters.py

@@ -75,7 +75,7 @@ class BasicHTML(Formatter):
             ul = ET.SubElement(self._body, "ul")
             for issue in issues_in_category:
                 li = ET.SubElement(ul, "li")
-                li.text = issue.message()
+                li.text = issue.as_message()
 
 
 FORMATTER_BY_NAME: dict[str, type[Formatter]] = {

cognite/neat/issues/warnings/__init__.py

@@ -0,0 +1,66 @@
+"""All warnings raised by the neat package are defined here. Note this module is called 'warnings' which
+conflicts with the built-in Python warnings module. However, it is expected to always be used in an absolute
+import, and should thus not cause a naming conflict."""
+
+from cognite.neat.issues._base import DefaultWarning, NeatWarning, _get_subclasses
+
+from . import user_modeling
+from ._external import (
+    FileItemNotSupportedWarning,
+    FileMissingRequiredFieldWarning,
+    FileReadWarning,
+    FileTypeUnexpectedWarning,
+)
+from ._general import NeatValueWarning, NotSupportedWarning, RegexViolationWarning
+from ._models import (
+    BreakingModelingPrincipleWarning,
+    CDFNotSupportedWarning,
+    NotSupportedHasDataFilterLimitWarning,
+    NotSupportedViewContainerLimitWarning,
+    PrincipleMatchingSpaceAndVersionWarning,
+    PrincipleOneModelOneSpaceWarning,
+    PrincipleSolutionBuildsOnEnterpriseWarning,
+    UserModelingWarning,
+)
+from ._properties import (
+    PropertyDefinitionDuplicatedWarning,
+    PropertyNotFoundWarning,
+    PropertyTypeNotSupportedWarning,
+)
+from ._resources import (
+    ResourceNeatWarning,
+    ResourceNotFoundWarning,
+    ResourceRetrievalWarning,
+    ResourcesDuplicatedWarning,
+    ResourceTypeNotSupportedWarning,
+)
+
+__all__ = [
+    "DefaultWarning",
+    "FileReadWarning",
+    "FileMissingRequiredFieldWarning",
+    "FileItemNotSupportedWarning",
+    "FileTypeUnexpectedWarning",
+    "NeatValueWarning",
+    "NotSupportedWarning",
+    "UserModelingWarning",
+    "CDFNotSupportedWarning",
+    "BreakingModelingPrincipleWarning",
+    "PropertyDefinitionDuplicatedWarning",
+    "PropertyTypeNotSupportedWarning",
+    "PropertyNotFoundWarning",
+    "ResourceNeatWarning",
+    "ResourcesDuplicatedWarning",
+    "RegexViolationWarning",
+    "ResourceNotFoundWarning",
+    "ResourceTypeNotSupportedWarning",
+    "ResourceRetrievalWarning",
+    "PrincipleOneModelOneSpaceWarning",
+    "PrincipleMatchingSpaceAndVersionWarning",
+    "PrincipleSolutionBuildsOnEnterpriseWarning",
+    "NotSupportedViewContainerLimitWarning",
+    "NotSupportedHasDataFilterLimitWarning",
+    "user_modeling",
+]
+
+_NEAT_WARNINGS_BY_NAME = {warning.__name__: warning for warning in _get_subclasses(NeatWarning, include_base=True)}

cognite/neat/issues/warnings/_external.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from cognite.neat.issues import NeatWarning
+
+
+@dataclass(frozen=True)
+class FileReadWarning(NeatWarning):
+    """Error when reading file, {filepath}: {reason}"""
+
+    filepath: Path
+    reason: str
+
+
+@dataclass(frozen=True)
+class FileMissingRequiredFieldWarning(NeatWarning):
+    """Missing required {field_name} in {filepath}: {field}. The file will be skipped"""
+
+    filepath: Path
+    field_name: str
+    field: str
+
+
+@dataclass(frozen=True)
+class FileTypeUnexpectedWarning(NeatWarning):
+    """Unexpected file type: {filepath}. Expected format: {expected_format}"""
+
+    extra = "Error: {error_message}"
+
+    filepath: Path
+    expected_format: frozenset[str]
+    error_message: str | None = None
+
+
+@dataclass(frozen=True)
+class FileItemNotSupportedWarning(NeatWarning):
+    """The item {item} in {filepath} is not supported. The item will be skipped"""
+
+    item: str
+    filepath: Path

cognite/neat/issues/warnings/_general.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+
+from cognite.neat.issues import NeatWarning
+
+
+@dataclass(frozen=True)
+class NeatValueWarning(NeatWarning):
+    """{value}"""
+
+    value: str
+
+
+@dataclass(frozen=True)
+class NotSupportedWarning(NeatWarning):
+    """{feature} is not supported"""
+
+    feature: str
+
+
+@dataclass(frozen=True)
+class RegexViolationWarning(NeatWarning):
+    """The value '{value}' of {identifier} does not match the {pattern_name} pattern '{pattern}'"""
+
+    extra = "{motivation}"
+    value: str
+    pattern: str
+    identifier: str
+    pattern_name: str
+    motivation: str | None = None

cognite/neat/issues/warnings/_models.py

@@ -0,0 +1,92 @@
+from abc import ABC
+from dataclasses import dataclass
+from typing import ClassVar
+
+from cognite.client.data_classes.data_modeling import ViewId
+
+from cognite.neat.constants import DMS_VIEW_CONTAINER_SIZE_LIMIT
+from cognite.neat.issues import NeatWarning
+
+_BASE_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/data-modeling-principles.html"
+
+
+@dataclass(frozen=True)
+class BreakingModelingPrincipleWarning(NeatWarning, ABC):
+    """{warning_class}: {specific} violates the {principle} principle.
+    See {url} for more information."""
+
+    url: ClassVar[str]
+    specific: str
+
+    def as_message(self) -> str:
+        principle = type(self).__name__.removesuffix("Warning")
+        url = f"{_BASE_URL}#{self.url}"
+        return (self.__doc__ or "").format(
+            warning_class=BreakingModelingPrincipleWarning.__name__,
+            specific=self.specific,
+            principle=principle,
+            url=url,
+        )
+
+
+@dataclass(frozen=True)
+class PrincipleOneModelOneSpaceWarning(BreakingModelingPrincipleWarning):
+    """{warning_class}: {specific} violates the {principle} principle.
+    See {url} for more information."""
+
+    url = "all-data-models-are-kept-in-its-own-space"
+
+
+@dataclass(frozen=True)
+class PrincipleMatchingSpaceAndVersionWarning(BreakingModelingPrincipleWarning):
+    """{warning_class}: {specific} violates the {principle} principle.
+    See {url} for more information."""
+
+    url = "all-views-of-a-data-models-have-the-same-version-and-space-as-the-data-model"
+
+
+@dataclass(frozen=True)
+class PrincipleSolutionBuildsOnEnterpriseWarning(BreakingModelingPrincipleWarning):
+    """{warning_class}: {specific} violates the {principle} principle.
+    See {url} for more information."""
+
+    url = "solution-data-models-should-always-be-referencing-the-enterprise-data-model"
+
+
+@dataclass(frozen=True)
+class UserModelingWarning(NeatWarning, ABC):
+    """This is a generic warning for user modeling issues.
+    These warnings will not cause the resulting model to be invalid, but
+    will likely lead to suboptimal performance, unnecessary complexity, or other issues."""
+
+    ...
+
+
+@dataclass(frozen=True)
+class CDFNotSupportedWarning(NeatWarning, ABC):
+    """This is a base class for warnings for modeling issues that will
+    likely lead to the CDF API rejecting the model."""
+
+    ...
+
+
+@dataclass(frozen=True)
+class NotSupportedViewContainerLimitWarning(CDFNotSupportedWarning):
+    """The view {view_id} maps, {count} containers, which is more than the limit {limit}."""
+
+    fix = "Reduce the number of containers the view maps to." ""
+
+    view_id: ViewId
+    count: int
+    limit: int = DMS_VIEW_CONTAINER_SIZE_LIMIT
+
+
+@dataclass(frozen=True)
+class NotSupportedHasDataFilterLimitWarning(CDFNotSupportedWarning):
+    """The view {view_id} uses a hasData filter applied to {count} containers, which is more than the limit {limit}."""
+
+    fix = "Do not map to more than {limit} containers."
+
+    view_id: ViewId
+    count: int
+    limit: int = DMS_VIEW_CONTAINER_SIZE_LIMIT

cognite/neat/issues/warnings/_properties.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass
+from typing import Generic
+
+from cognite.neat.issues._base import ResourceType
+
+from ._resources import ResourceNeatWarning, T_Identifier, T_ReferenceIdentifier
+
+
+@dataclass(frozen=True)
+class PropertyWarning(ResourceNeatWarning[T_Identifier]):
+    """Base class for property warnings {resource_type} with identifier {identifier}.{property_name}"""
+
+    property_name: str
+
+
+@dataclass(frozen=True)
+class PropertyTypeNotSupportedWarning(PropertyWarning[T_Identifier]):
+    """The {resource_type} with identifier {identifier} has a property {property_name}
+    of unsupported type {property_type}. This will be ignored."""
+
+    property_type: str
+
+
+@dataclass(frozen=True)
+class PropertyNotFoundWarning(PropertyWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
+    """The {resource_type} with identifier {identifier} does not have a property {property_name} referred
+    to by {referred_type} {referred_by} does not exist. This will be ignored.
+    """
+
+    fix = "Ensure the {resource_type} {identifier} has a {property_name} property"
+    referred_by: T_ReferenceIdentifier
+    referred_type: ResourceType
+
+
+@dataclass(frozen=True)
+class PropertyDefinitionDuplicatedWarning(PropertyWarning[T_Identifier]):
+    """Got multiple values for the {resource_type} {identifier}.{property_name} {values}.
+    {default_action}"""
+
+    extra = "Recommended action: {recommended_action}"
+
+    values: frozenset[str]
+    default_action: str
+    recommended_action: str | None = None

cognite/neat/issues/warnings/_resources.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+from typing import Generic
+
+from cognite.neat.issues._base import NeatWarning, ResourceType, T_Identifier, T_ReferenceIdentifier
+
+
+# Name ResourceNeatWarning to avoid conflicts with the built-in ResourceWarning
+@dataclass(frozen=True)
+class ResourceNeatWarning(NeatWarning, Generic[T_Identifier]):
+    """Base class for resource warnings {resource_type} with identifier {identifier}"""
+
+    identifier: T_Identifier
+    resource_type: ResourceType
+
+
+@dataclass(frozen=True)
+class ResourceNotFoundWarning(ResourceNeatWarning, Generic[T_Identifier, T_ReferenceIdentifier]):
+    """The {resource_type} with identifier {identifier} referred by {referred_type} {referred_by} does not exist.
+    This will be ignored."""
+
+    fix = "Create the {resource_type}"
+
+    referred_by: T_ReferenceIdentifier
+    referred_type: str
+
+
+@dataclass(frozen=True)
+class ResourcesDuplicatedWarning(NeatWarning, Generic[T_Identifier]):
+    """Duplicated {resource_type} with identifiers {resources} were found. {default_action}"""
+
+    fix = "Remove the duplicate resources"
+
+    resources: frozenset[T_Identifier]
+    resource_type: ResourceType
+    default_action: str
+
+
+@dataclass(frozen=True)
+class ResourceRetrievalWarning(NeatWarning, Generic[T_Identifier]):
+    """Failed to retrieve {resource_type} with identifiers {resources}. Continuing without
+    these resources."""
+
+    extra = "The error was: {error}"
+
+    fix = "Check the error."
+
+    resources: frozenset[T_Identifier]
+    resource_type: ResourceType
+    error: str | None = None
+
+
+class ResourceTypeNotSupportedWarning(ResourceNeatWarning[T_Identifier]):
+    """The {resource_type} with identifier {identifier} is not supported. This will be ignored."""
+
+    resource_type: str  # type: ignore[assignment]

cognite/neat/issues/warnings/user_modeling.py

@@ -0,0 +1,113 @@
+"""These are user modeling warnings. These warnings will not cause the resulting model
+to be invalid, but will likely lead to suboptimal performance, unnecessary complexity,
+or other issues."""
+# These warnings are usually only used once in the code base.
+
+from dataclasses import dataclass
+
+from cognite.client.data_classes.data_modeling import ContainerId, ViewId
+
+from cognite.neat.constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
+
+from ._models import UserModelingWarning
+
+__all__ = [
+    "DirectRelationMissingSourceWarning",
+    "EmptyContainerWarning",
+    "HasDataFilterOnNoPropertiesViewWarning",
+    "NodeTypeFilterOnParentViewWarning",
+    "HasDataFilterOnViewWithReferencesWarning",
+    "ViewPropertyLimitWarning",
+    "NotNeatSupportedFilterWarning",
+    "ParentInDifferentSpaceWarning",
+]
+
+
+@dataclass(frozen=True)
+class DirectRelationMissingSourceWarning(UserModelingWarning):
+    """The view {view_id}.{prop_name} is a direct relation without a source.
+    Direct relations in views should point to a single other view, if not, you end up
+    with a more complex model than necessary."""
+
+    fix = "Create the source view"
+
+    view_id: ViewId
+    prop_name: str
+
+
+@dataclass(frozen=True)
+class ParentInDifferentSpaceWarning(UserModelingWarning):
+    """The view {view_id} has multiple parents in different spaces.
+    Neat recommends maximum one implementation of a view from another space."""
+
+    fix = "Ensure all parents of the view are in the same space"
+
+    view_id: ViewId
+
+
+@dataclass(frozen=True)
+class EmptyContainerWarning(UserModelingWarning):
+    """Container {container_id} is empty and will be skipped.
+    The container does not have any properties."""
+
+    fix = "Add properties to the container or remove the container"
+
+    container_id: ContainerId
+
+
+@dataclass(frozen=True)
+class HasDataFilterOnNoPropertiesViewWarning(UserModelingWarning):
+    """Cannot set hasData filter on view {view_id}.
+    The view does not have properties in any containers.
+    Use a node type filter instead."""
+
+    fix = "Use a node type filter instead"
+
+    view_id: ViewId
+
+
+@dataclass(frozen=True)
+class NodeTypeFilterOnParentViewWarning(UserModelingWarning):
+    """Setting a node type filter on parent view {view_id}.
+    This is not recommended as parent views are typically used for multiple types of nodes."""
+
+    fix = "Use a HasData filter instead"
+
+    view_id: ViewId
+
+
+@dataclass(frozen=True)
+class HasDataFilterOnViewWithReferencesWarning(UserModelingWarning):
+    """Setting a hasData filter on view {view_id} which references other views {references}.
+    This is not recommended as it will lead to no nodes being returned when querying the solution view.
+    Use a NodeType filter instead."""
+
+    fix = "Use a NodeType filter instead"
+
+    view_id: ViewId
+    references: frozenset[ViewId]
+
+
+@dataclass(frozen=True)
+class ViewPropertyLimitWarning(UserModelingWarning):
+    """The number of properties in the {view_id} view is {count} which
+    is more than the API limit {limit} properties.
+    This can lead to performance issues.
+    Reduce the number of properties in the view."""
+
+    fix = "Reduce the number of properties in the view"
+
+    view_id: ViewId
+    count: int
+    limit: int = DMS_CONTAINER_PROPERTY_SIZE_LIMIT
+
+
+@dataclass(frozen=True)
+class NotNeatSupportedFilterWarning(UserModelingWarning):
+    """The view {view_id} uses a non-standard filter.
+    This will not be validated by Neat, and is thus not recommended.
+    If you can use a HasData or NoteType filter."""
+
+    fix = "Use a HasData or NoteType filter instead"
+
+    view_id: ViewId