cognite-neat 0.102.0__py3-none-any.whl → 0.103.1__py3-none-any.whl

cognite/neat/_graph/transformers/_value_type.py

@@ -1,8 +1,17 @@
-from rdflib import XSD, Graph, URIRef
+import warnings
+from collections.abc import Callable
+from typing import Any, cast
+from urllib.parse import quote
+
+import rdflib
+from rdflib import RDF, XSD, Graph, Namespace, URIRef
 
 from cognite.neat._constants import UNKNOWN_TYPE
 from cognite.neat._graph.queries import Queries
-from cognite.neat._utils.rdf_ import remove_namespace_from_uri
+from cognite.neat._issues.warnings import NeatValueWarning, PropertyDataTypeConversionWarning
+from cognite.neat._utils.auxiliary import string_to_ideal_type
+from cognite.neat._utils.collection_ import iterate_progress_bar
+from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
 
 from ._base import BaseTransformer
 
@@ -64,3 +73,188 @@ class SplitMultiValueProperty(BaseTransformer):
                     graph.remove((s, property_uri, o))
                     new_property = URIRef(f"{property_uri}_{remove_namespace_from_uri(value_type_uri)}")
                     graph.add((s, new_property, o))
+
+
+class ConvertLiteral(BaseTransformer):
+    description: str = "ConvertLiteral is a transformer that improve data typing of a literal value."
+    _use_only_once: bool = False
+    _need_changes = frozenset({})
+
+    _count_by_properties = """SELECT (COUNT(?value) AS ?valueCount)
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?value
+       FILTER(isLiteral(?value))
+    }}"""
+
+    _count_by_properties_uri = """SELECT (COUNT(?value) AS ?valueCount)
+        WHERE {{
+          ?instance a <{subject_type}> .
+          ?instance <{subject_predicate}> ?value
+           FILTER(isIRI(?value))
+        }}"""
+
+    _properties = """SELECT ?instance ?value
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?value
+
+      FILTER(isLiteral(?value))
+
+    }}"""
+
+    def __init__(
+        self,
+        subject_type: URIRef,
+        subject_predicate: URIRef,
+        conversion: Callable[[Any], Any] | None = None,
+    ) -> None:
+        self.subject_type = subject_type
+        self.subject_predicate = subject_predicate
+        self.conversion = conversion or string_to_ideal_type
+        self._type_name = remove_namespace_from_uri(subject_type)
+        self._property_name = remove_namespace_from_uri(subject_predicate)
+
+    def transform(self, graph: Graph) -> None:
+        count_connection_query = self._count_by_properties_uri.format(
+            subject_type=self.subject_type, subject_predicate=self.subject_predicate
+        )
+        connection_count_res = list(graph.query(count_connection_query))
+        connection_count = int(connection_count_res[0][0])  # type: ignore [index, arg-type]
+
+        if connection_count > 0:
+            warnings.warn(
+                NeatValueWarning(
+                    f"Skipping {connection_count} of {self._type_name}.{self._property_name} "
+                    f"as these are connections and not data values."
+                ),
+                stacklevel=2,
+            )
+
+        count_query = self._count_by_properties.format(
+            subject_type=self.subject_type, subject_predicate=self.subject_predicate
+        )
+
+        property_count_res = list(graph.query(count_query))
+        property_count = int(property_count_res[0][0])  # type: ignore [index, arg-type]
+        iterate_query = self._properties.format(
+            subject_type=self.subject_type, subject_predicate=self.subject_predicate
+        )
+
+        for instance, literal in iterate_progress_bar(  # type: ignore[misc]
+            graph.query(iterate_query),
+            total=property_count,
+            description=f"Converting {self._type_name}.{self._property_name}.",
+        ):
+            value = cast(rdflib.Literal, literal).toPython()
+            try:
+                converted_value = self.conversion(value)
+            except Exception as e:
+                warnings.warn(
+                    PropertyDataTypeConversionWarning(str(instance), self._type_name, self._property_name, str(e)),
+                    stacklevel=2,
+                )
+                continue
+
+            graph.add((instance, self.subject_predicate, rdflib.Literal(converted_value)))
+            graph.remove((instance, self.subject_predicate, literal))
+
+
+class LiteralToEntity(BaseTransformer):
+    description = "Converts a literal value to new entity"
+
+    _count_properties_of_type = """SELECT (COUNT(?property) AS ?propertyCount)
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?property
+      FILTER(isLiteral(?property))
+    }}"""
+    _count_connections_of_type = """SELECT (COUNT(?property) AS ?propertyCount)
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?property
+      FILTER(isIRI(?property))
+    }}"""
+
+    _properties_of_type = """SELECT ?instance ?property
+    WHERE {{
+      ?instance a <{subject_type}> .
+      ?instance <{subject_predicate}> ?property
+      FILTER(isLiteral(?property))
+    }}"""
+
+    _count_properties = """SELECT (COUNT(?property) AS ?propertyCount)
+    WHERE {{
+      ?instance <{subject_predicate}> ?property
+      FILTER(isLiteral(?property))
+    }}"""
+    _count_connections = """SELECT (COUNT(?property) AS ?propertyCount)
+        WHERE {{
+          ?instance <{subject_predicate}> ?property
+          FILTER(isIRI(?property))
+        }}"""
+    _properties = """SELECT ?instance ?property
+    WHERE {{
+      ?instance <{subject_predicate}> ?property
+      FILTER(isLiteral(?property))
+    }}"""
+
+    def __init__(
+        self, subject_type: URIRef | None, subject_predicate: URIRef, entity_type: str, new_property: str | None = None
+    ) -> None:
+        self.subject_type = subject_type
+        self.subject_predicate = subject_predicate
+        self.entity_type = entity_type
+        self.new_property = new_property
+
+    def transform(self, graph: Graph) -> None:
+        if self.subject_type is None:
+            count_query = self._count_properties.format(subject_predicate=self.subject_predicate)
+            iterate_query = self._properties.format(subject_predicate=self.subject_predicate)
+            connection_count_query = self._count_connections.format(subject_predicate=self.subject_predicate)
+        else:
+            count_query = self._count_properties_of_type.format(
+                subject_type=self.subject_type, subject_predicate=self.subject_predicate
+            )
+            iterate_query = self._properties_of_type.format(
+                subject_type=self.subject_type, subject_predicate=self.subject_predicate
+            )
+            connection_count_query = self._count_connections_of_type.format(
+                subject_type=self.subject_type, subject_predicate=self.subject_predicate
+            )
+
+        connection_count_res = list(graph.query(connection_count_query))
+        connection_count = int(connection_count_res[0][0])  # type: ignore [index, arg-type]
+        if connection_count > 0:
+            warnings.warn(
+                NeatValueWarning(
+                    f"Skipping {connection_count} of {remove_namespace_from_uri(self.subject_predicate)} "
+                    f"as these are connections and not data values."
+                ),
+                stacklevel=2,
+            )
+
+        property_count_res = list(graph.query(count_query))
+        property_count = int(property_count_res[0][0])  # type: ignore [index, arg-type]
+
+        instance: URIRef
+        description = f"Creating {remove_namespace_from_uri(self.subject_predicate)}."
+        if self.subject_type is not None:
+            description = (
+                f"Creating {remove_namespace_from_uri(self.subject_type)}."
+                f"{remove_namespace_from_uri(self.subject_predicate)}."
+            )
+        for instance, literal in iterate_progress_bar(  # type: ignore[misc, assignment]
+            graph.query(iterate_query),
+            total=property_count,
+            description=description,
+        ):
+            value = cast(rdflib.Literal, literal).toPython()
+            namespace = Namespace(get_namespace(instance))
+            entity_type = namespace[self.entity_type]
+            new_entity = namespace[f"{self.entity_type}_{quote(value)!s}"]
+            graph.add((new_entity, RDF.type, entity_type))
+            if self.new_property is not None:
+                graph.add((new_entity, namespace[self.new_property], rdflib.Literal(value)))
+            graph.add((instance, self.subject_predicate, new_entity))
+            graph.remove((instance, self.subject_predicate, literal))

cognite/neat/_issues/__init__.py

@@ -15,14 +15,14 @@ from ._base import (
 )
 
 __all__ = [
-    "NeatIssue",
-    "NeatError",
-    "NeatWarning",
     "DefaultWarning",
-    "NeatIssueList",
+    "FutureResult",
     "IssueList",
     "MultiValueError",
-    "catch_warnings",
+    "NeatError",
+    "NeatIssue",
+    "NeatIssueList",
+    "NeatWarning",
     "catch_issues",
-    "FutureResult",
+    "catch_warnings",
 ]

cognite/neat/_issues/_base.py

@@ -32,12 +32,12 @@ else:
 
 
 __all__ = [
-    "NeatIssue",
-    "NeatError",
-    "NeatWarning",
     "DefaultWarning",
-    "NeatIssueList",
     "MultiValueError",
+    "NeatError",
+    "NeatIssue",
+    "NeatIssueList",
+    "NeatWarning",
 ]
 
 T_Identifier = TypeVar("T_Identifier", bound=Hashable)

cognite/neat/_issues/errors/__init__.py

@@ -38,40 +38,40 @@ from ._workflow import (
 )
 
 __all__ = [
+    "AuthorizationError",
+    "CDFMissingClientError",
+    "DefaultPydanticError",
+    "FileMissingRequiredFieldError",
+    "FileNotAFileError",
+    "FileNotFoundNeatError",
+    "FileReadError",
+    "FileTypeUnexpectedError",
     "NeatError",
-    "NeatValueError",
     "NeatImportError",
-    "RegexViolationError",
-    "AuthorizationError",
+    "NeatTypeError",
+    "NeatValueError",
     "NeatYamlError",
-    "FileReadError",
-    "ResourceCreationError",
-    "FileNotFoundNeatError",
-    "FileMissingRequiredFieldError",
+    "PropertyDefinitionDuplicatedError",
     "PropertyDefinitionError",
-    "PropertyTypeNotSupportedError",
+    "PropertyMappingDuplicatedError",
     "PropertyNotFoundError",
-    "PropertyDefinitionDuplicatedError",
+    "PropertyTypeNotSupportedError",
+    "RegexViolationError",
     "ResourceChangedError",
+    "ResourceConversionError",
+    "ResourceCreationError",
     "ResourceDuplicatedError",
-    "ResourceRetrievalError",
-    "ResourceNotFoundError",
     "ResourceError",
-    "ResourceNotDefinedError",
     "ResourceMissingIdentifierError",
-    "ResourceConversionError",
-    "WorkflowConfigurationNotSetError",
+    "ResourceNotDefinedError",
+    "ResourceNotFoundError",
+    "ResourceRetrievalError",
+    "ReversedConnectionNotFeasibleError",
+    "RowError",
     "WorkFlowMissingDataError",
+    "WorkflowConfigurationNotSetError",
     "WorkflowStepNotInitializedError",
     "WorkflowStepOutputError",
-    "FileTypeUnexpectedError",
-    "FileNotAFileError",
-    "DefaultPydanticError",
-    "PropertyMappingDuplicatedError",
-    "RowError",
-    "NeatTypeError",
-    "ReversedConnectionNotFeasibleError",
-    "CDFMissingClientError",
 ]
 
 _NEAT_ERRORS_BY_NAME = {error.__name__: error for error in _get_subclasses(NeatError, include_base=True)}

cognite/neat/_issues/formatters.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from ._base import NeatError, NeatIssueList, NeatWarning
 
-__all__ = ["Formatter", "BasicHTML", "FORMATTER_BY_NAME"]
+__all__ = ["FORMATTER_BY_NAME", "BasicHTML", "Formatter"]
 
 
 class Formatter(ABC):

cognite/neat/_issues/warnings/__init__.py

@@ -26,6 +26,7 @@ from ._models import (
     UserModelingWarning,
 )
 from ._properties import (
+    PropertyDataTypeConversionWarning,
     PropertyDefinitionDuplicatedWarning,
     PropertyNotFoundWarning,
     PropertyOverwritingWarning,
@@ -43,38 +44,39 @@ from ._resources import (
 )
 
 __all__ = [
+    "BreakingModelingPrincipleWarning",
+    "CDFAuthWarning",
+    "CDFMaxIterationsWarning",
+    "CDFNotSupportedWarning",
     "DefaultWarning",
-    "FileReadWarning",
-    "FileMissingRequiredFieldWarning",
     "FileItemNotSupportedWarning",
+    "FileMissingRequiredFieldWarning",
+    "FileReadWarning",
     "FileTypeUnexpectedWarning",
     "NeatValueWarning",
+    "NotSupportedHasDataFilterLimitWarning",
+    "NotSupportedViewContainerLimitWarning",
     "NotSupportedWarning",
-    "UserModelingWarning",
-    "CDFNotSupportedWarning",
-    "BreakingModelingPrincipleWarning",
+    "PrincipleMatchingSpaceAndVersionWarning",
+    "PrincipleOneModelOneSpaceWarning",
+    "PrincipleSolutionBuildsOnEnterpriseWarning",
+    "PropertyDataTypeConversionWarning",
     "PropertyDefinitionDuplicatedWarning",
-    "PropertyTypeNotSupportedWarning",
     "PropertyNotFoundWarning",
-    "PropertyValueTypeUndefinedWarning",
     "PropertyOverwritingWarning",
     "PropertySkippedWarning",
-    "ResourceNeatWarning",
-    "ResourcesDuplicatedWarning",
+    "PropertyTypeNotSupportedWarning",
+    "PropertyValueTypeUndefinedWarning",
     "RegexViolationWarning",
+    "ResourceNeatWarning",
     "ResourceNotFoundWarning",
-    "ResourceTypeNotSupportedWarning",
-    "ResourceRetrievalWarning",
     "ResourceRegexViolationWarning",
-    "PrincipleOneModelOneSpaceWarning",
-    "PrincipleMatchingSpaceAndVersionWarning",
-    "PrincipleSolutionBuildsOnEnterpriseWarning",
-    "NotSupportedViewContainerLimitWarning",
-    "NotSupportedHasDataFilterLimitWarning",
+    "ResourceRetrievalWarning",
+    "ResourceTypeNotSupportedWarning",
+    "ResourcesDuplicatedWarning",
     "UndefinedViewWarning",
-    "CDFAuthWarning",
+    "UserModelingWarning",
     "user_modeling",
-    "CDFMaxIterationsWarning",
 ]
 
 _NEAT_WARNINGS_BY_NAME = {warning.__name__: warning for warning in _get_subclasses(NeatWarning, include_base=True)}

cognite/neat/_issues/warnings/_properties.py

@@ -70,3 +70,10 @@ class PropertySkippedWarning(PropertyWarning[T_Identifier]):
     which is skipped. {reason}."""
 
     reason: str
+
+
+@dataclass(unsafe_hash=True)
+class PropertyDataTypeConversionWarning(PropertyWarning[T_Identifier]):
+    """The {resource_type} with identifier {identifier} failed to convert the property {property_name}: {error}"""
+
+    error: str

cognite/neat/_issues/warnings/user_modeling.py

@@ -12,12 +12,12 @@ from cognite.neat._constants import DMS_CONTAINER_PROPERTY_SIZE_LIMIT
 from ._models import UserModelingWarning
 
 __all__ = [
+    "ContainerPropertyLimitWarning",
     "DirectRelationMissingSourceWarning",
     "EmptyContainerWarning",
     "HasDataFilterOnNoPropertiesViewWarning",
-    "NodeTypeFilterOnParentViewWarning",
     "HasDataFilterOnViewWithReferencesWarning",
-    "ContainerPropertyLimitWarning",
+    "NodeTypeFilterOnParentViewWarning",
     "NotNeatSupportedFilterWarning",
     "ParentInDifferentSpaceWarning",
 ]

cognite/neat/_rules/analysis/__init__.py

@@ -1,4 +1,4 @@
 from ._dms import DMSAnalysis
 from ._information import InformationAnalysis
 
-__all__ = ["InformationAnalysis", "DMSAnalysis"]
+__all__ = ["DMSAnalysis", "InformationAnalysis"]

cognite/neat/_rules/catalog/__init__.py

@@ -4,3 +4,4 @@ from pathlib import Path
 
 _CATALOG = Path(__file__).parent
 imf_attributes = _CATALOG / "info-rules-imf.xlsx"
+hello_world_pump = _CATALOG / "hello_world_pump.xlsx"

cognite/neat/_rules/exporters/__init__.py

@@ -7,15 +7,15 @@ from ._rules2yaml import YAMLExporter
 
 __all__ = [
     "BaseExporter",
-    "DMSExporter",
     "CDFExporter",
-    "SemanticDataModelExporter",
-    "OWLExporter",
+    "DMSExporter",
+    "ExcelExporter",
     "GraphExporter",
+    "InstanceTemplateExporter",
+    "OWLExporter",
     "SHACLExporter",
-    "ExcelExporter",
+    "SemanticDataModelExporter",
     "YAMLExporter",
-    "InstanceTemplateExporter",
 ]
 
 

cognite/neat/_rules/exporters/_rules2excel.py

@@ -117,7 +117,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
 
             main_header = self._main_header_by_sheet_name[sheet_name]
             sheet.append([main_header] + [""] * (len(headers) - 1))
-            sheet.merge_cells(start_row=1, start_column=1, end_row=1, end_column=len(headers))
+
             if headers[0] == "Neat ID":
                 # Move the Neat ID to the end of the columns
                 headers = headers[1:] + ["Neat ID"]
@@ -157,12 +157,13 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
                 # This freezes all rows above the given row
                 sheet.freeze_panes = sheet["A3"]
 
-                sheet["A1"].alignment = Alignment(horizontal="center")
+                sheet["A1"].alignment = Alignment(horizontal="left")
 
             if self._styling_level > 1:
                 # Make the header row bold, larger, and colored
-                sheet["A1"].font = Font(bold=True, size=20)
-                sheet["A1"].fill = PatternFill(fgColor="FFC000", patternType="solid")
+                for cell, *_ in sheet.iter_cols(min_row=1, max_row=1, min_col=1, max_col=len(headers)):
+                    cell.font = Font(bold=True, size=20)
+                    cell.fill = PatternFill(fgColor="FFC000", patternType="solid")
                 for cell in sheet["2"]:
                     cell.font = Font(bold=True, size=14)
 

cognite/neat/_rules/importers/__init__.py

@@ -7,14 +7,14 @@ from ._yaml2rules import YAMLImporter
 
 __all__ = [
     "BaseImporter",
-    "OWLImporter",
-    "IMFImporter",
     "DMSImporter",
+    "DTDLImporter",
     "ExcelImporter",
     "GoogleSheetImporter",
-    "DTDLImporter",
-    "YAMLImporter",
+    "IMFImporter",
     "InferenceImporter",
+    "OWLImporter",
+    "YAMLImporter",
 ]
 
 RulesImporters = (

cognite/neat/_rules/importers/_base.py

@@ -3,16 +3,18 @@ from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from contextlib import contextmanager, suppress
 from datetime import datetime
-from typing import Any, Generic, Literal
+from typing import TYPE_CHECKING, Any, Generic, Literal
 
 from pydantic import ValidationError
 
 from cognite.neat._constants import DEFAULT_NAMESPACE
 from cognite.neat._issues import IssueList, NeatError, NeatWarning
 from cognite.neat._rules._shared import ReadRules, T_InputRules
-from cognite.neat._store._provenance import Agent as ProvenanceAgent
 from cognite.neat._utils.auxiliary import class_html_doc
 
+if TYPE_CHECKING:
+    from cognite.neat._store._provenance import Agent as ProvenanceAgent
+
 
 class BaseImporter(ABC, Generic[T_InputRules]):
     """
@@ -48,8 +50,10 @@ class BaseImporter(ABC, Generic[T_InputRules]):
         return class_html_doc(cls)
 
     @property
-    def agent(self) -> ProvenanceAgent:
+    def agent(self) -> "ProvenanceAgent":
         """Provenance agent for the importer."""
+        from cognite.neat._store._provenance import Agent as ProvenanceAgent
+
         return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
 
 

cognite/neat/_rules/importers/_rdf/__init__.py

@@ -2,4 +2,4 @@ from ._imf2rules import IMFImporter
 from ._inference2rules import InferenceImporter
 from ._owl2rules import OWLImporter
 
-__all__ = ["IMFImporter", "OWLImporter", "InferenceImporter"]
+__all__ = ["IMFImporter", "InferenceImporter", "OWLImporter"]

cognite/neat/_rules/models/__init__.py

@@ -17,16 +17,16 @@ VERIFIED_RULES_BY_ROLE: dict[RoleTypes, type[InformationRules] | type[DMSRules]]
 
 
 __all__ = [
+    "INPUT_RULES_BY_ROLE",
     "DMSInputRules",
-    "InformationInputRules",
-    "InformationRules",
     "DMSRules",
-    "INPUT_RULES_BY_ROLE",
     "DMSSchema",
+    "DataModelType",
+    "ExtensionCategory",
+    "InformationInputRules",
+    "InformationRules",
     "RoleTypes",
     "SchemaCompleteness",
-    "ExtensionCategory",
-    "DataModelType",
     "SheetList",
     "SheetRow",
 ]

cognite/neat/_rules/models/_base_rules.py

@@ -149,7 +149,7 @@ class BaseMetadata(SchemaModel):
 
     role: ClassVar[RoleTypes] = Field(description="Role of the person creating the data model")
     aspect: ClassVar[DataModelAspect] = Field(description="Aspect of the data model")
-    space: SpaceType = Field(alias="prefix", description="The space where the data model is defined")
+    space: SpaceType = Field(description="The space where the data model is defined")
     external_id: DataModelExternalIdType = Field(
         alias="externalId", description="External identifier for the data model"
     )

cognite/neat/_rules/models/dms/__init__.py

@@ -13,20 +13,20 @@ from ._rules_input import (
 from ._validation import DMSValidation
 
 __all__ = [
-    "DMSRules",
-    "DMSSchema",
-    "DMSMetadata",
-    "DMSView",
-    "DMSProperty",
     "DMSContainer",
-    "DMSNode",
     "DMSEnum",
-    "DMSInputRules",
-    "DMSInputMetadata",
-    "DMSInputView",
-    "DMSInputProperty",
     "DMSInputContainer",
-    "DMSInputNode",
     "DMSInputEnum",
+    "DMSInputMetadata",
+    "DMSInputNode",
+    "DMSInputProperty",
+    "DMSInputRules",
+    "DMSInputView",
+    "DMSMetadata",
+    "DMSNode",
+    "DMSProperty",
+    "DMSRules",
+    "DMSSchema",
     "DMSValidation",
+    "DMSView",
 ]