cognite-neat 0.106.0__py3-none-any.whl → 0.108.0__py3-none-any.whl

cognite/neat/_graph/transformers/_rdfpath.py

@@ -35,8 +35,8 @@ class AddSelfReferenceProperty(BaseTransformer):
 
     def transform(self, graph: Graph) -> None:
         for property_ in self.properties:
-            prefix = property_.transformation.traversal.class_.prefix
-            suffix = property_.transformation.traversal.class_.suffix
+            prefix = property_.instance_source.traversal.class_.prefix
+            suffix = property_.instance_source.traversal.class_.suffix
 
             namespace = self.rules.prefixes[prefix] if prefix in self.rules.prefixes else self.rules.metadata.namespace
 
@@ -54,7 +54,7 @@ class AddSelfReferenceProperty(BaseTransformer):
                 property_=f"{self.rules.metadata.prefix}:{property_.property_}",
             )
 
-            property_.transformation = RDFPath(traversal=traversal)
+            property_.instance_source = RDFPath(traversal=traversal)
 
 
 class MakeConnectionOnExactMatch(BaseTransformerStandardised):

cognite/neat/_graph/transformers/_value_type.py

@@ -1,23 +1,21 @@
 import warnings
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
 from typing import Any, cast
 from urllib.parse import quote
 
 import rdflib
-from rdflib import RDF, XSD, Graph, Namespace, URIRef
+from rdflib import RDF, Literal, Namespace, URIRef
 from rdflib.query import ResultRow
 
-from cognite.neat._constants import UNKNOWN_TYPE
-from cognite.neat._graph.queries import Queries
+from cognite.neat._constants import NEAT
 from cognite.neat._issues.warnings import PropertyDataTypeConversionWarning
 from cognite.neat._utils.auxiliary import string_to_ideal_type
-from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
+from cognite.neat._utils.rdf_ import Triple, get_namespace, remove_namespace_from_uri
 
-from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
+from ._base import BaseTransformerStandardised, RowTransformationOutput
 
 
-# TODO: Standardise
-class SplitMultiValueProperty(BaseTransformer):
+class SplitMultiValueProperty(BaseTransformerStandardised):
     description: str = (
         "SplitMultiValueProperty is a transformer that splits a "
         "multi-value property into multiple single-value properties."
@@ -25,55 +23,67 @@ class SplitMultiValueProperty(BaseTransformer):
     _use_only_once: bool = True
     _need_changes = frozenset({})
 
-    _object_property_template: str = """SELECT ?s ?o WHERE{{
+    def __init__(self, unknown_type: URIRef | None = None) -> None:
+        self.unknown_type = unknown_type or NEAT.UnknownType
 
-                                ?s a <{subject_uri}> .
-                                ?s <{property_uri}> ?o .
-                                ?o a <{object_uri}> .
+    def _iterate_query(self) -> str:
+        query = """SELECT ?subjectType ?property
+                          (GROUP_CONCAT(DISTINCT STR(?valueType); SEPARATOR=",") AS ?valueTypes)
+
+                   WHERE {{
+                       ?s ?property ?o .
+                       ?s a ?subjectType .
+                       OPTIONAL {{ ?o a ?type }}
+
+                       # Key part to determine value type: either object, data or unknown
+                       BIND(   IF(isLiteral(?o),DATATYPE(?o),
+                               IF(BOUND(?type), ?type,
+                                               <{unknownType}>)) AS ?valueType)
+                   }}
+
+                   GROUP BY ?subjectType ?property
+                   HAVING (COUNT(DISTINCT ?valueType) > 1)"""
+
+        return query.format(unknownType=self.unknown_type)
+
+    def _count_query(self) -> str:
+        query = """SELECT (COUNT(*) AS ?tripleCount)
+                   WHERE {?s ?p ?o .}"""
+        return query
+
+    def _sub_iterate_query(self, type_: URIRef, property_: URIRef) -> str:
+        query = """ SELECT ?s ?p ?o ?valueType WHERE {{
+                           ?s a <{subject_uri}> .
+                           ?s <{property_uri}> ?o .
 
-                            }}"""
+                           OPTIONAL {{ ?o a ?type }}
 
-    _datatype_property_template: str = """SELECT ?s ?o WHERE {{
+                           BIND(<{property_uri}> AS ?p)
 
-                                ?s a <{subject_uri}> .
-                                ?s <{property_uri}> ?o .
-                                FILTER (datatype(?o) = <{object_uri}>)
+                           BIND(IF(isLiteral(?o),  DATATYPE(?o),
+                                   IF(BOUND(?type),?type,
+                                                   <{unknownType}>)) AS ?valueType)
 
-                                }}"""
+                                               }} """
 
-    _unknown_property_template: str = """SELECT ?s ?o WHERE {{
+        return query.format(unknownType=self.unknown_type, subject_uri=type_, property_uri=property_)
 
-                                ?s a <{subject_uri}> .
-                                ?s <{property_uri}> ?o .
-                                FILTER NOT EXISTS {{ ?o a ?objectType }}
-                                }}"""
+    def _iterator(self, graph) -> Iterator:
+        for type_, property_, _ in graph.query(self._iterate_query()):
+            yield from graph.query(self._sub_iterate_query(type_, property_))
 
-    def transform(self, graph: Graph) -> None:
-        # handle multi value type object properties
-        for subject_uri, property_uri, value_types in Queries(graph).multi_value_type_property():
-            for value_type_uri in value_types:
-                _args = {
-                    "subject_uri": subject_uri,
-                    "property_uri": property_uri,
-                    "object_uri": value_type_uri,
-                }
+    def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
+        row_output = RowTransformationOutput()
+        subject, old_property, object, value_type = query_result_row
 
-                # Case 1: Unknown value type
-                if value_type_uri == UNKNOWN_TYPE:
-                    iterator = graph.query(self._unknown_property_template.format(**_args))
+        new_property = URIRef(f"{old_property}_{remove_namespace_from_uri(value_type)}")
 
-                # Case 2: Datatype value type
-                elif value_type_uri.startswith(str(XSD)):
-                    iterator = graph.query(self._datatype_property_template.format(**_args))
+        row_output.add_triples.append(cast(Triple, (subject, new_property, object)))
+        row_output.remove_triples.append(cast(Triple, (subject, old_property, object)))
 
-                # Case 3: Object value type
-                else:
-                    iterator = graph.query(self._object_property_template.format(**_args))
+        row_output.instances_modified_count += 1
 
-                for s, o in iterator:  # type: ignore [misc]
-                    graph.remove((s, property_uri, o))
-                    new_property = URIRef(f"{property_uri}_{remove_namespace_from_uri(value_type_uri)}")
-                    graph.add((s, new_property, o))
+        return row_output
 
 
 class ConvertLiteral(BaseTransformerStandardised):
@@ -295,3 +305,54 @@ class ConnectionToLiteral(BaseTransformerStandardised):
         row_output.instances_modified_count += 1
 
         return row_output
+
+
+class SetNeatType(BaseTransformerStandardised):
+    description = "Set the sub type of an instance based on the property"
+
+    def __init__(
+        self, subject_type: URIRef, subject_predicate: URIRef, drop_property: bool, namespace: Namespace | None = None
+    ) -> None:
+        self.subject_type = subject_type
+        self.subject_predicate = subject_predicate
+        self.drop_property = drop_property
+        self._namespace = namespace or Namespace(get_namespace(subject_type))
+
+    def _count_query(self) -> str:
+        query = """SELECT (COUNT(?object) AS ?objectCount)
+                    WHERE {{
+                      ?instance a <{subject_type}> .
+                      ?instance <{subject_predicate}> ?object
+                      FILTER(isLiteral(?object))
+                    }}"""
+        return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
+
+    def _skip_count_query(self) -> str:
+        query = """SELECT (COUNT(?object) AS ?objectCount)
+                    WHERE {{
+                      ?instance a <{subject_type}> .
+                      ?instance <{subject_predicate}> ?object
+                      FILTER(isIRI(?object))
+                    }}"""
+        return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
+
+    def _iterate_query(self) -> str:
+        query = """SELECT ?instance ?object
+                    WHERE {{
+                      ?instance a <{subject_type}> .
+                      ?instance <{subject_predicate}> ?object
+                      FILTER(isLiteral(?object))
+                    }}"""
+        return query.format(subject_type=self.subject_type, subject_predicate=self.subject_predicate)
+
+    def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
+        row_output = RowTransformationOutput()
+
+        instance, object_literal = cast(tuple[URIRef, Literal], query_result_row)
+        if self.drop_property:
+            row_output.remove_triples.append((instance, self.subject_predicate, object_literal))
+
+        row_output.add_triples.append((instance, NEAT.type, self._namespace[str(object_literal.toPython())]))
+        row_output.instances_modified_count += 1
+
+        return row_output

cognite/neat/_issues/errors/_resources.py

@@ -30,7 +30,7 @@ class ResourceRetrievalError(ResourceError[T_Identifier]):
 
 @dataclass(unsafe_hash=True)
 class ResourceNotFoundError(ResourceError, Generic[T_Identifier, T_ReferenceIdentifier]):
-    """The {resource_type} with identifier {identifier} does not exist"""
+    """The {resource_type} with identifier '{identifier}' does not exist"""
 
     extra = " This is expected by {referred_type} {referred_by}."
 

cognite/neat/_issues/warnings/__init__.py

@@ -31,7 +31,6 @@ from ._properties import (
     PropertyDirectRelationLimitWarning,
     PropertyNotFoundWarning,
     PropertyOverwritingWarning,
-    PropertySkippedWarning,
     PropertyTypeNotSupportedWarning,
     PropertyValueTypeUndefinedWarning,
 )
@@ -68,7 +67,6 @@ __all__ = [
     "PropertyDirectRelationLimitWarning",
     "PropertyNotFoundWarning",
     "PropertyOverwritingWarning",
-    "PropertySkippedWarning",
     "PropertyTypeNotSupportedWarning",
     "PropertyValueTypeUndefinedWarning",
     "RegexViolationWarning",

cognite/neat/_issues/warnings/_models.py

@@ -74,7 +74,7 @@ class CDFNotSupportedWarning(NeatWarning, ABC):
 class NotSupportedViewContainerLimitWarning(CDFNotSupportedWarning):
     """The view {view_id} maps, {count} containers, which is more than the limit {limit}."""
 
-    fix = "Reduce the number of containers the view maps to." ""
+    fix = "Reduce the number of containers the view maps to."
 
     view_id: ViewId
     count: int

cognite/neat/_issues/warnings/_properties.py

@@ -65,14 +65,6 @@ class PropertyOverwritingWarning(PropertyWarning[T_Identifier]):
     overwriting: tuple[str, ...]
 
 
-@dataclass(unsafe_hash=True)
-class PropertySkippedWarning(PropertyWarning[T_Identifier]):
-    """The {resource_type} with identifier {identifier} has a property {property_name}
-    which is skipped. {reason}."""
-
-    reason: str
-
-
 @dataclass(unsafe_hash=True)
 class PropertyDataTypeConversionWarning(PropertyWarning[T_Identifier]):
     """The {resource_type} with identifier {identifier} failed to convert the property {property_name}: {error}"""

cognite/neat/_rules/analysis/_base.py

@@ -251,7 +251,7 @@ class BaseAnalysis(ABC, Generic[T_Rules, T_Class, T_Property, T_ClassEntity, T_P
                 if (
                     only_rdfpath
                     and isinstance(property_, InformationProperty)
-                    and isinstance(property_.transformation, RDFPath)
+                    and isinstance(property_.instance_source, RDFPath)
                 ) or not only_rdfpath:
                     processed_properties[prop_entity] = property_
             class_property_pairs[class_] = processed_properties

cognite/neat/_rules/analysis/_information.py

@@ -55,12 +55,13 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
 
     def has_hop_transformations(self):
         return any(
-            prop_.transformation and isinstance(prop_.transformation.traversal, Hop) for prop_ in self.rules.properties
+            prop_.instance_source and isinstance(prop_.instance_source.traversal, Hop)
+            for prop_ in self.rules.properties
         )
 
     def has_self_reference_property_transformations(self):
         return any(
-            prop_.transformation and isinstance(prop_.transformation.traversal, SelfReferenceProperty)
+            prop_.instance_source and isinstance(prop_.instance_source.traversal, SelfReferenceProperty)
             for prop_ in self.rules.properties
         )
 
@@ -68,7 +69,7 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
         return [
             prop_
             for prop_ in self.rules.properties
-            if prop_.transformation and isinstance(prop_.transformation.traversal, SelfReferenceProperty)
+            if prop_.instance_source and isinstance(prop_.instance_source.traversal, SelfReferenceProperty)
         ]
 
     def define_property_renaming_config(self, class_: ClassEntity) -> dict[str | URIRef, str]:
@@ -76,7 +77,7 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
 
         if definitions := self.class_property_pairs(only_rdfpath=True, consider_inheritance=True).get(class_, None):
             for property_id, definition in definitions.items():
-                transformation = cast(RDFPath, definition.transformation)
+                transformation = cast(RDFPath, definition.instance_source)
 
                 # use case we have a single property rdf path, and defined prefix
                 # in either metadata or prefixes of rules
@@ -101,26 +102,26 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
 
         return property_renaming_configuration
 
-    def neat_id_to_transformation_property_uri(self, property_neat_id: URIRef) -> URIRef | None:
+    def neat_id_to_instance_source_property_uri(self, property_neat_id: URIRef) -> URIRef | None:
         if (
             (property_ := self.properties_by_neat_id.get(property_neat_id))
-            and property_.transformation
+            and property_.instance_source
             and isinstance(
-                property_.transformation.traversal,
+                property_.instance_source.traversal,
                 SingleProperty,
             )
             and (
-                property_.transformation.traversal.property.prefix in self.rules.prefixes
-                or property_.transformation.traversal.property.prefix == self.rules.metadata.prefix
+                property_.instance_source.traversal.property.prefix in self.rules.prefixes
+                or property_.instance_source.traversal.property.prefix == self.rules.metadata.prefix
             )
         ):
             namespace = (
                 self.rules.metadata.namespace
-                if property_.transformation.traversal.property.prefix == self.rules.metadata.prefix
-                else self.rules.prefixes[property_.transformation.traversal.property.prefix]
+                if property_.instance_source.traversal.property.prefix == self.rules.metadata.prefix
+                else self.rules.prefixes[property_.instance_source.traversal.property.prefix]
             )
 
-            return namespace[property_.transformation.traversal.property.suffix]
+            return namespace[property_.instance_source.traversal.property.suffix]
         return None
 
     def property_types(self, class_: ClassEntity) -> dict[str, EntityTypes]:
@@ -137,7 +138,7 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
             class_, None
         ):
             for property_ in class_property_pairs.values():
-                classes.append(cast(RDFPath, property_.transformation).traversal.class_)
+                classes.append(cast(RDFPath, property_.instance_source).traversal.class_)
 
             return cast(ClassEntity, most_occurring_element(classes))
         else:

cognite/neat/_rules/catalog/__init__.py

@@ -5,3 +5,4 @@ from pathlib import Path
 _CATALOG = Path(__file__).parent
 imf_attributes = _CATALOG / "info-rules-imf.xlsx"
 hello_world_pump = _CATALOG / "hello_world_pump.xlsx"
+classic_model = _CATALOG / "classic_model.xlsx"

cognite/neat/_rules/exporters/_rules2instance_template.py

@@ -96,13 +96,13 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
 def _add_index_identifiers(workbook: Workbook, sheet: str, no_rows: int):
     """Adds index-based auto identifier to a sheet identifier column"""
     for i in range(no_rows):
-        workbook[sheet][f"A{i+2}"] = f'=IF(ISBLANK(B{i+2}), "","{sheet}-{i+1}")'
+        workbook[sheet][f"A{i + 2}"] = f'=IF(ISBLANK(B{i + 2}), "","{sheet}-{i + 1}")'
 
 
 def _add_uuid_identifiers(workbook: Workbook, sheet: str, no_rows: int):
     """Adds UUID-based auto identifier to a sheet identifier column"""
     for i in range(no_rows):
-        workbook[sheet][f"A{i+2}"] = f'=IF(ISBLANK(B{i+2}), "","{sheet}-{uuid.uuid4()}")'
+        workbook[sheet][f"A{i + 2}"] = f'=IF(ISBLANK(B{i + 2}), "","{sheet}-{uuid.uuid4()}")'
 
 
 def _add_drop_down_list(
@@ -122,7 +122,7 @@ def _add_drop_down_list(
     workbook[sheet].add_data_validation(drop_down_list)
 
     for i in range(no_rows):
-        drop_down_list.add(workbook[sheet][f"{column}{i+2}"])
+        drop_down_list.add(workbook[sheet][f"{column}{i + 2}"])
 
 
 def _adjust_column_width(workbook: Workbook):

cognite/neat/_rules/importers/__init__.py

@@ -1,7 +1,7 @@
 from ._base import BaseImporter
 from ._dms2rules import DMSImporter
 from ._dtdl2rules import DTDLImporter
-from ._rdf import IMFImporter, InferenceImporter, OWLImporter
+from ._rdf import IMFImporter, InferenceImporter, OWLImporter, SubclassInferenceImporter
 from ._spreadsheet2rules import ExcelImporter, GoogleSheetImporter
 from ._yaml2rules import YAMLImporter
 
@@ -14,6 +14,7 @@ __all__ = [
     "IMFImporter",
     "InferenceImporter",
     "OWLImporter",
+    "SubclassInferenceImporter",
     "YAMLImporter",
 ]
 
@@ -26,6 +27,7 @@ RulesImporters = (
     | DTDLImporter
     | YAMLImporter
     | InferenceImporter
+    | SubclassInferenceImporter
 )
 
 

cognite/neat/_rules/importers/_dms2rules.py

@@ -115,8 +115,8 @@ class DMSImporter(BaseImporter[DMSInputRules]):
         data_model_ids = [data_model_id]
         data_models = client.data_modeling.data_models.retrieve(data_model_ids, inline_views=True)
 
-        user_models = cls._find_model_in_list(data_models, data_model_id)
-        if len(user_models) == 0:
+        retrieved_models = cls._find_model_in_list(data_models, data_model_id)
+        if len(retrieved_models) == 0:
             return cls(
                 DMSSchema(),
                 [
@@ -127,16 +127,18 @@ class DMSImporter(BaseImporter[DMSInputRules]):
                     )
                 ],
             )
-        user_model = user_models.latest_version()
+        return cls.from_data_model(client, retrieved_models.latest_version())
 
+    @classmethod
+    def from_data_model(cls, client: NeatClient, model: dm.DataModel[dm.View]) -> "DMSImporter":
         issue_list = IssueList()
         with _handle_issues(issue_list) as result:
-            schema = NeatClient(client).schema.retrieve_data_model(user_model)
+            schema = client.schema.retrieve_data_model(model)
 
         if result.result == "failure" or issue_list.has_errors:
             return cls(DMSSchema(), issue_list)
 
-        metadata = cls._create_metadata_from_model(user_model)
+        metadata = cls._create_metadata_from_model(model)
 
         return cls(
             schema,

cognite/neat/_rules/importers/_dtdl2rules/spec.py

@@ -302,8 +302,7 @@ class Interface(DTDLBase):
     spec_version = frozenset(["2", "3"])
     default_context: ClassVar[IRI] = Field(
         "dtmi:dtdl:context;3",
-        description="This can be set directly on the class to change the "
-        "default context used when parsing a document.",
+        description="This can be set directly on the class to change the default context used when parsing a document.",
     )
     id_: DTMI = Field(alias="@id")  # type: ignore[assignment]
     context: IRI | None = Field(alias="@context")

cognite/neat/_rules/importers/_rdf/__init__.py

@@ -1,5 +1,5 @@
 from ._imf2rules import IMFImporter
-from ._inference2rules import InferenceImporter
+from ._inference2rules import InferenceImporter, SubclassInferenceImporter
 from ._owl2rules import OWLImporter
 
-__all__ = ["IMFImporter", "InferenceImporter", "OWLImporter"]
+__all__ = ["IMFImporter", "InferenceImporter", "OWLImporter", "SubclassInferenceImporter"]

cognite/neat/_rules/importers/_rdf/_base.py

@@ -72,7 +72,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
     ):
         return cls(
             IssueList(title=f"{cls.__name__} issues"),
-            store.graph,
+            store.dataset,
             data_model_id=data_model_id,
             max_number_of_instance=max_number_of_instance,
             non_existing_node_type=non_existing_node_type,
@@ -140,7 +140,7 @@ class BaseRDFImporter(BaseImporter[InformationInputRules]):
             prefixes: Dict of prefixes and namespaces
         """
         if Namespace(get_namespace(URI)) not in prefixes.values():
-            prefixes[f"prefix_{len(prefixes)+1}"] = Namespace(get_namespace(URI))
+            prefixes[f"prefix_{len(prefixes) + 1}"] = Namespace(get_namespace(URI))
 
     @property
     def _metadata(self) -> dict: