cognite-neat 0.123.2__py3-none-any.whl → 0.127.30__py3-none-any.whl

cognite/neat/{core/_data_model/exporters/_data_model2ontology.py → v0/core/_data_model/exporters/_data_model2semantic_model.py}

@@ -8,24 +8,22 @@ from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator
 from rdflib import DCTERMS, OWL, RDF, RDFS, XSD, BNode, Graph, Literal, Namespace, URIRef
 from rdflib.collection import Collection as GraphCollection
 
-from cognite.neat.core._constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
-from cognite.neat.core._data_model._constants import EntityTypes
-from cognite.neat.core._data_model.analysis import DataModelAnalysis
-from cognite.neat.core._data_model.models.conceptual import (
+from cognite.neat.v0.core._constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
+from cognite.neat.v0.core._data_model._constants import EntityTypes
+from cognite.neat.v0.core._data_model.analysis import DataModelAnalysis
+from cognite.neat.v0.core._data_model.models.conceptual import (
     Concept,
     ConceptualDataModel,
     ConceptualMetadata,
     ConceptualProperty,
 )
-from cognite.neat.core._data_model.models.conceptual._validation import duplicated_properties
-from cognite.neat.core._data_model.models.data_types import DataType
-from cognite.neat.core._data_model.models.entities import ConceptEntity
-from cognite.neat.core._issues import MultiValueError
-from cognite.neat.core._issues.errors import (
+from cognite.neat.v0.core._data_model.models.data_types import DataType
+from cognite.neat.v0.core._data_model.models.entities import ConceptEntity
+from cognite.neat.v0.core._issues.errors import (
     PropertyDefinitionDuplicatedError,
 )
-from cognite.neat.core._issues.warnings import PropertyDefinitionDuplicatedWarning
-from cognite.neat.core._utils.rdf_ import remove_namespace_from_uri
+from cognite.neat.v0.core._issues.warnings import PropertyDefinitionDuplicatedWarning
+from cognite.neat.v0.core._utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExporter
 
@@ -34,6 +32,8 @@ if sys.version_info >= (3, 11):
 else:
     from typing_extensions import Self
 
+SHACL = Namespace("http://www.w3.org/ns/shacl#")
+
 
 class GraphExporter(BaseExporter[ConceptualDataModel, Graph], ABC):
     def export_to_file(self, data_model: ConceptualDataModel, filepath: Path) -> None:
@@ -44,7 +44,7 @@ class OWLExporter(GraphExporter):
     """Exports verified conceptual data model to an OWL ontology."""
 
     def export(self, data_model: ConceptualDataModel) -> Graph:
-        return Ontology.from_data_model(data_model).as_owl()
+        return Ontology.from_data_model(data_model).graph
 
     @property
     def description(self) -> str:
@@ -55,44 +55,31 @@ class SHACLExporter(GraphExporter):
     """Exports data_model to a SHACL graph."""
 
     def export(self, data_model: ConceptualDataModel) -> Graph:
-        return Ontology.from_data_model(data_model).as_shacl()
-
-    @property
-    def description(self) -> str:
-        return "Export verified information model to SHACL."
-
-
-class SemanticDataModelExporter(GraphExporter):
-    """Exports verified information model to a semantic data model."""
-
-    def export(self, data_model: ConceptualDataModel) -> Graph:
-        return Ontology.from_data_model(data_model).as_semantic_data_model()
+        return ShaclShapes.from_data_model(data_model).graph
 
     @property
     def description(self) -> str:
-        return "Export verified information model to a semantic data model."
+        return "Export verified conceptual data model to SHACL."
 
 
-class OntologyModel(BaseModel):
+class _ModelConfig(BaseModel):
     model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True, strict=False, extra="allow")
 
 
-class Ontology(OntologyModel):
+class Ontology(_ModelConfig):
     """
     Represents an ontology. This class is used to generate an OWL ontology from conceptual data model.
 
     Args:
         properties: A list of OWL properties.
         classes: A list of OWL classes.
-        shapes: A list of SHACL node shapes.
         metadata: Metadata about the ontology.
         prefixes: A dictionary of prefixes and namespaces.
     """
 
+    metadata: "OWLMetadata"
     properties: list["OWLProperty"]
     classes: list["OWLClass"]
-    shapes: list["SHACLNodeShape"]
-    metadata: "OWLMetadata"
     prefixes: dict[str, Namespace]
 
     @classmethod
@@ -106,23 +93,7 @@ class Ontology(OntologyModel):
         Returns:
             An instance of Ontology.
         """
-        if duplicates := duplicated_properties(data_model.properties):
-            errors = []
-            for (concept, property_), definitions in duplicates.items():
-                errors.append(
-                    PropertyDefinitionDuplicatedError(
-                        concept,
-                        "concept",
-                        property_,
-                        frozenset({str(definition[1].value_type) for definition in definitions}),
-                        tuple(definition[0] for definition in definitions),
-                        "rows",
-                    )
-                )
-            raise MultiValueError(errors)
-
         analysis = DataModelAnalysis(data_model)
-        concept_by_suffix = analysis.concept_by_suffix()
         return cls(
             properties=[
                 OWLProperty.from_list_of_properties(definition, data_model.metadata.namespace)
@@ -132,46 +103,12 @@ class Ontology(OntologyModel):
                 OWLClass.from_concept(definition, data_model.metadata.namespace, data_model.prefixes)
                 for definition in data_model.concepts
             ],
-            shapes=[
-                SHACLNodeShape.from_data_model(
-                    concept_by_suffix[str(concept.suffix)],
-                    list(properties.values()),
-                    data_model.metadata.namespace,
-                )
-                for concept, properties in analysis.properties_by_id_by_concept().items()
-            ]
-            + [
-                SHACLNodeShape.from_data_model(
-                    concept,
-                    [],
-                    data_model.metadata.namespace,
-                )
-                for concept in concept_by_suffix.values()
-            ],
             metadata=OWLMetadata(**data_model.metadata.model_dump()),
             prefixes=data_model.prefixes,
         )
 
-    def as_shacl(self) -> Graph:
-        """
-        Generates a SHACL graph from the ontology.
-
-        Returns:
-            A SHACL graph.
-        """
-
-        shacl = Graph()
-        shacl.bind(self.metadata.prefix, self.metadata.namespace)
-        for prefix, namespace in self.prefixes.items():
-            shacl.bind(prefix, namespace)
-
-        for shape in self.shapes:
-            for triple in shape.triples:
-                shacl.add(triple)  # type: ignore[arg-type]
-
-        return shacl
-
-    def as_owl(self) -> Graph:
+    @property
+    def graph(self) -> Graph:
         """
         Generates an OWL graph from the ontology.
 
@@ -197,33 +134,6 @@ class Ontology(OntologyModel):
 
         return owl
 
-    def as_semantic_data_model(self) -> Graph:
-        return self.as_owl() + self.as_shacl()
-
-    @property
-    def owl_triples(self) -> list[tuple]:
-        return list(self.as_owl())
-
-    @property
-    def shacl_triples(self) -> list[tuple]:
-        return list(self.as_shacl())
-
-    @property
-    def triples(self) -> list[tuple]:
-        return self.owl_triples + self.shacl_triples
-
-    @property
-    def ontology(self) -> str:
-        return self.as_owl().serialize()
-
-    @property
-    def constraints(self) -> str:
-        return self.as_shacl().serialize()
-
-    @property
-    def semantic_data_model(self) -> str:
-        return (self.as_owl() + self.as_shacl()).serialize()
-
 
 class OWLMetadata(ConceptualMetadata):
     @property
@@ -250,7 +160,7 @@ class OWLMetadata(ConceptualMetadata):
         return triples
 
 
-class OWLClass(OntologyModel):
+class OWLClass(_ModelConfig):
     id_: URIRef
     type_: URIRef = OWL.Class
     label: str | None
@@ -323,7 +233,7 @@ class OWLClass(OntologyModel):
         )
 
 
-class OWLProperty(OntologyModel):
+class OWLProperty(_ModelConfig):
     id_: URIRef
     type_: set[URIRef]
     label: set[str]
@@ -527,10 +437,78 @@ class OWLProperty(OntologyModel):
         )
 
 
-SHACL = Namespace("http://www.w3.org/ns/shacl#")
+class ShaclShapes(_ModelConfig):
+    """
+    Represents a SHACL shapes. This class is used to generate a SHACL graph from conceptual data model.
+
+    Args:
+        shapes: A list of SHACL node shapes.
+        prefixes: A dictionary of prefixes and namespaces.
+    """
+
+    shapes: list["SHACLNodeShape"]
+    prefixes: dict[str, Namespace]
+
+    @classmethod
+    def from_data_model(cls, data_model: ConceptualDataModel) -> Self:
+        """
+        Generates shacl shapes from a conceptual data model.
+
+        Args:
+            data_model: The data_model to generate the shacl shapes from.
+
+        Returns:
+            An instance of ShaclShapes.
+        """
+        analysis = DataModelAnalysis(data_model)
+        concepts_by_concept_entity = analysis.concept_by_concept_entity
+        properties_by_concept_entity = analysis.properties_by_concepts()
+        return cls(
+            shapes=[
+                # shapes that have property shapes as well
+                SHACLNodeShape.from_concept(
+                    concept_entity,
+                    concepts_by_concept_entity,
+                    properties,
+                    data_model.metadata.namespace,
+                )
+                for concept_entity, properties in properties_by_concept_entity.items()
+            ]
+            + [
+                # shapes without any property shapes
+                SHACLNodeShape.from_concept(
+                    concept_entity,
+                    concepts_by_concept_entity,
+                    [],
+                    data_model.metadata.namespace,
+                )
+                for concept_entity in concepts_by_concept_entity.keys()
+                if concept_entity not in properties_by_concept_entity
+            ],
+            prefixes=data_model.prefixes,
+        )
+
+    @property
+    def graph(self) -> Graph:
+        """
+        Generates a SHACL graph from the class instance.
 
+        Returns:
+            A SHACL graph.
+        """
 
-class SHACLNodeShape(OntologyModel):
+        shacl = Graph()
+        for prefix, namespace in self.prefixes.items():
+            shacl.bind(prefix, namespace)
+
+        for shape in self.shapes:
+            for triple in shape.triples:
+                shacl.add(triple)  # type: ignore[arg-type]
+
+        return shacl
+
+
+class SHACLNodeShape(_ModelConfig):
     id_: URIRef
     type_: URIRef = SHACL.NodeShape
     target_class: URIRef
@@ -572,26 +550,33 @@ class SHACLNodeShape(OntologyModel):
         )
 
     @classmethod
-    def from_data_model(
+    def from_concept(
         cls,
-        concept_definition: Concept,
-        property_definitions: list[ConceptualProperty],
+        concept_entity: ConceptEntity,
+        concepts_by_concept_entity: dict[ConceptEntity, Concept],
+        properties: list[ConceptualProperty],
         namespace: Namespace,
     ) -> "SHACLNodeShape":
-        if concept_definition.implements:
-            parent = [namespace[str(parent.suffix) + "Shape"] for parent in concept_definition.implements]
+        if not (concept := concepts_by_concept_entity.get(concept_entity)):
+            raise ValueError(f"Concept {concept_entity} not found in data model!")
+
+        if concept.implements:
+            parent = [namespace[str(parent.suffix) + "Shape"] for parent in concept.implements]
         else:
             parent = None
         return cls(
-            id_=namespace[f"{concept_definition.concept.suffix!s}Shape"],
-            target_class=namespace[str(concept_definition.concept.suffix)],
+            id_=namespace[f"{concept.concept.suffix!s}Shape"],
+            target_class=concept.instance_source or namespace[str(concept.concept.suffix)],
             parent=parent,
-            property_shapes=[SHACLPropertyShape.from_property(prop, namespace) for prop in property_definitions],
+            property_shapes=[
+                SHACLPropertyShape.from_property(property_, concepts_by_concept_entity, namespace)
+                for property_ in properties
+            ],
             namespace=namespace,
         )
 
 
-class SHACLPropertyShape(OntologyModel):
+class SHACLPropertyShape(_ModelConfig):
     id_: BNode
     type_: URIRef = SHACL.property
     path: URIRef  # URIRef to property in OWL
@@ -631,23 +616,31 @@ class SHACLPropertyShape(OntologyModel):
         return self.path_triples + self.node_kind_triples + self.cardinality_triples
 
     @classmethod
-    def from_property(cls, definition: ConceptualProperty, namespace: Namespace) -> "SHACLPropertyShape":
-        # TODO requires PR to fix MultiValueType and UnknownValueType
-        if isinstance(definition.value_type, ConceptEntity):
-            expected_value_type = namespace[f"{definition.value_type.suffix}Shape"]
-        elif isinstance(definition.value_type, DataType):
-            expected_value_type = XSD[definition.value_type.xsd]
+    def from_property(
+        cls,
+        property_: ConceptualProperty,
+        concepts_by_concept_entity: dict[ConceptEntity, Concept],
+        namespace: Namespace,
+    ) -> "SHACLPropertyShape":
+        if isinstance(property_.value_type, ConceptEntity):
+            concept = concepts_by_concept_entity.get(property_.value_type)
+            value_type_uri = concept.instance_source if concept else None
+            expected_value_type = value_type_uri or namespace[f"{property_.value_type.suffix}"]
+        elif isinstance(property_.value_type, DataType):
+            expected_value_type = XSD[property_.value_type.xsd]
         else:
-            raise ValueError(f"Value type {definition.value_type.type_} is not supported")
+            raise NotImplementedError(f"Value type {property_.value_type.type_} is not supported yet")
 
         return cls(
             id_=BNode(),
-            path=namespace[definition.property_],
-            node_kind=SHACL.IRI if definition.type_ == EntityTypes.object_property else SHACL.Literal,
+            path=property_.instance_source[0]
+            if property_.instance_source and len(property_.instance_source) == 1
+            else namespace[property_.property_],
+            node_kind=SHACL.IRI if property_.type_ == EntityTypes.object_property else SHACL.Literal,
             expected_value_type=expected_value_type,
-            min_count=definition.min_count,
+            min_count=property_.min_count,
             max_count=(
-                int(definition.max_count) if definition.max_count and definition.max_count != float("inf") else None
+                int(property_.max_count) if property_.max_count and property_.max_count != float("inf") else None
             ),
             namespace=namespace,
         )

cognite/neat/{core → v0/core}/_data_model/exporters/_data_model2yaml.py

@@ -5,7 +5,7 @@ from typing import Literal, get_args
 
 import yaml
 
-from cognite.neat.core._data_model._shared import VerifiedDataModel
+from cognite.neat.v0.core._data_model._shared import VerifiedDataModel
 
 from ._base import BaseExporter
 

cognite/neat/{core → v0/core}/_data_model/importers/__init__.py

@@ -1,17 +1,16 @@
 from ._base import BaseImporter
 from ._dict2data_model import DictImporter
 from ._dms2data_model import DMSImporter
-from ._dtdl2data_model import DTDLImporter
-from ._rdf import IMFImporter, InferenceImporter, OWLImporter, SubclassInferenceImporter
+from ._graph2data_model import GraphImporter
+from ._rdf import InferenceImporter, OWLImporter, SubclassInferenceImporter
 from ._spreadsheet2data_model import ExcelImporter
 
 __all__ = [
     "BaseImporter",
     "DMSImporter",
-    "DTDLImporter",
     "DictImporter",
     "ExcelImporter",
-    "IMFImporter",
+    "GraphImporter",
     "InferenceImporter",
     "OWLImporter",
     "SubclassInferenceImporter",
@@ -19,13 +18,12 @@ __all__ = [
 
 DataModelImporters = (
     OWLImporter
-    | IMFImporter
     | DMSImporter
     | ExcelImporter
-    | DTDLImporter
     | DictImporter
     | InferenceImporter
     | SubclassInferenceImporter
+    | GraphImporter
 )
 
 

cognite/neat/{core → v0/core}/_data_model/importers/_base.py

@@ -5,15 +5,15 @@ from typing import TYPE_CHECKING, Any, Generic
 
 from rdflib import URIRef
 
-from cognite.neat.core._constants import DEFAULT_NAMESPACE
-from cognite.neat.core._data_model._shared import (
+from cognite.neat.v0.core._constants import DEFAULT_NAMESPACE
+from cognite.neat.v0.core._data_model._shared import (
     ImportedDataModel,
     T_UnverifiedDataModel,
 )
-from cognite.neat.core._utils.auxiliary import class_html_doc
+from cognite.neat.v0.core._utils.auxiliary import class_html_doc
 
 if TYPE_CHECKING:
-    from cognite.neat.core._store._provenance import Agent as ProvenanceAgent
+    from cognite.neat.v0.core._store._provenance import Agent as ProvenanceAgent
 
 
 class BaseImporter(ABC, Generic[T_UnverifiedDataModel]):
@@ -51,7 +51,7 @@ class BaseImporter(ABC, Generic[T_UnverifiedDataModel]):
     @property
     def agent(self) -> "ProvenanceAgent":
         """Provenance agent for the importer."""
-        from cognite.neat.core._store._provenance import Agent as ProvenanceAgent
+        from cognite.neat.v0.core._store._provenance import Agent as ProvenanceAgent
 
         return ProvenanceAgent(id_=DEFAULT_NAMESPACE[f"agent/{type(self).__name__}"])
 

cognite/neat/{core → v0/core}/_data_model/importers/_base_file_reader.py

@@ -1,8 +1,8 @@
 from pathlib import Path
 from typing import Protocol, TypeVar
 
-from cognite.neat.core._issues import NeatIssue
-from cognite.neat.core._issues.errors import (
+from cognite.neat.v0.core._issues import NeatIssue
+from cognite.neat.v0.core._issues.errors import (
     FileNotAFileError,
     FileNotFoundNeatError,
     FileTypeUnexpectedError,

cognite/neat/{core → v0/core}/_data_model/importers/_dict2data_model.py

@@ -3,20 +3,20 @@ from typing import Any, cast
 
 import yaml
 
-from cognite.neat.core._data_model._shared import (
+from cognite.neat.v0.core._data_model._shared import (
     ImportedDataModel,
     T_UnverifiedDataModel,
 )
-from cognite.neat.core._data_model.models import (
+from cognite.neat.v0.core._data_model.models import (
     UNVERIFIED_DATA_MODEL_BY_ROLE,
     RoleTypes,
 )
-from cognite.neat.core._issues import IssueList, MultiValueError, NeatIssue
-from cognite.neat.core._issues.errors import (
+from cognite.neat.v0.core._issues import IssueList, MultiValueError, NeatIssue
+from cognite.neat.v0.core._issues.errors import (
     FileMissingRequiredFieldError,
     FileReadError,
 )
-from cognite.neat.core._issues.warnings import NeatValueWarning
+from cognite.neat.v0.core._issues.warnings import NeatValueWarning
 
 from ._base import BaseImporter
 from ._base_file_reader import BaseFileReader
@@ -128,4 +128,4 @@ class DictImporter(BaseImporter[T_UnverifiedDataModel]):
         if self._read_issues.has_errors:
             raise MultiValueError(self._read_issues.errors)
 
-        return ImportedDataModel[T_UnverifiedDataModel](data_model, {})
+        return ImportedDataModel[T_UnverifiedDataModel](data_model)

cognite/neat/{core → v0/core}/_data_model/importers/_dms2data_model.py

@@ -28,23 +28,23 @@ from cognite.client.data_classes.data_modeling.views import (
 )
 from cognite.client.utils import ms_to_datetime
 
-from cognite.neat.core._client import NeatClient
-from cognite.neat.core._constants import (
+from cognite.neat.v0.core._client import NeatClient
+from cognite.neat.v0.core._constants import (
     DMS_DIRECT_RELATION_LIST_DEFAULT_LIMIT,
     DMS_PRIMITIVE_LIST_DEFAULT_LIMIT,
 )
-from cognite.neat.core._data_model._shared import ImportedDataModel
-from cognite.neat.core._data_model.importers._base import BaseImporter
-from cognite.neat.core._data_model.models import (
+from cognite.neat.v0.core._data_model._shared import ImportedDataModel
+from cognite.neat.v0.core._data_model.importers._base import BaseImporter
+from cognite.neat.v0.core._data_model.models import (
     DMSSchema,
     UnverifiedPhysicalDataModel,
 )
-from cognite.neat.core._data_model.models.conceptual import (
+from cognite.neat.v0.core._data_model.models.conceptual import (
     UnverifiedConcept,
     UnverifiedConceptualProperty,
 )
-from cognite.neat.core._data_model.models.data_types import DataType, Enum, String
-from cognite.neat.core._data_model.models.entities import (
+from cognite.neat.v0.core._data_model.models.data_types import DataType, Enum, String
+from cognite.neat.v0.core._data_model.models.entities import (
     ConceptEntity,
     ContainerEntity,
     ContainerIndexEntity,
@@ -54,7 +54,8 @@ from cognite.neat.core._data_model.models.entities import (
     ReverseConnectionEntity,
     ViewEntity,
 )
-from cognite.neat.core._data_model.models.physical import (
+from cognite.neat.v0.core._data_model.models.entities._single_value import ContainerConstraintEntity
+from cognite.neat.v0.core._data_model.models.physical import (
     UnverifiedPhysicalContainer,
     UnverifiedPhysicalEnum,
     UnverifiedPhysicalMetadata,
@@ -62,13 +63,13 @@ from cognite.neat.core._data_model.models.physical import (
     UnverifiedPhysicalProperty,
     UnverifiedPhysicalView,
 )
-from cognite.neat.core._issues import (
+from cognite.neat.v0.core._issues import (
     IssueList,
     MultiValueError,
     NeatIssue,
     catch_issues,
 )
-from cognite.neat.core._issues.errors import (
+from cognite.neat.v0.core._issues.errors import (
     FileNotFoundNeatError,
     FileTypeUnexpectedError,
     NeatValueError,
@@ -76,7 +77,7 @@ from cognite.neat.core._issues.errors import (
     ResourceMissingIdentifierError,
     ResourceRetrievalError,
 )
-from cognite.neat.core._issues.warnings import (
+from cognite.neat.v0.core._issues.warnings import (
     MissingCogniteClientWarning,
     NeatValueWarning,
     PropertyNotFoundWarning,
@@ -276,7 +277,7 @@ class DMSImporter(BaseImporter[UnverifiedPhysicalDataModel]):
         self.issue_list.trigger_warnings()
         if self.issue_list.has_errors:
             raise MultiValueError(self.issue_list.errors)
-        return ImportedDataModel(user_data_model, {})
+        return ImportedDataModel(user_data_model, None)
 
     def _create_data_model_components(
         self,
@@ -446,6 +447,8 @@ class DMSImporter(BaseImporter[UnverifiedPhysicalDataModel]):
                     return ViewEntity.from_id(prop.source)
             elif isinstance(prop_type, PropertyTypeWithUnit) and prop_type.unit:
                 return DataType.load(f"{prop_type._type}(unit={prop_type.unit.external_id})")
+            elif isinstance(prop_type, dm.Text) and prop_type.max_text_size is not None:
+                return DataType.load(f"{prop_type._type}(maxTextSize={prop_type.max_text_size})")
             elif isinstance(prop_type, DMSEnum):
                 if enum_collection_by_container_property is None:
                     return String()
@@ -569,17 +572,17 @@ class DMSImporter(BaseImporter[UnverifiedPhysicalDataModel]):
                 index.append(ContainerIndexEntity(prefix="inverted", suffix=index_name, order=order))
         return index or None
 
-    def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[str] | None:
+    def _get_constraint(self, prop: ViewPropertyApply, prop_id: str) -> list[ContainerConstraintEntity] | None:
         if not isinstance(prop, dm.MappedPropertyApply):
             return None
         container = self._all_containers_by_id[prop.container]
-        unique_constraints: list[str] = []
+        unique_constraints: list[ContainerConstraintEntity] = []
         for constraint_name, constraint_obj in (container.constraints or {}).items():
             if isinstance(constraint_obj, dm.RequiresConstraint):
                 # This is handled in the .from_container method of DMSContainer
                 continue
             elif isinstance(constraint_obj, dm.UniquenessConstraint) and prop_id in constraint_obj.properties:
-                unique_constraints.append(constraint_name)
+                unique_constraints.append(ContainerConstraintEntity(prefix="uniqueness", suffix=constraint_name))
             elif isinstance(constraint_obj, dm.UniquenessConstraint):
                 # This does not apply to this property
                 continue