cognite-neat 0.72.1__tar.gz → 0.72.3__tar.gz

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.72.1
+Version: 0.72.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.72.3/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.72.3"

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/app/api/routers/core.py

@@ -1,11 +1,13 @@
 import json
 import shutil
 import tempfile
+from copy import deepcopy
 from pathlib import Path
 from typing import cast
 
 from fastapi import APIRouter, UploadFile
 
+from cognite.neat.app.api.configuration import NEAT_APP
 from cognite.neat.rules import exporters, importers
 from cognite.neat.rules.models._rules import DMSRules
 from cognite.neat.rules.models._rules.base import RoleTypes
@@ -55,8 +57,36 @@ async def convert_data_model_to_rules(file: UploadFile):
 @router.post("/api/core/rules2dms")
 async def convert_rules_to_dms(rules: DMSRules):
     dms_schema = exporters.DMSExporter().export(rules)
+    containers = {f"{container.space}:{container.external_id}": container.dump() for container in dms_schema.containers}
+    views = {f"{view.space}:{view.external_id}": view.dump() for view in dms_schema.views}
+
+    if views and containers:
+        _to_visualization_compliant_views(views, containers)
 
     return {
-        "views": dms_schema.views.dump() if dms_schema.views else None,
-        "containers": dms_schema.containers.dump() if dms_schema.containers else None,
+        "views": list(views.values()) if views else None,
+        "containers": list(containers.values()) if containers else None,
     }
+
+
+def _to_visualization_compliant_views(views, containers):
+    for view in views.values():
+        for property in view["properties"].values():
+            # needs coping information from container:
+            if property.get("container", None) and property["container"]["type"] == "container":
+                container_id = f"{property['container']['space']}:{property['container']['externalId']}"
+                container_property_def = deepcopy(
+                    containers[container_id]["properties"][property["containerPropertyIdentifier"]]
+                )
+                property["type"] = container_property_def["type"]
+                container_property_def.pop("type")
+                property.update(container_property_def)
+
+
+@router.post("/api/core/publish-rules")
+async def publish_rules_as_data_model(rules: DMSRules):
+    if NEAT_APP.cdf_client:
+        uploaded = exporters.DMSExporter().export_to_cdf(rules, NEAT_APP.cdf_client)
+        return {"uploaded": uploaded}
+    else:
+        return {"uploaded": []}

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/constants.py

@@ -11,6 +11,7 @@ EXAMPLE_RULES = PACKAGE_DIRECTORY / "rules" / "examples"
 EXAMPLE_GRAPHS = PACKAGE_DIRECTORY / "graph" / "examples"
 EXAMPLE_WORKFLOWS = PACKAGE_DIRECTORY / "workflows" / "examples"
 
+DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/neat#")
 
 PREFIXES = {
     "rdf": RDF._NS,
@@ -27,10 +28,10 @@ PREFIXES = {
     "md": Namespace("http://iec.ch/TC57/61970-552/ModelDescription/1#"),
     "pti": Namespace("http://www.pti-us.com/PTI_CIM-schema-cim16#"),
     "tnt": Namespace("http://purl.org/cognite/tnt#"),
-    "neat": Namespace("http://purl.org/cognite/neat#"),
+    "neat": DEFAULT_NAMESPACE,
 }
 
-DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/app#")
+
 DEFAULT_URI = ""
 
 DEFAULT_DOCS_URL = "https://cognite-neat.readthedocs-hosted.com/en/latest/"

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/rules/exporters/_rules2dms.py

@@ -40,8 +40,6 @@ class DMSExporter(CDFExporter[DMSSchema]):
             If set, only export components in the given spaces. Defaults to None which means all spaces.
         existing_handling (Literal["fail", "skip", "update", "force"], optional): How to handle existing components.
             Defaults to "update". See below for details.
-        standardize_casing(bool, optional): Whether to standardize the casing. This means PascalCase for external ID
-            of views, containers, and data models, and camelCase for properties.
         export_pipeline (bool, optional): Whether to export the pipeline. Defaults to False. This means setting
             up transformations, RAW databases and tables to populate the data model.
         instance_space (str, optional): The space to use for the instance. Defaults to None.
@@ -59,14 +57,12 @@ class DMSExporter(CDFExporter[DMSSchema]):
         export_components: Component | Collection[Component] = "all",
         include_space: set[str] | None = None,
         existing_handling: Literal["fail", "skip", "update", "force"] = "update",
-        standardize_casing: bool = True,
         export_pipeline: bool = False,
         instance_space: str | None = None,
     ):
         self.export_components = {export_components} if isinstance(export_components, str) else set(export_components)
         self.include_space = include_space
         self.existing_handling = existing_handling
-        self.standardize_casing = standardize_casing
         self.export_pipeline = export_pipeline
         self.instance_space = instance_space
         self._schema: DMSSchema | None = None
@@ -119,11 +115,11 @@ class DMSExporter(CDFExporter[DMSSchema]):
         )
         is_new_model = dms_rules.reference is None
         if is_new_model or is_solution_model:
-            return dms_rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
+            return dms_rules.as_schema(self.export_pipeline, self.instance_space)
 
         # This is an extension of an existing model.
         reference_rules = cast(DMSRules, dms_rules.reference).copy(deep=True)
-        reference_schema = reference_rules.as_schema(self.standardize_casing, self.export_pipeline)
+        reference_schema = reference_rules.as_schema(self.export_pipeline)
 
         # Todo Move this to an appropriate location
         # Merging Reference with User Rules
@@ -146,7 +142,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
                 property_.reference = None
                 combined_rules.properties.append(property_)
 
-        schema = combined_rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
+        schema = combined_rules.as_schema(self.export_pipeline, self.instance_space)
 
         if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
             # We do not freeze views as they might be changed, even for addition,

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/rules/exporters/_rules2ontology.py

@@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator
 from rdflib import DCTERMS, OWL, RDF, RDFS, XSD, BNode, Graph, Literal, Namespace, URIRef
 from rdflib.collection import Collection as GraphCollection
 
+from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
 from cognite.neat.rules import exceptions
 from cognite.neat.rules._analysis._information_rules import InformationArchitectRulesAnalysis
 from cognite.neat.rules.models._rules import DMSRules
@@ -217,6 +218,7 @@ class OWLMetadata(InformationMetadata):
             (URIRef(self.namespace), DCTERMS.hasVersion, Literal(self.version)),
             (URIRef(self.namespace), OWL.versionInfo, Literal(self.version)),
             (URIRef(self.namespace), RDFS.label, Literal(self.name)),
+            (URIRef(self.namespace), NEAT_NAMESPACE.prefix, Literal(self.prefix)),
             (URIRef(self.namespace), DCTERMS.title, Literal(self.name)),
             (URIRef(self.namespace), DCTERMS.created, Literal(self.created, datatype=XSD.dateTime)),
             (URIRef(self.namespace), DCTERMS.description, Literal(self.description)),

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/rules/importers/_owl2rules/_owl2metadata.py

@@ -3,6 +3,7 @@ import re
 
 from rdflib import Graph, Namespace
 
+from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.rules.models._rules.base import RoleTypes, SchemaCompleteness
 from cognite.neat.rules.models.rules import (
     prefix_compliance_regex,
@@ -29,23 +30,23 @@ def parse_owl_metadata(graph: Graph, make_compliant: bool = False) -> dict:
     """
     # TODO: Move dataframe to dict representation
 
-    query = """SELECT ?namespace ?prefix ?version ?created ?updated ?title ?description ?creator ?rights ?license
-    WHERE {
+    query = f"""SELECT ?namespace ?prefix ?version ?created ?updated ?title ?description ?creator ?rights ?license
+    WHERE {{
         ?namespace a owl:Ontology .
-        OPTIONAL {?namespace owl:versionInfo ?version }.
-        OPTIONAL {?namespace dcterms:creator ?creator }.
-        OPTIONAL {?namespace dcterms:title|rdfs:label|skos:prefLabel ?title }.
-        OPTIONAL {?namespace dcterms:modified ?updated }.
-        OPTIONAL {?namespace dcterms:created ?created }.
-        OPTIONAL {?namespace dcterms:description ?description }.
-
-        OPTIONAL {?namespace dcterms:rights|dc:rights ?rights }.
-
-        OPTIONAL {?namespace dcterms:license|dc:license ?license }.
+        OPTIONAL {{?namespace owl:versionInfo ?version }}.
+        OPTIONAL {{?namespace dcterms:creator ?creator }}.
+        OPTIONAL {{?namespace <{DEFAULT_NAMESPACE.prefix}> ?prefix }}.
+        OPTIONAL {{?namespace dcterms:title|rdfs:label|skos:prefLabel ?title }}.
+        OPTIONAL {{?namespace dcterms:modified ?updated }}.
+        OPTIONAL {{?namespace dcterms:created ?created }}.
+        OPTIONAL {{?namespace dcterms:description ?description }}.
+        OPTIONAL {{?namespace dcterms:rights|dc:rights ?rights }}.
+
+        OPTIONAL {{?namespace dcterms:license|dc:license ?license }}.
         FILTER (!isBlank(?namespace))
         FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
         FILTER (!bound(?title) || LANG(?title) = "" || LANGMATCHES(LANG(?title), "en"))
-    }
+    }}
     """
 
     results = [{item for item in sublist} for sublist in list(zip(*graph.query(query), strict=True))]
@@ -72,7 +73,6 @@ def parse_owl_metadata(graph: Graph, make_compliant: bool = False) -> dict:
     )
 
     if make_compliant:
-        raw_metadata.pop("created")
         return make_metadata_compliant(raw_metadata)
 
     return raw_metadata
@@ -176,7 +176,7 @@ def fix_date(
     if date := metadata.get(date_type, None):
         try:
             if isinstance(date, datetime.datetime):
-                pass
+                return metadata
             elif isinstance(date, datetime.date):
                 metadata[date_type] = datetime.datetime.combine(metadata[date_type], datetime.datetime.min.time())
             elif isinstance(date, str):

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/rules/importers/_owl2rules/_owl2properties.py

@@ -40,6 +40,8 @@ def parse_owl_properties(graph: Graph, make_compliant: bool = False, language: s
         FILTER (!bound(?class) || !isBlank(?class))
         FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
         FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
+        BIND(IF(bound(?minCount), ?minCount, 0) AS ?minCount)
+        BIND(IF(bound(?maxCount), ?maxCount, 1) AS ?maxCount)
     }
     """
 

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/rules/importers/_owl2rules/_owl2rules.py

@@ -38,7 +38,7 @@ class OWLImporter(BaseImporter):
 
     """
 
-    def __init__(self, owl_filepath: Path, make_compliant: bool = True):
+    def __init__(self, owl_filepath: Path, make_compliant: bool = False):
         self.owl_filepath = owl_filepath
         self.make_compliant = make_compliant
 

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -4,6 +4,7 @@ generating a list of rules based on which nodes that form the graph are made.
 """
 
 from collections import UserDict, defaultdict
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Literal, cast, overload
 
@@ -49,6 +50,12 @@ class MetadataRaw(UserDict):
     def has_schema_field(self) -> bool:
         return self.get("schema") in [schema.value for schema in SchemaCompleteness.__members__.values()]
 
+    @property
+    def schema(self) -> SchemaCompleteness | None:
+        if not self.has_schema_field:
+            return None
+        return SchemaCompleteness(self["schema"])
+
     def is_valid(self, issue_list: IssueList, filepath: Path) -> bool:
         if not self.has_role_field:
             issue_list.append(issues.spreadsheet_file.RoleMissingOrUnsupportedError(filepath))
@@ -61,6 +68,14 @@ class MetadataRaw(UserDict):
         return True
 
 
+@dataclass
+class ReadResult:
+    sheets: dict[str, dict | list]
+    read_info_by_sheet: dict[str, SpreadsheetRead]
+    role: RoleTypes
+    schema: SchemaCompleteness | None
+
+
 class SpreadsheetReader:
     def __init__(self, issue_list: IssueList, is_reference: bool = False):
         self.issue_list = issue_list
@@ -79,7 +94,7 @@ class SpreadsheetReader:
     def to_reference_sheet(cls, sheet_name: str) -> str:
         return f"Ref{sheet_name}"
 
-    def read(self, filepath: Path) -> Rules | None:
+    def read(self, filepath: Path) -> None | ReadResult:
         with pd.ExcelFile(filepath) as excel_file:
             if self.metadata_sheet_name not in excel_file.sheet_names:
                 self.issue_list.append(
@@ -95,21 +110,10 @@ class SpreadsheetReader:
                 return None
 
             sheets, read_info_by_sheet = self._read_sheets(metadata, excel_file)
-            if self.issue_list.has_errors:
-                return None
-
-            rules_cls = RULES_PER_ROLE[metadata.role]
-            with _handle_issues(
-                self.issue_list,
-                error_cls=issues.spreadsheet.InvalidSheetError,
-                error_args={"read_info_by_sheet": read_info_by_sheet},
-            ) as future:
-                rules = rules_cls.model_validate(sheets)  # type: ignore[attr-defined]
-
-            if future.result == "failure" or self.issue_list.has_errors:
+            if sheets is None or self.issue_list.has_errors:
                 return None
 
-        return rules
+            return ReadResult(sheets, read_info_by_sheet, metadata.role, metadata.schema)
 
     def _read_sheets(
         self, metadata: MetadataRaw, excel_file: ExcelFile
@@ -167,43 +171,60 @@ class ExcelImporter(BaseImporter):
         is_reference: bool = False,
     ) -> tuple[Rules | None, IssueList] | Rules:
         issue_list = IssueList(title=f"'{self.filepath.name}'")
-
         if not self.filepath.exists():
             issue_list.append(issues.spreadsheet_file.SpreadsheetNotFoundError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        user_rules: Rules | None = None
+        user_result: ReadResult | None = None
         if not is_reference:
-            user_rules = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
-            if issue_list.has_errors:
+            user_result = SpreadsheetReader(issue_list, is_reference=False).read(self.filepath)
+            if user_result is None or issue_list.has_errors:
                 return self._return_or_raise(issue_list, errors)
 
-        reference_rules: Rules | None = None
+        reference_result: ReadResult | None = None
         if is_reference or (
-            user_rules
-            and user_rules.metadata.role != RoleTypes.domain_expert
-            and cast(DMSRules | InformationRules, user_rules).metadata.schema_ == SchemaCompleteness.extended
+            user_result
+            and user_result.role != RoleTypes.domain_expert
+            and user_result.schema == SchemaCompleteness.extended
         ):
-            reference_rules = SpreadsheetReader(issue_list, is_reference=True).read(self.filepath)
+            reference_result = SpreadsheetReader(issue_list, is_reference=True).read(self.filepath)
             if issue_list.has_errors:
                 return self._return_or_raise(issue_list, errors)
 
-        if user_rules and reference_rules and user_rules.metadata.role != reference_rules.metadata.role:
+        if user_result and reference_result and user_result.role != reference_result.role:
             issue_list.append(issues.spreadsheet_file.RoleMismatchError(self.filepath))
             return self._return_or_raise(issue_list, errors)
 
-        if user_rules and reference_rules:
-            rules = user_rules
-            rules.reference = reference_rules
-        elif user_rules:
-            rules = user_rules
-        elif reference_rules:
-            rules = reference_rules
+        if user_result and reference_result:
+            user_result.sheets["reference"] = reference_result.sheets
+            sheets = user_result.sheets
+            original_role = user_result.role
+            read_info_by_sheet = user_result.read_info_by_sheet
+            read_info_by_sheet.update(reference_result.read_info_by_sheet)
+        elif user_result:
+            sheets = user_result.sheets
+            original_role = user_result.role
+            read_info_by_sheet = user_result.read_info_by_sheet
+        elif reference_result:
+            sheets = reference_result.sheets
+            original_role = reference_result.role
+            read_info_by_sheet = reference_result.read_info_by_sheet
         else:
             raise ValueError(
                 "No rules were generated. This should have been caught earlier. " f"Bug in {type(self).__name__}."
             )
 
+        rules_cls = RULES_PER_ROLE[original_role]
+        with _handle_issues(
+            issue_list,
+            error_cls=issues.spreadsheet.InvalidSheetError,
+            error_args={"read_info_by_sheet": read_info_by_sheet},
+        ) as future:
+            rules = rules_cls.model_validate(sheets)  # type: ignore[attr-defined]
+
+        if future.result == "failure" or issue_list.has_errors:
+            return self._return_or_raise(issue_list, errors)
+
         return self._to_output(
             rules,
             issue_list,

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/rules/issues/dms.py

@@ -24,6 +24,8 @@ __all__ = [
     "EmptyContainerWarning",
     "UnsupportedRelationWarning",
     "MultipleReferenceWarning",
+    "HasDataFilterOnNoPropertiesViewWarning",
+    "NodeTypeFilterOnParentViewWarning",
 ]
 
 
@@ -329,3 +331,44 @@ class MultipleReferenceWarning(DMSSchemaWarning):
         output["view_id"] = self.view_id.dump()
         output["implements"] = [view.dump() for view in self.implements]
         return output
+
+
+@dataclass(frozen=True)
+class HasDataFilterOnNoPropertiesViewWarning(DMSSchemaWarning):
+    description = "Attempting to set a HasData filter on a view without properties."
+    fix = "Add properties to the view or use a node type filter"
+    error_name: ClassVar[str] = "HasDataFilterOnNoPropertiesViewWarning"
+    view_id: dm.ViewId
+
+    def message(self) -> str:
+        return (
+            f"Cannot set hasData filter on view {self.view_id} as it does not have properties in any containers. "
+            "Using a node type filter instead."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        return output
+
+
+@dataclass(frozen=True)
+class NodeTypeFilterOnParentViewWarning(DMSSchemaWarning):
+    description = (
+        "Setting a node type filter on a parent view. This is no "
+        "recommended as parent views are typically used for multiple type of nodes."
+    )
+    fix = "Use a HasData filter instead"
+    error_name: ClassVar[str] = "NodeTypeFilterOnParentViewWarning"
+    view_id: dm.ViewId
+
+    def message(self) -> str:
+        return (
+            f"Setting a node type filter on parent view {self.view_id}. This is not recommended as "
+            "parent views are typically used for multiple types of nodes."
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["view_id"] = self.view_id.dump()
+        return output

{cognite_neat-0.72.1 → cognite_neat-0.72.3}/cognite/neat/rules/models/_rules/_types/_base.py

@@ -14,7 +14,6 @@ from cognite.neat.rules.models.rdfpath import (
     TransformationRuleType,
     parse_rule,
 )
-from cognite.neat.utils.text import to_pascal
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
@@ -197,15 +196,14 @@ class ContainerEntity(Entity):
     def from_id(cls, container_id: ContainerId) -> "ContainerEntity":
         return ContainerEntity(prefix=container_id.space, suffix=container_id.external_id)
 
-    def as_id(self, default_space: str | None, standardize_casing: bool = True) -> ContainerId:
+    def as_id(self, default_space: str | None) -> ContainerId:
         if self.space is Undefined and default_space is None:
             raise ValueError("Space is Undefined! Set default_space!")
 
-        external_id = to_pascal(self.external_id) if standardize_casing else self.external_id
         if self.space is Undefined:
-            return ContainerId(space=cast(str, default_space), external_id=external_id)
+            return ContainerId(space=cast(str, default_space), external_id=self.external_id)
         else:
-            return ContainerId(space=self.space, external_id=external_id)
+            return ContainerId(space=self.space, external_id=self.external_id)
 
 
 class ViewEntity(Entity):
@@ -233,7 +231,6 @@ class ViewEntity(Entity):
         allow_none: Literal[False] = False,
         default_space: str | None = None,
         default_version: str | None = None,
-        standardize_casing: bool = True,
     ) -> ViewId:
         ...
 
@@ -243,7 +240,6 @@ class ViewEntity(Entity):
         allow_none: Literal[True],
         default_space: str | None = None,
         default_version: str | None = None,
-        standardize_casing: bool = True,
     ) -> ViewId | None:
         ...
 
@@ -252,7 +248,6 @@ class ViewEntity(Entity):
         allow_none: bool = False,
         default_space: str | None = None,
         default_version: str | None = None,
-        standardize_casing: bool = True,
     ) -> ViewId | None:
         if self.suffix is Unknown and allow_none:
             return None
@@ -265,8 +260,7 @@ class ViewEntity(Entity):
             raise ValueError("space is required")
         if version is None:
             raise ValueError("version is required")
-        external_id = to_pascal(self.external_id) if standardize_casing else self.external_id
-        return ViewId(space=space, external_id=external_id, version=version)
+        return ViewId(space=space, external_id=self.external_id, version=version)
 
 
 class ViewPropEntity(ViewEntity):
@@ -303,14 +297,10 @@ class ViewPropEntity(ViewEntity):
             property_=prop_id.property,
         )
 
-    def as_prop_id(
-        self, default_space: str | None = None, default_version: str | None = None, standardize_casing: bool = True
-    ) -> PropertyId:
+    def as_prop_id(self, default_space: str | None = None, default_version: str | None = None) -> PropertyId:
         if self.property_ is None:
             raise ValueError("property is required to create PropertyId")
-        return PropertyId(
-            source=self.as_id(False, default_space, default_version, standardize_casing), property=self.property_
-        )
+        return PropertyId(source=self.as_id(False, default_space, default_version), property=self.property_)
 
     @property
     def versioned_id(self) -> str: