cognite-neat 0.92.1__tar.gz → 0.92.3__tar.gz

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.92.1
+Version: 0.92.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.92.3/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.92.3"

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/graph/queries/_base.py

@@ -7,7 +7,7 @@ from rdflib import Literal as RdfLiteral
 from rdflib.query import ResultRow
 
 from cognite.neat.graph.models import InstanceType
-from cognite.neat.rules.models.entities import ClassEntity
+from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
 from cognite.neat.rules.models.information import InformationRules
 from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
@@ -98,13 +98,18 @@ class Queries:
     def describe(
         self,
         instance_id: URIRef,
+        instance_type: str | None = None,
         property_renaming_config: dict | None = None,
+        property_types: dict[str, EntityTypes] | None = None,
     ) -> tuple[str, dict[str | InstanceType, list[str]]] | None:
         """DESCRIBE instance for a given class from the graph store
 
         Args:
             instance_id: Instance id for which we want to generate query
-            property_renaming_config: Dictionary to rename properties, default None
+            instance_type: Type of the instance, default None (will be inferred from triples)
+            property_renaming_config: Dictionary to rename properties, default None (no renaming)
+            property_types: Dictionary of property types, default None (helper for removal of namespace)
+
 
         Returns:
             Dictionary of instance properties
@@ -119,21 +124,52 @@ class Queries:
                 "null",
             ]:
                 continue
-            # we are skipping deep validation with Pydantic to remove namespace here
-            # as it reduces time to process triples by 10-15x
-            value = remove_namespace_from_uri(object_, validation="prefix")
-
-            # use-case: calling describe without renaming properties
-            # losing the namespace from the predicate!
-            if not property_renaming_config and predicate != RDF.type:
-                property_values[remove_namespace_from_uri(predicate, validation="prefix")].append(value)
-            elif predicate == RDF.type:
-                property_values[RDF.type].append(value)
-            # use-case: calling describe with renaming properties
-            # renaming the property to the new name, if the property is defined
-            # in the RULES sheet
-            elif property_renaming_config and (property_ := property_renaming_config.get(predicate, None)):
+
+            # set property
+            if property_renaming_config and predicate != RDF.type:
+                property_ = property_renaming_config.get(
+                    predicate, remove_namespace_from_uri(predicate, validation="prefix")
+                )
+            elif not property_renaming_config and predicate != RDF.type:
+                property_ = remove_namespace_from_uri(predicate, validation="prefix")
+            else:
+                property_ = RDF.type
+
+            # set value
+            # if it is URIRef and property type is object property, we need to remove namespace
+            # if it URIref but we are doing this into data type property, we do not remove namespace
+            # case 1 for RDF type we remove namespace
+            if property_ == RDF.type:
+                value = remove_namespace_from_uri(object_, validation="prefix")
+
+            # case 2 for define object properties we remove namespace
+            elif (
+                isinstance(object_, URIRef)
+                and property_types
+                and property_types.get(property_, None) == EntityTypes.object_property
+            ):
+                value = remove_namespace_from_uri(object_, validation="prefix")
+
+            # case 3 when property type is not defined and returned value is URIRef we remove namespace
+            elif isinstance(object_, URIRef) and not property_types:
+                value = remove_namespace_from_uri(object_, validation="prefix")
+
+            # case 4 for data type properties we do not remove namespace but keep the entire value
+            # but we drop the datatype part, and keep everything to be string (data loader will do the conversion)
+            # for value type it expects (if possible)
+            else:
+                value = str(object_)
+
+            # add type to the dictionary
+            if predicate != RDF.type:
                 property_values[property_].append(value)
+            else:
+                # guarding against multiple rdf:type values as this is not allowed in CDF
+                if RDF.type not in property_values:
+                    property_values[RDF.type].append(instance_type if instance_type else value)
+                else:
+                    # we should not have multiple rdf:type values
+                    continue
 
         if property_values:
             return (

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/analysis/_information.py

@@ -1,4 +1,3 @@
-import logging
 import warnings
 from typing import Any, cast
 
@@ -12,12 +11,13 @@ from cognite.neat.rules.models._rdfpath import (
     SelfReferenceProperty,
     SingleProperty,
 )
-from cognite.neat.rules.models.entities import ClassEntity, ReferenceEntity
+from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ReferenceEntity
 from cognite.neat.rules.models.information import (
     InformationClass,
     InformationProperty,
     InformationRules,
 )
+from cognite.neat.utils.collection_ import most_occurring_element
 from cognite.neat.utils.rdf_ import get_inheritance_path
 
 from ._base import BaseAnalysis
@@ -105,6 +105,26 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
 
         return property_renaming_configuration
 
+    def property_types(self, class_: ClassEntity) -> dict[str, EntityTypes]:
+        property_types = {}
+        if definitions := self.class_property_pairs(consider_inheritance=True).get(class_, None):
+            for property_id, definition in definitions.items():
+                property_types[property_id] = definition.type_
+
+        return property_types
+
+    def most_occurring_class_in_transformations(self, class_: ClassEntity) -> ClassEntity | None:
+        classes = []
+        if class_property_pairs := self.class_property_pairs(consider_inheritance=True, only_rdfpath=True).get(
+            class_, None
+        ):
+            for property_ in class_property_pairs.values():
+                classes.append(cast(RDFPath, property_.transformation).traversal.class_)
+
+            return cast(ClassEntity, most_occurring_element(classes))
+        else:
+            return None
+
     def subset_rules(self, desired_classes: set[ClassEntity]) -> InformationRules:
         """
         Subset rules to only include desired classes and their properties.
@@ -148,11 +168,9 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
         possible_classes = possible_classes.union(parents)
 
         if not possible_classes:
-            logging.error("None of the desired classes are defined in the data model!")
             raise ValueError("None of the desired classes are defined in the data model!")
 
         if impossible_classes:
-            logging.warning(f"Could not find the following classes defined in the data model: {impossible_classes}")
             warnings.warn(
                 f"Could not find the following classes defined in the data model: {impossible_classes}",
                 stacklevel=2,
@@ -165,7 +183,6 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
             "properties": [],
         }
 
-        logging.info(f"Reducing data model to only include the following classes: {possible_classes}")
         for class_ in possible_classes:
             reduced_data_model["classes"].append(class_as_dict[str(class_.suffix)])
 
@@ -181,3 +198,31 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
             warnings.warn(f"Reduced data model is not complete: {e}", stacklevel=2)
             reduced_data_model["metadata"].schema_ = SchemaCompleteness.partial
             return type(self.rules).model_construct(**reduced_data_model)
+
+    def class_uri(self, class_: ClassEntity) -> URIRef | None:
+        """Get URI for a class entity based on the rules.
+
+        Args:
+            class_: instance of ClassEntity
+
+        Returns:
+            URIRef of the class entity or None if not found
+        """
+
+        # we need to handle optional renamings and we do this
+        # by checking if the most occurring class in transformations alternatively
+        # in cases when we are not specifying transformations we default to the class entity
+        if not (most_frequent_class := self.most_occurring_class_in_transformations(class_)):
+            most_frequent_class = class_
+
+        # case 1 class prefix in rules.prefixes
+        if most_frequent_class.prefix in self.rules.prefixes:
+            return self.rules.prefixes[cast(str, most_frequent_class.prefix)][most_frequent_class.suffix]
+
+        # case 2 class prefix equal to rules.metadata.prefix
+        elif most_frequent_class.prefix == self.rules.metadata.prefix:
+            return self.rules.metadata.namespace[most_frequent_class.suffix]
+
+        # case 3 when class prefix is not found in prefixes of rules
+        else:
+            return None

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/examples/__init__.py

@@ -7,3 +7,4 @@ from pathlib import Path
 
 _EXAMPLES = Path(__file__).parent
 wind_energy_ontology = _EXAMPLES / "wind-energy.owl"
+imf_info_rules = _EXAMPLES / "info-rules-imf.xlsx"

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/exporters/_rules2excel.py

@@ -11,6 +11,7 @@ from openpyxl import Workbook
 from openpyxl.cell import MergedCell
 from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
 from openpyxl.worksheet.worksheet import Worksheet
+from rdflib import Namespace
 
 from cognite.neat.rules._shared import VerifiedRules
 from cognite.neat.rules.models import (
@@ -22,6 +23,7 @@ from cognite.neat.rules.models import (
 from cognite.neat.rules.models.dms import DMSMetadata
 from cognite.neat.rules.models.domain import DomainMetadata
 from cognite.neat.rules.models.information import InformationMetadata
+from cognite.neat.rules.models.information._rules import InformationRules
 
 from ._base import BaseExporter
 
@@ -133,6 +135,9 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
             self._write_sheets(workbook, dumped_reference_rules, rules, sheet_prefix="Ref")
             self._write_metadata_sheet(workbook, dumped_reference_rules["Metadata"], sheet_prefix="Ref")
 
+        if isinstance(rules, InformationRules) and rules.prefixes:
+            self._write_prefixes_sheet(workbook, rules.prefixes)
+
         if self._styling_level > 0:
             self._adjust_column_widths(workbook)
 
@@ -209,6 +214,16 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
             for cell in metadata_sheet["A"]:
                 cell.font = Font(bold=True, size=12)
 
+    def _write_prefixes_sheet(self, workbook: Workbook, prefixes: dict[str, Namespace]) -> None:
+        metadata_sheet = workbook.create_sheet("Prefixes")
+        metadata_sheet.append(["Prefix", "Namespace"])
+        for key, value in prefixes.items():
+            metadata_sheet.append([key, value])
+
+        if self._styling_level > 1:
+            for cell in metadata_sheet["A"]:
+                cell.font = Font(bold=True, size=12)
+
     @classmethod
     def _get_item_class(cls, annotation: GenericAlias) -> type[SheetRow]:
         if not isinstance(annotation, GenericAlias):

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/importers/_base.py

@@ -1,8 +1,7 @@
-import getpass
 import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from datetime import datetime
 from typing import Any, Generic, Literal
 
@@ -25,6 +24,12 @@ class BaseImporter(ABC, Generic[T_InputRules]):
         raise NotImplementedError()
 
     def _default_metadata(self) -> dict[str, Any]:
+        creator = "UNKNOWN"
+        with suppress(KeyError, ImportError):
+            import getpass
+
+            creator = getpass.getuser()
+
         return {
             "prefix": "neat",
             "schema": "partial",
@@ -33,7 +38,7 @@ class BaseImporter(ABC, Generic[T_InputRules]):
             "title": "Neat Imported Data Model",
             "created": datetime.now().replace(microsecond=0).isoformat(),
             "updated": datetime.now().replace(microsecond=0).isoformat(),
-            "creator": getpass.getuser(),
+            "creator": creator,
             "description": f"Imported using {type(self).__name__}",
         }
 

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/importers/_spreadsheet2rules.py

@@ -11,6 +11,7 @@ from typing import Literal, cast
 import pandas as pd
 from cognite.client.utils._importing import local_import
 from pandas import ExcelFile
+from rdflib import Namespace
 
 from cognite.neat.issues import IssueList
 from cognite.neat.issues.errors import (
@@ -19,6 +20,7 @@ from cognite.neat.issues.errors import (
     FileReadError,
     PropertyDefinitionDuplicatedError,
 )
+from cognite.neat.issues.warnings import FileMissingRequiredFieldWarning
 from cognite.neat.rules._shared import ReadRules, T_InputRules
 from cognite.neat.rules.models import (
     INPUT_RULES_BY_ROLE,
@@ -93,6 +95,7 @@ class ReadResult:
     sheets: dict[str, dict | list]
     read_info_by_sheet: dict[str, SpreadsheetRead]
     metadata: MetadataRaw
+    prefixes: dict[str, Namespace] | None = None
 
     @property
     def role(self) -> RoleTypes:
@@ -122,6 +125,10 @@ class SpreadsheetReader:
     def metadata_sheet_name(self) -> str:
         return f"{self._sheet_prefix}Metadata"
 
+    @property
+    def prefixes_sheet_name(self) -> str:
+        return "Prefixes"
+
     @property
     def seen_sheets(self) -> set[str]:
         if not self._seen_files:
@@ -149,6 +156,14 @@ class SpreadsheetReader:
             return None
         sheets["Metadata"] = dict(metadata)
 
+        # Special case for reading prefixes as they are suppose to be read only once
+        if (
+            self.prefixes_sheet_name in excel_file.sheet_names
+            and not self._sheet_prefix
+            and (prefixes := self._read_prefixes(excel_file, filepath))
+        ):
+            sheets["Prefixes"] = prefixes
+
         return ReadResult(sheets, read_info_by_sheet, metadata)
 
     def _read_metadata(self, excel_file: ExcelFile, filepath: Path) -> MetadataRaw | None:
@@ -163,6 +178,25 @@ class SpreadsheetReader:
             return None
         return metadata
 
+    def _read_prefixes(self, excel_file: ExcelFile, filepath: Path) -> dict[str, Namespace] | None:
+        if self.prefixes_sheet_name not in excel_file.sheet_names:
+            return None
+
+        else:
+            prefixes = {}
+
+            for row in read_individual_sheet(excel_file, "Prefixes", expected_headers=["Prefix", "Namespace"]):
+                if "Prefix" in row and "Namespace" in row:
+                    prefixes[row["Prefix"]] = row["Namespace"]
+                else:
+                    if "Prefix" not in row:
+                        self.issue_list.append(FileMissingRequiredFieldWarning(filepath, "prefixes", "prefix"))
+                    if "Namespace" not in row:
+                        self.issue_list.append(FileMissingRequiredFieldWarning(filepath, "prefixes", "namespace"))
+                    return None
+
+            return prefixes
+
     def _read_sheets(
         self, excel_file: ExcelFile, read_role: RoleTypes
     ) -> tuple[dict[str, dict | list] | None, dict[str, SpreadsheetRead]]:

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/models/_base_input.py

@@ -66,11 +66,15 @@ class InputRules(Generic[T_BaseRules], ABC):
                 candidate = type_
             elif isinstance(type_, GenericAlias) and type_.__origin__ is list and is_dataclass(type_.__args__[0]):
                 candidate = type_.__args__[0]
+
+            # this handles prefixes
+            elif isinstance(type_, GenericAlias) and type_.__origin__ is dict:
+                candidate = type_
             else:
                 continue
 
-            if hasattr(candidate, "_load"):
-                output[field_.name] = candidate
+            output[field_.name] = candidate
+
         return output
 
     @classmethod
@@ -88,10 +92,16 @@ class InputRules(Generic[T_BaseRules], ABC):
             else:
                 continue
 
-            if isinstance(value, dict):
-                args[field_name] = field_type._load(value)  # type: ignore[attr-defined]
-            elif isinstance(value, list) and value and isinstance(value[0], dict):
-                args[field_name] = [field_type._load(item) for item in value]  # type: ignore[attr-defined]
+            # Handles the case where the field is a dataclass
+            if hasattr(field_type, "_load"):
+                if isinstance(value, dict):
+                    args[field_name] = field_type._load(value)  # type: ignore[attr-defined]
+                elif isinstance(value, list) and value and isinstance(value[0], dict):
+                    args[field_name] = [field_type._load(item) for item in value]  # type: ignore[attr-defined]
+            # Handles the case where the field holds non-dataclass values, e.g. a prefixes dict
+            else:
+                args[field_name] = value
+
         return cls(**args)
 
     def _dataclass_fields(self) -> list[Field]:

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/models/_base_rules.py

@@ -108,6 +108,14 @@ class SchemaModel(BaseModel):
         """Returns a set of mandatory fields for the model."""
         return _get_required_fields(cls, use_alias)
 
+    @field_validator("*", mode="before")
+    def strip_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        elif isinstance(value, list):
+            return [entry.strip() if isinstance(entry, str) else entry for entry in value]
+        return value
+
 
 class BaseMetadata(SchemaModel):
     """
@@ -279,12 +287,6 @@ class BaseRules(SchemaModel, ABC):
 
 
 class SheetRow(SchemaModel):
-    @field_validator("*", mode="before")
-    def strip_string(cls, value: Any) -> Any:
-        if isinstance(value, str):
-            return value.strip()
-        return value
-
     @abstractmethod
     def _identifier(self) -> tuple[Hashable, ...]:
         raise NotImplementedError()

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/models/dms/_rules.py

@@ -98,15 +98,15 @@ class DMSMetadata(BaseMetadata):
 
     @field_validator("schema_", mode="plain")
     def as_enum_schema(cls, value: str) -> SchemaCompleteness:
-        return SchemaCompleteness(value)
+        return SchemaCompleteness(value.strip())
 
     @field_validator("extension", mode="plain")
     def as_enum_extension(cls, value: str) -> ExtensionCategory:
-        return ExtensionCategory(value)
+        return ExtensionCategory(value.strip())
 
     @field_validator("data_model_type", mode="plain")
     def as_enum_model_type(cls, value: str) -> DataModelType:
-        return DataModelType(value)
+        return DataModelType(value.strip())
 
     @field_validator("description", mode="before")
     def nan_as_none(cls, value):

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/models/entities/_single_value.py

@@ -17,6 +17,7 @@ from cognite.client.data_classes.data_modeling.ids import (
 from pydantic import (
     BaseModel,
     Field,
+    field_validator,
     model_serializer,
     model_validator,
 )
@@ -97,6 +98,14 @@ class Entity(BaseModel, extra="ignore"):
     def as_str(self) -> str:
         return str(self)
 
+    @field_validator("*", mode="before")
+    def strip_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        elif isinstance(value, list):
+            return [entry.strip() if isinstance(entry, str) else entry for entry in value]
+        return value
+
     @classmethod
     def _parse(cls, raw: str, defaults: dict) -> dict:
         if not (result := ENTITY_PATTERN.match(raw)):

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/rules/models/information/_rules.py

@@ -100,15 +100,15 @@ class InformationMetadata(BaseMetadata):
 
     @field_validator("schema_", mode="plain")
     def as_enum_schema(cls, value: str) -> SchemaCompleteness:
-        return SchemaCompleteness(value)
+        return SchemaCompleteness(value.strip())
 
     @field_validator("extension", mode="plain")
     def as_enum_extension(cls, value: str) -> ExtensionCategory:
-        return ExtensionCategory(value)
+        return ExtensionCategory(value.strip())
 
     @field_validator("data_model_type", mode="plain")
     def as_enum_model_type(cls, value: str) -> DataModelType:
-        return DataModelType(value)
+        return DataModelType(value.strip())
 
     def as_identifier(self) -> str:
         return f"{self.prefix}:{self.name}"

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/cognite/neat/store/_base.py

@@ -186,9 +186,17 @@ class NeatGraphStore:
             warnings.warn("Desired type not found in graph!", stacklevel=2)
             return None
 
-        analysis = InformationAnalysis(self.rules)
-        has_hop_transformations = analysis.has_hop_transformations()
-        has_self_reference_transformations = analysis.has_self_reference_property_transformations()
+        if not (class_uri := InformationAnalysis(self.rules).class_uri(class_entity)):
+            warnings.warn(
+                f"Class {class_} does not have namespace defined for prefix {class_entity.prefix} Rules!",
+                stacklevel=2,
+            )
+            return None
+
+        has_hop_transformations = InformationAnalysis(self.rules).has_hop_transformations()
+        has_self_reference_transformations = InformationAnalysis(
+            self.rules
+        ).has_self_reference_property_transformations()
         if has_hop_transformations or has_self_reference_transformations:
             msg = (
                 f"Rules contain [{'Hop' if has_hop_transformations else '' }"
@@ -205,12 +213,22 @@ class NeatGraphStore:
             )
             return None
 
-        instance_ids = self.queries.list_instances_ids_of_class(self.rules.metadata.namespace[class_])
+        # get all the instances for give class_uri
+        instance_ids = self.queries.list_instances_ids_of_class(class_uri)
 
+        # get potential property renaming config
         property_renaming_config = InformationAnalysis(self.rules).define_property_renaming_config(class_entity)
 
+        # get property types to guide process of removing or not namespaces from results
+        property_types = InformationAnalysis(self.rules).property_types(class_entity)
+
         for instance_id in instance_ids:
-            if res := self.queries.describe(instance_id, property_renaming_config):
+            if res := self.queries.describe(
+                instance_id=instance_id,
+                instance_type=class_,
+                property_renaming_config=property_renaming_config,
+                property_types=property_types,
+            ):
                 yield res
 
     def _parse_file(

{cognite_neat-0.92.1 → cognite_neat-0.92.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.92.1"
+version = "0.92.3"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.92.1/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.92.1"