cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

cognite/neat/_rules/importers/_rdf/_shared.py

@@ -2,10 +2,9 @@ import datetime
 
 import numpy as np
 import pandas as pd
-from rdflib import OWL, Literal, Namespace
+from rdflib import Literal
 
 from cognite.neat._rules._constants import PATTERNS
-from cognite.neat._rules.models._base_rules import MatchType
 from cognite.neat._rules.models.data_types import _XSD_TYPES
 from cognite.neat._utils.rdf_ import remove_namespace_from_uri
 
@@ -17,10 +16,7 @@ def parse_raw_classes_dataframe(query_results: list[tuple]) -> pd.DataFrame:
             "Class",
             "Name",
             "Description",
-            "Parent Class",
-            "Reference",
-            "Match",
-            "Comment",
+            "Implements",
         ],
     )
 
@@ -31,9 +27,8 @@ def parse_raw_classes_dataframe(query_results: list[tuple]) -> pd.DataFrame:
     df.replace(np.nan, "", regex=True, inplace=True)
 
     df.Class = df.Class.apply(lambda x: remove_namespace_from_uri(x))
-    df["Match Type"] = len(df) * [MatchType.exact]
     df["Comment"] = len(df) * [None]
-    df["Parent Class"] = df["Parent Class"].apply(lambda x: remove_namespace_from_uri(x))
+    df["Implements"] = df["Implements"].apply(lambda x: remove_namespace_from_uri(x))
 
     return df
 
@@ -44,10 +39,7 @@ def clean_up_classes(df: pd.DataFrame) -> pd.DataFrame:
             "Class": class_,
             "Name": group_df["Name"].unique()[0],
             "Description": "\n".join(list(group_df.Description.unique())),
-            "Parent Class": ", ".join(list(group_df["Parent Class"].unique())),
-            "Reference": group_df["Reference"].unique()[0],
-            "Match Type": group_df["Match Type"].unique()[0],
-            "Comment": group_df["Comment"].unique()[0],
+            "Implements": ", ".join(list(group_df["Implements"].unique())),
         }
         for class_, group_df in df.groupby("Class")
     ]
@@ -57,8 +49,8 @@ def clean_up_classes(df: pd.DataFrame) -> pd.DataFrame:
     # bring NaNs back
     df.replace("", None, inplace=True)
 
-    # split Parent Class column back into list
-    df["Parent Class"] = df["Parent Class"].apply(lambda x: x.split(", ") if isinstance(x, str) else None)
+    # split Implements column back into list
+    df["Implements"] = df["Implements"].apply(lambda x: x.split(", ") if isinstance(x, str) else None)
 
     return df
 
@@ -76,18 +68,6 @@ def make_classes_compliant(classes: pd.DataFrame, importer: str = "RDF-based") -
         starts with a number, etc. This will cause issues when trying to create the class in CDF.
     """
 
-    # Replace empty or non-string values in "Match" column with "exact"
-    classes["Match Type"] = classes["Match Type"].fillna(MatchType.exact)
-    classes["Match Type"] = classes["Match Type"].apply(
-        lambda x: MatchType.exact if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Replace empty or non-string values in "Comment" column with a default value
-    classes["Comment"] = classes["Comment"].fillna(f"Imported using {importer} importer")
-    classes["Comment"] = classes["Comment"].apply(
-        lambda x: (f"Imported using {importer} importer" if not isinstance(x, str) or len(x) == 0 else x)
-    )
-
     # Add _object_property_class, _data_type_property_class, _thing_class to the dataframe
     classes = pd.concat(
         [
@@ -114,10 +94,7 @@ def object_property_class() -> dict:
         "Class": "ObjectProperty",
         "Name": None,
         "Description": "The class of object properties.",
-        "Parent Class": None,
-        "Reference": OWL.ObjectProperty,
-        "Match Type": MatchType.exact,
-        "Comment": "Added by NEAT based on owl:ObjectProperty but adapted to NEAT and use in CDF.",
+        "Implements": None,
     }
 
 
@@ -126,10 +103,7 @@ def data_type_property_class() -> dict:
         "Class": "DatatypeProperty",
         "Name": None,
         "Description": "The class of data properties.",
-        "Parent Class": None,
-        "Reference": OWL.DatatypeProperty,
-        "Match Type": MatchType.exact,
-        "Comment": "Added by NEAT based on owl:DatatypeProperty but adapted to NEAT and use in CDF.",
+        "Implements": None,
     }
 
 
@@ -138,24 +112,13 @@ def thing_class() -> dict:
         "Class": "Thing",
         "Name": None,
         "Description": "The class of holding class individuals.",
-        "Parent Class": None,
-        "Reference": OWL.Thing,
-        "Match Type": MatchType.exact,
-        "Comment": (
-            "Added by NEAT. "
-            "Imported from OWL base ontology, it is meant for use as a default"
-            " value type for object properties which miss a declared range."
-        ),
+        "Implements": None,
     }
 
 
 def add_parent_class(df: pd.DataFrame) -> list[dict]:
     parent_set = {
-        item
-        for sublist in df["Parent Class"].tolist()
-        if sublist
-        for item in sublist
-        if item != "" and item is not None
+        item for sublist in df["Implements"].tolist() if sublist for item in sublist if item != "" and item is not None
     }
     class_set = set(df["Class"].tolist())
 
@@ -166,14 +129,7 @@ def add_parent_class(df: pd.DataFrame) -> list[dict]:
                 "Class": missing_parent_class,
                 "Name": None,
                 "Description": None,
-                "Parent Class": None,
-                "Reference": None,
-                "Match Type": None,
-                "Comment": (
-                    "Added by NEAT. "
-                    "This is a parent class that is missing in the ontology. "
-                    "It is added by NEAT to make the ontology compliant with CDF."
-                ),
+                "Implements": None,
             }
         ]
 
@@ -192,9 +148,6 @@ def parse_raw_properties_dataframe(query_results: list[tuple]) -> pd.DataFrame:
             "Min Count",
             "Max Count",
             "Default",
-            "Reference",
-            "Match Type",
-            "Comment",
             "_property_type",
         ],
     )
@@ -206,8 +159,6 @@ def parse_raw_properties_dataframe(query_results: list[tuple]) -> pd.DataFrame:
     df.Class = df.Class.apply(lambda x: remove_namespace_from_uri(x))
     df.Property = df.Property.apply(lambda x: remove_namespace_from_uri(x))
     df["Value Type"] = df["Value Type"].apply(lambda x: remove_namespace_from_uri(x))
-    df["Match Type"] = len(df) * [MatchType.exact]
-    df["Comment"] = len(df) * [None]
     df["_property_type"] = df["_property_type"].apply(lambda x: remove_namespace_from_uri(x))
 
     return df
@@ -231,9 +182,6 @@ def clean_up_properties(df: pd.DataFrame) -> pd.DataFrame:
                     "Min Count": property_grouped_df["Min Count"].unique()[0],
                     "Max Count": property_grouped_df["Max Count"].unique()[0],
                     "Default": property_grouped_df["Default"].unique()[0],
-                    "Reference": property_grouped_df["Reference"].unique()[0],
-                    "Match Type": property_grouped_df["Match Type"].unique()[0],
-                    "Comment": property_grouped_df["Comment"].unique()[0],
                     "_property_type": property_grouped_df["_property_type"].unique()[0],
                 }
             ]
@@ -251,18 +199,6 @@ def make_properties_compliant(properties: pd.DataFrame, importer: str = "RDF-bas
     # default to 1 if "Max Count" is not specified
     properties["Max Count"] = properties["Max Count"].apply(lambda x: 1 if not isinstance(x, Literal) or x == "" else x)
 
-    # Replace empty or non-string values in "Match Type" column with "exact"
-    properties["Match Type"] = properties["Match Type"].fillna("exact")
-    properties["Match Type"] = properties["Match Type"].apply(
-        lambda x: "exact" if not isinstance(x, str) or len(x) == 0 else x
-    )
-
-    # Replace empty or non-string values in "Comment" column with a default value
-    properties["Comment"] = properties["Comment"].fillna(f"Imported using {importer} importer")
-    properties["Comment"] = properties["Comment"].apply(
-        lambda x: (f"Imported using {importer} importer" if not isinstance(x, str) or len(x) == 0 else x)
-    )
-
     # Reduce length of elements in the "Description" column to 1024 characters
     properties["Description"] = properties["Description"].apply(lambda x: x[:1024] if isinstance(x, str) else None)
 
@@ -335,8 +271,7 @@ def make_metadata_compliant(metadata: dict) -> dict:
         Dictionary containing metadata with fixed errors
     """
 
-    metadata = fix_namespace(metadata, default=Namespace("http://purl.org/cognite/neat#"))
-    metadata = fix_prefix(metadata)
+    metadata = fix_space(metadata)
     metadata = fix_version(metadata)
     metadata = fix_date(
         metadata,
@@ -348,37 +283,13 @@ def make_metadata_compliant(metadata: dict) -> dict:
         date_type="updated",
         default=datetime.datetime.now().replace(microsecond=0),
     )
-    metadata = fix_title(metadata)
+    metadata = fix_name(metadata)
     metadata = fix_description(metadata)
     metadata = fix_author(metadata, "creator")
-    metadata = fix_rights(metadata)
-    metadata = fix_license(metadata)
 
     return metadata
 
 
-def fix_license(metadata: dict, default: str = "Unknown license") -> dict:
-    if license := metadata.get("license", None):
-        if not isinstance(license, str):
-            metadata["license"] = default
-        elif isinstance(license, str) and len(license) == 0:
-            metadata["license"] = default
-    else:
-        metadata["license"] = default
-    return metadata
-
-
-def fix_rights(metadata: dict, default: str = "Unknown rights") -> dict:
-    if rights := metadata.get("rights", None):
-        if not isinstance(rights, str):
-            metadata["rights"] = default
-        elif isinstance(rights, str) and len(rights) == 0:
-            metadata["rights"] = default
-    else:
-        metadata["rights"] = default
-    return metadata
-
-
 def fix_author(metadata: dict, author_type: str = "creator", default: str = "NEAT") -> dict:
     if author := metadata.get(author_type, None):
         if not isinstance(author, str) or isinstance(author, list):
@@ -401,25 +312,12 @@ def fix_description(metadata: dict, default: str = "This model has been inferred
     return metadata
 
 
-def fix_prefix(metadata: dict, default: str = "neat") -> dict:
-    if prefix := metadata.get("prefix", None):
-        if not isinstance(prefix, str) or not PATTERNS.prefix_compliance.match(prefix):
-            metadata["prefix"] = default
+def fix_space(metadata: dict, default: str = "neat") -> dict:
+    if space := metadata.get("space", None):
+        if not isinstance(space, str) or not PATTERNS.space_compliance.match(space):
+            metadata["space"] = default
     else:
-        metadata["prefix"] = default
-    return metadata
-
-
-def fix_namespace(metadata: dict, default: Namespace) -> dict:
-    if namespace := metadata.get("namespace", None):
-        if not isinstance(namespace, Namespace):
-            try:
-                metadata["namespace"] = Namespace(namespace)
-            except Exception:
-                metadata["namespace"] = default
-    else:
-        metadata["namespace"] = default
-
+        metadata["space"] = default
     return metadata
 
 
@@ -456,18 +354,18 @@ def fix_version(metadata: dict, default: str = "1.0.0") -> dict:
     return metadata
 
 
-def fix_title(metadata: dict, default: str = "OWL Inferred Data Model") -> dict:
-    if title := metadata.get("title", None):
-        if not isinstance(title, str):
+def fix_name(metadata: dict, default: str = "OWL Inferred Data Model") -> dict:
+    if name := metadata.get("name", None):
+        if not isinstance(name, str):
             metadata["title"] = default
-        elif isinstance(title, str) and len(title) == 0:
+        elif isinstance(name, str) and len(name) == 0:
             metadata["title"] = default
-        elif isinstance(title, str) and len(title) > 255:
+        elif isinstance(name, str) and len(name) > 255:
             metadata["title"] = metadata["title"][:255]
         else:
             pass
     else:
-        metadata["title"] = default
+        metadata["name"] = default
 
     return metadata
 
@@ -539,11 +437,6 @@ def add_missing_value_types(components: dict) -> dict:
         components["Classes"].append(
             {
                 "Class": class_,
-                "Comment": (
-                    "Added by NEAT. "
-                    "This is a class that a domain of a property but was not defined in the ontology. "
-                    "It is added by NEAT to make the ontology compliant with CDF."
-                ),
             }
         )
 
@@ -561,25 +454,17 @@ def add_default_property_to_dangling_classes(components: dict[str, list[dict]])
     """
 
     dangling_classes = {
-        definition["Class"] for definition in components["Classes"] if not definition.get("Parent Class", None)
+        definition["Class"] for definition in components["Classes"] if not definition.get("Implements", None)
     } - {definition["Class"] for definition in components["Properties"]}
 
-    comment = (
-        "Added by NEAT. "
-        "This is property has been added to this class since otherwise it will create "
-        "dangling classes in the ontology."
-    )
-
     for class_ in dangling_classes:
         components["Properties"].append(
             {
                 "Class": class_,
                 "Property": "label",
                 "Value Type": "string",
-                "Comment": comment,
                 "Min Count": 0,
                 "Max Count": 1,
-                "Reference": "http://www.w3.org/2000/01/rdf-schema#label",
             }
         )
 

cognite/neat/_rules/importers/_spreadsheet2rules.py

@@ -18,7 +18,6 @@ from cognite.neat._issues.errors import (
     FileMissingRequiredFieldError,
     FileNotFoundNeatError,
     FileReadError,
-    PropertyDefinitionDuplicatedError,
 )
 from cognite.neat._issues.warnings import FileMissingRequiredFieldWarning
 from cognite.neat._rules._shared import ReadRules, T_InputRules
@@ -38,9 +37,7 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
         "Properties",
         "Properties",
         {
-            RoleTypes.domain_expert: "Property",
             RoleTypes.information: "Property",
-            RoleTypes.asset: "Property",
             RoleTypes.dms: "View Property",
         },
     ),
@@ -51,8 +48,17 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
     ("Nodes", "Nodes", "Node"),
 ]
 
+
 MANDATORY_SHEETS_BY_ROLE: dict[RoleTypes, set[str]] = {
-    role_type: {str(sheet_name) for sheet_name in VERIFIED_RULES_BY_ROLE[role_type].mandatory_fields(use_alias=True)}
+    role_type: {
+        str(sheet_name)
+        for sheet_name in (
+            VERIFIED_RULES_BY_ROLE.get(role_type).mandatory_fields(use_alias=True)  # type: ignore
+            if VERIFIED_RULES_BY_ROLE.get(role_type)
+            else []
+        )
+        if sheet_name is not None
+    }
     for role_type in RoleTypes.__members__.values()
 }
 
@@ -85,10 +91,6 @@ class MetadataRaw(UserDict):
             issue_list.append(FileMissingRequiredFieldError(filepath, "metadata", "role"))
             return False
 
-        # check if there is a schema field if role is not domain expert
-        if self.role != RoleTypes.domain_expert and not self.has_schema_field:
-            issue_list.append(FileMissingRequiredFieldError(filepath, "metadata", "schema"))
-            return False
         return True
 
 
@@ -261,45 +263,12 @@ class ExcelImporter(BaseImporter[T_InputRules]):
             if user_read is None or issue_list.has_errors:
                 return ReadRules(None, issue_list, {})
 
-            last_read: ReadResult | None = None
-            if any(sheet_name.startswith("Last") for sheet_name in user_reader.seen_sheets):
-                last_read = SpreadsheetReader(issue_list, required=False, sheet_prefix="Last").read(
-                    excel_file, self.filepath
-                )
-            reference_read: ReadResult | None = None
-            if any(sheet_name.startswith("Ref") for sheet_name in user_reader.seen_sheets):
-                reference_read = SpreadsheetReader(issue_list, sheet_prefix="Ref").read(excel_file, self.filepath)
-            elif any(sheet_name.startswith("CDMRef") for sheet_name in user_reader.seen_sheets):
-                reference_read = SpreadsheetReader(issue_list, sheet_prefix="CDMRef").read(excel_file, self.filepath)
-
         if issue_list.has_errors:
             return ReadRules(None, issue_list, {})
 
-        if reference_read and user_read.role != reference_read.role:
-            issue_list.append(
-                PropertyDefinitionDuplicatedError(
-                    self.filepath.as_posix(),
-                    "spreadsheet.metadata",  # type: ignore[arg-type]
-                    "role",
-                    frozenset({user_read.role, reference_read.role}),
-                    ("user", "reference"),
-                    "sheet",
-                )
-            )
-            return ReadRules(None, issue_list, {})
-
         sheets = user_read.sheets
         original_role = user_read.role
         read_info_by_sheet = user_read.read_info_by_sheet
-        if last_read:
-            sheets["last"] = last_read.sheets
-            read_info_by_sheet.update(last_read.read_info_by_sheet)
-            if reference_read:
-                # The last rules will also be validated against the reference rules
-                sheets["last"]["reference"] = reference_read.sheets  # type: ignore[call-overload]
-        if reference_read:
-            sheets["reference"] = reference_read.sheets
-            read_info_by_sheet.update(reference_read.read_info_by_sheet)
 
         rules_cls = INPUT_RULES_BY_ROLE[original_role]
         rules = cast(T_InputRules, rules_cls.load(sheets))

cognite/neat/_rules/models/__init__.py

@@ -1,39 +1,25 @@
-from cognite.neat._rules.models.asset._rules import AssetRules
-from cognite.neat._rules.models.asset._rules_input import AssetInputRules
-from cognite.neat._rules.models.domain import DomainInputRules, DomainRules
+from cognite.neat._client.data_classes.schema import DMSSchema
 from cognite.neat._rules.models.information._rules import InformationRules
 from cognite.neat._rules.models.information._rules_input import InformationInputRules
 
 from ._base_rules import DataModelType, ExtensionCategory, RoleTypes, SchemaCompleteness, SheetList, SheetRow
 from .dms._rules import DMSRules
 from .dms._rules_input import DMSInputRules
-from .dms._schema import DMSSchema
 
-INPUT_RULES_BY_ROLE: dict[
-    RoleTypes, type[InformationInputRules] | type[AssetInputRules] | type[DMSInputRules] | type[DomainInputRules]
-] = {
-    RoleTypes.domain_expert: DomainInputRules,
+INPUT_RULES_BY_ROLE: dict[RoleTypes, type[InformationInputRules] | type[DMSInputRules]] = {
     RoleTypes.information: InformationInputRules,
-    RoleTypes.asset: AssetInputRules,
     RoleTypes.dms: DMSInputRules,
 }
-VERIFIED_RULES_BY_ROLE: dict[
-    RoleTypes, type[InformationRules] | type[AssetRules] | type[DMSRules] | type[DomainRules]
-] = {
-    RoleTypes.domain_expert: DomainRules,
+VERIFIED_RULES_BY_ROLE: dict[RoleTypes, type[InformationRules] | type[DMSRules]] = {
     RoleTypes.information: InformationRules,
-    RoleTypes.asset: AssetRules,
     RoleTypes.dms: DMSRules,
 }
 
 
 __all__ = [
-    "DomainRules",
     "DMSInputRules",
     "InformationInputRules",
-    "AssetInputRules",
     "InformationRules",
-    "AssetRules",
     "DMSRules",
     "INPUT_RULES_BY_ROLE",
     "DMSSchema",