cognite-neat 0.109.3__py3-none-any.whl → 0.110.0__py3-none-any.whl

cognite/neat/_store/_graph_store.py

@@ -18,9 +18,6 @@ from cognite.neat._graph.queries import Queries
 from cognite.neat._graph.transformers import Transformers
 from cognite.neat._issues import IssueList, catch_issues
 from cognite.neat._issues.errors import OxigraphStorageLockedError
-from cognite.neat._rules.analysis import InformationAnalysis
-from cognite.neat._rules.models import InformationRules
-from cognite.neat._rules.models.entities import ClassEntity
 from cognite.neat._shared import InstanceType, Triple
 from cognite.neat._utils.auxiliary import local_import
 from cognite.neat._utils.rdf_ import add_triples_in_batch, remove_namespace_from_uri
@@ -55,9 +52,6 @@ class NeatGraphStore:
         dataset: Dataset,
         default_named_graph: URIRef | None = None,
     ):
-        self.rules: dict[URIRef, InformationRules] = {}
-        self.base_namespace: dict[URIRef, Namespace] = {}
-
         _start = datetime.now(timezone.utc)
         self.dataset = dataset
         self.provenance = Provenance[Entity](
@@ -72,8 +66,7 @@ class NeatGraphStore:
         )
 
         self.default_named_graph = default_named_graph or DATASET_DEFAULT_GRAPH_ID
-
-        self.queries = Queries(self.dataset, self.rules, self.default_named_graph)
+        self.queries = Queries(self.dataset, self.default_named_graph)
 
     def graph(self, named_graph: URIRef | None = None) -> Graph:
         """Get named graph from the dataset to query over"""
@@ -116,36 +109,6 @@ class NeatGraphStore:
         else:
             return self.dataset.serialize(format="ox-trig" if self.type_ == "OxigraphStore" else "trig")
 
-    def add_rules(self, rules: InformationRules, named_graph: URIRef | None = None) -> None:
-        """This method is used to add rules to a named graph stored in the graph store.
-
-        Args:
-            rules: InformationRules object containing rules to be added to the named graph
-            named_graph: URIRef of the named graph to store the rules in, by default None
-                        rules will be added to the default graph
-
-        """
-
-        named_graph = named_graph or self.default_named_graph
-
-        if named_graph in self.named_graphs:
-            # attaching appropriate namespace to the rules
-            # as well base_namespace
-            self.rules[named_graph] = rules
-            self.base_namespace[named_graph] = rules.metadata.namespace
-            self.queries = Queries(self.dataset, self.rules)
-            self.provenance.append(
-                Change.record(
-                    activity=f"{type(self)}.rules",
-                    start=datetime.now(timezone.utc),
-                    end=datetime.now(timezone.utc),
-                    description=f"Added {type(self.rules).__name__} to {named_graph} named graph",
-                )
-            )
-
-            if self.rules[named_graph].prefixes:
-                self._upsert_prefixes(self.rules[named_graph].prefixes, named_graph)
-
     def _upsert_prefixes(self, prefixes: dict[str, Namespace], named_graph: URIRef) -> None:
         """Adds prefixes to the graph store."""
         _start = datetime.now(timezone.utc)
@@ -271,226 +234,22 @@ class NeatGraphStore:
             last_change.target_entity.issues.extend(issue_list)
         return issue_list
 
-    def _read_via_rules_linkage(
+    def read(
         self,
-        class_neat_id: URIRef,
-        property_link_pairs: dict[str, URIRef] | None,
+        class_uri: URIRef,
         named_graph: URIRef | None = None,
-    ) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
-        named_graph = named_graph or self.default_named_graph
-
-        if named_graph not in self.named_graphs:
-            warnings.warn(
-                f"Named graph {named_graph} not found in graph store, cannot read",
-                stacklevel=2,
-            )
-            return
-
-        if not self.rules or named_graph not in self.rules:
-            warnings.warn(
-                f"Rules for named graph {named_graph} not found in graph store!",
-                stacklevel=2,
-            )
-            return
-
-        if self.multi_type_instances:
-            warnings.warn(
-                "Multi typed instances detected, issues with loading can occur!",
-                stacklevel=2,
-            )
-
-        analysis = InformationAnalysis(self.rules[named_graph])
-
-        if cls := analysis.classes_by_neat_id.get(class_neat_id):
-            if property_link_pairs:
-                property_renaming_config = {
-                    prop_uri: prop_name
-                    for prop_name, prop_neat_id in property_link_pairs.items()
-                    if (prop_uri := analysis.neat_id_to_instance_source_property_uri(prop_neat_id))
-                }
-                if information_properties := analysis.classes_with_properties(consider_inheritance=True).get(
-                    cls.class_
-                ):
-                    for prop in information_properties:
-                        if prop.neatId is None:
-                            continue
-                        # Include renaming done in the Information rules that are not present in the
-                        # property_link_pairs. The use case for this renaming to startNode and endNode
-                        # properties that are not part of DMSRules but will typically be present
-                        # in the Information rules.
-                        if (
-                            uri := analysis.neat_id_to_instance_source_property_uri(prop.neatId)
-                        ) and uri not in property_renaming_config:
-                            property_renaming_config[uri] = prop.property_
-
-                yield from self._read_via_class_entity(cls.class_, property_renaming_config)
-                return
-            else:
-                warnings.warn("Rules not linked", stacklevel=2)
-                return
-        else:
-            warnings.warn("Class with neat id {class_neat_id} found in rules", stacklevel=2)
-            return
-
-    def _read_via_class_entity(
-        self,
-        class_entity: ClassEntity,
         property_renaming_config: dict[URIRef, str] | None = None,
-        named_graph: URIRef | None = None,
     ) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
         named_graph = named_graph or self.default_named_graph
 
-        if named_graph not in self.named_graphs:
-            warnings.warn(
-                f"Named graph {named_graph} not found in graph store, cannot read",
-                stacklevel=2,
-            )
-            return
-
-        if not self.rules or named_graph not in self.rules:
-            warnings.warn(
-                f"Rules for named graph {named_graph} not found in graph store!",
-                stacklevel=2,
-            )
-            return
-        if self.multi_type_instances:
-            warnings.warn(
-                "Multi typed instances detected, issues with loading can occur!",
-                stacklevel=2,
-            )
-
-        if class_entity not in [definition.class_ for definition in self.rules[named_graph].classes]:
-            warnings.warn("Desired type not found in graph!", stacklevel=2)
-            return
-
-        if not (class_uri := InformationAnalysis(self.rules[named_graph]).class_uri(class_entity)):
-            warnings.warn(
-                f"Class {class_entity.suffix} does not have namespace defined for prefix {class_entity.prefix} Rules!",
-                stacklevel=2,
-            )
-            return
-
-        has_hop_transformations = InformationAnalysis(self.rules[named_graph]).has_hop_transformations()
-        has_self_reference_transformations = InformationAnalysis(
-            self.rules[named_graph]
-        ).has_self_reference_property_transformations()
-        if has_hop_transformations or has_self_reference_transformations:
-            msg = (
-                f"Rules contain [{'Hop' if has_hop_transformations else ''}"
-                f", {'SelfReferenceProperty' if has_self_reference_transformations else ''}]"
-                " rdfpath."
-                f" Run [{'ReduceHopTraversal' if has_hop_transformations else ''}"
-                f", {'AddSelfReferenceProperty' if has_self_reference_transformations else ''}]"
-                " transformer(s) first!"
-            )
-
-            warnings.warn(
-                msg,
-                stacklevel=2,
-            )
-            return
-
-        # get all the instances for give class_uri
-        instance_ids = self.queries.list_instances_ids_of_class(class_uri)
-
-        # get potential property renaming config
-        property_renaming_config = property_renaming_config or InformationAnalysis(
-            self.rules[named_graph]
-        ).define_property_renaming_config(class_entity)
+        instance_ids = self.queries.list_instances_ids_of_class(class_uri, named_graph=named_graph)
 
         for instance_id in instance_ids:
             if res := self.queries.describe(
-                instance_id=instance_id,
-                instance_type=class_entity.suffix,
-                property_renaming_config=property_renaming_config,
+                instance_id=instance_id, instance_type=class_uri, property_renaming_config=property_renaming_config
             ):
                 yield res
 
-    def read(
-        self, class_: str, named_graph: URIRef | None = None
-    ) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
-        """Read instances for given class from the graph store.
-
-        !!! note "Assumption"
-            This method assumes that the class_ belongs to the same (name)space as
-            the rules which are attached to the graph store.
-
-        """
-        named_graph = named_graph or self.default_named_graph
-
-        if named_graph not in self.named_graphs:
-            warnings.warn(
-                f"Named graph {named_graph} not found in graph store, cannot read",
-                stacklevel=2,
-            )
-            return
-
-        if not self.rules or named_graph not in self.rules:
-            warnings.warn(
-                f"Rules for named graph {named_graph} not found in graph store!",
-                stacklevel=2,
-            )
-            return
-        if self.multi_type_instances:
-            warnings.warn(
-                "Multi typed instances detected, issues with loading can occur!",
-                stacklevel=2,
-            )
-
-        class_entity = ClassEntity(prefix=self.rules[named_graph].metadata.prefix, suffix=class_)
-
-        if class_entity not in [definition.class_ for definition in self.rules[named_graph].classes]:
-            warnings.warn("Desired type not found in graph!", stacklevel=2)
-            return
-
-        yield from self._read_via_class_entity(class_entity)
-
-    def count_of_id(self, neat_id: URIRef, named_graph: URIRef | None = None) -> int:
-        """Count the number of instances of a given type
-
-        Args:
-            neat_id: Type for which instances are to be counted
-
-        Returns:
-            Number of instances
-        """
-        named_graph = named_graph or self.default_named_graph
-
-        if named_graph not in self.named_graphs:
-            warnings.warn(
-                f"Named graph {named_graph} not found in graph store, cannot count",
-                stacklevel=2,
-            )
-            return 0
-
-        if not self.rules or named_graph not in self.rules:
-            warnings.warn(
-                f"Rules for named graph {named_graph} not found in graph store!",
-                stacklevel=2,
-            )
-            return 0
-
-        class_entity = next(
-            (definition.class_ for definition in self.rules[named_graph].classes if definition.neatId == neat_id),
-            None,
-        )
-        if not class_entity:
-            warnings.warn("Desired type not found in graph!", stacklevel=2)
-            return 0
-
-        if not (class_uri := InformationAnalysis(self.rules[named_graph]).class_uri(class_entity)):
-            warnings.warn(
-                f"Class {class_entity.suffix} does not have namespace defined for prefix {class_entity.prefix} Rules!",
-                stacklevel=2,
-            )
-            return 0
-
-        return self.count_of_type(class_uri)
-
-    def count_of_type(self, class_uri: URIRef) -> int:
-        query = f"SELECT (COUNT(?instance) AS ?instanceCount) WHERE {{ ?instance a <{class_uri}> }}"
-        return int(next(iter(self.dataset.query(query)))[0])  # type: ignore[arg-type, index]
-
     def _parse_file(
         self,
         named_graph: URIRef,

cognite/neat/_utils/rdf_.py

@@ -154,7 +154,7 @@ def _traverse(hierarchy: dict, graph: dict, names: list[str]) -> dict:
     return hierarchy
 
 
-def get_inheritance_path(child: Any, child_parent: dict[Any, list[Any]]) -> list:
+def get_inheritance_path(child: Any, child_parent: dict[Any, set[Any]]) -> list[Any]:
     """Returns the inheritance path for a given child
 
     Args:
@@ -167,7 +167,7 @@ def get_inheritance_path(child: Any, child_parent: dict[Any, list[Any]]) -> list
     !!! note "No Circular Inheritance"
         This method assumes that the child_parent dictionary is a tree and does not contain any cycles.
     """
-    path = []
+    path: list[Any] = []
     if child in child_parent:
         path.extend(child_parent[child])
         for parent in child_parent[child]:

cognite/neat/_utils/spreadsheet.py

@@ -5,6 +5,8 @@ import pandas as pd
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
 
+from cognite.neat._rules._constants import get_internal_properties
+
 
 @dataclass
 class SpreadsheetRead:
@@ -16,6 +18,7 @@ class SpreadsheetRead:
 
     header_row: int = 1
     empty_rows: list[int] = field(default_factory=list)
+    skipped_rows: list[int] = field(default_factory=list)
     is_one_indexed: bool = True
 
     def __post_init__(self):
@@ -28,6 +31,13 @@ class SpreadsheetRead:
                 output += 1
             else:
                 break
+
+        for skipped_rows in self.skipped_rows:
+            if skipped_rows <= output:
+                output += 1
+            else:
+                break
+
         return output + self.header_row + (1 if self.is_one_indexed else 0)
 
 
@@ -63,19 +73,52 @@ def read_individual_sheet(
 
     raw = pd.read_excel(excel_file, sheet_name, skiprows=skiprows)
     is_na = raw.isnull().all(axis=1)
+    skip_rows = _find_rows_to_skip(raw)
     empty_rows = is_na[is_na].index.tolist()
 
+    if skip_rows:
+        raw = raw.drop(skip_rows)
+
     raw.dropna(axis=0, how="all", inplace=True)
+
     if "Value Type" in raw.columns:
         # Special handling for Value Type column, #N/A is treated specially by NEAT it means Unknown
         raw["Value Type"] = raw["Value Type"].replace(float("nan"), "#N/A")
+
     output = raw.replace(float("nan"), None).to_dict(orient="records")
     if return_read_info:
         # If no rows are skipped, row 1 is the header row.
-        return output, SpreadsheetRead(header_row=skiprows + 1, empty_rows=empty_rows, is_one_indexed=True)
+        return output, SpreadsheetRead(
+            header_row=skiprows + 1,
+            empty_rows=empty_rows,
+            is_one_indexed=True,
+            skipped_rows=skip_rows,
+        )
     return output
 
 
+def _find_rows_to_skip(
+    df: pd.DataFrame,
+) -> list:
+    """Find rows which are having all values as None except for internal properties."""
+    rows_to_skip = []
+
+    internal_cols = {val.lower() for val in get_internal_properties()}
+    for i, row in df.iterrows():
+        user_cols_state = []
+        internal_cols_state = []
+        for col in df.columns:
+            if col.lower() not in internal_cols:
+                user_cols_state.append(row[col] == "#N/A" or row[col].__str__().lower() in ["none", "nan"])
+            else:
+                internal_cols_state.append(row[col] is not None)
+
+        if all(user_cols_state) and any(internal_cols_state):
+            rows_to_skip.append(i)
+
+    return rows_to_skip
+
+
 def _get_row_number(sheet: Worksheet, values_to_find: list[str]) -> int | None:
     for row_number, row in enumerate(sheet.iter_rows(values_only=True), start=1):
         if any(value in row for value in values_to_find):

cognite/neat/_utils/text.py

@@ -3,7 +3,7 @@ from collections.abc import Collection
 from typing import Any
 
 
-def to_camel(string: str) -> str:
+def to_camel_case(string: str) -> str:
     """Convert snake_case_name to camelCaseName.
 
     Args:
@@ -12,22 +12,34 @@ def to_camel(string: str) -> str:
         camelCase of the input string.
 
     Examples:
-        >>> to_camel("a_b")
+        >>> to_camel_case("a_b")
         'aB'
-        >>> to_camel("ScenarioInstance_priceForecast")
+        >>> to_camel_case("ScenarioInstance_priceForecast")
         'scenarioInstancePriceForecast'
     """
-    string = re.sub(r"[\s_-]", "_", string)
+    string = re.sub(r"[^a-zA-Z0-9_]", "_", string)
     string = re.sub("_+", "_", string)
+    is_all_upper = string.upper() == string
+    is_first_upper = (
+        len(string) >= 2 and string[:2].upper() == string[:2] and "_" not in string[:2] and not is_all_upper
+    )
+    return _to_camel_case(string, is_all_upper, is_first_upper)
+
+
+def _to_camel_case(string, is_all_upper: bool, is_first_upper: bool):
     if "_" in string:
-        pascal_splits = [to_pascal(part) for part in string.split("_")]
+        pascal_splits = [
+            _to_pascal_case(part, is_all_upper, is_first_upper and no == 0)
+            for no, part in enumerate(string.split("_"), 0)
+        ]
     else:
         # Ensure pascal
-        string = string[0].upper() + string[1:]
+        if string:
+            string = string[0].upper() + string[1:]
         pascal_splits = [string]
     cleaned: list[str] = []
     for part in pascal_splits:
-        if part.upper() == part:
+        if part.upper() == part and is_all_upper:
             cleaned.append(part.capitalize())
         else:
             cleaned.append(part)
@@ -37,13 +49,16 @@ def to_camel(string: str) -> str:
         string_split.extend(re.findall(r"[A-Z][a-z0-9]*", part))
     if not string_split:
         string_split = [string]
-    try:
-        return string_split[0].casefold() + "".join(word.capitalize() for word in string_split[1:])
-    except IndexError:
+    if len(string_split) == 0:
         return ""
+    # The first word is a single letter, keep the original case
+    if is_first_upper:
+        return "".join(word for word in string_split)
+    else:
+        return string_split[0].casefold() + "".join(word for word in string_split[1:])
 
 
-def to_pascal(string: str) -> str:
+def to_pascal_case(string: str) -> str:
     """Convert string to PascalCaseName.
 
     Args:
@@ -52,16 +67,20 @@ def to_pascal(string: str) -> str:
         PascalCase of the input string.
 
     Examples:
-        >>> to_pascal("a_b")
+        >>> to_pascal_case("a_b")
         'AB'
-        >>> to_pascal('camel_case')
+        >>> to_pascal_case('camel_case')
         'CamelCase'
     """
-    camel = to_camel(string)
+    return _to_pascal_case(string, string == string.upper(), True)
+
+
+def _to_pascal_case(string: str, is_all_upper: bool, is_first_upper: bool) -> str:
+    camel = _to_camel_case(string, is_all_upper, is_first_upper)
     return f"{camel[0].upper()}{camel[1:]}" if camel else ""
 
 
-def to_snake(string: str) -> str:
+def to_snake_case(string: str) -> str:
     """
     Convert input string to snake_case
 
@@ -71,33 +90,33 @@ def to_snake(string: str) -> str:
         snake_case of the input string.
 
     Examples:
-        >>> to_snake("aB")
+        >>> to_snake_case("aB")
         'a_b'
-        >>> to_snake('CamelCase')
+        >>> to_snake_case('CamelCase')
         'camel_case'
-        >>> to_snake('camelCamelCase')
+        >>> to_snake_case('camelCamelCase')
         'camel_camel_case'
-        >>> to_snake('Camel2Camel2Case')
+        >>> to_snake_case('Camel2Camel2Case')
         'camel_2_camel_2_case'
-        >>> to_snake('getHTTPResponseCode')
+        >>> to_snake_case('getHTTPResponseCode')
         'get_http_response_code'
-        >>> to_snake('get200HTTPResponseCode')
+        >>> to_snake_case('get200HTTPResponseCode')
         'get_200_http_response_code'
-        >>> to_snake('getHTTP200ResponseCode')
+        >>> to_snake_case('getHTTP200ResponseCode')
         'get_http_200_response_code'
-        >>> to_snake('HTTPResponseCode')
+        >>> to_snake_case('HTTPResponseCode')
         'http_response_code'
-        >>> to_snake('ResponseHTTP')
+        >>> to_snake_case('ResponseHTTP')
         'response_http'
-        >>> to_snake('ResponseHTTP2')
+        >>> to_snake_case('ResponseHTTP2')
         'response_http_2'
-        >>> to_snake('Fun?!awesome')
+        >>> to_snake_case('Fun?!awesome')
         'fun_awesome'
-        >>> to_snake('Fun?!Awesome')
+        >>> to_snake_case('Fun?!Awesome')
         'fun_awesome'
-        >>> to_snake('10CoolDudes')
+        >>> to_snake_case('10CoolDudes')
         '10_cool_dudes'
-        >>> to_snake('20coolDudes')
+        >>> to_snake_case('20coolDudes')
         '20_cool_dudes'
     """
     pattern = re.compile(r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z][a-z]|\d|\W|$)|\d+")
@@ -117,7 +136,7 @@ def sentence_or_string_to_camel(string: str) -> str:
         except IndexError:
             return ""
     else:
-        return to_camel(string)
+        return to_camel_case(string)
 
 
 def replace_non_alphanumeric_with_underscore(text: str) -> str:
@@ -151,7 +170,7 @@ class NamingStandardization:
             # Underscore ensure that 'Class' it treated as a separate word
             # in the to_pascale function
             clean = f"Class_{clean}"
-        return to_pascal(clean)
+        return to_pascal_case(clean)
 
     @classmethod
     def standardize_property_str(cls, raw: str) -> str:
@@ -160,7 +179,7 @@ class NamingStandardization:
             # Underscore ensure that 'property' it treated as a separate word
             # in the to_camel function
             clean = f"property_{clean}"
-        return to_camel(clean)
+        return to_camel_case(clean)
 
     @classmethod
     def _clean_string(cls, raw: str) -> str:

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.109.3"
+__version__ = "0.110.0"
 __engine__ = "^2.0.3"

{cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.109.3
+Version: 0.110.0
 Summary: Knowledge graph transformation
 License: Apache-2.0
 Author: Nikola Vasiljevic