cognite-neat 0.108.0__tar.gz → 0.109.0__tar.gz

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.108.0
+Version: 0.109.0
 Summary: Knowledge graph transformation
 License: Apache-2.0
 Author: Nikola Vasiljevic

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_constants.py

@@ -86,8 +86,8 @@ class NEAT(DefinedNamespace):
     _fail = True
     _NS = Namespace("http://thisisneat.io/internal/")
 
-    type: URIRef  # type property used to express a type of a subject
     UnknownType: URIRef  # Unknown type used to express that the type of a subject is unknown
+    EmptyType: URIRef  # Empty type used to express that the type of a subject is empty
 
 
 def get_default_prefixes_and_namespaces() -> dict[str, Namespace]:

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_graph/extractors/_classic_cdf/_classic.py

@@ -369,8 +369,12 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
 
     def _lookup_dataset(self, dataset_id: int) -> str:
         if dataset_id not in self._dataset_external_ids_by_id:
-            if (dataset := self._client.data_sets.retrieve(id=dataset_id)) and dataset.external_id:
-                self._dataset_external_ids_by_id[dataset_id] = dataset.external_id
-            else:
-                raise KeyError(f"Could not find dataset with id {dataset_id}.")
+            try:
+                if (dataset := self._client.data_sets.retrieve(id=dataset_id)) and dataset.external_id:
+                    self._dataset_external_ids_by_id[dataset_id] = dataset.external_id
+                else:
+                    raise KeyError(f"Could not find dataset with id {dataset_id}.")
+            except CogniteAPIError as e:
+                warnings.warn(CDFAuthWarning("lookup dataset", str(e)), stacklevel=2)
+                return f"{InstanceIdPrefix.data_set}{dataset_id}"
         return self._dataset_external_ids_by_id[dataset_id]

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_graph/queries/_base.py

@@ -185,6 +185,10 @@ class Queries:
         query = f"ASK WHERE {{ ?s ?p ?o . FILTER(STRSTARTS(STR(?p), STR(<{namespace}>))) }}"
         return bool(self.graph(named_graph).query(query))
 
+    def has_data(self) -> bool:
+        """Check if the graph store has data"""
+        return cast(bool, next(iter(self.dataset.query("ASK WHERE { ?s ?p ?o }"))))
+
     def has_type(self, type_: URIRef, named_graph: URIRef | None = None) -> bool:
         """Check if a type exists in the graph store
 

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_graph/transformers/__init__.py

@@ -17,7 +17,7 @@ from ._prune_graph import (
     PruneTypes,
 )
 from ._rdfpath import AddSelfReferenceProperty, MakeConnectionOnExactMatch
-from ._value_type import ConnectionToLiteral, ConvertLiteral, LiteralToEntity, SetNeatType, SplitMultiValueProperty
+from ._value_type import ConnectionToLiteral, ConvertLiteral, LiteralToEntity, SetType, SplitMultiValueProperty
 
 __all__ = [
     "AddAssetDepth",
@@ -38,7 +38,7 @@ __all__ = [
     "PruneInstancesOfUnknownType",
     "PruneTypes",
     "RelationshipAsEdgeTransformer",
-    "SetNeatType",
+    "SetType",
     "SplitMultiValueProperty",
 ]
 
@@ -63,5 +63,5 @@ Transformers = (
     | ConnectionToLiteral
     | BaseTransformerStandardised
     | LookupRelationshipSourceTarget
-    | SetNeatType
+    | SetType
 )

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_graph/transformers/_base.py

@@ -12,14 +12,14 @@ from cognite.neat._shared import Triple
 from cognite.neat._utils.collection_ import iterate_progress_bar_if_above_config_threshold
 from cognite.neat._utils.graph_transformations_report import GraphTransformationResult
 
-To_Add_Triples: TypeAlias = list[Triple]
-To_Remove_Triples: TypeAlias = list[Triple]
+To_Add_Triples: TypeAlias = set[Triple]
+To_Remove_Triples: TypeAlias = set[Triple]
 
 
 @dataclasses.dataclass
 class RowTransformationOutput:
-    remove_triples: To_Remove_Triples = dataclasses.field(default_factory=list)
-    add_triples: To_Add_Triples = dataclasses.field(default_factory=list)
+    remove_triples: To_Remove_Triples = dataclasses.field(default_factory=set)
+    add_triples: To_Add_Triples = dataclasses.field(default_factory=set)
     instances_removed_count: int = 0
     instances_added_count: int = 0
     instances_modified_count: int = 0

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_graph/transformers/_classic_cdf.py

@@ -63,11 +63,11 @@ class AddAssetDepth(BaseTransformerStandardised):
         row_output = RowTransformationOutput()
         subject, object = query_result_row
 
-        row_output.add_triples.append(cast(Triple, (subject, DEFAULT_NAMESPACE.depth, object)))
+        row_output.add_triples.add(cast(Triple, (subject, DEFAULT_NAMESPACE.depth, object)))
 
         if self.depth_typing and (type_ := self.depth_typing.get(int(object), None)):
-            row_output.remove_triples.append(cast(Triple, (subject, RDF.type, self.asset_type)))
-            row_output.add_triples.append(cast(Triple, (subject, RDF.type, DEFAULT_NAMESPACE[type_])))
+            row_output.remove_triples.add(cast(Triple, (subject, RDF.type, self.asset_type)))
+            row_output.add_triples.add(cast(Triple, (subject, RDF.type, DEFAULT_NAMESPACE[type_])))
 
         row_output.instances_modified_count += 1
 
@@ -128,7 +128,7 @@ class BaseAssetConnector(BaseTransformerStandardised, ABC):
         row_output = RowTransformationOutput()
         subject, object = query_result_row
 
-        row_output.add_triples.append(cast(Triple, (subject, self.asset_to_resource_connection, object)))
+        row_output.add_triples.add(cast(Triple, (subject, self.asset_to_resource_connection, object)))
 
         row_output.instances_modified_count += 1
 
@@ -305,12 +305,12 @@ class AssetRelationshipConnector(BaseTransformerStandardised):
         row_output = RowTransformationOutput()
         source, relationship, target = query_result_row
 
-        row_output.add_triples.append(cast(Triple, (source, DEFAULT_NAMESPACE.relationship, target)))
-        row_output.add_triples.append(cast(Triple, (relationship, DEFAULT_NAMESPACE.source, source)))
-        row_output.add_triples.append(cast(Triple, (relationship, DEFAULT_NAMESPACE.target, target)))
+        row_output.add_triples.add(cast(Triple, (source, DEFAULT_NAMESPACE.relationship, target)))
+        row_output.add_triples.add(cast(Triple, (relationship, DEFAULT_NAMESPACE.source, source)))
+        row_output.add_triples.add(cast(Triple, (relationship, DEFAULT_NAMESPACE.target, target)))
 
-        row_output.remove_triples.append(cast(Triple, (relationship, self.relationship_source_xid_prop, None)))
-        row_output.remove_triples.append(cast(Triple, (relationship, self.relationship_target_xid_prop, None)))
+        row_output.remove_triples.add(cast(Triple, (relationship, self.relationship_source_xid_prop, None)))
+        row_output.remove_triples.add(cast(Triple, (relationship, self.relationship_target_xid_prop, None)))
 
         row_output.instances_modified_count += 2
 
@@ -578,10 +578,10 @@ WHERE {{
             warnings.warn(ResourceNotFoundWarning(target, "class", str(instance), "class"), stacklevel=2)
             return output
 
-        output.remove_triples.append((instance, self._namespace.sourceExternalId, source))
-        output.remove_triples.append((instance, self._namespace.targetExternalId, target))
-        output.add_triples.append((instance, self._namespace.sourceExternalId, source_id))
-        output.add_triples.append((instance, self._namespace.targetExternalId, target_id))
+        output.remove_triples.add((instance, self._namespace.sourceExternalId, source))
+        output.remove_triples.add((instance, self._namespace.targetExternalId, target))
+        output.add_triples.add((instance, self._namespace.sourceExternalId, source_id))
+        output.add_triples.add((instance, self._namespace.targetExternalId, target_id))
         output.instances_modified_count += 1
         return output
 

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_graph/transformers/_prune_graph.py

@@ -237,7 +237,7 @@ class PruneTypes(BaseTransformerStandardised):
         row_output = RowTransformationOutput()
 
         (subject,) = query_result_row
-        row_output.remove_triples.append((subject, None, None))  # type: ignore
+        row_output.remove_triples.add((subject, None, None))  # type: ignore
         row_output.instances_removed_count = 1
 
         return row_output
@@ -272,7 +272,7 @@ class PruneDeadEndEdges(BaseTransformerStandardised):
 
     def operation(self, row: ResultRow) -> RowTransformationOutput:
         row_output = RowTransformationOutput()
-        row_output.remove_triples.append(cast(Triple, row))
+        row_output.remove_triples.add(cast(Triple, row))
         row_output.instances_modified_count = 1
 
         return row_output
@@ -307,7 +307,7 @@ class PruneInstancesOfUnknownType(BaseTransformerStandardised):
     def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
         row_output = RowTransformationOutput()
         (subject,) = query_result_row
-        row_output.remove_triples.append(cast(Triple, (subject, None, None)))
+        row_output.remove_triples.add(cast(Triple, (subject, None, None)))
         row_output.instances_removed_count = 1
 
         return row_output

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_graph/transformers/_rdfpath.py

@@ -7,7 +7,6 @@ from rdflib.query import ResultRow
 from cognite.neat._rules.analysis import InformationAnalysis
 from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
 from cognite.neat._rules.models.information import InformationRules
-from cognite.neat._shared import Triple
 from cognite.neat._utils.rdf_ import get_namespace, remove_namespace_from_uri
 
 from ._base import BaseTransformer, BaseTransformerStandardised, RowTransformationOutput
@@ -76,7 +75,7 @@ class MakeConnectionOnExactMatch(BaseTransformerStandardised):
         self.object_type = object_type
         self.object_predicate = object_predicate
         subject_namespace = Namespace(get_namespace(subject_type))
-        self.connection = (
+        self.connection: URIRef = (
             subject_namespace[quote(connection.strip())]
             if isinstance(connection, str)
             else connection or subject_namespace[remove_namespace_from_uri(self.object_type).lower()]
@@ -125,8 +124,8 @@ class MakeConnectionOnExactMatch(BaseTransformerStandardised):
     def operation(self, query_result_row: ResultRow) -> RowTransformationOutput:
         row_output = RowTransformationOutput()
 
-        subject, object = query_result_row
+        subject, object = cast(tuple[URIRef, URIRef], query_result_row)
 
-        row_output.add_triples.append(cast(Triple, (subject, self.connection, object)))
+        row_output.add_triples.add((subject, self.connection, object))
         row_output.instances_modified_count += 1
         return row_output

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_graph/transformers/_value_type.py

@@ -4,7 +4,7 @@ from typing import Any, cast
 from urllib.parse import quote
 
 import rdflib
-from rdflib import RDF, Literal, Namespace, URIRef
+from rdflib import RDF, RDFS, Literal, Namespace, URIRef
 from rdflib.query import ResultRow
 
 from cognite.neat._constants import NEAT
@@ -78,8 +78,8 @@ class SplitMultiValueProperty(BaseTransformerStandardised):
 
         new_property = URIRef(f"{old_property}_{remove_namespace_from_uri(value_type)}")
 
-        row_output.add_triples.append(cast(Triple, (subject, new_property, object)))
-        row_output.remove_triples.append(cast(Triple, (subject, old_property, object)))
+        row_output.add_triples.add(cast(Triple, (subject, new_property, object)))
+        row_output.remove_triples.add(cast(Triple, (subject, old_property, object)))
 
         row_output.instances_modified_count += 1
 
@@ -143,8 +143,8 @@ class ConvertLiteral(BaseTransformerStandardised):
                 PropertyDataTypeConversionWarning(str(instance), self._type_name, self._property_name, str(e)),
                 stacklevel=2,
             )
-        row_output.add_triples.append((instance, self.subject_predicate, rdflib.Literal(converted_value)))  # type: ignore[arg-type]
-        row_output.remove_triples.append((instance, self.subject_predicate, literal))  # type: ignore[arg-type]
+        row_output.add_triples.add((instance, self.subject_predicate, rdflib.Literal(converted_value)))  # type: ignore[arg-type]
+        row_output.remove_triples.add((instance, self.subject_predicate, literal))  # type: ignore[arg-type]
         row_output.instances_modified_count += 1
 
         return row_output
@@ -221,15 +221,15 @@ class LiteralToEntity(BaseTransformerStandardised):
         namespace = Namespace(get_namespace(instance))  # type: ignore[arg-type]
         entity_type = namespace[self.entity_type]
         new_entity = namespace[f"{self.entity_type}_{quote(value)!s}"]
-        row_output.add_triples.append((new_entity, RDF.type, entity_type))
+        row_output.add_triples.add((new_entity, RDF.type, entity_type))
         row_output.instances_added_count += 1  # we add one new entity
 
         if self.new_property is not None:
-            row_output.add_triples.append((new_entity, namespace[self.new_property], rdflib.Literal(value)))  # type: ignore[arg-type]
+            row_output.add_triples.add((new_entity, namespace[self.new_property], rdflib.Literal(value)))  # type: ignore[arg-type]
             row_output.instances_modified_count += 1  # we modify the new entity
 
-        row_output.add_triples.append((instance, self.subject_predicate, new_entity))  # type: ignore[arg-type]
-        row_output.remove_triples.append((instance, self.subject_predicate, literal))  # type: ignore[arg-type]
+        row_output.add_triples.add((instance, self.subject_predicate, new_entity))  # type: ignore[arg-type]
+        row_output.remove_triples.add((instance, self.subject_predicate, literal))  # type: ignore[arg-type]
         row_output.instances_modified_count += 1  # we modify the old entity
 
         return row_output
@@ -300,18 +300,22 @@ class ConnectionToLiteral(BaseTransformerStandardised):
         instance, object_entity = cast(tuple[URIRef, URIRef], query_result_row)
         value = remove_namespace_from_uri(object_entity)
 
-        row_output.add_triples.append((instance, self.subject_predicate, rdflib.Literal(value)))
-        row_output.remove_triples.append((instance, self.subject_predicate, object_entity))
+        row_output.add_triples.add((instance, self.subject_predicate, rdflib.Literal(value)))
+        row_output.remove_triples.add((instance, self.subject_predicate, object_entity))
         row_output.instances_modified_count += 1
 
         return row_output
 
 
-class SetNeatType(BaseTransformerStandardised):
-    description = "Set the sub type of an instance based on the property"
+class SetType(BaseTransformerStandardised):
+    description = "Set the type of an instance based on a property"
 
     def __init__(
-        self, subject_type: URIRef, subject_predicate: URIRef, drop_property: bool, namespace: Namespace | None = None
+        self,
+        subject_type: URIRef,
+        subject_predicate: URIRef,
+        drop_property: bool = False,
+        namespace: Namespace | None = None,
     ) -> None:
         self.subject_type = subject_type
         self.subject_predicate = subject_predicate
@@ -350,9 +354,12 @@ class SetNeatType(BaseTransformerStandardised):
 
         instance, object_literal = cast(tuple[URIRef, Literal], query_result_row)
         if self.drop_property:
-            row_output.remove_triples.append((instance, self.subject_predicate, object_literal))
+            row_output.remove_triples.add((instance, self.subject_predicate, object_literal))
 
-        row_output.add_triples.append((instance, NEAT.type, self._namespace[str(object_literal.toPython())]))
+        row_output.remove_triples.add((instance, RDF.type, self.subject_type))
+        new_type = self._namespace[quote(object_literal.toPython())]
+        row_output.add_triples.add((instance, RDF.type, new_type))
+        row_output.add_triples.add((new_type, RDFS.subClassOf, self.subject_type))
         row_output.instances_modified_count += 1
 
         return row_output

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_issues/errors/__init__.py

@@ -9,6 +9,7 @@ from ._external import (
     FileReadError,
     FileTypeUnexpectedError,
     NeatYamlError,
+    OxigraphStorageLockedError,
 )
 from ._general import NeatImportError, NeatTypeError, NeatValueError, RegexViolationError
 from ._properties import (
@@ -51,6 +52,7 @@ __all__ = [
     "NeatTypeError",
     "NeatValueError",
     "NeatYamlError",
+    "OxigraphStorageLockedError",
     "PropertyDefinitionDuplicatedError",
     "PropertyDefinitionError",
     "PropertyMappingDuplicatedError",

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_issues/errors/_external.py

@@ -23,6 +23,14 @@ class FileReadError(NeatError, RuntimeError):
     reason: str
 
 
+@dataclass(unsafe_hash=True)
+class OxigraphStorageLockedError(NeatError, RuntimeError):
+    """Oxigraph on-disk storage locked at the path {filepath}"""
+
+    fix = "Delete LOCK file in the {filepath} folder to unlock the storage or use another storage path"
+    filepath: Path
+
+
 @dataclass(unsafe_hash=True)
 class FileNotFoundNeatError(NeatError, FileNotFoundError):
     """File {filepath} not found"""

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_issues/warnings/_resources.py

@@ -21,7 +21,7 @@ class ResourceRegexViolationWarning(ResourceNeatWarning):
 
     fix = (
         "Either export the data model and make the necessary changes manually"
-        " or run prepare.data_model.cdf_compliant_external_ids."
+        " or run fix.data_model.cdf_compliant_external_ids."
     )
 
     location: str

{cognite_neat-0.108.0 → cognite_neat-0.109.0}/cognite/neat/_rules/exporters/_rules2yaml.py

@@ -72,7 +72,7 @@ class YAMLExporter(BaseExporter[VerifiedRules, str]):
         """
         # model_dump_json ensures that the output is in JSON format,
         # if we don't do this, we will get Enums and other types that are not serializable to YAML
-        json_output = rules.dump(mode="json", exclude_none=True, exclude_unset=True)
+        json_output = rules.dump(mode="json", sort=True, exclude_none=True, exclude_unset=True)
         if self.output == "json":
             return json.dumps(json_output)
         elif self.output == "yaml":