cognite-neat 0.85.8__tar.gz → 0.85.10__tar.gz

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.85.8
+Version: 0.85.10
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0

cognite_neat-0.85.10/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.85.10"

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -41,7 +41,7 @@ class AssetsExtractor(BaseExtractor):
         to_type: Callable[[Asset], str | None] | None = None,
         total: int | None = None,
         limit: int | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.assets = assets
@@ -58,7 +58,7 @@ class AssetsExtractor(BaseExtractor):
         namespace: Namespace | None = None,
         to_type: Callable[[Asset], str | None] | None = None,
         limit: int | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         total = client.assets.aggregate_count(filter=AssetFilter(data_set_ids=[{"externalId": data_set_external_id}]))
 
@@ -82,7 +82,7 @@ class AssetsExtractor(BaseExtractor):
         namespace: Namespace | None = None,
         to_type: Callable[[Asset], str | None] | None = None,
         limit: int | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         total = client.assets.aggregate_count(
             filter=AssetFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
@@ -107,7 +107,7 @@ class AssetsExtractor(BaseExtractor):
         namespace: Namespace | None = None,
         to_type: Callable[[Asset], str] | None = None,
         limit: int | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(
             AssetList.load(Path(file_path).read_text()),

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/graph/extractors/_classic_cdf/_events.py

@@ -29,7 +29,7 @@ class EventsExtractor(BaseExtractor):
         self,
         events: Iterable[Event],
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.events = events
@@ -41,7 +41,7 @@ class EventsExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(
             cast(
@@ -57,7 +57,7 @@ class EventsExtractor(BaseExtractor):
         cls,
         file_path: str,
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(EventList.load(Path(file_path).read_text()), namespace, unpack_metadata)
 

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -30,7 +30,7 @@ class FilesExtractor(BaseExtractor):
         self,
         files_metadata: Iterable[FileMetadata],
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.files_metadata = files_metadata
@@ -42,7 +42,7 @@ class FilesExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(
             cast(
@@ -58,7 +58,7 @@ class FilesExtractor(BaseExtractor):
         cls,
         file_path: str,
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(
             FileMetadataList.load(Path(file_path).read_text()),

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/graph/extractors/_classic_cdf/_sequences.py

@@ -29,7 +29,7 @@ class SequencesExtractor(BaseExtractor):
         self,
         sequence: Iterable[Sequence],
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.sequence = sequence
@@ -41,7 +41,7 @@ class SequencesExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(
             cast(
@@ -57,7 +57,7 @@ class SequencesExtractor(BaseExtractor):
         cls,
         file_path: str,
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(SequenceList.load(Path(file_path).read_text()), namespace, unpack_metadata)
 

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/graph/extractors/_classic_cdf/_timeseries.py

@@ -29,7 +29,7 @@ class TimeSeriesExtractor(BaseExtractor):
         self,
         timeseries: Iterable[TimeSeries],
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.timeseries = timeseries
@@ -41,7 +41,7 @@ class TimeSeriesExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(
             cast(
@@ -57,7 +57,7 @@ class TimeSeriesExtractor(BaseExtractor):
         cls,
         file_path: str,
         namespace: Namespace | None = None,
-        unpack_metadata: bool = False,
+        unpack_metadata: bool = True,
     ):
         return cls(TimeSeriesList.load(Path(file_path).read_text()), namespace, unpack_metadata)
 

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/graph/queries/_base.py

@@ -2,6 +2,7 @@ import warnings
 from typing import Literal, cast, overload
 
 from rdflib import RDF, Graph, URIRef
+from rdflib import Literal as RdfLiteral
 from rdflib.query import ResultRow
 
 from cognite.neat.rules.models.entities import ClassEntity
@@ -18,6 +19,24 @@ class Queries:
         self.graph = graph
         self.rules = rules
 
+    def summarize_instances(self) -> list[tuple]:
+        """Summarize instances in the graph store by class and count"""
+
+        query_statement = """ SELECT ?class (COUNT(?instance) AS ?instanceCount)
+                             WHERE {
+                             ?instance a ?class .
+                             }
+                             GROUP BY ?class
+                             ORDER BY DESC(?instanceCount) """
+
+        return [
+            (
+                remove_namespace_from_uri(cast(URIRef, cast(tuple, res)[0])),
+                cast(RdfLiteral, cast(tuple, res)[1]).value,
+            )
+            for res in list(self.graph.query(query_statement))
+        ]
+
     def list_instances_ids_of_class(self, class_uri: URIRef, limit: int = -1) -> list[URIRef]:
         """Get instances ids for a given class
 
@@ -68,7 +87,10 @@ class Queries:
             # We cannot include the RDF.type in case there is a neat:type property
             return [remove_namespace_from_uri(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
         else:
-            warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
+            warnings.warn(
+                "No rules found for the graph store, returning empty list.",
+                stacklevel=2,
+            )
             return []
 
     def construct_instances_of_class(self, class_: str, properties_optional: bool = True) -> list[tuple[str, str, str]]:
@@ -95,7 +117,10 @@ class Queries:
             # We cannot include the RDF.type in case there is a neat:type property
             return [remove_namespace_from_uri(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
         else:
-            warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
+            warnings.warn(
+                "No rules found for the graph store, returning empty list.",
+                stacklevel=2,
+            )
             return []
 
     def list_triples(self, limit: int = 25) -> list[ResultRow]:

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/graph/stores/_base.py

@@ -5,6 +5,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast
 
+import pandas as pd
 from rdflib import Graph, Namespace, URIRef
 from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
 
@@ -283,12 +284,23 @@ class NeatGraphStore:
                 )
             )
 
+    @property
+    def summary(self) -> pd.DataFrame:
+        return pd.DataFrame(self.queries.summarize_instances(), columns=["Type", "Occurrence"])
+
     def _repr_html_(self) -> str:
         provenance = self.provenance._repr_html_()
+        summary: pd.DataFrame = self.summary
+        summary_text = (
+            "<br /><strong>Graph is empty</strong><br />"
+            if summary.empty
+            else f"<br /><strong>Graph content</strong><br />{cast(pd.DataFrame, summary)._repr_html_()}"  # type: ignore[operator]
+        )
 
         return (
             f"<strong>{type(self).__name__}</strong> A graph store is a container for storing triples. "
             "It can be queried and transformed to extract information.<br />"
             "<strong>Provenance</strong> Provenance is a record of changes that have occurred in the graph store.<br />"
             f"{provenance}"
+            f"{summary_text}"
         )

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/graph/transformers/_classic_cdf.py

@@ -1,6 +1,6 @@
 from typing import cast
 
-from rdflib import Graph, Literal, URIRef
+from rdflib import RDF, Graph, Literal, URIRef
 
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph import extractors
@@ -21,11 +21,16 @@ class AddAssetDepth(BaseTransformer):
                              <{asset_id}> <{root_prop}> ?root .}}"""
 
     def __init__(
-        self, asset_type: URIRef | None = None, root_prop: URIRef | None = None, parent_prop: URIRef | None = None
+        self,
+        asset_type: URIRef | None = None,
+        root_prop: URIRef | None = None,
+        parent_prop: URIRef | None = None,
+        depth_typing: dict[int, str] | None = None,
     ):
         self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
         self.root_prop = root_prop or DEFAULT_NAMESPACE.root
         self.parent_prop = parent_prop or DEFAULT_NAMESPACE.parent
+        self.depth_typing = depth_typing
 
     def transform(self, graph: Graph) -> None:
         """Adds depth of asset in the asset hierarchy to the graph."""
@@ -34,6 +39,13 @@ class AddAssetDepth(BaseTransformer):
             if depth := self.get_depth(graph, asset_id, self.root_prop, self.parent_prop):
                 graph.add((asset_id, DEFAULT_NAMESPACE.depth, Literal(depth)))
 
+                if self.depth_typing and (type_ := self.depth_typing.get(depth, None)):
+                    # remove existing type
+                    graph.remove((asset_id, RDF.type, None))
+
+                    # add new type
+                    graph.add((asset_id, RDF.type, DEFAULT_NAMESPACE[type_]))
+
     @classmethod
     def get_depth(
         cls,
@@ -62,7 +74,12 @@ class AddAssetDepth(BaseTransformer):
 class AssetTimeSeriesConnector(BaseTransformer):
     description: str = "Connects assets to timeseries, thus forming bi-directional connection"
     _use_only_once: bool = True
-    _need_changes = frozenset({str(extractors.AssetsExtractor.__name__), str(extractors.TimeSeriesExtractor.__name__)})
+    _need_changes = frozenset(
+        {
+            str(extractors.AssetsExtractor.__name__),
+            str(extractors.TimeSeriesExtractor.__name__),
+        }
+    )
     _asset_template: str = """SELECT ?asset_id WHERE {{
                               <{timeseries_id}> <{asset_prop}> ?asset_id .
                               ?asset_id a <{asset_type}>}}"""
@@ -100,7 +117,12 @@ class AssetTimeSeriesConnector(BaseTransformer):
 class AssetSequenceConnector(BaseTransformer):
     description: str = "Connects assets to sequences, thus forming bi-directional connection"
     _use_only_once: bool = True
-    _need_changes = frozenset({str(extractors.AssetsExtractor.__name__), str(extractors.SequencesExtractor.__name__)})
+    _need_changes = frozenset(
+        {
+            str(extractors.AssetsExtractor.__name__),
+            str(extractors.SequencesExtractor.__name__),
+        }
+    )
     _asset_template: str = """SELECT ?asset_id WHERE {{
                               <{sequence_id}> <{asset_prop}> ?asset_id .
                               ?asset_id a <{asset_type}>}}"""
@@ -138,7 +160,12 @@ class AssetSequenceConnector(BaseTransformer):
 class AssetFileConnector(BaseTransformer):
     description: str = "Connects assets to files, thus forming bi-directional connection"
     _use_only_once: bool = True
-    _need_changes = frozenset({str(extractors.AssetsExtractor.__name__), str(extractors.FilesExtractor.__name__)})
+    _need_changes = frozenset(
+        {
+            str(extractors.AssetsExtractor.__name__),
+            str(extractors.FilesExtractor.__name__),
+        }
+    )
     _asset_template: str = """SELECT ?asset_id WHERE {{
                               <{file_id}> <{asset_prop}> ?asset_id .
                               ?asset_id a <{asset_type}>}}"""
@@ -174,7 +201,12 @@ class AssetFileConnector(BaseTransformer):
 class AssetEventConnector(BaseTransformer):
     description: str = "Connects assets to events, thus forming bi-directional connection"
     _use_only_once: bool = True
-    _need_changes = frozenset({str(extractors.AssetsExtractor.__name__), str(extractors.EventsExtractor.__name__)})
+    _need_changes = frozenset(
+        {
+            str(extractors.AssetsExtractor.__name__),
+            str(extractors.EventsExtractor.__name__),
+        }
+    )
     _asset_template: str = """SELECT ?asset_id WHERE {{
                               <{event_id}> <{asset_prop}> ?asset_id .
                               ?asset_id a <{asset_type}>}}"""
@@ -211,7 +243,10 @@ class AssetRelationshipConnector(BaseTransformer):
     description: str = "Connects assets via relationships"
     _use_only_once: bool = True
     _need_changes = frozenset(
-        {str(extractors.AssetsExtractor.__name__), str(extractors.RelationshipsExtractor.__name__)}
+        {
+            str(extractors.AssetsExtractor.__name__),
+            str(extractors.RelationshipsExtractor.__name__),
+        }
     )
     _asset_template: str = """SELECT ?source ?target WHERE {{
                               <{relationship_id}> <{relationship_source_xid_prop}> ?source_xid .
@@ -256,8 +291,20 @@ class AssetRelationshipConnector(BaseTransformer):
                 # files can be connected to multiple assets in the graph
                 for source_asset_id, target_asset_id in cast(list[tuple], assets_id_res):
                     # create a relationship between the two assets
-                    graph.add((source_asset_id, DEFAULT_NAMESPACE.relationship, relationship_id))
-                    graph.add((target_asset_id, DEFAULT_NAMESPACE.relationship, relationship_id))
+                    graph.add(
+                        (
+                            source_asset_id,
+                            DEFAULT_NAMESPACE.relationship,
+                            relationship_id,
+                        )
+                    )
+                    graph.add(
+                        (
+                            target_asset_id,
+                            DEFAULT_NAMESPACE.relationship,
+                            relationship_id,
+                        )
+                    )
 
                     # add source and target to the relationship
                     graph.add((relationship_id, DEFAULT_NAMESPACE.source, source_asset_id))

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/rules/importers/_inference2rules.py

@@ -17,7 +17,11 @@ from cognite.neat.rules.models.information import (
     InformationMetadata,
     InformationRulesInput,
 )
-from cognite.neat.utils.utils import get_namespace, remove_namespace_from_uri, uri_to_short_form
+from cognite.neat.utils.utils import (
+    get_namespace,
+    remove_namespace_from_uri,
+    uri_to_short_form,
+)
 
 ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
                            WHERE { ?s a ?class . }
@@ -25,9 +29,21 @@ ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
 
 INSTANCES_OF_CLASS_QUERY = """SELECT ?s WHERE { ?s a <class> . }"""
 
+INSTANCE_PROPERTIES_JSON_DEFINITION = """SELECT ?property (count(?property) as ?occurrence) ?dataType ?objectType
+                                    WHERE {<instance_id> ?property ?value .
+
+                                           BIND(IF(REGEX(?value, "^\u007b(.*)\u007d$"),
+                                           <http://www.w3.org/2001/XMLSchema#json>,
+                                           datatype(?value)) AS ?dataType)
+
+                                           OPTIONAL {?value rdf:type ?objectType .}}
+                                    GROUP BY ?property ?dataType ?objectType"""
+
 INSTANCE_PROPERTIES_DEFINITION = """SELECT ?property (count(?property) as ?occurrence) ?dataType ?objectType
                                     WHERE {<instance_id> ?property ?value .
+
                                            BIND(datatype(?value) AS ?dataType)
+
                                            OPTIONAL {?value rdf:type ?objectType .}}
                                     GROUP BY ?property ?dataType ?objectType"""
 
@@ -44,27 +60,48 @@ class InferenceImporter(BaseImporter):
         graph: Knowledge graph
         max_number_of_instance: Maximum number of instances to be used in inference
         prefix: Prefix to be used for the inferred model
+        check_for_json_string: Check if values are JSON strings
     """
 
     def __init__(
-        self, issue_list: IssueList, graph: Graph, max_number_of_instance: int = -1, prefix: str = "inferred"
+        self,
+        issue_list: IssueList,
+        graph: Graph,
+        max_number_of_instance: int = -1,
+        prefix: str = "inferred",
+        check_for_json_string: bool = False,
     ) -> None:
         self.issue_list = issue_list
         self.graph = graph
         self.max_number_of_instance = max_number_of_instance
         self.prefix = prefix
+        self.check_for_json_string = check_for_json_string
 
     @classmethod
     def from_graph_store(
-        cls, store: NeatGraphStore, max_number_of_instance: int = -1, prefix: str = "inferred"
+        cls,
+        store: NeatGraphStore,
+        max_number_of_instance: int = -1,
+        prefix: str = "inferred",
+        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         issue_list = IssueList(title="Inferred from graph store")
 
-        return cls(issue_list, store.graph, max_number_of_instance=max_number_of_instance, prefix=prefix)
+        return cls(
+            issue_list,
+            store.graph,
+            max_number_of_instance=max_number_of_instance,
+            prefix=prefix,
+            check_for_json_string=check_for_json_string,
+        )
 
     @classmethod
     def from_rdf_file(
-        cls, filepath: Path, max_number_of_instance: int = -1, prefix: str = "inferred"
+        cls,
+        filepath: Path,
+        max_number_of_instance: int = -1,
+        prefix: str = "inferred",
+        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         issue_list = IssueList(title=f"'{filepath.name}'")
 
@@ -74,23 +111,41 @@ class InferenceImporter(BaseImporter):
         except Exception:
             issue_list.append(issues.fileread.FileReadError(filepath))
 
-        return cls(issue_list, graph, max_number_of_instance=max_number_of_instance, prefix=prefix)
+        return cls(
+            issue_list,
+            graph,
+            max_number_of_instance=max_number_of_instance,
+            prefix=prefix,
+            check_for_json_string=check_for_json_string,
+        )
 
     @classmethod
     def from_json_file(
-        cls, filepath: Path, max_number_of_instance: int = -1, prefix: str = "inferred"
+        cls,
+        filepath: Path,
+        max_number_of_instance: int = -1,
+        prefix: str = "inferred",
+        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         raise NotImplementedError("JSON file format is not supported yet.")
 
     @classmethod
     def from_yaml_file(
-        cls, filepath: Path, max_number_of_instance: int = -1, prefix: str = "inferred"
+        cls,
+        filepath: Path,
+        max_number_of_instance: int = -1,
+        prefix: str = "inferred",
+        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         raise NotImplementedError("YAML file format is not supported yet.")
 
     @classmethod
     def from_xml_file(
-        cls, filepath: Path, max_number_of_instance: int = -1, prefix: str = "inferred"
+        cls,
+        filepath: Path,
+        max_number_of_instance: int = -1,
+        prefix: str = "inferred",
+        check_for_json_string: bool = False,
     ) -> "InferenceImporter":
         raise NotImplementedError("JSON file format is not supported yet.")
 
@@ -149,6 +204,7 @@ class InferenceImporter(BaseImporter):
         properties: dict[str, dict] = {}
         prefixes: dict[str, Namespace] = PREFIXES.copy()
 
+        query = INSTANCE_PROPERTIES_JSON_DEFINITION if self.check_for_json_string else INSTANCE_PROPERTIES_DEFINITION
         # Adds default namespace to prefixes
         prefixes[self._default_metadata().prefix] = self._default_metadata().namespace
 
@@ -176,7 +232,7 @@ class InferenceImporter(BaseImporter):
                 + f" LIMIT {self.max_number_of_instance}"
             ):
                 for property_uri, occurrence, data_type_uri, object_type_uri in self.graph.query(  # type: ignore[misc]
-                    INSTANCE_PROPERTIES_DEFINITION.replace("instance_id", instance)
+                    query.replace("instance_id", instance)
                 ):  # type: ignore[misc]
                     property_id = remove_namespace_from_uri(property_uri)
                     self._add_uri_namespace_to_prefixes(cast(URIRef, property_uri), prefixes)

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/cognite/neat/rules/models/data_types.py

@@ -267,7 +267,7 @@ class Json(DataType):
     python = dms.Json
     dms = dms.Json
     graphql = "Json"
-    xsd = "string"
+    xsd = "json"
     sql = "STRING"
 
 

{cognite_neat-0.85.8 → cognite_neat-0.85.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.85.8"
+version = "0.85.10"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.85.8/cognite/neat/_version.py

@@ -1 +0,0 @@
-__version__ = "0.85.8"