cognite-neat 0.110.0__tar.gz → 0.111.0__tar.gz

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.110.0
+Version: 0.111.0
 Summary: Knowledge graph transformation
 License: Apache-2.0
 Author: Nikola Vasiljevic

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_alpha.py

@@ -15,3 +15,9 @@ class AlphaFlags:
     standardize_naming = AlphaWarning("standardize_naming")
     standardize_space_and_version = AlphaWarning("standardize_space_and_version")
     data_model_subsetting = AlphaWarning("data_model_subsetting")
+    ontology_read = AlphaWarning("ontology_read")
+    imf_read = AlphaWarning("imf_read")
+    dexpi_read = AlphaWarning("dexpi_read")
+    aml_read = AlphaWarning("aml_read")
+    csv_read = AlphaWarning("csv_read")
+    to_ontology = AlphaWarning("to_ontology")

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_client/_api/schema.py

@@ -1,5 +1,6 @@
 from collections import defaultdict
 from collections.abc import Iterable, Sequence
+from graphlib import CycleError, TopologicalSorter
 from typing import TYPE_CHECKING
 
 from cognite.client import data_modeling as dm
@@ -126,3 +127,28 @@ class SchemaAPI:
                 if is_hierarchy_property(prop.container, prop.container_property_identifier):
                     hierarchical_properties_by_view_id[view.as_id()].add(prop_id)
         return hierarchical_properties_by_view_id
+
+    @staticmethod
+    def get_view_order_by_direct_relation_constraints(views: Iterable[dm.View]) -> list[dm.ViewId]:
+        """Sorts the views by container constraints."""
+        view_sequence = list(views)
+        view_ids_by_container: dict[dm.ContainerId, set[dm.ViewId]] = defaultdict(set)
+        for view in view_sequence:
+            for container_id in view.referenced_containers():
+                view_ids_by_container[container_id].add(view.as_id())
+
+        view_by_dependency: dict[dm.ViewId, set[dm.ViewId]] = {}
+        for view in view_sequence:
+            view_id = view.as_id()
+            view_by_dependency[view_id] = set()
+            for prop in view.properties.values():
+                if (
+                    isinstance(prop, dm.MappedProperty)
+                    and isinstance(prop.type, dm.DirectRelation)
+                    and prop.type.container
+                ):
+                    view_by_dependency[view_id].update(view_ids_by_container[prop.type.container])
+        try:
+            return list(TopologicalSorter(view_by_dependency).static_order())
+        except CycleError as e:
+            raise NeatValueError("Cycle in direct relation constraints") from e

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_client/data_classes/schema.py

@@ -167,7 +167,7 @@ class DMSSchema:
                     if "." not in filename.stem:
                         continue
                     resource_type = filename.stem.rsplit(".", 1)[-1]
-                    if attr_name := cls._FIELD_NAME_BY_RESOURCE_TYPE.get(resource_type):
+                    if attr_name := cls._FIELD_NAME_BY_RESOURCE_TYPE.get(resource_type.casefold()):
                         data.setdefault(attr_name, [])
                         context.setdefault(attr_name, [])
                         try:

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_constants.py

@@ -1,3 +1,4 @@
+import re
 from collections.abc import Mapping
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -69,7 +70,9 @@ EXAMPLE_RULES = PACKAGE_DIRECTORY / "_rules" / "examples"
 EXAMPLE_GRAPHS = PACKAGE_DIRECTORY / "_graph" / "examples"
 EXAMPLE_WORKFLOWS = PACKAGE_DIRECTORY / "_workflows" / "examples"
 
-DEFAULT_SPACE_URI = "http://purl.org/cognite/{space}#"
+DEFAULT_SPACE_URI = "http://purl.org/cognite/space/{space}#"
+SPACE_URI_PATTERN = re.compile(r"http://purl.org/cognite/space/(?P<space>[^#]+)#$")
+DEFAULT_RAW_URI = "http://purl.org/cognite/raw#"
 DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/neat/")
 CDF_NAMESPACE = Namespace("https://cognitedata.com/")
 DEFAULT_BASE_URI = URIRef(DEFAULT_NAMESPACE)

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/__init__.py

@@ -11,10 +11,12 @@ from ._classic_cdf._relationships import RelationshipsExtractor
 from ._classic_cdf._sequences import SequencesExtractor
 from ._classic_cdf._timeseries import TimeSeriesExtractor
 from ._dexpi import DexpiExtractor
+from ._dict import DictExtractor
 from ._dms import DMSExtractor
 from ._dms_graph import DMSGraphExtractor
 from ._iodd import IODDExtractor
 from ._mock_graph_generator import MockGraphGenerator
+from ._raw import RAWExtractor
 from ._rdf_file import RdfFileExtractor
 
 __all__ = [
@@ -25,12 +27,14 @@ __all__ = [
     "DMSGraphExtractor",
     "DataSetExtractor",
     "DexpiExtractor",
+    "DictExtractor",
     "EventsExtractor",
     "FilesExtractor",
     "IODDExtractor",
     "KnowledgeGraphExtractor",
     "LabelsExtractor",
     "MockGraphGenerator",
+    "RAWExtractor",
     "RdfFileExtractor",
     "RelationshipsExtractor",
     "SequencesExtractor",

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/_classic_cdf/_base.py

@@ -61,8 +61,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
     Args:
         items (Iterable[T_CogniteResource]): An iterable of classic resource.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
-        to_type (Callable[[T_CogniteResource], str | None], optional): A function to convert an item to a type.
-            Defaults to None. If None or if the function returns None, the asset will be set to the default type.
         total (int, optional): The total number of items to load. If passed, you will get a progress bar if rich
             is installed. Defaults to None.
         limit (int, optional): The maximal number of items to load. Defaults to None. This is typically used for
@@ -87,7 +85,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         self,
         items: Iterable[T_CogniteResource],
         namespace: Namespace | None = None,
-        to_type: Callable[[T_CogniteResource], str | None] | None = None,
         total: int | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
@@ -99,7 +96,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.items = items
-        self.to_type = to_type
         self.total = total
         self.limit = min(limit, total) if limit and total else limit
         self.unpack_metadata = unpack_metadata
@@ -161,7 +157,7 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         if self._log_urirefs and hasattr(item, "external_id"):
             self._uriref_by_external_id[item.external_id] = id_
 
-        type_ = self._get_rdf_type(item)
+        type_ = self._get_rdf_type()
 
         # Set rdf type
         triples: list[Triple] = [(id_, RDF.type, self.namespace[type_])]
@@ -215,16 +211,14 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
                 if value and (self.skip_metadata_values is None or value.casefold() not in self.skip_metadata_values):
                     yield (
                         id_,
-                        self.namespace[key],
+                        self.namespace[urllib.parse.quote(key)],
                         Literal(string_to_ideal_type(value)),
                     )
         else:
             yield id_, self.namespace.metadata, Literal(json.dumps(metadata), datatype=XSD._NS["json"])
 
-    def _get_rdf_type(self, item: T_CogniteResource) -> str:
+    def _get_rdf_type(self) -> str:
         type_ = self._default_rdf_type
-        if self.to_type:
-            type_ = self.to_type(item) or type_
         if self.prefix:
             type_ = f"{self.prefix}{type_}"
         return self._SPACE_PATTERN.sub("_", type_)
@@ -263,7 +257,11 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
             "created_time",
             "last_updated_time",
         } and isinstance(raw, int):
-            return Literal(datetime.fromtimestamp(raw / 1000, timezone.utc), datatype=XSD.dateTime)
+            try:
+                return Literal(datetime.fromtimestamp(raw / 1000, timezone.utc), datatype=XSD.dateTime)
+            except (OSError, ValueError) as e:
+                warnings.warn(NeatValueWarning(f"Failed to convert timestamp {raw} to datetime: {e!s}"), stacklevel=2)
+                return Literal(raw)
         elif key == "labels":
             from ._labels import LabelsExtractor
 
@@ -284,7 +282,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
-        to_type: Callable[[T_CogniteResource], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -297,7 +294,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -321,7 +317,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         client: CogniteClient,
         root_asset_external_id: str,
         namespace: Namespace | None = None,
-        to_type: Callable[[T_CogniteResource], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -334,7 +329,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -357,7 +351,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         cls,
         file_path: str | Path,
         namespace: Namespace | None = None,
-        to_type: Callable[[T_CogniteResource], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -370,7 +363,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/_classic_cdf/_classic.py

@@ -102,6 +102,8 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         limit_per_type: int | None = None,
         prefix: str | None = None,
         identifier: typing.Literal["id", "externalId"] = "id",
+        unpack_metadata: bool = False,
+        skip_sequence_rows: bool = False,
     ):
         self._client = client
         if sum([bool(data_set_external_id), bool(root_asset_external_id)]) != 1:
@@ -111,7 +113,7 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         self._namespace = namespace or CLASSIC_CDF_NAMESPACE
         self._extractor_args = dict(
             namespace=self._namespace,
-            unpack_metadata=False,
+            unpack_metadata=unpack_metadata,
             as_write=True,
             camel_case=True,
             limit=limit_per_type,
@@ -121,6 +123,7 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         self._identifier = identifier
         self._prefix = prefix
         self._limit_per_type = limit_per_type
+        self._skip_sequence_rows = skip_sequence_rows
 
         self._uris_by_external_id_by_type: dict[InstanceIdPrefix, dict[str, URIRef]] = defaultdict(dict)
         self._source_external_ids_by_type: dict[InstanceIdPrefix, set[str]] = defaultdict(set)
@@ -133,6 +136,29 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         self._extracted_data_sets = False
         self._asset_external_ids_by_id: dict[int, str] = {}
         self._dataset_external_ids_by_id: dict[int, str] = {}
+        self.neat_prefix_by_predicate_uri: dict[URIRef, str] = {
+            self._namespace["dataSetId"]: InstanceIdPrefix.data_set,
+            self._namespace["assetId"]: InstanceIdPrefix.asset,
+            self._namespace["assetIds"]: InstanceIdPrefix.asset,
+            self._namespace["parentId"]: InstanceIdPrefix.asset,
+            self._namespace["rootId"]: InstanceIdPrefix.asset,
+            self._namespace["labels"]: InstanceIdPrefix.label,
+        }
+        self.neat_prefix_by_type_uri: dict[URIRef, str] = {}
+        for extractor_cls, type_prefix in [
+            (AssetsExtractor, InstanceIdPrefix.asset),
+            (TimeSeriesExtractor, InstanceIdPrefix.time_series),
+            (SequencesExtractor, InstanceIdPrefix.sequence),
+            (EventsExtractor, InstanceIdPrefix.event),
+            (FilesExtractor, InstanceIdPrefix.file),
+            (RelationshipsExtractor, InstanceIdPrefix.relationship),
+            (LabelsExtractor, InstanceIdPrefix.label),
+            (DataSetExtractor, InstanceIdPrefix.data_set),
+        ]:
+            rdf_type = extractor_cls._default_rdf_type  # type: ignore[attr-defined]
+            if prefix:
+                rdf_type = f"{prefix}{rdf_type}"
+            self.neat_prefix_by_type_uri[self._namespace[rdf_type]] = type_prefix
 
     def _get_activity_names(self) -> list[str]:
         activities = [data_access_object.extractor_cls.__name__ for data_access_object in self._classic_node_types] + [
@@ -237,14 +263,14 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
 
     def _extract_core_start_nodes(self):
         for core_node in self._classic_node_types:
+            kwargs = self._extractor_args.copy()
+            if core_node.extractor_cls == SequencesExtractor and self._skip_sequence_rows:
+                kwargs["skip_rows"] = True
+
             if self._data_set_external_id:
-                extractor = core_node.extractor_cls.from_dataset(
-                    self._client, self._data_set_external_id, **self._extractor_args
-                )
+                extractor = core_node.extractor_cls.from_dataset(self._client, self._data_set_external_id, **kwargs)
             elif self._root_asset_external_id:
-                extractor = core_node.extractor_cls.from_hierarchy(
-                    self._client, self._root_asset_external_id, **self._extractor_args
-                )
+                extractor = core_node.extractor_cls.from_hierarchy(self._client, self._root_asset_external_id, **kwargs)
             else:
                 raise ValueError("Exactly one of data_set_external_id or root_asset_external_id must be set.")
 
@@ -273,7 +299,9 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
                 # This is a private attribute, but we need to set it to log the target nodes.
                 extractor._log_target_nodes = True
                 if self._identifier == "id":
-                    extractor._uri_by_external_id_by_by_type = self._uris_by_external_id_by_type
+                    extractor._uri_by_external_id_by_type = self._uris_by_external_id_by_type
+                elif self._identifier == "externalId":
+                    extractor.lookup_dataset_external_id = self._lookup_dataset
 
                 yield from extractor.extract()
 
@@ -330,7 +358,9 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
     def _extract_labels(self):
         for chunk in self._chunk(list(self._labels), description="Extracting labels"):
             label_iterator = self._client.labels.retrieve(external_id=list(chunk), ignore_unknown_ids=True)
-            yield from LabelsExtractor(label_iterator, **self._extractor_args).extract()
+            extractor = LabelsExtractor(label_iterator, **self._extractor_args)
+            extractor.lookup_dataset_external_id = self._lookup_dataset
+            yield from extractor.extract()
 
     def _extract_data_sets(self):
         for chunk in self._chunk(list(self._data_set_ids), description="Extracting data sets"):

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/_classic_cdf/_relationships.py

@@ -1,7 +1,7 @@
 import typing
 import warnings
 from collections import defaultdict
-from collections.abc import Callable, Iterable, Set
+from collections.abc import Iterable, Set
 from pathlib import Path
 from typing import Any
 
@@ -26,7 +26,6 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
         self,
         items: Iterable[Relationship],
         namespace: Namespace | None = None,
-        to_type: Callable[[Relationship], str | None] | None = None,
         total: int | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
@@ -46,7 +45,6 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
         super().__init__(
             to_iterate,
             namespace=namespace,
-            to_type=to_type,
             total=total,
             limit=limit,
             unpack_metadata=unpack_metadata,
@@ -56,7 +54,7 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
             prefix=prefix,
             identifier=identifier,
         )
-        self._uri_by_external_id_by_by_type: dict[InstanceIdPrefix, dict[str, URIRef]] = defaultdict(dict)
+        self._uri_by_external_id_by_type: dict[InstanceIdPrefix, dict[str, URIRef]] = defaultdict(dict)
         self._target_triples: list[tuple[URIRef, URIRef, str, str]] = []
 
     def _log_target_nodes_if_set(self, item: Relationship) -> Relationship:
@@ -67,25 +65,33 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
         return item
 
     def _item2triples_special_cases(self, id_: URIRef, dumped: dict[str, Any]) -> list[Triple]:
-        if self.identifier == "externalId":
-            return []
         triples: list[Triple] = []
         if (source_external_id := dumped.pop("sourceExternalId")) and "sourceType" in dumped:
             source_type = dumped["sourceType"]
-            try:
-                source_uri = self._uri_by_external_id_by_by_type[InstanceIdPrefix.from_str(source_type)][
-                    source_external_id
-                ]
-            except KeyError:
-                warnings.warn(
-                    NeatValueWarning(f"Missing externalId {source_external_id} for {source_type}"), stacklevel=2
+            source_prefix = InstanceIdPrefix.from_str(source_type)
+            if self.identifier == "id":
+                try:
+                    source_uri = self._uri_by_external_id_by_type[source_prefix][source_external_id]
+                except KeyError:
+                    warnings.warn(
+                        NeatValueWarning(f"Missing externalId {source_external_id} for {source_type}"), stacklevel=2
+                    )
+                else:
+                    triples.append((id_, self.namespace["sourceExternalId"], source_uri))
+            elif self.identifier == "externalId":
+                triples.append(
+                    (id_, self.namespace["sourceExternalId"], self.namespace[f"{source_prefix}{source_external_id}"])
                 )
-            else:
-                triples.append((id_, self.namespace["sourceExternalId"], source_uri))
         if (target_external_id := dumped.pop("targetExternalId")) and "targetType" in dumped:
             target_type = dumped["targetType"]
-            # We do not yet have the target nodes, so we log them for later extraction.
-            self._target_triples.append((id_, self.namespace["targetExternalId"], target_type, target_external_id))
+            if self.identifier == "id":
+                # We do not yet have the target nodes, so we log them for later extraction.
+                self._target_triples.append((id_, self.namespace["targetExternalId"], target_type, target_external_id))
+            elif self.identifier == "externalId":
+                target_prefix = InstanceIdPrefix.from_str(target_type)
+                triples.append(
+                    (id_, self.namespace["targetExternalId"], self.namespace[f"{target_prefix}{target_external_id}"])
+                )
         return triples
 
     @classmethod

{cognite_neat-0.110.0 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/_classic_cdf/_sequences.py

@@ -1,7 +1,7 @@
 import itertools
 import json
 import typing
-from collections.abc import Callable, Iterable, Set
+from collections.abc import Iterable, Set
 from pathlib import Path
 from typing import Any
 
@@ -21,8 +21,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
     Args:
         items (Iterable[T_CogniteResource]): An iterable of classic resource.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
-        to_type (Callable[[T_CogniteResource], str | None], optional): A function to convert an item to a type.
-            Defaults to None. If None or if the function returns None, the asset will be set to the default type.
         total (int, optional): The total number of items to load. If passed, you will get a progress bar if rich
             is installed. Defaults to None.
         limit (int, optional): The maximal number of items to load. Defaults to None. This is typically used for
@@ -46,7 +44,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         self,
         items: Iterable[NeatSequence],
         namespace: Namespace | None = None,
-        to_type: Callable[[NeatSequence], str | None] | None = None,
         total: int | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
@@ -60,7 +57,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         super().__init__(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -78,7 +74,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
-        to_type: Callable[[NeatSequence], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -87,12 +82,12 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         prefix: str | None = None,
         identifier: typing.Literal["id", "externalId"] = "id",
         unpack_columns: bool = False,
+        skip_rows: bool = False,
     ):
-        total, items = cls._handle_no_access(lambda: cls._from_dataset(client, data_set_external_id))
+        total, items = cls._handle_no_access(lambda: cls._from_dataset(client, data_set_external_id, skip_rows))
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -110,7 +105,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         client: CogniteClient,
         root_asset_external_id: str,
         namespace: Namespace | None = None,
-        to_type: Callable[[NeatSequence], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -119,12 +113,12 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         prefix: str | None = None,
         identifier: typing.Literal["id", "externalId"] = "id",
         unpack_columns: bool = False,
+        skip_rows: bool = False,
     ):
-        total, items = cls._handle_no_access(lambda: cls._from_hierarchy(client, root_asset_external_id))
+        total, items = cls._handle_no_access(lambda: cls._from_hierarchy(client, root_asset_external_id, skip_rows))
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -141,7 +135,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         cls,
         file_path: str | Path,
         namespace: Namespace | None = None,
-        to_type: Callable[[NeatSequence], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -155,7 +148,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -169,23 +161,29 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
 
     @classmethod
     def _from_dataset(
-        cls, client: CogniteClient, data_set_external_id: str
+        cls, client: CogniteClient, data_set_external_id: str, skip_rows: bool = False
     ) -> tuple[int | None, Iterable[NeatSequence]]:
         total = client.sequences.aggregate_count(
             filter=SequenceFilter(data_set_ids=[{"externalId": data_set_external_id}])
         )
         items = client.sequences(data_set_external_ids=data_set_external_id)
-        return total, cls._lookup_rows(items, client)
+        if skip_rows:
+            return total, (NeatSequence.from_cognite_sequence(seq) for seq in items)
+        else:
+            return total, cls._lookup_rows(items, client)
 
     @classmethod
     def _from_hierarchy(
-        cls, client: CogniteClient, root_asset_external_id: str
+        cls, client: CogniteClient, root_asset_external_id: str, skip_rows: bool = False
     ) -> tuple[int | None, Iterable[NeatSequence]]:
         total = client.sequences.aggregate_count(
             filter=SequenceFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
         )
         items = client.sequences(asset_subtree_external_ids=[root_asset_external_id])
-        return total, cls._lookup_rows(items, client)
+        if skip_rows:
+            return total, (NeatSequence.from_cognite_sequence(seq) for seq in items)
+        else:
+            return total, cls._lookup_rows(items, client)
 
     @classmethod
     def _from_file(cls, file_path: str | Path) -> tuple[int | None, Iterable[NeatSequence]]:

cognite_neat-0.111.0/cognite/neat/_graph/extractors/_dict.py

@@ -0,0 +1,102 @@
+import json
+import urllib.parse
+from collections.abc import Callable, Iterable, Mapping, Set
+from typing import Any
+
+from cognite.client import data_modeling as dm
+from cognite.client.data_classes.data_modeling.instances import Instance
+from rdflib import XSD, Literal, Namespace, URIRef
+
+from cognite.neat._shared import Triple
+from cognite.neat._utils.auxiliary import string_to_ideal_type
+
+from ._base import BaseExtractor
+
+DEFAULT_EMPTY_VALUES = frozenset({"nan", "null", "none", "", " ", "nil", "n/a", "na", "unknown", "undefined"})
+
+
+class DictExtractor(BaseExtractor):
+    def __init__(
+        self,
+        id_: URIRef,
+        data: Mapping[str, Any],
+        namespace: Namespace,
+        uri_ref_keys: set[str] | None = None,
+        empty_values: Set[str] = DEFAULT_EMPTY_VALUES,
+        str_to_ideal_type: bool = False,
+        unpack_json: bool = False,
+    ) -> None:
+        self.id_ = id_
+        self.namespace = namespace
+        self.data = data
+        self.uri_ref_keys = uri_ref_keys or set()
+        self.empty_values = empty_values
+        self.str_to_ideal_type = str_to_ideal_type
+        self.unpack_json = unpack_json
+
+    def extract(self) -> Iterable[Triple]:
+        for key, value in self.data.items():
+            for predicate_str, object_ in self._get_predicate_objects_pair(key, value, self.unpack_json):
+                yield self.id_, self.namespace[urllib.parse.quote(predicate_str)], object_
+
+    def _get_predicate_objects_pair(
+        self, key: str, value: Any, unpack_json: bool
+    ) -> Iterable[tuple[str, Literal | URIRef]]:
+        if key in self.uri_ref_keys and not isinstance(value, dict | list):
+            yield key, URIRef(self.namespace[urllib.parse.quote(value)])
+        if isinstance(value, str | float | bool | int):
+            yield key, Literal(value)
+        elif isinstance(value, dict) and unpack_json:
+            yield from self._unpack_json(value)
+        elif isinstance(value, dict):
+            # This object is a json object.
+            yield key, Literal(json.dumps(value), datatype=XSD._NS["json"])
+        elif isinstance(value, list):
+            for item in value:
+                yield from self._get_predicate_objects_pair(key, item, False)
+
+    def _unpack_json(self, value: dict, parent: str | None = None) -> Iterable[tuple[str, Literal | URIRef]]:
+        for sub_key, sub_value in value.items():
+            key = f"{parent}_{sub_key}" if parent else sub_key
+            if isinstance(sub_value, str):
+                if sub_value.casefold() in self.empty_values:
+                    continue
+                if self.str_to_ideal_type:
+                    yield key, Literal(string_to_ideal_type(sub_value))
+                else:
+                    yield key, Literal(sub_value)
+            elif isinstance(sub_value, int | float | bool):
+                yield key, Literal(sub_value)
+            elif isinstance(sub_value, dict):
+                yield from self._unpack_json(sub_value, key)
+            elif isinstance(sub_value, list):
+                for no, item in enumerate(sub_value, 1):
+                    if isinstance(item, dict):
+                        yield from self._unpack_json(item, f"{key}_{no}")
+                    else:
+                        yield from self._get_predicate_objects_pair(key, item, self.unpack_json)
+            else:
+                yield key, Literal(str(sub_value))
+
+
+class DMSPropertyExtractor(DictExtractor):
+    def __init__(
+        self,
+        id_: URIRef,
+        data: Mapping[str, Any],
+        namespace: Namespace,
+        as_uri_ref: Callable[[Instance | dm.DirectRelationReference], URIRef],
+        empty_values: Set[str] = DEFAULT_EMPTY_VALUES,
+        str_to_ideal_type: bool = False,
+        unpack_json: bool = False,
+    ) -> None:
+        super().__init__(id_, data, namespace, None, empty_values, str_to_ideal_type, unpack_json)
+        self.as_uri_ref = as_uri_ref
+
+    def _get_predicate_objects_pair(
+        self, key: str, value: Any, unpack_json: bool
+    ) -> Iterable[tuple[str, Literal | URIRef]]:
+        if isinstance(value, dict) and "space" in value and "externalId" in value:
+            yield key, self.as_uri_ref(dm.DirectRelationReference.load(value))
+        else:
+            yield from super()._get_predicate_objects_pair(key, value, unpack_json)