cognite-neat 0.109.4__tar.gz → 0.111.0__tar.gz

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.109.4
+Version: 0.111.0
 Summary: Knowledge graph transformation
 License: Apache-2.0
 Author: Nikola Vasiljevic

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_alpha.py

@@ -13,3 +13,11 @@ class AlphaFlags:
     manual_rules_edit = AlphaWarning("enable_manual_edit")
     same_space_properties_only_export = AlphaWarning("same-space-properties-only")
     standardize_naming = AlphaWarning("standardize_naming")
+    standardize_space_and_version = AlphaWarning("standardize_space_and_version")
+    data_model_subsetting = AlphaWarning("data_model_subsetting")
+    ontology_read = AlphaWarning("ontology_read")
+    imf_read = AlphaWarning("imf_read")
+    dexpi_read = AlphaWarning("dexpi_read")
+    aml_read = AlphaWarning("aml_read")
+    csv_read = AlphaWarning("csv_read")
+    to_ontology = AlphaWarning("to_ontology")

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_client/_api/schema.py

@@ -1,10 +1,13 @@
-from collections.abc import Sequence
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+from graphlib import CycleError, TopologicalSorter
 from typing import TYPE_CHECKING
 
 from cognite.client import data_modeling as dm
 
 from cognite.neat._client.data_classes.data_modeling import ContainerApplyDict, SpaceApplyDict, ViewApplyDict
 from cognite.neat._client.data_classes.schema import DMSSchema
+from cognite.neat._constants import is_hierarchy_property
 from cognite.neat._issues.errors import NeatValueError
 
 if TYPE_CHECKING:
@@ -110,3 +113,42 @@ class SchemaAPI:
             views=view_write,
             containers=container_write,
         )
+
+    @staticmethod
+    def get_hierarchical_properties(
+        views: Iterable[dm.View | dm.ViewApply],
+    ) -> dict[dm.ViewId, set[str]]:
+        """Sorts the views by container constraints."""
+        hierarchical_properties_by_view_id: dict[dm.ViewId, set[str]] = defaultdict(set)
+        for view in views:
+            for prop_id, prop in (view.properties or {}).items():
+                if not isinstance(prop, dm.MappedProperty | dm.MappedPropertyApply):
+                    continue
+                if is_hierarchy_property(prop.container, prop.container_property_identifier):
+                    hierarchical_properties_by_view_id[view.as_id()].add(prop_id)
+        return hierarchical_properties_by_view_id
+
+    @staticmethod
+    def get_view_order_by_direct_relation_constraints(views: Iterable[dm.View]) -> list[dm.ViewId]:
+        """Sorts the views by container constraints."""
+        view_sequence = list(views)
+        view_ids_by_container: dict[dm.ContainerId, set[dm.ViewId]] = defaultdict(set)
+        for view in view_sequence:
+            for container_id in view.referenced_containers():
+                view_ids_by_container[container_id].add(view.as_id())
+
+        view_by_dependency: dict[dm.ViewId, set[dm.ViewId]] = {}
+        for view in view_sequence:
+            view_id = view.as_id()
+            view_by_dependency[view_id] = set()
+            for prop in view.properties.values():
+                if (
+                    isinstance(prop, dm.MappedProperty)
+                    and isinstance(prop.type, dm.DirectRelation)
+                    and prop.type.container
+                ):
+                    view_by_dependency[view_id].update(view_ids_by_container[prop.type.container])
+        try:
+            return list(TopologicalSorter(view_by_dependency).static_order())
+        except CycleError as e:
+            raise NeatValueError("Cycle in direct relation constraints") from e

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_client/data_classes/schema.py

@@ -34,7 +34,7 @@ from cognite.neat._issues.warnings import (
     FileTypeUnexpectedWarning,
     ResourcesDuplicatedWarning,
 )
-from cognite.neat._utils.text import to_camel
+from cognite.neat._utils.text import to_camel_case
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -167,7 +167,7 @@ class DMSSchema:
                     if "." not in filename.stem:
                         continue
                     resource_type = filename.stem.rsplit(".", 1)[-1]
-                    if attr_name := cls._FIELD_NAME_BY_RESOURCE_TYPE.get(resource_type):
+                    if attr_name := cls._FIELD_NAME_BY_RESOURCE_TYPE.get(resource_type.casefold()):
                         data.setdefault(attr_name, [])
                         context.setdefault(attr_name, [])
                         try:
@@ -238,7 +238,7 @@ class DMSSchema:
             data_dict = data
         loaded: dict[str, Any] = {}
         for attr in fields(cls):
-            if items := data_dict.get(attr.name) or data_dict.get(to_camel(attr.name)):
+            if items := data_dict.get(attr.name) or data_dict.get(to_camel_case(attr.name)):
                 if attr.name == "data_model":
                     if isinstance(items, list) and len(items) > 1:
                         try:
@@ -320,7 +320,7 @@ class DMSSchema:
         cls_fields = sorted(fields(self), key=lambda f: f.name) if sort else fields(self)
         for attr in cls_fields:
             if items := getattr(self, attr.name):
-                key = to_camel(attr.name) if camel_case else attr.name
+                key = to_camel_case(attr.name) if camel_case else attr.name
                 if isinstance(items, CogniteResourceDict):
                     if sort:
                         output[key] = [

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_constants.py

@@ -1,3 +1,4 @@
+import re
 from collections.abc import Mapping
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -69,7 +70,9 @@ EXAMPLE_RULES = PACKAGE_DIRECTORY / "_rules" / "examples"
 EXAMPLE_GRAPHS = PACKAGE_DIRECTORY / "_graph" / "examples"
 EXAMPLE_WORKFLOWS = PACKAGE_DIRECTORY / "_workflows" / "examples"
 
-DEFAULT_SPACE_URI = "http://purl.org/cognite/{space}#"
+DEFAULT_SPACE_URI = "http://purl.org/cognite/space/{space}#"
+SPACE_URI_PATTERN = re.compile(r"http://purl.org/cognite/space/(?P<space>[^#]+)#$")
+DEFAULT_RAW_URI = "http://purl.org/cognite/raw#"
 DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/neat/")
 CDF_NAMESPACE = Namespace("https://cognitedata.com/")
 DEFAULT_BASE_URI = URIRef(DEFAULT_NAMESPACE)
@@ -164,11 +167,22 @@ READONLY_PROPERTIES_BY_CONTAINER: Mapping[dm.ContainerId, frozenset[str]] = {
     dm.ContainerId("cdf_cdm", "CogniteFile"): frozenset({"isUploaded", "uploadedTime"}),
 }
 
+HIERARCHICAL_PROPERTIES_BY_CONTAINER: Mapping[dm.ContainerId, frozenset[str]] = {
+    dm.ContainerId("cdf_cdm", "CogniteAsset"): frozenset({"assetHierarchy_parent"})
+}
+
 
 def is_readonly_property(container: dm.ContainerId, property_: str) -> bool:
     return container in READONLY_PROPERTIES_BY_CONTAINER and property_ in READONLY_PROPERTIES_BY_CONTAINER[container]
 
 
+def is_hierarchy_property(container: dm.ContainerId, property_: str) -> bool:
+    return (
+        container in HIERARCHICAL_PROPERTIES_BY_CONTAINER
+        and property_ in HIERARCHICAL_PROPERTIES_BY_CONTAINER[container]
+    )
+
+
 DMS_RESERVED_PROPERTIES = frozenset(
     {
         "createdTime",

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/__init__.py

@@ -11,10 +11,12 @@ from ._classic_cdf._relationships import RelationshipsExtractor
 from ._classic_cdf._sequences import SequencesExtractor
 from ._classic_cdf._timeseries import TimeSeriesExtractor
 from ._dexpi import DexpiExtractor
+from ._dict import DictExtractor
 from ._dms import DMSExtractor
 from ._dms_graph import DMSGraphExtractor
 from ._iodd import IODDExtractor
 from ._mock_graph_generator import MockGraphGenerator
+from ._raw import RAWExtractor
 from ._rdf_file import RdfFileExtractor
 
 __all__ = [
@@ -25,12 +27,14 @@ __all__ = [
     "DMSGraphExtractor",
     "DataSetExtractor",
     "DexpiExtractor",
+    "DictExtractor",
     "EventsExtractor",
     "FilesExtractor",
     "IODDExtractor",
     "KnowledgeGraphExtractor",
     "LabelsExtractor",
     "MockGraphGenerator",
+    "RAWExtractor",
     "RdfFileExtractor",
     "RelationshipsExtractor",
     "SequencesExtractor",

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/_classic_cdf/_base.py

@@ -61,8 +61,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
     Args:
         items (Iterable[T_CogniteResource]): An iterable of classic resource.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
-        to_type (Callable[[T_CogniteResource], str | None], optional): A function to convert an item to a type.
-            Defaults to None. If None or if the function returns None, the asset will be set to the default type.
         total (int, optional): The total number of items to load. If passed, you will get a progress bar if rich
             is installed. Defaults to None.
         limit (int, optional): The maximal number of items to load. Defaults to None. This is typically used for
@@ -87,7 +85,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         self,
         items: Iterable[T_CogniteResource],
         namespace: Namespace | None = None,
-        to_type: Callable[[T_CogniteResource], str | None] | None = None,
         total: int | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
@@ -99,7 +96,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.items = items
-        self.to_type = to_type
         self.total = total
         self.limit = min(limit, total) if limit and total else limit
         self.unpack_metadata = unpack_metadata
@@ -161,7 +157,7 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         if self._log_urirefs and hasattr(item, "external_id"):
             self._uriref_by_external_id[item.external_id] = id_
 
-        type_ = self._get_rdf_type(item)
+        type_ = self._get_rdf_type()
 
         # Set rdf type
         triples: list[Triple] = [(id_, RDF.type, self.namespace[type_])]
@@ -215,16 +211,14 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
                 if value and (self.skip_metadata_values is None or value.casefold() not in self.skip_metadata_values):
                     yield (
                         id_,
-                        self.namespace[key],
+                        self.namespace[urllib.parse.quote(key)],
                         Literal(string_to_ideal_type(value)),
                     )
         else:
             yield id_, self.namespace.metadata, Literal(json.dumps(metadata), datatype=XSD._NS["json"])
 
-    def _get_rdf_type(self, item: T_CogniteResource) -> str:
+    def _get_rdf_type(self) -> str:
         type_ = self._default_rdf_type
-        if self.to_type:
-            type_ = self.to_type(item) or type_
         if self.prefix:
             type_ = f"{self.prefix}{type_}"
         return self._SPACE_PATTERN.sub("_", type_)
@@ -263,7 +257,11 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
             "created_time",
             "last_updated_time",
         } and isinstance(raw, int):
-            return Literal(datetime.fromtimestamp(raw / 1000, timezone.utc), datatype=XSD.dateTime)
+            try:
+                return Literal(datetime.fromtimestamp(raw / 1000, timezone.utc), datatype=XSD.dateTime)
+            except (OSError, ValueError) as e:
+                warnings.warn(NeatValueWarning(f"Failed to convert timestamp {raw} to datetime: {e!s}"), stacklevel=2)
+                return Literal(raw)
         elif key == "labels":
             from ._labels import LabelsExtractor
 
@@ -284,7 +282,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
-        to_type: Callable[[T_CogniteResource], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -297,7 +294,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -321,7 +317,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         client: CogniteClient,
         root_asset_external_id: str,
         namespace: Namespace | None = None,
-        to_type: Callable[[T_CogniteResource], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -334,7 +329,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -357,7 +351,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         cls,
         file_path: str | Path,
         namespace: Namespace | None = None,
-        to_type: Callable[[T_CogniteResource], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -370,7 +363,6 @@ class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/_classic_cdf/_classic.py

@@ -16,12 +16,10 @@ from cognite.neat._issues.warnings import CDFAuthWarning, NeatValueWarning
 from cognite.neat._rules._shared import ReadRules
 from cognite.neat._rules.catalog import classic_model
 from cognite.neat._rules.models import InformationInputRules, InformationRules
-from cognite.neat._rules.models._rdfpath import Entity as RDFPathEntity
-from cognite.neat._rules.models._rdfpath import RDFPath, SingleProperty
 from cognite.neat._shared import Triple
 from cognite.neat._utils.collection_ import chunker, iterate_progress_bar
 from cognite.neat._utils.rdf_ import remove_namespace_from_uri
-from cognite.neat._utils.text import to_snake
+from cognite.neat._utils.text import to_snake_case
 
 from ._assets import AssetsExtractor
 from ._base import ClassicCDFBaseExtractor, InstanceIdPrefix
@@ -104,6 +102,8 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         limit_per_type: int | None = None,
         prefix: str | None = None,
         identifier: typing.Literal["id", "externalId"] = "id",
+        unpack_metadata: bool = False,
+        skip_sequence_rows: bool = False,
     ):
         self._client = client
         if sum([bool(data_set_external_id), bool(root_asset_external_id)]) != 1:
@@ -113,7 +113,7 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         self._namespace = namespace or CLASSIC_CDF_NAMESPACE
         self._extractor_args = dict(
             namespace=self._namespace,
-            unpack_metadata=False,
+            unpack_metadata=unpack_metadata,
             as_write=True,
             camel_case=True,
             limit=limit_per_type,
@@ -123,6 +123,7 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         self._identifier = identifier
         self._prefix = prefix
         self._limit_per_type = limit_per_type
+        self._skip_sequence_rows = skip_sequence_rows
 
         self._uris_by_external_id_by_type: dict[InstanceIdPrefix, dict[str, URIRef]] = defaultdict(dict)
         self._source_external_ids_by_type: dict[InstanceIdPrefix, set[str]] = defaultdict(set)
@@ -135,6 +136,29 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         self._extracted_data_sets = False
         self._asset_external_ids_by_id: dict[int, str] = {}
         self._dataset_external_ids_by_id: dict[int, str] = {}
+        self.neat_prefix_by_predicate_uri: dict[URIRef, str] = {
+            self._namespace["dataSetId"]: InstanceIdPrefix.data_set,
+            self._namespace["assetId"]: InstanceIdPrefix.asset,
+            self._namespace["assetIds"]: InstanceIdPrefix.asset,
+            self._namespace["parentId"]: InstanceIdPrefix.asset,
+            self._namespace["rootId"]: InstanceIdPrefix.asset,
+            self._namespace["labels"]: InstanceIdPrefix.label,
+        }
+        self.neat_prefix_by_type_uri: dict[URIRef, str] = {}
+        for extractor_cls, type_prefix in [
+            (AssetsExtractor, InstanceIdPrefix.asset),
+            (TimeSeriesExtractor, InstanceIdPrefix.time_series),
+            (SequencesExtractor, InstanceIdPrefix.sequence),
+            (EventsExtractor, InstanceIdPrefix.event),
+            (FilesExtractor, InstanceIdPrefix.file),
+            (RelationshipsExtractor, InstanceIdPrefix.relationship),
+            (LabelsExtractor, InstanceIdPrefix.label),
+            (DataSetExtractor, InstanceIdPrefix.data_set),
+        ]:
+            rdf_type = extractor_cls._default_rdf_type  # type: ignore[attr-defined]
+            if prefix:
+                rdf_type = f"{prefix}{rdf_type}"
+            self.neat_prefix_by_type_uri[self._namespace[rdf_type]] = type_prefix
 
     def _get_activity_names(self) -> list[str]:
         activities = [data_access_object.extractor_cls.__name__ for data_access_object in self._classic_node_types] + [
@@ -194,13 +218,14 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
         for prop in verified.properties:
             prop_id = prop.property_
             if is_snake_case:
-                prop_id = to_snake(prop_id)
-            prop.instance_source = RDFPath(
-                traversal=SingleProperty(
-                    class_=RDFPathEntity(prefix=instance_prefix, suffix=prop.class_.suffix),
-                    property=RDFPathEntity(prefix=instance_prefix, suffix=prop_id),
-                )
-            )
+                prop_id = to_snake_case(prop_id)
+            prop.instance_source = [self._namespace[prop_id]]
+        for cls_ in verified.classes:
+            cls_id = cls_.class_.suffix
+            if is_snake_case:
+                cls_id = to_snake_case(cls_id)
+            cls_.instance_source = self._namespace[cls_id]
+
         return verified
 
     @property
@@ -238,14 +263,14 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
 
     def _extract_core_start_nodes(self):
         for core_node in self._classic_node_types:
+            kwargs = self._extractor_args.copy()
+            if core_node.extractor_cls == SequencesExtractor and self._skip_sequence_rows:
+                kwargs["skip_rows"] = True
+
             if self._data_set_external_id:
-                extractor = core_node.extractor_cls.from_dataset(
-                    self._client, self._data_set_external_id, **self._extractor_args
-                )
+                extractor = core_node.extractor_cls.from_dataset(self._client, self._data_set_external_id, **kwargs)
             elif self._root_asset_external_id:
-                extractor = core_node.extractor_cls.from_hierarchy(
-                    self._client, self._root_asset_external_id, **self._extractor_args
-                )
+                extractor = core_node.extractor_cls.from_hierarchy(self._client, self._root_asset_external_id, **kwargs)
             else:
                 raise ValueError("Exactly one of data_set_external_id or root_asset_external_id must be set.")
 
@@ -274,7 +299,9 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
                 # This is a private attribute, but we need to set it to log the target nodes.
                 extractor._log_target_nodes = True
                 if self._identifier == "id":
-                    extractor._uri_by_external_id_by_by_type = self._uris_by_external_id_by_type
+                    extractor._uri_by_external_id_by_type = self._uris_by_external_id_by_type
+                elif self._identifier == "externalId":
+                    extractor.lookup_dataset_external_id = self._lookup_dataset
 
                 yield from extractor.extract()
 
@@ -331,7 +358,9 @@ class ClassicGraphExtractor(KnowledgeGraphExtractor):
     def _extract_labels(self):
         for chunk in self._chunk(list(self._labels), description="Extracting labels"):
             label_iterator = self._client.labels.retrieve(external_id=list(chunk), ignore_unknown_ids=True)
-            yield from LabelsExtractor(label_iterator, **self._extractor_args).extract()
+            extractor = LabelsExtractor(label_iterator, **self._extractor_args)
+            extractor.lookup_dataset_external_id = self._lookup_dataset
+            yield from extractor.extract()
 
     def _extract_data_sets(self):
         for chunk in self._chunk(list(self._data_set_ids), description="Extracting data sets"):

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/_classic_cdf/_relationships.py

@@ -1,7 +1,7 @@
 import typing
 import warnings
 from collections import defaultdict
-from collections.abc import Callable, Iterable, Set
+from collections.abc import Iterable, Set
 from pathlib import Path
 from typing import Any
 
@@ -26,7 +26,6 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
         self,
         items: Iterable[Relationship],
         namespace: Namespace | None = None,
-        to_type: Callable[[Relationship], str | None] | None = None,
         total: int | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
@@ -46,7 +45,6 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
         super().__init__(
             to_iterate,
             namespace=namespace,
-            to_type=to_type,
             total=total,
             limit=limit,
             unpack_metadata=unpack_metadata,
@@ -56,7 +54,7 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
             prefix=prefix,
             identifier=identifier,
         )
-        self._uri_by_external_id_by_by_type: dict[InstanceIdPrefix, dict[str, URIRef]] = defaultdict(dict)
+        self._uri_by_external_id_by_type: dict[InstanceIdPrefix, dict[str, URIRef]] = defaultdict(dict)
         self._target_triples: list[tuple[URIRef, URIRef, str, str]] = []
 
     def _log_target_nodes_if_set(self, item: Relationship) -> Relationship:
@@ -67,25 +65,33 @@ class RelationshipsExtractor(ClassicCDFBaseExtractor[Relationship]):
         return item
 
     def _item2triples_special_cases(self, id_: URIRef, dumped: dict[str, Any]) -> list[Triple]:
-        if self.identifier == "externalId":
-            return []
         triples: list[Triple] = []
         if (source_external_id := dumped.pop("sourceExternalId")) and "sourceType" in dumped:
             source_type = dumped["sourceType"]
-            try:
-                source_uri = self._uri_by_external_id_by_by_type[InstanceIdPrefix.from_str(source_type)][
-                    source_external_id
-                ]
-            except KeyError:
-                warnings.warn(
-                    NeatValueWarning(f"Missing externalId {source_external_id} for {source_type}"), stacklevel=2
+            source_prefix = InstanceIdPrefix.from_str(source_type)
+            if self.identifier == "id":
+                try:
+                    source_uri = self._uri_by_external_id_by_type[source_prefix][source_external_id]
+                except KeyError:
+                    warnings.warn(
+                        NeatValueWarning(f"Missing externalId {source_external_id} for {source_type}"), stacklevel=2
+                    )
+                else:
+                    triples.append((id_, self.namespace["sourceExternalId"], source_uri))
+            elif self.identifier == "externalId":
+                triples.append(
+                    (id_, self.namespace["sourceExternalId"], self.namespace[f"{source_prefix}{source_external_id}"])
                 )
-            else:
-                triples.append((id_, self.namespace["sourceExternalId"], source_uri))
         if (target_external_id := dumped.pop("targetExternalId")) and "targetType" in dumped:
             target_type = dumped["targetType"]
-            # We do not yet have the target nodes, so we log them for later extraction.
-            self._target_triples.append((id_, self.namespace["targetExternalId"], target_type, target_external_id))
+            if self.identifier == "id":
+                # We do not yet have the target nodes, so we log them for later extraction.
+                self._target_triples.append((id_, self.namespace["targetExternalId"], target_type, target_external_id))
+            elif self.identifier == "externalId":
+                target_prefix = InstanceIdPrefix.from_str(target_type)
+                triples.append(
+                    (id_, self.namespace["targetExternalId"], self.namespace[f"{target_prefix}{target_external_id}"])
+                )
         return triples
 
     @classmethod

{cognite_neat-0.109.4 → cognite_neat-0.111.0}/cognite/neat/_graph/extractors/_classic_cdf/_sequences.py

@@ -1,7 +1,7 @@
 import itertools
 import json
 import typing
-from collections.abc import Callable, Iterable, Set
+from collections.abc import Iterable, Set
 from pathlib import Path
 from typing import Any
 
@@ -21,8 +21,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
     Args:
         items (Iterable[T_CogniteResource]): An iterable of classic resource.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
-        to_type (Callable[[T_CogniteResource], str | None], optional): A function to convert an item to a type.
-            Defaults to None. If None or if the function returns None, the asset will be set to the default type.
         total (int, optional): The total number of items to load. If passed, you will get a progress bar if rich
             is installed. Defaults to None.
         limit (int, optional): The maximal number of items to load. Defaults to None. This is typically used for
@@ -46,7 +44,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         self,
         items: Iterable[NeatSequence],
         namespace: Namespace | None = None,
-        to_type: Callable[[NeatSequence], str | None] | None = None,
         total: int | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
@@ -60,7 +57,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         super().__init__(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -78,7 +74,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
-        to_type: Callable[[NeatSequence], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -87,12 +82,12 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         prefix: str | None = None,
         identifier: typing.Literal["id", "externalId"] = "id",
         unpack_columns: bool = False,
+        skip_rows: bool = False,
     ):
-        total, items = cls._handle_no_access(lambda: cls._from_dataset(client, data_set_external_id))
+        total, items = cls._handle_no_access(lambda: cls._from_dataset(client, data_set_external_id, skip_rows))
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -110,7 +105,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         client: CogniteClient,
         root_asset_external_id: str,
         namespace: Namespace | None = None,
-        to_type: Callable[[NeatSequence], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -119,12 +113,12 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         prefix: str | None = None,
         identifier: typing.Literal["id", "externalId"] = "id",
         unpack_columns: bool = False,
+        skip_rows: bool = False,
     ):
-        total, items = cls._handle_no_access(lambda: cls._from_hierarchy(client, root_asset_external_id))
+        total, items = cls._handle_no_access(lambda: cls._from_hierarchy(client, root_asset_external_id, skip_rows))
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -141,7 +135,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         cls,
         file_path: str | Path,
         namespace: Namespace | None = None,
-        to_type: Callable[[NeatSequence], str | None] | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
         skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
@@ -155,7 +148,6 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
         return cls(
             items,
             namespace,
-            to_type,
             total,
             limit,
             unpack_metadata,
@@ -169,23 +161,29 @@ class SequencesExtractor(ClassicCDFBaseExtractor[NeatSequence]):
 
     @classmethod
     def _from_dataset(
-        cls, client: CogniteClient, data_set_external_id: str
+        cls, client: CogniteClient, data_set_external_id: str, skip_rows: bool = False
     ) -> tuple[int | None, Iterable[NeatSequence]]:
         total = client.sequences.aggregate_count(
             filter=SequenceFilter(data_set_ids=[{"externalId": data_set_external_id}])
         )
         items = client.sequences(data_set_external_ids=data_set_external_id)
-        return total, cls._lookup_rows(items, client)
+        if skip_rows:
+            return total, (NeatSequence.from_cognite_sequence(seq) for seq in items)
+        else:
+            return total, cls._lookup_rows(items, client)
 
     @classmethod
     def _from_hierarchy(
-        cls, client: CogniteClient, root_asset_external_id: str
+        cls, client: CogniteClient, root_asset_external_id: str, skip_rows: bool = False
     ) -> tuple[int | None, Iterable[NeatSequence]]:
         total = client.sequences.aggregate_count(
             filter=SequenceFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
         )
         items = client.sequences(asset_subtree_external_ids=[root_asset_external_id])
-        return total, cls._lookup_rows(items, client)
+        if skip_rows:
+            return total, (NeatSequence.from_cognite_sequence(seq) for seq in items)
+        else:
+            return total, cls._lookup_rows(items, client)
 
     @classmethod
     def _from_file(cls, file_path: str | Path) -> tuple[int | None, Iterable[NeatSequence]]: