cognite-neat 0.91.0__tar.gz → 0.92.1__tar.gz

{cognite_neat-0.91.0 → cognite_neat-0.92.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.91.0
+Version: 0.92.1
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -59,7 +59,7 @@ Description-Content-Type: text/markdown
 
 # kNowlEdge grAph Transformer (NEAT)
 
-[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/neat/release_pypi.yaml?style=for-the-badge)](https://github.com/cognitedata/neat/actions/workflows/release_pypi.yaml)
+[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/neat/release.yaml?style=for-the-badge)](https://github.com/cognitedata/neat/actions/workflows/release.yaml)
 [![Documentation Status](https://readthedocs.com/projects/cognite-neat/badge/?version=latest&style=for-the-badge)](https://cognite-neat.readthedocs-hosted.com/en/latest/?badge=latest)
 [![Github](https://shields.io/badge/github-cognite/neat-green?logo=github&style=for-the-badge)](https://github.com/cognitedata/neat)
 [![PyPI](https://img.shields.io/pypi/v/cognite-neat?style=for-the-badge)](https://pypi.org/project/cognite-neat/)

{cognite_neat-0.91.0 → cognite_neat-0.92.1}/README.md

@@ -1,6 +1,6 @@
 # kNowlEdge grAph Transformer (NEAT)
 
-[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/neat/release_pypi.yaml?style=for-the-badge)](https://github.com/cognitedata/neat/actions/workflows/release_pypi.yaml)
+[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/neat/release.yaml?style=for-the-badge)](https://github.com/cognitedata/neat/actions/workflows/release.yaml)
 [![Documentation Status](https://readthedocs.com/projects/cognite-neat/badge/?version=latest&style=for-the-badge)](https://cognite-neat.readthedocs-hosted.com/en/latest/?badge=latest)
 [![Github](https://shields.io/badge/github-cognite/neat-green?logo=github&style=for-the-badge)](https://github.com/cognitedata/neat)
 [![PyPI](https://img.shields.io/pypi/v/cognite-neat?style=for-the-badge)](https://pypi.org/project/cognite-neat/)

cognite_neat-0.92.1/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.92.1"

{cognite_neat-0.91.0 → cognite_neat-0.92.1}/cognite/neat/graph/extractors/__init__.py

@@ -1,5 +1,6 @@
 from ._base import BaseExtractor
 from ._classic_cdf._assets import AssetsExtractor
+from ._classic_cdf._classic import ClassicGraphExtractor
 from ._classic_cdf._events import EventsExtractor
 from ._classic_cdf._files import FilesExtractor
 from ._classic_cdf._labels import LabelsExtractor
@@ -15,6 +16,7 @@ from ._rdf_file import RdfFileExtractor
 __all__ = [
     "BaseExtractor",
     "AssetsExtractor",
+    "ClassicGraphExtractor",
     "MockGraphGenerator",
     "RelationshipsExtractor",
     "TimeSeriesExtractor",

{cognite_neat-0.91.0 → cognite_neat-0.92.1}/cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -8,12 +8,12 @@ from cognite.client.data_classes import Asset, AssetFilter, AssetList
 from rdflib import RDF, Literal, Namespace
 
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.auxiliary import create_sha256_hash
 
-from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFExtractor
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFBaseExtractor, InstanceIdPrefix
+from ._labels import LabelsExtractor
 
 
-class AssetsExtractor(ClassicCDFExtractor[Asset]):
+class AssetsExtractor(ClassicCDFBaseExtractor[Asset]):
     """Extract data from Cognite Data Fusions Assets into Neat.
 
     Args:
@@ -108,7 +108,7 @@ class AssetsExtractor(ClassicCDFExtractor[Asset]):
 
     def _item2triples(self, asset: Asset) -> list[Triple]:
         """Converts an asset to triples."""
-        id_ = self.namespace[f"Asset_{asset.id}"]
+        id_ = self.namespace[f"{InstanceIdPrefix.asset}{asset.id}"]
 
         type_ = self._get_rdf_type(asset)
 
@@ -151,7 +151,7 @@ class AssetsExtractor(ClassicCDFExtractor[Asset]):
                     (
                         id_,
                         self.namespace.label,
-                        self.namespace[f"Label_{create_sha256_hash(label.dump()['externalId'])}"],
+                        self.namespace[f"{InstanceIdPrefix.label}{LabelsExtractor._label_id(label)}"],
                     )
                 )
 
@@ -160,17 +160,17 @@ class AssetsExtractor(ClassicCDFExtractor[Asset]):
 
         # Create connections:
         if asset.parent_id:
-            triples.append((id_, self.namespace.parent, self.namespace[f"Asset_{asset.parent_id}"]))
+            triples.append((id_, self.namespace.parent, self.namespace[f"{InstanceIdPrefix.asset}{asset.parent_id}"]))
 
         if asset.root_id:
-            triples.append((id_, self.namespace.root, self.namespace[f"Asset_{asset.root_id}"]))
+            triples.append((id_, self.namespace.root, self.namespace[f"{InstanceIdPrefix.asset}{asset.root_id}"]))
 
         if asset.data_set_id:
             triples.append(
                 (
                     id_,
                     self.namespace.dataset,
-                    self.namespace[f"Dataset_{asset.data_set_id}"],
+                    self.namespace[f"{InstanceIdPrefix.data_set}{asset.data_set_id}"],
                 )
             )
 

{cognite_neat-0.91.0 → cognite_neat-0.92.1}/cognite/neat/graph/extractors/_classic_cdf/_base.py

@@ -1,5 +1,6 @@
 import json
 import re
+import sys
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterable, Set
 from typing import Generic, TypeVar
@@ -16,8 +17,32 @@ T_CogniteResource = TypeVar("T_CogniteResource", bound=CogniteResource)
 
 DEFAULT_SKIP_METADATA_VALUES = frozenset({"nan", "null", "none", ""})
 
+if sys.version_info >= (3, 11):
+    from enum import StrEnum
+else:
+    from backports.strenum import StrEnum
 
-class ClassicCDFExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
+
+class InstanceIdPrefix(StrEnum):
+    asset = "Asset_"
+    label = "Label_"
+    relationship = "Relationship_"
+    sequence = "Sequence_"
+    file = "File_"
+    time_series = "TimeSeries_"
+    event = "Event_"
+    data_set = "DataSet_"
+
+    @classmethod
+    def from_str(cls, raw: str) -> "InstanceIdPrefix":
+        raw = raw.title() + "_"
+        if raw == "Timeseries_":
+            return cls.time_series
+        else:
+            return cls(raw)
+
+
+class ClassicCDFBaseExtractor(BaseExtractor, ABC, Generic[T_CogniteResource]):
     """This is the Base Extractor for all classic CDF resources.
 
     A classic resource is recognized in that it has a metadata attribute of type dict[str, str].

cognite_neat-0.92.1/cognite/neat/graph/extractors/_classic_cdf/_classic.py

@@ -0,0 +1,208 @@
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+from typing import ClassVar, NamedTuple
+
+from cognite.client import CogniteClient
+from rdflib import Namespace
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors._base import BaseExtractor
+from cognite.neat.graph.models import Triple
+from cognite.neat.utils.collection_ import chunker
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
+
+from ._assets import AssetsExtractor
+from ._base import ClassicCDFBaseExtractor, InstanceIdPrefix
+from ._data_sets import DataSetExtractor
+from ._events import EventsExtractor
+from ._files import FilesExtractor
+from ._labels import LabelsExtractor
+from ._relationships import RelationshipsExtractor
+from ._sequences import SequencesExtractor
+from ._timeseries import TimeSeriesExtractor
+
+
+class _ClassicCoreType(NamedTuple):
+    extractor_cls: (
+        type[AssetsExtractor]
+        | type[TimeSeriesExtractor]
+        | type[SequencesExtractor]
+        | type[EventsExtractor]
+        | type[FilesExtractor]
+    )
+    resource_type: InstanceIdPrefix
+    api_name: str
+
+
+class ClassicGraphExtractor(BaseExtractor):
+    """This extractor extracts all classic CDF Resources.
+
+    The Classic Graph consists of the following core resource type.
+
+    Classic Node CDF Resources:
+     - Assets
+     - TimeSeries
+     - Sequences
+     - Events
+     - Files
+
+    All the classic node CDF resources can have one or more connections to one or more assets. This
+    will match a direct relationship in the data modeling of CDF.
+
+    In addition, you have relationships between the classic node CDF resources. This matches an edge
+    in the data modeling of CDF.
+
+    Finally, you have labels and data sets that to organize the graph. In which data sets have a similar,
+    but different, role as a space in data modeling. While labels can be compared to node types in data modeling,
+    used to quickly filter and find nodes/edges.
+
+    This extractor will extract the classic CDF graph into Neat starting from either a data set or a root asset.
+
+    It works as follows:
+
+    1. Extract all core nodes (assets, time series, sequences, events, files) filtered by the given data set or
+       root asset.
+    2. Extract all relationships starting from any of the extracted core nodes.
+    3. Extract all core nodes that are targets of the relationships that are not already extracted.
+    4. Extract all labels that are connected to the extracted core nodes/relationships.
+    5. Extract all data sets that are connected to the extracted core nodes/relationships.
+
+    Args:
+        client (CogniteClient): The Cognite client to use.
+        data_set_external_id (str, optional): The data set external id to extract from. Defaults to None.
+        root_asset_external_id (str, optional): The root asset external id to extract from. Defaults to None.
+        namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+    """
+
+    # These are the core resource types in the classic CDF.
+    _classic_node_types: ClassVar[tuple[_ClassicCoreType, ...]] = (
+        _ClassicCoreType(AssetsExtractor, InstanceIdPrefix.asset, "assets"),
+        _ClassicCoreType(TimeSeriesExtractor, InstanceIdPrefix.time_series, "time_series"),
+        _ClassicCoreType(SequencesExtractor, InstanceIdPrefix.sequence, "sequences"),
+        _ClassicCoreType(EventsExtractor, InstanceIdPrefix.event, "events"),
+        _ClassicCoreType(FilesExtractor, InstanceIdPrefix.file, "files"),
+    )
+
+    def __init__(
+        self,
+        client: CogniteClient,
+        data_set_external_id: str | None = None,
+        root_asset_external_id: str | None = None,
+        namespace: Namespace | None = None,
+    ):
+        self._client = client
+        if sum([bool(data_set_external_id), bool(root_asset_external_id)]) != 1:
+            raise ValueError("Exactly one of data_set_external_id or root_asset_external_id must be set.")
+        self._root_asset_external_id = root_asset_external_id
+        self._data_set_external_id = data_set_external_id
+        self._namespace = namespace or DEFAULT_NAMESPACE
+
+        self._source_external_ids_by_type: dict[InstanceIdPrefix, set[str]] = defaultdict(set)
+        self._target_external_ids_by_type: dict[InstanceIdPrefix, set[str]] = defaultdict(set)
+        self._labels: set[str] = set()
+        self._data_set_ids: set[int] = set()
+
+    def extract(self) -> Iterable[Triple]:
+        """Extracts all classic CDF Resources."""
+        yield from self._extract_core_start_nodes()
+
+        yield from self._extract_start_node_relationships()
+
+        yield from self._extract_core_end_nodes()
+
+        yield from self._extract_labels()
+        yield from self._extract_data_sets()
+
+    def _extract_core_start_nodes(self):
+        for core_node in self._classic_node_types:
+            if self._data_set_external_id:
+                extractor = core_node.extractor_cls.from_dataset(
+                    self._client, self._data_set_external_id, self._namespace, unpack_metadata=False
+                )
+            elif self._root_asset_external_id:
+                extractor = core_node.extractor_cls.from_hierarchy(
+                    self._client, self._root_asset_external_id, self._namespace, unpack_metadata=False
+                )
+            else:
+                raise ValueError("Exactly one of data_set_external_id or root_asset_external_id must be set.")
+
+            yield from self._extract_with_logging_label_dataset(extractor, core_node.resource_type)
+
+    def _extract_start_node_relationships(self):
+        for start_resource_type, source_external_ids in self._source_external_ids_by_type.items():
+            start_type = start_resource_type.removesuffix("_")
+            for chunk in self._chunk(list(source_external_ids), description=f"Extracting {start_type} relationships"):
+                relationship_iterator = self._client.relationships(
+                    source_external_ids=list(chunk), source_types=[start_type]
+                )
+                extractor = RelationshipsExtractor(relationship_iterator, self._namespace, unpack_metadata=False)
+                # This is a private attribute, but we need to set it to log the target nodes.
+                extractor._log_target_nodes = True
+
+                yield from extractor.extract()
+
+                # After the extraction is done, we need to update all the new target nodes so
+                # we can extract them in the next step.
+                for end_type, target_external_ids in extractor._target_external_ids_by_type.items():
+                    for external_id in target_external_ids:
+                        # We only want to extract the target nodes that are not already extracted.
+                        # Even though _source_external_ids_by_type is a defaultdict, we have to check if the key exists.
+                        # This is because we might not have extracted any nodes of that type yet, and looking up
+                        # a key that does not exist will create it. We are iterating of this dictionary, and
+                        # we do not want to create new keys while iterating.
+                        if (
+                            end_type not in self._source_external_ids_by_type
+                            or external_id not in self._source_external_ids_by_type[end_type]
+                        ):
+                            self._target_external_ids_by_type[end_type].add(external_id)
+
+    def _extract_core_end_nodes(self):
+        for core_node in self._classic_node_types:
+            target_external_ids = self._target_external_ids_by_type[core_node.resource_type]
+            api = getattr(self._client, core_node.api_name)
+            for chunk in self._chunk(
+                list(target_external_ids),
+                description=f"Extracting end nodes {core_node.resource_type.removesuffix('_')}",
+            ):
+                resource_iterator = api.retrieve_multiple(external_ids=list(chunk), ignore_unknown_ids=True)
+                extractor = core_node.extractor_cls(resource_iterator, self._namespace, unpack_metadata=False)
+                yield from self._extract_with_logging_label_dataset(extractor)
+
+    def _extract_labels(self):
+        for chunk in self._chunk(list(self._labels), description="Extracting labels"):
+            label_iterator = self._client.labels.retrieve(external_id=list(chunk), ignore_unknown_ids=True)
+            yield from LabelsExtractor(label_iterator, self._namespace).extract()
+
+    def _extract_data_sets(self):
+        for chunk in self._chunk(list(self._data_set_ids), description="Extracting data sets"):
+            data_set_iterator = self._client.data_sets.retrieve_multiple(ids=list(chunk), ignore_unknown_ids=True)
+            yield from DataSetExtractor(data_set_iterator, self._namespace, unpack_metadata=False).extract()
+
+    def _extract_with_logging_label_dataset(
+        self, extractor: ClassicCDFBaseExtractor, resource_type: InstanceIdPrefix | None = None
+    ) -> Iterable[Triple]:
+        for triple in extractor.extract():
+            if triple[1] == self._namespace.external_id and resource_type is not None:
+                self._source_external_ids_by_type[resource_type].add(remove_namespace_from_uri(triple[2]))
+            elif triple[1] == self._namespace.label:
+                self._labels.add(remove_namespace_from_uri(triple[2]).removeprefix(InstanceIdPrefix.label))
+            elif triple[1] == self._namespace.dataset:
+                self._data_set_ids.add(
+                    int(remove_namespace_from_uri(triple[2]).removeprefix(InstanceIdPrefix.data_set))
+                )
+            yield triple
+
+    @staticmethod
+    def _chunk(items: Sequence, description: str) -> Iterable:
+        to_iterate: Iterable = chunker(items, chunk_size=1000)
+        try:
+            from rich.progress import track
+        except ModuleNotFoundError:
+            ...
+        else:
+            to_iterate = track(
+                to_iterate,
+                total=(len(items) // 1000) + 1,
+                description=description,
+            )
+        return to_iterate

cognite_neat-0.92.1/cognite/neat/graph/extractors/_classic_cdf/_data_sets.py

@@ -0,0 +1,110 @@
+from collections.abc import Set
+from datetime import datetime, timezone
+from pathlib import Path
+
+from cognite.client import CogniteClient
+from cognite.client.data_classes import DataSet, DataSetList
+from cognite.client.utils.useful_types import SequenceNotStr
+from rdflib import RDF, Literal, Namespace
+
+from cognite.neat.graph.models import Triple
+
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFBaseExtractor, InstanceIdPrefix
+
+
+class DataSetExtractor(ClassicCDFBaseExtractor[DataSet]):
+    """Extract DataSets from Cognite Data Fusions into Neat.
+
+    Args:
+        items (Iterable[Asset]): An iterable of assets.
+        namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        to_type (Callable[[Asset], str | None], optional): A function to convert an asset to a type. Defaults to None.
+            If None or if the function returns None, the asset will be set to the default type "Asset".
+        total (int, optional): The total number of assets to load. If passed, you will get a progress bar if rich
+            is installed. Defaults to None.
+        limit (int, optional): The maximal number of assets to load. Defaults to None. This is typically used for
+            testing setup of the extractor. For example, if you are extracting 100 000 assets, you might want to
+            limit the extraction to 1000 assets to test the setup.
+        unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
+            a JSON string.
+        skip_metadata_values (set[str] | frozenset[str] | None, optional): A set of values to skip when unpacking
+            metadata. Defaults to frozenset({"nan", "null", "none", ""}).
+    """
+
+    _default_rdf_type = "DataSet"
+
+    @classmethod
+    def from_dataset(
+        cls,
+        client: CogniteClient,
+        data_set_external_id: SequenceNotStr[str],
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
+    ):
+        return cls(
+            client.data_sets.retrieve_multiple(external_ids=data_set_external_id),
+            namespace=namespace,
+            total=len(data_set_external_id),
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
+        )
+
+    @classmethod
+    def from_file(
+        cls,
+        file_path: str,
+        namespace: Namespace | None = None,
+        unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
+    ):
+        data_sets = DataSetList.load(Path(file_path).read_text())
+        return cls(
+            data_sets,
+            namespace=namespace,
+            total=len(data_sets),
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
+        )
+
+    def _item2triples(self, item: DataSet) -> list[Triple]:
+        """Converts an asset to triples."""
+        id_ = self.namespace[f"{InstanceIdPrefix.data_set}{item.id}"]
+
+        type_ = self._get_rdf_type(item)
+
+        triples: list[Triple] = [(id_, RDF.type, self.namespace[type_])]
+
+        # Create attributes
+        if item.name:
+            triples.append((id_, self.namespace.name, Literal(item.name)))
+
+        if item.description:
+            triples.append((id_, self.namespace.description, Literal(item.description)))
+
+        if item.external_id:
+            triples.append((id_, self.namespace.external_id, Literal(item.external_id)))
+
+        # properties' ref creation and update
+        triples.append(
+            (
+                id_,
+                self.namespace.created_time,
+                Literal(datetime.fromtimestamp(item.created_time / 1000, timezone.utc)),
+            )
+        )
+        triples.append(
+            (
+                id_,
+                self.namespace.last_updated_time,
+                Literal(datetime.fromtimestamp(item.last_updated_time / 1000, timezone.utc)),
+            )
+        )
+
+        if item.write_protected:
+            triples.append((id_, self.namespace.write_protected, Literal(item.write_protected)))
+
+        if item.metadata:
+            triples.extend(self._metadata_to_triples(id_, item.metadata))
+
+        return triples

{cognite_neat-0.91.0 → cognite_neat-0.92.1}/cognite/neat/graph/extractors/_classic_cdf/_events.py

@@ -8,10 +8,10 @@ from rdflib import RDF, Literal, Namespace
 
 from cognite.neat.graph.models import Triple
 
-from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFExtractor
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFBaseExtractor, InstanceIdPrefix
 
 
-class EventsExtractor(ClassicCDFExtractor[Event]):
+class EventsExtractor(ClassicCDFBaseExtractor[Event]):
     """Extract data from Cognite Data Fusions Events into Neat.
 
     Args:
@@ -55,6 +55,31 @@ class EventsExtractor(ClassicCDFExtractor[Event]):
             skip_metadata_values=skip_metadata_values,
         )
 
+    @classmethod
+    def from_hierarchy(
+        cls,
+        client: CogniteClient,
+        root_asset_external_id: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[Event], str | None] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
+    ):
+        total = client.events.aggregate_count(
+            filter=EventFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
+        )
+
+        return cls(
+            client.events(asset_subtree_external_ids=[root_asset_external_id]),
+            namespace,
+            to_type,
+            total,
+            limit,
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
+        )
+
     @classmethod
     def from_file(
         cls,
@@ -78,7 +103,7 @@ class EventsExtractor(ClassicCDFExtractor[Event]):
         )
 
     def _item2triples(self, event: Event) -> list[Triple]:
-        id_ = self.namespace[f"Event_{event.id}"]
+        id_ = self.namespace[f"{InstanceIdPrefix.event}{event.id}"]
 
         type_ = self._get_rdf_type(event)
 
@@ -146,12 +171,12 @@ class EventsExtractor(ClassicCDFExtractor[Event]):
                 (
                     id_,
                     self.namespace.data_set_id,
-                    self.namespace[f"Dataset_{event.data_set_id}"],
+                    self.namespace[f"{InstanceIdPrefix.data_set}{event.data_set_id}"],
                 )
             )
 
         if event.asset_ids:
             for asset_id in event.asset_ids:
-                triples.append((id_, self.namespace.asset, self.namespace[f"Asset_{asset_id}"]))
+                triples.append((id_, self.namespace.asset, self.namespace[f"{InstanceIdPrefix.asset}{asset_id}"]))
 
         return triples

{cognite_neat-0.91.0 → cognite_neat-0.92.1}/cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -1,18 +1,18 @@
 from collections.abc import Callable, Set
 from datetime import datetime, timezone
 from pathlib import Path
-from urllib.parse import quote
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes import FileMetadata, FileMetadataList
+from cognite.client.data_classes import FileMetadata, FileMetadataFilter, FileMetadataList
 from rdflib import RDF, Literal, Namespace
 
 from cognite.neat.graph.models import Triple
 
-from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFExtractor
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFBaseExtractor, InstanceIdPrefix
+from ._labels import LabelsExtractor
 
 
-class FilesExtractor(ClassicCDFExtractor[FileMetadata]):
+class FilesExtractor(ClassicCDFBaseExtractor[FileMetadata]):
     """Extract data from Cognite Data Fusions files metadata into Neat.
 
     Args:
@@ -53,6 +53,31 @@ class FilesExtractor(ClassicCDFExtractor[FileMetadata]):
             skip_metadata_values=skip_metadata_values,
         )
 
+    @classmethod
+    def from_hierarchy(
+        cls,
+        client: CogniteClient,
+        root_asset_external_id: str,
+        namespace: Namespace | None = None,
+        to_type: Callable[[FileMetadata], str | None] | None = None,
+        limit: int | None = None,
+        unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
+    ):
+        total = client.files.aggregate(
+            filter=FileMetadataFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
+        )[0].count
+
+        return cls(
+            client.files(asset_subtree_external_ids=[root_asset_external_id]),
+            namespace,
+            to_type,
+            total,
+            limit,
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
+        )
+
     @classmethod
     def from_file(
         cls,
@@ -75,7 +100,7 @@ class FilesExtractor(ClassicCDFExtractor[FileMetadata]):
         )
 
     def _item2triples(self, file: FileMetadata) -> list[Triple]:
-        id_ = self.namespace[f"File_{file.id}"]
+        id_ = self.namespace[f"{InstanceIdPrefix.file}{file.id}"]
 
         type_ = self._get_rdf_type(file)
 
@@ -153,7 +178,7 @@ class FilesExtractor(ClassicCDFExtractor[FileMetadata]):
                     (
                         id_,
                         self.namespace.label,
-                        self.namespace[f"Label_{quote(label.dump()['externalId'])}"],
+                        self.namespace[f"{InstanceIdPrefix.label}{LabelsExtractor._label_id(label)}"],
                     )
                 )
 
@@ -166,12 +191,12 @@ class FilesExtractor(ClassicCDFExtractor[FileMetadata]):
                 (
                     id_,
                     self.namespace.data_set_id,
-                    self.namespace[f"Dataset_{file.data_set_id}"],
+                    self.namespace[f"{InstanceIdPrefix.data_set}{file.data_set_id}"],
                 )
             )
 
         if file.asset_ids:
             for asset_id in file.asset_ids:
-                triples.append((id_, self.namespace.asset, self.namespace[f"Asset_{asset_id}"]))
+                triples.append((id_, self.namespace.asset, self.namespace[f"{InstanceIdPrefix.asset}{asset_id}"]))
 
         return triples

{cognite_neat-0.91.0 → cognite_neat-0.92.1}/cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -1,18 +1,18 @@
 from collections.abc import Callable, Set
 from datetime import datetime, timezone
 from pathlib import Path
+from urllib.parse import quote
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes import LabelDefinition, LabelDefinitionList
+from cognite.client.data_classes import Label, LabelDefinition, LabelDefinitionList
 from rdflib import RDF, Literal, Namespace
 
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.auxiliary import create_sha256_hash
 
-from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFExtractor
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFBaseExtractor, InstanceIdPrefix
 
 
-class LabelsExtractor(ClassicCDFExtractor[LabelDefinition]):
+class LabelsExtractor(ClassicCDFBaseExtractor[LabelDefinition]):
     """Extract data from Cognite Data Fusions Labels into Neat.
 
     Args:
@@ -78,7 +78,7 @@ class LabelsExtractor(ClassicCDFExtractor[LabelDefinition]):
         if not label.external_id:
             return []
 
-        id_ = self.namespace[f"Label_{create_sha256_hash(label.external_id)}"]
+        id_ = self.namespace[f"{InstanceIdPrefix.label}{self._label_id(label)}"]
 
         type_ = self._get_rdf_type(label)
         # Set rdf type
@@ -107,8 +107,16 @@ class LabelsExtractor(ClassicCDFExtractor[LabelDefinition]):
                 (
                     id_,
                     self.namespace.data_set_id,
-                    self.namespace[f"Dataset_{label.data_set_id}"],
+                    self.namespace[f"{InstanceIdPrefix.data_set}{label.data_set_id}"],
                 )
             )
 
         return triples
+
+    @staticmethod
+    def _label_id(label: Label | LabelDefinition) -> str:
+        # external_id can create ill-formed URIs, so we create websafe URIs
+        # since labels do not have internal ids, we use the external_id as the id
+        if label.external_id is None:
+            raise ValueError("External id must be set of the label")
+        return quote(label.external_id)