cognite-neat 0.120.0__tar.gz → 0.121.0__tar.gz

{cognite_neat-0.120.0 → cognite_neat-0.121.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-neat
-Version: 0.120.0
+Version: 0.121.0
 Summary: Knowledge graph transformation
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/

cognite_neat-0.121.0/cognite/neat/_version.py

@@ -0,0 +1,2 @@
+__version__ = "0.121.0"
+__engine__ = "^2.0.4"

{cognite_neat-0.120.0 → cognite_neat-0.121.0}/cognite/neat/core/_graph/extractors/_classic_cdf/_files.py

@@ -20,7 +20,7 @@ class FilesExtractor(ClassicCDFBaseExtractor[FileMetadata]):
         data_set_external_id: str,
     ) -> tuple[int | None, Iterable[FileMetadata]]:
         items = client.files(data_set_external_ids=data_set_external_id)
-        return None, items
+        return None, cls._filter_out_instance_id(items)
 
     @classmethod
     def _from_hierarchy(
@@ -30,9 +30,16 @@ class FilesExtractor(ClassicCDFBaseExtractor[FileMetadata]):
             filter=FileMetadataFilter(asset_subtree_ids=[{"externalId": root_asset_external_id}])
         )[0].count
         items = client.files(asset_subtree_external_ids=root_asset_external_id)
-        return total, items
+        return total, cls._filter_out_instance_id(items)
 
     @classmethod
     def _from_file(cls, file_path: str | Path) -> tuple[int | None, Iterable[FileMetadata]]:
         file_metadata = FileMetadataList.load(Path(file_path).read_text())
-        return len(file_metadata), file_metadata
+        return len(file_metadata), cls._filter_out_instance_id(file_metadata)
+
+    @classmethod
+    def _filter_out_instance_id(cls, items: Iterable[FileMetadata]) -> Iterable[FileMetadata]:
+        """Filter out TimeSeries with InstanceId."""
+        # If the InstanceId is not None, it means that the TimeSeries is already connected to CogniteTimeSeries in DMS.
+        # We do not want to download it again.
+        return (item for item in items if item.instance_id is None)

{cognite_neat-0.120.0 → cognite_neat-0.121.0}/cognite/neat/session/_read.py

@@ -468,6 +468,63 @@ class CDFClassicAPI(BaseReadAPI):
 
         return extract_issues
 
+    def file_metadata(self, data_set_external_id: str, identifier: Literal["id", "externalId"] = "id") -> IssueList:
+        """Read the file metadata from CDF into NEAT.
+
+        Note all files that have InstanceId set will be silently skipped. This method is for extracting
+        non-contextualized file medata only. If you want to include the potential connection from file metadata
+        to assets, use the `neat.read.cdf.graph()` method instead and select the asset hierarchy connected to this file.
+
+        Args:
+            data_set_external_id: The external id of the data set
+            identifier: The identifier to use for the file metadata. Note selecting "id" can cause issues
+                if the external ID of the file metadata is missing. Default is "id".
+
+        Returns:
+            IssueList: A list of issues that occurred during the extraction.
+
+        Example:
+            ```python
+            neat.read.cdf.time_series("data_set_external_id")
+            ```
+        """
+        namespace = CLASSIC_CDF_NAMESPACE
+        self._state._raise_exception_if_condition_not_met(
+            "Read time series",
+            empty_rules_store_required=True,
+            empty_instances_store_required=True,
+            client_required=True,
+        )
+        extractor = extractors.FilesExtractor.from_dataset(
+            cast(NeatClient, self._state.client),
+            data_set_external_id=data_set_external_id,
+            namespace=namespace,
+            identifier=identifier,
+            prefix="Classic",
+            skip_connections=True,
+        )
+        self._state.instances.neat_prefix_by_predicate_uri.update(
+            {
+                namespace["dataSetId"]: InstanceIdPrefix.data_set,
+                namespace["assetId"]: InstanceIdPrefix.asset,
+            }
+        )
+        self._state.instances.neat_prefix_by_type_uri.update(
+            {namespace[f"Classic{extractor._default_rdf_type}"]: InstanceIdPrefix.time_series}
+        )
+        extract_issues = self._state.instances.store.write(extractor)
+
+        if identifier == "externalId":
+            self._state.quoted_source_identifiers = True
+
+        self._state.instances.store.transform(
+            LiteralToEntity(None, namespace["source"], "ClassicSourceSystem", "name"),
+        )
+        # The above transformations creates a new type, so we need to update
+        self._state.instances.neat_prefix_by_type_uri.update({namespace["ClassicSourceSystem"]: "ClassicSourceSystem_"})
+
+        return extract_issues
+
 
 @session_class_wrapper
 class ExcelReadAPI(BaseReadAPI):

cognite_neat-0.121.0/last_changelog_entry.md

@@ -0,0 +1,5 @@
+
+### Added
+
+- Support for reading file metadata.
+`neat.read.cdf.classic.file_metdata(...)`

cognite_neat-0.121.0/last_git_message.txt

@@ -0,0 +1,18 @@
+[NEAT-924] 🗒Read Files (#1153)
+
+# Description
+
+In addition, added skipping of files that have `instanceId` set. These
+have a counterpart in DMS that should be ingested instead.
+
+## Bump
+
+- [ ] Patch
+- [x] Minor
+- [ ] Skip
+
+## Changelog
+### Added
+
+- Support for reading file metadata.
+`neat.read.cdf.classic.file_metdata(...)`

cognite_neat-0.121.0/last_version.txt

@@ -0,0 +1 @@
+0.120.0

{cognite_neat-0.120.0 → cognite_neat-0.121.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite-neat"
-version = "0.120.0"
+version = "0.121.0"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

cognite_neat-0.121.0/tests/data/_instances/asset_centric_cdf/files.yaml

@@ -0,0 +1,24 @@
+- externalId: geo_located_area
+  name: geo_located_area.nc
+  source: ACME
+  dataSetId: 6931754270713290
+  id: 146805958863362
+  createdTime: 1668777262326
+  lastUpdatedTime: 1668778466394
+  uploaded: true
+  uploadedTime: 1668778540224
+  assetIds:
+    - 5132527530441957
+    - 78504378486679
+- externalId: geo_location_area2
+  name: geo_location_area2.nc
+  source: ACME
+  dataSetId: 6931754270713290
+  id: 146805958863362
+  createdTime: 1668777262326
+  lastUpdatedTime: 1668778466394
+  uploaded: true
+  uploadedTime: 1668778540224
+  assetIds:
+    - 5132527530441957
+    - 78504378486679

{cognite_neat-0.120.0 → cognite_neat-0.121.0}/tests/tests_unit/test_graph/test_extractors/test_files_extractor.py

@@ -15,4 +15,4 @@ def test_timeseries_extractor():
     for triple in FilesExtractor.from_dataset(client_mock, data_set_external_id="some data set").extract():
         g.add(triple)
 
-    assert len(g) == 11
+    assert len(g) == 13

{cognite_neat-0.120.0 → cognite_neat-0.121.0}/tests/tests_unit/test_session/test_read.py

@@ -1,4 +1,4 @@
-from cognite.client.data_classes import TimeSeriesList
+from cognite.client.data_classes import FileMetadataList, TimeSeriesList
 from cognite.client.data_classes.data_modeling import NodeId
 
 from cognite.neat import NeatSession
@@ -51,3 +51,47 @@ class TestReadClassicTimeSeries:
             assert "isString" in properties
             value = properties["isString"][0]
             assert isinstance(value, str), f"The {instance_id} has not converted the isSting from bool to enum"
+
+    def test_read_file_metadata(self) -> None:
+        with monkeypatch_neat_client() as client:
+            file_metadata = FileMetadataList.load(InstanceData.AssetCentricCDF.files_yaml.read_text(encoding="utf-8"))
+            assert len(file_metadata) >= 2
+            # FileMetadata with InstanceId should be skipped
+            file_metadata[0].instance_id = NodeId("already", "connected")
+            expected_connection_drop = sum(
+                1 for fm in file_metadata for _ in fm.asset_ids or [] if fm.instance_id is None
+            )
+            client.files.return_value = file_metadata
+
+            neat: NeatSession = NeatSession(client)
+
+        issues = neat.read.cdf.classic.file_metadata("my_data_set", identifier="externalId")
+
+        dropped_connections: list[NeatValueWarning] = []
+        unexpected_issues: list[NeatIssue] = []
+        for issue in issues:
+            if isinstance(issue, NeatValueWarning) and issue.value.startswith("Skipping connection"):
+                dropped_connections.append(issue)
+            else:
+                unexpected_issues.append(issue)
+        assert not unexpected_issues
+        assert len(dropped_connections) == expected_connection_drop
+
+        instances_ids = sorted((id_ for id_, _ in neat._state.instances.store.queries.select.list_instances_ids()))
+
+        expected = sorted(
+            [
+                CLASSIC_CDF_NAMESPACE[f"{InstanceIdPrefix.file}{fm.external_id}"]
+                for fm in file_metadata
+                if fm.instance_id is None
+                # Neat automatically converts the source string to a new entity
+            ]
+            + list(
+                {
+                    CLASSIC_CDF_NAMESPACE[f"ClassicSourceSystem_{fm.source}"]
+                    for fm in file_metadata
+                    if fm.instance_id is None and fm.source
+                }
+            )
+        )
+        assert instances_ids == expected

{cognite_neat-0.120.0 → cognite_neat-0.121.0}/uv.lock

@@ -346,7 +346,7 @@ wheels = [
 
 [[package]]
 name = "cognite-neat"
-version = "0.120.0"
+version = "0.121.0"
 source = { editable = "." }
 dependencies = [
     { name = "backports-strenum", marker = "python_full_version < '3.11'" },

cognite_neat-0.120.0/cognite/neat/_version.py

@@ -1,2 +0,0 @@
-__version__ = "0.120.0"
-__engine__ = "^2.0.4"

cognite_neat-0.120.0/last_changelog_entry.md

@@ -1,5 +0,0 @@
-
-### Added
-
-- Support for reading time series.
-`neat.read.cdf.classic.time_series(...)`.

cognite_neat-0.120.0/last_git_message.txt

@@ -1,18 +0,0 @@
-[NEAT-923] ⏱ Read timeseries (#1148)
-
-# Description
-
-Open question, what to do about timeseries with `assetId`. Do we look it
-up or just skip that field?
-
-## Bump
-
-- [ ] Patch
-- [x] Minor
-- [ ] Skip
-
-## Changelog
-### Added
-
-- Support for reading time series.
-`neat.read.cdf.classic.time_series(...)`.

cognite_neat-0.120.0/last_version.txt

@@ -1 +0,0 @@
-0.119.9

cognite_neat-0.120.0/tests/data/_instances/asset_centric_cdf/files.yaml

@@ -1,12 +0,0 @@
-- externalId: geo_located_area
-  name: geo_located_area.nc
-  source: ACME
-  dataSetId: 6931754270713290
-  id: 146805958863362
-  createdTime: 1668777262326
-  lastUpdatedTime: 1668778466394
-  uploaded: true
-  uploadedTime: 1668778540224
-  assetIds:
-    - 5132527530441957
-    - 78504378486679