cognite-neat 0.86.0__tar.gz → 0.87.3__tar.gz

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.86.0
+Version: 0.87.3
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -21,7 +21,7 @@ Provides-Extra: oxi
 Provides-Extra: service
 Requires-Dist: PyYAML
 Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
-Requires-Dist: cognite-sdk (>=7.52.0,<8.0.0)
+Requires-Dist: cognite-sdk (>=7.54.6,<8.0.0)
 Requires-Dist: deepdiff
 Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
 Requires-Dist: fastapi (>=0,<1) ; extra == "service" or extra == "all"

cognite_neat-0.87.3/cognite/neat/_version.py

@@ -0,0 +1 @@
+__version__ = "0.87.3"

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/app/api/configuration.py

@@ -9,8 +9,6 @@ from fastapi import FastAPI
 from cognite import neat
 from cognite.neat.config import Config, configure_logging, copy_examples_to_directory, create_data_dir_structure
 from cognite.neat.constants import PACKAGE_DIRECTORY
-from cognite.neat.utils.cdf import ServiceCogniteClient
-from cognite.neat.utils.utils import get_cognite_client_from_config, get_cognite_client_from_token
 from cognite.neat.workflows.cdf_store import CdfStore
 from cognite.neat.workflows.manager import WorkflowManager
 from cognite.neat.workflows.triggers import TriggerManager
@@ -36,14 +34,7 @@ class NeatApp:
             self.config = config
         logging.info("Initializing global objects")
         if not self.cdf_client:
-            if isinstance(self.config.cdf_client, ServiceCogniteClient):
-                if self.config.cdf_client.client_id:
-                    self.cdf_client = get_cognite_client_from_config(self.config.cdf_client)
-                else:
-                    # If no client_id is provided, we assume that the token is provided instead of secret.
-                    self.cdf_client = get_cognite_client_from_token(self.config.cdf_client)
-            else:
-                raise ValueError("Only ServiceCogniteClient is supported at the moment.")
+            self.cdf_client = self.config.cdf_auth_config.get_client()
         self.cdf_store = CdfStore(
             self.cdf_client,
             self.config.cdf_default_dataset_id,

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/app/api/routers/data_exploration.py

@@ -17,7 +17,7 @@ from cognite.neat.app.api.data_classes.rest import (
 from cognite.neat.app.api.utils.data_mapping import rdf_result_to_api_response
 from cognite.neat.app.api.utils.query_templates import query_templates
 from cognite.neat.legacy.graph.transformations import query_generator
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
 
 router = APIRouter()

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/config.py

@@ -4,14 +4,14 @@ import os
 import shutil
 import sys
 from pathlib import Path
-from typing import Literal, cast
+from typing import Any, Literal, cast
 
 import yaml
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_serializer, model_validator
 from yaml import safe_load
 
 from cognite.neat.constants import EXAMPLE_GRAPHS, EXAMPLE_RULES, EXAMPLE_WORKFLOWS
-from cognite.neat.utils.cdf import InteractiveCogniteClient, ServiceCogniteClient
+from cognite.neat.utils.auth import EnvironmentVariables
 
 if sys.version_info >= (3, 11):
     from enum import StrEnum
@@ -41,7 +41,7 @@ class WorkflowsStoreType(StrEnum):
     URL = "url"
 
 
-class Config(BaseModel):
+class Config(BaseModel, arbitrary_types_allowed=True):
     workflows_store_type: WorkflowsStoreType = WorkflowsStoreType.FILE
     data_store_path: Path = Field(default_factory=lambda: Path.cwd() / "data")
 
@@ -51,7 +51,7 @@ class Config(BaseModel):
         default=None,
     )
 
-    cdf_client: InteractiveCogniteClient | ServiceCogniteClient = ServiceCogniteClient()
+    cdf_auth_config: EnvironmentVariables = Field(default_factory=EnvironmentVariables.default)
     cdf_default_dataset_id: int = 0
     load_examples: bool = True
 
@@ -60,9 +60,64 @@ class Config(BaseModel):
     download_workflows_from_cdf: bool = Field(
         default=False, description="Downloads all workflows from CDF automatically and stores them locally"
     )
-
     stop_on_error: bool = False
 
+    @model_validator(mode="before")
+    def backwards_compatible(cls, data: Any):
+        if not isinstance(data, dict):
+            return data
+        if "cdf_client" in data:
+            cdf_client = data["cdf_client"]
+            if isinstance(cdf_client, dict):
+                if "base_url" in cdf_client:
+                    base_url = cdf_client["base_url"]
+                    cluster = base_url.removeprefix("https://").removesuffix(".cognitedata.com")
+                else:
+                    base_url, cluster = "Missing", "Missing"
+                if "scopes" in cdf_client:
+                    scopes = cdf_client["scopes"]
+                    if isinstance(scopes, list):
+                        scopes = ",".join(scopes)
+                else:
+                    scopes = "Missing"
+                data["cdf_auth_config"] = EnvironmentVariables(
+                    CDF_PROJECT=cdf_client.get("project", "Missing"),
+                    CDF_CLUSTER=cluster,
+                    CDF_URL=base_url,
+                    IDP_CLIENT_ID=cdf_client.get("client_id", "Missing"),
+                    IDP_CLIENT_SECRET=cdf_client.get("client_secret", "Missing"),
+                    IDP_TOKEN_URL=cdf_client.get("token_url", "Missing"),
+                    IDP_SCOPES=scopes,
+                    CDF_TIMEOUT=int(cdf_client.get("timeout", 60)),
+                    CDF_MAX_WORKERS=int(cdf_client.get("max_workers", 3)),
+                )
+        return data
+
+    @staticmethod
+    @field_serializer("cdf_auth_config", when_used="always", return_type=dict)
+    def backwards_compatible_serialize(cdf_auth_config: EnvironmentVariables) -> dict[str, Any]:
+        output: dict[str, Any] = {}
+        config = cdf_auth_config
+        if config.CDF_PROJECT not in {"Missing", "NOT SET"}:
+            output["project"] = config.CDF_PROJECT
+        if config.CDF_CLUSTER not in {"Missing", "NOT SET"}:
+            output["cluster"] = config.CDF_CLUSTER
+        if config.CDF_URL:
+            output["base_url"] = config.CDF_URL
+        if config.IDP_CLIENT_ID:
+            output["client_id"] = config.IDP_CLIENT_ID
+        if config.IDP_CLIENT_SECRET:
+            output["client_secret"] = config.IDP_CLIENT_SECRET
+        if config.IDP_TOKEN_URL:
+            output["token_url"] = config.IDP_TOKEN_URL
+        if config.IDP_SCOPES:
+            output["scopes"] = config.idp_scopes
+        if config.CDF_TIMEOUT:
+            output["timeout"] = config.CDF_TIMEOUT
+        if config.CDF_MAX_WORKERS:
+            output["max_workers"] = config.CDF_MAX_WORKERS
+        return output
+
     @property
     def _dir_suffix(self) -> str:
         is_test_running = "pytest" in sys.modules
@@ -93,7 +148,7 @@ class Config(BaseModel):
 
     def to_yaml(self, filepath: Path):
         # Parse as json to avoid Path and Enum objects
-        dump = json.loads(self.json())
+        dump = json.loads(self.model_dump_json())
 
         with filepath.open("w") as f:
             yaml.safe_dump(dump, f)
@@ -101,15 +156,27 @@ class Config(BaseModel):
     @classmethod
     def from_env(cls) -> Self:
         missing = "Missing"
-        cdf_config = ServiceCogniteClient(
-            project=os.environ.get("NEAT_CDF_PROJECT", missing),
-            client_id=os.environ.get("NEAT_CDF_CLIENT_ID", missing),
-            client_secret=os.environ.get("NEAT_CDF_CLIENT_SECRET", missing),
-            base_url=os.environ.get("NEAT_CDF_BASE_URL", missing),
-            token_url=os.environ.get("NEAT_CDF_TOKEN_URL", missing),
-            scopes=[os.environ.get("NEAT_CDF_SCOPES", missing)],
-            timeout=int(os.environ.get("NEAT_CDF_CLIENT_TIMEOUT", "60")),
-            max_workers=int(os.environ.get("NEAT_CDF_CLIENT_MAX_WORKERS", "3")),
+        # This is to be backwards compatible with the old config
+
+        base_url: str | None = None
+        if "NEAT_CDF_BASE_URL" in os.environ:
+            base_url = os.environ["NEAT_CDF_BASE_URL"]
+        if isinstance(base_url, str):
+            cluster = base_url.removeprefix("https://").removesuffix(".cognitedata.com")
+        else:
+            cluster = missing
+        variables = EnvironmentVariables(
+            CDF_PROJECT=os.environ.get("NEAT_CDF_PROJECT", missing),
+            CDF_CLUSTER=cluster,
+            CDF_URL=base_url,
+            IDP_CLIENT_ID=os.environ.get("NEAT_CDF_CLIENT_ID"),
+            IDP_CLIENT_SECRET=os.environ.get("NEAT_CDF_CLIENT_SECRET"),
+            IDP_TOKEN_URL=os.environ.get("NEAT_CDF_TOKEN_URL"),
+            IDP_SCOPES=os.environ.get("NEAT_CDF_SCOPES"),
+            CDF_TIMEOUT=int(os.environ["NEAT_CDF_CLIENT_TIMEOUT"] if "NEAT_CDF_CLIENT_TIMEOUT" in os.environ else 60),
+            CDF_MAX_WORKERS=int(
+                os.environ["NEAT_CDF_CLIENT_MAX_WORKERS"] if "NEAT_CDF_CLIENT_MAX_WORKERS" in os.environ else 3
+            ),
         )
 
         if workflow_downloader_filter_value := os.environ.get("NEAT_WORKFLOW_DOWNLOADER_FILTER", None):
@@ -118,7 +185,7 @@ class Config(BaseModel):
             workflow_downloader_filter = None
 
         return cls(
-            cdf_client=cdf_config,
+            cdf_auth_config=variables,
             workflows_store_type=os.environ.get(  # type: ignore[arg-type]
                 "NEAT_WORKFLOWS_STORE_TYPE", WorkflowsStoreType.FILE
             ),

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/constants.py

@@ -14,15 +14,17 @@ EXAMPLE_WORKFLOWS = PACKAGE_DIRECTORY / "workflows" / "examples"
 
 DEFAULT_NAMESPACE = Namespace("http://purl.org/cognite/neat#")
 
-PREFIXES = {
-    "rdf": RDF._NS,
-    "rdfs": RDFS._NS,
-    "dct": DCTERMS._NS,
-    "skos": SKOS._NS,
-    "owl": OWL._NS,
-    "xsd": XSD._NS,
-    "pav": Namespace("http://purl.org/pav/"),
-}
+
+def get_default_prefixes() -> dict[str, Namespace]:
+    return {
+        "rdf": RDF._NS,
+        "rdfs": RDFS._NS,
+        "dct": DCTERMS._NS,
+        "skos": SKOS._NS,
+        "owl": OWL._NS,
+        "xsd": XSD._NS,
+        "pav": Namespace("http://purl.org/pav/"),
+    }
 
 
 DEFAULT_URI = ""

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -12,7 +12,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash, string_to_ideal_type
+from cognite.neat.utils.auxiliary import create_sha256_hash, string_to_ideal_type
 
 
 class AssetsExtractor(BaseExtractor):

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_events.py

@@ -12,7 +12,7 @@ from rdflib import RDF, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class EventsExtractor(BaseExtractor):

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -13,7 +13,7 @@ from rdflib import RDF, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class FilesExtractor(BaseExtractor):

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -10,7 +10,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash
+from cognite.neat.utils.auxiliary import create_sha256_hash
 
 
 class LabelsExtractor(BaseExtractor):

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -11,7 +11,7 @@ from rdflib import RDF, Literal, Namespace
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash
+from cognite.neat.utils.auxiliary import create_sha256_hash
 
 
 class RelationshipsExtractor(BaseExtractor):

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_sequences.py

@@ -12,7 +12,7 @@ from rdflib import RDF, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class SequencesExtractor(BaseExtractor):

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_classic_cdf/_timeseries.py

@@ -12,7 +12,7 @@ from rdflib import RDF, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class TimeSeriesExtractor(BaseExtractor):

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_dexpi.py

@@ -9,7 +9,7 @@ from rdflib import RDF, RDFS, XSD, Literal, Namespace, URIRef
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import as_neat_compliant_uri
+from cognite.neat.utils.rdf_ import as_neat_compliant_uri
 from cognite.neat.utils.xml_ import get_children, iterate_tree
 
 DEXPI = Namespace("http://sandbox.dexpi.org/rdl/")

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -12,12 +12,12 @@ import pandas as pd
 from rdflib import RDF, Literal, Namespace, URIRef
 
 from cognite.neat.graph.models import Triple
-from cognite.neat.rules.analysis import InformationArchitectRulesAnalysis
+from cognite.neat.rules.analysis import InformationAnalysis
 from cognite.neat.rules.models import DMSRules, InformationRules
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
 from cognite.neat.rules.models.information import InformationProperty
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExtractor
 
@@ -50,8 +50,7 @@ class MockGraphGenerator(BaseExtractor):
 
         if not class_count:
             self.class_count = {
-                class_: 1
-                for class_ in InformationArchitectRulesAnalysis(self.rules).defined_classes(consider_inheritance=True)
+                class_: 1 for class_ in InformationAnalysis(self.rules).defined_classes(consider_inheritance=True)
             }
         elif all(isinstance(key, str) for key in class_count.keys()):
             self.class_count = {
@@ -101,7 +100,7 @@ def generate_triples(
     """
 
     namespace = rules.metadata.namespace
-    defined_classes = InformationArchitectRulesAnalysis(rules).defined_classes(consider_inheritance=True)
+    defined_classes = InformationAnalysis(rules).defined_classes(consider_inheritance=True)
 
     if non_existing_classes := set(class_count.keys()) - defined_classes:
         msg = f"Class count contains classes {non_existing_classes} for which properties are not defined in Data Model!"
@@ -115,16 +114,16 @@ def generate_triples(
 
     # Subset data model to only classes that are defined in class count
     rules = (
-        InformationArchitectRulesAnalysis(rules).subset_rules(set(class_count.keys()))
+        InformationAnalysis(rules).subset_rules(set(class_count.keys()))
         if defined_classes != set(class_count.keys())
         else rules
     )
 
-    class_linkage = InformationArchitectRulesAnalysis(rules).class_linkage()
+    class_linkage = InformationAnalysis(rules).class_linkage().to_pandas()
 
     # Remove one of symmetric pairs from class linkage to maintain proper linking
     # among instances of symmetrically linked classes
-    if sym_pairs := InformationArchitectRulesAnalysis(rules).symmetrically_connected_classes():
+    if sym_pairs := InformationAnalysis(rules).symmetrically_connected_classes():
         class_linkage = _remove_higher_occurring_sym_pair(class_linkage, sym_pairs)
 
     # Remove any of symmetric pairs containing classes that are not present class count
@@ -134,7 +133,7 @@ def generate_triples(
     generation_order = _prettify_generation_order(_get_generation_order(class_linkage))
 
     # Generated simple view of data model
-    class_property_pairs = InformationArchitectRulesAnalysis(rules).classes_with_properties(consider_inheritance=True)
+    class_property_pairs = InformationAnalysis(rules).classes_with_properties(consider_inheritance=True)
 
     # pregenerate instance ids for each remaining class
     instance_ids = {

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/loaders/__init__.py

@@ -1,7 +1,8 @@
 from ._base import BaseLoader, CDFLoader
+from ._rdf2asset import AssetLoader
 from ._rdf2dms import DMSLoader
 
-__all__ = ["BaseLoader", "CDFLoader", "DMSLoader"]
+__all__ = ["BaseLoader", "CDFLoader", "DMSLoader", "AssetLoader"]
 
 
 def _repr_html_() -> str:
@@ -11,7 +12,9 @@ def _repr_html_() -> str:
         [
             {
                 "Loader": name,
-                "Description": globals()[name].__doc__.strip().split("\n")[0] if globals()[name].__doc__ else "Missing",
+                "Description": (
+                    globals()[name].__doc__.strip().split("\n")[0] if globals()[name].__doc__ else "Missing"
+                ),
             }
             for name in __all__
             if name not in ("BaseLoader", "CDFLoader")

{cognite_neat-0.86.0 → cognite_neat-0.87.3}/cognite/neat/graph/loaders/_base.py

@@ -15,6 +15,11 @@ from cognite.neat.utils.upload import UploadResult, UploadResultList
 T_Output = TypeVar("T_Output")
 
 
+# Sentinel value to indicate in the load method that all instances of a class have been loaded.
+# https://en.wikipedia.org/wiki/Sentinel_value
+class _END_OF_CLASS: ...
+
+
 class BaseLoader(ABC, Generic[T_Output]):
     _new_line = "\n"
     _encoding = "utf-8"
@@ -28,10 +33,10 @@ class BaseLoader(ABC, Generic[T_Output]):
 
     def load(self, stop_on_exception: bool = False) -> Iterable[T_Output | NeatIssue]:
         """Load the graph with data."""
-        return self._load(stop_on_exception)
+        return (item for item in self._load(stop_on_exception) if item is not _END_OF_CLASS)  # type: ignore[misc]
 
     @abstractmethod
-    def _load(self, stop_on_exception: bool = False) -> Iterable[T_Output | NeatIssue]:
+    def _load(self, stop_on_exception: bool = False) -> Iterable[T_Output | NeatIssue | type[_END_OF_CLASS]]:
         """Load the graph with data."""
         pass
 
@@ -58,13 +63,16 @@ class CDFLoader(BaseLoader[T_Output]):
 
         issues = NeatIssueList[NeatIssue]()
         items: list[T_Output] = []
-        for result in self.load(stop_on_exception=False):
+        for result in self._load(stop_on_exception=False):
             if isinstance(result, NeatIssue):
                 issues.append(result)
+            elif result is _END_OF_CLASS:
+                ...
             else:
-                items.append(result)
+                # MyPy does not understand else means the item will be of type T_Output
+                items.append(result)  # type: ignore[arg-type]
 
-            if len(items) >= self._UPLOAD_BATCH_SIZE:
+            if len(items) >= self._UPLOAD_BATCH_SIZE or result is _END_OF_CLASS:
                 yield from self._upload_to_cdf(client, items, dry_run, issues)
                 issues = NeatIssueList[NeatIssue]()
                 items = []