cognite-neat 0.101.0__py3-none-any.whl → 0.103.0__py3-none-any.whl

cognite/neat/_session/_show.py

@@ -21,6 +21,30 @@ from .exceptions import session_class_wrapper
 
 @session_class_wrapper
 class ShowAPI:
+    """Visualise a verified data model or instances contained in the graph store.
+    See, for example, `.data_model()` or `.instances()` for more.
+
+    Example:
+        Show instances
+        ```python
+        from cognite.neat import NeatSession
+        from cognite.neat import get_cognite_client
+
+        client = get_cognite_client(env_file_path=".env")
+        neat = NeatSession(client, storage="oxigraph") # Storage optimised for storage visualisation
+
+        # .... intermediate steps of reading, infering verifying and converting a data model and instances
+
+        neat.show.instances()
+        ```
+
+    Example:
+        Show data model
+        ```python
+        neat.show.data_model()
+        ```
+    """
+
     def __init__(self, state: SessionState) -> None:
         self._state = state
         self.data_model = ShowDataModelAPI(self._state)
@@ -65,6 +89,8 @@ class ShowBaseAPI:
 
 @session_class_wrapper
 class ShowDataModelAPI(ShowBaseAPI):
+    """Visualises the verified data model."""
+
     def __init__(self, state: SessionState) -> None:
         super().__init__(state)
         self._state = state
@@ -97,9 +123,18 @@ class ShowDataModelAPI(ShowBaseAPI):
         """Generate a DiGraph from the last verified DMS rules."""
         di_graph = nx.DiGraph()
 
+        # Views with properties or used as ValueType
+        # If a view is not used in properties or as ValueType, it is not added to the graph
+        # as we typically do not have the properties for it.
+        used_views = {prop_.view for prop_ in rules.properties} | {
+            prop_.value_type for prop_ in rules.properties if isinstance(prop_.value_type, ViewEntity)
+        }
+
         # Add nodes and edges from Views sheet
         for view in rules.views:
-            # if possible use human readable label coming from the view name
+            if view.view not in used_views:
+                continue
+            # if possible use humanreadable label coming from the view name
             if not di_graph.has_node(view.view.suffix):
                 di_graph.add_node(view.view.suffix, label=view.view.suffix)
 
@@ -123,7 +158,7 @@ class ShowDataModelAPI(ShowBaseAPI):
 
         # Add nodes and edges from Views sheet
         for class_ in rules.classes:
-            # if possible use human readable label coming from the view name
+            # if possible use humanreadable label coming from the view name
             if not di_graph.has_node(class_.class_.suffix):
                 di_graph.add_node(
                     class_.class_.suffix,
@@ -228,6 +263,8 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
 
 @session_class_wrapper
 class ShowDataModelProvenanceAPI(ShowBaseAPI):
+    """Visualises the provenance or steps that have been executed in the NeatSession."""
+
     def __init__(self, state: SessionState) -> None:
         super().__init__(state)
         self._state = state
@@ -288,6 +325,8 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
 
 @session_class_wrapper
 class ShowInstanceAPI(ShowBaseAPI):
+    """Visualise the instances contained in the graph store."""
+
     def __init__(self, state: SessionState) -> None:
         super().__init__(state)
         self._state = state

cognite/neat/_session/_to.py

@@ -19,6 +19,11 @@ from .exceptions import NeatSessionError, session_class_wrapper
 
 @session_class_wrapper
 class ToAPI:
+    """API used to write the contents of a NeatSession to a specified destination. For instance writing information
+    rules or DMS rules to a NEAT rules Excel spreadsheet, or writing a verified data model to CDF.
+
+    """
+
     def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
         self._state = state
         self._verbose = verbose
@@ -34,8 +39,22 @@ class ToAPI:
         Args:
             io: The file path or file-like object to write the Excel file to.
             model: The format of the data model to export. Defaults to None.
+
+        Example:
+            Export information model to excel rules sheet
+            ```python
+            information_rules_file_name = "information_rules.xlsx"
+            neat.to.excel(information_rules_file_name, model="information")
+            ```
+
+        Example:
+            Export data model to excel rules sheet
+            ```python
+            dms_rules_file_name = "dms_rules.xlsx"
+            neat.to.excel(information_rules_file_name, model="dms")
+            ```
         """
-        exporter = exporters.ExcelExporter()
+        exporter = exporters.ExcelExporter(styling="maximal")
         rules: VerifiedRules
         if model == "information" or model == "logical":
             rules = self._state.data_model.last_verified_information_rules[1]
@@ -63,13 +82,33 @@ class ToAPI:
             format: The format of the YAML file. Defaults to "neat".
             skip_system_spaces: If True, system spaces will be skipped. Defaults to True.
 
-        ... note::
-
+        !!! note "YAML formats"
             - "neat": This is the format Neat uses to store the data model.
             - "toolkit": This is the format used by Cognite Toolkit, that matches the CDF API.
 
         Returns:
             str | None: If io is None, the YAML string will be returned. Otherwise, None will be returned.
+
+        Example:
+            Export to yaml file in the case of "neat" format
+            ```python
+            your_yaml_file_name = "neat_rules.yaml"
+            neat.to.yaml(your_yaml_file_name, format="neat")
+            ```
+
+        Example:
+            Export yaml files as a zip folder in the case of "toolkit" format
+            ```python
+            your_zip_folder_name = "toolkit_data_model_files.zip"
+            neat.to.yaml(your_zip_folder_name, format="toolkit")
+            ```
+
+        Example:
+            Export yaml files to a folder in the case of "toolkit" format
+            ```python
+            your_folder_name = "my_project/data_model_files"
+            neat.to.yaml(your_folder_name, format="toolkit")
+            ```
         """
         if format == "neat":
             exporter = exporters.YAMLExporter()
@@ -97,12 +136,21 @@ class ToAPI:
 
 @session_class_wrapper
 class CDFToAPI:
+    """Write a verified Data Model and Instances to CDF."""
+
     def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
         self._client = client
         self._state = state
         self._verbose = verbose
 
     def instances(self, space: str | None = None) -> UploadResultList:
+        """Export the verified DMS instances to CDF.
+
+        Args:
+            space: Name of instance space to use. Default is to suffix the schema space with '_instances'.
+            Note this space is required to be different than the space with the data model.
+
+        """
         if not self._client:
             raise NeatSessionError("No CDF client provided!")
 
@@ -120,6 +168,7 @@ class CDFToAPI:
             self._state.data_model.last_verified_dms_rules[1],
             self._state.instances.store,
             instance_space=space,
+            client=self._client,
         )
         result = loader.load_into_cdf(self._client)
         self._state.instances.outcome.append(result)
@@ -144,13 +193,12 @@ class CDFToAPI:
                 Note this only applies to spaces and containers if they contain data.
             components: The components to export. If None, all components will be exported. Defaults to None.
 
-        ... note::
-
-        - "fail": If any component already exists, the export will fail.
-        - "skip": If any component already exists, it will be skipped.
-        - "update": If any component already exists, it will be updated.
-        - "force": If any component already exists, and the update fails, it will be deleted and recreated.
-        - "recreate": All components will be deleted and recreated. The exception is spaces, which will be updated.
+        !!! note "Data Model creation modes"
+            - "fail": If any component already exists, the export will fail.
+            - "skip": If any component already exists, it will be skipped.
+            - "update": If any component already exists, it will be updated.
+            - "force": If any component already exists, and the update fails, it will be deleted and recreated.
+            - "recreate": All components will be deleted and recreated. The exception is spaces, which will be updated.
 
         """
 

cognite/neat/_session/engine/__init__.py

@@ -1,4 +1,4 @@
 from ._import import import_engine
 from ._load import load_neat_engine
 
-__all__ = ["load_neat_engine", "import_engine"]
+__all__ = ["import_engine", "load_neat_engine"]

cognite/neat/_session/engine/_load.py

@@ -41,8 +41,10 @@ def load_neat_engine(client: CogniteClient | None, location: Literal["newest", "
 
     if location == "newest" or not candidates:
         # Loading in reverse order of priority
-        # 3. Downloads folder
+        # 4. Downloads folder
         candidates = _load_from_path(Path.home() / "Downloads", pattern, lower_bound, upper_bound)
+        # 3. Current working directory
+        candidates.update(_load_from_path(Path.cwd(), pattern, lower_bound, upper_bound))
         # 2. CDF
         if client:
             candidates.update(_load_from_cdf(client, pattern, lower_bound, upper_bound, cache_dir))

cognite/neat/_store/__init__.py

@@ -1,3 +1,4 @@
-from ._base import NeatGraphStore
+from ._graph_store import NeatGraphStore
+from ._rules_store import NeatRulesStore
 
-__all__ = ["NeatGraphStore"]
+__all__ = ["NeatGraphStore", "NeatRulesStore"]

cognite/neat/_store/{_base.py → _graph_store.py}

@@ -195,8 +195,13 @@ class NeatGraphStore:
         self, class_neat_id: URIRef, property_link_pairs: dict[str, URIRef] | None
     ) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
         if self.rules is None:
-            warnings.warn("Rules not found in graph store!", stacklevel=2)
+            warnings.warn("Rules not found in graph store! Aborting!", stacklevel=2)
             return
+        if self.multi_type_instances:
+            warnings.warn(
+                "Multi typed instances detected, issues with loading can occur!",
+                stacklevel=2,
+            )
 
         if cls := InformationAnalysis(self.rules).classes_by_neat_id.get(class_neat_id):
             if property_link_pairs:
@@ -225,6 +230,11 @@ class NeatGraphStore:
         if self.rules is None:
             warnings.warn("Rules not found in graph store!", stacklevel=2)
             return
+        if self.multi_type_instances:
+            warnings.warn(
+                "Multi typed instances detected, issues with loading can occur!",
+                stacklevel=2,
+            )
 
         if class_entity not in [definition.class_ for definition in self.rules.classes]:
             warnings.warn("Desired type not found in graph!", stacklevel=2)
@@ -267,7 +277,6 @@ class NeatGraphStore:
 
         # get property types to guide process of removing or not namespaces from results
         property_types = InformationAnalysis(self.rules).property_types(class_entity)
-
         for instance_id in instance_ids:
             if res := self.queries.describe(
                 instance_id=instance_id,
@@ -292,6 +301,11 @@ class NeatGraphStore:
         if not self.rules:
             warnings.warn("Rules not found in graph store!", stacklevel=2)
             return
+        if self.multi_type_instances:
+            warnings.warn(
+                "Multi typed instances detected, issues with loading can occur!",
+                stacklevel=2,
+            )
 
         class_entity = ClassEntity(prefix=self.rules.metadata.prefix, suffix=class_)
 
@@ -301,6 +315,39 @@ class NeatGraphStore:
 
         yield from self._read_via_class_entity(class_entity)
 
+    def count_of_id(self, neat_id: URIRef) -> int:
+        """Count the number of instances of a given type
+
+        Args:
+            neat_id: Type for which instances are to be counted
+
+        Returns:
+            Number of instances
+        """
+        if not self.rules:
+            warnings.warn("Rules not found in graph store!", stacklevel=2)
+            return 0
+
+        class_entity = next(
+            (definition.class_ for definition in self.rules.classes if definition.neatId == neat_id), None
+        )
+        if not class_entity:
+            warnings.warn("Desired type not found in graph!", stacklevel=2)
+            return 0
+
+        if not (class_uri := InformationAnalysis(self.rules).class_uri(class_entity)):
+            warnings.warn(
+                f"Class {class_entity.suffix} does not have namespace defined for prefix {class_entity.prefix} Rules!",
+                stacklevel=2,
+            )
+            return 0
+
+        return self.count_of_type(class_uri)
+
+    def count_of_type(self, class_uri: URIRef) -> int:
+        query = f"SELECT (COUNT(?instance) AS ?instanceCount) WHERE {{ ?instance a <{class_uri}> }}"
+        return int(next(iter(self.graph.query(query)))[0])  # type: ignore[arg-type, index]
+
     def _parse_file(
         self,
         filepath: Path,
@@ -390,6 +437,10 @@ class NeatGraphStore:
     def summary(self) -> pd.DataFrame:
         return pd.DataFrame(self.queries.summarize_instances(), columns=["Type", "Occurrence"])
 
+    @property
+    def multi_type_instances(self) -> dict[str, list[str]]:
+        return self.queries.multi_type_instances()
+
     def _repr_html_(self) -> str:
         provenance = self.provenance._repr_html_()
         summary: pd.DataFrame = self.summary
@@ -404,6 +455,9 @@ class NeatGraphStore:
                 f"{cast(pd.DataFrame, self._shorten_summary(summary))._repr_html_()}"  # type: ignore[operator]
             )
 
+        if self.multi_type_instances:
+            summary_text += "<br><strong>Multi value instances detected! Loading could have issues!</strong></br>"  # type: ignore
+
         return f"{summary_text}" f"{provenance}"
 
     def _shorten_summary(self, summary: pd.DataFrame) -> pd.DataFrame:

cognite/neat/_store/_provenance.py

@@ -20,12 +20,13 @@ import uuid
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Optional, TypeVar
+from typing import Generic, Optional, TypeVar
 
 from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
 from rdflib import PROV, RDF, Literal, URIRef
 
 from cognite.neat._constants import CDF_NAMESPACE, DEFAULT_NAMESPACE
+from cognite.neat._issues import IssueList
 from cognite.neat._rules._shared import JustRules, ReadRules, VerifiedRules
 from cognite.neat._shared import FrozenNeatObject, NeatList, Triple
 
@@ -50,6 +51,7 @@ UNKNOWN_AGENT = Agent(acted_on_behalf_of="UNKNOWN", id_=DEFAULT_NAMESPACE["unkno
 @dataclass(frozen=True)
 class Entity:
     was_attributed_to: Agent
+    issues: IssueList = field(default_factory=IssueList)
     was_generated_by: Optional["Activity"] = field(default=None, repr=False)
     id_: URIRef = DEFAULT_NAMESPACE["graph-store"]
 
@@ -117,6 +119,14 @@ class Entity:
         )
 
 
+T_Result = TypeVar("T_Result")
+
+
+@dataclass(frozen=True)
+class ModelEntity(Entity, Generic[T_Result]):
+    result: T_Result | None = None
+
+
 INSTANCES_ENTITY = Entity(was_attributed_to=NEAT_AGENT, id_=CDF_NAMESPACE["instances"])
 
 

cognite/neat/_store/_rules_store.py

@@ -0,0 +1,20 @@
+from cognite.neat._issues import IssueList
+from cognite.neat._rules.exporters import BaseExporter
+from cognite.neat._rules.importers import BaseImporter
+from cognite.neat._rules.transformers import RulesTransformer
+
+from ._provenance import Provenance
+
+
+class NeatRulesStore:
+    def __init__(self):
+        self._provenance = Provenance()
+
+    def write(self, importer: BaseImporter) -> IssueList:
+        raise NotImplementedError()
+
+    def transform(self, transformer: RulesTransformer) -> IssueList:
+        raise NotImplementedError()
+
+    def read(self, exporter: BaseExporter) -> IssueList:
+        raise NotImplementedError()

cognite/neat/_utils/auth.py

@@ -11,7 +11,7 @@ from cognite.client.credentials import CredentialProvider, OAuthClientCredential
 from cognite.neat import _version
 from cognite.neat._utils.auxiliary import local_import
 
-__all__ = ["get_cognite_client", "EnvironmentVariables"]
+__all__ = ["EnvironmentVariables", "get_cognite_client"]
 
 _LOGIN_FLOW: TypeAlias = Literal["infer", "client_credentials", "interactive", "token"]
 _VALID_LOGIN_FLOWS = get_args(_LOGIN_FLOW)
@@ -32,10 +32,7 @@ def get_cognite_client(env_file_name: str) -> CogniteClient:
     """
     if not env_file_name.endswith(".env"):
         raise ValueError("env_file_name must end with '.env'")
-    with suppress(KeyError):
-        variables = EnvironmentVariables.create_from_environ()
-        return variables.get_client()
-
+    # First try to load from .env file in repository root
     repo_root = _repo_root()
     if repo_root:
         with suppress(KeyError, FileNotFoundError, TypeError):
@@ -43,6 +40,11 @@ def get_cognite_client(env_file_name: str) -> CogniteClient:
             client = variables.get_client()
             print(f"Found {env_file_name} file in repository root. Loaded variables from {env_file_name} file.")
             return client
+    # Then try to load from environment variables
+    with suppress(KeyError):
+        variables = EnvironmentVariables.create_from_environ()
+        return variables.get_client()
+    # If not found, prompt the user
     variables = _prompt_user()
     if repo_root and _env_in_gitignore(repo_root, env_file_name):
         local_import("rich", "jupyter")

cognite/neat/_utils/reader/__init__.py

@@ -1,3 +1,3 @@
 from ._base import GitHubReader, HttpFileReader, NeatReader, PathReader
 
-__all__ = ["NeatReader", "PathReader", "GitHubReader", "HttpFileReader"]
+__all__ = ["GitHubReader", "HttpFileReader", "NeatReader", "PathReader"]

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.101.0"
-__engine__ = "^2.0.1"
+__version__ = "0.103.0"
+__engine__ = "^2.0.3"

cognite/neat/_workflows/__init__.py

@@ -9,9 +9,9 @@ from cognite.neat._workflows.model import (
 
 __all__ = [
     "BaseWorkflow",
-    "WorkflowStepDefinition",
+    "FlowMessage",
+    "WorkflowFullStateReport",
     "WorkflowManager",
+    "WorkflowStepDefinition",
     "WorkflowStepEvent",
-    "WorkflowFullStateReport",
-    "FlowMessage",
 ]

cognite/neat/_workflows/steps/lib/current/graph_extractor.py

@@ -10,7 +10,7 @@ from cognite.neat._workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat._workflows.steps.data_contracts import MultiRuleData, NeatGraph
 from cognite.neat._workflows.steps.step_model import Configurable, Step
 
-__all__ = ["GraphFromRdfFile", "GraphFromMockData"]
+__all__ = ["GraphFromMockData", "GraphFromRdfFile"]
 
 CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
 

cognite/neat/_workflows/steps/lib/current/rules_exporter.py

@@ -16,12 +16,12 @@ from cognite.neat._workflows.steps.data_contracts import CogniteClient, MultiRul
 from cognite.neat._workflows.steps.step_model import Configurable, Step
 
 __all__ = [
+    "DeleteDataModelFromCDF",
     "RulesToDMS",
     "RulesToExcel",
     "RulesToOntology",
     "RulesToSHACL",
     "RulesToSemanticDataModel",
-    "DeleteDataModelFromCDF",
 ]
 
 

cognite/neat/_workflows/steps/lib/current/rules_importer.py

@@ -19,10 +19,10 @@ from cognite.neat._workflows.steps.step_model import Configurable, Step
 CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
 
 __all__ = [
+    "DMSToRules",
     "ExcelToRules",
-    "OntologyToRules",
     "IMFToRules",
-    "DMSToRules",
+    "OntologyToRules",
     "RulesInferenceFromRdfFile",
 ]
 

cognite/neat/_workflows/steps/lib/io/io_steps.py

@@ -14,11 +14,11 @@ CATEGORY = "IO Steps"
 
 
 __all__ = [
-    "DownloadFileFromGitHub",
-    "UploadFileToGitHub",
+    "DownloadDataFromRestApiToFile",
     "DownloadFileFromCDF",
+    "DownloadFileFromGitHub",
     "UploadFileToCDF",
-    "DownloadDataFromRestApiToFile",
+    "UploadFileToGitHub",
 ]
 
 

{cognite_neat-0.101.0.dist-info → cognite_neat-0.103.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.101.0
+Version: 0.103.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0