cognite-neat 0.70.1__py3-none-any.whl → 0.127.19__py3-none-any.whl

cognite/neat/v0/session/_wizard.py

@@ -0,0 +1,44 @@
+from collections.abc import Sequence
+from typing import Literal, TypeVar, get_args
+
+from rich.prompt import IntPrompt, Prompt
+
+from cognite.neat.v0.core._data_model._constants import PATTERNS
+
+RDFFileType = Literal["Ontology", "IMF Types", "Inference"]
+NeatObjectType = Literal["Data Model", "Instances"]
+XMLFileType = Literal["dexpi", "aml"]
+
+
+def object_wizard(message: str = "Select object") -> NeatObjectType:
+    return _selection(message, get_args(NeatObjectType))
+
+
+def xml_format_wizard(message: str = "Select XML format") -> XMLFileType:
+    return _selection(message, get_args(XMLFileType))
+
+
+def rdf_dm_wizard(message: str = "Select source:") -> RDFFileType:
+    return _selection(message, get_args(RDFFileType))
+
+
+_T_Option = TypeVar("_T_Option")
+
+
+def _selection(message: str, options: Sequence[_T_Option]) -> _T_Option:
+    option_text = "\n  ".join([f"{i + 1}) {option}" for i, option in enumerate(options)])
+    selected_index = (
+        IntPrompt().ask(f"{message}\n  {option_text}\n", choices=list(map(str, range(1, len(options) + 1)))) - 1
+    )
+    return options[selected_index]
+
+
+def space_wizard(message: str = "Set space", space: str | None = None) -> str:
+    while True:
+        user_input = space or Prompt().ask(f"{message}:")
+        if PATTERNS.space_compliance.match(str(user_input)):
+            return user_input
+        else:
+            print(f"Invalid input. Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
+
+        space = ""

cognite/neat/v0/session/engine/__init__.py

@@ -0,0 +1,4 @@
+from ._import import import_engine
+from ._load import load_neat_engine
+
+__all__ = ["import_engine", "load_neat_engine"]

cognite/neat/v0/session/engine/_import.py

@@ -0,0 +1,7 @@
+from ._interface import NeatEngine
+
+
+def import_engine() -> NeatEngine:
+    from neatengine import NeatEngine  # type: ignore[import-not-found]
+
+    return NeatEngine()

cognite/neat/v0/session/engine/_interface.py

@@ -0,0 +1,25 @@
+from collections.abc import Iterable
+from typing import Any, Protocol
+
+from rdflib import Literal, URIRef
+
+
+class Extractor(Protocol):
+    def extract(self) -> Iterable[tuple[URIRef, URIRef, Literal | URIRef]]: ...
+
+
+class ConfigAPI(Protocol):
+    format: str | None = None
+    file: Any | None
+    type: str | None
+    primary_key: str | None
+    mapping: Any | None = None
+
+
+class NeatEngine(Protocol):
+    version: str = "2.0.5"
+
+    @property
+    def set(self) -> ConfigAPI: ...
+
+    def create_extractor(self) -> Extractor: ...

cognite/neat/v0/session/engine/_load.py

@@ -0,0 +1,131 @@
+import os
+import re
+import shutil
+import sys
+import tempfile
+import warnings
+from collections.abc import Callable
+from pathlib import Path
+from typing import Literal, cast
+
+from cognite.client import CogniteClient
+from packaging.version import Version
+from packaging.version import parse as parse_version
+
+from cognite.neat._version import __engine__
+from cognite.neat.v0.core._issues.errors import NeatValueError
+
+ENVIRONMENT_VARIABLE = "NEATENGINE"
+PACKAGE_NAME = "neatengine"
+PYVERSION = f"{sys.version_info.major}{sys.version_info.minor}"
+
+
+def load_neat_engine(client: CogniteClient | None, location: Literal["newest", "cache"]) -> str | None:
+    if location not in ["newest", "cache"]:
+        raise NeatValueError(f"Cannot load engine from location: {location}")
+
+    if not __engine__.startswith("^"):
+        # Using value error as this is a developer error
+        raise ValueError(f"Invalid engine version: {__engine__}")
+
+    lower_bound = parse_version(__engine__[1:])
+    upper_bound = Version(f"{lower_bound.major + 1}.0.0")
+
+    cache_dir = Path(tempfile.gettempdir()) / PACKAGE_NAME
+    cache_dir.mkdir(exist_ok=True)
+    pattern = re.compile(rf"{PACKAGE_NAME}-(\d+\.\d+\.\d+)-{PYVERSION}.zip")
+
+    candidates: dict[Version, Callable[[], Path]] = {}
+    if location == "cache" and cache_dir.exists():
+        candidates = _load_from_path(cache_dir, pattern, lower_bound, upper_bound)
+
+    if location == "newest" or not candidates:
+        # Loading in reverse order of priority
+        # 4. Downloads folder
+        candidates = _load_from_path(Path.home() / "Downloads", pattern, lower_bound, upper_bound)
+        # 3. Current working directory
+        candidates.update(_load_from_path(Path.cwd(), pattern, lower_bound, upper_bound))
+        # 2. CDF
+        if client:
+            candidates.update(_load_from_cdf(client, pattern, lower_bound, upper_bound, cache_dir))
+        # 1. Environment variable
+        if ENVIRONMENT_VARIABLE in os.environ:
+            environ_path = Path(os.environ[ENVIRONMENT_VARIABLE])
+            if environ_path.exists():
+                candidates.update(_load_from_path(environ_path, pattern, lower_bound, upper_bound))
+            else:
+                warnings.warn(
+                    f"Environment variable {ENVIRONMENT_VARIABLE} points to non-existing path: {environ_path}",
+                    UserWarning,
+                    stacklevel=2,
+                )
+
+    if not candidates:
+        return None
+
+    selected_version = max(candidates.keys(), default=None)
+    if not selected_version:
+        return None
+    source_path = candidates[selected_version]()
+    destination_path = cache_dir / source_path.name
+    if not destination_path.exists():
+        shutil.copy(source_path, destination_path)
+    sys.path.append(str(destination_path))
+    try:
+        from neatengine._version import __version__ as engine_version  # type: ignore[import-not-found]
+    except ImportError:
+        return None
+    return engine_version
+
+
+def _load_from_path(
+    path: Path, pattern: re.Pattern[str], lower_bound: Version, upper_bound: Version
+) -> dict[Version, Callable[[], Path]]:
+    if path.is_file() and (match := pattern.match(path.name)):
+        version = parse_version(match.group(1))
+        if lower_bound <= version < upper_bound:
+            return {parse_version(match.group(1)): lambda: path}
+        return {}
+    elif path.is_dir():
+        output: dict[Version, Callable[[], Path]] = {}
+        for candidate in path.iterdir():
+            if candidate.is_file() and (match := pattern.match(candidate.name)):
+                version = parse_version(match.group(1))
+                if lower_bound <= version < upper_bound:
+                    # Setting default value to ensure we use the candidate from the current iteration
+                    # If not set, the function will use the last candidate from the loop
+                    def return_path(the_path: Path = candidate) -> Path:
+                        return the_path
+
+                    output[parse_version(match.group(1))] = return_path
+
+        return output
+    return {}
+
+
+def _load_from_cdf(
+    client: CogniteClient, pattern: re.Pattern[str], lower_bound: Version, upper_bound: Version, cache_dir: Path
+) -> dict[Version, Callable[[], Path]]:
+    file_metadata = client.files.list(
+        limit=-1,
+        data_set_external_ids=PACKAGE_NAME,
+        external_id_prefix=PACKAGE_NAME,
+        metadata={"python_version": PYVERSION},
+    )
+    output: dict[Version, Callable[[], Path]] = {}
+    for file in file_metadata:
+        name = cast(str, file.name)
+
+        # Use function to lazily download file
+        # Setting default value to ensure we use the file_id from the current iteration
+        # If not set, the function will use the last file_id from the loop
+        def download_file(file_id: int = file.id, filename: str = name) -> Path:
+            client.files.download(cache_dir, file_id)
+            return cache_dir / filename
+
+        if match := pattern.match(name):
+            version = parse_version(match.group(1))
+            if lower_bound <= version < upper_bound:
+                output[version] = download_file
+
+    return output

cognite/neat/v0/session/exceptions.py

@@ -0,0 +1,103 @@
+import functools
+import warnings
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+from cognite.neat.v0.core._issues.errors import CDFMissingClientError, NeatImportError
+from cognite.neat.v0.core._issues.errors._external import OxigraphStorageLockedError
+from cognite.neat.v0.core._issues.errors._general import NeatValueError, WillExceedLimitError
+from cognite.neat.v0.plugins._issues import PluginError
+from cognite.neat.v0.session._experimental import ExperimentalFeatureWarning
+
+from ._collector import _COLLECTOR
+
+try:
+    from rich import print
+    from rich.markup import escape
+
+    _ERROR_PREFIX = "[bold red][ERROR][/bold red]"
+    _WARNING_PREFIX = "[bold bright_magenta][WARNING][/bold bright_magenta]"
+except ImportError:
+    _ERROR_PREFIX = "[ERROR]"
+    _WARNING_PREFIX = "[WARNING]"
+
+    def escape(x: Any, *_: Any, **__: Any) -> Any:  # type: ignore[misc]
+        return x
+
+
+class NeatSessionError(Exception):
+    """Base class for all exceptions raised by the NeatSession class."""
+
+    ...
+
+
+def _session_method_wrapper(func: Callable, cls_name: str) -> Any:
+    @functools.wraps(func)
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        _COLLECTOR.track_session_command(f"{cls_name}.{func.__name__}", *args, **kwargs)
+        try:
+            with warnings.catch_warnings(record=True) as w:
+                result = func(*args, **kwargs)
+                for warning in w:
+                    if isinstance(warning.message, ExperimentalFeatureWarning):
+                        print(f"{_WARNING_PREFIX} {warning.message}")
+
+            return result
+        except NeatSessionError as e:
+            action = _get_action()
+            print(f"{_ERROR_PREFIX} Cannot {action}: {e}")
+        except (
+            CDFMissingClientError,
+            NeatImportError,
+            NeatValueError,
+            OxigraphStorageLockedError,
+            PluginError,
+            WillExceedLimitError,
+        ) as e:
+            print(f"{_ERROR_PREFIX} {escape(e.as_message())}")
+        except ModuleNotFoundError as e:
+            if e.name == "neatengine":
+                action = _get_action()
+                print(f"{_ERROR_PREFIX} The functionality {action} requires the NeatEngine.")
+            else:
+                raise e
+
+    def _get_action() -> str:
+        action = func.__name__
+        if action == "__call__":
+            action = func.__qualname__.removesuffix(".__call__").removesuffix("API")
+        return action
+
+    return wrapper
+
+
+T_Class = TypeVar("T_Class", bound=object)
+
+
+def session_class_wrapper(cls: type[T_Class]) -> type[T_Class]:
+    """This decorator wraps all methods of a class.
+
+    It should be used with all composition classes used with the NeatSession class.
+
+    It does the following:
+        * Intercepts all NeatSession exceptions and prints them in a user-friendly way.
+        * Collects user metrics.
+
+    Args:
+        cls: NeatSession composition class
+
+    Returns:
+        cls: NeatSession composition class with all methods wrapped
+    """
+    to_check = [cls]
+    while to_check:
+        cls = to_check.pop()
+        for attr_name in dir(cls):
+            if attr_name.startswith("_") and not attr_name == "__call__":
+                continue
+            attr = getattr(cls, attr_name)
+            if callable(attr):
+                setattr(cls, attr_name, _session_method_wrapper(attr, cls.__name__))
+            elif isinstance(attr, type):
+                to_check.append(attr)
+    return cls

cognite/neat/v1.py

@@ -0,0 +1,3 @@
+from ._session import NeatSession
+
+__all__ = ["NeatSession"]

cognite_neat-0.127.19.dist-info/METADATA

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: cognite-neat
+Version: 0.127.19
+Summary: Knowledge graph transformation
+Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
+Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/
+Project-URL: GitHub, https://github.com/cognitedata/neat
+Project-URL: Changelog, https://github.com/cognitedata/neat/releases
+Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: backports-strenum<2.0.0,>=1.2; python_version < '3.11'
+Requires-Dist: cognite-sdk<8.0.0,>=7.83.0
+Requires-Dist: elementpath<5.0.0,>=4.0.0
+Requires-Dist: exceptiongroup<2.0.0,>=1.1.3; python_version < '3.11'
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: jsonpath-python<2.0.0,>=1.0.6
+Requires-Dist: mixpanel<5.0.0,>=4.10.1
+Requires-Dist: networkx<4.0.0,>=3.4.2
+Requires-Dist: openpyxl<4.0.0,>=3.0.10
+Requires-Dist: packaging>=22.0
+Requires-Dist: pandas<3.0.0,>=1.5.3
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: pyvis<1.0.0,>=0.3.2
+Requires-Dist: pyyaml<7.0.0,>=6.0.1
+Requires-Dist: rdflib<8.0.0,>=7.0.0
+Requires-Dist: requests<3.0.0,>=2.28.1
+Requires-Dist: rich[jupyter]<14.0.0,>=13.7.1
+Requires-Dist: tomli<3.0.0,>=2.0.1; python_version < '3.11'
+Requires-Dist: typing-extensions<5.0.0,>=4.8.0; python_version < '3.11'
+Requires-Dist: urllib3<3.0.0,>=1.26.15
+Provides-Extra: docs
+Requires-Dist: mistune==3.0.2; extra == 'docs'
+Requires-Dist: mkdocs-autorefs<1.0.0,>=0.5.0; extra == 'docs'
+Requires-Dist: mkdocs-git-authors-plugin<1.0.0,>=0.9.4; extra == 'docs'
+Requires-Dist: mkdocs-git-revision-date-localized-plugin; extra == 'docs'
+Requires-Dist: mkdocs-gitbook<1.0.0,>=0.0.1; extra == 'docs'
+Requires-Dist: mkdocs-glightbox<1.0.0,>=0.4.0; extra == 'docs'
+Requires-Dist: mkdocs-jupyter<1.0.0,>=0.25.1; extra == 'docs'
+Requires-Dist: mkdocs-material-extensions<2.0.0,>=1.3.1; extra == 'docs'
+Requires-Dist: mkdocs<2.0.0,>=1.4.0; extra == 'docs'
+Requires-Dist: mkdocstrings[python]<1.0.0,>=0.25.2; extra == 'docs'
+Requires-Dist: pymdown-extensions<11.0.0,>=10.14.3; extra == 'docs'
+Provides-Extra: google
+Requires-Dist: google-api-python-client<3.0.0,>=2.70.0; extra == 'google'
+Requires-Dist: google-auth-oauthlib<2.0.0,>=1.0.0; extra == 'google'
+Requires-Dist: gspread<6.0.0,>=5.0.0; extra == 'google'
+Provides-Extra: lxml
+Requires-Dist: lxml<6.0.0,>=5.3.0; extra == 'lxml'
+Provides-Extra: oxi
+Requires-Dist: oxrdflib<0.5.0,>=0.4.0; extra == 'oxi'
+Requires-Dist: pyoxigraph<0.5.0,>=0.4.3; extra == 'oxi'
+Description-Content-Type: text/markdown
+
+# kNowlEdge grAph Transformer (NEAT)
+
+[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/neat/release.yaml?style=for-the-badge)](https://github.com/cognitedata/neat/actions/workflows/release.yaml)
+[![Documentation Status](https://readthedocs.com/projects/cognite-neat/badge/?version=latest&style=for-the-badge)](https://cognite-neat.readthedocs-hosted.com/en/latest/?badge=latest)
+[![Github](https://shields.io/badge/github-cognite/neat-green?logo=github&style=for-the-badge)](https://github.com/cognitedata/neat)
+[![PyPI](https://img.shields.io/pypi/v/cognite-neat?style=for-the-badge)](https://pypi.org/project/cognite-neat/)
+[![Downloads](https://img.shields.io/pypi/dm/cognite-neat?style=for-the-badge)](https://pypistats.org/packages/cognite-neat)
+[![Docker Pulls](https://img.shields.io/docker/pulls/cognite/neat?style=for-the-badge)](https://hub.docker.com/r/cognite/neat)
+[![GitHub](https://img.shields.io/github/license/cognitedata/neat?style=for-the-badge)](https://github.com/cognitedata/neat/blob/master/LICENSE)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=for-the-badge)](https://github.com/ambv/black)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&style=for-the-badge)](https://github.com/astral-sh/ruff)
+[![mypy](https://img.shields.io/badge/mypy-checked-000000.svg?style=for-the-badge&color=blue)](http://mypy-lang.org)
+
+NEAT is a domain expert centric and developer friendly solution for rapid:
+
+- data modeling
+- extraction, transformation and loading of instances
+- and ingestion of the models and instances (i.e. knowledge graphs) into [Cognite Data Fusion](https://www.cognite.com/en/product/cognite_data_fusion_industrial_dataops_platform)
+
+NEAT is using open and globally recognized standards maintained by the [World Wide Web Consortium (W3C)](https://www.w3.org/RDF/).
+NEAT represents an essential tool for creation of standardized, machine-actionable, linked and semantic (meta)data.
+
+> NEAT is a funny acronym derived from k**N**owl**Ed**ge gr**A**ph **T**ransformer produced using [ACRONIMIFY](https://acronymify.com/NEAT/?q=knowledge+graph+transformer).
+
+
+## History
+
+NEAT emerged from years of experience in semantic tooling and knowledge graph development. The foundation was laid in 2020 with [sheet2rdf](https://github.com/nikokaoja/sheet2rdf), an Excel-based tool that trained data stewards to build domain-specific knowledge graphs and supported CI/CD processes in the Dutch Covid program and european wind energy community.
+
+By mid of 2022, sheet2rdf was used in several POCs in Cognite. As Cognite's Data Modeling Service (DMS) development progressed, the need for simplified data modeling experience led to demonstration of proto-NEAT, known as [sheet2fdm](https://github.com/cognitedata/sheet2fdm), an extension of sheet2rdf, enabling semantic data model definitions in OWL, SHACL, Python and GraphQL (see e.g., [wind energy data model](https://cognitedata.github.io/wind-energy-data-model/)) using a simplified version of sheet2rdf Excel template.
+
+Presented in various forums in 2022, this approach paved the way for NEAT’s formal development in November 2022 to enable cost-saving and empowerment of Cognite customers to self-sufficiently maintain and onboard knowledge graphs to Cognite Data Fusion.
+
+## Authorship
+
+### Authors
+The plot below shows the NEAT authorship from the start until present day.
+
+![NEAT authorship](./artifacts/figs/authorship.png)
+
+#### Current authors
+- [Nikola Vasiljević](www.linkedin.com/in/thisisnikola)
+- [Anders Albert](https://www.linkedin.com/in/anders-albert-00790483/)
+
+#### Former authors
+- [Aleksandrs Livincovs](https://www.linkedin.com/in/aleksandrslivincovs/)
+- [Julia Graham](https://www.linkedin.com/in/julia-graham-959a78a7/)
+- [Rogerio Júnior](https://www.linkedin.com/in/rogerio-saboia-j%C3%BAnior-087118a7/)
+
+### Contributors
+We are very grateful for the contributions made by:
+
+- [Kristina Tomičić](https://www.linkedin.com/in/kristina-tomicic-6bb443108/), who implemented Data Model and Instances visualization
+- [Marie Solvik Lepoutre](https://www.linkedin.com/in/mslepoutre/), who improved RDF triples projections to Cognite Data Fusion
+- [Bård Henning Tvedt](https://www.linkedin.com/in/bhtvedt/), who implemented IMF importer
+- [Hassan Gomaa](https://www.linkedin.com/in/dr-hassan-gomaa-232638121/), who extended the DEXPI extractor
+
+
+## Sponsors
+NEAT is developed and maintained by Cognite. We are grateful for the past support of our sponsors, who funded us to develop NEAT and to make it open source.
+
+- [Statnett](https://www.statnett.no/) - the MIMIR team ([Ola Hagen Øyan](https://www.linkedin.com/in/ola-%C3%B8yan-b0205b19/), [Olav Westeng Alstad](https://www.linkedin.com/in/olav-w-alstad-52329191/),[Andreas Kimsås](https://www.linkedin.com/in/andreas-kims%C3%A5s-964a0b2/) and [Anders Willersrud](https://www.linkedin.com/in/anders-willersrud-13a20220/)) – who supported the development of NEAT from end of 2022 to mid of 2023 and its integration with Statnett's infrastructure, where NEAT was battle-tested as a tool for non-sematic experts to define data models and transform large knowledge graphs representing the entire Norwegian power grid system. Without Statnett's support NEAT would not exist in the first place, and would not be open-source.
+- [Aker Solutions](https://www.akersolutions.com/) – the IMod Team (currently lead by [Maria Kenderkova](https://www.linkedin.com/in/maria-kenderkova/)), who funded development of NEAT from mid of 2023 till end of 2024 (multi-level and role-based data modeling, support for ontologies, IMF, DEXPI, AML,...) , as well who were early adopters and embraced NEAT and who were patient with us when things did not work so well. Aker Solutions was instrumental to elevating NEAT to a product level, and who selflessly advocate for NEAT globally.
+
+
+## Installation
+
+```bash
+pip install cognite-neat
+```
+
+## Usage
+
+The user interface for `NEAT` is a notebook-based environment. Once you have set up your notebook
+environment, you start by creating a `CogniteClient` and instantiate a `NeatSession` object.
+
+```python
+from cognite.neat import NeatSession, get_cognite_client
+
+client = get_cognite_client(".env")
+
+neat = NeatSession(client)
+
+neat.read.cdf.data_model(("my_space", "MyDataModel", "v1"))
+```
+
+## Documentation
+
+
+For more information, see the [documentation](https://cognite-neat.readthedocs-hosted.com/en/latest/)