cognite-neat 0.123.32__tar.gz → 0.123.34__tar.gz

cognite_neat-0.123.34/.gitignore

@@ -0,0 +1,182 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+.idea/
+config.yaml
+real_data/
+data/
+dev-data/
+dev-data-*/
+
+!cognite/neat/workflows/examples/config.yaml
+
+requirements.txt
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+!cognite/neat/workflows/steps/lib
+
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+!tests/data
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+
+# mypy
+.mypy_cache/
+data/.DS_Store
+.vscode/settings.json
+.DS_Store
+.vscode/
+.DS_Store
+default.profraw
+
+# private data
+data/cim-export
+data/temp
+
+docker/vol_data/*
+docker/vol_shared/*
+
+cognite/neat/ui/neat-app/node_modules
+!cognite/neat/ui/neat-app/build
+
+
+# local notebooks
+local/
+playground/
+
+tests/data/report.txt
+tests/data/transformation_rules.xlsx
+docs/tutorial/notebooks/power-grid-graph-capture.xls
+tests/data/970_0ca0c919-046a-4940-b191-f5cc4d0e6513.json
+build/*
+scripts/tmp/*
+scripts/config_manual_test.yaml
+real_cases/
+*.env
+
+# Excel chache files
+~*.xlsx
+docs/**/*.aml
+docs/**/*.xml
+tmp/

cognite_neat-0.123.34/PKG-INFO

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: cognite-neat
+Version: 0.123.34
+Summary: Knowledge graph transformation
+Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
+Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/
+Project-URL: GitHub, https://github.com/cognitedata/neat
+Project-URL: Changelog, https://github.com/cognitedata/neat/releases
+Author-email: Nikola Vasiljevic <nikola.vasiljevic@cognite.com>, Anders Albert <anders.albert@cognite.com>, Rogerio Júnior <rogerio.junior@cognite.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: backports-strenum<2.0.0,>=1.2; python_version < '3.11'
+Requires-Dist: cognite-sdk<8.0.0,>=7.83.0
+Requires-Dist: elementpath<5.0.0,>=4.0.0
+Requires-Dist: exceptiongroup<2.0.0,>=1.1.3; python_version < '3.11'
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: jsonpath-python<2.0.0,>=1.0.6
+Requires-Dist: mixpanel<5.0.0,>=4.10.1
+Requires-Dist: networkx<4.0.0,>=3.4.2
+Requires-Dist: openpyxl<4.0.0,>=3.0.10
+Requires-Dist: packaging>=22.0
+Requires-Dist: pandas<3.0.0,>=1.5.3
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: pyvis<1.0.0,>=0.3.2
+Requires-Dist: pyyaml<7.0.0,>=6.0.1
+Requires-Dist: rdflib<8.0.0,>=7.0.0
+Requires-Dist: requests<3.0.0,>=2.28.1
+Requires-Dist: rich[jupyter]<14.0.0,>=13.7.1
+Requires-Dist: tomli<3.0.0,>=2.0.1; python_version < '3.11'
+Requires-Dist: typing-extensions<5.0.0,>=4.8.0; python_version < '3.11'
+Requires-Dist: urllib3<3.0.0,>=1.26.15
+Provides-Extra: docs
+Requires-Dist: mistune==3.0.2; extra == 'docs'
+Requires-Dist: mkdocs-autorefs<1.0.0,>=0.5.0; extra == 'docs'
+Requires-Dist: mkdocs-git-authors-plugin<1.0.0,>=0.9.4; extra == 'docs'
+Requires-Dist: mkdocs-git-revision-date-localized-plugin; extra == 'docs'
+Requires-Dist: mkdocs-gitbook<1.0.0,>=0.0.1; extra == 'docs'
+Requires-Dist: mkdocs-glightbox<1.0.0,>=0.4.0; extra == 'docs'
+Requires-Dist: mkdocs-jupyter<1.0.0,>=0.25.1; extra == 'docs'
+Requires-Dist: mkdocs-material-extensions<2.0.0,>=1.3.1; extra == 'docs'
+Requires-Dist: mkdocs<2.0.0,>=1.4.0; extra == 'docs'
+Requires-Dist: mkdocstrings[python]<1.0.0,>=0.25.2; extra == 'docs'
+Requires-Dist: pymdown-extensions<11.0.0,>=10.14.3; extra == 'docs'
+Provides-Extra: google
+Requires-Dist: google-api-python-client<3.0.0,>=2.70.0; extra == 'google'
+Requires-Dist: google-auth-oauthlib<2.0.0,>=1.0.0; extra == 'google'
+Requires-Dist: gspread<6.0.0,>=5.0.0; extra == 'google'
+Provides-Extra: lxml
+Requires-Dist: lxml<6.0.0,>=5.3.0; extra == 'lxml'
+Provides-Extra: oxi
+Requires-Dist: oxrdflib<0.5.0,>=0.4.0; extra == 'oxi'
+Requires-Dist: pyoxigraph<0.5.0,>=0.4.3; extra == 'oxi'
+Description-Content-Type: text/markdown
+
+# kNowlEdge grAph Transformer (NEAT)
+
+[![release](https://img.shields.io/github/actions/workflow/status/cognitedata/neat/release.yaml?style=for-the-badge)](https://github.com/cognitedata/neat/actions/workflows/release.yaml)
+[![Documentation Status](https://readthedocs.com/projects/cognite-neat/badge/?version=latest&style=for-the-badge)](https://cognite-neat.readthedocs-hosted.com/en/latest/?badge=latest)
+[![Github](https://shields.io/badge/github-cognite/neat-green?logo=github&style=for-the-badge)](https://github.com/cognitedata/neat)
+[![PyPI](https://img.shields.io/pypi/v/cognite-neat?style=for-the-badge)](https://pypi.org/project/cognite-neat/)
+[![Downloads](https://img.shields.io/pypi/dm/cognite-neat?style=for-the-badge)](https://pypistats.org/packages/cognite-neat)
+[![Docker Pulls](https://img.shields.io/docker/pulls/cognite/neat?style=for-the-badge)](https://hub.docker.com/r/cognite/neat)
+[![GitHub](https://img.shields.io/github/license/cognitedata/neat?style=for-the-badge)](https://github.com/cognitedata/neat/blob/master/LICENSE)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=for-the-badge)](https://github.com/ambv/black)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&style=for-the-badge)](https://github.com/astral-sh/ruff)
+[![mypy](https://img.shields.io/badge/mypy-checked-000000.svg?style=for-the-badge&color=blue)](http://mypy-lang.org)
+
+NEAT is a domain expert centric and developer friendly solution for rapid:
+
+- data modeling
+- extraction, transformation and loading of instances
+- and ingestion of the models and instances (i.e. knowledge graphs) into [Cognite Data Fusion](https://www.cognite.com/en/product/cognite_data_fusion_industrial_dataops_platform)
+
+NEAT is using open and globally recognized standards maintained by the [World Wide Web Consortium (W3C)](https://www.w3.org/RDF/).
+NEAT represents an essential tool for creation of standardized, machine-actionable, linked and semantic (meta)data.
+
+> NEAT is a funny acronym derived from k**N**owl**Ed**ge gr**A**ph **T**ransformer produced using [ACRONIMIFY](https://acronymify.com/NEAT/?q=knowledge+graph+transformer).
+
+
+## History
+
+NEAT emerged from years of experience in semantic tooling and knowledge graph development. The foundation was laid in 2020 with [sheet2rdf](https://github.com/nikokaoja/sheet2rdf), an Excel-based tool that trained data stewards to build domain-specific knowledge graphs and supported CI/CD processes in the Dutch Covid program and european wind energy community.
+
+By mid of 2022, sheet2rdf was used in several POCs in Cognite. As Cognite's Data Modeling Service (DMS) development progressed, the need for simplified data modeling experience led to demonstration of proto-NEAT, known as [sheet2fdm](https://github.com/cognitedata/sheet2fdm), an extension of sheet2rdf, enabling semantic data model definitions in OWL, SHACL, Python and GraphQL (see e.g., [wind energy data model](https://cognitedata.github.io/wind-energy-data-model/)) using a simplified version of sheet2rdf Excel template.
+
+Presented in various forums in 2022, this approach paved the way for NEAT’s formal development in November 2022 to enable cost-saving and empowerment of Cognite customers to self-sufficiently maintain and onboard knowledge graphs to Cognite Data Fusion.
+
+## Authorship
+
+### Authors
+The plot below shows the NEAT authorship from the start until present day.
+
+![NEAT authorship](./artifacts/figs/authorship.png)
+
+#### Current authors
+- [Nikola Vasiljević](www.linkedin.com/in/thisisnikola)
+- [Anders Albert](https://www.linkedin.com/in/anders-albert-00790483/)
+- [Rogerio Júnior](https://www.linkedin.com/in/rogerio-saboia-j%C3%BAnior-087118a7/)
+
+#### Former authors
+- [Aleksandrs Livincovs](https://www.linkedin.com/in/aleksandrslivincovs/)
+- [Julia Graham](https://www.linkedin.com/in/julia-graham-959a78a7/)
+
+### Contributors
+We are very grateful for the contributions made by:
+
+
+- [Marie Solvik Lepoutre](https://www.linkedin.com/in/mslepoutre/), who improved RDF triples projections to Cognite Data Fusion
+- [Bård Henning Tvedt](https://www.linkedin.com/in/bhtvedt/), who implemented IMF importer
+- [Hassan Gomaa](https://www.linkedin.com/in/dr-hassan-gomaa-232638121/), who extended the DEXPI extractor
+- [Kristina Tomičić](https://www.linkedin.com/in/kristina-tomicic-6bb443108/), who implemented Data Model and Instances visualization
+
+
+## Sponsors
+NEAT is developed and maintained by Cognite. We are grateful for the past support of our sponsors, who funded us to develop NEAT and to make it open source.
+
+- [Statnett](https://www.statnett.no/) - the MIMIR team ([Ola Hagen Øyan](https://www.linkedin.com/in/ola-%C3%B8yan-b0205b19/), [Olav Westeng Alstad](https://www.linkedin.com/in/olav-w-alstad-52329191/),[Andreas Kimsås](https://www.linkedin.com/in/andreas-kims%C3%A5s-964a0b2/) and [Anders Willersrud](https://www.linkedin.com/in/anders-willersrud-13a20220/)) – who supported the development of NEAT from end of 2022 to mid of 2023 and its integration with Statnett's infrastructure, where NEAT was battle-tested as a tool for non-sematic experts to define data models and transform large knowledge graphs representing the entire Norwegian power grid system. Without Statnett's support NEAT would not exist in the first place, and would not be open-source.
+- [Aker Solutions](https://www.akersolutions.com/) – the IMod Team (currently lead by [Maria Kenderkova](https://www.linkedin.com/in/maria-kenderkova/)), who funded development of NEAT from mid of 2023 till end of 2024 (multi-level and role-based data modeling, support for ontologies, IMF, DEXPI, AML,...) , as well who were early adopters and embraced NEAT and who were patient with us when things did not work so well. Aker Solutions was instrumental to elevating NEAT to a product level, and who selflessly advocate for NEAT globally.
+
+
+## Installation
+
+```bash
+pip install cognite-neat
+```
+
+## Usage
+
+The user interface for `NEAT` is a notebook-based environment. Once you have set up your notebook
+environment, you start by creating a `CogniteClient` and instantiate a `NeatSession` object.
+
+```python
+from cognite.neat import NeatSession, get_cognite_client
+
+client = get_cognite_client(".env")
+
+neat = NeatSession(client)
+
+neat.read.cdf.data_model(("my_space", "MyDataModel", "v1"))
+```
+
+## Documentation
+
+For more information, see the [documentation](https://cognite-neat.readthedocs-hosted.com/en/latest/)

cognite_neat-0.123.34/cognite/neat/__init__.py

@@ -0,0 +1,6 @@
+from cognite.neat.v0.core._utils.auth import get_cognite_client
+from cognite.neat.v0.session import NeatSession
+
+from ._version import __version__
+
+__all__ = ["NeatSession", "__version__", "get_cognite_client"]

cognite_neat-0.123.34/cognite/neat/_data_model/_constants.py

@@ -0,0 +1,3 @@
+from ._identifiers import NameSpace
+
+XML_SCHEMA_NAMESPACE = NameSpace("http://www.w3.org/2001/XMLSchema#")

cognite_neat-0.123.34/cognite/neat/_data_model/_identifiers.py

@@ -0,0 +1,61 @@
+from pydantic import HttpUrl, RootModel, ValidationError
+
+from cognite.neat.v0.core._utils.auxiliary import local_import
+
+
+class URI(RootModel[str]):
+    def __init__(self, value: str):
+        try:
+            # Use Pydantic's AnyUrl to validate the URI
+            _ = HttpUrl(value)
+        except ValidationError as e:
+            raise ValueError(f"Invalid URI: {value}") from e
+        super().__init__(value)
+
+    def __str__(self) -> str:
+        return self.root
+
+    def __repr__(self) -> str:
+        return f"URI({self.root!r})"
+
+    def as_rdflib_uriref(self):  # type: ignore[no-untyped-def]
+        # rdflib is an optional dependency, so import here
+        local_import("rdflib", "rdflib")
+        from rdflib import URIRef
+
+        return URIRef(self.root)
+
+
+class NameSpace(RootModel[str]):
+    def __init__(self, value: str):
+        try:
+            # Use Pydantic's AnyUrl to validate the URI
+            _ = HttpUrl(value)
+        except ValidationError as e:
+            raise ValueError(f"Invalid Namespace: {value}") from e
+        super().__init__(value)
+
+    def __str__(self) -> str:
+        return self.root
+
+    def __repr__(self) -> str:
+        return f"NameSpace({self.root!r})"
+
+    def term(self, name: str) -> URI:
+        # need to handle slices explicitly because of __getitem__ override
+        return URI(self.root + (name if isinstance(name, str) else ""))
+
+    def __getitem__(self, key: str) -> URI:  # type: ignore[override]
+        return self.term(key)
+
+    def __getattr__(self, name: str) -> URI:
+        if name.startswith("__"):  # ignore any special Python names!
+            raise AttributeError
+        return self.term(name)
+
+    def as_rdflib_namespace(self):  # type: ignore[no-untyped-def]
+        # rdflib is an optional dependency, so import here
+        local_import("rdflib", "rdflib")
+        from rdflib import Namespace
+
+        return Namespace(self.root)

cognite_neat-0.123.34/cognite/neat/_data_model/models/dms/_space.py

@@ -0,0 +1,44 @@
+from abc import ABC
+
+from pydantic import Field, field_validator
+
+from cognite.neat.v0.core._utils.text import humanize_collection
+
+from ._base import WriteableResource
+from ._constants import FORBIDDEN_SPACES, SPACE_FORMAT_PATTERN
+
+
+class Space(WriteableResource["SpaceRequest"], ABC):
+    space: str = Field(
+        description="The Space identifier (id).",
+        min_length=1,
+        max_length=43,
+        pattern=SPACE_FORMAT_PATTERN,
+    )
+    name: str | None = Field(None, description="Name of the space.", max_length=1024)
+    description: str | None = Field(None, description="The description of the space.", max_length=255)
+
+    @field_validator("space")
+    def check_forbidden_space_value(cls, val: str) -> str:
+        """Check the space name not present in forbidden set"""
+        if val in FORBIDDEN_SPACES:
+            raise ValueError(f"{val!r} is a reserved space. Reserved Spaces: {humanize_collection(FORBIDDEN_SPACES)}")
+        return val
+
+    def as_request(self) -> "SpaceRequest":
+        return SpaceRequest.model_validate(self.model_dump(by_alias=True))
+
+
+class SpaceResponse(Space):
+    created_time: int = Field(
+        description="When the space was created. The number of milliseconds since 00:00:00 Thursday, 1 January 1970, "
+        "Coordinated Universal Time (UTC), minus leap seconds."
+    )
+    last_updated_time: int = Field(
+        description="When the space was last updated. The number of milliseconds since 00:00:00 Thursday, "
+        "1 January 1970, Coordinated Universal Time (UTC), minus leap seconds."
+    )
+    is_global: bool = Field(description="Whether the space is a global space.")
+
+
+class SpaceRequest(Space): ...

cognite_neat-0.123.34/cognite/neat/_data_model/models/entities/__init__.py

@@ -0,0 +1,46 @@
+from ._constants import Undefined, Unknown
+from ._data_types import (
+    AnyURI,
+    Boolean,
+    DataType,
+    Date,
+    DateTime,
+    DateTimeStamp,
+    Double,
+    Enum,
+    File,
+    Float,
+    Integer,
+    Json,
+    Long,
+    Sequence,
+    String,
+    Timeseries,
+)
+from ._identifiers import URI, NameSpace
+from ._parser import ParsedEntity, parse_entity
+
+__all__ = [
+    "URI",
+    "AnyURI",
+    "Boolean",
+    "DataType",
+    "Date",
+    "DateTime",
+    "DateTimeStamp",
+    "Double",
+    "Enum",
+    "File",
+    "Float",
+    "Integer",
+    "Json",
+    "Long",
+    "NameSpace",
+    "ParsedEntity",
+    "Sequence",
+    "String",
+    "Timeseries",
+    "Undefined",
+    "Unknown",
+    "parse_entity",
+]

cognite_neat-0.123.34/cognite/neat/_data_model/models/entities/_base.py

@@ -0,0 +1,100 @@
+from functools import total_ordering
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator, model_serializer
+
+from ._constants import (
+    Undefined,
+    Unknown,
+    _UndefinedType,
+    _UnknownType,
+)
+
+
+@total_ordering
+class Entity(BaseModel, extra="ignore", populate_by_name=True):
+    """Entity is a concept, class, property, datatype in semantics sense."""
+
+    prefix: str | _UndefinedType = Field(
+        default=Undefined, pattern=r"^[a-zA-Z][a-zA-Z0-9_-]{0,41}[a-zA-Z0-9]?$", min_length=1, max_length=43
+    )
+    suffix: str = Field(min_length=1, max_length=255, pattern=r"^[a-zA-Z0-9._~?@!$&'*+,;=%-]+$")
+
+    @model_serializer(when_used="unless-none", return_type=str)
+    def as_str(self) -> str:
+        return str(self)
+
+    @field_validator("*", mode="before")
+    def strip_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        elif isinstance(value, list):
+            return [entry.strip() if isinstance(entry, str) else entry for entry in value]
+        return value
+
+    def as_tuple(self) -> tuple[str, ...]:
+        # We haver overwritten the serialization to str, so we need to do it manually
+        extra: tuple[str, ...] = tuple(
+            [
+                str(v or "")
+                for field_name in self.model_fields.keys()
+                if (v := getattr(self, field_name)) and field_name not in {"prefix", "suffix"}
+            ]
+        )
+        if isinstance(self.prefix, _UndefinedType):
+            return str(self.suffix), *extra
+        else:
+            return self.prefix, str(self.suffix), *extra
+
+    def __lt__(self, other: object) -> bool:
+        if type(other) is not type(self):
+            return NotImplemented
+        return self.as_tuple() < other.as_tuple()  # type: ignore[attr-defined]
+
+    def __eq__(self, other: object) -> bool:
+        # We need to be explicit that we are not allowing comparison between different types
+        if type(other) is not type(self):
+            # requires explicit raising as NotImplemented would lead to running comparison
+            raise TypeError(f"'==' not supported between instances of {type(self).__name__} and {type(other).__name__}")
+        return self.as_tuple() == other.as_tuple()  # type: ignore[attr-defined]
+
+    def __hash__(self) -> int:
+        return hash(f"{type(self).__name__}({self})")
+
+    def __str__(self) -> str:
+        # there are three cases for string representation:
+        # 1. only suffix is present -> return str(suffix)
+        # 2. prefix and suffix are present -> return "prefix:suffix"
+        # 3. prefix, suffix and other fields are present -> return "prefix:suffix(field1=value1,field2=value2)"
+
+        model_dump = {k: v for k in self.model_fields if (v := getattr(self, k)) is not None}
+
+        base_str = f"{self.prefix}:{self.suffix}" if not isinstance(self.prefix, _UndefinedType) else str(self.suffix)
+
+        extra_fields = {
+            (self.model_fields[k].alias or k): v for k, v in model_dump.items() if k not in {"prefix", "suffix"}
+        }
+        if extra_fields:
+            extra_str = ",".join([f"{k}={v}" for k, v in extra_fields.items()])
+            return f"{base_str}({extra_str})"
+        else:
+            return base_str
+
+    def __repr__(self) -> str:
+        # We have overwritten the serialization to str, so we need to do it manually
+        model_dump = {k: v for k in self.model_fields if (v := getattr(self, k)) is not None}
+        args = ",".join([f"{k}={v!r}" for k, v in model_dump.items()])
+        return f"{type(self).__name__}({args})"
+
+
+class ConceptEntity(Entity):
+    version: str | None = None
+
+
+class UnknownEntity(ConceptEntity):
+    prefix: _UndefinedType = Undefined
+    suffix: _UnknownType = Unknown  # type: ignore[assignment]
+
+    @property
+    def id(self) -> str:
+        return str(Unknown)