cognite-neat 0.105.1__py3-none-any.whl → 0.106.0__py3-none-any.whl

cognite/neat/_session/_read.py

@@ -22,23 +22,22 @@ from .exceptions import NeatSessionError, session_class_wrapper
 class ReadAPI:
     """Read from a data source into NeatSession graph store."""
 
-    def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
         self._state = state
         self._verbose = verbose
-        self.cdf = CDFReadAPI(state, client, verbose)
-        self.rdf = RDFReadAPI(state, client, verbose)
-        self.excel = ExcelReadAPI(state, client, verbose)
-        self.csv = CSVReadAPI(state, client, verbose)
-        self.yaml = YamlReadAPI(state, client, verbose)
-        self.xml = XMLReadAPI(state, client, verbose)
+        self.cdf = CDFReadAPI(state, verbose)
+        self.rdf = RDFReadAPI(state, verbose)
+        self.excel = ExcelReadAPI(state, verbose)
+        self.csv = CSVReadAPI(state, verbose)
+        self.yaml = YamlReadAPI(state, verbose)
+        self.xml = XMLReadAPI(state, verbose)
 
 
 @session_class_wrapper
 class BaseReadAPI:
-    def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
         self._state = state
         self._verbose = verbose
-        self._client = client
 
 
 @session_class_wrapper
@@ -48,15 +47,15 @@ class CDFReadAPI(BaseReadAPI):
 
     """
 
-    def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
-        super().__init__(state, client, verbose)
-        self.classic = CDFClassicAPI(state, client, verbose)
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        super().__init__(state, verbose)
+        self.classic = CDFClassicAPI(state, verbose)
 
     @property
     def _get_client(self) -> NeatClient:
-        if self._client is None:
+        if self._state.client is None:
             raise NeatValueError("No client provided. Please provide a client to read a data model.")
-        return self._client
+        return self._state.client
 
     def data_model(self, data_model_id: DataModelIdentifier) -> IssueList:
         """Reads a Data Model from CDF to the knowledge graph.
@@ -89,11 +88,11 @@ class CDFClassicAPI(BaseReadAPI):
 
     @property
     def _get_client(self) -> NeatClient:
-        if self._client is None:
+        if self._state.client is None:
             raise ValueError("No client provided. Please provide a client to read a data model.")
-        return self._client
+        return self._state.client
 
-    def graph(self, root_asset_external_id: str, limit_per_type: int | None = None) -> None:
+    def graph(self, root_asset_external_id: str, limit_per_type: int | None = None) -> IssueList:
         """Reads the classic knowledge graph from CDF.
 
         The Classic Graph consists of the following core resource type.
@@ -129,14 +128,24 @@ class CDFClassicAPI(BaseReadAPI):
             root_asset_external_id: The external id of the root asset
             limit_per_type: The maximum number of nodes to extract per core node type. If None, all nodes are extracted.
 
+        Returns:
+            IssueList: A list of issues that occurred during the extraction.
+
+        Example:
+            ```python
+            neat.read.cdf.graph("root_asset_external_id")
+            ```
+
         """
         extractor = extractors.ClassicGraphExtractor(
             self._get_client, root_asset_external_id=root_asset_external_id, limit_per_type=limit_per_type
         )
 
-        self._state.instances.store.write(extractor)
-        if self._verbose:
-            print(f"Classic Graph {root_asset_external_id} read successfully")
+        issues = self._state.instances.store.write(extractor)
+        issues.action = "Read Classic Graph"
+        if issues:
+            print("Use the .inspect.issues() for more details")
+        return issues
 
 
 @session_class_wrapper
@@ -153,9 +162,9 @@ class ExcelReadAPI(BaseReadAPI):
         ```
     """
 
-    def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
-        super().__init__(state, client, verbose)
-        self.examples = ExcelExampleAPI(state, client, verbose)
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        super().__init__(state, verbose)
+        self.examples = ExcelExampleAPI(state, verbose)
 
     def __call__(self, io: Any) -> IssueList:
         """Reads a Neat Excel Rules sheet to the graph store. The rules sheet may stem from an Information architect,
@@ -201,7 +210,7 @@ class YamlReadAPI(BaseReadAPI):
         if format == "neat":
             importer = importers.YAMLImporter.from_file(path, source_name=f"{reader!s}")
         elif format == "toolkit":
-            dms_importer = importers.DMSImporter.from_path(path, self._client)
+            dms_importer = importers.DMSImporter.from_path(path, self._state.client)
             if dms_importer.issue_list.has_warning_type(MissingCogniteClientWarning):
                 raise NeatSessionError(
                     "No client provided. You are referencing Cognite containers in your data model, "
@@ -316,8 +325,8 @@ class RDFReadAPI(BaseReadAPI):
         io: file path or url to the RDF source
     """
 
-    def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
-        super().__init__(state, client, verbose)
+    def __init__(self, state: SessionState, verbose: bool) -> None:
+        super().__init__(state, verbose)
         self.examples = RDFExamples(state)
 
     def ontology(self, io: Any) -> IssueList:

cognite/neat/_session/_set.py

@@ -1,5 +1,7 @@
+from cognite.client import CogniteClient
 from cognite.client import data_modeling as dm
 
+from cognite.neat._client import NeatClient
 from cognite.neat._constants import COGNITE_MODELS
 from cognite.neat._issues import IssueList
 from cognite.neat._rules.models import DMSRules
@@ -35,3 +37,10 @@ class SetAPI:
                     " due to temporarily issue with the reverse direct relation interpretation"
                 )
         return self._state.rule_transform(SetIDDMSModel(new_model_id))
+
+    def client(self, client: CogniteClient) -> None:
+        """Sets the client to be used in the session."""
+        self._state.client = NeatClient(client)
+        if self._verbose:
+            print(f"Client set to {self._state.client.config.project} CDF project.")
+        return None

cognite/neat/_session/_state.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass, field
 from typing import Literal, cast
 
+from cognite.neat._client import NeatClient
 from cognite.neat._issues import IssueList
 from cognite.neat._rules.importers import BaseImporter, InferenceImporter
 from cognite.neat._rules.models import DMSRules, InformationRules
@@ -15,10 +16,11 @@ from .exceptions import NeatSessionError
 
 
 class SessionState:
-    def __init__(self, store_type: Literal["memory", "oxigraph"]) -> None:
+    def __init__(self, store_type: Literal["memory", "oxigraph"], client: NeatClient | None = None) -> None:
         self.instances = InstancesState(store_type)
         self.rule_store = NeatRulesStore()
         self.last_reference: DMSRules | InformationRules | None = None
+        self.client = client
 
     def rule_transform(self, *transformer: RulesTransformer) -> IssueList:
         if not transformer:

cognite/neat/_session/_to.py

@@ -4,7 +4,6 @@ from typing import Any, Literal, overload
 
 from cognite.client.data_classes.data_modeling import SpaceApply
 
-from cognite.neat._client import NeatClient
 from cognite.neat._constants import COGNITE_MODELS
 from cognite.neat._graph import loaders
 from cognite.neat._rules import exporters
@@ -25,10 +24,10 @@ class ToAPI:
 
     """
 
-    def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
+    def __init__(self, state: SessionState, verbose: bool) -> None:
         self._state = state
         self._verbose = verbose
-        self.cdf = CDFToAPI(state, client, verbose)
+        self.cdf = CDFToAPI(state, verbose)
 
     def excel(
         self,
@@ -153,8 +152,7 @@ class ToAPI:
 class CDFToAPI:
     """Write a verified Data Model and Instances to CDF."""
 
-    def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
-        self._client = client
+    def __init__(self, state: SessionState, verbose: bool) -> None:
         self._state = state
         self._verbose = verbose
 
@@ -166,9 +164,9 @@ class CDFToAPI:
             Note this space is required to be different than the space with the data model.
 
         """
-        if not self._client:
+        if not self._state.client:
             raise NeatSessionError("No CDF client provided!")
-
+        client = self._state.client
         space = space or f"{self._state.rule_store.last_verified_dms_rules.metadata.space}_instances"
 
         if space and space == self._state.rule_store.last_verified_dms_rules.metadata.space:
@@ -176,16 +174,16 @@ class CDFToAPI:
         elif not PATTERNS.space_compliance.match(str(space)):
             raise NeatSessionError("Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
 
-        if not self._client.data_modeling.spaces.retrieve(space):
-            self._client.data_modeling.spaces.apply(SpaceApply(space=space))
+        if not client.data_modeling.spaces.retrieve(space):
+            client.data_modeling.spaces.apply(SpaceApply(space=space))
 
         loader = loaders.DMSLoader.from_rules(
             self._state.rule_store.last_verified_dms_rules,
             self._state.instances.store,
             instance_space=space,
-            client=self._client,
+            client=client,
         )
-        result = loader.load_into_cdf(self._client)
+        result = loader.load_into_cdf(client)
         self._state.instances.outcome.append(result)
         print("You can inspect the details with the .inspect.outcome.instances(...) method.")
         return result
@@ -219,9 +217,9 @@ class CDFToAPI:
 
         exporter = exporters.DMSExporter(existing=existing, export_components=components, drop_data=drop_data)
 
-        if not self._client:
+        if not self._state.client:
             raise NeatSessionError("No client provided!")
 
-        result = self._state.rule_store.export_to_cdf(exporter, self._client, dry_run)
+        result = self._state.rule_store.export_to_cdf(exporter, self._state.client, dry_run)
         print("You can inspect the details with the .inspect.outcome.data_model(...) method.")
         return result

cognite/neat/_store/_graph_store.py

@@ -15,6 +15,7 @@ from cognite.neat._graph._shared import rdflib_to_oxi_type
 from cognite.neat._graph.extractors import RdfFileExtractor, TripleExtractors
 from cognite.neat._graph.queries import Queries
 from cognite.neat._graph.transformers import Transformers
+from cognite.neat._issues import IssueList, catch_issues
 from cognite.neat._rules.analysis import InformationAnalysis
 from cognite.neat._rules.models import InformationRules
 from cognite.neat._rules.models.entities import ClassEntity
@@ -157,39 +158,43 @@ class NeatGraphStore:
 
         return cls(graph, rules)
 
-    def write(self, extractor: TripleExtractors) -> None:
-        _start = datetime.now(timezone.utc)
-        success = True
-
-        if isinstance(extractor, RdfFileExtractor) and not extractor.issue_list.has_errors:
-            self._parse_file(extractor.filepath, cast(str, extractor.format), extractor.base_uri)
-        elif isinstance(extractor, RdfFileExtractor):
-            success = False
-            issue_text = "\n".join([issue.as_message() for issue in extractor.issue_list])
-            warnings.warn(
-                f"Cannot write to graph store with {type(extractor).__name__}, errors found in file:\n{issue_text}",
-                stacklevel=2,
-            )
-        else:
-            self._add_triples(extractor.extract())
-
-        if success:
-            _end = datetime.now(timezone.utc)
-            # Need to do the hasattr in case the extractor comes from NeatEngine.
-            activities = (
-                extractor._get_activity_names()
-                if hasattr(extractor, "_get_activity_names")
-                else [type(extractor).__name__]
-            )
-            for activity in activities:
-                self.provenance.append(
-                    Change.record(
+    def write(self, extractor: TripleExtractors) -> IssueList:
+        last_change: Change | None = None
+        with catch_issues() as issue_list:
+            _start = datetime.now(timezone.utc)
+            success = True
+
+            if isinstance(extractor, RdfFileExtractor) and not extractor.issue_list.has_errors:
+                self._parse_file(extractor.filepath, cast(str, extractor.format), extractor.base_uri)
+            elif isinstance(extractor, RdfFileExtractor):
+                success = False
+                issue_text = "\n".join([issue.as_message() for issue in extractor.issue_list])
+                warnings.warn(
+                    f"Cannot write to graph store with {type(extractor).__name__}, errors found in file:\n{issue_text}",
+                    stacklevel=2,
+                )
+            else:
+                self._add_triples(extractor.extract())
+
+            if success:
+                _end = datetime.now(timezone.utc)
+                # Need to do the hasattr in case the extractor comes from NeatEngine.
+                activities = (
+                    extractor._get_activity_names()
+                    if hasattr(extractor, "_get_activity_names")
+                    else [type(extractor).__name__]
+                )
+                for activity in activities:
+                    last_change = Change.record(
                         activity=activity,
                         start=_start,
                         end=_end,
                         description=f"Extracted triples to graph store using {type(extractor).__name__}",
                     )
-                )
+                    self.provenance.append(last_change)
+        if last_change:
+            last_change.target_entity.issues.extend(issue_list)
+        return issue_list
 
     def _read_via_rules_linkage(
         self, class_neat_id: URIRef, property_link_pairs: dict[str, URIRef] | None

cognite/neat/_utils/auth.py

@@ -23,8 +23,9 @@ def get_cognite_client(env_file_name: str) -> CogniteClient:
     be prompted to enter them.
 
     Args:
-        env_file_name: The name of the .env file to look for in the repository root. If the file is found, the variables
-            will be loaded from the file. If the file is not found, the user will be prompted to enter the variables.
+        env_file_name: The name of the .env file to look for in the repository root / current working directory. If
+        the file is found, the variables will be loaded from the file. If the file is not found, the user will
+        be prompted to enter the variables and the file will be created.
 
     Returns:
         CogniteClient: A CogniteClient instance.
@@ -40,26 +41,44 @@ def get_cognite_client(env_file_name: str) -> CogniteClient:
             client = variables.get_client()
             print(f"Found {env_file_name} file in repository root. Loaded variables from {env_file_name} file.")
             return client
+    elif (Path.cwd() / env_file_name).exists():
+        with suppress(KeyError, FileNotFoundError, TypeError):
+            variables = _from_dotenv(Path.cwd() / env_file_name)
+            client = variables.get_client()
+            print(
+                f"Found {env_file_name} file in current working directory. Loaded variables from {env_file_name} file."
+            )
+            return client
     # Then try to load from environment variables
     with suppress(KeyError):
         variables = EnvironmentVariables.create_from_environ()
+        print("Loaded variables from environment variables.")
         return variables.get_client()
     # If not found, prompt the user
     variables = _prompt_user()
     if repo_root and _env_in_gitignore(repo_root, env_file_name):
-        local_import("rich", "jupyter")
-        from rich.prompt import Prompt
-
         env_file = repo_root / env_file_name
-        answer = Prompt.ask(
-            f"Do you store the variables in an {env_file_name} file in the repository root for easy reuse?",
-            choices=["y", "n"],
-        )
-        if env_file.exists():
-            answer = Prompt.ask(f"{env_file} already exists. Overwrite?", choices=["y", "n"])
-        if answer == "y":
-            env_file.write_text(variables.create_env_file())
-            print(f"Created {env_file_name} file in repository root.")
+        location = "repository root"
+    elif repo_root:
+        # We do not offer to create the file in the repository root if it is in .gitignore
+        # as an inexperienced user might accidentally commit it.
+        print("Cannot create .env file in repository root as there is no .env entry in the .gitignore.")
+        return variables.get_client()
+    else:
+        env_file = Path.cwd() / env_file_name
+        location = "current working directory"
+    local_import("rich", "jupyter")
+    from rich.prompt import Prompt
+
+    answer = Prompt.ask(
+        f"Do you store the variables in an {env_file_name} file in the {location} for easy reuse?",
+        choices=["y", "n"],
+    )
+    if env_file.exists():
+        answer = Prompt.ask(f"{env_file} already exists. Overwrite?", choices=["y", "n"])
+    if answer == "y":
+        env_file.write_text(variables.create_env_file())
+        print(f"Created {env_file_name} file in {location}.")
 
     return variables.get_client()
 
@@ -320,7 +339,8 @@ def _prompt_cluster_and_project() -> EnvironmentVariables:
 def _repo_root() -> Path | None:
     with suppress(Exception):
         result = subprocess.run("git rev-parse --show-toplevel".split(), stdout=subprocess.PIPE)
-        return Path(result.stdout.decode().strip())
+        if (output := result.stdout.decode().strip()) != "":
+            return Path(output)
     return None
 
 

cognite/neat/_utils/collection_.py

@@ -2,6 +2,7 @@ from collections import Counter
 from collections.abc import Iterable, Sequence
 from typing import TypeVar
 
+from cognite.neat._config import GLOBAL_CONFIG
 from cognite.neat._constants import IN_PYODIDE
 
 T_Element = TypeVar("T_Element")
@@ -26,20 +27,40 @@ def remove_list_elements(input_list: list, elements_to_remove: list) -> list:
 
 
 def iterate_progress_bar(iterable: Iterable[T_Element], total: int, description: str) -> Iterable[T_Element]:
-    if IN_PYODIDE:
+    if IN_PYODIDE or GLOBAL_CONFIG.progress_bar in ("infer", "tqdm"):
         try:
-            from tqdm import tqdm  # type: ignore [import]
+            from tqdm import tqdm
         except ModuleNotFoundError:
             return iterable
         return tqdm(iterable, total=total, desc=description)
-    # Progress bar from rich requires multi-threading, which is not supported in Pyodide
-    try:
-        from rich.progress import track
-    except ModuleNotFoundError:
+
+    elif GLOBAL_CONFIG.progress_bar == "tqdm-notebook":
+        try:
+            from tqdm.notebook import tqdm as tqdm_notebook
+        except ModuleNotFoundError:
+            return iterable
+        return tqdm_notebook(iterable, total=total, desc=description)
+    elif GLOBAL_CONFIG.progress_bar in ("infer", "rich"):
+        # Progress bar from rich requires multi-threading, which is not supported in Pyodide
+        try:
+            from rich.progress import track
+        except ModuleNotFoundError:
+            return iterable
+
+        return track(
+            iterable,
+            total=total,
+            description=description,
+        )
+    elif GLOBAL_CONFIG.progress_bar is None:
         return iterable
+    else:
+        raise ValueError(f"Unsupported progress bar type: {GLOBAL_CONFIG.progress_bar}")
+
 
-    return track(
-        iterable,
-        total=total,
-        description=description,
-    )
+def iterate_progress_bar_if_above_config_threshold(
+    iterable: Iterable[T_Element], total: int, description: str
+) -> Iterable[T_Element]:
+    if GLOBAL_CONFIG.use_iterate_bar_threshold and total > GLOBAL_CONFIG.use_iterate_bar_threshold:
+        return iterate_progress_bar(iterable, total, description)
+    return iterable

cognite/neat/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.105.1"
+__version__ = "0.106.0"
 __engine__ = "^2.0.3"

{cognite_neat-0.105.1.dist-info → cognite_neat-0.106.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cognite-neat
-Version: 0.105.1
+Version: 0.106.0
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -17,13 +17,11 @@ Provides-Extra: all
 Provides-Extra: docs
 Provides-Extra: google
 Provides-Extra: oxi
-Provides-Extra: service
 Requires-Dist: PyYAML
 Requires-Dist: backports.strenum (>=1.2,<2.0) ; python_version < "3.11"
 Requires-Dist: cognite-sdk (>=7.71.2,<8.0.0)
 Requires-Dist: elementpath (>=4.0.0,<5.0.0)
 Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0) ; python_version < "3.11"
-Requires-Dist: fastapi (>=0,<1) ; extra == "service" or extra == "all"
 Requires-Dist: google-api-python-client ; extra == "google"
 Requires-Dist: google-auth-oauthlib ; extra == "google"
 Requires-Dist: gspread ; extra == "google"
@@ -45,20 +43,16 @@ Requires-Dist: openpyxl
 Requires-Dist: oxrdflib[oxigraph] (>=0.4.0,<0.5.0) ; extra == "oxi" or extra == "all"
 Requires-Dist: packaging (>=22.0,<25.0)
 Requires-Dist: pandas
-Requires-Dist: prometheus-client (>=0,<1) ; extra == "service" or extra == "all"
 Requires-Dist: pydantic (>=2,<3)
 Requires-Dist: pymdown-extensions ; extra == "docs"
 Requires-Dist: pyoxigraph (==0.4.3) ; extra == "oxi" or extra == "all"
-Requires-Dist: python-multipart (==0.0.9) ; extra == "service" or extra == "all"
 Requires-Dist: pyvis (>=0.3.2,<0.4.0)
 Requires-Dist: rdflib
 Requires-Dist: requests
 Requires-Dist: rich[jupyter] (>=13.7.1,<14.0.0)
-Requires-Dist: schedule (>=1,<2) ; extra == "service" or extra == "all"
 Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"
 Requires-Dist: typing_extensions (>=4.8,<5.0) ; python_version < "3.11"
 Requires-Dist: urllib3 (>=2,<3)
-Requires-Dist: uvicorn[standard] (>=0,<1) ; extra == "service" or extra == "all"
 Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
 Project-URL: Repository, https://github.com/cognitedata/neat
 Description-Content-Type: text/markdown