cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +10 -27
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +21 -45
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +3 -17
- cognite/neat/_rules/models/_base_rules.py +118 -62
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +20 -178
- cognite/neat/_rules/models/dms/_rules.py +65 -128
- cognite/neat/_rules/models/dms/_rules_input.py +72 -56
- cognite/neat/_rules/models/dms/_validation.py +16 -109
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +19 -122
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +3 -6
- cognite/neat/_rules/transformers/_converters.py +128 -206
- cognite/neat/_rules/transformers/_mapping.py +105 -34
- cognite/neat/_rules/transformers/_verification.py +5 -16
- cognite/neat/_session/_base.py +83 -21
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +22 -10
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +222 -27
- cognite/neat/_session/_read.py +109 -19
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +27 -14
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
cognite/neat/_session/_to.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
2
|
from typing import Any, Literal, overload
|
|
3
3
|
|
|
4
|
-
from cognite.client import CogniteClient
|
|
5
4
|
from cognite.client.data_classes.data_modeling import SpaceApply
|
|
6
5
|
|
|
6
|
+
from cognite.neat._client import NeatClient
|
|
7
7
|
from cognite.neat._graph import loaders
|
|
8
8
|
from cognite.neat._issues import IssueList, catch_warnings
|
|
9
9
|
from cognite.neat._rules import exporters
|
|
@@ -12,12 +12,12 @@ from cognite.neat._rules._shared import VerifiedRules
|
|
|
12
12
|
from cognite.neat._utils.upload import UploadResultCore, UploadResultList
|
|
13
13
|
|
|
14
14
|
from ._state import SessionState
|
|
15
|
-
from .exceptions import NeatSessionError,
|
|
15
|
+
from .exceptions import NeatSessionError, session_class_wrapper
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
@
|
|
18
|
+
@session_class_wrapper
|
|
19
19
|
class ToAPI:
|
|
20
|
-
def __init__(self, state: SessionState, client:
|
|
20
|
+
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
21
21
|
self._state = state
|
|
22
22
|
self._verbose = verbose
|
|
23
23
|
self.cdf = CDFToAPI(state, client, verbose)
|
|
@@ -46,23 +46,36 @@ class ToAPI:
|
|
|
46
46
|
return None
|
|
47
47
|
|
|
48
48
|
@overload
|
|
49
|
-
def yaml(self, io: None) -> str: ...
|
|
49
|
+
def yaml(self, io: None, format: Literal["neat"] = "neat") -> str: ...
|
|
50
50
|
|
|
51
51
|
@overload
|
|
52
|
-
def yaml(self, io: Any) -> None: ...
|
|
53
|
-
|
|
54
|
-
def yaml(self, io: Any | None = None) -> str | None:
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
52
|
+
def yaml(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> None: ...
|
|
53
|
+
|
|
54
|
+
def yaml(self, io: Any | None = None, format: Literal["neat", "toolkit"] = "neat") -> str | None:
|
|
55
|
+
if format == "neat":
|
|
56
|
+
exporter = exporters.YAMLExporter()
|
|
57
|
+
last_verified = self._state.data_model.last_verified_rule[1]
|
|
58
|
+
if io is None:
|
|
59
|
+
return exporter.export(last_verified)
|
|
60
|
+
|
|
61
|
+
exporter.export_to_file(last_verified, Path(io))
|
|
62
|
+
elif format == "toolkit":
|
|
63
|
+
if io is None or not isinstance(io, str | Path):
|
|
64
|
+
raise NeatSessionError(
|
|
65
|
+
"Please provide a zip file or directory path to write the YAML files to."
|
|
66
|
+
"This is required for the 'toolkit' format."
|
|
67
|
+
)
|
|
68
|
+
dms_rule = self._state.data_model.last_verified_dms_rules[1]
|
|
69
|
+
exporters.DMSExporter().export_to_file(dms_rule, Path(io))
|
|
70
|
+
else:
|
|
71
|
+
raise NeatSessionError("Please provide a valid format. {['neat', 'toolkit']}")
|
|
58
72
|
|
|
59
|
-
exporter.export_to_file(self._state.data_model.last_verified_rule[1], Path(io))
|
|
60
73
|
return None
|
|
61
74
|
|
|
62
75
|
|
|
63
|
-
@
|
|
76
|
+
@session_class_wrapper
|
|
64
77
|
class CDFToAPI:
|
|
65
|
-
def __init__(self, state: SessionState, client:
|
|
78
|
+
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
66
79
|
self._client = client
|
|
67
80
|
self._state = state
|
|
68
81
|
self._verbose = verbose
|
|
@@ -2,6 +2,8 @@ import functools
|
|
|
2
2
|
from collections.abc import Callable
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
|
+
from ._collector import _COLLECTOR
|
|
6
|
+
|
|
5
7
|
try:
|
|
6
8
|
from rich import print
|
|
7
9
|
|
|
@@ -16,9 +18,10 @@ class NeatSessionError(Exception):
|
|
|
16
18
|
...
|
|
17
19
|
|
|
18
20
|
|
|
19
|
-
def
|
|
21
|
+
def _session_method_wrapper(func: Callable, cls_name: str):
|
|
20
22
|
@functools.wraps(func)
|
|
21
23
|
def wrapper(*args: Any, **kwargs: Any):
|
|
24
|
+
_COLLECTOR.track_session_command(f"{cls_name}.{func.__name__}", *args, **kwargs)
|
|
22
25
|
try:
|
|
23
26
|
return func(*args, **kwargs)
|
|
24
27
|
except NeatSessionError as e:
|
|
@@ -40,7 +43,21 @@ def _intercept_session_exceptions(func: Callable):
|
|
|
40
43
|
return wrapper
|
|
41
44
|
|
|
42
45
|
|
|
43
|
-
def
|
|
46
|
+
def session_class_wrapper(cls: type):
|
|
47
|
+
"""This decorator wraps all methods of a class.
|
|
48
|
+
|
|
49
|
+
It should be used with all composition classes used with the NeatSession class.
|
|
50
|
+
|
|
51
|
+
It does the following:
|
|
52
|
+
* Intercepts all NeatSession exceptions and prints them in a user-friendly way.
|
|
53
|
+
* Collects user metrics.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
cls: NeatSession composition class
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
cls: NeatSession composition class with all methods wrapped
|
|
60
|
+
"""
|
|
44
61
|
to_check = [cls]
|
|
45
62
|
while to_check:
|
|
46
63
|
cls = to_check.pop()
|
|
@@ -49,7 +66,7 @@ def intercept_session_exceptions(cls: type):
|
|
|
49
66
|
continue
|
|
50
67
|
attr = getattr(cls, attr_name)
|
|
51
68
|
if callable(attr):
|
|
52
|
-
setattr(cls, attr_name,
|
|
69
|
+
setattr(cls, attr_name, _session_method_wrapper(attr, cls.__name__))
|
|
53
70
|
elif isinstance(attr, type):
|
|
54
71
|
to_check.append(attr)
|
|
55
72
|
return cls
|
cognite/neat/_store/_base.py
CHANGED
|
@@ -7,10 +7,11 @@ from typing import cast
|
|
|
7
7
|
|
|
8
8
|
import pandas as pd
|
|
9
9
|
from pandas import Index
|
|
10
|
-
from rdflib import
|
|
10
|
+
from rdflib import Dataset, Namespace, URIRef
|
|
11
11
|
from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
|
|
12
12
|
|
|
13
13
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
14
|
+
from cognite.neat._graph._shared import rdflib_to_oxi_type
|
|
14
15
|
from cognite.neat._graph.extractors import RdfFileExtractor, TripleExtractors
|
|
15
16
|
from cognite.neat._graph.queries import Queries
|
|
16
17
|
from cognite.neat._graph.transformers import Transformers
|
|
@@ -42,7 +43,7 @@ class NeatGraphStore:
|
|
|
42
43
|
|
|
43
44
|
def __init__(
|
|
44
45
|
self,
|
|
45
|
-
graph:
|
|
46
|
+
graph: Dataset,
|
|
46
47
|
rules: InformationRules | None = None,
|
|
47
48
|
):
|
|
48
49
|
self.rules: InformationRules | None = None
|
|
@@ -109,7 +110,7 @@ class NeatGraphStore:
|
|
|
109
110
|
|
|
110
111
|
@classmethod
|
|
111
112
|
def from_memory_store(cls, rules: InformationRules | None = None) -> "Self":
|
|
112
|
-
return cls(
|
|
113
|
+
return cls(Dataset(), rules)
|
|
113
114
|
|
|
114
115
|
@classmethod
|
|
115
116
|
def from_sparql_store(
|
|
@@ -127,7 +128,7 @@ class NeatGraphStore:
|
|
|
127
128
|
postAsEncoded=False,
|
|
128
129
|
autocommit=False,
|
|
129
130
|
)
|
|
130
|
-
graph =
|
|
131
|
+
graph = Dataset(store=store)
|
|
131
132
|
return cls(graph, rules)
|
|
132
133
|
|
|
133
134
|
@classmethod
|
|
@@ -150,9 +151,8 @@ class NeatGraphStore:
|
|
|
150
151
|
else:
|
|
151
152
|
raise Exception("Error initializing Oxigraph store")
|
|
152
153
|
|
|
153
|
-
graph =
|
|
154
|
+
graph = Dataset(
|
|
154
155
|
store=oxrdflib.OxigraphStore(store=oxi_store),
|
|
155
|
-
identifier=DEFAULT_NAMESPACE,
|
|
156
156
|
)
|
|
157
157
|
|
|
158
158
|
return cls(graph, rules)
|
|
@@ -162,7 +162,7 @@ class NeatGraphStore:
|
|
|
162
162
|
success = True
|
|
163
163
|
|
|
164
164
|
if isinstance(extractor, RdfFileExtractor) and not extractor.issue_list.has_errors:
|
|
165
|
-
self._parse_file(extractor.filepath, cast(str, extractor.
|
|
165
|
+
self._parse_file(extractor.filepath, cast(str, extractor.format), extractor.base_uri)
|
|
166
166
|
elif isinstance(extractor, RdfFileExtractor):
|
|
167
167
|
success = False
|
|
168
168
|
issue_text = "\n".join([issue.as_message() for issue in extractor.issue_list])
|
|
@@ -244,33 +244,36 @@ class NeatGraphStore:
|
|
|
244
244
|
def _parse_file(
|
|
245
245
|
self,
|
|
246
246
|
filepath: Path,
|
|
247
|
-
|
|
247
|
+
format: str = "turtle",
|
|
248
248
|
base_uri: URIRef | None = None,
|
|
249
249
|
) -> None:
|
|
250
250
|
"""Imports graph data from file.
|
|
251
251
|
|
|
252
252
|
Args:
|
|
253
253
|
filepath : File path to file containing graph data, by default None
|
|
254
|
-
|
|
255
|
-
base_uri :
|
|
254
|
+
format : rdflib format file containing RDF graph, by default "turtle"
|
|
255
|
+
base_uri : base URI to add to graph in case of relative URIs, by default None
|
|
256
|
+
|
|
257
|
+
!!! note "Oxigraph store"
|
|
258
|
+
By default we are using non-transactional mode for parsing RDF files.
|
|
259
|
+
This gives us a significant performance boost when importing large RDF files.
|
|
260
|
+
Underhood of rdflib we are triggering oxrdflib plugin which in respect
|
|
261
|
+
calls `bulk_load` method from oxigraph store. See more at:
|
|
262
|
+
https://pyoxigraph.readthedocs.io/en/stable/store.html#pyoxigraph.Store.bulk_load
|
|
256
263
|
"""
|
|
257
264
|
|
|
258
265
|
# Oxigraph store, do not want to type hint this as it is an optional dependency
|
|
259
266
|
if type(self.graph.store).__name__ == "OxigraphStore":
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
)
|
|
271
|
-
cast(pyoxigraph.Store, self.graph.store._store).optimize()
|
|
272
|
-
|
|
273
|
-
parse_to_oxi_store()
|
|
267
|
+
local_import("pyoxigraph", "oxi")
|
|
268
|
+
|
|
269
|
+
# this is necessary to trigger rdflib oxigraph plugin
|
|
270
|
+
self.graph.parse(
|
|
271
|
+
filepath,
|
|
272
|
+
format=rdflib_to_oxi_type(format),
|
|
273
|
+
transactional=False,
|
|
274
|
+
publicID=base_uri,
|
|
275
|
+
)
|
|
276
|
+
self.graph.store._store.optimize() # type: ignore[attr-defined]
|
|
274
277
|
|
|
275
278
|
# All other stores
|
|
276
279
|
else:
|
|
@@ -93,14 +93,14 @@ class Entity:
|
|
|
93
93
|
return cls(
|
|
94
94
|
was_attributed_to=agent,
|
|
95
95
|
was_generated_by=activity,
|
|
96
|
-
id_=rules.
|
|
96
|
+
id_=rules.metadata.identifier,
|
|
97
97
|
)
|
|
98
98
|
|
|
99
99
|
elif isinstance(rules, ReadRules | JustRules) and rules.rules is not None:
|
|
100
100
|
return cls(
|
|
101
101
|
was_attributed_to=agent,
|
|
102
102
|
was_generated_by=activity,
|
|
103
|
-
id_=rules.rules.
|
|
103
|
+
id_=rules.rules.metadata.identifier,
|
|
104
104
|
)
|
|
105
105
|
else:
|
|
106
106
|
return cls(
|
cognite/neat/_utils/auxiliary.py
CHANGED
|
@@ -140,3 +140,22 @@ def string_to_ideal_type(input_string: str) -> int | bool | float | datetime | s
|
|
|
140
140
|
except ValueError:
|
|
141
141
|
# Return the input string if no conversion is possible
|
|
142
142
|
return input_string
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def get_parameters_by_method(obj: object, prefix: str = "") -> dict[str, dict[str, type]]:
|
|
146
|
+
annotation_by_method = {}
|
|
147
|
+
namespace_dict = {**dict(obj.__class__.__dict__.items()), **vars(obj)}
|
|
148
|
+
for name, value in namespace_dict.items():
|
|
149
|
+
if name != "__call__" and (name.startswith("_") or isinstance(value, property)):
|
|
150
|
+
continue
|
|
151
|
+
if callable(value) and type(value).__name__ == "function":
|
|
152
|
+
annotation_by_method[f"{prefix}{name}"] = get_parameters(value)
|
|
153
|
+
elif isinstance(value, object) and type(value).__module__ != "builtins":
|
|
154
|
+
sub_prefix = f"{prefix}{name}." if prefix else f"{name}."
|
|
155
|
+
annotation_by_method.update(get_parameters_by_method(value, sub_prefix))
|
|
156
|
+
return annotation_by_method
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def get_parameters(obj: Callable) -> dict[str, type]:
|
|
160
|
+
annotations = inspect.get_annotations(obj)
|
|
161
|
+
return {name: annotations[name] for name in annotations if name != "return"}
|
cognite/neat/_utils/rdf_.py
CHANGED
|
@@ -173,7 +173,7 @@ def get_inheritance_path(child: Any, child_parent: dict[Any, list[Any]]) -> list
|
|
|
173
173
|
return path
|
|
174
174
|
|
|
175
175
|
|
|
176
|
-
def add_triples_in_batch(graph: Graph, triples: Iterable[Triple], batch_size: int = 10_000):
|
|
176
|
+
def add_triples_in_batch(graph: Graph, triples: Iterable[Triple], batch_size: int = 10_000) -> None:
|
|
177
177
|
"""Adds triples to the graph store in batches.
|
|
178
178
|
|
|
179
179
|
Args:
|
|
@@ -204,3 +204,30 @@ def add_triples_in_batch(graph: Graph, triples: Iterable[Triple], batch_size: in
|
|
|
204
204
|
check_commit()
|
|
205
205
|
|
|
206
206
|
check_commit(force_commit=True)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def remove_instance_ids_in_batch(graph: Graph, instance_ids: Iterable[URIRef], batch_size: int = 1_000) -> None:
|
|
210
|
+
"""Removes all triples related to the given instances in the graph store in batches.
|
|
211
|
+
|
|
212
|
+
Args:
|
|
213
|
+
graph: The graph store to remove triples from
|
|
214
|
+
instance_ids: list of instances to remove triples from
|
|
215
|
+
batch_size: Batch size of triples per commit, by default 10_000
|
|
216
|
+
|
|
217
|
+
"""
|
|
218
|
+
batch_count = 0
|
|
219
|
+
|
|
220
|
+
def check_commit(force_commit: bool = False):
|
|
221
|
+
"""Commit nodes to the graph if batch counter is reached or if force_commit is True"""
|
|
222
|
+
nonlocal batch_count
|
|
223
|
+
batch_count += 1
|
|
224
|
+
if force_commit or batch_count >= batch_size:
|
|
225
|
+
graph.commit()
|
|
226
|
+
batch_count = 0
|
|
227
|
+
return
|
|
228
|
+
|
|
229
|
+
for instance_id in instance_ids:
|
|
230
|
+
graph.remove((instance_id, None, None))
|
|
231
|
+
check_commit()
|
|
232
|
+
|
|
233
|
+
check_commit(force_commit=True)
|
cognite/neat/_version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.99.0"
|
|
2
2
|
__engine__ = "^1.0.3"
|
|
@@ -10,9 +10,7 @@ from cognite.client.data_classes import (
|
|
|
10
10
|
from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
|
|
11
11
|
|
|
12
12
|
from cognite.neat._rules.models import (
|
|
13
|
-
AssetRules,
|
|
14
13
|
DMSRules,
|
|
15
|
-
DomainRules,
|
|
16
14
|
InformationRules,
|
|
17
15
|
)
|
|
18
16
|
from cognite.neat._store import NeatGraphStore
|
|
@@ -20,19 +18,13 @@ from cognite.neat._workflows.steps.step_model import DataContract
|
|
|
20
18
|
|
|
21
19
|
|
|
22
20
|
class MultiRuleData(DataContract):
|
|
23
|
-
domain: DomainRules | None = None
|
|
24
21
|
information: InformationRules | None = None
|
|
25
|
-
asset: AssetRules | None = None
|
|
26
22
|
dms: DMSRules | None = None
|
|
27
23
|
|
|
28
24
|
@classmethod
|
|
29
|
-
def from_rules(cls, rules:
|
|
30
|
-
if isinstance(rules,
|
|
31
|
-
return cls(domain=rules)
|
|
32
|
-
elif isinstance(rules, InformationRules):
|
|
25
|
+
def from_rules(cls, rules: InformationRules | DMSRules):
|
|
26
|
+
if isinstance(rules, InformationRules):
|
|
33
27
|
return cls(information=rules)
|
|
34
|
-
elif isinstance(rules, AssetRules):
|
|
35
|
-
return cls(asset=rules)
|
|
36
28
|
elif isinstance(rules, DMSRules):
|
|
37
29
|
return cls(dms=rules)
|
|
38
30
|
else:
|
|
@@ -2,16 +2,14 @@ import time
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import ClassVar, Literal, cast
|
|
4
4
|
|
|
5
|
+
from cognite.neat._client import NeatClient
|
|
5
6
|
from cognite.neat._issues.errors import WorkflowStepNotInitializedError
|
|
6
7
|
from cognite.neat._rules import exporters
|
|
7
8
|
from cognite.neat._rules._shared import DMSRules, InformationRules, VerifiedRules
|
|
8
|
-
from cognite.neat._rules.models import
|
|
9
|
+
from cognite.neat._rules.models import RoleTypes
|
|
9
10
|
from cognite.neat._rules.transformers import (
|
|
10
|
-
AssetToInformation,
|
|
11
11
|
DMSToInformation,
|
|
12
|
-
InformationToAsset,
|
|
13
12
|
InformationToDMS,
|
|
14
|
-
RulesPipeline,
|
|
15
13
|
)
|
|
16
14
|
from cognite.neat._workflows.model import FlowMessage, StepExecutionStatus
|
|
17
15
|
from cognite.neat._workflows.steps.data_contracts import CogniteClient, MultiRuleData
|
|
@@ -82,7 +80,7 @@ class DeleteDataModelFromCDF(Step):
|
|
|
82
80
|
error_text="No DMS Schema components selected for removal! Please select minimum one!",
|
|
83
81
|
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
84
82
|
)
|
|
85
|
-
input_rules = rules.dms or rules.information
|
|
83
|
+
input_rules = rules.dms or rules.information
|
|
86
84
|
if input_rules is None:
|
|
87
85
|
return FlowMessage(
|
|
88
86
|
error_text="Missing DMS or Information rules in the input data! "
|
|
@@ -93,8 +91,6 @@ class DeleteDataModelFromCDF(Step):
|
|
|
93
91
|
dms_rules = input_rules
|
|
94
92
|
elif isinstance(input_rules, InformationRules):
|
|
95
93
|
dms_rules = InformationToDMS().transform(input_rules).rules
|
|
96
|
-
elif isinstance(input_rules, AssetRules):
|
|
97
|
-
dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
|
|
98
94
|
else:
|
|
99
95
|
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
100
96
|
|
|
@@ -105,7 +101,7 @@ class DeleteDataModelFromCDF(Step):
|
|
|
105
101
|
|
|
106
102
|
report_lines = ["# Data Model Deletion from CDF\n\n"]
|
|
107
103
|
errors = []
|
|
108
|
-
for result in dms_exporter.delete_from_cdf(rules=dms_rules, client=cdf_client, dry_run=dry_run):
|
|
104
|
+
for result in dms_exporter.delete_from_cdf(rules=dms_rules, client=NeatClient(cdf_client), dry_run=dry_run):
|
|
109
105
|
report_lines.append(str(result))
|
|
110
106
|
errors.extend(result.error_messages)
|
|
111
107
|
|
|
@@ -207,8 +203,6 @@ class RulesToDMS(Step):
|
|
|
207
203
|
dms_rules = input_rules
|
|
208
204
|
elif isinstance(input_rules, InformationRules):
|
|
209
205
|
dms_rules = InformationToDMS().transform(input_rules).rules
|
|
210
|
-
elif isinstance(input_rules, AssetRules):
|
|
211
|
-
dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
|
|
212
206
|
else:
|
|
213
207
|
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
214
208
|
|
|
@@ -220,18 +214,16 @@ class RulesToDMS(Step):
|
|
|
220
214
|
|
|
221
215
|
output_dir = self.config.staging_path
|
|
222
216
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
223
|
-
file_name =
|
|
224
|
-
input_rules.metadata.external_id
|
|
225
|
-
if isinstance(input_rules, DMSRules)
|
|
226
|
-
else input_rules.metadata.name.replace(" ", "_").lower()
|
|
227
|
-
)
|
|
217
|
+
file_name = input_rules.metadata.external_id
|
|
228
218
|
schema_zip = f"{file_name}.zip"
|
|
229
219
|
schema_full_path = output_dir / schema_zip
|
|
230
220
|
dms_exporter.export_to_file(dms_rules, schema_full_path)
|
|
231
221
|
|
|
232
222
|
report_lines = ["# DMS Schema Export to CDF\n\n"]
|
|
233
223
|
errors = []
|
|
234
|
-
for result in dms_exporter.export_to_cdf_iterable(
|
|
224
|
+
for result in dms_exporter.export_to_cdf_iterable(
|
|
225
|
+
rules=dms_rules, client=NeatClient(cdf_client), dry_run=dry_run
|
|
226
|
+
):
|
|
235
227
|
report_lines.append(str(result))
|
|
236
228
|
errors.extend(result.error_messages)
|
|
237
229
|
|
|
@@ -283,14 +275,6 @@ class RulesToExcel(Step):
|
|
|
283
275
|
"rules will be used.",
|
|
284
276
|
options=["input", *RoleTypes.__members__.keys()],
|
|
285
277
|
),
|
|
286
|
-
Configurable(
|
|
287
|
-
name="Dump Format",
|
|
288
|
-
value="user",
|
|
289
|
-
label="How to dump the rules to the Excel file.\n"
|
|
290
|
-
"'user' - just as is.\n'reference' - enterprise model used as basis for a solution model\n"
|
|
291
|
-
"'last' - used when updating a data model.",
|
|
292
|
-
options=list(exporters.ExcelExporter.dump_options),
|
|
293
|
-
),
|
|
294
278
|
Configurable(
|
|
295
279
|
name="New Data Model ID",
|
|
296
280
|
value="",
|
|
@@ -308,7 +292,6 @@ class RulesToExcel(Step):
|
|
|
308
292
|
if self.configs is None or self.data_store_path is None:
|
|
309
293
|
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
310
294
|
|
|
311
|
-
dump_format = self.configs.get("Dump Format", "user")
|
|
312
295
|
styling = cast(exporters.ExcelExporter.Style, self.configs.get("Styling", "default"))
|
|
313
296
|
role = self.configs.get("Output role format")
|
|
314
297
|
output_role: RoleTypes | None = None
|
|
@@ -325,18 +308,15 @@ class RulesToExcel(Step):
|
|
|
325
308
|
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
326
309
|
)
|
|
327
310
|
|
|
328
|
-
excel_exporter = exporters.ExcelExporter(styling=styling,
|
|
311
|
+
excel_exporter = exporters.ExcelExporter(styling=styling, new_model_id=new_model_id) # type: ignore[arg-type]
|
|
329
312
|
|
|
330
313
|
# Todo - Move the conversion to a separate workflow step.
|
|
331
314
|
rule_instance: VerifiedRules
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
elif rules.information:
|
|
315
|
+
|
|
316
|
+
if rules.information:
|
|
335
317
|
rule_instance = rules.information
|
|
336
318
|
elif rules.dms:
|
|
337
319
|
rule_instance = rules.dms
|
|
338
|
-
elif rules.asset:
|
|
339
|
-
rule_instance = rules.asset
|
|
340
320
|
else:
|
|
341
321
|
output_errors = "No rules provided for export!"
|
|
342
322
|
return FlowMessage(error_text=output_errors, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
|
|
@@ -346,26 +326,11 @@ class RulesToExcel(Step):
|
|
|
346
326
|
elif output_role is RoleTypes.dms:
|
|
347
327
|
if isinstance(rule_instance, InformationRules):
|
|
348
328
|
rule_instance = InformationToDMS().transform(rule_instance).rules
|
|
349
|
-
elif isinstance(rule_instance, AssetRules):
|
|
350
|
-
rule_instance = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(
|
|
351
|
-
rule_instance
|
|
352
|
-
)
|
|
353
329
|
else:
|
|
354
330
|
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
355
331
|
elif output_role is RoleTypes.information:
|
|
356
332
|
if isinstance(rule_instance, DMSRules):
|
|
357
333
|
rule_instance = DMSToInformation().transform(rule_instance).rules
|
|
358
|
-
elif isinstance(rule_instance, AssetRules):
|
|
359
|
-
rule_instance = AssetToInformation().transform(rule_instance).rules
|
|
360
|
-
else:
|
|
361
|
-
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
362
|
-
elif output_role is RoleTypes.asset:
|
|
363
|
-
if isinstance(rule_instance, InformationRules):
|
|
364
|
-
rule_instance = InformationToAsset().transform(rule_instance).rules
|
|
365
|
-
elif isinstance(rule_instance, DMSRules):
|
|
366
|
-
rule_instance = RulesPipeline[DMSRules, AssetRules]([DMSToInformation(), InformationToAsset()]).run(
|
|
367
|
-
rule_instance
|
|
368
|
-
)
|
|
369
334
|
else:
|
|
370
335
|
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
371
336
|
else:
|
|
@@ -604,8 +569,6 @@ class RulesToCDFTransformations(Step):
|
|
|
604
569
|
dms_rules = input_rules
|
|
605
570
|
elif isinstance(input_rules, InformationRules):
|
|
606
571
|
dms_rules = InformationToDMS().transform(input_rules).rules
|
|
607
|
-
elif isinstance(input_rules, AssetRules):
|
|
608
|
-
dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
|
|
609
572
|
else:
|
|
610
573
|
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
611
574
|
|
|
@@ -624,7 +587,9 @@ class RulesToCDFTransformations(Step):
|
|
|
624
587
|
|
|
625
588
|
report_lines = ["# DMS Schema Export to CDF\n\n"]
|
|
626
589
|
errors = []
|
|
627
|
-
for result in dms_exporter.export_to_cdf_iterable(
|
|
590
|
+
for result in dms_exporter.export_to_cdf_iterable(
|
|
591
|
+
rules=dms_rules, client=NeatClient(cdf_client), dry_run=dry_run
|
|
592
|
+
):
|
|
628
593
|
report_lines.append(str(result))
|
|
629
594
|
errors.extend(result.error_messages)
|
|
630
595
|
|
|
@@ -5,6 +5,7 @@ from typing import ClassVar
|
|
|
5
5
|
from cognite.client import CogniteClient
|
|
6
6
|
from cognite.client.data_classes.data_modeling import DataModelId
|
|
7
7
|
|
|
8
|
+
from cognite.neat._client import NeatClient
|
|
8
9
|
from cognite.neat._issues.errors import WorkflowStepNotInitializedError
|
|
9
10
|
from cognite.neat._issues.formatters import FORMATTER_BY_NAME
|
|
10
11
|
from cognite.neat._rules import importers
|
|
@@ -299,7 +300,9 @@ class DMSToRules(Step):
|
|
|
299
300
|
return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
|
|
300
301
|
ref_model_id = ref_model.as_id()
|
|
301
302
|
|
|
302
|
-
dms_importer = importers.DMSImporter.from_data_model_id(
|
|
303
|
+
dms_importer = importers.DMSImporter.from_data_model_id(
|
|
304
|
+
NeatClient(cdf_client), datamodel_entity.as_id(), ref_model_id
|
|
305
|
+
)
|
|
303
306
|
|
|
304
307
|
# if role is None, it will be inferred from the rules file
|
|
305
308
|
role = self.configs.get("Role")
|
|
@@ -5,11 +5,12 @@ from typing import ClassVar
|
|
|
5
5
|
|
|
6
6
|
from cognite.client import CogniteClient
|
|
7
7
|
|
|
8
|
+
from cognite.neat._client import NeatClient
|
|
9
|
+
from cognite.neat._client._api.data_modeling_loaders import ViewLoader
|
|
8
10
|
from cognite.neat._issues import NeatIssueList
|
|
9
11
|
from cognite.neat._issues.errors import ResourceNotFoundError, WorkflowStepNotInitializedError
|
|
10
12
|
from cognite.neat._issues.formatters import FORMATTER_BY_NAME
|
|
11
|
-
from cognite.neat._rules.models import DMSRules
|
|
12
|
-
from cognite.neat._utils.cdf.loaders import ViewLoader
|
|
13
|
+
from cognite.neat._rules.models import DMSRules
|
|
13
14
|
from cognite.neat._workflows.model import FlowMessage, StepExecutionStatus
|
|
14
15
|
from cognite.neat._workflows.steps.data_contracts import MultiRuleData
|
|
15
16
|
from cognite.neat._workflows.steps.step_model import Configurable, Step
|
|
@@ -49,12 +50,7 @@ class ValidateRulesAgainstCDF(Step):
|
|
|
49
50
|
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
50
51
|
)
|
|
51
52
|
dms_rules = rules.dms
|
|
52
|
-
|
|
53
|
-
return FlowMessage(
|
|
54
|
-
error_text="DMS rules are not partial. This step expects DMS rules to be a partial definition "
|
|
55
|
-
"with the rest of the definition being fetched from CDF.",
|
|
56
|
-
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
57
|
-
)
|
|
53
|
+
|
|
58
54
|
schema = dms_rules.as_schema()
|
|
59
55
|
errors = schema.validate()
|
|
60
56
|
if not errors:
|
|
@@ -80,7 +76,7 @@ class ValidateRulesAgainstCDF(Step):
|
|
|
80
76
|
retrieved_containers = cdf_client.data_modeling.containers.retrieve(missing_containers).as_write()
|
|
81
77
|
# Converting read format of views to write format requires to account for parents (implements)
|
|
82
78
|
# Thus we use the loader to convert the views to write format.
|
|
83
|
-
view_loader = ViewLoader(cdf_client)
|
|
79
|
+
view_loader = ViewLoader(NeatClient(cdf_client))
|
|
84
80
|
retrieved_views = [
|
|
85
81
|
view_loader.as_write(view) for view in cdf_client.data_modeling.views.retrieve(missing_views)
|
|
86
82
|
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: cognite-neat
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.99.0
|
|
4
4
|
Summary: Knowledge graph transformation
|
|
5
5
|
Home-page: https://cognite-neat.readthedocs-hosted.com/
|
|
6
6
|
License: Apache-2.0
|
|
@@ -29,6 +29,7 @@ Requires-Dist: google-auth-oauthlib ; extra == "google"
|
|
|
29
29
|
Requires-Dist: gspread ; extra == "google"
|
|
30
30
|
Requires-Dist: jsonpath-python (>=1.0.6,<2.0.0)
|
|
31
31
|
Requires-Dist: lxml (>=5.3.0,<6.0.0) ; extra == "all"
|
|
32
|
+
Requires-Dist: mixpanel (>=4.10.1,<5.0.0)
|
|
32
33
|
Requires-Dist: mkdocs ; extra == "docs"
|
|
33
34
|
Requires-Dist: mkdocs-autorefs (>=0.5.0,<0.6.0) ; extra == "docs"
|
|
34
35
|
Requires-Dist: mkdocs-git-authors-plugin ; extra == "docs"
|
|
@@ -40,13 +41,13 @@ Requires-Dist: mkdocs-material-extensions ; extra == "docs"
|
|
|
40
41
|
Requires-Dist: mkdocstrings[python] ; extra == "docs"
|
|
41
42
|
Requires-Dist: networkx (>=3.4.2,<4.0.0)
|
|
42
43
|
Requires-Dist: openpyxl
|
|
43
|
-
Requires-Dist: oxrdflib[oxigraph] (>=0.
|
|
44
|
+
Requires-Dist: oxrdflib[oxigraph] (>=0.4.0,<0.5.0) ; extra == "oxi" or extra == "all"
|
|
44
45
|
Requires-Dist: packaging (>=22.0,<25.0)
|
|
45
46
|
Requires-Dist: pandas
|
|
46
47
|
Requires-Dist: prometheus-client (>=0,<1) ; extra == "service" or extra == "all"
|
|
47
48
|
Requires-Dist: pydantic (>=2,<3)
|
|
48
49
|
Requires-Dist: pymdown-extensions ; extra == "docs"
|
|
49
|
-
Requires-Dist: pyoxigraph (==0.3
|
|
50
|
+
Requires-Dist: pyoxigraph (==0.4.3) ; extra == "oxi" or extra == "all"
|
|
50
51
|
Requires-Dist: python-multipart (==0.0.9) ; extra == "service" or extra == "all"
|
|
51
52
|
Requires-Dist: pyvis (>=0.3.2,<0.4.0)
|
|
52
53
|
Requires-Dist: rdflib
|