cognite-neat 0.109.4__py3-none-any.whl → 0.111.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_alpha.py +8 -0
- cognite/neat/_client/_api/schema.py +43 -1
- cognite/neat/_client/data_classes/schema.py +4 -4
- cognite/neat/_constants.py +15 -1
- cognite/neat/_graph/extractors/__init__.py +4 -0
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +8 -16
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +48 -19
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +23 -17
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +15 -17
- cognite/neat/_graph/extractors/_dict.py +102 -0
- cognite/neat/_graph/extractors/_dms.py +27 -40
- cognite/neat/_graph/extractors/_dms_graph.py +30 -3
- cognite/neat/_graph/extractors/_iodd.py +3 -3
- cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
- cognite/neat/_graph/extractors/_raw.py +67 -0
- cognite/neat/_graph/loaders/_base.py +20 -4
- cognite/neat/_graph/loaders/_rdf2dms.py +476 -383
- cognite/neat/_graph/queries/_base.py +163 -133
- cognite/neat/_graph/transformers/__init__.py +1 -3
- cognite/neat/_graph/transformers/_classic_cdf.py +6 -22
- cognite/neat/_graph/transformers/_rdfpath.py +2 -49
- cognite/neat/_issues/__init__.py +1 -6
- cognite/neat/_issues/_base.py +21 -252
- cognite/neat/_issues/_contextmanagers.py +46 -0
- cognite/neat/_issues/_factory.py +69 -0
- cognite/neat/_issues/errors/__init__.py +20 -4
- cognite/neat/_issues/errors/_external.py +7 -0
- cognite/neat/_issues/errors/_wrapper.py +81 -3
- cognite/neat/_issues/formatters.py +4 -4
- cognite/neat/_issues/warnings/__init__.py +3 -2
- cognite/neat/_issues/warnings/_properties.py +8 -0
- cognite/neat/_issues/warnings/user_modeling.py +12 -0
- cognite/neat/_rules/_constants.py +12 -0
- cognite/neat/_rules/_shared.py +3 -2
- cognite/neat/_rules/analysis/__init__.py +2 -3
- cognite/neat/_rules/analysis/_base.py +430 -259
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +3 -9
- cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
- cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
- cognite/neat/_rules/importers/_base.py +2 -47
- cognite/neat/_rules/importers/_dms2rules.py +7 -10
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +66 -26
- cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
- cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
- cognite/neat/_rules/models/_base_rules.py +0 -2
- cognite/neat/_rules/models/data_types.py +7 -0
- cognite/neat/_rules/models/dms/_exporter.py +9 -8
- cognite/neat/_rules/models/dms/_rules.py +29 -2
- cognite/neat/_rules/models/dms/_rules_input.py +9 -1
- cognite/neat/_rules/models/dms/_validation.py +115 -5
- cognite/neat/_rules/models/entities/_loaders.py +1 -1
- cognite/neat/_rules/models/entities/_multi_value.py +2 -2
- cognite/neat/_rules/models/entities/_single_value.py +8 -3
- cognite/neat/_rules/models/entities/_wrapped.py +2 -2
- cognite/neat/_rules/models/information/_rules.py +18 -17
- cognite/neat/_rules/models/information/_rules_input.py +3 -1
- cognite/neat/_rules/models/information/_validation.py +66 -17
- cognite/neat/_rules/transformers/__init__.py +8 -2
- cognite/neat/_rules/transformers/_converters.py +234 -44
- cognite/neat/_rules/transformers/_verification.py +5 -10
- cognite/neat/_session/_base.py +6 -4
- cognite/neat/_session/_explore.py +39 -0
- cognite/neat/_session/_inspect.py +25 -6
- cognite/neat/_session/_prepare.py +12 -0
- cognite/neat/_session/_read.py +88 -20
- cognite/neat/_session/_set.py +7 -1
- cognite/neat/_session/_show.py +11 -123
- cognite/neat/_session/_state.py +6 -2
- cognite/neat/_session/_subset.py +64 -0
- cognite/neat/_session/_to.py +177 -19
- cognite/neat/_store/_graph_store.py +9 -246
- cognite/neat/_utils/rdf_.py +36 -5
- cognite/neat/_utils/spreadsheet.py +44 -1
- cognite/neat/_utils/text.py +124 -37
- cognite/neat/_utils/upload.py +2 -0
- cognite/neat/_version.py +2 -2
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/RECORD +83 -82
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/WHEEL +1 -1
- cognite/neat/_graph/queries/_construct.py +0 -187
- cognite/neat/_graph/queries/_shared.py +0 -173
- cognite/neat/_rules/analysis/_dms.py +0 -57
- cognite/neat/_rules/analysis/_information.py +0 -249
- cognite/neat/_rules/models/_rdfpath.py +0 -372
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/entry_points.txt +0 -0
cognite/neat/_session/_to.py
CHANGED
|
@@ -5,14 +5,18 @@ from pathlib import Path
|
|
|
5
5
|
from typing import Any, Literal, overload
|
|
6
6
|
|
|
7
7
|
from cognite.client import data_modeling as dm
|
|
8
|
+
from cognite.client.data_classes.data_modeling import DataModelIdentifier
|
|
8
9
|
|
|
9
10
|
from cognite.neat._alpha import AlphaFlags
|
|
10
11
|
from cognite.neat._constants import COGNITE_MODELS
|
|
11
12
|
from cognite.neat._graph import loaders
|
|
13
|
+
from cognite.neat._issues import IssueList, NeatIssue, catch_issues
|
|
12
14
|
from cognite.neat._rules import exporters
|
|
13
15
|
from cognite.neat._rules._constants import PATTERNS
|
|
14
16
|
from cognite.neat._rules._shared import VerifiedRules
|
|
15
17
|
from cognite.neat._rules.exporters._rules2dms import Component
|
|
18
|
+
from cognite.neat._rules.importers import DMSImporter
|
|
19
|
+
from cognite.neat._rules.models import DMSRules, InformationRules
|
|
16
20
|
from cognite.neat._rules.models.dms import DMSMetadata
|
|
17
21
|
from cognite.neat._utils.upload import UploadResultList
|
|
18
22
|
|
|
@@ -31,14 +35,40 @@ class ToAPI:
|
|
|
31
35
|
self._state = state
|
|
32
36
|
self._verbose = verbose
|
|
33
37
|
self.cdf = CDFToAPI(state, verbose)
|
|
38
|
+
self._python = ToPythonAPI(state, verbose)
|
|
39
|
+
|
|
40
|
+
def ontology(self, io: Any) -> None:
|
|
41
|
+
"""Export the data model to ontology.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
io: The file path to file-like object to write the session to.
|
|
45
|
+
|
|
46
|
+
Example:
|
|
47
|
+
Export the session to a file
|
|
48
|
+
```python
|
|
49
|
+
ontology_file_name = "neat_session.ttl"
|
|
50
|
+
neat.to.ontology(ontology_file_name)
|
|
51
|
+
```
|
|
52
|
+
"""
|
|
53
|
+
warnings.filterwarnings("default")
|
|
54
|
+
AlphaFlags.to_ontology.warn()
|
|
55
|
+
|
|
56
|
+
filepath = Path(io)
|
|
57
|
+
if filepath.suffix != ".ttl":
|
|
58
|
+
warnings.warn("File extension is not .ttl, adding it to the file name", stacklevel=2)
|
|
59
|
+
filepath = filepath.with_suffix(".ttl")
|
|
60
|
+
|
|
61
|
+
exporter = exporters.OWLExporter()
|
|
62
|
+
self._state.rule_store.export_to_file(exporter, Path(io))
|
|
63
|
+
return None
|
|
34
64
|
|
|
35
65
|
def excel(
|
|
36
66
|
self,
|
|
37
67
|
io: Any,
|
|
38
|
-
include_reference: bool = True,
|
|
68
|
+
include_reference: bool | DataModelIdentifier = True,
|
|
39
69
|
include_properties: Literal["same-space", "all"] = "all",
|
|
40
70
|
add_empty_rows: bool = False,
|
|
41
|
-
) -> None:
|
|
71
|
+
) -> IssueList | None:
|
|
42
72
|
"""Export the verified data model to Excel.
|
|
43
73
|
|
|
44
74
|
Args:
|
|
@@ -46,6 +76,7 @@ class ToAPI:
|
|
|
46
76
|
include_reference: If True, the reference data model will be included. Defaults to True.
|
|
47
77
|
Note that this only applies if you have created the data model using the
|
|
48
78
|
create.enterprise_model(...), create.solution_model(), or create.data_product_model() methods.
|
|
79
|
+
You can also provide a DataModelIdentifier directly, which will be read from CDF
|
|
49
80
|
include_properties: The properties to include in the Excel file. Defaults to "all".
|
|
50
81
|
- "same-space": Only properties that are in the same space as the data model will be included.
|
|
51
82
|
add_empty_rows: If True, empty rows will be added between each component. Defaults to False.
|
|
@@ -71,19 +102,45 @@ class ToAPI:
|
|
|
71
102
|
dms_rules_file_name = "dms_rules.xlsx"
|
|
72
103
|
neat.to.excel(dms_rules_file_name, include_reference=True)
|
|
73
104
|
```
|
|
105
|
+
|
|
106
|
+
Example:
|
|
107
|
+
Read the data model ("my_space", "ISA95Model", "v5") and export it to an excel file with the
|
|
108
|
+
CogniteCore model in the reference sheets.
|
|
109
|
+
```python
|
|
110
|
+
client = CogniteClient()
|
|
111
|
+
neat = NeatSession(client)
|
|
112
|
+
|
|
113
|
+
neat.read.cdf(("my_space", "ISA95Model", "v5"))
|
|
114
|
+
dms_rules_file_name = "dms_rules.xlsx"
|
|
115
|
+
neat.to.excel(dms_rules_file_name, include_reference=("cdf_cdm", "CogniteCore", "v1"))
|
|
74
116
|
"""
|
|
75
117
|
reference_rules_with_prefix: tuple[VerifiedRules, str] | None = None
|
|
76
118
|
include_properties = include_properties.strip().lower()
|
|
77
119
|
|
|
78
|
-
if include_reference
|
|
79
|
-
if
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
prefix = "CDM"
|
|
120
|
+
if include_reference is not False:
|
|
121
|
+
if include_reference is True and self._state.last_reference is not None:
|
|
122
|
+
ref_rules: InformationRules | DMSRules | None = self._state.last_reference
|
|
123
|
+
elif include_reference is True:
|
|
124
|
+
ref_rules = None
|
|
84
125
|
else:
|
|
126
|
+
if not self._state.client:
|
|
127
|
+
raise NeatSessionError("No client provided!")
|
|
128
|
+
ref_rules = None
|
|
129
|
+
with catch_issues() as issues:
|
|
130
|
+
ref_read = DMSImporter.from_data_model_id(self._state.client, include_reference).to_rules()
|
|
131
|
+
if ref_read.rules is not None:
|
|
132
|
+
ref_rules = ref_read.rules.as_verified_rules()
|
|
133
|
+
if ref_rules is None or issues.has_errors:
|
|
134
|
+
issues.action = f"Read {include_reference}"
|
|
135
|
+
return issues
|
|
136
|
+
if ref_rules is not None:
|
|
85
137
|
prefix = "Ref"
|
|
86
|
-
|
|
138
|
+
if (
|
|
139
|
+
isinstance(ref_rules.metadata, DMSMetadata)
|
|
140
|
+
and ref_rules.metadata.as_data_model_id() in COGNITE_MODELS
|
|
141
|
+
):
|
|
142
|
+
prefix = "CDM"
|
|
143
|
+
reference_rules_with_prefix = ref_rules, prefix
|
|
87
144
|
|
|
88
145
|
if include_properties == "same-space":
|
|
89
146
|
warnings.filterwarnings("default")
|
|
@@ -95,7 +152,8 @@ class ToAPI:
|
|
|
95
152
|
add_empty_rows=add_empty_rows,
|
|
96
153
|
include_properties=include_properties, # type: ignore
|
|
97
154
|
)
|
|
98
|
-
|
|
155
|
+
self._state.rule_store.export_to_file(exporter, Path(io))
|
|
156
|
+
return None
|
|
99
157
|
|
|
100
158
|
def session(self, io: Any) -> None:
|
|
101
159
|
"""Export the current session to a file.
|
|
@@ -177,6 +235,7 @@ class ToAPI:
|
|
|
177
235
|
neat.to.yaml(your_folder_name, format="toolkit")
|
|
178
236
|
```
|
|
179
237
|
"""
|
|
238
|
+
|
|
180
239
|
if format == "neat":
|
|
181
240
|
exporter = exporters.YAMLExporter()
|
|
182
241
|
if io is None:
|
|
@@ -212,35 +271,67 @@ class CDFToAPI:
|
|
|
212
271
|
def instances(
|
|
213
272
|
self,
|
|
214
273
|
space: str | None = None,
|
|
274
|
+
space_property: str | None = None,
|
|
215
275
|
) -> UploadResultList:
|
|
216
276
|
"""Export the verified DMS instances to CDF.
|
|
217
277
|
|
|
218
278
|
Args:
|
|
219
279
|
space: Name of instance space to use. Default is to suffix the schema space with '_instances'.
|
|
220
|
-
|
|
280
|
+
Note this space is required to be different from the space with the data model.
|
|
281
|
+
space_property: This is an alternative to the 'space' argument. If provided, the space will set to the
|
|
282
|
+
value of the property with the given name for each instance. If the property is not found, the
|
|
283
|
+
'space' argument will be used. Defaults to None.
|
|
284
|
+
|
|
285
|
+
Returns:
|
|
286
|
+
UploadResultList: The result of the upload.
|
|
287
|
+
|
|
288
|
+
Example:
|
|
289
|
+
Export instances to CDF
|
|
290
|
+
```python
|
|
291
|
+
neat.to.cdf.instances()
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Export instances to CDF using the `dataSetId` property as the space
|
|
295
|
+
```python
|
|
296
|
+
neat.to.cdf.instances(space_property="dataSetId")
|
|
297
|
+
```
|
|
221
298
|
|
|
222
299
|
"""
|
|
300
|
+
return self._instances(instance_space=space, space_from_property=space_property)
|
|
301
|
+
|
|
302
|
+
def _instances(
|
|
303
|
+
self,
|
|
304
|
+
instance_space: str | None = None,
|
|
305
|
+
space_from_property: str | None = None,
|
|
306
|
+
use_source_space: bool = False,
|
|
307
|
+
) -> UploadResultList:
|
|
223
308
|
if not self._state.client:
|
|
224
309
|
raise NeatSessionError("No CDF client provided!")
|
|
225
310
|
client = self._state.client
|
|
226
|
-
|
|
311
|
+
dms_rules = self._state.rule_store.last_verified_dms_rules
|
|
312
|
+
instance_space = instance_space or f"{dms_rules.metadata.space}_instances"
|
|
227
313
|
|
|
228
|
-
if
|
|
314
|
+
if instance_space and instance_space == dms_rules.metadata.space:
|
|
229
315
|
raise NeatSessionError("Space for instances must be different from the data model space.")
|
|
230
|
-
elif not PATTERNS.space_compliance.match(str(
|
|
316
|
+
elif not PATTERNS.space_compliance.match(str(instance_space)):
|
|
231
317
|
raise NeatSessionError("Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
|
|
232
318
|
|
|
233
|
-
if not client.data_modeling.spaces.retrieve(
|
|
234
|
-
client.data_modeling.spaces.apply(dm.SpaceApply(space=
|
|
319
|
+
if not client.data_modeling.spaces.retrieve(instance_space):
|
|
320
|
+
client.data_modeling.spaces.apply(dm.SpaceApply(space=instance_space))
|
|
235
321
|
|
|
236
|
-
loader = loaders.DMSLoader
|
|
322
|
+
loader = loaders.DMSLoader(
|
|
237
323
|
self._state.rule_store.last_verified_dms_rules,
|
|
324
|
+
self._state.rule_store.last_verified_information_rules,
|
|
238
325
|
self._state.instances.store,
|
|
239
|
-
instance_space=
|
|
326
|
+
instance_space=instance_space,
|
|
240
327
|
client=client,
|
|
328
|
+
space_property=space_from_property,
|
|
329
|
+
use_source_space=use_source_space,
|
|
241
330
|
# In case urllib.parse.quote() was run on the extraction, we need to run
|
|
242
331
|
# urllib.parse.unquote() on the load.
|
|
243
|
-
unquote_external_ids=
|
|
332
|
+
unquote_external_ids=True,
|
|
333
|
+
neat_prefix_by_predicate_uri=self._state.instances.neat_prefix_by_predicate_uri,
|
|
334
|
+
neat_prefix_by_type_uri=self._state.instances.neat_prefix_by_type_uri,
|
|
244
335
|
)
|
|
245
336
|
|
|
246
337
|
result = loader.load_into_cdf(client)
|
|
@@ -283,3 +374,70 @@ class CDFToAPI:
|
|
|
283
374
|
result = self._state.rule_store.export_to_cdf(exporter, self._state.client, dry_run)
|
|
284
375
|
print("You can inspect the details with the .inspect.outcome.data_model(...) method.")
|
|
285
376
|
return result
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
@session_class_wrapper
|
|
380
|
+
class ToPythonAPI:
|
|
381
|
+
"""API used to write the contents of a NeatSession to Python objects"""
|
|
382
|
+
|
|
383
|
+
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
384
|
+
self._state = state
|
|
385
|
+
self._verbose = verbose
|
|
386
|
+
|
|
387
|
+
def instances(
|
|
388
|
+
self,
|
|
389
|
+
instance_space: str | None = None,
|
|
390
|
+
space_from_property: str | None = None,
|
|
391
|
+
use_source_space: bool = False,
|
|
392
|
+
) -> tuple[list[dm.InstanceApply], IssueList]:
|
|
393
|
+
"""Export the verified DMS instances to Python objects.
|
|
394
|
+
|
|
395
|
+
Args:
|
|
396
|
+
instance_space: The name of the instance space to use. Defaults to None.
|
|
397
|
+
space_from_property: This is an alternative to the 'instance_space' argument. If provided,
|
|
398
|
+
the space will be set to the value of the property with the given name for each instance.
|
|
399
|
+
If the property is not found, the 'instance_space' argument will be used. Defaults to None.
|
|
400
|
+
use_source_space: If True, the instance space will be set to the source space of the instance.
|
|
401
|
+
This is only relevant if the instances were extracted from CDF data models. Defaults to False.
|
|
402
|
+
|
|
403
|
+
Returns:
|
|
404
|
+
list[dm.InstanceApply]: The instances as Python objects.
|
|
405
|
+
|
|
406
|
+
Example:
|
|
407
|
+
Export instances to Python objects
|
|
408
|
+
```python
|
|
409
|
+
instances = neat.to._python.instances()
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
Export instances to Python objects using the `dataSetId` property as the space
|
|
413
|
+
```python
|
|
414
|
+
instances = neat.to._python.instances(space_from_property="dataSetId")
|
|
415
|
+
```
|
|
416
|
+
"""
|
|
417
|
+
dms_rules = self._state.rule_store.last_verified_dms_rules
|
|
418
|
+
instance_space = instance_space or f"{dms_rules.metadata.space}_instances"
|
|
419
|
+
|
|
420
|
+
if instance_space and instance_space == dms_rules.metadata.space:
|
|
421
|
+
raise NeatSessionError("Space for instances must be different from the data model space.")
|
|
422
|
+
elif not PATTERNS.space_compliance.match(str(instance_space)):
|
|
423
|
+
raise NeatSessionError(f"Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
|
|
424
|
+
|
|
425
|
+
loader = loaders.DMSLoader(
|
|
426
|
+
self._state.rule_store.last_verified_dms_rules,
|
|
427
|
+
self._state.rule_store.last_verified_information_rules,
|
|
428
|
+
self._state.instances.store,
|
|
429
|
+
instance_space=instance_space,
|
|
430
|
+
space_property=space_from_property,
|
|
431
|
+
use_source_space=use_source_space,
|
|
432
|
+
unquote_external_ids=True,
|
|
433
|
+
neat_prefix_by_predicate_uri=self._state.instances.neat_prefix_by_predicate_uri,
|
|
434
|
+
neat_prefix_by_type_uri=self._state.instances.neat_prefix_by_type_uri,
|
|
435
|
+
)
|
|
436
|
+
issue_list = IssueList()
|
|
437
|
+
instances: list[dm.InstanceApply] = []
|
|
438
|
+
for item in loader.load(stop_on_exception=False):
|
|
439
|
+
if isinstance(item, dm.InstanceApply):
|
|
440
|
+
instances.append(item)
|
|
441
|
+
elif isinstance(item, NeatIssue):
|
|
442
|
+
issue_list.append(item)
|
|
443
|
+
return instances, issue_list
|
|
@@ -3,7 +3,7 @@ import warnings
|
|
|
3
3
|
from collections.abc import Iterable
|
|
4
4
|
from datetime import datetime, timezone
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import cast, overload
|
|
6
|
+
from typing import Any, cast, overload
|
|
7
7
|
from zipfile import ZipExtFile
|
|
8
8
|
|
|
9
9
|
import pandas as pd
|
|
@@ -18,9 +18,6 @@ from cognite.neat._graph.queries import Queries
|
|
|
18
18
|
from cognite.neat._graph.transformers import Transformers
|
|
19
19
|
from cognite.neat._issues import IssueList, catch_issues
|
|
20
20
|
from cognite.neat._issues.errors import OxigraphStorageLockedError
|
|
21
|
-
from cognite.neat._rules.analysis import InformationAnalysis
|
|
22
|
-
from cognite.neat._rules.models import InformationRules
|
|
23
|
-
from cognite.neat._rules.models.entities import ClassEntity
|
|
24
21
|
from cognite.neat._shared import InstanceType, Triple
|
|
25
22
|
from cognite.neat._utils.auxiliary import local_import
|
|
26
23
|
from cognite.neat._utils.rdf_ import add_triples_in_batch, remove_namespace_from_uri
|
|
@@ -55,9 +52,6 @@ class NeatGraphStore:
|
|
|
55
52
|
dataset: Dataset,
|
|
56
53
|
default_named_graph: URIRef | None = None,
|
|
57
54
|
):
|
|
58
|
-
self.rules: dict[URIRef, InformationRules] = {}
|
|
59
|
-
self.base_namespace: dict[URIRef, Namespace] = {}
|
|
60
|
-
|
|
61
55
|
_start = datetime.now(timezone.utc)
|
|
62
56
|
self.dataset = dataset
|
|
63
57
|
self.provenance = Provenance[Entity](
|
|
@@ -72,8 +66,7 @@ class NeatGraphStore:
|
|
|
72
66
|
)
|
|
73
67
|
|
|
74
68
|
self.default_named_graph = default_named_graph or DATASET_DEFAULT_GRAPH_ID
|
|
75
|
-
|
|
76
|
-
self.queries = Queries(self.dataset, self.rules, self.default_named_graph)
|
|
69
|
+
self.queries = Queries(self.dataset, self.default_named_graph)
|
|
77
70
|
|
|
78
71
|
def graph(self, named_graph: URIRef | None = None) -> Graph:
|
|
79
72
|
"""Get named graph from the dataset to query over"""
|
|
@@ -116,36 +109,6 @@ class NeatGraphStore:
|
|
|
116
109
|
else:
|
|
117
110
|
return self.dataset.serialize(format="ox-trig" if self.type_ == "OxigraphStore" else "trig")
|
|
118
111
|
|
|
119
|
-
def add_rules(self, rules: InformationRules, named_graph: URIRef | None = None) -> None:
|
|
120
|
-
"""This method is used to add rules to a named graph stored in the graph store.
|
|
121
|
-
|
|
122
|
-
Args:
|
|
123
|
-
rules: InformationRules object containing rules to be added to the named graph
|
|
124
|
-
named_graph: URIRef of the named graph to store the rules in, by default None
|
|
125
|
-
rules will be added to the default graph
|
|
126
|
-
|
|
127
|
-
"""
|
|
128
|
-
|
|
129
|
-
named_graph = named_graph or self.default_named_graph
|
|
130
|
-
|
|
131
|
-
if named_graph in self.named_graphs:
|
|
132
|
-
# attaching appropriate namespace to the rules
|
|
133
|
-
# as well base_namespace
|
|
134
|
-
self.rules[named_graph] = rules
|
|
135
|
-
self.base_namespace[named_graph] = rules.metadata.namespace
|
|
136
|
-
self.queries = Queries(self.dataset, self.rules)
|
|
137
|
-
self.provenance.append(
|
|
138
|
-
Change.record(
|
|
139
|
-
activity=f"{type(self)}.rules",
|
|
140
|
-
start=datetime.now(timezone.utc),
|
|
141
|
-
end=datetime.now(timezone.utc),
|
|
142
|
-
description=f"Added {type(self.rules).__name__} to {named_graph} named graph",
|
|
143
|
-
)
|
|
144
|
-
)
|
|
145
|
-
|
|
146
|
-
if self.rules[named_graph].prefixes:
|
|
147
|
-
self._upsert_prefixes(self.rules[named_graph].prefixes, named_graph)
|
|
148
|
-
|
|
149
112
|
def _upsert_prefixes(self, prefixes: dict[str, Namespace], named_graph: URIRef) -> None:
|
|
150
113
|
"""Adds prefixes to the graph store."""
|
|
151
114
|
_start = datetime.now(timezone.utc)
|
|
@@ -271,226 +234,26 @@ class NeatGraphStore:
|
|
|
271
234
|
last_change.target_entity.issues.extend(issue_list)
|
|
272
235
|
return issue_list
|
|
273
236
|
|
|
274
|
-
def
|
|
237
|
+
def read(
|
|
275
238
|
self,
|
|
276
|
-
|
|
277
|
-
property_link_pairs: dict[str, URIRef] | None,
|
|
239
|
+
class_uri: URIRef,
|
|
278
240
|
named_graph: URIRef | None = None,
|
|
279
|
-
) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
|
|
280
|
-
named_graph = named_graph or self.default_named_graph
|
|
281
|
-
|
|
282
|
-
if named_graph not in self.named_graphs:
|
|
283
|
-
warnings.warn(
|
|
284
|
-
f"Named graph {named_graph} not found in graph store, cannot read",
|
|
285
|
-
stacklevel=2,
|
|
286
|
-
)
|
|
287
|
-
return
|
|
288
|
-
|
|
289
|
-
if not self.rules or named_graph not in self.rules:
|
|
290
|
-
warnings.warn(
|
|
291
|
-
f"Rules for named graph {named_graph} not found in graph store!",
|
|
292
|
-
stacklevel=2,
|
|
293
|
-
)
|
|
294
|
-
return
|
|
295
|
-
|
|
296
|
-
if self.multi_type_instances:
|
|
297
|
-
warnings.warn(
|
|
298
|
-
"Multi typed instances detected, issues with loading can occur!",
|
|
299
|
-
stacklevel=2,
|
|
300
|
-
)
|
|
301
|
-
|
|
302
|
-
analysis = InformationAnalysis(self.rules[named_graph])
|
|
303
|
-
|
|
304
|
-
if cls := analysis.classes_by_neat_id.get(class_neat_id):
|
|
305
|
-
if property_link_pairs:
|
|
306
|
-
property_renaming_config = {
|
|
307
|
-
prop_uri: prop_name
|
|
308
|
-
for prop_name, prop_neat_id in property_link_pairs.items()
|
|
309
|
-
if (prop_uri := analysis.neat_id_to_instance_source_property_uri(prop_neat_id))
|
|
310
|
-
}
|
|
311
|
-
if information_properties := analysis.classes_with_properties(consider_inheritance=True).get(
|
|
312
|
-
cls.class_
|
|
313
|
-
):
|
|
314
|
-
for prop in information_properties:
|
|
315
|
-
if prop.neatId is None:
|
|
316
|
-
continue
|
|
317
|
-
# Include renaming done in the Information rules that are not present in the
|
|
318
|
-
# property_link_pairs. The use case for this renaming to startNode and endNode
|
|
319
|
-
# properties that are not part of DMSRules but will typically be present
|
|
320
|
-
# in the Information rules.
|
|
321
|
-
if (
|
|
322
|
-
uri := analysis.neat_id_to_instance_source_property_uri(prop.neatId)
|
|
323
|
-
) and uri not in property_renaming_config:
|
|
324
|
-
property_renaming_config[uri] = prop.property_
|
|
325
|
-
|
|
326
|
-
yield from self._read_via_class_entity(cls.class_, property_renaming_config)
|
|
327
|
-
return
|
|
328
|
-
else:
|
|
329
|
-
warnings.warn("Rules not linked", stacklevel=2)
|
|
330
|
-
return
|
|
331
|
-
else:
|
|
332
|
-
warnings.warn("Class with neat id {class_neat_id} found in rules", stacklevel=2)
|
|
333
|
-
return
|
|
334
|
-
|
|
335
|
-
def _read_via_class_entity(
|
|
336
|
-
self,
|
|
337
|
-
class_entity: ClassEntity,
|
|
338
241
|
property_renaming_config: dict[URIRef, str] | None = None,
|
|
339
|
-
|
|
340
|
-
) -> Iterable[tuple[
|
|
242
|
+
remove_uri_namespace: bool = True,
|
|
243
|
+
) -> Iterable[tuple[URIRef, dict[str | InstanceType, list[Any]]]]:
|
|
341
244
|
named_graph = named_graph or self.default_named_graph
|
|
342
245
|
|
|
343
|
-
|
|
344
|
-
warnings.warn(
|
|
345
|
-
f"Named graph {named_graph} not found in graph store, cannot read",
|
|
346
|
-
stacklevel=2,
|
|
347
|
-
)
|
|
348
|
-
return
|
|
349
|
-
|
|
350
|
-
if not self.rules or named_graph not in self.rules:
|
|
351
|
-
warnings.warn(
|
|
352
|
-
f"Rules for named graph {named_graph} not found in graph store!",
|
|
353
|
-
stacklevel=2,
|
|
354
|
-
)
|
|
355
|
-
return
|
|
356
|
-
if self.multi_type_instances:
|
|
357
|
-
warnings.warn(
|
|
358
|
-
"Multi typed instances detected, issues with loading can occur!",
|
|
359
|
-
stacklevel=2,
|
|
360
|
-
)
|
|
361
|
-
|
|
362
|
-
if class_entity not in [definition.class_ for definition in self.rules[named_graph].classes]:
|
|
363
|
-
warnings.warn("Desired type not found in graph!", stacklevel=2)
|
|
364
|
-
return
|
|
365
|
-
|
|
366
|
-
if not (class_uri := InformationAnalysis(self.rules[named_graph]).class_uri(class_entity)):
|
|
367
|
-
warnings.warn(
|
|
368
|
-
f"Class {class_entity.suffix} does not have namespace defined for prefix {class_entity.prefix} Rules!",
|
|
369
|
-
stacklevel=2,
|
|
370
|
-
)
|
|
371
|
-
return
|
|
372
|
-
|
|
373
|
-
has_hop_transformations = InformationAnalysis(self.rules[named_graph]).has_hop_transformations()
|
|
374
|
-
has_self_reference_transformations = InformationAnalysis(
|
|
375
|
-
self.rules[named_graph]
|
|
376
|
-
).has_self_reference_property_transformations()
|
|
377
|
-
if has_hop_transformations or has_self_reference_transformations:
|
|
378
|
-
msg = (
|
|
379
|
-
f"Rules contain [{'Hop' if has_hop_transformations else ''}"
|
|
380
|
-
f", {'SelfReferenceProperty' if has_self_reference_transformations else ''}]"
|
|
381
|
-
" rdfpath."
|
|
382
|
-
f" Run [{'ReduceHopTraversal' if has_hop_transformations else ''}"
|
|
383
|
-
f", {'AddSelfReferenceProperty' if has_self_reference_transformations else ''}]"
|
|
384
|
-
" transformer(s) first!"
|
|
385
|
-
)
|
|
386
|
-
|
|
387
|
-
warnings.warn(
|
|
388
|
-
msg,
|
|
389
|
-
stacklevel=2,
|
|
390
|
-
)
|
|
391
|
-
return
|
|
392
|
-
|
|
393
|
-
# get all the instances for give class_uri
|
|
394
|
-
instance_ids = self.queries.list_instances_ids_of_class(class_uri)
|
|
395
|
-
|
|
396
|
-
# get potential property renaming config
|
|
397
|
-
property_renaming_config = property_renaming_config or InformationAnalysis(
|
|
398
|
-
self.rules[named_graph]
|
|
399
|
-
).define_property_renaming_config(class_entity)
|
|
246
|
+
instance_ids = self.queries.list_instances_ids(class_uri, named_graph=named_graph)
|
|
400
247
|
|
|
401
248
|
for instance_id in instance_ids:
|
|
402
249
|
if res := self.queries.describe(
|
|
403
250
|
instance_id=instance_id,
|
|
404
|
-
instance_type=
|
|
251
|
+
instance_type=class_uri,
|
|
405
252
|
property_renaming_config=property_renaming_config,
|
|
253
|
+
remove_uri_namespace=remove_uri_namespace,
|
|
406
254
|
):
|
|
407
255
|
yield res
|
|
408
256
|
|
|
409
|
-
def read(
|
|
410
|
-
self, class_: str, named_graph: URIRef | None = None
|
|
411
|
-
) -> Iterable[tuple[str, dict[str | InstanceType, list[str]]]]:
|
|
412
|
-
"""Read instances for given class from the graph store.
|
|
413
|
-
|
|
414
|
-
!!! note "Assumption"
|
|
415
|
-
This method assumes that the class_ belongs to the same (name)space as
|
|
416
|
-
the rules which are attached to the graph store.
|
|
417
|
-
|
|
418
|
-
"""
|
|
419
|
-
named_graph = named_graph or self.default_named_graph
|
|
420
|
-
|
|
421
|
-
if named_graph not in self.named_graphs:
|
|
422
|
-
warnings.warn(
|
|
423
|
-
f"Named graph {named_graph} not found in graph store, cannot read",
|
|
424
|
-
stacklevel=2,
|
|
425
|
-
)
|
|
426
|
-
return
|
|
427
|
-
|
|
428
|
-
if not self.rules or named_graph not in self.rules:
|
|
429
|
-
warnings.warn(
|
|
430
|
-
f"Rules for named graph {named_graph} not found in graph store!",
|
|
431
|
-
stacklevel=2,
|
|
432
|
-
)
|
|
433
|
-
return
|
|
434
|
-
if self.multi_type_instances:
|
|
435
|
-
warnings.warn(
|
|
436
|
-
"Multi typed instances detected, issues with loading can occur!",
|
|
437
|
-
stacklevel=2,
|
|
438
|
-
)
|
|
439
|
-
|
|
440
|
-
class_entity = ClassEntity(prefix=self.rules[named_graph].metadata.prefix, suffix=class_)
|
|
441
|
-
|
|
442
|
-
if class_entity not in [definition.class_ for definition in self.rules[named_graph].classes]:
|
|
443
|
-
warnings.warn("Desired type not found in graph!", stacklevel=2)
|
|
444
|
-
return
|
|
445
|
-
|
|
446
|
-
yield from self._read_via_class_entity(class_entity)
|
|
447
|
-
|
|
448
|
-
def count_of_id(self, neat_id: URIRef, named_graph: URIRef | None = None) -> int:
|
|
449
|
-
"""Count the number of instances of a given type
|
|
450
|
-
|
|
451
|
-
Args:
|
|
452
|
-
neat_id: Type for which instances are to be counted
|
|
453
|
-
|
|
454
|
-
Returns:
|
|
455
|
-
Number of instances
|
|
456
|
-
"""
|
|
457
|
-
named_graph = named_graph or self.default_named_graph
|
|
458
|
-
|
|
459
|
-
if named_graph not in self.named_graphs:
|
|
460
|
-
warnings.warn(
|
|
461
|
-
f"Named graph {named_graph} not found in graph store, cannot count",
|
|
462
|
-
stacklevel=2,
|
|
463
|
-
)
|
|
464
|
-
return 0
|
|
465
|
-
|
|
466
|
-
if not self.rules or named_graph not in self.rules:
|
|
467
|
-
warnings.warn(
|
|
468
|
-
f"Rules for named graph {named_graph} not found in graph store!",
|
|
469
|
-
stacklevel=2,
|
|
470
|
-
)
|
|
471
|
-
return 0
|
|
472
|
-
|
|
473
|
-
class_entity = next(
|
|
474
|
-
(definition.class_ for definition in self.rules[named_graph].classes if definition.neatId == neat_id),
|
|
475
|
-
None,
|
|
476
|
-
)
|
|
477
|
-
if not class_entity:
|
|
478
|
-
warnings.warn("Desired type not found in graph!", stacklevel=2)
|
|
479
|
-
return 0
|
|
480
|
-
|
|
481
|
-
if not (class_uri := InformationAnalysis(self.rules[named_graph]).class_uri(class_entity)):
|
|
482
|
-
warnings.warn(
|
|
483
|
-
f"Class {class_entity.suffix} does not have namespace defined for prefix {class_entity.prefix} Rules!",
|
|
484
|
-
stacklevel=2,
|
|
485
|
-
)
|
|
486
|
-
return 0
|
|
487
|
-
|
|
488
|
-
return self.count_of_type(class_uri)
|
|
489
|
-
|
|
490
|
-
def count_of_type(self, class_uri: URIRef) -> int:
|
|
491
|
-
query = f"SELECT (COUNT(?instance) AS ?instanceCount) WHERE {{ ?instance a <{class_uri}> }}"
|
|
492
|
-
return int(next(iter(self.dataset.query(query)))[0]) # type: ignore[arg-type, index]
|
|
493
|
-
|
|
494
257
|
def _parse_file(
|
|
495
258
|
self,
|
|
496
259
|
named_graph: URIRef,
|
cognite/neat/_utils/rdf_.py
CHANGED
|
@@ -7,6 +7,8 @@ from pydantic import HttpUrl, TypeAdapter, ValidationError
|
|
|
7
7
|
from rdflib import Graph, Namespace, URIRef
|
|
8
8
|
from rdflib import Literal as RdfLiteral
|
|
9
9
|
|
|
10
|
+
from cognite.neat._constants import SPACE_URI_PATTERN
|
|
11
|
+
|
|
10
12
|
Triple: TypeAlias = tuple[URIRef, URIRef, RdfLiteral | URIRef]
|
|
11
13
|
|
|
12
14
|
|
|
@@ -100,12 +102,41 @@ def get_namespace(URI: URIRef, special_separator: str = "#_") -> str:
|
|
|
100
102
|
str
|
|
101
103
|
Entity id without namespace
|
|
102
104
|
"""
|
|
105
|
+
return split_uri(URI, special_separator)[0]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def namespace_as_space(namespace: str) -> str | None:
|
|
109
|
+
if match := SPACE_URI_PATTERN.match(namespace):
|
|
110
|
+
return match.group("space")
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def split_uri(URI: URIRef, special_separator: str = "#_") -> tuple[str, str]:
|
|
115
|
+
"""Splits URI into namespace and entity name
|
|
116
|
+
|
|
117
|
+
Parameters
|
|
118
|
+
----------
|
|
119
|
+
URI : URIRef
|
|
120
|
+
URI of an entity
|
|
121
|
+
special_separator : str
|
|
122
|
+
Special separator to use instead of # or / if present in URI
|
|
123
|
+
Set by default to "#_" which covers special client use case
|
|
124
|
+
|
|
125
|
+
Returns
|
|
126
|
+
-------
|
|
127
|
+
tuple[str, str]
|
|
128
|
+
Tuple of namespace and entity name
|
|
129
|
+
"""
|
|
103
130
|
if special_separator in URI:
|
|
104
|
-
|
|
131
|
+
namespace, rest = URI.split(special_separator, maxsplit=1)
|
|
132
|
+
namespace += special_separator
|
|
105
133
|
elif "#" in URI:
|
|
106
|
-
|
|
134
|
+
namespace, rest = URI.split("#", maxsplit=1)
|
|
135
|
+
namespace += "#"
|
|
107
136
|
else:
|
|
108
|
-
|
|
137
|
+
namespace, rest = URI.rsplit("/", maxsplit=1)
|
|
138
|
+
namespace += "/"
|
|
139
|
+
return namespace, rest
|
|
109
140
|
|
|
110
141
|
|
|
111
142
|
def as_neat_compliant_uri(uri: URIRef) -> URIRef:
|
|
@@ -154,7 +185,7 @@ def _traverse(hierarchy: dict, graph: dict, names: list[str]) -> dict:
|
|
|
154
185
|
return hierarchy
|
|
155
186
|
|
|
156
187
|
|
|
157
|
-
def get_inheritance_path(child: Any, child_parent: dict[Any,
|
|
188
|
+
def get_inheritance_path(child: Any, child_parent: dict[Any, set[Any]]) -> list[Any]:
|
|
158
189
|
"""Returns the inheritance path for a given child
|
|
159
190
|
|
|
160
191
|
Args:
|
|
@@ -167,7 +198,7 @@ def get_inheritance_path(child: Any, child_parent: dict[Any, list[Any]]) -> list
|
|
|
167
198
|
!!! note "No Circular Inheritance"
|
|
168
199
|
This method assumes that the child_parent dictionary is a tree and does not contain any cycles.
|
|
169
200
|
"""
|
|
170
|
-
path = []
|
|
201
|
+
path: list[Any] = []
|
|
171
202
|
if child in child_parent:
|
|
172
203
|
path.extend(child_parent[child])
|
|
173
204
|
for parent in child_parent[child]:
|