cognite-neat 0.102.0__py3-none-any.whl → 0.103.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/__init__.py +1 -1
- cognite/neat/_app/api/routers/crud.py +1 -1
- cognite/neat/_client/__init__.py +1 -1
- cognite/neat/_client/_api/data_modeling_loaders.py +1 -1
- cognite/neat/_client/_api/schema.py +1 -1
- cognite/neat/_graph/_tracking/__init__.py +1 -1
- cognite/neat/_graph/extractors/__init__.py +8 -8
- cognite/neat/_graph/extractors/_mock_graph_generator.py +2 -3
- cognite/neat/_graph/loaders/_base.py +1 -1
- cognite/neat/_graph/loaders/_rdf2dms.py +165 -47
- cognite/neat/_graph/transformers/__init__.py +13 -9
- cognite/neat/_graph/transformers/_value_type.py +196 -2
- cognite/neat/_issues/__init__.py +6 -6
- cognite/neat/_issues/_base.py +4 -4
- cognite/neat/_issues/errors/__init__.py +22 -22
- cognite/neat/_issues/formatters.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +20 -18
- cognite/neat/_issues/warnings/_properties.py +7 -0
- cognite/neat/_issues/warnings/user_modeling.py +2 -2
- cognite/neat/_rules/analysis/__init__.py +1 -1
- cognite/neat/_rules/catalog/__init__.py +1 -0
- cognite/neat/_rules/catalog/hello_world_pump.xlsx +0 -0
- cognite/neat/_rules/exporters/__init__.py +5 -5
- cognite/neat/_rules/exporters/_rules2excel.py +5 -4
- cognite/neat/_rules/importers/__init__.py +4 -4
- cognite/neat/_rules/importers/_base.py +7 -3
- cognite/neat/_rules/importers/_rdf/__init__.py +1 -1
- cognite/neat/_rules/models/__init__.py +5 -5
- cognite/neat/_rules/models/_base_rules.py +1 -1
- cognite/neat/_rules/models/dms/__init__.py +11 -11
- cognite/neat/_rules/models/dms/_validation.py +18 -10
- cognite/neat/_rules/models/entities/__init__.py +26 -26
- cognite/neat/_rules/models/entities/_single_value.py +25 -5
- cognite/neat/_rules/models/information/__init__.py +5 -5
- cognite/neat/_rules/models/mapping/_classic2core.yaml +54 -8
- cognite/neat/_rules/transformers/__init__.py +12 -12
- cognite/neat/_rules/transformers/_pipelines.py +10 -5
- cognite/neat/_session/_base.py +71 -0
- cognite/neat/_session/_collector.py +3 -1
- cognite/neat/_session/_drop.py +10 -0
- cognite/neat/_session/_inspect.py +35 -1
- cognite/neat/_session/_mapping.py +5 -0
- cognite/neat/_session/_prepare.py +121 -15
- cognite/neat/_session/_read.py +180 -20
- cognite/neat/_session/_set.py +11 -1
- cognite/neat/_session/_show.py +50 -11
- cognite/neat/_session/_to.py +58 -10
- cognite/neat/_session/engine/__init__.py +1 -1
- cognite/neat/_store/__init__.py +3 -2
- cognite/neat/_store/{_base.py → _graph_store.py} +33 -0
- cognite/neat/_store/_provenance.py +11 -1
- cognite/neat/_store/_rules_store.py +20 -0
- cognite/neat/_utils/auth.py +1 -1
- cognite/neat/_utils/io_.py +11 -0
- cognite/neat/_utils/reader/__init__.py +1 -1
- cognite/neat/_version.py +2 -2
- cognite/neat/_workflows/__init__.py +3 -3
- cognite/neat/_workflows/steps/lib/current/graph_extractor.py +1 -1
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +1 -1
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +2 -2
- cognite/neat/_workflows/steps/lib/io/io_steps.py +3 -3
- {cognite_neat-0.102.0.dist-info → cognite_neat-0.103.1.dist-info}/METADATA +1 -1
- {cognite_neat-0.102.0.dist-info → cognite_neat-0.103.1.dist-info}/RECORD +66 -63
- {cognite_neat-0.102.0.dist-info → cognite_neat-0.103.1.dist-info}/LICENSE +0 -0
- {cognite_neat-0.102.0.dist-info → cognite_neat-0.103.1.dist-info}/WHEEL +0 -0
- {cognite_neat-0.102.0.dist-info → cognite_neat-0.103.1.dist-info}/entry_points.txt +0 -0
cognite/neat/_session/_read.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import tempfile
|
|
2
2
|
from datetime import datetime, timezone
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from typing import Any, Literal
|
|
4
|
+
from typing import Any, Literal, cast
|
|
5
5
|
|
|
6
6
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
|
|
7
7
|
|
|
@@ -11,7 +11,7 @@ from cognite.neat._graph import examples as instances_examples
|
|
|
11
11
|
from cognite.neat._graph import extractors
|
|
12
12
|
from cognite.neat._issues import IssueList
|
|
13
13
|
from cognite.neat._issues.errors import NeatValueError
|
|
14
|
-
from cognite.neat._rules import importers
|
|
14
|
+
from cognite.neat._rules import catalog, importers
|
|
15
15
|
from cognite.neat._rules._shared import ReadRules
|
|
16
16
|
from cognite.neat._rules.importers import BaseImporter
|
|
17
17
|
from cognite.neat._store._provenance import Activity as ProvenanceActivity
|
|
@@ -27,6 +27,8 @@ from .exceptions import NeatSessionError, session_class_wrapper
|
|
|
27
27
|
|
|
28
28
|
@session_class_wrapper
|
|
29
29
|
class ReadAPI:
|
|
30
|
+
"""Read from a data source into NeatSession graph store."""
|
|
31
|
+
|
|
30
32
|
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
31
33
|
self._state = state
|
|
32
34
|
self._verbose = verbose
|
|
@@ -68,6 +70,11 @@ class BaseReadAPI:
|
|
|
68
70
|
|
|
69
71
|
@session_class_wrapper
|
|
70
72
|
class CDFReadAPI(BaseReadAPI):
|
|
73
|
+
"""Reads from CDF Data Models.
|
|
74
|
+
Use the `.data_model()` method to load a CDF Data Model to the knowledge graph.
|
|
75
|
+
|
|
76
|
+
"""
|
|
77
|
+
|
|
71
78
|
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
72
79
|
super().__init__(state, client, verbose)
|
|
73
80
|
self.classic = CDFClassicAPI(state, client, verbose)
|
|
@@ -79,6 +86,18 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
79
86
|
return self._client
|
|
80
87
|
|
|
81
88
|
def data_model(self, data_model_id: DataModelIdentifier) -> IssueList:
|
|
89
|
+
"""Reads a Data Model from CDF to the knowledge graph.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
data_model_id: Tuple of strings with the id of a CDF Data Model.
|
|
93
|
+
Notation as follows (<name_of_space>, <name_of_data_model>, <data_model_version>)
|
|
94
|
+
|
|
95
|
+
Example:
|
|
96
|
+
```python
|
|
97
|
+
neat.read.cdf.data_model(("example_data_model_space", "EXAMPLE_DATA_MODEL", "v1"))
|
|
98
|
+
```
|
|
99
|
+
"""
|
|
100
|
+
|
|
82
101
|
data_model_id = DataModelId.load(data_model_id)
|
|
83
102
|
|
|
84
103
|
if not data_model_id.version:
|
|
@@ -113,6 +132,11 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
113
132
|
|
|
114
133
|
@session_class_wrapper
|
|
115
134
|
class CDFClassicAPI(BaseReadAPI):
|
|
135
|
+
"""Reads from the Classic Data Model from CDF.
|
|
136
|
+
Use the `.graph()` method to load CDF core resources to the knowledge graph.
|
|
137
|
+
|
|
138
|
+
"""
|
|
139
|
+
|
|
116
140
|
@property
|
|
117
141
|
def _get_client(self) -> NeatClient:
|
|
118
142
|
if self._client is None:
|
|
@@ -124,12 +148,12 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
124
148
|
|
|
125
149
|
The Classic Graph consists of the following core resource type.
|
|
126
150
|
|
|
127
|
-
Classic Node CDF Resources
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
151
|
+
!!! note "Classic Node CDF Resources"
|
|
152
|
+
- Assets
|
|
153
|
+
- TimeSeries
|
|
154
|
+
- Sequences
|
|
155
|
+
- Events
|
|
156
|
+
- Files
|
|
133
157
|
|
|
134
158
|
All the classic node CDF resources can have one or more connections to one or more assets. This
|
|
135
159
|
will match a direct relationship in the data modeling of CDF.
|
|
@@ -144,13 +168,12 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
144
168
|
This extractor will extract the classic CDF graph into Neat starting from either a data set or a root asset.
|
|
145
169
|
|
|
146
170
|
It works as follows:
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
5. Extract all data sets that are connected to the extracted core nodes/relationships.
|
|
171
|
+
1. Extract all core nodes (assets, time series, sequences, events, files) filtered by the given data set or
|
|
172
|
+
root asset.
|
|
173
|
+
2. Extract all relationships starting from any of the extracted core nodes.
|
|
174
|
+
3. Extract all core nodes that are targets of the relationships that are not already extracted.
|
|
175
|
+
4. Extract all labels that are connected to the extracted core nodes/relationships.
|
|
176
|
+
5. Extract all data sets that are connected to the extracted core nodes/relationships.
|
|
154
177
|
|
|
155
178
|
Args:
|
|
156
179
|
root_asset_external_id: The external id of the root asset
|
|
@@ -168,7 +191,29 @@ class CDFClassicAPI(BaseReadAPI):
|
|
|
168
191
|
|
|
169
192
|
@session_class_wrapper
|
|
170
193
|
class ExcelReadAPI(BaseReadAPI):
|
|
194
|
+
"""Reads a Neat Excel Rules sheet to the graph store. The rules sheet may stem from an Information architect,
|
|
195
|
+
or a DMS Architect.
|
|
196
|
+
|
|
197
|
+
Args:
|
|
198
|
+
io: file path to the Excel sheet
|
|
199
|
+
|
|
200
|
+
Example:
|
|
201
|
+
```python
|
|
202
|
+
neat.read.excel("information_or_dms_rules_sheet.xlsx")
|
|
203
|
+
```
|
|
204
|
+
"""
|
|
205
|
+
|
|
206
|
+
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
207
|
+
super().__init__(state, client, verbose)
|
|
208
|
+
self.examples = ExcelExampleAPI(state, client, verbose)
|
|
209
|
+
|
|
171
210
|
def __call__(self, io: Any) -> IssueList:
|
|
211
|
+
"""Reads a Neat Excel Rules sheet to the graph store. The rules sheet may stem from an Information architect,
|
|
212
|
+
or a DMS Architect.
|
|
213
|
+
|
|
214
|
+
Args:
|
|
215
|
+
io: file path to the Excel sheet
|
|
216
|
+
"""
|
|
172
217
|
reader = NeatReader.create(io)
|
|
173
218
|
start = datetime.now(timezone.utc)
|
|
174
219
|
if not isinstance(reader, PathReader):
|
|
@@ -190,8 +235,45 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
190
235
|
return input_rules.issues
|
|
191
236
|
|
|
192
237
|
|
|
238
|
+
@session_class_wrapper
|
|
239
|
+
class ExcelExampleAPI(BaseReadAPI):
|
|
240
|
+
"""Used as example for reading some data model into the NeatSession."""
|
|
241
|
+
|
|
242
|
+
@property
|
|
243
|
+
def pump_example(self) -> IssueList:
|
|
244
|
+
"""Reads the Nordic 44 knowledge graph into the NeatSession graph store."""
|
|
245
|
+
start = datetime.now(timezone.utc)
|
|
246
|
+
importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)
|
|
247
|
+
input_rules: ReadRules = importer.to_rules()
|
|
248
|
+
end = datetime.now(timezone.utc)
|
|
249
|
+
|
|
250
|
+
if input_rules.rules:
|
|
251
|
+
change = Change.from_rules_activity(
|
|
252
|
+
input_rules,
|
|
253
|
+
importer.agent,
|
|
254
|
+
start,
|
|
255
|
+
end,
|
|
256
|
+
description="Pump Example read as unverified data model",
|
|
257
|
+
)
|
|
258
|
+
self._store_rules(input_rules, change)
|
|
259
|
+
self._state.data_model.issue_lists.append(input_rules.issues)
|
|
260
|
+
return input_rules.issues
|
|
261
|
+
|
|
262
|
+
|
|
193
263
|
@session_class_wrapper
|
|
194
264
|
class YamlReadAPI(BaseReadAPI):
|
|
265
|
+
"""Reads a yaml with either neat rules, or several toolkit yaml files to import Data Model(s) into NeatSession.
|
|
266
|
+
|
|
267
|
+
Args:
|
|
268
|
+
io: file path to the Yaml file in the case of "neat" yaml, or path to a zip folder or directory with several
|
|
269
|
+
Yaml files in the case of "toolkit".
|
|
270
|
+
|
|
271
|
+
Example:
|
|
272
|
+
```python
|
|
273
|
+
neat.read.yaml("path_to_toolkit_yamls")
|
|
274
|
+
```
|
|
275
|
+
"""
|
|
276
|
+
|
|
195
277
|
def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
|
|
196
278
|
reader = NeatReader.create(io)
|
|
197
279
|
if not isinstance(reader, PathReader):
|
|
@@ -242,6 +324,23 @@ class YamlReadAPI(BaseReadAPI):
|
|
|
242
324
|
|
|
243
325
|
@session_class_wrapper
|
|
244
326
|
class CSVReadAPI(BaseReadAPI):
|
|
327
|
+
"""Reads a csv that contains a column to use as primary key which will be the unique identifier for the type of
|
|
328
|
+
data you want to read in. Ex. a csv can hold information about assets, and their identifiers are specified in
|
|
329
|
+
a "ASSET_TAG" column.
|
|
330
|
+
|
|
331
|
+
Args:
|
|
332
|
+
io: file path or url to the csv
|
|
333
|
+
type: string that specifies what type of data the csv contains. For instance "Asset" or "Equipment"
|
|
334
|
+
primary_key: string name of the column that should be used as the unique identifier for each row of data
|
|
335
|
+
|
|
336
|
+
Example:
|
|
337
|
+
```python
|
|
338
|
+
type_described_in_table = "Turbine"
|
|
339
|
+
column_with_identifier = "UNIQUE_TAG_NAME"
|
|
340
|
+
neat.read.csv("url_or_path_to_csv_file", type=type_described_in_table, primary_key=column_with_identifier)
|
|
341
|
+
```
|
|
342
|
+
"""
|
|
343
|
+
|
|
245
344
|
def __call__(self, io: Any, type: str, primary_key: str) -> None:
|
|
246
345
|
reader = NeatReader.create(io)
|
|
247
346
|
if isinstance(reader, HttpFileReader):
|
|
@@ -263,6 +362,13 @@ class CSVReadAPI(BaseReadAPI):
|
|
|
263
362
|
|
|
264
363
|
@session_class_wrapper
|
|
265
364
|
class XMLReadAPI(BaseReadAPI):
|
|
365
|
+
"""Reads an XML file that is either of DEXPI or AML format.
|
|
366
|
+
|
|
367
|
+
Args:
|
|
368
|
+
io: file path or url to the XML
|
|
369
|
+
format: can be either "dexpi" or "aml" are the currenly supported XML source types.
|
|
370
|
+
"""
|
|
371
|
+
|
|
266
372
|
def __call__(
|
|
267
373
|
self,
|
|
268
374
|
io: Any,
|
|
@@ -289,6 +395,16 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
289
395
|
raise NeatValueError("Only support XML files of DEXPI format at the moment.")
|
|
290
396
|
|
|
291
397
|
def dexpi(self, path):
|
|
398
|
+
"""Reads a DEXPI file into the NeatSession.
|
|
399
|
+
|
|
400
|
+
Args:
|
|
401
|
+
io: file path or url to the DEXPI file
|
|
402
|
+
|
|
403
|
+
Example:
|
|
404
|
+
```python
|
|
405
|
+
neat.read.xml.dexpi("url_or_path_to_dexpi_file")
|
|
406
|
+
```
|
|
407
|
+
"""
|
|
292
408
|
engine = import_engine()
|
|
293
409
|
engine.set.format = "dexpi"
|
|
294
410
|
engine.set.file = path
|
|
@@ -296,6 +412,16 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
296
412
|
self._state.instances.store.write(extractor)
|
|
297
413
|
|
|
298
414
|
def aml(self, path):
|
|
415
|
+
"""Reads an AML file into NeatSession.
|
|
416
|
+
|
|
417
|
+
Args:
|
|
418
|
+
io: file path or url to the AML file
|
|
419
|
+
|
|
420
|
+
Example:
|
|
421
|
+
```python
|
|
422
|
+
neat.read.xml.aml("url_or_path_to_aml_file")
|
|
423
|
+
```
|
|
424
|
+
"""
|
|
299
425
|
engine = import_engine()
|
|
300
426
|
engine.set.format = "aml"
|
|
301
427
|
engine.set.file = path
|
|
@@ -305,11 +431,27 @@ class XMLReadAPI(BaseReadAPI):
|
|
|
305
431
|
|
|
306
432
|
@session_class_wrapper
|
|
307
433
|
class RDFReadAPI(BaseReadAPI):
|
|
434
|
+
"""Reads an RDF source into NeatSession. Supported sources are "ontology" or "imf".
|
|
435
|
+
|
|
436
|
+
Args:
|
|
437
|
+
io: file path or url to the RDF source
|
|
438
|
+
"""
|
|
439
|
+
|
|
308
440
|
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
309
441
|
super().__init__(state, client, verbose)
|
|
310
442
|
self.examples = RDFExamples(state)
|
|
311
443
|
|
|
312
444
|
def ontology(self, io: Any) -> IssueList:
|
|
445
|
+
"""Reads an OWL ontology source into NeatSession.
|
|
446
|
+
|
|
447
|
+
Args:
|
|
448
|
+
io: file path or url to the OWL file
|
|
449
|
+
|
|
450
|
+
Example:
|
|
451
|
+
```python
|
|
452
|
+
neat.read.rdf.ontology("url_or_path_to_owl_source")
|
|
453
|
+
```
|
|
454
|
+
"""
|
|
313
455
|
start = datetime.now(timezone.utc)
|
|
314
456
|
reader = NeatReader.create(io)
|
|
315
457
|
if not isinstance(reader, PathReader):
|
|
@@ -331,6 +473,16 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
331
473
|
return input_rules.issues
|
|
332
474
|
|
|
333
475
|
def imf(self, io: Any) -> IssueList:
|
|
476
|
+
"""Reads IMF Types provided as SHACL shapes into NeatSession.
|
|
477
|
+
|
|
478
|
+
Args:
|
|
479
|
+
io: file path or url to the IMF file
|
|
480
|
+
|
|
481
|
+
Example:
|
|
482
|
+
```python
|
|
483
|
+
neat.read.rdf.imf("url_or_path_to_imf_source")
|
|
484
|
+
```
|
|
485
|
+
"""
|
|
334
486
|
start = datetime.now(timezone.utc)
|
|
335
487
|
reader = NeatReader.create(io)
|
|
336
488
|
if not isinstance(reader, PathReader):
|
|
@@ -360,16 +512,20 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
360
512
|
if type is None:
|
|
361
513
|
type = object_wizard()
|
|
362
514
|
|
|
363
|
-
|
|
515
|
+
type = type.lower()
|
|
516
|
+
|
|
517
|
+
if type == "data model":
|
|
364
518
|
source = source or rdf_dm_wizard("What type of data model is the RDF?")
|
|
365
|
-
|
|
519
|
+
source = cast(str, source).lower() # type: ignore
|
|
520
|
+
|
|
521
|
+
if source == "ontology":
|
|
366
522
|
return self.ontology(io)
|
|
367
|
-
elif source == "
|
|
523
|
+
elif source == "imf types":
|
|
368
524
|
return self.imf(io)
|
|
369
525
|
else:
|
|
370
|
-
raise ValueError(f"Expected ontology, imf or instances, got {source}")
|
|
526
|
+
raise ValueError(f"Expected ontology, imf types or instances, got {source}")
|
|
371
527
|
|
|
372
|
-
elif type
|
|
528
|
+
elif type == "instances":
|
|
373
529
|
reader = NeatReader.create(io)
|
|
374
530
|
if not isinstance(reader, PathReader):
|
|
375
531
|
raise NeatValueError("Only file paths are supported for RDF files")
|
|
@@ -380,11 +536,15 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
380
536
|
raise NeatSessionError(f"Expected data model or instances, got {type}")
|
|
381
537
|
|
|
382
538
|
|
|
539
|
+
@session_class_wrapper
|
|
383
540
|
class RDFExamples:
|
|
541
|
+
"""Used as example for reading some triples into the NeatSession knowledge grapgh."""
|
|
542
|
+
|
|
384
543
|
def __init__(self, state: SessionState) -> None:
|
|
385
544
|
self._state = state
|
|
386
545
|
|
|
387
546
|
@property
|
|
388
547
|
def nordic44(self) -> IssueList:
|
|
548
|
+
"""Reads the Nordic 44 knowledge graph into the NeatSession graph store."""
|
|
389
549
|
self._state.instances.store.write(extractors.RdfFileExtractor(instances_examples.nordic44_knowledge_graph))
|
|
390
550
|
return IssueList()
|
cognite/neat/_session/_set.py
CHANGED
|
@@ -12,12 +12,22 @@ from .exceptions import NeatSessionError, session_class_wrapper
|
|
|
12
12
|
|
|
13
13
|
@session_class_wrapper
|
|
14
14
|
class SetAPI:
|
|
15
|
+
"""Used to change the name of the data model from a data model id defined by neat to a user specified name."""
|
|
16
|
+
|
|
15
17
|
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
16
18
|
self._state = state
|
|
17
19
|
self._verbose = verbose
|
|
18
20
|
|
|
19
21
|
def data_model_id(self, new_model_id: dm.DataModelId | tuple[str, str, str]) -> None:
|
|
20
|
-
"""Sets the data model ID of the latest verified data model.
|
|
22
|
+
"""Sets the data model ID of the latest verified data model. Set the data model id as a tuple of strings
|
|
23
|
+
following the template (<data_model_space>, <data_model_name>, <data_model_version>).
|
|
24
|
+
|
|
25
|
+
Example:
|
|
26
|
+
Set a new data model id:
|
|
27
|
+
```python
|
|
28
|
+
neat.set.data_model_id(("my_data_model_space", "My_Data_Model", "v1"))
|
|
29
|
+
```
|
|
30
|
+
"""
|
|
21
31
|
if res := self._state.data_model.last_verified_dms_rules:
|
|
22
32
|
source_id, rules = res
|
|
23
33
|
|
cognite/neat/_session/_show.py
CHANGED
|
@@ -13,6 +13,7 @@ from cognite.neat._rules.models.dms._rules import DMSRules
|
|
|
13
13
|
from cognite.neat._rules.models.entities._single_value import ClassEntity, ViewEntity
|
|
14
14
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
15
15
|
from cognite.neat._session.exceptions import NeatSessionError
|
|
16
|
+
from cognite.neat._utils.io_ import to_directory_compatible
|
|
16
17
|
from cognite.neat._utils.rdf_ import remove_namespace_from_uri
|
|
17
18
|
|
|
18
19
|
from ._state import SessionState
|
|
@@ -21,6 +22,30 @@ from .exceptions import session_class_wrapper
|
|
|
21
22
|
|
|
22
23
|
@session_class_wrapper
|
|
23
24
|
class ShowAPI:
|
|
25
|
+
"""Visualise a verified data model or instances contained in the graph store.
|
|
26
|
+
See, for example, `.data_model()` or `.instances()` for more.
|
|
27
|
+
|
|
28
|
+
Example:
|
|
29
|
+
Show instances
|
|
30
|
+
```python
|
|
31
|
+
from cognite.neat import NeatSession
|
|
32
|
+
from cognite.neat import get_cognite_client
|
|
33
|
+
|
|
34
|
+
client = get_cognite_client(env_file_path=".env")
|
|
35
|
+
neat = NeatSession(client, storage="oxigraph") # Storage optimised for storage visualisation
|
|
36
|
+
|
|
37
|
+
# .... intermediate steps of reading, infering verifying and converting a data model and instances
|
|
38
|
+
|
|
39
|
+
neat.show.instances()
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Example:
|
|
43
|
+
Show data model
|
|
44
|
+
```python
|
|
45
|
+
neat.show.data_model()
|
|
46
|
+
```
|
|
47
|
+
"""
|
|
48
|
+
|
|
24
49
|
def __init__(self, state: SessionState) -> None:
|
|
25
50
|
self._state = state
|
|
26
51
|
self.data_model = ShowDataModelAPI(self._state)
|
|
@@ -65,6 +90,8 @@ class ShowBaseAPI:
|
|
|
65
90
|
|
|
66
91
|
@session_class_wrapper
|
|
67
92
|
class ShowDataModelAPI(ShowBaseAPI):
|
|
93
|
+
"""Visualises the verified data model."""
|
|
94
|
+
|
|
68
95
|
def __init__(self, state: SessionState) -> None:
|
|
69
96
|
super().__init__(state)
|
|
70
97
|
self._state = state
|
|
@@ -80,16 +107,16 @@ class ShowDataModelAPI(ShowBaseAPI):
|
|
|
80
107
|
rules = self._state.data_model.last_verified_rule[1]
|
|
81
108
|
|
|
82
109
|
if isinstance(rules, DMSRules):
|
|
83
|
-
di_graph = self._generate_dms_di_graph(
|
|
84
|
-
name = "dms_data_model.html"
|
|
110
|
+
di_graph = self._generate_dms_di_graph(rules)
|
|
85
111
|
elif isinstance(rules, InformationRules):
|
|
86
|
-
di_graph = self._generate_info_di_graph(
|
|
87
|
-
name = "information_data_model.html"
|
|
112
|
+
di_graph = self._generate_info_di_graph(rules)
|
|
88
113
|
else:
|
|
89
114
|
# This should never happen, but we need to handle it to satisfy mypy
|
|
90
115
|
raise NeatSessionError(
|
|
91
116
|
f"Unsupported type {type(rules) }. Make sure you have either information or DMS rules."
|
|
92
117
|
)
|
|
118
|
+
identifier = to_directory_compatible(str(rules.metadata.identifier))
|
|
119
|
+
name = f"{identifier}.html"
|
|
93
120
|
|
|
94
121
|
return self._generate_visualization(di_graph, name)
|
|
95
122
|
|
|
@@ -97,9 +124,18 @@ class ShowDataModelAPI(ShowBaseAPI):
|
|
|
97
124
|
"""Generate a DiGraph from the last verified DMS rules."""
|
|
98
125
|
di_graph = nx.DiGraph()
|
|
99
126
|
|
|
127
|
+
# Views with properties or used as ValueType
|
|
128
|
+
# If a view is not used in properties or as ValueType, it is not added to the graph
|
|
129
|
+
# as we typically do not have the properties for it.
|
|
130
|
+
used_views = {prop_.view for prop_ in rules.properties} | {
|
|
131
|
+
prop_.value_type for prop_ in rules.properties if isinstance(prop_.value_type, ViewEntity)
|
|
132
|
+
}
|
|
133
|
+
|
|
100
134
|
# Add nodes and edges from Views sheet
|
|
101
135
|
for view in rules.views:
|
|
102
|
-
|
|
136
|
+
if view.view not in used_views:
|
|
137
|
+
continue
|
|
138
|
+
# if possible use humanreadable label coming from the view name
|
|
103
139
|
if not di_graph.has_node(view.view.suffix):
|
|
104
140
|
di_graph.add_node(view.view.suffix, label=view.view.suffix)
|
|
105
141
|
|
|
@@ -123,7 +159,7 @@ class ShowDataModelAPI(ShowBaseAPI):
|
|
|
123
159
|
|
|
124
160
|
# Add nodes and edges from Views sheet
|
|
125
161
|
for class_ in rules.classes:
|
|
126
|
-
# if possible use
|
|
162
|
+
# if possible use humanreadable label coming from the view name
|
|
127
163
|
if not di_graph.has_node(class_.class_.suffix):
|
|
128
164
|
di_graph.add_node(
|
|
129
165
|
class_.class_.suffix,
|
|
@@ -160,17 +196,16 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
|
|
|
160
196
|
rules = self._state.data_model.last_verified_rule[1]
|
|
161
197
|
|
|
162
198
|
if isinstance(rules, DMSRules):
|
|
163
|
-
di_graph = self._generate_dms_di_graph(
|
|
164
|
-
name = "dms_data_model_implements.html"
|
|
199
|
+
di_graph = self._generate_dms_di_graph(rules)
|
|
165
200
|
elif isinstance(rules, InformationRules):
|
|
166
|
-
di_graph = self._generate_info_di_graph(
|
|
167
|
-
name = "information_data_model_implements.html"
|
|
201
|
+
di_graph = self._generate_info_di_graph(rules)
|
|
168
202
|
else:
|
|
169
203
|
# This should never happen, but we need to handle it to satisfy mypy
|
|
170
204
|
raise NeatSessionError(
|
|
171
205
|
f"Unsupported type {type(rules) }. Make sure you have either information or DMS rules."
|
|
172
206
|
)
|
|
173
|
-
|
|
207
|
+
identifier = to_directory_compatible(str(rules.metadata.identifier))
|
|
208
|
+
name = f"{identifier}_implements.html"
|
|
174
209
|
return self._generate_visualization(di_graph, name)
|
|
175
210
|
|
|
176
211
|
def _generate_dms_di_graph(self, rules: DMSRules) -> nx.DiGraph:
|
|
@@ -228,6 +263,8 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
|
|
|
228
263
|
|
|
229
264
|
@session_class_wrapper
|
|
230
265
|
class ShowDataModelProvenanceAPI(ShowBaseAPI):
|
|
266
|
+
"""Visualises the provenance or steps that have been executed in the NeatSession."""
|
|
267
|
+
|
|
231
268
|
def __init__(self, state: SessionState) -> None:
|
|
232
269
|
super().__init__(state)
|
|
233
270
|
self._state = state
|
|
@@ -288,6 +325,8 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
|
|
|
288
325
|
|
|
289
326
|
@session_class_wrapper
|
|
290
327
|
class ShowInstanceAPI(ShowBaseAPI):
|
|
328
|
+
"""Visualise the instances contained in the graph store."""
|
|
329
|
+
|
|
291
330
|
def __init__(self, state: SessionState) -> None:
|
|
292
331
|
super().__init__(state)
|
|
293
332
|
self._state = state
|
cognite/neat/_session/_to.py
CHANGED
|
@@ -19,6 +19,11 @@ from .exceptions import NeatSessionError, session_class_wrapper
|
|
|
19
19
|
|
|
20
20
|
@session_class_wrapper
|
|
21
21
|
class ToAPI:
|
|
22
|
+
"""API used to write the contents of a NeatSession to a specified destination. For instance writing information
|
|
23
|
+
rules or DMS rules to a NEAT rules Excel spreadsheet, or writing a verified data model to CDF.
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
|
|
22
27
|
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
23
28
|
self._state = state
|
|
24
29
|
self._verbose = verbose
|
|
@@ -34,8 +39,22 @@ class ToAPI:
|
|
|
34
39
|
Args:
|
|
35
40
|
io: The file path or file-like object to write the Excel file to.
|
|
36
41
|
model: The format of the data model to export. Defaults to None.
|
|
42
|
+
|
|
43
|
+
Example:
|
|
44
|
+
Export information model to excel rules sheet
|
|
45
|
+
```python
|
|
46
|
+
information_rules_file_name = "information_rules.xlsx"
|
|
47
|
+
neat.to.excel(information_rules_file_name, model="information")
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Example:
|
|
51
|
+
Export data model to excel rules sheet
|
|
52
|
+
```python
|
|
53
|
+
dms_rules_file_name = "dms_rules.xlsx"
|
|
54
|
+
neat.to.excel(information_rules_file_name, model="dms")
|
|
55
|
+
```
|
|
37
56
|
"""
|
|
38
|
-
exporter = exporters.ExcelExporter()
|
|
57
|
+
exporter = exporters.ExcelExporter(styling="maximal")
|
|
39
58
|
rules: VerifiedRules
|
|
40
59
|
if model == "information" or model == "logical":
|
|
41
60
|
rules = self._state.data_model.last_verified_information_rules[1]
|
|
@@ -63,13 +82,33 @@ class ToAPI:
|
|
|
63
82
|
format: The format of the YAML file. Defaults to "neat".
|
|
64
83
|
skip_system_spaces: If True, system spaces will be skipped. Defaults to True.
|
|
65
84
|
|
|
66
|
-
|
|
67
|
-
|
|
85
|
+
!!! note "YAML formats"
|
|
68
86
|
- "neat": This is the format Neat uses to store the data model.
|
|
69
87
|
- "toolkit": This is the format used by Cognite Toolkit, that matches the CDF API.
|
|
70
88
|
|
|
71
89
|
Returns:
|
|
72
90
|
str | None: If io is None, the YAML string will be returned. Otherwise, None will be returned.
|
|
91
|
+
|
|
92
|
+
Example:
|
|
93
|
+
Export to yaml file in the case of "neat" format
|
|
94
|
+
```python
|
|
95
|
+
your_yaml_file_name = "neat_rules.yaml"
|
|
96
|
+
neat.to.yaml(your_yaml_file_name, format="neat")
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Example:
|
|
100
|
+
Export yaml files as a zip folder in the case of "toolkit" format
|
|
101
|
+
```python
|
|
102
|
+
your_zip_folder_name = "toolkit_data_model_files.zip"
|
|
103
|
+
neat.to.yaml(your_zip_folder_name, format="toolkit")
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Example:
|
|
107
|
+
Export yaml files to a folder in the case of "toolkit" format
|
|
108
|
+
```python
|
|
109
|
+
your_folder_name = "my_project/data_model_files"
|
|
110
|
+
neat.to.yaml(your_folder_name, format="toolkit")
|
|
111
|
+
```
|
|
73
112
|
"""
|
|
74
113
|
if format == "neat":
|
|
75
114
|
exporter = exporters.YAMLExporter()
|
|
@@ -97,12 +136,21 @@ class ToAPI:
|
|
|
97
136
|
|
|
98
137
|
@session_class_wrapper
|
|
99
138
|
class CDFToAPI:
|
|
139
|
+
"""Write a verified Data Model and Instances to CDF."""
|
|
140
|
+
|
|
100
141
|
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
101
142
|
self._client = client
|
|
102
143
|
self._state = state
|
|
103
144
|
self._verbose = verbose
|
|
104
145
|
|
|
105
146
|
def instances(self, space: str | None = None) -> UploadResultList:
|
|
147
|
+
"""Export the verified DMS instances to CDF.
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
space: Name of instance space to use. Default is to suffix the schema space with '_instances'.
|
|
151
|
+
Note this space is required to be different than the space with the data model.
|
|
152
|
+
|
|
153
|
+
"""
|
|
106
154
|
if not self._client:
|
|
107
155
|
raise NeatSessionError("No CDF client provided!")
|
|
108
156
|
|
|
@@ -120,6 +168,7 @@ class CDFToAPI:
|
|
|
120
168
|
self._state.data_model.last_verified_dms_rules[1],
|
|
121
169
|
self._state.instances.store,
|
|
122
170
|
instance_space=space,
|
|
171
|
+
client=self._client,
|
|
123
172
|
)
|
|
124
173
|
result = loader.load_into_cdf(self._client)
|
|
125
174
|
self._state.instances.outcome.append(result)
|
|
@@ -144,13 +193,12 @@ class CDFToAPI:
|
|
|
144
193
|
Note this only applies to spaces and containers if they contain data.
|
|
145
194
|
components: The components to export. If None, all components will be exported. Defaults to None.
|
|
146
195
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
- "recreate": All components will be deleted and recreated. The exception is spaces, which will be updated.
|
|
196
|
+
!!! note "Data Model creation modes"
|
|
197
|
+
- "fail": If any component already exists, the export will fail.
|
|
198
|
+
- "skip": If any component already exists, it will be skipped.
|
|
199
|
+
- "update": If any component already exists, it will be updated.
|
|
200
|
+
- "force": If any component already exists, and the update fails, it will be deleted and recreated.
|
|
201
|
+
- "recreate": All components will be deleted and recreated. The exception is spaces, which will be updated.
|
|
154
202
|
|
|
155
203
|
"""
|
|
156
204
|
|
cognite/neat/_store/__init__.py
CHANGED
|
@@ -315,6 +315,39 @@ class NeatGraphStore:
|
|
|
315
315
|
|
|
316
316
|
yield from self._read_via_class_entity(class_entity)
|
|
317
317
|
|
|
318
|
+
def count_of_id(self, neat_id: URIRef) -> int:
|
|
319
|
+
"""Count the number of instances of a given type
|
|
320
|
+
|
|
321
|
+
Args:
|
|
322
|
+
neat_id: Type for which instances are to be counted
|
|
323
|
+
|
|
324
|
+
Returns:
|
|
325
|
+
Number of instances
|
|
326
|
+
"""
|
|
327
|
+
if not self.rules:
|
|
328
|
+
warnings.warn("Rules not found in graph store!", stacklevel=2)
|
|
329
|
+
return 0
|
|
330
|
+
|
|
331
|
+
class_entity = next(
|
|
332
|
+
(definition.class_ for definition in self.rules.classes if definition.neatId == neat_id), None
|
|
333
|
+
)
|
|
334
|
+
if not class_entity:
|
|
335
|
+
warnings.warn("Desired type not found in graph!", stacklevel=2)
|
|
336
|
+
return 0
|
|
337
|
+
|
|
338
|
+
if not (class_uri := InformationAnalysis(self.rules).class_uri(class_entity)):
|
|
339
|
+
warnings.warn(
|
|
340
|
+
f"Class {class_entity.suffix} does not have namespace defined for prefix {class_entity.prefix} Rules!",
|
|
341
|
+
stacklevel=2,
|
|
342
|
+
)
|
|
343
|
+
return 0
|
|
344
|
+
|
|
345
|
+
return self.count_of_type(class_uri)
|
|
346
|
+
|
|
347
|
+
def count_of_type(self, class_uri: URIRef) -> int:
|
|
348
|
+
query = f"SELECT (COUNT(?instance) AS ?instanceCount) WHERE {{ ?instance a <{class_uri}> }}"
|
|
349
|
+
return int(next(iter(self.graph.query(query)))[0]) # type: ignore[arg-type, index]
|
|
350
|
+
|
|
318
351
|
def _parse_file(
|
|
319
352
|
self,
|
|
320
353
|
filepath: Path,
|