cognite-neat 0.103.0__py3-none-any.whl → 0.104.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
- cognite/neat/_graph/transformers/_base.py +109 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +4 -0
- cognite/neat/_graph/transformers/_prune_graph.py +103 -47
- cognite/neat/_graph/transformers/_rdfpath.py +41 -17
- cognite/neat/_graph/transformers/_value_type.py +119 -154
- cognite/neat/_issues/_base.py +35 -8
- cognite/neat/_issues/warnings/_resources.py +1 -1
- cognite/neat/_rules/_shared.py +18 -34
- cognite/neat/_rules/exporters/_base.py +28 -2
- cognite/neat/_rules/exporters/_rules2dms.py +4 -0
- cognite/neat/_rules/exporters/_rules2excel.py +11 -0
- cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
- cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
- cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
- cognite/neat/_rules/importers/_base.py +9 -0
- cognite/neat/_rules/importers/_dms2rules.py +17 -5
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
- cognite/neat/_rules/importers/_rdf/_base.py +10 -8
- cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
- cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
- cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
- cognite/neat/_rules/importers/_yaml2rules.py +21 -7
- cognite/neat/_rules/models/_base_input.py +1 -1
- cognite/neat/_rules/models/_base_rules.py +5 -0
- cognite/neat/_rules/models/dms/_rules.py +4 -0
- cognite/neat/_rules/models/dms/_rules_input.py +9 -0
- cognite/neat/_rules/models/dms/_validation.py +2 -0
- cognite/neat/_rules/models/entities/_single_value.py +25 -5
- cognite/neat/_rules/models/information/_rules.py +4 -0
- cognite/neat/_rules/models/information/_rules_input.py +9 -0
- cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
- cognite/neat/_rules/transformers/__init__.py +5 -4
- cognite/neat/_rules/transformers/_base.py +41 -65
- cognite/neat/_rules/transformers/_converters.py +149 -62
- cognite/neat/_rules/transformers/_mapping.py +17 -12
- cognite/neat/_rules/transformers/_verification.py +50 -37
- cognite/neat/_session/_base.py +32 -121
- cognite/neat/_session/_inspect.py +3 -3
- cognite/neat/_session/_mapping.py +17 -105
- cognite/neat/_session/_prepare.py +36 -254
- cognite/neat/_session/_read.py +11 -130
- cognite/neat/_session/_set.py +6 -30
- cognite/neat/_session/_show.py +49 -30
- cognite/neat/_session/_state.py +49 -107
- cognite/neat/_session/_to.py +42 -31
- cognite/neat/_shared.py +23 -2
- cognite/neat/_store/_provenance.py +3 -82
- cognite/neat/_store/_rules_store.py +372 -10
- cognite/neat/_store/exceptions.py +23 -0
- cognite/neat/_utils/graph_transformations_report.py +36 -0
- cognite/neat/_utils/io_.py +11 -0
- cognite/neat/_utils/rdf_.py +8 -0
- cognite/neat/_utils/spreadsheet.py +5 -4
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +7 -7
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +24 -99
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/RECORD +63 -61
- cognite/neat/_rules/transformers/_pipelines.py +0 -70
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import copy
|
|
2
1
|
from collections.abc import Callable, Collection
|
|
3
|
-
from datetime import datetime, timezone
|
|
4
2
|
from typing import Any, Literal, cast
|
|
5
3
|
|
|
6
4
|
from cognite.client.data_classes.data_modeling import DataModelIdentifier
|
|
@@ -8,7 +6,6 @@ from rdflib import URIRef
|
|
|
8
6
|
|
|
9
7
|
from cognite.neat._client import NeatClient
|
|
10
8
|
from cognite.neat._constants import (
|
|
11
|
-
DEFAULT_NAMESPACE,
|
|
12
9
|
get_default_prefixes_and_namespaces,
|
|
13
10
|
)
|
|
14
11
|
from cognite.neat._graph.transformers import (
|
|
@@ -22,30 +19,19 @@ from cognite.neat._graph.transformers import (
|
|
|
22
19
|
Transformers,
|
|
23
20
|
)
|
|
24
21
|
from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
|
|
25
|
-
from cognite.neat.
|
|
26
|
-
from cognite.neat._rules.importers import DMSImporter
|
|
27
|
-
from cognite.neat._rules.models import DMSRules
|
|
28
|
-
from cognite.neat._rules.models.dms import DMSValidation
|
|
29
|
-
from cognite.neat._rules.models.entities import ClassEntity
|
|
30
|
-
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
22
|
+
from cognite.neat._issues import IssueList
|
|
31
23
|
from cognite.neat._rules.transformers import (
|
|
24
|
+
AddClassImplements,
|
|
25
|
+
IncludeReferenced,
|
|
32
26
|
PrefixEntities,
|
|
33
27
|
ReduceCogniteModel,
|
|
34
28
|
ToCompliantEntities,
|
|
35
29
|
ToExtension,
|
|
36
|
-
VerifyDMSRules,
|
|
37
30
|
)
|
|
38
|
-
from cognite.neat._store._provenance import Agent as ProvenanceAgent
|
|
39
|
-
from cognite.neat._store._provenance import Change
|
|
40
31
|
|
|
41
32
|
from ._state import SessionState
|
|
42
33
|
from .exceptions import NeatSessionError, session_class_wrapper
|
|
43
34
|
|
|
44
|
-
try:
|
|
45
|
-
from rich import print
|
|
46
|
-
except ImportError:
|
|
47
|
-
...
|
|
48
|
-
|
|
49
35
|
|
|
50
36
|
@session_class_wrapper
|
|
51
37
|
class PrepareAPI:
|
|
@@ -327,53 +313,18 @@ class DataModelPrepareAPI:
|
|
|
327
313
|
self._state = state
|
|
328
314
|
self._verbose = verbose
|
|
329
315
|
|
|
330
|
-
def cdf_compliant_external_ids(self) ->
|
|
316
|
+
def cdf_compliant_external_ids(self) -> IssueList:
|
|
331
317
|
"""Convert data model component external ids to CDF compliant entities."""
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
start = datetime.now(timezone.utc)
|
|
335
|
-
transformer = ToCompliantEntities()
|
|
336
|
-
output: ReadRules[InformationInputRules] = transformer.transform(rules)
|
|
337
|
-
end = datetime.now(timezone.utc)
|
|
338
|
-
|
|
339
|
-
change = Change.from_rules_activity(
|
|
340
|
-
output,
|
|
341
|
-
transformer.agent,
|
|
342
|
-
start,
|
|
343
|
-
end,
|
|
344
|
-
"Converted external ids to CDF compliant entities",
|
|
345
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
346
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
347
|
-
)
|
|
348
|
-
|
|
349
|
-
self._state.data_model.write(output, change)
|
|
318
|
+
return self._state.rule_transform(ToCompliantEntities())
|
|
350
319
|
|
|
351
|
-
def prefix(self, prefix: str) ->
|
|
320
|
+
def prefix(self, prefix: str) -> IssueList:
|
|
352
321
|
"""Prefix all views in the data model with the given prefix.
|
|
353
322
|
|
|
354
323
|
Args:
|
|
355
324
|
prefix: The prefix to add to the views in the data model.
|
|
356
325
|
|
|
357
326
|
"""
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
start = datetime.now(timezone.utc)
|
|
361
|
-
transformer = PrefixEntities(prefix)
|
|
362
|
-
new_rules = cast(InputRules, copy.deepcopy(rules.get_rules()))
|
|
363
|
-
output = transformer.transform(new_rules)
|
|
364
|
-
end = datetime.now(timezone.utc)
|
|
365
|
-
|
|
366
|
-
change = Change.from_rules_activity(
|
|
367
|
-
output,
|
|
368
|
-
transformer.agent,
|
|
369
|
-
start,
|
|
370
|
-
end,
|
|
371
|
-
"Added prefix to the data model views",
|
|
372
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
373
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
374
|
-
)
|
|
375
|
-
|
|
376
|
-
self._state.data_model.write(output, change)
|
|
327
|
+
return self._state.rule_transform(PrefixEntities(prefix))
|
|
377
328
|
|
|
378
329
|
def to_enterprise(
|
|
379
330
|
self,
|
|
@@ -381,7 +332,7 @@ class DataModelPrepareAPI:
|
|
|
381
332
|
org_name: str = "My",
|
|
382
333
|
dummy_property: str = "GUID",
|
|
383
334
|
move_connections: bool = False,
|
|
384
|
-
) ->
|
|
335
|
+
) -> IssueList:
|
|
385
336
|
"""Uses the current data model as a basis to create enterprise data model
|
|
386
337
|
|
|
387
338
|
Args:
|
|
@@ -405,34 +356,15 @@ class DataModelPrepareAPI:
|
|
|
405
356
|
views as the source and target views.
|
|
406
357
|
|
|
407
358
|
"""
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
start = datetime.now(timezone.utc)
|
|
412
|
-
transformer = ToExtension(
|
|
359
|
+
return self._state.rule_transform(
|
|
360
|
+
ToExtension(
|
|
413
361
|
new_model_id=data_model_id,
|
|
414
362
|
org_name=org_name,
|
|
415
363
|
type_="enterprise",
|
|
416
364
|
dummy_property=dummy_property,
|
|
417
365
|
move_connections=move_connections,
|
|
418
366
|
)
|
|
419
|
-
|
|
420
|
-
end = datetime.now(timezone.utc)
|
|
421
|
-
|
|
422
|
-
change = Change.from_rules_activity(
|
|
423
|
-
output,
|
|
424
|
-
transformer.agent,
|
|
425
|
-
start,
|
|
426
|
-
end,
|
|
427
|
-
(
|
|
428
|
-
f"Prepared data model {data_model_id} to be enterprise data "
|
|
429
|
-
f"model on top of {rules.metadata.as_data_model_id()}"
|
|
430
|
-
),
|
|
431
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
432
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
433
|
-
)
|
|
434
|
-
|
|
435
|
-
self._state.data_model.write(output.rules, change)
|
|
367
|
+
)
|
|
436
368
|
|
|
437
369
|
def to_solution(
|
|
438
370
|
self,
|
|
@@ -440,7 +372,7 @@ class DataModelPrepareAPI:
|
|
|
440
372
|
org_name: str = "My",
|
|
441
373
|
mode: Literal["read", "write"] = "read",
|
|
442
374
|
dummy_property: str = "GUID",
|
|
443
|
-
) ->
|
|
375
|
+
) -> IssueList:
|
|
444
376
|
"""Uses the current data model as a basis to create solution data model
|
|
445
377
|
|
|
446
378
|
Args:
|
|
@@ -461,34 +393,15 @@ class DataModelPrepareAPI:
|
|
|
461
393
|
the containers in the solution data model space.
|
|
462
394
|
|
|
463
395
|
"""
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
start = datetime.now(timezone.utc)
|
|
468
|
-
transformer = ToExtension(
|
|
396
|
+
return self._state.rule_transform(
|
|
397
|
+
ToExtension(
|
|
469
398
|
new_model_id=data_model_id,
|
|
470
399
|
org_name=org_name,
|
|
471
400
|
type_="solution",
|
|
472
401
|
mode=mode,
|
|
473
402
|
dummy_property=dummy_property,
|
|
474
403
|
)
|
|
475
|
-
|
|
476
|
-
end = datetime.now(timezone.utc)
|
|
477
|
-
|
|
478
|
-
change = Change.from_rules_activity(
|
|
479
|
-
output,
|
|
480
|
-
transformer.agent,
|
|
481
|
-
start,
|
|
482
|
-
end,
|
|
483
|
-
(
|
|
484
|
-
f"Prepared data model {data_model_id} to be solution data model "
|
|
485
|
-
f"on top of {rules.metadata.as_data_model_id()}"
|
|
486
|
-
),
|
|
487
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
488
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
489
|
-
)
|
|
490
|
-
|
|
491
|
-
self._state.data_model.write(output.rules, change)
|
|
404
|
+
)
|
|
492
405
|
|
|
493
406
|
def to_data_product(
|
|
494
407
|
self,
|
|
@@ -507,65 +420,25 @@ class DataModelPrepareAPI:
|
|
|
507
420
|
include: The views to include in the data product data model. Can be either "same-space" or "all".
|
|
508
421
|
If you set same-space, only the views in the same space as the data model will be included.
|
|
509
422
|
"""
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
importer = DMSImporter(schema)
|
|
524
|
-
reference_rules = importer.to_rules().rules
|
|
525
|
-
if reference_rules is not None:
|
|
526
|
-
imported = VerifyDMSRules("continue").transform(reference_rules)
|
|
527
|
-
if dms_ref := imported.rules:
|
|
528
|
-
rules = rules.model_copy(deep=True)
|
|
529
|
-
if rules.containers is None:
|
|
530
|
-
rules.containers = dms_ref.containers
|
|
531
|
-
else:
|
|
532
|
-
existing_containers = {c.container for c in rules.containers}
|
|
533
|
-
rules.containers.extend(
|
|
534
|
-
[c for c in dms_ref.containers or [] if c.container not in existing_containers]
|
|
535
|
-
)
|
|
536
|
-
existing_views = {v.view for v in rules.views}
|
|
537
|
-
rules.views.extend([v for v in dms_ref.views if v.view not in existing_views])
|
|
538
|
-
existing_properties = {(p.view, p.view_property) for p in rules.properties}
|
|
539
|
-
rules.properties.extend(
|
|
540
|
-
[p for p in dms_ref.properties if (p.view, p.view_property) not in existing_properties]
|
|
541
|
-
)
|
|
542
|
-
|
|
543
|
-
start = datetime.now(timezone.utc)
|
|
544
|
-
transformer = ToExtension(
|
|
545
|
-
new_model_id=data_model_id,
|
|
546
|
-
org_name=org_name,
|
|
547
|
-
type_="data_product",
|
|
548
|
-
include=include,
|
|
549
|
-
)
|
|
550
|
-
output = transformer.transform(rules)
|
|
551
|
-
end = datetime.now(timezone.utc)
|
|
552
|
-
|
|
553
|
-
change = Change.from_rules_activity(
|
|
554
|
-
output,
|
|
555
|
-
transformer.agent,
|
|
556
|
-
start,
|
|
557
|
-
end,
|
|
558
|
-
(
|
|
559
|
-
f"Prepared data model {data_model_id} to be data product model "
|
|
560
|
-
f"on top of {rules.metadata.as_data_model_id()}"
|
|
423
|
+
if self._client is None:
|
|
424
|
+
raise NeatSessionError(
|
|
425
|
+
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
426
|
+
"NEAT needs a client to lookup the definitions. "
|
|
427
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
428
|
+
)
|
|
429
|
+
transformers = [
|
|
430
|
+
IncludeReferenced(self._client, include_properties=True),
|
|
431
|
+
ToExtension(
|
|
432
|
+
new_model_id=data_model_id,
|
|
433
|
+
org_name=org_name,
|
|
434
|
+
type_="data_product",
|
|
435
|
+
include=include,
|
|
561
436
|
),
|
|
562
|
-
|
|
563
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
564
|
-
)
|
|
437
|
+
]
|
|
565
438
|
|
|
566
|
-
self._state.
|
|
439
|
+
self._state.rule_transform(*transformers)
|
|
567
440
|
|
|
568
|
-
def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"] | str]) ->
|
|
441
|
+
def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"] | str]) -> IssueList:
|
|
569
442
|
"""This is a special method that allow you to drop parts of the data model.
|
|
570
443
|
This only applies to Cognite Data Models.
|
|
571
444
|
|
|
@@ -574,90 +447,19 @@ class DataModelPrepareAPI:
|
|
|
574
447
|
drops multiple views at once. You can also pass externalIds of views to drop individual views.
|
|
575
448
|
|
|
576
449
|
"""
|
|
577
|
-
|
|
578
|
-
source_id, rules = input
|
|
579
|
-
start = datetime.now(timezone.utc)
|
|
580
|
-
|
|
581
|
-
transformer = ReduceCogniteModel(drop)
|
|
582
|
-
output = transformer.transform(rules)
|
|
583
|
-
output.rules.metadata.version = f"{rules.metadata.version}.reduced"
|
|
584
|
-
|
|
585
|
-
end = datetime.now(timezone.utc)
|
|
586
|
-
|
|
587
|
-
change = Change.from_rules_activity(
|
|
588
|
-
output,
|
|
589
|
-
transformer.agent,
|
|
590
|
-
start,
|
|
591
|
-
end,
|
|
592
|
-
(
|
|
593
|
-
f"Reduced data model {rules.metadata.as_data_model_id()}"
|
|
594
|
-
f"on top of {rules.metadata.as_data_model_id()}"
|
|
595
|
-
),
|
|
596
|
-
self._state.data_model.provenance.source_entity(source_id),
|
|
597
|
-
)
|
|
450
|
+
return self._state.rule_transform(ReduceCogniteModel(drop))
|
|
598
451
|
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
def include_referenced(self) -> None:
|
|
452
|
+
def include_referenced(self) -> IssueList:
|
|
602
453
|
"""Include referenced views and containers in the data model."""
|
|
603
|
-
start = datetime.now(timezone.utc)
|
|
604
|
-
|
|
605
|
-
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
606
|
-
view_ids, container_ids = DMSValidation(rules, self._client).imported_views_and_containers_ids()
|
|
607
|
-
if not (view_ids or container_ids):
|
|
608
|
-
print(
|
|
609
|
-
f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
|
|
610
|
-
f"that is not already included in the data model."
|
|
611
|
-
)
|
|
612
|
-
return
|
|
613
454
|
if self._client is None:
|
|
614
455
|
raise NeatSessionError(
|
|
615
456
|
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
616
457
|
"NEAT needs a client to lookup the definitions. "
|
|
617
458
|
"Please set the client in the session, NeatSession(client=client)."
|
|
618
459
|
)
|
|
619
|
-
|
|
620
|
-
copy_ = rules.model_copy(deep=True)
|
|
621
|
-
copy_.metadata.version = f"{rules.metadata.version}_completed"
|
|
622
|
-
importer = DMSImporter(schema)
|
|
623
|
-
imported = importer.to_rules()
|
|
624
|
-
if imported.rules is None:
|
|
625
|
-
self._state.data_model.issue_lists.append(imported.issues)
|
|
626
|
-
raise NeatSessionError(
|
|
627
|
-
"Could not import the referenced views and containers. "
|
|
628
|
-
"See `neat.inspect.issues()` for more information."
|
|
629
|
-
)
|
|
630
|
-
verified = VerifyDMSRules("continue", validate=False).transform(imported.rules)
|
|
631
|
-
if verified.rules is None:
|
|
632
|
-
self._state.data_model.issue_lists.append(verified.issues)
|
|
633
|
-
raise NeatSessionError(
|
|
634
|
-
"Could not verify the referenced views and containers. "
|
|
635
|
-
"See `neat.inspect.issues()` for more information."
|
|
636
|
-
)
|
|
637
|
-
if copy_.containers is None:
|
|
638
|
-
copy_.containers = verified.rules.containers
|
|
639
|
-
else:
|
|
640
|
-
existing_containers = {c.container for c in copy_.containers}
|
|
641
|
-
copy_.containers.extend(
|
|
642
|
-
[c for c in verified.rules.containers or [] if c.container not in existing_containers]
|
|
643
|
-
)
|
|
644
|
-
existing_views = {v.view for v in copy_.views}
|
|
645
|
-
copy_.views.extend([v for v in verified.rules.views if v.view not in existing_views])
|
|
646
|
-
end = datetime.now(timezone.utc)
|
|
647
|
-
|
|
648
|
-
change = Change.from_rules_activity(
|
|
649
|
-
copy_,
|
|
650
|
-
ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
|
|
651
|
-
start,
|
|
652
|
-
end,
|
|
653
|
-
(f"Included referenced views and containers in the data model {rules.metadata.as_data_model_id()}"),
|
|
654
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
655
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
656
|
-
)
|
|
657
|
-
|
|
658
|
-
self._state.data_model.write(copy_, change)
|
|
460
|
+
return self._state.rule_transform(IncludeReferenced(self._client))
|
|
659
461
|
|
|
660
|
-
def add_implements_to_classes(self, suffix: Literal["Edge"], implements: str = "Edge") ->
|
|
462
|
+
def add_implements_to_classes(self, suffix: Literal["Edge"], implements: str = "Edge") -> IssueList:
|
|
661
463
|
"""All classes with the suffix will have the implements property set to the given value.
|
|
662
464
|
|
|
663
465
|
Args:
|
|
@@ -665,24 +467,4 @@ class DataModelPrepareAPI:
|
|
|
665
467
|
implements: The value of the implements property to set.
|
|
666
468
|
|
|
667
469
|
"""
|
|
668
|
-
|
|
669
|
-
start = datetime.now(timezone.utc)
|
|
670
|
-
|
|
671
|
-
output = rules.model_copy(deep=True)
|
|
672
|
-
for class_ in output.classes:
|
|
673
|
-
if class_.class_.suffix.endswith(suffix):
|
|
674
|
-
class_.implements = [ClassEntity(prefix=class_.class_.prefix, suffix=implements)]
|
|
675
|
-
output.metadata.version = f"{rules.metadata.version}.implements_{implements}"
|
|
676
|
-
end = datetime.now(timezone.utc)
|
|
677
|
-
|
|
678
|
-
change = Change.from_rules_activity(
|
|
679
|
-
output,
|
|
680
|
-
ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
|
|
681
|
-
start,
|
|
682
|
-
end,
|
|
683
|
-
(f"Added implements property to classes with suffix {suffix}"),
|
|
684
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
685
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
686
|
-
)
|
|
687
|
-
|
|
688
|
-
self._state.data_model.write(output, change)
|
|
470
|
+
return self._state.rule_transform(AddClassImplements(implements, suffix))
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import tempfile
|
|
2
|
-
from datetime import datetime, timezone
|
|
3
2
|
from pathlib import Path
|
|
4
3
|
from typing import Any, Literal, cast
|
|
5
4
|
|
|
@@ -12,11 +11,7 @@ from cognite.neat._graph import extractors
|
|
|
12
11
|
from cognite.neat._issues import IssueList
|
|
13
12
|
from cognite.neat._issues.errors import NeatValueError
|
|
14
13
|
from cognite.neat._rules import catalog, importers
|
|
15
|
-
from cognite.neat._rules._shared import ReadRules
|
|
16
14
|
from cognite.neat._rules.importers import BaseImporter
|
|
17
|
-
from cognite.neat._store._provenance import Activity as ProvenanceActivity
|
|
18
|
-
from cognite.neat._store._provenance import Change
|
|
19
|
-
from cognite.neat._store._provenance import Entity as ProvenanceEntity
|
|
20
15
|
from cognite.neat._utils.reader import GitHubReader, HttpFileReader, NeatReader, PathReader
|
|
21
16
|
|
|
22
17
|
from ._state import SessionState
|
|
@@ -47,26 +42,6 @@ class BaseReadAPI:
|
|
|
47
42
|
self._verbose = verbose
|
|
48
43
|
self._client = client
|
|
49
44
|
|
|
50
|
-
def _store_rules(self, rules: ReadRules, change: Change) -> IssueList:
|
|
51
|
-
if self._verbose:
|
|
52
|
-
if rules.issues.has_errors:
|
|
53
|
-
print("Data model read failed")
|
|
54
|
-
else:
|
|
55
|
-
print("Data model read passed")
|
|
56
|
-
|
|
57
|
-
if rules.rules:
|
|
58
|
-
self._state.data_model.write(rules, change)
|
|
59
|
-
|
|
60
|
-
return rules.issues
|
|
61
|
-
|
|
62
|
-
def _return_filepath(self, io: Any) -> Path:
|
|
63
|
-
if isinstance(io, str):
|
|
64
|
-
return Path(io)
|
|
65
|
-
elif isinstance(io, Path):
|
|
66
|
-
return io
|
|
67
|
-
else:
|
|
68
|
-
raise NeatValueError(f"Expected str or Path, got {type(io)}")
|
|
69
|
-
|
|
70
45
|
|
|
71
46
|
@session_class_wrapper
|
|
72
47
|
class CDFReadAPI(BaseReadAPI):
|
|
@@ -103,31 +78,8 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
103
78
|
if not data_model_id.version:
|
|
104
79
|
raise NeatSessionError("Data model version is required to read a data model.")
|
|
105
80
|
|
|
106
|
-
# actual reading of data model
|
|
107
|
-
start = datetime.now(timezone.utc)
|
|
108
81
|
importer = importers.DMSImporter.from_data_model_id(self._get_client, data_model_id)
|
|
109
|
-
|
|
110
|
-
end = datetime.now(timezone.utc)
|
|
111
|
-
|
|
112
|
-
# provenance information
|
|
113
|
-
source_entity = ProvenanceEntity.from_data_model_id(data_model_id)
|
|
114
|
-
agent = importer.agent
|
|
115
|
-
activity = ProvenanceActivity(
|
|
116
|
-
was_associated_with=agent,
|
|
117
|
-
ended_at_time=end,
|
|
118
|
-
used=source_entity,
|
|
119
|
-
started_at_time=start,
|
|
120
|
-
)
|
|
121
|
-
target_entity = ProvenanceEntity.from_rules(rules, agent, activity)
|
|
122
|
-
change = Change(
|
|
123
|
-
source_entity=source_entity,
|
|
124
|
-
agent=agent,
|
|
125
|
-
activity=activity,
|
|
126
|
-
target_entity=target_entity,
|
|
127
|
-
description=f"DMS Data model {data_model_id.as_tuple()} read as unverified data model",
|
|
128
|
-
)
|
|
129
|
-
|
|
130
|
-
return self._store_rules(rules, change)
|
|
82
|
+
return self._state.rule_import(importer)
|
|
131
83
|
|
|
132
84
|
|
|
133
85
|
@session_class_wrapper
|
|
@@ -215,24 +167,9 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
215
167
|
io: file path to the Excel sheet
|
|
216
168
|
"""
|
|
217
169
|
reader = NeatReader.create(io)
|
|
218
|
-
start = datetime.now(timezone.utc)
|
|
219
170
|
if not isinstance(reader, PathReader):
|
|
220
171
|
raise NeatValueError("Only file paths are supported for Excel files")
|
|
221
|
-
|
|
222
|
-
input_rules: ReadRules = importer.to_rules()
|
|
223
|
-
end = datetime.now(timezone.utc)
|
|
224
|
-
|
|
225
|
-
if input_rules.rules:
|
|
226
|
-
change = Change.from_rules_activity(
|
|
227
|
-
input_rules,
|
|
228
|
-
importer.agent,
|
|
229
|
-
start,
|
|
230
|
-
end,
|
|
231
|
-
description=f"Excel file {reader!s} read as unverified data model",
|
|
232
|
-
)
|
|
233
|
-
self._store_rules(input_rules, change)
|
|
234
|
-
self._state.data_model.issue_lists.append(input_rules.issues)
|
|
235
|
-
return input_rules.issues
|
|
172
|
+
return self._state.rule_import(importers.ExcelImporter(reader.path))
|
|
236
173
|
|
|
237
174
|
|
|
238
175
|
@session_class_wrapper
|
|
@@ -241,23 +178,9 @@ class ExcelExampleAPI(BaseReadAPI):
|
|
|
241
178
|
|
|
242
179
|
@property
|
|
243
180
|
def pump_example(self) -> IssueList:
|
|
244
|
-
"""Reads the
|
|
245
|
-
start = datetime.now(timezone.utc)
|
|
181
|
+
"""Reads the Hello World pump example into the NeatSession."""
|
|
246
182
|
importer: importers.ExcelImporter = importers.ExcelImporter(catalog.hello_world_pump)
|
|
247
|
-
|
|
248
|
-
end = datetime.now(timezone.utc)
|
|
249
|
-
|
|
250
|
-
if input_rules.rules:
|
|
251
|
-
change = Change.from_rules_activity(
|
|
252
|
-
input_rules,
|
|
253
|
-
importer.agent,
|
|
254
|
-
start,
|
|
255
|
-
end,
|
|
256
|
-
description="Pump Example read as unverified data model",
|
|
257
|
-
)
|
|
258
|
-
self._store_rules(input_rules, change)
|
|
259
|
-
self._state.data_model.issue_lists.append(input_rules.issues)
|
|
260
|
-
return input_rules.issues
|
|
183
|
+
return self._state.rule_import(importer)
|
|
261
184
|
|
|
262
185
|
|
|
263
186
|
@session_class_wrapper
|
|
@@ -278,10 +201,9 @@ class YamlReadAPI(BaseReadAPI):
|
|
|
278
201
|
reader = NeatReader.create(io)
|
|
279
202
|
if not isinstance(reader, PathReader):
|
|
280
203
|
raise NeatValueError("Only file paths are supported for YAML files")
|
|
281
|
-
start = datetime.now(timezone.utc)
|
|
282
204
|
importer: BaseImporter
|
|
283
205
|
if format == "neat":
|
|
284
|
-
importer = importers.YAMLImporter.from_file(reader.path)
|
|
206
|
+
importer = importers.YAMLImporter.from_file(reader.path, source_name=f"{reader!s}")
|
|
285
207
|
elif format == "toolkit":
|
|
286
208
|
if reader.path.is_file():
|
|
287
209
|
dms_importer = importers.DMSImporter.from_zip_file(reader.path)
|
|
@@ -289,6 +211,7 @@ class YamlReadAPI(BaseReadAPI):
|
|
|
289
211
|
dms_importer = importers.DMSImporter.from_directory(reader.path)
|
|
290
212
|
else:
|
|
291
213
|
raise NeatValueError(f"Unsupported YAML format: {format}")
|
|
214
|
+
|
|
292
215
|
ref_containers = dms_importer.root_schema.referenced_container()
|
|
293
216
|
if system_container_ids := [
|
|
294
217
|
container_id for container_id in ref_containers if container_id.space in COGNITE_SPACES
|
|
@@ -305,21 +228,7 @@ class YamlReadAPI(BaseReadAPI):
|
|
|
305
228
|
importer = dms_importer
|
|
306
229
|
else:
|
|
307
230
|
raise NeatValueError(f"Unsupported YAML format: {format}")
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
end = datetime.now(timezone.utc)
|
|
311
|
-
|
|
312
|
-
if input_rules.rules:
|
|
313
|
-
change = Change.from_rules_activity(
|
|
314
|
-
input_rules,
|
|
315
|
-
importer.agent,
|
|
316
|
-
start,
|
|
317
|
-
end,
|
|
318
|
-
description=f"YAML file {reader!s} read as unverified data model",
|
|
319
|
-
)
|
|
320
|
-
self._store_rules(input_rules, change)
|
|
321
|
-
|
|
322
|
-
return input_rules.issues
|
|
231
|
+
return self._state.rule_import(importer)
|
|
323
232
|
|
|
324
233
|
|
|
325
234
|
@session_class_wrapper
|
|
@@ -452,25 +361,11 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
452
361
|
neat.read.rdf.ontology("url_or_path_to_owl_source")
|
|
453
362
|
```
|
|
454
363
|
"""
|
|
455
|
-
start = datetime.now(timezone.utc)
|
|
456
364
|
reader = NeatReader.create(io)
|
|
457
365
|
if not isinstance(reader, PathReader):
|
|
458
366
|
raise NeatValueError("Only file paths are supported for RDF files")
|
|
459
|
-
importer = importers.OWLImporter.from_file(reader.path)
|
|
460
|
-
|
|
461
|
-
end = datetime.now(timezone.utc)
|
|
462
|
-
|
|
463
|
-
if input_rules.rules:
|
|
464
|
-
change = Change.from_rules_activity(
|
|
465
|
-
input_rules,
|
|
466
|
-
importer.agent,
|
|
467
|
-
start,
|
|
468
|
-
end,
|
|
469
|
-
description=f"Ontology file {reader!s} read as unverified data model",
|
|
470
|
-
)
|
|
471
|
-
self._store_rules(input_rules, change)
|
|
472
|
-
|
|
473
|
-
return input_rules.issues
|
|
367
|
+
importer = importers.OWLImporter.from_file(reader.path, source_name=f"file {reader!s}")
|
|
368
|
+
return self._state.rule_import(importer)
|
|
474
369
|
|
|
475
370
|
def imf(self, io: Any) -> IssueList:
|
|
476
371
|
"""Reads IMF Types provided as SHACL shapes into NeatSession.
|
|
@@ -483,25 +378,11 @@ class RDFReadAPI(BaseReadAPI):
|
|
|
483
378
|
neat.read.rdf.imf("url_or_path_to_imf_source")
|
|
484
379
|
```
|
|
485
380
|
"""
|
|
486
|
-
start = datetime.now(timezone.utc)
|
|
487
381
|
reader = NeatReader.create(io)
|
|
488
382
|
if not isinstance(reader, PathReader):
|
|
489
383
|
raise NeatValueError("Only file paths are supported for RDF files")
|
|
490
|
-
importer = importers.IMFImporter.from_file(reader.path)
|
|
491
|
-
|
|
492
|
-
end = datetime.now(timezone.utc)
|
|
493
|
-
|
|
494
|
-
if input_rules.rules:
|
|
495
|
-
change = Change.from_rules_activity(
|
|
496
|
-
input_rules,
|
|
497
|
-
importer.agent,
|
|
498
|
-
start,
|
|
499
|
-
end,
|
|
500
|
-
description=f"IMF Types file {reader!s} read as unverified data model",
|
|
501
|
-
)
|
|
502
|
-
self._store_rules(input_rules, change)
|
|
503
|
-
|
|
504
|
-
return input_rules.issues
|
|
384
|
+
importer = importers.IMFImporter.from_file(reader.path, source_name=f"file {reader!s}")
|
|
385
|
+
return self._state.rule_import(importer)
|
|
505
386
|
|
|
506
387
|
def __call__(
|
|
507
388
|
self,
|
cognite/neat/_session/_set.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
from datetime import datetime, timezone
|
|
2
|
-
|
|
3
1
|
from cognite.client import data_modeling as dm
|
|
4
2
|
|
|
5
3
|
from cognite.neat._constants import COGNITE_MODELS
|
|
4
|
+
from cognite.neat._issues import IssueList
|
|
5
|
+
from cognite.neat._rules.models import DMSRules
|
|
6
6
|
from cognite.neat._rules.transformers import SetIDDMSModel
|
|
7
|
-
from cognite.neat._store._provenance import Change
|
|
8
7
|
|
|
9
8
|
from ._state import SessionState
|
|
10
9
|
from .exceptions import NeatSessionError, session_class_wrapper
|
|
@@ -18,7 +17,7 @@ class SetAPI:
|
|
|
18
17
|
self._state = state
|
|
19
18
|
self._verbose = verbose
|
|
20
19
|
|
|
21
|
-
def data_model_id(self, new_model_id: dm.DataModelId | tuple[str, str, str]) ->
|
|
20
|
+
def data_model_id(self, new_model_id: dm.DataModelId | tuple[str, str, str]) -> IssueList:
|
|
22
21
|
"""Sets the data model ID of the latest verified data model. Set the data model id as a tuple of strings
|
|
23
22
|
following the template (<data_model_space>, <data_model_name>, <data_model_version>).
|
|
24
23
|
|
|
@@ -28,34 +27,11 @@ class SetAPI:
|
|
|
28
27
|
neat.set.data_model_id(("my_data_model_space", "My_Data_Model", "v1"))
|
|
29
28
|
```
|
|
30
29
|
"""
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
rules = self._state.rule_store.get_last_successful_entity().result
|
|
31
|
+
if isinstance(rules, DMSRules):
|
|
34
32
|
if rules.metadata.as_data_model_id() in COGNITE_MODELS:
|
|
35
33
|
raise NeatSessionError(
|
|
36
34
|
"Cannot change the data model ID of a Cognite Data Model in NeatSession"
|
|
37
35
|
" due to temporarily issue with the reverse direct relation interpretation"
|
|
38
36
|
)
|
|
39
|
-
|
|
40
|
-
start = datetime.now(timezone.utc)
|
|
41
|
-
transformer = SetIDDMSModel(new_model_id)
|
|
42
|
-
|
|
43
|
-
output = transformer.transform(rules)
|
|
44
|
-
end = datetime.now(timezone.utc)
|
|
45
|
-
|
|
46
|
-
# Provenance
|
|
47
|
-
change = Change.from_rules_activity(
|
|
48
|
-
output.rules,
|
|
49
|
-
transformer.agent,
|
|
50
|
-
start,
|
|
51
|
-
end,
|
|
52
|
-
"Changed data model id",
|
|
53
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
54
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
self._state.data_model.write(output.rules, change)
|
|
58
|
-
if self._verbose:
|
|
59
|
-
print(f"Data model ID set to {new_model_id}")
|
|
60
|
-
else:
|
|
61
|
-
print("No verified DMS data model available")
|
|
37
|
+
return self._state.rule_transform(SetIDDMSModel(new_model_id))
|