cognite-neat 0.88.2__py3-none-any.whl → 0.89.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/constants.py +3 -0
- cognite/neat/graph/__init__.py +0 -3
- cognite/neat/graph/extractors/_mock_graph_generator.py +2 -1
- cognite/neat/graph/loaders/_base.py +3 -3
- cognite/neat/graph/loaders/_rdf2asset.py +24 -25
- cognite/neat/graph/loaders/_rdf2dms.py +20 -15
- cognite/neat/issues/__init__.py +1 -3
- cognite/neat/issues/_base.py +261 -71
- cognite/neat/issues/errors/__init__.py +73 -0
- cognite/neat/issues/errors/_external.py +67 -0
- cognite/neat/issues/errors/_general.py +35 -0
- cognite/neat/issues/errors/_properties.py +62 -0
- cognite/neat/issues/errors/_resources.py +111 -0
- cognite/neat/issues/errors/_workflow.py +36 -0
- cognite/neat/issues/formatters.py +1 -1
- cognite/neat/issues/warnings/__init__.py +66 -0
- cognite/neat/issues/warnings/_external.py +40 -0
- cognite/neat/issues/warnings/_general.py +29 -0
- cognite/neat/issues/warnings/_models.py +92 -0
- cognite/neat/issues/warnings/_properties.py +44 -0
- cognite/neat/issues/warnings/_resources.py +55 -0
- cognite/neat/issues/warnings/user_modeling.py +113 -0
- cognite/neat/rules/_shared.py +53 -2
- cognite/neat/rules/analysis/_base.py +1 -1
- cognite/neat/rules/exporters/_base.py +7 -18
- cognite/neat/rules/exporters/_rules2dms.py +17 -20
- cognite/neat/rules/exporters/_rules2excel.py +9 -16
- cognite/neat/rules/exporters/_rules2ontology.py +77 -64
- cognite/neat/rules/exporters/_rules2yaml.py +6 -9
- cognite/neat/rules/exporters/_validation.py +11 -96
- cognite/neat/rules/importers/_base.py +9 -58
- cognite/neat/rules/importers/_dms2rules.py +188 -135
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +48 -35
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +36 -45
- cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +8 -4
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +3 -3
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +18 -11
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2rules.py +12 -19
- cognite/neat/rules/importers/_rdf/_inference2rules.py +14 -37
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +9 -20
- cognite/neat/rules/importers/_rdf/_shared.py +4 -4
- cognite/neat/rules/importers/_spreadsheet2rules.py +46 -97
- cognite/neat/rules/importers/_yaml2rules.py +32 -58
- cognite/neat/rules/models/__init__.py +21 -5
- cognite/neat/rules/models/_base_input.py +162 -0
- cognite/neat/rules/models/{_base.py → _base_rules.py} +1 -12
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/{_types/_field.py → _types.py} +5 -10
- cognite/neat/rules/models/asset/__init__.py +5 -2
- cognite/neat/rules/models/asset/_rules.py +3 -23
- cognite/neat/rules/models/asset/_rules_input.py +40 -115
- cognite/neat/rules/models/asset/_validation.py +14 -10
- cognite/neat/rules/models/data_types.py +150 -44
- cognite/neat/rules/models/dms/__init__.py +19 -7
- cognite/neat/rules/models/dms/_exporter.py +102 -34
- cognite/neat/rules/models/dms/_rules.py +65 -162
- cognite/neat/rules/models/dms/_rules_input.py +186 -254
- cognite/neat/rules/models/dms/_schema.py +87 -78
- cognite/neat/rules/models/dms/_serializer.py +44 -3
- cognite/neat/rules/models/dms/_validation.py +106 -68
- cognite/neat/rules/models/domain.py +52 -1
- cognite/neat/rules/models/entities/__init__.py +63 -0
- cognite/neat/rules/models/entities/_constants.py +73 -0
- cognite/neat/rules/models/entities/_loaders.py +76 -0
- cognite/neat/rules/models/entities/_multi_value.py +67 -0
- cognite/neat/rules/models/{entities.py → entities/_single_value.py} +74 -232
- cognite/neat/rules/models/entities/_types.py +86 -0
- cognite/neat/rules/models/{wrapped_entities.py → entities/_wrapped.py} +1 -1
- cognite/neat/rules/models/information/__init__.py +10 -2
- cognite/neat/rules/models/information/_rules.py +10 -22
- cognite/neat/rules/models/information/_rules_input.py +57 -204
- cognite/neat/rules/models/information/_validation.py +48 -25
- cognite/neat/rules/transformers/__init__.py +21 -0
- cognite/neat/rules/transformers/_base.py +81 -0
- cognite/neat/rules/{models/information/_converter.py → transformers/_converters.py} +217 -21
- cognite/neat/rules/transformers/_map_onto.py +97 -0
- cognite/neat/rules/transformers/_pipelines.py +61 -0
- cognite/neat/rules/transformers/_verification.py +136 -0
- cognite/neat/{graph/stores → store}/_provenance.py +10 -1
- cognite/neat/utils/auxiliary.py +2 -35
- cognite/neat/utils/cdf/data_classes.py +20 -0
- cognite/neat/utils/regex_patterns.py +6 -0
- cognite/neat/utils/text.py +17 -0
- cognite/neat/workflows/base.py +4 -4
- cognite/neat/workflows/cdf_store.py +3 -3
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
- cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
- cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +116 -47
- cognite/neat/workflows/steps/lib/current/rules_importer.py +30 -28
- cognite/neat/workflows/steps/lib/current/rules_validator.py +5 -6
- cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
- cognite/neat/workflows/steps_registry.py +4 -5
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/RECORD +105 -106
- cognite/neat/exceptions.py +0 -145
- cognite/neat/graph/exceptions.py +0 -90
- cognite/neat/issues/errors/external.py +0 -21
- cognite/neat/issues/errors/properties.py +0 -75
- cognite/neat/issues/errors/resources.py +0 -123
- cognite/neat/issues/errors/schema.py +0 -0
- cognite/neat/issues/neat_warnings/__init__.py +0 -2
- cognite/neat/issues/neat_warnings/identifier.py +0 -27
- cognite/neat/issues/neat_warnings/models.py +0 -22
- cognite/neat/issues/neat_warnings/properties.py +0 -77
- cognite/neat/issues/neat_warnings/resources.py +0 -125
- cognite/neat/rules/issues/__init__.py +0 -22
- cognite/neat/rules/issues/base.py +0 -63
- cognite/neat/rules/issues/dms.py +0 -549
- cognite/neat/rules/issues/fileread.py +0 -197
- cognite/neat/rules/issues/ontology.py +0 -298
- cognite/neat/rules/issues/spreadsheet.py +0 -563
- cognite/neat/rules/issues/spreadsheet_file.py +0 -151
- cognite/neat/rules/issues/tables.py +0 -72
- cognite/neat/rules/models/_constants.py +0 -1
- cognite/neat/rules/models/_types/__init__.py +0 -19
- cognite/neat/rules/models/asset/_converter.py +0 -4
- cognite/neat/rules/models/dms/_converter.py +0 -145
- cognite/neat/workflows/_exceptions.py +0 -41
- /cognite/neat/{graph/stores → store}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/_base.py +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.2.dist-info → cognite_neat-0.89.0.dist-info}/entry_points.txt +0 -0
|
@@ -268,6 +268,10 @@ class ViewApplyDict(CogniteResourceDict[ViewId, ViewApply]):
|
|
|
268
268
|
def _as_id(cls, resource: ViewApply) -> ViewId:
|
|
269
269
|
return resource.as_id()
|
|
270
270
|
|
|
271
|
+
@classmethod
|
|
272
|
+
def from_iterable(cls, iterable: Iterable[ViewApply]) -> "ViewApplyDict":
|
|
273
|
+
return cls({view.as_id(): view for view in iterable})
|
|
274
|
+
|
|
271
275
|
|
|
272
276
|
class SpaceApplyDict(CogniteResourceDict[str, SpaceApply]):
|
|
273
277
|
_RESOURCE = SpaceApply
|
|
@@ -276,6 +280,10 @@ class SpaceApplyDict(CogniteResourceDict[str, SpaceApply]):
|
|
|
276
280
|
def _as_id(cls, resource: SpaceApply) -> str:
|
|
277
281
|
return resource.space
|
|
278
282
|
|
|
283
|
+
@classmethod
|
|
284
|
+
def from_iterable(cls, iterable: Iterable[SpaceApply]) -> "SpaceApplyDict":
|
|
285
|
+
return cls({space.space: space for space in iterable})
|
|
286
|
+
|
|
279
287
|
|
|
280
288
|
class ContainerApplyDict(CogniteResourceDict[ContainerId, ContainerApply]):
|
|
281
289
|
_RESOURCE = ContainerApply
|
|
@@ -284,6 +292,10 @@ class ContainerApplyDict(CogniteResourceDict[ContainerId, ContainerApply]):
|
|
|
284
292
|
def _as_id(cls, resource: ContainerApply) -> ContainerId:
|
|
285
293
|
return resource.as_id()
|
|
286
294
|
|
|
295
|
+
@classmethod
|
|
296
|
+
def from_iterable(cls, iterable: Iterable[ContainerApply]) -> "ContainerApplyDict":
|
|
297
|
+
return cls({container.as_id(): container for container in iterable})
|
|
298
|
+
|
|
287
299
|
|
|
288
300
|
class DataModelApplyDict(CogniteResourceDict[DataModelId, DataModelApply]):
|
|
289
301
|
_RESOURCE = DataModelApply
|
|
@@ -292,6 +304,10 @@ class DataModelApplyDict(CogniteResourceDict[DataModelId, DataModelApply]):
|
|
|
292
304
|
def _as_id(cls, resource: DataModelApply) -> DataModelId:
|
|
293
305
|
return resource.as_id()
|
|
294
306
|
|
|
307
|
+
@classmethod
|
|
308
|
+
def from_iterable(cls, iterable: Iterable[DataModelApply]) -> "DataModelApplyDict":
|
|
309
|
+
return cls({data_model.as_id(): data_model for data_model in iterable})
|
|
310
|
+
|
|
295
311
|
|
|
296
312
|
class NodeApplyDict(CogniteResourceDict[NodeId, NodeApply]):
|
|
297
313
|
_RESOURCE = NodeApply
|
|
@@ -299,3 +315,7 @@ class NodeApplyDict(CogniteResourceDict[NodeId, NodeApply]):
|
|
|
299
315
|
@classmethod
|
|
300
316
|
def _as_id(cls, resource: NodeApply) -> NodeId:
|
|
301
317
|
return resource.as_id()
|
|
318
|
+
|
|
319
|
+
@classmethod
|
|
320
|
+
def from_iterable(cls, iterable: Iterable[NodeApply]) -> "NodeApplyDict":
|
|
321
|
+
return cls({node.as_id(): node for node in iterable})
|
|
@@ -19,6 +19,12 @@ PROPERTY_ID_COMPLIANCE_REGEX = r"^(\*)|(?!^(Property|property)$)(^[a-zA-Z][a-zA-
|
|
|
19
19
|
VERSION_COMPLIANCE_REGEX = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
# This pattern ignores commas inside brackets
|
|
23
|
+
SPLIT_ON_COMMA_PATTERN = re.compile(r",(?![^(]*\))")
|
|
24
|
+
# This pattern ignores equal signs inside brackets
|
|
25
|
+
SPLIT_ON_EQUAL_PATTERN = re.compile(r"=(?![^(]*\))")
|
|
26
|
+
|
|
27
|
+
|
|
22
28
|
class _Patterns:
|
|
23
29
|
@cached_property
|
|
24
30
|
def more_than_one_alphanumeric(self) -> re.Pattern:
|
cognite/neat/utils/text.py
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import re
|
|
2
|
+
from collections.abc import Collection
|
|
3
|
+
from typing import Any
|
|
2
4
|
|
|
3
5
|
|
|
4
6
|
def to_camel(string: str) -> str:
|
|
@@ -106,3 +108,18 @@ def to_snake(string: str) -> str:
|
|
|
106
108
|
|
|
107
109
|
def replace_non_alphanumeric_with_underscore(text: str) -> str:
|
|
108
110
|
return re.sub(r"\W+", "_", text)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def humanize_collection(collection: Collection[Any], /, *, sort: bool = True) -> str:
|
|
114
|
+
if not collection:
|
|
115
|
+
return ""
|
|
116
|
+
elif len(collection) == 1:
|
|
117
|
+
return str(next(iter(collection)))
|
|
118
|
+
|
|
119
|
+
strings = (str(item) for item in collection)
|
|
120
|
+
if sort:
|
|
121
|
+
sequence = sorted(strings)
|
|
122
|
+
else:
|
|
123
|
+
sequence = list(strings)
|
|
124
|
+
|
|
125
|
+
return f"{', '.join(sequence[:-1])} and {sequence[-1]}"
|
cognite/neat/workflows/base.py
CHANGED
|
@@ -13,9 +13,9 @@ from prometheus_client import Gauge
|
|
|
13
13
|
|
|
14
14
|
from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
|
|
15
15
|
from cognite.neat.config import Config
|
|
16
|
+
from cognite.neat.issues.errors import WorkflowConfigurationNotSetError, WorkflowStepOutputError
|
|
16
17
|
from cognite.neat.utils.auxiliary import retry_decorator
|
|
17
18
|
from cognite.neat.workflows import cdf_store, utils
|
|
18
|
-
from cognite.neat.workflows._exceptions import ConfigurationNotSet, InvalidStepOutputException
|
|
19
19
|
from cognite.neat.workflows.cdf_store import CdfStore
|
|
20
20
|
from cognite.neat.workflows.model import (
|
|
21
21
|
FlowMessage,
|
|
@@ -316,7 +316,7 @@ class BaseWorkflow:
|
|
|
316
316
|
elif isinstance(out_obj, DataContract):
|
|
317
317
|
self.data[type(out_obj).__name__] = out_obj
|
|
318
318
|
else:
|
|
319
|
-
raise
|
|
319
|
+
raise WorkflowStepOutputError(step_type=type(out_obj).__name__)
|
|
320
320
|
|
|
321
321
|
elif step.stype == StepType.START_WORKFLOW_TASK_STEP:
|
|
322
322
|
if self.task_builder:
|
|
@@ -510,7 +510,7 @@ class BaseWorkflow:
|
|
|
510
510
|
if storage_type == "transformation_rules":
|
|
511
511
|
self.rules_storage_path = Path(storage_path)
|
|
512
512
|
if self.default_dataset_id is None:
|
|
513
|
-
raise
|
|
513
|
+
raise WorkflowConfigurationNotSetError("default_dataset_id")
|
|
514
514
|
self.cdf_store = cdf_store.CdfStore(
|
|
515
515
|
self.cdf_client, data_set_id=self.default_dataset_id, rules_storage_path=self.rules_storage_path
|
|
516
516
|
)
|
|
@@ -576,7 +576,7 @@ class BaseWorkflow:
|
|
|
576
576
|
|
|
577
577
|
file_list: list[Path] = []
|
|
578
578
|
if self.data_store_path is None:
|
|
579
|
-
raise
|
|
579
|
+
raise WorkflowConfigurationNotSetError("data_store_path")
|
|
580
580
|
workflow_data_path = Path(self.data_store_path) / "workflows" / self.name
|
|
581
581
|
try:
|
|
582
582
|
for root, _dirs, files in os.walk(workflow_data_path):
|
|
@@ -11,7 +11,7 @@ from cognite.client.data_classes import Event, FileMetadataUpdate
|
|
|
11
11
|
from fastapi.encoders import jsonable_encoder
|
|
12
12
|
from pydantic import BaseModel
|
|
13
13
|
|
|
14
|
-
from cognite.neat.
|
|
14
|
+
from cognite.neat.issues.errors import WorkflowConfigurationNotSetError
|
|
15
15
|
from cognite.neat.workflows.model import WorkflowFullStateReport, WorkflowState, WorkflowStepEvent
|
|
16
16
|
from cognite.neat.workflows.utils import get_file_hash
|
|
17
17
|
|
|
@@ -53,7 +53,7 @@ class CdfStore:
|
|
|
53
53
|
def package_workflow(self, workflow_name: str) -> str:
|
|
54
54
|
"""Creates a zip archive from a folder"""
|
|
55
55
|
if self.workflows_storage_path is None:
|
|
56
|
-
raise
|
|
56
|
+
raise WorkflowConfigurationNotSetError("workflows_storage_path")
|
|
57
57
|
folder_path = self.workflows_storage_path / workflow_name
|
|
58
58
|
archive_path = self.workflows_storage_path / f"{workflow_name}.zip"
|
|
59
59
|
# Make sure the folder exists
|
|
@@ -77,7 +77,7 @@ class CdfStore:
|
|
|
77
77
|
# Make sure the archive exists
|
|
78
78
|
workflow_name = workflow_name.replace(".zip", "")
|
|
79
79
|
if self.workflows_storage_path is None:
|
|
80
|
-
raise
|
|
80
|
+
raise WorkflowConfigurationNotSetError("workflows_storage_path")
|
|
81
81
|
package_full_path = Path(self.workflows_storage_path) / f"{workflow_name}.zip"
|
|
82
82
|
output_folder = Path(self.workflows_storage_path) / workflow_name
|
|
83
83
|
if not package_full_path.is_file():
|
|
@@ -9,13 +9,13 @@ from cognite.client.data_classes import (
|
|
|
9
9
|
)
|
|
10
10
|
from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
|
|
11
11
|
|
|
12
|
-
from cognite.neat.graph.stores import NeatGraphStore
|
|
13
12
|
from cognite.neat.rules.models import (
|
|
14
13
|
AssetRules,
|
|
15
14
|
DMSRules,
|
|
16
15
|
DomainRules,
|
|
17
16
|
InformationRules,
|
|
18
17
|
)
|
|
18
|
+
from cognite.neat.store import NeatGraphStore
|
|
19
19
|
from cognite.neat.workflows.steps.step_model import DataContract
|
|
20
20
|
|
|
21
21
|
|
|
@@ -8,8 +8,8 @@ from rdflib import URIRef
|
|
|
8
8
|
from cognite.neat.constants import DEFAULT_NAMESPACE
|
|
9
9
|
from cognite.neat.graph.extractors import RdfFileExtractor
|
|
10
10
|
from cognite.neat.graph.extractors._mock_graph_generator import MockGraphGenerator
|
|
11
|
+
from cognite.neat.issues.errors import WorkflowStepNotInitializedError
|
|
11
12
|
from cognite.neat.rules._shared import DMSRules, InformationRules
|
|
12
|
-
from cognite.neat.workflows._exceptions import StepNotInitialized
|
|
13
13
|
from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
|
|
14
14
|
from cognite.neat.workflows.steps.data_contracts import MultiRuleData, NeatGraph
|
|
15
15
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
@@ -45,7 +45,7 @@ class GraphFromMockData(Step):
|
|
|
45
45
|
self, rules: MultiRuleData, graph_store: NeatGraph
|
|
46
46
|
) -> FlowMessage:
|
|
47
47
|
if self.configs is None:
|
|
48
|
-
raise
|
|
48
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
49
49
|
|
|
50
50
|
if not rules.information and not rules.dms:
|
|
51
51
|
return FlowMessage(
|
|
@@ -110,7 +110,7 @@ class GraphFromRdfFile(Step):
|
|
|
110
110
|
|
|
111
111
|
def run(self, graph_store: NeatGraph) -> FlowMessage: # type: ignore[override, syntax]
|
|
112
112
|
if self.configs is None or self.data_store_path is None:
|
|
113
|
-
raise
|
|
113
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
114
114
|
|
|
115
115
|
if source_file := self.configs["File path"]:
|
|
116
116
|
NeatGraph.graph.write(
|
|
@@ -2,7 +2,7 @@ import time
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import ClassVar
|
|
4
4
|
|
|
5
|
-
from cognite.neat.
|
|
5
|
+
from cognite.neat.issues.errors import WorkflowStepNotInitializedError
|
|
6
6
|
from cognite.neat.workflows.model import FlowMessage
|
|
7
7
|
from cognite.neat.workflows.steps.data_contracts import NeatGraph
|
|
8
8
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
@@ -34,7 +34,7 @@ class GraphToRdfFile(Step):
|
|
|
34
34
|
self, graph: NeatGraph
|
|
35
35
|
) -> FlowMessage: # type: ignore[syntax]
|
|
36
36
|
if self.configs is None or self.data_store_path is None:
|
|
37
|
-
raise
|
|
37
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
38
38
|
|
|
39
39
|
storage_path = self.data_store_path / Path(self.configs["File path"])
|
|
40
40
|
relative_graph_file_path = str(storage_path).split("/data/")[1]
|
|
@@ -2,10 +2,17 @@ import time
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import ClassVar, Literal, cast
|
|
4
4
|
|
|
5
|
+
from cognite.neat.issues.errors import WorkflowStepNotInitializedError
|
|
5
6
|
from cognite.neat.rules import exporters
|
|
6
|
-
from cognite.neat.rules._shared import DMSRules, InformationRules,
|
|
7
|
-
from cognite.neat.rules.models import RoleTypes
|
|
8
|
-
from cognite.neat.
|
|
7
|
+
from cognite.neat.rules._shared import DMSRules, InformationRules, VerifiedRules
|
|
8
|
+
from cognite.neat.rules.models import AssetRules, RoleTypes
|
|
9
|
+
from cognite.neat.rules.transformers import (
|
|
10
|
+
AssetToInformation,
|
|
11
|
+
DMSToInformation,
|
|
12
|
+
InformationToAsset,
|
|
13
|
+
InformationToDMS,
|
|
14
|
+
RulesPipeline,
|
|
15
|
+
)
|
|
9
16
|
from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
|
|
10
17
|
from cognite.neat.workflows.steps.data_contracts import CogniteClient, MultiRuleData
|
|
11
18
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
@@ -61,7 +68,7 @@ class DeleteDataModelFromCDF(Step):
|
|
|
61
68
|
|
|
62
69
|
def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
|
|
63
70
|
if self.configs is None or self.data_store_path is None:
|
|
64
|
-
raise
|
|
71
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
65
72
|
components_to_delete = {
|
|
66
73
|
cast(Literal["all", "spaces", "data_models", "views", "containers"], key)
|
|
67
74
|
for key, value in self.complex_configs["Components"].items()
|
|
@@ -75,26 +82,30 @@ class DeleteDataModelFromCDF(Step):
|
|
|
75
82
|
error_text="No DMS Schema components selected for removal! Please select minimum one!",
|
|
76
83
|
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
77
84
|
)
|
|
78
|
-
input_rules = rules.dms or rules.information
|
|
85
|
+
input_rules = rules.dms or rules.information or rules.asset
|
|
79
86
|
if input_rules is None:
|
|
80
87
|
return FlowMessage(
|
|
81
88
|
error_text="Missing DMS or Information rules in the input data! "
|
|
82
89
|
"Please ensure that a DMS or Information rules is provided!",
|
|
83
90
|
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
84
91
|
)
|
|
92
|
+
if isinstance(input_rules, DMSRules):
|
|
93
|
+
dms_rules = input_rules
|
|
94
|
+
elif isinstance(input_rules, InformationRules):
|
|
95
|
+
dms_rules = InformationToDMS().transform(input_rules).rules
|
|
96
|
+
elif isinstance(input_rules, AssetRules):
|
|
97
|
+
dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
|
|
98
|
+
else:
|
|
99
|
+
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
85
100
|
|
|
86
101
|
dms_exporter = exporters.DMSExporter(
|
|
87
102
|
export_components=frozenset(components_to_delete),
|
|
88
|
-
include_space=(
|
|
89
|
-
None
|
|
90
|
-
if multi_space_components_delete
|
|
91
|
-
else {input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix}
|
|
92
|
-
),
|
|
103
|
+
include_space=(None if multi_space_components_delete else {dms_rules.metadata.space}),
|
|
93
104
|
)
|
|
94
105
|
|
|
95
106
|
report_lines = ["# Data Model Deletion from CDF\n\n"]
|
|
96
107
|
errors = []
|
|
97
|
-
for result in dms_exporter.delete_from_cdf(rules=
|
|
108
|
+
for result in dms_exporter.delete_from_cdf(rules=dms_rules, client=cdf_client, dry_run=dry_run):
|
|
98
109
|
report_lines.append(str(result))
|
|
99
110
|
errors.extend(result.error_messages)
|
|
100
111
|
|
|
@@ -168,7 +179,7 @@ class RulesToDMS(Step):
|
|
|
168
179
|
|
|
169
180
|
def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
|
|
170
181
|
if self.configs is None or self.data_store_path is None:
|
|
171
|
-
raise
|
|
182
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
172
183
|
existing_components_handling = cast(
|
|
173
184
|
Literal["fail", "update", "skip", "force"], self.configs["Existing component handling"]
|
|
174
185
|
)
|
|
@@ -192,14 +203,18 @@ class RulesToDMS(Step):
|
|
|
192
203
|
"Please ensure that a DMS or Information rules is provided!",
|
|
193
204
|
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
194
205
|
)
|
|
206
|
+
if isinstance(input_rules, DMSRules):
|
|
207
|
+
dms_rules = input_rules
|
|
208
|
+
elif isinstance(input_rules, InformationRules):
|
|
209
|
+
dms_rules = InformationToDMS().transform(input_rules).rules
|
|
210
|
+
elif isinstance(input_rules, AssetRules):
|
|
211
|
+
dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
|
|
212
|
+
else:
|
|
213
|
+
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
195
214
|
|
|
196
215
|
dms_exporter = exporters.DMSExporter(
|
|
197
216
|
export_components=frozenset(components_to_create),
|
|
198
|
-
include_space=(
|
|
199
|
-
None
|
|
200
|
-
if multi_space_components_create
|
|
201
|
-
else {input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix}
|
|
202
|
-
),
|
|
217
|
+
include_space=(None if multi_space_components_create else {dms_rules.metadata.space}),
|
|
203
218
|
existing_handling=existing_components_handling,
|
|
204
219
|
)
|
|
205
220
|
|
|
@@ -212,11 +227,11 @@ class RulesToDMS(Step):
|
|
|
212
227
|
)
|
|
213
228
|
schema_zip = f"{file_name}.zip"
|
|
214
229
|
schema_full_path = output_dir / schema_zip
|
|
215
|
-
dms_exporter.export_to_file(
|
|
230
|
+
dms_exporter.export_to_file(dms_rules, schema_full_path)
|
|
216
231
|
|
|
217
232
|
report_lines = ["# DMS Schema Export to CDF\n\n"]
|
|
218
233
|
errors = []
|
|
219
|
-
for result in dms_exporter.export_to_cdf_iterable(rules=
|
|
234
|
+
for result in dms_exporter.export_to_cdf_iterable(rules=dms_rules, client=cdf_client, dry_run=dry_run):
|
|
220
235
|
report_lines.append(str(result))
|
|
221
236
|
errors.extend(result.error_messages)
|
|
222
237
|
|
|
@@ -291,12 +306,12 @@ class RulesToExcel(Step):
|
|
|
291
306
|
|
|
292
307
|
def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
|
|
293
308
|
if self.configs is None or self.data_store_path is None:
|
|
294
|
-
raise
|
|
309
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
295
310
|
|
|
296
311
|
dump_format = self.configs.get("Dump Format", "user")
|
|
297
312
|
styling = cast(exporters.ExcelExporter.Style, self.configs.get("Styling", "default"))
|
|
298
313
|
role = self.configs.get("Output role format")
|
|
299
|
-
output_role = None
|
|
314
|
+
output_role: RoleTypes | None = None
|
|
300
315
|
if role != "input" and role is not None:
|
|
301
316
|
output_role = RoleTypes[role]
|
|
302
317
|
|
|
@@ -310,23 +325,52 @@ class RulesToExcel(Step):
|
|
|
310
325
|
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
311
326
|
)
|
|
312
327
|
|
|
313
|
-
excel_exporter = exporters.ExcelExporter(
|
|
314
|
-
styling=styling,
|
|
315
|
-
output_role=output_role,
|
|
316
|
-
dump_as=dump_format, # type: ignore[arg-type]
|
|
317
|
-
new_model_id=new_model_id,
|
|
318
|
-
)
|
|
328
|
+
excel_exporter = exporters.ExcelExporter(styling=styling, dump_as=dump_format, new_model_id=new_model_id) # type: ignore[arg-type]
|
|
319
329
|
|
|
320
|
-
|
|
330
|
+
# Todo - Move the conversion to a separate workflow step.
|
|
331
|
+
rule_instance: VerifiedRules
|
|
321
332
|
if rules.domain:
|
|
322
333
|
rule_instance = rules.domain
|
|
323
334
|
elif rules.information:
|
|
324
335
|
rule_instance = rules.information
|
|
325
336
|
elif rules.dms:
|
|
326
337
|
rule_instance = rules.dms
|
|
338
|
+
elif rules.asset:
|
|
339
|
+
rule_instance = rules.asset
|
|
327
340
|
else:
|
|
328
341
|
output_errors = "No rules provided for export!"
|
|
329
342
|
return FlowMessage(error_text=output_errors, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
|
|
343
|
+
|
|
344
|
+
if rule_instance.metadata.role is output_role or output_role is None:
|
|
345
|
+
...
|
|
346
|
+
elif output_role is RoleTypes.dms:
|
|
347
|
+
if isinstance(rule_instance, InformationRules):
|
|
348
|
+
rule_instance = InformationToDMS().transform(rule_instance).rules
|
|
349
|
+
elif isinstance(rule_instance, AssetRules):
|
|
350
|
+
rule_instance = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(
|
|
351
|
+
rule_instance
|
|
352
|
+
)
|
|
353
|
+
else:
|
|
354
|
+
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
355
|
+
elif output_role is RoleTypes.information:
|
|
356
|
+
if isinstance(rule_instance, DMSRules):
|
|
357
|
+
rule_instance = DMSToInformation().transform(rule_instance).rules
|
|
358
|
+
elif isinstance(rule_instance, AssetRules):
|
|
359
|
+
rule_instance = AssetToInformation().transform(rule_instance).rules
|
|
360
|
+
else:
|
|
361
|
+
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
362
|
+
elif output_role is RoleTypes.asset:
|
|
363
|
+
if isinstance(rule_instance, InformationRules):
|
|
364
|
+
rule_instance = InformationToAsset().transform(rule_instance).rules
|
|
365
|
+
elif isinstance(rule_instance, DMSRules):
|
|
366
|
+
rule_instance = RulesPipeline[DMSRules, AssetRules]([DMSToInformation(), InformationToAsset()]).run(
|
|
367
|
+
rule_instance
|
|
368
|
+
)
|
|
369
|
+
else:
|
|
370
|
+
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
371
|
+
else:
|
|
372
|
+
raise NotImplementedError(f"Role {output_role} is not supported for {type(rules).__name__} rules")
|
|
373
|
+
|
|
330
374
|
if output_role is None:
|
|
331
375
|
output_role = rule_instance.metadata.role
|
|
332
376
|
output_dir = self.data_store_path / Path("staging")
|
|
@@ -369,7 +413,7 @@ class RulesToOntology(Step):
|
|
|
369
413
|
|
|
370
414
|
def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
|
|
371
415
|
if self.configs is None or self.data_store_path is None:
|
|
372
|
-
raise
|
|
416
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
373
417
|
|
|
374
418
|
if not rules.information and not rules.dms:
|
|
375
419
|
return FlowMessage(
|
|
@@ -384,8 +428,16 @@ class RulesToOntology(Step):
|
|
|
384
428
|
)
|
|
385
429
|
storage_path.parent.mkdir(parents=True, exist_ok=True)
|
|
386
430
|
|
|
431
|
+
input_rules = rules.information or rules.dms
|
|
432
|
+
if isinstance(input_rules, DMSRules):
|
|
433
|
+
info_rules = DMSToInformation().transform(input_rules).rules
|
|
434
|
+
elif isinstance(input_rules, InformationRules):
|
|
435
|
+
info_rules = input_rules
|
|
436
|
+
else:
|
|
437
|
+
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
438
|
+
|
|
387
439
|
exporter = exporters.OWLExporter()
|
|
388
|
-
exporter.export_to_file(
|
|
440
|
+
exporter.export_to_file(info_rules, storage_path)
|
|
389
441
|
|
|
390
442
|
relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
|
|
391
443
|
|
|
@@ -420,7 +472,7 @@ class RulesToSHACL(Step):
|
|
|
420
472
|
|
|
421
473
|
def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
|
|
422
474
|
if self.configs is None or self.data_store_path is None:
|
|
423
|
-
raise
|
|
475
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
424
476
|
|
|
425
477
|
if not rules.information and not rules.dms:
|
|
426
478
|
return FlowMessage(
|
|
@@ -435,8 +487,16 @@ class RulesToSHACL(Step):
|
|
|
435
487
|
)
|
|
436
488
|
storage_path.parent.mkdir(parents=True, exist_ok=True)
|
|
437
489
|
|
|
490
|
+
input_rules = rules.information or rules.dms
|
|
491
|
+
if isinstance(input_rules, DMSRules):
|
|
492
|
+
info_rules = DMSToInformation().transform(input_rules).rules
|
|
493
|
+
elif isinstance(input_rules, InformationRules):
|
|
494
|
+
info_rules = input_rules
|
|
495
|
+
else:
|
|
496
|
+
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
497
|
+
|
|
438
498
|
exporter = exporters.SHACLExporter()
|
|
439
|
-
exporter.export_to_file(
|
|
499
|
+
exporter.export_to_file(info_rules, storage_path)
|
|
440
500
|
|
|
441
501
|
relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
|
|
442
502
|
|
|
@@ -471,7 +531,7 @@ class RulesToSemanticDataModel(Step):
|
|
|
471
531
|
|
|
472
532
|
def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
|
|
473
533
|
if self.configs is None or self.data_store_path is None:
|
|
474
|
-
raise
|
|
534
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
475
535
|
|
|
476
536
|
if not rules.information and not rules.dms:
|
|
477
537
|
return FlowMessage(
|
|
@@ -485,9 +545,15 @@ class RulesToSemanticDataModel(Step):
|
|
|
485
545
|
self.data_store_path / Path(self.configs["File path"]) if self.configs["File path"] else default_path
|
|
486
546
|
)
|
|
487
547
|
storage_path.parent.mkdir(parents=True, exist_ok=True)
|
|
488
|
-
|
|
548
|
+
input_rules = rules.information or rules.dms
|
|
549
|
+
if isinstance(input_rules, DMSRules):
|
|
550
|
+
info_rules = DMSToInformation().transform(input_rules).rules
|
|
551
|
+
elif isinstance(input_rules, InformationRules):
|
|
552
|
+
info_rules = input_rules
|
|
553
|
+
else:
|
|
554
|
+
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
489
555
|
exporter = exporters.SemanticDataModelExporter()
|
|
490
|
-
exporter.export_to_file(
|
|
556
|
+
exporter.export_to_file(info_rules, storage_path)
|
|
491
557
|
|
|
492
558
|
relative_file_path = "/".join(storage_path.relative_to(self.data_store_path).parts)
|
|
493
559
|
|
|
@@ -525,7 +591,7 @@ class RulesToCDFTransformations(Step):
|
|
|
525
591
|
|
|
526
592
|
def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
|
|
527
593
|
if self.configs is None or self.data_store_path is None:
|
|
528
|
-
raise
|
|
594
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
529
595
|
|
|
530
596
|
input_rules = rules.dms or rules.information
|
|
531
597
|
if input_rules is None:
|
|
@@ -534,28 +600,31 @@ class RulesToCDFTransformations(Step):
|
|
|
534
600
|
"Please ensure that a DMS or Information rules is provided!",
|
|
535
601
|
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
536
602
|
)
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
)
|
|
540
|
-
|
|
603
|
+
if isinstance(input_rules, DMSRules):
|
|
604
|
+
dms_rules = input_rules
|
|
605
|
+
elif isinstance(input_rules, InformationRules):
|
|
606
|
+
dms_rules = InformationToDMS().transform(input_rules).rules
|
|
607
|
+
elif isinstance(input_rules, AssetRules):
|
|
608
|
+
dms_rules = RulesPipeline[AssetRules, DMSRules]([AssetToInformation(), InformationToDMS()]).run(input_rules)
|
|
609
|
+
else:
|
|
610
|
+
raise NotImplementedError(f"Unsupported rules type {type(input_rules)}")
|
|
611
|
+
|
|
612
|
+
instance_space = self.configs.get("Instance space") or dms_rules.metadata.space
|
|
541
613
|
dry_run = self.configs.get("Dry run", "False") == "True"
|
|
542
614
|
dms_exporter = exporters.DMSExporter(
|
|
543
615
|
export_pipeline=True, instance_space=instance_space, export_components=["spaces"]
|
|
544
616
|
)
|
|
545
617
|
output_dir = self.config.staging_path
|
|
546
618
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
547
|
-
file_name = (
|
|
548
|
-
input_rules.metadata.external_id
|
|
549
|
-
if isinstance(input_rules, DMSRules)
|
|
550
|
-
else input_rules.metadata.name.replace(" ", "_").lower()
|
|
551
|
-
)
|
|
619
|
+
file_name = dms_rules.metadata.external_id.replace(":", "_")
|
|
552
620
|
schema_zip = f"{file_name}_pipeline.zip"
|
|
553
621
|
schema_full_path = output_dir / schema_zip
|
|
554
|
-
|
|
622
|
+
|
|
623
|
+
dms_exporter.export_to_file(dms_rules, schema_full_path)
|
|
555
624
|
|
|
556
625
|
report_lines = ["# DMS Schema Export to CDF\n\n"]
|
|
557
626
|
errors = []
|
|
558
|
-
for result in dms_exporter.export_to_cdf_iterable(rules=
|
|
627
|
+
for result in dms_exporter.export_to_cdf_iterable(rules=dms_rules, client=cdf_client, dry_run=dry_run):
|
|
559
628
|
report_lines.append(str(result))
|
|
560
629
|
errors.extend(result.error_messages)
|
|
561
630
|
|