cognite-neat 0.88.1__py3-none-any.whl → 0.88.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite/neat/_version.py +1 -1
- cognite/neat/graph/__init__.py +0 -3
- cognite/neat/graph/loaders/_base.py +6 -6
- cognite/neat/graph/loaders/_rdf2asset.py +28 -31
- cognite/neat/graph/loaders/_rdf2dms.py +24 -15
- cognite/neat/issues/__init__.py +14 -0
- cognite/neat/issues/_base.py +415 -0
- cognite/neat/issues/errors/__init__.py +72 -0
- cognite/neat/issues/errors/_external.py +67 -0
- cognite/neat/issues/errors/_general.py +28 -0
- cognite/neat/issues/errors/_properties.py +62 -0
- cognite/neat/issues/errors/_resources.py +111 -0
- cognite/neat/issues/errors/_workflow.py +36 -0
- cognite/neat/{rules/issues → issues}/formatters.py +10 -10
- cognite/neat/issues/warnings/__init__.py +66 -0
- cognite/neat/issues/warnings/_external.py +40 -0
- cognite/neat/issues/warnings/_general.py +29 -0
- cognite/neat/issues/warnings/_models.py +92 -0
- cognite/neat/issues/warnings/_properties.py +44 -0
- cognite/neat/issues/warnings/_resources.py +55 -0
- cognite/neat/issues/warnings/user_modeling.py +113 -0
- cognite/neat/rules/_shared.py +10 -2
- cognite/neat/rules/exporters/_base.py +6 -6
- cognite/neat/rules/exporters/_rules2dms.py +19 -11
- cognite/neat/rules/exporters/_rules2excel.py +4 -4
- cognite/neat/rules/exporters/_rules2ontology.py +74 -51
- cognite/neat/rules/exporters/_rules2yaml.py +3 -3
- cognite/neat/rules/exporters/_validation.py +11 -96
- cognite/neat/rules/importers/__init__.py +7 -3
- cognite/neat/rules/importers/_base.py +9 -13
- cognite/neat/rules/importers/_dms2rules.py +42 -24
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +49 -53
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +31 -23
- cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/__init__.py +3 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +82 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +34 -0
- cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +123 -0
- cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py} +24 -18
- cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py} +9 -9
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +58 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py +68 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +60 -0
- cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +76 -0
- cognite/neat/rules/importers/_rdf/_shared.py +586 -0
- cognite/neat/rules/importers/_spreadsheet2rules.py +35 -22
- cognite/neat/rules/importers/_yaml2rules.py +23 -21
- cognite/neat/rules/models/_constants.py +2 -1
- cognite/neat/rules/models/_rdfpath.py +4 -4
- cognite/neat/rules/models/_types/_field.py +9 -11
- cognite/neat/rules/models/asset/_rules.py +1 -3
- cognite/neat/rules/models/asset/_validation.py +14 -10
- cognite/neat/rules/models/dms/_converter.py +2 -4
- cognite/neat/rules/models/dms/_exporter.py +30 -8
- cognite/neat/rules/models/dms/_rules.py +23 -7
- cognite/neat/rules/models/dms/_schema.py +94 -62
- cognite/neat/rules/models/dms/_validation.py +105 -66
- cognite/neat/rules/models/entities.py +3 -0
- cognite/neat/rules/models/information/_converter.py +2 -2
- cognite/neat/rules/models/information/_rules.py +7 -8
- cognite/neat/rules/models/information/_validation.py +48 -25
- cognite/neat/rules/transformers/__init__.py +0 -0
- cognite/neat/rules/transformers/_base.py +15 -0
- cognite/neat/utils/auxiliary.py +2 -35
- cognite/neat/utils/text.py +17 -0
- cognite/neat/workflows/base.py +4 -4
- cognite/neat/workflows/cdf_store.py +3 -3
- cognite/neat/workflows/steps/data_contracts.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
- cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
- cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
- cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -10
- cognite/neat/workflows/steps/lib/current/rules_importer.py +78 -6
- cognite/neat/workflows/steps/lib/current/rules_validator.py +20 -9
- cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
- cognite/neat/workflows/steps_registry.py +4 -5
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/METADATA +1 -1
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/RECORD +86 -77
- cognite/neat/exceptions.py +0 -145
- cognite/neat/graph/exceptions.py +0 -90
- cognite/neat/graph/issues/loader.py +0 -104
- cognite/neat/issues.py +0 -158
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +0 -215
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +0 -209
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +0 -203
- cognite/neat/rules/issues/__init__.py +0 -26
- cognite/neat/rules/issues/base.py +0 -82
- cognite/neat/rules/issues/dms.py +0 -683
- cognite/neat/rules/issues/fileread.py +0 -197
- cognite/neat/rules/issues/importing.py +0 -423
- cognite/neat/rules/issues/ontology.py +0 -298
- cognite/neat/rules/issues/spreadsheet.py +0 -563
- cognite/neat/rules/issues/spreadsheet_file.py +0 -151
- cognite/neat/rules/issues/tables.py +0 -72
- cognite/neat/workflows/_exceptions.py +0 -41
- /cognite/neat/{graph/issues → rules/importers/_rdf}/__init__.py +0 -0
- /cognite/neat/rules/importers/{_owl2rules → _rdf/_owl2rules}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/__init__.py +0 -0
- /cognite/neat/{graph/stores → store}/_base.py +0 -0
- /cognite/neat/{graph/stores → store}/_provenance.py +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/entry_points.txt +0 -0
|
@@ -8,9 +8,7 @@ from pydantic.main import IncEx
|
|
|
8
8
|
from rdflib import Namespace
|
|
9
9
|
|
|
10
10
|
from cognite.neat.constants import get_default_prefixes
|
|
11
|
-
from cognite.neat.issues import
|
|
12
|
-
from cognite.neat.rules import issues
|
|
13
|
-
from cognite.neat.rules.issues.spreadsheet import DefaultValueTypeNotProperError
|
|
11
|
+
from cognite.neat.issues.errors import PropertyDefinitionError
|
|
14
12
|
from cognite.neat.rules.models._base import (
|
|
15
13
|
BaseMetadata,
|
|
16
14
|
BaseRules,
|
|
@@ -228,11 +226,12 @@ class InformationProperty(SheetEntity):
|
|
|
228
226
|
self.default = self.value_type.python(self.default)
|
|
229
227
|
|
|
230
228
|
except Exception:
|
|
231
|
-
raise
|
|
229
|
+
raise PropertyDefinitionError(
|
|
230
|
+
self.class_,
|
|
231
|
+
"Class",
|
|
232
232
|
self.property_,
|
|
233
|
-
type
|
|
234
|
-
|
|
235
|
-
).as_exception() from None
|
|
233
|
+
f"Default value {self.default} is not of type {self.value_type.python}",
|
|
234
|
+
) from None
|
|
236
235
|
return self
|
|
237
236
|
|
|
238
237
|
@property
|
|
@@ -306,7 +305,7 @@ class InformationRules(BaseRules):
|
|
|
306
305
|
if issue_list.warnings:
|
|
307
306
|
issue_list.trigger_warnings()
|
|
308
307
|
if issue_list.has_errors:
|
|
309
|
-
raise
|
|
308
|
+
raise issue_list.as_exception()
|
|
310
309
|
return self
|
|
311
310
|
|
|
312
311
|
def dump(
|
|
@@ -2,8 +2,8 @@ import itertools
|
|
|
2
2
|
from collections import Counter
|
|
3
3
|
from typing import cast
|
|
4
4
|
|
|
5
|
-
from cognite.neat.
|
|
6
|
-
from cognite.neat.
|
|
5
|
+
from cognite.neat.issues import IssueList
|
|
6
|
+
from cognite.neat.issues.errors import NeatValueError, ResourceNotDefinedError
|
|
7
7
|
from cognite.neat.rules.models._base import DataModelType, SchemaCompleteness
|
|
8
8
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
|
|
9
9
|
from cognite.neat.utils.rdf_ import get_inheritance_path
|
|
@@ -58,9 +58,9 @@ class InformationPostValidation:
|
|
|
58
58
|
):
|
|
59
59
|
dangling_classes.add(class_)
|
|
60
60
|
|
|
61
|
-
|
|
61
|
+
for class_ in dangling_classes:
|
|
62
62
|
self.issue_list.append(
|
|
63
|
-
|
|
63
|
+
NeatValueError(f"Class {class_} has no properties and is not a parent of any class with properties")
|
|
64
64
|
)
|
|
65
65
|
|
|
66
66
|
def _referenced_parent_classes_exist(self) -> None:
|
|
@@ -70,9 +70,15 @@ class InformationPostValidation:
|
|
|
70
70
|
parents = set(itertools.chain.from_iterable(class_parent_pairs.values()))
|
|
71
71
|
|
|
72
72
|
if undefined_parents := parents.difference(classes):
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
for parent in undefined_parents:
|
|
74
|
+
# Todo: include row and column number
|
|
75
|
+
self.issue_list.append(
|
|
76
|
+
ResourceNotDefinedError[ClassEntity](
|
|
77
|
+
resource_type="class",
|
|
78
|
+
identifier=parent,
|
|
79
|
+
location="Classes sheet",
|
|
80
|
+
)
|
|
81
|
+
)
|
|
76
82
|
|
|
77
83
|
def _referenced_classes_exist(self) -> None:
|
|
78
84
|
# needs to be complete for this validation to pass
|
|
@@ -83,21 +89,27 @@ class InformationPostValidation:
|
|
|
83
89
|
if self.metadata.schema_ == SchemaCompleteness.complete and (
|
|
84
90
|
missing_classes := referred_classes.difference(defined_classes)
|
|
85
91
|
):
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
[
|
|
92
|
+
for class_ in missing_classes:
|
|
93
|
+
self.issue_list.append(
|
|
94
|
+
ResourceNotDefinedError[ClassEntity](
|
|
95
|
+
resource_type="class",
|
|
96
|
+
identifier=class_,
|
|
97
|
+
location="Classes sheet",
|
|
98
|
+
)
|
|
89
99
|
)
|
|
90
|
-
)
|
|
91
100
|
|
|
92
101
|
# USE CASE: models are extended (user + last = complete)
|
|
93
102
|
if self.metadata.schema_ == SchemaCompleteness.extended:
|
|
94
103
|
defined_classes |= {class_.class_ for class_ in cast(InformationRules, self.rules.last).classes}
|
|
95
104
|
if missing_classes := referred_classes.difference(defined_classes):
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
[
|
|
105
|
+
for class_ in missing_classes:
|
|
106
|
+
self.issue_list.append(
|
|
107
|
+
ResourceNotDefinedError[ClassEntity](
|
|
108
|
+
resource_type="class",
|
|
109
|
+
identifier=class_,
|
|
110
|
+
location="Classes sheet",
|
|
111
|
+
)
|
|
99
112
|
)
|
|
100
|
-
)
|
|
101
113
|
|
|
102
114
|
def _referenced_value_types_exist(self) -> None:
|
|
103
115
|
# adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
|
|
@@ -112,21 +124,29 @@ class InformationPostValidation:
|
|
|
112
124
|
if self.metadata.schema_ == SchemaCompleteness.complete and (
|
|
113
125
|
missing_value_types := referred_object_types.difference(defined_classes)
|
|
114
126
|
):
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
127
|
+
# Todo: include row and column number
|
|
128
|
+
for missing in missing_value_types:
|
|
129
|
+
self.issue_list.append(
|
|
130
|
+
ResourceNotDefinedError[ClassEntity](
|
|
131
|
+
resource_type="class",
|
|
132
|
+
identifier=cast(ClassEntity, missing),
|
|
133
|
+
location="Classes sheet",
|
|
134
|
+
)
|
|
118
135
|
)
|
|
119
|
-
)
|
|
120
136
|
|
|
121
137
|
# USE CASE: models are extended (user + last = complete)
|
|
122
138
|
if self.metadata.schema_ == SchemaCompleteness.extended:
|
|
123
139
|
defined_classes |= {class_.class_ for class_ in cast(InformationRules, self.rules.last).classes}
|
|
124
140
|
if missing_value_types := referred_object_types.difference(defined_classes):
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
141
|
+
# Todo: include row and column number
|
|
142
|
+
for missing in missing_value_types:
|
|
143
|
+
self.issue_list.append(
|
|
144
|
+
ResourceNotDefinedError(
|
|
145
|
+
resource_type="class",
|
|
146
|
+
identifier=cast(ClassEntity, missing),
|
|
147
|
+
location="Classes sheet",
|
|
148
|
+
)
|
|
128
149
|
)
|
|
129
|
-
)
|
|
130
150
|
|
|
131
151
|
def _class_parent_pairs(self) -> dict[ClassEntity, list[ClassEntity]]:
|
|
132
152
|
class_subclass_pairs: dict[ClassEntity, list[ClassEntity]] = {}
|
|
@@ -173,7 +193,10 @@ class InformationPostValidation:
|
|
|
173
193
|
reused_namespaces = [value for value, count in Counter(prefixes.values()).items() if count > 1]
|
|
174
194
|
impacted_prefixes = [key for key, value in prefixes.items() if value in reused_namespaces]
|
|
175
195
|
self.issue_list.append(
|
|
176
|
-
|
|
177
|
-
prefixes
|
|
196
|
+
NeatValueError(
|
|
197
|
+
"Namespace collision detected. The following prefixes "
|
|
198
|
+
f"are assigned to the same namespace: {impacted_prefixes}"
|
|
199
|
+
f"\nImpacted namespaces: {reused_namespaces}"
|
|
200
|
+
"\nMake sure that each unique namespace is assigned to a unique prefix"
|
|
178
201
|
)
|
|
179
202
|
)
|
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Generic, TypeVar
|
|
3
|
+
|
|
4
|
+
from cognite.neat.rules._shared import Rules
|
|
5
|
+
|
|
6
|
+
T_RulesIn = TypeVar("T_RulesIn", bound=Rules)
|
|
7
|
+
T_RulesOut = TypeVar("T_RulesOut", bound=Rules)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class RulesTransformer(ABC, Generic[T_RulesIn, T_RulesOut]):
|
|
11
|
+
"""This is the base class for all rule transformers."""
|
|
12
|
+
|
|
13
|
+
@abstractmethod
|
|
14
|
+
def transform(self, rules: T_RulesIn) -> T_RulesOut:
|
|
15
|
+
raise NotImplementedError()
|
cognite/neat/utils/auxiliary.py
CHANGED
|
@@ -3,15 +3,14 @@ import importlib
|
|
|
3
3
|
import inspect
|
|
4
4
|
import logging
|
|
5
5
|
import time
|
|
6
|
-
from collections.abc import Callable
|
|
6
|
+
from collections.abc import Callable
|
|
7
7
|
from datetime import datetime
|
|
8
8
|
from functools import wraps
|
|
9
9
|
from types import ModuleType
|
|
10
10
|
|
|
11
11
|
from cognite.client.exceptions import CogniteDuplicatedError, CogniteReadTimeout
|
|
12
|
-
from pydantic_core import ErrorDetails
|
|
13
12
|
|
|
14
|
-
from cognite.neat.
|
|
13
|
+
from cognite.neat.issues.errors import NeatImportError
|
|
15
14
|
|
|
16
15
|
|
|
17
16
|
def local_import(module: str, extra: str) -> ModuleType:
|
|
@@ -119,38 +118,6 @@ def create_sha256_hash(string: str) -> str:
|
|
|
119
118
|
return hash_value
|
|
120
119
|
|
|
121
120
|
|
|
122
|
-
# Will likely be removed with legacy code
|
|
123
|
-
def generate_exception_report(exceptions: list[dict] | list[ErrorDetails] | None, category: str = "") -> str:
|
|
124
|
-
exceptions_as_dict = _order_expectations_by_type(exceptions) if exceptions else {}
|
|
125
|
-
report = ""
|
|
126
|
-
|
|
127
|
-
for exception_type in exceptions_as_dict.keys():
|
|
128
|
-
title = f"# {category}: {exception_type}" if category else ""
|
|
129
|
-
warnings = "\n- " + "\n- ".join(exceptions_as_dict[exception_type])
|
|
130
|
-
report += title + warnings + "\n\n"
|
|
131
|
-
|
|
132
|
-
return report
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
def _order_expectations_by_type(
|
|
136
|
-
exceptions: list[dict] | list[ErrorDetails],
|
|
137
|
-
) -> dict[str, list[str]]:
|
|
138
|
-
exception_dict: dict[str, list[str]] = {}
|
|
139
|
-
for exception in exceptions:
|
|
140
|
-
if not isinstance(exception["loc"], str) and isinstance(exception["loc"], Iterable):
|
|
141
|
-
location = f"[{'/'.join(str(e) for e in exception['loc'])}]"
|
|
142
|
-
else:
|
|
143
|
-
location = ""
|
|
144
|
-
|
|
145
|
-
issue = f"{exception['msg']} {location}"
|
|
146
|
-
|
|
147
|
-
if exception_dict.get(exception["type"]) is None:
|
|
148
|
-
exception_dict[exception["type"]] = [issue]
|
|
149
|
-
else:
|
|
150
|
-
exception_dict[exception["type"]].append(issue)
|
|
151
|
-
return exception_dict
|
|
152
|
-
|
|
153
|
-
|
|
154
121
|
def string_to_ideal_type(input_string: str) -> int | bool | float | datetime | str:
|
|
155
122
|
try:
|
|
156
123
|
# Try converting to int
|
cognite/neat/utils/text.py
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import re
|
|
2
|
+
from collections.abc import Collection
|
|
3
|
+
from typing import Any
|
|
2
4
|
|
|
3
5
|
|
|
4
6
|
def to_camel(string: str) -> str:
|
|
@@ -106,3 +108,18 @@ def to_snake(string: str) -> str:
|
|
|
106
108
|
|
|
107
109
|
def replace_non_alphanumeric_with_underscore(text: str) -> str:
|
|
108
110
|
return re.sub(r"\W+", "_", text)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def humanize_collection(collection: Collection[Any], /, *, sort: bool = True) -> str:
|
|
114
|
+
if not collection:
|
|
115
|
+
return ""
|
|
116
|
+
elif len(collection) == 1:
|
|
117
|
+
return str(next(iter(collection)))
|
|
118
|
+
|
|
119
|
+
strings = (str(item) for item in collection)
|
|
120
|
+
if sort:
|
|
121
|
+
sequence = sorted(strings)
|
|
122
|
+
else:
|
|
123
|
+
sequence = list(strings)
|
|
124
|
+
|
|
125
|
+
return f"{', '.join(sequence[:-1])} and {sequence[-1]}"
|
cognite/neat/workflows/base.py
CHANGED
|
@@ -13,9 +13,9 @@ from prometheus_client import Gauge
|
|
|
13
13
|
|
|
14
14
|
from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
|
|
15
15
|
from cognite.neat.config import Config
|
|
16
|
+
from cognite.neat.issues.errors import WorkflowConfigurationNotSetError, WorkflowStepOutputError
|
|
16
17
|
from cognite.neat.utils.auxiliary import retry_decorator
|
|
17
18
|
from cognite.neat.workflows import cdf_store, utils
|
|
18
|
-
from cognite.neat.workflows._exceptions import ConfigurationNotSet, InvalidStepOutputException
|
|
19
19
|
from cognite.neat.workflows.cdf_store import CdfStore
|
|
20
20
|
from cognite.neat.workflows.model import (
|
|
21
21
|
FlowMessage,
|
|
@@ -316,7 +316,7 @@ class BaseWorkflow:
|
|
|
316
316
|
elif isinstance(out_obj, DataContract):
|
|
317
317
|
self.data[type(out_obj).__name__] = out_obj
|
|
318
318
|
else:
|
|
319
|
-
raise
|
|
319
|
+
raise WorkflowStepOutputError(step_type=type(out_obj).__name__)
|
|
320
320
|
|
|
321
321
|
elif step.stype == StepType.START_WORKFLOW_TASK_STEP:
|
|
322
322
|
if self.task_builder:
|
|
@@ -510,7 +510,7 @@ class BaseWorkflow:
|
|
|
510
510
|
if storage_type == "transformation_rules":
|
|
511
511
|
self.rules_storage_path = Path(storage_path)
|
|
512
512
|
if self.default_dataset_id is None:
|
|
513
|
-
raise
|
|
513
|
+
raise WorkflowConfigurationNotSetError("default_dataset_id")
|
|
514
514
|
self.cdf_store = cdf_store.CdfStore(
|
|
515
515
|
self.cdf_client, data_set_id=self.default_dataset_id, rules_storage_path=self.rules_storage_path
|
|
516
516
|
)
|
|
@@ -576,7 +576,7 @@ class BaseWorkflow:
|
|
|
576
576
|
|
|
577
577
|
file_list: list[Path] = []
|
|
578
578
|
if self.data_store_path is None:
|
|
579
|
-
raise
|
|
579
|
+
raise WorkflowConfigurationNotSetError("data_store_path")
|
|
580
580
|
workflow_data_path = Path(self.data_store_path) / "workflows" / self.name
|
|
581
581
|
try:
|
|
582
582
|
for root, _dirs, files in os.walk(workflow_data_path):
|
|
@@ -11,7 +11,7 @@ from cognite.client.data_classes import Event, FileMetadataUpdate
|
|
|
11
11
|
from fastapi.encoders import jsonable_encoder
|
|
12
12
|
from pydantic import BaseModel
|
|
13
13
|
|
|
14
|
-
from cognite.neat.
|
|
14
|
+
from cognite.neat.issues.errors import WorkflowConfigurationNotSetError
|
|
15
15
|
from cognite.neat.workflows.model import WorkflowFullStateReport, WorkflowState, WorkflowStepEvent
|
|
16
16
|
from cognite.neat.workflows.utils import get_file_hash
|
|
17
17
|
|
|
@@ -53,7 +53,7 @@ class CdfStore:
|
|
|
53
53
|
def package_workflow(self, workflow_name: str) -> str:
|
|
54
54
|
"""Creates a zip archive from a folder"""
|
|
55
55
|
if self.workflows_storage_path is None:
|
|
56
|
-
raise
|
|
56
|
+
raise WorkflowConfigurationNotSetError("workflows_storage_path")
|
|
57
57
|
folder_path = self.workflows_storage_path / workflow_name
|
|
58
58
|
archive_path = self.workflows_storage_path / f"{workflow_name}.zip"
|
|
59
59
|
# Make sure the folder exists
|
|
@@ -77,7 +77,7 @@ class CdfStore:
|
|
|
77
77
|
# Make sure the archive exists
|
|
78
78
|
workflow_name = workflow_name.replace(".zip", "")
|
|
79
79
|
if self.workflows_storage_path is None:
|
|
80
|
-
raise
|
|
80
|
+
raise WorkflowConfigurationNotSetError("workflows_storage_path")
|
|
81
81
|
package_full_path = Path(self.workflows_storage_path) / f"{workflow_name}.zip"
|
|
82
82
|
output_folder = Path(self.workflows_storage_path) / workflow_name
|
|
83
83
|
if not package_full_path.is_file():
|
|
@@ -9,13 +9,13 @@ from cognite.client.data_classes import (
|
|
|
9
9
|
)
|
|
10
10
|
from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
|
|
11
11
|
|
|
12
|
-
from cognite.neat.graph.stores import NeatGraphStore
|
|
13
12
|
from cognite.neat.rules.models import (
|
|
14
13
|
AssetRules,
|
|
15
14
|
DMSRules,
|
|
16
15
|
DomainRules,
|
|
17
16
|
InformationRules,
|
|
18
17
|
)
|
|
18
|
+
from cognite.neat.store import NeatGraphStore
|
|
19
19
|
from cognite.neat.workflows.steps.step_model import DataContract
|
|
20
20
|
|
|
21
21
|
|
|
@@ -8,8 +8,8 @@ from rdflib import URIRef
|
|
|
8
8
|
from cognite.neat.constants import DEFAULT_NAMESPACE
|
|
9
9
|
from cognite.neat.graph.extractors import RdfFileExtractor
|
|
10
10
|
from cognite.neat.graph.extractors._mock_graph_generator import MockGraphGenerator
|
|
11
|
+
from cognite.neat.issues.errors import WorkflowStepNotInitializedError
|
|
11
12
|
from cognite.neat.rules._shared import DMSRules, InformationRules
|
|
12
|
-
from cognite.neat.workflows._exceptions import StepNotInitialized
|
|
13
13
|
from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
|
|
14
14
|
from cognite.neat.workflows.steps.data_contracts import MultiRuleData, NeatGraph
|
|
15
15
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
@@ -45,7 +45,7 @@ class GraphFromMockData(Step):
|
|
|
45
45
|
self, rules: MultiRuleData, graph_store: NeatGraph
|
|
46
46
|
) -> FlowMessage:
|
|
47
47
|
if self.configs is None:
|
|
48
|
-
raise
|
|
48
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
49
49
|
|
|
50
50
|
if not rules.information and not rules.dms:
|
|
51
51
|
return FlowMessage(
|
|
@@ -110,7 +110,7 @@ class GraphFromRdfFile(Step):
|
|
|
110
110
|
|
|
111
111
|
def run(self, graph_store: NeatGraph) -> FlowMessage: # type: ignore[override, syntax]
|
|
112
112
|
if self.configs is None or self.data_store_path is None:
|
|
113
|
-
raise
|
|
113
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
114
114
|
|
|
115
115
|
if source_file := self.configs["File path"]:
|
|
116
116
|
NeatGraph.graph.write(
|
|
@@ -2,7 +2,7 @@ import time
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import ClassVar
|
|
4
4
|
|
|
5
|
-
from cognite.neat.
|
|
5
|
+
from cognite.neat.issues.errors import WorkflowStepNotInitializedError
|
|
6
6
|
from cognite.neat.workflows.model import FlowMessage
|
|
7
7
|
from cognite.neat.workflows.steps.data_contracts import NeatGraph
|
|
8
8
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
@@ -34,7 +34,7 @@ class GraphToRdfFile(Step):
|
|
|
34
34
|
self, graph: NeatGraph
|
|
35
35
|
) -> FlowMessage: # type: ignore[syntax]
|
|
36
36
|
if self.configs is None or self.data_store_path is None:
|
|
37
|
-
raise
|
|
37
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
38
38
|
|
|
39
39
|
storage_path = self.data_store_path / Path(self.configs["File path"])
|
|
40
40
|
relative_graph_file_path = str(storage_path).split("/data/")[1]
|
|
@@ -2,10 +2,10 @@ import time
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import ClassVar, Literal, cast
|
|
4
4
|
|
|
5
|
+
from cognite.neat.issues.errors import WorkflowStepNotInitializedError
|
|
5
6
|
from cognite.neat.rules import exporters
|
|
6
|
-
from cognite.neat.rules._shared import DMSRules, InformationRules,
|
|
7
|
+
from cognite.neat.rules._shared import DMSRules, InformationRules, VerifiedRules
|
|
7
8
|
from cognite.neat.rules.models import RoleTypes
|
|
8
|
-
from cognite.neat.workflows._exceptions import StepNotInitialized
|
|
9
9
|
from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
|
|
10
10
|
from cognite.neat.workflows.steps.data_contracts import CogniteClient, MultiRuleData
|
|
11
11
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
@@ -61,7 +61,7 @@ class DeleteDataModelFromCDF(Step):
|
|
|
61
61
|
|
|
62
62
|
def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
|
|
63
63
|
if self.configs is None or self.data_store_path is None:
|
|
64
|
-
raise
|
|
64
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
65
65
|
components_to_delete = {
|
|
66
66
|
cast(Literal["all", "spaces", "data_models", "views", "containers"], key)
|
|
67
67
|
for key, value in self.complex_configs["Components"].items()
|
|
@@ -168,7 +168,7 @@ class RulesToDMS(Step):
|
|
|
168
168
|
|
|
169
169
|
def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
|
|
170
170
|
if self.configs is None or self.data_store_path is None:
|
|
171
|
-
raise
|
|
171
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
172
172
|
existing_components_handling = cast(
|
|
173
173
|
Literal["fail", "update", "skip", "force"], self.configs["Existing component handling"]
|
|
174
174
|
)
|
|
@@ -291,7 +291,7 @@ class RulesToExcel(Step):
|
|
|
291
291
|
|
|
292
292
|
def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
|
|
293
293
|
if self.configs is None or self.data_store_path is None:
|
|
294
|
-
raise
|
|
294
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
295
295
|
|
|
296
296
|
dump_format = self.configs.get("Dump Format", "user")
|
|
297
297
|
styling = cast(exporters.ExcelExporter.Style, self.configs.get("Styling", "default"))
|
|
@@ -317,7 +317,7 @@ class RulesToExcel(Step):
|
|
|
317
317
|
new_model_id=new_model_id,
|
|
318
318
|
)
|
|
319
319
|
|
|
320
|
-
rule_instance:
|
|
320
|
+
rule_instance: VerifiedRules
|
|
321
321
|
if rules.domain:
|
|
322
322
|
rule_instance = rules.domain
|
|
323
323
|
elif rules.information:
|
|
@@ -369,7 +369,7 @@ class RulesToOntology(Step):
|
|
|
369
369
|
|
|
370
370
|
def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
|
|
371
371
|
if self.configs is None or self.data_store_path is None:
|
|
372
|
-
raise
|
|
372
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
373
373
|
|
|
374
374
|
if not rules.information and not rules.dms:
|
|
375
375
|
return FlowMessage(
|
|
@@ -420,7 +420,7 @@ class RulesToSHACL(Step):
|
|
|
420
420
|
|
|
421
421
|
def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
|
|
422
422
|
if self.configs is None or self.data_store_path is None:
|
|
423
|
-
raise
|
|
423
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
424
424
|
|
|
425
425
|
if not rules.information and not rules.dms:
|
|
426
426
|
return FlowMessage(
|
|
@@ -471,7 +471,7 @@ class RulesToSemanticDataModel(Step):
|
|
|
471
471
|
|
|
472
472
|
def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
|
|
473
473
|
if self.configs is None or self.data_store_path is None:
|
|
474
|
-
raise
|
|
474
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
475
475
|
|
|
476
476
|
if not rules.information and not rules.dms:
|
|
477
477
|
return FlowMessage(
|
|
@@ -525,7 +525,7 @@ class RulesToCDFTransformations(Step):
|
|
|
525
525
|
|
|
526
526
|
def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
|
|
527
527
|
if self.configs is None or self.data_store_path is None:
|
|
528
|
-
raise
|
|
528
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
529
529
|
|
|
530
530
|
input_rules = rules.dms or rules.information
|
|
531
531
|
if input_rules is None:
|
|
@@ -5,11 +5,11 @@ from typing import ClassVar
|
|
|
5
5
|
from cognite.client import CogniteClient
|
|
6
6
|
from cognite.client.data_classes.data_modeling import DataModelId
|
|
7
7
|
|
|
8
|
+
from cognite.neat.issues.errors import WorkflowStepNotInitializedError
|
|
9
|
+
from cognite.neat.issues.formatters import FORMATTER_BY_NAME
|
|
8
10
|
from cognite.neat.rules import importers
|
|
9
|
-
from cognite.neat.rules.issues.formatters import FORMATTER_BY_NAME
|
|
10
11
|
from cognite.neat.rules.models import RoleTypes
|
|
11
12
|
from cognite.neat.rules.models.entities import DataModelEntity, DMSUnknownEntity
|
|
12
|
-
from cognite.neat.workflows._exceptions import StepNotInitialized
|
|
13
13
|
from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
|
|
14
14
|
from cognite.neat.workflows.steps.data_contracts import MultiRuleData
|
|
15
15
|
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
@@ -19,6 +19,7 @@ CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
|
|
|
19
19
|
__all__ = [
|
|
20
20
|
"ExcelToRules",
|
|
21
21
|
"OntologyToRules",
|
|
22
|
+
"IMFToRules",
|
|
22
23
|
"DMSToRules",
|
|
23
24
|
"RulesInferenceFromRdfFile",
|
|
24
25
|
]
|
|
@@ -54,7 +55,7 @@ class ExcelToRules(Step):
|
|
|
54
55
|
|
|
55
56
|
def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
|
|
56
57
|
if self.configs is None or self.data_store_path is None:
|
|
57
|
-
raise
|
|
58
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
58
59
|
|
|
59
60
|
file_name = self.configs.get("File name", None)
|
|
60
61
|
full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
|
|
@@ -123,7 +124,7 @@ class OntologyToRules(Step):
|
|
|
123
124
|
|
|
124
125
|
def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
|
|
125
126
|
if self.configs is None or self.data_store_path is None:
|
|
126
|
-
raise
|
|
127
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
127
128
|
|
|
128
129
|
file_name = self.configs.get("File name", None)
|
|
129
130
|
full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
|
|
@@ -162,6 +163,77 @@ class OntologyToRules(Step):
|
|
|
162
163
|
return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
|
|
163
164
|
|
|
164
165
|
|
|
166
|
+
class IMFToRules(Step):
|
|
167
|
+
"""This step import rules from the IMF-types and validates them."""
|
|
168
|
+
|
|
169
|
+
description = "This step imports rules from an RDF file "
|
|
170
|
+
version = "private-beta"
|
|
171
|
+
category = CATEGORY
|
|
172
|
+
configurables: ClassVar[list[Configurable]] = [
|
|
173
|
+
Configurable(
|
|
174
|
+
name="File name",
|
|
175
|
+
value="",
|
|
176
|
+
label="""Full file name of the RDF-file containing the IMF-types in the rules folder.
|
|
177
|
+
If not provided, step will attempt to get file name from payload
|
|
178
|
+
of 'File Uploader' step (if exist)""",
|
|
179
|
+
),
|
|
180
|
+
Configurable(
|
|
181
|
+
name="Report formatter",
|
|
182
|
+
value=next(iter(FORMATTER_BY_NAME.keys())),
|
|
183
|
+
label="The format of the report for the validation of the rules",
|
|
184
|
+
options=list(FORMATTER_BY_NAME),
|
|
185
|
+
),
|
|
186
|
+
Configurable(
|
|
187
|
+
name="Role",
|
|
188
|
+
value="infer",
|
|
189
|
+
label="For what role Rules are intended?",
|
|
190
|
+
options=["infer", *RoleTypes.__members__.keys()],
|
|
191
|
+
),
|
|
192
|
+
]
|
|
193
|
+
|
|
194
|
+
def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
|
|
195
|
+
if self.configs is None or self.data_store_path is None:
|
|
196
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
197
|
+
|
|
198
|
+
file_name = self.configs.get("File name", None)
|
|
199
|
+
|
|
200
|
+
full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
|
|
201
|
+
|
|
202
|
+
if file_name:
|
|
203
|
+
rules_file_path = self.config.rules_store_path / file_name
|
|
204
|
+
|
|
205
|
+
elif full_path:
|
|
206
|
+
rules_file_path = full_path
|
|
207
|
+
else:
|
|
208
|
+
error_text = "Expected either 'File name' in the step config or 'File uploader' step uploading Excel Rules."
|
|
209
|
+
return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
|
|
210
|
+
|
|
211
|
+
# if role is None, it will be inferred from the rules file
|
|
212
|
+
role = self.configs.get("Role")
|
|
213
|
+
role_enum = None
|
|
214
|
+
if role != "infer" and role is not None:
|
|
215
|
+
role_enum = RoleTypes[role]
|
|
216
|
+
|
|
217
|
+
ontology_importer = importers.IMFImporter(filepath=rules_file_path)
|
|
218
|
+
rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
|
|
219
|
+
|
|
220
|
+
if rules is None:
|
|
221
|
+
output_dir = self.config.staging_path
|
|
222
|
+
report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
|
|
223
|
+
report_writer.write_to_file(issues, file_or_dir_path=output_dir)
|
|
224
|
+
report_file = report_writer.default_file_name
|
|
225
|
+
error_text = (
|
|
226
|
+
"<p></p>"
|
|
227
|
+
f'<a href="/data/staging/{report_file}?{time.time()}" '
|
|
228
|
+
f'target="_blank">Failed to validate rules, click here for report</a>'
|
|
229
|
+
)
|
|
230
|
+
return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
|
|
231
|
+
|
|
232
|
+
output_text = "Rules validation passed successfully!"
|
|
233
|
+
|
|
234
|
+
return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
|
|
235
|
+
|
|
236
|
+
|
|
165
237
|
class DMSToRules(Step):
|
|
166
238
|
"""This step imports rules from CDF Data Model"""
|
|
167
239
|
|
|
@@ -199,7 +271,7 @@ class DMSToRules(Step):
|
|
|
199
271
|
|
|
200
272
|
def run(self, cdf_client: CogniteClient) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
|
|
201
273
|
if self.configs is None or self.data_store_path is None:
|
|
202
|
-
raise
|
|
274
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
203
275
|
|
|
204
276
|
datamodel_id_str = self.configs.get("Data model id")
|
|
205
277
|
if datamodel_id_str is None:
|
|
@@ -288,7 +360,7 @@ class RulesInferenceFromRdfFile(Step):
|
|
|
288
360
|
|
|
289
361
|
def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
|
|
290
362
|
if self.configs is None or self.data_store_path is None:
|
|
291
|
-
raise
|
|
363
|
+
raise WorkflowStepNotInitializedError(type(self).__name__)
|
|
292
364
|
|
|
293
365
|
file_path = self.configs.get("File path", None)
|
|
294
366
|
full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
|