cognite-neat 0.87.4__py3-none-any.whl → 0.88.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/data_classes/rest.py +0 -19
- cognite/neat/app/api/explorer.py +6 -4
- cognite/neat/app/api/routers/crud.py +11 -21
- cognite/neat/app/api/routers/workflows.py +24 -94
- cognite/neat/graph/extractors/_classic_cdf/_assets.py +8 -2
- cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
- cognite/neat/graph/loaders/_base.py +17 -12
- cognite/neat/graph/loaders/_rdf2asset.py +223 -58
- cognite/neat/graph/loaders/_rdf2dms.py +1 -1
- cognite/neat/graph/stores/_base.py +5 -0
- cognite/neat/rules/analysis/_asset.py +31 -1
- cognite/neat/rules/importers/_inference2rules.py +31 -35
- cognite/neat/rules/models/information/_rules.py +1 -1
- cognite/neat/workflows/steps/data_contracts.py +17 -43
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +28 -24
- cognite/neat/workflows/steps/lib/current/graph_loader.py +4 -21
- cognite/neat/workflows/steps/lib/current/graph_store.py +18 -134
- cognite/neat/workflows/steps_registry.py +5 -7
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/RECORD +24 -132
- cognite/neat/app/api/routers/core.py +0 -91
- cognite/neat/app/api/routers/data_exploration.py +0 -336
- cognite/neat/app/api/routers/rules.py +0 -203
- cognite/neat/legacy/__init__.py +0 -0
- cognite/neat/legacy/graph/__init__.py +0 -3
- cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -20182
- cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44.xml +0 -20163
- cognite/neat/legacy/graph/examples/__init__.py +0 -10
- cognite/neat/legacy/graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- cognite/neat/legacy/graph/exceptions.py +0 -90
- cognite/neat/legacy/graph/extractors/__init__.py +0 -6
- cognite/neat/legacy/graph/extractors/_base.py +0 -14
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -44
- cognite/neat/legacy/graph/extractors/_graph_capturing_sheet.py +0 -403
- cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +0 -361
- cognite/neat/legacy/graph/loaders/__init__.py +0 -23
- cognite/neat/legacy/graph/loaders/_asset_loader.py +0 -511
- cognite/neat/legacy/graph/loaders/_base.py +0 -67
- cognite/neat/legacy/graph/loaders/_exceptions.py +0 -85
- cognite/neat/legacy/graph/loaders/core/__init__.py +0 -0
- cognite/neat/legacy/graph/loaders/core/labels.py +0 -58
- cognite/neat/legacy/graph/loaders/core/models.py +0 -136
- cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +0 -1046
- cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +0 -559
- cognite/neat/legacy/graph/loaders/rdf_to_dms.py +0 -309
- cognite/neat/legacy/graph/loaders/validator.py +0 -87
- cognite/neat/legacy/graph/models.py +0 -6
- cognite/neat/legacy/graph/stores/__init__.py +0 -13
- cognite/neat/legacy/graph/stores/_base.py +0 -400
- cognite/neat/legacy/graph/stores/_graphdb_store.py +0 -52
- cognite/neat/legacy/graph/stores/_memory_store.py +0 -43
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +0 -151
- cognite/neat/legacy/graph/stores/_oxrdflib.py +0 -247
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +0 -42
- cognite/neat/legacy/graph/transformations/__init__.py +0 -0
- cognite/neat/legacy/graph/transformations/entity_matcher.py +0 -101
- cognite/neat/legacy/graph/transformations/query_generator/__init__.py +0 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +0 -575
- cognite/neat/legacy/graph/transformations/transformer.py +0 -322
- cognite/neat/legacy/rules/__init__.py +0 -0
- cognite/neat/legacy/rules/analysis.py +0 -231
- cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx +0 -0
- cognite/neat/legacy/rules/examples/Rules-Nordic44.xlsx +0 -0
- cognite/neat/legacy/rules/examples/__init__.py +0 -18
- cognite/neat/legacy/rules/examples/power-grid-containers.yaml +0 -124
- cognite/neat/legacy/rules/examples/power-grid-example.xlsx +0 -0
- cognite/neat/legacy/rules/examples/power-grid-model.yaml +0 -224
- cognite/neat/legacy/rules/examples/rules-template.xlsx +0 -0
- cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/skos-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/source-to-solution-mapping-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/wind-energy.owl +0 -1511
- cognite/neat/legacy/rules/exceptions.py +0 -2972
- cognite/neat/legacy/rules/exporters/__init__.py +0 -20
- cognite/neat/legacy/rules/exporters/_base.py +0 -45
- cognite/neat/legacy/rules/exporters/_core/__init__.py +0 -5
- cognite/neat/legacy/rules/exporters/_core/rules2labels.py +0 -24
- cognite/neat/legacy/rules/exporters/_rules2dms.py +0 -885
- cognite/neat/legacy/rules/exporters/_rules2excel.py +0 -213
- cognite/neat/legacy/rules/exporters/_rules2graphql.py +0 -183
- cognite/neat/legacy/rules/exporters/_rules2ontology.py +0 -524
- cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +0 -748
- cognite/neat/legacy/rules/exporters/_rules2rules.py +0 -105
- cognite/neat/legacy/rules/exporters/_rules2triples.py +0 -38
- cognite/neat/legacy/rules/exporters/_validation.py +0 -146
- cognite/neat/legacy/rules/importers/__init__.py +0 -22
- cognite/neat/legacy/rules/importers/_base.py +0 -66
- cognite/neat/legacy/rules/importers/_dict2rules.py +0 -158
- cognite/neat/legacy/rules/importers/_dms2rules.py +0 -194
- cognite/neat/legacy/rules/importers/_graph2rules.py +0 -308
- cognite/neat/legacy/rules/importers/_json2rules.py +0 -39
- cognite/neat/legacy/rules/importers/_owl2rules/__init__.py +0 -3
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +0 -239
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +0 -260
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +0 -217
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py +0 -290
- cognite/neat/legacy/rules/importers/_spreadsheet2rules.py +0 -45
- cognite/neat/legacy/rules/importers/_xsd2rules.py +0 -20
- cognite/neat/legacy/rules/importers/_yaml2rules.py +0 -39
- cognite/neat/legacy/rules/models/__init__.py +0 -5
- cognite/neat/legacy/rules/models/_base.py +0 -151
- cognite/neat/legacy/rules/models/raw_rules.py +0 -316
- cognite/neat/legacy/rules/models/rdfpath.py +0 -237
- cognite/neat/legacy/rules/models/rules.py +0 -1289
- cognite/neat/legacy/rules/models/tables.py +0 -9
- cognite/neat/legacy/rules/models/value_types.py +0 -118
- cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +0 -89
- cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
- cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
- cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
- cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +0 -65
- cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
- cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +0 -67
- cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
- cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +0 -95
- cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +0 -111
- cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
- cognite/neat/workflows/migration/__init__.py +0 -0
- cognite/neat/workflows/migration/steps.py +0 -91
- cognite/neat/workflows/migration/wf_manifests.py +0 -33
- cognite/neat/workflows/steps/lib/legacy/__init__.py +0 -7
- cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py +0 -82
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +0 -746
- cognite/neat/workflows/steps/lib/legacy/graph_loader.py +0 -606
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +0 -307
- cognite/neat/workflows/steps/lib/legacy/graph_transformer.py +0 -58
- cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +0 -511
- cognite/neat/workflows/steps/lib/legacy/rules_importer.py +0 -612
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,612 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import logging
|
|
3
|
-
import time
|
|
4
|
-
from datetime import datetime
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from typing import ClassVar, cast
|
|
7
|
-
|
|
8
|
-
import yaml
|
|
9
|
-
from prometheus_client import Gauge
|
|
10
|
-
from rdflib import Namespace
|
|
11
|
-
|
|
12
|
-
from cognite.neat.legacy.rules import exporters, importers
|
|
13
|
-
from cognite.neat.legacy.rules.models.rdfpath import TransformationRuleType
|
|
14
|
-
from cognite.neat.legacy.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
|
|
15
|
-
from cognite.neat.legacy.rules.models.value_types import ValueType
|
|
16
|
-
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
17
|
-
from cognite.neat.workflows import utils
|
|
18
|
-
from cognite.neat.workflows._exceptions import StepNotInitialized
|
|
19
|
-
from cognite.neat.workflows.cdf_store import CdfStore
|
|
20
|
-
from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
|
|
21
|
-
from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
|
|
22
|
-
from cognite.neat.workflows.steps.step_model import Configurable, Step
|
|
23
|
-
|
|
24
|
-
CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
|
|
25
|
-
|
|
26
|
-
__all__ = [
|
|
27
|
-
"ImportExcelToRules",
|
|
28
|
-
"ImportOpenApiToRules",
|
|
29
|
-
"ImportGraphToRules",
|
|
30
|
-
"ImportOntologyToRules",
|
|
31
|
-
]
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
class ImportExcelToRules(Step):
|
|
35
|
-
"""
|
|
36
|
-
This step import rules from the Excel file
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
description = "This step import rules from the Excel file"
|
|
40
|
-
version = "legacy"
|
|
41
|
-
category = CATEGORY
|
|
42
|
-
configurables: ClassVar[list[Configurable]] = [
|
|
43
|
-
Configurable(
|
|
44
|
-
name="validation_report_storage_dir",
|
|
45
|
-
value="rules_validation_report",
|
|
46
|
-
label="Directory to store validation report",
|
|
47
|
-
),
|
|
48
|
-
Configurable(
|
|
49
|
-
name="validation_report_file",
|
|
50
|
-
value="rules_validation_report.txt",
|
|
51
|
-
label="File name to store validation report",
|
|
52
|
-
),
|
|
53
|
-
Configurable(
|
|
54
|
-
name="file_name",
|
|
55
|
-
value="rules.xlsx",
|
|
56
|
-
label="Full name of the rules file in rules folder. If includes path, \
|
|
57
|
-
it will be relative to the neat data folder",
|
|
58
|
-
),
|
|
59
|
-
Configurable(name="version", value="", label="Optional version of the rules file"),
|
|
60
|
-
]
|
|
61
|
-
|
|
62
|
-
def run(self, cdf_store: CdfStore) -> (FlowMessage, RulesData): # type: ignore[syntax, override]
|
|
63
|
-
if self.configs is None or self.data_store_path is None:
|
|
64
|
-
raise StepNotInitialized(type(self).__name__)
|
|
65
|
-
store = cdf_store
|
|
66
|
-
# rules file
|
|
67
|
-
if self.configs is None:
|
|
68
|
-
raise ValueError(f"Step {type(self).__name__} has not been configured.")
|
|
69
|
-
rules_file = Path(self.configs["file_name"])
|
|
70
|
-
if str(rules_file.parent) == ".":
|
|
71
|
-
rules_file_path = self.config.rules_store_path / rules_file
|
|
72
|
-
else:
|
|
73
|
-
rules_file_path = Path(self.data_store_path) / rules_file
|
|
74
|
-
|
|
75
|
-
version = self.configs["version"]
|
|
76
|
-
|
|
77
|
-
# rules validation
|
|
78
|
-
report_file = self.configs["validation_report_file"]
|
|
79
|
-
report_dir_str = self.configs["validation_report_storage_dir"]
|
|
80
|
-
report_dir = self.data_store_path / Path(report_dir_str)
|
|
81
|
-
report_dir.mkdir(parents=True, exist_ok=True)
|
|
82
|
-
report_full_path = report_dir / report_file
|
|
83
|
-
|
|
84
|
-
if not rules_file_path.exists():
|
|
85
|
-
logging.info(f"Rules files doesn't exist in local fs {rules_file_path}")
|
|
86
|
-
|
|
87
|
-
if rules_file_path.exists() and not version:
|
|
88
|
-
logging.info(f"Loading rules from {rules_file_path}")
|
|
89
|
-
elif rules_file_path.exists() and version:
|
|
90
|
-
hash = utils.get_file_hash(rules_file_path)
|
|
91
|
-
if hash != version:
|
|
92
|
-
store.load_rules_file_from_cdf(str(rules_file), version)
|
|
93
|
-
else:
|
|
94
|
-
store.load_rules_file_from_cdf(str(rules_file), version)
|
|
95
|
-
|
|
96
|
-
raw_rules = importers.ExcelImporter(rules_file_path).to_raw_rules()
|
|
97
|
-
rules, errors, _ = raw_rules.to_rules(return_report=True, skip_validation=False)
|
|
98
|
-
report = "# RULES VALIDATION REPORT\n\n" + generate_exception_report(errors, "Errors")
|
|
99
|
-
|
|
100
|
-
report_full_path.write_text(report)
|
|
101
|
-
|
|
102
|
-
text_for_report = (
|
|
103
|
-
"<p></p>"
|
|
104
|
-
"Download rules validation report "
|
|
105
|
-
f'<a href="/data/{report_dir_str}/{report_file}?{time.time()}" '
|
|
106
|
-
f'target="_blank">here</a>'
|
|
107
|
-
)
|
|
108
|
-
|
|
109
|
-
if rules is None:
|
|
110
|
-
return FlowMessage(
|
|
111
|
-
error_text=f"Failed to load transformation rules! {text_for_report}",
|
|
112
|
-
step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
|
|
113
|
-
)
|
|
114
|
-
|
|
115
|
-
if self.metrics is None:
|
|
116
|
-
raise ValueError(f"Step {type(self).__name__} has not been configured.")
|
|
117
|
-
rules_metrics = cast(
|
|
118
|
-
Gauge,
|
|
119
|
-
self.metrics.register_metric(
|
|
120
|
-
"data_model_rules", "Transformation rules stats", m_type="gauge", metric_labels=["component"]
|
|
121
|
-
),
|
|
122
|
-
)
|
|
123
|
-
rules_metrics.labels({"component": "classes"}).set(len(rules.classes))
|
|
124
|
-
rules_metrics.labels({"component": "properties"}).set(len(rules.properties))
|
|
125
|
-
logging.info(f"Loaded prefixes {rules.prefixes!s} rules from {rules_file_path.name!r}.")
|
|
126
|
-
output_text = f"<p></p>Loaded {len(rules.properties)} rules! {text_for_report}"
|
|
127
|
-
|
|
128
|
-
return FlowMessage(output_text=output_text), RulesData(rules=rules)
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
class ImportOntologyToRules(Step):
|
|
132
|
-
"""The step extracts schema from OpenApi/Swagger specification and generates NEAT transformation rules object."""
|
|
133
|
-
|
|
134
|
-
description = "The step extracts NEAT rules object from OWL Ontology and \
|
|
135
|
-
exports them as an Excel rules files for further editing."
|
|
136
|
-
category = CATEGORY
|
|
137
|
-
version = "legacy"
|
|
138
|
-
configurables: ClassVar[list[Configurable]] = [
|
|
139
|
-
Configurable(
|
|
140
|
-
name="ontology_file_path", value="staging/ontology.ttl", label="Relative path to the OWL ontology file."
|
|
141
|
-
),
|
|
142
|
-
Configurable(
|
|
143
|
-
name="excel_file_path", value="staging/rules.xlsx", label="Relative path for the Excel rules storage."
|
|
144
|
-
),
|
|
145
|
-
Configurable(
|
|
146
|
-
name="make_compliant",
|
|
147
|
-
value="True",
|
|
148
|
-
label="Relative path for the Excel rules storage.",
|
|
149
|
-
options=["True", "False"],
|
|
150
|
-
),
|
|
151
|
-
]
|
|
152
|
-
|
|
153
|
-
def run(self) -> FlowMessage: # type: ignore[override, syntax]
|
|
154
|
-
ontology_file_path = self.data_store_path / Path(self.configs["ontology_file_path"])
|
|
155
|
-
excel_file_path = self.data_store_path / Path(self.configs["excel_file_path"])
|
|
156
|
-
report_file_path = excel_file_path.parent / f"report_{excel_file_path.stem}.txt"
|
|
157
|
-
|
|
158
|
-
make_compliant = self.configs["make_compliant"] == "True"
|
|
159
|
-
try:
|
|
160
|
-
rules = importers.OWLImporter(ontology_file_path).to_rules(make_compliant=make_compliant)
|
|
161
|
-
except Exception:
|
|
162
|
-
rules = importers.OWLImporter(ontology_file_path).to_rules(
|
|
163
|
-
skip_validation=True, make_compliant=make_compliant
|
|
164
|
-
)
|
|
165
|
-
assert isinstance(rules, Rules)
|
|
166
|
-
exporters.ExcelExporter.from_rules(rules).export_to_file(excel_file_path)
|
|
167
|
-
|
|
168
|
-
if report := importers.ExcelImporter(filepath=excel_file_path).to_raw_rules().validate_rules():
|
|
169
|
-
report_file_path.write_text(report)
|
|
170
|
-
|
|
171
|
-
relative_excel_file_path = str(excel_file_path).split("/data/")[1]
|
|
172
|
-
relative_report_file_path = str(report_file_path).split("/data/")[1]
|
|
173
|
-
|
|
174
|
-
output_text = (
|
|
175
|
-
"<p></p>"
|
|
176
|
-
"Rules imported from OWL Ontology can be downloaded here : "
|
|
177
|
-
f'<a href="/data/{relative_excel_file_path}?{time.time()}" '
|
|
178
|
-
f'target="_blank">{excel_file_path.stem}.xlsx</a>'
|
|
179
|
-
"<p></p>"
|
|
180
|
-
"Report can be downloaded here : "
|
|
181
|
-
f'<a href="/data/{relative_report_file_path}?{time.time()}" '
|
|
182
|
-
f'target="_blank">{report_file_path.stem}.txt</a>'
|
|
183
|
-
)
|
|
184
|
-
|
|
185
|
-
return FlowMessage(output_text=output_text)
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
class ImportOpenApiToRules(Step):
|
|
189
|
-
"""The step extracts schema from OpenApi/Swagger specification and generates NEAT transformation rules object."""
|
|
190
|
-
|
|
191
|
-
description = "The step extracts schema from OpenAPI specification and generates NEAT transformation rules object. \
|
|
192
|
-
The rules object can be serialized to excel file or used directly in other steps."
|
|
193
|
-
category = CATEGORY
|
|
194
|
-
version = "legacy"
|
|
195
|
-
configurables: ClassVar[list[Configurable]] = [
|
|
196
|
-
Configurable(
|
|
197
|
-
name="openapi_spec_file_path",
|
|
198
|
-
value="workflows/openapi_to_rules/source_data/openapi.json",
|
|
199
|
-
label="Relative path to the OpenAPI spec file.The file can be in either json or in yaml format.",
|
|
200
|
-
),
|
|
201
|
-
Configurable(
|
|
202
|
-
name="fdm_compatibility_mode",
|
|
203
|
-
value="True",
|
|
204
|
-
label="If set to True, the step will try to convert property names to FDM compatible names.",
|
|
205
|
-
options=["True", "False"],
|
|
206
|
-
),
|
|
207
|
-
]
|
|
208
|
-
|
|
209
|
-
def run(self) -> (FlowMessage, RulesData): # type: ignore[override, syntax]
|
|
210
|
-
openapi_file_path = self.data_store_path / Path(self.configs["openapi_spec_file_path"])
|
|
211
|
-
self.processed_classes_counter = 0
|
|
212
|
-
self.processed_properties_counter = 0
|
|
213
|
-
self.failed_classes_counter = 0
|
|
214
|
-
self.failed_properties_counter = 0
|
|
215
|
-
self.failed_classes: dict[str, str] = {}
|
|
216
|
-
self.failed_properties: dict[str, str] = {}
|
|
217
|
-
self.is_fdm_compatibility_mode = self.configs["fdm_compatibility_mode"] == "True"
|
|
218
|
-
rules = self.open_api_to_rules(openapi_file_path)
|
|
219
|
-
report = f" Generated Rules and source data model from OpenApi specs \
|
|
220
|
-
<p> Processed {self.processed_classes_counter} classes \
|
|
221
|
-
and {self.processed_properties_counter} properties. </p> \
|
|
222
|
-
<p> Failed to process {self.failed_classes_counter} classes and \
|
|
223
|
-
{self.failed_properties_counter} properties. </p>"
|
|
224
|
-
report_obj = {
|
|
225
|
-
"processed_classes_counter": self.processed_classes_counter,
|
|
226
|
-
"processed_properies_counter": self.processed_properties_counter,
|
|
227
|
-
"failed_classes": self.failed_classes,
|
|
228
|
-
"failed_properties": self.failed_properties,
|
|
229
|
-
}
|
|
230
|
-
return (FlowMessage(output_text=report, payload=report_obj), RulesData(rules=rules))
|
|
231
|
-
|
|
232
|
-
def open_api_to_rules(self, open_api_spec_file_path: Path) -> Rules:
|
|
233
|
-
"""Converts OpenAPI spec to NEAT transformation rules object."""
|
|
234
|
-
with open_api_spec_file_path.open("r") as openapi_file:
|
|
235
|
-
if open_api_spec_file_path.suffix == ".json":
|
|
236
|
-
openapi_spec = json.load(openapi_file)
|
|
237
|
-
elif open_api_spec_file_path.suffix == ".yaml":
|
|
238
|
-
openapi_spec = yaml.safe_load(openapi_file)
|
|
239
|
-
|
|
240
|
-
metadata = Metadata(
|
|
241
|
-
title="OpenAPI to DM transformation rules",
|
|
242
|
-
description="OpenAPI to DM transformation rules",
|
|
243
|
-
version="0.1",
|
|
244
|
-
creator="Cognite",
|
|
245
|
-
created=datetime.utcnow(),
|
|
246
|
-
namespace=Namespace("http://purl.org/cognite/neat#"),
|
|
247
|
-
prefix="neat",
|
|
248
|
-
suffix="OpenAPI",
|
|
249
|
-
)
|
|
250
|
-
|
|
251
|
-
classes = Classes()
|
|
252
|
-
properties = Properties()
|
|
253
|
-
|
|
254
|
-
# Loop through OpenAPI components
|
|
255
|
-
for component_name, component_info in openapi_spec.get("components", {}).get("schemas", {}).items():
|
|
256
|
-
if self.is_fdm_compatibility_mode:
|
|
257
|
-
class_name = get_dms_compatible_name(create_fdm_compatibility_class_name(component_name))
|
|
258
|
-
else:
|
|
259
|
-
class_name = component_name
|
|
260
|
-
class_id = class_name
|
|
261
|
-
logging.info(f" OpenAPi parser : Processing class {class_id} ")
|
|
262
|
-
try:
|
|
263
|
-
class_ = Class(
|
|
264
|
-
class_id=class_id,
|
|
265
|
-
class_name=class_name,
|
|
266
|
-
description=component_info.get("description", component_info.get("title", "empty")),
|
|
267
|
-
)
|
|
268
|
-
classes[class_id] = class_
|
|
269
|
-
self.processed_classes_counter += 1
|
|
270
|
-
# Loop through properties of OpenApi spec]
|
|
271
|
-
self.process_properies(properties, class_id, class_name, component_info)
|
|
272
|
-
|
|
273
|
-
except Exception as e:
|
|
274
|
-
logging.error(f" OpenAPi parser : Error creating class {class_id}: {e}")
|
|
275
|
-
self.failed_classes_counter += 1
|
|
276
|
-
self.failed_classes[class_id] = str(e)
|
|
277
|
-
|
|
278
|
-
rules = Rules(metadata=metadata, classes=classes, properties=properties, prefixes={}, instances=[])
|
|
279
|
-
|
|
280
|
-
return rules
|
|
281
|
-
|
|
282
|
-
def process_properies(
|
|
283
|
-
self,
|
|
284
|
-
rules_properties: Properties,
|
|
285
|
-
class_id: str,
|
|
286
|
-
class_name: str,
|
|
287
|
-
component: dict,
|
|
288
|
-
parent_property_name: str | None = None,
|
|
289
|
-
):
|
|
290
|
-
# component can have keys : type, description, properties, required, title, allOf, anyOf, oneOf§
|
|
291
|
-
logging.info(f" OpenAPi parser : Processing properties for class {class_id} , component {component}")
|
|
292
|
-
for component_name, component_info in component.items():
|
|
293
|
-
if component_name == "allOf" or component_name == "anyOf" or component_name == "oneOf":
|
|
294
|
-
for sub_component in component_info:
|
|
295
|
-
self.process_properies(rules_properties, class_id, class_name, sub_component, component_name)
|
|
296
|
-
elif component_name == "properties":
|
|
297
|
-
for prop_name, prop_info in component_info.items():
|
|
298
|
-
prop_id = prop_name
|
|
299
|
-
if prop_name == "allOf" or prop_name == "anyOf" or prop_name == "oneOf":
|
|
300
|
-
if isinstance(prop_info, list):
|
|
301
|
-
prop_type = prop_info[0].get("type", "string")
|
|
302
|
-
else:
|
|
303
|
-
logging.error(f" !!!!! prop info is not a list . its: {prop_info} ")
|
|
304
|
-
|
|
305
|
-
else:
|
|
306
|
-
prop_type = prop_info.get("type", "string")
|
|
307
|
-
if prop_type == "array":
|
|
308
|
-
self.process_properies(
|
|
309
|
-
rules_properties, class_id, class_name, prop_info.get("items", {}), prop_name
|
|
310
|
-
)
|
|
311
|
-
continue
|
|
312
|
-
expected_value_type = self.map_open_api_type(prop_type)
|
|
313
|
-
if prop_name == "$ref":
|
|
314
|
-
ref_class = self.get_ref_class_name(prop_info.get("$ref", ""))
|
|
315
|
-
expected_value_type = ref_class
|
|
316
|
-
try:
|
|
317
|
-
prop = Property(
|
|
318
|
-
class_id=class_id,
|
|
319
|
-
property_id=get_dms_compatible_name(prop_id) if self.is_fdm_compatibility_mode else prop_id,
|
|
320
|
-
property_name=(
|
|
321
|
-
get_dms_compatible_name(prop_name) if self.is_fdm_compatibility_mode else prop_name
|
|
322
|
-
),
|
|
323
|
-
property_type="ObjectProperty",
|
|
324
|
-
description=prop_info.get("description", prop_info.get("title", "empty")),
|
|
325
|
-
expected_value_type=ValueType(prefix="neat", suffix=expected_value_type),
|
|
326
|
-
cdf_resource_type=["Asset"],
|
|
327
|
-
resource_type_property="Asset", # type: ignore
|
|
328
|
-
rule_type=TransformationRuleType("rdfpath"),
|
|
329
|
-
rule=f"neat:{class_name}(neat:{prop_name})",
|
|
330
|
-
label="linked to",
|
|
331
|
-
)
|
|
332
|
-
self.processed_properties_counter += 1
|
|
333
|
-
rules_properties[class_id + prop_id] = prop
|
|
334
|
-
except Exception as e:
|
|
335
|
-
logging.error(f" OpenAPi parser : Error creating property {prop_id}: {e}")
|
|
336
|
-
self.failed_properties_counter += 1
|
|
337
|
-
self.failed_properties[prop_id] = str(e)
|
|
338
|
-
elif component_name == "$ref":
|
|
339
|
-
ref_class = self.get_ref_class_name(component_info)
|
|
340
|
-
if parent_property_name is not None:
|
|
341
|
-
prop = Property(
|
|
342
|
-
class_id=class_id,
|
|
343
|
-
property_id=parent_property_name,
|
|
344
|
-
property_name=parent_property_name,
|
|
345
|
-
property_type="ObjectProperty",
|
|
346
|
-
description="no",
|
|
347
|
-
expected_value_type=ValueType(prefix="neat", suffix=ref_class),
|
|
348
|
-
cdf_resource_type=["Asset"],
|
|
349
|
-
resource_type_property="Asset", # type: ignore
|
|
350
|
-
rule_type=TransformationRuleType("rdfpath"),
|
|
351
|
-
rule=f"neat:{class_name}(neat:{parent_property_name})",
|
|
352
|
-
label="linked to",
|
|
353
|
-
)
|
|
354
|
-
rules_properties[class_id + parent_property_name] = prop
|
|
355
|
-
|
|
356
|
-
def get_ref_class_name(self, ref: str) -> str:
|
|
357
|
-
ref_payload = ref.split("/")[-1]
|
|
358
|
-
if self.is_fdm_compatibility_mode:
|
|
359
|
-
return get_dms_compatible_name(create_fdm_compatibility_class_name(ref_payload))
|
|
360
|
-
return ref_payload
|
|
361
|
-
|
|
362
|
-
def map_open_api_type(self, openapi_type: str) -> str:
|
|
363
|
-
"""Map OpenAPI type to NEAT compatible types"""
|
|
364
|
-
if openapi_type == "object":
|
|
365
|
-
datatype = "json"
|
|
366
|
-
elif openapi_type == "array":
|
|
367
|
-
datatype = "sequence"
|
|
368
|
-
elif openapi_type == "number":
|
|
369
|
-
datatype = "float"
|
|
370
|
-
else:
|
|
371
|
-
return openapi_type # Default to string
|
|
372
|
-
return datatype
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
class ImportArbitraryJsonYamlToRules(Step):
|
|
376
|
-
"""The step extracts schema from arbitrary json or yaml file and generates NEAT transformation rules object."""
|
|
377
|
-
|
|
378
|
-
description = "The step extracts schema from arbitrary json file and generates NEAT transformation rules object."
|
|
379
|
-
category = CATEGORY
|
|
380
|
-
version = "legacy"
|
|
381
|
-
configurables: ClassVar[list[Configurable]] = [
|
|
382
|
-
Configurable(
|
|
383
|
-
name="file_path",
|
|
384
|
-
value="workflows/openapi_to_rules/data/data.json",
|
|
385
|
-
label="Relative path to the json file.The file can be in either json or in yaml format.",
|
|
386
|
-
),
|
|
387
|
-
Configurable(
|
|
388
|
-
name="fdm_compatibility_mode",
|
|
389
|
-
value="True",
|
|
390
|
-
label="If set to True, the step will try to convert property names to FDM compatible names.",
|
|
391
|
-
options=["True", "False"],
|
|
392
|
-
),
|
|
393
|
-
]
|
|
394
|
-
|
|
395
|
-
def run(self) -> (FlowMessage, RulesData): # type: ignore[override, syntax]
|
|
396
|
-
openapi_file_path = Path(self.data_store_path) / Path(self.configs["file_path"])
|
|
397
|
-
self.processed_classes_counter = 0
|
|
398
|
-
self.processed_properies_counter = 0
|
|
399
|
-
self.failed_classes_counter = 0
|
|
400
|
-
self.failed_properties_counter = 0
|
|
401
|
-
self.failed_classes: dict[str, str] = {}
|
|
402
|
-
self.failed_properties: dict[str, str] = {}
|
|
403
|
-
self.is_fdm_compatibility_mode = self.configs["fdm_compatibility_mode"] == "True"
|
|
404
|
-
|
|
405
|
-
rules = self.dict_to_rules(openapi_file_path)
|
|
406
|
-
report = f" Generated Rules and source data model from json/yaml \
|
|
407
|
-
<p> Processed {self.processed_classes_counter} classes and {self.processed_properies_counter} properties. </p> \
|
|
408
|
-
<p> Failed to process {self.failed_classes_counter} classes \
|
|
409
|
-
and {self.failed_properties_counter} properties. </p>"
|
|
410
|
-
report_obj = {
|
|
411
|
-
"processed_classes_counter": self.processed_classes_counter,
|
|
412
|
-
"processed_properies_counter": self.processed_properies_counter,
|
|
413
|
-
"failed_classes": self.failed_classes,
|
|
414
|
-
"failed_properties": self.failed_properties,
|
|
415
|
-
}
|
|
416
|
-
return FlowMessage(output_text=report, payload=report_obj), RulesData(rules=rules)
|
|
417
|
-
|
|
418
|
-
def dict_to_rules(self, open_api_spec_file_path: Path) -> Rules:
|
|
419
|
-
"""Converts OpenAPI spec to NEAT transformation rules object."""
|
|
420
|
-
with open_api_spec_file_path.open("r") as openapi_file:
|
|
421
|
-
if open_api_spec_file_path.suffix == ".json":
|
|
422
|
-
src_data_obj = json.load(openapi_file)
|
|
423
|
-
elif open_api_spec_file_path.suffix == ".yaml":
|
|
424
|
-
src_data_obj = yaml.safe_load(openapi_file)
|
|
425
|
-
|
|
426
|
-
metadata = Metadata(
|
|
427
|
-
title="OpenAPI to DM transformation rules",
|
|
428
|
-
description="OpenAPI to DM transformation rules",
|
|
429
|
-
version="0.1",
|
|
430
|
-
creator="Cognite",
|
|
431
|
-
created=datetime.utcnow(),
|
|
432
|
-
namespace=Namespace("http://purl.org/cognite/neat#"),
|
|
433
|
-
prefix="neat",
|
|
434
|
-
suffix="OpenAPI",
|
|
435
|
-
)
|
|
436
|
-
|
|
437
|
-
self.classes = Classes()
|
|
438
|
-
self.properties = Properties()
|
|
439
|
-
|
|
440
|
-
self.convert_dict_to_classes_and_props(src_data_obj, None)
|
|
441
|
-
rules = Rules(metadata=metadata, classes=self.classes, properties=self.properties, prefixes={}, instances=[])
|
|
442
|
-
|
|
443
|
-
return rules
|
|
444
|
-
|
|
445
|
-
def add_class(self, class_name: str, description: str | None = None, parent_class_name: str | None = None):
|
|
446
|
-
if class_name in self.classes:
|
|
447
|
-
return
|
|
448
|
-
if self.is_fdm_compatibility_mode:
|
|
449
|
-
class_name = get_dms_compatible_name(create_fdm_compatibility_class_name(class_name))
|
|
450
|
-
try:
|
|
451
|
-
class_ = Class(class_id=class_name, class_name=class_name, description=description)
|
|
452
|
-
if parent_class_name:
|
|
453
|
-
self.add_property(class_name, "parent", parent_class_name, None)
|
|
454
|
-
self.classes[class_name] = class_
|
|
455
|
-
self.processed_classes_counter += 1
|
|
456
|
-
except Exception as e:
|
|
457
|
-
logging.error(f" OpenAPi parser : Error creating class {class_name}: {e}")
|
|
458
|
-
self.failed_classes_counter += 1
|
|
459
|
-
self.failed_classes[class_name] = str(e)
|
|
460
|
-
return
|
|
461
|
-
|
|
462
|
-
def add_property(self, class_name: str, property_name: str, property_type: str, description: str | None = None):
|
|
463
|
-
if class_name + property_name in self.properties:
|
|
464
|
-
return
|
|
465
|
-
if self.is_fdm_compatibility_mode:
|
|
466
|
-
property_name = get_dms_compatible_name(property_name)
|
|
467
|
-
class_name = get_dms_compatible_name(create_fdm_compatibility_class_name(class_name))
|
|
468
|
-
try:
|
|
469
|
-
prop = Property(
|
|
470
|
-
class_id=class_name,
|
|
471
|
-
property_id=property_name,
|
|
472
|
-
property_name=property_name,
|
|
473
|
-
property_type="ObjectProperty",
|
|
474
|
-
description=description,
|
|
475
|
-
expected_value_type=ValueType(prefix="neat", suffix=property_type),
|
|
476
|
-
cdf_resource_type=["Asset"],
|
|
477
|
-
resource_type_property="Asset", # type: ignore
|
|
478
|
-
rule_type=TransformationRuleType("rdfpath"),
|
|
479
|
-
rule=f"neat:{class_name}(neat:{property_name})",
|
|
480
|
-
label="linked to",
|
|
481
|
-
)
|
|
482
|
-
self.properties[class_name + property_name] = prop
|
|
483
|
-
self.processed_properies_counter += 1
|
|
484
|
-
except Exception as e:
|
|
485
|
-
logging.error(f" OpenAPi parser : Error creating property {property_name}: {e}")
|
|
486
|
-
self.failed_properties_counter += 1
|
|
487
|
-
self.failed_properties[class_name + property_name] = str(e)
|
|
488
|
-
return
|
|
489
|
-
|
|
490
|
-
# Iterate through the JSON data and convert it to triples
|
|
491
|
-
def convert_dict_to_classes_and_props(self, data: dict, parent_property_name=None, grand_parent_property_name=None):
|
|
492
|
-
if isinstance(data, dict):
|
|
493
|
-
if len(data) == 0:
|
|
494
|
-
return
|
|
495
|
-
if parent_property_name is None:
|
|
496
|
-
for key, value in data.items():
|
|
497
|
-
self.convert_dict_to_classes_and_props(value, key)
|
|
498
|
-
else:
|
|
499
|
-
description = None
|
|
500
|
-
self.add_class(parent_property_name, description, grand_parent_property_name)
|
|
501
|
-
for key, value in data.items():
|
|
502
|
-
self.convert_dict_to_classes_and_props(value, key, parent_property_name)
|
|
503
|
-
elif isinstance(data, list):
|
|
504
|
-
for item in data:
|
|
505
|
-
self.convert_dict_to_classes_and_props(item, parent_property_name, grand_parent_property_name)
|
|
506
|
-
else:
|
|
507
|
-
# Convert scalar values to RDF literals
|
|
508
|
-
data_type = ""
|
|
509
|
-
if isinstance(data, bool):
|
|
510
|
-
data_type = "boolean"
|
|
511
|
-
elif isinstance(data, int):
|
|
512
|
-
data_type = "integer"
|
|
513
|
-
elif isinstance(data, float):
|
|
514
|
-
data_type = "float"
|
|
515
|
-
elif isinstance(data, str):
|
|
516
|
-
data_type = "string"
|
|
517
|
-
else:
|
|
518
|
-
data_type = "string"
|
|
519
|
-
if grand_parent_property_name is None:
|
|
520
|
-
logging.error(" grand_parent_property_name is None")
|
|
521
|
-
return
|
|
522
|
-
|
|
523
|
-
self.add_property(grand_parent_property_name, parent_property_name, data_type, None)
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
def get_dms_compatible_name(name: str) -> str:
|
|
527
|
-
"""Converts name to DMS compatible name.It applies both to class and property names"""
|
|
528
|
-
# reserverd words in DMS
|
|
529
|
-
reserved_words_mapping = {
|
|
530
|
-
"space": "src_space",
|
|
531
|
-
"externalId": "external_id",
|
|
532
|
-
"createdTime": "created_time",
|
|
533
|
-
"lastUpdatedTime": "last_updated_time",
|
|
534
|
-
"deletedTime": "deleted_time",
|
|
535
|
-
"edge_id": "src_edge_id",
|
|
536
|
-
"node_id": "src_node_id",
|
|
537
|
-
"project_id": "src_project_id",
|
|
538
|
-
"property_group": "src_property_group",
|
|
539
|
-
"seq": "src_seq",
|
|
540
|
-
"tg_table_name": "src__table_name",
|
|
541
|
-
"extensions": "src_extensions",
|
|
542
|
-
}
|
|
543
|
-
if name in reserved_words_mapping:
|
|
544
|
-
return reserved_words_mapping[name]
|
|
545
|
-
else:
|
|
546
|
-
return name.replace(".", "_")
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
def create_fdm_compatibility_class_name(input_string: str):
|
|
550
|
-
"""Remove underscores and capitalize each word in the string ,
|
|
551
|
-
the conversion is done to improve compliance with DMS naming conventions"""
|
|
552
|
-
|
|
553
|
-
if "_" in input_string:
|
|
554
|
-
words = input_string.split("_") # Split the string by underscores
|
|
555
|
-
result = "".join([word.capitalize() for word in words]) # Capitalize each word
|
|
556
|
-
return result
|
|
557
|
-
else:
|
|
558
|
-
return input_string
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
class ImportGraphToRules(Step):
|
|
562
|
-
"""The step extracts data model from RDF graph and generates NEAT transformation rules object."""
|
|
563
|
-
|
|
564
|
-
description = "The step extracts data model from RDF graph and generates NEAT transformation rules object. \
|
|
565
|
-
The rules object can be serialized to excel file or used directly in other steps."
|
|
566
|
-
category = CATEGORY
|
|
567
|
-
version = "legacy"
|
|
568
|
-
configurables: ClassVar[list[Configurable]] = [
|
|
569
|
-
Configurable(
|
|
570
|
-
name="excel_file_path", value="staging/rules.xlsx", label="Relative path for the Excel rules storage."
|
|
571
|
-
),
|
|
572
|
-
Configurable(
|
|
573
|
-
name="max_number_of_instances",
|
|
574
|
-
value="-1",
|
|
575
|
-
label="Maximum number of instances per class to process, -1 means all instances",
|
|
576
|
-
),
|
|
577
|
-
]
|
|
578
|
-
|
|
579
|
-
def run(self, graph_store: SourceGraph | SolutionGraph) -> FlowMessage: # type: ignore[override, syntax]
|
|
580
|
-
excel_file_path = self.data_store_path / Path(self.configs["excel_file_path"])
|
|
581
|
-
report_file_path = excel_file_path.parent / f"report_{excel_file_path.stem}.txt"
|
|
582
|
-
|
|
583
|
-
try:
|
|
584
|
-
rules = importers.GraphImporter(
|
|
585
|
-
graph_store.graph.graph, int(self.configs["max_number_of_instances"])
|
|
586
|
-
).to_rules()
|
|
587
|
-
except Exception:
|
|
588
|
-
rules = importers.GraphImporter(
|
|
589
|
-
graph_store.graph.graph, int(self.configs["max_number_of_instances"])
|
|
590
|
-
).to_rules(skip_validation=True)
|
|
591
|
-
|
|
592
|
-
assert isinstance(rules, Rules)
|
|
593
|
-
exporters.ExcelExporter.from_rules(rules).export_to_file(excel_file_path)
|
|
594
|
-
|
|
595
|
-
if report := importers.ExcelImporter(filepath=excel_file_path).to_raw_rules().validate_rules():
|
|
596
|
-
report_file_path.write_text(report)
|
|
597
|
-
|
|
598
|
-
relative_excel_file_path = str(excel_file_path).split("/data/")[1]
|
|
599
|
-
relative_report_file_path = str(report_file_path).split("/data/")[1]
|
|
600
|
-
|
|
601
|
-
output_text = (
|
|
602
|
-
"<p></p>"
|
|
603
|
-
"Rules imported from Graph can be downloaded here : "
|
|
604
|
-
f'<a href="/data/{relative_excel_file_path}?{time.time()}" '
|
|
605
|
-
f'target="_blank">{excel_file_path.stem}.xlsx</a>'
|
|
606
|
-
"<p></p>"
|
|
607
|
-
"Report can be downloaded here : "
|
|
608
|
-
f'<a href="/data/{relative_report_file_path}?{time.time()}" '
|
|
609
|
-
f'target="_blank">{report_file_path.stem}.txt</a>'
|
|
610
|
-
)
|
|
611
|
-
|
|
612
|
-
return FlowMessage(output_text=output_text)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|