cognite-neat 0.87.6__py3-none-any.whl → 0.88.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (125) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/data_classes/rest.py +0 -19
  3. cognite/neat/app/api/explorer.py +6 -4
  4. cognite/neat/app/api/routers/crud.py +11 -21
  5. cognite/neat/app/api/routers/workflows.py +24 -94
  6. cognite/neat/graph/stores/_base.py +5 -0
  7. cognite/neat/rules/importers/_inference2rules.py +31 -35
  8. cognite/neat/workflows/steps/data_contracts.py +17 -43
  9. cognite/neat/workflows/steps/lib/current/graph_extractor.py +28 -24
  10. cognite/neat/workflows/steps/lib/current/graph_loader.py +4 -21
  11. cognite/neat/workflows/steps/lib/current/graph_store.py +18 -134
  12. cognite/neat/workflows/steps_registry.py +5 -7
  13. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/METADATA +1 -1
  14. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/RECORD +17 -125
  15. cognite/neat/app/api/routers/core.py +0 -91
  16. cognite/neat/app/api/routers/data_exploration.py +0 -336
  17. cognite/neat/app/api/routers/rules.py +0 -203
  18. cognite/neat/legacy/__init__.py +0 -0
  19. cognite/neat/legacy/graph/__init__.py +0 -3
  20. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -20182
  21. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44.xml +0 -20163
  22. cognite/neat/legacy/graph/examples/__init__.py +0 -10
  23. cognite/neat/legacy/graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  24. cognite/neat/legacy/graph/exceptions.py +0 -90
  25. cognite/neat/legacy/graph/extractors/__init__.py +0 -6
  26. cognite/neat/legacy/graph/extractors/_base.py +0 -14
  27. cognite/neat/legacy/graph/extractors/_dexpi.py +0 -44
  28. cognite/neat/legacy/graph/extractors/_graph_capturing_sheet.py +0 -403
  29. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +0 -361
  30. cognite/neat/legacy/graph/loaders/__init__.py +0 -23
  31. cognite/neat/legacy/graph/loaders/_asset_loader.py +0 -511
  32. cognite/neat/legacy/graph/loaders/_base.py +0 -67
  33. cognite/neat/legacy/graph/loaders/_exceptions.py +0 -85
  34. cognite/neat/legacy/graph/loaders/core/__init__.py +0 -0
  35. cognite/neat/legacy/graph/loaders/core/labels.py +0 -58
  36. cognite/neat/legacy/graph/loaders/core/models.py +0 -136
  37. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +0 -1046
  38. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +0 -559
  39. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +0 -309
  40. cognite/neat/legacy/graph/loaders/validator.py +0 -87
  41. cognite/neat/legacy/graph/models.py +0 -6
  42. cognite/neat/legacy/graph/stores/__init__.py +0 -13
  43. cognite/neat/legacy/graph/stores/_base.py +0 -400
  44. cognite/neat/legacy/graph/stores/_graphdb_store.py +0 -52
  45. cognite/neat/legacy/graph/stores/_memory_store.py +0 -43
  46. cognite/neat/legacy/graph/stores/_oxigraph_store.py +0 -151
  47. cognite/neat/legacy/graph/stores/_oxrdflib.py +0 -247
  48. cognite/neat/legacy/graph/stores/_rdf_to_graph.py +0 -42
  49. cognite/neat/legacy/graph/transformations/__init__.py +0 -0
  50. cognite/neat/legacy/graph/transformations/entity_matcher.py +0 -101
  51. cognite/neat/legacy/graph/transformations/query_generator/__init__.py +0 -3
  52. cognite/neat/legacy/graph/transformations/query_generator/sparql.py +0 -575
  53. cognite/neat/legacy/graph/transformations/transformer.py +0 -322
  54. cognite/neat/legacy/rules/__init__.py +0 -0
  55. cognite/neat/legacy/rules/analysis.py +0 -231
  56. cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx +0 -0
  57. cognite/neat/legacy/rules/examples/Rules-Nordic44.xlsx +0 -0
  58. cognite/neat/legacy/rules/examples/__init__.py +0 -18
  59. cognite/neat/legacy/rules/examples/power-grid-containers.yaml +0 -124
  60. cognite/neat/legacy/rules/examples/power-grid-example.xlsx +0 -0
  61. cognite/neat/legacy/rules/examples/power-grid-model.yaml +0 -224
  62. cognite/neat/legacy/rules/examples/rules-template.xlsx +0 -0
  63. cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx +0 -0
  64. cognite/neat/legacy/rules/examples/skos-rules.xlsx +0 -0
  65. cognite/neat/legacy/rules/examples/source-to-solution-mapping-rules.xlsx +0 -0
  66. cognite/neat/legacy/rules/examples/wind-energy.owl +0 -1511
  67. cognite/neat/legacy/rules/exceptions.py +0 -2972
  68. cognite/neat/legacy/rules/exporters/__init__.py +0 -20
  69. cognite/neat/legacy/rules/exporters/_base.py +0 -45
  70. cognite/neat/legacy/rules/exporters/_core/__init__.py +0 -5
  71. cognite/neat/legacy/rules/exporters/_core/rules2labels.py +0 -24
  72. cognite/neat/legacy/rules/exporters/_rules2dms.py +0 -885
  73. cognite/neat/legacy/rules/exporters/_rules2excel.py +0 -213
  74. cognite/neat/legacy/rules/exporters/_rules2graphql.py +0 -183
  75. cognite/neat/legacy/rules/exporters/_rules2ontology.py +0 -524
  76. cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +0 -748
  77. cognite/neat/legacy/rules/exporters/_rules2rules.py +0 -105
  78. cognite/neat/legacy/rules/exporters/_rules2triples.py +0 -38
  79. cognite/neat/legacy/rules/exporters/_validation.py +0 -146
  80. cognite/neat/legacy/rules/importers/__init__.py +0 -22
  81. cognite/neat/legacy/rules/importers/_base.py +0 -66
  82. cognite/neat/legacy/rules/importers/_dict2rules.py +0 -158
  83. cognite/neat/legacy/rules/importers/_dms2rules.py +0 -194
  84. cognite/neat/legacy/rules/importers/_graph2rules.py +0 -308
  85. cognite/neat/legacy/rules/importers/_json2rules.py +0 -39
  86. cognite/neat/legacy/rules/importers/_owl2rules/__init__.py +0 -3
  87. cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +0 -239
  88. cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +0 -260
  89. cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +0 -217
  90. cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py +0 -290
  91. cognite/neat/legacy/rules/importers/_spreadsheet2rules.py +0 -45
  92. cognite/neat/legacy/rules/importers/_xsd2rules.py +0 -20
  93. cognite/neat/legacy/rules/importers/_yaml2rules.py +0 -39
  94. cognite/neat/legacy/rules/models/__init__.py +0 -5
  95. cognite/neat/legacy/rules/models/_base.py +0 -151
  96. cognite/neat/legacy/rules/models/raw_rules.py +0 -316
  97. cognite/neat/legacy/rules/models/rdfpath.py +0 -237
  98. cognite/neat/legacy/rules/models/rules.py +0 -1289
  99. cognite/neat/legacy/rules/models/tables.py +0 -9
  100. cognite/neat/legacy/rules/models/value_types.py +0 -118
  101. cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +0 -89
  102. cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
  103. cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
  104. cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  105. cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +0 -65
  106. cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
  107. cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +0 -67
  108. cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  109. cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +0 -95
  110. cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +0 -111
  111. cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  112. cognite/neat/workflows/migration/__init__.py +0 -0
  113. cognite/neat/workflows/migration/steps.py +0 -91
  114. cognite/neat/workflows/migration/wf_manifests.py +0 -33
  115. cognite/neat/workflows/steps/lib/legacy/__init__.py +0 -7
  116. cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py +0 -82
  117. cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +0 -746
  118. cognite/neat/workflows/steps/lib/legacy/graph_loader.py +0 -606
  119. cognite/neat/workflows/steps/lib/legacy/graph_store.py +0 -307
  120. cognite/neat/workflows/steps/lib/legacy/graph_transformer.py +0 -58
  121. cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +0 -511
  122. cognite/neat/workflows/steps/lib/legacy/rules_importer.py +0 -612
  123. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/LICENSE +0 -0
  124. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/WHEEL +0 -0
  125. {cognite_neat-0.87.6.dist-info → cognite_neat-0.88.0.dist-info}/entry_points.txt +0 -0
@@ -1,612 +0,0 @@
1
- import json
2
- import logging
3
- import time
4
- from datetime import datetime
5
- from pathlib import Path
6
- from typing import ClassVar, cast
7
-
8
- import yaml
9
- from prometheus_client import Gauge
10
- from rdflib import Namespace
11
-
12
- from cognite.neat.legacy.rules import exporters, importers
13
- from cognite.neat.legacy.rules.models.rdfpath import TransformationRuleType
14
- from cognite.neat.legacy.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
15
- from cognite.neat.legacy.rules.models.value_types import ValueType
16
- from cognite.neat.utils.auxiliary import generate_exception_report
17
- from cognite.neat.workflows import utils
18
- from cognite.neat.workflows._exceptions import StepNotInitialized
19
- from cognite.neat.workflows.cdf_store import CdfStore
20
- from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
21
- from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
22
- from cognite.neat.workflows.steps.step_model import Configurable, Step
23
-
24
- CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
25
-
26
- __all__ = [
27
- "ImportExcelToRules",
28
- "ImportOpenApiToRules",
29
- "ImportGraphToRules",
30
- "ImportOntologyToRules",
31
- ]
32
-
33
-
34
- class ImportExcelToRules(Step):
35
- """
36
- This step import rules from the Excel file
37
- """
38
-
39
- description = "This step import rules from the Excel file"
40
- version = "legacy"
41
- category = CATEGORY
42
- configurables: ClassVar[list[Configurable]] = [
43
- Configurable(
44
- name="validation_report_storage_dir",
45
- value="rules_validation_report",
46
- label="Directory to store validation report",
47
- ),
48
- Configurable(
49
- name="validation_report_file",
50
- value="rules_validation_report.txt",
51
- label="File name to store validation report",
52
- ),
53
- Configurable(
54
- name="file_name",
55
- value="rules.xlsx",
56
- label="Full name of the rules file in rules folder. If includes path, \
57
- it will be relative to the neat data folder",
58
- ),
59
- Configurable(name="version", value="", label="Optional version of the rules file"),
60
- ]
61
-
62
- def run(self, cdf_store: CdfStore) -> (FlowMessage, RulesData): # type: ignore[syntax, override]
63
- if self.configs is None or self.data_store_path is None:
64
- raise StepNotInitialized(type(self).__name__)
65
- store = cdf_store
66
- # rules file
67
- if self.configs is None:
68
- raise ValueError(f"Step {type(self).__name__} has not been configured.")
69
- rules_file = Path(self.configs["file_name"])
70
- if str(rules_file.parent) == ".":
71
- rules_file_path = self.config.rules_store_path / rules_file
72
- else:
73
- rules_file_path = Path(self.data_store_path) / rules_file
74
-
75
- version = self.configs["version"]
76
-
77
- # rules validation
78
- report_file = self.configs["validation_report_file"]
79
- report_dir_str = self.configs["validation_report_storage_dir"]
80
- report_dir = self.data_store_path / Path(report_dir_str)
81
- report_dir.mkdir(parents=True, exist_ok=True)
82
- report_full_path = report_dir / report_file
83
-
84
- if not rules_file_path.exists():
85
- logging.info(f"Rules files doesn't exist in local fs {rules_file_path}")
86
-
87
- if rules_file_path.exists() and not version:
88
- logging.info(f"Loading rules from {rules_file_path}")
89
- elif rules_file_path.exists() and version:
90
- hash = utils.get_file_hash(rules_file_path)
91
- if hash != version:
92
- store.load_rules_file_from_cdf(str(rules_file), version)
93
- else:
94
- store.load_rules_file_from_cdf(str(rules_file), version)
95
-
96
- raw_rules = importers.ExcelImporter(rules_file_path).to_raw_rules()
97
- rules, errors, _ = raw_rules.to_rules(return_report=True, skip_validation=False)
98
- report = "# RULES VALIDATION REPORT\n\n" + generate_exception_report(errors, "Errors")
99
-
100
- report_full_path.write_text(report)
101
-
102
- text_for_report = (
103
- "<p></p>"
104
- "Download rules validation report "
105
- f'<a href="/data/{report_dir_str}/{report_file}?{time.time()}" '
106
- f'target="_blank">here</a>'
107
- )
108
-
109
- if rules is None:
110
- return FlowMessage(
111
- error_text=f"Failed to load transformation rules! {text_for_report}",
112
- step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
113
- )
114
-
115
- if self.metrics is None:
116
- raise ValueError(f"Step {type(self).__name__} has not been configured.")
117
- rules_metrics = cast(
118
- Gauge,
119
- self.metrics.register_metric(
120
- "data_model_rules", "Transformation rules stats", m_type="gauge", metric_labels=["component"]
121
- ),
122
- )
123
- rules_metrics.labels({"component": "classes"}).set(len(rules.classes))
124
- rules_metrics.labels({"component": "properties"}).set(len(rules.properties))
125
- logging.info(f"Loaded prefixes {rules.prefixes!s} rules from {rules_file_path.name!r}.")
126
- output_text = f"<p></p>Loaded {len(rules.properties)} rules! {text_for_report}"
127
-
128
- return FlowMessage(output_text=output_text), RulesData(rules=rules)
129
-
130
-
131
- class ImportOntologyToRules(Step):
132
- """The step extracts schema from OpenApi/Swagger specification and generates NEAT transformation rules object."""
133
-
134
- description = "The step extracts NEAT rules object from OWL Ontology and \
135
- exports them as an Excel rules files for further editing."
136
- category = CATEGORY
137
- version = "legacy"
138
- configurables: ClassVar[list[Configurable]] = [
139
- Configurable(
140
- name="ontology_file_path", value="staging/ontology.ttl", label="Relative path to the OWL ontology file."
141
- ),
142
- Configurable(
143
- name="excel_file_path", value="staging/rules.xlsx", label="Relative path for the Excel rules storage."
144
- ),
145
- Configurable(
146
- name="make_compliant",
147
- value="True",
148
- label="Relative path for the Excel rules storage.",
149
- options=["True", "False"],
150
- ),
151
- ]
152
-
153
- def run(self) -> FlowMessage: # type: ignore[override, syntax]
154
- ontology_file_path = self.data_store_path / Path(self.configs["ontology_file_path"])
155
- excel_file_path = self.data_store_path / Path(self.configs["excel_file_path"])
156
- report_file_path = excel_file_path.parent / f"report_{excel_file_path.stem}.txt"
157
-
158
- make_compliant = self.configs["make_compliant"] == "True"
159
- try:
160
- rules = importers.OWLImporter(ontology_file_path).to_rules(make_compliant=make_compliant)
161
- except Exception:
162
- rules = importers.OWLImporter(ontology_file_path).to_rules(
163
- skip_validation=True, make_compliant=make_compliant
164
- )
165
- assert isinstance(rules, Rules)
166
- exporters.ExcelExporter.from_rules(rules).export_to_file(excel_file_path)
167
-
168
- if report := importers.ExcelImporter(filepath=excel_file_path).to_raw_rules().validate_rules():
169
- report_file_path.write_text(report)
170
-
171
- relative_excel_file_path = str(excel_file_path).split("/data/")[1]
172
- relative_report_file_path = str(report_file_path).split("/data/")[1]
173
-
174
- output_text = (
175
- "<p></p>"
176
- "Rules imported from OWL Ontology can be downloaded here : "
177
- f'<a href="/data/{relative_excel_file_path}?{time.time()}" '
178
- f'target="_blank">{excel_file_path.stem}.xlsx</a>'
179
- "<p></p>"
180
- "Report can be downloaded here : "
181
- f'<a href="/data/{relative_report_file_path}?{time.time()}" '
182
- f'target="_blank">{report_file_path.stem}.txt</a>'
183
- )
184
-
185
- return FlowMessage(output_text=output_text)
186
-
187
-
188
- class ImportOpenApiToRules(Step):
189
- """The step extracts schema from OpenApi/Swagger specification and generates NEAT transformation rules object."""
190
-
191
- description = "The step extracts schema from OpenAPI specification and generates NEAT transformation rules object. \
192
- The rules object can be serialized to excel file or used directly in other steps."
193
- category = CATEGORY
194
- version = "legacy"
195
- configurables: ClassVar[list[Configurable]] = [
196
- Configurable(
197
- name="openapi_spec_file_path",
198
- value="workflows/openapi_to_rules/source_data/openapi.json",
199
- label="Relative path to the OpenAPI spec file.The file can be in either json or in yaml format.",
200
- ),
201
- Configurable(
202
- name="fdm_compatibility_mode",
203
- value="True",
204
- label="If set to True, the step will try to convert property names to FDM compatible names.",
205
- options=["True", "False"],
206
- ),
207
- ]
208
-
209
- def run(self) -> (FlowMessage, RulesData): # type: ignore[override, syntax]
210
- openapi_file_path = self.data_store_path / Path(self.configs["openapi_spec_file_path"])
211
- self.processed_classes_counter = 0
212
- self.processed_properties_counter = 0
213
- self.failed_classes_counter = 0
214
- self.failed_properties_counter = 0
215
- self.failed_classes: dict[str, str] = {}
216
- self.failed_properties: dict[str, str] = {}
217
- self.is_fdm_compatibility_mode = self.configs["fdm_compatibility_mode"] == "True"
218
- rules = self.open_api_to_rules(openapi_file_path)
219
- report = f" Generated Rules and source data model from OpenApi specs \
220
- <p> Processed {self.processed_classes_counter} classes \
221
- and {self.processed_properties_counter} properties. </p> \
222
- <p> Failed to process {self.failed_classes_counter} classes and \
223
- {self.failed_properties_counter} properties. </p>"
224
- report_obj = {
225
- "processed_classes_counter": self.processed_classes_counter,
226
- "processed_properies_counter": self.processed_properties_counter,
227
- "failed_classes": self.failed_classes,
228
- "failed_properties": self.failed_properties,
229
- }
230
- return (FlowMessage(output_text=report, payload=report_obj), RulesData(rules=rules))
231
-
232
- def open_api_to_rules(self, open_api_spec_file_path: Path) -> Rules:
233
- """Converts OpenAPI spec to NEAT transformation rules object."""
234
- with open_api_spec_file_path.open("r") as openapi_file:
235
- if open_api_spec_file_path.suffix == ".json":
236
- openapi_spec = json.load(openapi_file)
237
- elif open_api_spec_file_path.suffix == ".yaml":
238
- openapi_spec = yaml.safe_load(openapi_file)
239
-
240
- metadata = Metadata(
241
- title="OpenAPI to DM transformation rules",
242
- description="OpenAPI to DM transformation rules",
243
- version="0.1",
244
- creator="Cognite",
245
- created=datetime.utcnow(),
246
- namespace=Namespace("http://purl.org/cognite/neat#"),
247
- prefix="neat",
248
- suffix="OpenAPI",
249
- )
250
-
251
- classes = Classes()
252
- properties = Properties()
253
-
254
- # Loop through OpenAPI components
255
- for component_name, component_info in openapi_spec.get("components", {}).get("schemas", {}).items():
256
- if self.is_fdm_compatibility_mode:
257
- class_name = get_dms_compatible_name(create_fdm_compatibility_class_name(component_name))
258
- else:
259
- class_name = component_name
260
- class_id = class_name
261
- logging.info(f" OpenAPi parser : Processing class {class_id} ")
262
- try:
263
- class_ = Class(
264
- class_id=class_id,
265
- class_name=class_name,
266
- description=component_info.get("description", component_info.get("title", "empty")),
267
- )
268
- classes[class_id] = class_
269
- self.processed_classes_counter += 1
270
- # Loop through properties of OpenApi spec]
271
- self.process_properies(properties, class_id, class_name, component_info)
272
-
273
- except Exception as e:
274
- logging.error(f" OpenAPi parser : Error creating class {class_id}: {e}")
275
- self.failed_classes_counter += 1
276
- self.failed_classes[class_id] = str(e)
277
-
278
- rules = Rules(metadata=metadata, classes=classes, properties=properties, prefixes={}, instances=[])
279
-
280
- return rules
281
-
282
- def process_properies(
283
- self,
284
- rules_properties: Properties,
285
- class_id: str,
286
- class_name: str,
287
- component: dict,
288
- parent_property_name: str | None = None,
289
- ):
290
- # component can have keys : type, description, properties, required, title, allOf, anyOf, oneOf§
291
- logging.info(f" OpenAPi parser : Processing properties for class {class_id} , component {component}")
292
- for component_name, component_info in component.items():
293
- if component_name == "allOf" or component_name == "anyOf" or component_name == "oneOf":
294
- for sub_component in component_info:
295
- self.process_properies(rules_properties, class_id, class_name, sub_component, component_name)
296
- elif component_name == "properties":
297
- for prop_name, prop_info in component_info.items():
298
- prop_id = prop_name
299
- if prop_name == "allOf" or prop_name == "anyOf" or prop_name == "oneOf":
300
- if isinstance(prop_info, list):
301
- prop_type = prop_info[0].get("type", "string")
302
- else:
303
- logging.error(f" !!!!! prop info is not a list . its: {prop_info} ")
304
-
305
- else:
306
- prop_type = prop_info.get("type", "string")
307
- if prop_type == "array":
308
- self.process_properies(
309
- rules_properties, class_id, class_name, prop_info.get("items", {}), prop_name
310
- )
311
- continue
312
- expected_value_type = self.map_open_api_type(prop_type)
313
- if prop_name == "$ref":
314
- ref_class = self.get_ref_class_name(prop_info.get("$ref", ""))
315
- expected_value_type = ref_class
316
- try:
317
- prop = Property(
318
- class_id=class_id,
319
- property_id=get_dms_compatible_name(prop_id) if self.is_fdm_compatibility_mode else prop_id,
320
- property_name=(
321
- get_dms_compatible_name(prop_name) if self.is_fdm_compatibility_mode else prop_name
322
- ),
323
- property_type="ObjectProperty",
324
- description=prop_info.get("description", prop_info.get("title", "empty")),
325
- expected_value_type=ValueType(prefix="neat", suffix=expected_value_type),
326
- cdf_resource_type=["Asset"],
327
- resource_type_property="Asset", # type: ignore
328
- rule_type=TransformationRuleType("rdfpath"),
329
- rule=f"neat:{class_name}(neat:{prop_name})",
330
- label="linked to",
331
- )
332
- self.processed_properties_counter += 1
333
- rules_properties[class_id + prop_id] = prop
334
- except Exception as e:
335
- logging.error(f" OpenAPi parser : Error creating property {prop_id}: {e}")
336
- self.failed_properties_counter += 1
337
- self.failed_properties[prop_id] = str(e)
338
- elif component_name == "$ref":
339
- ref_class = self.get_ref_class_name(component_info)
340
- if parent_property_name is not None:
341
- prop = Property(
342
- class_id=class_id,
343
- property_id=parent_property_name,
344
- property_name=parent_property_name,
345
- property_type="ObjectProperty",
346
- description="no",
347
- expected_value_type=ValueType(prefix="neat", suffix=ref_class),
348
- cdf_resource_type=["Asset"],
349
- resource_type_property="Asset", # type: ignore
350
- rule_type=TransformationRuleType("rdfpath"),
351
- rule=f"neat:{class_name}(neat:{parent_property_name})",
352
- label="linked to",
353
- )
354
- rules_properties[class_id + parent_property_name] = prop
355
-
356
- def get_ref_class_name(self, ref: str) -> str:
357
- ref_payload = ref.split("/")[-1]
358
- if self.is_fdm_compatibility_mode:
359
- return get_dms_compatible_name(create_fdm_compatibility_class_name(ref_payload))
360
- return ref_payload
361
-
362
- def map_open_api_type(self, openapi_type: str) -> str:
363
- """Map OpenAPI type to NEAT compatible types"""
364
- if openapi_type == "object":
365
- datatype = "json"
366
- elif openapi_type == "array":
367
- datatype = "sequence"
368
- elif openapi_type == "number":
369
- datatype = "float"
370
- else:
371
- return openapi_type # Default to string
372
- return datatype
373
-
374
-
375
- class ImportArbitraryJsonYamlToRules(Step):
376
- """The step extracts schema from arbitrary json or yaml file and generates NEAT transformation rules object."""
377
-
378
- description = "The step extracts schema from arbitrary json file and generates NEAT transformation rules object."
379
- category = CATEGORY
380
- version = "legacy"
381
- configurables: ClassVar[list[Configurable]] = [
382
- Configurable(
383
- name="file_path",
384
- value="workflows/openapi_to_rules/data/data.json",
385
- label="Relative path to the json file.The file can be in either json or in yaml format.",
386
- ),
387
- Configurable(
388
- name="fdm_compatibility_mode",
389
- value="True",
390
- label="If set to True, the step will try to convert property names to FDM compatible names.",
391
- options=["True", "False"],
392
- ),
393
- ]
394
-
395
- def run(self) -> (FlowMessage, RulesData): # type: ignore[override, syntax]
396
- openapi_file_path = Path(self.data_store_path) / Path(self.configs["file_path"])
397
- self.processed_classes_counter = 0
398
- self.processed_properies_counter = 0
399
- self.failed_classes_counter = 0
400
- self.failed_properties_counter = 0
401
- self.failed_classes: dict[str, str] = {}
402
- self.failed_properties: dict[str, str] = {}
403
- self.is_fdm_compatibility_mode = self.configs["fdm_compatibility_mode"] == "True"
404
-
405
- rules = self.dict_to_rules(openapi_file_path)
406
- report = f" Generated Rules and source data model from json/yaml \
407
- <p> Processed {self.processed_classes_counter} classes and {self.processed_properies_counter} properties. </p> \
408
- <p> Failed to process {self.failed_classes_counter} classes \
409
- and {self.failed_properties_counter} properties. </p>"
410
- report_obj = {
411
- "processed_classes_counter": self.processed_classes_counter,
412
- "processed_properies_counter": self.processed_properies_counter,
413
- "failed_classes": self.failed_classes,
414
- "failed_properties": self.failed_properties,
415
- }
416
- return FlowMessage(output_text=report, payload=report_obj), RulesData(rules=rules)
417
-
418
- def dict_to_rules(self, open_api_spec_file_path: Path) -> Rules:
419
- """Converts OpenAPI spec to NEAT transformation rules object."""
420
- with open_api_spec_file_path.open("r") as openapi_file:
421
- if open_api_spec_file_path.suffix == ".json":
422
- src_data_obj = json.load(openapi_file)
423
- elif open_api_spec_file_path.suffix == ".yaml":
424
- src_data_obj = yaml.safe_load(openapi_file)
425
-
426
- metadata = Metadata(
427
- title="OpenAPI to DM transformation rules",
428
- description="OpenAPI to DM transformation rules",
429
- version="0.1",
430
- creator="Cognite",
431
- created=datetime.utcnow(),
432
- namespace=Namespace("http://purl.org/cognite/neat#"),
433
- prefix="neat",
434
- suffix="OpenAPI",
435
- )
436
-
437
- self.classes = Classes()
438
- self.properties = Properties()
439
-
440
- self.convert_dict_to_classes_and_props(src_data_obj, None)
441
- rules = Rules(metadata=metadata, classes=self.classes, properties=self.properties, prefixes={}, instances=[])
442
-
443
- return rules
444
-
445
- def add_class(self, class_name: str, description: str | None = None, parent_class_name: str | None = None):
446
- if class_name in self.classes:
447
- return
448
- if self.is_fdm_compatibility_mode:
449
- class_name = get_dms_compatible_name(create_fdm_compatibility_class_name(class_name))
450
- try:
451
- class_ = Class(class_id=class_name, class_name=class_name, description=description)
452
- if parent_class_name:
453
- self.add_property(class_name, "parent", parent_class_name, None)
454
- self.classes[class_name] = class_
455
- self.processed_classes_counter += 1
456
- except Exception as e:
457
- logging.error(f" OpenAPi parser : Error creating class {class_name}: {e}")
458
- self.failed_classes_counter += 1
459
- self.failed_classes[class_name] = str(e)
460
- return
461
-
462
- def add_property(self, class_name: str, property_name: str, property_type: str, description: str | None = None):
463
- if class_name + property_name in self.properties:
464
- return
465
- if self.is_fdm_compatibility_mode:
466
- property_name = get_dms_compatible_name(property_name)
467
- class_name = get_dms_compatible_name(create_fdm_compatibility_class_name(class_name))
468
- try:
469
- prop = Property(
470
- class_id=class_name,
471
- property_id=property_name,
472
- property_name=property_name,
473
- property_type="ObjectProperty",
474
- description=description,
475
- expected_value_type=ValueType(prefix="neat", suffix=property_type),
476
- cdf_resource_type=["Asset"],
477
- resource_type_property="Asset", # type: ignore
478
- rule_type=TransformationRuleType("rdfpath"),
479
- rule=f"neat:{class_name}(neat:{property_name})",
480
- label="linked to",
481
- )
482
- self.properties[class_name + property_name] = prop
483
- self.processed_properies_counter += 1
484
- except Exception as e:
485
- logging.error(f" OpenAPi parser : Error creating property {property_name}: {e}")
486
- self.failed_properties_counter += 1
487
- self.failed_properties[class_name + property_name] = str(e)
488
- return
489
-
490
- # Iterate through the JSON data and convert it to triples
491
- def convert_dict_to_classes_and_props(self, data: dict, parent_property_name=None, grand_parent_property_name=None):
492
- if isinstance(data, dict):
493
- if len(data) == 0:
494
- return
495
- if parent_property_name is None:
496
- for key, value in data.items():
497
- self.convert_dict_to_classes_and_props(value, key)
498
- else:
499
- description = None
500
- self.add_class(parent_property_name, description, grand_parent_property_name)
501
- for key, value in data.items():
502
- self.convert_dict_to_classes_and_props(value, key, parent_property_name)
503
- elif isinstance(data, list):
504
- for item in data:
505
- self.convert_dict_to_classes_and_props(item, parent_property_name, grand_parent_property_name)
506
- else:
507
- # Convert scalar values to RDF literals
508
- data_type = ""
509
- if isinstance(data, bool):
510
- data_type = "boolean"
511
- elif isinstance(data, int):
512
- data_type = "integer"
513
- elif isinstance(data, float):
514
- data_type = "float"
515
- elif isinstance(data, str):
516
- data_type = "string"
517
- else:
518
- data_type = "string"
519
- if grand_parent_property_name is None:
520
- logging.error(" grand_parent_property_name is None")
521
- return
522
-
523
- self.add_property(grand_parent_property_name, parent_property_name, data_type, None)
524
-
525
-
526
- def get_dms_compatible_name(name: str) -> str:
527
- """Converts name to DMS compatible name.It applies both to class and property names"""
528
- # reserverd words in DMS
529
- reserved_words_mapping = {
530
- "space": "src_space",
531
- "externalId": "external_id",
532
- "createdTime": "created_time",
533
- "lastUpdatedTime": "last_updated_time",
534
- "deletedTime": "deleted_time",
535
- "edge_id": "src_edge_id",
536
- "node_id": "src_node_id",
537
- "project_id": "src_project_id",
538
- "property_group": "src_property_group",
539
- "seq": "src_seq",
540
- "tg_table_name": "src__table_name",
541
- "extensions": "src_extensions",
542
- }
543
- if name in reserved_words_mapping:
544
- return reserved_words_mapping[name]
545
- else:
546
- return name.replace(".", "_")
547
-
548
-
549
- def create_fdm_compatibility_class_name(input_string: str):
550
- """Remove underscores and capitalize each word in the string ,
551
- the conversion is done to improve compliance with DMS naming conventions"""
552
-
553
- if "_" in input_string:
554
- words = input_string.split("_") # Split the string by underscores
555
- result = "".join([word.capitalize() for word in words]) # Capitalize each word
556
- return result
557
- else:
558
- return input_string
559
-
560
-
561
- class ImportGraphToRules(Step):
562
- """The step extracts data model from RDF graph and generates NEAT transformation rules object."""
563
-
564
- description = "The step extracts data model from RDF graph and generates NEAT transformation rules object. \
565
- The rules object can be serialized to excel file or used directly in other steps."
566
- category = CATEGORY
567
- version = "legacy"
568
- configurables: ClassVar[list[Configurable]] = [
569
- Configurable(
570
- name="excel_file_path", value="staging/rules.xlsx", label="Relative path for the Excel rules storage."
571
- ),
572
- Configurable(
573
- name="max_number_of_instances",
574
- value="-1",
575
- label="Maximum number of instances per class to process, -1 means all instances",
576
- ),
577
- ]
578
-
579
- def run(self, graph_store: SourceGraph | SolutionGraph) -> FlowMessage: # type: ignore[override, syntax]
580
- excel_file_path = self.data_store_path / Path(self.configs["excel_file_path"])
581
- report_file_path = excel_file_path.parent / f"report_{excel_file_path.stem}.txt"
582
-
583
- try:
584
- rules = importers.GraphImporter(
585
- graph_store.graph.graph, int(self.configs["max_number_of_instances"])
586
- ).to_rules()
587
- except Exception:
588
- rules = importers.GraphImporter(
589
- graph_store.graph.graph, int(self.configs["max_number_of_instances"])
590
- ).to_rules(skip_validation=True)
591
-
592
- assert isinstance(rules, Rules)
593
- exporters.ExcelExporter.from_rules(rules).export_to_file(excel_file_path)
594
-
595
- if report := importers.ExcelImporter(filepath=excel_file_path).to_raw_rules().validate_rules():
596
- report_file_path.write_text(report)
597
-
598
- relative_excel_file_path = str(excel_file_path).split("/data/")[1]
599
- relative_report_file_path = str(report_file_path).split("/data/")[1]
600
-
601
- output_text = (
602
- "<p></p>"
603
- "Rules imported from Graph can be downloaded here : "
604
- f'<a href="/data/{relative_excel_file_path}?{time.time()}" '
605
- f'target="_blank">{excel_file_path.stem}.xlsx</a>'
606
- "<p></p>"
607
- "Report can be downloaded here : "
608
- f'<a href="/data/{relative_report_file_path}?{time.time()}" '
609
- f'target="_blank">{report_file_path.stem}.txt</a>'
610
- )
611
-
612
- return FlowMessage(output_text=output_text)