cognite-neat 0.87.4__py3-none-any.whl → 0.88.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (132) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/data_classes/rest.py +0 -19
  3. cognite/neat/app/api/explorer.py +6 -4
  4. cognite/neat/app/api/routers/crud.py +11 -21
  5. cognite/neat/app/api/routers/workflows.py +24 -94
  6. cognite/neat/graph/extractors/_classic_cdf/_assets.py +8 -2
  7. cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
  8. cognite/neat/graph/loaders/_base.py +17 -12
  9. cognite/neat/graph/loaders/_rdf2asset.py +223 -58
  10. cognite/neat/graph/loaders/_rdf2dms.py +1 -1
  11. cognite/neat/graph/stores/_base.py +5 -0
  12. cognite/neat/rules/analysis/_asset.py +31 -1
  13. cognite/neat/rules/importers/_inference2rules.py +31 -35
  14. cognite/neat/rules/models/information/_rules.py +1 -1
  15. cognite/neat/workflows/steps/data_contracts.py +17 -43
  16. cognite/neat/workflows/steps/lib/current/graph_extractor.py +28 -24
  17. cognite/neat/workflows/steps/lib/current/graph_loader.py +4 -21
  18. cognite/neat/workflows/steps/lib/current/graph_store.py +18 -134
  19. cognite/neat/workflows/steps_registry.py +5 -7
  20. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/METADATA +1 -1
  21. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/RECORD +24 -132
  22. cognite/neat/app/api/routers/core.py +0 -91
  23. cognite/neat/app/api/routers/data_exploration.py +0 -336
  24. cognite/neat/app/api/routers/rules.py +0 -203
  25. cognite/neat/legacy/__init__.py +0 -0
  26. cognite/neat/legacy/graph/__init__.py +0 -3
  27. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -20182
  28. cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44.xml +0 -20163
  29. cognite/neat/legacy/graph/examples/__init__.py +0 -10
  30. cognite/neat/legacy/graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
  31. cognite/neat/legacy/graph/exceptions.py +0 -90
  32. cognite/neat/legacy/graph/extractors/__init__.py +0 -6
  33. cognite/neat/legacy/graph/extractors/_base.py +0 -14
  34. cognite/neat/legacy/graph/extractors/_dexpi.py +0 -44
  35. cognite/neat/legacy/graph/extractors/_graph_capturing_sheet.py +0 -403
  36. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +0 -361
  37. cognite/neat/legacy/graph/loaders/__init__.py +0 -23
  38. cognite/neat/legacy/graph/loaders/_asset_loader.py +0 -511
  39. cognite/neat/legacy/graph/loaders/_base.py +0 -67
  40. cognite/neat/legacy/graph/loaders/_exceptions.py +0 -85
  41. cognite/neat/legacy/graph/loaders/core/__init__.py +0 -0
  42. cognite/neat/legacy/graph/loaders/core/labels.py +0 -58
  43. cognite/neat/legacy/graph/loaders/core/models.py +0 -136
  44. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +0 -1046
  45. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +0 -559
  46. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +0 -309
  47. cognite/neat/legacy/graph/loaders/validator.py +0 -87
  48. cognite/neat/legacy/graph/models.py +0 -6
  49. cognite/neat/legacy/graph/stores/__init__.py +0 -13
  50. cognite/neat/legacy/graph/stores/_base.py +0 -400
  51. cognite/neat/legacy/graph/stores/_graphdb_store.py +0 -52
  52. cognite/neat/legacy/graph/stores/_memory_store.py +0 -43
  53. cognite/neat/legacy/graph/stores/_oxigraph_store.py +0 -151
  54. cognite/neat/legacy/graph/stores/_oxrdflib.py +0 -247
  55. cognite/neat/legacy/graph/stores/_rdf_to_graph.py +0 -42
  56. cognite/neat/legacy/graph/transformations/__init__.py +0 -0
  57. cognite/neat/legacy/graph/transformations/entity_matcher.py +0 -101
  58. cognite/neat/legacy/graph/transformations/query_generator/__init__.py +0 -3
  59. cognite/neat/legacy/graph/transformations/query_generator/sparql.py +0 -575
  60. cognite/neat/legacy/graph/transformations/transformer.py +0 -322
  61. cognite/neat/legacy/rules/__init__.py +0 -0
  62. cognite/neat/legacy/rules/analysis.py +0 -231
  63. cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx +0 -0
  64. cognite/neat/legacy/rules/examples/Rules-Nordic44.xlsx +0 -0
  65. cognite/neat/legacy/rules/examples/__init__.py +0 -18
  66. cognite/neat/legacy/rules/examples/power-grid-containers.yaml +0 -124
  67. cognite/neat/legacy/rules/examples/power-grid-example.xlsx +0 -0
  68. cognite/neat/legacy/rules/examples/power-grid-model.yaml +0 -224
  69. cognite/neat/legacy/rules/examples/rules-template.xlsx +0 -0
  70. cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx +0 -0
  71. cognite/neat/legacy/rules/examples/skos-rules.xlsx +0 -0
  72. cognite/neat/legacy/rules/examples/source-to-solution-mapping-rules.xlsx +0 -0
  73. cognite/neat/legacy/rules/examples/wind-energy.owl +0 -1511
  74. cognite/neat/legacy/rules/exceptions.py +0 -2972
  75. cognite/neat/legacy/rules/exporters/__init__.py +0 -20
  76. cognite/neat/legacy/rules/exporters/_base.py +0 -45
  77. cognite/neat/legacy/rules/exporters/_core/__init__.py +0 -5
  78. cognite/neat/legacy/rules/exporters/_core/rules2labels.py +0 -24
  79. cognite/neat/legacy/rules/exporters/_rules2dms.py +0 -885
  80. cognite/neat/legacy/rules/exporters/_rules2excel.py +0 -213
  81. cognite/neat/legacy/rules/exporters/_rules2graphql.py +0 -183
  82. cognite/neat/legacy/rules/exporters/_rules2ontology.py +0 -524
  83. cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +0 -748
  84. cognite/neat/legacy/rules/exporters/_rules2rules.py +0 -105
  85. cognite/neat/legacy/rules/exporters/_rules2triples.py +0 -38
  86. cognite/neat/legacy/rules/exporters/_validation.py +0 -146
  87. cognite/neat/legacy/rules/importers/__init__.py +0 -22
  88. cognite/neat/legacy/rules/importers/_base.py +0 -66
  89. cognite/neat/legacy/rules/importers/_dict2rules.py +0 -158
  90. cognite/neat/legacy/rules/importers/_dms2rules.py +0 -194
  91. cognite/neat/legacy/rules/importers/_graph2rules.py +0 -308
  92. cognite/neat/legacy/rules/importers/_json2rules.py +0 -39
  93. cognite/neat/legacy/rules/importers/_owl2rules/__init__.py +0 -3
  94. cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +0 -239
  95. cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +0 -260
  96. cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +0 -217
  97. cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py +0 -290
  98. cognite/neat/legacy/rules/importers/_spreadsheet2rules.py +0 -45
  99. cognite/neat/legacy/rules/importers/_xsd2rules.py +0 -20
  100. cognite/neat/legacy/rules/importers/_yaml2rules.py +0 -39
  101. cognite/neat/legacy/rules/models/__init__.py +0 -5
  102. cognite/neat/legacy/rules/models/_base.py +0 -151
  103. cognite/neat/legacy/rules/models/raw_rules.py +0 -316
  104. cognite/neat/legacy/rules/models/rdfpath.py +0 -237
  105. cognite/neat/legacy/rules/models/rules.py +0 -1289
  106. cognite/neat/legacy/rules/models/tables.py +0 -9
  107. cognite/neat/legacy/rules/models/value_types.py +0 -118
  108. cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +0 -89
  109. cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
  110. cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
  111. cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  112. cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +0 -65
  113. cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
  114. cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +0 -67
  115. cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
  116. cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +0 -95
  117. cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +0 -111
  118. cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
  119. cognite/neat/workflows/migration/__init__.py +0 -0
  120. cognite/neat/workflows/migration/steps.py +0 -91
  121. cognite/neat/workflows/migration/wf_manifests.py +0 -33
  122. cognite/neat/workflows/steps/lib/legacy/__init__.py +0 -7
  123. cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py +0 -82
  124. cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +0 -746
  125. cognite/neat/workflows/steps/lib/legacy/graph_loader.py +0 -606
  126. cognite/neat/workflows/steps/lib/legacy/graph_store.py +0 -307
  127. cognite/neat/workflows/steps/lib/legacy/graph_transformer.py +0 -58
  128. cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +0 -511
  129. cognite/neat/workflows/steps/lib/legacy/rules_importer.py +0 -612
  130. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/LICENSE +0 -0
  131. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/WHEEL +0 -0
  132. {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/entry_points.txt +0 -0
@@ -1,91 +0,0 @@
1
- import json
2
- import shutil
3
- import tempfile
4
- from copy import deepcopy
5
- from pathlib import Path
6
- from typing import cast
7
-
8
- from fastapi import APIRouter, UploadFile
9
-
10
- from cognite.neat.app.api.configuration import NEAT_APP
11
- from cognite.neat.rules import exporters, importers
12
- from cognite.neat.rules.models import DMSRules, RoleTypes
13
-
14
- router = APIRouter()
15
-
16
-
17
- @router.post("/api/core/convert")
18
- async def convert_data_model_to_rules(file: UploadFile):
19
- suffix = Path(cast(str, file.filename)).suffix
20
-
21
- if suffix not in [".xlsx", ".ttl", ".owl", ".json", ".yaml"]:
22
- return {
23
- "filename": None,
24
- "content": None,
25
- "issues": [f"File type {suffix} not supported. Supported types are ['.xlsx', '.ttl', '.json', '.yaml']"],
26
- }
27
-
28
- with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp:
29
- shutil.copyfileobj(file.file, temp) # type: ignore # known issue with mypy
30
- temp_filepath = Path(temp.name)
31
-
32
- # read as Excel rules
33
- if suffix == ".xlsx":
34
- rules, issues = importers.ExcelImporter(filepath=temp_filepath).to_rules(role=RoleTypes.dms)
35
-
36
- # load as OWL
37
- elif suffix in [".ttl", ".owl"]:
38
- rules, issues = importers.OWLImporter(filepath=temp_filepath).to_rules(role=RoleTypes.dms)
39
-
40
- # load as YAML
41
- elif suffix in [".yml", ".yaml"]:
42
- rules, issues = importers.YAMLImporter.from_file(temp_filepath).to_rules(role=RoleTypes.dms)
43
-
44
- # load as JSON
45
- elif suffix == ".json":
46
- with temp_filepath.open() as temp:
47
- json_data = json.load(temp)
48
- rules, issues = importers.YAMLImporter(raw_data=json.loads(json_data)).to_rules(role=RoleTypes.dms)
49
-
50
- # Remove the temporary file
51
- temp_filepath.unlink()
52
-
53
- return {"filename": file.filename, "content": rules.model_dump(by_alias=True) if rules else None, "issues": issues}
54
-
55
-
56
- @router.post("/api/core/rules2dms")
57
- async def convert_rules_to_dms(rules: DMSRules):
58
- dms_schema = exporters.DMSExporter().export(rules)
59
- containers = {f"{container.space}:{container.external_id}": container.dump() for container in dms_schema.containers}
60
- views = {f"{view.space}:{view.external_id}": view.dump() for view in dms_schema.views}
61
-
62
- if views and containers:
63
- _to_visualization_compliant_views(views, containers)
64
-
65
- return {
66
- "views": list(views.values()) if views else None,
67
- "containers": list(containers.values()) if containers else None,
68
- }
69
-
70
-
71
- def _to_visualization_compliant_views(views, containers):
72
- for view in views.values():
73
- for property in view["properties"].values():
74
- # needs coping information from container:
75
- if property.get("container", None) and property["container"]["type"] == "container":
76
- container_id = f"{property['container']['space']}:{property['container']['externalId']}"
77
- container_property_def = deepcopy(
78
- containers[container_id]["properties"][property["containerPropertyIdentifier"]]
79
- )
80
- property["type"] = container_property_def["type"]
81
- container_property_def.pop("type")
82
- property.update(container_property_def)
83
-
84
-
85
- @router.post("/api/core/publish-rules")
86
- async def publish_rules_as_data_model(rules: DMSRules):
87
- if NEAT_APP.cdf_client:
88
- uploaded = exporters.DMSExporter().export_to_cdf(rules, NEAT_APP.cdf_client)
89
- return {"uploaded": uploaded}
90
- else:
91
- return {"uploaded": []}
@@ -1,336 +0,0 @@
1
- import logging
2
- import time
3
- import traceback
4
- from typing import Any, cast
5
-
6
- import rdflib
7
- from fastapi import APIRouter
8
-
9
- from cognite.neat.app.api.asgi.metrics import counter
10
- from cognite.neat.app.api.configuration import CACHE_STORE, NEAT_APP
11
- from cognite.neat.app.api.data_classes.rest import (
12
- DatatypePropertyRequest,
13
- NodesAndEdgesRequest,
14
- QueryRequest,
15
- RuleRequest,
16
- )
17
- from cognite.neat.app.api.utils.data_mapping import rdf_result_to_api_response
18
- from cognite.neat.app.api.utils.query_templates import query_templates
19
- from cognite.neat.legacy.graph.transformations import query_generator
20
- from cognite.neat.utils.rdf_ import remove_namespace_from_uri
21
- from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
22
-
23
- router = APIRouter()
24
-
25
-
26
- @router.get("/api/list-queries")
27
- def list_queries():
28
- counter.inc()
29
- return query_templates
30
-
31
-
32
- @router.post("/api/get-datatype-properties")
33
- def get_datatype_properties(request: DatatypePropertyRequest):
34
- logging.info("Querying datatype properties ordered by usage:")
35
- if "get_datatype_properties" in CACHE_STORE and request.cache:
36
- return CACHE_STORE["get_datatype_properties"]
37
- sparql_query: str = (
38
- "SELECT DISTINCT ?property (count(?o) as ?occurrence ) "
39
- "WHERE { ?s ?property ?o . FILTER(isLiteral(?o))} "
40
- "group by ?property order by DESC(?occurrence)"
41
- )
42
- if request.limit != -1:
43
- query = f"{sparql_query} LIMIT {request.limit}"
44
- else:
45
- query = sparql_query
46
-
47
- results = get_data_from_graph(query, request.graph_name, request.workflow_name)
48
-
49
- try:
50
- datatype_properties = [
51
- {
52
- "id": row[rdflib.Variable("property")],
53
- "count": int(row[rdflib.Variable("occurrence")]),
54
- "name": remove_namespace_from_uri(row[rdflib.Variable("property")]),
55
- }
56
- for row in results["rows"]
57
- ]
58
- except Exception as e:
59
- logging.error(f"Error while parsing datatype properties : {e}")
60
-
61
- merged_result = {
62
- "datatype_properties": datatype_properties,
63
- "error": "",
64
- "elapsed_time_sec": results["elapsed_time_sec"],
65
- "query": query,
66
- }
67
-
68
- if request.cache:
69
- CACHE_STORE["get_datatype_properties"] = merged_result
70
- return merged_result
71
-
72
-
73
- @router.post("/api/get-nodes-and-edges")
74
- def get_nodes_and_edges(request: NodesAndEdgesRequest):
75
- logging.info("Querying graph nodes and edges :")
76
- if "get_nodes_and_edges" in CACHE_STORE and request.cache:
77
- return CACHE_STORE["get_nodes_and_edges"]
78
- elapsed_time_sec: float
79
- edges_result: dict | list
80
- nodes_result: dict | list
81
- """
82
- "nodes": [
83
- {
84
- "node_id": "http://purl.org/nordic44#_a63ce14e-fba0-4f9e-8b59-7ef2fe887ff8",
85
- "node_class": "http://entsoe.eu/CIM/SchemaExtension/3/2#RateTemperature",
86
- "node_name": "-5degreeCelsius"
87
- }],
88
- "edges": [
89
- {
90
- "src_object_ref": "http://purl.org/nordic44#_f1769eaa-9aeb-11e5-91da-b8763fd99c5f",
91
- "conn": "http://iec.ch/TC57/2013/CIM-schema-cim16#OperationalLimitSet.Terminal",
92
- "dst_object_ref": "http://purl.org/nordic44#_2dd90186-bdfb-11e5-94fa-c8f73332c8f4"
93
- },
94
- """
95
- if request.sparql_query:
96
- mixed_result = get_data_from_graph(request.sparql_query, request.graph_name, request.workflow_name)
97
-
98
- edges_result = []
99
- nodes_result = []
100
- try:
101
- nodes_result = [
102
- {
103
- "node_id": v[rdflib.Variable("node_id")],
104
- "node_class": v[rdflib.Variable("node_class")],
105
- "node_name": v[rdflib.Variable("node_name")],
106
- }
107
- for v in mixed_result["rows"]
108
- ]
109
- except Exception as e:
110
- logging.error(f"Error while parsing nodes : {e}")
111
-
112
- try:
113
- edges_result = [
114
- {
115
- "src_object_ref": v[rdflib.Variable("src_object_ref")],
116
- "dst_object_ref": v[rdflib.Variable("dst_object_ref")],
117
- }
118
- for v in mixed_result["rows"]
119
- if rdflib.Variable("src_object_ref") in v and rdflib.Variable("dst_object_ref") in v
120
- ]
121
- except Exception as e:
122
- logging.error(f"Error while parsing edges : {e}")
123
-
124
- query = request.sparql_query
125
- elapsed_time_sec = mixed_result["elapsed_time_sec"]
126
- else:
127
- nodes_filter = ""
128
- edges_dst_filter = ""
129
- edges_src_filter = ""
130
-
131
- if len(request.node_class_filter) > 0:
132
- nodes_filter = "VALUES ?node_class { " + " ".join([f"<{x}>" for x in request.node_class_filter]) + " }"
133
-
134
- if request.node_name_property:
135
- nodes_query = f"SELECT DISTINCT ?node_id ?node_class ?node_name WHERE \
136
- {{ {nodes_filter} \
137
- ?node_id {request.node_name_property} ?node_name . ?node_id a ?node_class }} "
138
- else:
139
- nodes_query = f"SELECT DISTINCT ?node_id ?node_class (?node_id AS ?node_name) WHERE \
140
- {{ {nodes_filter} ?node_id a ?node_class }} "
141
-
142
- if len(request.src_edge_filter) > 0:
143
- edges_src_filter = (
144
- "VALUES ?src_object_class { " + " ".join([f"<{x}>" for x in request.src_edge_filter]) + " }"
145
- )
146
- if len(request.dst_edge_filter) > 0:
147
- edges_dst_filter = (
148
- "VALUES ?dst_object_class { " + " ".join([f"<{x}>" for x in request.dst_edge_filter]) + " }"
149
- )
150
-
151
- edges_query = f"SELECT ?src_object_ref ?conn ?dst_object_ref \
152
- WHERE {{ \
153
- {edges_src_filter} \
154
- {edges_dst_filter} \
155
- ?src_object_ref ?conn ?dst_object_ref . \
156
- ?src_object_ref a ?src_object_class . \
157
- ?dst_object_ref a ?dst_object_class .\
158
- }} "
159
- nodes_query = f"{nodes_query} LIMIT {request.limit}"
160
- edges_query = f"{edges_query} LIMIT {request.limit}"
161
- logging.info(f"Nodes query : {nodes_query}")
162
- logging.info(f"Edges query : {edges_query}")
163
- nodes_result = get_data_from_graph(nodes_query, request.graph_name, request.workflow_name)
164
- edges_result = get_data_from_graph(edges_query, request.graph_name, request.workflow_name)
165
- elapsed_time_sec = nodes_result["elapsed_time_sec"] + edges_result["elapsed_time_sec"]
166
- query = nodes_query + edges_query
167
- nodes_result = nodes_result["rows"]
168
- edges_result = edges_result["rows"]
169
-
170
- merged_result = {
171
- "nodes": nodes_result,
172
- "edges": edges_result,
173
- "error": "",
174
- "elapsed_time_sec": elapsed_time_sec,
175
- "query": query,
176
- }
177
-
178
- if request.cache:
179
- CACHE_STORE["get_nodes_and_edges"] = merged_result
180
- return merged_result
181
-
182
-
183
- @router.post("/api/query")
184
- def query_graph(request: QueryRequest):
185
- logging.info(f"Querying graph {request.graph_name} with query {request.query}")
186
- sparq_query = request.query
187
- return get_data_from_graph(sparq_query, request.graph_name, workflow_name=request.workflow_name)
188
-
189
-
190
- @router.post("/api/execute-rule")
191
- def execute_rule(request: RuleRequest):
192
- if NEAT_APP.workflow_manager is None:
193
- return {"error": "NeatApp is not initialized"}
194
- logging.debug(
195
- f"Executing rule type: { request.rule_type } rule : {request.rule} , workflow : {request.workflow_name} , "
196
- f"graph : {request.graph_name}"
197
- )
198
- workflow = NEAT_APP.workflow_manager.get_workflow(request.workflow_name)
199
- if workflow is None:
200
- return {"error": f"Workflow {request.workflow_name} not found"}
201
-
202
- api_result = {"error": ""}
203
- workflow_context = workflow.get_context()
204
- if workflow_context is None:
205
- return {"error": "Workflow context is not initialized"}
206
- if request.graph_name == "source":
207
- if "SourceGraph" in workflow_context:
208
- graph = cast(SourceGraph, workflow_context["SourceGraph"]).graph
209
- else:
210
- logging.info("Source graph is empty , please load the graph first")
211
- api_result["error"] = "source graph is empty , please load the graph first"
212
- elif request.graph_name == "solution":
213
- if "SolutionGraph" in workflow_context:
214
- graph = cast(SolutionGraph, workflow_context["SolutionGraph"]).graph
215
- else:
216
- logging.info("Solution graph is empty , please load the graph first")
217
- api_result["error"] = "solution graph is empty , please load the graph first"
218
- else:
219
- raise Exception("Unknown graph name")
220
-
221
- if request.rule_type == "rdfpath":
222
- start_time = time.perf_counter()
223
- rules = cast(RulesData, workflow_context["RulesData"]).rules
224
- sparql_query = query_generator.build_sparql_query(
225
- graph, # type: ignore[arg-type]
226
- request.rule,
227
- prefixes=rules.prefixes,
228
- )
229
- else:
230
- logging.error("Unknown rule type")
231
- return {"error": "Unknown rule type"}
232
- stop_time = time.perf_counter()
233
- elapsed_time_sec_1 = stop_time - start_time
234
- logging.info(f"Computed query : {sparql_query} in {elapsed_time_sec_1 * 1000} ms")
235
-
236
- api_result = get_data_from_graph(sparql_query, request.graph_name, workflow_name=request.workflow_name)
237
- api_result["elapsed_time_sec"] += elapsed_time_sec_1
238
- return api_result
239
-
240
-
241
- @router.get("/api/object-properties")
242
- def get_object_properties(reference: str, graph_name: str = "source", workflow_name: str = "default"):
243
- logging.info(f"Querying object-properties from {graph_name} :")
244
- query = f"SELECT ?property ?value WHERE {{ <{reference}> ?property ?value }} ORDER BY ?property"
245
- return get_data_from_graph(query, graph_name, workflow_name=workflow_name)
246
-
247
-
248
- @router.get("/api/search")
249
- def search(
250
- search_str: str, graph_name: str = "source", search_type: str = "value_exact_match", workflow_name: str = "default"
251
- ):
252
- logging.info(f"Search from {graph_name} :")
253
- if search_type == "reference":
254
- query = (
255
- f"SELECT ?class ?reference WHERE {{ {{?reference ?class <{search_str}> . }} "
256
- f"UNION {{ <{search_str}> ?class ?reference }} }} limit 10000"
257
- )
258
-
259
- elif search_type == "value_exact_match":
260
- query = (
261
- f"select ?object_ref ?type ?property ?value where "
262
- f"{{ ?object_ref ?property ?value . ?object_ref a ?type . "
263
- f'FILTER(?value="{search_str}") }} limit 10000'
264
- )
265
- elif search_type == "value_regexp":
266
- query = (
267
- f"select ?object_ref ?type ?property ?value where "
268
- f"{{ ?object_ref ?property ?value . ?object_ref a ?type . "
269
- f'FILTER regex(?value, "{search_str}","i") }} limit 10000'
270
- )
271
- return get_data_from_graph(query, graph_name, workflow_name=workflow_name)
272
-
273
-
274
- @router.get("/api/get-classes")
275
- def get_classes(graph_name: str = "source", workflow_name: str = "default", cache: bool = True):
276
- logging.info(f"Querying graph classes from graph name : {graph_name}, cache : {cache}")
277
- cache_key = f"classes_{graph_name}"
278
- if cache_key in CACHE_STORE and cache:
279
- return CACHE_STORE[cache_key]
280
- query = (
281
- "SELECT ?class (count(?s) as ?instances ) WHERE { ?s a ?class . } " "group by ?class order by DESC(?instances)"
282
- )
283
- api_result = get_data_from_graph(query, graph_name, workflow_name)
284
- CACHE_STORE["get_classes"] = api_result
285
- return api_result
286
-
287
-
288
- def get_data_from_graph(sparq_query: str, graph_name: str = "source", workflow_name: str = "default") -> dict[str, Any]:
289
- if NEAT_APP.workflow_manager is None:
290
- return {"error": "NeatApp is not initialized"}
291
- total_elapsed_time = 0.0
292
- api_result: dict[str, Any] = {"error": ""}
293
- result = None
294
-
295
- try:
296
- logging.info(f"Preparing query :{sparq_query} ")
297
- start_time = time.perf_counter()
298
- workflow = NEAT_APP.workflow_manager.get_workflow(workflow_name)
299
- if workflow is None:
300
- return {"error": f"Workflow {workflow_name} not found"}
301
- workflow_context = workflow.get_context()
302
-
303
- if graph_name == "source":
304
- if "SourceGraph" in workflow_context:
305
- result = cast(SourceGraph, workflow_context["SourceGraph"]).graph.query(sparq_query)
306
- else:
307
- logging.info("Source graph is empty , please load the graph first")
308
- api_result["error"] = "source graph is empty , please load the graph first"
309
- elif graph_name == "solution":
310
- if "SolutionGraph" in workflow_context:
311
- result = cast(SolutionGraph, workflow_context["SolutionGraph"]).graph.query(sparq_query)
312
- else:
313
- logging.info("Solution graph is empty , please load the graph first")
314
- api_result["error"] = "solution graph is empty , please load the graph first"
315
- else:
316
- raise Exception("Unknown graph name")
317
-
318
- stop_time = time.perf_counter()
319
- elapsed_time_sec_1 = stop_time - start_time
320
- logging.info(f"Query prepared in {elapsed_time_sec_1 * 1000} ms")
321
-
322
- start_time = time.perf_counter()
323
- if result:
324
- api_result = rdf_result_to_api_response(result)
325
- stop_time = time.perf_counter()
326
- elapsed_time_sec_2 = stop_time - start_time
327
- total_elapsed_time = elapsed_time_sec_1 + elapsed_time_sec_2
328
- except Exception as e:
329
- logging.error(f"Error while executing query :{e}")
330
- traceback.print_exc()
331
- api_result["error"] = str(e)
332
-
333
- api_result["query"] = sparq_query
334
- api_result["elapsed_time_sec"] = total_elapsed_time
335
- logging.info(f"Data fetched in {total_elapsed_time * 1000} ms")
336
- return api_result
@@ -1,203 +0,0 @@
1
- import logging
2
- from pathlib import Path
3
- from typing import Any, cast
4
-
5
- from fastapi import APIRouter, Response
6
- from rdflib import Namespace
7
-
8
- from cognite.neat.app.api.configuration import NEAT_APP
9
- from cognite.neat.app.api.data_classes.rest import TransformationRulesUpdateRequest
10
- from cognite.neat.legacy.rules import exporters as legacy_exporters
11
- from cognite.neat.legacy.rules import importers as legacy_importers
12
- from cognite.neat.legacy.rules.models._base import EntityTypes
13
- from cognite.neat.legacy.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
14
- from cognite.neat.rules import importers
15
- from cognite.neat.rules.models import RoleTypes
16
- from cognite.neat.workflows.steps.data_contracts import RulesData
17
- from cognite.neat.workflows.steps.lib.current.rules_exporter import RulesToExcel
18
- from cognite.neat.workflows.steps.lib.current.rules_importer import ExcelToRules
19
- from cognite.neat.workflows.steps.lib.legacy.rules_importer import ImportExcelToRules
20
- from cognite.neat.workflows.utils import get_file_hash
21
-
22
- router = APIRouter()
23
-
24
-
25
- @router.get("/api/rules/list")
26
- def get_rules_list():
27
- rules_dir = Path(NEAT_APP.config.rules_store_path)
28
- return {"result": [str(file.name) for file in rules_dir.glob("*.xlsx")]}
29
-
30
-
31
- @router.get("/api/rules")
32
- def get_rules(
33
- sheetname: str = "Properties",
34
- url: str | None = None,
35
- source_type: str | None = None,
36
- orient: str = "columns",
37
- workflow_name: str = "default",
38
- file_name: str | None = None,
39
- version: str | None = None,
40
- as_role: str | None = None,
41
- ) -> dict[str, Any]:
42
- rules_schema_version = ""
43
- if NEAT_APP.cdf_store is None or NEAT_APP.workflow_manager is None:
44
- return {"error": "NeatApp is not initialized"}
45
- if workflow_name != "undefined":
46
- workflow = NEAT_APP.workflow_manager.get_workflow(workflow_name)
47
- if workflow is None:
48
- return {"error": f"Workflow {workflow_name} is not found"}
49
- workflow_definition = workflow.get_workflow_definition()
50
- if not file_name:
51
- for step in workflow_definition.steps:
52
- if step.method == ImportExcelToRules.__name__:
53
- file_name = step.configs["file_name"]
54
- version = step.configs["version"]
55
- break
56
- if step.method == RulesToExcel.__name__ or step.method == ExcelToRules.__name__:
57
- rules_schema_version = "v2"
58
- as_role = step.configs.get("as_role", "")
59
- file_name = step.configs.get("File name", "")
60
- if file_name:
61
- break
62
- file_name = step.configs.get("File path", "")
63
- break
64
-
65
- if not file_name:
66
- return {"error": "File name is not provided"}
67
- rules_file = Path(file_name)
68
- if str(rules_file.parent) == ".":
69
- path = Path(NEAT_APP.config.rules_store_path) / rules_file
70
- else:
71
- path = Path(NEAT_APP.config.data_store_path) / rules_file
72
-
73
- src = "local"
74
- if url:
75
- path = Path(url)
76
-
77
- if path.exists() and not version:
78
- logging.info(f"Loading rules from {path}")
79
- elif path.exists() and version:
80
- hash_ = get_file_hash(path)
81
- if hash_ != version:
82
- NEAT_APP.cdf_store.load_rules_file_from_cdf(file_name, version)
83
- src = "cdf"
84
- else:
85
- NEAT_APP.cdf_store.load_rules_file_from_cdf(file_name, version)
86
- src = "cdf"
87
-
88
- error_text = ""
89
- properties = []
90
- classes = []
91
- remaped_rules = {}
92
- try:
93
- # Trying to load rules V1
94
- if rules_schema_version == "" or rules_schema_version == "v1":
95
- rules = cast(
96
- Rules, legacy_importers.ExcelImporter(path).to_rules(return_report=False, skip_validation=False)
97
- )
98
- properties = [
99
- {
100
- "class": value.class_id,
101
- "property": value.property_id,
102
- "property_description": value.description,
103
- "property_type": (
104
- value.expected_value_type.versioned_id
105
- if value.property_type == EntityTypes.object_property
106
- else value.expected_value_type.suffix
107
- ),
108
- "cdf_resource_type": value.cdf_resource_type,
109
- "cdf_metadata_type": value.resource_type_property,
110
- "rule_type": value.rule_type,
111
- "rule": value.rule,
112
- }
113
- for value in rules.properties.values()
114
- ]
115
-
116
- classes = [
117
- {
118
- "class": value.class_id,
119
- "class_description": value.description,
120
- "cdf_resource_type": value.cdf_resource_type,
121
- }
122
- for value in rules.classes.values()
123
- ]
124
- rules_schema_version = "v1"
125
- remaped_rules = {"properties": properties, "metadata": rules.metadata.model_dump(), "classes": classes}
126
- except Exception as e:
127
- error_text = str(e)
128
-
129
- if rules_schema_version == "" or rules_schema_version == "v2":
130
- try:
131
- role = RoleTypes(as_role) if as_role else None
132
- rules_v2, issues = importers.ExcelImporter(path).to_rules(role=role)
133
- error_text = ""
134
- rules_schema_version = "v2"
135
- if rules_v2:
136
- remaped_rules = rules_v2.model_dump()
137
- except Exception as e:
138
- error_text = str(e)
139
- rules_schema_version = "unknown"
140
-
141
- return {
142
- "rules": remaped_rules,
143
- "file_name": path.name,
144
- "hash": get_file_hash(path),
145
- "error_text": error_text,
146
- "src": src,
147
- "rules_schema_version": rules_schema_version,
148
- }
149
-
150
-
151
- @router.get("/api/rules/from_file")
152
- def get_original_rules_from_file(file_name: str):
153
- # """Endpoint for retrieving raw transformation from file"""
154
- path = Path(NEAT_APP.config.rules_store_path) / file_name
155
- rules = cast(
156
- Rules, legacy_importers.ExcelImporter(filepath=path).to_rules(return_report=False, skip_validation=False)
157
- )
158
- return Response(content=rules.model_dump_json(), media_type="application/json")
159
-
160
-
161
- @router.get("/api/rules/from_workflow")
162
- def get_original_rules_from_workflow(workflow_name: str):
163
- """Endpoing for retrieving transformation from memmory"""
164
- workflow = NEAT_APP.workflow_manager.get_workflow(workflow_name)
165
- if workflow is None:
166
- return {"error": f"Workflow {workflow_name} is not found"}
167
- context = workflow.get_context()
168
- rules_data = context["RulesData"]
169
- if type(rules_data) != RulesData:
170
- return {"error": "RulesData is not found in workflow context"}
171
-
172
- return Response(content=rules_data.rules.model_dump_json(), media_type="application/json")
173
-
174
-
175
- @router.post("/api/rules/model_and_transformations")
176
- def upsert_rules(request: TransformationRulesUpdateRequest):
177
- """Endpoing for updating transformation rules via API . This endpoint is still experimental"""
178
- rules = request.rules_object
179
- rules["metadata"]["namespace"] = Namespace(rules["metadata"]["namespace"])
180
- metadata = Metadata(**rules["metadata"])
181
- classes = Classes()
182
- for class_, val in rules["classes"].items():
183
- classes[class_] = Class(**val)
184
- properties = Properties()
185
-
186
- for prop, val in rules["properties"].items():
187
- val["resource_type_property"] = []
188
- properties[prop] = Property(**val)
189
-
190
- prefixes: dict[str, Namespace] = {}
191
- for prefix, val in rules["prefixes"].items():
192
- prefixes[prefix] = Namespace(val)
193
-
194
- rules = Rules(metadata=metadata, classes=classes, properties=properties, prefixes=prefixes, instances=[])
195
- if request.output_format == "excel":
196
- rules_file = Path(request.file_name)
197
- if str(rules_file.parent) == ".":
198
- path = Path(NEAT_APP.config.rules_store_path) / rules_file
199
- else:
200
- path = Path(NEAT_APP.config.data_store_path) / rules_file
201
-
202
- legacy_exporters.ExcelExporter(rules=rules).export_to_file(path)
203
- return {"status": "ok"}
File without changes
@@ -1,3 +0,0 @@
1
- from .stores import NeatGraphStoreBase
2
-
3
- __all__ = ["NeatGraphStoreBase"]