cognite-neat 0.87.4__py3-none-any.whl → 0.88.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/data_classes/rest.py +0 -19
- cognite/neat/app/api/explorer.py +6 -4
- cognite/neat/app/api/routers/crud.py +11 -21
- cognite/neat/app/api/routers/workflows.py +24 -94
- cognite/neat/graph/extractors/_classic_cdf/_assets.py +8 -2
- cognite/neat/graph/extractors/_mock_graph_generator.py +2 -2
- cognite/neat/graph/loaders/_base.py +17 -12
- cognite/neat/graph/loaders/_rdf2asset.py +223 -58
- cognite/neat/graph/loaders/_rdf2dms.py +1 -1
- cognite/neat/graph/stores/_base.py +5 -0
- cognite/neat/rules/analysis/_asset.py +31 -1
- cognite/neat/rules/importers/_inference2rules.py +31 -35
- cognite/neat/rules/models/information/_rules.py +1 -1
- cognite/neat/workflows/steps/data_contracts.py +17 -43
- cognite/neat/workflows/steps/lib/current/graph_extractor.py +28 -24
- cognite/neat/workflows/steps/lib/current/graph_loader.py +4 -21
- cognite/neat/workflows/steps/lib/current/graph_store.py +18 -134
- cognite/neat/workflows/steps_registry.py +5 -7
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/RECORD +24 -132
- cognite/neat/app/api/routers/core.py +0 -91
- cognite/neat/app/api/routers/data_exploration.py +0 -336
- cognite/neat/app/api/routers/rules.py +0 -203
- cognite/neat/legacy/__init__.py +0 -0
- cognite/neat/legacy/graph/__init__.py +0 -3
- cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44-dirty.xml +0 -20182
- cognite/neat/legacy/graph/examples/Knowledge-Graph-Nordic44.xml +0 -20163
- cognite/neat/legacy/graph/examples/__init__.py +0 -10
- cognite/neat/legacy/graph/examples/skos-capturing-sheet-wind-topics.xlsx +0 -0
- cognite/neat/legacy/graph/exceptions.py +0 -90
- cognite/neat/legacy/graph/extractors/__init__.py +0 -6
- cognite/neat/legacy/graph/extractors/_base.py +0 -14
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -44
- cognite/neat/legacy/graph/extractors/_graph_capturing_sheet.py +0 -403
- cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +0 -361
- cognite/neat/legacy/graph/loaders/__init__.py +0 -23
- cognite/neat/legacy/graph/loaders/_asset_loader.py +0 -511
- cognite/neat/legacy/graph/loaders/_base.py +0 -67
- cognite/neat/legacy/graph/loaders/_exceptions.py +0 -85
- cognite/neat/legacy/graph/loaders/core/__init__.py +0 -0
- cognite/neat/legacy/graph/loaders/core/labels.py +0 -58
- cognite/neat/legacy/graph/loaders/core/models.py +0 -136
- cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +0 -1046
- cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +0 -559
- cognite/neat/legacy/graph/loaders/rdf_to_dms.py +0 -309
- cognite/neat/legacy/graph/loaders/validator.py +0 -87
- cognite/neat/legacy/graph/models.py +0 -6
- cognite/neat/legacy/graph/stores/__init__.py +0 -13
- cognite/neat/legacy/graph/stores/_base.py +0 -400
- cognite/neat/legacy/graph/stores/_graphdb_store.py +0 -52
- cognite/neat/legacy/graph/stores/_memory_store.py +0 -43
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +0 -151
- cognite/neat/legacy/graph/stores/_oxrdflib.py +0 -247
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +0 -42
- cognite/neat/legacy/graph/transformations/__init__.py +0 -0
- cognite/neat/legacy/graph/transformations/entity_matcher.py +0 -101
- cognite/neat/legacy/graph/transformations/query_generator/__init__.py +0 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +0 -575
- cognite/neat/legacy/graph/transformations/transformer.py +0 -322
- cognite/neat/legacy/rules/__init__.py +0 -0
- cognite/neat/legacy/rules/analysis.py +0 -231
- cognite/neat/legacy/rules/examples/Rules-Nordic44-to-graphql.xlsx +0 -0
- cognite/neat/legacy/rules/examples/Rules-Nordic44.xlsx +0 -0
- cognite/neat/legacy/rules/examples/__init__.py +0 -18
- cognite/neat/legacy/rules/examples/power-grid-containers.yaml +0 -124
- cognite/neat/legacy/rules/examples/power-grid-example.xlsx +0 -0
- cognite/neat/legacy/rules/examples/power-grid-model.yaml +0 -224
- cognite/neat/legacy/rules/examples/rules-template.xlsx +0 -0
- cognite/neat/legacy/rules/examples/sheet2cdf-transformation-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/skos-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/source-to-solution-mapping-rules.xlsx +0 -0
- cognite/neat/legacy/rules/examples/wind-energy.owl +0 -1511
- cognite/neat/legacy/rules/exceptions.py +0 -2972
- cognite/neat/legacy/rules/exporters/__init__.py +0 -20
- cognite/neat/legacy/rules/exporters/_base.py +0 -45
- cognite/neat/legacy/rules/exporters/_core/__init__.py +0 -5
- cognite/neat/legacy/rules/exporters/_core/rules2labels.py +0 -24
- cognite/neat/legacy/rules/exporters/_rules2dms.py +0 -885
- cognite/neat/legacy/rules/exporters/_rules2excel.py +0 -213
- cognite/neat/legacy/rules/exporters/_rules2graphql.py +0 -183
- cognite/neat/legacy/rules/exporters/_rules2ontology.py +0 -524
- cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +0 -748
- cognite/neat/legacy/rules/exporters/_rules2rules.py +0 -105
- cognite/neat/legacy/rules/exporters/_rules2triples.py +0 -38
- cognite/neat/legacy/rules/exporters/_validation.py +0 -146
- cognite/neat/legacy/rules/importers/__init__.py +0 -22
- cognite/neat/legacy/rules/importers/_base.py +0 -66
- cognite/neat/legacy/rules/importers/_dict2rules.py +0 -158
- cognite/neat/legacy/rules/importers/_dms2rules.py +0 -194
- cognite/neat/legacy/rules/importers/_graph2rules.py +0 -308
- cognite/neat/legacy/rules/importers/_json2rules.py +0 -39
- cognite/neat/legacy/rules/importers/_owl2rules/__init__.py +0 -3
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +0 -239
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +0 -260
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +0 -217
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2rules.py +0 -290
- cognite/neat/legacy/rules/importers/_spreadsheet2rules.py +0 -45
- cognite/neat/legacy/rules/importers/_xsd2rules.py +0 -20
- cognite/neat/legacy/rules/importers/_yaml2rules.py +0 -39
- cognite/neat/legacy/rules/models/__init__.py +0 -5
- cognite/neat/legacy/rules/models/_base.py +0 -151
- cognite/neat/legacy/rules/models/raw_rules.py +0 -316
- cognite/neat/legacy/rules/models/rdfpath.py +0 -237
- cognite/neat/legacy/rules/models/rules.py +0 -1289
- cognite/neat/legacy/rules/models/tables.py +0 -9
- cognite/neat/legacy/rules/models/value_types.py +0 -118
- cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +0 -89
- cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
- cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
- cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
- cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +0 -65
- cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
- cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +0 -67
- cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +0 -64
- cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +0 -95
- cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +0 -111
- cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +0 -270
- cognite/neat/workflows/migration/__init__.py +0 -0
- cognite/neat/workflows/migration/steps.py +0 -91
- cognite/neat/workflows/migration/wf_manifests.py +0 -33
- cognite/neat/workflows/steps/lib/legacy/__init__.py +0 -7
- cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py +0 -82
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +0 -746
- cognite/neat/workflows/steps/lib/legacy/graph_loader.py +0 -606
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +0 -307
- cognite/neat/workflows/steps/lib/legacy/graph_transformer.py +0 -58
- cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +0 -511
- cognite/neat/workflows/steps/lib/legacy/rules_importer.py +0 -612
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.87.4.dist-info → cognite_neat-0.88.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import shutil
|
|
3
|
-
import tempfile
|
|
4
|
-
from copy import deepcopy
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from typing import cast
|
|
7
|
-
|
|
8
|
-
from fastapi import APIRouter, UploadFile
|
|
9
|
-
|
|
10
|
-
from cognite.neat.app.api.configuration import NEAT_APP
|
|
11
|
-
from cognite.neat.rules import exporters, importers
|
|
12
|
-
from cognite.neat.rules.models import DMSRules, RoleTypes
|
|
13
|
-
|
|
14
|
-
router = APIRouter()
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
@router.post("/api/core/convert")
|
|
18
|
-
async def convert_data_model_to_rules(file: UploadFile):
|
|
19
|
-
suffix = Path(cast(str, file.filename)).suffix
|
|
20
|
-
|
|
21
|
-
if suffix not in [".xlsx", ".ttl", ".owl", ".json", ".yaml"]:
|
|
22
|
-
return {
|
|
23
|
-
"filename": None,
|
|
24
|
-
"content": None,
|
|
25
|
-
"issues": [f"File type {suffix} not supported. Supported types are ['.xlsx', '.ttl', '.json', '.yaml']"],
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp:
|
|
29
|
-
shutil.copyfileobj(file.file, temp) # type: ignore # known issue with mypy
|
|
30
|
-
temp_filepath = Path(temp.name)
|
|
31
|
-
|
|
32
|
-
# read as Excel rules
|
|
33
|
-
if suffix == ".xlsx":
|
|
34
|
-
rules, issues = importers.ExcelImporter(filepath=temp_filepath).to_rules(role=RoleTypes.dms)
|
|
35
|
-
|
|
36
|
-
# load as OWL
|
|
37
|
-
elif suffix in [".ttl", ".owl"]:
|
|
38
|
-
rules, issues = importers.OWLImporter(filepath=temp_filepath).to_rules(role=RoleTypes.dms)
|
|
39
|
-
|
|
40
|
-
# load as YAML
|
|
41
|
-
elif suffix in [".yml", ".yaml"]:
|
|
42
|
-
rules, issues = importers.YAMLImporter.from_file(temp_filepath).to_rules(role=RoleTypes.dms)
|
|
43
|
-
|
|
44
|
-
# load as JSON
|
|
45
|
-
elif suffix == ".json":
|
|
46
|
-
with temp_filepath.open() as temp:
|
|
47
|
-
json_data = json.load(temp)
|
|
48
|
-
rules, issues = importers.YAMLImporter(raw_data=json.loads(json_data)).to_rules(role=RoleTypes.dms)
|
|
49
|
-
|
|
50
|
-
# Remove the temporary file
|
|
51
|
-
temp_filepath.unlink()
|
|
52
|
-
|
|
53
|
-
return {"filename": file.filename, "content": rules.model_dump(by_alias=True) if rules else None, "issues": issues}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
@router.post("/api/core/rules2dms")
|
|
57
|
-
async def convert_rules_to_dms(rules: DMSRules):
|
|
58
|
-
dms_schema = exporters.DMSExporter().export(rules)
|
|
59
|
-
containers = {f"{container.space}:{container.external_id}": container.dump() for container in dms_schema.containers}
|
|
60
|
-
views = {f"{view.space}:{view.external_id}": view.dump() for view in dms_schema.views}
|
|
61
|
-
|
|
62
|
-
if views and containers:
|
|
63
|
-
_to_visualization_compliant_views(views, containers)
|
|
64
|
-
|
|
65
|
-
return {
|
|
66
|
-
"views": list(views.values()) if views else None,
|
|
67
|
-
"containers": list(containers.values()) if containers else None,
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def _to_visualization_compliant_views(views, containers):
|
|
72
|
-
for view in views.values():
|
|
73
|
-
for property in view["properties"].values():
|
|
74
|
-
# needs coping information from container:
|
|
75
|
-
if property.get("container", None) and property["container"]["type"] == "container":
|
|
76
|
-
container_id = f"{property['container']['space']}:{property['container']['externalId']}"
|
|
77
|
-
container_property_def = deepcopy(
|
|
78
|
-
containers[container_id]["properties"][property["containerPropertyIdentifier"]]
|
|
79
|
-
)
|
|
80
|
-
property["type"] = container_property_def["type"]
|
|
81
|
-
container_property_def.pop("type")
|
|
82
|
-
property.update(container_property_def)
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
@router.post("/api/core/publish-rules")
|
|
86
|
-
async def publish_rules_as_data_model(rules: DMSRules):
|
|
87
|
-
if NEAT_APP.cdf_client:
|
|
88
|
-
uploaded = exporters.DMSExporter().export_to_cdf(rules, NEAT_APP.cdf_client)
|
|
89
|
-
return {"uploaded": uploaded}
|
|
90
|
-
else:
|
|
91
|
-
return {"uploaded": []}
|
|
@@ -1,336 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
import time
|
|
3
|
-
import traceback
|
|
4
|
-
from typing import Any, cast
|
|
5
|
-
|
|
6
|
-
import rdflib
|
|
7
|
-
from fastapi import APIRouter
|
|
8
|
-
|
|
9
|
-
from cognite.neat.app.api.asgi.metrics import counter
|
|
10
|
-
from cognite.neat.app.api.configuration import CACHE_STORE, NEAT_APP
|
|
11
|
-
from cognite.neat.app.api.data_classes.rest import (
|
|
12
|
-
DatatypePropertyRequest,
|
|
13
|
-
NodesAndEdgesRequest,
|
|
14
|
-
QueryRequest,
|
|
15
|
-
RuleRequest,
|
|
16
|
-
)
|
|
17
|
-
from cognite.neat.app.api.utils.data_mapping import rdf_result_to_api_response
|
|
18
|
-
from cognite.neat.app.api.utils.query_templates import query_templates
|
|
19
|
-
from cognite.neat.legacy.graph.transformations import query_generator
|
|
20
|
-
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
21
|
-
from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
|
|
22
|
-
|
|
23
|
-
router = APIRouter()
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
@router.get("/api/list-queries")
|
|
27
|
-
def list_queries():
|
|
28
|
-
counter.inc()
|
|
29
|
-
return query_templates
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
@router.post("/api/get-datatype-properties")
|
|
33
|
-
def get_datatype_properties(request: DatatypePropertyRequest):
|
|
34
|
-
logging.info("Querying datatype properties ordered by usage:")
|
|
35
|
-
if "get_datatype_properties" in CACHE_STORE and request.cache:
|
|
36
|
-
return CACHE_STORE["get_datatype_properties"]
|
|
37
|
-
sparql_query: str = (
|
|
38
|
-
"SELECT DISTINCT ?property (count(?o) as ?occurrence ) "
|
|
39
|
-
"WHERE { ?s ?property ?o . FILTER(isLiteral(?o))} "
|
|
40
|
-
"group by ?property order by DESC(?occurrence)"
|
|
41
|
-
)
|
|
42
|
-
if request.limit != -1:
|
|
43
|
-
query = f"{sparql_query} LIMIT {request.limit}"
|
|
44
|
-
else:
|
|
45
|
-
query = sparql_query
|
|
46
|
-
|
|
47
|
-
results = get_data_from_graph(query, request.graph_name, request.workflow_name)
|
|
48
|
-
|
|
49
|
-
try:
|
|
50
|
-
datatype_properties = [
|
|
51
|
-
{
|
|
52
|
-
"id": row[rdflib.Variable("property")],
|
|
53
|
-
"count": int(row[rdflib.Variable("occurrence")]),
|
|
54
|
-
"name": remove_namespace_from_uri(row[rdflib.Variable("property")]),
|
|
55
|
-
}
|
|
56
|
-
for row in results["rows"]
|
|
57
|
-
]
|
|
58
|
-
except Exception as e:
|
|
59
|
-
logging.error(f"Error while parsing datatype properties : {e}")
|
|
60
|
-
|
|
61
|
-
merged_result = {
|
|
62
|
-
"datatype_properties": datatype_properties,
|
|
63
|
-
"error": "",
|
|
64
|
-
"elapsed_time_sec": results["elapsed_time_sec"],
|
|
65
|
-
"query": query,
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
if request.cache:
|
|
69
|
-
CACHE_STORE["get_datatype_properties"] = merged_result
|
|
70
|
-
return merged_result
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
@router.post("/api/get-nodes-and-edges")
|
|
74
|
-
def get_nodes_and_edges(request: NodesAndEdgesRequest):
|
|
75
|
-
logging.info("Querying graph nodes and edges :")
|
|
76
|
-
if "get_nodes_and_edges" in CACHE_STORE and request.cache:
|
|
77
|
-
return CACHE_STORE["get_nodes_and_edges"]
|
|
78
|
-
elapsed_time_sec: float
|
|
79
|
-
edges_result: dict | list
|
|
80
|
-
nodes_result: dict | list
|
|
81
|
-
"""
|
|
82
|
-
"nodes": [
|
|
83
|
-
{
|
|
84
|
-
"node_id": "http://purl.org/nordic44#_a63ce14e-fba0-4f9e-8b59-7ef2fe887ff8",
|
|
85
|
-
"node_class": "http://entsoe.eu/CIM/SchemaExtension/3/2#RateTemperature",
|
|
86
|
-
"node_name": "-5degreeCelsius"
|
|
87
|
-
}],
|
|
88
|
-
"edges": [
|
|
89
|
-
{
|
|
90
|
-
"src_object_ref": "http://purl.org/nordic44#_f1769eaa-9aeb-11e5-91da-b8763fd99c5f",
|
|
91
|
-
"conn": "http://iec.ch/TC57/2013/CIM-schema-cim16#OperationalLimitSet.Terminal",
|
|
92
|
-
"dst_object_ref": "http://purl.org/nordic44#_2dd90186-bdfb-11e5-94fa-c8f73332c8f4"
|
|
93
|
-
},
|
|
94
|
-
"""
|
|
95
|
-
if request.sparql_query:
|
|
96
|
-
mixed_result = get_data_from_graph(request.sparql_query, request.graph_name, request.workflow_name)
|
|
97
|
-
|
|
98
|
-
edges_result = []
|
|
99
|
-
nodes_result = []
|
|
100
|
-
try:
|
|
101
|
-
nodes_result = [
|
|
102
|
-
{
|
|
103
|
-
"node_id": v[rdflib.Variable("node_id")],
|
|
104
|
-
"node_class": v[rdflib.Variable("node_class")],
|
|
105
|
-
"node_name": v[rdflib.Variable("node_name")],
|
|
106
|
-
}
|
|
107
|
-
for v in mixed_result["rows"]
|
|
108
|
-
]
|
|
109
|
-
except Exception as e:
|
|
110
|
-
logging.error(f"Error while parsing nodes : {e}")
|
|
111
|
-
|
|
112
|
-
try:
|
|
113
|
-
edges_result = [
|
|
114
|
-
{
|
|
115
|
-
"src_object_ref": v[rdflib.Variable("src_object_ref")],
|
|
116
|
-
"dst_object_ref": v[rdflib.Variable("dst_object_ref")],
|
|
117
|
-
}
|
|
118
|
-
for v in mixed_result["rows"]
|
|
119
|
-
if rdflib.Variable("src_object_ref") in v and rdflib.Variable("dst_object_ref") in v
|
|
120
|
-
]
|
|
121
|
-
except Exception as e:
|
|
122
|
-
logging.error(f"Error while parsing edges : {e}")
|
|
123
|
-
|
|
124
|
-
query = request.sparql_query
|
|
125
|
-
elapsed_time_sec = mixed_result["elapsed_time_sec"]
|
|
126
|
-
else:
|
|
127
|
-
nodes_filter = ""
|
|
128
|
-
edges_dst_filter = ""
|
|
129
|
-
edges_src_filter = ""
|
|
130
|
-
|
|
131
|
-
if len(request.node_class_filter) > 0:
|
|
132
|
-
nodes_filter = "VALUES ?node_class { " + " ".join([f"<{x}>" for x in request.node_class_filter]) + " }"
|
|
133
|
-
|
|
134
|
-
if request.node_name_property:
|
|
135
|
-
nodes_query = f"SELECT DISTINCT ?node_id ?node_class ?node_name WHERE \
|
|
136
|
-
{{ {nodes_filter} \
|
|
137
|
-
?node_id {request.node_name_property} ?node_name . ?node_id a ?node_class }} "
|
|
138
|
-
else:
|
|
139
|
-
nodes_query = f"SELECT DISTINCT ?node_id ?node_class (?node_id AS ?node_name) WHERE \
|
|
140
|
-
{{ {nodes_filter} ?node_id a ?node_class }} "
|
|
141
|
-
|
|
142
|
-
if len(request.src_edge_filter) > 0:
|
|
143
|
-
edges_src_filter = (
|
|
144
|
-
"VALUES ?src_object_class { " + " ".join([f"<{x}>" for x in request.src_edge_filter]) + " }"
|
|
145
|
-
)
|
|
146
|
-
if len(request.dst_edge_filter) > 0:
|
|
147
|
-
edges_dst_filter = (
|
|
148
|
-
"VALUES ?dst_object_class { " + " ".join([f"<{x}>" for x in request.dst_edge_filter]) + " }"
|
|
149
|
-
)
|
|
150
|
-
|
|
151
|
-
edges_query = f"SELECT ?src_object_ref ?conn ?dst_object_ref \
|
|
152
|
-
WHERE {{ \
|
|
153
|
-
{edges_src_filter} \
|
|
154
|
-
{edges_dst_filter} \
|
|
155
|
-
?src_object_ref ?conn ?dst_object_ref . \
|
|
156
|
-
?src_object_ref a ?src_object_class . \
|
|
157
|
-
?dst_object_ref a ?dst_object_class .\
|
|
158
|
-
}} "
|
|
159
|
-
nodes_query = f"{nodes_query} LIMIT {request.limit}"
|
|
160
|
-
edges_query = f"{edges_query} LIMIT {request.limit}"
|
|
161
|
-
logging.info(f"Nodes query : {nodes_query}")
|
|
162
|
-
logging.info(f"Edges query : {edges_query}")
|
|
163
|
-
nodes_result = get_data_from_graph(nodes_query, request.graph_name, request.workflow_name)
|
|
164
|
-
edges_result = get_data_from_graph(edges_query, request.graph_name, request.workflow_name)
|
|
165
|
-
elapsed_time_sec = nodes_result["elapsed_time_sec"] + edges_result["elapsed_time_sec"]
|
|
166
|
-
query = nodes_query + edges_query
|
|
167
|
-
nodes_result = nodes_result["rows"]
|
|
168
|
-
edges_result = edges_result["rows"]
|
|
169
|
-
|
|
170
|
-
merged_result = {
|
|
171
|
-
"nodes": nodes_result,
|
|
172
|
-
"edges": edges_result,
|
|
173
|
-
"error": "",
|
|
174
|
-
"elapsed_time_sec": elapsed_time_sec,
|
|
175
|
-
"query": query,
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
if request.cache:
|
|
179
|
-
CACHE_STORE["get_nodes_and_edges"] = merged_result
|
|
180
|
-
return merged_result
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
@router.post("/api/query")
|
|
184
|
-
def query_graph(request: QueryRequest):
|
|
185
|
-
logging.info(f"Querying graph {request.graph_name} with query {request.query}")
|
|
186
|
-
sparq_query = request.query
|
|
187
|
-
return get_data_from_graph(sparq_query, request.graph_name, workflow_name=request.workflow_name)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
@router.post("/api/execute-rule")
|
|
191
|
-
def execute_rule(request: RuleRequest):
|
|
192
|
-
if NEAT_APP.workflow_manager is None:
|
|
193
|
-
return {"error": "NeatApp is not initialized"}
|
|
194
|
-
logging.debug(
|
|
195
|
-
f"Executing rule type: { request.rule_type } rule : {request.rule} , workflow : {request.workflow_name} , "
|
|
196
|
-
f"graph : {request.graph_name}"
|
|
197
|
-
)
|
|
198
|
-
workflow = NEAT_APP.workflow_manager.get_workflow(request.workflow_name)
|
|
199
|
-
if workflow is None:
|
|
200
|
-
return {"error": f"Workflow {request.workflow_name} not found"}
|
|
201
|
-
|
|
202
|
-
api_result = {"error": ""}
|
|
203
|
-
workflow_context = workflow.get_context()
|
|
204
|
-
if workflow_context is None:
|
|
205
|
-
return {"error": "Workflow context is not initialized"}
|
|
206
|
-
if request.graph_name == "source":
|
|
207
|
-
if "SourceGraph" in workflow_context:
|
|
208
|
-
graph = cast(SourceGraph, workflow_context["SourceGraph"]).graph
|
|
209
|
-
else:
|
|
210
|
-
logging.info("Source graph is empty , please load the graph first")
|
|
211
|
-
api_result["error"] = "source graph is empty , please load the graph first"
|
|
212
|
-
elif request.graph_name == "solution":
|
|
213
|
-
if "SolutionGraph" in workflow_context:
|
|
214
|
-
graph = cast(SolutionGraph, workflow_context["SolutionGraph"]).graph
|
|
215
|
-
else:
|
|
216
|
-
logging.info("Solution graph is empty , please load the graph first")
|
|
217
|
-
api_result["error"] = "solution graph is empty , please load the graph first"
|
|
218
|
-
else:
|
|
219
|
-
raise Exception("Unknown graph name")
|
|
220
|
-
|
|
221
|
-
if request.rule_type == "rdfpath":
|
|
222
|
-
start_time = time.perf_counter()
|
|
223
|
-
rules = cast(RulesData, workflow_context["RulesData"]).rules
|
|
224
|
-
sparql_query = query_generator.build_sparql_query(
|
|
225
|
-
graph, # type: ignore[arg-type]
|
|
226
|
-
request.rule,
|
|
227
|
-
prefixes=rules.prefixes,
|
|
228
|
-
)
|
|
229
|
-
else:
|
|
230
|
-
logging.error("Unknown rule type")
|
|
231
|
-
return {"error": "Unknown rule type"}
|
|
232
|
-
stop_time = time.perf_counter()
|
|
233
|
-
elapsed_time_sec_1 = stop_time - start_time
|
|
234
|
-
logging.info(f"Computed query : {sparql_query} in {elapsed_time_sec_1 * 1000} ms")
|
|
235
|
-
|
|
236
|
-
api_result = get_data_from_graph(sparql_query, request.graph_name, workflow_name=request.workflow_name)
|
|
237
|
-
api_result["elapsed_time_sec"] += elapsed_time_sec_1
|
|
238
|
-
return api_result
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
@router.get("/api/object-properties")
|
|
242
|
-
def get_object_properties(reference: str, graph_name: str = "source", workflow_name: str = "default"):
|
|
243
|
-
logging.info(f"Querying object-properties from {graph_name} :")
|
|
244
|
-
query = f"SELECT ?property ?value WHERE {{ <{reference}> ?property ?value }} ORDER BY ?property"
|
|
245
|
-
return get_data_from_graph(query, graph_name, workflow_name=workflow_name)
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
@router.get("/api/search")
|
|
249
|
-
def search(
|
|
250
|
-
search_str: str, graph_name: str = "source", search_type: str = "value_exact_match", workflow_name: str = "default"
|
|
251
|
-
):
|
|
252
|
-
logging.info(f"Search from {graph_name} :")
|
|
253
|
-
if search_type == "reference":
|
|
254
|
-
query = (
|
|
255
|
-
f"SELECT ?class ?reference WHERE {{ {{?reference ?class <{search_str}> . }} "
|
|
256
|
-
f"UNION {{ <{search_str}> ?class ?reference }} }} limit 10000"
|
|
257
|
-
)
|
|
258
|
-
|
|
259
|
-
elif search_type == "value_exact_match":
|
|
260
|
-
query = (
|
|
261
|
-
f"select ?object_ref ?type ?property ?value where "
|
|
262
|
-
f"{{ ?object_ref ?property ?value . ?object_ref a ?type . "
|
|
263
|
-
f'FILTER(?value="{search_str}") }} limit 10000'
|
|
264
|
-
)
|
|
265
|
-
elif search_type == "value_regexp":
|
|
266
|
-
query = (
|
|
267
|
-
f"select ?object_ref ?type ?property ?value where "
|
|
268
|
-
f"{{ ?object_ref ?property ?value . ?object_ref a ?type . "
|
|
269
|
-
f'FILTER regex(?value, "{search_str}","i") }} limit 10000'
|
|
270
|
-
)
|
|
271
|
-
return get_data_from_graph(query, graph_name, workflow_name=workflow_name)
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
@router.get("/api/get-classes")
|
|
275
|
-
def get_classes(graph_name: str = "source", workflow_name: str = "default", cache: bool = True):
|
|
276
|
-
logging.info(f"Querying graph classes from graph name : {graph_name}, cache : {cache}")
|
|
277
|
-
cache_key = f"classes_{graph_name}"
|
|
278
|
-
if cache_key in CACHE_STORE and cache:
|
|
279
|
-
return CACHE_STORE[cache_key]
|
|
280
|
-
query = (
|
|
281
|
-
"SELECT ?class (count(?s) as ?instances ) WHERE { ?s a ?class . } " "group by ?class order by DESC(?instances)"
|
|
282
|
-
)
|
|
283
|
-
api_result = get_data_from_graph(query, graph_name, workflow_name)
|
|
284
|
-
CACHE_STORE["get_classes"] = api_result
|
|
285
|
-
return api_result
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
def get_data_from_graph(sparq_query: str, graph_name: str = "source", workflow_name: str = "default") -> dict[str, Any]:
|
|
289
|
-
if NEAT_APP.workflow_manager is None:
|
|
290
|
-
return {"error": "NeatApp is not initialized"}
|
|
291
|
-
total_elapsed_time = 0.0
|
|
292
|
-
api_result: dict[str, Any] = {"error": ""}
|
|
293
|
-
result = None
|
|
294
|
-
|
|
295
|
-
try:
|
|
296
|
-
logging.info(f"Preparing query :{sparq_query} ")
|
|
297
|
-
start_time = time.perf_counter()
|
|
298
|
-
workflow = NEAT_APP.workflow_manager.get_workflow(workflow_name)
|
|
299
|
-
if workflow is None:
|
|
300
|
-
return {"error": f"Workflow {workflow_name} not found"}
|
|
301
|
-
workflow_context = workflow.get_context()
|
|
302
|
-
|
|
303
|
-
if graph_name == "source":
|
|
304
|
-
if "SourceGraph" in workflow_context:
|
|
305
|
-
result = cast(SourceGraph, workflow_context["SourceGraph"]).graph.query(sparq_query)
|
|
306
|
-
else:
|
|
307
|
-
logging.info("Source graph is empty , please load the graph first")
|
|
308
|
-
api_result["error"] = "source graph is empty , please load the graph first"
|
|
309
|
-
elif graph_name == "solution":
|
|
310
|
-
if "SolutionGraph" in workflow_context:
|
|
311
|
-
result = cast(SolutionGraph, workflow_context["SolutionGraph"]).graph.query(sparq_query)
|
|
312
|
-
else:
|
|
313
|
-
logging.info("Solution graph is empty , please load the graph first")
|
|
314
|
-
api_result["error"] = "solution graph is empty , please load the graph first"
|
|
315
|
-
else:
|
|
316
|
-
raise Exception("Unknown graph name")
|
|
317
|
-
|
|
318
|
-
stop_time = time.perf_counter()
|
|
319
|
-
elapsed_time_sec_1 = stop_time - start_time
|
|
320
|
-
logging.info(f"Query prepared in {elapsed_time_sec_1 * 1000} ms")
|
|
321
|
-
|
|
322
|
-
start_time = time.perf_counter()
|
|
323
|
-
if result:
|
|
324
|
-
api_result = rdf_result_to_api_response(result)
|
|
325
|
-
stop_time = time.perf_counter()
|
|
326
|
-
elapsed_time_sec_2 = stop_time - start_time
|
|
327
|
-
total_elapsed_time = elapsed_time_sec_1 + elapsed_time_sec_2
|
|
328
|
-
except Exception as e:
|
|
329
|
-
logging.error(f"Error while executing query :{e}")
|
|
330
|
-
traceback.print_exc()
|
|
331
|
-
api_result["error"] = str(e)
|
|
332
|
-
|
|
333
|
-
api_result["query"] = sparq_query
|
|
334
|
-
api_result["elapsed_time_sec"] = total_elapsed_time
|
|
335
|
-
logging.info(f"Data fetched in {total_elapsed_time * 1000} ms")
|
|
336
|
-
return api_result
|
|
@@ -1,203 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
from typing import Any, cast
|
|
4
|
-
|
|
5
|
-
from fastapi import APIRouter, Response
|
|
6
|
-
from rdflib import Namespace
|
|
7
|
-
|
|
8
|
-
from cognite.neat.app.api.configuration import NEAT_APP
|
|
9
|
-
from cognite.neat.app.api.data_classes.rest import TransformationRulesUpdateRequest
|
|
10
|
-
from cognite.neat.legacy.rules import exporters as legacy_exporters
|
|
11
|
-
from cognite.neat.legacy.rules import importers as legacy_importers
|
|
12
|
-
from cognite.neat.legacy.rules.models._base import EntityTypes
|
|
13
|
-
from cognite.neat.legacy.rules.models.rules import Class, Classes, Metadata, Properties, Property, Rules
|
|
14
|
-
from cognite.neat.rules import importers
|
|
15
|
-
from cognite.neat.rules.models import RoleTypes
|
|
16
|
-
from cognite.neat.workflows.steps.data_contracts import RulesData
|
|
17
|
-
from cognite.neat.workflows.steps.lib.current.rules_exporter import RulesToExcel
|
|
18
|
-
from cognite.neat.workflows.steps.lib.current.rules_importer import ExcelToRules
|
|
19
|
-
from cognite.neat.workflows.steps.lib.legacy.rules_importer import ImportExcelToRules
|
|
20
|
-
from cognite.neat.workflows.utils import get_file_hash
|
|
21
|
-
|
|
22
|
-
router = APIRouter()
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
@router.get("/api/rules/list")
|
|
26
|
-
def get_rules_list():
|
|
27
|
-
rules_dir = Path(NEAT_APP.config.rules_store_path)
|
|
28
|
-
return {"result": [str(file.name) for file in rules_dir.glob("*.xlsx")]}
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
@router.get("/api/rules")
|
|
32
|
-
def get_rules(
|
|
33
|
-
sheetname: str = "Properties",
|
|
34
|
-
url: str | None = None,
|
|
35
|
-
source_type: str | None = None,
|
|
36
|
-
orient: str = "columns",
|
|
37
|
-
workflow_name: str = "default",
|
|
38
|
-
file_name: str | None = None,
|
|
39
|
-
version: str | None = None,
|
|
40
|
-
as_role: str | None = None,
|
|
41
|
-
) -> dict[str, Any]:
|
|
42
|
-
rules_schema_version = ""
|
|
43
|
-
if NEAT_APP.cdf_store is None or NEAT_APP.workflow_manager is None:
|
|
44
|
-
return {"error": "NeatApp is not initialized"}
|
|
45
|
-
if workflow_name != "undefined":
|
|
46
|
-
workflow = NEAT_APP.workflow_manager.get_workflow(workflow_name)
|
|
47
|
-
if workflow is None:
|
|
48
|
-
return {"error": f"Workflow {workflow_name} is not found"}
|
|
49
|
-
workflow_definition = workflow.get_workflow_definition()
|
|
50
|
-
if not file_name:
|
|
51
|
-
for step in workflow_definition.steps:
|
|
52
|
-
if step.method == ImportExcelToRules.__name__:
|
|
53
|
-
file_name = step.configs["file_name"]
|
|
54
|
-
version = step.configs["version"]
|
|
55
|
-
break
|
|
56
|
-
if step.method == RulesToExcel.__name__ or step.method == ExcelToRules.__name__:
|
|
57
|
-
rules_schema_version = "v2"
|
|
58
|
-
as_role = step.configs.get("as_role", "")
|
|
59
|
-
file_name = step.configs.get("File name", "")
|
|
60
|
-
if file_name:
|
|
61
|
-
break
|
|
62
|
-
file_name = step.configs.get("File path", "")
|
|
63
|
-
break
|
|
64
|
-
|
|
65
|
-
if not file_name:
|
|
66
|
-
return {"error": "File name is not provided"}
|
|
67
|
-
rules_file = Path(file_name)
|
|
68
|
-
if str(rules_file.parent) == ".":
|
|
69
|
-
path = Path(NEAT_APP.config.rules_store_path) / rules_file
|
|
70
|
-
else:
|
|
71
|
-
path = Path(NEAT_APP.config.data_store_path) / rules_file
|
|
72
|
-
|
|
73
|
-
src = "local"
|
|
74
|
-
if url:
|
|
75
|
-
path = Path(url)
|
|
76
|
-
|
|
77
|
-
if path.exists() and not version:
|
|
78
|
-
logging.info(f"Loading rules from {path}")
|
|
79
|
-
elif path.exists() and version:
|
|
80
|
-
hash_ = get_file_hash(path)
|
|
81
|
-
if hash_ != version:
|
|
82
|
-
NEAT_APP.cdf_store.load_rules_file_from_cdf(file_name, version)
|
|
83
|
-
src = "cdf"
|
|
84
|
-
else:
|
|
85
|
-
NEAT_APP.cdf_store.load_rules_file_from_cdf(file_name, version)
|
|
86
|
-
src = "cdf"
|
|
87
|
-
|
|
88
|
-
error_text = ""
|
|
89
|
-
properties = []
|
|
90
|
-
classes = []
|
|
91
|
-
remaped_rules = {}
|
|
92
|
-
try:
|
|
93
|
-
# Trying to load rules V1
|
|
94
|
-
if rules_schema_version == "" or rules_schema_version == "v1":
|
|
95
|
-
rules = cast(
|
|
96
|
-
Rules, legacy_importers.ExcelImporter(path).to_rules(return_report=False, skip_validation=False)
|
|
97
|
-
)
|
|
98
|
-
properties = [
|
|
99
|
-
{
|
|
100
|
-
"class": value.class_id,
|
|
101
|
-
"property": value.property_id,
|
|
102
|
-
"property_description": value.description,
|
|
103
|
-
"property_type": (
|
|
104
|
-
value.expected_value_type.versioned_id
|
|
105
|
-
if value.property_type == EntityTypes.object_property
|
|
106
|
-
else value.expected_value_type.suffix
|
|
107
|
-
),
|
|
108
|
-
"cdf_resource_type": value.cdf_resource_type,
|
|
109
|
-
"cdf_metadata_type": value.resource_type_property,
|
|
110
|
-
"rule_type": value.rule_type,
|
|
111
|
-
"rule": value.rule,
|
|
112
|
-
}
|
|
113
|
-
for value in rules.properties.values()
|
|
114
|
-
]
|
|
115
|
-
|
|
116
|
-
classes = [
|
|
117
|
-
{
|
|
118
|
-
"class": value.class_id,
|
|
119
|
-
"class_description": value.description,
|
|
120
|
-
"cdf_resource_type": value.cdf_resource_type,
|
|
121
|
-
}
|
|
122
|
-
for value in rules.classes.values()
|
|
123
|
-
]
|
|
124
|
-
rules_schema_version = "v1"
|
|
125
|
-
remaped_rules = {"properties": properties, "metadata": rules.metadata.model_dump(), "classes": classes}
|
|
126
|
-
except Exception as e:
|
|
127
|
-
error_text = str(e)
|
|
128
|
-
|
|
129
|
-
if rules_schema_version == "" or rules_schema_version == "v2":
|
|
130
|
-
try:
|
|
131
|
-
role = RoleTypes(as_role) if as_role else None
|
|
132
|
-
rules_v2, issues = importers.ExcelImporter(path).to_rules(role=role)
|
|
133
|
-
error_text = ""
|
|
134
|
-
rules_schema_version = "v2"
|
|
135
|
-
if rules_v2:
|
|
136
|
-
remaped_rules = rules_v2.model_dump()
|
|
137
|
-
except Exception as e:
|
|
138
|
-
error_text = str(e)
|
|
139
|
-
rules_schema_version = "unknown"
|
|
140
|
-
|
|
141
|
-
return {
|
|
142
|
-
"rules": remaped_rules,
|
|
143
|
-
"file_name": path.name,
|
|
144
|
-
"hash": get_file_hash(path),
|
|
145
|
-
"error_text": error_text,
|
|
146
|
-
"src": src,
|
|
147
|
-
"rules_schema_version": rules_schema_version,
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
@router.get("/api/rules/from_file")
|
|
152
|
-
def get_original_rules_from_file(file_name: str):
|
|
153
|
-
# """Endpoint for retrieving raw transformation from file"""
|
|
154
|
-
path = Path(NEAT_APP.config.rules_store_path) / file_name
|
|
155
|
-
rules = cast(
|
|
156
|
-
Rules, legacy_importers.ExcelImporter(filepath=path).to_rules(return_report=False, skip_validation=False)
|
|
157
|
-
)
|
|
158
|
-
return Response(content=rules.model_dump_json(), media_type="application/json")
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
@router.get("/api/rules/from_workflow")
|
|
162
|
-
def get_original_rules_from_workflow(workflow_name: str):
|
|
163
|
-
"""Endpoing for retrieving transformation from memmory"""
|
|
164
|
-
workflow = NEAT_APP.workflow_manager.get_workflow(workflow_name)
|
|
165
|
-
if workflow is None:
|
|
166
|
-
return {"error": f"Workflow {workflow_name} is not found"}
|
|
167
|
-
context = workflow.get_context()
|
|
168
|
-
rules_data = context["RulesData"]
|
|
169
|
-
if type(rules_data) != RulesData:
|
|
170
|
-
return {"error": "RulesData is not found in workflow context"}
|
|
171
|
-
|
|
172
|
-
return Response(content=rules_data.rules.model_dump_json(), media_type="application/json")
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
@router.post("/api/rules/model_and_transformations")
|
|
176
|
-
def upsert_rules(request: TransformationRulesUpdateRequest):
|
|
177
|
-
"""Endpoing for updating transformation rules via API . This endpoint is still experimental"""
|
|
178
|
-
rules = request.rules_object
|
|
179
|
-
rules["metadata"]["namespace"] = Namespace(rules["metadata"]["namespace"])
|
|
180
|
-
metadata = Metadata(**rules["metadata"])
|
|
181
|
-
classes = Classes()
|
|
182
|
-
for class_, val in rules["classes"].items():
|
|
183
|
-
classes[class_] = Class(**val)
|
|
184
|
-
properties = Properties()
|
|
185
|
-
|
|
186
|
-
for prop, val in rules["properties"].items():
|
|
187
|
-
val["resource_type_property"] = []
|
|
188
|
-
properties[prop] = Property(**val)
|
|
189
|
-
|
|
190
|
-
prefixes: dict[str, Namespace] = {}
|
|
191
|
-
for prefix, val in rules["prefixes"].items():
|
|
192
|
-
prefixes[prefix] = Namespace(val)
|
|
193
|
-
|
|
194
|
-
rules = Rules(metadata=metadata, classes=classes, properties=properties, prefixes=prefixes, instances=[])
|
|
195
|
-
if request.output_format == "excel":
|
|
196
|
-
rules_file = Path(request.file_name)
|
|
197
|
-
if str(rules_file.parent) == ".":
|
|
198
|
-
path = Path(NEAT_APP.config.rules_store_path) / rules_file
|
|
199
|
-
else:
|
|
200
|
-
path = Path(NEAT_APP.config.data_store_path) / rules_file
|
|
201
|
-
|
|
202
|
-
legacy_exporters.ExcelExporter(rules=rules).export_to_file(path)
|
|
203
|
-
return {"status": "ok"}
|
cognite/neat/legacy/__init__.py
DELETED
|
File without changes
|