mustrd 0.1.7__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mustrd/README.adoc +15 -6
- mustrd/model/catalog-v001.xml +5 -0
- mustrd/model/mustrdTestOntology.ttl +51 -0
- mustrd/model/mustrdTestShapes.ttl +24 -0
- mustrd/model/ontology.ttl +2 -1
- mustrd/model/triplestoreOntology.ttl +174 -0
- mustrd/model/triplestoreshapes.ttl +42 -0
- mustrd/mustrd.py +48 -39
- mustrd/mustrdAnzo.py +61 -77
- mustrd/mustrdQueryProcessor.py +136 -0
- mustrd/mustrdTestPlugin.py +125 -58
- mustrd/namespace.py +36 -29
- mustrd/run.py +2 -2
- mustrd/spec_component.py +60 -52
- mustrd/steprunner.py +15 -15
- mustrd/test/test_mustrd.py +5 -0
- {mustrd-0.1.7.dist-info → mustrd-0.2.0.dist-info}/METADATA +1 -1
- mustrd-0.2.0.dist-info/RECORD +32 -0
- mustrd-0.1.7.dist-info/RECORD +0 -25
- {mustrd-0.1.7.dist-info → mustrd-0.2.0.dist-info}/LICENSE +0 -0
- {mustrd-0.1.7.dist-info → mustrd-0.2.0.dist-info}/WHEEL +0 -0
- {mustrd-0.1.7.dist-info → mustrd-0.2.0.dist-info}/entry_points.txt +0 -0
mustrd/mustrdAnzo.py
CHANGED
@@ -28,7 +28,6 @@ from rdflib import Graph, ConjunctiveGraph, Literal, URIRef
|
|
28
28
|
from requests import ConnectTimeout, Response, HTTPError, RequestException, ConnectionError
|
29
29
|
from bs4 import BeautifulSoup
|
30
30
|
import logging
|
31
|
-
from .namespace import MUST
|
32
31
|
|
33
32
|
|
34
33
|
# https://github.com/Semantic-partners/mustrd/issues/73
|
@@ -43,7 +42,6 @@ def manage_anzo_response(response: Response) -> str:
|
|
43
42
|
else:
|
44
43
|
raise RequestException(f"Anzo error, status code: {response.status_code}, content: {content_string}")
|
45
44
|
|
46
|
-
|
47
45
|
def query_with_bindings(bindings: dict, when: str) -> str:
|
48
46
|
values = ""
|
49
47
|
for key, value in bindings.items():
|
@@ -52,68 +50,70 @@ def query_with_bindings(bindings: dict, when: str) -> str:
|
|
52
50
|
return f"{split_query[0].strip()} WHERE {{ {values} {split_query[1].strip()}"
|
53
51
|
|
54
52
|
def execute_select (triple_store: dict, when: str, bindings: dict = None) -> str:
|
53
|
+
if bindings:
|
54
|
+
when = query_with_bindings(bindings, when)
|
55
|
+
# Just remove ${fromSources} if we are executing a query step; the sources are defined using http parameters
|
56
|
+
when = when.replace("${fromSources}", "")
|
57
|
+
return execute_sparql(triple_store, False, when, triple_store['input_graph'])
|
58
|
+
|
59
|
+
PARAMS = {
|
60
|
+
# Update parameters for INSERT / DELETE
|
61
|
+
True: {
|
62
|
+
"default-graph-param": "using-graph-uri",
|
63
|
+
"named-graph-param":"using-named-graph-uri",
|
64
|
+
"Content-Type": "application/sparql-update"
|
65
|
+
},
|
66
|
+
# Query parameters for SELECT / CONSTRUCT
|
67
|
+
False: {
|
68
|
+
"default-graph-param": "default-graph-uri",
|
69
|
+
"named-graph-param":"named-graph-uri",
|
70
|
+
"Content-Type": "application/sparql-query"
|
71
|
+
}
|
72
|
+
}
|
73
|
+
def execute_sparql(triple_store: dict, is_update: bool, sparql, graph: str, format: str = "application/sparql-results+json"):
|
74
|
+
params = {
|
75
|
+
"format" : format,
|
76
|
+
"datasourceURI" : triple_store['gqe_uri'],
|
77
|
+
"skipCache": "true",
|
78
|
+
# Default and named datasets have different query param for query and update
|
79
|
+
PARAMS[is_update]["default-graph-param"] : graph,
|
80
|
+
PARAMS[is_update]["named-graph-param"] : graph
|
81
|
+
}
|
82
|
+
headers={"Content-Type": PARAMS[is_update]["Content-Type"]}
|
55
83
|
try:
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
return
|
64
|
-
|
65
|
-
|
66
|
-
verify=False))
|
67
|
-
except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout):
|
84
|
+
response = manage_anzo_response(requests.post(url=f"https://{triple_store['url']}:{triple_store['port']}/sparql",
|
85
|
+
params=params,
|
86
|
+
auth=(triple_store['username'], triple_store['password']),
|
87
|
+
headers=headers,
|
88
|
+
data=sparql,
|
89
|
+
verify=False))
|
90
|
+
logging.debug(f'response {response}')
|
91
|
+
return response
|
92
|
+
except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout) as e:
|
93
|
+
logging.error(f'response {e}')
|
68
94
|
raise
|
69
95
|
|
70
96
|
def execute_update(triple_store: dict, when: str, bindings: dict = None) -> Graph:
|
71
97
|
logging.debug(f"updating in anzo! {triple_store=} {when=}")
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
response = manage_anzo_response(requests.post(url=url,
|
82
|
-
auth=(triple_store['username'],
|
83
|
-
triple_store['password']),
|
84
|
-
data=data,
|
85
|
-
verify=False))
|
86
|
-
logging.debug(f'response {response}')
|
87
|
-
check_data = {'datasourceURI': triple_store['gqe_uri'], 'query': "construct {?s ?p ?o} { ?s ?p ?o }",
|
88
|
-
'default-graph-uri': output_graph, 'skipCache': 'true'}
|
89
|
-
everything_response = manage_anzo_response(requests.post(url=url,
|
90
|
-
auth=(triple_store['username'],
|
91
|
-
triple_store['password']),
|
92
|
-
data=check_data,
|
93
|
-
verify=False))
|
94
|
-
# todo deal with error responses
|
95
|
-
new_graph = Graph().parse(data=everything_response)
|
98
|
+
|
99
|
+
# FIXME If query doesn't contain ${targetGraph}, then graph should be defined explicitly in the query
|
100
|
+
substituted_query = when.replace("${usingSources}", "").replace(
|
101
|
+
"${targetGraph}", f"<{triple_store['output_graph']}>")
|
102
|
+
|
103
|
+
execute_sparql(triple_store, True, substituted_query, triple_store['input_graph'], "ttl")
|
104
|
+
|
105
|
+
new_graph = execute_construct(triple_store, "construct {?s ?p ?o} { ?s ?p ?o }")
|
106
|
+
|
96
107
|
logging.debug(f"new_graph={new_graph.serialize(format='ttl')}")
|
108
|
+
|
97
109
|
return new_graph
|
98
110
|
|
99
111
|
|
100
112
|
def execute_construct(triple_store: dict, when: str, bindings: dict = None) -> Graph:
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
'default-graph-uri': triple_store['input_graph'], 'skipCache': 'true'}
|
106
|
-
url = f"https://{triple_store['url']}:{triple_store['port']}/sparql?format=ttl"
|
107
|
-
response = requests.post(url=url,
|
108
|
-
auth=(triple_store['username'],
|
109
|
-
triple_store['password']),
|
110
|
-
data=data,
|
111
|
-
verify=False)
|
112
|
-
logging.debug(f'response {response}')
|
113
|
-
return Graph().parse(data=manage_anzo_response(response))
|
114
|
-
except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout) as e:
|
115
|
-
logging.error(f'response {e}')
|
116
|
-
raise
|
113
|
+
if bindings:
|
114
|
+
when = query_with_bindings(bindings, when)
|
115
|
+
response = execute_sparql(triple_store, False, when, triple_store['input_graph'], "ttl")
|
116
|
+
return Graph().parse(data=response)
|
117
117
|
|
118
118
|
|
119
119
|
# Get Given or then from the content of a graphmart
|
@@ -207,30 +207,14 @@ SELECT ?query ?param_query ?query_template
|
|
207
207
|
|
208
208
|
def upload_given(triple_store: dict, given: Graph):
|
209
209
|
logging.debug(f"upload_given {triple_store} {given}")
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
serialized_given = given.serialize(format="nt")
|
217
|
-
insert_query = f"INSERT DATA {{graph <{triple_store['input_graph']}>{{{serialized_given}}}}}"
|
218
|
-
data = {'datasourceURI': triple_store['gqe_uri'], 'update': insert_query}
|
219
|
-
response = requests.post(url=f"https://{triple_store['url']}:{triple_store['port']}/sparql",
|
220
|
-
auth=(triple_store['username'], triple_store['password']), data=data, verify=False)
|
221
|
-
manage_anzo_response(response)
|
222
|
-
except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout):
|
223
|
-
raise
|
210
|
+
clear_graph(triple_store, triple_store['input_graph'])
|
211
|
+
clear_graph(triple_store, triple_store['output_graph'])
|
212
|
+
serialized_given = given.serialize(format="nt")
|
213
|
+
|
214
|
+
insert_query = f"INSERT DATA {{graph <{triple_store['input_graph']}>{{{serialized_given}}}}}"
|
215
|
+
execute_sparql(triple_store, True, insert_query, None, None)
|
224
216
|
|
225
217
|
|
226
218
|
def clear_graph(triple_store: dict, graph_uri: str):
|
227
|
-
|
228
|
-
clear_query = f"CLEAR GRAPH <{graph_uri}>"
|
229
|
-
data = {'datasourceURI': triple_store['gqe_uri'], 'update': clear_query}
|
230
|
-
url = f"https://{triple_store['url']}:{triple_store['port']}/sparql"
|
231
|
-
response = requests.post(url=url,
|
232
|
-
auth=(triple_store['username'], triple_store['password']), data=data, verify=False)
|
233
|
-
manage_anzo_response(response)
|
234
|
-
except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout):
|
235
|
-
raise
|
219
|
+
execute_sparql(triple_store, True, f"CLEAR GRAPH <{graph_uri}>", None, None)
|
236
220
|
|
@@ -0,0 +1,136 @@
|
|
1
|
+
|
2
|
+
from pyparsing import ParseResults
|
3
|
+
from rdflib import RDF, Graph, URIRef, Variable, Literal, XSD, util, BNode
|
4
|
+
from rdflib.plugins.sparql.parser import parseQuery, parseUpdate
|
5
|
+
from rdflib.plugins.sparql.algebra import translateQuery, translateUpdate, translateAlgebra
|
6
|
+
from rdflib.plugins.sparql.sparql import Query
|
7
|
+
from rdflib.plugins.sparql.parserutils import CompValue, value, Expr
|
8
|
+
from rdflib.namespace import DefinedNamespace, Namespace
|
9
|
+
from rdflib.term import Identifier
|
10
|
+
from typing import Union
|
11
|
+
|
12
|
+
from builtins import list, set, tuple, str
|
13
|
+
|
14
|
+
|
15
|
+
namespace = "https://mustrd.com/query/"
|
16
|
+
|
17
|
+
class MustrdQueryProcessor:
|
18
|
+
original_query : Union [Query, ParseResults]
|
19
|
+
current_query : Union [Query, ParseResults]
|
20
|
+
graph : Graph
|
21
|
+
algebra_mode: bool = False
|
22
|
+
graph_mode: bool = True
|
23
|
+
|
24
|
+
def __init__(self, query_str: str, algebra_mode: bool = False, graph_mode: bool = True):
|
25
|
+
parsetree = parseQuery(query_str)
|
26
|
+
# Init original query to algebra or parsed query
|
27
|
+
self.original_query = (algebra_mode and translateQuery(parsetree)) or parsetree
|
28
|
+
self.current_query = self.original_query
|
29
|
+
self.algebra_mode = algebra_mode
|
30
|
+
self.graph_mode = graph_mode
|
31
|
+
self.graph = Graph()
|
32
|
+
if graph_mode:
|
33
|
+
self.query_to_graph((algebra_mode and self.original_query.algebra) or parsetree._toklist, BNode())
|
34
|
+
|
35
|
+
def query_to_graph(self, part: CompValue, partBnode):
|
36
|
+
if not part or not partBnode:
|
37
|
+
return
|
38
|
+
self.graph.add((partBnode, RDF.type, URIRef(namespace + type(part).__name__)))
|
39
|
+
self.graph.add((partBnode, QUERY.has_class , Literal(str(part.__class__.__name__))))
|
40
|
+
if isinstance(part, CompValue) or isinstance(part, ParseResults):
|
41
|
+
self.graph.add((partBnode, QUERY.name , Literal(part.name)))
|
42
|
+
if isinstance(part, CompValue):
|
43
|
+
for key, sub_part in part.items():
|
44
|
+
sub_part_bnode = BNode()
|
45
|
+
self.graph.add((partBnode, URIRef(namespace + str(key)) , sub_part_bnode))
|
46
|
+
self.query_to_graph(sub_part, sub_part_bnode)
|
47
|
+
elif hasattr(part, '__iter__') and not isinstance(part, Identifier) and not isinstance(part, str):
|
48
|
+
for sub_part in part:
|
49
|
+
sub_part_bnode = BNode()
|
50
|
+
self.graph.add((partBnode, QUERY.has_list , sub_part_bnode))
|
51
|
+
self.query_to_graph(sub_part, sub_part_bnode)
|
52
|
+
elif isinstance(part, Identifier) or isinstance(part, str):
|
53
|
+
self.graph.add((partBnode, QUERY.has_value, Literal(part)))
|
54
|
+
|
55
|
+
def serialize_graph(self):
|
56
|
+
if not self.graph_mode:
|
57
|
+
raise Exception("Not able to execute that function if graph mode is not activated: cannot work with two sources of truth")
|
58
|
+
return self.graph.serialize(format = "ttl")
|
59
|
+
|
60
|
+
def query_graph(self, meta_query: str):
|
61
|
+
if not self.graph_mode:
|
62
|
+
raise Exception("Not able to execute that function if graph mode is not activated: cannot work with two sources of truth")
|
63
|
+
return self.graph.query(meta_query)
|
64
|
+
|
65
|
+
def update(self, meta_query: str):
|
66
|
+
if not self.graph_mode:
|
67
|
+
# Implement update directly on objects: self.current_query
|
68
|
+
pass
|
69
|
+
return self.graph.update(meta_query)
|
70
|
+
|
71
|
+
def get_query(self):
|
72
|
+
if self.graph_mode:
|
73
|
+
roots = self.graph.query("SELECT DISTINCT ?sub WHERE {?sub ?prop ?obj FILTER NOT EXISTS {?s ?p ?sub}}")
|
74
|
+
if len(roots) != 1:
|
75
|
+
raise Exception("query graph has more than one root: invalid")
|
76
|
+
|
77
|
+
for root in roots:
|
78
|
+
new_query = self.graph_to_query(root.sub)
|
79
|
+
if not self.algebra_mode:
|
80
|
+
new_query = ParseResults(toklist=new_query, name=self.original_query.name)
|
81
|
+
new_query = translateQuery(new_query)
|
82
|
+
else:
|
83
|
+
new_query = Query(algebra=new_query, prologue=self.original_query.prologue)
|
84
|
+
else:
|
85
|
+
if not self.algebra_mode:
|
86
|
+
new_query = translateQuery(self.current_query)
|
87
|
+
else:
|
88
|
+
new_query = self.current_query
|
89
|
+
return translateAlgebra(new_query)
|
90
|
+
|
91
|
+
|
92
|
+
|
93
|
+
def graph_to_query(self, subject):
|
94
|
+
subject_dict = self.get_subject_dict(subject)
|
95
|
+
if QUERY.has_class in subject_dict:
|
96
|
+
class_name = str(subject_dict[QUERY.has_class])
|
97
|
+
subject_dict.pop(QUERY.has_class)
|
98
|
+
if class_name in globals():
|
99
|
+
clazz = globals()[class_name]
|
100
|
+
if clazz in (CompValue, Expr):
|
101
|
+
comp_name = str(subject_dict[QUERY.name])
|
102
|
+
subject_dict.pop(QUERY.name)
|
103
|
+
subject_dict.pop(RDF.type)
|
104
|
+
new_dict = dict(map(lambda kv: [str(kv[0]).replace(str(QUERY._NS), ""),
|
105
|
+
self.graph_to_query(kv[1])] ,
|
106
|
+
subject_dict.items()))
|
107
|
+
return clazz(comp_name, **new_dict)
|
108
|
+
elif clazz in (set, list, tuple) and QUERY.has_list in subject_dict:
|
109
|
+
return clazz(map(lambda item: self.graph_to_query(item), subject_dict[QUERY.has_list]))
|
110
|
+
elif clazz == ParseResults and QUERY.has_list in subject_dict:
|
111
|
+
return ParseResults(toklist=list(map(lambda item: self.graph_to_query(item), subject_dict[QUERY.has_list])))
|
112
|
+
elif clazz in (Literal, Variable, URIRef, str) and QUERY.has_value in subject_dict:
|
113
|
+
return clazz(str(subject_dict[QUERY.has_value]))
|
114
|
+
|
115
|
+
|
116
|
+
def get_subject_dict(self, subject):
|
117
|
+
dict = {}
|
118
|
+
for key, value in self.graph.predicate_objects(subject):
|
119
|
+
# If key already exists: create or add to a list
|
120
|
+
if key == QUERY.has_list:
|
121
|
+
if key in dict:
|
122
|
+
dict[key].append(value)
|
123
|
+
else:
|
124
|
+
dict[key] = [value]
|
125
|
+
else:
|
126
|
+
dict[key] = value
|
127
|
+
return dict
|
128
|
+
|
129
|
+
|
130
|
+
|
131
|
+
class QUERY(DefinedNamespace):
|
132
|
+
_NS = Namespace("https://mustrd.com/query/")
|
133
|
+
has_class : URIRef
|
134
|
+
has_list : URIRef
|
135
|
+
name : URIRef
|
136
|
+
has_value: URIRef
|
mustrd/mustrdTestPlugin.py
CHANGED
@@ -29,22 +29,30 @@ from pathlib import Path
|
|
29
29
|
from rdflib.namespace import Namespace
|
30
30
|
from rdflib import Graph, RDF
|
31
31
|
from pytest import Session
|
32
|
-
from typing import Dict
|
33
32
|
|
34
33
|
from mustrd.TestResult import ResultList, TestResult, get_result_list
|
35
34
|
from mustrd.utils import get_mustrd_root
|
36
35
|
from mustrd.mustrd import get_triple_store_graph, get_triple_stores
|
37
|
-
from mustrd.mustrd import SpecSkipped, validate_specs, get_specs, SpecPassed, run_spec
|
38
|
-
from mustrd.namespace import MUST
|
39
|
-
from
|
36
|
+
from mustrd.mustrd import Specification, SpecSkipped, validate_specs, get_specs, SpecPassed, run_spec
|
37
|
+
from mustrd.namespace import MUST, TRIPLESTORE, MUSTRDTEST
|
38
|
+
from typing import Union
|
39
|
+
from pyshacl import validate
|
40
40
|
|
41
41
|
spnamespace = Namespace("https://semanticpartners.com/data/test/")
|
42
42
|
|
43
43
|
mustrd_root = get_mustrd_root()
|
44
44
|
|
45
|
+
MUSTRD_PYTEST_PATH = "mustrd_tests/"
|
46
|
+
|
45
47
|
|
46
48
|
def pytest_addoption(parser):
|
47
|
-
group = parser.getgroup("
|
49
|
+
group = parser.getgroup("mustrd option")
|
50
|
+
group.addoption(
|
51
|
+
"--mustrd",
|
52
|
+
action="store_true",
|
53
|
+
dest="mustrd",
|
54
|
+
help="Activate/deactivate mustrd test generation.",
|
55
|
+
)
|
48
56
|
group.addoption(
|
49
57
|
"--md",
|
50
58
|
action="store",
|
@@ -59,7 +67,6 @@ def pytest_addoption(parser):
|
|
59
67
|
dest="configpath",
|
60
68
|
metavar="pathToTestConfig",
|
61
69
|
default=None,
|
62
|
-
required=True,
|
63
70
|
help="Ttl file containing the list of test to construct.",
|
64
71
|
)
|
65
72
|
group.addoption(
|
@@ -68,7 +75,6 @@ def pytest_addoption(parser):
|
|
68
75
|
dest="secrets",
|
69
76
|
metavar="Secrets",
|
70
77
|
default=None,
|
71
|
-
required=False,
|
72
78
|
help="Give the secrets by command line in order to be able to store secrets safely in CI tools",
|
73
79
|
)
|
74
80
|
return
|
@@ -76,23 +82,40 @@ def pytest_addoption(parser):
|
|
76
82
|
|
77
83
|
def pytest_configure(config) -> None:
|
78
84
|
# Read configuration file
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
85
|
+
if config.getoption("mustrd"):
|
86
|
+
test_configs = parse_config(config.getoption("configpath"))
|
87
|
+
config.pluginmanager.register(MustrdTestPlugin(config.getoption("mdpath"),
|
88
|
+
test_configs, config.getoption("secrets")))
|
89
|
+
|
90
|
+
def parse_config(config_path):
|
91
|
+
test_configs = []
|
92
|
+
config_graph = Graph().parse(config_path)
|
93
|
+
shacl_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdTestShapes.ttl")))
|
94
|
+
ont_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdTestOntology.ttl")))
|
95
|
+
conforms, results_graph, results_text = validate(
|
96
|
+
data_graph= config_graph,
|
97
|
+
shacl_graph = shacl_graph,
|
98
|
+
ont_graph = ont_graph,
|
99
|
+
advanced= True,
|
100
|
+
inference= 'none'
|
101
|
+
)
|
102
|
+
if not conforms:
|
103
|
+
raise ValueError(f"Mustrd test configuration not conform to the shapes. SHACL report: {results_text}", results_graph)
|
104
|
+
|
105
|
+
for test_config_subject in config_graph.subjects(predicate=RDF.type, object=MUSTRDTEST.MustrdTest):
|
106
|
+
spec_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasSpecPath, str)
|
107
|
+
data_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasDataPath, str)
|
108
|
+
triplestore_spec_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.triplestoreSpecPath, str)
|
109
|
+
pytest_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasPytestPath, str)
|
86
110
|
filter_on_tripleStore = list(config_graph.objects(subject=test_config_subject,
|
87
|
-
|
111
|
+
predicate=MUSTRDTEST.filterOnTripleStore))
|
88
112
|
|
89
|
-
test_configs
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
test_configs, config.getoption("secrets")))
|
113
|
+
test_configs.append(TestConfig(spec_path=spec_path, data_path=data_path,
|
114
|
+
triplestore_spec_path=triplestore_spec_path,
|
115
|
+
pytest_path = pytest_path,
|
116
|
+
filter_on_tripleStore=filter_on_tripleStore))
|
117
|
+
return test_configs
|
118
|
+
|
96
119
|
|
97
120
|
|
98
121
|
def get_config_param(config_graph, config_subject, config_param, convert_function):
|
@@ -102,65 +125,109 @@ def get_config_param(config_graph, config_subject, config_param, convert_functio
|
|
102
125
|
|
103
126
|
@dataclass
|
104
127
|
class TestConfig:
|
105
|
-
test_function: str
|
106
128
|
spec_path: str
|
107
129
|
data_path: str
|
108
130
|
triplestore_spec_path: str
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
filter_on_tripleStore: str = None):
|
113
|
-
self.test_function = test_function
|
114
|
-
self.spec_path = spec_path
|
115
|
-
self.data_path = data_path
|
116
|
-
self.triplestore_spec_path = triplestore_spec_path
|
117
|
-
self.filter_on_tripleStore = filter_on_tripleStore
|
131
|
+
pytest_path: str
|
132
|
+
filter_on_tripleStore: str = None
|
133
|
+
|
118
134
|
|
135
|
+
@dataclass
|
136
|
+
class TestParamWrapper:
|
137
|
+
test_config: TestConfig
|
138
|
+
unit_test: Union[Specification, SpecSkipped]
|
119
139
|
|
120
140
|
class MustrdTestPlugin:
|
121
141
|
md_path: str
|
122
|
-
test_configs:
|
142
|
+
test_configs: list
|
123
143
|
secrets: str
|
144
|
+
unit_tests: Union[Specification, SpecSkipped]
|
145
|
+
items: list
|
124
146
|
|
125
147
|
def __init__(self, md_path, test_configs, secrets):
|
126
148
|
self.md_path = md_path
|
127
149
|
self.test_configs = test_configs
|
128
150
|
self.secrets = secrets
|
151
|
+
self.items = []
|
152
|
+
|
153
|
+
@pytest.hookimpl(tryfirst=True)
|
154
|
+
def pytest_collection(self, session):
|
155
|
+
self.unit_tests = []
|
156
|
+
args = session.config.args
|
157
|
+
if len(args) > 0:
|
158
|
+
file_name = self.get_file_name_from_arg(args[0])
|
159
|
+
# Filter test to collect only specified path
|
160
|
+
config_to_collect = list(filter(lambda config:
|
161
|
+
# Case we want to collect everything
|
162
|
+
MUSTRD_PYTEST_PATH not in args[0]
|
163
|
+
# Case we want to collect a test or sub test
|
164
|
+
or (config.pytest_path or "") in args[0]
|
165
|
+
# Case we want to collect a whole test folder
|
166
|
+
or args[0].replace(f"./{MUSTRD_PYTEST_PATH}", "") in config.pytest_path,
|
167
|
+
self.test_configs))
|
168
|
+
|
169
|
+
# Redirect everything to test_mustrd.py, no need to filter on specified test: Only specified test will be collected anyway
|
170
|
+
session.config.args[0] = os.path.join(mustrd_root, "test/test_mustrd.py")
|
171
|
+
# Collecting only relevant tests
|
172
|
+
|
173
|
+
for one_test_config in config_to_collect:
|
174
|
+
triple_stores = self.get_triple_stores_from_file(one_test_config)
|
175
|
+
|
176
|
+
if one_test_config.filter_on_tripleStore and not triple_stores:
|
177
|
+
self.unit_tests.extend(list(map(lambda triple_store:
|
178
|
+
TestParamWrapper(test_config = one_test_config, unit_test=SpecSkipped(MUST.TestSpec, triple_store, "No triplestore found")),
|
179
|
+
one_test_config.filter_on_tripleStore)))
|
180
|
+
else:
|
181
|
+
specs = self.generate_tests_for_config({"spec_path": Path(one_test_config.spec_path),
|
182
|
+
"data_path": Path(one_test_config.data_path)},
|
183
|
+
triple_stores, file_name)
|
184
|
+
self.unit_tests.extend(list(map(lambda spec: TestParamWrapper(test_config = one_test_config, unit_test=spec),specs)))
|
185
|
+
|
186
|
+
def get_file_name_from_arg(self, arg):
|
187
|
+
if arg and len(arg) > 0 and "[" in arg and ".mustrd.ttl@" in arg:
|
188
|
+
return arg[arg.index("[") + 1: arg.index(".mustrd.ttl@")]
|
189
|
+
return None
|
190
|
+
|
191
|
+
|
192
|
+
@pytest.hookimpl(hookwrapper=True)
|
193
|
+
def pytest_pycollect_makeitem(self, collector, name, obj):
|
194
|
+
report = yield
|
195
|
+
if name == "test_unit":
|
196
|
+
items = report.get_result()
|
197
|
+
new_results = []
|
198
|
+
for item in items:
|
199
|
+
virtual_path = MUSTRD_PYTEST_PATH + (item.callspec.params["unit_tests"].test_config.pytest_path or "default")
|
200
|
+
item.fspath = Path(virtual_path)
|
201
|
+
item._nodeid = virtual_path + "::" + item.name
|
202
|
+
self.items.append(item)
|
203
|
+
new_results.append(item)
|
204
|
+
return new_results
|
205
|
+
|
129
206
|
|
130
207
|
# Hook called at collection time: reads the configuration of the tests, and generate pytests from it
|
131
208
|
def pytest_generate_tests(self, metafunc):
|
132
|
-
|
133
209
|
if len(metafunc.fixturenames) > 0:
|
134
|
-
if metafunc.function.__name__
|
135
|
-
one_test_config = self.test_configs[metafunc.function.__name__]
|
136
|
-
|
137
|
-
triple_stores = self.get_triple_stores_from_file(one_test_config)
|
138
|
-
|
139
|
-
unit_tests = []
|
140
|
-
if one_test_config.filter_on_tripleStore and not triple_stores:
|
141
|
-
unit_tests = list(map(lambda triple_store:
|
142
|
-
SpecSkipped(MUST.TestSpec, triple_store, "No triplestore found"),
|
143
|
-
one_test_config.filter_on_tripleStore))
|
144
|
-
else:
|
145
|
-
unit_tests = self.generate_tests_for_config({"spec_path": Path(one_test_config.spec_path),
|
146
|
-
"data_path": Path(one_test_config.data_path)},
|
147
|
-
triple_stores)
|
148
|
-
|
210
|
+
if metafunc.function.__name__ == "test_unit":
|
149
211
|
# Create the test in itself
|
150
|
-
if unit_tests:
|
151
|
-
metafunc.parametrize(metafunc.fixturenames[0], unit_tests,
|
212
|
+
if self.unit_tests:
|
213
|
+
metafunc.parametrize(metafunc.fixturenames[0], self.unit_tests,
|
214
|
+
ids=lambda test_param: (test_param.unit_test.spec_file_name or "") + "@" +
|
215
|
+
(test_param.test_config.pytest_path or ""))
|
152
216
|
else:
|
153
217
|
metafunc.parametrize(metafunc.fixturenames[0],
|
154
218
|
[SpecSkipped(MUST.TestSpec, None, "No triplestore found")],
|
155
219
|
ids=lambda x: "No configuration found for this test")
|
220
|
+
|
221
|
+
|
222
|
+
|
156
223
|
|
157
224
|
# Generate test for each triple store available
|
158
|
-
def generate_tests_for_config(self, config, triple_stores):
|
225
|
+
def generate_tests_for_config(self, config, triple_stores, file_name):
|
159
226
|
|
160
227
|
shacl_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdShapes.ttl")))
|
161
228
|
ont_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/ontology.ttl")))
|
162
229
|
valid_spec_uris, spec_graph, invalid_spec_results = validate_specs(config, triple_stores,
|
163
|
-
shacl_graph, ont_graph)
|
230
|
+
shacl_graph, ont_graph, file_name or "*")
|
164
231
|
|
165
232
|
specs, skipped_spec_results = \
|
166
233
|
get_specs(valid_spec_uris, spec_graph, triple_stores, config)
|
@@ -186,15 +253,15 @@ class MustrdTestPlugin:
|
|
186
253
|
triple_stores = get_triple_stores(get_triple_store_graph(Path(test_config.triplestore_spec_path),
|
187
254
|
self.secrets))
|
188
255
|
except Exception as e:
|
189
|
-
print(f"""
|
190
|
-
|
191
|
-
triple_stores = [{'type':
|
256
|
+
print(f"""Triplestore configuration parsing failed {test_config.triplestore_spec_path}.
|
257
|
+
Only rdflib will be executed""", e)
|
258
|
+
triple_stores = [{'type': TRIPLESTORE.RdfLib, 'uri': TRIPLESTORE.RdfLib}]
|
192
259
|
else:
|
193
260
|
print("No triple store configuration required: using embedded rdflib")
|
194
|
-
triple_stores = [{'type':
|
261
|
+
triple_stores = [{'type': TRIPLESTORE.RdfLib, 'uri': TRIPLESTORE.RdfLib}]
|
195
262
|
|
196
263
|
if test_config.filter_on_tripleStore:
|
197
|
-
triple_stores = list(filter(lambda triple_store: (triple_store["
|
264
|
+
triple_stores = list(filter(lambda triple_store: (triple_store["uri"] in test_config.filter_on_tripleStore),
|
198
265
|
triple_stores))
|
199
266
|
return triple_stores
|
200
267
|
|