mustrd 0.1.6__py3-none-any.whl → 0.1.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mustrd/README.adoc +6 -6
- mustrd/model/catalog-v001.xml +5 -0
- mustrd/model/mustrdTestOntology.ttl +51 -0
- mustrd/model/mustrdTestShapes.ttl +24 -0
- mustrd/model/ontology.ttl +2 -1
- mustrd/model/triplestoreOntology.ttl +174 -0
- mustrd/model/triplestoreshapes.ttl +42 -0
- mustrd/mustrd.py +49 -39
- mustrd/mustrdTestPlugin.py +168 -41
- mustrd/namespace.py +36 -29
- mustrd/run.py +2 -2
- mustrd/spec_component.py +11 -11
- mustrd/steprunner.py +15 -15
- mustrd/test/test_mustrd.py +5 -0
- {mustrd-0.1.6.dist-info → mustrd-0.1.8.dist-info}/METADATA +1 -1
- mustrd-0.1.8.dist-info/RECORD +31 -0
- mustrd-0.1.8.dist-info/entry_points.txt +3 -0
- mustrd/plugin.py +0 -87
- mustrd-0.1.6.dist-info/RECORD +0 -26
- mustrd-0.1.6.dist-info/entry_points.txt +0 -3
- {mustrd-0.1.6.dist-info → mustrd-0.1.8.dist-info}/LICENSE +0 -0
- {mustrd-0.1.6.dist-info → mustrd-0.1.8.dist-info}/WHEEL +0 -0
mustrd/mustrdTestPlugin.py
CHANGED
@@ -27,81 +27,207 @@ import pytest
|
|
27
27
|
import os
|
28
28
|
from pathlib import Path
|
29
29
|
from rdflib.namespace import Namespace
|
30
|
-
from rdflib import Graph
|
30
|
+
from rdflib import Graph, RDF
|
31
31
|
from pytest import Session
|
32
|
-
from typing import Dict
|
33
32
|
|
34
33
|
from mustrd.TestResult import ResultList, TestResult, get_result_list
|
35
34
|
from mustrd.utils import get_mustrd_root
|
36
|
-
from mustrd.mustrd import get_triple_store_graph, get_triple_stores
|
37
|
-
from mustrd.
|
35
|
+
from mustrd.mustrd import get_triple_store_graph, get_triple_stores
|
36
|
+
from mustrd.mustrd import Specification, SpecSkipped, validate_specs, get_specs, SpecPassed, run_spec
|
37
|
+
from mustrd.namespace import MUST, TRIPLESTORE, MUSTRDTEST
|
38
|
+
from typing import Union
|
39
|
+
from pyshacl import validate
|
38
40
|
|
39
41
|
spnamespace = Namespace("https://semanticpartners.com/data/test/")
|
40
42
|
|
41
43
|
mustrd_root = get_mustrd_root()
|
42
44
|
|
45
|
+
MUSTRD_PYTEST_PATH = "mustrd_tests/"
|
46
|
+
|
47
|
+
|
48
|
+
def pytest_addoption(parser):
|
49
|
+
group = parser.getgroup("mustrd option")
|
50
|
+
group.addoption(
|
51
|
+
"--mustrd",
|
52
|
+
action="store_true",
|
53
|
+
dest="mustrd",
|
54
|
+
help="Activate/deactivate mustrd test generation.",
|
55
|
+
)
|
56
|
+
group.addoption(
|
57
|
+
"--md",
|
58
|
+
action="store",
|
59
|
+
dest="mdpath",
|
60
|
+
metavar="pathToMdSummary",
|
61
|
+
default=None,
|
62
|
+
help="create md summary file at that path.",
|
63
|
+
)
|
64
|
+
group.addoption(
|
65
|
+
"--config",
|
66
|
+
action="store",
|
67
|
+
dest="configpath",
|
68
|
+
metavar="pathToTestConfig",
|
69
|
+
default=None,
|
70
|
+
help="Ttl file containing the list of test to construct.",
|
71
|
+
)
|
72
|
+
group.addoption(
|
73
|
+
"--secrets",
|
74
|
+
action="store",
|
75
|
+
dest="secrets",
|
76
|
+
metavar="Secrets",
|
77
|
+
default=None,
|
78
|
+
help="Give the secrets by command line in order to be able to store secrets safely in CI tools",
|
79
|
+
)
|
80
|
+
return
|
81
|
+
|
82
|
+
|
83
|
+
def pytest_configure(config) -> None:
|
84
|
+
# Read configuration file
|
85
|
+
if config.getoption("mustrd"):
|
86
|
+
test_configs = parse_config(config.getoption("configpath"))
|
87
|
+
config.pluginmanager.register(MustrdTestPlugin(config.getoption("mdpath"),
|
88
|
+
test_configs, config.getoption("secrets")))
|
89
|
+
|
90
|
+
def parse_config(config_path):
|
91
|
+
test_configs = []
|
92
|
+
config_graph = Graph().parse(config_path)
|
93
|
+
shacl_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdTestShapes.ttl")))
|
94
|
+
ont_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdTestOntology.ttl")))
|
95
|
+
conforms, results_graph, results_text = validate(
|
96
|
+
data_graph= config_graph,
|
97
|
+
shacl_graph = shacl_graph,
|
98
|
+
ont_graph = ont_graph,
|
99
|
+
advanced= True,
|
100
|
+
inference= 'none'
|
101
|
+
)
|
102
|
+
if not conforms:
|
103
|
+
raise ValueError(f"Mustrd test configuration not conform to the shapes. SHACL report: {results_text}", results_graph)
|
104
|
+
|
105
|
+
for test_config_subject in config_graph.subjects(predicate=RDF.type, object=MUSTRDTEST.MustrdTest):
|
106
|
+
spec_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasSpecPath, str)
|
107
|
+
data_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasDataPath, str)
|
108
|
+
triplestore_spec_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.triplestoreSpecPath, str)
|
109
|
+
pytest_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasPytestPath, str)
|
110
|
+
filter_on_tripleStore = list(config_graph.objects(subject=test_config_subject,
|
111
|
+
predicate=MUSTRDTEST.filterOnTripleStore))
|
112
|
+
|
113
|
+
test_configs.append(TestConfig(spec_path=spec_path, data_path=data_path,
|
114
|
+
triplestore_spec_path=triplestore_spec_path,
|
115
|
+
pytest_path = pytest_path,
|
116
|
+
filter_on_tripleStore=filter_on_tripleStore))
|
117
|
+
return test_configs
|
118
|
+
|
119
|
+
|
120
|
+
|
121
|
+
def get_config_param(config_graph, config_subject, config_param, convert_function):
|
122
|
+
raw_value = config_graph.value(subject=config_subject, predicate=config_param, any=True)
|
123
|
+
return convert_function(raw_value) if raw_value else None
|
124
|
+
|
43
125
|
|
44
126
|
@dataclass
|
45
127
|
class TestConfig:
|
46
|
-
test_function: str
|
47
128
|
spec_path: str
|
48
129
|
data_path: str
|
49
130
|
triplestore_spec_path: str
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
filter_on_tripleStore: str = None):
|
54
|
-
self.test_function = test_function
|
55
|
-
self.spec_path = spec_path
|
56
|
-
self.data_path = data_path
|
57
|
-
self.triplestore_spec_path = triplestore_spec_path
|
58
|
-
self.filter_on_tripleStore = filter_on_tripleStore
|
131
|
+
pytest_path: str
|
132
|
+
filter_on_tripleStore: str = None
|
133
|
+
|
59
134
|
|
135
|
+
@dataclass
|
136
|
+
class TestParamWrapper:
|
137
|
+
test_config: TestConfig
|
138
|
+
unit_test: Union[Specification, SpecSkipped]
|
60
139
|
|
61
140
|
class MustrdTestPlugin:
|
62
141
|
md_path: str
|
63
|
-
test_configs:
|
142
|
+
test_configs: list
|
64
143
|
secrets: str
|
144
|
+
unit_tests: Union[Specification, SpecSkipped]
|
145
|
+
items: list
|
65
146
|
|
66
147
|
def __init__(self, md_path, test_configs, secrets):
|
67
148
|
self.md_path = md_path
|
68
149
|
self.test_configs = test_configs
|
69
150
|
self.secrets = secrets
|
151
|
+
self.items = []
|
152
|
+
|
153
|
+
@pytest.hookimpl(tryfirst=True)
|
154
|
+
def pytest_collection(self, session):
|
155
|
+
self.unit_tests = []
|
156
|
+
args = session.config.args
|
157
|
+
if len(args) > 0:
|
158
|
+
file_name = self.get_file_name_from_arg(args[0])
|
159
|
+
# Filter test to collect only specified path
|
160
|
+
config_to_collect = list(filter(lambda config:
|
161
|
+
# Case we want to collect everything
|
162
|
+
MUSTRD_PYTEST_PATH not in args[0]
|
163
|
+
# Case we want to collect a test or sub test
|
164
|
+
or (config.pytest_path or "") in args[0]
|
165
|
+
# Case we want to collect a whole test folder
|
166
|
+
or args[0].replace(f"./{MUSTRD_PYTEST_PATH}", "") in config.pytest_path,
|
167
|
+
self.test_configs))
|
168
|
+
|
169
|
+
# Redirect everything to test_mustrd.py, no need to filter on specified test: Only specified test will be collected anyway
|
170
|
+
session.config.args[0] = os.path.join(mustrd_root, "test/test_mustrd.py")
|
171
|
+
# Collecting only relevant tests
|
172
|
+
|
173
|
+
for one_test_config in config_to_collect:
|
174
|
+
triple_stores = self.get_triple_stores_from_file(one_test_config)
|
175
|
+
|
176
|
+
if one_test_config.filter_on_tripleStore and not triple_stores:
|
177
|
+
self.unit_tests.extend(list(map(lambda triple_store:
|
178
|
+
TestParamWrapper(test_config = one_test_config, unit_test=SpecSkipped(MUST.TestSpec, triple_store, "No triplestore found")),
|
179
|
+
one_test_config.filter_on_tripleStore)))
|
180
|
+
else:
|
181
|
+
specs = self.generate_tests_for_config({"spec_path": Path(one_test_config.spec_path),
|
182
|
+
"data_path": Path(one_test_config.data_path)},
|
183
|
+
triple_stores, file_name)
|
184
|
+
self.unit_tests.extend(list(map(lambda spec: TestParamWrapper(test_config = one_test_config, unit_test=spec),specs)))
|
185
|
+
|
186
|
+
def get_file_name_from_arg(self, arg):
|
187
|
+
if arg and len(arg) > 0 and "[" in arg and ".mustrd.ttl@" in arg:
|
188
|
+
return arg[arg.index("[") + 1: arg.index(".mustrd.ttl@")]
|
189
|
+
return None
|
190
|
+
|
191
|
+
|
192
|
+
@pytest.hookimpl(hookwrapper=True)
|
193
|
+
def pytest_pycollect_makeitem(self, collector, name, obj):
|
194
|
+
report = yield
|
195
|
+
if name == "test_unit":
|
196
|
+
items = report.get_result()
|
197
|
+
new_results = []
|
198
|
+
for item in items:
|
199
|
+
virtual_path = MUSTRD_PYTEST_PATH + (item.callspec.params["unit_tests"].test_config.pytest_path or "default")
|
200
|
+
item.fspath = Path(virtual_path)
|
201
|
+
item._nodeid = virtual_path + "::" + item.name
|
202
|
+
self.items.append(item)
|
203
|
+
new_results.append(item)
|
204
|
+
return new_results
|
205
|
+
|
70
206
|
|
71
207
|
# Hook called at collection time: reads the configuration of the tests, and generate pytests from it
|
72
208
|
def pytest_generate_tests(self, metafunc):
|
73
|
-
|
74
209
|
if len(metafunc.fixturenames) > 0:
|
75
|
-
if metafunc.function.__name__
|
76
|
-
one_test_config = self.test_configs[metafunc.function.__name__]
|
77
|
-
|
78
|
-
triple_stores = self.get_triple_stores_from_file(one_test_config)
|
79
|
-
|
80
|
-
unit_tests = []
|
81
|
-
if one_test_config.filter_on_tripleStore and not triple_stores:
|
82
|
-
unit_tests = list(map(lambda triple_store:
|
83
|
-
SpecSkipped(MUST.TestSpec, triple_store, "No triplestore found"),
|
84
|
-
one_test_config.filter_on_tripleStore))
|
85
|
-
else:
|
86
|
-
unit_tests = self.generate_tests_for_config({"spec_path": Path(one_test_config.spec_path),
|
87
|
-
"data_path": Path(one_test_config.data_path)},
|
88
|
-
triple_stores)
|
89
|
-
|
210
|
+
if metafunc.function.__name__ == "test_unit":
|
90
211
|
# Create the test in itself
|
91
|
-
if unit_tests:
|
92
|
-
metafunc.parametrize(metafunc.fixturenames[0], unit_tests,
|
212
|
+
if self.unit_tests:
|
213
|
+
metafunc.parametrize(metafunc.fixturenames[0], self.unit_tests,
|
214
|
+
ids=lambda test_param: (test_param.unit_test.spec_file_name or "") + "@" +
|
215
|
+
(test_param.test_config.pytest_path or ""))
|
93
216
|
else:
|
94
217
|
metafunc.parametrize(metafunc.fixturenames[0],
|
95
218
|
[SpecSkipped(MUST.TestSpec, None, "No triplestore found")],
|
96
219
|
ids=lambda x: "No configuration found for this test")
|
220
|
+
|
221
|
+
|
222
|
+
|
97
223
|
|
98
224
|
# Generate test for each triple store available
|
99
|
-
def generate_tests_for_config(self, config, triple_stores):
|
225
|
+
def generate_tests_for_config(self, config, triple_stores, file_name):
|
100
226
|
|
101
227
|
shacl_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdShapes.ttl")))
|
102
228
|
ont_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/ontology.ttl")))
|
103
229
|
valid_spec_uris, spec_graph, invalid_spec_results = validate_specs(config, triple_stores,
|
104
|
-
shacl_graph, ont_graph)
|
230
|
+
shacl_graph, ont_graph, file_name or "*")
|
105
231
|
|
106
232
|
specs, skipped_spec_results = \
|
107
233
|
get_specs(valid_spec_uris, spec_graph, triple_stores, config)
|
@@ -124,17 +250,18 @@ class MustrdTestPlugin:
|
|
124
250
|
def get_triple_stores_from_file(self, test_config):
|
125
251
|
if test_config.triplestore_spec_path:
|
126
252
|
try:
|
127
|
-
triple_stores = get_triple_stores(get_triple_store_graph(Path(test_config.triplestore_spec_path),
|
253
|
+
triple_stores = get_triple_stores(get_triple_store_graph(Path(test_config.triplestore_spec_path),
|
254
|
+
self.secrets))
|
128
255
|
except Exception as e:
|
129
|
-
print(f"""
|
130
|
-
|
131
|
-
triple_stores = [{'type':
|
256
|
+
print(f"""Triplestore configuration parsing failed {test_config.triplestore_spec_path}.
|
257
|
+
Only rdflib will be executed""", e)
|
258
|
+
triple_stores = [{'type': TRIPLESTORE.RdfLib, 'uri': TRIPLESTORE.RdfLib}]
|
132
259
|
else:
|
133
260
|
print("No triple store configuration required: using embedded rdflib")
|
134
|
-
triple_stores = [{'type':
|
261
|
+
triple_stores = [{'type': TRIPLESTORE.RdfLib, 'uri': TRIPLESTORE.RdfLib}]
|
135
262
|
|
136
263
|
if test_config.filter_on_tripleStore:
|
137
|
-
triple_stores = list(filter(lambda triple_store: (triple_store["
|
264
|
+
triple_stores = list(filter(lambda triple_store: (triple_store["uri"] in test_config.filter_on_tripleStore),
|
138
265
|
triple_stores))
|
139
266
|
return triple_stores
|
140
267
|
|
mustrd/namespace.py
CHANGED
@@ -25,18 +25,10 @@ SOFTWARE.
|
|
25
25
|
from rdflib import URIRef
|
26
26
|
from rdflib.namespace import DefinedNamespace, Namespace
|
27
27
|
|
28
|
-
|
28
|
+
# Namespace for the test specifications
|
29
29
|
class MUST(DefinedNamespace):
|
30
30
|
_NS = Namespace("https://mustrd.com/model/")
|
31
|
-
|
32
|
-
# test configuration
|
33
|
-
TestConfig: URIRef
|
34
|
-
hasTestFunction: URIRef
|
35
|
-
hasSpecPath: URIRef
|
36
|
-
hasDataPath: URIRef
|
37
|
-
triplestoreSpecPath: URIRef
|
38
|
-
filterOnTripleStore: URIRef
|
39
|
-
|
31
|
+
|
40
32
|
# Specification classes
|
41
33
|
TestSpec: URIRef
|
42
34
|
SelectSparql: URIRef
|
@@ -50,8 +42,6 @@ class MUST(DefinedNamespace):
|
|
50
42
|
given: URIRef
|
51
43
|
when: URIRef
|
52
44
|
then: URIRef
|
53
|
-
inputGraph: URIRef
|
54
|
-
outputGraph: URIRef # anzo specials?
|
55
45
|
dataSource: URIRef
|
56
46
|
file: URIRef
|
57
47
|
fileName: URIRef
|
@@ -84,22 +74,10 @@ class MUST(DefinedNamespace):
|
|
84
74
|
fileSource: URIRef
|
85
75
|
loadedFromFile: URIRef
|
86
76
|
specSourceFile: URIRef
|
77
|
+
specFileName: URIRef
|
87
78
|
|
88
79
|
# Triple store config parameters
|
89
|
-
url: URIRef
|
90
|
-
port: URIRef
|
91
|
-
username: URIRef
|
92
|
-
password: URIRef
|
93
|
-
inputGraph: URIRef
|
94
|
-
repository: URIRef
|
95
|
-
|
96
|
-
# RDFLib
|
97
|
-
RdfLib: URIRef
|
98
|
-
RdfLibConfig: URIRef
|
99
|
-
|
100
80
|
# Anzo
|
101
|
-
Anzo: URIRef
|
102
|
-
AnzoConfig: URIRef
|
103
81
|
AnzoGraphmartDataset: URIRef
|
104
82
|
AnzoQueryBuilderSparqlSource: URIRef
|
105
83
|
AnzoGraphmartStepSparqlSource: URIRef
|
@@ -107,12 +85,41 @@ class MUST(DefinedNamespace):
|
|
107
85
|
AnzoGraphmartQueryDrivenTemplatedStepSparqlSource: URIRef
|
108
86
|
anzoQueryStep: URIRef
|
109
87
|
anzoGraphmartLayer: URIRef
|
110
|
-
|
111
88
|
|
112
89
|
graphmart: URIRef
|
113
90
|
layer: URIRef
|
114
|
-
gqeURI: URIRef
|
115
91
|
|
116
|
-
|
92
|
+
|
93
|
+
# FIXME: There is nothing to do that by default?
|
94
|
+
@classmethod
|
95
|
+
def get_local_name(cls, uri: URIRef):
|
96
|
+
return str(uri).split(cls._NS)[1]
|
97
|
+
|
98
|
+
# Namespace for triplestores
|
99
|
+
class TRIPLESTORE(DefinedNamespace):
|
100
|
+
_NS = Namespace("https://mustrd.com/triplestore/")
|
101
|
+
RdfLib: URIRef
|
117
102
|
GraphDb: URIRef
|
118
|
-
|
103
|
+
Anzo: URIRef
|
104
|
+
ExternalTripleStore: URIRef
|
105
|
+
InternalTripleStore: URIRef
|
106
|
+
|
107
|
+
gqeURI: URIRef
|
108
|
+
inputGraph: URIRef
|
109
|
+
outputGraph: URIRef # anzo specials? # Triple store config parameters
|
110
|
+
url: URIRef
|
111
|
+
port: URIRef
|
112
|
+
username: URIRef
|
113
|
+
password: URIRef
|
114
|
+
repository: URIRef
|
115
|
+
|
116
|
+
# namespace for pytest_mustrd config
|
117
|
+
class MUSTRDTEST(DefinedNamespace):
|
118
|
+
_NS = Namespace("https://mustrd.com/mustrdTest/")
|
119
|
+
MustrdTest: URIRef
|
120
|
+
hasSpecPath: URIRef
|
121
|
+
hasDataPath: URIRef
|
122
|
+
triplestoreSpecPath: URIRef
|
123
|
+
hasPytestPath: URIRef
|
124
|
+
filterOnTripleStore: URIRef
|
125
|
+
|
mustrd/run.py
CHANGED
@@ -29,7 +29,7 @@ import os
|
|
29
29
|
from rdflib import Graph
|
30
30
|
from .mustrd import get_triple_store_graph, run_specs, get_triple_stores, review_results, validate_specs, get_specs
|
31
31
|
from pathlib import Path
|
32
|
-
from .namespace import
|
32
|
+
from .namespace import TRIPLESTORE
|
33
33
|
from .utils import get_project_root
|
34
34
|
import logging
|
35
35
|
log = logger_setup.setup_logger(__name__)
|
@@ -68,7 +68,7 @@ def main(argv):
|
|
68
68
|
triple_stores = get_triple_stores(get_triple_store_graph(triplestore_spec_path))
|
69
69
|
else:
|
70
70
|
log.info(f"No triple store configuration added, running default configuration")
|
71
|
-
triple_stores = [{'type':
|
71
|
+
triple_stores = [{'type': TRIPLESTORE.RdfLib}]
|
72
72
|
log.info("Triple Stores: " + str(triple_stores))
|
73
73
|
if args.data:
|
74
74
|
run_config["data_path"] = Path(args.data)
|
mustrd/spec_component.py
CHANGED
@@ -37,7 +37,7 @@ import logging
|
|
37
37
|
|
38
38
|
from . import logger_setup
|
39
39
|
from .mustrdAnzo import get_queries_for_layer, get_queries_from_templated_step, get_spec_component_from_graphmart, get_query_from_querybuilder, get_query_from_step
|
40
|
-
from .namespace import MUST
|
40
|
+
from .namespace import MUST, TRIPLESTORE
|
41
41
|
from multimethods import MultiMethod, Default
|
42
42
|
|
43
43
|
log = logger_setup.setup_logger(__name__)
|
@@ -417,7 +417,7 @@ def _get_spec_component_StatementsDataset(spec_component_details: SpecComponentD
|
|
417
417
|
def _get_spec_component_AnzoGraphmartDataset(spec_component_details: SpecComponentDetails) -> SpecComponent:
|
418
418
|
spec_component = init_spec_component(spec_component_details.predicate)
|
419
419
|
|
420
|
-
if spec_component_details.mustrd_triple_store["type"] ==
|
420
|
+
if spec_component_details.mustrd_triple_store["type"] == TRIPLESTORE.Anzo:
|
421
421
|
# Get GIVEN or THEN from anzo graphmart
|
422
422
|
graphmart = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
423
423
|
predicate=MUST.graphmart)
|
@@ -428,7 +428,7 @@ def _get_spec_component_AnzoGraphmartDataset(spec_component_details: SpecCompone
|
|
428
428
|
graphmart=graphmart,
|
429
429
|
layer=layer)
|
430
430
|
else:
|
431
|
-
raise ValueError(f"You must define {
|
431
|
+
raise ValueError(f"You must define {TRIPLESTORE.Anzo} to use {MUST.AnzoGraphmartDataset}")
|
432
432
|
|
433
433
|
return spec_component
|
434
434
|
|
@@ -438,7 +438,7 @@ def _get_spec_component_AnzoQueryBuilderSparqlSource(spec_component_details: Spe
|
|
438
438
|
spec_component = init_spec_component(spec_component_details.predicate)
|
439
439
|
|
440
440
|
# Get WHEN specComponent from query builder
|
441
|
-
if spec_component_details.mustrd_triple_store["type"] ==
|
441
|
+
if spec_component_details.mustrd_triple_store["type"] == TRIPLESTORE.Anzo:
|
442
442
|
query_folder = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
443
443
|
predicate=MUST.queryFolder)
|
444
444
|
query_name = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
@@ -448,7 +448,7 @@ def _get_spec_component_AnzoQueryBuilderSparqlSource(spec_component_details: Spe
|
|
448
448
|
query_name=query_name)
|
449
449
|
# If anzo specific function is called but no anzo defined
|
450
450
|
else:
|
451
|
-
raise ValueError(f"You must define {
|
451
|
+
raise ValueError(f"You must define {TRIPLESTORE.Anzo} to use {MUST.AnzoQueryBuilderSparqlSource}")
|
452
452
|
|
453
453
|
spec_component.queryType = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
454
454
|
predicate=MUST.queryType)
|
@@ -460,14 +460,14 @@ def _get_spec_component_AnzoGraphmartStepSparqlSource(spec_component_details: Sp
|
|
460
460
|
spec_component = init_spec_component(spec_component_details.predicate)
|
461
461
|
|
462
462
|
# Get WHEN specComponent from query builder
|
463
|
-
if spec_component_details.mustrd_triple_store["type"] ==
|
463
|
+
if spec_component_details.mustrd_triple_store["type"] == TRIPLESTORE.Anzo:
|
464
464
|
query_step_uri = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
465
465
|
predicate=MUST.anzoQueryStep)
|
466
466
|
spec_component.value = get_query_from_step(triple_store=spec_component_details.mustrd_triple_store,
|
467
467
|
query_step_uri=query_step_uri)
|
468
468
|
# If anzo specific function is called but no anzo defined
|
469
469
|
else:
|
470
|
-
raise ValueError(f"You must define {
|
470
|
+
raise ValueError(f"You must define {TRIPLESTORE.Anzo} to use {MUST.AnzoGraphmartStepSparqlSource}")
|
471
471
|
|
472
472
|
spec_component.queryType = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
473
473
|
predicate=MUST.queryType)
|
@@ -478,7 +478,7 @@ def _get_spec_component_AnzoGraphmartQueryDrivenTemplatedStepSparqlSource(spec_c
|
|
478
478
|
spec_component = init_spec_component(spec_component_details.predicate, spec_component_details.mustrd_triple_store["type"] )
|
479
479
|
|
480
480
|
# Get WHEN specComponent from query builder
|
481
|
-
if spec_component_details.mustrd_triple_store["type"] ==
|
481
|
+
if spec_component_details.mustrd_triple_store["type"] == TRIPLESTORE.Anzo:
|
482
482
|
query_step_uri = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
483
483
|
predicate=MUST.anzoQueryStep)
|
484
484
|
queries = get_queries_from_templated_step(triple_store=spec_component_details.mustrd_triple_store,
|
@@ -487,7 +487,7 @@ def _get_spec_component_AnzoGraphmartQueryDrivenTemplatedStepSparqlSource(spec_c
|
|
487
487
|
spec_component.queryTemplate = queries["query_template"]
|
488
488
|
# If anzo specific function is called but no anzo defined
|
489
489
|
else:
|
490
|
-
raise ValueError(f"You must define {
|
490
|
+
raise ValueError(f"You must define {TRIPLESTORE.Anzo} to use {MUST.AnzoGraphmartQueryDrivenTemplatedStepSparqlSource}")
|
491
491
|
|
492
492
|
spec_component.queryType = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
493
493
|
predicate=MUST.queryType)
|
@@ -497,7 +497,7 @@ def _get_spec_component_AnzoGraphmartQueryDrivenTemplatedStepSparqlSource(spec_c
|
|
497
497
|
def _get_spec_component_AnzoGraphmartLayerSparqlSource(spec_component_details: SpecComponentDetails) -> list:
|
498
498
|
spec_components = []
|
499
499
|
# Get the ordered WHEN specComponents which is the transform and query driven template queries for the Layer
|
500
|
-
if spec_component_details.mustrd_triple_store["type"] ==
|
500
|
+
if spec_component_details.mustrd_triple_store["type"] == TRIPLESTORE.Anzo:
|
501
501
|
graphmart_layer_uri = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
502
502
|
predicate=MUST.anzoGraphmartLayer)
|
503
503
|
queries = get_queries_for_layer(triple_store=spec_component_details.mustrd_triple_store,
|
@@ -529,7 +529,7 @@ def init_spec_component(predicate: URIRef, triple_store_type: URIRef = None ) ->
|
|
529
529
|
if predicate == MUST.given:
|
530
530
|
spec_component = GivenSpec()
|
531
531
|
elif predicate == MUST.when:
|
532
|
-
if triple_store_type ==
|
532
|
+
if triple_store_type == TRIPLESTORE.Anzo:
|
533
533
|
spec_component = AnzoWhenSpec()
|
534
534
|
else:
|
535
535
|
spec_component = WhenSpec()
|
mustrd/steprunner.py
CHANGED
@@ -26,7 +26,7 @@ import json
|
|
26
26
|
|
27
27
|
from . import logger_setup
|
28
28
|
from multimethods import MultiMethod, Default
|
29
|
-
from .namespace import MUST
|
29
|
+
from .namespace import MUST, TRIPLESTORE
|
30
30
|
from rdflib import Graph, URIRef
|
31
31
|
from .mustrdRdfLib import execute_select as execute_select_rdflib
|
32
32
|
from .mustrdRdfLib import execute_construct as execute_construct_rdflib
|
@@ -53,17 +53,17 @@ def dispatch_upload_given(triple_store: dict, given: Graph):
|
|
53
53
|
upload_given = MultiMethod('upload_given', dispatch_upload_given)
|
54
54
|
|
55
55
|
|
56
|
-
@upload_given.method(
|
56
|
+
@upload_given.method(TRIPLESTORE.RdfLib)
|
57
57
|
def _upload_given_rdflib(triple_store: dict, given: Graph):
|
58
58
|
triple_store["given"] = given
|
59
59
|
|
60
60
|
|
61
|
-
@upload_given.method(
|
61
|
+
@upload_given.method(TRIPLESTORE.GraphDb)
|
62
62
|
def _upload_given_graphdb(triple_store: dict, given: Graph):
|
63
63
|
upload_given_graphdb(triple_store, given)
|
64
64
|
|
65
65
|
|
66
|
-
@upload_given.method(
|
66
|
+
@upload_given.method(TRIPLESTORE.Anzo)
|
67
67
|
def _upload_given_anzo(triple_store: dict, given: Graph):
|
68
68
|
upload_given_anzo(triple_store, given)
|
69
69
|
|
@@ -78,52 +78,52 @@ def dispatch_run_when(spec_uri: URIRef, triple_store: dict, when: WhenSpec):
|
|
78
78
|
run_when = MultiMethod('run_when', dispatch_run_when)
|
79
79
|
|
80
80
|
|
81
|
-
@run_when.method((
|
81
|
+
@run_when.method((TRIPLESTORE.Anzo, MUST.UpdateSparql))
|
82
82
|
def _anzo_run_when_update(spec_uri: URIRef, triple_store: dict, when: AnzoWhenSpec):
|
83
83
|
return execute_update_anzo(triple_store, when.value, when.bindings)
|
84
84
|
|
85
85
|
|
86
|
-
@run_when.method((
|
86
|
+
@run_when.method((TRIPLESTORE.Anzo, MUST.ConstructSparql))
|
87
87
|
def _anzo_run_when_construct(spec_uri: URIRef, triple_store: dict, when: AnzoWhenSpec):
|
88
88
|
return execute_construct_anzo(triple_store, when.value, when.bindings)
|
89
89
|
|
90
90
|
|
91
|
-
@run_when.method((
|
91
|
+
@run_when.method((TRIPLESTORE.Anzo, MUST.SelectSparql))
|
92
92
|
def _anzo_run_when_select(spec_uri: URIRef, triple_store: dict, when: AnzoWhenSpec):
|
93
93
|
return execute_select_anzo(triple_store, when.value, when.bindings)
|
94
94
|
|
95
95
|
|
96
|
-
@run_when.method((
|
96
|
+
@run_when.method((TRIPLESTORE.GraphDb, MUST.UpdateSparql))
|
97
97
|
def _graphdb_run_when_update(spec_uri: URIRef, triple_store: dict, when: WhenSpec):
|
98
98
|
return execute_update_graphdb(triple_store, when.value, when.bindings)
|
99
99
|
|
100
100
|
|
101
|
-
@run_when.method((
|
101
|
+
@run_when.method((TRIPLESTORE.GraphDb, MUST.ConstructSparql))
|
102
102
|
def _graphdb_run_when_construct(spec_uri: URIRef, triple_store: dict, when: WhenSpec):
|
103
103
|
return execute_construct_graphdb(triple_store, when.value, when.bindings)
|
104
104
|
|
105
105
|
|
106
|
-
@run_when.method((
|
106
|
+
@run_when.method((TRIPLESTORE.GraphDb, MUST.SelectSparql))
|
107
107
|
def _graphdb_run_when_select(spec_uri: URIRef, triple_store: dict, when: WhenSpec):
|
108
108
|
return execute_select_graphdb(triple_store, when.value, when.bindings)
|
109
109
|
|
110
110
|
|
111
|
-
@run_when.method((
|
111
|
+
@run_when.method((TRIPLESTORE.RdfLib, MUST.UpdateSparql))
|
112
112
|
def _rdflib_run_when_update(spec_uri: URIRef, triple_store: dict, when: WhenSpec):
|
113
113
|
return execute_update_rdflib(triple_store, triple_store["given"], when.value, when.bindings)
|
114
114
|
|
115
115
|
|
116
|
-
@run_when.method((
|
116
|
+
@run_when.method((TRIPLESTORE.RdfLib, MUST.ConstructSparql))
|
117
117
|
def _rdflib_run_when_construct(spec_uri: URIRef, triple_store: dict, when: WhenSpec):
|
118
118
|
return execute_construct_rdflib(triple_store, triple_store["given"], when.value, when.bindings)
|
119
119
|
|
120
120
|
|
121
|
-
@run_when.method((
|
121
|
+
@run_when.method((TRIPLESTORE.RdfLib, MUST.SelectSparql))
|
122
122
|
def _rdflib_run_when_select(spec_uri: URIRef, triple_store: dict, when: WhenSpec):
|
123
123
|
return execute_select_rdflib(triple_store, triple_store["given"], when.value, when.bindings)
|
124
124
|
|
125
125
|
|
126
|
-
@run_when.method((
|
126
|
+
@run_when.method((TRIPLESTORE.Anzo, MUST.AnzoQueryDrivenUpdateSparql))
|
127
127
|
def _multi_run_when_anzo_query_driven_update(spec_uri: URIRef, triple_store: dict, when: AnzoWhenSpec):
|
128
128
|
# run the parameters query to obtain the values for the template step and put them into a dictionary
|
129
129
|
query_parameters = json.loads(execute_select_anzo(triple_store, when.paramQuery, None))
|
@@ -158,7 +158,7 @@ def _multi_run_when_default(spec_uri: URIRef, triple_store: dict, when: WhenSpec
|
|
158
158
|
elif when.queryType == MUST.DescribeSparql:
|
159
159
|
log.warning(f"Skipping {spec_uri}, SPARQL DESCRIBE not implemented.")
|
160
160
|
msg = "SPARQL DESCRIBE not implemented."
|
161
|
-
elif triple_store['type'] not in [
|
161
|
+
elif triple_store['type'] not in [TRIPLESTORE.Anzo, TRIPLESTORE.GraphDb, TRIPLESTORE.RdfLib]:
|
162
162
|
msg = f"{when.queryType} not implemented for {triple_store['type']}"
|
163
163
|
else:
|
164
164
|
log.warning(f"Skipping {spec_uri}, {when.queryType} is not a valid SPARQL query type.")
|
@@ -0,0 +1,31 @@
|
|
1
|
+
mustrd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
+
mustrd/logger_setup.py,sha256=AXTXiiyQ4E_SyqNaKlLsZlkkKr_YJfJZ0N4PEoN7d7E,1630
|
3
|
+
mustrd/model/catalog-v001.xml,sha256=jgD6VOxR6A8izzr3ZmL4AzaTZ5cVwsvt618MTuRlGBk,386
|
4
|
+
mustrd/model/mustrdShapes.ttl,sha256=BR-9fAwGpQt6glfPpDB0Xx3vFXMt_UTQwi469o6aWUo,9709
|
5
|
+
mustrd/model/mustrdTestOntology.ttl,sha256=W7IRbPKrhpYFweZvc9QH8gdjBiuZMHKETIuHBepLnYw,2034
|
6
|
+
mustrd/model/mustrdTestShapes.ttl,sha256=Bcof6p7MtzO2L6H1DWB1hibLSix7Dxo-z6HlFcG31M8,842
|
7
|
+
mustrd/model/ontology.ttl,sha256=JF6LG5-T7tbi3QfmJcEFQcFo-rQT0CTYGg7PEmuzINY,17303
|
8
|
+
mustrd/model/test-resources/resources.ttl,sha256=7z1P4qJwuqRdmHzKd_mhnUIs-aQoSQxas7yDCwKYarc,1558
|
9
|
+
mustrd/model/triplestoreOntology.ttl,sha256=87leTcKot45AgcCGfJLEwbY9rsuRGHbYbnTFeijrzOk,6199
|
10
|
+
mustrd/model/triplestoreshapes.ttl,sha256=qiLzbclGDzueH5GCKANEwMM9VxOmswu42FiSnITATeQ,1876
|
11
|
+
mustrd/mustrd.py,sha256=-obRBvgFU5QiI5k4zTaf473aHkgvl3ZHQ2xVkdSxS8c,35099
|
12
|
+
mustrd/mustrdAnzo.py,sha256=bQ-IHCnsEK7paJ31GgS9IH3hGd4o_5zeNrpn0Sb8WIk,11822
|
13
|
+
mustrd/mustrdGraphDb.py,sha256=KNXXYyZjYm_IZaeuuxlPBP-XyX9hZ5UvrdxshKnZhwo,5599
|
14
|
+
mustrd/mustrdRdfLib.py,sha256=LeS-uArlxQwsM-7lrJyp8O8m-6VgxsR1BlpxsOCNePs,2129
|
15
|
+
mustrd/mustrdTestPlugin.py,sha256=SsTkwrs0n1lvHvuiyQs-z4dsXnjB7p3d6aFWR8_jLV8,15035
|
16
|
+
mustrd/namespace.py,sha256=3zJbdWEji_2SkhasHK3J2UO3am_phNKDwAFk2htXRYM,3765
|
17
|
+
mustrd/README.adoc,sha256=56W59hX4xyttspTXUXRJAgqemBSVavWHpfs6Lar_-qo,11995
|
18
|
+
mustrd/run.py,sha256=r3FS_iAuXYpENbbrLfgymGUP4dx-ZGJ_Y2ZvnZdL9dg,4380
|
19
|
+
mustrd/spec_component.py,sha256=0D81UlZEpUGk9FZfzUElp8a6kEoFsfkr7bGasKDTdSU,32546
|
20
|
+
mustrd/steprunner.py,sha256=n3k4O2SjsLUEG5QOxLG8Z8D3iH9PJzzR63dIV2KHcBE,7555
|
21
|
+
mustrd/templates/md_ResultList_leaf_template.jinja,sha256=un1XMeNO9xLcry-DmTQ2QoWoTM5Q5CkVUV_LS4O7sHM,526
|
22
|
+
mustrd/templates/md_ResultList_template.jinja,sha256=RrTaPN1obsTEpG5BjQp7BV9iBFzZOXHwslgvyC0hDKY,211
|
23
|
+
mustrd/templates/md_stats_template.jinja,sha256=DlNfH9eoEFVHQmvp6QNbQv36cqRLyBkqG1ITOHb_Yu8,157
|
24
|
+
mustrd/test/test_mustrd.py,sha256=Q2qTTY--I0wfhgHW7hnkIoV3t_sUTXm8xMmirJaFc7o,129
|
25
|
+
mustrd/TestResult.py,sha256=q5BnvRxhfCEEY062rHasnA61AeKR4oXXs5z8lASvx1o,4986
|
26
|
+
mustrd/utils.py,sha256=fuvdLE20MdkJtY5atBiU8-u7sVcq1mzIYF0gTcw1UMk,1424
|
27
|
+
mustrd-0.1.8.dist-info/entry_points.txt,sha256=v7V7sN0_L1aB4Ug_9io5axlQSeJ1C0tNrQWwdXdV58s,50
|
28
|
+
mustrd-0.1.8.dist-info/LICENSE,sha256=OwjAcbL4HsENdMen1iYR23HH7OEn899oCt100xUo0Zo,1099
|
29
|
+
mustrd-0.1.8.dist-info/METADATA,sha256=lxU94amogEOoqPHnF1cdKqK8ceXahLbgpXudBQ3pGCg,4130
|
30
|
+
mustrd-0.1.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
31
|
+
mustrd-0.1.8.dist-info/RECORD,,
|