mustrd 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mustrd/README.adoc +0 -177
- mustrd/TestResult.py +1 -1
- mustrd/mustrd.py +90 -99
- mustrd/mustrdAnzo.py +30 -15
- mustrd/mustrdTestPlugin.py +48 -41
- mustrd/namespace.py +9 -8
- mustrd/run.py +3 -4
- mustrd/spec_component.py +109 -98
- mustrd/test/test_mustrd.py +1 -1
- mustrd/utils.py +1 -0
- {mustrd-0.2.1.dist-info → mustrd-0.2.2.dist-info}/METADATA +10 -5
- {mustrd-0.2.1.dist-info → mustrd-0.2.2.dist-info}/RECORD +15 -15
- {mustrd-0.2.1.dist-info → mustrd-0.2.2.dist-info}/LICENSE +0 -0
- {mustrd-0.2.1.dist-info → mustrd-0.2.2.dist-info}/WHEEL +0 -0
- {mustrd-0.2.1.dist-info → mustrd-0.2.2.dist-info}/entry_points.txt +0 -0
mustrd/spec_component.py
CHANGED
@@ -29,16 +29,17 @@ from typing import Tuple, List, Type
|
|
29
29
|
|
30
30
|
import pandas
|
31
31
|
import requests
|
32
|
-
from rdflib import RDF, Graph, URIRef, Variable, Literal, XSD, util
|
32
|
+
from rdflib import RDF, Graph, URIRef, Variable, Literal, XSD, util, ConjunctiveGraph
|
33
33
|
from rdflib.exceptions import ParserError
|
34
34
|
from rdflib.term import Node
|
35
|
-
import
|
35
|
+
from rdflib.plugins.stores.memory import Memory
|
36
36
|
|
37
37
|
from . import logger_setup
|
38
|
-
from .mustrdAnzo import get_queries_for_layer, get_queries_from_templated_step, get_spec_component_from_graphmart
|
38
|
+
from .mustrdAnzo import get_queries_for_layer, get_queries_from_templated_step, get_spec_component_from_graphmart
|
39
|
+
from .mustrdAnzo import get_query_from_querybuilder, get_query_from_step
|
39
40
|
from .namespace import MUST, TRIPLESTORE
|
40
41
|
from multimethods import MultiMethod, Default
|
41
|
-
from .utils import
|
42
|
+
from .utils import get_mustrd_root
|
42
43
|
|
43
44
|
log = logger_setup.setup_logger(__name__)
|
44
45
|
|
@@ -50,7 +51,7 @@ class SpecComponent:
|
|
50
51
|
|
51
52
|
@dataclass
|
52
53
|
class GivenSpec(SpecComponent):
|
53
|
-
value:
|
54
|
+
value: ConjunctiveGraph = None
|
54
55
|
|
55
56
|
|
56
57
|
@dataclass
|
@@ -59,11 +60,13 @@ class WhenSpec(SpecComponent):
|
|
59
60
|
queryType: URIRef = None
|
60
61
|
bindings: dict = None
|
61
62
|
|
63
|
+
|
62
64
|
@dataclass
|
63
65
|
class AnzoWhenSpec(WhenSpec):
|
64
66
|
paramQuery: str = None
|
65
67
|
queryTemplate: str = None
|
66
68
|
|
69
|
+
|
67
70
|
@dataclass
|
68
71
|
class ThenSpec(SpecComponent):
|
69
72
|
value: Graph = Graph()
|
@@ -86,22 +89,21 @@ class SpecComponentDetails:
|
|
86
89
|
run_config: dict
|
87
90
|
root_paths: list
|
88
91
|
|
92
|
+
|
89
93
|
def get_path(path_type: str, file_name, spec_component_details: SpecComponentDetails) -> Path:
|
90
94
|
if path_type in spec_component_details.run_config:
|
91
95
|
relative_path = os.path.join(spec_component_details.run_config[path_type], file_name)
|
92
96
|
else:
|
93
97
|
relative_path = file_name
|
94
98
|
return get_file_absolute_path(spec_component_details, relative_path)
|
95
|
-
|
99
|
+
|
96
100
|
|
97
101
|
def parse_spec_component(subject: URIRef,
|
98
102
|
predicate: URIRef,
|
99
103
|
spec_graph: Graph,
|
100
104
|
run_config: dict,
|
101
105
|
mustrd_triple_store: dict) -> GivenSpec | WhenSpec | ThenSpec | TableThenSpec:
|
102
|
-
# print(f"parse_spec_component {subject=} {predicate=} ")
|
103
106
|
spec_component_nodes = get_spec_component_nodes(subject, predicate, spec_graph)
|
104
|
-
# all_data_source_types = []
|
105
107
|
spec_components = []
|
106
108
|
for spec_component_node in spec_component_nodes:
|
107
109
|
data_source_types = get_data_source_types(subject, predicate, spec_graph, spec_component_node)
|
@@ -113,41 +115,41 @@ def parse_spec_component(subject: URIRef,
|
|
113
115
|
mustrd_triple_store=mustrd_triple_store,
|
114
116
|
spec_component_node=spec_component_node,
|
115
117
|
data_source_type=data_source_type,
|
116
|
-
run_config=run_config,
|
118
|
+
run_config=run_config,
|
117
119
|
root_paths=get_components_roots(spec_graph, subject, run_config))
|
118
120
|
spec_component = get_spec_component(spec_component_details)
|
119
|
-
if
|
120
|
-
spec_components += spec_component
|
121
|
+
if isinstance(spec_component, list):
|
122
|
+
spec_components += spec_component
|
121
123
|
else:
|
122
124
|
spec_components += [spec_component]
|
123
|
-
|
124
|
-
# all_data_source_types.extend(data_source_types)
|
125
|
-
# return all_data_source_types
|
126
125
|
# merge multiple graphs into one, give error if spec config is a TableThen
|
127
126
|
# print(f"calling multimethod with {spec_components}")
|
128
127
|
return combine_specs(spec_components)
|
129
128
|
|
129
|
+
|
130
130
|
# Here we retrieve all the possible root paths for a specification component.
|
131
131
|
# This defines the order of priority between root paths which is:
|
132
132
|
# 1) Path where the spec is located
|
133
133
|
# 2) spec_path defined in mustrd test configuration files or cmd line argument
|
134
134
|
# 3) data_path defined in mustrd test configuration files or cmd line argument
|
135
|
-
# 4) Mustrd source folder: In case of default resources packaged with mustrd source
|
135
|
+
# 4) Mustrd source folder: In case of default resources packaged with mustrd source
|
136
|
+
# (will be in venv when mustrd is called as library)
|
136
137
|
# We intentionally don't try for absolute files, but you should feel free to argue that we should do
|
137
138
|
def get_components_roots(spec_graph: Graph, subject: URIRef, run_config: dict):
|
138
139
|
where_did_i_load_this_spec_from = spec_graph.value(subject=subject,
|
139
|
-
|
140
|
-
roots = []
|
141
|
-
if
|
142
|
-
log.error(f"{where_did_i_load_this_spec_from=} was None for test_spec={subject},
|
140
|
+
predicate=MUST.specSourceFile)
|
141
|
+
roots = []
|
142
|
+
if not where_did_i_load_this_spec_from:
|
143
|
+
log.error(f"""{where_did_i_load_this_spec_from=} was None for test_spec={subject},
|
144
|
+
we didn't set the test specifications specSourceFile when loading, spec_graph={spec_graph}""")
|
143
145
|
else:
|
144
|
-
roots.append(Path(os.path.dirname(where_did_i_load_this_spec_from)))
|
145
|
-
if run_config and'spec_path' in run_config:
|
146
|
+
roots.append(Path(os.path.dirname(where_did_i_load_this_spec_from)))
|
147
|
+
if run_config and 'spec_path' in run_config:
|
146
148
|
roots.append(Path(run_config['spec_path']))
|
147
|
-
if run_config and 'data_path'
|
149
|
+
if run_config and 'data_path' in run_config:
|
148
150
|
roots.append(run_config['data_path'])
|
149
151
|
roots.append(get_mustrd_root())
|
150
|
-
|
152
|
+
|
151
153
|
return roots
|
152
154
|
|
153
155
|
|
@@ -155,18 +157,20 @@ def get_components_roots(spec_graph: Graph, subject: URIRef, run_config: dict):
|
|
155
157
|
def get_file_absolute_path(spec_component_details: SpecComponentDetails, relative_file_path: str):
|
156
158
|
if not relative_file_path:
|
157
159
|
raise ValueError("Cannot get absolute path of None")
|
158
|
-
absolute_file_paths = list(map(lambda root_path: Path(os.path.join(root_path, relative_file_path)),
|
160
|
+
absolute_file_paths = list(map(lambda root_path: Path(os.path.join(root_path, relative_file_path)),
|
161
|
+
spec_component_details.root_paths))
|
159
162
|
for absolute_file_path in absolute_file_paths:
|
160
163
|
if (os.path.exists(absolute_file_path)):
|
161
164
|
return absolute_file_path
|
162
165
|
raise FileNotFoundError(f"Could not find file {relative_file_path=} in any of the {absolute_file_paths=}")
|
163
166
|
|
167
|
+
|
164
168
|
def get_spec_component_type(spec_components: List[SpecComponent]) -> Type[SpecComponent]:
|
165
169
|
# Get the type of the first object in the list
|
166
170
|
spec_type = type(spec_components[0])
|
167
171
|
# Loop through the remaining objects in the list and check their types
|
168
172
|
for spec_component in spec_components[1:]:
|
169
|
-
if
|
173
|
+
if not isinstance(spec_component, spec_type):
|
170
174
|
# If an object has a different type, raise an error
|
171
175
|
raise ValueError("All spec components must be of the same type")
|
172
176
|
|
@@ -216,7 +220,7 @@ def _combine_then_specs(spec_components: List[ThenSpec]) -> ThenSpec:
|
|
216
220
|
@combine_specs.method(TableThenSpec)
|
217
221
|
def _combine_table_then_specs(spec_components: List[TableThenSpec]) -> TableThenSpec:
|
218
222
|
if len(spec_components) != 1:
|
219
|
-
raise ValueError(
|
223
|
+
raise ValueError("Parsing of multiple components of MUST.then for tables not implemented")
|
220
224
|
return spec_components[0]
|
221
225
|
|
222
226
|
|
@@ -224,6 +228,7 @@ def _combine_table_then_specs(spec_components: List[TableThenSpec]) -> TableThen
|
|
224
228
|
def _combine_specs_default(spec_components: List[SpecComponent]):
|
225
229
|
raise ValueError(f"Parsing of multiple components of this type not implemented {spec_components}")
|
226
230
|
|
231
|
+
|
227
232
|
def get_data_source_types(subject: URIRef, predicate: URIRef, spec_graph: Graph, source_node: Node) -> List[Node]:
|
228
233
|
data_source_types = []
|
229
234
|
for data_source_type in spec_graph.objects(subject=source_node, predicate=RDF.type):
|
@@ -233,6 +238,7 @@ def get_data_source_types(subject: URIRef, predicate: URIRef, spec_graph: Graph,
|
|
233
238
|
raise ValueError(f"Node has no rdf type {subject} {predicate}")
|
234
239
|
return data_source_types
|
235
240
|
|
241
|
+
|
236
242
|
# https://github.com/Semantic-partners/mustrd/issues/99
|
237
243
|
def get_spec_component_dispatch(spec_component_details: SpecComponentDetails) -> Tuple[Node, URIRef]:
|
238
244
|
return spec_component_details.data_source_type, spec_component_details.predicate
|
@@ -254,7 +260,7 @@ def _get_spec_component_folderdatasource_given(spec_component_details: SpecCompo
|
|
254
260
|
file_name = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
255
261
|
predicate=MUST.fileName)
|
256
262
|
|
257
|
-
path = get_path('given_path', file_name,spec_component_details)
|
263
|
+
path = get_path('given_path', file_name, spec_component_details)
|
258
264
|
try:
|
259
265
|
spec_component.value = Graph().parse(data=get_spec_component_from_file(path))
|
260
266
|
except ParserError as e:
|
@@ -270,10 +276,11 @@ def _get_spec_component_foldersparqlsource_when(spec_component_details: SpecComp
|
|
270
276
|
file_name = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
271
277
|
predicate=MUST.fileName)
|
272
278
|
|
273
|
-
path = get_path('when_path', file_name,spec_component_details)
|
279
|
+
path = get_path('when_path', file_name, spec_component_details)
|
274
280
|
spec_component.value = get_spec_component_from_file(path)
|
275
|
-
spec_component.queryType = spec_component_details.spec_graph.value(
|
276
|
-
|
281
|
+
spec_component.queryType = spec_component_details.spec_graph.value(
|
282
|
+
subject=spec_component_details.spec_component_node,
|
283
|
+
predicate=MUST.queryType)
|
277
284
|
return spec_component
|
278
285
|
|
279
286
|
|
@@ -283,21 +290,23 @@ def _get_spec_component_folderdatasource_then(spec_component_details: SpecCompon
|
|
283
290
|
|
284
291
|
file_name = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
285
292
|
predicate=MUST.fileName)
|
286
|
-
path = get_path('then_path', file_name,spec_component_details)
|
293
|
+
path = get_path('then_path', file_name, spec_component_details)
|
287
294
|
|
288
295
|
return load_dataset_from_file(path, spec_component)
|
289
296
|
|
297
|
+
|
290
298
|
@get_spec_component.method((MUST.FileDataset, MUST.given))
|
291
299
|
@get_spec_component.method((MUST.FileDataset, MUST.then))
|
292
300
|
def _get_spec_component_filedatasource(spec_component_details: SpecComponentDetails) -> GivenSpec:
|
293
301
|
spec_component = init_spec_component(spec_component_details.predicate)
|
294
302
|
return load_spec_component(spec_component_details, spec_component)
|
295
303
|
|
304
|
+
|
296
305
|
def load_spec_component(spec_component_details, spec_component):
|
297
306
|
file_path = Path(str(spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
298
307
|
predicate=MUST.file)))
|
299
308
|
return load_dataset_from_file(get_file_absolute_path(spec_component_details, file_path), spec_component)
|
300
|
-
|
309
|
+
|
301
310
|
|
302
311
|
def load_dataset_from_file(path: Path, spec_component: ThenSpec) -> ThenSpec:
|
303
312
|
if path.is_dir():
|
@@ -326,34 +335,21 @@ def load_dataset_from_file(path: Path, spec_component: ThenSpec) -> ThenSpec:
|
|
326
335
|
return spec_component
|
327
336
|
|
328
337
|
|
329
|
-
|
330
338
|
@get_spec_component.method((MUST.FileSparqlSource, MUST.when))
|
331
339
|
def _get_spec_component_filedatasource_when(spec_component_details: SpecComponentDetails) -> SpecComponent:
|
332
340
|
spec_component = init_spec_component(spec_component_details.predicate)
|
333
341
|
|
334
342
|
file_path = Path(str(spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
335
|
-
predicate=MUST.file)))
|
343
|
+
predicate=MUST.file)))
|
336
344
|
spec_component.value = get_spec_component_from_file(get_file_absolute_path(spec_component_details, file_path))
|
337
345
|
|
338
|
-
spec_component.queryType = spec_component_details.spec_graph.value(
|
339
|
-
|
346
|
+
spec_component.queryType = spec_component_details.spec_graph.value(
|
347
|
+
subject=spec_component_details.spec_component_node,
|
348
|
+
predicate=MUST.queryType)
|
340
349
|
|
341
350
|
return spec_component
|
342
351
|
|
343
352
|
|
344
|
-
# @get_spec_component.method((MUST.FileDataset, MUST.then))
|
345
|
-
# def _get_spec_component_filedatasource_then(spec_component_details: SpecComponentDetails) -> SpecComponent:
|
346
|
-
# spec_component = init_spec_component(spec_component_details.predicate)
|
347
|
-
|
348
|
-
# file_path = Path(str(spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
349
|
-
# predicate=MUST.file)))
|
350
|
-
# if str(file_path).startswith("/"): # absolute path
|
351
|
-
# path = file_path
|
352
|
-
# else: #relative path
|
353
|
-
# path = Path(os.path.join(spec_component_details.run_config['spec_path'], file_path))
|
354
|
-
# return get_then_from_file(path, spec_component)
|
355
|
-
|
356
|
-
|
357
353
|
@get_spec_component.method((MUST.TextSparqlSource, MUST.when))
|
358
354
|
def _get_spec_component_TextSparqlSource(spec_component_details: SpecComponentDetails) -> SpecComponent:
|
359
355
|
spec_component = init_spec_component(spec_component_details.predicate)
|
@@ -364,8 +360,9 @@ def _get_spec_component_TextSparqlSource(spec_component_details: SpecComponentDe
|
|
364
360
|
predicate=MUST.queryText))
|
365
361
|
|
366
362
|
spec_component.bindings = get_when_bindings(spec_component_details.subject, spec_component_details.spec_graph)
|
367
|
-
spec_component.queryType = spec_component_details.spec_graph.value(
|
368
|
-
|
363
|
+
spec_component.queryType = spec_component_details.spec_graph.value(
|
364
|
+
subject=spec_component_details.spec_component_node,
|
365
|
+
predicate=MUST.queryType)
|
369
366
|
return spec_component
|
370
367
|
|
371
368
|
|
@@ -379,9 +376,10 @@ def _get_spec_component_HttpDataset(spec_component_details: SpecComponentDetails
|
|
379
376
|
# Get specComponent with http GET protocol
|
380
377
|
spec_component.value = requests.get(str(
|
381
378
|
spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
382
|
-
predicate=MUST.dataSourceUrl)).content)
|
383
|
-
spec_component.queryType = spec_component_details.spec_graph.value(
|
384
|
-
|
379
|
+
predicate=MUST.dataSourceUrl)).content)
|
380
|
+
spec_component.queryType = spec_component_details.spec_graph.value(
|
381
|
+
subject=spec_component_details.spec_component_node,
|
382
|
+
predicate=MUST.queryType)
|
385
383
|
return spec_component
|
386
384
|
|
387
385
|
|
@@ -413,10 +411,14 @@ def _get_spec_component_EmptyGraph(spec_component_details: SpecComponentDetails)
|
|
413
411
|
@get_spec_component.method((MUST.StatementsDataset, MUST.then))
|
414
412
|
def _get_spec_component_StatementsDataset(spec_component_details: SpecComponentDetails) -> SpecComponent:
|
415
413
|
spec_component = init_spec_component(spec_component_details.predicate)
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
414
|
+
store = Memory()
|
415
|
+
g = URIRef("http://localhost:7200/test-graph")
|
416
|
+
spec_component.value = ConjunctiveGraph(store=store)
|
417
|
+
spec_graph = Graph(store=store, identifier=g)
|
418
|
+
|
419
|
+
data = get_spec_from_statements(spec_component_details.subject, spec_component_details.predicate,
|
420
|
+
spec_component_details.spec_graph)
|
421
|
+
spec_graph.parse(data=data)
|
420
422
|
return spec_component
|
421
423
|
|
422
424
|
|
@@ -458,8 +460,9 @@ def _get_spec_component_AnzoQueryBuilderSparqlSource(spec_component_details: Spe
|
|
458
460
|
else:
|
459
461
|
raise ValueError(f"You must define {TRIPLESTORE.Anzo} to use {MUST.AnzoQueryBuilderSparqlSource}")
|
460
462
|
|
461
|
-
spec_component.queryType = spec_component_details.spec_graph.value(
|
462
|
-
|
463
|
+
spec_component.queryType = spec_component_details.spec_graph.value(
|
464
|
+
subject=spec_component_details.spec_component_node,
|
465
|
+
predicate=MUST.queryType)
|
463
466
|
return spec_component
|
464
467
|
|
465
468
|
|
@@ -470,62 +473,72 @@ def _get_spec_component_AnzoGraphmartStepSparqlSource(spec_component_details: Sp
|
|
470
473
|
# Get WHEN specComponent from query builder
|
471
474
|
if spec_component_details.mustrd_triple_store["type"] == TRIPLESTORE.Anzo:
|
472
475
|
query_step_uri = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
473
|
-
|
476
|
+
predicate=MUST.anzoQueryStep)
|
474
477
|
spec_component.value = get_query_from_step(triple_store=spec_component_details.mustrd_triple_store,
|
475
|
-
|
478
|
+
query_step_uri=query_step_uri)
|
476
479
|
# If anzo specific function is called but no anzo defined
|
477
480
|
else:
|
478
481
|
raise ValueError(f"You must define {TRIPLESTORE.Anzo} to use {MUST.AnzoGraphmartStepSparqlSource}")
|
479
482
|
|
480
|
-
spec_component.queryType = spec_component_details.spec_graph.value(
|
481
|
-
|
483
|
+
spec_component.queryType = spec_component_details.spec_graph.value(
|
484
|
+
subject=spec_component_details.spec_component_node,
|
485
|
+
predicate=MUST.queryType)
|
482
486
|
return spec_component
|
483
487
|
|
488
|
+
|
484
489
|
@get_spec_component.method((MUST.AnzoGraphmartQueryDrivenTemplatedStepSparqlSource, MUST.when))
|
485
|
-
def _get_spec_component_AnzoGraphmartQueryDrivenTemplatedStepSparqlSource(spec_component_details: SpecComponentDetails) -> SpecComponent:
|
486
|
-
spec_component = init_spec_component(
|
490
|
+
def _get_spec_component_AnzoGraphmartQueryDrivenTemplatedStepSparqlSource(spec_component_details: SpecComponentDetails) -> SpecComponent: # noqa
|
491
|
+
spec_component = init_spec_component(
|
492
|
+
spec_component_details.predicate, spec_component_details.mustrd_triple_store["type"])
|
487
493
|
|
488
494
|
# Get WHEN specComponent from query builder
|
489
495
|
if spec_component_details.mustrd_triple_store["type"] == TRIPLESTORE.Anzo:
|
490
496
|
query_step_uri = spec_component_details.spec_graph.value(subject=spec_component_details.spec_component_node,
|
491
|
-
|
497
|
+
predicate=MUST.anzoQueryStep)
|
492
498
|
queries = get_queries_from_templated_step(triple_store=spec_component_details.mustrd_triple_store,
|
493
|
-
|
494
|
-
spec_component.paramQuery= queries["param_query"]
|
499
|
+
query_step_uri=query_step_uri)
|
500
|
+
spec_component.paramQuery = queries["param_query"]
|
495
501
|
spec_component.queryTemplate = queries["query_template"]
|
496
502
|
# If anzo specific function is called but no anzo defined
|
497
503
|
else:
|
498
|
-
raise ValueError(f"You must define {TRIPLESTORE.Anzo}
|
504
|
+
raise ValueError(f"""You must define {TRIPLESTORE.Anzo}
|
505
|
+
to use {MUST.AnzoGraphmartQueryDrivenTemplatedStepSparqlSource}""")
|
499
506
|
|
500
|
-
spec_component.queryType = spec_component_details.spec_graph.value(
|
501
|
-
|
507
|
+
spec_component.queryType = spec_component_details.spec_graph.value(
|
508
|
+
subject=spec_component_details.spec_component_node,
|
509
|
+
predicate=MUST.queryType)
|
502
510
|
return spec_component
|
503
511
|
|
512
|
+
|
504
513
|
@get_spec_component.method((MUST.AnzoGraphmartLayerSparqlSource, MUST.when))
|
505
514
|
def _get_spec_component_AnzoGraphmartLayerSparqlSource(spec_component_details: SpecComponentDetails) -> list:
|
506
515
|
spec_components = []
|
507
516
|
# Get the ordered WHEN specComponents which is the transform and query driven template queries for the Layer
|
508
517
|
if spec_component_details.mustrd_triple_store["type"] == TRIPLESTORE.Anzo:
|
509
|
-
graphmart_layer_uri = spec_component_details.spec_graph.value(
|
510
|
-
|
518
|
+
graphmart_layer_uri = spec_component_details.spec_graph.value(
|
519
|
+
subject=spec_component_details.spec_component_node,
|
520
|
+
predicate=MUST.anzoGraphmartLayer)
|
511
521
|
queries = get_queries_for_layer(triple_store=spec_component_details.mustrd_triple_store,
|
512
|
-
|
522
|
+
graphmart_layer_uri=graphmart_layer_uri)
|
513
523
|
# If anzo specific function is called but no anzo defined
|
514
524
|
else:
|
515
|
-
raise ValueError(
|
525
|
+
raise ValueError("This test specification is specific to Anzo and can only be run against that platform.")
|
516
526
|
for query in queries:
|
517
|
-
spec_component = init_spec_component(spec_component_details.predicate,
|
518
|
-
|
527
|
+
spec_component = init_spec_component(spec_component_details.predicate,
|
528
|
+
spec_component_details.mustrd_triple_store["type"])
|
529
|
+
spec_component.value = query.get("query")
|
519
530
|
spec_component.paramQuery = query.get("param_query")
|
520
531
|
spec_component.queryTemplate = query.get("query_template")
|
521
532
|
if spec_component.value:
|
522
|
-
spec_component.queryType = spec_component_details.spec_graph.value(
|
523
|
-
|
533
|
+
spec_component.queryType = spec_component_details.spec_graph.value(
|
534
|
+
subject=spec_component_details.spec_component_node,
|
535
|
+
predicate=MUST.queryType)
|
524
536
|
else:
|
525
|
-
spec_component.queryType
|
537
|
+
spec_component.queryType = MUST.AnzoQueryDrivenUpdateSparql
|
526
538
|
spec_components += [spec_component]
|
527
539
|
return spec_components
|
528
540
|
|
541
|
+
|
529
542
|
@get_spec_component.method(Default)
|
530
543
|
def _get_spec_component_default(spec_component_details: SpecComponentDetails) -> SpecComponent:
|
531
544
|
raise ValueError(
|
@@ -533,7 +546,7 @@ def _get_spec_component_default(spec_component_details: SpecComponentDetails) ->
|
|
533
546
|
f"spec component ({spec_component_details.predicate})")
|
534
547
|
|
535
548
|
|
536
|
-
def init_spec_component(predicate: URIRef, triple_store_type: URIRef = None
|
549
|
+
def init_spec_component(predicate: URIRef, triple_store_type: URIRef = None) -> GivenSpec | WhenSpec | ThenSpec | TableThenSpec: # noqa
|
537
550
|
if predicate == MUST.given:
|
538
551
|
spec_component = GivenSpec()
|
539
552
|
elif predicate == MUST.when:
|
@@ -559,9 +572,6 @@ def get_spec_component_nodes(subject: URIRef, predicate: URIRef, spec_graph: Gra
|
|
559
572
|
|
560
573
|
|
561
574
|
def get_spec_component_from_file(path: Path) -> str:
|
562
|
-
# project_root = get_project_root()
|
563
|
-
# file_path = Path(os.path.join(project_root, path))
|
564
|
-
|
565
575
|
if path.is_dir():
|
566
576
|
raise ValueError(f"Path {path} is a directory, expected a file")
|
567
577
|
|
@@ -576,14 +586,13 @@ def get_spec_from_statements(subject: URIRef,
|
|
576
586
|
predicate: URIRef,
|
577
587
|
spec_graph: Graph) -> Graph:
|
578
588
|
statements_query = f"""
|
579
|
-
prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
589
|
+
prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
580
590
|
|
581
591
|
CONSTRUCT {{ ?s ?p ?o }}
|
582
592
|
{{
|
583
593
|
<{subject}> <{predicate}> [
|
584
594
|
a <{MUST.StatementsDataset}> ;
|
585
595
|
<{MUST.hasStatement}> [
|
586
|
-
a rdf:Statement ;
|
587
596
|
rdf:subject ?s ;
|
588
597
|
rdf:predicate ?p ;
|
589
598
|
rdf:object ?o ;
|
@@ -601,17 +610,17 @@ def get_spec_from_table(subject: URIRef,
|
|
601
610
|
spec_graph: Graph) -> pandas.DataFrame:
|
602
611
|
# query the spec to get the expected result to convert to dataframe for comparison
|
603
612
|
then_query = f"""
|
604
|
-
prefix sh: <http://www.w3.org/ns/shacl#>
|
613
|
+
prefix sh: <http://www.w3.org/ns/shacl#>
|
605
614
|
SELECT ?row ?variable ?binding ?order
|
606
|
-
WHERE {{
|
615
|
+
WHERE {{
|
607
616
|
<{subject}> <{predicate}> [
|
608
617
|
a <{MUST.TableDataset}> ;
|
609
618
|
<{MUST.hasRow}> ?row ].
|
610
619
|
?row <{MUST.hasBinding}> [
|
611
620
|
<{MUST.variable}> ?variable ;
|
612
621
|
<{MUST.boundValue}> ?binding ; ] .
|
613
|
-
OPTIONAL {{ ?row sh:order ?order . }}
|
614
|
-
.}}
|
622
|
+
OPTIONAL {{ ?row sh:order ?order . }}
|
623
|
+
.}}
|
615
624
|
ORDER BY ?order"""
|
616
625
|
|
617
626
|
expected_results = spec_graph.query(then_query)
|
@@ -630,7 +639,7 @@ def get_spec_from_table(subject: URIRef,
|
|
630
639
|
for row in expected_results:
|
631
640
|
df.loc[str(row.row), row.variable.value] = str(row.binding)
|
632
641
|
df.loc[str(row.row), "order"] = row.order
|
633
|
-
if
|
642
|
+
if isinstance(row.binding, Literal):
|
634
643
|
literal_type = str(XSD.string)
|
635
644
|
if hasattr(row.binding, "datatype") and row.binding.datatype:
|
636
645
|
literal_type = str(row.binding.datatype)
|
@@ -648,7 +657,9 @@ def get_spec_from_table(subject: URIRef,
|
|
648
657
|
|
649
658
|
def get_when_bindings(subject: URIRef,
|
650
659
|
spec_graph: Graph) -> dict:
|
651
|
-
when_bindings_query = f"""SELECT ?variable ?binding {{ <{subject}> <{MUST.when}> [ a <{MUST.TextSparqlSource}> ;
|
660
|
+
when_bindings_query = f"""SELECT ?variable ?binding {{ <{subject}> <{MUST.when}> [ a <{MUST.TextSparqlSource}> ;
|
661
|
+
<{MUST.hasBinding}> [ <{MUST.variable}> ?variable ;
|
662
|
+
<{MUST.boundValue}> ?binding ; ] ; ] ;}}"""
|
652
663
|
when_bindings = spec_graph.query(when_bindings_query)
|
653
664
|
|
654
665
|
if len(when_bindings.bindings) == 0:
|
@@ -663,24 +674,24 @@ def get_when_bindings(subject: URIRef,
|
|
663
674
|
def is_then_select_ordered(subject: URIRef, predicate: URIRef, spec_graph: Graph) -> bool:
|
664
675
|
ask_select_ordered = f"""
|
665
676
|
ASK {{
|
666
|
-
{{SELECT (count(?binding) as ?totalBindings) {{
|
677
|
+
{{SELECT (count(?binding) as ?totalBindings) {{
|
667
678
|
<{subject}> <{predicate}> [
|
668
679
|
a <{MUST.TableDataset}> ;
|
669
680
|
<{MUST.hasRow}> [ <{MUST.hasBinding}> [
|
670
681
|
<{MUST.variable}> ?variable ;
|
671
682
|
<{MUST.boundValue}> ?binding ;
|
672
|
-
] ;
|
683
|
+
] ;
|
673
684
|
]
|
674
685
|
]
|
675
686
|
}} }}
|
676
|
-
{{SELECT (count(?binding) as ?orderedBindings) {{
|
687
|
+
{{SELECT (count(?binding) as ?orderedBindings) {{
|
677
688
|
<{subject}> <{predicate}> [
|
678
689
|
a <{MUST.TableDataset}> ;
|
679
690
|
<{MUST.hasRow}> [ sh:order ?order ;
|
680
|
-
<{MUST.hasBinding}> [
|
691
|
+
<{MUST.hasBinding}> [
|
681
692
|
<{MUST.variable}> ?variable ;
|
682
693
|
<{MUST.boundValue}> ?binding ;
|
683
|
-
] ;
|
694
|
+
] ;
|
684
695
|
]
|
685
696
|
]
|
686
697
|
}} }}
|
mustrd/test/test_mustrd.py
CHANGED
mustrd/utils.py
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: mustrd
|
3
|
-
Version: 0.2.
|
3
|
+
Version: 0.2.2
|
4
4
|
Summary: A Spec By Example framework for RDF and SPARQL, Inspired by Cucumber.
|
5
5
|
Home-page: https://github.com/Semantic-partners/mustrd
|
6
6
|
License: MIT
|
7
7
|
Author: John Placek
|
8
8
|
Author-email: john.placek@semanticpartners.com
|
9
|
-
Requires-Python: >=3.11
|
9
|
+
Requires-Python: >=3.11,<4.0
|
10
10
|
Classifier: Framework :: Pytest
|
11
11
|
Classifier: License :: OSI Approved :: MIT License
|
12
12
|
Classifier: Natural Language :: English
|
13
13
|
Classifier: Programming Language :: Python
|
14
14
|
Classifier: Programming Language :: Python :: 3
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
15
16
|
Classifier: Programming Language :: Python :: 3.12
|
16
17
|
Classifier: Topic :: Software Development :: Quality Assurance
|
17
18
|
Classifier: Topic :: Software Development :: Testing
|
@@ -23,7 +24,7 @@ Requires-Dist: colorlog (>=6.7.0,<7.0.0)
|
|
23
24
|
Requires-Dist: coverage (==7.4.3)
|
24
25
|
Requires-Dist: flake8 (==7.0.0)
|
25
26
|
Requires-Dist: multimethods-py (>=0.5.3,<0.6.0)
|
26
|
-
Requires-Dist: numpy (>=1.26.0,<
|
27
|
+
Requires-Dist: numpy (>=1.26.0,<1.27.0)
|
27
28
|
Requires-Dist: openpyxl (>=3.1.2,<4.0.0)
|
28
29
|
Requires-Dist: pandas (>=1.5.2,<2.0.0)
|
29
30
|
Requires-Dist: pyanzo (>=3.3.10,<4.0.0)
|
@@ -34,7 +35,7 @@ Requires-Dist: requests (>=2.28.2,<3.0.0)
|
|
34
35
|
Requires-Dist: tabulate (>=0.9.0,<0.10.0)
|
35
36
|
Requires-Dist: toml (>=0.10.2,<0.11.0)
|
36
37
|
Requires-Dist: tomli (>=2.0.1,<3.0.0)
|
37
|
-
Requires-Dist: urllib3 (==1.26.
|
38
|
+
Requires-Dist: urllib3 (==1.26.19)
|
38
39
|
Project-URL: Repository, https://github.com/Semantic-partners/mustrd
|
39
40
|
Description-Content-Type: text/plain
|
40
41
|
|
@@ -95,5 +96,9 @@ We invite you to try it, see where it doesn't fit, and raise an issue, or even b
|
|
95
96
|
We're a specialist consultancy in Semantic Tech, we're putting this out in case it's useful, but if you need more support, kindly contact our business team on info@semanticpartners.com
|
96
97
|
|
97
98
|
// tag::body[]
|
98
|
-
|
99
|
+
|
100
|
+
include::GETSTARTED.adoc[Get started]
|
101
|
+
|
102
|
+
Developer doc:
|
103
|
+
include::mustrd/README.adoc[tags=body]
|
99
104
|
|
@@ -1,5 +1,5 @@
|
|
1
|
-
mustrd/README.adoc,sha256=
|
2
|
-
mustrd/TestResult.py,sha256=
|
1
|
+
mustrd/README.adoc,sha256=E5KuShPEl2U6NmRzEAwZtAhBoyJsOvjGS1CM2UsbSQ4,1394
|
2
|
+
mustrd/TestResult.py,sha256=K4yth-rYrK6Pl7SFiTAZkUStBIzHlgq0oxSSpq5F34M,4849
|
3
3
|
mustrd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
4
4
|
mustrd/logger_setup.py,sha256=pNtG5egdd7OiFdn-Qx-tlf5JITka1yitbzYBpHzrP_4,1582
|
5
5
|
mustrd/model/catalog-v001.xml,sha256=IEtaw3FK4KGQWKancEe1HqzUQTrKdk89vNxnoLKGF4o,381
|
@@ -10,22 +10,22 @@ mustrd/model/ontology.ttl,sha256=AK27qyT8GW_SwJDwTFvtfxndBPEfYT6Y1jiW26Hyjg0,168
|
|
10
10
|
mustrd/model/test-resources/resources.ttl,sha256=1Dsp1nuNxauj9bxeX-HShQsiO-CVy5Irwm2y2x0cdjI,1498
|
11
11
|
mustrd/model/triplestoreOntology.ttl,sha256=jK9zJOInlJ8uXzbAsQarBBrZgGrMrMXbrsuXGMbp020,6025
|
12
12
|
mustrd/model/triplestoreshapes.ttl,sha256=VqdItpI9qFQSj4leZuwbIoAjWJehhR8bao0EPvtGMR0,1835
|
13
|
-
mustrd/mustrd.py,sha256=
|
14
|
-
mustrd/mustrdAnzo.py,sha256=
|
13
|
+
mustrd/mustrd.py,sha256=TFppEvQraOU2G1D2R1-MAatlY_qtYAosGl4oGQ3uLa8,33953
|
14
|
+
mustrd/mustrdAnzo.py,sha256=AfXKcU17_PQwoG1RfUVYgctcXMJPrfZdmEXzVi7E3HM,12155
|
15
15
|
mustrd/mustrdGraphDb.py,sha256=ySYYGvKUgXIS8QLLS7FEcppLlSb0qZ3lLlYzPf_Sr3Y,5474
|
16
16
|
mustrd/mustrdRdfLib.py,sha256=CvrzEl1-lEPugYVe3y_Ip8JMzUxv6IeWauLOa_WA-XI,2073
|
17
|
-
mustrd/mustrdTestPlugin.py,sha256=
|
18
|
-
mustrd/namespace.py,sha256=
|
19
|
-
mustrd/run.py,sha256=
|
20
|
-
mustrd/spec_component.py,sha256=
|
17
|
+
mustrd/mustrdTestPlugin.py,sha256=zQ6oqLsl4KbyTyCF27-SLiZdiP0o6-tMbGin-qBNvTE,14913
|
18
|
+
mustrd/namespace.py,sha256=xGfH-nRP3t_I4rmoGJPqlSGbI0G5HUMlUeAl7l_yW-s,3622
|
19
|
+
mustrd/run.py,sha256=5xZUgKPMBQ-03cWROAnwtbOs2Nb0Vat6n8Fi6EyfS-k,4257
|
20
|
+
mustrd/spec_component.py,sha256=7BmMjJMy0ACbVBAEZpe5ZprnDZYs6h5f7-fI38l3jAk,31483
|
21
21
|
mustrd/steprunner.py,sha256=YKrMSYhethWCAQEI63hK6jXh3ljyoM87w7eHyoRd8rc,7389
|
22
22
|
mustrd/templates/md_ResultList_leaf_template.jinja,sha256=IzwZjliCx7-viipATDQK6MQg_5q1kLMKdeNSZg1sXXY,508
|
23
23
|
mustrd/templates/md_ResultList_template.jinja,sha256=_8joJ7vtw_qoqxv3HhUtBgRfhOeqmgfaRFwEo4MROvQ,203
|
24
24
|
mustrd/templates/md_stats_template.jinja,sha256=96W62cMWu9UGLNv65ZQ8RYLjkxKHhJy-FlUtXgud6XY,155
|
25
|
-
mustrd/test/test_mustrd.py,sha256=
|
26
|
-
mustrd/utils.py,sha256=
|
27
|
-
mustrd-0.2.
|
28
|
-
mustrd-0.2.
|
29
|
-
mustrd-0.2.
|
30
|
-
mustrd-0.2.
|
31
|
-
mustrd-0.2.
|
25
|
+
mustrd/test/test_mustrd.py,sha256=ku4-UQ2railFo6z9apaIkPFOk980XSwCPBEHjhwWZUE,126
|
26
|
+
mustrd/utils.py,sha256=OGdLvw7GvjrFgTJo0J97Xwdh-_ZgSmapmOistrEchO0,1387
|
27
|
+
mustrd-0.2.2.dist-info/LICENSE,sha256=r8nmh5fUct9h2w8_RDl13EIscvmwCLoarPr1kg35MnA,1078
|
28
|
+
mustrd-0.2.2.dist-info/METADATA,sha256=PhkQpayIladpIEDKm-Q6JvGsoYUEPL1x7X1FSPBJB3w,4333
|
29
|
+
mustrd-0.2.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
30
|
+
mustrd-0.2.2.dist-info/entry_points.txt,sha256=v7V7sN0_L1aB4Ug_9io5axlQSeJ1C0tNrQWwdXdV58s,50
|
31
|
+
mustrd-0.2.2.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|