mustrd 0.2.0__py3-none-any.whl → 0.2.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mustrd/README.adoc +201 -210
- mustrd/execute_update_spec.py +18 -0
- mustrd/logger_setup.py +48 -48
- mustrd/mustrd.py +842 -787
- mustrd/mustrdAnzo.py +208 -220
- mustrd/mustrdGraphDb.py +128 -125
- mustrd/mustrdRdfLib.py +56 -56
- mustrd/namespace.py +104 -125
- mustrd/run.py +95 -106
- mustrd/spec_component.py +617 -690
- mustrd/triple_store_dispatch.py +115 -0
- mustrd/utils.py +30 -38
- {mustrd-0.2.0.dist-info → mustrd-0.2.0a1.dist-info}/LICENSE +21 -21
- mustrd-0.2.0a1.dist-info/METADATA +24 -0
- mustrd-0.2.0a1.dist-info/RECORD +17 -0
- {mustrd-0.2.0.dist-info → mustrd-0.2.0a1.dist-info}/WHEEL +1 -1
- mustrd/TestResult.py +0 -136
- mustrd/model/catalog-v001.xml +0 -5
- mustrd/model/mustrdShapes.ttl +0 -253
- mustrd/model/mustrdTestOntology.ttl +0 -51
- mustrd/model/mustrdTestShapes.ttl +0 -24
- mustrd/model/ontology.ttl +0 -494
- mustrd/model/test-resources/resources.ttl +0 -60
- mustrd/model/triplestoreOntology.ttl +0 -174
- mustrd/model/triplestoreshapes.ttl +0 -42
- mustrd/mustrdQueryProcessor.py +0 -136
- mustrd/mustrdTestPlugin.py +0 -328
- mustrd/steprunner.py +0 -166
- mustrd/templates/md_ResultList_leaf_template.jinja +0 -19
- mustrd/templates/md_ResultList_template.jinja +0 -9
- mustrd/templates/md_stats_template.jinja +0 -3
- mustrd/test/test_mustrd.py +0 -5
- mustrd-0.2.0.dist-info/METADATA +0 -97
- mustrd-0.2.0.dist-info/RECORD +0 -32
- mustrd-0.2.0.dist-info/entry_points.txt +0 -3
@@ -1,174 +0,0 @@
|
|
1
|
-
@base <https://mustrd.com/triplestore/> .
|
2
|
-
@prefix : <https://mustrd.com/triplestore/> .
|
3
|
-
@prefix dc: <http://purl.org/dc/elements/1.1/> .
|
4
|
-
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
5
|
-
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
6
|
-
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
7
|
-
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
|
8
|
-
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
9
|
-
@prefix foaf: <http://xmlns.com/foaf/spec/> .
|
10
|
-
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
11
|
-
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
|
12
|
-
#
|
13
|
-
#
|
14
|
-
# #################################################################
|
15
|
-
# #
|
16
|
-
# # Data properties
|
17
|
-
# #
|
18
|
-
# #################################################################
|
19
|
-
#
|
20
|
-
#
|
21
|
-
# https://mustrd.com/triplestore/gqeURI
|
22
|
-
#
|
23
|
-
# https://mustrd.com/triplestore/inputGraph
|
24
|
-
#
|
25
|
-
# https://mustrd.com/triplestore/outputGraph
|
26
|
-
#
|
27
|
-
# https://mustrd.com/triplestore/password
|
28
|
-
#
|
29
|
-
# https://mustrd.com/triplestore/port
|
30
|
-
#
|
31
|
-
# https://mustrd.com/triplestore/repository
|
32
|
-
#
|
33
|
-
# https://mustrd.com/triplestore/url
|
34
|
-
#
|
35
|
-
# https://mustrd.com/triplestore/username
|
36
|
-
#
|
37
|
-
#
|
38
|
-
#
|
39
|
-
# #################################################################
|
40
|
-
# #
|
41
|
-
# # Classes
|
42
|
-
# #
|
43
|
-
# #################################################################
|
44
|
-
#
|
45
|
-
#
|
46
|
-
# https://mustrd.com/triplestore/Anzo
|
47
|
-
#
|
48
|
-
# https://mustrd.com/triplestore/ExternalTripleStore
|
49
|
-
#
|
50
|
-
# https://mustrd.com/triplestore/GraphDb
|
51
|
-
#
|
52
|
-
# https://mustrd.com/triplestore/InternalTripleStore
|
53
|
-
#
|
54
|
-
# https://mustrd.com/triplestore/RdfLib
|
55
|
-
#
|
56
|
-
# https://mustrd.com/triplestore/TripleStore
|
57
|
-
#
|
58
|
-
# Generated by the OWL API (version 4.5.26.2023-07-17T20:34:13Z) https://github.com/owlcs/owlapi
|
59
|
-
|
60
|
-
<> a owl:Ontology;
|
61
|
-
rdfs:label "triple store ontology" .
|
62
|
-
|
63
|
-
:gqeURI a owl:DatatypeProperty;
|
64
|
-
rdfs:domain :Anzo;
|
65
|
-
rdfs:range xsd:string;
|
66
|
-
rdfs:label "gqeURI" .
|
67
|
-
|
68
|
-
:inputGraph a owl:DatatypeProperty;
|
69
|
-
rdfs:domain :TripleStore;
|
70
|
-
rdfs:range xsd:string;
|
71
|
-
rdfs:label "inputGraph" .
|
72
|
-
|
73
|
-
:outputGraph a owl:DatatypeProperty;
|
74
|
-
rdfs:domain :TripleStore;
|
75
|
-
rdfs:range xsd:string;
|
76
|
-
rdfs:comment """Uri of the graph where to insert/delete the triples
|
77
|
-
|
78
|
-
In some triple stores there is a default insert graph like in graphDB:
|
79
|
-
In graphDB the default insert graph is called: <http://www.openrdf.org/schema/sesame#nil>
|
80
|
-
or
|
81
|
-
<http://rdf4j.org/schema/rdf4j#nil>
|
82
|
-
|
83
|
-
Other triple stores do not define default insert graph like anzograph.
|
84
|
-
For those triple stores, this property must be mandatory""";
|
85
|
-
rdfs:label "outputGraph" .
|
86
|
-
|
87
|
-
:password a owl:DatatypeProperty;
|
88
|
-
rdfs:domain :ExternalTripleStore;
|
89
|
-
rdfs:range xsd:string;
|
90
|
-
rdfs:label "password" .
|
91
|
-
|
92
|
-
:port a owl:DatatypeProperty;
|
93
|
-
rdfs:domain :ExternalTripleStore;
|
94
|
-
rdfs:range xsd:string;
|
95
|
-
rdfs:comment "Triple store port";
|
96
|
-
rdfs:label "port" .
|
97
|
-
|
98
|
-
:repository a owl:DatatypeProperty;
|
99
|
-
rdfs:domain :GraphDb;
|
100
|
-
rdfs:range xsd:string;
|
101
|
-
rdfs:label "repository" .
|
102
|
-
|
103
|
-
:url a owl:DatatypeProperty;
|
104
|
-
rdfs:domain :ExternalTripleStore;
|
105
|
-
rdfs:range xsd:string;
|
106
|
-
rdfs:comment "triple store URL";
|
107
|
-
rdfs:label "url" .
|
108
|
-
|
109
|
-
:username a owl:DatatypeProperty;
|
110
|
-
rdfs:domain :ExternalTripleStore;
|
111
|
-
rdfs:range xsd:string;
|
112
|
-
rdfs:label "username" .
|
113
|
-
|
114
|
-
:Anzo a owl:Class;
|
115
|
-
rdfs:subClassOf :ExternalTripleStore;
|
116
|
-
rdfs:comment """TODO: model this:
|
117
|
-
|
118
|
-
definition of RDF dataset:
|
119
|
-
|
120
|
-
default graph=
|
121
|
-
RDF merge of default-graph-uri if exists
|
122
|
-
else RDF merge of FROM if exists
|
123
|
-
else all graphs in anzo graph (if you query a graphmart, default-graph-uri will be automatically set to the layers of the graphmart), if you are not sysadmin and you do not define the default graph, then the query will fail since you don't have permissions on all graphs
|
124
|
-
if default-graph-uri is defined then FROM clauses are ignored
|
125
|
-
|
126
|
-
default named graph =
|
127
|
-
RDF merge of named-graph-uri if exists
|
128
|
-
else RDF merge of FROM NAMED if exists
|
129
|
-
else default graph if exists
|
130
|
-
all graphs in anzo graph (if you query a graphmart, default-graph-uri will be automatically set to the layers of the graphmart), if you are not sysadmin and you do not define the default graph, then the query will fail since you don't have permissions on all graphs
|
131
|
-
if named-graph-uri is defined then FROM NAMED clauses are ignored
|
132
|
-
|
133
|
-
There is no default insert graph. If you try to insert/delete without graph clause, your query will fail.""";
|
134
|
-
rdfs:label "Anzo" .
|
135
|
-
|
136
|
-
:ExternalTripleStore a owl:Class;
|
137
|
-
rdfs:subClassOf :TripleStore .
|
138
|
-
|
139
|
-
:GraphDb a owl:Class;
|
140
|
-
rdfs:subClassOf :ExternalTripleStore;
|
141
|
-
rdfs:comment """TODO: model this:
|
142
|
-
|
143
|
-
Definition of RDF dataset in graphDB:
|
144
|
-
|
145
|
-
default graph (in the sense of W3C, called default dataset in graphDB) =
|
146
|
-
Virtual graph that exist only for a query and represent all triples accessible outside of a graph clause in a sparql query =
|
147
|
-
RDF merge of default-graph-uri if exists
|
148
|
-
else RDF merge of FROM if exists
|
149
|
-
else all graphs in the repository, including default insert graph
|
150
|
-
you can query default insert graph by including it (<http://www.openrdf.org/schema/sesame#nil> or <http://rdf4j.org/schema/rdf4j#nil>) in a default-graph-uri parameter or a FROM clause
|
151
|
-
If default-graph-uri is defined, then FROM clause is ignored
|
152
|
-
|
153
|
-
default named graph =
|
154
|
-
All triples accessible inside a graph clause:
|
155
|
-
RDF merge of named-graph-uri if exists
|
156
|
-
else RDF merge of FROM NAMED if exist
|
157
|
-
else all graphs in the repository, EXCLUDING default insert graph
|
158
|
-
(To be exact you can query the default insert graph only if you name it GRAPH <http://www.openrdf.org/schema/sesame#nil> or GRAPH <http://rdf4j.org/schema/rdf4j#nil>, but you won't query it with GRAPH ?graph)
|
159
|
-
default insert graph can be added to named-graph-uri or FROM NAMED
|
160
|
-
If a default-graph-uri is defined but no named-graph-uri, then default named graph = void
|
161
|
-
|
162
|
-
default insert graph (called \"The default graph\" in graphDB) =
|
163
|
-
Graph where triples are inserted / deleted when no graph clause is given in an INSERT or DELETE clause""";
|
164
|
-
rdfs:label "GraphDb" .
|
165
|
-
|
166
|
-
:InternalTripleStore a owl:Class;
|
167
|
-
rdfs:subClassOf :TripleStore .
|
168
|
-
|
169
|
-
:RdfLib a owl:Class;
|
170
|
-
rdfs:subClassOf :InternalTripleStore;
|
171
|
-
rdfs:label "RdfLib" .
|
172
|
-
|
173
|
-
:TripleStore a owl:Class;
|
174
|
-
rdfs:label "TripleStore" .
|
@@ -1,42 +0,0 @@
|
|
1
|
-
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
2
|
-
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
3
|
-
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
4
|
-
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
5
|
-
@prefix ex: <http://www.example.org/#> .
|
6
|
-
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
7
|
-
@prefix triplestore: <https://mustrd.com/triplestore/> .
|
8
|
-
|
9
|
-
triplestore:ExternalTripleStoreShape
|
10
|
-
a sh:NodeShape ;
|
11
|
-
sh:targetClass triplestore:ExternalTripleStore ;
|
12
|
-
sh:property [ sh:path triplestore:url ;
|
13
|
-
sh:minCount 1 ;
|
14
|
-
sh:maxCount 1 ],
|
15
|
-
[ sh:path triplestore:port ;
|
16
|
-
sh:minCount 1 ;
|
17
|
-
sh:maxCount 1 ],
|
18
|
-
[ sh:path triplestore:username ;
|
19
|
-
sh:maxCount 1 ],
|
20
|
-
[ sh:path triplestore:password ;
|
21
|
-
sh:maxCount 1 ] .
|
22
|
-
|
23
|
-
triplestore:AnzoShape
|
24
|
-
a sh:NodeShape ;
|
25
|
-
sh:targetClass triplestore:Anzo ;
|
26
|
-
sh:property [ sh:path triplestore:gqeURI ;
|
27
|
-
sh:minCount 1 ;
|
28
|
-
sh:maxCount 1 ],
|
29
|
-
[ sh:path triplestore:outputGraph ;
|
30
|
-
sh:minCount 1 ;
|
31
|
-
sh:maxCount 1 ],
|
32
|
-
# For anzo the input graph is not really necessary if the user is sysadmin
|
33
|
-
# but querying all graphs in AZG is usually not a good idea, so for the moment this is forbidden
|
34
|
-
[ sh:path triplestore:inputGraph ;
|
35
|
-
sh:minCount 1 ] .
|
36
|
-
|
37
|
-
triplestore:GraphDbShape
|
38
|
-
a sh:NodeShape ;
|
39
|
-
sh:targetClass triplestore:GraphDb ;
|
40
|
-
sh:property [ sh:path triplestore:repository ;
|
41
|
-
sh:minCount 1 ;
|
42
|
-
sh:maxCount 1 ] .
|
mustrd/mustrdQueryProcessor.py
DELETED
@@ -1,136 +0,0 @@
|
|
1
|
-
|
2
|
-
from pyparsing import ParseResults
|
3
|
-
from rdflib import RDF, Graph, URIRef, Variable, Literal, XSD, util, BNode
|
4
|
-
from rdflib.plugins.sparql.parser import parseQuery, parseUpdate
|
5
|
-
from rdflib.plugins.sparql.algebra import translateQuery, translateUpdate, translateAlgebra
|
6
|
-
from rdflib.plugins.sparql.sparql import Query
|
7
|
-
from rdflib.plugins.sparql.parserutils import CompValue, value, Expr
|
8
|
-
from rdflib.namespace import DefinedNamespace, Namespace
|
9
|
-
from rdflib.term import Identifier
|
10
|
-
from typing import Union
|
11
|
-
|
12
|
-
from builtins import list, set, tuple, str
|
13
|
-
|
14
|
-
|
15
|
-
namespace = "https://mustrd.com/query/"
|
16
|
-
|
17
|
-
class MustrdQueryProcessor:
|
18
|
-
original_query : Union [Query, ParseResults]
|
19
|
-
current_query : Union [Query, ParseResults]
|
20
|
-
graph : Graph
|
21
|
-
algebra_mode: bool = False
|
22
|
-
graph_mode: bool = True
|
23
|
-
|
24
|
-
def __init__(self, query_str: str, algebra_mode: bool = False, graph_mode: bool = True):
|
25
|
-
parsetree = parseQuery(query_str)
|
26
|
-
# Init original query to algebra or parsed query
|
27
|
-
self.original_query = (algebra_mode and translateQuery(parsetree)) or parsetree
|
28
|
-
self.current_query = self.original_query
|
29
|
-
self.algebra_mode = algebra_mode
|
30
|
-
self.graph_mode = graph_mode
|
31
|
-
self.graph = Graph()
|
32
|
-
if graph_mode:
|
33
|
-
self.query_to_graph((algebra_mode and self.original_query.algebra) or parsetree._toklist, BNode())
|
34
|
-
|
35
|
-
def query_to_graph(self, part: CompValue, partBnode):
|
36
|
-
if not part or not partBnode:
|
37
|
-
return
|
38
|
-
self.graph.add((partBnode, RDF.type, URIRef(namespace + type(part).__name__)))
|
39
|
-
self.graph.add((partBnode, QUERY.has_class , Literal(str(part.__class__.__name__))))
|
40
|
-
if isinstance(part, CompValue) or isinstance(part, ParseResults):
|
41
|
-
self.graph.add((partBnode, QUERY.name , Literal(part.name)))
|
42
|
-
if isinstance(part, CompValue):
|
43
|
-
for key, sub_part in part.items():
|
44
|
-
sub_part_bnode = BNode()
|
45
|
-
self.graph.add((partBnode, URIRef(namespace + str(key)) , sub_part_bnode))
|
46
|
-
self.query_to_graph(sub_part, sub_part_bnode)
|
47
|
-
elif hasattr(part, '__iter__') and not isinstance(part, Identifier) and not isinstance(part, str):
|
48
|
-
for sub_part in part:
|
49
|
-
sub_part_bnode = BNode()
|
50
|
-
self.graph.add((partBnode, QUERY.has_list , sub_part_bnode))
|
51
|
-
self.query_to_graph(sub_part, sub_part_bnode)
|
52
|
-
elif isinstance(part, Identifier) or isinstance(part, str):
|
53
|
-
self.graph.add((partBnode, QUERY.has_value, Literal(part)))
|
54
|
-
|
55
|
-
def serialize_graph(self):
|
56
|
-
if not self.graph_mode:
|
57
|
-
raise Exception("Not able to execute that function if graph mode is not activated: cannot work with two sources of truth")
|
58
|
-
return self.graph.serialize(format = "ttl")
|
59
|
-
|
60
|
-
def query_graph(self, meta_query: str):
|
61
|
-
if not self.graph_mode:
|
62
|
-
raise Exception("Not able to execute that function if graph mode is not activated: cannot work with two sources of truth")
|
63
|
-
return self.graph.query(meta_query)
|
64
|
-
|
65
|
-
def update(self, meta_query: str):
|
66
|
-
if not self.graph_mode:
|
67
|
-
# Implement update directly on objects: self.current_query
|
68
|
-
pass
|
69
|
-
return self.graph.update(meta_query)
|
70
|
-
|
71
|
-
def get_query(self):
|
72
|
-
if self.graph_mode:
|
73
|
-
roots = self.graph.query("SELECT DISTINCT ?sub WHERE {?sub ?prop ?obj FILTER NOT EXISTS {?s ?p ?sub}}")
|
74
|
-
if len(roots) != 1:
|
75
|
-
raise Exception("query graph has more than one root: invalid")
|
76
|
-
|
77
|
-
for root in roots:
|
78
|
-
new_query = self.graph_to_query(root.sub)
|
79
|
-
if not self.algebra_mode:
|
80
|
-
new_query = ParseResults(toklist=new_query, name=self.original_query.name)
|
81
|
-
new_query = translateQuery(new_query)
|
82
|
-
else:
|
83
|
-
new_query = Query(algebra=new_query, prologue=self.original_query.prologue)
|
84
|
-
else:
|
85
|
-
if not self.algebra_mode:
|
86
|
-
new_query = translateQuery(self.current_query)
|
87
|
-
else:
|
88
|
-
new_query = self.current_query
|
89
|
-
return translateAlgebra(new_query)
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
def graph_to_query(self, subject):
|
94
|
-
subject_dict = self.get_subject_dict(subject)
|
95
|
-
if QUERY.has_class in subject_dict:
|
96
|
-
class_name = str(subject_dict[QUERY.has_class])
|
97
|
-
subject_dict.pop(QUERY.has_class)
|
98
|
-
if class_name in globals():
|
99
|
-
clazz = globals()[class_name]
|
100
|
-
if clazz in (CompValue, Expr):
|
101
|
-
comp_name = str(subject_dict[QUERY.name])
|
102
|
-
subject_dict.pop(QUERY.name)
|
103
|
-
subject_dict.pop(RDF.type)
|
104
|
-
new_dict = dict(map(lambda kv: [str(kv[0]).replace(str(QUERY._NS), ""),
|
105
|
-
self.graph_to_query(kv[1])] ,
|
106
|
-
subject_dict.items()))
|
107
|
-
return clazz(comp_name, **new_dict)
|
108
|
-
elif clazz in (set, list, tuple) and QUERY.has_list in subject_dict:
|
109
|
-
return clazz(map(lambda item: self.graph_to_query(item), subject_dict[QUERY.has_list]))
|
110
|
-
elif clazz == ParseResults and QUERY.has_list in subject_dict:
|
111
|
-
return ParseResults(toklist=list(map(lambda item: self.graph_to_query(item), subject_dict[QUERY.has_list])))
|
112
|
-
elif clazz in (Literal, Variable, URIRef, str) and QUERY.has_value in subject_dict:
|
113
|
-
return clazz(str(subject_dict[QUERY.has_value]))
|
114
|
-
|
115
|
-
|
116
|
-
def get_subject_dict(self, subject):
|
117
|
-
dict = {}
|
118
|
-
for key, value in self.graph.predicate_objects(subject):
|
119
|
-
# If key already exists: create or add to a list
|
120
|
-
if key == QUERY.has_list:
|
121
|
-
if key in dict:
|
122
|
-
dict[key].append(value)
|
123
|
-
else:
|
124
|
-
dict[key] = [value]
|
125
|
-
else:
|
126
|
-
dict[key] = value
|
127
|
-
return dict
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
class QUERY(DefinedNamespace):
|
132
|
-
_NS = Namespace("https://mustrd.com/query/")
|
133
|
-
has_class : URIRef
|
134
|
-
has_list : URIRef
|
135
|
-
name : URIRef
|
136
|
-
has_value: URIRef
|
mustrd/mustrdTestPlugin.py
DELETED
@@ -1,328 +0,0 @@
|
|
1
|
-
"""
|
2
|
-
MIT License
|
3
|
-
|
4
|
-
Copyright (c) 2023 Semantic Partners Ltd
|
5
|
-
|
6
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
7
|
-
of this software and associated documentation files (the "Software"), to deal
|
8
|
-
in the Software without restriction, including without limitation the rights
|
9
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
10
|
-
copies of the Software, and to permit persons to whom the Software is
|
11
|
-
furnished to do so, subject to the following conditions:
|
12
|
-
|
13
|
-
The above copyright notice and this permission notice shall be included in all
|
14
|
-
copies or substantial portions of the Software.
|
15
|
-
|
16
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
17
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
18
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
19
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
20
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
21
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
22
|
-
SOFTWARE.
|
23
|
-
"""
|
24
|
-
|
25
|
-
from dataclasses import dataclass
|
26
|
-
import pytest
|
27
|
-
import os
|
28
|
-
from pathlib import Path
|
29
|
-
from rdflib.namespace import Namespace
|
30
|
-
from rdflib import Graph, RDF
|
31
|
-
from pytest import Session
|
32
|
-
|
33
|
-
from mustrd.TestResult import ResultList, TestResult, get_result_list
|
34
|
-
from mustrd.utils import get_mustrd_root
|
35
|
-
from mustrd.mustrd import get_triple_store_graph, get_triple_stores
|
36
|
-
from mustrd.mustrd import Specification, SpecSkipped, validate_specs, get_specs, SpecPassed, run_spec
|
37
|
-
from mustrd.namespace import MUST, TRIPLESTORE, MUSTRDTEST
|
38
|
-
from typing import Union
|
39
|
-
from pyshacl import validate
|
40
|
-
|
41
|
-
spnamespace = Namespace("https://semanticpartners.com/data/test/")
|
42
|
-
|
43
|
-
mustrd_root = get_mustrd_root()
|
44
|
-
|
45
|
-
MUSTRD_PYTEST_PATH = "mustrd_tests/"
|
46
|
-
|
47
|
-
|
48
|
-
def pytest_addoption(parser):
|
49
|
-
group = parser.getgroup("mustrd option")
|
50
|
-
group.addoption(
|
51
|
-
"--mustrd",
|
52
|
-
action="store_true",
|
53
|
-
dest="mustrd",
|
54
|
-
help="Activate/deactivate mustrd test generation.",
|
55
|
-
)
|
56
|
-
group.addoption(
|
57
|
-
"--md",
|
58
|
-
action="store",
|
59
|
-
dest="mdpath",
|
60
|
-
metavar="pathToMdSummary",
|
61
|
-
default=None,
|
62
|
-
help="create md summary file at that path.",
|
63
|
-
)
|
64
|
-
group.addoption(
|
65
|
-
"--config",
|
66
|
-
action="store",
|
67
|
-
dest="configpath",
|
68
|
-
metavar="pathToTestConfig",
|
69
|
-
default=None,
|
70
|
-
help="Ttl file containing the list of test to construct.",
|
71
|
-
)
|
72
|
-
group.addoption(
|
73
|
-
"--secrets",
|
74
|
-
action="store",
|
75
|
-
dest="secrets",
|
76
|
-
metavar="Secrets",
|
77
|
-
default=None,
|
78
|
-
help="Give the secrets by command line in order to be able to store secrets safely in CI tools",
|
79
|
-
)
|
80
|
-
return
|
81
|
-
|
82
|
-
|
83
|
-
def pytest_configure(config) -> None:
|
84
|
-
# Read configuration file
|
85
|
-
if config.getoption("mustrd"):
|
86
|
-
test_configs = parse_config(config.getoption("configpath"))
|
87
|
-
config.pluginmanager.register(MustrdTestPlugin(config.getoption("mdpath"),
|
88
|
-
test_configs, config.getoption("secrets")))
|
89
|
-
|
90
|
-
def parse_config(config_path):
|
91
|
-
test_configs = []
|
92
|
-
config_graph = Graph().parse(config_path)
|
93
|
-
shacl_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdTestShapes.ttl")))
|
94
|
-
ont_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdTestOntology.ttl")))
|
95
|
-
conforms, results_graph, results_text = validate(
|
96
|
-
data_graph= config_graph,
|
97
|
-
shacl_graph = shacl_graph,
|
98
|
-
ont_graph = ont_graph,
|
99
|
-
advanced= True,
|
100
|
-
inference= 'none'
|
101
|
-
)
|
102
|
-
if not conforms:
|
103
|
-
raise ValueError(f"Mustrd test configuration not conform to the shapes. SHACL report: {results_text}", results_graph)
|
104
|
-
|
105
|
-
for test_config_subject in config_graph.subjects(predicate=RDF.type, object=MUSTRDTEST.MustrdTest):
|
106
|
-
spec_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasSpecPath, str)
|
107
|
-
data_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasDataPath, str)
|
108
|
-
triplestore_spec_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.triplestoreSpecPath, str)
|
109
|
-
pytest_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasPytestPath, str)
|
110
|
-
filter_on_tripleStore = list(config_graph.objects(subject=test_config_subject,
|
111
|
-
predicate=MUSTRDTEST.filterOnTripleStore))
|
112
|
-
|
113
|
-
test_configs.append(TestConfig(spec_path=spec_path, data_path=data_path,
|
114
|
-
triplestore_spec_path=triplestore_spec_path,
|
115
|
-
pytest_path = pytest_path,
|
116
|
-
filter_on_tripleStore=filter_on_tripleStore))
|
117
|
-
return test_configs
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
def get_config_param(config_graph, config_subject, config_param, convert_function):
|
122
|
-
raw_value = config_graph.value(subject=config_subject, predicate=config_param, any=True)
|
123
|
-
return convert_function(raw_value) if raw_value else None
|
124
|
-
|
125
|
-
|
126
|
-
@dataclass
|
127
|
-
class TestConfig:
|
128
|
-
spec_path: str
|
129
|
-
data_path: str
|
130
|
-
triplestore_spec_path: str
|
131
|
-
pytest_path: str
|
132
|
-
filter_on_tripleStore: str = None
|
133
|
-
|
134
|
-
|
135
|
-
@dataclass
|
136
|
-
class TestParamWrapper:
|
137
|
-
test_config: TestConfig
|
138
|
-
unit_test: Union[Specification, SpecSkipped]
|
139
|
-
|
140
|
-
class MustrdTestPlugin:
|
141
|
-
md_path: str
|
142
|
-
test_configs: list
|
143
|
-
secrets: str
|
144
|
-
unit_tests: Union[Specification, SpecSkipped]
|
145
|
-
items: list
|
146
|
-
|
147
|
-
def __init__(self, md_path, test_configs, secrets):
|
148
|
-
self.md_path = md_path
|
149
|
-
self.test_configs = test_configs
|
150
|
-
self.secrets = secrets
|
151
|
-
self.items = []
|
152
|
-
|
153
|
-
@pytest.hookimpl(tryfirst=True)
|
154
|
-
def pytest_collection(self, session):
|
155
|
-
self.unit_tests = []
|
156
|
-
args = session.config.args
|
157
|
-
if len(args) > 0:
|
158
|
-
file_name = self.get_file_name_from_arg(args[0])
|
159
|
-
# Filter test to collect only specified path
|
160
|
-
config_to_collect = list(filter(lambda config:
|
161
|
-
# Case we want to collect everything
|
162
|
-
MUSTRD_PYTEST_PATH not in args[0]
|
163
|
-
# Case we want to collect a test or sub test
|
164
|
-
or (config.pytest_path or "") in args[0]
|
165
|
-
# Case we want to collect a whole test folder
|
166
|
-
or args[0].replace(f"./{MUSTRD_PYTEST_PATH}", "") in config.pytest_path,
|
167
|
-
self.test_configs))
|
168
|
-
|
169
|
-
# Redirect everything to test_mustrd.py, no need to filter on specified test: Only specified test will be collected anyway
|
170
|
-
session.config.args[0] = os.path.join(mustrd_root, "test/test_mustrd.py")
|
171
|
-
# Collecting only relevant tests
|
172
|
-
|
173
|
-
for one_test_config in config_to_collect:
|
174
|
-
triple_stores = self.get_triple_stores_from_file(one_test_config)
|
175
|
-
|
176
|
-
if one_test_config.filter_on_tripleStore and not triple_stores:
|
177
|
-
self.unit_tests.extend(list(map(lambda triple_store:
|
178
|
-
TestParamWrapper(test_config = one_test_config, unit_test=SpecSkipped(MUST.TestSpec, triple_store, "No triplestore found")),
|
179
|
-
one_test_config.filter_on_tripleStore)))
|
180
|
-
else:
|
181
|
-
specs = self.generate_tests_for_config({"spec_path": Path(one_test_config.spec_path),
|
182
|
-
"data_path": Path(one_test_config.data_path)},
|
183
|
-
triple_stores, file_name)
|
184
|
-
self.unit_tests.extend(list(map(lambda spec: TestParamWrapper(test_config = one_test_config, unit_test=spec),specs)))
|
185
|
-
|
186
|
-
def get_file_name_from_arg(self, arg):
|
187
|
-
if arg and len(arg) > 0 and "[" in arg and ".mustrd.ttl@" in arg:
|
188
|
-
return arg[arg.index("[") + 1: arg.index(".mustrd.ttl@")]
|
189
|
-
return None
|
190
|
-
|
191
|
-
|
192
|
-
@pytest.hookimpl(hookwrapper=True)
|
193
|
-
def pytest_pycollect_makeitem(self, collector, name, obj):
|
194
|
-
report = yield
|
195
|
-
if name == "test_unit":
|
196
|
-
items = report.get_result()
|
197
|
-
new_results = []
|
198
|
-
for item in items:
|
199
|
-
virtual_path = MUSTRD_PYTEST_PATH + (item.callspec.params["unit_tests"].test_config.pytest_path or "default")
|
200
|
-
item.fspath = Path(virtual_path)
|
201
|
-
item._nodeid = virtual_path + "::" + item.name
|
202
|
-
self.items.append(item)
|
203
|
-
new_results.append(item)
|
204
|
-
return new_results
|
205
|
-
|
206
|
-
|
207
|
-
# Hook called at collection time: reads the configuration of the tests, and generate pytests from it
|
208
|
-
def pytest_generate_tests(self, metafunc):
|
209
|
-
if len(metafunc.fixturenames) > 0:
|
210
|
-
if metafunc.function.__name__ == "test_unit":
|
211
|
-
# Create the test in itself
|
212
|
-
if self.unit_tests:
|
213
|
-
metafunc.parametrize(metafunc.fixturenames[0], self.unit_tests,
|
214
|
-
ids=lambda test_param: (test_param.unit_test.spec_file_name or "") + "@" +
|
215
|
-
(test_param.test_config.pytest_path or ""))
|
216
|
-
else:
|
217
|
-
metafunc.parametrize(metafunc.fixturenames[0],
|
218
|
-
[SpecSkipped(MUST.TestSpec, None, "No triplestore found")],
|
219
|
-
ids=lambda x: "No configuration found for this test")
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
# Generate test for each triple store available
|
225
|
-
def generate_tests_for_config(self, config, triple_stores, file_name):
|
226
|
-
|
227
|
-
shacl_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdShapes.ttl")))
|
228
|
-
ont_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/ontology.ttl")))
|
229
|
-
valid_spec_uris, spec_graph, invalid_spec_results = validate_specs(config, triple_stores,
|
230
|
-
shacl_graph, ont_graph, file_name or "*")
|
231
|
-
|
232
|
-
specs, skipped_spec_results = \
|
233
|
-
get_specs(valid_spec_uris, spec_graph, triple_stores, config)
|
234
|
-
|
235
|
-
# Return normal specs + skipped results
|
236
|
-
return specs + skipped_spec_results + invalid_spec_results
|
237
|
-
|
238
|
-
# Function called to generate the name of the test
|
239
|
-
def get_test_name(self, spec):
|
240
|
-
# FIXME: SpecSkipped should have the same structure?
|
241
|
-
if isinstance(spec, SpecSkipped):
|
242
|
-
triple_store = spec.triple_store
|
243
|
-
else:
|
244
|
-
triple_store = spec.triple_store['type']
|
245
|
-
triple_store_name = triple_store.replace("https://mustrd.com/model/", "")
|
246
|
-
test_name = spec.spec_uri.replace(spnamespace, "").replace("_", " ")
|
247
|
-
return triple_store_name + ": " + test_name
|
248
|
-
|
249
|
-
# Get triple store configuration or default
|
250
|
-
def get_triple_stores_from_file(self, test_config):
|
251
|
-
if test_config.triplestore_spec_path:
|
252
|
-
try:
|
253
|
-
triple_stores = get_triple_stores(get_triple_store_graph(Path(test_config.triplestore_spec_path),
|
254
|
-
self.secrets))
|
255
|
-
except Exception as e:
|
256
|
-
print(f"""Triplestore configuration parsing failed {test_config.triplestore_spec_path}.
|
257
|
-
Only rdflib will be executed""", e)
|
258
|
-
triple_stores = [{'type': TRIPLESTORE.RdfLib, 'uri': TRIPLESTORE.RdfLib}]
|
259
|
-
else:
|
260
|
-
print("No triple store configuration required: using embedded rdflib")
|
261
|
-
triple_stores = [{'type': TRIPLESTORE.RdfLib, 'uri': TRIPLESTORE.RdfLib}]
|
262
|
-
|
263
|
-
if test_config.filter_on_tripleStore:
|
264
|
-
triple_stores = list(filter(lambda triple_store: (triple_store["uri"] in test_config.filter_on_tripleStore),
|
265
|
-
triple_stores))
|
266
|
-
return triple_stores
|
267
|
-
|
268
|
-
# Hook function. Initialize the list of result in session
|
269
|
-
def pytest_sessionstart(self, session):
|
270
|
-
session.results = dict()
|
271
|
-
|
272
|
-
# Hook function called each time a report is generated by a test
|
273
|
-
# The report is added to a list in the session
|
274
|
-
# so it can be used later in pytest_sessionfinish to generate the global report md file
|
275
|
-
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
|
276
|
-
def pytest_runtest_makereport(self, item, call):
|
277
|
-
outcome = yield
|
278
|
-
result = outcome.get_result()
|
279
|
-
|
280
|
-
if result.when == 'call':
|
281
|
-
# Add the result of the test to the session
|
282
|
-
item.session.results[item] = result
|
283
|
-
|
284
|
-
# Take all the test results in session, parse them, split them in mustrd and standard pytest and generate md file
|
285
|
-
def pytest_sessionfinish(self, session: Session, exitstatus):
|
286
|
-
# if md path has not been defined in argument, then do not generate md file
|
287
|
-
if not self.md_path:
|
288
|
-
return
|
289
|
-
|
290
|
-
test_results = []
|
291
|
-
for test_conf, result in session.results.items():
|
292
|
-
# Case auto generated tests
|
293
|
-
if test_conf.originalname != test_conf.name:
|
294
|
-
module_name = test_conf.parent.name
|
295
|
-
class_name = test_conf.originalname
|
296
|
-
test_name = test_conf.name.replace(class_name, "").replace("[", "").replace("]", "")
|
297
|
-
is_mustrd = True
|
298
|
-
# Case normal unit tests
|
299
|
-
else:
|
300
|
-
module_name = test_conf.parent.parent.name
|
301
|
-
class_name = test_conf.parent.name
|
302
|
-
test_name = test_conf.originalname
|
303
|
-
is_mustrd = False
|
304
|
-
|
305
|
-
test_results.append(TestResult(test_name, class_name, module_name, result.outcome, is_mustrd))
|
306
|
-
|
307
|
-
result_list = ResultList(None, get_result_list(test_results,
|
308
|
-
lambda result: result.type,
|
309
|
-
lambda result: result.module_name,
|
310
|
-
lambda result: result.class_name),
|
311
|
-
False)
|
312
|
-
|
313
|
-
md = result_list.render()
|
314
|
-
with open(self.md_path, 'w') as file:
|
315
|
-
file.write(md)
|
316
|
-
|
317
|
-
|
318
|
-
# Function called in the test to actually run it
|
319
|
-
def run_test_spec(test_spec):
|
320
|
-
if isinstance(test_spec, SpecSkipped):
|
321
|
-
pytest.skip(f"Invalid configuration, error : {test_spec.message}")
|
322
|
-
result = run_spec(test_spec)
|
323
|
-
|
324
|
-
result_type = type(result)
|
325
|
-
if result_type == SpecSkipped:
|
326
|
-
# FIXME: Better exception management
|
327
|
-
pytest.skip("Unsupported configuration")
|
328
|
-
return result_type == SpecPassed
|