mustrd 0.1.7__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mustrd-0.1.7 → mustrd-0.2.0}/PKG-INFO +1 -1
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/README.adoc +15 -6
- mustrd-0.2.0/mustrd/model/catalog-v001.xml +5 -0
- mustrd-0.2.0/mustrd/model/mustrdTestOntology.ttl +51 -0
- mustrd-0.2.0/mustrd/model/mustrdTestShapes.ttl +24 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/model/ontology.ttl +2 -1
- mustrd-0.2.0/mustrd/model/triplestoreOntology.ttl +174 -0
- mustrd-0.2.0/mustrd/model/triplestoreshapes.ttl +42 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/mustrd.py +48 -39
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/mustrdAnzo.py +61 -77
- mustrd-0.2.0/mustrd/mustrdQueryProcessor.py +136 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/mustrdTestPlugin.py +125 -58
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/namespace.py +36 -29
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/run.py +2 -2
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/spec_component.py +60 -52
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/steprunner.py +15 -15
- mustrd-0.2.0/mustrd/test/test_mustrd.py +5 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/pyproject.toml +1 -1
- {mustrd-0.1.7 → mustrd-0.2.0}/LICENSE +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/README.adoc +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/TestResult.py +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/__init__.py +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/logger_setup.py +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/model/mustrdShapes.ttl +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/model/test-resources/resources.ttl +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/mustrdGraphDb.py +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/mustrdRdfLib.py +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/templates/md_ResultList_leaf_template.jinja +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/templates/md_ResultList_template.jinja +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/templates/md_stats_template.jinja +0 -0
- {mustrd-0.1.7 → mustrd-0.2.0}/mustrd/utils.py +0 -0
@@ -26,6 +26,15 @@ Run `pytest` from the project root.
|
|
26
26
|
|
27
27
|
If you have got this far then you are probably ready to create your own specifications to test your application SPARQL queries. These will be executed against the default RDFLib triplestore unless you configure one or more alternatives. The instructions for this are included in <<Configuring external triplestores>> below.
|
28
28
|
|
29
|
+
=== Paths
|
30
|
+
All paths are consired relative. That way mustrd tests can be versionned and shared easily.
|
31
|
+
To get absolute path from relative path in a spec file, we prefix it with the first existing result in:
|
32
|
+
1) Path where the spec is located
|
33
|
+
2) spec_path defined in mustrd test configuration files or cmd line argument
|
34
|
+
3) data_path defined in mustrd test configuration files or cmd line argument
|
35
|
+
4) Mustrd folder: In case of default resources packaged with mustrd source (will be in venv when mustrd is called as library)
|
36
|
+
We intentionally use the same method to build paths in all spec components to avoid confusion.
|
37
|
+
|
29
38
|
=== Givens
|
30
39
|
These are used to specify the dataset against which the SPARQL statement will be run.
|
31
40
|
They can be generated from external sources such as an existing graph, or a file or folder containing serialised RDF. It is also possible to specify the dataset as reified RDF directly in the test step. Currently tabular data sources such as csv files or TableDatasets are not supported.
|
@@ -178,16 +187,16 @@ The configuration file should be serialised RDF. An example in Turtle format is
|
|
178
187
|
must:GraphDbConfig1 a must:GraphDbConfig ;
|
179
188
|
must:url "http://localhost";
|
180
189
|
must:port "7200";
|
181
|
-
must:username "test/triplestore_config/tripleStoreCredentials.toml" ;
|
182
|
-
must:password "test/triplestore_config/tripleStoreCredentials.toml" ;
|
183
190
|
must:inputGraph "http://localhost:7200/test-graph" ;
|
184
191
|
must:repository "mustrd" .
|
185
192
|
----
|
186
|
-
|
193
|
+
To avoid versioning secrets when you want to version triplestore configuration (for example in case you want to run mustrd in CI), you have to configure user/password in a different file.
|
194
|
+
This file must be named as the triple store configuration file, but with "_secrets" just before the extension. For example triplestores.ttl -> triplestores_secrets.ttl
|
195
|
+
Subjects in the two files must match, no need to redefine the type, for example:
|
187
196
|
----
|
188
|
-
|
189
|
-
|
190
|
-
|
197
|
+
@prefix must: <https://mustrd.com/model/> .
|
198
|
+
must:GraphDbConfig1 must:username 'test' ;
|
199
|
+
must:password 'test' .
|
191
200
|
----
|
192
201
|
|
193
202
|
== Additional Notes for Developers
|
@@ -0,0 +1,5 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
2
|
+
<catalog prefer="public" xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog">
|
3
|
+
<uri id="User Entered Import Resolution" name="https://mustrd.com/triplestore/" uri="triplestoreOntology.ttl"/>
|
4
|
+
<group id="Folder Repository, directory=, recursive=true, Auto-Update=true, version=2" prefer="public" xml:base=""/>
|
5
|
+
</catalog>
|
@@ -0,0 +1,51 @@
|
|
1
|
+
@prefix : <https://mustrd.com/mustrdTest/> .
|
2
|
+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
3
|
+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
4
|
+
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
|
5
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
6
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
7
|
+
@prefix triplestore: <https://mustrd.com/triplestore/> .
|
8
|
+
@base <https://mustrd.com/mustrdTest/> .
|
9
|
+
|
10
|
+
<https://mustrd.com/mustrdTest/> rdf:type owl:Ontology ;
|
11
|
+
owl:imports triplestore: ;
|
12
|
+
rdfs:comment "Model user input to generate mustrd pytests" ;
|
13
|
+
rdfs:label "Mustrd test ontology" .
|
14
|
+
|
15
|
+
#################################################################
|
16
|
+
# Object Properties
|
17
|
+
#################################################################
|
18
|
+
|
19
|
+
### https://mustrd.com/mustrdTest/filterOnTripleStore
|
20
|
+
:filterOnTripleStore rdf:type owl:ObjectProperty ;
|
21
|
+
rdfs:domain :MustrdTest ;
|
22
|
+
rdfs:range triplestore:TripleStore .
|
23
|
+
|
24
|
+
|
25
|
+
#################################################################
|
26
|
+
# Data properties
|
27
|
+
#################################################################
|
28
|
+
|
29
|
+
### https://mustrd.com/mustrdTest/hasDataPath
|
30
|
+
:hasDataPath rdf:type owl:DatatypeProperty ;
|
31
|
+
rdfs:domain :MustrdTest ;
|
32
|
+
rdfs:range xsd:string .
|
33
|
+
|
34
|
+
|
35
|
+
### https://mustrd.com/mustrdTest/hasSpecPath
|
36
|
+
:hasSpecPath rdf:type owl:DatatypeProperty ;
|
37
|
+
rdfs:domain :MustrdTest ;
|
38
|
+
rdfs:range xsd:string .
|
39
|
+
|
40
|
+
|
41
|
+
#################################################################
|
42
|
+
# Classes
|
43
|
+
#################################################################
|
44
|
+
|
45
|
+
### https://mustrd.com/mustrdTest/MustrdTest
|
46
|
+
:MustrdTest rdf:type owl:Class ;
|
47
|
+
rdfs:comment "Contains user input to parameter a mustrd pytest" ;
|
48
|
+
rdfs:label "Mustrd test" .
|
49
|
+
|
50
|
+
|
51
|
+
### Generated by the OWL API (version 4.5.26.2023-07-17T20:34:13Z) https://github.com/owlcs/owlapi
|
@@ -0,0 +1,24 @@
|
|
1
|
+
@prefix : <https://mustrd.com/mustrdTest/> .
|
2
|
+
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
3
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
4
|
+
@prefix triplestore: <https://mustrd.com/triplestore/> .
|
5
|
+
@base <https://mustrd.com/mustrdTest/> .
|
6
|
+
|
7
|
+
|
8
|
+
|
9
|
+
|
10
|
+
:MustrdTestShape
|
11
|
+
a sh:NodeShape ;
|
12
|
+
sh:targetClass :MustrdTest ;
|
13
|
+
sh:property [ sh:path :hasDataPath ;
|
14
|
+
sh:datatype xsd:string ;
|
15
|
+
sh:minCount 1 ;
|
16
|
+
sh:maxCount 1 ],
|
17
|
+
[ sh:path :hasSpecPath ;
|
18
|
+
sh:datatype xsd:string ;
|
19
|
+
sh:minCount 1 ;
|
20
|
+
sh:maxCount 1 ],
|
21
|
+
[ sh:path :hasPytestPath ;
|
22
|
+
sh:datatype xsd:string ;
|
23
|
+
sh:maxCount 1 ] .
|
24
|
+
|
@@ -0,0 +1,174 @@
|
|
1
|
+
@base <https://mustrd.com/triplestore/> .
|
2
|
+
@prefix : <https://mustrd.com/triplestore/> .
|
3
|
+
@prefix dc: <http://purl.org/dc/elements/1.1/> .
|
4
|
+
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
5
|
+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
6
|
+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
7
|
+
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
|
8
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
9
|
+
@prefix foaf: <http://xmlns.com/foaf/spec/> .
|
10
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
11
|
+
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
|
12
|
+
#
|
13
|
+
#
|
14
|
+
# #################################################################
|
15
|
+
# #
|
16
|
+
# # Data properties
|
17
|
+
# #
|
18
|
+
# #################################################################
|
19
|
+
#
|
20
|
+
#
|
21
|
+
# https://mustrd.com/triplestore/gqeURI
|
22
|
+
#
|
23
|
+
# https://mustrd.com/triplestore/inputGraph
|
24
|
+
#
|
25
|
+
# https://mustrd.com/triplestore/outputGraph
|
26
|
+
#
|
27
|
+
# https://mustrd.com/triplestore/password
|
28
|
+
#
|
29
|
+
# https://mustrd.com/triplestore/port
|
30
|
+
#
|
31
|
+
# https://mustrd.com/triplestore/repository
|
32
|
+
#
|
33
|
+
# https://mustrd.com/triplestore/url
|
34
|
+
#
|
35
|
+
# https://mustrd.com/triplestore/username
|
36
|
+
#
|
37
|
+
#
|
38
|
+
#
|
39
|
+
# #################################################################
|
40
|
+
# #
|
41
|
+
# # Classes
|
42
|
+
# #
|
43
|
+
# #################################################################
|
44
|
+
#
|
45
|
+
#
|
46
|
+
# https://mustrd.com/triplestore/Anzo
|
47
|
+
#
|
48
|
+
# https://mustrd.com/triplestore/ExternalTripleStore
|
49
|
+
#
|
50
|
+
# https://mustrd.com/triplestore/GraphDb
|
51
|
+
#
|
52
|
+
# https://mustrd.com/triplestore/InternalTripleStore
|
53
|
+
#
|
54
|
+
# https://mustrd.com/triplestore/RdfLib
|
55
|
+
#
|
56
|
+
# https://mustrd.com/triplestore/TripleStore
|
57
|
+
#
|
58
|
+
# Generated by the OWL API (version 4.5.26.2023-07-17T20:34:13Z) https://github.com/owlcs/owlapi
|
59
|
+
|
60
|
+
<> a owl:Ontology;
|
61
|
+
rdfs:label "triple store ontology" .
|
62
|
+
|
63
|
+
:gqeURI a owl:DatatypeProperty;
|
64
|
+
rdfs:domain :Anzo;
|
65
|
+
rdfs:range xsd:string;
|
66
|
+
rdfs:label "gqeURI" .
|
67
|
+
|
68
|
+
:inputGraph a owl:DatatypeProperty;
|
69
|
+
rdfs:domain :TripleStore;
|
70
|
+
rdfs:range xsd:string;
|
71
|
+
rdfs:label "inputGraph" .
|
72
|
+
|
73
|
+
:outputGraph a owl:DatatypeProperty;
|
74
|
+
rdfs:domain :TripleStore;
|
75
|
+
rdfs:range xsd:string;
|
76
|
+
rdfs:comment """Uri of the graph where to insert/delete the triples
|
77
|
+
|
78
|
+
In some triple stores there is a default insert graph like in graphDB:
|
79
|
+
In graphDB the default insert graph is called: <http://www.openrdf.org/schema/sesame#nil>
|
80
|
+
or
|
81
|
+
<http://rdf4j.org/schema/rdf4j#nil>
|
82
|
+
|
83
|
+
Other triple stores do not define default insert graph like anzograph.
|
84
|
+
For those triple stores, this property must be mandatory""";
|
85
|
+
rdfs:label "outputGraph" .
|
86
|
+
|
87
|
+
:password a owl:DatatypeProperty;
|
88
|
+
rdfs:domain :ExternalTripleStore;
|
89
|
+
rdfs:range xsd:string;
|
90
|
+
rdfs:label "password" .
|
91
|
+
|
92
|
+
:port a owl:DatatypeProperty;
|
93
|
+
rdfs:domain :ExternalTripleStore;
|
94
|
+
rdfs:range xsd:string;
|
95
|
+
rdfs:comment "Triple store port";
|
96
|
+
rdfs:label "port" .
|
97
|
+
|
98
|
+
:repository a owl:DatatypeProperty;
|
99
|
+
rdfs:domain :GraphDb;
|
100
|
+
rdfs:range xsd:string;
|
101
|
+
rdfs:label "repository" .
|
102
|
+
|
103
|
+
:url a owl:DatatypeProperty;
|
104
|
+
rdfs:domain :ExternalTripleStore;
|
105
|
+
rdfs:range xsd:string;
|
106
|
+
rdfs:comment "triple store URL";
|
107
|
+
rdfs:label "url" .
|
108
|
+
|
109
|
+
:username a owl:DatatypeProperty;
|
110
|
+
rdfs:domain :ExternalTripleStore;
|
111
|
+
rdfs:range xsd:string;
|
112
|
+
rdfs:label "username" .
|
113
|
+
|
114
|
+
:Anzo a owl:Class;
|
115
|
+
rdfs:subClassOf :ExternalTripleStore;
|
116
|
+
rdfs:comment """TODO: model this:
|
117
|
+
|
118
|
+
definition of RDF dataset:
|
119
|
+
|
120
|
+
default graph=
|
121
|
+
RDF merge of default-graph-uri if exists
|
122
|
+
else RDF merge of FROM if exists
|
123
|
+
else all graphs in anzo graph (if you query a graphmart, default-graph-uri will be automatically set to the layers of the graphmart), if you are not sysadmin and you do not define the default graph, then the query will fail since you don't have permissions on all graphs
|
124
|
+
if default-graph-uri is defined then FROM clauses are ignored
|
125
|
+
|
126
|
+
default named graph =
|
127
|
+
RDF merge of named-graph-uri if exists
|
128
|
+
else RDF merge of FROM NAMED if exists
|
129
|
+
else default graph if exists
|
130
|
+
all graphs in anzo graph (if you query a graphmart, default-graph-uri will be automatically set to the layers of the graphmart), if you are not sysadmin and you do not define the default graph, then the query will fail since you don't have permissions on all graphs
|
131
|
+
if named-graph-uri is defined then FROM NAMED clauses are ignored
|
132
|
+
|
133
|
+
There is no default insert graph. If you try to insert/delete without graph clause, your query will fail.""";
|
134
|
+
rdfs:label "Anzo" .
|
135
|
+
|
136
|
+
:ExternalTripleStore a owl:Class;
|
137
|
+
rdfs:subClassOf :TripleStore .
|
138
|
+
|
139
|
+
:GraphDb a owl:Class;
|
140
|
+
rdfs:subClassOf :ExternalTripleStore;
|
141
|
+
rdfs:comment """TODO: model this:
|
142
|
+
|
143
|
+
Definition of RDF dataset in graphDB:
|
144
|
+
|
145
|
+
default graph (in the sense of W3C, called default dataset in graphDB) =
|
146
|
+
Virtual graph that exist only for a query and represent all triples accessible outside of a graph clause in a sparql query =
|
147
|
+
RDF merge of default-graph-uri if exists
|
148
|
+
else RDF merge of FROM if exists
|
149
|
+
else all graphs in the repository, including default insert graph
|
150
|
+
you can query default insert graph by including it (<http://www.openrdf.org/schema/sesame#nil> or <http://rdf4j.org/schema/rdf4j#nil>) in a default-graph-uri parameter or a FROM clause
|
151
|
+
If default-graph-uri is defined, then FROM clause is ignored
|
152
|
+
|
153
|
+
default named graph =
|
154
|
+
All triples accessible inside a graph clause:
|
155
|
+
RDF merge of named-graph-uri if exists
|
156
|
+
else RDF merge of FROM NAMED if exist
|
157
|
+
else all graphs in the repository, EXCLUDING default insert graph
|
158
|
+
(To be exact you can query the default insert graph only if you name it GRAPH <http://www.openrdf.org/schema/sesame#nil> or GRAPH <http://rdf4j.org/schema/rdf4j#nil>, but you won't query it with GRAPH ?graph)
|
159
|
+
default insert graph can be added to named-graph-uri or FROM NAMED
|
160
|
+
If a default-graph-uri is defined but no named-graph-uri, then default named graph = void
|
161
|
+
|
162
|
+
default insert graph (called \"The default graph\" in graphDB) =
|
163
|
+
Graph where triples are inserted / deleted when no graph clause is given in an INSERT or DELETE clause""";
|
164
|
+
rdfs:label "GraphDb" .
|
165
|
+
|
166
|
+
:InternalTripleStore a owl:Class;
|
167
|
+
rdfs:subClassOf :TripleStore .
|
168
|
+
|
169
|
+
:RdfLib a owl:Class;
|
170
|
+
rdfs:subClassOf :InternalTripleStore;
|
171
|
+
rdfs:label "RdfLib" .
|
172
|
+
|
173
|
+
:TripleStore a owl:Class;
|
174
|
+
rdfs:label "TripleStore" .
|
@@ -0,0 +1,42 @@
|
|
1
|
+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
2
|
+
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
3
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
4
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
5
|
+
@prefix ex: <http://www.example.org/#> .
|
6
|
+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
7
|
+
@prefix triplestore: <https://mustrd.com/triplestore/> .
|
8
|
+
|
9
|
+
triplestore:ExternalTripleStoreShape
|
10
|
+
a sh:NodeShape ;
|
11
|
+
sh:targetClass triplestore:ExternalTripleStore ;
|
12
|
+
sh:property [ sh:path triplestore:url ;
|
13
|
+
sh:minCount 1 ;
|
14
|
+
sh:maxCount 1 ],
|
15
|
+
[ sh:path triplestore:port ;
|
16
|
+
sh:minCount 1 ;
|
17
|
+
sh:maxCount 1 ],
|
18
|
+
[ sh:path triplestore:username ;
|
19
|
+
sh:maxCount 1 ],
|
20
|
+
[ sh:path triplestore:password ;
|
21
|
+
sh:maxCount 1 ] .
|
22
|
+
|
23
|
+
triplestore:AnzoShape
|
24
|
+
a sh:NodeShape ;
|
25
|
+
sh:targetClass triplestore:Anzo ;
|
26
|
+
sh:property [ sh:path triplestore:gqeURI ;
|
27
|
+
sh:minCount 1 ;
|
28
|
+
sh:maxCount 1 ],
|
29
|
+
[ sh:path triplestore:outputGraph ;
|
30
|
+
sh:minCount 1 ;
|
31
|
+
sh:maxCount 1 ],
|
32
|
+
# For anzo the input graph is not really necessary if the user is sysadmin
|
33
|
+
# but querying all graphs in AZG is usually not a good idea, so for the moment this is forbidden
|
34
|
+
[ sh:path triplestore:inputGraph ;
|
35
|
+
sh:minCount 1 ] .
|
36
|
+
|
37
|
+
triplestore:GraphDbShape
|
38
|
+
a sh:NodeShape ;
|
39
|
+
sh:targetClass triplestore:GraphDb ;
|
40
|
+
sh:property [ sh:path triplestore:repository ;
|
41
|
+
sh:minCount 1 ;
|
42
|
+
sh:maxCount 1 ] .
|
@@ -40,13 +40,13 @@ from rdflib import Graph, URIRef, RDF, XSD, SH, Literal
|
|
40
40
|
from rdflib.compare import isomorphic, graph_diff
|
41
41
|
import pandas
|
42
42
|
|
43
|
-
from .namespace import MUST
|
43
|
+
from .namespace import MUST, TRIPLESTORE
|
44
44
|
import requests
|
45
45
|
import json
|
46
46
|
from pandas import DataFrame
|
47
47
|
|
48
48
|
from .spec_component import TableThenSpec, parse_spec_component, WhenSpec, ThenSpec
|
49
|
-
from .utils import is_json
|
49
|
+
from .utils import is_json,get_mustrd_root
|
50
50
|
from colorama import Fore, Style
|
51
51
|
from tabulate import tabulate
|
52
52
|
from collections import defaultdict
|
@@ -93,6 +93,7 @@ class Specification:
|
|
93
93
|
given: Graph
|
94
94
|
when: WhenSpec
|
95
95
|
then: ThenSpec
|
96
|
+
spec_file_name: str = "default.mustrd.ttl"
|
96
97
|
|
97
98
|
|
98
99
|
@dataclass
|
@@ -152,6 +153,7 @@ class TripleStoreConnectionError(SpecResult):
|
|
152
153
|
@dataclass
|
153
154
|
class SpecSkipped(SpecResult):
|
154
155
|
message: str
|
156
|
+
spec_file_name: str = "default.mustrd.ttl"
|
155
157
|
|
156
158
|
|
157
159
|
@dataclass
|
@@ -176,16 +178,15 @@ class UpdateSparqlQuery(SparqlAction):
|
|
176
178
|
|
177
179
|
# https://github.com/Semantic-partners/mustrd/issues/19
|
178
180
|
|
179
|
-
def validate_specs(run_config: dict, triple_stores: List, shacl_graph: Graph, ont_graph: Graph)\
|
181
|
+
def validate_specs(run_config: dict, triple_stores: List, shacl_graph: Graph, ont_graph: Graph, file_name: str = "*")\
|
180
182
|
-> Tuple[List, Graph, List]:
|
181
183
|
spec_graph = Graph()
|
182
184
|
subject_uris = set()
|
183
185
|
focus_uris = set()
|
184
186
|
invalid_specs = []
|
185
|
-
|
186
|
-
ttl_files = list(run_config['spec_path'].glob('**/*.mustrd.ttl'))
|
187
|
+
ttl_files = list(run_config['spec_path'].glob(f'**/{file_name}.mustrd.ttl'))
|
187
188
|
ttl_files.sort()
|
188
|
-
log.info(f"Found {len(ttl_files)} mustrd.ttl files in {run_config['spec_path']}")
|
189
|
+
log.info(f"Found {len(ttl_files)} {file_name}.mustrd.ttl files in {run_config['spec_path']}")
|
189
190
|
|
190
191
|
for file in ttl_files:
|
191
192
|
error_messages = []
|
@@ -233,7 +234,7 @@ def validate_specs(run_config: dict, triple_stores: List, shacl_graph: Graph, on
|
|
233
234
|
if len(error_messages) > 0:
|
234
235
|
error_messages.sort()
|
235
236
|
error_message = "\n".join(msg for msg in error_messages)
|
236
|
-
invalid_specs += [SpecSkipped(subject_uri, triple_store["type"], error_message) for triple_store in
|
237
|
+
invalid_specs += [SpecSkipped(subject_uri, triple_store["type"], error_message, file.name) for triple_store in
|
237
238
|
triple_stores]
|
238
239
|
else:
|
239
240
|
subject_uris.add(subject_uri)
|
@@ -241,13 +242,12 @@ def validate_specs(run_config: dict, triple_stores: List, shacl_graph: Graph, on
|
|
241
242
|
this_spec_graph.parse(file)
|
242
243
|
spec_uris_in_this_file = list(this_spec_graph.subjects(RDF.type, MUST.TestSpec))
|
243
244
|
for spec in spec_uris_in_this_file:
|
244
|
-
tripleToAdd = [spec, MUST.specSourceFile, Literal(file)]
|
245
245
|
# print(f"adding {tripleToAdd}")
|
246
|
-
this_spec_graph.add(
|
246
|
+
this_spec_graph.add([spec, MUST.specSourceFile, Literal(file)])
|
247
|
+
this_spec_graph.add([spec, MUST.specFileName, Literal(file.name)])
|
247
248
|
# print(f"beforeadd: {spec_graph}" )
|
248
249
|
# print(f"beforeadd: {str(this_spec_graph.serialize())}" )
|
249
250
|
spec_graph += this_spec_graph
|
250
|
-
# print(f"afteradd: {str(spec_graph.serialize())}" )
|
251
251
|
|
252
252
|
|
253
253
|
sourceFiles = list(spec_graph.subject_objects(MUST.specSourceFile))
|
@@ -276,14 +276,14 @@ def get_specs(spec_uris: List[URIRef], spec_graph: Graph, triple_stores: List[di
|
|
276
276
|
for triple_store in triple_stores:
|
277
277
|
if "error" in triple_store:
|
278
278
|
log.error(f"{triple_store['error']}. No specs run for this triple store.")
|
279
|
-
skipped_results += [SpecSkipped(spec_uri, triple_store['type'], triple_store['error']) for spec_uri in
|
279
|
+
skipped_results += [SpecSkipped(spec_uri, triple_store['type'], triple_store['error'], get_spec_file(spec_uri, spec_graph)) for spec_uri in
|
280
280
|
spec_uris]
|
281
281
|
else:
|
282
282
|
for spec_uri in spec_uris:
|
283
283
|
try:
|
284
284
|
specs += [get_spec(spec_uri, spec_graph, run_config, triple_store)]
|
285
285
|
except (ValueError, FileNotFoundError, ConnectionError) as e:
|
286
|
-
skipped_results += [SpecSkipped(spec_uri, triple_store['type'], e)]
|
286
|
+
skipped_results += [SpecSkipped(spec_uri, triple_store['type'], e, get_spec_file(spec_uri, spec_graph))]
|
287
287
|
|
288
288
|
except (BadSyntax, FileNotFoundError) as e:
|
289
289
|
template = "An exception of type {0} occurred when trying to parse the triple store configuration file. " \
|
@@ -303,11 +303,13 @@ def run_specs(specs) -> List[SpecResult]:
|
|
303
303
|
results.append(run_spec(specification))
|
304
304
|
return results
|
305
305
|
|
306
|
+
def get_spec_file(spec_uri: URIRef, spec_graph: Graph):
|
307
|
+
return str(spec_graph.value(subject = spec_uri, predicate = MUST.specFileName, default = "default.mustrd.ttl"))
|
306
308
|
|
307
309
|
def get_spec(spec_uri: URIRef, spec_graph: Graph, run_config: dict, mustrd_triple_store: dict = None) -> Specification:
|
308
310
|
try:
|
309
311
|
if mustrd_triple_store is None:
|
310
|
-
mustrd_triple_store = {"type":
|
312
|
+
mustrd_triple_store = {"type": TRIPLESTORE.RdfLib}
|
311
313
|
components = []
|
312
314
|
for predicate in MUST.given, MUST.when, MUST.then:
|
313
315
|
components.append(parse_spec_component(subject=spec_uri,
|
@@ -316,9 +318,9 @@ def get_spec(spec_uri: URIRef, spec_graph: Graph, run_config: dict, mustrd_tripl
|
|
316
318
|
run_config=run_config,
|
317
319
|
mustrd_triple_store=mustrd_triple_store))
|
318
320
|
|
319
|
-
|
321
|
+
spec_file_name = get_spec_file(spec_uri, spec_graph)
|
320
322
|
# https://github.com/Semantic-partners/mustrd/issues/92
|
321
|
-
return Specification(spec_uri, mustrd_triple_store, components[0].value, components[1], components[2])
|
323
|
+
return Specification(spec_uri, mustrd_triple_store, components[0].value, components[1], components[2], spec_file_name)
|
322
324
|
|
323
325
|
except (ValueError, FileNotFoundError) as e:
|
324
326
|
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
|
@@ -354,7 +356,7 @@ def run_spec(spec: Specification) -> SpecResult:
|
|
354
356
|
log.debug(f"{given_as_turtle}")
|
355
357
|
upload_given(triple_store, spec.given)
|
356
358
|
else:
|
357
|
-
if triple_store['type'] ==
|
359
|
+
if triple_store['type'] == TRIPLESTORE.RdfLib:
|
358
360
|
return SpecSkipped(spec_uri, triple_store['type'], "Unable to run Inherited State tests on Rdflib")
|
359
361
|
try:
|
360
362
|
for when in spec.when:
|
@@ -386,57 +388,64 @@ def get_triple_store_graph(triple_store_graph_path: Path, secrets: str):
|
|
386
388
|
return Graph().parse(triple_store_graph_path).parse(data = secrets)
|
387
389
|
else:
|
388
390
|
secret_path = triple_store_graph_path.parent / Path(triple_store_graph_path.stem + "_secrets" + triple_store_graph_path.suffix)
|
389
|
-
return Graph().parse(triple_store_graph_path).parse(
|
391
|
+
return Graph().parse(triple_store_graph_path).parse(secret_path)
|
390
392
|
|
391
393
|
|
392
394
|
def get_triple_stores(triple_store_graph: Graph) -> list[dict]:
|
393
395
|
triple_stores = []
|
396
|
+
shacl_graph = Graph().parse(Path(os.path.join(get_mustrd_root(), "model/triplestoreshapes.ttl")))
|
397
|
+
ont_graph = Graph().parse(Path(os.path.join(get_mustrd_root(), "model/triplestoreOntology.ttl")))
|
398
|
+
conforms, results_graph, results_text = validate(
|
399
|
+
data_graph= triple_store_graph,
|
400
|
+
shacl_graph = shacl_graph,
|
401
|
+
ont_graph = ont_graph,
|
402
|
+
advanced= True,
|
403
|
+
inference= 'none'
|
404
|
+
)
|
405
|
+
if not conforms:
|
406
|
+
raise ValueError(f"Triple store configuration not conform to the shapes. SHACL report: {results_text}", results_graph)
|
394
407
|
for triple_store_config, rdf_type, triple_store_type in triple_store_graph.triples((None, RDF.type, None)):
|
395
408
|
triple_store = {}
|
396
|
-
|
397
|
-
|
398
|
-
triple_store["type"] = MUST.RdfLib
|
409
|
+
triple_store["type"] = triple_store_type
|
410
|
+
triple_store["uri"] = triple_store_config
|
399
411
|
# Anzo graph via anzo
|
400
|
-
|
401
|
-
triple_store["
|
402
|
-
triple_store["
|
403
|
-
triple_store["port"] = triple_store_graph.value(subject=triple_store_config, predicate=MUST.port)
|
412
|
+
if triple_store_type == TRIPLESTORE.Anzo:
|
413
|
+
triple_store["url"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.url)
|
414
|
+
triple_store["port"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.port)
|
404
415
|
try:
|
405
|
-
triple_store["username"] = str(triple_store_graph.value(subject=triple_store_config, predicate=
|
406
|
-
triple_store["password"] = str(triple_store_graph.value(subject=triple_store_config, predicate=
|
416
|
+
triple_store["username"] = str(triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.username))
|
417
|
+
triple_store["password"] = str(triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.password))
|
407
418
|
except (FileNotFoundError, ValueError) as e:
|
408
419
|
triple_store["error"] = e
|
409
|
-
triple_store["gqe_uri"] = triple_store_graph.value(subject=triple_store_config, predicate=
|
420
|
+
triple_store["gqe_uri"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.gqeURI)
|
410
421
|
triple_store["input_graph"] = triple_store_graph.value(subject=triple_store_config,
|
411
|
-
predicate=
|
422
|
+
predicate=TRIPLESTORE.inputGraph)
|
412
423
|
triple_store["output_graph"] = triple_store_graph.value(subject=triple_store_config,
|
413
|
-
predicate=
|
424
|
+
predicate=TRIPLESTORE.outputGraph)
|
414
425
|
try:
|
415
426
|
check_triple_store_params(triple_store, ["url", "port", "username", "password", "input_graph"])
|
416
427
|
except ValueError as e:
|
417
428
|
triple_store["error"] = e
|
418
429
|
# GraphDB
|
419
|
-
elif triple_store_type ==
|
420
|
-
triple_store["
|
421
|
-
triple_store["
|
422
|
-
triple_store["port"] = triple_store_graph.value(subject=triple_store_config, predicate=MUST.port)
|
430
|
+
elif triple_store_type == TRIPLESTORE.GraphDb:
|
431
|
+
triple_store["url"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.url)
|
432
|
+
triple_store["port"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.port)
|
423
433
|
try:
|
424
|
-
triple_store["username"] = str(triple_store_graph.value(subject=triple_store_config, predicate=
|
425
|
-
triple_store["password"] = str(triple_store_graph.value(subject=triple_store_config, predicate=
|
434
|
+
triple_store["username"] = str(triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.username))
|
435
|
+
triple_store["password"] = str(triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.password))
|
426
436
|
except (FileNotFoundError, ValueError) as e:
|
427
437
|
log.error(f"Credential retrieval failed {e}")
|
428
438
|
triple_store["error"] = e
|
429
439
|
triple_store["repository"] = triple_store_graph.value(subject=triple_store_config,
|
430
|
-
predicate=
|
440
|
+
predicate=TRIPLESTORE.repository)
|
431
441
|
triple_store["input_graph"] = triple_store_graph.value(subject=triple_store_config,
|
432
|
-
predicate=
|
442
|
+
predicate=TRIPLESTORE.inputGraph)
|
433
443
|
|
434
444
|
try:
|
435
445
|
check_triple_store_params(triple_store, ["url", "port", "repository"])
|
436
446
|
except ValueError as e:
|
437
447
|
triple_store["error"] = e
|
438
|
-
|
439
|
-
triple_store["type"] = triple_store_type
|
448
|
+
elif triple_store_type != TRIPLESTORE.RdfLib:
|
440
449
|
triple_store["error"] = f"Triple store not implemented: {triple_store_type}"
|
441
450
|
|
442
451
|
triple_stores.append(triple_store)
|