mustrd 0.1.6__tar.gz → 0.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mustrd-0.1.6 → mustrd-0.1.8}/PKG-INFO +1 -1
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/README.adoc +6 -6
- mustrd-0.1.8/mustrd/model/catalog-v001.xml +5 -0
- mustrd-0.1.8/mustrd/model/mustrdTestOntology.ttl +51 -0
- mustrd-0.1.8/mustrd/model/mustrdTestShapes.ttl +24 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/model/ontology.ttl +2 -1
- mustrd-0.1.8/mustrd/model/triplestoreOntology.ttl +174 -0
- mustrd-0.1.8/mustrd/model/triplestoreshapes.ttl +42 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/mustrd.py +49 -39
- mustrd-0.1.8/mustrd/mustrdTestPlugin.py +328 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/namespace.py +36 -29
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/run.py +2 -2
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/spec_component.py +11 -11
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/steprunner.py +15 -15
- mustrd-0.1.8/mustrd/test/test_mustrd.py +5 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/pyproject.toml +2 -2
- mustrd-0.1.6/mustrd/mustrdTestPlugin.py +0 -201
- mustrd-0.1.6/mustrd/plugin.py +0 -87
- {mustrd-0.1.6 → mustrd-0.1.8}/LICENSE +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/README.adoc +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/TestResult.py +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/__init__.py +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/logger_setup.py +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/model/mustrdShapes.ttl +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/model/test-resources/resources.ttl +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/mustrdAnzo.py +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/mustrdGraphDb.py +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/mustrdRdfLib.py +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/templates/md_ResultList_leaf_template.jinja +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/templates/md_ResultList_template.jinja +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/templates/md_stats_template.jinja +0 -0
- {mustrd-0.1.6 → mustrd-0.1.8}/mustrd/utils.py +0 -0
@@ -178,16 +178,16 @@ The configuration file should be serialised RDF. An example in Turtle format is
|
|
178
178
|
must:GraphDbConfig1 a must:GraphDbConfig ;
|
179
179
|
must:url "http://localhost";
|
180
180
|
must:port "7200";
|
181
|
-
must:username "test/triplestore_config/tripleStoreCredentials.toml" ;
|
182
|
-
must:password "test/triplestore_config/tripleStoreCredentials.toml" ;
|
183
181
|
must:inputGraph "http://localhost:7200/test-graph" ;
|
184
182
|
must:repository "mustrd" .
|
185
183
|
----
|
186
|
-
|
184
|
+
To avoid versioning secrets when you want to version triplestore configuration (for example in case you want to run mustrd in CI), you have to configure user/password in a different file.
|
185
|
+
This file must be named as the triple store configuration file, but with "_secrets" just before the extension. For example triplestores.ttl -> triplestores_secrets.ttl
|
186
|
+
Subjects in the two files must match, no need to redefine the type, for example:
|
187
187
|
----
|
188
|
-
|
189
|
-
|
190
|
-
|
188
|
+
@prefix must: <https://mustrd.com/model/> .
|
189
|
+
must:GraphDbConfig1 must:username 'test' ;
|
190
|
+
must:password 'test' .
|
191
191
|
----
|
192
192
|
|
193
193
|
== Additional Notes for Developers
|
@@ -0,0 +1,5 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
2
|
+
<catalog prefer="public" xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog">
|
3
|
+
<uri id="User Entered Import Resolution" name="https://mustrd.com/triplestore/" uri="triplestoreOntology.ttl"/>
|
4
|
+
<group id="Folder Repository, directory=, recursive=true, Auto-Update=true, version=2" prefer="public" xml:base=""/>
|
5
|
+
</catalog>
|
@@ -0,0 +1,51 @@
|
|
1
|
+
@prefix : <https://mustrd.com/mustrdTest/> .
|
2
|
+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
3
|
+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
4
|
+
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
|
5
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
6
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
7
|
+
@prefix triplestore: <https://mustrd.com/triplestore/> .
|
8
|
+
@base <https://mustrd.com/mustrdTest/> .
|
9
|
+
|
10
|
+
<https://mustrd.com/mustrdTest/> rdf:type owl:Ontology ;
|
11
|
+
owl:imports triplestore: ;
|
12
|
+
rdfs:comment "Model user input to generate mustrd pytests" ;
|
13
|
+
rdfs:label "Mustrd test ontology" .
|
14
|
+
|
15
|
+
#################################################################
|
16
|
+
# Object Properties
|
17
|
+
#################################################################
|
18
|
+
|
19
|
+
### https://mustrd.com/mustrdTest/filterOnTripleStore
|
20
|
+
:filterOnTripleStore rdf:type owl:ObjectProperty ;
|
21
|
+
rdfs:domain :MustrdTest ;
|
22
|
+
rdfs:range triplestore:TripleStore .
|
23
|
+
|
24
|
+
|
25
|
+
#################################################################
|
26
|
+
# Data properties
|
27
|
+
#################################################################
|
28
|
+
|
29
|
+
### https://mustrd.com/mustrdTest/hasDataPath
|
30
|
+
:hasDataPath rdf:type owl:DatatypeProperty ;
|
31
|
+
rdfs:domain :MustrdTest ;
|
32
|
+
rdfs:range xsd:string .
|
33
|
+
|
34
|
+
|
35
|
+
### https://mustrd.com/mustrdTest/hasSpecPath
|
36
|
+
:hasSpecPath rdf:type owl:DatatypeProperty ;
|
37
|
+
rdfs:domain :MustrdTest ;
|
38
|
+
rdfs:range xsd:string .
|
39
|
+
|
40
|
+
|
41
|
+
#################################################################
|
42
|
+
# Classes
|
43
|
+
#################################################################
|
44
|
+
|
45
|
+
### https://mustrd.com/mustrdTest/MustrdTest
|
46
|
+
:MustrdTest rdf:type owl:Class ;
|
47
|
+
rdfs:comment "Contains user input to parameter a mustrd pytest" ;
|
48
|
+
rdfs:label "Mustrd test" .
|
49
|
+
|
50
|
+
|
51
|
+
### Generated by the OWL API (version 4.5.26.2023-07-17T20:34:13Z) https://github.com/owlcs/owlapi
|
@@ -0,0 +1,24 @@
|
|
1
|
+
@prefix : <https://mustrd.com/mustrdTest/> .
|
2
|
+
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
3
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
4
|
+
@prefix triplestore: <https://mustrd.com/triplestore/> .
|
5
|
+
@base <https://mustrd.com/mustrdTest/> .
|
6
|
+
|
7
|
+
|
8
|
+
|
9
|
+
|
10
|
+
:MustrdTestShape
|
11
|
+
a sh:NodeShape ;
|
12
|
+
sh:targetClass :MustrdTest ;
|
13
|
+
sh:property [ sh:path :hasDataPath ;
|
14
|
+
sh:datatype xsd:string ;
|
15
|
+
sh:minCount 1 ;
|
16
|
+
sh:maxCount 1 ],
|
17
|
+
[ sh:path :hasSpecPath ;
|
18
|
+
sh:datatype xsd:string ;
|
19
|
+
sh:minCount 1 ;
|
20
|
+
sh:maxCount 1 ],
|
21
|
+
[ sh:path :hasPytestPath ;
|
22
|
+
sh:datatype xsd:string ;
|
23
|
+
sh:maxCount 1 ] .
|
24
|
+
|
@@ -0,0 +1,174 @@
|
|
1
|
+
@base <https://mustrd.com/triplestore/> .
|
2
|
+
@prefix : <https://mustrd.com/triplestore/> .
|
3
|
+
@prefix dc: <http://purl.org/dc/elements/1.1/> .
|
4
|
+
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
5
|
+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
6
|
+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
7
|
+
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
|
8
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
9
|
+
@prefix foaf: <http://xmlns.com/foaf/spec/> .
|
10
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
11
|
+
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
|
12
|
+
#
|
13
|
+
#
|
14
|
+
# #################################################################
|
15
|
+
# #
|
16
|
+
# # Data properties
|
17
|
+
# #
|
18
|
+
# #################################################################
|
19
|
+
#
|
20
|
+
#
|
21
|
+
# https://mustrd.com/triplestore/gqeURI
|
22
|
+
#
|
23
|
+
# https://mustrd.com/triplestore/inputGraph
|
24
|
+
#
|
25
|
+
# https://mustrd.com/triplestore/outputGraph
|
26
|
+
#
|
27
|
+
# https://mustrd.com/triplestore/password
|
28
|
+
#
|
29
|
+
# https://mustrd.com/triplestore/port
|
30
|
+
#
|
31
|
+
# https://mustrd.com/triplestore/repository
|
32
|
+
#
|
33
|
+
# https://mustrd.com/triplestore/url
|
34
|
+
#
|
35
|
+
# https://mustrd.com/triplestore/username
|
36
|
+
#
|
37
|
+
#
|
38
|
+
#
|
39
|
+
# #################################################################
|
40
|
+
# #
|
41
|
+
# # Classes
|
42
|
+
# #
|
43
|
+
# #################################################################
|
44
|
+
#
|
45
|
+
#
|
46
|
+
# https://mustrd.com/triplestore/Anzo
|
47
|
+
#
|
48
|
+
# https://mustrd.com/triplestore/ExternalTripleStore
|
49
|
+
#
|
50
|
+
# https://mustrd.com/triplestore/GraphDb
|
51
|
+
#
|
52
|
+
# https://mustrd.com/triplestore/InternalTripleStore
|
53
|
+
#
|
54
|
+
# https://mustrd.com/triplestore/RdfLib
|
55
|
+
#
|
56
|
+
# https://mustrd.com/triplestore/TripleStore
|
57
|
+
#
|
58
|
+
# Generated by the OWL API (version 4.5.26.2023-07-17T20:34:13Z) https://github.com/owlcs/owlapi
|
59
|
+
|
60
|
+
<> a owl:Ontology;
|
61
|
+
rdfs:label "triple store ontology" .
|
62
|
+
|
63
|
+
:gqeURI a owl:DatatypeProperty;
|
64
|
+
rdfs:domain :Anzo;
|
65
|
+
rdfs:range xsd:string;
|
66
|
+
rdfs:label "gqeURI" .
|
67
|
+
|
68
|
+
:inputGraph a owl:DatatypeProperty;
|
69
|
+
rdfs:domain :TripleStore;
|
70
|
+
rdfs:range xsd:string;
|
71
|
+
rdfs:label "inputGraph" .
|
72
|
+
|
73
|
+
:outputGraph a owl:DatatypeProperty;
|
74
|
+
rdfs:domain :TripleStore;
|
75
|
+
rdfs:range xsd:string;
|
76
|
+
rdfs:comment """Uri of the graph where to insert/delete the triples
|
77
|
+
|
78
|
+
In some triple stores there is a default insert graph like in graphDB:
|
79
|
+
In graphDB the default insert graph is called: <http://www.openrdf.org/schema/sesame#nil>
|
80
|
+
or
|
81
|
+
<http://rdf4j.org/schema/rdf4j#nil>
|
82
|
+
|
83
|
+
Other triple stores do not define default insert graph like anzograph.
|
84
|
+
For those triple stores, this property must be mandatory""";
|
85
|
+
rdfs:label "outputGraph" .
|
86
|
+
|
87
|
+
:password a owl:DatatypeProperty;
|
88
|
+
rdfs:domain :ExternalTripleStore;
|
89
|
+
rdfs:range xsd:string;
|
90
|
+
rdfs:label "password" .
|
91
|
+
|
92
|
+
:port a owl:DatatypeProperty;
|
93
|
+
rdfs:domain :ExternalTripleStore;
|
94
|
+
rdfs:range xsd:string;
|
95
|
+
rdfs:comment "Triple store port";
|
96
|
+
rdfs:label "port" .
|
97
|
+
|
98
|
+
:repository a owl:DatatypeProperty;
|
99
|
+
rdfs:domain :GraphDb;
|
100
|
+
rdfs:range xsd:string;
|
101
|
+
rdfs:label "repository" .
|
102
|
+
|
103
|
+
:url a owl:DatatypeProperty;
|
104
|
+
rdfs:domain :ExternalTripleStore;
|
105
|
+
rdfs:range xsd:string;
|
106
|
+
rdfs:comment "triple store URL";
|
107
|
+
rdfs:label "url" .
|
108
|
+
|
109
|
+
:username a owl:DatatypeProperty;
|
110
|
+
rdfs:domain :ExternalTripleStore;
|
111
|
+
rdfs:range xsd:string;
|
112
|
+
rdfs:label "username" .
|
113
|
+
|
114
|
+
:Anzo a owl:Class;
|
115
|
+
rdfs:subClassOf :ExternalTripleStore;
|
116
|
+
rdfs:comment """TODO: model this:
|
117
|
+
|
118
|
+
definition of RDF dataset:
|
119
|
+
|
120
|
+
default graph=
|
121
|
+
RDF merge of default-graph-uri if exists
|
122
|
+
else RDF merge of FROM if exists
|
123
|
+
else all graphs in anzo graph (if you query a graphmart, default-graph-uri will be automatically set to the layers of the graphmart), if you are not sysadmin and you do not define the default graph, then the query will fail since you don't have permissions on all graphs
|
124
|
+
if default-graph-uri is defined then FROM clauses are ignored
|
125
|
+
|
126
|
+
default named graph =
|
127
|
+
RDF merge of named-graph-uri if exists
|
128
|
+
else RDF merge of FROM NAMED if exists
|
129
|
+
else default graph if exists
|
130
|
+
all graphs in anzo graph (if you query a graphmart, default-graph-uri will be automatically set to the layers of the graphmart), if you are not sysadmin and you do not define the default graph, then the query will fail since you don't have permissions on all graphs
|
131
|
+
if named-graph-uri is defined then FROM NAMED clauses are ignored
|
132
|
+
|
133
|
+
There is no default insert graph. If you try to insert/delete without graph clause, your query will fail.""";
|
134
|
+
rdfs:label "Anzo" .
|
135
|
+
|
136
|
+
:ExternalTripleStore a owl:Class;
|
137
|
+
rdfs:subClassOf :TripleStore .
|
138
|
+
|
139
|
+
:GraphDb a owl:Class;
|
140
|
+
rdfs:subClassOf :ExternalTripleStore;
|
141
|
+
rdfs:comment """TODO: model this:
|
142
|
+
|
143
|
+
Definition of RDF dataset in graphDB:
|
144
|
+
|
145
|
+
default graph (in the sense of W3C, called default dataset in graphDB) =
|
146
|
+
Virtual graph that exist only for a query and represent all triples accessible outside of a graph clause in a sparql query =
|
147
|
+
RDF merge of default-graph-uri if exists
|
148
|
+
else RDF merge of FROM if exists
|
149
|
+
else all graphs in the repository, including default insert graph
|
150
|
+
you can query default insert graph by including it (<http://www.openrdf.org/schema/sesame#nil> or <http://rdf4j.org/schema/rdf4j#nil>) in a default-graph-uri parameter or a FROM clause
|
151
|
+
If default-graph-uri is defined, then FROM clause is ignored
|
152
|
+
|
153
|
+
default named graph =
|
154
|
+
All triples accessible inside a graph clause:
|
155
|
+
RDF merge of named-graph-uri if exists
|
156
|
+
else RDF merge of FROM NAMED if exist
|
157
|
+
else all graphs in the repository, EXCLUDING default insert graph
|
158
|
+
(To be exact you can query the default insert graph only if you name it GRAPH <http://www.openrdf.org/schema/sesame#nil> or GRAPH <http://rdf4j.org/schema/rdf4j#nil>, but you won't query it with GRAPH ?graph)
|
159
|
+
default insert graph can be added to named-graph-uri or FROM NAMED
|
160
|
+
If a default-graph-uri is defined but no named-graph-uri, then default named graph = void
|
161
|
+
|
162
|
+
default insert graph (called \"The default graph\" in graphDB) =
|
163
|
+
Graph where triples are inserted / deleted when no graph clause is given in an INSERT or DELETE clause""";
|
164
|
+
rdfs:label "GraphDb" .
|
165
|
+
|
166
|
+
:InternalTripleStore a owl:Class;
|
167
|
+
rdfs:subClassOf :TripleStore .
|
168
|
+
|
169
|
+
:RdfLib a owl:Class;
|
170
|
+
rdfs:subClassOf :InternalTripleStore;
|
171
|
+
rdfs:label "RdfLib" .
|
172
|
+
|
173
|
+
:TripleStore a owl:Class;
|
174
|
+
rdfs:label "TripleStore" .
|
@@ -0,0 +1,42 @@
|
|
1
|
+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
2
|
+
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
3
|
+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
4
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
5
|
+
@prefix ex: <http://www.example.org/#> .
|
6
|
+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
7
|
+
@prefix triplestore: <https://mustrd.com/triplestore/> .
|
8
|
+
|
9
|
+
triplestore:ExternalTripleStoreShape
|
10
|
+
a sh:NodeShape ;
|
11
|
+
sh:targetClass triplestore:ExternalTripleStore ;
|
12
|
+
sh:property [ sh:path triplestore:url ;
|
13
|
+
sh:minCount 1 ;
|
14
|
+
sh:maxCount 1 ],
|
15
|
+
[ sh:path triplestore:port ;
|
16
|
+
sh:minCount 1 ;
|
17
|
+
sh:maxCount 1 ],
|
18
|
+
[ sh:path triplestore:username ;
|
19
|
+
sh:maxCount 1 ],
|
20
|
+
[ sh:path triplestore:password ;
|
21
|
+
sh:maxCount 1 ] .
|
22
|
+
|
23
|
+
triplestore:AnzoShape
|
24
|
+
a sh:NodeShape ;
|
25
|
+
sh:targetClass triplestore:Anzo ;
|
26
|
+
sh:property [ sh:path triplestore:gqeURI ;
|
27
|
+
sh:minCount 1 ;
|
28
|
+
sh:maxCount 1 ],
|
29
|
+
[ sh:path triplestore:outputGraph ;
|
30
|
+
sh:minCount 1 ;
|
31
|
+
sh:maxCount 1 ],
|
32
|
+
# For anzo the input graph is not really necessary if the user is sysadmin
|
33
|
+
# but querying all graphs in AZG is usually not a good idea, so for the moment this is forbidden
|
34
|
+
[ sh:path triplestore:inputGraph ;
|
35
|
+
sh:minCount 1 ] .
|
36
|
+
|
37
|
+
triplestore:GraphDbShape
|
38
|
+
a sh:NodeShape ;
|
39
|
+
sh:targetClass triplestore:GraphDb ;
|
40
|
+
sh:property [ sh:path triplestore:repository ;
|
41
|
+
sh:minCount 1 ;
|
42
|
+
sh:maxCount 1 ] .
|
@@ -40,13 +40,13 @@ from rdflib import Graph, URIRef, RDF, XSD, SH, Literal
|
|
40
40
|
from rdflib.compare import isomorphic, graph_diff
|
41
41
|
import pandas
|
42
42
|
|
43
|
-
from .namespace import MUST
|
43
|
+
from .namespace import MUST, TRIPLESTORE
|
44
44
|
import requests
|
45
45
|
import json
|
46
46
|
from pandas import DataFrame
|
47
47
|
|
48
48
|
from .spec_component import TableThenSpec, parse_spec_component, WhenSpec, ThenSpec
|
49
|
-
from .utils import is_json
|
49
|
+
from .utils import is_json,get_mustrd_root
|
50
50
|
from colorama import Fore, Style
|
51
51
|
from tabulate import tabulate
|
52
52
|
from collections import defaultdict
|
@@ -93,6 +93,7 @@ class Specification:
|
|
93
93
|
given: Graph
|
94
94
|
when: WhenSpec
|
95
95
|
then: ThenSpec
|
96
|
+
spec_file_name: str = "default.mustrd.ttl"
|
96
97
|
|
97
98
|
|
98
99
|
@dataclass
|
@@ -152,6 +153,7 @@ class TripleStoreConnectionError(SpecResult):
|
|
152
153
|
@dataclass
|
153
154
|
class SpecSkipped(SpecResult):
|
154
155
|
message: str
|
156
|
+
spec_file_name: str = "default.mustrd.ttl"
|
155
157
|
|
156
158
|
|
157
159
|
@dataclass
|
@@ -176,16 +178,15 @@ class UpdateSparqlQuery(SparqlAction):
|
|
176
178
|
|
177
179
|
# https://github.com/Semantic-partners/mustrd/issues/19
|
178
180
|
|
179
|
-
def validate_specs(run_config: dict, triple_stores: List, shacl_graph: Graph, ont_graph: Graph)\
|
181
|
+
def validate_specs(run_config: dict, triple_stores: List, shacl_graph: Graph, ont_graph: Graph, file_name: str = "*")\
|
180
182
|
-> Tuple[List, Graph, List]:
|
181
183
|
spec_graph = Graph()
|
182
184
|
subject_uris = set()
|
183
185
|
focus_uris = set()
|
184
186
|
invalid_specs = []
|
185
|
-
|
186
|
-
ttl_files = list(run_config['spec_path'].glob('**/*.mustrd.ttl'))
|
187
|
+
ttl_files = list(run_config['spec_path'].glob(f'**/{file_name}.mustrd.ttl'))
|
187
188
|
ttl_files.sort()
|
188
|
-
log.info(f"Found {len(ttl_files)} mustrd.ttl files in {run_config['spec_path']}")
|
189
|
+
log.info(f"Found {len(ttl_files)} {file_name}.mustrd.ttl files in {run_config['spec_path']}")
|
189
190
|
|
190
191
|
for file in ttl_files:
|
191
192
|
error_messages = []
|
@@ -233,7 +234,7 @@ def validate_specs(run_config: dict, triple_stores: List, shacl_graph: Graph, on
|
|
233
234
|
if len(error_messages) > 0:
|
234
235
|
error_messages.sort()
|
235
236
|
error_message = "\n".join(msg for msg in error_messages)
|
236
|
-
invalid_specs += [SpecSkipped(subject_uri, triple_store["type"], error_message) for triple_store in
|
237
|
+
invalid_specs += [SpecSkipped(subject_uri, triple_store["type"], error_message, file.name) for triple_store in
|
237
238
|
triple_stores]
|
238
239
|
else:
|
239
240
|
subject_uris.add(subject_uri)
|
@@ -241,13 +242,12 @@ def validate_specs(run_config: dict, triple_stores: List, shacl_graph: Graph, on
|
|
241
242
|
this_spec_graph.parse(file)
|
242
243
|
spec_uris_in_this_file = list(this_spec_graph.subjects(RDF.type, MUST.TestSpec))
|
243
244
|
for spec in spec_uris_in_this_file:
|
244
|
-
tripleToAdd = [spec, MUST.specSourceFile, Literal(file)]
|
245
245
|
# print(f"adding {tripleToAdd}")
|
246
|
-
this_spec_graph.add(
|
246
|
+
this_spec_graph.add([spec, MUST.specSourceFile, Literal(file)])
|
247
|
+
this_spec_graph.add([spec, MUST.specFileName, Literal(file.name)])
|
247
248
|
# print(f"beforeadd: {spec_graph}" )
|
248
249
|
# print(f"beforeadd: {str(this_spec_graph.serialize())}" )
|
249
250
|
spec_graph += this_spec_graph
|
250
|
-
# print(f"afteradd: {str(spec_graph.serialize())}" )
|
251
251
|
|
252
252
|
|
253
253
|
sourceFiles = list(spec_graph.subject_objects(MUST.specSourceFile))
|
@@ -276,14 +276,14 @@ def get_specs(spec_uris: List[URIRef], spec_graph: Graph, triple_stores: List[di
|
|
276
276
|
for triple_store in triple_stores:
|
277
277
|
if "error" in triple_store:
|
278
278
|
log.error(f"{triple_store['error']}. No specs run for this triple store.")
|
279
|
-
skipped_results += [SpecSkipped(spec_uri, triple_store['type'], triple_store['error']) for spec_uri in
|
279
|
+
skipped_results += [SpecSkipped(spec_uri, triple_store['type'], triple_store['error'], get_spec_file(spec_uri, spec_graph)) for spec_uri in
|
280
280
|
spec_uris]
|
281
281
|
else:
|
282
282
|
for spec_uri in spec_uris:
|
283
283
|
try:
|
284
284
|
specs += [get_spec(spec_uri, spec_graph, run_config, triple_store)]
|
285
285
|
except (ValueError, FileNotFoundError, ConnectionError) as e:
|
286
|
-
skipped_results += [SpecSkipped(spec_uri, triple_store['type'], e)]
|
286
|
+
skipped_results += [SpecSkipped(spec_uri, triple_store['type'], e, get_spec_file(spec_uri, spec_graph))]
|
287
287
|
|
288
288
|
except (BadSyntax, FileNotFoundError) as e:
|
289
289
|
template = "An exception of type {0} occurred when trying to parse the triple store configuration file. " \
|
@@ -303,11 +303,14 @@ def run_specs(specs) -> List[SpecResult]:
|
|
303
303
|
results.append(run_spec(specification))
|
304
304
|
return results
|
305
305
|
|
306
|
+
def get_spec_file(spec_uri: URIRef, spec_graph: Graph):
|
307
|
+
test = str(spec_graph.value(subject = spec_uri, predicate = MUST.specFileName, default = "default.mustrd.ttl"))
|
308
|
+
return test
|
306
309
|
|
307
310
|
def get_spec(spec_uri: URIRef, spec_graph: Graph, run_config: dict, mustrd_triple_store: dict = None) -> Specification:
|
308
311
|
try:
|
309
312
|
if mustrd_triple_store is None:
|
310
|
-
mustrd_triple_store = {"type":
|
313
|
+
mustrd_triple_store = {"type": TRIPLESTORE.RdfLib}
|
311
314
|
components = []
|
312
315
|
for predicate in MUST.given, MUST.when, MUST.then:
|
313
316
|
components.append(parse_spec_component(subject=spec_uri,
|
@@ -316,9 +319,9 @@ def get_spec(spec_uri: URIRef, spec_graph: Graph, run_config: dict, mustrd_tripl
|
|
316
319
|
run_config=run_config,
|
317
320
|
mustrd_triple_store=mustrd_triple_store))
|
318
321
|
|
319
|
-
|
322
|
+
spec_file_name = get_spec_file(spec_uri, spec_graph)
|
320
323
|
# https://github.com/Semantic-partners/mustrd/issues/92
|
321
|
-
return Specification(spec_uri, mustrd_triple_store, components[0].value, components[1], components[2])
|
324
|
+
return Specification(spec_uri, mustrd_triple_store, components[0].value, components[1], components[2], spec_file_name)
|
322
325
|
|
323
326
|
except (ValueError, FileNotFoundError) as e:
|
324
327
|
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
|
@@ -354,7 +357,7 @@ def run_spec(spec: Specification) -> SpecResult:
|
|
354
357
|
log.debug(f"{given_as_turtle}")
|
355
358
|
upload_given(triple_store, spec.given)
|
356
359
|
else:
|
357
|
-
if triple_store['type'] ==
|
360
|
+
if triple_store['type'] == TRIPLESTORE.RdfLib:
|
358
361
|
return SpecSkipped(spec_uri, triple_store['type'], "Unable to run Inherited State tests on Rdflib")
|
359
362
|
try:
|
360
363
|
for when in spec.when:
|
@@ -386,57 +389,64 @@ def get_triple_store_graph(triple_store_graph_path: Path, secrets: str):
|
|
386
389
|
return Graph().parse(triple_store_graph_path).parse(data = secrets)
|
387
390
|
else:
|
388
391
|
secret_path = triple_store_graph_path.parent / Path(triple_store_graph_path.stem + "_secrets" + triple_store_graph_path.suffix)
|
389
|
-
return Graph().parse(triple_store_graph_path).parse(
|
392
|
+
return Graph().parse(triple_store_graph_path).parse(secret_path)
|
390
393
|
|
391
394
|
|
392
395
|
def get_triple_stores(triple_store_graph: Graph) -> list[dict]:
|
393
396
|
triple_stores = []
|
397
|
+
shacl_graph = Graph().parse(Path(os.path.join(get_mustrd_root(), "model/triplestoreshapes.ttl")))
|
398
|
+
ont_graph = Graph().parse(Path(os.path.join(get_mustrd_root(), "model/triplestoreOntology.ttl")))
|
399
|
+
conforms, results_graph, results_text = validate(
|
400
|
+
data_graph= triple_store_graph,
|
401
|
+
shacl_graph = shacl_graph,
|
402
|
+
ont_graph = ont_graph,
|
403
|
+
advanced= True,
|
404
|
+
inference= 'none'
|
405
|
+
)
|
406
|
+
if not conforms:
|
407
|
+
raise ValueError(f"Triple store configuration not conform to the shapes. SHACL report: {results_text}", results_graph)
|
394
408
|
for triple_store_config, rdf_type, triple_store_type in triple_store_graph.triples((None, RDF.type, None)):
|
395
409
|
triple_store = {}
|
396
|
-
|
397
|
-
|
398
|
-
triple_store["type"] = MUST.RdfLib
|
410
|
+
triple_store["type"] = triple_store_type
|
411
|
+
triple_store["uri"] = triple_store_config
|
399
412
|
# Anzo graph via anzo
|
400
|
-
|
401
|
-
triple_store["
|
402
|
-
triple_store["
|
403
|
-
triple_store["port"] = triple_store_graph.value(subject=triple_store_config, predicate=MUST.port)
|
413
|
+
if triple_store_type == TRIPLESTORE.Anzo:
|
414
|
+
triple_store["url"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.url)
|
415
|
+
triple_store["port"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.port)
|
404
416
|
try:
|
405
|
-
triple_store["username"] = str(triple_store_graph.value(subject=triple_store_config, predicate=
|
406
|
-
triple_store["password"] = str(triple_store_graph.value(subject=triple_store_config, predicate=
|
417
|
+
triple_store["username"] = str(triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.username))
|
418
|
+
triple_store["password"] = str(triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.password))
|
407
419
|
except (FileNotFoundError, ValueError) as e:
|
408
420
|
triple_store["error"] = e
|
409
|
-
triple_store["gqe_uri"] = triple_store_graph.value(subject=triple_store_config, predicate=
|
421
|
+
triple_store["gqe_uri"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.gqeURI)
|
410
422
|
triple_store["input_graph"] = triple_store_graph.value(subject=triple_store_config,
|
411
|
-
predicate=
|
423
|
+
predicate=TRIPLESTORE.inputGraph)
|
412
424
|
triple_store["output_graph"] = triple_store_graph.value(subject=triple_store_config,
|
413
|
-
predicate=
|
425
|
+
predicate=TRIPLESTORE.outputGraph)
|
414
426
|
try:
|
415
427
|
check_triple_store_params(triple_store, ["url", "port", "username", "password", "input_graph"])
|
416
428
|
except ValueError as e:
|
417
429
|
triple_store["error"] = e
|
418
430
|
# GraphDB
|
419
|
-
elif triple_store_type ==
|
420
|
-
triple_store["
|
421
|
-
triple_store["
|
422
|
-
triple_store["port"] = triple_store_graph.value(subject=triple_store_config, predicate=MUST.port)
|
431
|
+
elif triple_store_type == TRIPLESTORE.GraphDb:
|
432
|
+
triple_store["url"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.url)
|
433
|
+
triple_store["port"] = triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.port)
|
423
434
|
try:
|
424
|
-
triple_store["username"] = str(triple_store_graph.value(subject=triple_store_config, predicate=
|
425
|
-
triple_store["password"] = str(triple_store_graph.value(subject=triple_store_config, predicate=
|
435
|
+
triple_store["username"] = str(triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.username))
|
436
|
+
triple_store["password"] = str(triple_store_graph.value(subject=triple_store_config, predicate=TRIPLESTORE.password))
|
426
437
|
except (FileNotFoundError, ValueError) as e:
|
427
438
|
log.error(f"Credential retrieval failed {e}")
|
428
439
|
triple_store["error"] = e
|
429
440
|
triple_store["repository"] = triple_store_graph.value(subject=triple_store_config,
|
430
|
-
predicate=
|
441
|
+
predicate=TRIPLESTORE.repository)
|
431
442
|
triple_store["input_graph"] = triple_store_graph.value(subject=triple_store_config,
|
432
|
-
predicate=
|
443
|
+
predicate=TRIPLESTORE.inputGraph)
|
433
444
|
|
434
445
|
try:
|
435
446
|
check_triple_store_params(triple_store, ["url", "port", "repository"])
|
436
447
|
except ValueError as e:
|
437
448
|
triple_store["error"] = e
|
438
|
-
|
439
|
-
triple_store["type"] = triple_store_type
|
449
|
+
elif triple_store_type != TRIPLESTORE.RdfLib:
|
440
450
|
triple_store["error"] = f"Triple store not implemented: {triple_store_type}"
|
441
451
|
|
442
452
|
triple_stores.append(triple_store)
|