mustrd 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

mustrd/mustrdAnzo.py

@@ -51,13 +51,16 @@ def query_with_bindings(bindings: dict, when: str) -> str:
     split_query = when.lower().split("where {", 1)
     return f"{split_query[0].strip()} WHERE {{ {values} {split_query[1].strip()}"
 
-def execute_select (triple_store: dict,  when: str, bindings: dict = None) -> str:
+
+def execute_select(triple_store: dict,  when: str, bindings: dict = None) -> str:
     try:
         if bindings:
             when = query_with_bindings(bindings, when)
         when = when.replace("${fromSources}", f"FROM <{triple_store['input_graph']}>\nFROM <{triple_store['output_graph']}>").replace(
         "${targetGraph}", f"<{triple_store['output_graph']}>")
         data = {'datasourceURI': triple_store['gqe_uri'], 'query': when,
+                'default-graph-uri': triple_store['input_graph'],
+                'named-graph-uri': triple_store['input_graph'],
                  'skipCache': 'true'}
         url = f"https://{triple_store['url']}:{triple_store['port']}/sparql?format=application/sparql-results+json"
         return manage_anzo_response(requests.post(url=url,
@@ -67,6 +70,7 @@ def execute_select (triple_store: dict,  when: str, bindings: dict = None) -> st
     except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout):
         raise
 
+
 def execute_update(triple_store: dict, when: str, bindings: dict = None) -> Graph:
     logging.debug(f"updating in anzo! {triple_store=} {when=}")
     input_graph = triple_store['input_graph']
@@ -75,8 +79,11 @@ def execute_update(triple_store: dict, when: str, bindings: dict = None) -> Grap
     substituted_query = when.replace("${usingSources}", f"USING <{triple_store['input_graph']}> \nUSING <{triple_store['output_graph']}>").replace(
         "${targetGraph}", f"<{output_graph}>")
    
-    data = {'datasourceURI': triple_store['gqe_uri'], 'update': substituted_query,
-                'default-graph-uri': input_graph, 'skipCache': 'true'}
+    data = {'datasourceURI': triple_store['gqe_uri'],
+            'update': substituted_query,
+            'using-graph-uri': [output_graph, input_graph],
+            'using-named-graph-uri': [output_graph, input_graph],
+            'skipCache': 'true'}
     url = f"https://{triple_store['url']}:{triple_store['port']}/sparql?format=ttl"
     response = manage_anzo_response(requests.post(url=url,
             auth=(triple_store['username'],
@@ -85,7 +92,9 @@ def execute_update(triple_store: dict, when: str, bindings: dict = None) -> Grap
             verify=False))
     logging.debug(f'response {response}')
     check_data = {'datasourceURI': triple_store['gqe_uri'], 'query': "construct {?s ?p ?o} { ?s ?p ?o }",
-                'default-graph-uri': output_graph, 'skipCache': 'true'}
+                'default-graph-uri': output_graph,
+                'named-graph-uri': output_graph,
+                  'skipCache': 'true'}
     everything_response = manage_anzo_response(requests.post(url=url,
             auth=(triple_store['username'],
                 triple_store['password']),
@@ -102,7 +111,9 @@ def execute_construct(triple_store: dict, when: str, bindings: dict = None) -> G
         if bindings:
             when = query_with_bindings(bindings, when)
         data = {'datasourceURI': triple_store['gqe_uri'], 'query': when,
-                'default-graph-uri': triple_store['input_graph'], 'skipCache': 'true'}
+                'default-graph-uri': triple_store['input_graph'],
+                'named-graph-uri': triple_store['input_graph'],
+                'skipCache': 'true'}
         url = f"https://{triple_store['url']}:{triple_store['port']}/sparql?format=ttl"
         response = requests.post(url=url,
             auth=(triple_store['username'],
@@ -110,7 +121,9 @@ def execute_construct(triple_store: dict, when: str, bindings: dict = None) -> G
             data=data,
             verify=False)
         logging.debug(f'response {response}')
-        return Graph().parse(data=manage_anzo_response(response))
+        g = Graph().parse(data=manage_anzo_response(response))
+        logging.debug(f"Actual Result = {g.serialize(format='ttl')}")
+        return g
     except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout) as e:
         logging.error(f'response {e}')
         raise
@@ -119,7 +132,7 @@ def execute_construct(triple_store: dict, when: str, bindings: dict = None) -> G
 # Get Given or then from the content of a graphmart
 def get_spec_component_from_graphmart(triple_store: dict, graphmart: URIRef, layer: URIRef = None) -> ConjunctiveGraph:
     try:
-        anzo_client = AnzoClient(triple_store['url'], triple_store['port'], 
+        anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
                                  username=triple_store['username'],
                                  password=triple_store['password'])
         return anzo_client.query_graphmart(graphmart=graphmart,
@@ -140,10 +153,10 @@ def get_query_from_querybuilder(triple_store: dict, folder_name: Literal, query_
             ?queryFolder a <http://www.cambridgesemantics.com/ontologies/QueryPlayground#QueryFolder>;
                         <http://purl.org/dc/elements/1.1/title> "{folder_name}"
     }}"""
-    anzo_client = AnzoClient(triple_store['url'], triple_store['port'], 
+    anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
                              username=triple_store['username'],
                              password=triple_store['password'])
-    
+
     result = anzo_client.query_journal(query_string=query).as_table_results().as_record_dictionaries()
     if len(result) == 0:
         raise FileNotFoundError(f"Query {query_name} not found in folder {folder_name}")
@@ -158,7 +171,7 @@ def get_query_from_step(triple_store: dict, query_step_uri: URIRef) -> str:
                      <http://cambridgesemantics.com/ontologies/Graphmarts#transformQuery> ?query
     }}
     # """
-    anzo_client = AnzoClient(triple_store['url'], triple_store['port'], 
+    anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
                              username=triple_store['username'],
                              password=triple_store['password'])
     record_dictionaries = anzo_client.query_journal(query_string=query).as_table_results().as_record_dictionaries()
@@ -175,7 +188,7 @@ def get_queries_from_templated_step(triple_store: dict, query_step_uri: URIRef)
 					    <http://cambridgesemantics.com/ontologies/Graphmarts#template> ?query_template .
     }}
     """
-    anzo_client = AnzoClient(triple_store['url'], triple_store['port'], 
+    anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
                              username=triple_store['username'],
                              password=triple_store['password'])
     record_dictionaries = anzo_client.query_journal(query_string=query).as_table_results().as_record_dictionaries()
@@ -189,7 +202,7 @@ SELECT ?query ?param_query ?query_template
   {{ <{graphmart_layer_uri}> graphmarts:step ?step .
   ?step         anzo:index ?index ;
                 anzo:orderedValue ?query_step .
-  ?query_step graphmarts:enabled true ; 
+  ?query_step graphmarts:enabled true ;
   OPTIONAL {{  ?query_step
    				graphmarts:parametersTemplate ?param_query ;
            		graphmarts:template ?query_template ;
@@ -199,7 +212,7 @@ SELECT ?query ?param_query ?query_template
       . }}
   }}
   ORDER BY ?index"""
-    anzo_client = AnzoClient(triple_store['url'], triple_store['port'], 
+    anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
                              username=triple_store['username'],
                              password=triple_store['password'])
     return anzo_client.query_journal(query_string=query).as_table_results().as_record_dictionaries()
@@ -215,7 +228,10 @@ def upload_given(triple_store: dict, given: Graph):
         clear_graph(triple_store, output_graph)
         serialized_given = given.serialize(format="nt")
         insert_query = f"INSERT DATA {{graph <{triple_store['input_graph']}>{{{serialized_given}}}}}"
-        data = {'datasourceURI': triple_store['gqe_uri'], 'update': insert_query}
+        data = {'datasourceURI': triple_store['gqe_uri'],
+                'update': insert_query,
+                'using-graph-uri': input_graph,
+                'using-named-graph-uri': input_graph}
         response = requests.post(url=f"https://{triple_store['url']}:{triple_store['port']}/sparql",
                                     auth=(triple_store['username'], triple_store['password']), data=data, verify=False)
         manage_anzo_response(response)
@@ -233,4 +249,3 @@ def clear_graph(triple_store: dict, graph_uri: str):
         manage_anzo_response(response)
     except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout):
         raise
-

mustrd/mustrdTestPlugin.py

@@ -85,37 +85,44 @@ def pytest_configure(config) -> None:
     if config.getoption("mustrd"):
         test_configs = parse_config(config.getoption("configpath"))
         config.pluginmanager.register(MustrdTestPlugin(config.getoption("mdpath"),
-                                                    test_configs, config.getoption("secrets")))
-    
+                                                       test_configs, config.getoption("secrets")))
+
+
 def parse_config(config_path):
     test_configs = []
     config_graph = Graph().parse(config_path)
     shacl_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdTestShapes.ttl")))
     ont_graph = Graph().parse(Path(os.path.join(mustrd_root, "model/mustrdTestOntology.ttl")))
     conforms, results_graph, results_text = validate(
-            data_graph= config_graph,
-            shacl_graph = shacl_graph,
-            ont_graph  = ont_graph,
-            advanced= True,
-            inference= 'none'
+            data_graph=config_graph,
+            shacl_graph=shacl_graph,
+            ont_graph=ont_graph,
+            advanced=True,
+            inference='none'
         )
     if not conforms:
-        raise ValueError(f"Mustrd test configuration not conform to the shapes. SHACL report: {results_text}", results_graph)
-        
+        raise ValueError(f"Mustrd test configuration not conform to the shapes. SHACL report: {results_text}",
+                         results_graph)
+
     for test_config_subject in config_graph.subjects(predicate=RDF.type, object=MUSTRDTEST.MustrdTest):
         spec_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasSpecPath, str)
         data_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasDataPath, str)
         triplestore_spec_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.triplestoreSpecPath, str)
         pytest_path = get_config_param(config_graph, test_config_subject, MUSTRDTEST.hasPytestPath, str)
         filter_on_tripleStore = list(config_graph.objects(subject=test_config_subject,
-                                                        predicate=MUSTRDTEST.filterOnTripleStore))
+                                                          predicate=MUSTRDTEST.filterOnTripleStore))
+
+        # Root path is the mustrd test config path
+        root_path = Path(config_path).parent
+        spec_path = root_path / Path(spec_path) if spec_path else None
+        data_path = root_path / Path(data_path) if data_path else None
+        triplestore_spec_path = root_path / Path(triplestore_spec_path) if triplestore_spec_path else None
 
         test_configs.append(TestConfig(spec_path=spec_path, data_path=data_path,
-                                                triplestore_spec_path=triplestore_spec_path,
-                                                pytest_path = pytest_path,
-                                                filter_on_tripleStore=filter_on_tripleStore))
+                                       triplestore_spec_path=triplestore_spec_path,
+                                       pytest_path=pytest_path,
+                                       filter_on_tripleStore=filter_on_tripleStore))
     return test_configs
-    
 
 
 def get_config_param(config_graph, config_subject, config_param, convert_function):
@@ -125,18 +132,19 @@ def get_config_param(config_graph, config_subject, config_param, convert_functio
 
 @dataclass
 class TestConfig:
-    spec_path: str
-    data_path: str
-    triplestore_spec_path: str
+    spec_path: Path
+    data_path: Path
+    triplestore_spec_path: Path
     pytest_path: str
     filter_on_tripleStore: str = None
-        
+
 
 @dataclass
 class TestParamWrapper:
     test_config: TestConfig
     unit_test: Union[Specification, SpecSkipped]
 
+
 class MustrdTestPlugin:
     md_path: str
     test_configs: list
@@ -149,7 +157,7 @@ class MustrdTestPlugin:
         self.test_configs = test_configs
         self.secrets = secrets
         self.items = []
-    
+
     @pytest.hookimpl(tryfirst=True)
     def pytest_collection(self, session):
         self.unit_tests = []
@@ -157,7 +165,7 @@ class MustrdTestPlugin:
         if len(args) > 0:
             file_name = self.get_file_name_from_arg(args[0])
             # Filter test to collect only specified path
-            config_to_collect = list(filter(lambda config: 
+            config_to_collect = list(filter(lambda config:
                                             # Case we want to collect everything
                                             MUSTRD_PYTEST_PATH not in args[0]
                                             # Case we want to collect a test or sub test
@@ -165,30 +173,33 @@ class MustrdTestPlugin:
                                             # Case we want to collect a whole test folder
                                             or args[0].replace(f"./{MUSTRD_PYTEST_PATH}", "") in config.pytest_path,
                                             self.test_configs))
-                
-            # Redirect everything to test_mustrd.py, no need to filter on specified test: Only specified test will be collected anyway
+
+            # Redirect everything to test_mustrd.py,
+            # no need to filter on specified test: Only specified test will be collected anyway
             session.config.args[0] = os.path.join(mustrd_root, "test/test_mustrd.py")
         # Collecting only relevant tests
-        
-        for one_test_config in config_to_collect: 
+
+        for one_test_config in config_to_collect:
             triple_stores = self.get_triple_stores_from_file(one_test_config)
 
             if one_test_config.filter_on_tripleStore and not triple_stores:
-                self.unit_tests.extend(list(map(lambda triple_store:
-                                TestParamWrapper(test_config = one_test_config, unit_test=SpecSkipped(MUST.TestSpec, triple_store, "No triplestore found")),
-                                one_test_config.filter_on_tripleStore)))
+                self.unit_tests.extend(list(map(
+                    lambda triple_store:
+                        TestParamWrapper(test_config=one_test_config,
+                                         unit_test=SpecSkipped(MUST.TestSpec, triple_store, "No triplestore found")),
+                        one_test_config.filter_on_tripleStore)))
             else:
-                specs = self.generate_tests_for_config({"spec_path": Path(one_test_config.spec_path),
-                                                            "data_path": Path(one_test_config.data_path)},
-                                                            triple_stores, file_name)
-                self.unit_tests.extend(list(map(lambda spec: TestParamWrapper(test_config = one_test_config, unit_test=spec),specs)))
-        
+                specs = self.generate_tests_for_config({"spec_path": one_test_config.spec_path,
+                                                        "data_path": one_test_config.data_path},
+                                                       triple_stores, file_name)
+                self.unit_tests.extend(list(map(
+                    lambda spec: TestParamWrapper(test_config=one_test_config, unit_test=spec), specs)))
+
     def get_file_name_from_arg(self, arg):
         if arg and len(arg) > 0 and "[" in arg and ".mustrd.ttl@" in arg:
             return arg[arg.index("[") + 1: arg.index(".mustrd.ttl@")]
         return None
-        
-        
+
     @pytest.hookimpl(hookwrapper=True)
     def pytest_pycollect_makeitem(self, collector, name, obj):
         report = yield
@@ -196,18 +207,17 @@ class MustrdTestPlugin:
             items = report.get_result()
             new_results = []
             for item in items:
-                virtual_path =  MUSTRD_PYTEST_PATH + (item.callspec.params["unit_tests"].test_config.pytest_path or "default")
+                virtual_path = MUSTRD_PYTEST_PATH + (item.callspec.params["unit_tests"].test_config.pytest_path or "default")
                 item.fspath = Path(virtual_path)
                 item._nodeid = virtual_path + "::" + item.name
                 self.items.append(item)
                 new_results.append(item)
             return new_results
-        
 
     # Hook called at collection time: reads the configuration of the tests, and generate pytests from it
     def pytest_generate_tests(self, metafunc):
         if len(metafunc.fixturenames) > 0:
-            if metafunc.function.__name__  == "test_unit":
+            if metafunc.function.__name__ == "test_unit":
                 # Create the test in itself
                 if self.unit_tests:
                     metafunc.parametrize(metafunc.fixturenames[0], self.unit_tests,
@@ -217,9 +227,6 @@ class MustrdTestPlugin:
                 metafunc.parametrize(metafunc.fixturenames[0],
                                      [SpecSkipped(MUST.TestSpec, None, "No triplestore found")],
                                      ids=lambda x: "No configuration found for this test")
-    
-                
-            
 
     # Generate test for each triple store available
     def generate_tests_for_config(self, config, triple_stores, file_name):
@@ -250,7 +257,7 @@ class MustrdTestPlugin:
     def get_triple_stores_from_file(self, test_config):
         if test_config.triplestore_spec_path:
             try:
-                triple_stores = get_triple_stores(get_triple_store_graph(Path(test_config.triplestore_spec_path),
+                triple_stores = get_triple_stores(get_triple_store_graph(test_config.triplestore_spec_path,
                                                                          self.secrets))
             except Exception as e:
                 print(f"""Triplestore configuration parsing failed {test_config.triplestore_spec_path}.

mustrd/namespace.py

@@ -25,10 +25,11 @@ SOFTWARE.
 from rdflib import URIRef
 from rdflib.namespace import DefinedNamespace, Namespace
 
+
 # Namespace for the test specifications
 class MUST(DefinedNamespace):
     _NS = Namespace("https://mustrd.com/model/")
-    
+
     # Specification classes
     TestSpec: URIRef
     SelectSparql: URIRef
@@ -55,7 +56,7 @@ class MUST(DefinedNamespace):
     hasBinding: URIRef
     variable: URIRef
     boundValue: URIRef
-    focus:URIRef
+    focus: URIRef
 
     # Specification data sources
     TableDataset: URIRef
@@ -85,16 +86,16 @@ class MUST(DefinedNamespace):
     AnzoGraphmartQueryDrivenTemplatedStepSparqlSource: URIRef
     anzoQueryStep: URIRef
     anzoGraphmartLayer: URIRef
-    
+
     graphmart: URIRef
     layer: URIRef
 
-    
     # FIXME: There is nothing to do that by default?
     @classmethod
     def get_local_name(cls, uri: URIRef):
         return str(uri).split(cls._NS)[1]
 
+
 # Namespace for triplestores
 class TRIPLESTORE(DefinedNamespace):
     _NS = Namespace("https://mustrd.com/triplestore/")
@@ -103,16 +104,17 @@ class TRIPLESTORE(DefinedNamespace):
     Anzo: URIRef
     ExternalTripleStore: URIRef
     InternalTripleStore: URIRef
-    
+
     gqeURI: URIRef
-    inputGraph: URIRef 
-    outputGraph: URIRef # anzo specials?     # Triple store config parameters
+    inputGraph: URIRef
+    outputGraph: URIRef  # anzo specials?     # Triple store config parameters
     url: URIRef
     port: URIRef
     username: URIRef
     password: URIRef
     repository: URIRef
 
+
 # namespace for pytest_mustrd config
 class MUSTRDTEST(DefinedNamespace):
     _NS = Namespace("https://mustrd.com/mustrdTest/")
@@ -122,4 +124,3 @@ class MUSTRDTEST(DefinedNamespace):
     triplestoreSpecPath: URIRef
     hasPytestPath: URIRef
     filterOnTripleStore: URIRef
-    

mustrd/run.py

@@ -31,7 +31,6 @@ from .mustrd import get_triple_store_graph, run_specs, get_triple_stores, review
 from pathlib import Path
 from .namespace import TRIPLESTORE
 from .utils import get_project_root
-import logging 
 log = logger_setup.setup_logger(__name__)
 
 
@@ -50,7 +49,7 @@ def parse_args() -> argparse.Namespace:
 
 # https://github.com/Semantic-partners/mustrd/issues/108
 def main(argv):
-    #given_path = when_path = then_path = None
+    # Given_path = when_path = then_path = None
     project_root = get_project_root()
     run_config = {}
     args = parse_args()
@@ -59,7 +58,7 @@ def main(argv):
 
     verbose = args.verbose
     if verbose:
-        log.info(f"Verbose set")
+        log.info("Verbose set")
         run_config["verbose"] = True
 
     if args.config:
@@ -67,7 +66,7 @@ def main(argv):
         log.info(f"Path for triple store configuration is {triplestore_spec_path}")
         triple_stores = get_triple_stores(get_triple_store_graph(triplestore_spec_path))
     else:
-        log.info(f"No triple store configuration added, running default configuration")
+        log.info("No triple store configuration added, running default configuration")
         triple_stores = [{'type': TRIPLESTORE.RdfLib}]
     log.info("Triple Stores: " + str(triple_stores))
     if args.data: