mustrd 0.2.6.1__py3-none-any.whl → 0.2.6.3__py3-none-any.whl

mustrd/mustrdAnzo.py

@@ -22,51 +22,24 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 """
 
-import requests
-from pyanzo import AnzoClient
 from rdflib import Graph, ConjunctiveGraph, Literal, URIRef
-from requests import ConnectTimeout, Response, HTTPError, RequestException, ConnectionError
-from bs4 import BeautifulSoup
+from requests import ConnectTimeout, HTTPError, ConnectionError
 import logging
-from .namespace import MUST
-
-
-# https://github.com/Semantic-partners/mustrd/issues/73
-def manage_anzo_response(response: Response) -> str:
-    content_string = response.content.decode("utf-8")
-    if response.status_code == 200:
-        return content_string
-    elif response.status_code == 403:
-        html = BeautifulSoup(content_string, 'html.parser')
-        title_tag = html.title.string
-        raise HTTPError(f"Anzo authentication error, status code: {response.status_code}, content: {title_tag}")
-    else:
-        raise RequestException(f"Anzo error, status code: {response.status_code}, content: {content_string}")
-
-
-def query_with_bindings(bindings: dict, when: str) -> str:
-    values = ""
-    for key, value in bindings.items():
-        values += f"VALUES ?{key} {{{value.n3()}}} "
-    split_query = when.lower().split("where {", 1)
-    return f"{split_query[0].strip()} WHERE {{ {values} {split_query[1].strip()}"
+from mustrd.anzo_utils import query_azg, query_graphmart
+from mustrd.anzo_utils import query_configuration, json_to_dictlist, ttl_to_graph
 
 
 def execute_select(triple_store: dict,  when: str, bindings: dict = None) -> str:
     try:
         if bindings:
             when = query_with_bindings(bindings, when)
-        when = when.replace("${fromSources}", f"FROM <{triple_store['input_graph']}>\nFROM <{triple_store['output_graph']}>").replace(
-        "${targetGraph}", f"<{triple_store['output_graph']}>")
-        data = {'datasourceURI': triple_store['gqe_uri'], 'query': when,
-                'default-graph-uri': triple_store['input_graph'],
-                'named-graph-uri': triple_store['input_graph'],
-                 'skipCache': 'true'}
-        url = f"https://{triple_store['url']}:{triple_store['port']}/sparql?format=application/sparql-results+json"
-        return manage_anzo_response(requests.post(url=url,
-                                                  auth=(triple_store['username'], triple_store['password']),
-                                                  data=data,
-                                                  verify=False))
+        # FIXME: why do we have those tokens in a select query? in particular ${targetGraph}?
+        # FIXME: why do we also query the output graph?
+        when = when.replace("${fromSources}",
+                            f"FROM <{triple_store['input_graph']}>\nFROM <{triple_store['output_graph']}>").replace(
+                                "${targetGraph}", f"<{triple_store['output_graph']}>")
+        # TODO: manage results here
+        return query_azg(anzo_config=triple_store, query=when)
     except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout):
         raise
 
@@ -76,32 +49,20 @@ def execute_update(triple_store: dict, when: str, bindings: dict = None) -> Grap
     input_graph = triple_store['input_graph']
     output_graph = triple_store['output_graph']
 
-    substituted_query = when.replace("${usingSources}", f"USING <{triple_store['input_graph']}> \nUSING <{triple_store['output_graph']}>").replace(
-        "${targetGraph}", f"<{output_graph}>")
-   
-    data = {'datasourceURI': triple_store['gqe_uri'],
-            'update': substituted_query,
-            'using-graph-uri': [output_graph, input_graph],
-            'using-named-graph-uri': [output_graph, input_graph],
-            'skipCache': 'true'}
-    url = f"https://{triple_store['url']}:{triple_store['port']}/sparql?format=ttl"
-    response = manage_anzo_response(requests.post(url=url,
-            auth=(triple_store['username'],
-                triple_store['password']),
-            data=data,
-            verify=False))
+    # FIXME: that will only work with steps.
+    # We could replace USING clauses with using-graph-uri parameter
+    # But there is no parameter for default insert graphs.
+    substituted_query = when.replace("${usingSources}",
+                                     f"""USING <{input_graph}>
+USING <{triple_store['output_graph']}>""").replace(
+                                         "${targetGraph}", f"<{output_graph}>")
+
+    response = query_azg(anzo_config=triple_store, query=substituted_query, is_update=True,
+                         data_layers=[input_graph, output_graph], format="ttl")
     logging.debug(f'response {response}')
-    check_data = {'datasourceURI': triple_store['gqe_uri'], 'query': "construct {?s ?p ?o} { ?s ?p ?o }",
-                'default-graph-uri': output_graph,
-                'named-graph-uri': output_graph,
-                  'skipCache': 'true'}
-    everything_response = manage_anzo_response(requests.post(url=url,
-            auth=(triple_store['username'],
-                triple_store['password']),
-            data=check_data,
-            verify=False))
-    # todo deal with error responses
-    new_graph = Graph().parse(data=everything_response)
+    # TODO: deal with error responses
+    new_graph = ttl_to_graph(query_azg(anzo_config=triple_store, query="construct {?s ?p ?o} { ?s ?p ?o }",
+                                       format="ttl", data_layers=output_graph))
     logging.debug(f"new_graph={new_graph.serialize(format='ttl')}")
     return new_graph
 
@@ -110,35 +71,27 @@ def execute_construct(triple_store: dict, when: str, bindings: dict = None) -> G
     try:
         if bindings:
             when = query_with_bindings(bindings, when)
-        data = {'datasourceURI': triple_store['gqe_uri'], 'query': when,
-                'default-graph-uri': triple_store['input_graph'],
-                'named-graph-uri': triple_store['input_graph'],
-                'skipCache': 'true'}
-        url = f"https://{triple_store['url']}:{triple_store['port']}/sparql?format=ttl"
-        response = requests.post(url=url,
-            auth=(triple_store['username'],
-                triple_store['password']),
-            data=data,
-            verify=False)
-        logging.debug(f'response {response}')
-        g = Graph().parse(data=manage_anzo_response(response))
-        logging.debug(f"Actual Result = {g.serialize(format='ttl')}")
-        return g
+        return ttl_to_graph(query_azg(anzo_config=triple_store, query=when, format="ttl",
+                                      data_layers=triple_store['input_graph']))
     except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout) as e:
         logging.error(f'response {e}')
         raise
 
 
+def query_with_bindings(bindings: dict, when: str) -> str:
+    values = ""
+    for key, value in bindings.items():
+        values += f"VALUES ?{key} {{{value.n3()}}} "
+    split_query = when.lower().split("where {", 1)
+    return f"{split_query[0].strip()} WHERE {{ {values} {split_query[1].strip()}"
+
+
 # Get Given or then from the content of a graphmart
 def get_spec_component_from_graphmart(triple_store: dict, graphmart: URIRef, layer: URIRef = None) -> ConjunctiveGraph:
     try:
-        anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
-                                 username=triple_store['username'],
-                                 password=triple_store['password'])
-        return anzo_client.query_graphmart(graphmart=graphmart,
-                                           data_layers=layer,
-                                           query_string="CONSTRUCT {?s ?p ?o} WHERE {?s ?p ?o}",
-                                           skip_cache=True).as_quad_store().as_rdflib_graph()
+        return ttl_to_graph(query_graphmart(anzo_config=triple_store, graphmart=graphmart,
+                                            query="CONSTRUCT {?s ?p ?o} WHERE {?s ?p ?o}",
+                                            data_layers=layer, format="ttl"))
     except RuntimeError as e:
         raise ConnectionError(f"Anzo connection error, {e}")
 
@@ -153,46 +106,33 @@ def get_query_from_querybuilder(triple_store: dict, folder_name: Literal, query_
             ?queryFolder a <http://www.cambridgesemantics.com/ontologies/QueryPlayground#QueryFolder>;
                         <http://purl.org/dc/elements/1.1/title> "{folder_name}"
     }}"""
-    anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
-                             username=triple_store['username'],
-                             password=triple_store['password'])
-
-    result = anzo_client.query_journal(query_string=query).as_table_results().as_record_dictionaries()
+    result = json_to_dictlist(query_configuration(
+        anzo_config=triple_store, query=query))
     if len(result) == 0:
-        raise FileNotFoundError(f"Query {query_name} not found in folder {folder_name}")
+        raise FileNotFoundError(
+            f"Query {query_name} not found in folder {folder_name}")
     return result[0].get("query")
 
 
 # https://github.com/Semantic-partners/mustrd/issues/102
 def get_query_from_step(triple_store: dict, query_step_uri: URIRef) -> str:
-    query = f"""SELECT ?stepUri ?query WHERE {{
+    query = f"""SELECT ?query WHERE {{
         BIND(<{query_step_uri}> as ?stepUri)
             ?stepUri a <http://cambridgesemantics.com/ontologies/Graphmarts#Step>;
                      <http://cambridgesemantics.com/ontologies/Graphmarts#transformQuery> ?query
-    }}
-    # """
-    anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
-                             username=triple_store['username'],
-                             password=triple_store['password'])
-    record_dictionaries = anzo_client.query_journal(query_string=query).as_table_results().as_record_dictionaries()
+    }}"""
+    return json_to_dictlist(query_configuration(anzo_config=triple_store, query=query))[0]['query']
 
-    return record_dictionaries[0].get(
-        "query")
 
 def get_queries_from_templated_step(triple_store: dict, query_step_uri: URIRef) -> dict:
-
-    query = f"""SELECT ?stepUri ?param_query ?query_template WHERE {{
+    query = f"""SELECT ?param_query ?query_template WHERE {{
         BIND(<{query_step_uri}> as ?stepUri)
             ?stepUri    a <http://cambridgesemantics.com/ontologies/Graphmarts#Step> ;
-   					    <http://cambridgesemantics.com/ontologies/Graphmarts#parametersTemplate> ?param_query ;
-					    <http://cambridgesemantics.com/ontologies/Graphmarts#template> ?query_template .
+                        <http://cambridgesemantics.com/ontologies/Graphmarts#parametersTemplate> ?param_query ;
+                        <http://cambridgesemantics.com/ontologies/Graphmarts#template> ?query_template .
     }}
     """
-    anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
-                             username=triple_store['username'],
-                             password=triple_store['password'])
-    record_dictionaries = anzo_client.query_journal(query_string=query).as_table_results().as_record_dictionaries()
-    return record_dictionaries[0]
+    return json_to_dictlist(query_configuration(anzo_config=triple_store, query=query))[0]
 
 
 def get_queries_for_layer(triple_store: dict, graphmart_layer_uri: URIRef):
@@ -204,37 +144,26 @@ SELECT ?query ?param_query ?query_template
                 anzo:orderedValue ?query_step .
   ?query_step graphmarts:enabled true ;
   OPTIONAL {{  ?query_step
-   				graphmarts:parametersTemplate ?param_query ;
-           		graphmarts:template ?query_template ;
+                graphmarts:parametersTemplate ?param_query ;
+                graphmarts:template ?query_template ;
       . }}
   OPTIONAL {{  ?query_step
-   				graphmarts:transformQuery ?query ;
+                graphmarts:transformQuery ?query ;
       . }}
   }}
   ORDER BY ?index"""
-    anzo_client = AnzoClient(triple_store['url'], triple_store['port'],
-                             username=triple_store['username'],
-                             password=triple_store['password'])
-    return anzo_client.query_journal(query_string=query).as_table_results().as_record_dictionaries()
+    return json_to_dictlist(query_configuration(anzo_config=triple_store, query=query))
 
 
 def upload_given(triple_store: dict, given: Graph):
     logging.debug(f"upload_given {triple_store} {given}")
 
     try:
-        input_graph = triple_store['input_graph']
-        output_graph = triple_store['output_graph']
-        clear_graph(triple_store, input_graph)
-        clear_graph(triple_store, output_graph)
+        clear_graph(triple_store, triple_store['input_graph'])
+        clear_graph(triple_store, triple_store['output_graph'])
         serialized_given = given.serialize(format="nt")
         insert_query = f"INSERT DATA {{graph <{triple_store['input_graph']}>{{{serialized_given}}}}}"
-        data = {'datasourceURI': triple_store['gqe_uri'],
-                'update': insert_query,
-                'using-graph-uri': input_graph,
-                'using-named-graph-uri': input_graph}
-        response = requests.post(url=f"https://{triple_store['url']}:{triple_store['port']}/sparql",
-                                    auth=(triple_store['username'], triple_store['password']), data=data, verify=False)
-        manage_anzo_response(response)
+        query_azg(anzo_config=triple_store, query=insert_query, is_update=True)
     except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout):
         raise
 
@@ -242,10 +171,6 @@ def upload_given(triple_store: dict, given: Graph):
 def clear_graph(triple_store: dict, graph_uri: str):
     try:
         clear_query = f"CLEAR GRAPH <{graph_uri}>"
-        data = {'datasourceURI': triple_store['gqe_uri'], 'update': clear_query}
-        url = f"https://{triple_store['url']}:{triple_store['port']}/sparql"
-        response = requests.post(url=url,
-                                 auth=(triple_store['username'], triple_store['password']), data=data, verify=False)
-        manage_anzo_response(response)
+        query_azg(anzo_config=triple_store, query=clear_query, is_update=True)
     except (ConnectionError, TimeoutError, HTTPError, ConnectTimeout):
         raise

mustrd/mustrdGraphDb.py

@@ -49,7 +49,7 @@ def upload_given(triple_store: dict, given: Graph):
             graph = "default"
             if triple_store['input_graph']:
                 graph = urllib.parse.urlencode({'graph': triple_store['input_graph']})
-            url = f"{triple_store['url']}:{triple_store['port']}/repositories/{triple_store['repository']}" \
+            url = f"{triple_store['url']}/repositories/{triple_store['repository']}" \
                   f"/rdf-graphs/service?{graph}"
             # graph store PUT drop silently the graph or default and upload the payload
             # https://www.w3.org/TR/sparql11-http-rdf-update/#http-put
@@ -82,7 +82,7 @@ def post_update_query(triple_store: dict, query: str, params: dict = None) -> st
     params = add_graph_to_params(params, triple_store["input_graph"])
     try:
         return manage_graphdb_response(requests.post(
-            url=f"{triple_store['url']}:{triple_store['port']}/repositories/{triple_store['repository']}/statements",
+            url=f"{triple_store['url']}/repositories/{triple_store['repository']}/statements",
             data=query,
             params=params,
             auth=(triple_store['username'], triple_store['password']),
@@ -99,7 +99,7 @@ def post_query(triple_store: dict, query: str, accept: str, params: dict = None)
     params = add_graph_to_params(params, triple_store["input_graph"])
     try:
         return manage_graphdb_response(
-            requests.post(url=f"{triple_store['url']}:{triple_store['port']}/repositories/{triple_store['repository']}",
+            requests.post(url=f"{triple_store['url']}/repositories/{triple_store['repository']}",
                           data=query,
                           params=params,
                           auth=(triple_store['username'], triple_store['password']),