snapquery 0.0.3__tar.gz → 0.0.4__tar.gz

{snapquery-0.0.3 → snapquery-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snapquery
-Version: 0.0.3
+Version: 0.0.4
 Summary: snapquery: Introduce Named Queries and Named Query Middleware to wikidata
 Project-URL: Home, https://github.com/WolfgangFahl/snapquery
 Project-URL: Documentation, https://wiki.bitplan.com/index.php/snapquery
@@ -25,7 +25,9 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: User Interfaces
 Requires-Python: >=3.9
 Requires-Dist: ngwidgets>=0.14.2
-Requires-Dist: pylodstorage>=0.10.4
+Requires-Dist: pylodstorage>=0.10.5
+Requires-Dist: requests
+Requires-Dist: wikitextparser
 Provides-Extra: test
 Requires-Dist: green; extra == 'test'
 Description-Content-Type: text/markdown

{snapquery-0.0.3 → snapquery-0.0.4}/pyproject.toml

@@ -19,8 +19,10 @@ license = {text = "Apache-2.0"}
 dependencies = [
 	# https://github.com/WolfgangFahl/nicegui_widgets
 	"ngwidgets>=0.14.2",
-  # https://pypi.org/project/pyLodStorage/
-  "pyLodStorage>=0.10.4"
+    # https://pypi.org/project/pyLodStorage/
+    "pyLodStorage>=0.10.5",
+    "wikitextparser",
+    "requests"
 ]
 
 requires-python = ">=3.9"

snapquery-0.0.4/snapquery/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.4"

snapquery-0.0.4/snapquery/scholia.py

@@ -0,0 +1,49 @@
+"""
+Created on 2024-05-04
+
+@author: wf
+"""
+import requests
+from snapquery.snapquery_core import NamedQueryManager
+class ScholiaQueries:
+    """
+    scholia queries
+    """
+    
+    @classmethod
+    def get(cls,db_path:str=None,debug:bool=False,limit:int=None):
+        nqm = NamedQueryManager.from_samples(db_path=db_path)
+        repository_url = "https://api.github.com/repos/WDscholia/scholia/contents/scholia/app/templates"
+        headers = {'Accept': 'application/vnd.github.v3+json'}
+        response = requests.get(repository_url, headers=headers)
+        response.raise_for_status()  # Ensure we notice bad responses
+        
+        exclude = ["author_topics", "event_proceedings", "publisher_editors", "works_topics"]
+        lod=[]
+        file_list_json=response.json()
+        for i,file_info in enumerate(file_list_json,start=1):
+            file_name = file_info['name']
+            if file_name.endswith('.sparql') and file_name[:-7] not in exclude:
+                file_response = requests.get(file_info['download_url'])
+                file_response.raise_for_status()
+                query_str = file_response.text
+                name=file_name[:-7]
+                record={ 
+                      "namespace":"scholia",
+                      "name": name,
+                      "url": file_info['download_url'],
+                      "title": name,
+                      "sparql":query_str
+                }
+                lod.append(record)
+                if debug:
+                    if i % 80 == 0:
+                        print(f"{i}")
+                    print(".", end='', flush=True)
+                if limit and len(lod)>=limit:
+                    break
+        if debug:
+            print(f"found {len(lod)} scholia queries")
+        nqm.store(lod)
+        return nqm
+        

{snapquery-0.0.3 → snapquery-0.0.4}/snapquery/snapquery_cmd.py

@@ -3,10 +3,11 @@ Created on 2024-05-03
 
 @author: wf
 """
+
 import sys
 from argparse import ArgumentParser
 
-from lodstorage.query import EndpointManager, Format, Query
+from lodstorage.query import EndpointManager, Format
 from ngwidgets.cmd import WebserverCmd
 
 from snapquery.snapquery_core import NamedQueryManager
@@ -34,7 +35,7 @@ class SnapQueryCmd(WebserverCmd):
             "-en",
             "--endpointName",
             default="wikidata",
-            help=f"Name of the endpoint to use for queries - use --listEndpoints to list available endpoints",
+            help="Name of the endpoint to use for queries - use --listEndpoints to list available endpoints",
         )
         parser.add_argument(
             "-le",
@@ -46,7 +47,10 @@ class SnapQueryCmd(WebserverCmd):
             "--limit", type=int, default=None, help="set limit parameter of query"
         )
         parser.add_argument(
-            "--namespace", type=str, default="wikidata-examples", help="namespace to filter queries"
+            "--namespace",
+            type=str,
+            default="wikidata-examples",
+            help="namespace to filter queries",
         )
         parser.add_argument("-qn", "--queryName", help="run a named query")
         parser.add_argument("-f", "--format", type=Format, choices=list(Format))
@@ -66,13 +70,20 @@ class SnapQueryCmd(WebserverCmd):
                 print(endpoint)
             handled = True  # Operation handled
         elif self.args.queryName is not None:
-            nqm = NamedQueryManager()
-            namespace=self.args.namespace
+            nqm = NamedQueryManager.from_samples()
+            namespace = self.args.namespace
             name = self.args.queryName
             endpoint_name = self.args.endpointName
             r_format = self.args.format
-            limit=self.args.limit
-            formatted_result=nqm.formatted_query(name, namespace=namespace,endpoint_name=endpoint_name, r_format=r_format,limit=limit)
+            limit = self.args.limit
+            qb = nqm.get_query(
+                name=name, namespace=namespace, endpoint_name=endpoint_name, limit=limit
+            )
+            if r_format==Format.raw:
+                formatted_result=qb.raw_query()
+            else:
+                qlod = qb.get_lod()
+                formatted_result = qb.format_result(qlod=qlod, r_format=r_format)
             print(formatted_result)
             return handled
 

{snapquery-0.0.3 → snapquery-0.0.4}/snapquery/snapquery_core.py

@@ -3,9 +3,11 @@ Created on 2024-05-03
 
 @author: wf
 """
+
 import json
-import re
 import os
+import re
+import requests
 from dataclasses import field
 from pathlib import Path
 from typing import Any, Dict, List, Optional
@@ -38,13 +40,15 @@ class NamedQuery:
     namespace: str
     # name/id
     name: str
-    # one line title
-    title: str
-    # multiline description
-    description: str
     # sparql query (to be hidden later)
     sparql: str
-
+    # the url of the source code of the query 
+    url: Optional[str]=None
+    # one line title
+    title: Optional[str]=None
+    # multiline description
+    description: Optional[str]=None
+  
     def __post_init__(self):
         """
         Post-initialization processing to construct a unique identifier for the query
@@ -53,6 +57,9 @@ class NamedQuery:
         self.query_id = f"{self.namespace}.{self.name}"
 
     def as_link(self) -> str:
+        """
+        get me as a link
+        """
         url = f"/query/{self.namespace}/{self.name}"
         text = self.name
         tooltip = "query details"
@@ -68,9 +75,21 @@ class NamedQuery:
             namespace=record["namespace"],
             name=record["name"],
             title=record["title"],
+            url=record["url"],
             description=record["description"],
             sparql=record["sparql"],
         )
+        
+    def as_record(self)->Dict:
+        record = {
+            "namespace": self.namespace,
+            "name": self.name,
+            "url": self.url,
+            "title": self.title,
+            "description": self.description,
+            "sparql": self.sparql,
+        }
+        return record
 
     def as_viewrecord(self) -> Dict:
         """
@@ -81,9 +100,123 @@ class NamedQuery:
             "namespace": self.namespace,
             "title": self.title,
             "description": self.description,
+            "url": self.url
         }
 
 
+class QueryBundle:
+    """
+    Bundles a named query, a query, and an endpoint into a single manageable object, facilitating the execution of SPARQL queries.
+
+    Attributes:
+        named_query (NamedQuery): The named query object, which includes metadata about the query.
+        query (Query): The actual query object that contains the SPARQL query string.
+        endpoint (Endpoint): The endpoint object where the SPARQL query should be executed.
+        sparql (SPARQL): A SPARQL service object initialized with the endpoint URL.
+    """
+
+    def __init__(self, named_query: NamedQuery, query: Query, endpoint: Endpoint):
+        """
+        Initializes a new instance of the QueryBundle class.
+
+        Args:
+            named_query (NamedQuery): An instance of NamedQuery that provides a named reference to the query.
+            query (Query): An instance of Query containing the SPARQL query string.
+            endpoint (Endpoint): An instance of Endpoint representing the SPARQL endpoint URL.
+        """
+        self.named_query = named_query
+        self.query = query
+        self.endpoint = endpoint
+        self.sparql = SPARQL(endpoint.endpoint)
+        
+    def raw_query(self,resultFormat, mime_type:str=None, timeout:float=10.0):
+        """
+        returns raw result of the endpoint
+
+        Args:
+            resultFormat(str): format of the result
+            mime_type(str): mime_type to use (if any)
+            timeout(float): timeout in seconds
+
+        Returns:
+            raw result of the query
+        """
+        params = {"query": self.query.query, "format": resultFormat}
+        payload = {}
+        if mime_type:
+            headers = {"Accept": mime_type}
+        else:
+            headers = {}
+        endpoint_url = self.endpoint.endpoint
+        method = self.endpoint.method
+        response = requests.request(
+            method, endpoint_url, headers=headers, data=payload, params=params,timeout=timeout
+        )
+        return response.text
+
+    def get_lod(self) -> List[dict]:
+        """
+        Executes the stored query using the SPARQL service and returns the results as a list of dictionaries.
+
+        Returns:
+            List[dict]: A list where each dictionary represents a row of results from the SPARQL query.
+        """
+        lod = self.sparql.queryAsListOfDicts(self.query.query)
+        return lod
+
+    def format_result(
+        self,
+        qlod: List[Dict[str, Any]] = None,
+        r_format: Format = Format.json,
+    ) -> Optional[str]:
+        """
+        Formats the query results based on the specified format and prints them.
+
+        Args:
+            qlod (List[Dict[str, Any]]): The list of dictionaries that represent the query results.
+            query (Query): The query object which contains details like the endpoint and the database.
+            r_format(Format): The format in which to print the results.
+
+        Returns:
+            Optional[str]: The formatted string representation of the query results, or None if printed directly.
+        """
+        if qlod is None:
+            qlod = self.get_lod()
+        if r_format is None:
+            r_format = Format.json
+        if r_format == Format.csv:
+            csv_output = CSV.toCSV(qlod)
+            return csv_output
+        elif r_format in [Format.latex, Format.github, Format.mediawiki, Format.html]:
+            doc = self.query.documentQueryResult(
+                qlod, tablefmt=str(r_format), floatfmt=".0f"
+            )
+            return doc.asText()
+        elif r_format == Format.json:
+            return json.dumps(qlod, indent=2, sort_keys=True, default=str)
+        return None  # In case no format is matched or needed
+
+    def set_limit(self, limit: int = None):
+        """
+        set the limit of my query
+        
+        Args:
+            limit(int): the limit to set - default: None
+        """
+        if limit:
+            sparql_query = self.query.query
+            # @TODO - this is too naive for cases where
+            # there are SPARQL elements hat have a "limit" in the name e.g. "height_limit"
+            # or if there is a LIMIT in a subquery
+            if "limit" in sparql_query or "LIMIT" in sparql_query:
+                sparql_query = re.sub(
+                    r"(limit|LIMIT)\s+(\d+)", f"LIMIT {limit}", sparql_query
+                )
+            else:
+                sparql_query += f"\nLIMIT {limit}"
+            self.query.query = sparql_query
+
+
 class NamedQueryManager:
     """
     Manages the storage, retrieval, and execution of named SPARQL queries.
@@ -136,23 +269,33 @@ class NamedQueryManager:
         nqm = NamedQueryManager(debug=debug)
         path_obj = Path(db_path)
         if not path_obj.exists() or path_obj.stat().st_size == 0:
-            sample_queries = cls.get_samples()
-            list_of_records = []
-            for _query_name, named_query in sample_queries.items():
-                record = {
-                    "namespace": named_query.namespace,
-                    "name": named_query.name,
-                    "title": named_query.title,
-                    "description": named_query.description,
-                    "sparql": named_query.sparql,
-                }
-                list_of_records.append(record)
+            sample_records=cls.get_sample_records()
             entityInfo = EntityInfo(
-                list_of_records, name="NamedQuery", primaryKey="query_id"
+                sample_records, name="NamedQuery", primaryKey="query_id"
             )
-            nqm.sql_db.createTable(list_of_records, "NamedQuery", withDrop=True)
-            nqm.sql_db.store(list_of_records, entityInfo)
+            nqm.sql_db.createTable(sample_records, "NamedQuery", withDrop=True)
+            nqm.sql_db.store(sample_records, entityInfo)
         return nqm
+    
+    def store(self,lod):
+        """
+        store the given list of dicts
+        """
+        sample_records=NamedQueryManager.get_sample_records()
+        entityInfo = EntityInfo(
+            sample_records, name="NamedQuery", primaryKey="query_id"
+        )
+        self.sql_db.store(lod, entityInfo,fixNone=True,replace=True)
+        
+    @classmethod
+    def get_sample_records(cls):
+        sample_queries = cls.get_samples()
+        list_of_records = []
+        for _query_name, named_query in sample_queries.items():
+            record=named_query.as_record()
+            list_of_records.append(record)
+        return list_of_records
+
 
     @classmethod
     def get_samples(cls) -> dict[str, NamedQuery]:
@@ -163,6 +306,7 @@ class NamedQueryManager:
             "wikidata-examples.cats": NamedQuery(
                 namespace="wikidata-examples",
                 name="cats",
+                url="https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples#Cats",
                 title="Cats on Wikidata",
                 description="This query retrieves all items classified under 'house cat' (Q146) on Wikidata.",
                 sparql="""
@@ -176,6 +320,7 @@ WHERE {
             "wikidata-examples.horses": NamedQuery(
                 namespace="wikidata-examples",
                 name="horses",
+                url="https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples#Horses_(showing_some_info_about_them)",
                 title="Horses on Wikidata",
                 description="This query retrieves information about horses, including parents, gender, and approximate birth and death years.",
                 sparql="""
@@ -198,14 +343,20 @@ ORDER BY ?horse
         }
         return samples
 
-    def get_sparql(
-        self,
-        name: str,
-        namespace: str = "wikidata-examples",
-        endpoint_name: str = "wikidata",
-    ) -> str:
-        sql_query = f"""SELECT 
-    sparql 
+    def lookup(self, name: str, namespace: str) -> NamedQuery:
+        """
+        lookup the named query for the given name and namespace
+
+
+        Args:
+            name (str): The name of the named query to execute.
+            namespace (str): The namespace of the named query, default is 'wikidata-examples'.
+
+        Returns:
+            NamedQuery: the named query
+        """
+        sql_query = """SELECT 
+    *
 FROM 
     NamedQuery 
 WHERE 
@@ -221,18 +372,19 @@ WHERE
             msg = f"multiple entries ({query_count}) for name '{name}' and namespace '{namespace}'"
             raise ValueError(msg)
 
-        sparql_query = query_records[0]["sparql"]
-        return sparql_query
+        record = query_records[0]
+        named_query = NamedQuery.from_record(record)
+        return named_query
 
-    def query(
+    def get_query(
         self,
         name: str,
         namespace: str = "wikidata-examples",
         endpoint_name: str = "wikidata",
-        limit:int=None
-    ):
+        limit: int = None,
+    ) -> QueryBundle:
         """
-        Execute a named SPARQL query using a specified endpoint and return the results.
+        get the query for the given parameters
 
         Args:
             name (str): The name of the named query to execute.
@@ -241,114 +393,28 @@ WHERE
             limit(int): the query limit (if any)
 
         Returns:
-            Dict[str, Any]: The results of the SPARQL query in JSON format.
-
-        Raises:
-            ValueError: If no named query matches the given name and namespace.
-            Exception: If the SPARQL query execution fails or the endpoint returns an error.
+            QueryBundle: named_query, query and endpoint
         """
-        sparql_query = self.get_sparql(name, namespace, endpoint_name)
-
-        if not endpoint_name in self.endpoints:
+        named_query = self.lookup(name, namespace)
+        if endpoint_name not in self.endpoints:
             msg = f"Invalid endpoint {endpoint_name}"
             ValueError(msg)
         endpoint = self.endpoints.get(endpoint_name)
-        self.query = Query(name=name, query=sparql_query, lang="sparql",limit=limit)
- 
-        sparql = SPARQL(endpoint.endpoint)
-        if limit: 
-            # @TODO - this is to naive for cases where there are 
-            # SPARQL elements hat have a "limit" in the name e.g. "height_limit"
-            if "limit" in sparql_query or "LIMIT" in sparql_query:
-                sparql_query = re.sub(
-                    r"(limit|LIMIT)\s+(\d+)", f"LIMIT {limit}", sparql_query
-                )
-            else:
-                sparql_query += f"\nLIMIT {limit}"
-        lod = sparql.queryAsListOfDicts(sparql_query)
-        return lod
-    
-    def formatted_query(self,
-        name: str,
-        namespace: str = "wikidata-examples",
-        endpoint_name: str = "wikidata",
-        r_format: Format = Format.html,
-        limit:int=None)->str:
-        """
-        Execute a named SPARQL query using a specified endpoint and return the results formatted .
-
-        Args:
-            name (str): The name of the named query to execute.
-            namespace (str): The namespace of the named query, default is 'wikidata-examples'.
-            endpoint_name (str): The name of the endpoint to send the SPARQL query to, default is 'wikidata'.
-            r_format: (Format): the format to result should be returned in
-            limit(int): the query limit (if any)
-        Returns:
-            str: The results of the SPARQL query in the given format.
-
-        Raises:
-            ValueError: If no named query matches the given name and namespace.
-            Exception: If the SPARQL query execution fails or the endpoint returns an error.
-        """
-        qlod = self.query(endpoint_name=endpoint_name, name=name,namespace=namespace,limit=limit)
-        formatted_result=self.format_result(qlod=qlod, query=self.query, r_format=r_format, endpoint_name=endpoint_name)
-        return formatted_result
-
-    def format_result(
-        self,
-        qlod: List[Dict[str, Any]],
-        query: Query,
-        r_format: Format,
-        endpoint_name: str = "wikidata",
-    ) -> Optional[str]:
-        """
-        Formats the query results based on the specified format and prints them.
-
-        Args:
-            qlod (List[Dict[str, Any]]): The list of dictionaries that represent the query results.
-            query (Query): The query object which contains details like the endpoint and the database.
-            r_format_str(Format): The format in which to print the results.
-            endpoint_name (str): The name of the endpoint to be used, defaults to "wikidata".
-
-        Returns:
-            Optional[str]: The formatted string representation of the query results, or None if printed directly.
-        """
-        # Retrieve endpoint configuration using the provided endpoint name
+        sparql_query = named_query.sparql
+        query = Query(
+            name=name,
+            query=sparql_query,
+            lang="sparql",
+            endpoint=endpoint.endpoint,
+            limit=limit,
+        )
         endpointConf = self.endpoints.get(endpoint_name, Endpoint.getDefault())
         query.tryItUrl = endpointConf.website
         query.database = endpointConf.database
-
-        if r_format == Format.csv:
-            csv_output = CSV.toCSV(qlod)
-            return csv_output
-        elif r_format in [Format.latex, Format.github, Format.mediawiki, Format.html]:
-            doc = query.documentQueryResult(
-                qlod, tablefmt=str(r_format), floatfmt=".0f"
-            )
-            return doc.asText()
-        elif r_format == Format.json:
-            return json.dumps(qlod, indent=2, sort_keys=True, default=str)
-        return None  # In case no format is matched or needed
-
-    def print_result(
-        self,
-        qlod: List[Dict[str, Any]],
-        query: Query,
-        r_format_str: str,
-        endpoint_name: str = "wikidata",
-    ) -> None:
-        """
-        Prints the formatted results of a query.
-
-        Args:
-            qlod (List[Dict[str, Any]]): The list of dictionaries that represent the query results.
-            query (Query): The query object which contains details such as the endpoint and the database.
-            r_format_str: The format in which to print the results.
-            endpoint_name (str): The name of the endpoint to be used, defaults to "wikidata".
-
-        Returns:
-            None
-        """
-        result_str = self.format_result(qlod, query, r_format_str, endpoint_name)
-        if result_str is not None:
-            print(result_str)
+        query_bundle = QueryBundle(
+            named_query=named_query, 
+            query=query, 
+            endpoint=endpoint
+        )
+        query_bundle.set_limit(limit)
+        return query_bundle

{snapquery-0.0.3 → snapquery-0.0.4}/snapquery/snapquery_view.py

@@ -3,14 +3,89 @@ Created on 2024-05-03
 
 @author: wf
 """
+
 import asyncio
 from typing import List
 
 from ngwidgets.input_webserver import InputWebSolution
 from ngwidgets.lod_grid import ListOfDictsGrid
-from nicegui import ui
+from ngwidgets.widgets import Link
+from nicegui import ui, run, background_tasks
+
+from snapquery.snapquery_core import NamedQuery, QueryBundle
+
+
+class NamedQueryView:
+    """
+    display a named Query
+    """
+
+    def __init__(
+        self,
+        solution: InputWebSolution,
+        query_bundle: QueryBundle,
+        r_format_str: str = "html",
+    ):
+        self.solution = solution
+        self.nqm = self.solution.nqm
+        self.query_bundle = query_bundle
+        self.r_format_str = r_format_str
+        self.load_task = None
+        self.limit = 200
+        self.timeout = 20.0
+        self.setup_ui()
+
+    def setup_ui(self):
+        """
+        setup my user interface
+        """
+        nq = self.query_bundle.named_query
+        url = self.query_bundle.query.tryItUrl
+        text = nq.title
+        tooltip = "try it!"
+        link = Link.create(url, text, tooltip, target="_blank")
+        with self.solution.container:
+            with ui.row() as self.query_parms_row:
+                ui.number(label="limit").bind_value(self, "limit")
+                ui.number(label="time out").bind_value(self, "timeout")
+            with ui.row() as self.query_row:
+                self.try_it_link = ui.html(link)
+                ui.label(nq.description)
+                ui.button(icon="play_arrow", on_click=self.run_query)
+            self.grid_row = ui.row()
+            pass
+
+    async def load_query_results(self):
+        """
+        (re) load the query results
+        """
+        self.query_bundle.set_limit(int(self.limit))
+        lod = await run.io_bound(self.query_bundle.get_lod)
+        self.grid_row.clear()
+        with self.grid_row:
+            self.lod_grid = ListOfDictsGrid()
+            self.lod_grid.load_lod(lod)
+        self.grid_row.update()
+
+    async def run_query(self, _args):
+        """
+        run the current query
+        """
+
+        def cancel_running():
+            if self.load_task:
+                self.load_task.cancel()
 
-from snapquery.snapquery_core import NamedQuery
+        self.grid_row.clear()
+        with self.grid_row:
+            ui.spinner()
+        self.grid_row.update()
+        # cancel task still running
+        cancel_running()
+        # cancel task if it takes too long
+        ui.timer(self.timeout, lambda: cancel_running(), once=True)
+        # run task in background
+        self.load_task = background_tasks.create(self.load_query_results())
 
 
 class NamedQuerySearch:
@@ -24,7 +99,7 @@ class NamedQuerySearch:
         self.search_debounce_task = None
         self.keystroke_time = 0.65  # minimum time in seconds to wait between keystrokes before starting searching
         self.setup_ui()
-        self.namespace = "wikidata-examples"
+        self.namespace = ""
         self.name = ""
 
     def setup_ui(self):

{snapquery-0.0.3 → snapquery-0.0.4}/snapquery/snapquery_webserver.py

@@ -2,15 +2,16 @@
 Created on 2024-05-03
 @author: wf
 """
+
 from fastapi import HTTPException
 from fastapi.responses import HTMLResponse, PlainTextResponse
-from lodstorage.query import Query, Format
+from lodstorage.query import Format
 from ngwidgets.input_webserver import InputWebserver, InputWebSolution, WebserverConfig
-from nicegui import app
+from nicegui import app, ui
 from nicegui.client import Client
 
 from snapquery.snapquery_core import NamedQueryManager
-from snapquery.snapquery_view import NamedQuerySearch
+from snapquery.snapquery_view import NamedQuerySearch, NamedQueryView
 from snapquery.version import Version
 
 
@@ -40,33 +41,68 @@ class SnapQueryWebServer(InputWebserver):
         InputWebserver.__init__(self, config=SnapQueryWebServer.get_config())
         self.nqm = NamedQueryManager.from_samples()
 
-        @app.get("/sparql/{namespace}/{name}")
-        def sparql(namespace: str, name: str) -> PlainTextResponse:
+        @ui.page("/query/{namespace}/{name}")
+        async def query_page(
+            client: Client,
+            namespace: str,
+            name: str,
+            endpoint_name: str = "wikidata",
+            limit: int = None,
+            format: str = "html",
+        ):
+            """
+            show the query page for the given namespace and name
+            """
+            return await self.page(
+                client,
+                SnapQuerySolution.query_page,
+                namespace=namespace,
+                name=name,
+                endpoint_name=endpoint_name,
+                limit=limit,
+                r_format_str=format,
+            )
+
+        @app.get("/api/sparql/{namespace}/{name}")
+        def sparql(
+            namespace: str,
+            name: str,
+            endpoint_name: str = "wikidata",
+            limit: int = None,
+        ) -> PlainTextResponse:
             """
             Gets a SPARQL query by name within a specified namespace
 
             Args:
                 namespace (str): The namespace identifying the group or category of the query.
                 name (str): The specific name of the query to be executed.
-
+                endpoint_name(str): the name of the endpoint to use
+                limit(int): a limit to set, default=None
             Returns:
                 HTMLResponse: The plain text SPARQL code
 
             Raises:
                 HTTPException: If the query cannot be found or fails to execute.
             """
-            endpoint_name = "wikidata"
-            sparql_query = self.nqm.get_sparql(name, namespace, endpoint_name)
+            qb = self.nqm.get_query(name, namespace, endpoint_name, limit)
+            sparql_query = qb.query.query
             return PlainTextResponse(sparql_query)
 
-        @app.get("/query/{namespace}/{name}")
-        def query(namespace: str, name: str) -> HTMLResponse:
+        @app.get("/api/query/{namespace}/{name}")
+        def query(
+            namespace: str,
+            name: str,
+            endpoint_name: str = "wikidata",
+            limit: int = None,
+        ) -> HTMLResponse:
             """
             Executes a SPARQL query by name within a specified namespace, formats the results, and returns them as an HTML response.
 
             Args:
                 namespace (str): The namespace identifying the group or category of the query.
                 name (str): The specific name of the query to be executed.
+                endpoint_name(str): the name of the endpoint to use
+                limit(int): a limit to set, default=None
 
             Returns:
                 HTMLResponse: The HTML formatted response containing the results of the query execution.
@@ -74,16 +110,34 @@ class SnapQueryWebServer(InputWebserver):
             Raises:
                 HTTPException: If the query cannot be found or fails to execute.
             """
-            content = self.query(namespace, name)
+            content = self.query(
+                namespace, name, endpoint_name=endpoint_name, limit=limit
+            )
             if not content:
                 raise HTTPException(status_code=500, detail="Could not create result")
 
             # Return the content as an HTML response
             return HTMLResponse(content)
 
-    def query(self, namespace: str, name: str):
-        """ """
-        endpoint_name = "wikidata"
+    def query(
+        self,
+        namespace: str,
+        name: str,
+        endpoint_name: str = "wikidata",
+        limit: int = None,
+    ) -> str:
+        """
+        Queries an external API to retrieve data based on a given namespace and name.
+
+        Args:
+            namespace (str): The namespace to which the query belongs. It helps in categorizing the data.
+            name (str): The name identifier of the data to be queried.
+            endpoint_name (str): The name of the endpoint to be used for the query. Defaults to 'wikidata'.
+            limit(int): the limit for the query default: None
+
+            Returns:
+                str: the content retrieved
+        """
         # content negotiation
         # Determine response format by extension in the name or Accept header
         if "." in name:
@@ -92,24 +146,16 @@ class SnapQueryWebServer(InputWebserver):
         else:
             r_format_str = "html"
 
-        # Retrieve the SPARQL query string using the namespace and name.
         try:
-            sparql_query = self.nqm.get_sparql(name, namespace, endpoint_name)
-            qlod = self.nqm.query(
-                name=name, namespace=namespace, endpoint_name=endpoint_name
-            )
-            query = Query(name=name, query=sparql_query, lang="sparql")
+            r_format = Format[r_format_str]
+            qb = self.nqm.get_query(name, namespace, endpoint_name, limit)
+            qlod = qb.get_lod()
+            content = qb.format_result(qlod, r_format)
+            return content
         except Exception as e:
             # Handling specific exceptions can be more detailed based on what nqm.get_sparql and nqm.query can raise
             raise HTTPException(status_code=404, detail=str(e))
 
-        # Format the results and generate HTML content
-        r_format = Format[r_format_str]
-        content = self.nqm.format_result(
-            qlod, query, r_format, endpoint_name=endpoint_name
-        )
-        return content
-
 
 class SnapQuerySolution(InputWebSolution):
     """
@@ -150,3 +196,19 @@ class SnapQuerySolution(InputWebSolution):
         svg display
         """
         await self.setup_content_div(self.setup_ui)
+
+    async def query_page(
+        self,
+        namespace: str,
+        name: str,
+        endpoint_name: str = "wikidata",
+        limit: int = None,
+        r_format_str: str = "html",
+    ):
+        def show():
+            query_bundle = self.nqm.get_query(name, namespace, endpoint_name, limit)
+            self.named_query_view = NamedQueryView(
+                self, query_bundle=query_bundle, r_format_str=r_format_str
+            )
+
+        await self.setup_content_div(show)

{snapquery-0.0.3 → snapquery-0.0.4}/snapquery/version.py

@@ -3,6 +3,7 @@ Created on 2024-05-01
 
 @author: wf
 """
+
 from dataclasses import dataclass
 
 import snapquery
@@ -26,7 +27,7 @@ class Version:
     chat_url = "https://github.com/WolfgangFahl/snapquery/discussions"
     cm_url = "https://github.com/WolfgangFahl/snapquery"
 
-    license = f"""Copyright 2024 contributors. All rights reserved.
+    license = """Copyright 2024 contributors. All rights reserved.
 
   Licensed under the Apache License 2.0
   http://www.apache.org/licenses/LICENSE-2.0

snapquery-0.0.4/tests/test_cmdline.py

@@ -0,0 +1,75 @@
+"""
+Created on 2024-05-04
+
+@author: wf
+"""
+
+import json
+import sys
+from io import StringIO
+
+from ngwidgets.basetest import Basetest
+
+from snapquery import snapquery_cmd
+
+
+class TestCommandLine(Basetest):
+    """
+    test the command line handling
+    """
+
+    def setUp(self, debug=False, profile=True):
+        Basetest.setUp(self, debug=debug, profile=profile)
+
+    def capture_stdout(self, callback, *args, **kwargs):
+        """
+        Executes a callback with given arguments and captures stdout.
+        """
+        capture = StringIO()
+        original_stdout = sys.stdout
+        sys.stdout = capture
+        try:
+            callback(*args, **kwargs)
+            return capture.getvalue()
+        finally:
+            sys.stdout = original_stdout
+
+    def test_list_endpoints(self):
+        """
+        test listing endpoints
+
+        """
+
+        def run_cmd():
+            snapquery_cmd.main(["--listEndpoints"])
+
+        # Capture the output of running the command
+        output = self.capture_stdout(run_cmd)
+        if self.debug:
+            print(output)
+        self.assertTrue("wikidata:https://query.wikidata.org" in output)
+
+    def test_namedquery(self):
+        """
+        test a named query via command line
+        """
+        limit = 10
+
+        def run_cmd():
+            snapquery_cmd.main(["-d", "-qn", "cats", "--limit", f"{limit}"])
+
+        # Capture the output of running the command
+        output = self.capture_stdout(run_cmd)
+        if self.debug:
+            print(output)
+        # Parse the JSON output
+        result = json.loads(output)
+        # Check that the result has exactly 10 entries
+        self.assertEqual(
+            len(result), limit, "The number of results should be exactly 10."
+        )
+
+        # Check that each entry has 'item' and 'itemLabel'
+        for entry in result:
+            self.assertIn("item", entry, "Each entry must include an 'item'.")
+            self.assertIn("itemLabel", entry, "Each entry must include an 'itemLabel'.")

{snapquery-0.0.3 → snapquery-0.0.4}/tests/test_endpoints.py

@@ -3,6 +3,7 @@ Created on 2024-05-03
 
 @author: wf
 """
+
 from lodstorage.query import EndpointManager
 from ngwidgets.basetest import Basetest
 

{snapquery-0.0.3 → snapquery-0.0.4}/tests/test_namedqueries.py

@@ -3,6 +3,7 @@ Created on 2024-05-03
 
 @author: wf
 """
+
 import json
 
 from ngwidgets.basetest import Basetest
@@ -12,7 +13,7 @@ from snapquery.snapquery_core import NamedQueryManager
 
 class TestNamedQueryManager(Basetest):
     """
-    test the named query Manage
+    test the named query Manager
     """
 
     def setUp(self, debug=False, profile=True):
@@ -26,7 +27,8 @@ class TestNamedQueryManager(Basetest):
         nqm = NamedQueryManager.from_samples(db_path=db_path)
         for name, ex_count in [("x-invalid", -1), ("cats", 223)]:
             try:
-                lod = nqm.query(name)
+                query_bundle = nqm.get_query(name)
+                lod = query_bundle.get_lod()
                 if self.debug:
                     print(f"{name}:")
                     print(json.dumps(lod, default=str, indent=2))

{snapquery-0.0.3 → snapquery-0.0.4}/tests/test_restful_api.py

@@ -3,6 +3,7 @@ Created on 2024-05-03
 
 @author: wf
 """
+
 from ngwidgets.webserver_test import WebserverTest
 
 from snapquery.snapquery_cmd import SnapQueryCmd
@@ -32,7 +33,7 @@ class TestRestFulApi(WebserverTest):
         test the plaintext SPARQL API
         """
         for example in ["cats", "horses"]:
-            path = f"/sparql/wikidata-examples/{example}"
+            path = f"/api/sparql/wikidata-examples/{example}?limit=10"
             sparql_query = self.getHtml(path)
             if self.debug:
                 print(sparql_query)
@@ -42,8 +43,8 @@ class TestRestFulApi(WebserverTest):
         """
         test the RESTFul Query API
         """
-        for r_format in ["mediawiki", "github", "latex", "html"]:
-            path = f"/query/wikidata-examples/cats.{r_format}"
-            html = self.getHtml(path)
+        for r_format in ["mediawiki", "github", "latex", "html", "json"]:
+            path = f"/api/query/wikidata-examples/cats.{r_format}?limit=10"
+            result = self.getHtml(path)
             if self.debug:
-                print(html)
+                print(result)

snapquery-0.0.4/tests/test_scholia.py

@@ -0,0 +1,26 @@
+"""
+Created on 2024-05-04
+
+@author: wf
+"""
+from ngwidgets.basetest import Basetest
+from snapquery.scholia import ScholiaQueries
+class TestScholia(Basetest):
+    """
+    test scholia queries
+    """
+
+    def setUp(self, debug=True, profile=True):
+        Basetest.setUp(self, debug=debug, profile=profile)
+        
+    def test_scholia_queries(self):
+        """
+        test retrieving scholia queries
+        """
+        if self.inPublicCI():
+            db_path = "/tmp/scholia_queries.db"
+            limit=10
+        else:
+            db_path=None
+            limit=None
+        nqm=ScholiaQueries.get(db_path,debug=self.debug,limit=limit)

snapquery-0.0.4/tests/test_wd_query_parsing.py

@@ -0,0 +1,90 @@
+"""
+Created on 2024-05-04
+
+@author: tholzheim
+"""
+
+import pprint
+import re
+import requests
+import wikitextparser as wtp
+from wikitextparser import Section, Template
+from ngwidgets.basetest import Basetest
+from snapquery.snapquery_core import NamedQuery, NamedQueryManager
+
+class TestWdQueryParsing(Basetest):
+    """
+    test wikidata query parsing
+    """
+    
+    def setUp(self, debug=True, profile=True):
+        Basetest.setUp(self, debug=debug, profile=profile)
+        self.base_url = "https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples"
+        
+    def _get_examples_wikitext(self) -> str:
+        """Get wiki text with SPARQL query examples"""
+        res = requests.get(f"{self.base_url}?action=raw")
+        return res.text
+
+    def _extract_query_from_section(self, section: Section) -> NamedQuery:
+        """
+        Extract named query from section
+        """
+        query = self._get_sparql_template(section.templates)
+        named_query = None
+        if query:
+            desc = section.plain_text()
+            title = self._sanitize_title(section.title).strip()
+            named_query = NamedQuery(
+                namespace="wikidata-examples",
+                name=title,
+                title=title,
+                description=desc,
+                url=self.base_url,
+                sparql=query.arguments[0].value,
+            )
+        return named_query
+
+    def _sanitize_title(self, section_title: str) -> str:
+        """
+        Sanitize section title by removing translate tags
+        """
+        pattern = r"""<translate><!--T:\d+-->(?P<name>[\w\d\s.\-,\\=<>/|{}()"'&]+)<\/translate>"""
+        match = re.search(pattern, section_title)
+        if match:
+            title = match.group("name")
+        else:
+            title = section_title
+        return title
+
+    def _get_sparql_template(self, templates: list[Template]) -> Template:
+        """
+        Get sparql template of the list of templates
+        If the list contains more than one sparql template None is returned.
+        """
+        queries = []
+        for template in templates:
+            if template.name == "SPARQL":
+                queries.append(template)
+        # ToDo: Handle sections with multiple queries
+        return queries[0] if len(queries) == 1 else None
+
+    def test_wikidata_examples_query_extraction(self):
+        """
+        test extraction and parsing of wikidata example queries to NamedQueries
+
+        Queries can be found at https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples
+        """
+        wikitext = self._get_examples_wikitext()
+        parsed = wtp.parse(wikitext)
+        lod = []
+        for section in parsed.sections:
+            nq = self._extract_query_from_section(section)
+            if nq:
+                lod.append(nq.as_record())
+                if self.debug:
+                    pprint.pprint(nq)
+        if self.debug:
+            print(f"found {len(lod)} queries")
+        nqm = NamedQueryManager.from_samples()
+        nqm.store(lod)

snapquery-0.0.3/snapquery/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.3"