snapquery 0.0.2__tar.gz → 0.0.4__tar.gz

{snapquery-0.0.2 → snapquery-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snapquery
-Version: 0.0.2
+Version: 0.0.4
 Summary: snapquery: Introduce Named Queries and Named Query Middleware to wikidata
 Project-URL: Home, https://github.com/WolfgangFahl/snapquery
 Project-URL: Documentation, https://wiki.bitplan.com/index.php/snapquery
@@ -25,7 +25,9 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: User Interfaces
 Requires-Python: >=3.9
 Requires-Dist: ngwidgets>=0.14.2
-Requires-Dist: pylodstorage>=0.10.4
+Requires-Dist: pylodstorage>=0.10.5
+Requires-Dist: requests
+Requires-Dist: wikitextparser
 Provides-Extra: test
 Requires-Dist: green; extra == 'test'
 Description-Content-Type: text/markdown

{snapquery-0.0.2 → snapquery-0.0.4}/pyproject.toml

@@ -19,8 +19,10 @@ license = {text = "Apache-2.0"}
 dependencies = [
 	# https://github.com/WolfgangFahl/nicegui_widgets
 	"ngwidgets>=0.14.2",
-  # https://pypi.org/project/pyLodStorage/
-  "pyLodStorage>=0.10.4"
+    # https://pypi.org/project/pyLodStorage/
+    "pyLodStorage>=0.10.5",
+    "wikitextparser",
+    "requests"
 ]
 
 requires-python = ">=3.9"

snapquery-0.0.4/snapquery/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.4"

snapquery-0.0.4/snapquery/scholia.py

@@ -0,0 +1,49 @@
+"""
+Created on 2024-05-04
+
+@author: wf
+"""
+import requests
+from snapquery.snapquery_core import NamedQueryManager
+class ScholiaQueries:
+    """
+    scholia queries
+    """
+    
+    @classmethod
+    def get(cls,db_path:str=None,debug:bool=False,limit:int=None):
+        nqm = NamedQueryManager.from_samples(db_path=db_path)
+        repository_url = "https://api.github.com/repos/WDscholia/scholia/contents/scholia/app/templates"
+        headers = {'Accept': 'application/vnd.github.v3+json'}
+        response = requests.get(repository_url, headers=headers)
+        response.raise_for_status()  # Ensure we notice bad responses
+        
+        exclude = ["author_topics", "event_proceedings", "publisher_editors", "works_topics"]
+        lod=[]
+        file_list_json=response.json()
+        for i,file_info in enumerate(file_list_json,start=1):
+            file_name = file_info['name']
+            if file_name.endswith('.sparql') and file_name[:-7] not in exclude:
+                file_response = requests.get(file_info['download_url'])
+                file_response.raise_for_status()
+                query_str = file_response.text
+                name=file_name[:-7]
+                record={ 
+                      "namespace":"scholia",
+                      "name": name,
+                      "url": file_info['download_url'],
+                      "title": name,
+                      "sparql":query_str
+                }
+                lod.append(record)
+                if debug:
+                    if i % 80 == 0:
+                        print(f"{i}")
+                    print(".", end='', flush=True)
+                if limit and len(lod)>=limit:
+                    break
+        if debug:
+            print(f"found {len(lod)} scholia queries")
+        nqm.store(lod)
+        return nqm
+        

{snapquery-0.0.2 → snapquery-0.0.4}/snapquery/snapquery_cmd.py

@@ -3,10 +3,11 @@ Created on 2024-05-03
 
 @author: wf
 """
+
 import sys
 from argparse import ArgumentParser
 
-from lodstorage.query import EndpointManager, Format, Query
+from lodstorage.query import EndpointManager, Format
 from ngwidgets.cmd import WebserverCmd
 
 from snapquery.snapquery_core import NamedQueryManager
@@ -34,7 +35,7 @@ class SnapQueryCmd(WebserverCmd):
             "-en",
             "--endpointName",
             default="wikidata",
-            help=f"Name of the endpoint to use for queries - use --listEndpoints to list available endpoints",
+            help="Name of the endpoint to use for queries - use --listEndpoints to list available endpoints",
         )
         parser.add_argument(
             "-le",
@@ -42,6 +43,15 @@ class SnapQueryCmd(WebserverCmd):
             action="store_true",
             help="show the list of available endpoints",
         )
+        parser.add_argument(
+            "--limit", type=int, default=None, help="set limit parameter of query"
+        )
+        parser.add_argument(
+            "--namespace",
+            type=str,
+            default="wikidata-examples",
+            help="namespace to filter queries",
+        )
         parser.add_argument("-qn", "--queryName", help="run a named query")
         parser.add_argument("-f", "--format", type=Format, choices=list(Format))
         return parser
@@ -60,15 +70,22 @@ class SnapQueryCmd(WebserverCmd):
                 print(endpoint)
             handled = True  # Operation handled
         elif self.args.queryName is not None:
-            nqm = NamedQueryManager()
+            nqm = NamedQueryManager.from_samples()
+            namespace = self.args.namespace
             name = self.args.queryName
             endpoint_name = self.args.endpointName
             r_format = self.args.format
-            sparql_query = nqm.get_sparql(name=name, endpoint_name=endpoint_name)
-            qlod = nqm.query(endpoint_name=endpoint_name, name=name)
-            query = Query(name=name, query=sparql_query, lang="sparql")
-            nqm.format_result(qlod, query, r_format, endpoint_name=endpoint_name)
-        return handled
+            limit = self.args.limit
+            qb = nqm.get_query(
+                name=name, namespace=namespace, endpoint_name=endpoint_name, limit=limit
+            )
+            if r_format==Format.raw:
+                formatted_result=qb.raw_query()
+            else:
+                qlod = qb.get_lod()
+                formatted_result = qb.format_result(qlod=qlod, r_format=r_format)
+            print(formatted_result)
+            return handled
 
 
 def main(argv: list = None):

{snapquery-0.0.2 → snapquery-0.0.4}/snapquery/snapquery_core.py

@@ -3,8 +3,11 @@ Created on 2024-05-03
 
 @author: wf
 """
+
 import json
 import os
+import re
+import requests
 from dataclasses import field
 from pathlib import Path
 from typing import Any, Dict, List, Optional
@@ -14,6 +17,7 @@ from lodstorage.query import Endpoint, EndpointManager, Format, Query
 from lodstorage.sparql import SPARQL
 from lodstorage.sql import SQLDB, EntityInfo
 from lodstorage.yamlable import lod_storable
+from ngwidgets.widgets import Link
 
 
 @lod_storable
@@ -36,13 +40,15 @@ class NamedQuery:
     namespace: str
     # name/id
     name: str
-    # one line title
-    title: str
-    # multiline description
-    description: str
     # sparql query (to be hidden later)
     sparql: str
-
+    # the url of the source code of the query 
+    url: Optional[str]=None
+    # one line title
+    title: Optional[str]=None
+    # multiline description
+    description: Optional[str]=None
+  
     def __post_init__(self):
         """
         Post-initialization processing to construct a unique identifier for the query
@@ -50,6 +56,166 @@ class NamedQuery:
         """
         self.query_id = f"{self.namespace}.{self.name}"
 
+    def as_link(self) -> str:
+        """
+        get me as a link
+        """
+        url = f"/query/{self.namespace}/{self.name}"
+        text = self.name
+        tooltip = "query details"
+        link = Link.create(url, text, tooltip)
+        return link
+
+    @classmethod
+    def from_record(cls, record: Dict) -> "NamedQuery":
+        """
+        Class method to instantiate NamedQuery from a dictionary record.
+        """
+        return cls(
+            namespace=record["namespace"],
+            name=record["name"],
+            title=record["title"],
+            url=record["url"],
+            description=record["description"],
+            sparql=record["sparql"],
+        )
+        
+    def as_record(self)->Dict:
+        record = {
+            "namespace": self.namespace,
+            "name": self.name,
+            "url": self.url,
+            "title": self.title,
+            "description": self.description,
+            "sparql": self.sparql,
+        }
+        return record
+
+    def as_viewrecord(self) -> Dict:
+        """
+        Return a dictionary representing the NamedQuery with keys ordered as Name, Namespace, Title, Description.
+        """
+        return {
+            "name": self.as_link(),
+            "namespace": self.namespace,
+            "title": self.title,
+            "description": self.description,
+            "url": self.url
+        }
+
+
+class QueryBundle:
+    """
+    Bundles a named query, a query, and an endpoint into a single manageable object, facilitating the execution of SPARQL queries.
+
+    Attributes:
+        named_query (NamedQuery): The named query object, which includes metadata about the query.
+        query (Query): The actual query object that contains the SPARQL query string.
+        endpoint (Endpoint): The endpoint object where the SPARQL query should be executed.
+        sparql (SPARQL): A SPARQL service object initialized with the endpoint URL.
+    """
+
+    def __init__(self, named_query: NamedQuery, query: Query, endpoint: Endpoint):
+        """
+        Initializes a new instance of the QueryBundle class.
+
+        Args:
+            named_query (NamedQuery): An instance of NamedQuery that provides a named reference to the query.
+            query (Query): An instance of Query containing the SPARQL query string.
+            endpoint (Endpoint): An instance of Endpoint representing the SPARQL endpoint URL.
+        """
+        self.named_query = named_query
+        self.query = query
+        self.endpoint = endpoint
+        self.sparql = SPARQL(endpoint.endpoint)
+        
+    def raw_query(self,resultFormat, mime_type:str=None, timeout:float=10.0):
+        """
+        returns raw result of the endpoint
+
+        Args:
+            resultFormat(str): format of the result
+            mime_type(str): mime_type to use (if any)
+            timeout(float): timeout in seconds
+
+        Returns:
+            raw result of the query
+        """
+        params = {"query": self.query.query, "format": resultFormat}
+        payload = {}
+        if mime_type:
+            headers = {"Accept": mime_type}
+        else:
+            headers = {}
+        endpoint_url = self.endpoint.endpoint
+        method = self.endpoint.method
+        response = requests.request(
+            method, endpoint_url, headers=headers, data=payload, params=params,timeout=timeout
+        )
+        return response.text
+
+    def get_lod(self) -> List[dict]:
+        """
+        Executes the stored query using the SPARQL service and returns the results as a list of dictionaries.
+
+        Returns:
+            List[dict]: A list where each dictionary represents a row of results from the SPARQL query.
+        """
+        lod = self.sparql.queryAsListOfDicts(self.query.query)
+        return lod
+
+    def format_result(
+        self,
+        qlod: List[Dict[str, Any]] = None,
+        r_format: Format = Format.json,
+    ) -> Optional[str]:
+        """
+        Formats the query results based on the specified format and prints them.
+
+        Args:
+            qlod (List[Dict[str, Any]]): The list of dictionaries that represent the query results.
+            query (Query): The query object which contains details like the endpoint and the database.
+            r_format(Format): The format in which to print the results.
+
+        Returns:
+            Optional[str]: The formatted string representation of the query results, or None if printed directly.
+        """
+        if qlod is None:
+            qlod = self.get_lod()
+        if r_format is None:
+            r_format = Format.json
+        if r_format == Format.csv:
+            csv_output = CSV.toCSV(qlod)
+            return csv_output
+        elif r_format in [Format.latex, Format.github, Format.mediawiki, Format.html]:
+            doc = self.query.documentQueryResult(
+                qlod, tablefmt=str(r_format), floatfmt=".0f"
+            )
+            return doc.asText()
+        elif r_format == Format.json:
+            return json.dumps(qlod, indent=2, sort_keys=True, default=str)
+        return None  # In case no format is matched or needed
+
+    def set_limit(self, limit: int = None):
+        """
+        set the limit of my query
+        
+        Args:
+            limit(int): the limit to set - default: None
+        """
+        if limit:
+            sparql_query = self.query.query
+            # @TODO - this is too naive for cases where
+            # there are SPARQL elements hat have a "limit" in the name e.g. "height_limit"
+            # or if there is a LIMIT in a subquery
+            if "limit" in sparql_query or "LIMIT" in sparql_query:
+                sparql_query = re.sub(
+                    r"(limit|LIMIT)\s+(\d+)", f"LIMIT {limit}", sparql_query
+                )
+            else:
+                sparql_query += f"\nLIMIT {limit}"
+            self.query.query = sparql_query
+
 
 class NamedQueryManager:
     """
@@ -72,7 +238,7 @@ class NamedQueryManager:
         if db_path is None:
             db_path = NamedQueryManager.get_cache_path()
         self.debug = debug
-        self.sql_db = SQLDB(dbname=db_path,check_same_thread=False, debug=debug)
+        self.sql_db = SQLDB(dbname=db_path, check_same_thread=False, debug=debug)
         self.endpoints = EndpointManager.getEndpoints(lang="sparql")
 
     @classmethod
@@ -103,23 +269,33 @@ class NamedQueryManager:
         nqm = NamedQueryManager(debug=debug)
         path_obj = Path(db_path)
         if not path_obj.exists() or path_obj.stat().st_size == 0:
-            sample_queries = cls.get_samples()
-            list_of_records = []
-            for _query_name, named_query in sample_queries.items():
-                record = {
-                    "namespace": named_query.namespace,
-                    "name": named_query.name,
-                    "title": named_query.title,
-                    "description": named_query.description,
-                    "sparql": named_query.sparql,
-                }
-                list_of_records.append(record)
+            sample_records=cls.get_sample_records()
             entityInfo = EntityInfo(
-                list_of_records, name="NamedQuery", primaryKey="query_id"
+                sample_records, name="NamedQuery", primaryKey="query_id"
             )
-            nqm.sql_db.createTable(list_of_records, "NamedQuery", withDrop=True)
-            nqm.sql_db.store(list_of_records, entityInfo)
+            nqm.sql_db.createTable(sample_records, "NamedQuery", withDrop=True)
+            nqm.sql_db.store(sample_records, entityInfo)
         return nqm
+    
+    def store(self,lod):
+        """
+        store the given list of dicts
+        """
+        sample_records=NamedQueryManager.get_sample_records()
+        entityInfo = EntityInfo(
+            sample_records, name="NamedQuery", primaryKey="query_id"
+        )
+        self.sql_db.store(lod, entityInfo,fixNone=True,replace=True)
+        
+    @classmethod
+    def get_sample_records(cls):
+        sample_queries = cls.get_samples()
+        list_of_records = []
+        for _query_name, named_query in sample_queries.items():
+            record=named_query.as_record()
+            list_of_records.append(record)
+        return list_of_records
+
 
     @classmethod
     def get_samples(cls) -> dict[str, NamedQuery]:
@@ -130,6 +306,7 @@ class NamedQueryManager:
             "wikidata-examples.cats": NamedQuery(
                 namespace="wikidata-examples",
                 name="cats",
+                url="https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples#Cats",
                 title="Cats on Wikidata",
                 description="This query retrieves all items classified under 'house cat' (Q146) on Wikidata.",
                 sparql="""
@@ -143,6 +320,7 @@ WHERE {
             "wikidata-examples.horses": NamedQuery(
                 namespace="wikidata-examples",
                 name="horses",
+                url="https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples#Horses_(showing_some_info_about_them)",
                 title="Horses on Wikidata",
                 description="This query retrieves information about horses, including parents, gender, and approximate birth and death years.",
                 sparql="""
@@ -165,14 +343,20 @@ ORDER BY ?horse
         }
         return samples
 
-    def get_sparql(
-        self,
-        name: str,
-        namespace: str = "wikidata-examples",
-        endpoint_name: str = "wikidata",
-    ) -> str:
-        sql_query = f"""SELECT 
-    sparql 
+    def lookup(self, name: str, namespace: str) -> NamedQuery:
+        """
+        lookup the named query for the given name and namespace
+
+
+        Args:
+            name (str): The name of the named query to execute.
+            namespace (str): The namespace of the named query, default is 'wikidata-examples'.
+
+        Returns:
+            NamedQuery: the named query
+        """
+        sql_query = """SELECT 
+    *
 FROM 
     NamedQuery 
 WHERE 
@@ -188,96 +372,49 @@ WHERE
             msg = f"multiple entries ({query_count}) for name '{name}' and namespace '{namespace}'"
             raise ValueError(msg)
 
-        sparql_query = query_records[0]["sparql"]
-        return sparql_query
+        record = query_records[0]
+        named_query = NamedQuery.from_record(record)
+        return named_query
 
-    def query(
+    def get_query(
         self,
         name: str,
         namespace: str = "wikidata-examples",
         endpoint_name: str = "wikidata",
-    ):
+        limit: int = None,
+    ) -> QueryBundle:
         """
-        Execute a named SPARQL query using a specified endpoint and return the results.
+        get the query for the given parameters
 
         Args:
             name (str): The name of the named query to execute.
             namespace (str): The namespace of the named query, default is 'wikidata-examples'.
             endpoint_name (str): The name of the endpoint to send the SPARQL query to, default is 'wikidata'.
+            limit(int): the query limit (if any)
 
         Returns:
-            Dict[str, Any]: The results of the SPARQL query in JSON format.
-
-        Raises:
-            ValueError: If no named query matches the given name and namespace.
-            Exception: If the SPARQL query execution fails or the endpoint returns an error.
+            QueryBundle: named_query, query and endpoint
         """
-        sparql_query = self.get_sparql(name, namespace, endpoint_name)
-
-        if not endpoint_name in self.endpoints:
+        named_query = self.lookup(name, namespace)
+        if endpoint_name not in self.endpoints:
             msg = f"Invalid endpoint {endpoint_name}"
             ValueError(msg)
         endpoint = self.endpoints.get(endpoint_name)
-        sparql = SPARQL(endpoint.endpoint)
-        lod = sparql.queryAsListOfDicts(sparql_query)
-        return lod
-
-    def format_result(
-        self,
-        qlod: List[Dict[str, Any]],
-        query: Query,
-        r_format_str: str,
-        endpoint_name: str = "wikidata",
-    ) -> Optional[str]:
-        """
-        Formats the query results based on the specified format and prints them.
-
-        Args:
-            qlod (List[Dict[str, Any]]): The list of dictionaries that represent the query results.
-            query (Query): The query object which contains details like the endpoint and the database.
-            r_format_str: The format in which to print the results.
-            endpoint_name (str): The name of the endpoint to be used, defaults to "wikidata".
-
-        Returns:
-            Optional[str]: The formatted string representation of the query results, or None if printed directly.
-        """
-        # Retrieve endpoint configuration using the provided endpoint name
+        sparql_query = named_query.sparql
+        query = Query(
+            name=name,
+            query=sparql_query,
+            lang="sparql",
+            endpoint=endpoint.endpoint,
+            limit=limit,
+        )
         endpointConf = self.endpoints.get(endpoint_name, Endpoint.getDefault())
         query.tryItUrl = endpointConf.website
         query.database = endpointConf.database
-        r_format = Format[r_format_str] 
-
-        if r_format == Format.csv:
-            csv_output = CSV.toCSV(qlod)
-            return csv_output
-        elif r_format in [Format.latex, Format.github, Format.mediawiki, Format.html]:
-            doc = query.documentQueryResult(
-                qlod, tablefmt=str(r_format), floatfmt=".0f"
-            )
-            return doc.asText()
-        elif r_format == Format.json:
-            return json.dumps(qlod, indent=2, sort_keys=True, default=str)
-        return None  # In case no format is matched or needed
-
-    def print_result(
-        self,
-        qlod: List[Dict[str, Any]],
-        query: Query,
-        r_format_str: str,
-        endpoint_name: str = "wikidata",
-    ) -> None:
-        """
-        Prints the formatted results of a query.
-
-        Args:
-            qlod (List[Dict[str, Any]]): The list of dictionaries that represent the query results.
-            query (Query): The query object which contains details such as the endpoint and the database.
-            r_format_str: The format in which to print the results.
-            endpoint_name (str): The name of the endpoint to be used, defaults to "wikidata".
-
-        Returns:
-            None
-        """
-        result_str = self.format_result(qlod, query, r_format_str, endpoint_name)
-        if result_str is not None:
-            print(result_str)
+        query_bundle = QueryBundle(
+            named_query=named_query, 
+            query=query, 
+            endpoint=endpoint
+        )
+        query_bundle.set_limit(limit)
+        return query_bundle