snapquery 0.0.1__tar.gz → 0.0.2__tar.gz

{snapquery-0.0.1 → snapquery-0.0.2}/.project

@@ -5,6 +5,7 @@
 	<projects>
 		<project>nicegui</project>
 		<project>nicegui_widgets</project>
+		<project>pyLoDStorage</project>
 	</projects>
 	<buildSpec>
 		<buildCommand>

{snapquery-0.0.1 → snapquery-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snapquery
-Version: 0.0.1
+Version: 0.0.2
 Summary: snapquery: Introduce Named Queries and Named Query Middleware to wikidata
 Project-URL: Home, https://github.com/WolfgangFahl/snapquery
 Project-URL: Documentation, https://wiki.bitplan.com/index.php/snapquery
@@ -25,7 +25,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: User Interfaces
 Requires-Python: >=3.9
 Requires-Dist: ngwidgets>=0.14.2
-Requires-Dist: pylodstorage>=0.10.3
+Requires-Dist: pylodstorage>=0.10.4
 Provides-Extra: test
 Requires-Dist: green; extra == 'test'
 Description-Content-Type: text/markdown

{snapquery-0.0.1 → snapquery-0.0.2}/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
 	# https://github.com/WolfgangFahl/nicegui_widgets
 	"ngwidgets>=0.14.2",
   # https://pypi.org/project/pyLodStorage/
-  "pyLodStorage>=0.10.3"
+  "pyLodStorage>=0.10.4"
 ]
 
 requires-python = ">=3.9"

snapquery-0.0.2/snapquery/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.2"

snapquery-0.0.2/snapquery/snapquery_cmd.py

@@ -0,0 +1,90 @@
+"""
+Created on 2024-05-03
+
+@author: wf
+"""
+import sys
+from argparse import ArgumentParser
+
+from lodstorage.query import EndpointManager, Format, Query
+from ngwidgets.cmd import WebserverCmd
+
+from snapquery.snapquery_core import NamedQueryManager
+from snapquery.snapquery_webserver import SnapQueryWebServer
+
+
+class SnapQueryCmd(WebserverCmd):
+    """
+    Command line for diagrams server
+    """
+
+    def getArgParser(self, description: str, version_msg) -> ArgumentParser:
+        """
+        override the default argparser call
+        """
+        parser = super().getArgParser(description, version_msg)
+        # see https://github.com/WolfgangFahl/pyLoDStorage/blob/master/lodstorage/querymain.py
+        parser.add_argument(
+            "-ep",
+            "--endpointPath",
+            default=None,
+            help="path to yaml file to configure endpoints to use for queries",
+        )
+        parser.add_argument(
+            "-en",
+            "--endpointName",
+            default="wikidata",
+            help=f"Name of the endpoint to use for queries - use --listEndpoints to list available endpoints",
+        )
+        parser.add_argument(
+            "-le",
+            "--listEndpoints",
+            action="store_true",
+            help="show the list of available endpoints",
+        )
+        parser.add_argument("-qn", "--queryName", help="run a named query")
+        parser.add_argument("-f", "--format", type=Format, choices=list(Format))
+        return parser
+
+    def handle_args(self) -> bool:
+        """
+        handle the command line args
+        """
+        # Call the superclass handle_args to maintain base class behavior
+        handled = super().handle_args()
+        endpoints = EndpointManager.getEndpoints(self.args.endpointPath, lang="sparql")
+        # Check if listing of endpoints is requested
+        if self.args.listEndpoints:
+            # List endpoints
+            for endpoint in endpoints.values():
+                print(endpoint)
+            handled = True  # Operation handled
+        elif self.args.queryName is not None:
+            nqm = NamedQueryManager()
+            name = self.args.queryName
+            endpoint_name = self.args.endpointName
+            r_format = self.args.format
+            sparql_query = nqm.get_sparql(name=name, endpoint_name=endpoint_name)
+            qlod = nqm.query(endpoint_name=endpoint_name, name=name)
+            query = Query(name=name, query=sparql_query, lang="sparql")
+            nqm.format_result(qlod, query, r_format, endpoint_name=endpoint_name)
+        return handled
+
+
+def main(argv: list = None):
+    """
+    main call
+    """
+    cmd = SnapQueryCmd(
+        config=SnapQueryWebServer.get_config(),
+        webserver_cls=SnapQueryWebServer,
+    )
+    exit_code = cmd.cmd_main(argv)
+    return exit_code
+
+
+DEBUG = 0
+if __name__ == "__main__":
+    if DEBUG:
+        sys.argv.append("-d")
+    sys.exit(main())

snapquery-0.0.2/snapquery/snapquery_core.py

@@ -0,0 +1,283 @@
+"""
+Created on 2024-05-03
+
+@author: wf
+"""
+import json
+import os
+from dataclasses import field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from lodstorage.csv import CSV
+from lodstorage.query import Endpoint, EndpointManager, Format, Query
+from lodstorage.sparql import SPARQL
+from lodstorage.sql import SQLDB, EntityInfo
+from lodstorage.yamlable import lod_storable
+
+
+@lod_storable
+class NamedQuery:
+    """
+    A named query that encapsulates the details and SPARQL query for a specific purpose.
+
+    Attributes:
+        namespace (str): The namespace of the query, which helps in categorizing the query.
+        name (str): The unique name or identifier of the query within its namespace.
+        title (str): A brief one-line title that describes the query.
+        description (str): A detailed multiline description of what the query does and the data it accesses.
+        sparql (str): The SPARQL query string. This might be hidden in future to encapsulate query details.
+        query_id (str): A unique identifier for the query, generated from namespace and name, used as a primary key.
+    """
+
+    query_id: str = field(init=False)
+
+    # namespace
+    namespace: str
+    # name/id
+    name: str
+    # one line title
+    title: str
+    # multiline description
+    description: str
+    # sparql query (to be hidden later)
+    sparql: str
+
+    def __post_init__(self):
+        """
+        Post-initialization processing to construct a unique identifier for the query
+        based on its namespace and name.
+        """
+        self.query_id = f"{self.namespace}.{self.name}"
+
+
+class NamedQueryManager:
+    """
+    Manages the storage, retrieval, and execution of named SPARQL queries.
+    """
+
+    def __init__(self, db_path: str = None, debug: bool = False):
+        """
+        Initializes the NamedQueryManager with a specific database path and a debug mode.
+
+        Args:
+            db_path (Optional[str]): The file path to the SQLite database. If None, the default cache path is used.
+            debug (bool): If True, enables debug mode which may provide additional logging and error reporting.
+
+        Attributes:
+            debug (bool): Stores the debug state.
+            sql_db (SQLDB): An instance of SQLDB to manage the SQLite database interactions.
+            endpoints (dict): A dictionary of SPARQL endpoints configured for use.
+        """
+        if db_path is None:
+            db_path = NamedQueryManager.get_cache_path()
+        self.debug = debug
+        self.sql_db = SQLDB(dbname=db_path,check_same_thread=False, debug=debug)
+        self.endpoints = EndpointManager.getEndpoints(lang="sparql")
+
+    @classmethod
+    def get_cache_path(cls) -> str:
+        home = str(Path.home())
+        cache_dir = f"{home}/.solutions/snapquery/storage"
+        os.makedirs(cache_dir, exist_ok=True)
+        cache_path = f"{cache_dir}/named_queries.db"
+        return cache_path
+
+    @classmethod
+    def from_samples(
+        cls, db_path: Optional[str] = None, debug: bool = False
+    ) -> "NamedQueryManager":
+        """
+        Creates and returns an instance of NamedQueryManager, optionally initializing it from sample data.
+
+        Args:
+            db_path (Optional[str]): Path to the database file. If None, the default path is used.
+            debug (bool): If True, enables debug mode which may provide additional logging.
+
+        Returns:
+            NamedQueryManager: An instance of the manager initialized with the database at `db_path`.
+        """
+        if db_path is None:
+            db_path = cls.get_cache_path()
+
+        nqm = NamedQueryManager(debug=debug)
+        path_obj = Path(db_path)
+        if not path_obj.exists() or path_obj.stat().st_size == 0:
+            sample_queries = cls.get_samples()
+            list_of_records = []
+            for _query_name, named_query in sample_queries.items():
+                record = {
+                    "namespace": named_query.namespace,
+                    "name": named_query.name,
+                    "title": named_query.title,
+                    "description": named_query.description,
+                    "sparql": named_query.sparql,
+                }
+                list_of_records.append(record)
+            entityInfo = EntityInfo(
+                list_of_records, name="NamedQuery", primaryKey="query_id"
+            )
+            nqm.sql_db.createTable(list_of_records, "NamedQuery", withDrop=True)
+            nqm.sql_db.store(list_of_records, entityInfo)
+        return nqm
+
+    @classmethod
+    def get_samples(cls) -> dict[str, NamedQuery]:
+        """
+        get samples
+        """
+        samples = {
+            "wikidata-examples.cats": NamedQuery(
+                namespace="wikidata-examples",
+                name="cats",
+                title="Cats on Wikidata",
+                description="This query retrieves all items classified under 'house cat' (Q146) on Wikidata.",
+                sparql="""
+SELECT ?item ?itemLabel
+WHERE {
+  ?item wdt:P31 wd:Q146. # Must be a cat
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
+}
+""",
+            ),
+            "wikidata-examples.horses": NamedQuery(
+                namespace="wikidata-examples",
+                name="horses",
+                title="Horses on Wikidata",
+                description="This query retrieves information about horses, including parents, gender, and approximate birth and death years.",
+                sparql="""
+SELECT DISTINCT ?horse ?horseLabel ?mother ?motherLabel ?father ?fatherLabel 
+(year(?birthdate) as ?birthyear) (year(?deathdate) as ?deathyear) ?genderLabel
+WHERE {
+  ?horse wdt:P31/wdt:P279* wd:Q726 .     # Instance and subclasses of horse (Q726)
+  OPTIONAL{?horse wdt:P25 ?mother .}     # Mother
+  OPTIONAL{?horse wdt:P22 ?father .}     # Father
+  OPTIONAL{?horse wdt:P569 ?birthdate .} # Birth date
+  OPTIONAL{?horse wdt:P570 ?deathdate .} # Death date
+  OPTIONAL{?horse wdt:P21 ?gender .}     # Gender
+  SERVICE wikibase:label {
+    bd:serviceParam wikibase:language "[AUTO_LANGUAGE],fr,ar,be,bg,bn,ca,cs,da,de,el,en,es,et,fa,fi,he,hi,hu,hy,id,it,ja,jv,ko,nb,nl,eo,pa,pl,pt,ro,ru,sh,sk,sr,sv,sw,te,th,tr,uk,yue,vec,vi,zh"
+  }
+}
+ORDER BY ?horse
+""",
+            ),
+        }
+        return samples
+
+    def get_sparql(
+        self,
+        name: str,
+        namespace: str = "wikidata-examples",
+        endpoint_name: str = "wikidata",
+    ) -> str:
+        sql_query = f"""SELECT 
+    sparql 
+FROM 
+    NamedQuery 
+WHERE 
+    name = ? AND namespace = ?"""
+        query_records = self.sql_db.query(sql_query, (name, namespace))
+
+        if not query_records:
+            msg = f"NamedQuery not found for the specified name '{name}' and namespace '{namespace}'."
+            raise ValueError(msg)
+
+        query_count = len(query_records)
+        if query_count != 1:
+            msg = f"multiple entries ({query_count}) for name '{name}' and namespace '{namespace}'"
+            raise ValueError(msg)
+
+        sparql_query = query_records[0]["sparql"]
+        return sparql_query
+
+    def query(
+        self,
+        name: str,
+        namespace: str = "wikidata-examples",
+        endpoint_name: str = "wikidata",
+    ):
+        """
+        Execute a named SPARQL query using a specified endpoint and return the results.
+
+        Args:
+            name (str): The name of the named query to execute.
+            namespace (str): The namespace of the named query, default is 'wikidata-examples'.
+            endpoint_name (str): The name of the endpoint to send the SPARQL query to, default is 'wikidata'.
+
+        Returns:
+            Dict[str, Any]: The results of the SPARQL query in JSON format.
+
+        Raises:
+            ValueError: If no named query matches the given name and namespace.
+            Exception: If the SPARQL query execution fails or the endpoint returns an error.
+        """
+        sparql_query = self.get_sparql(name, namespace, endpoint_name)
+
+        if not endpoint_name in self.endpoints:
+            msg = f"Invalid endpoint {endpoint_name}"
+            ValueError(msg)
+        endpoint = self.endpoints.get(endpoint_name)
+        sparql = SPARQL(endpoint.endpoint)
+        lod = sparql.queryAsListOfDicts(sparql_query)
+        return lod
+
+    def format_result(
+        self,
+        qlod: List[Dict[str, Any]],
+        query: Query,
+        r_format_str: str,
+        endpoint_name: str = "wikidata",
+    ) -> Optional[str]:
+        """
+        Formats the query results based on the specified format and prints them.
+
+        Args:
+            qlod (List[Dict[str, Any]]): The list of dictionaries that represent the query results.
+            query (Query): The query object which contains details like the endpoint and the database.
+            r_format_str: The format in which to print the results.
+            endpoint_name (str): The name of the endpoint to be used, defaults to "wikidata".
+
+        Returns:
+            Optional[str]: The formatted string representation of the query results, or None if printed directly.
+        """
+        # Retrieve endpoint configuration using the provided endpoint name
+        endpointConf = self.endpoints.get(endpoint_name, Endpoint.getDefault())
+        query.tryItUrl = endpointConf.website
+        query.database = endpointConf.database
+        r_format = Format[r_format_str] 
+
+        if r_format == Format.csv:
+            csv_output = CSV.toCSV(qlod)
+            return csv_output
+        elif r_format in [Format.latex, Format.github, Format.mediawiki, Format.html]:
+            doc = query.documentQueryResult(
+                qlod, tablefmt=str(r_format), floatfmt=".0f"
+            )
+            return doc.asText()
+        elif r_format == Format.json:
+            return json.dumps(qlod, indent=2, sort_keys=True, default=str)
+        return None  # In case no format is matched or needed
+
+    def print_result(
+        self,
+        qlod: List[Dict[str, Any]],
+        query: Query,
+        r_format_str: str,
+        endpoint_name: str = "wikidata",
+    ) -> None:
+        """
+        Prints the formatted results of a query.
+
+        Args:
+            qlod (List[Dict[str, Any]]): The list of dictionaries that represent the query results.
+            query (Query): The query object which contains details such as the endpoint and the database.
+            r_format_str: The format in which to print the results.
+            endpoint_name (str): The name of the endpoint to be used, defaults to "wikidata".
+
+        Returns:
+            None
+        """
+        result_str = self.format_result(qlod, query, r_format_str, endpoint_name)
+        if result_str is not None:
+            print(result_str)

snapquery-0.0.2/snapquery/snapquery_webserver.py

@@ -0,0 +1,106 @@
+"""
+Created on 2024-05-03
+@author: wf
+"""
+from fastapi import Request, Header,HTTPException
+from fastapi.responses import HTMLResponse
+from lodstorage.query import Query
+from ngwidgets.input_webserver import InputWebserver, InputWebSolution, WebserverConfig
+from nicegui import app
+from nicegui.client import Client
+
+from snapquery.snapquery_core import NamedQueryManager
+from snapquery.version import Version
+
+
+class SnapQueryWebServer(InputWebserver):
+    """
+    server to supply named Queries
+    """
+
+    @classmethod
+    def get_config(cls) -> WebserverConfig:
+        """
+        get the configuration for this Webserver
+        """
+        copy_right = ""
+        config = WebserverConfig(
+            short_name="snapquery",
+            copy_right=copy_right,
+            version=Version(),
+            default_port=9862,
+        )
+        server_config = WebserverConfig.get(config)
+        server_config.solution_class = SnapQuerySolution
+        return server_config
+
+    def __init__(self):
+        """Constructs all the necessary attributes for the WebServer object."""
+        InputWebserver.__init__(self, config=SnapQueryWebServer.get_config())
+        self.nqm = NamedQueryManager.from_samples()
+
+        @app.get("/query/{namespace}/{name}", response_class=HTMLResponse)
+        def query(namespace: str, name: str):
+            """
+            Executes a SPARQL query by name within a specified namespace, formats the results, and returns them as an HTML response.
+
+            Args:
+                namespace (str): The namespace identifying the group or category of the query.
+                name (str): The specific name of the query to be executed.
+
+            Returns:
+                HTMLResponse: The HTML formatted response containing the results of the query execution.
+
+            Raises:
+                HTTPException: If the query cannot be found or fails to execute.
+            """
+            content=self.query(namespace,name)
+            if not content:
+                raise HTTPException(status_code=500, detail="Could not create result")
+
+            # Return the content as an HTML response
+            return HTMLResponse(content)
+        
+    def query(self,namespace:str,name:str):
+        endpoint_name = "wikidata"
+        # content negotiation
+        # Determine response format by extension in the name or Accept header
+        if '.' in name:
+            r_format = name.split('.')[-1]
+            name = name[:name.rfind('.')]
+        else:
+            r_format = "html"
+
+        # Retrieve the SPARQL query string using the namespace and name.
+        try:
+            sparql_query = self.nqm.get_sparql(name, namespace, endpoint_name)
+            qlod = self.nqm.query(
+                name=name, namespace=namespace, endpoint_name=endpoint_name
+            )
+            query = Query(name=name, query=sparql_query, lang="sparql")
+        except Exception as e:
+            # Handling specific exceptions can be more detailed based on what nqm.get_sparql and nqm.query can raise
+            raise HTTPException(status_code=404, detail=str(e))
+
+        # Format the results and generate HTML content
+        content = self.nqm.format_result(
+            qlod, query, r_format, endpoint_name=endpoint_name
+        )
+        return content
+
+
+class SnapQuerySolution(InputWebSolution):
+    """
+    the Snap Query solution
+    """
+
+    def __init__(self, webserver: SnapQueryWebServer, client: Client):
+        """
+        Initialize the solution
+
+        Calls the constructor of the base solution
+        Args:
+            webserver (SnapQueryWebServer): The webserver instance associated with this context.
+            client (Client): The client instance this context is associated with.
+        """
+        super().__init__(webserver, client)  # Call to the superclass constructor

{snapquery-0.0.1 → snapquery-0.0.2}/tests/test_endpoints.py

@@ -1,24 +1,25 @@
-'''
+"""
 Created on 2024-05-03
 
 @author: wf
-'''
-from ngwidgets.basetest import Basetest
+"""
 from lodstorage.query import EndpointManager
+from ngwidgets.basetest import Basetest
+
 
 class TestEndpoints(Basetest):
     """
     test endpoint handling according to https://github.com/WolfgangFahl/snapquery/issues/1
     """
-    
+
     def setUp(self, debug=False, profile=True):
         Basetest.setUp(self, debug=debug, profile=profile)
-        
+
     def testEndpoints(self):
         """
         test the endpoint handling
         """
-        em=EndpointManager()
-        ep_names=em.getEndpointNames()
+        em = EndpointManager()
+        ep_names = em.getEndpointNames()
         self.assertTrue("wikidata" in ep_names)
         pass

snapquery-0.0.2/tests/test_namedqueries.py

@@ -0,0 +1,37 @@
+"""
+Created on 2024-05-03
+
+@author: wf
+"""
+import json
+
+from ngwidgets.basetest import Basetest
+
+from snapquery.snapquery_core import NamedQueryManager
+
+
+class TestNamedQueryManager(Basetest):
+    """
+    test the named query Manage
+    """
+
+    def setUp(self, debug=False, profile=True):
+        Basetest.setUp(self, debug=debug, profile=profile)
+
+    def testNamedQueries(self):
+        """
+        test getting a named query manager
+        """
+        db_path = "/tmp/named_queries.db"
+        nqm = NamedQueryManager.from_samples(db_path=db_path)
+        for name, ex_count in [("x-invalid", -1), ("cats", 223)]:
+            try:
+                lod = nqm.query(name)
+                if self.debug:
+                    print(f"{name}:")
+                    print(json.dumps(lod, default=str, indent=2))
+                self.assertEqual(ex_count, len(lod))
+            except Exception as ex:
+                if self.debug:
+                    print(f"{name}:Exception {str(ex)}")
+                self.assertEqual(-1, ex_count)

snapquery-0.0.2/tests/test_restful_api.py

@@ -0,0 +1,34 @@
+"""
+Created on 2024-05-03
+
+@author: wf
+"""
+from snapquery.snapquery_webserver import SnapQueryWebServer
+from snapquery.snapquery_cmd import SnapQueryCmd
+from ngwidgets.webserver_test import WebserverTest
+
+
+class TestRestFullApi(WebserverTest):
+    """
+    test endpoint handling according to https://github.com/WolfgangFahl/snapquery/issues/1
+    """
+    def setUp(self, debug=True, profile=True):
+        server_class = SnapQueryWebServer
+        cmd_class = SnapQueryCmd
+        WebserverTest.setUp(self, server_class, cmd_class, debug=debug, profile=profile)
+
+   
+    def testDemoWebserver(self):
+        """
+        test API docs access
+        """
+        # self.debug=True
+        html = self.getHtml("/docs")
+        self.assertTrue("Swagger" in html)
+        
+    def testQueryApi(self):
+        for r_format in ["mediawiki","github","latex","html"]:
+            path=f"/query/wikidata-examples/cats.{r_format}"
+            html=self.getHtml(path)
+            if self.debug:
+                print(html)

snapquery-0.0.1/snapquery/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.1"

snapquery-0.0.1/snapquery/snapquery_cmd.py

@@ -1,43 +0,0 @@
-"""
-Created on 2024-05-03
-
-@author: wf
-"""
-import sys
-from argparse import ArgumentParser
-
-from ngwidgets.cmd import WebserverCmd
-
-from snapquery.snapquery_webserver import SnapQueryWebServer
-
-
-class SnapQueryCmd(WebserverCmd):
-    """
-    Command line for diagrams server
-    """
-
-    def getArgParser(self, description: str, version_msg) -> ArgumentParser:
-        """
-        override the default argparser call
-        """
-        parser = super().getArgParser(description, version_msg)
-        return parser
-
-
-def main(argv: list = None):
-    """
-    main call
-    """
-    cmd = SnapQueryCmd(
-        config=SnapQueryWebServer.get_config(),
-        webserver_cls=SnapQueryWebServer,
-    )
-    exit_code = cmd.cmd_main(argv)
-    return exit_code
-
-
-DEBUG = 0
-if __name__ == "__main__":
-    if DEBUG:
-        sys.argv.append("-d")
-    sys.exit(main())

snapquery-0.0.1/snapquery/snapquery_core.py

@@ -1,5 +0,0 @@
-"""
-Created on 2024-05-03
-
-@author: wf
-"""

snapquery-0.0.1/snapquery/snapquery_webserver.py

@@ -1,51 +0,0 @@
-"""
-Created on 2024-05-03
-@author: wf
-"""
-from ngwidgets.input_webserver import InputWebserver, InputWebSolution, WebserverConfig
-from nicegui.client import Client
-
-from snapquery.version import Version
-
-
-class SnapQueryWebServer(InputWebserver):
-    """
-    server to supply named Queries
-    """
-
-    @classmethod
-    def get_config(cls) -> WebserverConfig:
-        """
-        get the configuration for this Webserver
-        """
-        copy_right = ""
-        config = WebserverConfig(
-            short_name="snapquery",
-            copy_right=copy_right,
-            version=Version(),
-            default_port=9862,
-        )
-        server_config = WebserverConfig.get(config)
-        server_config.solution_class = SnapQuerySolution
-        return server_config
-
-    def __init__(self):
-        """Constructs all the necessary attributes for the WebServer object."""
-        InputWebserver.__init__(self, config=SnapQueryWebServer.get_config())
-
-
-class SnapQuerySolution(InputWebSolution):
-    """
-    the Snap Query solution
-    """
-
-    def __init__(self, webserver: SnapQueryWebServer, client: Client):
-        """
-        Initialize the solution
-
-        Calls the constructor of the base solution
-        Args:
-            webserver (SnapQueryWebServer): The webserver instance associated with this context.
-            client (Client): The client instance this context is associated with.
-        """
-        super().__init__(webserver, client)  # Call to the superclass constructor