snapquery 0.2.3__tar.gz → 0.2.5__tar.gz

{snapquery-0.2.3 → snapquery-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snapquery
-Version: 0.2.3
+Version: 0.2.5
 Summary: Just Query wikidata and other SPARQL endpoints by query name - a frontend to Introduce Named Queries and Named Query Middleware to wikidata and other SPARQL endpoints 
 Project-URL: Home, https://github.com/WolfgangFahl/snapquery
 Project-URL: Documentation, https://wiki.bitplan.com/index.php/snapquery
@@ -26,14 +26,14 @@ Classifier: Topic :: Software Development :: User Interfaces
 Requires-Python: >=3.10
 Requires-Dist: mwoauth>=0.4.0
 Requires-Dist: nameparser>=1.1.3
-Requires-Dist: ngwidgets>=0.30.0
+Requires-Dist: ngwidgets>=0.30.3
 Requires-Dist: nicegui[plotly]==2.24.2
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: py-3rdparty-mediawiki>=0.18.1
 Requires-Dist: py-ez-wikidata>=0.3.2
-Requires-Dist: pybasemkit>=0.1.3
-Requires-Dist: pylodstorage>=0.18.5
-Requires-Dist: pyopensourceprojects>=0.3.0
+Requires-Dist: pybasemkit>=0.1.6
+Requires-Dist: pylodstorage>=0.18.6
+Requires-Dist: pyopensourceprojects>=0.4.1
 Requires-Dist: python-slugify>=8.0.4
 Requires-Dist: ratelimit>=2.2.1
 Requires-Dist: requests

{snapquery-0.2.3 → snapquery-0.2.5}/pyproject.toml

@@ -22,11 +22,11 @@ dependencies = [
 	# https://pypi.org/project/pybasemkit/
 	# https://github.com/WolfgangFahl/pybasemkit
 	# Python base module kit: YAML/JSON I/O, structured logging, CLI tooling, shell execution, and remote pydevd debug support.
-	"pybasemkit>=0.1.3",
+	"pybasemkit>=0.1.6",
 	# https://github.com/WolfgangFahl/nicegui_widgets
-	"ngwidgets>=0.30.0",
+	"ngwidgets>=0.30.3",
     # https://pypi.org/project/pyLodStorage/
-    "pyLodStorage>=0.18.5",
+    "pyLodStorage>=0.18.6",
     # https://pypi.org/project/py-3rdparty-mediawiki/
     "py-3rdparty-mediawiki>=0.18.1",
     "wikitextparser",
@@ -51,7 +51,7 @@ dependencies = [
     # https://pypi.org/project/nameparser/
     "nameparser>=1.1.3",
     # https://github.com/WolfgangFahl/pyOpenSourceProjects
-    "pyOpenSourceProjects>=0.3.0"
+    "pyOpenSourceProjects>=0.4.1"
 ]
 
 requires-python = ">=3.10"

snapquery-0.2.5/snapquery/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.5"

{snapquery-0.2.3 → snapquery-0.2.5}/snapquery/github_access.py

@@ -4,9 +4,10 @@ GitHub API client for accessing repository contents.
 Created on 2025-12-02
 @author: wf
 """
+
 import json
-from  pathlib import Path
-from typing import List, Dict, Any, Optional, Union
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
 
 import requests
 
@@ -16,28 +17,38 @@ class GitHub:
     A simple GitHub API client for accessing repository contents.
     """
 
-    def __init__(self, owner: str, repo: str, token: Optional[str] = None, session: Optional[requests.Session] = None):
+    def __init__(
+        self,
+        owner: str,
+        repo: str,
+        branch: Optional[str] = None,
+        token: Optional[str] = None,
+        session: Optional[requests.Session] = None
+    ):
         """
         Initialize GitHub client.
 
         Args:
             owner: Repository owner (username or organization)
             repo: Repository name
+            branch: Optional specific branch or commit SHA (default: default repo branch)
             token: Optional GitHub API token for authentication
             session: Optional custom requests.Session
         """
         self.owner = owner
         self.repo = repo
-        self.token = token
+        self.branch = branch
+
+
         self.base_url = f"https://api.github.com/repos/{owner}/{repo}"
-        # Use provided token or read from file (compatible with GitHubApi)
+
+        # Use provided token or read from file
         self.token = token if token is not None else self._read_token()
 
         # Use custom session or create new one
         self.session = session or requests.Session()
 
 
-
     def _headers(self) -> Dict[str, str]:
         headers = {"Accept": "application/vnd.github.v3+json"}
         if self.token:
@@ -55,14 +66,13 @@ class GitHub:
         token_path = Path.home() / ".github" / "access_token.json"
         if token_path.exists():
             try:
-                with open(token_path, 'r') as f:
+                with open(token_path, "r") as f:
                     data = json.load(f)
                     return data.get("access_token")
             except (json.JSONDecodeError, OSError):
                 pass
         return None
 
-
     def get_contents(self, path: str = "") -> Union[List[Dict[str, Any]], Dict[str, Any]]:
         """
         Get contents of a directory or file from the repository.
@@ -74,10 +84,15 @@ class GitHub:
             List of dictionaries for directories, or content for files
         """
         url = f"{self.base_url}/contents/{path}"
-        response = self.session.get(url, headers=self._headers(), timeout=30)
+        params = {}
+        if self.branch:
+            params["ref"] = self.branch
+
+        response = self.session.get(url, headers=self._headers(), params=params, timeout=30)
         response.raise_for_status()
         return response.json()
 
+
     def list_files_recursive(self, path: str = "", suffix: Optional[str] = None) -> List[Dict[str, Any]]:
         """
         Recursively list files under a given path. Optionally filter by file suffix.
@@ -116,4 +131,4 @@ class GitHub:
         """
         response = self.session.get(download_url, headers=self._headers(), timeout=30)
         response.raise_for_status()
-        return response.text
+        return response.text

snapquery-0.2.5/snapquery/qlever.py

@@ -0,0 +1,187 @@
+"""
+Created on 2024-06-20
+
+@author: wf
+"""
+
+import re
+from typing import Dict, List
+
+from bs4 import BeautifulSoup
+from osprojects.osproject import OsProject, Ticket
+from tqdm import tqdm
+
+from snapquery.snapquery_core import NamedQuery, NamedQuerySet
+from snapquery.wd_short_url import ShortUrl
+
+
+class QLeverUrl(ShortUrl):
+    """
+    Handles operations related to QLever short URLs.
+    """
+
+    def __init__(self, short_url: str):
+        super().__init__(short_url, scheme="https", netloc="qlever.cs.uni-freiburg.de")
+        self.sparql = None
+
+    def read_query(self) -> str:
+        """
+        Read a query from a QLever short URL.
+        It follows redirects and parses the HTML to find the query in the textarea.
+
+        Returns:
+            str: The SPARQL query extracted from the short URL.
+        """
+        # Ensure we have the final URL (handles redirects)
+        self.fetch_final_url(self.short_url)
+
+        if self.url:
+            try:
+                # Assuming ShortUrl or its parent has a get_content or similar method.
+                # If not, specific request logic would go here.
+                # Using get_wikitext as per your snippet pattern, assuming it returns page content.
+                text = self.get_wikitext(self.url)
+                if text:
+                    soup = BeautifulSoup(text, "html.parser")
+                    query_element = soup.find("textarea", {"id": "query"})
+                    if query_element and query_element.text:
+                        self.sparql = query_element.text.strip()
+            except Exception as ex:
+                self.error = ex
+        return self.sparql
+
+
+class QLever:
+    """
+    Handle https://github.com/ad-freiburg/qlever specifics
+    and interaction with QLever instances.
+    """
+
+    def __init__(
+        self,
+        with_progress: bool = True,
+        debug: bool = False,
+    ):
+        """
+        constructor
+        """
+        self.url = "https://github.com/ad-freiburg/qlever"
+        self.with_progress = with_progress
+        self.debug = debug
+        # Regex pattern to find URLs starting with the specified prefix
+        self.wd_url_pattern = re.compile(r"https://(?:qlever\.cs\.uni-freiburg\.de|qlever\.dev)/wikidata/[A-Za-z0-9]+")
+        self.osproject = OsProject.fromUrl(self.url)
+
+    def wd_urls_for_ticket(self, ticket: Ticket) -> List[str]:
+        """
+        Extracts and returns all URLs from a ticket's body and comments
+        that match the QLever Wikidata pattern.
+        """
+        extracted_urls = []
+
+        # Extract URLs from the ticket body
+        if ticket.body:
+            found_urls = self.wd_url_pattern.findall(ticket.body)
+            extracted_urls.extend(found_urls)
+
+        # Fetch and extract URLs from comments
+        comments = self.osproject.getComments(ticket.number)
+        for comment in comments:
+            if "body" in comment:
+                found_urls = self.wd_url_pattern.findall(comment["body"])
+                extracted_urls.extend(found_urls)
+
+        # distinct list
+        ticket_urls = list(set(extracted_urls))
+        return ticket_urls
+
+    def get_issues_query_set(self, limit: int = None, progress: bool = False) -> NamedQuerySet:
+        """
+        Orchestrates the retrieval of tickets, extraction of URLs,
+        resolving of SPARQL queries, and creation of a NamedQuerySet.
+
+        Args:
+            limit (int): Max number of tickets to process.
+            progress (bool): Whether to show a CLI progress bar.
+
+        Returns:
+            NamedQuerySet: The populated query set.
+        """
+        # Get the full dictionary
+        tickets_map = self.osproject.getAllTickets(limit=limit)
+        items_iterable = tickets_map.items()
+
+        if progress:
+            iterator = tqdm(
+                items_iterable, total=len(tickets_map), desc="Scanning tickets for QLever URLs", unit="ticket"
+            )
+        else:
+            iterator = items_iterable
+
+        ticket_dict = {}
+
+        # 1. Scan tickets for URLs
+        for i, (t_id, ticket) in enumerate(iterator):
+            msg = f"fetching short urls for ticket {i}:#{t_id} {ticket.title}"
+            if self.debug:
+                print(msg)
+            urls = self.wd_urls_for_ticket(ticket)
+            if urls:
+                ticket_dict[ticket] = urls
+
+        # 2. Convert to NamedQuerySet
+        # (This resolves the short URLs to actual SPARQL)
+        nq_set = self.named_queries_for_tickets(ticket_dict)
+        return nq_set
+
+    def named_queries_for_tickets(self, ticket_dict: Dict[Ticket, List[str]]) -> NamedQuerySet:
+        """
+        Create named queries for each ticket's extracted URLs.
+
+        Args:
+            ticket_dict (dict): Dictionary mapping tickets to a list of URLs.
+
+        Returns:
+            NamedQuerySet: A set of named queries generated from the URLs.
+        """
+        named_query_set = NamedQuerySet(
+            domain="qlever.dev",
+            namespace="issues.wikidata",
+            target_graph_name="wikidata",
+        )
+
+        total_urls = sum(len(urls) for urls in ticket_dict.values())
+
+        # We might want progress here if there are many URLs to resolve
+        if self.with_progress and total_urls > 0:
+            pbar = tqdm(total=total_urls, desc="Resolving QLever URLs", unit="url")
+        else:
+            pbar = None
+
+        for ticket, urls in ticket_dict.items():
+            for i, url in enumerate(urls, 1):
+                # Resolve the Short URL
+                qlever_url = QLeverUrl(url)
+                sparql = qlever_url.read_query()
+
+                if sparql:
+                    # Construct the NamedQuery
+                    query = NamedQuery(
+                        domain=named_query_set.domain,
+                        name=f"Issue{ticket.number}_query{i}",
+                        namespace=named_query_set.namespace,
+                        url=url,
+                        sparql=sparql,
+                        title=f"QLever Issue #{ticket.number} Query {i}",
+                        description=ticket.title,
+                        comment=f"Source: Ticket {ticket.url}, QLever Link: {url}",
+                    )
+                    named_query_set.queries.append(query)
+
+                if pbar:
+                    pbar.update(1)
+
+        if pbar:
+            pbar.close()
+
+        return named_query_set

snapquery-0.2.5/snapquery/query_parameterizer.py

@@ -0,0 +1,246 @@
+"""
+Created on 2025-12-16
+
+@author: wf
+"""
+import os
+import re
+import textwrap
+from typing import Optional, Dict, Any, List
+
+from lodstorage.params import Param
+from lodstorage.query import Query
+from snapquery.snapquery_core import NamedQuerySet, NamedQuery
+
+
+class QueryParameterizer:
+    """
+    Parameterizer for Scholia Query Sets.
+    Transforms Jinja2-based Scholia templates into standard
+    LoDStorage Parameterized Queries.
+    """
+
+    # Replacement for Scholia's label macro
+    LABEL_SERVICE = 'SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }'
+
+    # Defaults for specific Scholia Aspects
+    # derived from the prompt list
+    ASPECT_DEFAULTS = {
+        "author": "Q80",          # Tim Berners-Lee
+        "authors": "Q80",
+        "award": "Q616568",       # Turing Award
+        "catalogue": "Q2463660",  # British Library catalogue
+        "chemical": "Q18216",     # Aspirin
+        "cito": "Q1054942",       # citation intent
+        "clinical": "Q12136",     # Common Cold (disease context)
+        "complex": "Q417724",     # Protein complex
+        "countries": "Q183",      # Germany
+        "country": "Q183",
+        "dataset": "Q1172284",    # Data set
+        "disease": "Q12136",      # Common Cold
+        "event": "Q2020153",      # WWW Conference
+        "gene": "Q7191",          # p53
+        "index": "Q1",            # Universe (generic fallback)
+        "language": "Q1860",      # English
+        "lexeme": "Q151326",      # Word
+        "license": "Q20007257",   # CC BY 4.0
+        "location": "Q64",        # Berlin
+        "ontology": "Q324254",    # Ontology
+        "organization": "Q95",    # Google
+        "organizations": "Q95",
+        "pathway": "Q26487",      # Metabolic pathway
+        "podcast": "Q24634210",   # The Ozymandias Project
+        "printer": "Q11226",      # Printer (the device/person)
+        "project": "Q5562725",    # WikiProject
+        "property": "P31",        # Instance of
+        "protein": "Q11060",      # Green fluorescent protein
+        "publisher": "Q203019",   # Elsevier
+        "retraction": "Q45263695", # Retracted paper example
+        "series": "Q245209",      # Book series
+        "software": "Q28865",     # Python
+        "sponsor": "Q95",         # Google
+        "taxon": "Q140",          # Lion
+        "topic": "Q21198",        # Computer Science
+        "topics": "Q21198",
+        "use": "Q1",
+        "uses": "Q1",
+        "venue": "Q180445",       # Nature (Journal)
+        "venues": "Q180445",
+        "wikiproject": "Q5562725",
+        "work": "Q13442814",      # Scientific Article
+        "works": "Q13442814"
+    }
+
+    def __init__(self, debug: bool = False):
+        self.debug = debug
+
+    def _infer_param_details(self, var_name: str, aspect: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Heuristics to determine type and default value based on variable name
+        and the query aspect (category).
+
+        Args:
+            var_name: The variable name found in the jinja template (e.g. 'q', 'target').
+            aspect: The scholia aspect derived from the query name (e.g. 'venue', 'author').
+        """
+        # Global Default
+        details = {"type": "string", "default": "Q80"}
+
+        # 1. Identify Type based on variable name
+        if var_name in ['q', 'target', 'item', 'person', 'work', 'venue', 'gene', 'disease']:
+            details["type"] = "WikidataItem"
+
+            # 2. Assign Default Value based on Aspect
+            if aspect and aspect in self.ASPECT_DEFAULTS:
+                details["default"] = self.ASPECT_DEFAULTS[aspect]
+
+            # 3. Fallback defaults if aspect is missing or generic
+            elif var_name == 'person':
+                details["default"] = "Q80"  # Tim Berners-Lee
+            elif var_name == 'work':
+                details["default"] = "Q13442814"
+            else:
+                details["default"] = "Q80"
+
+        elif var_name in ['p', 'property']:
+            details["type"] = "WikidataProperty"
+            details["default"] = "P31"
+        elif var_name in ['limit']:
+            details["type"] = "int"
+            details["default"] = 10
+        elif var_name in ['lang', 'language']:
+            details["type"] = "string"
+            details["default"] = "en"
+
+        return details
+
+    def get_queries_by_aspect(self,query_list:List[Query])->Dict[str,List[Query]]:
+        """
+        get queries by aspect
+        """
+        queries_by_aspect = {}
+        for query in query_list:
+            aspect=self.get_aspect(query.name)
+            if aspect in self.ASPECT_DEFAULTS:
+                if aspect not in queries_by_aspect:
+                    queries_by_aspect[aspect] = []
+                queries_by_aspect[aspect].append(query)
+        return queries_by_aspect
+
+    def store_query_list(self,query_list:List[Query],tag:str,path:str="/tmp/queries"):
+        """
+        Store query list sorted by aspect into separate YAML files.
+
+        Args:
+            query_list (List[Query]): List of Query objects with aspect metadata.
+            tag (str): Tag to include in filename.
+            path (str): Directory path where YAML files will be stored.
+        """
+        queries_by_aspect=self.get_queries_by_aspect(query_list)
+        # Create directory if it doesn't exist
+        os.makedirs(path, exist_ok=True)
+
+        # Store each aspect's queries in a separate YAML file
+        for aspect, queries in sorted(queries_by_aspect.items()):
+            filename = f"{tag}_{aspect}_queries.yaml"
+            filepath = os.path.join(path, filename)
+
+            with open(filepath, "w", encoding="utf-8") as f:
+                for query in queries:
+                    # work around https://github.com/yaml/pyyaml/issues/411
+                    if query.sparql:
+                        clean_sparql = textwrap.dedent(query.sparql).strip()
+                    yaml_str=query.to_yaml()
+                    f.write(yaml_str)
+            pass
+
+
+    def parameterize_query_set(self, query_set: NamedQuerySet) -> List[Query]:
+        """
+        Parameterize the given query set to produce a list of LoDStorage queries.
+
+        Args:
+            query_set (NamedQuerySet): The set of named queries to transform.
+
+        Returns:
+            List[Query]: A list of transformed LoDStorage Query objects.
+        """
+        query_list = []
+        for nq in query_set.queries:
+            query = self.parameterize_query(nq)
+            query_list.append(query)
+        return query_list
+
+    def get_aspect(self,name:str)->str:
+        aspect = None
+        if name:
+            # Split by hyphen first, as that is the standard separator in the example
+            parts = re.split(r'[-_]', name)
+            if parts:
+                aspect = parts[0].lower()
+        return aspect
+
+    def parameterize_query(self, nq: NamedQuery) -> Query:
+        """
+        Parameterize a single named query.
+
+        1. Identifies the "Aspect" from the query name (e.g. 'venue-...' -> 'venue').
+        2. Cleans Jinja2 specific imports.
+        3. Replaces Scholia Macros with standard SPARQL.
+        4. Detects variables and creates Param objects using aspect-specific defaults.
+
+        Args:
+            nq (NamedQuery): The named query input.
+
+        Returns:
+            Query: The transformed LoDStorage Query object.
+        """
+        # 0. Determine Aspect
+        # Naming convention: {aspect}-{subname} or {aspect}_{subname}
+        # e.g., "venue-curation_missing-author-item" -> "venue"
+        aspect=self.get_aspect(nq.name)
+
+        raw_sparql = nq.sparql
+
+        # 1. Remove Jinja imports
+        # e.g., {% import 'sparql-helpers.sparql' as sparql_helpers -%}
+        txt = re.sub(r'\{%\s*import\s*.*?%\}', '', raw_sparql, flags=re.DOTALL)
+
+        # 2. Replace Label/Description Macros
+        # Matches: {{ sparql_helpers.labels(["?mol"], languages) }}
+        txt = re.sub(
+            r"\{\{\s*sparql_helpers\.(labels|descriptions).*?\}\}",
+            self.LABEL_SERVICE,
+            txt,
+            flags=re.DOTALL
+        )
+
+        # 3. Detect Variables to build Param list
+        found_vars = set(re.findall(r"\{\{\s*([a-zA-Z0-9_]+)\s*\}\}", txt))
+        found_vars.discard('sparql_helpers')
+
+        # Create lodstorage Query object
+        clean_sparql = txt.strip()
+
+        query_obj = Query(
+            name=nq.name,
+            query=clean_sparql,
+            title=nq.title,
+            description=nq.description,
+            lang='sparql'
+        )
+
+        # 4. Attach Parameters with Aspect context
+        param_list = []
+        for var in sorted(found_vars):
+            details = self._infer_param_details(var, aspect=aspect)
+            p = Param(
+                name=var,
+                type=details["type"],
+                default_value=details["default"]
+            )
+            param_list.append(p)
+
+        query_obj.param_list = param_list
+
+        return query_obj