snapquery 0.2.3__tar.gz → 0.2.4__tar.gz

{snapquery-0.2.3 → snapquery-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snapquery
-Version: 0.2.3
+Version: 0.2.4
 Summary: Just Query wikidata and other SPARQL endpoints by query name - a frontend to Introduce Named Queries and Named Query Middleware to wikidata and other SPARQL endpoints 
 Project-URL: Home, https://github.com/WolfgangFahl/snapquery
 Project-URL: Documentation, https://wiki.bitplan.com/index.php/snapquery
@@ -26,14 +26,14 @@ Classifier: Topic :: Software Development :: User Interfaces
 Requires-Python: >=3.10
 Requires-Dist: mwoauth>=0.4.0
 Requires-Dist: nameparser>=1.1.3
-Requires-Dist: ngwidgets>=0.30.0
+Requires-Dist: ngwidgets>=0.30.3
 Requires-Dist: nicegui[plotly]==2.24.2
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: py-3rdparty-mediawiki>=0.18.1
 Requires-Dist: py-ez-wikidata>=0.3.2
-Requires-Dist: pybasemkit>=0.1.3
-Requires-Dist: pylodstorage>=0.18.5
-Requires-Dist: pyopensourceprojects>=0.3.0
+Requires-Dist: pybasemkit>=0.1.6
+Requires-Dist: pylodstorage>=0.18.6
+Requires-Dist: pyopensourceprojects>=0.4.1
 Requires-Dist: python-slugify>=8.0.4
 Requires-Dist: ratelimit>=2.2.1
 Requires-Dist: requests

{snapquery-0.2.3 → snapquery-0.2.4}/pyproject.toml

@@ -22,11 +22,11 @@ dependencies = [
 	# https://pypi.org/project/pybasemkit/
 	# https://github.com/WolfgangFahl/pybasemkit
 	# Python base module kit: YAML/JSON I/O, structured logging, CLI tooling, shell execution, and remote pydevd debug support.
-	"pybasemkit>=0.1.3",
+	"pybasemkit>=0.1.6",
 	# https://github.com/WolfgangFahl/nicegui_widgets
-	"ngwidgets>=0.30.0",
+	"ngwidgets>=0.30.3",
     # https://pypi.org/project/pyLodStorage/
-    "pyLodStorage>=0.18.5",
+    "pyLodStorage>=0.18.6",
     # https://pypi.org/project/py-3rdparty-mediawiki/
     "py-3rdparty-mediawiki>=0.18.1",
     "wikitextparser",
@@ -51,7 +51,7 @@ dependencies = [
     # https://pypi.org/project/nameparser/
     "nameparser>=1.1.3",
     # https://github.com/WolfgangFahl/pyOpenSourceProjects
-    "pyOpenSourceProjects>=0.3.0"
+    "pyOpenSourceProjects>=0.4.1"
 ]
 
 requires-python = ">=3.10"

snapquery-0.2.4/snapquery/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.4"

{snapquery-0.2.3 → snapquery-0.2.4}/snapquery/github_access.py

@@ -4,9 +4,10 @@ GitHub API client for accessing repository contents.
 Created on 2025-12-02
 @author: wf
 """
+
 import json
-from  pathlib import Path
-from typing import List, Dict, Any, Optional, Union
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
 
 import requests
 
@@ -16,28 +17,38 @@ class GitHub:
     A simple GitHub API client for accessing repository contents.
     """
 
-    def __init__(self, owner: str, repo: str, token: Optional[str] = None, session: Optional[requests.Session] = None):
+    def __init__(
+        self,
+        owner: str,
+        repo: str,
+        branch: Optional[str] = None,
+        token: Optional[str] = None,
+        session: Optional[requests.Session] = None
+    ):
         """
         Initialize GitHub client.
 
         Args:
             owner: Repository owner (username or organization)
             repo: Repository name
+            branch: Optional specific branch or commit SHA (default: default repo branch)
             token: Optional GitHub API token for authentication
             session: Optional custom requests.Session
         """
         self.owner = owner
         self.repo = repo
-        self.token = token
+        self.branch = branch
+
+
         self.base_url = f"https://api.github.com/repos/{owner}/{repo}"
-        # Use provided token or read from file (compatible with GitHubApi)
+
+        # Use provided token or read from file
         self.token = token if token is not None else self._read_token()
 
         # Use custom session or create new one
         self.session = session or requests.Session()
 
 
-
     def _headers(self) -> Dict[str, str]:
         headers = {"Accept": "application/vnd.github.v3+json"}
         if self.token:
@@ -55,14 +66,13 @@ class GitHub:
         token_path = Path.home() / ".github" / "access_token.json"
         if token_path.exists():
             try:
-                with open(token_path, 'r') as f:
+                with open(token_path, "r") as f:
                     data = json.load(f)
                     return data.get("access_token")
             except (json.JSONDecodeError, OSError):
                 pass
         return None
 
-
     def get_contents(self, path: str = "") -> Union[List[Dict[str, Any]], Dict[str, Any]]:
         """
         Get contents of a directory or file from the repository.
@@ -74,10 +84,15 @@ class GitHub:
             List of dictionaries for directories, or content for files
         """
         url = f"{self.base_url}/contents/{path}"
-        response = self.session.get(url, headers=self._headers(), timeout=30)
+        params = {}
+        if self.branch:
+            params["ref"] = self.branch
+
+        response = self.session.get(url, headers=self._headers(), params=params, timeout=30)
         response.raise_for_status()
         return response.json()
 
+
     def list_files_recursive(self, path: str = "", suffix: Optional[str] = None) -> List[Dict[str, Any]]:
         """
         Recursively list files under a given path. Optionally filter by file suffix.
@@ -116,4 +131,4 @@ class GitHub:
         """
         response = self.session.get(download_url, headers=self._headers(), timeout=30)
         response.raise_for_status()
-        return response.text
+        return response.text

snapquery-0.2.4/snapquery/qlever.py

@@ -0,0 +1,187 @@
+"""
+Created on 2024-06-20
+
+@author: wf
+"""
+
+import re
+from typing import Dict, List
+
+from bs4 import BeautifulSoup
+from osprojects.osproject import OsProject, Ticket
+from tqdm import tqdm
+
+from snapquery.snapquery_core import NamedQuery, NamedQuerySet
+from snapquery.wd_short_url import ShortUrl
+
+
+class QLeverUrl(ShortUrl):
+    """
+    Handles operations related to QLever short URLs.
+    """
+
+    def __init__(self, short_url: str):
+        super().__init__(short_url, scheme="https", netloc="qlever.cs.uni-freiburg.de")
+        self.sparql = None
+
+    def read_query(self) -> str:
+        """
+        Read a query from a QLever short URL.
+        It follows redirects and parses the HTML to find the query in the textarea.
+
+        Returns:
+            str: The SPARQL query extracted from the short URL.
+        """
+        # Ensure we have the final URL (handles redirects)
+        self.fetch_final_url(self.short_url)
+
+        if self.url:
+            try:
+                # Assuming ShortUrl or its parent has a get_content or similar method.
+                # If not, specific request logic would go here.
+                # Using get_wikitext as per your snippet pattern, assuming it returns page content.
+                text = self.get_wikitext(self.url)
+                if text:
+                    soup = BeautifulSoup(text, "html.parser")
+                    query_element = soup.find("textarea", {"id": "query"})
+                    if query_element and query_element.text:
+                        self.sparql = query_element.text.strip()
+            except Exception as ex:
+                self.error = ex
+        return self.sparql
+
+
+class QLever:
+    """
+    Handle https://github.com/ad-freiburg/qlever specifics
+    and interaction with QLever instances.
+    """
+
+    def __init__(
+        self,
+        with_progress: bool = True,
+        debug: bool = False,
+    ):
+        """
+        constructor
+        """
+        self.url = "https://github.com/ad-freiburg/qlever"
+        self.with_progress = with_progress
+        self.debug = debug
+        # Regex pattern to find URLs starting with the specified prefix
+        self.wd_url_pattern = re.compile(r"https://(?:qlever\.cs\.uni-freiburg\.de|qlever\.dev)/wikidata/[A-Za-z0-9]+")
+        self.osproject = OsProject.fromUrl(self.url)
+
+    def wd_urls_for_ticket(self, ticket: Ticket) -> List[str]:
+        """
+        Extracts and returns all URLs from a ticket's body and comments
+        that match the QLever Wikidata pattern.
+        """
+        extracted_urls = []
+
+        # Extract URLs from the ticket body
+        if ticket.body:
+            found_urls = self.wd_url_pattern.findall(ticket.body)
+            extracted_urls.extend(found_urls)
+
+        # Fetch and extract URLs from comments
+        comments = self.osproject.getComments(ticket.number)
+        for comment in comments:
+            if "body" in comment:
+                found_urls = self.wd_url_pattern.findall(comment["body"])
+                extracted_urls.extend(found_urls)
+
+        # distinct list
+        ticket_urls = list(set(extracted_urls))
+        return ticket_urls
+
+    def get_issues_query_set(self, limit: int = None, progress: bool = False) -> NamedQuerySet:
+        """
+        Orchestrates the retrieval of tickets, extraction of URLs,
+        resolving of SPARQL queries, and creation of a NamedQuerySet.
+
+        Args:
+            limit (int): Max number of tickets to process.
+            progress (bool): Whether to show a CLI progress bar.
+
+        Returns:
+            NamedQuerySet: The populated query set.
+        """
+        # Get the full dictionary
+        tickets_map = self.osproject.getAllTickets(limit=limit)
+        items_iterable = tickets_map.items()
+
+        if progress:
+            iterator = tqdm(
+                items_iterable, total=len(tickets_map), desc="Scanning tickets for QLever URLs", unit="ticket"
+            )
+        else:
+            iterator = items_iterable
+
+        ticket_dict = {}
+
+        # 1. Scan tickets for URLs
+        for i, (t_id, ticket) in enumerate(iterator):
+            msg = f"fetching short urls for ticket {i}:#{t_id} {ticket.title}"
+            if self.debug:
+                print(msg)
+            urls = self.wd_urls_for_ticket(ticket)
+            if urls:
+                ticket_dict[ticket] = urls
+
+        # 2. Convert to NamedQuerySet
+        # (This resolves the short URLs to actual SPARQL)
+        nq_set = self.named_queries_for_tickets(ticket_dict)
+        return nq_set
+
+    def named_queries_for_tickets(self, ticket_dict: Dict[Ticket, List[str]]) -> NamedQuerySet:
+        """
+        Create named queries for each ticket's extracted URLs.
+
+        Args:
+            ticket_dict (dict): Dictionary mapping tickets to a list of URLs.
+
+        Returns:
+            NamedQuerySet: A set of named queries generated from the URLs.
+        """
+        named_query_set = NamedQuerySet(
+            domain="qlever.dev",
+            namespace="issues.wikidata",
+            target_graph_name="wikidata",
+        )
+
+        total_urls = sum(len(urls) for urls in ticket_dict.values())
+
+        # We might want progress here if there are many URLs to resolve
+        if self.with_progress and total_urls > 0:
+            pbar = tqdm(total=total_urls, desc="Resolving QLever URLs", unit="url")
+        else:
+            pbar = None
+
+        for ticket, urls in ticket_dict.items():
+            for i, url in enumerate(urls, 1):
+                # Resolve the Short URL
+                qlever_url = QLeverUrl(url)
+                sparql = qlever_url.read_query()
+
+                if sparql:
+                    # Construct the NamedQuery
+                    query = NamedQuery(
+                        domain=named_query_set.domain,
+                        name=f"Issue{ticket.number}_query{i}",
+                        namespace=named_query_set.namespace,
+                        url=url,
+                        sparql=sparql,
+                        title=f"QLever Issue #{ticket.number} Query {i}",
+                        description=ticket.title,
+                        comment=f"Source: Ticket {ticket.url}, QLever Link: {url}",
+                    )
+                    named_query_set.queries.append(query)
+
+                if pbar:
+                    pbar.update(1)
+
+        if pbar:
+            pbar.close()
+
+        return named_query_set