snapquery 0.2.0__tar.gz → 0.2.1__tar.gz

{snapquery-0.2.0 → snapquery-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snapquery
-Version: 0.2.0
+Version: 0.2.1
 Summary: Just Query wikidata and other SPARQL endpoints by query name - a frontend to Introduce Named Queries and Named Query Middleware to wikidata and other SPARQL endpoints 
 Project-URL: Home, https://github.com/WolfgangFahl/snapquery
 Project-URL: Documentation, https://wiki.bitplan.com/index.php/snapquery

snapquery-0.2.1/snapquery/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

snapquery-0.2.1/snapquery/github_access.py

@@ -0,0 +1,119 @@
+"""
+GitHub API client for accessing repository contents.
+
+Created on 2025-12-02
+@author: wf
+"""
+import json
+from  pathlib import Path
+from typing import List, Dict, Any, Optional, Union
+
+import requests
+
+
+class GitHub:
+    """
+    A simple GitHub API client for accessing repository contents.
+    """
+
+    def __init__(self, owner: str, repo: str, token: Optional[str] = None, session: Optional[requests.Session] = None):
+        """
+        Initialize GitHub client.
+
+        Args:
+            owner: Repository owner (username or organization)
+            repo: Repository name
+            token: Optional GitHub API token for authentication
+            session: Optional custom requests.Session
+        """
+        self.owner = owner
+        self.repo = repo
+        self.token = token
+        self.base_url = f"https://api.github.com/repos/{owner}/{repo}"
+        # Use provided token or read from file (compatible with GitHubApi)
+        self.token = token if token is not None else self._read_token()
+
+        # Use custom session or create new one
+        self.session = session or requests.Session()
+
+
+
+    def _headers(self) -> Dict[str, str]:
+        headers = {"Accept": "application/vnd.github.v3+json"}
+        if self.token:
+            headers["Authorization"] = f"token {self.token}"
+        return headers
+
+    def _read_token(self) -> Optional[str]:
+        """
+        Read GitHub token from ~/.github/access_token.json
+        (compatible with GitHubApi token storage format).
+
+        Returns:
+            GitHub token or None if not found
+        """
+        token_path = Path.home() / ".github" / "access_token.json"
+        if token_path.exists():
+            try:
+                with open(token_path, 'r') as f:
+                    data = json.load(f)
+                    return data.get("access_token")
+            except (json.JSONDecodeError, OSError):
+                pass
+        return None
+
+
+    def get_contents(self, path: str = "") -> Union[List[Dict[str, Any]], Dict[str, Any]]:
+        """
+        Get contents of a directory or file from the repository.
+
+        Args:
+            path: Path within the repository
+
+        Returns:
+            List of dictionaries for directories, or content for files
+        """
+        url = f"{self.base_url}/contents/{path}"
+        response = self.session.get(url, headers=self._headers(), timeout=30)
+        response.raise_for_status()
+        return response.json()
+
+    def list_files_recursive(self, path: str = "", suffix: Optional[str] = None) -> List[Dict[str, Any]]:
+        """
+        Recursively list files under a given path. Optionally filter by file suffix.
+
+        Args:
+            path: starting path within the repository
+            suffix: optional filename suffix filter, e.g., ".ttl"
+
+        Returns:
+            A flat list of GitHub content item dicts for files.
+        """
+        items = self.get_contents(path)
+        if isinstance(items, dict):
+            items = [items]
+
+        files: List[Dict[str, Any]] = []
+        for item in items:
+            item_type = item.get("type")
+            item_path = item.get("path", "")
+            if item_type == "file":
+                if suffix is None or item_path.endswith(suffix):
+                    files.append(item)
+            elif item_type == "dir":
+                files.extend(self.list_files_recursive(item_path, suffix=suffix))
+        return files
+
+    def download(self, download_url: str) -> str:
+        """
+        Download raw file content via a download_url.
+
+        Args:
+            download_url: The GitHub-provided raw download URL
+
+        Returns:
+            The text content of the file
+        """
+        response = self.session.get(download_url, headers=self._headers(), timeout=30)
+        response.raise_for_status()
+        return response.text

{snapquery-0.2.0 → snapquery-0.2.1}/snapquery/samples/endpoints.yaml

@@ -76,6 +76,31 @@ endpoints:
       - blazegraph
       - commons
 
+  wikidata-qlever:
+    description: QLever Freiburg Wikidata.
+    lang: sparql
+    method: POST
+    database: qlever
+    endpoint: https://qlever.dev/api/wikidata
+    website: https://qlever.dev/wikidata
+    calls_per_minute: 60
+    prefix_sets:
+      - rdf
+      - wikidata
+      - commons
+
+  wikidata-qlever-dbis:
+    description: QLever RWTH Wikidata.
+    lang: sparql
+    method: POST
+    database: qlever
+    endpoint: https://qlever-api.wikidata.dbis.rwth-aachen.de/
+    website: https://qlever.wikidata.dbis.rwth-aachen.de/wikidata
+    calls_per_minute: 60
+    prefix_sets:
+      - rdf
+      - wikidata
+
   dblp:
     description: DBLP official (qlever - reliable).
     lang: sparql

{snapquery-0.2.0 → snapquery-0.2.1}/snapquery/samples/endpoints_optional.yaml

@@ -26,30 +26,6 @@ endpoints:
       - rdf
       - wikidata
 
-  wikidata-qlever:
-    description: QLever Freiburg Wikidata.
-    lang: sparql
-    method: POST
-    database: qlever
-    endpoint: https://qlever.cs.uni-freiburg.de/api/wikidata
-    website: https://qlever.cs.uni-freiburg.de/wikidata
-    calls_per_minute: 60
-    prefix_sets:
-      - rdf
-      - wikidata
-      - commons
-
-  wikidata-qlever-dbis:
-    description: QLever RWTH Wikidata.
-    lang: sparql
-    method: POST
-    database: qlever
-    endpoint: https://qlever-api.wikidata.dbis.rwth-aachen.de/
-    website: https://qlever.wikidata.dbis.rwth-aachen.de/wikidata
-    calls_per_minute: 60
-    prefix_sets:
-      - rdf
-      - wikidata
 
   wikidata-scatter:
     description: Scatter experimental.

snapquery-0.2.1/snapquery/sib_sparql_examples.py

@@ -0,0 +1,167 @@
+"""
+SIB SPARQL Examples Fetcher
+Refactored to use a reusable GitHub API client and improved rdflib parsing.
+
+Handles nested /examples/<service>/<NNN>.ttl structure.
+Extracts sh:select, rdfs:comment, schema:target/labels as NamedQuery.
+Exports to YAML (user request).
+
+Modified: 2025-12-02 by wf
+"""
+
+import os
+from typing import Optional, List, Dict, Any, Set
+
+import rdflib
+
+from snapquery.github_access import GitHub
+from snapquery.snapquery_core import NamedQuery, NamedQueryManager, NamedQuerySet
+
+class SibSparqlExamples:
+    """
+    Fetch & parse SIB sparql-examples from GitHub API (remote, no clone).
+    """
+
+    def __init__(
+        self,
+        nqm: NamedQueryManager,
+        github: Optional[GitHub] = None,
+        debug: bool = False,
+    ):
+        self.nqm = nqm
+        self.github = github or GitHub(owner="sib-swiss", repo="sparql-examples", token=os.getenv("GITHUB_TOKEN"))
+        self.named_query_set = NamedQuerySet(
+            domain="sparql-examples.sib.swiss",
+            namespace="sib-examples",
+            target_graph_name="various",  # varies per query (schema:target)
+        )
+        self.debug = debug
+        self.cache: Dict[str, str] = {}  # TTL content cache
+        self.stored_queries: List[NamedQuery] = []
+
+    def collect_ttl_items(self, path: str = "examples") -> List[Dict[str, Any]]:
+        """Recursively collect all .ttl file items under given path using GitHub client."""
+        ttl_files= self.github.list_files_recursive(path, suffix=".ttl")
+        return ttl_files
+
+    def get_ttl_content(self, download_url: str) -> str:
+        """Fetch TTL content from GitHub download_url, with cache."""
+        if download_url in self.cache:
+            return self.cache[download_url]
+        content = self.github.download(download_url)
+        self.cache[download_url] = content
+        return content
+
+    def get_html_url(self, file_path: str) -> str:
+        """
+        Construct GitHub HTML URL for a file path.
+
+        Args:
+            file_path: Path like "examples/Bgee/001.ttl"
+
+        Returns:
+            URL like "https://github.com/sib-swiss/sparql-examples/blob/main/examples/Bgee/001.ttl"
+        """
+        url=f"https://github.com/{self.github.owner}/{self.github.repo}/blob/main/{file_path}"
+        return url
+
+    def parse_ttl_to_namedqueries(self, ttl_content: str, ttl_path: str) -> List[NamedQuery]:
+        """
+        Parse TTL → NamedQuery using rdflib + SHACL shapes.
+        Matches your Bgee sample exactly.
+        """
+        parsed_queries: List[NamedQuery] = []
+        try:
+            g = rdflib.Graph()
+            g.parse(data=ttl_content, format="turtle")
+
+            # path e.g. "examples/Bgee/001.ttl" → endpoint_path="Bgee/001"
+            endpoint_path = ttl_path.replace("examples/", "").rstrip(".ttl")
+            html_url = self.get_html_url(ttl_path)
+
+            # Fixed SPARQL: direct sh:select on sh:SPARQLSelectExecutable (matches sample)
+            qtext = """
+            PREFIX sh: <http://www.w3.org/ns/shacl#>
+            PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+            PREFIX schema: <http://schema.org/>
+
+            SELECT ?s ?query ?comment ?label WHERE {
+              ?s a sh:SPARQLSelectExecutable ;
+                 sh:select ?query .
+              OPTIONAL { ?s rdfs:comment ?comment }
+              OPTIONAL { ?s rdfs:label ?label }
+              OPTIONAL { ?s schema:name ?label }  # Fallback label
+            }
+            """
+
+            rows = list(g.query(qtext))
+            if not rows:
+                if self.debug:
+                    print(f"No SHACL queries in {ttl_path}")
+                return []
+
+            seen_queries: Set[str] = set()
+            fallback_title = endpoint_path.split("/")[-1].replace("_", " ").title()
+
+            for i, row in enumerate(rows):
+                query_str = str(row.query)
+                if not query_str or query_str in seen_queries:
+                    continue
+                seen_queries.add(query_str)
+
+                comment = str(row.comment) if row.comment else ""
+                label = str(row.label) if row.label else fallback_title
+
+                title = label
+                description = comment or f"SIB example: {endpoint_path} ({title})"
+
+                # Name: "Bgee:001" → slugify to "bgee-001"
+                name_suffix = f"_{i+1}" if len(rows) > 1 else ""
+                sub_name = endpoint_path.replace("/", ":") + name_suffix
+                name = sub_name  # No duplicate prefix!
+
+                base_url = f"https://sparql-examples.sib.swiss/{endpoint_path.replace('/', '/')}{name_suffix}"
+
+                nq = NamedQuery(
+                    domain=self.named_query_set.domain,
+                    namespace=self.named_query_set.namespace,
+                    name=name,
+                    url=html_url,  # GitHub source
+                    title=title,
+                    description=description,
+                    sparql=query_str,
+                )
+                parsed_queries.append(nq)
+
+        except Exception as e:
+            if self.debug:
+                print(f"Error parsing {ttl_path}: {e}")
+        return parsed_queries
+
+    def extract_queries(self, limit: Optional[int] = None, debug_print: bool = False) -> List[NamedQuery]:
+        """
+        Main: Fetch/parse/store all TTL → NamedQuery/DB/Set.
+        """
+        self.stored_queries.clear()
+        ttl_items = self.collect_ttl_items("examples")
+        if limit:
+            ttl_items = ttl_items[:limit]
+
+        for item in ttl_items:
+            if debug_print:
+                print(f"Processing {item['path']}...")
+            content = self.get_ttl_content(item["download_url"])
+            nqs = self.parse_ttl_to_namedqueries(content, item["path"])
+            for nq in nqs:
+                self.nqm.add_and_store(nq)  # FIXED: stores to DB + QueryDetails
+                self.named_query_set.add(nq)  # FIXED: add (not add_query)
+                self.stored_queries.append(nq)
+        return self.stored_queries
+
+
+    def save_to_yaml(self, filepath: str) -> Dict[str, Any]:
+        """
+        Export stored queries to a YAML file.
+        """
+        self.named_query_set.save_to_yaml_file(filepath)
+

{snapquery-0.2.0 → snapquery-0.2.1}/snapquery/snapquery_cmd.py

@@ -3,7 +3,7 @@ Created on 2024-05-03
 
 @author: wf
 """
-
+from tqdm import tqdm
 import logging
 import sys
 from argparse import ArgumentParser
@@ -87,6 +87,12 @@ class SnapQueryCmd(WebserverCmd):
             action=StoreDictKeyPair,
             help="query parameters as Key-value pairs in the format key1=value1,key2=value2",
         )
+        parser.add_argument(
+            "--progress",
+            action="store_true",
+            help="show progress bars when testing queries (--testQueries)",
+        )
+
         parser.add_argument(
             "--domain",
             type=str,
@@ -144,6 +150,56 @@ class SnapQueryCmd(WebserverCmd):
             self.args.func(self.args)
         return self.args
 
+    def handle_test_queries(self):
+        """
+        Handle the --testQueries option by executing queries against endpoints.
+        The endpoint is the outer loop, queries are the inner loop.
+        """
+        # Determine which endpoints to use
+        if self.args.endpointName:
+            endpoint_names = [self.args.endpointName]
+        else:
+            endpoint_names = list(self.nqm.endpoints.keys())
+
+        # Get all queries to test
+        queries = self.nqm.get_all_queries(domain=self.args.domain, namespace=self.args.namespace)
+
+        # Create execution instance
+        execution = Execution(self.nqm, debug=self.args.debug)
+
+        # Outer loop: endpoints
+        endpoint_iter = tqdm(endpoint_names, desc="Testing endpoints") if self.args.progress else endpoint_names
+        for endpoint_name in endpoint_iter:
+            # Inner loop: queries
+            query_iter = tqdm(queries, desc=f"Queries for {endpoint_name}", leave=False) if self.args.progress else queries
+            for i, nq in enumerate(query_iter, start=1):
+                execution.execute(
+                    nq,
+                    endpoint_name=endpoint_name,
+                    context=self.args.context,
+                    title=f"{endpoint_name}::query {i:3}/{len(queries)}",
+                    prefix_merger=QueryPrefixMerger.get_by_name(self.args.prefix_merger),
+                )
+
+    def handle_test_queries_no_progress_version(self):
+        if self.args.endpointName:
+            endpoint_names = [self.args.endpointName]
+        else:
+            endpoint_names = list(self.nqm.endpoints.keys())
+        queries = self.nqm.get_all_queries(domain=self.args.domain, namespace=self.args.namespace)
+        execution = Execution(self.nqm, debug=self.args.debug)
+        query_iter = tqdm(queries, desc="Testing queries") if self.args.progress else queries
+        for i, nq in enumerate(query_iter, start=1):
+            for endpoint_name in endpoint_names:
+                execution.execute(
+                    nq,
+                    endpoint_name=endpoint_name,
+                    context=self.args.context,
+                    title=f"query {i:3}/{len(queries)}::{endpoint_name}",
+                    prefix_merger=QueryPrefixMerger.get_by_name(self.args.prefix_merger),
+                )
+
+
     def handle_args(self, args) -> bool:
         """
         handle the command line args
@@ -174,21 +230,8 @@ class SnapQueryCmd(WebserverCmd):
                 print(f"{namespace}:{count}")
             handled = True
         elif self.args.testQueries:
-            if self.args.endpointName:
-                endpoint_names = [self.args.endpointName]
-            else:
-                endpoint_names = list(nqm.endpoints.keys())
-            queries = self.nqm.get_all_queries(domain=self.args.domain, namespace=self.args.namespace)
-            execution = Execution(self.nqm, debug=self.args.debug)
-            for i, nq in enumerate(queries, start=1):
-                for endpoint_name in endpoint_names:
-                    execution.execute(
-                        nq,
-                        endpoint_name=endpoint_name,
-                        context=self.args.context,
-                        title=f"query {i:3}/{len(queries)}::{endpoint_name}",
-                        prefix_merger=QueryPrefixMerger.get_by_name(self.args.prefix_merger),
-                    )
+            self.handle_test_queries()
+            handled = True
         elif self.args.queryName is not None or self.args.query_id is not None:
             if self.args.query_id is not None:
                 query_name = QueryName.from_query_id(self.args.query_id)

{snapquery-0.2.0 → snapquery-0.2.1}/snapquery/snapquery_core.py

@@ -728,7 +728,7 @@ class NamedQueryManager:
             for source_class, pk in [
                 (NamedQuery, "query_id"),
                 (QueryStats, "stats_id"),
-                (QueryDetails, "quer_id"),
+                (QueryDetails, "query_id"),
             ]:
                 # Fetch sample records from the specified class
                 sample_records = cls.get_sample_records(source_class=source_class)

snapquery-0.2.1/tests/test_sib_examples.py

@@ -0,0 +1,74 @@
+"""
+Test SIB SPARQL Examples fetching.
+Verifies integration between SIB fetcher and Snapquery Core.
+
+Created on 2025-12-02
+@author: wf
+"""
+import os
+import unittest
+from basemkit.basetest import Basetest
+from snapquery.snapquery_core import NamedQueryManager
+from snapquery.sib_sparql_examples import SibSparqlExamples
+
+class TestSibExamples(Basetest):
+    """
+    Test retrieving SIB SPARQL examples using GitHub cache/api.
+    """
+
+    def setUp(self, debug=True, profile=True):
+        Basetest.setUp(self, debug=debug, profile=profile)
+
+    @unittest.skipIf(Basetest.inPublicCI(), "avoid github rate limit in CI")
+    def test_sib_examples_fetch_and_store(self):
+        """
+        Test retrieving SIB examples, populating the DB, and exporting to YAML.
+        """
+        db_path = "/tmp/sib_examples.db"
+        yaml_path = "/tmp/sib_examples.yaml"
+
+        if os.path.exists(db_path):
+            os.remove(db_path)
+        if os.path.exists(yaml_path):
+            os.remove(yaml_path)
+
+        nqm = NamedQueryManager.from_samples(db_path=db_path)
+        sib_fetcher = SibSparqlExamples(nqm, debug=self.debug)
+
+        # Limit for testing efficiency
+        limit = 7 # if self.inPublicCI() else None
+        if self.debug:
+            print(f"Fetching SIB examples (limit={limit})...")
+
+        loaded_queries = sib_fetcher.extract_queries(limit=limit, debug_print=self.debug)
+
+        self.assertTrue(len(loaded_queries) > 0, "Should have loaded at least one query")
+        self.assertEqual(len(sib_fetcher.named_query_set), len(loaded_queries))
+
+        # Verify SQL Database Storage
+        records = nqm.sql_db.query(
+            """
+            SELECT count(*) as count
+            FROM NamedQuery
+            WHERE namespace=? AND domain=?
+            """,
+            (sib_fetcher.named_query_set.namespace, sib_fetcher.named_query_set.domain)
+        )
+        db_count = records[0]['count']
+        self.assertEqual(db_count, len(loaded_queries), "DB count should match loaded count")
+
+        # Verify YAML Export
+        if self.debug:
+            print(f"Exporting to {yaml_path}...")
+        sib_fetcher.save_to_yaml(yaml_path)
+
+        self.assertTrue(os.path.exists(yaml_path))
+
+        # Optional: Read back to verify YamlAble structure
+        with open(yaml_path, 'r') as f:
+            content = f.read()
+            self.assertIn("sib-examples", content)
+            self.assertIn("queries:", content)
+
+        if self.debug:
+            print(f"Successfully processed {db_count} queries.")

snapquery-0.2.0/snapquery/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.0"

snapquery-0.2.0/snapquery/github_access.py

@@ -1,46 +0,0 @@
-"""
-GitHub API client for accessing repository contents.
-
-Created on 2025-12-02
-@author: wf
-"""
-import requests
-
-class GitHub:
-    """
-    A simple GitHub API client for accessing repository contents.
-    """
-
-    def __init__(self, owner: str, repo: str, token: str = None):
-        """
-        Initialize GitHub client.
-
-        Args:
-            owner: Repository owner (username or organization)
-            repo: Repository name
-            token: Optional GitHub API token for authentication
-        """
-        self.owner = owner
-        self.repo = repo
-        self.token = token
-        self.base_url = f"https://api.github.com/repos/{owner}/{repo}"
-
-    def get_contents(self, path: str = ""):
-        """
-        Get contents of a directory or file from the repository.
-
-        Args:
-            path: Path within the repository
-
-        Returns:
-            List of dictionaries for directories, or content for files
-        """
-        url = f"{self.base_url}/contents/{path}"
-        headers = {"Accept": "application/vnd.github.v3+json"}
-
-        if self.token:
-            headers["Authorization"] = f"token {self.token}"
-
-        response = requests.get(url, headers=headers)
-        response.raise_for_status()
-        return response.json()

snapquery-0.2.0/tests/test_sib_examples.py

@@ -1,35 +0,0 @@
-"""
-
-SIB Swiss Institute of Bioinformatics
-
-sparql examples
-
-@author wf
-"""
-import unittest
-
-from basemkit.basetest import Basetest
-from rdflib import Graph
-
-
-class TestSibExamples(Basetest):
-    """
-    Test for Issue #59
-    https://github.com/WolfgangFahl/snapquery/issues/59
-    https://github.com/sib-swiss/sparql-examples/
-
-    Snap query should consider using the sparql-examples style of encoding queries as their own entities as the basis of it's data interchange
-    """
-
-    def setUp(self, debug=False, profile=True):
-        Basetest.setUp(self, debug=debug, profile=profile)
-
-
-    @unittest.skip("needs github clone to work -postpone")
-    def testBgee(self):
-        """
-        test a single example
-        """
-        g= Graph().parse("examples/Bgee/001.ttl", format="turtle")
-        query = g.value(None, g.namespace("sh")["select"])  # Gets sh:select literal
-        print(query)  # Full SPARQL text