querysource 5.1.0__tar.gz → 5.1.2__tar.gz

{querysource-5.1.0 → querysource-5.1.2}/.claude/agents/sdd-worker.md

@@ -23,7 +23,7 @@ description: |
 model: sonnet
 color: blue
 permissionMode: bypassPermissions
-tools: Read, Write, Edit, MultiEdit, Bash, Glob, Grep, Agent, mcp__parrot-sdd-coder__coder_begin_execution, mcp__parrot-sdd-coder__coder_end_execution, mcp__parrot-sdd-coder__coder_suspend_model, mcp__parrot-sdd-coder__coder_plan, mcp__parrot-sdd-coder__coder_run_chunk, mcp__parrot-sdd-coder__coder_prepare_native, mcp__parrot-sdd-coder__coder_merge, mcp__parrot-sdd-coder__coder_wait, mcp__parrot-sdd-coder__coder_status, mcp__parrot-sdd-coder__coder_cleanup, mcp__parrot-sdd-coder__coder_record_feedback, mcp__parrot-sdd-coder__coder_record_review, mcp__parrot-sdd-coder__coder_feedback_report, mcp__parrot-sdd-coder__coder_record_native_observation, mcp__parrot-sdd-coder__coder_task_context, mcp__parrot-sdd-coder__coder_delivery_report, mcp__parrot-sdd-coder__coder_read_artifact, mcp__parrot-sdd-coder__coder_bg_status, mcp__parrot-sdd-coder__coder_run_validation, mcp__parrot-bounded-source__source_inspect_batch, mcp__wikitoolkit__ledger_open, mcp__wikitoolkit__ledger_context
+tools: Read, Write, Edit, MultiEdit, Bash, Glob, Grep, Agent, SendMessage, mcp__parrot-sdd-coder__coder_begin_execution, mcp__parrot-sdd-coder__coder_end_execution, mcp__parrot-sdd-coder__coder_suspend_model, mcp__parrot-sdd-coder__coder_plan, mcp__parrot-sdd-coder__coder_run_chunk, mcp__parrot-sdd-coder__coder_prepare_native, mcp__parrot-sdd-coder__coder_merge, mcp__parrot-sdd-coder__coder_wait, mcp__parrot-sdd-coder__coder_status, mcp__parrot-sdd-coder__coder_cleanup, mcp__parrot-sdd-coder__coder_record_feedback, mcp__parrot-sdd-coder__coder_record_review, mcp__parrot-sdd-coder__coder_feedback_report, mcp__parrot-sdd-coder__coder_record_native_observation, mcp__parrot-sdd-coder__coder_task_context, mcp__parrot-sdd-coder__coder_delivery_report, mcp__parrot-sdd-coder__coder_read_artifact, mcp__parrot-sdd-coder__coder_bg_status, mcp__parrot-sdd-coder__coder_run_validation, mcp__parrot-bounded-source__source_inspect_batch, mcp__wikitoolkit__ledger_open, mcp__wikitoolkit__ledger_context
 hooks:
   PreToolUse:
     - matcher: "Bash|Write|Edit|MultiEdit|NotebookEdit"

{querysource-5.1.0 → querysource-5.1.2}/.github/workflows/release.yml

@@ -51,6 +51,9 @@ jobs:
             cd {project}/rust && rm -rf target/wheels &&
             maturin build --release --manylinux off --interpreter python3 &&
             python3 -c "import zipfile,glob,os,shutil;whl=glob.glob('target/wheels/*.whl')[0];zf=zipfile.ZipFile(whl);so=[n for n in zf.namelist() if n.endswith('.so') and '_qs_parsers' in n][0];zf.extract(so,'/tmp/_qs');shutil.copy2(os.path.join('/tmp/_qs',so),'{project}/querysource/qs_parsers/')" &&
+            cd {project}/rust/qsurl && rm -rf target/wheels &&
+            maturin build --release --manylinux off --interpreter python3 &&
+            python3 -c "import zipfile,glob,os,shutil;whl=glob.glob('target/wheels/*.whl')[0];zf=zipfile.ZipFile(whl);so=[n for n in zf.namelist() if n.endswith('.so') and '_qsurl' in n][0];zf.extract(so,'/tmp/_qsurl');shutil.copy2(os.path.join('/tmp/_qsurl',so),'{project}/querysource/qsurl/')" &&
             cd {project}
           CIBW_ENVIRONMENT: "PATH=/root/.cargo/bin:$PATH"
           CIBW_BUILD: ${{ matrix.cibw-build }}

{querysource-5.1.0 → querysource-5.1.2}/Makefile

@@ -62,6 +62,7 @@ develop:
 # root pyproject.toml (which uses setuptools for the rest of the project).
 build-rust:
 	$(MATURIN) develop --release --manifest-path rust/Cargo.toml
+	$(MATURIN) develop --release --manifest-path rust/qsurl/Cargo.toml
 
 # Stage the compiled Rust extension (.so) INTO the source tree so that
 # `uv build` (setuptools backend) bundles it into the wheel. setuptools only
@@ -80,6 +81,15 @@ stage-rust:
 	  find "$$tmp" -name '_qs_parsers*.so' -exec cp {} querysource/qs_parsers/ \; ; \
 	  rm -rf "$$tmp"; \
 	  ls -la querysource/qs_parsers/_qs_parsers*.so
+	$(MATURIN) build --release -i python --manifest-path rust/qsurl/Cargo.toml --out $(RUST_WHEEL_OUT)
+	@whl=$$(ls -t $(RUST_WHEEL_OUT)/qsurl-*.whl | head -1); \
+	  test -n "$$whl" || { echo "ERROR: maturin produced no qsurl wheel in $(RUST_WHEEL_OUT)"; exit 1; }; \
+	  echo "Staging qsurl extension from $$whl"; \
+	  tmp=$$(mktemp -d); \
+	  unzip -o -q "$$whl" -d "$$tmp"; \
+	  find "$$tmp" -name '_qsurl*.so' -exec cp {} querysource/qsurl/ \; ; \
+	  rm -rf "$$tmp"; \
+	  ls -la querysource/qsurl/_qsurl*.so
 
 # Alternative: install without lock file (faster for development)
 develop-fast:

{querysource-5.1.0 → querysource-5.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: querysource
-Version: 5.1.0
+Version: 5.1.2
 Summary: Aiohttp web service for querying several databases easily
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: BSD-3-Clause
@@ -92,6 +92,7 @@ Requires-Dist: altair==5.5.0
 Requires-Dist: tqdm<=4.67.1,>=4.65.0
 Requires-Dist: phonenumbers==9.0.9
 Requires-Dist: navigator-auth>=0.15.8
+Requires-Dist: lark>=1.3.1
 Provides-Extra: analytics
 Requires-Dist: pygwalker>=0.4.8.9; extra == "analytics"
 Requires-Dist: numpy<2.4,>=2.0.0; extra == "analytics"

{querysource-5.1.0 → querysource-5.1.2}/pyproject.toml

@@ -116,6 +116,7 @@ dependencies = [
     "tqdm>=4.65.0,<=4.67.1",
     "phonenumbers==9.0.9",
     "navigator-auth>=0.15.8",
+    "lark>=1.3.1",
 ]
 
 [project.optional-dependencies]
@@ -192,6 +193,7 @@ include = ["querysource", "querysource.*"]
 [tool.setuptools.package-data]
 querysource = ["py.typed"]
 "querysource.qs_parsers" = ["*.so", "*.pyd"]
+"querysource.qsurl" = ["*.so", "*.pyd", "*.lark"]
 
 # Exclude Cython-generated intermediate sources from the wheel.
 # These .c/.cpp files are build artifacts (the compiled .so already ships)

{querysource-5.1.0 → querysource-5.1.2}/querysource/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '5.1.0'
-__version_tuple__ = version_tuple = (5, 1, 0)
+__version__ = version = '5.1.2'
+__version_tuple__ = version_tuple = (5, 1, 2)
 
-__commit_id__ = commit_id = 'g12466af48'
+__commit_id__ = commit_id = 'g09925518a'

{querysource-5.1.0 → querysource-5.1.2}/querysource/conf.py

@@ -1,11 +1,11 @@
 # Import Config Class
-import sys
 import os
+import sys
 from pathlib import Path
+
 from navconfig import BASE_DIR, config
 from navconfig.logging import logging
 
-
 ### Matplotlib Configuration
 logging.getLogger(name='matplotlib').setLevel(logging.WARNING)
 logging.getLogger(name='matplotlib.font_manager').setLevel(logging.ERROR)
@@ -378,6 +378,9 @@ DEFAULT_QUERY_FORMAT = config.get(
     fallback='native'
 )
 
+## qsurl (FEAT-152): rows a pushdown result may have before an in-memory residual plan runs.
+QSURL_MAX_RESIDUAL_ROWS = config.getint("QSURL_MAX_RESIDUAL_ROWS", fallback=50000)
+
 ## Query parameters that must never reach the query parser.
 # Authentication/transport-related params (e.g. ?auth=..., ?apikey=...) get
 # appended to API URLs by upstream gateways/clients and would otherwise be

{querysource-5.1.0 → querysource-5.1.2}/querysource/handlers/__init__.py

@@ -8,6 +8,7 @@ from .executor import QueryExecutor
 from .log import LoggingService
 from .manager import QueryManager
 from .multi import QueryHandler
+from .qsurl import QSUrlService
 from .scheduler import SchedulerJobsView
 from .service import QueryService
 from .tenant import TenantQueryHandler
@@ -15,6 +16,7 @@ from .variables import VariablesService
 
 __all__ = (
     'LoggingService',
+    'QSUrlService',
     'QueryDescribe',
     'QueryExecutor',
     'QueryHandler',

{querysource-5.1.0 → querysource-5.1.2}/querysource/handlers/abstract.py

@@ -136,7 +136,8 @@ class AbstractHandler(BaseHandler):
         message: str = None,
         exception: BaseException = None,
         stacktrace: str = None,
-        code: int = 400
+        code: int = 400,
+        detail: dict | None = None,
     ) -> HTTPException:
         """Error.
 
@@ -155,6 +156,8 @@ class AbstractHandler(BaseHandler):
             stacktrace (str, optional): Pre-captured traceback string.
                 Forwarded to the server log only; never in the client body.
             code (int, optional): HTTP error code. Defaults to 400.
+            detail (dict, optional): client-safe structured error; when given, ``message`` is
+                public in every mode and ``detail`` is emitted as the payload's ``detail``.
         """
         # Map HTTP status code to a formatter category
         if code == 404:
@@ -170,7 +173,8 @@ class AbstractHandler(BaseHandler):
             exception=exception,
             debug=self.debug,
             logger=self.logger,
-            public_message=message if self.debug else None,
+            public_message=message if (self.debug or detail is not None) else None,
+            public_detail=detail,
         )
         args = {
             "reason": payload["error"],

querysource-5.1.2/querysource/handlers/qsurl.py

@@ -0,0 +1,287 @@
+"""qsurl HTTP handler: GET /api/v1/services/qsurl/{path:.*} (FEAT-152)."""
+from __future__ import annotations
+
+import re
+
+from aiohttp import web
+from asyncdb.exceptions import ConnectionTimeout, ProviderError
+
+from ..auth import ResourceType
+from ..conf import CSV_DEFAULT_DELIMITER, CSV_DEFAULT_QUOTING
+from ..exceptions import DriverError, ParserError, QueryException, SlugNotFound
+from ..outputs import DataOutput
+from ..qsurl import QSUrlError, parse
+from ..qsurl.translate import split
+from ..queries.qs import QS
+from ..tenant_errors import TenantError
+from ..tenants import QueryIdentity
+from ..types import graph_ouputs, mime_supported
+from .abstract import AbstractHandler
+
+_BARE_SLUG = re.compile(r"^[A-Za-z0-9_-]+$")
+
+
+class QSUrlService(AbstractHandler):
+    """Parse a qsurl query, enforce PBAC and execute it through QS."""
+
+    async def query(self, request: web.Request) -> web.StreamResponse:
+        """Handle one qsurl read.
+
+        Steps: decode-once source assembly, PBAC ``slug:execute``, output
+        negotiation, capability probe + ``translate.split``, ``QS`` execution
+        with the residual plan, ``datasource:use``/``driver:use`` PBAC, then
+        ``DataOutput``.
+
+        Raises:
+            web.HTTPException: 400 (qsurl errors, with ``detail``), 404 (PBAC deny), others as QueryService.
+        """
+        params = self.query_parameters(request)
+        args = self.match_parameters(request)
+        try:
+            options = await self.json_data(request) or {}
+        except (TypeError, ValueError):
+            options = {}
+        try:
+            source = self._source(args.pop("path", ""), params.pop("q", None))
+            ir = parse(source)
+            slug = ir["slug"]
+            tenant = request.get("qs_tenant")
+            await self._enforce_slug_execute(request, slug, tenant)
+            queryformat, output_args = self._output_args(request, params, options)
+            caps, scan = await self.resolve_capabilities(request, slug, tenant)
+            pushdown, plan = split(ir, caps, residual_scan=scan)
+            conditions = {**options, **params, **pushdown}
+            self.logger.debug(
+                f'qsurl Slug: {slug}, format: {queryformat}, conditions: {conditions}'
+            )
+            if query := await self.get_source(
+                request, slug, conditions, driver=args, tenant=tenant,
+                definition=request.get('qs_definition'), residual=plan,
+            ):
+                try:
+                    await query.build_provider()
+                except SlugNotFound as err:
+                    raise self.Error(
+                        message=f"Slug Not Found: {slug}",
+                        exception=err,
+                        code=400
+                    ) from err
+                except TenantError as err:
+                    raise self.Error(
+                        message=str(err),
+                        exception=err,
+                        code=err.code
+                    ) from err
+                except ParserError as err:
+                    raise self.Error(
+                        message=f"Error parsing Query Slug {slug}",
+                        exception=err
+                    ) from err
+                except (ProviderError, DriverError) as err:
+                    raise self.Error(
+                        message="Connection Error",
+                        exception=err
+                    ) from err
+                except Exception as ex:
+                    raise self.Except(
+                        message="Unknown Error on Query",
+                        exception=ex
+                    ) from ex
+                # PBAC: enforce datasource:use and driver:use after provider is resolved.
+                _ds_name = getattr(
+                    getattr(getattr(query, '_qs', None), '_definition', None),
+                    'provider',
+                    'db'
+                ) or 'db'
+                _drv_name = getattr(
+                    getattr(query, '_provider', None),
+                    'driver',
+                    'db'
+                ) or 'db'
+                await self._enforce_pbac(
+                    request,
+                    resource_type=ResourceType.DATASOURCE,
+                    resource_name=_ds_name,
+                    action="datasource:use",
+                )
+                await self._enforce_pbac(
+                    request,
+                    resource_type=ResourceType.DRIVER,
+                    resource_name=_drv_name,
+                    action="driver:use",
+                )
+                if not queryformat:
+                    if ctype := query.accepts():
+                        queryformat = mime_supported[ctype]
+                try:
+                    output = DataOutput(
+                        request,
+                        query=query,
+                        ctype=queryformat,
+                        slug=slug,
+                        **output_args
+                    )
+                    return await output.response()
+                except ConnectionTimeout as err:
+                    raise DriverError(
+                        f"Connection Timeout: {err}"
+                    ) from err
+                except (ProviderError, DriverError) as err:
+                    raise self.Error(
+                        message="DataOutput Error",
+                        exception=err,
+                        code=402
+                    ) from err
+                except QSUrlError:
+                    raise
+                except (QueryException, Exception) as ex:
+                    raise self.Except(
+                        message=f"Error on Query: {slug}",
+                        exception=ex
+                    ) from ex
+            else:
+                raise self.Error(
+                    message=f"Unable to get Provider for slug: {slug}"
+                )
+        except QSUrlError as err:
+            raise self.Error(message=err.message, exception=err, code=400, detail=err.to_dict()) from err
+        except web.HTTPException:
+            raise
+        except (ProviderError, DriverError) as err:
+            raise self.Error(
+                message='Query Failed',
+                exception=err
+            ) from err
+        except (QueryException, Exception) as ex:
+            raise self.Except(
+                message="Uncaught Error on Query",
+                exception=ex
+            ) from ex
+
+    @staticmethod
+    def _source(path: str, q: str | None) -> str:
+        """Return the qsurl source; never re-decodes (aiohttp decoded once)."""
+        if q is None:
+            return path
+        if not _BARE_SLUG.match(path):
+            raise QSUrlError("parse", "with ?q= the path must be a bare slug; q carries everything after it")
+        return f"{path}{q}"
+
+    async def _enforce_slug_execute(self, request: web.Request, slug: str, tenant: str | None) -> None:
+        """PBAC slug:execute, tenant-aware — verbatim logic of service.py:200-225."""
+        registry = request.app.get("qs_tenant_registry")
+        if registry is not None:
+            try:
+                store = registry.resolve(tenant)
+            except web.HTTPNotFound:
+                raise
+            except Exception as exc:  # pylint: disable=W0703
+                self.logger.warning(
+                    "QSUrlService ownership pre-flight error (fail-closed): %s",
+                    exc,
+                )
+                raise web.HTTPNotFound() from exc
+            identity = QueryIdentity(store=store, slug=slug)
+            await self._enforce_owned_slug(
+                request,
+                identity=identity,
+                action="slug:execute",
+            )
+        else:
+            await self._enforce_pbac(
+                request,
+                resource_type=ResourceType.SLUG,
+                resource_name=slug,
+                action="slug:execute",
+            )
+
+    def _output_args(self, request: web.Request, params: dict, options: dict) -> tuple[str | None, dict]:
+        """queryformat + DataOutput kwargs — verbatim logic of service.py:227-289 (pops its keys from params/options)."""
+        _format: str | None = None
+        writer_options: dict = {}
+        try:
+            _format = params['queryformat']
+            del params['queryformat']
+        except KeyError:
+            pass
+        try:
+            _format, tpl = _format.split('=')
+        except (AttributeError, ValueError):
+            tpl = None
+        if tpl:
+            try:
+                report = options['_report_options']
+            except (TypeError, KeyError):
+                report = {}
+            writer_options = {
+                "template": tpl,
+                **report
+            }
+        queryformat = self.format(request, params, _format)
+        try:
+            download = params['_download']
+            del params['_download']
+        except KeyError:
+            download = False
+        try:
+            filename = params['_filename']
+            del params['_filename']
+        except KeyError:
+            filename = None
+        if _format == 'csv':
+            try:
+                writer_options = options['_csv_options']
+                del options['_csv_options']
+            except (TypeError, KeyError):
+                writer_options = {
+                    "delimiter": CSV_DEFAULT_DELIMITER,
+                    "quoting": CSV_DEFAULT_QUOTING
+                }
+        else:
+            try:
+                writer_options = options['_output_options']
+                del options['_output_options']
+            except (TypeError, KeyError):
+                pass
+            try:
+                del options['_csv_options']
+            except (TypeError, KeyError):
+                pass
+        if queryformat in graph_ouputs:
+            writer_options = {}
+        try:
+            writer_options = options['_graph_options']
+            del options['_graph_options']
+        except (TypeError, KeyError):
+            pass
+        output_args = {
+            "filename": filename,
+            "download": download,
+            "writer_options": writer_options,
+        }
+        return queryformat, output_args
+
+    async def resolve_capabilities(self, request: web.Request, slug: str, tenant: str | None) -> tuple[frozenset[str], bool]:
+        """Resolve (capabilities, residual_scan) of the provider that will execute ``slug`` via a lazy probe QS."""
+        probe = QS(slug=slug, conditions={}, request=request, tenant=tenant,
+                   definition=request.get("qs_definition"), lazy=True)
+        try:
+            await probe.build_provider()
+            return probe._qs.capabilities, probe._qs.residual_scan  # pylint: disable=protected-access
+        except SlugNotFound as err:
+            raise self.Error(
+                message=f"Slug Not Found: {slug}",
+                exception=err,
+                code=400
+            ) from err
+        except TenantError as err:
+            raise self.Error(
+                message=str(err),
+                exception=err,
+                code=err.code
+            ) from err
+        finally:
+            try:
+                await probe.close()
+            except Exception:  # pylint: disable=W0703
+                pass

{querysource-5.1.0 → querysource-5.1.2}/querysource/handlers/tenant.py

@@ -292,13 +292,13 @@ class TenantQueryHandler(AbstractHandler):
             if is_multi:
                 from .multi import QueryHandler
 
-                return await QueryHandler(request).query(request)
+                return await QueryHandler().query(request)
             from .service import QueryService
 
-            return await QueryService(request).query(request)
+            return await QueryService().query(request)
         from .multi import QueryHandler
 
-        handler = QueryHandler(request)
+        handler = QueryHandler()
         return await handler.query(request)
 
     async def columns(self, request: web.Request) -> web.StreamResponse:
@@ -310,10 +310,10 @@ class TenantQueryHandler(AbstractHandler):
         if is_multi:
             from .multi import QueryHandler
 
-            return await QueryHandler(request).columns(request)
+            return await QueryHandler().columns(request)
         from .service import QueryService
 
-        handler = QueryService(request)
+        handler = QueryService()
         if request.method == "HEAD":
             return await handler.get_columns(request)
         return await handler.columns(request)
@@ -327,8 +327,8 @@ class TenantQueryHandler(AbstractHandler):
         if is_multi:
             from .multi import QueryHandler
 
-            return await QueryHandler(request).test_slug(request)
+            return await QueryHandler().test_slug(request)
         from .service import QueryService
 
-        handler = QueryService(request)
+        handler = QueryService()
         return await handler.test_slug(request)

{querysource-5.1.0 → querysource-5.1.2}/querysource/outputs/output.py

@@ -14,6 +14,7 @@ from ..exceptions import (
 )
 from ..interfaces.queries import AbstractQuery
 from ..ownership_logging import implicit_artifact_name
+from ..qsurl.errors import QSUrlError
 from ..utils.errors import build_error_payload
 from .writers import (
     BokehWriter,
@@ -234,6 +235,8 @@ class DataOutput:
             ### Return data on Output:
             try:
                 await writer.get_result()
+            except QSUrlError:
+                raise  # FEAT-152: the qsurl handler answers 400 with the structured detail
             except (NoDataFound, DataNotFound) as err:
                 _msg = f"{err!s}" if DEBUG else "Data not found"
                 headers = {