querysource 4.3.0__tar.gz → 4.4.0__tar.gz

{querysource-4.3.0 → querysource-4.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: querysource
-Version: 4.3.0
+Version: 4.4.0
 Summary: Aiohttp web service for querying several databases easily
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: BSD-3-Clause

{querysource-4.3.0 → querysource-4.4.0}/qs-parser/Cargo.lock

@@ -119,9 +119,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.23.5"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"
+checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229"
 dependencies = [
  "cfg-if",
  "indoc",
@@ -137,9 +137,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.23.5"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"
+checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -147,9 +147,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.23.5"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"
+checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -157,9 +157,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.23.5"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"
+checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -169,9 +169,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.23.5"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"
+checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -311,9 +311,9 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.16"
+version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "unicode-ident"

{querysource-4.3.0 → querysource-4.4.0}/qs-parser/Cargo.toml

@@ -10,7 +10,7 @@ name = "qs_parser"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.23.4", features = ["extension-module"] }
+pyo3 = { version = "0.24.1", features = ["extension-module"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 rayon = "1.10"

{querysource-4.3.0 → querysource-4.4.0}/querysource/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '4.3.1.dev0+gec7dea294.d20260523'
-__version_tuple__ = version_tuple = (4, 3, 1, 'dev0', 'gec7dea294.d20260523')
+__version__ = version = '4.4.1.dev0+g74f4a954b.d20260527'
+__version_tuple__ = version_tuple = (4, 4, 1, 'dev0', 'g74f4a954b.d20260527')
 
-__commit_id__ = commit_id = 'gec7dea294'
+__commit_id__ = commit_id = 'g74f4a954b'

{querysource-4.3.0 → querysource-4.4.0}/querysource/conf.py

@@ -280,6 +280,20 @@ JIRA_CERT = config.get('JIRA_CERT')
 HTTPCLIENT_MAX_SEMAPHORE = config.getint("HTTPCLIENT_MAX_SEMAPHORE", fallback=5)
 HTTPCLIENT_MAX_WORKERS = config.getint("HTTPCLIENT_MAX_WORKERS", fallback=1)
 
+## MultiQS thread guardrails:
+MULTIQS_MAX_CONCURRENT_THREADS = config.getint(
+    "MULTIQS_MAX_CONCURRENT_THREADS",
+    fallback=4,
+)
+MULTIQS_MAX_SOURCES_PER_REQUEST = config.getint(
+    "MULTIQS_MAX_SOURCES_PER_REQUEST",
+    fallback=25,
+)
+MULTIQS_SOURCE_TIMEOUT_SECONDS = config.getint(
+    "MULTIQS_SOURCE_TIMEOUT_SECONDS",
+    fallback=30,
+)
+
 ## Google API:
 GOOGLE_API_KEY = config.get('GOOGLE_API_KEY')
 GOOGLE_SEARCH_API_KEY = config.get('GOOGLE_SEARCH_API_KEY')
@@ -439,6 +453,33 @@ QS_AIRTABLE_OAUTH_ENABLED = config.getboolean(
     'QS_AIRTABLE_OAUTH_ENABLED', fallback=False
 )
 
+# ── QWorker Remote Execution (FEAT-101) ───────────────────────────────────────
+# Host/port of the remote qworker server used when MultiQS queries have
+# ``remote: true``.  QWORKER_HOST=None means remote execution is not configured
+# and any ``remote: true`` query without an explicit ``worker:`` key will raise
+# DriverError at dispatch time.
+QWORKER_HOST = config.get('QWORKER_HOST', fallback=None)
+QWORKER_PORT = config.getint('QWORKER_PORT', fallback=8888)
+# TCP connection timeout in seconds (how long QClient waits to establish a
+# connection to the qworker server).
+QWORKER_TIMEOUT = config.getint('QWORKER_TIMEOUT', fallback=5)
+# Query execution timeout in seconds (wraps the entire client.run() call via
+# asyncio.wait_for; independent from the TCP connection timeout above).
+QWORKER_QUERY_TIMEOUT = config.getint('QWORKER_QUERY_TIMEOUT', fallback=60)
+
+# Pre-parsed worker list for QClient.
+# Supports "host1:port1,host2:port2" or plain "host1,host2" (uses QWORKER_PORT).
+_qworker_hosts_raw = config.get('QWORKER_HOST', fallback=None)
+QWORKER_WORKERS: list = []
+if _qworker_hosts_raw:
+    for _h in _qworker_hosts_raw.split(','):
+        _h = _h.strip()
+        if ':' in _h:
+            _hh, _hp = _h.rsplit(':', 1)
+            QWORKER_WORKERS.append((_hh.strip(), int(_hp)))
+        else:
+            QWORKER_WORKERS.append((_h, QWORKER_PORT))
+
 try:
     from settings.settings import *  # pylint: disable=W0614,W0401 # noqa
 except ImportError:

{querysource-4.3.0 → querysource-4.4.0}/querysource/handlers/multi.py

@@ -231,15 +231,25 @@ class QueryHandler(AbstractHandler):
             result, options = await qs.query()
         except DataNotFound as dnf:
             total_time = time.monotonic() - started_at
+            _remote_queries_on_err = getattr(qs, '_remote_queries', [])
+            if _remote_queries_on_err:
+                self.logger.warning(
+                    "MultiQuery DataNotFound after remote queries %s: %s",
+                    _remote_queries_on_err,
+                    dnf,
+                )
+            _err_headers = {
+                'Content-Type': 'application/json',
+                'X-Slug': slug,
+                'X-Format': queryformat,
+                'X-Total-Time': f'{total_time:.2f} seconds',
+                'X-Error': str(dnf),
+            }
+            if _remote_queries_on_err:
+                _err_headers['X-Remote-Queries'] = ','.join(_remote_queries_on_err)
             return self.NoData(
                 message=str(dnf),
-                headers={
-                    'Content-Type': 'application/json',
-                    'X-Slug': slug,
-                    'X-Format': queryformat,
-                    'X-Total-Time': f'{total_time:.2f} seconds',
-                    'X-Error': str(dnf),
-                },
+                headers=_err_headers,
             )
         except SlugNotFound as snf:
             raise self.Error(
@@ -255,6 +265,13 @@ class QueryHandler(AbstractHandler):
             )
         except (QueryException, DriverError) as qe:
             trace = traceback.format_exc()
+            _remote_queries_on_err = getattr(qs, '_remote_queries', [])
+            if _remote_queries_on_err:
+                self.logger.warning(
+                    "MultiQuery error after remote queries %s: %s",
+                    _remote_queries_on_err,
+                    qe,
+                )
             self.logger.exception(qe, stack_info=True)
             raise self.Error(
                 message="Query Error",
@@ -264,6 +281,13 @@ class QueryHandler(AbstractHandler):
             )
         except Exception as ex:
             trace = traceback.format_exc()
+            _remote_queries_on_err = getattr(qs, '_remote_queries', [])
+            if _remote_queries_on_err:
+                self.logger.warning(
+                    "MultiQuery unexpected error after remote queries %s: %s",
+                    _remote_queries_on_err,
+                    ex,
+                )
             self.logger.exception(ex, stack_info=True)
             raise self.Except(
                 message=f"Unknown Error on Query: {ex!s}",
@@ -381,7 +405,12 @@ class QueryHandler(AbstractHandler):
             self.logger.debug(
                 f'Query Duration: {total_time:.2f} seconds'
             )
-            return await output.response()
+            response = await output.response()
+            # Add X-Remote-Queries header if any queries ran remotely (FEAT-101).
+            remote_queries = getattr(qs, '_remote_queries', [])
+            if remote_queries:
+                response.headers['X-Remote-Queries'] = ','.join(remote_queries)
+            return response
         except (DataNotFound) as ex:
             return self.NoData(
                 message="No Data was Found",

querysource-4.4.0/querysource/handlers/scheduler.py

@@ -0,0 +1,307 @@
+"""Scheduler jobstore handler — FEAT-100.
+
+HTTP surface on top of QSScheduler's APScheduler jobstore.
+
+Routes (registered in querysource/services.py when ENABLE_QS_SCHEDULER=True):
+    GET    /api/v1/qs/scheduler/jobs            -> list all jobs
+    GET    /api/v1/qs/scheduler/jobs/{job_id}   -> single job, or 404
+    POST   /api/v1/qs/scheduler/jobs            -> sync a slug's job(s) live
+    DELETE /api/v1/qs/scheduler/jobs/{job_id}   -> remove a job live
+    PATCH  /api/v1/qs/scheduler/jobs/{job_id}   -> pause/resume a job live
+
+The write verbs mutate the running APScheduler so schedules take effect
+without a restart. The DB (public.queries) stays the source of truth:
+startup rebuilds every job, and POST mirrors a single slug's row into the
+live scheduler.
+
+Note: Per-process visibility only. MemoryJobStore lives in the running
+process — multiple QS instances each report (and mutate) their own jobs.
+"""
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Optional
+
+from aiohttp import web
+from navigator.views import BaseView
+
+if TYPE_CHECKING:
+    from apscheduler.job import Job
+    from querysource.scheduler import QSScheduler
+
+logger = logging.getLogger("QS.SchedulerJobsView")
+
+
+def _kind_from_id(job_id: str) -> str:
+    """Map job ID prefix to its kind.
+
+    Args:
+        job_id: APScheduler job ID, e.g. ``"query_foo"``.
+
+    Returns:
+        One of ``"query"``, ``"multi"``, ``"cache"``, ``"unknown"``.
+    """
+    if job_id.startswith("query_"):
+        return "query"
+    if job_id.startswith("multi_"):
+        return "multi"
+    if job_id.startswith("cache_"):
+        return "cache"
+    return "unknown"
+
+
+class SchedulerJobsView(BaseView):
+    """Class-based aiohttp view exposing the QSScheduler jobstore.
+
+        GET     /api/v1/qs/scheduler/jobs            — list all jobs
+        GET     /api/v1/qs/scheduler/jobs/{job_id}   — single job by ID
+        POST    /api/v1/qs/scheduler/jobs            — sync a slug's job(s) live
+        DELETE  /api/v1/qs/scheduler/jobs/{job_id}   — remove a job live
+        PATCH   /api/v1/qs/scheduler/jobs/{job_id}   — pause/resume a job live
+
+    Note: Per-process visibility only. MemoryJobStore lives in the running
+    process — if multiple QS instances run with the scheduler enabled, each
+    instance answers only for (and mutates) its own jobstore.
+    """
+
+    def _get_scheduler(self) -> Optional["QSScheduler"]:
+        """Retrieve the QSScheduler instance from the app state.
+
+        Returns:
+            The QSScheduler stored in ``app["qs_scheduler"]``, or None if
+            not present (e.g. during startup or if scheduler was not
+            initialised).
+        """
+        return self.request.app.get("qs_scheduler")
+
+    def _serialize_job(self, job: "Job") -> dict:
+        """Serialize an APScheduler Job to a plain dict.
+
+        Only extracts ``slug`` from ``job.kwargs`` — never includes the raw
+        kwargs dict because it contains non-serialisable internal objects
+        such as ``notification_manager``.
+
+        Args:
+            job: An APScheduler ``Job`` instance.
+
+        Returns:
+            A dict with keys: ``id``, ``name``, ``kind``, ``slug``,
+            ``next_run_time``, ``trigger``, ``coalesce``, ``max_instances``,
+            ``misfire_grace_time``, ``pending``.
+        """
+        # APScheduler Job uses __slots__; some slots are only set after the
+        # scheduler starts (e.g. next_run_time). Use getattr with a default
+        # to avoid AttributeError when the scheduler is pending/unstarted.
+        nrt = getattr(job, "next_run_time", None)
+        trigger_cls_name = type(job.trigger).__name__  # e.g. "IntervalTrigger"
+        # Strip the "Trigger" suffix and lowercase to get "interval" / "cron"
+        trigger_type = trigger_cls_name.lower()
+        if trigger_type.endswith("trigger"):
+            trigger_type = trigger_type[: -len("trigger")]
+
+        slug: Optional[str] = (job.kwargs or {}).get("slug")
+
+        return {
+            "id": job.id,
+            "name": job.name,
+            "kind": _kind_from_id(job.id),
+            "slug": slug,
+            "next_run_time": nrt.isoformat() if nrt is not None else None,
+            "trigger": {
+                "type": trigger_type,
+                "repr": str(job.trigger),
+            },
+            "coalesce": bool(getattr(job, "coalesce", False)),
+            "max_instances": int(getattr(job, "max_instances", 1)),
+            "misfire_grace_time": getattr(job, "misfire_grace_time", None),
+            "pending": bool(getattr(job, "pending", True)),
+        }
+
+    async def get(self) -> web.Response:
+        """Handle GET requests for the scheduler jobs endpoint.
+
+        Dispatches based on the presence of ``{job_id}`` in the URL:
+
+        - ``GET /api/v1/qs/scheduler/jobs``            — list all jobs.
+        - ``GET /api/v1/qs/scheduler/jobs/{job_id}``   — single job lookup.
+
+        Returns:
+            - **200** with the list envelope or single job dict.
+            - **404** when ``job_id`` is provided but no matching job exists.
+            - **503** when ``app["qs_scheduler"]`` is missing (e.g.
+              mid-startup race between route registration and
+              ``QSScheduler.startup``).
+        """
+        scheduler = self._get_scheduler()
+        if scheduler is None:
+            return self._unavailable_response()
+
+        aps = scheduler._scheduler  # AsyncIOScheduler instance
+        params = self.match_parameters(self.request)
+        job_id = params.get("job_id")
+
+        if job_id:
+            job = aps.get_job(job_id) if aps is not None else None
+            if job is None:
+                logger.debug("Job '%s' not found in scheduler.", job_id)
+                return self.json_response(
+                    response={"error": f"Job '{job_id}' not found."},
+                    status=404,
+                )
+            return self.json_response(
+                response=self._serialize_job(job),
+                status=200,
+            )
+
+        # List all jobs
+        jobs = aps.get_jobs() if aps is not None else []
+        logger.debug("Listing %d scheduler job(s).", len(jobs))
+        return self.json_response(
+            response={
+                "scheduler": {
+                    "enabled": True,
+                    "running": bool(aps.running) if aps is not None else False,
+                    "timezone": str(scheduler._timezone),
+                    "job_count": len(jobs),
+                },
+                "jobs": [self._serialize_job(j) for j in jobs],
+            },
+            status=200,
+        )
+
+    def _unavailable_response(self) -> web.Response:
+        """503 envelope used when the scheduler isn't in app state yet."""
+        logger.warning(
+            "scheduler endpoint called but 'qs_scheduler' is not in app state — "
+            "scheduler may still be starting up."
+        )
+        return self.json_response(
+            response={
+                "error": (
+                    "QSScheduler is not available yet — the scheduler is "
+                    "enabled but the application has not finished starting up."
+                ),
+                "scheduler": {"enabled": True, "running": False},
+            },
+            status=503,
+        )
+
+    async def post(self) -> web.Response:
+        """Register/sync a slug's scheduled job(s) into the live scheduler.
+
+        Body: ``{"slug": "<query_slug>"}``.
+
+        Reads the slug's current ``public.queries`` row and (re)registers its
+        query/multi/cache jobs without a restart. If the slug no longer has a
+        scheduler definition, its jobs are removed. The DB row written by the
+        slug-management API is the source of truth; this only mirrors it into
+        the running scheduler.
+
+        Returns:
+            - **200** ``{status, slug, registered: [...], removed: [...]}``.
+            - **400** when the body is not JSON or ``slug`` is missing.
+            - **503** when the scheduler isn't available yet.
+        """
+        scheduler = self._get_scheduler()
+        if scheduler is None:
+            return self._unavailable_response()
+
+        try:
+            body = await self.request.json()
+        except Exception:  # noqa: BLE001
+            return self.json_response(
+                response={"error": "Request body must be valid JSON."},
+                status=400,
+            )
+        slug = body.get("slug") if isinstance(body, dict) else None
+        if not slug or not isinstance(slug, str):
+            return self.json_response(
+                response={"error": "Body must include a non-empty 'slug' string."},
+                status=400,
+            )
+
+        try:
+            result = await scheduler.register_slug(slug)
+        except Exception as exc:  # noqa: BLE001
+            logger.error("Error registering slug '%s': %s", slug, exc)
+            return self.json_response(
+                response={"error": str(exc)},
+                status=500,
+            )
+        return self.json_response(response={"status": "ok", **result}, status=200)
+
+    async def delete(self) -> web.Response:
+        """Remove a single job from the live scheduler.
+
+        ``DELETE /api/v1/qs/scheduler/jobs/{job_id}``.
+
+        Returns:
+            - **200** ``{status: "removed", id}`` on success.
+            - **400** when ``job_id`` is missing from the path.
+            - **404** when no such job exists.
+            - **503** when the scheduler isn't available yet.
+        """
+        scheduler = self._get_scheduler()
+        if scheduler is None:
+            return self._unavailable_response()
+
+        job_id = self.match_parameters(self.request).get("job_id")
+        if not job_id:
+            return self.json_response(
+                response={"error": "job_id is required in the path."},
+                status=400,
+            )
+        if not scheduler.remove_job(job_id):
+            return self.json_response(
+                response={"error": f"Job '{job_id}' not found."},
+                status=404,
+            )
+        return self.json_response(
+            response={"status": "removed", "id": job_id},
+            status=200,
+        )
+
+    async def patch(self) -> web.Response:
+        """Pause or resume a live job.
+
+        ``PATCH /api/v1/qs/scheduler/jobs/{job_id}`` with body
+        ``{"action": "pause" | "resume"}``.
+
+        Returns:
+            - **200** ``{status: "paused"|"resumed", id}`` on success.
+            - **400** when ``job_id`` or a valid ``action`` is missing.
+            - **404** when no such job exists.
+            - **503** when the scheduler isn't available yet.
+        """
+        scheduler = self._get_scheduler()
+        if scheduler is None:
+            return self._unavailable_response()
+
+        job_id = self.match_parameters(self.request).get("job_id")
+        if not job_id:
+            return self.json_response(
+                response={"error": "job_id is required in the path."},
+                status=400,
+            )
+        try:
+            body = await self.request.json()
+        except Exception:  # noqa: BLE001
+            return self.json_response(
+                response={"error": "Request body must be valid JSON."},
+                status=400,
+            )
+        action = body.get("action") if isinstance(body, dict) else None
+        if action not in ("pause", "resume"):
+            return self.json_response(
+                response={"error": "Body 'action' must be 'pause' or 'resume'."},
+                status=400,
+            )
+        if not scheduler.set_job_paused(job_id, action == "pause"):
+            return self.json_response(
+                response={"error": f"Job '{job_id}' not found."},
+                status=404,
+            )
+        return self.json_response(
+            response={"status": f"{action}d", "id": job_id},
+            status=200,
+        )

{querysource-4.3.0 → querysource-4.4.0}/querysource/outputs/writers/json.py

@@ -15,11 +15,27 @@ class jsonWriter(AbstractWriter):
             from pandas import DataFrame
             is_dataframe = isinstance(self.data, DataFrame)
         except ImportError:
+            DataFrame = None  # type: ignore
             is_dataframe = False
+        # Multi-source pipelines without an operator (no Join/Concat/Melt/Merge)
+        # return a dict of DataFrames (one per source). Serialize each frame to
+        # records so the frontend can render per-source tabs.
+        is_multi_df = (
+            DataFrame is not None
+            and isinstance(self.data, dict)
+            and len(self.data) > 0
+            and all(isinstance(v, DataFrame) for v in self.data.values())
+        )
         if is_dataframe:
             # Convert to a list of dictionaries
             data_dict = self.data.to_dict(orient='records')
             data = self._json.dumps(data_dict)
+        elif is_multi_df:
+            data_dict = {
+                key: frame.to_dict(orient='records')
+                for key, frame in self.data.items()
+            }
+            data = self._json.dumps(data_dict)
         else:
             try:
                 data = self._json.dumps(self.data)