querysource 4.5.13__tar.gz → 4.5.14__tar.gz

querysource-4.5.14/.parrot/wiki.json

@@ -0,0 +1,29 @@
+{
+  "wiki_name": "querysource",
+  "storage_dir": ".parrot/wiki",
+  "backend": "sqlite",
+  "include_suffixes": [],
+  "exclude_dirs": [
+    ".parrot/wiki"
+  ],
+  "body_max_chars": 16000,
+  "max_file_kb": 512,
+  "claude": {
+    "nudge_cooldown_seconds": 60,
+    "nudge_tools": [
+      "Grep",
+      "Glob",
+      "Read",
+      "Bash"
+    ]
+  },
+  "sync_graph": false,
+  "arango_database": null,
+  "arango_credentials_env": "ARANGODB",
+  "arango_text_analyzer": "text_en",
+  "vault_dir": null,
+  "namespaces": {},
+  "obsidian_sync": null,
+  "symbol_depth": 2,
+  "structural_backend": true
+}

querysource-4.5.14/.parrot/wiki.local.json

@@ -0,0 +1,3 @@
+{
+  "backend": "sqlite"
+}

{querysource-4.5.13 → querysource-4.5.14}/CHANGES.rst

@@ -1,6 +1,38 @@
 Unreleased
 ==========
 
+Row-oriented outputs — DataFrame results and swallowed errors
+-------------------------------------------------------------
+
+The ``iter`` output format now honours its contract and returns a list of
+dictionaries for ``pandas.DataFrame`` results (``bigquery``, ``deltatbl``,
+``iceberg`` providers) instead of passing the frame through. Iterating a
+DataFrame yields column names, not rows, so every writer declaring
+``output_format = 'iter'`` received garbage: ``aiocsv`` raised
+``AttributeError`` in ``CSVWriter``/``TSVWriter`` and the writers'
+``TmpFile.__aexit__`` returned a truthy value, suppressing the exception —
+``slug:csv`` answered HTTP 200 with only the header line. ``slug:txt`` and the
+report writers (``slug:html``, ``slug:pdf``) were broken by the same cause.
+
+- ``iterFormat.serialize`` converts DataFrames to records
+  (``NaN``/``NaT``/``NA`` -> ``None``) via the new
+  ``querysource.utils.dataframes`` helpers.
+- ``CSVWriter``/``TSVWriter`` apply the same normalisation defensively, for
+  callers handing a frame straight to a writer.
+- ``TmpFile.__aexit__`` no longer suppresses exceptions, so writer errors
+  surface as HTTP 500 instead of a silent, truncated 200.
+Drop invalid ``Content-Range`` header from streamed responses
+------------------------------------------------------------
+
+``AbstractWriter.stream_response()`` advertised
+``Content-Range: bytes 0-16384/<content-length>`` on ``200`` responses that
+carry the **full** body. The header is only defined for ``206``/``416``
+(RFC 9110 s14.4), the range is off by one (inclusive positions, so
+``0-16384`` spans 16385 bytes) and for any body under 16385 bytes the
+last-byte-pos exceeded the complete length, making the field value invalid.
+QuerySource does not honour request ``Range`` headers at all, so nothing
+relied on it. ``Content-Length`` is unchanged.
+
 FEAT-090 — Query Slug list pagination
 -------------------------------------
 

{querysource-4.5.13 → querysource-4.5.14}/Makefile

@@ -28,9 +28,14 @@ venv:
 	@echo 'run `source .venv/bin/activate` to start develop with QuerySource.'
 
 # Install production dependencies using lock file
-install: build-rust
+# NOTE: uv sync must run BEFORE build-rust — build-rust invokes .venv/bin/maturin,
+# which only exists once it has been installed. maturin is a `dev`-only extra
+# (see pyproject.toml), so it is excluded from `--no-dev` and installed explicitly.
+install:
 	uv sync --frozen --no-dev --extra analytics --extra vectors
 	uv pip install navigator-api[uvloop,locale]
+	uv pip install "maturin>=1.7,<2.0"
+	$(MAKE) build-rust
 	@echo "Production dependencies installed. Use 'make develop' for development setup."
 
 # Generate lock files (uv only)
@@ -42,8 +47,13 @@ else
 endif
 
 # Install all dependencies including dev dependencies
-develop: build-rust
+# NOTE: uv sync must run BEFORE build-rust — build-rust invokes .venv/bin/maturin,
+# which is installed by the `dev` extra during uv sync. Making build-rust a
+# prerequisite (instead of a recipe step) would run it first and fail on a
+# fresh .venv where maturin isn't installed yet.
+develop:
 	uv sync --frozen --extra analytics --extra dev
+	$(MAKE) build-rust
 	$(MAKE) build-inplace
 
 # Build the Rust-accelerated parser extension (querysource.qs_parsers._qs_parsers)

{querysource-4.5.13 → querysource-4.5.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: querysource
-Version: 4.5.13
+Version: 4.5.14
 Summary: Aiohttp web service for querying several databases easily
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: BSD-3-Clause

{querysource-4.5.13 → querysource-4.5.14}/querysource/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '4.5.13'
-__version_tuple__ = version_tuple = (4, 5, 13)
+__version__ = version = '4.5.14'
+__version_tuple__ = version_tuple = (4, 5, 14)
 
-__commit_id__ = commit_id = 'g31553f851'
+__commit_id__ = commit_id = 'g5872d5ef4'

{querysource-4.5.13 → querysource-4.5.14}/querysource/outputs/dt/iter.py

@@ -8,6 +8,7 @@ try:
     from google.cloud.bigquery.table import RowIterator
 except ImportError:
     RowIterator = None
+from ...utils.dataframes import df_to_records
 from .abstract import OutputFormat
 
 
@@ -17,7 +18,11 @@ class iterFormat(OutputFormat):
     """
     async def serialize(self, result, error, *args, **kwargs):
         if isinstance(result, pandas.DataFrame):
-            data = result
+            # pandas-backed providers (bigquery, deltatbl, iceberg) return a
+            # DataFrame. Iterating it yields column names, not rows, so every
+            # row-oriented consumer (csv, tsv, txt, report/pdf) has to receive
+            # records here — this format's contract is a list of dictionaries.
+            data = df_to_records(result)
         elif isinstance(result, (RowIterator, list)):
             data = [dict(row) for row in result]
         else:

{querysource-4.5.13 → querysource-4.5.14}/querysource/outputs/writers/abstract.py

@@ -15,6 +15,7 @@ from aiohttp.web_exceptions import (
 from ...interfaces.queries import AbstractQuery
 from ...libs.encoders import DefaultEncoder
 from ...utils.functions import check_empty
+from ...utils.dataframes import df_to_records
 from ...utils.errors import build_error_payload
 from ...exceptions import (
     CacheException,
@@ -157,10 +158,9 @@ class AbstractWriter(ABC):
         content_length = len(data)
         response.content_length = content_length
         if self.response_type == 'stream':  # an stream response:
+            # NOTE: no Content-Range header here: the full body is sent with a
+            # 200 status and range requests are not supported.
             chunk_size = 16384
-            response.headers[
-                "Content-Range"
-            ] = f"bytes 0-{chunk_size}/{content_length}"
             try:
                 i = 0
                 await response.prepare(self.request)
@@ -305,8 +305,9 @@ class AbstractWriter(ABC):
                     from pandas import DataFrame
                     if isinstance(self.query, DataFrame):
                         if self.output_format == 'iter':
-                             # convert dataframe into a list of dictionaries:
-                            self.data = self.query.to_dict(orient='records')
+                            # convert dataframe into a list of dictionaries,
+                            # same normalisation the 'iter' format applies:
+                            self.data = df_to_records(self.query)
                         else:
                             self.data = self.query
                         error = None

{querysource-4.5.13 → querysource-4.5.14}/querysource/outputs/writers/csv.py

@@ -6,6 +6,7 @@ from ...conf import (
     CSV_DEFAULT_DELIMITER,
     CSV_DEFAULT_QUOTING
 )
+from ...utils.dataframes import df_to_records, is_dataframe
 from .abstract import AbstractWriter
 
 
@@ -22,7 +23,10 @@ class TmpFile:
 
     async def __aexit__(self, exc_type, exc, tb):
         self.output.seek(0)
-        return self
+        # A truthy return from __aexit__ suppresses the in-flight exception:
+        # any error raised while writing rows was silently dropped and the
+        # response went out as HTTP 200 with only the header line.
+        return False
 
     def get(self):
         return self.output.getvalue()
@@ -36,6 +40,10 @@ class CSVWriter(AbstractWriter):
     async def get_response(self) -> web.StreamResponse:
         try:
             await self.get_buffer()
+            if is_dataframe(self.data):
+                # Defence in depth: the 'iter' output format already normalises
+                # DataFrames, but a writer can also be handed one directly.
+                self.data = df_to_records(self.data)
             if 'delimiter' in self.kwargs:
                 delimiter = self.kwargs['delimiter']
             else:

{querysource-4.5.13 → querysource-4.5.14}/querysource/outputs/writers/tsv.py

@@ -2,6 +2,7 @@ import csv
 from io import StringIO
 from aiocsv import AsyncDictWriter
 from aiohttp import web
+from ...utils.dataframes import df_to_records, is_dataframe
 from .abstract import AbstractWriter
 
 
@@ -18,7 +19,10 @@ class TmpFile:
 
     async def __aexit__(self, exc_type, exc, tb):
         self.output.seek(0)
-        return self
+        # A truthy return from __aexit__ suppresses the in-flight exception:
+        # any error raised while writing rows was silently dropped and the
+        # response went out as HTTP 200 with only the header line.
+        return False
 
     def get(self):
         return self.output.getvalue()
@@ -32,6 +36,10 @@ class TSVWriter(AbstractWriter):
     async def get_response(self) -> web.StreamResponse:
         try:
             await self.get_buffer()
+            if is_dataframe(self.data):
+                # Defence in depth: the 'iter' output format already normalises
+                # DataFrames, but a writer can also be handed one directly.
+                self.data = df_to_records(self.data)
             tmp = TmpFile()
             async with tmp.open_buffer() as afp:
                 writer = AsyncDictWriter(afp, self.columns, restval="NULL", quoting=csv.QUOTE_NONE, delimiter='\t', skipinitialspace=True)

querysource-4.5.14/querysource/utils/dataframes.py

@@ -0,0 +1,41 @@
+"""
+DataFrame helpers.
+
+Shared normalisation used by the ``iter`` output format and by the row-oriented
+writers, so a pandas-backed provider (``bigquery``, ``deltatbl``, ``iceberg``)
+and a plain ``list[dict]`` provider hand the same shape downstream.
+"""
+from typing import Any
+
+
+def is_dataframe(data: Any) -> bool:
+    """Duck-typed check for a pandas-like DataFrame.
+
+    Avoids importing pandas just to test the type, matching the check already
+    used by :meth:`querysource.outputs.writers.abstract.AbstractWriter.get_buffer`.
+
+    Args:
+        data: Object to inspect.
+
+    Returns:
+        bool: True when *data* exposes both ``to_dict`` and ``columns``.
+    """
+    return hasattr(data, 'to_dict') and hasattr(data, 'columns')
+
+
+def df_to_records(df: Any) -> list[dict]:
+    """Convert a DataFrame into a list of row dictionaries.
+
+    Iterating a DataFrame yields its *column names*, not its rows, so any
+    consumer expecting an iterable of mappings (``csv.DictWriter``,
+    ``aiocsv.AsyncDictWriter``, Jinja2 report templates) silently receives
+    garbage. Missing values (``NaN``/``NaT``/``pandas.NA``) become ``None`` so
+    they serialise as empty/null instead of the string ``"nan"``.
+
+    Args:
+        df: A pandas DataFrame (or DataFrame-like object).
+
+    Returns:
+        list[dict]: One dictionary per row, keyed by column name.
+    """
+    return df.astype(object).where(df.notna(), None).to_dict(orient='records')

{querysource-4.5.13 → querysource-4.5.14}/querysource/version.py

@@ -6,7 +6,7 @@ __description__ = (
     'QuerySource Query parser and generator.'
     'Supports SQL, NoSQL, MongoDB, Cassandra, Elasticsearch, etc.'
 )
-__version__ = '4.5.13'
+__version__ = '4.5.14'
 __url__ = 'https://github.com/phenobarbital/querysource'
 __copyright__ = 'Copyright (c) 2018-2025 Jesus Lara'
 __author__ = 'Jesus Lara'

{querysource-4.5.13 → querysource-4.5.14}/querysource.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: querysource
-Version: 4.5.13
+Version: 4.5.14
 Summary: Aiohttp web service for querying several databases easily
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: BSD-3-Clause

{querysource-4.5.13 → querysource-4.5.14}/querysource.egg-info/SOURCES.txt

@@ -66,6 +66,8 @@ tox.ini
 .github/workflows/codeql-analysis.yml
 .github/workflows/release.yml
 .jupyter/jupyter_notebook_config.py
+.parrot/wiki.json
+.parrot/wiki.local.json
 bin/README.md
 generated/AirtableSource.json
 generated/Concat.json
@@ -469,6 +471,7 @@ querysource/types/dt/filters.py
 querysource/types/dt/transforms.py
 querysource/utils/__init__.py
 querysource/utils/cache_serialization.py
+querysource/utils/dataframes.py
 querysource/utils/errors.py
 querysource/utils/events.py
 querysource/utils/fn.py
@@ -597,6 +600,7 @@ tests/handlers/test_queryexecutor_pbac_smoke.py
 tests/handlers/test_querymanager_pagination.py
 tests/handlers/test_queryservice_pbac_smoke.py
 tests/integration/__init__.py
+tests/integration/test_csv_stream_response.py
 tests/integration/test_datasource_endpoint.py
 tests/integration/test_multiquery_output_errors.py
 tests/integration/test_pbac_credentials.py
@@ -620,10 +624,12 @@ tests/scheduler/test_parse_trigger_shorthands.py
 tests/scheduler/test_shorthand_loaders.py
 tests/services/__init__.py
 tests/services/test_querysource_setup_pbac.py
+tests/unit/test_csv_writer_dataframe.py
 tests/unit/test_cython_filter_fallback.py
 tests/unit/test_datasource_redaction.py
 tests/unit/test_error_payload.py
 tests/unit/test_handler_output_status.py
+tests/unit/test_iter_format_dataframe.py
 tests/unit/test_multiqs_output_raise.py
 tests/unit/test_output_error.py
 tests/unit/test_provider_raw_query_validated.py

{querysource-4.5.13 → querysource-4.5.14}/querysource.egg-info/scm_file_list.json

@@ -47,6 +47,8 @@
     ".gitignore",
     ".isort.cfg",
     ".jupyter/jupyter_notebook_config.py",
+    ".parrot/wiki.json",
+    ".parrot/wiki.local.json",
     ".pre-commit-config.yaml",
     ".pylintrc",
     "CHANGES.rst",
@@ -463,6 +465,7 @@
     "querysource/types/validators.pyx",
     "querysource/utils/__init__.py",
     "querysource/utils/cache_serialization.py",
+    "querysource/utils/dataframes.py",
     "querysource/utils/errors.py",
     "querysource/utils/events.py",
     "querysource/utils/fn.py",
@@ -853,6 +856,7 @@
     "tests/handlers/test_querymanager_pagination.py",
     "tests/handlers/test_queryservice_pbac_smoke.py",
     "tests/integration/__init__.py",
+    "tests/integration/test_csv_stream_response.py",
     "tests/integration/test_datasource_endpoint.py",
     "tests/integration/test_multiquery_output_errors.py",
     "tests/integration/test_pbac_credentials.py",
@@ -947,10 +951,12 @@
     "tests/test_thread_source_base.py",
     "tests/test_threadquery_executor.py",
     "tests/test_zammad_search.py",
+    "tests/unit/test_csv_writer_dataframe.py",
     "tests/unit/test_cython_filter_fallback.py",
     "tests/unit/test_datasource_redaction.py",
     "tests/unit/test_error_payload.py",
     "tests/unit/test_handler_output_status.py",
+    "tests/unit/test_iter_format_dataframe.py",
     "tests/unit/test_multiqs_output_raise.py",
     "tests/unit/test_output_error.py",
     "tests/unit/test_provider_raw_query_validated.py",

querysource-4.5.14/querysource.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "4.5.14",
+  "distance": 0,
+  "node": "g5872d5ef4fdafc79f9636c954f28e1e2144a7a63",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-11"
+}

querysource-4.5.14/tests/integration/test_csv_stream_response.py

@@ -0,0 +1,36 @@
+"""Verify full CSV responses do not advertise partial byte ranges."""
+
+import pytest
+from aiohttp import web
+from aiohttp.test_utils import TestClient, TestServer
+
+from querysource.outputs.output import DataOutput
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("body_size", [421, 16384, 16385, 32769])
+async def test_full_csv_stream_has_no_content_range(body_size: int) -> None:
+    value = "x" * (body_size - len(b"name\r\n\r\n"))
+    expected = f"name\r\n{value}\r\n".encode("utf-8")
+
+    async def handler(request: web.Request) -> web.StreamResponse:
+        output = DataOutput(
+            request,
+            query=[{"name": value}],
+            ctype="csv",
+            slug="pokemon_warehouse_list_count",
+            writer_options={"delimiter": ",", "quoting": "minimal"},
+        )
+        return await output.response()
+
+    app = web.Application()
+    app.router.add_get("/csv", handler)
+    async with TestClient(TestServer(app)) as client:
+        async with client.get(
+            "/csv", headers={"Accept-Encoding": "gzip, deflate, br"}
+        ) as response:
+            assert response.status == 200
+            assert response.content_type == "text/csv"
+            assert response.content_length == body_size
+            assert await response.read() == expected
+            assert "Content-Range" not in response.headers

querysource-4.5.14/tests/unit/test_csv_writer_dataframe.py

@@ -0,0 +1,82 @@
+"""CSVWriter / TSVWriter must serialise a pandas DataFrame row by row and must
+not swallow writer exceptions.
+
+Regression: slugs served by pandas-backed providers (bigquery, deltatbl,
+iceberg) exported as ``:csv`` returned HTTP 200 with only the header line —
+``writerows(DataFrame)`` iterated column names, raised ``AttributeError`` and
+``TmpFile.__aexit__`` (returning ``self``) suppressed it.
+"""
+import pandas
+import pytest
+
+from querysource.outputs.writers.csv import CSVWriter, TmpFile
+from querysource.outputs.writers.tsv import TSVWriter
+
+
+class _Req:
+    headers = {}
+
+
+class _Resp:
+    def __init__(self):
+        self.headers = {}
+        self.content_length = None
+
+
+def _writer(cls, **kwargs):
+    writer = cls(
+        request=_Req(), resultset=[{"x": 1}], filename="slug",
+        response_type="stream", download=False, compression="gzip",
+        ctype=cls.ctype, **kwargs,
+    )
+
+    async def fake_response(response_type):
+        return _Resp()
+
+    async def fake_stream(response, data):
+        return data
+
+    writer.response = fake_response
+    writer.stream_response = fake_stream
+    return writer
+
+
+DF = pandas.DataFrame([{"a": 1, "b": "x"}, {"a": 2, "b": None}])
+
+
+async def test_csv_dataframe_writes_every_row():
+    writer = _writer(CSVWriter, delimiter=",", quoting="string")
+    writer.data = DF
+    out = await writer.get_response()
+    lines = out.decode().splitlines()
+    assert lines == ['"a","b"', '1,"x"', '2,""']
+
+
+async def test_tsv_dataframe_writes_every_row():
+    writer = _writer(TSVWriter)
+    writer.data = DF
+    out = await writer.get_response()
+    lines = out.decode().splitlines()
+    assert lines == ["a\tb", "1\tx", "2\t"]
+
+
+async def test_csv_list_of_dicts_unchanged():
+    writer = _writer(CSVWriter, delimiter=",", quoting="string")
+    writer.data = DF.to_dict(orient="records")
+    out = await writer.get_response()
+    assert out.decode().splitlines()[1] == '1,"x"'
+
+
+async def test_writer_errors_are_not_swallowed():
+    writer = _writer(CSVWriter, delimiter=",", quoting="string")
+    writer.data = ["not-a-mapping"]
+    writer.columns = ["a"]
+    with pytest.raises(AttributeError):
+        await writer.get_response()
+
+
+async def test_tmpfile_propagates_exceptions():
+    tmp = TmpFile()
+    with pytest.raises(RuntimeError):
+        async with tmp.open_buffer():
+            raise RuntimeError("boom")

querysource-4.5.14/tests/unit/test_iter_format_dataframe.py

@@ -0,0 +1,66 @@
+"""The ``iter`` output format must honour its contract: a list of dictionaries.
+
+Regression: pandas-backed providers (``bigquery``, ``deltatbl``, ``iceberg``)
+returned a ``pandas.DataFrame`` unchanged from ``iterFormat.serialize``. Every
+row-oriented writer declaring ``output_format = 'iter'`` (csv, tsv, txt,
+report/pdf) then iterated the frame, which yields *column names* instead of
+rows — ``slug:csv`` answered HTTP 200 with only the header line.
+"""
+import pandas
+import pytest
+
+from querysource.outputs.dt.iter import iterFormat
+from querysource.utils.dataframes import df_to_records, is_dataframe
+
+DF = pandas.DataFrame(
+    [
+        {"a": 1, "b": "x", "ts": pandas.Timestamp("2020-01-01")},
+        {"a": 2, "b": None, "ts": pandas.NaT},
+    ]
+)
+
+
+async def test_iter_serialises_dataframe_to_records():
+    data, error = await iterFormat().serialize(DF, None)
+    assert error is None
+    assert isinstance(data, list)
+    assert data == [
+        {"a": 1, "b": "x", "ts": pandas.Timestamp("2020-01-01")},
+        {"a": 2, "b": None, "ts": None},
+    ]
+
+
+async def test_iter_dataframe_rows_are_mappings():
+    # The actual failure mode: iterating a DataFrame yields column names.
+    data, _ = await iterFormat().serialize(DF, None)
+    assert [sorted(row.keys()) for row in data] == [["a", "b", "ts"]] * 2
+
+
+async def test_iter_list_of_dicts_unchanged():
+    rows = [{"a": 1}, {"a": 2}]
+    data, _ = await iterFormat().serialize(rows, None)
+    assert data == rows
+
+
+async def test_iter_propagates_error():
+    err = RuntimeError("boom")
+    data, error = await iterFormat().serialize([{"a": 1}], err)
+    assert error is err
+    assert data == [{"a": 1}]
+
+
+def test_df_to_records_maps_missing_values_to_none():
+    records = df_to_records(DF)
+    assert records[1]["b"] is None
+    assert records[1]["ts"] is None
+    # NaN in a float column must not leak as the string "nan"
+    floats = pandas.DataFrame([{"v": 1.5}, {"v": float("nan")}])
+    assert df_to_records(floats) == [{"v": 1.5}, {"v": None}]
+
+
+@pytest.mark.parametrize(
+    "value, expected",
+    [(DF, True), ([{"a": 1}], False), ({"a": 1}, False), ("text", False)],
+)
+def test_is_dataframe(value, expected):
+    assert is_dataframe(value) is expected

querysource-4.5.13/querysource.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "4.5.13",
-  "distance": 0,
-  "node": "g31553f85159327ea2f3835803f906af990841c22",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-08-25"
-}