queryapigate 0.5.0__py3-none-any.whl

queryapigate/engine.py

@@ -0,0 +1,166 @@
+"""Ties the pieces together: look up the connection, validate the SQL, run it, wrap the page."""
+import hashlib
+import logging
+import time
+
+from . import config, metrics, store
+from .errors import ApiError
+from .formats import ResultSetDTO
+from .pool import get_pool
+from .runners import RUNNERS, STREAM_RUNNERS
+from .sqltools import validate_sql
+
+log = logging.getLogger('queryapigate')
+
+
+def _sql_hash(sql):
+    """A correlation key for "did this same query run elsewhere/before" without re-reading the SQL text
+    itself - same idea (and same full-length sha256 hex) pool.py and cache.py already use for their own
+    keys. Logged *alongside* the full SQL text (see the log.info() calls below), not instead of it - the
+    hash is for a log aggregator to filter/group on; the full text is still there for a human reading logs
+    locally."""
+    return hashlib.sha256(sql.encode('utf-8')).hexdigest()
+
+
+def execute_sql(sql, connection_name, limit, offset, params=None, timeout=None, allow_writes=True, key_name='-',
+                allowed_write_ops=None):
+    """Run ``sql`` on a named connection and return the requested page as a ResultSetDTO.
+
+    ``allow_writes`` is the caller's own permission (e.g. a scoped API key); the connection is only ever
+    writable when both that *and* the server-wide QUERYAPIGATE_ALLOW_WRITES allow it - a caller can narrow the
+    server's setting, never widen it. ``key_name`` is only for the query counter in ``/metrics`` (audit: which
+    key touched which connection); it plays no part in what the query is allowed to do. ``allowed_write_ops``
+    is a key's own narrower allow-list of write keywords, if it has one - see ``sqltools.validate_sql()``.
+    """
+    details = store.get_connection(connection_name)
+    effective_allow_writes = config.allow_writes() and allow_writes
+    sql = validate_sql(sql, dialect=details['db'], allow_writes=effective_allow_writes,
+                       allowed_write_ops=allowed_write_ops)
+    log.info('Executing on %s (%s), limit=%s offset=%s timeout=%s: %s',
+             connection_name, details['db'], limit, offset, timeout, sql,
+             extra={'connection': connection_name, 'dialect': details['db'], 'limit': limit, 'offset': offset,
+                   'timeout': timeout, 'sql_hash': _sql_hash(sql)})
+    started = time.monotonic()
+    status = 'error'
+    metrics.inc_active_query()
+    try:
+        columns, rows = RUNNERS[details['db']](details, sql, params, limit, offset, not effective_allow_writes,
+                                             timeout, get_pool())
+        status = 'success'
+    except ApiError:
+        raise
+    except ImportError as error:
+        log.exception('Missing database driver')
+        raise ApiError(f"The driver for '{details['db']}' is not installed", 500, detail=str(error)) from error
+    except Exception as error:
+        log.exception('Query on %s failed', connection_name)
+        raise ApiError('An error occurred while executing the SQL query', 500, detail=str(error)) from error
+    finally:
+        metrics.dec_active_query()
+        elapsed = time.monotonic() - started
+        metrics.observe_query(connection_name, details['db'], status, elapsed, key_name)
+        threshold = config.slow_query_threshold()
+        if threshold and elapsed >= threshold:
+            log.warning('Slow query on %s (%s): %.1fms - %s', connection_name, details['db'], elapsed * 1000, sql,
+                       extra={'connection': connection_name, 'dialect': details['db'],
+                              'duration_ms': round(elapsed * 1000, 1)})
+    has_more = len(rows) > limit
+    result_rows = rows[:limit]
+    metrics.observe_rows(connection_name, details['db'], key_name, len(result_rows))
+    return ResultSetDTO(result_rows, columns, has_more=has_more)
+
+
+def stream_sql(sql, connection_name, params=None, timeout=None, key_name='-'):
+    """Like execute_sql, but for the whole result rather than one page - and, unlike execute_sql, always
+    read-only regardless of QUERYAPIGATE_ALLOW_WRITES or the caller's own permission. A large export has no
+    business mutating data, and forcing this sidesteps a lot of incidental complexity around commit timing
+    on a connection that may stay checked out for a long time - see runners.py's "Streaming" section for
+    what that already involves per dialect without adding writes into the mix too.
+
+    Returns (columns, rows): ``columns`` is available immediately (the underlying generator is primed once
+    to get it, surfacing a connection or SQL error here just like execute_sql does), ``rows`` is a lazy
+    generator - the connection checked out (or freshly opened) for it is released only once that generator
+    is exhausted, errors, or a client disconnect closes it early (see runners._make_stream_runner).
+    """
+    details = store.get_connection(connection_name)
+    sql = validate_sql(sql, dialect=details['db'], allow_writes=False)
+    log.info('Streaming from %s (%s): %s', connection_name, details['db'], sql,
+             extra={'connection': connection_name, 'dialect': details['db'], 'sql_hash': _sql_hash(sql)})
+    status = 'error'
+    metrics.inc_active_query()
+    try:
+        stream = STREAM_RUNNERS[details['db']](details, sql, params, timeout, get_pool())
+        columns = next(stream)
+        status = 'success'
+    except ApiError:
+        raise
+    except ImportError as error:
+        log.exception('Missing database driver')
+        raise ApiError(f"The driver for '{details['db']}' is not installed", 500, detail=str(error)) from error
+    except Exception as error:
+        log.exception('Streaming query on %s failed', connection_name)
+        raise ApiError('An error occurred while executing the SQL query', 500, detail=str(error)) from error
+    finally:
+        # On success, the query stays active until _drain() below finishes consuming it - the decrement
+        # (and the row/stream-status metrics) move there with it, not here.
+        if status == 'error':
+            metrics.observe_stream(connection_name, details['db'], status, key_name)
+            metrics.dec_active_query()
+    return columns, _drain(stream, connection_name, details['db'], key_name)
+
+
+def _drain(rows, connection_name, dialect, key_name):
+    """Wraps the row generator stream_sql() returns: records the export's final status once it is fully
+    consumed (success) or fails partway through (error - client disconnects are not failures, so
+    GeneratorExit is excluded), and makes sure a mid-stream failure lands in the log. Once the response has
+    started, a failure here can no longer change its status or body shape - the client just sees the
+    connection end early - so logging clearly is the most this can do about it. Also where the query
+    started by stream_sql() actually stops being "active" (see inc_active_query() there), and where rows
+    that made it out are counted - even a partial count on failure or disconnect is data that left the
+    server, not nothing.
+
+    Also enforces QUERYAPIGATE_STREAM_MAX_ROWS (config.stream_max_rows()), if set: unlike a paged response,
+    ?stream=true has no ceiling of its own otherwise - the whole point is not buffering the result, so
+    nothing else naturally bounds how much a single export can return. Breaking out of the ``for`` loop
+    below (rather than letting it run to exhaustion) still reaches the ``else`` clause normally - a ``break``
+    only skips a ``for``'s own ``else``, not the surrounding ``try``'s - so the usual success bookkeeping
+    still runs; ``rows.close()`` is what actually releases the still-open connection early, the same
+    GeneratorExit-based cleanup a client disconnecting mid-stream already triggers (see
+    runners._make_stream_runner), just initiated from here instead of by the client closing the response.
+    """
+    limit = config.stream_max_rows()
+    truncated = False
+    count = 0
+    try:
+        for row in rows:
+            count += 1
+            yield row
+            if limit is not None and count >= limit:
+                truncated = True
+                break
+    except GeneratorExit:
+        metrics.observe_rows(connection_name, dialect, key_name, count)
+        metrics.dec_active_query()
+        raise
+    except Exception:
+        log.exception('Streaming from %s failed partway through', connection_name)
+        metrics.observe_stream(connection_name, dialect, 'error', key_name)
+        metrics.observe_rows(connection_name, dialect, key_name, count)
+        metrics.dec_active_query()
+        raise
+    else:
+        if truncated:
+            rows.close()
+            log.warning('Streaming from %s truncated at %s rows (QUERYAPIGATE_STREAM_MAX_ROWS)',
+                       connection_name, limit,
+                       extra={'connection': connection_name, 'dialect': dialect, 'row_limit': limit})
+        metrics.observe_stream(connection_name, dialect, 'truncated' if truncated else 'success', key_name)
+        metrics.observe_rows(connection_name, dialect, key_name, count)
+        metrics.dec_active_query()
+
+
+def timed(func, *args, **kwargs):
+    """Call ``func`` and return (result, elapsed milliseconds)."""
+    started = time.monotonic()
+    result = func(*args, **kwargs)
+    return result, round((time.monotonic() - started) * 1000)

queryapigate/errors.py

@@ -0,0 +1,8 @@
+class ApiError(Exception):
+    """An error that maps directly onto an HTTP response."""
+
+    def __init__(self, message, status=400, **extra):
+        super().__init__(message)
+        self.message = message
+        self.status = status
+        self.extra = extra

queryapigate/formats.py

@@ -0,0 +1,193 @@
+"""Result sets and their output formats (JSON, NDJSON, CSV, TSV, XML, YAML, XLSX)."""
+import csv
+import json
+import re
+import xml.etree.ElementTree as ET
+from datetime import date, datetime, time
+from decimal import Decimal
+from io import BytesIO, StringIO
+
+import yaml
+from flask import Response, jsonify
+from openpyxl import Workbook
+from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
+
+
+def json_default(obj):
+    """``default=`` hook for json.dumps covering the types database drivers return."""
+    if isinstance(obj, datetime):
+        return obj.strftime('%Y-%m-%d %H:%M:%S')
+    if isinstance(obj, (date, time)):
+        return obj.isoformat()
+    if isinstance(obj, Decimal):
+        return float(obj)
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return bytes(obj).decode('utf-8', 'replace')
+    return str(obj)
+
+
+def _unique(names):
+    """Disambiguate repeated column names (e.g. ``SELECT a.id, b.id``) so no data is lost."""
+    seen, result = {}, []
+    for name in names:
+        name = str(name)
+        seen[name] = seen.get(name, 0) + 1
+        result.append(name if seen[name] == 1 else f'{name}_{seen[name]}')
+    return result
+
+
+def _cell(value):
+    """Coerce a value into plain built-in types that CSV, YAML and Excel writers all accept.
+
+    Some drivers (e.g. H2 through JPype) return subclasses of int/float/str, which PyYAML's safe
+    dumper rejects because it only recognises the exact built-in types.
+    """
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return bool(value)
+    if isinstance(value, int):
+        return int(value)
+    if isinstance(value, float):
+        return float(value)
+    if isinstance(value, str):
+        return str(value)
+    if isinstance(value, Decimal):
+        return float(value)
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        return bytes(value).decode('utf-8', 'replace')
+    if isinstance(value, datetime):
+        return value.replace(tzinfo=None) if value.tzinfo else value
+    if isinstance(value, (date, time)):
+        return value
+    return str(value)
+
+
+def _xml_name(name):
+    name = re.sub(r'[^\w.\-]', '_', name)
+    return name if re.match(r'[A-Za-z_]', name) else f'_{name}'
+
+
+class ResultSetDTO:
+    """A page of query results (column names + rows) with output-format converters."""
+
+    def __init__(self, rows, columns, has_more=False):
+        self.columns = _unique(columns)
+        self.rows = [[_cell(v) for v in row] for row in rows]
+        self.has_more = has_more
+
+    def __bool__(self):
+        return bool(self.rows)
+
+    def as_dicts(self):
+        return [dict(zip(self.columns, row)) for row in self.rows]
+
+    def _to_delimited(self, delimiter, mimetype):
+        data_io = StringIO()
+        writer = csv.writer(data_io, delimiter=delimiter)
+        writer.writerow(self.columns)
+        writer.writerows(self.rows)
+        return Response(data_io.getvalue(), mimetype=mimetype)
+
+    def to_csv(self):
+        return self._to_delimited(',', 'text/csv')
+
+    def to_tsv(self):
+        return self._to_delimited('\t', 'text/tab-separated-values')
+
+    def to_json(self):
+        return jsonify(self.as_dicts())
+
+    def to_ndjson(self):
+        lines = (json.dumps(row, default=json_default) for row in self.as_dicts())
+        return Response('\n'.join(lines) + '\n', mimetype='application/x-ndjson')
+
+    def to_xml(self):
+        root = ET.Element('data')
+        tags = [_xml_name(c) for c in self.columns]
+        for row in self.rows:
+            item = ET.SubElement(root, 'item')
+            for tag, value in zip(tags, row):
+                ET.SubElement(item, tag).text = '' if value is None else str(value)
+        return Response(ET.tostring(root, encoding='unicode', method='xml'), mimetype='application/xml')
+
+    def to_yaml(self):
+        return Response(yaml.safe_dump(self.as_dicts(), default_flow_style=False, sort_keys=False,
+                                       allow_unicode=True),
+                        mimetype='application/x-yaml')
+
+    def to_xlsx(self):
+        wb = Workbook()
+        ws = wb.active
+        ws.append(self.columns)
+        for row in self.rows:
+            ws.append([ILLEGAL_CHARACTERS_RE.sub('', v) if isinstance(v, str) else v for v in row])
+        excel_data = BytesIO()
+        wb.save(excel_data)
+        return Response(
+            excel_data.getvalue(),
+            mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            headers={'Content-Disposition': 'attachment;filename=result.xlsx'},
+        )
+
+
+FORMATTERS = {
+    'json': ResultSetDTO.to_json,
+    'ndjson': ResultSetDTO.to_ndjson,
+    'csv': ResultSetDTO.to_csv,
+    'tsv': ResultSetDTO.to_tsv,
+    'xml': ResultSetDTO.to_xml,
+    'yaml': ResultSetDTO.to_yaml,
+    'xlsx': ResultSetDTO.to_xlsx,
+}
+
+# --------------------------------------------------------------------------------------
+# Streaming: unlike ResultSetDTO above, these never hold more than one row (or one small
+# write buffer) in memory at a time - `rows` is a generator pulling batches from a live
+# database cursor, not a pre-materialised list. Only CSV and NDJSON are genuinely streamable
+# text formats; XML/YAML/XLSX all need the whole document structure in memory to write
+# correctly (a closing root tag, a single top-level list, a zip's central directory), so
+# they stay page-at-a-time via ResultSetDTO/FORMATTERS above.
+
+STREAM_FORMATTERS = frozenset({'csv', 'tsv', 'ndjson'})
+STREAM_MIMETYPES = {'csv': 'text/csv', 'tsv': 'text/tab-separated-values', 'ndjson': 'application/x-ndjson'}
+STREAM_DELIMITERS = {'csv': ',', 'tsv': '\t'}
+
+
+def _stream_delimited(columns, rows, delimiter):
+    buf = StringIO()
+    writer = csv.writer(buf, delimiter=delimiter)
+    writer.writerow(columns)
+    yield buf.getvalue()
+    for row in rows:
+        buf.seek(0)
+        buf.truncate(0)
+        writer.writerow([_cell(v) for v in row])
+        yield buf.getvalue()
+
+
+def _stream_ndjson(columns, rows):
+    for row in rows:
+        yield json.dumps(dict(zip(columns, (_cell(v) for v in row))), default=json_default) + '\n'
+
+
+def iter_stream_chunks(output_format, columns, rows):
+    """The text chunks for ``output_format`` in ``STREAM_FORMATTERS``, generated from ``rows`` as they
+    arrive rather than built up front - shared by stream_response() (an HTTP response body) and the
+    ``queryapigate export`` CLI command (written straight to a file), so there is exactly one place that knows
+    how to turn a row stream into CSV/TSV/NDJSON text."""
+    columns = _unique(columns)
+    if output_format == 'ndjson':
+        return _stream_ndjson(columns, rows)
+    return _stream_delimited(columns, rows, STREAM_DELIMITERS[output_format])
+
+
+def stream_response(output_format, columns, rows, filename):
+    """A chunked ``Response`` for ``output_format`` in ``STREAM_FORMATTERS``, generated from ``rows`` as
+    they arrive rather than built up front - the point being that a result far larger than fits in memory
+    can still be exported, at the cost of the usual conveniences a fully-buffered response gets for free
+    (a reliable Content-Length, the ability to retry a failed write, an ETag for caching)."""
+    body = iter_stream_chunks(output_format, columns, rows)
+    response = Response(body, mimetype=STREAM_MIMETYPES[output_format])
+    response.headers['Content-Disposition'] = f'attachment; filename="{filename}.{output_format}"'
+    return response

queryapigate/logging_setup.py

@@ -0,0 +1,75 @@
+"""One log line per event from the ``queryapigate`` logger, tagged with the request ID and the calling API key's
+name that caused it (both ``-`` outside a request, e.g. at startup; key is also ``-`` when no key is
+configured at all, and ``admin`` for ``QUERYAPIGATE_API_KEY`` - see ``apikeys.Permission``). Plain text by
+default; ``QUERYAPIGATE_JSON_LOGS=1`` switches to one JSON object per line for log aggregators.
+
+In JSON mode, any caller-supplied ``extra={...}`` on a ``log.info()``/``log.warning()`` call becomes its
+own top-level JSON key, not just text folded into ``message`` - see ``_JsonFormatter``. ``engine.py`` and
+``app.py`` use this for fields a log aggregator would otherwise have to parse out of the message string
+(``connection``, ``dialect``, ``status``, ``duration_ms``, ...), so a query or request can be filtered on
+those directly rather than only by ``request_id``/``key``, which were already real fields before this.
+
+``configure()`` is called from ``create_app()`` - the one place every entry point (``queryapigate serve``,
+gunicorn/WSGI, the test suite) goes through - so logging is set up the same way regardless of how the app
+is run, instead of relying on the CLI's ``logging.basicConfig()``, which never runs under a WSGI server.
+"""
+import json
+import logging
+
+from flask import g
+
+from . import config
+
+_PLAIN_FORMAT = '%(asctime)s %(levelname)s %(name)s [%(request_id)s key=%(key)s]: %(message)s'
+# Every attribute a plain LogRecord carries with no `extra` at all - used to spot which attributes on a
+# given record were added via `extra={...}` (see _JsonFormatter below), without hardcoding their names.
+_BASE_RECORD_ATTRS = frozenset(vars(logging.makeLogRecord({}))) | {'message', 'asctime'}
+
+
+class _RequestContextFilter(logging.Filter):
+    def filter(self, record):
+        try:
+            record.request_id = g.request_id
+        except RuntimeError:  # outside a Flask request context (startup, CLI, background threads)
+            record.request_id = '-'
+            record.key = '-'
+            return True
+        permission = g.get('permission')  # unset before the key is resolved, e.g. a CORS preflight
+        record.key = (permission.name if permission else None) or '-'
+        return True
+
+
+class _JsonFormatter(logging.Formatter):
+    def format(self, record):
+        payload = {
+            'time': self.formatTime(record, '%Y-%m-%dT%H:%M:%S'),
+            'level': record.levelname,
+            'logger': record.name,
+            'request_id': getattr(record, 'request_id', '-'),
+            'key': getattr(record, 'key', '-'),
+            'message': record.getMessage(),
+        }
+        # request_id/key are already set above (from _RequestContextFilter, not a caller's `extra`); every
+        # other attribute a record doesn't get by default came from an explicit `extra={...}` at the call
+        # site, and becomes its own field here rather than staying folded into `message` above.
+        payload.update({k: v for k, v in vars(record).items()
+                        if k not in _BASE_RECORD_ATTRS and k not in ('request_id', 'key')})
+        if record.exc_info:
+            payload['exception'] = self.formatException(record.exc_info)
+        return json.dumps(payload)
+
+
+def configure(logger):
+    """Attach one handler to ``logger``, replacing any handler set by an earlier call of this function (safe to
+    call repeatedly - once per ``create_app()``, including in tests) but leaving any other handler alone, e.g.
+    one ``unittest.TestCase.assertLogs`` installs temporarily while a test is running."""
+    for existing in list(logger.handlers):
+        if getattr(existing, '_queryapigate_managed', False):
+            logger.removeHandler(existing)
+    handler = logging.StreamHandler()
+    handler._queryapigate_managed = True
+    handler.addFilter(_RequestContextFilter())
+    handler.setFormatter(_JsonFormatter() if config.json_logs() else logging.Formatter(_PLAIN_FORMAT))
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+    logger.propagate = False

queryapigate/metrics.py

@@ -0,0 +1,215 @@
+"""In-process metrics, rendered as Prometheus text exposition format at ``/metrics``.
+
+Counters and histograms live in this process's memory - correct for the image this project ships (gunicorn
+with a single worker; see the Dockerfile, which explains why in a comment next to ``--workers 1``) but not
+for a multi-process deployment, which would need a shared backing store instead (e.g. ``prometheus_client``
+in its multiprocess mode) - nothing here does that.
+"""
+import bisect
+import threading
+
+from . import pool
+
+_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0)
+
+_lock = threading.Lock()
+_request_counts: dict[tuple, int] = {}  # (method, endpoint, status, key) -> int
+_request_hist: dict[tuple, list] = {}   # (method, endpoint) -> [count per bucket in _BUCKETS..., +Inf count]
+_request_sum: dict[tuple, float] = {}   # (method, endpoint) -> total seconds
+_query_counts: dict[tuple, int] = {}    # (connection, dialect, status, key) -> int
+_query_hist: dict[tuple, list] = {}     # (connection, dialect) -> [count per bucket...]
+_query_sum: dict[tuple, float] = {}     # (connection, dialect) -> total seconds
+_stream_counts: dict[tuple, int] = {}   # (connection, dialect, status, key) -> int
+_row_counts: dict[tuple, int] = {}      # (connection, dialect, key) -> total rows returned
+_serialization_hist: dict[tuple, list] = {}  # (format,) -> [count per bucket...]
+_serialization_sum: dict[tuple, float] = {}  # (format,) -> total seconds
+_active_queries = 0                     # queries currently executing (paged or mid-stream)
+_rate_limit_rejections = 0
+
+
+def _bucket_index(elapsed):
+    return bisect.bisect_left(_BUCKETS, elapsed)
+
+
+def observe_request(method, endpoint, status, elapsed, key='-'):
+    """`key` is the calling API key's name ('admin' for QUERYAPIGATE_API_KEY, '-' when none is configured), kept
+    on the request/query counters (audit: who did what) but not the latency histograms, so the number of
+    distinct keys never multiplies the size of the bucketed output."""
+    with _lock:
+        count_key = (method, endpoint, status, key)
+        _request_counts[count_key] = _request_counts.get(count_key, 0) + 1
+        hkey = (method, endpoint)
+        counts = _request_hist.setdefault(hkey, [0] * (len(_BUCKETS) + 1))
+        counts[_bucket_index(elapsed)] += 1
+        _request_sum[hkey] = _request_sum.get(hkey, 0.0) + elapsed
+
+
+def observe_query(connection_name, dialect, status, elapsed, key='-'):
+    with _lock:
+        count_key = (connection_name, dialect, status, key)
+        _query_counts[count_key] = _query_counts.get(count_key, 0) + 1
+        hkey = (connection_name, dialect)
+        counts = _query_hist.setdefault(hkey, [0] * (len(_BUCKETS) + 1))
+        counts[_bucket_index(elapsed)] += 1
+        _query_sum[hkey] = _query_sum.get(hkey, 0.0) + elapsed
+
+
+def observe_stream(connection_name, dialect, status, key='-'):
+    """A separate counter from observe_query(): a streaming export can run for minutes, so mixing its
+    duration into the regular per-query latency histogram would make that histogram's percentiles
+    meaningless for the fast, page-at-a-time queries it actually describes. There is deliberately no
+    equivalent streaming latency histogram here for the same reason - just how many started and how they
+    ended."""
+    with _lock:
+        count_key = (connection_name, dialect, status, key)
+        _stream_counts[count_key] = _stream_counts.get(count_key, 0) + 1
+
+
+def observe_rows(connection_name, dialect, key, count):
+    """Rows actually returned to the caller - the trimmed page for a paged query, or however many made it
+    out before a streaming export finished or failed partway through (see engine._drain()); not split by
+    status, since a partial streamed count is still data that left the server, not nothing."""
+    with _lock:
+        row_key = (connection_name, dialect, key)
+        _row_counts[row_key] = _row_counts.get(row_key, 0) + count
+
+
+def observe_serialization(output_format, elapsed):
+    """How long formats.render() took to build the response body - JSON/CSV/TSV/XML/YAML/XLSX encoding,
+    after the query itself has already finished. Only ever called for a paged (non-streaming) response:
+    streaming formats it row by row, interleaved with network I/O, so there's no single "serialization
+    happened here" span to measure the way there is for a page built in memory up front."""
+    with _lock:
+        hkey = (output_format,)
+        counts = _serialization_hist.setdefault(hkey, [0] * (len(_BUCKETS) + 1))
+        counts[_bucket_index(elapsed)] += 1
+        _serialization_sum[hkey] = _serialization_sum.get(hkey, 0.0) + elapsed
+
+
+def inc_active_query():
+    global _active_queries
+    with _lock:
+        _active_queries += 1
+
+
+def dec_active_query():
+    """Pairs with inc_active_query(). For a paged query the whole span is one function call
+    (engine.execute_sql); for a streaming export the connection stays checked out for as long as the client
+    keeps reading, so the increment happens once in engine.stream_sql() but the decrement is deferred to
+    engine._drain() - whichever of the two actually turns out to be where the query stops being active."""
+    global _active_queries
+    with _lock:
+        _active_queries -= 1
+
+
+def inc_rate_limit_rejection():
+    global _rate_limit_rejections
+    with _lock:
+        _rate_limit_rejections += 1
+
+
+def summary_for_key(name):
+    """Live usage for one API key's name, aggregated from the same counters /metrics renders - queries run,
+    of those how many failed, and rows returned. No latency figure: the request/query duration histograms
+    are deliberately not split by key (see observe_request()'s docstring), so there is nothing to average
+    per key without changing that trade-off."""
+    with _lock:
+        queries = sum(count for (_, _, _, key), count in _query_counts.items() if key == name)
+        errors = sum(count for (_, _, status, key), count in _query_counts.items()
+                     if key == name and status == 'error')
+        rows = sum(count for (_, _, key), count in _row_counts.items() if key == name)
+    return {'queries': queries, 'errors': errors, 'rows': rows}
+
+
+def summary_for_connection(name):
+    """Live usage for one connection name, aggregated the same way as summary_for_key() - plus an average
+    query latency, which *is* available here: the duration histograms are keyed by (connection, dialect),
+    and a connection has exactly one dialect, so this is an unambiguous per-connection average."""
+    with _lock:
+        queries = sum(count for (conn, _, _, _), count in _query_counts.items() if conn == name)
+        errors = sum(count for (conn, _, status, _), count in _query_counts.items()
+                     if conn == name and status == 'error')
+        rows = sum(count for (conn, _, _), count in _row_counts.items() if conn == name)
+        elapsed = sum(total for (conn, _), total in _query_sum.items() if conn == name)
+        sampled = sum(sum(counts) for (conn, _), counts in _query_hist.items() if conn == name)
+    avg_duration_ms = round(elapsed / sampled * 1000, 1) if sampled else None
+    return {'queries': queries, 'errors': errors, 'rows': rows, 'avg_duration_ms': avg_duration_ms}
+
+
+def _escape(value):
+    return str(value).replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')
+
+
+def _labels(pairs):
+    return '{' + ','.join(f'{name}="{_escape(value)}"' for name, value in pairs) + '}'
+
+
+def _render_counter(lines, name, help_text, label_names, counts):
+    lines.append(f'# HELP {name} {help_text}')
+    lines.append(f'# TYPE {name} counter')
+    for key in sorted(counts):
+        lines.append(f'{name}{_labels(zip(label_names, key))} {counts[key]}')
+
+
+def _render_histogram(lines, name, help_text, label_names, hist, totals):
+    lines.append(f'# HELP {name} {help_text}')
+    lines.append(f'# TYPE {name} histogram')
+    for key in sorted(hist):
+        labels = list(zip(label_names, key))
+        cumulative = 0
+        for edge, count in zip(_BUCKETS, hist[key]):
+            cumulative += count
+            lines.append(f'{name}_bucket{_labels([*labels, ("le", edge)])} {cumulative}')
+        cumulative += hist[key][-1]  # the +Inf bucket
+        lines.append(f'{name}_bucket{_labels([*labels, ("le", "+Inf")])} {cumulative}')
+        lines.append(f'{name}_sum{_labels(labels)} {totals.get(key, 0.0)}')
+        lines.append(f'{name}_count{_labels(labels)} {cumulative}')
+
+
+def render():
+    """The current metrics as Prometheus text exposition format."""
+    with _lock:
+        request_counts = dict(_request_counts)
+        request_hist = {k: list(v) for k, v in _request_hist.items()}
+        request_sum = dict(_request_sum)
+        query_counts = dict(_query_counts)
+        query_hist = {k: list(v) for k, v in _query_hist.items()}
+        query_sum = dict(_query_sum)
+        stream_counts = dict(_stream_counts)
+        row_counts = dict(_row_counts)
+        serialization_hist = {k: list(v) for k, v in _serialization_hist.items()}
+        serialization_sum = dict(_serialization_sum)
+        active_queries = _active_queries
+        rejections = _rate_limit_rejections
+
+    lines = []
+    _render_counter(lines, 'queryapigate_requests_total', 'Total HTTP requests.',
+                    ('method', 'endpoint', 'status', 'key'), request_counts)
+    _render_histogram(lines, 'queryapigate_request_duration_seconds', 'HTTP request latency in seconds.',
+                      ('method', 'endpoint'), request_hist, request_sum)
+    _render_counter(lines, 'queryapigate_queries_total', 'Total SQL queries executed.',
+                    ('connection', 'dialect', 'status', 'key'), query_counts)
+    _render_histogram(lines, 'queryapigate_query_duration_seconds', 'SQL query latency in seconds.',
+                      ('connection', 'dialect'), query_hist, query_sum)
+    _render_counter(lines, 'queryapigate_stream_exports_total', 'Total streaming (stream=true) exports started.',
+                    ('connection', 'dialect', 'status', 'key'), stream_counts)
+    _render_counter(lines, 'queryapigate_rows_returned_total', 'Total rows returned by SQL queries.',
+                    ('connection', 'dialect', 'key'), row_counts)
+    _render_histogram(lines, 'queryapigate_serialization_duration_seconds',
+                      'Response body serialization latency in seconds, by output format (paged responses only).',
+                      ('format',), serialization_hist, serialization_sum)
+
+    lines.append('# HELP queryapigate_active_queries SQL queries currently executing (paged or mid-stream).')
+    lines.append('# TYPE queryapigate_active_queries gauge')
+    lines.append(f'queryapigate_active_queries {active_queries}')
+
+    lines.append('# HELP queryapigate_pool_idle_connections Idle pooled database connections currently held.')
+    lines.append('# TYPE queryapigate_pool_idle_connections gauge')
+    shared = pool.get_pool()
+    lines.append(f'queryapigate_pool_idle_connections {shared.idle_count() if shared else 0}')
+
+    lines.append('# HELP queryapigate_rate_limit_rejections_total Requests rejected by the rate limiter.')
+    lines.append('# TYPE queryapigate_rate_limit_rejections_total counter')
+    lines.append(f'queryapigate_rate_limit_rejections_total {rejections}')
+
+    return '\n'.join(lines) + '\n'