queryapigate 0.5.0__py3-none-any.whl

queryapigate/app.py

@@ -0,0 +1,844 @@
+"""The Flask application: HTTP routes on top of the store, engine and formatters."""
+import logging
+import math
+import time
+import uuid
+
+from flask import Blueprint, Flask, Response, current_app, g, jsonify, redirect, request, url_for
+from flask.json.provider import DefaultJSONProvider
+from werkzeug.exceptions import HTTPException
+from werkzeug.middleware.proxy_fix import ProxyFix
+
+from . import apikeys, cache, config, cors, engine, logging_setup, metrics, openapi, pool, schema, sqltools, store, ui
+from . import params as param_rules
+from .errors import ApiError
+from .formats import FORMATTERS, STREAM_FORMATTERS, json_default, stream_response
+from .ratelimit import KeyRateLimiters, RateLimiter
+
+log = logging.getLogger('queryapigate')
+bp = Blueprint('api', __name__)
+
+# Query-string arguments that control a request rather than supplying query parameters.
+RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version', 'timeout', 'stream'}
+PUBLIC_ENDPOINTS = {'api.index', 'api.favicon', 'api.health', 'api.docs', 'api.openapi_spec', 'api.admin_ui',
+                    'api.metrics_endpoint'}
+RATE_LIMIT_EXEMPT = {'api.health', 'api.metrics_endpoint'}  # monitoring keeps working while a client is throttled
+ACCESS_LOG_QUIET = {'api.health', 'api.metrics_endpoint'}  # polled too often to log every hit
+
+
+class JSONProvider(DefaultJSONProvider):
+    default = staticmethod(json_default)
+    sort_keys = False  # keep result columns in the order the query returned them
+
+
+def create_app():
+    from . import __version__
+    config.check_settings()
+    logging_setup.configure(log)
+    app = Flask(__name__)
+    hops = config.proxy_hops()
+    if hops:  # behind reverse proxies: take the client address and scheme from their X-Forwarded-* headers
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_for=hops, x_proto=hops, x_host=hops)
+    app.extensions['queryapigate_limiter'] = RateLimiter()
+    app.extensions['queryapigate_key_limiter'] = KeyRateLimiters()
+    app.extensions['queryapigate_cache'] = cache.ResponseCache()
+    if config.cors_origins() == '*' and not config.api_key():
+        log.warning('QUERYAPIGATE_CORS_ORIGINS=* without QUERYAPIGATE_API_KEY: any website a user visits can call '
+                    'this API from their browser and reach every active connection. Set an API key or list the '
+                    'origins.')
+    if apikeys.any_configured() and not config.api_key():
+        log.warning('Scoped API keys exist but QUERYAPIGATE_API_KEY is not set: no key can manage connections, saved '
+                    'queries or other API keys until it is - only a scoped key\'s own allowed connections work.')
+    if config.secret_key():
+        # Encrypt any literal password already on disk immediately, rather than waiting for its next
+        # PATCH /connections - a connection saved before QUERYAPIGATE_SECRET_KEY existed benefits right away.
+        store.encrypt_plaintext_passwords_in_place()
+    else:
+        encrypted = store.encrypted_password_connections()
+        if encrypted:
+            log.warning('Connection(s) %s have an encrypted password but QUERYAPIGATE_SECRET_KEY is not set - '
+                        'they cannot be used until the key that encrypted them is restored.',
+                        ', '.join(encrypted))
+    plaintext = store.plaintext_password_connections()
+    if plaintext:
+        log.warning("Connection(s) %s store a literal password in %s. Consider a \"${VAR}\" reference to an "
+                    "environment variable instead - it reads the same way but keeps the secret out of the file, "
+                    "or set QUERYAPIGATE_SECRET_KEY to encrypt it at rest automatically.",
+                    ', '.join(plaintext), config.connections_file())
+    app.json = JSONProvider(app)
+    app.config['QUERYAPIGATE_VERSION'] = __version__
+
+    @app.errorhandler(ApiError)
+    def handle_api_error(error):
+        return jsonify({'error': error.message, **error.extra}), error.status
+
+    @app.errorhandler(HTTPException)
+    def handle_http_error(error):
+        return jsonify({'error': error.description}), error.code
+
+    @app.errorhandler(Exception)
+    def handle_unexpected_error(error):
+        log.exception('Unhandled error')
+        return jsonify({'error': 'An error occurred'}), 500
+
+    @app.before_request
+    def gate():
+        # Set first so every request - including one rejected below - gets an ID and a measured duration.
+        g.request_id = uuid.uuid4().hex[:12]
+        g.request_started = time.monotonic()
+        # Order matters: a browser's preflight cannot carry the API key, and rate limiting comes before the key
+        # check so that guessing keys is throttled too.
+        if cors.is_preflight(request):
+            return cors.preflight_response(request.headers.get('Origin'))
+        limited = check_rate_limit()
+        if limited is not None:
+            return limited
+        g.permission = resolve_permission()
+        if request.endpoint not in PUBLIC_ENDPOINTS and g.permission is None:
+            return jsonify({'error': 'Unauthorized'}), 401
+        limited = check_key_rate_limit()
+        if limited is not None:
+            return limited
+
+    @app.after_request
+    def decorate(response):
+        cors.add_headers(response, request.headers.get('Origin'))
+        if g.get('rate_limit'):
+            limit, remaining = g.rate_limit
+            response.headers['X-RateLimit-Limit'] = str(limit)
+            response.headers['X-RateLimit-Remaining'] = str(remaining)
+        if g.get('key_rate_limit'):
+            limit, remaining = g.key_rate_limit
+            response.headers['X-RateLimit-Key-Limit'] = str(limit)
+            response.headers['X-RateLimit-Key-Remaining'] = str(remaining)
+        response.headers['X-Request-Id'] = g.get('request_id', '-')
+        elapsed = time.monotonic() - g.get('request_started', time.monotonic())
+        endpoint = request.endpoint or 'unmatched'
+        metrics.observe_request(request.method, endpoint, str(response.status_code), elapsed, caller_key_name())
+        if endpoint not in ACCESS_LOG_QUIET:
+            extra = {'method': request.method, 'path': request.path, 'status': response.status_code,
+                    'duration_ms': round(elapsed * 1000, 1)}
+            if g.get('serialization_ms') is not None:  # only set for a paged response - see render()
+                extra['serialization_ms'] = g.serialization_ms
+            log.info('%s %s -> %s in %.1fms', request.method, request.path, response.status_code, elapsed * 1000,
+                    extra=extra)
+        return response
+
+    app.register_blueprint(bp)
+    return app
+
+
+# --------------------------------------------------------------------------------------
+# Request helpers
+# --------------------------------------------------------------------------------------
+
+def check_rate_limit():
+    """Count this request against its client's quota; returns a 429 response when it is over the limit."""
+    limit = config.rate_limit()
+    if limit is None or request.method == 'OPTIONS' or request.endpoint in RATE_LIMIT_EXEMPT:
+        return None
+    count, period = limit
+    client = request.remote_addr or 'unknown'
+    allowed, remaining, retry_after = current_app.extensions['queryapigate_limiter'].hit(client, count, period)
+    g.rate_limit = (count, remaining)
+    if allowed:
+        return None
+    metrics.inc_rate_limit_rejection()
+    response = jsonify({'error': 'Rate limit exceeded', 'retry_after': retry_after})
+    response.status_code = 429
+    response.headers['Retry-After'] = str(retry_after)
+    return response
+
+
+def check_key_rate_limit():
+    """Like check_rate_limit() above, but for a key's own optional `rate_limit` grant - checked in
+    *addition* to the server-wide, IP-based limit, never instead of it (see apikeys.py's module docstring).
+    Runs after g.permission is resolved, unlike the IP-based check, since there is no per-key identity to
+    limit by before then. Naturally a no-op for the admin key and the open/no-key case, both of which
+    always resolve rate_limit=None - no special-casing needed for either."""
+    permission = g.get('permission')
+    if permission is None or permission.rate_limit is None or request.method == 'OPTIONS' \
+            or request.endpoint in RATE_LIMIT_EXEMPT:
+        return None
+    count, period = permission.rate_limit
+    limiter = current_app.extensions['queryapigate_key_limiter']
+    allowed, remaining, retry_after = limiter.hit(permission.name, count, period)
+    g.key_rate_limit = (count, remaining)
+    if allowed:
+        return None
+    metrics.inc_rate_limit_rejection()
+    response = jsonify({'error': "Rate limit exceeded for this API key", 'retry_after': retry_after})
+    response.status_code = 429
+    response.headers['Retry-After'] = str(retry_after)
+    return response
+
+
+def resolve_permission():
+    """The caller's Permission: a matched key (admin or scoped), the unrestricted OPEN default when no key
+    is configured anywhere, or None when a key is required but missing or wrong."""
+    permission = apikeys.authenticate(request.headers.get('X-API-Key', ''), client_ip=request.remote_addr)
+    if permission is not None:
+        return permission
+    return None if apikeys.auth_required() else apikeys.OPEN
+
+
+def require_admin():
+    """Only the admin key (QUERYAPIGATE_API_KEY, or no key at all when nothing is configured) manages the
+    server's own configuration - connections, saved queries and other API keys."""
+    if not g.permission.admin:
+        raise ApiError('This API key is not authorized to manage the server configuration', 403)
+
+
+def require_connection(connection_name):
+    if not apikeys.can_use(g.permission, connection_name):
+        raise ApiError(f"This API key is not permitted to use the connection '{connection_name}'", 403)
+
+
+def caller_key_name():
+    """The calling API key's name for logs/metrics ('admin', a scoped key's name, or '-'); safe to call even
+    before permission is resolved, e.g. while rendering a CORS preflight or a 429 in decorate()."""
+    permission = g.get('permission')
+    return (permission.name if permission else None) or '-'
+
+
+def _dict_diff(before, after):
+    """{field: {'from':.., 'to':..}} for every field that differs between two dicts (union of both sets of
+    keys) - used to build audit_log 'changes' for an update. Only the fields that actually changed, not the
+    whole entry, so a reviewer doesn't have to spot the difference themselves."""
+    keys = set(before or {}) | set(after or {})
+    return {k: {'from': (before or {}).get(k), 'to': (after or {}).get(k)}
+            for k in keys if (before or {}).get(k) != (after or {}).get(k)}
+
+
+def _connection_audit_changes(before, after):
+    """Like _dict_diff(), but for a connection's raw (unmasked) stored fields specifically: 'password' is
+    reported only as the literal string 'changed' when it differs, in either direction - never the actual
+    value, before or after masking, since this is what gets persisted to audit_log.json. Every other field
+    (host, port, user, db, database, active, ...) is not a secret and is shown as given. `before=None`
+    means the connection didn't exist yet - the caller records that as a 'create_connection' snapshot
+    instead of calling this."""
+    diff = _dict_diff(before, after)
+    if 'password' in diff:
+        diff['password'] = 'changed'
+    return diff
+
+
+def get_json_body(required=True):
+    data = request.get_json(silent=True)
+    if data is None and not required:
+        return {}
+    if not isinstance(data, dict):
+        raise ApiError('Request body must be a JSON object')
+    return data
+
+
+def get_object(value, label):
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ApiError(f'{label} must be a JSON object')
+    return value
+
+
+def get_int(value, label):
+    if value is None or value == '':
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        raise ApiError(f'{label} must be an integer') from None
+
+
+def get_pagination():
+    """Return (limit, offset, page) from the ?page and ?page_size query parameters."""
+    page = get_int(request.args.get('page'), 'page')
+    page_size = get_int(request.args.get('page_size'), 'page_size')
+    page = 1 if page is None else page
+    page_size = 10 if page_size is None else page_size
+    if page < 1 or page_size < 1:
+        raise ApiError('page and page_size must be positive')
+    if page_size > config.max_page_size():
+        raise ApiError(f'page_size must not exceed {config.max_page_size()}')
+    return page_size, (page - 1) * page_size, page
+
+
+def get_output_format(body=None):
+    output_format = str(request.args.get('format') or (body or {}).get('format') or 'json').lower()
+    if output_format not in FORMATTERS:
+        raise ApiError(f"Unsupported format '{output_format}'. Supported formats: {', '.join(FORMATTERS)}")
+    return output_format
+
+
+def get_timeout(body=None):
+    """Seconds allowed for the query: ?timeout= may lower the server limit but never raise it."""
+    raw = request.args.get('timeout', (body or {}).get('timeout'))
+    if raw in (None, ''):
+        return config.effective_timeout(None)
+    try:
+        value = float(raw)
+    except (TypeError, ValueError):
+        raise ApiError('timeout must be a number of seconds') from None
+    if not math.isfinite(value) or value <= 0:
+        raise ApiError('timeout must be a positive number of seconds')
+    return config.effective_timeout(value)
+
+
+def render(result, output_format, page, page_size):
+    """Build the paged response body and record how long that took (JSON/CSV/TSV/XML/YAML/XLSX encoding) as
+    its own metric and access-log field, separate from query execution time - see
+    metrics.observe_serialization(). Streaming responses never go through here; there's no equivalent
+    single serialization span to measure for those (see that function's docstring)."""
+    started = time.monotonic()
+    response = jsonify({'message': 'No results returned'}) if not result else FORMATTERS[output_format](result)
+    elapsed = time.monotonic() - started
+    metrics.observe_serialization(output_format, elapsed)
+    g.serialization_ms = round(elapsed * 1000, 1)
+    response.headers['X-Page'] = str(page)
+    response.headers['X-Page-Size'] = str(page_size)
+    response.headers['X-Has-More'] = 'true' if result.has_more else 'false'
+    return response
+
+
+def get_stream_flag():
+    """Whether ?stream=true was requested: the whole result, streamed as it comes off the cursor instead of
+    one page built in memory first. Only csv/tsv/ndjson support it - see formats.STREAM_FORMATTERS."""
+    return (request.args.get('stream') or '').strip().lower() in ('1', 'true', 'yes', 'on')
+
+
+def stream_sql_response(sql, connection_name, params, timeout, output_format, filename, saved=None):
+    """Shared by execute_sql_endpoint() and run_saved(): validates the stream=true-specific constraints
+    (format, no pagination) and returns the chunked Response. ``saved``, when given, is (path, version) for
+    a saved query whose run should still be recorded in its execution_history once the stream finishes."""
+    if output_format not in STREAM_FORMATTERS:
+        raise ApiError(f"stream=true only supports these formats: {', '.join(sorted(STREAM_FORMATTERS))}")
+    if request.args.get('page') or request.args.get('page_size'):
+        raise ApiError('stream=true exports the whole result and does not accept page/page_size')
+    key_name = caller_key_name()
+    columns, rows = engine.stream_sql(sql, connection_name, params, timeout, key_name=key_name)
+    if saved is not None:
+        path, number = saved
+        # Captured here, not inside _record_stream_history(): that generator's body runs lazily, as the
+        # response streams out - by then the request/app context this view function runs in is long gone
+        # (no flask.stream_with_context() wrapping is used), so g and caller_key_name() are only safe to
+        # read up front, while still inside the request that's actually issuing the query.
+        rows = _record_stream_history(rows, path, number, connection_name, g.get('request_id'), key_name)
+    return stream_response(output_format, columns, rows, filename)
+
+
+def _record_stream_history(rows, path, number, connection_name, request_id, key_name):
+    """Records a saved query's streamed run in its execution_history once fully drained or failed partway
+    through (not on a client disconnect, GeneratorExit) - counting rows as they pass through, since the
+    total is not known up front, the same trade-off engine._drain() makes for the streaming metric."""
+    entry = {'executed_at': store.now(), 'connection_name': connection_name,
+             'request_id': request_id, 'key_name': key_name}
+    count = 0
+    try:
+        for row in rows:
+            count += 1
+            yield row
+    except GeneratorExit:
+        raise
+    except ApiError as error:
+        store.record_execution(path, number, {**entry, 'status': 'error', 'error': error.message, 'rows': count})
+        raise
+    except Exception as error:
+        store.record_execution(path, number, {**entry, 'status': 'error', 'error': str(error), 'rows': count})
+        raise
+    else:
+        store.record_execution(path, number, {**entry, 'status': 'success', 'rows': count})
+
+
+# --------------------------------------------------------------------------------------
+# Query execution
+# --------------------------------------------------------------------------------------
+
+@bp.route('/execute_sql', methods=['POST'])
+def execute_sql_endpoint():
+    data = get_json_body()
+    if not data.get('sql'):
+        raise ApiError('SQL query is missing')
+    if not data.get('connection_name'):
+        raise ApiError('Connection name is missing')
+    require_connection(data['connection_name'])
+    params = get_object(data.get('params'), 'params')
+    output_format = get_output_format(data)
+    timeout = get_timeout(data)
+    if get_stream_flag():
+        return stream_sql_response(data['sql'], data['connection_name'], params, timeout, output_format,
+                                   filename=data['connection_name'])
+    limit, offset, page = get_pagination()
+    result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params, timeout,
+                                allow_writes=g.permission.allow_writes, key_name=caller_key_name(),
+                                allowed_write_ops=g.permission.allowed_write_ops)
+    return render(result, output_format, page, limit)
+
+
+def cache_lookup(cache_key, ttl):
+    """A live entry rendered as a response (304 if the client already has it), or None on a cache miss."""
+    hit = current_app.extensions['queryapigate_cache'].get(cache_key)
+    if hit is None:
+        return None
+    body, content_type, headers, etag = hit
+    quoted = f'"{etag}"'
+    if request.headers.get('If-None-Match') == quoted:
+        response = Response(status=304)
+    else:
+        response = Response(body, content_type=content_type)
+        for name, value in headers:
+            response.headers[name] = value
+    response.headers['ETag'] = quoted
+    response.headers['Cache-Control'] = f'max-age={ttl}'
+    response.headers['X-Cache'] = 'HIT'
+    return response
+
+
+def cache_store(cache_key, response, ttl):
+    """Store `response` under `cache_key` for `ttl` seconds and tag it as a fresh cache MISS."""
+    replay_headers = [(name, value) for name, value in response.headers.items()
+                      if name.lower() not in ('content-type', 'content-length')]
+    etag = current_app.extensions['queryapigate_cache'].set(cache_key, response.get_data(), response.content_type,
+                                                       replay_headers, ttl)
+    response.headers['ETag'] = f'"{etag}"'
+    response.headers['Cache-Control'] = f'max-age={ttl}'
+    response.headers['X-Cache'] = 'MISS'
+    return response
+
+
+def run_saved(ref, body, url_params):
+    """Execute a saved query (latest version unless one is requested) and record the run - or, for one with
+    a cache_ttl whose SQL is read-only, serve a cached response instead."""
+    path = store.resolve_saved_file(ref)
+    number, saved = store.select_version(store.load_versions(path),
+                                         get_int(request.args.get('version') or body.get('version'), 'version'))
+    connection_name = request.args.get('connection_name') or body.get('connection_name') \
+        or saved.get('connection_name')
+    if not connection_name:
+        raise ApiError('Connection name is missing')
+    query_name = store.query_name(path)
+    # A key's `queries` grant is additive on top of `connections` (see apikeys.py's module docstring): it
+    # can name this exact saved query without needing connection access at all, so only fall back to the
+    # ordinary per-connection check when the query itself was not specifically granted.
+    if not apikeys.can_use_query(g.permission, query_name):
+        require_connection(connection_name)
+    if not isinstance(saved.get('sql_query'), str):
+        raise ApiError('Saved query has no SQL', 500)
+    # Per-query write curation (apikeys.can_write_query()) only ever adds write reach for this one named
+    # query on top of whatever the key's blanket allow_writes already grants - never the other way round.
+    effective_allow_writes = g.permission.allow_writes or apikeys.can_write_query(g.permission, query_name)
+
+    raw = {**url_params, **get_object(body.get('params'), 'params'),
+           **get_object(body.get('placeholders'), 'placeholders')}
+    used = set(sqltools.placeholder_names(saved['sql_query']))
+    values = param_rules.resolve(saved.get('query_parameters'), raw, used=used)
+    sql = sqltools.fill_placeholders(saved['sql_query'], values)
+    output_format = get_output_format(body)
+    timeout = get_timeout(body)
+
+    if get_stream_flag():
+        # No caching for a streamed export - caching would require materialising the whole body anyway,
+        # defeating the point - but the run is still recorded once the stream finishes, same as any other.
+        return stream_sql_response(sql, connection_name, values, timeout, output_format, filename=ref,
+                                   saved=(path, number))
+
+    limit, offset, page = get_pagination()
+
+    # A saved query is only ever cached when it declares a cache_ttl *and* its SQL is read-only - never a
+    # write, no matter the setting, since serving a cached response would silently skip that write.
+    ttl = saved.get('cache_ttl') or 0
+    cache_key = None
+    if ttl > 0:
+        dialect = store.get_connection(connection_name)['db']
+        if sqltools.first_keyword(sql, dialect) in sqltools.READ_ONLY_STATEMENTS:
+            cache_key = cache.ResponseCache.key(name=ref, version=number, connection=connection_name,
+                                                values=values, format=output_format, page=page, page_size=limit)
+            cached = cache_lookup(cache_key, ttl)
+            if cached is not None:
+                return cached
+
+    entry = {'executed_at': store.now(), 'connection_name': connection_name,
+             'request_id': g.get('request_id'), 'key_name': caller_key_name()}
+    try:
+        result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset, values, timeout,
+                                          allow_writes=effective_allow_writes, key_name=caller_key_name(),
+                                          allowed_write_ops=g.permission.allowed_write_ops)
+    except ApiError as error:
+        store.record_execution(path, number, {**entry, 'status': 'error', 'error': error.message})
+        raise
+    response = render(result, output_format, page, limit)  # sets g.serialization_ms - see render()
+    store.record_execution(path, number, {**entry, 'status': 'success', 'rows': len(result.rows),
+                                          'duration_ms': elapsed_ms, 'serialization_ms': g.serialization_ms})
+    if cache_key is not None:
+        response = cache_store(cache_key, response, ttl)
+    return response
+
+
+@bp.route('/execute_sql_from_file', methods=['POST'])
+@bp.route('/execute_sql_with_parameters_from_file', methods=['POST'])
+def execute_sql_from_file():
+    body = get_json_body()
+    return run_saved(body.get('filepath'), body, {})
+
+
+@bp.route('/q/<name>', methods=['GET', 'POST'])
+def run_named_query(name):
+    body = get_json_body(required=False) if request.method == 'POST' else {}
+    url_params = {k: v for k, v in request.args.items() if k not in RESERVED_ARGS}
+    return run_saved(name, body, url_params)
+
+
+# --------------------------------------------------------------------------------------
+# Saved queries
+# --------------------------------------------------------------------------------------
+
+@bp.route('/view_file_content', methods=['GET'])
+def view_file_content():
+    require_admin()
+    path = store.resolve_saved_file(request.args.get('filename'))
+    with open(path, 'r') as f:
+        return jsonify({'content': f.read()}), 200
+
+
+@bp.route('/save_sql_to_file', methods=['PATCH'])
+def save_sql_to_file():
+    require_admin()
+    data = get_json_body()
+    for field, label in (('author', 'Author'), ('description', 'Description'),
+                         ('sql_query', 'SQL query'), ('filename', 'Filename')):
+        if not data.get(field) or not isinstance(data[field], str):
+            raise ApiError(f'{label} is missing')
+    tags = data.get('tags', [])
+    if not isinstance(tags, (list, str)):
+        raise ApiError('tags must be a string or a list')
+    query_parameters = get_object(data.get('query_parameters'), 'query_parameters')
+    param_rules.parse_definitions(query_parameters)
+    unused = sorted(set(query_parameters) - set(sqltools.placeholder_names(data['sql_query'])))
+    if unused:
+        raise ApiError(f"query_parameters declares {', '.join(unused)}, which sql_query does not use "
+                       '(write :name in the SQL, or remove the declaration)')
+    connection_name = data.get('connection_name')
+    if connection_name is not None and not isinstance(connection_name, str):
+        raise ApiError('connection_name must be a string')
+    cache_ttl = data.get('cache_ttl')
+    if cache_ttl is not None and (not isinstance(cache_ttl, int) or isinstance(cache_ttl, bool) or cache_ttl < 0):
+        raise ApiError('cache_ttl must be a non-negative integer number of seconds')
+
+    query_uuid, version = store.save_version(data['filename'], {
+        'sql_query': data['sql_query'],
+        'author': data['author'],
+        'description': data['description'],
+        'tags': tags,
+        'query_parameters': query_parameters,
+        **({'connection_name': connection_name} if connection_name else {}),
+        **({'cache_ttl': cache_ttl} if cache_ttl else {}),
+    })
+    store.record_audit(caller_key_name(), 'save_query', data['filename'],
+                       {'version': version, 'connection_name': connection_name, 'author': data['author']})
+    return jsonify({'message': 'SQL query saved successfully', 'filename': data['filename'],
+                    'uuid': query_uuid, 'version': version}), 200
+
+
+@bp.route('/saved_sql/<name>', methods=['DELETE'])
+def delete_saved_query(name):
+    require_admin()
+    version = get_int(request.args.get('version'), 'version')
+    store.delete_saved(name, version)
+    store.record_audit(caller_key_name(), 'delete_query', name,
+                       {'version': version} if version is not None else None)
+    what = f'Version {version} of {name}' if version is not None else name
+    return jsonify({'message': f'{what} deleted'}), 200
+
+
+@bp.route('/list_files', methods=['GET'])
+def list_files():
+    require_admin()
+    sort_by = request.args.get('sort_by', 'name')
+    if sort_by not in ('name', 'modified'):
+        raise ApiError("sort_by must be 'name' or 'modified'")
+    sort_order = request.args.get('sort_order', 'asc')
+    if sort_order not in ('asc', 'desc'):
+        raise ApiError("sort_order must be 'asc' or 'desc'")
+
+    files = store.list_saved()
+    if sort_by == 'name':
+        def key(f):
+            return f['filename'].lower()
+    else:
+        def key(f):
+            return (f['versions'][-1].get('last_modified_at') or '') if f['versions'] else ''
+    files.sort(key=key, reverse=sort_order == 'desc')
+    return jsonify({'files': files}), 200
+
+
+# --------------------------------------------------------------------------------------
+# Connections
+# --------------------------------------------------------------------------------------
+
+@bp.route('/connections', methods=['GET'])
+def get_connections():
+    require_admin()
+    connections = store.mask_passwords(store.read_connections())
+    for name, conn in connections.items():
+        conn['usage'] = metrics.summary_for_connection(name)
+    return jsonify({'connections': connections}), 200
+
+
+@bp.route('/connections', methods=['PATCH'])
+def update_connections():
+    require_admin()
+    connections = get_json_body().get('connections')
+    if not connections or not isinstance(connections, dict):
+        raise ApiError('Connections data is missing')
+    before = store.read_connections()  # raw, unmasked - in memory only, never itself logged; see below
+    store.update_connections(connections)
+    after = store.read_connections()
+    actor = caller_key_name()
+    for name in connections:
+        if name not in before:
+            store.record_audit(actor, 'create_connection', name, store.mask_passwords({name: after[name]})[name])
+        else:
+            changes = _connection_audit_changes(before.get(name), after.get(name))
+            if changes:
+                store.record_audit(actor, 'update_connection', name, changes)
+    pool.close_pooled_connections()  # new settings or credentials must not be served by old connections
+    return jsonify({'message': 'Connections updated successfully'}), 200
+
+
+@bp.route('/connections/<name>', methods=['DELETE'])
+def delete_connection(name):
+    require_admin()
+    before = store.read_connections().get(name)
+    store.delete_connection(name)
+    if before is not None:
+        store.record_audit(caller_key_name(), 'delete_connection', name, store.mask_passwords({name: before})[name])
+    pool.close_pooled_connections()  # a removed connection must not keep serving from idle sockets
+    return jsonify({'message': f"Connection '{name}' deleted"}), 200
+
+
+@bp.route('/connections/<name>/schema', methods=['GET'])
+def connection_schema(name):
+    require_connection(name)
+    return jsonify(schema.fetch_schema(name)), 200
+
+
+# --------------------------------------------------------------------------------------
+# API keys
+# --------------------------------------------------------------------------------------
+
+@bp.route('/api_keys', methods=['GET'])
+def get_api_keys():
+    require_admin()
+    keys = apikeys.list_keys()
+    for name, key in keys.items():
+        key['usage'] = metrics.summary_for_key(name)
+    return jsonify({'keys': keys}), 200
+
+
+@bp.route('/api_keys', methods=['POST'])
+def create_api_key():
+    require_admin()
+    data = get_json_body()
+    secret = apikeys.create_key(data.get('name'), connections=data.get('connections'),
+                                allow_writes=data.get('allow_writes'), queries=data.get('queries'),
+                                expires_at=data.get('expires_at'), rate_limit=data.get('rate_limit'),
+                                allowed_ips=data.get('allowed_ips'),
+                                allowed_write_ops=data.get('allowed_write_ops'), role=data.get('role'))
+    store.record_audit(caller_key_name(), 'create_key', data.get('name'), apikeys.list_keys().get(data.get('name')))
+    return jsonify({'name': data.get('name'), 'key': secret,
+                    'message': "Store this key now - it can't be shown again."}), 200
+
+
+@bp.route('/api_keys/<name>', methods=['PATCH'])
+def update_api_key(name):
+    require_admin()
+    data = get_json_body()
+    before = apikeys.list_keys().get(name)
+    # expires_at, rate_limit, allowed_ips and allowed_write_ops all need a real presence check, not .get():
+    # an explicit null in the request body means "clear it", which must be distinguishable from the field
+    # being absent ("leave it alone") - see apikeys.update_key's _UNSET sentinel.
+    unset_kwargs = {k: data[k] for k in ('expires_at', 'rate_limit', 'allowed_ips', 'allowed_write_ops')
+                    if k in data}
+    apikeys.update_key(name, connections=data.get('connections'), allow_writes=data.get('allow_writes'),
+                       active=data.get('active'), queries=data.get('queries'), **unset_kwargs)
+    changes = _dict_diff(before, apikeys.list_keys().get(name))
+    if changes:
+        store.record_audit(caller_key_name(), 'update_key', name, changes)
+    return jsonify({'message': f"API key '{name}' updated"}), 200
+
+
+@bp.route('/api_keys/<name>', methods=['DELETE'])
+def delete_api_key(name):
+    require_admin()
+    before = apikeys.list_keys().get(name)
+    apikeys.delete_key(name)
+    store.record_audit(caller_key_name(), 'delete_key', name, before)
+    return jsonify({'message': f"API key '{name}' deleted"}), 200
+
+
+@bp.route('/roles', methods=['GET'])
+def get_roles():
+    require_admin()
+    return jsonify({'roles': apikeys.list_roles()}), 200
+
+
+@bp.route('/roles', methods=['POST'])
+def create_role_endpoint():
+    require_admin()
+    data = get_json_body()
+    apikeys.create_role(data.get('name'), connections=data.get('connections'),
+                        allow_writes=bool(data.get('allow_writes', False)), queries=data.get('queries'),
+                        rate_limit=data.get('rate_limit'), allowed_ips=data.get('allowed_ips'),
+                        allowed_write_ops=data.get('allowed_write_ops'))
+    store.record_audit(caller_key_name(), 'create_role', data.get('name'), apikeys.list_roles().get(data.get('name')))
+    return jsonify({'message': f"Role '{data.get('name')}' created"}), 200
+
+
+@bp.route('/roles/<name>', methods=['PATCH'])
+def update_role_endpoint(name):
+    require_admin()
+    data = get_json_body()
+    before = apikeys.list_roles().get(name)
+    # Same _UNSET-sentinel presence check update_api_key() already uses for these three fields.
+    unset_kwargs = {k: data[k] for k in ('rate_limit', 'allowed_ips', 'allowed_write_ops') if k in data}
+    apikeys.update_role(name, connections=data.get('connections'), allow_writes=data.get('allow_writes'),
+                        queries=data.get('queries'), **unset_kwargs)
+    changes = _dict_diff(before, apikeys.list_roles().get(name))
+    if changes:
+        store.record_audit(caller_key_name(), 'update_role', name, changes)
+    return jsonify({'message': f"Role '{name}' updated"}), 200
+
+
+@bp.route('/roles/<name>', methods=['DELETE'])
+def delete_role_endpoint(name):
+    require_admin()
+    before = apikeys.list_roles().get(name)
+    apikeys.delete_role(name)
+    store.record_audit(caller_key_name(), 'delete_role', name, before)
+    return jsonify({'message': f"Role '{name}' deleted"}), 200
+
+
+@bp.route('/audit_log', methods=['GET'])
+def audit_log_endpoint():
+    """A durable record of administrative changes - who created, changed or removed an API key, connection
+    or saved query, and when (see store.record_audit()). Admin only, like everything else that reveals the
+    server's own configuration; newest entry first, capped at config.audit_log_limit()."""
+    require_admin()
+    return jsonify({'entries': list(reversed(store.read_audit_log()))}), 200
+
+
+# --------------------------------------------------------------------------------------
+# Service endpoints
+# --------------------------------------------------------------------------------------
+
+@bp.route('/', methods=['GET'])
+def index():
+    return redirect(url_for('api.docs'))
+
+
+@bp.route('/favicon.ico', methods=['GET'])
+def favicon():
+    return '', 204
+
+
+@bp.route('/health', methods=['GET'])
+def health():
+    from flask import current_app
+    return jsonify({'status': 'ok', 'version': current_app.config['QUERYAPIGATE_VERSION']})
+
+
+@bp.route('/metrics', methods=['GET'])
+def metrics_endpoint():
+    return Response(metrics.render(), mimetype='text/plain; version=0.0.4; charset=utf-8')
+
+
+def describe_saved_queries(permission):
+    """What the OpenAPI document needs to know about each saved query (never its SQL text) that
+    ``permission`` may actually call - the same test run_saved() applies, so a key scoped to specific
+    queries (see apikeys.py) sees only its own approved list here, not the whole internal catalogue; an
+    unrestricted (admin, or connection-wide) key sees everything, unchanged from before this filter."""
+    described = []
+    for name, number, data in store.latest_versions():
+        sql = data.get('sql_query')
+        if not isinstance(sql, str):
+            continue
+        connection_name = data.get('connection_name')
+        if not (apikeys.can_use_query(permission, name)
+                or (connection_name and apikeys.can_use(permission, connection_name))):
+            continue
+        declared = param_rules.read_definitions(data.get('query_parameters'))
+        used = sqltools.placeholder_names(sql)
+        parameters = {}
+        for param in used:  # what the SQL needs, in order; undeclared ones are plain required text
+            parameters[param] = declared.get(param) or param_rules.read_definition({})
+        described.append({'name': name, 'version': number, 'description': data.get('description'),
+                          'tags': data.get('tags'), 'connection_name': data.get('connection_name'),
+                          'parameters': parameters})
+    return described
+
+
+@bp.route('/openapi.json', methods=['GET'])
+def openapi_spec():
+    from flask import current_app
+    # The generic API description is public. The list of saved queries (names, descriptions, parameters - never
+    # the SQL itself) is shown to any authenticated caller, admin or scoped, same as any other endpoint they
+    # could call through /q/<name> - a scoped key still needs to know a query's parameters to use it.
+    saved = describe_saved_queries(g.permission) if g.permission is not None else None
+    return jsonify(openapi.build_spec(current_app.config['QUERYAPIGATE_VERSION'], saved))
+
+
+def describe_catalog(permission):
+    """Like describe_saved_queries() (same reachability rule, same parameter shape - kept as its own loop
+    rather than sharing one, since the two describe different things to different consumers: OpenAPI's
+    request/response shape versus this endpoint's governance terms - and OpenAPI's shape is a stable
+    contract other tooling parses, not something to risk changing by threading new fields through it), plus
+    the governance facts OpenAPI has no field for: whether this query is cached, and whether *this specific
+    caller* can write through it (apikeys.can_write_query() - independent of their blanket allow_writes)."""
+    catalog = []
+    for name, number, data in store.latest_versions():
+        sql = data.get('sql_query')
+        if not isinstance(sql, str):
+            continue
+        connection_name = data.get('connection_name')
+        if not (apikeys.can_use_query(permission, name)
+                or (connection_name and apikeys.can_use(permission, connection_name))):
+            continue
+        declared = param_rules.read_definitions(data.get('query_parameters'))
+        used = sqltools.placeholder_names(sql)
+        parameters = {}
+        for param in used:
+            parameters[param] = declared.get(param) or param_rules.read_definition({})
+        catalog.append({'name': name, 'version': number, 'description': data.get('description'),
+                        'tags': data.get('tags'), 'connection_name': connection_name,
+                        'parameters': parameters, 'cache_ttl': data.get('cache_ttl') or None,
+                        'can_write': apikeys.can_write_query(permission, name)})
+    return catalog
+
+
+@bp.route('/catalog', methods=['GET'])
+def catalog():
+    """Everything this caller can reach through /q/<name>, and the terms it's offered under, in one place -
+    closing the gap where that information exists (cache_ttl on a saved query, a key's own rate_limit and
+    write curation) but was scattered across admin-only screens a scoped key can never reach. Requires the
+    same authentication any other functional endpoint does (unlike /openapi.json, this is never public) -
+    the whole point is answering 'what can *I* use,' which needs a resolved caller to mean anything."""
+    permission = g.permission
+    return jsonify({
+        'queries': describe_catalog(permission),
+        'caller': {
+            'name': permission.name,
+            'admin': permission.admin,
+            'allow_writes': permission.allow_writes,
+            'allowed_write_ops': permission.allowed_write_ops,
+            'rate_limit': config.format_rate_limit(permission.rate_limit),
+            'server_rate_limit': config.format_rate_limit(config.rate_limit()),
+        },
+    }), 200
+
+
+@bp.route('/docs', methods=['GET'])
+def docs():
+    return Response(openapi.DOCS_HTML, mimetype='text/html')
+
+
+@bp.route('/ui', methods=['GET'])
+def admin_ui():
+    return Response(ui.UI_HTML, mimetype='text/html')