sql2api 0.1.0__py3-none-any.whl

sql2api/openapi.py

@@ -0,0 +1,127 @@
+"""OpenAPI description of the API, served at /openapi.json and rendered at /docs."""
+from . import config
+
+_FORMAT_PARAM = {'name': 'format', 'in': 'query', 'schema': {
+    'type': 'string', 'enum': ['json', 'ndjson', 'csv', 'tsv', 'xml', 'yaml', 'xlsx'], 'default': 'json'}}
+_PAGE_PARAMS = [
+    {'name': 'page', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 1}},
+    {'name': 'page_size', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 10},
+     'description': 'Upper limit is SQL2API_MAX_PAGE_SIZE (default 1000).'},
+]
+_ROWS = {'description': 'The requested page. Header X-Has-More says whether another page follows.',
+         'headers': {'X-Page': {'schema': {'type': 'integer'}}, 'X-Page-Size': {'schema': {'type': 'integer'}},
+                     'X-Has-More': {'schema': {'type': 'string', 'enum': ['true', 'false']}}},
+         'content': {'application/json': {'schema': {'type': 'array', 'items': {'type': 'object'}}}}}
+_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '500')}
+
+
+def _body(properties, required):
+    return {'required': True, 'content': {'application/json': {'schema': {
+        'type': 'object', 'required': required, 'properties': properties}}}}
+
+
+_EXEC_PROPS = {
+    'connection_name': {'type': 'string'},
+    'format': {'type': 'string'},
+    'placeholders': {'type': 'object', 'description': 'Values for :name (bound) and {name} (text) parameters.'},
+    'params': {'type': 'object', 'description': 'Alias of placeholders.'},
+    'version': {'type': 'integer', 'description': 'Saved version to run (default: latest).'},
+}
+
+
+def build_spec(version):
+    saved = {'filepath': {'type': 'string', 'description': 'Saved query name or path inside saved_sql/.'},
+             **_EXEC_PROPS}
+    return {
+        'openapi': '3.0.3',
+        'info': {'title': 'SQL2API', 'version': version,
+                 'description': 'Run SQL against configured databases and get the results back over HTTP.'},
+        'components': {
+            'securitySchemes': {'ApiKey': {'type': 'apiKey', 'in': 'header', 'name': 'X-API-Key'}},
+            'responses': {'Error': {'description': 'Error', 'content': {'application/json': {'schema': {
+                'type': 'object', 'properties': {'error': {'type': 'string'}, 'detail': {'type': 'string'}}}}}}},
+        },
+        'security': [{}, {'ApiKey': []}],
+        'paths': {
+            '/execute_sql': {'post': {
+                'summary': 'Execute SQL', 'tags': ['Query'],
+                'parameters': [_FORMAT_PARAM, *_PAGE_PARAMS],
+                'requestBody': _body({'sql': {'type': 'string'}, 'connection_name': {'type': 'string'},
+                                      'params': {'type': 'object',
+                                                 'description': 'Values for :name bound parameters.'}},
+                                     ['sql', 'connection_name']),
+                'responses': {'200': _ROWS, **_ERRORS}}},
+            '/q/{name}': {
+                'parameters': [{'name': 'name', 'in': 'path', 'required': True, 'schema': {'type': 'string'}}],
+                'get': {'summary': 'Run a saved query; extra query-string arguments become parameters',
+                        'tags': ['Saved queries'],
+                        'parameters': [_FORMAT_PARAM, *_PAGE_PARAMS,
+                                       {'name': 'connection_name', 'in': 'query', 'schema': {'type': 'string'}},
+                                       {'name': 'version', 'in': 'query', 'schema': {'type': 'integer'}}],
+                        'responses': {'200': _ROWS, **_ERRORS}},
+                'post': {'summary': 'Run a saved query with a JSON body of parameters', 'tags': ['Saved queries'],
+                         'parameters': [_FORMAT_PARAM, *_PAGE_PARAMS],
+                         'requestBody': _body(_EXEC_PROPS, []),
+                         'responses': {'200': _ROWS, **_ERRORS}}},
+            '/execute_sql_from_file': {'post': {
+                'summary': 'Run the latest version of a saved query', 'tags': ['Saved queries'],
+                'parameters': [_FORMAT_PARAM, *_PAGE_PARAMS],
+                'requestBody': _body(saved, ['filepath']), 'responses': {'200': _ROWS, **_ERRORS}}},
+            '/execute_sql_with_parameters_from_file': {'post': {
+                'summary': 'Run a saved query with parameter values', 'tags': ['Saved queries'],
+                'parameters': [_FORMAT_PARAM, *_PAGE_PARAMS],
+                'requestBody': _body(saved, ['filepath']), 'responses': {'200': _ROWS, **_ERRORS}}},
+            '/save_sql_to_file': {'patch': {
+                'summary': 'Save a query (creates the next version)', 'tags': ['Saved queries'],
+                'requestBody': _body({
+                    'filename': {'type': 'string'}, 'sql_query': {'type': 'string'}, 'author': {'type': 'string'},
+                    'description': {'type': 'string'}, 'tags': {'type': 'array', 'items': {'type': 'string'}},
+                    'query_parameters': {'type': 'object', 'example': {'actor_id': 'int'}},
+                    'connection_name': {'type': 'string', 'description': 'Default connection for /q/{name}.'}},
+                    ['filename', 'sql_query', 'author', 'description']),
+                'responses': {'200': {'description': 'Saved'}, **_ERRORS}}},
+            '/list_files': {'get': {
+                'summary': 'List saved queries', 'tags': ['Saved queries'],
+                'parameters': [
+                    {'name': 'sort_by', 'in': 'query', 'schema': {'type': 'string', 'enum': ['name', 'modified']}},
+                    {'name': 'sort_order', 'in': 'query', 'schema': {'type': 'string', 'enum': ['asc', 'desc']}}],
+                'responses': {'200': {'description': 'Saved queries with version metadata'}, **_ERRORS}}},
+            '/view_file_content': {'get': {
+                'summary': 'Raw content of a saved query file', 'tags': ['Saved queries'],
+                'parameters': [{'name': 'filename', 'in': 'query', 'required': True, 'schema': {'type': 'string'}}],
+                'responses': {'200': {'description': 'File content'}, **_ERRORS}}},
+            '/saved_sql/{name}': {'delete': {
+                'summary': 'Delete a saved query, or one version with ?version=', 'tags': ['Saved queries'],
+                'parameters': [{'name': 'name', 'in': 'path', 'required': True, 'schema': {'type': 'string'}},
+                               {'name': 'version', 'in': 'query', 'schema': {'type': 'integer'}}],
+                'responses': {'200': {'description': 'Deleted'}, **_ERRORS}}},
+            '/connections': {
+                'get': {'summary': 'List connections (passwords masked)', 'tags': ['Connections'],
+                        'responses': {'200': {'description': 'Connections'}, **_ERRORS}},
+                'patch': {'summary': 'Add or update connections', 'tags': ['Connections'],
+                          'requestBody': _body({'connections': {
+                              'type': 'object', 'description': 'Map of name to connection details.',
+                              'additionalProperties': {'type': 'object', 'properties': {
+                                  'db': {'type': 'string', 'enum': list(config.SUPPORTED_DB_TYPES)},
+                                  'host': {'type': 'string'}, 'port': {'type': 'integer'},
+                                  'user': {'type': 'string'}, 'password': {'type': 'string'},
+                                  'database': {'type': 'string'}, 'active': {'type': 'boolean'}}}}},
+                              ['connections']),
+                          'responses': {'200': {'description': 'Updated'}, **_ERRORS}}},
+            '/connections/{name}': {'delete': {
+                'summary': 'Delete a connection', 'tags': ['Connections'],
+                'parameters': [{'name': 'name', 'in': 'path', 'required': True, 'schema': {'type': 'string'}}],
+                'responses': {'200': {'description': 'Deleted'}, **_ERRORS}}},
+            '/health': {'get': {'summary': 'Liveness check', 'tags': ['Service'],
+                                'responses': {'200': {'description': 'OK'}}}},
+        },
+    }
+
+
+DOCS_HTML = """<!doctype html>
+<html><head><meta charset="utf-8"><title>SQL2API docs</title>
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css"></head>
+<body><div id="ui"></div>
+<script src="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui-bundle.js"></script>
+<script>SwaggerUIBundle({url: 'openapi.json', dom_id: '#ui'});</script></body></html>
+"""

sql2api/runners.py

@@ -0,0 +1,142 @@
+"""One runner per database type. Each connects, runs one statement, closes, and returns (columns, rows).
+
+Runners fetch ``limit + 1`` rows: the extra row is how the caller learns whether another page exists.
+Database drivers are imported lazily so only the ones you actually use need to be installed.
+"""
+import os
+import sqlite3
+from pathlib import Path
+
+from . import config
+from .errors import ApiError
+from .sqltools import bind_parameters, is_paginated, paginate
+
+
+def _connect_args(details, **renames):
+    """Pick the standard connection fields out of a stored connection, renaming keys per driver."""
+    args = {}
+    for key in ('host', 'port', 'user', 'password', 'database'):
+        if details.get(key) not in (None, ''):
+            args[renames.get(key, key)] = details[key]
+    return args
+
+
+def _prepare(sql, params, style, limit, offset):
+    """Return (sql, args, sliced) - the statement to send, its bound arguments and whether the
+    database already applied the requested window (LIMIT/OFFSET) for us."""
+    window = limit + 1
+    if is_paginated(sql):
+        sql, sliced = paginate(sql, window, offset), True
+    else:
+        sliced = False
+    sql, args = bind_parameters(sql, params or {}, style)
+    return sql, args, sliced
+
+
+def _fetch_page(cursor, sql, params, style, limit, offset):
+    sql, args, sliced = _prepare(sql, params, style, limit, offset)
+    if args is None:
+        cursor.execute(sql)
+    else:
+        cursor.execute(sql, args)
+    if not cursor.description:
+        return [], []
+    rows = cursor.fetchall() if sliced else cursor.fetchmany(offset + limit + 1)[offset:]
+    return [d[0] for d in cursor.description], rows
+
+
+def _run_mysql(details, sql, params, limit, offset, read_only):
+    import mysql.connector
+    conn = mysql.connector.connect(connection_timeout=config.CONNECT_TIMEOUT, **_connect_args(details))
+    try:
+        cursor = conn.cursor()
+        if read_only:
+            cursor.execute('SET SESSION TRANSACTION READ ONLY')
+        result = _fetch_page(cursor, sql, params, 'format', limit, offset)
+        if not read_only:
+            conn.commit()
+        return result
+    finally:
+        conn.close()
+
+
+def _run_postgres(details, sql, params, limit, offset, read_only):
+    import psycopg2
+    conn = psycopg2.connect(connect_timeout=config.CONNECT_TIMEOUT, **_connect_args(details, database='dbname'))
+    try:
+        if read_only:
+            conn.set_session(readonly=True)
+        with conn.cursor() as cursor:
+            result = _fetch_page(cursor, sql, params, 'format', limit, offset)
+        if not read_only:
+            conn.commit()
+        return result
+    finally:
+        conn.close()
+
+
+def _run_clickhouse(details, sql, params, limit, offset, read_only):
+    from clickhouse_driver import Client
+    client = Client(connect_timeout=config.CONNECT_TIMEOUT, **_connect_args(details))
+    try:
+        sql, args, sliced = _prepare(sql, params, 'pyformat', limit, offset)
+        result = client.execute(sql, args, with_column_types=True,
+                                settings={'readonly': 1} if read_only else None)
+        # Statements without a result set (DDL, INSERT) may not return a (rows, types) pair.
+        rows, column_types = result if isinstance(result, tuple) else ([], [])
+        if not sliced:
+            rows = rows[offset:offset + limit + 1]
+        return [c[0] for c in column_types or []], rows
+    finally:
+        client.disconnect()
+
+
+def _run_sqlite(details, sql, params, limit, offset, read_only):
+    path = details.get('database')
+    if not path:
+        raise ApiError('Database file path not provided')
+    if not os.path.isabs(path):
+        path = os.path.join(config.home(), path)
+    if not os.path.isfile(path):
+        raise ApiError('SQLite database file not found', 404)
+    if read_only:
+        conn = sqlite3.connect(f'{Path(path).as_uri()}?mode=ro', uri=True, timeout=config.CONNECT_TIMEOUT)
+    else:
+        conn = sqlite3.connect(path, timeout=config.CONNECT_TIMEOUT)
+    try:
+        cursor = conn.cursor()
+        result = _fetch_page(cursor, sql, params, 'qmark', limit, offset)
+        if not read_only:
+            conn.commit()
+        return result
+    finally:
+        conn.close()
+
+
+def _run_h2(details, sql, params, limit, offset, read_only):
+    import jaydebeapi
+    host = details.get('host') or 'localhost'
+    if details.get('port') and ':' not in host:
+        host = f"{host}:{details['port']}"
+    url = f"jdbc:h2:tcp://{host}/~/{details.get('database')}"
+    conn = jaydebeapi.connect('org.h2.Driver', url, [details.get('user'), details.get('password')],
+                              [config.h2_jar()])
+    try:
+        # Unlike the other drivers, H2's JDBC setReadOnly() is only a hint and does not block writes,
+        # so here the read-only guarantee rests on validate_sql() alone.
+        cursor = conn.cursor()
+        result = _fetch_page(cursor, sql, params, 'qmark', limit, offset)
+        if not read_only:
+            conn.commit()
+        return result
+    finally:
+        conn.close()
+
+
+RUNNERS = {
+    'mysql': _run_mysql,
+    'postgres': _run_postgres,
+    'clickhouse': _run_clickhouse,
+    'sqlite': _run_sqlite,
+    'h2': _run_h2,
+}

sql2api/sqltools.py

@@ -0,0 +1,156 @@
+"""Pure SQL text handling: validation, pagination, and parameter substitution/binding."""
+import math
+import re
+
+from . import config
+from .errors import ApiError
+
+# Quoted strings, quoted identifiers and comments - text inside these is never inspected.
+_LITERAL = r"'(?:[^']|'')*'|\"(?:[^\"]|\"\")*\"|`[^`]*`|--[^\n]*|/\*.*?\*/"
+_LITERALS_RE = re.compile(_LITERAL, re.S)
+# A ":name" marker, skipping literals and Postgres "::" casts.
+_PARAM_RE = re.compile(rf'(?P<skip>{_LITERAL}|::)|:(?P<name>[A-Za-z_]\w*)', re.S)
+_BRACE_RE = re.compile(r'\{(\w+)\}')
+_TRAILING_LIMIT_RE = re.compile(r'\s+LIMIT\s+\d+(?:\s*,\s*\d+|\s+OFFSET\s+\d+)?\s*$', re.I)
+_SAFE_TEXT_RE = re.compile(r'^[\w\s.,:@%+/\-]*$')
+
+READ_ONLY_STATEMENTS = ('select', 'with', 'show', 'describe', 'desc', 'explain', 'values', 'table')
+PAGINATED_STATEMENTS = ('select', 'with')
+
+
+def first_keyword(sql):
+    match = re.match(r'[\s(]*([A-Za-z]+)', _LITERALS_RE.sub(' ', sql))
+    return match.group(1).lower() if match else ''
+
+
+def is_paginated(sql):
+    return first_keyword(sql) in PAGINATED_STATEMENTS
+
+
+def validate_sql(sql):
+    """Normalise a client-supplied statement and enforce the single-statement/read-only rules."""
+    if not isinstance(sql, str) or not sql.strip():
+        raise ApiError('SQL query is missing')
+    sql = re.sub(r'[\s;]+$', '', sql.strip())
+    if ';' in _LITERALS_RE.sub(' ', sql):
+        raise ApiError('Only a single SQL statement can be executed at a time')
+    if not config.allow_writes():
+        if first_keyword(sql) not in READ_ONLY_STATEMENTS or '/*!' in sql:
+            raise ApiError('Only read-only statements (SELECT, WITH, SHOW, DESCRIBE, EXPLAIN) are allowed. '
+                           'Set SQL2API_ALLOW_WRITES=1 to lift this restriction.', 403)
+    return sql
+
+
+def paginate(sql, limit, offset):
+    """Replace any trailing LIMIT/OFFSET on a SELECT with the requested window."""
+    sql = _TRAILING_LIMIT_RE.sub('', sql)
+    return f'{sql}\nLIMIT {int(limit)} OFFSET {int(offset)}'
+
+
+# --------------------------------------------------------------------------------------
+# Parameters
+# --------------------------------------------------------------------------------------
+
+def _to_bool(text):
+    lowered = text.strip().lower()
+    if lowered in ('true', '1', 'yes'):
+        return True
+    if lowered in ('false', '0', 'no'):
+        return False
+    raise ValueError(text)
+
+
+_COERCERS = {'int': int, 'integer': int, 'float': float, 'number': float, 'str': str, 'string': str,
+             'bool': _to_bool, 'boolean': _to_bool}
+
+
+def coerce_params(declared, params):
+    """Convert string values (e.g. from a query string) to the types a saved query declares.
+
+    ``declared`` is the saved version's ``query_parameters``: ``{"actor_id": "int"}`` or
+    ``{"actor_id": {"type": "int"}}``. Undeclared parameters are left untouched.
+    """
+    result = dict(params)
+    for name, spec in (declared or {}).items():
+        type_name = spec.get('type') if isinstance(spec, dict) else spec
+        coerce = _COERCERS.get(str(type_name).lower())
+        if coerce and isinstance(result.get(name), str):
+            try:
+                result[name] = coerce(result[name])
+            except ValueError:
+                raise ApiError(f"Parameter '{name}' must be of type {type_name}") from None
+    return result
+
+
+def _check_value(name, value):
+    if value is None or isinstance(value, (bool, str)):
+        return
+    if isinstance(value, (int, float)) and math.isfinite(value):
+        return
+    raise ApiError(f"Parameter '{name}' must be a string, finite number, boolean or null")
+
+
+def fill_placeholders(sql, values):
+    """Substitute ``{name}`` text placeholders (legacy style; prefer bound ``:name`` parameters).
+
+    Because the value becomes part of the SQL text it is restricted to numbers, booleans and
+    plain text so it cannot break out of the query. Only placeholders present in the SQL are
+    validated, so the same dict can also carry values for bound parameters.
+    """
+    for key, value in values.items():
+        token = f'{{{key}}}'
+        if token not in sql:
+            continue
+        if isinstance(value, bool):
+            text = 'TRUE' if value else 'FALSE'
+        elif isinstance(value, (int, float)):
+            if not math.isfinite(value):
+                raise ApiError(f"Placeholder '{key}' must be a finite number")
+            text = str(value)
+        elif isinstance(value, str) and _SAFE_TEXT_RE.match(value) and '--' not in value:
+            text = value
+        else:
+            raise ApiError(f"Placeholder '{key}' is substituted into the SQL text, so it must be a number, "
+                           "boolean or a string containing only letters, digits, whitespace and . , : @ % + / - "
+                           f"(use a bound :{key} parameter for arbitrary text)")
+        sql = sql.replace(token, text)
+    missing = sorted(set(_BRACE_RE.findall(sql)))
+    if missing:
+        raise ApiError(f"No value provided for placeholder(s): {', '.join(missing)}")
+    return sql
+
+
+def named_parameters(sql):
+    """Names of the ``:name`` bound parameters used by ``sql``, in order of appearance."""
+    return [m.group('name') for m in _PARAM_RE.finditer(sql) if m.group('name')]
+
+
+_MARKERS = {'qmark': lambda name: '?', 'format': lambda name: '%s', 'pyformat': lambda name: f'%({name})s'}
+
+
+def bind_parameters(sql, params, style):
+    """Rewrite ``:name`` markers into a driver's paramstyle and return ``(sql, args)``.
+
+    ``style`` is 'qmark' (``?``, positional), 'format' (``%s``, positional) or 'pyformat'
+    (``%(name)s``, mapping). Values never become part of the SQL text. Returns ``(sql, None)``
+    when the statement has no parameters, so drivers do not apply ``%`` formatting to it.
+    """
+    matches = [m for m in _PARAM_RE.finditer(sql) if m.group('name')]
+    if not matches:
+        return sql, None
+    names = [m.group('name') for m in matches]
+    missing = sorted(set(names) - set(params))
+    if missing:
+        raise ApiError(f"No value provided for parameter(s): {', '.join(missing)}")
+    for name in set(names):
+        _check_value(name, params[name])
+
+    escape = (lambda text: text.replace('%', '%%')) if style in ('format', 'pyformat') else (lambda text: text)
+    pieces, position = [], 0
+    for match in matches:
+        pieces.append(escape(sql[position:match.start()]))
+        pieces.append(_MARKERS[style](match.group('name')))
+        position = match.end()
+    pieces.append(escape(sql[position:]))
+    args = {name: params[name] for name in names} if style == 'pyformat' else [params[name] for name in names]
+    return ''.join(pieces), args