sql2api 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

sql2api/__init__.py

@@ -1,5 +1,5 @@
 """SQL2API - expose SQL databases as a REST API."""
-__version__ = '0.1.0'
+__version__ = '0.2.0'
 
 from .app import create_app  # noqa: E402  (app imports __version__)
 

sql2api/app.py

@@ -1,12 +1,13 @@
 """The Flask application: HTTP routes on top of the store, engine and formatters."""
 import hmac
 import logging
+import math
 
 from flask import Blueprint, Flask, Response, jsonify, redirect, request, url_for
 from flask.json.provider import DefaultJSONProvider
 from werkzeug.exceptions import HTTPException
 
-from . import config, engine, openapi, sqltools, store
+from . import config, engine, openapi, pool, sqltools, store
 from .errors import ApiError
 from .formats import FORMATTERS, json_default
 
@@ -14,7 +15,7 @@ log = logging.getLogger('sql2api')
 bp = Blueprint('api', __name__)
 
 # Query-string arguments that control a request rather than supplying query parameters.
-RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version'}
+RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version', 'timeout'}
 PUBLIC_ENDPOINTS = {'api.index', 'api.favicon', 'api.health', 'api.docs', 'api.openapi_spec'}
 
 
@@ -105,6 +106,20 @@ def get_output_format(body=None):
     return output_format
 
 
+def get_timeout(body=None):
+    """Seconds allowed for the query: ?timeout= may lower the server limit but never raise it."""
+    raw = request.args.get('timeout', (body or {}).get('timeout'))
+    if raw in (None, ''):
+        return config.effective_timeout(None)
+    try:
+        value = float(raw)
+    except (TypeError, ValueError):
+        raise ApiError('timeout must be a number of seconds') from None
+    if not math.isfinite(value) or value <= 0:
+        raise ApiError('timeout must be a positive number of seconds')
+    return config.effective_timeout(value)
+
+
 def render(result, output_format, page, page_size):
     response = jsonify({'message': 'No results returned'}) if not result else FORMATTERS[output_format](result)
     response.headers['X-Page'] = str(page)
@@ -126,8 +141,9 @@ def execute_sql_endpoint():
         raise ApiError('Connection name is missing')
     params = get_object(data.get('params'), 'params')
     output_format = get_output_format(data)
+    timeout = get_timeout(data)
     limit, offset, page = get_pagination()
-    result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params)
+    result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params, timeout)
     return render(result, output_format, page, limit)
 
 
@@ -148,11 +164,12 @@ def run_saved(ref, body, url_params):
     params = sqltools.coerce_params(saved.get('query_parameters'), raw)
     sql = sqltools.fill_placeholders(saved['sql_query'], params)
     output_format = get_output_format(body)
+    timeout = get_timeout(body)
     limit, offset, page = get_pagination()
 
     entry = {'executed_at': store.now(), 'connection_name': connection_name}
     try:
-        result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset, params)
+        result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset, params, timeout)
     except ApiError as error:
         store.record_execution(path, number, {**entry, 'status': 'error', 'error': error.message})
         raise
@@ -256,12 +273,14 @@ def update_connections():
     if not connections or not isinstance(connections, dict):
         raise ApiError('Connections data is missing')
     store.update_connections(connections)
+    pool.close_pooled_connections()  # new settings or credentials must not be served by old connections
     return jsonify({'message': 'Connections updated successfully'}), 200
 
 
 @bp.route('/connections/<name>', methods=['DELETE'])
 def delete_connection(name):
     store.delete_connection(name)
+    pool.close_pooled_connections()  # a removed connection must not keep serving from idle sockets
     return jsonify({'message': f"Connection '{name}' deleted"}), 200
 
 

sql2api/config.py

@@ -6,6 +6,9 @@ SUPPORTED_DB_TYPES = ('mysql', 'postgres', 'clickhouse', 'sqlite', 'h2')
 PASSWORD_MASK = '********'
 CONNECT_TIMEOUT = 10  # seconds
 HISTORY_LIMIT = 50  # executions remembered per saved-query version
+DEFAULT_QUERY_TIMEOUT = 30.0  # seconds
+DEFAULT_POOL_SIZE = 5  # idle connections kept per distinct connection
+DEFAULT_POOL_IDLE_TIMEOUT = 300.0  # seconds
 
 # Ships with the package; override with SQL2API_H2_JAR to use a different H2 version.
 BUNDLED_H2_JAR = Path(__file__).parent / 'lib' / 'h2-2.2.224.jar'
@@ -55,6 +58,42 @@ def api_key():
     return os.environ.get('SQL2API_API_KEY') or None
 
 
+def query_timeout():
+    """Server-wide statement time limit in seconds (SQL2API_QUERY_TIMEOUT); None when disabled (set to 0)."""
+    try:
+        value = float(os.environ.get('SQL2API_QUERY_TIMEOUT', DEFAULT_QUERY_TIMEOUT))
+    except ValueError:
+        return DEFAULT_QUERY_TIMEOUT
+    if value == 0:
+        return None
+    return value if value > 0 else DEFAULT_QUERY_TIMEOUT
+
+
+def effective_timeout(requested=None):
+    """The limit to enforce: a request may ask for less time than the server allows, never more."""
+    limit = query_timeout()
+    if requested is None:
+        return limit
+    return requested if limit is None else min(requested, limit)
+
+
+def pool_size():
+    """Idle connections kept per distinct connection (SQL2API_POOL_SIZE); 0 disables pooling."""
+    try:
+        return max(0, int(os.environ.get('SQL2API_POOL_SIZE', DEFAULT_POOL_SIZE)))
+    except ValueError:
+        return DEFAULT_POOL_SIZE
+
+
+def pool_idle_timeout():
+    """Seconds an idle pooled connection is kept before it is closed (SQL2API_POOL_IDLE_TIMEOUT)."""
+    try:
+        value = float(os.environ.get('SQL2API_POOL_IDLE_TIMEOUT', DEFAULT_POOL_IDLE_TIMEOUT))
+    except ValueError:
+        return DEFAULT_POOL_IDLE_TIMEOUT
+    return value if value > 0 else DEFAULT_POOL_IDLE_TIMEOUT
+
+
 def max_page_size():
     try:
         return max(1, int(os.environ.get('SQL2API_MAX_PAGE_SIZE', 1000)))

sql2api/engine.py

@@ -5,19 +5,22 @@ import time
 from . import config, store
 from .errors import ApiError
 from .formats import ResultSetDTO
+from .pool import get_pool
 from .runners import RUNNERS
 from .sqltools import validate_sql
 
 log = logging.getLogger('sql2api')
 
 
-def execute_sql(sql, connection_name, limit, offset, params=None):
+def execute_sql(sql, connection_name, limit, offset, params=None, timeout=None):
     """Run ``sql`` on a named connection and return the requested page as a ResultSetDTO."""
     details = store.get_connection(connection_name)
     sql = validate_sql(sql)
-    log.info('Executing on %s (%s), limit=%s offset=%s: %s', connection_name, details['db'], limit, offset, sql)
+    log.info('Executing on %s (%s), limit=%s offset=%s timeout=%s: %s',
+             connection_name, details['db'], limit, offset, timeout, sql)
     try:
-        columns, rows = RUNNERS[details['db']](details, sql, params, limit, offset, not config.allow_writes())
+        columns, rows = RUNNERS[details['db']](details, sql, params, limit, offset, not config.allow_writes(), timeout,
+                                             get_pool())
     except ApiError:
         raise
     except ImportError as error:

sql2api/openapi.py

@@ -7,12 +7,15 @@ _PAGE_PARAMS = [
     {'name': 'page', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 1}},
     {'name': 'page_size', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 10},
      'description': 'Upper limit is SQL2API_MAX_PAGE_SIZE (default 1000).'},
+    {'name': 'timeout', 'in': 'query', 'schema': {'type': 'number', 'exclusiveMinimum': 0},
+     'description': 'Seconds the query may run before it is cancelled (504). Can lower, never raise, the server '
+                    'limit SQL2API_QUERY_TIMEOUT (default 30; 0 disables the server limit).'},
 ]
 _ROWS = {'description': 'The requested page. Header X-Has-More says whether another page follows.',
          'headers': {'X-Page': {'schema': {'type': 'integer'}}, 'X-Page-Size': {'schema': {'type': 'integer'}},
                      'X-Has-More': {'schema': {'type': 'string', 'enum': ['true', 'false']}}},
          'content': {'application/json': {'schema': {'type': 'array', 'items': {'type': 'object'}}}}}
-_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '500')}
+_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '500', '504')}
 
 
 def _body(properties, required):

sql2api/pool.py

@@ -0,0 +1,146 @@
+"""A small thread-safe pool of live database connections.
+
+Only *idle* connections are pooled: a connection is checked out exclusively by one request, and on the way
+back it is reset (its transaction ended) and kept for reuse - unless the request failed, in which case its
+state is unknown and it is closed instead. The number of concurrent connections is not limited here; the
+pool only bounds how many idle ones are kept around.
+"""
+import atexit
+import hashlib
+import json
+import logging
+import threading
+import time
+from collections import deque
+from contextlib import contextmanager
+
+from . import config
+
+log = logging.getLogger('sql2api')
+
+# A connection that has been idle for less than this is trusted without a round trip to check it.
+VALIDATE_AFTER = 5.0  # seconds
+
+
+class Session:
+    """A connection plus what the driver wants to remember about it (e.g. limits already applied)."""
+
+    def __init__(self, conn, driver):
+        self.conn = conn
+        self.driver = driver
+        self.state = {}
+        self.idle_since = time.monotonic()
+
+    def close(self):
+        try:
+            self.driver.close(self)
+        except Exception:  # a connection that cannot be closed cleanly is gone anyway
+            log.debug('Error while closing a connection', exc_info=True)
+
+
+def fingerprint(details, read_only):
+    """Pool key: connections are interchangeable when their settings and read-only mode are identical."""
+    blob = json.dumps(details, sort_keys=True, default=str)
+    return hashlib.sha256(blob.encode('utf-8')).hexdigest(), bool(read_only)
+
+
+class ConnectionPool:
+    def __init__(self):
+        self._idle = {}  # key -> deque of Sessions, least recently used on the left
+        self._lock = threading.Lock()
+
+    @contextmanager
+    def checkout(self, driver, details, read_only):
+        key = fingerprint(details, read_only)
+        session = self._acquire(key)
+        if session is None:
+            session = Session(driver.connect(details, read_only), driver)
+        try:
+            yield session
+        except BaseException:
+            session.close()  # after an error its state is unknown - never reuse it
+            raise
+        self._release(key, session)
+
+    # ---- internals ------------------------------------------------------------------
+
+    def _acquire(self, key):
+        while True:
+            with self._lock:
+                queue = self._idle.get(key)
+                if not queue:
+                    return None
+                session = queue.pop()  # most recently used first, so spare connections age out
+                if not queue:
+                    del self._idle[key]
+            idle_for = time.monotonic() - session.idle_since
+            if idle_for > config.pool_idle_timeout():
+                session.close()
+                continue
+            if idle_for > VALIDATE_AFTER:
+                try:
+                    alive = session.driver.is_alive(session)
+                except Exception:
+                    alive = False
+                if not alive:
+                    log.info('Discarding a dead pooled connection')
+                    session.close()
+                    continue
+            return session
+
+    def _release(self, key, session):
+        try:
+            session.driver.reset(session)  # ends the transaction so the next user sees fresh data
+        except Exception:
+            log.debug('Could not reset a connection; closing it', exc_info=True)
+            session.close()
+            return
+        session.idle_since = time.monotonic()
+        surplus = []
+        with self._lock:
+            queue = self._idle.setdefault(key, deque())
+            queue.append(session)
+            while len(queue) > config.pool_size():
+                surplus.append(queue.popleft())
+            surplus.extend(self._expired_locked())
+        for old in surplus:
+            old.close()
+
+    def _expired_locked(self):
+        """Remove and return idle sessions past the idle timeout, across every key (caller holds the lock)."""
+        cutoff = time.monotonic() - config.pool_idle_timeout()
+        expired = []
+        for key in list(self._idle):
+            queue = self._idle[key]
+            while queue and queue[0].idle_since < cutoff:
+                expired.append(queue.popleft())
+            if not queue:
+                del self._idle[key]
+        return expired
+
+    # ---- housekeeping ---------------------------------------------------------------
+
+    def idle_count(self):
+        with self._lock:
+            return sum(len(q) for q in self._idle.values())
+
+    def close_all(self):
+        with self._lock:
+            sessions = [s for queue in self._idle.values() for s in queue]
+            self._idle.clear()
+        for session in sessions:
+            session.close()
+
+
+_default = ConnectionPool()
+atexit.register(_default.close_all)
+
+
+def close_pooled_connections():
+    """Close every idle connection in the shared pool (also runs at interpreter exit)."""
+    _default.close_all()
+
+
+def get_pool():
+    """The shared pool, or None when pooling is disabled (SQL2API_POOL_SIZE=0)."""
+    return _default if config.pool_size() > 0 else None

sql2api/runners.py

@@ -1,16 +1,35 @@
-"""One runner per database type. Each connects, runs one statement, closes, and returns (columns, rows).
+"""One runner per database type: connect (or borrow a pooled connection), run one statement, return (columns, rows).
 
 Runners fetch ``limit + 1`` rows: the extra row is how the caller learns whether another page exists.
 Database drivers are imported lazily so only the ones you actually use need to be installed.
+
+Each network database is a small ``_Driver`` subclass. ``_make_runner`` turns it into the callable stored in
+``RUNNERS``, which either opens a connection for the single call or borrows one from the pool.
 """
+import atexit
+import logging
+import math
 import os
 import sqlite3
+import time
 from pathlib import Path
 
 from . import config
 from .errors import ApiError
+from .pool import Session, close_pooled_connections
 from .sqltools import bind_parameters, is_paginated, paginate
 
+log = logging.getLogger('sql2api')
+
+
+def _timed_out(timeout):
+    return ApiError(f'The query exceeded the time limit of {timeout:g} seconds and was cancelled', 504,
+                    timeout=timeout)
+
+
+def _millis(timeout):
+    return max(1, int(timeout * 1000))
+
 
 def _connect_args(details, **renames):
     """Pick the standard connection fields out of a stored connection, renaming keys per driver."""
@@ -45,53 +64,250 @@ def _fetch_page(cursor, sql, params, style, limit, offset):
     return [d[0] for d in cursor.description], rows
 
 
-def _run_mysql(details, sql, params, limit, offset, read_only):
-    import mysql.connector
-    conn = mysql.connector.connect(connection_timeout=config.CONNECT_TIMEOUT, **_connect_args(details))
-    try:
-        cursor = conn.cursor()
+# --------------------------------------------------------------------------------------
+# Drivers
+# --------------------------------------------------------------------------------------
+
+class _Driver:
+    """How to connect to, check, reset, query and close one kind of database."""
+
+    def connect(self, details, read_only):
+        raise NotImplementedError
+
+    def is_alive(self, session):
+        """Cheap check that a connection that sat idle is still usable."""
+        return True
+
+    def reset(self, session):
+        """Put a used connection back into a clean state: end its transaction so no stale snapshot lingers."""
+
+    def close(self, session):
+        session.conn.close()
+
+    def query(self, session, sql, params, limit, offset, read_only, timeout):
+        raise NotImplementedError
+
+
+# MySQL reports 3024 (ER_QUERY_TIMEOUT), MariaDB 1969 (ER_STATEMENT_TIMEOUT)
+_MYSQL_TIMEOUT_ERRNOS = (3024, 1969)
+
+
+class _MySQL(_Driver):
+    def connect(self, details, read_only):
+        import mysql.connector
+        conn = mysql.connector.connect(connection_timeout=config.CONNECT_TIMEOUT, **_connect_args(details))
         if read_only:
-            cursor.execute('SET SESSION TRANSACTION READ ONLY')
-        result = _fetch_page(cursor, sql, params, 'format', limit, offset)
-        if not read_only:
-            conn.commit()
-        return result
-    finally:
-        conn.close()
+            try:
+                cursor = conn.cursor()
+                cursor.execute('SET SESSION TRANSACTION READ ONLY')
+                cursor.close()
+            except Exception:
+                conn.close()
+                raise
+        return conn
 
+    def is_alive(self, session):
+        return session.conn.is_connected()
 
-def _run_postgres(details, sql, params, limit, offset, read_only):
-    import psycopg2
-    conn = psycopg2.connect(connect_timeout=config.CONNECT_TIMEOUT, **_connect_args(details, database='dbname'))
-    try:
+    def reset(self, session):
+        session.conn.rollback()
+
+    def query(self, session, sql, params, limit, offset, read_only, timeout):
+        import mysql.connector
+        conn = session.conn
+        cursor = conn.cursor()
+        try:
+            if timeout != session.state.get('timeout'):
+                # MySQL limits SELECT statements in milliseconds; MariaDB uses a differently named variable
+                # in seconds. 0 removes a limit that an earlier request applied to this pooled connection.
+                try:
+                    cursor.execute(f'SET SESSION max_execution_time = {_millis(timeout) if timeout else 0}')
+                except mysql.connector.Error:
+                    try:
+                        cursor.execute(f'SET SESSION max_statement_time = {timeout:g}' if timeout
+                                       else 'SET SESSION max_statement_time = 0')
+                    except mysql.connector.Error:
+                        log.warning('This MySQL server supports neither max_execution_time nor '
+                                    'max_statement_time; the query time limit is not enforced')
+                session.state['timeout'] = timeout
+            try:
+                result = _fetch_page(cursor, sql, params, 'format', limit, offset)
+            except mysql.connector.Error as error:
+                if getattr(error, 'errno', None) in _MYSQL_TIMEOUT_ERRNOS:
+                    raise _timed_out(timeout) from None
+                raise
+            if not read_only:
+                conn.commit()
+            return result
+        finally:
+            cursor.close()
+
+
+class _Postgres(_Driver):
+    def connect(self, details, read_only):
+        import psycopg2
+        conn = psycopg2.connect(connect_timeout=config.CONNECT_TIMEOUT, **_connect_args(details, database='dbname'))
         if read_only:
-            conn.set_session(readonly=True)
+            try:
+                conn.set_session(readonly=True)
+            except Exception:
+                conn.close()
+                raise
+        return conn
+
+    def is_alive(self, session):
+        if session.conn.closed:
+            return False
+        with session.conn.cursor() as cursor:
+            cursor.execute('SELECT 1')
+        return True
+
+    def reset(self, session):
+        session.conn.rollback()
+
+    def query(self, session, sql, params, limit, offset, read_only, timeout):
+        import psycopg2.errors
+        conn = session.conn
         with conn.cursor() as cursor:
-            result = _fetch_page(cursor, sql, params, 'format', limit, offset)
+            if timeout:
+                # LOCAL scopes the limit to this transaction, so nothing leaks to the connection's next user
+                cursor.execute(f'SET LOCAL statement_timeout = {_millis(timeout)}')
+            try:
+                result = _fetch_page(cursor, sql, params, 'format', limit, offset)
+            except psycopg2.errors.QueryCanceled:
+                raise _timed_out(timeout) from None
         if not read_only:
             conn.commit()
         return result
-    finally:
-        conn.close()
 
 
-def _run_clickhouse(details, sql, params, limit, offset, read_only):
-    from clickhouse_driver import Client
-    client = Client(connect_timeout=config.CONNECT_TIMEOUT, **_connect_args(details))
-    try:
+_CLICKHOUSE_TIMEOUT_EXCEEDED = 159
+
+
+class _ClickHouse(_Driver):
+    # The native-protocol Client pings before each query and reconnects by itself, so it needs no
+    # is_alive check, and every setting is sent per query, so there is no session state to reset.
+    def connect(self, details, read_only):
+        from clickhouse_driver import Client
+        client_args = {}
+        limit = config.query_timeout()
+        if limit:
+            # Backstop in case the server never answers; the server-side limit below is what normally fires.
+            client_args['send_receive_timeout'] = math.ceil(limit) + 5
+        return Client(connect_timeout=config.CONNECT_TIMEOUT, **client_args, **_connect_args(details))
+
+    def close(self, session):
+        session.conn.disconnect()
+
+    def query(self, session, sql, params, limit, offset, read_only, timeout):
+        from clickhouse_driver.errors import ServerException
         sql, args, sliced = _prepare(sql, params, 'pyformat', limit, offset)
-        result = client.execute(sql, args, with_column_types=True,
-                                settings={'readonly': 1} if read_only else None)
+        settings = {}
+        if timeout:
+            settings['max_execution_time'] = math.ceil(timeout)  # whole seconds
+        if read_only:
+            settings['readonly'] = 1  # last: it forbids changing settings after it
+        try:
+            result = session.conn.execute(sql, args, with_column_types=True, settings=settings or None)
+        except ServerException as error:
+            if error.code == _CLICKHOUSE_TIMEOUT_EXCEEDED:
+                raise _timed_out(timeout) from None
+            raise
         # Statements without a result set (DDL, INSERT) may not return a (rows, types) pair.
         rows, column_types = result if isinstance(result, tuple) else ([], [])
         if not sliced:
             rows = rows[offset:offset + limit + 1]
         return [c[0] for c in column_types or []], rows
-    finally:
-        client.disconnect()
 
 
-def _run_sqlite(details, sql, params, limit, offset, read_only):
+def _attach_thread_as_daemon():
+    """Make the calling thread a *daemon* thread as far as the JVM is concerned.
+
+    jaydebeapi attaches every thread that uses H2 to the JVM as a non-daemon thread, and JPype's shutdown then
+    waits for those threads forever - so a server that has handled concurrent H2 requests would hang on exit.
+    Attaching them as daemons first (jaydebeapi only attaches threads that are not attached yet) avoids that.
+    """
+    import jpype
+    if jpype.isJVMStarted() and not jpype.java.lang.Thread.isAttached():
+        jpype.java.lang.Thread.attachAsDaemon()
+
+
+class _H2(_Driver):
+    _exit_hook_registered = False
+
+    def connect(self, details, read_only):
+        import jaydebeapi
+        _attach_thread_as_daemon()
+        host = details.get('host') or 'localhost'
+        if details.get('port') and ':' not in host:
+            host = f"{host}:{details['port']}"
+        url = f"jdbc:h2:tcp://{host}/~/{details.get('database')}"
+        # Unlike the other drivers, H2's JDBC setReadOnly() is only a hint and does not block writes,
+        # so here the read-only guarantee rests on validate_sql() alone.
+        conn = jaydebeapi.connect('org.h2.Driver', url, [details.get('user'), details.get('password')],
+                                  [config.h2_jar()])
+        if not _H2._exit_hook_registered:
+            # JPype shuts the JVM down from its own atexit hook, which it registers when the JVM starts (just
+            # now). Hooks run last-in first-out, so registering ours after that makes pooled connections close
+            # while the JVM is still alive - closing them afterwards would hang the interpreter at exit.
+            atexit.register(close_pooled_connections)
+            _H2._exit_hook_registered = True
+        return conn
+
+    def is_alive(self, session):
+        _attach_thread_as_daemon()
+        return bool(session.conn.jconn.isValid(2))
+
+    def reset(self, session):
+        _attach_thread_as_daemon()
+        session.conn.rollback()
+
+    def close(self, session):
+        _attach_thread_as_daemon()
+        session.conn.close()
+
+    def query(self, session, sql, params, limit, offset, read_only, timeout):
+        import jaydebeapi
+        _attach_thread_as_daemon()
+        conn = session.conn
+        cursor = conn.cursor()
+        if timeout != session.state.get('timeout'):
+            cursor.execute(f'SET QUERY_TIMEOUT {_millis(timeout) if timeout else 0}')
+            session.state['timeout'] = timeout
+        try:
+            result = _fetch_page(cursor, sql, params, 'qmark', limit, offset)
+        except jaydebeapi.Error as error:
+            # H2: "Statement was canceled or the session timed out" (error 57014 / 90051)
+            if timeout and any(word in str(error).lower() for word in ('canceled', 'timed out')):
+                raise _timed_out(timeout) from None
+            raise
+        if not read_only:
+            conn.commit()
+        return result
+
+
+def _make_runner(driver):
+    def run(details, sql, params, limit, offset, read_only, timeout=None, pool=None):
+        if pool is None:
+            session = Session(driver.connect(details, read_only), driver)
+            try:
+                return driver.query(session, sql, params, limit, offset, read_only, timeout)
+            finally:
+                session.close()
+        with pool.checkout(driver, details, read_only) as session:
+            return driver.query(session, sql, params, limit, offset, read_only, timeout)
+
+    return run
+
+
+_run_mysql = _make_runner(_MySQL())
+_run_postgres = _make_runner(_Postgres())
+_run_clickhouse = _make_runner(_ClickHouse())
+_run_h2 = _make_runner(_H2())
+
+
+def _run_sqlite(details, sql, params, limit, offset, read_only, timeout=None, pool=None):
+    """SQLite is a local file, so opening it per call is cheap and it is never pooled."""
     path = details.get('database')
     if not path:
         raise ApiError('Database file path not provided')
@@ -104,28 +320,24 @@ def _run_sqlite(details, sql, params, limit, offset, read_only):
     else:
         conn = sqlite3.connect(path, timeout=config.CONNECT_TIMEOUT)
     try:
-        cursor = conn.cursor()
-        result = _fetch_page(cursor, sql, params, 'qmark', limit, offset)
-        if not read_only:
-            conn.commit()
-        return result
-    finally:
-        conn.close()
+        expired = []
+        if timeout:
+            deadline = time.monotonic() + timeout
 
+            def check_deadline():
+                if time.monotonic() > deadline:
+                    expired.append(True)
+                    return 1  # non-zero aborts the running statement
+                return 0
 
-def _run_h2(details, sql, params, limit, offset, read_only):
-    import jaydebeapi
-    host = details.get('host') or 'localhost'
-    if details.get('port') and ':' not in host:
-        host = f"{host}:{details['port']}"
-    url = f"jdbc:h2:tcp://{host}/~/{details.get('database')}"
-    conn = jaydebeapi.connect('org.h2.Driver', url, [details.get('user'), details.get('password')],
-                              [config.h2_jar()])
-    try:
-        # Unlike the other drivers, H2's JDBC setReadOnly() is only a hint and does not block writes,
-        # so here the read-only guarantee rests on validate_sql() alone.
+            conn.set_progress_handler(check_deadline, 10000)  # called every 10 000 VM instructions
         cursor = conn.cursor()
-        result = _fetch_page(cursor, sql, params, 'qmark', limit, offset)
+        try:
+            result = _fetch_page(cursor, sql, params, 'qmark', limit, offset)
+        except sqlite3.OperationalError:
+            if expired:
+                raise _timed_out(timeout) from None
+            raise
         if not read_only:
             conn.commit()
         return result

{sql2api-0.1.0.dist-info → sql2api-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql2api
-Version: 0.1.0
+Version: 0.2.0
 Summary: Turn SQL into a REST API: run queries against MySQL, PostgreSQL, ClickHouse, SQLite or H2 over HTTP and get JSON, CSV, XML, YAML or XLSX back.
 Author-email: Anantha Raju C <arcswdev@gmail.com>
 License: MIT
@@ -75,6 +75,10 @@ film_id,title,rating,length
 - **Bound parameters** - write `WHERE id = :id` and the value is sent to the database separately from the SQL, so it
   cannot inject anything. Declare types (`{"id": "int"}`) and query-string values are converted for you.
 - **Pagination** - `?page=2&page_size=50`, with `X-Has-More` telling you whether another page exists.
+- **Connection pooling** - MySQL, PostgreSQL, ClickHouse and H2 connections are reused between requests instead of
+  opened for each one (about 30x lower per-request overhead on MySQL and H2 against a local server; more over a network).
+- **Query time limit** - runaway queries are cancelled on the database (30 s by default, `?timeout=` per request) so
+  they cannot tie up the service.
 - **Read-only by default** - only single `SELECT`/`WITH`/`SHOW`/`DESCRIBE`/`EXPLAIN` statements run, and sessions are
   opened read-only where the database supports it.
 - **Secrets stay out of files** - `"password": "${PG_PASSWORD}"` in `db_connections.json` reads the environment.
@@ -136,6 +140,9 @@ Everything is configured through environment variables (all optional):
 | `SQL2API_ALLOW_WRITES` | off | Allow `INSERT`/`UPDATE`/DDL. Otherwise only single read-only statements are accepted. |
 | `SQL2API_API_KEY` | unset | When set, every request (except `/health` and `/docs`) needs a matching `X-API-Key` header. |
 | `SQL2API_MAX_PAGE_SIZE` | `1000` | Upper limit for `page_size`. |
+| `SQL2API_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |
+| `SQL2API_POOL_IDLE_TIMEOUT` | `300` | Seconds an idle pooled connection is kept before it is closed. |
+| `SQL2API_QUERY_TIMEOUT` | `30` | Seconds a query may run before it is cancelled (HTTP 504). `0` disables the limit. A request can lower it with `?timeout=`, never raise it. |
 | `SQL2API_HOST` / `SQL2API_PORT` | `127.0.0.1` / `5000` | Bind address for `sql2api serve`. |
 | `SQL2API_DEBUG` | off | Flask debug mode. Never enable on a reachable host. |
 | `SQL2API_H2_JAR` | bundled | Path to a different H2 JDBC jar. |

sql2api-0.2.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+sql2api/__init__.py,sha256=aVq6e6s7KZt7JlbdHpxY4q_3dOZSuzf8wFrLLggZyQc,185
+sql2api/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+sql2api/app.py,sha256=AcYUNu31OJ_aW9_kbXtuYJDAJH5UW3paNuv52UgMWOs,12335
+sql2api/cli.py,sha256=1L3RYiCgnh1w_speXB4RPqxNWs5kR7kT68X9L-aqDLk,2712
+sql2api/config.py,sha256=ACMeLisww_6WmzdbIysmZjBui6QxQLWh1evv8z1l9aw,3663
+sql2api/engine.py,sha256=hys-ArxL4n3WWIsrk8hcERaLB11io_f5Wpr5x9zVBr4,1691
+sql2api/errors.py,sha256=nJddO1gAjxQZH7-IXf3Bt8ruxv3EhVw6t6viFKfluG4,264
+sql2api/formats.py,sha256=SbLDZ-yQIt7pcWXYi8WNYEmG6JNqoeeGeRwIg5XHmos,4817
+sql2api/openapi.py,sha256=O7iRvEic1L2OXAp-P3N1FXpPdpTM1w0rAEShWEcpjfk,8905
+sql2api/pool.py,sha256=hIxkaVkDnjQEV_4f62yq_HtZkCEHiq_qR_fUGyHGhiA,5170
+sql2api/runners.py,sha256=ANYrGluet1HOV_Fi1IvJFZMEj1CFtjKEbQzuksqLi0g,13782
+sql2api/sqltools.py,sha256=HPNC2hCSkBqiJ8O-WYWj7gwILnNOtyVD5A_58SG3lik,6657
+sql2api/store.py,sha256=OHLE1PU7ZrqQcPJec4jJbqNeFBDSt7TOUO8NfH7QLJg,9890
+sql2api/lib/h2-2.2.224.jar,sha256=udjxk1itqCpPbrWxdMbP4yCjdbWpy1pP5FbWI-blVJc,2614933
+sql2api-0.2.0.dist-info/licenses/LICENSE,sha256=ufgSbLEx8zM6J0kFgDoK-6NNb0gqrsFfCb-CnNbZGg0,1071
+sql2api-0.2.0.dist-info/METADATA,sha256=YgkgbcA2O-BWhcS77nr8YzltW3YB8PO9NftPfBoEQmU,10241
+sql2api-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+sql2api-0.2.0.dist-info/entry_points.txt,sha256=63gZjCA84qTkErE2iiuTiGS7GtdamvOqi2elEXwy_zE,45
+sql2api-0.2.0.dist-info/top_level.txt,sha256=dicWZ6j0AE1xBDKr0-qh2CkuP7Yc1CWQf_T6r8LS6ms,8
+sql2api-0.2.0.dist-info/RECORD,,

sql2api-0.1.0.dist-info/RECORD

@@ -1,19 +0,0 @@
-sql2api/__init__.py,sha256=FeKzfwtM7UszmZ7HBQ8AQi_WNHabfDeAIpWARwhqfys,185
-sql2api/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
-sql2api/app.py,sha256=VwoiN9TC2a8wPbCyf3IFMuWsnjteH5scrItLKIgbOoI,11447
-sql2api/cli.py,sha256=1L3RYiCgnh1w_speXB4RPqxNWs5kR7kT68X9L-aqDLk,2712
-sql2api/config.py,sha256=goaJF810BKzyeENR-6lc1l6k1OdSaF4-3CJkpMWv4gU,2222
-sql2api/engine.py,sha256=b4uS0FZAWuUipr_CpAWjyVYPyzTtiDD2HDnfvI1PNP0,1551
-sql2api/errors.py,sha256=nJddO1gAjxQZH7-IXf3Bt8ruxv3EhVw6t6viFKfluG4,264
-sql2api/formats.py,sha256=SbLDZ-yQIt7pcWXYi8WNYEmG6JNqoeeGeRwIg5XHmos,4817
-sql2api/openapi.py,sha256=evoO6KsPN0UjvdxYR0dwHTMygAI-7FIPxhURh7i4rXY,8597
-sql2api/runners.py,sha256=tiP4-zNaD-bSdTycr1OBiHx00A-KESpLXXudhAhwBFw,5218
-sql2api/sqltools.py,sha256=HPNC2hCSkBqiJ8O-WYWj7gwILnNOtyVD5A_58SG3lik,6657
-sql2api/store.py,sha256=OHLE1PU7ZrqQcPJec4jJbqNeFBDSt7TOUO8NfH7QLJg,9890
-sql2api/lib/h2-2.2.224.jar,sha256=udjxk1itqCpPbrWxdMbP4yCjdbWpy1pP5FbWI-blVJc,2614933
-sql2api-0.1.0.dist-info/licenses/LICENSE,sha256=ufgSbLEx8zM6J0kFgDoK-6NNb0gqrsFfCb-CnNbZGg0,1071
-sql2api-0.1.0.dist-info/METADATA,sha256=xrC-ppNP7rufNkYTdi_eFDmkieHx_jEjirZOFsf8F38,9461
-sql2api-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-sql2api-0.1.0.dist-info/entry_points.txt,sha256=63gZjCA84qTkErE2iiuTiGS7GtdamvOqi2elEXwy_zE,45
-sql2api-0.1.0.dist-info/top_level.txt,sha256=dicWZ6j0AE1xBDKr0-qh2CkuP7Yc1CWQf_T6r8LS6ms,8
-sql2api-0.1.0.dist-info/RECORD,,