sql2api 0.1.0__tar.gz → 0.2.0__tar.gz

{sql2api-0.1.0/sql2api.egg-info → sql2api-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql2api
-Version: 0.1.0
+Version: 0.2.0
 Summary: Turn SQL into a REST API: run queries against MySQL, PostgreSQL, ClickHouse, SQLite or H2 over HTTP and get JSON, CSV, XML, YAML or XLSX back.
 Author-email: Anantha Raju C <arcswdev@gmail.com>
 License: MIT
@@ -75,6 +75,10 @@ film_id,title,rating,length
 - **Bound parameters** - write `WHERE id = :id` and the value is sent to the database separately from the SQL, so it
   cannot inject anything. Declare types (`{"id": "int"}`) and query-string values are converted for you.
 - **Pagination** - `?page=2&page_size=50`, with `X-Has-More` telling you whether another page exists.
+- **Connection pooling** - MySQL, PostgreSQL, ClickHouse and H2 connections are reused between requests instead of
+  opened for each one (about 30x lower per-request overhead on MySQL and H2 against a local server; more over a network).
+- **Query time limit** - runaway queries are cancelled on the database (30 s by default, `?timeout=` per request) so
+  they cannot tie up the service.
 - **Read-only by default** - only single `SELECT`/`WITH`/`SHOW`/`DESCRIBE`/`EXPLAIN` statements run, and sessions are
   opened read-only where the database supports it.
 - **Secrets stay out of files** - `"password": "${PG_PASSWORD}"` in `db_connections.json` reads the environment.
@@ -136,6 +140,9 @@ Everything is configured through environment variables (all optional):
 | `SQL2API_ALLOW_WRITES` | off | Allow `INSERT`/`UPDATE`/DDL. Otherwise only single read-only statements are accepted. |
 | `SQL2API_API_KEY` | unset | When set, every request (except `/health` and `/docs`) needs a matching `X-API-Key` header. |
 | `SQL2API_MAX_PAGE_SIZE` | `1000` | Upper limit for `page_size`. |
+| `SQL2API_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |
+| `SQL2API_POOL_IDLE_TIMEOUT` | `300` | Seconds an idle pooled connection is kept before it is closed. |
+| `SQL2API_QUERY_TIMEOUT` | `30` | Seconds a query may run before it is cancelled (HTTP 504). `0` disables the limit. A request can lower it with `?timeout=`, never raise it. |
 | `SQL2API_HOST` / `SQL2API_PORT` | `127.0.0.1` / `5000` | Bind address for `sql2api serve`. |
 | `SQL2API_DEBUG` | off | Flask debug mode. Never enable on a reachable host. |
 | `SQL2API_H2_JAR` | bundled | Path to a different H2 JDBC jar. |

{sql2api-0.1.0 → sql2api-0.2.0}/README.md

@@ -34,6 +34,10 @@ film_id,title,rating,length
 - **Bound parameters** - write `WHERE id = :id` and the value is sent to the database separately from the SQL, so it
   cannot inject anything. Declare types (`{"id": "int"}`) and query-string values are converted for you.
 - **Pagination** - `?page=2&page_size=50`, with `X-Has-More` telling you whether another page exists.
+- **Connection pooling** - MySQL, PostgreSQL, ClickHouse and H2 connections are reused between requests instead of
+  opened for each one (about 30x lower per-request overhead on MySQL and H2 against a local server; more over a network).
+- **Query time limit** - runaway queries are cancelled on the database (30 s by default, `?timeout=` per request) so
+  they cannot tie up the service.
 - **Read-only by default** - only single `SELECT`/`WITH`/`SHOW`/`DESCRIBE`/`EXPLAIN` statements run, and sessions are
   opened read-only where the database supports it.
 - **Secrets stay out of files** - `"password": "${PG_PASSWORD}"` in `db_connections.json` reads the environment.
@@ -95,6 +99,9 @@ Everything is configured through environment variables (all optional):
 | `SQL2API_ALLOW_WRITES` | off | Allow `INSERT`/`UPDATE`/DDL. Otherwise only single read-only statements are accepted. |
 | `SQL2API_API_KEY` | unset | When set, every request (except `/health` and `/docs`) needs a matching `X-API-Key` header. |
 | `SQL2API_MAX_PAGE_SIZE` | `1000` | Upper limit for `page_size`. |
+| `SQL2API_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |
+| `SQL2API_POOL_IDLE_TIMEOUT` | `300` | Seconds an idle pooled connection is kept before it is closed. |
+| `SQL2API_QUERY_TIMEOUT` | `30` | Seconds a query may run before it is cancelled (HTTP 504). `0` disables the limit. A request can lower it with `?timeout=`, never raise it. |
 | `SQL2API_HOST` / `SQL2API_PORT` | `127.0.0.1` / `5000` | Bind address for `sql2api serve`. |
 | `SQL2API_DEBUG` | off | Flask debug mode. Never enable on a reachable host. |
 | `SQL2API_H2_JAR` | bundled | Path to a different H2 JDBC jar. |

{sql2api-0.1.0 → sql2api-0.2.0}/sql2api/__init__.py

@@ -1,5 +1,5 @@
 """SQL2API - expose SQL databases as a REST API."""
-__version__ = '0.1.0'
+__version__ = '0.2.0'
 
 from .app import create_app  # noqa: E402  (app imports __version__)
 

{sql2api-0.1.0 → sql2api-0.2.0}/sql2api/app.py

@@ -1,12 +1,13 @@
 """The Flask application: HTTP routes on top of the store, engine and formatters."""
 import hmac
 import logging
+import math
 
 from flask import Blueprint, Flask, Response, jsonify, redirect, request, url_for
 from flask.json.provider import DefaultJSONProvider
 from werkzeug.exceptions import HTTPException
 
-from . import config, engine, openapi, sqltools, store
+from . import config, engine, openapi, pool, sqltools, store
 from .errors import ApiError
 from .formats import FORMATTERS, json_default
 
@@ -14,7 +15,7 @@ log = logging.getLogger('sql2api')
 bp = Blueprint('api', __name__)
 
 # Query-string arguments that control a request rather than supplying query parameters.
-RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version'}
+RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version', 'timeout'}
 PUBLIC_ENDPOINTS = {'api.index', 'api.favicon', 'api.health', 'api.docs', 'api.openapi_spec'}
 
 
@@ -105,6 +106,20 @@ def get_output_format(body=None):
     return output_format
 
 
+def get_timeout(body=None):
+    """Seconds allowed for the query: ?timeout= may lower the server limit but never raise it."""
+    raw = request.args.get('timeout', (body or {}).get('timeout'))
+    if raw in (None, ''):
+        return config.effective_timeout(None)
+    try:
+        value = float(raw)
+    except (TypeError, ValueError):
+        raise ApiError('timeout must be a number of seconds') from None
+    if not math.isfinite(value) or value <= 0:
+        raise ApiError('timeout must be a positive number of seconds')
+    return config.effective_timeout(value)
+
+
 def render(result, output_format, page, page_size):
     response = jsonify({'message': 'No results returned'}) if not result else FORMATTERS[output_format](result)
     response.headers['X-Page'] = str(page)
@@ -126,8 +141,9 @@ def execute_sql_endpoint():
         raise ApiError('Connection name is missing')
     params = get_object(data.get('params'), 'params')
     output_format = get_output_format(data)
+    timeout = get_timeout(data)
     limit, offset, page = get_pagination()
-    result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params)
+    result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params, timeout)
     return render(result, output_format, page, limit)
 
 
@@ -148,11 +164,12 @@ def run_saved(ref, body, url_params):
     params = sqltools.coerce_params(saved.get('query_parameters'), raw)
     sql = sqltools.fill_placeholders(saved['sql_query'], params)
     output_format = get_output_format(body)
+    timeout = get_timeout(body)
     limit, offset, page = get_pagination()
 
     entry = {'executed_at': store.now(), 'connection_name': connection_name}
     try:
-        result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset, params)
+        result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset, params, timeout)
     except ApiError as error:
         store.record_execution(path, number, {**entry, 'status': 'error', 'error': error.message})
         raise
@@ -256,12 +273,14 @@ def update_connections():
     if not connections or not isinstance(connections, dict):
         raise ApiError('Connections data is missing')
     store.update_connections(connections)
+    pool.close_pooled_connections()  # new settings or credentials must not be served by old connections
     return jsonify({'message': 'Connections updated successfully'}), 200
 
 
 @bp.route('/connections/<name>', methods=['DELETE'])
 def delete_connection(name):
     store.delete_connection(name)
+    pool.close_pooled_connections()  # a removed connection must not keep serving from idle sockets
     return jsonify({'message': f"Connection '{name}' deleted"}), 200
 
 

{sql2api-0.1.0 → sql2api-0.2.0}/sql2api/config.py

@@ -6,6 +6,9 @@ SUPPORTED_DB_TYPES = ('mysql', 'postgres', 'clickhouse', 'sqlite', 'h2')
 PASSWORD_MASK = '********'
 CONNECT_TIMEOUT = 10  # seconds
 HISTORY_LIMIT = 50  # executions remembered per saved-query version
+DEFAULT_QUERY_TIMEOUT = 30.0  # seconds
+DEFAULT_POOL_SIZE = 5  # idle connections kept per distinct connection
+DEFAULT_POOL_IDLE_TIMEOUT = 300.0  # seconds
 
 # Ships with the package; override with SQL2API_H2_JAR to use a different H2 version.
 BUNDLED_H2_JAR = Path(__file__).parent / 'lib' / 'h2-2.2.224.jar'
@@ -55,6 +58,42 @@ def api_key():
     return os.environ.get('SQL2API_API_KEY') or None
 
 
+def query_timeout():
+    """Server-wide statement time limit in seconds (SQL2API_QUERY_TIMEOUT); None when disabled (set to 0)."""
+    try:
+        value = float(os.environ.get('SQL2API_QUERY_TIMEOUT', DEFAULT_QUERY_TIMEOUT))
+    except ValueError:
+        return DEFAULT_QUERY_TIMEOUT
+    if value == 0:
+        return None
+    return value if value > 0 else DEFAULT_QUERY_TIMEOUT
+
+
+def effective_timeout(requested=None):
+    """The limit to enforce: a request may ask for less time than the server allows, never more."""
+    limit = query_timeout()
+    if requested is None:
+        return limit
+    return requested if limit is None else min(requested, limit)
+
+
+def pool_size():
+    """Idle connections kept per distinct connection (SQL2API_POOL_SIZE); 0 disables pooling."""
+    try:
+        return max(0, int(os.environ.get('SQL2API_POOL_SIZE', DEFAULT_POOL_SIZE)))
+    except ValueError:
+        return DEFAULT_POOL_SIZE
+
+
+def pool_idle_timeout():
+    """Seconds an idle pooled connection is kept before it is closed (SQL2API_POOL_IDLE_TIMEOUT)."""
+    try:
+        value = float(os.environ.get('SQL2API_POOL_IDLE_TIMEOUT', DEFAULT_POOL_IDLE_TIMEOUT))
+    except ValueError:
+        return DEFAULT_POOL_IDLE_TIMEOUT
+    return value if value > 0 else DEFAULT_POOL_IDLE_TIMEOUT
+
+
 def max_page_size():
     try:
         return max(1, int(os.environ.get('SQL2API_MAX_PAGE_SIZE', 1000)))

{sql2api-0.1.0 → sql2api-0.2.0}/sql2api/engine.py

@@ -5,19 +5,22 @@ import time
 from . import config, store
 from .errors import ApiError
 from .formats import ResultSetDTO
+from .pool import get_pool
 from .runners import RUNNERS
 from .sqltools import validate_sql
 
 log = logging.getLogger('sql2api')
 
 
-def execute_sql(sql, connection_name, limit, offset, params=None):
+def execute_sql(sql, connection_name, limit, offset, params=None, timeout=None):
     """Run ``sql`` on a named connection and return the requested page as a ResultSetDTO."""
     details = store.get_connection(connection_name)
     sql = validate_sql(sql)
-    log.info('Executing on %s (%s), limit=%s offset=%s: %s', connection_name, details['db'], limit, offset, sql)
+    log.info('Executing on %s (%s), limit=%s offset=%s timeout=%s: %s',
+             connection_name, details['db'], limit, offset, timeout, sql)
     try:
-        columns, rows = RUNNERS[details['db']](details, sql, params, limit, offset, not config.allow_writes())
+        columns, rows = RUNNERS[details['db']](details, sql, params, limit, offset, not config.allow_writes(), timeout,
+                                             get_pool())
     except ApiError:
         raise
     except ImportError as error:

{sql2api-0.1.0 → sql2api-0.2.0}/sql2api/openapi.py

@@ -7,12 +7,15 @@ _PAGE_PARAMS = [
     {'name': 'page', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 1}},
     {'name': 'page_size', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 10},
      'description': 'Upper limit is SQL2API_MAX_PAGE_SIZE (default 1000).'},
+    {'name': 'timeout', 'in': 'query', 'schema': {'type': 'number', 'exclusiveMinimum': 0},
+     'description': 'Seconds the query may run before it is cancelled (504). Can lower, never raise, the server '
+                    'limit SQL2API_QUERY_TIMEOUT (default 30; 0 disables the server limit).'},
 ]
 _ROWS = {'description': 'The requested page. Header X-Has-More says whether another page follows.',
          'headers': {'X-Page': {'schema': {'type': 'integer'}}, 'X-Page-Size': {'schema': {'type': 'integer'}},
                      'X-Has-More': {'schema': {'type': 'string', 'enum': ['true', 'false']}}},
          'content': {'application/json': {'schema': {'type': 'array', 'items': {'type': 'object'}}}}}
-_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '500')}
+_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '500', '504')}
 
 
 def _body(properties, required):

sql2api-0.2.0/sql2api/pool.py

@@ -0,0 +1,146 @@
+"""A small thread-safe pool of live database connections.
+
+Only *idle* connections are pooled: a connection is checked out exclusively by one request, and on the way
+back it is reset (its transaction ended) and kept for reuse - unless the request failed, in which case its
+state is unknown and it is closed instead. The number of concurrent connections is not limited here; the
+pool only bounds how many idle ones are kept around.
+"""
+import atexit
+import hashlib
+import json
+import logging
+import threading
+import time
+from collections import deque
+from contextlib import contextmanager
+
+from . import config
+
+log = logging.getLogger('sql2api')
+
+# A connection that has been idle for less than this is trusted without a round trip to check it.
+VALIDATE_AFTER = 5.0  # seconds
+
+
+class Session:
+    """A connection plus what the driver wants to remember about it (e.g. limits already applied)."""
+
+    def __init__(self, conn, driver):
+        self.conn = conn
+        self.driver = driver
+        self.state = {}
+        self.idle_since = time.monotonic()
+
+    def close(self):
+        try:
+            self.driver.close(self)
+        except Exception:  # a connection that cannot be closed cleanly is gone anyway
+            log.debug('Error while closing a connection', exc_info=True)
+
+
+def fingerprint(details, read_only):
+    """Pool key: connections are interchangeable when their settings and read-only mode are identical."""
+    blob = json.dumps(details, sort_keys=True, default=str)
+    return hashlib.sha256(blob.encode('utf-8')).hexdigest(), bool(read_only)
+
+
+class ConnectionPool:
+    def __init__(self):
+        self._idle = {}  # key -> deque of Sessions, least recently used on the left
+        self._lock = threading.Lock()
+
+    @contextmanager
+    def checkout(self, driver, details, read_only):
+        key = fingerprint(details, read_only)
+        session = self._acquire(key)
+        if session is None:
+            session = Session(driver.connect(details, read_only), driver)
+        try:
+            yield session
+        except BaseException:
+            session.close()  # after an error its state is unknown - never reuse it
+            raise
+        self._release(key, session)
+
+    # ---- internals ------------------------------------------------------------------
+
+    def _acquire(self, key):
+        while True:
+            with self._lock:
+                queue = self._idle.get(key)
+                if not queue:
+                    return None
+                session = queue.pop()  # most recently used first, so spare connections age out
+                if not queue:
+                    del self._idle[key]
+            idle_for = time.monotonic() - session.idle_since
+            if idle_for > config.pool_idle_timeout():
+                session.close()
+                continue
+            if idle_for > VALIDATE_AFTER:
+                try:
+                    alive = session.driver.is_alive(session)
+                except Exception:
+                    alive = False
+                if not alive:
+                    log.info('Discarding a dead pooled connection')
+                    session.close()
+                    continue
+            return session
+
+    def _release(self, key, session):
+        try:
+            session.driver.reset(session)  # ends the transaction so the next user sees fresh data
+        except Exception:
+            log.debug('Could not reset a connection; closing it', exc_info=True)
+            session.close()
+            return
+        session.idle_since = time.monotonic()
+        surplus = []
+        with self._lock:
+            queue = self._idle.setdefault(key, deque())
+            queue.append(session)
+            while len(queue) > config.pool_size():
+                surplus.append(queue.popleft())
+            surplus.extend(self._expired_locked())
+        for old in surplus:
+            old.close()
+
+    def _expired_locked(self):
+        """Remove and return idle sessions past the idle timeout, across every key (caller holds the lock)."""
+        cutoff = time.monotonic() - config.pool_idle_timeout()
+        expired = []
+        for key in list(self._idle):
+            queue = self._idle[key]
+            while queue and queue[0].idle_since < cutoff:
+                expired.append(queue.popleft())
+            if not queue:
+                del self._idle[key]
+        return expired
+
+    # ---- housekeeping ---------------------------------------------------------------
+
+    def idle_count(self):
+        with self._lock:
+            return sum(len(q) for q in self._idle.values())
+
+    def close_all(self):
+        with self._lock:
+            sessions = [s for queue in self._idle.values() for s in queue]
+            self._idle.clear()
+        for session in sessions:
+            session.close()
+
+
+_default = ConnectionPool()
+atexit.register(_default.close_all)
+
+
+def close_pooled_connections():
+    """Close every idle connection in the shared pool (also runs at interpreter exit)."""
+    _default.close_all()
+
+
+def get_pool():
+    """The shared pool, or None when pooling is disabled (SQL2API_POOL_SIZE=0)."""
+    return _default if config.pool_size() > 0 else None