sql2api 0.1.0__py3-none-any.whl

sql2api/__init__.py

@@ -0,0 +1,6 @@
+"""SQL2API - expose SQL databases as a REST API."""
+__version__ = '0.1.0'
+
+from .app import create_app  # noqa: E402  (app imports __version__)
+
+__all__ = ['create_app', '__version__']

sql2api/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

sql2api/app.py

@@ -0,0 +1,296 @@
+"""The Flask application: HTTP routes on top of the store, engine and formatters."""
+import hmac
+import logging
+
+from flask import Blueprint, Flask, Response, jsonify, redirect, request, url_for
+from flask.json.provider import DefaultJSONProvider
+from werkzeug.exceptions import HTTPException
+
+from . import config, engine, openapi, sqltools, store
+from .errors import ApiError
+from .formats import FORMATTERS, json_default
+
+log = logging.getLogger('sql2api')
+bp = Blueprint('api', __name__)
+
+# Query-string arguments that control a request rather than supplying query parameters.
+RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version'}
+PUBLIC_ENDPOINTS = {'api.index', 'api.favicon', 'api.health', 'api.docs', 'api.openapi_spec'}
+
+
+class JSONProvider(DefaultJSONProvider):
+    default = staticmethod(json_default)
+    sort_keys = False  # keep result columns in the order the query returned them
+
+
+def create_app():
+    from . import __version__
+    app = Flask(__name__)
+    app.json = JSONProvider(app)
+    app.config['SQL2API_VERSION'] = __version__
+
+    @app.errorhandler(ApiError)
+    def handle_api_error(error):
+        return jsonify({'error': error.message, **error.extra}), error.status
+
+    @app.errorhandler(HTTPException)
+    def handle_http_error(error):
+        return jsonify({'error': error.description}), error.code
+
+    @app.errorhandler(Exception)
+    def handle_unexpected_error(error):
+        log.exception('Unhandled error')
+        return jsonify({'error': 'An error occurred'}), 500
+
+    @app.before_request
+    def require_api_key():
+        expected = config.api_key()
+        if expected and request.endpoint not in PUBLIC_ENDPOINTS:
+            # compare_digest rejects non-ASCII str, so compare bytes
+            supplied = request.headers.get('X-API-Key', '').encode('utf-8', 'replace')
+            if not hmac.compare_digest(supplied, expected.encode('utf-8')):
+                return jsonify({'error': 'Unauthorized'}), 401
+
+    app.register_blueprint(bp)
+    return app
+
+
+# --------------------------------------------------------------------------------------
+# Request helpers
+# --------------------------------------------------------------------------------------
+
+def get_json_body(required=True):
+    data = request.get_json(silent=True)
+    if data is None and not required:
+        return {}
+    if not isinstance(data, dict):
+        raise ApiError('Request body must be a JSON object')
+    return data
+
+
+def get_object(value, label):
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ApiError(f'{label} must be a JSON object')
+    return value
+
+
+def get_int(value, label):
+    if value is None or value == '':
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        raise ApiError(f'{label} must be an integer') from None
+
+
+def get_pagination():
+    """Return (limit, offset, page) from the ?page and ?page_size query parameters."""
+    page = get_int(request.args.get('page'), 'page')
+    page_size = get_int(request.args.get('page_size'), 'page_size')
+    page = 1 if page is None else page
+    page_size = 10 if page_size is None else page_size
+    if page < 1 or page_size < 1:
+        raise ApiError('page and page_size must be positive')
+    if page_size > config.max_page_size():
+        raise ApiError(f'page_size must not exceed {config.max_page_size()}')
+    return page_size, (page - 1) * page_size, page
+
+
+def get_output_format(body=None):
+    output_format = str(request.args.get('format') or (body or {}).get('format') or 'json').lower()
+    if output_format not in FORMATTERS:
+        raise ApiError(f"Unsupported format '{output_format}'. Supported formats: {', '.join(FORMATTERS)}")
+    return output_format
+
+
+def render(result, output_format, page, page_size):
+    response = jsonify({'message': 'No results returned'}) if not result else FORMATTERS[output_format](result)
+    response.headers['X-Page'] = str(page)
+    response.headers['X-Page-Size'] = str(page_size)
+    response.headers['X-Has-More'] = 'true' if result.has_more else 'false'
+    return response
+
+
+# --------------------------------------------------------------------------------------
+# Query execution
+# --------------------------------------------------------------------------------------
+
+@bp.route('/execute_sql', methods=['POST'])
+def execute_sql_endpoint():
+    data = get_json_body()
+    if not data.get('sql'):
+        raise ApiError('SQL query is missing')
+    if not data.get('connection_name'):
+        raise ApiError('Connection name is missing')
+    params = get_object(data.get('params'), 'params')
+    output_format = get_output_format(data)
+    limit, offset, page = get_pagination()
+    result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params)
+    return render(result, output_format, page, limit)
+
+
+def run_saved(ref, body, url_params):
+    """Execute a saved query (latest version unless one is requested) and record the run."""
+    path = store.resolve_saved_file(ref)
+    number, saved = store.select_version(store.load_versions(path),
+                                         get_int(request.args.get('version') or body.get('version'), 'version'))
+    connection_name = request.args.get('connection_name') or body.get('connection_name') \
+        or saved.get('connection_name')
+    if not connection_name:
+        raise ApiError('Connection name is missing')
+    if not isinstance(saved.get('sql_query'), str):
+        raise ApiError('Saved query has no SQL', 500)
+
+    raw = {**url_params, **get_object(body.get('params'), 'params'),
+           **get_object(body.get('placeholders'), 'placeholders')}
+    params = sqltools.coerce_params(saved.get('query_parameters'), raw)
+    sql = sqltools.fill_placeholders(saved['sql_query'], params)
+    output_format = get_output_format(body)
+    limit, offset, page = get_pagination()
+
+    entry = {'executed_at': store.now(), 'connection_name': connection_name}
+    try:
+        result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset, params)
+    except ApiError as error:
+        store.record_execution(path, number, {**entry, 'status': 'error', 'error': error.message})
+        raise
+    store.record_execution(path, number, {**entry, 'status': 'success', 'rows': len(result.rows),
+                                          'duration_ms': elapsed_ms})
+    return render(result, output_format, page, limit)
+
+
+@bp.route('/execute_sql_from_file', methods=['POST'])
+@bp.route('/execute_sql_with_parameters_from_file', methods=['POST'])
+def execute_sql_from_file():
+    body = get_json_body()
+    return run_saved(body.get('filepath'), body, {})
+
+
+@bp.route('/q/<name>', methods=['GET', 'POST'])
+def run_named_query(name):
+    body = get_json_body(required=False) if request.method == 'POST' else {}
+    url_params = {k: v for k, v in request.args.items() if k not in RESERVED_ARGS}
+    return run_saved(name, body, url_params)
+
+
+# --------------------------------------------------------------------------------------
+# Saved queries
+# --------------------------------------------------------------------------------------
+
+@bp.route('/view_file_content', methods=['GET'])
+def view_file_content():
+    path = store.resolve_saved_file(request.args.get('filename'))
+    with open(path, 'r') as f:
+        return jsonify({'content': f.read()}), 200
+
+
+@bp.route('/save_sql_to_file', methods=['PATCH'])
+def save_sql_to_file():
+    data = get_json_body()
+    for field, label in (('author', 'Author'), ('description', 'Description'),
+                         ('sql_query', 'SQL query'), ('filename', 'Filename')):
+        if not data.get(field) or not isinstance(data[field], str):
+            raise ApiError(f'{label} is missing')
+    tags = data.get('tags', [])
+    if not isinstance(tags, (list, str)):
+        raise ApiError('tags must be a string or a list')
+    query_parameters = get_object(data.get('query_parameters'), 'query_parameters')
+    connection_name = data.get('connection_name')
+    if connection_name is not None and not isinstance(connection_name, str):
+        raise ApiError('connection_name must be a string')
+
+    query_uuid, version = store.save_version(data['filename'], {
+        'sql_query': data['sql_query'],
+        'author': data['author'],
+        'description': data['description'],
+        'tags': tags,
+        'query_parameters': query_parameters,
+        **({'connection_name': connection_name} if connection_name else {}),
+    })
+    return jsonify({'message': 'SQL query saved successfully', 'filename': data['filename'],
+                    'uuid': query_uuid, 'version': version}), 200
+
+
+@bp.route('/saved_sql/<name>', methods=['DELETE'])
+def delete_saved_query(name):
+    version = get_int(request.args.get('version'), 'version')
+    store.delete_saved(name, version)
+    what = f'Version {version} of {name}' if version is not None else name
+    return jsonify({'message': f'{what} deleted'}), 200
+
+
+@bp.route('/list_files', methods=['GET'])
+def list_files():
+    sort_by = request.args.get('sort_by', 'name')
+    if sort_by not in ('name', 'modified'):
+        raise ApiError("sort_by must be 'name' or 'modified'")
+    sort_order = request.args.get('sort_order', 'asc')
+    if sort_order not in ('asc', 'desc'):
+        raise ApiError("sort_order must be 'asc' or 'desc'")
+
+    files = store.list_saved()
+    if sort_by == 'name':
+        def key(f):
+            return f['filename'].lower()
+    else:
+        def key(f):
+            return (f['versions'][-1].get('last_modified_at') or '') if f['versions'] else ''
+    files.sort(key=key, reverse=sort_order == 'desc')
+    return jsonify({'files': files}), 200
+
+
+# --------------------------------------------------------------------------------------
+# Connections
+# --------------------------------------------------------------------------------------
+
+@bp.route('/connections', methods=['GET'])
+def get_connections():
+    return jsonify({'connections': store.mask_passwords(store.read_connections())}), 200
+
+
+@bp.route('/connections', methods=['PATCH'])
+def update_connections():
+    connections = get_json_body().get('connections')
+    if not connections or not isinstance(connections, dict):
+        raise ApiError('Connections data is missing')
+    store.update_connections(connections)
+    return jsonify({'message': 'Connections updated successfully'}), 200
+
+
+@bp.route('/connections/<name>', methods=['DELETE'])
+def delete_connection(name):
+    store.delete_connection(name)
+    return jsonify({'message': f"Connection '{name}' deleted"}), 200
+
+
+# --------------------------------------------------------------------------------------
+# Service endpoints
+# --------------------------------------------------------------------------------------
+
+@bp.route('/', methods=['GET'])
+def index():
+    return redirect(url_for('api.docs'))
+
+
+@bp.route('/favicon.ico', methods=['GET'])
+def favicon():
+    return '', 204
+
+
+@bp.route('/health', methods=['GET'])
+def health():
+    from flask import current_app
+    return jsonify({'status': 'ok', 'version': current_app.config['SQL2API_VERSION']})
+
+
+@bp.route('/openapi.json', methods=['GET'])
+def openapi_spec():
+    from flask import current_app
+    return jsonify(openapi.build_spec(current_app.config['SQL2API_VERSION']))
+
+
+@bp.route('/docs', methods=['GET'])
+def docs():
+    return Response(openapi.DOCS_HTML, mimetype='text/html')

sql2api/cli.py

@@ -0,0 +1,65 @@
+"""Command line entry point: ``sql2api serve`` and ``sql2api init``."""
+import argparse
+import logging
+import os
+
+from . import __version__, config, store
+from .app import create_app
+
+LOOPBACK_HOSTS = ('127.0.0.1', 'localhost', '::1')
+
+
+def _serve(args):
+    app = create_app()
+    if args.host not in LOOPBACK_HOSTS and not config.api_key():
+        logging.getLogger('sql2api').warning(
+            'Listening on %s without SQL2API_API_KEY set: anyone who can reach this port can run SQL '
+            'on your active connections.', args.host)
+    logging.getLogger('sql2api').info('Using %s (connections: %s)', config.home(), config.connections_file().name)
+    app.run(host=args.host, port=args.port, debug=args.debug)
+    return 0
+
+
+def _init(args):
+    home = config.home()
+    home.mkdir(parents=True, exist_ok=True)
+    (home / 'saved_sql').mkdir(exist_ok=True)
+    target = config.connections_file()
+    if target.exists():
+        print(f'{target} already exists - left untouched')
+    else:
+        store.write_json_atomic(str(target), config.EXAMPLE_CONNECTIONS)
+        print(f'Created {target}\nEdit it, set "active": true on the connections you want, then run: sql2api serve')
+    return 0
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(prog='sql2api', description='Expose SQL databases as a REST API.')
+    parser.add_argument('--version', action='version', version=f'sql2api {__version__}')
+    commands = parser.add_subparsers(dest='command')
+
+    serve = commands.add_parser('serve', help='start the HTTP server (default)')
+    serve.add_argument('--host', default=os.environ.get('SQL2API_HOST', '127.0.0.1'))
+    serve.add_argument('--port', type=int, default=int(os.environ.get('SQL2API_PORT', 5000)))
+    serve.add_argument('--debug', action='store_true', default=config.env_flag('SQL2API_DEBUG'),
+                       help='Flask debug mode - never use on a reachable host')
+    serve.set_defaults(func=_serve)
+
+    init = commands.add_parser('init', help='create db_connections.json and saved_sql/ in the home folder')
+    init.set_defaults(func=_init)
+
+    for sub in (serve, init):
+        sub.add_argument('--home', help='folder holding db_connections.json and saved_sql/ '
+                                        '(default: $SQL2API_HOME or the current directory)')
+    return parser
+
+
+def main(argv=None):
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.command is None:
+        args = parser.parse_args(['serve', *(argv or [])])
+    if getattr(args, 'home', None):
+        os.environ['SQL2API_HOME'] = args.home
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s: %(message)s')
+    return args.func(args)

sql2api/config.py

@@ -0,0 +1,62 @@
+"""Runtime configuration, read from environment variables at call time."""
+import os
+from pathlib import Path
+
+SUPPORTED_DB_TYPES = ('mysql', 'postgres', 'clickhouse', 'sqlite', 'h2')
+PASSWORD_MASK = '********'
+CONNECT_TIMEOUT = 10  # seconds
+HISTORY_LIMIT = 50  # executions remembered per saved-query version
+
+# Ships with the package; override with SQL2API_H2_JAR to use a different H2 version.
+BUNDLED_H2_JAR = Path(__file__).parent / 'lib' / 'h2-2.2.224.jar'
+
+# Shown by `sql2api init`. Every entry starts inactive so nothing connects until you opt in.
+EXAMPLE_CONNECTIONS = {
+    'connections': {
+        'example-sqlite': {'db': 'sqlite', 'database': 'example.db', 'active': False},
+        'example-postgres': {'db': 'postgres', 'host': 'localhost', 'port': 5432, 'database': 'postgres',
+                             'user': 'postgres', 'password': '${POSTGRES_PASSWORD}', 'active': False},
+        'example-mysql': {'db': 'mysql', 'host': 'localhost', 'port': 3306, 'database': 'mydb',
+                          'user': 'root', 'password': '${MYSQL_PASSWORD}', 'active': False},
+        'example-clickhouse': {'db': 'clickhouse', 'host': 'localhost', 'port': 9000, 'database': 'default',
+                               'user': 'default', 'password': '', 'active': False},
+        'example-h2': {'db': 'h2', 'host': 'localhost', 'database': 'test', 'user': 'SA', 'password': '',
+                       'active': False},
+    }
+}
+
+
+def env_flag(name):
+    return os.environ.get(name, '').strip().lower() in ('1', 'true', 'yes', 'on')
+
+
+def home():
+    """Folder holding db_connections.json and saved_sql/ (SQL2API_HOME, default: current directory)."""
+    return Path(os.environ.get('SQL2API_HOME') or os.getcwd()).resolve()
+
+
+def connections_file():
+    return home() / 'db_connections.json'
+
+
+def saved_sql_dir():
+    return home() / 'saved_sql'
+
+
+def h2_jar():
+    return os.environ.get('SQL2API_H2_JAR') or str(BUNDLED_H2_JAR)
+
+
+def allow_writes():
+    return env_flag('SQL2API_ALLOW_WRITES')
+
+
+def api_key():
+    return os.environ.get('SQL2API_API_KEY') or None
+
+
+def max_page_size():
+    try:
+        return max(1, int(os.environ.get('SQL2API_MAX_PAGE_SIZE', 1000)))
+    except ValueError:
+        return 1000

sql2api/engine.py

@@ -0,0 +1,37 @@
+"""Ties the pieces together: look up the connection, validate the SQL, run it, wrap the page."""
+import logging
+import time
+
+from . import config, store
+from .errors import ApiError
+from .formats import ResultSetDTO
+from .runners import RUNNERS
+from .sqltools import validate_sql
+
+log = logging.getLogger('sql2api')
+
+
+def execute_sql(sql, connection_name, limit, offset, params=None):
+    """Run ``sql`` on a named connection and return the requested page as a ResultSetDTO."""
+    details = store.get_connection(connection_name)
+    sql = validate_sql(sql)
+    log.info('Executing on %s (%s), limit=%s offset=%s: %s', connection_name, details['db'], limit, offset, sql)
+    try:
+        columns, rows = RUNNERS[details['db']](details, sql, params, limit, offset, not config.allow_writes())
+    except ApiError:
+        raise
+    except ImportError as error:
+        log.exception('Missing database driver')
+        raise ApiError(f"The driver for '{details['db']}' is not installed", 500, detail=str(error)) from error
+    except Exception as error:
+        log.exception('Query on %s failed', connection_name)
+        raise ApiError('An error occurred while executing the SQL query', 500, detail=str(error)) from error
+    has_more = len(rows) > limit
+    return ResultSetDTO(rows[:limit], columns, has_more=has_more)
+
+
+def timed(func, *args, **kwargs):
+    """Call ``func`` and return (result, elapsed milliseconds)."""
+    started = time.monotonic()
+    result = func(*args, **kwargs)
+    return result, round((time.monotonic() - started) * 1000)

sql2api/errors.py

@@ -0,0 +1,8 @@
+class ApiError(Exception):
+    """An error that maps directly onto an HTTP response."""
+
+    def __init__(self, message, status=400, **extra):
+        super().__init__(message)
+        self.message = message
+        self.status = status
+        self.extra = extra

sql2api/formats.py

@@ -0,0 +1,142 @@
+"""Result sets and their output formats (JSON, NDJSON, CSV, TSV, XML, YAML, XLSX)."""
+import csv
+import json
+import re
+import xml.etree.ElementTree as ET
+from datetime import date, datetime, time
+from decimal import Decimal
+from io import BytesIO, StringIO
+
+import yaml
+from flask import Response, jsonify
+from openpyxl import Workbook
+from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
+
+
+def json_default(obj):
+    """``default=`` hook for json.dumps covering the types database drivers return."""
+    if isinstance(obj, datetime):
+        return obj.strftime('%Y-%m-%d %H:%M:%S')
+    if isinstance(obj, (date, time)):
+        return obj.isoformat()
+    if isinstance(obj, Decimal):
+        return float(obj)
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return bytes(obj).decode('utf-8', 'replace')
+    return str(obj)
+
+
+def _unique(names):
+    """Disambiguate repeated column names (e.g. ``SELECT a.id, b.id``) so no data is lost."""
+    seen, result = {}, []
+    for name in names:
+        name = str(name)
+        seen[name] = seen.get(name, 0) + 1
+        result.append(name if seen[name] == 1 else f'{name}_{seen[name]}')
+    return result
+
+
+def _cell(value):
+    """Coerce a value into plain built-in types that CSV, YAML and Excel writers all accept.
+
+    Some drivers (e.g. H2 through JPype) return subclasses of int/float/str, which PyYAML's safe
+    dumper rejects because it only recognises the exact built-in types.
+    """
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return bool(value)
+    if isinstance(value, int):
+        return int(value)
+    if isinstance(value, float):
+        return float(value)
+    if isinstance(value, str):
+        return str(value)
+    if isinstance(value, Decimal):
+        return float(value)
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        return bytes(value).decode('utf-8', 'replace')
+    if isinstance(value, datetime):
+        return value.replace(tzinfo=None) if value.tzinfo else value
+    if isinstance(value, (date, time)):
+        return value
+    return str(value)
+
+
+def _xml_name(name):
+    name = re.sub(r'[^\w.\-]', '_', name)
+    return name if re.match(r'[A-Za-z_]', name) else f'_{name}'
+
+
+class ResultSetDTO:
+    """A page of query results (column names + rows) with output-format converters."""
+
+    def __init__(self, rows, columns, has_more=False):
+        self.columns = _unique(columns)
+        self.rows = [[_cell(v) for v in row] for row in rows]
+        self.has_more = has_more
+
+    def __bool__(self):
+        return bool(self.rows)
+
+    def as_dicts(self):
+        return [dict(zip(self.columns, row)) for row in self.rows]
+
+    def _to_delimited(self, delimiter, mimetype):
+        data_io = StringIO()
+        writer = csv.writer(data_io, delimiter=delimiter)
+        writer.writerow(self.columns)
+        writer.writerows(self.rows)
+        return Response(data_io.getvalue(), mimetype=mimetype)
+
+    def to_csv(self):
+        return self._to_delimited(',', 'text/csv')
+
+    def to_tsv(self):
+        return self._to_delimited('\t', 'text/tab-separated-values')
+
+    def to_json(self):
+        return jsonify(self.as_dicts())
+
+    def to_ndjson(self):
+        lines = (json.dumps(row, default=json_default) for row in self.as_dicts())
+        return Response('\n'.join(lines) + '\n', mimetype='application/x-ndjson')
+
+    def to_xml(self):
+        root = ET.Element('data')
+        tags = [_xml_name(c) for c in self.columns]
+        for row in self.rows:
+            item = ET.SubElement(root, 'item')
+            for tag, value in zip(tags, row):
+                ET.SubElement(item, tag).text = '' if value is None else str(value)
+        return Response(ET.tostring(root, encoding='unicode', method='xml'), mimetype='application/xml')
+
+    def to_yaml(self):
+        return Response(yaml.safe_dump(self.as_dicts(), default_flow_style=False, sort_keys=False,
+                                       allow_unicode=True),
+                        mimetype='application/x-yaml')
+
+    def to_xlsx(self):
+        wb = Workbook()
+        ws = wb.active
+        ws.append(self.columns)
+        for row in self.rows:
+            ws.append([ILLEGAL_CHARACTERS_RE.sub('', v) if isinstance(v, str) else v for v in row])
+        excel_data = BytesIO()
+        wb.save(excel_data)
+        return Response(
+            excel_data.getvalue(),
+            mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            headers={'Content-Disposition': 'attachment;filename=result.xlsx'},
+        )
+
+
+FORMATTERS = {
+    'json': ResultSetDTO.to_json,
+    'ndjson': ResultSetDTO.to_ndjson,
+    'csv': ResultSetDTO.to_csv,
+    'tsv': ResultSetDTO.to_tsv,
+    'xml': ResultSetDTO.to_xml,
+    'yaml': ResultSetDTO.to_yaml,
+    'xlsx': ResultSetDTO.to_xlsx,
+}