queryapigate 0.5.0__py3-none-any.whl

queryapigate/schema.py

@@ -0,0 +1,99 @@
+"""Schema introspection: list a connection's tables/views and their columns.
+
+One query per dialect against its catalogue (``information_schema`` for MySQL/PostgreSQL/H2/DuckDB,
+``system.tables``/``system.columns`` for ClickHouse, ``sqlite_master``/``pragma_table_info`` for SQLite),
+run through the normal execution pipeline (the SQL guard, pagination, pooling) like any other query - there
+is no separate code path or driver logic to introspect a database. Nothing here is user-controlled, so the
+per-dialect SQL is a plain literal string, not a bound parameter.
+
+Drivers disagree on the case of unaliased and even aliased result column names (MySQL and H2 report
+catalogue columns in upper case; PostgreSQL, ClickHouse and SQLite report them as written), so rows are
+matched up case-insensitively rather than by relying on any one driver's convention.
+"""
+from . import engine, store
+from .errors import ApiError
+
+# Large enough that no real schema is ever truncated, small enough to bound one request's memory use.
+ROW_CAP = 5000
+
+_QUERIES = {
+    'mysql': """
+        SELECT t.table_name AS table_name, t.table_type AS table_type, c.column_name AS column_name,
+               c.data_type AS data_type, c.is_nullable AS is_nullable, c.ordinal_position AS ordinal_position
+        FROM information_schema.tables t
+        JOIN information_schema.columns c
+          ON c.table_schema = t.table_schema AND c.table_name = t.table_name
+        WHERE t.table_schema = DATABASE()
+        ORDER BY t.table_name, c.ordinal_position
+    """,
+    'postgres': """
+        SELECT t.table_name AS table_name, t.table_type AS table_type, c.column_name AS column_name,
+               c.data_type AS data_type, c.is_nullable AS is_nullable, c.ordinal_position AS ordinal_position
+        FROM information_schema.tables t
+        JOIN information_schema.columns c
+          ON c.table_schema = t.table_schema AND c.table_name = t.table_name
+        WHERE t.table_schema NOT IN ('pg_catalog', 'information_schema')
+        ORDER BY t.table_name, c.ordinal_position
+    """,
+    'clickhouse': """
+        SELECT t.name AS table_name, t.engine AS table_type, c.name AS column_name, c.type AS data_type,
+               if(startsWith(c.type, 'Nullable('), 'YES', 'NO') AS is_nullable, c.position AS ordinal_position
+        FROM system.tables t
+        JOIN system.columns c ON c.database = t.database AND c.table = t.name
+        WHERE t.database = currentDatabase()
+        ORDER BY t.name, c.position
+    """,
+    'sqlite': """
+        SELECT m.name AS table_name, m.type AS table_type, ti.name AS column_name, ti.type AS data_type,
+               CASE ti."notnull" WHEN 0 THEN 'YES' ELSE 'NO' END AS is_nullable, ti.cid + 1 AS ordinal_position
+        FROM sqlite_master m
+        JOIN pragma_table_info(m.name) ti
+        WHERE m.type IN ('table', 'view') AND m.name NOT LIKE 'sqlite_%'
+        ORDER BY m.name, ti.cid
+    """,
+    'h2': """
+        SELECT t.table_name AS table_name, t.table_type AS table_type, c.column_name AS column_name,
+               c.data_type AS data_type, c.is_nullable AS is_nullable, c.ordinal_position AS ordinal_position
+        FROM information_schema.tables t
+        JOIN information_schema.columns c
+          ON c.table_schema = t.table_schema AND c.table_name = t.table_name
+        WHERE t.table_schema NOT IN ('INFORMATION_SCHEMA')
+        ORDER BY t.table_name, c.ordinal_position
+    """,
+    'duckdb': """
+        SELECT t.table_name AS table_name, t.table_type AS table_type, c.column_name AS column_name,
+               c.data_type AS data_type, c.is_nullable AS is_nullable, c.ordinal_position AS ordinal_position
+        FROM information_schema.tables t
+        JOIN information_schema.columns c
+          ON c.table_schema = t.table_schema AND c.table_name = t.table_name
+        WHERE t.table_schema = 'main'
+        ORDER BY t.table_name, c.ordinal_position
+    """,
+}
+
+
+def _normalize_table_type(value):
+    """'BASE TABLE'/'table' -> table; 'VIEW'/'View'/'MaterializedView' (ClickHouse engine) -> view."""
+    return 'view' if 'view' in (value or '').lower() else 'table'
+
+
+def fetch_schema(connection_name):
+    """Return {'tables': [{'name', 'type', 'columns': [{'name', 'type', 'nullable', 'position'}]}], 'truncated'}."""
+    dialect = store.get_connection(connection_name)['db']
+    if dialect not in _QUERIES:
+        raise ApiError(f"Schema introspection isn't supported for '{dialect}' connections yet")
+    result = engine.execute_sql(_QUERIES[dialect], connection_name, ROW_CAP, 0)
+    lower_columns = [str(c).lower() for c in result.columns]
+    tables = {}
+    order = []
+    for row in result.rows:
+        record = dict(zip(lower_columns, row))
+        name = record['table_name']
+        if name not in tables:
+            tables[name] = {'name': name, 'type': _normalize_table_type(record['table_type']), 'columns': []}
+            order.append(name)
+        tables[name]['columns'].append({
+            'name': record['column_name'], 'type': record['data_type'],
+            'nullable': str(record['is_nullable']).upper() == 'YES', 'position': record['ordinal_position'],
+        })
+    return {'tables': [tables[name] for name in order], 'truncated': result.has_more}

queryapigate/sqltools.py

@@ -0,0 +1,184 @@
+"""Pure SQL text handling: validation, pagination, and parameter substitution/binding."""
+import math
+import re
+
+from . import config
+from .errors import ApiError
+
+# Quoted strings, quoted identifiers and comments - text inside these is never inspected.
+#
+# Two variants exist because MySQL and ClickHouse honour a C-style backslash escape inside '...' and "..."
+# string literals by default (\' does not close the string), while PostgreSQL, SQLite and H2 do not (under
+# their standard/default settings, a quote only escapes via doubling, '' or ""). Using the doubling-only
+# rule for a backslash-honouring database under-counts how long a string stays open: a value ending in an
+# odd number of backslashes before a quote can make that database treat what we consider "outside the
+# string" as still inside it, or vice versa, letting a semicolon or a second statement hide from this
+# guard's semicolon check while the database executes it as separate, live SQL. This was verified against
+# real MySQL and ClickHouse servers, see tests/test_sql_guard_fuzz.py. `--` comments also require the
+# character after `--` to be whitespace or end of input, matching real SQL comment syntax; a bare `--x`
+# is not a comment to any of our dialects and must stay visible to this guard.
+_COMMENT = r'--(?=[ \t\n]|\Z)[^\n]*|/\*.*?\*/'
+_ANSI_LITERAL = rf"'(?:[^']|'')*'|\"(?:[^\"]|\"\")*\"|`(?:[^`]|``)*`|{_COMMENT}"
+_BACKSLASH_LITERAL = rf"'(?:[^'\\]|\\.|'')*'|\"(?:[^\"\\]|\\.|\"\")*\"|`(?:[^`]|``)*`|{_COMMENT}"
+_ANSI_LITERALS_RE = re.compile(_ANSI_LITERAL, re.S)
+_BACKSLASH_LITERALS_RE = re.compile(_BACKSLASH_LITERAL, re.S)
+# Dialects whose string literals honour backslash escapes (\') by default - see the comment above.
+BACKSLASH_ESCAPE_DIALECTS = frozenset({'mysql', 'clickhouse'})
+# A ":name" marker, skipping literals and Postgres "::" casts. Built per dialect for the same reason.
+_PARAM_DIALECT_LITERALS = ((None, _ANSI_LITERAL), *((d, _BACKSLASH_LITERAL) for d in BACKSLASH_ESCAPE_DIALECTS))
+_PARAM_RE = {dialect: re.compile(rf'(?P<skip>{literal}|::)|:(?P<name>[A-Za-z_]\w*)', re.S)
+            for dialect, literal in _PARAM_DIALECT_LITERALS}
+_BRACE_RE = re.compile(r'\{(\w+)\}')
+_TRAILING_LIMIT_RE = re.compile(r'\s+LIMIT\s+\d+(?:\s*,\s*\d+|\s+OFFSET\s+\d+)?\s*$', re.I)
+_SAFE_TEXT_RE = re.compile(r'^[\w\s.,:@%+/\-]*$')
+
+READ_ONLY_STATEMENTS = ('select', 'with', 'show', 'describe', 'desc', 'explain', 'values', 'table')
+PAGINATED_STATEMENTS = ('select', 'with')
+
+
+def _literals_re(dialect):
+    return _BACKSLASH_LITERALS_RE if dialect in BACKSLASH_ESCAPE_DIALECTS else _ANSI_LITERALS_RE
+
+
+def _param_re(dialect):
+    return _PARAM_RE[dialect] if dialect in _PARAM_RE else _PARAM_RE[None]
+
+
+def first_keyword(sql, dialect=None):
+    match = re.match(r'[\s(]*([A-Za-z]+)', _literals_re(dialect).sub(' ', sql))
+    return match.group(1).lower() if match else ''
+
+
+def is_paginated(sql, dialect=None):
+    """Whether the runner should append its own LIMIT/OFFSET. Always False for a generic 'jdbc' connection:
+    that syntax is not portable across arbitrary vendors (Oracle and SQL Server use OFFSET/FETCH, not
+    LIMIT), so paging for those connections instead falls back to fetching the window and slicing it in
+    Python - correct for any vendor, just not server-side-optimal. See runners._JDBC."""
+    return dialect != 'jdbc' and first_keyword(sql, dialect) in PAGINATED_STATEMENTS
+
+
+def validate_sql(sql, dialect=None, allow_writes=None, allowed_write_ops=None):
+    """Normalise a client-supplied statement and enforce the single-statement/read-only rules.
+
+    ``dialect`` should be the target connection's ``db`` value, so literals are read with the rules that
+    database actually applies (see the module docstring above _ANSI_LITERAL). ``allow_writes`` defaults to
+    the server-wide setting; a caller may pass an already-narrower effective value (e.g. a scoped API key
+    without write access) but never a more permissive one - see ``engine.execute_sql``.
+
+    ``allowed_write_ops`` (a key's own grant - see apikeys.py's module docstring) narrows *which* write
+    statements are allowed once writes are otherwise permitted - e.g. INSERT but not DELETE/DROP on the
+    same key. It never restricts a read-only statement, only a write one; ``None`` (the default) means no
+    such narrowing, exactly today's behaviour.
+    """
+    if not isinstance(sql, str) or not sql.strip():
+        raise ApiError('SQL query is missing')
+    sql = re.sub(r'[\s;]+$', '', sql.strip())
+    if ';' in _literals_re(dialect).sub(' ', sql):
+        raise ApiError('Only a single SQL statement can be executed at a time')
+    if not (config.allow_writes() if allow_writes is None else allow_writes):
+        if first_keyword(sql, dialect) not in READ_ONLY_STATEMENTS or '/*!' in sql:
+            raise ApiError('Only read-only statements (SELECT, WITH, SHOW, DESCRIBE, EXPLAIN) are allowed. '
+                           'Set QUERYAPIGATE_ALLOW_WRITES=1 to lift this restriction.', 403)
+    elif allowed_write_ops is not None:
+        keyword = first_keyword(sql, dialect)
+        if keyword not in READ_ONLY_STATEMENTS and keyword not in allowed_write_ops:
+            raise ApiError(f"This API key may only perform these write operations: "
+                           f"{', '.join(sorted(allowed_write_ops))}.", 403)
+    return sql
+
+
+def paginate(sql, limit, offset):
+    """Replace any trailing LIMIT/OFFSET on a SELECT with the requested window."""
+    sql = _TRAILING_LIMIT_RE.sub('', sql)
+    return f'{sql}\nLIMIT {int(limit)} OFFSET {int(offset)}'
+
+
+# --------------------------------------------------------------------------------------
+# Parameters
+# --------------------------------------------------------------------------------------
+
+def _check_value(name, value):
+    if value is None or isinstance(value, (bool, str)):
+        return
+    if isinstance(value, (int, float)) and math.isfinite(value):
+        return
+    raise ApiError(f"Parameter '{name}' must be a string, finite number, boolean or null")
+
+
+def fill_placeholders(sql, values):
+    """Substitute ``{name}`` text placeholders (legacy style; prefer bound ``:name`` parameters).
+
+    Because the value becomes part of the SQL text it is restricted to numbers, booleans and
+    plain text so it cannot break out of the query. Only placeholders present in the SQL are
+    validated, so the same dict can also carry values for bound parameters.
+    """
+    for key, value in values.items():
+        token = f'{{{key}}}'
+        if token not in sql:
+            continue
+        if isinstance(value, bool):
+            text = 'TRUE' if value else 'FALSE'
+        elif isinstance(value, (int, float)):
+            if not math.isfinite(value):
+                raise ApiError(f"Placeholder '{key}' must be a finite number")
+            text = str(value)
+        elif isinstance(value, str) and _SAFE_TEXT_RE.match(value) and '--' not in value:
+            text = value
+        else:
+            raise ApiError(f"Placeholder '{key}' is substituted into the SQL text, so it must be a number, "
+                           "boolean or a string containing only letters, digits, whitespace and . , : @ % + / - "
+                           f"(use a bound :{key} parameter for arbitrary text)")
+        sql = sql.replace(token, text)
+    missing = sorted(set(_BRACE_RE.findall(sql)))
+    if missing:
+        raise ApiError(f"No value provided for placeholder(s): {', '.join(missing)}")
+    return sql
+
+
+def named_parameters(sql, dialect=None):
+    """Names of the ``:name`` bound parameters used by ``sql``, in order of appearance."""
+    return [m.group('name') for m in _param_re(dialect).finditer(sql) if m.group('name')]
+
+
+def placeholder_names(sql, dialect=None):
+    """Every parameter name ``sql`` uses - bound ``:name`` first, then ``{name}`` text placeholders - once each.
+
+    app.py calls this without a dialect (bookkeeping: which declared parameters need a value, and the
+    OpenAPI catalogue) even where a connection - and so a dialect - is known, because getting the dialect
+    wrong here can only make that bookkeeping slightly inaccurate in a rare, contrived edge case; it never
+    affects what SQL is actually sent to a database. The SQL that is actually executed always goes through
+    engine.execute_sql -> validate_sql and runners.py's bind_parameters, both of which do receive the
+    connection's real dialect. See the module docstring above _ANSI_LITERAL for why the dialect matters.
+    """
+    return list(dict.fromkeys(named_parameters(sql, dialect) + _BRACE_RE.findall(sql)))
+
+
+_MARKERS = {'qmark': lambda name: '?', 'format': lambda name: '%s', 'pyformat': lambda name: f'%({name})s'}
+
+
+def bind_parameters(sql, params, style, dialect=None):
+    """Rewrite ``:name`` markers into a driver's paramstyle and return ``(sql, args)``.
+
+    ``style`` is 'qmark' (``?``, positional), 'format' (``%s``, positional) or 'pyformat'
+    (``%(name)s``, mapping). Values never become part of the SQL text. Returns ``(sql, None)``
+    when the statement has no parameters, so drivers do not apply ``%`` formatting to it.
+    """
+    matches = [m for m in _param_re(dialect).finditer(sql) if m.group('name')]
+    if not matches:
+        return sql, None
+    names = [m.group('name') for m in matches]
+    missing = sorted(set(names) - set(params))
+    if missing:
+        raise ApiError(f"No value provided for parameter(s): {', '.join(missing)}")
+    for name in set(names):
+        _check_value(name, params[name])
+
+    escape = (lambda text: text.replace('%', '%%')) if style in ('format', 'pyformat') else (lambda text: text)
+    pieces, position = [], 0
+    for match in matches:
+        pieces.append(escape(sql[position:match.start()]))
+        pieces.append(_MARKERS[style](match.group('name')))
+        position = match.end()
+    pieces.append(escape(sql[position:]))
+    args = {name: params[name] for name in names} if style == 'pyformat' else [params[name] for name in names]
+    return ''.join(pieces), args