sql2api 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sql2api/__init__.py +1 -1
- sql2api/app.py +112 -15
- sql2api/cli.py +6 -1
- sql2api/config.py +94 -0
- sql2api/cors.py +51 -0
- sql2api/engine.py +6 -3
- sql2api/openapi.py +100 -8
- sql2api/params.py +275 -0
- sql2api/pool.py +146 -0
- sql2api/ratelimit.py +56 -0
- sql2api/runners.py +261 -49
- sql2api/sqltools.py +5 -31
- sql2api/store.py +18 -0
- {sql2api-0.1.0.dist-info → sql2api-0.3.0.dist-info}/METADATA +70 -6
- sql2api-0.3.0.dist-info/RECORD +23 -0
- sql2api-0.1.0.dist-info/RECORD +0 -19
- {sql2api-0.1.0.dist-info → sql2api-0.3.0.dist-info}/WHEEL +0 -0
- {sql2api-0.1.0.dist-info → sql2api-0.3.0.dist-info}/entry_points.txt +0 -0
- {sql2api-0.1.0.dist-info → sql2api-0.3.0.dist-info}/licenses/LICENSE +0 -0
- {sql2api-0.1.0.dist-info → sql2api-0.3.0.dist-info}/top_level.txt +0 -0
sql2api/__init__.py
CHANGED
sql2api/app.py
CHANGED
|
@@ -1,21 +1,26 @@
|
|
|
1
1
|
"""The Flask application: HTTP routes on top of the store, engine and formatters."""
|
|
2
2
|
import hmac
|
|
3
3
|
import logging
|
|
4
|
+
import math
|
|
4
5
|
|
|
5
|
-
from flask import Blueprint, Flask, Response, jsonify, redirect, request, url_for
|
|
6
|
+
from flask import Blueprint, Flask, Response, current_app, g, jsonify, redirect, request, url_for
|
|
6
7
|
from flask.json.provider import DefaultJSONProvider
|
|
7
8
|
from werkzeug.exceptions import HTTPException
|
|
9
|
+
from werkzeug.middleware.proxy_fix import ProxyFix
|
|
8
10
|
|
|
9
|
-
from . import config, engine, openapi, sqltools, store
|
|
11
|
+
from . import config, cors, engine, openapi, pool, sqltools, store
|
|
12
|
+
from . import params as param_rules
|
|
10
13
|
from .errors import ApiError
|
|
11
14
|
from .formats import FORMATTERS, json_default
|
|
15
|
+
from .ratelimit import RateLimiter
|
|
12
16
|
|
|
13
17
|
log = logging.getLogger('sql2api')
|
|
14
18
|
bp = Blueprint('api', __name__)
|
|
15
19
|
|
|
16
20
|
# Query-string arguments that control a request rather than supplying query parameters.
|
|
17
|
-
RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version'}
|
|
21
|
+
RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version', 'timeout'}
|
|
18
22
|
PUBLIC_ENDPOINTS = {'api.index', 'api.favicon', 'api.health', 'api.docs', 'api.openapi_spec'}
|
|
23
|
+
RATE_LIMIT_EXEMPT = {'api.health'} # so monitoring keeps working while a client is being throttled
|
|
19
24
|
|
|
20
25
|
|
|
21
26
|
class JSONProvider(DefaultJSONProvider):
|
|
@@ -25,7 +30,15 @@ class JSONProvider(DefaultJSONProvider):
|
|
|
25
30
|
|
|
26
31
|
def create_app():
|
|
27
32
|
from . import __version__
|
|
33
|
+
config.check_settings()
|
|
28
34
|
app = Flask(__name__)
|
|
35
|
+
hops = config.proxy_hops()
|
|
36
|
+
if hops: # behind reverse proxies: take the client address and scheme from their X-Forwarded-* headers
|
|
37
|
+
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=hops, x_proto=hops, x_host=hops)
|
|
38
|
+
app.extensions['sql2api_limiter'] = RateLimiter()
|
|
39
|
+
if config.cors_origins() == '*' and not config.api_key():
|
|
40
|
+
log.warning('SQL2API_CORS_ORIGINS=* without SQL2API_API_KEY: any website a user visits can call this API '
|
|
41
|
+
'from their browser and reach every active connection. Set an API key or list the origins.')
|
|
29
42
|
app.json = JSONProvider(app)
|
|
30
43
|
app.config['SQL2API_VERSION'] = __version__
|
|
31
44
|
|
|
@@ -43,13 +56,25 @@ def create_app():
|
|
|
43
56
|
return jsonify({'error': 'An error occurred'}), 500
|
|
44
57
|
|
|
45
58
|
@app.before_request
|
|
46
|
-
def
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
59
|
+
def gate():
|
|
60
|
+
# Order matters: a browser's preflight cannot carry the API key, and rate limiting comes before the key
|
|
61
|
+
# check so that guessing keys is throttled too.
|
|
62
|
+
if cors.is_preflight(request):
|
|
63
|
+
return cors.preflight_response(request.headers.get('Origin'))
|
|
64
|
+
limited = check_rate_limit()
|
|
65
|
+
if limited is not None:
|
|
66
|
+
return limited
|
|
67
|
+
if request.endpoint not in PUBLIC_ENDPOINTS and not has_valid_key():
|
|
68
|
+
return jsonify({'error': 'Unauthorized'}), 401
|
|
69
|
+
|
|
70
|
+
@app.after_request
|
|
71
|
+
def decorate(response):
|
|
72
|
+
cors.add_headers(response, request.headers.get('Origin'))
|
|
73
|
+
if g.get('rate_limit'):
|
|
74
|
+
limit, remaining = g.rate_limit
|
|
75
|
+
response.headers['X-RateLimit-Limit'] = str(limit)
|
|
76
|
+
response.headers['X-RateLimit-Remaining'] = str(remaining)
|
|
77
|
+
return response
|
|
53
78
|
|
|
54
79
|
app.register_blueprint(bp)
|
|
55
80
|
return app
|
|
@@ -59,6 +84,33 @@ def create_app():
|
|
|
59
84
|
# Request helpers
|
|
60
85
|
# --------------------------------------------------------------------------------------
|
|
61
86
|
|
|
87
|
+
def check_rate_limit():
|
|
88
|
+
"""Count this request against its client's quota; returns a 429 response when it is over the limit."""
|
|
89
|
+
limit = config.rate_limit()
|
|
90
|
+
if limit is None or request.method == 'OPTIONS' or request.endpoint in RATE_LIMIT_EXEMPT:
|
|
91
|
+
return None
|
|
92
|
+
count, period = limit
|
|
93
|
+
client = request.remote_addr or 'unknown'
|
|
94
|
+
allowed, remaining, retry_after = current_app.extensions['sql2api_limiter'].hit(client, count, period)
|
|
95
|
+
g.rate_limit = (count, remaining)
|
|
96
|
+
if allowed:
|
|
97
|
+
return None
|
|
98
|
+
response = jsonify({'error': 'Rate limit exceeded', 'retry_after': retry_after})
|
|
99
|
+
response.status_code = 429
|
|
100
|
+
response.headers['Retry-After'] = str(retry_after)
|
|
101
|
+
return response
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def has_valid_key():
|
|
105
|
+
"""True when no API key is configured, or the request carries the right X-API-Key header."""
|
|
106
|
+
expected = config.api_key()
|
|
107
|
+
if not expected:
|
|
108
|
+
return True
|
|
109
|
+
# compare_digest rejects non-ASCII str, so compare bytes
|
|
110
|
+
supplied = request.headers.get('X-API-Key', '').encode('utf-8', 'replace')
|
|
111
|
+
return hmac.compare_digest(supplied, expected.encode('utf-8'))
|
|
112
|
+
|
|
113
|
+
|
|
62
114
|
def get_json_body(required=True):
|
|
63
115
|
data = request.get_json(silent=True)
|
|
64
116
|
if data is None and not required:
|
|
@@ -105,6 +157,20 @@ def get_output_format(body=None):
|
|
|
105
157
|
return output_format
|
|
106
158
|
|
|
107
159
|
|
|
160
|
+
def get_timeout(body=None):
|
|
161
|
+
"""Seconds allowed for the query: ?timeout= may lower the server limit but never raise it."""
|
|
162
|
+
raw = request.args.get('timeout', (body or {}).get('timeout'))
|
|
163
|
+
if raw in (None, ''):
|
|
164
|
+
return config.effective_timeout(None)
|
|
165
|
+
try:
|
|
166
|
+
value = float(raw)
|
|
167
|
+
except (TypeError, ValueError):
|
|
168
|
+
raise ApiError('timeout must be a number of seconds') from None
|
|
169
|
+
if not math.isfinite(value) or value <= 0:
|
|
170
|
+
raise ApiError('timeout must be a positive number of seconds')
|
|
171
|
+
return config.effective_timeout(value)
|
|
172
|
+
|
|
173
|
+
|
|
108
174
|
def render(result, output_format, page, page_size):
|
|
109
175
|
response = jsonify({'message': 'No results returned'}) if not result else FORMATTERS[output_format](result)
|
|
110
176
|
response.headers['X-Page'] = str(page)
|
|
@@ -126,8 +192,9 @@ def execute_sql_endpoint():
|
|
|
126
192
|
raise ApiError('Connection name is missing')
|
|
127
193
|
params = get_object(data.get('params'), 'params')
|
|
128
194
|
output_format = get_output_format(data)
|
|
195
|
+
timeout = get_timeout(data)
|
|
129
196
|
limit, offset, page = get_pagination()
|
|
130
|
-
result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params)
|
|
197
|
+
result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params, timeout)
|
|
131
198
|
return render(result, output_format, page, limit)
|
|
132
199
|
|
|
133
200
|
|
|
@@ -145,14 +212,16 @@ def run_saved(ref, body, url_params):
|
|
|
145
212
|
|
|
146
213
|
raw = {**url_params, **get_object(body.get('params'), 'params'),
|
|
147
214
|
**get_object(body.get('placeholders'), 'placeholders')}
|
|
148
|
-
|
|
149
|
-
|
|
215
|
+
used = set(sqltools.placeholder_names(saved['sql_query']))
|
|
216
|
+
values = param_rules.resolve(saved.get('query_parameters'), raw, used=used)
|
|
217
|
+
sql = sqltools.fill_placeholders(saved['sql_query'], values)
|
|
150
218
|
output_format = get_output_format(body)
|
|
219
|
+
timeout = get_timeout(body)
|
|
151
220
|
limit, offset, page = get_pagination()
|
|
152
221
|
|
|
153
222
|
entry = {'executed_at': store.now(), 'connection_name': connection_name}
|
|
154
223
|
try:
|
|
155
|
-
result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset,
|
|
224
|
+
result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset, values, timeout)
|
|
156
225
|
except ApiError as error:
|
|
157
226
|
store.record_execution(path, number, {**entry, 'status': 'error', 'error': error.message})
|
|
158
227
|
raise
|
|
@@ -197,6 +266,11 @@ def save_sql_to_file():
|
|
|
197
266
|
if not isinstance(tags, (list, str)):
|
|
198
267
|
raise ApiError('tags must be a string or a list')
|
|
199
268
|
query_parameters = get_object(data.get('query_parameters'), 'query_parameters')
|
|
269
|
+
param_rules.parse_definitions(query_parameters)
|
|
270
|
+
unused = sorted(set(query_parameters) - set(sqltools.placeholder_names(data['sql_query'])))
|
|
271
|
+
if unused:
|
|
272
|
+
raise ApiError(f"query_parameters declares {', '.join(unused)}, which sql_query does not use "
|
|
273
|
+
'(write :name in the SQL, or remove the declaration)')
|
|
200
274
|
connection_name = data.get('connection_name')
|
|
201
275
|
if connection_name is not None and not isinstance(connection_name, str):
|
|
202
276
|
raise ApiError('connection_name must be a string')
|
|
@@ -256,12 +330,14 @@ def update_connections():
|
|
|
256
330
|
if not connections or not isinstance(connections, dict):
|
|
257
331
|
raise ApiError('Connections data is missing')
|
|
258
332
|
store.update_connections(connections)
|
|
333
|
+
pool.close_pooled_connections() # new settings or credentials must not be served by old connections
|
|
259
334
|
return jsonify({'message': 'Connections updated successfully'}), 200
|
|
260
335
|
|
|
261
336
|
|
|
262
337
|
@bp.route('/connections/<name>', methods=['DELETE'])
|
|
263
338
|
def delete_connection(name):
|
|
264
339
|
store.delete_connection(name)
|
|
340
|
+
pool.close_pooled_connections() # a removed connection must not keep serving from idle sockets
|
|
265
341
|
return jsonify({'message': f"Connection '{name}' deleted"}), 200
|
|
266
342
|
|
|
267
343
|
|
|
@@ -285,10 +361,31 @@ def health():
|
|
|
285
361
|
return jsonify({'status': 'ok', 'version': current_app.config['SQL2API_VERSION']})
|
|
286
362
|
|
|
287
363
|
|
|
364
|
+
def describe_saved_queries():
|
|
365
|
+
"""What the OpenAPI document needs to know about each saved query (never its SQL text)."""
|
|
366
|
+
described = []
|
|
367
|
+
for name, number, data in store.latest_versions():
|
|
368
|
+
sql = data.get('sql_query')
|
|
369
|
+
if not isinstance(sql, str):
|
|
370
|
+
continue
|
|
371
|
+
declared = param_rules.read_definitions(data.get('query_parameters'))
|
|
372
|
+
used = sqltools.placeholder_names(sql)
|
|
373
|
+
parameters = {}
|
|
374
|
+
for param in used: # what the SQL needs, in order; undeclared ones are plain required text
|
|
375
|
+
parameters[param] = declared.get(param) or param_rules.read_definition({})
|
|
376
|
+
described.append({'name': name, 'version': number, 'description': data.get('description'),
|
|
377
|
+
'tags': data.get('tags'), 'connection_name': data.get('connection_name'),
|
|
378
|
+
'parameters': parameters})
|
|
379
|
+
return described
|
|
380
|
+
|
|
381
|
+
|
|
288
382
|
@bp.route('/openapi.json', methods=['GET'])
|
|
289
383
|
def openapi_spec():
|
|
290
384
|
from flask import current_app
|
|
291
|
-
|
|
385
|
+
# The generic API description is public. The list of saved queries (names, descriptions, parameters) is only
|
|
386
|
+
# shown to callers who could list them anyway, so an API key protects it too.
|
|
387
|
+
saved = describe_saved_queries() if has_valid_key() else None
|
|
388
|
+
return jsonify(openapi.build_spec(current_app.config['SQL2API_VERSION'], saved))
|
|
292
389
|
|
|
293
390
|
|
|
294
391
|
@bp.route('/docs', methods=['GET'])
|
sql2api/cli.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import argparse
|
|
3
3
|
import logging
|
|
4
4
|
import os
|
|
5
|
+
import sys
|
|
5
6
|
|
|
6
7
|
from . import __version__, config, store
|
|
7
8
|
from .app import create_app
|
|
@@ -10,7 +11,11 @@ LOOPBACK_HOSTS = ('127.0.0.1', 'localhost', '::1')
|
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def _serve(args):
|
|
13
|
-
|
|
14
|
+
try:
|
|
15
|
+
app = create_app()
|
|
16
|
+
except ValueError as error: # a malformed setting, e.g. SQL2API_RATE_LIMIT
|
|
17
|
+
print(f'sql2api: {error}', file=sys.stderr)
|
|
18
|
+
return 2
|
|
14
19
|
if args.host not in LOOPBACK_HOSTS and not config.api_key():
|
|
15
20
|
logging.getLogger('sql2api').warning(
|
|
16
21
|
'Listening on %s without SQL2API_API_KEY set: anyone who can reach this port can run SQL '
|
sql2api/config.py
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
"""Runtime configuration, read from environment variables at call time."""
|
|
2
2
|
import os
|
|
3
|
+
import re
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
SUPPORTED_DB_TYPES = ('mysql', 'postgres', 'clickhouse', 'sqlite', 'h2')
|
|
6
7
|
PASSWORD_MASK = '********'
|
|
7
8
|
CONNECT_TIMEOUT = 10 # seconds
|
|
8
9
|
HISTORY_LIMIT = 50 # executions remembered per saved-query version
|
|
10
|
+
DEFAULT_QUERY_TIMEOUT = 30.0 # seconds
|
|
11
|
+
DEFAULT_POOL_SIZE = 5 # idle connections kept per distinct connection
|
|
12
|
+
DEFAULT_POOL_IDLE_TIMEOUT = 300.0 # seconds
|
|
9
13
|
|
|
10
14
|
# Ships with the package; override with SQL2API_H2_JAR to use a different H2 version.
|
|
11
15
|
BUNDLED_H2_JAR = Path(__file__).parent / 'lib' / 'h2-2.2.224.jar'
|
|
@@ -55,6 +59,96 @@ def api_key():
|
|
|
55
59
|
return os.environ.get('SQL2API_API_KEY') or None
|
|
56
60
|
|
|
57
61
|
|
|
62
|
+
def query_timeout():
|
|
63
|
+
"""Server-wide statement time limit in seconds (SQL2API_QUERY_TIMEOUT); None when disabled (set to 0)."""
|
|
64
|
+
try:
|
|
65
|
+
value = float(os.environ.get('SQL2API_QUERY_TIMEOUT', DEFAULT_QUERY_TIMEOUT))
|
|
66
|
+
except ValueError:
|
|
67
|
+
return DEFAULT_QUERY_TIMEOUT
|
|
68
|
+
if value == 0:
|
|
69
|
+
return None
|
|
70
|
+
return value if value > 0 else DEFAULT_QUERY_TIMEOUT
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def effective_timeout(requested=None):
|
|
74
|
+
"""The limit to enforce: a request may ask for less time than the server allows, never more."""
|
|
75
|
+
limit = query_timeout()
|
|
76
|
+
if requested is None:
|
|
77
|
+
return limit
|
|
78
|
+
return requested if limit is None else min(requested, limit)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def pool_size():
|
|
82
|
+
"""Idle connections kept per distinct connection (SQL2API_POOL_SIZE); 0 disables pooling."""
|
|
83
|
+
try:
|
|
84
|
+
return max(0, int(os.environ.get('SQL2API_POOL_SIZE', DEFAULT_POOL_SIZE)))
|
|
85
|
+
except ValueError:
|
|
86
|
+
return DEFAULT_POOL_SIZE
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def pool_idle_timeout():
|
|
90
|
+
"""Seconds an idle pooled connection is kept before it is closed (SQL2API_POOL_IDLE_TIMEOUT)."""
|
|
91
|
+
try:
|
|
92
|
+
value = float(os.environ.get('SQL2API_POOL_IDLE_TIMEOUT', DEFAULT_POOL_IDLE_TIMEOUT))
|
|
93
|
+
except ValueError:
|
|
94
|
+
return DEFAULT_POOL_IDLE_TIMEOUT
|
|
95
|
+
return value if value > 0 else DEFAULT_POOL_IDLE_TIMEOUT
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def cors_origins():
|
|
99
|
+
"""Origins allowed to call the API from a browser (SQL2API_CORS_ORIGINS): None (off), '*' or a frozenset.
|
|
100
|
+
|
|
101
|
+
Entries are compared case-insensitively and without a trailing slash, e.g. https://app.example.com.
|
|
102
|
+
"""
|
|
103
|
+
raw = os.environ.get('SQL2API_CORS_ORIGINS', '').strip()
|
|
104
|
+
if not raw:
|
|
105
|
+
return None
|
|
106
|
+
origins = [item.strip().rstrip('/').lower() for item in raw.split(',') if item.strip()]
|
|
107
|
+
if '*' in origins:
|
|
108
|
+
return '*'
|
|
109
|
+
return frozenset(origins) or None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
_RATE_PERIODS = {'second': 1, 'minute': 60, 'hour': 3600, 'day': 86400}
|
|
113
|
+
_RATE_RE = re.compile(r'^\s*(\d+)\s*/\s*(second|minute|hour|day)s?\s*$', re.I)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def parse_rate_limit(text):
|
|
117
|
+
"""Parse '60/minute' (also second, hour, day) into (requests, seconds); raises ValueError when malformed."""
|
|
118
|
+
match = _RATE_RE.match(text or '')
|
|
119
|
+
if not match or int(match.group(1)) < 1:
|
|
120
|
+
raise ValueError(f"SQL2API_RATE_LIMIT must look like '60/minute' (a positive count, then second, minute, "
|
|
121
|
+
f"hour or day), not {text!r}")
|
|
122
|
+
return int(match.group(1)), _RATE_PERIODS[match.group(2).lower()]
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def rate_limit():
|
|
126
|
+
"""(requests, seconds) allowed per client (SQL2API_RATE_LIMIT), or None when limiting is off."""
|
|
127
|
+
raw = os.environ.get('SQL2API_RATE_LIMIT', '').strip()
|
|
128
|
+
if not raw:
|
|
129
|
+
return None
|
|
130
|
+
try:
|
|
131
|
+
return parse_rate_limit(raw)
|
|
132
|
+
except ValueError:
|
|
133
|
+
return None # create_app() rejects a malformed value at startup; never limit by accident afterwards
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def proxy_hops():
|
|
137
|
+
"""Reverse proxies in front of the app whose X-Forwarded-* headers can be trusted (SQL2API_TRUST_PROXY)."""
|
|
138
|
+
try:
|
|
139
|
+
return max(0, int(os.environ.get('SQL2API_TRUST_PROXY', 0)))
|
|
140
|
+
except ValueError:
|
|
141
|
+
return 0
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def check_settings():
|
|
145
|
+
"""Raise ValueError for a malformed setting, so a typo fails at startup instead of silently switching off a
|
|
146
|
+
protection."""
|
|
147
|
+
raw = os.environ.get('SQL2API_RATE_LIMIT', '').strip()
|
|
148
|
+
if raw:
|
|
149
|
+
parse_rate_limit(raw)
|
|
150
|
+
|
|
151
|
+
|
|
58
152
|
def max_page_size():
|
|
59
153
|
try:
|
|
60
154
|
return max(1, int(os.environ.get('SQL2API_MAX_PAGE_SIZE', 1000)))
|
sql2api/cors.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Cross-origin (CORS) support for browser clients, off unless SQL2API_CORS_ORIGINS is set.
|
|
2
|
+
|
|
3
|
+
Without it a web page on another origin cannot call the API: browsers refuse to send the JSON requests and to read
|
|
4
|
+
the answers. CORS only tells the *browser* which sites may call; it is not authentication (use SQL2API_API_KEY).
|
|
5
|
+
"""
|
|
6
|
+
from flask import Response
|
|
7
|
+
|
|
8
|
+
from . import config
|
|
9
|
+
|
|
10
|
+
ALLOWED_METHODS = 'GET, POST, PATCH, DELETE, OPTIONS'
|
|
11
|
+
ALLOWED_HEADERS = 'Content-Type, X-API-Key'
|
|
12
|
+
# Browsers hide response headers a page has not been told about, and pagination depends on these.
|
|
13
|
+
EXPOSED_HEADERS = 'X-Page, X-Page-Size, X-Has-More, X-RateLimit-Limit, X-RateLimit-Remaining, Retry-After'
|
|
14
|
+
PREFLIGHT_MAX_AGE = '600'
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def allow_origin_value(origin):
|
|
18
|
+
"""The Access-Control-Allow-Origin value for a request from ``origin``, or None when it is not allowed."""
|
|
19
|
+
allowed = config.cors_origins()
|
|
20
|
+
if allowed is None or not origin:
|
|
21
|
+
return None
|
|
22
|
+
if allowed == '*':
|
|
23
|
+
return '*'
|
|
24
|
+
return origin if origin.rstrip('/').lower() in allowed else None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def is_preflight(request):
|
|
28
|
+
return (request.method == 'OPTIONS' and 'Access-Control-Request-Method' in request.headers
|
|
29
|
+
and config.cors_origins() is not None)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def preflight_response(origin):
|
|
33
|
+
"""Answer a browser's permission check before it sends the real request (no auth: it carries no key)."""
|
|
34
|
+
response = Response(status=204)
|
|
35
|
+
value = allow_origin_value(origin)
|
|
36
|
+
if value:
|
|
37
|
+
response.headers['Access-Control-Allow-Methods'] = ALLOWED_METHODS
|
|
38
|
+
response.headers['Access-Control-Allow-Headers'] = ALLOWED_HEADERS
|
|
39
|
+
response.headers['Access-Control-Max-Age'] = PREFLIGHT_MAX_AGE
|
|
40
|
+
add_headers(response, origin)
|
|
41
|
+
return response
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def add_headers(response, origin):
|
|
45
|
+
value = allow_origin_value(origin)
|
|
46
|
+
if value:
|
|
47
|
+
response.headers['Access-Control-Allow-Origin'] = value
|
|
48
|
+
response.headers['Access-Control-Expose-Headers'] = EXPOSED_HEADERS
|
|
49
|
+
if value != '*':
|
|
50
|
+
response.headers.add('Vary', 'Origin') # the answer depends on who asked
|
|
51
|
+
return response
|
sql2api/engine.py
CHANGED
|
@@ -5,19 +5,22 @@ import time
|
|
|
5
5
|
from . import config, store
|
|
6
6
|
from .errors import ApiError
|
|
7
7
|
from .formats import ResultSetDTO
|
|
8
|
+
from .pool import get_pool
|
|
8
9
|
from .runners import RUNNERS
|
|
9
10
|
from .sqltools import validate_sql
|
|
10
11
|
|
|
11
12
|
log = logging.getLogger('sql2api')
|
|
12
13
|
|
|
13
14
|
|
|
14
|
-
def execute_sql(sql, connection_name, limit, offset, params=None):
|
|
15
|
+
def execute_sql(sql, connection_name, limit, offset, params=None, timeout=None):
|
|
15
16
|
"""Run ``sql`` on a named connection and return the requested page as a ResultSetDTO."""
|
|
16
17
|
details = store.get_connection(connection_name)
|
|
17
18
|
sql = validate_sql(sql)
|
|
18
|
-
log.info('Executing on %s (%s), limit=%s offset=%s: %s',
|
|
19
|
+
log.info('Executing on %s (%s), limit=%s offset=%s timeout=%s: %s',
|
|
20
|
+
connection_name, details['db'], limit, offset, timeout, sql)
|
|
19
21
|
try:
|
|
20
|
-
columns, rows = RUNNERS[details['db']](details, sql, params, limit, offset, not config.allow_writes()
|
|
22
|
+
columns, rows = RUNNERS[details['db']](details, sql, params, limit, offset, not config.allow_writes(), timeout,
|
|
23
|
+
get_pool())
|
|
21
24
|
except ApiError:
|
|
22
25
|
raise
|
|
23
26
|
except ImportError as error:
|
sql2api/openapi.py
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
"""OpenAPI description of the API, served at /openapi.json and rendered at /docs."""
|
|
2
|
+
import re
|
|
3
|
+
from urllib.parse import quote
|
|
4
|
+
|
|
2
5
|
from . import config
|
|
6
|
+
from .params import json_schema
|
|
3
7
|
|
|
4
8
|
_FORMAT_PARAM = {'name': 'format', 'in': 'query', 'schema': {
|
|
5
9
|
'type': 'string', 'enum': ['json', 'ndjson', 'csv', 'tsv', 'xml', 'yaml', 'xlsx'], 'default': 'json'}}
|
|
@@ -7,17 +11,27 @@ _PAGE_PARAMS = [
|
|
|
7
11
|
{'name': 'page', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 1}},
|
|
8
12
|
{'name': 'page_size', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 10},
|
|
9
13
|
'description': 'Upper limit is SQL2API_MAX_PAGE_SIZE (default 1000).'},
|
|
14
|
+
{'name': 'timeout', 'in': 'query', 'schema': {'type': 'number', 'minimum': 0, 'exclusiveMinimum': True},
|
|
15
|
+
'description': 'Seconds the query may run before it is cancelled (504). Can lower, never raise, the server '
|
|
16
|
+
'limit SQL2API_QUERY_TIMEOUT (default 30; 0 disables the server limit).'},
|
|
10
17
|
]
|
|
11
18
|
_ROWS = {'description': 'The requested page. Header X-Has-More says whether another page follows.',
|
|
12
19
|
'headers': {'X-Page': {'schema': {'type': 'integer'}}, 'X-Page-Size': {'schema': {'type': 'integer'}},
|
|
13
20
|
'X-Has-More': {'schema': {'type': 'string', 'enum': ['true', 'false']}}},
|
|
14
21
|
'content': {'application/json': {'schema': {'type': 'array', 'items': {'type': 'object'}}}}}
|
|
15
|
-
_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '500')}
|
|
22
|
+
_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '429', '500', '504')}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _object_schema(properties, required=()):
|
|
26
|
+
"""An object schema; OpenAPI 3.0 forbids an empty ``required`` list, so it is only included when non-empty."""
|
|
27
|
+
schema = {'type': 'object', 'properties': properties}
|
|
28
|
+
if required:
|
|
29
|
+
schema['required'] = list(required)
|
|
30
|
+
return schema
|
|
16
31
|
|
|
17
32
|
|
|
18
33
|
def _body(properties, required):
|
|
19
|
-
return {'required': True, 'content': {'application/json': {'schema':
|
|
20
|
-
'type': 'object', 'required': required, 'properties': properties}}}}
|
|
34
|
+
return {'required': True, 'content': {'application/json': {'schema': _object_schema(properties, required)}}}
|
|
21
35
|
|
|
22
36
|
|
|
23
37
|
_EXEC_PROPS = {
|
|
@@ -29,10 +43,58 @@ _EXEC_PROPS = {
|
|
|
29
43
|
}
|
|
30
44
|
|
|
31
45
|
|
|
32
|
-
def
|
|
46
|
+
def _saved_query_paths(queries):
|
|
47
|
+
"""One documented endpoint per saved query, with its declared parameters and rules."""
|
|
48
|
+
paths, seen = {}, set()
|
|
49
|
+
for query in queries:
|
|
50
|
+
name = query['name']
|
|
51
|
+
has_default_connection = bool(query.get('connection_name'))
|
|
52
|
+
properties, required, query_params = {}, [], []
|
|
53
|
+
for param, spec in query['parameters'].items():
|
|
54
|
+
schema = json_schema(spec)
|
|
55
|
+
entry = {'name': param, 'in': 'query', 'required': spec['required'], 'schema': schema}
|
|
56
|
+
if spec['description']:
|
|
57
|
+
entry['description'] = spec['description']
|
|
58
|
+
query_params.append(entry)
|
|
59
|
+
properties[param] = {**schema, **({'description': spec['description']} if spec['description'] else {})}
|
|
60
|
+
if spec['required']:
|
|
61
|
+
required.append(param)
|
|
62
|
+
connection = {'name': 'connection_name', 'in': 'query', 'required': not has_default_connection,
|
|
63
|
+
'schema': {'type': 'string', **({'default': query['connection_name']}
|
|
64
|
+
if has_default_connection else {})}}
|
|
65
|
+
common = [connection, {'name': 'version', 'in': 'query', 'schema': {'type': 'integer'},
|
|
66
|
+
'description': f"Saved version to run (default: latest, currently {query['version']})."},
|
|
67
|
+
_FORMAT_PARAM, *_PAGE_PARAMS]
|
|
68
|
+
summary = query.get('description') or f'Run the saved query {name}'
|
|
69
|
+
tags = query.get('tags')
|
|
70
|
+
detail = f"Runs version {query['version']} of the saved query '{name}'."
|
|
71
|
+
if tags:
|
|
72
|
+
detail += ' Tags: ' + (tags if isinstance(tags, str) else ', '.join(map(str, tags))) + '.'
|
|
73
|
+
operation_id = 'run_' + re.sub(r'\W', '_', name)
|
|
74
|
+
while operation_id in seen:
|
|
75
|
+
operation_id += '_'
|
|
76
|
+
seen.add(operation_id)
|
|
77
|
+
body_properties = {'params': _object_schema(properties, required),
|
|
78
|
+
'connection_name': connection['schema'], 'version': {'type': 'integer'},
|
|
79
|
+
'format': {'type': 'string'}, 'timeout': {'type': 'number'}}
|
|
80
|
+
paths['/q/' + quote(name, safe='')] = {
|
|
81
|
+
'get': {'summary': summary, 'description': detail, 'tags': ['Saved queries (live)'],
|
|
82
|
+
'operationId': operation_id, 'parameters': [*query_params, *common],
|
|
83
|
+
'responses': {'200': _ROWS, **_ERRORS}},
|
|
84
|
+
'post': {'summary': summary + ' (parameters in a JSON body)', 'description': detail,
|
|
85
|
+
'tags': ['Saved queries (live)'], 'operationId': operation_id + '_post',
|
|
86
|
+
'parameters': [_FORMAT_PARAM, *_PAGE_PARAMS],
|
|
87
|
+
'requestBody': {'required': bool(required), 'content': {'application/json': {
|
|
88
|
+
'schema': _object_schema(body_properties)}}},
|
|
89
|
+
'responses': {'200': _ROWS, **_ERRORS}},
|
|
90
|
+
}
|
|
91
|
+
return paths
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def build_spec(version, saved_queries=None):
|
|
33
95
|
saved = {'filepath': {'type': 'string', 'description': 'Saved query name or path inside saved_sql/.'},
|
|
34
96
|
**_EXEC_PROPS}
|
|
35
|
-
|
|
97
|
+
spec = {
|
|
36
98
|
'openapi': '3.0.3',
|
|
37
99
|
'info': {'title': 'SQL2API', 'version': version,
|
|
38
100
|
'description': 'Run SQL against configured databases and get the results back over HTTP.'},
|
|
@@ -116,12 +178,42 @@ def build_spec(version):
|
|
|
116
178
|
'responses': {'200': {'description': 'OK'}}}},
|
|
117
179
|
},
|
|
118
180
|
}
|
|
181
|
+
if saved_queries:
|
|
182
|
+
spec['paths'].update(_saved_query_paths(saved_queries))
|
|
183
|
+
spec['tags'] = [{'name': 'Saved queries (live)',
|
|
184
|
+
'description': 'One endpoint per saved query, generated from its latest version: its '
|
|
185
|
+
'declared parameters and their rules, and its default connection.'}]
|
|
186
|
+
return spec
|
|
119
187
|
|
|
120
188
|
|
|
121
189
|
DOCS_HTML = """<!doctype html>
|
|
122
190
|
<html><head><meta charset="utf-8"><title>SQL2API docs</title>
|
|
123
|
-
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css"
|
|
124
|
-
<
|
|
191
|
+
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css">
|
|
192
|
+
<style>
|
|
193
|
+
body { margin: 0; font-family: sans-serif; }
|
|
194
|
+
#key-bar { padding: 8px 16px; background: #f4f4f4; border-bottom: 1px solid #ddd; font-size: 14px; }
|
|
195
|
+
#key-bar input { width: 260px; padding: 4px; }
|
|
196
|
+
</style></head>
|
|
197
|
+
<body>
|
|
198
|
+
<div id="key-bar">
|
|
199
|
+
API key (only needed if the server sets SQL2API_API_KEY; it reveals your saved queries below and is
|
|
200
|
+
sent with "Try it out" requests; kept for this browser tab only):
|
|
201
|
+
<input id="key" type="password" autocomplete="off" placeholder="X-API-Key">
|
|
202
|
+
<button id="save-key">Apply</button>
|
|
203
|
+
</div>
|
|
204
|
+
<div id="ui"></div>
|
|
125
205
|
<script src="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui-bundle.js"></script>
|
|
126
|
-
<script>
|
|
206
|
+
<script>
|
|
207
|
+
var stored = '';
|
|
208
|
+
try { stored = sessionStorage.getItem('sql2api-key') || ''; } catch (e) {}
|
|
209
|
+
document.getElementById('key').value = stored;
|
|
210
|
+
document.getElementById('save-key').onclick = function () {
|
|
211
|
+
try { sessionStorage.setItem('sql2api-key', document.getElementById('key').value); } catch (e) {}
|
|
212
|
+
location.reload();
|
|
213
|
+
};
|
|
214
|
+
SwaggerUIBundle({
|
|
215
|
+
url: 'openapi.json', dom_id: '#ui',
|
|
216
|
+
requestInterceptor: function (req) { if (stored) { req.headers['X-API-Key'] = stored; } return req; }
|
|
217
|
+
});
|
|
218
|
+
</script></body></html>
|
|
127
219
|
"""
|