sql2api 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sql2api/__init__.py +1 -1
- sql2api/app.py +91 -13
- sql2api/cli.py +6 -1
- sql2api/config.py +55 -0
- sql2api/cors.py +51 -0
- sql2api/openapi.py +98 -9
- sql2api/params.py +275 -0
- sql2api/ratelimit.py +56 -0
- sql2api/sqltools.py +5 -31
- sql2api/store.py +18 -0
- {sql2api-0.2.0.dist-info → sql2api-0.3.0.dist-info}/METADATA +63 -6
- sql2api-0.3.0.dist-info/RECORD +23 -0
- sql2api-0.2.0.dist-info/RECORD +0 -20
- {sql2api-0.2.0.dist-info → sql2api-0.3.0.dist-info}/WHEEL +0 -0
- {sql2api-0.2.0.dist-info → sql2api-0.3.0.dist-info}/entry_points.txt +0 -0
- {sql2api-0.2.0.dist-info → sql2api-0.3.0.dist-info}/licenses/LICENSE +0 -0
- {sql2api-0.2.0.dist-info → sql2api-0.3.0.dist-info}/top_level.txt +0 -0
sql2api/__init__.py
CHANGED
sql2api/app.py
CHANGED
|
@@ -3,13 +3,16 @@ import hmac
|
|
|
3
3
|
import logging
|
|
4
4
|
import math
|
|
5
5
|
|
|
6
|
-
from flask import Blueprint, Flask, Response, jsonify, redirect, request, url_for
|
|
6
|
+
from flask import Blueprint, Flask, Response, current_app, g, jsonify, redirect, request, url_for
|
|
7
7
|
from flask.json.provider import DefaultJSONProvider
|
|
8
8
|
from werkzeug.exceptions import HTTPException
|
|
9
|
+
from werkzeug.middleware.proxy_fix import ProxyFix
|
|
9
10
|
|
|
10
|
-
from . import config, engine, openapi, pool, sqltools, store
|
|
11
|
+
from . import config, cors, engine, openapi, pool, sqltools, store
|
|
12
|
+
from . import params as param_rules
|
|
11
13
|
from .errors import ApiError
|
|
12
14
|
from .formats import FORMATTERS, json_default
|
|
15
|
+
from .ratelimit import RateLimiter
|
|
13
16
|
|
|
14
17
|
log = logging.getLogger('sql2api')
|
|
15
18
|
bp = Blueprint('api', __name__)
|
|
@@ -17,6 +20,7 @@ bp = Blueprint('api', __name__)
|
|
|
17
20
|
# Query-string arguments that control a request rather than supplying query parameters.
|
|
18
21
|
RESERVED_ARGS = {'format', 'page', 'page_size', 'connection_name', 'version', 'timeout'}
|
|
19
22
|
PUBLIC_ENDPOINTS = {'api.index', 'api.favicon', 'api.health', 'api.docs', 'api.openapi_spec'}
|
|
23
|
+
RATE_LIMIT_EXEMPT = {'api.health'} # so monitoring keeps working while a client is being throttled
|
|
20
24
|
|
|
21
25
|
|
|
22
26
|
class JSONProvider(DefaultJSONProvider):
|
|
@@ -26,7 +30,15 @@ class JSONProvider(DefaultJSONProvider):
|
|
|
26
30
|
|
|
27
31
|
def create_app():
|
|
28
32
|
from . import __version__
|
|
33
|
+
config.check_settings()
|
|
29
34
|
app = Flask(__name__)
|
|
35
|
+
hops = config.proxy_hops()
|
|
36
|
+
if hops: # behind reverse proxies: take the client address and scheme from their X-Forwarded-* headers
|
|
37
|
+
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=hops, x_proto=hops, x_host=hops)
|
|
38
|
+
app.extensions['sql2api_limiter'] = RateLimiter()
|
|
39
|
+
if config.cors_origins() == '*' and not config.api_key():
|
|
40
|
+
log.warning('SQL2API_CORS_ORIGINS=* without SQL2API_API_KEY: any website a user visits can call this API '
|
|
41
|
+
'from their browser and reach every active connection. Set an API key or list the origins.')
|
|
30
42
|
app.json = JSONProvider(app)
|
|
31
43
|
app.config['SQL2API_VERSION'] = __version__
|
|
32
44
|
|
|
@@ -44,13 +56,25 @@ def create_app():
|
|
|
44
56
|
return jsonify({'error': 'An error occurred'}), 500
|
|
45
57
|
|
|
46
58
|
@app.before_request
|
|
47
|
-
def
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
59
|
+
def gate():
|
|
60
|
+
# Order matters: a browser's preflight cannot carry the API key, and rate limiting comes before the key
|
|
61
|
+
# check so that guessing keys is throttled too.
|
|
62
|
+
if cors.is_preflight(request):
|
|
63
|
+
return cors.preflight_response(request.headers.get('Origin'))
|
|
64
|
+
limited = check_rate_limit()
|
|
65
|
+
if limited is not None:
|
|
66
|
+
return limited
|
|
67
|
+
if request.endpoint not in PUBLIC_ENDPOINTS and not has_valid_key():
|
|
68
|
+
return jsonify({'error': 'Unauthorized'}), 401
|
|
69
|
+
|
|
70
|
+
@app.after_request
|
|
71
|
+
def decorate(response):
|
|
72
|
+
cors.add_headers(response, request.headers.get('Origin'))
|
|
73
|
+
if g.get('rate_limit'):
|
|
74
|
+
limit, remaining = g.rate_limit
|
|
75
|
+
response.headers['X-RateLimit-Limit'] = str(limit)
|
|
76
|
+
response.headers['X-RateLimit-Remaining'] = str(remaining)
|
|
77
|
+
return response
|
|
54
78
|
|
|
55
79
|
app.register_blueprint(bp)
|
|
56
80
|
return app
|
|
@@ -60,6 +84,33 @@ def create_app():
|
|
|
60
84
|
# Request helpers
|
|
61
85
|
# --------------------------------------------------------------------------------------
|
|
62
86
|
|
|
87
|
+
def check_rate_limit():
|
|
88
|
+
"""Count this request against its client's quota; returns a 429 response when it is over the limit."""
|
|
89
|
+
limit = config.rate_limit()
|
|
90
|
+
if limit is None or request.method == 'OPTIONS' or request.endpoint in RATE_LIMIT_EXEMPT:
|
|
91
|
+
return None
|
|
92
|
+
count, period = limit
|
|
93
|
+
client = request.remote_addr or 'unknown'
|
|
94
|
+
allowed, remaining, retry_after = current_app.extensions['sql2api_limiter'].hit(client, count, period)
|
|
95
|
+
g.rate_limit = (count, remaining)
|
|
96
|
+
if allowed:
|
|
97
|
+
return None
|
|
98
|
+
response = jsonify({'error': 'Rate limit exceeded', 'retry_after': retry_after})
|
|
99
|
+
response.status_code = 429
|
|
100
|
+
response.headers['Retry-After'] = str(retry_after)
|
|
101
|
+
return response
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def has_valid_key():
|
|
105
|
+
"""True when no API key is configured, or the request carries the right X-API-Key header."""
|
|
106
|
+
expected = config.api_key()
|
|
107
|
+
if not expected:
|
|
108
|
+
return True
|
|
109
|
+
# compare_digest rejects non-ASCII str, so compare bytes
|
|
110
|
+
supplied = request.headers.get('X-API-Key', '').encode('utf-8', 'replace')
|
|
111
|
+
return hmac.compare_digest(supplied, expected.encode('utf-8'))
|
|
112
|
+
|
|
113
|
+
|
|
63
114
|
def get_json_body(required=True):
|
|
64
115
|
data = request.get_json(silent=True)
|
|
65
116
|
if data is None and not required:
|
|
@@ -161,15 +212,16 @@ def run_saved(ref, body, url_params):
|
|
|
161
212
|
|
|
162
213
|
raw = {**url_params, **get_object(body.get('params'), 'params'),
|
|
163
214
|
**get_object(body.get('placeholders'), 'placeholders')}
|
|
164
|
-
|
|
165
|
-
|
|
215
|
+
used = set(sqltools.placeholder_names(saved['sql_query']))
|
|
216
|
+
values = param_rules.resolve(saved.get('query_parameters'), raw, used=used)
|
|
217
|
+
sql = sqltools.fill_placeholders(saved['sql_query'], values)
|
|
166
218
|
output_format = get_output_format(body)
|
|
167
219
|
timeout = get_timeout(body)
|
|
168
220
|
limit, offset, page = get_pagination()
|
|
169
221
|
|
|
170
222
|
entry = {'executed_at': store.now(), 'connection_name': connection_name}
|
|
171
223
|
try:
|
|
172
|
-
result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset,
|
|
224
|
+
result, elapsed_ms = engine.timed(engine.execute_sql, sql, connection_name, limit, offset, values, timeout)
|
|
173
225
|
except ApiError as error:
|
|
174
226
|
store.record_execution(path, number, {**entry, 'status': 'error', 'error': error.message})
|
|
175
227
|
raise
|
|
@@ -214,6 +266,11 @@ def save_sql_to_file():
|
|
|
214
266
|
if not isinstance(tags, (list, str)):
|
|
215
267
|
raise ApiError('tags must be a string or a list')
|
|
216
268
|
query_parameters = get_object(data.get('query_parameters'), 'query_parameters')
|
|
269
|
+
param_rules.parse_definitions(query_parameters)
|
|
270
|
+
unused = sorted(set(query_parameters) - set(sqltools.placeholder_names(data['sql_query'])))
|
|
271
|
+
if unused:
|
|
272
|
+
raise ApiError(f"query_parameters declares {', '.join(unused)}, which sql_query does not use "
|
|
273
|
+
'(write :name in the SQL, or remove the declaration)')
|
|
217
274
|
connection_name = data.get('connection_name')
|
|
218
275
|
if connection_name is not None and not isinstance(connection_name, str):
|
|
219
276
|
raise ApiError('connection_name must be a string')
|
|
@@ -304,10 +361,31 @@ def health():
|
|
|
304
361
|
return jsonify({'status': 'ok', 'version': current_app.config['SQL2API_VERSION']})
|
|
305
362
|
|
|
306
363
|
|
|
364
|
+
def describe_saved_queries():
|
|
365
|
+
"""What the OpenAPI document needs to know about each saved query (never its SQL text)."""
|
|
366
|
+
described = []
|
|
367
|
+
for name, number, data in store.latest_versions():
|
|
368
|
+
sql = data.get('sql_query')
|
|
369
|
+
if not isinstance(sql, str):
|
|
370
|
+
continue
|
|
371
|
+
declared = param_rules.read_definitions(data.get('query_parameters'))
|
|
372
|
+
used = sqltools.placeholder_names(sql)
|
|
373
|
+
parameters = {}
|
|
374
|
+
for param in used: # what the SQL needs, in order; undeclared ones are plain required text
|
|
375
|
+
parameters[param] = declared.get(param) or param_rules.read_definition({})
|
|
376
|
+
described.append({'name': name, 'version': number, 'description': data.get('description'),
|
|
377
|
+
'tags': data.get('tags'), 'connection_name': data.get('connection_name'),
|
|
378
|
+
'parameters': parameters})
|
|
379
|
+
return described
|
|
380
|
+
|
|
381
|
+
|
|
307
382
|
@bp.route('/openapi.json', methods=['GET'])
|
|
308
383
|
def openapi_spec():
|
|
309
384
|
from flask import current_app
|
|
310
|
-
|
|
385
|
+
# The generic API description is public. The list of saved queries (names, descriptions, parameters) is only
|
|
386
|
+
# shown to callers who could list them anyway, so an API key protects it too.
|
|
387
|
+
saved = describe_saved_queries() if has_valid_key() else None
|
|
388
|
+
return jsonify(openapi.build_spec(current_app.config['SQL2API_VERSION'], saved))
|
|
311
389
|
|
|
312
390
|
|
|
313
391
|
@bp.route('/docs', methods=['GET'])
|
sql2api/cli.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import argparse
|
|
3
3
|
import logging
|
|
4
4
|
import os
|
|
5
|
+
import sys
|
|
5
6
|
|
|
6
7
|
from . import __version__, config, store
|
|
7
8
|
from .app import create_app
|
|
@@ -10,7 +11,11 @@ LOOPBACK_HOSTS = ('127.0.0.1', 'localhost', '::1')
|
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def _serve(args):
|
|
13
|
-
|
|
14
|
+
try:
|
|
15
|
+
app = create_app()
|
|
16
|
+
except ValueError as error: # a malformed setting, e.g. SQL2API_RATE_LIMIT
|
|
17
|
+
print(f'sql2api: {error}', file=sys.stderr)
|
|
18
|
+
return 2
|
|
14
19
|
if args.host not in LOOPBACK_HOSTS and not config.api_key():
|
|
15
20
|
logging.getLogger('sql2api').warning(
|
|
16
21
|
'Listening on %s without SQL2API_API_KEY set: anyone who can reach this port can run SQL '
|
sql2api/config.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Runtime configuration, read from environment variables at call time."""
|
|
2
2
|
import os
|
|
3
|
+
import re
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
SUPPORTED_DB_TYPES = ('mysql', 'postgres', 'clickhouse', 'sqlite', 'h2')
|
|
@@ -94,6 +95,60 @@ def pool_idle_timeout():
|
|
|
94
95
|
return value if value > 0 else DEFAULT_POOL_IDLE_TIMEOUT
|
|
95
96
|
|
|
96
97
|
|
|
98
|
+
def cors_origins():
|
|
99
|
+
"""Origins allowed to call the API from a browser (SQL2API_CORS_ORIGINS): None (off), '*' or a frozenset.
|
|
100
|
+
|
|
101
|
+
Entries are compared case-insensitively and without a trailing slash, e.g. https://app.example.com.
|
|
102
|
+
"""
|
|
103
|
+
raw = os.environ.get('SQL2API_CORS_ORIGINS', '').strip()
|
|
104
|
+
if not raw:
|
|
105
|
+
return None
|
|
106
|
+
origins = [item.strip().rstrip('/').lower() for item in raw.split(',') if item.strip()]
|
|
107
|
+
if '*' in origins:
|
|
108
|
+
return '*'
|
|
109
|
+
return frozenset(origins) or None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
_RATE_PERIODS = {'second': 1, 'minute': 60, 'hour': 3600, 'day': 86400}
|
|
113
|
+
_RATE_RE = re.compile(r'^\s*(\d+)\s*/\s*(second|minute|hour|day)s?\s*$', re.I)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def parse_rate_limit(text):
|
|
117
|
+
"""Parse '60/minute' (also second, hour, day) into (requests, seconds); raises ValueError when malformed."""
|
|
118
|
+
match = _RATE_RE.match(text or '')
|
|
119
|
+
if not match or int(match.group(1)) < 1:
|
|
120
|
+
raise ValueError(f"SQL2API_RATE_LIMIT must look like '60/minute' (a positive count, then second, minute, "
|
|
121
|
+
f"hour or day), not {text!r}")
|
|
122
|
+
return int(match.group(1)), _RATE_PERIODS[match.group(2).lower()]
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def rate_limit():
|
|
126
|
+
"""(requests, seconds) allowed per client (SQL2API_RATE_LIMIT), or None when limiting is off."""
|
|
127
|
+
raw = os.environ.get('SQL2API_RATE_LIMIT', '').strip()
|
|
128
|
+
if not raw:
|
|
129
|
+
return None
|
|
130
|
+
try:
|
|
131
|
+
return parse_rate_limit(raw)
|
|
132
|
+
except ValueError:
|
|
133
|
+
return None # create_app() rejects a malformed value at startup; never limit by accident afterwards
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def proxy_hops():
|
|
137
|
+
"""Reverse proxies in front of the app whose X-Forwarded-* headers can be trusted (SQL2API_TRUST_PROXY)."""
|
|
138
|
+
try:
|
|
139
|
+
return max(0, int(os.environ.get('SQL2API_TRUST_PROXY', 0)))
|
|
140
|
+
except ValueError:
|
|
141
|
+
return 0
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def check_settings():
|
|
145
|
+
"""Raise ValueError for a malformed setting, so a typo fails at startup instead of silently switching off a
|
|
146
|
+
protection."""
|
|
147
|
+
raw = os.environ.get('SQL2API_RATE_LIMIT', '').strip()
|
|
148
|
+
if raw:
|
|
149
|
+
parse_rate_limit(raw)
|
|
150
|
+
|
|
151
|
+
|
|
97
152
|
def max_page_size():
|
|
98
153
|
try:
|
|
99
154
|
return max(1, int(os.environ.get('SQL2API_MAX_PAGE_SIZE', 1000)))
|
sql2api/cors.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Cross-origin (CORS) support for browser clients, off unless SQL2API_CORS_ORIGINS is set.
|
|
2
|
+
|
|
3
|
+
Without it a web page on another origin cannot call the API: browsers refuse to send the JSON requests and to read
|
|
4
|
+
the answers. CORS only tells the *browser* which sites may call; it is not authentication (use SQL2API_API_KEY).
|
|
5
|
+
"""
|
|
6
|
+
from flask import Response
|
|
7
|
+
|
|
8
|
+
from . import config
|
|
9
|
+
|
|
10
|
+
ALLOWED_METHODS = 'GET, POST, PATCH, DELETE, OPTIONS'
|
|
11
|
+
ALLOWED_HEADERS = 'Content-Type, X-API-Key'
|
|
12
|
+
# Browsers hide response headers a page has not been told about, and pagination depends on these.
|
|
13
|
+
EXPOSED_HEADERS = 'X-Page, X-Page-Size, X-Has-More, X-RateLimit-Limit, X-RateLimit-Remaining, Retry-After'
|
|
14
|
+
PREFLIGHT_MAX_AGE = '600'
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def allow_origin_value(origin):
|
|
18
|
+
"""The Access-Control-Allow-Origin value for a request from ``origin``, or None when it is not allowed."""
|
|
19
|
+
allowed = config.cors_origins()
|
|
20
|
+
if allowed is None or not origin:
|
|
21
|
+
return None
|
|
22
|
+
if allowed == '*':
|
|
23
|
+
return '*'
|
|
24
|
+
return origin if origin.rstrip('/').lower() in allowed else None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def is_preflight(request):
|
|
28
|
+
return (request.method == 'OPTIONS' and 'Access-Control-Request-Method' in request.headers
|
|
29
|
+
and config.cors_origins() is not None)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def preflight_response(origin):
|
|
33
|
+
"""Answer a browser's permission check before it sends the real request (no auth: it carries no key)."""
|
|
34
|
+
response = Response(status=204)
|
|
35
|
+
value = allow_origin_value(origin)
|
|
36
|
+
if value:
|
|
37
|
+
response.headers['Access-Control-Allow-Methods'] = ALLOWED_METHODS
|
|
38
|
+
response.headers['Access-Control-Allow-Headers'] = ALLOWED_HEADERS
|
|
39
|
+
response.headers['Access-Control-Max-Age'] = PREFLIGHT_MAX_AGE
|
|
40
|
+
add_headers(response, origin)
|
|
41
|
+
return response
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def add_headers(response, origin):
|
|
45
|
+
value = allow_origin_value(origin)
|
|
46
|
+
if value:
|
|
47
|
+
response.headers['Access-Control-Allow-Origin'] = value
|
|
48
|
+
response.headers['Access-Control-Expose-Headers'] = EXPOSED_HEADERS
|
|
49
|
+
if value != '*':
|
|
50
|
+
response.headers.add('Vary', 'Origin') # the answer depends on who asked
|
|
51
|
+
return response
|
sql2api/openapi.py
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
"""OpenAPI description of the API, served at /openapi.json and rendered at /docs."""
|
|
2
|
+
import re
|
|
3
|
+
from urllib.parse import quote
|
|
4
|
+
|
|
2
5
|
from . import config
|
|
6
|
+
from .params import json_schema
|
|
3
7
|
|
|
4
8
|
_FORMAT_PARAM = {'name': 'format', 'in': 'query', 'schema': {
|
|
5
9
|
'type': 'string', 'enum': ['json', 'ndjson', 'csv', 'tsv', 'xml', 'yaml', 'xlsx'], 'default': 'json'}}
|
|
@@ -7,7 +11,7 @@ _PAGE_PARAMS = [
|
|
|
7
11
|
{'name': 'page', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 1}},
|
|
8
12
|
{'name': 'page_size', 'in': 'query', 'schema': {'type': 'integer', 'minimum': 1, 'default': 10},
|
|
9
13
|
'description': 'Upper limit is SQL2API_MAX_PAGE_SIZE (default 1000).'},
|
|
10
|
-
{'name': 'timeout', 'in': 'query', 'schema': {'type': 'number', 'exclusiveMinimum':
|
|
14
|
+
{'name': 'timeout', 'in': 'query', 'schema': {'type': 'number', 'minimum': 0, 'exclusiveMinimum': True},
|
|
11
15
|
'description': 'Seconds the query may run before it is cancelled (504). Can lower, never raise, the server '
|
|
12
16
|
'limit SQL2API_QUERY_TIMEOUT (default 30; 0 disables the server limit).'},
|
|
13
17
|
]
|
|
@@ -15,12 +19,19 @@ _ROWS = {'description': 'The requested page. Header X-Has-More says whether anot
|
|
|
15
19
|
'headers': {'X-Page': {'schema': {'type': 'integer'}}, 'X-Page-Size': {'schema': {'type': 'integer'}},
|
|
16
20
|
'X-Has-More': {'schema': {'type': 'string', 'enum': ['true', 'false']}}},
|
|
17
21
|
'content': {'application/json': {'schema': {'type': 'array', 'items': {'type': 'object'}}}}}
|
|
18
|
-
_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '500', '504')}
|
|
22
|
+
_ERRORS = {c: {'$ref': '#/components/responses/Error'} for c in ('400', '401', '403', '404', '429', '500', '504')}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _object_schema(properties, required=()):
|
|
26
|
+
"""An object schema; OpenAPI 3.0 forbids an empty ``required`` list, so it is only included when non-empty."""
|
|
27
|
+
schema = {'type': 'object', 'properties': properties}
|
|
28
|
+
if required:
|
|
29
|
+
schema['required'] = list(required)
|
|
30
|
+
return schema
|
|
19
31
|
|
|
20
32
|
|
|
21
33
|
def _body(properties, required):
|
|
22
|
-
return {'required': True, 'content': {'application/json': {'schema':
|
|
23
|
-
'type': 'object', 'required': required, 'properties': properties}}}}
|
|
34
|
+
return {'required': True, 'content': {'application/json': {'schema': _object_schema(properties, required)}}}
|
|
24
35
|
|
|
25
36
|
|
|
26
37
|
_EXEC_PROPS = {
|
|
@@ -32,10 +43,58 @@ _EXEC_PROPS = {
|
|
|
32
43
|
}
|
|
33
44
|
|
|
34
45
|
|
|
35
|
-
def
|
|
46
|
+
def _saved_query_paths(queries):
|
|
47
|
+
"""One documented endpoint per saved query, with its declared parameters and rules."""
|
|
48
|
+
paths, seen = {}, set()
|
|
49
|
+
for query in queries:
|
|
50
|
+
name = query['name']
|
|
51
|
+
has_default_connection = bool(query.get('connection_name'))
|
|
52
|
+
properties, required, query_params = {}, [], []
|
|
53
|
+
for param, spec in query['parameters'].items():
|
|
54
|
+
schema = json_schema(spec)
|
|
55
|
+
entry = {'name': param, 'in': 'query', 'required': spec['required'], 'schema': schema}
|
|
56
|
+
if spec['description']:
|
|
57
|
+
entry['description'] = spec['description']
|
|
58
|
+
query_params.append(entry)
|
|
59
|
+
properties[param] = {**schema, **({'description': spec['description']} if spec['description'] else {})}
|
|
60
|
+
if spec['required']:
|
|
61
|
+
required.append(param)
|
|
62
|
+
connection = {'name': 'connection_name', 'in': 'query', 'required': not has_default_connection,
|
|
63
|
+
'schema': {'type': 'string', **({'default': query['connection_name']}
|
|
64
|
+
if has_default_connection else {})}}
|
|
65
|
+
common = [connection, {'name': 'version', 'in': 'query', 'schema': {'type': 'integer'},
|
|
66
|
+
'description': f"Saved version to run (default: latest, currently {query['version']})."},
|
|
67
|
+
_FORMAT_PARAM, *_PAGE_PARAMS]
|
|
68
|
+
summary = query.get('description') or f'Run the saved query {name}'
|
|
69
|
+
tags = query.get('tags')
|
|
70
|
+
detail = f"Runs version {query['version']} of the saved query '{name}'."
|
|
71
|
+
if tags:
|
|
72
|
+
detail += ' Tags: ' + (tags if isinstance(tags, str) else ', '.join(map(str, tags))) + '.'
|
|
73
|
+
operation_id = 'run_' + re.sub(r'\W', '_', name)
|
|
74
|
+
while operation_id in seen:
|
|
75
|
+
operation_id += '_'
|
|
76
|
+
seen.add(operation_id)
|
|
77
|
+
body_properties = {'params': _object_schema(properties, required),
|
|
78
|
+
'connection_name': connection['schema'], 'version': {'type': 'integer'},
|
|
79
|
+
'format': {'type': 'string'}, 'timeout': {'type': 'number'}}
|
|
80
|
+
paths['/q/' + quote(name, safe='')] = {
|
|
81
|
+
'get': {'summary': summary, 'description': detail, 'tags': ['Saved queries (live)'],
|
|
82
|
+
'operationId': operation_id, 'parameters': [*query_params, *common],
|
|
83
|
+
'responses': {'200': _ROWS, **_ERRORS}},
|
|
84
|
+
'post': {'summary': summary + ' (parameters in a JSON body)', 'description': detail,
|
|
85
|
+
'tags': ['Saved queries (live)'], 'operationId': operation_id + '_post',
|
|
86
|
+
'parameters': [_FORMAT_PARAM, *_PAGE_PARAMS],
|
|
87
|
+
'requestBody': {'required': bool(required), 'content': {'application/json': {
|
|
88
|
+
'schema': _object_schema(body_properties)}}},
|
|
89
|
+
'responses': {'200': _ROWS, **_ERRORS}},
|
|
90
|
+
}
|
|
91
|
+
return paths
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def build_spec(version, saved_queries=None):
|
|
36
95
|
saved = {'filepath': {'type': 'string', 'description': 'Saved query name or path inside saved_sql/.'},
|
|
37
96
|
**_EXEC_PROPS}
|
|
38
|
-
|
|
97
|
+
spec = {
|
|
39
98
|
'openapi': '3.0.3',
|
|
40
99
|
'info': {'title': 'SQL2API', 'version': version,
|
|
41
100
|
'description': 'Run SQL against configured databases and get the results back over HTTP.'},
|
|
@@ -119,12 +178,42 @@ def build_spec(version):
|
|
|
119
178
|
'responses': {'200': {'description': 'OK'}}}},
|
|
120
179
|
},
|
|
121
180
|
}
|
|
181
|
+
if saved_queries:
|
|
182
|
+
spec['paths'].update(_saved_query_paths(saved_queries))
|
|
183
|
+
spec['tags'] = [{'name': 'Saved queries (live)',
|
|
184
|
+
'description': 'One endpoint per saved query, generated from its latest version: its '
|
|
185
|
+
'declared parameters and their rules, and its default connection.'}]
|
|
186
|
+
return spec
|
|
122
187
|
|
|
123
188
|
|
|
124
189
|
DOCS_HTML = """<!doctype html>
|
|
125
190
|
<html><head><meta charset="utf-8"><title>SQL2API docs</title>
|
|
126
|
-
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css"
|
|
127
|
-
<
|
|
191
|
+
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css">
|
|
192
|
+
<style>
|
|
193
|
+
body { margin: 0; font-family: sans-serif; }
|
|
194
|
+
#key-bar { padding: 8px 16px; background: #f4f4f4; border-bottom: 1px solid #ddd; font-size: 14px; }
|
|
195
|
+
#key-bar input { width: 260px; padding: 4px; }
|
|
196
|
+
</style></head>
|
|
197
|
+
<body>
|
|
198
|
+
<div id="key-bar">
|
|
199
|
+
API key (only needed if the server sets SQL2API_API_KEY; it reveals your saved queries below and is
|
|
200
|
+
sent with "Try it out" requests; kept for this browser tab only):
|
|
201
|
+
<input id="key" type="password" autocomplete="off" placeholder="X-API-Key">
|
|
202
|
+
<button id="save-key">Apply</button>
|
|
203
|
+
</div>
|
|
204
|
+
<div id="ui"></div>
|
|
128
205
|
<script src="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui-bundle.js"></script>
|
|
129
|
-
<script>
|
|
206
|
+
<script>
|
|
207
|
+
var stored = '';
|
|
208
|
+
try { stored = sessionStorage.getItem('sql2api-key') || ''; } catch (e) {}
|
|
209
|
+
document.getElementById('key').value = stored;
|
|
210
|
+
document.getElementById('save-key').onclick = function () {
|
|
211
|
+
try { sessionStorage.setItem('sql2api-key', document.getElementById('key').value); } catch (e) {}
|
|
212
|
+
location.reload();
|
|
213
|
+
};
|
|
214
|
+
SwaggerUIBundle({
|
|
215
|
+
url: 'openapi.json', dom_id: '#ui',
|
|
216
|
+
requestInterceptor: function (req) { if (stored) { req.headers['X-API-Key'] = stored; } return req; }
|
|
217
|
+
});
|
|
218
|
+
</script></body></html>
|
|
130
219
|
"""
|
sql2api/params.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Parameter definitions of a saved query: checking the definitions, and enforcing them on request values.
|
|
2
|
+
|
|
3
|
+
A saved query declares its parameters in ``query_parameters``. The short form is just a type
|
|
4
|
+
(``{"id": "int"}``); the full form adds rules::
|
|
5
|
+
|
|
6
|
+
{
|
|
7
|
+
"rating": {"type": "str", "enum": ["G", "PG", "R"], "default": "PG", "description": "MPAA rating"},
|
|
8
|
+
"max_length": {"type": "int", "min": 1, "max": 600},
|
|
9
|
+
"title": {"type": "str", "required": false, "min_length": 2, "pattern": "[A-Za-z ]+"}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
Rules: ``type`` (int, float, str, bool), ``required`` (default: true unless a ``default`` is given), ``default``,
|
|
13
|
+
``enum``, ``min``/``max`` (numbers), ``min_length``/``max_length``/``pattern`` (text) and ``description``.
|
|
14
|
+
An optional parameter that is neither supplied nor defaulted is bound as NULL.
|
|
15
|
+
|
|
16
|
+
Definitions are validated strictly when a query is saved. At run time they are read leniently, so older saved
|
|
17
|
+
files with unusual definitions keep working.
|
|
18
|
+
"""
|
|
19
|
+
import math
|
|
20
|
+
import re
|
|
21
|
+
|
|
22
|
+
from .errors import ApiError
|
|
23
|
+
|
|
24
|
+
_TYPES = {'int': 'integer', 'integer': 'integer', 'float': 'number', 'number': 'number',
|
|
25
|
+
'str': 'string', 'string': 'string', 'bool': 'boolean', 'boolean': 'boolean'}
|
|
26
|
+
_RULE_KEYS = {'type', 'required', 'default', 'enum', 'min', 'max', 'min_length', 'max_length', 'pattern',
|
|
27
|
+
'description'}
|
|
28
|
+
_NAME_RE = re.compile(r'^[A-Za-z_]\w*$')
|
|
29
|
+
_MAX_PATTERN_LENGTH = 500
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class _Invalid(Exception):
|
|
33
|
+
"""A value or a definition that breaks a rule; the message completes the sentence '<name> ...'."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _to_bool(text):
|
|
37
|
+
lowered = text.strip().lower()
|
|
38
|
+
if lowered in ('true', '1', 'yes'):
|
|
39
|
+
return True
|
|
40
|
+
if lowered in ('false', '0', 'no'):
|
|
41
|
+
return False
|
|
42
|
+
raise ValueError(text)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _is_number(value):
|
|
46
|
+
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _plain(spec):
|
|
50
|
+
return {'type': None, 'required': True, 'has_default': False, 'default': None, 'enum': None, 'min': None,
|
|
51
|
+
'max': None, 'min_length': None, 'max_length': None, 'pattern': None, 'description': None, **spec}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _coerce(type_name, value):
|
|
55
|
+
"""Convert a value (usually a query-string text) to the declared type, or raise _Invalid."""
|
|
56
|
+
if type_name == 'integer':
|
|
57
|
+
if isinstance(value, str):
|
|
58
|
+
try:
|
|
59
|
+
return int(value)
|
|
60
|
+
except ValueError:
|
|
61
|
+
raise _Invalid('must be an integer') from None
|
|
62
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
63
|
+
return value
|
|
64
|
+
raise _Invalid('must be an integer')
|
|
65
|
+
if type_name == 'number':
|
|
66
|
+
if isinstance(value, str):
|
|
67
|
+
try:
|
|
68
|
+
value = float(value)
|
|
69
|
+
except ValueError:
|
|
70
|
+
raise _Invalid('must be a number') from None
|
|
71
|
+
if _is_number(value):
|
|
72
|
+
return value
|
|
73
|
+
raise _Invalid('must be a number')
|
|
74
|
+
if type_name == 'boolean':
|
|
75
|
+
if isinstance(value, str):
|
|
76
|
+
try:
|
|
77
|
+
return _to_bool(value)
|
|
78
|
+
except ValueError:
|
|
79
|
+
raise _Invalid('must be true or false') from None
|
|
80
|
+
if isinstance(value, bool):
|
|
81
|
+
return value
|
|
82
|
+
raise _Invalid('must be true or false')
|
|
83
|
+
if type_name == 'string':
|
|
84
|
+
if isinstance(value, str):
|
|
85
|
+
return value
|
|
86
|
+
raise _Invalid('must be text')
|
|
87
|
+
if isinstance(value, (str, bool)) or _is_number(value): # untyped: any scalar
|
|
88
|
+
return value
|
|
89
|
+
raise _Invalid('must be text, a number or true/false')
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _check(spec, value):
|
|
93
|
+
"""Return the value converted to the declared type after applying every rule, or raise _Invalid."""
|
|
94
|
+
value = _coerce(spec['type'], value)
|
|
95
|
+
if spec['enum'] is not None and value not in spec['enum']:
|
|
96
|
+
raise _Invalid('must be one of: ' + ', '.join(str(v) for v in spec['enum']))
|
|
97
|
+
if _is_number(value):
|
|
98
|
+
if spec['min'] is not None and value < spec['min']:
|
|
99
|
+
raise _Invalid(f"must be at least {spec['min']:g}")
|
|
100
|
+
if spec['max'] is not None and value > spec['max']:
|
|
101
|
+
raise _Invalid(f"must be at most {spec['max']:g}")
|
|
102
|
+
if isinstance(value, str):
|
|
103
|
+
if spec['min_length'] is not None and len(value) < spec['min_length']:
|
|
104
|
+
raise _Invalid(f"must be at least {spec['min_length']} characters long")
|
|
105
|
+
if spec['max_length'] is not None and len(value) > spec['max_length']:
|
|
106
|
+
raise _Invalid(f"must be at most {spec['max_length']} characters long")
|
|
107
|
+
if spec['pattern'] is not None and not re.fullmatch(spec['pattern'], value):
|
|
108
|
+
raise _Invalid(f"must match the pattern {spec['pattern']}")
|
|
109
|
+
return value
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# --------------------------------------------------------------------------------------
|
|
113
|
+
# Definitions
|
|
114
|
+
# --------------------------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
def _parse_strict(spec):
|
|
117
|
+
if isinstance(spec, str):
|
|
118
|
+
spec = {'type': spec}
|
|
119
|
+
if not isinstance(spec, dict):
|
|
120
|
+
raise _Invalid("must be a type name such as 'int' or an object of rules")
|
|
121
|
+
unknown = sorted(set(spec) - _RULE_KEYS)
|
|
122
|
+
if unknown:
|
|
123
|
+
raise _Invalid(f"has unknown rule(s): {', '.join(unknown)} (allowed: {', '.join(sorted(_RULE_KEYS))})")
|
|
124
|
+
|
|
125
|
+
type_name = None
|
|
126
|
+
if spec.get('type') is not None:
|
|
127
|
+
type_name = _TYPES.get(str(spec['type']).lower())
|
|
128
|
+
if type_name is None:
|
|
129
|
+
raise _Invalid(f"has unknown type '{spec['type']}' (use int, float, str or bool)")
|
|
130
|
+
parsed = _plain({'type': type_name})
|
|
131
|
+
|
|
132
|
+
if 'required' in spec:
|
|
133
|
+
if not isinstance(spec['required'], bool):
|
|
134
|
+
raise _Invalid("'required' must be true or false")
|
|
135
|
+
parsed['required'] = spec['required']
|
|
136
|
+
elif 'default' in spec:
|
|
137
|
+
parsed['required'] = False
|
|
138
|
+
if 'description' in spec:
|
|
139
|
+
if not isinstance(spec['description'], str):
|
|
140
|
+
raise _Invalid("'description' must be text")
|
|
141
|
+
parsed['description'] = spec['description']
|
|
142
|
+
|
|
143
|
+
numeric = type_name in (None, 'integer', 'number')
|
|
144
|
+
textual = type_name in (None, 'string')
|
|
145
|
+
for key in ('min', 'max'):
|
|
146
|
+
if key in spec:
|
|
147
|
+
if not numeric:
|
|
148
|
+
raise _Invalid(f"'{key}' only applies to numbers (use min_length/max_length for text)")
|
|
149
|
+
if not _is_number(spec[key]):
|
|
150
|
+
raise _Invalid(f"'{key}' must be a number")
|
|
151
|
+
parsed[key] = spec[key]
|
|
152
|
+
if parsed['min'] is not None and parsed['max'] is not None and parsed['min'] > parsed['max']:
|
|
153
|
+
raise _Invalid("'min' must not be greater than 'max'")
|
|
154
|
+
for key in ('min_length', 'max_length'):
|
|
155
|
+
if key in spec:
|
|
156
|
+
if not textual:
|
|
157
|
+
raise _Invalid(f"'{key}' only applies to text")
|
|
158
|
+
if not isinstance(spec[key], int) or isinstance(spec[key], bool) or spec[key] < 0:
|
|
159
|
+
raise _Invalid(f"'{key}' must be a non-negative integer")
|
|
160
|
+
parsed[key] = spec[key]
|
|
161
|
+
if (parsed['min_length'] is not None and parsed['max_length'] is not None
|
|
162
|
+
and parsed['min_length'] > parsed['max_length']):
|
|
163
|
+
raise _Invalid("'min_length' must not be greater than 'max_length'")
|
|
164
|
+
if 'pattern' in spec:
|
|
165
|
+
if not textual:
|
|
166
|
+
raise _Invalid("'pattern' only applies to text")
|
|
167
|
+
if not isinstance(spec['pattern'], str) or len(spec['pattern']) > _MAX_PATTERN_LENGTH:
|
|
168
|
+
raise _Invalid(f"'pattern' must be text of at most {_MAX_PATTERN_LENGTH} characters")
|
|
169
|
+
try:
|
|
170
|
+
re.compile(spec['pattern'])
|
|
171
|
+
except re.error as error:
|
|
172
|
+
raise _Invalid(f"'pattern' is not a valid regular expression ({error})") from None
|
|
173
|
+
parsed['pattern'] = spec['pattern']
|
|
174
|
+
|
|
175
|
+
if 'enum' in spec:
|
|
176
|
+
if not isinstance(spec['enum'], list) or not spec['enum']:
|
|
177
|
+
raise _Invalid("'enum' must be a non-empty list")
|
|
178
|
+
try:
|
|
179
|
+
parsed['enum'] = [_coerce(type_name, v) for v in spec['enum']]
|
|
180
|
+
except _Invalid as error:
|
|
181
|
+
raise _Invalid(f"has an 'enum' value that {error}") from None
|
|
182
|
+
if 'default' in spec:
|
|
183
|
+
if spec.get('required') is True:
|
|
184
|
+
raise _Invalid("cannot be both 'required' and have a 'default'")
|
|
185
|
+
try:
|
|
186
|
+
parsed['default'] = _check(parsed, spec['default'])
|
|
187
|
+
except _Invalid as error:
|
|
188
|
+
raise _Invalid(f"has a 'default' that {error}") from None
|
|
189
|
+
parsed['has_default'] = True
|
|
190
|
+
return parsed
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def parse_definitions(query_parameters):
|
|
194
|
+
"""Validate the ``query_parameters`` of a query about to be saved; raises ApiError(400) listing every problem."""
|
|
195
|
+
if not isinstance(query_parameters, dict):
|
|
196
|
+
raise ApiError('query_parameters must be a JSON object')
|
|
197
|
+
errors = {}
|
|
198
|
+
for name, spec in query_parameters.items():
|
|
199
|
+
if not _NAME_RE.match(name):
|
|
200
|
+
errors[name] = 'is not a valid parameter name (letters, digits and underscores; not starting with a digit)'
|
|
201
|
+
continue
|
|
202
|
+
try:
|
|
203
|
+
_parse_strict(spec)
|
|
204
|
+
except _Invalid as error:
|
|
205
|
+
errors[name] = str(error)
|
|
206
|
+
if errors:
|
|
207
|
+
raise ApiError('Invalid query_parameters: ' + '; '.join(f'{n} {m}' for n, m in errors.items()), 400,
|
|
208
|
+
errors=errors)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def read_definition(spec):
|
|
212
|
+
"""A definition as the normalised rule dict; anything unusable degrades to an untyped, required parameter."""
|
|
213
|
+
try:
|
|
214
|
+
return _parse_strict(spec)
|
|
215
|
+
except _Invalid:
|
|
216
|
+
if isinstance(spec, str):
|
|
217
|
+
return _plain({'type': _TYPES.get(spec.lower())})
|
|
218
|
+
return _plain({})
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def read_definitions(query_parameters):
|
|
222
|
+
if not isinstance(query_parameters, dict):
|
|
223
|
+
return {}
|
|
224
|
+
return {name: read_definition(spec) for name, spec in query_parameters.items()}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
# --------------------------------------------------------------------------------------
|
|
228
|
+
# Values
|
|
229
|
+
# --------------------------------------------------------------------------------------
|
|
230
|
+
|
|
231
|
+
def resolve(declared, supplied, used=None):
|
|
232
|
+
"""Apply a saved query's parameter definitions to the values a request supplied.
|
|
233
|
+
|
|
234
|
+
Returns every value to bind: supplied ones converted and checked, defaults filled in, optional ones without
|
|
235
|
+
a value set to None. Values for undeclared parameters pass through untouched. Declared parameters that the
|
|
236
|
+
SQL does not use (``used`` is the set of names it does use) are not enforced. Raises ApiError(400) listing
|
|
237
|
+
every violation.
|
|
238
|
+
"""
|
|
239
|
+
values = dict(supplied)
|
|
240
|
+
errors = {}
|
|
241
|
+
for name, spec in read_definitions(declared).items():
|
|
242
|
+
if used is not None and name not in used:
|
|
243
|
+
continue
|
|
244
|
+
if name not in values:
|
|
245
|
+
if spec['has_default']:
|
|
246
|
+
values[name] = spec['default']
|
|
247
|
+
elif spec['required']:
|
|
248
|
+
errors[name] = 'is required'
|
|
249
|
+
else:
|
|
250
|
+
values[name] = None
|
|
251
|
+
continue
|
|
252
|
+
if values[name] is None:
|
|
253
|
+
if spec['required']:
|
|
254
|
+
errors[name] = 'must not be null'
|
|
255
|
+
continue
|
|
256
|
+
try:
|
|
257
|
+
values[name] = _check(spec, values[name])
|
|
258
|
+
except _Invalid as error:
|
|
259
|
+
errors[name] = str(error)
|
|
260
|
+
if errors:
|
|
261
|
+
raise ApiError('Invalid parameters: ' + '; '.join(f'{n} {m}' for n, m in errors.items()), 400,
|
|
262
|
+
errors=errors)
|
|
263
|
+
return values
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def json_schema(spec):
|
|
267
|
+
"""The OpenAPI/JSON-schema description of a parameter's type and rules."""
|
|
268
|
+
schema = {'type': spec['type'] or 'string'}
|
|
269
|
+
for rule, key in (('enum', 'enum'), ('min', 'minimum'), ('max', 'maximum'), ('min_length', 'minLength'),
|
|
270
|
+
('max_length', 'maxLength'), ('pattern', 'pattern')):
|
|
271
|
+
if spec[rule] is not None:
|
|
272
|
+
schema[key] = spec[rule]
|
|
273
|
+
if spec['has_default']:
|
|
274
|
+
schema['default'] = spec['default']
|
|
275
|
+
return schema
|
sql2api/ratelimit.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""A small in-memory token-bucket rate limiter, off unless SQL2API_RATE_LIMIT is set (e.g. ``60/minute``).
|
|
2
|
+
|
|
3
|
+
Each client (by IP address) has a bucket holding up to the full quota, refilled continuously, so short bursts are
|
|
4
|
+
allowed but the sustained rate is capped. State lives in this process: with several worker processes each has its
|
|
5
|
+
own buckets, so the effective limit is multiplied by the number of workers.
|
|
6
|
+
"""
|
|
7
|
+
import math
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
from collections import OrderedDict
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RateLimiter:
|
|
14
|
+
MAX_CLIENTS = 100_000 # bounds memory; the least recently seen clients are forgotten first
|
|
15
|
+
|
|
16
|
+
def __init__(self, clock=time.monotonic):
|
|
17
|
+
self._clock = clock
|
|
18
|
+
self._lock = threading.Lock()
|
|
19
|
+
self._buckets = OrderedDict() # client -> [tokens, last_seen]; least recently seen first
|
|
20
|
+
self._spec = None
|
|
21
|
+
self._last_sweep = clock()
|
|
22
|
+
|
|
23
|
+
def hit(self, client, count, period):
|
|
24
|
+
"""Record a request. Returns (allowed, remaining, retry_after_seconds)."""
|
|
25
|
+
now = self._clock()
|
|
26
|
+
with self._lock:
|
|
27
|
+
if self._spec != (count, period): # limit changed: start again rather than mix old and new rules
|
|
28
|
+
self._buckets.clear()
|
|
29
|
+
self._spec = (count, period)
|
|
30
|
+
self._sweep(now, period)
|
|
31
|
+
tokens, seen = self._buckets.pop(client, (float(count), now))
|
|
32
|
+
tokens = min(float(count), tokens + (now - seen) * count / period)
|
|
33
|
+
allowed = tokens >= 1
|
|
34
|
+
if allowed:
|
|
35
|
+
tokens -= 1
|
|
36
|
+
self._buckets[client] = (tokens, now) # re-inserted last: most recently seen
|
|
37
|
+
if len(self._buckets) > self.MAX_CLIENTS:
|
|
38
|
+
self._buckets.popitem(last=False)
|
|
39
|
+
# round first: 20.000000000000004 seconds must not become a 21 second wait
|
|
40
|
+
retry_after = 0 if allowed else max(1, math.ceil(round((1 - tokens) * period / count, 6)))
|
|
41
|
+
return allowed, int(tokens), retry_after
|
|
42
|
+
|
|
43
|
+
def _sweep(self, now, period):
|
|
44
|
+
"""Forget clients idle for a full period: their bucket has refilled, so they look brand new anyway."""
|
|
45
|
+
if now - self._last_sweep < period:
|
|
46
|
+
return
|
|
47
|
+
self._last_sweep = now
|
|
48
|
+
while self._buckets:
|
|
49
|
+
client, (_, seen) = next(iter(self._buckets.items()))
|
|
50
|
+
if now - seen < period:
|
|
51
|
+
break
|
|
52
|
+
del self._buckets[client]
|
|
53
|
+
|
|
54
|
+
def size(self):
|
|
55
|
+
with self._lock:
|
|
56
|
+
return len(self._buckets)
|
sql2api/sqltools.py
CHANGED
|
@@ -51,37 +51,6 @@ def paginate(sql, limit, offset):
|
|
|
51
51
|
# Parameters
|
|
52
52
|
# --------------------------------------------------------------------------------------
|
|
53
53
|
|
|
54
|
-
def _to_bool(text):
|
|
55
|
-
lowered = text.strip().lower()
|
|
56
|
-
if lowered in ('true', '1', 'yes'):
|
|
57
|
-
return True
|
|
58
|
-
if lowered in ('false', '0', 'no'):
|
|
59
|
-
return False
|
|
60
|
-
raise ValueError(text)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
_COERCERS = {'int': int, 'integer': int, 'float': float, 'number': float, 'str': str, 'string': str,
|
|
64
|
-
'bool': _to_bool, 'boolean': _to_bool}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def coerce_params(declared, params):
|
|
68
|
-
"""Convert string values (e.g. from a query string) to the types a saved query declares.
|
|
69
|
-
|
|
70
|
-
``declared`` is the saved version's ``query_parameters``: ``{"actor_id": "int"}`` or
|
|
71
|
-
``{"actor_id": {"type": "int"}}``. Undeclared parameters are left untouched.
|
|
72
|
-
"""
|
|
73
|
-
result = dict(params)
|
|
74
|
-
for name, spec in (declared or {}).items():
|
|
75
|
-
type_name = spec.get('type') if isinstance(spec, dict) else spec
|
|
76
|
-
coerce = _COERCERS.get(str(type_name).lower())
|
|
77
|
-
if coerce and isinstance(result.get(name), str):
|
|
78
|
-
try:
|
|
79
|
-
result[name] = coerce(result[name])
|
|
80
|
-
except ValueError:
|
|
81
|
-
raise ApiError(f"Parameter '{name}' must be of type {type_name}") from None
|
|
82
|
-
return result
|
|
83
|
-
|
|
84
|
-
|
|
85
54
|
def _check_value(name, value):
|
|
86
55
|
if value is None or isinstance(value, (bool, str)):
|
|
87
56
|
return
|
|
@@ -125,6 +94,11 @@ def named_parameters(sql):
|
|
|
125
94
|
return [m.group('name') for m in _PARAM_RE.finditer(sql) if m.group('name')]
|
|
126
95
|
|
|
127
96
|
|
|
97
|
+
def placeholder_names(sql):
|
|
98
|
+
"""Every parameter name ``sql`` uses - bound ``:name`` first, then ``{name}`` text placeholders - once each."""
|
|
99
|
+
return list(dict.fromkeys(named_parameters(sql) + _BRACE_RE.findall(sql)))
|
|
100
|
+
|
|
101
|
+
|
|
128
102
|
_MARKERS = {'qmark': lambda name: '?', 'format': lambda name: '%s', 'pyformat': lambda name: f'%({name})s'}
|
|
129
103
|
|
|
130
104
|
|
sql2api/store.py
CHANGED
|
@@ -226,6 +226,24 @@ def record_execution(path, version, entry):
|
|
|
226
226
|
pass
|
|
227
227
|
|
|
228
228
|
|
|
229
|
+
def latest_versions():
|
|
230
|
+
"""(name, version number, data) for the newest version of every saved query; unreadable files are skipped."""
|
|
231
|
+
saved_dir = str(config.saved_sql_dir())
|
|
232
|
+
if not os.path.isdir(saved_dir):
|
|
233
|
+
return []
|
|
234
|
+
found = []
|
|
235
|
+
for filename in sorted(os.listdir(saved_dir)):
|
|
236
|
+
path = os.path.join(saved_dir, filename)
|
|
237
|
+
if not (filename.endswith('.json') and os.path.isfile(path)):
|
|
238
|
+
continue
|
|
239
|
+
try:
|
|
240
|
+
number, data = select_version(load_versions(path))
|
|
241
|
+
except ApiError:
|
|
242
|
+
continue
|
|
243
|
+
found.append((filename[:-5], number, data))
|
|
244
|
+
return found
|
|
245
|
+
|
|
246
|
+
|
|
229
247
|
def list_saved():
|
|
230
248
|
"""Return every saved query with its versions' metadata (SQL text is not included)."""
|
|
231
249
|
saved_dir = str(config.saved_sql_dir())
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sql2api
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Turn SQL into a REST API: run queries against MySQL, PostgreSQL, ClickHouse, SQLite or H2 over HTTP and get JSON, CSV, XML, YAML or XLSX back.
|
|
5
5
|
Author-email: Anantha Raju C <arcswdev@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -37,6 +37,7 @@ Requires-Dist: gunicorn>=21; extra == "server"
|
|
|
37
37
|
Provides-Extra: dev
|
|
38
38
|
Requires-Dist: sql2api[all]; extra == "dev"
|
|
39
39
|
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
40
|
+
Requires-Dist: openapi-spec-validator>=0.7; extra == "dev"
|
|
40
41
|
Dynamic: license-file
|
|
41
42
|
|
|
42
43
|
# SQL2API
|
|
@@ -74,6 +75,10 @@ film_id,title,rating,length
|
|
|
74
75
|
(or `?version=1`). Each run is recorded in the query's execution history.
|
|
75
76
|
- **Bound parameters** - write `WHERE id = :id` and the value is sent to the database separately from the SQL, so it
|
|
76
77
|
cannot inject anything. Declare types (`{"id": "int"}`) and query-string values are converted for you.
|
|
78
|
+
- **Parameter rules** - saved queries can declare defaults, optional parameters, allowed values, numeric ranges and
|
|
79
|
+
text patterns. Bad input is rejected with a field-by-field `400` before it reaches the database.
|
|
80
|
+
- **A live catalogue of your endpoints** - `/docs` lists every saved query as its own endpoint with its parameters and
|
|
81
|
+
rules. The SQL itself is never shown, and with an API key set the list is hidden from anonymous readers.
|
|
77
82
|
- **Pagination** - `?page=2&page_size=50`, with `X-Has-More` telling you whether another page exists.
|
|
78
83
|
- **Connection pooling** - MySQL, PostgreSQL, ClickHouse and H2 connections are reused between requests instead of
|
|
79
84
|
opened for each one (about 30x lower per-request overhead on MySQL and H2 against a local server; more over a network).
|
|
@@ -120,14 +125,19 @@ every supported database) and `saved_sql/`. Edit the file, set `"active": true`,
|
|
|
120
125
|
curl -X PATCH http://127.0.0.1:5000/save_sql_to_file -H 'Content-Type: application/json' -d '{
|
|
121
126
|
"filename": "actor_by_id",
|
|
122
127
|
"sql_query": "SELECT * FROM actor WHERE actor_id = :id",
|
|
123
|
-
"query_parameters": {"id": "int"},
|
|
128
|
+
"query_parameters": {"id": {"type": "int", "min": 1, "max": 200, "description": "Actor id"}},
|
|
124
129
|
"connection_name": "sakila-sqlite",
|
|
125
130
|
"author": "me", "description": "Look up an actor"
|
|
126
131
|
}'
|
|
127
132
|
|
|
128
133
|
curl 'http://127.0.0.1:5000/q/actor_by_id?id=7&format=yaml'
|
|
134
|
+
curl 'http://127.0.0.1:5000/q/actor_by_id?id=0'
|
|
135
|
+
# {"error": "Invalid parameters: id must be at least 1", "errors": {"id": "must be at least 1"}}
|
|
129
136
|
~~~
|
|
130
137
|
|
|
138
|
+
Rules: `type` (`int`, `float`, `str`, `bool`), `default`, `required`, `enum`, `min`/`max`, `min_length`/`max_length`,
|
|
139
|
+
`pattern` and `description` - see [the API reference](documentation/API.md#parameter-rules).
|
|
140
|
+
|
|
131
141
|
Saving again under the same name adds version 2; `DELETE /saved_sql/actor_by_id?version=1` removes one version.
|
|
132
142
|
|
|
133
143
|
## Configuration
|
|
@@ -140,6 +150,9 @@ Everything is configured through environment variables (all optional):
|
|
|
140
150
|
| `SQL2API_ALLOW_WRITES` | off | Allow `INSERT`/`UPDATE`/DDL. Otherwise only single read-only statements are accepted. |
|
|
141
151
|
| `SQL2API_API_KEY` | unset | When set, every request (except `/health` and `/docs`) needs a matching `X-API-Key` header. |
|
|
142
152
|
| `SQL2API_MAX_PAGE_SIZE` | `1000` | Upper limit for `page_size`. |
|
|
153
|
+
| `SQL2API_CORS_ORIGINS` | unset | Websites allowed to call the API from a browser: comma-separated origins such as `https://app.example.com`, or `*`. Off by default. |
|
|
154
|
+
| `SQL2API_RATE_LIMIT` | unset | Requests allowed per client address, e.g. `60/minute` (also `second`, `hour`, `day`). Off by default; a malformed value stops startup. |
|
|
155
|
+
| `SQL2API_TRUST_PROXY` | `0` | Number of reverse proxies in front of the app whose `X-Forwarded-*` headers are trusted. Set it (usually `1`) behind nginx, a load balancer or a platform router, or every client looks like the proxy. |
|
|
143
156
|
| `SQL2API_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |
|
|
144
157
|
| `SQL2API_POOL_IDLE_TIMEOUT` | `300` | Seconds an idle pooled connection is kept before it is closed. |
|
|
145
158
|
| `SQL2API_QUERY_TIMEOUT` | `30` | Seconds a query may run before it is cancelled (HTTP 504). `0` disables the limit. A request can lower it with `?timeout=`, never raise it. |
|
|
@@ -160,15 +173,59 @@ SQL2API runs whatever SQL it is given against your databases, so it ships locked
|
|
|
160
173
|
|
|
161
174
|
See [SECURITY.md](SECURITY.md) to report a vulnerability.
|
|
162
175
|
|
|
176
|
+
### Calling the API from a browser
|
|
177
|
+
|
|
178
|
+
Browsers refuse cross-origin JSON calls unless the server allows them. List the sites that may call the API:
|
|
179
|
+
|
|
180
|
+
~~~bash
|
|
181
|
+
SQL2API_API_KEY=change-me SQL2API_CORS_ORIGINS=https://app.example.com sql2api serve
|
|
182
|
+
~~~
|
|
183
|
+
|
|
184
|
+
Preflight checks are answered automatically, and the pagination headers (`X-Has-More` etc.) are exposed to the page's
|
|
185
|
+
JavaScript. CORS only tells the *browser* which sites may call; it is not authentication, so keep the API key. Avoid
|
|
186
|
+
`*` without a key: any website a visitor opens could then reach your databases through their browser (the server logs
|
|
187
|
+
a warning if you start that way).
|
|
188
|
+
|
|
189
|
+
### Rate limiting
|
|
190
|
+
|
|
191
|
+
`SQL2API_RATE_LIMIT=60/minute` gives each client address a bucket of 60 requests that refills steadily, so short bursts
|
|
192
|
+
work but the sustained rate is capped. Over the limit, requests get `429` with a `Retry-After` header, and every
|
|
193
|
+
response carries `X-RateLimit-Limit` and `X-RateLimit-Remaining`. The limit is applied before the API key check, so
|
|
194
|
+
guessing keys is throttled too; `/health` and CORS preflights are never counted. State is per process: with several
|
|
195
|
+
workers, the effective limit is multiplied by the number of workers.
|
|
196
|
+
|
|
163
197
|
## Docker
|
|
164
198
|
|
|
199
|
+
Every release is published to GitHub Container Registry for `linux/amd64` and `linux/arm64`:
|
|
200
|
+
|
|
201
|
+
~~~bash
|
|
202
|
+
docker run -p 5000:5000 -v "$PWD/data:/data" -e SQL2API_API_KEY=change-me ghcr.io/anantharajuc/sql2api:latest
|
|
203
|
+
~~~
|
|
204
|
+
|
|
205
|
+
| Tag | Contents |
|
|
206
|
+
|-----|----------|
|
|
207
|
+
| `X.Y.Z`, `latest` | SQL2API with the MySQL, PostgreSQL and ClickHouse drivers (SQLite is built in) |
|
|
208
|
+
| `X.Y.Z-h2`, `latest-h2` | The same plus Java and the H2 driver |
|
|
209
|
+
|
|
210
|
+
The container keeps `db_connections.json` and `saved_sql/` in `/data` (create a starter with
|
|
211
|
+
`docker run --rm -v "$PWD/data:/data" ghcr.io/anantharajuc/sql2api sql2api init`). It runs as a non-root user under
|
|
212
|
+
gunicorn with one worker (the files are protected by an in-process lock) and a health check on `/health`. Behind a
|
|
213
|
+
reverse proxy or load balancer, set `SQL2API_TRUST_PROXY=1`. To build it yourself:
|
|
214
|
+
`docker build -t sql2api .` (add `--build-arg WITH_H2=true` for H2).
|
|
215
|
+
|
|
216
|
+
### Try it with one command
|
|
217
|
+
|
|
218
|
+
[`docker-compose.yml`](docker-compose.yml) starts SQL2API in front of a PostgreSQL database seeded with sample films:
|
|
219
|
+
|
|
165
220
|
~~~bash
|
|
166
|
-
docker
|
|
167
|
-
|
|
221
|
+
docker compose up --build
|
|
222
|
+
curl -H 'X-API-Key: demo-key' 'http://127.0.0.1:5000/q/films_by_rating?rating=PG&max_length=90'
|
|
168
223
|
~~~
|
|
169
224
|
|
|
170
|
-
|
|
171
|
-
|
|
225
|
+
Open <http://127.0.0.1:5000/docs>, paste `demo-key` into the box at the top, and both saved queries appear as endpoints.
|
|
226
|
+
The demo listens on localhost only, mounts its configuration read-only, and reads the database password from an
|
|
227
|
+
environment variable (`${DEMO_DB_PASSWORD}` in [`demo/data/db_connections.json`](demo/data/db_connections.json)).
|
|
228
|
+
Clean up with `docker compose down -v`.
|
|
172
229
|
|
|
173
230
|
## API overview
|
|
174
231
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
sql2api/__init__.py,sha256=E0GamNI-uTZw-GcL8le7gE0DrJApGpI-v_wlIekgyN4,185
|
|
2
|
+
sql2api/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
|
|
3
|
+
sql2api/app.py,sha256=9WGV13ZSL0LydQ-vLitm1t9wvgDSAePfVQP6y37wK74,16342
|
|
4
|
+
sql2api/cli.py,sha256=-5wb3UHl1N2gFNY4ypyhOPasZQQgacf2P7XRmw3eWNo,2885
|
|
5
|
+
sql2api/config.py,sha256=5Oi3GDxv0xWjVymvfAMcJ4Na4biulwpZbpPoN_i5_sE,5708
|
|
6
|
+
sql2api/cors.py,sha256=lEZSZk7sQCRaKqPd6oGmbXamFGtstDVKxAO5IKO84KU,2101
|
|
7
|
+
sql2api/engine.py,sha256=hys-ArxL4n3WWIsrk8hcERaLB11io_f5Wpr5x9zVBr4,1691
|
|
8
|
+
sql2api/errors.py,sha256=nJddO1gAjxQZH7-IXf3Bt8ruxv3EhVw6t6viFKfluG4,264
|
|
9
|
+
sql2api/formats.py,sha256=SbLDZ-yQIt7pcWXYi8WNYEmG6JNqoeeGeRwIg5XHmos,4817
|
|
10
|
+
sql2api/openapi.py,sha256=iK2l4kZD-3Q9jteRyHq76Y0LnUVXLljfycNzzgqk624,13591
|
|
11
|
+
sql2api/params.py,sha256=y_muqwY7ZIhBfUjTpAyw1MpvCyVCmXRrIAUYN9pmB0c,11775
|
|
12
|
+
sql2api/pool.py,sha256=hIxkaVkDnjQEV_4f62yq_HtZkCEHiq_qR_fUGyHGhiA,5170
|
|
13
|
+
sql2api/ratelimit.py,sha256=Cc5J8_RIpnBXmyJ5QiGinjyc47kLdOx-PN43FUBQdm8,2483
|
|
14
|
+
sql2api/runners.py,sha256=ANYrGluet1HOV_Fi1IvJFZMEj1CFtjKEbQzuksqLi0g,13782
|
|
15
|
+
sql2api/sqltools.py,sha256=1C1P-U-rMGHbm0mP6zZ9SkxrnGCfQgXv2aCzLgQi0Vc,5742
|
|
16
|
+
sql2api/store.py,sha256=nHbt1wok8PwAHLetmZeIZeYyCfCp0NXsXNo4hErgTiE,10525
|
|
17
|
+
sql2api/lib/h2-2.2.224.jar,sha256=udjxk1itqCpPbrWxdMbP4yCjdbWpy1pP5FbWI-blVJc,2614933
|
|
18
|
+
sql2api-0.3.0.dist-info/licenses/LICENSE,sha256=ufgSbLEx8zM6J0kFgDoK-6NNb0gqrsFfCb-CnNbZGg0,1071
|
|
19
|
+
sql2api-0.3.0.dist-info/METADATA,sha256=f_VRSImaSTefFnQU6YeVbVbQ240eijsdBbVGbMolxc8,14139
|
|
20
|
+
sql2api-0.3.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
21
|
+
sql2api-0.3.0.dist-info/entry_points.txt,sha256=63gZjCA84qTkErE2iiuTiGS7GtdamvOqi2elEXwy_zE,45
|
|
22
|
+
sql2api-0.3.0.dist-info/top_level.txt,sha256=dicWZ6j0AE1xBDKr0-qh2CkuP7Yc1CWQf_T6r8LS6ms,8
|
|
23
|
+
sql2api-0.3.0.dist-info/RECORD,,
|
sql2api-0.2.0.dist-info/RECORD
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
sql2api/__init__.py,sha256=aVq6e6s7KZt7JlbdHpxY4q_3dOZSuzf8wFrLLggZyQc,185
|
|
2
|
-
sql2api/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
|
|
3
|
-
sql2api/app.py,sha256=AcYUNu31OJ_aW9_kbXtuYJDAJH5UW3paNuv52UgMWOs,12335
|
|
4
|
-
sql2api/cli.py,sha256=1L3RYiCgnh1w_speXB4RPqxNWs5kR7kT68X9L-aqDLk,2712
|
|
5
|
-
sql2api/config.py,sha256=ACMeLisww_6WmzdbIysmZjBui6QxQLWh1evv8z1l9aw,3663
|
|
6
|
-
sql2api/engine.py,sha256=hys-ArxL4n3WWIsrk8hcERaLB11io_f5Wpr5x9zVBr4,1691
|
|
7
|
-
sql2api/errors.py,sha256=nJddO1gAjxQZH7-IXf3Bt8ruxv3EhVw6t6viFKfluG4,264
|
|
8
|
-
sql2api/formats.py,sha256=SbLDZ-yQIt7pcWXYi8WNYEmG6JNqoeeGeRwIg5XHmos,4817
|
|
9
|
-
sql2api/openapi.py,sha256=O7iRvEic1L2OXAp-P3N1FXpPdpTM1w0rAEShWEcpjfk,8905
|
|
10
|
-
sql2api/pool.py,sha256=hIxkaVkDnjQEV_4f62yq_HtZkCEHiq_qR_fUGyHGhiA,5170
|
|
11
|
-
sql2api/runners.py,sha256=ANYrGluet1HOV_Fi1IvJFZMEj1CFtjKEbQzuksqLi0g,13782
|
|
12
|
-
sql2api/sqltools.py,sha256=HPNC2hCSkBqiJ8O-WYWj7gwILnNOtyVD5A_58SG3lik,6657
|
|
13
|
-
sql2api/store.py,sha256=OHLE1PU7ZrqQcPJec4jJbqNeFBDSt7TOUO8NfH7QLJg,9890
|
|
14
|
-
sql2api/lib/h2-2.2.224.jar,sha256=udjxk1itqCpPbrWxdMbP4yCjdbWpy1pP5FbWI-blVJc,2614933
|
|
15
|
-
sql2api-0.2.0.dist-info/licenses/LICENSE,sha256=ufgSbLEx8zM6J0kFgDoK-6NNb0gqrsFfCb-CnNbZGg0,1071
|
|
16
|
-
sql2api-0.2.0.dist-info/METADATA,sha256=YgkgbcA2O-BWhcS77nr8YzltW3YB8PO9NftPfBoEQmU,10241
|
|
17
|
-
sql2api-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
18
|
-
sql2api-0.2.0.dist-info/entry_points.txt,sha256=63gZjCA84qTkErE2iiuTiGS7GtdamvOqi2elEXwy_zE,45
|
|
19
|
-
sql2api-0.2.0.dist-info/top_level.txt,sha256=dicWZ6j0AE1xBDKr0-qh2CkuP7Yc1CWQf_T6r8LS6ms,8
|
|
20
|
-
sql2api-0.2.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|