queryapigate 0.8.0__tar.gz → 0.9.0__tar.gz

{queryapigate-0.8.0/queryapigate.egg-info → queryapigate-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queryapigate
-Version: 0.8.0
+Version: 0.9.0
 Summary: Turn SQL into a REST API: run queries against MySQL, PostgreSQL, ClickHouse, SQLite or H2 over HTTP and get JSON, CSV, XML, YAML or XLSX back.
 Author-email: Anantha Raju C <arcswdev@gmail.com>
 License-Expression: FSL-1.1-MIT
@@ -41,10 +41,12 @@ Requires-Dist: JayDeBeApi>=1.2; extra == "h2"
 Requires-Dist: JPype1>=1.4; extra == "h2"
 Provides-Extra: duckdb
 Requires-Dist: duckdb>=1.0; extra == "duckdb"
+Provides-Extra: mongo
+Requires-Dist: pymongo>=4.0; extra == "mongo"
 Provides-Extra: encryption
 Requires-Dist: cryptography>=41; extra == "encryption"
 Provides-Extra: all
-Requires-Dist: queryapigate[clickhouse,duckdb,encryption,h2,mysql,postgres]; extra == "all"
+Requires-Dist: queryapigate[clickhouse,duckdb,encryption,h2,mongo,mysql,postgres]; extra == "all"
 Provides-Extra: server
 Requires-Dist: gunicorn>=21; extra == "server"
 Provides-Extra: docs

{queryapigate-0.8.0 → queryapigate-0.9.0}/pyproject.toml

@@ -35,8 +35,9 @@ postgres = ["psycopg2-binary>=2.9"]
 clickhouse = ["clickhouse-driver>=0.2"]
 h2 = ["JayDeBeApi>=1.2", "JPype1>=1.4"]  # also needs a Java runtime
 duckdb = ["duckdb>=1.0"]  # Connection.interrupt()/InterruptException, relied on for query timeouts, verified from here
+mongo = ["pymongo>=4.0"]
 encryption = ["cryptography>=41"]  # only needed when QUERYAPIGATE_SECRET_KEY is set - see store.py's _get_fernet()
-all = ["queryapigate[mysql,postgres,clickhouse,h2,duckdb,encryption]"]
+all = ["queryapigate[mysql,postgres,clickhouse,h2,duckdb,mongo,encryption]"]
 server = ["gunicorn>=21"]
 docs = ["mkdocs-material>=9,<10"]
 dev = ["queryapigate[all]", "ruff>=0.5", "openapi-spec-validator>=0.7", "hypothesis>=6.100", "mypy>=1.10",

{queryapigate-0.8.0 → queryapigate-0.9.0}/queryapigate/__init__.py

@@ -1,5 +1,5 @@
 """QueryAPIGate - expose SQL databases as a REST API."""
-__version__ = '0.8.0'
+__version__ = '0.9.0'
 
 from .app import create_app  # noqa: E402  (app imports __version__)
 

{queryapigate-0.8.0 → queryapigate-0.9.0}/queryapigate/app.py

@@ -21,6 +21,7 @@ from . import (
     examples,
     logging_setup,
     metrics,
+    mongotools,
     openapi,
     pool,
     postman,
@@ -411,6 +412,12 @@ def execute_sql_endpoint():
     if not data.get('connection_name'):
         raise ApiError('Connection name is missing')
     require_connection(data['connection_name'])
+    database = data.get('database')
+    if database and not g.permission.admin:
+        # Every other scoped-key boundary is per connection, never per database within one - a key granted a
+        # connection may run any SQL that connection's own configured database allows, but not redirect that
+        # same connection at a sibling database on the same server the admin never listed it for.
+        raise ApiError('Only the admin key may run a query against a different database on this connection', 403)
     params = get_object(data.get('params'), 'params')
     output_format = get_output_format(data)
     timeout = get_timeout(data)
@@ -420,7 +427,29 @@ def execute_sql_endpoint():
     limit, offset, page = get_pagination()
     result = engine.execute_sql(data['sql'], data['connection_name'], limit, offset, params, timeout,
                                 allow_writes=g.permission.allow_writes, key_name=caller_key_name(),
-                                allowed_write_ops=g.permission.allowed_write_ops)
+                                allowed_write_ops=g.permission.allowed_write_ops, database=database)
+    return render(result, output_format, page, limit)
+
+
+@bp.route('/execute_mongo', methods=['POST'])
+def execute_mongo_endpoint():
+    """The Mongo sibling of /execute_sql - a find() query, ad-hoc, against a saved connection. Read-only,
+    full stop (BACKLOG #36: MongoDB support is find-only in this version) - unlike /execute_sql there is no
+    write permission to check."""
+    data = get_json_body()
+    if not data.get('collection'):
+        raise ApiError('Collection is missing')
+    if not data.get('connection_name'):
+        raise ApiError('Connection name is missing')
+    require_connection(data['connection_name'])
+    filter_doc = mongotools.validate_filter(get_object(data.get('filter'), 'filter'))
+    params = get_object(data.get('params'), 'params')
+    output_format = get_output_format(data)
+    timeout = get_timeout(data)
+    limit, offset, page = get_pagination()
+    result = engine.execute_mongo(data['collection'], filter_doc, data['connection_name'], limit, offset,
+                                  params, timeout, projection=data.get('projection'), sort=data.get('sort'),
+                                  key_name=caller_key_name())
     return render(result, output_format, page, limit)
 
 
@@ -477,19 +506,24 @@ def run_saved(ref, body, url_params):
                        or apikeys.can_use_collection(g.permission, store.read_collection(content)))
     if not (granted_by_name and connection_name == saved.get('connection_name')):
         require_connection(connection_name)
+
+    raw = {**url_params, **get_object(body.get('params'), 'params'),
+           **get_object(body.get('placeholders'), 'placeholders')}
+    output_format = get_output_format(body)
+    timeout = get_timeout(body)
+
+    if saved.get('query_type') == 'mongo':
+        return run_saved_mongo(saved, path, number, ref, connection_name, raw, output_format, timeout)
+
     if not isinstance(saved.get('sql_query'), str):
         raise ApiError('Saved query has no SQL', 500)
     # Per-query write curation (apikeys.can_write_query()) only ever adds write reach for this one named
     # query on top of whatever the key's blanket allow_writes already grants - never the other way round.
     effective_allow_writes = g.permission.allow_writes or apikeys.can_write_query(g.permission, query_name)
 
-    raw = {**url_params, **get_object(body.get('params'), 'params'),
-           **get_object(body.get('placeholders'), 'placeholders')}
     used = set(sqltools.placeholder_names(saved['sql_query']))
     values = param_rules.resolve(saved.get('query_parameters'), raw, used=used)
     sql = sqltools.fill_placeholders(saved['sql_query'], values)
-    output_format = get_output_format(body)
-    timeout = get_timeout(body)
 
     if get_stream_flag():
         # No caching for a streamed export - caching would require materialising the whole body anyway,
@@ -529,6 +563,32 @@ def run_saved(ref, body, url_params):
     return response
 
 
+def run_saved_mongo(saved, path, number, ref, connection_name, raw, output_format, timeout):
+    """The mongo query_type branch of run_saved(): a find() query instead of SQL. No caching (cache_ttl is
+    not yet supported for a mongo saved query) and no streaming (?stream=true) in this version - see
+    BACKLOG #36. Always read-only, same as the ad-hoc /execute_mongo."""
+    if get_stream_flag():
+        raise ApiError('Streaming is not supported for Mongo queries yet', 400)
+    used = set(mongotools.placeholder_names(saved.get('mongo_filter') or {}))
+    values = param_rules.resolve(saved.get('query_parameters'), raw, used=used)
+    filter_doc = mongotools.fill_placeholders(saved.get('mongo_filter') or {}, values)
+    limit, offset, page = get_pagination()
+    entry = {'executed_at': store.now(), 'connection_name': connection_name,
+             'request_id': g.get('request_id'), 'key_name': caller_key_name()}
+    try:
+        result, elapsed_ms = engine.timed(engine.execute_mongo, saved['mongo_collection'], filter_doc,
+                                          connection_name, limit, offset, timeout=timeout,
+                                          projection=saved.get('mongo_projection'), sort=saved.get('mongo_sort'),
+                                          key_name=caller_key_name())
+    except ApiError as error:
+        store.record_execution(path, number, {**entry, 'status': 'error', 'error': error.message})
+        raise
+    response = render(result, output_format, page, limit)  # sets g.serialization_ms - see render()
+    store.record_execution(path, number, {**entry, 'status': 'success', 'rows': len(result.rows),
+                                          'duration_ms': elapsed_ms, 'serialization_ms': g.serialization_ms})
+    return response
+
+
 @bp.route('/execute_sql_from_file', methods=['POST'])
 @bp.route('/execute_sql_with_parameters_from_file', methods=['POST'])
 def execute_sql_from_file():
@@ -722,13 +782,63 @@ def update_connections():
     return jsonify({'message': 'Connections updated successfully'}), 200
 
 
+@bp.route('/connections/databases', methods=['POST'])
+def list_databases_route():
+    """Every database on the server a connection points at - saved, or not yet saved. Give 'name' alone for
+    an already-saved connection (Run SQL's database picker); give the connection's own fields (as
+    /connections/test does) to probe one that isn't saved yet - the New/Edit connection form's 'Default
+    database' dropdown - or to try changed fields before saving them. Admin only, like every other endpoint
+    that reveals the server's own configuration or reaches into a connection's server ahead of a real query."""
+    require_admin()
+    body = get_json_body()
+    if not isinstance(body, dict):
+        raise ApiError('Connection fields are missing')
+    if body.get('name') and not body.get('db'):
+        databases = schema.list_databases(body['name'])
+    else:
+        if body.get('db') not in config.SUPPORTED_DB_TYPES:
+            raise ApiError(f"'db' must be one of: {', '.join(config.SUPPORTED_DB_TYPES)}")
+        if body.get('password') == config.PASSWORD_MASK:
+            existing = store.read_connections().get(body.get('name') or '', {})
+            body = {**body, 'password': existing.get('password', '')}
+        databases = engine.list_databases(store.resolve_ad_hoc(body))
+    return jsonify({'databases': databases}), 200
+
+
+@bp.route('/connections/test', methods=['POST'])
+def test_connection_route():
+    """Try to connect with the given fields - not a saved connection's name, the fields themselves, exactly
+    as the New/Edit connection form has them right now. Nothing is written anywhere; a masked password
+    (unchanged from an existing connection's form) is resolved to the real stored value first, the same way
+    update_connections() already treats one."""
+    require_admin()
+    details = get_json_body()
+    if not isinstance(details, dict) or details.get('db') not in config.SUPPORTED_DB_TYPES:
+        raise ApiError(f"'db' must be one of: {', '.join(config.SUPPORTED_DB_TYPES)}")
+    if details.get('password') == config.PASSWORD_MASK:
+        existing = store.read_connections().get(details.get('name') or '', {})
+        details = {**details, 'password': existing.get('password', '')}
+    result = engine.test_connection(store.resolve_ad_hoc(details))
+    return jsonify({'message': 'Connected', **result}), 200
+
+
 @bp.route('/connections/<name>', methods=['DELETE'])
 def delete_connection(name):
+    """Deleting a connection is destructive and hard to reason about after the fact - every saved query that
+    used it stops working - so the caller must say why. The reason is stored nowhere but the audit log (there
+    is no separate "deleted connections" table); the admin UI's own Deleted tab reads it straight from there,
+    the same as everything else on this route already does."""
     require_admin()
     before = store.read_connections().get(name)
+    if before is None:
+        raise ApiError(f"Connection '{name}' not found", 404)
+    data = get_json_body(required=False)
+    reason = (data.get('reason') or '').strip()
+    if not reason:
+        raise ApiError('A reason is required to delete a connection')
     store.delete_connection(name)
-    if before is not None:
-        store.record_audit(caller_key_name(), 'delete_connection', name, store.mask_passwords({name: before})[name])
+    changes = {**store.mask_passwords({name: before})[name], 'deleted_reason': reason}
+    store.record_audit(caller_key_name(), 'delete_connection', name, changes)
     pool.close_pooled_connections()  # a removed connection must not keep serving from idle sockets
     return jsonify({'message': f"Connection '{name}' deleted"}), 200
 
@@ -736,7 +846,10 @@ def delete_connection(name):
 @bp.route('/connections/<name>/schema', methods=['GET'])
 def connection_schema(name):
     require_connection(name)
-    return jsonify(schema.fetch_schema(name)), 200
+    database = request.args.get('database') or None
+    if database and not g.permission.admin:
+        raise ApiError('Only the admin key may browse a different database on this connection', 403)
+    return jsonify(schema.fetch_schema(name, database=database)), 200
 
 
 # --------------------------------------------------------------------------------------
@@ -881,6 +994,14 @@ def metrics_endpoint():
     return Response(metrics.render(), mimetype='text/plain; version=0.0.4; charset=utf-8')
 
 
+def _is_runnable(data):
+    """Whether a saved-query version has something to actually execute - a SQL string, or (query_type
+    'mongo') a Mongo collection - so a definition left mid-edit or otherwise malformed never gets listed
+    somewhere a caller might then try to run it."""
+    return isinstance(data.get('sql_query'), str) or (data.get('query_type') == 'mongo'
+                                                       and isinstance(data.get('mongo_collection'), str))
+
+
 def describe_saved_queries(permission):
     """What the OpenAPI document needs to know about each saved query (never its SQL text) that
     ``permission`` may actually call - the same test run_saved() applies, so a key scoped to specific
@@ -888,8 +1009,7 @@ def describe_saved_queries(permission):
     unrestricted (admin, or connection-wide) key sees everything, unchanged from before this filter."""
     described = []
     for name, number, data, collection in store.latest_versions():
-        sql = data.get('sql_query')
-        if not isinstance(sql, str):
+        if not _is_runnable(data):
             continue
         connection_name = data.get('connection_name')
         if not apikeys.can_run_saved(permission, name, collection, connection_name):
@@ -920,8 +1040,7 @@ def describe_catalog(permission):
     caller* can write through it (apikeys.can_write_query() - independent of their blanket allow_writes)."""
     catalog = []
     for name, number, data, collection in store.latest_versions():
-        sql = data.get('sql_query')
-        if not isinstance(sql, str):
+        if not _is_runnable(data):
             continue
         connection_name = data.get('connection_name')
         if not apikeys.can_run_saved(permission, name, collection, connection_name):

{queryapigate-0.8.0 → queryapigate-0.9.0}/queryapigate/bundle.py

@@ -18,7 +18,8 @@ FORMAT_VERSION = 1
 POLICIES = ('fail', 'skip', 'new-version')
 # Everything an entry may carry. Anything else is an error rather than silently dropped: a field the importer
 # ignores is a field whose loss nobody notices.
-ENTRY_FIELDS = {'name', 'sql_query', 'author', 'description', 'tags', 'query_parameters', 'connection_name',
+ENTRY_FIELDS = {'name', 'sql_query', 'query_type', 'mongo_collection', 'mongo_filter', 'mongo_projection',
+                'mongo_sort', 'author', 'description', 'tags', 'query_parameters', 'connection_name',
                 'cache_ttl'}
 
 

{queryapigate-0.8.0 → queryapigate-0.9.0}/queryapigate/config.py

@@ -3,7 +3,7 @@ import os
 import re
 from pathlib import Path
 
-SUPPORTED_DB_TYPES = ('mysql', 'postgres', 'clickhouse', 'sqlite', 'h2', 'jdbc', 'duckdb')
+SUPPORTED_DB_TYPES = ('mysql', 'postgres', 'clickhouse', 'sqlite', 'h2', 'jdbc', 'duckdb', 'mongo')
 PASSWORD_MASK = '********'
 CONNECT_TIMEOUT = 10  # seconds
 HISTORY_LIMIT = 50  # executions remembered per saved-query version

queryapigate-0.9.0/queryapigate/definitions.py

@@ -0,0 +1,98 @@
+"""Validation of a saved-query definition - shared by everything that creates one (``PATCH /save_sql_to_file``
+and ``queryapigate collection import``), so a query is held to the same rules however it arrives."""
+from . import mongotools, sqltools, store
+from . import params as param_rules
+from .errors import ApiError
+
+NO_COLLECTION_GIVEN = store._UNSET  # the request said nothing about a collection: leave the query's as it is
+
+
+def as_object(value, label):
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ApiError(f'{label} must be a JSON object')
+    return value
+
+
+def validate_definition(data):
+    """Check ``data`` (a decoded request body or bundle entry) and return ``(fields, collection)``: the fields
+    to store on the new version, and the collection to file the query under - ``NO_COLLECTION_GIVEN`` when
+    ``data`` has no ``collection`` key, None to remove it from any, or a valid collection name.
+
+    ``query_type`` (default ``'sql'``) picks which shape the query itself takes: ``sql_query`` (a SQL
+    string) or, for a Mongo find query, ``mongo_collection`` + ``mongo_filter`` (a JSON filter document,
+    optionally ``mongo_projection``/``mongo_sort``) - see mongotools.py and BACKLOG #36. Everything else
+    (author/description/tags/query_parameters/connection_name/cache_ttl/collection) is shared between the
+    two, unchanged either way.
+    """
+    query_type = data.get('query_type', 'sql')
+    if query_type not in ('sql', 'mongo'):
+        raise ApiError("query_type must be 'sql' or 'mongo'")
+    required = [('author', 'Author'), ('description', 'Description'), ('filename', 'Filename')]
+    required.append(('sql_query', 'SQL query') if query_type == 'sql' else ('mongo_collection', 'Mongo collection'))
+    for field, label in required:
+        if not data.get(field) or not isinstance(data[field], str):
+            raise ApiError(f'{label} is missing')
+    tags = data.get('tags', [])
+    if not isinstance(tags, (list, str)):
+        raise ApiError('tags must be a string or a list')
+    query_parameters = as_object(data.get('query_parameters'), 'query_parameters')
+    param_rules.parse_definitions(query_parameters)
+    if query_type == 'mongo':
+        mongo_filter = mongotools.validate_filter(as_object(data.get('mongo_filter'), 'mongo_filter'))
+        used = set(mongotools.placeholder_names(mongo_filter))
+    else:
+        used = set(sqltools.placeholder_names(data['sql_query']))
+    unused = sorted(set(query_parameters) - used)
+    if unused:
+        where = 'the filter' if query_type == 'mongo' else 'sql_query'
+        raise ApiError(f"query_parameters declares {', '.join(unused)}, which {where} does not use "
+                       '(write :name in it, or remove the declaration)')
+    connection_name = data.get('connection_name')
+    if connection_name is not None and not isinstance(connection_name, str):
+        raise ApiError('connection_name must be a string')
+    cache_ttl = data.get('cache_ttl')
+    if cache_ttl is not None and (not isinstance(cache_ttl, int) or isinstance(cache_ttl, bool) or cache_ttl < 0):
+        raise ApiError('cache_ttl must be a non-negative integer number of seconds')
+    collection = NO_COLLECTION_GIVEN
+    if 'collection' in data:
+        collection = data['collection']
+        if collection is not None:
+            store.validate_collection_name(collection)
+
+    fields = {
+        'author': data['author'],
+        'description': data['description'],
+        'tags': tags,
+        'query_parameters': query_parameters,
+        **({'connection_name': connection_name} if connection_name else {}),
+        **({'cache_ttl': cache_ttl} if cache_ttl else {}),
+    }
+    if query_type == 'mongo':
+        # No 'query_type' field at all for a plain SQL query (query_type's own default) - every saved query
+        # ever written before this version is missing it too, and every check of it elsewhere
+        # (saved.get('query_type') == 'mongo') already treats that absence as "sql", so this keeps a SQL
+        # query's stored shape byte-for-byte unchanged, not just behaviourally equivalent.
+        fields['query_type'] = 'mongo'
+        fields['mongo_collection'] = data['mongo_collection']
+        fields['mongo_filter'] = mongo_filter
+        for extra in ('mongo_projection', 'mongo_sort'):
+            if data.get(extra) is not None:
+                fields[extra] = as_object(data.get(extra), extra)
+    else:
+        fields['sql_query'] = data['sql_query']
+    return fields, collection
+
+
+def effective_parameters(saved):
+    """A saved version's parameters as callers must supply them: every placeholder its query actually uses, in
+    the order it uses them, with the declared rules (an undeclared one is plain required text). The one place
+    this is derived - the OpenAPI document, the catalogue and the Postman export all describe a query's
+    parameters, and must not each work them out separately."""
+    declared = param_rules.read_definitions(saved.get('query_parameters'))
+    if saved.get('query_type') == 'mongo':
+        names = mongotools.placeholder_names(saved.get('mongo_filter') or {})
+    else:
+        names = sqltools.placeholder_names(saved['sql_query'])
+    return {name: declared.get(name) or param_rules.read_definition({}) for name in names}