queryapigate 0.24.0__tar.gz → 0.26.0__tar.gz

{queryapigate-0.24.0/queryapigate.egg-info → queryapigate-0.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queryapigate
-Version: 0.24.0
+Version: 0.26.0
 Summary: Turn SQL into governed REST APIs over MySQL, PostgreSQL, ClickHouse, SQLite, DuckDB, H2, MongoDB, and Parquet/CSV/JSON files - JSON, CSV, Parquet or Excel out, with scoped keys, an admin Console and MCP for AI agents.
 Author-email: Anantha Raju C <arcswdev@gmail.com>
 License-Expression: FSL-1.1-MIT
@@ -663,6 +663,9 @@ Everything is configured through environment variables (all optional):
 | `QUERYAPIGATE_CORS_ORIGINS` | unset | Websites allowed to call the API from a browser: comma-separated origins such as `https://app.example.com`, or `*`. Off by default. |
 | `QUERYAPIGATE_RATE_LIMIT` | unset | Requests allowed per client address, e.g. `60/minute` (also `second`, `hour`, `day`). Off by default; a malformed value stops startup. |
 | `QUERYAPIGATE_TRUST_PROXY` | `0` | Number of reverse proxies in front of the app whose `X-Forwarded-*` headers are trusted. Set it (usually `1`) behind nginx, a load balancer or a platform router, or every client looks like the proxy. |
+| `QUERYAPIGATE_AUDIT_SYSLOG_URL` | *(unset)* | Stream every audit entry to a SIEM over syslog: `tls://host:6514` or `tcp://host:514` (experimental). With `QUERYAPIGATE_AUDIT_SYSLOG_CA`, and `_CERT`/`_KEY` for mutual TLS. |
+| `QUERYAPIGATE_AUDIT_OTLP` | `false` | Stream every audit entry as OTLP log records to the OTLP logs endpoint (experimental). |
+| `QUERYAPIGATE_AUDIT_KEY` | *(unset)* | HMAC key (32+ characters) the audit log's hash chain is signed with, so an edit made with only the store can't be re-signed. Several, comma-separated, while rotating it. Unset: plain SHA-256. |
 | `QUERYAPIGATE_HSTS_MAX_AGE` | *(unset)* | Seconds browsers should use only HTTPS for this host (`Strict-Transport-Security`), sent only on requests that arrived over HTTPS. Unset: not sent - set it once every client reaches the server over HTTPS. |
 | `QUERYAPIGATE_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |
 | `QUERYAPIGATE_POOL_IDLE_TIMEOUT` | `300` | Seconds an idle pooled connection is kept before it is closed. |

{queryapigate-0.24.0 → queryapigate-0.26.0}/README.md

@@ -589,6 +589,9 @@ Everything is configured through environment variables (all optional):
 | `QUERYAPIGATE_CORS_ORIGINS` | unset | Websites allowed to call the API from a browser: comma-separated origins such as `https://app.example.com`, or `*`. Off by default. |
 | `QUERYAPIGATE_RATE_LIMIT` | unset | Requests allowed per client address, e.g. `60/minute` (also `second`, `hour`, `day`). Off by default; a malformed value stops startup. |
 | `QUERYAPIGATE_TRUST_PROXY` | `0` | Number of reverse proxies in front of the app whose `X-Forwarded-*` headers are trusted. Set it (usually `1`) behind nginx, a load balancer or a platform router, or every client looks like the proxy. |
+| `QUERYAPIGATE_AUDIT_SYSLOG_URL` | *(unset)* | Stream every audit entry to a SIEM over syslog: `tls://host:6514` or `tcp://host:514` (experimental). With `QUERYAPIGATE_AUDIT_SYSLOG_CA`, and `_CERT`/`_KEY` for mutual TLS. |
+| `QUERYAPIGATE_AUDIT_OTLP` | `false` | Stream every audit entry as OTLP log records to the OTLP logs endpoint (experimental). |
+| `QUERYAPIGATE_AUDIT_KEY` | *(unset)* | HMAC key (32+ characters) the audit log's hash chain is signed with, so an edit made with only the store can't be re-signed. Several, comma-separated, while rotating it. Unset: plain SHA-256. |
 | `QUERYAPIGATE_HSTS_MAX_AGE` | *(unset)* | Seconds browsers should use only HTTPS for this host (`Strict-Transport-Security`), sent only on requests that arrived over HTTPS. Unset: not sent - set it once every client reaches the server over HTTPS. |
 | `QUERYAPIGATE_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |
 | `QUERYAPIGATE_POOL_IDLE_TIMEOUT` | `300` | Seconds an idle pooled connection is kept before it is closed. |

{queryapigate-0.24.0 → queryapigate-0.26.0}/queryapigate/__init__.py

@@ -1,5 +1,5 @@
 """QueryAPIGate - expose SQL databases as a REST API."""
-__version__ = '0.24.0'
+__version__ = "0.26.0"
 
 from .app import create_app  # noqa: E402  (app imports __version__)
 

{queryapigate-0.24.0 → queryapigate-0.26.0}/queryapigate/adminroles.py

@@ -71,6 +71,7 @@ OPERATIONS = {
     'v1.clear_cache': 'cache.write', 'v1.delete_cache_entry': 'cache.write',
     'v1.load_examples': 'examples.write', 'v1.unload_examples': 'examples.write',
     'v1.list_alerts': 'observe', 'v1.list_audit': 'observe', 'v1.search_history': 'observe',
+    'v1.verify_audit': 'observe', 'v1.audit_delivery': 'observe',
     'v1.list_instances': 'observe', 'v1.agent_usage': 'observe', 'v1.quota_usage': 'observe',
     'v1.list_write_requests': 'writes.approve', 'v1.get_write_request': 'writes.approve',
     'v1.approve_write_request': 'writes.approve', 'v1.reject_write_request': 'writes.approve',

{queryapigate-0.24.0 → queryapigate-0.26.0}/queryapigate/alerts.py

@@ -58,7 +58,7 @@ KINDS = ('open_server', 'break_glass_used', 'connection_failing', 'key_expired',
          'query_timeouts', 'query_slow', 'key_rate_limited', 'client_rate_limited', 'history_failed', 'history_dropped',
          'instances_not_shared', 'instances_versions_differ', 'rate_limits_not_shared', 'export_failing', 'key_unused',
          'agent_unusual_volume', 'sso_unreachable', 'writes_awaiting_approval', 'encryption_key_rotation_incomplete',
-         'quota_nearly_used')
+         'quota_nearly_used', 'audit_chain_broken', 'audit_delivery_behind')
 
 
 def _alert(severity, kind, subject, title, detail, target=None, since=None):
@@ -75,9 +75,8 @@ def _plural(n, word):
 def collect(now=None):
     """Every alert that holds now, most severe first."""
     now = now or datetime.now()
-    found = [*_server(now), *_sso(), *_encryption(), *_writes(), *_quotas(), *_keys(now), *_runs(now),
-             *_rate_limited(),
-             *_history_health()]
+    found = [*_server(now), *_sso(), *_encryption(), *_audit_chain(), *_audit_delivery(now), *_writes(),
+             *_quotas(), *_keys(now), *_runs(now), *_rate_limited(), *_history_health()]
     return sorted(found, key=lambda a: (SEVERITY_ORDER[a['severity']], a['kind'], a['id']))
 
 
@@ -121,6 +120,51 @@ def _encryption():
                    'the first. Remove the others from every instance to finish the rotation.', target)]
 
 
+def _audit_chain():
+    """The audit log's hash chain doesn't verify (ADR 0012): an entry changed, removed, inserted or reordered."""
+    from . import auditchain
+    try:
+        report = auditchain.cached_verify(db.connection())
+    except db.Error:
+        return []
+    if report['intact']:
+        return []
+    found = report['break']
+    where = f"entry {found.get('seq')} ({found.get('action')}, {found.get('timestamp')})"
+    problem = _CHAIN_PROBLEMS.get(found.get('problem'), found.get('problem'))
+    return [_alert('critical', 'audit_chain_broken', 'audit_log', 'The audit log has been tampered with',
+                   f"Its hash chain breaks at {where}: {problem}. Compare it with your SIEM's copy, or the export "
+                   'file (queryapigate audit verify --file).', {'type': 'audit', 'name': 'verify'},
+                   found.get('timestamp'))]
+
+
+def _audit_delivery(now):
+    """A SIEM output more than five minutes behind, or failing (ADR 0012)."""
+    from . import auditstream
+    if not auditstream.outputs():
+        return []
+    found = []
+    for out in auditstream.status():
+        oldest = out['oldest_undelivered_at'] if out['behind'] else None
+        late = oldest is not None and oldest < (now - timedelta(minutes=5)).strftime('%Y-%m-%d %H:%M:%S')
+        if not late and not (out['behind'] and out['last_error']):
+            continue
+        severity = 'critical' if late and out['last_error'] else 'warning'
+        detail = (f"{out['behind']} audit entries are waiting, the oldest since {oldest}. "
+                  + (f"Last error: {out['last_error']} ({out['last_error_at']}). " if out['last_error'] else '')
+                  + 'They are kept, past the rolling cap, until delivered.')
+        found.append(_alert(severity, 'audit_delivery_behind', out['output'],
+                            f"The audit log isn't reaching {out['target'] or out['output']}", detail,
+                            {'type': 'audit', 'name': 'delivery'}, oldest))
+    return found
+
+
+_CHAIN_PROBLEMS = {'changed': 'it was changed after it was written',
+                   'changed_before': 'the entry before it was changed or removed',
+                   'missing': 'entries before it are missing', 'out_of_order': 'it is out of order',
+                   'unkeyed_after_keyed': 'it was re-written without the audit key'}
+
+
 def _quotas():
     """Keys that have used 80% of a usage quota this period (warning), or all of it (critical) - ADR 0011."""
     from . import quotas

{queryapigate-0.24.0 → queryapigate-0.26.0}/queryapigate/app.py

@@ -187,6 +187,9 @@ def create_app():
                     "environment variable instead - it reads the same way but keeps the secret out of the file, "
                     "or set QUERYAPIGATE_SECRET_KEY to encrypt it at rest automatically.",
                     ', '.join(plaintext), config.db_file())
+    _note_audit_key_rotation()
+    from . import auditstream
+    auditstream.start()  # the audit log to a SIEM, when an output is set (ADR 0012)
     if config.load_examples():
         load_examples_at_startup()
     app.json = JSONProvider(app)
@@ -313,6 +316,26 @@ def create_app():
     return app
 
 
+def _note_audit_key_rotation():
+    """Put a change of QUERYAPIGATE_AUDIT_KEY in the chain (ADR 0012): `audit_key_set` when one is set and the newest
+    entry isn't keyed yet - so a key set means the newest entry is keyed, which verify() holds the chain to - and
+    `audit_key_rotated` when the first key changed since the newest entry was signed."""
+    from . import auditchain
+    if not auditchain.keys():
+        return
+    try:
+        conn = db.connection()
+        if not auditchain.head_keyed(conn):  # a key set for the first time: from here on, every entry is keyed
+            # How many came before, unkeyed: expected once, when the key is first set. Again later - after a key was
+            # set long ago - it means the keyed history was replaced, which the SIEM's copy shows (ADR 0012).
+            unkeyed = conn.execute('SELECT COUNT(*) FROM audit_log').fetchone()[0]
+            store.record_audit('startup', 'audit_key_set', 'audit_log', {'entries_before_unkeyed': unkeyed})
+        elif auditchain.head_signed_by(conn) not in (None, 0):
+            store.record_audit('startup', 'audit_key_rotated', 'audit_log', {'keys': len(auditchain.keys())})
+    except db.Error:
+        log.warning('Could not check the audit key against the audit log', exc_info=True)
+
+
 def error_body(message, status, extra=None, code=None):
     """The JSON body of every error response (errors.py, BACKLOG #69): the message, a stable `code` (the raiser's
     own, or the one for the status), whatever extras the raiser attached, and the `request_id` that finds the

queryapigate-0.26.0/queryapigate/auditchain.py

@@ -0,0 +1,215 @@
+"""The audit log as a hash chain (ADR 0012, BACKLOG #96): each entry carries `seq`, the previous entry's `hash` as its
+`prev_hash`, and its own `hash` over both and the entry - so changing, removing, inserting or reordering an entry
+breaks every hash after it.
+
+The hash is HMAC-SHA256 under QUERYAPIGATE_AUDIT_KEY when that is set ("hmac-sha256:<hex>"): someone who can write to
+the store but hasn't the key can't re-sign an edit. Without it, plain SHA-256 ("sha256:<hex>"): consistent, but anyone
+who knows to can recompute it - a SIEM copy outside the store is what contradicts that. Several keys,
+comma-separated, while rotating: the first signs, every one verifies. Once the chain is keyed, an unkeyed entry after
+a keyed one is a break, not a change of mind.
+
+Entries are appended inside the insert's transaction, which holds the store's write lock (db.transaction()), so
+instances writing at once still make one chain. The rolling cap (QUERYAPIGATE_AUDIT_LOG_LIMIT) trims the oldest; the
+anchor (audit_chain) remembers the newest entry trimmed, so the chain's first kept entry must follow it - a deletion
+from the head looks different from a trim.
+"""
+import hashlib
+import hmac
+import json
+import os
+
+GENESIS = '0' * 64
+ANCHOR = 'anchor'
+
+
+def keys():
+    """QUERYAPIGATE_AUDIT_KEY's keys, first to last - empty when unset."""
+    return [k.strip() for k in os.environ.get('QUERYAPIGATE_AUDIT_KEY', '').split(',') if k.strip()]
+
+
+def check_keys():
+    """At startup: each key long enough to be one (32 characters or more), none twice."""
+    found = keys()
+    for i, key in enumerate(found):
+        which = ('QUERYAPIGATE_AUDIT_KEY' if len(found) == 1
+                 else f'Key {i + 1} of {len(found)} in QUERYAPIGATE_AUDIT_KEY')
+        if len(key) < 32:
+            raise ValueError(f'{which} must be at least 32 characters - e.g. from '
+                             '`python -c "import secrets; print(secrets.token_hex(32))"`')
+        if key in found[:i]:
+            raise ValueError(f'{which} is the same as key {found.index(key) + 1} - list each key once')
+
+
+def canonical(entry):
+    """The entry as hashed: sorted keys, no whitespace - the same bytes from the dict written and the JSON read back."""
+    return json.dumps(entry, sort_keys=True, separators=(',', ':'), default=str, ensure_ascii=False)
+
+
+def _message(prev_hash, seq, entry):
+    return f'{prev_hash}\n{seq}\n{canonical(entry)}'.encode()
+
+
+def compute(prev_hash, seq, entry, key=None):
+    if key is None:
+        return 'sha256:' + hashlib.sha256(_message(prev_hash, seq, entry)).hexdigest()
+    return 'hmac-sha256:' + hmac.new(key.encode(), _message(prev_hash, seq, entry), hashlib.sha256).hexdigest()
+
+
+def _matches(stored, prev_hash, seq, entry, listed):
+    """Whether `stored` is this entry's hash - under any listed key, or unkeyed. Returns (ok, keyed, known): `known`
+    False for a keyed hash with no key to check it against."""
+    if stored.startswith('sha256:'):
+        return hmac.compare_digest(stored, compute(prev_hash, seq, entry)), False, True
+    if stored.startswith('hmac-sha256:'):
+        if not listed:
+            return True, True, False
+        return any(hmac.compare_digest(stored, compute(prev_hash, seq, entry, k)) for k in listed), True, True
+    return False, False, True
+
+
+def _head(conn):
+    row = conn.execute('SELECT seq, hash FROM audit_log WHERE seq IS NOT NULL ORDER BY seq DESC LIMIT 1').fetchone()
+    if row is not None:
+        return row[0], row[1]
+    anchor = conn.execute('SELECT seq, hash FROM audit_chain WHERE name = ?', (ANCHOR,)).fetchone()
+    return (anchor[0], anchor[1]) if anchor is not None else (0, GENESIS)
+
+
+def append(conn, entry, limit):
+    """Insert `entry` as the chain's next link, inside the caller's transaction, and trim to `limit`, moving the anchor.
+    Returns (seq, prev_hash, hash). Entries written meanwhile without a link - by an instance still on 0.24, during a
+    rolling upgrade - are linked in first, in the order they were written."""
+    chain_existing(conn)
+    seq, prev_hash = _head(conn)
+    seq += 1
+    listed = keys()
+    digest = compute(prev_hash, seq, entry, listed[0] if listed else None)
+    conn.execute('INSERT INTO audit_log (timestamp, entry_json, seq, prev_hash, hash) VALUES (?, ?, ?, ?, ?)',
+                 (entry['timestamp'], json.dumps(entry, default=str), seq, prev_hash, digest))
+    cut = seq - limit  # keep the newest `limit`: everything up to `cut` goes, and `cut` becomes the anchor
+    from . import auditstream
+    floor = auditstream.delivery_floor(conn)
+    if floor is not None and floor < cut:  # not delivered to a SIEM yet: kept - up to ten times the cap
+        cut = max(floor, seq - limit * auditstream.MAX_BACKLOG_FACTOR)
+    if cut > 0:
+        row = conn.execute('SELECT seq, hash FROM audit_log WHERE seq = ?', (cut,)).fetchone()
+        if row is not None:
+            _set_anchor(conn, row[0], row[1])
+            conn.execute('DELETE FROM audit_log WHERE seq <= ?', (cut,))
+    return seq, prev_hash, digest
+
+
+def _set_anchor(conn, seq, digest):
+    if conn.execute('SELECT 1 FROM audit_chain WHERE name = ?', (ANCHOR,)).fetchone() is None:
+        conn.execute('INSERT INTO audit_chain (name, seq, hash) VALUES (?, ?, ?)', (ANCHOR, seq, digest))
+    else:
+        conn.execute('UPDATE audit_chain SET seq = ?, hash = ? WHERE name = ?', (seq, digest, ANCHOR))
+
+
+def chain_existing(conn):
+    """Schema 14: link the entries already stored, oldest first, as they are now - the chain proves nothing about them
+    before this moment, which the `audit_chain_started` entry that follows records."""
+    rows = conn.execute('SELECT id, entry_json FROM audit_log WHERE seq IS NULL ORDER BY id').fetchall()
+    seq, prev_hash = _head(conn)
+    listed = keys()
+    for row in rows:
+        seq += 1
+        entry = json.loads(row['entry_json'])
+        digest = compute(prev_hash, seq, entry, listed[0] if listed else None)
+        conn.execute('UPDATE audit_log SET seq = ?, prev_hash = ?, hash = ? WHERE id = ?',
+                     (seq, prev_hash, digest, row['id']))
+        prev_hash = digest
+    return len(rows)
+
+
+def _walk(links, anchor):
+    """Verify `links` - (seq, prev_hash, hash, entry), in seq order - from `anchor` (seq, hash). Returns the report."""
+    listed = keys()
+    expected_seq, expected_prev = anchor
+    keyed_seen = unchecked = 0
+    for checked, (seq, prev_hash, stored, entry) in enumerate(links):
+        where = {'seq': seq, 'timestamp': entry.get('timestamp'), 'action': entry.get('action')}
+        if seq != expected_seq + 1:
+            problem = 'missing' if seq > expected_seq + 1 else 'out_of_order'
+            return {'intact': False, 'checked': checked, 'break': {**where, 'problem': problem,
+                                                                    'expected_seq': expected_seq + 1}}
+        if prev_hash != expected_prev:
+            return {'intact': False, 'checked': checked, 'break': {**where, 'problem': 'changed_before'}}
+        ok, keyed, known = _matches(stored or '', prev_hash, seq, entry, listed)
+        if not ok:
+            return {'intact': False, 'checked': checked, 'break': {**where, 'problem': 'changed'}}
+        if keyed:
+            keyed_seen += 1
+            unchecked += 0 if known else 1
+        elif keyed_seen:  # unkeyed after keyed: re-written without the key
+            return {'intact': False, 'checked': checked, 'break': {**where, 'problem': 'unkeyed_after_keyed'}}
+        expected_seq, expected_prev = seq, stored
+    if listed and links and not (links[-1][2] or '').startswith('hmac-sha256:'):
+        # With a key set the server signs every entry, and says so when it starts (audit_key_set) - so a newest entry
+        # that isn't keyed means the chain was re-written without the key, from the start if need be.
+        seq, _prev, _stored, entry = links[-1]
+        return {'intact': False, 'checked': len(links), 'break': {'seq': seq, 'timestamp': entry.get('timestamp'),
+                                                                   'action': entry.get('action'),
+                                                                   'problem': 'unkeyed_after_keyed'}}
+    return {'intact': True, 'entries': expected_seq - anchor[0], 'first_seq': anchor[0] + 1, 'last_seq': expected_seq,
+            'keyed': bool(keyed_seen), 'unchecked_without_key': unchecked}
+
+
+def verify(conn):
+    """The stored chain, from the anchor: {'intact': True, ...} or {'intact': False, 'break': {...}}."""
+    anchor = conn.execute('SELECT seq, hash FROM audit_chain WHERE name = ?', (ANCHOR,)).fetchone()
+    rows = conn.execute('SELECT seq, prev_hash, hash, entry_json FROM audit_log ORDER BY seq').fetchall()
+    links = [(r['seq'], r['prev_hash'], r['hash'], json.loads(r['entry_json'])) for r in rows]
+    report = _walk(links, (anchor[0], anchor[1]) if anchor is not None else (0, GENESIS))
+    report['entries_stored'] = len(links)
+    return report
+
+
+def verify_file(path):
+    """An export file (QUERYAPIGATE_AUDIT_LOG_EXPORT_FILE), one entry per line with its chain fields - from its first
+    line, whose `prev_hash` is taken as given (the file may have started after the chain did)."""
+    links = []
+    with open(path) as f:
+        for number, line in enumerate(f, 1):
+            if not line.strip():
+                continue
+            try:
+                data = json.loads(line)
+            except ValueError:
+                return {'intact': False, 'break': {'line': number, 'problem': 'not_json'}}
+            if 'seq' not in data:
+                continue  # written before the chain existed
+            chain = (data.pop('seq'), data.pop('prev_hash', None), data.pop('hash', None))
+            links.append((*chain, data))
+    if not links:
+        return {'intact': True, 'entries': 0}
+    return _walk(links, (links[0][0] - 1, links[0][1]))
+
+
+def head_keyed(conn):
+    """Whether the newest entry is signed with a key at all (False for none, or an empty log)."""
+    row = conn.execute('SELECT hash FROM audit_log ORDER BY seq DESC LIMIT 1').fetchone()
+    return row is not None and (row['hash'] or '').startswith('hmac-sha256:')
+
+
+def head_signed_by(conn):
+    """Which listed key signed the newest entry - 0 for the first, None for none (unkeyed, or no key verifies it)."""
+    row = conn.execute('SELECT seq, prev_hash, hash, entry_json FROM audit_log ORDER BY seq DESC LIMIT 1').fetchone()
+    if row is None or not (row['hash'] or '').startswith('hmac-sha256:'):
+        return None
+    entry = json.loads(row['entry_json'])
+    for i, key in enumerate(keys()):
+        if hmac.compare_digest(row['hash'], compute(row['prev_hash'], row['seq'], entry, key)):
+            return i
+    return None
+
+
+_checked = {'at': 0.0, 'report': None}
+
+
+def cached_verify(conn, max_age=300.0):
+    """verify(), at most once every `max_age` seconds per process - for the alert, which the Console asks for often."""
+    import time
+    if _checked['report'] is None or time.monotonic() - _checked['at'] >= max_age:
+        _checked['report'], _checked['at'] = verify(conn), time.monotonic()
+    return _checked['report']