queryapigate 0.31.0__tar.gz → 0.32.0__tar.gz

{queryapigate-0.31.0/queryapigate.egg-info → queryapigate-0.32.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queryapigate
-Version: 0.31.0
+Version: 0.32.0
 Summary: Turn SQL into governed REST APIs over MySQL, PostgreSQL, ClickHouse, SQLite, DuckDB, H2, MongoDB, and Parquet/CSV/JSON files - JSON, CSV, Parquet or Excel out, with scoped keys, an admin Console and MCP for AI agents.
 Author-email: Anantha Raju C <arcswdev@gmail.com>
 License-Expression: FSL-1.1-MIT
@@ -667,8 +667,8 @@ Everything is configured through environment variables (all optional):
 | `QUERYAPIGATE_CORS_ORIGINS` | unset | Websites allowed to call the API from a browser: comma-separated origins such as `https://app.example.com`, or `*`. Off by default. |
 | `QUERYAPIGATE_RATE_LIMIT` | unset | Requests allowed per client address, e.g. `60/minute` (also `second`, `hour`, `day`). Off by default; a malformed value stops startup. |
 | `QUERYAPIGATE_TRUST_PROXY` | `0` | Number of reverse proxies in front of the app whose `X-Forwarded-*` headers are trusted. Set it (usually `1`) behind nginx, a load balancer or a platform router, or every client looks like the proxy. |
-| `QUERYAPIGATE_AUDIT_SYSLOG_URL` | *(unset)* | Stream every audit entry to a SIEM over syslog: `tls://host:6514` or `tcp://host:514` (experimental). With `QUERYAPIGATE_AUDIT_SYSLOG_CA`, and `_CERT`/`_KEY` for mutual TLS. |
-| `QUERYAPIGATE_AUDIT_OTLP` | `false` | Stream every audit entry as OTLP log records to the OTLP logs endpoint (experimental). |
+| `QUERYAPIGATE_AUDIT_SYSLOG_URL` | *(unset)* | Stream every audit entry to a SIEM over syslog: `tls://host:6514` or `tcp://host:514`. With `QUERYAPIGATE_AUDIT_SYSLOG_CA`, and `_CERT`/`_KEY` for mutual TLS. |
+| `QUERYAPIGATE_AUDIT_OTLP` | `false` | Stream every audit entry as OTLP log records to the OTLP logs endpoint. |
 | `QUERYAPIGATE_AUDIT_KEY` | *(unset)* | HMAC key (32+ characters) the audit log's hash chain is signed with, so an edit made with only the store can't be re-signed. Several, comma-separated, while rotating it. Unset: plain SHA-256. |
 | `QUERYAPIGATE_HSTS_MAX_AGE` | *(unset)* | Seconds browsers should use only HTTPS for this host (`Strict-Transport-Security`), sent only on requests that arrived over HTTPS. Unset: not sent - set it once every client reaches the server over HTTPS. |
 | `QUERYAPIGATE_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |

{queryapigate-0.31.0 → queryapigate-0.32.0}/README.md

@@ -589,8 +589,8 @@ Everything is configured through environment variables (all optional):
 | `QUERYAPIGATE_CORS_ORIGINS` | unset | Websites allowed to call the API from a browser: comma-separated origins such as `https://app.example.com`, or `*`. Off by default. |
 | `QUERYAPIGATE_RATE_LIMIT` | unset | Requests allowed per client address, e.g. `60/minute` (also `second`, `hour`, `day`). Off by default; a malformed value stops startup. |
 | `QUERYAPIGATE_TRUST_PROXY` | `0` | Number of reverse proxies in front of the app whose `X-Forwarded-*` headers are trusted. Set it (usually `1`) behind nginx, a load balancer or a platform router, or every client looks like the proxy. |
-| `QUERYAPIGATE_AUDIT_SYSLOG_URL` | *(unset)* | Stream every audit entry to a SIEM over syslog: `tls://host:6514` or `tcp://host:514` (experimental). With `QUERYAPIGATE_AUDIT_SYSLOG_CA`, and `_CERT`/`_KEY` for mutual TLS. |
-| `QUERYAPIGATE_AUDIT_OTLP` | `false` | Stream every audit entry as OTLP log records to the OTLP logs endpoint (experimental). |
+| `QUERYAPIGATE_AUDIT_SYSLOG_URL` | *(unset)* | Stream every audit entry to a SIEM over syslog: `tls://host:6514` or `tcp://host:514`. With `QUERYAPIGATE_AUDIT_SYSLOG_CA`, and `_CERT`/`_KEY` for mutual TLS. |
+| `QUERYAPIGATE_AUDIT_OTLP` | `false` | Stream every audit entry as OTLP log records to the OTLP logs endpoint. |
 | `QUERYAPIGATE_AUDIT_KEY` | *(unset)* | HMAC key (32+ characters) the audit log's hash chain is signed with, so an edit made with only the store can't be re-signed. Several, comma-separated, while rotating it. Unset: plain SHA-256. |
 | `QUERYAPIGATE_HSTS_MAX_AGE` | *(unset)* | Seconds browsers should use only HTTPS for this host (`Strict-Transport-Security`), sent only on requests that arrived over HTTPS. Unset: not sent - set it once every client reaches the server over HTTPS. |
 | `QUERYAPIGATE_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |

{queryapigate-0.31.0 → queryapigate-0.32.0}/queryapigate/__init__.py

@@ -1,5 +1,5 @@
 """QueryAPIGate - expose SQL databases as a REST API."""
-__version__ = "0.31.0"
+__version__ = "0.32.0"
 
 from .app import create_app  # noqa: E402  (app imports __version__)
 

{queryapigate-0.31.0 → queryapigate-0.32.0}/queryapigate/auditchain.py

@@ -165,25 +165,64 @@ def verify(conn):
     return report
 
 
+def _entry_in(line):
+    """The JSON object a line holds: the whole line (an export file), or the message at its end - a syslog line, as
+    rsyslog or syslog-ng write what the stream sent (`<110>1 ... [meta sequenceId="7"] {...}`). None if it has none."""
+    try:
+        data = json.loads(line)
+        return data if isinstance(data, dict) else None
+    except ValueError:
+        pass
+    decoder = json.JSONDecoder()
+    start = line.find('{')
+    while start != -1:
+        try:
+            data, end = decoder.raw_decode(line, start)
+        except ValueError:
+            start = line.find('{', start + 1)
+            continue
+        if isinstance(data, dict) and not line[end:].strip():
+            return data
+        start = line.find('{', start + 1)
+    return None
+
+
 def verify_file(path):
-    """An export file (QUERYAPIGATE_AUDIT_LOG_EXPORT_FILE), one entry per line with its chain fields - from its first
-    line, whose `prev_hash` is taken as given (the file may have started after the chain did)."""
-    links = []
+    """A copy of the chain, one entry per line - an export file (QUERYAPIGATE_AUDIT_LOG_EXPORT_FILE), or what a SIEM
+    received (ADR 0019): JSON lines, or syslog lines whose message is the entry. Walked in `seq` order from its first
+    entry, whose `prev_hash` is taken as given (the copy may have started after the chain did).
+
+    Delivery is at least once, so a SIEM's copy may hold an entry twice: an exact repeat - the same `seq` and `hash` -
+    is dropped, and counted (`repeats`). A repeat whose `hash` differs is never dropped: `conflicting_copies`, naming
+    both - one of them isn't what was written."""
+    by_seq = {}
+    repeats = 0
     with open(path) as f:
         for number, line in enumerate(f, 1):
             if not line.strip():
                 continue
-            try:
-                data = json.loads(line)
-            except ValueError:
+            data = _entry_in(line)
+            if data is None:
                 return {'intact': False, 'break': {'line': number, 'problem': 'not_json'}}
             if 'seq' not in data:
                 continue  # written before the chain existed
-            chain = (data.pop('seq'), data.pop('prev_hash', None), data.pop('hash', None))
-            links.append((*chain, data))
+            seq, prev_hash, digest = data.pop('seq'), data.pop('prev_hash', None), data.pop('hash', None)
+            seen = by_seq.get(seq)
+            if seen is None:
+                by_seq[seq] = (seq, prev_hash, digest, data, number)
+                continue
+            if (seen[1], seen[2], seen[3]) == (prev_hash, digest, data):
+                repeats += 1
+                continue
+            return {'intact': False, 'break': {'seq': seq, 'timestamp': data.get('timestamp'),
+                                               'action': data.get('action'), 'problem': 'conflicting_copies',
+                                               'lines': [seen[4], number]}}
+    links = [link[:4] for _seq, link in sorted(by_seq.items())]
     if not links:
-        return {'intact': True, 'entries': 0}
-    return _walk(links, (links[0][0] - 1, links[0][1]))
+        return {'intact': True, 'entries': 0, 'repeats': repeats}
+    report = _walk(links, (links[0][0] - 1, links[0][1]))
+    report['repeats'] = repeats
+    return report
 
 
 def head_keyed(conn):

{queryapigate-0.31.0 → queryapigate-0.32.0}/queryapigate/auditstream.py

@@ -1,4 +1,5 @@
-"""The audit log, streamed to a SIEM (ADR 0012, BACKLOG #96) - experimental.
+"""The audit log, streamed to a SIEM (ADR 0012, BACKLOG #96) - stable since 0.32, tested against real receivers
+(ADR 0019).
 
 Two outputs, each turned on by one setting:
 

{queryapigate-0.31.0 → queryapigate-0.32.0}/queryapigate/cli.py

@@ -716,12 +716,17 @@ def _audit_verify(args):
         how = ('keyed with QUERYAPIGATE_AUDIT_KEY' if report.get('keyed')
                else 'unkeyed (plain SHA-256: set QUERYAPIGATE_AUDIT_KEY so an edit cannot be re-signed)')
         print(f"Intact: entries {report['first_seq']} to {report['last_seq']}, {how}.")
+        if report.get('repeats'):
+            print(f"  {report['repeats']} repeated entries dropped - the same seq and hash, as at-least-once delivery "
+                  'can send them')
         if report.get('unchecked_without_key'):
             print(f"  {report['unchecked_without_key']} keyed entries not checked - QUERYAPIGATE_AUDIT_KEY is not set")
         return 0
     found = report['break']
     where = f"line {found['line']}" if 'line' in found else (
         f"entry {found['seq']} ({found.get('action')}, {found.get('timestamp')})")
+    if found['problem'] == 'conflicting_copies':
+        where += f" - lines {found['lines'][0]} and {found['lines'][1]} differ"
     print(f"BROKEN at {where}: {found['problem'].replace('_', ' ')}", file=sys.stderr)
     return 1
 
@@ -915,7 +920,9 @@ def build_parser():
     audit_actions = audit_parser.add_subparsers(dest='action')
     audit_verify = audit_actions.add_parser('verify', help='walk the chain and say whether it is intact, or where it '
                                                           'first breaks (exit 0 or 1)')
-    audit_verify.add_argument('--file', help='an export file (QUERYAPIGATE_AUDIT_LOG_EXPORT_FILE) instead of the store')
+    audit_verify.add_argument('--file', help='a copy instead of the store: an export file '
+                                             '(QUERYAPIGATE_AUDIT_LOG_EXPORT_FILE), or what a SIEM received - JSON or '
+                                             'syslog lines, repeats allowed')
     audit_verify.set_defaults(func=_audit_verify)
 
     history_parser = commands.add_parser('history', help='run history: forget a person (ADR 0016)')

{queryapigate-0.31.0 → queryapigate-0.32.0}/queryapigate/experimental.py

@@ -9,8 +9,8 @@ docs say so at each feature's own section.
 A feature graduates - its entry deleted here, its callout replaced by a note that it's stable and what may still be
 added - when it meets every one of GRADUATION. Alerts, remote files through DuckDB, exports and column masking did in
 0.20; descriptions, agent usage, usage quotas, configuration as code, OAuth for the MCP server, write tools and
-Console sign-in through SSO in 0.30 (ADR 0017), with the Helm chart. The connection types have their own terms
-(databases.GRADUATION).
+Console sign-in through SSO in 0.30 (ADR 0017), with the Helm chart; the audit log streamed to a SIEM in 0.32, once
+tested against real receivers (ADR 0019). The connection types have their own terms (databases.GRADUATION).
 """
 from typing import Any
 
@@ -41,14 +41,6 @@ FEATURES: dict[str, dict[str, Any]] = {
         'db_types': ('h2', 'jdbc', 'mongo'),
         'docs': ('documentation/DATABASE_CONNECTION_CONFIGURATION.md', 'Experimental: h2, jdbc and mongo'),
     },
-    'audit_stream': {
-        'name': 'The audit log streamed to a SIEM',
-        'why': 'new in 0.25 - its settings and message format may change as it meets real SIEMs (ADR 0012)',
-        'settings': ('QUERYAPIGATE_AUDIT_SYSLOG_URL', 'QUERYAPIGATE_AUDIT_SYSLOG_CA', 'QUERYAPIGATE_AUDIT_SYSLOG_CERT',
-                     'QUERYAPIGATE_AUDIT_SYSLOG_KEY', 'QUERYAPIGATE_AUDIT_OTLP'),
-        'operations': (('get', '/api/v1/audit/delivery'),),
-        'docs': ('documentation/API.md', 'Streaming the audit log to a SIEM'),
-    },
     'tracing': {
         'name': 'Tracing with OpenTelemetry',
         'why': 'new in 0.23 - span names and attributes may change as the OpenTelemetry conventions for databases and '

{queryapigate-0.31.0 → queryapigate-0.32.0/queryapigate.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queryapigate
-Version: 0.31.0
+Version: 0.32.0
 Summary: Turn SQL into governed REST APIs over MySQL, PostgreSQL, ClickHouse, SQLite, DuckDB, H2, MongoDB, and Parquet/CSV/JSON files - JSON, CSV, Parquet or Excel out, with scoped keys, an admin Console and MCP for AI agents.
 Author-email: Anantha Raju C <arcswdev@gmail.com>
 License-Expression: FSL-1.1-MIT
@@ -667,8 +667,8 @@ Everything is configured through environment variables (all optional):
 | `QUERYAPIGATE_CORS_ORIGINS` | unset | Websites allowed to call the API from a browser: comma-separated origins such as `https://app.example.com`, or `*`. Off by default. |
 | `QUERYAPIGATE_RATE_LIMIT` | unset | Requests allowed per client address, e.g. `60/minute` (also `second`, `hour`, `day`). Off by default; a malformed value stops startup. |
 | `QUERYAPIGATE_TRUST_PROXY` | `0` | Number of reverse proxies in front of the app whose `X-Forwarded-*` headers are trusted. Set it (usually `1`) behind nginx, a load balancer or a platform router, or every client looks like the proxy. |
-| `QUERYAPIGATE_AUDIT_SYSLOG_URL` | *(unset)* | Stream every audit entry to a SIEM over syslog: `tls://host:6514` or `tcp://host:514` (experimental). With `QUERYAPIGATE_AUDIT_SYSLOG_CA`, and `_CERT`/`_KEY` for mutual TLS. |
-| `QUERYAPIGATE_AUDIT_OTLP` | `false` | Stream every audit entry as OTLP log records to the OTLP logs endpoint (experimental). |
+| `QUERYAPIGATE_AUDIT_SYSLOG_URL` | *(unset)* | Stream every audit entry to a SIEM over syslog: `tls://host:6514` or `tcp://host:514`. With `QUERYAPIGATE_AUDIT_SYSLOG_CA`, and `_CERT`/`_KEY` for mutual TLS. |
+| `QUERYAPIGATE_AUDIT_OTLP` | `false` | Stream every audit entry as OTLP log records to the OTLP logs endpoint. |
 | `QUERYAPIGATE_AUDIT_KEY` | *(unset)* | HMAC key (32+ characters) the audit log's hash chain is signed with, so an edit made with only the store can't be re-signed. Several, comma-separated, while rotating it. Unset: plain SHA-256. |
 | `QUERYAPIGATE_HSTS_MAX_AGE` | *(unset)* | Seconds browsers should use only HTTPS for this host (`Strict-Transport-Security`), sent only on requests that arrived over HTTPS. Unset: not sent - set it once every client reaches the server over HTTPS. |
 | `QUERYAPIGATE_POOL_SIZE` | `5` | Idle connections kept per distinct connection setting. `0` turns pooling off. |

{queryapigate-0.31.0 → queryapigate-0.32.0}/queryapigate.egg-info/SOURCES.txt

@@ -136,6 +136,7 @@ tests/test_api_keys.py
 tests/test_api_v1.py
 tests/test_audit_chain.py
 tests/test_audit_log.py
+tests/test_audit_siem.py
 tests/test_audit_stream.py
 tests/test_aws.py
 tests/test_backup_restore.py

{queryapigate-0.31.0 → queryapigate-0.32.0}/tests/test_audit_chain.py

@@ -233,6 +233,70 @@ class ExportFileTests(ChainTestCase):
         self.assertEqual(auditchain.verify_file(path)['break']['problem'], 'changed')
 
 
+class SiemCopyTests(ChainTestCase):
+    """What a SIEM received (ADR 0019): delivery is at least once, so its copy may hold an entry twice, and after a
+    retry not in order - an untouched copy still verifies; a copy someone edited still doesn't."""
+
+    def copy(self, lines, name='siem.log'):
+        path = os.path.join(self.tmp.name, name)
+        with open(path, 'w') as f:
+            f.writelines(line + '\n' for line in lines)
+        return path
+
+    def entries(self, n=5):
+        path = os.path.join(self.tmp.name, 'audit.jsonl')
+        with mock.patch.dict(os.environ, {'QUERYAPIGATE_AUDIT_LOG_EXPORT_FILE': path}):
+            self.write(n)
+        return [line.rstrip('\n') for line in open(path)]
+
+    @staticmethod
+    def as_syslog(line):
+        entry = json.loads(line)
+        return (f'2026-10-13T10:00:00+00:00 api-1 queryapigate[42]: <110>1 2026-10-13T10:00:00Z api-1 queryapigate 42 '
+                f'{entry["action"]} [meta sequenceId="{entry["seq"]}"] {line}')
+
+    def test_exact_repeats_are_dropped_and_counted(self):
+        lines = self.entries()
+        report = auditchain.verify_file(self.copy(lines[:3] + lines[1:3] + lines[3:]))  # 2 and 3 sent again
+        self.assertTrue(report['intact'], report)
+        self.assertEqual((report['first_seq'], report['last_seq'], report['repeats']), (1, 5, 2))
+
+    def test_entries_are_walked_in_seq_order_whatever_order_they_arrived_in(self):
+        lines = self.entries()
+        self.assertTrue(auditchain.verify_file(self.copy([lines[0], lines[3], lines[1], lines[2], lines[4]]))['intact'])
+
+    def test_a_repeat_that_differs_is_never_dropped(self):
+        lines = self.entries()
+        forged = json.loads(lines[2])
+        forged['target'] = 'forged'
+        report = auditchain.verify_file(self.copy([*lines, json.dumps(forged)]))
+        self.assertFalse(report['intact'])
+        self.assertEqual(report['break']['problem'], 'conflicting_copies')
+        self.assertEqual((report['break']['seq'], report['break']['lines']), (3, [3, 6]))
+
+    def test_syslog_lines_as_a_receiver_writes_them(self):
+        lines = [self.as_syslog(line) for line in self.entries()]
+        self.assertTrue(auditchain.verify_file(self.copy(lines + lines[:1]))['intact'])
+        edited = lines[1].replace('"seq": 2', '"seq": 2, "note": "added"')
+        self.assertNotEqual(edited, lines[1])
+        report = auditchain.verify_file(self.copy([lines[0], edited, *lines[2:]]))
+        self.assertEqual(report['break']['problem'], 'changed')
+
+    def test_a_missing_entry_is_still_missing(self):
+        lines = self.entries()
+        report = auditchain.verify_file(self.copy(lines[:2] + lines[3:]))
+        self.assertEqual((report['break']['problem'], report['break']['seq']), ('missing', 4))
+
+    def test_the_cli_says_how_many_repeats_it_dropped(self):
+        lines = self.entries(3)
+        path = self.copy(lines + lines[1:2])
+        out, err = io.StringIO(), io.StringIO()
+        with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+            self.assertEqual(cli.main(['audit', 'verify', '--file', path]), 0)
+        self.assertIn('Intact: entries 1 to 3', out.getvalue())
+        self.assertIn('1 repeated entries dropped', out.getvalue())
+
+
 class SurfaceTests(ChainTestCase):
     def run_cli(self, *argv):
         out, err = io.StringIO(), io.StringIO()

queryapigate-0.32.0/tests/test_audit_siem.py

@@ -0,0 +1,246 @@
+"""The audit log's stream against real receivers (ADR 0019): rsyslog and syslog-ng taking RFC 5425 syslog over mutual
+TLS, and an OpenTelemetry Collector taking OTLP/HTTP logs - started by .github/scripts/siem_receivers.sh, which
+QUERYAPIGATE_TEST_SIEM names the directory of. Each test reads what the receiver itself parsed and wrote, never our own
+parse: MSGID, structured data, the message - and checks that copy with `audit verify --file`, as an auditor would.
+
+CI's integration job runs these with every receiver up, and fails if any test skips."""
+import json
+import os
+import subprocess
+import tempfile
+import time
+import unittest
+import uuid
+from unittest import mock
+
+from queryapigate import auditchain, auditstream, db, store
+
+SIEM = os.environ.get('QUERYAPIGATE_TEST_SIEM')
+RECEIVERS = json.load(open(os.path.join(SIEM, 'receivers.json'))) if SIEM else {}
+SKIP = 'set QUERYAPIGATE_TEST_SIEM to the directory .github/scripts/siem_receivers.sh started'
+
+
+def _docker(*args):
+    subprocess.run(['docker', *args], check=True, capture_output=True, timeout=120)
+
+
+class SiemTestCase(unittest.TestCase):
+    receiver = None   # 'rsyslog', 'syslog-ng' or 'otelcol'
+
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.marker = uuid.uuid4().hex[:12]   # every receiver's file is shared between tests: ours are marked
+        certs = RECEIVERS['certs']
+        env = {'QUERYAPIGATE_HOME': self.tmp.name, 'QUERYAPIGATE_AUDIT_KEY': '', 'QUERYAPIGATE_AUDIT_LOG_LIMIT': '500',
+               'QUERYAPIGATE_AUDIT_SYSLOG_URL': '', 'QUERYAPIGATE_AUDIT_OTLP': '',
+               'QUERYAPIGATE_AUDIT_SYSLOG_CA': f'{certs}/ca.pem',
+               'QUERYAPIGATE_AUDIT_SYSLOG_CERT': f'{certs}/client.pem',
+               'QUERYAPIGATE_AUDIT_SYSLOG_KEY': f'{certs}/client.key'}
+        if self.receiver == 'otelcol':
+            env.update(QUERYAPIGATE_AUDIT_OTLP='1', OTEL_EXPORTER_OTLP_ENDPOINT=RECEIVERS['otelcol']['url'],
+                       OTEL_EXPORTER_OTLP_LOGS_ENDPOINT='', OTEL_EXPORTER_OTLP_HEADERS='')
+        else:
+            env['QUERYAPIGATE_AUDIT_SYSLOG_URL'] = RECEIVERS[self.receiver]['url']
+        patcher = mock.patch.dict(os.environ, env)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        auditstream.AUTO_START = False
+        self.addCleanup(setattr, auditstream, 'AUTO_START', True)
+        self.addCleanup(self._close_senders)
+        self._close_senders()
+        db.init_schema()
+
+    @staticmethod
+    def _close_senders():
+        for sender in auditstream._senders.values():
+            sender.close()
+        auditstream._senders.clear()
+
+    @property
+    def output(self):
+        return 'otlp' if self.receiver == 'otelcol' else 'syslog'
+
+    def write(self, count, start=0):
+        for i in range(start, start + count):
+            store.record_audit('alice', 'update_key', f'{self.marker}-key-{i}', {'n': i})
+
+    def deliver(self):
+        return auditstream.deliver_once(self.output)
+
+    def received(self, count, timeout=30.0):
+        """What the receiver wrote of this test's entries, once it has `count` - [(app, msgid, sd, entry)] for syslog,
+        [(record attributes, entry)] for the Collector."""
+        deadline = time.monotonic() + timeout
+        found = []
+        while time.monotonic() < deadline:
+            found = self._read()
+            if len(found) >= count:
+                return found
+            time.sleep(0.25)
+        self.fail(f'{self.receiver} wrote {len(found)} of {count} entries in {timeout:.0f}s')
+
+    def _read(self):
+        path = RECEIVERS[self.receiver]['out']
+        if not os.path.exists(path):
+            return []
+        with open(path, encoding='utf-8') as f:
+            lines = f.read().splitlines()
+        if self.receiver == 'otelcol':
+            records = []
+            for line in lines:
+                for resource in json.loads(line)['resourceLogs']:
+                    for scope in resource['scopeLogs']:
+                        for record in scope['logRecords']:
+                            entry = json.loads(record['body']['stringValue'])
+                            if entry['target'].startswith(self.marker):
+                                attributes = {a['key']: next(iter(a['value'].values())) for a in record['attributes']}
+                                records.append((attributes, entry))
+            return records
+        rows = []
+        for line in lines:
+            app, msgid, sd, message = line.split('\t', 3)
+            entry = json.loads(message.lstrip(' '))
+            if entry['target'].startswith(self.marker):
+                rows.append((app, msgid, sd, entry))
+        return rows
+
+    def copy_as_received(self, rows, edit=None):
+        """The receiver's copy of this test's entries, as a file `audit verify --file` reads - one entry a line."""
+        path = os.path.join(self.tmp.name, f'{self.receiver}-copy.jsonl')
+        with open(path, 'w') as f:
+            for row in rows:
+                entry = dict(row[-1])
+                if edit is not None and entry['seq'] == edit:
+                    entry['changes'] = {'n': 'forged'}
+                f.write(json.dumps(entry) + '\n')
+        return path
+
+
+class SyslogReceiverTests:
+    """For rsyslog and syslog-ng alike."""
+
+    def test_every_entry_arrives_in_order_parsed_as_meant(self):
+        self.write(5)
+        self.assertEqual(self.deliver(), 5)
+        rows = self.received(5)
+        self.assertEqual([r[3]['seq'] for r in rows], [1, 2, 3, 4, 5])
+        for app, msgid, sd, entry in rows:
+            self.assertEqual(app, 'queryapigate')
+            self.assertEqual(msgid, 'update_key')    # the action, as the receiver parsed it
+            self.assertEqual(sd, f'[meta sequenceId="{entry["seq"]}"]')
+        stored = [r[0] for r in db.connection().execute('SELECT hash FROM audit_log ORDER BY seq')]
+        self.assertEqual([r[3]['hash'] for r in rows], stored)
+
+    def test_its_copy_verifies_and_an_edit_in_it_is_found(self):
+        self.write(4)
+        self.deliver()
+        rows = self.received(4)
+        report = auditchain.verify_file(self.copy_as_received(rows))
+        self.assertEqual((report['intact'], report['first_seq'], report['last_seq']), (True, 1, 4), report)
+        broken = auditchain.verify_file(self.copy_as_received(rows, edit=3))
+        self.assertEqual((broken['break']['problem'], broken['break']['seq']), ('changed', 3))
+
+    def test_without_a_client_certificate_it_refuses_and_nothing_is_lost(self):
+        with mock.patch.dict(os.environ, {'QUERYAPIGATE_AUDIT_SYSLOG_CERT': '', 'QUERYAPIGATE_AUDIT_SYSLOG_KEY': ''}):
+            self.write(2)
+            with self.assertRaises(OSError) as caught:
+                self.deliver()
+        self.assertIn('CERTIFICATE_REQUIRED', str(caught.exception))
+        [status] = auditstream.status()
+        self.assertEqual((status['delivered_seq'], status['behind']), (0, 2))
+        self.assertIn('CERTIFICATE_REQUIRED', status['last_error'])
+        self._close_senders()
+        self.assertEqual(self.deliver(), 2)   # with its certificate again: both, nothing lost
+        self.assertEqual([r[3]['seq'] for r in self.received(2)], [1, 2])
+
+    def test_a_client_certificate_from_another_ca_is_refused(self):
+        certs = RECEIVERS['certs']
+        with mock.patch.dict(os.environ, {'QUERYAPIGATE_AUDIT_SYSLOG_CERT': f'{certs}/stranger.pem',
+                                          'QUERYAPIGATE_AUDIT_SYSLOG_KEY': f'{certs}/stranger.key'}):
+            self.write(1)
+            with self.assertRaises(OSError) as caught:
+                self.deliver()
+        self.assertIn('UNKNOWN_CA', str(caught.exception))
+        self.assertEqual(auditstream.status()[0]['delivered_seq'], 0)
+
+    def test_an_outage_loses_nothing(self):
+        container = RECEIVERS[self.receiver]['container']
+        self.write(2)
+        self.assertEqual(self.deliver(), 2)
+        self.received(2)
+        _docker('stop', container)   # QueryAPIGate's connection to it still open: noticed, not written into
+        try:
+            self.write(3, start=2)
+            with self.assertRaises(OSError):
+                self.deliver()
+            [status] = auditstream.status()
+            self.assertEqual((status['delivered_seq'], status['behind']), (2, 3))
+        finally:
+            _docker('start', container)
+        deadline = time.monotonic() + 60
+        while True:   # until it listens again
+            try:
+                delivered = self.deliver()
+                break
+            except OSError:
+                if time.monotonic() > deadline:
+                    raise
+                time.sleep(1)
+        self.assertEqual(delivered, 3)
+        rows = self.received(5)
+        self.assertEqual(sorted({r[3]['seq'] for r in rows}), [1, 2, 3, 4, 5])
+        self.assertTrue(auditchain.verify_file(self.copy_as_received(rows))['intact'])
+
+
+@unittest.skipUnless(SIEM, SKIP)
+class RsyslogTests(SyslogReceiverTests, SiemTestCase):
+    receiver = 'rsyslog'
+
+
+@unittest.skipUnless(SIEM, SKIP)
+class SyslogNgTests(SyslogReceiverTests, SiemTestCase):
+    receiver = 'syslog-ng'
+
+
+@unittest.skipUnless(SIEM, SKIP)
+class OtelCollectorTests(SiemTestCase):
+    receiver = 'otelcol'
+
+    def test_every_entry_arrives_as_a_log_record_with_its_attributes(self):
+        self.write(3)
+        self.assertEqual(self.deliver(), 3)
+        records = self.received(3)
+        self.assertEqual([e['seq'] for _a, e in records], [1, 2, 3])
+        for attributes, entry in records:
+            self.assertEqual(int(attributes['queryapigate.audit.seq']), entry['seq'])
+            self.assertEqual(attributes['queryapigate.audit.hash'], entry['hash'])
+            self.assertEqual(attributes['queryapigate.audit.action'], 'update_key')
+            self.assertEqual(attributes['queryapigate.audit.actor'], 'alice')
+        self.assertTrue(auditchain.verify_file(self.copy_as_received(records))['intact'])
+
+    def test_an_outage_loses_nothing(self):
+        container = RECEIVERS['otelcol']['container']
+        _docker('stop', container)
+        try:
+            self.write(2)
+            with self.assertRaises(OSError):
+                self.deliver()
+            self.assertEqual(auditstream.status()[0]['delivered_seq'], 0)
+        finally:
+            _docker('start', container)
+        deadline = time.monotonic() + 60
+        while True:
+            try:
+                delivered = self.deliver()
+                break
+            except OSError:
+                if time.monotonic() > deadline:
+                    raise
+                time.sleep(1)
+        self.assertEqual(delivered, 2)
+        self.assertEqual([e['seq'] for _a, e in self.received(2)], [1, 2])
+
+
+if __name__ == '__main__':
+    unittest.main()

{queryapigate-0.31.0 → queryapigate-0.32.0}/tests/test_experimental.py

@@ -65,18 +65,22 @@ class MarkedEverywhereTests(unittest.TestCase):
                 note = read(path)[match.start():match.start() + 900].replace('\n>', ' ')
                 self.assertIn('may still be added', ' '.join(note.split()))
 
-    def test_features_graduated_in_0_30_say_what_may_be_added_and_their_limits(self):
-        """ADR 0017: each section opens with the promise, what may still be added, and its limits."""
+    def test_features_graduated_since_0_30_say_what_may_be_added_and_their_limits(self):
+        """ADR 0017 and after: each section opens with the promise and when it began, what may still be added, and its
+        limits."""
         api = 'documentation/API.md'
-        for path, heading in ((api, 'Descriptions'), (api, 'Agent usage'), (api, 'Usage quotas'),
-                              (api, 'Configuration as code'), (api, 'Console sign-in through SSO'),
-                              ('documentation/MCP.md', 'Sign-in for AI clients (OAuth)'),
-                              ('documentation/MCP.md', 'Write tools: agents that write, approved by a person'),
-                              ('documentation/DEPLOYMENT.md', 'Kubernetes (Helm)')):
+        for path, heading, since in ((api, 'Descriptions', '0.30'), (api, 'Agent usage', '0.30'),
+                                     (api, 'Usage quotas', '0.30'), (api, 'Configuration as code', '0.30'),
+                                     (api, 'Console sign-in through SSO', '0.30'),
+                                     (api, 'Streaming the audit log to a SIEM', '0.32'),
+                                     ('documentation/MCP.md', 'Sign-in for AI clients (OAuth)', '0.30'),
+                                     ('documentation/MCP.md', 'Write tools: agents that write, approved by a person',
+                                      '0.30'),
+                                     ('documentation/DEPLOYMENT.md', 'Kubernetes (Helm)', '0.30')):
             with self.subTest(heading=heading):
                 text = read(path)
                 match = re.search(rf'^#+ {re.escape(heading)}\n+(.+)', text, re.M)
-                self.assertTrue(match.group(1).startswith('> **Stable** since 0.30'), match.group(1))
+                self.assertTrue(match.group(1).startswith(f'> **Stable** since {since}'), match.group(1))
                 callout = text[match.start(1):].split('\n\n', 1)[0].replace('\n>', ' ')
                 self.assertIn('may still be added', ' '.join(callout.split()))
                 self.assertIn('Limits:', callout)

{queryapigate-0.31.0 → queryapigate-0.32.0}/tests/test_supply_chain.py

@@ -37,6 +37,15 @@ class PinnedTests(unittest.TestCase):
             with self.subTest(image=image):
                 self.assertRegex(image, r'@sha256:[0-9a-f]{64}$')
 
+    def test_the_siem_receivers_ci_tests_against_are_pinned_by_digest(self):
+        """ADR 0019: rsyslog, syslog-ng and the OpenTelemetry Collector, as siem_receivers.sh runs them."""
+        with open(os.path.join(ROOT, '.github', 'scripts', 'siem_receivers.sh')) as f:
+            images = dict(re.findall(r'^(RSYSLOG|SYSLOG_NG|OTELCOL)=(\S+)$', f.read(), re.M))
+        self.assertEqual(set(images), {'RSYSLOG', 'SYSLOG_NG', 'OTELCOL'})
+        for name, image in images.items():
+            with self.subTest(name=name):
+                self.assertRegex(image, r':[\w.-]+@sha256:[0-9a-f]{64}$')
+
     def test_dependabot_proposes_the_updates(self):
         with open(os.path.join(ROOT, '.github', 'dependabot.yml')) as f:
             ecosystems = {u['package-ecosystem'] for u in yaml.safe_load(f)['updates']}