queryapigate 0.48.0__tar.gz → 0.49.0__tar.gz

{queryapigate-0.48.0/queryapigate.egg-info → queryapigate-0.49.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queryapigate
-Version: 0.48.0
+Version: 0.49.0
 Summary: Turn SQL into governed REST APIs over MySQL, PostgreSQL, ClickHouse, SQLite, DuckDB, H2, MongoDB, and Parquet/CSV/JSON files - JSON, CSV, Parquet or Excel out, with scoped keys, an admin Console and MCP for AI agents.
 Author-email: Anantha Raju C <arcswdev@gmail.com>
 License-Expression: Apache-2.0

{queryapigate-0.48.0 → queryapigate-0.49.0}/queryapigate/__init__.py

@@ -1,5 +1,5 @@
 """QueryAPIGate - expose SQL databases as a REST API."""
-__version__ = "0.48.0"
+__version__ = "0.49.0"
 
 from .app import create_app  # noqa: E402  (app imports __version__)
 

{queryapigate-0.48.0 → queryapigate-0.49.0}/queryapigate/experimental.py

@@ -12,7 +12,8 @@ added - when it meets every one of GRADUATION. Alerts, remote files through Duck
 Console sign-in through SSO in 0.30 (ADR 0017), with the Helm chart; the audit log streamed to a SIEM in 0.32, once
 tested against real receivers (ADR 0019); live events, and secrets from files and Vault, in 0.34 (ADR 0021) - AWS's
 stay, until tested against AWS. The connection types have their own terms (databases.GRADUATION): MongoDB met them
-in 0.37, with streaming exports (ADR 0022).
+in 0.37, with streaming exports (ADR 0022). The ClickHouse run store graduated in 0.49 (ADR 0029), once upgrades from
+each release that had it were tested.
 """
 from typing import Any
 
@@ -75,13 +76,6 @@ FEATURES: dict[str, dict[str, Any]] = {
         'settings': ('QUERYAPIGATE_DATAPLANE_SECRET',),
         'docs': ('documentation/DEPLOYMENT.md', 'A Rust data plane in front of /q/'),
     },
-    'run_store': {
-        'name': 'A ClickHouse run store for history, the event log and usage',
-        'why': 'a prototype (ADR 0029, decided go): its tables, its event ids and how it is switched on may change '
-               'before it graduates',
-        'settings': ('QUERYAPIGATE_RUN_STORE_URL',),
-        'docs': ('documentation/DEPLOYMENT.md', 'A ClickHouse run store (prototype)'),
-    },
 }
 
 DB_TYPES = frozenset(t for f in FEATURES.values() for t in f.get('db_types', ()))

{queryapigate-0.48.0 → queryapigate-0.49.0}/queryapigate/masking.py

@@ -404,7 +404,10 @@ def _store_hash_key():
 def _hash(value, key):
     if value is None:
         return None
-    text = value.decode('utf-8', 'replace') if isinstance(value, (bytes, bytearray)) else str(value)
+    # Binary values as their text, as JSON shows them - a memoryview too, which psycopg2 hands over for bytea: str() of
+    # one is its address, not its value
+    text = bytes(value).decode('utf-8', 'replace') if isinstance(value, (bytes, bytearray, memoryview)) \
+        else str(value)
     return hmac.new(key, text.encode('utf-8'), hashlib.sha256).hexdigest()[:16]
 
 

{queryapigate-0.48.0 → queryapigate-0.49.0}/queryapigate/runstore.py

@@ -1,4 +1,4 @@
-"""The run store (ADR 0029, experimental): run history, the event log and usage rollups in ClickHouse.
+"""The run store (ADR 0029): run history, the event log and usage rollups in ClickHouse.
 
 With QUERYAPIGATE_RUN_STORE_URL set, history.py's writer thread sends its batches here instead of to the metadata
 store, and everything that reads runs or events - GET /api/v1/history, the Console's runs per version, usage

{queryapigate-0.48.0 → queryapigate-0.49.0/queryapigate.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queryapigate
-Version: 0.48.0
+Version: 0.49.0
 Summary: Turn SQL into governed REST APIs over MySQL, PostgreSQL, ClickHouse, SQLite, DuckDB, H2, MongoDB, and Parquet/CSV/JSON files - JSON, CSV, Parquet or Excel out, with scoped keys, an admin Console and MCP for AI agents.
 Author-email: Anantha Raju C <arcswdev@gmail.com>
 License-Expression: Apache-2.0

{queryapigate-0.48.0 → queryapigate-0.49.0}/tests/test_db.py

@@ -15,7 +15,7 @@ from unittest import mock
 
 from queryapigate import config, db, events, history
 from queryapigate.store import now as store_now
-from tests import TEST_DATABASE_URL
+from tests import TEST_DATABASE_URL, TEST_RUN_STORE_URL
 from tests.helpers import save_query, write_connections
 
 sqlite_only = unittest.skipIf(TEST_DATABASE_URL, 'inspects the SQLite file itself')
@@ -316,6 +316,7 @@ class PostgresDbTests(unittest.TestCase):
         self.assertIs(db.connection(), parent)
         self.assertEqual(parent.execute('SELECT 1').fetchone()[0], 1)  # the parent's is still usable
 
+    @unittest.skipIf(TEST_RUN_STORE_URL, 'a run store keeps history by time, not by count')
     def test_concurrent_history_writes_are_all_recorded_then_trimmed(self):
         """Runs recorded from many threads at once (queued, then batched by history.py, whose writes skip the
         store-wide lock on Postgres) must all land, and the per-version cap must still hold afterwards."""
@@ -456,6 +457,7 @@ class MigrateToPostgresTests(unittest.TestCase):
             db.close()
         return snapshot
 
+    @unittest.skipIf(TEST_RUN_STORE_URL, "the SQLite store's runs are in the run store, which the copy leaves there")
     def test_copies_everything_and_the_api_sees_the_same_store(self):
         before = os.path.getmtime(self.sqlite_path), os.path.getsize(self.sqlite_path)
         copied = db.migrate_sqlite_to_postgres(self.sqlite_path)
@@ -467,6 +469,7 @@ class MigrateToPostgresTests(unittest.TestCase):
         self.assertEqual((os.path.getmtime(self.sqlite_path), os.path.getsize(self.sqlite_path)), before)
         self.assertEqual(copied['events'], 3)
 
+    @unittest.skipIf(TEST_RUN_STORE_URL, "the SQLite store's events are in the run store, which the copy leaves there")
     def test_event_ids_and_their_sequence_come_across(self):
         """An id handed out and since swept is not handed out again: the sequence is SQLite's, not MAX(id)."""
         source = sqlite3.connect(self.sqlite_path)

{queryapigate-0.48.0 → queryapigate-0.49.0}/tests/test_events_server.py

@@ -247,6 +247,7 @@ class DeliveryTests(EventServerTestCase):
         self.assertTrue(stream.nothing_more())
 
     @unittest.skipUnless(TEST_DATABASE_URL, 'set QUERYAPIGATE_TEST_DATABASE_URL to run')
+    @unittest.skipIf(TEST_RUN_STORE_URL, "the metadata store's NOTIFY: with a run store, the events server tails it")
     def test_postgres_notify_delivers_without_waiting_for_the_poll(self):
         self.start_server(poll_interval=30)  # only a NOTIFY can wake this one in time
         stream = self.open('admin')

{queryapigate-0.48.0 → queryapigate-0.49.0}/tests/test_experimental.py

@@ -80,7 +80,8 @@ class MarkedEverywhereTests(unittest.TestCase):
                                      ('documentation/MCP.md', 'Sign-in for AI clients (OAuth)', '0.30'),
                                      ('documentation/MCP.md', 'Write tools: agents that write, approved by a person',
                                       '0.30'),
-                                     ('documentation/DEPLOYMENT.md', 'Kubernetes (Helm)', '0.30')):
+                                     ('documentation/DEPLOYMENT.md', 'Kubernetes (Helm)', '0.30'),
+                                     ('documentation/DEPLOYMENT.md', 'A ClickHouse run store', '0.49')):
             with self.subTest(heading=heading):
                 text = read(path)
                 match = re.search(rf'^#+ {re.escape(heading)}\n+(.+)', text, re.M)

{queryapigate-0.48.0 → queryapigate-0.49.0}/tests/test_history.py

@@ -239,6 +239,7 @@ class RetentionTests(HistoryTestCase):
             self.assertEqual(self.stored(), 2)
 
     @unittest.skipUnless(TEST_DATABASE_URL, 'several instances only share a PostgreSQL store')
+    @unittest.skipIf(TEST_RUN_STORE_URL, "a run store's TTL, not a sweep, keeps its retention")
     def test_while_another_instance_sweeps_this_one_skips(self):
         with mock.patch.dict(os.environ, {'QUERYAPIGATE_HISTORY_RETENTION_DAYS': '7'}):
             now = datetime(2026, 3, 1, 12, 0, 0)

{queryapigate-0.48.0 → queryapigate-0.49.0}/tests/test_masking.py

@@ -221,6 +221,17 @@ class ApplyTests(unittest.TestCase):
         with mock.patch.dict(os.environ, {'QUERYAPIGATE_MASK_KEY': 'k2'}):
             self.assertNotEqual(hashed('jane@example.com'), before)
 
+    def test_binary_values_hash_by_their_value_whatever_the_driver_hands_over(self):
+        # psycopg2 gives a memoryview for bytea, mysql-connector bytes for a BLOB: the same value, the same hash -
+        # and two memoryviews of one value too, which str() would have told apart by their addresses
+        def hashed(value):
+            return list(masking.apply(['v'], [(value,)], ['hash'])[1])[0][0]
+        value = b'\x48\x65\x6c\x6c\x6f'
+        self.assertEqual(hashed(memoryview(value)), hashed(value))
+        self.assertEqual(hashed(memoryview(value)), hashed(memoryview(bytes(value))))
+        self.assertEqual(hashed(bytearray(value)), hashed('Hello'))
+        self.assertNotEqual(hashed(memoryview(b'Hello')), hashed(memoryview(b'World')))
+
     def test_mask(self):
         for value, expected in (('jane@example.com', 'j***@example.com'), ('@x.org', '***@x.org'), ('Bo', 'B***'),
                                 ('', None), (12345, None), (None, None)):

{queryapigate-0.48.0 → queryapigate-0.49.0}/tests/test_rolling_upgrade.py

@@ -7,7 +7,9 @@ What it can't do is start: a release refuses a store a newer one has upgraded (d
 restarts mid-upgrade must come back on the new version.
 
 This runs the real previous release, installed from PyPI - QUERYAPIGATE_TEST_OLD_BIN is its `queryapigate` - beside
-the current code, on one store. Runs when that and QUERYAPIGATE_TEST_DATABASE_URL are set (CI's PostgreSQL job).
+the current code, on one store. Runs when that and QUERYAPIGATE_TEST_DATABASE_URL are set (CI's PostgreSQL job). With
+QUERYAPIGATE_TEST_RUN_STORE_URL set too (CI's run store job on PostgreSQL), both keep their runs in one ClickHouse run
+store (ADR 0029): each node's runs land there, with ids of its own, whichever release it is.
 
 The one exception: 0.38's store change (schema 17, workspaces - ADR 0023) isn't additive, so a 0.37 instance can't
 write to a store 0.38 has upgraded, and that upgrade stops every instance first (CHANGELOG). Against a previous release
@@ -23,6 +25,7 @@ import uuid
 from tests.test_multinode import ADMIN, call, eventually
 
 DATABASE_URL = os.environ.get('QUERYAPIGATE_TEST_DATABASE_URL')
+RUN_STORE_URL = os.environ.get('QUERYAPIGATE_TEST_RUN_STORE_URL', '').strip() or None
 OLD_BIN = os.environ.get('QUERYAPIGATE_TEST_OLD_BIN')
 NOT_ADDITIVE_UNTIL = (0, 38, 0)  # the release whose store change wasn't additive: one before it can't serve beside it
 
@@ -58,8 +61,10 @@ class RollingUpgradeTests(unittest.TestCase):
         separator = '&' if '?' in DATABASE_URL else '?'
         self.env = {**os.environ, 'QUERYAPIGATE_API_KEY': ADMIN, 'QUERYAPIGATE_HISTORY_FLUSH_INTERVAL': '0.1',
                     'QUERYAPIGATE_DATABASE_URL': f'{DATABASE_URL}{separator}options=-csearch_path%3D{self.schema}'}
-        for name in ('QUERYAPIGATE_TEST_DATABASE_URL', 'QUERYAPIGATE_REDIS_URL'):
+        for name in ('QUERYAPIGATE_TEST_DATABASE_URL', 'QUERYAPIGATE_TEST_RUN_STORE_URL', 'QUERYAPIGATE_REDIS_URL'):
             self.env.pop(name, None)
+        if RUN_STORE_URL:  # a database of its own, named as the schema is
+            self.env['QUERYAPIGATE_RUN_STORE_URL'] = f'{RUN_STORE_URL.rstrip("/")}/{self.schema}'
         self.processes = []
         self.data = os.path.join(self.tmp.name, 'data.db')
         import sqlite3
@@ -82,6 +87,9 @@ class RollingUpgradeTests(unittest.TestCase):
         conn.autocommit = True
         conn.cursor().execute(f'DROP SCHEMA IF EXISTS {self.schema} CASCADE')
         conn.close()
+        if RUN_STORE_URL:
+            from queryapigate import runstore
+            runstore.execute(f'DROP DATABASE IF EXISTS {self.schema} SYNC', db_name='default')
         self.tmp.cleanup()
 
     def start(self, command, name):
@@ -146,6 +154,15 @@ class RollingUpgradeTests(unittest.TestCase):
         self.assertEqual(status, 201, body)
         self.assertEqual(call(old, 'GET', '/q/total', key=body['secret'])[1], [{'s': 30}])
         self.assertTrue(eventually(lambda: len(call(new, 'GET', '/api/v1/history?query=total')[1]['items']) >= 3))
+        if RUN_STORE_URL:
+            from queryapigate import runstore
+            self.assertEqual(runstore.query('SELECT max(version) AS v FROM run_store_schema', db_name=self.schema),
+                             [{'v': runstore._SCHEMA_VERSION}])
+            [runs] = runstore.query("SELECT count() AS n, uniqExact(id) AS ids, groupUniqArray(key_name) AS keys "
+                                    "FROM events WHERE query_name = 'total'", db_name=self.schema)
+            self.assertGreaterEqual(runs['n'], 3)
+            self.assertEqual(runs['ids'], runs['n'])  # the two releases' minters never handed out one id twice
+            self.assertEqual(sorted(runs['keys']), ['admin', 'made-by-new', 'made-by-old'])
 
         # A node that restarts mid-upgrade comes back on the old release only while the store is still at that
         # release's schema; once the new code has moved it on, the old release refuses, saying why. (Right after a

{queryapigate-0.48.0 → queryapigate-0.49.0}/tests/test_upgrades.py

@@ -9,10 +9,17 @@ A new release adds its fixture; this test then covers it with no change here.
 Releases with the PostgreSQL store (0.12.0 on) also have <version>.postgres.tar.gz: the same estate in a PostgreSQL
 schema, dumped as plain SQL. When the suite runs against PostgreSQL, each is restored into the test's own schema and
 checked the same way - which is also a test of restoring a pg_dump (BACKLOG #71).
+
+Releases with the ClickHouse run store (0.45.0 on, ADR 0029) also have <version>.runstore.tar.gz: the home, whose
+metadata store says its runs are in ClickHouse, and runstore/, those tables and their rows. When the suite runs with
+a run store, each is loaded into the test's own ClickHouse database and checked the same way, and its tables must be
+at this code's run store schema: 0.45.0's are at version 1, so this tests that migration on a real release's tables.
 """
 import glob
+import hashlib
 import json
 import os
+import re
 import shutil
 import sqlite3
 import tarfile
@@ -20,7 +27,7 @@ import tempfile
 import unittest
 from unittest import mock
 
-from queryapigate import create_app, db, events, history
+from queryapigate import create_app, db, events, history, runstore
 from tests import TEST_DATABASE_URL, TEST_RUN_STORE_URL
 from tests.helpers import needs_sqlite_335
 
@@ -52,6 +59,34 @@ def restore_postgres(sql):
         conn.close()
 
 
+def restore_run_store(home):
+    """Load a fixture's run store (runstore/ in its home) into this test's own ClickHouse database: the tables, their
+    rows, then the views that write to them, so no row is counted twice. TTL merges stay stopped on it: the fixture's
+    events are older than the event log keeps, and ClickHouse would drop them partway through a test. The metadata
+    store's marker names the ClickHouse the fixture was made on; it's pointed at this one, as the connection is
+    pointed at this home's data.db."""
+    name = runstore.database()
+    runstore.execute(f'DROP DATABASE IF EXISTS {name} SYNC', db_name='default')
+    runstore.execute(f'CREATE DATABASE {name}', db_name='default')
+    with open(os.path.join(home, 'runstore', 'tables.json')) as f:
+        tables = json.load(f)
+    for table in tables:
+        ddl = table['ddl'].replace('qag_fixture.', name + '.')
+        runstore.execute(re.sub(r'DEFINER = \S+', 'DEFINER = CURRENT_USER', ddl), db_name=name)
+        if table['engine'].endswith('MergeTree'):
+            runstore.execute(f'SYSTEM STOP TTL MERGES {name}.{table["name"]}', db_name=name)
+        rows = os.path.join(home, 'runstore', f'{table["name"]}.native')
+        if os.path.exists(rows):
+            with open(rows, 'rb') as f:
+                runstore._client().request({'database': name, 'query': f'INSERT INTO {table["name"]} FORMAT Native'},
+                                           f.read())
+    conn = sqlite3.connect(os.path.join(home, 'queryapigate.db'))
+    conn.execute('UPDATE run_store SET location = ?', (runstore.location(),))
+    conn.commit()
+    conn.close()
+    return name
+
+
 class FixtureTestCase(unittest.TestCase):
     def open_home(self, version, kind=''):
         """A fresh home holding that version's store, with the test's QUERYAPIGATE_HOME pointed at it."""
@@ -69,9 +104,22 @@ class FixtureTestCase(unittest.TestCase):
 
 
 class StoreChecks(FixtureTestCase):
-    """What every fixture must read back, whichever store it is in - run by the two classes below."""
+    """What every fixture must read back, whichever store it is in - run by the three classes below."""
     kind = ''
 
+    def restore(self, home):
+        """Put a fixture's PostgreSQL dump or run store where this run's settings look for them."""
+        if self.kind == '.postgres':
+            with open(os.path.join(home, 'store.sql')) as f:
+                restore_postgres(f.read())
+        elif self.kind == '.runstore':
+            name = restore_run_store(home)
+
+            def drop():
+                history.flush()  # this test's last runs first, or they'd be written to a database just dropped
+                runstore.execute(f'DROP DATABASE IF EXISTS {name} SYNC', db_name='default')
+            self.addCleanup(drop)
+
     def check_every_release(self):
         for version in fixture_versions(self.kind):
             with self.subTest(version=version):
@@ -91,9 +139,7 @@ class StoreChecks(FixtureTestCase):
                 create_app()
             return
         self.assertEqual(expected['skipped'], [])
-        if self.kind == '.postgres':
-            with open(os.path.join(home, 'store.sql')) as f:
-                restore_postgres(f.read())
+        self.restore(home)
         client = create_app().test_client()
 
         connection = self.get(client, '/api/v1/connections/films')
@@ -121,16 +167,36 @@ class StoreChecks(FixtureTestCase):
         self.assertEqual([row['title'] for row in res.get_json()], ['Alpha', 'Gamma'])
 
         self.assertTrue(self.get(client, '/api/v1/audit')['items'])
+        if self.kind == '.runstore':
+            self.check_run_store(client, expected)
+
+    def check_run_store(self, client, expected):
+        """A run store's tables at this code's schema, whichever release made them, and nothing lost on the way."""
+        history.flush()
+        self.assertEqual(runstore.query('SELECT max(version) AS v FROM run_store_schema')[0]['v'],
+                         runstore._SCHEMA_VERSION)
+        # The ad-hoc run's statement, in the hourly rollups as its hash whether the release kept the text (0.45) or not
+        sql = "SELECT title FROM film WHERE rating = 'R'"
+        self.assertEqual([row['statement'] for row in runstore.query(
+            'SELECT DISTINCT statement FROM run_hours WHERE query_name IS NULL')],
+            ['sha:' + hashlib.sha256(sql.encode()).hexdigest()])
+        self.assertEqual(sum(row['runs'] for row in runstore.query('SELECT runs FROM run_hours')),
+                         expected['run_store']['rows']['events'] + 1)
+        runs = self.get(client, '/api/v1/history')['items']
+        self.assertEqual(sorted(r['kind'] for r in runs), ['adhoc', 'saved', 'saved'])  # and the run just made
+        self.assertEqual([r['sql'] for r in runs if r['kind'] == 'adhoc'], [sql])
 
     def check_event_ids(self, version):
         """ADR 0002 §3, ADR 0020 §4: the run's history row id was its event id, and stays so. A client holding the id
         before it resumes with the run; the next event's id is above every id the old release could have sent."""
         home, _ = self.open_home(version, self.kind)
-        if self.kind == '.postgres':
-            with open(os.path.join(home, 'store.sql')) as f:
-                restore_postgres(f.read())
-        newest = db.connection().execute('SELECT MAX(rowid) FROM execution_history').fetchone()[0]
-        with mock.patch.dict(os.environ, {'QUERYAPIGATE_EVENTS_RETENTION_HOURS': '1000000'}):  # however old the run
+        self.restore(home)
+        if self.kind == '.runstore':
+            newest = runstore.query('SELECT max(id) AS id FROM events')[0]['id']
+        else:
+            newest = db.connection().execute('SELECT MAX(rowid) FROM execution_history').fetchone()[0]
+        # However old the run: eleven years, within ClickHouse's DateTime, which ends in 2106
+        with mock.patch.dict(os.environ, {'QUERYAPIGATE_EVENTS_RETENTION_HOURS': '100000'}):
             client = create_app().test_client()
         [(event_id, payload_json)] = events._replay_page(newest - 1, None, {})
         payload = json.loads(payload_json)
@@ -140,7 +206,11 @@ class StoreChecks(FixtureTestCase):
         self.assertEqual(res.status_code, 200, res.get_data(as_text=True))
         self.assertEqual(client.get('/q/films_by_rating?rating=PG', headers=ADMIN).status_code, 200)
         history.flush()
-        self.assertEqual([row[0] for row in events._replay_page(newest, None, {})], [newest + 1])
+        if self.kind == '.runstore':  # a run store's ids are minted from the clock: above the old one, not next to it
+            [(after, _)] = events._replay_page(newest, None, {}, upto=runstore.MAX_ID)
+            self.assertGreater(after, newest)
+        else:
+            self.assertEqual([row[0] for row in events._replay_page(newest, None, {})], [newest + 1])
 
 
 @unittest.skipIf(TEST_DATABASE_URL, 'the SQLite stores and JSON homes')
@@ -175,6 +245,23 @@ class PostgresUpgradeTests(StoreChecks):
                 self.check_event_ids(version)
 
 
+@unittest.skipIf(TEST_DATABASE_URL, "the fixtures' metadata stores are SQLite")
+@unittest.skipUnless(TEST_RUN_STORE_URL, 'set QUERYAPIGATE_TEST_RUN_STORE_URL to run')
+class RunStoreUpgradeTests(StoreChecks):
+    kind = '.runstore'
+
+    def test_there_is_a_fixture_for_every_release_with_the_run_store(self):
+        self.assertLessEqual({'0.45.0', '0.46.0', '0.47.0', '0.48.0'}, set(fixture_versions(self.kind)))
+
+    def test_every_release_store_starts_with_its_data_intact(self):
+        self.check_every_release()
+
+    def test_every_release_store_keeps_its_event_ids(self):
+        for version in fixture_versions(self.kind):
+            with self.subTest(version=version):
+                self.check_event_ids(version)
+
+
 @unittest.skipIf(TEST_DATABASE_URL, 'the fixtures are SQLite stores')
 class RefusalTests(FixtureTestCase):
     """A store this version can't run on stops startup with a message saying what is wrong and what to do - before