flow.record 3.15.dev4__tar.gz → 3.15.dev6__tar.gz

{flow.record-3.15.dev4/flow.record.egg-info → flow.record-3.15.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.15.dev4
+Version: 3.15.dev6
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -32,10 +32,14 @@ Provides-Extra: geoip
 Requires-Dist: maxminddb; extra == "geoip"
 Provides-Extra: avro
 Requires-Dist: fastavro[snappy]; extra == "avro"
+Provides-Extra: duckdb
+Requires-Dist: duckdb; extra == "duckdb"
+Requires-Dist: pytz; extra == "duckdb"
 Provides-Extra: test
-Requires-Dist: lz4; extra == "test"
-Requires-Dist: zstandard; extra == "test"
-Requires-Dist: fastavro; extra == "test"
+Requires-Dist: flow.record[compression]; extra == "test"
+Requires-Dist: flow.record[avro]; extra == "test"
+Requires-Dist: duckdb; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
+Requires-Dist: pytz; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
 
 # flow.record
 

flow.record-3.15.dev6/flow/record/adapter/duckdb.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import logging
+
+import duckdb
+
+from flow.record.adapter.sqlite import (
+    Selector,
+    SqliteReader,
+    SqliteWriter,
+    make_selector,
+)
+
+logger = logging.getLogger(__name__)
+
+__usage__ = """
+DuckDB adapter
+---
+Write usage: rdump -w duckdb://[PATH]?batch_size=[BATCH_SIZE]
+Read usage: rdump duckdb://[PATH]?batch_size=[BATCH_SIZE]
+[PATH]: path to DuckDB database file
+
+Optional parameters:
+    [BATCH_SIZE]: number of records to read or write in a single transaction (default: 1000)
+"""
+
+
+class DuckdbReader(SqliteReader):
+    """DuckDB reader, subclasses from SQLite reader."""
+
+    logger = logger
+
+    def __init__(self, path: str, *, batch_size: str | int = 1000, selector: Selector | str | None = None, **kwargs):
+        self.selector = make_selector(selector)
+        self.descriptors_seen = set()
+        self.con = duckdb.connect(path)
+        self.count = 0
+        self.batch_size = int(batch_size)
+
+
+class DuckdbWriter(SqliteWriter):
+    """DuckDB writer, subclasses from SQLite writer."""
+
+    logger = logger
+
+    def __init__(self, path: str, *, batch_size: str | int = 1000, **kwargs):
+        self.descriptors_seen = set()
+        self.con = None
+        self.con = duckdb.connect(path)
+        self.count = 0
+        self.batch_size = int(batch_size)
+        self.con.begin()
+
+    def tx_cycle(self) -> None:
+        self.con.commit()
+        self.con.begin()

{flow.record-3.15.dev4 → flow.record-3.15.dev6}/flow/record/adapter/sqlite.py

@@ -18,7 +18,7 @@ SQLite adapter
 ---
 Write usage: rdump -w sqlite://[PATH]?batch_size=[BATCH_SIZE]
 Read usage: rdump sqlite://[PATH]?batch_size=[BATCH_SIZE]
-[PATH]: path to sqlite database file
+[PATH]: path to SQLite database file
 
 Optional parameters:
     [BATCH_SIZE]: number of records to read or write in a single transaction (default: 1000)
@@ -28,12 +28,12 @@ Optional parameters:
 FIELD_MAP = {
     "int": "INTEGER",
     "uint32": "INTEGER",
-    "varint": "INTEGER",
+    "varint": "BIGINT",
     "float": "REAL",
     "boolean": "INTEGER",
     "bytes": "BLOB",
-    "filesize": "INTEGER",
-    "datetime": "TIMESTAMP",
+    "filesize": "BIGINT",
+    "datetime": "TIMESTAMPTZ",
 }
 
 
@@ -41,12 +41,15 @@ FIELD_MAP = {
 SQLITE_FIELD_MAP = {
     "VARCHAR": "string",
     "INTEGER": "varint",
+    "BIGINT": "varint",
     "BLOB": "bytes",
     "REAL": "float",
     "DOUBLE": "float",
     "BOOLEAN": "boolean",
     "DATETIME": "datetime",
     "TIMESTAMP": "datetime",
+    "TIMESTAMPTZ": "datetime",
+    "TIMESTAMP WITH TIME ZONE": "datetime",
 }
 
 
@@ -58,11 +61,11 @@ def create_descriptor_table(con: sqlite3.Connection, descriptor: RecordDescripto
     column_defs = []
     for column_name, fieldset in descriptor.get_all_fields().items():
         column_type = FIELD_MAP.get(fieldset.typename, "TEXT")
-        column_defs.append(f"   `{column_name}` {column_type}")
+        column_defs.append(f'   "{column_name}" {column_type}')
     sql_columns = ",\n".join(column_defs)
 
     # Create the descriptor table
-    sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (\n{sql_columns}\n)"
+    sql = f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n{sql_columns}\n)'
     logger.debug(sql)
     con.execute(sql)
 
@@ -72,7 +75,7 @@ def update_descriptor_columns(con: sqlite3.Connection, descriptor: RecordDescrip
     table_name = descriptor.name
 
     # Get existing columns
-    cursor = con.execute(f"PRAGMA table_info(`{table_name}`)")
+    cursor = con.execute(f'PRAGMA table_info("{table_name}")')
     column_names = set(row[1] for row in cursor.fetchall())
 
     # Add missing columns
@@ -81,23 +84,23 @@ def update_descriptor_columns(con: sqlite3.Connection, descriptor: RecordDescrip
         if column_name in column_names:
             continue
         column_type = FIELD_MAP.get(fieldset.typename, "TEXT")
-        column_defs.append(f"  ALTER TABLE `{table_name}` ADD COLUMN `{column_name}` {column_type}")
+        column_defs.append(f'  ALTER TABLE "{table_name}" ADD COLUMN "{column_name}" {column_type}')
 
     # No missing columns
     if not column_defs:
         return None
 
     # Add the new columns
-    sql = ";\n".join(column_defs)
-    con.executescript(sql)
+    for col_def in column_defs:
+        con.execute(col_def)
 
 
 @lru_cache(maxsize=1000)
 def prepare_insert_sql(table_name: str, field_names: tuple[str]) -> str:
     """Return (cached) prepared SQL statement for inserting a record based on table name and field names."""
-    column_names = ", ".join(f"`{name}`" for name in field_names)
+    column_names = ", ".join(f'"{name}"' for name in field_names)
     value_placeholder = ", ".join(["?"] * len(field_names))
-    return f"INSERT INTO `{table_name}` ({column_names}) VALUES ({value_placeholder})"
+    return f'INSERT INTO "{table_name}" ({column_names}) VALUES ({value_placeholder})'
 
 
 def db_insert_record(con: sqlite3.Connection, record: Record) -> None:
@@ -123,7 +126,11 @@ def db_insert_record(con: sqlite3.Connection, record: Record) -> None:
 
 
 class SqliteReader(AbstractReader):
-    def __init__(self, path: str, batch_size: str | int = 1000, selector: Selector | str | None = None, **kwargs):
+    """SQLite reader."""
+
+    logger = logger
+
+    def __init__(self, path: str, *, batch_size: str | int = 1000, selector: Selector | str | None = None, **kwargs):
         self.selector = make_selector(selector)
         self.descriptors_seen = set()
         self.con = sqlite3.connect(path)
@@ -140,7 +147,7 @@ class SqliteReader(AbstractReader):
 
         # flow.record is quite strict with what is allowed in fieldnames or decriptor name.
         # While SQLite is less strict, we need to sanitize the names to make them compatible.
-        table_name_org = table_name
+        table_name_org = table_name.replace('"', '""')
         table_name = normalize_fieldname(table_name)
 
         schema = self.con.execute(
@@ -161,8 +168,8 @@ class SqliteReader(AbstractReader):
             fnames.append(fname)
 
         descriptor_cls = RecordDescriptor(table_name, fields)
-        table_name_org = table_name_org.replace("`", r"\\\`")
-        cursor = self.con.execute(f"SELECT * FROM `{table_name_org}`")
+        table_name_org = table_name_org.replace('"', '""')
+        cursor = self.con.execute(f'SELECT * FROM "{table_name_org}"')
         while True:
             rows = cursor.fetchmany(self.batch_size)
             if not rows:
@@ -186,19 +193,24 @@ class SqliteReader(AbstractReader):
     def __iter__(self) -> Iterator[Record]:
         """Iterate over all tables in the database and yield records."""
         for table_name in self.table_names():
-            logging.debug("Reading table: %s", table_name)
+            self.logger.debug("Reading table: %s", table_name)
             for record in self.read_table(table_name):
                 if not self.selector or self.selector.match(record):
                     yield record
 
 
 class SqliteWriter(AbstractWriter):
-    def __init__(self, path: str, batch_size: str | int = 1000, **kwargs):
+    """SQLite writer."""
+
+    logger = logger
+
+    def __init__(self, path: str, *, batch_size: str | int = 1000, **kwargs):
         self.descriptors_seen = set()
         self.con = None
-        self.con = sqlite3.connect(path)
+        self.con = sqlite3.connect(path, isolation_level=None)
         self.count = 0
         self.batch_size = int(batch_size)
+        self.tx_cycle()
 
     def write(self, record: Record) -> None:
         """Write a record to the database"""
@@ -207,17 +219,23 @@ class SqliteWriter(AbstractWriter):
             self.descriptors_seen.add(desc)
             create_descriptor_table(self.con, desc)
             update_descriptor_columns(self.con, desc)
+            self.flush()
 
         db_insert_record(self.con, record)
         self.count += 1
 
         # Commit every batch_size records
         if self.count % self.batch_size == 0:
-            self.con.commit()
+            self.flush()
+
+    def tx_cycle(self) -> None:
+        if self.con.in_transaction:
+            self.con.execute("COMMIT")
+        self.con.execute("BEGIN")
 
     def flush(self) -> None:
         if self.con:
-            self.con.commit()
+            self.tx_cycle()
 
     def close(self) -> None:
         if self.con:

{flow.record-3.15.dev4 → flow.record-3.15.dev6}/flow/record/stream.py

@@ -256,7 +256,8 @@ class RecordFieldRewriter:
         self.exclude = exclude or []
         self.expression = compile(expression, "<string>", "exec") if expression else None
 
-    @lru_cache(maxsize=256)
+        self.record_descriptor_for_fields = lru_cache(256)(self.record_descriptor_for_fields)
+
     def record_descriptor_for_fields(self, descriptor, fields=None, exclude=None, new_fields=None):
         if not fields and not exclude and not new_fields:
             return descriptor

{flow.record-3.15.dev4 → flow.record-3.15.dev6}/flow/record/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.15.dev4'
-__version_tuple__ = version_tuple = (3, 15, 'dev4')
+__version__ = version = '3.15.dev6'
+__version_tuple__ = version_tuple = (3, 15, 'dev6')

{flow.record-3.15.dev4 → flow.record-3.15.dev6/flow.record.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.15.dev4
+Version: 3.15.dev6
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -32,10 +32,14 @@ Provides-Extra: geoip
 Requires-Dist: maxminddb; extra == "geoip"
 Provides-Extra: avro
 Requires-Dist: fastavro[snappy]; extra == "avro"
+Provides-Extra: duckdb
+Requires-Dist: duckdb; extra == "duckdb"
+Requires-Dist: pytz; extra == "duckdb"
 Provides-Extra: test
-Requires-Dist: lz4; extra == "test"
-Requires-Dist: zstandard; extra == "test"
-Requires-Dist: fastavro; extra == "test"
+Requires-Dist: flow.record[compression]; extra == "test"
+Requires-Dist: flow.record[avro]; extra == "test"
+Requires-Dist: duckdb; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
+Requires-Dist: pytz; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
 
 # flow.record
 

{flow.record-3.15.dev4 → flow.record-3.15.dev6}/flow.record.egg-info/SOURCES.txt

@@ -29,6 +29,7 @@ flow/record/adapter/archive.py
 flow/record/adapter/avro.py
 flow/record/adapter/broker.py
 flow/record/adapter/csvfile.py
+flow/record/adapter/duckdb.py
 flow/record/adapter/elastic.py
 flow/record/adapter/jsonfile.py
 flow/record/adapter/line.py
@@ -71,7 +72,7 @@ tests/test_record_descriptor.py
 tests/test_regression.py
 tests/test_selector.py
 tests/test_splunk_adapter.py
-tests/test_sqlite_adapter.py
+tests/test_sqlite_duckdb_adapter.py
 tests/utils_inspect.py
 tests/docs/Makefile
 tests/docs/conf.py

{flow.record-3.15.dev4 → flow.record-3.15.dev6}/flow.record.egg-info/requires.txt

@@ -13,6 +13,10 @@ fastavro[snappy]
 lz4
 zstandard
 
+[duckdb]
+duckdb
+pytz
+
 [elastic]
 elasticsearch
 
@@ -20,6 +24,9 @@ elasticsearch
 maxminddb
 
 [test]
-lz4
-zstandard
-fastavro
+flow.record[compression]
+flow.record[avro]
+
+[test:platform_python_implementation != "PyPy" and python_version < "3.12"]
+duckdb
+pytz

{flow.record-3.15.dev4 → flow.record-3.15.dev6}/pyproject.toml

@@ -49,10 +49,15 @@ geoip = [
 avro = [
     "fastavro[snappy]",
 ]
+duckdb = [
+    "duckdb",
+    "pytz", # duckdb requires pytz for timezone support
+]
 test = [
-    "lz4",
-    "zstandard",
-    "fastavro",
+    "flow.record[compression]",
+    "flow.record[avro]",
+    "duckdb; platform_python_implementation != 'PyPy' and python_version < '3.12'", # duckdb
+    "pytz; platform_python_implementation != 'PyPy' and python_version < '3.12'", # duckdb
 ]
 
 [project.scripts]

flow.record-3.15.dev4/tests/test_sqlite_adapter.py → flow.record-3.15.dev6/tests/test_sqlite_duckdb_adapter.py

@@ -1,7 +1,12 @@
 import sqlite3
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Iterator
+from typing import Any, Iterator, NamedTuple
+
+try:
+    import duckdb
+except ModuleNotFoundError:
+    duckdb = None
 
 import pytest
 
@@ -11,6 +16,26 @@ from flow.record.base import normalize_fieldname
 from flow.record.exceptions import RecordDescriptorError
 
 
+class Database(NamedTuple):
+    scheme: str
+    connector: Any
+
+
+# We test for sqlite3 and duckdb (if available)
+if duckdb is None:
+    databases = [
+        Database("sqlite", sqlite3),
+    ]
+else:
+    databases = [
+        Database("sqlite", sqlite3),
+        Database("duckdb", duckdb),
+    ]
+
+# pytest fixture that will run the test for each database in the databases list
+sqlite_duckdb_parametrize = pytest.mark.parametrize("db", databases, ids=[db.scheme for db in databases])
+
+
 def generate_records(amount: int) -> Iterator[Record]:
     """Generates some test records"""
     TestRecordWithFooBar = RecordDescriptor(
@@ -34,11 +59,12 @@ def generate_records(amount: int) -> Iterator[Record]:
         "_my_movies",
     ],
 )
-def test_table_name_sanitization(tmp_path: Path, table_name: str) -> None:
+@sqlite_duckdb_parametrize
+def test_table_name_sanitization(tmp_path: Path, table_name: str, db: Database) -> None:
     """Ensure that we can read table names that are technically invalid in flow.record."""
-    db = tmp_path / "records.db"
-    con = sqlite3.connect(db)
-    con.execute(f"CREATE TABLE '{table_name}' (title TEXT, year INTEGER, score REAL)")
+    db_path = tmp_path / "records.db"
+    con = db.connector.connect(str(db_path))
+    con.execute(f"CREATE TABLE '{table_name}' (title TEXT, year INTEGER, score DOUBLE)")
     data = [
         ("Monty Python Live at the Hollywood Bowl", 1982, 7.9),
         ("Monty Python's The Meaning of Life", 1983, 7.5),
@@ -49,7 +75,7 @@ def test_table_name_sanitization(tmp_path: Path, table_name: str) -> None:
     con.close()
 
     data_records = []
-    with RecordReader(f"sqlite://{db}") as reader:
+    with RecordReader(f"{db.scheme}://{db_path}") as reader:
         data_records = [(record.title, record.year, record.score) for record in reader]
     assert data == data_records
 
@@ -63,11 +89,12 @@ def test_table_name_sanitization(tmp_path: Path, table_name: str) -> None:
         "1337_starting_with_number",
     ],
 )
-def test_field_name_sanitization(tmp_path: Path, field_name: str) -> None:
+@sqlite_duckdb_parametrize
+def test_field_name_sanitization(tmp_path: Path, field_name: str, db: Database) -> None:
     """Ensure that we can read field names that are technically invalid in flow.record."""
-    db = tmp_path / "records.db"
-    con = sqlite3.connect(db)
-    con.execute(f"CREATE TABLE 'my_table' ('{field_name}' TEXT)")
+    db_path = tmp_path / "records.db"
+    con = db.connector.connect(str(db_path))
+    con.execute(f'CREATE TABLE "my_table" ("{field_name}" TEXT)')
     data = [
         ("hello",),
         ("world",),
@@ -81,7 +108,7 @@ def test_field_name_sanitization(tmp_path: Path, field_name: str) -> None:
     data_records = []
     sanitized_field_name = normalize_fieldname(field_name)
 
-    with RecordReader(f"sqlite://{db}") as reader:
+    with RecordReader(f"{db.scheme}://{db_path}") as reader:
         data_records = [(getattr(record, sanitized_field_name),) for record in reader]
     assert data == data_records
 
@@ -95,20 +122,21 @@ def test_field_name_sanitization(tmp_path: Path, field_name: str) -> None:
         2000,
     ],
 )
-def test_write_to_sqlite(tmp_path: Path, count: int) -> None:
+@sqlite_duckdb_parametrize
+def test_write_to_sqlite(tmp_path: Path, count: int, db: Database) -> None:
     """Tests writing records to a SQLite database."""
-    db = tmp_path / "records.db"
-    with RecordWriter(f"sqlite://{db}") as writer:
+    db_path = tmp_path / "records.db"
+    with RecordWriter(f"{db.scheme}://{db_path}") as writer:
         for record in generate_records(count):
             writer.write(record)
 
     record_count = 0
-    with sqlite3.connect(db) as con:
+    with db.connector.connect(str(db_path)) as con:
         cursor = con.execute("SELECT COUNT(*) FROM 'test/record'")
         record_count = cursor.fetchone()[0]
 
         cursor = con.execute("SELECT * FROM 'test/record'")
-        for index, row in enumerate(cursor):
+        for index, row in enumerate(cursor.fetchall()):
             assert row[0] == f"record{index}"
             assert row[1] == "bar"
             assert row[2] == "baz"
@@ -119,18 +147,19 @@ def test_write_to_sqlite(tmp_path: Path, count: int) -> None:
     assert record_count == count
 
 
-def test_read_from_sqlite(tmp_path: Path) -> None:
+@sqlite_duckdb_parametrize
+def test_read_from_sqlite(tmp_path: Path, db: Database) -> None:
     """Tests basic reading from a SQLite database."""
     # Generate a SQLite database
-    db = tmp_path / "records.db"
-    with sqlite3.connect(db) as con:
+    db_path = tmp_path / "records.db"
+    with db.connector.connect(str(db_path)) as con:
         con.execute(
             """
             CREATE TABLE 'test/record' (
                 name TEXT,
                 data BLOB,
-                datetime DATETIME,
-                score REAL
+                datetime TIMESTAMPTZ,
+                score DOUBLE
             )
             """
         )
@@ -143,7 +172,7 @@ def test_read_from_sqlite(tmp_path: Path) -> None:
             )
 
     # Read the SQLite database using flow.record
-    with RecordReader(f"sqlite://{db}") as reader:
+    with RecordReader(f"{db.scheme}://{db_path}") as reader:
         for i, record in enumerate(reader, start=1):
             assert isinstance(record.name, str)
             assert isinstance(record.datetime, datetime)
@@ -153,12 +182,14 @@ def test_read_from_sqlite(tmp_path: Path) -> None:
             assert record.name == f"record{i}"
             assert record.data == f"foobar{i}".encode()
             assert record.datetime == datetime(2023, 10, i, 13, 37, tzinfo=timezone.utc)
+            assert str(record.datetime) == f"2023-10-{i:02d} 13:37:00+00:00"
             assert record.score == 3.14 + i
 
 
-def test_write_dynamic_descriptor(tmp_path: Path) -> None:
+@sqlite_duckdb_parametrize
+def test_write_dynamic_descriptor(tmp_path: Path, db: Database) -> None:
     """Test the ability to write records with different descriptors to the same table."""
-    db = tmp_path / "records.db"
+    db_path = tmp_path / "records.db"
     TestRecord = RecordDescriptor(
         "test/dynamic",
         [
@@ -179,7 +210,7 @@ def test_write_dynamic_descriptor(tmp_path: Path) -> None:
     )
 
     # We should be able to write records with different descriptors to the same table
-    with RecordWriter(f"sqlite://{db}") as writer:
+    with RecordWriter(f"{db.scheme}://{db_path}") as writer:
         record1 = TestRecord(name="record1", foo="bar", bar="baz")
         writer.write(record1)
         record2 = TestRecord_extra(name="record2", foo="bar", bar="baz", extra="extra", extra2="extra2")
@@ -187,7 +218,7 @@ def test_write_dynamic_descriptor(tmp_path: Path) -> None:
 
     # The read table should be a combination of both descriptors
     record_count = 0
-    with RecordReader(f"sqlite://{db}") as reader:
+    with RecordReader(f"{db.scheme}://{db_path}") as reader:
         for record_count, record in enumerate(reader, start=1):
             assert record._desc.get_field_tuples() == (
                 ("string", "name"),
@@ -206,14 +237,15 @@ def test_write_dynamic_descriptor(tmp_path: Path) -> None:
     assert record_count == 2
 
 
-def test_write_zero_records(tmp_path: Path) -> None:
+@sqlite_duckdb_parametrize
+def test_write_zero_records(tmp_path: Path, db: Database) -> None:
     """Test writing zero records."""
-    db = tmp_path / "records.db"
-    with RecordWriter(f"sqlite://{db}") as writer:
+    db_path = tmp_path / "records.db"
+    with RecordWriter(f"{db.scheme}://{db_path}") as writer:
         assert writer
 
     # test if it's a valid database
-    with sqlite3.connect(db) as con:
+    with db.connector.connect(str(db_path)) as con:
         assert con.execute("SELECT * FROM sqlite_master").fetchall() == []
 
 
@@ -295,7 +327,7 @@ def test_invalid_field_names_quoting(tmp_path: Path, invalid_field_name: str) ->
 def test_prepare_insert_sql():
     table_name = "my_table"
     field_names = ("name", "age", "email")
-    expected_sql = "INSERT INTO `my_table` (`name`, `age`, `email`) VALUES (?, ?, ?)"
+    expected_sql = 'INSERT INTO "my_table" ("name", "age", "email") VALUES (?, ?, ?)'
     assert prepare_insert_sql(table_name, field_names) == expected_sql
 
 
@@ -308,17 +340,24 @@ def test_prepare_insert_sql():
         (1000, 0, 1000),
     ],
 )
-def test_batch_size(tmp_path: Path, batch_size: int, expected_first: int, expected_second: int) -> None:
+@sqlite_duckdb_parametrize
+def test_batch_size(
+    tmp_path: Path,
+    batch_size: int,
+    expected_first: int,
+    expected_second: int,
+    db: Database,
+) -> None:
     """Test that batch_size is respected when writing records."""
     records = generate_records(batch_size + 100)
     db_path = tmp_path / "records.db"
-    with RecordWriter(f"sqlite://{db_path}?batch_size={batch_size}") as writer:
+    with RecordWriter(f"{db.scheme}://{db_path}?batch_size={batch_size}") as writer:
         # write a single record, should not be flushed yet if batch_size > 1
         writer.write(next(records))
 
         # test count of records in table (no flush yet if batch_size > 1)
-        with sqlite3.connect(db_path) as con:
-            x = con.execute("select count(*) from `test/record`")
+        with db.connector.connect(str(db_path)) as con:
+            x = con.execute('SELECT COUNT(*) FROM "test/record"')
             assert x.fetchone()[0] is expected_first
 
         # write at least batch_size records, should be flushed due to batch_size
@@ -326,23 +365,24 @@ def test_batch_size(tmp_path: Path, batch_size: int, expected_first: int, expect
             writer.write(next(records))
 
         # test count of records in table after flush
-        with sqlite3.connect(db_path) as con:
-            x = con.execute("select count(*) from `test/record`")
+        with db.connector.connect(str(db_path)) as con:
+            x = con.execute('SELECT COUNT(*) FROM "test/record"')
             assert x.fetchone()[0] == expected_second
 
 
-def test_selector(tmp_path: Path) -> None:
+@sqlite_duckdb_parametrize
+def test_selector(tmp_path: Path, db: Database) -> None:
     """Test selector when reading records."""
     db_path = tmp_path / "records.db"
-    with RecordWriter(f"sqlite://{db_path}") as writer:
+    with RecordWriter(f"{db.scheme}://{db_path}") as writer:
         for record in generate_records(10):
             writer.write(record)
 
-    with RecordReader(f"sqlite://{db_path}", selector="r.name == 'record5'") as reader:
+    with RecordReader(f"{db.scheme}://{db_path}", selector="r.name == 'record5'") as reader:
         records = list(reader)
         assert len(records) == 1
         assert records[0].name == "record5"
 
-    with RecordReader(f"sqlite://{db_path}", selector="r.name == 'record12345'") as reader:
+    with RecordReader(f"{db.scheme}://{db_path}", selector="r.name == 'record12345'") as reader:
         records = list(reader)
         assert len(records) == 0

{flow.record-3.15.dev4 → flow.record-3.15.dev6}/tox.ini

@@ -4,7 +4,7 @@ envlist = lint, py3, pypy3
 # requires if they are not available on the host system. This requires the
 # locally installed tox to have a minimum version 3.3.0. This means the names
 # of the configuration options are still according to the tox 3.x syntax.
-minversion = 4.2.4
+minversion = 4.11.4
 # This version of virtualenv will install setuptools version 65.5.0 and pip
 # 22.3. These versions fully support python projects defined only through a
 # pyproject.toml file (PEP-517/PEP-518/PEP-621)