flow.record 3.22.dev4__tar.gz → 3.22.dev6__tar.gz

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flow.record
-Version: 3.22.dev4
+Version: 3.22.dev6
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: AGPL-3.0-or-later

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/flow/record/adapter/sqlite.py

@@ -204,6 +204,11 @@ class SqliteReader(AbstractReader):
                 if match_record_with_context(record, selector, ctx):
                     yield record
 
+    def close(self) -> None:
+        if self.con:
+            self.con.close()
+        self.con = None
+
 
 class SqliteWriter(AbstractWriter):
     """SQLite writer."""

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/flow/record/jsonpacker.py

@@ -49,12 +49,7 @@ class JsonRecordPacker:
                 serial["_type"] = "record"
                 serial["_recorddescriptor"] = obj._desc.identifier
 
-            for field_type, field_name in obj._desc.get_field_tuples():
-                # Boolean field types should be cast to a bool instead of staying ints
-                if field_type == "boolean" and isinstance(serial[field_name], int):
-                    serial[field_name] = bool(serial[field_name])
-
-            return serial
+            return self.convert_basic_types(serial)
         if isinstance(obj, RecordDescriptor):
             return {
                 "_type": "recorddescriptor",
@@ -102,7 +97,19 @@ class JsonRecordPacker:
                 return RecordDescriptor._unpack(*data)
         return obj
 
-    def pack(self, obj: Record | RecordDescriptor) -> str:
+    def convert_basic_types(self, obj: Any) -> Any:
+        """Explicitly convert some basic types when packing to JSON."""
+        if isinstance(obj, fieldtypes.boolean):
+            return bool(obj)
+        if isinstance(obj, dict):
+            return {k: self.convert_basic_types(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [self.convert_basic_types(item) for item in obj]
+        return obj
+
+    def pack(self, obj: Record | RecordDescriptor | dict) -> str:
+        if isinstance(obj, dict):
+            obj = self.convert_basic_types(obj)
         return json.dumps(obj, default=self.pack_obj, indent=self.indent)
 
     def unpack(self, d: str) -> RecordDescriptor | Record:

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/flow/record/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.22.dev4'
-__version_tuple__ = version_tuple = (3, 22, 'dev4')
+__version__ = version = '3.22.dev6'
+__version_tuple__ = version_tuple = (3, 22, 'dev6')
 
-__commit_id__ = commit_id = 'g4bd45720d'
+__commit_id__ = commit_id = 'g668138538'

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/flow.record.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flow.record
-Version: 3.22.dev4
+Version: 3.22.dev6
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: AGPL-3.0-or-later

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/flow.record.egg-info/SOURCES.txt

@@ -76,6 +76,7 @@ tests/adapter/test_sqlite_duckdb.py
 tests/adapter/test_text.py
 tests/adapter/test_xlsx.py
 tests/fieldtypes/__init__.py
+tests/fieldtypes/test_boolean.py
 tests/fieldtypes/test_fieldtypes.py
 tests/fieldtypes/test_ip.py
 tests/packer/__init__.py

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/tests/adapter/test_sqlite_duckdb.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sqlite3
+from contextlib import closing
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any, NamedTuple
 
@@ -136,7 +137,7 @@ def test_write_to_sqlite(tmp_path: Path, count: int, db: Database) -> None:
             writer.write(record)
 
     record_count = 0
-    with db.connector.connect(str(db_path)) as con:
+    with closing(db.connector.connect(str(db_path))) as con:
         cursor = con.execute("SELECT COUNT(*) FROM 'test/record'")
         record_count = cursor.fetchone()[0]
 
@@ -157,7 +158,7 @@ def test_read_from_sqlite(tmp_path: Path, db: Database) -> None:
     """Tests basic reading from a SQLite database."""
     # Generate a SQLite database
     db_path = tmp_path / "records.db"
-    with db.connector.connect(str(db_path)) as con:
+    with closing(db.connector.connect(str(db_path))) as con:
         con.execute(
             """
             CREATE TABLE 'test/record' (
@@ -176,6 +177,7 @@ def test_read_from_sqlite(tmp_path: Path, db: Database) -> None:
                 """,
                 (f"record{i}", f"foobar{i}".encode(), dt_isoformat, 3.14 + i),
             )
+        con.commit()
 
     # Read the SQLite database using flow.record
     with RecordReader(f"{db.scheme}://{db_path}") as reader:
@@ -251,7 +253,7 @@ def test_write_zero_records(tmp_path: Path, db: Database) -> None:
         assert writer
 
     # test if it's a valid database
-    with db.connector.connect(str(db_path)) as con:
+    with closing(db.connector.connect(str(db_path))) as con:
         assert con.execute("SELECT * FROM sqlite_master").fetchall() == []
 
 
@@ -272,9 +274,10 @@ def test_write_zero_records(tmp_path: Path, db: Database) -> None:
 def test_non_strict_sqlite_fields(tmp_path: Path, sqlite_coltype: str, sqlite_value: Any, expected_value: Any) -> None:
     """SQLite by default is non strict, meaning that the value could be of different type than the column type."""
     db = tmp_path / "records.db"
-    with sqlite3.connect(db) as con:
+    with closing(sqlite3.connect(db)) as con:
         con.execute(f"CREATE TABLE 'strict-test' (field {sqlite_coltype})")
         con.execute("INSERT INTO 'strict-test' VALUES(?)", (sqlite_value,))
+        con.commit()
 
     with RecordReader(f"sqlite://{db}") as reader:
         record = next(iter(reader))
@@ -294,10 +297,11 @@ def test_invalid_table_names_quoting(tmp_path: Path, invalid_table_name: str) ->
 
     # Creating the tables with these invalid_table_names in SQLite is no problem
     db = tmp_path / "records.db"
-    with sqlite3.connect(db) as con:
+    with closing(sqlite3.connect(db)) as con:
         con.execute(f"CREATE TABLE [{invalid_table_name}] (field TEXT, field2 TEXT)")
         con.execute(f"INSERT INTO [{invalid_table_name}] VALUES(?, ?)", ("hello", "world"))
         con.execute(f"INSERT INTO [{invalid_table_name}] VALUES(?, ?)", ("goodbye", "planet"))
+        con.commit()
 
     # However, these invalid_table_names should raise an exception when reading
     with (
@@ -320,10 +324,11 @@ def test_invalid_field_names_quoting(tmp_path: Path, invalid_field_name: str) ->
 
     # Creating the table with invalid field name in SQLite is no problem
     db = tmp_path / "records.db"
-    with sqlite3.connect(db) as con:
+    with closing(sqlite3.connect(db)) as con:
         con.execute(f"CREATE TABLE [test] (field TEXT, [{invalid_field_name}] TEXT)")
         con.execute("INSERT INTO [test] VALUES(?, ?)", ("hello", "world"))
         con.execute("INSERT INTO [test] VALUES(?, ?)", ("goodbye", "planet"))
+        con.commit()
 
     # However, these field names are invalid in flow.record and should raise an exception
     with (
@@ -365,7 +370,7 @@ def test_batch_size(
         writer.write(next(records))
 
         # test count of records in table (no flush yet if batch_size > 1)
-        with db.connector.connect(str(db_path)) as con:
+        with closing(db.connector.connect(str(db_path))) as con:
             x = con.execute('SELECT COUNT(*) FROM "test/record"')
             assert x.fetchone()[0] is expected_first
 
@@ -374,7 +379,7 @@ def test_batch_size(
             writer.write(next(records))
 
         # test count of records in table after flush
-        with db.connector.connect(str(db_path)) as con:
+        with closing(db.connector.connect(str(db_path))) as con:
             x = con.execute('SELECT COUNT(*) FROM "test/record"')
             assert x.fetchone()[0] == expected_second
 

flow_record-3.22.dev6/tests/fieldtypes/test_boolean.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import pytest
+
+from flow.record.base import RecordDescriptor
+
+
+def test_boolean() -> None:
+    TestRecord = RecordDescriptor(
+        "test/boolean",
+        [
+            ("boolean", "booltrue"),
+            ("boolean", "boolfalse"),
+        ],
+    )
+
+    r = TestRecord(True, False)
+    assert bool(r.booltrue) is True
+    assert bool(r.boolfalse) is False
+
+    r = TestRecord(1, 0)
+    assert bool(r.booltrue) is True
+    assert bool(r.boolfalse) is False
+
+    assert str(r.booltrue) == "True"
+    assert str(r.boolfalse) == "False"
+
+    assert repr(r.booltrue) == "True"
+    assert repr(r.boolfalse) == "False"
+
+    with pytest.raises(ValueError, match="Value not a valid boolean value"):
+        TestRecord(2, -1)
+
+    with pytest.raises(ValueError, match="invalid literal for int"):
+        TestRecord("True", "False")

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/tests/fieldtypes/test_fieldtypes.py

@@ -293,36 +293,6 @@ def test_dictlist() -> None:
     assert r.hits[1]["b"] == 4
 
 
-def test_boolean() -> None:
-    TestRecord = RecordDescriptor(
-        "test/boolean",
-        [
-            ("boolean", "booltrue"),
-            ("boolean", "boolfalse"),
-        ],
-    )
-
-    r = TestRecord(True, False)
-    assert bool(r.booltrue) is True
-    assert bool(r.boolfalse) is False
-
-    r = TestRecord(1, 0)
-    assert bool(r.booltrue) is True
-    assert bool(r.boolfalse) is False
-
-    assert str(r.booltrue) == "True"
-    assert str(r.boolfalse) == "False"
-
-    assert repr(r.booltrue) == "True"
-    assert repr(r.boolfalse) == "False"
-
-    with pytest.raises(ValueError, match="Value not a valid boolean value"):
-        r = TestRecord(2, -1)
-
-    with pytest.raises(ValueError, match="invalid literal for int"):
-        r = TestRecord("True", "False")
-
-
 def test_float() -> None:
     TestRecord = RecordDescriptor(
         "test/float",

{flow_record-3.22.dev4 → flow_record-3.22.dev6}/tests/packer/test_json_packer.py

@@ -93,6 +93,16 @@ def test_record_pack_bool_regression() -> None:
     # pack the json string back to a record and make sure it is the same as before
     assert packer.unpack(data) == record
 
+    # Make sure the same applies to an OrderedDict, which is how JsonRecordPacker is invoked for
+    # the Elastic adapter.
+    rdict = record._asdict()
+    data = packer.pack(rdict)
+    assert data.startswith('{"some_varint": 1, "some_uint": 0, "some_boolean": false, ')
+
+    # test that packer.pack has no side effects on rdict
+    assert rdict == record._asdict()
+    assert isinstance(rdict["some_boolean"], fieldtypes.boolean)
+
 
 def test_record_pack_surrogateescape() -> None:
     TestRecord = RecordDescriptor(