flow.record 3.22.dev3__tar.gz → 3.22.dev5__tar.gz

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flow.record
-Version: 3.22.dev3
+Version: 3.22.dev5
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: AGPL-3.0-or-later

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/flow/record/base.py

@@ -893,6 +893,8 @@ def RecordAdapter(
                             "entering record text, rather than a record stream? This can be fixed by using "
                             "'rdump -w -' to write a record stream to stdout."
                         )
+                    if not peek_data:
+                        raise EOFError("Empty input stream")
                     raise RecordAdapterNotFound("Could not find adapter for file-like object")
 
             # Now that we found an adapter, we will fall back into the same code path as when a URL is given. As the url

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/flow/record/jsonpacker.py

@@ -49,12 +49,7 @@ class JsonRecordPacker:
                 serial["_type"] = "record"
                 serial["_recorddescriptor"] = obj._desc.identifier
 
-            for field_type, field_name in obj._desc.get_field_tuples():
-                # Boolean field types should be cast to a bool instead of staying ints
-                if field_type == "boolean" and isinstance(serial[field_name], int):
-                    serial[field_name] = bool(serial[field_name])
-
-            return serial
+            return self.convert_basic_types(serial)
         if isinstance(obj, RecordDescriptor):
             return {
                 "_type": "recorddescriptor",
@@ -102,7 +97,19 @@ class JsonRecordPacker:
                 return RecordDescriptor._unpack(*data)
         return obj
 
-    def pack(self, obj: Record | RecordDescriptor) -> str:
+    def convert_basic_types(self, obj: Any) -> Any:
+        """Explicitly convert some basic types when packing to JSON."""
+        if isinstance(obj, fieldtypes.boolean):
+            return bool(obj)
+        if isinstance(obj, dict):
+            return {k: self.convert_basic_types(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [self.convert_basic_types(item) for item in obj]
+        return obj
+
+    def pack(self, obj: Record | RecordDescriptor | dict) -> str:
+        if isinstance(obj, dict):
+            obj = self.convert_basic_types(obj)
         return json.dumps(obj, default=self.pack_obj, indent=self.indent)
 
     def unpack(self, d: str) -> RecordDescriptor | Record:

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/flow/record/stream.py

@@ -164,11 +164,13 @@ def record_stream(sources: list[str], selector: str | None = None) -> Iterator[R
             print("[reading from stdin]", file=sys.stderr)
 
         # Initial value for reader, in case of exception message
-        reader = "RecordReader"
+        reader: str | AbstractReader = "RecordReader"
         try:
             reader = RecordReader(src, selector=selector)
             yield from reader
-            reader.close()
+        except EOFError as e:
+            # End of file reached, likely no records in source
+            log.warning("%s(%r): %s", reader, src, e)
         except IOError as e:
             if len(sources) == 1:
                 raise
@@ -184,6 +186,9 @@ def record_stream(sources: list[str], selector: str | None = None) -> Iterator[R
             else:
                 log.warning("Exception in %r for %r: %s -- skipping to next reader", reader, src, aRepr.repr(e))
                 continue
+        finally:
+            if isinstance(reader, AbstractReader):
+                reader.close()
 
 
 class PathTemplateWriter:

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/flow/record/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.22.dev3'
-__version_tuple__ = version_tuple = (3, 22, 'dev3')
+__version__ = version = '3.22.dev5'
+__version_tuple__ = version_tuple = (3, 22, 'dev5')
 
-__commit_id__ = commit_id = 'g6fab237c7'
+__commit_id__ = commit_id = 'gc3f8cd8c6'

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/flow.record.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flow.record
-Version: 3.22.dev3
+Version: 3.22.dev5
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: AGPL-3.0-or-later

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/flow.record.egg-info/SOURCES.txt

@@ -76,6 +76,7 @@ tests/adapter/test_sqlite_duckdb.py
 tests/adapter/test_text.py
 tests/adapter/test_xlsx.py
 tests/fieldtypes/__init__.py
+tests/fieldtypes/test_boolean.py
 tests/fieldtypes/test_fieldtypes.py
 tests/fieldtypes/test_ip.py
 tests/packer/__init__.py

flow_record-3.22.dev5/tests/fieldtypes/test_boolean.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import pytest
+
+from flow.record.base import RecordDescriptor
+
+
+def test_boolean() -> None:
+    TestRecord = RecordDescriptor(
+        "test/boolean",
+        [
+            ("boolean", "booltrue"),
+            ("boolean", "boolfalse"),
+        ],
+    )
+
+    r = TestRecord(True, False)
+    assert bool(r.booltrue) is True
+    assert bool(r.boolfalse) is False
+
+    r = TestRecord(1, 0)
+    assert bool(r.booltrue) is True
+    assert bool(r.boolfalse) is False
+
+    assert str(r.booltrue) == "True"
+    assert str(r.boolfalse) == "False"
+
+    assert repr(r.booltrue) == "True"
+    assert repr(r.boolfalse) == "False"
+
+    with pytest.raises(ValueError, match="Value not a valid boolean value"):
+        TestRecord(2, -1)
+
+    with pytest.raises(ValueError, match="invalid literal for int"):
+        TestRecord("True", "False")

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/tests/fieldtypes/test_fieldtypes.py

@@ -293,36 +293,6 @@ def test_dictlist() -> None:
     assert r.hits[1]["b"] == 4
 
 
-def test_boolean() -> None:
-    TestRecord = RecordDescriptor(
-        "test/boolean",
-        [
-            ("boolean", "booltrue"),
-            ("boolean", "boolfalse"),
-        ],
-    )
-
-    r = TestRecord(True, False)
-    assert bool(r.booltrue) is True
-    assert bool(r.boolfalse) is False
-
-    r = TestRecord(1, 0)
-    assert bool(r.booltrue) is True
-    assert bool(r.boolfalse) is False
-
-    assert str(r.booltrue) == "True"
-    assert str(r.boolfalse) == "False"
-
-    assert repr(r.booltrue) == "True"
-    assert repr(r.boolfalse) == "False"
-
-    with pytest.raises(ValueError, match="Value not a valid boolean value"):
-        r = TestRecord(2, -1)
-
-    with pytest.raises(ValueError, match="invalid literal for int"):
-        r = TestRecord("True", "False")
-
-
 def test_float() -> None:
     TestRecord = RecordDescriptor(
         "test/float",

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/tests/packer/test_json_packer.py

@@ -93,6 +93,16 @@ def test_record_pack_bool_regression() -> None:
     # pack the json string back to a record and make sure it is the same as before
     assert packer.unpack(data) == record
 
+    # Make sure the same applies to an OrderedDict, which is how JsonRecordPacker is invoked for
+    # the Elastic adapter.
+    rdict = record._asdict()
+    data = packer.pack(rdict)
+    assert data.startswith('{"some_varint": 1, "some_uint": 0, "some_boolean": false, ')
+
+    # test that packer.pack has no side effects on rdict
+    assert rdict == record._asdict()
+    assert isinstance(rdict["some_boolean"], fieldtypes.boolean)
+
 
 def test_record_pack_surrogateescape() -> None:
     TestRecord = RecordDescriptor(

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/tests/record/test_adapter.py

@@ -499,3 +499,11 @@ def test_file_like_writer_reader() -> None:
     assert len(read_records) == 10
     for idx, record in enumerate(read_records):
         assert record == test_records[idx]
+
+
+def test_empty_stdin(monkeypatch: pytest.MonkeyPatch) -> None:
+    # Mock stdin to be empty
+    monkeypatch.setattr(sys, "stdin", BytesIO(b""))
+
+    with pytest.raises(EOFError, match="Empty input stream"):
+        RecordAdapter()

{flow_record-3.22.dev3 → flow_record-3.22.dev5}/tests/tools/test_rdump.py

@@ -797,3 +797,76 @@ def test_rdump_catch_sigpipe(tmp_path: Path) -> None:
     assert "test/record count=0" in stdout
     assert "test/record count=1" in stdout
     assert len(stdout.splitlines()) == 2
+
+
+def test_rdump_empty_records_pipe(tmp_path: Path) -> None:
+    """Test that rdump handles empty records as input gracefully."""
+
+    # create an empty records file
+    path = tmp_path / "empty.records"
+    with RecordWriter(path):
+        pass
+
+    # although the records file is empty, it should exist and have a RECORDSTREAM header
+    assert path.exists()
+    assert b"RECORDSTREAM" in path.read_bytes()
+
+    # rdump empty.records | rdump -l
+    p1 = subprocess.Popen(["rdump", str(path)], stdout=subprocess.PIPE)
+    p2 = subprocess.Popen(
+        ["rdump", "-l"],
+        stdin=p1.stdout,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = p2.communicate()
+    assert p2.returncode == 0
+    assert b"RecordReader('-'): Empty input stream" in stderr
+    assert b"Processed 0 records (matched=0, unmatched=0)" in stdout
+
+
+@pytest.mark.parametrize(
+    "stdin_bytes",
+    [
+        b"",
+        None,
+    ],
+)
+def test_rdump_empty_stdin_pipe(stdin_bytes: bytes | None) -> None:
+    """Test that rdump handles empty stdin as input gracefully."""
+
+    # rdump -l (with empty stdin)
+    pipe = subprocess.Popen(
+        ["rdump", "-l"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = pipe.communicate(input=None)
+    assert pipe.returncode == 0
+    assert b"RecordReader('-'): Empty input stream" in stderr
+    assert b"Processed 0 records (matched=0, unmatched=0)" in stdout
+
+
+@pytest.mark.parametrize(
+    "stdin_bytes",
+    [
+        b"\n",
+        b"this is not a valid record stream",
+        b"RANDOMDATA",
+    ],
+)
+def test_rdump_invalid_stdin_pipe(stdin_bytes: bytes) -> None:
+    """Test that rdump handles invalid stdin as an error"""
+
+    # rdump -l (with invalid stdin)
+    pipe = subprocess.Popen(
+        ["rdump", "-l"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = pipe.communicate(input=stdin_bytes)
+    assert pipe.returncode == 1, "rdump should exit with error code 1 on invalid input"
+    assert b"rdump encountered a fatal error: Could not find adapter for file-like object" in stderr
+    assert b"Processed 0 records (matched=0, unmatched=0)" in stdout