flow.record 3.14.dev4__tar.gz → 3.15.dev1__tar.gz

{flow.record-3.14.dev4/flow.record.egg-info → flow.record-3.15.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.14.dev4
+Version: 3.15.dev1
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

flow.record-3.15.dev1/flow/record/adapter/line.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from functools import lru_cache
+
+from flow.record import Record, RecordDescriptor, open_path_or_stream
+from flow.record.adapter import AbstractWriter
+from flow.record.utils import is_stdout
+
+__usage__ = """
+Line output format adapter (writer only)
+---
+Write usage: rdump -w line://[PATH]?verbose=[VERBOSE]
+[PATH]: path to file. Leave empty or "-" to output to stdout
+
+Optional arguments:
+    [VERBOSE]: Also show fieldtype in line output (default: False)
+"""
+
+
+@lru_cache(maxsize=1024)
+def field_types_for_record_descriptor(desc: RecordDescriptor) -> dict[str, str]:
+    """Return dictionary of fieldname -> fieldtype for given RecordDescriptor.
+
+    Args:
+        desc: RecordDescriptor to get fieldtypes for
+    Returns:
+        Dictionary of fieldname -> fieldtype
+    """
+    return {fname: fieldset.typename for fname, fieldset in desc.get_all_fields().items()}
+
+
+class LineWriter(AbstractWriter):
+    """Prints all fields and values of the Record on a separate line."""
+
+    fp = None
+
+    def __init__(
+        self,
+        path: str,
+        *,
+        fields: list[str] | str | None = None,
+        exclude: list[str] | str | None = None,
+        verbose: bool = False,
+        **kwargs,
+    ):
+        self.fp = open_path_or_stream(path, "wb")
+        self.count = 0
+        self.fields = fields
+        self.exclude = exclude
+        self.verbose = verbose
+        if isinstance(self.fields, str):
+            self.fields = self.fields.split(",")
+        if isinstance(self.exclude, str):
+            self.exclude = self.exclude.split(",")
+
+    def write(self, rec: Record) -> None:
+        rdict = rec._asdict(fields=self.fields, exclude=self.exclude)
+        rdict_types = field_types_for_record_descriptor(rec._desc) if self.verbose else None
+
+        self.count += 1
+        self.fp.write(f"--[ RECORD {self.count} ]--\n".encode())
+        if rdict:
+            if rdict_types:
+                # also account for extra characters for fieldtype and whitespace + parenthesis
+                width = max(len(k + rdict_types[k]) for k in rdict) + 3
+            else:
+                width = max(len(k) for k in rdict)
+            fmt = "{{:>{width}}} = {{}}\n".format(width=width)
+        for key, value in rdict.items():
+            if rdict_types:
+                key = f"{key} ({rdict_types[key]})"
+            self.fp.write(fmt.format(key, value).encode())
+
+    def flush(self) -> None:
+        if self.fp:
+            self.fp.flush()
+
+    def close(self) -> None:
+        if self.fp and not is_stdout(self.fp):
+            self.fp.close()
+        self.fp = None

{flow.record-3.14.dev4 → flow.record-3.15.dev1}/flow/record/base.py

@@ -499,7 +499,7 @@ class RecordDescriptor:
             "_source": RecordField("_source", "string"),
             "_classification": RecordField("_classification", "datetime"),
             "_generated": RecordField("_generated", "datetime"),
-            "_version": RecordField("_version", "vaeint"),
+            "_version": RecordField("_version", "varint"),
         }
 
         Returns:

{flow.record-3.14.dev4 → flow.record-3.15.dev1}/flow/record/fieldtypes/__init__.py

@@ -32,6 +32,7 @@ NATIVE_UNICODE = isinstance("", str)
 UTC = timezone.utc
 
 PY_311 = sys.version_info >= (3, 11, 0)
+PY_312 = sys.version_info >= (3, 12, 0)
 
 PATH_POSIX = 0
 PATH_WINDOWS = 1
@@ -645,28 +646,31 @@ class path(pathlib.PurePath, FieldType):
             for path_part in args:
                 if isinstance(path_part, pathlib.PureWindowsPath):
                     cls = windows_path
-                    # The (string) representation of a pathlib.PureWindowsPath is not round trip equivalent if a
-                    # path starts with a \ or / followed by a drive letter, e.g.: \C:\...
-                    # Meaning:
-                    #
-                    # str(PureWindowsPath(r"\C:\WINDOWS/Temp")) !=
-                    # str(PureWindowsPath(PureWindowsPath(r"\C:\WINDOWS/Temp"))),
-                    #
-                    # repr(PureWindowsPath(r"\C:\WINDOWS/Temp")) !=
-                    # repr(PureWindowsPath(PureWindowsPath(r"\C:\WINDOWS/Temp"))),
-                    #
-                    # This would be the case though when using PurePosixPath instead.
-                    #
-                    # This construction works around that by converting all path parts
-                    # to strings first.
-                    args = (str(arg) for arg in args)
+                    if not PY_312:
+                        # For Python < 3.12, the (string) representation of a
+                        # pathlib.PureWindowsPath is not round trip equivalent if a path
+                        # starts with a \ or / followed by a drive letter, e.g.: \C:\...
+                        # Meaning:
+                        #
+                        # str(PureWindowsPath(r"\C:\WINDOWS/Temp")) !=
+                        # str(PureWindowsPath(PureWindowsPath(r"\C:\WINDOWS/Temp"))),
+                        #
+                        # repr(PureWindowsPath(r"\C:\WINDOWS/Temp")) !=
+                        # repr(PureWindowsPath(PureWindowsPath(r"\C:\WINDOWS/Temp"))),
+                        #
+                        # This would be the case though when using PurePosixPath instead.
+                        #
+                        # This construction works around that by converting all path parts
+                        # to strings first.
+                        args = (str(arg) for arg in args)
                 elif isinstance(path_part, pathlib.PurePosixPath):
                     cls = posix_path
                 elif _is_windowslike_path(path_part):
                     # This handles any custom PurePath based implementations that have a windows
                     # like path separator (\).
                     cls = windows_path
-                    args = (str(arg) for arg in args)
+                    if not PY_312:
+                        args = (str(arg) for arg in args)
                 elif _is_posixlike_path(path_part):
                     # This handles any custom PurePath based implementations that don't have a
                     # windows like path separator (\).
@@ -675,7 +679,11 @@ class path(pathlib.PurePath, FieldType):
                     continue
                 break
 
-        return cls._from_parts(args)
+        if PY_312:
+            obj = super().__new__(cls)
+        else:
+            obj = cls._from_parts(args)
+        return obj
 
     def __eq__(self, other: Any) -> bool:
         if isinstance(other, str):

{flow.record-3.14.dev4 → flow.record-3.15.dev1}/flow/record/tools/rdump.py

@@ -98,7 +98,9 @@ def main(argv=None):
     output.add_argument("-c", "--count", type=int, help="Exit after COUNT records")
     output.add_argument("--skip", metavar="COUNT", type=int, default=0, help="Skip the first COUNT records")
     output.add_argument("-w", "--writer", metavar="OUTPUT", default=None, help="Write records to output")
-    output.add_argument("-m", "--mode", default=None, choices=("csv", "json", "jsonlines", "line"), help="Output mode")
+    output.add_argument(
+        "-m", "--mode", default=None, choices=("csv", "json", "jsonlines", "line", "line-verbose"), help="Output mode"
+    )
     output.add_argument(
         "--split", metavar="COUNT", default=None, type=int, help="Write record files smaller than COUNT records"
     )
@@ -155,6 +157,15 @@ def main(argv=None):
         default=argparse.SUPPRESS,
         help="Short for --mode=line",
     )
+    aliases.add_argument(
+        "-Lv",
+        "--line-verbose",
+        action="store_const",
+        const="line-verbose",
+        dest="mode",
+        default=argparse.SUPPRESS,
+        help="Short for --mode=line-verbose",
+    )
 
     args = parser.parse_args(argv)
 
@@ -176,6 +187,7 @@ def main(argv=None):
             "json": "jsonfile://?indent=2&descriptors=false",
             "jsonlines": "jsonfile://?descriptors=false",
             "line": "line://",
+            "line-verbose": "line://?verbose=true",
         }
         uri = mode_to_uri.get(args.mode, uri)
         qparams = {

{flow.record-3.14.dev4 → flow.record-3.15.dev1}/flow/record/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.14.dev4'
-__version_tuple__ = version_tuple = (3, 14, 'dev4')
+__version__ = version = '3.15.dev1'
+__version_tuple__ = version_tuple = (3, 15, 'dev1')

{flow.record-3.14.dev4 → flow.record-3.15.dev1/flow.record.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.14.dev4
+Version: 3.15.dev1
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{flow.record-3.14.dev4 → flow.record-3.15.dev1}/tests/test_fieldtypes.py

@@ -3,6 +3,8 @@
 import hashlib
 import os
 import pathlib
+import posixpath
+import types
 from datetime import datetime, timedelta, timezone
 
 import pytest
@@ -12,6 +14,7 @@ from flow.record import RecordDescriptor, RecordReader, RecordWriter
 from flow.record.fieldtypes import (
     PATH_POSIX,
     PATH_WINDOWS,
+    PY_312,
     _is_posixlike_path,
     _is_windowslike_path,
 )
@@ -527,12 +530,29 @@ def test_digest():
 
 
 def custom_pure_path(sep, altsep):
-    class CustomFlavour(pathlib._PosixFlavour):
-        def __new__(cls):
-            instance = pathlib._PosixFlavour.__new__(cls)
-            instance.sep = sep
-            instance.altsep = altsep
-            return instance
+    # Starting from Python 3.12, pathlib._Flavours are removed as you can
+    # now properly subclass pathlib.Path
+    # The flavour property of Path's is replaced by a link to e.g.
+    # posixpath or ntpath.
+    # See also: https://github.com/python/cpython/issues/88302
+    if PY_312:
+
+        class CustomFlavour:
+            def __new__(cls, *args, **kwargs):
+                flavour = types.ModuleType("mockpath")
+                flavour.__dict__.update(posixpath.__dict__)
+                flavour.sep = sep
+                flavour.altsep = altsep
+                return flavour
+
+    else:
+
+        class CustomFlavour(pathlib._PosixFlavour):
+            def __new__(cls):
+                instance = super().__new__(cls)
+                instance.sep = sep
+                instance.altsep = altsep
+                return instance
 
     class PureCustomPath(pathlib.PurePath):
         _flavour = CustomFlavour()

{flow.record-3.14.dev4 → flow.record-3.15.dev1}/tests/test_rdump.py

@@ -624,3 +624,48 @@ def test_flow_record_invalid_tz(tmp_path, capsys):
 
     # restore DISPLAY_TZINFO just in case
     flow.record.fieldtypes.DISPLAY_TZINFO = flow_record_tz(default_tz="UTC")
+
+
+@pytest.mark.parametrize(
+    "rdump_params",
+    [
+        ["--mode=line-verbose"],
+        ["--line-verbose"],
+        ["-Lv"],
+        ["-w", "line://?verbose=true"],
+        ["-w", "line://?verbose=1"],
+        ["-w", "line://?verbose=True"],
+    ],
+)
+def test_rdump_line_verbose(tmp_path, capsys, rdump_params):
+    TestRecord = RecordDescriptor(
+        "test/rdump/line_verbose",
+        [
+            ("datetime", "stamp"),
+            ("bytes", "data"),
+            ("uint32", "counter"),
+            ("string", "foo"),
+        ],
+    )
+    record_path = tmp_path / "test.records"
+
+    with RecordWriter(record_path) as writer:
+        writer.write(TestRecord(counter=1))
+        writer.write(TestRecord(counter=2))
+        writer.write(TestRecord(counter=3))
+
+    from flow.record.adapter.line import field_types_for_record_descriptor
+
+    field_types_for_record_descriptor.cache_clear()
+    assert field_types_for_record_descriptor.cache_info().currsize == 0
+    rdump.main([str(record_path)] + rdump_params)
+    assert field_types_for_record_descriptor.cache_info().misses == 1
+    assert field_types_for_record_descriptor.cache_info().hits == 2
+    assert field_types_for_record_descriptor.cache_info().currsize == 1
+
+    captured = capsys.readouterr()
+    assert captured.err == ""
+    assert "stamp (datetime) =" in captured.out
+    assert "data (bytes) =" in captured.out
+    assert "counter (uint32) =" in captured.out
+    assert "foo (string) =" in captured.out

flow.record-3.14.dev4/flow/record/adapter/line.py

@@ -1,44 +0,0 @@
-from flow.record import open_path_or_stream
-from flow.record.adapter import AbstractWriter
-from flow.record.utils import is_stdout
-
-__usage__ = """
-Line output format adapter (writer only)
----
-Write usage: rdump -w line://[PATH]
-[PATH]: path to file. Leave empty or "-" to output to stdout
-"""
-
-
-class LineWriter(AbstractWriter):
-    """Prints all fields and values of the Record on a separate line."""
-
-    fp = None
-
-    def __init__(self, path, fields=None, exclude=None, **kwargs):
-        self.fp = open_path_or_stream(path, "wb")
-        self.count = 0
-        self.fields = fields
-        self.exclude = exclude
-        if isinstance(self.fields, str):
-            self.fields = self.fields.split(",")
-        if isinstance(self.exclude, str):
-            self.exclude = self.exclude.split(",")
-
-    def write(self, rec):
-        rdict = rec._asdict(fields=self.fields, exclude=self.exclude)
-        self.count += 1
-        self.fp.write("--[ RECORD {} ]--\n".format(self.count).encode())
-        if rdict:
-            fmt = "{{:>{width}}} = {{}}\n".format(width=max(len(k) for k in rdict))
-        for key, value in rdict.items():
-            self.fp.write(fmt.format(key, value).encode())
-
-    def flush(self):
-        if self.fp:
-            self.fp.flush()
-
-    def close(self):
-        if self.fp and not is_stdout(self.fp):
-            self.fp.close()
-        self.fp = None