flow.record 3.14.dev2__tar.gz → 3.14.dev4__tar.gz

{flow.record-3.14.dev2/flow.record.egg-info → flow.record-3.14.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.14.dev2
+Version: 3.14.dev4
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{flow.record-3.14.dev2 → flow.record-3.14.dev4}/flow/record/adapter/csvfile.py

@@ -5,6 +5,7 @@ import sys
 
 from flow.record import RecordDescriptor
 from flow.record.adapter import AbstractReader, AbstractWriter
+from flow.record.base import normalize_fieldname
 from flow.record.selector import make_selector
 from flow.record.utils import is_stdout
 
@@ -14,15 +15,16 @@ Comma-separated values (CSV) adapter
 Write usage: rdump -w csvfile://[PATH]?lineterminator=[TERMINATOR]
 Read usage: rdump csvfile://[PATH]?fields=[FIELDS]
 [PATH]: path to file. Leave empty or "-" to output to stdout
-[TERMINATOR]: line terminator, default is \\r\\n
-[FIELDS]: comma-separated list of CSV fields (in case of missing CSV header)
+
+Optional parameters:
+    [TERMINATOR]: line terminator, default is \\r\\n
+    [FIELDS]: comma-separated list of CSV fields (in case of missing CSV header)
 """
 
 
 class CsvfileWriter(AbstractWriter):
-    fp = None
-
     def __init__(self, path, fields=None, exclude=None, lineterminator=None, **kwargs):
+        self.fp = None
         if path in (None, "", "-"):
             self.fp = sys.stdout
         else:
@@ -58,15 +60,19 @@ class CsvfileWriter(AbstractWriter):
 
 
 class CsvfileReader(AbstractReader):
-    fp = None
-
     def __init__(self, path, selector=None, fields=None, **kwargs):
+        self.fp = None
         self.selector = make_selector(selector)
         if path in (None, "", "-"):
             self.fp = sys.stdin
         else:
             self.fp = open(path, "r", newline="")
-        self.reader = csv.reader(self.fp)
+
+        self.dialect = "excel"
+        if self.fp.seekable():
+            self.dialect = csv.Sniffer().sniff(self.fp.read(1024))
+            self.fp.seek(0)
+        self.reader = csv.reader(self.fp, dialect=self.dialect)
 
         if isinstance(fields, str):
             # parse fields from fields argument (comma-separated string)
@@ -75,8 +81,11 @@ class CsvfileReader(AbstractReader):
             # parse fields from first CSV row
             self.fields = next(self.reader)
 
-        # Create RecordDescriptor from fields
-        self.desc = RecordDescriptor("csv/reader", [("string", col) for col in self.fields])
+        # clean field names
+        self.fields = [normalize_fieldname(col) for col in self.fields]
+
+        # Create RecordDescriptor from fields, skipping fields starting with "_" (reserved for internal use)
+        self.desc = RecordDescriptor("csv/reader", [("string", col) for col in self.fields if not col.startswith("_")])
 
     def close(self):
         if self.fp:
@@ -85,6 +94,7 @@ class CsvfileReader(AbstractReader):
 
     def __iter__(self):
         for row in self.reader:
-            record = self.desc(*row)
+            rdict = dict(zip(self.fields, row))
+            record = self.desc.init_from_dict(rdict)
             if not self.selector or self.selector.match(record):
                 yield record

{flow.record-3.14.dev2 → flow.record-3.14.dev4}/flow/record/adapter/sqlite.py

@@ -8,7 +8,7 @@ from typing import Iterator
 
 from flow.record import Record, RecordDescriptor
 from flow.record.adapter import AbstractReader, AbstractWriter
-from flow.record.base import RESERVED_FIELDS
+from flow.record.base import RESERVED_FIELDS, normalize_fieldname
 from flow.record.selector import Selector, make_selector
 
 logger = logging.getLogger(__name__)
@@ -50,27 +50,6 @@ SQLITE_FIELD_MAP = {
 }
 
 
-def sanitized_name(name: str) -> str:
-    """Returns a sanitized version of name.
-
-    Some (field) names are not allowed in flow.record, while they can be allowed in SQLite.
-    This sanitizes the name so it can still be used in flow.record.
-
-        >>> sanitized_name("my-variable-name-with-dashes")
-        'my_variable_name_with_dashes'
-        >>> sanitized_name("_my_name_starting_with_underscore")
-        'n__my_name_starting_with_underscore'
-        >>> sanitized_name("1337")
-        'n_1337'
-    """
-
-    if name not in RESERVED_FIELDS:
-        name = name.replace("-", "_")
-        if name.startswith("_") or name[0].isdecimal():
-            name = "n_" + name
-    return name
-
-
 def create_descriptor_table(con: sqlite3.Connection, descriptor: RecordDescriptor) -> None:
     """Create table for a RecordDescriptor if it doesn't exists yet."""
     table_name = descriptor.name
@@ -162,7 +141,7 @@ class SqliteReader(AbstractReader):
         # flow.record is quite strict with what is allowed in fieldnames or decriptor name.
         # While SQLite is less strict, we need to sanitize the names to make them compatible.
         table_name_org = table_name
-        table_name = sanitized_name(table_name)
+        table_name = normalize_fieldname(table_name)
 
         schema = self.con.execute(
             "SELECT c.type, c.name FROM pragma_table_info(?) c",
@@ -174,7 +153,7 @@ class SqliteReader(AbstractReader):
         fname_to_type = {}
         for idx, row in enumerate(schema):
             ftype, fname = row
-            fname = sanitized_name(fname)
+            fname = normalize_fieldname(fname)
             ftype = SQLITE_FIELD_MAP.get(ftype, "string")
             fname_to_type[fname] = ftype
             if fname not in RESERVED_FIELDS:
@@ -182,6 +161,7 @@ class SqliteReader(AbstractReader):
             fnames.append(fname)
 
         descriptor_cls = RecordDescriptor(table_name, fields)
+        table_name_org = table_name_org.replace("`", r"\\\`")
         cursor = self.con.execute(f"SELECT * FROM `{table_name_org}`")
         while True:
             rows = cursor.fetchmany(self.batch_size)

{flow.record-3.14.dev2 → flow.record-3.14.dev4}/flow/record/base.py

@@ -971,6 +971,36 @@ def extend_record(
     return ExtendedRecord.init_from_dict(collections.ChainMap(*kv_maps))
 
 
+@functools.lru_cache(maxsize=4096)
+def normalize_fieldname(field_name: str) -> str:
+    """Returns a normalized version of ``field_name``.
+
+    Some (field) names are not allowed in flow.record, while they can be allowed in other formats.
+    This normalizes the name so it can still be used in flow.record.
+    Reserved field_names are not normalized.
+
+        >>> normalize_fieldname("my-variable-name-with-dashes")
+        'my_variable_name_with_dashes'
+        >>> normalize_fieldname("_my_name_starting_with_underscore")
+        'x__my_name_starting_with_underscore'
+        >>> normalize_fieldname("1337")
+        'x_1337'
+        >>> normalize_fieldname("my name with spaces")
+        'my_name_with_spaces'
+        >>> normalize_fieldname("my name (with) parentheses")
+        'my_name__with__parentheses'
+        >>> normalize_fieldname("_generated")
+        '_generated'
+    """
+
+    if field_name not in RESERVED_FIELDS:
+        field_name = re.sub(r"[- ()]", "_", field_name)
+        # prepend `n_` if field_name is empty or starts with underscore or digit
+        if len(field_name) == 0 or field_name.startswith("_") or field_name[0].isdecimal():
+            field_name = "x_" + field_name
+    return field_name
+
+
 class DynamicFieldtypeModule:
     def __init__(self, path=""):
         self.path = path

{flow.record-3.14.dev2 → flow.record-3.14.dev4}/flow/record/stream.py

@@ -3,6 +3,7 @@ from __future__ import print_function
 import datetime
 import logging
 import os
+import reprlib
 import struct
 import sys
 from collections import ChainMap
@@ -17,6 +18,9 @@ from .packer import RecordPacker
 
 log = logging.getLogger(__package__)
 
+aRepr = reprlib.Repr()
+aRepr.maxother = 255
+
 
 def RecordOutput(fp):
     """Return a RecordPrinter if `fp` is a tty otherwise a RecordStreamWriter."""
@@ -156,7 +160,7 @@ def record_stream(sources, selector=None):
         except KeyboardInterrupt:
             raise
         except Exception as e:  # noqa: B902
-            log.warning("Exception in {!r} for {!r}: {!r} -- skipping to next reader".format(reader, src, e))
+            log.warning("Exception in %r for %r: %s -- skipping to next reader", reader, src, aRepr.repr(e))
             continue
 
 

{flow.record-3.14.dev2 → flow.record-3.14.dev4}/flow/record/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.14.dev2'
-__version_tuple__ = version_tuple = (3, 14, 'dev2')
+__version__ = version = '3.14.dev4'
+__version_tuple__ = version_tuple = (3, 14, 'dev4')

{flow.record-3.14.dev2 → flow.record-3.14.dev4/flow.record.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.14.dev2
+Version: 3.14.dev4
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{flow.record-3.14.dev2 → flow.record-3.14.dev4}/flow.record.egg-info/SOURCES.txt

@@ -56,6 +56,7 @@ tests/standalone_test.py
 tests/test_avro.py
 tests/test_avro_adapter.py
 tests/test_compiled_selector.py
+tests/test_csv_adapter.py
 tests/test_deprecations.py
 tests/test_fieldtype_ip.py
 tests/test_fieldtypes.py

flow.record-3.14.dev4/tests/test_csv_adapter.py

@@ -0,0 +1,75 @@
+from datetime import datetime, timezone
+from pathlib import Path
+
+import pytest
+
+from flow.record import RecordReader
+
+
+@pytest.mark.parametrize("delimiter", [",", ";", "\t", "|"])
+def test_csv_sniff(tmp_path: Path, delimiter: str) -> None:
+    """Test CSV adapter with sniffing the dialect."""
+    input_data = delimiter.join(["title", "year", "imdb"]) + "\n"
+    input_data += delimiter.join(["The Shawshank Redemption", "1994", "tt0111161"]) + "\n"
+    input_data += delimiter.join(["The Matrix", "1998", "tt0133093"]) + "\n"
+
+    csv_path = tmp_path / "test.csv"
+    csv_path.write_text(input_data)
+
+    with RecordReader(csv_path) as reader:
+        records = list(reader)
+        assert len(records) == 2
+
+        assert records[0].title == "The Shawshank Redemption"
+        assert records[0].year == "1994"
+        assert records[0].imdb == "tt0111161"
+
+        assert records[1].title == "The Matrix"
+        assert records[1].year == "1998"
+        assert records[1].imdb == "tt0133093"
+
+
+def test_csv_non_standard_headers(tmp_path: Path) -> None:
+    """Test CSV adapter with header names that need to be cleaned up."""
+    input_data = "Filename,Full Path,Size (bytes)\n"
+    input_data += "passwd,/etc/passwd,2370\n"
+    input_data += "shadow,/etc/shadow,1290\n"
+
+    csv_path = tmp_path / "test.csv"
+    csv_path.write_text(input_data)
+
+    with RecordReader(csv_path) as reader:
+        records = list(reader)
+        assert len(records) == 2
+
+        assert records[0].Filename == "passwd"
+        assert records[0].Full_Path == "/etc/passwd"
+        assert records[0].Size__bytes_ == "2370"
+
+        assert records[1].Filename == "shadow"
+        assert records[1].Full_Path == "/etc/shadow"
+        assert records[1].Size__bytes_ == "1290"
+
+
+def test_csv_read_reserved_fields(tmp_path: Path) -> None:
+    """Test CSV adapter with reading reserved field names."""
+    input_data = "_generated,_source,foo,bar\n"
+    input_data += "2023-11-11 11:11:11.111111+11:11,single,hello,world\n"
+    input_data += "2023-11-14T22:13:20+00:00,epoch,goodbye,planet\n"
+
+    csv_path = tmp_path / "test.csv"
+    csv_path.write_text(input_data)
+
+    with RecordReader(csv_path) as reader:
+        records = list(reader)
+        assert len(records) == 2
+
+        assert records[0]._generated == datetime.fromisoformat("2023-11-11 11:11:11.111111+11:11")
+        assert records[0]._source == "single"
+        assert records[0].foo == "hello"
+        assert records[0].bar == "world"
+
+        assert records[1]._generated == datetime.fromtimestamp(1700000000, tz=timezone.utc)
+        assert records[1]._source == "epoch"
+        assert records[1].foo == "goodbye"
+        assert records[1].bar == "planet"

{flow.record-3.14.dev2 → flow.record-3.14.dev4}/tests/test_record.py

@@ -15,7 +15,7 @@ from flow.record import (
     fieldtypes,
     record_stream,
 )
-from flow.record.base import merge_record_descriptors
+from flow.record.base import merge_record_descriptors, normalize_fieldname
 from flow.record.exceptions import RecordDescriptorError
 from flow.record.stream import RecordFieldRewriter
 
@@ -781,3 +781,14 @@ def test_merge_record_descriptor_name():
     assert MergedRecord.name == "test/ip_record"
     record = MergedRecord()
     assert record._desc.name == "test/ip_record"
+
+
+def test_normalize_fieldname():
+    assert normalize_fieldname("hello") == "hello"
+    assert normalize_fieldname("my-variable-name-with-dashes") == "my_variable_name_with_dashes"
+    assert normalize_fieldname("_my_name_starting_with_underscore") == "x__my_name_starting_with_underscore"
+    assert normalize_fieldname("1337") == "x_1337"
+    assert normalize_fieldname("my name with spaces") == "my_name_with_spaces"
+    assert normalize_fieldname("my name (with) parentheses") == "my_name__with__parentheses"
+    assert normalize_fieldname("_generated") == "_generated"
+    assert normalize_fieldname("_source") == "_source"

{flow.record-3.14.dev2 → flow.record-3.14.dev4}/tests/test_sqlite_adapter.py

@@ -6,7 +6,8 @@ from typing import Any, Iterator
 import pytest
 
 from flow.record import Record, RecordDescriptor, RecordReader, RecordWriter
-from flow.record.adapter.sqlite import prepare_insert_sql, sanitized_name
+from flow.record.adapter.sqlite import prepare_insert_sql
+from flow.record.base import normalize_fieldname
 from flow.record.exceptions import RecordDescriptorError
 
 
@@ -78,7 +79,7 @@ def test_field_name_sanitization(tmp_path: Path, field_name: str) -> None:
     con.close()
 
     data_records = []
-    sanitized_field_name = sanitized_name(field_name)
+    sanitized_field_name = normalize_fieldname(field_name)
 
     with RecordReader(f"sqlite://{db}") as reader:
         data_records = [(getattr(record, sanitized_field_name),) for record in reader]