flow.record 3.17.dev4__tar.gz → 3.17.dev6__tar.gz

{flow_record-3.17.dev4/flow.record.egg-info → flow_record-3.17.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.17.dev4
+Version: 3.17.dev6
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/adapter/elastic.py

@@ -106,7 +106,7 @@ class ElasticWriter(AbstractWriter):
         }
 
         if self.hash_record:
-            document["_id"] = hashlib.md5(document["_source"].encode()).hexdigest()
+            document["_id"] = hashlib.md5(document["_source"].encode(errors="surrogateescape")).hexdigest()
 
         return document
 

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/adapter/line.py

@@ -69,7 +69,7 @@ class LineWriter(AbstractWriter):
         for key, value in rdict.items():
             if rdict_types:
                 key = f"{key} ({rdict_types[key]})"
-            self.fp.write(fmt.format(key, value).encode())
+            self.fp.write(fmt.format(key, value).encode(errors="surrogateescape"))
 
     def flush(self) -> None:
         if self.fp:

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/adapter/sqlite.py

@@ -187,7 +187,7 @@ class SqliteReader(AbstractReader):
                         if value == 0:
                             row[idx] = None
                         elif isinstance(value, str):
-                            row[idx] = value.encode("utf-8")
+                            row[idx] = value.encode(errors="surrogateescape")
                 yield descriptor_cls.init_from_dict(dict(zip(fnames, row)))
 
     def __iter__(self) -> Iterator[Record]:

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/adapter/text.py

@@ -41,7 +41,7 @@ class TextWriter(AbstractWriter):
             buf = self.format_spec.format_map(DefaultMissing(rec._asdict()))
         else:
             buf = repr(rec)
-        self.fp.write(buf.encode() + b"\n")
+        self.fp.write(buf.encode(errors="surrogateescape") + b"\n")
 
         # because stdout is usually line buffered we force flush here if wanted
         if self.auto_flush:

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/adapter/xlsx.py

@@ -36,7 +36,7 @@ def sanitize_fieldvalues(values: Iterator[Any]) -> Iterator[Any]:
         elif isinstance(value, bytes):
             base64_encode = False
             try:
-                new_value = 'b"' + value.decode() + '"'
+                new_value = 'b"' + value.decode(errors="surrogateescape") + '"'
                 if ILLEGAL_CHARACTERS_RE.search(new_value):
                     base64_encode = True
                 else:
@@ -142,7 +142,7 @@ class XlsxReader(AbstractReader):
                     if field_types[idx] == "bytes":
                         if value[1] == '"':  # If so, we know this is b""
                             # Cut of the b" at the start and the trailing "
-                            value = value[2:-1].encode()
+                            value = value[2:-1].encode(errors="surrogateescape")
                         else:
                             # If not, we know it is base64 encoded (so we cut of the starting 'base64:')
                             value = b64decode(value[7:])

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/base.py

@@ -61,7 +61,7 @@ except ImportError:
 
 from collections import OrderedDict
 
-from .utils import to_native_str, to_str
+from .utils import to_str
 from .whitelist import WHITELIST, WHITELIST_TREE
 
 log = logging.getLogger(__package__)
@@ -513,7 +513,7 @@ class RecordDescriptor:
             name, fields = parse_def(name)
 
         self.name = name
-        self._field_tuples = tuple([(to_native_str(k), to_str(v)) for k, v in fields])
+        self._field_tuples = tuple([(to_str(k), to_str(v)) for k, v in fields])
         self.recordType = _generate_record_class(name, self._field_tuples)
         self.recordType._desc = self
 

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/fieldtypes/__init__.py

@@ -28,7 +28,6 @@ except ImportError:
 from flow.record.base import FieldType
 
 RE_NORMALIZE_PATH = re.compile(r"[\\/]+")
-NATIVE_UNICODE = isinstance("", str)
 
 UTC = timezone.utc
 
@@ -207,10 +206,7 @@ class stringlist(list, FieldType):
 class string(string_type, FieldType):
     def __new__(cls, value):
         if isinstance(value, bytes_type):
-            value = cls._decode(value, "utf-8")
-            if isinstance(value, bytes_type):
-                # Still bytes, so decoding failed (Python 2)
-                return bytes(value)
+            value = value.decode(errors="surrogateescape")
         return super().__new__(cls, value)
 
     def _pack(self):
@@ -221,27 +217,6 @@ class string(string_type, FieldType):
             return defang(self)
         return str.__format__(self, spec)
 
-    @classmethod
-    def _decode(cls, data, encoding):
-        """Decode a byte-string into a unicode-string.
-
-        Python 3: When `data` contains invalid unicode characters a `UnicodeDecodeError` is raised.
-        Python 2: When `data` contains invalid unicode characters the original byte-string is returned.
-        """
-        if NATIVE_UNICODE:
-            # Raises exception on decode error
-            return data.decode(encoding)
-        try:
-            return data.decode(encoding)
-        except UnicodeDecodeError:
-            # Fallback to bytes (Python 2 only)
-            preview = data[:16].encode("hex_codec") + (".." if len(data) > 16 else "")
-            warnings.warn(
-                "Got binary data in string field (hex: {}). Compatibility is not guaranteed.".format(preview),
-                RuntimeWarning,
-            )
-            return data
-
 
 # Alias for backwards compatibility
 wstring = string
@@ -278,7 +253,7 @@ class datetime(_dt, FieldType):
         if len(args) == 1 and not kwargs:
             arg = args[0]
             if isinstance(arg, bytes_type):
-                arg = arg.decode("utf-8")
+                arg = arg.decode(errors="surrogateescape")
             if isinstance(arg, string_type):
                 # If we are on Python 3.11 or newer, we can use fromisoformat() to parse the string (fast path)
                 #

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/fieldtypes/net/ip.py

@@ -1,38 +1,54 @@
-from ipaddress import ip_address, ip_network
+from __future__ import annotations
+
+from ipaddress import (
+    IPv4Address,
+    IPv4Network,
+    IPv6Address,
+    IPv6Network,
+    ip_address,
+    ip_network,
+)
+from typing import Union
 
 from flow.record.base import FieldType
 from flow.record.fieldtypes import defang
 
+_IPNetwork = Union[IPv4Network, IPv6Network]
+_IPAddress = Union[IPv4Address, IPv6Address]
+
 
 class ipaddress(FieldType):
     val = None
     _type = "net.ipaddress"
 
-    def __init__(self, addr):
+    def __init__(self, addr: str | int | bytes):
         self.val = ip_address(addr)
 
-    def __eq__(self, b):
+    def __eq__(self, b: str | int | bytes | _IPAddress) -> bool:
         try:
             return self.val == ip_address(b)
         except ValueError:
             return False
 
-    def __str__(self):
+    def __hash__(self) -> int:
+        return hash(self.val)
+
+    def __str__(self) -> str:
         return str(self.val)
 
-    def __repr__(self):
-        return "{}({!r})".format(self._type, str(self))
+    def __repr__(self) -> str:
+        return f"{self._type}({str(self)!r})"
 
-    def __format__(self, spec):
+    def __format__(self, spec: str) -> str:
         if spec == "defang":
             return defang(str(self))
         return str.__format__(str(self), spec)
 
-    def _pack(self):
+    def _pack(self) -> int:
         return int(self.val)
 
     @staticmethod
-    def _unpack(data):
+    def _unpack(data: int) -> ipaddress:
         return ipaddress(data)
 
 
@@ -40,17 +56,20 @@ class ipnetwork(FieldType):
     val = None
     _type = "net.ipnetwork"
 
-    def __init__(self, addr):
+    def __init__(self, addr: str | int | bytes):
         self.val = ip_network(addr)
 
-    def __eq__(self, b):
+    def __eq__(self, b: str | int | bytes | _IPNetwork) -> bool:
         try:
             return self.val == ip_network(b)
         except ValueError:
             return False
 
+    def __hash__(self) -> int:
+        return hash(self.val)
+
     @staticmethod
-    def _is_subnet_of(a, b):
+    def _is_subnet_of(a: _IPNetwork, b: _IPNetwork) -> bool:
         try:
             # Always false if one is v4 and the other is v6.
             if a._version != b._version:
@@ -59,23 +78,23 @@ class ipnetwork(FieldType):
         except AttributeError:
             raise TypeError("Unable to test subnet containment " "between {} and {}".format(a, b))
 
-    def __contains__(self, b):
+    def __contains__(self, b: str | int | bytes | _IPAddress) -> bool:
         try:
             return self._is_subnet_of(ip_network(b), self.val)
         except (ValueError, TypeError):
             return False
 
-    def __str__(self):
+    def __str__(self) -> str:
         return str(self.val)
 
-    def __repr__(self):
-        return "{}({!r})".format(self._type, str(self))
+    def __repr__(self) -> str:
+        return f"{self._type}({str(self)!r})"
 
-    def _pack(self):
+    def _pack(self) -> str:
         return self.val.compressed
 
     @staticmethod
-    def _unpack(data):
+    def _unpack(data: str) -> ipnetwork:
         return ipnetwork(data)
 
 

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/fieldtypes/net/ipv4.py

@@ -3,7 +3,6 @@ import struct
 import warnings
 
 from flow.record import FieldType
-from flow.record.utils import to_native_str
 
 
 def addr_long(s):
@@ -45,9 +44,6 @@ class subnet(FieldType):
             DeprecationWarning,
             stacklevel=5,
         )
-        if isinstance(addr, type("")):
-            addr = to_native_str(addr)
-
         if not isinstance(addr, str):
             raise TypeError("Subnet() argument 1 must be string, not {}".format(type(addr).__name__))
 
@@ -67,9 +63,6 @@ class subnet(FieldType):
         if addr is None:
             return False
 
-        if isinstance(addr, type("")):
-            addr = to_native_str(addr)
-
         if isinstance(addr, str):
             addr = addr_long(addr)
 

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/jsonpacker.py

@@ -47,12 +47,8 @@ class JsonRecordPacker:
                 serial["_recorddescriptor"] = obj._desc.identifier
 
             for field_type, field_name in obj._desc.get_field_tuples():
-                # PYTHON2: Because "bytes" are also "str" we have to handle this here
-                if field_type == "bytes" and isinstance(serial[field_name], str):
-                    serial[field_name] = base64.b64encode(serial[field_name]).decode()
-
                 # Boolean field types should be cast to a bool instead of staying ints
-                elif field_type == "boolean" and isinstance(serial[field_name], int):
+                if field_type == "boolean" and isinstance(serial[field_name], int):
                     serial[field_name] = bool(serial[field_name])
 
             return serial

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/utils.py

@@ -3,13 +3,10 @@ from __future__ import annotations
 import base64
 import os
 import sys
+import warnings
 from functools import wraps
 from typing import BinaryIO, TextIO
 
-_native = str
-_unicode = type("")
-_bytes = type(b"")
-
 
 def get_stdout(binary: bool = False) -> TextIO | BinaryIO:
     """Return the stdout stream as binary or text stream.
@@ -50,33 +47,32 @@ def is_stdout(fp: TextIO | BinaryIO) -> bool:
 
 def to_bytes(value):
     """Convert a value to a byte string."""
-    if value is None or isinstance(value, _bytes):
+    if value is None or isinstance(value, bytes):
         return value
-    if isinstance(value, _unicode):
-        return value.encode("utf-8")
-    return _bytes(value)
+    if isinstance(value, str):
+        return value.encode(errors="surrogateescape")
+    return bytes(value)
 
 
 def to_str(value):
     """Convert a value to a unicode string."""
-    if value is None or isinstance(value, _unicode):
+    if value is None or isinstance(value, str):
         return value
-    if isinstance(value, _bytes):
-        return value.decode("utf-8")
-    return _unicode(value)
+    if isinstance(value, bytes):
+        return value.decode(errors="surrogateescape")
+    return str(value)
 
 
 def to_native_str(value):
-    """Convert a value to a native `str`."""
-    if value is None or isinstance(value, _native):
-        return value
-    if isinstance(value, _unicode):
-        # Python 2: unicode -> str
-        return value.encode("utf-8")
-    if isinstance(value, _bytes):
-        # Python 3: bytes -> str
-        return value.decode("utf-8")
-    return _native(value)
+    warnings.warn(
+        (
+            "The to_native_str() function is deprecated, "
+            "this function will be removed in flow.record 3.20, "
+            "use to_str() instead"
+        ),
+        DeprecationWarning,
+    )
+    return to_str(value)
 
 
 def to_base64(value):

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow/record/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.17.dev4'
-__version_tuple__ = version_tuple = (3, 17, 'dev4')
+__version__ = version = '3.17.dev6'
+__version_tuple__ = version_tuple = (3, 17, 'dev6')

{flow_record-3.17.dev4 → flow_record-3.17.dev6/flow.record.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.17.dev4
+Version: 3.17.dev6
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/flow.record.egg-info/SOURCES.txt

@@ -54,6 +54,8 @@ tests/__init__.py
 tests/_utils.py
 tests/selector_explain_example.py
 tests/standalone_test.py
+tests/test_adapter_line.py
+tests/test_adapter_text.py
 tests/test_avro.py
 tests/test_avro_adapter.py
 tests/test_compiled_selector.py
@@ -75,7 +77,6 @@ tests/test_selector.py
 tests/test_splunk_adapter.py
 tests/test_sqlite_duckdb_adapter.py
 tests/test_xlsx_adapter.py
-tests/utils_inspect.py
 tests/docs/Makefile
 tests/docs/conf.py
 tests/docs/index.rst

flow_record-3.17.dev6/tests/test_adapter_line.py

@@ -0,0 +1,29 @@
+from io import BytesIO
+
+from flow.record import RecordDescriptor
+from flow.record.adapter.line import LineWriter
+
+
+def test_line_writer_write_surrogateescape():
+    output = BytesIO()
+
+    lw = LineWriter(
+        path=output,
+        fields="name",
+    )
+
+    TestRecord = RecordDescriptor(
+        "test/string",
+        [
+            ("string", "name"),
+        ],
+    )
+
+    # construct from 'bytes' but with invalid unicode bytes
+    record = TestRecord(b"R\xc3\xa9\xeamy")
+    lw.write(record)
+
+    output.seek(0)
+    data = output.read()
+
+    assert data == b"--[ RECORD 1 ]--\nname = R\xc3\xa9\xeamy\n"

flow_record-3.17.dev6/tests/test_adapter_text.py

@@ -0,0 +1,28 @@
+from io import BytesIO
+
+from flow.record import RecordDescriptor
+from flow.record.adapter.text import TextWriter
+
+
+def test_text_writer_write_surrogateescape():
+    output = BytesIO()
+
+    tw = TextWriter(
+        path=output,
+    )
+
+    TestRecord = RecordDescriptor(
+        "test/string",
+        [
+            ("string", "name"),
+        ],
+    )
+
+    # construct from 'bytes' but with invalid unicode bytes
+    record = TestRecord(b"R\xc3\xa9\xeamy")
+    tw.write(record)
+
+    output.seek(0)
+    data = output.read()
+
+    assert data == b"<test/string name='R\xc3\xa9\\udceamy'>\n"

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/tests/test_fieldtype_ip.py

@@ -48,12 +48,19 @@ def test_record_ipaddress():
     assert TestRecord("0.0.0.0").ip == "0.0.0.0"
     assert TestRecord("192.168.0.1").ip == "192.168.0.1"
     assert TestRecord("255.255.255.255").ip == "255.255.255.255"
+    assert hash(TestRecord("192.168.0.1").ip) == hash(net.ipaddress("192.168.0.1"))
 
     # ipv6
     assert TestRecord("::1").ip == "::1"
     assert TestRecord("2001:4860:4860::8888").ip == "2001:4860:4860::8888"
     assert TestRecord("2001:4860:4860::4444").ip == "2001:4860:4860::4444"
 
+    # Test whether it functions in a set
+    data = {TestRecord(ip).ip for ip in ["192.168.0.1", "192.168.0.1", "::1", "::1"]}
+    assert len(data) == 2
+    assert net.ipaddress("::1") in data
+    assert net.ipaddress("192.168.0.1") in data
+
     # instantiate from different types
     assert TestRecord(1).ip == "0.0.0.1"
     assert TestRecord(0x7F0000FF).ip == "127.0.0.255"
@@ -90,6 +97,7 @@ def test_record_ipnetwork():
     assert "192.168.1.1" not in r.subnet
     assert isinstance(r.subnet, net.ipnetwork)
     assert repr(r.subnet) == "net.ipnetwork('192.168.0.0/24')"
+    assert hash(r.subnet) == hash(net.ipnetwork("192.168.0.0/24"))
 
     r = TestRecord("192.168.1.1/32")
     assert r.subnet == "192.168.1.1"
@@ -111,6 +119,13 @@ def test_record_ipnetwork():
     assert "64:ff9b::0.0.0.0" in r.subnet
     assert "64:ff9b::255.255.255.255" in r.subnet
 
+    # Test whether it functions in a set
+    data = {TestRecord(x).subnet for x in ["192.168.0.0/24", "192.168.0.0/24", "::1", "::1"]}
+    assert len(data) == 2
+    assert net.ipnetwork("::1") in data
+    assert net.ipnetwork("192.168.0.0/24") in data
+    assert "::1" not in data
+
 
 @pytest.mark.parametrize("PSelector", [Selector, CompiledSelector])
 def test_selector_ipaddress(PSelector):

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/tests/test_fieldtypes.py

@@ -213,15 +213,8 @@ def test_string():
     assert r.name == "Rémy"
 
     # construct from 'bytes' but with invalid unicode bytes
-    if isinstance("", str):
-        # Python 3
-        with pytest.raises(UnicodeDecodeError):
-            TestRecord(b"R\xc3\xa9\xeamy")
-    else:
-        # Python 2
-        with pytest.warns(RuntimeWarning):
-            r = TestRecord(b"R\xc3\xa9\xeamy")
-            assert r.name
+    r = TestRecord(b"R\xc3\xa9\xeamy")
+    assert r.name == "Ré\udceamy"
 
 
 def test_wstring():

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/tests/test_json_packer.py

@@ -90,3 +90,23 @@ def test_record_pack_bool_regression() -> None:
 
     # pack the json string back to a record and make sure it is the same as before
     assert packer.unpack(data) == record
+
+
+def test_record_pack_surrogateescape() -> None:
+    TestRecord = RecordDescriptor(
+        "test/string",
+        [
+            ("string", "name"),
+        ],
+    )
+
+    record = TestRecord(b"R\xc3\xa9\xeamy")
+    packer = JsonRecordPacker()
+
+    data = packer.pack(record)
+
+    # pack to json string and check if the 3rd and 4th byte are properly surrogate escaped
+    assert data.startswith('{"name": "R\\u00e9\\udceamy",')
+
+    # pack the json string back to a record and make sure it is the same as before
+    assert packer.unpack(data) == record

{flow_record-3.17.dev4 → flow_record-3.17.dev6}/tests/test_record.py

@@ -1,4 +1,5 @@
 import importlib
+import inspect
 import os
 import sys
 from unittest.mock import patch
@@ -27,8 +28,6 @@ from flow.record.base import (
 from flow.record.exceptions import RecordDescriptorError
 from flow.record.stream import RecordFieldRewriter
 
-from . import utils_inspect as inspect
-
 
 def test_record_creation():
     TestRecord = RecordDescriptor(
@@ -288,8 +287,30 @@ def test_record_printer_stdout(capsys):
     writer.write(record)
 
     out, err = capsys.readouterr()
-    modifier = "" if isinstance("", str) else "u"
-    expected = "<test/a a_string={u}'hello' common={u}'world' a_count=10>\n".format(u=modifier)
+    expected = "<test/a a_string='hello' common='world' a_count=10>\n"
+    assert out == expected
+
+
+def test_record_printer_stdout_surrogateescape(capsys):
+    Record = RecordDescriptor(
+        "test/a",
+        [
+            ("string", "name"),
+        ],
+    )
+    record = Record(b"R\xc3\xa9\xeamy")
+
+    # fake capsys to be a tty.
+    def isatty():
+        return True
+
+    capsys._capture.out.tmpfile.isatty = isatty
+
+    writer = RecordPrinter(getattr(sys.stdout, "buffer", sys.stdout))
+    writer.write(record)
+
+    out, err = capsys.readouterr()
+    expected = "<test/a name='Ré\\udceamy'>\n"
     assert out == expected
 
 

flow_record-3.17.dev4/tests/utils_inspect.py

@@ -1,58 +0,0 @@
-"""
-Backport of `inspect.signature` for Python 2.
-
-Based on: https://github.com/python/cpython/blob/3.7/Lib/inspect.py
-"""
-
-import collections
-import inspect
-
-
-class _empty:
-    pass
-
-
-class Parameter:
-    POSITIONAL_ONLY = 0
-    POSITIONAL_OR_KEYWORD = 1
-    VAR_POSITIONAL = 2
-    KEYWORD_ONLY = 3
-    VAR_KEYWORD = 4
-
-    empty = _empty
-
-    def __init__(self, name, kind, default=_empty):
-        self.name = name
-        self.kind = kind
-        self.default = default
-
-
-class Signature:
-    empty = _empty
-
-    def __init__(self, parameters=None):
-        self.parameters = parameters
-
-
-def signature(obj):
-    try:
-        # Python 3
-        return inspect.signature(obj)
-    except AttributeError:
-        # Python 2
-        spec = inspect.getargspec(obj)
-
-        # Create parameter objects which are compatible with python 3 objects
-        parameters = collections.OrderedDict()
-        for i in range(0, len(spec.args)):
-            arg = spec.args[i]
-            default = _empty
-            if spec.defaults and (len(spec.args) - i <= len(spec.defaults)):
-                default = spec.defaults[i - len(spec.args)]
-            parameters[arg] = Parameter(name=arg, default=default, kind=Parameter.POSITIONAL_OR_KEYWORD)
-        if spec.varargs:
-            parameters[spec.varargs] = Parameter(name=spec.varargs, kind=Parameter.VAR_POSITIONAL)
-        if spec.keywords:
-            parameters[spec.keywords] = Parameter(name=spec.keywords, kind=Parameter.VAR_KEYWORD)
-
-        return Signature(parameters=parameters)