flow.record 3.19.dev7__tar.gz → 3.19.dev9__tar.gz

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: flow.record
-Version: 3.19.dev7
+Version: 3.19.dev9
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/flow/record/adapter/elastic.py

@@ -6,8 +6,14 @@ import queue
 import threading
 from typing import TYPE_CHECKING
 
-import elasticsearch
-import elasticsearch.helpers
+try:
+    import elasticsearch
+    import elasticsearch.helpers
+
+    HAS_ELASTIC = True
+
+except ImportError:
+    HAS_ELASTIC = False
 
 from flow.record.adapter import AbstractReader, AbstractWriter
 from flow.record.base import Record, RecordDescriptor
@@ -33,6 +39,8 @@ Optional arguments:
   [INDEX]: name of the index to use (default: records)
   [VERIFY_CERTS]: verify certs of Elasticsearch instance (default: True)
   [HASH_RECORD]: make record unique by hashing record [slow] (default: False)
+  [REQUEST_TIMEOUT]: maximum duration in seconds for a request to Elastic (default: 30)
+  [MAX_RETRIES]: maximum retries before a record is marked as failed (default: 3)
   [_META_*]: record metadata fields (default: None)
 """
 
@@ -49,8 +57,19 @@ class ElasticWriter(AbstractWriter):
         hash_record: str | bool = False,
         api_key: str | None = None,
         queue_size: int = 100000,
+        request_timeout: int = 30,
+        max_retries: int = 3,
         **kwargs,
     ) -> None:
+        """Initialize the ElasticWriter.
+
+        Resources:
+            - https://elasticsearch-py.readthedocs.io/en/v8.17.1/api/elasticsearch.html
+        """
+
+        if not HAS_ELASTIC:
+            raise RuntimeError("Required dependency 'elasticsearch' missing")
+
         self.index = index
         self.uri = uri
         verify_certs = str(verify_certs).lower() in ("1", "true")
@@ -63,20 +82,23 @@ class ElasticWriter(AbstractWriter):
 
         self.queue: queue.Queue[Record | StopIteration] = queue.Queue(maxsize=queue_size)
         self.event = threading.Event()
+        self.exception: Exception | None = None
+        threading.excepthook = self.excepthook
 
         self.es = elasticsearch.Elasticsearch(
             uri,
             verify_certs=verify_certs,
             http_compress=http_compress,
             api_key=api_key,
+            request_timeout=request_timeout,
+            retry_on_timeout=True,
+            max_retries=max_retries,
         )
 
         self.json_packer = JsonRecordPacker()
 
         self.thread = threading.Thread(target=self.streaming_bulk_thread)
         self.thread.start()
-        self.exception: Exception | None = None
-        threading.excepthook = self.excepthook
 
         if not verify_certs:
             # Disable InsecureRequestWarning of urllib3, caused by the verify_certs flag.
@@ -140,20 +162,28 @@ class ElasticWriter(AbstractWriter):
             yield self.record_to_document(record, index=self.index)
 
     def streaming_bulk_thread(self) -> None:
-        """Thread that streams the documents to ES via the bulk api"""
+        """Thread that streams the documents to ES via the bulk api.
 
-        for ok, item in elasticsearch.helpers.streaming_bulk(
+        Resources:
+            - https://elasticsearch-py.readthedocs.io/en/v8.17.1/helpers.html#elasticsearch.helpers.streaming_bulk
+            - https://github.com/elastic/elasticsearch-py/blob/main/elasticsearch/helpers/actions.py#L362
+        """
+        for _ok, _item in elasticsearch.helpers.streaming_bulk(
             self.es,
             self.document_stream(),
-            raise_on_error=False,
-            raise_on_exception=False,
+            raise_on_error=True,
+            raise_on_exception=True,
+            # Some settings have to be redefined because streaming_bulk does not inherit them from the self.es instance.
+            max_retries=3,
         ):
-            if not ok:
-                log.error("Failed to insert %r", item)
+            pass
 
         self.event.set()
 
     def write(self, record: Record) -> None:
+        if self.exception:
+            raise self.exception
+
         self.queue.put(record)
 
     def flush(self) -> None:
@@ -179,6 +209,8 @@ class ElasticReader(AbstractReader):
         http_compress: str | bool = True,
         selector: None | Selector | CompiledSelector = None,
         api_key: str | None = None,
+        request_timeout: int = 30,
+        max_retries: int = 3,
         **kwargs,
     ) -> None:
         self.index = index
@@ -195,6 +227,9 @@ class ElasticReader(AbstractReader):
             verify_certs=verify_certs,
             http_compress=http_compress,
             api_key=api_key,
+            request_timeout=request_timeout,
+            retry_on_timeout=True,
+            max_retries=max_retries,
         )
 
         if not verify_certs:

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/flow/record/fieldtypes/net/__init__.py

@@ -1,12 +1,19 @@
 from __future__ import annotations
 
 from flow.record.fieldtypes import string
-from flow.record.fieldtypes.net.ip import IPAddress, IPNetwork, ipaddress, ipnetwork
+from flow.record.fieldtypes.net.ip import (
+    IPAddress,
+    IPNetwork,
+    ipaddress,
+    ipinterface,
+    ipnetwork,
+)
 
 __all__ = [
     "IPAddress",
     "IPNetwork",
     "ipaddress",
+    "ipinterface",
     "ipnetwork",
 ]
 

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/flow/record/fieldtypes/net/ip.py

@@ -2,10 +2,13 @@ from __future__ import annotations
 
 from ipaddress import (
     IPv4Address,
+    IPv4Interface,
     IPv4Network,
     IPv6Address,
+    IPv6Interface,
     IPv6Network,
     ip_address,
+    ip_interface,
     ip_network,
 )
 from typing import Union
@@ -15,16 +18,19 @@ from flow.record.fieldtypes import defang
 
 _IPNetwork = Union[IPv4Network, IPv6Network]
 _IPAddress = Union[IPv4Address, IPv6Address]
+_IPInterface = Union[IPv4Interface, IPv6Interface]
+_ConversionTypes = Union[str, int, bytes]
+_IPTypes = Union[_IPNetwork, _IPAddress, _IPInterface]
 
 
 class ipaddress(FieldType):
-    val = None
+    val: _IPAddress = None
     _type = "net.ipaddress"
 
-    def __init__(self, addr: str | int | bytes):
+    def __init__(self, addr: _ConversionTypes | _IPAddress):
         self.val = ip_address(addr)
 
-    def __eq__(self, b: str | int | bytes | _IPAddress) -> bool:
+    def __eq__(self, b: _ConversionTypes | _IPAddress) -> bool:
         try:
             return self.val == ip_address(b)
         except ValueError:
@@ -53,13 +59,13 @@ class ipaddress(FieldType):
 
 
 class ipnetwork(FieldType):
-    val = None
+    val: _IPNetwork = None
     _type = "net.ipnetwork"
 
-    def __init__(self, addr: str | int | bytes):
+    def __init__(self, addr: _ConversionTypes | _IPNetwork):
         self.val = ip_network(addr)
 
-    def __eq__(self, b: str | int | bytes | _IPNetwork) -> bool:
+    def __eq__(self, b: _ConversionTypes | _IPNetwork) -> bool:
         try:
             return self.val == ip_network(b)
         except ValueError:
@@ -98,6 +104,52 @@ class ipnetwork(FieldType):
     def _unpack(data: str) -> ipnetwork:
         return ipnetwork(data)
 
+    @property
+    def netmask(self) -> ipaddress:
+        return ipaddress(self.val.netmask)
+
+
+class ipinterface(FieldType):
+    val: _IPInterface = None
+    _type = "net.ipinterface"
+
+    def __init__(self, addr: _ConversionTypes | _IPTypes) -> None:
+        self.val = ip_interface(addr)
+
+    def __eq__(self, b: _ConversionTypes | _IPTypes) -> bool:
+        try:
+            return self.val == ip_interface(b)
+        except ValueError:
+            return False
+
+    def __hash__(self) -> int:
+        return hash(self.val)
+
+    def __str__(self) -> str:
+        return str(self.val)
+
+    def __repr__(self) -> str:
+        return f"{self._type}({str(self)!r})"
+
+    @property
+    def ip(self) -> ipaddress:
+        return ipaddress(self.val.ip)
+
+    @property
+    def network(self) -> ipnetwork:
+        return ipnetwork(self.val.network)
+
+    @property
+    def netmask(self) -> ipaddress:
+        return ipaddress(self.val.netmask)
+
+    def _pack(self) -> str:
+        return self.val.compressed
+
+    @staticmethod
+    def _unpack(data: str) -> ipinterface:
+        return ipinterface(data)
+
 
 # alias: net.IPAddress -> net.ipaddress
 # alias: net.IPNetwork -> net.ipnetwork

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/flow/record/jsonpacker.py

@@ -68,7 +68,7 @@ class JsonRecordPacker:
                 "sha1": obj.sha1,
                 "sha256": obj.sha256,
             }
-        if isinstance(obj, (fieldtypes.net.ipaddress, fieldtypes.net.ipnetwork)):
+        if isinstance(obj, (fieldtypes.net.ipaddress, fieldtypes.net.ipnetwork, fieldtypes.net.ipinterface)):
             return str(obj)
         if isinstance(obj, bytes):
             return base64.b64encode(obj).decode()

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/flow/record/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.19.dev7'
-__version_tuple__ = version_tuple = (3, 19, 'dev7')
+__version__ = version = '3.19.dev9'
+__version_tuple__ = version_tuple = (3, 19, 'dev9')

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/flow/record/whitelist.py

@@ -24,6 +24,7 @@ WHITELIST = [
     "bytes",
     "record",
     "net.ipaddress",
+    "net.ipinterface",
     "net.ipnetwork",
     "net.IPAddress",
     "net.IPNetwork",

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/flow.record.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: flow.record
-Version: 3.19.dev7
+Version: 3.19.dev9
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{flow_record-3.19.dev7 → flow_record-3.19.dev9}/tests/test_fieldtype_ip.py

@@ -128,6 +128,85 @@ def test_record_ipnetwork() -> None:
     assert "::1" not in data
 
 
+def test_record_ipinterface() -> None:
+    TestRecord = RecordDescriptor(
+        "test/ipinterface",
+        [
+            ("net.ipinterface", "interface"),
+        ],
+    )
+
+    # ipv4
+    r = TestRecord("192.168.0.0/24")
+    assert r.interface == "192.168.0.0/24"
+    assert "bad.ip" not in r.interface.network
+    assert "192.168.0.1" in r.interface.network
+    assert isinstance(r.interface, net.ipinterface)
+    assert repr(r.interface) == "net.ipinterface('192.168.0.0/24')"
+    assert hash(r.interface) == hash(net.ipinterface("192.168.0.0/24"))
+
+    r = TestRecord("192.168.1.1")
+    assert r.interface.ip == "192.168.1.1"
+    assert r.interface.network == "192.168.1.1/32"
+    assert r.interface == "192.168.1.1/32"
+    assert r.interface.netmask == "255.255.255.255"
+
+    r = TestRecord("192.168.1.24/255.255.255.0")
+    assert r.interface == "192.168.1.24/24"
+    assert r.interface.ip == "192.168.1.24"
+    assert r.interface.network == "192.168.1.0/24"
+    assert r.interface.netmask == "255.255.255.0"
+
+    # ipv6 - https://en.wikipedia.org/wiki/IPv6_address
+    r = TestRecord("::1")
+    assert r.interface == "::1"
+    assert r.interface == "::1/128"
+
+    r = TestRecord("64:ff9b::2/96")
+    assert r.interface == "64:ff9b::2/96"
+    assert r.interface.ip == "64:ff9b::2"
+    assert r.interface.network == "64:ff9b::/96"
+    assert r.interface.netmask == "ffff:ffff:ffff:ffff:ffff:ffff::"
+
+    # instantiate from different types
+    assert TestRecord(1).interface == "0.0.0.1/32"
+    assert TestRecord(0x7F0000FF).interface == "127.0.0.255/32"
+    assert TestRecord(b"\x7f\xff\xff\xff").interface == "127.255.255.255/32"
+
+    # Test whether it functions in a set
+    data = {TestRecord(x).interface for x in ["192.168.0.0/24", "192.168.0.0/24", "::1", "::1"]}
+    assert len(data) == 2
+    assert net.ipinterface("::1") in data
+    assert net.ipinterface("192.168.0.0/24") in data
+    assert "::1" not in data
+
+
+def test_record_ipinterface_types() -> None:
+    TestRecord = RecordDescriptor(
+        "test/ipinterface",
+        [
+            (
+                "net.ipinterface",
+                "interface",
+            )
+        ],
+    )
+
+    r = TestRecord("192.168.0.255/24")
+    _if = r.interface
+    assert isinstance(_if, net.ipinterface)
+    assert isinstance(_if.ip, net.ipaddress)
+    assert isinstance(_if.network, net.ipnetwork)
+    assert isinstance(_if.netmask, net.ipaddress)
+
+    r = TestRecord("64:ff9b::/96")
+    _if = r.interface
+    assert isinstance(_if, net.ipinterface)
+    assert isinstance(_if.ip, net.ipaddress)
+    assert isinstance(_if.network, net.ipnetwork)
+    assert isinstance(_if.netmask, net.ipaddress)
+
+
 @pytest.mark.parametrize("PSelector", [Selector, CompiledSelector])
 def test_selector_ipaddress(PSelector: type[Selector]) -> None:
     TestRecord = RecordDescriptor(