flow.record 3.15.dev13__tar.gz → 3.15.dev15__tar.gz

{flow.record-3.15.dev13/flow.record.egg-info → flow.record-3.15.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.15.dev13
+Version: 3.15.dev15
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -35,6 +35,8 @@ Requires-Dist: fastavro[snappy]; extra == "avro"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Requires-Dist: pytz; extra == "duckdb"
+Provides-Extra: splunk
+Requires-Dist: httpx; extra == "splunk"
 Provides-Extra: test
 Requires-Dist: flow.record[compression]; extra == "test"
 Requires-Dist: flow.record[avro]; extra == "test"

flow.record-3.15.dev15/flow/record/adapter/splunk.py

@@ -0,0 +1,282 @@
+import json
+import logging
+import socket
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+from urllib.parse import urlparse
+
+try:
+    import httpx
+
+    HAS_HTTPX = True
+except ImportError:
+    HAS_HTTPX = False
+
+from flow.record.adapter import AbstractReader, AbstractWriter
+from flow.record.base import Record
+from flow.record.jsonpacker import JsonRecordPacker
+from flow.record.utils import to_base64, to_bytes, to_str
+
+__usage__ = """
+Splunk output adapter (writer only)
+---
+Write usage: rdump -w splunk+[PROTOCOL]://[IP]:[PORT]?tag=[TAG]&token=[TOKEN]&sourcetype=[SOURCETYPE]
+[PROTOCOL]: Protocol to use for forwarding data. Can be tcp, http or https, defaults to tcp if omitted.
+[IP]:[PORT]: ip and port to a splunk instance
+[TAG]: optional value to add as "rdtag" output field when writing
+[TOKEN]: Authentication token for sending data over HTTP(S)
+[SOURCETYPE]: Set sourcetype of data. Defaults to records, but can also be set to JSON.
+[SSL_VERIFY]: Whether to verify the server certificate when sending data over HTTP(S). Defaults to True.
+"""
+
+log = logging.getLogger(__package__)
+
+# Amount of records to bundle into a single request when sending data over HTTP(S).
+RECORD_BUFFER_LIMIT = 20
+
+# https://docs.splunk.com/Documentation/Splunk/7.3.1/Data/Configureindex-timefieldextraction
+RESERVED_SPLUNK_FIELDS = [
+    "_indextime",
+    "_time",
+    "index",
+    "punct",
+    "source",
+    "sourcetype",
+    "tag",
+    "type",
+]
+
+RESERVED_RECORD_FIELDS = ["_classification", "_generated", "_source"]
+
+PREFIX_WITH_RD = set(RESERVED_SPLUNK_FIELDS + RESERVED_RECORD_FIELDS)
+
+
+class Protocol(Enum):
+    HTTP = "http"
+    HTTPS = "https"
+    TCP = "tcp"
+
+
+class SourceType(Enum):
+    JSON = "json"
+    RECORDS = "records"
+
+
+def splunkify_key_value(record: Record, tag: Optional[str] = None) -> str:
+    ret = []
+
+    ret.append(f'rdtype="{record._desc.name}"')
+
+    if tag is None:
+        ret.append("rdtag=None")
+    else:
+        ret.append(f'rdtag="{tag}"')
+
+    for field in record._desc.get_all_fields():
+        # Omit the _version field as the Splunk adapter has no reader support for deserialising records back.
+        if field == "_version":
+            continue
+
+        val = getattr(record, field)
+
+        if field in PREFIX_WITH_RD:
+            field = f"rd_{field}"
+
+        if val is None:
+            ret.append(f"{field}=None")
+        else:
+            val = to_base64(val) if isinstance(val, bytes) else to_str(val)
+            val = val.replace("\\", "\\\\").replace('"', '\\"')
+            ret.append(f'{field}="{val}"')
+
+    return " ".join(ret)
+
+
+def splunkify_json(packer: JsonRecordPacker, record: Record, tag: Optional[str] = None) -> str:
+    ret = {}
+
+    indexer_fields = [
+        ("host", "host"),
+        ("host", "hostname"),
+        ("time", "ts"),
+    ]
+
+    # When converting a record to json text for splunk, we distinguish between the 'event' (containing the data) and a
+    # few other fields that are splunk-specific for indexing. We add those 'indexer_fields' to the return object first.
+    for splunk_name, field_name in indexer_fields:
+        if hasattr(record, field_name):
+            val = getattr(record, field_name)
+            if val:
+                if isinstance(val, datetime):
+                    # Convert datetime objects to epoch timestamp for reserved fields.
+                    ret[splunk_name] = val.timestamp()
+                    continue
+                ret[splunk_name] = to_str(val)
+
+    record_as_dict = packer.pack_obj(record)
+
+    # Omit the _version field as the Splunk adapter has no reader support for deserialising records back.
+    del record_as_dict["_version"]
+
+    # These fields end up in the 'event', but we have a few reserved field names. If those field names are in the
+    # record, we prefix them with 'rd_' (short for record descriptor)
+    for field in PREFIX_WITH_RD:
+        if field not in record_as_dict:
+            continue
+        new_field = f"rd_{field}"
+
+        record_as_dict[new_field] = record_as_dict[field]
+        del record_as_dict[field]
+
+    # Almost done, just have to add the tag and the type (i.e the record descriptor's name) to the event.
+    record_as_dict["rdtag"] = tag
+
+    # Yes.
+    record_as_dict["rdtype"] = record._desc.name
+
+    ret["event"] = record_as_dict
+    return json.dumps(ret, default=packer.pack_obj)
+
+
+class SplunkWriter(AbstractWriter):
+    sock = None
+    session = None
+
+    def __init__(
+        self,
+        uri: str,
+        tag: Optional[str] = None,
+        token: Optional[str] = None,
+        sourcetype: Optional[str] = None,
+        ssl_verify: bool = True,
+        **kwargs,
+    ):
+        # If the writer is initiated without a protocol, we assume we will be writing over tcp
+        if "://" not in uri:
+            uri = f"tcp://{uri}"
+
+        if sourcetype is None:
+            log.warning("No sourcetype provided, assuming 'records' sourcetype")
+            sourcetype = SourceType.RECORDS
+
+        parsed_url = urlparse(uri)
+        url_scheme = parsed_url.scheme.lower()
+
+        self.sourcetype = SourceType(sourcetype)
+        self.protocol = Protocol(url_scheme)
+
+        if self.protocol == Protocol.TCP and self.sourcetype != SourceType.RECORDS:
+            raise ValueError("For sending data to Splunk over TCP, only the 'records' sourcetype is allowed")
+
+        self.host = parsed_url.hostname
+        self.port = parsed_url.port
+        self.tag = tag
+        self.record_buffer = []
+        self._warned = False
+        self.packer = None
+
+        if self.sourcetype == SourceType.JSON:
+            self.packer = JsonRecordPacker(indent=4, pack_descriptors=False)
+
+        if self.protocol == Protocol.TCP:
+            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.SOL_TCP)
+            self.sock.connect((self.host, self.port))
+            self._send = self._send_tcp
+        elif self.protocol in (Protocol.HTTP, Protocol.HTTPS):
+            if not HAS_HTTPX:
+                raise ImportError("The httpx library is required for sending data over HTTP(S)")
+
+            scheme = self.protocol.value
+            self.token = token
+            if not self.token:
+                raise ValueError("An authorization token is required for the HTTP collector")
+            if not self.token.startswith("Splunk "):
+                self.token = f"Splunk {self.token}"
+
+            # Assume verify=True unless specified otherwise.
+            self.verify = str(ssl_verify).lower() not in ("0", "false")
+            if not self.verify:
+                log.warning("Certificate verification is disabled")
+
+            endpoint = "event" if self.sourcetype != SourceType.RECORDS else "raw"
+            port = f":{self.port}" if self.port else ""
+            self.url = f"{scheme}://{self.host}{port}/services/collector/{endpoint}?auto_extract_timestamp=true"
+
+            self.headers = {
+                "Authorization": self.token,
+                # A randomized value so that Splunk can loadbalance between different incoming datastreams
+                "X-Splunk-Request-Channel": str(uuid.uuid4()),
+            }
+
+            self.session = httpx.Client(verify=self.verify, headers=self.headers)
+
+            self._send = self._send_http
+
+    def _cache_records_for_http(self, data: Optional[bytes] = None, flush: bool = False) -> Optional[bytes]:
+        # It's possible to call this function without any data, purely to flush. Hence this check.
+        if data:
+            self.record_buffer.append(data)
+        if len(self.record_buffer) < RECORD_BUFFER_LIMIT and not flush:
+            # Buffer limit not exceeded yet, so we do not return a buffer yet, unless buffer is explicitly flushed.
+            return
+        buf = b"".join(self.record_buffer)
+        if not buf:
+            return
+
+        # We're going to be returning a buffer for the writer to send, so we can clear the internal record buffer.
+        self.record_buffer.clear()
+        return buf
+
+    def _send(self, data: bytes) -> None:
+        raise RuntimeError("This method should be overridden at runtime")
+
+    def _send_http(self, data: Optional[bytes] = None, flush: bool = False) -> None:
+        buf = self._cache_records_for_http(data, flush)
+        if not buf:
+            return
+        response = self.session.post(self.url, data=buf)
+        if response.status_code != 200:
+            raise ConnectionError(f"{response.text} ({response.status_code})")
+
+    def _send_tcp(self, data: bytes) -> None:
+        self.sock.sendall(data)
+
+    def write(self, record: Record) -> None:
+        if not self._warned and "rdtag" in record._desc.fields:
+            self._warned = True
+            log.warning(
+                "Record has 'rdtag' field which conflicts with the Splunk adapter -- "
+                "Splunk output will have duplicate 'rdtag' fields",
+            )
+
+        if self.sourcetype == SourceType.RECORDS:
+            rec = splunkify_key_value(record, self.tag)
+        else:
+            rec = splunkify_json(self.packer, record, self.tag)
+
+        # Trail with a newline for line breaking.
+        data = to_bytes(rec) + b"\n"
+
+        self._send(data)
+
+    def flush(self) -> None:
+        if self.protocol in [Protocol.HTTP, Protocol.HTTPS]:
+            self._send_http(flush=True)
+
+    def close(self) -> None:
+        # For TCP
+        if self.sock:
+            self.sock.close()
+        self.sock = None
+
+        if self.session:
+            self.flush()
+            self.session.close()
+        self.session = None
+
+
+class SplunkReader(AbstractReader):
+    def __init__(self, path, selector=None, **kwargs):
+        raise NotImplementedError()

{flow.record-3.15.dev13 → flow.record-3.15.dev15}/flow/record/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.15.dev13'
-__version_tuple__ = version_tuple = (3, 15, 'dev13')
+__version__ = version = '3.15.dev15'
+__version_tuple__ = version_tuple = (3, 15, 'dev15')

{flow.record-3.15.dev13 → flow.record-3.15.dev15/flow.record.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.15.dev13
+Version: 3.15.dev15
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -35,6 +35,8 @@ Requires-Dist: fastavro[snappy]; extra == "avro"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Requires-Dist: pytz; extra == "duckdb"
+Provides-Extra: splunk
+Requires-Dist: httpx; extra == "splunk"
 Provides-Extra: test
 Requires-Dist: flow.record[compression]; extra == "test"
 Requires-Dist: flow.record[avro]; extra == "test"

{flow.record-3.15.dev13 → flow.record-3.15.dev15}/flow.record.egg-info/requires.txt

@@ -23,6 +23,9 @@ elasticsearch
 [geoip]
 maxminddb
 
+[splunk]
+httpx
+
 [test]
 flow.record[compression]
 flow.record[avro]

{flow.record-3.15.dev13 → flow.record-3.15.dev15}/pyproject.toml

@@ -53,6 +53,9 @@ duckdb = [
     "duckdb",
     "pytz", # duckdb requires pytz for timezone support
 ]
+splunk = [
+    "httpx",
+]
 test = [
     "flow.record[compression]",
     "flow.record[avro]",

flow.record-3.15.dev15/tests/test_splunk_adapter.py

@@ -0,0 +1,403 @@
+import datetime
+import json
+import sys
+from typing import Iterator
+from unittest.mock import ANY, MagicMock, patch
+
+import pytest
+
+import flow.record.adapter.splunk
+from flow.record import RecordDescriptor
+from flow.record.adapter.splunk import (
+    Protocol,
+    SplunkWriter,
+    splunkify_json,
+    splunkify_key_value,
+)
+from flow.record.jsonpacker import JsonRecordPacker
+
+BASE_FIELD_VALUES = {
+    "rd__source": None,
+    "rd__classification": None,
+    "rd__generated": ANY,
+}
+
+JSON_PACKER = JsonRecordPacker(pack_descriptors=False)
+
+# Reserved fields is an ordered dict so we can make assertions with a static order of reserved fields.
+RESERVED_FIELDS_KEY_VALUE_SUFFIX = 'rd__source=None rd__classification=None rd__generated="'
+
+
+@pytest.fixture
+def mock_httpx_package(monkeypatch: pytest.MonkeyPatch) -> Iterator[MagicMock]:
+    with monkeypatch.context() as m:
+        mock_httpx = MagicMock()
+        m.setitem(sys.modules, "httpx", mock_httpx)
+
+        yield mock_httpx
+
+
+def test_splunkify_reserved_field():
+    with patch.object(
+        flow.record.adapter.splunk,
+        "PREFIX_WITH_RD",
+        set(["foo", "_generated", "_classification", "_source"]),
+    ):
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("string", "foo")],
+        )
+
+        test_record = test_record_descriptor(foo="bar")
+
+        output_key_value = splunkify_key_value(test_record)
+        output_json = splunkify_json(JSON_PACKER, test_record)
+
+        assert output_key_value.startswith(
+            f'rdtype="test/record" rdtag=None rd_foo="bar" {RESERVED_FIELDS_KEY_VALUE_SUFFIX}'
+        )
+
+        assert json.loads(output_json) == {
+            "event": dict(
+                {
+                    "rdtag": None,
+                    "rdtype": "test/record",
+                    "rd_foo": "bar",
+                },
+                **BASE_FIELD_VALUES,
+            )
+        }
+
+
+def test_splunkify_normal_field():
+    with patch.object(
+        flow.record.adapter.splunk,
+        "RESERVED_SPLUNK_FIELDS",
+        set(),
+    ):
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("string", "foo")],
+        )
+
+        test_record = test_record_descriptor(foo="bar")
+
+        output_key_value = splunkify_key_value(test_record)
+        output_json = splunkify_json(JSON_PACKER, test_record)
+        assert output_key_value.startswith(
+            f'rdtype="test/record" rdtag=None foo="bar" {RESERVED_FIELDS_KEY_VALUE_SUFFIX}'
+        )
+        assert json.loads(output_json) == {
+            "event": dict(
+                {
+                    "rdtag": None,
+                    "rdtype": "test/record",
+                    "foo": "bar",
+                },
+                **BASE_FIELD_VALUES,
+            )
+        }
+
+
+def test_splunkify_source_field():
+    with patch.object(
+        flow.record.adapter.splunk,
+        "RESERVED_SPLUNK_FIELDS",
+        set(),
+    ):
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("string", "source")],
+        )
+
+        test_record = test_record_descriptor(source="file_on_target")
+        test_record._source = "path_of_target"
+
+        output_key_value = splunkify_key_value(test_record)
+        output_json = splunkify_json(JSON_PACKER, test_record)
+        assert output_key_value.startswith(
+            'rdtype="test/record" rdtag=None rd_source="file_on_target" rd__source="path_of_target"'
+        )
+
+        expected_base_field_values = BASE_FIELD_VALUES.copy()
+        expected_base_field_values["rd__source"] = "path_of_target"
+
+        assert json.loads(output_json) == {
+            "event": dict(
+                {
+                    "rdtag": None,
+                    "rdtype": "test/record",
+                    "rd_source": "file_on_target",
+                    "rd__source": "path_of_target",
+                    "rd__generated": ANY,
+                    "rd__classification": None,
+                },
+            ),
+        }
+
+
+def test_splunkify_rdtag_field():
+    with patch.object(
+        flow.record.adapter.splunk,
+        "RESERVED_SPLUNK_FIELDS",
+        set(),
+    ):
+        test_record_descriptor = RecordDescriptor("test/record", [])
+
+        test_record = test_record_descriptor()
+
+        output_key_value = splunkify_key_value(test_record, tag="bar")
+        output_json = splunkify_json(JSON_PACKER, test_record, tag="bar")
+        assert output_key_value.startswith(f'rdtype="test/record" rdtag="bar" {RESERVED_FIELDS_KEY_VALUE_SUFFIX}')
+        assert json.loads(output_json) == {
+            "event": dict(
+                {
+                    "rdtag": "bar",
+                    "rdtype": "test/record",
+                },
+                **BASE_FIELD_VALUES,
+            )
+        }
+
+
+def test_splunkify_none_field():
+    with patch.object(
+        flow.record.adapter.splunk,
+        "RESERVED_SPLUNK_FIELDS",
+        set(),
+    ):
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("string", "foo")],
+        )
+
+        test_record = test_record_descriptor()
+
+        output_key_value = splunkify_key_value(test_record)
+        output_json = splunkify_json(JSON_PACKER, test_record)
+        assert output_key_value.startswith(
+            f'rdtype="test/record" rdtag=None foo=None {RESERVED_FIELDS_KEY_VALUE_SUFFIX}'
+        )
+        assert json.loads(output_json) == {
+            "event": dict(
+                {
+                    "rdtag": None,
+                    "rdtype": "test/record",
+                    "foo": None,
+                },
+                **BASE_FIELD_VALUES,
+            )
+        }
+
+
+def test_splunkify_byte_field():
+    with patch.object(
+        flow.record.adapter.splunk,
+        "RESERVED_SPLUNK_FIELDS",
+        set(),
+    ):
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("bytes", "foo")],
+        )
+
+        test_record = test_record_descriptor(foo=b"bar")
+
+        output_key_value = splunkify_key_value(test_record)
+        output_json = splunkify_json(JSON_PACKER, test_record)
+        assert output_key_value.startswith(
+            f'rdtype="test/record" rdtag=None foo="YmFy" {RESERVED_FIELDS_KEY_VALUE_SUFFIX}'
+        )
+        assert json.loads(output_json) == {
+            "event": dict(
+                {
+                    "rdtag": None,
+                    "rdtype": "test/record",
+                    "foo": "YmFy",
+                },
+                **BASE_FIELD_VALUES,
+            )
+        }
+
+
+def test_splunkify_backslash_quote_field():
+    with patch.object(
+        flow.record.adapter.splunk,
+        "RESERVED_SPLUNK_FIELDS",
+        set(),
+    ):
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("string", "foo")],
+        )
+
+        test_record = test_record_descriptor(foo=b'\\"')
+
+        output = splunkify_key_value(test_record)
+        output_json = splunkify_json(JSON_PACKER, test_record)
+        assert output.startswith(f'rdtype="test/record" rdtag=None foo="\\\\\\"" {RESERVED_FIELDS_KEY_VALUE_SUFFIX}')
+        assert json.loads(output_json) == {
+            "event": dict(
+                {
+                    "rdtag": None,
+                    "rdtype": "test/record",
+                    "foo": '\\"',
+                },
+                **BASE_FIELD_VALUES,
+            )
+        }
+
+
+def test_splunkify_json_special_fields():
+    with patch.object(
+        flow.record.adapter.splunk,
+        "RESERVED_SPLUNK_FIELDS",
+        set(),
+    ):
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [
+                ("datetime", "ts"),
+                ("string", "hostname"),
+                ("string", "foo"),
+            ],
+        )
+
+        # Datetimes should be converted to epoch
+        test_record = test_record_descriptor(ts=datetime.datetime(1970, 1, 1, 4, 0), hostname="RECYCLOPS", foo="bar")
+
+        output = splunkify_json(JSON_PACKER, test_record)
+        assert '"time": 14400.0,' in output
+        assert '"host": "RECYCLOPS"' in output
+
+
+def test_tcp_protocol():
+    with patch("socket.socket") as mock_socket:
+        tcp_writer = SplunkWriter("splunk:1337")
+        assert tcp_writer.host == "splunk"
+        assert tcp_writer.port == 1337
+        assert tcp_writer.protocol == Protocol.TCP
+
+        mock_socket.assert_called()
+        mock_socket.return_value.connect.assert_called_with(("splunk", 1337))
+
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("string", "foo")],
+        )
+
+        test_record = test_record_descriptor(foo="bar")
+        tcp_writer.write(test_record)
+
+        args, _ = mock_socket.return_value.sendall.call_args
+        written_to_splunk = args[0]
+
+        assert written_to_splunk.startswith(
+            b'rdtype="test/record" rdtag=None foo="bar" ' + RESERVED_FIELDS_KEY_VALUE_SUFFIX.encode()
+        )
+        assert written_to_splunk.endswith(b'"\n')
+
+
+def test_https_protocol_records_sourcetype(mock_httpx_package: MagicMock):
+    if "flow.record.adapter.splunk" in sys.modules:
+        del sys.modules["flow.record.adapter.splunk"]
+
+    from flow.record.adapter.splunk import Protocol, SourceType, SplunkWriter
+
+    with patch.object(
+        flow.record.adapter.splunk,
+        "HAS_HTTPX",
+        True,
+    ):
+        mock_httpx_package.Client.return_value.post.return_value.status_code = 200
+        https_writer = SplunkWriter("https://splunk:8088", token="password123")
+
+        assert https_writer.host == "splunk"
+        assert https_writer.protocol == Protocol.HTTPS
+        assert https_writer.sourcetype == SourceType.RECORDS
+        assert https_writer.verify is True
+        assert https_writer.url == "https://splunk:8088/services/collector/raw?auto_extract_timestamp=true"
+
+        _, kwargs = mock_httpx_package.Client.call_args
+        assert kwargs["verify"] is True
+
+        given_headers = kwargs["headers"]
+        assert given_headers["Authorization"] == "Splunk password123"
+        assert "X-Splunk-Request-Channel" in given_headers
+
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("string", "foo")],
+        )
+
+        test_record = test_record_descriptor(foo="bar")
+        https_writer.write(test_record)
+
+        mock_httpx_package.Client.return_value.post.assert_not_called()
+
+        https_writer.close()
+        mock_httpx_package.Client.return_value.post.assert_called_with(
+            "https://splunk:8088/services/collector/raw?auto_extract_timestamp=true",
+            data=ANY,
+        )
+        _, kwargs = mock_httpx_package.Client.return_value.post.call_args
+        sent_data = kwargs["data"]
+        assert sent_data.startswith(
+            b'rdtype="test/record" rdtag=None foo="bar" ' + RESERVED_FIELDS_KEY_VALUE_SUFFIX.encode()
+        )
+        assert sent_data.endswith(b'"\n')
+
+
+def test_https_protocol_json_sourcetype(mock_httpx_package: MagicMock):
+    if "flow.record.adapter.splunk" in sys.modules:
+        del sys.modules["flow.record.adapter.splunk"]
+
+    from flow.record.adapter.splunk import SplunkWriter
+
+    with patch.object(
+        flow.record.adapter.splunk,
+        "HAS_HTTPX",
+        True,
+    ):
+        mock_httpx_package.Client.return_value.post.return_value.status_code = 200
+
+        https_writer = SplunkWriter("https://splunk:8088", token="password123", sourcetype="json")
+
+        test_record_descriptor = RecordDescriptor(
+            "test/record",
+            [("string", "foo")],
+        )
+
+        https_writer.write(test_record_descriptor(foo="bar"))
+        https_writer.write(test_record_descriptor(foo="baz"))
+        mock_httpx_package.Client.return_value.post.assert_not_called()
+
+        https_writer.close()
+        mock_httpx_package.Client.return_value.post.assert_called_with(
+            "https://splunk:8088/services/collector/event?auto_extract_timestamp=true",
+            data=ANY,
+        )
+
+        _, kwargs = mock_httpx_package.Client.return_value.post.call_args
+        sent_data = kwargs["data"]
+        first_record_json, _, second_record_json = sent_data.partition(b"\n")
+        assert json.loads(first_record_json) == {
+            "event": dict(
+                {
+                    "rdtag": None,
+                    "rdtype": "test/record",
+                    "foo": "bar",
+                },
+                **BASE_FIELD_VALUES,
+            )
+        }
+        assert json.loads(second_record_json) == {
+            "event": dict(
+                {
+                    "rdtag": None,
+                    "rdtype": "test/record",
+                    "foo": "baz",
+                },
+                **BASE_FIELD_VALUES,
+            )
+        }

flow.record-3.15.dev13/flow/record/adapter/splunk.py

@@ -1,90 +0,0 @@
-import logging
-import socket
-
-from flow.record.adapter import AbstractReader, AbstractWriter
-from flow.record.utils import to_base64, to_bytes, to_str
-
-__usage__ = """
-Splunk output adapter (writer only)
----
-Write usage: rdump -w splunk://[IP]:[PORT]?tag=[TAG]
-[IP]:[PORT]: ip and port to a splunk instance
-[TAG]: optional value to add as "rdtag" output field when writing
-"""
-
-log = logging.getLogger(__package__)
-
-RESERVED_SPLUNK_FIELDS = set(
-    [
-        "_indextime",
-        "_time",
-        "index",
-        "punct",
-        "source",
-        "sourcetype",
-        "tag",
-    ]
-)
-
-
-def splunkify(record, tag=None):
-    ret = []
-
-    ret.append(f'type="{record._desc.name}"')
-
-    if tag is None:
-        ret.append("rdtag=None")
-    else:
-        ret.append(f'rdtag="{tag}"')
-
-    for field in record._desc.fields:
-        val = getattr(record, field)
-        if val is None:
-            ret.append(f"{field}=None")
-        else:
-            val = to_base64(val) if isinstance(val, bytes) else to_str(val)
-            val = val.replace("\\", "\\\\").replace('"', '\\"')
-            if field in RESERVED_SPLUNK_FIELDS:
-                field = f"rd_{field}"
-            ret.append(f'{field}="{val}"')
-
-    return " ".join(ret)
-
-
-class SplunkWriter(AbstractWriter):
-    sock = None
-
-    def __init__(self, path, tag=None, **kwargs):
-        p = path.strip("/").split("/")
-        host, port = p[0].split(":")
-        port = int(port)
-
-        self.tag = tag
-        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.SOL_TCP)
-        self.sock.connect((host, port))
-        self.descriptors = {}
-        self._warned = False
-
-    def write(self, record):
-        if not self._warned and "rdtag" in record._desc.fields:
-            self._warned = True
-            log.warning(
-                "Record has 'rdtag' field which conflicts with the Splunk adapter -- "
-                "Splunk output will have duplicate 'rdtag' fields",
-            )
-        rec = splunkify(record, tag=self.tag)
-        data = to_bytes(rec) + b"\n"
-        self.sock.sendall(data)
-
-    def flush(self):
-        pass
-
-    def close(self):
-        if self.sock:
-            self.sock.close()
-        self.sock = None
-
-
-class SplunkReader(AbstractReader):
-    def __init__(self, path, selector=None, **kwargs):
-        raise NotImplementedError()

flow.record-3.15.dev13/tests/test_splunk_adapter.py

@@ -1,104 +0,0 @@
-from unittest import mock
-
-import flow.record.adapter.splunk
-from flow.record import RecordDescriptor
-from flow.record.adapter.splunk import splunkify
-
-
-def test_splunkify_reserved_field():
-    with mock.patch.object(
-        flow.record.adapter.splunk,
-        "RESERVED_SPLUNK_FIELDS",
-        set(["foo"]),
-    ):
-        test_record_descriptor = RecordDescriptor(
-            "test/record",
-            [("string", "foo")],
-        )
-
-        test_record = test_record_descriptor(foo="bar")
-
-        output = splunkify(test_record)
-        assert output == 'type="test/record" rdtag=None rd_foo="bar"'
-
-
-def test_splunkify_normal_field():
-    with mock.patch.object(
-        flow.record.adapter.splunk,
-        "RESERVED_SPLUNK_FIELDS",
-        set(),
-    ):
-        test_record_descriptor = RecordDescriptor(
-            "test/record",
-            [("string", "foo")],
-        )
-
-        test_record = test_record_descriptor(foo="bar")
-
-        output = splunkify(test_record)
-        assert output == 'type="test/record" rdtag=None foo="bar"'
-
-
-def test_splunkify_rdtag_field():
-    with mock.patch.object(
-        flow.record.adapter.splunk,
-        "RESERVED_SPLUNK_FIELDS",
-        set(),
-    ):
-        test_record_descriptor = RecordDescriptor("test/record", [])
-
-        test_record = test_record_descriptor()
-
-        output = splunkify(test_record, tag="bar")
-        assert output == 'type="test/record" rdtag="bar"'
-
-
-def test_splunkify_none_field():
-    with mock.patch.object(
-        flow.record.adapter.splunk,
-        "RESERVED_SPLUNK_FIELDS",
-        set(),
-    ):
-        test_record_descriptor = RecordDescriptor(
-            "test/record",
-            [("string", "foo")],
-        )
-
-        test_record = test_record_descriptor()
-
-        output = splunkify(test_record)
-        assert output == 'type="test/record" rdtag=None foo=None'
-
-
-def test_splunkify_byte_field():
-    with mock.patch.object(
-        flow.record.adapter.splunk,
-        "RESERVED_SPLUNK_FIELDS",
-        set(),
-    ):
-        test_record_descriptor = RecordDescriptor(
-            "test/record",
-            [("bytes", "foo")],
-        )
-
-        test_record = test_record_descriptor(foo=b"bar")
-
-        output = splunkify(test_record)
-        assert output == 'type="test/record" rdtag=None foo="YmFy"'
-
-
-def test_splunkify_backslash_quote_field():
-    with mock.patch.object(
-        flow.record.adapter.splunk,
-        "RESERVED_SPLUNK_FIELDS",
-        set(),
-    ):
-        test_record_descriptor = RecordDescriptor(
-            "test/record",
-            [("string", "foo")],
-        )
-
-        test_record = test_record_descriptor(foo=b'\\"')
-
-        output = splunkify(test_record)
-        assert output == 'type="test/record" rdtag=None foo="\\\\\\""'