flow.record 3.14.dev4__tar.gz → 3.15__tar.gz

{flow.record-3.14.dev4/flow.record.egg-info → flow_record-3.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.14.dev4
+Version: 3.15
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -32,10 +32,16 @@ Provides-Extra: geoip
 Requires-Dist: maxminddb; extra == "geoip"
 Provides-Extra: avro
 Requires-Dist: fastavro[snappy]; extra == "avro"
+Provides-Extra: duckdb
+Requires-Dist: duckdb; extra == "duckdb"
+Requires-Dist: pytz; extra == "duckdb"
+Provides-Extra: splunk
+Requires-Dist: httpx; extra == "splunk"
 Provides-Extra: test
-Requires-Dist: lz4; extra == "test"
-Requires-Dist: zstandard; extra == "test"
-Requires-Dist: fastavro; extra == "test"
+Requires-Dist: flow.record[compression]; extra == "test"
+Requires-Dist: flow.record[avro]; extra == "test"
+Requires-Dist: duckdb; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
+Requires-Dist: pytz; (platform_python_implementation != "PyPy" and python_version < "3.12") and extra == "test"
 
 # flow.record
 

{flow.record-3.14.dev4 → flow_record-3.15}/flow/record/__init__.py

@@ -2,6 +2,7 @@ import gzip
 import os
 
 from flow.record.base import (
+    IGNORE_FIELDS_FOR_COMPARISON,
     RECORD_VERSION,
     RECORDSTREAM_MAGIC,
     DynamicDescriptor,
@@ -20,6 +21,7 @@ from flow.record.base import (
     open_path,
     open_path_or_stream,
     open_stream,
+    set_ignored_fields_for_comparison,
     stream,
 )
 from flow.record.jsonpacker import JsonRecordPacker
@@ -35,6 +37,7 @@ from flow.record.stream import (
 )
 
 __all__ = [
+    "IGNORE_FIELDS_FOR_COMPARISON",
     "RECORD_VERSION",
     "RECORDSTREAM_MAGIC",
     "FieldType",
@@ -54,6 +57,7 @@ __all__ = [
     "open_path_or_stream",
     "open_path",
     "open_stream",
+    "set_ignored_fields_for_comparison",
     "stream",
     "dynamic_fieldtype",
     "DynamicDescriptor",

flow_record-3.15/flow/record/adapter/duckdb.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import logging
+
+import duckdb
+
+from flow.record.adapter.sqlite import (
+    Selector,
+    SqliteReader,
+    SqliteWriter,
+    make_selector,
+)
+
+logger = logging.getLogger(__name__)
+
+__usage__ = """
+DuckDB adapter
+---
+Write usage: rdump -w duckdb://[PATH]?batch_size=[BATCH_SIZE]
+Read usage: rdump duckdb://[PATH]?batch_size=[BATCH_SIZE]
+[PATH]: path to DuckDB database file
+
+Optional parameters:
+    [BATCH_SIZE]: number of records to read or write in a single transaction (default: 1000)
+"""
+
+
+class DuckdbReader(SqliteReader):
+    """DuckDB reader, subclasses from SQLite reader."""
+
+    logger = logger
+
+    def __init__(self, path: str, *, batch_size: str | int = 1000, selector: Selector | str | None = None, **kwargs):
+        self.selector = make_selector(selector)
+        self.descriptors_seen = set()
+        self.con = duckdb.connect(path)
+        self.count = 0
+        self.batch_size = int(batch_size)
+
+
+class DuckdbWriter(SqliteWriter):
+    """DuckDB writer, subclasses from SQLite writer."""
+
+    logger = logger
+
+    def __init__(self, path: str, *, batch_size: str | int = 1000, **kwargs):
+        self.descriptors_seen = set()
+        self.con = None
+        self.con = duckdb.connect(path)
+        self.count = 0
+        self.batch_size = int(batch_size)
+        self.con.begin()
+
+    def tx_cycle(self) -> None:
+        self.con.commit()
+        self.con.begin()

flow_record-3.15/flow/record/adapter/line.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from functools import lru_cache
+
+from flow.record import Record, RecordDescriptor, open_path_or_stream
+from flow.record.adapter import AbstractWriter
+from flow.record.utils import is_stdout
+
+__usage__ = """
+Line output format adapter (writer only)
+---
+Write usage: rdump -w line://[PATH]?verbose=[VERBOSE]
+[PATH]: path to file. Leave empty or "-" to output to stdout
+
+Optional arguments:
+    [VERBOSE]: Also show fieldtype in line output (default: False)
+"""
+
+
+@lru_cache(maxsize=1024)
+def field_types_for_record_descriptor(desc: RecordDescriptor) -> dict[str, str]:
+    """Return dictionary of fieldname -> fieldtype for given RecordDescriptor.
+
+    Args:
+        desc: RecordDescriptor to get fieldtypes for
+    Returns:
+        Dictionary of fieldname -> fieldtype
+    """
+    return {fname: fieldset.typename for fname, fieldset in desc.get_all_fields().items()}
+
+
+class LineWriter(AbstractWriter):
+    """Prints all fields and values of the Record on a separate line."""
+
+    fp = None
+
+    def __init__(
+        self,
+        path: str,
+        *,
+        fields: list[str] | str | None = None,
+        exclude: list[str] | str | None = None,
+        verbose: bool = False,
+        **kwargs,
+    ):
+        self.fp = open_path_or_stream(path, "wb")
+        self.count = 0
+        self.fields = fields
+        self.exclude = exclude
+        self.verbose = verbose
+        if isinstance(self.fields, str):
+            self.fields = self.fields.split(",")
+        if isinstance(self.exclude, str):
+            self.exclude = self.exclude.split(",")
+
+    def write(self, rec: Record) -> None:
+        rdict = rec._asdict(fields=self.fields, exclude=self.exclude)
+        rdict_types = field_types_for_record_descriptor(rec._desc) if self.verbose else None
+
+        self.count += 1
+        self.fp.write(f"--[ RECORD {self.count} ]--\n".encode())
+        if rdict:
+            if rdict_types:
+                # also account for extra characters for fieldtype and whitespace + parenthesis
+                width = max(len(k + rdict_types[k]) for k in rdict) + 3
+            else:
+                width = max(len(k) for k in rdict)
+            fmt = "{{:>{width}}} = {{}}\n".format(width=width)
+        for key, value in rdict.items():
+            if rdict_types:
+                key = f"{key} ({rdict_types[key]})"
+            self.fp.write(fmt.format(key, value).encode())
+
+    def flush(self) -> None:
+        if self.fp:
+            self.fp.flush()
+
+    def close(self) -> None:
+        if self.fp and not is_stdout(self.fp):
+            self.fp.close()
+        self.fp = None

flow_record-3.15/flow/record/adapter/splunk.py

@@ -0,0 +1,282 @@
+import json
+import logging
+import socket
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+from urllib.parse import urlparse
+
+try:
+    import httpx
+
+    HAS_HTTPX = True
+except ImportError:
+    HAS_HTTPX = False
+
+from flow.record.adapter import AbstractReader, AbstractWriter
+from flow.record.base import Record
+from flow.record.jsonpacker import JsonRecordPacker
+from flow.record.utils import to_base64, to_bytes, to_str
+
+__usage__ = """
+Splunk output adapter (writer only)
+---
+Write usage: rdump -w splunk+[PROTOCOL]://[IP]:[PORT]?tag=[TAG]&token=[TOKEN]&sourcetype=[SOURCETYPE]
+[PROTOCOL]: Protocol to use for forwarding data. Can be tcp, http or https, defaults to tcp if omitted.
+[IP]:[PORT]: ip and port to a splunk instance
+[TAG]: optional value to add as "rdtag" output field when writing
+[TOKEN]: Authentication token for sending data over HTTP(S)
+[SOURCETYPE]: Set sourcetype of data. Defaults to records, but can also be set to JSON.
+[SSL_VERIFY]: Whether to verify the server certificate when sending data over HTTP(S). Defaults to True.
+"""
+
+log = logging.getLogger(__package__)
+
+# Amount of records to bundle into a single request when sending data over HTTP(S).
+RECORD_BUFFER_LIMIT = 20
+
+# https://docs.splunk.com/Documentation/Splunk/7.3.1/Data/Configureindex-timefieldextraction
+RESERVED_SPLUNK_FIELDS = [
+    "_indextime",
+    "_time",
+    "index",
+    "punct",
+    "source",
+    "sourcetype",
+    "tag",
+    "type",
+]
+
+RESERVED_RECORD_FIELDS = ["_classification", "_generated", "_source"]
+
+PREFIX_WITH_RD = set(RESERVED_SPLUNK_FIELDS + RESERVED_RECORD_FIELDS)
+
+
+class Protocol(Enum):
+    HTTP = "http"
+    HTTPS = "https"
+    TCP = "tcp"
+
+
+class SourceType(Enum):
+    JSON = "json"
+    RECORDS = "records"
+
+
+def splunkify_key_value(record: Record, tag: Optional[str] = None) -> str:
+    ret = []
+
+    ret.append(f'rdtype="{record._desc.name}"')
+
+    if tag is None:
+        ret.append("rdtag=None")
+    else:
+        ret.append(f'rdtag="{tag}"')
+
+    for field in record._desc.get_all_fields():
+        # Omit the _version field as the Splunk adapter has no reader support for deserialising records back.
+        if field == "_version":
+            continue
+
+        val = getattr(record, field)
+
+        if field in PREFIX_WITH_RD:
+            field = f"rd_{field}"
+
+        if val is None:
+            ret.append(f"{field}=None")
+        else:
+            val = to_base64(val) if isinstance(val, bytes) else to_str(val)
+            val = val.replace("\\", "\\\\").replace('"', '\\"')
+            ret.append(f'{field}="{val}"')
+
+    return " ".join(ret)
+
+
+def splunkify_json(packer: JsonRecordPacker, record: Record, tag: Optional[str] = None) -> str:
+    ret = {}
+
+    indexer_fields = [
+        ("host", "host"),
+        ("host", "hostname"),
+        ("time", "ts"),
+    ]
+
+    # When converting a record to json text for splunk, we distinguish between the 'event' (containing the data) and a
+    # few other fields that are splunk-specific for indexing. We add those 'indexer_fields' to the return object first.
+    for splunk_name, field_name in indexer_fields:
+        if hasattr(record, field_name):
+            val = getattr(record, field_name)
+            if val:
+                if isinstance(val, datetime):
+                    # Convert datetime objects to epoch timestamp for reserved fields.
+                    ret[splunk_name] = val.timestamp()
+                    continue
+                ret[splunk_name] = to_str(val)
+
+    record_as_dict = packer.pack_obj(record)
+
+    # Omit the _version field as the Splunk adapter has no reader support for deserialising records back.
+    del record_as_dict["_version"]
+
+    # These fields end up in the 'event', but we have a few reserved field names. If those field names are in the
+    # record, we prefix them with 'rd_' (short for record descriptor)
+    for field in PREFIX_WITH_RD:
+        if field not in record_as_dict:
+            continue
+        new_field = f"rd_{field}"
+
+        record_as_dict[new_field] = record_as_dict[field]
+        del record_as_dict[field]
+
+    # Almost done, just have to add the tag and the type (i.e the record descriptor's name) to the event.
+    record_as_dict["rdtag"] = tag
+
+    # Yes.
+    record_as_dict["rdtype"] = record._desc.name
+
+    ret["event"] = record_as_dict
+    return json.dumps(ret, default=packer.pack_obj)
+
+
+class SplunkWriter(AbstractWriter):
+    sock = None
+    session = None
+
+    def __init__(
+        self,
+        uri: str,
+        tag: Optional[str] = None,
+        token: Optional[str] = None,
+        sourcetype: Optional[str] = None,
+        ssl_verify: bool = True,
+        **kwargs,
+    ):
+        # If the writer is initiated without a protocol, we assume we will be writing over tcp
+        if "://" not in uri:
+            uri = f"tcp://{uri}"
+
+        if sourcetype is None:
+            log.warning("No sourcetype provided, assuming 'records' sourcetype")
+            sourcetype = SourceType.RECORDS
+
+        parsed_url = urlparse(uri)
+        url_scheme = parsed_url.scheme.lower()
+
+        self.sourcetype = SourceType(sourcetype)
+        self.protocol = Protocol(url_scheme)
+
+        if self.protocol == Protocol.TCP and self.sourcetype != SourceType.RECORDS:
+            raise ValueError("For sending data to Splunk over TCP, only the 'records' sourcetype is allowed")
+
+        self.host = parsed_url.hostname
+        self.port = parsed_url.port
+        self.tag = tag
+        self.record_buffer = []
+        self._warned = False
+        self.packer = None
+
+        if self.sourcetype == SourceType.JSON:
+            self.packer = JsonRecordPacker(indent=4, pack_descriptors=False)
+
+        if self.protocol == Protocol.TCP:
+            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.SOL_TCP)
+            self.sock.connect((self.host, self.port))
+            self._send = self._send_tcp
+        elif self.protocol in (Protocol.HTTP, Protocol.HTTPS):
+            if not HAS_HTTPX:
+                raise ImportError("The httpx library is required for sending data over HTTP(S)")
+
+            scheme = self.protocol.value
+            self.token = token
+            if not self.token:
+                raise ValueError("An authorization token is required for the HTTP collector")
+            if not self.token.startswith("Splunk "):
+                self.token = f"Splunk {self.token}"
+
+            # Assume verify=True unless specified otherwise.
+            self.verify = str(ssl_verify).lower() not in ("0", "false")
+            if not self.verify:
+                log.warning("Certificate verification is disabled")
+
+            endpoint = "event" if self.sourcetype != SourceType.RECORDS else "raw"
+            port = f":{self.port}" if self.port else ""
+            self.url = f"{scheme}://{self.host}{port}/services/collector/{endpoint}?auto_extract_timestamp=true"
+
+            self.headers = {
+                "Authorization": self.token,
+                # A randomized value so that Splunk can loadbalance between different incoming datastreams
+                "X-Splunk-Request-Channel": str(uuid.uuid4()),
+            }
+
+            self.session = httpx.Client(verify=self.verify, headers=self.headers)
+
+            self._send = self._send_http
+
+    def _cache_records_for_http(self, data: Optional[bytes] = None, flush: bool = False) -> Optional[bytes]:
+        # It's possible to call this function without any data, purely to flush. Hence this check.
+        if data:
+            self.record_buffer.append(data)
+        if len(self.record_buffer) < RECORD_BUFFER_LIMIT and not flush:
+            # Buffer limit not exceeded yet, so we do not return a buffer yet, unless buffer is explicitly flushed.
+            return
+        buf = b"".join(self.record_buffer)
+        if not buf:
+            return
+
+        # We're going to be returning a buffer for the writer to send, so we can clear the internal record buffer.
+        self.record_buffer.clear()
+        return buf
+
+    def _send(self, data: bytes) -> None:
+        raise RuntimeError("This method should be overridden at runtime")
+
+    def _send_http(self, data: Optional[bytes] = None, flush: bool = False) -> None:
+        buf = self._cache_records_for_http(data, flush)
+        if not buf:
+            return
+        response = self.session.post(self.url, data=buf)
+        if response.status_code != 200:
+            raise ConnectionError(f"{response.text} ({response.status_code})")
+
+    def _send_tcp(self, data: bytes) -> None:
+        self.sock.sendall(data)
+
+    def write(self, record: Record) -> None:
+        if not self._warned and "rdtag" in record._desc.fields:
+            self._warned = True
+            log.warning(
+                "Record has 'rdtag' field which conflicts with the Splunk adapter -- "
+                "Splunk output will have duplicate 'rdtag' fields",
+            )
+
+        if self.sourcetype == SourceType.RECORDS:
+            rec = splunkify_key_value(record, self.tag)
+        else:
+            rec = splunkify_json(self.packer, record, self.tag)
+
+        # Trail with a newline for line breaking.
+        data = to_bytes(rec) + b"\n"
+
+        self._send(data)
+
+    def flush(self) -> None:
+        if self.protocol in [Protocol.HTTP, Protocol.HTTPS]:
+            self._send_http(flush=True)
+
+    def close(self) -> None:
+        # For TCP
+        if self.sock:
+            self.sock.close()
+        self.sock = None
+
+        if self.session:
+            self.flush()
+            self.session.close()
+        self.session = None
+
+
+class SplunkReader(AbstractReader):
+    def __init__(self, path, selector=None, **kwargs):
+        raise NotImplementedError()

{flow.record-3.14.dev4 → flow_record-3.15}/flow/record/adapter/sqlite.py

@@ -18,7 +18,7 @@ SQLite adapter
 ---
 Write usage: rdump -w sqlite://[PATH]?batch_size=[BATCH_SIZE]
 Read usage: rdump sqlite://[PATH]?batch_size=[BATCH_SIZE]
-[PATH]: path to sqlite database file
+[PATH]: path to SQLite database file
 
 Optional parameters:
     [BATCH_SIZE]: number of records to read or write in a single transaction (default: 1000)
@@ -28,12 +28,12 @@ Optional parameters:
 FIELD_MAP = {
     "int": "INTEGER",
     "uint32": "INTEGER",
-    "varint": "INTEGER",
+    "varint": "BIGINT",
     "float": "REAL",
     "boolean": "INTEGER",
     "bytes": "BLOB",
-    "filesize": "INTEGER",
-    "datetime": "TIMESTAMP",
+    "filesize": "BIGINT",
+    "datetime": "TIMESTAMPTZ",
 }
 
 
@@ -41,12 +41,15 @@ FIELD_MAP = {
 SQLITE_FIELD_MAP = {
     "VARCHAR": "string",
     "INTEGER": "varint",
+    "BIGINT": "varint",
     "BLOB": "bytes",
     "REAL": "float",
     "DOUBLE": "float",
     "BOOLEAN": "boolean",
     "DATETIME": "datetime",
     "TIMESTAMP": "datetime",
+    "TIMESTAMPTZ": "datetime",
+    "TIMESTAMP WITH TIME ZONE": "datetime",
 }
 
 
@@ -58,11 +61,11 @@ def create_descriptor_table(con: sqlite3.Connection, descriptor: RecordDescripto
     column_defs = []
     for column_name, fieldset in descriptor.get_all_fields().items():
         column_type = FIELD_MAP.get(fieldset.typename, "TEXT")
-        column_defs.append(f"   `{column_name}` {column_type}")
+        column_defs.append(f'   "{column_name}" {column_type}')
     sql_columns = ",\n".join(column_defs)
 
     # Create the descriptor table
-    sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (\n{sql_columns}\n)"
+    sql = f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n{sql_columns}\n)'
     logger.debug(sql)
     con.execute(sql)
 
@@ -72,7 +75,7 @@ def update_descriptor_columns(con: sqlite3.Connection, descriptor: RecordDescrip
     table_name = descriptor.name
 
     # Get existing columns
-    cursor = con.execute(f"PRAGMA table_info(`{table_name}`)")
+    cursor = con.execute(f'PRAGMA table_info("{table_name}")')
     column_names = set(row[1] for row in cursor.fetchall())
 
     # Add missing columns
@@ -81,23 +84,23 @@ def update_descriptor_columns(con: sqlite3.Connection, descriptor: RecordDescrip
         if column_name in column_names:
             continue
         column_type = FIELD_MAP.get(fieldset.typename, "TEXT")
-        column_defs.append(f"  ALTER TABLE `{table_name}` ADD COLUMN `{column_name}` {column_type}")
+        column_defs.append(f'  ALTER TABLE "{table_name}" ADD COLUMN "{column_name}" {column_type}')
 
     # No missing columns
     if not column_defs:
         return None
 
     # Add the new columns
-    sql = ";\n".join(column_defs)
-    con.executescript(sql)
+    for col_def in column_defs:
+        con.execute(col_def)
 
 
 @lru_cache(maxsize=1000)
 def prepare_insert_sql(table_name: str, field_names: tuple[str]) -> str:
     """Return (cached) prepared SQL statement for inserting a record based on table name and field names."""
-    column_names = ", ".join(f"`{name}`" for name in field_names)
+    column_names = ", ".join(f'"{name}"' for name in field_names)
     value_placeholder = ", ".join(["?"] * len(field_names))
-    return f"INSERT INTO `{table_name}` ({column_names}) VALUES ({value_placeholder})"
+    return f'INSERT INTO "{table_name}" ({column_names}) VALUES ({value_placeholder})'
 
 
 def db_insert_record(con: sqlite3.Connection, record: Record) -> None:
@@ -123,7 +126,11 @@ def db_insert_record(con: sqlite3.Connection, record: Record) -> None:
 
 
 class SqliteReader(AbstractReader):
-    def __init__(self, path: str, batch_size: str | int = 1000, selector: Selector | str | None = None, **kwargs):
+    """SQLite reader."""
+
+    logger = logger
+
+    def __init__(self, path: str, *, batch_size: str | int = 1000, selector: Selector | str | None = None, **kwargs):
         self.selector = make_selector(selector)
         self.descriptors_seen = set()
         self.con = sqlite3.connect(path)
@@ -140,7 +147,7 @@ class SqliteReader(AbstractReader):
 
         # flow.record is quite strict with what is allowed in fieldnames or decriptor name.
         # While SQLite is less strict, we need to sanitize the names to make them compatible.
-        table_name_org = table_name
+        table_name_org = table_name.replace('"', '""')
         table_name = normalize_fieldname(table_name)
 
         schema = self.con.execute(
@@ -161,8 +168,8 @@ class SqliteReader(AbstractReader):
             fnames.append(fname)
 
         descriptor_cls = RecordDescriptor(table_name, fields)
-        table_name_org = table_name_org.replace("`", r"\\\`")
-        cursor = self.con.execute(f"SELECT * FROM `{table_name_org}`")
+        table_name_org = table_name_org.replace('"', '""')
+        cursor = self.con.execute(f'SELECT * FROM "{table_name_org}"')
         while True:
             rows = cursor.fetchmany(self.batch_size)
             if not rows:
@@ -186,19 +193,24 @@ class SqliteReader(AbstractReader):
     def __iter__(self) -> Iterator[Record]:
         """Iterate over all tables in the database and yield records."""
         for table_name in self.table_names():
-            logging.debug("Reading table: %s", table_name)
+            self.logger.debug("Reading table: %s", table_name)
             for record in self.read_table(table_name):
                 if not self.selector or self.selector.match(record):
                     yield record
 
 
 class SqliteWriter(AbstractWriter):
-    def __init__(self, path: str, batch_size: str | int = 1000, **kwargs):
+    """SQLite writer."""
+
+    logger = logger
+
+    def __init__(self, path: str, *, batch_size: str | int = 1000, **kwargs):
         self.descriptors_seen = set()
         self.con = None
-        self.con = sqlite3.connect(path)
+        self.con = sqlite3.connect(path, isolation_level=None)
         self.count = 0
         self.batch_size = int(batch_size)
+        self.tx_cycle()
 
     def write(self, record: Record) -> None:
         """Write a record to the database"""
@@ -207,17 +219,23 @@ class SqliteWriter(AbstractWriter):
             self.descriptors_seen.add(desc)
             create_descriptor_table(self.con, desc)
             update_descriptor_columns(self.con, desc)
+            self.flush()
 
         db_insert_record(self.con, record)
         self.count += 1
 
         # Commit every batch_size records
         if self.count % self.batch_size == 0:
-            self.con.commit()
+            self.flush()
+
+    def tx_cycle(self) -> None:
+        if self.con.in_transaction:
+            self.con.execute("COMMIT")
+        self.con.execute("BEGIN")
 
     def flush(self) -> None:
         if self.con:
-            self.con.commit()
+            self.tx_cycle()
 
     def close(self) -> None:
         if self.con: