flow.record 3.6.dev15__tar.gz → 3.10.dev3__tar.gz

flow.record-3.10.dev3/MANIFEST.in

@@ -0,0 +1,2 @@
+exclude .gitignore
+recursive-exclude .github/ *

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/PKG-INFO

@@ -1,17 +1,20 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.6.dev15
+Version: 3.10.dev3
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
-Home-page: https://github.com/fox-it/flow.record
-Author: Dissect Team
-Author-email: dissect@fox-it.com
+Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
+Project-URL: homepage, https://dissect.tools
+Project-URL: documentation, https://docs.dissect.tools/en/latest/projects/flow.record
+Project-URL: repository, https://github.com/fox-it/flow.record
 Classifier: Programming Language :: Python :: 3
 Requires-Python: ~=3.7
 Description-Content-Type: text/markdown
 Provides-Extra: compression
 Provides-Extra: elastic
 Provides-Extra: geoip
+Provides-Extra: avro
+Provides-Extra: test
 License-File: LICENSE
 License-File: COPYRIGHT
 

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/__init__.py

@@ -15,6 +15,7 @@ from flow.record.base import (
     RecordWriter,
     dynamic_fieldtype,
     extend_record,
+    iter_timestamped_records,
     open_path,
     stream,
 )
@@ -55,6 +56,7 @@ __all__ = [
     "RecordDescriptorError",
     "record_stream",
     "extend_record",
+    "iter_timestamped_records",
 ]
 
 

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/adapter/__init__.py

@@ -4,6 +4,7 @@ import abc
 
 def with_metaclass(meta, *bases):
     """Create a base class with a metaclass. Python 2 and 3 compatible."""
+
     # This requires a bit of explanation: the basic idea is to make a dummy
     # metaclass for one level of class instantiation that replaces itself with
     # the actual metaclass.

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/adapter/avro.py

@@ -71,8 +71,13 @@ class AvroWriter(AbstractWriter):
         self.writer.write(r._packdict())
 
     def flush(self):
-        if self.writer:
-            self.writer.flush()
+        if not self.writer:
+            self.writer = fastavro.write.Writer(
+                self.fp,
+                fastavro.parse_schema({"type": "record", "name": "empty"}),
+                codec=self.codec,
+            )
+        self.writer.flush()
 
     def close(self):
         if self.fp and not is_stdout(self.fp):
@@ -89,7 +94,7 @@ class AvroReader(AbstractReader):
         self.selector = make_selector(selector)
 
         self.reader = fastavro.reader(self.fp)
-        self.schema = self.reader.schema
+        self.schema = self.reader.writer_schema
         if not self.schema:
             raise Exception("Missing Avro schema")
 
@@ -185,7 +190,7 @@ def avro_type_to_flow_type(ftype):
                 return "{}[]".format(item_type)
             else:
                 logical_type = t.get("logicalType")
-                if logical_type and "time" in logical_type or "date" in logical_type:
+                if logical_type and ("time" in logical_type or "date" in logical_type):
                     return "datetime"
 
         if t == "null":

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/adapter/broker.py

@@ -1,6 +1,7 @@
-from flow.record.adapter import AbstractWriter, AbstractReader
 from flow.broker import Publisher, Subscriber
 
+from flow.record.adapter import AbstractReader, AbstractWriter
+
 __usage__ = """
 PubSub adapter using flow.broker
 ---

flow.record-3.10.dev3/flow/record/adapter/csvfile.py

@@ -0,0 +1,90 @@
+from __future__ import absolute_import
+
+import csv
+import sys
+
+from flow.record import RecordDescriptor
+from flow.record.adapter import AbstractReader, AbstractWriter
+from flow.record.selector import make_selector
+from flow.record.utils import is_stdout
+
+__usage__ = """
+Comma-separated values (CSV) adapter
+---
+Write usage: rdump -w csvfile://[PATH]?lineterminator=[TERMINATOR]
+Read usage: rdump csvfile://[PATH]?fields=[FIELDS]
+[PATH]: path to file. Leave empty or "-" to output to stdout
+[TERMINATOR]: line terminator, default is \\r\\n
+[FIELDS]: comma-separated list of CSV fields (in case of missing CSV header)
+"""
+
+
+class CsvfileWriter(AbstractWriter):
+    fp = None
+
+    def __init__(self, path, fields=None, exclude=None, lineterminator=None, **kwargs):
+        if path in (None, "", "-"):
+            self.fp = sys.stdout
+        else:
+            self.fp = open(path, "w", newline="")
+        self.lineterminator = lineterminator or "\r\n"
+        for r, n in ((r"\r", "\r"), (r"\n", "\n"), (r"\t", "\t")):
+            self.lineterminator = self.lineterminator.replace(r, n)
+        self.desc = None
+        self.writer = None
+        self.fields = fields
+        self.exclude = exclude
+        if isinstance(self.fields, str):
+            self.fields = self.fields.split(",")
+        if isinstance(self.exclude, str):
+            self.exclude = self.exclude.split(",")
+
+    def write(self, r):
+        rdict = r._asdict(fields=self.fields, exclude=self.exclude)
+        if not self.desc or self.desc != r._desc:
+            self.desc = r._desc
+            self.writer = csv.DictWriter(self.fp, rdict, lineterminator=self.lineterminator)
+            self.writer.writeheader()
+        self.writer.writerow(rdict)
+
+    def flush(self):
+        if self.fp:
+            self.fp.flush()
+
+    def close(self):
+        if self.fp and not is_stdout(self.fp):
+            self.fp.close()
+        self.fp = None
+
+
+class CsvfileReader(AbstractReader):
+    fp = None
+
+    def __init__(self, path, selector=None, fields=None, **kwargs):
+        self.selector = make_selector(selector)
+        if path in (None, "", "-"):
+            self.fp = sys.stdin
+        else:
+            self.fp = open(path, "r", newline="")
+        self.reader = csv.reader(self.fp)
+
+        if isinstance(fields, str):
+            # parse fields from fields argument (comma-separated string)
+            self.fields = fields.split(",")
+        else:
+            # parse fields from first CSV row
+            self.fields = next(self.reader)
+
+        # Create RecordDescriptor from fields
+        self.desc = RecordDescriptor("csv/reader", [("string", col) for col in self.fields])
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+        self.fp = None
+
+    def __iter__(self):
+        for row in self.reader:
+            record = self.desc(*row)
+            if not self.selector or self.selector.match(record):
+                yield record

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/adapter/jsonfile.py

@@ -1,10 +1,11 @@
 import json
+
 from flow import record
 from flow.record import JsonRecordPacker
-from flow.record.utils import is_stdout
-from flow.record.selector import make_selector
-from flow.record.adapter import AbstractWriter, AbstractReader
+from flow.record.adapter import AbstractReader, AbstractWriter
 from flow.record.fieldtypes import fieldtype_for_value
+from flow.record.selector import make_selector
+from flow.record.utils import is_stdout
 
 __usage__ = """
 JSON adapter

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/adapter/line.py

@@ -1,5 +1,5 @@
-from flow.record.adapter import AbstractWriter
 from flow.record import open_path
+from flow.record.adapter import AbstractWriter
 from flow.record.utils import is_stdout
 
 __usage__ = """
@@ -31,7 +31,7 @@ class LineWriter(AbstractWriter):
         self.fp.write("--[ RECORD {} ]--\n".format(self.count).encode())
         if rdict:
             fmt = "{{:>{width}}} = {{}}\n".format(width=max(len(k) for k in rdict))
-        for (key, value) in rdict.items():
+        for key, value in rdict.items():
             self.fp.write(fmt.format(key, value).encode())
 
     def flush(self):

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/adapter/mongo.py

@@ -1,8 +1,9 @@
 import bson
+from pymongo import MongoClient
+
 from flow import record
 from flow.record.adapter import AbstractReader, AbstractWriter
 from flow.record.selector import make_selector
-from pymongo import MongoClient
 
 __usage__ = """
 MongoDB adapter

flow.record-3.10.dev3/flow/record/adapter/split.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+from urllib.parse import urlparse
+
+from flow.record.adapter import AbstractWriter
+from flow.record.base import RecordWriter
+
+DEFAULT_RECORD_COUNT = 1000
+DEFAULT_SUFFIX_LENGTH = 2
+
+__usage__ = f"""
+Record split adapter, splits records into multiple destination files (writer only)
+---
+Write usage: rdump -w split://[PATH]?count=[COUNT]&suffix-length=[SUFFIX-LENGTH]
+[PATH]: output path or uri
+[COUNT]: maximum record count per file (default: {DEFAULT_RECORD_COUNT})
+[SUFFIX-LENGTH]: length of suffix (default: {DEFAULT_SUFFIX_LENGTH})
+"""
+
+
+class SplitWriter(AbstractWriter):
+    writer = None
+
+    def __init__(self, path, **kwargs):
+        self.path = str(path)
+        self.kwargs = kwargs
+
+        self.written = 0
+        self.count = int(kwargs.get("count", DEFAULT_RECORD_COUNT))
+        self.suffix_length = int(kwargs.get("suffix-length", DEFAULT_SUFFIX_LENGTH))
+        self.file_count = 0
+
+        parsed = urlparse(self.path)
+        self.is_stdout = parsed.netloc in ("", "-") and parsed.path == ""
+
+        self.writer = RecordWriter(self._next_path(), **self.kwargs)
+
+    def _next_path(self):
+        if self.is_stdout:
+            return self.path
+
+        path = self.path
+        scheme = ""
+        sep = ""
+        if "://" in path:
+            scheme, sep, path = path.partition("://")
+
+        suffix = str(self.file_count).rjust(self.suffix_length, "0")
+        path = Path(path)
+        path = path.with_suffix(f".{suffix}{path.suffix}")
+
+        self.file_count += 1
+        return scheme + sep + str(path)
+
+    def write(self, r):
+        self.writer.write(r)
+
+        if self.is_stdout:
+            return
+
+        self.written += 1
+        if self.written >= self.count:
+            self.flush()
+            self.close()
+            self.written = 0
+            self.writer = RecordWriter(self._next_path(), **self.kwargs)
+
+    def flush(self):
+        if self.writer:
+            self.writer.flush()
+
+    def close(self):
+        if self.writer:
+            self.writer.close()
+        self.writer = None

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/adapter/splunk.py

@@ -1,8 +1,8 @@
-import socket
 import logging
+import socket
 
 from flow.record.adapter import AbstractReader, AbstractWriter
-from flow.record.utils import to_str, to_bytes, to_base64
+from flow.record.utils import to_base64, to_bytes, to_str
 
 __usage__ = """
 Splunk output adapter (writer only)

{flow.record-3.6.dev15 → flow.record-3.10.dev3}/flow/record/adapter/xlsx.py

@@ -1,9 +1,9 @@
 import openpyxl
 
 from flow import record
-from flow.record.utils import is_stdout
+from flow.record.adapter import AbstractReader, AbstractWriter
 from flow.record.selector import make_selector
-from flow.record.adapter import AbstractWriter, AbstractReader
+from flow.record.utils import is_stdout
 
 __usage__ = """
 Microsoft Excel spreadsheet adapter