flow.record 3.10.dev1__tar.gz → 3.10.dev3__tar.gz

{flow.record-3.10.dev1/flow.record.egg-info → flow.record-3.10.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.10.dev1
+Version: 3.10.dev3
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -13,6 +13,8 @@ Description-Content-Type: text/markdown
 Provides-Extra: compression
 Provides-Extra: elastic
 Provides-Extra: geoip
+Provides-Extra: avro
+Provides-Extra: test
 License-File: LICENSE
 License-File: COPYRIGHT
 

{flow.record-3.10.dev1 → flow.record-3.10.dev3}/flow/record/adapter/avro.py

@@ -71,8 +71,13 @@ class AvroWriter(AbstractWriter):
         self.writer.write(r._packdict())
 
     def flush(self):
-        if self.writer:
-            self.writer.flush()
+        if not self.writer:
+            self.writer = fastavro.write.Writer(
+                self.fp,
+                fastavro.parse_schema({"type": "record", "name": "empty"}),
+                codec=self.codec,
+            )
+        self.writer.flush()
 
     def close(self):
         if self.fp and not is_stdout(self.fp):
@@ -89,7 +94,7 @@ class AvroReader(AbstractReader):
         self.selector = make_selector(selector)
 
         self.reader = fastavro.reader(self.fp)
-        self.schema = self.reader.schema
+        self.schema = self.reader.writer_schema
         if not self.schema:
             raise Exception("Missing Avro schema")
 
@@ -185,7 +190,7 @@ def avro_type_to_flow_type(ftype):
                 return "{}[]".format(item_type)
             else:
                 logical_type = t.get("logicalType")
-                if logical_type and "time" in logical_type or "date" in logical_type:
+                if logical_type and ("time" in logical_type or "date" in logical_type):
                     return "datetime"
 
         if t == "null":

flow.record-3.10.dev3/flow/record/version.py

@@ -0,0 +1,4 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+__version__ = version = '3.10.dev3'
+__version_tuple__ = version_tuple = (3, 10, 'dev3')

{flow.record-3.10.dev1 → flow.record-3.10.dev3/flow.record.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flow.record
-Version: 3.10.dev1
+Version: 3.10.dev3
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -13,6 +13,8 @@ Description-Content-Type: text/markdown
 Provides-Extra: compression
 Provides-Extra: elastic
 Provides-Extra: geoip
+Provides-Extra: avro
+Provides-Extra: test
 License-File: LICENSE
 License-File: COPYRIGHT
 

{flow.record-3.10.dev1 → flow.record-3.10.dev3}/flow.record.egg-info/SOURCES.txt

@@ -49,8 +49,10 @@ flow/record/tools/__init__.py
 flow/record/tools/geoip.py
 flow/record/tools/rdump.py
 tests/__init__.py
+tests/_utils.py
 tests/selector_explain_example.py
 tests/standalone_test.py
+tests/test_avro_adapter.py
 tests/test_compiled_selector.py
 tests/test_deprecations.py
 tests/test_fieldtype_ip.py

{flow.record-3.10.dev1 → flow.record-3.10.dev3}/flow.record.egg-info/requires.txt

@@ -1,5 +1,8 @@
 msgpack>=0.5.2
 
+[avro]
+fastavro[snappy]
+
 [compression]
 lz4
 zstandard
@@ -9,3 +12,8 @@ elasticsearch
 
 [geoip]
 maxminddb
+
+[test]
+lz4
+zstandard
+fastavro

{flow.record-3.10.dev1 → flow.record-3.10.dev3}/pyproject.toml

@@ -36,6 +36,14 @@ elastic = [
 geoip = [
     "maxminddb",
 ]
+avro = [
+    "fastavro[snappy]",
+]
+test = [
+    "lz4",
+    "zstandard",
+    "fastavro",
+]
 
 [project.scripts]
 rdump = "flow.record.tools.rdump:main"

flow.record-3.10.dev3/tests/_utils.py

@@ -0,0 +1,36 @@
+import datetime
+
+from flow.record import RecordDescriptor
+
+
+def generate_records(count=100):
+    TestRecordEmbedded = RecordDescriptor(
+        "test/embedded_record",
+        [
+            ("datetime", "dt"),
+        ],
+    )
+    TestRecord = RecordDescriptor(
+        "test/adapter",
+        [
+            ("uint32", "number"),
+            ("record", "record"),
+        ],
+    )
+
+    for i in range(count):
+        embedded = TestRecordEmbedded(datetime.datetime.utcnow())
+        yield TestRecord(number=i, record=embedded)
+
+
+def generate_plain_records(count=100):
+    TestRecord = RecordDescriptor(
+        "test/adapter/plain",
+        [
+            ("uint32", "number"),
+            ("datetime", "dt"),
+        ],
+    )
+
+    for i in range(count):
+        yield TestRecord(number=i, dt=datetime.datetime.utcnow())

flow.record-3.10.dev3/tests/test_avro_adapter.py

@@ -0,0 +1,51 @@
+from flow.record import RecordReader, RecordWriter
+
+from ._utils import generate_plain_records
+
+
+def test_avro_adapter(tmpdir):
+    json_file = tmpdir.join("records.avro")
+    record_adapter_path = "avro://{}".format(json_file)
+    writer = RecordWriter(record_adapter_path)
+    nr_records = 1337
+
+    for record in generate_plain_records(nr_records):
+        writer.write(record)
+    writer.flush()
+
+    nr_received_records = 0
+    reader = RecordReader(record_adapter_path)
+    for _ in reader:
+        nr_received_records += 1
+
+    assert nr_records == nr_received_records
+
+
+def test_avro_adapter_contextmanager(tmpdir):
+    json_file = tmpdir.join("records.avro")
+    record_adapter_path = "avro://{}".format(json_file)
+    with RecordWriter(record_adapter_path) as writer:
+        nr_records = 1337
+        for record in generate_plain_records(nr_records):
+            writer.write(record)
+
+    nr_received_records = 0
+    with RecordReader(record_adapter_path) as reader:
+        for _ in reader:
+            nr_received_records += 1
+
+        assert nr_records == nr_received_records
+
+
+def test_avro_adapter_empty(tmpdir):
+    json_file = tmpdir.join("records.avro")
+    record_adapter_path = "avro://{}".format(json_file)
+    with RecordWriter(record_adapter_path):
+        pass
+
+    nr_received_records = 0
+    with RecordReader(record_adapter_path) as reader:
+        for _ in reader:
+            nr_received_records += 1
+
+        assert nr_received_records == 0

{flow.record-3.10.dev1 → flow.record-3.10.dev3}/tests/test_json_record_adapter.py

@@ -1,29 +1,10 @@
-import datetime
 import json
 
 import pytest
 
-from flow.record import RecordDescriptor, RecordReader, RecordWriter
-
-
-def generate_records(count=100):
-    TestRecordEmbedded = RecordDescriptor(
-        "test/embedded_record",
-        [
-            ("datetime", "dt"),
-        ],
-    )
-    TestRecord = RecordDescriptor(
-        "test/adapter",
-        [
-            ("uint32", "number"),
-            ("record", "record"),
-        ],
-    )
-
-    for i in range(count):
-        embedded = TestRecordEmbedded(datetime.datetime.utcnow())
-        yield TestRecord(number=i, record=embedded)
+from flow.record import RecordReader, RecordWriter
+
+from ._utils import generate_records
 
 
 def test_json_adapter(tmpdir):

{flow.record-3.10.dev1 → flow.record-3.10.dev3}/tests/test_record_adapter.py

@@ -1,4 +1,5 @@
 import datetime
+import platform
 import sys
 
 import pytest
@@ -27,25 +28,7 @@ from flow.record.base import (
 )
 from flow.record.selector import CompiledSelector, Selector
 
-
-def generate_records(count=100):
-    TestRecordEmbedded = RecordDescriptor(
-        "test/embedded_record",
-        [
-            ("datetime", "dt"),
-        ],
-    )
-    TestRecord = RecordDescriptor(
-        "test/adapter",
-        [
-            ("uint32", "number"),
-            ("record", "record"),
-        ],
-    )
-
-    for i in range(count):
-        embedded = TestRecordEmbedded(datetime.datetime.utcnow())
-        yield TestRecord(number=i, record=embedded)
+from ._utils import generate_records
 
 
 def test_stream_writer_reader():
@@ -85,6 +68,11 @@ def test_compressed_writer_reader(tmpdir, compression):
     if compression == "zstd" and not HAS_ZSTD:
         pytest.skip("zstandard module not installed")
 
+    if compression == "lz4" and platform.python_implementation() == "PyPy":
+        pytest.skip("lz4 module not supported on PyPy")
+    if compression == "zstd" and platform.python_implementation() == "PyPy":
+        pytest.skip("zstandard module not supported on PyPy")
+
     p = tmpdir.mkdir("{}-test".format(compression))
     path = str(p.join("test.records.{}".format(compression)))
 

{flow.record-3.10.dev1 → flow.record-3.10.dev3}/tox.ini

@@ -15,6 +15,7 @@ deps =
     pytest
     pytest-cov
     coverage
+extras = test
 commands =
 # Capturing output will fail on pypy, possibly due to this issue: https://github.com/pytest-dev/pytest/issues/5502
     pytest --basetemp="{envtmpdir}" {posargs:--color=yes --capture=no --cov=flow --cov-report=term-missing -v tests}

flow.record-3.10.dev1/flow/record/version.py

@@ -1,4 +0,0 @@
-# file generated by setuptools_scm
-# don't change, don't track in version control
-__version__ = version = '3.10.dev1'
-__version_tuple__ = version_tuple = (3, 10, 'dev1')