flow.record 3.16.dev2__tar.gz → 3.16.dev4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flow_record-3.16.dev2/flow.record.egg-info → flow_record-3.16.dev4}/PKG-INFO +1 -1
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/base.py +3 -5
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/stream.py +2 -1
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/utils.py +38 -2
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/version.py +2 -2
- {flow_record-3.16.dev2 → flow_record-3.16.dev4/flow.record.egg-info}/PKG-INFO +1 -1
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tox.ini +5 -4
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/COPYRIGHT +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/LICENSE +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/MANIFEST.in +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/README.md +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/examples/filesystem.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/examples/passivedns.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/examples/records.json +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/examples/tcpconn.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/__init__.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/__init__.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/archive.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/avro.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/broker.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/csvfile.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/duckdb.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/elastic.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/jsonfile.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/line.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/mongo.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/split.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/splunk.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/sqlite.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/stream.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/text.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/adapter/xlsx.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/exceptions.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/fieldtypes/__init__.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/fieldtypes/credential.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/fieldtypes/net/__init__.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/fieldtypes/net/ip.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/fieldtypes/net/ipv4.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/fieldtypes/net/tcp.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/fieldtypes/net/udp.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/jsonpacker.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/packer.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/selector.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/tools/__init__.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/tools/geoip.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/tools/rdump.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow/record/whitelist.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow.record.egg-info/SOURCES.txt +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow.record.egg-info/dependency_links.txt +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow.record.egg-info/entry_points.txt +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow.record.egg-info/requires.txt +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/flow.record.egg-info/top_level.txt +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/pyproject.toml +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/setup.cfg +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/__init__.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/_utils.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/docs/Makefile +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/docs/conf.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/docs/index.rst +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/selector_explain_example.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/standalone_test.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_avro.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_avro_adapter.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_compiled_selector.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_csv_adapter.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_deprecations.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_elastic_adapter.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_fieldtype_ip.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_fieldtypes.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_json_packer.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_json_record_adapter.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_multi_timestamp.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_packer.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_rdump.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_record.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_record_adapter.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_record_descriptor.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_regression.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_selector.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_splunk_adapter.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/test_sqlite_duckdb_adapter.py +0 -0
- {flow_record-3.16.dev2 → flow_record-3.16.dev4}/tests/utils_inspect.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: flow.record
|
|
3
|
-
Version: 3.16.
|
|
3
|
+
Version: 3.16.dev4
|
|
4
4
|
Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
|
|
5
5
|
Author-email: Dissect Team <dissect@fox-it.com>
|
|
6
6
|
License: Affero General Public License v3
|
|
@@ -31,6 +31,7 @@ from urllib.parse import parse_qsl, urlparse
|
|
|
31
31
|
|
|
32
32
|
from flow.record.adapter import AbstractReader, AbstractWriter
|
|
33
33
|
from flow.record.exceptions import RecordAdapterNotFound, RecordDescriptorError
|
|
34
|
+
from flow.record.utils import get_stdin, get_stdout
|
|
34
35
|
|
|
35
36
|
try:
|
|
36
37
|
import lz4.frame as lz4
|
|
@@ -812,10 +813,7 @@ def open_path(path: str, mode: str, clobber: bool = True) -> IO:
|
|
|
812
813
|
# normal file or stdio for reading or writing
|
|
813
814
|
if not fp:
|
|
814
815
|
if is_stdio:
|
|
815
|
-
if binary
|
|
816
|
-
fp = getattr(sys.stdout, "buffer", sys.stdout) if out else getattr(sys.stdin, "buffer", sys.stdin)
|
|
817
|
-
else:
|
|
818
|
-
fp = sys.stdout if out else sys.stdin
|
|
816
|
+
fp = get_stdout(binary=binary) if out else get_stdin(binary=binary)
|
|
819
817
|
else:
|
|
820
818
|
fp = io.open(path, mode)
|
|
821
819
|
# check if we are reading a compressed stream
|
|
@@ -867,7 +865,7 @@ def RecordAdapter(
|
|
|
867
865
|
if url in ("-", "", None) and fileobj is None:
|
|
868
866
|
# For reading stdin, we cannot rely on an extension to know what sort of stream is incoming. Thus, we will
|
|
869
867
|
# treat it as a 'fileobj', where we can peek into the stream and try to select the appropriate adapter.
|
|
870
|
-
fileobj =
|
|
868
|
+
fileobj = get_stdin(binary=True)
|
|
871
869
|
if fileobj is not None:
|
|
872
870
|
# This record adapter has received a file-like object for record reading
|
|
873
871
|
# We just need to find the right adapter by peeking into the first few bytes.
|
|
@@ -12,6 +12,7 @@ from functools import lru_cache
|
|
|
12
12
|
from flow.record import RECORDSTREAM_MAGIC, RecordWriter
|
|
13
13
|
from flow.record.fieldtypes import fieldtype_for_value
|
|
14
14
|
from flow.record.selector import make_selector
|
|
15
|
+
from flow.record.utils import is_stdout
|
|
15
16
|
|
|
16
17
|
from .base import RecordDescriptor, RecordReader
|
|
17
18
|
from .packer import RecordPacker
|
|
@@ -70,7 +71,7 @@ class RecordStreamWriter:
|
|
|
70
71
|
self.write(descriptor)
|
|
71
72
|
|
|
72
73
|
def close(self):
|
|
73
|
-
if self.fp and self.fp
|
|
74
|
+
if self.fp and not is_stdout(self.fp):
|
|
74
75
|
self.fp.close()
|
|
75
76
|
self.fp = None
|
|
76
77
|
|
|
@@ -1,15 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import base64
|
|
2
4
|
import os
|
|
3
5
|
import sys
|
|
4
6
|
from functools import wraps
|
|
7
|
+
from typing import BinaryIO, TextIO
|
|
5
8
|
|
|
6
9
|
_native = str
|
|
7
10
|
_unicode = type("")
|
|
8
11
|
_bytes = type(b"")
|
|
9
12
|
|
|
10
13
|
|
|
11
|
-
def
|
|
12
|
-
|
|
14
|
+
def get_stdout(binary: bool = False) -> TextIO | BinaryIO:
|
|
15
|
+
"""Return the stdout stream as binary or text stream.
|
|
16
|
+
|
|
17
|
+
This function is the preferred way to get the stdout stream in flow.record.
|
|
18
|
+
|
|
19
|
+
Arguments:
|
|
20
|
+
binary: Whether to return the stream as binary stream.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
The stdout stream.
|
|
24
|
+
"""
|
|
25
|
+
fp = getattr(sys.stdout, "buffer", sys.stdout) if binary else sys.stdout
|
|
26
|
+
fp._is_stdout = True
|
|
27
|
+
return fp
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_stdin(binary: bool = False) -> TextIO | BinaryIO:
|
|
31
|
+
"""Return the stdin stream as binary or text stream.
|
|
32
|
+
|
|
33
|
+
This function is the preferred way to get the stdin stream in flow.record.
|
|
34
|
+
|
|
35
|
+
Arguments:
|
|
36
|
+
binary: Whether to return the stream as binary stream.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
The stdin stream.
|
|
40
|
+
"""
|
|
41
|
+
fp = getattr(sys.stdin, "buffer", sys.stdin) if binary else sys.stdin
|
|
42
|
+
fp._is_stdin = True
|
|
43
|
+
return fp
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def is_stdout(fp: TextIO | BinaryIO) -> bool:
|
|
47
|
+
"""Returns True if ``fp`` is the stdout stream."""
|
|
48
|
+
return fp in (sys.stdout, sys.stdout.buffer) or hasattr(fp, "_is_stdout")
|
|
13
49
|
|
|
14
50
|
|
|
15
51
|
def to_bytes(value):
|
|
@@ -12,5 +12,5 @@ __version__: str
|
|
|
12
12
|
__version_tuple__: VERSION_TUPLE
|
|
13
13
|
version_tuple: VERSION_TUPLE
|
|
14
14
|
|
|
15
|
-
__version__ = version = '3.16.
|
|
16
|
-
__version_tuple__ = version_tuple = (3, 16, '
|
|
15
|
+
__version__ = version = '3.16.dev4'
|
|
16
|
+
__version_tuple__ = version_tuple = (3, 16, 'dev4')
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: flow.record
|
|
3
|
-
Version: 3.16.
|
|
3
|
+
Version: 3.16.dev4
|
|
4
4
|
Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
|
|
5
5
|
Author-email: Dissect Team <dissect@fox-it.com>
|
|
6
6
|
License: Affero General Public License v3
|
|
@@ -5,10 +5,11 @@ envlist = lint, py3, pypy3
|
|
|
5
5
|
# locally installed tox to have a minimum version 3.3.0. This means the names
|
|
6
6
|
# of the configuration options are still according to the tox 3.x syntax.
|
|
7
7
|
minversion = 4.11.4
|
|
8
|
-
# This version of virtualenv will install setuptools version
|
|
9
|
-
#
|
|
10
|
-
# pyproject.toml file (PEP-517/PEP-518/PEP-621)
|
|
11
|
-
|
|
8
|
+
# This version of virtualenv will install setuptools version 68.2.2 and pip
|
|
9
|
+
# 23.3.1. These versions fully support python projects defined only through a
|
|
10
|
+
# pyproject.toml file (PEP-517/PEP-518/PEP-621). This pip version also support
|
|
11
|
+
# the proper version resolving with (sub-)dependencies defining dev extra's.
|
|
12
|
+
requires = virtualenv>=20.24.6
|
|
12
13
|
|
|
13
14
|
[testenv]
|
|
14
15
|
deps =
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|