flow.record 3.17.dev3__tar.gz → 3.17.dev4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flow_record-3.17.dev3/flow.record.egg-info → flow_record-3.17.dev4}/PKG-INFO +1 -1
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/base.py +46 -35
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/version.py +2 -2
- {flow_record-3.17.dev3 → flow_record-3.17.dev4/flow.record.egg-info}/PKG-INFO +1 -1
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/COPYRIGHT +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/LICENSE +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/MANIFEST.in +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/README.md +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/examples/filesystem.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/examples/passivedns.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/examples/records.json +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/examples/tcpconn.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/__init__.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/__init__.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/archive.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/avro.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/broker.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/csvfile.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/duckdb.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/elastic.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/jsonfile.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/line.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/mongo.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/split.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/splunk.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/sqlite.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/stream.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/text.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/adapter/xlsx.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/exceptions.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/fieldtypes/__init__.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/fieldtypes/credential.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/fieldtypes/net/__init__.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/fieldtypes/net/ip.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/fieldtypes/net/ipv4.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/fieldtypes/net/tcp.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/fieldtypes/net/udp.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/jsonpacker.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/packer.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/selector.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/stream.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/tools/__init__.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/tools/geoip.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/tools/rdump.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/utils.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow/record/whitelist.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow.record.egg-info/SOURCES.txt +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow.record.egg-info/dependency_links.txt +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow.record.egg-info/entry_points.txt +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow.record.egg-info/requires.txt +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/flow.record.egg-info/top_level.txt +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/pyproject.toml +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/setup.cfg +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/__init__.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/_utils.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/docs/Makefile +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/docs/conf.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/docs/index.rst +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/selector_explain_example.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/standalone_test.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_avro.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_avro_adapter.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_compiled_selector.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_csv_adapter.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_deprecations.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_elastic_adapter.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_fieldtype_ip.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_fieldtypes.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_json_packer.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_json_record_adapter.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_multi_timestamp.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_packer.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_rdump.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_record.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_record_adapter.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_record_descriptor.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_regression.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_selector.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_splunk_adapter.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_sqlite_duckdb_adapter.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/test_xlsx_adapter.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tests/utils_inspect.py +0 -0
- {flow_record-3.17.dev3 → flow_record-3.17.dev4}/tox.ini +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: flow.record
|
|
3
|
-
Version: 3.17.
|
|
3
|
+
Version: 3.17.dev4
|
|
4
4
|
Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
|
|
5
5
|
Author-email: Dissect Team <dissect@fox-it.com>
|
|
6
6
|
License: Affero General Public License v3
|
|
@@ -523,12 +523,14 @@ class RecordDescriptor:
|
|
|
523
523
|
"""
|
|
524
524
|
Get required fields mapping. eg:
|
|
525
525
|
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
526
|
+
.. code-block:: text
|
|
527
|
+
|
|
528
|
+
{
|
|
529
|
+
"_source": RecordField("_source", "string"),
|
|
530
|
+
"_classification": RecordField("_classification", "datetime"),
|
|
531
|
+
"_generated": RecordField("_generated", "datetime"),
|
|
532
|
+
"_version": RecordField("_version", "vaeint"),
|
|
533
|
+
}
|
|
532
534
|
|
|
533
535
|
Returns:
|
|
534
536
|
Mapping of required fields
|
|
@@ -540,10 +542,12 @@ class RecordDescriptor:
|
|
|
540
542
|
"""
|
|
541
543
|
Get fields mapping (without required fields). eg:
|
|
542
544
|
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
545
|
+
.. code-block:: text
|
|
546
|
+
|
|
547
|
+
{
|
|
548
|
+
"foo": RecordField("foo", "string"),
|
|
549
|
+
"bar": RecordField("bar", "varint"),
|
|
550
|
+
}
|
|
547
551
|
|
|
548
552
|
Returns:
|
|
549
553
|
Mapping of Record fields
|
|
@@ -556,15 +560,17 @@ class RecordDescriptor:
|
|
|
556
560
|
"""
|
|
557
561
|
Get all fields including required meta fields. eg:
|
|
558
562
|
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
563
|
+
.. code-block:: text
|
|
564
|
+
|
|
565
|
+
{
|
|
566
|
+
"ts": RecordField("ts", "datetime"),
|
|
567
|
+
"foo": RecordField("foo", "string"),
|
|
568
|
+
"bar": RecordField("bar", "varint"),
|
|
569
|
+
"_source": RecordField("_source", "string"),
|
|
570
|
+
"_classification": RecordField("_classification", "datetime"),
|
|
571
|
+
"_generated": RecordField("_generated", "datetime"),
|
|
572
|
+
"_version": RecordField("_version", "varint"),
|
|
573
|
+
}
|
|
568
574
|
|
|
569
575
|
Returns:
|
|
570
576
|
Mapping of all Record fields
|
|
@@ -591,18 +597,18 @@ class RecordDescriptor:
|
|
|
591
597
|
return RecordFieldSet(field for field in self.fields.values() if field.typename == name)
|
|
592
598
|
|
|
593
599
|
def __call__(self, *args, **kwargs) -> Record:
|
|
594
|
-
"""Create a new Record initialized with
|
|
600
|
+
"""Create a new Record initialized with ``args`` and ``kwargs``."""
|
|
595
601
|
return self.recordType(*args, **kwargs)
|
|
596
602
|
|
|
597
603
|
def init_from_dict(self, rdict: dict[str, Any], raise_unknown=False) -> Record:
|
|
598
|
-
"""Create a new Record initialized with key, value pairs from
|
|
604
|
+
"""Create a new Record initialized with key, value pairs from ``rdict``.
|
|
599
605
|
|
|
600
|
-
If
|
|
606
|
+
If ``raise_unknown=True`` then fields on ``rdict`` that are unknown to this
|
|
601
607
|
RecordDescriptor will raise a TypeError exception due to initializing
|
|
602
608
|
with unknown keyword arguments. (default: False)
|
|
603
609
|
|
|
604
610
|
Returns:
|
|
605
|
-
Record with data from
|
|
611
|
+
Record with data from ``rdict``
|
|
606
612
|
"""
|
|
607
613
|
|
|
608
614
|
if not raise_unknown:
|
|
@@ -610,14 +616,14 @@ class RecordDescriptor:
|
|
|
610
616
|
return self.recordType(**rdict)
|
|
611
617
|
|
|
612
618
|
def init_from_record(self, record: Record, raise_unknown=False) -> Record:
|
|
613
|
-
"""Create a new Record initialized with data from another
|
|
619
|
+
"""Create a new Record initialized with data from another ``record``.
|
|
614
620
|
|
|
615
|
-
If
|
|
621
|
+
If ``raise_unknown=True`` then fields on ``record`` that are unknown to this
|
|
616
622
|
RecordDescriptor will raise a TypeError exception due to initializing
|
|
617
623
|
with unknown keyword arguments. (default: False)
|
|
618
624
|
|
|
619
625
|
Returns:
|
|
620
|
-
Record with data from
|
|
626
|
+
Record with data from ``record``
|
|
621
627
|
"""
|
|
622
628
|
return self.init_from_dict(record._asdict(), raise_unknown=raise_unknown)
|
|
623
629
|
|
|
@@ -633,7 +639,9 @@ class RecordDescriptor:
|
|
|
633
639
|
def get_field_tuples(self) -> tuple[tuple[str, str]]:
|
|
634
640
|
"""Returns a tuple containing the (typename, name) tuples, eg:
|
|
635
641
|
|
|
636
|
-
|
|
642
|
+
.. code-block:: text
|
|
643
|
+
|
|
644
|
+
(('boolean', 'foo'), ('string', 'bar'))
|
|
637
645
|
|
|
638
646
|
Returns:
|
|
639
647
|
Tuple of (typename, name) tuples
|
|
@@ -676,7 +684,7 @@ class RecordDescriptor:
|
|
|
676
684
|
def definition(self, reserved: bool = True) -> str:
|
|
677
685
|
"""Return the RecordDescriptor as Python definition string.
|
|
678
686
|
|
|
679
|
-
If
|
|
687
|
+
If ``reserved`` is True it will also return the reserved fields.
|
|
680
688
|
|
|
681
689
|
Returns:
|
|
682
690
|
Descriptor definition string
|
|
@@ -769,7 +777,7 @@ def open_path(path: str, mode: str, clobber: bool = True) -> IO:
|
|
|
769
777
|
Args:
|
|
770
778
|
path: Filename or path to filename to open
|
|
771
779
|
mode: Could be "r", "rb" to open file for reading, "w", "wb" for writing
|
|
772
|
-
clobber: Overwrite file if it already exists if
|
|
780
|
+
clobber: Overwrite file if it already exists if ``clobber=True``, else raises IOError.
|
|
773
781
|
|
|
774
782
|
"""
|
|
775
783
|
binary = "b" in mode
|
|
@@ -1040,6 +1048,8 @@ def normalize_fieldname(field_name: str) -> str:
|
|
|
1040
1048
|
This normalizes the name so it can still be used in flow.record.
|
|
1041
1049
|
Reserved field_names are not normalized.
|
|
1042
1050
|
|
|
1051
|
+
.. code-block:: text
|
|
1052
|
+
|
|
1043
1053
|
>>> normalize_fieldname("my-variable-name-with-dashes")
|
|
1044
1054
|
'my_variable_name_with_dashes'
|
|
1045
1055
|
>>> normalize_fieldname("_my_name_starting_with_underscore")
|
|
@@ -1100,25 +1110,26 @@ TimestampRecord = RecordDescriptor(
|
|
|
1100
1110
|
|
|
1101
1111
|
|
|
1102
1112
|
def iter_timestamped_records(record: Record) -> Iterator[Record]:
|
|
1103
|
-
"""Yields timestamped annotated records for each
|
|
1104
|
-
If
|
|
1113
|
+
"""Yields timestamped annotated records for each ``datetime`` fieldtype in ``record``.
|
|
1114
|
+
If ``record`` does not have any ``datetime`` fields the original record is returned.
|
|
1105
1115
|
|
|
1106
1116
|
Args:
|
|
1107
1117
|
record: Record to add timestamp fields for.
|
|
1108
1118
|
|
|
1109
1119
|
Yields:
|
|
1110
|
-
Record annotated with
|
|
1120
|
+
Record annotated with ``ts`` and ``ts_description`` fields for each ``datetime`` fieldtype.
|
|
1111
1121
|
"""
|
|
1112
|
-
|
|
1122
|
+
|
|
1123
|
+
# get all ``datetime`` fields. (excluding _generated).
|
|
1113
1124
|
dt_fields = record._desc.getfields("datetime")
|
|
1114
1125
|
if not dt_fields:
|
|
1115
1126
|
yield record
|
|
1116
1127
|
return
|
|
1117
1128
|
|
|
1118
|
-
# yield a new record for each
|
|
1129
|
+
# yield a new record for each ``datetime`` field assigned as ``ts``.
|
|
1119
1130
|
record_name = record._desc.name
|
|
1120
1131
|
for field in dt_fields:
|
|
1121
1132
|
ts_record = TimestampRecord(getattr(record, field.name), field.name)
|
|
1122
|
-
# we extend
|
|
1133
|
+
# we extend ``ts_record`` with original ``record`` so TSRecord info goes first.
|
|
1123
1134
|
record = extend_record(ts_record, [record], name=record_name)
|
|
1124
1135
|
yield record
|
|
@@ -12,5 +12,5 @@ __version__: str
|
|
|
12
12
|
__version_tuple__: VERSION_TUPLE
|
|
13
13
|
version_tuple: VERSION_TUPLE
|
|
14
14
|
|
|
15
|
-
__version__ = version = '3.17.
|
|
16
|
-
__version_tuple__ = version_tuple = (3, 17, '
|
|
15
|
+
__version__ = version = '3.17.dev4'
|
|
16
|
+
__version_tuple__ = version_tuple = (3, 17, 'dev4')
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: flow.record
|
|
3
|
-
Version: 3.17.
|
|
3
|
+
Version: 3.17.dev4
|
|
4
4
|
Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
|
|
5
5
|
Author-email: Dissect Team <dissect@fox-it.com>
|
|
6
6
|
License: Affero General Public License v3
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|