flow.record 3.22.dev1__tar.gz → 3.22.dev3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/PKG-INFO +1 -1
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/fieldtypes/__init__.py +90 -72
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/jsonpacker.py +1 -4
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/version.py +3 -3
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow.record.egg-info/PKG-INFO +1 -1
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_json.py +38 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_xlsx.py +2 -2
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/fieldtypes/test_fieldtypes.py +148 -35
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/.git-blame-ignore-revs +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/.gitattributes +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/COPYRIGHT +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/LICENSE +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/MANIFEST.in +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/README.md +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/examples/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/examples/filesystem.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/examples/passivedns.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/examples/records.json +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/examples/selectors.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/examples/tcpconn.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/archive.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/avro.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/broker.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/csvfile.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/duckdb.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/elastic.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/jsonfile.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/line.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/mongo.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/split.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/splunk.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/sqlite.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/stream.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/text.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/adapter/xlsx.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/base.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/context.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/exceptions.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/fieldtypes/credential.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/fieldtypes/net/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/fieldtypes/net/ip.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/fieldtypes/net/ipv4.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/fieldtypes/net/tcp.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/fieldtypes/net/udp.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/packer.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/selector.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/stream.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/tools/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/tools/geoip.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/tools/rdump.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/utils.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow/record/whitelist.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow.record.egg-info/SOURCES.txt +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow.record.egg-info/dependency_links.txt +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow.record.egg-info/entry_points.txt +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow.record.egg-info/requires.txt +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/flow.record.egg-info/top_level.txt +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/pyproject.toml +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/setup.cfg +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/_data/.gitkeep +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/_docs/Makefile +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/_docs/conf.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/_docs/index.rst +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/_utils.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_avro.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_csv.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_elastic.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_line.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_splunk.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_sqlite_duckdb.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/adapter/test_text.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/conftest.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/fieldtypes/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/fieldtypes/test_ip.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/packer/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/packer/test_json_packer.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/packer/test_packer.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/record/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/record/test_adapter.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/record/test_context.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/record/test_descriptor.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/record/test_multi_timestamp.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/record/test_record.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/selector/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/selector/test_compiled.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/selector/test_selectors.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/test_deprecations.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/test_regressions.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/test_utils.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/tools/__init__.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tests/tools/test_rdump.py +0 -0
- {flow_record-3.22.dev1 → flow_record-3.22.dev3}/tox.ini +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flow.record
|
|
3
|
-
Version: 3.22.
|
|
3
|
+
Version: 3.22.dev3
|
|
4
4
|
Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
|
|
5
5
|
Author-email: Dissect Team <dissect@fox-it.com>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -28,6 +28,7 @@ except ImportError:
|
|
|
28
28
|
from flow.record.base import FieldType, Record
|
|
29
29
|
|
|
30
30
|
RE_NORMALIZE_PATH = re.compile(r"[\\/]+")
|
|
31
|
+
RE_WINDOWS_PATH = re.compile(r"^[a-zA-Z]:[\\/]")
|
|
31
32
|
|
|
32
33
|
UTC = timezone.utc
|
|
33
34
|
|
|
@@ -602,6 +603,10 @@ def _is_posixlike_path(path: Any) -> bool:
|
|
|
602
603
|
if isinstance(path, pathlib.PurePath):
|
|
603
604
|
obj = getattr(path, "parser", None) or path._flavour
|
|
604
605
|
return "\\" not in (obj.sep, obj.altsep)
|
|
606
|
+
if isinstance(path, str):
|
|
607
|
+
if RE_WINDOWS_PATH.match(path):
|
|
608
|
+
return False
|
|
609
|
+
return "/" in path and "\\" not in path
|
|
605
610
|
return False
|
|
606
611
|
|
|
607
612
|
|
|
@@ -609,6 +614,10 @@ def _is_windowslike_path(path: Any) -> bool:
|
|
|
609
614
|
if isinstance(path, pathlib.PurePath):
|
|
610
615
|
obj = getattr(path, "parser", None) or path._flavour
|
|
611
616
|
return "\\" in (obj.sep, obj.altsep)
|
|
617
|
+
if isinstance(path, str):
|
|
618
|
+
if RE_WINDOWS_PATH.match(path):
|
|
619
|
+
return True
|
|
620
|
+
return "\\" in path
|
|
612
621
|
return False
|
|
613
622
|
|
|
614
623
|
|
|
@@ -743,41 +752,39 @@ class windows_path(pathlib.PureWindowsPath, path):
|
|
|
743
752
|
|
|
744
753
|
|
|
745
754
|
class command(FieldType):
|
|
746
|
-
|
|
747
|
-
args: list[str] | None = None
|
|
755
|
+
"""The command fieldtype splits a command string into an ``executable`` and its arguments.
|
|
748
756
|
|
|
749
|
-
|
|
750
|
-
|
|
757
|
+
Args:
|
|
758
|
+
value: the string that contains the command and arguments
|
|
759
|
+
path_type: When specified it forces the command to use a specific path type
|
|
751
760
|
|
|
752
|
-
|
|
753
|
-
if cls is not command:
|
|
754
|
-
return super().__new__(cls)
|
|
761
|
+
Example:
|
|
755
762
|
|
|
756
|
-
|
|
757
|
-
|
|
763
|
+
.. code-block:: text
|
|
764
|
+
|
|
765
|
+
'c:\\windows\\malware.exe /info' -> windows_path('c:\\windows\\malware.exe) ['/info']
|
|
766
|
+
'/usr/bin/env bash' -> posix_path('/usr/bin/env') ['bash']
|
|
758
767
|
|
|
759
|
-
#
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
# the strip and check for ":" on the second line is for `<drive_letter>:`
|
|
764
|
-
stripped_value = value.lstrip("\"'")
|
|
765
|
-
windows = value.startswith((r"\\", "%")) or (len(stripped_value) >= 2 and stripped_value[1] == ":")
|
|
768
|
+
# In this situation, the executable path needs to be quoted.
|
|
769
|
+
'c:\\user\\John Doe\\malware.exe /all /the /things' -> windows_path('c:\\user\\John')
|
|
770
|
+
['Doe\\malware.exe /all /the /things']
|
|
771
|
+
"""
|
|
766
772
|
|
|
767
|
-
|
|
768
|
-
|
|
773
|
+
__executable: path
|
|
774
|
+
__args: tuple[str, ...]
|
|
769
775
|
|
|
770
|
-
|
|
771
|
-
if value is None:
|
|
772
|
-
return
|
|
776
|
+
__path_type: type[path]
|
|
773
777
|
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
778
|
+
def __init__(self, value: str = "", *, path_type: type[path] | None = None):
|
|
779
|
+
if not isinstance(value, str):
|
|
780
|
+
raise TypeError(f"Expected a value of type 'str' not {type(value)}")
|
|
781
|
+
|
|
782
|
+
raw = value.strip()
|
|
777
783
|
|
|
778
|
-
|
|
779
|
-
self.
|
|
780
|
-
|
|
784
|
+
# Detect the kind of path from value if not specified
|
|
785
|
+
self.__path_type = path_type or type(path(raw.lstrip("\"'")))
|
|
786
|
+
|
|
787
|
+
self.executable, self.args = self._split(raw)
|
|
781
788
|
|
|
782
789
|
def __repr__(self) -> str:
|
|
783
790
|
return f"(executable={self.executable!r}, args={self.args})"
|
|
@@ -786,66 +793,77 @@ class command(FieldType):
|
|
|
786
793
|
if isinstance(other, command):
|
|
787
794
|
return self.executable == other.executable and self.args == other.args
|
|
788
795
|
if isinstance(other, str):
|
|
789
|
-
return self.
|
|
796
|
+
return self.raw == other
|
|
790
797
|
if isinstance(other, (tuple, list)):
|
|
791
|
-
return self.executable == other[0] and self.args ==
|
|
798
|
+
return self.executable == other[0] and self.args == (*other[1:],)
|
|
792
799
|
|
|
793
800
|
return False
|
|
794
801
|
|
|
795
|
-
def _split(self, value: str) -> tuple[str,
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
return self._path_type(executable), args
|
|
802
|
+
def _split(self, value: str) -> tuple[str, tuple[str, ...]]:
|
|
803
|
+
if not value:
|
|
804
|
+
return "", ()
|
|
800
805
|
|
|
801
|
-
|
|
802
|
-
return
|
|
806
|
+
executable, *args = shlex.split(value, posix=self.__path_type is posix_path)
|
|
807
|
+
return executable.strip("'\" "), (*args,)
|
|
803
808
|
|
|
804
|
-
def _pack(self) -> tuple[
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
_exec, _ = self.executable._pack()
|
|
808
|
-
return ((_exec, self.args), command_type)
|
|
809
|
-
return (None, command_type)
|
|
810
|
-
|
|
811
|
-
@classmethod
|
|
812
|
-
def _unpack(cls, data: tuple[tuple[str, tuple] | None, int]) -> command:
|
|
813
|
-
_value, _type = data
|
|
814
|
-
if _type == TYPE_WINDOWS:
|
|
815
|
-
return windows_command(_value)
|
|
816
|
-
|
|
817
|
-
return posix_command(_value)
|
|
809
|
+
def _pack(self) -> tuple[str, int]:
|
|
810
|
+
path_type = TYPE_WINDOWS if self.__path_type is windows_path else TYPE_POSIX
|
|
811
|
+
return self.raw, path_type
|
|
818
812
|
|
|
819
813
|
@classmethod
|
|
820
|
-
def
|
|
821
|
-
|
|
814
|
+
def _unpack(cls, data: tuple[str, int]) -> command:
|
|
815
|
+
raw_str, path_type = data
|
|
816
|
+
if path_type == TYPE_POSIX:
|
|
817
|
+
return command(raw_str, path_type=posix_path)
|
|
818
|
+
if path_type == TYPE_WINDOWS:
|
|
819
|
+
return command(raw_str, path_type=windows_path)
|
|
820
|
+
# default, infer type of path from str
|
|
821
|
+
return command(raw_str)
|
|
822
822
|
|
|
823
|
-
@
|
|
824
|
-
def
|
|
825
|
-
return
|
|
823
|
+
@property
|
|
824
|
+
def executable(self) -> path:
|
|
825
|
+
return self.__executable
|
|
826
826
|
|
|
827
|
+
@property
|
|
828
|
+
def args(self) -> tuple[str, ...]:
|
|
829
|
+
return self.__args
|
|
827
830
|
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
+
@executable.setter
|
|
832
|
+
def executable(self, val: str | path | None) -> None:
|
|
833
|
+
self.__executable = self.__path_type(val)
|
|
831
834
|
|
|
835
|
+
@args.setter
|
|
836
|
+
def args(self, val: str | tuple[str, ...] | list[str] | None) -> None:
|
|
837
|
+
if val is None:
|
|
838
|
+
self.__args = ()
|
|
839
|
+
return
|
|
832
840
|
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
841
|
+
if isinstance(val, str):
|
|
842
|
+
self.__args = tuple(shlex.split(val, posix=self.__path_type is posix_path))
|
|
843
|
+
elif isinstance(val, list):
|
|
844
|
+
self.__args = tuple(val)
|
|
845
|
+
else:
|
|
846
|
+
self.__args = val
|
|
836
847
|
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
args = [" ".join(args)]
|
|
848
|
+
@property
|
|
849
|
+
def raw(self) -> str:
|
|
850
|
+
exe = str(self.executable)
|
|
841
851
|
|
|
842
|
-
|
|
852
|
+
if " " in exe:
|
|
853
|
+
exe = shlex.quote(exe)
|
|
843
854
|
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
855
|
+
result = [exe]
|
|
856
|
+
# Only quote on posix paths as shlex doesn't remove the quotes on non posix paths
|
|
857
|
+
if self.__path_type is posix_path:
|
|
858
|
+
result.extend(shlex.quote(part) if " " in part else part for part in self.args)
|
|
859
|
+
else:
|
|
860
|
+
result.extend(self.args)
|
|
861
|
+
return " ".join(result)
|
|
847
862
|
|
|
848
|
-
|
|
849
|
-
|
|
863
|
+
@classmethod
|
|
864
|
+
def from_posix(cls, value: str) -> command:
|
|
865
|
+
return command(value, path_type=posix_path)
|
|
850
866
|
|
|
851
|
-
|
|
867
|
+
@classmethod
|
|
868
|
+
def from_windows(cls, value: str) -> command:
|
|
869
|
+
return command(value, path_type=windows_path)
|
|
@@ -75,10 +75,7 @@ class JsonRecordPacker:
|
|
|
75
75
|
if isinstance(obj, fieldtypes.path):
|
|
76
76
|
return str(obj)
|
|
77
77
|
if isinstance(obj, fieldtypes.command):
|
|
78
|
-
return
|
|
79
|
-
"executable": obj.executable,
|
|
80
|
-
"args": obj.args,
|
|
81
|
-
}
|
|
78
|
+
return obj.raw
|
|
82
79
|
|
|
83
80
|
raise TypeError(f"Unpackable type {type(obj)}")
|
|
84
81
|
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '3.22.
|
|
32
|
-
__version_tuple__ = version_tuple = (3, 22, '
|
|
31
|
+
__version__ = version = '3.22.dev3'
|
|
32
|
+
__version_tuple__ = version_tuple = (3, 22, 'dev3')
|
|
33
33
|
|
|
34
|
-
__commit_id__ = commit_id = '
|
|
34
|
+
__commit_id__ = commit_id = 'g6fab237c7'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flow.record
|
|
3
|
-
Version: 3.22.
|
|
3
|
+
Version: 3.22.dev3
|
|
4
4
|
Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
|
|
5
5
|
Author-email: Dissect Team <dissect@fox-it.com>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING
|
|
|
6
6
|
import pytest
|
|
7
7
|
|
|
8
8
|
from flow.record import RecordReader, RecordWriter
|
|
9
|
+
from flow.record.base import RecordDescriptor
|
|
9
10
|
from tests._utils import generate_records
|
|
10
11
|
|
|
11
12
|
if TYPE_CHECKING:
|
|
@@ -117,3 +118,40 @@ def test_json_adapter_with_record_descriptors(tmp_path: Path, record_adapter_pat
|
|
|
117
118
|
elif record["_type"] == "record":
|
|
118
119
|
assert "_recorddescriptor" in record
|
|
119
120
|
assert descriptor_seen == 2
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_json_command_fieldtype(tmp_path: Path) -> None:
|
|
124
|
+
json_file = tmp_path.joinpath("records.json")
|
|
125
|
+
record_adapter_path = f"jsonfile://{json_file}"
|
|
126
|
+
writer = RecordWriter(record_adapter_path)
|
|
127
|
+
|
|
128
|
+
TestRecord = RecordDescriptor(
|
|
129
|
+
"test/command",
|
|
130
|
+
[
|
|
131
|
+
("command", "commando"),
|
|
132
|
+
],
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
writer.write(
|
|
136
|
+
TestRecord(
|
|
137
|
+
commando="C:\\help.exe data",
|
|
138
|
+
)
|
|
139
|
+
)
|
|
140
|
+
writer.write(
|
|
141
|
+
TestRecord(
|
|
142
|
+
commando="/usr/bin/env bash",
|
|
143
|
+
)
|
|
144
|
+
)
|
|
145
|
+
writer.write(TestRecord())
|
|
146
|
+
writer.flush()
|
|
147
|
+
|
|
148
|
+
reader = RecordReader(record_adapter_path)
|
|
149
|
+
records = list(reader)
|
|
150
|
+
|
|
151
|
+
assert records[0].commando.executable == "C:\\help.exe"
|
|
152
|
+
assert records[0].commando.args == ("data",)
|
|
153
|
+
|
|
154
|
+
assert records[1].commando.executable == "/usr/bin/env"
|
|
155
|
+
assert records[1].commando.args == ("bash",)
|
|
156
|
+
|
|
157
|
+
assert len(records) == 3
|
|
@@ -39,9 +39,9 @@ def test_sanitize_field_values(mock_openpyxl_package: MagicMock) -> None:
|
|
|
39
39
|
fieldtypes.net.ipaddress("13.37.13.37"),
|
|
40
40
|
["Shaken", "Not", "Stirred"],
|
|
41
41
|
fieldtypes.posix_path("/home/user"),
|
|
42
|
-
fieldtypes.
|
|
42
|
+
fieldtypes.command.from_posix("/bin/bash -c 'echo hello world'"),
|
|
43
43
|
fieldtypes.windows_path("C:\\Users\\user\\Desktop"),
|
|
44
|
-
fieldtypes.
|
|
44
|
+
fieldtypes.command.from_windows("C:\\Some.exe /?"),
|
|
45
45
|
]
|
|
46
46
|
)
|
|
47
47
|
) == [
|
|
@@ -22,10 +22,8 @@ from flow.record.fieldtypes import (
|
|
|
22
22
|
command,
|
|
23
23
|
fieldtype_for_value,
|
|
24
24
|
net,
|
|
25
|
-
posix_command,
|
|
26
25
|
posix_path,
|
|
27
26
|
uri,
|
|
28
|
-
windows_command,
|
|
29
27
|
windows_path,
|
|
30
28
|
)
|
|
31
29
|
from flow.record.fieldtypes import datetime as dt
|
|
@@ -578,7 +576,10 @@ def custom_pure_path(sep: str, altsep: str) -> pathlib.PurePath:
|
|
|
578
576
|
(custom_pure_path(sep="/", altsep="")("/foo/bar"), True),
|
|
579
577
|
(custom_pure_path(sep="\\", altsep="/")(r"C:\foo\bar"), False),
|
|
580
578
|
(custom_pure_path(sep=":", altsep="\\")(r"C:\foo\bar"), False),
|
|
581
|
-
("/foo/bar",
|
|
579
|
+
("/foo/bar", True),
|
|
580
|
+
(r"C:\foo\bar", False),
|
|
581
|
+
(r"C:/foo/bar", False),
|
|
582
|
+
(r"\??\C:\Windows\System32\calc.exe", False),
|
|
582
583
|
],
|
|
583
584
|
)
|
|
584
585
|
def test__is_posixlike_path(path_: pathlib.PurePath | str, is_posix: bool) -> None:
|
|
@@ -594,6 +595,9 @@ def test__is_posixlike_path(path_: pathlib.PurePath | str, is_posix: bool) -> No
|
|
|
594
595
|
(custom_pure_path(sep="\\", altsep="/")(r"C:\foo\bar"), True),
|
|
595
596
|
(custom_pure_path(sep=":", altsep="\\")(r"C:\foo\bar"), True),
|
|
596
597
|
("/foo/bar", False),
|
|
598
|
+
(r"C:\foo\bar", True),
|
|
599
|
+
(r"C:/foo/bar", True),
|
|
600
|
+
(r"\??\C:\Windows\System32\calc.exe", True),
|
|
597
601
|
],
|
|
598
602
|
)
|
|
599
603
|
def test__is_windowslike_path(path_: pathlib.PurePath, is_windows: bool) -> None:
|
|
@@ -677,7 +681,7 @@ def test_path() -> None:
|
|
|
677
681
|
),
|
|
678
682
|
(
|
|
679
683
|
("/some/path", pathlib.PureWindowsPath("win/path"), pathlib.PurePosixPath("pos/path")),
|
|
680
|
-
flow.record.fieldtypes.
|
|
684
|
+
flow.record.fieldtypes.posix_path,
|
|
681
685
|
),
|
|
682
686
|
(
|
|
683
687
|
(pathlib.PurePosixPath("pos/path"), pathlib.PureWindowsPath("win/path")),
|
|
@@ -1027,6 +1031,20 @@ def test_datetime_comparisions() -> None:
|
|
|
1027
1031
|
assert dt("2023-01-02") != datetime(2023, 3, 4, tzinfo=UTC)
|
|
1028
1032
|
|
|
1029
1033
|
|
|
1034
|
+
def test_empty_command() -> None:
|
|
1035
|
+
command = fieldtypes.command()
|
|
1036
|
+
assert command.executable == ""
|
|
1037
|
+
assert command.args == ()
|
|
1038
|
+
|
|
1039
|
+
command = fieldtypes.command("")
|
|
1040
|
+
assert command.executable == ""
|
|
1041
|
+
assert command.args == ()
|
|
1042
|
+
|
|
1043
|
+
command = fieldtypes.command(" ")
|
|
1044
|
+
assert command.executable == ""
|
|
1045
|
+
assert command.args == ()
|
|
1046
|
+
|
|
1047
|
+
|
|
1030
1048
|
def test_command_record() -> None:
|
|
1031
1049
|
TestRecord = RecordDescriptor(
|
|
1032
1050
|
"test/command",
|
|
@@ -1035,15 +1053,19 @@ def test_command_record() -> None:
|
|
|
1035
1053
|
],
|
|
1036
1054
|
)
|
|
1037
1055
|
|
|
1056
|
+
# path defaults to type depending on the os it runs on, so we emulate this here
|
|
1057
|
+
_type = windows_path if os.name == "nt" else posix_path
|
|
1058
|
+
|
|
1038
1059
|
record = TestRecord(commando="help.exe -h")
|
|
1039
|
-
assert isinstance(record.commando,
|
|
1060
|
+
assert isinstance(record.commando.executable, _type)
|
|
1040
1061
|
assert record.commando.executable == "help.exe"
|
|
1041
|
-
assert record.commando.args ==
|
|
1062
|
+
assert record.commando.args == ("-h",)
|
|
1042
1063
|
|
|
1043
1064
|
record = TestRecord(commando="something.so -h -q -something")
|
|
1044
|
-
|
|
1065
|
+
args = ("-h", "-q", "-something")
|
|
1066
|
+
assert isinstance(record.commando.executable, _type)
|
|
1045
1067
|
assert record.commando.executable == "something.so"
|
|
1046
|
-
assert record.commando.args ==
|
|
1068
|
+
assert record.commando.args == args
|
|
1047
1069
|
|
|
1048
1070
|
|
|
1049
1071
|
def test_command_integration(tmp_path: pathlib.Path) -> None:
|
|
@@ -1055,15 +1077,16 @@ def test_command_integration(tmp_path: pathlib.Path) -> None:
|
|
|
1055
1077
|
)
|
|
1056
1078
|
|
|
1057
1079
|
with RecordWriter(tmp_path / "command_record") as writer:
|
|
1058
|
-
record = TestRecord(commando=r"
|
|
1080
|
+
record = TestRecord(commando=r"\.\?\some_command.exe -h,help /d quiet")
|
|
1059
1081
|
writer.write(record)
|
|
1060
|
-
assert record.commando.executable == r"
|
|
1061
|
-
assert record.commando.args ==
|
|
1082
|
+
assert record.commando.executable == r"\.\?\some_command.exe"
|
|
1083
|
+
assert record.commando.args == (r"-h,help", "/d", "quiet")
|
|
1062
1084
|
|
|
1063
1085
|
with RecordReader(tmp_path / "command_record") as reader:
|
|
1064
1086
|
for record in reader:
|
|
1065
|
-
assert record.commando.executable == r"
|
|
1066
|
-
assert record.commando.args ==
|
|
1087
|
+
assert record.commando.executable == r"\.\?\some_command.exe"
|
|
1088
|
+
assert record.commando.args == (r"-h,help", "/d", "quiet")
|
|
1089
|
+
assert record.commando.raw == r"\?\some_command.exe -h,help /d quiet"
|
|
1067
1090
|
|
|
1068
1091
|
|
|
1069
1092
|
def test_command_integration_none(tmp_path: pathlib.Path) -> None:
|
|
@@ -1074,44 +1097,78 @@ def test_command_integration_none(tmp_path: pathlib.Path) -> None:
|
|
|
1074
1097
|
],
|
|
1075
1098
|
)
|
|
1076
1099
|
|
|
1100
|
+
# None
|
|
1077
1101
|
with RecordWriter(tmp_path / "command_record") as writer:
|
|
1078
|
-
record = TestRecord(commando=
|
|
1102
|
+
record = TestRecord(commando=None)
|
|
1079
1103
|
writer.write(record)
|
|
1080
1104
|
with RecordReader(tmp_path / "command_record") as reader:
|
|
1081
1105
|
for record in reader:
|
|
1082
|
-
assert record.commando
|
|
1083
|
-
|
|
1106
|
+
assert record.commando is None
|
|
1107
|
+
|
|
1108
|
+
# empty string
|
|
1109
|
+
with RecordWriter(tmp_path / "command_record") as writer:
|
|
1110
|
+
record = TestRecord(commando="")
|
|
1111
|
+
writer.write(record)
|
|
1112
|
+
with RecordReader(tmp_path / "command_record") as reader:
|
|
1113
|
+
for record in reader:
|
|
1114
|
+
assert record.commando == ""
|
|
1115
|
+
assert record.commando.executable == ""
|
|
1116
|
+
assert record.commando.args == ()
|
|
1117
|
+
assert record.commando.raw == ""
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
def test_integration_correct_path(tmp_path: pathlib.Path) -> None:
|
|
1121
|
+
TestRecord = RecordDescriptor(
|
|
1122
|
+
"test/command",
|
|
1123
|
+
[
|
|
1124
|
+
("command", "commando"),
|
|
1125
|
+
],
|
|
1126
|
+
)
|
|
1127
|
+
|
|
1128
|
+
with RecordWriter(tmp_path / "command_record") as writer:
|
|
1129
|
+
record = TestRecord(commando=command.from_windows("hello.exe -d"))
|
|
1130
|
+
writer.write(record)
|
|
1131
|
+
with RecordReader(tmp_path / "command_record") as reader:
|
|
1132
|
+
for record in reader:
|
|
1133
|
+
assert record.commando.executable == "hello.exe"
|
|
1134
|
+
assert isinstance(record.commando.executable, windows_path)
|
|
1135
|
+
assert record.commando.args == ("-d",)
|
|
1084
1136
|
|
|
1085
1137
|
|
|
1086
1138
|
@pytest.mark.parametrize(
|
|
1087
1139
|
("command_string", "expected_executable", "expected_argument"),
|
|
1088
1140
|
[
|
|
1089
1141
|
# Test relative windows paths
|
|
1090
|
-
("windows.exe something,or,somethingelse", "windows.exe",
|
|
1142
|
+
("windows.exe something,or,somethingelse", "windows.exe", ("something,or,somethingelse",)),
|
|
1091
1143
|
# Test weird command strings for windows
|
|
1092
|
-
("windows.dll something,or,somethingelse", "windows.dll",
|
|
1144
|
+
("windows.dll something,or,somethingelse", "windows.dll", ("something,or,somethingelse",)),
|
|
1093
1145
|
# Test environment variables
|
|
1094
|
-
(
|
|
1095
|
-
|
|
1096
|
-
|
|
1146
|
+
(
|
|
1147
|
+
r"%WINDIR%\\windows.dll something,or,somethingelse",
|
|
1148
|
+
r"%WINDIR%\\windows.dll",
|
|
1149
|
+
("something,or,somethingelse",),
|
|
1150
|
+
),
|
|
1151
|
+
# Test a single quoted path
|
|
1152
|
+
(r"'c:\path to some exe' /d /a", r"c:\path to some exe", ("/d", "/a")),
|
|
1153
|
+
# Test a double quoted path
|
|
1154
|
+
(r'"c:\path to some exe" /d /a', r"c:\path to some exe", ("/d", "/a")),
|
|
1097
1155
|
# Test a unquoted path
|
|
1098
|
-
(r"\Users\test\hello.exe", r"\Users\test\hello.exe",
|
|
1156
|
+
(r"\Users\test\hello.exe", r"\Users\test\hello.exe", ()),
|
|
1099
1157
|
# Test an unquoted path with a path as argument
|
|
1100
|
-
(r"\Users\test\hello.exe c:\startmepls.exe", r"\Users\test\hello.exe",
|
|
1158
|
+
(r"\Users\test\hello.exe c:\startmepls.exe", r"\Users\test\hello.exe", (r"c:\startmepls.exe",)),
|
|
1101
1159
|
# Test a quoted UNC path
|
|
1102
|
-
(r"'\\192.168.1.2\Program Files\hello.exe'", r"\\192.168.1.2\Program Files\hello.exe",
|
|
1160
|
+
(r"'\\192.168.1.2\Program Files\hello.exe'", r"\\192.168.1.2\Program Files\hello.exe", ()),
|
|
1103
1161
|
# Test an unquoted UNC path
|
|
1104
|
-
(r"\\192.168.1.2\Users\test\hello.exe /d /a", r"\\192.168.1.2\Users\test\hello.exe",
|
|
1162
|
+
(r"\\192.168.1.2\Users\test\hello.exe /d /a", r"\\192.168.1.2\Users\test\hello.exe", ("/d", "/a")),
|
|
1105
1163
|
# Test an empty command string
|
|
1106
|
-
(r"''", r"",
|
|
1107
|
-
# Test None
|
|
1108
|
-
(None, None, None),
|
|
1164
|
+
(r"''", r"", ()),
|
|
1109
1165
|
],
|
|
1110
1166
|
)
|
|
1111
|
-
def test_command_windows(command_string: str, expected_executable: str, expected_argument:
|
|
1112
|
-
cmd =
|
|
1167
|
+
def test_command_windows(command_string: str, expected_executable: str, expected_argument: tuple[str, ...]) -> None:
|
|
1168
|
+
cmd = command.from_windows(command_string)
|
|
1113
1169
|
|
|
1114
1170
|
assert cmd.executable == expected_executable
|
|
1171
|
+
assert isinstance(cmd.executable, windows_path)
|
|
1115
1172
|
assert cmd.args == expected_argument
|
|
1116
1173
|
|
|
1117
1174
|
|
|
@@ -1119,15 +1176,21 @@ def test_command_windows(command_string: str, expected_executable: str, expected
|
|
|
1119
1176
|
("command_string", "expected_executable", "expected_argument"),
|
|
1120
1177
|
[
|
|
1121
1178
|
# Test relative posix command
|
|
1122
|
-
("some_file.so -h asdsad -f asdsadas", "some_file.so",
|
|
1179
|
+
("some_file.so -h asdsad -f asdsadas", "some_file.so", ("-h", "asdsad", "-f", "asdsadas")),
|
|
1123
1180
|
# Test command with spaces
|
|
1124
|
-
(r"/bin/hello\ world -h -word", r"/bin/hello world",
|
|
1181
|
+
(r"/bin/hello\ world -h -word", r"/bin/hello world", ("-h", "-word")),
|
|
1182
|
+
(r" /bin/hello\ world", r"/bin/hello world", ()),
|
|
1183
|
+
# Test single quoted command
|
|
1184
|
+
(r"'/tmp/ /test/hello' -h -word", r"/tmp/ /test/hello", ("-h", "-word")),
|
|
1185
|
+
# Test double quoted command
|
|
1186
|
+
(r'"/tmp/ /test/hello" -h -word', r"/tmp/ /test/hello", ("-h", "-word")),
|
|
1125
1187
|
],
|
|
1126
1188
|
)
|
|
1127
1189
|
def test_command_posix(command_string: str, expected_executable: str, expected_argument: list[str]) -> None:
|
|
1128
|
-
cmd =
|
|
1190
|
+
cmd = command.from_posix(command_string)
|
|
1129
1191
|
|
|
1130
1192
|
assert cmd.executable == expected_executable
|
|
1193
|
+
assert isinstance(cmd.executable, fieldtypes.posix_path)
|
|
1131
1194
|
assert cmd.args == expected_argument
|
|
1132
1195
|
|
|
1133
1196
|
|
|
@@ -1144,8 +1207,13 @@ def test_command_equal() -> None:
|
|
|
1144
1207
|
# Compare paths that contain spaces
|
|
1145
1208
|
assert command("'/home/some folder/file' -h") == "'/home/some folder/file' -h"
|
|
1146
1209
|
assert command("'c:\\Program files\\some.dll' -h -q") == "'c:\\Program files\\some.dll' -h -q"
|
|
1147
|
-
assert command("'c:\\program files\\some.dll' -h -q") == ["c:\\program files\\some.dll", "-h -q"]
|
|
1148
|
-
assert command("'c:\\Program files\\some.dll' -h -q") == ("c:\\Program files\\some.dll", "-h -q")
|
|
1210
|
+
assert command("'c:\\program files\\some.dll' -h -q") == ["c:\\program files\\some.dll", "-h", "-q"]
|
|
1211
|
+
assert command("'c:\\Program files\\some.dll' -h -q") == ("c:\\Program files\\some.dll", "-h", "-q")
|
|
1212
|
+
|
|
1213
|
+
assert (
|
|
1214
|
+
command(r"'c:\Program Files\some.dll' --command 'hello world'")
|
|
1215
|
+
== r"'c:\Program Files\some.dll' --command 'hello world'"
|
|
1216
|
+
)
|
|
1149
1217
|
|
|
1150
1218
|
# Test failure conditions
|
|
1151
1219
|
assert command("hello.so -h") != 1
|
|
@@ -1154,6 +1222,51 @@ def test_command_equal() -> None:
|
|
|
1154
1222
|
assert command("hello.so") != ("hello.so", "")
|
|
1155
1223
|
|
|
1156
1224
|
|
|
1225
|
+
def test_command_assign_posix() -> None:
|
|
1226
|
+
_command = command("/")
|
|
1227
|
+
|
|
1228
|
+
assert _command.raw == "/"
|
|
1229
|
+
|
|
1230
|
+
# Test whether we can assign executable
|
|
1231
|
+
_command.executable = "/path/to/home dir/"
|
|
1232
|
+
assert _command.raw == "'/path/to/home dir'"
|
|
1233
|
+
|
|
1234
|
+
# Test whether it uses the underlying path
|
|
1235
|
+
_command.executable = fieldtypes.windows_path("path\\to\\dir")
|
|
1236
|
+
assert _command.raw == r"path/to/dir"
|
|
1237
|
+
|
|
1238
|
+
# As it is windows, this should change to be one value
|
|
1239
|
+
_command.args = ["command", "arguments", "for", "posix"]
|
|
1240
|
+
assert _command.args == ("command", "arguments", "for", "posix")
|
|
1241
|
+
assert _command.raw == r"path/to/dir command arguments for posix"
|
|
1242
|
+
|
|
1243
|
+
_command.args = ("command", "-c", "command string")
|
|
1244
|
+
assert _command.raw == "path/to/dir command -c 'command string'"
|
|
1245
|
+
|
|
1246
|
+
_command.args = "command -c 'command string2'"
|
|
1247
|
+
assert _command.args == ("command", "-c", "command string2")
|
|
1248
|
+
assert _command.raw == "path/to/dir command -c 'command string2'"
|
|
1249
|
+
|
|
1250
|
+
|
|
1251
|
+
def test_command_assign_windows() -> None:
|
|
1252
|
+
_command = command("c:\\")
|
|
1253
|
+
|
|
1254
|
+
assert _command.raw == "c:\\"
|
|
1255
|
+
|
|
1256
|
+
_command.executable = r"c:\windows\path"
|
|
1257
|
+
assert _command.raw == r"c:\windows\path"
|
|
1258
|
+
|
|
1259
|
+
_command.executable = r"c:\windows\path to file"
|
|
1260
|
+
assert _command.raw == r"'c:\windows\path to file'"
|
|
1261
|
+
|
|
1262
|
+
_command.args = ("command", "arguments", "for", "windows")
|
|
1263
|
+
assert _command.args == ("command", "arguments", "for", "windows")
|
|
1264
|
+
assert _command.raw == r"'c:\windows\path to file' command arguments for windows"
|
|
1265
|
+
|
|
1266
|
+
_command.args = "command arguments for windows2"
|
|
1267
|
+
assert _command.args == ("command", "arguments", "for", "windows2")
|
|
1268
|
+
|
|
1269
|
+
|
|
1157
1270
|
def test_command_failed() -> None:
|
|
1158
1271
|
with pytest.raises(TypeError, match="Expected a value of type 'str'"):
|
|
1159
1272
|
command(b"failed")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|