flow.record 3.22.dev2__tar.gz → 3.22.dev4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/PKG-INFO +1 -1
  2. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/base.py +2 -0
  3. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/fieldtypes/__init__.py +81 -72
  4. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/jsonpacker.py +1 -4
  5. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/stream.py +7 -2
  6. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/version.py +3 -3
  7. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow.record.egg-info/PKG-INFO +1 -1
  8. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_json.py +38 -0
  9. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_xlsx.py +2 -2
  10. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/fieldtypes/test_fieldtypes.py +140 -33
  11. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/record/test_adapter.py +8 -0
  12. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/tools/test_rdump.py +73 -0
  13. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/.git-blame-ignore-revs +0 -0
  14. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/.gitattributes +0 -0
  15. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/COPYRIGHT +0 -0
  16. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/LICENSE +0 -0
  17. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/MANIFEST.in +0 -0
  18. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/README.md +0 -0
  19. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/examples/__init__.py +0 -0
  20. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/examples/filesystem.py +0 -0
  21. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/examples/passivedns.py +0 -0
  22. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/examples/records.json +0 -0
  23. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/examples/selectors.py +0 -0
  24. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/examples/tcpconn.py +0 -0
  25. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/__init__.py +0 -0
  26. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/__init__.py +0 -0
  27. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/archive.py +0 -0
  28. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/avro.py +0 -0
  29. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/broker.py +0 -0
  30. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/csvfile.py +0 -0
  31. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/duckdb.py +0 -0
  32. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/elastic.py +0 -0
  33. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/jsonfile.py +0 -0
  34. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/line.py +0 -0
  35. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/mongo.py +0 -0
  36. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/split.py +0 -0
  37. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/splunk.py +0 -0
  38. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/sqlite.py +0 -0
  39. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/stream.py +0 -0
  40. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/text.py +0 -0
  41. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/adapter/xlsx.py +0 -0
  42. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/context.py +0 -0
  43. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/exceptions.py +0 -0
  44. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/fieldtypes/credential.py +0 -0
  45. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/fieldtypes/net/__init__.py +0 -0
  46. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/fieldtypes/net/ip.py +0 -0
  47. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/fieldtypes/net/ipv4.py +0 -0
  48. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/fieldtypes/net/tcp.py +0 -0
  49. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/fieldtypes/net/udp.py +0 -0
  50. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/packer.py +0 -0
  51. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/selector.py +0 -0
  52. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/tools/__init__.py +0 -0
  53. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/tools/geoip.py +0 -0
  54. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/tools/rdump.py +0 -0
  55. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/utils.py +0 -0
  56. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow/record/whitelist.py +0 -0
  57. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow.record.egg-info/SOURCES.txt +0 -0
  58. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow.record.egg-info/dependency_links.txt +0 -0
  59. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow.record.egg-info/entry_points.txt +0 -0
  60. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow.record.egg-info/requires.txt +0 -0
  61. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/flow.record.egg-info/top_level.txt +0 -0
  62. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/pyproject.toml +0 -0
  63. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/setup.cfg +0 -0
  64. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/__init__.py +0 -0
  65. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/_data/.gitkeep +0 -0
  66. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/_docs/Makefile +0 -0
  67. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/_docs/conf.py +0 -0
  68. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/_docs/index.rst +0 -0
  69. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/_utils.py +0 -0
  70. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/__init__.py +0 -0
  71. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_avro.py +0 -0
  72. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_csv.py +0 -0
  73. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_elastic.py +0 -0
  74. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_line.py +0 -0
  75. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_splunk.py +0 -0
  76. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_sqlite_duckdb.py +0 -0
  77. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/adapter/test_text.py +0 -0
  78. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/conftest.py +0 -0
  79. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/fieldtypes/__init__.py +0 -0
  80. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/fieldtypes/test_ip.py +0 -0
  81. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/packer/__init__.py +0 -0
  82. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/packer/test_json_packer.py +0 -0
  83. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/packer/test_packer.py +0 -0
  84. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/record/__init__.py +0 -0
  85. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/record/test_context.py +0 -0
  86. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/record/test_descriptor.py +0 -0
  87. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/record/test_multi_timestamp.py +0 -0
  88. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/record/test_record.py +0 -0
  89. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/selector/__init__.py +0 -0
  90. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/selector/test_compiled.py +0 -0
  91. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/selector/test_selectors.py +0 -0
  92. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/test_deprecations.py +0 -0
  93. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/test_regressions.py +0 -0
  94. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/test_utils.py +0 -0
  95. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tests/tools/__init__.py +0 -0
  96. {flow_record-3.22.dev2 → flow_record-3.22.dev4}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flow.record
3
- Version: 3.22.dev2
3
+ Version: 3.22.dev4
4
4
  Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
5
5
  Author-email: Dissect Team <dissect@fox-it.com>
6
6
  License-Expression: AGPL-3.0-or-later
@@ -893,6 +893,8 @@ def RecordAdapter(
893
893
  "entering record text, rather than a record stream? This can be fixed by using "
894
894
  "'rdump -w -' to write a record stream to stdout."
895
895
  )
896
+ if not peek_data:
897
+ raise EOFError("Empty input stream")
896
898
  raise RecordAdapterNotFound("Could not find adapter for file-like object")
897
899
 
898
900
  # Now that we found an adapter, we will fall back into the same code path as when a URL is given. As the url
@@ -752,41 +752,39 @@ class windows_path(pathlib.PureWindowsPath, path):
752
752
 
753
753
 
754
754
  class command(FieldType):
755
- executable: path | None = None
756
- args: list[str] | None = None
755
+ """The command fieldtype splits a command string into an ``executable`` and its arguments.
757
756
 
758
- _path_type: type[path] = None
759
- _posix: bool
757
+ Args:
758
+ value: the string that contains the command and arguments
759
+ path_type: When specified it forces the command to use a specific path type
760
760
 
761
- def __new__(cls, value: str):
762
- if cls is not command:
763
- return super().__new__(cls)
761
+ Example:
764
762
 
765
- if not isinstance(value, str):
766
- raise TypeError(f"Expected a value of type 'str' not {type(value)}")
763
+ .. code-block:: text
764
+
765
+ 'c:\\windows\\malware.exe /info' -> windows_path('c:\\windows\\malware.exe) ['/info']
766
+ '/usr/bin/env bash' -> posix_path('/usr/bin/env') ['bash']
767
767
 
768
- # pre checking for windows like paths
769
- # This checks for windows like starts of a path:
770
- # an '%' for an environment variable
771
- # r'\\' for a UNC path
772
- # the strip and check for ":" on the second line is for `<drive_letter>:`
773
- stripped_value = value.lstrip("\"'")
774
- windows = value.startswith((r"\\", "%")) or (len(stripped_value) >= 2 and stripped_value[1] == ":")
768
+ # In this situation, the executable path needs to be quoted.
769
+ 'c:\\user\\John Doe\\malware.exe /all /the /things' -> windows_path('c:\\user\\John')
770
+ ['Doe\\malware.exe /all /the /things']
771
+ """
775
772
 
776
- cls = windows_command if windows else posix_command
777
- return super().__new__(cls)
773
+ __executable: path
774
+ __args: tuple[str, ...]
778
775
 
779
- def __init__(self, value: str | tuple[str, tuple[str]] | None):
780
- if value is None:
781
- return
776
+ __path_type: type[path]
782
777
 
783
- if isinstance(value, str):
784
- self.executable, self.args = self._split(value)
785
- return
778
+ def __init__(self, value: str = "", *, path_type: type[path] | None = None):
779
+ if not isinstance(value, str):
780
+ raise TypeError(f"Expected a value of type 'str' not {type(value)}")
781
+
782
+ raw = value.strip()
786
783
 
787
- executable, self.args = value
788
- self.executable = self._path_type(executable)
789
- self.args = list(self.args)
784
+ # Detect the kind of path from value if not specified
785
+ self.__path_type = path_type or type(path(raw.lstrip("\"'")))
786
+
787
+ self.executable, self.args = self._split(raw)
790
788
 
791
789
  def __repr__(self) -> str:
792
790
  return f"(executable={self.executable!r}, args={self.args})"
@@ -795,66 +793,77 @@ class command(FieldType):
795
793
  if isinstance(other, command):
796
794
  return self.executable == other.executable and self.args == other.args
797
795
  if isinstance(other, str):
798
- return self._join() == other
796
+ return self.raw == other
799
797
  if isinstance(other, (tuple, list)):
800
- return self.executable == other[0] and self.args == list(other[1:])
798
+ return self.executable == other[0] and self.args == (*other[1:],)
801
799
 
802
800
  return False
803
801
 
804
- def _split(self, value: str) -> tuple[str, list[str]]:
805
- executable, *args = shlex.split(value, posix=self._posix)
806
- executable = executable.strip("'\" ")
807
-
808
- return self._path_type(executable), args
802
+ def _split(self, value: str) -> tuple[str, tuple[str, ...]]:
803
+ if not value:
804
+ return "", ()
809
805
 
810
- def _join(self) -> str:
811
- return shlex.join([str(self.executable), *self.args])
806
+ executable, *args = shlex.split(value, posix=self.__path_type is posix_path)
807
+ return executable.strip("'\" "), (*args,)
812
808
 
813
- def _pack(self) -> tuple[tuple[str, list], str]:
814
- command_type = TYPE_WINDOWS if isinstance(self, windows_command) else TYPE_POSIX
815
- if self.executable:
816
- _exec, _ = self.executable._pack()
817
- return ((_exec, self.args), command_type)
818
- return (None, command_type)
819
-
820
- @classmethod
821
- def _unpack(cls, data: tuple[tuple[str, tuple] | None, int]) -> command:
822
- _value, _type = data
823
- if _type == TYPE_WINDOWS:
824
- return windows_command(_value)
825
-
826
- return posix_command(_value)
809
+ def _pack(self) -> tuple[str, int]:
810
+ path_type = TYPE_WINDOWS if self.__path_type is windows_path else TYPE_POSIX
811
+ return self.raw, path_type
827
812
 
828
813
  @classmethod
829
- def from_posix(cls, value: str) -> command:
830
- return posix_command(value)
814
+ def _unpack(cls, data: tuple[str, int]) -> command:
815
+ raw_str, path_type = data
816
+ if path_type == TYPE_POSIX:
817
+ return command(raw_str, path_type=posix_path)
818
+ if path_type == TYPE_WINDOWS:
819
+ return command(raw_str, path_type=windows_path)
820
+ # default, infer type of path from str
821
+ return command(raw_str)
831
822
 
832
- @classmethod
833
- def from_windows(cls, value: str) -> command:
834
- return windows_command(value)
823
+ @property
824
+ def executable(self) -> path:
825
+ return self.__executable
835
826
 
827
+ @property
828
+ def args(self) -> tuple[str, ...]:
829
+ return self.__args
836
830
 
837
- class posix_command(command):
838
- _posix = True
839
- _path_type = posix_path
831
+ @executable.setter
832
+ def executable(self, val: str | path | None) -> None:
833
+ self.__executable = self.__path_type(val)
840
834
 
835
+ @args.setter
836
+ def args(self, val: str | tuple[str, ...] | list[str] | None) -> None:
837
+ if val is None:
838
+ self.__args = ()
839
+ return
841
840
 
842
- class windows_command(command):
843
- _posix = False
844
- _path_type = windows_path
841
+ if isinstance(val, str):
842
+ self.__args = tuple(shlex.split(val, posix=self.__path_type is posix_path))
843
+ elif isinstance(val, list):
844
+ self.__args = tuple(val)
845
+ else:
846
+ self.__args = val
845
847
 
846
- def _split(self, value: str) -> tuple[str, list[str]]:
847
- executable, args = super()._split(value)
848
- if args:
849
- args = [" ".join(args)]
848
+ @property
849
+ def raw(self) -> str:
850
+ exe = str(self.executable)
850
851
 
851
- return executable, args
852
+ if " " in exe:
853
+ exe = shlex.quote(exe)
852
854
 
853
- def _join(self) -> str:
854
- arg = f" {self.args[0]}" if self.args else ""
855
- executable_str = str(self.executable)
855
+ result = [exe]
856
+ # Only quote on posix paths as shlex doesn't remove the quotes on non posix paths
857
+ if self.__path_type is posix_path:
858
+ result.extend(shlex.quote(part) if " " in part else part for part in self.args)
859
+ else:
860
+ result.extend(self.args)
861
+ return " ".join(result)
856
862
 
857
- if " " in executable_str:
858
- return f"'{executable_str}'{arg}"
863
+ @classmethod
864
+ def from_posix(cls, value: str) -> command:
865
+ return command(value, path_type=posix_path)
859
866
 
860
- return f"{executable_str}{arg}"
867
+ @classmethod
868
+ def from_windows(cls, value: str) -> command:
869
+ return command(value, path_type=windows_path)
@@ -75,10 +75,7 @@ class JsonRecordPacker:
75
75
  if isinstance(obj, fieldtypes.path):
76
76
  return str(obj)
77
77
  if isinstance(obj, fieldtypes.command):
78
- return {
79
- "executable": obj.executable,
80
- "args": obj.args,
81
- }
78
+ return obj.raw
82
79
 
83
80
  raise TypeError(f"Unpackable type {type(obj)}")
84
81
 
@@ -164,11 +164,13 @@ def record_stream(sources: list[str], selector: str | None = None) -> Iterator[R
164
164
  print("[reading from stdin]", file=sys.stderr)
165
165
 
166
166
  # Initial value for reader, in case of exception message
167
- reader = "RecordReader"
167
+ reader: str | AbstractReader = "RecordReader"
168
168
  try:
169
169
  reader = RecordReader(src, selector=selector)
170
170
  yield from reader
171
- reader.close()
171
+ except EOFError as e:
172
+ # End of file reached, likely no records in source
173
+ log.warning("%s(%r): %s", reader, src, e)
172
174
  except IOError as e:
173
175
  if len(sources) == 1:
174
176
  raise
@@ -184,6 +186,9 @@ def record_stream(sources: list[str], selector: str | None = None) -> Iterator[R
184
186
  else:
185
187
  log.warning("Exception in %r for %r: %s -- skipping to next reader", reader, src, aRepr.repr(e))
186
188
  continue
189
+ finally:
190
+ if isinstance(reader, AbstractReader):
191
+ reader.close()
187
192
 
188
193
 
189
194
  class PathTemplateWriter:
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '3.22.dev2'
32
- __version_tuple__ = version_tuple = (3, 22, 'dev2')
31
+ __version__ = version = '3.22.dev4'
32
+ __version_tuple__ = version_tuple = (3, 22, 'dev4')
33
33
 
34
- __commit_id__ = commit_id = 'gcd69cf171'
34
+ __commit_id__ = commit_id = 'g4bd45720d'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flow.record
3
- Version: 3.22.dev2
3
+ Version: 3.22.dev4
4
4
  Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
5
5
  Author-email: Dissect Team <dissect@fox-it.com>
6
6
  License-Expression: AGPL-3.0-or-later
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING
6
6
  import pytest
7
7
 
8
8
  from flow.record import RecordReader, RecordWriter
9
+ from flow.record.base import RecordDescriptor
9
10
  from tests._utils import generate_records
10
11
 
11
12
  if TYPE_CHECKING:
@@ -117,3 +118,40 @@ def test_json_adapter_with_record_descriptors(tmp_path: Path, record_adapter_pat
117
118
  elif record["_type"] == "record":
118
119
  assert "_recorddescriptor" in record
119
120
  assert descriptor_seen == 2
121
+
122
+
123
+ def test_json_command_fieldtype(tmp_path: Path) -> None:
124
+ json_file = tmp_path.joinpath("records.json")
125
+ record_adapter_path = f"jsonfile://{json_file}"
126
+ writer = RecordWriter(record_adapter_path)
127
+
128
+ TestRecord = RecordDescriptor(
129
+ "test/command",
130
+ [
131
+ ("command", "commando"),
132
+ ],
133
+ )
134
+
135
+ writer.write(
136
+ TestRecord(
137
+ commando="C:\\help.exe data",
138
+ )
139
+ )
140
+ writer.write(
141
+ TestRecord(
142
+ commando="/usr/bin/env bash",
143
+ )
144
+ )
145
+ writer.write(TestRecord())
146
+ writer.flush()
147
+
148
+ reader = RecordReader(record_adapter_path)
149
+ records = list(reader)
150
+
151
+ assert records[0].commando.executable == "C:\\help.exe"
152
+ assert records[0].commando.args == ("data",)
153
+
154
+ assert records[1].commando.executable == "/usr/bin/env"
155
+ assert records[1].commando.args == ("bash",)
156
+
157
+ assert len(records) == 3
@@ -39,9 +39,9 @@ def test_sanitize_field_values(mock_openpyxl_package: MagicMock) -> None:
39
39
  fieldtypes.net.ipaddress("13.37.13.37"),
40
40
  ["Shaken", "Not", "Stirred"],
41
41
  fieldtypes.posix_path("/home/user"),
42
- fieldtypes.posix_command("/bin/bash -c 'echo hello world'"),
42
+ fieldtypes.command.from_posix("/bin/bash -c 'echo hello world'"),
43
43
  fieldtypes.windows_path("C:\\Users\\user\\Desktop"),
44
- fieldtypes.windows_command("C:\\Some.exe /?"),
44
+ fieldtypes.command.from_windows("C:\\Some.exe /?"),
45
45
  ]
46
46
  )
47
47
  ) == [
@@ -22,10 +22,8 @@ from flow.record.fieldtypes import (
22
22
  command,
23
23
  fieldtype_for_value,
24
24
  net,
25
- posix_command,
26
25
  posix_path,
27
26
  uri,
28
- windows_command,
29
27
  windows_path,
30
28
  )
31
29
  from flow.record.fieldtypes import datetime as dt
@@ -1033,6 +1031,20 @@ def test_datetime_comparisions() -> None:
1033
1031
  assert dt("2023-01-02") != datetime(2023, 3, 4, tzinfo=UTC)
1034
1032
 
1035
1033
 
1034
+ def test_empty_command() -> None:
1035
+ command = fieldtypes.command()
1036
+ assert command.executable == ""
1037
+ assert command.args == ()
1038
+
1039
+ command = fieldtypes.command("")
1040
+ assert command.executable == ""
1041
+ assert command.args == ()
1042
+
1043
+ command = fieldtypes.command(" ")
1044
+ assert command.executable == ""
1045
+ assert command.args == ()
1046
+
1047
+
1036
1048
  def test_command_record() -> None:
1037
1049
  TestRecord = RecordDescriptor(
1038
1050
  "test/command",
@@ -1041,15 +1053,19 @@ def test_command_record() -> None:
1041
1053
  ],
1042
1054
  )
1043
1055
 
1056
+ # path defaults to type depending on the os it runs on, so we emulate this here
1057
+ _type = windows_path if os.name == "nt" else posix_path
1058
+
1044
1059
  record = TestRecord(commando="help.exe -h")
1045
- assert isinstance(record.commando, posix_command)
1060
+ assert isinstance(record.commando.executable, _type)
1046
1061
  assert record.commando.executable == "help.exe"
1047
- assert record.commando.args == ["-h"]
1062
+ assert record.commando.args == ("-h",)
1048
1063
 
1049
1064
  record = TestRecord(commando="something.so -h -q -something")
1050
- assert isinstance(record.commando, posix_command)
1065
+ args = ("-h", "-q", "-something")
1066
+ assert isinstance(record.commando.executable, _type)
1051
1067
  assert record.commando.executable == "something.so"
1052
- assert record.commando.args == ["-h", "-q", "-something"]
1068
+ assert record.commando.args == args
1053
1069
 
1054
1070
 
1055
1071
  def test_command_integration(tmp_path: pathlib.Path) -> None:
@@ -1061,15 +1077,16 @@ def test_command_integration(tmp_path: pathlib.Path) -> None:
1061
1077
  )
1062
1078
 
1063
1079
  with RecordWriter(tmp_path / "command_record") as writer:
1064
- record = TestRecord(commando=r"\\.\\?\some_command.exe -h,help /d quiet")
1080
+ record = TestRecord(commando=r"\.\?\some_command.exe -h,help /d quiet")
1065
1081
  writer.write(record)
1066
- assert record.commando.executable == r"\\.\\?\some_command.exe"
1067
- assert record.commando.args == [r"-h,help /d quiet"]
1082
+ assert record.commando.executable == r"\.\?\some_command.exe"
1083
+ assert record.commando.args == (r"-h,help", "/d", "quiet")
1068
1084
 
1069
1085
  with RecordReader(tmp_path / "command_record") as reader:
1070
1086
  for record in reader:
1071
- assert record.commando.executable == r"\\.\\?\some_command.exe"
1072
- assert record.commando.args == [r"-h,help /d quiet"]
1087
+ assert record.commando.executable == r"\.\?\some_command.exe"
1088
+ assert record.commando.args == (r"-h,help", "/d", "quiet")
1089
+ assert record.commando.raw == r"\?\some_command.exe -h,help /d quiet"
1073
1090
 
1074
1091
 
1075
1092
  def test_command_integration_none(tmp_path: pathlib.Path) -> None:
@@ -1080,44 +1097,78 @@ def test_command_integration_none(tmp_path: pathlib.Path) -> None:
1080
1097
  ],
1081
1098
  )
1082
1099
 
1100
+ # None
1101
+ with RecordWriter(tmp_path / "command_record") as writer:
1102
+ record = TestRecord(commando=None)
1103
+ writer.write(record)
1104
+ with RecordReader(tmp_path / "command_record") as reader:
1105
+ for record in reader:
1106
+ assert record.commando is None
1107
+
1108
+ # empty string
1109
+ with RecordWriter(tmp_path / "command_record") as writer:
1110
+ record = TestRecord(commando="")
1111
+ writer.write(record)
1112
+ with RecordReader(tmp_path / "command_record") as reader:
1113
+ for record in reader:
1114
+ assert record.commando == ""
1115
+ assert record.commando.executable == ""
1116
+ assert record.commando.args == ()
1117
+ assert record.commando.raw == ""
1118
+
1119
+
1120
+ def test_integration_correct_path(tmp_path: pathlib.Path) -> None:
1121
+ TestRecord = RecordDescriptor(
1122
+ "test/command",
1123
+ [
1124
+ ("command", "commando"),
1125
+ ],
1126
+ )
1127
+
1083
1128
  with RecordWriter(tmp_path / "command_record") as writer:
1084
- record = TestRecord(commando=command.from_posix(None))
1129
+ record = TestRecord(commando=command.from_windows("hello.exe -d"))
1085
1130
  writer.write(record)
1086
1131
  with RecordReader(tmp_path / "command_record") as reader:
1087
1132
  for record in reader:
1088
- assert record.commando.executable is None
1089
- assert record.commando.args is None
1133
+ assert record.commando.executable == "hello.exe"
1134
+ assert isinstance(record.commando.executable, windows_path)
1135
+ assert record.commando.args == ("-d",)
1090
1136
 
1091
1137
 
1092
1138
  @pytest.mark.parametrize(
1093
1139
  ("command_string", "expected_executable", "expected_argument"),
1094
1140
  [
1095
1141
  # Test relative windows paths
1096
- ("windows.exe something,or,somethingelse", "windows.exe", ["something,or,somethingelse"]),
1142
+ ("windows.exe something,or,somethingelse", "windows.exe", ("something,or,somethingelse",)),
1097
1143
  # Test weird command strings for windows
1098
- ("windows.dll something,or,somethingelse", "windows.dll", ["something,or,somethingelse"]),
1144
+ ("windows.dll something,or,somethingelse", "windows.dll", ("something,or,somethingelse",)),
1099
1145
  # Test environment variables
1100
- (r"%WINDIR%\\windows.dll something,or,somethingelse", r"%WINDIR%\\windows.dll", ["something,or,somethingelse"]),
1101
- # Test a quoted path
1102
- (r"'c:\path to some exe' /d /a", r"c:\path to some exe", [r"/d /a"]),
1146
+ (
1147
+ r"%WINDIR%\\windows.dll something,or,somethingelse",
1148
+ r"%WINDIR%\\windows.dll",
1149
+ ("something,or,somethingelse",),
1150
+ ),
1151
+ # Test a single quoted path
1152
+ (r"'c:\path to some exe' /d /a", r"c:\path to some exe", ("/d", "/a")),
1153
+ # Test a double quoted path
1154
+ (r'"c:\path to some exe" /d /a', r"c:\path to some exe", ("/d", "/a")),
1103
1155
  # Test a unquoted path
1104
- (r"\Users\test\hello.exe", r"\Users\test\hello.exe", []),
1156
+ (r"\Users\test\hello.exe", r"\Users\test\hello.exe", ()),
1105
1157
  # Test an unquoted path with a path as argument
1106
- (r"\Users\test\hello.exe c:\startmepls.exe", r"\Users\test\hello.exe", [r"c:\startmepls.exe"]),
1158
+ (r"\Users\test\hello.exe c:\startmepls.exe", r"\Users\test\hello.exe", (r"c:\startmepls.exe",)),
1107
1159
  # Test a quoted UNC path
1108
- (r"'\\192.168.1.2\Program Files\hello.exe'", r"\\192.168.1.2\Program Files\hello.exe", []),
1160
+ (r"'\\192.168.1.2\Program Files\hello.exe'", r"\\192.168.1.2\Program Files\hello.exe", ()),
1109
1161
  # Test an unquoted UNC path
1110
- (r"\\192.168.1.2\Users\test\hello.exe /d /a", r"\\192.168.1.2\Users\test\hello.exe", [r"/d /a"]),
1162
+ (r"\\192.168.1.2\Users\test\hello.exe /d /a", r"\\192.168.1.2\Users\test\hello.exe", ("/d", "/a")),
1111
1163
  # Test an empty command string
1112
- (r"''", r"", []),
1113
- # Test None
1114
- (None, None, None),
1164
+ (r"''", r"", ()),
1115
1165
  ],
1116
1166
  )
1117
- def test_command_windows(command_string: str, expected_executable: str, expected_argument: list[str]) -> None:
1118
- cmd = windows_command(command_string)
1167
+ def test_command_windows(command_string: str, expected_executable: str, expected_argument: tuple[str, ...]) -> None:
1168
+ cmd = command.from_windows(command_string)
1119
1169
 
1120
1170
  assert cmd.executable == expected_executable
1171
+ assert isinstance(cmd.executable, windows_path)
1121
1172
  assert cmd.args == expected_argument
1122
1173
 
1123
1174
 
@@ -1125,15 +1176,21 @@ def test_command_windows(command_string: str, expected_executable: str, expected
1125
1176
  ("command_string", "expected_executable", "expected_argument"),
1126
1177
  [
1127
1178
  # Test relative posix command
1128
- ("some_file.so -h asdsad -f asdsadas", "some_file.so", ["-h", "asdsad", "-f", "asdsadas"]),
1179
+ ("some_file.so -h asdsad -f asdsadas", "some_file.so", ("-h", "asdsad", "-f", "asdsadas")),
1129
1180
  # Test command with spaces
1130
- (r"/bin/hello\ world -h -word", r"/bin/hello world", ["-h", "-word"]),
1181
+ (r"/bin/hello\ world -h -word", r"/bin/hello world", ("-h", "-word")),
1182
+ (r" /bin/hello\ world", r"/bin/hello world", ()),
1183
+ # Test single quoted command
1184
+ (r"'/tmp/ /test/hello' -h -word", r"/tmp/ /test/hello", ("-h", "-word")),
1185
+ # Test double quoted command
1186
+ (r'"/tmp/ /test/hello" -h -word', r"/tmp/ /test/hello", ("-h", "-word")),
1131
1187
  ],
1132
1188
  )
1133
1189
  def test_command_posix(command_string: str, expected_executable: str, expected_argument: list[str]) -> None:
1134
- cmd = posix_command(command_string)
1190
+ cmd = command.from_posix(command_string)
1135
1191
 
1136
1192
  assert cmd.executable == expected_executable
1193
+ assert isinstance(cmd.executable, fieldtypes.posix_path)
1137
1194
  assert cmd.args == expected_argument
1138
1195
 
1139
1196
 
@@ -1150,8 +1207,13 @@ def test_command_equal() -> None:
1150
1207
  # Compare paths that contain spaces
1151
1208
  assert command("'/home/some folder/file' -h") == "'/home/some folder/file' -h"
1152
1209
  assert command("'c:\\Program files\\some.dll' -h -q") == "'c:\\Program files\\some.dll' -h -q"
1153
- assert command("'c:\\program files\\some.dll' -h -q") == ["c:\\program files\\some.dll", "-h -q"]
1154
- assert command("'c:\\Program files\\some.dll' -h -q") == ("c:\\Program files\\some.dll", "-h -q")
1210
+ assert command("'c:\\program files\\some.dll' -h -q") == ["c:\\program files\\some.dll", "-h", "-q"]
1211
+ assert command("'c:\\Program files\\some.dll' -h -q") == ("c:\\Program files\\some.dll", "-h", "-q")
1212
+
1213
+ assert (
1214
+ command(r"'c:\Program Files\some.dll' --command 'hello world'")
1215
+ == r"'c:\Program Files\some.dll' --command 'hello world'"
1216
+ )
1155
1217
 
1156
1218
  # Test failure conditions
1157
1219
  assert command("hello.so -h") != 1
@@ -1160,6 +1222,51 @@ def test_command_equal() -> None:
1160
1222
  assert command("hello.so") != ("hello.so", "")
1161
1223
 
1162
1224
 
1225
+ def test_command_assign_posix() -> None:
1226
+ _command = command("/")
1227
+
1228
+ assert _command.raw == "/"
1229
+
1230
+ # Test whether we can assign executable
1231
+ _command.executable = "/path/to/home dir/"
1232
+ assert _command.raw == "'/path/to/home dir'"
1233
+
1234
+ # Test whether it uses the underlying path
1235
+ _command.executable = fieldtypes.windows_path("path\\to\\dir")
1236
+ assert _command.raw == r"path/to/dir"
1237
+
1238
+ # As it is windows, this should change to be one value
1239
+ _command.args = ["command", "arguments", "for", "posix"]
1240
+ assert _command.args == ("command", "arguments", "for", "posix")
1241
+ assert _command.raw == r"path/to/dir command arguments for posix"
1242
+
1243
+ _command.args = ("command", "-c", "command string")
1244
+ assert _command.raw == "path/to/dir command -c 'command string'"
1245
+
1246
+ _command.args = "command -c 'command string2'"
1247
+ assert _command.args == ("command", "-c", "command string2")
1248
+ assert _command.raw == "path/to/dir command -c 'command string2'"
1249
+
1250
+
1251
+ def test_command_assign_windows() -> None:
1252
+ _command = command("c:\\")
1253
+
1254
+ assert _command.raw == "c:\\"
1255
+
1256
+ _command.executable = r"c:\windows\path"
1257
+ assert _command.raw == r"c:\windows\path"
1258
+
1259
+ _command.executable = r"c:\windows\path to file"
1260
+ assert _command.raw == r"'c:\windows\path to file'"
1261
+
1262
+ _command.args = ("command", "arguments", "for", "windows")
1263
+ assert _command.args == ("command", "arguments", "for", "windows")
1264
+ assert _command.raw == r"'c:\windows\path to file' command arguments for windows"
1265
+
1266
+ _command.args = "command arguments for windows2"
1267
+ assert _command.args == ("command", "arguments", "for", "windows2")
1268
+
1269
+
1163
1270
  def test_command_failed() -> None:
1164
1271
  with pytest.raises(TypeError, match="Expected a value of type 'str'"):
1165
1272
  command(b"failed")
@@ -499,3 +499,11 @@ def test_file_like_writer_reader() -> None:
499
499
  assert len(read_records) == 10
500
500
  for idx, record in enumerate(read_records):
501
501
  assert record == test_records[idx]
502
+
503
+
504
+ def test_empty_stdin(monkeypatch: pytest.MonkeyPatch) -> None:
505
+ # Mock stdin to be empty
506
+ monkeypatch.setattr(sys, "stdin", BytesIO(b""))
507
+
508
+ with pytest.raises(EOFError, match="Empty input stream"):
509
+ RecordAdapter()
@@ -797,3 +797,76 @@ def test_rdump_catch_sigpipe(tmp_path: Path) -> None:
797
797
  assert "test/record count=0" in stdout
798
798
  assert "test/record count=1" in stdout
799
799
  assert len(stdout.splitlines()) == 2
800
+
801
+
802
+ def test_rdump_empty_records_pipe(tmp_path: Path) -> None:
803
+ """Test that rdump handles empty records as input gracefully."""
804
+
805
+ # create an empty records file
806
+ path = tmp_path / "empty.records"
807
+ with RecordWriter(path):
808
+ pass
809
+
810
+ # although the records file is empty, it should exist and have a RECORDSTREAM header
811
+ assert path.exists()
812
+ assert b"RECORDSTREAM" in path.read_bytes()
813
+
814
+ # rdump empty.records | rdump -l
815
+ p1 = subprocess.Popen(["rdump", str(path)], stdout=subprocess.PIPE)
816
+ p2 = subprocess.Popen(
817
+ ["rdump", "-l"],
818
+ stdin=p1.stdout,
819
+ stdout=subprocess.PIPE,
820
+ stderr=subprocess.PIPE,
821
+ )
822
+ stdout, stderr = p2.communicate()
823
+ assert p2.returncode == 0
824
+ assert b"RecordReader('-'): Empty input stream" in stderr
825
+ assert b"Processed 0 records (matched=0, unmatched=0)" in stdout
826
+
827
+
828
+ @pytest.mark.parametrize(
829
+ "stdin_bytes",
830
+ [
831
+ b"",
832
+ None,
833
+ ],
834
+ )
835
+ def test_rdump_empty_stdin_pipe(stdin_bytes: bytes | None) -> None:
836
+ """Test that rdump handles empty stdin as input gracefully."""
837
+
838
+ # rdump -l (with empty stdin)
839
+ pipe = subprocess.Popen(
840
+ ["rdump", "-l"],
841
+ stdin=subprocess.PIPE,
842
+ stdout=subprocess.PIPE,
843
+ stderr=subprocess.PIPE,
844
+ )
845
+ stdout, stderr = pipe.communicate(input=None)
846
+ assert pipe.returncode == 0
847
+ assert b"RecordReader('-'): Empty input stream" in stderr
848
+ assert b"Processed 0 records (matched=0, unmatched=0)" in stdout
849
+
850
+
851
+ @pytest.mark.parametrize(
852
+ "stdin_bytes",
853
+ [
854
+ b"\n",
855
+ b"this is not a valid record stream",
856
+ b"RANDOMDATA",
857
+ ],
858
+ )
859
+ def test_rdump_invalid_stdin_pipe(stdin_bytes: bytes) -> None:
860
+ """Test that rdump handles invalid stdin as an error"""
861
+
862
+ # rdump -l (with invalid stdin)
863
+ pipe = subprocess.Popen(
864
+ ["rdump", "-l"],
865
+ stdin=subprocess.PIPE,
866
+ stdout=subprocess.PIPE,
867
+ stderr=subprocess.PIPE,
868
+ )
869
+ stdout, stderr = pipe.communicate(input=stdin_bytes)
870
+ assert pipe.returncode == 1, "rdump should exit with error code 1 on invalid input"
871
+ assert b"rdump encountered a fatal error: Could not find adapter for file-like object" in stderr
872
+ assert b"Processed 0 records (matched=0, unmatched=0)" in stdout
File without changes
File without changes