traffic-taffy 0.8.1__py3-none-any.whl → 0.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. traffic_taffy/__init__.py +1 -1
  2. traffic_taffy/algorithms/__init__.py +14 -7
  3. traffic_taffy/algorithms/comparecorrelation.py +164 -0
  4. traffic_taffy/algorithms/comparecorrelationchanges.py +210 -0
  5. traffic_taffy/algorithms/compareseries.py +117 -0
  6. traffic_taffy/algorithms/compareslices.py +116 -0
  7. traffic_taffy/algorithms/statistical.py +9 -9
  8. traffic_taffy/compare.py +149 -159
  9. traffic_taffy/comparison.py +18 -4
  10. traffic_taffy/config.py +133 -0
  11. traffic_taffy/dissection.py +171 -6
  12. traffic_taffy/dissectmany.py +26 -16
  13. traffic_taffy/dissector.py +189 -77
  14. traffic_taffy/dissector_engine/scapy.py +41 -8
  15. traffic_taffy/graph.py +54 -53
  16. traffic_taffy/graphdata.py +13 -2
  17. traffic_taffy/hooks/ip2asn.py +20 -7
  18. traffic_taffy/hooks/labels.py +45 -0
  19. traffic_taffy/hooks/psl.py +21 -3
  20. traffic_taffy/iana/tables.msgpak +0 -0
  21. traffic_taffy/output/__init__.py +8 -48
  22. traffic_taffy/output/console.py +37 -25
  23. traffic_taffy/output/fsdb.py +24 -18
  24. traffic_taffy/reports/__init__.py +5 -0
  25. traffic_taffy/reports/compareslicesreport.py +85 -0
  26. traffic_taffy/reports/correlationchangereport.py +54 -0
  27. traffic_taffy/reports/correlationreport.py +42 -0
  28. traffic_taffy/taffy_config.py +44 -0
  29. traffic_taffy/tests/test_compare_results.py +22 -7
  30. traffic_taffy/tests/test_config.py +149 -0
  31. traffic_taffy/tests/test_global_config.py +33 -0
  32. traffic_taffy/tests/test_normalize.py +1 -0
  33. traffic_taffy/tests/test_pcap_dissector.py +12 -2
  34. traffic_taffy/tests/test_pcap_splitter.py +21 -10
  35. traffic_taffy/tools/cache_info.py +3 -2
  36. traffic_taffy/tools/compare.py +32 -24
  37. traffic_taffy/tools/config.py +83 -0
  38. traffic_taffy/tools/dissect.py +51 -59
  39. traffic_taffy/tools/explore.py +5 -4
  40. traffic_taffy/tools/export.py +28 -17
  41. traffic_taffy/tools/graph.py +25 -27
  42. {traffic_taffy-0.8.1.dist-info → traffic_taffy-0.9.dist-info}/METADATA +4 -1
  43. traffic_taffy-0.9.dist-info/RECORD +56 -0
  44. {traffic_taffy-0.8.1.dist-info → traffic_taffy-0.9.dist-info}/entry_points.txt +1 -0
  45. traffic_taffy/report.py +0 -12
  46. traffic_taffy/tests/test_dpkt_engine.py +0 -15
  47. traffic_taffy-0.8.1.dist-info/RECORD +0 -43
  48. {traffic_taffy-0.8.1.dist-info → traffic_taffy-0.9.dist-info}/WHEEL +0 -0
  49. {traffic_taffy-0.8.1.dist-info → traffic_taffy-0.9.dist-info}/licenses/LICENSE.txt +0 -0
@@ -4,54 +4,149 @@ from __future__ import annotations
4
4
 
5
5
  import sys
6
6
  from collections import Counter, defaultdict
7
- from logging import error, warning
7
+ from logging import error, warning, debug, info
8
8
  from typing import List
9
9
  import importlib
10
+ from typing import TYPE_CHECKING, Any
10
11
 
11
12
  from rich import print
12
13
 
13
14
  from traffic_taffy.dissection import Dissection, PCAPDissectorLevel
14
15
  from traffic_taffy.hooks import call_hooks
16
+ from traffic_taffy.taffy_config import TaffyConfig, taffy_default
17
+
18
+ if TYPE_CHECKING:
19
+ from argparse import Parser
20
+
21
+
22
+ class TTD_CFG:
23
+ KEY_DISSECTOR: str = "dissect"
24
+
25
+ BIN_SIZE: str = "bin_size"
26
+ CACHE_FILE_SUFFIX: str = "cache_file_suffix"
27
+ CACHE_PCAP_RESULTS: str = "cache_pcap_results"
28
+ DISSECTION_LEVEL: str = "dissection_level"
29
+ FILTER: str = "filter"
30
+ FILTER_ARGUMENTS: str = "filter_arguments"
31
+ FORCE_LOAD: str = "force_load"
32
+ FORCE_OVERWRITE: str = "force_overwrite"
33
+ IGNORE_LIST: str = "ignore_list"
34
+ LAYERS: str = "layers"
35
+ MERGE: str = "merge"
36
+ MODULES: str = "use_modules"
37
+ PACKET_COUNT: str = "packet_count"
38
+
39
+
40
+ class TTL_CFG:
41
+ KEY_LIMITOR: str = "limit_output"
42
+
43
+ MATCH_EXPRESSION: str = "match_expression"
44
+ MATCH_STRING: str = "match_string"
45
+ MATCH_VALUE: str = "match_value"
46
+ MINIMUM_COUNT: str = "minimum_count"
47
+
15
48
 
16
49
  POST_DISSECT_HOOK: str = "post_dissect"
17
50
 
18
51
 
52
+ def dissector_default(name: str, value: Any) -> None:
53
+ taffy_default(TTD_CFG.KEY_DISSECTOR + "." + name, value)
54
+
55
+
56
+ dissector_default("dissection_level", PCAPDissectorLevel.THROUGH_IP.value)
57
+ dissector_default("packet_count", 0)
58
+ dissector_default("bin_size", None)
59
+ dissector_default("filter", None)
60
+ dissector_default("layers", [])
61
+ dissector_default("use_modules", None)
62
+ dissector_default("merge", False)
63
+ dissector_default("cache_pcap_results", False)
64
+ dissector_default("force_overwrite", False)
65
+ dissector_default("force_load", False)
66
+ dissector_default("cache_file_suffix", "taffy")
67
+ dissector_default("maximum_cores", 20) # TODO(hardaker): fix double forking
68
+
69
+
70
+ def limitor_default(name: str, value: Any) -> None:
71
+ taffy_default(TTL_CFG.KEY_LIMITOR + "." + name, value)
72
+
73
+
74
+ limitor_default("match_string", None)
75
+ limitor_default("match_value", None)
76
+ limitor_default("match_expression", None)
77
+ limitor_default("minimum_count", None)
78
+
79
+ dissector_default(
80
+ "ignore_list",
81
+ [
82
+ "Ethernet_IP_TCP_seq",
83
+ "Ethernet_IP_TCP_ack",
84
+ "Ethernet_IPv6_TCP_seq",
85
+ "Ethernet_IPv6_TCP_ack",
86
+ "Ethernet_IPv6_TCP_Raw_load",
87
+ "Ethernet_IP_UDP_Raw_load",
88
+ "Ethernet_IP_UDP_DNS_id",
89
+ "Ethernet_IP_ICMP_IP in ICMP_UDP in ICMP_chksum",
90
+ "Ethernet_IP_ICMP_IP in ICMP_UDP in ICMP_Raw_load",
91
+ "Ethernet_IP_ICMP_IP in ICMP_chksum",
92
+ "Ethernet_IP_ICMP_IP in ICMP_id",
93
+ "Ethernet_IP_TCP_DNS_id",
94
+ "Ethernet_IPv6_UDP_DNS_id",
95
+ "Ethernet_IPv6_TCP_DNS_id",
96
+ "Ethernet_IP_id",
97
+ "Ethernet_IP_chksum",
98
+ "Ethernet_IP_UDP_chksum",
99
+ "Ethernet_IP_TCP_chksum",
100
+ "Ethernet_IP_TCP_window",
101
+ "Ethernet_IP_TCP_Raw_load",
102
+ "Ethernet_IP_UDP_Raw_load",
103
+ "Ethernet_IPv6_UDP_chksum",
104
+ "Ethernet_IPv6_fl",
105
+ "Ethernet_IP_ICMP_chksum",
106
+ "Ethernet_IP_ICMP_id",
107
+ "Ethernet_IP_ICMP_seq",
108
+ "Ethernet_IP_TCP_Padding_load",
109
+ "Ethernet_IP_TCP_window",
110
+ "Ethernet_IPv6_TCP_chksum",
111
+ "Ethernet_IPv6_plen",
112
+ "Ethernet_IP_TCP_Encrypted Content_load",
113
+ "Ethernet_IP_TCP_TLS_TLS_Raw_load",
114
+ ],
115
+ )
116
+
117
+
19
118
  class PCAPDissector:
20
119
  """loads a pcap file and counts the contents in both time and depth."""
21
120
 
22
121
  def __init__(
23
122
  self,
24
123
  pcap_file: str,
25
- bin_size: int = 0,
26
- maximum_count: int = 0,
27
- dissector_level: PCAPDissectorLevel = PCAPDissectorLevel.DETAILED,
28
- pcap_filter: str | None = None,
29
- cache_results: bool = False,
30
- cache_file_suffix: str = "taffy",
31
- ignore_list: list | None = None,
32
- layers: List[str] | None = None,
33
- force_overwrite: bool = False,
34
- force_load: bool = False,
35
- merge_files: bool = False, # Note: unused for a single load
124
+ config: TaffyConfig | None = None,
36
125
  ) -> None:
37
126
  """Create a dissector object."""
38
- if ignore_list is None:
39
- ignore_list = []
40
127
  self.pcap_file = pcap_file
41
- self.dissector_level = dissector_level
42
- self.pcap_filter = pcap_filter
43
- self.maximum_count = maximum_count
44
- self.cache_results = cache_results
45
- self.bin_size = bin_size
46
- if cache_file_suffix[0] != ".":
47
- cache_file_suffix = "." + cache_file_suffix
48
- self.cache_file_suffix = cache_file_suffix
49
- self.ignore_list = ignore_list
50
- self.layers = layers
51
- self.force_overwrite = force_overwrite
52
- self.force_load = force_load
53
-
54
- if dissector_level == PCAPDissectorLevel.COUNT_ONLY and bin_size == 0:
128
+ self.config = config
129
+ if not self.config:
130
+ config = TaffyConfig()
131
+
132
+ dissection_config = config[TTD_CFG.KEY_DISSECTOR]
133
+
134
+ self.dissector_level = dissection_config[TTD_CFG.DISSECTION_LEVEL]
135
+ self.pcap_filter = dissection_config[TTD_CFG.FILTER]
136
+ self.maximum_count = dissection_config[TTD_CFG.PACKET_COUNT]
137
+ self.cache_results = dissection_config[TTD_CFG.CACHE_PCAP_RESULTS]
138
+ self.bin_size = dissection_config[TTD_CFG.BIN_SIZE]
139
+ self.cache_file_suffix = dissection_config[TTD_CFG.CACHE_FILE_SUFFIX]
140
+ if self.cache_file_suffix[0] != ".":
141
+ self.cache_file_suffix = "." + self.cache_file_suffix
142
+ self.ignore_list = dissection_config[TTD_CFG.IGNORE_LIST]
143
+ if self.ignore_list is None:
144
+ self.ignore_list = []
145
+ self.layers = dissection_config[TTD_CFG.LAYERS]
146
+ self.force_overwrite = dissection_config[TTD_CFG.FORCE_OVERWRITE]
147
+ self.force_load = dissection_config[TTD_CFG.FORCE_LOAD]
148
+
149
+ if self.dissector_level == PCAPDissectorLevel.COUNT_ONLY and self.bin_size == 0:
55
150
  warning("counting packets only with no binning is unlikely to be helpful")
56
151
 
57
152
  @property
@@ -79,6 +174,7 @@ class PCAPDissector:
79
174
  def load_from_cache(
80
175
  self: PCAPDissector, force_overwrite: bool = False, force_load: bool = False
81
176
  ) -> Dissection:
177
+ """Load dissector contents from a cached file."""
82
178
  if self.cache_results:
83
179
  args = self.dissection_args()
84
180
  self.dissection = Dissection(*args)
@@ -123,7 +219,9 @@ class PCAPDissector:
123
219
 
124
220
  engine = DissectionEngineDpkt(*args)
125
221
 
222
+ debug(f"dissecting using {engine}")
126
223
  self.dissection = engine.load()
224
+ debug("done dissecting")
127
225
  call_hooks(POST_DISSECT_HOOK, dissection=self.dissection)
128
226
 
129
227
  if self.cache_results:
@@ -136,6 +234,7 @@ class PCAPDissector:
136
234
  match_string: str | None = None,
137
235
  match_value: str | None = None,
138
236
  minimum_count: int | None = None,
237
+ match_expression: str | None = None,
139
238
  ) -> None:
140
239
  """Print the results to the console."""
141
240
  if timestamps is None:
@@ -146,6 +245,7 @@ class PCAPDissector:
146
245
  match_value=match_value,
147
246
  minimum_count=minimum_count,
148
247
  make_printable=True,
248
+ match_expression=match_expression,
149
249
  ):
150
250
  print(f"{key:<30} {subkey:<30} {value}")
151
251
 
@@ -155,6 +255,7 @@ class PCAPDissector:
155
255
  match_string: str | None = None,
156
256
  match_value: str | None = None,
157
257
  minimum_count: int | None = None,
258
+ match_expression: str | None = None,
158
259
  ) -> None:
159
260
  """Output the results in an FSDB file."""
160
261
  if timestamps is None:
@@ -172,19 +273,28 @@ class PCAPDissector:
172
273
  match_value=match_value,
173
274
  minimum_count=minimum_count,
174
275
  make_printable=True,
276
+ match_expression=match_expression,
175
277
  ):
176
278
  fh.append([key, subkey, value])
177
279
  fh.close()
178
280
 
179
281
 
180
- def dissector_add_parseargs(parser, add_subgroup: bool = True):
282
+ def dissector_add_parseargs(
283
+ parser: Parser, config: TaffyConfig | None = None, add_subgroup: bool = True
284
+ ) -> None:
285
+ """Add arguments related to disection."""
181
286
  if add_subgroup:
182
- parser = parser.add_argument_group("Parsing Options")
287
+ parser = parser.add_argument_group("Dissection Options", config_path="dissect")
288
+
289
+ if not config:
290
+ config = TaffyConfig()
183
291
 
292
+ dissection_config = config[TTD_CFG.KEY_DISSECTOR]
184
293
  parser.add_argument(
185
294
  "-d",
186
295
  "--dissection-level",
187
- default=PCAPDissectorLevel.THROUGH_IP.value,
296
+ config_path=TTD_CFG.DISSECTION_LEVEL,
297
+ default=dissection_config[TTD_CFG.DISSECTION_LEVEL],
188
298
  type=int,
189
299
  help="Dump to various levels of detail (1-10, with 10 is the most detailed and slowest)",
190
300
  )
@@ -192,40 +302,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
192
302
  parser.add_argument(
193
303
  "-I",
194
304
  "--ignore-list",
195
- default=[
196
- "Ethernet_IP_TCP_seq",
197
- "Ethernet_IP_TCP_ack",
198
- "Ethernet_IPv6_TCP_seq",
199
- "Ethernet_IPv6_TCP_ack",
200
- "Ethernet_IPv6_TCP_Raw_load",
201
- "Ethernet_IP_UDP_Raw_load",
202
- "Ethernet_IP_UDP_DNS_id",
203
- "Ethernet_IP_ICMP_IP in ICMP_UDP in ICMP_chksum",
204
- "Ethernet_IP_ICMP_IP in ICMP_UDP in ICMP_Raw_load",
205
- "Ethernet_IP_ICMP_IP in ICMP_chksum",
206
- "Ethernet_IP_ICMP_IP in ICMP_id",
207
- "Ethernet_IP_TCP_DNS_id",
208
- "Ethernet_IPv6_UDP_DNS_id",
209
- "Ethernet_IPv6_TCP_DNS_id",
210
- "Ethernet_IP_id",
211
- "Ethernet_IP_chksum",
212
- "Ethernet_IP_UDP_chksum",
213
- "Ethernet_IP_TCP_chksum",
214
- "Ethernet_IP_TCP_window",
215
- "Ethernet_IP_TCP_Raw_load",
216
- "Ethernet_IP_UDP_Raw_load",
217
- "Ethernet_IPv6_UDP_chksum",
218
- "Ethernet_IPv6_fl",
219
- "Ethernet_IP_ICMP_chksum",
220
- "Ethernet_IP_ICMP_id",
221
- "Ethernet_IP_ICMP_seq",
222
- "Ethernet_IP_TCP_Padding_load",
223
- "Ethernet_IP_TCP_window",
224
- "Ethernet_IPv6_TCP_chksum",
225
- "Ethernet_IPv6_plen",
226
- "Ethernet_IP_TCP_Encrypted Content_load",
227
- "Ethernet_IP_TCP_TLS_TLS_Raw_load",
228
- ],
305
+ config_path=TTD_CFG.IGNORE_LIST,
306
+ default=dissection_config[TTD_CFG.IGNORE_LIST],
229
307
  nargs="*",
230
308
  type=str,
231
309
  help="A list of (unlikely to be useful) packet fields to ignore",
@@ -234,7 +312,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
234
312
  parser.add_argument(
235
313
  "-n",
236
314
  "--packet-count",
237
- default=0,
315
+ config_path=TTD_CFG.PACKET_COUNT,
316
+ default=dissection_config[TTD_CFG.PACKET_COUNT],
238
317
  type=int,
239
318
  help="Maximum number of packets to analyze",
240
319
  )
@@ -242,6 +321,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
242
321
  parser.add_argument(
243
322
  "-b",
244
323
  "--bin-size",
324
+ config_path=TTD_CFG.BIN_SIZE,
325
+ default=dissection_config[TTD_CFG.BIN_SIZE],
245
326
  type=int,
246
327
  help="Bin results into this many seconds",
247
328
  )
@@ -249,7 +330,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
249
330
  parser.add_argument(
250
331
  "-F",
251
332
  "--filter",
252
- default=None,
333
+ config_path=TTD_CFG.FILTER,
334
+ default=dissection_config[TTD_CFG.FILTER],
253
335
  type=str,
254
336
  help="filter to apply to the pcap file when processing",
255
337
  )
@@ -257,7 +339,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
257
339
  parser.add_argument(
258
340
  "-L",
259
341
  "--layers",
260
- default=[],
342
+ config_path=TTD_CFG.LAYERS,
343
+ default=dissection_config[TTD_CFG.LAYERS],
261
344
  type=str,
262
345
  nargs="*",
263
346
  help="List of extra layers to load (eg: tls, http, etc)",
@@ -266,7 +349,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
266
349
  parser.add_argument(
267
350
  "-x",
268
351
  "--modules",
269
- default=None,
352
+ config_path=TTD_CFG.MODULES,
353
+ default=dissection_config[TTD_CFG.MODULES],
270
354
  type=str,
271
355
  nargs="*",
272
356
  help="Extra processing modules to load (currently: psl) ",
@@ -275,6 +359,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
275
359
  parser.add_argument(
276
360
  "--merge",
277
361
  "--merge-files",
362
+ config_path=TTD_CFG.MERGE,
363
+ default=dissection_config[TTD_CFG.MERGE],
278
364
  action="store_true",
279
365
  help="Dissect multiple files as one. (compare by time)",
280
366
  )
@@ -282,6 +368,7 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
282
368
  parser.add_argument(
283
369
  "-C",
284
370
  "--cache-pcap-results",
371
+ config_path=TTD_CFG.CACHE_PCAP_RESULTS,
285
372
  action="store_true",
286
373
  help="Cache and use PCAP results into/from a cache file file",
287
374
  )
@@ -290,33 +377,45 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
290
377
  "--cache-file-suffix",
291
378
  "--cs",
292
379
  type=str,
293
- default="taffy",
380
+ config_path=TTD_CFG.CACHE_FILE_SUFFIX,
381
+ default=dissection_config[TTD_CFG.CACHE_FILE_SUFFIX],
294
382
  help="The suffix file to use when creating cache files",
295
383
  )
296
384
 
297
385
  parser.add_argument(
298
386
  "--force-overwrite",
299
387
  action="store_true",
388
+ config_path="force_overwrite",
300
389
  help="Force continuing with an incompatible cache (and rewriting it)",
301
390
  )
302
391
 
303
392
  parser.add_argument(
304
393
  "--force-load",
305
394
  action="store_true",
395
+ config_path="force_load",
306
396
  help="Force continuing with an incompatible cache (trying to load it anyway)",
307
397
  )
308
398
 
309
399
  return parser
310
400
 
311
401
 
312
- def limitor_add_parseargs(parser, add_subgroup: bool = True):
402
+ def limitor_add_parseargs(
403
+ parser, config: TaffyConfig = None, add_subgroup: bool = True
404
+ ):
313
405
  if add_subgroup:
314
- parser = parser.add_argument_group("Limiting options")
406
+ parser = parser.add_argument_group(
407
+ "Limiting options", config_path=TTL_CFG.KEY_LIMITOR
408
+ )
409
+
410
+ if not config:
411
+ config = TaffyConfig()
315
412
 
413
+ limitor_config = config[TTL_CFG.KEY_LIMITOR]
316
414
  parser.add_argument(
317
415
  "-m",
318
416
  "--match-string",
319
- default=None,
417
+ config_path=TTL_CFG.MATCH_STRING,
418
+ default=limitor_config[TTL_CFG.MATCH_STRING],
320
419
  type=str,
321
420
  help="Only report on data with this substring in the header",
322
421
  )
@@ -324,16 +423,27 @@ def limitor_add_parseargs(parser, add_subgroup: bool = True):
324
423
  parser.add_argument(
325
424
  "-M",
326
425
  "--match-value",
327
- default=None,
426
+ config_path=TTL_CFG.MATCH_VALUE,
427
+ default=limitor_config[TTL_CFG.MATCH_VALUE],
328
428
  type=str,
329
429
  nargs="*",
330
430
  help="Only report on data with this substring in the packet value field",
331
431
  )
332
432
 
433
+ parser.add_argument(
434
+ "-E",
435
+ "--match-expression",
436
+ config_path=TTL_CFG.MATCH_EXPRESSION,
437
+ default=limitor_config[TTL_CFG.MATCH_EXPRESSION],
438
+ type=str,
439
+ help="Match expression to be evaluated at runtime for returning data",
440
+ )
441
+
333
442
  parser.add_argument(
334
443
  "-c",
335
444
  "--minimum-count",
336
- default=None,
445
+ config_path=TTL_CFG.MINIMUM_COUNT,
446
+ default=limitor_config[TTL_CFG.MINIMUM_COUNT],
337
447
  type=float,
338
448
  help="Don't include results without this high of a record count",
339
449
  )
@@ -342,22 +452,24 @@ def limitor_add_parseargs(parser, add_subgroup: bool = True):
342
452
 
343
453
 
344
454
  def dissector_handle_arguments(args) -> None:
455
+ """Handle checking and loading arguments."""
345
456
  check_dissector_level(args.dissection_level)
346
457
  dissector_load_extra_modules(args.modules)
347
458
 
348
459
 
349
460
  def dissector_load_extra_modules(modules: List[str]) -> None:
350
- """Loads extra modules"""
461
+ """Load extra modules."""
351
462
  if not modules:
352
463
  return
353
464
  for module in modules:
354
465
  try:
355
466
  importlib.import_module(f"traffic_taffy.hooks.{module}")
467
+ info(f"loaded module: {module}")
356
468
  except Exception as exp:
357
469
  error(f"failed to load module {module}: {exp}")
358
470
 
359
471
 
360
- def check_dissector_level(level: int):
472
+ def check_dissector_level(level: int) -> bool:
361
473
  """Check that the dissector level is legal."""
362
474
  current_dissection_levels = [
363
475
  PCAPDissectorLevel.COUNT_ONLY.value,
@@ -6,6 +6,12 @@ from pcap_parallel import PCAPParallel
6
6
  from logging import warning
7
7
 
8
8
  from scapy.all import sniff, load_layer
9
+ from tempfile import NamedTemporaryFile
10
+ from traffic_taffy.taffy_config import TaffyConfig, taffy_default
11
+
12
+
13
+ taffy_default("dissect.engines.scapy.use_temp_files", False)
14
+ taffy_default("dissect.engines.scapy.temp_file_directory", None)
9
15
 
10
16
 
11
17
  class DissectionEngineScapy(DissectionEngine):
@@ -15,6 +21,8 @@ class DissectionEngineScapy(DissectionEngine):
15
21
  """Create a scapy engine class."""
16
22
  super().__init__(*args, **kwargs)
17
23
 
24
+ self.taffy_config = TaffyConfig()
25
+
18
26
  def load_data(self) -> None:
19
27
  """Load a pcap file into a nested dictionary of statistical counts."""
20
28
  if isinstance(self.pcap_file, str):
@@ -22,18 +30,43 @@ class DissectionEngineScapy(DissectionEngine):
22
30
  else:
23
31
  load_this = self.pcap_file
24
32
 
33
+ use_temp_files: bool = self.taffy_config.get_dotnest(
34
+ "dissect.engines.scapy.use_temp_files"
35
+ )
36
+ if self.pcap_filter is not None and self.pcap_filter != "":
37
+ # somehow scapy hangs when a filter is applied to a memory object
38
+ use_temp_files = True
39
+
25
40
  if self.layers:
26
41
  for layer in self.layers:
27
42
  load_layer(layer)
28
43
 
29
- sniff(
30
- offline=load_this,
31
- prn=self.callback,
32
- store=0,
33
- count=self.maximum_count,
34
- filter=self.pcap_filter,
35
- )
36
- # TODO(hardaker): for some reason this fails on xz compressed files when processing in parallel
44
+ if use_temp_files:
45
+ tmp_directory = self.taffy_config.get_dotnest(
46
+ "dissect.engines.scapy.temp_file_directory"
47
+ )
48
+ with NamedTemporaryFile(dir=tmp_directory) as tmpf:
49
+ tmpf.write(load_this.read())
50
+ tmpf.flush()
51
+
52
+ sniff(
53
+ offline=tmpf.name,
54
+ prn=self.callback,
55
+ store=0,
56
+ count=self.maximum_count,
57
+ filter=self.pcap_filter,
58
+ )
59
+
60
+ else:
61
+ sniff(
62
+ offline=load_this,
63
+ prn=self.callback,
64
+ store=0,
65
+ count=self.maximum_count,
66
+ filter=self.pcap_filter,
67
+ )
68
+
69
+ # TODO(hardaker): for some reason this fails on xz compressed files when processing in parallel
37
70
 
38
71
  def add_item(self, field_value: str | int, prefix: str) -> None:
39
72
  """Add an item to the self.dissection regardless of it's various types"""
traffic_taffy/graph.py CHANGED
@@ -5,11 +5,23 @@ from __future__ import annotations
5
5
  import seaborn as sns
6
6
  import matplotlib.pyplot as plt
7
7
  from logging import debug, info
8
- from typing import List
9
8
 
10
- from traffic_taffy.dissector import PCAPDissectorLevel
11
9
  from traffic_taffy.dissectmany import PCAPDissectMany
12
10
  from traffic_taffy.graphdata import PcapGraphData
11
+ from traffic_taffy.taffy_config import TaffyConfig, taffy_default
12
+ from traffic_taffy.dissector import TTD_CFG, TTL_CFG
13
+
14
+
15
+ class TTG_CFG:
16
+ KEY_GRAPH: str = "graph"
17
+ OUTPUT_FILE: str = "output_file"
18
+ BY_PERCENTAGE: str = "by_percentage"
19
+ INTERACTIVE: str = "interactive"
20
+
21
+
22
+ taffy_default("graph.output_file", None)
23
+ taffy_default("graph.by_percentage", False)
24
+ taffy_default("graph.interactive", False)
13
25
 
14
26
 
15
27
  class PcapGraph(PcapGraphData):
@@ -19,46 +31,37 @@ class PcapGraph(PcapGraphData):
19
31
  self,
20
32
  pcap_files: str,
21
33
  output_file: str,
22
- maximum_count: int | None = None,
23
- minimum_count: int | None = None,
24
- bin_size: int | None = None,
25
- match_string: str | None = None,
26
- match_value: str | None = None,
27
- cache_pcap_results: bool = False,
28
- dissector_level: PCAPDissectorLevel = PCAPDissectorLevel.COUNT_ONLY,
29
- interactive: bool = False,
30
- ignore_list: List[str] | None = None,
31
- by_percentage: bool = False,
32
- pcap_filter: str | None = None,
33
- cache_file_suffix: str = "taffy",
34
- layers: List[str] | None = None,
35
- force_overwrite: bool = False,
36
- force_load: bool = False,
37
- merge_files: bool = False, # unused
34
+ config: TaffyConfig(),
38
35
  ):
39
36
  """Create an instance of a graphing object."""
37
+ self.config = config
38
+
39
+ dissector_config = config[TTD_CFG.KEY_DISSECTOR]
40
+ limitor_config = config[TTL_CFG.KEY_LIMITOR]
41
+ graph_config = config[TTG_CFG.KEY_GRAPH]
42
+ super().__init__(
43
+ match_string=limitor_config[TTL_CFG.MATCH_STRING],
44
+ match_value=limitor_config[TTL_CFG.MATCH_VALUE],
45
+ minimum_count=limitor_config[TTL_CFG.MINIMUM_COUNT],
46
+ match_expression=limitor_config[TTL_CFG.MATCH_EXPRESSION],
47
+ )
48
+
40
49
  self.pcap_files = pcap_files
41
50
  self.output_file = output_file
42
- self.maximum_count = maximum_count
43
- self.minimum_count = minimum_count
44
- self.bin_size = bin_size
45
- self.subsections = None
46
- self.pcap_filter = None
47
- self.match_string = match_string
48
- self.match_value = match_value
49
- self.cache_pcap_results = cache_pcap_results
50
- self.dissector_level = dissector_level
51
- self.interactive = interactive
52
- self.ignore_list = ignore_list or []
53
- self.by_percentage = by_percentage
54
- self.pcap_filter = pcap_filter
55
- self.cache_file_suffix = cache_file_suffix
56
- self.layers = layers
57
- self.force_overwrite = force_overwrite
58
- self.force_load = force_load
59
- self.merge_files = merge_files
60
-
61
- super().__init__()
51
+ self.maximum_count = dissector_config[TTD_CFG.PACKET_COUNT]
52
+ self.bin_size = dissector_config[TTD_CFG.BIN_SIZE]
53
+ self.pcap_filter = dissector_config[TTD_CFG.FILTER]
54
+ self.cache_pcap_results = dissector_config[TTD_CFG.CACHE_PCAP_RESULTS]
55
+ self.dissector_level = dissector_config[TTD_CFG.DISSECTION_LEVEL]
56
+ self.ignore_list = dissector_config[TTD_CFG.IGNORE_LIST]
57
+ self.cache_file_suffix = dissector_config[TTD_CFG.CACHE_FILE_SUFFIX]
58
+ self.layers = dissector_config[TTD_CFG.LAYERS]
59
+ self.force_overwrite = dissector_config[TTD_CFG.FORCE_OVERWRITE]
60
+ self.force_load = dissector_config[TTD_CFG.FORCE_LOAD]
61
+ self.merge_files = dissector_config[TTD_CFG.MERGE]
62
+
63
+ self.interactive = graph_config[TTG_CFG.INTERACTIVE]
64
+ self.by_percentage = graph_config[TTG_CFG.BY_PERCENTAGE]
62
65
 
63
66
  def load_pcaps(self) -> None:
64
67
  """Load the pcap and counts things into bins."""
@@ -67,35 +70,33 @@ class PcapGraph(PcapGraphData):
67
70
  info("reading pcap files")
68
71
  pdm = PCAPDissectMany(
69
72
  self.pcap_files,
70
- bin_size=self.bin_size,
71
- maximum_count=self.maximum_count,
72
- dissector_level=self.dissector_level,
73
- pcap_filter=self.pcap_filter,
74
- cache_results=self.cache_pcap_results,
75
- ignore_list=self.ignore_list,
76
- cache_file_suffix=self.cache_file_suffix,
77
- layers=self.layers,
78
- force_overwrite=self.force_overwrite,
79
- force_load=self.force_load,
80
- merge_files=self.merge_files,
73
+ self.config,
81
74
  )
82
75
  self.dissections = pdm.load_all()
83
76
  info("done reading pcap files")
84
77
 
85
- def create_graph(self) -> None:
78
+ def create_graph(self, options: dict | None = None) -> None:
86
79
  """Create the graph itself and save it."""
87
- df = self.get_dataframe(merge=True, calculate_load_fraction=self.by_percentage)
80
+ if not options:
81
+ options = {}
88
82
 
83
+ df = self.get_dataframe(merge=True, calculate_load_fraction=self.by_percentage)
89
84
  hue_variable = "index"
90
85
  if df[hue_variable].nunique() == 1:
91
86
  hue_variable = None
92
87
 
93
88
  if self.by_percentage:
94
- df["load_fraction"]
95
89
  y_column = "load_fraction"
96
90
  else:
97
91
  y_column = "count"
98
92
 
93
+ # TODO(hardaker): support re-indexing and hole filling (this doesn't work)
94
+ # str(self.bin_size or 1) + "s"
95
+ # df = df.set_index("time")
96
+ # df.index = df.index.to_period(freq=freq)
97
+ # timeindex = pd.period_range(min(df.index), max(df.index), freq=freq)
98
+ # df = df.reindex(timeindex) # , fill_value=0
99
+
99
100
  ax = sns.relplot(
100
101
  data=df,
101
102
  kind="line",
@@ -104,7 +105,7 @@ class PcapGraph(PcapGraphData):
104
105
  hue=hue_variable,
105
106
  aspect=1.77,
106
107
  )
107
- ax.set(xlabel="time", ylabel=y_column)
108
+ ax.set(xlabel="time", ylabel=options.get("ylabel", y_column))
108
109
  plt.xticks(rotation=45)
109
110
 
110
111
  info(f"saving graph to {self.output_file}")