traffic-taffy 0.8.1__py3-none-any.whl → 0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- traffic_taffy/__init__.py +1 -1
- traffic_taffy/algorithms/__init__.py +14 -7
- traffic_taffy/algorithms/comparecorrelation.py +164 -0
- traffic_taffy/algorithms/comparecorrelationchanges.py +210 -0
- traffic_taffy/algorithms/compareseries.py +117 -0
- traffic_taffy/algorithms/compareslices.py +116 -0
- traffic_taffy/algorithms/statistical.py +9 -9
- traffic_taffy/compare.py +149 -159
- traffic_taffy/comparison.py +18 -4
- traffic_taffy/config.py +133 -0
- traffic_taffy/dissection.py +171 -6
- traffic_taffy/dissectmany.py +26 -16
- traffic_taffy/dissector.py +189 -77
- traffic_taffy/dissector_engine/scapy.py +41 -8
- traffic_taffy/graph.py +54 -53
- traffic_taffy/graphdata.py +13 -2
- traffic_taffy/hooks/ip2asn.py +20 -7
- traffic_taffy/hooks/labels.py +45 -0
- traffic_taffy/hooks/psl.py +21 -3
- traffic_taffy/iana/tables.msgpak +0 -0
- traffic_taffy/output/__init__.py +8 -48
- traffic_taffy/output/console.py +37 -25
- traffic_taffy/output/fsdb.py +24 -18
- traffic_taffy/reports/__init__.py +5 -0
- traffic_taffy/reports/compareslicesreport.py +85 -0
- traffic_taffy/reports/correlationchangereport.py +54 -0
- traffic_taffy/reports/correlationreport.py +42 -0
- traffic_taffy/taffy_config.py +44 -0
- traffic_taffy/tests/test_compare_results.py +22 -7
- traffic_taffy/tests/test_config.py +149 -0
- traffic_taffy/tests/test_global_config.py +33 -0
- traffic_taffy/tests/test_normalize.py +1 -0
- traffic_taffy/tests/test_pcap_dissector.py +12 -2
- traffic_taffy/tests/test_pcap_splitter.py +21 -10
- traffic_taffy/tools/cache_info.py +3 -2
- traffic_taffy/tools/compare.py +32 -24
- traffic_taffy/tools/config.py +83 -0
- traffic_taffy/tools/dissect.py +51 -59
- traffic_taffy/tools/explore.py +5 -4
- traffic_taffy/tools/export.py +28 -17
- traffic_taffy/tools/graph.py +25 -27
- {traffic_taffy-0.8.1.dist-info → traffic_taffy-0.9.dist-info}/METADATA +4 -1
- traffic_taffy-0.9.dist-info/RECORD +56 -0
- {traffic_taffy-0.8.1.dist-info → traffic_taffy-0.9.dist-info}/entry_points.txt +1 -0
- traffic_taffy/report.py +0 -12
- traffic_taffy/tests/test_dpkt_engine.py +0 -15
- traffic_taffy-0.8.1.dist-info/RECORD +0 -43
- {traffic_taffy-0.8.1.dist-info → traffic_taffy-0.9.dist-info}/WHEEL +0 -0
- {traffic_taffy-0.8.1.dist-info → traffic_taffy-0.9.dist-info}/licenses/LICENSE.txt +0 -0
traffic_taffy/dissector.py
CHANGED
@@ -4,54 +4,149 @@ from __future__ import annotations
|
|
4
4
|
|
5
5
|
import sys
|
6
6
|
from collections import Counter, defaultdict
|
7
|
-
from logging import error, warning
|
7
|
+
from logging import error, warning, debug, info
|
8
8
|
from typing import List
|
9
9
|
import importlib
|
10
|
+
from typing import TYPE_CHECKING, Any
|
10
11
|
|
11
12
|
from rich import print
|
12
13
|
|
13
14
|
from traffic_taffy.dissection import Dissection, PCAPDissectorLevel
|
14
15
|
from traffic_taffy.hooks import call_hooks
|
16
|
+
from traffic_taffy.taffy_config import TaffyConfig, taffy_default
|
17
|
+
|
18
|
+
if TYPE_CHECKING:
|
19
|
+
from argparse import Parser
|
20
|
+
|
21
|
+
|
22
|
+
class TTD_CFG:
|
23
|
+
KEY_DISSECTOR: str = "dissect"
|
24
|
+
|
25
|
+
BIN_SIZE: str = "bin_size"
|
26
|
+
CACHE_FILE_SUFFIX: str = "cache_file_suffix"
|
27
|
+
CACHE_PCAP_RESULTS: str = "cache_pcap_results"
|
28
|
+
DISSECTION_LEVEL: str = "dissection_level"
|
29
|
+
FILTER: str = "filter"
|
30
|
+
FILTER_ARGUMENTS: str = "filter_arguments"
|
31
|
+
FORCE_LOAD: str = "force_load"
|
32
|
+
FORCE_OVERWRITE: str = "force_overwrite"
|
33
|
+
IGNORE_LIST: str = "ignore_list"
|
34
|
+
LAYERS: str = "layers"
|
35
|
+
MERGE: str = "merge"
|
36
|
+
MODULES: str = "use_modules"
|
37
|
+
PACKET_COUNT: str = "packet_count"
|
38
|
+
|
39
|
+
|
40
|
+
class TTL_CFG:
|
41
|
+
KEY_LIMITOR: str = "limit_output"
|
42
|
+
|
43
|
+
MATCH_EXPRESSION: str = "match_expression"
|
44
|
+
MATCH_STRING: str = "match_string"
|
45
|
+
MATCH_VALUE: str = "match_value"
|
46
|
+
MINIMUM_COUNT: str = "minimum_count"
|
47
|
+
|
15
48
|
|
16
49
|
POST_DISSECT_HOOK: str = "post_dissect"
|
17
50
|
|
18
51
|
|
52
|
+
def dissector_default(name: str, value: Any) -> None:
|
53
|
+
taffy_default(TTD_CFG.KEY_DISSECTOR + "." + name, value)
|
54
|
+
|
55
|
+
|
56
|
+
dissector_default("dissection_level", PCAPDissectorLevel.THROUGH_IP.value)
|
57
|
+
dissector_default("packet_count", 0)
|
58
|
+
dissector_default("bin_size", None)
|
59
|
+
dissector_default("filter", None)
|
60
|
+
dissector_default("layers", [])
|
61
|
+
dissector_default("use_modules", None)
|
62
|
+
dissector_default("merge", False)
|
63
|
+
dissector_default("cache_pcap_results", False)
|
64
|
+
dissector_default("force_overwrite", False)
|
65
|
+
dissector_default("force_load", False)
|
66
|
+
dissector_default("cache_file_suffix", "taffy")
|
67
|
+
dissector_default("maximum_cores", 20) # TODO(hardaker): fix double forking
|
68
|
+
|
69
|
+
|
70
|
+
def limitor_default(name: str, value: Any) -> None:
|
71
|
+
taffy_default(TTL_CFG.KEY_LIMITOR + "." + name, value)
|
72
|
+
|
73
|
+
|
74
|
+
limitor_default("match_string", None)
|
75
|
+
limitor_default("match_value", None)
|
76
|
+
limitor_default("match_expression", None)
|
77
|
+
limitor_default("minimum_count", None)
|
78
|
+
|
79
|
+
dissector_default(
|
80
|
+
"ignore_list",
|
81
|
+
[
|
82
|
+
"Ethernet_IP_TCP_seq",
|
83
|
+
"Ethernet_IP_TCP_ack",
|
84
|
+
"Ethernet_IPv6_TCP_seq",
|
85
|
+
"Ethernet_IPv6_TCP_ack",
|
86
|
+
"Ethernet_IPv6_TCP_Raw_load",
|
87
|
+
"Ethernet_IP_UDP_Raw_load",
|
88
|
+
"Ethernet_IP_UDP_DNS_id",
|
89
|
+
"Ethernet_IP_ICMP_IP in ICMP_UDP in ICMP_chksum",
|
90
|
+
"Ethernet_IP_ICMP_IP in ICMP_UDP in ICMP_Raw_load",
|
91
|
+
"Ethernet_IP_ICMP_IP in ICMP_chksum",
|
92
|
+
"Ethernet_IP_ICMP_IP in ICMP_id",
|
93
|
+
"Ethernet_IP_TCP_DNS_id",
|
94
|
+
"Ethernet_IPv6_UDP_DNS_id",
|
95
|
+
"Ethernet_IPv6_TCP_DNS_id",
|
96
|
+
"Ethernet_IP_id",
|
97
|
+
"Ethernet_IP_chksum",
|
98
|
+
"Ethernet_IP_UDP_chksum",
|
99
|
+
"Ethernet_IP_TCP_chksum",
|
100
|
+
"Ethernet_IP_TCP_window",
|
101
|
+
"Ethernet_IP_TCP_Raw_load",
|
102
|
+
"Ethernet_IP_UDP_Raw_load",
|
103
|
+
"Ethernet_IPv6_UDP_chksum",
|
104
|
+
"Ethernet_IPv6_fl",
|
105
|
+
"Ethernet_IP_ICMP_chksum",
|
106
|
+
"Ethernet_IP_ICMP_id",
|
107
|
+
"Ethernet_IP_ICMP_seq",
|
108
|
+
"Ethernet_IP_TCP_Padding_load",
|
109
|
+
"Ethernet_IP_TCP_window",
|
110
|
+
"Ethernet_IPv6_TCP_chksum",
|
111
|
+
"Ethernet_IPv6_plen",
|
112
|
+
"Ethernet_IP_TCP_Encrypted Content_load",
|
113
|
+
"Ethernet_IP_TCP_TLS_TLS_Raw_load",
|
114
|
+
],
|
115
|
+
)
|
116
|
+
|
117
|
+
|
19
118
|
class PCAPDissector:
|
20
119
|
"""loads a pcap file and counts the contents in both time and depth."""
|
21
120
|
|
22
121
|
def __init__(
|
23
122
|
self,
|
24
123
|
pcap_file: str,
|
25
|
-
|
26
|
-
maximum_count: int = 0,
|
27
|
-
dissector_level: PCAPDissectorLevel = PCAPDissectorLevel.DETAILED,
|
28
|
-
pcap_filter: str | None = None,
|
29
|
-
cache_results: bool = False,
|
30
|
-
cache_file_suffix: str = "taffy",
|
31
|
-
ignore_list: list | None = None,
|
32
|
-
layers: List[str] | None = None,
|
33
|
-
force_overwrite: bool = False,
|
34
|
-
force_load: bool = False,
|
35
|
-
merge_files: bool = False, # Note: unused for a single load
|
124
|
+
config: TaffyConfig | None = None,
|
36
125
|
) -> None:
|
37
126
|
"""Create a dissector object."""
|
38
|
-
if ignore_list is None:
|
39
|
-
ignore_list = []
|
40
127
|
self.pcap_file = pcap_file
|
41
|
-
self.
|
42
|
-
self.
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
self.
|
49
|
-
self.
|
50
|
-
self.
|
51
|
-
self.
|
52
|
-
self.
|
53
|
-
|
54
|
-
|
128
|
+
self.config = config
|
129
|
+
if not self.config:
|
130
|
+
config = TaffyConfig()
|
131
|
+
|
132
|
+
dissection_config = config[TTD_CFG.KEY_DISSECTOR]
|
133
|
+
|
134
|
+
self.dissector_level = dissection_config[TTD_CFG.DISSECTION_LEVEL]
|
135
|
+
self.pcap_filter = dissection_config[TTD_CFG.FILTER]
|
136
|
+
self.maximum_count = dissection_config[TTD_CFG.PACKET_COUNT]
|
137
|
+
self.cache_results = dissection_config[TTD_CFG.CACHE_PCAP_RESULTS]
|
138
|
+
self.bin_size = dissection_config[TTD_CFG.BIN_SIZE]
|
139
|
+
self.cache_file_suffix = dissection_config[TTD_CFG.CACHE_FILE_SUFFIX]
|
140
|
+
if self.cache_file_suffix[0] != ".":
|
141
|
+
self.cache_file_suffix = "." + self.cache_file_suffix
|
142
|
+
self.ignore_list = dissection_config[TTD_CFG.IGNORE_LIST]
|
143
|
+
if self.ignore_list is None:
|
144
|
+
self.ignore_list = []
|
145
|
+
self.layers = dissection_config[TTD_CFG.LAYERS]
|
146
|
+
self.force_overwrite = dissection_config[TTD_CFG.FORCE_OVERWRITE]
|
147
|
+
self.force_load = dissection_config[TTD_CFG.FORCE_LOAD]
|
148
|
+
|
149
|
+
if self.dissector_level == PCAPDissectorLevel.COUNT_ONLY and self.bin_size == 0:
|
55
150
|
warning("counting packets only with no binning is unlikely to be helpful")
|
56
151
|
|
57
152
|
@property
|
@@ -79,6 +174,7 @@ class PCAPDissector:
|
|
79
174
|
def load_from_cache(
|
80
175
|
self: PCAPDissector, force_overwrite: bool = False, force_load: bool = False
|
81
176
|
) -> Dissection:
|
177
|
+
"""Load dissector contents from a cached file."""
|
82
178
|
if self.cache_results:
|
83
179
|
args = self.dissection_args()
|
84
180
|
self.dissection = Dissection(*args)
|
@@ -123,7 +219,9 @@ class PCAPDissector:
|
|
123
219
|
|
124
220
|
engine = DissectionEngineDpkt(*args)
|
125
221
|
|
222
|
+
debug(f"dissecting using {engine}")
|
126
223
|
self.dissection = engine.load()
|
224
|
+
debug("done dissecting")
|
127
225
|
call_hooks(POST_DISSECT_HOOK, dissection=self.dissection)
|
128
226
|
|
129
227
|
if self.cache_results:
|
@@ -136,6 +234,7 @@ class PCAPDissector:
|
|
136
234
|
match_string: str | None = None,
|
137
235
|
match_value: str | None = None,
|
138
236
|
minimum_count: int | None = None,
|
237
|
+
match_expression: str | None = None,
|
139
238
|
) -> None:
|
140
239
|
"""Print the results to the console."""
|
141
240
|
if timestamps is None:
|
@@ -146,6 +245,7 @@ class PCAPDissector:
|
|
146
245
|
match_value=match_value,
|
147
246
|
minimum_count=minimum_count,
|
148
247
|
make_printable=True,
|
248
|
+
match_expression=match_expression,
|
149
249
|
):
|
150
250
|
print(f"{key:<30} {subkey:<30} {value}")
|
151
251
|
|
@@ -155,6 +255,7 @@ class PCAPDissector:
|
|
155
255
|
match_string: str | None = None,
|
156
256
|
match_value: str | None = None,
|
157
257
|
minimum_count: int | None = None,
|
258
|
+
match_expression: str | None = None,
|
158
259
|
) -> None:
|
159
260
|
"""Output the results in an FSDB file."""
|
160
261
|
if timestamps is None:
|
@@ -172,19 +273,28 @@ class PCAPDissector:
|
|
172
273
|
match_value=match_value,
|
173
274
|
minimum_count=minimum_count,
|
174
275
|
make_printable=True,
|
276
|
+
match_expression=match_expression,
|
175
277
|
):
|
176
278
|
fh.append([key, subkey, value])
|
177
279
|
fh.close()
|
178
280
|
|
179
281
|
|
180
|
-
def dissector_add_parseargs(
|
282
|
+
def dissector_add_parseargs(
|
283
|
+
parser: Parser, config: TaffyConfig | None = None, add_subgroup: bool = True
|
284
|
+
) -> None:
|
285
|
+
"""Add arguments related to disection."""
|
181
286
|
if add_subgroup:
|
182
|
-
parser = parser.add_argument_group("
|
287
|
+
parser = parser.add_argument_group("Dissection Options", config_path="dissect")
|
288
|
+
|
289
|
+
if not config:
|
290
|
+
config = TaffyConfig()
|
183
291
|
|
292
|
+
dissection_config = config[TTD_CFG.KEY_DISSECTOR]
|
184
293
|
parser.add_argument(
|
185
294
|
"-d",
|
186
295
|
"--dissection-level",
|
187
|
-
|
296
|
+
config_path=TTD_CFG.DISSECTION_LEVEL,
|
297
|
+
default=dissection_config[TTD_CFG.DISSECTION_LEVEL],
|
188
298
|
type=int,
|
189
299
|
help="Dump to various levels of detail (1-10, with 10 is the most detailed and slowest)",
|
190
300
|
)
|
@@ -192,40 +302,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
192
302
|
parser.add_argument(
|
193
303
|
"-I",
|
194
304
|
"--ignore-list",
|
195
|
-
|
196
|
-
|
197
|
-
"Ethernet_IP_TCP_ack",
|
198
|
-
"Ethernet_IPv6_TCP_seq",
|
199
|
-
"Ethernet_IPv6_TCP_ack",
|
200
|
-
"Ethernet_IPv6_TCP_Raw_load",
|
201
|
-
"Ethernet_IP_UDP_Raw_load",
|
202
|
-
"Ethernet_IP_UDP_DNS_id",
|
203
|
-
"Ethernet_IP_ICMP_IP in ICMP_UDP in ICMP_chksum",
|
204
|
-
"Ethernet_IP_ICMP_IP in ICMP_UDP in ICMP_Raw_load",
|
205
|
-
"Ethernet_IP_ICMP_IP in ICMP_chksum",
|
206
|
-
"Ethernet_IP_ICMP_IP in ICMP_id",
|
207
|
-
"Ethernet_IP_TCP_DNS_id",
|
208
|
-
"Ethernet_IPv6_UDP_DNS_id",
|
209
|
-
"Ethernet_IPv6_TCP_DNS_id",
|
210
|
-
"Ethernet_IP_id",
|
211
|
-
"Ethernet_IP_chksum",
|
212
|
-
"Ethernet_IP_UDP_chksum",
|
213
|
-
"Ethernet_IP_TCP_chksum",
|
214
|
-
"Ethernet_IP_TCP_window",
|
215
|
-
"Ethernet_IP_TCP_Raw_load",
|
216
|
-
"Ethernet_IP_UDP_Raw_load",
|
217
|
-
"Ethernet_IPv6_UDP_chksum",
|
218
|
-
"Ethernet_IPv6_fl",
|
219
|
-
"Ethernet_IP_ICMP_chksum",
|
220
|
-
"Ethernet_IP_ICMP_id",
|
221
|
-
"Ethernet_IP_ICMP_seq",
|
222
|
-
"Ethernet_IP_TCP_Padding_load",
|
223
|
-
"Ethernet_IP_TCP_window",
|
224
|
-
"Ethernet_IPv6_TCP_chksum",
|
225
|
-
"Ethernet_IPv6_plen",
|
226
|
-
"Ethernet_IP_TCP_Encrypted Content_load",
|
227
|
-
"Ethernet_IP_TCP_TLS_TLS_Raw_load",
|
228
|
-
],
|
305
|
+
config_path=TTD_CFG.IGNORE_LIST,
|
306
|
+
default=dissection_config[TTD_CFG.IGNORE_LIST],
|
229
307
|
nargs="*",
|
230
308
|
type=str,
|
231
309
|
help="A list of (unlikely to be useful) packet fields to ignore",
|
@@ -234,7 +312,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
234
312
|
parser.add_argument(
|
235
313
|
"-n",
|
236
314
|
"--packet-count",
|
237
|
-
|
315
|
+
config_path=TTD_CFG.PACKET_COUNT,
|
316
|
+
default=dissection_config[TTD_CFG.PACKET_COUNT],
|
238
317
|
type=int,
|
239
318
|
help="Maximum number of packets to analyze",
|
240
319
|
)
|
@@ -242,6 +321,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
242
321
|
parser.add_argument(
|
243
322
|
"-b",
|
244
323
|
"--bin-size",
|
324
|
+
config_path=TTD_CFG.BIN_SIZE,
|
325
|
+
default=dissection_config[TTD_CFG.BIN_SIZE],
|
245
326
|
type=int,
|
246
327
|
help="Bin results into this many seconds",
|
247
328
|
)
|
@@ -249,7 +330,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
249
330
|
parser.add_argument(
|
250
331
|
"-F",
|
251
332
|
"--filter",
|
252
|
-
|
333
|
+
config_path=TTD_CFG.FILTER,
|
334
|
+
default=dissection_config[TTD_CFG.FILTER],
|
253
335
|
type=str,
|
254
336
|
help="filter to apply to the pcap file when processing",
|
255
337
|
)
|
@@ -257,7 +339,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
257
339
|
parser.add_argument(
|
258
340
|
"-L",
|
259
341
|
"--layers",
|
260
|
-
|
342
|
+
config_path=TTD_CFG.LAYERS,
|
343
|
+
default=dissection_config[TTD_CFG.LAYERS],
|
261
344
|
type=str,
|
262
345
|
nargs="*",
|
263
346
|
help="List of extra layers to load (eg: tls, http, etc)",
|
@@ -266,7 +349,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
266
349
|
parser.add_argument(
|
267
350
|
"-x",
|
268
351
|
"--modules",
|
269
|
-
|
352
|
+
config_path=TTD_CFG.MODULES,
|
353
|
+
default=dissection_config[TTD_CFG.MODULES],
|
270
354
|
type=str,
|
271
355
|
nargs="*",
|
272
356
|
help="Extra processing modules to load (currently: psl) ",
|
@@ -275,6 +359,8 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
275
359
|
parser.add_argument(
|
276
360
|
"--merge",
|
277
361
|
"--merge-files",
|
362
|
+
config_path=TTD_CFG.MERGE,
|
363
|
+
default=dissection_config[TTD_CFG.MERGE],
|
278
364
|
action="store_true",
|
279
365
|
help="Dissect multiple files as one. (compare by time)",
|
280
366
|
)
|
@@ -282,6 +368,7 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
282
368
|
parser.add_argument(
|
283
369
|
"-C",
|
284
370
|
"--cache-pcap-results",
|
371
|
+
config_path=TTD_CFG.CACHE_PCAP_RESULTS,
|
285
372
|
action="store_true",
|
286
373
|
help="Cache and use PCAP results into/from a cache file file",
|
287
374
|
)
|
@@ -290,33 +377,45 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
290
377
|
"--cache-file-suffix",
|
291
378
|
"--cs",
|
292
379
|
type=str,
|
293
|
-
|
380
|
+
config_path=TTD_CFG.CACHE_FILE_SUFFIX,
|
381
|
+
default=dissection_config[TTD_CFG.CACHE_FILE_SUFFIX],
|
294
382
|
help="The suffix file to use when creating cache files",
|
295
383
|
)
|
296
384
|
|
297
385
|
parser.add_argument(
|
298
386
|
"--force-overwrite",
|
299
387
|
action="store_true",
|
388
|
+
config_path="force_overwrite",
|
300
389
|
help="Force continuing with an incompatible cache (and rewriting it)",
|
301
390
|
)
|
302
391
|
|
303
392
|
parser.add_argument(
|
304
393
|
"--force-load",
|
305
394
|
action="store_true",
|
395
|
+
config_path="force_load",
|
306
396
|
help="Force continuing with an incompatible cache (trying to load it anyway)",
|
307
397
|
)
|
308
398
|
|
309
399
|
return parser
|
310
400
|
|
311
401
|
|
312
|
-
def limitor_add_parseargs(
|
402
|
+
def limitor_add_parseargs(
|
403
|
+
parser, config: TaffyConfig = None, add_subgroup: bool = True
|
404
|
+
):
|
313
405
|
if add_subgroup:
|
314
|
-
parser = parser.add_argument_group(
|
406
|
+
parser = parser.add_argument_group(
|
407
|
+
"Limiting options", config_path=TTL_CFG.KEY_LIMITOR
|
408
|
+
)
|
409
|
+
|
410
|
+
if not config:
|
411
|
+
config = TaffyConfig()
|
315
412
|
|
413
|
+
limitor_config = config[TTL_CFG.KEY_LIMITOR]
|
316
414
|
parser.add_argument(
|
317
415
|
"-m",
|
318
416
|
"--match-string",
|
319
|
-
|
417
|
+
config_path=TTL_CFG.MATCH_STRING,
|
418
|
+
default=limitor_config[TTL_CFG.MATCH_STRING],
|
320
419
|
type=str,
|
321
420
|
help="Only report on data with this substring in the header",
|
322
421
|
)
|
@@ -324,16 +423,27 @@ def limitor_add_parseargs(parser, add_subgroup: bool = True):
|
|
324
423
|
parser.add_argument(
|
325
424
|
"-M",
|
326
425
|
"--match-value",
|
327
|
-
|
426
|
+
config_path=TTL_CFG.MATCH_VALUE,
|
427
|
+
default=limitor_config[TTL_CFG.MATCH_VALUE],
|
328
428
|
type=str,
|
329
429
|
nargs="*",
|
330
430
|
help="Only report on data with this substring in the packet value field",
|
331
431
|
)
|
332
432
|
|
433
|
+
parser.add_argument(
|
434
|
+
"-E",
|
435
|
+
"--match-expression",
|
436
|
+
config_path=TTL_CFG.MATCH_EXPRESSION,
|
437
|
+
default=limitor_config[TTL_CFG.MATCH_EXPRESSION],
|
438
|
+
type=str,
|
439
|
+
help="Match expression to be evaluated at runtime for returning data",
|
440
|
+
)
|
441
|
+
|
333
442
|
parser.add_argument(
|
334
443
|
"-c",
|
335
444
|
"--minimum-count",
|
336
|
-
|
445
|
+
config_path=TTL_CFG.MINIMUM_COUNT,
|
446
|
+
default=limitor_config[TTL_CFG.MINIMUM_COUNT],
|
337
447
|
type=float,
|
338
448
|
help="Don't include results without this high of a record count",
|
339
449
|
)
|
@@ -342,22 +452,24 @@ def limitor_add_parseargs(parser, add_subgroup: bool = True):
|
|
342
452
|
|
343
453
|
|
344
454
|
def dissector_handle_arguments(args) -> None:
|
455
|
+
"""Handle checking and loading arguments."""
|
345
456
|
check_dissector_level(args.dissection_level)
|
346
457
|
dissector_load_extra_modules(args.modules)
|
347
458
|
|
348
459
|
|
349
460
|
def dissector_load_extra_modules(modules: List[str]) -> None:
|
350
|
-
"""
|
461
|
+
"""Load extra modules."""
|
351
462
|
if not modules:
|
352
463
|
return
|
353
464
|
for module in modules:
|
354
465
|
try:
|
355
466
|
importlib.import_module(f"traffic_taffy.hooks.{module}")
|
467
|
+
info(f"loaded module: {module}")
|
356
468
|
except Exception as exp:
|
357
469
|
error(f"failed to load module {module}: {exp}")
|
358
470
|
|
359
471
|
|
360
|
-
def check_dissector_level(level: int):
|
472
|
+
def check_dissector_level(level: int) -> bool:
|
361
473
|
"""Check that the dissector level is legal."""
|
362
474
|
current_dissection_levels = [
|
363
475
|
PCAPDissectorLevel.COUNT_ONLY.value,
|
@@ -6,6 +6,12 @@ from pcap_parallel import PCAPParallel
|
|
6
6
|
from logging import warning
|
7
7
|
|
8
8
|
from scapy.all import sniff, load_layer
|
9
|
+
from tempfile import NamedTemporaryFile
|
10
|
+
from traffic_taffy.taffy_config import TaffyConfig, taffy_default
|
11
|
+
|
12
|
+
|
13
|
+
taffy_default("dissect.engines.scapy.use_temp_files", False)
|
14
|
+
taffy_default("dissect.engines.scapy.temp_file_directory", None)
|
9
15
|
|
10
16
|
|
11
17
|
class DissectionEngineScapy(DissectionEngine):
|
@@ -15,6 +21,8 @@ class DissectionEngineScapy(DissectionEngine):
|
|
15
21
|
"""Create a scapy engine class."""
|
16
22
|
super().__init__(*args, **kwargs)
|
17
23
|
|
24
|
+
self.taffy_config = TaffyConfig()
|
25
|
+
|
18
26
|
def load_data(self) -> None:
|
19
27
|
"""Load a pcap file into a nested dictionary of statistical counts."""
|
20
28
|
if isinstance(self.pcap_file, str):
|
@@ -22,18 +30,43 @@ class DissectionEngineScapy(DissectionEngine):
|
|
22
30
|
else:
|
23
31
|
load_this = self.pcap_file
|
24
32
|
|
33
|
+
use_temp_files: bool = self.taffy_config.get_dotnest(
|
34
|
+
"dissect.engines.scapy.use_temp_files"
|
35
|
+
)
|
36
|
+
if self.pcap_filter is not None and self.pcap_filter != "":
|
37
|
+
# somehow scapy hangs when a filter is applied to a memory object
|
38
|
+
use_temp_files = True
|
39
|
+
|
25
40
|
if self.layers:
|
26
41
|
for layer in self.layers:
|
27
42
|
load_layer(layer)
|
28
43
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
44
|
+
if use_temp_files:
|
45
|
+
tmp_directory = self.taffy_config.get_dotnest(
|
46
|
+
"dissect.engines.scapy.temp_file_directory"
|
47
|
+
)
|
48
|
+
with NamedTemporaryFile(dir=tmp_directory) as tmpf:
|
49
|
+
tmpf.write(load_this.read())
|
50
|
+
tmpf.flush()
|
51
|
+
|
52
|
+
sniff(
|
53
|
+
offline=tmpf.name,
|
54
|
+
prn=self.callback,
|
55
|
+
store=0,
|
56
|
+
count=self.maximum_count,
|
57
|
+
filter=self.pcap_filter,
|
58
|
+
)
|
59
|
+
|
60
|
+
else:
|
61
|
+
sniff(
|
62
|
+
offline=load_this,
|
63
|
+
prn=self.callback,
|
64
|
+
store=0,
|
65
|
+
count=self.maximum_count,
|
66
|
+
filter=self.pcap_filter,
|
67
|
+
)
|
68
|
+
|
69
|
+
# TODO(hardaker): for some reason this fails on xz compressed files when processing in parallel
|
37
70
|
|
38
71
|
def add_item(self, field_value: str | int, prefix: str) -> None:
|
39
72
|
"""Add an item to the self.dissection regardless of it's various types"""
|
traffic_taffy/graph.py
CHANGED
@@ -5,11 +5,23 @@ from __future__ import annotations
|
|
5
5
|
import seaborn as sns
|
6
6
|
import matplotlib.pyplot as plt
|
7
7
|
from logging import debug, info
|
8
|
-
from typing import List
|
9
8
|
|
10
|
-
from traffic_taffy.dissector import PCAPDissectorLevel
|
11
9
|
from traffic_taffy.dissectmany import PCAPDissectMany
|
12
10
|
from traffic_taffy.graphdata import PcapGraphData
|
11
|
+
from traffic_taffy.taffy_config import TaffyConfig, taffy_default
|
12
|
+
from traffic_taffy.dissector import TTD_CFG, TTL_CFG
|
13
|
+
|
14
|
+
|
15
|
+
class TTG_CFG:
|
16
|
+
KEY_GRAPH: str = "graph"
|
17
|
+
OUTPUT_FILE: str = "output_file"
|
18
|
+
BY_PERCENTAGE: str = "by_percentage"
|
19
|
+
INTERACTIVE: str = "interactive"
|
20
|
+
|
21
|
+
|
22
|
+
taffy_default("graph.output_file", None)
|
23
|
+
taffy_default("graph.by_percentage", False)
|
24
|
+
taffy_default("graph.interactive", False)
|
13
25
|
|
14
26
|
|
15
27
|
class PcapGraph(PcapGraphData):
|
@@ -19,46 +31,37 @@ class PcapGraph(PcapGraphData):
|
|
19
31
|
self,
|
20
32
|
pcap_files: str,
|
21
33
|
output_file: str,
|
22
|
-
|
23
|
-
minimum_count: int | None = None,
|
24
|
-
bin_size: int | None = None,
|
25
|
-
match_string: str | None = None,
|
26
|
-
match_value: str | None = None,
|
27
|
-
cache_pcap_results: bool = False,
|
28
|
-
dissector_level: PCAPDissectorLevel = PCAPDissectorLevel.COUNT_ONLY,
|
29
|
-
interactive: bool = False,
|
30
|
-
ignore_list: List[str] | None = None,
|
31
|
-
by_percentage: bool = False,
|
32
|
-
pcap_filter: str | None = None,
|
33
|
-
cache_file_suffix: str = "taffy",
|
34
|
-
layers: List[str] | None = None,
|
35
|
-
force_overwrite: bool = False,
|
36
|
-
force_load: bool = False,
|
37
|
-
merge_files: bool = False, # unused
|
34
|
+
config: TaffyConfig(),
|
38
35
|
):
|
39
36
|
"""Create an instance of a graphing object."""
|
37
|
+
self.config = config
|
38
|
+
|
39
|
+
dissector_config = config[TTD_CFG.KEY_DISSECTOR]
|
40
|
+
limitor_config = config[TTL_CFG.KEY_LIMITOR]
|
41
|
+
graph_config = config[TTG_CFG.KEY_GRAPH]
|
42
|
+
super().__init__(
|
43
|
+
match_string=limitor_config[TTL_CFG.MATCH_STRING],
|
44
|
+
match_value=limitor_config[TTL_CFG.MATCH_VALUE],
|
45
|
+
minimum_count=limitor_config[TTL_CFG.MINIMUM_COUNT],
|
46
|
+
match_expression=limitor_config[TTL_CFG.MATCH_EXPRESSION],
|
47
|
+
)
|
48
|
+
|
40
49
|
self.pcap_files = pcap_files
|
41
50
|
self.output_file = output_file
|
42
|
-
self.maximum_count =
|
43
|
-
self.
|
44
|
-
self.
|
45
|
-
self.
|
46
|
-
self.
|
47
|
-
self.
|
48
|
-
self.
|
49
|
-
self.
|
50
|
-
self.
|
51
|
-
self.
|
52
|
-
self.
|
53
|
-
|
54
|
-
self.
|
55
|
-
self.
|
56
|
-
self.layers = layers
|
57
|
-
self.force_overwrite = force_overwrite
|
58
|
-
self.force_load = force_load
|
59
|
-
self.merge_files = merge_files
|
60
|
-
|
61
|
-
super().__init__()
|
51
|
+
self.maximum_count = dissector_config[TTD_CFG.PACKET_COUNT]
|
52
|
+
self.bin_size = dissector_config[TTD_CFG.BIN_SIZE]
|
53
|
+
self.pcap_filter = dissector_config[TTD_CFG.FILTER]
|
54
|
+
self.cache_pcap_results = dissector_config[TTD_CFG.CACHE_PCAP_RESULTS]
|
55
|
+
self.dissector_level = dissector_config[TTD_CFG.DISSECTION_LEVEL]
|
56
|
+
self.ignore_list = dissector_config[TTD_CFG.IGNORE_LIST]
|
57
|
+
self.cache_file_suffix = dissector_config[TTD_CFG.CACHE_FILE_SUFFIX]
|
58
|
+
self.layers = dissector_config[TTD_CFG.LAYERS]
|
59
|
+
self.force_overwrite = dissector_config[TTD_CFG.FORCE_OVERWRITE]
|
60
|
+
self.force_load = dissector_config[TTD_CFG.FORCE_LOAD]
|
61
|
+
self.merge_files = dissector_config[TTD_CFG.MERGE]
|
62
|
+
|
63
|
+
self.interactive = graph_config[TTG_CFG.INTERACTIVE]
|
64
|
+
self.by_percentage = graph_config[TTG_CFG.BY_PERCENTAGE]
|
62
65
|
|
63
66
|
def load_pcaps(self) -> None:
|
64
67
|
"""Load the pcap and counts things into bins."""
|
@@ -67,35 +70,33 @@ class PcapGraph(PcapGraphData):
|
|
67
70
|
info("reading pcap files")
|
68
71
|
pdm = PCAPDissectMany(
|
69
72
|
self.pcap_files,
|
70
|
-
|
71
|
-
maximum_count=self.maximum_count,
|
72
|
-
dissector_level=self.dissector_level,
|
73
|
-
pcap_filter=self.pcap_filter,
|
74
|
-
cache_results=self.cache_pcap_results,
|
75
|
-
ignore_list=self.ignore_list,
|
76
|
-
cache_file_suffix=self.cache_file_suffix,
|
77
|
-
layers=self.layers,
|
78
|
-
force_overwrite=self.force_overwrite,
|
79
|
-
force_load=self.force_load,
|
80
|
-
merge_files=self.merge_files,
|
73
|
+
self.config,
|
81
74
|
)
|
82
75
|
self.dissections = pdm.load_all()
|
83
76
|
info("done reading pcap files")
|
84
77
|
|
85
|
-
def create_graph(self) -> None:
|
78
|
+
def create_graph(self, options: dict | None = None) -> None:
|
86
79
|
"""Create the graph itself and save it."""
|
87
|
-
|
80
|
+
if not options:
|
81
|
+
options = {}
|
88
82
|
|
83
|
+
df = self.get_dataframe(merge=True, calculate_load_fraction=self.by_percentage)
|
89
84
|
hue_variable = "index"
|
90
85
|
if df[hue_variable].nunique() == 1:
|
91
86
|
hue_variable = None
|
92
87
|
|
93
88
|
if self.by_percentage:
|
94
|
-
df["load_fraction"]
|
95
89
|
y_column = "load_fraction"
|
96
90
|
else:
|
97
91
|
y_column = "count"
|
98
92
|
|
93
|
+
# TODO(hardaker): support re-indexing and hole filling (this doesn't work)
|
94
|
+
# str(self.bin_size or 1) + "s"
|
95
|
+
# df = df.set_index("time")
|
96
|
+
# df.index = df.index.to_period(freq=freq)
|
97
|
+
# timeindex = pd.period_range(min(df.index), max(df.index), freq=freq)
|
98
|
+
# df = df.reindex(timeindex) # , fill_value=0
|
99
|
+
|
99
100
|
ax = sns.relplot(
|
100
101
|
data=df,
|
101
102
|
kind="line",
|
@@ -104,7 +105,7 @@ class PcapGraph(PcapGraphData):
|
|
104
105
|
hue=hue_variable,
|
105
106
|
aspect=1.77,
|
106
107
|
)
|
107
|
-
ax.set(xlabel="time", ylabel=y_column)
|
108
|
+
ax.set(xlabel="time", ylabel=options.get("ylabel", y_column))
|
108
109
|
plt.xticks(rotation=45)
|
109
110
|
|
110
111
|
info(f"saving graph to {self.output_file}")
|