traffic-taffy 0.3.6__py3-none-any.whl → 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- traffic_taffy/cache_info.py +0 -6
- traffic_taffy/compare.py +154 -250
- traffic_taffy/comparison.py +26 -0
- traffic_taffy/dissection.py +383 -0
- traffic_taffy/dissectmany.py +20 -18
- traffic_taffy/dissector.py +128 -476
- traffic_taffy/dissector_engine/__init__.py +35 -0
- traffic_taffy/dissector_engine/dpkt.py +98 -0
- traffic_taffy/dissector_engine/scapy.py +98 -0
- traffic_taffy/graph.py +23 -90
- traffic_taffy/graphdata.py +35 -20
- traffic_taffy/output/__init__.py +118 -0
- traffic_taffy/output/console.py +72 -0
- traffic_taffy/output/fsdb.py +50 -0
- traffic_taffy/output/memory.py +51 -0
- traffic_taffy/pcap_splitter.py +17 -36
- traffic_taffy/tools/cache_info.py +65 -0
- traffic_taffy/tools/compare.py +110 -0
- traffic_taffy/tools/dissect.py +77 -0
- traffic_taffy/tools/explore.py +686 -0
- traffic_taffy/tools/graph.py +85 -0
- {traffic_taffy-0.3.6.dist-info → traffic_taffy-0.4.1.dist-info}/METADATA +1 -1
- traffic_taffy-0.4.1.dist-info/RECORD +29 -0
- traffic_taffy-0.4.1.dist-info/entry_points.txt +6 -0
- pcap_compare/cache_info.py +0 -46
- pcap_compare/compare.py +0 -288
- pcap_compare/dissectmany.py +0 -21
- pcap_compare/dissector.py +0 -512
- pcap_compare/dissectorresults.py +0 -21
- pcap_compare/graph.py +0 -210
- traffic_taffy/explore.py +0 -221
- traffic_taffy-0.3.6.dist-info/RECORD +0 -22
- traffic_taffy-0.3.6.dist-info/entry_points.txt +0 -5
- {pcap_compare → traffic_taffy/tools}/__init__.py +0 -0
- {traffic_taffy-0.3.6.dist-info → traffic_taffy-0.4.1.dist-info}/WHEEL +0 -0
- {traffic_taffy-0.3.6.dist-info → traffic_taffy-0.4.1.dist-info}/top_level.txt +0 -0
traffic_taffy/dissector.py
CHANGED
@@ -1,450 +1,90 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
import os
|
4
|
-
import pickle
|
5
|
-
import ipaddress
|
6
|
-
from enum import Enum
|
7
|
-
from logging import warning, info, error, debug
|
1
|
+
import sys
|
2
|
+
from logging import warning, error
|
8
3
|
from collections import Counter, defaultdict
|
9
|
-
from
|
10
|
-
from typing import Any, List
|
11
|
-
import dpkt
|
4
|
+
from typing import List
|
12
5
|
from rich import print
|
13
|
-
from
|
14
|
-
|
15
|
-
|
16
|
-
class PCAPDissectorType(Enum):
|
17
|
-
COUNT_ONLY = 1
|
18
|
-
THROUGH_IP = 2
|
19
|
-
DETAILED = 10
|
6
|
+
from traffic_taffy.dissection import PCAPDissectorLevel, Dissection
|
20
7
|
|
21
8
|
|
22
9
|
class PCAPDissector:
|
23
10
|
"loads a pcap file and counts the contents in both time and depth"
|
24
11
|
|
25
|
-
TOTAL_COUNT: str = "__TOTAL__"
|
26
|
-
TOTAL_SUBKEY: str = "packet"
|
27
|
-
WIDTH_SUBKEY: str = "__WIDTH__"
|
28
|
-
DISSECTION_KEY: str = "PCAP_DISSECTION_VERSION"
|
29
|
-
DISSECTION_VERSION: int = 4
|
30
|
-
|
31
|
-
def print_mac_address(value):
|
32
|
-
"Converts bytes to ethernet mac style address"
|
33
|
-
|
34
|
-
# TODO: certainly inefficient
|
35
|
-
def two_hex(value):
|
36
|
-
return f"{value:02x}"
|
37
|
-
|
38
|
-
return ":".join(map(two_hex, value))
|
39
|
-
|
40
|
-
display_transformers = {
|
41
|
-
"Ethernet.IP.src": ipaddress.ip_address,
|
42
|
-
"Ethernet.IP.dst": ipaddress.ip_address,
|
43
|
-
"Ethernet.IP6.src": ipaddress.ip_address,
|
44
|
-
"Ethernet.IP6.dst": ipaddress.ip_address,
|
45
|
-
"Ethernet.src": print_mac_address,
|
46
|
-
"Ethernet.dst": print_mac_address,
|
47
|
-
}
|
48
|
-
|
49
12
|
def __init__(
|
50
13
|
self,
|
51
14
|
pcap_file: str,
|
52
15
|
bin_size: int = 0,
|
53
16
|
maximum_count: int = 0,
|
54
|
-
dissector_level:
|
17
|
+
dissector_level: PCAPDissectorLevel = PCAPDissectorLevel.DETAILED,
|
55
18
|
pcap_filter: str | None = None,
|
56
19
|
cache_results: bool = False,
|
20
|
+
cache_file_suffix: str = "taffy",
|
21
|
+
ignore_list: list = [],
|
57
22
|
):
|
58
23
|
self.pcap_file = pcap_file
|
59
|
-
self.bin_size = bin_size
|
60
24
|
self.dissector_level = dissector_level
|
61
25
|
self.pcap_filter = pcap_filter
|
62
26
|
self.maximum_count = maximum_count
|
63
27
|
self.cache_results = cache_results
|
28
|
+
self.bin_size = bin_size
|
29
|
+
if cache_file_suffix[0] != ".":
|
30
|
+
cache_file_suffix = "." + cache_file_suffix
|
31
|
+
self.cache_file_suffix = cache_file_suffix
|
32
|
+
self.ignore_list = ignore_list
|
64
33
|
|
65
|
-
|
66
|
-
"pcap_file",
|
67
|
-
"bin_size",
|
68
|
-
"dissector_level",
|
69
|
-
"pcap_filter",
|
70
|
-
"maximum_count",
|
71
|
-
]
|
72
|
-
|
73
|
-
self.settable_from_cache = ["bin_size", "dissector_level", "maximum_count"]
|
74
|
-
|
75
|
-
# TODO: convert to a factory
|
76
|
-
self.data = {0: defaultdict(Counter)}
|
77
|
-
|
78
|
-
if dissector_level == PCAPDissectorType.COUNT_ONLY and bin_size == 0:
|
34
|
+
if dissector_level == PCAPDissectorLevel.COUNT_ONLY and bin_size == 0:
|
79
35
|
warning("counting packets only with no binning is unlikely to be helpful")
|
80
36
|
|
81
37
|
@property
|
82
|
-
def
|
83
|
-
return self.
|
84
|
-
|
85
|
-
@
|
86
|
-
def
|
87
|
-
self.
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
if not timestamps:
|
99
|
-
timestamps = data.keys()
|
100
|
-
for timestamp in timestamps:
|
101
|
-
for key in sorted(data[timestamp]):
|
102
|
-
if match_string and match_string not in key:
|
103
|
-
continue
|
104
|
-
|
105
|
-
for subkey, count in sorted(
|
106
|
-
data[timestamp][key].items(), key=lambda x: x[1], reverse=True
|
107
|
-
):
|
108
|
-
if minimum_count and abs(count) < minimum_count:
|
109
|
-
continue
|
110
|
-
|
111
|
-
if make_printable:
|
112
|
-
subkey = PCAPDissector.make_printable(key, subkey)
|
113
|
-
count = PCAPDissector.make_printable(None, count)
|
114
|
-
|
115
|
-
if match_value and match_value not in subkey:
|
116
|
-
continue
|
117
|
-
|
118
|
-
yield (timestamp, key, subkey, count)
|
119
|
-
|
120
|
-
@staticmethod
|
121
|
-
def calculate_metadata(data):
|
122
|
-
"Calculates things like the number of value entries within each key/subkey"
|
123
|
-
# TODO: do we do this with or without key and value matches?
|
124
|
-
for timestamp in data.keys():
|
125
|
-
for key in data[timestamp]:
|
126
|
-
if PCAPDissector.WIDTH_SUBKEY in data[timestamp][key]:
|
127
|
-
# make sure to avoid counting itself
|
128
|
-
del data[timestamp][key][PCAPDissector.WIDTH_SUBKEY]
|
129
|
-
data[timestamp][key][PCAPDissector.WIDTH_SUBKEY] = len(
|
130
|
-
data[timestamp][key]
|
131
|
-
)
|
132
|
-
|
133
|
-
def incr(self, key: str, value: Any, count: int = 1):
|
134
|
-
# always save a total count at the zero bin
|
135
|
-
# note: there should be no recorded tcpdump files from 1970 Jan 01 :-)
|
136
|
-
self.data[0][key][value] += count
|
137
|
-
if self.timestamp:
|
138
|
-
if self.timestamp not in self.data:
|
139
|
-
self.data[self.timestamp] = defaultdict(Counter)
|
140
|
-
self.data[self.timestamp][key][value] += count
|
141
|
-
|
142
|
-
def load_from_cache(self) -> dict | None:
|
143
|
-
if not self.pcap_file or not isinstance(self.pcap_file, str):
|
144
|
-
return None
|
145
|
-
if not (self.cache_results and os.path.exists(self.pcap_file + ".pkl")):
|
146
|
-
return None
|
147
|
-
|
148
|
-
cached_file = self.pcap_file + ".pkl"
|
149
|
-
cached_contents = self.load_saved(cached_file, dont_overwrite=True)
|
150
|
-
|
151
|
-
ok_to_load = True
|
152
|
-
|
153
|
-
if cached_contents[self.DISSECTION_KEY] != self.DISSECTION_VERSION:
|
154
|
-
debug(
|
155
|
-
"dissection cache version ({cached_contents[self.DISSECTION_KEY]}) differs from code version {self.DISSECTION_VERSION}"
|
156
|
-
)
|
157
|
-
ok_to_load = False
|
158
|
-
|
159
|
-
# a zero really is a 1 since bin(0) still does int(timestamp)
|
160
|
-
if (
|
161
|
-
cached_contents["parameters"]["bin_size"] == 0
|
162
|
-
or cached_contents["parameters"]["bin_size"] is None
|
163
|
-
):
|
164
|
-
cached_contents["parameters"]["bin_size"] = 1
|
165
|
-
|
166
|
-
for parameter in self.parameters:
|
167
|
-
specified = getattr(self, parameter)
|
168
|
-
cached = cached_contents["parameters"][parameter]
|
169
|
-
|
170
|
-
if not specified and parameter in self.settable_from_cache:
|
171
|
-
# inherit from the cache
|
172
|
-
setattr(self, parameter, cached)
|
173
|
-
continue
|
174
|
-
|
175
|
-
if specified and specified != cached:
|
176
|
-
# special checks for certain types of parameters:
|
177
|
-
|
178
|
-
if parameter == "dissector_level":
|
179
|
-
debug("------------ here 1")
|
180
|
-
if parameter == "dissector_level" and specified <= cached:
|
181
|
-
debug(f"here with dissector_level {specified} and {cached}")
|
182
|
-
# loading a more detailed cache is ok
|
183
|
-
continue
|
184
|
-
|
185
|
-
if parameter == "pcap_file" and os.path.basename(
|
186
|
-
specified
|
187
|
-
) == os.path.basename(cached):
|
188
|
-
# as long as the basename is ok, we'll assume it's a different path
|
189
|
-
# TODO: only store basename?
|
190
|
-
continue
|
191
|
-
|
192
|
-
debug(
|
193
|
-
f"parameter {parameter} doesn't match: specified={specified} != cached={cached}"
|
194
|
-
)
|
195
|
-
ok_to_load = False
|
196
|
-
|
197
|
-
if ok_to_load:
|
198
|
-
info(f"loading cached pcap contents from {cached_file}")
|
199
|
-
self.load_saved_contents(cached_contents)
|
200
|
-
return self.data
|
201
|
-
|
202
|
-
error(f"Failed to load cached data for {self.pcap_file} due to differences")
|
203
|
-
error("refusing to continue -- remove the cache to recreate it")
|
204
|
-
raise ValueError(
|
205
|
-
"INCOMPATIBLE CACHE: remove the cache or don't use it to continue"
|
38
|
+
def dissection(self):
|
39
|
+
return self._dissection
|
40
|
+
|
41
|
+
@dissection.setter
|
42
|
+
def dissection(self, new_dissection):
|
43
|
+
self._dissection = new_dissection
|
44
|
+
|
45
|
+
def dissection_args(self):
|
46
|
+
return (
|
47
|
+
self.pcap_file,
|
48
|
+
self.pcap_filter,
|
49
|
+
self.maximum_count,
|
50
|
+
self.bin_size,
|
51
|
+
self.dissector_level,
|
52
|
+
self.cache_file_suffix,
|
53
|
+
set(self.ignore_list),
|
206
54
|
)
|
207
55
|
|
208
|
-
def
|
56
|
+
def load_from_cache(self, force: bool = False):
|
57
|
+
if self.cache_results:
|
58
|
+
args = self.dissection_args()
|
59
|
+
self.dissection = Dissection(*args)
|
60
|
+
cached_data = self.dissection.load_from_cache(force=force)
|
61
|
+
if cached_data:
|
62
|
+
return cached_data
|
63
|
+
|
64
|
+
def load(self, force: bool = False) -> dict:
|
209
65
|
"Loads data from a pcap file or its cached results"
|
210
|
-
cached_data = self.load_from_cache()
|
66
|
+
cached_data = self.load_from_cache(force=force)
|
211
67
|
if cached_data:
|
212
68
|
return cached_data
|
213
69
|
|
70
|
+
engine = None
|
71
|
+
args = self.dissection_args()
|
214
72
|
if (
|
215
|
-
self.dissector_level ==
|
216
|
-
or self.dissector_level ==
|
73
|
+
self.dissector_level == PCAPDissectorLevel.DETAILED
|
74
|
+
or self.dissector_level == PCAPDissectorLevel.DETAILED.value
|
217
75
|
):
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
def dpkt_callback(self, timestamp: float, packet: bytes):
|
223
|
-
# if binning is requested, save it in a binned time slot
|
224
|
-
self.timestamp = int(timestamp)
|
225
|
-
if self.bin_size:
|
226
|
-
self.timestamp = self.timestamp - self.timestamp % self.bin_size
|
227
|
-
self.incr(self.TOTAL_COUNT, self.TOTAL_SUBKEY)
|
228
|
-
|
229
|
-
if self.dissector_level.value >= PCAPDissectorType.THROUGH_IP.value:
|
230
|
-
eth = dpkt.ethernet.Ethernet(packet)
|
231
|
-
# these names are designed to match scapy names
|
232
|
-
self.incr("Ethernet.dst", eth.dst)
|
233
|
-
self.incr("Ethernet.src", eth.src)
|
234
|
-
self.incr("Ethernet.type", eth.type)
|
235
|
-
|
236
|
-
if isinstance(eth.data, dpkt.ip.IP):
|
237
|
-
ip = eth.data
|
238
|
-
|
239
|
-
IPVER = "IP"
|
240
|
-
if ip.v == 6:
|
241
|
-
IPVER = "IPv6"
|
242
|
-
|
243
|
-
# TODO: make sure all these match scapy
|
244
|
-
self.incr(f"Ethernet.{IPVER}.dst", ip.dst)
|
245
|
-
self.incr(f"Ethernet.{IPVER}.src", ip.src)
|
246
|
-
self.incr(f"Ethernet.{IPVER}.df", ip.df)
|
247
|
-
self.incr(f"Ethernet.{IPVER}.offset", ip.offset)
|
248
|
-
self.incr(f"Ethernet.{IPVER}.tos", ip.tos)
|
249
|
-
self.incr(f"Ethernet.{IPVER}.len", ip.len)
|
250
|
-
self.incr(f"Ethernet.{IPVER}.id", ip.id)
|
251
|
-
self.incr(f"Ethernet.{IPVER}.hl", ip.hl)
|
252
|
-
self.incr(f"Ethernet.{IPVER}.rf", ip.rf)
|
253
|
-
self.incr(f"Ethernet.{IPVER}.p", ip.p)
|
254
|
-
self.incr(f"Ethernet.{IPVER}.chksum", ip.sum)
|
255
|
-
self.incr(f"Ethernet.{IPVER}.tos", ip.tos)
|
256
|
-
self.incr(f"Ethernet.{IPVER}.version", ip.v)
|
257
|
-
self.incr(f"Ethernet.{IPVER}.ttl", ip.ttl)
|
258
|
-
|
259
|
-
if isinstance(ip.data, dpkt.udp.UDP):
|
260
|
-
udp = ip.data
|
261
|
-
self.incr(f"Ethernet.{IPVER}.UDP.sport", udp.sport)
|
262
|
-
self.incr(f"Ethernet.{IPVER}.UDP.dport", udp.dport)
|
263
|
-
self.incr(f"Ethernet.{IPVER}.UDP.len", udp.ulen)
|
264
|
-
self.incr(f"Ethernet.{IPVER}.UDP.chksum", udp.sum)
|
265
|
-
|
266
|
-
# TODO: handle DNS and others for level 3
|
267
|
-
|
268
|
-
elif isinstance(ip.data, dpkt.tcp.TCP):
|
269
|
-
# TODO
|
270
|
-
tcp = ip.data
|
271
|
-
self.incr(f"Ethernet.{IPVER}.TCP.sport", tcp.sport)
|
272
|
-
self.incr(f"Ethernet.{IPVER}.TCP.dport", tcp.dport)
|
273
|
-
self.incr(f"Ethernet.{IPVER}.TCP.seq", tcp.seq)
|
274
|
-
self.incr(f"Ethernet.{IPVER}.TCP.flags", tcp.flags)
|
275
|
-
# self.incr(f"Ethernet.{IPVER}.TCP.reserved", tcp.reserved)
|
276
|
-
self.incr(f"Ethernet.{IPVER}.TCP.window", tcp.win)
|
277
|
-
self.incr(f"Ethernet.{IPVER}.TCP.chksum", tcp.sum)
|
278
|
-
self.incr(f"Ethernet.{IPVER}.TCP.options", tcp.opts)
|
279
|
-
|
280
|
-
# TODO: handle DNS and others for level 3
|
281
|
-
|
282
|
-
def load_via_dpkt(self) -> dict:
|
283
|
-
self.data = {0: defaultdict(Counter)}
|
284
|
-
if isinstance(self.pcap_file, str):
|
285
|
-
pcap = dpkt.pcap.Reader(pcapp.open_maybe_compressed(self.pcap_file))
|
286
|
-
else:
|
287
|
-
# it's an open handle already
|
288
|
-
pcap = dpkt.pcap.Reader(self.pcap_file)
|
289
|
-
if self.pcap_filter:
|
290
|
-
pcap.setfilter(self.pcap_filter)
|
291
|
-
pcap.dispatch(self.maximum_count, self.dpkt_callback)
|
292
|
-
|
293
|
-
self.calculate_metadata(self.data)
|
294
|
-
self.save_to_cache()
|
295
|
-
return self.data
|
296
|
-
|
297
|
-
def add_scapy_item(self, field_value, prefix: str) -> None:
|
298
|
-
"Adds an item to the self.data regardless of it's various types"
|
299
|
-
if isinstance(field_value, list):
|
300
|
-
if len(field_value) > 0:
|
301
|
-
# if it's a list of tuples, count the (eg TCP option) names
|
302
|
-
# TODO: values can be always the same or things like timestamps
|
303
|
-
# that will always change or are too unique
|
304
|
-
if isinstance(field_value[0], tuple):
|
305
|
-
for item in field_value:
|
306
|
-
self.incr(prefix, item[0])
|
307
|
-
else:
|
308
|
-
for item in field_value:
|
309
|
-
self.add_scapy_item(item, prefix)
|
310
|
-
# else:
|
311
|
-
# debug(f"ignoring empty-list: {field_value}")
|
312
|
-
elif (
|
313
|
-
isinstance(field_value, str)
|
314
|
-
or isinstance(field_value, int)
|
315
|
-
or isinstance(field_value, float)
|
316
|
-
):
|
317
|
-
self.incr(prefix, field_value)
|
318
|
-
|
319
|
-
elif isinstance(field_value, bytes):
|
320
|
-
try:
|
321
|
-
converted = field_value.decode("utf-8")
|
322
|
-
self.incr(prefix, converted)
|
323
|
-
except Exception:
|
324
|
-
converted = "0x" + field_value.hex()
|
325
|
-
self.incr(prefix, converted)
|
326
|
-
|
327
|
-
def add_scapy_layer(self, layer, prefix: str | None = "") -> None:
|
328
|
-
"Analyzes a layer to add counts to each layer sub-component"
|
329
|
-
|
330
|
-
if hasattr(layer, "fields_desc"):
|
331
|
-
name_list = [field.name for field in layer.fields_desc]
|
332
|
-
elif hasattr(layer, "fields"):
|
333
|
-
name_list = [field.name for field in layer.fields]
|
76
|
+
from traffic_taffy.dissector_engine.scapy import DissectionEngineScapy
|
77
|
+
|
78
|
+
engine = DissectionEngineScapy(*args)
|
334
79
|
else:
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
else:
|
344
|
-
self.add_scapy_item(field_value, prefix + field_name)
|
345
|
-
except Exception:
|
346
|
-
warning(f"scapy error at '{prefix}' in field '{field_name}'")
|
347
|
-
|
348
|
-
def scapy_callback(self, packet):
|
349
|
-
prefix = "."
|
350
|
-
self.timestamp = int(packet.time)
|
351
|
-
if self.bin_size:
|
352
|
-
self.timestamp = self.timestamp - self.timestamp % self.bin_size
|
353
|
-
|
354
|
-
self.incr(self.TOTAL_COUNT, self.TOTAL_SUBKEY)
|
355
|
-
for payload in packet.iterpayloads():
|
356
|
-
prefix = f"{prefix}{payload.name}."
|
357
|
-
self.add_scapy_layer(payload, prefix[1:])
|
358
|
-
|
359
|
-
def load_via_scapy(self) -> dict:
|
360
|
-
"Loads a pcap file into a nested dictionary of statistical counts"
|
361
|
-
load_this = self.pcap_file
|
362
|
-
if isinstance(self.pcap_file, str):
|
363
|
-
load_this = pcapp.open_maybe_compressed(self.pcap_file)
|
364
|
-
sniff(
|
365
|
-
offline=load_this,
|
366
|
-
prn=self.scapy_callback,
|
367
|
-
store=0,
|
368
|
-
count=self.maximum_count,
|
369
|
-
filter=self.pcap_filter,
|
370
|
-
)
|
371
|
-
self.calculate_metadata(self.data)
|
372
|
-
self.save_to_cache()
|
373
|
-
return self.data
|
374
|
-
|
375
|
-
def save_to_cache(self):
|
376
|
-
if self.pcap_file and isinstance(self.pcap_file, str) and self.cache_results:
|
377
|
-
self.save(self.pcap_file + ".pkl")
|
378
|
-
|
379
|
-
def save(self, where: str) -> None:
|
380
|
-
"Saves a generated dissection to a pickle file"
|
381
|
-
|
382
|
-
# wrap the report in a version header
|
383
|
-
versioned_cache = {
|
384
|
-
self.DISSECTION_KEY: self.DISSECTION_VERSION,
|
385
|
-
"file": self.pcap_file,
|
386
|
-
"parameters": {},
|
387
|
-
"dissection": self.data,
|
388
|
-
}
|
389
|
-
|
390
|
-
for parameter in self.parameters:
|
391
|
-
versioned_cache["parameters"][parameter] = getattr(self, parameter)
|
392
|
-
# TODO: fix this hack
|
393
|
-
|
394
|
-
# basically, bin_size of 0 is 1... but it may be faster
|
395
|
-
# to leave it at zero to avoid the bin_size math of 1,
|
396
|
-
# which is actually a math noop that will still consume
|
397
|
-
# cycles. We save it as 1 though since the math is past
|
398
|
-
# us and a 1 value is more informative to the user.
|
399
|
-
if parameter == "bin_size" and self.bin_size == 0:
|
400
|
-
versioned_cache["parameters"][parameter] = 1
|
401
|
-
|
402
|
-
# save it
|
403
|
-
info(f"caching PCAP data to '{where}'")
|
404
|
-
pickle.dump(versioned_cache, open(where, "wb"))
|
405
|
-
|
406
|
-
def load_saved_contents(self, versioned_cache):
|
407
|
-
# set the local parameters from the cache
|
408
|
-
for parameter in self.parameters:
|
409
|
-
setattr(self, parameter, versioned_cache["parameters"][parameter])
|
410
|
-
|
411
|
-
# load the data
|
412
|
-
self.data = versioned_cache["dissection"]
|
413
|
-
|
414
|
-
def load_saved(self, where: str, dont_overwrite: bool = False) -> dict:
|
415
|
-
"Loads a previous saved report from a file instead of re-parsing pcaps"
|
416
|
-
contents = pickle.load(open(where, "rb"))
|
417
|
-
|
418
|
-
# check that the version header matches something we understand
|
419
|
-
if contents["PCAP_DISSECTION_VERSION"] != self.DISSECTION_VERSION:
|
420
|
-
raise ValueError(
|
421
|
-
"improper saved dissection version: report version = "
|
422
|
-
+ str(contents["PCAP_COMPARE_VERSION"])
|
423
|
-
+ ", our version: "
|
424
|
-
+ str(self.DISSECTION_VERSION)
|
425
|
-
)
|
426
|
-
|
427
|
-
if not dont_overwrite:
|
428
|
-
self.load_saved_contents(contents)
|
429
|
-
|
430
|
-
return contents
|
431
|
-
|
432
|
-
@staticmethod
|
433
|
-
def make_printable(value_type: str, value: Any) -> str:
|
434
|
-
try:
|
435
|
-
if isinstance(value, bytes):
|
436
|
-
if value_type in PCAPDissector.display_transformers:
|
437
|
-
value = str(PCAPDissector.display_transformers[value_type](value))
|
438
|
-
else:
|
439
|
-
value = "0x" + value.hex()
|
440
|
-
else:
|
441
|
-
value = str(value)
|
442
|
-
except Exception:
|
443
|
-
if isinstance(value, bytes):
|
444
|
-
value = "0x" + value.hex()
|
445
|
-
else:
|
446
|
-
value = "[unprintable]"
|
447
|
-
return value
|
80
|
+
from traffic_taffy.dissector_engine.dpkt import DissectionEngineDpkt
|
81
|
+
|
82
|
+
engine = DissectionEngineDpkt(*args)
|
83
|
+
|
84
|
+
self.dissection = engine.load()
|
85
|
+
if self.cache_results:
|
86
|
+
self.dissection.save_to_cache()
|
87
|
+
return self.dissection
|
448
88
|
|
449
89
|
def print(
|
450
90
|
self,
|
@@ -453,8 +93,7 @@ class PCAPDissector:
|
|
453
93
|
match_value: str | None = None,
|
454
94
|
minimum_count: int | None = None,
|
455
95
|
) -> None:
|
456
|
-
for timestamp, key, subkey, value in self.find_data(
|
457
|
-
self._data,
|
96
|
+
for timestamp, key, subkey, value in self.dissection.find_data(
|
458
97
|
timestamps=timestamps,
|
459
98
|
match_string=match_string,
|
460
99
|
match_value=match_value,
|
@@ -463,6 +102,30 @@ class PCAPDissector:
|
|
463
102
|
):
|
464
103
|
print(f"{key:<30} {subkey:<30} {value}")
|
465
104
|
|
105
|
+
def print_to_fsdb(
|
106
|
+
self,
|
107
|
+
timestamps: List[int] | None = [0],
|
108
|
+
match_string: str | None = None,
|
109
|
+
match_value: str | None = None,
|
110
|
+
minimum_count: int | None = None,
|
111
|
+
) -> None:
|
112
|
+
import pyfsdb
|
113
|
+
|
114
|
+
fh = pyfsdb.Fsdb(
|
115
|
+
out_file_handle=sys.stdout,
|
116
|
+
out_column_names=["key", "subkey", "value"],
|
117
|
+
converters={"value": int},
|
118
|
+
)
|
119
|
+
for timestamp, key, subkey, value in self.dissection.find_data(
|
120
|
+
timestamps=timestamps,
|
121
|
+
match_string=match_string,
|
122
|
+
match_value=match_value,
|
123
|
+
minimum_count=minimum_count,
|
124
|
+
make_printable=True,
|
125
|
+
):
|
126
|
+
fh.append([key, subkey, value])
|
127
|
+
fh.close()
|
128
|
+
|
466
129
|
|
467
130
|
def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
468
131
|
if add_subgroup:
|
@@ -471,15 +134,44 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
471
134
|
parser.add_argument(
|
472
135
|
"-d",
|
473
136
|
"--dissection-level",
|
474
|
-
default=
|
137
|
+
default=PCAPDissectorLevel.THROUGH_IP.value,
|
475
138
|
type=int,
|
476
139
|
help="Dump to various levels of detail (1-10, with 10 is the most detailed and slowest)",
|
477
140
|
)
|
478
141
|
|
142
|
+
parser.add_argument(
|
143
|
+
"-I",
|
144
|
+
"--ignore-list",
|
145
|
+
default=[
|
146
|
+
"Ethernet.IP.TCP.seq",
|
147
|
+
"Ethernet.IP.TCP.ack",
|
148
|
+
"Ethernet.IPv6.TCP.seq",
|
149
|
+
"Ethernet.IPv6.TCP.ack",
|
150
|
+
"Ethernet.IP.UDP.DNS.id",
|
151
|
+
"Ethernet.IP.TCP.DNS.id",
|
152
|
+
"Ethernet.IPv6.UDP.DNS.id",
|
153
|
+
"Ethernet.IPv6.TCP.DNS.id",
|
154
|
+
"Ethernet.IP.id",
|
155
|
+
"Ethernet.IP.chksum",
|
156
|
+
"Ethernet.IP.UDP.chksum",
|
157
|
+
"Ethernet.IP.TCP.chksum",
|
158
|
+
"Ethernet.IPv6.UDP.chksum" "Ethernet.IPv6.fl",
|
159
|
+
"Ethernet.IP.ICMP.chksum",
|
160
|
+
"Ethernet.IP.ICMP.id",
|
161
|
+
"Ethernet.IP.ICMP.seq",
|
162
|
+
"Ethernet.IP.TCP.Padding.load",
|
163
|
+
"Ethernet.IPv6.TCP.chksum",
|
164
|
+
"Ethernet.IPv6.plen",
|
165
|
+
],
|
166
|
+
nargs="*",
|
167
|
+
type=str,
|
168
|
+
help="A list of (unlikely to be useful) packet fields to ignore",
|
169
|
+
)
|
170
|
+
|
479
171
|
parser.add_argument(
|
480
172
|
"-n",
|
481
173
|
"--packet-count",
|
482
|
-
default
|
174
|
+
default=0,
|
483
175
|
type=int,
|
484
176
|
help="Maximum number of packets to analyze",
|
485
177
|
)
|
@@ -488,7 +180,6 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
488
180
|
"-b",
|
489
181
|
"--bin-size",
|
490
182
|
type=int,
|
491
|
-
default=3600,
|
492
183
|
help="Bin results into this many seconds",
|
493
184
|
)
|
494
185
|
|
@@ -496,7 +187,21 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
|
|
496
187
|
"-C",
|
497
188
|
"--cache-pcap-results",
|
498
189
|
action="store_true",
|
499
|
-
help="Cache and use PCAP results into/from a
|
190
|
+
help="Cache and use PCAP results into/from a cache file file",
|
191
|
+
)
|
192
|
+
|
193
|
+
parser.add_argument(
|
194
|
+
"--cache-file-suffix",
|
195
|
+
"--cs",
|
196
|
+
type=str,
|
197
|
+
default="taffy",
|
198
|
+
help="The suffix file to use when creating cache files",
|
199
|
+
)
|
200
|
+
|
201
|
+
parser.add_argument(
|
202
|
+
"--force",
|
203
|
+
action="store_true",
|
204
|
+
help="Force continuing with an incompatible cache (and rewriting it)",
|
500
205
|
)
|
501
206
|
|
502
207
|
return parser
|
@@ -535,9 +240,9 @@ def limitor_add_parseargs(parser, add_subgroup: bool = True):
|
|
535
240
|
|
536
241
|
def check_dissector_level(level: int):
|
537
242
|
current_dissection_levels = [
|
538
|
-
|
539
|
-
|
540
|
-
|
243
|
+
PCAPDissectorLevel.COUNT_ONLY.value,
|
244
|
+
PCAPDissectorLevel.THROUGH_IP.value,
|
245
|
+
PCAPDissectorLevel.DETAILED.value,
|
541
246
|
]
|
542
247
|
if level not in current_dissection_levels:
|
543
248
|
error(f"currently supported dissection levels: {current_dissection_levels}")
|
@@ -553,56 +258,3 @@ def pcap_data_merge(d1: dict, d2: dict):
|
|
553
258
|
d1[key] = defaultdict(Counter)
|
554
259
|
d1[key][subkey] += d2[key][subkey]
|
555
260
|
return d1
|
556
|
-
|
557
|
-
|
558
|
-
def main():
|
559
|
-
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
560
|
-
import logging
|
561
|
-
|
562
|
-
def parse_args():
|
563
|
-
"Parse the command line arguments."
|
564
|
-
parser = ArgumentParser(
|
565
|
-
formatter_class=ArgumentDefaultsHelpFormatter,
|
566
|
-
description=__doc__,
|
567
|
-
epilog="Exmaple Usage: ",
|
568
|
-
)
|
569
|
-
|
570
|
-
parser.add_argument(
|
571
|
-
"--log-level",
|
572
|
-
"--ll",
|
573
|
-
default="info",
|
574
|
-
help="Define the logging verbosity level (debug, info, warning, error, fotal, critical).",
|
575
|
-
)
|
576
|
-
|
577
|
-
dissector_add_parseargs(parser)
|
578
|
-
limitor_add_parseargs(parser)
|
579
|
-
|
580
|
-
parser.add_argument("input_file", type=str, help="input pcap file")
|
581
|
-
|
582
|
-
args = parser.parse_args()
|
583
|
-
log_level = args.log_level.upper()
|
584
|
-
logging.basicConfig(level=log_level, format="%(levelname)-10s:\t%(message)s")
|
585
|
-
return args
|
586
|
-
|
587
|
-
args = parse_args()
|
588
|
-
|
589
|
-
check_dissector_level(args.dissection_level)
|
590
|
-
|
591
|
-
pd = PCAPDissector(
|
592
|
-
args.input_file,
|
593
|
-
bin_size=args.bin_size,
|
594
|
-
dissector_level=args.dissection_level,
|
595
|
-
maximum_count=args.packet_count,
|
596
|
-
cache_results=args.cache_pcap_results,
|
597
|
-
)
|
598
|
-
pd.load()
|
599
|
-
pd.print(
|
600
|
-
timestamps=[0],
|
601
|
-
match_string=args.match_string,
|
602
|
-
match_value=args.match_value,
|
603
|
-
minimum_count=args.minimum_count,
|
604
|
-
)
|
605
|
-
|
606
|
-
|
607
|
-
if __name__ == "__main__":
|
608
|
-
main()
|