traffic-taffy 0.5.8__py3-none-any.whl → 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- traffic_taffy/__init__.py +1 -1
- traffic_taffy/compare.py +86 -54
- traffic_taffy/comparison.py +13 -5
- traffic_taffy/dissection.py +31 -17
- traffic_taffy/dissectmany.py +52 -27
- traffic_taffy/dissector.py +22 -1
- traffic_taffy/dissector_engine/__init__.py +24 -1
- traffic_taffy/dissector_engine/dnstap.py +121 -0
- traffic_taffy/dissector_engine/dpkt.py +207 -41
- traffic_taffy/dissector_engine/scapy.py +27 -32
- traffic_taffy/graph.py +22 -13
- traffic_taffy/output/__init__.py +36 -20
- traffic_taffy/output/console.py +32 -17
- traffic_taffy/output/fsdb.py +17 -12
- traffic_taffy/output/memory.py +23 -16
- traffic_taffy/tests/test_compare_results.py +37 -37
- traffic_taffy/tests/test_dpkt_engine.py +15 -0
- traffic_taffy/tests/test_pcap_dissector.py +4 -4
- traffic_taffy/tests/test_pcap_splitter.py +0 -1
- traffic_taffy/tools/cache_info.py +10 -7
- traffic_taffy/tools/compare.py +10 -16
- traffic_taffy/tools/dissect.py +8 -6
- traffic_taffy/tools/explore.py +20 -46
- traffic_taffy/tools/graph.py +6 -5
- {traffic_taffy-0.5.8.dist-info → traffic_taffy-0.6.1.dist-info}/METADATA +11 -2
- traffic_taffy-0.6.1.dist-info/RECORD +35 -0
- traffic_taffy-0.6.1.dist-info/licenses/LICENSE.txt +204 -0
- traffic_taffy/dissectorresults.py +0 -21
- traffic_taffy/tests/test_result_storage.py +0 -16
- traffic_taffy-0.5.8.dist-info/RECORD +0 -34
- {traffic_taffy-0.5.8.dist-info → traffic_taffy-0.6.1.dist-info}/WHEEL +0 -0
- {traffic_taffy-0.5.8.dist-info → traffic_taffy-0.6.1.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,121 @@
|
|
1
|
+
"""A dissection engine for quickly parsing and counting packets."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from logging import error, debug
|
6
|
+
from traffic_taffy.dissector_engine.dpkt import DissectionEngineDpkt
|
7
|
+
from traffic_taffy.dissection import Dissection, PCAPDissectorLevel
|
8
|
+
|
9
|
+
import dnstap_pb
|
10
|
+
import fstrm
|
11
|
+
|
12
|
+
|
13
|
+
class DissectionEngineDNStap(DissectionEngineDpkt):
|
14
|
+
"""A dissection engine for parsing saved dnscap files."""
|
15
|
+
|
16
|
+
# should be larger than any potentially stored DNS message
|
17
|
+
READ_SIZE = 8192
|
18
|
+
|
19
|
+
def __init__(self, *args: list, **kwargs: dict):
|
20
|
+
"""Create a dissection engine for parsing dnscap files."""
|
21
|
+
super().__init__(*args, **kwargs)
|
22
|
+
|
23
|
+
def load_data(self) -> Dissection:
|
24
|
+
"""loads the dnstap file into memory."""
|
25
|
+
|
26
|
+
# technically a dnstap file, not a pcap_file
|
27
|
+
with open(self.pcap_file, "rb") as fh:
|
28
|
+
data = fh.read(self.READ_SIZE)
|
29
|
+
|
30
|
+
# create the framing stream decoded
|
31
|
+
stream = fstrm.FstrmCodec()
|
32
|
+
success = stream.append_and_process(data)
|
33
|
+
if not success:
|
34
|
+
error("failed to read {self.pcap_file} as a fstrm")
|
35
|
+
raise ValueError("failed to read input file")
|
36
|
+
|
37
|
+
# the base header is just extra data about the file's producer
|
38
|
+
# header =
|
39
|
+
header = stream.decode()
|
40
|
+
debug(f"header: {header[1]}")
|
41
|
+
# read more data (see comment below)
|
42
|
+
stream.append(fh.read(self.READ_SIZE - len(stream.buf)))
|
43
|
+
|
44
|
+
# Create the dnstap protobuf parser
|
45
|
+
dd = dnstap_pb.Dnstap()
|
46
|
+
|
47
|
+
dissection = self.dissection
|
48
|
+
level = self.dissector_level
|
49
|
+
count = 0
|
50
|
+
|
51
|
+
# loop through the stream and process each subsequent frame
|
52
|
+
while stream.process():
|
53
|
+
# pull the next frame out
|
54
|
+
body = stream.decode()
|
55
|
+
|
56
|
+
# parse it as a dnstap protobuf object
|
57
|
+
dd.ParseFromString(body[2])
|
58
|
+
message = dd.message
|
59
|
+
|
60
|
+
self.start_packet(message.query_time_sec)
|
61
|
+
|
62
|
+
# keep the buffer at the required size.
|
63
|
+
stream.append(fh.read(self.READ_SIZE - len(stream.buf)))
|
64
|
+
|
65
|
+
# determine if it's IPv4 or IPv6
|
66
|
+
if message.socket_family == 1:
|
67
|
+
IP = "IP"
|
68
|
+
self.incr(dissection, "Ethernet_type", 2048)
|
69
|
+
self.incr(dissection, "Ethernet_IP_version", 4)
|
70
|
+
elif message.socket_family == 2:
|
71
|
+
IP = "IPv6"
|
72
|
+
self.incr(dissection, "Ethernet_type", 34525)
|
73
|
+
self.incr(dissection, "Ethernet_IP_version", 6)
|
74
|
+
else:
|
75
|
+
raise ValueError("unknown IP protocol in dnstap")
|
76
|
+
prefix = "Ethernet_" + IP + "_"
|
77
|
+
|
78
|
+
# set the source/dest addresses
|
79
|
+
self.incr(dissection, prefix + "src", message.query_address)
|
80
|
+
self.incr(dissection, prefix + "dst", message.response_address)
|
81
|
+
|
82
|
+
# Determine the transport protocol
|
83
|
+
# TODO(hardaker): read these names from the protobuf spec directly
|
84
|
+
if message.socket_protocol == 1:
|
85
|
+
protocol_prefix = prefix + "UDP_"
|
86
|
+
elif message.socket_protocol == 2:
|
87
|
+
protocol_prefix = prefix + "TCP_"
|
88
|
+
elif message.socket_protocol == 3:
|
89
|
+
protocol_prefix = prefix + "DOT_"
|
90
|
+
elif message.socket_protocol == 4:
|
91
|
+
protocol_prefix = prefix + "DOH_"
|
92
|
+
elif message.socket_protocol == 5:
|
93
|
+
protocol_prefix = prefix + "DNSCryptUDP_"
|
94
|
+
elif message.socket_protocol == 6:
|
95
|
+
protocol_prefix = prefix + "DNSCryptTCP_"
|
96
|
+
elif message.socket_protocol == 7:
|
97
|
+
protocol_prefix = prefix + "DOQ_"
|
98
|
+
else:
|
99
|
+
raise ValueError("unknown DNS socket protocol in dnstap")
|
100
|
+
# TODO(hardaker): get full protocol list here
|
101
|
+
|
102
|
+
self.incr(dissection, protocol_prefix + "sport", message.query_port)
|
103
|
+
self.incr(dissection, protocol_prefix + "dport", message.response_port)
|
104
|
+
|
105
|
+
if message.type & 0x01 == 1: # query
|
106
|
+
self.incr(
|
107
|
+
dissection, protocol_prefix + "DNS_qr", 0
|
108
|
+
) # query = 0 in the protocol
|
109
|
+
else:
|
110
|
+
self.incr(
|
111
|
+
dissection, protocol_prefix + "DNS_qr", 1
|
112
|
+
) # response = 1 in the protocol
|
113
|
+
|
114
|
+
count += 1
|
115
|
+
if self.maximum_count and count >= self.maximum_count:
|
116
|
+
break
|
117
|
+
|
118
|
+
if level >= PCAPDissectorLevel.COMMON_LAYERS.value:
|
119
|
+
self.dissect_dns(message.query_message, protocol_prefix + "DNS_")
|
120
|
+
|
121
|
+
return dissection
|
@@ -1,18 +1,31 @@
|
|
1
|
+
"""A dissection engine for quickly parsing and counting packets."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from logging import debug
|
1
6
|
from traffic_taffy.dissector_engine import DissectionEngine
|
2
7
|
from traffic_taffy.dissection import Dissection, PCAPDissectorLevel
|
3
|
-
from pcap_parallel import PCAPParallel
|
8
|
+
from pcap_parallel import PCAPParallel
|
4
9
|
|
5
10
|
import dpkt
|
6
11
|
|
7
12
|
|
8
13
|
class DissectionEngineDpkt(DissectionEngine):
|
9
|
-
|
14
|
+
"""A dissection engine for quickly parsing and counting packets."""
|
15
|
+
|
16
|
+
DNS_PORT: int = 53
|
17
|
+
HTTP_PORT: int = 80
|
18
|
+
IPV6_VERSION: int = 6
|
19
|
+
|
20
|
+
def __init__(self, *args: list, **kwargs: dict):
|
21
|
+
"""Create a dissection engine for quickly parsing and counting packets."""
|
10
22
|
super().__init__(*args, **kwargs)
|
11
23
|
|
12
|
-
def
|
13
|
-
|
24
|
+
def load_data(self) -> None:
|
25
|
+
"""Load the specified PCAP into memory."""
|
26
|
+
# Note: called from self.load() after initializing
|
14
27
|
if isinstance(self.pcap_file, str):
|
15
|
-
pcap = dpkt.pcap.Reader(
|
28
|
+
pcap = dpkt.pcap.Reader(PCAPParallel.open_maybe_compressed(self.pcap_file))
|
16
29
|
else:
|
17
30
|
# it's an open handle already
|
18
31
|
pcap = dpkt.pcap.Reader(self.pcap_file)
|
@@ -20,45 +33,144 @@ class DissectionEngineDpkt(DissectionEngine):
|
|
20
33
|
pcap.setfilter(self.pcap_filter)
|
21
34
|
pcap.dispatch(self.maximum_count, self.callback)
|
22
35
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
def incr(self, dissection, name, value):
|
36
|
+
def incr(self, dissection: Dissection, name: str, value: str | int) -> None:
|
37
|
+
"""Increment a given name and value counter."""
|
27
38
|
if name not in self.ignore_list:
|
28
39
|
dissection.incr(name, value)
|
29
40
|
|
30
|
-
def
|
41
|
+
def dissect_dns(self, dns_data: bytes, prefix: str = None) -> None:
|
42
|
+
try:
|
43
|
+
dns = dpkt.dns.DNS(dns_data)
|
44
|
+
except dpkt.dpkt.UnpackError:
|
45
|
+
self.incr(self.dissection, prefix + "unparsable_dns", "PARSE_ERROR")
|
46
|
+
debug("DPKT unparsable DNS data")
|
47
|
+
return
|
48
|
+
except UnicodeDecodeError:
|
49
|
+
self.incr(self.dissection, prefix + "unparsable_utf8", "PARSE_ERROR")
|
50
|
+
debug("DPKT unparsable UTF8 data")
|
51
|
+
return
|
52
|
+
|
53
|
+
dissection = self.dissection
|
54
|
+
|
55
|
+
self.incr(dissection, prefix + "id", dns.id)
|
56
|
+
self.incr(dissection, prefix + "opcode", dns.op)
|
57
|
+
# self.incr(dissection, prefix + "qd", dns.qd)
|
58
|
+
# self.incr(dissection, prefix + "an", dns.an)
|
59
|
+
# self.incr(dissection, prefix + "ns", dns.ns)
|
60
|
+
# self.incr(dissection, prefix + "ar", dns.ar)
|
61
|
+
|
62
|
+
# flags and headers
|
63
|
+
self.incr(dissection, prefix + "rcode", dns.rcode)
|
64
|
+
self.incr(dissection, prefix + "ra", dns.ra)
|
65
|
+
self.incr(dissection, prefix + "rd", dns.rd)
|
66
|
+
self.incr(dissection, prefix + "tc", dns.tc)
|
67
|
+
self.incr(dissection, prefix + "z", dns.zero)
|
68
|
+
self.incr(dissection, prefix + "opcode", dns.opcode)
|
69
|
+
self.incr(dissection, prefix + "qr", dns.qr)
|
70
|
+
self.incr(dissection, prefix + "aa", dns.aa)
|
71
|
+
# self.incr(dissection, prefix + "ad", dns.ad)
|
72
|
+
|
73
|
+
# record counts
|
74
|
+
self.incr(dissection, prefix + "qdcount", len(dns.qd))
|
75
|
+
self.incr(dissection, prefix + "ancount", len(dns.an))
|
76
|
+
self.incr(dissection, prefix + "nscount", len(dns.ns))
|
77
|
+
self.incr(dissection, prefix + "arcount", len(dns.ar))
|
78
|
+
|
79
|
+
for record in dns.qd:
|
80
|
+
self.incr(dissection, prefix + "qd_qname", record.name + ".")
|
81
|
+
self.incr(dissection, prefix + "qd_qtype", record.type)
|
82
|
+
self.incr(dissection, prefix + "qd_qclass", record.cls)
|
83
|
+
|
84
|
+
for record in dns.an:
|
85
|
+
self.incr(dissection, prefix + "an_rrname", record.name + ".")
|
86
|
+
self.incr(dissection, prefix + "an_type", record.type)
|
87
|
+
self.incr(dissection, prefix + "an_rclass", record.cls)
|
88
|
+
self.incr(dissection, prefix + "an_rdlen", record.rlen)
|
89
|
+
self.incr(dissection, prefix + "an_ttl", record.ttl)
|
90
|
+
|
91
|
+
# concepts from dpkt.dns.DNS_upnack_rdata()
|
92
|
+
if record.type == dpkt.dns.DNS_A:
|
93
|
+
# TODO(hardaker): decode this hex streem to an IP
|
94
|
+
self.incr(dissection, prefix + "an_rdata", record.ip)
|
95
|
+
elif record.type == dpkt.dns.DNS_AAAA:
|
96
|
+
# TODO(hardaker): decode this hex streem to an IP(v6)
|
97
|
+
self.incr(dissection, prefix + "an_rdata", record.ip6)
|
98
|
+
elif record.type == dpkt.dns.DNS_NS:
|
99
|
+
self.incr(dissection, prefix + "an_nsname", record.nsname)
|
100
|
+
elif record.type == dpkt.dns.DNS_CNAME:
|
101
|
+
self.incr(dissection, prefix + "an_cname", record.cname)
|
102
|
+
elif record.type == dpkt.dns.DNS_CNAME:
|
103
|
+
self.incr(dissection, prefix + "an_ptrname", record.ptrname)
|
104
|
+
elif record.type == dpkt.dns.DNS_MX:
|
105
|
+
self.incr(
|
106
|
+
dissection,
|
107
|
+
prefix + "an_preference",
|
108
|
+
record.preference,
|
109
|
+
)
|
110
|
+
self.incr(dissection, prefix + "an_mxname", record.mxname)
|
111
|
+
elif record.type == dpkt.dns.DNS_SRV:
|
112
|
+
self.incr(dissection, prefix + "an_priority", record.priority)
|
113
|
+
self.incr(dissection, prefix + "an_weight", record.weight)
|
114
|
+
self.incr(dissection, prefix + "an_port", record.port)
|
115
|
+
self.incr(dissection, prefix + "an_srvname", record.srvname)
|
116
|
+
self.incr(dissection, prefix + "an_off", record.off)
|
117
|
+
elif record.type in (dpkt.dns.DNS_TXT, dpkt.dns.DNS_HINFO):
|
118
|
+
for text_record in record:
|
119
|
+
self.incr(dissection, prefix + "an_text", text_record)
|
120
|
+
elif record.type == dpkt.dns.DNS_SOA:
|
121
|
+
self.incr(dissection, prefix + "an_mname", record.mname + ".")
|
122
|
+
self.incr(dissection, prefix + "an_rname", record.rname)
|
123
|
+
self.incr(dissection, prefix + "an_serial", record.serial)
|
124
|
+
self.incr(dissection, prefix + "an_refresh", record.refresh)
|
125
|
+
self.incr(dissection, prefix + "an_refresh", record.refresh)
|
126
|
+
self.incr(dissection, prefix + "an_retry", record.retry)
|
127
|
+
self.incr(dissection, prefix + "an_expire", record.expire)
|
128
|
+
self.incr(dissection, prefix + "an_minimum", record.minimum)
|
129
|
+
|
130
|
+
for record in dns.ns:
|
131
|
+
self.incr(dissection, prefix + "ns_rrname", record.name + ".")
|
132
|
+
self.incr(dissection, prefix + "ns_type", record.type)
|
133
|
+
self.incr(dissection, prefix + "ns_rclass", record.cls)
|
134
|
+
# self.incr(dissection, prefix + "ns_rdata", record.nsname)
|
135
|
+
self.incr(dissection, prefix + "ns_ttl", record.ttl)
|
136
|
+
|
137
|
+
for record in dns.ar:
|
138
|
+
self.incr(dissection, prefix + "ar_rrname", record.name + "_")
|
139
|
+
self.incr(dissection, prefix + "ar_type", record.type)
|
140
|
+
self.incr(dissection, prefix + "ar_rclass", record.cls)
|
141
|
+
self.incr(dissection, prefix + "ar_ttl", record.ttl)
|
142
|
+
self.incr(dissection, prefix + "ar_rdlen", record.rlen)
|
143
|
+
|
144
|
+
def callback(self, timestamp: float, packet: bytes) -> None:
|
145
|
+
"""Dissect and count one packet."""
|
31
146
|
# if binning is requested, save it in a binned time slot
|
32
147
|
dissection: Dissection = self.dissection
|
33
148
|
|
34
|
-
|
35
|
-
if dissection.bin_size:
|
36
|
-
dissection.timestamp = (
|
37
|
-
dissection.timestamp - dissection.timestamp % dissection.bin_size
|
38
|
-
)
|
39
|
-
|
40
|
-
dissection.incr(Dissection.TOTAL_COUNT, dissection.TOTAL_SUBKEY)
|
149
|
+
self.start_packet(int(timestamp), dissection)
|
41
150
|
|
42
151
|
level = self.dissector_level
|
43
152
|
if isinstance(level, PCAPDissectorLevel):
|
44
153
|
level = level.value
|
154
|
+
|
45
155
|
if level >= PCAPDissectorLevel.THROUGH_IP.value:
|
46
156
|
eth = dpkt.ethernet.Ethernet(packet)
|
47
157
|
# these names are designed to match scapy names
|
48
|
-
self.incr(dissection, "
|
49
|
-
self.incr(dissection, "
|
50
|
-
self.incr(dissection, "
|
158
|
+
self.incr(dissection, "Ethernet_dst", eth.dst)
|
159
|
+
self.incr(dissection, "Ethernet_src", eth.src)
|
160
|
+
self.incr(dissection, "Ethernet_type", eth.type)
|
51
161
|
|
52
162
|
if isinstance(eth.data, dpkt.ip.IP):
|
53
163
|
ip = eth.data
|
164
|
+
udp = None
|
165
|
+
tcp = None
|
54
166
|
|
55
|
-
|
56
|
-
if ip.v ==
|
57
|
-
|
167
|
+
ipver = "IP"
|
168
|
+
if ip.v == DissectionEngineDpkt.IPV6_VERSION:
|
169
|
+
ipver = "IPv6"
|
58
170
|
|
59
|
-
prefix = f"
|
171
|
+
prefix = f"Ethernet_{ipver}_"
|
60
172
|
|
61
|
-
# TODO: make sure all these match scapy
|
173
|
+
# TODO(hardaker): make sure all these match scapy
|
62
174
|
self.incr(dissection, prefix + "dst", ip.dst)
|
63
175
|
self.incr(dissection, prefix + "src", ip.src)
|
64
176
|
self.incr(dissection, prefix + "df", ip.df)
|
@@ -76,23 +188,77 @@ class DissectionEngineDpkt(DissectionEngine):
|
|
76
188
|
|
77
189
|
if isinstance(ip.data, dpkt.udp.UDP):
|
78
190
|
udp = ip.data
|
79
|
-
self.incr(dissection, prefix + "
|
80
|
-
self.incr(dissection, prefix + "
|
81
|
-
self.incr(dissection, prefix + "
|
82
|
-
self.incr(dissection, prefix + "
|
191
|
+
self.incr(dissection, prefix + "UDP_sport", udp.sport)
|
192
|
+
self.incr(dissection, prefix + "UDP_dport", udp.dport)
|
193
|
+
self.incr(dissection, prefix + "UDP_len", udp.ulen)
|
194
|
+
self.incr(dissection, prefix + "UDP_chksum", udp.sum)
|
83
195
|
|
84
|
-
# TODO: handle DNS and others for level 3
|
196
|
+
# TODO(hardaker): handle DNS and others for level 3
|
85
197
|
|
86
198
|
elif isinstance(ip.data, dpkt.tcp.TCP):
|
87
|
-
# TODO
|
88
199
|
tcp = ip.data
|
89
|
-
self.incr(dissection, prefix + "
|
90
|
-
self.incr(dissection, prefix + "
|
91
|
-
self.incr(dissection, prefix + "
|
92
|
-
self.incr(dissection, prefix + "
|
93
|
-
# self.incr(dissection, prefix + "
|
94
|
-
self.incr(dissection, prefix + "
|
95
|
-
self.incr(dissection, prefix + "
|
96
|
-
self.incr(dissection, prefix + "
|
97
|
-
|
98
|
-
|
200
|
+
self.incr(dissection, prefix + "TCP_sport", tcp.sport)
|
201
|
+
self.incr(dissection, prefix + "TCP_dport", tcp.dport)
|
202
|
+
self.incr(dissection, prefix + "TCP_seq", tcp.seq)
|
203
|
+
self.incr(dissection, prefix + "TCP_flags", tcp.flags)
|
204
|
+
# self.incr(dissection, prefix + "TCP_reserved", tcp.reserved)
|
205
|
+
self.incr(dissection, prefix + "TCP_window", tcp.win)
|
206
|
+
self.incr(dissection, prefix + "TCP_chksum", tcp.sum)
|
207
|
+
self.incr(dissection, prefix + "TCP_options", tcp.opts)
|
208
|
+
|
209
|
+
if level >= PCAPDissectorLevel.COMMON_LAYERS.value:
|
210
|
+
http = None
|
211
|
+
if udp and DissectionEngineDpkt.DNS_PORT in (udp.sport, udp.dport):
|
212
|
+
self.dissect_dns(udp.data, prefix + "UDP_DNS_")
|
213
|
+
return
|
214
|
+
|
215
|
+
if tcp and DissectionEngineDpkt.DNS_PORT in (tcp.sport, tcp.dport):
|
216
|
+
self.dissect_dns(tcp.data, prefix + "TCP_DNS_")
|
217
|
+
return
|
218
|
+
|
219
|
+
if (
|
220
|
+
tcp
|
221
|
+
and DissectionEngineDpkt.HTTP_PORT in (tcp.sport, tcp.dport)
|
222
|
+
and len(tcp.data) > 0
|
223
|
+
):
|
224
|
+
try:
|
225
|
+
(command, _, content) = tcp.data.partition(b"\r\n")
|
226
|
+
http = dpkt.http.Message(content)
|
227
|
+
prefix += "TCP_HTTP 1_"
|
228
|
+
|
229
|
+
(method, _, remaining) = command.partition(b" ")
|
230
|
+
method = method.decode("utf-8")
|
231
|
+
if method in [
|
232
|
+
"GET",
|
233
|
+
"POST",
|
234
|
+
"HEAD",
|
235
|
+
"PUT",
|
236
|
+
"DELETE",
|
237
|
+
"CONNECT",
|
238
|
+
"TRACE",
|
239
|
+
"PATCH",
|
240
|
+
"OPTIONS",
|
241
|
+
]:
|
242
|
+
prefix += "Request_"
|
243
|
+
self.incr(dissection, prefix + "Method", method)
|
244
|
+
else:
|
245
|
+
prefix += "Response_"
|
246
|
+
|
247
|
+
except dpkt.dpkt.UnpackError:
|
248
|
+
self.incr(
|
249
|
+
dissection,
|
250
|
+
prefix + "TCP_HTTP_unparsable",
|
251
|
+
"PARSE_ERROR",
|
252
|
+
)
|
253
|
+
debug("DPKT unparsable HTTP data")
|
254
|
+
return
|
255
|
+
|
256
|
+
if http:
|
257
|
+
for header in http.headers:
|
258
|
+
parts = http.headers[header]
|
259
|
+
if not isinstance(parts, list):
|
260
|
+
parts = [parts]
|
261
|
+
for value in parts:
|
262
|
+
self.incr(
|
263
|
+
dissection, prefix + header, value.capitalize()
|
264
|
+
)
|
@@ -1,21 +1,26 @@
|
|
1
|
+
"""A scapy engine for deeply parsing and counting packets."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
1
4
|
from traffic_taffy.dissector_engine import DissectionEngine
|
2
|
-
from
|
3
|
-
from pcap_parallel import PCAPParallel as pcapp
|
5
|
+
from pcap_parallel import PCAPParallel
|
4
6
|
from logging import warning
|
5
7
|
|
6
8
|
from scapy.all import sniff, load_layer
|
7
9
|
|
8
10
|
|
9
11
|
class DissectionEngineScapy(DissectionEngine):
|
10
|
-
|
11
|
-
|
12
|
+
"""A scapy engine class for deeply parsing and counting packets."""
|
13
|
+
|
14
|
+
def __init__(self, *args: list, **kwargs: dict):
|
15
|
+
"""Create a scapy engine class."""
|
16
|
+
super().__init__(*args, **kwargs)
|
12
17
|
|
13
|
-
def
|
14
|
-
"
|
15
|
-
self.init_dissection()
|
16
|
-
load_this = self.pcap_file
|
18
|
+
def load_data(self) -> None:
|
19
|
+
"""Load a pcap file into a nested dictionary of statistical counts."""
|
17
20
|
if isinstance(self.pcap_file, str):
|
18
|
-
load_this =
|
21
|
+
load_this = PCAPParallel.open_maybe_compressed(self.pcap_file)
|
22
|
+
else:
|
23
|
+
load_this = self.pcap_file
|
19
24
|
|
20
25
|
if self.layers:
|
21
26
|
for layer in self.layers:
|
@@ -28,17 +33,15 @@ class DissectionEngineScapy(DissectionEngine):
|
|
28
33
|
count=self.maximum_count,
|
29
34
|
filter=self.pcap_filter,
|
30
35
|
)
|
31
|
-
|
32
|
-
# TODO: for some reason this fails on xz compressed files when processing in parallel
|
33
|
-
return self.dissection
|
34
|
-
|
35
|
-
def add_item(self, field_value, prefix: str) -> None:
|
36
|
-
"Adds an item to the self.dissection regardless of it's various types"
|
36
|
+
# TODO(hardaker): for some reason this fails on xz compressed files when processing in parallel
|
37
37
|
|
38
|
+
def add_item(self, field_value: str | int, prefix: str) -> None:
|
39
|
+
"""Add an item to the self.dissection regardless of it's various types"""
|
38
40
|
if isinstance(field_value, list):
|
39
41
|
if len(field_value) > 0:
|
40
42
|
# if it's a list of tuples, count the (eg TCP option) names
|
41
|
-
#
|
43
|
+
#
|
44
|
+
# TODO(hardaker): values can be always the same or things like timestamps
|
42
45
|
# that will always change or are too unique
|
43
46
|
if isinstance(field_value[0], tuple):
|
44
47
|
for item in field_value:
|
@@ -48,11 +51,7 @@ class DissectionEngineScapy(DissectionEngine):
|
|
48
51
|
self.add_item(item, prefix)
|
49
52
|
# else:
|
50
53
|
# debug(f"ignoring empty-list: {field_value}")
|
51
|
-
elif (
|
52
|
-
isinstance(field_value, str)
|
53
|
-
or isinstance(field_value, int)
|
54
|
-
or isinstance(field_value, float)
|
55
|
-
):
|
54
|
+
elif isinstance(field_value, (str, int, float)):
|
56
55
|
self.dissection.incr(prefix, field_value)
|
57
56
|
|
58
57
|
elif isinstance(field_value, bytes):
|
@@ -64,8 +63,7 @@ class DissectionEngineScapy(DissectionEngine):
|
|
64
63
|
self.dissection.incr(prefix, converted)
|
65
64
|
|
66
65
|
def add_layer(self, layer, prefix: str | None = "") -> None:
|
67
|
-
"
|
68
|
-
|
66
|
+
"""Analyze a layer to add counts to each layer sub-component."""
|
69
67
|
if hasattr(layer, "fields_desc"):
|
70
68
|
name_list = [field.name for field in layer.fields_desc]
|
71
69
|
elif hasattr(layer, "fields"):
|
@@ -83,21 +81,18 @@ class DissectionEngineScapy(DissectionEngine):
|
|
83
81
|
try:
|
84
82
|
field_value = getattr(layer, field_name)
|
85
83
|
if hasattr(field_value, "fields"):
|
86
|
-
self.add_layer(field_value, new_prefix + "
|
84
|
+
self.add_layer(field_value, new_prefix + "_")
|
87
85
|
else:
|
88
86
|
self.add_item(field_value, new_prefix)
|
89
87
|
except Exception as e:
|
90
88
|
warning(f"scapy error at '{prefix}' in field '{field_name}'")
|
91
89
|
warning(e)
|
92
90
|
|
93
|
-
def callback(self, packet):
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
self.timestamp = self.timestamp - self.timestamp % self.bin_size
|
91
|
+
def callback(self, packet) -> None:
|
92
|
+
"""Handle one packet to dissect."""
|
93
|
+
prefix = "_"
|
94
|
+
self.start_packet(int(packet.time))
|
98
95
|
|
99
|
-
self.dissection.timestamp = int(self.timestamp)
|
100
|
-
self.dissection.incr(Dissection.TOTAL_COUNT, Dissection.TOTAL_SUBKEY)
|
101
96
|
for payload in packet.iterpayloads():
|
102
|
-
prefix = f"{prefix}{payload.name}
|
97
|
+
prefix = f"{prefix}{payload.name}_"
|
103
98
|
self.add_layer(payload, prefix[1:])
|
traffic_taffy/graph.py
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
"""Create an output graph from a dissection data."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
1
5
|
import seaborn as sns
|
2
6
|
import matplotlib.pyplot as plt
|
3
7
|
from logging import debug, info
|
@@ -9,19 +13,21 @@ from traffic_taffy.graphdata import PcapGraphData
|
|
9
13
|
|
10
14
|
|
11
15
|
class PcapGraph(PcapGraphData):
|
16
|
+
"""Create an output graph from a dissection data."""
|
17
|
+
|
12
18
|
def __init__(
|
13
19
|
self,
|
14
20
|
pcap_files: str,
|
15
21
|
output_file: str,
|
16
|
-
maximum_count: int = None,
|
17
|
-
minimum_count: int = None,
|
18
|
-
bin_size: int = None,
|
19
|
-
match_string: str = None,
|
20
|
-
match_value: str = None,
|
22
|
+
maximum_count: int | None = None,
|
23
|
+
minimum_count: int | None = None,
|
24
|
+
bin_size: int | None = None,
|
25
|
+
match_string: str | None = None,
|
26
|
+
match_value: str | None = None,
|
21
27
|
cache_pcap_results: bool = False,
|
22
28
|
dissector_level: PCAPDissectorLevel = PCAPDissectorLevel.COUNT_ONLY,
|
23
29
|
interactive: bool = False,
|
24
|
-
ignore_list: List[str] =
|
30
|
+
ignore_list: List[str] | None = None,
|
25
31
|
by_percentage: bool = False,
|
26
32
|
pcap_filter: str | None = None,
|
27
33
|
cache_file_suffix: str = "taffy",
|
@@ -29,6 +35,7 @@ class PcapGraph(PcapGraphData):
|
|
29
35
|
force_overwrite: bool = False,
|
30
36
|
force_load: bool = False,
|
31
37
|
):
|
38
|
+
"""Create an instance of a graphing object."""
|
32
39
|
self.pcap_files = pcap_files
|
33
40
|
self.output_file = output_file
|
34
41
|
self.maximum_count = maximum_count
|
@@ -41,7 +48,7 @@ class PcapGraph(PcapGraphData):
|
|
41
48
|
self.cache_pcap_results = cache_pcap_results
|
42
49
|
self.dissector_level = dissector_level
|
43
50
|
self.interactive = interactive
|
44
|
-
self.ignore_list = ignore_list
|
51
|
+
self.ignore_list = ignore_list or []
|
45
52
|
self.by_percentage = by_percentage
|
46
53
|
self.pcap_filter = pcap_filter
|
47
54
|
self.cache_file_suffix = cache_file_suffix
|
@@ -51,8 +58,8 @@ class PcapGraph(PcapGraphData):
|
|
51
58
|
|
52
59
|
super().__init__()
|
53
60
|
|
54
|
-
def load_pcaps(self):
|
55
|
-
"
|
61
|
+
def load_pcaps(self) -> None:
|
62
|
+
"""Load the pcap and counts things into bins."""
|
56
63
|
self.data = {}
|
57
64
|
|
58
65
|
info("reading pcap files")
|
@@ -72,7 +79,8 @@ class PcapGraph(PcapGraphData):
|
|
72
79
|
self.dissections = pdm.load_all()
|
73
80
|
info("done reading pcap files")
|
74
81
|
|
75
|
-
def create_graph(self):
|
82
|
+
def create_graph(self) -> None:
|
83
|
+
"""Create the graph itself and save it."""
|
76
84
|
df = self.get_dataframe(merge=True, calculate_load_fraction=self.by_percentage)
|
77
85
|
|
78
86
|
hue_variable = "index"
|
@@ -102,8 +110,8 @@ class PcapGraph(PcapGraphData):
|
|
102
110
|
else:
|
103
111
|
plt.show()
|
104
112
|
|
105
|
-
def show_graph(self):
|
106
|
-
"Graph the results of the data collection"
|
113
|
+
def show_graph(self) -> None:
|
114
|
+
"""Graph the results of the data collection."""
|
107
115
|
debug("creating the graph")
|
108
116
|
sns.set_theme()
|
109
117
|
|
@@ -119,7 +127,8 @@ class PcapGraph(PcapGraphData):
|
|
119
127
|
if not self.match_string and not self.match_value:
|
120
128
|
self.interactive = False
|
121
129
|
|
122
|
-
def graph_it(self):
|
130
|
+
def graph_it(self) -> None:
|
131
|
+
"""Load the pcaps and graph it."""
|
123
132
|
debug("--- loading pcaps")
|
124
133
|
self.load_pcaps()
|
125
134
|
debug("--- creating graph")
|