traffic-taffy 0.6.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

traffic_taffy/__init__.py

@@ -1 +1 @@
-__VERSION__ = "0.6.4"
+__VERSION__ = "0.8.1"

traffic_taffy/algorithms/__init__.py

@@ -0,0 +1,14 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    pass
+
+
+class ComparisonAlgorithm:
+    def __init__(self):
+        pass
+
+    def compare_dissections(left_side: dict, right_side: dict) -> dict:
+        raise ValueError(
+            "code failure: base class compare_dissections should never be called"
+        )

traffic_taffy/algorithms/statistical.py

@@ -0,0 +1,100 @@
+from traffic_taffy.algorithms import ComparisonAlgorithm
+from traffic_taffy.comparison import Comparison
+from traffic_taffy.dissection import Dissection
+from traffic_taffy.report import Report
+
+
+class ComparisonStatistical(ComparisonAlgorithm):
+    def __init__(self):
+        super().__init__()
+
+    def compare_dissections(self, left_side: dict, right_side: dict) -> Comparison:
+        """Compare two dissections."""
+        report = {}
+
+        keys = set(left_side.keys())
+        keys = keys.union(right_side.keys())
+        for key in keys:
+            report[key] = {}
+
+            if key not in left_side:
+                left_side[key] = {}
+            left_side_total = sum(left_side[key].values())
+
+            if key not in right_side:
+                right_side[key] = {}
+            right_side_total = sum(right_side[key].values())
+
+            new_left_count = 0
+            for subkey in left_side[key]:
+                delta_percentage = 0.0
+                total = 0
+                if subkey in right_side[key]:
+                    left_percentage = left_side[key][subkey] / left_side_total
+                    right_percentage = right_side[key][subkey] / right_side_total
+                    delta_percentage = right_percentage - left_percentage
+                    total = right_side[key][subkey] + left_side[key][subkey]
+                    left_count = left_side[key][subkey]
+                    right_count = right_side[key][subkey]
+                else:
+                    delta_percentage = -1.0
+                    left_percentage = left_side[key][subkey] / left_side_total
+                    right_percentage = 0.0
+                    total = -left_side[key][subkey]
+                    left_count = left_side[key][subkey]
+                    right_count = 0
+                    new_left_count += 1
+
+                delta_absolute = right_count - left_count
+                report[key][subkey] = Report(
+                    delta_percentage=delta_percentage,
+                    delta_absolute=delta_absolute,
+                    total=total,
+                    left_count=left_count,
+                    right_count=right_count,
+                    left_percentage=left_percentage,
+                    right_percentage=right_percentage,
+                )
+
+            new_right_count = 0
+            for subkey in right_side[key]:
+                if subkey not in report[key]:
+                    delta_percentage = 1.0
+                    total = right_side[key][subkey]
+                    left_count = 0
+                    right_count = right_side[key][subkey]
+                    left_percentage = 0.0
+                    right_percentage = right_side[key][subkey] / right_side_total
+                    new_right_count += 1  # this value wasn't in the left
+
+                    report[key][subkey] = Report(
+                        delta_percentage=delta_percentage,
+                        delta_absolute=right_count,
+                        total=total,
+                        left_count=left_count,
+                        right_count=right_count,
+                        left_percentage=left_percentage,
+                        right_percentage=right_percentage,
+                    )
+
+            if right_side_total == 0:
+                right_percent = 100
+            else:
+                right_percent = new_right_count / right_side_total
+
+            if left_side_total == 0:
+                left_percent = 100
+            else:
+                left_percent = new_left_count / left_side_total
+
+            report[key][Dissection.NEW_RIGHT_SUBKEY] = Report(
+                delta_absolute=new_right_count - new_left_count,
+                total=new_left_count + new_right_count,
+                left_count=new_left_count,
+                right_count=new_right_count,
+                left_percentage=left_percent,
+                right_percentage=right_percent,
+                delta_percentage=right_percent - left_percent,
+            )
+
+        return Comparison(report)

traffic_taffy/compare.py

@@ -3,8 +3,9 @@
 from __future__ import annotations
 from logging import debug, error
 from typing import List, TYPE_CHECKING
-import datetime as dt
 from datetime import datetime
+import datetime as dt
+import itertools
 
 if TYPE_CHECKING:
     from argparse import ArgumentParser, Namespace
@@ -13,19 +14,7 @@ from traffic_taffy.comparison import Comparison
 from traffic_taffy.dissectmany import PCAPDissectMany
 from traffic_taffy.dissector import PCAPDissectorLevel
 from traffic_taffy.dissection import Dissection
-
-from dataclasses import dataclass
-
-
-@dataclass
-class Report:
-    delta_percentage: float
-    delta_absolute: int
-    total: int
-    left_count: int
-    right_count: int
-    left_percentage: float
-    right_percentage: float
+from traffic_taffy.algorithms.statistical import ComparisonStatistical
 
 
 class PcapCompare:
@@ -48,6 +37,7 @@ class PcapCompare:
         layers: List[str] | None = None,
         force_load: bool = False,
         force_overwrite: bool = False,
+        merge_files: bool = False,
     ) -> None:
         """Create a compare object."""
         self.pcap_files = pcap_files
@@ -63,6 +53,9 @@ class PcapCompare:
         self.layers = layers
         self.force_overwrite = force_overwrite
         self.force_load = force_load
+        self.merge_files = merge_files
+
+        self.algorithm = ComparisonStatistical()
 
     @property
     def pcap_files(self) -> List[str]:
@@ -82,97 +75,6 @@ class PcapCompare:
     def reports(self, newvalue: List[dict]) -> None:
         self._reports = newvalue
 
-    def compare_dissections(self, left_side: dict, right_side: dict) -> dict:
-        """Compare two dissections."""
-        report = {}
-
-        keys = set(left_side.keys())
-        keys = keys.union(right_side.keys())
-        for key in keys:
-            report[key] = {}
-
-            if key not in left_side:
-                left_side[key] = {}
-            left_side_total = sum(left_side[key].values())
-
-            if key not in right_side:
-                right_side[key] = {}
-            right_side_total = sum(right_side[key].values())
-
-            new_left_count = 0
-            for subkey in left_side[key]:
-                delta_percentage = 0.0
-                total = 0
-                if subkey in right_side[key]:
-                    left_percentage = left_side[key][subkey] / left_side_total
-                    right_percentage = right_side[key][subkey] / right_side_total
-                    delta_percentage = right_percentage - left_percentage
-                    total = right_side[key][subkey] + left_side[key][subkey]
-                    left_count = left_side[key][subkey]
-                    right_count = right_side[key][subkey]
-                else:
-                    delta_percentage = -1.0
-                    left_percentage = left_side[key][subkey] / left_side_total
-                    right_percentage = 0.0
-                    total = -left_side[key][subkey]
-                    left_count = left_side[key][subkey]
-                    right_count = 0
-                    new_left_count += 1
-
-                delta_absolute = right_count - left_count
-                report[key][subkey] = Report(
-                    delta_percentage=delta_percentage,
-                    delta_absolute=delta_absolute,
-                    total=total,
-                    left_count=left_count,
-                    right_count=right_count,
-                    left_percentage=left_percentage,
-                    right_percentage=right_percentage,
-                )
-
-            new_right_count = 0
-            for subkey in right_side[key]:
-                if subkey not in report[key]:
-                    delta_percentage = 1.0
-                    total = right_side[key][subkey]
-                    left_count = 0
-                    right_count = right_side[key][subkey]
-                    left_percentage = 0.0
-                    right_percentage = right_side[key][subkey] / right_side_total
-                    new_right_count += 1  # this value wasn't in the left
-
-                    report[key][subkey] = Report(
-                        delta_percentage=delta_percentage,
-                        delta_absolute=right_count,
-                        total=total,
-                        left_count=left_count,
-                        right_count=right_count,
-                        left_percentage=left_percentage,
-                        right_percentage=right_percentage,
-                    )
-
-            if right_side_total == 0:
-                right_percent = 100
-            else:
-                right_percent = new_right_count / right_side_total
-
-            if left_side_total == 0:
-                left_percent = 100
-            else:
-                left_percent = new_left_count / left_side_total
-
-            report[key][Dissection.NEW_RIGHT_SUBKEY] = Report(
-                delta_absolute=new_right_count - new_left_count,
-                total=new_left_count + new_right_count,
-                left_count=new_left_count,
-                right_count=new_right_count,
-                left_percentage=left_percent,
-                right_percentage=right_percent,
-                delta_percentage=right_percent - left_percent,
-            )
-
-        return Comparison(report)
-
     def load_pcaps(self) -> None:
         """Load all pcaps into memory and dissect them."""
         # load the first as a reference pcap
@@ -188,6 +90,7 @@ class PcapCompare:
             layers=self.layers,
             force_load=self.force_load,
             force_overwrite=self.force_overwrite,
+            merge_files=self.merge_files,
         )
         return pdm.load_all()
 
@@ -200,19 +103,34 @@ class PcapCompare:
     def compare_all(self, dissections: List[Dissection]) -> List[Comparison]:
         """Compare all loaded pcaps."""
         reports = []
-        if len(self.pcap_files) > 1:
+
+        # hack to figure out if there is at least two instances of a generator
+        # without actually extracting them all
+        # (since it could be memory expensive)
+        reference = next(dissections)
+        other = None
+        multiple = True
+        try:
+            other = next(dissections)
+            dissections = itertools.chain([other], dissections)
+        except Exception as e:
+            print(e)
+            multiple = False
+
+        if multiple:
             # multiple file comparison
-            reference = next(dissections)
             for other in dissections:
                 # compare the two global summaries
 
-                report = self.compare_dissections(reference.data[0], other.data[0])
+                report = self.algorithm.compare_dissections(
+                    reference.data[0], other.data[0]
+                )
                 report.title = f"{reference.pcap_file} vs {other.pcap_file}"
 
                 reports.append(report)
         else:
             # deal with timestamps within a single file
-            reference = list(dissections)[0].data
+            reference = reference.data
             timestamps = list(reference.keys())
             if len(timestamps) <= 2:  # just 0-summary plus a single stamp
                 error(
@@ -240,7 +158,7 @@ class PcapCompare:
 
                 debug(f"comparing timestamps {time_left} and {time_right}")
 
-                report = self.compare_dissections(
+                report = self.algorithm.compare_dissections(
                     reference[time_left],
                     reference[time_right],
                 )
@@ -293,7 +211,7 @@ def compare_add_parseargs(
     )
 
     compare_parser.add_argument(
-        "-x",
+        "-R",
         "--top-records",
         default=None,
         type=int,
@@ -338,4 +256,5 @@ def get_comparison_args(args: Namespace) -> dict:
         "top_records": args.top_records,
         "reverse_sort": args.reverse_sort,
         "sort_by": args.sort_by,
+        "merge_files": args.merge,
     }

traffic_taffy/dissection.py

@@ -416,7 +416,9 @@ class Dissection:
         try:
             if isinstance(value, bytes):
                 if value_type in Dissection.DISPLAY_TRANSFORMERS:
-                    value = str(Dissection.DISPLAY_TRANSFORMERS[value_type](value))
+                    value = str(
+                        Dissection.DISPLAY_TRANSFORMERS[value_type](value_type, value)
+                    )
                 else:
                     value = "0x" + value.hex()
             else:
@@ -432,7 +434,7 @@ class Dissection:
         return value
 
     @staticmethod
-    def print_mac_address(value: bytes) -> str:
+    def print_mac_address(value_type: str, value: bytes) -> str:
         """Convert bytes to ethernet mac style address."""
 
         # TODO(hardaker): certainly inefficient
@@ -441,12 +443,17 @@ class Dissection:
 
         return ":".join(map(two_hex, value))
 
+    @staticmethod
+    def print_ip_address(value_type: str, value: bytes) -> str:
+        """Convert binary bytes to IP addresses (v4 and v6)."""
+        return ipaddress.ip_address(value)
+
     # has to go at the end to pick up the above function names
     DISPLAY_TRANSFORMERS: ClassVar[Dict[str, callable]] = {
-        "Ethernet_IP_src": ipaddress.ip_address,
-        "Ethernet_IP_dst": ipaddress.ip_address,
-        "Ethernet_IP6_src": ipaddress.ip_address,
-        "Ethernet_IP6_dst": ipaddress.ip_address,
+        "Ethernet_IP_src": print_ip_address,
+        "Ethernet_IP_dst": print_ip_address,
+        "Ethernet_IP6_src": print_ip_address,
+        "Ethernet_IP6_dst": print_ip_address,
         "Ethernet_src": print_mac_address,
         "Ethernet_dst": print_mac_address,
     }

traffic_taffy/dissectmany.py

@@ -120,6 +120,16 @@ class PCAPDissectMany:
         # use all available resources
         with ProcessPoolExecutor() as executor:
             dissections = executor.map(self.load_pcap, self.pcap_files)
-            if return_as_list:  # convert from generator
+
+            # all loaded files should be merged as if they are one
+            if self.kwargs["merge_files"]:
+                dissection = next(dissections)
+                for to_be_merged in dissections:
+                    dissection.merge(to_be_merged)
+
+                dissections = [dissection]
+
+            elif return_as_list:  # convert from generator
                 dissections = list(dissections)
+
             return dissections

traffic_taffy/dissector.py

@@ -6,10 +6,14 @@ import sys
 from collections import Counter, defaultdict
 from logging import error, warning
 from typing import List
+import importlib
 
 from rich import print
 
 from traffic_taffy.dissection import Dissection, PCAPDissectorLevel
+from traffic_taffy.hooks import call_hooks
+
+POST_DISSECT_HOOK: str = "post_dissect"
 
 
 class PCAPDissector:
@@ -28,6 +32,7 @@ class PCAPDissector:
         layers: List[str] | None = None,
         force_overwrite: bool = False,
         force_load: bool = False,
+        merge_files: bool = False,  # Note: unused for a single load
     ) -> None:
         """Create a dissector object."""
         if ignore_list is None:
@@ -119,6 +124,8 @@ class PCAPDissector:
             engine = DissectionEngineDpkt(*args)
 
         self.dissection = engine.load()
+        call_hooks(POST_DISSECT_HOOK, dissection=self.dissection)
+
         if self.cache_results:
             self.dissection.save_to_cache()
         return self.dissection
@@ -256,6 +263,22 @@ def dissector_add_parseargs(parser, add_subgroup: bool = True):
         help="List of extra layers to load (eg: tls, http, etc)",
     )
 
+    parser.add_argument(
+        "-x",
+        "--modules",
+        default=None,
+        type=str,
+        nargs="*",
+        help="Extra processing modules to load (currently: psl) ",
+    )
+
+    parser.add_argument(
+        "--merge",
+        "--merge-files",
+        action="store_true",
+        help="Dissect multiple files as one.  (compare by time)",
+    )
+
     parser.add_argument(
         "-C",
         "--cache-pcap-results",
@@ -318,6 +341,22 @@ def limitor_add_parseargs(parser, add_subgroup: bool = True):
     return parser
 
 
+def dissector_handle_arguments(args) -> None:
+    check_dissector_level(args.dissection_level)
+    dissector_load_extra_modules(args.modules)
+
+
+def dissector_load_extra_modules(modules: List[str]) -> None:
+    """Loads extra modules"""
+    if not modules:
+        return
+    for module in modules:
+        try:
+            importlib.import_module(f"traffic_taffy.hooks.{module}")
+        except Exception as exp:
+            error(f"failed to load module {module}: {exp}")
+
+
 def check_dissector_level(level: int):
     """Check that the dissector level is legal."""
     current_dissection_levels = [

traffic_taffy/graph.py

@@ -34,6 +34,7 @@ class PcapGraph(PcapGraphData):
         layers: List[str] | None = None,
         force_overwrite: bool = False,
         force_load: bool = False,
+        merge_files: bool = False,  # unused
     ):
         """Create an instance of a graphing object."""
         self.pcap_files = pcap_files
@@ -55,6 +56,7 @@ class PcapGraph(PcapGraphData):
         self.layers = layers
         self.force_overwrite = force_overwrite
         self.force_load = force_load
+        self.merge_files = merge_files
 
         super().__init__()
 
@@ -75,6 +77,7 @@ class PcapGraph(PcapGraphData):
             layers=self.layers,
             force_overwrite=self.force_overwrite,
             force_load=self.force_load,
+            merge_files=self.merge_files,
         )
         self.dissections = pdm.load_all()
         info("done reading pcap files")

traffic_taffy/hooks/__init__.py

@@ -0,0 +1,38 @@
+from collections import defaultdict
+
+# __path__ = extend_path(__path__, __name__)
+
+from functools import wraps
+
+hooks = defaultdict(list)
+
+
+def register_hook(hook):
+    def decorator(function):
+        hooks[hook].append(function)
+
+        @wraps(function)
+        def _wrap(*args, **kwargs):
+            return function(*args, **kwargs)
+
+        return _wrap
+
+    return decorator
+
+
+def call_hooks(spot, *args, **kwargs):
+    for hook in hooks[spot]:
+        hook(*args, **kwargs)
+
+
+def main():
+    @register_hook("hookspot")
+    def test_hook():
+        print("world!!!")
+
+    print("hello")
+    call_hooks("hookspot")
+
+
+if __name__ == "__main__":
+    main()

traffic_taffy/hooks/ip2asn.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+from logging import error, info, debug
+import ip2asn
+
+from traffic_taffy.hooks import register_hook
+from traffic_taffy.dissector import POST_DISSECT_HOOK
+from traffic_taffy.dissection import Dissection
+
+if not Path("ip2asn-combined.tsv").exists():
+    error("The ip2asn plugin requires a ip2asn-combined.tsv in this directory")
+    error("Please download it from https://iptoasn.com/")
+
+info("loading ip2asn-combined.tsv")
+i2a = ip2asn.IP2ASN("ip2asn-combined.tsv")
+info("  ... loaded")
+
+
+@register_hook(POST_DISSECT_HOOK)
+def ip_to_asn(dissection: Dissection, **kwargs):
+    timestamps = dissection.data.keys()
+
+    for timestamp in timestamps:
+        keys = list(dissection.data[timestamp].keys())
+
+        for key in keys:
+            key = str(key)
+            if (
+                key.endswith("IP_src")
+                or key.endswith("IP_dst")
+                or key.endswith("IPv6_src")
+                or key.endswith("IPv6_dst")
+            ):
+                for value in dissection.data[timestamp][key]:
+                    count = dissection.data[timestamp][key][value]
+                    details = None
+                    try:
+                        details = i2a.lookup_address(value)
+                    except Exception:
+                        debug("failed to parse address: {value}")
+                    if not details:
+                        continue
+
+                    dissection.data[timestamp][key + "_ASN"][details["ASN"]] += count
+                    dissection.data[timestamp][key + "_country"][
+                        details["country"]
+                    ] += count
+                    dissection.data[timestamp][key + "_owner"][
+                        details["owner"]
+                    ] += count

traffic_taffy/hooks/psl.py

@@ -0,0 +1,42 @@
+from traffic_taffy.hooks import register_hook
+from traffic_taffy.dissector import POST_DISSECT_HOOK
+from traffic_taffy.dissection import Dissection
+
+import dnssplitter
+
+splitter = dnssplitter.DNSSplitter()
+splitter.init_tree()
+
+
+@register_hook(POST_DISSECT_HOOK)
+def split_dns_names(dissection: Dissection, **kwargs):
+    timestamps = dissection.data.keys()
+
+    for timestamp in timestamps:
+        keys = list(dissection.data[timestamp].keys())
+
+        for key in keys:
+            key = str(key)
+            if (
+                key.endswith("_qname")
+                or key.endswith("_mname")
+                or key.endswith("_rrname")
+            ):
+                for value in dissection.data[timestamp][key]:
+                    count = dissection.data[timestamp][key][value]
+                    results = splitter.search_tree(value)
+                    if not results or not results[2]:
+                        continue
+                    (
+                        prefix,
+                        registered_domain,
+                        registration_point,
+                    ) = results
+                    if registration_point:
+                        dissection.data[timestamp][key + "_prefix"][prefix] += count
+                        dissection.data[timestamp][key + "_domain"][
+                            registered_domain
+                        ] += count
+                        dissection.data[timestamp][key + "_psl"][
+                            registration_point
+                        ] += count

traffic_taffy/report.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Report:
+    delta_percentage: float
+    delta_absolute: int
+    total: int
+    left_count: int
+    right_count: int
+    left_percentage: float
+    right_percentage: float

traffic_taffy/tests/test_compare_results.py

@@ -1,8 +1,9 @@
 from collections import Counter
-from traffic_taffy.compare import PcapCompare, Report
+from traffic_taffy.report import Report
+from traffic_taffy.algorithms.statistical import ComparisonStatistical
 
 
-def test_compare_results():
+def test_compare_statistical_algorithm():
     left_data = {0: {"src": Counter({"a": 5, "b": 10})}}  # total = 15
     right_data = {0: {"src": Counter({"a": 15, "c": 15})}}  # total = 30
 
@@ -48,7 +49,7 @@ def test_compare_results():
         }
     }
 
-    pc = PcapCompare([1, 2])  # bogus file names
-    report = pc.compare_dissections(left_data[0], right_data[0])
+    algorithm = ComparisonStatistical()  # bogus file names
+    report = algorithm.compare_dissections(left_data[0], right_data[0])
 
     assert report.contents == expected

traffic_taffy/tests/test_dpkt_engine.py

@@ -0,0 +1,15 @@
+import os
+from traffic_taffy.dissection import PCAPDissectorLevel
+from traffic_taffy.dissector_engine.dpkt import DissectionEngineDpkt
+
+def test_dpkt_engine():
+    test_pcap = "dns.pcap"
+    test_pcap = "port53-2023-30-31_20.pcap"
+    test_pcap = "airplane-wireless.pcap"
+    if not os.path.exists(test_pcap):
+        return
+
+    engine = DissectionEngineDpkt(test_pcap,
+                                  dissector_level = PCAPDissectorLevel.COMMON_LAYERS)
+    dissection = engine.load()
+

traffic_taffy/tests/test_hooks.py

@@ -0,0 +1,46 @@
+from traffic_taffy.hooks import register_hook, call_hooks
+
+into_hook = 1
+
+
+def test_register_and_call_hook():
+    @register_hook("testhook")
+    def hook_callback():
+        global into_hook
+        into_hook += 1
+
+    call_hooks("testhookDNE")
+    assert into_hook == 1
+
+    call_hooks("testhook")
+    assert into_hook == 2
+
+
+def test_register_and_call_hook_with_args():
+    @register_hook("testhook_storage")
+    def hook_callback(storage, key, value):
+        storage[key] = value
+
+    the_storage = {}
+
+    call_hooks("testhook_storage", the_storage, "testkey", "testvalue")
+    assert the_storage == {"testkey": "testvalue"}
+
+    call_hooks("testhook_storage", the_storage, "otherkey", 4)
+    assert the_storage == {"testkey": "testvalue", "otherkey": 4}
+
+
+def test_register_and_call_hook_with_kwargs():
+    @register_hook("testhook_storage")
+    def hook_callback(storage={}, key=None, value=None):
+        storage[key] = value
+
+    the_storage = {}
+
+    call_hooks(
+        "testhook_storage", storage=the_storage, key="testkey", value="testvalue"
+    )
+    assert the_storage == {"testkey": "testvalue"}
+
+    call_hooks("testhook_storage", key="otherkey", value=4, storage=the_storage)
+    assert the_storage == {"testkey": "testvalue", "otherkey": 4}

traffic_taffy/tests/test_splitter.py

@@ -0,0 +1,45 @@
+from traffic_taffy.dissection import Dissection
+from traffic_taffy.dissector import POST_DISSECT_HOOK
+from traffic_taffy.hooks import call_hooks
+import traffic_taffy.hooks.psl
+
+
+def test_splitter_module():
+    dissection = Dissection("bogus")
+    dissection.incr("foo", "bar")
+    dissection.incr("foo_qname", "www.example.com")
+    dissection.incr("foo_qname", "www.example.net")
+    dissection.incr("foo_mname", "www.example.co.uk")
+    dissection.incr("foo_qname", "bogus.__doesntexist")
+
+    # bogus to avoid ruff removing the import
+    traffic_taffy.hooks.psl.splitter = traffic_taffy.hooks.psl.splitter
+
+    assert dissection.data[0] == {
+        "foo": {"bar": 1},
+        "foo_qname": {
+            "www.example.com": 1,
+            "www.example.net": 1,
+            "bogus.__doesntexist": 1,
+        },
+        "foo_mname": {"www.example.co.uk": 1},
+    }
+
+    call_hooks(POST_DISSECT_HOOK, dissection)
+
+    test_result = {
+        "foo": {"bar": 1},
+        "foo_mname": {"www.example.co.uk": 1},
+        "foo_mname_prefix": {"www": 1},
+        "foo_mname_domain": {"example.co.uk": 1},
+        "foo_mname_psl": {"co.uk": 1},
+        "foo_qname": {
+            "www.example.com": 1,
+            "www.example.net": 1,
+            "bogus.__doesntexist": 1,
+        },
+        "foo_qname_prefix": {"www": 2},
+        "foo_qname_domain": {"example.com": 1, "example.net": 1},
+        "foo_qname_psl": {"com": 1, "net": 1},
+    }
+    assert dissection.data[0] == test_result

traffic_taffy/tests/test_value_printing.py

@@ -3,7 +3,7 @@ from traffic_taffy.dissection import Dissection
 
 def test_printable():
     assert (
-        Dissection.make_printable("Ethernet.IP.dst", b"\x7f\x00\x00\x01") == "127.0.0.1"
+        Dissection.make_printable("Ethernet_IP_dst", b"\x7f\x00\x00\x01") == "127.0.0.1"
     )
 
     assert Dissection.make_printable("badtype", b"\x7f\x00\x00\x01") == "0x7f000001"

traffic_taffy/tools/compare.py

@@ -3,6 +3,7 @@
 import sys
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, Namespace
 import logging
+from logging import error
 from traffic_taffy.output.console import Console
 from traffic_taffy.output.fsdb import Fsdb
 
@@ -10,7 +11,7 @@ from traffic_taffy.compare import compare_add_parseargs, get_comparison_args
 from traffic_taffy.dissector import (
     dissector_add_parseargs,
     limitor_add_parseargs,
-    check_dissector_level,
+    dissector_handle_arguments,
 )
 from traffic_taffy.compare import PcapCompare
 
@@ -50,7 +51,7 @@ def parse_args() -> Namespace:
     log_level = args.log_level.upper()
     logging.basicConfig(level=log_level, format="%(levelname)-10s:\t%(message)s")
 
-    check_dissector_level(args.dissection_level)
+    dissector_handle_arguments(args)
 
     return args
 
@@ -89,12 +90,14 @@ def main() -> None:
             layers=args.layers,
             force_load=args.force_load,
             force_overwrite=args.force_overwrite,
+            merge_files=args.merge,
         )
 
         # compare the pcaps
         try:
             reports = pc.compare()
-        except ValueError:
+        except ValueError as e:
+            error(e)
             sys.exit()
 
         if args.fsdb:

traffic_taffy/tools/dissect.py

@@ -1,9 +1,11 @@
 """Performs generic dissection of a PCAP file."""
+import sys
 import logging
+from logging import error
 from traffic_taffy.dissector import (
     dissector_add_parseargs,
     limitor_add_parseargs,
-    check_dissector_level,
+    dissector_handle_arguments,
     PCAPDissector,
 )
 from traffic_taffy.dissectmany import PCAPDissectMany
@@ -53,7 +55,7 @@ def main() -> None:
 
     args = parse_args()
 
-    check_dissector_level(args.dissection_level)
+    dissector_handle_arguments(args)
 
     # load all the files
     pdm = PCAPDissectMany(
@@ -68,8 +70,13 @@ def main() -> None:
         layers=args.layers,
         force_overwrite=args.force_overwrite,
         force_load=args.force_load,
+        merge_files=args.merge,
     )
-    dissections = pdm.load_all(return_as_list=True, dont_fork=args.dont_fork)
+    try:
+        dissections = pdm.load_all(return_as_list=True, dont_fork=args.dont_fork)
+    except ValueError as e:
+        error(e)
+        sys.exit()
 
     # merge them into a single dissection
     dissection = dissections.pop(0)

traffic_taffy/tools/explore.py

@@ -10,7 +10,7 @@ from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, Namespace
 from traffic_taffy.dissector import (
     dissector_add_parseargs,
     limitor_add_parseargs,
-    check_dissector_level,
+    dissector_handle_arguments,
 )
 from traffic_taffy.dissection import Dissection
 from traffic_taffy.graphdata import PcapGraphData
@@ -180,6 +180,7 @@ class TaffyExplorer(QDialog, PcapGraphData):
             layers=self.args.layers,
             force_load=self.args.force_load,
             force_overwrite=self.args.force_overwrite,
+            merge_files=self.args.merge,
         )
 
         # create the graph data storage
@@ -638,7 +639,7 @@ def parse_args() -> Namespace:
     log_level = args.log_level.upper()
     logging.basicConfig(level=log_level, format="%(levelname)-10s:\t%(message)s")
 
-    check_dissector_level(args.dissection_level)
+    dissector_handle_arguments(args)
 
     return args
 

traffic_taffy/tools/export.py

@@ -8,9 +8,9 @@ import pyfsdb
 
 from traffic_taffy.dissectmany import PCAPDissectMany
 from traffic_taffy.dissector import (
-    check_dissector_level,
     dissector_add_parseargs,
     limitor_add_parseargs,
+    dissector_handle_arguments,
 )
 
 
@@ -52,7 +52,7 @@ def main() -> None:
     """Export traffic-taffy data into an FSDB file."""
     args = parse_args()
 
-    check_dissector_level(args.dissection_level)
+    dissector_handle_arguments(args)
 
     pdm = PCAPDissectMany(
         args.input_pcaps,
@@ -66,6 +66,7 @@ def main() -> None:
         layers=args.layers,
         force_load=args.force_load,
         force_overwrite=args.force_overwrite,
+        merge_files=args.merge,
     )
 
     dissections = pdm.load_all(return_as_list=True)

traffic_taffy/tools/graph.py

@@ -5,7 +5,7 @@ from traffic_taffy.graph import PcapGraph
 from traffic_taffy.dissector import (
     dissector_add_parseargs,
     limitor_add_parseargs,
-    check_dissector_level,
+    dissector_handle_arguments,
 )
 import logging
 
@@ -63,7 +63,7 @@ def main() -> None:
     """Run taffy-graph."""
     args = parse_args()
 
-    check_dissector_level(args.dissection_level)
+    dissector_handle_arguments(args)
 
     pc = PcapGraph(
         args.input_pcaps,
@@ -83,6 +83,7 @@ def main() -> None:
         layers=args.layers,
         force_overwrite=args.force_overwrite,
         force_load=args.force_load,
+        merge_files=args.merge,
     )
     pc.graph_it()
 

{traffic_taffy-0.6.4.dist-info → traffic_taffy-0.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: traffic-taffy
-Version: 0.6.4
+Version: 0.8.1
 Summary: A tool for doing differential analysis of pcap files
 Project-URL: Homepage, https://traffic-taffy.github.io/
 Author-email: Wes Hardaker <opensource@hardakers.net>
@@ -9,7 +9,9 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Requires-Dist: cryptography
+Requires-Dist: dnssplitter
 Requires-Dist: dpkt
+Requires-Dist: ip2asn
 Requires-Dist: msgpack
 Requires-Dist: pandas
 Requires-Dist: pcap-parallel

traffic_taffy-0.8.1.dist-info/RECORD

@@ -0,0 +1,43 @@
+traffic_taffy/__init__.py,sha256=cGUURt3LgcflHlV_5CG6SItz7kLJVNNBZFU8RU7RSJY,22
+traffic_taffy/compare.py,sha256=4T25Ygp6rQw7J3syn4Tam6tEw3ieSzl-lELPARvNnVw,8641
+traffic_taffy/comparison.py,sha256=MmCxK7E3RbVLS8gJ_JJOdqa2MHQ7np1Pq_VB0Scir8U,933
+traffic_taffy/dissection.py,sha256=-jXt4PYCi_lvPh86KC_84VywRZqhZ1cGgLdV2glCQC8,17390
+traffic_taffy/dissectmany.py,sha256=kJO2XF8QzOmLDsKDjZIkQBmxC97wFpi-IisCURLOmfU,4700
+traffic_taffy/dissector.py,sha256=JHewf3vhvYjE6CGuRzO6_SnR5QuHZjzx-hZjpkAPHl4,11685
+traffic_taffy/graph.py,sha256=lH20RjnY3rMU8g2zyOkpS5pfvgCcYa4GCtO22uArzs8,4510
+traffic_taffy/graphdata.py,sha256=lHpLuzzypEPuz3QPfkvs0IAbplTLzKcqHaLyoRidmiE,2971
+traffic_taffy/report.py,sha256=Yzb27hUWcWL-RxWpSQmRyM8NyWxQGT0l0jUCGHoYDSY,224
+traffic_taffy/algorithms/__init__.py,sha256=wfzoujmw_Huwi69UnDgAcBQyXeKqz2tPW4RFSqTr3iU,318
+traffic_taffy/algorithms/statistical.py,sha256=J9eI2a4S3GXezWySjjXZQ_sxNEnSc6ePiDSL3cp-UjQ,4023
+traffic_taffy/dissector_engine/__init__.py,sha256=Hu-UQtz7yhivmQLUP5b8tFQLEhy2bfvrRV3Q4aZp6vg,2202
+traffic_taffy/dissector_engine/dnstap.py,sha256=rBzVlB0D3YVhHOsr17cbnCIZU13g20srgR4sE7ZfNUE,4810
+traffic_taffy/dissector_engine/dpkt.py,sha256=YgFceo_6cy1VN-ODIijSsOfH3w8OzHPbpUS463is3YI,10949
+traffic_taffy/dissector_engine/scapy.py,sha256=5lE6y4wzULfULTFcTAo6qINazqoBVwzbRzP0alP4ATc,3730
+traffic_taffy/hooks/__init__.py,sha256=Bvhl6RnyBqQkWuCU6TS0O_ZHe4qCQsC4HE8FELigWPw,661
+traffic_taffy/hooks/ip2asn.py,sha256=8KyqwKTY3YK9GZEDW_3PM_6zuvrzkDAzIceIodNAepY,1765
+traffic_taffy/hooks/psl.py,sha256=6Xmbl2OigDIe17ChumU35iuE0B6Q7ttBp2iBruxWFg4,1529
+traffic_taffy/output/__init__.py,sha256=dSpW9xDno3nKaYOwThS2fVkcIHJVAJE9DlEiLfdHXfg,4555
+traffic_taffy/output/console.py,sha256=Ziq4MKtzSdP9oVaZrWHiw0Bpsm2Dj9QFLJtDK38mxaY,2911
+traffic_taffy/output/fsdb.py,sha256=po--ldHeRPTXVTkP68qtI_BYV2cEOab-kFVWv0ymj5M,1704
+traffic_taffy/output/memory.py,sha256=86tgJ-jMt3UVX31eP6U02YbbYRoqbYhhR4kXJQmYzO4,1870
+traffic_taffy/tests/test_compare_results.py,sha256=LRAAsel2vwSLZmSGxzdGurga77dhBsLkTCMRjybb10A,1921
+traffic_taffy/tests/test_dict_merge.py,sha256=t3rZSQQ0AlBxRKfLborx9SxYN53cCAQQzZ2w-__WT2Y,1429
+traffic_taffy/tests/test_dpkt_engine.py,sha256=512Wfq7D1qVkfhGwf1u2QSgZooWqZQWV9L4OhpAr4AE,489
+traffic_taffy/tests/test_hooks.py,sha256=amjEbtMwOZZCg_RCJ0wQR7aOqNfwz3IG3WY-9CwjSF4,1260
+traffic_taffy/tests/test_normalize.py,sha256=k5y3XmYitnF1aTkB-9dZ7WZPQB__l_iEzC6atKKywfw,2601
+traffic_taffy/tests/test_pcap_dissector.py,sha256=sxJJ3seO9pjzilM0b6iu_ggQbUI4E7WbMFdwm4m9Gz0,2027
+traffic_taffy/tests/test_pcap_splitter.py,sha256=Pjv2CS_WXqK1wQ1rdsxLU-M1Z5fuHsciEhiDJpujFnI,2568
+traffic_taffy/tests/test_splitter.py,sha256=bSb84QOCxRuRXy4BGLFDs5YYRBDgZrx9InhtE3DZsFg,1481
+traffic_taffy/tests/test_value_printing.py,sha256=rhmCUqnh1Lk1TTZvZi7ksvUWm4XDB4g2aUihjLpoqHI,266
+traffic_taffy/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+traffic_taffy/tools/cache_info.py,sha256=N4L-xjO3i-p3QD5DIytSt3GBnT4JsJh3V2dl5Tktumk,2084
+traffic_taffy/tools/compare.py,sha256=BkcWza0lN6KeBb_8oyr88eh0WoVqp5LM3yK8nRY16bc,3476
+traffic_taffy/tools/dissect.py,sha256=Z_5JJ92E4168_GZM_9Iu-m6lDfETBt8Gv-vD_Mu3Cfg,3559
+traffic_taffy/tools/explore.py,sha256=Hb4x9HZkFoYnR8BJKF5OLwZWKGR3RX3MdZhX4Oo-NDU,24180
+traffic_taffy/tools/export.py,sha256=BuAnZcOszj9ZpMxHmRjDqptR15dMRigOhp2-BFJHgWA,2707
+traffic_taffy/tools/graph.py,sha256=gARv5-7N5MUBSJJ8Uj5XLx4xEonXgIMADKwN573jxyk,2555
+traffic_taffy-0.8.1.dist-info/METADATA,sha256=MSgtWi6_e2R7wxBETCrzuPu_hI5QMXJa_n5ATjmJR6c,1933
+traffic_taffy-0.8.1.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
+traffic_taffy-0.8.1.dist-info/entry_points.txt,sha256=ySz30b1Cu03CtCGMRqqg0NZJpzrVI91T5HjbEaTNnpQ,314
+traffic_taffy-0.8.1.dist-info/licenses/LICENSE.txt,sha256=hiV1DJgDQeSM1r7P-ez5oxily11S5nsCedU0jKzKKzo,11338
+traffic_taffy-0.8.1.dist-info/RECORD,,

traffic_taffy-0.6.4.dist-info/RECORD

@@ -1,34 +0,0 @@
-traffic_taffy/__init__.py,sha256=nWnFwYowE5aRF4-qDmemCNsKthCCZqKs1fEUqJ-LX9E,22
-traffic_taffy/compare.py,sha256=VwwxfJUYMXSVByX4HTAk3IoQz5yxBUzcWYw36f5bj4s,11855
-traffic_taffy/comparison.py,sha256=MmCxK7E3RbVLS8gJ_JJOdqa2MHQ7np1Pq_VB0Scir8U,933
-traffic_taffy/dissection.py,sha256=Y1uiMHrD4TWQqu85dptI3joGOX6O4Dmn0MRnaH3gIPI,17141
-traffic_taffy/dissectmany.py,sha256=M3myhPiatVLJrINAuHkX4bx9X7rhpJy37T_H7p1TI8w,4394
-traffic_taffy/dissector.py,sha256=ysUVcsdQyx4LMFabSvln1vS7FFGShj67wBQfvnrSpAs,10609
-traffic_taffy/graph.py,sha256=Jj5Ga43I-zbetGjAIT-rzuEHr_ZZwFhGP76vkEgwNmg,4384
-traffic_taffy/graphdata.py,sha256=lHpLuzzypEPuz3QPfkvs0IAbplTLzKcqHaLyoRidmiE,2971
-traffic_taffy/dissector_engine/__init__.py,sha256=Hu-UQtz7yhivmQLUP5b8tFQLEhy2bfvrRV3Q4aZp6vg,2202
-traffic_taffy/dissector_engine/dnstap.py,sha256=rBzVlB0D3YVhHOsr17cbnCIZU13g20srgR4sE7ZfNUE,4810
-traffic_taffy/dissector_engine/dpkt.py,sha256=YgFceo_6cy1VN-ODIijSsOfH3w8OzHPbpUS463is3YI,10949
-traffic_taffy/dissector_engine/scapy.py,sha256=5lE6y4wzULfULTFcTAo6qINazqoBVwzbRzP0alP4ATc,3730
-traffic_taffy/output/__init__.py,sha256=dSpW9xDno3nKaYOwThS2fVkcIHJVAJE9DlEiLfdHXfg,4555
-traffic_taffy/output/console.py,sha256=Ziq4MKtzSdP9oVaZrWHiw0Bpsm2Dj9QFLJtDK38mxaY,2911
-traffic_taffy/output/fsdb.py,sha256=po--ldHeRPTXVTkP68qtI_BYV2cEOab-kFVWv0ymj5M,1704
-traffic_taffy/output/memory.py,sha256=86tgJ-jMt3UVX31eP6U02YbbYRoqbYhhR4kXJQmYzO4,1870
-traffic_taffy/tests/test_compare_results.py,sha256=aUxy6UdtGhWJYNuQn8_UzkPRQpbEfzKXWz7DShpmRG4,1832
-traffic_taffy/tests/test_dict_merge.py,sha256=t3rZSQQ0AlBxRKfLborx9SxYN53cCAQQzZ2w-__WT2Y,1429
-traffic_taffy/tests/test_normalize.py,sha256=k5y3XmYitnF1aTkB-9dZ7WZPQB__l_iEzC6atKKywfw,2601
-traffic_taffy/tests/test_pcap_dissector.py,sha256=sxJJ3seO9pjzilM0b6iu_ggQbUI4E7WbMFdwm4m9Gz0,2027
-traffic_taffy/tests/test_pcap_splitter.py,sha256=Pjv2CS_WXqK1wQ1rdsxLU-M1Z5fuHsciEhiDJpujFnI,2568
-traffic_taffy/tests/test_value_printing.py,sha256=3k-d1bpa6tD8qcrgNVoZhhG7kwylI2AMVNFPbRE2vds,266
-traffic_taffy/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-traffic_taffy/tools/cache_info.py,sha256=N4L-xjO3i-p3QD5DIytSt3GBnT4JsJh3V2dl5Tktumk,2084
-traffic_taffy/tools/compare.py,sha256=mVChkw56pzU6HCY67pNSI7qrlaU5N65BdLoP8upmsXU,3395
-traffic_taffy/tools/dissect.py,sha256=1-b6fnIE3DPJ2xvsImPrVynKPwerQU0PrYvaMJC85ZA,3420
-traffic_taffy/tools/explore.py,sha256=TEUqfiknqKwEq0zswfgwvKPLMSOlTUWqC8NhcwPfQO8,24146
-traffic_taffy/tools/export.py,sha256=jk5ck-K9m9_Rno72fDWgfS67WKUumDL1KQpMG_j0-G8,2682
-traffic_taffy/tools/graph.py,sha256=5pcO_FsASddi_zEYB9h24syuqLCaDzByIPAiR4aq6gg,2530
-traffic_taffy-0.6.4.dist-info/METADATA,sha256=m9iXCaqdmC3SC9hb24ZIUnjdWESXR2cp6UdjHIpOOSs,1884
-traffic_taffy-0.6.4.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
-traffic_taffy-0.6.4.dist-info/entry_points.txt,sha256=ySz30b1Cu03CtCGMRqqg0NZJpzrVI91T5HjbEaTNnpQ,314
-traffic_taffy-0.6.4.dist-info/licenses/LICENSE.txt,sha256=hiV1DJgDQeSM1r7P-ez5oxily11S5nsCedU0jKzKKzo,11338
-traffic_taffy-0.6.4.dist-info/RECORD,,