traffic-taffy 0.5.4__tar.gz → 0.5.6__tar.gz

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: traffic-taffy
-Version: 0.5.4
+Version: 0.5.6
 Summary: A tool for doing differential analysis of pcap files
 Project-URL: Homepage, https://github.com/hardaker/traffic-taffy
 Author-email: Wes Hardaker <opensource@hardakers.net>

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/pyproject.toml

@@ -50,8 +50,23 @@ include = [
 ]
 
 [tool.ruff]
-# long lines, unsorted imports, too many args
-ignore = ["E501", "I001", "PLR0913"]
+# long lines, unsorted imports, too many args, don't type selfs, or return types
+ignore = ["E501", "I001", "PLR0913", "ANN101", "ANN204",
+# allow boolean typed position arguments/defaults
+"FBT001", "FBT002",
+# don't require issues,
+"TD003",
+# trailing commas is often wrong because of black formatting
+"COM812",
+# generic typing (list vs List) doesn't work until python 3.8
+"UP006",
+# Logging statement uses f-string
+"G004",
+# complaining about simple if/else
+"SIM108",
+# allow blind exceptions
+"BLE001"
+]
 fixable = ["ALL"]  # gulp
 # select = ["ALL"]
 # select = ["A", "ARG", "C4", "COM", "EM", "E", "ANN"]

traffic_taffy-0.5.6/traffic_taffy/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "0.5.6"

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/dissection.py

@@ -39,7 +39,7 @@ class Dissection:
         bin_size: int = 0,
         dissector_level: PCAPDissectorLevel = PCAPDissectorLevel.DETAILED,
         cache_file_suffix: str = "taffy",
-        ignore_list: list = [],
+        ignore_list: list | None = None,
         *args: list,
         **kwargs: dict,
     ) -> Dissection:
@@ -52,7 +52,7 @@ class Dissection:
         self.dissector_level = dissector_level
         self.maximum_count = maximum_count
         self.pcap_filter = pcap_filter
-        self.ignore_list = ignore_list
+        self.ignore_list = ignore_list or []
 
         self.parameters = [
             "pcap_file",
@@ -81,10 +81,11 @@ class Dissection:
 
     @property
     def timestamp(self) -> int:
+        """Timestamp currently being worked on."""
         return self._timestamp
 
     @timestamp.setter
-    def timestamp(self: Dissection, newval):
+    def timestamp(self: Dissection, newval: int) -> None:
         self._timestamp = newval
 
     @property
@@ -93,19 +94,19 @@ class Dissection:
         return self._data
 
     @data.setter
-    def data(self: Dissection, newval):
+    def data(self: Dissection, newval: dict) -> None:
         self._data = newval
 
     @property
-    def pcap_file(self: Dissection):
-        """The PCAP file name of this dissection"""
+    def pcap_file(self: Dissection) -> str:
+        """The PCAP file name of this dissection."""
         return self._pcap_file
 
     @pcap_file.setter
-    def pcap_file(self: Dissection, newval):
+    def pcap_file(self: Dissection, newval: str) -> None:
         self._pcap_file = newval
 
-    def incr(self: Dissection, key: str, value: Any, count: int = 1):
+    def incr(self: Dissection, key: str, value: Any, count: int = 1) -> None:
         """Increase one field within the counter."""
         # always save a total count at the zero bin
         # note: there should be no recorded tcpdump files from 1970 Jan 01 :-)
@@ -117,8 +118,8 @@ class Dissection:
 
     def calculate_metadata(self: Dissection) -> None:
         """Calculate thing like the number of value entries within each key/subkey."""
-        # TODO: do we do this with or without key and value matches?
-        for timestamp in self.data.keys():
+        # TODO(hardaker): do we do this with or without key and value matches?
+        for timestamp in self.data:
             for key in self.data[timestamp]:
                 if self.WIDTH_SUBKEY in self.data[timestamp][key]:
                     # make sure to avoid counting itself
@@ -131,12 +132,12 @@ class Dissection:
                     # don't count the NEW subkey either
                     self.data[timestamp][key] -= 1
 
-    def merge(self: Dissection, other_dissection) -> None:
-        "merges counters in two dissections into self -- note destructive to self"
+    def merge(self: Dissection, other_dissection: Dissection) -> None:
+        """Merge counters from another dissection into self."""
         for timestamp in other_dissection.data:
             for key in other_dissection.data[timestamp]:
                 for subkey in other_dissection.data[timestamp][key]:
-                    # TODO: this is horribly inefficient
+                    # TODO(hardaker): this is horribly inefficient
                     if timestamp not in self.data:
                         self.data[timestamp] = defaultdict(Counter)
                     elif key not in self.data[timestamp]:
@@ -151,11 +152,13 @@ class Dissection:
                     ][key][subkey]
 
     def merge_all(self: Dissection, other_dissections: List[Dissection]) -> None:
+        """Merge multiple dissection contents into this one."""
         for dissection in other_dissections:
             self.merge(dissection)
 
     @staticmethod
-    def subdict_producer():
+    def subdict_producer() -> defaultdict:
+        """Create a factory for creating a producer."""
         return defaultdict(Counter)
 
     #
@@ -165,6 +168,7 @@ class Dissection:
     def load_from_cache(
         self: Dissection, force_overwrite: bool = False, force_load: bool = True
     ) -> dict | None:
+        """Load the dissection data from a cache."""
         if not self.pcap_file or not isinstance(self.pcap_file, str):
             return None
         if not os.path.exists(self.pcap_file + self.cache_file_suffix):
@@ -233,19 +237,18 @@ class Dissection:
 
         error(f"Failed to load cached data for {self.pcap_file} due to differences")
         error("refusing to continue -- remove the cache to recreate it")
-        raise ValueError(
-            "INCOMPATIBLE CACHE: remove the cache or don't use it to continue"
-        )
+        msg = "INCOMPATIBLE CACHE: remove the cache or don't use it to continue"
+        raise ValueError(msg)
 
     def save_to_cache(self: Dissection, where: str | None = None) -> None:
+        """Save the dissection contents to a cache."""
         if not where and self.pcap_file and isinstance(self.pcap_file, str):
             where = self.pcap_file + self.cache_file_suffix
         if where:
             self.save(where)
 
     def save(self: Dissection, where: str) -> None:
-        "Saves a generated dissection to a msgpack file"
-
+        """Saves a generated dissection to a msgpack file."""
         # wrap the report in a version header
         versioned_cache = {
             self.DISSECTION_KEY: self.DISSECTION_VERSION,
@@ -256,7 +259,7 @@ class Dissection:
 
         for parameter in self.parameters:
             versioned_cache["parameters"][parameter] = getattr(self, parameter)
-            # TODO: fix this hack
+            # TODO(hardaker): fix this hack
 
             # basically, bin_size of 0 is 1...  but it may be faster
             # to leave it at zero to avoid the bin_size math of 1,
@@ -280,9 +283,31 @@ class Dissection:
 
         # save it
         info(f"caching PCAP data to '{where}'")
-        msgpack.dump(versioned_cache, open(where, "wb"))
 
-    def load_saved_contents(self: Dissection, versioned_cache):
+        # convert int keys that are too large
+        for timestamp in versioned_cache["dissection"]:
+            for key in versioned_cache["dissection"][timestamp]:
+                versioned_cache["dissection"][timestamp][key] = dict(
+                    versioned_cache["dissection"][timestamp][key]
+                )
+                # sigh -- msgpack can't handle large int based dictionary keys
+                fix_list = []
+                for subkey in versioned_cache["dissection"][timestamp][key]:
+                    if isinstance(subkey, int) and subkey > 2**32 - 1:
+                        info(f"converting {key} {subkey}")
+                        fix_list.append(subkey)
+
+                for subkey in fix_list:
+                    versioned_cache["dissection"][timestamp][key][
+                        str(subkey)
+                    ] = versioned_cache["dissection"][timestamp][key][subkey]
+                    del versioned_cache["dissection"][timestamp][key][subkey]
+
+        with open(where, "wb") as saveto:
+            msgpack.dump(versioned_cache, saveto)
+
+    def load_saved_contents(self: Dissection, versioned_cache: dict):
+        """Set parameters from the cache."""
         # set the local parameters from the cache
         for parameter in self.parameters:
             setattr(self, parameter, versioned_cache["parameters"][parameter])
@@ -291,8 +316,9 @@ class Dissection:
         self.data = versioned_cache["dissection"]
 
     def load_saved(self: Dissection, where: str, dont_overwrite: bool = False) -> dict:
-        "Loads a previous saved report from a file instead of re-parsing pcaps"
-        contents = msgpack.load(open(where, "rb"), strict_map_key=False)
+        """Load a saved report from a cache file."""
+        with open(where, "rb") as cache_file:
+            contents = msgpack.load(cache_file, strict_map_key=False)
 
         # convert the ignore list to a set (msgpack doesn't do sets)
         contents["parameters"]["ignore_list"] = set(
@@ -320,14 +346,15 @@ class Dissection:
         match_value: str | None = None,
         minimum_count: int | None = None,
         make_printable: bool = False,
-    ):
+    ) -> list:
+        """Search through data for appropriate records."""
         data = self.data
 
         if not timestamps:
             timestamps = data.keys()
 
         # find timestamps/key values with at least one item above count
-        # TODO: we should really use pandas for this
+        # TODO(hardaker): we should really use pandas for this
         usable = defaultdict(set)
         for timestamp in timestamps:
             for key in data[timestamp]:
@@ -345,7 +372,7 @@ class Dissection:
                     ):
                         usable[key].add(subkey)
 
-        # TODO: move the timestamp inside the other fors for faster
+        # TODO(hardaker): move the timestamp inside the other fors for faster
         # processing of skipped key/subkeys
         for timestamp in timestamps:
             for key in sorted(data[timestamp]):
@@ -370,10 +397,11 @@ class Dissection:
 
     @staticmethod
     def make_printable(value_type: str, value: Any) -> str:
+        """Turn a value into a printable version if needed."""
         try:
             if isinstance(value, bytes):
-                if value_type in Dissection.display_transformers:
-                    value = str(Dissection.display_transformers[value_type](value))
+                if value_type in Dissection.DISPLAY_TRANSFORMERS:
+                    value = str(Dissection.DISPLAY_TRANSFORMERS[value_type](value))
                 else:
                     value = "0x" + value.hex()
             else:
@@ -388,16 +416,17 @@ class Dissection:
         return value
 
     @staticmethod
-    def print_mac_address(value):
-        "Converts bytes to ethernet mac style address"
+    def print_mac_address(value: bytes) -> str:
+        """Convert bytes to ethernet mac style address."""
 
-        # TODO: certainly inefficient
-        def two_hex(value):
+        # TODO(hardaker): certainly inefficient
+        def two_hex(value: bytes) -> str:
             return f"{value:02x}"
 
         return ":".join(map(two_hex, value))
 
-    display_transformers = {
+    # has to go at the end to pick up the above function names
+    DISPLAY_TRANSFORMERS: dict = {
         "Ethernet.IP.src": ipaddress.ip_address,
         "Ethernet.IP.dst": ipaddress.ip_address,
         "Ethernet.IP6.src": ipaddress.ip_address,

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/dissectmany.py

@@ -83,7 +83,16 @@ class PCAPDissectMany:
 
         return dissection
 
-    def load_all(self, return_as_list: bool = False):
+    def load_all(self, return_as_list: bool = False, dont_fork: bool = False):
+        if dont_fork:
+            # handle each one individually -- typically for inserting debugging stops
+            dissections = []
+            for pcap_file in self.pcap_files:
+                dissection = self.load_pcap(pcap_file)
+                dissections.append(dissection)
+            return dissections
+
+        # use all available resources
         with ProcessPoolExecutor() as executor:
             dissections = executor.map(self.load_pcap, self.pcap_files)
             if return_as_list:  # convert from generator

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/graphdata.py

@@ -1,29 +1,36 @@
+"""A module for storing/transforming data (frequently to be graphed)."""
+
 import os
 from pandas import DataFrame, to_datetime, concat
+from traffic_taffy.dissection import Dissection
 
 
 class PcapGraphData:
+    """A base class for storing/transforming data (frequently to be graphed)."""
+
     def __init__(self):
+        """Create an instance of a PcapGraphData."""
         self.dissections = []
-        pass
 
     @property
-    def dissections(self):
+    def dissections(self) -> list:
+        """Dissections stored within the PcapGraphData instance."""
         return self._dissections
 
     @dissections.setter
-    def dissections(self, newvalue):
+    def dissections(self, newvalue: list) -> None:
         self._dissections = newvalue
 
-    def normalize_bins(self, dissection):
-        results = {}
-        time_keys = list(dissection.data.keys())
+    def normalize_bins(self, dissection: Dissection) -> dict:
+        """Transform a dissection's list of data into a dictionary."""
+        results: dict = {}
+        time_keys: list = list(dissection.data.keys())
         if time_keys[0] == 0:  # likely always
             time_keys.pop(0)
 
-        results = {"time": [], "count": [], "index": [], "key": [], "subkey": []}
+        results: dict = {"time": [], "count": [], "index": [], "key": [], "subkey": []}
 
-        # TODO: this could likely be made much more efficient and needs hole-filling
+        # TODO(hardaker): this could likely be made much more efficient and needs hole-filling
         for timestamp, key, subkey, value in dissection.find_data(
             timestamps=time_keys,
             match_string=self.match_string,
@@ -40,7 +47,10 @@ class PcapGraphData:
 
         return results
 
-    def get_dataframe(self, merge=False, calculate_load_fraction=False):
+    def get_dataframe(
+        self, merge: bool = False, calculate_load_fraction: bool = False
+    ) -> DataFrame:
+        """Create a pandas dataframe from stored dissections."""
         datasets = []
         if merge:
             dissection = next(self.dissections).clone()

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/tools/cache_info.py

@@ -11,7 +11,7 @@ def parse_args():
     parser = ArgumentParser(
         formatter_class=ArgumentDefaultsHelpFormatter,
         description=__doc__,
-        epilog="Exmaple Usage: ",
+        epilog="Example Usage: taffy-cache-info something.taffy",
     )
 
     parser.add_argument(

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/tools/compare.py

@@ -19,7 +19,7 @@ def parse_args():
     parser = ArgumentParser(
         formatter_class=ArgumentDefaultsHelpFormatter,
         description=__doc__,
-        epilog="Exmaple Usage: ",
+        epilog="Example Usage: taffy-compare -C file1.pcap file2.pcap",
     )
 
     output_options = parser.add_argument_group("Output format")
@@ -93,6 +93,8 @@ def main():
             ignore_list=args.ignore_list,
             pcap_filter=args.filter,
             layers=args.layers,
+            force_load=args.force_load,
+            force_overwrite=args.force_overwrite,
         )
 
         # compare the pcaps

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/tools/dissect.py

@@ -16,7 +16,7 @@ def main():
         parser = ArgumentParser(
             formatter_class=ArgumentDefaultsHelpFormatter,
             description=__doc__,
-            epilog="Exmaple Usage: ",
+            epilog="Example Usage: taffy-dissect -C -d 10 -n 10000 file.pcap",
         )
 
         parser.add_argument(
@@ -33,10 +33,16 @@ def main():
             help="Print results in an FSDB formatted output",
         )
 
+        parser.add_argument(
+            "--dont-fork",
+            action="store_true",
+            help="Do not fork into multiple processes per file (still fork per file)",
+        )
+
         dissector_add_parseargs(parser)
         limitor_add_parseargs(parser)
 
-        parser.add_argument("input_file", type=str, help="input pcap file")
+        parser.add_argument("input_pcaps", type=str, help="input pcap file", nargs="*")
 
         args = parser.parse_args()
         log_level = args.log_level.upper()
@@ -47,8 +53,9 @@ def main():
 
     check_dissector_level(args.dissection_level)
 
+    # load all the files
     pdm = PCAPDissectMany(
-        args.input_file,
+        args.input_pcaps,
         bin_size=args.bin_size,
         dissector_level=args.dissection_level,
         maximum_count=args.packet_count,
@@ -60,14 +67,15 @@ def main():
         force_overwrite=args.force_overwrite,
         force_load=args.force_load,
     )
-    dissection = pdm.load_pcap(
-        args.input_file,
-        maximum_count=args.packet_count,
-        force_overwrite=args.force_overwrite,
-        force_load=args.force_load,
-    )
+    dissections = pdm.load_all(True, dont_fork=args.dont_fork)
+
+    # merge them into a single dissection
+    dissection = dissections.pop(0)
+    dissection.merge_all(dissections)
+
+    # put the dissection into a dissector for reporting
     pd = PCAPDissector(
-        args.input_file,
+        args.input_pcaps[0],
         bin_size=args.bin_size,
         dissector_level=args.dissection_level,
         maximum_count=args.packet_count,
@@ -81,6 +89,7 @@ def main():
     )
     pd.dissection = dissection
 
+    # output as requested
     if args.fsdb:
         pd.print_to_fsdb(
             timestamps=[0],

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/tools/explore.py

@@ -641,7 +641,7 @@ def parse_args():
     parser = ArgumentParser(
         formatter_class=ArgumentDefaultsHelpFormatter,
         description=__doc__,
-        epilog="Exmaple Usage: ",
+        epilog="Example Usage: taffy-explore -C file1.pcap file2.pcap",
     )
 
     limitor_parser = limitor_add_parseargs(parser)

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/tools/export.py

@@ -19,7 +19,7 @@ def parse_args() -> Namespace:
     parser = ArgumentParser(
         formatter_class=ArgumentDefaultsHelpFormatter,
         description=__doc__,
-        epilog="Exmaple Usage: ",
+        epilog="Example Usage: taffy-export -C -m IP.UDP.sport file.pcap",
     )
 
     parser.add_argument(
@@ -40,7 +40,7 @@ def parse_args() -> Namespace:
         help="Where to store output data",
     )
 
-    parser.add_argument("input_files", nargs="*", type=str, help="input pcap file")
+    parser.add_argument("input_pcaps", nargs="*", type=str, help="input pcap file")
 
     args = parser.parse_args()
     log_level = args.log_level.upper()
@@ -55,7 +55,7 @@ def main() -> None:
     check_dissector_level(args.dissection_level)
 
     pdm = PCAPDissectMany(
-        args.input_files,
+        args.input_pcaps,
         bin_size=args.bin_size,
         dissector_level=args.dissection_level,
         maximum_count=args.packet_count,
@@ -64,6 +64,8 @@ def main() -> None:
         ignore_list=args.ignore_list,
         pcap_filter=args.filter,
         layers=args.layers,
+        force_load=args.force_load,
+        force_overwrite=args.force_overwrite,
     )
 
     dissections = pdm.load_all(return_as_list=True)

{traffic_taffy-0.5.4 → traffic_taffy-0.5.6}/traffic_taffy/tools/graph.py

@@ -15,7 +15,7 @@ def parse_args():
     parser = ArgumentParser(
         formatter_class=ArgumentDefaultsHelpFormatter,
         description=__doc__,
-        epilog="Exmaple Usage: ",
+        epilog="Example Usage: taffy-graph -C -m TOTAL -M packet -o graph.png file.pcap",
     )
 
     parser.add_argument(
@@ -50,7 +50,7 @@ def parse_args():
     dissector_add_parseargs(parser)
     limitor_add_parseargs(parser)
 
-    parser.add_argument("input_file", type=str, help="PCAP file to graph", nargs="+")
+    parser.add_argument("input_pcaps", type=str, help="PCAP file to graph", nargs="+")
 
     args = parser.parse_args()
     log_level = args.log_level.upper()
@@ -65,7 +65,7 @@ def main():
     check_dissector_level(args.dissection_level)
 
     pc = PcapGraph(
-        args.input_file,
+        args.input_pcaps,
         args.output_file,
         maximum_count=args.packet_count,
         minimum_count=args.minimum_count,

traffic_taffy-0.5.4/traffic_taffy/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "0.5.4"