traffic-taffy 0.3.6__py3-none-any.whl → 0.4.1__py3-none-any.whl

traffic_taffy/cache_info.py

@@ -47,12 +47,6 @@ def main():
     for key in contents["parameters"]:
         print(f"    {key:<16} {contents['parameters'][key]}")
 
-    print("data info:")
-    timestamps = list(contents["dissection"].keys())
-    print(f"    timestamps:      {len(timestamps)}")
-    print(f"    first:           {timestamps[1]}")  # skips 0 = global
-    print(f"    last:            {timestamps[-1]}")
-
 
 if __name__ == "__main__":
     main()

traffic_taffy/compare.py

@@ -1,18 +1,12 @@
-"""Takes a set of pcap files to compare and creates a report"""
-
-import logging
-from logging import info, debug
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+from logging import debug
 from typing import List
-from rich.console import Console
+import datetime as dt
+from datetime import datetime
+
+from traffic_taffy.comparison import Comparison
 from traffic_taffy.dissectmany import PCAPDissectMany
-from traffic_taffy.dissector import (
-    PCAPDissectorType,
-    dissector_add_parseargs,
-    limitor_add_parseargs,
-    PCAPDissector,
-    check_dissector_level,
-)
+from traffic_taffy.dissector import PCAPDissectorLevel
+from traffic_taffy.dissection import Dissection
 
 
 class PcapCompare:
@@ -22,34 +16,35 @@ class PcapCompare:
 
     def __init__(
         self,
-        pcaps: List[str],
-        maximum_count: int | None = None,
+        pcap_files: List[str],
+        maximum_count: int = 0,  # where 0 == all
         deep: bool = True,
-        print_threshold: float = 0.0,
-        print_minimum_count: int | None = None,
-        print_match_string: str | None = None,
         pkt_filter: str | None = None,
-        only_positive: bool = False,
-        only_negative: bool = False,
         cache_results: bool = False,
-        bin_size: int | None = 3600,
-        dissection_level: PCAPDissectorType = PCAPDissectorType.COUNT_ONLY,
+        cache_file_suffix: str = "taffy",
+        bin_size: int | None = None,
+        dissection_level: PCAPDissectorLevel = PCAPDissectorLevel.COUNT_ONLY,
         between_times: List[int] | None = None,
+        ignore_list: List[str] = [],
     ) -> None:
-        self.pcaps = pcaps
+        self.pcap_files = pcap_files
         self.deep = deep
         self.maximum_count = maximum_count
-        self.print_threshold = print_threshold
-        self.print_minimum_count = print_minimum_count
-        self.print_match_string = print_match_string
         self.pkt_filter = pkt_filter
-        self.only_positive = only_positive
-        self.only_negative = only_negative
         self.cache_results = cache_results
         self.dissection_level = dissection_level
         self.between_times = between_times
         self.bin_size = bin_size
-        self.console = None
+        self.cache_file_suffix = cache_file_suffix
+        self.ignore_list = ignore_list
+
+    @property
+    def pcap_files(self):
+        return self._pcap_files
+
+    @pcap_files.setter
+    def pcap_files(self, new_pcap_files):
+        self._pcap_files = new_pcap_files
 
     @property
     def reports(self):
@@ -59,210 +54,135 @@ class PcapCompare:
     def reports(self, newvalue):
         self._reports = newvalue
 
-    def compare_dissections(self, dissection1: dict, dissection2: dict) -> dict:
+    def compare_dissections(self, left_side: dict, right_side: dict) -> dict:
         "compares the results from two reports"
 
         report = {}
 
-        # TODO: missing key in dissection2 (major items added)
-        keys = set(dissection1.keys())
-        keys = keys.union(dissection2.keys())
+        # TODO: missing key in right_side (major items added)
+        keys = set(left_side.keys())
+        keys = keys.union(right_side.keys())
         for key in keys:
-            dissection1_total = dissection1[key].total()
-            dissection2_total = dissection2[key].total()
             report[key] = {}
 
-            for subkey in dissection1[key].keys():
-                delta = 0.0
+            if key not in left_side:
+                left_side[key] = {}
+            left_side_total = sum(left_side[key].values())
+
+            if key not in right_side:
+                right_side[key] = {}
+            right_side_total = sum(right_side[key].values())
+
+            new_left_count = 0
+            for subkey in left_side[key].keys():
+                delta_percentage = 0.0
                 total = 0
-                if subkey in dissection1[key] and subkey in dissection2[key]:
-                    delta = (
-                        dissection2[key][subkey] / dissection2_total
-                        - dissection1[key][subkey] / dissection1_total
-                    )
-                    total = dissection2[key][subkey] + dissection1[key][subkey]
-                    ref_count = dissection1[key][subkey]
-                    comp_count = dissection2[key][subkey]
+                if subkey in right_side[key]:
+                    left_percentage = left_side[key][subkey] / left_side_total
+                    right_percentage = right_side[key][subkey] / right_side_total
+                    delta_percentage = right_percentage - left_percentage
+                    total = right_side[key][subkey] + left_side[key][subkey]
+                    left_count = left_side[key][subkey]
+                    right_count = right_side[key][subkey]
                 else:
-                    delta = -1.0
-                    total = dissection1[key][subkey]
-                    ref_count = dissection1[key][subkey]
-                    comp_count = 0
-
+                    delta_percentage = -1.0
+                    left_percentage = left_side[key][subkey] / left_side_total
+                    right_percentage = 0.0
+                    total = -left_side[key][subkey]
+                    left_count = left_side[key][subkey]
+                    right_count = 0
+                    new_left_count += 1
+
+                delta_absolute = right_count - left_count
                 report[key][subkey] = {
-                    "delta": delta,
+                    "delta_percentage": delta_percentage,
+                    "delta_absolute": delta_absolute,
                     "total": total,
-                    "ref_count": ref_count,
-                    "comp_count": comp_count,
+                    "left_count": left_count,
+                    "right_count": right_count,
+                    "left_percentage": left_percentage,
+                    "right_percentage": right_percentage,
                 }
 
-            for subkey in dissection2[key].keys():
+            new_right_count = 0
+            for subkey in right_side[key].keys():
                 if subkey not in report[key]:
-                    delta = 1.0
-                    total = dissection2[key][subkey]
-                    ref_count = 0
-                    comp_count = dissection2[key][subkey]
+                    delta_percentage = 1.0
+                    total = right_side[key][subkey]
+                    left_count = 0
+                    right_count = right_side[key][subkey]
+                    left_percentage = 0.0
+                    right_percentage = right_side[key][subkey] / right_side_total
+                    new_right_count += 1  # this value wasn't in the left
 
                     report[key][subkey] = {
-                        "delta": delta,
+                        "delta_percentage": delta_percentage,
+                        "delta_absolute": right_count,
                         "total": total,
-                        "ref_count": ref_count,
-                        "comp_count": comp_count,
+                        "left_count": left_count,
+                        "right_count": right_count,
+                        "left_percentage": left_percentage,
+                        "right_percentage": right_percentage,
                     }
 
-        return report
-
-    def filter_check(self, data: dict) -> bool:
-        "Returns true if we should include it"
-        delta: float = data["delta"]
-        total: int = data["total"]
-
-        if self.only_positive and delta <= 0:
-            return False
-
-        if self.only_negative and delta >= 0:
-            return False
-
-        if not self.print_threshold and not self.print_minimum_count:
-            # always print
-            return True
-
-        if self.print_threshold and not self.print_minimum_count:
-            # check print_threshold as a fraction
-            if abs(delta) > self.print_threshold:
-                return True
-        elif not self.print_threshold and self.print_minimum_count:
-            # just check print_minimum_count
-            if total > self.print_minimum_count:
-                return True
-        else:
-            # require both
-            if total > self.print_minimum_count and abs(delta) > self.print_threshold:
-                return True
-
-        return False
-
-    def init_console(self):
-        if not self.console:
-            self.console = Console()
-
-    def print_report(self, report: dict) -> None:
-        "prints a report to the console"
-
-        self.init_console()
-        for key in sorted(report):
-            reported: bool = False
-
-            if self.print_match_string and self.print_match_string not in key:
-                continue
-
-            for subkey, data in sorted(
-                report[key].items(), key=lambda x: x[1]["delta"], reverse=True
-            ):
-                if not self.filter_check(data):
-                    continue
-
-                # print the header
-                if not reported:
-                    print(f"====== {key}")
-                    reported = True
-
-                delta: float = data["delta"]
-
-                # apply some fancy styling
-                style = ""
-                if delta < -0.5:
-                    style = "[bold red]"
-                elif delta < 0.0:
-                    style = "[red]"
-                elif delta > 0.5:
-                    style = "[bold green]"
-                elif delta > 0.0:
-                    style = "[green]"
-                endstyle = style.replace("[", "[/")
-
-                # construct the output line with styling
-                subkey = PCAPDissector.make_printable(key, subkey)
-                line = f"  {style}{subkey:<50}{endstyle}"
-                line += f"{100*delta:>7.2f} {data['total']:>8} "
-                line += f"{data['ref_count']:>8} {data['comp_count']:>8}"
-
-                # print it to the rich console
-                self.console.print(line)
-
-    def print_header(self):
-        # This should match the spacing in print_report()
-        self.init_console()
-
-        style = ""
-        subkey = "Value"
-        endstyle = ""
-        delta = "Delta %"
-        total = "Total"
-        ref_count = "Left"
-        comp_count = "Right"
-
-        line = f"  {style}{subkey:<50}{endstyle}"
-        line += f"{delta:>7} {total:>8} "
-        line += f"{ref_count:>8} {comp_count:>8}"
-
-        self.console.print(line)
-
-    def print(self) -> None:
-        "outputs the results"
-        self.print_header()
-        for n, report in enumerate(self.reports):
-            title = report.get("title", f"report #{n}")
-            print(f"************ {title}")
-            self.print_report(report["report"])
+            if right_side_total == 0:
+                right_percent = 100
+            else:
+                right_percent = new_right_count / right_side_total
+            report[key][Dissection.NEW_RIGHT_SUBKEY] = {
+                "delta_absolute": new_right_count - new_left_count,
+                "total": new_left_count + new_right_count,
+                "left_count": new_left_count,
+                "right_count": new_right_count,
+                "left_percentage": new_left_count / left_side_total,
+                "right_percentage": right_percent,
+                "delta_percentage": (right_percent - new_left_count / left_side_total),
+            }
+
+        return Comparison(report)
 
     def load_pcaps(self) -> None:
         # load the first as a reference pcap
-        info(f"reading pcap files using level={self.dissection_level}")
         pdm = PCAPDissectMany(
-            self.pcaps,
+            self.pcap_files,
             bin_size=self.bin_size,
             maximum_count=self.maximum_count,
             pcap_filter=self.pkt_filter,
             cache_results=self.cache_results,
+            cache_file_suffix=self.cache_file_suffix,
             dissector_level=self.dissection_level,
+            ignore_list=self.ignore_list,
         )
         results = pdm.load_all()
         return results
 
-    def compare(self) -> None:
+    def compare(self) -> List[Comparison]:
         "Compares each pcap against the original source"
 
-        results = self.load_pcaps()
-        self.compare_all(results)
+        dissections = self.load_pcaps()
+        self.compare_all(dissections)
+        return self.reports
 
-    def compare_all(self, results):
+    def compare_all(self, dissections) -> List[Comparison]:
         reports = []
-        if len(self.pcaps) > 1:
+        if len(self.pcap_files) > 1:
             # multiple file comparison
-            reference = next(results)
-            for other in results:
+            reference = next(dissections)
+            for other in dissections:
                 # compare the two global summaries
-                reports.append(
-                    {
-                        "report": self.compare_dissections(
-                            reference["data"][0], other["data"][0]
-                        ),
-                        "title": f"{reference['file']} vs {other['file']}",
-                    }
-                )
 
+                report = self.compare_dissections(reference.data[0], other.data[0])
+                report.title = f"{reference.pcap_file} vs {other.pcap_file}"
+
+                reports.append(report)
         else:
             # deal with timestamps within a single file
-            results = list(results)
-            reference = results[0]
-            timestamps = list(reference["data"].keys())
+            reference = list(dissections)[0].data
+            timestamps = list(reference.keys())
             debug(
                 f"found {len(timestamps)} timestamps from {timestamps[2]} to {timestamps[-1]}"
             )
 
-            self.print_header()
-
             for timestamp in range(
                 2, len(timestamps)
             ):  # second real non-zero timestamp to last
@@ -280,12 +200,19 @@ class PcapCompare:
                 debug(f"comparing timestamps {time_left} and {time_right}")
 
                 report = self.compare_dissections(
-                    reference["data"][time_left], reference["data"][time_right]
+                    reference[time_left],
+                    reference[time_right],
+                )
+
+                title_left = datetime.fromtimestamp(time_left, dt.UTC).strftime(
+                    "%Y-%m-%d %H:%M:%S"
+                )
+                title_right = datetime.fromtimestamp(time_right, dt.UTC).strftime(
+                    "%Y-%m-%d %H:%M:%S"
                 )
 
-                title = f"time {time_left} vs time {time_right}"
-                print(f"************ {title}")
-                self.print_report(report)
+                report.title = f"time {title_left} vs time {title_right}"
+                reports.append(report)
 
                 continue
 
@@ -298,19 +225,14 @@ class PcapCompare:
                 # )
 
         self.reports = reports
+        return reports
 
 
-def parse_args():
-    "Parse the command line arguments."
-    parser = ArgumentParser(
-        formatter_class=ArgumentDefaultsHelpFormatter,
-        description=__doc__,
-        epilog="Exmaple Usage: ",
-    )
-
-    limiting_parser = limitor_add_parseargs(parser)
+def compare_add_parseargs(compare_parser, add_subgroup: bool = True):
+    if add_subgroup:
+        compare_parser = compare_parser.add_argument_group("Comparison result options")
 
-    limiting_parser.add_argument(
+    compare_parser.add_argument(
         "-t",
         "--print-threshold",
         default=0.0,
@@ -318,66 +240,48 @@ def parse_args():
         help="Don't print results with abs(percent) less than this threshold",
     )
 
-    limiting_parser.add_argument(
+    compare_parser.add_argument(
         "-P", "--only-positive", action="store_true", help="Only show positive entries"
     )
 
-    limiting_parser.add_argument(
+    compare_parser.add_argument(
         "-N", "--only-negative", action="store_true", help="Only show negative entries"
     )
 
-    limiting_parser.add_argument(
-        "-T",
-        "--between-times",
-        nargs=2,
+    compare_parser.add_argument(
+        "-x",
+        "--top-records",
+        default=None,
         type=int,
-        help="For single files, only display results between these timestamps",
+        help="Show the top N records from each section.",
     )
 
-    dissector_add_parseargs(parser)
-
-    debugging_group = parser.add_argument_group("Debugging options")
-
-    debugging_group.add_argument(
-        "--log-level",
-        "--ll",
-        default="info",
-        help="Define the logging verbosity level (debug, info, warning, error, ...).",
+    compare_parser.add_argument(
+        "-r",
+        "--reverse_sort",
+        action="store_true",
+        help="Reverse the sort order of reports",
     )
 
-    parser.add_argument("pcap_files", type=str, nargs="*", help="PCAP files to analyze")
-
-    args = parser.parse_args()
-    log_level = args.log_level.upper()
-    logging.basicConfig(level=log_level, format="%(levelname)-10s:\t%(message)s")
-
-    check_dissector_level(args.dissection_level)
-
-    return args
-
-
-def main():
-    args = parse_args()
-    pc = PcapCompare(
-        args.pcap_files,
-        maximum_count=args.packet_count,
-        print_threshold=float(args.print_threshold) / 100.0,
-        print_minimum_count=args.minimum_count,
-        print_match_string=args.match_string,
-        only_positive=args.only_positive,
-        only_negative=args.only_negative,
-        cache_results=args.cache_pcap_results,
-        dissection_level=args.dissection_level,
-        between_times=args.between_times,
-        bin_size=args.bin_size,
+    compare_parser.add_argument(
+        "-T",
+        "--between-times",
+        nargs=2,
+        type=int,
+        help="For single files, only display results between these timestamps",
     )
 
-    # compare the pcaps
-    pc.compare()
-
-    # print the results
-    pc.print()
+    return compare_parser
 
 
-if __name__ == "__main__":
-    main()
+def get_comparison_args(args):
+    return {
+        "maximum_count": args.packet_count or 0,
+        "print_threshold": float(args.print_threshold) / 100.0,
+        "minimum_count": args.minimum_count,
+        "match_string": args.match_string,
+        "only_positive": args.only_positive,
+        "only_negative": args.only_negative,
+        "top_records": args.top_records,
+        "reverse_sort": args.reverse_sort,
+    }

traffic_taffy/comparison.py

@@ -0,0 +1,26 @@
+from typing import Dict
+
+
+class Comparison:
+    def __init__(self, contents: list, title: str = ""):
+        self.contents = contents
+        self.title: str = title
+        self.printing_arguments: Dict[str] = {}
+
+    # title
+    @property
+    def title(self) -> str:
+        return self._title
+
+    @title.setter
+    def title(self, new_title):
+        self._title = new_title
+
+    # report contents -- actual data
+    @property
+    def contents(self):
+        return self._contents
+
+    @contents.setter
+    def contents(self, new_contents):
+        self._contents = new_contents