traffic-taffy 0.8.5__py3-none-any.whl → 0.9__py3-none-any.whl

traffic_taffy/tests/test_compare_results.py

@@ -1,5 +1,5 @@
 from collections import Counter
-from traffic_taffy.report import Report
+from traffic_taffy.reports.compareslicesreport import CompareSlicesReport
 from traffic_taffy.algorithms.statistical import ComparisonStatistical
 
 
@@ -10,7 +10,7 @@ def test_compare_statistical_algorithm():
     # this should be positive when right_data is larger
     expected = {
         "src": {
-            "a": Report(
+            "a": CompareSlicesReport(
                 total=20,
                 left_count=5,
                 right_count=15,
@@ -19,7 +19,7 @@ def test_compare_statistical_algorithm():
                 right_percentage=15.0 / 30.0,
                 delta_percentage=15.0 / 30.0 - 5.0 / 15.0,
             ),
-            "b": Report(
+            "b": CompareSlicesReport(
                 total=-10,  # only in 1
                 left_count=10,
                 right_count=0,
@@ -28,7 +28,7 @@ def test_compare_statistical_algorithm():
                 right_percentage=0.0,
                 delta_percentage=-1.0,
             ),
-            "c": Report(
+            "c": CompareSlicesReport(
                 total=15,  # only in 2
                 left_count=0,
                 right_count=15,
@@ -37,7 +37,7 @@ def test_compare_statistical_algorithm():
                 right_percentage=15.0 / 30.0,
                 delta_percentage=1.0,
             ),
-            "__NEW_VALUES__": Report(
+            "__NEW_VALUES__": CompareSlicesReport(
                 total=2,  # 1 on each side
                 left_count=1,  # b
                 right_count=1,  # c
@@ -50,6 +50,21 @@ def test_compare_statistical_algorithm():
     }
 
     algorithm = ComparisonStatistical()  # bogus file names
-    report = algorithm.compare_dissections(left_data[0], right_data[0])
 
-    assert report.contents == expected
+    class FakeDissection:
+        def __init__(self, data):
+            self._data = data
+
+        @property
+        def data(self):
+            return self._data
+
+        @property
+        def pcap_file(self):
+            return "bogus"
+
+    left = FakeDissection(left_data)
+    right = FakeDissection(right_data)
+
+    report = algorithm.compare_dissections(iter([left, right]))
+    assert report[0].contents == expected

traffic_taffy/tests/test_config.py

@@ -0,0 +1,149 @@
+from io import StringIO
+from traffic_taffy.config import Config
+from argparse import Namespace
+from tempfile import NamedTemporaryFile
+
+from argparse import ArgumentParser
+
+TESTCONFIG: str = """
+name: foo
+value: bar
+arry:
+  - 1
+  - 2
+"""
+
+
+def test_loading():
+    contents = StringIO(TESTCONFIG)
+
+    cfg = Config()
+    cfg.load_stream(contents)
+    assert cfg["name"] == "foo"
+    assert cfg["arry"][0] == 1  # truly sic!
+
+
+def test_namespace_loading():
+    cfg = Config()
+
+    arguments: Namespace = Namespace()
+    arguments.test_arg_one = 12
+    arguments.test_arg_two = {"a": "hello", "b": "world"}
+
+    cfg.load_namespace(arguments)
+
+    assert cfg["test_arg_one"] == 12
+    assert cfg["test_arg_two"]["b"] == "world"
+
+
+def test_as_namespace():
+    contents = StringIO(TESTCONFIG)
+
+    cfg = Config()
+    cfg.load_stream(contents)
+    assert cfg["name"] == "foo"
+    assert cfg["arry"][0] == 1  # truly sic!
+
+    args = cfg.as_namespace()
+
+    assert args.name == "foo"
+    assert args.arry[0] == 1
+
+
+def test_namespace_loading_and_mapping():
+    cfg = Config()
+
+    arguments: Namespace = Namespace()
+    arguments.test_arg_one = 12
+    arguments.test_arg_two = {"a": "hello", "b": "world"}
+
+    remap: dict = {"test_arg_one": "new_arg_one"}
+
+    cfg.load_namespace(arguments, mapping=remap)
+
+    assert cfg["new_arg_one"] == 12
+    assert cfg["test_arg_two"]["b"] == "world"
+
+    assert "test_arg_one" not in cfg
+
+
+def test_config_commandline_option():
+    cfg = Config()
+
+    with NamedTemporaryFile("w", suffix="yml") as fileh:
+        fileh.write(TESTCONFIG)
+        fileh.flush()
+
+        cfg.read_configfile_from_arguments(
+            ["foo", "bar", "-in-the-way", "--config", fileh.name, "--other", "-arg"]
+        )
+
+        assert cfg["name"] == "foo"
+        assert cfg["arry"][0] == 1
+
+
+def test_expected_full_usage():
+    # Create configuration in a yaml file
+    with NamedTemporaryFile("w", suffix="yml") as fileh:
+        fileh.write("question: 'how many roads must a man walk down?'\n")
+        fileh.write("reference: hitchhikers\n")
+        fileh.write("options:\n  - 1\n  - 2\n  - 3\n")
+        fileh.flush()
+
+        # set some application hard-code defaults
+        cfg = Config()
+        cfg["answer"] = 42
+        cfg["options"] = ["a", "b", "c"]
+
+        assert cfg == {"answer": 42, "options": ["a", "b", "c"]}
+
+        # define the arguments we want to pass (potentially overriding other variables)
+        passed_arguments = [
+            "--question",
+            "What do you get when you multiply six by seven?",
+            "--config",
+            fileh.name,
+            "-r",
+            "The guide",
+        ]
+
+        # now parse these to just read the config file
+        cfg.read_configfile_from_arguments(passed_arguments)
+
+        # ensure the configuration has been updated from the file contents, but not CLI args
+
+        assert cfg == {
+            "answer": 42,  # note: same
+            "options": [1, 2, 3],  # note: overwritten
+            "question": "how many roads must a man walk down?",  # note: same
+            "reference": "hitchhikers",  # note: same
+        }
+
+        # set up the command line options
+        parser = ArgumentParser()
+
+        parser.add_argument("-q", "--question", default=cfg["question"], type=str)
+        parser.add_argument("-a", "--answer", default=cfg["answer"], type=int)
+        parser.add_argument(
+            "-o", "--options", default=cfg["options"], nargs="+", type=int
+        )
+        parser.add_argument("-r", "--reference", default=cfg["reference"], type=str)
+        parser.add_argument("-c", "--config", type=str)
+        parser.add_argument("--only-unused-argument", "--", type=str)
+
+        args = parser.parse_args(passed_arguments)
+        cfg.load_namespace(args)
+
+        del cfg[
+            "config"
+        ]  # this will always be random tmp file and we don't need to check it
+        assert (
+            cfg
+            == {
+                "answer": 42,  # note: still a default
+                "options": [1, 2, 3],  # note: from config
+                "question": "What do you get when you multiply six by seven?",  # note: from cli
+                "reference": "The guide",  # note: from cli
+                "only_unused_argument": None,
+            }
+        )

traffic_taffy/tests/test_global_config.py

@@ -0,0 +1,33 @@
+from traffic_taffy.config import Config
+from traffic_taffy.taffy_config import TaffyConfig, taffy_default
+
+
+def test_multi_config():
+    c1 = Config()
+    c2 = Config()
+
+    c1["foo"] = 2
+    c2["foo"] = 3
+    assert c1["foo"] == 2
+
+
+def test_global_config():
+    c1 = TaffyConfig()
+    c2 = TaffyConfig()
+
+    c1["foo"] = 2
+    c2["foo"] = 3
+    assert c1["foo"] == 3
+
+
+def test_defaults():
+    taffy_default("a", "b")
+
+    c = TaffyConfig()
+    assert c["a"] == "b"
+
+    c["a"] = "c"  # override
+    assert c["a"] == "c"
+
+    taffy_default("a", "d")  # ignore overrides
+    assert c["a"] == "c"

traffic_taffy/tests/test_normalize.py

@@ -9,6 +9,7 @@ class ParentFaker(PcapGraphData):
         self.match_value = None
         self.minimum_count = 0
         self.bin_size = 1
+        self.match_expression = None
 
         super().__init__()
 

traffic_taffy/tests/test_pcap_dissector.py

@@ -50,10 +50,20 @@ def test_dissector_simple_callback() -> None:
     dpkt_engine.dissection.save(save_file)
 
     # create a new one to make sure it's blank
+    from traffic_taffy.taffy_config import TaffyConfig
+
+    config = TaffyConfig(
+        {
+            "dissect": {
+                "dissection_level": PCAPDissectorLevel.COUNT_ONLY.value,
+                "cache_results": True,
+            }
+        }
+    )
+
     pd = PCAPDissector(
         base_pcap,
-        dissector_level=PCAPDissectorLevel.COUNT_ONLY.value,
-        cache_results=True,
+        config,
     )
 
     pd.load()

traffic_taffy/tests/test_pcap_splitter.py

@@ -4,17 +4,33 @@ import logging
 from logging import debug
 from traffic_taffy.dissector import PCAPDissector, pcap_data_merge
 from traffic_taffy.dissection import Dissection
+from traffic_taffy.taffy_config import TaffyConfig
 from pcap_parallel import PCAPParallel
 
 test_pkl = "/tmp/test.pcap.pkl"
 
+default_config = TaffyConfig(
+    {
+        "dissect": {
+            "dissection_level": 10,
+            "filter": None,
+            "packet_count": 0,
+            "cache_pcap_results": False,
+            "bin_size": 1,
+            "cache_file_suffix": "taffy",
+            "ignore_list": [],
+            "layers": [],
+            "force_overwrite": False,
+            "force_load": False,
+        }
+    }
+)
+
 
 def buffer_callback(pcap_io_buffer):
     pd = PCAPDissector(
         pcap_io_buffer,
-        bin_size=0,
-        dissector_level=10,
-        cache_results=False,
+        default_config,
     )
     pd.load()
     return pd.dissection.data
@@ -62,9 +78,7 @@ def test_pcap_splitter():
         # create a bogus dissector
         pd = PCAPDissector(
             None,
-            bin_size=0,
-            dissector_level=10,
-            cache_results=False,
+            default_config,
         )
         pd.dissection = dissection
         dissection.save(test_pkl)
@@ -75,10 +89,7 @@ def test_pcap_splitter():
         normal_start_time = time.time()
         pd = PCAPDissector(
             test_pcap,
-            bin_size=0,
-            dissector_level=10,
-            cache_results=False,
-            maximum_count=maximum_count,
+            default_config,
         )
         pd.load()
         data2 = pd.dissection.data

traffic_taffy/tools/cache_info.py

@@ -1,6 +1,7 @@
 """Loads the cached data for a file to display the results about it."""
 
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, Namespace
+from argparse import ArgumentParser, Namespace
+from rich_argparse import RichHelpFormatter
 from pathlib import Path
 from rich import print
 import logging
@@ -10,7 +11,7 @@ import msgpack
 def parse_args() -> Namespace:
     """Parse the command line arguments."""
     parser = ArgumentParser(
-        formatter_class=ArgumentDefaultsHelpFormatter,
+        formatter_class=RichHelpFormatter,
         description=__doc__,
         epilog="Example Usage: taffy-cache-info something.taffy",
     )

traffic_taffy/tools/compare.py

@@ -1,27 +1,37 @@
 """Takes a set of pcap files to compare and creates a report."""
 
 import sys
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, Namespace
+from argparse import Namespace
+from argparse_with_config import ArgumentParserWithConfig
+from rich_argparse import RichHelpFormatter
 import logging
 from logging import error
 from traffic_taffy.output.console import Console
 from traffic_taffy.output.fsdb import Fsdb
+from traffic_taffy.taffy_config import TaffyConfig, taffy_default
 
 from traffic_taffy.compare import compare_add_parseargs, get_comparison_args
 from traffic_taffy.dissector import (
     dissector_add_parseargs,
     limitor_add_parseargs,
     dissector_handle_arguments,
+    TTD_CFG,
 )
 from traffic_taffy.compare import PcapCompare
 
+taffy_default("compare.fsdb", False)
 
-def parse_args() -> Namespace:
+
+def compare_parse_args() -> Namespace:
     """Parse the command line arguments."""
-    parser = ArgumentParser(
-        formatter_class=ArgumentDefaultsHelpFormatter,
+
+    config: TaffyConfig = TaffyConfig()
+
+    parser = ArgumentParserWithConfig(
+        formatter_class=RichHelpFormatter,
         description=__doc__,
         epilog="Example Usage: taffy-compare -C file1.pcap file2.pcap",
+        default_config=config,
     )
 
     output_options = parser.add_argument_group("Output format")
@@ -29,12 +39,13 @@ def parse_args() -> Namespace:
         "-f",
         "--fsdb",
         action="store_true",
+        config_path="compare.output_fsdb",
         help="Print results in an FSDB formatted output",
     )
 
-    limitor_parser = limitor_add_parseargs(parser)
-    compare_add_parseargs(limitor_parser, add_subgroup=False)
-    dissector_add_parseargs(parser)
+    limitor_add_parseargs(parser, config)
+    compare_add_parseargs(parser, config)
+    dissector_add_parseargs(parser, config)
 
     debugging_group = parser.add_argument_group("Debugging options")
 
@@ -42,6 +53,7 @@ def parse_args() -> Namespace:
         "--log-level",
         "--ll",
         default="info",
+        config_path="log_level",
         help="Define the logging verbosity level (debug, info, warning, error, ...).",
     )
 
@@ -53,15 +65,17 @@ def parse_args() -> Namespace:
 
     dissector_handle_arguments(args)
 
-    return args
+    return parser.config, args
 
 
 def main() -> None:
     """Run taffy-compare."""
-    args = parse_args()
+    config, args = compare_parse_args()
 
     # setup output options
-    printing_arguments = get_comparison_args(args)
+    config[TTD_CFG.KEY_DISSECTOR][TTD_CFG.FILTER_ARGUMENTS] = get_comparison_args(
+        config
+    )
 
     # get our files to compare (maybe just one)
     left = args.pcap_files.pop(0)
@@ -79,18 +93,7 @@ def main() -> None:
 
         pc = PcapCompare(
             files,
-            cache_results=args.cache_pcap_results,
-            cache_file_suffix=args.cache_file_suffix,
-            maximum_count=printing_arguments["maximum_count"],
-            dissection_level=args.dissection_level,
-            # between_times=args.between_times,  # TODO(hardaker): TBD
-            bin_size=args.bin_size,
-            ignore_list=args.ignore_list,
-            pcap_filter=args.filter,
-            layers=args.layers,
-            force_load=args.force_load,
-            force_overwrite=args.force_overwrite,
-            merge_files=args.merge,
+            config,
         )
 
         # compare the pcaps
@@ -101,9 +104,11 @@ def main() -> None:
             sys.exit()
 
         if args.fsdb:
-            output = Fsdb(None, printing_arguments)
+            output = Fsdb(None, config[TTD_CFG.KEY_DISSECTOR][TTD_CFG.FILTER_ARGUMENTS])
         else:
-            output = Console(None, printing_arguments)
+            output = Console(
+                None, config[TTD_CFG.KEY_DISSECTOR][TTD_CFG.FILTER_ARGUMENTS]
+            )
 
         for report in reports:
             # output results to the console
@@ -120,3 +125,6 @@ def main() -> None:
 
 if __name__ == "__main__":
     main()
+    config = TaffyConfig()
+    if config.get("dump", False):
+        config.dump()

traffic_taffy/tools/config.py

@@ -0,0 +1,83 @@
+"""Performs generic dissection of a PCAP file."""
+import sys
+import logging
+import yaml
+from traffic_taffy.taffy_config import TaffyConfig, TT_CFG
+from rich_argparse import RichHelpFormatter
+from argparse import ArgumentParser, Namespace
+
+# these force configuration token loading in a way ruff won't "fix"
+from traffic_taffy.dissector import TTD_CFG as TTD_CFG
+from traffic_taffy.compare import TTC_CFG as TTC_CFG
+from traffic_taffy.graph import TTG_CFG as TTG_CFG
+from traffic_taffy.tools.compare import compare_parse_args as compare_parse_args
+
+
+# we try to load a number of modules, but if the missing requirements aren't available
+# we don't fail here
+try:
+    from traffic_taffy.dissector_engine.scapy import (
+        DissectionEngineScapy as DissectionEngineScapy,
+    )
+except ModuleNotFoundError:
+    logging.debug("scapy module not loadable")
+
+try:
+    from traffic_taffy.hooks.ip2asn import ip_to_asn as ip_to_asn
+except ModuleNotFoundError:
+    logging.debug("ip2asn module not loadable")
+
+try:
+    from traffic_taffy.hooks.psl import split_dns_names as split_dns_names
+except ModuleNotFoundError:
+    logging.debug("psl module not loadable")
+
+
+def main() -> None:
+    """Dissect a pcap file and report contents."""
+
+    def parse_args() -> Namespace:
+        """Parse the command line arguments."""
+
+        config: TaffyConfig = TaffyConfig()
+        config.config_option_names = ["-y", "--config"]
+        config[TT_CFG.LOG_LEVEL] = "info"
+
+        config.read_configfile_from_arguments(sys.argv)
+
+        parser = ArgumentParser(
+            formatter_class=RichHelpFormatter,
+            description=__doc__,
+            epilog="Example Usage: taffy-config > defaults.yml",
+        )
+
+        parser.add_argument(
+            "-y",
+            "--config",
+            default=None,
+            type=str,
+            help="Configuration file (YAML) to load.",
+        )
+
+        parser.add_argument(
+            "--log-level",
+            "--ll",
+            default="info",
+            help="Define the logging verbosity level (debug, info, warning, error, fotal, critical).",
+        )
+
+        args = parser.parse_args()
+        log_level = args.log_level.upper()
+        logging.basicConfig(level=log_level, format="%(levelname)-10s:\t%(message)s")
+
+        config.load_namespace(args)
+        return config
+
+    config = parse_args()
+    config.as_namespace()
+
+    print(yaml.dump(dict(config)))
+
+
+if __name__ == "__main__":
+    main()