traffic-taffy 0.8.1__py3-none-any.whl → 0.9__py3-none-any.whl

traffic_taffy/graphdata.py

@@ -8,9 +8,19 @@ from traffic_taffy.dissection import Dissection
 class PcapGraphData:
     """A base class for storing/transforming data (frequently to be graphed)."""
 
-    def __init__(self):
+    def __init__(
+        self,
+        match_string: str = None,
+        match_value: str = None,
+        minimum_count: int = None,
+        match_expression: str = None,
+    ):
         """Create an instance of a PcapGraphData."""
         self.dissections = []
+        self.match_string = match_string
+        self.match_value = match_value
+        self.minimum_count = minimum_count
+        self.match_expression = match_expression
 
     @property
     def dissections(self) -> list:
@@ -21,7 +31,7 @@ class PcapGraphData:
     def dissections(self, newvalue: list) -> None:
         self._dissections = newvalue
 
-    def normalize_bins(self, dissection: Dissection) -> dict:
+    def normalize_bins(self, dissection: Dissection, minimalize: bool = False) -> dict:
         """Transform a dissection's list of data into a dictionary."""
         results: dict = {}
         time_keys: list = list(dissection.data.keys())
@@ -37,6 +47,7 @@ class PcapGraphData:
             match_value=self.match_value,
             minimum_count=self.minimum_count,
             make_printable=True,
+            match_expression=self.match_expression,
         ):
             index = key + "=" + subkey
             results["count"].append(int(value))

traffic_taffy/hooks/ip2asn.py

@@ -5,18 +5,29 @@ import ip2asn
 from traffic_taffy.hooks import register_hook
 from traffic_taffy.dissector import POST_DISSECT_HOOK
 from traffic_taffy.dissection import Dissection
+from traffic_taffy.taffy_config import taffy_default, TaffyConfig
 
-if not Path("ip2asn-combined.tsv").exists():
-    error("The ip2asn plugin requires a ip2asn-combined.tsv in this directory")
-    error("Please download it from https://iptoasn.com/")
+i2a = None
 
-info("loading ip2asn-combined.tsv")
-i2a = ip2asn.IP2ASN("ip2asn-combined.tsv")
-info("  ... loaded")
+taffy_default("modules.ip2asn.database", "ip2asn-combined.tsv")
 
 
 @register_hook(POST_DISSECT_HOOK)
 def ip_to_asn(dissection: Dissection, **kwargs):
+    global i2a
+
+    if i2a is None:
+        config = TaffyConfig()
+        db_path = config.get_dotnest("modules.ip2asn.database")
+
+        if not Path(db_path).exists():
+            error("The ip2asn plugin requires a ip2asn-combined.tsv in this directory")
+            error("Please download it from https://iptoasn.com/")
+
+        info(f"loading {db_path}")
+        i2a = ip2asn.IP2ASN(db_path)
+        info("  ... loaded")
+
     timestamps = dissection.data.keys()
 
     for timestamp in timestamps:
@@ -33,10 +44,12 @@ def ip_to_asn(dissection: Dissection, **kwargs):
                 for value in dissection.data[timestamp][key]:
                     count = dissection.data[timestamp][key][value]
                     details = None
+
+                    # TODO(hardaker): doesn't handle bytes addresses from dpkt
                     try:
                         details = i2a.lookup_address(value)
                     except Exception:
-                        debug("failed to parse address: {value}")
+                        debug(f"failed to parse address: {value}")
                     if not details:
                         continue
 

traffic_taffy/hooks/labels.py

@@ -0,0 +1,45 @@
+"""A traffic taffy module to split the last five labels and count them"""
+from traffic_taffy.hooks import register_hook
+from traffic_taffy.dissector import POST_DISSECT_HOOK
+from traffic_taffy.dissection import Dissection
+
+import dnssplitter
+
+splitter = dnssplitter.DNSSplitter()
+splitter.init_tree()
+
+
+@register_hook(POST_DISSECT_HOOK)
+def split_dns_names(dissection: Dissection, **kwargs):
+    """Split a DNS name into pieces and count the last 5 labels"""
+    timestamps = dissection.data.keys()
+
+    for timestamp in timestamps:
+        keys = list(dissection.data[timestamp].keys())
+
+        for key in keys:
+            key = str(key)
+            if (
+                key.endswith("_qname")
+                or key.endswith("_mname")
+                or key.endswith("_rrname")
+            ):
+                for value in dissection.data[timestamp][key]:
+                    count = dissection.data[timestamp][key][value]
+
+                    parts = value.split(".")
+                    if parts[-1] == "":
+                        parts = parts[:-1]  # drop the empty end "." split
+                    dissection.data[timestamp][key + "_tld"][parts[-1]] += count
+                    if len(parts) > 1:
+                        dissection.data[timestamp][key + "_sld"][parts[-2]] += count
+                        if len(parts) > 2:
+                            dissection.data[timestamp][key + "_3ld"][parts[-3]] += count
+                            if len(parts) > 3:
+                                dissection.data[timestamp][key + "_4ld"][
+                                    parts[-4]
+                                ] += count
+                                if len(parts) > 4:
+                                    dissection.data[timestamp][key + "_5ld"][
+                                        parts[-5]
+                                    ] += count

traffic_taffy/hooks/psl.py

@@ -1,15 +1,33 @@
+from logging import info
+import dnssplitter
+
 from traffic_taffy.hooks import register_hook
 from traffic_taffy.dissector import POST_DISSECT_HOOK
 from traffic_taffy.dissection import Dissection
+from traffic_taffy.taffy_config import taffy_default, TaffyConfig
 
-import dnssplitter
+splitter = None
 
-splitter = dnssplitter.DNSSplitter()
-splitter.init_tree()
+taffy_default("modules.psl.database", "__internal__")
 
 
 @register_hook(POST_DISSECT_HOOK)
 def split_dns_names(dissection: Dissection, **kwargs):
+    global splitter
+
+    if not splitter:
+        config = TaffyConfig()
+        splitter = dnssplitter.DNSSplitter()
+
+        path = config.get_dotnest("modules.psl.database")
+
+        if path == "__internal__":
+            splitter.init_tree()
+            info("loading PSL data from internal")
+        else:
+            info(f"loading PSL from {path}")
+            splitter.load_psl_file(path)
+
     timestamps = dissection.data.keys()
 
     for timestamp in timestamps:

traffic_taffy/output/__init__.py

@@ -33,11 +33,11 @@ class Output:
     def output_options(self, new_output_options: dict) -> None:
         self._output_options = new_output_options
 
-    def output(self, report: Comparison | None = None) -> None:
-        """Dump a report to the output stream."""
-        if not report:
-            report = self.report
-        contents = report.contents
+    def output(self, comparison: Comparison | None = None) -> None:
+        """Dump a comparison to the output stream."""
+        if not comparison:
+            comparison = self.report
+        contents = comparison.contents
 
         first_of_anything: bool = True
 
@@ -72,19 +72,20 @@ class Output:
 
             # TODO(hardaker): we don't do match_value here?
 
+            sort_by = comparison.sort_by
             record_count = 0
             for subkey, data in sorted(
                 contents[key].items(),
                 key=lambda x: getattr(x[1], sort_by),
                 reverse=sort_order,
             ):
-                if not self.filter_check(data):
+                if not data.filter_check(self.output_options):
                     continue
 
                 # print the header
                 if not reported:
                     if first_of_anything:
-                        self.output_start(report)
+                        self.output_start(comparison, contents[key][subkey])
                         first_of_anything = False
 
                     self.output_new_section(key)
@@ -106,44 +107,3 @@ class Output:
     def output_close(self) -> None:
         """Close the output stream."""
         return
-
-    def filter_check(self, data: dict) -> bool:
-        """Return true if we should include it."""
-        delta: float = data.delta_percentage
-        total: int = data.total
-
-        if self.output_options["only_positive"] and delta <= 0:
-            return False
-
-        if self.output_options["only_negative"] and delta >= 0:
-            return False
-
-        if (
-            not self.output_options["print_threshold"]
-            and not self.output_options["minimum_count"]
-        ):
-            # always print
-            return True
-
-        if (
-            self.output_options["print_threshold"]
-            and not self.output_options["minimum_count"]
-        ):
-            # check output_options["print_threshold"] as a fraction
-            if abs(delta) > self.output_options["print_threshold"]:
-                return True
-        elif (
-            not self.output_options["print_threshold"]
-            and self.output_options["minimum_count"]
-        ):
-            # just check output_options["minimum_count"]
-            if total > self.output_options["minimum_count"]:
-                return True
-        elif (
-            total > self.output_options["minimum_count"]
-            and abs(delta) > self.output_options["print_threshold"]
-        ):
-            # require both
-            return True
-
-        return False

traffic_taffy/output/console.py

@@ -7,8 +7,11 @@ from rich.console import Console as RichConsole
 from traffic_taffy.output import Output
 from traffic_taffy.dissection import Dissection
 
+import dataclasses
+
 if TYPE_CHECKING:
     from traffic_taffy.comparison import Comparison
+    from traffic_taffy.reports import Report
 
 
 class Console(Output):
@@ -29,12 +32,12 @@ class Console(Output):
         if not self.console:
             self.console = RichConsole()
 
-    def output_start(self, report: Comparison) -> None:
+    def output_start(self, comparison: Comparison, report: Report) -> None:
         """Print the header about columns being displayed."""
         # This should match the spacing in print_contents()
         self.init_console()
 
-        self.console.print(f"======== {report.title}")
+        self.console.print(f"======== {comparison.title}")
         if self.have_done_header:
             return
 
@@ -43,17 +46,15 @@ class Console(Output):
         style = ""
         subkey = "Value"
         endstyle = ""
-        left_count = "Left"
-        right_count = "Right"
-        actual_delta = "Delta"
 
-        left_percent = "Left %"
-        right_percent = "Right %"
-        percent_delta = "Delta-%"
+        field_values = {field.name: field.name for field in dataclasses.fields(report)}
 
-        line = f"  {style}{subkey:<50}{endstyle}"
-        line += f" {left_count:>8} {right_count:>8} {actual_delta:>8}"
-        line += f" {left_percent:>8} {right_percent:>8}  {percent_delta:>7}"
+        line = report.header_string.format(
+            style=style,
+            endstyle=endstyle,
+            subkey=subkey,
+            **field_values,
+        )
 
         self.console.print(line)
 
@@ -63,25 +64,36 @@ class Console(Output):
 
     def output_record(self, key: str, subkey: Any, data: Dict[str, Any]) -> None:
         """Print a report to the console."""
-        delta_percentage: float = data.delta_percentage
 
-        # apply some fancy styling
         style = ""
-        if delta_percentage < -Console.BOLD_LIMIT:
-            style = "[bold red]"
-        elif delta_percentage < Console.POSITIVE:
-            style = "[red]"
-        elif delta_percentage > Console.BOLD_LIMIT:
-            style = "[bold green]"
-        elif delta_percentage > Console.POSITIVE:
-            style = "[green]"
-        endstyle = style.replace("[", "[/")
+        endstyle = ""
+        if getattr(data, "delta_percentage", None):
+            delta_percentage: float = data.delta_percentage
+
+            # apply some styling depending on range
+            if delta_percentage < -Console.BOLD_LIMIT:
+                style = "[bold red]"
+            elif delta_percentage < Console.POSITIVE:
+                style = "[red]"
+            elif delta_percentage > Console.BOLD_LIMIT:
+                style = "[bold green]"
+            elif delta_percentage > Console.POSITIVE:
+                style = "[green]"
+            endstyle = style.replace("[", "[/")
 
         # construct the output line with styling
         subkey = Dissection.make_printable(key, subkey)
-        line = f"  {style}{subkey:<50}{endstyle}"
-        line += f" {data.left_count:>8} {data.right_count:>8} {data.delta_absolute:>8}"
-        line += f" {100*data.left_percentage:>7.2f} {100*data.right_percentage:>7.2f}  {100*delta_percentage:>7.2f}"
+
+        field_values = {
+            field.name: getattr(data, field.name) for field in dataclasses.fields(data)
+        }
+
+        line = data.format_string.format(
+            style=style,
+            endstyle=endstyle,
+            subkey=subkey,
+            **field_values,
+        )
 
         # print it to the rich console
         self.console.print(line)

traffic_taffy/output/fsdb.py

@@ -6,6 +6,9 @@ from typing import Any
 from traffic_taffy.output import Output
 from traffic_taffy.dissection import Dissection
 from traffic_taffy.comparison import Comparison
+from traffic_taffy.reports import Report
+
+import dataclasses
 
 
 class Fsdb(Output):
@@ -17,39 +20,42 @@ class Fsdb(Output):
         self.console = None
         self.have_done_header = False
         self.in_report = None
+        self.fsdb = None
+
+    def init_fsdb(self, firstreport):
+        self.fields = dataclasses.fields(firstreport)
+        self.columns = []
+        self.converters = []
+
+        for field in self.fields:
+            self.columns.append(field.name)
+            self.converters.append(field.type)
 
         self.fsdb = pyfsdb.Fsdb(out_file_handle=sys.stdout)
+
         self.fsdb.out_column_names = [
             "report",
-            "Key",
+            "key",
             "subkey",
-            "left",
-            "right",
-            "delta",
-            "left_fraction",
-            "right_fraction",
-            "delta_fraction",
-        ]
-        self.fsdb.converters = [str, str, str, int, int, int, float, float, float]
-
-    def output_start(self, report: Comparison) -> None:
+        ] + self.columns
+        self.fsdb.converters = [str, str, str] + self.converters
+
+    def output_start(self, comparison: Comparison, report: Report) -> None:
         """Print the header about columns being displayed."""
         # This should match the spacing in print_contents()
-        self.in_report = report.title
+        self.in_report = comparison.title
 
     def output_record(self, key: str, subkey: Any, data: dict) -> None:
         """Print a report to the console."""
+        if self.fsdb is None:
+            self.init_fsdb(data)
+
         subkey = Dissection.make_printable(key, subkey)
         self.fsdb.append(
             [
                 self.in_report,
                 key,
                 subkey,
-                data.left_count,
-                data.right_count,
-                data.delta_absolute,
-                data.left_percentage,
-                data.right_percentage,
-                data.delta_percentage,
             ]
+            + [getattr(data, field.name) for field in dataclasses.fields(data)]
         )

traffic_taffy/reports/__init__.py

@@ -0,0 +1,5 @@
+"""A base class for report containers."""
+
+
+class Report:
+    """A bogus base class for containers for typing"""

traffic_taffy/reports/compareslicesreport.py

@@ -0,0 +1,85 @@
+"""Report for storing a comparison between two different dissections."""
+
+from __future__ import annotations
+from typing import Dict
+from dataclasses import dataclass
+from traffic_taffy.reports import Report
+
+
+@dataclass
+class CompareSlicesReport(Report):
+    """Report for storing a comparison between two different dissections."""
+
+    delta_percentage: float
+    delta_absolute: int
+    total: int
+    left_count: int
+    right_count: int
+    left_percentage: float
+    right_percentage: float
+
+    @property
+    def formatting(self) -> Dict[str, str]:
+        """Formatting field recommendations."""
+        return {
+            "delta_percentage": ">7.2f",
+            "delta_absolute": ">8",
+            "total": None,
+            "left_count": ">8",
+            "right_count": ">8",
+            "left_percentage": ">7.2f",
+            "right_percentage": ">7.2f",
+        }
+
+    @property
+    def header_string(self) -> str:
+        """Header string."""
+        line = "  {style}{subkey:<50}{endstyle}"
+        line += " {left_count:>8} {right_count:>8} {delta_absolute:>8}"
+        line += " {left_percentage:>7} {right_percentage:>7}  {delta_percentage:>7}"
+
+        return line
+
+    @property
+    def format_string(self) -> str:
+        """Formatting string for each printed line."""
+        line = "  {style}{subkey:<50}{endstyle}"
+        line += " {left_count:>8} {right_count:>8} {delta_absolute:>8}"
+        line += " {left_percentage:>7.2f} {right_percentage:>7.2f}  {delta_percentage:>7.2f}"
+
+        return line
+
+    def filter_check(self, output_options: dict) -> bool:
+        """Return true if we should include it."""
+        delta: float = self.delta_percentage
+        total: int = self.total
+
+        if output_options["only_positive"] and delta <= 0:
+            return False
+
+        if output_options["only_negative"] and delta >= 0:
+            return False
+
+        if (
+            not output_options["print_threshold"]
+            and not output_options["minimum_count"]
+        ):
+            # always print
+            return True
+
+        if output_options["print_threshold"] and not output_options["minimum_count"]:
+            # check output_options["print_threshold"] as a fraction
+            if abs(delta) > output_options["print_threshold"]:
+                return True
+        elif not output_options["print_threshold"] and output_options["minimum_count"]:
+            # just check output_options["minimum_count"]
+            if total > output_options["minimum_count"]:
+                return True
+        elif (
+            total > output_options["minimum_count"]
+            and abs(delta) > output_options["print_threshold"]
+        ):
+            # require both
+            return True
+
+        return False

traffic_taffy/reports/correlationchangereport.py

@@ -0,0 +1,54 @@
+"""Report for storing correlations between signals."""
+
+from __future__ import annotations
+from typing import Dict
+from dataclasses import dataclass
+from traffic_taffy.reports import Report
+
+
+@dataclass
+class CorrelationChangeReport(Report):
+    """Report for storing correlations between signals."""
+
+    left_correlation: float
+    right_correlation: float
+    delta_correlation: float
+    timestamp: int
+
+    @property
+    def formatting(self) -> Dict[str, str]:
+        """Formatting field recommendations."""
+        return {
+            "timestamp": ">7",
+            "left_correlation": ">7.2f",
+            "right_correlation": ">7.2f",
+            "delta_correlation": ">7.2f",
+        }
+
+    @property
+    def header_string(self) -> str:
+        """Formatting string for each printed line."""
+        line = "  {style}{subkey:<50}{endstyle}"
+        line += " {timestamp:>10}"
+        line += " {left_correlation:>17}"
+        line += " {right_correlation:>17}"
+        line += " {delta_correlation:>17}"
+
+        return line
+
+    @property
+    def format_string(self) -> str:
+        """Formatting string for each printed line."""
+        line = "  {style}{subkey:<50}{endstyle}"
+        line += " {timestamp:>10}"
+        line += " {left_correlation:>17.2f}"
+        line += " {right_correlation:>17.2f}"
+        line += " {delta_correlation:>17.2f}"
+
+        return line
+
+    def filter_check(self, output_options: dict) -> bool:
+        """Return true if we should include it
+
+        (which is always as we pre-filter for correlations)."""
+        return True

traffic_taffy/reports/correlationreport.py

@@ -0,0 +1,42 @@
+"""Report for storing correlations between signals."""
+
+from __future__ import annotations
+from typing import Dict
+from dataclasses import dataclass
+from traffic_taffy.reports import Report
+
+
+@dataclass
+class CorrelationReport(Report):
+    """Report for storing correlations between signals."""
+
+    correlation: float
+
+    @property
+    def formatting(self) -> Dict[str, str]:
+        """Formatting field recommendations."""
+        return {
+            "correlation": ">7.2f",
+        }
+
+    @property
+    def header_string(self) -> str:
+        """Formatting string for each printed line."""
+        line = "  {style}{subkey:<50}{endstyle}"
+        line += " {correlation:>11}"
+
+        return line
+
+    @property
+    def format_string(self) -> str:
+        """Formatting string for each printed line."""
+        line = "  {style}{subkey:<50}{endstyle}"
+        line += " {correlation:>11.2f}"
+
+        return line
+
+    def filter_check(self, output_options: dict) -> bool:
+        """Return true if we should include it
+
+        (which is always as we pre-filter for correlations)."""
+        return True

traffic_taffy/taffy_config.py

@@ -0,0 +1,44 @@
+"""A global configuration storage class that can be easily accessed."""
+
+from typing import Any
+
+from traffic_taffy.config import Config
+
+
+class TT_CFG:
+    LOG_LEVEL: str = "log_level"
+    CACHE_RESULTS: str = "cache_results"
+
+
+taffy_global_defaults = {"log_level": "info"}
+
+
+class TaffyConfig(object):
+    """A global configuration storage class that can be easily accessed."""
+
+    _instance = None
+
+    def __new__(class_obj, *args, **kwargs):
+        if not isinstance(class_obj._instance, Config):
+            class_obj._instance = Config(*args, **kwargs)
+            class_obj._instance.update(taffy_global_defaults)
+        return class_obj._instance
+
+
+def taffy_default(parameter: str, value: Any) -> bool:
+    """"""
+    config = TaffyConfig()
+    try:
+        value = (
+            config.get_dotnest(parameter, return_none=False) is not None
+        )  # ignore any value
+    except ValueError:
+        # a value doesn't exist, so create it
+        config.set_dotnest(parameter, value)
+        return True
+
+    if value is None:
+        config.set_dotnest(parameter, value)
+        return True
+
+    return False