traffic-taffy 0.8.1__tar.gz → 0.9__tar.gz

{traffic_taffy-0.8.1 → traffic_taffy-0.9}/.gitignore

@@ -7,3 +7,4 @@ build
 dist
 *.png
 *.rej
+*.taffy

{traffic_taffy-0.8.1 → traffic_taffy-0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: traffic-taffy
-Version: 0.8.1
+Version: 0.9
 Summary: A tool for doing differential analysis of pcap files
 Project-URL: Homepage, https://traffic-taffy.github.io/
 Author-email: Wes Hardaker <opensource@hardakers.net>
@@ -8,8 +8,10 @@ License-File: LICENSE.txt
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
+Requires-Dist: argparse-with-config>=1.1.4
 Requires-Dist: cryptography
 Requires-Dist: dnssplitter
+Requires-Dist: dotnest>=1.0
 Requires-Dist: dpkt
 Requires-Dist: ip2asn
 Requires-Dist: msgpack
@@ -19,6 +21,7 @@ Requires-Dist: pyfsdb
 Requires-Dist: pyopenssl==22.1.0
 Requires-Dist: pyqt6-charts
 Requires-Dist: rich
+Requires-Dist: rich-argparse
 Requires-Dist: scapy
 Requires-Dist: seaborn
 Description-Content-Type: text/markdown

{traffic_taffy-0.8.1 → traffic_taffy-0.9}/pyproject.toml

@@ -24,14 +24,20 @@ dependencies = [
     "pyfsdb",
     "PyQt6-Charts",
     "rich",
+    "rich_argparse",
     "scapy",
     "seaborn",
     "cryptography",
     "pyOpenSSL==22.1.0",
     "dnssplitter",
     "ip2asn",
+    "dotnest>=1.0",
+    "argparse-with-config>=1.1.4",
 ]
 
+[project.package_data]
+"traffic_taffy.iana" = ['tables.msgpak']
+
 [project.scripts]
 taffy-cache-info = "traffic_taffy.tools.cache_info:main"
 taffy-compare = "traffic_taffy.tools.compare:main"
@@ -39,6 +45,7 @@ taffy-dissect = "traffic_taffy.tools.dissect:main"
 taffy-explorer = "traffic_taffy.tools.explorer:main"
 taffy-graph = "traffic_taffy.tools.graph:main"
 taffy-export = "traffic_taffy.tools.export:main"
+taffy-config = "traffic_taffy.tools.config:main"
 
 [project.urls]
 Homepage = "https://traffic-taffy.github.io/"
@@ -70,6 +77,10 @@ ignore = ["E501", "I001", "PLR0913", "ANN101", "ANN204",
 "BLE001",
 # allow for loop variable overrides
 "PLW2901",
+# disable "no blank line before class"
+"D203",
+# disable multi-line-summary-second-line
+"D213",
 ]
 fixable = ["ALL"]  # gulp
 # select = ["ALL"]

traffic_taffy-0.9/traffic_taffy/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "0.9"

traffic_taffy-0.9/traffic_taffy/algorithms/__init__.py

@@ -0,0 +1,21 @@
+"""traffic-taffy algorithm produce comparisons between different datasets."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+from logging import error
+
+if TYPE_CHECKING:
+    from traffic_taffy.dissection import Dissection
+    from traffic_taffy.reports import Report
+
+
+class ComparisonAlgorithm:
+    """A base class for all comparison algorithms."""
+
+    def __init__(self):
+        """Construct a ComparisonAlgorithm."""
+
+    def compare_dissections(self, _dissections: List[Dissection]) -> List[Report]:
+        """Compare dissections base function just to warn things are not implemented."""
+        error("code failure: base class compare_two_dissections should never be called")
+        raise ValueError

traffic_taffy-0.9/traffic_taffy/algorithms/comparecorrelation.py

@@ -0,0 +1,164 @@
+"""Compares datasets using DataFrame's correlation."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+import pandas as pd
+import numpy as np
+
+from logging import debug, warning, info
+
+from traffic_taffy.algorithms.compareseries import ComparisonSeriesAlgorithm
+from traffic_taffy.reports.correlationreport import CorrelationReport
+from traffic_taffy.comparison import Comparison, OrganizedReports
+from traffic_taffy.taffy_config import TaffyConfig, taffy_default
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
+    from numpy import ndarray
+
+taffy_default("algorithms.correlation.minimum_correlation", 0.8)
+taffy_default("algorithms.correlation.correlation_method", "spearman")
+taffy_default("algorithms.correlation.max_pivot", 1000)
+
+
+class CompareCorrelation(ComparisonSeriesAlgorithm):
+    """Compare series using the pandas correlation."""
+
+    def __init__(
+        self,
+        timestamps: List[int] | None = None,
+        match_string: str | None = None,
+        match_value: str | None = None,
+        minimum_count: int | None = None,
+        make_printable: bool = False,
+        match_expression: str | None = None,
+    ):
+        """Create a CompareCorrelation instance.
+
+        Valid methods: kendall, pearson, spearman, corrcoef
+
+        speed-wise; pearson < spearman < corrcoef < kendall
+
+        accuracy-wise:
+            corrcoef: not great (uses np.corrcoef)
+            pearson: better but, not good
+            spearman: best
+            kendall: best
+        """
+        super().__init__(
+            timestamps,
+            match_string,
+            match_value,
+            minimum_count,
+            make_printable,
+            match_expression,
+        )
+        self.method = None
+
+    def compare_series(
+        self, df: DataFrame, indexes: ndarray | None = None
+    ) -> List[CorrelationReport]:
+        """Compare a bunch of series using correlation.
+
+        This tries to do a comparison in a faster path if the number
+        of keys are reasonable (for if not a pivot will consume all
+        available memory)
+        """
+
+        config = TaffyConfig()
+        minimum_correlation = float(
+            config.get_dotnest("algorithms.correlation.minimum_correlation")
+        )
+        self.minimum_correlation = minimum_correlation
+
+        max_pivot = int(config.get_dotnest("algorithms.correlation.max_pivot"))
+        method = config.get_dotnest("algorithms.correlation.correlation_method")
+        self.method = method
+
+        indexes = df["index"].unique()
+        num_indexes = len(indexes)
+        if num_indexes > max_pivot:
+            # we assume this is arbitrarily too large
+            # use the slower parent version instead
+            warning(
+                f"too many indexes ({num_indexes} > {max_pivot}) == using slower routine to conserve memory"
+            )
+            return super().compare_series(df, indexes)
+
+        info(f"Studying correlation of {num_indexes} indexes")
+
+        for key in ["subkey", "index", "filename"]:
+            del df[key]
+        df = df.pivot_table(
+            columns=["key"], index=["time"], values="count", fill_value=0
+        )
+
+        # indexes have changed
+        indexes = df.columns.to_list()
+
+        # use pandas internal kendall
+        # TODO(hardaker): np.corrcoef is multi-core but is pearsons
+        # TODO(hardaker): scipy.stat.kendalltau is kendall,
+        #                 but can only do one at a time
+
+        # TODO(hardaker): df.corr() returns different numbers here
+        # than inside compare_two_series!!
+
+        reports: OrganizedReports = {}
+
+        if method == "corrcoef":
+            np_array = df.to_numpy()
+            results = np.corrcoef(np_array)
+            for numx, column_left in enumerate(indexes):
+                for numy, column_right in enumerate(indexes[numx + 1 :]):
+                    value = results[numx][numy]
+                    # if value > minimum_correlation:
+                    #     print(
+                    #         f"{column_left:<30} similar to {column_right:<30}: {value}"
+                    #     )
+            return reports
+
+        # default to using the datafram corr method instead
+        df.fillna(0, inplace=True)
+        results = df.corr(method=method)
+
+        for num, column_left in enumerate(indexes):
+            for column_right in indexes[num + 1 :]:
+                value = results[column_left][column_right]
+                if value > minimum_correlation:
+                    # print(f"{column_left:<30} similar to {column_right:<30}: {value}")
+                    if column_left not in reports:
+                        reports[column_left] = {}
+                    reports[column_left][column_right] = CorrelationReport(
+                        value,
+                    )
+        return [Comparison(reports, "Correlation Report", "correlation")]
+
+    def compare_two_series(
+        self,
+        column_left: str,
+        series_left: list,
+        column_right: str,
+        series_right: list,
+        reports: OrganizedReports = None,
+    ) -> dict:
+        """Compare two series using the dataframe correlation algorithms."""
+        debug(f"correlation comparing {column_left} and {column_right}")
+        both = pd.concat([series_left, series_right], axis=1)
+        both.fillna(0, inplace=True)
+
+        # Note actually faster -- about the same as df.corr
+        # import scipy
+        # results = scipy.stats.kendalltau(both['left'], both['right'])
+        # value = results.statistic
+
+        results = both.corr(method=self.method)
+        value = results["left"][1]
+        debug(f"{column_left:<30} similar to {column_right:<30}: {value}")
+
+        if value > self.minimum_correlation:
+            # print(f"{column_left:<30} similar to {column_right:<30}: {value}")
+
+            return CorrelationReport(value)
+
+        return

traffic_taffy-0.9/traffic_taffy/algorithms/comparecorrelationchanges.py

@@ -0,0 +1,210 @@
+"""Compares datasets using DataFrame's correlation."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+import pandas as pd
+import numpy as np
+
+from logging import debug, warning, info
+
+from traffic_taffy.algorithms.compareseries import ComparisonSeriesAlgorithm
+from traffic_taffy.reports.correlationchangereport import CorrelationChangeReport
+from traffic_taffy.comparison import Comparison, OrganizedReports
+from traffic_taffy.taffy_config import TaffyConfig, taffy_default
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
+    from numpy import ndarray
+
+taffy_default("algorithms.correlationchanges.minimum_change", 0.5)
+taffy_default("algorithms.correlationchanges.correlation_method", "spearman")
+taffy_default("algorithms.correlationchanges.comparison_width", 15)
+taffy_default("algorithms.correlationchanges.slide_length", None)
+
+
+class CompareCorrelationChanges(ComparisonSeriesAlgorithm):
+    """Compare series using the pandas correlation."""
+
+    MAX_PIVOT = 1000
+
+    def __init__(
+        self,
+        timestamps: List[int] | None = None,
+        match_string: str | None = None,
+        match_value: str | None = None,
+        minimum_count: int | None = None,
+        make_printable: bool = False,
+        match_expression: str | None = None,
+    ):
+        """Create a CompareCorrelationChanges instance.
+
+        Valid methods: kendall, pearson, spearman, corrcoef
+
+        speed-wise; pearson < spearman < corrcoef < kendall
+
+        accuracy-wise:
+            corrcoef: not great (uses np.corrcoef)
+            pearson: better but, not good
+            spearman: best
+            kendall: best
+        """
+        super().__init__(
+            timestamps,
+            match_string,
+            match_value,
+            minimum_count,
+            make_printable,
+            match_expression,
+        )
+        self.method = None
+
+    def compare_series(
+        self, df: DataFrame, indexes: ndarray | None = None
+    ) -> List[CorrelationChangeReport]:
+        """Compare a bunch of series looking for changes in correlation.
+
+        This tries to do a comparison in a faster path if the number
+        of keys are reasonable (for if not a pivot will consume all
+        available memory)
+        """
+
+        self.sort_by = "delta_correlation"
+
+        config = TaffyConfig()
+        minimum_change = float(
+            config.get_dotnest("algorithms.correlationchanges.minimum_change", 0.3)
+        )
+        self.minimum_change = minimum_change
+
+        method = config.get_dotnest("algorithms.correlationchanges.correlation_method")
+        self.method = method
+
+        comparison_width = config.get_dotnest(
+            "algorithms.correlationchanges.comparison_width"
+        )
+        self.comparison_width = comparison_width
+
+        slide_length = config.get_dotnest("algorithms.correlationchanges.slide_length")
+        if not slide_length:
+            slide_length = comparison_width
+        self.slide_length = slide_length
+
+        indexes = df["index"].unique()
+        num_indexes = len(indexes)
+        info(
+            f"starting correlation changes comparison: num_indexes={num_indexes}, min_change={self.minimum_change}"
+        )
+
+        # TODO(hardaker): use a full sweeping comparison for faster correlations
+        # now we just revert to the slower non-pivot method for proof of concept
+        return super().compare_series(df, indexes)
+
+        if num_indexes > self.MAX_PIVOT:
+            # we assume this is arbitrarily too large
+            # use the slower parent version instead
+            warning(
+                f"too many indexes ({num_indexes} > {self.MAX_PIVOT}) == using slower routine to conserve memory"
+            )
+            return super().compare_series(df, indexes)
+
+        for key in ["subkey", "index", "filename"]:
+            del df[key]
+        df = df.pivot_table(
+            columns=["key"], index=["time"], values="count", fill_value=0
+        )
+
+        # indexes have changed
+        indexes = df.columns.to_list()
+
+        # use pandas internal kendall
+        # TODO(hardaker): np.corrcoef is multi-core but is pearsons
+        # TODO(hardaker): scipy.stat.kendalltau is kendall,
+        #                 but can only do one at a time
+
+        # TODO(hardaker): df.corr() returns different numbers here
+        # than inside compare_two_series!!
+
+        reports: OrganizedReports = {}
+
+        if method == "corrcoef":
+            np_array = df.to_numpy()
+            results = np.corrcoef(np_array)
+            for numx, column_left in enumerate(indexes):
+                for numy, column_right in enumerate(indexes[numx + 1 :]):
+                    value = results[numx][numy]
+                    # if value > minimum_value:
+                    #     print(
+                    #         f"{column_left:<30} similar to {column_right:<30}: {value}"
+                    #     )
+            return reports
+
+        # default to using the datafram corr method instead
+        results = df.corr(method=method)
+
+        # TODO(hardaker): this doesn't actually do anything
+        # need to break correlation into pieces and run multiple passes
+
+        for num, column_left in enumerate(indexes):
+            for column_right in indexes[num + 1 :]:
+                value = results[column_left][column_right]
+                if value > self.minimum_change:
+                    # print(f"{column_left:<30} similar to {column_right:<30}: {value}")
+                    if column_left not in reports:
+                        reports[column_left] = {}
+                    reports[column_left][column_right] = CorrelationChangeReport(
+                        value,
+                    )
+
+        return [Comparison(reports, "Correlation Report", "delta_correlation")]
+
+    def compare_two_series(
+        self,
+        column_left: str,
+        series_left: list,
+        column_right: str,
+        series_right: list,
+    ) -> CorrelationChangeReport | None:
+        """Compare two series using the dataframe correlation algorithms."""
+        debug(f"correlation comparing {column_left} and {column_right}")
+        both = pd.concat([series_left, series_right], axis=1)
+        both.fillna(0, inplace=True)
+
+        # Note actually faster -- about the same as df.corr
+        # import scipy
+        # results = scipy.stats.kendalltau(both['left'], both['right'])
+        # value = results.statistic
+
+        start_index: int = 0
+        middle_index: int = self.comparison_width
+        end_index: int = 2 * self.comparison_width
+
+        data_length = len(both)
+
+        while end_index < data_length:
+            left_correlation = both[start_index:middle_index].corr(self.method)["left"][
+                "right"
+            ]
+            right_correlation = both[middle_index:end_index].corr(self.method)["left"][
+                "right"
+            ]
+            delta_correlation = right_correlation - left_correlation
+
+            # well this is ugly:
+            timestamp = (
+                both[middle_index : middle_index + 1]
+                .index.to_pydatetime()[0]
+                .timestamp()
+            )
+
+            debug(f"  {right_correlation} - {left_correlation} = {delta_correlation}")
+            if abs(delta_correlation) >= self.minimum_change:
+                return CorrelationChangeReport(
+                    left_correlation, right_correlation, delta_correlation, timestamp
+                )
+
+            start_index += self.slide_length
+            middle_index += self.slide_length
+            end_index += self.slide_length
+
+        # if we get here there are no change points found
+        return

traffic_taffy-0.9/traffic_taffy/algorithms/compareseries.py

@@ -0,0 +1,117 @@
+"""Compares datasets in time-series rather than by series."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+from traffic_taffy.algorithms import ComparisonAlgorithm
+from traffic_taffy.graphdata import PcapGraphData
+from traffic_taffy.comparison import Comparison, OrganizedReports
+
+from logging import error
+
+if TYPE_CHECKING:
+    from traffic_taffy.dissection import Dissection
+    from pandas import DataFrame
+    from numpy import ndarray
+
+
+class ComparisonSeriesAlgorithm(ComparisonAlgorithm):
+    """A base class for algorithms that compare left/right series."""
+
+    def __init__(
+        self,
+        timestamps: List[int] | None = None,
+        match_string: str | None = None,
+        match_value: str | None = None,
+        minimum_count: int | None = None,
+        make_printable: bool = False,
+        match_expression: str | None = None,
+    ):
+        """Create a ComparisonAlgorithm."""
+        self.timestamps = timestamps
+        self.match_string = match_string
+        self.match_value = match_value
+        self.minimum_count = minimum_count
+        self.make_printable = make_printable
+        self.match_expression = (match_expression,)
+        self.sort_by = "correlation"
+
+    def compare_two_series(
+        self,
+        _column_left: str,
+        _series_left: list,
+        _column_right: str,
+        _series_right: list,
+    ) -> dict:
+        """Error catching base class function for comparing two columnar series."""
+        error("code failure: base class compare_two_series should never be called")
+        raise ValueError
+
+    def compare_dissections(self, dissections: List[Dissection]) -> List[Comparison]:
+        """Compare all the column series."""
+        # hack to figure out if there is at least two instances of a generator
+        # without actually extracting them all
+        # (since it could be memory expensive)
+
+        # merge all dissections together into one
+        # TODO(hardaker): ideally this should be a parameter
+        #                 forced upward into dissectmany
+        dissection = next(dissections)
+        for to_be_merged in dissections:
+            dissection.merge(to_be_merged)
+
+        # filter downward
+        dissection = dissection.filter(
+            self.timestamps,
+            self.match_string,
+            self.match_value,
+            self.minimum_count,
+            self.make_printable,
+            self.match_expression,
+        )
+
+        data = PcapGraphData()
+        data.dissections = [dissection]
+        # data.normalize_bins() ?
+        df = data.get_dataframe()
+
+        return self.compare_series(df)
+
+    def compare_series(
+        self, df: DataFrame, indexes: ndarray | None = None
+    ) -> List[Comparison]:
+        """Compares the series found in a dataframe, two at a time."""
+
+        reports: OrganizedReports = {}
+
+        if indexes is None:
+            indexes = df["index"].unique()
+
+        for num, column_left in enumerate(indexes):
+            series_left = df[df["index"] == column_left]
+            series_left = series_left.set_index("time")
+            series_left = series_left["count"]
+            series_left.name = "left"
+
+            # TODO(hardaker): n^2 is bad
+            for column_right in indexes[num + 1 :]:
+                if column_left == column_right:
+                    continue
+
+                series_right = df[df["index"] == column_right]
+                series_right = series_right.set_index("time")
+                series_right = series_right["count"]
+                series_right.name = "right"
+
+                report = self.compare_two_series(
+                    column_left, series_left, column_right, series_right
+                )
+                if column_left not in reports:
+                    reports[column_left] = {}
+
+                if isinstance(report, list):
+                    # TODO(hardaker): we don't actually handle arrays yet
+                    reports[column_left][column_right].extend(report)
+                elif report:
+                    reports[column_left][column_right] = report
+
+        return [Comparison(reports, "Correlation Report", self.sort_by)]