traffic-taffy 0.8.1__py3-none-any.whl → 0.9__py3-none-any.whl

traffic_taffy/__init__.py

@@ -1 +1 @@
-__VERSION__ = "0.8.1"
+__VERSION__ = "0.9"

traffic_taffy/algorithms/__init__.py

@@ -1,14 +1,21 @@
-from typing import TYPE_CHECKING
+"""traffic-taffy algorithm produce comparisons between different datasets."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+from logging import error
 
 if TYPE_CHECKING:
-    pass
+    from traffic_taffy.dissection import Dissection
+    from traffic_taffy.reports import Report
 
 
 class ComparisonAlgorithm:
+    """A base class for all comparison algorithms."""
+
     def __init__(self):
-        pass
+        """Construct a ComparisonAlgorithm."""
 
-    def compare_dissections(left_side: dict, right_side: dict) -> dict:
-        raise ValueError(
-            "code failure: base class compare_dissections should never be called"
-        )
+    def compare_dissections(self, _dissections: List[Dissection]) -> List[Report]:
+        """Compare dissections base function just to warn things are not implemented."""
+        error("code failure: base class compare_two_dissections should never be called")
+        raise ValueError

traffic_taffy/algorithms/comparecorrelation.py

@@ -0,0 +1,164 @@
+"""Compares datasets using DataFrame's correlation."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+import pandas as pd
+import numpy as np
+
+from logging import debug, warning, info
+
+from traffic_taffy.algorithms.compareseries import ComparisonSeriesAlgorithm
+from traffic_taffy.reports.correlationreport import CorrelationReport
+from traffic_taffy.comparison import Comparison, OrganizedReports
+from traffic_taffy.taffy_config import TaffyConfig, taffy_default
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
+    from numpy import ndarray
+
+taffy_default("algorithms.correlation.minimum_correlation", 0.8)
+taffy_default("algorithms.correlation.correlation_method", "spearman")
+taffy_default("algorithms.correlation.max_pivot", 1000)
+
+
+class CompareCorrelation(ComparisonSeriesAlgorithm):
+    """Compare series using the pandas correlation."""
+
+    def __init__(
+        self,
+        timestamps: List[int] | None = None,
+        match_string: str | None = None,
+        match_value: str | None = None,
+        minimum_count: int | None = None,
+        make_printable: bool = False,
+        match_expression: str | None = None,
+    ):
+        """Create a CompareCorrelation instance.
+
+        Valid methods: kendall, pearson, spearman, corrcoef
+
+        speed-wise; pearson < spearman < corrcoef < kendall
+
+        accuracy-wise:
+            corrcoef: not great (uses np.corrcoef)
+            pearson: better but, not good
+            spearman: best
+            kendall: best
+        """
+        super().__init__(
+            timestamps,
+            match_string,
+            match_value,
+            minimum_count,
+            make_printable,
+            match_expression,
+        )
+        self.method = None
+
+    def compare_series(
+        self, df: DataFrame, indexes: ndarray | None = None
+    ) -> List[CorrelationReport]:
+        """Compare a bunch of series using correlation.
+
+        This tries to do a comparison in a faster path if the number
+        of keys are reasonable (for if not a pivot will consume all
+        available memory)
+        """
+
+        config = TaffyConfig()
+        minimum_correlation = float(
+            config.get_dotnest("algorithms.correlation.minimum_correlation")
+        )
+        self.minimum_correlation = minimum_correlation
+
+        max_pivot = int(config.get_dotnest("algorithms.correlation.max_pivot"))
+        method = config.get_dotnest("algorithms.correlation.correlation_method")
+        self.method = method
+
+        indexes = df["index"].unique()
+        num_indexes = len(indexes)
+        if num_indexes > max_pivot:
+            # we assume this is arbitrarily too large
+            # use the slower parent version instead
+            warning(
+                f"too many indexes ({num_indexes} > {max_pivot}) == using slower routine to conserve memory"
+            )
+            return super().compare_series(df, indexes)
+
+        info(f"Studying correlation of {num_indexes} indexes")
+
+        for key in ["subkey", "index", "filename"]:
+            del df[key]
+        df = df.pivot_table(
+            columns=["key"], index=["time"], values="count", fill_value=0
+        )
+
+        # indexes have changed
+        indexes = df.columns.to_list()
+
+        # use pandas internal kendall
+        # TODO(hardaker): np.corrcoef is multi-core but is pearsons
+        # TODO(hardaker): scipy.stat.kendalltau is kendall,
+        #                 but can only do one at a time
+
+        # TODO(hardaker): df.corr() returns different numbers here
+        # than inside compare_two_series!!
+
+        reports: OrganizedReports = {}
+
+        if method == "corrcoef":
+            np_array = df.to_numpy()
+            results = np.corrcoef(np_array)
+            for numx, column_left in enumerate(indexes):
+                for numy, column_right in enumerate(indexes[numx + 1 :]):
+                    value = results[numx][numy]
+                    # if value > minimum_correlation:
+                    #     print(
+                    #         f"{column_left:<30} similar to {column_right:<30}: {value}"
+                    #     )
+            return reports
+
+        # default to using the datafram corr method instead
+        df.fillna(0, inplace=True)
+        results = df.corr(method=method)
+
+        for num, column_left in enumerate(indexes):
+            for column_right in indexes[num + 1 :]:
+                value = results[column_left][column_right]
+                if value > minimum_correlation:
+                    # print(f"{column_left:<30} similar to {column_right:<30}: {value}")
+                    if column_left not in reports:
+                        reports[column_left] = {}
+                    reports[column_left][column_right] = CorrelationReport(
+                        value,
+                    )
+        return [Comparison(reports, "Correlation Report", "correlation")]
+
+    def compare_two_series(
+        self,
+        column_left: str,
+        series_left: list,
+        column_right: str,
+        series_right: list,
+        reports: OrganizedReports = None,
+    ) -> dict:
+        """Compare two series using the dataframe correlation algorithms."""
+        debug(f"correlation comparing {column_left} and {column_right}")
+        both = pd.concat([series_left, series_right], axis=1)
+        both.fillna(0, inplace=True)
+
+        # Note actually faster -- about the same as df.corr
+        # import scipy
+        # results = scipy.stats.kendalltau(both['left'], both['right'])
+        # value = results.statistic
+
+        results = both.corr(method=self.method)
+        value = results["left"][1]
+        debug(f"{column_left:<30} similar to {column_right:<30}: {value}")
+
+        if value > self.minimum_correlation:
+            # print(f"{column_left:<30} similar to {column_right:<30}: {value}")
+
+            return CorrelationReport(value)
+
+        return

traffic_taffy/algorithms/comparecorrelationchanges.py

@@ -0,0 +1,210 @@
+"""Compares datasets using DataFrame's correlation."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+import pandas as pd
+import numpy as np
+
+from logging import debug, warning, info
+
+from traffic_taffy.algorithms.compareseries import ComparisonSeriesAlgorithm
+from traffic_taffy.reports.correlationchangereport import CorrelationChangeReport
+from traffic_taffy.comparison import Comparison, OrganizedReports
+from traffic_taffy.taffy_config import TaffyConfig, taffy_default
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
+    from numpy import ndarray
+
+taffy_default("algorithms.correlationchanges.minimum_change", 0.5)
+taffy_default("algorithms.correlationchanges.correlation_method", "spearman")
+taffy_default("algorithms.correlationchanges.comparison_width", 15)
+taffy_default("algorithms.correlationchanges.slide_length", None)
+
+
+class CompareCorrelationChanges(ComparisonSeriesAlgorithm):
+    """Compare series using the pandas correlation."""
+
+    MAX_PIVOT = 1000
+
+    def __init__(
+        self,
+        timestamps: List[int] | None = None,
+        match_string: str | None = None,
+        match_value: str | None = None,
+        minimum_count: int | None = None,
+        make_printable: bool = False,
+        match_expression: str | None = None,
+    ):
+        """Create a CompareCorrelationChanges instance.
+
+        Valid methods: kendall, pearson, spearman, corrcoef
+
+        speed-wise; pearson < spearman < corrcoef < kendall
+
+        accuracy-wise:
+            corrcoef: not great (uses np.corrcoef)
+            pearson: better but, not good
+            spearman: best
+            kendall: best
+        """
+        super().__init__(
+            timestamps,
+            match_string,
+            match_value,
+            minimum_count,
+            make_printable,
+            match_expression,
+        )
+        self.method = None
+
+    def compare_series(
+        self, df: DataFrame, indexes: ndarray | None = None
+    ) -> List[CorrelationChangeReport]:
+        """Compare a bunch of series looking for changes in correlation.
+
+        This tries to do a comparison in a faster path if the number
+        of keys are reasonable (for if not a pivot will consume all
+        available memory)
+        """
+
+        self.sort_by = "delta_correlation"
+
+        config = TaffyConfig()
+        minimum_change = float(
+            config.get_dotnest("algorithms.correlationchanges.minimum_change", 0.3)
+        )
+        self.minimum_change = minimum_change
+
+        method = config.get_dotnest("algorithms.correlationchanges.correlation_method")
+        self.method = method
+
+        comparison_width = config.get_dotnest(
+            "algorithms.correlationchanges.comparison_width"
+        )
+        self.comparison_width = comparison_width
+
+        slide_length = config.get_dotnest("algorithms.correlationchanges.slide_length")
+        if not slide_length:
+            slide_length = comparison_width
+        self.slide_length = slide_length
+
+        indexes = df["index"].unique()
+        num_indexes = len(indexes)
+        info(
+            f"starting correlation changes comparison: num_indexes={num_indexes}, min_change={self.minimum_change}"
+        )
+
+        # TODO(hardaker): use a full sweeping comparison for faster correlations
+        # now we just revert to the slower non-pivot method for proof of concept
+        return super().compare_series(df, indexes)
+
+        if num_indexes > self.MAX_PIVOT:
+            # we assume this is arbitrarily too large
+            # use the slower parent version instead
+            warning(
+                f"too many indexes ({num_indexes} > {self.MAX_PIVOT}) == using slower routine to conserve memory"
+            )
+            return super().compare_series(df, indexes)
+
+        for key in ["subkey", "index", "filename"]:
+            del df[key]
+        df = df.pivot_table(
+            columns=["key"], index=["time"], values="count", fill_value=0
+        )
+
+        # indexes have changed
+        indexes = df.columns.to_list()
+
+        # use pandas internal kendall
+        # TODO(hardaker): np.corrcoef is multi-core but is pearsons
+        # TODO(hardaker): scipy.stat.kendalltau is kendall,
+        #                 but can only do one at a time
+
+        # TODO(hardaker): df.corr() returns different numbers here
+        # than inside compare_two_series!!
+
+        reports: OrganizedReports = {}
+
+        if method == "corrcoef":
+            np_array = df.to_numpy()
+            results = np.corrcoef(np_array)
+            for numx, column_left in enumerate(indexes):
+                for numy, column_right in enumerate(indexes[numx + 1 :]):
+                    value = results[numx][numy]
+                    # if value > minimum_value:
+                    #     print(
+                    #         f"{column_left:<30} similar to {column_right:<30}: {value}"
+                    #     )
+            return reports
+
+        # default to using the datafram corr method instead
+        results = df.corr(method=method)
+
+        # TODO(hardaker): this doesn't actually do anything
+        # need to break correlation into pieces and run multiple passes
+
+        for num, column_left in enumerate(indexes):
+            for column_right in indexes[num + 1 :]:
+                value = results[column_left][column_right]
+                if value > self.minimum_change:
+                    # print(f"{column_left:<30} similar to {column_right:<30}: {value}")
+                    if column_left not in reports:
+                        reports[column_left] = {}
+                    reports[column_left][column_right] = CorrelationChangeReport(
+                        value,
+                    )
+
+        return [Comparison(reports, "Correlation Report", "delta_correlation")]
+
+    def compare_two_series(
+        self,
+        column_left: str,
+        series_left: list,
+        column_right: str,
+        series_right: list,
+    ) -> CorrelationChangeReport | None:
+        """Compare two series using the dataframe correlation algorithms."""
+        debug(f"correlation comparing {column_left} and {column_right}")
+        both = pd.concat([series_left, series_right], axis=1)
+        both.fillna(0, inplace=True)
+
+        # Note actually faster -- about the same as df.corr
+        # import scipy
+        # results = scipy.stats.kendalltau(both['left'], both['right'])
+        # value = results.statistic
+
+        start_index: int = 0
+        middle_index: int = self.comparison_width
+        end_index: int = 2 * self.comparison_width
+
+        data_length = len(both)
+
+        while end_index < data_length:
+            left_correlation = both[start_index:middle_index].corr(self.method)["left"][
+                "right"
+            ]
+            right_correlation = both[middle_index:end_index].corr(self.method)["left"][
+                "right"
+            ]
+            delta_correlation = right_correlation - left_correlation
+
+            # well this is ugly:
+            timestamp = (
+                both[middle_index : middle_index + 1]
+                .index.to_pydatetime()[0]
+                .timestamp()
+            )
+
+            debug(f"  {right_correlation} - {left_correlation} = {delta_correlation}")
+            if abs(delta_correlation) >= self.minimum_change:
+                return CorrelationChangeReport(
+                    left_correlation, right_correlation, delta_correlation, timestamp
+                )
+
+            start_index += self.slide_length
+            middle_index += self.slide_length
+            end_index += self.slide_length
+
+        # if we get here there are no change points found
+        return

traffic_taffy/algorithms/compareseries.py

@@ -0,0 +1,117 @@
+"""Compares datasets in time-series rather than by series."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+from traffic_taffy.algorithms import ComparisonAlgorithm
+from traffic_taffy.graphdata import PcapGraphData
+from traffic_taffy.comparison import Comparison, OrganizedReports
+
+from logging import error
+
+if TYPE_CHECKING:
+    from traffic_taffy.dissection import Dissection
+    from pandas import DataFrame
+    from numpy import ndarray
+
+
+class ComparisonSeriesAlgorithm(ComparisonAlgorithm):
+    """A base class for algorithms that compare left/right series."""
+
+    def __init__(
+        self,
+        timestamps: List[int] | None = None,
+        match_string: str | None = None,
+        match_value: str | None = None,
+        minimum_count: int | None = None,
+        make_printable: bool = False,
+        match_expression: str | None = None,
+    ):
+        """Create a ComparisonAlgorithm."""
+        self.timestamps = timestamps
+        self.match_string = match_string
+        self.match_value = match_value
+        self.minimum_count = minimum_count
+        self.make_printable = make_printable
+        self.match_expression = (match_expression,)
+        self.sort_by = "correlation"
+
+    def compare_two_series(
+        self,
+        _column_left: str,
+        _series_left: list,
+        _column_right: str,
+        _series_right: list,
+    ) -> dict:
+        """Error catching base class function for comparing two columnar series."""
+        error("code failure: base class compare_two_series should never be called")
+        raise ValueError
+
+    def compare_dissections(self, dissections: List[Dissection]) -> List[Comparison]:
+        """Compare all the column series."""
+        # hack to figure out if there is at least two instances of a generator
+        # without actually extracting them all
+        # (since it could be memory expensive)
+
+        # merge all dissections together into one
+        # TODO(hardaker): ideally this should be a parameter
+        #                 forced upward into dissectmany
+        dissection = next(dissections)
+        for to_be_merged in dissections:
+            dissection.merge(to_be_merged)
+
+        # filter downward
+        dissection = dissection.filter(
+            self.timestamps,
+            self.match_string,
+            self.match_value,
+            self.minimum_count,
+            self.make_printable,
+            self.match_expression,
+        )
+
+        data = PcapGraphData()
+        data.dissections = [dissection]
+        # data.normalize_bins() ?
+        df = data.get_dataframe()
+
+        return self.compare_series(df)
+
+    def compare_series(
+        self, df: DataFrame, indexes: ndarray | None = None
+    ) -> List[Comparison]:
+        """Compares the series found in a dataframe, two at a time."""
+
+        reports: OrganizedReports = {}
+
+        if indexes is None:
+            indexes = df["index"].unique()
+
+        for num, column_left in enumerate(indexes):
+            series_left = df[df["index"] == column_left]
+            series_left = series_left.set_index("time")
+            series_left = series_left["count"]
+            series_left.name = "left"
+
+            # TODO(hardaker): n^2 is bad
+            for column_right in indexes[num + 1 :]:
+                if column_left == column_right:
+                    continue
+
+                series_right = df[df["index"] == column_right]
+                series_right = series_right.set_index("time")
+                series_right = series_right["count"]
+                series_right.name = "right"
+
+                report = self.compare_two_series(
+                    column_left, series_left, column_right, series_right
+                )
+                if column_left not in reports:
+                    reports[column_left] = {}
+
+                if isinstance(report, list):
+                    # TODO(hardaker): we don't actually handle arrays yet
+                    reports[column_left][column_right].extend(report)
+                elif report:
+                    reports[column_left][column_right] = report
+
+        return [Comparison(reports, "Correlation Report", self.sort_by)]

traffic_taffy/algorithms/compareslices.py

@@ -0,0 +1,116 @@
+"""Compares datasets in time-slices rather than by series."""
+
+from __future__ import annotations
+from typing import List, TYPE_CHECKING
+from traffic_taffy.algorithms import ComparisonAlgorithm
+import itertools
+import datetime as dt
+
+from logging import debug, error, exception
+
+if TYPE_CHECKING:
+    from traffic_taffy.dissection import Dissection
+    from traffic_taffy.comparison import Comparison
+
+
+class ComparisonSlicesAlgorithm(ComparisonAlgorithm):
+    """A base class for algorithms that compare left/right slices."""
+
+    def __init__(
+        self,
+        timestamps: List[int] | None = None,
+        match_string: str | None = None,
+        match_value: str | None = None,
+        minimum_count: int | None = None,
+        make_printable: bool = False,
+        match_expression: str | None = None,
+    ):
+        """Create a ComparisonAlgorithm."""
+        self.timestamps = timestamps
+        self.match_string = match_string
+        self.match_value = match_value
+        self.minimum_count = minimum_count
+        self.make_printable = make_printable
+        self.match_expression = (match_expression,)
+
+    def compare_two_dissections(
+        self, _left_side: Dissection, _right_side: Dissection
+    ) -> Comparison:
+        """Error catching base class function for comparing two alogirthms."""
+        error("code failure: base class compare_two_dissections should never be called")
+        raise ValueError
+
+    def compare_dissections(self, dissections: List[Dissection]) -> List[Comparison]:
+        """Compare all the dissections in slices."""
+        comparisons = []
+        # hack to figure out if there is at least two instances of a generator
+        # without actually extracting them all
+        # (since it could be memory expensive)
+
+        reference = next(dissections)
+        other = None
+        multiple = True
+        try:
+            other = next(dissections)
+            dissections = itertools.chain([other], dissections)
+        except Exception:
+            exception("failed to create a chain of dissections")
+            multiple = False
+
+        if multiple:
+            # multiple file comparison
+            for other in dissections:
+                # compare the two global summaries
+
+                comparison = self.compare_two_dissections(
+                    reference.data[0], other.data[0]
+                )
+                comparison.title = f"{reference.pcap_file} vs {other.pcap_file}"
+
+                comparisons.append(comparison)
+        else:
+            # deal with timestamps within a single file
+            reference = reference.data
+            timestamps = list(reference.keys())
+            if len(timestamps) == 1:  # just 0-summary plus a single stamp
+                error(
+                    "the requested pcap data was not long enough to compare against itself"
+                )
+                errorstr: str = "not large enough pcap file"
+                raise ValueError(errorstr)
+            debug(
+                f"found {len(timestamps)} timestamps from {timestamps[2]} to {timestamps[-1]}"
+            )
+
+            for timestamp in range(
+                2, len(timestamps)
+            ):  # second real non-zero timestamp to last
+                time_left = timestamps[timestamp - 1]
+                time_right = timestamps[timestamp]
+
+                # see if we were asked to only use particular time ranges
+                # if self.between_times and (
+                #     time_left < self.between_times[0]
+                #     or time_right > self.between_times[1]
+                # ):
+                #     continue
+
+                debug(f"comparing timestamps {time_left} and {time_right}")
+
+                comparison = self.compare_two_dissections(
+                    reference[time_left],
+                    reference[time_right],
+                )
+
+                title_left = dt.datetime.fromtimestamp(time_left, dt.UTC).strftime(
+                    "%Y-%m-%d %H:%M:%S"
+                )
+                title_right = dt.datetime.fromtimestamp(time_right, dt.UTC).strftime(
+                    "%Y-%m-%d %H:%M:%S"
+                )
+
+                comparison.title = f"time {title_left} vs time {title_right}"
+                comparisons.append(comparison)
+
+        # return our collected results
+        return comparisons

traffic_taffy/algorithms/statistical.py

@@ -1,14 +1,14 @@
-from traffic_taffy.algorithms import ComparisonAlgorithm
+from traffic_taffy.algorithms.compareslices import ComparisonSlicesAlgorithm
 from traffic_taffy.comparison import Comparison
 from traffic_taffy.dissection import Dissection
-from traffic_taffy.report import Report
+from traffic_taffy.reports.compareslicesreport import CompareSlicesReport
 
 
-class ComparisonStatistical(ComparisonAlgorithm):
-    def __init__(self):
-        super().__init__()
+class ComparisonStatistical(ComparisonSlicesAlgorithm):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
 
-    def compare_dissections(self, left_side: dict, right_side: dict) -> Comparison:
+    def compare_two_dissections(self, left_side: dict, right_side: dict) -> Comparison:
         """Compare two dissections."""
         report = {}
 
@@ -46,7 +46,7 @@ class ComparisonStatistical(ComparisonAlgorithm):
                     new_left_count += 1
 
                 delta_absolute = right_count - left_count
-                report[key][subkey] = Report(
+                report[key][subkey] = CompareSlicesReport(
                     delta_percentage=delta_percentage,
                     delta_absolute=delta_absolute,
                     total=total,
@@ -67,7 +67,7 @@ class ComparisonStatistical(ComparisonAlgorithm):
                     right_percentage = right_side[key][subkey] / right_side_total
                     new_right_count += 1  # this value wasn't in the left
 
-                    report[key][subkey] = Report(
+                    report[key][subkey] = CompareSlicesReport(
                         delta_percentage=delta_percentage,
                         delta_absolute=right_count,
                         total=total,
@@ -87,7 +87,7 @@ class ComparisonStatistical(ComparisonAlgorithm):
             else:
                 left_percent = new_left_count / left_side_total
 
-            report[key][Dissection.NEW_RIGHT_SUBKEY] = Report(
+            report[key][Dissection.NEW_RIGHT_SUBKEY] = CompareSlicesReport(
                 delta_absolute=new_right_count - new_left_count,
                 total=new_left_count + new_right_count,
                 left_count=new_left_count,