tsadmetrics 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

tsadmetrics/_tsadeval/nabscore.py

@@ -0,0 +1,311 @@
+# ----------------------------------------------------------------------
+# Copyright (C) 2014-2015, Numenta, Inc.  Unless you have an agreement
+# with Numenta, Inc., for a separate license for this software code, the
+# following terms and conditions apply:
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero Public License for more details.
+#
+# You should have received a copy of the GNU Affero Public License
+# along with this program.  If not, see http://www.gnu.org/licenses.
+#
+# http://numenta.org/licenses/
+# ----------------------------------------------------------------------
+from collections import namedtuple
+import logging
+import math
+
+logger = logging.getLogger(__name__)
+AnomalyPoint = namedtuple(
+  "AnomalyPoint",
+  ["timestamp", "anomalyScore", "sweepScore", "windowName"]
+)
+ThresholdScore = namedtuple(
+  "ThresholdScore",
+  ["threshold", "score", "tp", "tn", "fp", "fn", "total"]
+)
+
+
+def sigmoid(x):
+  """Standard sigmoid function."""
+  return 1 / (1 + math.exp(-x))
+
+
+def scaledSigmoid(relativePositionInWindow):
+  """Return a scaled sigmoid function given a relative position within a
+  labeled window.  The function is computed as follows:
+  A relative position of -1.0 is the far left edge of the anomaly window and
+  corresponds to S = 2*sigmoid(5) - 1.0 = 0.98661.  This is the earliest to be
+  counted as a true positive.
+  A relative position of -0.5 is halfway into the anomaly window and
+  corresponds to S = 2*sigmoid(0.5*5) - 1.0 = 0.84828.
+  A relative position of 0.0 consists of the right edge of the window and
+  corresponds to S = 2*sigmoid(0) - 1 = 0.0.
+  Relative positions > 0 correspond to false positives increasingly far away
+  from the right edge of the window. A relative position of 1.0 is past the
+  right  edge of the  window and corresponds to a score of 2*sigmoid(-5) - 1.0 =
+  -0.98661.
+  @param  relativePositionInWindow (float)  A relative position
+                                            within a window calculated per the
+                                            rules above.
+  @return (float)
+  """
+  if relativePositionInWindow > 3.0:
+    # FP well behind window
+    val = -1.0
+  else:
+    val = 2*sigmoid(-5*relativePositionInWindow) - 1.0
+
+  return val
+
+
+def prepAnomalyListForScoring(inputAnomalyList):
+  """
+  Sort by anomaly score and filter all rows with 'probationary' window name
+  """
+  return sorted(
+    [x for x in inputAnomalyList if x.windowName != 'probationary'],
+    key=lambda x: x.anomalyScore,
+    reverse=True)
+
+class Sweeper(object):
+  """Class used to iterate over all anomaly scores in a data set, generating
+  threshold-score pairs for use in threshold optimization or dataset scoring.
+  """
+
+  def __init__(self, probationPercent=0.15, costMatrix=None):
+    self.probationPercent = probationPercent
+
+    self.tpWeight = 0
+    self.fpWeight = 0
+    self.fnWeight = 0
+
+    if costMatrix is not None:
+      self.setCostMatrix(costMatrix)
+
+
+  def setCostMatrix(self, costMatrix):
+    self.tpWeight = costMatrix["tpWeight"]
+    self.fpWeight = costMatrix["fpWeight"]
+    self.fnWeight = costMatrix["fnWeight"]
+
+
+  def _getProbationaryLength(self, numRows):
+    return min(
+      math.floor(self.probationPercent * numRows),
+      self.probationPercent * 5000
+    )
+
+
+  def _prepareScoreByThresholdParts(self, inputAnomalyList):
+    scoreParts = {"fp": 0}
+    for row in inputAnomalyList:
+      if row.windowName not in ('probationary', None):
+        scoreParts[row.windowName] = -self.fnWeight
+    return scoreParts
+
+
+  def calcSweepScore(
+      self, timestamps, anomalyScores, windowLimits, dataSetName):
+    """
+    Given a single file's rows, return a list of AnomalyPoints.
+    Each AnomalyPoint contains the row's timestamp, anomaly score,
+    calculated NAB score, and window name. These lists may be passed
+    to `calcScoreByThreshold()` directly in order to score or optimize
+    a single file, or combined together prior to being passed to
+    `calcScoreByThreshold()` in order to score / calculate multiple
+    files / an entire corpus.
+    @param timestamps:    (list)  `datetime` objects
+    @param anomalyScores: (list)  `float` objects in the range [0.0, 1.0]
+    @param windowLimits:  (list)  `tuple` objects of window limits
+    @param dataSetName:   (list)  `string` name of dataset, often filename
+    @return   (list) List of AnomalyPoint objects
+    """
+    assert len(timestamps) == len(anomalyScores), \
+      "timestamps and anomalyScores should not be different lengths!"
+    timestamps = list(timestamps)
+    windowLimits = list(windowLimits)  # Copy because we mutate this list
+    # The final list of anomaly points returned from this function.
+    # Used for threshold optimization and scoring in other functions.
+    anomalyList = []
+
+    # One-time config variables
+    maxTP = scaledSigmoid(-1.0)
+    probationaryLength = self._getProbationaryLength(len(timestamps))
+
+    # Iteration variables - these update as we iterate through the data
+    curWindowLimits = None
+    curWindowName = None
+    curWindowWidth = None
+    curWindowRightIndex = None
+    prevWindowWidth = None
+    prevWindowRightIndex = None
+
+    for i, (curTime, curAnomaly) in enumerate(zip(timestamps, anomalyScores)):
+      unweightedScore = None
+      weightedScore = None
+
+      # If not in a window, check if we've just entered one
+      if windowLimits and curTime == windowLimits[0][0]:
+        curWindowLimits = windowLimits.pop(0)
+        curWindowName = "%s|%s" % (dataSetName, curWindowLimits[0])
+        curWindowRightIndex = timestamps.index(curWindowLimits[1])
+        curWindowWidth = float(curWindowRightIndex -
+                               timestamps.index(curWindowLimits[0]) + 1)
+
+        logger.debug(
+          "Entering window: %s (%s)", curWindowName, str(curWindowLimits))
+
+      # If in a window, score as if true positive
+      if curWindowLimits is not None:
+        positionInWindow = -(curWindowRightIndex - i + 1) / curWindowWidth
+        unweightedScore = scaledSigmoid(positionInWindow)
+        weightedScore = unweightedScore * self.tpWeight / maxTP
+
+      # If outside a window, score as if false positive
+      else:
+        if prevWindowRightIndex is None:
+          # No preceding window, so return score as is we were just really
+          # far away from the nearest window.
+          unweightedScore = -1.0
+        else:
+          numerator = abs(prevWindowRightIndex - i)
+          denominator = float(prevWindowWidth - 1)
+          positionPastWindow = numerator / denominator
+          unweightedScore = scaledSigmoid(positionPastWindow)
+
+        weightedScore = unweightedScore * self.fpWeight
+
+      if i >= probationaryLength:
+        pointWindowName = curWindowName
+      else:
+        pointWindowName = "probationary"
+
+      point = AnomalyPoint(curTime, curAnomaly, weightedScore, pointWindowName)
+
+      anomalyList.append(point)
+
+      # If at right-edge of window, exit window.
+      # This happens after processing the current point and appending it
+      # to the list.
+      if curWindowLimits is not None and curTime == curWindowLimits[1]:
+        logger.debug("Exiting window: %s", curWindowName)
+        prevWindowRightIndex = i
+        prevWindowWidth = curWindowWidth
+        curWindowLimits = None
+        curWindowName = None
+        curWindowWidth = None
+        curWindowRightIndex = None
+
+    return anomalyList
+
+
+  def calcScoreByThreshold(self, anomalyList):
+    """
+    Find NAB scores for each threshold in `anomalyList`.
+    @param anomalyList  (list) `AnomalyPoint` objects from `calcSweepScore()`
+    @return (list)  List of `ThresholdScore` objects
+    """
+    scorableList = prepAnomalyListForScoring(anomalyList)
+    scoreParts = self._prepareScoreByThresholdParts(scorableList)
+    scoresByThreshold = []  # The final list we return
+
+    # The current threshold above which an anomaly score is considered
+    # an anomaly prediction. This starts above 1.0 so that all points
+    # are skipped, which gives us a full false-negative score.
+    curThreshold = 1.1
+
+    # Initialize counts:
+    # * every point in a window is a false negative
+    # * every point outside a window is a true negative
+    tn = sum(1 if x.windowName is None else 0 for x in scorableList)
+    fn = sum(1 if x.windowName is not None else 0 for x in scorableList)
+    tp = 0
+    fp = 0
+
+    # Iterate through every data point, starting with highest anomaly scores
+    # and working down. Whenever we reach a new anomaly score, we save the
+    # current score and begin calculating the score for the new, lower
+    # threshold. Every data point we iterate over is 'active' for the current
+    # threshold level, so the point is either:
+    #   * a true positive (has a `windowName`)
+    #   * a false positive (`windowName is None`).
+    for dataPoint in scorableList:
+      # If we've reached a new anomaly threshold, store the current
+      # threshold+score pair.
+      if dataPoint.anomalyScore != curThreshold:
+        curScore = sum(scoreParts.values())
+        totalCount = tp + tn + fp + fn
+        s = ThresholdScore(curThreshold, curScore, tp, tn, fp, fn, totalCount)
+        scoresByThreshold.append(s)
+        curThreshold = dataPoint.anomalyScore
+
+      # Adjust counts
+      if dataPoint.windowName is not None:
+        tp += 1
+        fn -= 1
+      else:
+        fp += 1
+        tn -= 1
+
+      if dataPoint.windowName is None:
+        scoreParts["fp"] += dataPoint.sweepScore
+      else:
+        scoreParts[dataPoint.windowName] = max(
+          scoreParts[dataPoint.windowName],
+          dataPoint.sweepScore
+        )
+
+    # Make sure to save the score for the last threshold
+    curScore = sum(scoreParts.values())
+    totalCount = tp + tn + fp + fn
+    s = ThresholdScore(curThreshold, curScore, tp, tn, fp, fn, totalCount)
+    scoresByThreshold.append(s)
+
+    return scoresByThreshold
+
+
+  def scoreDataSet(
+      self, timestamps, anomalyScores, windowLimits, dataSetName, threshold):
+    """Function called to score each dataset in the corpus.
+    @param timestamps     (tuple) tuple of timestamps
+    @param anomalyScores  (tuple) tuple of anomaly scores (floats [0, 1.0])
+    @param windowLimits   (tuple) tuple of window limit tuples
+    @param dataSetName    (string) name of this dataset, usually a file path.
+      Used to name the windows in this dataset, which is important when scoring
+      more than one data set, as each window in all data sets needs to be
+      uniquely named.
+    @param threshold      (float) the threshold at which an anomaly score is
+      considered to be an anomaly prediction.
+    @return
+    :return:  (tuple) Contains:
+      scores      (list) List of per-row scores, to be saved in score file
+      matchingRow (ThresholdScore)
+    """
+    anomalyList = self.calcSweepScore(
+      timestamps, anomalyScores, windowLimits, dataSetName)
+    scoresByThreshold = self.calcScoreByThreshold(anomalyList)
+
+    matchingRow = None
+    prevRow = None
+    for thresholdScore in scoresByThreshold:
+      if thresholdScore.threshold == threshold:
+        matchingRow = thresholdScore
+        break
+      elif thresholdScore.threshold < threshold:
+        matchingRow = prevRow
+        break
+
+      prevRow = thresholdScore
+
+    # Return sweepScore for each row, to be added to score file
+    return (
+      [x.sweepScore for x in anomalyList],
+      matchingRow
+    )

tsadmetrics/_tsadeval/tests.py

@@ -0,0 +1,376 @@
+from .metrics import *
+
+import unittest
+
+
+class Binary_detection_tester(unittest.TestCase):
+    def test_unsorted(self):
+        self.assertRaises(AssertionError, Binary_detection, 10, [2, 3, 4], [3, 4, 2])
+        self.assertRaises(AssertionError, Binary_detection, 10, [3, 4, 2], [2, 3, 4])
+        self.assertRaises(AssertionError, Binary_detection, 10, [[1, 8]], [[5, 6], [1, 2]])
+        self.assertRaises(AssertionError, Binary_detection, 10, [[5, 6], [1, 2]], [[1, 8]])
+
+    def test_nonunique(self):
+        self.assertRaises(AssertionError, Binary_detection, 10, [2, 4, 4], [2, 3, 4])
+        self.assertRaises(AssertionError, Binary_detection, 10, [2, 3, 4], [2, 4, 4])
+
+    def test_long_anom(self):
+        self.assertRaises(AssertionError, Binary_detection, 4, [1], [2, 3, 4])
+        self.assertRaises(AssertionError, Binary_detection, 4, [[2, 4]], [1])
+        self.assertRaises(AssertionError, Binary_detection, 4, [-1], [1])
+
+    def test_point_to_seq(self):
+        anom1 = [3, 4, 5, 7, 8, 11]
+        anom2 = [[3, 5], [7, 8], [11, 11]]
+        d = Binary_detection(12, anom1, anom2)
+
+        self.assertTrue(np.array_equal(np.array(anom1), d.get_predicted_anomalies_ptwise()))
+        self.assertTrue(np.array_equal(np.array(anom2), d.get_gt_anomalies_segmentwise()))
+
+    def test_anomaly_full_seires(self):
+        anom1 = [3, 4, 5, 7, 8, 11]
+        d = Binary_detection(12, anom1, anom1)
+
+        self.assertTrue(
+            np.array_equal(d.get_gt_anomalies_full_series(), np.array([0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1]))
+        )
+        self.assertTrue(
+            np.array_equal(d.get_predicted_anomalies_full_series(), np.array([0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1]))
+        )
+
+    def test_empty_anom(self):
+        anom1 = [3, 4, 5, 7, 8]
+        anom2 = []
+
+        d = Binary_detection(12, anom1, anom2)
+        self.assertEqual(0, len(d.get_predicted_anomalies_ptwise()))
+        self.assertEqual(0, len(d.get_predicted_anomalies_segmentwise()))
+
+
+class Confusion_metrics_tester(unittest.TestCase):
+    def test_metrics(self):
+        self.assertEqual(0.75, recall(tp=3, fn=1))
+        self.assertEqual(0.75, precision(tp=3, fp=1))
+        self.assertEqual(0.6, f1_score(tp=3, fn=1, fp=3))
+        self.assertEqual(0.625, f1_from_pr(p=1, r=0.25, beta=0.5))
+        self.assertAlmostEqual(5 / 9, f1_from_pr(p=1, r=0.5, beta=2))
+
+    def test_requires_names(self):
+        self.assertRaises(TypeError, recall, 3, 4)
+        self.assertRaises(TypeError, precision, 3, 4)
+        self.assertRaises(TypeError, f1_score, 3, 4, 5)
+
+    def test_zerodivision(self):
+        self.assertEqual(0, recall(tp=0, fn=0))
+        self.assertEqual(0, precision(tp=0, fp=0))
+        self.assertEqual(0, f1_score(tp=0, fp=1, fn=1))
+
+
+class Metrics_tester(unittest.TestCase):
+    def test_PW(self):
+        pw = Pointwise_metrics(10, [1, 2, 3, 4], [4, 5, 6])
+
+        self.assertEqual(pw.tp, 1)
+        self.assertEqual(pw.fp, 2)
+        self.assertEqual(pw.fn, 3)
+
+    def test_PA(self):
+        pa = PointAdjust(10, [1, 2, 3, 4, 9], [4, 5, 6])
+        self.assertEqual(pa.tp, 4)
+        self.assertEqual(pa.fp, 2)
+        self.assertEqual(pa.fn, 1)
+
+        pa = PointAdjust(10, [1, 2, 3, 4, 5, 6, 7], [4])
+        self.assertEqual(pa.get_score(), 1)
+
+        pa = PointAdjust(10, [1, 2, 3, 4, 5, 6, 7], [9])
+        self.assertEqual(pa.get_score(), 0)
+
+    def test_dtPA(self):
+        pa = DelayThresholdedPointAdjust(10, [1, 2, 3, 4, 9], [4, 5, 6], k=3)
+        self.assertEqual(pa.tp, 4)
+        self.assertEqual(pa.fp, 2)
+        self.assertEqual(pa.fn, 1)
+
+        pa = DelayThresholdedPointAdjust(10, [1, 2, 3, 4, 9], [4, 5, 6], k=2)
+        self.assertEqual(pa.tp, 0)
+        self.assertEqual(pa.fp, 2)
+        self.assertEqual(pa.fn, 5)
+
+        pa = DelayThresholdedPointAdjust(10, [1, 2, 3, 4, 5, 6, 7], [4], k=3)
+        self.assertEqual(pa.get_score(), 1)
+
+        pa = DelayThresholdedPointAdjust(10, [1, 2, 3, 4, 5, 6, 7], [4], k=2)
+        self.assertEqual(pa.get_score(), 0)
+
+    def test_pakf(self):
+        pa = PointAdjustKPercent(10, [1, 2, 3, 4, 9], [4, 5, 6], k=0.5)
+        self.assertEqual(pa.tp, 1)
+        self.assertEqual(pa.fp, 2)
+        self.assertEqual(pa.fn, 4)
+
+        pa = PointAdjustKPercent(10, [1, 2, 3, 4, 9], [4, 5, 6], k=0.1)
+        self.assertEqual(pa.tp, 4)
+        self.assertEqual(pa.fp, 2)
+        self.assertEqual(pa.fn, 1)
+
+    def test_lspa(self):
+        pa = LatencySparsityAware(10, [2, 3, 4, 5, 9], [4, 7], tw=1)
+        self.assertAlmostEqual(pa.get_score(), f1_score(tp=pa.tp, fn=pa.fn, fp=pa.fp), 4)
+        self.assertEqual(pa.tp, 2)
+        self.assertEqual(pa.fp, 1)
+        self.assertEqual(pa.fn, 3)
+
+        pa = LatencySparsityAware(10, [2, 3, 4, 5, 9], [4, 7], tw=2)
+        self.assertAlmostEqual(pa.get_score(), f1_score(tp=pa.tp, fn=pa.fn, fp=pa.fp), 4)
+        self.assertEqual(pa.tp, 1)
+        self.assertEqual(pa.fp, 1)
+        self.assertEqual(pa.fn, 2)
+
+    def test_Segment(self):
+        s = Segmentwise_metrics(10, [[1, 2], [4, 4], [7, 9]], [[0, 6]])
+        self.assertEqual(s.tp, 2)
+        self.assertEqual(s.fp, 0)
+        self.assertEqual(s.fn, 1)
+
+        s = Segmentwise_metrics(10, [[1, 2], [4, 4], [7, 9]], [[6, 6], [8, 8]])
+        self.assertEqual(s.tp, 1)
+        self.assertEqual(s.fp, 1)
+        self.assertEqual(s.fn, 2)
+
+        s = Segmentwise_metrics(10, [[1, 2], [4, 4], [7, 9]], [])
+        self.assertEqual(s.tp, 0)
+        self.assertEqual(s.fp, 0)
+        self.assertEqual(s.fn, 3)
+
+        s = Segmentwise_metrics(10, [[1, 2], [4, 4], [7, 9]], [[0, 9]])
+        self.assertEqual(s.tp, 3)
+        self.assertEqual(s.fp, 0)
+        self.assertEqual(s.fn, 0)
+
+    def test_CF(self):
+        c = Composite_f(10, [0, 2, 3, 5, 7, 9], [3, 6])
+        f = c.get_score()
+        self.assertEqual(c.p, 0.5)
+        self.assertEqual(c.r, 0.2)
+
+    def test_affiliation(self):
+        a = Affiliation(10, [2, 3], [2])
+        f = a.get_score()
+        self.assertEqual(a.p, 1)
+        self.assertTrue(a.r < 1)
+
+        a = Affiliation(10, [2, 3], [2, 3, 4])
+        f = a.get_score()
+        self.assertTrue(a.p < 1)
+        self.assertEqual(a.r, 1)
+
+    def test_range_pr(self):
+        r = Range_PR(10, [2, 3], [2])
+        f = r.get_score()
+        self.assertEqual(r.p, 1)
+        self.assertTrue(r.r < 1)
+
+        r2 = Range_PR(10, [2, 3], [2, 3])
+        f2 = r2.get_score()
+        self.assertTrue(f2 > f)
+
+        r = Range_PR(10, [2, 3], [2, 3, 4])
+        f = r.get_score()
+        self.assertTrue(r.p < 1)
+        self.assertEqual(r.r, 1)
+
+    def test_NAB(self):
+        n = NAB_score(10, [[3, 6]], [3])
+        self.assertAlmostEqual(n.get_score(), 100)
+
+        n = NAB_score(10, [[3, 6]], [])
+        self.assertAlmostEqual(n.get_score(), 0)
+
+        n = NAB_score(10, [[3, 6]], [1])
+        self.assertAlmostEqual(n.get_score(), -100 * 0.11 / 2)
+
+        n = NAB_score(10, [3, 6], [1])
+        self.assertTrue(np.isnan(n.get_score()))
+
+    def test_ttol(self):
+        t = Time_Tolerant(10, [3, 4, 8], [1, 2, 3], d=2)
+        self.assertAlmostEqual(t.recall(), 2 / 3)
+        self.assertAlmostEqual(t.precision(), 1)
+
+        t = Time_Tolerant(10, [4, 5], [6], d=1)
+        self.assertAlmostEqual(t.recall(), 1 / 2)
+        self.assertAlmostEqual(t.precision(), 1)
+
+    def test_TaF(self):
+        t = TaF(10, [4, 5, 6], [4, 5, 6])
+        self.assertEqual(t.get_score(), 1)
+
+        t = TaF(10, [4, 5, 6], [1, 2, 3])
+        self.assertEqual(t.get_score(), 0)
+
+        t = TaF(10, [4, 5, 6], [7, 8, 9])
+        self.assertEqual(t.get_score(), 0)
+        t = TaF(10, [4, 5, 6], [7, 8, 9], delta=1)
+        self.assertTrue(t.get_score() > 0)
+
+        t1 = TaF(10, [4, 5, 8, 9], [4, 5])
+        t2 = TaF(10, [4, 5, 8, 9], [5, 8])
+        self.assertTrue(t1.get_score() < t2.get_score())
+
+    def test_eTaF(self):
+        t = eTaF(10, [4, 5, 6], [4, 5, 6])
+        self.assertEqual(t.get_score(), 1)
+
+        t = eTaF(10, [4, 5, 6], [1, 2, 3])
+        self.assertEqual(t.get_score(), 0)
+
+        t = eTaF(10, [4, 5, 6], [7, 8, 9])
+        self.assertTrue(t.get_score() == 0)
+
+        t1 = eTaF(10, [4, 5, 8, 9], [4, 5])
+        t2 = eTaF(10, [4, 5, 8, 9], [5, 8])
+        self.assertTrue(t1.get_score() < t2.get_score())
+
+    def test_temp_dist(self):
+        t = Temporal_Distance(10, [4, 5, 6], [4, 5, 6])
+        self.assertEqual(t.get_score(), 0)
+
+        t = Temporal_Distance(10, [4, 6], [4, 5, 6])
+        self.assertEqual(t.get_score(), 1)
+
+        t = Temporal_Distance(10, [4], [4, 5, 6])
+        self.assertEqual(t.get_score(), 3)
+
+        t = Temporal_Distance(10, [4, 5, 6], [8])
+        self.assertEqual(t.get_score(), 11)
+
+        t = Temporal_Distance(10, [4, 5, 6], [])
+        self.assertEqual(t.get_score(), 30)
+
+
+class Threshold_metric_tester(unittest.TestCase):
+    #    def test_roc(self):
+    #        a = aucroc(true = [0,0,1,1], score = [0.1,0.4,0.35,0.8])
+    def test_auc_pr(self):
+        gt = [[2, 3]]
+        anomaly_score = [1, 3, 2, 4]
+        auc_pr = AUC_PR_pw(gt, anomaly_score)
+
+        score = auc_pr.get_score()
+        self.assertAlmostEqual(score, 0.83, 2)
+
+        anomaly_score = [1, 2, 3, 4]
+        auc_pr = AUC_PR_pw(gt, anomaly_score)
+        score = auc_pr.get_score()
+        self.assertEqual(score, 1)
+
+        anomaly_score = [4, 3, 1, 1]
+        auc_pr = AUC_PR_pw(gt, anomaly_score)
+        score = auc_pr.get_score()
+        self.assertEqual(score, 0.5)
+
+    def test_auc_roc(self):
+        gt = [[2, 3]]
+        anomaly_score = [1, 3, 2, 4]
+        auc_roc = AUC_ROC(gt, anomaly_score)
+
+        score = auc_roc.get_score()
+        self.assertAlmostEqual(score, 0.75, 2)
+
+        anomaly_score = [1, 2, 3, 4]
+        auc_roc = AUC_ROC(gt, anomaly_score)
+        score = auc_roc.get_score()
+        self.assertEqual(score, 1)
+
+        anomaly_score = [4, 4, 4, 4]
+        auc_roc = AUC_ROC(gt, anomaly_score)
+        score = auc_roc.get_score()
+        self.assertEqual(score, 0.5)
+
+    def test_vus_pr(self):
+        gt = [[0, 1]]
+        anomaly_score = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+        vus_pr = VUS_PR(gt, anomaly_score, max_window=4)
+
+        score = vus_pr.get_score()
+        self.assertTrue(score <= 0.2)
+
+        gt = [[1, 3]]
+        anomaly_score = [8, 0, 9, 1, 7, 2, 3, 4, 5, 6]
+        vus_pr = VUS_PR(gt, anomaly_score, max_window=4)
+        score = vus_pr.get_score()
+        self.assertTrue(score > 0.5)
+        vus_pr = VUS_PR(gt, anomaly_score, max_window=0)
+        score = vus_pr.get_score()
+        self.assertTrue(score < 0.5)
+
+    def test_vus_roc(self):
+        gt = [[0, 1]]
+        anomaly_score = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+        vus = VUS_ROC(gt, anomaly_score, max_window=4)
+
+        score = vus.get_score()
+        self.assertTrue(score <= 0.1)
+
+        gt = [[1, 3]]
+        anomaly_score = [8, 0, 9, 1, 7, 2, 3, 4, 5, 6]
+        vus = VUS_ROC(gt, anomaly_score, max_window=4)
+        score = vus.get_score()
+        self.assertTrue(score > 0.4)
+        vus = VUS_ROC(gt, anomaly_score, max_window=0)
+        score = vus.get_score()
+        self.assertTrue(score < 0.4)
+
+    def test_PatK(self):
+        gt = [[2, 3]]
+
+        anomaly_score = [1, 4, 2, 3]
+        patk = PatK_pw(gt, anomaly_score)
+        score = patk.get_score()
+        self.assertEqual(score, 0.5)
+
+        anomaly_score = [1, 2, 3, 4]
+        patk = PatK_pw(gt, anomaly_score)
+        score = patk.get_score()
+        self.assertEqual(score, 1)
+
+        anomaly_score = [3, 4, 1, 2]
+        patk = PatK_pw(gt, anomaly_score)
+        score = patk.get_score()
+        self.assertEqual(score, 0)
+
+        anomaly_score = [3, 4, 1, 2]
+        patk = PatK_pw([1, 2, 3], anomaly_score)
+        score = patk.get_score()
+        self.assertAlmostEqual(score, 2 / 3)
+
+        anomaly_score = [2, 1, 1, 0]
+        patk = PatK_pw([0, 1], anomaly_score)
+        score = patk.get_score()
+        self.assertAlmostEqual(score, 2 / 3)
+
+        patk = PatK_pw([], [0, 1, 2, 4])
+        self.assertRaises(AssertionError, patk.get_score)
+
+    def test_best_threshold_pw(self):
+        gt = [[2, 3]]
+
+        anomaly_score = [1, 3, 2, 4]
+        metric = Best_threshold_pw(gt, anomaly_score)
+        score = metric.get_score()
+        self.assertAlmostEqual(score, 2 * 2 / 3 * 1 / (1 + 2 / 3))
+
+        anomaly_score = [2, 3, 1, 4]
+        metric = Best_threshold_pw(gt, anomaly_score)
+        score = metric.get_score()
+        self.assertAlmostEqual(score, 2 * 1 / 2 * 1 / (1 + 1 / 2))
+
+        anomaly_score = [4, 3, 1, 2]
+        metric = Best_threshold_pw(gt, anomaly_score)
+        score = metric.get_score()
+        self.assertAlmostEqual(score, 2 * 1 / 2 * 1 / (1 + 1 / 2))
+
+
+if __name__ == "__main__":
+    unittest.main()