sciveo 0.1.30__tar.gz → 0.1.32__tar.gz

{sciveo-0.1.30 → sciveo-0.1.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.30
+Version: 0.1.32
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.30 → sciveo-0.1.32}/sciveo/ml/evaluation/object_detection.py

@@ -21,6 +21,10 @@ from sciveo.ml.images.object_detection import *
 
 Object Detection Evaluation
 
+Using AP and FP for object detector evaluation.
+Need to define max allowed false positives (for example 0.03).
+Usually max AP has relatively high FP rate, so maxing FP is on lower AP.
+
 """
 class EvalObjectDetection:
   def __init__(self, predictions, labels, class_names):
@@ -79,8 +83,7 @@ class EvalObjectDetection:
 
     return inter_area / union_area
 
-  # Compute the Average Precision (AP)
-  def compute_ap(self, class_name, iou_threshold=0.0):
+  def compute_ap(self, class_name, confidence_threshold=0.0):
     """
     Calculate the Average Precision based on IoU and confidence scores.
     Returns the AP for the current class.
@@ -89,7 +92,6 @@ class EvalObjectDetection:
     count_labels = 0
     true_positives = []
     false_positives = []
-    false_negatives = []
     detected = []
 
     for i, prediction in enumerate(self.converted_predictions):
@@ -97,9 +99,11 @@ class EvalObjectDetection:
       current_labels = self.converted_labels[i].get(class_name, [])
 
       count_labels += len(current_labels)
-      TP = 0
 
       for prediction_box in current_predictions:
+        if prediction_box[4] < confidence_threshold:
+          continue
+
         max_iou = 0
         gt_match = None
 
@@ -109,55 +113,91 @@ class EvalObjectDetection:
             max_iou = iou
             gt_match = label_box
 
-        if max_iou >= iou_threshold and gt_match not in detected:
+        if gt_match is not None and gt_match not in detected:
           true_positives.append(1)
           false_positives.append(0)
           detected.append(gt_match)
-          TP += 1
         else:
           true_positives.append(0)
           false_positives.append(1)
 
-      FN = len(current_labels) - TP
-      false_negatives.append(FN)
-
-    # Convert to cumulative sums
     tp_cumsum = np.cumsum(true_positives)
     fp_cumsum = np.cumsum(false_positives)
 
-    # Compute recall and precision
-    recall = tp_cumsum / count_labels
-    precision = tp_cumsum / (tp_cumsum + fp_cumsum)
+    recall = tp_cumsum / (count_labels + 1e-20)
+    precision = tp_cumsum / (tp_cumsum + fp_cumsum + 1e-20)
+
+    FP = np.sum(false_positives) / (count_labels + 1e-20)
 
     # Compute AP using the trapezoidal rule (integrating precision over recall)
     ap = np.trapz(precision, recall)
-    return ap, np.sum(false_negatives) / count_labels
+    return ap, FP
 
-  def threshold(self, class_name, max_false_negative):
+  # TODO: Use simple gradient-based threshold optimisation instead of current grid search.
+  def calc_thresholds(self, class_name):
     list_ap = []
-    list_fn = []
+    list_FP = []
     list_thresholds = np.linspace(0.0, 1.0, 101).tolist()
-    for i, iou_threshold in enumerate(list_thresholds):
-      ap, FN = self.compute_ap(class_name, iou_threshold)
-      #debug("AP", ap, "FN", FN)
-      if FN > max_false_negative:
-        ap = 0.0
+    for i, threshold in enumerate(list_thresholds):
+      ap, FP = self.compute_ap(class_name, threshold)
       list_ap.append(ap)
-      list_fn.append(FN)
+      list_FP.append(FP)
+      if i % 10 == 0:
+        debug(class_name, "threshold", threshold, "ap", ap, "FP", FP)
+    return list_thresholds, list_ap, list_FP
+
+  def threshold(self, class_name, list_thresholds, list_ap, list_FP, max_fp=0.05):
     idx = list_ap.index(max(list_ap))
-    debug("Threshold", class_name, list_thresholds[idx], "AP", list_ap[idx], "FN", list_fn[idx])
-    return list_thresholds[idx], list_ap[idx], list_fn[idx]
+    if list_FP[idx] <= max_fp:
+      return list_thresholds[idx]
+
+    for i in range(len(list_thresholds)):
+      if list_FP[i] <= max_fp:
+        debug("Threshold", class_name, list_thresholds[i], "AP", list_ap[i], "max_fp", max_fp, "FP", list_FP[i])
+        idx = i
+        break
+
+    return list_thresholds[idx]
+
+  def thresholds(self):
+    """
+      Calculate best confidence thresholds for every class.
+      Predictions should be non-thresholded.
+      Thresholds precission is 1/100.
+    """
+
+    result = {}
+    list_thresholds = {}
+    list_ap = {}
+    list_FP = {}
+
+    for class_name in self.class_names:
+      list_thresholds[class_name], list_ap[class_name], list_FP[class_name] = self.calc_thresholds(class_name)
+      idx = list_ap[class_name].index(max(list_ap[class_name]))
+      debug("Threshold", class_name, list_thresholds[class_name][idx], "max AP", list_ap[class_name][idx])
 
-  def evaluate(self, iou_thresholds={"default": 0.0}):
+    for max_fp in [0.01, 0.03, 0.05, 0.1]:
+      result[max_fp] = {"default": 0.5}
+      for class_name in self.class_names:
+        class_threshold = self.threshold(class_name, list_thresholds[class_name], list_ap[class_name], list_FP[class_name], max_fp)
+        result[max_fp][class_name] = class_threshold
+
+    return result
+
+  def evaluate(self, confidence_thresholds={"default": 0.0}):
     """
     Calculate metrics like mAP based on IoU.
     """
+
     aps = []
     class_ap = {}
+    class_FP = {}
+
     for class_name in self.class_names:
-      ap, false_negatives = self.compute_ap(class_name, iou_thresholds.get(class_name, iou_thresholds["default"]))
+      ap, FP = self.compute_ap(class_name, confidence_thresholds.get(class_name, confidence_thresholds["default"]))
       aps.append(ap)
       class_ap[class_name] = ap
+      class_FP[class_name] = FP
 
     mAP = np.mean(aps)
-    return {'mAP': mAP, 'AP per class': class_ap}
+    return {'mAP': mAP, 'AP per class': class_ap, "FP per class": class_FP}

{sciveo-0.1.30 → sciveo-0.1.32}/sciveo/tools/logger.py

@@ -28,7 +28,11 @@ def _sciveo_get_logger(name):
   if not logger.hasHandlers():
     with _sciveo_log_lock:
       if not logger.hasHandlers():
-        logger.setLevel(logging.getLevelName(_sciveo_log_min_level))
+        log_min_level = logging.getLevelName(_sciveo_log_min_level)
+        if log_min_level.startswith("Level"):
+          log_min_level = "DEBUG"
+        logger.setLevel(log_min_level)
+
         formatter = logging.Formatter('%(asctime)s [%(thread)d] [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
         ch = logging.StreamHandler()
         ch.setFormatter(formatter)

sciveo-0.1.32/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.32'

{sciveo-0.1.30 → sciveo-0.1.32}/sciveo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.30
+Version: 0.1.32
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

sciveo-0.1.30/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.30'