sciveo 0.1.31__tar.gz → 0.1.32__tar.gz

{sciveo-0.1.31 → sciveo-0.1.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.31
+Version: 0.1.32
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.31 → sciveo-0.1.32}/sciveo/ml/evaluation/object_detection.py

@@ -21,6 +21,10 @@ from sciveo.ml.images.object_detection import *
 
 Object Detection Evaluation
 
+Using AP and FP for object detector evaluation.
+Need to define max allowed false positives (for example 0.03).
+Usually max AP has relatively high FP rate, so maxing FP is on lower AP.
+
 """
 class EvalObjectDetection:
   def __init__(self, predictions, labels, class_names):
@@ -79,7 +83,6 @@ class EvalObjectDetection:
 
     return inter_area / union_area
 
-  # Compute the Average Precision (AP)
   def compute_ap(self, class_name, confidence_threshold=0.0):
     """
     Calculate the Average Precision based on IoU and confidence scores.
@@ -118,40 +121,83 @@ class EvalObjectDetection:
           true_positives.append(0)
           false_positives.append(1)
 
-    # Convert to cumulative sums
     tp_cumsum = np.cumsum(true_positives)
     fp_cumsum = np.cumsum(false_positives)
 
-    # Compute recall and precision
     recall = tp_cumsum / (count_labels + 1e-20)
     precision = tp_cumsum / (tp_cumsum + fp_cumsum + 1e-20)
 
+    FP = np.sum(false_positives) / (count_labels + 1e-20)
+
     # Compute AP using the trapezoidal rule (integrating precision over recall)
     ap = np.trapz(precision, recall)
-    return ap
+    return ap, FP
 
-  def threshold(self, class_name):
+  # TODO: Use simple gradient-based threshold optimisation instead of current grid search.
+  def calc_thresholds(self, class_name):
     list_ap = []
+    list_FP = []
     list_thresholds = np.linspace(0.0, 1.0, 101).tolist()
     for i, threshold in enumerate(list_thresholds):
-      ap = self.compute_ap(class_name, threshold)
+      ap, FP = self.compute_ap(class_name, threshold)
       list_ap.append(ap)
+      list_FP.append(FP)
       if i % 10 == 0:
-        debug(class_name, "threshold", threshold, "ap", ap)
+        debug(class_name, "threshold", threshold, "ap", ap, "FP", FP)
+    return list_thresholds, list_ap, list_FP
+
+  def threshold(self, class_name, list_thresholds, list_ap, list_FP, max_fp=0.05):
     idx = list_ap.index(max(list_ap))
-    debug("Threshold", class_name, list_thresholds[idx], "AP", list_ap[idx])
-    return list_thresholds[idx], list_ap[idx]
+    if list_FP[idx] <= max_fp:
+      return list_thresholds[idx]
+
+    for i in range(len(list_thresholds)):
+      if list_FP[i] <= max_fp:
+        debug("Threshold", class_name, list_thresholds[i], "AP", list_ap[i], "max_fp", max_fp, "FP", list_FP[i])
+        idx = i
+        break
 
-  def evaluate(self, iou_thresholds={"default": 0.0}):
+    return list_thresholds[idx]
+
+  def thresholds(self):
+    """
+      Calculate best confidence thresholds for every class.
+      Predictions should be non-thresholded.
+      Thresholds precission is 1/100.
+    """
+
+    result = {}
+    list_thresholds = {}
+    list_ap = {}
+    list_FP = {}
+
+    for class_name in self.class_names:
+      list_thresholds[class_name], list_ap[class_name], list_FP[class_name] = self.calc_thresholds(class_name)
+      idx = list_ap[class_name].index(max(list_ap[class_name]))
+      debug("Threshold", class_name, list_thresholds[class_name][idx], "max AP", list_ap[class_name][idx])
+
+    for max_fp in [0.01, 0.03, 0.05, 0.1]:
+      result[max_fp] = {"default": 0.5}
+      for class_name in self.class_names:
+        class_threshold = self.threshold(class_name, list_thresholds[class_name], list_ap[class_name], list_FP[class_name], max_fp)
+        result[max_fp][class_name] = class_threshold
+
+    return result
+
+  def evaluate(self, confidence_thresholds={"default": 0.0}):
     """
     Calculate metrics like mAP based on IoU.
     """
+
     aps = []
     class_ap = {}
+    class_FP = {}
+
     for class_name in self.class_names:
-      ap = self.compute_ap(class_name, iou_thresholds.get(class_name, iou_thresholds["default"]))
+      ap, FP = self.compute_ap(class_name, confidence_thresholds.get(class_name, confidence_thresholds["default"]))
       aps.append(ap)
       class_ap[class_name] = ap
+      class_FP[class_name] = FP
 
     mAP = np.mean(aps)
-    return {'mAP': mAP, 'AP per class': class_ap}
+    return {'mAP': mAP, 'AP per class': class_ap, "FP per class": class_FP}

{sciveo-0.1.31 → sciveo-0.1.32}/sciveo/tools/logger.py

@@ -28,7 +28,11 @@ def _sciveo_get_logger(name):
   if not logger.hasHandlers():
     with _sciveo_log_lock:
       if not logger.hasHandlers():
-        logger.setLevel(logging.getLevelName(_sciveo_log_min_level))
+        log_min_level = logging.getLevelName(_sciveo_log_min_level)
+        if log_min_level.startswith("Level"):
+          log_min_level = "DEBUG"
+        logger.setLevel(log_min_level)
+
         formatter = logging.Formatter('%(asctime)s [%(thread)d] [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
         ch = logging.StreamHandler()
         ch.setFormatter(formatter)

sciveo-0.1.32/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.32'

{sciveo-0.1.31 → sciveo-0.1.32}/sciveo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.31
+Version: 0.1.32
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

sciveo-0.1.31/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.31'