sciveo 0.1.46__tar.gz → 0.1.48__tar.gz

{sciveo-0.1.46 → sciveo-0.1.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.46
+Version: 0.1.48
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.46 → sciveo-0.1.48}/sciveo/cli.py

@@ -11,6 +11,7 @@
 #
 
 import os
+import time
 import json
 import argparse
 
@@ -26,7 +27,8 @@ def main():
     'command',
     choices=[
       'init', 'monitor', 'scan', 'nvr',
-      'media-server', 'media-run'
+      'media-server', 'media-run',
+      'maintenance'
     ],
     help='Command to execute')
 
@@ -47,6 +49,9 @@ def main():
   parser.add_argument('--processor', type=str, help='Processor name')
   parser.add_argument('--src', type=str, default=None, help='Source')
   parser.add_argument('--dst', type=str, default=None, help='Destination')
+  parser.add_argument('--value', type=float, help='Value')
+  parser.add_argument('--threshold', type=float, help='Threshold')
+  parser.add_argument('--execute', type=str, default=None, help='Execute command')
 
   args = parser.parse_args()
 
@@ -85,6 +90,15 @@ def main():
     if args.processor == "audio-plot":
       from sciveo.media.pipelines.processors.audio.audio_extractor_process import plot_audio
       plot_audio(args.width, args.height, args.rate, args.input_path, args.output_path)
+  elif args.command == 'maintenance':
+    from sciveo.monitoring.process.memory import MemoryMonitorCommandDaemon
+    daemons = []
+    if args.execute is not None:
+      daemons.append(MemoryMonitorCommandDaemon(threshold_percent=args.threshold, period=args.period, command=args.execute))
+    for daemon in daemons:
+      daemon.start()
+    while(True):
+      time.sleep(3600)
   else:
     warning(args.command, "not implemented")
 

{sciveo-0.1.46 → sciveo-0.1.48}/sciveo/ml/dataset/object_detection.py

@@ -18,23 +18,33 @@ import math
 import numpy as np
 
 from sciveo.tools.logger import *
+from sciveo.ml.evaluation.object_detection import *
 
 
-def bbox_from_yolo(yolo_bbox):
-  yolo_bbox = [float(a) for a in yolo_bbox]
-  n, cx, cy, w, h = yolo_bbox
+def xcycwh_to_xyxy(bbox):
+  cx, cy, w, h = bbox
   x1 = cx - w / 2
   y1 = cy - h / 2
   x2 = cx + w / 2
   y2 = cy + h / 2
   return [x1, y1, x2, y2]
 
-def bbox_to_yolo(bbox):
-  n, x1, y1, x2, y2 = bbox
+def xyxy_to_xcycwh(bbox):
+  x1, y1, x2, y2 = bbox
   cx = (x1 + x2) / 2
   cy = (y1 + y2) / 2
   w = x2 - x1
   h = y2 - y1
+  return [cx, cy, w, h]
+
+def bbox_from_yolo(yolo_bbox):
+  yolo_bbox = [float(a) for a in yolo_bbox]
+  n, cx, cy, w, h = yolo_bbox
+  return xcycwh_to_xyxy([cx, cy, w, h])
+
+def bbox_to_yolo(bbox):
+  n, x1, y1, x2, y2 = bbox
+  cx, cy, w, h = xyxy_to_xcycwh([x1, y1, x2, y2])
   return [n, cx, cy, w, h]
 
 
@@ -222,22 +232,42 @@ class ObjectDetectionDataset:
     for split in self.splits:
       result["distribution"].setdefault(split, [])
       for class_name in self.classes:
-         result["distribution"][split].append({class_name: result["datasets"][split][class_name] / result["count"][split]})
+        result["distribution"][split].append({class_name: result["datasets"][split][class_name] / result["count"][split]})
       result["distribution"][split] = sorted(result["distribution"][split], key=lambda x: list(x.values())[0], reverse=True)
 
     return result
 
 
-
-if __name__ == "__main__":
-  # ds = YOLODataset()
-  # ds.load("/Users/sgeorgiev/develop/data/newspapers/test_yolo")
-  # debug("classes", ds.classes)
-  # ds.save(f"/Users/sgeorgiev/develop/data/newspapers/tmp/job-out", classes=["block", "vertical"], copy_images=True)
-
-  ds = ObjectDetectionDataset()
-  ds.from_yolo("/Users/sgeorgiev/develop/data/newspapers/test_yolo")
-  stats = ds.stats()
-  debug("stats", stats)
-  info("distribution", stats["distribution"])
-  # ds.save("/Users/sgeorgiev/develop/data/newspapers/tmp/od-1", copy_images=False)
+class EvalObjectDetectionDataset(EvalObjectDetection):
+  def __init__(self, predictions, labels, split="test"):
+    self.split = split
+    super().__init__(predictions.image_labels[split], labels.image_labels[split], predictions.classes)
+    self.from_datasets()
+
+  """
+    Converted predictions of type [{"class 1": [x1,y1,x2,y2, confidence]}, ...]
+    Boxes of type xyxyn + confidence
+
+    Converted Labels of type [{"class 1": [[x1,y1,x2,y2],...], "class 2": [[], [],...]...}]
+    Boxes of type xyxyn
+  """
+  def from_datasets(self):
+    self.converted_predictions = []
+    self.converted_labels = []
+
+    list_prediction_images = list(self.predictions.keys())
+    list_prediction_images.sort()
+    list_labels_images = list(self.labels.keys())
+    list_labels_images.sort()
+    if not all(a == b for a, b in zip(list_prediction_images, list_labels_images)):
+      error("predictions and labels differ")
+      return
+
+    for image_name in list_prediction_images:
+      self.converted_labels.append(self.labels[image_name])
+      current_image_prediction = {}
+      for class_name, class_prediction in self.predictions[image_name].items():
+        current_image_prediction[class_name] = class_prediction
+        for i, current_class_prediction in enumerate(current_image_prediction[class_name]):
+          current_image_prediction[class_name][i].append(1.0)
+      self.converted_predictions.append(current_image_prediction)

{sciveo-0.1.46 → sciveo-0.1.48}/sciveo/ml/evaluation/object_detection.py

@@ -17,6 +17,28 @@ from sciveo.tools.logger import *
 from sciveo.ml.images.object_detection import *
 
 
+def compute_iou_xyxy(box1, box2):
+  """
+    box1, box2: [x1, y1, x2, y2] format.
+    Returns the IoU between the two boxes.
+  """
+  x1_inter = max(box1[0], box2[0])
+  y1_inter = max(box1[1], box2[1])
+  x2_inter = min(box1[2], box2[2])
+  y2_inter = min(box1[3], box2[3])
+
+  inter_area = max(0, x2_inter - x1_inter) * max(0, y2_inter - y1_inter)
+  box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1])
+  box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1])
+  union_area = box1_area + box2_area - inter_area
+
+  if union_area == 0:
+    return 0.0
+
+  return inter_area / union_area
+
+
+
 """
 
 Object Detection Evaluation
@@ -39,6 +61,10 @@ class EvalObjectDetection:
       boxes = predicted.boxes
       class_names = predicted.names
 
+      """
+        Converted predictions of type [{"class 1": [x1,y1,x2,y2, confidence]}, ...]
+        Boxes of type xyxyn + confidence
+      """
       converted_prediction = {}
       for i in range(len(boxes)):
         class_id = int(boxes.cls[i].item())
@@ -53,7 +79,14 @@ class EvalObjectDetection:
         converted_prediction[label_class].append(box_confidence)
       self.converted_predictions.append(converted_prediction)
 
-    # TODO: currently labels of type [ {"class 1": [ ['x1':x1,'y1':y1,'x2':x2,'y2':y2] ... ], "class 2": [ ['x1':x1,'y1':y1,'x2':x2,'y2':y2] ... ]} ... ]
+    self._convert_labels_dict()
+
+  """
+    Labels of type [ {"class 1": [ ['x1':x1,'y1':y1,'x2':x2,'y2':y2] ... ], "class 2": [ ['x1':x1,'y1':y1,'x2':x2,'y2':y2] ... ]} ... ]
+    Converted Labels of type [{"class 1": [[x1,y1,x2,y2],...], "class 2": [[], [],...]...}]
+    Boxes of type xyxyn
+  """
+  def _convert_labels_dict(self):
     self.converted_labels = []
     for label in self.labels:
       converted_label = {}
@@ -63,26 +96,6 @@ class EvalObjectDetection:
           converted_label[class_name].append([box['x1'], box['y1'], box['x2'], box['y2']])
       self.converted_labels.append(converted_label)
 
-  def compute_iou(self, box1, box2):
-    """
-    box1, box2: [x1, y1, x2, y2] format.
-    Returns the IoU between the two boxes.
-    """
-    x1_inter = max(box1[0], box2[0])
-    y1_inter = max(box1[1], box2[1])
-    x2_inter = min(box1[2], box2[2])
-    y2_inter = min(box1[3], box2[3])
-
-    inter_area = max(0, x2_inter - x1_inter) * max(0, y2_inter - y1_inter)
-    box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1])
-    box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1])
-    union_area = box1_area + box2_area - inter_area
-
-    if union_area == 0:
-      return 0.0
-
-    return inter_area / union_area
-
   def compute_ap(self, class_name, confidence_threshold=0.0):
     """
     Calculate the Average Precision based on IoU and confidence scores.
@@ -108,7 +121,7 @@ class EvalObjectDetection:
         gt_match = None
 
         for label_box in current_labels:
-          iou = self.compute_iou(prediction_box[:4], label_box)
+          iou = compute_iou_xyxy(prediction_box[:4], label_box)
           if iou > 0.5 and iou >= max_iou:
             max_iou = iou
             gt_match = label_box

{sciveo-0.1.46 → sciveo-0.1.48}/sciveo/ml/nlp/embeddings.py

@@ -132,7 +132,7 @@ class TextEmbedding(BaseML):
 
 
 class SentenceEmbedding(TextEmbedding):
-  def __init__(self, model_name='BAAI/bge-m3', cache_dir=None, device=None) -> None:
+  def __init__(self, model_name='softel/sentence-bge-v1.3', cache_dir=None, device=None) -> None:
     super().__init__(model_name=model_name, cache_dir=cache_dir, device=device)
     self.model = None
     self.max_tokens = 8192

sciveo-0.1.48/sciveo/monitoring/process/memory.py

@@ -0,0 +1,37 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2025
+#
+
+
+import psutil
+import subprocess
+import time
+import threading
+
+from sciveo.tools.daemon import *
+from sciveo.tools.logger import *
+from sciveo.tools.simple_counter import RunCounter
+
+
+class MemoryMonitorCommandDaemon(DaemonBase):
+  def __init__(self, threshold_percent=90, period=5, command="echo 'Low Memory!'"):
+    super().__init__(period=period)
+    self.threshold_percent = threshold_percent
+    self.command = command
+    self.used_percent = 0
+    self.printer = RunCounter(60, lambda: debug(f"Memory usage: {self.used_percent}%", f"threshold: {self.threshold_percent}%"))
+
+  def loop(self):
+    mem = psutil.virtual_memory()
+    self.used_percent = mem.percent
+    self.printer.run()
+    if self.used_percent > self.threshold_percent:
+      warning(f"⚠️ Memory usage {self.used_percent}% exceeded {self.threshold_percent}%, executing command: {self.command}")
+      subprocess.run(self.command, shell=True)

sciveo-0.1.48/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.48'

{sciveo-0.1.46 → sciveo-0.1.48}/sciveo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.46
+Version: 0.1.48
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.46 → sciveo-0.1.48}/sciveo.egg-info/SOURCES.txt

@@ -112,6 +112,8 @@ sciveo/ml/video/description.py
 sciveo/monitoring/__init__.py
 sciveo/monitoring/monitor.py
 sciveo/monitoring/start.py
+sciveo/monitoring/process/__init__.py
+sciveo/monitoring/process/memory.py
 sciveo/network/__init__.py
 sciveo/network/camera.py
 sciveo/network/sniffer.py

sciveo-0.1.48/test/test_ml_datasets.py

@@ -0,0 +1,46 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import unittest
+import json
+
+from sciveo.ml.dataset.object_detection import *
+
+
+class TestMLDatasets(unittest.TestCase):
+  def test_object_detection(self):
+    HOME_PATH = os.path.expanduser("~")
+
+    # ds = YOLODataset()
+    # ds.load(f"{HOME_PATH}/data/test_yolo")
+    # debug("classes", ds.classes)
+    # ds.save(f"{HOME_PATH}/data/tmp/yolo-out", classes=["block", "vertical"], copy_images=False)
+
+    ds1 = ObjectDetectionDataset()
+    ds1.from_yolo(f"{HOME_PATH}/data/test_yolo")
+    stats = ds1.stats()
+    debug("stats", stats)
+    info("distribution", stats["distribution"])
+    # ds.save(f"{HOME_PATH}/data/tmp/od-1", copy_images=False)
+
+    ds2 = ObjectDetectionDataset()
+    ds2.from_yolo(f"{HOME_PATH}/data/job-1")
+    stats2 = ds2.stats()
+    debug("stats", stats2)
+    info("distribution", stats2["distribution"])
+
+    evaluator = EvalObjectDetectionDataset(ds1, ds2)
+    scores = evaluator.evaluate()
+    debug("eval", scores)
+
+
+if __name__ == '__main__':
+  unittest.main()

sciveo-0.1.46/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.46'

sciveo-0.1.46/test/test_ml_datasets.py

@@ -1,36 +0,0 @@
-#
-# Pavlin Georgiev, Softel Labs
-#
-# This is a proprietary file and may not be copied,
-# distributed, or modified without express permission
-# from the owner. For licensing inquiries, please
-# contact pavlin@softel.bg.
-#
-# 2024
-#
-
-import unittest
-import json
-
-from sciveo.ml.dataset.object_detection import *
-
-
-class TestMLDatasets(unittest.TestCase):
-  def test_object_detection(self):
-    HOME_PATH = os.path.expanduser("~")
-
-    ds = YOLODataset()
-    ds.load(f"{HOME_PATH}/data/test_yolo")
-    debug("classes", ds.classes)
-    ds.save(f"{HOME_PATH}/data/tmp/yolo-out", classes=["block", "vertical"], copy_images=True)
-
-    ds = ObjectDetectionDataset()
-    ds.from_yolo(f"{HOME_PATH}/data/test_yolo")
-    stats = ds.stats()
-    debug("stats", stats)
-    info("distribution", stats["distribution"])
-    ds.save(f"{HOME_PATH}/data/tmp/od-1", copy_images=False)
-
-
-if __name__ == '__main__':
-  unittest.main()