sciveo 0.1.45__tar.gz → 0.1.47__tar.gz

{sciveo-0.1.45 → sciveo-0.1.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.45
+Version: 0.1.47
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.45 → sciveo-0.1.47}/sciveo/cli.py

@@ -11,6 +11,7 @@
 #
 
 import os
+import time
 import json
 import argparse
 
@@ -26,7 +27,8 @@ def main():
     'command',
     choices=[
       'init', 'monitor', 'scan', 'nvr',
-      'media-server', 'media-run'
+      'media-server', 'media-run',
+      'maintenance'
     ],
     help='Command to execute')
 
@@ -47,6 +49,9 @@ def main():
   parser.add_argument('--processor', type=str, help='Processor name')
   parser.add_argument('--src', type=str, default=None, help='Source')
   parser.add_argument('--dst', type=str, default=None, help='Destination')
+  parser.add_argument('--value', type=float, help='Value')
+  parser.add_argument('--threshold', type=float, help='Threshold')
+  parser.add_argument('--execute', type=str, default=None, help='Execute command')
 
   args = parser.parse_args()
 
@@ -85,6 +90,15 @@ def main():
     if args.processor == "audio-plot":
       from sciveo.media.pipelines.processors.audio.audio_extractor_process import plot_audio
       plot_audio(args.width, args.height, args.rate, args.input_path, args.output_path)
+  elif args.command == 'maintenance':
+    from sciveo.monitoring.process.memory import MemoryMonitorCommandDaemon
+    daemons = []
+    if args.execute is not None:
+      daemons.append(MemoryMonitorCommandDaemon(threshold_percent=args.threshold, period=args.period, command=args.execute))
+    for daemon in daemons:
+      daemon.start()
+    while(True):
+      time.sleep(3600)
   else:
     warning(args.command, "not implemented")
 

{sciveo-0.1.45 → sciveo-0.1.47}/sciveo/media/pipelines/server.py

@@ -16,18 +16,26 @@ from sciveo.tools.logger import *
 from sciveo.tools.daemon import DaemonBase
 from sciveo.tools.os import *
 from sciveo.tools.simple_counter import RunCounter
+from sciveo.tools.configuration import ConfigurationArguments
 from sciveo.tools.aws.priority_queue import SQSPriorityQueue
 from sciveo.media.pipelines.pipeline import *
 from sciveo.media.pipelines.queues import MediaJobState
 
 
 class MaintenanceDaemon(DaemonBase):
-  def __init__(self, period=3600):
-    super().__init__(period=period)
+  def __init__(self, **kwargs):
+    self.arguments = ConfigurationArguments({
+      "period": 3600,
+      "run_period_files": 24,
+      "retention_period_files": 30,
+    }, **kwargs)
+
+    super().__init__(period=self.arguments["period"])
+
     self.base_tmp_path = os.environ["WEB_MEDIA_PIPELINES_TMP_PATH"]
 
     self.timers = [
-      RunCounter(24, lambda: run_system_cmd(f"find {self.base_tmp_path} -mtime +30 -type f -delete")),
+      RunCounter(self.arguments['run_period_files'], lambda: run_system_cmd(f"find {self.base_tmp_path} -mtime +{self.arguments['retention_period_files']} -type f -delete")),
       # RunCounter(3, self.print_me),
     ]
 
@@ -59,7 +67,7 @@ class MediaJobQueueDaemon(DaemonBase):
 
 def __START_SCIVEO_MEDIA_SERVER__():
   daemons = [
-    MaintenanceDaemon(period=3600),
+    MaintenanceDaemon(period=3600, run_period_files=24, retention_period_files=30),
     MediaJobQueueDaemon(),
   ]
 

sciveo-0.1.47/sciveo/ml/dataset/object_detection.py

@@ -0,0 +1,273 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import os
+import shutil
+import yaml
+import json
+import time
+import math
+import numpy as np
+
+from sciveo.tools.logger import *
+from sciveo.ml.evaluation.object_detection import *
+
+
+def xcycwh_to_xyxy(bbox):
+  cx, cy, w, h = bbox
+  x1 = cx - w / 2
+  y1 = cy - h / 2
+  x2 = cx + w / 2
+  y2 = cy + h / 2
+  return [x1, y1, x2, y2]
+
+def xyxy_to_xcycwh(bbox):
+  x1, y1, x2, y2 = bbox
+  cx = (x1 + x2) / 2
+  cy = (y1 + y2) / 2
+  w = x2 - x1
+  h = y2 - y1
+  return [cx, cy, w, h]
+
+def bbox_from_yolo(yolo_bbox):
+  yolo_bbox = [float(a) for a in yolo_bbox]
+  n, cx, cy, w, h = yolo_bbox
+  return xcycwh_to_xyxy([cx, cy, w, h])
+
+def bbox_to_yolo(bbox):
+  n, x1, y1, x2, y2 = bbox
+  cx, cy, w, h = xyxy_to_xcycwh([x1, y1, x2, y2])
+  return [n, cx, cy, w, h]
+
+
+class YOLODataset:
+  def __init__(self):
+    self.src_path = None
+    self.dst_path = None
+
+    self.classes = {}
+    self.splits = []
+    self.labels = {}
+
+  def load(self, base_path):
+    self.src_path = base_path
+    yaml_path = os.path.join(base_path, "data.yaml")
+    if not os.path.exists(yaml_path):
+      raise FileNotFoundError(f"data.yaml not found in {base_path}")
+
+    with open(yaml_path, "r") as f:
+      self.data_yaml = yaml.safe_load(f)
+
+    self.classes = self.data_yaml.get("names", {})
+    self.splits = [key for key in self.data_yaml.keys() if key not in ["names", "nc", "path"]]
+    self.labels = {split: self._load_labels_for_split(base_path, split) for split in self.splits}
+
+  def _load_labels_for_split(self, base_path, split: str):
+    images_dir = self.data_yaml.get(split)
+    labels_dir = images_dir.replace("images", "labels")
+    if not labels_dir:
+      return {}
+
+    images_path = os.path.join(base_path, images_dir)
+    labels_path = os.path.join(base_path, labels_dir)
+    if not os.path.exists(labels_path):
+      return {}
+
+    annotations = {}
+    for label_file in os.listdir(labels_path):
+      if label_file.endswith(".txt"):
+        image_id = label_file.replace(".txt", ".jpg")
+        label_path = os.path.join(labels_path, label_file)
+
+        with open(label_path, "r") as f:
+          annotations[image_id] = [line.strip().split() for line in f.readlines()]
+
+    return annotations
+
+  def save(self, base_path, classes=[], copy_images=True):
+    self.dst_path = base_path
+    os.makedirs(self.dst_path, exist_ok=True)
+
+    yaml_path = os.path.join(base_path, "data.yaml")
+
+    if not classes:
+      selected_classes = self.classes
+    else:
+      selected_classes = {idx: name for idx, name in self.classes.items() if name in classes}
+
+    class_mapping = {name: new_idx for new_idx, (idx, name) in enumerate(selected_classes.items())}
+
+    # data_yaml = {
+    #   "path": "./",
+    #   "names": {idx: name for idx, name in selected_classes.items()}
+    # }
+    # for split in self.splits:
+    #   data_yaml[split] = f"images/{split}"
+    # with open(yaml_path, "w") as f:
+    #   yaml.dump(data_yaml, f, default_flow_style=False)
+
+    with open(yaml_path, "w") as fp:
+      fp.write("  path: ./ # dataset root dir\n")
+      for split in self.splits:
+        fp.write(f"  test: images/{split}\n")
+      fp.write("  names:\n")
+      for class_id, class_name in selected_classes.items():
+        fp.write(f"    {class_id}: {class_name}\n")
+
+    for split, images in self.labels.items():
+      labels_path = os.path.join(base_path, "labels", split)
+      images_path = os.path.join(base_path, "images", split)
+      os.makedirs(labels_path, exist_ok=True)
+      os.makedirs(images_path, exist_ok=True)
+
+      for image_name, annotations in images.items():
+        label_file_path = os.path.join(labels_path, image_name.replace(".jpg", ".txt"))
+
+        with open(label_file_path, "w") as f:
+          for annotation in annotations:
+            class_id, *bbox = map(float, annotation)
+            class_id = int(class_id)
+
+            class_name = self.classes.get(class_id)
+            if class_name in class_mapping:
+              new_class_id = class_mapping[class_name]
+              f.write(f"{new_class_id} {' '.join(map(str, bbox))}\n")
+
+        if copy_images and self.src_path:
+          src_image_path = os.path.join(self.src_path, "images", split, image_name)
+          dst_image_path = os.path.join(images_path, image_name)
+          if not os.path.isfile(dst_image_path):
+            shutil.copy(src_image_path, dst_image_path)
+
+
+"""
+Object Detection Dataset
+
+labels in splits (for example train/val/test)
+every dataset split has image name as key which points to dict of classes as keys
+
+labels[split][image_name][class_name] is a list of bounding boxes of same class
+bboxes are of type nxyxy which means normalized [x1, y1, x2, y2] coordinates (upper left and down right point of the rectangle)
+
+"""
+class ObjectDetectionDataset:
+  def __init__(self):
+    self.classes = []
+    self.splits = []
+    self.labels = {}
+
+    self.src_path = None
+    self.dst_path = None
+
+  def save(self, base_path, copy_images=True, src_path=None):
+    if src_path is not None:
+      self.src_path = src_path
+
+    self.dst_path = base_path
+    for split in self.splits:
+      images_path = os.path.join(self.dst_path, split, "images")
+      os.makedirs(images_path, exist_ok=True)
+
+      labels_path = os.path.join(self.dst_path, split, "labels.json")
+      with open(labels_path, 'w') as fp:
+        json.dump(self.labels[split], fp, indent=2)
+
+      if copy_images and self.src_path:
+        src_images_path = os.path.join(self.src_path, "images", split)
+        for image_path, subdirs, image_files in os.walk(src_images_path):
+          for image_name in image_files:
+            src_path = os.path.join(image_path, image_name)
+            dst_path = os.path.join(images_path, image_name)
+            if not os.path.isfile(dst_path):
+              shutil.copy(src_path, dst_path)
+
+  def from_yolo(self, base_path):
+    self.src_path = base_path
+
+    self.raw = YOLODataset()
+    self.raw.load(base_path)
+
+    self.classes = []
+    for i in range(len(self.raw.classes)):
+      self.classes.append(self.raw.classes[i])
+    self.splits = self.raw.splits
+    self.image_labels = {}
+    self.labels = {}
+    for split, split_labels in self.raw.labels.items():
+      self.image_labels.setdefault(split, {})
+      self.labels.setdefault(split, {})
+      for image_id, image_labels in split_labels.items():
+        self.image_labels[split].setdefault(image_id, {})
+        for label in image_labels:
+          class_id = int(label[0])
+          class_name = self.raw.classes[class_id]
+          bbox = bbox_from_yolo(label)
+
+          self.image_labels[split][image_id].setdefault(class_name, [])
+          self.image_labels[split][image_id][class_name].append(bbox)
+
+          self.labels[split].setdefault(class_name, {})
+          self.labels[split][class_name].setdefault(image_id, [])
+          self.labels[split][class_name][image_id].append(bbox)
+
+  def stats(self):
+    result = {"datasets": {}, "count": {}, "distribution": {}}
+    for split in self.splits:
+      result["datasets"].setdefault(split, {})
+      result["count"].setdefault(split, 0)
+      for class_name in self.classes:
+        result["datasets"][split].setdefault(class_name, 0)
+        for image_name, image_labels in self.labels[split][class_name].items():
+          result["count"][split] += len(image_labels)
+          result["datasets"][split][class_name] += len(image_labels)
+
+    for split in self.splits:
+      result["distribution"].setdefault(split, [])
+      for class_name in self.classes:
+        result["distribution"][split].append({class_name: result["datasets"][split][class_name] / result["count"][split]})
+      result["distribution"][split] = sorted(result["distribution"][split], key=lambda x: list(x.values())[0], reverse=True)
+
+    return result
+
+
+class EvalObjectDetectionDataset(EvalObjectDetection):
+  def __init__(self, predictions, labels, split="test"):
+    self.split = split
+    super().__init__(predictions.image_labels[split], labels.image_labels[split], predictions.classes)
+    self.from_datasets()
+
+  """
+    Converted predictions of type [{"class 1": [x1,y1,x2,y2, confidence]}, ...]
+    Boxes of type xyxyn + confidence
+
+    Converted Labels of type [{"class 1": [[x1,y1,x2,y2],...], "class 2": [[], [],...]...}]
+    Boxes of type xyxyn
+  """
+  def from_datasets(self):
+    self.converted_predictions = []
+    self.converted_labels = []
+
+    list_prediction_images = list(self.predictions.keys())
+    list_prediction_images.sort()
+    list_labels_images = list(self.labels.keys())
+    list_labels_images.sort()
+    if not all(a == b for a, b in zip(list_prediction_images, list_labels_images)):
+      error("predictions and labels differ")
+      return
+
+    for image_name in list_prediction_images:
+      self.converted_labels.append(self.labels[image_name])
+      current_image_prediction = {}
+      for class_name, class_prediction in self.predictions[image_name].items():
+        current_image_prediction[class_name] = class_prediction
+        for i, current_class_prediction in enumerate(current_image_prediction[class_name]):
+          current_image_prediction[class_name][i].append(1.0)
+      self.converted_predictions.append(current_image_prediction)

{sciveo-0.1.45 → sciveo-0.1.47}/sciveo/ml/evaluation/object_detection.py

@@ -17,6 +17,28 @@ from sciveo.tools.logger import *
 from sciveo.ml.images.object_detection import *
 
 
+def compute_iou_xyxy(box1, box2):
+  """
+    box1, box2: [x1, y1, x2, y2] format.
+    Returns the IoU between the two boxes.
+  """
+  x1_inter = max(box1[0], box2[0])
+  y1_inter = max(box1[1], box2[1])
+  x2_inter = min(box1[2], box2[2])
+  y2_inter = min(box1[3], box2[3])
+
+  inter_area = max(0, x2_inter - x1_inter) * max(0, y2_inter - y1_inter)
+  box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1])
+  box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1])
+  union_area = box1_area + box2_area - inter_area
+
+  if union_area == 0:
+    return 0.0
+
+  return inter_area / union_area
+
+
+
 """
 
 Object Detection Evaluation
@@ -39,6 +61,10 @@ class EvalObjectDetection:
       boxes = predicted.boxes
       class_names = predicted.names
 
+      """
+        Converted predictions of type [{"class 1": [x1,y1,x2,y2, confidence]}, ...]
+        Boxes of type xyxyn + confidence
+      """
       converted_prediction = {}
       for i in range(len(boxes)):
         class_id = int(boxes.cls[i].item())
@@ -53,7 +79,14 @@ class EvalObjectDetection:
         converted_prediction[label_class].append(box_confidence)
       self.converted_predictions.append(converted_prediction)
 
-    # TODO: currently labels of type [ {"class 1": [ ['x1':x1,'y1':y1,'x2':x2,'y2':y2] ... ], "class 2": [ ['x1':x1,'y1':y1,'x2':x2,'y2':y2] ... ]} ... ]
+    self._convert_labels_dict()
+
+  """
+    Labels of type [ {"class 1": [ ['x1':x1,'y1':y1,'x2':x2,'y2':y2] ... ], "class 2": [ ['x1':x1,'y1':y1,'x2':x2,'y2':y2] ... ]} ... ]
+    Converted Labels of type [{"class 1": [[x1,y1,x2,y2],...], "class 2": [[], [],...]...}]
+    Boxes of type xyxyn
+  """
+  def _convert_labels_dict(self):
     self.converted_labels = []
     for label in self.labels:
       converted_label = {}
@@ -63,26 +96,6 @@ class EvalObjectDetection:
           converted_label[class_name].append([box['x1'], box['y1'], box['x2'], box['y2']])
       self.converted_labels.append(converted_label)
 
-  def compute_iou(self, box1, box2):
-    """
-    box1, box2: [x1, y1, x2, y2] format.
-    Returns the IoU between the two boxes.
-    """
-    x1_inter = max(box1[0], box2[0])
-    y1_inter = max(box1[1], box2[1])
-    x2_inter = min(box1[2], box2[2])
-    y2_inter = min(box1[3], box2[3])
-
-    inter_area = max(0, x2_inter - x1_inter) * max(0, y2_inter - y1_inter)
-    box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1])
-    box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1])
-    union_area = box1_area + box2_area - inter_area
-
-    if union_area == 0:
-      return 0.0
-
-    return inter_area / union_area
-
   def compute_ap(self, class_name, confidence_threshold=0.0):
     """
     Calculate the Average Precision based on IoU and confidence scores.
@@ -108,7 +121,7 @@ class EvalObjectDetection:
         gt_match = None
 
         for label_box in current_labels:
-          iou = self.compute_iou(prediction_box[:4], label_box)
+          iou = compute_iou_xyxy(prediction_box[:4], label_box)
           if iou > 0.5 and iou >= max_iou:
             max_iou = iou
             gt_match = label_box

{sciveo-0.1.45 → sciveo-0.1.47}/sciveo/ml/nlp/embeddings.py

@@ -132,7 +132,7 @@ class TextEmbedding(BaseML):
 
 
 class SentenceEmbedding(TextEmbedding):
-  def __init__(self, model_name='BAAI/bge-m3', cache_dir=None, device=None) -> None:
+  def __init__(self, model_name='softel/sentence-bge-v1.3', cache_dir=None, device=None) -> None:
     super().__init__(model_name=model_name, cache_dir=cache_dir, device=device)
     self.model = None
     self.max_tokens = 8192

{sciveo-0.1.45 → sciveo-0.1.47}/sciveo/monitoring/start.py

@@ -13,24 +13,22 @@ import os
 import time
 
 from sciveo.tools.logger import *
+from sciveo.tools.configuration import ConfigurationArguments
 from sciveo.monitoring.monitor import BaseMonitor
 
 
 class MonitorStart:
   def __init__(self, **kwargs):
-    self.default_arguments = {
+    self.arguments = ConfigurationArguments({
       "period": 120,
       "block": True,
       "fork": False,
       "fork_type": 0,
       "output_path": None,
-    }
-    self.arguments = {}
-    for k, v in self.default_arguments.items():
-      self.arguments[k] = kwargs.get(k, v)
+    }, **kwargs)
 
     if len(kwargs) == 0:
-      info("sciveo monitoring default options", self.default_arguments)
+      info("sciveo monitoring default options", self.arguments)
 
   def __call__(self):
     if self.arguments["fork"] and self.arguments["block"]:

{sciveo-0.1.45 → sciveo-0.1.47}/sciveo/tools/configuration.py

@@ -78,4 +78,21 @@ class GlobalConfiguration:
           if file_extension == ".json":
             self.read_json(file_path)
           else:
-            self.read_file(file_path)
+            self.read_file(file_path)
+
+
+class ConfigurationArguments:
+  def __init__(self, default, **kwargs) -> None:
+    self.default = default
+    self.arguments = {}
+    for k, v in self.default.items():
+      self.arguments[k] = kwargs.get(k, v)
+
+  def __getitem__(self, key):
+    return self.arguments[key]
+
+  def __len__(self):
+    return len(self.arguments)
+
+  def __repr__(self):
+    return self.arguments

sciveo-0.1.47/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.47'

{sciveo-0.1.45 → sciveo-0.1.47}/sciveo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.45
+Version: 0.1.47
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.45 → sciveo-0.1.47}/sciveo.egg-info/SOURCES.txt

@@ -91,6 +91,8 @@ sciveo/media/tools/__init__.py
 sciveo/media/tools/nvr.py
 sciveo/ml/__init__.py
 sciveo/ml/base.py
+sciveo/ml/dataset/__init__.py
+sciveo/ml/dataset/object_detection.py
 sciveo/ml/evaluation/__init__.py
 sciveo/ml/evaluation/object_detection.py
 sciveo/ml/images/__init__.py
@@ -140,6 +142,7 @@ test/test_complexity.py
 test/test_compress.py
 test/test_configuration.py
 test/test_crypto.py
+test/test_ml_datasets.py
 test/test_monitoring.py
 test/test_runner.py
 test/test_sampling.py

{sciveo-0.1.45 → sciveo-0.1.47}/test/test_compress.py

@@ -1,3 +1,14 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
 import unittest
 import json
 

{sciveo-0.1.45 → sciveo-0.1.47}/test/test_crypto.py

@@ -1,3 +1,14 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
 import unittest
 
 from sciveo.tools.crypto import *

sciveo-0.1.47/test/test_ml_datasets.py

@@ -0,0 +1,46 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import unittest
+import json
+
+from sciveo.ml.dataset.object_detection import *
+
+
+class TestMLDatasets(unittest.TestCase):
+  def test_object_detection(self):
+    HOME_PATH = os.path.expanduser("~")
+
+    # ds = YOLODataset()
+    # ds.load(f"{HOME_PATH}/data/test_yolo")
+    # debug("classes", ds.classes)
+    # ds.save(f"{HOME_PATH}/data/tmp/yolo-out", classes=["block", "vertical"], copy_images=False)
+
+    ds1 = ObjectDetectionDataset()
+    ds1.from_yolo(f"{HOME_PATH}/data/test_yolo")
+    stats = ds1.stats()
+    debug("stats", stats)
+    info("distribution", stats["distribution"])
+    # ds.save(f"{HOME_PATH}/data/tmp/od-1", copy_images=False)
+
+    ds2 = ObjectDetectionDataset()
+    ds2.from_yolo(f"{HOME_PATH}/data/job-1")
+    stats2 = ds2.stats()
+    debug("stats", stats2)
+    info("distribution", stats2["distribution"])
+
+    evaluator = EvalObjectDetectionDataset(ds1, ds2)
+    scores = evaluator.evaluate()
+    debug("eval", scores)
+
+
+if __name__ == '__main__':
+  unittest.main()

sciveo-0.1.45/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.45'