sciveo 0.1.44__tar.gz → 0.1.46__tar.gz

{sciveo-0.1.44 → sciveo-0.1.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.44
+Version: 0.1.46
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.44 → sciveo-0.1.46}/sciveo/cli.py

@@ -25,7 +25,7 @@ def main():
   parser.add_argument(
     'command',
     choices=[
-      'init', 'monitor', 'scan',
+      'init', 'monitor', 'scan', 'nvr',
       'media-server', 'media-run'
     ],
     help='Command to execute')
@@ -75,6 +75,9 @@ def main():
           fp.write(line + '\n')
     else:
       info(f"init, [{base_path}] already there")
+  elif args.command == 'nvr':
+    from sciveo.media.tools.nvr import VideoRecorder
+    VideoRecorder(args.input_path).start()
   elif args.command == 'media-server':
     from sciveo.media.pipelines.server import __START_SCIVEO_MEDIA_SERVER__
     __START_SCIVEO_MEDIA_SERVER__()

{sciveo-0.1.44 → sciveo-0.1.46}/sciveo/media/pipelines/server.py

@@ -16,18 +16,26 @@ from sciveo.tools.logger import *
 from sciveo.tools.daemon import DaemonBase
 from sciveo.tools.os import *
 from sciveo.tools.simple_counter import RunCounter
+from sciveo.tools.configuration import ConfigurationArguments
 from sciveo.tools.aws.priority_queue import SQSPriorityQueue
 from sciveo.media.pipelines.pipeline import *
 from sciveo.media.pipelines.queues import MediaJobState
 
 
 class MaintenanceDaemon(DaemonBase):
-  def __init__(self, period=3600):
-    super().__init__(period=period)
+  def __init__(self, **kwargs):
+    self.arguments = ConfigurationArguments({
+      "period": 3600,
+      "run_period_files": 24,
+      "retention_period_files": 30,
+    }, **kwargs)
+
+    super().__init__(period=self.arguments["period"])
+
     self.base_tmp_path = os.environ["WEB_MEDIA_PIPELINES_TMP_PATH"]
 
     self.timers = [
-      RunCounter(24, lambda: run_system_cmd(f"find {self.base_tmp_path} -mtime +30 -type f -delete")),
+      RunCounter(self.arguments['run_period_files'], lambda: run_system_cmd(f"find {self.base_tmp_path} -mtime +{self.arguments['retention_period_files']} -type f -delete")),
       # RunCounter(3, self.print_me),
     ]
 
@@ -59,7 +67,7 @@ class MediaJobQueueDaemon(DaemonBase):
 
 def __START_SCIVEO_MEDIA_SERVER__():
   daemons = [
-    MaintenanceDaemon(period=3600),
+    MaintenanceDaemon(period=3600, run_period_files=24, retention_period_files=30),
     MediaJobQueueDaemon(),
   ]
 

sciveo-0.1.46/sciveo/media/tools/nvr.py

@@ -0,0 +1,143 @@
+#
+# Stanislav Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact s.georgiev@softel.bg.
+#
+# 2025
+#
+
+import os
+import subprocess as sp
+import time
+import re
+import json
+import cv2
+import datetime
+
+from sciveo.tools.logger import *
+from sciveo.tools.daemon import DaemonBase
+from sciveo.tools.queue import TouchedFilePathQueue
+from sciveo.tools.simple_counter import RunCounter
+
+
+class VideoCameraCaptureDaemon(DaemonBase):
+  def __init__(self, cam_id, url, dst_path, max_video_len=60, transport="tcp"):
+    super().__init__()
+    self.cam_id = cam_id
+    self.url = url
+    self.dst_path = dst_path
+    self.max_video_len = max_video_len
+    self.transport = transport
+    self.cmd = [
+      "ffmpeg",
+      "-rtsp_transport", self.transport,
+      "-i", self.url,
+      "-c", "copy",
+      "-acodec", "aac",
+      "-f", "segment",
+      "-segment_time", f"{self.max_video_len}",
+      "-reset_timestamps", "1",
+      "-strftime", "1",
+      f"{self.dst_path}/{self.cam_id}___%Y-%m-%d___%H-%M-%S.mp4"
+    ]
+
+  def clear(self):
+    os.system(f"pgrep -f \"{self.url}\" |xargs kill -9")
+
+    files = [f for f in os.listdir(self.dst_path) if os.path.isfile(os.path.join(self.dst_path, f))]
+    for file_name in files:
+      if file_name.startswith(f"{self.cam_id}___"):
+        file_path = os.path.join(self.dst_path, file_name)
+        info("RM", file_path)
+        os.remove(file_path)
+
+  def loop(self):
+    info("start", self.cam_id)
+    while(True):
+      self.clear()
+
+      p = sp.Popen(self.cmd, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
+      p.wait()
+
+      self.clear()
+      warning(self.cam_id, "streaming interrupted, wait to retry...")
+      time.sleep(5)
+
+
+class VideoRecorder:
+  def __init__(self, path_configuration):
+    with open(path_configuration, 'r') as fp:
+      self.configuration = json.load(fp)
+    self.cams = []
+
+    for cam_id, cam_config in self.configuration["cam"].items():
+      cam = VideoCameraCaptureDaemon(
+        cam_id, cam_config["url"],
+        self.configuration["path"]["tmp"],
+        self.configuration.get("max_video_len", 60),
+        self.configuration.get("transport", "tcp")
+      )
+      self.cams.append(cam)
+
+    self.queue = TouchedFilePathQueue(self.configuration["path"]["tmp"], period=5, touched_timeout=5)
+    self.cleaner_timer = RunCounter(1000, self.clean_old_videos)
+
+  def start(self):
+    for cam in self.cams:
+      cam.start()
+
+    time.sleep(10)
+
+    while(True):
+      try:
+        file_name, file_path = self.queue.pop()
+        debug("pop", file_name, file_path)
+        self.process_file(file_name, file_path)
+        self.cleaner_timer.run()
+      except Exception as e:
+        exception(e)
+        time.sleep(1)
+
+  def process_file(self, file_name, file_path):
+    split = file_name.split("___")
+    if len(split) == 3:
+      cam_id = split[0]
+      video_date = split[1]
+      video_file_name = split[2]
+    else:
+      warning("wrong file format, removing", file_name, file_path)
+      os.remove(file_path)
+
+    match = re.match(r"(\d{2})\-(\d{2})\-(\d{2})\.mp4", video_file_name)
+    if not match:
+      warning("Invalid filename format")
+      video_file_name_split = video_file_name.split(".")
+      video_file_name = f"{video_file_name_split[0]}-{video_file_name_split[0]}.{video_file_name_split[1]}"
+    else:
+      hh, mm, ss = map(int, match.groups())
+      start_time = datetime.datetime(2000, 1, 1, hh, mm, ss)
+      end_time = start_time + datetime.timedelta(seconds=self.configuration["max_video_len"])
+      video_file_name = f"{start_time.strftime('%H.%M.%S')}-{end_time.strftime('%H.%M.%S')}.mp4"
+
+    video_base_path = os.path.join(self.configuration["path"]["video"], cam_id, video_date)
+    video_file_path = os.path.join(video_base_path, video_file_name)
+
+    debug("MV", file_path, "=>", video_file_path)
+    os.makedirs(video_base_path, exist_ok=True)
+    os.rename(file_path, video_file_path)
+
+  def clean_old_videos(self):
+    try:
+      days = self.configuration.get("video_retention_period", 7)
+      cmd = "find {} -mtime +{} -type f -delete".format(self.configuration["path"]["video"], days)
+      debug("cmd", cmd)
+      os.system(cmd)
+    except Exception as e:
+      excprint([self], e, cmd)
+
+
+if __name__ == '__main__':
+  VideoRecorder("./cams.json").start()

sciveo-0.1.46/sciveo/ml/dataset/object_detection.py

@@ -0,0 +1,243 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import os
+import shutil
+import yaml
+import json
+import time
+import math
+import numpy as np
+
+from sciveo.tools.logger import *
+
+
+def bbox_from_yolo(yolo_bbox):
+  yolo_bbox = [float(a) for a in yolo_bbox]
+  n, cx, cy, w, h = yolo_bbox
+  x1 = cx - w / 2
+  y1 = cy - h / 2
+  x2 = cx + w / 2
+  y2 = cy + h / 2
+  return [x1, y1, x2, y2]
+
+def bbox_to_yolo(bbox):
+  n, x1, y1, x2, y2 = bbox
+  cx = (x1 + x2) / 2
+  cy = (y1 + y2) / 2
+  w = x2 - x1
+  h = y2 - y1
+  return [n, cx, cy, w, h]
+
+
+class YOLODataset:
+  def __init__(self):
+    self.src_path = None
+    self.dst_path = None
+
+    self.classes = {}
+    self.splits = []
+    self.labels = {}
+
+  def load(self, base_path):
+    self.src_path = base_path
+    yaml_path = os.path.join(base_path, "data.yaml")
+    if not os.path.exists(yaml_path):
+      raise FileNotFoundError(f"data.yaml not found in {base_path}")
+
+    with open(yaml_path, "r") as f:
+      self.data_yaml = yaml.safe_load(f)
+
+    self.classes = self.data_yaml.get("names", {})
+    self.splits = [key for key in self.data_yaml.keys() if key not in ["names", "nc", "path"]]
+    self.labels = {split: self._load_labels_for_split(base_path, split) for split in self.splits}
+
+  def _load_labels_for_split(self, base_path, split: str):
+    images_dir = self.data_yaml.get(split)
+    labels_dir = images_dir.replace("images", "labels")
+    if not labels_dir:
+      return {}
+
+    images_path = os.path.join(base_path, images_dir)
+    labels_path = os.path.join(base_path, labels_dir)
+    if not os.path.exists(labels_path):
+      return {}
+
+    annotations = {}
+    for label_file in os.listdir(labels_path):
+      if label_file.endswith(".txt"):
+        image_id = label_file.replace(".txt", ".jpg")
+        label_path = os.path.join(labels_path, label_file)
+
+        with open(label_path, "r") as f:
+          annotations[image_id] = [line.strip().split() for line in f.readlines()]
+
+    return annotations
+
+  def save(self, base_path, classes=[], copy_images=True):
+    self.dst_path = base_path
+    os.makedirs(self.dst_path, exist_ok=True)
+
+    yaml_path = os.path.join(base_path, "data.yaml")
+
+    if not classes:
+      selected_classes = self.classes
+    else:
+      selected_classes = {idx: name for idx, name in self.classes.items() if name in classes}
+
+    class_mapping = {name: new_idx for new_idx, (idx, name) in enumerate(selected_classes.items())}
+
+    # data_yaml = {
+    #   "path": "./",
+    #   "names": {idx: name for idx, name in selected_classes.items()}
+    # }
+    # for split in self.splits:
+    #   data_yaml[split] = f"images/{split}"
+    # with open(yaml_path, "w") as f:
+    #   yaml.dump(data_yaml, f, default_flow_style=False)
+
+    with open(yaml_path, "w") as fp:
+      fp.write("  path: ./ # dataset root dir\n")
+      for split in self.splits:
+        fp.write(f"  test: images/{split}\n")
+      fp.write("  names:\n")
+      for class_id, class_name in selected_classes.items():
+        fp.write(f"    {class_id}: {class_name}\n")
+
+    for split, images in self.labels.items():
+      labels_path = os.path.join(base_path, "labels", split)
+      images_path = os.path.join(base_path, "images", split)
+      os.makedirs(labels_path, exist_ok=True)
+      os.makedirs(images_path, exist_ok=True)
+
+      for image_name, annotations in images.items():
+        label_file_path = os.path.join(labels_path, image_name.replace(".jpg", ".txt"))
+
+        with open(label_file_path, "w") as f:
+          for annotation in annotations:
+            class_id, *bbox = map(float, annotation)
+            class_id = int(class_id)
+
+            class_name = self.classes.get(class_id)
+            if class_name in class_mapping:
+              new_class_id = class_mapping[class_name]
+              f.write(f"{new_class_id} {' '.join(map(str, bbox))}\n")
+
+        if copy_images and self.src_path:
+          src_image_path = os.path.join(self.src_path, "images", split, image_name)
+          dst_image_path = os.path.join(images_path, image_name)
+          if not os.path.isfile(dst_image_path):
+            shutil.copy(src_image_path, dst_image_path)
+
+
+"""
+Object Detection Dataset
+
+labels in splits (for example train/val/test)
+every dataset split has image name as key which points to dict of classes as keys
+
+labels[split][image_name][class_name] is a list of bounding boxes of same class
+bboxes are of type nxyxy which means normalized [x1, y1, x2, y2] coordinates (upper left and down right point of the rectangle)
+
+"""
+class ObjectDetectionDataset:
+  def __init__(self):
+    self.classes = []
+    self.splits = []
+    self.labels = {}
+
+    self.src_path = None
+    self.dst_path = None
+
+  def save(self, base_path, copy_images=True, src_path=None):
+    if src_path is not None:
+      self.src_path = src_path
+
+    self.dst_path = base_path
+    for split in self.splits:
+      images_path = os.path.join(self.dst_path, split, "images")
+      os.makedirs(images_path, exist_ok=True)
+
+      labels_path = os.path.join(self.dst_path, split, "labels.json")
+      with open(labels_path, 'w') as fp:
+        json.dump(self.labels[split], fp, indent=2)
+
+      if copy_images and self.src_path:
+        src_images_path = os.path.join(self.src_path, "images", split)
+        for image_path, subdirs, image_files in os.walk(src_images_path):
+          for image_name in image_files:
+            src_path = os.path.join(image_path, image_name)
+            dst_path = os.path.join(images_path, image_name)
+            if not os.path.isfile(dst_path):
+              shutil.copy(src_path, dst_path)
+
+  def from_yolo(self, base_path):
+    self.src_path = base_path
+
+    self.raw = YOLODataset()
+    self.raw.load(base_path)
+
+    self.classes = []
+    for i in range(len(self.raw.classes)):
+      self.classes.append(self.raw.classes[i])
+    self.splits = self.raw.splits
+    self.image_labels = {}
+    self.labels = {}
+    for split, split_labels in self.raw.labels.items():
+      self.image_labels.setdefault(split, {})
+      self.labels.setdefault(split, {})
+      for image_id, image_labels in split_labels.items():
+        self.image_labels[split].setdefault(image_id, {})
+        for label in image_labels:
+          class_id = int(label[0])
+          class_name = self.raw.classes[class_id]
+          bbox = bbox_from_yolo(label)
+
+          self.image_labels[split][image_id].setdefault(class_name, [])
+          self.image_labels[split][image_id][class_name].append(bbox)
+
+          self.labels[split].setdefault(class_name, {})
+          self.labels[split][class_name].setdefault(image_id, [])
+          self.labels[split][class_name][image_id].append(bbox)
+
+  def stats(self):
+    result = {"datasets": {}, "count": {}, "distribution": {}}
+    for split in self.splits:
+      result["datasets"].setdefault(split, {})
+      result["count"].setdefault(split, 0)
+      for class_name in self.classes:
+        result["datasets"][split].setdefault(class_name, 0)
+        for image_name, image_labels in self.labels[split][class_name].items():
+          result["count"][split] += len(image_labels)
+          result["datasets"][split][class_name] += len(image_labels)
+
+    for split in self.splits:
+      result["distribution"].setdefault(split, [])
+      for class_name in self.classes:
+         result["distribution"][split].append({class_name: result["datasets"][split][class_name] / result["count"][split]})
+      result["distribution"][split] = sorted(result["distribution"][split], key=lambda x: list(x.values())[0], reverse=True)
+
+    return result
+
+
+
+if __name__ == "__main__":
+  # ds = YOLODataset()
+  # ds.load("/Users/sgeorgiev/develop/data/newspapers/test_yolo")
+  # debug("classes", ds.classes)
+  # ds.save(f"/Users/sgeorgiev/develop/data/newspapers/tmp/job-out", classes=["block", "vertical"], copy_images=True)
+
+  ds = ObjectDetectionDataset()
+  ds.from_yolo("/Users/sgeorgiev/develop/data/newspapers/test_yolo")
+  stats = ds.stats()
+  debug("stats", stats)
+  info("distribution", stats["distribution"])
+  # ds.save("/Users/sgeorgiev/develop/data/newspapers/tmp/od-1", copy_images=False)

{sciveo-0.1.44 → sciveo-0.1.46}/sciveo/monitoring/start.py

@@ -13,24 +13,22 @@ import os
 import time
 
 from sciveo.tools.logger import *
+from sciveo.tools.configuration import ConfigurationArguments
 from sciveo.monitoring.monitor import BaseMonitor
 
 
 class MonitorStart:
   def __init__(self, **kwargs):
-    self.default_arguments = {
+    self.arguments = ConfigurationArguments({
       "period": 120,
       "block": True,
       "fork": False,
       "fork_type": 0,
       "output_path": None,
-    }
-    self.arguments = {}
-    for k, v in self.default_arguments.items():
-      self.arguments[k] = kwargs.get(k, v)
+    }, **kwargs)
 
     if len(kwargs) == 0:
-      info("sciveo monitoring default options", self.default_arguments)
+      info("sciveo monitoring default options", self.arguments)
 
   def __call__(self):
     if self.arguments["fork"] and self.arguments["block"]:

{sciveo-0.1.44 → sciveo-0.1.46}/sciveo/tools/configuration.py

@@ -78,4 +78,21 @@ class GlobalConfiguration:
           if file_extension == ".json":
             self.read_json(file_path)
           else:
-            self.read_file(file_path)
+            self.read_file(file_path)
+
+
+class ConfigurationArguments:
+  def __init__(self, default, **kwargs) -> None:
+    self.default = default
+    self.arguments = {}
+    for k, v in self.default.items():
+      self.arguments[k] = kwargs.get(k, v)
+
+  def __getitem__(self, key):
+    return self.arguments[key]
+
+  def __len__(self):
+    return len(self.arguments)
+
+  def __repr__(self):
+    return self.arguments

{sciveo-0.1.44 → sciveo-0.1.46}/sciveo/tools/queue.py

@@ -29,3 +29,19 @@ class FilePathQueue:
         if self.filter(file_name, file_path):
           return file_name, file_path
       time.sleep(self.period)
+
+
+class TouchedFilePathQueue(FilePathQueue):
+  def __init__(self, path, period, touched_timeout):
+    super().__init__(path, period)
+    self.touched_timeout = touched_timeout
+
+  def filter(self, file_name, file_path):
+    return self.touched(file_path) > self.touched_timeout
+
+  def touched(self, path):
+    try:
+      last_touched = time.time() - os.path.getmtime(path)
+    except Exception:
+      last_touched = sys.maxsize
+    return last_touched

sciveo-0.1.46/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.46'

{sciveo-0.1.44 → sciveo-0.1.46}/sciveo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.44
+Version: 0.1.46
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.44 → sciveo-0.1.46}/sciveo.egg-info/SOURCES.txt

@@ -87,8 +87,12 @@ sciveo/media/pipelines/processors/video/video_frames.py
 sciveo/media/pipelines/processors/video/video_resample.py
 sciveo/media/pipelines/web/__init__.py
 sciveo/media/pipelines/web/server.py
+sciveo/media/tools/__init__.py
+sciveo/media/tools/nvr.py
 sciveo/ml/__init__.py
 sciveo/ml/base.py
+sciveo/ml/dataset/__init__.py
+sciveo/ml/dataset/object_detection.py
 sciveo/ml/evaluation/__init__.py
 sciveo/ml/evaluation/object_detection.py
 sciveo/ml/images/__init__.py
@@ -138,6 +142,7 @@ test/test_complexity.py
 test/test_compress.py
 test/test_configuration.py
 test/test_crypto.py
+test/test_ml_datasets.py
 test/test_monitoring.py
 test/test_runner.py
 test/test_sampling.py

{sciveo-0.1.44 → sciveo-0.1.46}/test/test_compress.py

@@ -1,3 +1,14 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
 import unittest
 import json
 

{sciveo-0.1.44 → sciveo-0.1.46}/test/test_crypto.py

@@ -1,3 +1,14 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
 import unittest
 
 from sciveo.tools.crypto import *

sciveo-0.1.46/test/test_ml_datasets.py

@@ -0,0 +1,36 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import unittest
+import json
+
+from sciveo.ml.dataset.object_detection import *
+
+
+class TestMLDatasets(unittest.TestCase):
+  def test_object_detection(self):
+    HOME_PATH = os.path.expanduser("~")
+
+    ds = YOLODataset()
+    ds.load(f"{HOME_PATH}/data/test_yolo")
+    debug("classes", ds.classes)
+    ds.save(f"{HOME_PATH}/data/tmp/yolo-out", classes=["block", "vertical"], copy_images=True)
+
+    ds = ObjectDetectionDataset()
+    ds.from_yolo(f"{HOME_PATH}/data/test_yolo")
+    stats = ds.stats()
+    debug("stats", stats)
+    info("distribution", stats["distribution"])
+    ds.save(f"{HOME_PATH}/data/tmp/od-1", copy_images=False)
+
+
+if __name__ == '__main__':
+  unittest.main()

sciveo-0.1.44/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.44'