sciveo 0.1.32__tar.gz → 0.1.34__tar.gz

{sciveo-0.1.32 → sciveo-0.1.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.32
+Version: 0.1.34
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo/cli.py

@@ -31,7 +31,7 @@ def main():
   parser.add_argument('--port', type=int, default=22, help='Host port number, used for network ops')
   parser.add_argument('--localhost', type=bool, default=False, help='Add localhost to list of hosts')
   parser.add_argument('--input-path', type=str, help='Input Path')
-  parser.add_argument('--output-path', type=str, help='Output Path')
+  parser.add_argument('--output-path', type=str, default=None, help='Output Path')
   parser.add_argument('--width', type=str, default=None, help='width')
   parser.add_argument('--height', type=str, default=None, help='height')
   parser.add_argument('--rate', type=int, help='Rate number')
@@ -41,7 +41,7 @@ def main():
 
   if args.command == 'monitor':
     from sciveo.monitoring.start import MonitorStart
-    MonitorStart(period=args.period, block=args.block)()
+    MonitorStart(period=args.period, block=args.block, output_path=args.output_path)()
   elif args.command == 'scan':
     from sciveo.network.tools import NetworkTools
     NetworkTools(timeout=args.timeout, localhost=args.localhost).scan_port(port=args.port, network=args.net)
@@ -53,7 +53,7 @@ def main():
       default_lines = [
         "secret_access_key=<your secret access key>",
         "api_base_url=https://sciveo.com",
-        "log_min_level=DEBUG"
+        "sci_log_level=DEBUG"
       ]
       with open(os.path.join(base_path, "default"), 'w') as fp:
         for line in default_lines:

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo/media/pipelines/pipeline.py

@@ -31,6 +31,7 @@ from sciveo.media.pipelines.processors.image.depth_esimation import *
 from sciveo.media.pipelines.processors.image.mask import *
 from sciveo.media.pipelines.processors.image.segmentation import *
 from sciveo.media.pipelines.processors.image.watermark import *
+from sciveo.media.pipelines.processors.image.object_detection import ImageObjectDetectionProcessor
 
 from sciveo.media.pipelines.processors.file.archive import *
 
@@ -84,6 +85,7 @@ class MediaPipeline:
       "image-segmentation": ImageSegmentation,
       "image-depth-estimation": ImageDepthEstimation,
       "image-embedding": ImageEmbedding,
+      "image-object-detection": ImageObjectDetectionProcessor,
 
       "sci-timeseries-predictor": TimeSeriesPredictorProcessor,
       "sci-timeseries-trainer": TimeSeriesTrainerProcessor,

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo/media/pipelines/processors/base.py

@@ -27,6 +27,9 @@ class BaseProcessor(BaseContentProcessor):
   def post_process(self):
     pass
 
+  def init_run(self):
+    pass
+
   def run(self, job, input):
     self.job = job
     self.job_id = job["id"]
@@ -35,6 +38,9 @@ class BaseProcessor(BaseContentProcessor):
       return []
     progress_per_media = self.max_progress / max(1, len(input))
     debug("run", job["id"], progress_per_media, "input", input)
+
+    self.init_run()
+
     for i, media in enumerate(input):
       if self.is_processor_run(job, media):
         try:

sciveo-0.1.34/sciveo/media/pipelines/processors/image/object_detection.py

@@ -0,0 +1,186 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import numpy as np
+import cv2
+
+from sciveo.tools.logger import *
+from sciveo.tools.simple_counter import Timer
+from sciveo.tools.common import *
+from sciveo.media.pipelines.processors.base import *
+
+
+class ObjectDetectorBase:
+  def __init__(self, model_path, device='cpu', colors=None):
+    self.model_path = model_path
+    self.device = device
+    if colors is None:
+      self.colors = [
+        (60, 180, 75),    # Green
+        (255, 255, 255),   # White
+        (245, 130, 48),   # Orange
+        (255, 225, 25),   # Yellow
+        (0, 130, 200),    # Blue
+        (230, 25, 75),    # Red
+        (145, 30, 180),   # Purple
+        (70, 240, 240),   # Cyan
+        (240, 50, 230),   # Magenta
+        (210, 245, 60),   # Lime
+        (250, 190, 212),  # Pink
+        (0, 128, 128),    # Teal
+        (220, 190, 255),  # Lavender
+        (170, 110, 40),   # Brown
+        (128, 0, 0),      # Maroon
+        (0, 0, 128),      # Navy
+        (128, 128, 0),    # Olive
+        (255, 215, 180),  # Peach
+        (255, 250, 200),  # Ivory
+        (170, 255, 195),  # Mint
+      ]
+    else:
+      self.colors = colors
+
+  def resize(self, image, h):
+    ratio = max(image.shape[0], image.shape[1]) / h
+    h, w = int(image.shape[0] / ratio), int(image.shape[1] / ratio)
+    return cv2.resize(image, (w, h))
+
+  def load(self, image_path):
+    image = cv2.imread(image_path)
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    return image
+
+  def read_images(self, images_paths):
+    X = []
+    for image_path in images_paths:
+      image = self.load(image_path)
+      image = self.resize(image)
+      X.append(image)
+    return X
+
+  def draw_label_inverted(self, frame, label_text, x_min, y_min, color, font=cv2.FONT_HERSHEY_SIMPLEX, font_scale=0.3, font_thickness=1):
+    text_size = cv2.getTextSize(label_text, font, font_scale, font_thickness)[0]
+    y_offset = text_size[1] + 4
+    x_offset = text_size[0] + 4
+    if y_min - y_offset >= 0:
+      text_background_top_left = (x_min, y_min - y_offset)
+      text_background_bottom_right = (x_min + x_offset, y_min)
+    else:
+      text_background_top_left = (x_min, y_min)
+      text_background_bottom_right = (x_min + x_offset, y_min + y_offset)
+
+    cv2.rectangle(frame, text_background_top_left, text_background_bottom_right, color, cv2.FILLED)
+    cv2.putText(frame, label_text, (x_min + 2, text_background_bottom_right[1] - 2), font, font_scale, (0,0,0), font_thickness)
+
+  def draw_object_rectangle_xyxy(self, frame, box, label, color, alpha=0.2, filled=True):
+    (x1, y1, x2, y2) = box
+
+    if filled:
+      rectangle = np.zeros((y2 - y1, x2 - x1, 3), dtype=np.uint8)
+      rectangle[:] = color
+      frame[y1:y2, x1:x2] = cv2.addWeighted(rectangle, alpha, frame[y1:y2, x1:x2], 1 - alpha, 0)
+
+    cv2.rectangle(frame, (x1, y1), (x2, y2), color, thickness=1)
+
+    self.draw_label_inverted(frame, label, x1, y1, color=color)
+
+  def draw_object_rectangle_xywh(frame, box, label, color, alpha=0.2, filled=True):
+    (x, y, w, h) = box
+    (x1, y1, x2, y2) = x, y, x + w, y + h
+    self.draw_object_rectangle_xyxy(frame, (x1, y1, x2, y2), label, color, alpha=alpha, filled=filled)
+
+
+class ObjectDetectorYOLO(ObjectDetectorBase):
+  def __init__(self, model_path="yolo11m.pt", device='cpu', colors=None):
+    super().__init__(model_path, device=device, colors=colors)
+    from ultralytics import YOLO
+    self.model = YOLO(self.model_path)
+
+  def predict_one(self, x, confidence_threshold=0.5):
+    return self.model.predict(x, device=self.device, conf=confidence_threshold, verbose=False)
+
+  def predict(self, X, max_n=64, confidence_threshold=0.5):
+    predictions = []
+
+    num_batches = math.ceil(len(X) / max_n)
+
+    for batch_idx in range(num_batches):
+      timer = Timer()
+      start_idx = batch_idx * max_n
+      end_idx = min((batch_idx + 1) * max_n, len(X))
+
+      batch_images = X[start_idx:end_idx]
+      batch_predictions = self.model.predict(batch_images, device=self.device, conf=confidence_threshold, verbose=False)
+      predictions.extend(batch_predictions)
+
+      elapsed = timer.stop()
+      FPS = len(batch_images) / elapsed
+      debug(f"batch {batch_idx} / {num_batches}", "elapsed", elapsed, "FPS", FPS, "len", len(batch_images))
+
+    return predictions
+
+  def resize(self, image):
+    return super().resize(image, 640)
+
+  def draw(self, image, detections, colors=None):
+    if colors is None:
+      colors = self.colors
+    height, width = image.shape[:2]
+    boxes = detections.boxes
+    class_names = detections.names
+    for i, box in enumerate(boxes):
+      bbox = (np.array(box.xyxyn[0]) * np.array([width, height, width, height])).astype(int).tolist()
+
+      confidence = box.conf[0].item()
+      class_id = int(box.cls[0].item())
+      label = class_names[class_id]
+      label_text = f"{label} {int(confidence * 100)}%"
+
+      color = colors[i % len(colors)]
+      self.draw_object_rectangle_xyxy(image, bbox, label_text, color)
+
+
+class ImageObjectDetectionProcessor(BaseProcessor):
+  def __init__(self, processor_config, max_progress) -> None:
+    super().__init__(processor_config, max_progress)
+    self.default.update({"JPEG_QUALITY": 80, "min_confidence": 0.5, "model_type": 0, "height": 720})
+
+  def init_run(self):
+    TPU = os.environ.get("MEDIA_PROCESSING_BACKEND", "cpu")
+    self.predictor = ObjectDetectorYOLO(model_path=["yolo11x.pt", "yolo11l.pt", "yolo11m.pt", "yolo11s.pt"][self["model_type"]], device=TPU)
+
+  def process(self, media):
+    try:
+      self.media = media
+      self.local_path = media["local_path"]
+
+      tag = "object-detections"
+      image = self.predictor.load(self.local_path)
+      image_resized = self.predictor.resize(image)
+
+      detections = self.predictor.predict_one([image_resized], confidence_threshold=self["min_confidence"])
+
+      # image_resized = self.predictor.super().resize(image, h=self["height"])
+      image_resized = cv2.cvtColor(image_resized, cv2.COLOR_RGB2BGR)
+      self.predictor.draw(image_resized, detections[0])
+      result_image_local_path = self.add_suffix_to_filename(self.local_path, tag)
+      cv2.imwrite(result_image_local_path, image_resized, [cv2.IMWRITE_JPEG_QUALITY, self["JPEG_QUALITY"]])
+
+      self.next_content(self.media, tag, result_image_local_path, w=image_resized.shape[1], h=image_resized.shape[0])
+    except Exception as e:
+      exception(e, self.media)
+    return self.media
+
+  def content_type(self):
+    return "image"
+
+  def name(self):
+    return "image-object-detection"

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo/monitoring/monitor.py

@@ -12,6 +12,7 @@
 import os
 import subprocess
 import time
+import json
 import datetime
 import socket
 import psutil
@@ -43,8 +44,11 @@ class MonitorTools:
 
 
 class BaseMonitor(DaemonBase):
-  def __init__(self, period=5):
+  def __init__(self, period=5, output_path=None):
     super().__init__(period=period)
+    self.output_path = output_path
+    if self.output_path is not None and os.path.isdir(self.output_path):
+      self.output_path = os.path.join(self.output_path, "sciveo_monitor.json")
 
     self.data = {
       "CPU": {},
@@ -78,6 +82,17 @@ class BaseMonitor(DaemonBase):
   def __call__(self):
     return self.data
 
+  def output_write(self):
+    if os.path.isfile(self.output_path):
+      with open(self.output_path, "rb+") as fp:
+        fp.seek(-2, 2)
+        fp.truncate()
+      with open(self.output_path, "a") as fp:
+        fp.write(f", {json.dumps(self.data)}]\n")
+    else:
+      with open(self.output_path, "w") as fp:
+        fp.write(json.dumps([self.data]) + "\n")
+
   def loop(self):
     self.get_cpu_usage()
     self.get_temperatures()
@@ -91,6 +106,12 @@ class BaseMonitor(DaemonBase):
 
     api_result = self.api.POST_SCI("monitor", {"data": self.data})
 
+    try:
+      if self.output_path is not None:
+        self.output_write()
+    except Exception as e:
+      error(e, "writing failed for", self.output_path)
+
     debug(self(), "api_result", api_result)
 
   def tail_logs(self):

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo/monitoring/start.py

@@ -22,7 +22,8 @@ class MonitorStart:
       "period": 120,
       "block": True,
       "fork": False,
-      "fork_type": 0
+      "fork_type": 0,
+      "output_path": None,
     }
     self.arguments = {}
     for k, v in self.default_arguments.items():
@@ -59,7 +60,7 @@ class MonitorStart:
 
   def start(self):
     period = max(self.arguments["period"], 5)
-    mon = BaseMonitor(period=period)
+    mon = BaseMonitor(period=period, output_path=self.arguments["output_path"])
     mon.start()
     mon.join()
 

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo/tools/logger.py

@@ -20,7 +20,7 @@ from sciveo.tools.configuration import GlobalConfiguration
 SCIVEO_LOGGER_NAME = "sciveo-log"
 
 _sciveo_global_config = GlobalConfiguration.get()
-_sciveo_log_min_level = _sciveo_global_config["LOG_MIN_LEVEL"]
+_sciveo_log_min_level = _sciveo_global_config["SCI_LOG_LEVEL"]
 _sciveo_log_lock = Lock()
 
 def _sciveo_get_logger(name):
@@ -29,7 +29,7 @@ def _sciveo_get_logger(name):
     with _sciveo_log_lock:
       if not logger.hasHandlers():
         log_min_level = logging.getLevelName(_sciveo_log_min_level)
-        if log_min_level.startswith("Level"):
+        if (isinstance(log_min_level, str) and log_min_level.startswith("Level")) or isinstance(log_min_level, int):
           log_min_level = "DEBUG"
         logger.setLevel(log_min_level)
 

sciveo-0.1.34/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.34'

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.32
+Version: 0.1.34
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo.egg-info/SOURCES.txt

@@ -66,6 +66,7 @@ sciveo/media/pipelines/processors/image/filters.py
 sciveo/media/pipelines/processors/image/generators.py
 sciveo/media/pipelines/processors/image/histogram.py
 sciveo/media/pipelines/processors/image/mask.py
+sciveo/media/pipelines/processors/image/object_detection.py
 sciveo/media/pipelines/processors/image/resize.py
 sciveo/media/pipelines/processors/image/segmentation.py
 sciveo/media/pipelines/processors/image/watermark.py

{sciveo-0.1.32 → sciveo-0.1.34}/sciveo.egg-info/requires.txt

@@ -56,6 +56,7 @@ diffusers>=0.0.0
 transformers>=0.0.0
 accelerate>=0.0.0
 annoy>=0.0.0
+ultralytics
 
 [media-ml]
 tensorflow>=0.0.0
@@ -66,6 +67,7 @@ diffusers>=0.0.0
 transformers>=0.0.0
 accelerate>=0.0.0
 annoy>=0.0.0
+ultralytics
 
 [media-server]
 fastapi

{sciveo-0.1.32 → sciveo-0.1.34}/setup.py

@@ -30,6 +30,7 @@ extras_require = {
         'tensorflow>=0.0.0', 'keras>=0.0.0',
         'torch>=0.0.0', 'torchvision>=0.0.0',
         'diffusers>=0.0.0', 'transformers>=0.0.0', 'accelerate>=0.0.0', 'annoy>=0.0.0',
+        'ultralytics'
       ]
 }
 

sciveo-0.1.32/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.32'