sciveo 0.1.21__tar.gz → 0.1.23__tar.gz

{sciveo-0.1.21 → sciveo-0.1.23}/PKG-INFO

@@ -1,10 +1,14 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.21
+Version: 0.1.23
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net
+Provides-Extra: media
+Provides-Extra: media-server
+Provides-Extra: media-ml
 Provides-Extra: all
+Provides-Extra: media-all
 
 # SCIVEO - ML/AI and Scientific tools
 

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/__init__.py

@@ -16,8 +16,8 @@ try:
   import os
   import time
 
-  from sciveo.common.tools.logger import *
-  from sciveo.common.tools.daemon import TasksDaemon, __upload_content__
+  from sciveo.tools.logger import *
+  from sciveo.tools.daemon import TasksDaemon, __upload_content__
   from sciveo.content.runner import ProjectRunner
   from sciveo.content.dataset import Dataset
   from sciveo.monitoring.start import MonitorStart

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/api/base.py

@@ -14,8 +14,8 @@ import json
 from urllib import request, parse
 from urllib.error import HTTPError
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.configuration import GlobalConfiguration
+from sciveo.tools.logger import *
+from sciveo.tools.configuration import GlobalConfiguration
 
 
 class APIRemoteClient:

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/api/upload.py

@@ -15,7 +15,7 @@ from urllib import request, parse
 from urllib.error import HTTPError
 import requests
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 from sciveo.api.base import *
 
 

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/cli.py

@@ -13,18 +13,18 @@
 import os
 import argparse
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.timers import Timer
+from sciveo.tools.logger import *
+from sciveo.tools.timers import Timer
 from sciveo.monitoring.start import MonitorStart
 from sciveo.network.tools import NetworkTools
-from sciveo.common.tools.configuration import GlobalConfiguration
+from sciveo.tools.configuration import GlobalConfiguration
 
 
 def main():
   config = GlobalConfiguration.get()
 
   parser = argparse.ArgumentParser(description='sciveo CLI')
-  parser.add_argument('command', choices=['init', 'monitor', 'scan'], help='Command to execute')
+  parser.add_argument('command', choices=['init', 'monitor', 'scan', 'media-server'], help='Command to execute')
   parser.add_argument('--period', type=int, default=120, help='Period in seconds')
   parser.add_argument('--block', type=bool, default=True, help='Block flag')
   parser.add_argument('--auth', type=str, default=config['secret_access_key'], help='Auth secret access key')
@@ -53,6 +53,9 @@ def main():
           fp.write(line + '\n')
     else:
       info(f"init, [{base_path}] already there")
+  elif args.command == 'media-server':
+    from sciveo.media.pipelines.server import __START_SCIVEO_MEDIA_SERVER__
+    __START_SCIVEO_MEDIA_SERVER__()
   else:
     warning(args.command, "not implemented")
 

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/common/configuration.py

@@ -11,7 +11,7 @@
 
 import json
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 
 
 list_hyperparameters = list(set([

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/common/model.py

@@ -15,7 +15,7 @@ import datetime
 
 import numpy as np
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 
 
 class ModelInfo:

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/common/optimizers.py

@@ -11,7 +11,7 @@
 
 import numpy as np
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 
 
 class BaseOptimizer:

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/common/sampling.py

@@ -12,7 +12,7 @@
 from itertools import product
 import numpy as np
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 from sciveo.common.configuration import Configuration
 from sciveo.common.optimizers import BaseOptimizer, AdamOptimizer
 

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/content/dataset.py

@@ -14,7 +14,7 @@ import time
 
 import numpy as np
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 
 
 class Dataset:

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/content/experiment.py

@@ -15,10 +15,10 @@ import datetime
 
 import numpy as np
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.daemon import __upload_content__ as UPLOAD
-from sciveo.common.tools.formating import format_elapsed_time
-from sciveo.common.tools.hardware import HardwareInfo
+from sciveo.tools.logger import *
+from sciveo.tools.daemon import __upload_content__ as UPLOAD
+from sciveo.tools.formating import format_elapsed_time
+from sciveo.tools.hardware import HardwareInfo
 from sciveo.common.configuration import Configuration
 from sciveo.common.model import ModelInfo
 from sciveo.content.dataset import Dataset

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/content/project.py

@@ -14,7 +14,7 @@ import time
 
 import numpy as np
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 from sciveo.api.base import *
 from sciveo.common.configuration import Configuration
 from sciveo.content.experiment import Experiment, RemoteExperiment

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/content/runner.py

@@ -11,7 +11,7 @@
 
 import os
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 from sciveo.common.sampling import RandomSampler, GridSampler, AutoSampler
 from sciveo.content.project import RemoteProject, LocalProject
 

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/monitoring/monitor.py

@@ -20,10 +20,10 @@ import re
 import uuid
 import numpy as np
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.daemon import DaemonBase
-from sciveo.common.tools.hardware import *
-from sciveo.common.tools.formating import format_memory_size
+from sciveo.tools.logger import *
+from sciveo.tools.daemon import DaemonBase
+from sciveo.tools.hardware import *
+from sciveo.tools.formating import format_memory_size
 from sciveo.api.base import APIRemoteClient
 
 

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/monitoring/start.py

@@ -12,7 +12,7 @@
 import os
 import time
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 from sciveo.monitoring.monitor import BaseMonitor
 
 

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/network/camera.py

@@ -15,8 +15,8 @@ import cv2
 import numpy as np
 from scapy.all import sniff, IP, TCP, UDP
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.timers import FPSCounter
+from sciveo.tools.logger import *
+from sciveo.tools.timers import FPSCounter
 from sciveo.network.tools import StreamSniffer
 
 

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/network/sniffer.py

@@ -13,8 +13,8 @@ import socket
 import threading
 from scapy.all import sniff, IP, TCP
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.timers import Timer
+from sciveo.tools.logger import *
+from sciveo.tools.timers import Timer
 
 
 class StreamSniffer:

{sciveo-0.1.21 → sciveo-0.1.23}/sciveo/network/tools.py

@@ -13,8 +13,8 @@ import socket
 import ipaddress
 import threading
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.timers import Timer
+from sciveo.tools.logger import *
+from sciveo.tools.timers import Timer
 
 
 class NetworkTools:

sciveo-0.1.23/sciveo/tools/array.py

@@ -0,0 +1,149 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import numpy as np
+import copy
+
+
+def split_sized(arr, chunk_size, residual=True):
+  result = []
+  num_sets = int(len(arr) / chunk_size)
+  from_idx = 0
+  to_idx = 0
+  for i in range(num_sets):
+    to_idx = from_idx + chunk_size
+    result.append(arr[from_idx:to_idx])
+    from_idx = to_idx
+
+  if residual:
+    result.append(arr[from_idx:])
+
+  return result
+
+
+def split_ratios(arr, ratios=[0.75, 0.20, 0.05]):
+  if sum(ratios) != 1.0:
+    raise Exception("invalid ratios sum: {}".format(sum(ratios)))
+
+  ratios_idx = []
+  current_id = 0
+  for r in ratios:
+    ratios_idx.append(current_id)
+    current_id += int(len(arr) * r)
+
+  result = []
+  for i in range(1, len(ratios_idx)):
+    result.append(arr[ratios_idx[i - 1] : ratios_idx[i]])
+  result.append(arr[ratios_idx[i]:])
+
+  return result
+
+
+def split_dataset(arr, ratios={"train": 0.85, "val": 0.10, "test": 0.05}):
+  ratios_keys = []
+  ratios_values = []
+  for k, v in ratios.items():
+    ratios_keys.append(k)
+    ratios_values.append(v)
+
+  split_keys = split_ratios(arr, ratios_values)
+
+  dataset = {}
+  for i, k in enumerate(ratios_keys):
+    dataset[k] = split_keys[i]
+  return dataset
+
+
+def copy_dict_values(src, dst, keys):
+  for k in keys:
+    if k in src:
+      dst[k] = src[k]
+
+
+"""
+ Search d2 keys into d1, when found, merge these keys recursive into the section found.
+"""
+def merge_dicts(d1, d2):
+  for k, v in d1.items():
+    if k in d2:
+      if isinstance(v, dict):
+        merge_dicts(v, d2[k])
+      else:
+        d1[k] = d2[k]
+    else:
+      if isinstance(v, dict):
+        merge_dicts(v, d2)
+  return d1
+
+def merge_dicts2(d1, d2, r={}):
+  for k, v in d1.items():
+    r[k] = copy.deepcopy(v)
+    if k in d2:
+      if isinstance(v, dict):
+        merge_dicts2(v, d2[k], r[k])
+      else:
+        d1[k] = d2[k]
+        r[k] = copy.deepcopy(d2[k])
+    else:
+      if isinstance(v, dict):
+        merge_dicts2(v, d2, r[k])
+      else:
+        for k2, v2 in d2.items():
+          if k2 not in d1:
+            r[k2] = copy.deepcopy(v2)
+  return r
+
+
+def split_sized(arr, chunk_size, residual=True):
+  result = []
+  num_sets = int(len(arr) / chunk_size)
+  from_idx = 0
+  to_idx = 0
+  for i in range(num_sets):
+    to_idx = from_idx + chunk_size
+    result.append(arr[from_idx:to_idx])
+    from_idx = to_idx
+
+  if residual:
+    result.append(arr[from_idx:])
+
+  return result
+
+def split_ratios(arr, ratios=[0.75, 0.20, 0.05]):
+  if sum(ratios) != 1.0:
+    raise Exception("invalid ratios sum: {}".format(sum(ratios)))
+
+  ratios_idx = []
+  current_id = 0
+  for r in ratios:
+    ratios_idx.append(current_id)
+    current_id += int(len(arr) * r)
+
+  result = []
+  for i in range(1, len(ratios_idx)):
+    result.append(arr[ratios_idx[i - 1] : ratios_idx[i]])
+  result.append(arr[ratios_idx[i]:])
+
+  return result
+
+def split_dataset(arr, ratios={"train": 0.85, "val": 0.10, "test": 0.05}):
+  ratios_keys = []
+  ratios_values = []
+  for k, v in ratios.items():
+    ratios_keys.append(k)
+    ratios_values.append(v)
+
+  split_keys = split_ratios(arr, ratios_values)
+
+  dataset = {}
+  for i, k in enumerate(ratios_keys):
+    dataset[k] = split_keys[i]
+  return dataset

sciveo-0.1.23/sciveo/tools/common.py

@@ -0,0 +1,98 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import re
+import datetime
+import os
+import os.path
+import json
+import cv2
+import skvideo.io
+
+from sciveo.tools.logger import *
+
+
+def get_resize_resoulution(FH, FW, max_dim):
+  if FH <= max_dim:
+    debug("get_resize_resoulution", [FH, FW], max_dim)
+    return FH, FW
+  h = max_dim
+  w = int(FW * h / FH)
+  if w % 2 == 1:
+    w += 1
+  return h, w
+
+def get_frame_resolution_max_dim(frame, max_dim):
+  FH = frame.shape[0]
+  FW = frame.shape[1]
+
+  if FH > FW:
+    return get_resize_resoulution(FH, FW, max_dim)
+  else:
+    w, h = get_resize_resoulution(FW, FH, max_dim)
+    return h, w
+
+def get_frame_resolution(frame, h):
+  FH = frame.shape[0]
+  FW = frame.shape[1]
+  return get_resize_resoulution(FH, FW, h)
+
+
+class VideoWriterBase:
+  def __init__(self, local_video_path, w=640):
+    self.local_video_path = local_video_path
+    self.w = w
+    self.draw_frame = True
+
+  def use_frame_draw(self):
+    return self.draw_frame
+
+  def write(self, frame, detections=None):
+    pass
+
+  def close(self):
+    pass
+
+class VideoWriterDummy(VideoWriterBase):
+  def __init__(self, local_video_path, w=640):
+    super().__init__(local_video_path, w)
+    debug("init", self.local_video_path)
+
+  def write(self, frame, detections=None):
+    pass
+
+  def close(self):
+    debug("close", self.local_video_path)
+    pass
+
+class VideoWriterFFMPEG(VideoWriterBase):
+  def __init__(self, local_video_path, w=640, crf='35', aux_params={}):
+    super().__init__(local_video_path, w)
+
+    output_params = {'-vcodec':'libx264', '-crf':crf, '-preset':'medium', '-vf':'format=yuv420p', '-profile:v':'baseline', '-level':'3.0'}
+    output_params = {**output_params, **aux_params}
+    info(self.local_video_path, output_params)
+    self.out = skvideo.io.FFmpegWriter(self.local_video_path, outputdict=output_params)
+
+  def write(self, frame, detections=None):
+    self.out.writeFrame(frame[:,:,::-1])
+
+  def close(self):
+    self.out.close()
+    debug("closed", self.local_video_path)
+
+  @staticmethod
+  def new(local_video_path, w=640, crf='35'):
+    if int(os.environ.setdefault('PRODUCTION_MODE', "1")):
+      result = VideoWriterFFMPEG(local_video_path, w, crf)
+    else:
+      result = VideoWriterDummy(local_video_path, w)
+    return result

{sciveo-0.1.21/sciveo/common → sciveo-0.1.23/sciveo}/tools/compress.py

@@ -1,8 +1,19 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
 import json
 import gzip
 import base64
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 
 
 class CompressJsonData:

{sciveo-0.1.21/sciveo/common → sciveo-0.1.23/sciveo}/tools/crypto.py

@@ -1,9 +1,20 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
 import json
 import base64
 from Crypto.Cipher import AES
 from Crypto.Random import get_random_bytes
 
-from sciveo.common.tools.logger import *
+from sciveo.tools.logger import *
 
 
 class CryptoJsonData:

{sciveo-0.1.21/sciveo/common → sciveo-0.1.23/sciveo}/tools/daemon.py

@@ -13,8 +13,8 @@ import threading
 import signal
 import time
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.synchronized import ListQueue
+from sciveo.tools.logger import *
+from sciveo.tools.synchronized import ListQueue
 from sciveo.api.upload import APIFileUploader
 
 

{sciveo-0.1.21/sciveo/common → sciveo-0.1.23/sciveo}/tools/hardware.py

@@ -15,8 +15,8 @@ import uuid
 import random
 import string
 
-from sciveo.common.tools.logger import *
-from sciveo.common.tools.formating import format_memory_size
+from sciveo.tools.logger import *
+from sciveo.tools.formating import format_memory_size
 
 
 def new_guid_uuid():

sciveo-0.1.23/sciveo/tools/http.py

@@ -0,0 +1,134 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import json
+from urllib import request, parse
+from urllib.error import HTTPError
+import requests
+
+from sciveo.tools.logger import *
+from sciveo.tools.synchronized import ListQueue
+
+
+def POST_JSON(url, data_json, headers={}, timeout=30):
+  result = False
+  try:
+    #debug("POST", url, data_json, headers)
+    response = requests.post(url, json=data_json, headers=headers, timeout=timeout)
+    result = response.json()
+    debug("POST", result)
+  except HTTPError as e:
+    error(e, data_json)
+  except Exception as e:
+    error(e, data_json)
+  return result
+
+def POST(url, data, headers={}, timeout=30):
+  result = False
+  try:
+    debug("POST", url, data, headers)
+    data = parse.urlencode(data).encode("utf-8")
+    resp = request.urlopen(request.Request(url, data=data, headers=headers), timeout=timeout)
+    result = json.loads(resp.read())
+    debug("POST", resp.status, result)
+  except HTTPError as e:
+    error(e, data)
+  except Exception as e:
+    error(e, data)
+  return result
+
+def GET(url, headers={}, timeout=30):
+  result = False
+  try:
+    # debug("GET", url)
+    response = requests.get(url, headers=headers, timeout=timeout)
+    result = response.json()
+    # debug("GET", result)
+  except HTTPError as e:
+    error(e)
+  except Exception as e:
+    error(e)
+  return result
+
+
+class HTTPQueueSynchronized:
+  _queue = None
+
+  @staticmethod
+  def queue():
+    if HTTPQueueSynchronized._queue is None:
+      HTTPQueueSynchronized._queue = ListQueue("HTTPQueue")
+    return HTTPQueueSynchronized._queue
+
+  @staticmethod
+  def size():
+    return HTTPQueueSynchronized.queue().size()
+
+  @staticmethod
+  def get(url,
+          url_prefix="{}/api/v1".format(os.environ.get('API_BASE_URL', 'api.smiveo.com')),
+          access_token=os.environ.get('AUTH_KEY_SENSORS_API', "none")
+          ):
+    debug("HTTPQueueSynchronized::GET", url)
+    try:
+      url = url_prefix + url
+      url = HTTPQueueSynchronized.url_append(url, {"access_token": access_token})
+      HTTPQueueSynchronized.queue().push({"method": "GET", "url": url, "data": None})
+    except Exception as e:
+      error(e, "HTTPQueueSynchronized::GET", url)
+
+  @staticmethod
+  def post(url, data,
+           url_prefix="{}/api/v1".format(os.environ.get('API_BASE_URL', 'api.smiveo.com')),
+           access_token=os.environ.get('AUTH_KEY_SENSORS_API', "none")
+          ):
+    debug("HTTPQueueSynchronized::POST", url, data)
+    try:
+      url = url_prefix + url
+      data["access_token"] = access_token
+      HTTPQueueSynchronized.queue().push({"method": "POST", "url": url, "data": parse.urlencode(data).encode("utf-8")})
+    except Exception as e:
+      error(e, "HTTPQueueSynchronized::POST", url, data)
+
+  @staticmethod
+  def pop(block=True, timeout=None, retries=5):
+    req = HTTPQueueSynchronized.queue().pop(timeout=timeout)
+    try:
+      while(retries > 0):
+        if HTTPQueueSynchronized.save(req["method"], req["url"], req["data"]):
+          break
+        retries -= 1
+    except Exception as e:
+      error(e, "HTTPQueueSynchronized::POP")
+
+  @staticmethod
+  def url_append(url, url_args):
+    for k, v in url_args.items():
+      if "?" in url:
+        and_token = "&"
+      else:
+        and_token = "?"
+      url += "{}{}={}".format(and_token, k, v)
+    return url
+
+  @staticmethod
+  def save(method, url, data, timeout=10):
+    try:
+      debug("HTTPQueueSynchronized", method, url, data)
+      resp = request.urlopen(request.Request(url, data=data), timeout=timeout)
+      debug("HTTPQueueSynchronized", resp.status, resp.read())
+    except HTTPError as e:
+      error(e, "HTTPQueueSynchronized::HTTPError", method, url, data)
+      return False
+    except Exception as e:
+      error(e, "HTTPQueueSynchronized::Exception", method, url, data)
+      return False
+    return True