sciveo 0.1.43__tar.gz → 0.1.45__tar.gz

{sciveo-0.1.43 → sciveo-0.1.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.43
+Version: 0.1.45
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.43 → sciveo-0.1.45}/sciveo/cli.py

@@ -11,6 +11,7 @@
 #
 
 import os
+import json
 import argparse
 
 from sciveo.tools.logger import *
@@ -24,7 +25,7 @@ def main():
   parser.add_argument(
     'command',
     choices=[
-      'init', 'monitor', 'scan',
+      'init', 'monitor', 'scan', 'nvr',
       'media-server', 'media-run'
     ],
     help='Command to execute')
@@ -34,7 +35,9 @@ def main():
   parser.add_argument('--auth', type=str, default=config['secret_access_key'], help='Auth secret access key')
   parser.add_argument('--timeout', type=float, default=1.0, help='Timeout')
   parser.add_argument('--net', type=str, default=None, help='Network like 192.168.10.0/24')
+  parser.add_argument('--host', type=str, default=None, help='Host ip or name')
   parser.add_argument('--port', type=int, default=22, help='Host port number, used for network ops')
+  parser.add_argument('--ports', type=str, default="[]", help='Host ports list')
   parser.add_argument('--localhost', type=bool, default=False, help='Add localhost to list of hosts')
   parser.add_argument('--input-path', type=str, help='Input Path')
   parser.add_argument('--output-path', type=str, default=None, help='Output Path')
@@ -52,7 +55,11 @@ def main():
     MonitorStart(period=args.period, block=args.block, output_path=args.output_path)()
   elif args.command == 'scan':
     from sciveo.network.tools import NetworkTools
-    NetworkTools(timeout=args.timeout, localhost=args.localhost).scan_port(port=args.port, network=args.net)
+    host=args.host
+    if host is None:
+      NetworkTools(timeout=args.timeout, localhost=args.localhost).scan_port(port=args.port, network=args.net)
+    else:
+      NetworkTools(timeout=args.timeout, ports=json.loads(args.ports)).scan_host(host)
   elif args.command == 'init':
     home = os.path.expanduser('~')
     base_path = os.path.join(home, '.sciveo')
@@ -68,6 +75,9 @@ def main():
           fp.write(line + '\n')
     else:
       info(f"init, [{base_path}] already there")
+  elif args.command == 'nvr':
+    from sciveo.media.tools.nvr import VideoRecorder
+    VideoRecorder(args.input_path).start()
   elif args.command == 'media-server':
     from sciveo.media.pipelines.server import __START_SCIVEO_MEDIA_SERVER__
     __START_SCIVEO_MEDIA_SERVER__()

sciveo-0.1.45/sciveo/media/tools/nvr.py

@@ -0,0 +1,143 @@
+#
+# Stanislav Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact s.georgiev@softel.bg.
+#
+# 2025
+#
+
+import os
+import subprocess as sp
+import time
+import re
+import json
+import cv2
+import datetime
+
+from sciveo.tools.logger import *
+from sciveo.tools.daemon import DaemonBase
+from sciveo.tools.queue import TouchedFilePathQueue
+from sciveo.tools.simple_counter import RunCounter
+
+
+class VideoCameraCaptureDaemon(DaemonBase):
+  def __init__(self, cam_id, url, dst_path, max_video_len=60, transport="tcp"):
+    super().__init__()
+    self.cam_id = cam_id
+    self.url = url
+    self.dst_path = dst_path
+    self.max_video_len = max_video_len
+    self.transport = transport
+    self.cmd = [
+      "ffmpeg",
+      "-rtsp_transport", self.transport,
+      "-i", self.url,
+      "-c", "copy",
+      "-acodec", "aac",
+      "-f", "segment",
+      "-segment_time", f"{self.max_video_len}",
+      "-reset_timestamps", "1",
+      "-strftime", "1",
+      f"{self.dst_path}/{self.cam_id}___%Y-%m-%d___%H-%M-%S.mp4"
+    ]
+
+  def clear(self):
+    os.system(f"pgrep -f \"{self.url}\" |xargs kill -9")
+
+    files = [f for f in os.listdir(self.dst_path) if os.path.isfile(os.path.join(self.dst_path, f))]
+    for file_name in files:
+      if file_name.startswith(f"{self.cam_id}___"):
+        file_path = os.path.join(self.dst_path, file_name)
+        info("RM", file_path)
+        os.remove(file_path)
+
+  def loop(self):
+    info("start", self.cam_id)
+    while(True):
+      self.clear()
+
+      p = sp.Popen(self.cmd, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
+      p.wait()
+
+      self.clear()
+      warning(self.cam_id, "streaming interrupted, wait to retry...")
+      time.sleep(5)
+
+
+class VideoRecorder:
+  def __init__(self, path_configuration):
+    with open(path_configuration, 'r') as fp:
+      self.configuration = json.load(fp)
+    self.cams = []
+
+    for cam_id, cam_config in self.configuration["cam"].items():
+      cam = VideoCameraCaptureDaemon(
+        cam_id, cam_config["url"],
+        self.configuration["path"]["tmp"],
+        self.configuration.get("max_video_len", 60),
+        self.configuration.get("transport", "tcp")
+      )
+      self.cams.append(cam)
+
+    self.queue = TouchedFilePathQueue(self.configuration["path"]["tmp"], period=5, touched_timeout=5)
+    self.cleaner_timer = RunCounter(1000, self.clean_old_videos)
+
+  def start(self):
+    for cam in self.cams:
+      cam.start()
+
+    time.sleep(10)
+
+    while(True):
+      try:
+        file_name, file_path = self.queue.pop()
+        debug("pop", file_name, file_path)
+        self.process_file(file_name, file_path)
+        self.cleaner_timer.run()
+      except Exception as e:
+        exception(e)
+        time.sleep(1)
+
+  def process_file(self, file_name, file_path):
+    split = file_name.split("___")
+    if len(split) == 3:
+      cam_id = split[0]
+      video_date = split[1]
+      video_file_name = split[2]
+    else:
+      warning("wrong file format, removing", file_name, file_path)
+      os.remove(file_path)
+
+    match = re.match(r"(\d{2})\-(\d{2})\-(\d{2})\.mp4", video_file_name)
+    if not match:
+      warning("Invalid filename format")
+      video_file_name_split = video_file_name.split(".")
+      video_file_name = f"{video_file_name_split[0]}-{video_file_name_split[0]}.{video_file_name_split[1]}"
+    else:
+      hh, mm, ss = map(int, match.groups())
+      start_time = datetime.datetime(2000, 1, 1, hh, mm, ss)
+      end_time = start_time + datetime.timedelta(seconds=self.configuration["max_video_len"])
+      video_file_name = f"{start_time.strftime('%H.%M.%S')}-{end_time.strftime('%H.%M.%S')}.mp4"
+
+    video_base_path = os.path.join(self.configuration["path"]["video"], cam_id, video_date)
+    video_file_path = os.path.join(video_base_path, video_file_name)
+
+    debug("MV", file_path, "=>", video_file_path)
+    os.makedirs(video_base_path, exist_ok=True)
+    os.rename(file_path, video_file_path)
+
+  def clean_old_videos(self):
+    try:
+      days = self.configuration.get("video_retention_period", 7)
+      cmd = "find {} -mtime +{} -type f -delete".format(self.configuration["path"]["video"], days)
+      debug("cmd", cmd)
+      os.system(cmd)
+    except Exception as e:
+      excprint([self], e, cmd)
+
+
+if __name__ == '__main__':
+  VideoRecorder("./cams.json").start()

sciveo-0.1.45/sciveo/network/tools.py

@@ -0,0 +1,204 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import socket
+import ipaddress
+import threading
+import json
+
+from sciveo.tools.logger import *
+from sciveo.tools.timers import Timer
+
+
+class NetworkTools:
+  def __init__(self, **kwargs):
+    self.default_arguments = {
+      "timeout": 1.0,
+      "localhost": False,
+      "host": None,
+      "ports": []
+    }
+
+    self.arguments = {}
+    for k, v in self.default_arguments.items():
+      self.arguments[k] = kwargs.get(k, v)
+
+    self.net_classes = ["192.168.", "10."]
+    for i in range(16, 32):
+      self.net_classes.append(f"172.{i}.")
+
+    self.data = {}
+    self.data_lock = threading.Lock()
+
+    # self.known_ports = [20, 21, 22, 23, 25, 53, 67, 68, 69, 80, 110, 123, 143, 161, 162, 179, 194, 389, 443, 465, 514, 587, 636, 993, 995, 1080, 1194, 1433, 1521, 1723, 2049, 2082, 2083, 2086, 2087, 2095, 2096, 2181, 2375, 2376, 3000, 3306, 3389, 5432, 5900, 5984, 6379, 8080, 8443, 8888, 9000, 9090, 9200, 9300, 11211, 27017]
+    self.known_ports = {
+      20: socket.SOCK_STREAM,  # FTP Data (TCP)
+      21: socket.SOCK_STREAM,  # FTP Control (TCP)
+      22: socket.SOCK_STREAM,  # SSH (TCP)
+      23: socket.SOCK_STREAM,  # Telnet (TCP)
+      25: socket.SOCK_STREAM,  # SMTP (TCP)
+      53: socket.SOCK_DGRAM,   # DNS (UDP, but also uses TCP for large queries)
+      67: socket.SOCK_DGRAM,   # DHCP Server (UDP)
+      68: socket.SOCK_DGRAM,   # DHCP Client (UDP)
+      69: socket.SOCK_DGRAM,   # TFTP (UDP)
+      80: socket.SOCK_STREAM,  # HTTP (TCP)
+      110: socket.SOCK_STREAM, # POP3 (TCP)
+      123: socket.SOCK_DGRAM,  # NTP (UDP)
+      143: socket.SOCK_STREAM, # IMAP (TCP)
+      161: socket.SOCK_DGRAM,  # SNMP (UDP)
+      162: socket.SOCK_DGRAM,  # SNMP Trap (UDP)
+      179: socket.SOCK_STREAM, # BGP (TCP)
+      194: socket.SOCK_STREAM, # IRC (TCP)
+      389: socket.SOCK_STREAM, # LDAP (TCP)
+      443: socket.SOCK_STREAM, # HTTPS (TCP)
+      465: socket.SOCK_STREAM, # SMTPS (TCP)
+      514: socket.SOCK_DGRAM,  # Syslog (UDP)
+      587: socket.SOCK_STREAM, # SMTP Submission (TCP)
+      636: socket.SOCK_STREAM, # LDAPS (TCP)
+      993: socket.SOCK_STREAM, # IMAPS (TCP)
+      995: socket.SOCK_STREAM, # POP3S (TCP)
+      1080: socket.SOCK_STREAM, # SOCKS (TCP)
+      1194: socket.SOCK_DGRAM,  # OpenVPN (UDP)
+      1433: socket.SOCK_STREAM, # MS-SQL (TCP)
+      1521: socket.SOCK_STREAM, # Oracle DB (TCP)
+      1723: socket.SOCK_STREAM, # PPTP (TCP)
+      2049: socket.SOCK_DGRAM,  # NFS (UDP)
+      2082: socket.SOCK_STREAM, # cPanel (TCP)
+      2083: socket.SOCK_STREAM, # cPanel SSL (TCP)
+      2086: socket.SOCK_STREAM, # WHM (TCP)
+      2087: socket.SOCK_STREAM, # WHM SSL (TCP)
+      2095: socket.SOCK_STREAM, # Webmail (TCP)
+      2096: socket.SOCK_STREAM, # Webmail SSL (TCP)
+      2181: socket.SOCK_STREAM, # ZooKeeper (TCP)
+      2375: socket.SOCK_STREAM, # Docker (TCP)
+      2376: socket.SOCK_STREAM, # Docker SSL (TCP)
+      3000: socket.SOCK_STREAM, # Node.js (TCP)
+      3306: socket.SOCK_STREAM, # MySQL (TCP)
+      3389: socket.SOCK_STREAM, # RDP (TCP)
+      5432: socket.SOCK_STREAM, # PostgreSQL (TCP)
+      5900: socket.SOCK_STREAM, # VNC (TCP)
+      5984: socket.SOCK_STREAM, # CouchDB (TCP)
+      6379: socket.SOCK_STREAM, # Redis (TCP)
+      8080: socket.SOCK_STREAM, # HTTP Alt (TCP)
+      8443: socket.SOCK_STREAM, # HTTPS Alt (TCP)
+      8888: socket.SOCK_STREAM, # cPanel Alt (TCP)
+      9000: socket.SOCK_STREAM, # PHP-FPM (TCP)
+      9090: socket.SOCK_STREAM, # Prometheus (TCP)
+      9200: socket.SOCK_STREAM, # Elasticsearch (TCP)
+      9300: socket.SOCK_STREAM, # Elasticsearch Cluster (TCP)
+      11211: socket.SOCK_DGRAM, # Memcached (UDP)
+      27017: socket.SOCK_STREAM, # MongoDB (TCP)
+    }
+
+  def get_local_nets(self):
+    list_local_ips = []
+    try:
+      import netifaces
+      interfaces = netifaces.interfaces()
+      for interface in interfaces:
+        addrs = netifaces.ifaddresses(interface)
+        if netifaces.AF_INET in addrs:
+          ip = addrs[netifaces.AF_INET][0]['addr']
+          for net_class in self.net_classes:
+            if ip.startswith(net_class):
+              list_local_ips.append(ip)
+    except Exception as e:
+      warning("netifaces not installed")
+    return list_local_ips
+
+  def generate_ip_list(self, base_ip):
+    octets = base_ip.split('.')
+    network_prefix = '.'.join(octets[:3])
+    return [f'{network_prefix}.{i}' for i in range(1, 255)]
+
+  def scan_port(self, port=22, network=None):
+    t = Timer()
+    self.data["scan"] = {}
+    self.data["scan"].setdefault(port, [])
+
+    if network is None:
+      list_local_ips = self.get_local_nets()
+      # debug("scan_port", "list_local_ips", list_local_ips)
+      for local_ip in list_local_ips:
+        list_ip = self.generate_ip_list(local_ip)
+        self.scan_port_hosts(list_ip, port)
+    else:
+      list_ip = []
+      net = ipaddress.ip_network(network, strict=False)
+      for ip in net.hosts():
+        list_ip.append(str(ip))
+      self.scan_port_hosts(list_ip, port)
+
+    if self.arguments["localhost"]:
+      self.scan_port_hosts(["127.0.0.1"], port)
+    self.data["scan"][port].sort(key=lambda ip: int(ip.split('.')[-1]))
+    info(f"scan_port [{port}] elapsed time {t.stop():.1f}s", self.data["scan"][port])
+    return self.data["scan"][port]
+
+  def scan_port_hosts(self, list_ip, port=22):
+    timeout = self.arguments["timeout"]
+    list_threads = []
+    for ip in list_ip:
+      t = threading.Thread(target=self.scan_host_port, args=(ip, port, timeout))
+      t.start()
+      list_threads.append(t)
+    for t in list_threads:
+      t.join()
+
+  def test_socket(self, ip, port, timeout, transport=socket.SOCK_STREAM):
+    try:
+      with socket.socket(socket.AF_INET, transport) as sock:
+        sock.settimeout(timeout)
+        if transport == socket.SOCK_STREAM:
+          result = sock.connect_ex((ip, port))
+          if result == 0:
+            return True
+        elif transport == socket.SOCK_DGRAM:
+          sock.sendto(b"", (ip, port))
+          try:
+            data, addr = sock.recvfrom(1)
+            return True
+          except socket.timeout:
+            # warning("sock timeout", (ip, port, transport))
+            pass
+    except socket.error:
+      pass
+    return False
+
+  def scan_host_port(self, ip, port, timeout):
+    transport = self.known_ports.get(port, socket.SOCK_STREAM)
+    if self.test_socket(ip, port, timeout, transport):
+      with self.data_lock:
+        if "scan" in self.data:
+          self.data["scan"][port].append(ip)
+        if "host" in self.data:
+          self.data["host"][ip].append(port)
+
+  def scan_host(self, host):
+    timer = Timer()
+    self.data.setdefault("host", {})
+    self.data["host"][host] = []
+    timeout = self.arguments["timeout"]
+    list_threads = []
+    if len(self.arguments["ports"]) == 0:
+      list_ports = list(self.known_ports.keys())
+    else:
+      list_ports = self.arguments["ports"]
+    for port in list_ports:
+      thr = threading.Thread(target=self.scan_host_port, args=(host, port, timeout))
+      thr.start()
+      list_threads.append(thr)
+    for thr in list_threads:
+      thr.join()
+
+    self.data["host"][host].sort(key=lambda port: int(port))
+    info(f"scan host [{host}] elapsed time {timer.stop():.1f}s", self.data["host"][host])
+    return self.data["host"][host]

{sciveo-0.1.43 → sciveo-0.1.45}/sciveo/tools/queue.py

@@ -29,3 +29,19 @@ class FilePathQueue:
         if self.filter(file_name, file_path):
           return file_name, file_path
       time.sleep(self.period)
+
+
+class TouchedFilePathQueue(FilePathQueue):
+  def __init__(self, path, period, touched_timeout):
+    super().__init__(path, period)
+    self.touched_timeout = touched_timeout
+
+  def filter(self, file_name, file_path):
+    return self.touched(file_path) > self.touched_timeout
+
+  def touched(self, path):
+    try:
+      last_touched = time.time() - os.path.getmtime(path)
+    except Exception:
+      last_touched = sys.maxsize
+    return last_touched

sciveo-0.1.45/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.45'

{sciveo-0.1.43 → sciveo-0.1.45}/sciveo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.43
+Version: 0.1.45
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.43 → sciveo-0.1.45}/sciveo.egg-info/SOURCES.txt

@@ -87,6 +87,8 @@ sciveo/media/pipelines/processors/video/video_frames.py
 sciveo/media/pipelines/processors/video/video_resample.py
 sciveo/media/pipelines/web/__init__.py
 sciveo/media/pipelines/web/server.py
+sciveo/media/tools/__init__.py
+sciveo/media/tools/nvr.py
 sciveo/ml/__init__.py
 sciveo/ml/base.py
 sciveo/ml/evaluation/__init__.py

sciveo-0.1.43/sciveo/network/tools.py

@@ -1,102 +0,0 @@
-#
-# Pavlin Georgiev, Softel Labs
-#
-# This is a proprietary file and may not be copied,
-# distributed, or modified without express permission
-# from the owner. For licensing inquiries, please
-# contact pavlin@softel.bg.
-#
-# 2024
-#
-
-import socket
-import ipaddress
-import threading
-
-from sciveo.tools.logger import *
-from sciveo.tools.timers import Timer
-
-
-class NetworkTools:
-  def __init__(self, **kwargs):
-    self.default_arguments = {
-      "timeout": 1.0,
-      "localhost": False,
-    }
-
-    self.arguments = {}
-    for k, v in self.default_arguments.items():
-      self.arguments[k] = kwargs.get(k, v)
-
-    self.net_classes = ["192.168.", "10."]
-    for i in range(16, 32):
-      self.net_classes.append(f"172.{i}.")
-
-    self.data = {"scan": {}}
-    self.data_lock = threading.Lock()
-
-  def get_local_nets(self):
-    list_local_ips = []
-    try:
-      import netifaces
-      interfaces = netifaces.interfaces()
-      for interface in interfaces:
-        addrs = netifaces.ifaddresses(interface)
-        if netifaces.AF_INET in addrs:
-          ip = addrs[netifaces.AF_INET][0]['addr']
-          for net_class in self.net_classes:
-            if ip.startswith(net_class):
-              list_local_ips.append(ip)
-    except Exception as e:
-      warning("netifaces not installed")
-    return list_local_ips
-
-  def generate_ip_list(self, base_ip):
-    octets = base_ip.split('.')
-    network_prefix = '.'.join(octets[:3])
-    return [f'{network_prefix}.{i}' for i in range(1, 255)]
-
-  def scan_port(self, port=22, network=None):
-    t = Timer()
-    self.data["scan"].setdefault(port, [])
-
-    if network is None:
-      list_local_ips = self.get_local_nets()
-      # debug("scan_port", "list_local_ips", list_local_ips)
-      for local_ip in list_local_ips:
-        list_ip = self.generate_ip_list(local_ip)
-        self.scan_port_hosts(list_ip, port)
-    else:
-      list_ip = []
-      net = ipaddress.ip_network(network, strict=False)
-      for ip in net.hosts():
-        list_ip.append(str(ip))
-      self.scan_port_hosts(list_ip, port)
-
-    if self.arguments["localhost"]:
-      self.scan_port_hosts(["127.0.0.1"], port)
-    self.data["scan"][port].sort(key=lambda ip: int(ip.split('.')[-1]))
-    info(f"scan_port [{port}] elapsed time {t.stop():.1f}s", self.data["scan"][port])
-    return self.data["scan"][port]
-
-  def scan_port_hosts(self, list_ip, port=22):
-    timeout = self.arguments["timeout"]
-    list_threads = []
-    for ip in list_ip:
-      t = threading.Thread(target=self.scan_host_port, args=(ip, port, timeout))
-      t.start()
-      list_threads.append(t)
-    for t in list_threads:
-      t.join()
-
-  def scan_host_port(self, ip, port, timeout):
-    try:
-      with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.settimeout(timeout)
-        result = sock.connect_ex((ip, port))
-        if result == 0:
-          with self.data_lock:
-            self.data["scan"][port].append(ip)
-        # debug("scan_ports", ip, port, result)
-    except socket.error:
-      pass

sciveo-0.1.43/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.43'