sciveo 0.1.72__tar.gz → 0.1.74__tar.gz

{sciveo-0.1.72 → sciveo-0.1.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.72
+Version: 0.1.74
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net
@@ -8,6 +8,8 @@ Provides-Extra: server
 Provides-Extra: media
 Provides-Extra: media-ml
 Provides-Extra: web
+Provides-Extra: db
+Provides-Extra: power
 Provides-Extra: all
 Provides-Extra: ml
 

{sciveo-0.1.72 → sciveo-0.1.74}/sciveo/cli.py

@@ -29,7 +29,7 @@ def main():
     choices=[
       'init', 'monitor', 'scan', 'nvr', 'predictors-server',
       'media-server', 'media-run',
-      'watchdog'
+      'watchdog',
     ],
     help='Command to execute')
 
@@ -54,12 +54,20 @@ def main():
   parser.add_argument('--threshold', type=float, default=None, help='Threshold')
   parser.add_argument('--execute', type=str, default=None, help='Execute command')
   parser.add_argument('--pid', type=int, default=None, help='Process PID')
+  parser.add_argument('--serial', type=str, default=None, help='Serial Name')
 
   args = parser.parse_args()
 
   if args.command == 'monitor':
-    from sciveo.monitoring.start import MonitorStart
-    MonitorStart(period=args.period, block=args.block, output_path=args.output_path)()
+    if args.src is None:
+      from sciveo.monitoring.start import MonitorStart
+      MonitorStart(period=args.period, block=args.block, output_path=args.output_path)()
+    elif args.src.startswith("power"):
+      from sciveo.monitoring.power.ems300 import PowerEMS300
+      mon = PowerEMS300(serial=args.serial, host=args.host, port=args.port, period=args.period, output_path=args.output_path)
+      mon.start()
+      while(True):
+        time.sleep(3600)
   elif args.command == 'scan':
     from sciveo.network.tools import NetworkTools
     host=args.host

sciveo-0.1.74/sciveo/db/read_table.py

@@ -0,0 +1,204 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2025
+#
+
+import os
+import json
+import psycopg2
+import numpy as np
+import pandas as pd
+from sciveo.tools.logger import *
+
+
+class BaseTable:
+  def __init__(self, table_name, config=None, batch_size=10_000, save_path=None, id_col="id"):
+    self.table_name = table_name
+    self.batch_size = batch_size
+    self.save_path = save_path
+    self.id_col = id_col
+    self.last_id = None
+
+    if config is None:
+      self.config = {
+        'dbname':   os.environ.get('DB_NAME'),
+        'user':     os.environ.get('DB_USERNAME'),
+        'password': os.environ.get('DB_PASSWORD'),
+        'host':     os.environ.get('DB_HOST', 'localhost'),
+        'port':     int(os.environ.get('DB_PORT', 5432)),
+      }
+    else:
+      self.config = config
+
+    self.conn = psycopg2.connect(**self.config)
+    self.cursor = self.conn.cursor()
+    self.df = pd.DataFrame()
+
+  def load_file(self, file_path=None):
+    if not os.path.exists(file_path):
+      warning(f"File not found: {file_path}")
+      self.latest_df = pd.DataFrame()
+      return self.latest_df
+
+    if file_path.endswith(".parquet"):
+      self.latest_df = pd.read_parquet(file_path)
+    else:
+      self.latest_df = pd.read_csv(file_path)
+    info(f"Loaded {len(self.latest_df)} records from {file_path}")
+    return self.latest_df
+
+  def load(self, base_path=None):
+    if base_path is None and self.save_path is not None:
+      base_path = self.save_path
+
+    if base_path is None:
+      self.df = pd.DataFrame()
+      return self.df
+
+    files = [os.path.join(base_path, f) for f in os.listdir(base_path) if f.startswith(self.table_name) and f.endswith(".parquet")]
+
+    if not files:
+      info(f"There are no files in {base_path} to read")
+      self.df = pd.DataFrame()
+      return self.df
+
+    dfs = []
+    for file_path in sorted(files):
+      try:
+        df_part = self.load_file(file_path)
+        from_id = int(df_part[self.id_col].min())
+        to_id = int(df_part[self.id_col].max())
+        dfs.append(df_part)
+        debug(f"Loaded {len(df_part)} records from {file_path} id: [{from_id} - {to_id}]")
+      except Exception as e:
+        warning(f"Failed to load {file_path}: {e}")
+
+    if dfs:
+      self.df = pd.concat(dfs, ignore_index=True)
+    else:
+      self.df = pd.DataFrame()
+
+    self.last_id = int(self.df[self.id_col].max()) if not self.df.empty else None
+
+    info(f"Total loaded records: {len(self.df)} last_id: {self.last_id}")
+    return self.df
+
+  def _sanitize_df(self, df):
+    PANDAS_MIN_YEAR = 1677
+    PANDAS_MAX_YEAR = 2262
+    datetime_cols = []
+
+    # datetime cleanup
+    for col in df.columns:
+      if pd.api.types.is_datetime64_any_dtype(df[col]) or 'date' in col.lower() or 'time' in col.lower():
+        datetime_cols.append(col)
+
+    for col in datetime_cols:
+      def safe_dt(x):
+        if pd.isna(x):
+          return pd.NaT
+        if isinstance(x, str):
+          return pd.to_datetime(x, errors='coerce')
+        if hasattr(x, 'year'):
+          year = x.year
+        else:
+          return pd.NaT
+        if PANDAS_MIN_YEAR <= year <= PANDAS_MAX_YEAR:
+          return pd.Timestamp(x)
+        return pd.NaT
+      df[col] = df[col].apply(safe_dt)
+
+    # drop rows with invalid datetime
+    df = df.dropna(subset=datetime_cols)
+
+    # object columns: convert dict/list to JSON string
+    for col in df.columns:
+      if df[col].dtype == object:
+        df[col] = df[col].apply(lambda x: json.dumps(x) if isinstance(x, (dict, list)) else str(x) if x is not None else None)
+
+    df = df.reset_index(drop=True)
+    return df
+
+  def fetch_next_batch(self, last_id=None):
+    query = f"SELECT * FROM {self.table_name}"
+    if last_id is not None:
+      query += f" WHERE id > {last_id}"
+    query += f" ORDER BY id ASC LIMIT {self.batch_size};"
+
+    self.cursor.execute(query)
+    rows = self.cursor.fetchall()
+    if not rows:
+      return pd.DataFrame()
+
+    colnames = [desc[0] for desc in self.cursor.description]
+    df_new = pd.DataFrame(rows, columns=colnames)
+    return self._sanitize_df(df_new)
+
+  def append(self, df_new):
+    if not df_new.empty:
+      self.df = pd.concat([self.df, df_new], ignore_index=True)
+      debug(f"Added {len(df_new)} new rows (total {len(self.df)})")
+    self.last_id = int(self.df[self.id_col].max()) if not self.df.empty else None
+    debug(f"last {self.id_col} [{self.last_id}]")
+
+  def sync_incremental(self, last_id=None):
+    if last_id is None:
+      last_id = int(self.df[self.id_col].max()) if not self.df.empty else None
+    debug("reading from", last_id)
+    df_new = self.fetch_next_batch(last_id)
+    self.append(df_new)
+    if df_new.empty:
+      info("No new rows found")
+    return df_new
+
+  def save(self, file_path=None):
+    if self.save_path is not None and self.last_id is not None:
+      from_id = int(self.latest_df[self.id_col].min())
+      to_id = int(self.latest_df[self.id_col].max())
+      file_path = os.path.join(self.save_path, f"{self.table_name}--{from_id}-{to_id}.parquet")
+
+    if file_path.endswith(".parquet"):
+      self.latest_df.to_parquet(file_path, index=False)
+    else:
+      self.latest_df.to_csv(file_path, index=False)
+    info(f"Saved {len(self.latest_df)} rows to {file_path}")
+
+  def update(self, last_id=None):
+    if self.df.empty:
+      self.load()
+    self.latest_df = self.sync_incremental(last_id=last_id)
+    self.save()
+    return self.latest_df
+
+  def close(self):
+    self.cursor.close()
+    self.conn.close()
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    self.close()
+
+
+if __name__ == "__main__":
+  with BaseTable("video_record_predictions", batch_size=10_000, save_path="/home/ml/video_predictions/data") as T:
+
+    T.update(last_id=1_000_000)
+    T.load()
+    if not T.df.empty:
+      debug(T.df.columns)
+      debug(T.df[["key", "people", "is_valid", "updated_at", "data"]].head())
+
+    while(True):
+      df = T.update()
+      if df.empty:
+        info("Finished", T.df.shape)
+        break
+      info("read", df.shape, df[["key", "people"]].head())

sciveo-0.1.74/sciveo/monitoring/power/ems300.py

@@ -0,0 +1,107 @@
+#
+# Stanislav Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact s.georgiev@softel.bg.
+#
+# 2025
+#
+
+import os
+import json
+import datetime
+import time
+from pymodbus.client import ModbusTcpClient
+from sciveo.tools.logger import *
+from sciveo.tools.daemon import DaemonBase
+# from sciveo.api.base import APIRemoteClient
+from sciveo.monitoring.power.tools import *
+
+
+class PowerEMS300(DaemonBase):
+  def __init__(self, serial, host, port=502, device_id=247, delay=0.01, period=30, output_path=None):
+    super().__init__(period=period)
+    self.serial = serial
+    self.host = host
+    self.port = port
+    self.device_id = device_id
+    self.delay = delay
+    self.client = None
+
+    if output_path is not None and os.path.isdir(output_path):
+      self.output_path = os.path.join(output_path, f"{self.serial}.json")
+    else:
+      self.output_path = output_path
+
+    # self.api = APIRemoteClient()
+    self.client = ModbusTcpClient(host=self.host, port=self.port)
+
+  def connect(self):
+    if not self.client.connected:
+      if self.client.connect():
+        info(type(self.client).__name__, "connected", (self.host, self.port))
+      else:
+        error("Connect FAIL", (self.host, self.port))
+    return self.client.connected
+
+  def send(self, data):
+    api_result = self.api.POST_SCI("monitor", {"data": data})
+    debug(data, "api_result", api_result)
+
+  def output_write(self, data):
+    if os.path.isfile(self.output_path):
+      with open(self.output_path, "rb+") as fp:
+        fp.seek(-2, 2)
+        fp.truncate()
+      with open(self.output_path, "a") as fp:
+        fp.write(f", {json.dumps(data)}]\n")
+    else:
+      with open(self.output_path, "w") as fp:
+        fp.write(json.dumps([data]) + "\n")
+
+  def save(self, data):
+    try:
+      if self.output_path is not None:
+        info("save", self.output_path, data)
+        self.output_write(data)
+    except Exception as e:
+      error(e, "writing failed for", self.output_path)
+
+  def loop(self):
+    if not self.connect():
+      warning("Not connected", (self.host, self.port))
+      return
+
+    try:
+      data = {
+        "local_time": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        "serial": self.serial,
+      }
+
+      current_info = [8026, 8044, 8060, 8776, 8778, 10101, 10103, 10105, 10713, 10715, 10716, 10945]
+      for addr in current_info:
+        result = read_input_registers(self.client, EMS300_REG_MAP, addr, self.device_id)
+        if result is not None:
+          value, name, dtype = result
+          debug(f"{name}: {value}")
+          data[name] = value
+        time.sleep(self.delay)
+
+      # self.send(data)
+      self.save(data)
+    except Exception as e:
+      exception(e)
+
+  def close(self):
+    self.client.close()
+
+
+if __name__ == "__main__":
+
+  mon = PowerEMS300(serial="EMS300-1", host="192.168.86.184", port=502, period=30, output_path="ems300.json")
+  mon.start()
+
+  while(True):
+    time.sleep(30)

sciveo-0.1.74/sciveo/monitoring/power/sim.py

@@ -0,0 +1,71 @@
+#
+# Stanislav Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact s.georgiev@softel.bg.
+#
+# 2025
+#
+
+import time
+import threading
+import time
+from pymodbus.server import StartTcpServer
+from pymodbus.datastore import ModbusServerContext, ModbusSparseDataBlock
+from threading import Thread
+from sciveo.tools.logger import *
+from sciveo.tools.daemon import DaemonBase
+
+
+# Example read-only registers
+# INITIAL_REGISTERS = {
+#     8044: 12500,   # Plant Active Power
+#     8060: 8200,    # PV Active Power
+#     8038: -1500,   # Storage Power
+#     8048: 600,     # Reactive Power
+#     9082: 3000,    # Dynamic Allowable Charge Power
+#     9084: 3500,    # Dynamic Allowable Discharge Power
+# }
+INITIAL_REGISTERS = {
+  8060: 8200,    # PV Active Power
+}
+
+class SimulatorEMS300(DaemonBase):
+  def __init__(self, host="127.0.0.1", port=5020, period=1):
+    super().__init__(num_threads=1, period=period)
+    self.host = host
+    self.port = port
+
+    # Only input registers
+    self.ir_block = ModbusSparseDataBlock(INITIAL_REGISTERS.copy())
+
+    self.context = ModbusServerContext(
+      { 247: {  # unit_id -> datastore dict
+          'di': ModbusSparseDataBlock({}),   # discrete inputs
+          'co': ModbusSparseDataBlock({}),   # coils
+          'hr': ModbusSparseDataBlock({}),   # holding registers
+          'ir': self.ir_block                # input registers
+        }
+      },
+      single=False  # multiple units not used here, keep False
+    )
+
+  def loop(self):
+    # Slowly vary PV Active Power
+    pv = self.ir_block.getValues(8060, count=1)[0]
+    pv = (pv + 50) % 15000
+    self.ir_block.setValues(8060, [pv])
+    info(f"Simulated PV Active Power: {pv}")
+
+  def start_server(self):
+    StartTcpServer(self.context, address=(self.host, self.port))
+
+
+if __name__ == "__main__":
+    sim = SimulatorEMS300(host="127.0.0.1", port=1502)
+    sim.start()
+    sim.start_server()
+    while True:
+      time.sleep(10)