sciveo 0.1.72__tar.gz → 0.1.73__tar.gz

{sciveo-0.1.72 → sciveo-0.1.73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.72
+Version: 0.1.73
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net
@@ -8,6 +8,8 @@ Provides-Extra: server
 Provides-Extra: media
 Provides-Extra: media-ml
 Provides-Extra: web
+Provides-Extra: db
+Provides-Extra: power
 Provides-Extra: all
 Provides-Extra: ml
 

sciveo-0.1.73/sciveo/db/read_table.py

@@ -0,0 +1,204 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2025
+#
+
+import os
+import json
+import psycopg2
+import numpy as np
+import pandas as pd
+from sciveo.tools.logger import *
+
+
+class BaseTable:
+  def __init__(self, table_name, config=None, batch_size=10_000, save_path=None, id_col="id"):
+    self.table_name = table_name
+    self.batch_size = batch_size
+    self.save_path = save_path
+    self.id_col = id_col
+    self.last_id = None
+
+    if config is None:
+      self.config = {
+        'dbname':   os.environ.get('DB_NAME'),
+        'user':     os.environ.get('DB_USERNAME'),
+        'password': os.environ.get('DB_PASSWORD'),
+        'host':     os.environ.get('DB_HOST', 'localhost'),
+        'port':     int(os.environ.get('DB_PORT', 5432)),
+      }
+    else:
+      self.config = config
+
+    self.conn = psycopg2.connect(**self.config)
+    self.cursor = self.conn.cursor()
+    self.df = pd.DataFrame()
+
+  def load_file(self, file_path=None):
+    if not os.path.exists(file_path):
+      warning(f"File not found: {file_path}")
+      self.latest_df = pd.DataFrame()
+      return self.latest_df
+
+    if file_path.endswith(".parquet"):
+      self.latest_df = pd.read_parquet(file_path)
+    else:
+      self.latest_df = pd.read_csv(file_path)
+    info(f"Loaded {len(self.latest_df)} records from {file_path}")
+    return self.latest_df
+
+  def load(self, base_path=None):
+    if base_path is None and self.save_path is not None:
+      base_path = self.save_path
+
+    if base_path is None:
+      self.df = pd.DataFrame()
+      return self.df
+
+    files = [os.path.join(base_path, f) for f in os.listdir(base_path) if f.startswith(self.table_name) and f.endswith(".parquet")]
+
+    if not files:
+      info(f"There are no files in {base_path} to read")
+      self.df = pd.DataFrame()
+      return self.df
+
+    dfs = []
+    for file_path in sorted(files):
+      try:
+        df_part = self.load_file(file_path)
+        from_id = int(df_part[self.id_col].min())
+        to_id = int(df_part[self.id_col].max())
+        dfs.append(df_part)
+        debug(f"Loaded {len(df_part)} records from {file_path} id: [{from_id} - {to_id}]")
+      except Exception as e:
+        warning(f"Failed to load {file_path}: {e}")
+
+    if dfs:
+      self.df = pd.concat(dfs, ignore_index=True)
+    else:
+      self.df = pd.DataFrame()
+
+    self.last_id = int(self.df[self.id_col].max()) if not self.df.empty else None
+
+    info(f"Total loaded records: {len(self.df)} last_id: {self.last_id}")
+    return self.df
+
+  def _sanitize_df(self, df):
+    PANDAS_MIN_YEAR = 1677
+    PANDAS_MAX_YEAR = 2262
+    datetime_cols = []
+
+    # datetime cleanup
+    for col in df.columns:
+      if pd.api.types.is_datetime64_any_dtype(df[col]) or 'date' in col.lower() or 'time' in col.lower():
+        datetime_cols.append(col)
+
+    for col in datetime_cols:
+      def safe_dt(x):
+        if pd.isna(x):
+          return pd.NaT
+        if isinstance(x, str):
+          return pd.to_datetime(x, errors='coerce')
+        if hasattr(x, 'year'):
+          year = x.year
+        else:
+          return pd.NaT
+        if PANDAS_MIN_YEAR <= year <= PANDAS_MAX_YEAR:
+          return pd.Timestamp(x)
+        return pd.NaT
+      df[col] = df[col].apply(safe_dt)
+
+    # drop rows with invalid datetime
+    df = df.dropna(subset=datetime_cols)
+
+    # object columns: convert dict/list to JSON string
+    for col in df.columns:
+      if df[col].dtype == object:
+        df[col] = df[col].apply(lambda x: json.dumps(x) if isinstance(x, (dict, list)) else str(x) if x is not None else None)
+
+    df = df.reset_index(drop=True)
+    return df
+
+  def fetch_next_batch(self, last_id=None):
+    query = f"SELECT * FROM {self.table_name}"
+    if last_id is not None:
+      query += f" WHERE id > {last_id}"
+    query += f" ORDER BY id ASC LIMIT {self.batch_size};"
+
+    self.cursor.execute(query)
+    rows = self.cursor.fetchall()
+    if not rows:
+      return pd.DataFrame()
+
+    colnames = [desc[0] for desc in self.cursor.description]
+    df_new = pd.DataFrame(rows, columns=colnames)
+    return self._sanitize_df(df_new)
+
+  def append(self, df_new):
+    if not df_new.empty:
+      self.df = pd.concat([self.df, df_new], ignore_index=True)
+      debug(f"Added {len(df_new)} new rows (total {len(self.df)})")
+    self.last_id = int(self.df[self.id_col].max()) if not self.df.empty else None
+    debug(f"last {self.id_col} [{self.last_id}]")
+
+  def sync_incremental(self, last_id=None):
+    if last_id is None:
+      last_id = int(self.df[self.id_col].max()) if not self.df.empty else None
+    debug("reading from", last_id)
+    df_new = self.fetch_next_batch(last_id)
+    self.append(df_new)
+    if df_new.empty:
+      info("No new rows found")
+    return df_new
+
+  def save(self, file_path=None):
+    if self.save_path is not None and self.last_id is not None:
+      from_id = int(self.latest_df[self.id_col].min())
+      to_id = int(self.latest_df[self.id_col].max())
+      file_path = os.path.join(self.save_path, f"{self.table_name}--{from_id}-{to_id}.parquet")
+
+    if file_path.endswith(".parquet"):
+      self.latest_df.to_parquet(file_path, index=False)
+    else:
+      self.latest_df.to_csv(file_path, index=False)
+    info(f"Saved {len(self.latest_df)} rows to {file_path}")
+
+  def update(self, last_id=None):
+    if self.df.empty:
+      self.load()
+    self.latest_df = self.sync_incremental(last_id=last_id)
+    self.save()
+    return self.latest_df
+
+  def close(self):
+    self.cursor.close()
+    self.conn.close()
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    self.close()
+
+
+if __name__ == "__main__":
+  with BaseTable("video_record_predictions", batch_size=10_000, save_path="/home/ml/video_predictions/data") as T:
+
+    T.update(last_id=1_000_000)
+    T.load()
+    if not T.df.empty:
+      debug(T.df.columns)
+      debug(T.df[["key", "people", "is_valid", "updated_at", "data"]].head())
+
+    while(True):
+      df = T.update()
+      if df.empty:
+        info("Finished", T.df.shape)
+        break
+      info("read", df.shape, df[["key", "people"]].head())

sciveo-0.1.73/sciveo/monitoring/power/ems300.py

@@ -0,0 +1,108 @@
+#
+# Stanislav Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact s.georgiev@softel.bg.
+#
+# 2025
+#
+
+import time
+from pymodbus.client import ModbusTcpClient
+from sciveo.tools.logger import *
+from sciveo.tools.daemon import DaemonBase
+from sciveo.monitoring.power.tools import *
+
+
+class PowerEMS300(DaemonBase):
+  def __init__(self, host, port=502, device_id=247, delay=0.01, period=30):
+    super().__init__(period=period)
+    self.host = host
+    self.port = port
+    self.device_id = device_id
+    self.delay = delay
+    self.client = None
+
+    self.client = ModbusTcpClient(host=self.host, port=self.port)
+    self.connected = False
+
+  def connect(self):
+    if not (self.connected or self.client.connect()):
+      error("Connect FAIL", (self.host, self.port))
+      self.connected = False
+    else:
+      self.connected = True
+    return self.connected
+
+  def loop(self):
+    self.connect()
+    if not self.connected:
+      warning("Not connected", (self.host, self.port))
+      return
+
+    try:
+      # iterate known addresses in map (sorted)
+      for addr in sorted(REG_MAP.keys()):
+        dtype, factor, name = REG_MAP[addr]
+        count = count_for_type(dtype)
+        protocol_addr = addr - 1  # doc addresses start at 1; pymodbus expects zero-based
+        try:
+          rr = self.client.read_input_registers(address=protocol_addr, count=count, device_id=self.device_id)
+          if rr is None:
+            warning(f"No response for {addr} ({name})")
+            continue
+          if hasattr(rr, "isError") and rr.isError():
+            warning(f"Modbus exception at {addr} ({name}): {rr}")
+            continue
+          regs = getattr(rr, "registers", None)
+          if not regs or len(regs) < count:
+            warning(f"Incomplete registers at {addr} ({name}): {regs}")
+            continue
+
+          val_raw = decode_registers(regs, dtype)
+          if val_raw is None:
+            warning(f"Unable to decode {addr} ({name}) regs={regs}")
+            continue
+          # apply factor
+          value = val_raw * factor
+          # format integer-looking floats without trailing .0
+          if isinstance(value, float) and value.is_integer():
+            value = int(value)
+          info(f"{addr} {name}: raw={regs} -> {value} (factor={factor}, type={dtype})")
+        except Exception as e:
+          error(f"Error reading {addr} ({name}): {e}")
+        time.sleep(self.delay)
+
+      # Optionally scan remaining addresses 8000..8200 for any non-zero single-register values
+      info("\nQuick scan for single-register non-zero values (8000..8200):")
+      for addr in range(8000, 8201):
+        if addr in REG_MAP:
+          continue
+        try:
+          rr = self.client.read_input_registers(address=addr - 1, count=1, device_id=self.device_id)
+          if rr is None:
+            continue
+          if hasattr(rr, "isError") and rr.isError():
+            continue
+          regs = getattr(rr, "registers", None)
+          if regs and regs[0] != 0:
+            info(f"{addr}: {regs[0]} (0x{regs[0]:04X})")
+        except Exception:
+          pass
+        time.sleep(0.005)
+    except Exception as e:
+      exception(e)
+
+  def close(self):
+    self.client.close()
+
+
+if __name__ == "__main__":
+
+  mon = PowerEMS300(host="localhost", port=1502, period=10)
+  mon.start()
+
+  while(True):
+    time.sleep(30)

sciveo-0.1.73/sciveo/monitoring/power/sim.py

@@ -0,0 +1,71 @@
+#
+# Stanislav Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact s.georgiev@softel.bg.
+#
+# 2025
+#
+
+import time
+import threading
+import time
+from pymodbus.server import StartTcpServer
+from pymodbus.datastore import ModbusServerContext, ModbusSparseDataBlock
+from threading import Thread
+from sciveo.tools.logger import *
+from sciveo.tools.daemon import DaemonBase
+
+
+# Example read-only registers
+# INITIAL_REGISTERS = {
+#     8044: 12500,   # Plant Active Power
+#     8060: 8200,    # PV Active Power
+#     8038: -1500,   # Storage Power
+#     8048: 600,     # Reactive Power
+#     9082: 3000,    # Dynamic Allowable Charge Power
+#     9084: 3500,    # Dynamic Allowable Discharge Power
+# }
+INITIAL_REGISTERS = {
+  8060: 8200,    # PV Active Power
+}
+
+class SimulatorEMS300(DaemonBase):
+  def __init__(self, host="127.0.0.1", port=5020, period=1):
+    super().__init__(num_threads=1, period=period)
+    self.host = host
+    self.port = port
+
+    # Only input registers
+    self.ir_block = ModbusSparseDataBlock(INITIAL_REGISTERS.copy())
+
+    self.context = ModbusServerContext(
+      { 247: {  # unit_id -> datastore dict
+          'di': ModbusSparseDataBlock({}),   # discrete inputs
+          'co': ModbusSparseDataBlock({}),   # coils
+          'hr': ModbusSparseDataBlock({}),   # holding registers
+          'ir': self.ir_block                # input registers
+        }
+      },
+      single=False  # multiple units not used here, keep False
+    )
+
+  def loop(self):
+    # Slowly vary PV Active Power
+    pv = self.ir_block.getValues(8060, count=1)[0]
+    pv = (pv + 50) % 15000
+    self.ir_block.setValues(8060, [pv])
+    info(f"Simulated PV Active Power: {pv}")
+
+  def start_server(self):
+    StartTcpServer(self.context, address=(self.host, self.port))
+
+
+if __name__ == "__main__":
+    sim = SimulatorEMS300(host="127.0.0.1", port=1502)
+    sim.start()
+    sim.start_server()
+    while True:
+      time.sleep(10)