micrOSDevToolKit 2.1.5__py3-none-any.whl → 2.26.1__py3-none-any.whl

micrOS/source/Hooks.py

@@ -17,12 +17,21 @@ Designed by Marcell Ban aka BxNxM
 #################################################################
 #                           IMPORTS                             #
 #################################################################
-from Config import cfgget
+from Config import cfgget, cfgput
 from microIO import detect_platform
-from Debug import console_write
+from Debug import console_write, syslog
 from Tasks import exec_lm_pipe
+from Auth import resolve_secret
 from micropython import mem_info
 from machine import freq
+try:
+    from machine import reset_cause, PWRON_RESET, HARD_RESET, WDT_RESET, DEEPSLEEP_RESET, SOFT_RESET
+except ImportError:
+    reset_cause = None
+try:
+    from gc import mem_free
+except ImportError:
+    from simgc import mem_free
 
 #################################################################
 #                          FUNCTIONS                            #
@@ -34,29 +43,58 @@ def bootup():
     Executes when system boots up.
     """
     # Execute LMs from boothook config parameter
-    console_write("[BOOT] EXECUTION ...")
+    console_write("[BOOT] EXECUTION...")
     bootasks = cfgget('boothook')
     if bootasks is not None and bootasks.lower() != 'n/a':
         console_write(f"|-[BOOT] TASKS: {bootasks}")
-        if exec_lm_pipe(bootasks):
+        if exec_lm_pipe(resolve_secret(bootasks)):
             console_write("|-[BOOT] DONE")
         else:
             console_write("|-[BOOT] ERROR")
 
-    # Set boostmd (boost mode)
+    # Load and Save boot cause
+    boot_cause()
+    # Autotune queue size
+    _tune_queue_size()
+    # Configure CPU performance
+    _tune_performance()
+
+
+def _tune_queue_size():
+    """
+    Tune queue size based on available ram
+    between 5-50
+    """
+    min_queue, max_queue, task_req_kb = 5, 20, 20       # 400kb max for tasks management
+    est_queue = int(mem_free()/1000/task_req_kb)
+    est_queue = max(est_queue, min_queue)
+    est_queue = min(est_queue, max_queue)
+    current_queue = cfgget('aioqueue')
+    if est_queue > current_queue:
+        cfgput('aioqueue', est_queue)
+
+
+def _tune_performance():
+    # {(platforms, ...): (min_clock, max_clock), ...}
+    cpu_clocks = {
+        ('esp32c3', 'esp32c6'): (80_000_000, 160_000_000),
+        ('esp32',): (160_000_000, 240_000_000),  # default
+    }
     platform = detect_platform()
-    if cfgget('boostmd') is True:
-        console_write(f"[BOOT HOOKS] Set up CPU high Hz - boostmd: {cfgget('boostmd')}")
-        if platform == 'esp32c3':
-            freq(160_000_000)   # 160 Mhz (max)
-        elif 'esp32' in platform:
-            freq(240_000_000)   # 240 Mhz (max)
+    cpu_min_max = cpu_clocks[('esp32',)]
+    for platforms, clocks in cpu_clocks.items():
+        if platform in platforms:
+            cpu_min_max = clocks
+            break
+    # Set boosted (boost mode)
+    if cfgget('boostmd'):
+        max_hz = cpu_min_max[1]
+        console_write(f"[BOOT HOOKS] CPU boost mode ON: {max_hz} Hz")
+        freq(max_hz)
     else:
-        console_write(f"[BOOT HOOKS] Set up CPU low Hz - boostmd: {cfgget('boostmd')}")
-        if platform == 'esp32c3':
-            freq(80_000_000)   # 80 Mhz / Half the max CPU clock
-        elif 'esp32' in platform:
-            freq(160_000_000)   # 160 Mhz / Half the max CPU clock
+        min_hz = cpu_min_max[0]
+        console_write(f"[BOOT HOOKS] CPU boost mode OFF: {min_hz} Hz")
+        freq(min_hz)
 
 
 def profiling_info(label=""):
@@ -67,3 +105,37 @@ def profiling_info(label=""):
         console_write(f"{'~'*5} [PROFILING INFO] - {label} {'~'*5}")
         mem_info()
         console_write("~"*30)
+
+
+def boot_cause():
+    reason = 0, "-Unknown"
+    if callable(reset_cause):
+        reason = 0, "Unknown"
+        reset_reason = reset_cause()
+        if reset_reason == PWRON_RESET:
+            reason = 1, "PowerOn"
+        elif reset_reason == HARD_RESET:
+            reason = 2, "HardReset"
+        elif reset_reason == WDT_RESET:
+            reason = 3, "WDTWakeUp"
+        elif reset_reason == DEEPSLEEP_RESET:
+            reason = 4, "DSWakeUp"
+        elif reset_reason == SOFT_RESET:
+            reason = 5, "SoftReset"
+    syslog(f"[BOOT] info: {reason[1]}")
+    return reason
+
+
+def enableESPNow() -> str:
+    """
+    Enable ESP-NOW communication
+    """
+    if cfgget('espnow'):
+        try:
+            from Espnow import ESPNowSS
+            verdict = ESPNowSS().start_server()
+            console_write(f"[TASK] Start ESPNow-InterCon server: {verdict}")
+        except Exception as e:
+            syslog(f"[ERR] Start ESPNow-InterCon server: {e}")
+            return str(e)
+    return "ESPNow disabled"

micrOS/source/InterConnect.py

@@ -1,14 +1,35 @@
+"""
+Module is responsible for device-device communication
+dedicated to micrOS framework.
+Built-in-function:
+- Socket InterConnect interface
+- ESPNow redirection
+
+Designed by 
+    Marcell Ban aka BxNxM
+    Kristof Kasza aka KKristof452
+"""
+
 from socket import getaddrinfo, SOCK_STREAM
-from re import compile
-import uasyncio as asyncio
-from Debug import errlog_add
+from re import compile as re_compile
+from json import loads
+from binascii import hexlify
+from uasyncio import open_connection
+
+from Debug import syslog
 from Config import cfgget
-from Server import SocketServer
+from Server import Server
 from Tasks import NativeTask
 
+if cfgget('espnow'):
+    from Espnow import ESPNowSS
+else:
+    ESPNowSS = None
+
 
 class InterCon:
-    CONN_MAP = {}
+    CONN_MAP: dict[str, str] = {}   # hostname: IP address pairs
+    NO_ESPNOW: list[str] = []       # disabled ESPNow hostname list (cache for fallback speed-up)
     PORT = cfgget('socport')
 
     def __init__(self):
@@ -19,10 +40,10 @@ class InterCon:
 
     @staticmethod
     def validate_ipv4(str_in):
-        pattern = compile(r'^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])$')
+        pattern = re_compile(r'^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])$')
         return bool(pattern.match(str_in))
 
-    async def send_cmd(self, host, cmd):
+    async def send_cmd(self, host:str, cmd:list):
         """
         Async Main method to implement device-device communication with
         - dhcp host resolve and IP caching
@@ -34,14 +55,19 @@ class InterCon:
         # Check if host is a hostname (example.local) and resolve its IP address
         if not InterCon.validate_ipv4(host):
             hostname = host
-            # Retrieve IP address by hostname dynamically
+            # Lookup hostname without .domain (sub-hostname matching)
+            if '.' not in hostname:
+                _hosts = list([d for d in InterCon.CONN_MAP if hostname in d])
+                if len(_hosts) > 0:
+                    hostname = _hosts[0]
+            # Retrieve IP address by hostname.domain dynamically
             if InterCon.CONN_MAP.get(hostname, None) is None:
                 try:
                     addr_info = getaddrinfo(host, InterCon.PORT, 0, SOCK_STREAM)
                     host = addr_info[-1][4][0]
                 except OSError as e:
-                    SocketServer.reply_all(f"[intercon] NoHost: {e}")
-                    errlog_add(f"[intercon] send_cmd {host} oserr: {e}")
+                    Server.reply_all(f"[intercon] NoHost: {e}")
+                    syslog(f"[intercon] send_cmd {host} oserr: {e}")
                     return ''
             else:
                 # Restore IP from cache by hostname
@@ -50,12 +76,12 @@ class InterCon:
         if InterCon.validate_ipv4(host):
             try:
                 # Create socket object
-                self.reader, self.writer = await asyncio.open_connection(host, InterCon.PORT)
+                self.reader, self.writer = await open_connection(host, InterCon.PORT)
                 # Send command over TCP/IP
                 output = await self.__run_command(cmd, hostname)
             except OSError as e:
-                SocketServer.reply_all(f"[intercon] NoHost: {e}")
-                errlog_add(f"[intercon] send_cmd {host} oserr: {e}")
+                Server.reply_all(f"[intercon] NoHost: {e}")
+                syslog(f"[intercon] send_cmd {host} oserr: {e}")
                 output = None
             finally:
                 if self.writer:
@@ -68,26 +94,23 @@ class InterCon:
                 InterCon.CONN_MAP[hostname] = None if output is None else host
             # None: ServerBusy(or \0) or Prompt mismatch (auto delete cached IP), STR: valid comm. output
             return output
-        else:
-            errlog_add(f"[ERR][intercon] Invalid host: {host}")
+        syslog(f"[ERR][intercon] Invalid host: {host}")
         return ''
 
-    async def __run_command(self, cmd, hostname):
+    async def __run_command(self, cmd:list, hostname:str):
         """
         Implements receive data on open connection, command query and result collection
         :param cmd: command string to server socket shell
         :param hostname: hostname for prompt checking
         Return None here will trigger retry mechanism... + deletes cached IP
         """
-        cmd = str.encode(cmd)
         data, prompt = await self.__receive_data()
         if "Connection is busy. Bye!" in prompt:
             return None
         # Compare prompt |node01 $| with hostname 'node01.local'
         if hostname is None or prompt is None or str(prompt).replace('$', '').strip() == str(hostname).split('.')[0]:
             # Run command on validated device
-            # TODO: Handle multiple cmd as input, separated by ; (????)
-            self.writer.write(cmd)
+            self.writer.write(str.encode(' '.join(cmd)))
             await self.writer.drain()
             data, _ = await self.__receive_data(prompt=prompt)
             if data == '\0':
@@ -95,7 +118,7 @@ class InterCon:
             # Successful data receive, return data
             return data
         # Skip command run: prompt and host not the same!
-        SocketServer.reply_all(f"[intercon] prompt mismatch, hostname: {hostname} prompt: {prompt}")
+        Server.reply_all(f"[intercon] prompt mismatch, hostname: {hostname} prompt: {prompt}")
         return None
 
     async def __auth_handshake(self, prompt):
@@ -104,13 +127,13 @@ class InterCon:
             await self.writer.drain()
             data, prompt = await self.__receive_data(prompt=prompt)
         except Exception as e:
-            errlog_add(f'[intercon][ERR] Auth: {e}')
+            syslog(f'[intercon][ERR] Auth: {e}')
             data = 'AuthFailed'
         if 'AuthOk' in data:
             return True             # AuthOk
         return False                # AuthFailed
 
-    async def __receive_data(self, prompt=None):
+    async def __receive_data(self, prompt=None) -> tuple[str, str]:
         """
         Implements data receive loop until prompt / [configure] / Bye!
         :param prompt: socket shell prompt
@@ -140,8 +163,34 @@ class InterCon:
         data = data.replace(prompt, '').replace('\n', ' ')
         return data, prompt
 
+    async def auto_espnow_handshake(self, host:str) -> dict:
+        """
+        [1] Check espnow.server running on host
+        [2] Get MAC address for host (from system info)
+        [3] Execute ESPNowSS.handshake
+        """
+        response = await self.send_cmd(host, ["task", "list", ">json"])
+        if not response:
+            return {None: f"[ERR] ESPNow auto handshake: task list >>{host}: {response}"}
+
+        active_tasks = loads(response).get("active")
+        if active_tasks and "espnow.server" in active_tasks:
+            response = await self.send_cmd(host, ["system", "info", ">json"])
+            if not response:
+                return {None: "[ERR] ESPNow auto handshake: system info"}
+            try:
+                host_mac = loads(response).get("mac")
+            except Exception as ex:
+                return {None: f"[ERR] ESPNow auto handshake: {ex}"}
+
+            return ESPNowSS().handshake(host_mac)
+
+        if not InterCon.validate_ipv4(host):
+            InterCon.NO_ESPNOW.append(str(host).split(".")[0])   # host.local -> host
+        return {None: f"ESPNow auto handshake: espnow disabled on host {host}"}
+
 
-async def _send_cmd(host, cmd, com_obj):
+async def _socket_send_cmd(host:str, cmd:list, com_obj:InterCon) -> None:
     """
     Async send command wrapper for further async task integration and sync send_cmd usage (main)
     :param host: hostname / IP address
@@ -149,42 +198,77 @@ async def _send_cmd(host, cmd, com_obj):
     :param com_obj: InterCon object to utilize send_cmd method and task status updates
     """
     # Send command
+    for _ in range(0, 2):                           # Retry mechanism
+        out = await com_obj.send_cmd(host, cmd)     # Send CMD
+        if out is not None:                         # Retry mechanism
+            break
+        await com_obj.task.feed(sleep_ms=100)       # Retry mechanism
+    com_obj.task.out = '' if out is None else out
+
+
+async def _send_cmd(host:str, cmd:list|str, com_obj:InterCon):
+    """
+    Top level InterConnect callback function
+        [1] node01.domain -> ESPNow, Socket
+                (domain: .local, .net, etc.)
+        [2] node01 -> ESPNow, Socket fallback in case found in InterConnect cache
+        [3] IP address -> Socket
+    """
     with com_obj.task:
-        for _ in range(0, 4):                           # Retry mechanism
-            out = await com_obj.send_cmd(host, cmd)     # Send CMD
-            if out is not None:                         # Retry mechanism
-                break
-            await asyncio.sleep_ms(100)                 # Retry mechanism
-        com_obj.task.out = '' if out is None else out
-    return com_obj.task.out
+        if ESPNowSS:
+            # [1] ESPNow Active
+            name = str(host).split(".")[0]   # host.local -> host
+            if name not in InterCon.NO_ESPNOW and name in list(ESPNowSS().devices.values()):
+                com_obj.task.out = "Redirected to ESPNow"
+                if isinstance(cmd, list):
+                    cmd = ' '.join(cmd)
+                # Send command and retrieve result
+                sender = ESPNowSS().send(peer=name, msg=cmd)
+                sender_task = NativeTask.TASKS.get(list(sender.keys())[0])
+                result = await sender_task.await_result(timeout=10)
+                if result != "Timeout has beed exceeded":
+                    # Successful command execution
+                    com_obj.task.out = result                       # Output mirroring: Child -> Parent
+                    sender_task.out = "Redirected to ParentTask"    # Remove redundant data in embedded mode
+                    return
 
+        # Handle legacy string input
+        if isinstance(cmd, str):
+            cmd = cmd.split()
+        # [1][2] Socket send (default and fallback)
+        await _socket_send_cmd(host, cmd, com_obj)
 
-def send_cmd(host, cmd):
+        if ESPNowSS and name not in InterCon.NO_ESPNOW:
+            # [3] Automatic ESPNow handshake
+            verdict = await com_obj.auto_espnow_handshake(host)
+            if list(verdict.keys())[0] is None:
+                syslog(str(list(verdict.values())[0]))
+
+
+def send_cmd(host:str, cmd:list|str) -> dict:
     """
-    Sync wrapper of async _send_cmd (InterCon.send_cmd consumer with retry)
-    :param host: hostname / IP address
-    :param cmd: command string to server socket shell
+    Top level InterConnect send task creation
+        Handles ESPNow and socket communication
     """
     def _tagify():
         nonlocal host, cmd
-        _mod = cmd.split(' ')[0].strip()
+        _mod = cmd[0]
         if InterCon.validate_ipv4(host):
             return f"{'.'.join(host.split('.')[-2:])}.{_mod}"
-        return f"{host.replace('.local', '')}.{_mod}"
+        target = ".".join(host.split(".")[0:-1]) if "." in host else host
+        return f"{target}.{_mod}"
 
     com_obj = InterCon()
-    tag = f"con.{_tagify()}"
-    started = com_obj.task.create(callback=_send_cmd(host, cmd, com_obj), tag=tag)
-    if started:
-        result = {"verdict": f"Task started: task show {tag}", "tag": tag}
-    else:
-        result = {"verdict": "Task is Busy", "tag": tag}
-    return result
+    task_id = f"con.{_tagify()}"            # CHECK TASK ID CONFLICT
+    return com_obj.task.create(callback=_send_cmd(host, cmd, com_obj), tag=task_id)
 
 
-def host_cache():
+def host_cache() -> dict:
     """
     Dump InterCon connection cache
     """
-    return InterCon.CONN_MAP
-
+    if ESPNowSS is None:
+        return InterCon.CONN_MAP
+    all_devs = dict({name:hexlify(mac, ':').decode() for mac, name in ESPNowSS().devices.items()})
+    all_devs.update(InterCon.CONN_MAP)
+    return all_devs

micrOS/source/Interrupts.py

@@ -19,9 +19,9 @@ Reference: https://docs.micropython.org/en/latest/library/machine.Pin.html
 from machine import Pin
 from utime import ticks_ms, ticks_diff
 from Config import cfgget
-from Debug import console_write, errlog_add
+from Debug import console_write, syslog
 from Tasks import exec_lm_pipe_schedule
-from microIO import physical_pin
+from microIO import resolve_pin
 if cfgget('cron'):
     # Only import when enabled - memory usage optimization
     from Scheduler import scheduler
@@ -71,8 +71,9 @@ def enableInterrupt():
 
 def enableCron():
     """
-    Set time stump based scheduler aka cron in timer1
+    Set time stump based scheduler aka cron on Timer1
     Input: cron(bool), crontasks(str)
+    This is for low frequency sampling, like 12 or 6 / minute (due to low power mode compatibility)
     """
     timer_period = 5000         # Timer period ms: 12 check/min
     console_write(f"[IRQ] CRON IRQ SETUP: {cfgget('cron')} SEQ: {timer_period}")
@@ -80,11 +81,10 @@ def enableCron():
     if cfgget("cron") and cfgget('crontasks').lower() != 'n/a':
         from machine import Timer
         # INIT TIMER 1 IRQ with callback function wrapper
-        lm_byte = bytearray(cfgget('crontasks'), 'utf-8')           # store as bytearray (cache optimization)
-        sample = int(timer_period/1000)
+        sampling = int(timer_period/1000)
         timer = Timer(1)
         timer.init(period=timer_period, mode=Timer.PERIODIC,
-                   callback=lambda timer: scheduler(lm_byte, sample))
+                   callback=lambda timer: scheduler(sampling))
 
 
 #################################################################
@@ -155,9 +155,9 @@ def initEventIRQs():
         Resolve pin by name
         """
         try:
-            return physical_pin(_p)
+            return resolve_pin(_p)
         except Exception as e:
-            errlog_add(f'[ERR][!] EVENT {_p} IO error: {e}')
+            syslog(f'[ERR][!] EVENT {_p} IO error: {e}')
         return None
 
     # Load External IRQ (1-4) execution data set from node config

micrOS/source/Logger.py

@@ -0,0 +1,131 @@
+"""
+Module is responsible for System and User logging
+- built-in log rotation
+
+Designed by Marcell Ban aka BxNxM
+"""
+from time import localtime
+from re import match
+from uos import remove
+from Files import OSPath, path_join, ilist_fs, is_dir
+
+#############################################
+#        LOGGING WITH DATA ROTATION         #
+#############################################
+
+def _init_logger():
+    """ Init /logs folder """
+    if not is_dir(OSPath.LOGS):
+        OSPath.LOGS = OSPath._ROOT
+    return OSPath.LOGS
+
+
+def _dir_select(f_name:str) -> str:
+    """
+    Select log dir based on file extension
+    :param f_name: filename with extension to detect target dir
+    """
+    if f_name.endswith(".log"):
+        return OSPath.LOGS
+    return OSPath.DATA
+
+
+def logger(data, f_name:str, limit:int):
+    """
+    Generic logger function with line rotation and time
+    :param data: data to log
+    :param f_name: file name to use
+    :param limit: line limit for log rotation
+    return write verdict - true / false
+    INFO: hardcoded max data number = 30
+    """
+    def _logger(f_mode='r+'):
+        nonlocal data, f_path, limit
+        limit = min(limit, 30)  # Hardcoded max data line = 30
+        # [1] GET TIME STUMP
+        ts_buff = [str(k) for k in localtime()]
+        ts = ".".join(ts_buff[0:3]) + "-" + ":".join(ts_buff[3:6])
+        # [2] OPEN FILE - WRITE DATA WITH TS
+        with open(f_path, f_mode) as f:
+            _data = f"{ts} {data}\n"
+            # read file lines and filter by time stump chunks (hack for replace truncate)
+            lines = [_l for _l in f.readlines() if '-' in _l and '.' in _l]
+            # get file params
+            lines_len = len(lines)
+            lines.append(_data)
+            f.seek(0)
+            # line data rotate
+            if lines_len >= limit:
+                lines = lines[-limit:]
+            # write file
+            f.write(''.join(lines))
+
+    f_path = path_join(_dir_select(f_name), f_name)
+    # Run logger
+    try:
+        # There is file - append 'r+'
+        _logger()
+    except:
+        try:
+            # There is no file - create 'a+'
+            _logger('a+')
+        except:
+            return False
+    return True
+
+
+def log_get(f_name:str, msgobj=None):
+    """
+    Generic file getter for .log files
+    - log content critical [ERR] counter
+    """
+    f_path = path_join(_dir_select(f_name), f_name)
+    err_cnt = 0
+    try:
+        if msgobj is not None:
+            msgobj(f_path)
+        with open(f_path, 'r') as f:
+            eline = f.readline().strip()
+            while eline:
+                # GET error from log line (tag: [ERR])
+                err_cnt += 1 if "[ERR]" in eline else 0
+                # GIVE BACK .log file contents
+                if msgobj is not None:
+                    msgobj(f"\t{eline}")
+                eline = f.readline().strip()
+    except:
+        pass
+    return err_cnt
+
+
+def syslog(data=None, msgobj=None):
+    """
+    System log setter/getter
+    :param data: None - read logs, str - write logs
+    :param msgobj: function to stream .log files
+    """
+    if data is None:
+        # READ LOGS
+        err_cnt = sum([log_get(f, msgobj) for f in ilist_fs(OSPath.LOGS, type_filter='f') if f.endswith(".sys.log")])
+        return err_cnt
+    # WRITE LOGS - [target].sys.log automatic log level detection
+    _match = match(r"^\[([^\[\]]+)\]", data)
+    log_lvl = _match.group(1).lower() if _match else 'user'
+    f_name = f"{log_lvl}.sys.log" if log_lvl in ("err", "warn", "boot", "info") else 'user.sys.log'
+    return logger(data, f_name, limit=4)
+
+
+def log_clean(msgobj=None):
+    """
+    Clean logs folder
+    """
+    logs_dir = OSPath.LOGS
+    to_del = [file for file in ilist_fs(logs_dir, type_filter='f') if file.endswith('.log')]
+    for _del in to_del:
+        _del = path_join(logs_dir, _del)
+        if msgobj is not None:
+            msgobj(f" Delete: {_del}")
+        remove(_del)
+
+# Init log folder at module load
+_init_logger()