micrOSDevToolKit 2.9.1__py3-none-any.whl → 2.26.1__py3-none-any.whl

micrOS/source/Files.py

@@ -0,0 +1,207 @@
+"""
+Module is responsible high level micropython file system opeartions
+[IMPORTANT] This module must never use any micrOS specific functions or classes.
+"""
+
+from uos import ilistdir, remove, stat, getcwd, mkdir, rmdir
+from sys import path as upath
+
+################################   Helper functions   #####################################
+
+def _filter(path:str='/', ext:tuple=None, prefix:tuple=None, hide_core:bool=True) -> bool:
+    """
+    Filter files in the micrOS filesystem.
+
+    :param path: file path to check
+    :param ext: tuple of extensions to filter by, default: None (all)
+    :param prefix: tuple of prefixes to match (e.g. ('LM', 'IO')), default: None
+    :param hide_core: if True, hides core .py/.mpy files in the root (current) directory
+    :return: bool, whether the file passes the filter
+    """
+    parent = "/".join(path.split("/")[:-1]) or "/"
+    fname = path.split("/")[-1]
+    _ext = fname.split(".")[-1]
+
+    # --- Hide core logic ---
+    # Core = any .py/.mpy in the current (root) working directory
+    if hide_core and _ext in ("mpy", "py") and parent in ('/', ""):
+        return False
+
+    # --- General matching rules ---
+    if ext is None and prefix is None:
+        return True
+    if isinstance(prefix, tuple) and fname.split("_")[0] in prefix:
+        return True
+    if isinstance(ext, tuple) and _ext in ext:
+        return True
+    return False
+
+
+def is_protected(path:str='/') -> bool:
+    """
+    Check is file/dir protected
+        - protected root folders and files: /*
+        - protected system files by name
+    """
+    protected_files = ("node_config.json", "LM_system.mpy", "LM_pacman.mpy", "LM_cluster.mpy")
+    # Detect parent directory
+    parent = "/".join(path.split("/")[:-1]) or "/"
+    # Get file/folder name
+    fname = path.split("/")[-1]
+    # Disallow: root folders and protected files
+    return parent in ("/", "") or fname in protected_files
+
+
+def _type_mask_to_str(item_type:int=None) -> str:
+    # Map the raw bit-mask to a single character
+    if item_type & 0x4000:              # Dir bit-mask
+        item_type = 'd'
+    elif item_type & 0x8000:            # File bit-mask
+        item_type = 'f'
+    else:
+        item_type = 'o'
+    return item_type
+
+###########################   Public functions   #############################
+def is_dir(path):
+    try:
+        return stat(path)[0] & 0x4000
+    except OSError:
+        return False
+
+
+def is_file(path):
+    try:
+        return stat(path)[0] & 0x8000
+    except OSError:
+        return False
+
+
+def ilist_fs(path:str="/", type_filter:str='*', select:str='*', core:bool=False):
+    """
+    Linux like ls command - list app resources and app folders
+    :param path: path to list, default: /
+    :param type_filter: content type, default all (*), f-file, d-dir can be selected
+    :param select: select specific application resource type by prefix: LM or IO
+    :param core: list core files resources as well, default: False
+    return iterator:
+        when content is all (*) output: [(item_type, item), ...]
+        OR
+        content type was selected (not *) output: [item, ...]
+    """
+    path = path if path.endswith('/') else f"{path}/"
+    # Info: uos.ilistdir: (name, type, inode[, size])
+    for item, item_type, *_ in ilistdir(path):
+        item_type = _type_mask_to_str(item_type)
+        if type_filter in ("*", item_type):
+            # Mods only
+            _select = None if select == "*" else (select,)
+            if item_type == 'f' and not _filter(path_join(path, item), prefix=_select, hide_core=not core):
+                continue
+            if select != '*' and item_type == 'd':
+                continue
+            # Create result
+            if type_filter == "*":
+                yield item_type, item
+            else:
+                yield item
+
+
+def list_fs(path:str="/", type_filter:str='*', select:str='*', core:bool=False) -> list[str,] | list[tuple[str, str],]:
+    """
+    Wrapper of ilist_fs
+    Return list
+    """
+    return list(ilist_fs(path, type_filter, select, core))
+
+
+def remove_file(path, force=False):
+    """
+    Linux like rm command - delete app resources and folders
+    :param path: file to delete
+    :param force: pypass file protection check - sudo mode
+    """
+    # protect some resources
+    if not force and is_protected(path):
+        return f'Protected resource, skip deletion: {path}'
+    if is_file(path):
+        remove(path)
+        return f"{path} deleted"
+    return f"Cannot delete (not a file): {path}"
+
+
+def remove_dir(path, force=False):
+    """
+    Recursively delete a folder and all its contents.
+    :param path: folder to delete
+    :param force: pypass dir protection check - sudo mode
+    """
+    # protect some resources
+    if not force and is_protected(path):
+        return f'Protected resource, skip deletion: {path}'
+    for entry in ilistdir(path):
+        content_path = path_join(path, entry[0])
+        if is_dir(content_path):            # directory flag
+            remove_dir(content_path)
+        else:
+            remove(content_path)
+    rmdir(path)
+    return f"{path} deleted"
+
+
+def path_join(*parts):
+    path = "/".join(part.strip("/") for part in parts if part)
+    if parts and parts[0].startswith("/"):
+        path = path if path.startswith("/") else "/" + path
+    return path
+
+
+def abs_path(path):
+    parts = path.split("/")
+    stack = []
+    for p in parts:
+        if not p or p == ".":
+            continue
+        if p == "..":
+            if stack:
+                stack.pop()   # prevent escaping root
+        else:
+            stack.append(p)
+    return "/" + "/".join(stack)
+
+
+# micrOS system file structure
+class OSPath:
+    _ROOT = getcwd()
+    LOGS = path_join(_ROOT, '/logs')        # Logs (.log)
+    DATA = path_join(_ROOT,'/data')         # Application data (.dat, .cache, etc.)
+    WEB = path_join(_ROOT,'/web')           # Web resources (.html, .css, .js, .json, etc.)
+    MODULES = path_join(_ROOT, '/modules')  # Application modules (.mpy, .py)
+    CONFIG = path_join(_ROOT, '/config')    # System configuration files (node_config.json, etc.)
+    LIB = path_join(_ROOT, '/lib')          # Official and Custom package installation target path
+
+
+def init_micros_dirs():
+    """
+    Init micrOS root file system directories
+    """
+    # ENABLE MODULES ACCESS
+    if OSPath.MODULES not in upath:
+        upath.insert(0, OSPath.MODULES)
+    # ENABLE LIB ACCESS
+    if OSPath.LIB not in upath:
+        upath.insert(0, OSPath.LIB)
+
+    root_dirs = [
+        getattr(OSPath, key)
+        for key in dir(OSPath)
+        if not key.startswith("_") and isinstance(getattr(OSPath, key), str)
+    ]
+    print(f"[BOOT] rootFS validation: {root_dirs}")
+    for dir_path in root_dirs:
+        if not is_dir(dir_path):
+            try:
+                mkdir(dir_path)
+                print(f"[BOOT] init dir: {dir_path}")
+            except Exception as e:
+                print(f"[ERR][BOOT] cannot init dir {dir_path}: {e}")

micrOS/source/Hooks.py

@@ -19,8 +19,9 @@ Designed by Marcell Ban aka BxNxM
 #################################################################
 from Config import cfgget, cfgput
 from microIO import detect_platform
-from Debug import console_write, errlog_add
+from Debug import console_write, syslog
 from Tasks import exec_lm_pipe
+from Auth import resolve_secret
 from micropython import mem_info
 from machine import freq
 try:
@@ -43,34 +44,23 @@ def bootup():
     """
     # Execute LMs from boothook config parameter
     console_write("[BOOT] EXECUTION...")
-    boot_cause()
     bootasks = cfgget('boothook')
     if bootasks is not None and bootasks.lower() != 'n/a':
         console_write(f"|-[BOOT] TASKS: {bootasks}")
-        if exec_lm_pipe(bootasks):
+        if exec_lm_pipe(resolve_secret(bootasks)):
             console_write("|-[BOOT] DONE")
         else:
             console_write("|-[BOOT] ERROR")
 
-    # Set boostmd (boost mode)
-    platform = detect_platform()
-    if cfgget('boostmd') is True:
-        console_write(f"[BOOT HOOKS] Set up CPU high Hz - boostmd: {cfgget('boostmd')}")
-        if platform == 'esp32c3':
-            freq(160_000_000)   # 160 Mhz (max)
-        elif 'esp32' in platform:
-            freq(240_000_000)   # 240 Mhz (max)
-    else:
-        console_write(f"[BOOT HOOKS] Set up CPU low Hz - boostmd: {cfgget('boostmd')}")
-        if platform == 'esp32c3':
-            freq(80_000_000)   # 80 Mhz / Half the max CPU clock
-        elif 'esp32' in platform:
-            freq(160_000_000)   # 160 Mhz / Half the max CPU clock
+    # Load and Save boot cause
+    boot_cause()
     # Autotune queue size
-    tune_queue_size()
+    _tune_queue_size()
+    # Configure CPU performance
+    _tune_performance()
 
 
-def tune_queue_size():
+def _tune_queue_size():
     """
     Tune queue size based on available ram
     between 5-50
@@ -84,6 +74,29 @@ def tune_queue_size():
         cfgput('aioqueue', est_queue)
 
 
+def _tune_performance():
+    # {(platforms, ...): (min_clock, max_clock), ...}
+    cpu_clocks = {
+        ('esp32c3', 'esp32c6'): (80_000_000, 160_000_000),
+        ('esp32',): (160_000_000, 240_000_000),  # default
+    }
+    platform = detect_platform()
+    cpu_min_max = cpu_clocks[('esp32',)]
+    for platforms, clocks in cpu_clocks.items():
+        if platform in platforms:
+            cpu_min_max = clocks
+            break
+    # Set boosted (boost mode)
+    if cfgget('boostmd'):
+        max_hz = cpu_min_max[1]
+        console_write(f"[BOOT HOOKS] CPU boost mode ON: {max_hz} Hz")
+        freq(max_hz)
+    else:
+        min_hz = cpu_min_max[0]
+        console_write(f"[BOOT HOOKS] CPU boost mode OFF: {min_hz} Hz")
+        freq(min_hz)
+
+
 def profiling_info(label=""):
     """
     Runtime memory measurements
@@ -109,5 +122,20 @@ def boot_cause():
             reason = 4, "DSWakeUp"
         elif reset_reason == SOFT_RESET:
             reason = 5, "SoftReset"
-    errlog_add(f"[BOOT] info: {reason[1]}")
+    syslog(f"[BOOT] info: {reason[1]}")
     return reason
+
+
+def enableESPNow() -> str:
+    """
+    Enable ESP-NOW communication
+    """
+    if cfgget('espnow'):
+        try:
+            from Espnow import ESPNowSS
+            verdict = ESPNowSS().start_server()
+            console_write(f"[TASK] Start ESPNow-InterCon server: {verdict}")
+        except Exception as e:
+            syslog(f"[ERR] Start ESPNow-InterCon server: {e}")
+            return str(e)
+    return "ESPNow disabled"

micrOS/source/InterConnect.py

@@ -1,14 +1,35 @@
+"""
+Module is responsible for device-device communication
+dedicated to micrOS framework.
+Built-in-function:
+- Socket InterConnect interface
+- ESPNow redirection
+
+Designed by 
+    Marcell Ban aka BxNxM
+    Kristof Kasza aka KKristof452
+"""
+
 from socket import getaddrinfo, SOCK_STREAM
-from re import compile
-import uasyncio as asyncio
-from Debug import errlog_add
+from re import compile as re_compile
+from json import loads
+from binascii import hexlify
+from uasyncio import open_connection
+
+from Debug import syslog
 from Config import cfgget
 from Server import Server
 from Tasks import NativeTask
 
+if cfgget('espnow'):
+    from Espnow import ESPNowSS
+else:
+    ESPNowSS = None
+
 
 class InterCon:
-    CONN_MAP = {}
+    CONN_MAP: dict[str, str] = {}   # hostname: IP address pairs
+    NO_ESPNOW: list[str] = []       # disabled ESPNow hostname list (cache for fallback speed-up)
     PORT = cfgget('socport')
 
     def __init__(self):
@@ -19,10 +40,10 @@ class InterCon:
 
     @staticmethod
     def validate_ipv4(str_in):
-        pattern = compile(r'^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])$')
+        pattern = re_compile(r'^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])$')
         return bool(pattern.match(str_in))
 
-    async def send_cmd(self, host, cmd):
+    async def send_cmd(self, host:str, cmd:list):
         """
         Async Main method to implement device-device communication with
         - dhcp host resolve and IP caching
@@ -34,14 +55,19 @@ class InterCon:
         # Check if host is a hostname (example.local) and resolve its IP address
         if not InterCon.validate_ipv4(host):
             hostname = host
-            # Retrieve IP address by hostname dynamically
+            # Lookup hostname without .domain (sub-hostname matching)
+            if '.' not in hostname:
+                _hosts = list([d for d in InterCon.CONN_MAP if hostname in d])
+                if len(_hosts) > 0:
+                    hostname = _hosts[0]
+            # Retrieve IP address by hostname.domain dynamically
             if InterCon.CONN_MAP.get(hostname, None) is None:
                 try:
                     addr_info = getaddrinfo(host, InterCon.PORT, 0, SOCK_STREAM)
                     host = addr_info[-1][4][0]
                 except OSError as e:
                     Server.reply_all(f"[intercon] NoHost: {e}")
-                    errlog_add(f"[intercon] send_cmd {host} oserr: {e}")
+                    syslog(f"[intercon] send_cmd {host} oserr: {e}")
                     return ''
             else:
                 # Restore IP from cache by hostname
@@ -50,12 +76,12 @@ class InterCon:
         if InterCon.validate_ipv4(host):
             try:
                 # Create socket object
-                self.reader, self.writer = await asyncio.open_connection(host, InterCon.PORT)
+                self.reader, self.writer = await open_connection(host, InterCon.PORT)
                 # Send command over TCP/IP
                 output = await self.__run_command(cmd, hostname)
             except OSError as e:
                 Server.reply_all(f"[intercon] NoHost: {e}")
-                errlog_add(f"[intercon] send_cmd {host} oserr: {e}")
+                syslog(f"[intercon] send_cmd {host} oserr: {e}")
                 output = None
             finally:
                 if self.writer:
@@ -68,26 +94,23 @@ class InterCon:
                 InterCon.CONN_MAP[hostname] = None if output is None else host
             # None: ServerBusy(or \0) or Prompt mismatch (auto delete cached IP), STR: valid comm. output
             return output
-        else:
-            errlog_add(f"[ERR][intercon] Invalid host: {host}")
+        syslog(f"[ERR][intercon] Invalid host: {host}")
         return ''
 
-    async def __run_command(self, cmd, hostname):
+    async def __run_command(self, cmd:list, hostname:str):
         """
         Implements receive data on open connection, command query and result collection
         :param cmd: command string to server socket shell
         :param hostname: hostname for prompt checking
         Return None here will trigger retry mechanism... + deletes cached IP
         """
-        cmd = str.encode(cmd)
         data, prompt = await self.__receive_data()
         if "Connection is busy. Bye!" in prompt:
             return None
         # Compare prompt |node01 $| with hostname 'node01.local'
         if hostname is None or prompt is None or str(prompt).replace('$', '').strip() == str(hostname).split('.')[0]:
             # Run command on validated device
-            # TODO: Handle multiple cmd as input, separated by ; (????)
-            self.writer.write(cmd)
+            self.writer.write(str.encode(' '.join(cmd)))
             await self.writer.drain()
             data, _ = await self.__receive_data(prompt=prompt)
             if data == '\0':
@@ -104,13 +127,13 @@ class InterCon:
             await self.writer.drain()
             data, prompt = await self.__receive_data(prompt=prompt)
         except Exception as e:
-            errlog_add(f'[intercon][ERR] Auth: {e}')
+            syslog(f'[intercon][ERR] Auth: {e}')
             data = 'AuthFailed'
         if 'AuthOk' in data:
             return True             # AuthOk
         return False                # AuthFailed
 
-    async def __receive_data(self, prompt=None):
+    async def __receive_data(self, prompt=None) -> tuple[str, str]:
         """
         Implements data receive loop until prompt / [configure] / Bye!
         :param prompt: socket shell prompt
@@ -140,8 +163,34 @@ class InterCon:
         data = data.replace(prompt, '').replace('\n', ' ')
         return data, prompt
 
+    async def auto_espnow_handshake(self, host:str) -> dict:
+        """
+        [1] Check espnow.server running on host
+        [2] Get MAC address for host (from system info)
+        [3] Execute ESPNowSS.handshake
+        """
+        response = await self.send_cmd(host, ["task", "list", ">json"])
+        if not response:
+            return {None: f"[ERR] ESPNow auto handshake: task list >>{host}: {response}"}
+
+        active_tasks = loads(response).get("active")
+        if active_tasks and "espnow.server" in active_tasks:
+            response = await self.send_cmd(host, ["system", "info", ">json"])
+            if not response:
+                return {None: "[ERR] ESPNow auto handshake: system info"}
+            try:
+                host_mac = loads(response).get("mac")
+            except Exception as ex:
+                return {None: f"[ERR] ESPNow auto handshake: {ex}"}
+
+            return ESPNowSS().handshake(host_mac)
+
+        if not InterCon.validate_ipv4(host):
+            InterCon.NO_ESPNOW.append(str(host).split(".")[0])   # host.local -> host
+        return {None: f"ESPNow auto handshake: espnow disabled on host {host}"}
+
 
-async def _send_cmd(host, cmd, com_obj):
+async def _socket_send_cmd(host:str, cmd:list, com_obj:InterCon) -> None:
     """
     Async send command wrapper for further async task integration and sync send_cmd usage (main)
     :param host: hostname / IP address
@@ -149,42 +198,77 @@ async def _send_cmd(host, cmd, com_obj):
     :param com_obj: InterCon object to utilize send_cmd method and task status updates
     """
     # Send command
+    for _ in range(0, 2):                           # Retry mechanism
+        out = await com_obj.send_cmd(host, cmd)     # Send CMD
+        if out is not None:                         # Retry mechanism
+            break
+        await com_obj.task.feed(sleep_ms=100)       # Retry mechanism
+    com_obj.task.out = '' if out is None else out
+
+
+async def _send_cmd(host:str, cmd:list|str, com_obj:InterCon):
+    """
+    Top level InterConnect callback function
+        [1] node01.domain -> ESPNow, Socket
+                (domain: .local, .net, etc.)
+        [2] node01 -> ESPNow, Socket fallback in case found in InterConnect cache
+        [3] IP address -> Socket
+    """
     with com_obj.task:
-        for _ in range(0, 4):                           # Retry mechanism
-            out = await com_obj.send_cmd(host, cmd)     # Send CMD
-            if out is not None:                         # Retry mechanism
-                break
-            await asyncio.sleep_ms(100)                 # Retry mechanism
-        com_obj.task.out = '' if out is None else out
-    return com_obj.task.out
+        if ESPNowSS:
+            # [1] ESPNow Active
+            name = str(host).split(".")[0]   # host.local -> host
+            if name not in InterCon.NO_ESPNOW and name in list(ESPNowSS().devices.values()):
+                com_obj.task.out = "Redirected to ESPNow"
+                if isinstance(cmd, list):
+                    cmd = ' '.join(cmd)
+                # Send command and retrieve result
+                sender = ESPNowSS().send(peer=name, msg=cmd)
+                sender_task = NativeTask.TASKS.get(list(sender.keys())[0])
+                result = await sender_task.await_result(timeout=10)
+                if result != "Timeout has beed exceeded":
+                    # Successful command execution
+                    com_obj.task.out = result                       # Output mirroring: Child -> Parent
+                    sender_task.out = "Redirected to ParentTask"    # Remove redundant data in embedded mode
+                    return
 
+        # Handle legacy string input
+        if isinstance(cmd, str):
+            cmd = cmd.split()
+        # [1][2] Socket send (default and fallback)
+        await _socket_send_cmd(host, cmd, com_obj)
 
-def send_cmd(host, cmd):
+        if ESPNowSS and name not in InterCon.NO_ESPNOW:
+            # [3] Automatic ESPNow handshake
+            verdict = await com_obj.auto_espnow_handshake(host)
+            if list(verdict.keys())[0] is None:
+                syslog(str(list(verdict.values())[0]))
+
+
+def send_cmd(host:str, cmd:list|str) -> dict:
     """
-    Sync wrapper of async _send_cmd (InterCon.send_cmd consumer with retry)
-    :param host: hostname / IP address
-    :param cmd: command string to server socket shell
+    Top level InterConnect send task creation
+        Handles ESPNow and socket communication
     """
     def _tagify():
         nonlocal host, cmd
-        _mod = cmd.split(' ')[0].strip()
+        _mod = cmd[0]
         if InterCon.validate_ipv4(host):
             return f"{'.'.join(host.split('.')[-2:])}.{_mod}"
-        return f"{host.replace('.local', '')}.{_mod}"
+        target = ".".join(host.split(".")[0:-1]) if "." in host else host
+        return f"{target}.{_mod}"
 
     com_obj = InterCon()
-    tag = f"con.{_tagify()}"
-    started = com_obj.task.create(callback=_send_cmd(host, cmd, com_obj), tag=tag)
-    if started:
-        result = {"verdict": f"Task started: task show {tag}", "tag": tag}
-    else:
-        result = {"verdict": "Task is Busy", "tag": tag}
-    return result
+    task_id = f"con.{_tagify()}"            # CHECK TASK ID CONFLICT
+    return com_obj.task.create(callback=_send_cmd(host, cmd, com_obj), tag=task_id)
 
 
-def host_cache():
+def host_cache() -> dict:
     """
     Dump InterCon connection cache
     """
-    return InterCon.CONN_MAP
-
+    if ESPNowSS is None:
+        return InterCon.CONN_MAP
+    all_devs = dict({name:hexlify(mac, ':').decode() for mac, name in ESPNowSS().devices.items()})
+    all_devs.update(InterCon.CONN_MAP)
+    return all_devs

micrOS/source/Interrupts.py

@@ -19,7 +19,7 @@ Reference: https://docs.micropython.org/en/latest/library/machine.Pin.html
 from machine import Pin
 from utime import ticks_ms, ticks_diff
 from Config import cfgget
-from Debug import console_write, errlog_add
+from Debug import console_write, syslog
 from Tasks import exec_lm_pipe_schedule
 from microIO import resolve_pin
 if cfgget('cron'):
@@ -71,8 +71,9 @@ def enableInterrupt():
 
 def enableCron():
     """
-    Set time stump based scheduler aka cron in timer1
+    Set time stump based scheduler aka cron on Timer1
     Input: cron(bool), crontasks(str)
+    This is for low frequency sampling, like 12 or 6 / minute (due to low power mode compatibility)
     """
     timer_period = 5000         # Timer period ms: 12 check/min
     console_write(f"[IRQ] CRON IRQ SETUP: {cfgget('cron')} SEQ: {timer_period}")
@@ -80,11 +81,10 @@ def enableCron():
     if cfgget("cron") and cfgget('crontasks').lower() != 'n/a':
         from machine import Timer
         # INIT TIMER 1 IRQ with callback function wrapper
-        lm_byte = bytearray(cfgget('crontasks'), 'utf-8')           # store as bytearray (cache optimization)
-        sample = int(timer_period/1000)
+        sampling = int(timer_period/1000)
         timer = Timer(1)
         timer.init(period=timer_period, mode=Timer.PERIODIC,
-                   callback=lambda timer: scheduler(lm_byte, sample))
+                   callback=lambda timer: scheduler(sampling))
 
 
 #################################################################
@@ -157,7 +157,7 @@ def initEventIRQs():
         try:
             return resolve_pin(_p)
         except Exception as e:
-            errlog_add(f'[ERR][!] EVENT {_p} IO error: {e}')
+            syslog(f'[ERR][!] EVENT {_p} IO error: {e}')
         return None
 
     # Load External IRQ (1-4) execution data set from node config