micrOSDevToolKit 2.13.0__py3-none-any.whl → 2.17.0__py3-none-any.whl

toolkit/DevEnvOTA.py

@@ -13,18 +13,16 @@ try:
     from .lib import LocalMachine
     from .lib.TerminalColors import Colors
     from .lib.SafeInput import input_with_timeout
-    from .lib.file_extensions import check_all_extensions, check_web_extensions, check_python_extensions
+    from .lib.MicrosFiles import check_all_extensions, check_web_extensions, check_python_extensions, micros_resource_list
     from .lib.Repository import git_clone_archive, git_clone
-    from .MicrosFiles import micros_resource_list
 except Exception as e:
     print("Import warning __name__:{}: {}".format(__name__, e))
     from DevEnvCompile import Compile
     from lib import LocalMachine
     from lib.TerminalColors import Colors
     from lib.SafeInput import input_with_timeout
-    from lib.file_extensions import check_all_extensions, check_web_extensions, check_python_extensions
+    from lib.MicrosFiles import check_all_extensions, check_web_extensions, check_python_extensions, micros_resource_list
     from lib.Repository import git_clone_archive, git_clone
-    from MicrosFiles import micros_resource_list
 
 sys.path.append(MYPATH)
 import socketClient
@@ -406,6 +404,30 @@ class OTA(Compile):
             print(f"[SIM] 'OTA' COPY FILES... {source} -> {target}")
             LocalMachine.FileHandler().copy(source, target)
 
+    @staticmethod
+    def _dynamic_file_target(source_name, force_lm):
+        """
+        Dynamic file target path based on file extension.
+        -> web
+        -> modules
+        -> config
+        -> data
+        """
+        # Force LM update - user load modules - drag n drop files
+        if force_lm and not source_name.startswith('LM_') and source_name.endswith('py'):
+            source_name = 'LM_{}'.format(source_name)
+        # Drag-n-Drop file upload file type check and folder adjustment
+        if check_web_extensions(source_name):
+            source_name_target = os.path.join('web', source_name)
+        elif check_python_extensions(source_name):
+            source_name_target = os.path.join('modules', source_name)
+        elif source_name.endswith("node_config.json"):
+            source_name_target = os.path.join('config', source_name)
+        else:
+            # Copy file to micrOS data folder (not web, not module, not config)
+            source_name_target = os.path.join('data', source_name)
+        return source_name_target
+
     def ota_webrepl_update_core(self, device=None, upload_path_list=None, ota_password='ADmin123',
                                 force_lm=False, upload_root_dir=None):
         """
@@ -468,22 +490,11 @@ class OTA(Compile):
             exitcode = -1
             source_name = os.path.basename(source)
             if upload_root_dir is None:
-                # Drag-n-Drop file upload file type check and folder adjustment
-                if check_web_extensions(source_name):
-                    source_name_target = os.path.join('web', source_name)
-                elif not check_python_extensions(source_name) and not source_name.endswith("node_config.json"):
-                    source_name_target = os.path.join('data', source_name)
-                else:
-                    # Copy file to micrOS root folder (.mpy and .py or node_config.json)
-                    source_name_target = source_name
+                source_name_target = self._dynamic_file_target(source_name, force_lm)
             else:
                 # MAIN USE-CASE
                 source_name_target = source.replace(upload_root_dir, '')
 
-            # Force LM update - user load modules - drag n drop files
-            if force_lm and not source_name.startswith('LM_') and source_name.endswith('.py'):
-                source_name_target = 'LM_{}'.format(source_name)
-
             command = '{python} {api} -p {pwd} {input_file} {host}:{target_path}'.format(
                 python=self.python_interpreter,
                 api=self.webreplcli_repo_path.replace(" ", "\ "),

toolkit/DevEnvUSB.py

@@ -13,14 +13,14 @@ try:
     from .lib import LocalMachine
     from .lib.TerminalColors import Colors
     from .lib.SerialDriverHandler import install_usb_serial_driver
-    from .MicrosFiles import micros_resource_list
+    from .lib.MicrosFiles import micros_resource_list
 except Exception as e:
     print("Import warning __name__:{}: {}".format(__name__, e))
     from DevEnvCompile import Compile
     from lib import LocalMachine
     from lib.TerminalColors import Colors
     from lib.SerialDriverHandler import install_usb_serial_driver
-    from MicrosFiles import micros_resource_list
+    from lib.MicrosFiles import micros_resource_list
 
 
 class USB(Compile):
@@ -35,6 +35,7 @@ class USB(Compile):
         self.micrOS_node_config_archive = os.path.join(MYPATH, "user_data/node_config_archive")
         self.node_config_profiles_path = os.path.join(MYPATH, "../micrOS/release_info/node_config_profiles/")
         self.esptool_interface = self.get_valid_esptool_cmd()
+        self.micrOS_node_config_path = "/config/node_config.json"
         self.dev_types_and_cmds = \
             {'esp32':
                  {'erase': '{esptool_interface} --port {dev} erase_flash',
@@ -436,10 +437,14 @@ class USB(Compile):
         selected_index = int(input("Select index: "))
         # Use (already existing) selected config to restore
         selected_config = conf_list[selected_index]
-        if '-' in selected_config:
+        if selected_config.endswith(".json"):
             # Restore saved config
-            target_path = os.path.join(self.precompiled_micrOS_dir_path, selected_config.split('-')[1])
+            selected_config_target = selected_config
+            if '-' in selected_config:
+                # Remove prefix (device name / profile name)
+                selected_config_target = selected_config.split('-')[-1]
             source_path = os.path.join(self.micrOS_node_config_archive, selected_config)
+            target_path = os.path.join(self.precompiled_micrOS_dir_path, selected_config_target)
         elif selected_index == len(conf_list) - 1:
             # SKIP restore config - use the local version in mpy-micrOS folder
             target_path = os.path.join(self.precompiled_micrOS_dir_path, 'node_config.json')
@@ -448,7 +453,7 @@ class USB(Compile):
             # Create new config - from micrOS folder path -> mpy-micrOS folder
             target_path = os.path.join(self.precompiled_micrOS_dir_path, selected_config)
             source_path = os.path.join(self.micrOS_dir_path, selected_config)
-        self.console("Restore config: {} -> {}".format(source_path, target_path))
+        self.console(f"Restore config: {source_path} -> {target_path}")
         if source_path is not None:
             LocalMachine.FileHandler.copy(source_path, target_path)
 
@@ -485,9 +490,11 @@ class USB(Compile):
         else:
             self.console("Profile was selected: {}{}{}".format(Colors.OK, profile_list[int(profile)], Colors.NC))
         # Read default conf
-        default_conf_path = os.path.join(self.micros_sim_workspace, 'node_config.json')
+        default_conf_path = os.path.join(self.micrOS_dir_path, 'node_config.json')
         if not os.path.isfile(default_conf_path):
-            self.micrOS_sim_default_conf_create()
+            if not self.__generate_default_config():
+                self.console(f"Missing default config: {default_conf_path}", state="warn")
+                return None
         with open(default_conf_path, 'r') as f:
             default_conf_dict = json.load(f)
         # Read profile
@@ -650,21 +657,28 @@ class USB(Compile):
         return processid
 
     def __get_node_config(self):
+
+        def _get_config(path):
+            mpremote_cmd = self.dev_types_and_cmds[self.selected_device_type]['mpremote_cmd']
+            if mpremote_cmd is None:
+                # Legacy ampy command (esp32 auto reboot tolerance...)
+                _command = (self.dev_types_and_cmds[self.selected_device_type]['ampy_cmd']
+                           .format(dev=device, args=f'get {path}'))
+            else:
+                _command = f'{mpremote_cmd} fs cat {path}'  # new mpremote <1.24.1
+            _exitcode, _stdout, _stderr = LocalMachine.CommandHandler.run_command(_command, shell=True)
+            return _exitcode, _stdout, _stderr
+
         device = self.get_devices()[0]
         self.console(f"Get node config over USB: {device}")
-        mpremote_cmd = self.dev_types_and_cmds[self.selected_device_type]['mpremote_cmd']
-        if mpremote_cmd is None:
-            # Legacy ampy command (esp32 auto reboot tolerance...)
-            command = self.dev_types_and_cmds[self.selected_device_type]['ampy_cmd'].format(dev=device, args='get node_config.json')
+        if self.dry_run:
+            exitcode, stdout, stderr = 0, '{"key": "Dummy stdout"}', ''
         else:
-            command = f'{mpremote_cmd} fs cat node_config.json'     # new mpremote <1.24.1
-        if not self.dry_run:
-            exitcode, stdout, stderr = LocalMachine.CommandHandler.run_command(command, shell=True)
+            exitcode, stdout, stderr = _get_config("node_config.json")                  # Legacy
+            if exitcode != 0:
+                exitcode, stdout, stderr = _get_config(self.micrOS_node_config_path)    # New
             self._archive_node_config()
-        else:
-            exitcode = 0
-            stdout = '{"key": "Dummy stdout"}'
-            stderr = ''
+
         if '\n' in stdout:
             stdout = stdout.strip().splitlines()
             stdout = str([line for line in stdout if '{' in line and '}' in line][0])
@@ -687,7 +701,7 @@ class USB(Compile):
 
     def _archive_node_config(self):
         self.console("ARCHIVE NODE_CONFIG.JSON")
-        local_node_config = os.path.join(self.precompiled_micrOS_dir_path, 'node_config.json')
+        local_node_config = self.precompiled_micrOS_dir_path + self.micrOS_node_config_path
         if os.path.isfile(local_node_config):
             with open(local_node_config, 'r') as f:
                 node_devfid = json.load(f)['devfid']
@@ -699,8 +713,8 @@ class USB(Compile):
                 LocalMachine.FileHandler.copy(local_node_config, archive_node_config)
 
     def __override_local_config_from_node(self, node_config=None):
-        node_config_path = os.path.join(self.precompiled_micrOS_dir_path, 'node_config.json')
-        self.console("Overwrite node_config.json with connected node config: {}".format(node_config_path), state='ok')
+        node_config_path = self.precompiled_micrOS_dir_path + self.micrOS_node_config_path
+        self.console(f"Overwrite node_config.json with connected node config: {node_config_path}", state='ok')
         if not self.dry_run and node_config is not None:
             with open(node_config_path, 'w') as f:
                 f.write(node_config)

toolkit/LM_to_compile.dat

@@ -11,7 +11,6 @@ LM_bme280.py
 LM_co2.py
 LM_dht11.py
 LM_light_sensor.py
-LM_L298N_DCmotor.py
 LM_intercon.py
 LM_tinyrgb.py
 LM_repair.py
@@ -41,3 +40,6 @@ LM_haptic.py
 LM_distance.py
 LM_veml7700.py
 LM_neomatrix.py
+LM_L298N.py
+LM_mqtt_client.py
+LM_cluster.py

toolkit/MicrOSDevEnv.py

@@ -142,6 +142,20 @@ class MicrOSDevTool(OTA, USB):
         [!] name dependency with micrOS internal manual provider
         """
 
+        def _is_private_func(_line):
+            nonlocal decorator, line
+            if 'def ' in line and 'def _' not in line:
+                if "async" in line and decorator is not None and "@micro_task" in decorator:
+                    # Enable (adjust) async task method with publish_micro_task decorator
+                    line = line.replace("async", "").strip()
+                    return False
+                if '(self' in line or '(cls' in _line or (decorator is not None and "staticmethod" in decorator):
+                    # Ignore class methods
+                    return True
+                return False
+            # Ignore non functions, and hidden functions (starts with _)
+            return True
+
         if not os.path.isdir(self.sfuncman_output_path):
             self.console('DOC GEN DISABLED', state="WARN")
             return
@@ -151,11 +165,12 @@ class MicrOSDevTool(OTA, USB):
         static_help_html_path = os.path.join(self.sfuncman_output_path, 'sfuncman.html')
 
         # [PARSING] Collect Load Module function structure buffer
-        modules_to_doc = (i.split('.')[0] for i in LocalMachine.FileHandler.list_dir(self.micrOS_dir_path) if
+        modules_path = os.path.join(self.micrOS_dir_path, "modules")
+        modules_to_doc = (i.split('.')[0] for i in LocalMachine.FileHandler.list_dir(modules_path) if
                           i.startswith('LM_') and (i.endswith('.py')))
         module_function_dict = {}
         for LM in modules_to_doc:
-            LMpath = '{}/{}.py'.format(self.micrOS_dir_path, LM)
+            LMpath = '{}/{}.py'.format(modules_path, LM)
             try:
                 module_name = LM.replace('LM_', '')
                 module_function_dict[module_name] = {}
@@ -165,25 +180,26 @@ class MicrOSDevTool(OTA, USB):
                         line = f.readline()
                         if not line:
                             break
-                        if 'def ' in line and 'def _' not in line:
-                            if '(self' in line or '(cls' in line or (decorator is not None and "staticmethod" in decorator):
-                                continue
-                            # Gen proper func name
-                            command = '{}'.format(line.split(')')[0]).replace("def", '').strip()
-                            command = command.replace('(', ' ') \
-                                .replace(',', '') \
-                                .replace('msgobj=None', '') \
-                                .replace('force=True', '')
-                            func = command.strip().split()[0]
-                            param = ' '.join(command.strip().split()[1:])
-                            # Save record
-                            if module_function_dict[module_name].get(func, None) is None:
-                                module_function_dict[module_name][func] = {}
-                            module_function_dict[module_name][func]['param(s)'] = param if len(param) > 0 else ""
-                        elif line.strip().startswith("@"):
-                            decorator = line.strip()
-                        else:
-                            decorator = None
+                        if _is_private_func(line):
+                            if line.strip().startswith("@"):
+                                decorator = line.strip()
+                            else:
+                                decorator = None
+                            continue
+
+                        # Gen proper func name
+                        command = '{}'.format(line.split(')')[0]).replace("def", '').strip()
+                        command = command.replace('(', ' ') \
+                            .replace(',', '') \
+                            .replace('msgobj=None', '') \
+                            .replace('force=True', '')
+                        func = command.strip().split()[0]
+                        param = ' '.join(command.strip().split()[1:])
+                        # Save record
+                        if module_function_dict[module_name].get(func, None) is None:
+                            module_function_dict[module_name][func] = {}
+                        module_function_dict[module_name][func]['param(s)'] = param if len(param) > 0 else ""
+
                 # Create / update module data fields
                 module_function_dict[module_name]['img'] = f"https://github.com/BxNxM/micrOS/blob/master/media/lms/{module_name}.png?raw=true"
             except Exception as e:

toolkit/dashboard_apps/QMI8685_GYRO.py

@@ -26,7 +26,7 @@ def print_table(data):
 
 def app(devfid=None, pwd=None):
     CLIENT = AppBase(device=devfid, password=pwd)
-    measure_cmd = 'qmi8685 measure >json'
+    measure_cmd = 'qmi8658 measure >json'
 
     mp.set_start_method('spawn', force=True)
     queue = mp.Queue()

toolkit/dashboard_apps/SystemTest.py

@@ -551,8 +551,8 @@ def webcli_test():
             out = CLIENT.execute(['conf', 'devip'], tout=TIMEOUT_SEC)
             if out[0] and out[1] is not None:
                 devip = out[1]
-                endpoints.append(f'http://{devip}')
                 endpoints.append(f'http://{devip}/rest')
+                endpoints.append(f'http://{devip}')
 
             for endpoint in endpoints:
                 _start_t = time.time()
@@ -562,12 +562,15 @@ def webcli_test():
                     delta_t = round(time.time() - _start_t, 2)
                     # Check if the request was successful
                     if not (response.status_code == 200 and ('<!DOCTYPE html>' in str(response.content) or '"micrOS"' in str(response.content))):
-                        verdict += f" Endpoint: {endpoint} [{Colors.ERR}NOK{Colors.NC}]({delta_t}s)"
-                        print(response.content)
-                        state = False
+                        if response.status_code == 400 and "Low memory" in str(response.content):
+                            verdict += f" Endpoint: {endpoint} - Low memory mode [{Colors.WARN}OK{Colors.NC}]({delta_t}s)"
+                            print(response.content)
+                        else:
+                            verdict += f" Endpoint: {endpoint} [{Colors.ERR}NOK{Colors.NC}]({delta_t}s)"
+                            print(response.content)
+                            state = False
                     else:
                         verdict += f" Endpoint: {endpoint} [{Colors.OK}OK{Colors.NC}]({delta_t}s)"
-
                         metrics_name = 'landingpage' if "." in endpoint.split('/')[-1] else endpoint.split('/')[-1]
                         _add_metrics(f"web_{metrics_name}_response_ms", int(delta_t*1000))
                 except Exception as e:

toolkit/{MicrosFiles.py → lib/MicrosFiles.py}

@@ -1,12 +1,32 @@
 import os
 
 try:
-    from .lib import LocalMachine
-    from .lib.file_extensions import check_all_extensions, check_web_extensions
+    from . import LocalMachine
 except Exception as e:
     print("Import warning __name__:{}: {}".format(__name__, e))
-    from lib import LocalMachine
-    from lib.file_extensions import check_all_extensions, check_web_extensions
+    import LocalMachine
+
+PYTHON_EXTENSIONS = ('py', 'mpy')
+WEB_ONLY = ('js', 'html', 'css', 'json', 'ico', 'jpeg', 'png')
+ENABLED_EXTENSIONS = PYTHON_EXTENSIONS + WEB_ONLY
+
+def check_all_extensions(path):
+    extension = path.split('.')[-1]
+    if extension in ENABLED_EXTENSIONS:
+        return True
+    return False
+
+def check_web_extensions(path):
+    extension = path.split('.')[-1]
+    if extension in WEB_ONLY:
+        return True
+    return False
+
+def check_python_extensions(path):
+    extension = path.split('.')[-1]
+    if extension in PYTHON_EXTENSIONS:
+        return True
+    return False
 
 def micros_resource_list(root_folder):
     resources_path = []

toolkit/lib/pip_package_installer.py

@@ -66,8 +66,11 @@ def install_package(name, additional_pip_param=None):
             print(f"{TerminalColors.Colors.OK}[PIP] install {name} OK{TerminalColors.Colors.NC}")
         else:
             print(f"{TerminalColors.Colors.WARN}[PIP] install {name} NOK{TerminalColors.Colors.NC}")
-            # Support brew installed python
-            if "This environment is externally managed" in out:
+            if "--break-system-packages" in out:
+                # (Retry) Support system wide package install
+                install_package(name, additional_pip_param="--break-system-packages")
+            elif "This environment is externally managed" in out:
+                # (Retry-Fallback) Support brew installed python
                 if "brew install" in out:
                     print(f"Install optional dependency (brew): {name}")
                     state, out = run_subprocess(["brew", "install", name])

toolkit/micrOSdashboard.py

@@ -29,14 +29,14 @@ try:
     from . import socketClient
     from . import dashboard_apps
     from .lib import LocalMachine
-    from .lib.file_extensions import check_all_extensions
+    from .lib.MicrosFiles import check_all_extensions
 except Exception as e:
     print("Import warning: {}".format(e))
     import MicrOSDevEnv
     import socketClient
     import dashboard_apps
     from lib import LocalMachine
-    from lib.file_extensions import check_all_extensions
+    from lib.MicrosFiles import check_all_extensions
 
 APP_DIR = os.path.join(MYPATH, 'dashboard_apps')
 

toolkit/micrOSlint.py

@@ -18,7 +18,7 @@ MICROS_SOURCE_DIR = os.path.join(MYPATH, '../micrOS/source')
 RELEASE_INFO_PATH = os.path.join(MYPATH, '../micrOS/release_info/micrOS_ReleaseInfo')
 
 # MICROS LINTER CONFIG
-ALLOWED_LM_DEP_WARNS = 6        # ALLOWED NUMBER OF LM CORE DEPENDENCY (less is better)
+ALLOWED_LM_DEP_WARNS = 4        # ALLOWED NUMBER OF LM CORE DEPENDENCY (less is better)
 
 def parse_micros_file_categories(verbose=True):
     """
@@ -27,11 +27,17 @@ def parse_micros_file_categories(verbose=True):
     file_ignore_list = ['.DS_Store']
     # micrOS file categories
     categories = {'core': [], 'load_module': [], 'pin_maps': [], 'other': []}
-    files = [f for f in LocalMachine.FileHandler.list_dir(MICROS_SOURCE_DIR) if f not in file_ignore_list]
+    modules_path = os.path.join(MICROS_SOURCE_DIR, "modules")
+    web_path = os.path.join(MICROS_SOURCE_DIR, "web")
+    check_file = lambda _b, _f: os.path.isfile(os.path.join(_b, _f)) and _f not in file_ignore_list
+    root_files = [f for f in LocalMachine.FileHandler.list_dir(MICROS_SOURCE_DIR) if check_file(MICROS_SOURCE_DIR, f)]
+    module_files = [f"modules/{f}" for f in LocalMachine.FileHandler.list_dir(modules_path) if check_file(modules_path, f)]
+    web_files = [f"web/{f}" for f in LocalMachine.FileHandler.list_dir(web_path) if check_file(web_path, f)]
+    files = root_files + module_files + web_files
     for f in files:
-        if f.startswith('LM_') and f.endswith('.py'):
+        if f.endswith('.py') and "LM_" in f:
             categories['load_module'].append(f)
-        elif f.startswith('IO_') and f.endswith('.py'):
+        elif f.endswith('.py') and "IO_" in f:
             categories['pin_maps'].append(f)
         elif f.endswith('.py'):
             categories['core'].append(f)
@@ -179,7 +185,8 @@ def load_module_checker(categories, verbose=True):
                 _allowed.append(_allow)
         return _relation, _allowed
 
-    lm_god_mode = ['LM_system.py', 'LM_pacman.py', 'LM_intercon.py']
+    lm_god_mode = ['modules/LM_system.py', 'modules/LM_pacman.py',
+                   'modules/LM_espnow.py', 'modules/LM_cluster.py']
     state_lm_dep = True
     verdict = []
     lm_resources = categories['load_module']
@@ -224,7 +231,10 @@ def _run_pylint(file_name):
         '--disable=too-many-return-statements',     # :D I don't think so :D
         '--disable=too-many-branches',              # :D I don't think so :D
         '--disable=too-many-positional-arguments',  # :D I don't think so :D
-        '--disable=too-many-instance-attributes'    # :D I don't think so :D
+        '--disable=too-many-instance-attributes',   # :D I don't think so :D
+        '--disable=wrong-import-position',          # :D I don't think so :D !!! It is needed due to optimizations
+        '--disable=too-many-arguments',             # :D I know what I am doing...
+        '--disable=multiple-statements',            # :D I line oneliners
     ]
     if file_name in ['Tasks.py', 'microIO.py', 'Types.py']:
         pylint_opts.append('--disable=exec-used')   # Disable micrOS execution core exec/eval warning
@@ -469,7 +479,7 @@ def short_report(categories, states, verbose=True):
     print(f"load_module_checker:                load module dependency check (no core): {c_OK if lm_dep[0] else c_NOK} {'' if lm_dep[1] == 0 else f'{lm_dep[1]}{_vis(lm_dep_diff)} warning(s)'}")
     print(f"  core pylint score:                {core_pylint}{_pyl_vis(core_score_diff)}")
     print(f"load module pylint score:           {lm_pylint}{_pyl_vis(load_score_diff)}")
-    print(f"pylint resource check (syntax?):     {c_OK if pylint_check[0] else f'{c_NOK}: {pylint_check[1]}' }")
+    print(f"pylint resource check (syntax):     {c_OK if pylint_check[0] else f'{c_NOK}: {pylint_check[1]}' }")
 
     exitcode = sum([1 for k, v in states.items() if not v[0]])
     print(f"micrOSlint verdict: {c_OK if exitcode == 0 else c_NOK}")

toolkit/simulator_lib/mqtt_as/Note.md

@@ -0,0 +1,15 @@
+# Test LM_mqtt_client.py in simulator
+
+## mqtt_as dependency
+
+```
+https://github.com/peterhinch/micropython-mqtt/tree/master/mqtt_as
+```
+
+## TODO
+
+```
+Optimize module dependency for micrOS usage
+- remove ssid and password usage -> micrOS handles that
+- remove nework conenction check -> micrOS makes that
+```