micrOSDevToolKit 2.11.0__py3-none-any.whl → 2.13.1__py3-none-any.whl

toolkit/DevEnvOTA.py

@@ -5,22 +5,26 @@ MYPATH = os.path.dirname(__file__)
 print("Module [DevEnvOTA] path: {} __package__: {} __name__: {} __file__: {}".format(
     sys.path[0], __package__, __name__, MYPATH))
 
+from pprint import pprint
+
 try:
     from .DevEnvCompile import Compile
     from . import socketClient
     from .lib import LocalMachine
     from .lib.TerminalColors import Colors
     from .lib.SafeInput import input_with_timeout
-    from .lib.file_extensions import check_all_extensions
+    from .lib.file_extensions import check_all_extensions, check_web_extensions, check_python_extensions
     from .lib.Repository import git_clone_archive, git_clone
+    from .MicrosFiles import micros_resource_list
 except Exception as e:
     print("Import warning __name__:{}: {}".format(__name__, e))
     from DevEnvCompile import Compile
     from lib import LocalMachine
     from lib.TerminalColors import Colors
     from lib.SafeInput import input_with_timeout
-    from lib.file_extensions import check_all_extensions
+    from lib.file_extensions import check_all_extensions, check_web_extensions, check_python_extensions
     from lib.Repository import git_clone_archive, git_clone
+    from MicrosFiles import micros_resource_list
 
 sys.path.append(MYPATH)
 import socketClient
@@ -241,9 +245,9 @@ class OTA(Compile):
         self.console("  loader update: {}".format(force_mode), state='OK')
 
         # Parse files from precompiled dir
-        resource_list_to_upload = [os.path.join(self.precompiled_micrOS_dir_path, pysource) for pysource in
-                                   LocalMachine.FileHandler.list_dir(self.precompiled_micrOS_dir_path)
-                                   if check_all_extensions(pysource)]
+        resource_list_to_upload, dir_list_to_create = micros_resource_list(self.precompiled_micrOS_dir_path)
+        # LIMITATION: WEBREPL Cannot create directories with remote command...
+
         # Apply upload settings on parsed resources
         for index, source in enumerate(resource_list_to_upload):
             source_name = os.path.basename(source)
@@ -268,7 +272,8 @@ class OTA(Compile):
             # Add source to upload
             upload_path_list.append(source)
         # Upload files / sources
-        return self.ota_webrepl_update_core(device, upload_path_list=upload_path_list, ota_password=webrepl_password)
+        return self.ota_webrepl_update_core(device, upload_path_list=upload_path_list,
+                                            ota_password=webrepl_password, upload_root_dir=self.precompiled_micrOS_dir_path)
 
     def _enable_micros_ota_update_via_webrepl(self, device=None, ota_password=None):
         # Get specific device from device list
@@ -401,7 +406,8 @@ class OTA(Compile):
             print(f"[SIM] 'OTA' COPY FILES... {source} -> {target}")
             LocalMachine.FileHandler().copy(source, target)
 
-    def ota_webrepl_update_core(self, device=None, upload_path_list=[], ota_password='ADmin123', force_lm=False):
+    def ota_webrepl_update_core(self, device=None, upload_path_list=None, ota_password='ADmin123',
+                                force_lm=False, upload_root_dir=None):
         """
         Generic file uploader for micrOS - over webrepl
             info: https://techoverflow.net/2020/02/22/how-to-upload-files-to-micropython-using-webrepl-using-webrepl_cli-py/
@@ -410,7 +416,11 @@ class OTA(Compile):
         upload_path_list: file path list to upload
         ota_password - accessing webrepl to upload files
         force_lm - use prefix as 'LM_' for every file - for user file upload / GUI drag n drop
+        upload_root_dir - root directory to upload files (subdir support)
         """
+        if upload_path_list is None:
+            upload_path_list = []
+
         if device[0] == "__simulator__":
             OTA.sim_ota_update(upload_path_list, force_lm)
             return
@@ -457,7 +467,18 @@ class OTA(Compile):
             # Copy retry mechanism
             exitcode = -1
             source_name = os.path.basename(source)
-            source_name_target = source_name
+            if upload_root_dir is None:
+                # Drag-n-Drop file upload file type check and folder adjustment
+                if check_web_extensions(source_name):
+                    source_name_target = os.path.join('web', source_name)
+                elif not check_python_extensions(source_name) and not source_name.endswith("node_config.json"):
+                    source_name_target = os.path.join('data', source_name)
+                else:
+                    # Copy file to micrOS root folder (.mpy and .py or node_config.json)
+                    source_name_target = source_name
+            else:
+                # MAIN USE-CASE
+                source_name_target = source.replace(upload_root_dir, '')
 
             # Force LM update - user load modules - drag n drop files
             if force_lm and not source_name.startswith('LM_') and source_name.endswith('.py'):

toolkit/DevEnvUSB.py

@@ -13,12 +13,14 @@ try:
     from .lib import LocalMachine
     from .lib.TerminalColors import Colors
     from .lib.SerialDriverHandler import install_usb_serial_driver
+    from .MicrosFiles import micros_resource_list
 except Exception as e:
     print("Import warning __name__:{}: {}".format(__name__, e))
     from DevEnvCompile import Compile
     from lib import LocalMachine
     from lib.TerminalColors import Colors
     from lib.SerialDriverHandler import install_usb_serial_driver
+    from MicrosFiles import micros_resource_list
 
 
 class USB(Compile):
@@ -207,6 +209,38 @@ class USB(Compile):
             self.console("Deployment failed.\n{} - {}".format(stdout, stderr), state='err')
             return False
 
+    def _mkdir_on_dev(self, folders:list):
+        mpremote_cmd = self.dev_types_and_cmds[self.selected_device_type]['mpremote_cmd']
+        device = self.get_devices()[0]
+        if mpremote_cmd is None:
+            # Legacy ampy command (esp32 auto reboot tolerance...)
+            mkdir_cmd = self.dev_types_and_cmds[self.selected_device_type]['ampy_cmd'].format(dev=device, args=f'mkdir')
+        else:
+            mkdir_cmd = f'{mpremote_cmd} fs mkdir'
+
+        status = 0
+        for folder in folders:
+            _mkdir_cmd = f"{mkdir_cmd} {folder}"
+            self.console(f"Create directory on device: {_mkdir_cmd}")
+            if self.dry_run:
+                pass
+            else:
+                try:
+                    exitcode, stdout, stderr = LocalMachine.CommandHandler.run_command(_mkdir_cmd, shell=True)
+                    if exitcode != 0:
+                        verdict = stdout + stderr
+                        if "File exists" in verdict or "Directory already exists" in verdict:
+                            exitcode = 0
+                        else:
+                            self.console(f"MKDIR ERROR:\n{stdout}\n{stderr}", state="err")
+                except Exception as e:
+                    self.console(f"MKDIR ERROR {_mkdir_cmd}: {e}", state="err")
+                    exitcode = 1
+                status += exitcode
+
+        return True if status == 0 else False
+
+
     def put_micros_to_dev(self):
         self.select_board_n_micropython()
         status = True
@@ -223,17 +257,20 @@ class USB(Compile):
                 self.console(f"... wait for reset {10-k} sec", state='imp')
                 time.sleep(1)
 
-        mpremote_cmd = self.dev_types_and_cmds[self.selected_device_type]['mpremote_cmd']
-        device = self.get_devices()[0]
-        source_to_put_device = LocalMachine.FileHandler.list_dir(self.precompiled_micrOS_dir_path)
+        # Parse micrOS resources with folders
+        _source_to_put_device, dir_list_to_create = micros_resource_list(self.precompiled_micrOS_dir_path)
+        self.console(f"CREATE FOLDERS: {dir_list_to_create}", state="ok")
+        # Create sub folders
+        if not self._mkdir_on_dev(dir_list_to_create):
+            self.console(f"Error creating directories on device: {dir_list_to_create}")
+            sys.exit(1)
+        # Generate resource list to be put on the device
+        source_to_put_device = list([s.replace(self.precompiled_micrOS_dir_path + "/", '') for s in _source_to_put_device])
         # Set source order - main, boot
-        source_to_put_device.append(source_to_put_device.pop(source_to_put_device.index('main.py')))
-        try:
-            # PIP deployment generates this ...
-            source_to_put_device.remove('__pycache__')      # remove if accidentally left here
-        except:
-            pass
+        source_to_put_device.append(source_to_put_device.pop(source_to_put_device.index("main.py")))
 
+        mpremote_cmd = self.dev_types_and_cmds[self.selected_device_type]['mpremote_cmd']
+        device = self.get_devices()[0]
         # Change workdir
         workdir_handler = LocalMachine.SimplePopPushd()
         workdir_handler.pushd(self.precompiled_micrOS_dir_path)
@@ -243,7 +280,7 @@ class USB(Compile):
             self.console("[{}%] micrOS deploy via USB - {}".format(percent, device))
             if mpremote_cmd is None:
                 # Legacy ampy command (esp32 auto reboot tolerance...)
-                command = self.dev_types_and_cmds[self.selected_device_type]['ampy_cmd'].format(dev=device, args=f'put {source}')
+                command = self.dev_types_and_cmds[self.selected_device_type]['ampy_cmd'].format(dev=device, args=f'put {source} /{source}')
             else:
                 command = f'{mpremote_cmd} fs cp {source} :{source}'     # new mpremote <1.24.1
             if ' ' in source:

toolkit/MicrOSDevEnv.py

@@ -62,8 +62,16 @@ class MicrOSDevTool(OTA, USB):
                 if f.endswith('.json'):
                     continue
                 f_path = os.path.join(self.micrOS_dir_path, f)
-                self.console("[SIM] Copy micrOS resources: {} -> {}".format(f_path, self.micros_sim_workspace))
-                LocalMachine.FileHandler().copy(f_path, self.micros_sim_workspace)
+                if f.startswith("_") or f.startswith("."):
+                    # SKIP files startswith `_` and `.`
+                    continue
+                _, f_type = LocalMachine.FileHandler.path_is_exists(f_path)
+                target_dir = self.micros_sim_workspace
+                if f_type == "d":
+                    target_dir = os.path.join(self.micros_sim_workspace, f)
+                self.console(f"[SIM] Copy micrOS resources: {f_path} -> {target_dir}")
+                if not LocalMachine.FileHandler().copy(f_path, target_dir):
+                    self.console(f"[ERROR] Failed to copy: {f_path}")
 
             if prepare_only:
                 # In case of automatic node_conf creation

toolkit/MicrosFiles.py

@@ -0,0 +1,26 @@
+import os
+
+try:
+    from .lib import LocalMachine
+    from .lib.file_extensions import check_all_extensions, check_web_extensions
+except Exception as e:
+    print("Import warning __name__:{}: {}".format(__name__, e))
+    from lib import LocalMachine
+    from lib.file_extensions import check_all_extensions, check_web_extensions
+
+def micros_resource_list(root_folder):
+    resources_path = []
+    subfolders = []
+    for source in LocalMachine.FileHandler.list_dir(root_folder):
+        source_full_path = os.path.join(root_folder, source)
+        # [1] / Root directory source files and folders.
+        if check_all_extensions(source):
+            resources_path.append(source_full_path)
+        # [2] /dir Handle sub dictionary sources
+        elif LocalMachine.FileHandler.path_is_exists(source_full_path)[1] == 'd':
+            subfolders.append(source)
+            for sub_source in LocalMachine.FileHandler.list_dir(source_full_path):
+                sub_source_full_path = os.path.join(source_full_path, sub_source)
+                if check_all_extensions(sub_source):
+                    resources_path.append(sub_source_full_path)
+    return resources_path, subfolders

toolkit/lib/LocalMachine.py

@@ -186,11 +186,13 @@ class FileHandler:
                     os.remove(path)
                 elif type_ == 'd':
                     shutil.rmtree(path)
+                return True
         except Exception as e:
             if ignore:
                 debug_print("[DEBUG] Removing " + path + " is forced to ignore failure: " + str(e))
             else:
                 raise Exception("Cannot remove " + path + ": " + str(e))
+        return False
 
     @staticmethod
     def extract_tar(targz, extract_path):
@@ -210,7 +212,10 @@ class FileHandler:
     def copy(from_path, to_path):
         debug_print("[DEBUG] Copy " + from_path + " to " + to_path)
         try:
-            shutil.copy(from_path, to_path)
+            if os.path.isdir(from_path):
+                shutil.copytree(from_path, to_path, dirs_exist_ok=True)
+            else:
+                shutil.copy(from_path, to_path)
             return True
         except Exception as e:
             debug_print("Copy error: {}".format(e))

toolkit/lib/file_extensions.py

@@ -1,7 +1,7 @@
 
-
-ENABLED_EXTENSIONS = ('py', 'mpy', 'js', 'css', 'html')
-WEB_ONLY = ('js', 'html', 'css')
+PYTHON_EXTENSIONS = ('py', 'mpy')
+WEB_ONLY = ('js', 'html', 'css', 'json', 'ico', 'jpeg', 'png')
+ENABLED_EXTENSIONS = PYTHON_EXTENSIONS + WEB_ONLY
 
 def check_all_extensions(path):
     extension = path.split('.')[-1]
@@ -13,4 +13,10 @@ def check_web_extensions(path):
     extension = path.split('.')[-1]
     if extension in WEB_ONLY:
         return True
+    return False
+
+def check_python_extensions(path):
+    extension = path.split('.')[-1]
+    if extension in PYTHON_EXTENSIONS:
+        return True
     return False

toolkit/lib/pip_package_installer.py

@@ -66,8 +66,11 @@ def install_package(name, additional_pip_param=None):
             print(f"{TerminalColors.Colors.OK}[PIP] install {name} OK{TerminalColors.Colors.NC}")
         else:
             print(f"{TerminalColors.Colors.WARN}[PIP] install {name} NOK{TerminalColors.Colors.NC}")
-            # Support brew installed python
-            if "This environment is externally managed" in out:
+            if "--break-system-packages" in out:
+                # (Retry) Support system wide package install
+                install_package(name, additional_pip_param="--break-system-packages")
+            elif "This environment is externally managed" in out:
+                # (Retry-Fallback) Support brew installed python
                 if "brew install" in out:
                     print(f"Install optional dependency (brew): {name}")
                     state, out = run_subprocess(["brew", "install", name])

toolkit/micrOSlint.py

@@ -222,7 +222,9 @@ def _run_pylint(file_name):
         '--disable=broad-exception-caught',         # Disable BROAD exception
         '--disable=broad-exception-raised',         # Disable BROAD exception
         '--disable=too-many-return-statements',     # :D I don't think so :D
-        '--disable=too-many-branches'               # :D I don't think so :D
+        '--disable=too-many-branches',              # :D I don't think so :D
+        '--disable=too-many-positional-arguments',  # :D I don't think so :D
+        '--disable=too-many-instance-attributes'    # :D I don't think so :D
     ]
     if file_name in ['Tasks.py', 'microIO.py', 'Types.py']:
         pylint_opts.append('--disable=exec-used')   # Disable micrOS execution core exec/eval warning

toolkit/workspace/precompiled/LM_dashboard_be.py

@@ -1,4 +1,4 @@
-from Common import web_endpoint, syslog
+from Common import web_dir, web_endpoint, syslog
 
 ENDPOINT_INITED = False
 
@@ -8,7 +8,7 @@ def load():
 
 def _dashboard_clb():
     try:
-        with open('dashboard.html', 'r') as html:
+        with open(web_dir('dashboard.html'), 'r') as html:
             html_content = html.read()
         return 'text/html', html_content
     except Exception as e: