mpytool 1.1.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

mpytool/__init__.py

@@ -2,5 +2,7 @@
 
 from mpytool.conn import ConnError, Timeout
 from mpytool.conn_serial import ConnSerial
+from mpytool.conn_socket import ConnSocket
 from mpytool.mpy_comm import MpyError, CmdError
 from mpytool.mpy import Mpy
+from mpytool.logger import SimpleColorLogger

mpytool/conn.py

@@ -1,5 +1,8 @@
 """MicroPython tool: abstract connector"""
 
+import time as _time
+import select as _select
+
 
 class ConnError(Exception):
     """General connection error"""
@@ -12,16 +15,86 @@ class Timeout(ConnError):
 class Conn():
     def __init__(self, log=None):
         self._log = log
+        self._buffer = bytearray(b'')
+
+    @property
+    def fd(self):
+        """Return file descriptor for select()"""
+        return None
+
+    def _has_data(self, timeout=0):
+        """Check if data is available to read using select()"""
+        fd = self.fd
+        if fd is None:
+            return False
+        readable, _, _ = _select.select([fd], [], [], timeout)
+        return bool(readable)
+
+    def _read_available(self):
+        """Read available data from device (must be implemented by subclass)"""
+        raise NotImplementedError
+
+    def _write_raw(self, data):
+        """Write data to device, return bytes written (must be implemented by subclass)"""
+        raise NotImplementedError
 
-    def write(self, data, chunk_size=128, delay=0.01):
-        """Write to device
+    def _read_to_buffer(self, wait_timeout=0):
+        """Read available data into buffer
+
+        Arguments:
+            wait_timeout: how long to wait for data (0 = non-blocking)
         """
+        if self._has_data(wait_timeout):
+            data = self._read_available()
+            if data:
+                self._buffer += data
+                return True
+        return False
+
+    def flush(self):
+        """Flush and return buffer contents"""
+        buffer = bytes(self._buffer)
+        del self._buffer[:]
+        return buffer
+
+    def read(self):
+        """Read available data from device"""
+        if self._has_data():
+            return self._read_available()
+        return None
+
+    def write(self, data):
+        """Write data to device"""
+        if self._log:
+            self._log.debug("wr: %s", bytes(data))
+        while data:
+            count = self._write_raw(data)
+            data = data[count:]
 
     def read_until(self, end, timeout=1):
-        """Read until
-        """
+        """Read until end marker is found"""
+        if self._log:
+            self._log.debug("wait for %s", end)
+        start_time = _time.time()
+        while True:
+            # Use select() with 1ms timeout instead of sleep - wakes immediately on data
+            if self._read_to_buffer(wait_timeout=0.001):
+                start_time = _time.time()  # reset timeout on data received
+            if end in self._buffer:
+                break
+            if timeout is not None and start_time + timeout < _time.time():
+                if self._buffer:
+                    raise Timeout(
+                        f"During timeout received: {bytes(self._buffer)}")
+                raise Timeout("No data received")
+        index = self._buffer.index(end)
+        data = self._buffer[:index]
+        del self._buffer[:index + len(end)]
+        if self._log:
+            self._log.debug("rd: %s", bytes(data + end))
+        return data
 
     def read_line(self, timeout=None):
-        """Read signle line"""
+        """Read single line"""
         line = self.read_until(b'\n', timeout)
         return line.strip(b'\r')

mpytool/conn_serial.py

@@ -1,6 +1,5 @@
 """MicroPython tool: serial connector"""
 
-import time as _time
 import serial as _serial
 import mpytool.conn as _conn
 
@@ -8,42 +7,28 @@ import mpytool.conn as _conn
 class ConnSerial(_conn.Conn):
     def __init__(self, log=None, **serial_config):
         super().__init__(log)
-        self._serial = _serial.Serial(**serial_config)
-        self._buffer = b''
+        try:
+            self._serial = _serial.Serial(**serial_config)
+        except _serial.serialutil.SerialException as err:
+            self._serial = None
+            raise _conn.ConnError(
+                f"Error opening serial port {serial_config['port']}") from err
 
-    def _read_to_buffer(self):
+    def __del__(self):
+        if self._serial:
+            self._serial.close()
+
+    @property
+    def fd(self):
+        return self._serial.fd if self._serial else None
+
+    def _read_available(self):
+        """Read available data from serial port"""
         in_waiting = self._serial.in_waiting
         if in_waiting > 0:
-            self._buffer += self._serial.read(in_waiting)
-            return True
-        return False
-
-    def write(self, data, chunk_size=128, delay=0.01):
-        if self._log:
-            self._log.debug(f"wr: {data}")
-        while data:
-            chunk = data[:chunk_size]
-            count = self._serial.write(chunk)
-            data = data[count:]
-            _time.sleep(delay)
+            return self._serial.read(in_waiting)
+        return None
 
-    def read_until(self, end, timeout=1):
-        if self._log:
-            self._log.debug(f'wait for {end}')
-        start_time = _time.time()
-        while True:
-            if self._read_to_buffer():
-                start_time = _time.time()
-            if end in self._buffer:
-                break
-            if timeout is not None and start_time + timeout < _time.time():
-                if self._buffer:
-                    raise _conn.Timeout(
-                        f"During timeout received: {bytes(self._buffer)}")
-                raise _conn.Timeout("No data received")
-            _time.sleep(.01)
-        data, self._buffer = self._buffer.split(end, 1)
-        if self._log:
-            self._log.debug(f"rd: {data + end}")
-        data = data.rstrip(end)
-        return data
+    def _write_raw(self, data):
+        """Write data to serial port"""
+        return self._serial.write(data)

mpytool/conn_socket.py

@@ -0,0 +1,53 @@
+"""MicroPython tool: socket connector"""
+
+import socket as _socket
+import mpytool.conn as _conn
+
+
+class ConnSocket(_conn.Conn):
+    def __init__(self, address, log=None):
+        super().__init__(log)
+        self._socket = None
+        sock = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
+        sock.settimeout(5)
+        if ':' in address:
+            host, port = address.split(':')
+            port = int(port)
+        else:
+            host = address
+            port = 23
+        if log:
+            log.info(f"Connecting to: {host}:{port}")
+        try:
+            sock.connect((host, port))
+        except _socket.timeout as err:
+            raise _conn.ConnError(f"Timeout connecting to {host}:{port}") from err
+        except _socket.error as err:
+            raise _conn.ConnError(f"Cannot connect to {host}:{port}: {err}") from err
+        sock.settimeout(None)
+        sock.setblocking(False)
+        self._socket = sock
+        if log:
+            log.info("connected")
+
+    def __del__(self):
+        if self._socket:
+            self._socket.close()
+
+    @property
+    def fd(self):
+        return self._socket.fileno() if self._socket else None
+
+    def _read_available(self):
+        """Read available data from socket"""
+        try:
+            data = self._socket.recv(4096)
+            if data:
+                return data
+        except BlockingIOError:
+            pass
+        return None
+
+    def _write_raw(self, data):
+        """Write data to socket"""
+        return self._socket.send(data)

mpytool/logger.py

@@ -0,0 +1,87 @@
+"""Simple color logger for mpytool"""
+
+import os as _os
+import sys as _sys
+
+
+class SimpleColorLogger():
+    # ANSI color codes
+    _RESET = '\033[0m'
+    _BOLD_RED = '\033[1;31m'
+    _BOLD_YELLOW = '\033[1;33m'
+    _BOLD_MAGENTA = '\033[1;35m'
+    _BOLD_BLUE = '\033[1;34m'
+    _BOLD_GREEN = '\033[1;32m'
+    _BOLD_CYAN = '\033[1;36m'
+    _CLEAR_LINE = '\033[K'
+
+    # Color names for verbose()
+    COLORS = {
+        'red': _BOLD_RED,
+        'yellow': _BOLD_YELLOW,
+        'magenta': _BOLD_MAGENTA,
+        'blue': _BOLD_BLUE,
+        'green': _BOLD_GREEN,
+        'cyan': _BOLD_CYAN,
+    }
+
+    def __init__(self, loglevel=1, verbose_level=0):
+        self._loglevel = loglevel
+        self._verbose_level = verbose_level
+        self._is_tty = (
+            _sys.stderr.isatty()
+            and _os.environ.get('NO_COLOR') is None
+            and _os.environ.get('TERM') != 'dumb'
+            and _os.environ.get('CI') is None
+        )
+
+    def log(self, msg):
+        print(msg, file=_sys.stderr)
+
+    def error(self, msg, *args):
+        if args:
+            msg = msg % args
+        if self._loglevel >= 1:
+            if self._is_tty:
+                self.log(f"{self._BOLD_RED}{msg}{self._RESET}")
+            else:
+                self.log(f"E: {msg}")
+
+    def warning(self, msg, *args):
+        if args:
+            msg = msg % args
+        if self._loglevel >= 2:
+            if self._is_tty:
+                self.log(f"{self._BOLD_YELLOW}{msg}{self._RESET}")
+            else:
+                self.log(f"W: {msg}")
+
+    def info(self, msg, *args):
+        if args:
+            msg = msg % args
+        if self._loglevel >= 3:
+            if self._is_tty:
+                self.log(f"{self._BOLD_MAGENTA}{msg}{self._RESET}")
+            else:
+                self.log(f"I: {msg}")
+
+    def debug(self, msg, *args):
+        if args:
+            msg = msg % args
+        if self._loglevel >= 4:
+            if self._is_tty:
+                self.log(f"{self._BOLD_BLUE}{msg}{self._RESET}")
+            else:
+                self.log(f"D: {msg}")
+
+    def verbose(self, msg, level=1, color='green', end='\n', overwrite=False):
+        """Print verbose message if verbose_level >= level"""
+        if self._verbose_level < level:
+            return
+        # Skip progress updates (overwrite without newline) in non-TTY mode
+        if overwrite and not self._is_tty and end != '\n':
+            return
+        color_code = self.COLORS.get(color, self._BOLD_GREEN) if self._is_tty else ''
+        reset_code = self._RESET if self._is_tty else ''
+        clear = f'\r{self._CLEAR_LINE}' if self._is_tty and overwrite else ''
+        print(f"{clear}{color_code}{msg}{reset_code}", end=end, file=_sys.stderr, flush=True)

mpytool/mpy.py

@@ -3,6 +3,11 @@
 import mpytool.mpy_comm as _mpy_comm
 
 
+def _escape_path(path: str) -> str:
+    """Escape path for use in Python string literal"""
+    return path.replace("\\", "\\\\").replace("'", "\\'")
+
+
 class PathNotFound(_mpy_comm.MpyError):
     """File not found"""
     def __init__(self, file_name):
@@ -26,7 +31,7 @@ class DirNotFound(PathNotFound):
 
 
 class Mpy():
-    _CHUNK = 4096
+    _CHUNK = 512
     _ATTR_DIR = 0x4000
     _ATTR_FILE = 0x8000
     _HELPERS = {
@@ -91,6 +96,18 @@ def _mpytool_rmdir(path):
         elif attr == {_ATTR_DIR}:
             _mpytool_rmdir(path + '/' + name)
     os.rmdir(path)
+""",
+        'hashfile': """
+def _mpytool_hashfile(path):
+    import hashlib
+    h = hashlib.sha256()
+    with open(path, 'rb') as f:
+        while True:
+            chunk = f.read(512)
+            if not chunk:
+                break
+            h.update(chunk)
+    return h.digest()
 """}
 
     def __init__(self, conn, log=None):
@@ -112,6 +129,14 @@ def _mpytool_rmdir(path):
         """
         return self._mpy_comm
 
+    def reset_state(self):
+        """Reset internal state after device reset
+
+        Call this after soft_reset() to clear cached helper/import state.
+        """
+        self._imported = []
+        self._load_helpers = []
+
     def load_helper(self, helper):
         """Load helper function to MicroPython
 
@@ -147,7 +172,7 @@ def _mpytool_rmdir(path):
         """
         self.import_module('os')
         self.load_helper('stat')
-        return self._mpy_comm.exec_eval(f"_mpytool_stat('{path}')")
+        return self._mpy_comm.exec_eval(f"_mpytool_stat('{_escape_path(path)}')")
 
     def ls(self, path=None):
         """List files on path
@@ -166,7 +191,7 @@ def _mpytool_rmdir(path):
             path = ''
         try:
             result = self._mpy_comm.exec_eval(
-                f"tuple(os.ilistdir('{path}'))")
+                f"tuple(os.ilistdir('{_escape_path(path)}'))")
             res_dir = []
             res_file = []
             for entry in result:
@@ -199,14 +224,14 @@ def _mpytool_rmdir(path):
         if path is None:
             path = ''
         if path in ('', '.', '/'):
-            return self._mpy_comm.exec_eval(f"_mpytool_tree('{path}')")
+            return self._mpy_comm.exec_eval(f"_mpytool_tree('{_escape_path(path)}')")
         # check if path exists
         result = self.stat(path)
         if result is None:
             raise DirNotFound(path)
         if result == -1:
-            return self._mpy_comm.exec_eval(f"_mpytool_tree('{path}')")
-        return((path, result[6], None))
+            return self._mpy_comm.exec_eval(f"_mpytool_tree('{_escape_path(path)}')")
+        return (path, result, None)
 
     def mkdir(self, path):
         """make directory (also create all parents)
@@ -216,7 +241,7 @@ def _mpytool_rmdir(path):
         """
         self.import_module('os')
         self.load_helper('mkdir')
-        if self._mpy_comm.exec_eval(f"_mpytool_mkdir('{path}')"):
+        if self._mpy_comm.exec_eval(f"_mpytool_mkdir('{_escape_path(path)}')"):
             raise _mpy_comm.MpyError(f'Error creating directory, this is file: {path}')
 
     def delete(self, path):
@@ -231,21 +256,53 @@ def _mpytool_rmdir(path):
         if result == -1:
             self.import_module('os')
             self.load_helper('rmdir')
-            self._mpy_comm.exec(f"_mpytool_rmdir('{path}')")
+            self._mpy_comm.exec(f"_mpytool_rmdir('{_escape_path(path)}')", 20)
         else:
-            self._mpy_comm.exec(f"os.remove('{path}')")
+            self._mpy_comm.exec(f"os.remove('{_escape_path(path)}')")
+
+    def rename(self, src, dst):
+        """Rename/move file or directory
 
-    def get(self, path):
+        Arguments:
+            src: source path
+            dst: destination path
+        """
+        self.import_module('os')
+        self._mpy_comm.exec(f"os.rename('{_escape_path(src)}', '{_escape_path(dst)}')")
+
+    def hashfile(self, path):
+        """Compute SHA256 hash of file
+
+        Arguments:
+            path: file path
+
+        Returns:
+            bytes with SHA256 hash (32 bytes) or None if hashlib not available
+        """
+        self.load_helper('hashfile')
+        try:
+            return self._mpy_comm.exec_eval(f"_mpytool_hashfile('{_escape_path(path)}')")
+        except _mpy_comm.CmdError:
+            return None
+
+    def get(self, path, progress_callback=None):
         """Read file
 
         Arguments:
             path: file path to read
+            progress_callback: optional callback(transferred, total) for progress
 
         Returns:
             bytes with file content
         """
+        # Get file size first if callback provided
+        total_size = 0
+        if progress_callback:
+            total_size = self.stat(path)
+            if total_size is None or total_size < 0:
+                total_size = 0
         try:
-            self._mpy_comm.exec(f"f = open('{path}', 'rb')")
+            self._mpy_comm.exec(f"f = open('{_escape_path(path)}', 'rb')")
         except _mpy_comm.CmdError as err:
             raise FileNotFound(path) from err
         data = b''
@@ -254,19 +311,27 @@ def _mpytool_rmdir(path):
             if not result:
                 break
             data += result
+            if progress_callback:
+                progress_callback(len(data), total_size)
         self._mpy_comm.exec("f.close()")
         return data
 
-    def put(self, data, path):
-        """Read file
+    def put(self, data, path, progress_callback=None):
+        """Write file to device
 
         Arguments:
             data: bytes with file content
             path: file path to write
+            progress_callback: optional callback(transferred, total) for progress
         """
-        self._mpy_comm.exec(f"f = open('{path}', 'wb')")
+        total_size = len(data)
+        transferred = 0
+        self._mpy_comm.exec(f"f = open('{_escape_path(path)}', 'wb')")
         while data:
             chunk = data[:self._CHUNK]
             count = self._mpy_comm.exec_eval(f"f.write({chunk})", timeout=10)
             data = data[count:]
+            transferred += count
+            if progress_callback:
+                progress_callback(transferred, total_size)
         self._mpy_comm.exec("f.close()")

mpytool/mpy_comm.py

@@ -8,7 +8,7 @@ class MpyError(Exception):
 
 
 class CmdError(MpyError):
-    """Timeout"""
+    """Command execution error on device"""
     def __init__(self, cmd, result, error):
         self._cmd = cmd
         self._result = result
@@ -47,24 +47,53 @@ class MpyComm():
         return self._conn
 
     def stop_current_operation(self):
+        """Stop any running operation and get to known REPL state.
+
+        Some USB/serial converters reset the device when port opens.
+        We send multiple Ctrl-C/Ctrl-B with short timeouts to catch the device
+        after reset completes and program starts running.
+
+        Returns:
+            True if we're in a known REPL state, False if recovery failed
+        """
         if self._repl_mode is not None:
-            return
+            return True
         if self._log:
             self._log.info('STOP CURRENT OPERATION')
-        self._conn.write(b'\x03')
-        try:
-            # wait for prompt
-            self._conn.read_until(b'\r\n>>> ', timeout=1)
-        except _conn.Timeout:
-            # probably is in RAW repl
-            if self._log:
-                self._log.warning("Timeout while stopping program")
-            self.exit_raw_repl()
 
-    def enter_raw_repl(self):
+        # Flush any pending data first
+        self._conn.flush()
+
+        # Try multiple attempts with short timeouts
+        # Interleave Ctrl-C (interrupt program) and Ctrl-B (exit raw REPL)
+        # This handles USB/serial converters that reset device on port open
+        for attempt in range(15):
+            # Alternate: Ctrl-C, Ctrl-C, Ctrl-B, repeat
+            if attempt % 3 == 2:
+                self._conn.write(b'\x02')  # Ctrl-B - exit raw REPL
+            else:
+                self._conn.write(b'\x03')  # Ctrl-C - interrupt program
+            try:
+                self._conn.read_until(b'\r\n>>> ', timeout=0.2)
+                self._repl_mode = False
+                return True
+            except _conn.Timeout:
+                pass
+
+        if self._log:
+            self._log.warning("Could not establish REPL state")
+        return False
+
+    def enter_raw_repl(self, max_retries=3):
         if self._repl_mode is True:
             return
-        self.stop_current_operation()
+        retries = 0
+        while not self.stop_current_operation():
+            retries += 1
+            if retries >= max_retries:
+                raise MpyError("Could not establish REPL connection")
+            if self._log:
+                self._log.warning('..retry %d/%d', retries, max_retries)
         if self._log:
             self._log.info('ENTER RAW REPL')
         self._conn.write(b'\x01')
@@ -105,14 +134,14 @@ class MpyComm():
         # wait for prompt
         self.enter_raw_repl()
         if self._log:
-            self._log.info(f"CMD: {command}")
+            self._log.info("CMD: %s", command)
         self._conn.write(bytes(command, 'utf-8'))
         self._conn.write(b'\x04')
         self._conn.read_until(b'OK', timeout)
         result = self._conn.read_until(b'\x04', timeout)
         if result:
             if self._log:
-                self._log.info(f'RES: {result}')
+                self._log.info('RES: %s', bytes(result))
         err = self._conn.read_until(b'\x04>', timeout)
         if err:
             raise CmdError(command, result, err)