mpflash 0.7.7__py3-none-any.whl → 0.8.1__py3-none-any.whl

mpflash/cli_group.py

@@ -3,8 +3,6 @@ Main entry point for the CLI group.
 Additional comands are added in the submodules.
 """
 
-import sys
-
 import rich_click as click
 
 from .config import config
@@ -14,22 +12,17 @@ from .logger import make_quiet, set_loglevel
 def cb_verbose(ctx, param, value):
     """Callback to set the log level to DEBUG if verbose is set"""
     if value and not config.quiet:
-        set_loglevel("DEBUG")
         config.verbose = True
+        if value > 1:
+            set_loglevel("TRACE")
+        else:
+            set_loglevel("DEBUG")
     else:
         set_loglevel("INFO")
         config.verbose = False
     return value
 
 
-def cb_ignore(ctx, param, value):
-    if value:
-        config.ignore_ports = list(value)
-        if sys.platform == "win32":
-            config.ignore_ports = [port.upper() for port in config.ignore_ports]
-    return value
-
-
 def cb_interactive(ctx, param, value):
     if value:
         config.interactive = value
@@ -74,22 +67,10 @@ def cb_quiet(ctx, param, value):
     "-V",
     "--verbose",
     is_eager=True,
-    is_flag=True,
+    count=True,
     help="Enables verbose mode.",
     callback=cb_verbose,
 )
-@click.option(
-    "--ignore",
-    "-i",
-    is_eager=True,
-    help="Serial port(s) to ignore. Defaults to MPFLASH_IGNORE.",
-    callback=cb_ignore,
-    multiple=True,
-    default=[],
-    envvar="MPFLASH_IGNORE",
-    show_default=True,
-    metavar="SERIALPORT",
-)
 @click.option(
     "--test",
     is_eager=True,

mpflash/cli_list.py

@@ -1,11 +1,13 @@
 import json
+from typing import List
 
 import rich_click as click
 from rich import print
 
 from .cli_group import cli
-from .list import list_mcus, show_mcus
+from .list import show_mcus
 from .logger import make_quiet
+from .connected import list_mcus
 
 
 @cli.command("list", help="List the connected MCU boards.")
@@ -18,21 +20,55 @@ from .logger import make_quiet
     show_default=True,
     help="""Output in json format""",
 )
+@click.option(
+    "--serial",
+    "--serial-port",
+    "-s",
+    "serial",
+    default=["*"],
+    multiple=True,
+    show_default=True,
+    help="Which serial port(s) to list. ",
+    metavar="SERIALPORT",
+)
+@click.option(
+    "--ignore",
+    "-i",
+    is_eager=True,
+    help="Serial port(s) to ignore. Defaults to MPFLASH_IGNORE.",
+    multiple=True,
+    default=[],
+    envvar="MPFLASH_IGNORE",
+    show_default=True,
+    metavar="SERIALPORT",
+)
+@click.option(
+    "--bluetooth/--no-bluetooth",
+    "-b/-nb",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="""Include bluetooth ports in the list""",
+)
 @click.option(
     "--progress/--no-progress",
+    # "-p/-np", -p is already used for --port
     "progress",
     is_flag=True,
     default=True,
     show_default=True,
     help="""Show progress""",
 )
-def cli_list_mcus(as_json: bool, progress: bool = True) -> int:
+def cli_list_mcus(serial: List[str], ignore: List[str], bluetooth: bool, as_json: bool, progress: bool = True) -> int:
     """List the connected MCU boards, and output in a nice table or json."""
+    serial = list(serial)
+    ignore = list(ignore)
     if as_json:
         # avoid noise in json output
         make_quiet()
+    # TODO? Ask user to select a serialport if [?] is given ?
 
-    conn_mcus = list_mcus()
+    conn_mcus = list_mcus(ignore=ignore, include=serial, bluetooth=bluetooth)
     if as_json:
         print(json.dumps([mcu.__dict__ for mcu in conn_mcus], indent=4))
         progress = False

mpflash/cli_main.py

@@ -2,6 +2,8 @@
 
 # import rich_click as click
 
+import os
+
 import click
 from loguru import logger as log
 
@@ -12,19 +14,24 @@ from .cli_list import cli_list_mcus
 
 
 def mpflash():
-    cli.add_command(cli_flash_board)
     cli.add_command(cli_list_mcus)
     cli.add_command(cli_download)
+    cli.add_command(cli_flash_board)
+
     # cli(auto_envvar_prefix="MPFLASH")
-    try:
+    if False and os.environ.get("COMPUTERNAME") == "JOSVERL-S4":
+        # intentional less error suppression on dev machine
         result = cli(standalone_mode=False)
-        exit(result)
-    except AttributeError as e:
-        log.error(f"Error: {e}")
-        exit(-1)
-    except click.exceptions.ClickException as e:
-        log.error(f"Error: {e}")
-        exit(-2)
+    else:
+        try:
+            result = cli(standalone_mode=False)
+            exit(result)
+        except AttributeError as e:
+            log.error(f"Error: {e}")
+            exit(-1)
+        except click.exceptions.ClickException as e:
+            log.error(f"Error: {e}")
+            exit(-2)
 
 
 if __name__ == "__main__":

mpflash/common.py

@@ -1,11 +1,16 @@
+import fnmatch
 import os
-import time
-from typing import TypedDict
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional, Union
 
 from github import Auth, Github
-from rich.progress import track
+from serial.tools import list_ports
+from serial.tools.list_ports_common import ListPortInfo
+
+from .logger import log
 
-from mpflash.errors import MPFlashError
 # from mpflash.mpremoteboard import MPRemoteBoard
 
 PORT_FWTYPES = {
@@ -14,6 +19,7 @@ PORT_FWTYPES = {
     "esp8266": [".bin"],
     "rp2": [".uf2"],
     "samd": [".uf2"],
+    # below this not yet implemented / tested
     "mimxrt": [".hex"],
     "nrf": [".uf2"],
     "renesas-ra": [".hex"],
@@ -28,11 +34,118 @@ PAT = os.environ.get("GITHUB_TOKEN") or PAT_NO_ACCESS
 GH_CLIENT = Github(auth=Auth.Token(PAT))
 
 
-class FWInfo(TypedDict):
-    filename: str
-    port: str
-    board: str
-    variant: str
-    preview: bool
-    version: str
-    build: str
+@dataclass
+class FWInfo:
+    """
+    Downloaded Firmware information
+    is somewhat related to the BOARD class in the mpboard_id module
+    """
+
+    port: str  # MicroPython port
+    board: str  # MicroPython board
+    filename: str = field(default="")  # relative filename of the firmware image
+    firmware: str = field(default="")  # url or path to original firmware image
+    variant: str = field(default="")  # MicroPython variant
+    preview: bool = field(default=False)  # True if the firmware is a preview version
+    version: str = field(default="")  # MicroPython version
+    url: str = field(default="")  # url to the firmware image download folder
+    build: str = field(default="0")  # The build = number of commits since the last release
+    ext: str = field(default="")  # the file extension of the firmware
+    family: str = field(default="micropython")  # The family of the firmware
+    custom: bool = field(default=False)  # True if the firmware is a custom build
+    description: str = field(default="")  # Description used by this firmware (custom only)
+
+    def to_dict(self) -> dict:
+        """Convert the object to a dictionary"""
+        return self.__dict__
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "FWInfo":
+        """Create a FWInfo object from a dictionary"""
+        # add missing keys
+        if "ext" not in data:
+            data["ext"] = Path(data["firmware"]).suffix
+        if "family" not in data:
+            data["family"] = "micropython"
+        return cls(**data)
+
+
+@dataclass
+class Params:
+    """Common parameters for downloading and flashing firmware"""
+
+    ports: List[str] = field(default_factory=list)
+    boards: List[str] = field(default_factory=list)
+    versions: List[str] = field(default_factory=list)
+    fw_folder: Path = Path()
+    serial: List[str] = field(default_factory=list)
+    ignore: List[str] = field(default_factory=list)
+
+
+@dataclass
+class DownloadParams(Params):
+    """Parameters for downloading firmware"""
+
+    clean: bool = False
+    force: bool = False
+
+
+@dataclass
+class FlashParams(Params):
+    """Parameters for flashing a board"""
+
+    erase: bool = True
+    bootloader: bool = True
+    cpu: str = ""
+
+
+ParamType = Union[DownloadParams, FlashParams]
+
+
+def filtered_comports(
+    ignore: Optional[List[str]] = None,
+    include: Optional[List[str]] = None,
+    bluetooth: bool = False,
+) -> List[ListPortInfo]:  # sourcery skip: assign-if-exp
+    """
+    Get a list of filtered comports.
+    """
+    if not ignore:
+        ignore = []
+    elif not isinstance(ignore, list):  # type: ignore
+        ignore = list(ignore)
+    if not include:
+        include = ["*"]
+    elif not isinstance(include, list):  # type: ignore
+        include = list(include)
+
+    # remove ports that are to be ignored
+    log.trace(f"{include=}, {ignore=}, {bluetooth=}")
+    comports = [p for p in list_ports.comports() if not any(fnmatch.fnmatch(p.device, i) for i in ignore)]
+    log.trace(f"comports: {[p.device for p in comports]}")
+    # remove bluetooth ports
+
+    if include != ["*"]:
+        # if there are explicit ports to include, add them to the list
+        explicit = [p for p in list_ports.comports() if any(fnmatch.fnmatch(p.device, i) for i in include)]
+        log.trace(f"explicit: {[p.device for p in explicit]}")
+        if ignore == []:
+            # if nothing to ignore, just use the explicit list as a sinple sane default
+            comports = explicit
+        else:
+            # if there are ports to ignore, add the explicit list to the filtered list
+            comports = list(set(explicit) | set(comports))
+    if not bluetooth:
+        # filter out bluetooth ports
+        comports = [p for p in comports if "bluetooth" not in p.description.lower()]
+        comports = [p for p in comports if "BTHENUM" not in p.hwid]
+        if sys.platform == "darwin":
+            comports = [p for p in comports if ".Bluetooth" not in p.device]
+        log.trace(f"no Bluetooth: {[p.device for p in comports]}")
+    log.debug(f"filtered_comports: {[p.device for p in comports]}")
+    # sort
+    if sys.platform == "win32":
+        # Windows sort of comports by number - but fallback to device name
+        return sorted(comports, key=lambda x: int(x.device.split()[0][3:]) if x.device.split()[0][3:].isdigit() else x)
+    # sort by device name
+    return sorted(comports, key=lambda x: x.device)

mpflash/connected.py

@@ -0,0 +1,74 @@
+from typing import List, Tuple
+
+from rich import print
+from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+from rich.table import Column
+
+from mpflash.mpremoteboard import MPRemoteBoard
+
+from .common import filtered_comports
+
+
+def connected_ports_boards(
+    *, include: List[str], ignore: List[str]
+) -> Tuple[List[str], List[str], List[MPRemoteBoard]]:
+    """
+    Returns a tuple containing lists of unique ports and boards from the connected MCUs.
+    Boards that are physically connected, but give no tangible response are ignored.
+
+    Returns:
+        A tuple containing three lists:
+            - A list of unique ports where MCUs are connected.
+            - A list of unique board names of the connected MCUs.
+            - A list of MPRemoteBoard instances of the connected MCUs.
+    """
+    mpr_boards = [b for b in list_mcus(include=include, ignore=ignore) if b.connected]
+    ports = list({b.port for b in mpr_boards})
+    boards = list({b.board for b in mpr_boards})
+    return (ports, boards, mpr_boards)
+
+
+# #########################################################################################################
+rp_spinner = SpinnerColumn(finished_text="✅")
+rp_text = TextColumn("{task.description} {task.fields[device]}", table_column=Column())
+rp_bar = BarColumn(bar_width=None, table_column=Column())
+
+
+def list_mcus(*, ignore: List[str], include: List[str], bluetooth: bool = False):
+    """
+    Retrieves information about connected microcontroller boards.
+
+    Returns:
+        List[MPRemoteBoard]: A list of MPRemoteBoard instances with board information.
+    Raises:
+        ConnectionError: If there is an error connecting to a board.
+    """
+    # conn_mcus = [MPRemoteBoard(sp) for sp in MPRemoteBoard.connected_boards(bluetooth) if sp not in config.ignore_ports]
+
+    comports = filtered_comports(
+        ignore=ignore,
+        include=include,
+        bluetooth=bluetooth,
+    )
+    conn_mcus = [MPRemoteBoard(c.device) for c in comports]
+
+    # a lot of boilerplate to show a progress bar with the comport currently scanned
+    # low update rate to facilitate screen readers/narration
+    with Progress(rp_spinner, rp_text, rp_bar, TimeElapsedColumn(), refresh_per_second=2) as progress:
+        tsk_scan = progress.add_task("[green]Scanning", visible=False, total=None)
+        progress.tasks[tsk_scan].fields["device"] = "..."
+        progress.tasks[tsk_scan].visible = True
+        progress.start_task(tsk_scan)
+        try:
+            for mcu in conn_mcus:
+                progress.update(tsk_scan, device=mcu.serialport.replace("/dev/", ""))
+                try:
+                    mcu.get_mcu_info()
+                except ConnectionError as e:
+                    print(f"Error: {e}")
+                    continue
+        finally:
+            # transient
+            progress.stop_task(tsk_scan)
+            progress.tasks[tsk_scan].visible = False
+    return conn_mcus

mpflash/download.py

@@ -18,10 +18,11 @@ from bs4 import BeautifulSoup
 from loguru import logger as log
 from rich.progress import track
 
-from mpflash.common import PORT_FWTYPES
+from mpflash.common import PORT_FWTYPES, FWInfo
 from mpflash.errors import MPFlashError
 from mpflash.mpboard_id import get_known_ports
 
+# avoid conflict with the ujson used by MicroPython
 jsonlines.ujson = None  # type: ignore
 # #########################################################################################################
 
@@ -31,8 +32,18 @@ MICROPYTHON_ORG_URL = "https://micropython.org/"
 # Regexes to remove dates and hashes in the filename that just get in the way
 RE_DATE = r"(-\d{8}-)"
 RE_HASH = r"(.g[0-9a-f]+\.)"
-# regex to extract the version from the firmware filename
-RE_VERSION_PREVIEW = r"(\d+\.\d+(\.\d+)?(-\w+.\d+)?)"
+# regex to extract the version and the build from the firmware filename
+# group 1 is the version, group 2 is the build
+RE_VERSION_PREVIEW = r"v([\d\.]+)-?(?:preview\.)?(\d+)?\."
+# 'RPI_PICO_W-v1.23.uf2'
+# 'RPI_PICO_W-v1.23.0.uf2'
+# 'RPI_PICO_W-v1.23.0-406.uf2'
+# 'RPI_PICO_W-v1.23.0-preview.406.uf2'
+# 'RPI_PICO_W-v1.23.0-preview.4.uf2'
+# 'RPI_PICO_W-v1.23.0.uf2'
+# 'https://micropython.org/resources/firmware/RPI_PICO_W-20240531-v1.24.0-preview.10.gc1a6b95bf.uf2'
+# 'https://micropython.org/resources/firmware/RPI_PICO_W-20240531-v1.24.0-preview.10.uf2'
+# 'RPI_PICO_W-v1.24.0-preview.10.gc1a6b95bf.uf2'
 
 
 # use functools.lru_cache to avoid needing to download pages multiple times
@@ -51,6 +62,10 @@ def get_board_urls(page_url: str) -> List[Dict[str, str]]:
 
     Args:
         page_url (str): The url of the page to get the board urls from.
+
+    Returns:
+        List[Dict[str, str]]: A list of dictionaries containing the board name and url.
+
     """
     downloads_html = get_page(page_url)
     soup = BeautifulSoup(downloads_html, "html.parser")
@@ -89,14 +104,11 @@ def board_firmware_urls(board_url: str, base_url: str, ext: str) -> List[str]:
     return links
 
 
-# type alias for the firmware info
-FirmwareInfo = Dict[str, str]
-
-
 # boards we are interested in ( this avoids getting a lot of boards we don't care about)
 # The first run takes ~60 seconds to run for 4 ports , all boards
 # so it makes sense to cache the results and skip boards as soon as possible
-def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FirmwareInfo]:
+def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FWInfo]:
+    # sourcery skip: use-getitem-for-re-match-groups
     """
     Retrieves a list of firmware information for the specified ports and boards.
 
@@ -106,59 +118,70 @@ def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[Firmwar
         clean (bool): A flag indicating whether to perform a clean retrieval.
 
     Returns:
-        List[FirmwareInfo]: A list of firmware information for the specified ports and boards.
+        List[FWInfo]: A list of firmware information for the specified ports and boards.
 
     """
-    board_urls: List[FirmwareInfo] = []
+    board_urls: List[FWInfo] = []
     if ports is None:
         ports = get_known_ports()
     for port in ports:
         download_page_url = f"{MICROPYTHON_ORG_URL}download/?port={port}"
-        _urls = get_board_urls(download_page_url)
+        urls = get_board_urls(download_page_url)
         # filter out boards we don't care about
-        _urls = [board for board in _urls if board["board"] in boards]
+        urls = [board for board in urls if board["board"] in boards]
         # add the port to the board urls
-        for board in _urls:
+        for board in urls:
             board["port"] = port
 
-        for board in track(_urls, description=f"Checking {port} download pages", transient=True,refresh_per_second=2):
+        for board in track(urls, description=f"Checking {port} download pages", transient=True, refresh_per_second=2):
             # add a board to the list for each firmware found
-            firmwares = []
+            firmware_urls: List[str] = []
             for ext in PORT_FWTYPES[port]:
-                firmwares += board_firmware_urls(board["url"], MICROPYTHON_ORG_URL, ext)
-
-            for _url in firmwares:
+                firmware_urls += board_firmware_urls(board["url"], MICROPYTHON_ORG_URL, ext)
+            for _url in firmware_urls:
                 board["firmware"] = _url
-                board["preview"] = "preview" in _url  # type: ignore
-                if ver_match := re.search(RE_VERSION_PREVIEW, _url):
-                    board["version"] = ver_match[1]
-                else:
-                    board["version"] = ""
-                if "preview." in board["version"]:
-                    board["build"] = board["version"].split("preview.")[-1]
-                else:
-                    board["build"] = "0"
                 fname = Path(board["firmware"]).name
                 if clean:
                     # remove date from firmware name
                     fname = re.sub(RE_DATE, "-", fname)
                     # remove hash from firmware name
                     fname = re.sub(RE_HASH, ".", fname)
-                board["filename"] = fname
-                board["ext"] = Path(board["firmware"]).suffix
-                board["variant"] = board["filename"].split("-v")[0] if "-v" in board["filename"] else ""
-                board_urls.append(board.copy())
+                fw_info = FWInfo(
+                    filename=fname,
+                    port=port,
+                    board=board["board"],
+                    preview="preview" in _url,
+                    firmware=_url,
+                    version="",
+                )
+                # board["firmware"] = _url
+                # board["preview"] = "preview" in _url  # type: ignore
+                if ver_match := re.search(RE_VERSION_PREVIEW, _url):
+                    fw_info.version = ver_match.group(1)
+                    fw_info.build = ver_match.group(2) or "0"
+                fw_info.preview = fw_info.build != "0"
+                # # else:
+                # #     board.$1= ""
+                # if "preview." in fw_info.version:
+                #     fw_info.build = fw_info.version.split("preview.")[-1]
+                # else:
+                #     fw_info.build = "0"
+
+                fw_info.ext = Path(fw_info.firmware).suffix
+                fw_info.variant = fw_info.filename.split("-v")[0] if "-v" in fw_info.filename else ""
+
+                board_urls.append(fw_info)
     return board_urls
 
 
-def key_fw_ver_pre_ext_bld(x: FirmwareInfo):
+def key_fw_ver_pre_ext_bld(x: FWInfo):
     "sorting key for the retrieved board urls"
-    return x["variant"], x["version"], x["preview"], x["ext"], x["build"]
+    return x.variant, x.version, x.preview, x.ext, x.build
 
 
-def key_fw_var_pre_ext(x: FirmwareInfo):
+def key_fw_var_pre_ext(x: FWInfo):
     "Grouping key for the retrieved board urls"
-    return x["variant"], x["preview"], x["ext"]
+    return x.variant, x.preview, x.ext
 
 
 def download_firmwares(
@@ -177,32 +200,37 @@ def download_firmwares(
     unique_boards = get_firmware_list(ports, boards, versions, clean)
 
     for b in unique_boards:
-        log.debug(b["filename"])
+        log.debug(b.filename)
     # relevant
 
     log.info(f"Found {len(unique_boards)} relevant unique firmwares")
+    if not unique_boards:
+        log.error("No relevant firmwares could be found on https://micropython.org/download")
+        log.info(f"{versions=} {ports=} {boards=}")
+        log.info("Please check the website for the latest firmware files or try the preview version.")
+        return 0
 
     firmware_folder.mkdir(exist_ok=True)
 
     with jsonlines.open(firmware_folder / "firmware.jsonl", "a") as writer:
         for board in unique_boards:
-            filename = firmware_folder / board["port"] / board["filename"]
+            filename = firmware_folder / board.port / board.filename
             filename.parent.mkdir(exist_ok=True)
             if filename.exists() and not force:
                 skipped += 1
                 log.debug(f" {filename} already exists, skip download")
                 continue
-            log.info(f"Downloading {board['firmware']}")
+            log.info(f"Downloading {board.firmware}")
             log.info(f"         to {filename}")
             try:
-                r = requests.get(board["firmware"], allow_redirects=True)
+                r = requests.get(board.firmware, allow_redirects=True)
                 with open(filename, "wb") as fw:
                     fw.write(r.content)
-                board["filename"] = str(filename.relative_to(firmware_folder))
+                board.filename = str(filename.relative_to(firmware_folder))
             except requests.RequestException as e:
                 log.exception(e)
                 continue
-            writer.write(board)
+            writer.write(board.to_dict())
             downloaded += 1
     log.info(f"Downloaded {downloaded} firmwares, skipped {skipped} existing files.")
     return downloaded + skipped
@@ -219,7 +247,7 @@ def get_firmware_list(ports: List[str], boards: List[str], versions: List[str],
         clean : A flag indicating whether to perform a clean check.
 
     Returns:
-        List[FirmwareInfo]: A list of unique firmware information.
+        List[FWInfo]: A list of unique firmware information.
 
     """
 
@@ -228,15 +256,18 @@ def get_firmware_list(ports: List[str], boards: List[str], versions: List[str],
     board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
 
     log.debug(f"Total {len(board_urls)} firmwares")
+
     relevant = [
         board
         for board in board_urls
-        if board["board"] in boards and (board["version"] in versions or board["preview"] and preview)
-        # and b["port"] in ["esp32", "rp2"]
+        if board.version in versions and board.build == "0" and board.board in boards and not board.preview
     ]
+
+    if preview:
+        relevant.extend([board for board in board_urls if board.board in boards and board.preview])
     log.debug(f"Matching firmwares: {len(relevant)}")
     # select the unique boards
-    unique_boards: List[FirmwareInfo] = []
+    unique_boards: List[FWInfo] = []
     for _, g in itertools.groupby(relevant, key=key_fw_var_pre_ext):
         # list is aleady sorted by build so we can just get the last item
         sub_list = list(g)
@@ -280,5 +311,10 @@ def download(
     except (PermissionError, FileNotFoundError) as e:
         log.critical(f"Could not create folder {destination}")
         raise MPFlashError(f"Could not create folder {destination}") from e
+    try:
+        result = download_firmwares(destination, ports, boards, versions, force=force, clean=clean)
+    except requests.exceptions.RequestException as e:
+        log.exception(e)
+        raise MPFlashError("Could not connect to micropython.org") from e
 
-    return download_firmwares(destination, ports, boards, versions, force=force, clean=clean)
+    return result