micropython-stubber 1.17.6__py3-none-any.whl → 1.20.0__py3-none-any.whl

mpflash/mpflash/{downloader.py → download.py}

@@ -5,20 +5,24 @@ Uses the micropython.org website to get the available versions and locations to
 
 import functools
 import itertools
-import json
 import re
 from pathlib import Path
 from typing import Dict, List, Optional
 from urllib.parse import urljoin
 
+# #########################################################################################################
+# make sure that jsonlines does not mistake the MicroPython ujson for the CPython ujson
+import jsonlines
 import requests
-import rich_click as click
 from bs4 import BeautifulSoup
 from loguru import logger as log
 from rich.progress import track
 
-from .cli_group import cli
-from .common import DEFAULT_FW_PATH, PORT_FWTYPES, clean_version
+from mpflash.common import PORT_FWTYPES
+
+jsonlines.ujson = None  # type: ignore
+# #########################################################################################################
+
 
 MICROPYTHON_ORG_URL = "https://micropython.org/"
 
@@ -28,21 +32,6 @@ RE_HASH = r"(.g[0-9a-f]+\.)"
 # regex to extract the version from the firmware filename
 RE_VERSION_PREVIEW = r"(\d+\.\d+(\.\d+)?(-\w+.\d+)?)"
 
-# boards we are interested in ( this avoids getting a lot of boards that are not relevant)
-DEFAULT_BOARDS = [
-    "PYBV11",
-    "ESP8266_GENERIC",
-    "ESP32_GENERIC",
-    "ESP32_GENERIC_S3",
-    "RPI_PICO",
-    "RPI_PICO_W",
-    "ADAFRUIT_QTPY_RP2040",
-    "ARDUINO_NANO_RP2040_CONNECT",
-    "PIMORONI_PICOLIPO_16MB",
-    "SEEED_WIO_TERMINAL",
-    "PARTICLE_XENON",
-]
-
 
 # use functools.lru_cache to avoid needing to download pages multiple times
 @functools.lru_cache(maxsize=500)
@@ -57,6 +46,9 @@ def get_board_urls(page_url: str) -> List[Dict[str, str]]:
     """
     Get the urls to all the board pages listed on this page.
     Assumes that all links to firmware  have "class": "board-card"
+
+    Args:
+        page_url (str): The url of the page to get the board urls from.
     """
     downloads_html = get_page(page_url)
     soup = BeautifulSoup(downloads_html, "html.parser")
@@ -68,8 +60,17 @@ def get_board_urls(page_url: str) -> List[Dict[str, str]]:
     return [{"board": board, "url": page_url + board} for board in boards]
 
 
-def firmware_list(board_url: str, base_url: str, ext: str) -> List[str]:
-    """Get the urls to all the firmware files for a board."""
+def board_firmware_urls(board_url: str, base_url: str, ext: str) -> List[str]:
+    """
+    Get the urls to all the firmware files for a board.
+    Args:
+        page_url (str): The url of the page to get the board urls from.
+    ??? base_url (str): The base url to join the relative urls to.
+        ext (str): The extension of the firmware files to get. (with or withouth leading .)
+
+    the urls are relative urls to the site root
+
+    """
     html = get_page(board_url)
     soup = BeautifulSoup(html, "html.parser")
     # get all the a tags:
@@ -93,20 +94,35 @@ FirmwareInfo = Dict[str, str]
 # boards we are interested in ( this avoids getting a lot of boards we don't care about)
 # The first run takes ~60 seconds to run for 4 ports , all boards
 # so it makes sense to cache the results and skip boards as soon as possible
-def get_boards(fw_types: Dict[str, str], board_list: List[str], clean: bool) -> List[FirmwareInfo]:
+def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FirmwareInfo]:
+    """
+    Retrieves a list of firmware information for the specified ports and boards.
+
+    Args:
+        ports (List[str]): The list of ports to check for firmware.
+        boards (List[str]): The list of boards to retrieve firmware information for.
+        clean (bool): A flag indicating whether to perform a clean retrieval.
+
+    Returns:
+        List[FirmwareInfo]: A list of firmware information for the specified ports and boards.
+
+    """
     board_urls: List[FirmwareInfo] = []
-    for port in fw_types:
+    for port in ports:
         download_page_url = f"{MICROPYTHON_ORG_URL}download/?port={port}"
         _urls = get_board_urls(download_page_url)
         # filter out boards we don't care about
-        _urls = [board for board in _urls if board["board"] in board_list]
+        _urls = [board for board in _urls if board["board"] in boards]
         # add the port to the board urls
         for board in _urls:
             board["port"] = port
 
         for board in track(_urls, description=f"Checking {port} download pages", transient=True):
             # add a board to the list for each firmware found
-            firmwares = firmware_list(board["url"], MICROPYTHON_ORG_URL, fw_types[port])
+            firmwares = []
+            for ext in PORT_FWTYPES[port]:
+                firmwares += board_firmware_urls(board["url"], MICROPYTHON_ORG_URL, ext)
+
             for _url in firmwares:
                 board["firmware"] = _url
                 board["preview"] = "preview" in _url  # type: ignore
@@ -125,34 +141,35 @@ def get_boards(fw_types: Dict[str, str], board_list: List[str], clean: bool) ->
                     # remove hash from firmware name
                     fname = re.sub(RE_HASH, ".", fname)
                 board["filename"] = fname
+                board["ext"] = Path(board["firmware"]).suffix
                 board["variant"] = board["filename"].split("-v")[0] if "-v" in board["filename"] else ""
                 board_urls.append(board.copy())
     return board_urls
 
 
-def key_fw_variant_ver(x: FirmwareInfo):
+def key_fw_ver_pre_ext_bld(x: FirmwareInfo):
     "sorting key for the retrieved board urls"
-    return x["variant"], x["version"], x["preview"], x["build"]
+    return x["variant"], x["version"], x["preview"], x["ext"], x["build"]
 
 
-def key_fw_variant(x: FirmwareInfo):
+def key_fw_var_pre_ext(x: FirmwareInfo):
     "Grouping key for the retrieved board urls"
-    return x["variant"], x["preview"]
+    return x["variant"], x["preview"], x["ext"]
 
 
 def download_firmwares(
     firmware_folder: Path,
-    board_list: List[str],
-    version_list: Optional[List[str]] = None,
+    ports: List[str],
+    boards: List[str],
+    versions: Optional[List[str]] = None,
     *,
-    preview: bool = False,
     force: bool = False,
     clean: bool = True,
 ):
     skipped = downloaded = 0
-    if version_list is None:
-        version_list = []
-    unique_boards = get_firmware_list(board_list, version_list, preview, clean)
+    if versions is None:
+        versions = []
+    unique_boards = get_firmware_list(ports, boards, versions, clean)
 
     for b in unique_boards:
         log.debug(b["filename"])
@@ -162,7 +179,7 @@ def download_firmwares(
 
     firmware_folder.mkdir(exist_ok=True)
 
-    with open(firmware_folder / "firmware.jsonl", "a", encoding="utf-8", buffering=1) as f_jsonl:
+    with jsonlines.open(firmware_folder / "firmware.jsonl", "a") as writer:
         for board in unique_boards:
             filename = firmware_folder / board["port"] / board["filename"]
             filename.parent.mkdir(exist_ok=True)
@@ -170,7 +187,8 @@ def download_firmwares(
                 skipped += 1
                 log.debug(f" {filename} already exists, skip download")
                 continue
-            log.info(f"Downloading {board['firmware']} to {filename}")
+            log.info(f"Downloading {board['firmware']}")
+            log.info(f"         to {filename}")
             try:
                 r = requests.get(board["firmware"], allow_redirects=True)
                 with open(filename, "wb") as fw:
@@ -179,29 +197,41 @@ def download_firmwares(
             except requests.RequestException as e:
                 log.exception(e)
                 continue
-            # add the firmware to the jsonl file
-            json_str = json.dumps(board) + "\n"
-            f_jsonl.write(json_str)
+            writer.write(board)
             downloaded += 1
     log.info(f"Downloaded {downloaded} firmwares, skipped {skipped} existing files.")
 
 
-def get_firmware_list(board_list: List[str], version_list: List[str], preview: bool, clean: bool):
-    log.trace("Checking MicroPython download pages")
+def get_firmware_list(ports: List[str], boards: List[str], versions: List[str], clean: bool):
+    """
+    Retrieves a list of unique firmware information based on the specified ports, boards, versions, and clean flag.
+
+    Args:
+        ports : The list of ports to check for firmware.
+        boards : The list of boards to filter the firmware by.
+        versions : The list of versions to filter the firmware by.
+        clean : A flag indicating whether to perform a clean check.
 
-    board_urls = sorted(get_boards(PORT_FWTYPES, board_list, clean), key=key_fw_variant_ver)
+    Returns:
+        List[FirmwareInfo]: A list of unique firmware information.
+
+    """
+
+    log.trace("Checking MicroPython download pages")
+    preview = "preview" in versions
+    board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
 
     log.debug(f"Total {len(board_urls)} firmwares")
     relevant = [
         board
         for board in board_urls
-        if board["board"] in board_list and (board["version"] in version_list or board["preview"] and preview)
+        if board["board"] in boards and (board["version"] in versions or board["preview"] and preview)
         # and b["port"] in ["esp32", "rp2"]
     ]
     log.debug(f"Matching firmwares: {len(relevant)}")
     # select the unique boards
     unique_boards: List[FirmwareInfo] = []
-    for _, g in itertools.groupby(relevant, key=key_fw_variant):
+    for _, g in itertools.groupby(relevant, key=key_fw_var_pre_ext):
         # list is aleady sorted by build so we can just get the last item
         sub_list = list(g)
         unique_boards.append(sub_list[-1])
@@ -209,55 +239,39 @@ def get_firmware_list(board_list: List[str], version_list: List[str], preview: b
     return unique_boards
 
 
-@cli.command(
-    "download",
-    help="Download MicroPython firmware for specific ports, boards and versions.",
-)
-@click.option(
-    "--destination",
-    "-d",
-    type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
-    default=DEFAULT_FW_PATH,
-    show_default=True,
-    help="The folder to download the firmware to.",
-)
-@click.option(
-    "--version",
-    "-v",
-    "versions",
-    multiple=True,
-    help="The version of MicroPython to to download. Use 'preview' to include preview versions.",
-    show_default=True,
-    default=["stable"],
-)
-@click.option(
-    "--board",
-    "-b",
-    "boards",
-    multiple=True,
-    show_default=True,
-    help="The board(s) to download the firmware for.",  # Use '--board all' to download all boards.",
-)
-@click.option(
-    "--clean/--no-clean",
-    default=True,
-    show_default=True,
-    help="""Remove dates and hashes from the downloaded firmware filenames.""",
-)
-@click.option(
-    "--force",
-    default=False,
-    is_flag=True,
-    help="""Force download of firmware even if it already exists.""",
-    show_default=True,
-)
-def download(destination: Path, boards: List[str], versions: List[str], force: bool, clean: bool):
-    versions = list(versions)
-    # preview is not a version, it is an option to include preview versions
-    preview = "preview" in versions
-    versions = [v for v in versions if v != "preview"]
+def download(
+    destination: Path,
+    ports: List[str],
+    boards: List[str],
+    versions: List[str],
+    force: bool,
+    clean: bool,
+):
+    """
+    Downloads firmware files based on the specified destination, ports, boards, versions, force flag, and clean flag.
+
+    Args:
+        destination : The destination folder to save the downloaded firmware files.
+        ports : The list of ports to check for firmware.
+        boards : The list of boards to download firmware for.
+        versions : The list of versions to download firmware for.
+        force : A flag indicating whether to force the download even if the firmware file already exists.
+        clean : A flag indicating whether to perform a clean download.
 
-    boards = list(boards) or DEFAULT_BOARDS
-    versions = [clean_version(v, drop_v=True) for v in versions]  # remove leading v from version
-    destination.mkdir(exist_ok=True)
-    download_firmwares(destination, boards, versions, preview=preview, force=force, clean=clean)
+    Returns:
+        None
+
+    Raises:
+        SystemExit: If no boards are found or specified.
+
+    """
+    if not boards:
+        log.critical("No boards found, please connect a board or specify boards to download firmware for.")
+        exit(1)
+    # versions = [clean_version(v, drop_v=True) for v in versions]  # remove leading v from version
+    try:
+        destination.mkdir(exist_ok=True, parents=True)
+    except (PermissionError, FileNotFoundError) as e:
+        log.critical(f"Could not create folder {destination}\n{e}")
+        exit(1)
+    download_firmwares(destination, ports, boards, versions, force=force, clean=clean)

mpflash/mpflash/downloaded.py

@@ -0,0 +1,108 @@
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import jsonlines
+from loguru import logger as log
+
+from mpflash.common import FWInfo
+from mpflash.vendor.versions import clean_version
+
+from .config import config
+
+
+# #########################################################################################################
+def downloaded_firmwares(fw_folder: Path) -> List[FWInfo]:
+    """Load a list of locally downloaded firmwares from the jsonl file"""
+    firmwares: List[FWInfo] = []
+    try:
+        with jsonlines.open(fw_folder / "firmware.jsonl") as reader:
+            firmwares.extend(iter(reader))
+    except FileNotFoundError:
+        log.error(f"No firmware.jsonl found in {fw_folder}")
+    # sort by filename
+    firmwares.sort(key=lambda x: x["filename"])
+    return firmwares
+
+
+def find_downloaded_firmware(
+    *,
+    board_id: str,
+    version: str = "",
+    port: str = "",
+    variants: bool = False,
+    fw_folder: Optional[Path] = None,
+    trie: int = 1,
+    selector: Optional[Dict[str, str]] = None,
+) -> List[FWInfo]:
+    if selector is None:
+        selector = {}
+    fw_folder = fw_folder or config.firmware_folder
+    # Use the information in firmwares.jsonl to find the firmware file
+    fw_list = downloaded_firmwares(fw_folder)
+    if not fw_list:
+        log.error("No firmware files found. Please download the firmware first.")
+        return []
+    # filter by version
+    version = clean_version(version, drop_v=True)
+    fw_list = filter_downloaded_fwlist(fw_list, board_id, version, port, variants, selector)
+
+    if not fw_list and trie < 3:
+        log.info(f"Try ({trie+1}) to find a firmware for the board {board_id}")
+        if trie == 1:
+            # ESP board naming conventions have changed by adding a PORT refix
+            if port.startswith("esp") and not board_id.startswith(port.upper()):
+                board_id = f"{port.upper()}_{board_id}"
+            # RP2 board naming conventions have changed by adding a _RPIprefix
+            if port == "rp2" and not board_id.startswith("RPI_"):
+                board_id = f"RPI_{board_id}"
+        elif trie == 2:
+            board_id = board_id.replace("_", "-")
+
+        fw_list = find_downloaded_firmware(
+            fw_folder=fw_folder,
+            board_id=board_id,
+            version=version,
+            port=port,
+            trie=trie + 1,
+            selector=selector,
+        )
+        # hope we have a match now for the board
+    # sort by filename
+    fw_list.sort(key=lambda x: x["filename"])
+    return fw_list
+
+
+def filter_downloaded_fwlist(
+    fw_list: List[FWInfo],
+    board_id: str,
+    version: str,
+    port: str,
+    # preview: bool,
+    variants: bool,
+    selector: dict,
+) -> List[FWInfo]:
+    """Filter the downloaded firmware list based on the provided parameters"""
+    if "preview" in version:
+        # never get a preview for an older version
+        fw_list = [fw for fw in fw_list if fw["preview"]]
+    else:
+        fw_list = [fw for fw in fw_list if fw["version"] == version]
+
+    # filter by port
+    if port:
+        fw_list = [fw for fw in fw_list if fw["port"] == port]
+
+    if board_id:
+        if variants:
+            # any variant of this board_id
+            fw_list = [fw for fw in fw_list if fw["board"] == board_id]
+        else:
+            # the firmware variant should match exactly the board_id
+            fw_list = [fw for fw in fw_list if fw["variant"] == board_id]
+    if selector and port in selector:
+        fw_list = [fw for fw in fw_list if fw["filename"].endswith(selector[port])]
+    return fw_list
+
+
+# #########################################################################################################
+#

mpflash/mpflash/errors.py

@@ -0,0 +1,5 @@
+"""Custom exceptions for the MPFlash package."""
+
+
+class MPFlashError(Exception):
+    pass

mpflash/mpflash/flash.py

@@ -0,0 +1,69 @@
+import time
+from pathlib import Path
+
+from loguru import logger as log
+
+from mpflash.common import PORT_FWTYPES
+from mpflash.mpremoteboard import MPRemoteBoard
+
+from .flash_esp import flash_esp
+from .flash_stm32 import flash_stm32
+from .flash_uf2 import flash_uf2
+from .worklist import WorkList
+
+# #########################################################################################################
+
+
+def flash_list(
+    todo: WorkList,
+    fw_folder: Path,
+    erase: bool,
+    bootloader: bool,
+):
+    """Flash a list of boards with the specified firmware."""
+    flashed = []
+    for mcu, fw_info in todo:
+        fw_file = fw_folder / fw_info["filename"]  # type: ignore
+        if not fw_file.exists():
+            log.error(f"File {fw_file} does not exist, skipping {mcu.board} on {mcu.serialport}")
+            continue
+        log.info(f"Updating {mcu.board} on {mcu.serialport} to {fw_info['version']}")
+        updated = None
+        # try:
+        if mcu.port in [port for port, exts in PORT_FWTYPES.items() if ".uf2" in exts] and fw_file.suffix == ".uf2":
+            # any .uf2 port ["samd", "rp2", "nrf"]:
+            if bootloader:
+                enter_bootloader(mcu)
+            updated = flash_uf2(mcu, fw_file=fw_file, erase=erase)
+        elif mcu.port in ["stm32"]:
+            if bootloader:
+                enter_bootloader(mcu)
+            updated = flash_stm32(mcu, fw_file, erase=erase)
+        elif mcu.port in ["esp32", "esp8266"]:
+            #  bootloader is handled by esptool for esp32/esp8266
+            updated = flash_esp(mcu, fw_file=fw_file, erase=erase)
+        else:
+            log.error(f"Don't (yet) know how to flash {mcu.port}-{mcu.board} on {mcu.serialport}")
+
+        if updated:
+            flashed.append(updated)
+        else:
+            log.error(f"Failed to flash {mcu.board} on {mcu.serialport}")
+    return flashed
+
+
+def enter_bootloader(mcu: MPRemoteBoard, timeout: int = 10, wait_after: int = 2):
+    """Enter the bootloader mode for the board"""
+    log.info(f"Entering bootloader on {mcu.board} on {mcu.serialport}")
+    mcu.run_command("bootloader", timeout=timeout)
+    time.sleep(wait_after)
+
+
+# TODO:
+# flash from some sort of queue to allow different images to be flashed to the same board
+#  - flash variant 1
+#  - stub variant 1
+#  - flash variant 2
+#  - stub variant 2
+#
+# JIT download / download any missing firmwares based on the detected boards

mpflash/mpflash/flash_esp.py

@@ -4,47 +4,42 @@
 # #########################################################################################################
 """
 
-import time
 from pathlib import Path
 from typing import List, Optional
 
 import esptool
 from loguru import logger as log
 
-from stubber.bulk.mpremoteboard import MPRemoteBoard
+from mpflash.mpboard_id import find_stored_board
+from mpflash.mpremoteboard import MPRemoteBoard
 
 
-def flash_esp(
-    mcu: MPRemoteBoard, fw_file: Path, *, erase: bool = True
-) -> Optional[MPRemoteBoard]:
+def flash_esp(mcu: MPRemoteBoard, fw_file: Path, *, erase: bool = True) -> Optional[MPRemoteBoard]:
     if mcu.port not in ["esp32", "esp8266"] or mcu.board in ["ARDUINO_NANO_ESP32"]:
-        log.error(
-            f"esptool not supported for {mcu.port} {mcu.board} on {mcu.serialport}"
-        )
+        log.error(f"esptool not supported for {mcu.port} {mcu.board} on {mcu.serialport}")
         return None
 
     log.info(f"Flashing {fw_file} on {mcu.board} on {mcu.serialport}")
-    if mcu.port == "esp8266":
-        baud_rate = str(460_800)
-    else:
-        baud_rate = str(512_000)
-        # baud_rate = str(115_200)
+    if not mcu.cpu:
+        # Lookup CPU based on the board name
+        mcu.cpu = find_stored_board(mcu.board)["cpu"]
+
     cmds: List[List[str]] = []
     if erase:
-        cmds.append(
-            f"esptool --chip {mcu.cpu} --port {mcu.serialport} erase_flash".split()
-        )
+        cmds.append(f"esptool --chip {mcu.cpu} --port {mcu.serialport} erase_flash".split())
 
-    if mcu.cpu.upper() in ("ESP32", "ESP32S2"):
-        start_addr = "0x1000"
-    elif mcu.cpu.upper() in ("ESP32S3", "ESP32C3"):
-        start_addr = "0x0"
     if mcu.cpu.upper().startswith("ESP32"):
+        baud_rate = str(921_600)
+        if mcu.cpu.upper() in ("ESP32", "ESP32S2"):
+            start_addr = "0x1000"
+        elif mcu.cpu.upper() in ("ESP32S3", "ESP32C3"):
+            start_addr = "0x0"
         cmds.append(
             f"esptool --chip {mcu.cpu} --port {mcu.serialport} -b {baud_rate} write_flash --compress {start_addr}".split()
             + [str(fw_file)]
         )
     elif mcu.cpu.upper() == "ESP8266":
+        baud_rate = str(460_800)
         start_addr = "0x0"
         cmds.append(
             f"esptool --chip {mcu.cpu} --port {mcu.serialport} -b {baud_rate} write_flash --flash_size=detect {start_addr}".split()
@@ -59,7 +54,6 @@ def flash_esp(
         return None
 
     log.info("Done flashing, resetting the board and wait for it to restart")
-    time.sleep(5)
-    mcu.get_mcu_info()
-    log.success(f"Flashed {mcu.version} to {mcu.board} on {mcu.serialport} done")
+    mcu.wait_for_restart()
+    log.success(f"Flashed {mcu.serialport} to {mcu.board} {mcu.version}")
     return mcu