mpflash 0.4.0.post3__py3-none-any.whl → 0.5.0__py3-none-any.whl

mpflash/common.py

@@ -1,7 +1,9 @@
+import os
 import time
+from functools import lru_cache
 from typing import TypedDict
 
-from github import Github
+from github import Auth, Github
 from loguru import logger as log
 from packaging.version import parse
 from rich.progress import track
@@ -19,6 +21,14 @@ PORT_FWTYPES = {
     "renesas-ra": [".hex"],
 }
 
+# Token with no permissions to avoid throttling
+# https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#getting-a-higher-rate-limit
+PAT_NO_ACCESS = (
+    "github_pat" + "_11AAHPVFQ0qAkDnSUaMKSp" + "_ZkDl5NRRwBsUN6EYg9ahp1Dvj4FDDONnXVgimxC2EtpY7Q7BUKBoQ0Jq72X"
+)
+PAT = os.environ.get("GITHUB_TOKEN") or PAT_NO_ACCESS
+GH_CLIENT = Github(auth=Auth.Token(PAT))
+
 
 class FWInfo(TypedDict):
     filename: str
@@ -27,6 +37,7 @@ class FWInfo(TypedDict):
     variant: str
     preview: bool
     version: str
+    build: str
 
 
 #############################################################
@@ -52,7 +63,7 @@ def clean_version(
     if version in {"", "-"}:
         return version
     if version.lower() == "stable":
-        _v = get_stable_version()
+        _v = get_stable_mp_version()
         if not _v:
             log.warning("Could not determine the latest stable version")
             return "stable"
@@ -89,12 +100,12 @@ def clean_version(
     return version
 
 
-def micropython_versions(minver: str = "v1.9.2"):
+@lru_cache(maxsize=10)
+def micropython_versions(minver: str = "v1.20"):
     """Get the list of micropython versions from github tags"""
     try:
-        g = Github()
-        _ = 1 / 0
-        repo = g.get_repo("micropython/micropython")
+        gh_client = GH_CLIENT
+        repo = gh_client.get_repo("micropython/micropython")
         versions = [tag.name for tag in repo.get_tags() if parse(tag.name) >= parse(minver)]
     except Exception:
         versions = [
@@ -116,19 +127,23 @@ def micropython_versions(minver: str = "v1.9.2"):
             "v1.12",
             "v1.11",
             "v1.10",
-            "v1.9.4",
-            "v1.9.3",
         ]
         versions = [v for v in versions if parse(v) >= parse(minver)]
     return sorted(versions)
 
 
-def get_stable_version() -> str:
+def get_stable_mp_version() -> str:
     # read the versions from the git tags
     all_versions = micropython_versions(minver="v1.17")
     return [v for v in all_versions if not v.endswith(V_PREVIEW)][-1]
 
 
+def get_preview_mp_version() -> str:
+    # read the versions from the git tags
+    all_versions = micropython_versions(minver="v1.17")
+    return [v for v in all_versions if v.endswith(V_PREVIEW)][-1]
+
+
 #############################################################
 def wait_for_restart(mcu: MPRemoteBoard, timeout: int = 10):
     """wait for the board to restart"""

mpflash/config.py

@@ -1,16 +1,18 @@
 """centralized configuration for mpflash"""
 
-import platformdirs
-
-from typing import List
 from pathlib import Path
+from typing import List
+
+import platformdirs
 
 
 class MPtoolConfig:
     """Centralized configuration for mpflash"""
 
     quiet: bool = False
+    verbose: bool = False
     ignore_ports: List[str] = []
+    interactive: bool = True
     firmware_folder: Path = platformdirs.user_downloads_path() / "firmware"
 
 

mpflash/download.py

@@ -5,19 +5,24 @@ Uses the micropython.org website to get the available versions and locations to
 
 import functools
 import itertools
-import json
 import re
 from pathlib import Path
 from typing import Dict, List, Optional
 from urllib.parse import urljoin
 
-from mpflash.common import PORT_FWTYPES
+# #########################################################################################################
+# make sure that jsonlines does not mistake the MicroPython ujson for the CPython ujson
+import jsonlines
 import requests
 from bs4 import BeautifulSoup
 from loguru import logger as log
 from rich.progress import track
 
-# from .common import PORT_FWTYPES, clean_version
+from mpflash.common import PORT_FWTYPES
+
+jsonlines.ujson = None  # type: ignore
+# #########################################################################################################
+
 
 MICROPYTHON_ORG_URL = "https://micropython.org/"
 
@@ -78,6 +83,18 @@ FirmwareInfo = Dict[str, str]
 # The first run takes ~60 seconds to run for 4 ports , all boards
 # so it makes sense to cache the results and skip boards as soon as possible
 def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FirmwareInfo]:
+    """
+    Retrieves a list of firmware information for the specified ports and boards.
+
+    Args:
+        ports (List[str]): The list of ports to check for firmware.
+        boards (List[str]): The list of boards to retrieve firmware information for.
+        clean (bool): A flag indicating whether to perform a clean retrieval.
+
+    Returns:
+        List[FirmwareInfo]: A list of firmware information for the specified ports and boards.
+
+    """
     board_urls: List[FirmwareInfo] = []
     for port in ports:
         download_page_url = f"{MICROPYTHON_ORG_URL}download/?port={port}"
@@ -134,14 +151,13 @@ def download_firmwares(
     boards: List[str],
     versions: Optional[List[str]] = None,
     *,
-    preview: bool = False,
     force: bool = False,
     clean: bool = True,
 ):
     skipped = downloaded = 0
     if versions is None:
         versions = []
-    unique_boards = get_firmware_list(ports, boards, versions, preview, clean)
+    unique_boards = get_firmware_list(ports, boards, versions, clean)
 
     for b in unique_boards:
         log.debug(b["filename"])
@@ -151,7 +167,7 @@ def download_firmwares(
 
     firmware_folder.mkdir(exist_ok=True)
 
-    with open(firmware_folder / "firmware.jsonl", "a", encoding="utf-8", buffering=1) as f_jsonl:
+    with jsonlines.open(firmware_folder / "firmware.jsonl", "a") as writer:
         for board in unique_boards:
             filename = firmware_folder / board["port"] / board["filename"]
             filename.parent.mkdir(exist_ok=True)
@@ -169,16 +185,28 @@ def download_firmwares(
             except requests.RequestException as e:
                 log.exception(e)
                 continue
-            # add the firmware to the jsonl file
-            json_str = json.dumps(board) + "\n"
-            f_jsonl.write(json_str)
+            writer.write(board)
             downloaded += 1
     log.info(f"Downloaded {downloaded} firmwares, skipped {skipped} existing files.")
 
 
-def get_firmware_list(ports: List[str], boards: List[str], versions: List[str], preview: bool, clean: bool):
-    log.trace("Checking MicroPython download pages")
+def get_firmware_list(ports: List[str], boards: List[str], versions: List[str], clean: bool):
+    """
+    Retrieves a list of unique firmware information based on the specified ports, boards, versions, and clean flag.
 
+    Args:
+        ports : The list of ports to check for firmware.
+        boards : The list of boards to filter the firmware by.
+        versions : The list of versions to filter the firmware by.
+        clean : A flag indicating whether to perform a clean check.
+
+    Returns:
+        List[FirmwareInfo]: A list of unique firmware information.
+
+    """
+
+    log.trace("Checking MicroPython download pages")
+    preview = "preview" in versions
     board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
 
     log.debug(f"Total {len(board_urls)} firmwares")
@@ -206,8 +234,25 @@ def download(
     versions: List[str],
     force: bool,
     clean: bool,
-    preview: bool,
 ):
+    """
+    Downloads firmware files based on the specified destination, ports, boards, versions, force flag, and clean flag.
+
+    Args:
+        destination : The destination folder to save the downloaded firmware files.
+        ports : The list of ports to check for firmware.
+        boards : The list of boards to download firmware for.
+        versions : The list of versions to download firmware for.
+        force : A flag indicating whether to force the download even if the firmware file already exists.
+        clean : A flag indicating whether to perform a clean download.
+
+    Returns:
+        None
+
+    Raises:
+        SystemExit: If no boards are found or specified.
+
+    """
     if not boards:
         log.critical("No boards found, please connect a board or specify boards to download firmware for.")
         exit(1)
@@ -217,4 +262,4 @@ def download(
     except (PermissionError, FileNotFoundError) as e:
         log.critical(f"Could not create folder {destination}\n{e}")
         exit(1)
-    download_firmwares(destination, ports, boards, versions, preview=preview, force=force, clean=clean)
+    download_firmwares(destination, ports, boards, versions, force=force, clean=clean)

mpflash/flash.py

@@ -5,9 +5,10 @@ from typing import Dict, List, Optional, Tuple
 import jsonlines
 from loguru import logger as log
 
+from mpflash.mpremoteboard import MPRemoteBoard
+
 from .common import FWInfo, clean_version
 from .config import config
-from .mpremoteboard import MPRemoteBoard
 
 # #########################################################################################################
 WorkList = List[Tuple[MPRemoteBoard, FWInfo]]
@@ -32,12 +33,11 @@ def find_firmware(
     board: str,
     version: str = "",
     port: str = "",
-    preview: bool = False,
     variants: bool = False,
     fw_folder: Optional[Path] = None,
     trie: int = 1,
     selector: Optional[Dict[str, str]] = None,
-):
+) -> List[FWInfo]:
     if selector is None:
         selector = {}
     fw_folder = fw_folder or config.firmware_folder
@@ -48,24 +48,8 @@ def find_firmware(
         return []
     # filter by version
     version = clean_version(version, drop_v=True)
-    if preview or "preview" in version:
-        # never get a preview for an older version
-        fw_list = [fw for fw in fw_list if fw["preview"]]
-    else:
-        fw_list = [fw for fw in fw_list if fw["version"] == version]
+    fw_list = filter_fwlist(fw_list, board, version, port, variants, selector)
 
-    # filter by port
-    if port:
-        fw_list = [fw for fw in fw_list if fw["port"] == port]
-
-    if board:
-        if variants:
-            fw_list = [fw for fw in fw_list if fw["board"] == board]
-        else:
-            # the variant should match exactly the board name
-            fw_list = [fw for fw in fw_list if fw["variant"] == board]
-    if selector and port in selector:
-        fw_list = [fw for fw in fw_list if fw["filename"].endswith(selector[port])]
     if not fw_list and trie < 2:
         board_id = board.replace("_", "-")
         # ESP board naming conventions have changed by adding a PORT refix
@@ -75,13 +59,12 @@ def find_firmware(
         if port == "rp2" and not board_id.startswith("RPI_"):
             board_id = f"RPI_{board_id}"
 
-        log.warning(f"Trying to find a firmware for the board {board_id}")
+        log.info(f"Try ({trie}) to find a firmware for the board {board_id}")
         fw_list = find_firmware(
             fw_folder=fw_folder,
             board=board_id,
             version=version,
             port=port,
-            preview=preview,
             trie=trie + 1,
             selector=selector,
         )
@@ -91,6 +74,37 @@ def find_firmware(
     return fw_list
 
 
+def filter_fwlist(
+    fw_list: List[FWInfo],
+    board: str,
+    version: str,
+    port: str,
+    # preview: bool,
+    variants: bool,
+    selector: dict,
+) -> List[FWInfo]:
+    """Filter the firmware list based on the provided parameters"""
+    if "preview" in version:
+        # never get a preview for an older version
+        fw_list = [fw for fw in fw_list if fw["preview"]]
+    else:
+        fw_list = [fw for fw in fw_list if fw["version"] == version]
+
+    # filter by port
+    if port:
+        fw_list = [fw for fw in fw_list if fw["port"] == port]
+
+    if board:
+        if variants:
+            fw_list = [fw for fw in fw_list if fw["board"] == board]
+        else:
+            # the variant should match exactly the board name
+            fw_list = [fw for fw in fw_list if fw["variant"] == board]
+    if selector and port in selector:
+        fw_list = [fw for fw in fw_list if fw["filename"].endswith(selector[port])]
+    return fw_list
+
+
 # #########################################################################################################
 #
 
@@ -100,7 +114,6 @@ def auto_update(
     target_version: str,
     fw_folder: Path,
     *,
-    preview: bool = False,
     selector: Optional[Dict[str, str]] = None,
 ) -> WorkList:
     """Builds a list of boards to update based on the connected boards and the firmware available"""
@@ -118,7 +131,6 @@ def auto_update(
             board=mcu.board,
             version=target_version,
             port=mcu.port,
-            preview=preview or "preview" in target_version,
             selector=selector,
         )
 

mpflash/flash_esp.py

@@ -4,26 +4,28 @@
 # #########################################################################################################
 """
 
-import time
 from pathlib import Path
 from typing import List, Optional
 
 import esptool
 from loguru import logger as log
 
-from .mpremoteboard import MPRemoteBoard
+from mpflash.mpboard_id.api import find_mp_board
+from mpflash.mpremoteboard import MPRemoteBoard
+
 from .common import wait_for_restart
 
-def flash_esp(
-    mcu: MPRemoteBoard, fw_file: Path, *, erase: bool = True
-) -> Optional[MPRemoteBoard]:
+
+def flash_esp(mcu: MPRemoteBoard, fw_file: Path, *, erase: bool = True) -> Optional[MPRemoteBoard]:
     if mcu.port not in ["esp32", "esp8266"] or mcu.board in ["ARDUINO_NANO_ESP32"]:
-        log.error(
-            f"esptool not supported for {mcu.port} {mcu.board} on {mcu.serialport}"
-        )
+        log.error(f"esptool not supported for {mcu.port} {mcu.board} on {mcu.serialport}")
         return None
 
     log.info(f"Flashing {fw_file} on {mcu.board} on {mcu.serialport}")
+    if not mcu.cpu:
+        # Lookup CPU based on the board name
+        mcu.cpu = find_mp_board(mcu.board)["cpu"]
+
     if mcu.port == "esp8266":
         baud_rate = str(460_800)
     else:
@@ -31,9 +33,7 @@ def flash_esp(
         # baud_rate = str(115_200)
     cmds: List[List[str]] = []
     if erase:
-        cmds.append(
-            f"esptool --chip {mcu.cpu} --port {mcu.serialport} erase_flash".split()
-        )
+        cmds.append(f"esptool --chip {mcu.cpu} --port {mcu.serialport} erase_flash".split())
 
     if mcu.cpu.upper() in ("ESP32", "ESP32S2"):
         start_addr = "0x1000"

mpflash/flash_stm32.py

@@ -8,7 +8,7 @@ from mpflash.common import wait_for_restart
 
 # from .flash_stm32_cube import flash_stm32_cubecli
 from .flash_stm32_dfu import dfu_init, flash_stm32_dfu
-from .mpremoteboard import MPRemoteBoard
+from mpflash.mpremoteboard import MPRemoteBoard
 
 
 def flash_stm32(mcu: MPRemoteBoard, fw_file: Path, *, erase: bool, stm32_dfu: bool = True):

mpflash/flash_stm32_dfu.py

@@ -1,26 +1,23 @@
 import platform
-import sys
-import time
 from pathlib import Path
 from typing import Optional
 
 from loguru import logger as log
 
-from .common import wait_for_restart
-from .mpremoteboard import MPRemoteBoard
+from mpflash.mpremoteboard import MPRemoteBoard
 
 
-def init_libusb_windows():
+def init_libusb_windows() -> bool:
     # on windows we need to initialze the libusb backend with the correct dll
-    import libusb
+    import libusb  # type: ignore
     import usb.backend.libusb1 as libusb1
 
     arch = "x64" if platform.architecture()[0] == "64bit" else "x86"
     libusb1_dll = Path(libusb.__file__).parent / f"_platform\\_windows\\{arch}\\libusb-1.0.dll"
     if not libusb1_dll.exists():
         raise FileNotFoundError(f"libusb1.dll not found at {libusb1_dll}")
-
     backend = libusb1.get_backend(find_library=lambda x: libusb1_dll.as_posix())
+    return backend is not None
 
 
 try:

mpflash/flash_uf2.py

@@ -11,7 +11,7 @@ from typing import Optional
 from loguru import logger as log
 from rich.progress import track
 
-from .mpremoteboard import MPRemoteBoard
+from mpflash.mpremoteboard import MPRemoteBoard
 
 from .common import PORT_FWTYPES
 from .flash_uf2_linux import dismount_uf2, wait_for_UF2_linux

mpflash/mpboard_id/api.py

@@ -1,9 +1,9 @@
 import json
 from functools import lru_cache
 from pathlib import Path
-from typing import List, Tuple, TypedDict, Union
+from typing import List, Optional, Tuple, TypedDict, Union
 
-from mpflash.common import PORT_FWTYPES
+from mpflash.common import PORT_FWTYPES, clean_version
 
 
 # Board  based on the dataclass Board but changed to TypedDict
@@ -18,31 +18,72 @@ class Board(TypedDict):
     mcu_name: str
     path: Union[Path, str]
     version: str
+    cpu: str
 
 
 @lru_cache(maxsize=None)
 def read_boardinfo() -> List[Board]:
+    """Reads the board_info.json file and returns the data as a list of Board objects"""
     with open(Path(__file__).parent / "board_info.json", "r") as file:
         return json.load(file)
 
 
 def known_mp_ports() -> List[str]:
     # TODO: Filter for Version
-    info = read_boardinfo()
+    mp_boards = read_boardinfo()
     # select the unique ports from info
-    ports = set({board["port"] for board in info if board["port"] in PORT_FWTYPES.keys()})
+    ports = set({board["port"] for board in mp_boards if board["port"] in PORT_FWTYPES.keys()})
     return sorted(list(ports))
 
 
-def known_mp_boards(port: str) -> List[Tuple[str, str]]:
-    info = read_boardinfo()
-    boards = set({(board["description"], board["board"]) for board in info if board["port"] == port})
+def get_mp_boards_for_port(port: str, versions: Optional[List[str]] = None):
+    """
+    Returns a list of boards for the given port and version(s)
+
+    port : str : The Micropython port to filter for
+    versions : List[str] : The Micropython versions to filter for (actual versions required)"""
+    mp_boards = read_boardinfo()
+
+    # filter for 'preview' as they are not in the board_info.json
+    # instead use stable version
+    versions = versions or []
+    if "preview" in versions:
+        versions.remove("preview")
+        versions.append("stable")
+    if versions:
+        # make sure of the v prefix
+        versions = [clean_version(v) for v in versions]
+        # filter for the version(s)
+        mp_boards = [board for board in mp_boards if board["version"] in versions]
+    # filter for the port
+    mp_boards = [board for board in mp_boards if board["port"] == port]
+    return mp_boards
+
+
+def known_mp_boards(port: str, versions: Optional[List[str]] = None) -> List[Tuple[str, str]]:
+    """
+    Returns a list of tuples with the description and board name for the given port and version
+
+    port : str : The Micropython port to filter for
+    versions : List[str] : The Micropython versions to filter for (actual versions required)
+    """
+    mp_boards = get_mp_boards_for_port(port, versions)
+
+    boards = set(
+        {(f'{board["description"]} [board["board"]] {board["version"]}', board["board"]) for board in mp_boards}
+    )
     return sorted(list(boards))
 
 
-def find_mp_port(board: str) -> str:
+def find_mp_board(board: str) -> Board:
+    """Find the board for the given board"""
     info = read_boardinfo()
     for board_info in info:
         if board_info["board"] == board:
-            return board_info["port"]
-    return ""
+            if "cpu" not in board_info or not board_info["cpu"]:
+                if " with " in board_info["description"]:
+                    board_info["cpu"] = board_info["description"].split(" with ")[-1]
+                else:
+                    board_info["cpu"] = board_info["port"]
+            return board_info
+    raise LookupError(f"Board {board} not found")