micropython-stubber 1.17.5__py3-none-any.whl → 1.19.0__py3-none-any.whl

mpflash/mpflash/common.py

@@ -1,127 +1,38 @@
-from pathlib import Path
-from typing import Dict, Union
+import os
+import time
+from typing import TypedDict
 
-from github import Github
-from loguru import logger as log
-from packaging.version import parse
+from github import Auth, Github
+from rich.progress import track
+
+from mpflash.errors import MPFlashError
+# from mpflash.mpremoteboard import MPRemoteBoard
 
 PORT_FWTYPES = {
-    "stm32": ".hex",
-    "esp32": ".bin",
-    "esp8266": ".bin",
-    "rp2": ".uf2",
-    "samd": ".uf2",
-    "mimxrt": ".hex",
-    "nrf": ".hex",
-    "renesas-ra": ".hex",
+    "stm32": [".dfu"],  # need .dfu for pydfu.py - .hex for cube cli/GUI
+    "esp32": [".bin"],
+    "esp8266": [".bin"],
+    "rp2": [".uf2"],
+    "samd": [".uf2"],
+    "mimxrt": [".hex"],
+    "nrf": [".uf2"],
+    "renesas-ra": [".hex"],
 }
 
-DEFAULT_FW_PATH = Path.cwd() / "firmware"
-# DEFAULT_FW_PATH = Path.home() / "mp_firmware"
-
-FWInfo = Dict[str, Union[str, bool]]
-
-#############################################################
-# Version handling copied from stubber/utils/versions.py
-#############################################################
-V_PREVIEW = "preview"
-"Latest preview version"
-
-SET_PREVIEW = {"preview", "latest", "master"}
-
-
-def clean_version(
-    version: str,
-    *,
-    build: bool = False,
-    patch: bool = False,
-    commit: bool = False,
-    drop_v: bool = False,
-    flat: bool = False,
-):
-    "Clean up and transform the many flavours of versions"
-    # 'v1.13.0-103-gb137d064e' --> 'v1.13-103'
-    if version in {"", "-"}:
-        return version
-    if version.lower() == "stable":
-        _v = get_stable_version()
-        if not _v:
-            log.warning("Could not determine the latest stable version")
-            return "stable"
-        version = _v
-        log.info(f"Using latest stable version: {version}")
-    is_preview = "-preview" in version
-    nibbles = version.split("-")
-    ver_ = nibbles[0].lower().lstrip("v")
-    if not patch and ver_ >= "1.10.0" and ver_ < "1.20.0" and ver_.endswith(".0"):
-        # remove the last ".0" - but only for versions between 1.10 and 1.20 (because)
-        nibbles[0] = nibbles[0][:-2]
-    if len(nibbles) == 1:
-        version = nibbles[0]
-    elif build and not is_preview:
-        version = "-".join(nibbles) if commit else "-".join(nibbles[:-1])
-    else:
-        # version = "-".join((nibbles[0], LATEST))
-        # HACK: this is not always right, but good enough most of the time
-        if is_preview:
-            version = "-".join((nibbles[0], V_PREVIEW))
-        else:
-            version = V_PREVIEW
-    if flat:
-        version = version.strip().replace(".", "_").replace("-", "_")
-    else:
-        version = version.strip().replace("_preview", "-preview").replace("_", ".")
-
-    if drop_v:
-        version = version.lstrip("v")
-    elif not version.startswith("v") and version.lower() not in SET_PREVIEW:
-        version = "v" + version
-    if version in SET_PREVIEW:
-        version = V_PREVIEW
-    return version
-
-
-def micropython_versions(minver: str = "v1.9.2"):
-    """Get the list of micropython versions from github tags"""
-    try:
-        g = Github()
-        _ = 1 / 0
-        repo = g.get_repo("micropython/micropython")
-        versions = [
-            tag.name for tag in repo.get_tags() if parse(tag.name) >= parse(minver)
-        ]
-    except Exception:
-        versions = [
-            "v9.99.9-preview",
-            "v1.22.2",
-            "v1.22.1",
-            "v1.22.0",
-            "v1.21.1",
-            "v1.21.0",
-            "v1.20.0",
-            "v1.19.1",
-            "v1.19",
-            "v1.18",
-            "v1.17",
-            "v1.16",
-            "v1.15",
-            "v1.14",
-            "v1.13",
-            "v1.12",
-            "v1.11",
-            "v1.10",
-            "v1.9.4",
-            "v1.9.3",
-        ]
-        versions = [v for v in versions if parse(v) >= parse(minver)]
-    return sorted(versions)
-
-
-def get_stable_version() -> str:
-    # read the versions from the git tags
-    all_versions = micropython_versions(minver="v1.17")
-    stable_version = [v for v in all_versions if not v.endswith(V_PREVIEW)][-1]
-    return stable_version
-
-
-#############################################################
+# Token with no permissions to avoid throttling
+# https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#getting-a-higher-rate-limit
+PAT_NO_ACCESS = (
+    "github_pat" + "_11AAHPVFQ0qAkDnSUaMKSp" + "_ZkDl5NRRwBsUN6EYg9ahp1Dvj4FDDONnXVgimxC2EtpY7Q7BUKBoQ0Jq72X"
+)
+PAT = os.environ.get("GITHUB_TOKEN") or PAT_NO_ACCESS
+GH_CLIENT = Github(auth=Auth.Token(PAT))
+
+
+class FWInfo(TypedDict):
+    filename: str
+    port: str
+    board: str
+    variant: str
+    preview: bool
+    version: str
+    build: str

mpflash/mpflash/config.py

@@ -1,10 +1,19 @@
 """centralized configuration for mpflash"""
 
+from pathlib import Path
 from typing import List
 
+import platformdirs
+
 
 class MPtoolConfig:
+    """Centralized configuration for mpflash"""
+
+    quiet: bool = False
+    verbose: bool = False
     ignore_ports: List[str] = []
+    interactive: bool = True
+    firmware_folder: Path = platformdirs.user_downloads_path() / "firmware"
 
 
 config = MPtoolConfig()

mpflash/mpflash/{downloader.py → download.py}

@@ -5,20 +5,24 @@ Uses the micropython.org website to get the available versions and locations to
 
 import functools
 import itertools
-import json
 import re
 from pathlib import Path
 from typing import Dict, List, Optional
 from urllib.parse import urljoin
 
+# #########################################################################################################
+# make sure that jsonlines does not mistake the MicroPython ujson for the CPython ujson
+import jsonlines
 import requests
-import rich_click as click
 from bs4 import BeautifulSoup
 from loguru import logger as log
 from rich.progress import track
 
-from .cli_group import cli
-from .common import PORT_FWTYPES, clean_version
+from mpflash.common import PORT_FWTYPES
+
+jsonlines.ujson = None  # type: ignore
+# #########################################################################################################
+
 
 MICROPYTHON_ORG_URL = "https://micropython.org/"
 
@@ -28,20 +32,6 @@ RE_HASH = r"(.g[0-9a-f]+\.)"
 # regex to extract the version from the firmware filename
 RE_VERSION_PREVIEW = r"(\d+\.\d+(\.\d+)?(-\w+.\d+)?)"
 
-# boards we are interested in ( this avoids getting a lot of boards that are not relevant)
-DEFAULT_BOARDS = [
-    "PYBV11",
-    "ESP8266_GENERIC",
-    "ESP32_GENERIC",
-    "ESP32_GENERIC_S3",
-    "RPI_PICO",
-    "RPI_PICO_W",
-    "ADAFRUIT_QTPY_RP2040",
-    "ARDUINO_NANO_RP2040_CONNECT",
-    "PIMORONI_PICOLIPO_16MB",
-    "SEEED_WIO_TERMINAL",
-]
-
 
 # use functools.lru_cache to avoid needing to download pages multiple times
 @functools.lru_cache(maxsize=500)
@@ -56,6 +46,9 @@ def get_board_urls(page_url: str) -> List[Dict[str, str]]:
     """
     Get the urls to all the board pages listed on this page.
     Assumes that all links to firmware  have "class": "board-card"
+
+    Args:
+        page_url (str): The url of the page to get the board urls from.
     """
     downloads_html = get_page(page_url)
     soup = BeautifulSoup(downloads_html, "html.parser")
@@ -67,8 +60,17 @@ def get_board_urls(page_url: str) -> List[Dict[str, str]]:
     return [{"board": board, "url": page_url + board} for board in boards]
 
 
-def firmware_list(board_url: str, base_url: str, ext: str) -> List[str]:
-    """Get the urls to all the firmware files for a board."""
+def board_firmware_urls(board_url: str, base_url: str, ext: str) -> List[str]:
+    """
+    Get the urls to all the firmware files for a board.
+    Args:
+        page_url (str): The url of the page to get the board urls from.
+    ??? base_url (str): The base url to join the relative urls to.
+        ext (str): The extension of the firmware files to get. (with or withouth leading .)
+
+    the urls are relative urls to the site root
+
+    """
     html = get_page(board_url)
     soup = BeautifulSoup(html, "html.parser")
     # get all the a tags:
@@ -77,9 +79,7 @@ def firmware_list(board_url: str, base_url: str, ext: str) -> List[str]:
     tags = soup.findAll(
         "a",
         recursive=True,
-        attrs={
-            "href": re.compile(r"^/resources/firmware/.*\." + ext.lstrip(".") + "$")
-        },
+        attrs={"href": re.compile(r"^/resources/firmware/.*\." + ext.lstrip(".") + "$")},
     )
     if "?" in base_url:
         base_url = base_url.split("?")[0]
@@ -94,24 +94,35 @@ FirmwareInfo = Dict[str, str]
 # boards we are interested in ( this avoids getting a lot of boards we don't care about)
 # The first run takes ~60 seconds to run for 4 ports , all boards
 # so it makes sense to cache the results and skip boards as soon as possible
-def get_boards(
-    fw_types: Dict[str, str], board_list: List[str], clean: bool
-) -> List[FirmwareInfo]:
+def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FirmwareInfo]:
+    """
+    Retrieves a list of firmware information for the specified ports and boards.
+
+    Args:
+        ports (List[str]): The list of ports to check for firmware.
+        boards (List[str]): The list of boards to retrieve firmware information for.
+        clean (bool): A flag indicating whether to perform a clean retrieval.
+
+    Returns:
+        List[FirmwareInfo]: A list of firmware information for the specified ports and boards.
+
+    """
     board_urls: List[FirmwareInfo] = []
-    for port in fw_types:
+    for port in ports:
         download_page_url = f"{MICROPYTHON_ORG_URL}download/?port={port}"
         _urls = get_board_urls(download_page_url)
         # filter out boards we don't care about
-        _urls = [board for board in _urls if board["board"] in board_list]
+        _urls = [board for board in _urls if board["board"] in boards]
         # add the port to the board urls
         for board in _urls:
             board["port"] = port
 
-        for board in track(
-            _urls, description="Checking download pages", transient=True
-        ):
+        for board in track(_urls, description=f"Checking {port} download pages", transient=True):
             # add a board to the list for each firmware found
-            firmwares = firmware_list(board["url"], MICROPYTHON_ORG_URL, fw_types[port])
+            firmwares = []
+            for ext in PORT_FWTYPES[port]:
+                firmwares += board_firmware_urls(board["url"], MICROPYTHON_ORG_URL, ext)
+
             for _url in firmwares:
                 board["firmware"] = _url
                 board["preview"] = "preview" in _url  # type: ignore
@@ -130,38 +141,35 @@ def get_boards(
                     # remove hash from firmware name
                     fname = re.sub(RE_HASH, ".", fname)
                 board["filename"] = fname
-                board["variant"] = (
-                    board["filename"].split("-v")[0]
-                    if "-v" in board["filename"]
-                    else ""
-                )
+                board["ext"] = Path(board["firmware"]).suffix
+                board["variant"] = board["filename"].split("-v")[0] if "-v" in board["filename"] else ""
                 board_urls.append(board.copy())
     return board_urls
 
 
-def key_fw_variant_ver(x: FirmwareInfo):
+def key_fw_ver_pre_ext_bld(x: FirmwareInfo):
     "sorting key for the retrieved board urls"
-    return x["variant"], x["version"], x["preview"], x["build"]
+    return x["variant"], x["version"], x["preview"], x["ext"], x["build"]
 
 
-def key_fw_variant(x: FirmwareInfo):
+def key_fw_var_pre_ext(x: FirmwareInfo):
     "Grouping key for the retrieved board urls"
-    return x["variant"], x["preview"]
+    return x["variant"], x["preview"], x["ext"]
 
 
 def download_firmwares(
     firmware_folder: Path,
-    board_list: List[str],
-    version_list: Optional[List[str]] = None,
+    ports: List[str],
+    boards: List[str],
+    versions: Optional[List[str]] = None,
     *,
-    preview: bool = False,
     force: bool = False,
     clean: bool = True,
 ):
     skipped = downloaded = 0
-    if version_list is None:
-        version_list = []
-    unique_boards = get_firmware_list(board_list, version_list, preview, clean)
+    if versions is None:
+        versions = []
+    unique_boards = get_firmware_list(ports, boards, versions, clean)
 
     for b in unique_boards:
         log.debug(b["filename"])
@@ -171,9 +179,7 @@ def download_firmwares(
 
     firmware_folder.mkdir(exist_ok=True)
 
-    with open(
-        firmware_folder / "firmware.jsonl", "a", encoding="utf-8", buffering=1
-    ) as f_jsonl:
+    with jsonlines.open(firmware_folder / "firmware.jsonl", "a") as writer:
         for board in unique_boards:
             filename = firmware_folder / board["port"] / board["filename"]
             filename.parent.mkdir(exist_ok=True)
@@ -181,7 +187,8 @@ def download_firmwares(
                 skipped += 1
                 log.debug(f" {filename} already exists, skip download")
                 continue
-            log.info(f"Downloading {board['firmware']} to {filename}")
+            log.info(f"Downloading {board['firmware']}")
+            log.info(f"         to {filename}")
             try:
                 r = requests.get(board["firmware"], allow_redirects=True)
                 with open(filename, "wb") as fw:
@@ -190,34 +197,41 @@ def download_firmwares(
             except requests.RequestException as e:
                 log.exception(e)
                 continue
-            # add the firmware to the jsonl file
-            json_str = json.dumps(board) + "\n"
-            f_jsonl.write(json_str)
+            writer.write(board)
             downloaded += 1
     log.info(f"Downloaded {downloaded} firmwares, skipped {skipped} existing files.")
 
 
-def get_firmware_list(
-    board_list: List[str], version_list: List[str], preview: bool, clean: bool
-):
-    log.trace("Checking MicroPython download pages")
+def get_firmware_list(ports: List[str], boards: List[str], versions: List[str], clean: bool):
+    """
+    Retrieves a list of unique firmware information based on the specified ports, boards, versions, and clean flag.
 
-    board_urls = sorted(
-        get_boards(PORT_FWTYPES, board_list, clean), key=key_fw_variant_ver
-    )
+    Args:
+        ports : The list of ports to check for firmware.
+        boards : The list of boards to filter the firmware by.
+        versions : The list of versions to filter the firmware by.
+        clean : A flag indicating whether to perform a clean check.
+
+    Returns:
+        List[FirmwareInfo]: A list of unique firmware information.
+
+    """
+
+    log.trace("Checking MicroPython download pages")
+    preview = "preview" in versions
+    board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
 
     log.debug(f"Total {len(board_urls)} firmwares")
     relevant = [
         board
         for board in board_urls
-        if board["board"] in board_list
-        and (board["version"] in version_list or board["preview"] and preview)
+        if board["board"] in boards and (board["version"] in versions or board["preview"] and preview)
         # and b["port"] in ["esp32", "rp2"]
     ]
     log.debug(f"Matching firmwares: {len(relevant)}")
     # select the unique boards
     unique_boards: List[FirmwareInfo] = []
-    for _, g in itertools.groupby(relevant, key=key_fw_variant):
+    for _, g in itertools.groupby(relevant, key=key_fw_var_pre_ext):
         # list is aleady sorted by build so we can just get the last item
         sub_list = list(g)
         unique_boards.append(sub_list[-1])
@@ -225,61 +239,39 @@ def get_firmware_list(
     return unique_boards
 
 
-@cli.command(
-    "download",
-    help="Download MicroPython firmware for specific ports, boards and versions.",
-)
-@click.option(
-    "--destination",
-    "-d",
-    type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
-    default="./firmware",
-    show_default=True,
-    help="The folder to download the firmware to.",
-)
-@click.option(
-    "--version",
-    "-v",
-    "versions",
-    multiple=True,
-    help="The version of MicroPython to to download. Use 'preview' to include preview versions.",
-    show_default=True,
-    default=["stable"],
-)
-@click.option(
-    "--board",
-    "-b",
-    "boards",
-    multiple=True,
-    show_default=True,
-    help="The board(s) to download the firmware for.",  # Use '--board all' to download all boards.",
-)
-@click.option(
-    "--clean/--no-clean",
-    default=True,
-    show_default=True,
-    help="""Remove dates and hashes from the downloaded firmware filenames.""",
-)
-@click.option(
-    "--force",
-    default=False,
-    is_flag=True,
-    help="""Force download of firmware even if it already exists.""",
-    show_default=True,
-)
 def download(
-    destination: Path, boards: List[str], versions: List[str], force: bool, clean: bool
+    destination: Path,
+    ports: List[str],
+    boards: List[str],
+    versions: List[str],
+    force: bool,
+    clean: bool,
 ):
-    versions = list(versions)
-    # preview is not a version, it is an option to include preview versions
-    preview = "preview" in versions
-    versions = [v for v in versions if v != "preview"]
-
-    boards = list(boards) or DEFAULT_BOARDS
-    versions = [
-        clean_version(v, drop_v=True) for v in versions
-    ]  # remove leading v from version
-    destination.mkdir(exist_ok=True)
-    download_firmwares(
-        destination, boards, versions, preview=preview, force=force, clean=clean
-    )
+    """
+    Downloads firmware files based on the specified destination, ports, boards, versions, force flag, and clean flag.
+
+    Args:
+        destination : The destination folder to save the downloaded firmware files.
+        ports : The list of ports to check for firmware.
+        boards : The list of boards to download firmware for.
+        versions : The list of versions to download firmware for.
+        force : A flag indicating whether to force the download even if the firmware file already exists.
+        clean : A flag indicating whether to perform a clean download.
+
+    Returns:
+        None
+
+    Raises:
+        SystemExit: If no boards are found or specified.
+
+    """
+    if not boards:
+        log.critical("No boards found, please connect a board or specify boards to download firmware for.")
+        exit(1)
+    # versions = [clean_version(v, drop_v=True) for v in versions]  # remove leading v from version
+    try:
+        destination.mkdir(exist_ok=True, parents=True)
+    except (PermissionError, FileNotFoundError) as e:
+        log.critical(f"Could not create folder {destination}\n{e}")
+        exit(1)
+    download_firmwares(destination, ports, boards, versions, force=force, clean=clean)

mpflash/mpflash/downloaded.py

@@ -0,0 +1,108 @@
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import jsonlines
+from loguru import logger as log
+
+from mpflash.common import FWInfo
+from mpflash.vendor.versions import clean_version
+
+from .config import config
+
+
+# #########################################################################################################
+def downloaded_firmwares(fw_folder: Path) -> List[FWInfo]:
+    """Load a list of locally downloaded firmwares from the jsonl file"""
+    firmwares: List[FWInfo] = []
+    try:
+        with jsonlines.open(fw_folder / "firmware.jsonl") as reader:
+            firmwares.extend(iter(reader))
+    except FileNotFoundError:
+        log.error(f"No firmware.jsonl found in {fw_folder}")
+    # sort by filename
+    firmwares.sort(key=lambda x: x["filename"])
+    return firmwares
+
+
+def find_downloaded_firmware(
+    *,
+    board_id: str,
+    version: str = "",
+    port: str = "",
+    variants: bool = False,
+    fw_folder: Optional[Path] = None,
+    trie: int = 1,
+    selector: Optional[Dict[str, str]] = None,
+) -> List[FWInfo]:
+    if selector is None:
+        selector = {}
+    fw_folder = fw_folder or config.firmware_folder
+    # Use the information in firmwares.jsonl to find the firmware file
+    fw_list = downloaded_firmwares(fw_folder)
+    if not fw_list:
+        log.error("No firmware files found. Please download the firmware first.")
+        return []
+    # filter by version
+    version = clean_version(version, drop_v=True)
+    fw_list = filter_downloaded_fwlist(fw_list, board_id, version, port, variants, selector)
+
+    if not fw_list and trie < 3:
+        log.info(f"Try ({trie+1}) to find a firmware for the board {board_id}")
+        if trie == 1:
+            # ESP board naming conventions have changed by adding a PORT refix
+            if port.startswith("esp") and not board_id.startswith(port.upper()):
+                board_id = f"{port.upper()}_{board_id}"
+            # RP2 board naming conventions have changed by adding a _RPIprefix
+            if port == "rp2" and not board_id.startswith("RPI_"):
+                board_id = f"RPI_{board_id}"
+        elif trie == 2:
+            board_id = board_id.replace("_", "-")
+
+        fw_list = find_downloaded_firmware(
+            fw_folder=fw_folder,
+            board_id=board_id,
+            version=version,
+            port=port,
+            trie=trie + 1,
+            selector=selector,
+        )
+        # hope we have a match now for the board
+    # sort by filename
+    fw_list.sort(key=lambda x: x["filename"])
+    return fw_list
+
+
+def filter_downloaded_fwlist(
+    fw_list: List[FWInfo],
+    board_id: str,
+    version: str,
+    port: str,
+    # preview: bool,
+    variants: bool,
+    selector: dict,
+) -> List[FWInfo]:
+    """Filter the downloaded firmware list based on the provided parameters"""
+    if "preview" in version:
+        # never get a preview for an older version
+        fw_list = [fw for fw in fw_list if fw["preview"]]
+    else:
+        fw_list = [fw for fw in fw_list if fw["version"] == version]
+
+    # filter by port
+    if port:
+        fw_list = [fw for fw in fw_list if fw["port"] == port]
+
+    if board_id:
+        if variants:
+            # any variant of this board_id
+            fw_list = [fw for fw in fw_list if fw["board"] == board_id]
+        else:
+            # the firmware variant should match exactly the board_id
+            fw_list = [fw for fw in fw_list if fw["variant"] == board_id]
+    if selector and port in selector:
+        fw_list = [fw for fw in fw_list if fw["filename"].endswith(selector[port])]
+    return fw_list
+
+
+# #########################################################################################################
+#

mpflash/mpflash/errors.py

@@ -0,0 +1,5 @@
+"""Custom exceptions for the MPFlash package."""
+
+
+class MPFlashError(Exception):
+    pass