mpflash 1.25.0.post1__py3-none-any.whl → 1.25.0rc1__py3-none-any.whl

mpflash/download/fwinfo.py

@@ -0,0 +1,41 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class FWInfo:
+    """
+    Downloaded Firmware information
+    is somewhat related to the BOARD class in the mpboard_id module
+    """
+
+    port: str  # MicroPython port
+    board: str  # MicroPython board
+    filename: str = field(default="")  # relative filename of the firmware image
+    url: str = field(default="")  # url or path to original firmware image
+    variant: str = field(default="")  # MicroPython variant
+    preview: bool = field(default=False)  # True if the firmware is a preview version
+    version: str = field(default="")  # MicroPython version (NO v prefix)
+    url: str = field(default="")  # url to the firmware image download folder
+    build: str = field(default="0")  # The build = number of commits since the last release
+    ext: str = field(default="")  # the file extension of the firmware
+    family: str = field(default="micropython")  # The family of the firmware
+    custom: bool = field(default=False)  # True if the firmware is a custom build
+    description: str = field(default="")  # Description used by this firmware (custom only)
+    mcu: str = field(default="")
+
+    def to_dict(self) -> dict:
+        """Convert the object to a dictionary"""
+        return self.__dict__
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "FWInfo":
+        """Create a FWInfo object from a dictionary"""
+        valid_keys = {field.name for field in FWInfo.__dataclass_fields__.values()}
+        filtered_data = {k: v for k, v in data.items() if k in valid_keys}
+        # add missing keys
+        if "ext" not in data:
+            data["ext"] = Path(data["filename"]).suffix
+        if "family" not in data:
+            data["family"] = "micropython"
+        return FWInfo(**filtered_data)        

mpflash/download/jid.py

@@ -0,0 +1,53 @@
+# Just In-time Download of firmware if not already available
+from loguru import logger as log
+
+from mpflash.common import Params
+from mpflash.download import download
+from mpflash.downloaded import find_downloaded_firmware
+from mpflash.errors import MPFlashError
+from mpflash.flash.worklist import WorkList
+from mpflash.mpboard_id.alternate import alternate_board_names
+
+
+def ensure_firmware_downloaded(worklist: WorkList, version: str) -> None:
+    """
+    Ensure all firmware in the worklist is downloaded for the given version.
+
+    Iterates over the worklist, downloads missing firmware, and updates the worklist
+    with the downloaded firmware.
+
+    Raises MPFlashError if download fails.
+    """
+    # iterate over the worklist ann update missing firmware
+    newlist: WorkList = []
+    for mcu, firmware in worklist:
+        if firmware:
+            # firmware is already downloaded
+            newlist.append((mcu, firmware))
+            continue
+        # check if the firmware is already downloaded
+        board_firmwares = find_downloaded_firmware(
+            board_id=f"{mcu.board}-{mcu.variant}" if mcu.variant else mcu.board,
+            version=version,
+            port=mcu.port,
+        )
+        if not board_firmwares:
+            # download the firmware
+            log.info(f"Downloading {version} firmware for {mcu.board} on {mcu.serialport}.")
+            download(ports=[mcu.port], boards=alternate_board_names(mcu.board, mcu.port), versions=[version], force=True, clean=True)
+            new_firmware = find_downloaded_firmware(
+                board_id=f"{mcu.board}-{mcu.variant}" if mcu.variant else mcu.board,
+                version=version,
+                port=mcu.port,
+            )
+            if not new_firmware:
+                raise MPFlashError(f"Failed to download {version} firmware for {mcu.board} on {mcu.serialport}.")
+            newlist.append((mcu, new_firmware[0]))
+        else:
+            log.info(f"Found {version} firmware {board_firmwares[-1].firmware_file} for {mcu.board} on {mcu.serialport}.")
+            newlist.append((mcu, firmware))
+
+    worklist.clear()
+    worklist.extend(newlist)
+
+    pass

mpflash/downloaded.py

@@ -1,113 +1,99 @@
+import os
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import List
 
-import jsonlines
 from loguru import logger as log
 
-from mpflash.common import PORT_FWTYPES, FWInfo
-from mpflash.db.downloads import downloaded
+from mpflash.config import config
+from mpflash.db.core import Session
+from mpflash.db.models import Firmware
+from mpflash.mpboard_id.alternate import alternate_board_names
 from mpflash.versions import clean_version
 
-from .config import config
-
 # #########################################################################################################
 
 
-def clean_downloaded_firmwares(fw_folder: Path) -> None:
+def clean_downloaded_firmwares() -> None:
+    """
+    - Check if all firmware records in the database are still available on disk.
+        - If not, remove the record from the database.
+    - For all firmware files on disk that are not in the database:
+        - loag a warning message.
+        - Check if the file is a valid firmware file.
+        - If so, add it to the database.
+
     """
-    Remove duplicate entries from the firmware.jsonl file, keeping the latest one
-    uniqueness is based on the filename
     """
-    # Duplication should no longer happen,
-    # but is would be a good idea to check the consistence between the DB and the downloads folder sometimes 
-    pass 
+    Get all firmware files in the firmware directory and its subfolders.
+    """
 
+    firmware_dir = Path(config.firmware_folder)
 
-def find_downloaded_firmware(
-    *,
-    board_id: str,
-    version: str = "",  # v1.2.3
-    port: str = "",
-    variants: bool = False,
-    db_path: Optional[Path] = None,
-    trie: int = 1,
-    selector: Optional[Dict[str, str]] = None,
-) -> List[FWInfo]:
-    if selector is None:
-        selector = {}
-    
-    # Use the information in firmwares.jsonl to find the firmware file
-    log.debug(f"{trie}] Looking for firmware for {board_id} {version} ")
-    fw_list = downloaded()
-    if not fw_list:
-        log.error("No firmware files found. Please download the firmware first.")
-        return []
-    # filter by version
-    version = clean_version(version)
-    fw_list = filter_downloaded_fwlist(fw_list, board_id, version, port, variants, selector)
+    """
+    Returns a set of firmware file paths (relative to firmware_dir) found on disk.
+    Uses a generator for performance and includes files in subfolders.
+    Skips files with certain extensions.
+    """
+    firmware_files_on_disk = {
+        str(f.relative_to(firmware_dir)) for f in firmware_dir.rglob("*") if f.is_file() and f.suffix not in {".db", ".bak", ".jsonl"}
+    }
 
-    if not fw_list and trie < 3:
-        log.info(f"Try ({trie+1}) to find a firmware for the board {board_id}")
-        if trie == 1:
-            # ESP board naming conventions have changed by adding a PORT prefix
-            if port.startswith("esp") and not board_id.startswith(port.upper()):
-                board_id = f"{port.upper()}_{board_id}"
-            # RP2 board naming conventions have changed by adding a _RPI prefix
-            if port == "rp2" and not board_id.startswith("RPI_"):
-                board_id = f"RPI_{board_id}"
-        elif trie == 2:
-            board_id = board_id.replace("_", "-")
+    with Session() as session:
+        db_firmwares = session.query(Firmware).all()
+        db_firmware_files = {fw.firmware_file for fw in db_firmwares}
 
-        fw_list = find_downloaded_firmware(
-            db_path=db_path,
-            board_id=board_id,
-            version=version,
-            port=port,
-            trie=trie + 1,
-            selector=selector,
-        )
-        # hope we have a match now for the board
-    # sort by filename
-    fw_list.sort(key=lambda x: x.filename)
-    return fw_list
+        # Remove DB records for files not on disk
+        for fw in db_firmwares:
+            if fw.firmware_file not in firmware_files_on_disk:
+                log.warning(f"Firmware file missing on disk, removing DB record: {fw.firmware_file}")
+                session.delete(fw)
+        session.commit()
 
+        # Warn about files on disk not in DB
+        for fw_file in firmware_files_on_disk - db_firmware_files:
+            log.warning(f"Found file in firmware folder but not in DB: {fw_file}")
 
-def filter_downloaded_fwlist(
-    fw_list: List[FWInfo],
-    board_id: str,
-    version: str,  # v1.2.3
-    port: str,
-    # preview: bool,
-    variants: bool,
-    selector: dict,
-) -> List[FWInfo]:
-    """Filter the downloaded firmware list based on the provided parameters"""
-    if "preview" in version:
-        # never get a preview for an older version
-        fw_list = [fw for fw in fw_list if fw.preview]
-    else:
-        # older FWInfo version has no v1.2.3 prefix
-        either = [clean_version(version, drop_v=False), clean_version(version, drop_v=True)]
-        fw_list = [fw for fw in fw_list if fw.version in either]
-    log.trace(f"Filtering firmware for {version} : {len(fw_list)} found.")
-    # filter by port
-    if port:
-        fw_list = [fw for fw in fw_list if fw.port == port and Path(fw.firmware).suffix in PORT_FWTYPES[port]]
-        log.trace(f"Filtering firmware for {port} : {len(fw_list)} found.")
 
-    if board_id:
-        if variants:
-            # any variant of this board_id
-            fw_list = [fw for fw in fw_list if fw.board == board_id]
+def find_downloaded_firmware(
+    board_id: str,
+    version: str = "",
+    port: str = "",
+    variants: bool = False,
+) -> List[Firmware]:
+    version = clean_version(version)
+    log.debug(f"Looking for firmware for {board_id} {version} ")
+    # Special handling for preview versions
+    with Session() as session:
+        if version == "preview" or "preview" in version:
+            # Find all preview firmwares for this board/port, return the latest (highest build)
+            query = session.query(Firmware).filter(Firmware.board_id == board_id)
+            if port:
+                query = query.filter(Firmware.port == port)
+            query = query.filter(Firmware.firmware_file.contains("preview")).order_by(Firmware.build.desc())
+            fw_list = query.all()
+            if fw_list:
+                return [fw_list[0]]  # Return the latest preview only
         else:
-            # the firmware variant should match exactly the board_id
-            fw_list = [fw for fw in fw_list if fw.variant == board_id]
-        log.trace(f"Filtering firmware for {board_id} : {len(fw_list)} found.")
-
-    if selector and port in selector:
-        fw_list = [fw for fw in fw_list if fw.filename.endswith(selector[port])]
-    return fw_list
-
+            fw_list = session.query(Firmware).filter(Firmware.board_id == board_id, Firmware.version == version).all()
+            if fw_list:
+                return fw_list
+    #
+    more_board_ids = alternate_board_names(board_id, port)
+    #
+    log.debug(f"2nd search with renamed board_id :{board_id}")
+    with Session() as session:
+        if version == "preview" or "preview" in version:
+            query = session.query(Firmware).filter(Firmware.board_id.in_(more_board_ids))
+            if port:
+                query = query.filter(Firmware.port == port)
+            query = query.filter(Firmware.firmware_file.contains("preview")).order_by(Firmware.build.desc())
+            fw_list = query.all()
+            if fw_list:
+                return [fw_list[0]]
+        else:
+            fw_list = session.query(Firmware).filter(Firmware.board_id.in_(more_board_ids), Firmware.version == version).all()
+            if fw_list:
+                return fw_list
+    log.warning(f"No firmware files found for board {board_id} version {version}")
+    return []
 
-# #########################################################################################################
-#

mpflash/flash/__init__.py

@@ -3,7 +3,8 @@ from pathlib import Path
 from loguru import logger as log
 
 from mpflash.bootloader.activate import enter_bootloader
-from mpflash.common import PORT_FWTYPES, BootloaderMethod, UF2_PORTS
+from mpflash.common import PORT_FWTYPES, UF2_PORTS, BootloaderMethod
+from mpflash.config import config
 from mpflash.errors import MPFlashError
 
 from .esp import flash_esp
@@ -15,7 +16,6 @@ from .worklist import WorkList
 
 def flash_list(
     todo: WorkList,
-    fw_folder: Path,
     erase: bool,
     bootloader: BootloaderMethod,
     **kwargs
@@ -23,7 +23,11 @@ def flash_list(
     """Flash a list of boards with the specified firmware."""
     flashed = []
     for mcu, fw_info in todo:
-        fw_file = fw_folder / fw_info.filename
+        if not fw_info:
+            log.error(f"Firmware not found for {mcu.board} on {mcu.serialport}, skipping")
+            continue
+
+        fw_file = config.firmware_folder / fw_info.firmware_file
         if not fw_file.exists():
             log.error(f"File {fw_file} does not exist, skipping {mcu.board} on {mcu.serialport}")
             continue

mpflash/flash/esp.py

@@ -15,6 +15,7 @@ from mpflash.mpremoteboard import MPRemoteBoard
 
 FlashMode = Literal["keep", "qio", "qout", "dio", "dout"]
 
+
 def flash_esp(
     mcu: MPRemoteBoard,
     fw_file: Path,
@@ -30,7 +31,7 @@ def flash_esp(
     log.info(f"Flashing {fw_file} on {mcu.board} on {mcu.serialport}")
     if not mcu.cpu:
         # Lookup CPU based on the board name
-        mcu.cpu = find_known_board(mcu.board).cpu
+        mcu.cpu = find_known_board(mcu.board).mcu
 
     cmds: List[List[str]] = []
 

mpflash/flash/worklist.py

@@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Tuple
 
 from loguru import logger as log
 
-from mpflash.common import FWInfo, filtered_comports
+from mpflash.common import filtered_comports
+from mpflash.db.models import Firmware
 from mpflash.downloaded import find_downloaded_firmware
 from mpflash.errors import MPFlashError
 from mpflash.list import show_mcus
@@ -13,45 +14,38 @@ from mpflash.mpboard_id import find_known_board
 from mpflash.mpremoteboard import MPRemoteBoard
 
 # #########################################################################################################
-WorkList = List[Tuple[MPRemoteBoard, FWInfo]]
+WorkList = List[Tuple[MPRemoteBoard, Optional[Firmware]]]
 # #########################################################################################################
 
 
 def auto_update(
     conn_boards: List[MPRemoteBoard],
     target_version: str,
-    fw_folder: Path,
-    *,
-    selector: Optional[Dict[str, str]] = None,
 ) -> WorkList:
     """Builds a list of boards to update based on the connected boards and the firmwares available locally in the firmware folder.
 
     Args:
         conn_boards (List[MPRemoteBoard]): List of connected boards
         target_version (str): Target firmware version
-        fw_folder (Path): Path to the firmware folder
         selector (Optional[Dict[str, str]], optional): Selector for filtering firmware. Defaults to None.
 
     Returns:
         WorkList: List of boards and firmware information to update
     """
-    if selector is None:
-        selector = {}
     wl: WorkList = []
     for mcu in conn_boards:
         if mcu.family not in ("micropython", "unknown"):
             log.warning(f"Skipping flashing {mcu.family} {mcu.port} {mcu.board} on {mcu.serialport} as it is not a MicroPython firmware")
             continue
         board_firmwares = find_downloaded_firmware(
-            db_path=fw_folder,
-            board_id=mcu.board if not mcu.variant else f"{mcu.board}-{mcu.variant}",
+            board_id=f"{mcu.board}-{mcu.variant}" if mcu.variant else mcu.board,
             version=target_version,
             port=mcu.port,
-            selector=selector,
         )
 
         if not board_firmwares:
-            log.error(f"No {target_version} firmware found for {mcu.board} on {mcu.serialport}.")
+            log.warning(f"No {target_version} firmware found for {mcu.board} on {mcu.serialport}.")
+            wl.append((mcu, None))
             continue
 
         if len(board_firmwares) > 1:
@@ -59,7 +53,7 @@ def auto_update(
 
         # just use the last firmware
         fw_info = board_firmwares[-1]
-        log.info(f"Found {target_version} firmware {fw_info.filename} for {mcu.board} on {mcu.serialport}.")
+        log.info(f"Found {target_version} firmware {fw_info.firmware_file} for {mcu.board} on {mcu.serialport}.")
         wl.append((mcu, fw_info))
     return wl
 
@@ -69,7 +63,6 @@ def manual_worklist(
     *,
     board_id: str,
     version: str,
-    fw_folder: Path,
 ) -> WorkList:
     """Create a worklist for a single board specified manually.
 
@@ -77,7 +70,6 @@ def manual_worklist(
         serial (str): Serial port of the board
         board (str): Board_ID
         version (str): Firmware version
-        fw_folder (Path): Path to the firmware folder
 
     Returns:
         WorkList: List of boards and firmware information to update
@@ -89,13 +81,13 @@ def manual_worklist(
         info = find_known_board(board_id)
         mcu.port = info.port
         # need the CPU type for the esptool
-        mcu.cpu = info.cpu
+        mcu.cpu = info.mcu
     except (LookupError, MPFlashError) as e:
         log.error(f"Board {board_id} not found in board database")
         log.exception(e)
         return []
     mcu.board = board_id
-    firmwares = find_downloaded_firmware(db_path=fw_folder, board_id=board_id, version=version, port=mcu.port)
+    firmwares = find_downloaded_firmware(board_id=board_id, version=version, port=mcu.port)
     if not firmwares:
         log.error(f"No firmware found for {mcu.port} {board_id} version {version}")
         return []
@@ -107,32 +99,29 @@ def single_auto_worklist(
     serial: str,
     *,
     version: str,
-    fw_folder: Path,
 ) -> WorkList:
     """Create a worklist for a single serial-port.
 
     Args:
         serial_port (str): Serial port of the board
         version (str): Firmware version
-        fw_folder (Path): Path to the firmware folder
 
     Returns:
         WorkList: List of boards and firmware information to update
     """
     log.trace(f"Auto updating {serial} to {version}")
     conn_boards = [MPRemoteBoard(serial)]
-    todo = auto_update(conn_boards, version, fw_folder)  # type: ignore # List / list
-    show_mcus(conn_boards)  # type: ignore
+    todo = auto_update(conn_boards, version)  # type: ignore # List / list
+    show_mcus(conn_boards)
     return todo
 
 
 def full_auto_worklist(
     all_boards: List[MPRemoteBoard],
-     *, 
-     include: List[str], 
-     ignore: List[str], 
-     version: str, 
-     fw_folder: Path, 
+    *,
+    include: List[str],
+    ignore: List[str],
+    version: str,
 ) -> WorkList:
     """
     Create a worklist for all connected micropython boards based on the information retrieved from the board.
@@ -140,14 +129,13 @@ def full_auto_worklist(
 
     Args:
         version (str): Firmware version
-        fw_folder (Path): Path to the firmware folder
 
     Returns:
         WorkList: List of boards and firmware information to update
     """
     log.trace(f"Auto updating all boards to {version}")
     if selected_boards := filter_boards(all_boards, include=include, ignore=ignore):
-        return auto_update(selected_boards, version, fw_folder)
+        return auto_update(selected_boards, version)
     else:
         return []
 

mpflash/list.py

@@ -16,7 +16,7 @@ def show_mcus(
     refresh: bool = True,
     *,
     title_style="magenta",
-    header_style="bold magenta",    
+    header_style="bold magenta",
 ):
     console.print(mcu_table(conn_mcus, title, refresh, title_style=title_style, header_style=header_style))
 
@@ -38,7 +38,7 @@ def mcu_table(
 ):
     """
     builds a rich table with the connected boards information
-    The columns of the table are adjusted to the terminal width > 90 
+    The columns of the table are adjusted to the terminal width > 90
     the columns are :
                 Narrow      Wide
     - Serial    Yes         Yes
@@ -83,7 +83,7 @@ def mcu_table(
         table.add_column("Port")
     table.add_column("Board", overflow="fold")
     if needs_variant:
-        table.add_column("Variant") 
+        table.add_column("Variant")
     if is_wide:
         table.add_column("CPU")
     table.add_column("Version", overflow="fold", min_width=5, max_width=16)

mpflash/logger.py

@@ -11,8 +11,8 @@ console = Console()
 def _log_formatter(record: dict) -> str:
     """Log message formatter to combine loguru and rich formatting."""
     color_map = {
-        "TRACE": "dim blue",
-        "DEBUG": "cyan",
+        "TRACE": "cyan",
+        "DEBUG": "orange",
         "INFO": "bold",
         "SUCCESS": "bold green",
         "WARNING": "yellow",

mpflash/mpboard_id/__init__.py

@@ -3,15 +3,9 @@ Access to the micropython port and board information that is stored in the board
 that is included in the module.
 
 """
-
-from functools import lru_cache
-from typing import List, Optional, Tuple
-import importlib
 from mpflash.errors import MPFlashError
-from .board import Board
-
 from mpflash.versions import clean_version
-from .store import read_known_boardinfo
-from .known import get_known_ports, get_known_boards_for_port
-from .known import known_stored_boards, find_known_board
 
+from .known import (find_known_board, get_known_boards_for_port, known_ports,
+                    known_stored_boards)
+from .resolve import resolve_board_ids

mpflash/mpboard_id/alternate.py

@@ -0,0 +1,56 @@
+from typing import Dict, List, Optional
+
+from loguru import logger as log
+
+
+def alternate_board_names(board_id, port="") -> List[str]:
+    more = [board_id]
+
+    log.debug("try for renamed board_id")
+
+    if board_id.startswith("PICO"):
+        more.append(board_id.replace("PICO", "RPI_PICO"))
+    elif board_id.startswith("RPI_"):
+        more.append(board_id.replace("RPI_", ""))
+    elif board_id.startswith("GENERIC"):
+        if port:
+            more.append(board_id.replace("GENERIC", f"{port.upper()}_GENERIC"))
+        else:
+            # just add both of them
+            more.append(board_id.replace("GENERIC", f"ESP32_GENERIC"))
+            more.append(board_id.replace("GENERIC", f"ESP8266_GENERIC"))
+    elif board_id.startswith("ESP32_"):
+        more.append(board_id.replace("ESP32_", ""))
+    elif board_id.startswith("ESP8266_"):
+        more.append(board_id.replace("ESP8266_", ""))
+
+    # VARIANT
+    variant_suffixes = ["SPIRAM", "THREAD"]
+    for board in more:
+        if any(suffix in board for suffix in variant_suffixes):
+            for suffix in variant_suffixes:
+                if board.endswith(f"_{suffix}"):
+                    more.append(board.replace(f"_{suffix}", ""))
+                    # more.append(board.replace(f"_{suffix}", f"-{suffix}"))
+                    break  # first one found
+
+    return more
+
+
+def add_renamed_boards(boards: List[str]) -> List[str]:
+    """
+    Adds the renamed boards to the list of boards.
+
+    Args:
+        boards : The list of boards to add the renamed boards to.
+
+    Returns:
+        List[str]: The list of boards with the renamed boards added.
+    """
+
+    _boards = boards.copy()
+    for board in boards:
+        _boards.extend(alternate_board_names(board))
+        if board != board.upper():
+            _boards.extend(alternate_board_names(board.upper()))
+    return _boards