mpflash 1.25.1__py3-none-any.whl → 1.25.2__py3-none-any.whl

mpflash/db/core.py

@@ -2,7 +2,8 @@ from pathlib import Path
 from sqlite3 import DatabaseError, OperationalError
 
 from loguru import logger as log
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, text
+from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import sessionmaker
 
 from mpflash.config import config
@@ -13,18 +14,90 @@ from .models import Base
 
 TRACE = False
 connect_str = f"sqlite:///{config.db_path.as_posix()}"
+log.debug(f"Connecting to database at {connect_str}")
 engine = create_engine(connect_str, echo=TRACE)
 Session = sessionmaker(bind=engine)
 
 
+def get_schema_version() -> int:
+    """Get current database schema version."""
+    from .meta import get_metadata_value
+
+    version = get_metadata_value("schema_version")
+    return int(version) if version else 0
+
+
+def set_schema_version(version: int) -> None:
+    """Set database schema version."""
+    from .meta import set_metadata_value
+
+    set_metadata_value("schema_version", str(version))
+
+
+def migration_001_add_custom_id() -> None:
+    """Add custom_id column to firmwares table."""
+    log.info("Running migration 001: Add custom_id column to firmwares table")
+
+    with Session() as session:
+        # Check if column already exists
+        result = session.execute(text("PRAGMA table_info(firmwares)")).fetchall()
+
+        columns = [row[1] for row in result]
+        if "custom_id" not in columns:
+            # Add column without UNIQUE constraint
+            session.execute(text("ALTER TABLE firmwares ADD COLUMN custom_id VARCHAR(40)"))
+
+            # Create regular index for performance
+            session.execute(text("CREATE INDEX IF NOT EXISTS idx_firmwares_custom_id ON firmwares(custom_id)"))
+
+            session.commit()
+            log.info("Added custom_id column and index to firmwares table")
+        else:
+            log.info("custom_id column already exists in firmwares table")
+
+            # Check if index exists and create if needed
+            result = session.execute(text("PRAGMA index_list(firmwares)")).fetchall()
+
+            index_names = [row[1] for row in result]
+            if "idx_firmwares_custom_id" not in index_names:
+                session.execute(text("CREATE INDEX IF NOT EXISTS idx_firmwares_custom_id ON firmwares(custom_id)"))
+                session.commit()
+                log.info("Added index to existing custom_id column")
+
+
+def run_schema_migrations() -> None:
+    """Run all pending database schema migrations."""
+    current_version = get_schema_version()
+    target_version = 1  # Current latest version
+
+    if current_version >= target_version:
+        log.debug(f"Database schema is up to date (version {current_version})")
+        return
+
+    log.info(f"Upgrading database schema from version {current_version} to {target_version}")
+
+    try:
+        # Run migrations in order
+        if current_version < 1:
+            migration_001_add_custom_id()
+
+        # Update schema version
+        set_schema_version(target_version)
+        log.info(f"Database schema upgraded to version {target_version}")
+
+    except SQLAlchemyError as e:
+        log.error(f"Migration failed: {e}")
+        raise MPFlashError(f"Failed to migrate database schema: {e}")
+
+
 def migrate_database(boards: bool = True, firmwares: bool = True):
-    """Migrate from 1.24.x to 1.25.x"""
+    """Migrate from 1.24.x to 1.25.x and run schema migrations."""
     # Move import here to avoid circular import
     from .loader import load_jsonl_to_db, update_boards
 
     # get the location of the database from the session
     with Session() as session:
-        db_location = session.get_bind().url.database # type: ignore
+        db_location = session.get_bind().url.database  # type: ignore
         log.debug(f"Database location: {Path(db_location)}")  # type: ignore
 
     try:
@@ -33,6 +106,10 @@ def migrate_database(boards: bool = True, firmwares: bool = True):
         log.error(f"Error creating database: {e}")
         log.error("Database might already exist, trying to migrate.")
         raise MPFlashError("Database migration failed. Please check the logs for more details.") from e
+
+    # Run schema migrations after creating database
+    run_schema_migrations()
+
     if boards:
         update_boards()
     if firmwares:
@@ -54,8 +131,9 @@ def migrate_database(boards: bool = True, firmwares: bool = True):
 
 
 def create_database():
-    """
-    Create the SQLite database and tables if they don't exist.
-    """
+    """Create the SQLite database and tables if they don't exist."""
     # Create the database and tables if they don't exist
     Base.metadata.create_all(engine)
+
+    # Run schema migrations after table creation
+    run_schema_migrations()

mpflash/db/gather_boards.py

@@ -1,11 +1,11 @@
 from os import path
 from pathlib import Path
-from mpflash.mpremoteboard import HERE
-from mpflash.vendor.board_database import Database
-from mpflash.logger import log
-
 from typing import List
+
 import mpflash.basicgit as git
+from mpflash.logger import log
+from mpflash.mpremoteboard import HERE
+from mpflash.vendor.board_database import Database
 from mpflash.versions import micropython_versions
 
 HERE = Path(__file__).parent.resolve()
@@ -79,6 +79,7 @@ def create_zip_file(longlist, zip_file: Path):
     """Create a ZIP file containing the CSV data."""
     # lazy import
     import zipfile
+
     import pandas as pd
 
     csv_filename = "micropython_boards.csv"
@@ -96,6 +97,7 @@ def create_zip_file(longlist, zip_file: Path):
 
 def package_repo(mpy_path: Path):
     mpy_path = mpy_path or Path("../repos/micropython")
+    log.info(f"Packaging Micropython boards from {mpy_path}")
     mp_versions = micropython_versions(minver="1.18")
     longlist = boardlist_from_repo(
         versions=mp_versions,

mpflash/db/loader.py

@@ -24,7 +24,7 @@ def load_data_from_zip(zip_file: Path) -> int:
     if not zip_file.exists() or not zip_file.is_file():
         log.error(f"Zip file {zip_file} not found.")
         return 0
-    count = 0 
+    count = 0
     # Load data directly from the zip file
     with zipfile.ZipFile(zip_file, "r") as zipf:
         # Read the CSV file from the zip
@@ -109,14 +109,15 @@ def load_jsonl_to_db(jsonl_path: Path):
 
 
 def update_boards():
+    boards_version = "v1.25.2"
     try:
         meta = get_metadata()
         log.debug(f"Metadata: {meta}")
-        if meta.get("boards_version", "") < "v1.25.0":
+        if meta.get("boards_version", "") < boards_version:
             log.info("Update boards from CSV to SQLite database.")
             # Load data from the zip file into the database
             load_data_from_zip(HERE / "micropython_boards.zip")
-            set_metadata_value("boards_version", "v1.25.0")
+            set_metadata_value("boards_version", boards_version)
             meta = get_metadata()
     except Exception as e:
         raise MPFlashError(f"Error updating boards table: {e}") from e

mpflash/db/meta.py

@@ -1,4 +1,3 @@
-
 from typing import Optional
 
 from loguru import logger as log
@@ -18,10 +17,11 @@ def get_metadata() -> dict:
         with Session() as session:
             metadata = session.query(Metadata).all()
             return {m.name: m.value for m in metadata}
-    except (DatabaseError, OperationalError)  as e:
+    except (DatabaseError, OperationalError) as e:
         log.error(f"Error retrieving metadata: {e}")
         return {}
-    
+
+
 def set_metadata(metadata: dict):
     """
     Set metadata in the database.
@@ -56,6 +56,7 @@ def get_metadata_value(name: str) -> Optional[str]:
         metadata = session.query(Metadata).filter(Metadata.name == name).first()
     return metadata.value if metadata else None
 
+
 def set_metadata_value(name: str, value: str):
     """
     Set metadata value by name.

mpflash/db/models.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Union
 
 import sqlalchemy as sa
 from sqlalchemy import Index, String
@@ -78,6 +79,7 @@ class Firmware(Base):
     source: Mapped[str] = mapped_column()
     build: Mapped[int] = mapped_column(default=0, comment="Build number")
     custom: Mapped[bool] = mapped_column(default=False, comment="True if this is a custom firmware")
+    custom_id: Mapped[Union[str, None]] = mapped_column(String(40), nullable=True, default=None)
 
     @property
     def preview(self) -> bool:

mpflash/download/__init__.py

@@ -4,6 +4,7 @@ Uses the micropython.org website to get the available versions and locations to
 """
 
 import itertools
+import re
 from pathlib import Path
 from typing import Dict, List, Optional
 
@@ -16,7 +17,7 @@ from rich.progress import track
 from mpflash.common import PORT_FWTYPES
 from mpflash.config import config
 from mpflash.db.core import Session
-from mpflash.db.models import Firmware
+from mpflash.db.models import Firmware, Board
 from mpflash.downloaded import clean_downloaded_firmwares
 from mpflash.errors import MPFlashError
 from mpflash.mpboard_id.alternate import add_renamed_boards
@@ -36,6 +37,56 @@ def key_fw_boardid_preview_ext(fw: Firmware):
     return fw.board_id, fw.preview, fw.ext
 
 
+
+
+# Cache for variant suffixes - will be populated on first use
+_PATTERN = ""
+
+def _get_variant_pattern():
+    """
+    Query the database for all known variant suffixes.
+    This is done only once and the result is cached.
+    
+    Returns:
+        List of known variant suffixes.
+    """
+    global _PATTERN
+    
+    if _PATTERN:
+        # If the pattern is already set, return it
+        return _PATTERN
+
+    with Session() as session:
+        # Query distinct variants
+        variants = session.query(Board.variant).distinct().all()
+
+
+    _VARIANT_SUFFIXES = [f"_{v[0]}" for v in variants if v[0] and v[0].strip()]
+    # workaround for the fact that the variant names do not always match the suffixes
+    # e.g. 'PIMORONI_PICOLIPO' has the variant 'FLASH_16MB' but the suffix is '_16MB'
+    _VARIANT_SUFFIXES.extend( ["_2MB", "_4MB", "_8MB", "_16MB"] )  # add common SPIRAM size suffixes
+
+    # return _VARIANT_SUFFIXES
+    _PATTERN  = f"({'|'.join(re.escape(v) for v in _VARIANT_SUFFIXES)})$"
+    return _PATTERN
+
+    
+
+def strip_variant(board: str) -> str:
+    """
+    Strips the variant suffix from the board name based on variants in the database.
+    For example, 'RPI_PICO_W_SPIRAM' becomes 'RPI_PICO_W'.
+
+    Args:
+        board: The board name to process.
+
+    Returns:
+        The board name without the variant suffix.
+    """
+    pattern = _get_variant_pattern()
+    return re.sub(pattern, "", board)
+
+
 def download_firmwares(
     firmware_folder: Path,
     ports: List[str],
@@ -60,9 +111,15 @@ def download_firmwares(
 
     downloaded = 0
     versions = [] if versions is None else [clean_version(v) for v in versions]
+    
+    # remove the known variant suffixes from the boards
+    # TODO: IS THIS REALLY NEEDED ?
+    # boards = [strip_variant(b) for b in boards]
+    
     # handle downloading firmware for renamed boards
     boards = add_renamed_boards(boards)
 
+
     available_firmwares = get_firmware_list(ports, boards, versions, clean)
 
     for b in available_firmwares:

mpflash/download/fwinfo.py

@@ -38,4 +38,4 @@ class FWInfo:
             data["ext"] = Path(data["filename"]).suffix
         if "family" not in data:
             data["family"] = "micropython"
-        return FWInfo(**filtered_data)        
+        return FWInfo(**filtered_data)

mpflash/downloaded.py

@@ -59,17 +59,22 @@ def find_downloaded_firmware(
     version: str = "",
     port: str = "",
     variants: bool = False,
+    custom: bool = False,
 ) -> List[Firmware]:
     version = clean_version(version)
     log.debug(f"Looking for firmware for {board_id} {version} ")
     # Special handling for preview versions
     with Session() as session:
-        if version == "preview" or "preview" in version:
+        if "preview" in version:
             # Find all preview firmwares for this board/port, return the latest (highest build)
-            query = session.query(Firmware).filter(Firmware.board_id == board_id)
+            if custom:
+                query = session.query(Firmware).filter(Firmware.custom_id == board_id)
+            else:
+                query = session.query(Firmware).filter(Firmware.board_id == board_id)
             if port:
                 query = query.filter(Firmware.port == port)
             query = query.filter(Firmware.firmware_file.contains("preview")).order_by(Firmware.build.desc())
+            log.trace(f"Querying for preview firmware: {query}")
             fw_list = query.all()
             if fw_list:
                 return [fw_list[0]]  # Return the latest preview only
@@ -82,7 +87,7 @@ def find_downloaded_firmware(
     #
     log.debug(f"2nd search with renamed board_id :{board_id}")
     with Session() as session:
-        if version == "preview" or "preview" in version:
+        if "preview" in version:
             query = session.query(Firmware).filter(Firmware.board_id.in_(more_board_ids))
             if port:
                 query = query.filter(Firmware.port == port)
@@ -96,4 +101,3 @@ def find_downloaded_firmware(
                 return fw_list
     log.warning(f"No firmware files found for board {board_id} version {version}")
     return []
-

mpflash/flash/__init__.py

@@ -39,6 +39,15 @@ def flash_list(
             log.error(f"Failed to flash {mcu.board} on {mcu.serialport}: {e}")
             continue
         if updated:
+            if fw_info.custom:
+                # Add / Update board_info.toml with the custom_id and Description
+                mcu.get_board_info_toml()
+                if fw_info.description:
+                    mcu.toml["description"] = fw_info.description
+                mcu.toml["mpflash"]["board_id"] = fw_info.board_id
+                mcu.toml["mpflash"]["custom_id"] = fw_info.custom_id
+                mcu.set_board_info_toml()
+
             flashed.append(updated)
         else:
             log.error(f"Failed to flash {mcu.board} on {mcu.serialport}")
@@ -51,7 +60,7 @@ def flash_mcu(
         fw_file: Path,
         erase: bool = False,
         bootloader: BootloaderMethod = BootloaderMethod.AUTO,
-        **kwargs        
+        **kwargs
     ):
         """Flash a single MCU with the specified firmware."""
         updated = None

mpflash/flash/worklist.py

@@ -34,6 +34,7 @@ def auto_update_worklist(
     Returns:
         WorkList: List of boards and firmware information to update
     """
+    log.debug(f"auto_update_worklist: {len(conn_boards)} boards, target version: {target_version}")
     wl: WorkList = []
     for mcu in conn_boards:
         if mcu.family not in ("micropython", "unknown"):
@@ -65,12 +66,14 @@ def manual_worklist(
     *,
     board_id: str,
     version: str,
+    custom: bool = False,
 ) -> WorkList:
     """Create a worklist for manually specified boards."""
+    log.debug(f"manual_worklist: {len(serial)} serial ports, board_id: {board_id}, version: {version}")
     wl: WorkList = []
     for comport in serial:
         log.trace(f"Manual updating {comport} to {board_id} {version}")
-        wl.append(manual_board(comport, board_id=board_id, version=version))
+        wl.append(manual_board(comport, board_id=board_id, version=version, custom=custom))
     return wl
 
 
@@ -79,6 +82,7 @@ def manual_board(
     *,
     board_id: str,
     version: str,
+    custom: bool = False,
 ) -> FlashItem:
     """Create a Flash work item for a single board specified manually.
 
@@ -90,7 +94,7 @@ def manual_board(
     Returns:
         FlashItem: Board and firmware information to update
     """
-    log.trace(f"Manual updating {serial} to {board_id} {version}")
+    log.debug(f"manual_board: {serial} {board_id} {version}")
     mcu = MPRemoteBoard(serial)
     # Lookup the matching port and cpu in board_info based in the board name
     try:
@@ -103,7 +107,7 @@ def manual_board(
         log.exception(e)
         return (mcu, None)
     mcu.board = board_id
-    firmwares = find_downloaded_firmware(board_id=board_id, version=version, port=mcu.port)
+    firmwares = find_downloaded_firmware(board_id=board_id, version=version, port=mcu.port, custom=custom)
     if not firmwares:
         log.trace(f"No firmware found for {mcu.port} {board_id} version {version}")
         return (mcu, None)
@@ -125,6 +129,7 @@ def single_auto_worklist(
     Returns:
         WorkList: List of boards and firmware information to update
     """
+    log.debug(f"single_auto_worklist: {serial} version: {version}")
     log.trace(f"Auto updating {serial} to {version}")
     conn_boards = [MPRemoteBoard(serial)]
     todo = auto_update_worklist(conn_boards, version)  # type: ignore # List / list
@@ -149,7 +154,7 @@ def full_auto_worklist(
     Returns:
         WorkList: List of boards and firmware information to update
     """
-    log.trace(f"Auto updating all boards to {version}")
+    log.debug(f"full_auto_worklist: {len(all_boards)} boards, include: {include}, ignore: {ignore}, version: {version}")
     if selected_boards := filter_boards(all_boards, include=include, ignore=ignore):
         return auto_update_worklist(selected_boards, version)
     else:

mpflash/list.py

@@ -90,6 +90,7 @@ def mcu_table(
     if needs_build:
         table.add_column("Build" if is_wide else "Bld", justify="right")
     if config.usb:
+        table.add_column("vid:pid", overflow="fold", max_width=14)
         table.add_column("Location", overflow="fold", max_width=60)
     # fill the table with the data
     for mcu in conn_mcus:
@@ -111,6 +112,7 @@ def mcu_table(
         if needs_build:
             row.append(mcu.build)
         if config.usb:
+            row.append(f"{mcu.vid:04x}:{mcu.pid:04x}")
             row.append(mcu.location)
 
         table.add_row(*row)

mpflash/logger.py

@@ -15,6 +15,7 @@ from .config import config
 
 console = Console()
 
+
 # Detect if the output encoding supports Unicode (UTF-8)
 @functools.lru_cache(maxsize=1)
 def _is_utf8_encoding() -> bool:
@@ -26,11 +27,13 @@ def _is_utf8_encoding() -> bool:
     except BaseException:
         return False
 
+
 def _log_formatter(record: dict) -> str:
     """
     Log message formatter for loguru and rich.
 
     Removes Unicode icons if console encoding is not UTF-8.
+    Handles messages containing curly braces safely.
     """
     color_map = {
         "TRACE": "cyan",
@@ -47,21 +50,34 @@ def _log_formatter(record: dict) -> str:
         icon = record["level"].icon
     else:
         icon = record["level"].name  # fallback to text
-    # Insert color directly using f-string
-    return f"[not bold green]{{time:HH:mm:ss}}[/not bold green] | {icon} [{lvl_color}]{record['message']}[/{lvl_color}]"
+    # Escape curly braces in the message to prevent format conflicts
+    safe_message = record["message"].replace("{", "{{").replace("}", "}}")
+    # Use string concatenation to avoid f-string format conflicts
+    time_part = "[not bold green]{time:HH:mm:ss}[/not bold green]"
+    message_part = f"[{lvl_color}]{safe_message}[/{lvl_color}]"
+    return f"{time_part} | {icon} {message_part}"
 
 
 def set_loglevel(loglevel: str) -> None:
     """
     Set the log level for the logger.
 
-    Ensures Unicode safety for log output.
+    Ensures Unicode safety for log output and handles format errors.
     """
     try:
         log.remove()
     except ValueError:
         pass
-    log.add(console.print, level=loglevel.upper(), colorize=False, format=_log_formatter)  # type: ignore
+
+    # Add error handling for format issues
+    def safe_format_wrapper(message):
+        try:
+            console.print(message)
+        except (KeyError, ValueError) as e:
+            # Fallback to simple text output if formatting fails
+            console.print(f"[LOG FORMAT ERROR] {message} (Error: {e})")
+
+    log.add(safe_format_wrapper, level=loglevel.upper(), colorize=False, format=_log_formatter)  # type: ignore
 
 
 def make_quiet() -> None:

mpflash/mpboard_id/board_id.py

@@ -2,7 +2,6 @@
 Translate board description to board designator
 """
 
-
 from typing import List, Optional
 
 from mpflash.db.core import Session
@@ -17,25 +16,24 @@ def find_board_id_by_description(
     short_descr: str,
     *,
     version: str,
-) -> Optional[str]:
+) -> str:
     """Find the MicroPython BOARD_ID based on the description in the firmware"""
     version = clean_version(version) if version else ""
-    try:
+    boards = _find_board_id_by_description(
+        descr=descr,
+        short_descr=short_descr,
+        version=version,
+    )
+    if not boards:
+        log.debug(f"Version {version} not found in board info, using any version")
         boards = _find_board_id_by_description(
             descr=descr,
             short_descr=short_descr,
-            version=version,
+            version="%",  # any version
         )
-        if not boards:
-            log.debug(f"Version {version} not found in board info, using any version")
-            boards = _find_board_id_by_description(
-                descr=descr,
-                short_descr=short_descr,
-                version="%",  # any version
-            )
-        return boards[0].board_id if boards else None
-    except MPFlashError:
-        return "UNKNOWN_BOARD"
+    if not boards:
+        raise MPFlashError(f"No board info found for description '{descr}' or '{short_descr}'")
+    return boards[0].board_id
 
 
 def _find_board_id_by_description(
@@ -63,7 +61,5 @@ def _find_board_id_by_description(
         )
         boards = qry.all()
 
-    if not boards:
-        raise MPFlashError(f"No board info found for description '{descr}' or '{short_descr}'")
-    return boards
 
+    return boards

mpflash/mpboard_id/known.py

@@ -64,9 +64,15 @@ def known_stored_boards(port: str, versions: List[str] = []) -> List[Tuple[str,
 
 
 def find_known_board(board_id: str, version="") -> Board:
-    """Find the board for the given BOARD_ID or 'board description' and return the board info as a Board object"""
+    """
+    Find the board for the given BOARD_ID or 'board description'
+    if the board_id is not found, it will try to find it by description.
+
+    if the board_id contains an @, it will split it and use the first part as the board_id
+    Returns the board info as a Board object
+    """
     with Session() as session:
-        qry = session.query(Board).filter(Board.board_id == board_id)
+        qry = session.query(Board).filter(Board.board_id == board_id.split("@")[0])
         if version:
             qry = qry.filter(Board.version == version)
         board = qry.first()