mpflash 1.25.0rc4__py3-none-any.whl → 1.25.2__py3-none-any.whl

mpflash/custom/__init__.py

@@ -0,0 +1,144 @@
+import shutil
+import sqlite3
+from pathlib import Path
+from typing import Union
+
+import jsonlines
+import requests
+from loguru import logger as log
+# re-use logic from mpremote
+from mpremote.mip import _rewrite_url as rewrite_url  # type: ignore
+
+from mpflash.config import config
+from mpflash.db.core import Session
+from mpflash.db.models import Firmware
+from mpflash.errors import MPFlashError
+from mpflash.versions import get_preview_mp_version, get_stable_mp_version
+
+from .naming import (custom_fw_from_path, extract_commit_count,
+                     port_and_boardid_from_path)
+
+# 
+# github.com/<owner>/<repo>@<branch>#<commit>
+# $remote_url = git remote get-url origin
+# $branch = git rev-parse --abbrev-ref HEAD
+# $commit = git rev-parse --short HEAD
+# if ($remote_url -match "github.com[:/](.+)/(.+?)(\.git)?$") {
+#     $owner = $matches[1]
+#     $repo = $matches[2]
+#     "github.com/$owner/$repo@$branch#$commit"
+# }
+
+
+# 1) local path 
+
+
+def add_firmware(
+    source: Path,  
+    fw_info: dict,
+    *,
+    force: bool = False,
+    custom: bool = False,
+) -> bool:
+    """
+    Add a firmware to the database , and firmware folder.
+    stored in the port folder, with the filename.
+
+    fw_info is a dict with the following keys:
+    - board_id: str, required
+    - version: str, required
+    - port: str, required
+    - firmware_file: str, required, the filename to store in the firmware folder
+    - source: str, optional, the source of the firmware, can be a local path
+    - description: str, optional, a description of the firmware
+    - custom: bool, optional, if the firmware is a custom firmware, default False
+    """
+    try:
+        source = source.expanduser().absolute()
+        if not source.exists() or not source.is_file():
+            log.error(f"Source file {source} does not exist or is not a file")
+            return False
+        with Session() as session:
+            # Check minimal info needed
+            new_fw = Firmware(**fw_info)
+            if custom:
+                new_fw.custom = True
+
+            if not new_fw.board_id:
+                log.error("board_id is required")
+                return False
+
+            # assume the the firmware_file has already been prepared 
+            fw_filename = config.firmware_folder / new_fw.firmware_file
+
+            if not copy_firmware(source, fw_filename, force):
+                log.error(f"Failed to copy {source} to {fw_filename}")
+                return False
+            # add to inventory
+            # check if the firmware already exists
+            if custom:
+                qry = session.query(Firmware).filter(Firmware.custom_id == new_fw.custom_id) 
+            else:
+                qry = session.query(Firmware).filter(Firmware.board_id == new_fw.board_id) 
+
+            qry = qry.filter(
+                    Firmware.board_id == new_fw.board_id,
+                    Firmware.version == new_fw.version,
+                    Firmware.port == new_fw.port,
+                    Firmware.custom == new_fw.custom,
+                )
+            existing_fw = qry.first()
+        
+            if existing_fw:
+                if not force:
+                    log.warning(f"Firmware {existing_fw} already exists")
+                    return False
+                # update the existing firmware
+                existing_fw.firmware_file = new_fw.firmware_file
+                existing_fw.source = new_fw.source
+                existing_fw.description = new_fw.description
+                existing_fw.custom = custom
+                if custom:
+                    existing_fw.custom_id = new_fw.custom_id
+            else:
+                session.add(new_fw)
+            session.commit()
+
+        return True
+    except sqlite3.DatabaseError as e:
+        raise MPFlashError(
+            f"Failed to add firmware {fw_info['firmware_file']}: {e}"
+        ) from e
+
+
+def copy_firmware(source: Path, fw_filename: Path, force: bool = False):
+    """Add a firmware to the firmware folder.
+    stored in the port folder, with the same filename as the source.
+    """
+    if fw_filename.exists() and not force:
+        log.error(f" {fw_filename} already exists. Use --force to overwrite")
+        return False
+    fw_filename.parent.mkdir(parents=True, exist_ok=True)
+    if isinstance(source, Path):
+        if not source.exists():
+            log.error(f"File {source} does not exist")
+            return False
+        # file copy
+        log.debug(f"Copy {source} to {fw_filename}")
+        shutil.copy(source, fw_filename)
+        return True
+    # TODO: handle github urls
+    # url = rewrite_url(source)
+    # if str(source).startswith("http://") or str(source).startswith("https://"):
+    #     log.debug(f"Download {url} to {fw_filename}")
+    #     response = requests.get(url)
+
+    #     if response.status_code == 200:
+    #         with open(fw_filename, "wb") as file:
+    #             file.write(response.content)
+    #             log.info("File downloaded and saved successfully.")
+    #             return True
+    #     else:
+    #         print("Failed to download the file.")
+    #         return False
+    # return False

mpflash/custom/naming.py

@@ -0,0 +1,91 @@
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import mpflash.basicgit as git
+from mpflash.logger import log
+
+
+def custom_fw_from_path(fw_path: Path) -> Dict[str, Union[str, int, bool]]:
+    """Generate a custom name for the firmware file based on its path.
+
+    Args:
+        firmware_path: Path to firmware file
+
+    Returns:
+        Custom name for the firmware file
+    """
+    repo_path = fw_path.expanduser().absolute().parent
+    port, board_id = port_and_boardid_from_path(fw_path)
+    if not port or not board_id:
+        raise ValueError(f"Could not extract port and board_id from path: {fw_path}")
+    if "wsl.localhost" in str(repo_path):
+        log.info("Accessing WSL path; please note that it may take a few seconds to get git info across filesystems")
+    version = git.get_local_tag(repo_path) or "unknown"
+    describe = git.get_git_describe(repo_path)
+    if describe:
+        build = extract_commit_count(describe)
+    else:
+        build = 0
+    branch = git.get_current_branch(repo_path)
+    if branch:
+        branch = branch.split("/")[-1]  # Use only last part of the branch name (?)
+    build_str = f".{build}" if build > 0 else ""
+    branch_str = f"@{branch}" if branch else ""
+    new_fw_path = Path(port) / f"{board_id}{branch_str}-{version}{build_str}{fw_path.suffix}"
+
+    return {
+        "port": port,
+        "board_id": f"{board_id}",
+        "custom_id": f"{board_id}{branch_str}",
+        "version": version,
+        "build": build,
+        "custom": True,
+        "firmware_file": new_fw_path.as_posix(),
+        "source": fw_path.expanduser().absolute().as_uri() if isinstance(fw_path, Path) else fw_path,  # Use URI for local files
+    }
+
+
+def port_and_boardid_from_path(firmware_path: Path) -> Tuple[Optional[str], Optional[str]]:
+    """Extract port and board_id from firmware path.
+
+    Args:
+        firmware_path: Path to firmware file
+
+    Returns:
+        Tuple of (port, board_id) or (None, None) if not found
+    """
+    path_str = str(firmware_path).replace("\\", "/")  # Normalize path for regex matching
+
+    # Pattern: /path/to/micropython/ports/{port}/build-{board_id}/firmware.ext
+    build_match = re.search(r"/ports/([^/]+)/build-([^/]+)/", path_str)
+    if build_match:
+        port = build_match.group(1)
+        board_id = build_match.group(2)
+        # Remove variant suffix (e.g., GENERIC_S3-SPIRAM_OCT -> GENERIC_S3)
+        board_id = board_id.split("-")[0]
+        return port, board_id
+
+    # Pattern: /path/to/micropython/ports/{port}/firmware.ext
+    port_match = re.search(r"/ports/([^/]+)/[^/]*firmware\.[^/]*$", path_str)
+    if port_match:
+        port = port_match.group(1)
+        return port, None
+
+    return None, None
+
+
+def extract_commit_count(git_describe: str) -> int:
+    """Extract commit count from git describe string.
+
+    Args:
+        git_describe: Git describe output like 'v1.26.0-preview-214-ga56a1eec7b-dirty'
+
+    Returns:
+        Commit count as integer or None if not found
+    """
+    # Match patterns like v1.26.0-preview-214-g... or v1.26.0-214-g...
+    match = re.search(r"-(\d+)-g[a-f0-9]+", git_describe)
+    if match:
+        return int(match.group(1))
+    return 0

mpflash/db/core.py

@@ -2,7 +2,8 @@ from pathlib import Path
 from sqlite3 import DatabaseError, OperationalError
 
 from loguru import logger as log
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, text
+from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import sessionmaker
 
 from mpflash.config import config
@@ -13,18 +14,90 @@ from .models import Base
 
 TRACE = False
 connect_str = f"sqlite:///{config.db_path.as_posix()}"
+log.debug(f"Connecting to database at {connect_str}")
 engine = create_engine(connect_str, echo=TRACE)
 Session = sessionmaker(bind=engine)
 
 
+def get_schema_version() -> int:
+    """Get current database schema version."""
+    from .meta import get_metadata_value
+
+    version = get_metadata_value("schema_version")
+    return int(version) if version else 0
+
+
+def set_schema_version(version: int) -> None:
+    """Set database schema version."""
+    from .meta import set_metadata_value
+
+    set_metadata_value("schema_version", str(version))
+
+
+def migration_001_add_custom_id() -> None:
+    """Add custom_id column to firmwares table."""
+    log.info("Running migration 001: Add custom_id column to firmwares table")
+
+    with Session() as session:
+        # Check if column already exists
+        result = session.execute(text("PRAGMA table_info(firmwares)")).fetchall()
+
+        columns = [row[1] for row in result]
+        if "custom_id" not in columns:
+            # Add column without UNIQUE constraint
+            session.execute(text("ALTER TABLE firmwares ADD COLUMN custom_id VARCHAR(40)"))
+
+            # Create regular index for performance
+            session.execute(text("CREATE INDEX IF NOT EXISTS idx_firmwares_custom_id ON firmwares(custom_id)"))
+
+            session.commit()
+            log.info("Added custom_id column and index to firmwares table")
+        else:
+            log.info("custom_id column already exists in firmwares table")
+
+            # Check if index exists and create if needed
+            result = session.execute(text("PRAGMA index_list(firmwares)")).fetchall()
+
+            index_names = [row[1] for row in result]
+            if "idx_firmwares_custom_id" not in index_names:
+                session.execute(text("CREATE INDEX IF NOT EXISTS idx_firmwares_custom_id ON firmwares(custom_id)"))
+                session.commit()
+                log.info("Added index to existing custom_id column")
+
+
+def run_schema_migrations() -> None:
+    """Run all pending database schema migrations."""
+    current_version = get_schema_version()
+    target_version = 1  # Current latest version
+
+    if current_version >= target_version:
+        log.debug(f"Database schema is up to date (version {current_version})")
+        return
+
+    log.info(f"Upgrading database schema from version {current_version} to {target_version}")
+
+    try:
+        # Run migrations in order
+        if current_version < 1:
+            migration_001_add_custom_id()
+
+        # Update schema version
+        set_schema_version(target_version)
+        log.info(f"Database schema upgraded to version {target_version}")
+
+    except SQLAlchemyError as e:
+        log.error(f"Migration failed: {e}")
+        raise MPFlashError(f"Failed to migrate database schema: {e}")
+
+
 def migrate_database(boards: bool = True, firmwares: bool = True):
-    """Migrate from 1.24.x to 1.25.x"""
+    """Migrate from 1.24.x to 1.25.x and run schema migrations."""
     # Move import here to avoid circular import
     from .loader import load_jsonl_to_db, update_boards
 
     # get the location of the database from the session
     with Session() as session:
-        db_location = session.get_bind().url.database # type: ignore
+        db_location = session.get_bind().url.database  # type: ignore
         log.debug(f"Database location: {Path(db_location)}")  # type: ignore
 
     try:
@@ -33,6 +106,10 @@ def migrate_database(boards: bool = True, firmwares: bool = True):
         log.error(f"Error creating database: {e}")
         log.error("Database might already exist, trying to migrate.")
         raise MPFlashError("Database migration failed. Please check the logs for more details.") from e
+
+    # Run schema migrations after creating database
+    run_schema_migrations()
+
     if boards:
         update_boards()
     if firmwares:
@@ -54,8 +131,9 @@ def migrate_database(boards: bool = True, firmwares: bool = True):
 
 
 def create_database():
-    """
-    Create the SQLite database and tables if they don't exist.
-    """
+    """Create the SQLite database and tables if they don't exist."""
     # Create the database and tables if they don't exist
     Base.metadata.create_all(engine)
+
+    # Run schema migrations after table creation
+    run_schema_migrations()

mpflash/db/gather_boards.py

@@ -1,11 +1,11 @@
 from os import path
 from pathlib import Path
-from mpflash.mpremoteboard import HERE
-from mpflash.vendor.board_database import Database
-from mpflash.logger import log
-
 from typing import List
+
 import mpflash.basicgit as git
+from mpflash.logger import log
+from mpflash.mpremoteboard import HERE
+from mpflash.vendor.board_database import Database
 from mpflash.versions import micropython_versions
 
 HERE = Path(__file__).parent.resolve()
@@ -79,6 +79,7 @@ def create_zip_file(longlist, zip_file: Path):
     """Create a ZIP file containing the CSV data."""
     # lazy import
     import zipfile
+
     import pandas as pd
 
     csv_filename = "micropython_boards.csv"
@@ -96,6 +97,7 @@ def create_zip_file(longlist, zip_file: Path):
 
 def package_repo(mpy_path: Path):
     mpy_path = mpy_path or Path("../repos/micropython")
+    log.info(f"Packaging Micropython boards from {mpy_path}")
     mp_versions = micropython_versions(minver="1.18")
     longlist = boardlist_from_repo(
         versions=mp_versions,

mpflash/db/loader.py

@@ -24,7 +24,7 @@ def load_data_from_zip(zip_file: Path) -> int:
     if not zip_file.exists() or not zip_file.is_file():
         log.error(f"Zip file {zip_file} not found.")
         return 0
-    count = 0 
+    count = 0
     # Load data directly from the zip file
     with zipfile.ZipFile(zip_file, "r") as zipf:
         # Read the CSV file from the zip
@@ -109,14 +109,15 @@ def load_jsonl_to_db(jsonl_path: Path):
 
 
 def update_boards():
+    boards_version = "v1.25.2"
     try:
         meta = get_metadata()
         log.debug(f"Metadata: {meta}")
-        if meta.get("boards_version", "") < "v1.25.0":
+        if meta.get("boards_version", "") < boards_version:
             log.info("Update boards from CSV to SQLite database.")
             # Load data from the zip file into the database
             load_data_from_zip(HERE / "micropython_boards.zip")
-            set_metadata_value("boards_version", "v1.25.0")
+            set_metadata_value("boards_version", boards_version)
             meta = get_metadata()
     except Exception as e:
         raise MPFlashError(f"Error updating boards table: {e}") from e

mpflash/db/meta.py

@@ -1,4 +1,3 @@
-
 from typing import Optional
 
 from loguru import logger as log
@@ -18,10 +17,11 @@ def get_metadata() -> dict:
         with Session() as session:
             metadata = session.query(Metadata).all()
             return {m.name: m.value for m in metadata}
-    except (DatabaseError, OperationalError)  as e:
+    except (DatabaseError, OperationalError) as e:
         log.error(f"Error retrieving metadata: {e}")
         return {}
-    
+
+
 def set_metadata(metadata: dict):
     """
     Set metadata in the database.
@@ -56,6 +56,7 @@ def get_metadata_value(name: str) -> Optional[str]:
         metadata = session.query(Metadata).filter(Metadata.name == name).first()
     return metadata.value if metadata else None
 
+
 def set_metadata_value(name: str, value: str):
     """
     Set metadata value by name.

mpflash/db/models.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Union
 
 import sqlalchemy as sa
 from sqlalchemy import Index, String
@@ -78,6 +79,7 @@ class Firmware(Base):
     source: Mapped[str] = mapped_column()
     build: Mapped[int] = mapped_column(default=0, comment="Build number")
     custom: Mapped[bool] = mapped_column(default=False, comment="True if this is a custom firmware")
+    custom_id: Mapped[Union[str, None]] = mapped_column(String(40), nullable=True, default=None)
 
     @property
     def preview(self) -> bool:

mpflash/download/__init__.py

@@ -4,6 +4,7 @@ Uses the micropython.org website to get the available versions and locations to
 """
 
 import itertools
+import re
 from pathlib import Path
 from typing import Dict, List, Optional
 
@@ -16,7 +17,7 @@ from rich.progress import track
 from mpflash.common import PORT_FWTYPES
 from mpflash.config import config
 from mpflash.db.core import Session
-from mpflash.db.models import Firmware
+from mpflash.db.models import Firmware, Board
 from mpflash.downloaded import clean_downloaded_firmwares
 from mpflash.errors import MPFlashError
 from mpflash.mpboard_id.alternate import add_renamed_boards
@@ -36,6 +37,56 @@ def key_fw_boardid_preview_ext(fw: Firmware):
     return fw.board_id, fw.preview, fw.ext
 
 
+
+
+# Cache for variant suffixes - will be populated on first use
+_PATTERN = ""
+
+def _get_variant_pattern():
+    """
+    Query the database for all known variant suffixes.
+    This is done only once and the result is cached.
+    
+    Returns:
+        List of known variant suffixes.
+    """
+    global _PATTERN
+    
+    if _PATTERN:
+        # If the pattern is already set, return it
+        return _PATTERN
+
+    with Session() as session:
+        # Query distinct variants
+        variants = session.query(Board.variant).distinct().all()
+
+
+    _VARIANT_SUFFIXES = [f"_{v[0]}" for v in variants if v[0] and v[0].strip()]
+    # workaround for the fact that the variant names do not always match the suffixes
+    # e.g. 'PIMORONI_PICOLIPO' has the variant 'FLASH_16MB' but the suffix is '_16MB'
+    _VARIANT_SUFFIXES.extend( ["_2MB", "_4MB", "_8MB", "_16MB"] )  # add common SPIRAM size suffixes
+
+    # return _VARIANT_SUFFIXES
+    _PATTERN  = f"({'|'.join(re.escape(v) for v in _VARIANT_SUFFIXES)})$"
+    return _PATTERN
+
+    
+
+def strip_variant(board: str) -> str:
+    """
+    Strips the variant suffix from the board name based on variants in the database.
+    For example, 'RPI_PICO_W_SPIRAM' becomes 'RPI_PICO_W'.
+
+    Args:
+        board: The board name to process.
+
+    Returns:
+        The board name without the variant suffix.
+    """
+    pattern = _get_variant_pattern()
+    return re.sub(pattern, "", board)
+
+
 def download_firmwares(
     firmware_folder: Path,
     ports: List[str],
@@ -60,9 +111,15 @@ def download_firmwares(
 
     downloaded = 0
     versions = [] if versions is None else [clean_version(v) for v in versions]
+    
+    # remove the known variant suffixes from the boards
+    # TODO: IS THIS REALLY NEEDED ?
+    # boards = [strip_variant(b) for b in boards]
+    
     # handle downloading firmware for renamed boards
     boards = add_renamed_boards(boards)
 
+
     available_firmwares = get_firmware_list(ports, boards, versions, clean)
 
     for b in available_firmwares:

mpflash/download/fwinfo.py

@@ -38,4 +38,4 @@ class FWInfo:
             data["ext"] = Path(data["filename"]).suffix
         if "family" not in data:
             data["family"] = "micropython"
-        return FWInfo(**filtered_data)        
+        return FWInfo(**filtered_data)

mpflash/download/jid.py

@@ -48,7 +48,7 @@ def ensure_firmware_downloaded(worklist: WorkList, version: str, force: bool) ->
             newlist.append((mcu, new_firmware[0]))
         else:
             log.info(f"Found {version} firmware {board_firmwares[-1].firmware_file} for {mcu.board} on {mcu.serialport}.")
-            newlist.append((mcu, firmware))
+            newlist.append((mcu, board_firmwares[0]))
 
     worklist.clear()
     worklist.extend(newlist)

mpflash/downloaded.py

@@ -59,17 +59,22 @@ def find_downloaded_firmware(
     version: str = "",
     port: str = "",
     variants: bool = False,
+    custom: bool = False,
 ) -> List[Firmware]:
     version = clean_version(version)
     log.debug(f"Looking for firmware for {board_id} {version} ")
     # Special handling for preview versions
     with Session() as session:
-        if version == "preview" or "preview" in version:
+        if "preview" in version:
             # Find all preview firmwares for this board/port, return the latest (highest build)
-            query = session.query(Firmware).filter(Firmware.board_id == board_id)
+            if custom:
+                query = session.query(Firmware).filter(Firmware.custom_id == board_id)
+            else:
+                query = session.query(Firmware).filter(Firmware.board_id == board_id)
             if port:
                 query = query.filter(Firmware.port == port)
             query = query.filter(Firmware.firmware_file.contains("preview")).order_by(Firmware.build.desc())
+            log.trace(f"Querying for preview firmware: {query}")
             fw_list = query.all()
             if fw_list:
                 return [fw_list[0]]  # Return the latest preview only
@@ -82,7 +87,7 @@ def find_downloaded_firmware(
     #
     log.debug(f"2nd search with renamed board_id :{board_id}")
     with Session() as session:
-        if version == "preview" or "preview" in version:
+        if "preview" in version:
             query = session.query(Firmware).filter(Firmware.board_id.in_(more_board_ids))
             if port:
                 query = query.filter(Firmware.port == port)
@@ -96,4 +101,3 @@ def find_downloaded_firmware(
                 return fw_list
     log.warning(f"No firmware files found for board {board_id} version {version}")
     return []
-

mpflash/flash/__init__.py

@@ -39,6 +39,15 @@ def flash_list(
             log.error(f"Failed to flash {mcu.board} on {mcu.serialport}: {e}")
             continue
         if updated:
+            if fw_info.custom:
+                # Add / Update board_info.toml with the custom_id and Description
+                mcu.get_board_info_toml()
+                if fw_info.description:
+                    mcu.toml["description"] = fw_info.description
+                mcu.toml["mpflash"]["board_id"] = fw_info.board_id
+                mcu.toml["mpflash"]["custom_id"] = fw_info.custom_id
+                mcu.set_board_info_toml()
+
             flashed.append(updated)
         else:
             log.error(f"Failed to flash {mcu.board} on {mcu.serialport}")
@@ -51,7 +60,7 @@ def flash_mcu(
         fw_file: Path,
         erase: bool = False,
         bootloader: BootloaderMethod = BootloaderMethod.AUTO,
-        **kwargs        
+        **kwargs
     ):
         """Flash a single MCU with the specified firmware."""
         updated = None

mpflash/flash/uf2/windows.py

@@ -7,6 +7,7 @@ import time
 from pathlib import Path
 from typing import Optional
 
+import psutil
 from rich.progress import track
 
 from .boardid import get_board_id
@@ -14,7 +15,6 @@ from .boardid import get_board_id
 
 def wait_for_UF2_windows(board_id: str, s_max: int = 10) -> Optional[Path]:
     """Wait for the MCU to mount as a drive"""
-    import psutil
 
     if s_max < 1:
         s_max = 10