mpflash 1.25.1__py3-none-any.whl → 1.26.0__py3-none-any.whl

mpflash/basicgit.py

@@ -135,6 +135,25 @@ def get_local_tags(repo: Optional[Path] = None, minver: Optional[str] = None) ->
     return sorted(tags)
 
 
+def get_current_branch(repo: Optional[Union[Path, str]] = None) -> Optional[str]:
+    """
+    Get the current branch name of a local repository.
+
+    Args:
+        repo: Path to the repository directory
+
+    Returns:
+        Current branch name or None if error
+    """
+    cmd = ["git", "branch", "--show-current"]
+    result = _run_local_git(cmd, repo=repo, expect_stderr=True)
+    if not result:
+        return None
+
+    branch = result.stdout.strip()
+    return branch if branch else None
+
+
 @cachetools.func.ttl_cache(maxsize=16, ttl=60)  # 60 seconds
 def get_tags(repo: str, minver: Optional[str] = None) -> List[str]:
     """
@@ -242,7 +261,7 @@ def fetch(repo: Union[Path, str]) -> bool:
     return result.returncode == 0 if result else False
 
 
-def pull(repo: Union[Path, str], branch: str = "main") -> bool:
+def pull(repo: Union[Path, str], branch: str = "main", force: bool = True) -> bool:
     """
     pull a repo origin into main
     repo should be in the form of : path/.git
@@ -253,7 +272,9 @@ def pull(repo: Union[Path, str], branch: str = "main") -> bool:
         raise NotADirectoryError
     repo = Path(repo)
     # first checkout HEAD
-    cmd = ["git", "checkout", branch, "--quiet", "--force"]
+    cmd = ["git", "checkout", branch, "--quiet"]
+    if force:
+        cmd.append("--force")
     result = _run_local_git(cmd, repo=repo, expect_stderr=True)
     if not result:
         log.error("error during git checkout main", result)

mpflash/cli_add.py

@@ -0,0 +1,131 @@
+"""CLI to add a custom MicroPython firmware."""
+
+from pathlib import Path
+from typing import Union
+
+import rich_click as click
+from loguru import logger as log
+
+from mpflash.connected import connected_ports_boards_variants
+from mpflash.custom import Firmware, add_firmware, custom_fw_from_path
+from mpflash.downloaded import clean_downloaded_firmwares
+from mpflash.errors import MPFlashError
+from mpflash.mpboard_id import find_known_board
+from mpflash.mpboard_id.alternate import add_renamed_boards
+from mpflash.versions import clean_version
+
+from .ask_input import ask_missing_params
+from .cli_group import cli
+from .config import config
+from .download import download
+
+
+@cli.command(
+    "add",
+    help="Add a custom MicroPython firmware.",
+)
+# @click.option(
+#     "--version",
+#     "-v",
+#     "versions",
+#     default=["stable"],
+#     multiple=False,
+#     show_default=True,
+#     help="The version of MicroPython to to download.",
+#     metavar="SEMVER, 'stable', 'preview' or '?'",
+# )
+@click.option(
+    "--path",
+    "-p",
+    "fw_path",
+    multiple=False,
+    default="",
+    show_default=False,
+    help="a local path to the firmware file to add.",
+    metavar="FIRMWARE_PATH",
+)
+@click.option(
+    "--description",
+    "-d",
+    "description",
+    default="",
+    help="An Optional description for the firmware.",
+    metavar="TXT",
+)
+# @click.option(
+#     "--board",
+#     "-b",
+#     "boards",
+#     multiple=True,
+#     default=[],
+#     show_default=True,
+#     help="The board(s) to download the firmware for.",
+#     metavar="BOARD_ID or ?",
+# )
+# @click.option(
+#     "--serial",
+#     "--serial-port",
+#     "-s",
+#     "serial",
+#     default=["*"],
+#     show_default=True,
+#     multiple=True,
+#     help="Which serial port(s) (or globs) to flash",
+#     metavar="SERIALPORT",
+# )
+# @click.option(
+#     "--ignore",
+#     "-i",
+#     is_eager=True,
+#     help="Serial port(s) to ignore. Defaults to MPFLASH_IGNORE.",
+#     multiple=True,
+#     default=[],
+#     envvar="MPFLASH_IGNORE",
+#     show_default=True,
+#     metavar="SERIALPORT",
+# )
+# @click.option(
+#     "--clean/--no-clean",
+#     default=True,
+#     show_default=True,
+#     help="""Remove dates and hashes from the downloaded firmware filenames.""",
+# )
+@click.option(
+    "--force",
+    "-f",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="""Overwrite existing firmware.""",
+)
+def cli_add_custom(
+    fw_path: Union[Path, str],
+    force: bool = False,
+    description: str = "",
+) -> int:
+    """Add a custom MicroPython firmware from a local file."""
+    if not fw_path:
+        log.error("No firmware path provided. Use --path to specify a firmware file.")
+        return 1
+    fw_path = Path(fw_path).expanduser().resolve()
+    if not fw_path.exists():
+        log.error(f"Firmware file does not exist: {fw_path}")
+        return 1
+
+    try:
+        fw_dict = custom_fw_from_path(fw_path)
+        if description:
+            fw_dict["description"] = description
+        if add_firmware(
+            source=fw_path,
+            fw_info=fw_dict,
+            custom=True,
+            force=force,
+        ):
+            log.success(f"Added custom firmware: {fw_dict['custom_id']} for {fw_dict['firmware_file']}")
+            return 0
+        else:
+            return 1
+    except MPFlashError as e:
+        log.error(f"{e}")
+        return 1

mpflash/cli_download.py

@@ -2,10 +2,11 @@
 
 from pathlib import Path
 
+from pytest import param
 import rich_click as click
 from loguru import logger as log
 
-from mpflash.connected import connected_ports_boards
+from mpflash.connected import connected_ports_boards_variants
 from mpflash.downloaded import clean_downloaded_firmwares
 from mpflash.errors import MPFlashError
 from mpflash.mpboard_id import find_known_board
@@ -97,7 +98,7 @@ def cli_download(**kwargs) -> int:
                         log.error(f"{e}")
     else:
         # no boards specified - detect connected ports and boards
-        params.ports, params.boards, _ = connected_ports_boards(include=params.serial, ignore=params.ignore)
+        params.ports, params.boards, _, _ = connected_ports_boards_variants(include=params.serial, ignore=params.ignore)
 
     params = ask_missing_params(params)
     if not params:  # Cancelled by user

mpflash/cli_flash.py

@@ -6,7 +6,7 @@ from loguru import logger as log
 import mpflash.download.jid as jid
 import mpflash.mpboard_id as mpboard_id
 from mpflash.ask_input import ask_missing_params
-from mpflash.cli_download import connected_ports_boards
+from mpflash.cli_download import connected_ports_boards_variants
 from mpflash.cli_group import cli
 from mpflash.cli_list import show_mcus
 from mpflash.common import BootloaderMethod, FlashParams, filtered_comports
@@ -93,7 +93,6 @@ from mpflash.versions import clean_version
 @click.option(
     "--cpu",
     "--chip",
-    "-c",
     "cpu",
     help="The CPU type to flash. If not specified will try to read the CPU from the connected MCU.",
     metavar="CPU",
@@ -129,6 +128,14 @@ from mpflash.versions import clean_version
     show_default=True,
     help="""Flash mode for ESP boards. (default: keep)""",
 )
+@click.option(
+    "--custom",
+    "-c",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="""Flash a custom firmware""",
+)
 def cli_flash_board(**kwargs) -> int:
     # version to versions, board to boards
     kwargs["versions"] = [kwargs.pop("version")] if kwargs["version"] is not None else []
@@ -158,9 +165,13 @@ def cli_flash_board(**kwargs) -> int:
     all_boards: List[MPRemoteBoard] = []
     if not params.boards:
         # nothing specified - detect connected boards
-        params.ports, params.boards, all_boards = connected_ports_boards(
-            include=params.ports, ignore=params.ignore, bluetooth=params.bluetooth
+        params.ports, params.boards, variants, all_boards = connected_ports_boards_variants(
+            include=params.ports,
+            ignore=params.ignore,
+            bluetooth=params.bluetooth,
         )
+        if variants and len(variants) >= 1:
+            params.variant = variants[0]
         if params.boards == []:
             # No MicroPython boards detected, but it could be unflashed or in bootloader mode
             # Ask for serial port and board_id to flash
@@ -188,7 +199,7 @@ def cli_flash_board(**kwargs) -> int:
         # A one or more serial port including the board / variant
         comports = filtered_comports(
             ignore=params.ignore,
-            include=params.ports,
+            include=params.serial,
             bluetooth=params.bluetooth,
         )
         board_id = f"{params.boards[0]}-{params.variant}" if params.variant else params.boards[0]
@@ -198,12 +209,13 @@ def cli_flash_board(**kwargs) -> int:
             comports,
             board_id=board_id,
             version=params.versions[0],
+            custom=params.custom,
         )
     # if serial port == auto and there are one or more specified/detected boards
     elif params.serial == ["*"] and params.boards:
         if not all_boards:
             log.trace("No boards detected yet, scanning for connected boards")
-            _, _, all_boards = connected_ports_boards(include=params.ports, ignore=params.ignore)
+            _, _, _, all_boards = connected_ports_boards_variants(include=params.ports, ignore=params.ignore)
         # if variant id provided on the cmdline, treat is as an override
         if params.variant:
             for b in all_boards:
@@ -233,7 +245,8 @@ def cli_flash_board(**kwargs) -> int:
             serial=params.serial[0],
             version=params.versions[0],
         )
-    jid.ensure_firmware_downloaded(worklist, version=params.versions[0], force=params.force)
+    if not params.custom:
+        jid.ensure_firmware_downloaded(worklist, version=params.versions[0], force=params.force)
     if flashed := flash_list(
         worklist,
         params.erase,
@@ -246,5 +259,3 @@ def cli_flash_board(**kwargs) -> int:
     else:
         log.error("No boards were flashed")
         return 1
-
-

mpflash/cli_group.py

@@ -14,6 +14,7 @@ from .logger import log, make_quiet, set_loglevel
 set_loglevel("INFO")
 config.verbose = False
 
+
 def cb_verbose(ctx, param, value):
     """Callback to set the log level to DEBUG if verbose is set"""
     if value and not config.quiet:

mpflash/cli_main.py

@@ -5,6 +5,9 @@ import os
 import click.exceptions as click_exceptions
 from loguru import logger as log
 
+from mpflash.errors import MPFlashError
+
+from .cli_add import cli_add_custom
 from .cli_download import cli_download
 from .cli_flash import cli_flash_board
 from .cli_group import cli
@@ -19,6 +22,7 @@ def mpflash():
     cli.add_command(cli_list_mcus)
     cli.add_command(cli_download)
     cli.add_command(cli_flash_board)
+    cli.add_command(cli_add_custom)
 
     # cli(auto_envvar_prefix="MPFLASH")
     if False and os.environ.get("COMPUTERNAME").upper().startswith("JOSVERL"):
@@ -37,6 +41,9 @@ def mpflash():
         except click_exceptions.Abort:
             # Aborted - Ctrl-C
             exit(-3)
+        except MPFlashError as e:
+            log.error(f"MPFlashError: {e}")
+            exit(-4)
 
 
 if __name__ == "__main__":

mpflash/common.py

@@ -4,7 +4,6 @@ import os
 import platform
 from dataclasses import dataclass, field
 from enum import Enum
-from pathlib import Path
 from typing import List, Optional, Union
 
 from serial.tools import list_ports
@@ -65,6 +64,7 @@ class FlashParams(Params):
     bootloader: BootloaderMethod = BootloaderMethod.NONE
     cpu: str = ""
     flash_mode: str = "keep"  # keep, qio, qout, dio, dout
+    custom: bool = False
 
     def __post_init__(self):
         if isinstance(self.bootloader, str):
@@ -85,6 +85,7 @@ def filtered_comports(
     """
     return [p.device for p in filtered_portinfos(ignore, include, bluetooth)]
 
+
 def filtered_portinfos(
     ignore: Optional[List[str]] = None,
     include: Optional[List[str]] = None,
@@ -94,6 +95,7 @@ def filtered_portinfos(
     Get a list of filtered comports using the include and ignore lists.
     both can be globs (e.g. COM*) or exact port names (e.g. COM1)
     """
+    log.trace(f"filtered_portinfos: {ignore=}, {include=}, {bluetooth=}")
     if not ignore:
         ignore = []
     elif not isinstance(ignore, list):  # type: ignore

mpflash/config.py

@@ -4,6 +4,7 @@ import os
 from importlib.metadata import version
 from pathlib import Path
 from typing import List, Optional
+
 import platformdirs
 
 from mpflash.errors import MPFlashError
@@ -46,14 +47,17 @@ class MPFlashConfig:
         """The folder where firmware files are stored"""
         if not self._firmware_folder:
             from mpflash.logger import log
+
             # Check if MPFLASH_FIRMWARE environment variable is set
             env_firmware_path = os.getenv("MPFLASH_FIRMWARE")
             if env_firmware_path:
-                firmware_path = Path(env_firmware_path)
+                firmware_path = Path(env_firmware_path).expanduser().resolve()
                 if firmware_path.exists() and firmware_path.is_dir():
                     self._firmware_folder = firmware_path
                 else:
-                    log.warning(f"Environment variable MPFLASH_FIRMWARE points to invalid directory: {env_firmware_path}. Using default location.")
+                    log.warning(
+                        f"Environment variable MPFLASH_FIRMWARE points to invalid directory: {env_firmware_path}. Using default location."
+                    )
             # allow testing in CI
             if Path(os.getenv("GITHUB_ACTIONS", "")).as_posix().lower() == "true":
                 workspace = os.getenv("GITHUB_WORKSPACE")
@@ -83,6 +87,7 @@ class MPFlashConfig:
     def db_path(self) -> Path:
         """The path to the database file"""
         return self.firmware_folder / "mpflash.db"
+
     @property
     def db_version(self) -> str:
         return "1.24.1"

mpflash/connected.py

@@ -8,9 +8,9 @@ from mpflash.common import filtered_portinfos, find_serial_by_path
 from mpflash.mpremoteboard import MPRemoteBoard
 
 
-def connected_ports_boards(
+def connected_ports_boards_variants(
     *, include: List[str], ignore: List[str], bluetooth: bool = False
-) -> Tuple[List[str], List[str], List[MPRemoteBoard]]:
+) -> Tuple[List[str], List[str], List[str], List[MPRemoteBoard]]:
     """
     Returns a tuple containing lists of unique ports and boards from the connected MCUs.
     Boards that are physically connected, but give no tangible response are ignored.
@@ -21,14 +21,15 @@ def connected_ports_boards(
             - A list of unique board names of the connected MCUs.
             - A list of MPRemoteBoard instances of the connected MCUs.
     """
-    conn_mcus = [b for b in list_mcus(include=include, ignore=ignore, bluetooth=bluetooth)]
+    # conn_mcus = [b for b in list_mcus(include=include, ignore=ignore, bluetooth=bluetooth)]
     conn_mcus = [b for b in list_mcus(include=include, ignore=ignore, bluetooth=bluetooth) if b.connected]
     # ignore boards that have the [mpflash] ignore flag set
     conn_mcus = [item for item in conn_mcus if not (item.toml.get("mpflash", {}).get("ignore", False))]
 
     ports = list({b.port for b in conn_mcus})
     boards = list({b.board for b in conn_mcus})
-    return (ports, boards, conn_mcus)
+    variants = list({b.variant for b in conn_mcus if b.variant})
+    return (ports, boards, variants, conn_mcus)
 
 
 # #########################################################################################################
@@ -47,7 +48,7 @@ def list_mcus(*, ignore: List[str], include: List[str], bluetooth: bool = False)
         ConnectionError: If there is an error connecting to a board.
     """
     # conn_mcus = [MPRemoteBoard(sp) for sp in MPRemoteBoard.connected_boards(bluetooth) if sp not in config.ignore_ports]
-
+    vid_pid = True
     comports = filtered_portinfos(
         ignore=ignore,
         include=include,

mpflash/custom/__init__.py

@@ -0,0 +1,144 @@
+import shutil
+import sqlite3
+from pathlib import Path
+from typing import Union
+
+import jsonlines
+import requests
+from loguru import logger as log
+# re-use logic from mpremote
+from mpremote.mip import _rewrite_url as rewrite_url  # type: ignore
+
+from mpflash.config import config
+from mpflash.db.core import Session
+from mpflash.db.models import Firmware
+from mpflash.errors import MPFlashError
+from mpflash.versions import get_preview_mp_version, get_stable_mp_version
+
+from .naming import (custom_fw_from_path, extract_commit_count,
+                     port_and_boardid_from_path)
+
+# 
+# github.com/<owner>/<repo>@<branch>#<commit>
+# $remote_url = git remote get-url origin
+# $branch = git rev-parse --abbrev-ref HEAD
+# $commit = git rev-parse --short HEAD
+# if ($remote_url -match "github.com[:/](.+)/(.+?)(\.git)?$") {
+#     $owner = $matches[1]
+#     $repo = $matches[2]
+#     "github.com/$owner/$repo@$branch#$commit"
+# }
+
+
+# 1) local path 
+
+
+def add_firmware(
+    source: Path,  
+    fw_info: dict,
+    *,
+    force: bool = False,
+    custom: bool = False,
+) -> bool:
+    """
+    Add a firmware to the database , and firmware folder.
+    stored in the port folder, with the filename.
+
+    fw_info is a dict with the following keys:
+    - board_id: str, required
+    - version: str, required
+    - port: str, required
+    - firmware_file: str, required, the filename to store in the firmware folder
+    - source: str, optional, the source of the firmware, can be a local path
+    - description: str, optional, a description of the firmware
+    - custom: bool, optional, if the firmware is a custom firmware, default False
+    """
+    try:
+        source = source.expanduser().absolute()
+        if not source.exists() or not source.is_file():
+            log.error(f"Source file {source} does not exist or is not a file")
+            return False
+        with Session() as session:
+            # Check minimal info needed
+            new_fw = Firmware(**fw_info)
+            if custom:
+                new_fw.custom = True
+
+            if not new_fw.board_id:
+                log.error("board_id is required")
+                return False
+
+            # assume the the firmware_file has already been prepared 
+            fw_filename = config.firmware_folder / new_fw.firmware_file
+
+            if not copy_firmware(source, fw_filename, force):
+                log.error(f"Failed to copy {source} to {fw_filename}")
+                return False
+            # add to inventory
+            # check if the firmware already exists
+            if custom:
+                qry = session.query(Firmware).filter(Firmware.custom_id == new_fw.custom_id) 
+            else:
+                qry = session.query(Firmware).filter(Firmware.board_id == new_fw.board_id) 
+
+            qry = qry.filter(
+                    Firmware.board_id == new_fw.board_id,
+                    Firmware.version == new_fw.version,
+                    Firmware.port == new_fw.port,
+                    Firmware.custom == new_fw.custom,
+                )
+            existing_fw = qry.first()
+        
+            if existing_fw:
+                if not force:
+                    log.warning(f"Firmware {existing_fw} already exists")
+                    return False
+                # update the existing firmware
+                existing_fw.firmware_file = new_fw.firmware_file
+                existing_fw.source = new_fw.source
+                existing_fw.description = new_fw.description
+                existing_fw.custom = custom
+                if custom:
+                    existing_fw.custom_id = new_fw.custom_id
+            else:
+                session.add(new_fw)
+            session.commit()
+
+        return True
+    except sqlite3.DatabaseError as e:
+        raise MPFlashError(
+            f"Failed to add firmware {fw_info['firmware_file']}: {e}"
+        ) from e
+
+
+def copy_firmware(source: Path, fw_filename: Path, force: bool = False):
+    """Add a firmware to the firmware folder.
+    stored in the port folder, with the same filename as the source.
+    """
+    if fw_filename.exists() and not force:
+        log.error(f" {fw_filename} already exists. Use --force to overwrite")
+        return False
+    fw_filename.parent.mkdir(parents=True, exist_ok=True)
+    if isinstance(source, Path):
+        if not source.exists():
+            log.error(f"File {source} does not exist")
+            return False
+        # file copy
+        log.debug(f"Copy {source} to {fw_filename}")
+        shutil.copy(source, fw_filename)
+        return True
+    # TODO: handle github urls
+    # url = rewrite_url(source)
+    # if str(source).startswith("http://") or str(source).startswith("https://"):
+    #     log.debug(f"Download {url} to {fw_filename}")
+    #     response = requests.get(url)
+
+    #     if response.status_code == 200:
+    #         with open(fw_filename, "wb") as file:
+    #             file.write(response.content)
+    #             log.info("File downloaded and saved successfully.")
+    #             return True
+    #     else:
+    #         print("Failed to download the file.")
+    #         return False
+    # return False

mpflash/custom/naming.py

@@ -0,0 +1,91 @@
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import mpflash.basicgit as git
+from mpflash.logger import log
+
+
+def custom_fw_from_path(fw_path: Path) -> Dict[str, Union[str, int, bool]]:
+    """Generate a custom name for the firmware file based on its path.
+
+    Args:
+        firmware_path: Path to firmware file
+
+    Returns:
+        Custom name for the firmware file
+    """
+    repo_path = fw_path.expanduser().absolute().parent
+    port, board_id = port_and_boardid_from_path(fw_path)
+    if not port or not board_id:
+        raise ValueError(f"Could not extract port and board_id from path: {fw_path}")
+    if "wsl.localhost" in str(repo_path):
+        log.info("Accessing WSL path; please note that it may take a few seconds to get git info across filesystems")
+    version = git.get_local_tag(repo_path) or "unknown"
+    describe = git.get_git_describe(repo_path)
+    if describe:
+        build = extract_commit_count(describe)
+    else:
+        build = 0
+    branch = git.get_current_branch(repo_path)
+    if branch:
+        branch = branch.split("/")[-1]  # Use only last part of the branch name (?)
+    build_str = f".{build}" if build > 0 else ""
+    branch_str = f"@{branch}" if branch else ""
+    new_fw_path = Path(port) / f"{board_id}{branch_str}-{version}{build_str}{fw_path.suffix}"
+
+    return {
+        "port": port,
+        "board_id": f"{board_id}",
+        "custom_id": f"{board_id}{branch_str}",
+        "version": version,
+        "build": build,
+        "custom": True,
+        "firmware_file": new_fw_path.as_posix(),
+        "source": fw_path.expanduser().absolute().as_uri() if isinstance(fw_path, Path) else fw_path,  # Use URI for local files
+    }
+
+
+def port_and_boardid_from_path(firmware_path: Path) -> Tuple[Optional[str], Optional[str]]:
+    """Extract port and board_id from firmware path.
+
+    Args:
+        firmware_path: Path to firmware file
+
+    Returns:
+        Tuple of (port, board_id) or (None, None) if not found
+    """
+    path_str = str(firmware_path).replace("\\", "/")  # Normalize path for regex matching
+
+    # Pattern: /path/to/micropython/ports/{port}/build-{board_id}/firmware.ext
+    build_match = re.search(r"/ports/([^/]+)/build-([^/]+)/", path_str)
+    if build_match:
+        port = build_match.group(1)
+        board_id = build_match.group(2)
+        # Remove variant suffix (e.g., GENERIC_S3-SPIRAM_OCT -> GENERIC_S3)
+        board_id = board_id.split("-")[0]
+        return port, board_id
+
+    # Pattern: /path/to/micropython/ports/{port}/firmware.ext
+    port_match = re.search(r"/ports/([^/]+)/[^/]*firmware\.[^/]*$", path_str)
+    if port_match:
+        port = port_match.group(1)
+        return port, None
+
+    return None, None
+
+
+def extract_commit_count(git_describe: str) -> int:
+    """Extract commit count from git describe string.
+
+    Args:
+        git_describe: Git describe output like 'v1.26.0-preview-214-ga56a1eec7b-dirty'
+
+    Returns:
+        Commit count as integer or None if not found
+    """
+    # Match patterns like v1.26.0-preview-214-g... or v1.26.0-214-g...
+    match = re.search(r"-(\d+)-g[a-f0-9]+", git_describe)
+    if match:
+        return int(match.group(1))
+    return 0