mpflash 1.25.0__py3-none-any.whl → 1.25.0rc1__py3-none-any.whl

mpflash/add_firmware.py

@@ -5,18 +5,29 @@ from typing import Union
 import jsonlines
 import requests
 from loguru import logger as log
-
 # re-use logic from mpremote
 from mpremote.mip import _rewrite_url as rewrite_url  # type: ignore
+from pytest import Session
 
-from mpflash.common import FWInfo
 from mpflash.config import config
+from mpflash.db.core import Session
+from mpflash.db.models import Firmware
 from mpflash.versions import get_preview_mp_version, get_stable_mp_version
 
+# github.com/<owner>/<repo>@<branch>#<commit>
+# $remote_url = git remote get-url origin
+# $branch = git rev-parse --abbrev-ref HEAD
+# $commit = git rev-parse --short HEAD
+# if ($remote_url -match "github.com[:/](.+)/(.+?)(\.git)?$") {
+#     $owner = $matches[1]
+#     $repo = $matches[2]
+#     "github.com/$owner/$repo@$branch#$commit"
+# }
+
 
 def add_firmware(
     source: Union[Path, str],
-    new_fw: FWInfo,
+    new_fw: Firmware,
     *,
     force: bool = False,
     custom: bool = False,
@@ -25,11 +36,11 @@ def add_firmware(
     """Add a firmware to the firmware folder.
 
     stored in the port folder, with the same filename as the source.
-
     """
     # Check minimal info needed
-    if not new_fw.port or not new_fw.board:
-        log.error("Port and board are required")
+
+    if not new_fw.board_id or not new_fw.board or not new_fw.port:
+        log.error("board_id, board and port are required")
         return False
     if not isinstance(source, Path) and not source.startswith("http"):
         log.error(f"Invalid source {source}")
@@ -37,31 +48,47 @@ def add_firmware(
 
     # use sensible defaults
     source_2 = Path(source)
-    new_fw.ext = new_fw.ext or source_2.suffix
-    new_fw.variant = new_fw.variant or new_fw.board
+    # new_fw.variant = new_fw.variant or new_fw.board
     new_fw.custom = new_fw.custom or custom
-    new_fw.description = new_fw.description or description
     if not new_fw.version:
         # TODO: Get version from filename
         # or use the last preview version
-        new_fw.version = get_preview_mp_version() if new_fw.preview else get_stable_mp_version()
+        new_fw.version = get_preview_mp_version()
 
     config.firmware_folder.mkdir(exist_ok=True)
 
     fw_filename = config.firmware_folder / new_fw.port / source_2.name
 
-    new_fw.filename = str(fw_filename.relative_to(config.firmware_folder))
-    new_fw.firmware = source.as_uri() if isinstance(source, Path) else source
+    new_fw.firmware_file = str(fw_filename.relative_to(config.firmware_folder))
+    new_fw.source = source.as_uri() if isinstance(source, Path) else source
 
     if not copy_firmware(source, fw_filename, force):
         log.error(f"Failed to copy {source} to {fw_filename}")
         return False
     # add to inventory
-    with jsonlines.open(config.firmware_folder / "firmware.jsonl", "a") as writer:
-        log.info(f"Adding {new_fw.port} {new_fw.board}")
-        log.info(f"    to {fw_filename}")
+    with Session() as session:
+        # check if the firmware already exists
+        existing_fw = (
+            session.query(Firmware)
+            .filter(
+                Firmware.board_id == new_fw.board_id,
+                Firmware.version == new_fw.version,
+                Firmware.port == new_fw.port,
+            )
+            .first()
+        )
+        if existing_fw:
+            log.warning(f"Firmware {existing_fw} already exists")
+            if not force:
+                return False
+            # update the existing firmware
+            existing_fw.firmware_file = new_fw.firmware_file
+            existing_fw.source = new_fw.source
+            existing_fw.custom = custom
+            existing_fw.description = description
+        else:
+            session.add(new_fw)
 
-        writer.write(new_fw.to_dict())
     return True
 
 

mpflash/ask_input.py

@@ -11,7 +11,8 @@ from loguru import logger as log
 
 from .common import DownloadParams, FlashParams, ParamType
 from .config import config
-from .mpboard_id import get_known_boards_for_port, get_known_ports, known_stored_boards
+from .mpboard_id import (get_known_boards_for_port, known_ports,
+                         known_stored_boards)
 from .mpremoteboard import MPRemoteBoard
 from .versions import micropython_versions
 
@@ -106,7 +107,7 @@ def filter_matching_boards(answers: dict) -> Sequence[Tuple[str, str]]:
     Returns:
         Sequence[Tuple[str, str]]: The filtered boards.
     """
-    versions = None
+    versions = []
     # if version is not asked ; then need to get the version from the inputs
     if "versions" in answers:
         versions = list(answers["versions"])
@@ -151,7 +152,7 @@ def ask_port_board(*, multi_select: bool, action: str):
         inquirer.List(
             "port",
             message="Which port do you want to {action} " + "to {serial} ?" if action == "flash" else "?",
-            choices=get_known_ports(),
+            choices=known_ports(),
             # autocomplete=True,
         ),
         inquirer_ux(

mpflash/basicgit.py

@@ -24,6 +24,7 @@ from mpflash.config import config
 
 # GH_CLIENT = None
 
+
 def _run_local_git(
     cmd: List[str],
     repo: Optional[Union[Path, str]] = None,
@@ -180,7 +181,6 @@ def checkout_tag(tag: str, repo: Optional[Union[str, Path]] = None) -> bool:
     return True
 
 
-
 def checkout_commit(commit_hash: str, repo: Optional[Union[Path, str]] = None) -> bool:
     """
     Checkout a specific commit
@@ -267,7 +267,7 @@ def pull(repo: Union[Path, str], branch: str = "main") -> bool:
     return result.returncode == 0
 
 
-def get_git_describe(folder: Optional[Path | str] = None):
+def get_git_describe(folder: Optional[Union[Path, str]] = None):
     """
     Based on MicroPython makeversionhdr.py
     returns : current git tag, commits ,commit hash : "v1.19.1-841-g3446"

mpflash/bootloader/manual.py

@@ -55,7 +55,7 @@ def enter_bootloader_manual(mcu: MPRemoteBoard, timeout: int = 10):
     message: str
     if mcu.port == "rp2":
         message = f"""\
-Please put your {" ".join([mcu.port,mcu.board])} device into bootloader mode by either:
+Please put your {" ".join([mcu.port, mcu.board])} device into bootloader mode by either:
 Method 1:
   1. Unplug the USB cable, 
   2. Press and hold the BOOTSEL button on the device, 

mpflash/cli_download.py

@@ -6,8 +6,10 @@ import rich_click as click
 from loguru import logger as log
 
 from mpflash.connected import connected_ports_boards
+from mpflash.downloaded import clean_downloaded_firmwares
 from mpflash.errors import MPFlashError
 from mpflash.mpboard_id import find_known_board
+from mpflash.mpboard_id.alternate import add_renamed_boards
 from mpflash.versions import clean_version
 
 from .ask_input import ask_missing_params
@@ -26,8 +28,8 @@ from .download import download
     "-d",
     "fw_folder",
     type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
-    default=config.firmware_folder,
-    show_default=True,
+    default=None,
+    show_default=False,
     help="The folder to download the firmware to.",
 )
 @click.option(
@@ -92,7 +94,8 @@ def cli_download(**kwargs) -> int:
     params.boards = list(params.boards)
     params.serial = list(params.serial)
     params.ignore = list(params.ignore)
-
+    if params.fw_folder: 
+        config.firmware_folder = Path(params.fw_folder)
     # all_boards: List[MPRemoteBoard] = []
     if params.boards:
         if not params.ports:
@@ -116,13 +119,13 @@ def cli_download(**kwargs) -> int:
 
     try:
         download(
-            params.fw_folder,
             params.ports,
-            params.boards,
+            add_renamed_boards(params.boards),
             params.versions,
             params.force,
             params.clean,
         )
+        clean_downloaded_firmwares()
         return 0
     except MPFlashError as e:
         log.error(f"{e}")

mpflash/cli_flash.py

@@ -4,6 +4,8 @@ from typing import List
 import rich_click as click
 from loguru import logger as log
 
+import mpflash.download.jid as jid
+import mpflash.mpboard_id as mpboard_id
 from mpflash.ask_input import ask_missing_params
 from mpflash.cli_download import connected_ports_boards
 from mpflash.cli_group import cli
@@ -13,7 +15,6 @@ from mpflash.config import config
 from mpflash.errors import MPFlashError
 from mpflash.flash import flash_list
 from mpflash.flash.worklist import WorkList, full_auto_worklist, manual_worklist, single_auto_worklist
-from mpflash.mpboard_id import find_known_board
 from mpflash.mpremoteboard import MPRemoteBoard
 from mpflash.versions import clean_version
 
@@ -31,8 +32,8 @@ from mpflash.versions import clean_version
     "-f",
     "fw_folder",
     type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
-    default=config.firmware_folder,
-    show_default=True,
+    default=None,
+    show_default=False,
     help="The folder to retrieve the firmware from.",
 )
 @click.option(
@@ -69,7 +70,7 @@ from mpflash.versions import clean_version
 )
 @click.option(
     "--bluetooth/--no-bluetooth",
-    "-b/-nb",
+    "--bt/--no-bt",
     is_flag=True,
     default=False,
     show_default=True,
@@ -116,7 +117,7 @@ from mpflash.versions import clean_version
 )
 @click.option(
     "--bootloader",
-    "-bl",
+    "--bl",
     "bootloader",
     type=click.Choice([e.value for e in BootloaderMethod]),
     default="auto",
@@ -124,7 +125,8 @@ from mpflash.versions import clean_version
     help="""How to enter the (MicroPython) bootloader before flashing.""",
 )
 @click.option(
-    "--flash_mode", "-fm",
+    "--flash_mode",
+    "-fm",
     type=click.Choice(["keep", "qio", "qout", "dio", "dout"]),
     default="keep",
     show_default=True,
@@ -137,7 +139,7 @@ def cli_flash_board(**kwargs) -> int:
         kwargs["boards"] = []
         kwargs.pop("board")
     else:
-        kwargs["boards"] = [kwargs.pop("board")]   
+        kwargs["boards"] = [kwargs.pop("board")]
 
     params = FlashParams(**kwargs)
     params.versions = list(params.versions)
@@ -150,7 +152,8 @@ def cli_flash_board(**kwargs) -> int:
     # make it simple for the user to flash one board by asking for the serial port if not specified
     if params.boards == ["?"] and params.serial == "*":
         params.serial = ["?"]
-
+    if params.fw_folder: 
+        config.firmware_folder = Path(params.fw_folder)
     # Detect connected boards if not specified,
     # and ask for input if boards cannot be detected
     all_boards: List[MPRemoteBoard] = []
@@ -167,14 +170,12 @@ def cli_flash_board(**kwargs) -> int:
             # assume manual mode if no board is detected
             params.bootloader = BootloaderMethod("manual")
     else:
-        resolve_board_ids(params)
+        mpboard_id.resolve_board_ids(params)
 
     # Ask for missing input if needed
     params = ask_missing_params(params)
     if not params:  # Cancelled by user
         return 2
-    # TODO: Just in time Download of firmware
-
     assert isinstance(params, FlashParams)
 
     if len(params.versions) > 1:
@@ -183,6 +184,7 @@ def cli_flash_board(**kwargs) -> int:
 
     params.versions = [clean_version(v) for v in params.versions]
     worklist: WorkList = []
+
     # if serial port == auto and there are one or more specified/detected boards
     if params.serial == ["*"] and params.boards:
         if not all_boards:
@@ -191,37 +193,33 @@ def cli_flash_board(**kwargs) -> int:
         # if variant id provided on the cmdline, treat is as an override
         if params.variant:
             for b in all_boards:
-                b.variant = params.variant if (params.variant.lower() not in {"-","none"}) else ""
+                b.variant = params.variant if (params.variant.lower() not in {"-", "none"}) else ""
 
         worklist = full_auto_worklist(
             all_boards=all_boards,
             version=params.versions[0],
-            fw_folder=params.fw_folder,
             include=params.serial,
             ignore=params.ignore,
         )
     elif params.versions[0] and params.boards[0] and params.serial:
-        # A one or more  serial port including the board / variant
+        # A one or more serial port including the board / variant
         worklist = manual_worklist(
             params.serial[0],
             board_id=params.boards[0],
             version=params.versions[0],
-            fw_folder=params.fw_folder,
         )
     else:
         # just this serial port on auto
         worklist = single_auto_worklist(
             serial=params.serial[0],
             version=params.versions[0],
-            fw_folder=params.fw_folder,
         )
-
+    jid.ensure_firmware_downloaded(worklist, version=params.versions[0])
     if flashed := flash_list(
         worklist,
-        params.fw_folder,
         params.erase,
         params.bootloader,
-        flash_mode = params.flash_mode,
+        flash_mode=params.flash_mode,
     ):
         log.info(f"Flashed {len(flashed)} boards")
         show_mcus(flashed, title="Updated boards after flashing")
@@ -231,17 +229,3 @@ def cli_flash_board(**kwargs) -> int:
         return 1
 
 
-def resolve_board_ids(params: Params):
-    """Resolve board descriptions to board_id, and remove empty strings from list of boards"""
-    for board_id in params.boards:
-        if board_id == "":
-            params.boards.remove(board_id)
-            continue
-        if " " in board_id:
-            try:
-                if info := find_known_board(board_id):
-                    log.info(f"Resolved board description: {info.board_id}")
-                    params.boards.remove(board_id)
-                    params.boards.append(info.board_id)
-            except Exception as e:
-                log.warning(f"Unable to resolve board description: {e}")

mpflash/cli_group.py

@@ -10,6 +10,9 @@ from mpflash.vendor.click_aliases import ClickAliasedGroup
 from .config import __version__, config
 from .logger import log, make_quiet, set_loglevel
 
+# default log level
+set_loglevel("INFO")
+config.verbose = False
 
 def cb_verbose(ctx, param, value):
     """Callback to set the log level to DEBUG if verbose is set"""

mpflash/cli_list.py

@@ -48,7 +48,7 @@ from .logger import make_quiet
 )
 @click.option(
     "--bluetooth/--no-bluetooth",
-    "-b/-nb",
+    "--bt/--no-bt",
     is_flag=True,
     default=False,
     show_default=True,
@@ -77,7 +77,7 @@ def cli_list_mcus(serial: List[str], ignore: List[str], bluetooth: bool, as_json
     conn_mcus = [item for item in conn_mcus if not (item.toml.get("mpflash", {}).get("ignore", False))]
     if as_json:
         print(json.dumps([mcu.to_dict() for mcu in conn_mcus], indent=4))
-        
+
     if progress:
         show_mcus(conn_mcus, refresh=False)
     for mcu in conn_mcus:

mpflash/cli_main.py

@@ -9,9 +9,13 @@ from .cli_download import cli_download
 from .cli_flash import cli_flash_board
 from .cli_group import cli
 from .cli_list import cli_list_mcus
+from .db.core import migrate_database
 
 
 def mpflash():
+    """Main entry point for the mpflash CLI."""
+    migrate_database(boards=True, firmwares=True)
+
     cli.add_command(cli_list_mcus)
     cli.add_command(cli_download)
     cli.add_command(cli_flash_board)

mpflash/common.py

@@ -28,41 +28,6 @@ PORT_FWTYPES = {
 
 UF2_PORTS = [port for port, exts in PORT_FWTYPES.items() if ".uf2" in exts]
 
-@dataclass
-class FWInfo:
-    """
-    Downloaded Firmware information
-    is somewhat related to the BOARD class in the mpboard_id module
-    """
-
-    port: str  # MicroPython port
-    board: str  # MicroPython board
-    filename: str = field(default="")  # relative filename of the firmware image
-    firmware: str = field(default="")  # url or path to original firmware image
-    variant: str = field(default="")  # MicroPython variant
-    preview: bool = field(default=False)  # True if the firmware is a preview version
-    version: str = field(default="")  # MicroPython version (NO v prefix)
-    url: str = field(default="")  # url to the firmware image download folder
-    build: str = field(default="0")  # The build = number of commits since the last release
-    ext: str = field(default="")  # the file extension of the firmware
-    family: str = field(default="micropython")  # The family of the firmware
-    custom: bool = field(default=False)  # True if the firmware is a custom build
-    description: str = field(default="")  # Description used by this firmware (custom only)
-
-    def to_dict(self) -> dict:
-        """Convert the object to a dictionary"""
-        return self.__dict__
-
-    @classmethod
-    def from_dict(cls, data: dict) -> "FWInfo":
-        """Create a FWInfo object from a dictionary"""
-        # add missing keys
-        if "ext" not in data:
-            data["ext"] = Path(data["firmware"]).suffix
-        if "family" not in data:
-            data["family"] = "micropython"
-        return cls(**data)
-
 
 @dataclass
 class Params:
@@ -72,7 +37,7 @@ class Params:
     boards: List[str] = field(default_factory=list)
     variant: str = ""
     versions: List[str] = field(default_factory=list)
-    fw_folder: Path = Path()
+    fw_folder: Optional[Path] = None
     serial: List[str] = field(default_factory=list)
     ignore: List[str] = field(default_factory=list)
     bluetooth: bool = False

mpflash/config.py

@@ -31,6 +31,7 @@ class MPFlashConfig:
         # No interactions in CI
         if os.getenv("GITHUB_ACTIONS") == "true":
             from mpflash.logger import log
+
             log.warning("Disabling interactive mode in CI")
             return False
         return self._interactive
@@ -44,18 +45,38 @@ class MPFlashConfig:
         """The folder where firmware files are stored"""
         if not self._firmware_folder:
             self._firmware_folder = platformdirs.user_downloads_path() / "firmware"
+            # allow testing in CI
+            if Path(os.getenv("GITHUB_ACTIONS", "")).as_posix().lower() == "true":
+                workspace = os.getenv("GITHUB_WORKSPACE")
+                if workspace:
+                    ws_path = Path(workspace) / "firmware"
+                    ws_path.mkdir(parents=True, exist_ok=True)
+                    print(f"Detected GitHub Actions environment. Using workspace path: {ws_path}")
+                    self._firmware_folder = ws_path
         return self._firmware_folder
 
+    @firmware_folder.setter
+    def firmware_folder(self, value: Path):
+        """Set the firmware folder"""
+        if value.exists() and value.is_dir():
+            self._firmware_folder = value
+        else:
+            raise ValueError(f"Invalid firmware folder: {value}. It must be a valid directory.")
+
     @property
     def db_path(self) -> Path:
         """The path to the database file"""
         return self.firmware_folder / "mpflash.db"
+    @property
+    def db_version(self) -> str:
+        return "1.24.1"
 
     @property
     def gh_client(self):
         """The gh client to use"""
         if not self._gh_client:
             from github import Auth, Github
+
             # Token with no permissions to avoid throttling
             # https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#getting-a-higher-rate-limit
             PAT_NO_ACCESS = "github_pat_" + "11AAHPVFQ0G4NTaQ73Bw5J" + "_fAp7K9sZ1qL8VFnI9g78eUlCdmOXHB3WzSdj2jtEYb4XF3N7PDJBl32qIxq"

mpflash/db/__init__.py

@@ -0,0 +1,2 @@
+
+from .models import Base, Board, Firmware

mpflash/db/core.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+from sqlite3 import DatabaseError, OperationalError
+
+from loguru import logger as log
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from mpflash.config import config
+
+# TODO:  lazy import to avoid slowdowns ?
+from .models import Base
+
+TRACE = False
+connect_str = f"sqlite:///{config.db_path.as_posix()}"
+engine = create_engine(connect_str, echo=TRACE)
+Session = sessionmaker(bind=engine)
+
+
+def migrate_database(boards: bool = True, firmwares: bool = True):
+    """Migrate from 1.24.x to 1.25.x"""
+    # Move import here to avoid circular import
+    from .loader import load_jsonl_to_db, update_boards
+
+    # get the location of the database from the session
+    with Session() as session:
+        db_location = session.get_bind().url.database # type: ignore
+        log.debug(f"Database location: {Path(db_location)}") # type: ignore
+
+    create_database()
+    if boards:
+
+        update_boards()
+    if firmwares:
+        jsonl_file = config.firmware_folder / "firmware.jsonl"
+        if jsonl_file.exists():
+            log.info(f"Migrating JSONL data {jsonl_file} to SQLite database.")
+            load_jsonl_to_db(jsonl_file)
+            # rename the jsonl file to jsonl.bak
+            log.info(f"Renaming {jsonl_file} to {jsonl_file.with_suffix('.jsonl.bak')}")
+            try:
+                jsonl_file.rename(jsonl_file.with_suffix(".jsonl.bak"))
+            except OSError as e:
+                for i in range(1, 10):
+                    try:
+                        jsonl_file.rename(jsonl_file.with_suffix(f".jsonl.{i}.bak"))
+                        break
+                    except OSError:
+                        continue
+
+
+def create_database():
+    """
+    Create the SQLite database and tables if they don't exist.
+    """
+    # Create the database and tables if they don't exist
+    Base.metadata.create_all(engine)