mpflash 1.24.8__py3-none-any.whl → 1.25.0__py3-none-any.whl

mpflash/downloaded.py

@@ -5,27 +5,12 @@ import jsonlines
 from loguru import logger as log
 
 from mpflash.common import PORT_FWTYPES, FWInfo
+from mpflash.db.downloads import downloaded
 from mpflash.versions import clean_version
 
 from .config import config
 
-
 # #########################################################################################################
-def downloaded_firmwares(fw_folder: Path) -> List[FWInfo]:
-    """Load a list of locally downloaded firmwares from the jsonl file"""
-    firmwares: List[FWInfo] = []
-    log.debug(f"Reading  {fw_folder / 'firmware.jsonl' }")
-    try:
-        with jsonlines.open(fw_folder / "firmware.jsonl") as reader:
-            firmwares = [FWInfo.from_dict(item) for item in reader]
-    except FileNotFoundError:
-        log.error(f"No firmware.jsonl found in {fw_folder}")
-    except jsonlines.InvalidLineError as e:
-        log.error(f"Invalid firmware.jsonl found in {fw_folder} : {e}")
-
-    # sort by filename
-    firmwares.sort(key=lambda x: x.filename)
-    return firmwares
 
 
 def clean_downloaded_firmwares(fw_folder: Path) -> None:
@@ -33,15 +18,9 @@ def clean_downloaded_firmwares(fw_folder: Path) -> None:
     Remove duplicate entries from the firmware.jsonl file, keeping the latest one
     uniqueness is based on the filename
     """
-    firmwares = downloaded_firmwares(fw_folder)
-    if not firmwares:
-        return
-    # keep the latest entry
-    unique_fw = {fw.filename: fw for fw in firmwares}
-    with jsonlines.open(fw_folder / "firmware.jsonl", "w") as writer:
-        for fw in unique_fw.values():
-            writer.write(fw.to_dict())
-    log.info(f"Removed duplicate entries from firmware.jsonl in {fw_folder}")
+    # Duplication should no longer happen,
+    # but is would be a good idea to check the consistence between the DB and the downloads folder sometimes 
+    pass 
 
 
 def find_downloaded_firmware(
@@ -50,16 +29,16 @@ def find_downloaded_firmware(
     version: str = "",  # v1.2.3
     port: str = "",
     variants: bool = False,
-    fw_folder: Optional[Path] = None,
+    db_path: Optional[Path] = None,
     trie: int = 1,
     selector: Optional[Dict[str, str]] = None,
 ) -> List[FWInfo]:
     if selector is None:
         selector = {}
-    fw_folder = fw_folder or config.firmware_folder
+    
     # Use the information in firmwares.jsonl to find the firmware file
     log.debug(f"{trie}] Looking for firmware for {board_id} {version} ")
-    fw_list = downloaded_firmwares(fw_folder)
+    fw_list = downloaded()
     if not fw_list:
         log.error("No firmware files found. Please download the firmware first.")
         return []
@@ -80,7 +59,7 @@ def find_downloaded_firmware(
             board_id = board_id.replace("_", "-")
 
         fw_list = find_downloaded_firmware(
-            fw_folder=fw_folder,
+            db_path=db_path,
             board_id=board_id,
             version=version,
             port=port,

mpflash/flash/worklist.py

@@ -43,7 +43,7 @@ def auto_update(
             log.warning(f"Skipping flashing {mcu.family} {mcu.port} {mcu.board} on {mcu.serialport} as it is not a MicroPython firmware")
             continue
         board_firmwares = find_downloaded_firmware(
-            fw_folder=fw_folder,
+            db_path=fw_folder,
             board_id=mcu.board if not mcu.variant else f"{mcu.board}-{mcu.variant}",
             version=target_version,
             port=mcu.port,
@@ -91,11 +91,11 @@ def manual_worklist(
         # need the CPU type for the esptool
         mcu.cpu = info.cpu
     except (LookupError, MPFlashError) as e:
-        log.error(f"Board {board_id} not found in board_info.zip")
+        log.error(f"Board {board_id} not found in board database")
         log.exception(e)
         return []
     mcu.board = board_id
-    firmwares = find_downloaded_firmware(fw_folder=fw_folder, board_id=board_id, version=version, port=mcu.port)
+    firmwares = find_downloaded_firmware(db_path=fw_folder, board_id=board_id, version=version, port=mcu.port)
     if not firmwares:
         log.error(f"No firmware found for {mcu.port} {board_id} version {version}")
         return []

mpflash/mpboard_id/board.py

@@ -21,17 +21,20 @@ class Board:
     mcu_name: str = field(default="")
     cpu: str = field(default="")
     variant: str = field(default="")
+    mcu: str = field(default="")
 
     def __post_init__(self):
         if not self.cpu:
             if " with " in self.description:
-                self.cpu = self.description.split(" with ")[-1]
+                self.cpu = self.description.rsplit(" with ")[-1]
             else:
                 self.cpu = self.port
 
     @staticmethod
     def from_dict(data: dict) -> "Board":
-        return Board(**data)
+        valid_keys = {field.name for field in Board.__dataclass_fields__.values()}
+        filtered_data = {k: v for k, v in data.items() if k in valid_keys}
+        return Board(**filtered_data)
 
     def to_dict(self) -> dict:
         return self.__dict__

mpflash/mpboard_id/board_id.py

@@ -3,11 +3,15 @@ Translate board description to board designator
 """
 
 import functools
+import re
+import sqlite3
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
+from mpflash.config import config
 from mpflash.errors import MPFlashError
 from mpflash.logger import log
+from mpflash.mpboard_id.board import Board
 from mpflash.mpboard_id.store import read_known_boardinfo
 from mpflash.versions import clean_version, get_preview_mp_version, get_stable_mp_version
 
@@ -20,21 +24,78 @@ def find_board_id_by_description(
     board_info: Optional[Path] = None,
 ) -> Optional[str]:
     """Find the MicroPython BOARD_ID based on the description in the firmware"""
-
+    version = clean_version(version) if version else ""
     try:
         boards = _find_board_id_by_description(
             descr=descr,
             short_descr=short_descr,
-            board_info=board_info,
-            version=clean_version(version) if version else None,
+            db_path=board_info,
+            version=version,
         )
-        return boards[-1].board_id
+        if not boards:
+            log.debug(f"Version {version} not found in board info, using any version")
+            boards = _find_board_id_by_description(
+                descr=descr,
+                short_descr=short_descr,
+                db_path=board_info,
+                version="%",  # any version
+            )
+        return boards[0].board_id if boards else None
     except MPFlashError:
         return "UNKNOWN_BOARD"
 
 
-@functools.lru_cache(maxsize=20)
 def _find_board_id_by_description(
+    *,
+    descr: str,
+    short_descr: str,
+    version: Optional[str] = None,
+    variant: str = "",
+    db_path: Optional[Path] = None,
+):
+    short_descr = short_descr or ""
+    boards: List[Board] = []
+    version = clean_version(version) if version else "%"
+    if "-preview" in version:
+        version = version.replace("-preview", "%")
+    descriptions = [descr, short_descr]
+    if descr.startswith("Generic"):
+        descriptions.append(descr[8:])
+        descriptions.append(short_descr[8:])
+
+    try:
+        with sqlite3.connect(db_path or config.db_path) as conn:
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            qry = f"""
+            SELECT 
+                *
+            FROM board_downloaded
+            WHERE 
+                board_id IN (
+                    SELECT  DISTINCT board_id
+                    FROM board_downloaded
+                    WHERE description IN {str(tuple(descriptions))}
+                )
+                AND version like '{version}'
+                AND variant like '{variant}'
+            """
+            cursor.execute(qry)
+            rows = cursor.fetchall()
+            for row in rows:
+                r = dict(row)
+
+                boards.append(Board.from_dict(dict(row)))
+    except sqlite3.OperationalError as e:
+        raise MPFlashError("Database error") from e
+    if not boards:
+        raise MPFlashError(f"No board info found for description '{descr}' or '{short_descr}'")
+    return boards
+
+
+@functools.lru_cache(maxsize=20)
+def _find_board_id_by_description_xx(
     *,
     descr: str,
     short_descr: str,

mpflash/mpboard_id/known.py

@@ -7,8 +7,10 @@ This module provides access to the board info and the known ports and boards."""
 from functools import lru_cache
 from typing import List, Optional, Tuple
 
+from mpflash.db.boards import find_board_id, find_board_info
 from mpflash.errors import MPFlashError
 from mpflash.versions import clean_version
+from mpflash.logger import log
 
 from .board import Board
 from .store import read_known_boardinfo
@@ -16,6 +18,7 @@ from .store import read_known_boardinfo
 
 def get_known_ports() -> List[str]:
     # TODO: Filter for Version
+    log.warning("get_known_ports() is deprecated")
     mp_boards = read_known_boardinfo()
     # select the unique ports from info
     ports = set({board.port for board in mp_boards if board.port})
@@ -41,6 +44,9 @@ def get_known_boards_for_port(port: Optional[str] = "", versions: Optional[List[
     if "preview" in versions:
         versions.remove("preview")
         versions.append("stable")
+    # filter for the port
+    if port:
+        mp_boards = [board for board in mp_boards if board.port == port]
     if versions:
         # make sure of the v prefix
         versions = [clean_version(v) for v in versions]
@@ -53,9 +59,6 @@ def get_known_boards_for_port(port: Optional[str] = "", versions: Optional[List[
             last_known_version = sorted({b.version for b in mp_boards})[-1]
             mp_boards = [board for board in mp_boards if board.version == last_known_version]
 
-    # filter for the port
-    if port:
-        mp_boards = [board for board in mp_boards if board.port == port]
     return mp_boards
 
 
@@ -73,22 +76,33 @@ def known_stored_boards(port: str, versions: Optional[List[str]] = None) -> List
 
 
 @lru_cache(maxsize=20)
-def find_known_board(board_id: str) -> Board:
+def find_known_board(board_id: str, version ="") -> Board:
     """Find the board for the given BOARD_ID or 'board description' and return the board info as a Board object"""
     # Some functional overlap with:
     # mpboard_id\board_id.py _find_board_id_by_description
-    info = read_known_boardinfo()
-    for board_info in info:
-        if board_id in (
-            board_info.board_id,
-            board_info.description,
-        ) or board_info.description.startswith(board_id):
-            if not board_info.cpu:
-                # failsafe for older board_info.json files
-                print(f"Board {board_id} has no CPU info, using port as CPU")
-                if " with " in board_info.description:
-                    board_info.cpu = board_info.description.split(" with ")[-1]
-                else:
-                    board_info.cpu = board_info.port
-            return board_info
+    # TODO: Refactor to search the SQLite DB instead of the JSON file
+    board_ids = find_board_id(board_id = board_id, version = version or "%")
+    boards = []
+    for board_id in board_ids:
+        # if we have a board_id, use it to find the board info
+        boards += [Board.from_dict(dict(r)) for r in find_board_info(board_id = board_id)]
+  
+
+    # if board_ids:
+    #     # if we have a board_id, use it to find the board info
+    #     board_id = board_ids[0]
+    # info = read_known_boardinfo()
+    # for board_info in info:
+    #     if board_id in (
+    #         board_info.board_id,
+    #         board_info.description,
+    #     ) or board_info.description.startswith(board_id):
+    #         if not board_info.cpu:
+    #             # failsafe for older board_info.json files
+    #             print(f"Board {board_id} has no CPU info, using port as CPU")
+    #             if " with " in board_info.description:
+    #                 board_info.cpu = board_info.description.split(" with ")[-1]
+    #             else:
+    #                 board_info.cpu = board_info.port
+    #         return board_info
     raise MPFlashError(f"Board {board_id} not found")

mpflash/mpboard_id/store.py

@@ -2,6 +2,7 @@ import functools
 import zipfile
 from pathlib import Path
 from typing import Final, List, Optional
+from mpflash.logger import log
 
 import jsons
 
@@ -32,7 +33,7 @@ def write_boardinfo_json(board_list: List[Board], *, folder: Optional[Path] = No
 @functools.lru_cache(maxsize=20)
 def read_known_boardinfo(board_info: Optional[Path] = None) -> List[Board]:
     """Reads the board information from a JSON file in a zip file."""
-
+    log.warning("read_known_boardinfo() is deprecated")
 
     if not board_info:
         board_info = HERE / "board_info.zip"

mpflash/mpremoteboard/__init__.py

@@ -2,7 +2,6 @@
 Module to run mpremote commands, and retry on failure or timeout
 """
 
-
 import contextlib
 import sys
 import time
@@ -35,7 +34,9 @@ RETRIES = 3
 class MPRemoteBoard:
     """Class to run mpremote commands"""
 
-    def __init__(self, serialport: str = "", update: bool = False, *, location: str = ""):
+    def __init__(
+        self, serialport: str = "", update: bool = False, *, location: str = ""
+    ):
         """
         Initialize MPRemoteBoard object.
 
@@ -43,6 +44,8 @@ class MPRemoteBoard:
         - serialport (str): The serial port to connect to. Default is an empty string.
         - update (bool): Whether to update the MCU information. Default is False.
         """
+        self._board_id = ""
+
         self.serialport: str = serialport
         self.firmware = {}
 
@@ -52,8 +55,6 @@ class MPRemoteBoard:
         self.description = ""
         self.version = ""
         self.port = ""
-        self.board = ""
-        self.variant= ""
         self.cpu = ""
         self.arch = ""
         self.mpy = ""
@@ -63,6 +64,33 @@ class MPRemoteBoard:
         if update:
             self.get_mcu_info()
 
+    ###################################
+    # board_id := board[-variant]
+    @property
+    def board_id(self) -> str:
+        return self._board_id
+
+    @board_id.setter
+    def board_id(self, value: str) -> None:
+        self._board_id = value.rstrip("-")
+
+    @property
+    def board(self) -> str:
+        return self._board_id.split("-")[0]
+
+    @board.setter
+    def board(self, value: str) -> None:
+        self.board_id = f"{value}-{self.variant}" if self.variant else value
+
+    @property
+    def variant(self) -> str:
+        return self._board_id.split("-")[1] if "-" in self._board_id else ""
+
+    @variant.setter
+    def variant(self, value: str) -> None:
+        self.board_id = f"{self.board}-{value}"
+
+    ###################################
     def __str__(self):
         """
         Return a string representation of the MPRemoteBoard object.
@@ -73,7 +101,9 @@ class MPRemoteBoard:
         return f"MPRemoteBoard({self.serialport}, {self.family} {self.port}, {self.board}{f'-{self.variant}' if self.variant else ''}, {self.version})"
 
     @staticmethod
-    def connected_boards(bluetooth: bool = False, description: bool = False) -> List[str]:
+    def connected_boards(
+        bluetooth: bool = False, description: bool = False
+    ) -> List[str]:
         # TODO: rename to connected_comports
         """
         Get a list of connected comports.
@@ -101,7 +131,10 @@ class MPRemoteBoard:
         if sys.platform == "win32":
             # Windows sort of comports by number - but fallback to device name
             return sorted(
-                output, key=lambda x: int(x.split()[0][3:]) if x.split()[0][3:].isdigit() else x
+                output,
+                key=lambda x: int(x.split()[0][3:])
+                if x.split()[0][3:].isdigit()
+                else x,
             )
         # sort by device name
         return sorted(output)
@@ -123,11 +156,11 @@ class MPRemoteBoard:
             timeout=timeout,
             resume=False,  # Avoid restarts
         )
-        if rc:
+        if rc not in (0, 1):  ## WORKAROUND - SUDDEN RETURN OF 1 on success
             log.debug(f"rc: {rc}, result: {result}")
             raise ConnectionError(f"Failed to get mcu_info for {self.serialport}")
         # Ok we have the info, now parse it
-        raw_info = result[0].strip()
+        raw_info = result[0].strip() if result else ""
         if raw_info.startswith("{") and raw_info.endswith("}"):
             info = eval(raw_info)
             self.family = info["family"]
@@ -140,17 +173,19 @@ class MPRemoteBoard:
             self.description = descr = info["board"]
             pos = descr.rfind(" with")
             short_descr = descr[:pos].strip() if pos != -1 else ""
-            if info["_build"]: 
-                self.board = info["_build"].split('-')[0]
-                self.variant = info["_build"].split('-')[1] if '-' in info["_build"] else ""
-            elif board_name := find_board_id_by_description(
-                    descr, short_descr, version=self.version
-                ):
-                self.board = board_name
+            if info.get("board_id", None):
+                # we have a board_id - so use that to get the board name
+                self.board_id = info["board_id"]
             else:
-                self.board = "UNKNOWN_BOARD"
+                self.board_id = f"{info['board']}-{info.get('variant', '')}"
+                board_name = find_board_id_by_description(
+                    descr, short_descr, version=self.version
+                )
+                self.board_id = board_name or "UNKNOWN_BOARD"
+                # TODO: Get the variant as well
             # get the board_info.toml
             self.get_board_info_toml()
+            # TODO: get board_id from the toml file if it exists
         # now we know the board is connected
         self.connected = True
 
@@ -174,7 +209,9 @@ class MPRemoteBoard:
                 log_errors=False,
             )
         except Exception as e:
-            raise ConnectionError(f"Failed to get board_info.toml for {self.serialport}:") from e
+            raise ConnectionError(
+                f"Failed to get board_info.toml for {self.serialport}:"
+            ) from e
         # this is optional - so only parse if we got the file
         self.toml = {}
         if rc in [OK]:  # sometimes we get an -9 ???
@@ -273,3 +310,30 @@ class MPRemoteBoard:
             with contextlib.suppress(ConnectionError, MPFlashError):
                 self.get_mcu_info()
                 break
+
+    def to_dict(self) -> dict:
+        """
+        Serialize the MPRemoteBoard object to JSON, including all attributes and readable properties.
+
+        Returns:
+        - str: A JSON string representation of the object.
+        """
+
+        def get_properties(obj):
+            """Helper function to get all readable properties."""
+            return {
+                name: getattr(obj, name)
+                for name in dir(obj.__class__)
+                if isinstance(getattr(obj.__class__, name, None), property)
+            }
+
+        # Combine instance attributes, readable properties, and private attributes
+        data = {**self.__dict__, **get_properties(self)}
+
+        # remove the path and firmware attibutes from the json output as they are always empty
+        del data["_board_id"]  # dup of board_id
+        del data["connected"]
+        del data["path"]
+        del data["firmware"]
+
+        return data

mpflash/mpremoteboard/mpy_fw_info.py

@@ -3,7 +3,7 @@ import os
 import sys
 
 
-def _build(s):
+def get_build(s):
     # extract build from sys.version or os.uname().version if available
     # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f'
     # sys.implementation.version: 'v1.13-103-gb137d064e'
@@ -49,10 +49,14 @@ def _info():  # type:() -> dict[str, str]
     except AttributeError:
         pass
     try:
-        machine = sys.implementation._machine if "_machine" in dir(sys.implementation) else os.uname().machine  # type: ignore
-        info["board"] = machine.strip()
-        info["_build"] = sys.implementation._build if "_build" in dir(sys.implementation) else ""
-        info["cpu"] = machine.split("with")[-1].strip() if "with" in machine else ""
+        _machine = sys.implementation._machine if "_machine" in dir(sys.implementation) else os.uname().machine  # type: ignore
+        info["board"] = _machine.strip()
+        si_build = sys.implementation._build if "_build" in dir(sys.implementation) else ""
+        if si_build:
+            info["board"] = si_build.split("-")[0]
+            info["variant"] = si_build.split("-")[1] if "-" in si_build else ""
+        info["board_id"] = si_build
+        info["cpu"] = _machine.split("with")[-1].strip() if "with" in _machine else ""
         info["mpy"] = (
             sys.implementation._mpy
             if "_mpy" in dir(sys.implementation)
@@ -65,12 +69,12 @@ def _info():  # type:() -> dict[str, str]
 
     try:
         if hasattr(sys, "version"):
-            info["build"] = _build(sys.version)
+            info["build"] = get_build(sys.version)
         elif hasattr(os, "uname"):
-            info["build"] = _build(os.uname()[3])  # type: ignore
+            info["build"] = get_build(os.uname()[3])  # type: ignore
             if not info["build"]:
                 # extract build from uname().release if available
-                info["build"] = _build(os.uname()[2])  # type: ignore
+                info["build"] = get_build(os.uname()[2])  # type: ignore
     except (AttributeError, IndexError):
         pass
     # avoid  build hashes
@@ -146,4 +150,4 @@ def _info():  # type:() -> dict[str, str]
 
 
 print(_info())
-del _info, _build, _version_str
+del _info, get_build, _version_str