mpflash 1.25.0.post1__py3-none-any.whl → 1.25.0rc1__py3-none-any.whl

mpflash/db/loader.py

@@ -0,0 +1,120 @@
+import csv
+import io
+import json
+import re
+import zipfile
+from pathlib import Path
+from turtle import up
+
+from loguru import logger as log
+
+from mpflash.errors import MPFlashError
+
+from .core import Session
+from .meta import get_metadata, set_metadata_value
+from .models import Board, Firmware
+
+HERE = Path(__file__).parent.resolve()
+
+
+def load_data_from_zip(zip_file: Path) -> int:
+    log.debug("Loading data from zip file")
+    csv_filename = "micropython_boards.csv"  # name of the .csv inside the .zip
+    # Check if the zip file exists
+    if not zip_file.exists() or not zip_file.is_file():
+        log.error(f"Zip file {zip_file} not found.")
+        return 0
+    count = 0 
+    # Load data directly from the zip file
+    with zipfile.ZipFile(zip_file, "r") as zipf:
+        # Read the CSV file from the zip
+        with zipf.open(csv_filename) as csv_file:
+            log.info("Reading CSV data...")
+            reader = csv.DictReader(io.TextIOWrapper(csv_file, "utf-8"))
+
+            # save to database
+            with Session() as session:
+                for row in reader:
+                    # Create a board instance from the row data
+                    board = Board(**row)
+                    # Use merge to update existing or insert new record
+                    # based on primary key (board_id and version)
+                    session.merge(board)
+                    count += 1
+                session.commit()
+    log.info(f"Loaded {count} boards from {zip_file}")
+    return count
+
+
+def load_jsonl_to_db(jsonl_path: Path):
+    """
+    Load a JSONL file into a SQLite database
+
+    Args:
+        jsonl_path (Path): Path to the JSONL file
+        conn (sqlite3.Connection): SQLite database connection
+        table_name (str): Name of the table to insert data into
+
+    Returns:
+        int: Number of records imported
+    """
+    log.debug("Loading JSONL file into database")
+    # Ensure file exists
+    if not jsonl_path.exists():
+        raise FileNotFoundError(f"JSONL file not found: {jsonl_path}")
+    num_records = 0
+    with jsonl_path.open("r", encoding="utf-8") as file:
+        with Session() as session:
+            for line in file:
+                record = json.loads(line.strip())
+                # Clean up the record
+
+                if "variant" in record:
+                    record["board_id"] = record.pop("variant")  # Rename 'variant' to 'board_id'
+                if "firmware" in record:
+                    record["source"] = record.pop("firmware")  # Rename 'firmware' to 'source'
+                if "preview" in record:
+                    record["version"] = f"{record['version']}-preview" if record["preview"] else record["version"]
+                    record.pop("preview", None)  # Remove 'preview' column
+                firmware_file = str(Path(record["filename"]).as_posix()) if record["filename"] else ""
+
+                # Check if Firmware with this firmware_file exists
+                existing_fw = session.query(Firmware).filter_by(firmware_file=firmware_file).first()
+                if existing_fw:
+                    # Update fields
+                    existing_fw.board_id = record["board_id"]
+                    existing_fw.version = record["version"]
+                    existing_fw.source = record["source"]
+                    existing_fw.build = record["build"]
+                    existing_fw.custom = record["custom"]
+                    existing_fw.port = record["port"]
+                else:
+                    # Add new Firmware
+                    fw = Firmware(
+                        board_id=record["board_id"],
+                        version=record["version"],
+                        firmware_file=firmware_file,
+                        source=record["source"],
+                        build=record["build"],
+                        custom=record["custom"],
+                        port=record["port"],
+                    )
+                    session.merge(fw)
+                num_records += 1
+            # commit once after all records are processed
+            session.commit()
+    return num_records
+
+
+def update_boards():
+    try:
+        meta = get_metadata()
+        log.debug(f"Metadata: {meta}")
+        if meta.get("boards_version", "") < "v1.25.0":
+            log.info("Update boards from CSV to SQLite database.")
+            # Load data from the zip file into the database
+            load_data_from_zip(HERE / "micropython_boards.zip")
+            set_metadata_value("boards_version", "v1.25.0")
+            meta = get_metadata()
+    except Exception as e:
+        raise MPFlashError(f"Error updating boards table: {e}") from e

mpflash/db/meta.py

@@ -0,0 +1,78 @@
+
+from typing import Optional
+
+from loguru import logger as log
+
+from .core import DatabaseError, OperationalError, Session
+from .models import Metadata
+
+
+def get_metadata() -> dict:
+    """
+    Get all metadata from the database.
+
+    Returns:
+        dict: Dictionary of metadata name-value pairs.
+    """
+    try:
+        with Session() as session:
+            metadata = session.query(Metadata).all()
+            return {m.name: m.value for m in metadata}
+    except (DatabaseError, OperationalError)  as e:
+        log.error(f"Error retrieving metadata: {e}")
+        return {}
+    
+def set_metadata(metadata: dict):
+    """
+    Set metadata in the database.
+    Args:
+        metadata (dict): Dictionary of metadata name-value pairs.
+    Returns:
+        None
+    """
+    with Session() as session:
+        for name, value in metadata.items():
+            existing_metadata = session.query(Metadata).filter(Metadata.name == name).first()
+            if existing_metadata:
+                existing_metadata.value = value
+            else:
+                new_metadata = Metadata(name=name, value=value)
+                session.add(new_metadata)
+        session.commit()
+
+
+def get_metadata_value(name: str) -> Optional[str]:
+    """
+    Get metadata value by name.
+
+    Args:
+        session (Session): SQLAlchemy session.
+        name (str): Name of the metadata.
+
+    Returns:
+        Optional[str]: Metadata value or None if not found.
+    """
+    with Session() as session:
+        metadata = session.query(Metadata).filter(Metadata.name == name).first()
+    return metadata.value if metadata else None
+
+def set_metadata_value(name: str, value: str):
+    """
+    Set metadata value by name.
+
+    Args:
+        session (Session): SQLAlchemy session.
+        name (str): Name of the metadata.
+        value (str): Value to set.
+
+    Returns:
+        None
+    """
+    with Session() as session:
+        metadata = session.query(Metadata).filter(Metadata.name == name).first()
+        if metadata:
+            metadata.value = value
+        else:
+            new_metadata = Metadata(name=name, value=value)
+            session.add(new_metadata)
+        session.commit()

mpflash/db/models.py

@@ -0,0 +1,93 @@
+from pathlib import Path
+
+import sqlalchemy as sa
+from sqlalchemy import Index, String
+from sqlalchemy.orm import DeclarativeBase, Mapped, composite, mapped_column, relationship
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class Metadata(Base):
+    """
+    Configuration information.
+    """
+
+    __tablename__ = "metadata"
+    name: Mapped[str] = mapped_column(primary_key=True, unique=True)
+    value: Mapped[str] = mapped_column()
+
+    def __repr__(self) -> str:
+        return f"Config(boards_version={self.name!r}, schema_version={self.value!r})"
+
+
+class Board(Base):
+    """
+    All known Boards model for storing board information.
+    """
+
+    __tablename__ = "boards"
+    __table_args__ = (sa.UniqueConstraint("board_id", "version"),)
+
+    board_id: Mapped[str] = mapped_column(String(40), primary_key=True, unique=False)
+    version: Mapped[str] = mapped_column(String(12), primary_key=True, unique=False)
+    board_name: Mapped[str] = mapped_column()
+    mcu: Mapped[str] = mapped_column()
+    variant: Mapped[str] = mapped_column(default="")
+    port: Mapped[str] = mapped_column(String(30))
+    path: Mapped[str] = mapped_column(comment="Path in micropyton repo as_posix()")
+    description: Mapped[str] = mapped_column()
+    family: Mapped[str] = mapped_column(default="micropython")
+    custom: Mapped[bool] = mapped_column(default=False, comment="True if this is a custom board")
+    firmwares = relationship(
+        "Firmware",
+        back_populates="board",
+        lazy="joined",
+    )
+
+    board_key = composite(lambda board_id, version: f"{board_id}_{version}", board_id, version)
+
+    def __repr__(self) -> str:
+        return f"Board(board_id={self.board_id!r}, version={self.version!r}, board_name={self.board_name!r})"
+
+
+class Firmware(Base):
+    """
+    Firmware model for storing firmware information.
+    """
+
+    __tablename__ = "firmwares"
+    __table_args__ = (
+        sa.ForeignKeyConstraint(["board_id", "version"], ["boards.board_id", "boards.version"]),
+        {"sqlite_autoincrement": False},
+    )
+
+    board_id: Mapped[str] = mapped_column(String(40), primary_key=True)
+    version: Mapped[str] = mapped_column(String(12), primary_key=True)
+    firmware_file: Mapped[str] = mapped_column(String, primary_key=True, index=True, comment="Path to the firmware file")
+    # Relationship to Board
+    board: Mapped["Board"] = relationship(
+        "Board",
+        back_populates="firmwares",
+        lazy="joined",
+        primaryjoin="and_(Firmware.board_id==Board.board_id, Firmware.version==Board.version)",
+    )
+    port: Mapped[str] = mapped_column(String(20), default="")  # duplicate of board.port
+    description: Mapped[str] = mapped_column(default="")
+    source: Mapped[str] = mapped_column()
+    build: Mapped[int] = mapped_column(default=0, comment="Build number")
+    custom: Mapped[bool] = mapped_column(default=False, comment="True if this is a custom firmware")
+
+    @property
+    def preview(self) -> bool:
+        "Check if the firmware is a preview version."
+        return "preview" in self.firmware_file
+
+    @property
+    def ext(self) -> str:
+        "Get the file extension of the firmware file."
+        return Path(self.firmware_file).suffix
+
+    def __repr__(self) -> str:
+        return f"Firmware(board_id={self.board_id!r}, version={self.version!r}, firmware_file={self.firmware_file!r})"

mpflash/db/tools.py

@@ -0,0 +1,27 @@
+import sqlite3
+from pathlib import Path
+from typing import List
+
+from mpflash.config import config
+from mpflash.errors import MPFlashError
+from mpflash.logger import log
+
+
+def backup_db(source_db: Path, backup_path: Path):
+    """
+    Backup the SQLite database to a specified path.
+
+    Args:
+        conn (sqlite3.Connection): SQLite connection object
+        backup_path (str or Path): Path to save the backup file
+    """
+
+    # Ensure the backup directory exists
+    backup_path.parent.mkdir(parents=True, exist_ok=True)
+    with sqlite3.connect(source_db) as conn:
+        # Perform the backup
+        with open(backup_path, "wb") as f:
+            for line in conn.iterdump():
+                f.write(f"{line}\n".encode("utf-8"))
+
+    log.info(f"Backup created at {backup_path}")

mpflash/download/__init__.py

@@ -5,7 +5,6 @@ Uses the micropython.org website to get the available versions and locations to
 
 import itertools
 from pathlib import Path
-import sqlite3
 from typing import Dict, List, Optional
 
 # #########################################################################################################
@@ -14,27 +13,27 @@ import jsonlines
 from loguru import logger as log
 from rich.progress import track
 
-from mpflash.common import PORT_FWTYPES, FWInfo
-from .from_web import get_boards, fetch_firmware_files
+from mpflash.common import PORT_FWTYPES
+from mpflash.config import config
+from mpflash.db.core import Session
+from mpflash.db.models import Firmware
 from mpflash.downloaded import clean_downloaded_firmwares
 from mpflash.errors import MPFlashError
+from mpflash.mpboard_id.alternate import add_renamed_boards
 from mpflash.versions import clean_version
-from mpflash.config import config
-from mpflash.db.downloads import upsert_download
+
+from .from_web import fetch_firmware_files, get_boards
+from .fwinfo import FWInfo
+
 # avoid conflict with the ujson used by MicroPython
 jsonlines.ujson = None  # type: ignore
 # #########################################################################################################
 
 
 
-def key_fw_ver_pre_ext_bld(x: FWInfo):
-    "sorting key for the retrieved board urls"
-    return x.variant, x.version, x.preview, x.ext, x.build
-
-
-def key_fw_var_pre_ext(x: FWInfo):
+def key_fw_boardid_preview_ext(fw: Firmware):
     "Grouping key for the retrieved board urls"
-    return x.variant, x.preview, x.ext
+    return fw.board_id, fw.preview, fw.ext
 
 
 def download_firmwares(
@@ -59,18 +58,18 @@ def download_firmwares(
     """
 
 
-    skipped = downloaded = 0
+    downloaded = 0
     versions = [] if versions is None else [clean_version(v) for v in versions]
-    # handle renamed boards
+    # handle downloading firmware for renamed boards
     boards = add_renamed_boards(boards)
 
     available_firmwares = get_firmware_list(ports, boards, versions, clean)
 
     for b in available_firmwares:
-        log.debug(b.filename)
+        log.debug(b.firmware_file)
     # relevant
 
-    log.info(f"Found {len(available_firmwares)} relevant unique firmwares")
+    log.info(f"Found {len(available_firmwares)} potentially relevant firmwares")
     if not available_firmwares:
         log.error("No relevant firmwares could be found on https://micropython.org/download")
         log.info(f"{versions=} {ports=} {boards=}")
@@ -83,7 +82,7 @@ def download_firmwares(
     log.success(f"Downloaded {downloaded} firmware images." )
     return downloaded 
 
-def download_firmware_files(available_firmwares :List[FWInfo],firmware_folder:Path, force:bool ):
+def download_firmware_files(available_firmwares :List[Firmware],firmware_folder:Path, force:bool ):
     """
     Downloads the firmware files to the specified folder.
     Args:
@@ -94,23 +93,23 @@ def download_firmware_files(available_firmwares :List[FWInfo],firmware_folder:Pa
     """
 
     # with jsonlines.open(firmware_folder / "firmware.jsonl", "a") as writer:
-    with sqlite3.connect(config.db_path) as conn:
+    with Session() as session:
         # skipped, downloaded = fetch_firmware_files(available_firmwares, firmware_folder, force, requests, writer)
         downloaded = 0
         for fw in fetch_firmware_files(available_firmwares, firmware_folder, force):
-            upsert_download(conn, fw)
-            # writer.write(fw)
-            log.debug(f" {fw.filename} downloaded")
+            session.merge(fw)
+            log.debug(f" {fw.firmware_file} downloaded")
             downloaded += 1
+        session.commit()
     if downloaded > 0:
-        clean_downloaded_firmwares(firmware_folder)
+        clean_downloaded_firmwares()
     return downloaded
 
 
 
 def get_firmware_list(ports: List[str], boards: List[str], versions: List[str], clean: bool = True):
     """
-    Retrieves a list of unique firmware files available om micropython.org > downloads
+    Retrieves a list of unique firmware files potentially  available on micropython.org > downloads
     based on the specified ports, boards, versions, and clean flag.
 
     Args:
@@ -126,36 +125,43 @@ def get_firmware_list(ports: List[str], boards: List[str], versions: List[str],
 
     log.trace("Checking MicroPython download pages")
     versions = [clean_version(v, drop_v=False) for v in versions]
-    preview = "preview" in versions
+    preview = any("preview" in v for v in versions)
 
-    board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
+    # board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
+    board_urls = get_boards(ports, boards, clean)
 
     log.debug(f"Total {len(board_urls)} firmwares")
+    if versions:
+        # filter out the boards that are not in the versions list
+        relevant = [
+            board for board in board_urls if ( 
+                board.version in versions 
+                # or (preview and board.preview )
+                # and board.board_id in boards 
+                # and board.build == "0" 
+                # and not board.preview
+            )
+        ]
+    else:
+        relevant = board_urls
 
-    relevant = [
-        board for board in board_urls if board.version in versions and board.build == "0" and board.board in boards and not board.preview
-    ]
-
-    if preview:
-        relevant.extend([board for board in board_urls if board.board in boards and board.preview])
     log.debug(f"Matching firmwares: {len(relevant)}")
     # select the unique boards
-    unique_boards: List[FWInfo] = []
-    for _, g in itertools.groupby(relevant, key=key_fw_var_pre_ext):
-        # list is aleady sorted by build so we can just get the last item
+    unique_boards: List[Firmware] = []
+    for _, g in itertools.groupby(relevant, key=key_fw_boardid_preview_ext):
+        # list is aleady sorted by build (desc)  so we can just get the first item
         sub_list = list(g)
-        unique_boards.append(sub_list[-1])
-    log.debug(f"Last preview only: {len(unique_boards)}")
+        unique_boards.append(sub_list[0])
+    log.debug(f"Including preview: {len(unique_boards)}")
     return unique_boards
 
 
 def download(
-    destination: Path,
     ports: List[str],
     boards: List[str],
     versions: List[str],
-    force: bool,
-    clean: bool,
+    force: bool = False,
+    clean: bool = True,
 ) -> int:
     """
     Downloads firmware files based on the specified destination, ports, boards, versions, force flag, and clean flag.
@@ -177,7 +183,7 @@ def download(
     """
     # Just in time import
     import requests
-
+    destination = config.firmware_folder
     if not boards:
         log.critical("No boards found, please connect a board or specify boards to download firmware for.")
         raise MPFlashError("No boards found")
@@ -195,27 +201,3 @@ def download(
 
     return result
 
-
-def add_renamed_boards(boards: List[str]) -> List[str]:
-    """
-    Adds the renamed boards to the list of boards.
-
-    Args:
-        boards : The list of boards to add the renamed boards to.
-
-    Returns:
-        List[str]: The list of boards with the renamed boards added.
-    """
-
-    renamed = {
-        "PICO": ["RPI_PICO"],
-        "PICO_W": ["RPI_PICO_W"],
-        "GENERIC": ["ESP32_GENERIC", "ESP8266_GENERIC"],  # just add both of them
-    }
-    _boards = boards.copy()
-    for board in boards:
-        if board in renamed and renamed[board] not in boards:
-            _boards.extend(renamed[board])
-        if board != board.upper() and board.upper() not in boards:
-            _boards.append(board.upper())
-    return _boards

mpflash/download/from_web.py

@@ -5,18 +5,16 @@ from pathlib import Path
 from typing import Dict, List, Optional
 from urllib.parse import urljoin
 
-
 from loguru import logger as log
 from rich.progress import track
 
-from mpflash.common import PORT_FWTYPES, FWInfo
+from mpflash.common import PORT_FWTYPES
+from mpflash.db.models import Firmware
 from mpflash.downloaded import clean_downloaded_firmwares
 from mpflash.errors import MPFlashError
-from mpflash.mpboard_id import get_known_ports
+from mpflash.mpboard_id import known_ports
 from mpflash.versions import clean_version
 
-
-
 MICROPYTHON_ORG_URL = "https://micropython.org/"
 
 
@@ -24,8 +22,8 @@ MICROPYTHON_ORG_URL = "https://micropython.org/"
 RE_DATE = r"(-\d{8}-)"
 RE_HASH = r"(.g[0-9a-f]+\.)"
 # regex to extract the version and the build from the firmware filename
-# group 1 is the version, group 2 is the build
-RE_VERSION_PREVIEW = r"v([\d\.]+)-?(?:preview\.)?(\d+)?\."
+# group 1 is the version+Preview , gr2 just the version, group 3 is the build
+RE_VERSION_PREVIEW = r"v(([\d\.]+)(?:-preview)?)\.?(\d+)?\."
 
 
 # 'RPI_PICO_W-v1.23.uf2'
@@ -107,7 +105,7 @@ def board_firmware_urls(board_url: str, base_url: str, ext: str) -> List[str]:
 # boards we are interested in ( this avoids getting a lot of boards we don't care about)
 # The first run takes ~60 seconds to run for 4 ports , all boards
 # so it makes sense to cache the results and skip boards as soon as possible
-def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FWInfo]:
+def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[Firmware]:
     # sourcery skip: use-getitem-for-re-match-groups
     """
     Retrieves a list of firmware information for the specified ports and boards.
@@ -115,15 +113,15 @@ def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FWInfo]
     Args:
         ports (List[str]): The list of ports to check for firmware.
         boards (List[str]): The list of boards to retrieve firmware information for.
-        clean (bool): A flag indicating whether to perform a clean retrieval.
+        clean (bool): Remove date and hash from the firmware name.
 
     Returns:
         List[FWInfo]: A list of firmware information for the specified ports and boards.
 
     """
-    board_urls: List[FWInfo] = []
+    board_urls: List[Firmware] = []
     if ports is None:
-        ports = get_known_ports()
+        ports = known_ports()
     for port in ports:
         download_page_url = f"{MICROPYTHON_ORG_URL}download/?port={port}"
         urls = get_board_urls(download_page_url)
@@ -152,52 +150,44 @@ def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FWInfo]
                     fname = re.sub(RE_DATE, "-", fname)
                     # remove hash from firmware name
                     fname = re.sub(RE_HASH, ".", fname)
-                fw_info = FWInfo(
-                    filename=fname,
+                fw_info = Firmware(
+                    firmware_file=fname,
                     port=port,
-                    board=board["board"],
-                    preview="preview" in _url,
-                    firmware=_url,
+                    board_id=board["board"],
+                    source=_url,
                     version="",
+                    custom=False,
+                    description="",  # todo : add description from download page
                 )
-                # board["firmware"] = _url
-                # board["preview"] = "preview" in _url  # type: ignore
                 if ver_match := re.search(RE_VERSION_PREVIEW, _url):
-                    fw_info.version = clean_version(ver_match.group(1))
-                    fw_info.build = ver_match.group(2) or "0"
-                fw_info.preview = fw_info.build != "0"
-                # # else:
-                # #     board.$1= ""
-                # if "preview." in fw_info.version:
-                #     fw_info.build = fw_info.version.split("preview.")[-1]
-                # else:
-                #     fw_info.build = "0"
-
-                fw_info.ext = Path(fw_info.firmware).suffix
-                fw_info.variant = fw_info.filename.split("-v")[0] if "-v" in fw_info.filename else ""
+                    fw_info.version = clean_version(ver_match[1])
+                    fw_info.build = int(ver_match[3] or 0)
+                if "-v" in fname:
+                    # get the full board_id[-variant] from the filename
+                    # filename : 'ESP32_GENERIC-v1.25.0.bin'
+                    fw_info.board_id = fname.split("-v")[0]
 
                 board_urls.append(fw_info)
     return board_urls
 
 
-
-def fetch_firmware_files(available_firmwares: List[FWInfo], firmware_folder: Path, force: bool):
+def fetch_firmware_files(available_firmwares: List[Firmware], firmware_folder: Path, force: bool):
     # Just in time import
     import requests
 
     for board in available_firmwares:
-        filename = firmware_folder / board.port / board.filename
+        filename = firmware_folder / board.port / board.firmware_file
         filename.parent.mkdir(exist_ok=True)
         if filename.exists() and not force:
             log.debug(f" {filename} already exists, skip download")
             continue
-        log.info(f"Downloading {board.firmware}")
+        log.info(f"Downloading {board.source}")
         log.info(f"         to {filename}")
         try:
-            r = requests.get(board.firmware, allow_redirects=True)
+            r = requests.get(board.source, allow_redirects=True)
             with open(filename, "wb") as fw:
                 fw.write(r.content)
-            board.filename = str(filename.relative_to(firmware_folder))
+            board.firmware_file = str(filename.relative_to(firmware_folder))
         except requests.RequestException as e:
             log.exception(e)
             continue