mpflash 1.25.0.post1__py3-none-any.whl → 1.25.0rc2__py3-none-any.whl

mpflash/db/gather_boards.py

@@ -0,0 +1,112 @@
+from os import path
+from pathlib import Path
+from mpflash.mpremoteboard import HERE
+from mpflash.vendor.board_database import Database
+from mpflash.logger import log
+
+from typing import List
+import mpflash.basicgit as git
+from mpflash.versions import micropython_versions
+
+HERE = Path(__file__).parent.resolve()
+
+
+## iterator to flatten the board database into a list of tuples
+def iter_boards(db: Database, version: str = ""):
+    """Iterate over the boards in the database and yield tuples of board information."""
+    version = version.strip()
+    for b in db.boards:
+        board = db.boards[b]
+        yield (
+            version,
+            board.name,
+            board.name,
+            board.mcu,
+            "",  # no variant
+            board.port.name if board.port else "",
+            board.path.split("/micropython/", 1)[1],  # TODO - remove hack
+            board.description,
+            "micropython",  # family
+        )
+        if board.variants:
+            for v in board.variants:
+                yield (
+                    version,
+                    f"{board.name}-{v.name}",
+                    board.name,
+                    board.mcu,
+                    v.name,
+                    board.port.name if board.port else "",
+                    board.path.split("/micropython/", 1)[1],  # TODO - remove hack
+                    v.description,
+                    "micropython",  # family
+                )
+
+
+def boardlist_from_repo(
+    versions: List[str],
+    mpy_dir: Path,
+):
+    longlist = []
+    if not mpy_dir.is_dir():
+        log.error(f"Directory {mpy_dir} not found")
+        return longlist
+    for version in versions:
+        build_nr = ""
+        if "preview" in version:
+            ok = git.checkout_tag("master", mpy_dir)
+            if describe := git.get_git_describe(mpy_dir):
+                parts = describe.split("-", 3)
+                if len(parts) >= 3:
+                    build_nr = parts[2]
+        else:
+            ok = git.checkout_tag(version, mpy_dir)
+        if not ok:
+            log.warning(f"Failed to checkout {version} in {mpy_dir}")
+            continue
+
+        log.info(f"{git.get_git_describe(mpy_dir)} - {build_nr}")
+        # un-cached database
+        db = Database(mpy_dir)
+        shortlist = list(iter_boards(db, version=version))
+        log.info(f"boards found {len(db.boards.keys())}")
+        log.info(f"boards-variants found {len(shortlist) - len(db.boards.keys())}")
+        longlist.extend(shortlist)
+    return longlist
+
+
+def create_zip_file(longlist, zip_file: Path):
+    """Create a ZIP file containing the CSV data."""
+    # lazy import
+    import zipfile
+    import pandas as pd
+
+    csv_filename = "micropython_boards.csv"
+
+    columns = ["version", "board_id", "board_name", "mcu", "variant", "port", "path", "description", "family"]
+    df = pd.DataFrame(longlist, columns=columns)
+
+    # Create the ZIP file and add the CSV data directly without creating an intermediate file
+    with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zipf:
+        # Create a temporary in-memory CSV string
+        csv_data = df.to_csv(index=False)
+        # Write the CSV data directly to the zip file
+        zipf.writestr(csv_filename, csv_data)
+
+
+def package_repo(mpy_path: Path):
+    mpy_path = mpy_path or Path("../repos/micropython")
+    mp_versions = micropython_versions(minver="1.18")
+    longlist = boardlist_from_repo(
+        versions=mp_versions,
+        mpy_dir=mpy_path,
+    )
+    log.info(f"Total boards-variants: {len(longlist)}")
+    zip_file = HERE / "micropython_boards.zip"
+    create_zip_file(longlist, zip_file=zip_file)
+
+    assert zip_file.is_file(), f"Failed to create {zip_file}"
+
+
+if __name__ == "__main__":
+    package_repo(Path("D:\\mypython\\mpflash\\repos\\micropython"))

mpflash/db/loader.py

@@ -0,0 +1,122 @@
+import csv
+import io
+import json
+import re
+import zipfile
+from pathlib import Path
+from turtle import up
+
+from loguru import logger as log
+
+from mpflash.errors import MPFlashError
+
+from .core import Session
+from .meta import get_metadata, set_metadata_value
+from .models import Board, Firmware
+
+HERE = Path(__file__).parent.resolve()
+
+
+def load_data_from_zip(zip_file: Path) -> int:
+    log.debug("Loading data from zip file")
+    csv_filename = "micropython_boards.csv"  # name of the .csv inside the .zip
+    # Check if the zip file exists
+    if not zip_file.exists() or not zip_file.is_file():
+        log.error(f"Zip file {zip_file} not found.")
+        return 0
+    count = 0 
+    # Load data directly from the zip file
+    with zipfile.ZipFile(zip_file, "r") as zipf:
+        # Read the CSV file from the zip
+        with zipf.open(csv_filename) as csv_file:
+            log.info("Reading CSV data...")
+            reader = csv.DictReader(io.TextIOWrapper(csv_file, "utf-8"))
+
+            # save to database
+            with Session() as session:
+                for row in reader:
+                    # Create a board instance from the row data
+                    board = Board(**row)
+                    # Use merge to update existing or insert new record
+                    # based on primary key (board_id and version)
+                    session.merge(board)
+                    count += 1
+                session.commit()
+    log.info(f"Loaded {count} boards from {zip_file}")
+    return count
+
+
+def load_jsonl_to_db(jsonl_path: Path):
+    """
+    Load a JSONL file into a SQLite database
+
+    Args:
+        jsonl_path (Path): Path to the JSONL file
+        conn (sqlite3.Connection): SQLite database connection
+        table_name (str): Name of the table to insert data into
+
+    Returns:
+        int: Number of records imported
+    """
+    log.debug("Loading JSONL file into database")
+    # Ensure file exists
+    if not jsonl_path.exists():
+        raise FileNotFoundError(f"JSONL file not found: {jsonl_path}")
+    num_records = 0
+    with jsonl_path.open("r", encoding="utf-8") as file:
+        with Session() as session:
+            for line in file:
+                record = json.loads(line.strip())
+                # Clean up the record
+
+                if "variant" in record:
+                    record["board_id"] = record.pop("variant")  # Rename 'variant' to 'board_id'
+                if "firmware" in record:
+                    record["source"] = record.pop("firmware")  # Rename 'firmware' to 'source'
+                if "preview" in record:
+                    record["version"] = f"{record['version']}-preview" if record["preview"] else record["version"]
+                    record.pop("preview", None)  # Remove 'preview' column
+                if not "custom" in record:
+                    record["custom"] = False
+                firmware_file = str(Path(record["filename"]).as_posix()) if record["filename"] else ""
+
+                # Check if Firmware with this firmware_file exists
+                existing_fw = session.query(Firmware).filter_by(firmware_file=firmware_file).first()
+                if existing_fw:
+                    # Update fields
+                    existing_fw.board_id = record["board_id"]
+                    existing_fw.version = record["version"]
+                    existing_fw.source = record["source"]
+                    existing_fw.build = record["build"]
+                    existing_fw.custom = record["custom"]
+                    existing_fw.port = record["port"]
+                else:
+                    # Add new Firmware
+                    fw = Firmware(
+                        board_id=record["board_id"],
+                        version=record["version"],
+                        firmware_file=firmware_file,
+                        source=record["source"],
+                        build=record["build"],
+                        custom=record["custom"],
+                        port=record["port"],
+                    )
+                    session.merge(fw)
+                num_records += 1
+            # commit once after all records are processed
+            session.commit()
+    return num_records
+
+
+def update_boards():
+    try:
+        meta = get_metadata()
+        log.debug(f"Metadata: {meta}")
+        if meta.get("boards_version", "") < "v1.25.0":
+            log.info("Update boards from CSV to SQLite database.")
+            # Load data from the zip file into the database
+            load_data_from_zip(HERE / "micropython_boards.zip")
+            set_metadata_value("boards_version", "v1.25.0")
+            meta = get_metadata()
+    except Exception as e:
+        raise MPFlashError(f"Error updating boards table: {e}") from e

mpflash/db/meta.py

@@ -0,0 +1,78 @@
+
+from typing import Optional
+
+from loguru import logger as log
+
+from .core import DatabaseError, OperationalError, Session
+from .models import Metadata
+
+
+def get_metadata() -> dict:
+    """
+    Get all metadata from the database.
+
+    Returns:
+        dict: Dictionary of metadata name-value pairs.
+    """
+    try:
+        with Session() as session:
+            metadata = session.query(Metadata).all()
+            return {m.name: m.value for m in metadata}
+    except (DatabaseError, OperationalError)  as e:
+        log.error(f"Error retrieving metadata: {e}")
+        return {}
+    
+def set_metadata(metadata: dict):
+    """
+    Set metadata in the database.
+    Args:
+        metadata (dict): Dictionary of metadata name-value pairs.
+    Returns:
+        None
+    """
+    with Session() as session:
+        for name, value in metadata.items():
+            existing_metadata = session.query(Metadata).filter(Metadata.name == name).first()
+            if existing_metadata:
+                existing_metadata.value = value
+            else:
+                new_metadata = Metadata(name=name, value=value)
+                session.add(new_metadata)
+        session.commit()
+
+
+def get_metadata_value(name: str) -> Optional[str]:
+    """
+    Get metadata value by name.
+
+    Args:
+        session (Session): SQLAlchemy session.
+        name (str): Name of the metadata.
+
+    Returns:
+        Optional[str]: Metadata value or None if not found.
+    """
+    with Session() as session:
+        metadata = session.query(Metadata).filter(Metadata.name == name).first()
+    return metadata.value if metadata else None
+
+def set_metadata_value(name: str, value: str):
+    """
+    Set metadata value by name.
+
+    Args:
+        session (Session): SQLAlchemy session.
+        name (str): Name of the metadata.
+        value (str): Value to set.
+
+    Returns:
+        None
+    """
+    with Session() as session:
+        metadata = session.query(Metadata).filter(Metadata.name == name).first()
+        if metadata:
+            metadata.value = value
+        else:
+            new_metadata = Metadata(name=name, value=value)
+            session.add(new_metadata)
+        session.commit()

mpflash/db/models.py

@@ -0,0 +1,93 @@
+from pathlib import Path
+
+import sqlalchemy as sa
+from sqlalchemy import Index, String
+from sqlalchemy.orm import DeclarativeBase, Mapped, composite, mapped_column, relationship
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class Metadata(Base):
+    """
+    Configuration information.
+    """
+
+    __tablename__ = "metadata"
+    name: Mapped[str] = mapped_column(primary_key=True, unique=True)
+    value: Mapped[str] = mapped_column()
+
+    def __repr__(self) -> str:
+        return f"Config(boards_version={self.name!r}, schema_version={self.value!r})"
+
+
+class Board(Base):
+    """
+    All known Boards model for storing board information.
+    """
+
+    __tablename__ = "boards"
+    __table_args__ = (sa.UniqueConstraint("board_id", "version"),)
+
+    board_id: Mapped[str] = mapped_column(String(40), primary_key=True, unique=False)
+    version: Mapped[str] = mapped_column(String(12), primary_key=True, unique=False)
+    board_name: Mapped[str] = mapped_column()
+    mcu: Mapped[str] = mapped_column()
+    variant: Mapped[str] = mapped_column(default="")
+    port: Mapped[str] = mapped_column(String(30))
+    path: Mapped[str] = mapped_column(comment="Path in micropyton repo as_posix()")
+    description: Mapped[str] = mapped_column()
+    family: Mapped[str] = mapped_column(default="micropython")
+    custom: Mapped[bool] = mapped_column(default=False, comment="True if this is a custom board")
+    firmwares = relationship(
+        "Firmware",
+        back_populates="board",
+        lazy="joined",
+    )
+
+    board_key = composite(lambda board_id, version: f"{board_id}_{version}", board_id, version)
+
+    def __repr__(self) -> str:
+        return f"Board(board_id={self.board_id!r}, version={self.version!r}, board_name={self.board_name!r})"
+
+
+class Firmware(Base):
+    """
+    Firmware model for storing firmware information.
+    """
+
+    __tablename__ = "firmwares"
+    __table_args__ = (
+        sa.ForeignKeyConstraint(["board_id", "version"], ["boards.board_id", "boards.version"]),
+        {"sqlite_autoincrement": False},
+    )
+
+    board_id: Mapped[str] = mapped_column(String(40), primary_key=True)
+    version: Mapped[str] = mapped_column(String(12), primary_key=True)
+    firmware_file: Mapped[str] = mapped_column(String, primary_key=True, index=True, comment="Path to the firmware file")
+    # Relationship to Board
+    board: Mapped["Board"] = relationship(
+        "Board",
+        back_populates="firmwares",
+        lazy="joined",
+        primaryjoin="and_(Firmware.board_id==Board.board_id, Firmware.version==Board.version)",
+    )
+    port: Mapped[str] = mapped_column(String(20), default="")  # duplicate of board.port
+    description: Mapped[str] = mapped_column(default="")
+    source: Mapped[str] = mapped_column()
+    build: Mapped[int] = mapped_column(default=0, comment="Build number")
+    custom: Mapped[bool] = mapped_column(default=False, comment="True if this is a custom firmware")
+
+    @property
+    def preview(self) -> bool:
+        "Check if the firmware is a preview version."
+        return "preview" in self.firmware_file
+
+    @property
+    def ext(self) -> str:
+        "Get the file extension of the firmware file."
+        return Path(self.firmware_file).suffix
+
+    def __repr__(self) -> str:
+        return f"Firmware(board_id={self.board_id!r}, version={self.version!r}, firmware_file={self.firmware_file!r})"

mpflash/db/tools.py

@@ -0,0 +1,27 @@
+import sqlite3
+from pathlib import Path
+from typing import List
+
+from mpflash.config import config
+from mpflash.errors import MPFlashError
+from mpflash.logger import log
+
+
+def backup_db(source_db: Path, backup_path: Path):
+    """
+    Backup the SQLite database to a specified path.
+
+    Args:
+        conn (sqlite3.Connection): SQLite connection object
+        backup_path (str or Path): Path to save the backup file
+    """
+
+    # Ensure the backup directory exists
+    backup_path.parent.mkdir(parents=True, exist_ok=True)
+    with sqlite3.connect(source_db) as conn:
+        # Perform the backup
+        with open(backup_path, "wb") as f:
+            for line in conn.iterdump():
+                f.write(f"{line}\n".encode("utf-8"))
+
+    log.info(f"Backup created at {backup_path}")

mpflash/download/__init__.py

@@ -5,7 +5,6 @@ Uses the micropython.org website to get the available versions and locations to
 
 import itertools
 from pathlib import Path
-import sqlite3
 from typing import Dict, List, Optional
 
 # #########################################################################################################
@@ -14,27 +13,27 @@ import jsonlines
 from loguru import logger as log
 from rich.progress import track
 
-from mpflash.common import PORT_FWTYPES, FWInfo
-from .from_web import get_boards, fetch_firmware_files
+from mpflash.common import PORT_FWTYPES
+from mpflash.config import config
+from mpflash.db.core import Session
+from mpflash.db.models import Firmware
 from mpflash.downloaded import clean_downloaded_firmwares
 from mpflash.errors import MPFlashError
+from mpflash.mpboard_id.alternate import add_renamed_boards
 from mpflash.versions import clean_version
-from mpflash.config import config
-from mpflash.db.downloads import upsert_download
+
+from .from_web import fetch_firmware_files, get_boards
+from .fwinfo import FWInfo
+
 # avoid conflict with the ujson used by MicroPython
 jsonlines.ujson = None  # type: ignore
 # #########################################################################################################
 
 
 
-def key_fw_ver_pre_ext_bld(x: FWInfo):
-    "sorting key for the retrieved board urls"
-    return x.variant, x.version, x.preview, x.ext, x.build
-
-
-def key_fw_var_pre_ext(x: FWInfo):
+def key_fw_boardid_preview_ext(fw: Firmware):
     "Grouping key for the retrieved board urls"
-    return x.variant, x.preview, x.ext
+    return fw.board_id, fw.preview, fw.ext
 
 
 def download_firmwares(
@@ -59,18 +58,18 @@ def download_firmwares(
     """
 
 
-    skipped = downloaded = 0
+    downloaded = 0
     versions = [] if versions is None else [clean_version(v) for v in versions]
-    # handle renamed boards
+    # handle downloading firmware for renamed boards
     boards = add_renamed_boards(boards)
 
     available_firmwares = get_firmware_list(ports, boards, versions, clean)
 
     for b in available_firmwares:
-        log.debug(b.filename)
+        log.debug(b.firmware_file)
     # relevant
 
-    log.info(f"Found {len(available_firmwares)} relevant unique firmwares")
+    log.info(f"Found {len(available_firmwares)} potentially relevant firmwares")
     if not available_firmwares:
         log.error("No relevant firmwares could be found on https://micropython.org/download")
         log.info(f"{versions=} {ports=} {boards=}")
@@ -83,7 +82,7 @@ def download_firmwares(
     log.success(f"Downloaded {downloaded} firmware images." )
     return downloaded 
 
-def download_firmware_files(available_firmwares :List[FWInfo],firmware_folder:Path, force:bool ):
+def download_firmware_files(available_firmwares :List[Firmware],firmware_folder:Path, force:bool ):
     """
     Downloads the firmware files to the specified folder.
     Args:
@@ -94,23 +93,23 @@ def download_firmware_files(available_firmwares :List[FWInfo],firmware_folder:Pa
     """
 
     # with jsonlines.open(firmware_folder / "firmware.jsonl", "a") as writer:
-    with sqlite3.connect(config.db_path) as conn:
+    with Session() as session:
         # skipped, downloaded = fetch_firmware_files(available_firmwares, firmware_folder, force, requests, writer)
         downloaded = 0
         for fw in fetch_firmware_files(available_firmwares, firmware_folder, force):
-            upsert_download(conn, fw)
-            # writer.write(fw)
-            log.debug(f" {fw.filename} downloaded")
+            session.merge(fw)
+            log.debug(f" {fw.firmware_file} downloaded")
             downloaded += 1
+        session.commit()
     if downloaded > 0:
-        clean_downloaded_firmwares(firmware_folder)
+        clean_downloaded_firmwares()
     return downloaded
 
 
 
 def get_firmware_list(ports: List[str], boards: List[str], versions: List[str], clean: bool = True):
     """
-    Retrieves a list of unique firmware files available om micropython.org > downloads
+    Retrieves a list of unique firmware files potentially  available on micropython.org > downloads
     based on the specified ports, boards, versions, and clean flag.
 
     Args:
@@ -126,36 +125,43 @@ def get_firmware_list(ports: List[str], boards: List[str], versions: List[str],
 
     log.trace("Checking MicroPython download pages")
     versions = [clean_version(v, drop_v=False) for v in versions]
-    preview = "preview" in versions
+    preview = any("preview" in v for v in versions)
 
-    board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
+    # board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
+    board_urls = get_boards(ports, boards, clean)
 
     log.debug(f"Total {len(board_urls)} firmwares")
+    if versions:
+        # filter out the boards that are not in the versions list
+        relevant = [
+            board for board in board_urls if ( 
+                board.version in versions 
+                # or (preview and board.preview )
+                # and board.board_id in boards 
+                # and board.build == "0" 
+                # and not board.preview
+            )
+        ]
+    else:
+        relevant = board_urls
 
-    relevant = [
-        board for board in board_urls if board.version in versions and board.build == "0" and board.board in boards and not board.preview
-    ]
-
-    if preview:
-        relevant.extend([board for board in board_urls if board.board in boards and board.preview])
     log.debug(f"Matching firmwares: {len(relevant)}")
     # select the unique boards
-    unique_boards: List[FWInfo] = []
-    for _, g in itertools.groupby(relevant, key=key_fw_var_pre_ext):
-        # list is aleady sorted by build so we can just get the last item
+    unique_boards: List[Firmware] = []
+    for _, g in itertools.groupby(relevant, key=key_fw_boardid_preview_ext):
+        # list is aleady sorted by build (desc)  so we can just get the first item
         sub_list = list(g)
-        unique_boards.append(sub_list[-1])
-    log.debug(f"Last preview only: {len(unique_boards)}")
+        unique_boards.append(sub_list[0])
+    log.debug(f"Including preview: {len(unique_boards)}")
     return unique_boards
 
 
 def download(
-    destination: Path,
     ports: List[str],
     boards: List[str],
     versions: List[str],
-    force: bool,
-    clean: bool,
+    force: bool = False,
+    clean: bool = True,
 ) -> int:
     """
     Downloads firmware files based on the specified destination, ports, boards, versions, force flag, and clean flag.
@@ -177,7 +183,7 @@ def download(
     """
     # Just in time import
     import requests
-
+    destination = config.firmware_folder
     if not boards:
         log.critical("No boards found, please connect a board or specify boards to download firmware for.")
         raise MPFlashError("No boards found")
@@ -195,27 +201,3 @@ def download(
 
     return result
 
-
-def add_renamed_boards(boards: List[str]) -> List[str]:
-    """
-    Adds the renamed boards to the list of boards.
-
-    Args:
-        boards : The list of boards to add the renamed boards to.
-
-    Returns:
-        List[str]: The list of boards with the renamed boards added.
-    """
-
-    renamed = {
-        "PICO": ["RPI_PICO"],
-        "PICO_W": ["RPI_PICO_W"],
-        "GENERIC": ["ESP32_GENERIC", "ESP8266_GENERIC"],  # just add both of them
-    }
-    _boards = boards.copy()
-    for board in boards:
-        if board in renamed and renamed[board] not in boards:
-            _boards.extend(renamed[board])
-        if board != board.upper() and board.upper() not in boards:
-            _boards.append(board.upper())
-    return _boards