mpflash 1.24.7__py3-none-any.whl → 1.25.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mpflash/ask_input.py +7 -7
- mpflash/basicgit.py +26 -59
- mpflash/bootloader/__init__.py +0 -2
- mpflash/bootloader/detect.py +1 -2
- mpflash/bootloader/manual.py +0 -1
- mpflash/bootloader/touch1200.py +2 -2
- mpflash/cli_flash.py +28 -5
- mpflash/cli_group.py +1 -0
- mpflash/cli_list.py +7 -8
- mpflash/cli_main.py +2 -2
- mpflash/common.py +6 -14
- mpflash/config.py +30 -6
- mpflash/connected.py +6 -14
- mpflash/db/boards.py +63 -0
- mpflash/db/downloads.py +87 -0
- mpflash/download/__init__.py +221 -0
- mpflash/download/from_web.py +204 -0
- mpflash/downloaded.py +9 -34
- mpflash/flash/__init__.py +33 -18
- mpflash/flash/esp.py +39 -8
- mpflash/flash/uf2/linux.py +4 -9
- mpflash/flash/uf2/macos.py +1 -1
- mpflash/flash/uf2/windows.py +1 -1
- mpflash/flash/worklist.py +10 -5
- mpflash/list.py +17 -6
- mpflash/logger.py +1 -3
- mpflash/mpboard_id/__init__.py +6 -87
- mpflash/mpboard_id/add_boards.py +3 -8
- mpflash/mpboard_id/board.py +5 -2
- mpflash/mpboard_id/board_id.py +67 -7
- mpflash/mpboard_id/board_info.json +30974 -0
- mpflash/mpboard_id/board_info.zip +0 -0
- mpflash/mpboard_id/known.py +108 -0
- mpflash/mpboard_id/store.py +2 -3
- mpflash/mpremoteboard/__init__.py +85 -17
- mpflash/mpremoteboard/mpy_fw_info.py +23 -22
- mpflash/py.typed +0 -0
- mpflash/vendor/board_database.py +86 -1
- mpflash/vendor/click_aliases.py +64 -0
- mpflash/vendor/dfu.py +2 -8
- mpflash/vendor/pydfu.py +3 -14
- mpflash/versions.py +16 -6
- {mpflash-1.24.7.dist-info → mpflash-1.25.0.dist-info}/METADATA +71 -13
- mpflash-1.25.0.dist-info/RECORD +62 -0
- {mpflash-1.24.7.dist-info → mpflash-1.25.0.dist-info}/WHEEL +1 -1
- mpflash/download.py +0 -364
- mpflash-1.24.7.dist-info/RECORD +0 -56
- {mpflash-1.24.7.dist-info → mpflash-1.25.0.dist-info}/LICENSE +0 -0
- {mpflash-1.24.7.dist-info → mpflash-1.25.0.dist-info}/entry_points.txt +0 -0
mpflash/db/downloads.py
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import sqlite3
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import List
|
6
|
+
|
7
|
+
from mpflash.common import FWInfo
|
8
|
+
from mpflash.config import config
|
9
|
+
from mpflash.logger import log
|
10
|
+
|
11
|
+
|
12
|
+
def upsert_download(conn: sqlite3.Connection, board: FWInfo):
|
13
|
+
"""
|
14
|
+
Adds a row to the downloaded firmware table in the database.
|
15
|
+
- downloads.board_id <-- FWInfo.variant
|
16
|
+
- downloads.source <-- FWInfo.firmware
|
17
|
+
|
18
|
+
Args:
|
19
|
+
conn : The database connection to use.
|
20
|
+
board : The firmware information to add to the database.
|
21
|
+
|
22
|
+
"""
|
23
|
+
with conn:
|
24
|
+
conn.execute(
|
25
|
+
"""
|
26
|
+
INSERT INTO downloads
|
27
|
+
(port, board, filename, source, board_id, version, build, ext, family, custom, description)
|
28
|
+
VALUES
|
29
|
+
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
30
|
+
ON CONFLICT(filename) DO UPDATE SET
|
31
|
+
port=excluded.port,
|
32
|
+
board=excluded.board,
|
33
|
+
source=excluded.source,
|
34
|
+
board_id=excluded.board_id,
|
35
|
+
version=excluded.version,
|
36
|
+
build=excluded.build,
|
37
|
+
ext=excluded.ext,
|
38
|
+
family=excluded.family,
|
39
|
+
custom=excluded.custom,
|
40
|
+
description=excluded.description
|
41
|
+
""",
|
42
|
+
(
|
43
|
+
board.port,
|
44
|
+
board.board,
|
45
|
+
board.filename,
|
46
|
+
board.firmware,
|
47
|
+
board.variant,
|
48
|
+
board.version,
|
49
|
+
board.build,
|
50
|
+
board.ext,
|
51
|
+
board.family,
|
52
|
+
board.custom,
|
53
|
+
board.description,
|
54
|
+
),
|
55
|
+
)
|
56
|
+
conn.commit()
|
57
|
+
|
58
|
+
def downloaded(db_path: Path | None = None) -> List[FWInfo]:
|
59
|
+
"""Load a list of locally downloaded firmwares from the database"""
|
60
|
+
db_path = db_path or config.db_path
|
61
|
+
with sqlite3.connect(db_path) as conn:
|
62
|
+
firmwares: List[FWInfo] = []
|
63
|
+
try:
|
64
|
+
conn.row_factory = sqlite3.Row
|
65
|
+
cursor = conn.cursor()
|
66
|
+
cursor.execute("SELECT * FROM downloads")
|
67
|
+
rows = cursor.fetchall()
|
68
|
+
for row in rows:
|
69
|
+
fw_info = FWInfo.from_dict(
|
70
|
+
{
|
71
|
+
"filename": row["filename"],
|
72
|
+
"version": row["version"],
|
73
|
+
"board": row["board"],
|
74
|
+
"variant": row["board_id"],
|
75
|
+
"port": row["port"],
|
76
|
+
"firmware": row["source"],
|
77
|
+
"build": row["build"],
|
78
|
+
"preview": 1 if int(row["build"]) > 0 else 0,
|
79
|
+
}
|
80
|
+
)
|
81
|
+
firmwares.append(fw_info)
|
82
|
+
except sqlite3.Error as e:
|
83
|
+
log.error(f"Database error: {e}")
|
84
|
+
|
85
|
+
# sort by filename
|
86
|
+
firmwares.sort(key=lambda x: x.filename)
|
87
|
+
return firmwares
|
@@ -0,0 +1,221 @@
|
|
1
|
+
"""
|
2
|
+
Module to download MicroPython firmware for specific boards and versions.
|
3
|
+
Uses the micropython.org website to get the available versions and locations to download firmware files.
|
4
|
+
"""
|
5
|
+
|
6
|
+
import itertools
|
7
|
+
from pathlib import Path
|
8
|
+
import sqlite3
|
9
|
+
from typing import Dict, List, Optional
|
10
|
+
|
11
|
+
# #########################################################################################################
|
12
|
+
# make sure that jsonlines does not mistake the MicroPython ujson for the CPython ujson
|
13
|
+
import jsonlines
|
14
|
+
from loguru import logger as log
|
15
|
+
from rich.progress import track
|
16
|
+
|
17
|
+
from mpflash.common import PORT_FWTYPES, FWInfo
|
18
|
+
from .from_web import get_boards, fetch_firmware_files
|
19
|
+
from mpflash.downloaded import clean_downloaded_firmwares
|
20
|
+
from mpflash.errors import MPFlashError
|
21
|
+
from mpflash.versions import clean_version
|
22
|
+
from mpflash.config import config
|
23
|
+
from mpflash.db.downloads import upsert_download
|
24
|
+
# avoid conflict with the ujson used by MicroPython
|
25
|
+
jsonlines.ujson = None # type: ignore
|
26
|
+
# #########################################################################################################
|
27
|
+
|
28
|
+
|
29
|
+
|
30
|
+
def key_fw_ver_pre_ext_bld(x: FWInfo):
|
31
|
+
"sorting key for the retrieved board urls"
|
32
|
+
return x.variant, x.version, x.preview, x.ext, x.build
|
33
|
+
|
34
|
+
|
35
|
+
def key_fw_var_pre_ext(x: FWInfo):
|
36
|
+
"Grouping key for the retrieved board urls"
|
37
|
+
return x.variant, x.preview, x.ext
|
38
|
+
|
39
|
+
|
40
|
+
def download_firmwares(
|
41
|
+
firmware_folder: Path,
|
42
|
+
ports: List[str],
|
43
|
+
boards: List[str],
|
44
|
+
versions: Optional[List[str]] = None,
|
45
|
+
*,
|
46
|
+
force: bool = False,
|
47
|
+
clean: bool = True,
|
48
|
+
) -> int:
|
49
|
+
"""
|
50
|
+
Downloads firmware files based on the specified firmware folder, ports, boards, versions, force flag, and clean flag.
|
51
|
+
|
52
|
+
Args:
|
53
|
+
firmware_folder : The folder to save the downloaded firmware files.
|
54
|
+
ports : The list of ports to check for firmware.
|
55
|
+
boards : The list of boards to download firmware for.
|
56
|
+
versions : The list of versions to download firmware for.
|
57
|
+
force : A flag indicating whether to force the download even if the firmware file already exists.
|
58
|
+
clean : A flag indicating to clean the date from the firmware filename.
|
59
|
+
"""
|
60
|
+
|
61
|
+
|
62
|
+
skipped = downloaded = 0
|
63
|
+
versions = [] if versions is None else [clean_version(v) for v in versions]
|
64
|
+
# handle renamed boards
|
65
|
+
boards = add_renamed_boards(boards)
|
66
|
+
|
67
|
+
available_firmwares = get_firmware_list(ports, boards, versions, clean)
|
68
|
+
|
69
|
+
for b in available_firmwares:
|
70
|
+
log.debug(b.filename)
|
71
|
+
# relevant
|
72
|
+
|
73
|
+
log.info(f"Found {len(available_firmwares)} relevant unique firmwares")
|
74
|
+
if not available_firmwares:
|
75
|
+
log.error("No relevant firmwares could be found on https://micropython.org/download")
|
76
|
+
log.info(f"{versions=} {ports=} {boards=}")
|
77
|
+
log.info("Please check the website for the latest firmware files or try the preview version.")
|
78
|
+
return 0
|
79
|
+
|
80
|
+
firmware_folder.mkdir(exist_ok=True)
|
81
|
+
|
82
|
+
downloaded = download_firmware_files(available_firmwares, firmware_folder, force )
|
83
|
+
log.success(f"Downloaded {downloaded} firmware images." )
|
84
|
+
return downloaded
|
85
|
+
|
86
|
+
def download_firmware_files(available_firmwares :List[FWInfo],firmware_folder:Path, force:bool ):
|
87
|
+
"""
|
88
|
+
Downloads the firmware files to the specified folder.
|
89
|
+
Args:
|
90
|
+
firmware_folder : The folder to save the downloaded firmware files.
|
91
|
+
force : A flag indicating whether to force the download even if the firmware file already exists.
|
92
|
+
requests : The requests module to use for downloading the firmware files.
|
93
|
+
unique_boards : The list of unique firmware information to download.
|
94
|
+
"""
|
95
|
+
|
96
|
+
# with jsonlines.open(firmware_folder / "firmware.jsonl", "a") as writer:
|
97
|
+
with sqlite3.connect(config.db_path) as conn:
|
98
|
+
# skipped, downloaded = fetch_firmware_files(available_firmwares, firmware_folder, force, requests, writer)
|
99
|
+
downloaded = 0
|
100
|
+
for fw in fetch_firmware_files(available_firmwares, firmware_folder, force):
|
101
|
+
upsert_download(conn, fw)
|
102
|
+
# writer.write(fw)
|
103
|
+
log.debug(f" {fw.filename} downloaded")
|
104
|
+
downloaded += 1
|
105
|
+
if downloaded > 0:
|
106
|
+
clean_downloaded_firmwares(firmware_folder)
|
107
|
+
return downloaded
|
108
|
+
|
109
|
+
|
110
|
+
|
111
|
+
def get_firmware_list(ports: List[str], boards: List[str], versions: List[str], clean: bool = True):
|
112
|
+
"""
|
113
|
+
Retrieves a list of unique firmware files available om micropython.org > downloads
|
114
|
+
based on the specified ports, boards, versions, and clean flag.
|
115
|
+
|
116
|
+
Args:
|
117
|
+
ports : One or more ports to check for firmware.
|
118
|
+
boards : One or more boards to filter the firmware by.
|
119
|
+
versions : One or more versions to filter the firmware by.
|
120
|
+
clean : Remove date-stamp and Git Hash from the firmware name.
|
121
|
+
|
122
|
+
Returns:
|
123
|
+
List[FWInfo]: A list of unique firmware information.
|
124
|
+
|
125
|
+
"""
|
126
|
+
|
127
|
+
log.trace("Checking MicroPython download pages")
|
128
|
+
versions = [clean_version(v, drop_v=False) for v in versions]
|
129
|
+
preview = "preview" in versions
|
130
|
+
|
131
|
+
board_urls = sorted(get_boards(ports, boards, clean), key=key_fw_ver_pre_ext_bld)
|
132
|
+
|
133
|
+
log.debug(f"Total {len(board_urls)} firmwares")
|
134
|
+
|
135
|
+
relevant = [
|
136
|
+
board for board in board_urls if board.version in versions and board.build == "0" and board.board in boards and not board.preview
|
137
|
+
]
|
138
|
+
|
139
|
+
if preview:
|
140
|
+
relevant.extend([board for board in board_urls if board.board in boards and board.preview])
|
141
|
+
log.debug(f"Matching firmwares: {len(relevant)}")
|
142
|
+
# select the unique boards
|
143
|
+
unique_boards: List[FWInfo] = []
|
144
|
+
for _, g in itertools.groupby(relevant, key=key_fw_var_pre_ext):
|
145
|
+
# list is aleady sorted by build so we can just get the last item
|
146
|
+
sub_list = list(g)
|
147
|
+
unique_boards.append(sub_list[-1])
|
148
|
+
log.debug(f"Last preview only: {len(unique_boards)}")
|
149
|
+
return unique_boards
|
150
|
+
|
151
|
+
|
152
|
+
def download(
|
153
|
+
destination: Path,
|
154
|
+
ports: List[str],
|
155
|
+
boards: List[str],
|
156
|
+
versions: List[str],
|
157
|
+
force: bool,
|
158
|
+
clean: bool,
|
159
|
+
) -> int:
|
160
|
+
"""
|
161
|
+
Downloads firmware files based on the specified destination, ports, boards, versions, force flag, and clean flag.
|
162
|
+
|
163
|
+
Args:
|
164
|
+
destination : The destination folder to save the downloaded firmware files.
|
165
|
+
ports : The list of ports to check for firmware.
|
166
|
+
boards : The list of boards to download firmware for.
|
167
|
+
versions : The list of versions to download firmware for.
|
168
|
+
force : A flag indicating whether to force the download even if the firmware file already exists.
|
169
|
+
clean : A flag indicating whether to clean the date from the firmware filename.
|
170
|
+
|
171
|
+
Returns:
|
172
|
+
int: The number of downloaded firmware files.
|
173
|
+
|
174
|
+
Raises:
|
175
|
+
MPFlashError : If no boards are found or specified.
|
176
|
+
|
177
|
+
"""
|
178
|
+
# Just in time import
|
179
|
+
import requests
|
180
|
+
|
181
|
+
if not boards:
|
182
|
+
log.critical("No boards found, please connect a board or specify boards to download firmware for.")
|
183
|
+
raise MPFlashError("No boards found")
|
184
|
+
|
185
|
+
try:
|
186
|
+
destination.mkdir(exist_ok=True, parents=True)
|
187
|
+
except (PermissionError, FileNotFoundError) as e:
|
188
|
+
log.critical(f"Could not create folder {destination}")
|
189
|
+
raise MPFlashError(f"Could not create folder {destination}") from e
|
190
|
+
try:
|
191
|
+
result = download_firmwares(destination, ports, boards, versions, force=force, clean=clean)
|
192
|
+
except requests.exceptions.RequestException as e:
|
193
|
+
log.exception(e)
|
194
|
+
raise MPFlashError("Could not connect to micropython.org") from e
|
195
|
+
|
196
|
+
return result
|
197
|
+
|
198
|
+
|
199
|
+
def add_renamed_boards(boards: List[str]) -> List[str]:
|
200
|
+
"""
|
201
|
+
Adds the renamed boards to the list of boards.
|
202
|
+
|
203
|
+
Args:
|
204
|
+
boards : The list of boards to add the renamed boards to.
|
205
|
+
|
206
|
+
Returns:
|
207
|
+
List[str]: The list of boards with the renamed boards added.
|
208
|
+
"""
|
209
|
+
|
210
|
+
renamed = {
|
211
|
+
"PICO": ["RPI_PICO"],
|
212
|
+
"PICO_W": ["RPI_PICO_W"],
|
213
|
+
"GENERIC": ["ESP32_GENERIC", "ESP8266_GENERIC"], # just add both of them
|
214
|
+
}
|
215
|
+
_boards = boards.copy()
|
216
|
+
for board in boards:
|
217
|
+
if board in renamed and renamed[board] not in boards:
|
218
|
+
_boards.extend(renamed[board])
|
219
|
+
if board != board.upper() and board.upper() not in boards:
|
220
|
+
_boards.append(board.upper())
|
221
|
+
return _boards
|
@@ -0,0 +1,204 @@
|
|
1
|
+
import functools
|
2
|
+
import itertools
|
3
|
+
import re
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import Dict, List, Optional
|
6
|
+
from urllib.parse import urljoin
|
7
|
+
|
8
|
+
|
9
|
+
from loguru import logger as log
|
10
|
+
from rich.progress import track
|
11
|
+
|
12
|
+
from mpflash.common import PORT_FWTYPES, FWInfo
|
13
|
+
from mpflash.downloaded import clean_downloaded_firmwares
|
14
|
+
from mpflash.errors import MPFlashError
|
15
|
+
from mpflash.mpboard_id import get_known_ports
|
16
|
+
from mpflash.versions import clean_version
|
17
|
+
|
18
|
+
|
19
|
+
|
20
|
+
MICROPYTHON_ORG_URL = "https://micropython.org/"
|
21
|
+
|
22
|
+
|
23
|
+
# Regexes to remove dates and hashes in the filename that just get in the way
|
24
|
+
RE_DATE = r"(-\d{8}-)"
|
25
|
+
RE_HASH = r"(.g[0-9a-f]+\.)"
|
26
|
+
# regex to extract the version and the build from the firmware filename
|
27
|
+
# group 1 is the version, group 2 is the build
|
28
|
+
RE_VERSION_PREVIEW = r"v([\d\.]+)-?(?:preview\.)?(\d+)?\."
|
29
|
+
|
30
|
+
|
31
|
+
# 'RPI_PICO_W-v1.23.uf2'
|
32
|
+
# 'RPI_PICO_W-v1.23.0.uf2'
|
33
|
+
# 'RPI_PICO_W-v1.23.0-406.uf2'
|
34
|
+
# 'RPI_PICO_W-v1.23.0-preview.406.uf2'
|
35
|
+
# 'RPI_PICO_W-v1.23.0-preview.4.uf2'
|
36
|
+
# 'RPI_PICO_W-v1.23.0.uf2'
|
37
|
+
# 'https://micropython.org/resources/firmware/RPI_PICO_W-20240531-v1.24.0-preview.10.gc1a6b95bf.uf2'
|
38
|
+
# 'https://micropython.org/resources/firmware/RPI_PICO_W-20240531-v1.24.0-preview.10.uf2'
|
39
|
+
# 'RPI_PICO_W-v1.24.0-preview.10.gc1a6b95bf.uf2'
|
40
|
+
# use functools.lru_cache to avoid needing to download pages multiple times
|
41
|
+
@functools.lru_cache(maxsize=500)
|
42
|
+
def get_page(page_url: str) -> str:
|
43
|
+
"""Get the HTML of a page and return it as a string."""
|
44
|
+
# Just in time import
|
45
|
+
import requests
|
46
|
+
|
47
|
+
response = requests.get(page_url)
|
48
|
+
return response.content.decode()
|
49
|
+
|
50
|
+
|
51
|
+
@functools.lru_cache(maxsize=500)
|
52
|
+
def get_board_urls(page_url: str) -> List[Dict[str, str]]:
|
53
|
+
"""
|
54
|
+
Get the urls to all the board pages listed on this page.
|
55
|
+
Assumes that all links to firmware have "class": "board-card"
|
56
|
+
|
57
|
+
Args:
|
58
|
+
page_url (str): The url of the page to get the board urls from.
|
59
|
+
|
60
|
+
Returns:
|
61
|
+
List[Dict[str, str]]: A list of dictionaries containing the board name and url.
|
62
|
+
|
63
|
+
"""
|
64
|
+
# Just in time import
|
65
|
+
from bs4 import BeautifulSoup
|
66
|
+
|
67
|
+
downloads_html = get_page(page_url)
|
68
|
+
soup = BeautifulSoup(downloads_html, "html.parser")
|
69
|
+
tags = soup.find_all("a", recursive=True, attrs={"class": "board-card"})
|
70
|
+
# assumes that all links are relative to the page url
|
71
|
+
boards = [tag.get("href") for tag in tags] # type: ignore
|
72
|
+
if "?" in page_url:
|
73
|
+
page_url = page_url.split("?")[0]
|
74
|
+
return [{"board": board, "url": page_url + board} for board in boards] # type: ignore
|
75
|
+
|
76
|
+
|
77
|
+
def board_firmware_urls(board_url: str, base_url: str, ext: str) -> List[str]:
|
78
|
+
"""
|
79
|
+
Get the urls to all the firmware files for a board.
|
80
|
+
Args:
|
81
|
+
page_url (str): The url of the page to get the board urls from.
|
82
|
+
??? base_url (str): The base url to join the relative urls to.
|
83
|
+
ext (str): The extension of the firmware files to get. (with or withouth leading .)
|
84
|
+
|
85
|
+
the urls are relative urls to the site root
|
86
|
+
|
87
|
+
"""
|
88
|
+
# Just in time import
|
89
|
+
from bs4 import BeautifulSoup
|
90
|
+
|
91
|
+
html = get_page(board_url)
|
92
|
+
soup = BeautifulSoup(html, "html.parser")
|
93
|
+
# get all the a tags:
|
94
|
+
# 1. that have a url that starts with `/resources/firmware/`
|
95
|
+
# 2. end with a matching extension for this port.
|
96
|
+
tags = soup.find_all(
|
97
|
+
"a",
|
98
|
+
recursive=True,
|
99
|
+
attrs={"href": re.compile(r"^/resources/firmware/.*\." + ext.lstrip(".") + "$")},
|
100
|
+
)
|
101
|
+
if "?" in base_url:
|
102
|
+
base_url = base_url.split("?")[0]
|
103
|
+
links: List = [urljoin(base_url, tag.get("href")) for tag in tags] # type: ignore
|
104
|
+
return links
|
105
|
+
|
106
|
+
|
107
|
+
# boards we are interested in ( this avoids getting a lot of boards we don't care about)
|
108
|
+
# The first run takes ~60 seconds to run for 4 ports , all boards
|
109
|
+
# so it makes sense to cache the results and skip boards as soon as possible
|
110
|
+
def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FWInfo]:
|
111
|
+
# sourcery skip: use-getitem-for-re-match-groups
|
112
|
+
"""
|
113
|
+
Retrieves a list of firmware information for the specified ports and boards.
|
114
|
+
|
115
|
+
Args:
|
116
|
+
ports (List[str]): The list of ports to check for firmware.
|
117
|
+
boards (List[str]): The list of boards to retrieve firmware information for.
|
118
|
+
clean (bool): A flag indicating whether to perform a clean retrieval.
|
119
|
+
|
120
|
+
Returns:
|
121
|
+
List[FWInfo]: A list of firmware information for the specified ports and boards.
|
122
|
+
|
123
|
+
"""
|
124
|
+
board_urls: List[FWInfo] = []
|
125
|
+
if ports is None:
|
126
|
+
ports = get_known_ports()
|
127
|
+
for port in ports:
|
128
|
+
download_page_url = f"{MICROPYTHON_ORG_URL}download/?port={port}"
|
129
|
+
urls = get_board_urls(download_page_url)
|
130
|
+
# filter out boards we don't care about
|
131
|
+
urls = [board for board in urls if board["board"] in boards]
|
132
|
+
# add the port to the board urls
|
133
|
+
for board in urls:
|
134
|
+
board["port"] = port
|
135
|
+
|
136
|
+
for board in track(
|
137
|
+
urls,
|
138
|
+
description=f"Checking {port} download pages",
|
139
|
+
transient=True,
|
140
|
+
refresh_per_second=1,
|
141
|
+
show_speed=False,
|
142
|
+
):
|
143
|
+
# add a board to the list for each firmware found
|
144
|
+
firmware_urls: List[str] = []
|
145
|
+
for ext in PORT_FWTYPES[port]:
|
146
|
+
firmware_urls += board_firmware_urls(board["url"], MICROPYTHON_ORG_URL, ext)
|
147
|
+
for _url in firmware_urls:
|
148
|
+
board["firmware"] = _url
|
149
|
+
fname = Path(board["firmware"]).name
|
150
|
+
if clean:
|
151
|
+
# remove date from firmware name
|
152
|
+
fname = re.sub(RE_DATE, "-", fname)
|
153
|
+
# remove hash from firmware name
|
154
|
+
fname = re.sub(RE_HASH, ".", fname)
|
155
|
+
fw_info = FWInfo(
|
156
|
+
filename=fname,
|
157
|
+
port=port,
|
158
|
+
board=board["board"],
|
159
|
+
preview="preview" in _url,
|
160
|
+
firmware=_url,
|
161
|
+
version="",
|
162
|
+
)
|
163
|
+
# board["firmware"] = _url
|
164
|
+
# board["preview"] = "preview" in _url # type: ignore
|
165
|
+
if ver_match := re.search(RE_VERSION_PREVIEW, _url):
|
166
|
+
fw_info.version = clean_version(ver_match.group(1))
|
167
|
+
fw_info.build = ver_match.group(2) or "0"
|
168
|
+
fw_info.preview = fw_info.build != "0"
|
169
|
+
# # else:
|
170
|
+
# # board.$1= ""
|
171
|
+
# if "preview." in fw_info.version:
|
172
|
+
# fw_info.build = fw_info.version.split("preview.")[-1]
|
173
|
+
# else:
|
174
|
+
# fw_info.build = "0"
|
175
|
+
|
176
|
+
fw_info.ext = Path(fw_info.firmware).suffix
|
177
|
+
fw_info.variant = fw_info.filename.split("-v")[0] if "-v" in fw_info.filename else ""
|
178
|
+
|
179
|
+
board_urls.append(fw_info)
|
180
|
+
return board_urls
|
181
|
+
|
182
|
+
|
183
|
+
|
184
|
+
def fetch_firmware_files(available_firmwares: List[FWInfo], firmware_folder: Path, force: bool):
|
185
|
+
# Just in time import
|
186
|
+
import requests
|
187
|
+
|
188
|
+
for board in available_firmwares:
|
189
|
+
filename = firmware_folder / board.port / board.filename
|
190
|
+
filename.parent.mkdir(exist_ok=True)
|
191
|
+
if filename.exists() and not force:
|
192
|
+
log.debug(f" {filename} already exists, skip download")
|
193
|
+
continue
|
194
|
+
log.info(f"Downloading {board.firmware}")
|
195
|
+
log.info(f" to {filename}")
|
196
|
+
try:
|
197
|
+
r = requests.get(board.firmware, allow_redirects=True)
|
198
|
+
with open(filename, "wb") as fw:
|
199
|
+
fw.write(r.content)
|
200
|
+
board.filename = str(filename.relative_to(firmware_folder))
|
201
|
+
except requests.RequestException as e:
|
202
|
+
log.exception(e)
|
203
|
+
continue
|
204
|
+
yield board
|
mpflash/downloaded.py
CHANGED
@@ -5,27 +5,12 @@ import jsonlines
|
|
5
5
|
from loguru import logger as log
|
6
6
|
|
7
7
|
from mpflash.common import PORT_FWTYPES, FWInfo
|
8
|
+
from mpflash.db.downloads import downloaded
|
8
9
|
from mpflash.versions import clean_version
|
9
10
|
|
10
11
|
from .config import config
|
11
12
|
|
12
|
-
|
13
13
|
# #########################################################################################################
|
14
|
-
def downloaded_firmwares(fw_folder: Path) -> List[FWInfo]:
|
15
|
-
"""Load a list of locally downloaded firmwares from the jsonl file"""
|
16
|
-
firmwares: List[FWInfo] = []
|
17
|
-
log.debug(f"Reading {fw_folder / 'firmware.jsonl' }")
|
18
|
-
try:
|
19
|
-
with jsonlines.open(fw_folder / "firmware.jsonl") as reader:
|
20
|
-
firmwares = [FWInfo.from_dict(item) for item in reader]
|
21
|
-
except FileNotFoundError:
|
22
|
-
log.error(f"No firmware.jsonl found in {fw_folder}")
|
23
|
-
except jsonlines.InvalidLineError as e:
|
24
|
-
log.error(f"Invalid firmware.jsonl found in {fw_folder} : {e}")
|
25
|
-
|
26
|
-
# sort by filename
|
27
|
-
firmwares.sort(key=lambda x: x.filename)
|
28
|
-
return firmwares
|
29
14
|
|
30
15
|
|
31
16
|
def clean_downloaded_firmwares(fw_folder: Path) -> None:
|
@@ -33,15 +18,9 @@ def clean_downloaded_firmwares(fw_folder: Path) -> None:
|
|
33
18
|
Remove duplicate entries from the firmware.jsonl file, keeping the latest one
|
34
19
|
uniqueness is based on the filename
|
35
20
|
"""
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
# keep the latest entry
|
40
|
-
unique_fw = {fw.filename: fw for fw in firmwares}
|
41
|
-
with jsonlines.open(fw_folder / "firmware.jsonl", "w") as writer:
|
42
|
-
for fw in unique_fw.values():
|
43
|
-
writer.write(fw.to_dict())
|
44
|
-
log.info(f"Removed duplicate entries from firmware.jsonl in {fw_folder}")
|
21
|
+
# Duplication should no longer happen,
|
22
|
+
# but is would be a good idea to check the consistence between the DB and the downloads folder sometimes
|
23
|
+
pass
|
45
24
|
|
46
25
|
|
47
26
|
def find_downloaded_firmware(
|
@@ -50,16 +29,16 @@ def find_downloaded_firmware(
|
|
50
29
|
version: str = "", # v1.2.3
|
51
30
|
port: str = "",
|
52
31
|
variants: bool = False,
|
53
|
-
|
32
|
+
db_path: Optional[Path] = None,
|
54
33
|
trie: int = 1,
|
55
34
|
selector: Optional[Dict[str, str]] = None,
|
56
35
|
) -> List[FWInfo]:
|
57
36
|
if selector is None:
|
58
37
|
selector = {}
|
59
|
-
|
38
|
+
|
60
39
|
# Use the information in firmwares.jsonl to find the firmware file
|
61
40
|
log.debug(f"{trie}] Looking for firmware for {board_id} {version} ")
|
62
|
-
fw_list =
|
41
|
+
fw_list = downloaded()
|
63
42
|
if not fw_list:
|
64
43
|
log.error("No firmware files found. Please download the firmware first.")
|
65
44
|
return []
|
@@ -80,7 +59,7 @@ def find_downloaded_firmware(
|
|
80
59
|
board_id = board_id.replace("_", "-")
|
81
60
|
|
82
61
|
fw_list = find_downloaded_firmware(
|
83
|
-
|
62
|
+
db_path=db_path,
|
84
63
|
board_id=board_id,
|
85
64
|
version=version,
|
86
65
|
port=port,
|
@@ -113,11 +92,7 @@ def filter_downloaded_fwlist(
|
|
113
92
|
log.trace(f"Filtering firmware for {version} : {len(fw_list)} found.")
|
114
93
|
# filter by port
|
115
94
|
if port:
|
116
|
-
fw_list = [
|
117
|
-
fw
|
118
|
-
for fw in fw_list
|
119
|
-
if fw.port == port and Path(fw.firmware).suffix in PORT_FWTYPES[port]
|
120
|
-
]
|
95
|
+
fw_list = [fw for fw in fw_list if fw.port == port and Path(fw.firmware).suffix in PORT_FWTYPES[port]]
|
121
96
|
log.trace(f"Filtering firmware for {port} : {len(fw_list)} found.")
|
122
97
|
|
123
98
|
if board_id:
|
mpflash/flash/__init__.py
CHANGED
@@ -2,8 +2,9 @@ from pathlib import Path
|
|
2
2
|
|
3
3
|
from loguru import logger as log
|
4
4
|
|
5
|
-
from mpflash.bootloader import enter_bootloader
|
6
|
-
from mpflash.common import PORT_FWTYPES, BootloaderMethod
|
5
|
+
from mpflash.bootloader.activate import enter_bootloader
|
6
|
+
from mpflash.common import PORT_FWTYPES, BootloaderMethod, UF2_PORTS
|
7
|
+
from mpflash.errors import MPFlashError
|
7
8
|
|
8
9
|
from .esp import flash_esp
|
9
10
|
from .stm32 import flash_stm32
|
@@ -12,44 +13,58 @@ from .worklist import WorkList
|
|
12
13
|
|
13
14
|
# #########################################################################################################
|
14
15
|
|
15
|
-
|
16
|
-
|
17
16
|
def flash_list(
|
18
17
|
todo: WorkList,
|
19
18
|
fw_folder: Path,
|
20
19
|
erase: bool,
|
21
20
|
bootloader: BootloaderMethod,
|
22
|
-
|
21
|
+
**kwargs
|
22
|
+
): # sourcery skip: use-named-expression
|
23
23
|
"""Flash a list of boards with the specified firmware."""
|
24
|
-
UF2_PORTS = [port for port, exts in PORT_FWTYPES.items() if ".uf2" in exts]
|
25
24
|
flashed = []
|
26
25
|
for mcu, fw_info in todo:
|
27
26
|
fw_file = fw_folder / fw_info.filename
|
28
27
|
if not fw_file.exists():
|
29
28
|
log.error(f"File {fw_file} does not exist, skipping {mcu.board} on {mcu.serialport}")
|
30
29
|
continue
|
30
|
+
|
31
31
|
log.info(f"Updating {mcu.board} on {mcu.serialport} to {fw_info.version}")
|
32
|
+
try:
|
33
|
+
updated = flash_mcu(mcu, fw_file=fw_file, erase=erase, bootloader=bootloader, **kwargs)
|
34
|
+
except MPFlashError as e:
|
35
|
+
log.error(f"Failed to flash {mcu.board} on {mcu.serialport}: {e}")
|
36
|
+
continue
|
37
|
+
if updated:
|
38
|
+
flashed.append(updated)
|
39
|
+
else:
|
40
|
+
log.error(f"Failed to flash {mcu.board} on {mcu.serialport}")
|
41
|
+
return flashed
|
42
|
+
|
43
|
+
|
44
|
+
def flash_mcu(
|
45
|
+
mcu,
|
46
|
+
*,
|
47
|
+
fw_file: Path,
|
48
|
+
erase: bool = False,
|
49
|
+
bootloader: BootloaderMethod = BootloaderMethod.AUTO,
|
50
|
+
**kwargs
|
51
|
+
):
|
52
|
+
"""Flash a single MCU with the specified firmware."""
|
32
53
|
updated = None
|
33
54
|
try:
|
34
55
|
if mcu.port in UF2_PORTS and fw_file.suffix == ".uf2":
|
35
56
|
if not enter_bootloader(mcu, bootloader):
|
36
|
-
|
57
|
+
raise MPFlashError(f"Failed to enter bootloader for {mcu.board} on {mcu.serialport}")
|
37
58
|
updated = flash_uf2(mcu, fw_file=fw_file, erase=erase)
|
38
59
|
elif mcu.port in ["stm32"]:
|
39
60
|
if not enter_bootloader(mcu, bootloader):
|
40
|
-
|
61
|
+
raise MPFlashError(f"Failed to enter bootloader for {mcu.board} on {mcu.serialport}")
|
41
62
|
updated = flash_stm32(mcu, fw_file, erase=erase)
|
42
63
|
elif mcu.port in ["esp32", "esp8266"]:
|
43
64
|
# bootloader is handled by esptool for esp32/esp8266
|
44
|
-
updated = flash_esp(mcu, fw_file=fw_file, erase=erase)
|
65
|
+
updated = flash_esp(mcu, fw_file=fw_file, erase=erase, **kwargs)
|
45
66
|
else:
|
46
|
-
|
67
|
+
raise MPFlashError(f"Don't (yet) know how to flash {mcu.port}-{mcu.board} on {mcu.serialport}")
|
47
68
|
except Exception as e:
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
if updated:
|
52
|
-
flashed.append(updated)
|
53
|
-
else:
|
54
|
-
log.error(f"Failed to flash {mcu.board} on {mcu.serialport}")
|
55
|
-
return flashed
|
69
|
+
raise MPFlashError(f"Failed to flash {mcu.board} on {mcu.serialport}") from e
|
70
|
+
return updated
|