micropython-stubber 1.23.1.post1__py3-none-any.whl → 1.23.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. {micropython_stubber-1.23.1.post1.dist-info → micropython_stubber-1.23.3.dist-info}/LICENSE +30 -30
  2. {micropython_stubber-1.23.1.post1.dist-info → micropython_stubber-1.23.3.dist-info}/METADATA +5 -5
  3. micropython_stubber-1.23.3.dist-info/RECORD +158 -0
  4. mpflash/README.md +220 -220
  5. mpflash/libusb_flash.ipynb +203 -203
  6. mpflash/mpflash/add_firmware.py +98 -98
  7. mpflash/mpflash/ask_input.py +236 -236
  8. mpflash/mpflash/basicgit.py +304 -284
  9. mpflash/mpflash/bootloader/__init__.py +2 -2
  10. mpflash/mpflash/bootloader/activate.py +60 -60
  11. mpflash/mpflash/bootloader/detect.py +82 -82
  12. mpflash/mpflash/bootloader/manual.py +101 -101
  13. mpflash/mpflash/bootloader/micropython.py +12 -12
  14. mpflash/mpflash/bootloader/touch1200.py +36 -36
  15. mpflash/mpflash/cli_download.py +129 -129
  16. mpflash/mpflash/cli_flash.py +224 -216
  17. mpflash/mpflash/cli_group.py +111 -111
  18. mpflash/mpflash/cli_list.py +87 -87
  19. mpflash/mpflash/cli_main.py +39 -39
  20. mpflash/mpflash/common.py +207 -166
  21. mpflash/mpflash/config.py +44 -44
  22. mpflash/mpflash/connected.py +96 -77
  23. mpflash/mpflash/download.py +364 -364
  24. mpflash/mpflash/downloaded.py +130 -130
  25. mpflash/mpflash/errors.py +9 -9
  26. mpflash/mpflash/flash/__init__.py +55 -55
  27. mpflash/mpflash/flash/esp.py +59 -59
  28. mpflash/mpflash/flash/stm32.py +19 -19
  29. mpflash/mpflash/flash/stm32_dfu.py +104 -104
  30. mpflash/mpflash/flash/uf2/__init__.py +88 -88
  31. mpflash/mpflash/flash/uf2/boardid.py +15 -15
  32. mpflash/mpflash/flash/uf2/linux.py +136 -130
  33. mpflash/mpflash/flash/uf2/macos.py +42 -42
  34. mpflash/mpflash/flash/uf2/uf2disk.py +12 -12
  35. mpflash/mpflash/flash/uf2/windows.py +43 -43
  36. mpflash/mpflash/flash/worklist.py +170 -170
  37. mpflash/mpflash/list.py +106 -106
  38. mpflash/mpflash/logger.py +41 -41
  39. mpflash/mpflash/mpboard_id/__init__.py +93 -93
  40. mpflash/mpflash/mpboard_id/add_boards.py +251 -251
  41. mpflash/mpflash/mpboard_id/board.py +37 -37
  42. mpflash/mpflash/mpboard_id/board_id.py +86 -86
  43. mpflash/mpflash/mpboard_id/store.py +48 -43
  44. mpflash/mpflash/mpremoteboard/__init__.py +266 -266
  45. mpflash/mpflash/mpremoteboard/mpy_fw_info.py +152 -141
  46. mpflash/mpflash/mpremoteboard/runner.py +140 -140
  47. mpflash/mpflash/vendor/click_aliases.py +91 -91
  48. mpflash/mpflash/vendor/dfu.py +165 -165
  49. mpflash/mpflash/vendor/pydfu.py +605 -605
  50. mpflash/mpflash/vendor/readme.md +2 -2
  51. mpflash/mpflash/versions.py +135 -135
  52. mpflash/poetry.lock +1687 -1599
  53. mpflash/pyproject.toml +65 -65
  54. mpflash/stm32_udev_rules.md +62 -62
  55. stubber/__init__.py +3 -3
  56. stubber/board/board_info.csv +193 -193
  57. stubber/board/boot.py +34 -34
  58. stubber/board/createstubs.py +986 -986
  59. stubber/board/createstubs_db.py +825 -825
  60. stubber/board/createstubs_db_min.py +331 -331
  61. stubber/board/createstubs_lvgl.py +741 -741
  62. stubber/board/createstubs_lvgl_min.py +741 -741
  63. stubber/board/createstubs_mem.py +766 -766
  64. stubber/board/createstubs_mem_min.py +306 -306
  65. stubber/board/createstubs_min.py +294 -294
  66. stubber/board/fw_info.py +141 -141
  67. stubber/board/info.py +183 -183
  68. stubber/board/main.py +19 -19
  69. stubber/board/modulelist.txt +247 -247
  70. stubber/board/pyrightconfig.json +34 -34
  71. stubber/bulk/mcu_stubber.py +437 -437
  72. stubber/codemod/_partials/__init__.py +48 -48
  73. stubber/codemod/_partials/db_main.py +147 -147
  74. stubber/codemod/_partials/lvgl_main.py +77 -77
  75. stubber/codemod/_partials/modules_reader.py +80 -80
  76. stubber/codemod/add_comment.py +53 -53
  77. stubber/codemod/add_method.py +65 -65
  78. stubber/codemod/board.py +317 -317
  79. stubber/codemod/enrich.py +145 -145
  80. stubber/codemod/merge_docstub.py +284 -284
  81. stubber/codemod/modify_list.py +54 -54
  82. stubber/codemod/utils.py +56 -56
  83. stubber/commands/build_cmd.py +94 -94
  84. stubber/commands/cli.py +49 -49
  85. stubber/commands/clone_cmd.py +78 -78
  86. stubber/commands/config_cmd.py +29 -29
  87. stubber/commands/enrich_folder_cmd.py +71 -71
  88. stubber/commands/get_core_cmd.py +71 -71
  89. stubber/commands/get_docstubs_cmd.py +92 -92
  90. stubber/commands/get_frozen_cmd.py +117 -117
  91. stubber/commands/get_mcu_cmd.py +102 -102
  92. stubber/commands/merge_cmd.py +66 -66
  93. stubber/commands/publish_cmd.py +118 -118
  94. stubber/commands/stub_cmd.py +31 -31
  95. stubber/commands/switch_cmd.py +62 -62
  96. stubber/commands/variants_cmd.py +48 -48
  97. stubber/cst_transformer.py +178 -178
  98. stubber/data/board_info.csv +193 -193
  99. stubber/data/board_info.json +1729 -1729
  100. stubber/data/micropython_tags.csv +15 -15
  101. stubber/data/requirements-core-micropython.txt +38 -38
  102. stubber/data/requirements-core-pycopy.txt +39 -39
  103. stubber/downloader.py +37 -37
  104. stubber/freeze/common.py +72 -72
  105. stubber/freeze/freeze_folder.py +69 -69
  106. stubber/freeze/freeze_manifest_2.py +126 -126
  107. stubber/freeze/get_frozen.py +131 -131
  108. stubber/get_cpython.py +112 -112
  109. stubber/get_lobo.py +59 -59
  110. stubber/minify.py +423 -423
  111. stubber/publish/bump.py +86 -86
  112. stubber/publish/candidates.py +275 -275
  113. stubber/publish/database.py +18 -18
  114. stubber/publish/defaults.py +40 -40
  115. stubber/publish/enums.py +24 -24
  116. stubber/publish/helpers.py +29 -29
  117. stubber/publish/merge_docstubs.py +132 -132
  118. stubber/publish/missing_class_methods.py +51 -51
  119. stubber/publish/package.py +150 -150
  120. stubber/publish/pathnames.py +51 -51
  121. stubber/publish/publish.py +120 -120
  122. stubber/publish/pypi.py +42 -42
  123. stubber/publish/stubpackage.py +1051 -1051
  124. stubber/rst/__init__.py +9 -9
  125. stubber/rst/classsort.py +78 -78
  126. stubber/rst/lookup.py +531 -531
  127. stubber/rst/output_dict.py +401 -401
  128. stubber/rst/reader.py +814 -814
  129. stubber/rst/report_return.py +77 -77
  130. stubber/rst/rst_utils.py +541 -541
  131. stubber/stubber.py +38 -38
  132. stubber/stubs_from_docs.py +90 -90
  133. stubber/tools/manifestfile.py +654 -654
  134. stubber/tools/readme.md +6 -6
  135. stubber/update_fallback.py +117 -117
  136. stubber/update_module_list.py +123 -123
  137. stubber/utils/__init__.py +6 -6
  138. stubber/utils/config.py +137 -137
  139. stubber/utils/makeversionhdr.py +54 -54
  140. stubber/utils/manifest.py +90 -90
  141. stubber/utils/post.py +80 -80
  142. stubber/utils/repos.py +156 -156
  143. stubber/utils/stubmaker.py +139 -139
  144. stubber/utils/typed_config_toml.py +80 -80
  145. stubber/variants.py +106 -106
  146. micropython_stubber-1.23.1.post1.dist-info/RECORD +0 -159
  147. mpflash/basicgit.py +0 -288
  148. {micropython_stubber-1.23.1.post1.dist-info → micropython_stubber-1.23.3.dist-info}/WHEEL +0 -0
  149. {micropython_stubber-1.23.1.post1.dist-info → micropython_stubber-1.23.3.dist-info}/entry_points.txt +0 -0
@@ -1,437 +1,437 @@
1
- """
2
- This script creates stubs on and for a connected micropython MCU board.
3
- """
4
-
5
- import json
6
- import shutil
7
- import sys
8
- import time
9
- from enum import Enum
10
- from pathlib import Path
11
- from tempfile import mkdtemp
12
- from typing import List, Optional, Tuple
13
-
14
- from rich.console import Console
15
- from rich.table import Table
16
- from tenacity import retry, stop_after_attempt, wait_fixed
17
-
18
- from mpflash.connected import list_mcus
19
- from mpflash.list import show_mcus
20
- from mpflash.logger import log
21
- from mpflash.mpremoteboard import ERROR, OK, MPRemoteBoard
22
- from stubber import utils
23
- from stubber.publish.merge_docstubs import merge_all_docstubs
24
- from stubber.publish.pathnames import board_folder_name
25
- from stubber.publish.publish import build_multiple
26
- from stubber.utils.config import CONFIG
27
-
28
- HERE = Path(__file__).parent
29
- ###############################################################################################
30
- # TODO: promote to cmdline params
31
- LOCAL_FILES = False
32
- reset_before = True
33
- TESTING = False
34
- ###############################################################################################
35
-
36
-
37
- ###############################################################################################
38
-
39
-
40
- class Variant(str, Enum):
41
- """Variants to generate stubs on a MCU"""
42
-
43
- full = "full"
44
- mem = "mem"
45
- db = "db"
46
-
47
-
48
- class Form(str, Enum):
49
- """Optimization forms of scripts"""
50
-
51
- py = "py"
52
- min = "min"
53
- mpy = "mpy"
54
-
55
-
56
- def copy_createstubs_to_board(board: MPRemoteBoard, variant: Variant, form: Form) -> bool:
57
- # sourcery skip: assign-if-exp, boolean-if-exp-identity, remove-unnecessary-cast
58
- """Copy createstubs to the board"""
59
- # copy createstubs.py to the destination folder
60
- origin = "./src/stubber/board"
61
-
62
- _py = [
63
- "rm :lib/createstubs.mpy",
64
- "rm :lib/createstubs_mem.mpy",
65
- "rm :lib/createstubs_db.mpy",
66
- "rm :lib/createstubs.py",
67
- "rm :lib/createstubs_mem.py",
68
- "rm :lib/createstubs_db.py",
69
- f"cp {origin}/createstubs.py :lib/createstubs.py",
70
- f"cp {origin}/createstubs_mem.py :lib/createstubs_mem.py",
71
- f"cp {origin}/createstubs_db.py :lib/createstubs_db.py",
72
- ]
73
-
74
- # copy createstubs*_min.py to the destination folder
75
- _min = [
76
- f"cp {origin}/createstubs_min.py :lib/createstubs.py",
77
- f"cp {origin}/createstubs_mem_min.py :lib/createstubs_mem.py",
78
- f"cp {origin}/createstubs_db_min.py :lib/createstubs_db.py",
79
- ]
80
- # copy createstubs*_mpy.mpy to the destination folder
81
- _mpy = [
82
- "rm :lib/createstubs.py",
83
- "rm :lib/createstubs_mem.py",
84
- "rm :lib/createstubs_db.py",
85
- f"cp {origin}/createstubs_mpy.mpy :lib/createstubs.mpy",
86
- f"cp {origin}/createstubs_mem_mpy.mpy :lib/createstubs_mem.mpy",
87
- f"cp {origin}/createstubs_db_mpy.mpy :lib/createstubs_db.mpy",
88
- ]
89
-
90
- _lib = [
91
- [
92
- "exec",
93
- "import os;os.mkdir('lib') if not ('lib' in os.listdir()) else print('folder lib already exists')",
94
- ]
95
- ]
96
-
97
- _get_ready = [
98
- "rm :modulelist.done",
99
- "rm :no_auto_stubber.txt",
100
- f"cp {origin}/modulelist.txt :lib/modulelist.txt",
101
- ]
102
- if form == Form.py:
103
- do = _lib + _py + _get_ready
104
- elif form == Form.min:
105
- do = _lib + _min + _get_ready
106
- else:
107
- do = _lib + _mpy + _get_ready
108
-
109
- # assume all ok, unless one is not ok
110
- for cmd in do:
111
- if isinstance(cmd, str) and cmd.startswith("rm "):
112
- log_errors = False
113
- else:
114
- log_errors = True
115
- rc, _ = board.run_command(cmd, log_errors=log_errors)
116
- if rc != OK and "rm" not in cmd:
117
- log.error(f"Error during copy createstubs running command: {cmd}")
118
- return False
119
- return True
120
-
121
-
122
- @retry(stop=stop_after_attempt(4), wait=wait_fixed(2))
123
- def hard_reset(board: MPRemoteBoard) -> bool:
124
- """Reset the board"""
125
- # do not run "exec", "import machine;machine.reset()" as it will hang an esp32
126
- rc, _ = board.run_command(["reset"], timeout=5)
127
- board.connected = False
128
- return rc == OK
129
-
130
-
131
- @retry(stop=stop_after_attempt(10), wait=wait_fixed(15))
132
- def run_createstubs(dest: Path, mcu: MPRemoteBoard, variant: Variant = Variant.db):
133
- """
134
- Run a createstubs[variant] on the provided board.
135
- Retry running the command up to 10 times, with a 15 second timeout between retries.
136
- this should allow for the boards with little memory to complete even if they run out of memory.
137
- """
138
- # add the lib folder to the path
139
- cmd_path = [
140
- "exec",
141
- 'import sys;sys.path.append("/lib") if "/lib" not in sys.path else "/lib already in path"',
142
- ]
143
- mcu.run_command(cmd_path, timeout=5)
144
-
145
- if reset_before:
146
- log.info(f"Resetting {mcu.serialport} {mcu.description}")
147
- mcu.run_command("reset", timeout=5)
148
- time.sleep(2)
149
-
150
- log.info(f"Running createstubs {variant.value} on {mcu.serialport} {mcu.description} using temp path: {dest}")
151
- cmd = build_cmd(dest, variant)
152
- log.info(f"Running : mpremote {' '.join(cmd)}")
153
- mcu.run_command.retry.wait = wait_fixed(15)
154
- # some boards need 2-3 minutes to run createstubs - so increase the default timeout
155
- # esp32s3 > 240 seconds with mounted fs
156
- # but slows down esp8266 restarts so keep that to 90 seconds
157
- timeout = 90 if mcu.port == "esp8266" else 6 * 60 # type: ignore
158
- rc, out = mcu.run_command(cmd, timeout=timeout)
159
- # check last line for exception or error and raise that if found
160
- if rc != OK and out and ":" in out[-1] and not out[-1].startswith("INFO") and not out[-1].startswith("WARN"):
161
- log.warning(f"createstubs: {out[-1]}")
162
- raise RuntimeError(out[-1]) from eval(out[-1].split(":")[0])
163
-
164
- if rc != OK and variant == Variant.db:
165
- # assume createstubs ran out of memory and try again
166
- raise MemoryError("Memory error, try again")
167
- return rc, out
168
-
169
-
170
- def build_cmd(dest: Path, variant: Variant = Variant.db):
171
- """Build the import createstubs[_??] command to run on the board"""
172
- cmd = ["mount", str(dest)] if dest else []
173
- if variant == Variant.db:
174
- cmd += ["exec", "import createstubs_db"]
175
- elif variant == Variant.mem:
176
- cmd += ["exec", "import createstubs_mem"]
177
- else:
178
- cmd += ["exec", "import createstubs"]
179
- return cmd
180
-
181
-
182
- def generate_board_stubs(
183
- dest: Path,
184
- mcu: MPRemoteBoard,
185
- variant: Variant = Variant.db,
186
- form: Form = Form.mpy,
187
- host_mounted: bool = True,
188
- ) -> Tuple[int, Optional[Path]]:
189
- """
190
- Generate the MCU stubs for this MCU board.
191
- Parameters
192
- ----------
193
- dest : Path
194
- The destination folder for the stubs
195
- port : str
196
- The port the board is connected to
197
- """
198
-
199
- # HOST -> MCU : copy createstubs to board
200
- ok = copy_scripts_to_board(mcu, variant, form)
201
- if not ok and not TESTING:
202
- log.warning("Error copying createstubs to board")
203
- return ERROR, None
204
-
205
- copy_boardname_to_board(mcu)
206
-
207
- rc, out = run_createstubs(dest, mcu, variant) # , host_mounted=host_mounted)
208
-
209
- if rc != OK:
210
- log.warning("Error running createstubs: %s", out)
211
- return ERROR, None
212
-
213
- if not host_mounted:
214
- # Waiting for MPRemote to support copying folder from board to host
215
- raise NotImplementedError("TODO: Copy stubs from board to host")
216
-
217
- # Find the output starting with 'Path: '
218
- folder = get_stubfolder(out)
219
- if not folder:
220
- return ERROR, None
221
-
222
- stubs_path = dest / folder
223
- mcu.path = stubs_path
224
- # read the modules.json file into a dict
225
- try:
226
- with open(stubs_path / "modules.json") as fp:
227
- modules_json = json.load(fp)
228
- mcu.firmware = modules_json["firmware"]
229
- except FileNotFoundError:
230
- log.warning("Could not load modules.json, Assuming error in createstubs")
231
- return ERROR, None
232
-
233
- # check the number of stubs generated
234
- if len(list(stubs_path.glob("*.p*"))) < 10:
235
- log.warning("Error generating stubs, too few (<10)stubs were generated")
236
- return ERROR, None
237
-
238
- stubgen_needed = any(stubs_path.glob("*.py"))
239
- utils.do_post_processing([stubs_path], stubgen=stubgen_needed, black=True, autoflake=True)
240
-
241
- return OK, stubs_path
242
-
243
-
244
- def copy_boardname_to_board(mcu: MPRemoteBoard):
245
- """
246
- Copies the board name to the board by writing it to the 'boardname.py' file.
247
-
248
- Args:
249
- mcu: The MCU object representing the microcontroller.
250
-
251
- Returns:
252
- None
253
- """
254
- if mcu.board:
255
- cmd = [
256
- "exec",
257
- f"with open('lib/boardname.py', 'w') as f: f.write('BOARDNAME=\"{mcu.board}\"')",
258
- ]
259
- log.info(f"Writing BOARDNAME='{mcu.board}' to boardname.py")
260
- else:
261
- cmd = ["rm", "boardname.py"]
262
- rc, _ = mcu.run_command(cmd)
263
- if rc != OK and "rm" not in cmd:
264
- log.error(f"Error during copy createstubs running command: {cmd}")
265
-
266
-
267
- def copy_scripts_to_board(mcu: MPRemoteBoard, variant: Variant, form: Form):
268
- """
269
- Copy scripts to the board.
270
-
271
- Args:
272
- mcu (str): The microcontroller unit.
273
- variant (str): The variant of the board.
274
- form (Form): The form of the scripts to be copied.
275
-
276
- Returns:
277
- bool: True if the scripts are successfully copied, False otherwise.
278
- """
279
- if LOCAL_FILES:
280
- return copy_createstubs_to_board(mcu, variant, form)
281
- if form == Form.min:
282
- location = "github:josverl/micropython-stubber/mip/minified.json"
283
- elif form == Form.mpy:
284
- location = "github:josverl/micropython-stubber/mip/mpy_v6.json"
285
- else:
286
- location = "github:josverl/micropython-stubber/mip/full.json"
287
-
288
- return mcu.mip_install(location)
289
-
290
-
291
- def get_stubfolder(out: List[str]):
292
- return lines[-1].split("/remote/")[-1].strip() if (lines := [l for l in out if l.startswith("INFO : Path: ")]) else ""
293
-
294
-
295
- def set_loglevel(verbose: int) -> str:
296
- """Set log level based on verbose level
297
- Get the level from the verbose setting (0=INFO, 1=DEBUG, 2=TRACE)
298
- Set the format string, based on the level.
299
- Add the handler to the logger, with the level and format string.
300
- Return the level
301
- """
302
- log.remove()
303
- level = {0: "INFO", 1: "DEBUG", 2: "TRACE"}.get(verbose, "TRACE")
304
- if level == "INFO":
305
- format_str = "<green>{time:HH:mm:ss}</green>|<level>{level: <8}</level>|<cyan>{module: <20}</cyan> - <level>{message}</level>"
306
- else:
307
- format_str = "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green>|<level>{level: <8}</level>|<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
308
-
309
- log.add(sys.stderr, level=level, backtrace=True, diagnose=True, colorize=True, format=format_str)
310
- # log.info(f"micropython-stubber {__version__}")
311
- return level
312
-
313
-
314
- def copy_to_repo(source: Path, fw: dict) -> Optional[Path]:
315
- """Copy the generated stubs to the stubs repo.
316
- If the destination folder exists, it is first emptied
317
- when successful: returns the destination path - None otherwise
318
- """
319
- destination = CONFIG.stub_path / board_folder_name(fw)
320
- try:
321
- if destination.exists() and destination.is_dir():
322
- # first clean the destination folder
323
- shutil.rmtree(destination)
324
- # copy all files and folder from the source to the destination
325
- shutil.copytree(source, destination, dirs_exist_ok=True)
326
- return destination
327
- except OSError as e:
328
- log.error(f"Error: {source} : {e.strerror}")
329
- return None
330
-
331
-
332
- def stub_connected_mcus(
333
- variant: str,
334
- format: str,
335
- debug: bool,
336
- serial: List[str],
337
- ignore: List[str],
338
- bluetooth: bool,
339
- ) -> int:
340
- """
341
- Runs the stubber to generate stubs for connected MicroPython boards.
342
-
343
- Args:
344
- variant (str): The variant of the createstubs script.
345
- format (str): The format of the createstubs script.
346
- debug (bool): Flag indicating whether to enable debug mode.
347
-
348
- Returns:
349
- None
350
- """
351
-
352
- if debug:
353
- set_loglevel(1)
354
- else:
355
- set_loglevel(0)
356
- variant = Variant(variant.lower())
357
- form = Form(format.lower())
358
- tempdir = mkdtemp(prefix="board_stubber")
359
- temp_path = Path(tempdir)
360
-
361
- all_built = []
362
-
363
- # scan boards and just work with the ones that respond with understandable data
364
- connected_mcus = list_mcus(ignore=ignore, include=serial, bluetooth=bluetooth)
365
- # ignore boards that have the [micropython-stubber] ignore flag set
366
- connected_mcus = [item for item in connected_mcus if not (item.toml.get("micropython-stubber", {}).get("ignore", False))]
367
-
368
- if not connected_mcus:
369
- log.error("No micropython boards were found")
370
- return ERROR
371
-
372
- show_mcus(connected_mcus, refresh=False)
373
-
374
- # scan boards and generate stubs
375
- for board in connected_mcus:
376
- log.info(f"Connecting using {board.serialport} to {board.port} {board.board} {board.version}: {board.description}")
377
- # remove the modulelist.done file before starting createstubs on each board
378
- (temp_path / "modulelist.done").unlink(missing_ok=True)
379
-
380
- rc, my_stubs = generate_board_stubs(temp_path, board, variant, form)
381
- if rc != OK:
382
- log.error(f"Failed to generate stubs for {board.serialport}")
383
- continue
384
- if my_stubs:
385
- log.success(f'Stubs generated for {board.firmware["port"]}-{board.firmware["board"]}')
386
- if destination := copy_to_repo(my_stubs, board.firmware):
387
- log.success(f"Stubs copied to {destination}")
388
- # Also merge the stubs with the docstubs
389
- log.info(f"Merging stubs with docstubs : {board.firmware}")
390
-
391
- merged = merge_all_docstubs(
392
- versions=board.firmware["version"],
393
- family=board.firmware["family"],
394
- boards=board.firmware["board"],
395
- ports=board.firmware["port"],
396
- mpy_path=CONFIG.mpy_path,
397
- )
398
- if not merged:
399
- log.error(f"Failed to merge stubs for {board.serialport}")
400
- continue
401
- # Then Build the package
402
- log.info(f"Building package for {board.firmware}")
403
- built = build_multiple(
404
- versions=board.firmware["version"],
405
- family=board.firmware["family"],
406
- boards=board.firmware["board"],
407
- ports=board.firmware["port"],
408
- )
409
- all_built.extend(built)
410
-
411
- if all_built:
412
- print_result_table(all_built)
413
- log.success("Done")
414
- return OK
415
- log.error(f"Failed to generate stubs for {board.serialport}")
416
- return ERROR
417
-
418
-
419
- def print_result_table(all_built: List, console: Optional[Console] = None):
420
- if not console:
421
- console = Console()
422
- # create a rich table of the results and print it'
423
- table = Table(title="Results")
424
-
425
- table.add_column("Result", style="cyan")
426
- table.add_column("Name/Path", style="cyan")
427
- table.add_column("Version", style="green")
428
- table.add_column("Error", style="red")
429
-
430
- for result in all_built:
431
- table.add_row(
432
- result["result"],
433
- (result["name"] + "\n" + result["path"]).strip(),
434
- result["version"],
435
- result["error"],
436
- )
437
- console.print(table)
1
+ """
2
+ This script creates stubs on and for a connected micropython MCU board.
3
+ """
4
+
5
+ import json
6
+ import shutil
7
+ import sys
8
+ import time
9
+ from enum import Enum
10
+ from pathlib import Path
11
+ from tempfile import mkdtemp
12
+ from typing import List, Optional, Tuple
13
+
14
+ from rich.console import Console
15
+ from rich.table import Table
16
+ from tenacity import retry, stop_after_attempt, wait_fixed
17
+
18
+ from mpflash.connected import list_mcus
19
+ from mpflash.list import show_mcus
20
+ from mpflash.logger import log
21
+ from mpflash.mpremoteboard import ERROR, OK, MPRemoteBoard
22
+ from stubber import utils
23
+ from stubber.publish.merge_docstubs import merge_all_docstubs
24
+ from stubber.publish.pathnames import board_folder_name
25
+ from stubber.publish.publish import build_multiple
26
+ from stubber.utils.config import CONFIG
27
+
28
+ HERE = Path(__file__).parent
29
+ ###############################################################################################
30
+ # TODO: promote to cmdline params
31
+ LOCAL_FILES = False
32
+ reset_before = True
33
+ TESTING = False
34
+ ###############################################################################################
35
+
36
+
37
+ ###############################################################################################
38
+
39
+
40
+ class Variant(str, Enum):
41
+ """Variants to generate stubs on a MCU"""
42
+
43
+ full = "full"
44
+ mem = "mem"
45
+ db = "db"
46
+
47
+
48
+ class Form(str, Enum):
49
+ """Optimization forms of scripts"""
50
+
51
+ py = "py"
52
+ min = "min"
53
+ mpy = "mpy"
54
+
55
+
56
+ def copy_createstubs_to_board(board: MPRemoteBoard, variant: Variant, form: Form) -> bool:
57
+ # sourcery skip: assign-if-exp, boolean-if-exp-identity, remove-unnecessary-cast
58
+ """Copy createstubs to the board"""
59
+ # copy createstubs.py to the destination folder
60
+ origin = "./src/stubber/board"
61
+
62
+ _py = [
63
+ "rm :lib/createstubs.mpy",
64
+ "rm :lib/createstubs_mem.mpy",
65
+ "rm :lib/createstubs_db.mpy",
66
+ "rm :lib/createstubs.py",
67
+ "rm :lib/createstubs_mem.py",
68
+ "rm :lib/createstubs_db.py",
69
+ f"cp {origin}/createstubs.py :lib/createstubs.py",
70
+ f"cp {origin}/createstubs_mem.py :lib/createstubs_mem.py",
71
+ f"cp {origin}/createstubs_db.py :lib/createstubs_db.py",
72
+ ]
73
+
74
+ # copy createstubs*_min.py to the destination folder
75
+ _min = [
76
+ f"cp {origin}/createstubs_min.py :lib/createstubs.py",
77
+ f"cp {origin}/createstubs_mem_min.py :lib/createstubs_mem.py",
78
+ f"cp {origin}/createstubs_db_min.py :lib/createstubs_db.py",
79
+ ]
80
+ # copy createstubs*_mpy.mpy to the destination folder
81
+ _mpy = [
82
+ "rm :lib/createstubs.py",
83
+ "rm :lib/createstubs_mem.py",
84
+ "rm :lib/createstubs_db.py",
85
+ f"cp {origin}/createstubs_mpy.mpy :lib/createstubs.mpy",
86
+ f"cp {origin}/createstubs_mem_mpy.mpy :lib/createstubs_mem.mpy",
87
+ f"cp {origin}/createstubs_db_mpy.mpy :lib/createstubs_db.mpy",
88
+ ]
89
+
90
+ _lib = [
91
+ [
92
+ "exec",
93
+ "import os;os.mkdir('lib') if not ('lib' in os.listdir()) else print('folder lib already exists')",
94
+ ]
95
+ ]
96
+
97
+ _get_ready = [
98
+ "rm :modulelist.done",
99
+ "rm :no_auto_stubber.txt",
100
+ f"cp {origin}/modulelist.txt :lib/modulelist.txt",
101
+ ]
102
+ if form == Form.py:
103
+ do = _lib + _py + _get_ready
104
+ elif form == Form.min:
105
+ do = _lib + _min + _get_ready
106
+ else:
107
+ do = _lib + _mpy + _get_ready
108
+
109
+ # assume all ok, unless one is not ok
110
+ for cmd in do:
111
+ if isinstance(cmd, str) and cmd.startswith("rm "):
112
+ log_errors = False
113
+ else:
114
+ log_errors = True
115
+ rc, _ = board.run_command(cmd, log_errors=log_errors)
116
+ if rc != OK and "rm" not in cmd:
117
+ log.error(f"Error during copy createstubs running command: {cmd}")
118
+ return False
119
+ return True
120
+
121
+
122
+ @retry(stop=stop_after_attempt(4), wait=wait_fixed(2))
123
+ def hard_reset(board: MPRemoteBoard) -> bool:
124
+ """Reset the board"""
125
+ # do not run "exec", "import machine;machine.reset()" as it will hang an esp32
126
+ rc, _ = board.run_command(["reset"], timeout=5)
127
+ board.connected = False
128
+ return rc == OK
129
+
130
+
131
+ @retry(stop=stop_after_attempt(10), wait=wait_fixed(15))
132
+ def run_createstubs(dest: Path, mcu: MPRemoteBoard, variant: Variant = Variant.db):
133
+ """
134
+ Run a createstubs[variant] on the provided board.
135
+ Retry running the command up to 10 times, with a 15 second timeout between retries.
136
+ this should allow for the boards with little memory to complete even if they run out of memory.
137
+ """
138
+ # add the lib folder to the path
139
+ cmd_path = [
140
+ "exec",
141
+ 'import sys;sys.path.append("/lib") if "/lib" not in sys.path else "/lib already in path"',
142
+ ]
143
+ mcu.run_command(cmd_path, timeout=5)
144
+
145
+ if reset_before:
146
+ log.info(f"Resetting {mcu.serialport} {mcu.description}")
147
+ mcu.run_command("reset", timeout=5)
148
+ time.sleep(2)
149
+
150
+ log.info(f"Running createstubs {variant.value} on {mcu.serialport} {mcu.description} using temp path: {dest}")
151
+ cmd = build_cmd(dest, variant)
152
+ log.info(f"Running : mpremote {' '.join(cmd)}")
153
+ mcu.run_command.retry.wait = wait_fixed(15)
154
+ # some boards need 2-3 minutes to run createstubs - so increase the default timeout
155
+ # esp32s3 > 240 seconds with mounted fs
156
+ # but slows down esp8266 restarts so keep that to 90 seconds
157
+ timeout = 90 if mcu.port == "esp8266" else 6 * 60 # type: ignore
158
+ rc, out = mcu.run_command(cmd, timeout=timeout)
159
+ # check last line for exception or error and raise that if found
160
+ if rc != OK and out and ":" in out[-1] and not out[-1].startswith("INFO") and not out[-1].startswith("WARN"):
161
+ log.warning(f"createstubs: {out[-1]}")
162
+ raise RuntimeError(out[-1]) from eval(out[-1].split(":")[0])
163
+
164
+ if rc != OK and variant == Variant.db:
165
+ # assume createstubs ran out of memory and try again
166
+ raise MemoryError("Memory error, try again")
167
+ return rc, out
168
+
169
+
170
+ def build_cmd(dest: Path, variant: Variant = Variant.db):
171
+ """Build the import createstubs[_??] command to run on the board"""
172
+ cmd = ["mount", str(dest)] if dest else []
173
+ if variant == Variant.db:
174
+ cmd += ["exec", "import createstubs_db"]
175
+ elif variant == Variant.mem:
176
+ cmd += ["exec", "import createstubs_mem"]
177
+ else:
178
+ cmd += ["exec", "import createstubs"]
179
+ return cmd
180
+
181
+
182
+ def generate_board_stubs(
183
+ dest: Path,
184
+ mcu: MPRemoteBoard,
185
+ variant: Variant = Variant.db,
186
+ form: Form = Form.mpy,
187
+ host_mounted: bool = True,
188
+ ) -> Tuple[int, Optional[Path]]:
189
+ """
190
+ Generate the MCU stubs for this MCU board.
191
+ Parameters
192
+ ----------
193
+ dest : Path
194
+ The destination folder for the stubs
195
+ port : str
196
+ The port the board is connected to
197
+ """
198
+
199
+ # HOST -> MCU : copy createstubs to board
200
+ ok = copy_scripts_to_board(mcu, variant, form)
201
+ if not ok and not TESTING:
202
+ log.warning("Error copying createstubs to board")
203
+ return ERROR, None
204
+
205
+ copy_boardname_to_board(mcu)
206
+
207
+ rc, out = run_createstubs(dest, mcu, variant) # , host_mounted=host_mounted)
208
+
209
+ if rc != OK:
210
+ log.warning("Error running createstubs: %s", out)
211
+ return ERROR, None
212
+
213
+ if not host_mounted:
214
+ # Waiting for MPRemote to support copying folder from board to host
215
+ raise NotImplementedError("TODO: Copy stubs from board to host")
216
+
217
+ # Find the output starting with 'Path: '
218
+ folder = get_stubfolder(out)
219
+ if not folder:
220
+ return ERROR, None
221
+
222
+ stubs_path = dest / folder
223
+ mcu.path = stubs_path
224
+ # read the modules.json file into a dict
225
+ try:
226
+ with open(stubs_path / "modules.json") as fp:
227
+ modules_json = json.load(fp)
228
+ mcu.firmware = modules_json["firmware"]
229
+ except FileNotFoundError:
230
+ log.warning("Could not load modules.json, Assuming error in createstubs")
231
+ return ERROR, None
232
+
233
+ # check the number of stubs generated
234
+ if len(list(stubs_path.glob("*.p*"))) < 10:
235
+ log.warning("Error generating stubs, too few (<10)stubs were generated")
236
+ return ERROR, None
237
+
238
+ stubgen_needed = any(stubs_path.glob("*.py"))
239
+ utils.do_post_processing([stubs_path], stubgen=stubgen_needed, black=True, autoflake=True)
240
+
241
+ return OK, stubs_path
242
+
243
+
244
+ def copy_boardname_to_board(mcu: MPRemoteBoard):
245
+ """
246
+ Copies the board name to the board by writing it to the 'boardname.py' file.
247
+
248
+ Args:
249
+ mcu: The MCU object representing the microcontroller.
250
+
251
+ Returns:
252
+ None
253
+ """
254
+ if mcu.board:
255
+ cmd = [
256
+ "exec",
257
+ f"with open('lib/boardname.py', 'w') as f: f.write('BOARDNAME=\"{mcu.board}\"')",
258
+ ]
259
+ log.info(f"Writing BOARDNAME='{mcu.board}' to boardname.py")
260
+ else:
261
+ cmd = ["rm", "boardname.py"]
262
+ rc, _ = mcu.run_command(cmd)
263
+ if rc != OK and "rm" not in cmd:
264
+ log.error(f"Error during copy createstubs running command: {cmd}")
265
+
266
+
267
+ def copy_scripts_to_board(mcu: MPRemoteBoard, variant: Variant, form: Form):
268
+ """
269
+ Copy scripts to the board.
270
+
271
+ Args:
272
+ mcu (str): The microcontroller unit.
273
+ variant (str): The variant of the board.
274
+ form (Form): The form of the scripts to be copied.
275
+
276
+ Returns:
277
+ bool: True if the scripts are successfully copied, False otherwise.
278
+ """
279
+ if LOCAL_FILES:
280
+ return copy_createstubs_to_board(mcu, variant, form)
281
+ if form == Form.min:
282
+ location = "github:josverl/micropython-stubber/mip/minified.json"
283
+ elif form == Form.mpy:
284
+ location = "github:josverl/micropython-stubber/mip/mpy_v6.json"
285
+ else:
286
+ location = "github:josverl/micropython-stubber/mip/full.json"
287
+
288
+ return mcu.mip_install(location)
289
+
290
+
291
+ def get_stubfolder(out: List[str]):
292
+ return lines[-1].split("/remote/")[-1].strip() if (lines := [l for l in out if l.startswith("INFO : Path: ")]) else ""
293
+
294
+
295
+ def set_loglevel(verbose: int) -> str:
296
+ """Set log level based on verbose level
297
+ Get the level from the verbose setting (0=INFO, 1=DEBUG, 2=TRACE)
298
+ Set the format string, based on the level.
299
+ Add the handler to the logger, with the level and format string.
300
+ Return the level
301
+ """
302
+ log.remove()
303
+ level = {0: "INFO", 1: "DEBUG", 2: "TRACE"}.get(verbose, "TRACE")
304
+ if level == "INFO":
305
+ format_str = "<green>{time:HH:mm:ss}</green>|<level>{level: <8}</level>|<cyan>{module: <20}</cyan> - <level>{message}</level>"
306
+ else:
307
+ format_str = "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green>|<level>{level: <8}</level>|<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
308
+
309
+ log.add(sys.stderr, level=level, backtrace=True, diagnose=True, colorize=True, format=format_str)
310
+ # log.info(f"micropython-stubber {__version__}")
311
+ return level
312
+
313
+
314
+ def copy_to_repo(source: Path, fw: dict) -> Optional[Path]:
315
+ """Copy the generated stubs to the stubs repo.
316
+ If the destination folder exists, it is first emptied
317
+ when successful: returns the destination path - None otherwise
318
+ """
319
+ destination = CONFIG.stub_path / board_folder_name(fw)
320
+ try:
321
+ if destination.exists() and destination.is_dir():
322
+ # first clean the destination folder
323
+ shutil.rmtree(destination)
324
+ # copy all files and folder from the source to the destination
325
+ shutil.copytree(source, destination, dirs_exist_ok=True)
326
+ return destination
327
+ except OSError as e:
328
+ log.error(f"Error: {source} : {e.strerror}")
329
+ return None
330
+
331
+
332
+ def stub_connected_mcus(
333
+ variant: str,
334
+ format: str,
335
+ debug: bool,
336
+ serial: List[str],
337
+ ignore: List[str],
338
+ bluetooth: bool,
339
+ ) -> int:
340
+ """
341
+ Runs the stubber to generate stubs for connected MicroPython boards.
342
+
343
+ Args:
344
+ variant (str): The variant of the createstubs script.
345
+ format (str): The format of the createstubs script.
346
+ debug (bool): Flag indicating whether to enable debug mode.
347
+
348
+ Returns:
349
+ None
350
+ """
351
+
352
+ if debug:
353
+ set_loglevel(1)
354
+ else:
355
+ set_loglevel(0)
356
+ variant = Variant(variant.lower())
357
+ form = Form(format.lower())
358
+ tempdir = mkdtemp(prefix="board_stubber")
359
+ temp_path = Path(tempdir)
360
+
361
+ all_built = []
362
+
363
+ # scan boards and just work with the ones that respond with understandable data
364
+ connected_mcus = list_mcus(ignore=ignore, include=serial, bluetooth=bluetooth)
365
+ # ignore boards that have the [micropython-stubber] ignore flag set
366
+ connected_mcus = [item for item in connected_mcus if not (item.toml.get("micropython-stubber", {}).get("ignore", False))]
367
+
368
+ if not connected_mcus:
369
+ log.error("No micropython boards were found")
370
+ return ERROR
371
+
372
+ show_mcus(connected_mcus, refresh=False)
373
+
374
+ # scan boards and generate stubs
375
+ for board in connected_mcus:
376
+ log.info(f"Connecting using {board.serialport} to {board.port} {board.board} {board.version}: {board.description}")
377
+ # remove the modulelist.done file before starting createstubs on each board
378
+ (temp_path / "modulelist.done").unlink(missing_ok=True)
379
+
380
+ rc, my_stubs = generate_board_stubs(temp_path, board, variant, form)
381
+ if rc != OK:
382
+ log.error(f"Failed to generate stubs for {board.serialport}")
383
+ continue
384
+ if my_stubs:
385
+ log.success(f'Stubs generated for {board.firmware["port"]}-{board.firmware["board"]}')
386
+ if destination := copy_to_repo(my_stubs, board.firmware):
387
+ log.success(f"Stubs copied to {destination}")
388
+ # Also merge the stubs with the docstubs
389
+ log.info(f"Merging stubs with docstubs : {board.firmware}")
390
+
391
+ merged = merge_all_docstubs(
392
+ versions=board.firmware["version"],
393
+ family=board.firmware["family"],
394
+ boards=board.firmware["board"],
395
+ ports=board.firmware["port"],
396
+ mpy_path=CONFIG.mpy_path,
397
+ )
398
+ if not merged:
399
+ log.error(f"Failed to merge stubs for {board.serialport}")
400
+ continue
401
+ # Then Build the package
402
+ log.info(f"Building package for {board.firmware}")
403
+ built = build_multiple(
404
+ versions=board.firmware["version"],
405
+ family=board.firmware["family"],
406
+ boards=board.firmware["board"],
407
+ ports=board.firmware["port"],
408
+ )
409
+ all_built.extend(built)
410
+
411
+ if all_built:
412
+ print_result_table(all_built)
413
+ log.success("Done")
414
+ return OK
415
+ log.error(f"Failed to generate stubs for {board.serialport}")
416
+ return ERROR
417
+
418
+
419
+ def print_result_table(all_built: List, console: Optional[Console] = None):
420
+ if not console:
421
+ console = Console()
422
+ # create a rich table of the results and print it'
423
+ table = Table(title="Results")
424
+
425
+ table.add_column("Result", style="cyan")
426
+ table.add_column("Name/Path", style="cyan")
427
+ table.add_column("Version", style="green")
428
+ table.add_column("Error", style="red")
429
+
430
+ for result in all_built:
431
+ table.add_row(
432
+ result["result"],
433
+ (result["name"] + "\n" + result["path"]).strip(),
434
+ result["version"],
435
+ result["error"],
436
+ )
437
+ console.print(table)