micropython-stubber 1.23.1__py3-none-any.whl → 1.23.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. {micropython_stubber-1.23.1.dist-info → micropython_stubber-1.23.2.dist-info}/LICENSE +30 -30
  2. {micropython_stubber-1.23.1.dist-info → micropython_stubber-1.23.2.dist-info}/METADATA +32 -15
  3. micropython_stubber-1.23.2.dist-info/RECORD +158 -0
  4. micropython_stubber-1.23.2.dist-info/entry_points.txt +5 -0
  5. mpflash/README.md +220 -194
  6. mpflash/libusb_flash.ipynb +203 -203
  7. mpflash/mpflash/add_firmware.py +98 -98
  8. mpflash/mpflash/ask_input.py +236 -236
  9. mpflash/mpflash/basicgit.py +284 -284
  10. mpflash/mpflash/bootloader/__init__.py +2 -2
  11. mpflash/mpflash/bootloader/activate.py +60 -60
  12. mpflash/mpflash/bootloader/detect.py +82 -82
  13. mpflash/mpflash/bootloader/manual.py +101 -101
  14. mpflash/mpflash/bootloader/micropython.py +12 -12
  15. mpflash/mpflash/bootloader/touch1200.py +36 -36
  16. mpflash/mpflash/cli_download.py +129 -129
  17. mpflash/mpflash/cli_flash.py +224 -219
  18. mpflash/mpflash/cli_group.py +111 -111
  19. mpflash/mpflash/cli_list.py +87 -81
  20. mpflash/mpflash/cli_main.py +39 -39
  21. mpflash/mpflash/common.py +210 -165
  22. mpflash/mpflash/config.py +44 -44
  23. mpflash/mpflash/connected.py +96 -78
  24. mpflash/mpflash/download.py +364 -364
  25. mpflash/mpflash/downloaded.py +130 -130
  26. mpflash/mpflash/errors.py +9 -9
  27. mpflash/mpflash/flash/__init__.py +55 -55
  28. mpflash/mpflash/flash/esp.py +59 -59
  29. mpflash/mpflash/flash/stm32.py +19 -19
  30. mpflash/mpflash/flash/stm32_dfu.py +104 -104
  31. mpflash/mpflash/flash/uf2/__init__.py +88 -88
  32. mpflash/mpflash/flash/uf2/boardid.py +15 -15
  33. mpflash/mpflash/flash/uf2/linux.py +136 -130
  34. mpflash/mpflash/flash/uf2/macos.py +42 -42
  35. mpflash/mpflash/flash/uf2/uf2disk.py +12 -12
  36. mpflash/mpflash/flash/uf2/windows.py +43 -43
  37. mpflash/mpflash/flash/worklist.py +170 -170
  38. mpflash/mpflash/list.py +106 -99
  39. mpflash/mpflash/logger.py +41 -41
  40. mpflash/mpflash/mpboard_id/__init__.py +93 -93
  41. mpflash/mpflash/mpboard_id/add_boards.py +251 -251
  42. mpflash/mpflash/mpboard_id/board.py +37 -37
  43. mpflash/mpflash/mpboard_id/board_id.py +86 -86
  44. mpflash/mpflash/mpboard_id/store.py +43 -43
  45. mpflash/mpflash/mpremoteboard/__init__.py +266 -222
  46. mpflash/mpflash/mpremoteboard/mpy_fw_info.py +141 -141
  47. mpflash/mpflash/mpremoteboard/runner.py +140 -140
  48. mpflash/mpflash/vendor/click_aliases.py +91 -91
  49. mpflash/mpflash/vendor/dfu.py +165 -165
  50. mpflash/mpflash/vendor/pydfu.py +605 -605
  51. mpflash/mpflash/vendor/readme.md +2 -2
  52. mpflash/mpflash/versions.py +135 -135
  53. mpflash/poetry.lock +1599 -1599
  54. mpflash/pyproject.toml +65 -65
  55. mpflash/stm32_udev_rules.md +62 -62
  56. stubber/__init__.py +3 -3
  57. stubber/board/board_info.csv +193 -193
  58. stubber/board/boot.py +34 -34
  59. stubber/board/createstubs.py +1004 -986
  60. stubber/board/createstubs_db.py +826 -825
  61. stubber/board/createstubs_db_min.py +332 -331
  62. stubber/board/createstubs_db_mpy.mpy +0 -0
  63. stubber/board/createstubs_lvgl.py +741 -741
  64. stubber/board/createstubs_lvgl_min.py +741 -741
  65. stubber/board/createstubs_mem.py +767 -766
  66. stubber/board/createstubs_mem_min.py +307 -306
  67. stubber/board/createstubs_mem_mpy.mpy +0 -0
  68. stubber/board/createstubs_min.py +295 -294
  69. stubber/board/createstubs_mpy.mpy +0 -0
  70. stubber/board/fw_info.py +141 -141
  71. stubber/board/info.py +183 -183
  72. stubber/board/main.py +19 -19
  73. stubber/board/modulelist.txt +247 -247
  74. stubber/board/pyrightconfig.json +34 -34
  75. stubber/bulk/mcu_stubber.py +437 -454
  76. stubber/codemod/_partials/__init__.py +48 -48
  77. stubber/codemod/_partials/db_main.py +147 -147
  78. stubber/codemod/_partials/lvgl_main.py +77 -77
  79. stubber/codemod/_partials/modules_reader.py +80 -80
  80. stubber/codemod/add_comment.py +53 -53
  81. stubber/codemod/add_method.py +65 -65
  82. stubber/codemod/board.py +317 -317
  83. stubber/codemod/enrich.py +151 -145
  84. stubber/codemod/merge_docstub.py +284 -284
  85. stubber/codemod/modify_list.py +54 -54
  86. stubber/codemod/utils.py +56 -56
  87. stubber/commands/build_cmd.py +94 -94
  88. stubber/commands/cli.py +49 -55
  89. stubber/commands/clone_cmd.py +78 -78
  90. stubber/commands/config_cmd.py +29 -29
  91. stubber/commands/enrich_folder_cmd.py +71 -71
  92. stubber/commands/get_core_cmd.py +71 -71
  93. stubber/commands/get_docstubs_cmd.py +92 -89
  94. stubber/commands/get_frozen_cmd.py +117 -114
  95. stubber/commands/get_mcu_cmd.py +102 -61
  96. stubber/commands/merge_cmd.py +66 -66
  97. stubber/commands/publish_cmd.py +118 -118
  98. stubber/commands/stub_cmd.py +31 -31
  99. stubber/commands/switch_cmd.py +62 -62
  100. stubber/commands/variants_cmd.py +48 -48
  101. stubber/cst_transformer.py +178 -178
  102. stubber/data/board_info.csv +193 -193
  103. stubber/data/board_info.json +1729 -1729
  104. stubber/data/micropython_tags.csv +15 -15
  105. stubber/data/requirements-core-micropython.txt +38 -38
  106. stubber/data/requirements-core-pycopy.txt +39 -39
  107. stubber/downloader.py +37 -36
  108. stubber/freeze/common.py +72 -68
  109. stubber/freeze/freeze_folder.py +69 -69
  110. stubber/freeze/freeze_manifest_2.py +126 -113
  111. stubber/freeze/get_frozen.py +131 -127
  112. stubber/get_cpython.py +112 -101
  113. stubber/get_lobo.py +59 -59
  114. stubber/minify.py +423 -419
  115. stubber/publish/bump.py +86 -86
  116. stubber/publish/candidates.py +275 -256
  117. stubber/publish/database.py +18 -18
  118. stubber/publish/defaults.py +40 -40
  119. stubber/publish/enums.py +24 -24
  120. stubber/publish/helpers.py +29 -29
  121. stubber/publish/merge_docstubs.py +136 -130
  122. stubber/publish/missing_class_methods.py +51 -49
  123. stubber/publish/package.py +150 -146
  124. stubber/publish/pathnames.py +51 -51
  125. stubber/publish/publish.py +120 -120
  126. stubber/publish/pypi.py +42 -38
  127. stubber/publish/stubpackage.py +1055 -1027
  128. stubber/rst/__init__.py +9 -9
  129. stubber/rst/classsort.py +78 -77
  130. stubber/rst/lookup.py +533 -530
  131. stubber/rst/output_dict.py +401 -401
  132. stubber/rst/reader.py +814 -814
  133. stubber/rst/report_return.py +77 -69
  134. stubber/rst/rst_utils.py +541 -540
  135. stubber/stubber.py +38 -38
  136. stubber/stubs_from_docs.py +90 -90
  137. stubber/tools/manifestfile.py +654 -654
  138. stubber/tools/readme.md +6 -6
  139. stubber/update_fallback.py +117 -117
  140. stubber/update_module_list.py +123 -123
  141. stubber/utils/__init__.py +6 -6
  142. stubber/utils/config.py +137 -125
  143. stubber/utils/makeversionhdr.py +54 -54
  144. stubber/utils/manifest.py +90 -90
  145. stubber/utils/post.py +80 -79
  146. stubber/utils/repos.py +156 -150
  147. stubber/utils/stubmaker.py +139 -139
  148. stubber/utils/typed_config_toml.py +80 -77
  149. stubber/variants.py +106 -106
  150. micropython_stubber-1.23.1.dist-info/RECORD +0 -159
  151. micropython_stubber-1.23.1.dist-info/entry_points.txt +0 -3
  152. mpflash/basicgit.py +0 -288
  153. {micropython_stubber-1.23.1.dist-info → micropython_stubber-1.23.2.dist-info}/WHEEL +0 -0
stubber/minify.py CHANGED
@@ -1,419 +1,423 @@
1
- """
2
- Processing for createstubs.py
3
- Minimizes and cross-compiles a MicroPyton file.
4
- """
5
-
6
- import itertools
7
- import subprocess
8
- import tempfile
9
- from contextlib import ExitStack
10
- from io import BytesIO, IOBase, StringIO, TextIOWrapper
11
- from pathlib import Path
12
- from typing import List, Tuple, Union
13
-
14
- try:
15
- import python_minifier
16
- except ImportError:
17
- python_minifier = None
18
-
19
- from loguru import logger as log
20
-
21
- from mpflash.versions import SET_PREVIEW, V_PREVIEW
22
-
23
- # Type Aliases for minify
24
- StubSource = Union[Path, str, StringIO, TextIOWrapper]
25
- XCompileDest = Union[Path, BytesIO]
26
- LineEdits = List[Tuple[str, str]]
27
-
28
-
29
- def get_whitespace_context(content: List[str], index: int):
30
- """Get whitespace count of lines surrounding index"""
31
- if not content:
32
- raise ValueError()
33
- if index < 0 or index > len(content):
34
- raise IndexError()
35
- if len(content) == 1:
36
- return [0, 0]
37
-
38
- def count_ws(line: str):
39
- return sum(1 for _ in itertools.takewhile(str.isspace, line))
40
-
41
- lines = content[max(0, index) : min(index + 2, len(content))]
42
- context = [count_ws(l) for l in lines]
43
- if len(context) < 2:
44
- context.append(context[0])
45
- return context
46
-
47
-
48
- def edit_lines(content: str, edits: LineEdits, diff: bool = False):
49
- # sourcery skip: no-long-functions
50
- """Edit string by list of edits
51
-
52
- Args:
53
- content (str): content to edit
54
- edits ([(str, str)]): List of edits to make.
55
- The first string in the tuple representsthe type of edit to make, can be either:
56
- - comment - comment text out (removed on minify)
57
- - rprint - replace text with print
58
- - rpass - replace text with pass
59
- The second string is the matching text to replace
60
- diff (bool, optional): Prints diff of each edit.
61
- Defaults to False.
62
-
63
- Returns:
64
- str: edited string
65
- """
66
-
67
- def comment(l: str, x: str):
68
- """Comment out line, so it will be removed on minify"""
69
- return l.replace(x, f"# {x}")
70
-
71
- def rprint(l: str, x: str): # type: ignore
72
- """Replace (logging) with print"""
73
- split = l.split("(")
74
- if len(split) > 1:
75
- return l.replace(split[0].strip(), "print")
76
- return l.replace(x, "print")
77
-
78
- def keepprint(l: str, x: str): # type: ignore
79
- """Replace 'print' with 'print '"""
80
- return l.replace("print(", "print (")
81
-
82
- def rpass(l: str, x: str): # type: ignore
83
- """Replace with pass"""
84
- return l.replace(x, "pass")
85
-
86
- def handle_multiline(content: List[str], index: int):
87
- """Handles edits that require multiline comments
88
-
89
- Example:
90
- self.log.debug("info: {} {}".format(
91
- 1,
92
- 2
93
- ))
94
- Here, only commenting out the first self.log line will raise
95
- an error. So this function returns all lines that need to
96
- be commented out instead.
97
-
98
- It also checks for situations such as this:
99
- if condition:
100
- self.log.debug('message')
101
-
102
- Here, since the only functionality of the conditional is the call log,
103
- both lines would be returned to comment out.
104
-
105
- """
106
- line = content[index]
107
- open_cnt = line.count("(")
108
- close_cnt = line.count(")")
109
- ahead_index = 1
110
- look_ahead = 0
111
- while open_cnt != close_cnt: # pragma: no cover
112
- look_ahead = l_index + ahead_index
113
- ahead_index += 1
114
- next_l = content[look_ahead]
115
- open_cnt += next_l.count("(")
116
- close_cnt += next_l.count(")")
117
- if ahead_index > 1: # pragma: no cover
118
- return range(index, look_ahead + 1)
119
- prev = content[index - 1]
120
- line_ws, post_ws = get_whitespace_context(content, index)
121
- prev_words = prev.strip().strip(":").split()
122
- check = any(
123
- t
124
- in (
125
- "if",
126
- "else",
127
- )
128
- for t in prev_words
129
- )
130
- return range(index - 1, index + 1) if check and line_ws != post_ws else None
131
-
132
- def handle_try_except(content: List[str], index: int) -> bool:
133
- """Checks if line at index is in try/except block
134
-
135
- Handles situations like this:
136
- try:
137
- something()
138
- except:
139
- self.log.debug('some message')
140
-
141
- Simply removing the self.log call would create a syntax error,
142
- which is what this function checks for.
143
-
144
- """
145
- prev = content[index - 1]
146
- line_ws, post_ws = get_whitespace_context(content, index)
147
- return "except" in prev and line_ws != post_ws
148
-
149
- lines = []
150
- multilines = set()
151
- content_l = content.splitlines(keepends=True)
152
- for line in content_l:
153
- _line = line
154
- for edit_action, match_text in edits:
155
- if match_text in line:
156
- if edit_action == "comment":
157
- l_index = content_l.index(line)
158
- # Check if edit spans multiple lines
159
- mline = handle_multiline(content_l, l_index)
160
- if mline:
161
- multilines.update(mline)
162
- break
163
- # Check if line is only statement in try/except
164
- if handle_try_except(content_l, l_index):
165
- edit_action = "rpass"
166
- match_text = line.strip()
167
- func = eval(edit_action) # pylint: disable= eval-used
168
- line = func(line, match_text)
169
- if line != _line:
170
- if diff:
171
- print(f"\n- {_line.strip()}")
172
- print(f"+ {line.strip()}")
173
- break
174
- lines.append(line)
175
- for line_num in multilines:
176
- # Go back and comment out multilines
177
- line = lines[line_num]
178
- lines[line_num] = comment(line, line.strip())
179
- stripped = "".join(lines)
180
- return stripped
181
-
182
-
183
- def minify_script(source_script: StubSource, keep_report: bool = True, diff: bool = False) -> str:
184
- """
185
- Minifies createstubs.py and variants
186
-
187
- Args:
188
- source_script:
189
- - (str): content to edit
190
- - (Path): path to file to edit
191
- - (IOBase): file-like object to edit
192
- keep_report (bool, optional): Keeps single report line in createstubs
193
- Defaults to True.
194
- diff (bool, optional): Print diff from edits. Defaults to False.
195
-
196
- Returns:
197
- str: minified source text
198
- """
199
-
200
- source_content = ""
201
- if isinstance(source_script, Path):
202
- source_content = source_script.read_text(encoding="utf-8")
203
- elif isinstance(source_script, (StringIO, TextIOWrapper)):
204
- source_content = "".join(source_script.readlines())
205
- elif isinstance(source_script, str): # type: ignore
206
- source_content = source_script
207
- else:
208
- raise TypeError(f"source_script must be str, Path, or file-like object, not {type(source_script)}")
209
-
210
- if not source_content:
211
- raise ValueError("No source content")
212
- len_1 = len(source_content)
213
-
214
- if 0:
215
- min_source = reduce_log_print(keep_report, diff, source_content)
216
- else:
217
- min_source = source_content
218
- len_2 = len(min_source)
219
-
220
- if not python_minifier:
221
- log.warning("python_minifier not found, skipping minification")
222
- else:
223
- # use python_minifier to minify the source if it is successfully imported
224
- min_source = python_minifier.minify(
225
- min_source,
226
- filename=getattr(source_script, "name", None),
227
- combine_imports=True,
228
- remove_literal_statements=True, # no Docstrings
229
- remove_annotations=True, # not used runtime anyways
230
- hoist_literals=True, # remove redundant strings
231
- rename_locals=True, # short names save memory
232
- preserve_locals=["stubber", "path"], # names to keep
233
- rename_globals=True, # short names save memory
234
- # keep these globals to allow testing/mocking to work against the minified not compiled version
235
- preserve_globals=[
236
- "main",
237
- "Stubber",
238
- "read_path",
239
- "get_root",
240
- "_info",
241
- "os",
242
- "sys",
243
- "__version__",
244
- ],
245
- # remove_pass=True, # no dead code
246
- # convert_posargs_to_args=True, # Does not save any space
247
- )
248
- len_3 = len(min_source)
249
- if 1:
250
- # write to temp file for debugging
251
- with open("tmp_minified.py", "w+") as f:
252
- f.write(min_source)
253
-
254
- log.info(f"Original length : {len_1}")
255
- log.info(f"Reduced length : {len_2}")
256
- log.info(f"Minified length : {len_3}")
257
- log.info(f"Reduced by : {len_1-len_3} ")
258
- return min_source
259
-
260
-
261
- def reduce_log_print(keep_report, diff, source_content):
262
- edits: LineEdits = [
263
- ("keepprint", "print('Debug: "),
264
- ("keepprint", "print('DEBUG: "),
265
- ("keepprint", 'print("Debug: '),
266
- ("keepprint", 'print("DEBUG: '),
267
- ("comment", "print("),
268
- ("comment", "import logging"),
269
- # report keepers may be inserted here
270
- # do report errors
271
- ("rprint", "self.log.error"),
272
- ("rprint", "log.error"),
273
- ]
274
- if keep_report:
275
- # insert report keepers after the comment modifiers
276
- edits += [
277
- # keepers
278
- ("rprint", 'self.log.info("Stub module: '),
279
- ("rprint", 'self.log.warning("{}Skip module:'),
280
- ("rprint", 'self.log.info("Clean/remove files in folder:'),
281
- ("rprint", 'self.log.info("Created stubs for'),
282
- ("rprint", 'self.log.info("Family: '),
283
- ("rprint", 'self.log.info("Version: '),
284
- ("rprint", 'self.log.info("Port: '),
285
- ("rprint", 'self.log.info("Board: '),
286
- # all others
287
- ("comment", "self.log"),
288
- ("comment", "_log."),
289
- ("comment", "_log ="),
290
- ]
291
- else:
292
- edits += [
293
- # remove first full
294
- ("comment", "self.log ="),
295
- ("comment", "self.log("),
296
- ("comment", "self.log.debug"),
297
- ("comment", "self.log.info"),
298
- ("comment", "self.log.warning"),
299
- # then short versions
300
- ("comment", "_log ="),
301
- ("comment", "_log.debug"),
302
- ("comment", "_log.info"),
303
- ("comment", "_log.warning"),
304
- ] + edits
305
-
306
- content = edit_lines(source_content, edits, diff=diff)
307
- return content
308
-
309
-
310
- def minify(
311
- source: StubSource,
312
- target: StubSource,
313
- keep_report: bool = True,
314
- diff: bool = False,
315
- ):
316
- """Minifies and compiles a script"""
317
- source_buf = None
318
- target_buf = None
319
-
320
- with ExitStack() as stack:
321
- if isinstance(source, Path):
322
- source_buf = stack.enter_context(source.open("r"))
323
- elif isinstance(source, (StringIO, str)):
324
- # different types of file-like objects are both acepted by minify_script
325
- source_buf = source
326
- else:
327
- raise TypeError(f"source must be str, Path, or file-like object, not {type(source)}")
328
-
329
- if isinstance(target, Path):
330
- if target.is_dir():
331
- if isinstance(source, Path):
332
- target = target / source.name
333
- else:
334
- target = target / "minified.py" # or raise error?
335
- target_buf = stack.enter_context(target.open("w+"))
336
- elif isinstance(target, IOBase): # type: ignore
337
- target_buf = target
338
- try:
339
- minified = minify_script(source_script=source_buf, keep_report=keep_report, diff=diff)
340
- target_buf.write(minified)
341
- except Exception as e: # pragma: no cover
342
- log.exception(e)
343
- return 0
344
-
345
-
346
- def cross_compile(
347
- source: StubSource,
348
- target: XCompileDest,
349
- version: str = "",
350
- ): # sourcery skip: assign-if-exp
351
- """Runs mpy-cross on a (minified) script"""
352
- # Sources can be a file, a string, or a file-like object
353
- if isinstance(source, Path):
354
- source_file = source
355
- elif isinstance(source, str):
356
- # create a temp file and write the source to it
357
- source_file = write_to_temp_file(source)
358
- elif isinstance(source, StringIO):
359
- source_file = write_to_temp_file(source.getvalue())
360
- else:
361
- raise TypeError(f"source must be str, Path, or file-like object, not {type(source)}")
362
-
363
- _target = None
364
- if isinstance(target, Path):
365
- if target.is_dir():
366
- target = target / source.name if isinstance(source, Path) else target / "minified.mpy"
367
- _target = target.with_suffix(".mpy")
368
- else:
369
- # target must be a Path object
370
- _target = get_temp_file(suffix=".mpy")
371
- result = pipx_mpy_cross(version, source_file, _target)
372
- if result.stderr and "No matching distribution found for mpy-cross==" in result.stderr:
373
- log.warning(f"mpy-cross=={version} not found, using default version.")
374
- result = pipx_mpy_cross(V_PREVIEW, source_file, _target)
375
-
376
- if result.returncode == 0:
377
- log.debug(f"mpy-cross compiled to : {_target.name}")
378
- else:
379
- log.error(f"mpy-cross failed to compile:{result.returncode} \n{result.stderr}")
380
-
381
- if isinstance(target, BytesIO):
382
- # copy the byte contents of the temp file to the target file-like object
383
- with _target.open("rb") as f:
384
- target.write(f.read())
385
- # _target.unlink()
386
-
387
- return result.returncode
388
-
389
-
390
- def pipx_mpy_cross(version: str, source_file, _target):
391
- """Run mpy-cross using pipx"""
392
-
393
- log.info(f"Compiling with mpy-cross version: {version}")
394
- if version in SET_PREVIEW:
395
- version = ""
396
- if version:
397
- version = "==" + version
398
-
399
- cmd = ["pipx", "run", f"mpy-cross{version}"] if version else ["pipx", "run", "mpy-cross"]
400
- # Add params
401
- cmd += ["-O2", str(source_file), "-o", str(_target), "-s", "createstubs.py"]
402
- log.trace(" ".join(cmd))
403
- result = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8") # Specify the encoding
404
- return result
405
-
406
-
407
- def write_to_temp_file(source: str):
408
- """Writes a string to a temp file and returns the Path object"""
409
- _, temp_file = tempfile.mkstemp(suffix=".py", prefix="mpy_cross_")
410
- temp_file = Path(temp_file)
411
- temp_file.write_text(source)
412
- return temp_file
413
-
414
-
415
- def get_temp_file(prefix: str = "mpy_cross_", suffix: str = ".py"):
416
- """Get temp file and returns the Path object"""
417
- _, temp_file = tempfile.mkstemp(prefix=prefix, suffix=suffix)
418
- temp_file = Path(temp_file)
419
- return temp_file
1
+ """
2
+ Processing for createstubs.py
3
+ Minimizes and cross-compiles a MicroPyton file.
4
+ """
5
+
6
+ import itertools
7
+ import subprocess
8
+ import tempfile
9
+ from contextlib import ExitStack
10
+ from io import BytesIO, IOBase, StringIO, TextIOWrapper
11
+ from pathlib import Path
12
+ from typing import List, Tuple, Union
13
+
14
+ try:
15
+ import python_minifier
16
+ except ImportError:
17
+ python_minifier = None
18
+
19
+ from mpflash.logger import log
20
+
21
+ from mpflash.versions import SET_PREVIEW, V_PREVIEW
22
+
23
+ # Type Aliases for minify
24
+ StubSource = Union[Path, str, StringIO, TextIOWrapper]
25
+ XCompileDest = Union[Path, BytesIO]
26
+ LineEdits = List[Tuple[str, str]]
27
+
28
+
29
+ def get_whitespace_context(content: List[str], index: int):
30
+ """Get whitespace count of lines surrounding index"""
31
+ if not content:
32
+ raise ValueError()
33
+ if index < 0 or index > len(content):
34
+ raise IndexError()
35
+ if len(content) == 1:
36
+ return [0, 0]
37
+
38
+ def count_ws(line: str):
39
+ return sum(1 for _ in itertools.takewhile(str.isspace, line))
40
+
41
+ lines = content[max(0, index) : min(index + 2, len(content))]
42
+ context = [count_ws(l) for l in lines]
43
+ if len(context) < 2:
44
+ context.append(context[0])
45
+ return context
46
+
47
+
48
+ def edit_lines(content: str, edits: LineEdits, diff: bool = False):
49
+ # sourcery skip: no-long-functions
50
+ """Edit string by list of edits
51
+
52
+ Args:
53
+ content (str): content to edit
54
+ edits ([(str, str)]): List of edits to make.
55
+ The first string in the tuple representsthe type of edit to make, can be either:
56
+ - comment - comment text out (removed on minify)
57
+ - rprint - replace text with print
58
+ - rpass - replace text with pass
59
+ The second string is the matching text to replace
60
+ diff (bool, optional): Prints diff of each edit.
61
+ Defaults to False.
62
+
63
+ Returns:
64
+ str: edited string
65
+ """
66
+
67
+ def comment(l: str, x: str):
68
+ """Comment out line, so it will be removed on minify"""
69
+ return l.replace(x, f"# {x}")
70
+
71
+ def rprint(l: str, x: str): # type: ignore
72
+ """Replace (logging) with print"""
73
+ split = l.split("(")
74
+ if len(split) > 1:
75
+ return l.replace(split[0].strip(), "print")
76
+ return l.replace(x, "print")
77
+
78
+ def keepprint(l: str, x: str): # type: ignore
79
+ """Replace 'print' with 'print '"""
80
+ return l.replace("print(", "print (")
81
+
82
+ def rpass(l: str, x: str): # type: ignore
83
+ """Replace with pass"""
84
+ return l.replace(x, "pass")
85
+
86
+ def handle_multiline(content: List[str], index: int):
87
+ """Handles edits that require multiline comments
88
+
89
+ Example:
90
+ self.log.debug("info: {} {}".format(
91
+ 1,
92
+ 2
93
+ ))
94
+ Here, only commenting out the first self.log line will raise
95
+ an error. So this function returns all lines that need to
96
+ be commented out instead.
97
+
98
+ It also checks for situations such as this:
99
+ if condition:
100
+ self.log.debug('message')
101
+
102
+ Here, since the only functionality of the conditional is the call log,
103
+ both lines would be returned to comment out.
104
+
105
+ """
106
+ line = content[index]
107
+ open_cnt = line.count("(")
108
+ close_cnt = line.count(")")
109
+ ahead_index = 1
110
+ look_ahead = 0
111
+ while open_cnt != close_cnt: # pragma: no cover
112
+ look_ahead = l_index + ahead_index
113
+ ahead_index += 1
114
+ next_l = content[look_ahead]
115
+ open_cnt += next_l.count("(")
116
+ close_cnt += next_l.count(")")
117
+ if ahead_index > 1: # pragma: no cover
118
+ return range(index, look_ahead + 1)
119
+ prev = content[index - 1]
120
+ line_ws, post_ws = get_whitespace_context(content, index)
121
+ prev_words = prev.strip().strip(":").split()
122
+ check = any(
123
+ t
124
+ in (
125
+ "if",
126
+ "else",
127
+ )
128
+ for t in prev_words
129
+ )
130
+ return range(index - 1, index + 1) if check and line_ws != post_ws else None
131
+
132
+ def handle_try_except(content: List[str], index: int) -> bool:
133
+ """Checks if line at index is in try/except block
134
+
135
+ Handles situations like this:
136
+ try:
137
+ something()
138
+ except:
139
+ self.log.debug('some message')
140
+
141
+ Simply removing the self.log call would create a syntax error,
142
+ which is what this function checks for.
143
+
144
+ """
145
+ prev = content[index - 1]
146
+ line_ws, post_ws = get_whitespace_context(content, index)
147
+ return "except" in prev and line_ws != post_ws
148
+
149
+ lines = []
150
+ multilines = set()
151
+ content_l = content.splitlines(keepends=True)
152
+ for line in content_l:
153
+ _line = line
154
+ for edit_action, match_text in edits:
155
+ if match_text in line:
156
+ if edit_action == "comment":
157
+ l_index = content_l.index(line)
158
+ # Check if edit spans multiple lines
159
+ mline = handle_multiline(content_l, l_index)
160
+ if mline:
161
+ multilines.update(mline)
162
+ break
163
+ # Check if line is only statement in try/except
164
+ if handle_try_except(content_l, l_index):
165
+ edit_action = "rpass"
166
+ match_text = line.strip()
167
+ func = eval(edit_action) # pylint: disable= eval-used
168
+ line = func(line, match_text)
169
+ if line != _line:
170
+ if diff:
171
+ print(f"\n- {_line.strip()}")
172
+ print(f"+ {line.strip()}")
173
+ break
174
+ lines.append(line)
175
+ for line_num in multilines:
176
+ # Go back and comment out multilines
177
+ line = lines[line_num]
178
+ lines[line_num] = comment(line, line.strip())
179
+ stripped = "".join(lines)
180
+ return stripped
181
+
182
+
183
+ def minify_script(source_script: StubSource, keep_report: bool = True, diff: bool = False) -> str:
184
+ """
185
+ Minifies createstubs.py and variants
186
+
187
+ Args:
188
+ source_script:
189
+ - (str): content to edit
190
+ - (Path): path to file to edit
191
+ - (IOBase): file-like object to edit
192
+ keep_report (bool, optional): Keeps single report line in createstubs
193
+ Defaults to True.
194
+ diff (bool, optional): Print diff from edits. Defaults to False.
195
+
196
+ Returns:
197
+ str: minified source text
198
+ """
199
+
200
+ source_content = ""
201
+ if isinstance(source_script, Path):
202
+ source_content = source_script.read_text(encoding="utf-8")
203
+ elif isinstance(source_script, (StringIO, TextIOWrapper)):
204
+ source_content = "".join(source_script.readlines())
205
+ elif isinstance(source_script, str): # type: ignore
206
+ source_content = source_script
207
+ else:
208
+ raise TypeError(
209
+ f"source_script must be str, Path, or file-like object, not {type(source_script)}"
210
+ )
211
+
212
+ if not source_content:
213
+ raise ValueError("No source content")
214
+ len_1 = len(source_content)
215
+
216
+ if 0:
217
+ min_source = reduce_log_print(keep_report, diff, source_content)
218
+ else:
219
+ min_source = source_content
220
+ len_2 = len(min_source)
221
+
222
+ if not python_minifier:
223
+ log.warning("python_minifier not found, skipping minification")
224
+ else:
225
+ # use python_minifier to minify the source if it is successfully imported
226
+ min_source = python_minifier.minify(
227
+ min_source,
228
+ filename=getattr(source_script, "name", None),
229
+ combine_imports=True,
230
+ remove_literal_statements=True, # no Docstrings
231
+ remove_annotations=True, # not used runtime anyways
232
+ hoist_literals=True, # remove redundant strings
233
+ rename_locals=True, # short names save memory
234
+ preserve_locals=["stubber", "path"], # names to keep
235
+ rename_globals=True, # short names save memory
236
+ # keep these globals to allow testing/mocking to work against the minified not compiled version
237
+ preserve_globals=[
238
+ "main",
239
+ "Stubber",
240
+ "read_path",
241
+ "get_root",
242
+ "_info",
243
+ "os",
244
+ "sys",
245
+ "__version__",
246
+ ],
247
+ # remove_pass=True, # no dead code
248
+ # convert_posargs_to_args=True, # Does not save any space
249
+ )
250
+ len_3 = len(min_source)
251
+ # if 1:
252
+ # # write to temp file for debugging
253
+ # with open("tmp_minified.py", "w+") as f:
254
+ # f.write(min_source)
255
+
256
+ log.info(f"Original length : {len_1}")
257
+ log.info(f"Reduced length : {len_2}")
258
+ log.info(f"Minified length : {len_3}")
259
+ log.info(f"Reduced by : {len_1-len_3} ")
260
+ return min_source
261
+
262
+
263
+ def reduce_log_print(keep_report, diff, source_content):
264
+ edits: LineEdits = [
265
+ ("keepprint", "print('Debug: "),
266
+ ("keepprint", "print('DEBUG: "),
267
+ ("keepprint", 'print("Debug: '),
268
+ ("keepprint", 'print("DEBUG: '),
269
+ ("comment", "print("),
270
+ ("comment", "import logging"),
271
+ # report keepers may be inserted here
272
+ # do report errors
273
+ ("rprint", "self.log.error"),
274
+ ("rprint", "log.error"),
275
+ ]
276
+ if keep_report:
277
+ # insert report keepers after the comment modifiers
278
+ edits += [
279
+ # keepers
280
+ ("rprint", 'self.log.info("Stub module: '),
281
+ ("rprint", 'self.log.warning("{}Skip module:'),
282
+ ("rprint", 'self.log.info("Clean/remove files in folder:'),
283
+ ("rprint", 'self.log.info("Created stubs for'),
284
+ ("rprint", 'self.log.info("Family: '),
285
+ ("rprint", 'self.log.info("Version: '),
286
+ ("rprint", 'self.log.info("Port: '),
287
+ ("rprint", 'self.log.info("Board: '),
288
+ # all others
289
+ ("comment", "self.log"),
290
+ ("comment", "_log."),
291
+ ("comment", "_log ="),
292
+ ]
293
+ else:
294
+ edits += [
295
+ # remove first full
296
+ ("comment", "self.log ="),
297
+ ("comment", "self.log("),
298
+ ("comment", "self.log.debug"),
299
+ ("comment", "self.log.info"),
300
+ ("comment", "self.log.warning"),
301
+ # then short versions
302
+ ("comment", "_log ="),
303
+ ("comment", "_log.debug"),
304
+ ("comment", "_log.info"),
305
+ ("comment", "_log.warning"),
306
+ ] + edits
307
+
308
+ content = edit_lines(source_content, edits, diff=diff)
309
+ return content
310
+
311
+
312
+ def minify(
313
+ source: StubSource,
314
+ target: StubSource,
315
+ keep_report: bool = True,
316
+ diff: bool = False,
317
+ ):
318
+ """Minifies and compiles a script"""
319
+ source_buf = None
320
+ target_buf = None
321
+
322
+ with ExitStack() as stack:
323
+ if isinstance(source, Path):
324
+ source_buf = stack.enter_context(source.open("r"))
325
+ elif isinstance(source, (StringIO, str)):
326
+ # different types of file-like objects are both acepted by minify_script
327
+ source_buf = source
328
+ else:
329
+ raise TypeError(f"source must be str, Path, or file-like object, not {type(source)}")
330
+
331
+ if isinstance(target, Path):
332
+ if target.is_dir():
333
+ if isinstance(source, Path):
334
+ target = target / source.name
335
+ else:
336
+ target = target / "minified.py" # or raise error?
337
+ target_buf = stack.enter_context(target.open("w+"))
338
+ elif isinstance(target, IOBase): # type: ignore
339
+ target_buf = target
340
+ try:
341
+ minified = minify_script(source_script=source_buf, keep_report=keep_report, diff=diff)
342
+ target_buf.write(minified)
343
+ except Exception as e: # pragma: no cover
344
+ log.exception(e)
345
+ return 0
346
+
347
+
348
+ def cross_compile(
349
+ source: StubSource,
350
+ target: XCompileDest,
351
+ version: str = "",
352
+ ): # sourcery skip: assign-if-exp
353
+ """Runs mpy-cross on a (minified) script"""
354
+ # Sources can be a file, a string, or a file-like object
355
+ if isinstance(source, Path):
356
+ source_file = source
357
+ elif isinstance(source, str):
358
+ # create a temp file and write the source to it
359
+ source_file = write_to_temp_file(source)
360
+ elif isinstance(source, StringIO):
361
+ source_file = write_to_temp_file(source.getvalue())
362
+ else:
363
+ raise TypeError(f"source must be str, Path, or file-like object, not {type(source)}")
364
+
365
+ _target = None
366
+ if isinstance(target, Path):
367
+ if target.is_dir():
368
+ target = target / source.name if isinstance(source, Path) else target / "minified.mpy"
369
+ _target = target.with_suffix(".mpy")
370
+ else:
371
+ # target must be a Path object
372
+ _target = get_temp_file(suffix=".mpy")
373
+ result = pipx_mpy_cross(version, source_file, _target)
374
+ if result.stderr and "No matching distribution found for mpy-cross==" in result.stderr:
375
+ log.warning(f"mpy-cross=={version} not found, using default version.")
376
+ result = pipx_mpy_cross(V_PREVIEW, source_file, _target)
377
+
378
+ if result.returncode == 0:
379
+ log.debug(f"mpy-cross compiled to : {_target.name}")
380
+ else:
381
+ log.error(f"mpy-cross failed to compile:{result.returncode} \n{result.stderr}")
382
+
383
+ if isinstance(target, BytesIO):
384
+ # copy the byte contents of the temp file to the target file-like object
385
+ with _target.open("rb") as f:
386
+ target.write(f.read())
387
+ # _target.unlink()
388
+
389
+ return result.returncode
390
+
391
+
392
+ def pipx_mpy_cross(version: str, source_file, _target):
393
+ """Run mpy-cross using pipx"""
394
+
395
+ log.info(f"Compiling with mpy-cross version: {version}")
396
+ if version in SET_PREVIEW:
397
+ version = ""
398
+ if version:
399
+ version = "==" + version
400
+
401
+ cmd = ["pipx", "run", f"mpy-cross{version}"] if version else ["pipx", "run", "mpy-cross"]
402
+ # Add params
403
+ cmd += ["-O2", str(source_file), "-o", str(_target), "-s", "createstubs.py"]
404
+ log.trace(" ".join(cmd))
405
+ result = subprocess.run(
406
+ cmd, capture_output=True, text=True, encoding="utf-8"
407
+ ) # Specify the encoding
408
+ return result
409
+
410
+
411
+ def write_to_temp_file(source: str):
412
+ """Writes a string to a temp file and returns the Path object"""
413
+ _, temp_file = tempfile.mkstemp(suffix=".py", prefix="mpy_cross_")
414
+ temp_file = Path(temp_file)
415
+ temp_file.write_text(source)
416
+ return temp_file
417
+
418
+
419
+ def get_temp_file(prefix: str = "mpy_cross_", suffix: str = ".py"):
420
+ """Get temp file and returns the Path object"""
421
+ _, temp_file = tempfile.mkstemp(prefix=prefix, suffix=suffix)
422
+ temp_file = Path(temp_file)
423
+ return temp_file