micropython-stubber 1.20.5__py3-none-any.whl → 1.23.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. {micropython_stubber-1.20.5.dist-info → micropython_stubber-1.23.0.dist-info}/LICENSE +30 -30
  2. {micropython_stubber-1.20.5.dist-info → micropython_stubber-1.23.0.dist-info}/METADATA +1 -1
  3. micropython_stubber-1.23.0.dist-info/RECORD +159 -0
  4. mpflash/README.md +184 -184
  5. mpflash/libusb_flash.ipynb +203 -203
  6. mpflash/mpflash/add_firmware.py +98 -98
  7. mpflash/mpflash/ask_input.py +236 -236
  8. mpflash/mpflash/bootloader/__init__.py +37 -36
  9. mpflash/mpflash/bootloader/manual.py +102 -102
  10. mpflash/mpflash/bootloader/micropython.py +10 -10
  11. mpflash/mpflash/bootloader/touch1200.py +45 -45
  12. mpflash/mpflash/cli_download.py +129 -129
  13. mpflash/mpflash/cli_flash.py +219 -219
  14. mpflash/mpflash/cli_group.py +98 -98
  15. mpflash/mpflash/cli_list.py +81 -81
  16. mpflash/mpflash/cli_main.py +41 -41
  17. mpflash/mpflash/common.py +164 -164
  18. mpflash/mpflash/config.py +43 -47
  19. mpflash/mpflash/connected.py +74 -74
  20. mpflash/mpflash/download.py +360 -360
  21. mpflash/mpflash/downloaded.py +130 -129
  22. mpflash/mpflash/errors.py +9 -9
  23. mpflash/mpflash/flash.py +55 -52
  24. mpflash/mpflash/flash_esp.py +59 -59
  25. mpflash/mpflash/flash_stm32.py +18 -24
  26. mpflash/mpflash/flash_stm32_cube.py +111 -111
  27. mpflash/mpflash/flash_stm32_dfu.py +104 -101
  28. mpflash/mpflash/flash_uf2.py +89 -67
  29. mpflash/mpflash/flash_uf2_boardid.py +15 -15
  30. mpflash/mpflash/flash_uf2_linux.py +129 -123
  31. mpflash/mpflash/flash_uf2_macos.py +37 -34
  32. mpflash/mpflash/flash_uf2_windows.py +38 -34
  33. mpflash/mpflash/list.py +89 -89
  34. mpflash/mpflash/logger.py +41 -41
  35. mpflash/mpflash/mpboard_id/__init__.py +93 -93
  36. mpflash/mpflash/mpboard_id/add_boards.py +255 -255
  37. mpflash/mpflash/mpboard_id/board.py +37 -37
  38. mpflash/mpflash/mpboard_id/board_id.py +86 -86
  39. mpflash/mpflash/mpboard_id/store.py +43 -43
  40. mpflash/mpflash/mpremoteboard/__init__.py +226 -221
  41. mpflash/mpflash/mpremoteboard/mpy_fw_info.py +141 -141
  42. mpflash/mpflash/mpremoteboard/runner.py +140 -140
  43. mpflash/mpflash/uf2disk.py +12 -12
  44. mpflash/mpflash/vendor/basicgit.py +288 -288
  45. mpflash/mpflash/vendor/click_aliases.py +91 -91
  46. mpflash/mpflash/vendor/dfu.py +165 -165
  47. mpflash/mpflash/vendor/pydfu.py +605 -605
  48. mpflash/mpflash/vendor/readme.md +2 -2
  49. mpflash/mpflash/vendor/versions.py +119 -117
  50. mpflash/mpflash/worklist.py +171 -170
  51. mpflash/poetry.lock +1588 -1588
  52. mpflash/pyproject.toml +64 -60
  53. mpflash/stm32_udev_rules.md +62 -62
  54. stubber/__init__.py +3 -3
  55. stubber/basicgit.py +294 -288
  56. stubber/board/board_info.csv +193 -193
  57. stubber/board/boot.py +34 -34
  58. stubber/board/createstubs.py +986 -986
  59. stubber/board/createstubs_db.py +825 -825
  60. stubber/board/createstubs_db_min.py +331 -331
  61. stubber/board/createstubs_db_mpy.mpy +0 -0
  62. stubber/board/createstubs_lvgl.py +741 -741
  63. stubber/board/createstubs_lvgl_min.py +741 -741
  64. stubber/board/createstubs_mem.py +766 -766
  65. stubber/board/createstubs_mem_min.py +306 -306
  66. stubber/board/createstubs_mem_mpy.mpy +0 -0
  67. stubber/board/createstubs_min.py +294 -294
  68. stubber/board/createstubs_mpy.mpy +0 -0
  69. stubber/board/fw_info.py +141 -141
  70. stubber/board/info.py +183 -183
  71. stubber/board/main.py +19 -19
  72. stubber/board/modulelist.txt +247 -247
  73. stubber/board/pyrightconfig.json +34 -34
  74. stubber/bulk/mcu_stubber.py +454 -454
  75. stubber/codemod/_partials/__init__.py +48 -48
  76. stubber/codemod/_partials/db_main.py +147 -147
  77. stubber/codemod/_partials/lvgl_main.py +77 -77
  78. stubber/codemod/_partials/modules_reader.py +80 -80
  79. stubber/codemod/add_comment.py +53 -53
  80. stubber/codemod/add_method.py +65 -65
  81. stubber/codemod/board.py +317 -317
  82. stubber/codemod/enrich.py +145 -145
  83. stubber/codemod/merge_docstub.py +284 -284
  84. stubber/codemod/modify_list.py +54 -54
  85. stubber/codemod/utils.py +57 -57
  86. stubber/commands/build_cmd.py +94 -94
  87. stubber/commands/cli.py +55 -51
  88. stubber/commands/clone_cmd.py +77 -66
  89. stubber/commands/config_cmd.py +29 -29
  90. stubber/commands/enrich_folder_cmd.py +71 -70
  91. stubber/commands/get_core_cmd.py +71 -69
  92. stubber/commands/get_docstubs_cmd.py +89 -87
  93. stubber/commands/get_frozen_cmd.py +114 -112
  94. stubber/commands/get_mcu_cmd.py +61 -56
  95. stubber/commands/merge_cmd.py +67 -66
  96. stubber/commands/publish_cmd.py +119 -119
  97. stubber/commands/stub_cmd.py +31 -30
  98. stubber/commands/switch_cmd.py +62 -54
  99. stubber/commands/variants_cmd.py +49 -48
  100. stubber/cst_transformer.py +178 -178
  101. stubber/data/board_info.csv +193 -193
  102. stubber/data/board_info.json +1729 -1729
  103. stubber/data/micropython_tags.csv +15 -15
  104. stubber/data/requirements-core-micropython.txt +38 -38
  105. stubber/data/requirements-core-pycopy.txt +39 -39
  106. stubber/downloader.py +36 -36
  107. stubber/freeze/common.py +68 -68
  108. stubber/freeze/freeze_folder.py +69 -69
  109. stubber/freeze/freeze_manifest_2.py +113 -113
  110. stubber/freeze/get_frozen.py +127 -127
  111. stubber/get_cpython.py +101 -101
  112. stubber/get_lobo.py +59 -59
  113. stubber/minify.py +418 -418
  114. stubber/publish/bump.py +86 -86
  115. stubber/publish/candidates.py +262 -262
  116. stubber/publish/database.py +18 -18
  117. stubber/publish/defaults.py +45 -45
  118. stubber/publish/enums.py +24 -24
  119. stubber/publish/helpers.py +29 -29
  120. stubber/publish/merge_docstubs.py +130 -130
  121. stubber/publish/missing_class_methods.py +49 -49
  122. stubber/publish/package.py +146 -146
  123. stubber/publish/pathnames.py +51 -51
  124. stubber/publish/publish.py +120 -120
  125. stubber/publish/pypi.py +38 -38
  126. stubber/publish/stubpackage.py +1029 -1029
  127. stubber/rst/__init__.py +9 -9
  128. stubber/rst/classsort.py +77 -77
  129. stubber/rst/lookup.py +530 -530
  130. stubber/rst/output_dict.py +401 -401
  131. stubber/rst/reader.py +822 -822
  132. stubber/rst/report_return.py +69 -69
  133. stubber/rst/rst_utils.py +540 -540
  134. stubber/stubber.py +38 -38
  135. stubber/stubs_from_docs.py +90 -90
  136. stubber/tools/manifestfile.py +655 -610
  137. stubber/tools/readme.md +7 -6
  138. stubber/update_fallback.py +117 -117
  139. stubber/update_module_list.py +123 -123
  140. stubber/utils/__init__.py +5 -5
  141. stubber/utils/config.py +127 -127
  142. stubber/utils/makeversionhdr.py +54 -54
  143. stubber/utils/manifest.py +92 -92
  144. stubber/utils/post.py +79 -79
  145. stubber/utils/repos.py +157 -154
  146. stubber/utils/stubmaker.py +139 -139
  147. stubber/utils/typed_config_toml.py +77 -77
  148. stubber/utils/versions.py +128 -120
  149. stubber/variants.py +106 -106
  150. micropython_stubber-1.20.5.dist-info/RECORD +0 -159
  151. {micropython_stubber-1.20.5.dist-info → micropython_stubber-1.23.0.dist-info}/WHEEL +0 -0
  152. {micropython_stubber-1.20.5.dist-info → micropython_stubber-1.23.0.dist-info}/entry_points.txt +0 -0
stubber/minify.py CHANGED
@@ -1,418 +1,418 @@
1
- """
2
- Processing for createstubs.py
3
- Minimizes and cross-compiles a MicroPyton file.
4
- """
5
- import itertools
6
- import subprocess
7
- import tempfile
8
- from contextlib import ExitStack
9
- from io import BytesIO, IOBase, StringIO, TextIOWrapper
10
- from pathlib import Path
11
- from typing import List, Tuple, Union
12
-
13
- try:
14
- import python_minifier
15
- except ImportError:
16
- python_minifier = None
17
-
18
- from loguru import logger as log
19
-
20
- from stubber.utils.versions import SET_PREVIEW, V_PREVIEW
21
-
22
- # Type Aliases for minify
23
- StubSource = Union[Path, str, StringIO, TextIOWrapper]
24
- XCompileDest = Union[Path, BytesIO]
25
- LineEdits = List[Tuple[str, str]]
26
-
27
-
28
- def get_whitespace_context(content: List[str], index: int):
29
- """Get whitespace count of lines surrounding index"""
30
- if not content:
31
- raise ValueError()
32
- if index < 0 or index > len(content):
33
- raise IndexError()
34
- if len(content) == 1:
35
- return [0, 0]
36
-
37
- def count_ws(line: str):
38
- return sum(1 for _ in itertools.takewhile(str.isspace, line))
39
-
40
- lines = content[max(0, index) : min(index + 2, len(content))]
41
- context = [count_ws(l) for l in lines]
42
- if len(context) < 2:
43
- context.append(context[0])
44
- return context
45
-
46
-
47
- def edit_lines(content: str, edits: LineEdits, diff: bool = False):
48
- # sourcery skip: no-long-functions
49
- """Edit string by list of edits
50
-
51
- Args:
52
- content (str): content to edit
53
- edits ([(str, str)]): List of edits to make.
54
- The first string in the tuple representsthe type of edit to make, can be either:
55
- - comment - comment text out (removed on minify)
56
- - rprint - replace text with print
57
- - rpass - replace text with pass
58
- The second string is the matching text to replace
59
- diff (bool, optional): Prints diff of each edit.
60
- Defaults to False.
61
-
62
- Returns:
63
- str: edited string
64
- """
65
-
66
- def comment(l: str, x: str):
67
- """Comment out line, so it will be removed on minify"""
68
- return l.replace(x, f"# {x}")
69
-
70
- def rprint(l: str, x: str): # type: ignore
71
- """Replace (logging) with print"""
72
- split = l.split("(")
73
- if len(split) > 1:
74
- return l.replace(split[0].strip(), "print")
75
- return l.replace(x, "print")
76
-
77
- def keepprint(l: str, x: str): # type: ignore
78
- """Replace 'print' with 'print '"""
79
- return l.replace("print(", "print (")
80
-
81
- def rpass(l: str, x: str): # type: ignore
82
- """Replace with pass"""
83
- return l.replace(x, "pass")
84
-
85
- def handle_multiline(content: List[str], index: int):
86
- """Handles edits that require multiline comments
87
-
88
- Example:
89
- self.log.debug("info: {} {}".format(
90
- 1,
91
- 2
92
- ))
93
- Here, only commenting out the first self.log line will raise
94
- an error. So this function returns all lines that need to
95
- be commented out instead.
96
-
97
- It also checks for situations such as this:
98
- if condition:
99
- self.log.debug('message')
100
-
101
- Here, since the only functionality of the conditional is the call log,
102
- both lines would be returned to comment out.
103
-
104
- """
105
- line = content[index]
106
- open_cnt = line.count("(")
107
- close_cnt = line.count(")")
108
- ahead_index = 1
109
- look_ahead = 0
110
- while open_cnt != close_cnt: # pragma: no cover
111
- look_ahead = l_index + ahead_index
112
- ahead_index += 1
113
- next_l = content[look_ahead]
114
- open_cnt += next_l.count("(")
115
- close_cnt += next_l.count(")")
116
- if ahead_index > 1: # pragma: no cover
117
- return range(index, look_ahead + 1)
118
- prev = content[index - 1]
119
- line_ws, post_ws = get_whitespace_context(content, index)
120
- prev_words = prev.strip().strip(":").split()
121
- check = any(
122
- t
123
- in (
124
- "if",
125
- "else",
126
- )
127
- for t in prev_words
128
- )
129
- return range(index - 1, index + 1) if check and line_ws != post_ws else None
130
-
131
- def handle_try_except(content: List[str], index: int) -> bool:
132
- """Checks if line at index is in try/except block
133
-
134
- Handles situations like this:
135
- try:
136
- something()
137
- except:
138
- self.log.debug('some message')
139
-
140
- Simply removing the self.log call would create a syntax error,
141
- which is what this function checks for.
142
-
143
- """
144
- prev = content[index - 1]
145
- line_ws, post_ws = get_whitespace_context(content, index)
146
- return "except" in prev and line_ws != post_ws
147
-
148
- lines = []
149
- multilines = set()
150
- content_l = content.splitlines(keepends=True)
151
- for line in content_l:
152
- _line = line
153
- for edit_action, match_text in edits:
154
- if match_text in line:
155
- if edit_action == "comment":
156
- l_index = content_l.index(line)
157
- # Check if edit spans multiple lines
158
- mline = handle_multiline(content_l, l_index)
159
- if mline:
160
- multilines.update(mline)
161
- break
162
- # Check if line is only statement in try/except
163
- if handle_try_except(content_l, l_index):
164
- edit_action = "rpass"
165
- match_text = line.strip()
166
- func = eval(edit_action) # pylint: disable= eval-used
167
- line = func(line, match_text)
168
- if line != _line:
169
- if diff:
170
- print(f"\n- {_line.strip()}")
171
- print(f"+ {line.strip()}")
172
- break
173
- lines.append(line)
174
- for line_num in multilines:
175
- # Go back and comment out multilines
176
- line = lines[line_num]
177
- lines[line_num] = comment(line, line.strip())
178
- stripped = "".join(lines)
179
- return stripped
180
-
181
-
182
- def minify_script(source_script: StubSource, keep_report: bool = True, diff: bool = False) -> str:
183
- """
184
- Minifies createstubs.py and variants
185
-
186
- Args:
187
- source_script:
188
- - (str): content to edit
189
- - (Path): path to file to edit
190
- - (IOBase): file-like object to edit
191
- keep_report (bool, optional): Keeps single report line in createstubs
192
- Defaults to True.
193
- diff (bool, optional): Print diff from edits. Defaults to False.
194
-
195
- Returns:
196
- str: minified source text
197
- """
198
-
199
- source_content = ""
200
- if isinstance(source_script, Path):
201
- source_content = source_script.read_text(encoding="utf-8")
202
- elif isinstance(source_script, (StringIO, TextIOWrapper)):
203
- source_content = "".join(source_script.readlines())
204
- elif isinstance(source_script, str): # type: ignore
205
- source_content = source_script
206
- else:
207
- raise TypeError(f"source_script must be str, Path, or file-like object, not {type(source_script)}")
208
-
209
- if not source_content:
210
- raise ValueError("No source content")
211
- len_1 = len(source_content)
212
-
213
- if 0:
214
- min_source = reduce_log_print(keep_report, diff, source_content)
215
- else:
216
- min_source = source_content
217
- len_2 = len(min_source)
218
-
219
- if not python_minifier:
220
- log.warning("python_minifier not found, skipping minification")
221
- else:
222
- # use python_minifier to minify the source if it is successfully imported
223
- min_source = python_minifier.minify(
224
- min_source,
225
- filename=getattr(source_script, "name", None),
226
- combine_imports=True,
227
- remove_literal_statements=True, # no Docstrings
228
- remove_annotations=True, # not used runtime anyways
229
- hoist_literals=True, # remove redundant strings
230
- rename_locals=True, # short names save memory
231
- preserve_locals=["stubber", "path"], # names to keep
232
- rename_globals=True, # short names save memory
233
- # keep these globals to allow testing/mocking to work against the minified not compiled version
234
- preserve_globals=[
235
- "main",
236
- "Stubber",
237
- "read_path",
238
- "get_root",
239
- "_info",
240
- "os",
241
- "sys",
242
- "__version__",
243
- ],
244
- # remove_pass=True, # no dead code
245
- # convert_posargs_to_args=True, # Does not save any space
246
- )
247
- len_3 = len(min_source)
248
- if 1:
249
- # write to temp file for debugging
250
- with open("tmp_minified.py", "w+") as f:
251
- f.write(min_source)
252
-
253
- log.info(f"Original length : {len_1}")
254
- log.info(f"Reduced length : {len_2}")
255
- log.info(f"Minified length : {len_3}")
256
- log.info(f"Reduced by : {len_1-len_3} ")
257
- return min_source
258
-
259
-
260
- def reduce_log_print(keep_report, diff, source_content):
261
- edits: LineEdits = [
262
- ("keepprint", "print('Debug: "),
263
- ("keepprint", "print('DEBUG: "),
264
- ("keepprint", 'print("Debug: '),
265
- ("keepprint", 'print("DEBUG: '),
266
- ("comment", "print("),
267
- ("comment", "import logging"),
268
- # report keepers may be inserted here
269
- # do report errors
270
- ("rprint", "self.log.error"),
271
- ("rprint", "log.error"),
272
- ]
273
- if keep_report:
274
- # insert report keepers after the comment modifiers
275
- edits += [
276
- # keepers
277
- ("rprint", 'self.log.info("Stub module: '),
278
- ("rprint", 'self.log.warning("{}Skip module:'),
279
- ("rprint", 'self.log.info("Clean/remove files in folder:'),
280
- ("rprint", 'self.log.info("Created stubs for'),
281
- ("rprint", 'self.log.info("Family: '),
282
- ("rprint", 'self.log.info("Version: '),
283
- ("rprint", 'self.log.info("Port: '),
284
- ("rprint", 'self.log.info("Board: '),
285
- # all others
286
- ("comment", "self.log"),
287
- ("comment", "_log."),
288
- ("comment", "_log ="),
289
- ]
290
- else:
291
- edits += [
292
- # remove first full
293
- ("comment", "self.log ="),
294
- ("comment", "self.log("),
295
- ("comment", "self.log.debug"),
296
- ("comment", "self.log.info"),
297
- ("comment", "self.log.warning"),
298
- # then short versions
299
- ("comment", "_log ="),
300
- ("comment", "_log.debug"),
301
- ("comment", "_log.info"),
302
- ("comment", "_log.warning"),
303
- ] + edits
304
-
305
- content = edit_lines(source_content, edits, diff=diff)
306
- return content
307
-
308
-
309
- def minify(
310
- source: StubSource,
311
- target: StubSource,
312
- keep_report: bool = True,
313
- diff: bool = False,
314
- ):
315
- """Minifies and compiles a script"""
316
- source_buf = None
317
- target_buf = None
318
-
319
- with ExitStack() as stack:
320
- if isinstance(source, Path):
321
- source_buf = stack.enter_context(source.open("r"))
322
- elif isinstance(source, (StringIO, str)):
323
- # different types of file-like objects are both acepted by minify_script
324
- source_buf = source
325
- else:
326
- raise TypeError(f"source must be str, Path, or file-like object, not {type(source)}")
327
-
328
- if isinstance(target, Path):
329
- if target.is_dir():
330
- if isinstance(source, Path):
331
- target = target / source.name
332
- else:
333
- target = target / "minified.py" # or raise error?
334
- target_buf = stack.enter_context(target.open("w+"))
335
- elif isinstance(target, IOBase): # type: ignore
336
- target_buf = target
337
- try:
338
- minified = minify_script(source_script=source_buf, keep_report=keep_report, diff=diff)
339
- target_buf.write(minified)
340
- except Exception as e: # pragma: no cover
341
- log.exception(e)
342
- return 0
343
-
344
-
345
- def cross_compile(
346
- source: StubSource,
347
- target: XCompileDest,
348
- version: str = "",
349
- ): # sourcery skip: assign-if-exp
350
- """Runs mpy-cross on a (minified) script"""
351
- # Sources can be a file, a string, or a file-like object
352
- if isinstance(source, Path):
353
- source_file = source
354
- elif isinstance(source, str):
355
- # create a temp file and write the source to it
356
- source_file = write_to_temp_file(source)
357
- elif isinstance(source, StringIO):
358
- source_file = write_to_temp_file(source.getvalue())
359
- else:
360
- raise TypeError(f"source must be str, Path, or file-like object, not {type(source)}")
361
-
362
- _target = None
363
- if isinstance(target, Path):
364
- if target.is_dir():
365
- target = target / source.name if isinstance(source, Path) else target / "minified.mpy"
366
- _target = target.with_suffix(".mpy")
367
- else:
368
- # target must be a Path object
369
- _target = get_temp_file(suffix=".mpy")
370
- result = pipx_mpy_cross(version, source_file, _target)
371
- if result.stderr and "No matching distribution found for mpy-cross==" in result.stderr:
372
- log.warning(f"mpy-cross=={version} not found, using latest")
373
- result = pipx_mpy_cross(V_PREVIEW, source_file, _target)
374
-
375
- if result.returncode == 0:
376
- log.debug(f"mpy-cross compiled to : {_target.name}")
377
- else:
378
- log.error(f"mpy-cross failed to compile:{result.returncode} \n{result.stderr}")
379
-
380
- if isinstance(target, BytesIO):
381
- # copy the byte contents of the temp file to the target file-like object
382
- with _target.open("rb") as f:
383
- target.write(f.read())
384
- # _target.unlink()
385
-
386
- return result.returncode
387
-
388
-
389
- def pipx_mpy_cross(version: str, source_file, _target):
390
- """Run mpy-cross using pipx"""
391
-
392
- log.info(f"Compiling with mpy-cross version: {version}")
393
- if version in SET_PREVIEW:
394
- version = ""
395
- if version:
396
- version = "==" + version
397
-
398
- cmd = ["pipx", "run", f"mpy-cross{version}"] if version else ["pipx", "run", "mpy-cross"]
399
- # Add params
400
- cmd += ["-O2", str(source_file), "-o", str(_target), "-s", "createstubs.py"]
401
- log.trace(" ".join(cmd))
402
- result = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8") # Specify the encoding
403
- return result
404
-
405
-
406
- def write_to_temp_file(source: str):
407
- """Writes a string to a temp file and returns the Path object"""
408
- _, temp_file = tempfile.mkstemp(suffix=".py", prefix="mpy_cross_")
409
- temp_file = Path(temp_file)
410
- temp_file.write_text(source)
411
- return temp_file
412
-
413
-
414
- def get_temp_file(prefix: str = "mpy_cross_", suffix: str = ".py"):
415
- """Get temp file and returns the Path object"""
416
- _, temp_file = tempfile.mkstemp(prefix=prefix, suffix=suffix)
417
- temp_file = Path(temp_file)
418
- return temp_file
1
+ """
2
+ Processing for createstubs.py
3
+ Minimizes and cross-compiles a MicroPyton file.
4
+ """
5
+ import itertools
6
+ import subprocess
7
+ import tempfile
8
+ from contextlib import ExitStack
9
+ from io import BytesIO, IOBase, StringIO, TextIOWrapper
10
+ from pathlib import Path
11
+ from typing import List, Tuple, Union
12
+
13
+ try:
14
+ import python_minifier
15
+ except ImportError:
16
+ python_minifier = None
17
+
18
+ from loguru import logger as log
19
+
20
+ from stubber.utils.versions import SET_PREVIEW, V_PREVIEW
21
+
22
+ # Type Aliases for minify
23
+ StubSource = Union[Path, str, StringIO, TextIOWrapper]
24
+ XCompileDest = Union[Path, BytesIO]
25
+ LineEdits = List[Tuple[str, str]]
26
+
27
+
28
+ def get_whitespace_context(content: List[str], index: int):
29
+ """Get whitespace count of lines surrounding index"""
30
+ if not content:
31
+ raise ValueError()
32
+ if index < 0 or index > len(content):
33
+ raise IndexError()
34
+ if len(content) == 1:
35
+ return [0, 0]
36
+
37
+ def count_ws(line: str):
38
+ return sum(1 for _ in itertools.takewhile(str.isspace, line))
39
+
40
+ lines = content[max(0, index) : min(index + 2, len(content))]
41
+ context = [count_ws(l) for l in lines]
42
+ if len(context) < 2:
43
+ context.append(context[0])
44
+ return context
45
+
46
+
47
+ def edit_lines(content: str, edits: LineEdits, diff: bool = False):
48
+ # sourcery skip: no-long-functions
49
+ """Edit string by list of edits
50
+
51
+ Args:
52
+ content (str): content to edit
53
+ edits ([(str, str)]): List of edits to make.
54
+ The first string in the tuple representsthe type of edit to make, can be either:
55
+ - comment - comment text out (removed on minify)
56
+ - rprint - replace text with print
57
+ - rpass - replace text with pass
58
+ The second string is the matching text to replace
59
+ diff (bool, optional): Prints diff of each edit.
60
+ Defaults to False.
61
+
62
+ Returns:
63
+ str: edited string
64
+ """
65
+
66
+ def comment(l: str, x: str):
67
+ """Comment out line, so it will be removed on minify"""
68
+ return l.replace(x, f"# {x}")
69
+
70
+ def rprint(l: str, x: str): # type: ignore
71
+ """Replace (logging) with print"""
72
+ split = l.split("(")
73
+ if len(split) > 1:
74
+ return l.replace(split[0].strip(), "print")
75
+ return l.replace(x, "print")
76
+
77
+ def keepprint(l: str, x: str): # type: ignore
78
+ """Replace 'print' with 'print '"""
79
+ return l.replace("print(", "print (")
80
+
81
+ def rpass(l: str, x: str): # type: ignore
82
+ """Replace with pass"""
83
+ return l.replace(x, "pass")
84
+
85
+ def handle_multiline(content: List[str], index: int):
86
+ """Handles edits that require multiline comments
87
+
88
+ Example:
89
+ self.log.debug("info: {} {}".format(
90
+ 1,
91
+ 2
92
+ ))
93
+ Here, only commenting out the first self.log line will raise
94
+ an error. So this function returns all lines that need to
95
+ be commented out instead.
96
+
97
+ It also checks for situations such as this:
98
+ if condition:
99
+ self.log.debug('message')
100
+
101
+ Here, since the only functionality of the conditional is the call log,
102
+ both lines would be returned to comment out.
103
+
104
+ """
105
+ line = content[index]
106
+ open_cnt = line.count("(")
107
+ close_cnt = line.count(")")
108
+ ahead_index = 1
109
+ look_ahead = 0
110
+ while open_cnt != close_cnt: # pragma: no cover
111
+ look_ahead = l_index + ahead_index
112
+ ahead_index += 1
113
+ next_l = content[look_ahead]
114
+ open_cnt += next_l.count("(")
115
+ close_cnt += next_l.count(")")
116
+ if ahead_index > 1: # pragma: no cover
117
+ return range(index, look_ahead + 1)
118
+ prev = content[index - 1]
119
+ line_ws, post_ws = get_whitespace_context(content, index)
120
+ prev_words = prev.strip().strip(":").split()
121
+ check = any(
122
+ t
123
+ in (
124
+ "if",
125
+ "else",
126
+ )
127
+ for t in prev_words
128
+ )
129
+ return range(index - 1, index + 1) if check and line_ws != post_ws else None
130
+
131
+ def handle_try_except(content: List[str], index: int) -> bool:
132
+ """Checks if line at index is in try/except block
133
+
134
+ Handles situations like this:
135
+ try:
136
+ something()
137
+ except:
138
+ self.log.debug('some message')
139
+
140
+ Simply removing the self.log call would create a syntax error,
141
+ which is what this function checks for.
142
+
143
+ """
144
+ prev = content[index - 1]
145
+ line_ws, post_ws = get_whitespace_context(content, index)
146
+ return "except" in prev and line_ws != post_ws
147
+
148
+ lines = []
149
+ multilines = set()
150
+ content_l = content.splitlines(keepends=True)
151
+ for line in content_l:
152
+ _line = line
153
+ for edit_action, match_text in edits:
154
+ if match_text in line:
155
+ if edit_action == "comment":
156
+ l_index = content_l.index(line)
157
+ # Check if edit spans multiple lines
158
+ mline = handle_multiline(content_l, l_index)
159
+ if mline:
160
+ multilines.update(mline)
161
+ break
162
+ # Check if line is only statement in try/except
163
+ if handle_try_except(content_l, l_index):
164
+ edit_action = "rpass"
165
+ match_text = line.strip()
166
+ func = eval(edit_action) # pylint: disable= eval-used
167
+ line = func(line, match_text)
168
+ if line != _line:
169
+ if diff:
170
+ print(f"\n- {_line.strip()}")
171
+ print(f"+ {line.strip()}")
172
+ break
173
+ lines.append(line)
174
+ for line_num in multilines:
175
+ # Go back and comment out multilines
176
+ line = lines[line_num]
177
+ lines[line_num] = comment(line, line.strip())
178
+ stripped = "".join(lines)
179
+ return stripped
180
+
181
+
182
+ def minify_script(source_script: StubSource, keep_report: bool = True, diff: bool = False) -> str:
183
+ """
184
+ Minifies createstubs.py and variants
185
+
186
+ Args:
187
+ source_script:
188
+ - (str): content to edit
189
+ - (Path): path to file to edit
190
+ - (IOBase): file-like object to edit
191
+ keep_report (bool, optional): Keeps single report line in createstubs
192
+ Defaults to True.
193
+ diff (bool, optional): Print diff from edits. Defaults to False.
194
+
195
+ Returns:
196
+ str: minified source text
197
+ """
198
+
199
+ source_content = ""
200
+ if isinstance(source_script, Path):
201
+ source_content = source_script.read_text(encoding="utf-8")
202
+ elif isinstance(source_script, (StringIO, TextIOWrapper)):
203
+ source_content = "".join(source_script.readlines())
204
+ elif isinstance(source_script, str): # type: ignore
205
+ source_content = source_script
206
+ else:
207
+ raise TypeError(f"source_script must be str, Path, or file-like object, not {type(source_script)}")
208
+
209
+ if not source_content:
210
+ raise ValueError("No source content")
211
+ len_1 = len(source_content)
212
+
213
+ if 0:
214
+ min_source = reduce_log_print(keep_report, diff, source_content)
215
+ else:
216
+ min_source = source_content
217
+ len_2 = len(min_source)
218
+
219
+ if not python_minifier:
220
+ log.warning("python_minifier not found, skipping minification")
221
+ else:
222
+ # use python_minifier to minify the source if it is successfully imported
223
+ min_source = python_minifier.minify(
224
+ min_source,
225
+ filename=getattr(source_script, "name", None),
226
+ combine_imports=True,
227
+ remove_literal_statements=True, # no Docstrings
228
+ remove_annotations=True, # not used runtime anyways
229
+ hoist_literals=True, # remove redundant strings
230
+ rename_locals=True, # short names save memory
231
+ preserve_locals=["stubber", "path"], # names to keep
232
+ rename_globals=True, # short names save memory
233
+ # keep these globals to allow testing/mocking to work against the minified not compiled version
234
+ preserve_globals=[
235
+ "main",
236
+ "Stubber",
237
+ "read_path",
238
+ "get_root",
239
+ "_info",
240
+ "os",
241
+ "sys",
242
+ "__version__",
243
+ ],
244
+ # remove_pass=True, # no dead code
245
+ # convert_posargs_to_args=True, # Does not save any space
246
+ )
247
+ len_3 = len(min_source)
248
+ if 1:
249
+ # write to temp file for debugging
250
+ with open("tmp_minified.py", "w+") as f:
251
+ f.write(min_source)
252
+
253
+ log.info(f"Original length : {len_1}")
254
+ log.info(f"Reduced length : {len_2}")
255
+ log.info(f"Minified length : {len_3}")
256
+ log.info(f"Reduced by : {len_1-len_3} ")
257
+ return min_source
258
+
259
+
260
+ def reduce_log_print(keep_report, diff, source_content):
261
+ edits: LineEdits = [
262
+ ("keepprint", "print('Debug: "),
263
+ ("keepprint", "print('DEBUG: "),
264
+ ("keepprint", 'print("Debug: '),
265
+ ("keepprint", 'print("DEBUG: '),
266
+ ("comment", "print("),
267
+ ("comment", "import logging"),
268
+ # report keepers may be inserted here
269
+ # do report errors
270
+ ("rprint", "self.log.error"),
271
+ ("rprint", "log.error"),
272
+ ]
273
+ if keep_report:
274
+ # insert report keepers after the comment modifiers
275
+ edits += [
276
+ # keepers
277
+ ("rprint", 'self.log.info("Stub module: '),
278
+ ("rprint", 'self.log.warning("{}Skip module:'),
279
+ ("rprint", 'self.log.info("Clean/remove files in folder:'),
280
+ ("rprint", 'self.log.info("Created stubs for'),
281
+ ("rprint", 'self.log.info("Family: '),
282
+ ("rprint", 'self.log.info("Version: '),
283
+ ("rprint", 'self.log.info("Port: '),
284
+ ("rprint", 'self.log.info("Board: '),
285
+ # all others
286
+ ("comment", "self.log"),
287
+ ("comment", "_log."),
288
+ ("comment", "_log ="),
289
+ ]
290
+ else:
291
+ edits += [
292
+ # remove first full
293
+ ("comment", "self.log ="),
294
+ ("comment", "self.log("),
295
+ ("comment", "self.log.debug"),
296
+ ("comment", "self.log.info"),
297
+ ("comment", "self.log.warning"),
298
+ # then short versions
299
+ ("comment", "_log ="),
300
+ ("comment", "_log.debug"),
301
+ ("comment", "_log.info"),
302
+ ("comment", "_log.warning"),
303
+ ] + edits
304
+
305
+ content = edit_lines(source_content, edits, diff=diff)
306
+ return content
307
+
308
+
309
+ def minify(
310
+ source: StubSource,
311
+ target: StubSource,
312
+ keep_report: bool = True,
313
+ diff: bool = False,
314
+ ):
315
+ """Minifies and compiles a script"""
316
+ source_buf = None
317
+ target_buf = None
318
+
319
+ with ExitStack() as stack:
320
+ if isinstance(source, Path):
321
+ source_buf = stack.enter_context(source.open("r"))
322
+ elif isinstance(source, (StringIO, str)):
323
+ # different types of file-like objects are both acepted by minify_script
324
+ source_buf = source
325
+ else:
326
+ raise TypeError(f"source must be str, Path, or file-like object, not {type(source)}")
327
+
328
+ if isinstance(target, Path):
329
+ if target.is_dir():
330
+ if isinstance(source, Path):
331
+ target = target / source.name
332
+ else:
333
+ target = target / "minified.py" # or raise error?
334
+ target_buf = stack.enter_context(target.open("w+"))
335
+ elif isinstance(target, IOBase): # type: ignore
336
+ target_buf = target
337
+ try:
338
+ minified = minify_script(source_script=source_buf, keep_report=keep_report, diff=diff)
339
+ target_buf.write(minified)
340
+ except Exception as e: # pragma: no cover
341
+ log.exception(e)
342
+ return 0
343
+
344
+
345
+ def cross_compile(
346
+ source: StubSource,
347
+ target: XCompileDest,
348
+ version: str = "",
349
+ ): # sourcery skip: assign-if-exp
350
+ """Runs mpy-cross on a (minified) script"""
351
+ # Sources can be a file, a string, or a file-like object
352
+ if isinstance(source, Path):
353
+ source_file = source
354
+ elif isinstance(source, str):
355
+ # create a temp file and write the source to it
356
+ source_file = write_to_temp_file(source)
357
+ elif isinstance(source, StringIO):
358
+ source_file = write_to_temp_file(source.getvalue())
359
+ else:
360
+ raise TypeError(f"source must be str, Path, or file-like object, not {type(source)}")
361
+
362
+ _target = None
363
+ if isinstance(target, Path):
364
+ if target.is_dir():
365
+ target = target / source.name if isinstance(source, Path) else target / "minified.mpy"
366
+ _target = target.with_suffix(".mpy")
367
+ else:
368
+ # target must be a Path object
369
+ _target = get_temp_file(suffix=".mpy")
370
+ result = pipx_mpy_cross(version, source_file, _target)
371
+ if result.stderr and "No matching distribution found for mpy-cross==" in result.stderr:
372
+ log.warning(f"mpy-cross=={version} not found, using default version.")
373
+ result = pipx_mpy_cross(V_PREVIEW, source_file, _target)
374
+
375
+ if result.returncode == 0:
376
+ log.debug(f"mpy-cross compiled to : {_target.name}")
377
+ else:
378
+ log.error(f"mpy-cross failed to compile:{result.returncode} \n{result.stderr}")
379
+
380
+ if isinstance(target, BytesIO):
381
+ # copy the byte contents of the temp file to the target file-like object
382
+ with _target.open("rb") as f:
383
+ target.write(f.read())
384
+ # _target.unlink()
385
+
386
+ return result.returncode
387
+
388
+
389
+ def pipx_mpy_cross(version: str, source_file, _target):
390
+ """Run mpy-cross using pipx"""
391
+
392
+ log.info(f"Compiling with mpy-cross version: {version}")
393
+ if version in SET_PREVIEW:
394
+ version = ""
395
+ if version:
396
+ version = "==" + version
397
+
398
+ cmd = ["pipx", "run", f"mpy-cross{version}"] if version else ["pipx", "run", "mpy-cross"]
399
+ # Add params
400
+ cmd += ["-O2", str(source_file), "-o", str(_target), "-s", "createstubs.py"]
401
+ log.trace(" ".join(cmd))
402
+ result = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8") # Specify the encoding
403
+ return result
404
+
405
+
406
+ def write_to_temp_file(source: str):
407
+ """Writes a string to a temp file and returns the Path object"""
408
+ _, temp_file = tempfile.mkstemp(suffix=".py", prefix="mpy_cross_")
409
+ temp_file = Path(temp_file)
410
+ temp_file.write_text(source)
411
+ return temp_file
412
+
413
+
414
+ def get_temp_file(prefix: str = "mpy_cross_", suffix: str = ".py"):
415
+ """Get temp file and returns the Path object"""
416
+ _, temp_file = tempfile.mkstemp(prefix=prefix, suffix=suffix)
417
+ temp_file = Path(temp_file)
418
+ return temp_file