micropython-stubber 1.20.5__py3-none-any.whl → 1.23.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. {micropython_stubber-1.20.5.dist-info → micropython_stubber-1.23.0.dist-info}/LICENSE +30 -30
  2. {micropython_stubber-1.20.5.dist-info → micropython_stubber-1.23.0.dist-info}/METADATA +1 -1
  3. micropython_stubber-1.23.0.dist-info/RECORD +159 -0
  4. mpflash/README.md +184 -184
  5. mpflash/libusb_flash.ipynb +203 -203
  6. mpflash/mpflash/add_firmware.py +98 -98
  7. mpflash/mpflash/ask_input.py +236 -236
  8. mpflash/mpflash/bootloader/__init__.py +37 -36
  9. mpflash/mpflash/bootloader/manual.py +102 -102
  10. mpflash/mpflash/bootloader/micropython.py +10 -10
  11. mpflash/mpflash/bootloader/touch1200.py +45 -45
  12. mpflash/mpflash/cli_download.py +129 -129
  13. mpflash/mpflash/cli_flash.py +219 -219
  14. mpflash/mpflash/cli_group.py +98 -98
  15. mpflash/mpflash/cli_list.py +81 -81
  16. mpflash/mpflash/cli_main.py +41 -41
  17. mpflash/mpflash/common.py +164 -164
  18. mpflash/mpflash/config.py +43 -47
  19. mpflash/mpflash/connected.py +74 -74
  20. mpflash/mpflash/download.py +360 -360
  21. mpflash/mpflash/downloaded.py +130 -129
  22. mpflash/mpflash/errors.py +9 -9
  23. mpflash/mpflash/flash.py +55 -52
  24. mpflash/mpflash/flash_esp.py +59 -59
  25. mpflash/mpflash/flash_stm32.py +18 -24
  26. mpflash/mpflash/flash_stm32_cube.py +111 -111
  27. mpflash/mpflash/flash_stm32_dfu.py +104 -101
  28. mpflash/mpflash/flash_uf2.py +89 -67
  29. mpflash/mpflash/flash_uf2_boardid.py +15 -15
  30. mpflash/mpflash/flash_uf2_linux.py +129 -123
  31. mpflash/mpflash/flash_uf2_macos.py +37 -34
  32. mpflash/mpflash/flash_uf2_windows.py +38 -34
  33. mpflash/mpflash/list.py +89 -89
  34. mpflash/mpflash/logger.py +41 -41
  35. mpflash/mpflash/mpboard_id/__init__.py +93 -93
  36. mpflash/mpflash/mpboard_id/add_boards.py +255 -255
  37. mpflash/mpflash/mpboard_id/board.py +37 -37
  38. mpflash/mpflash/mpboard_id/board_id.py +86 -86
  39. mpflash/mpflash/mpboard_id/store.py +43 -43
  40. mpflash/mpflash/mpremoteboard/__init__.py +226 -221
  41. mpflash/mpflash/mpremoteboard/mpy_fw_info.py +141 -141
  42. mpflash/mpflash/mpremoteboard/runner.py +140 -140
  43. mpflash/mpflash/uf2disk.py +12 -12
  44. mpflash/mpflash/vendor/basicgit.py +288 -288
  45. mpflash/mpflash/vendor/click_aliases.py +91 -91
  46. mpflash/mpflash/vendor/dfu.py +165 -165
  47. mpflash/mpflash/vendor/pydfu.py +605 -605
  48. mpflash/mpflash/vendor/readme.md +2 -2
  49. mpflash/mpflash/vendor/versions.py +119 -117
  50. mpflash/mpflash/worklist.py +171 -170
  51. mpflash/poetry.lock +1588 -1588
  52. mpflash/pyproject.toml +64 -60
  53. mpflash/stm32_udev_rules.md +62 -62
  54. stubber/__init__.py +3 -3
  55. stubber/basicgit.py +294 -288
  56. stubber/board/board_info.csv +193 -193
  57. stubber/board/boot.py +34 -34
  58. stubber/board/createstubs.py +986 -986
  59. stubber/board/createstubs_db.py +825 -825
  60. stubber/board/createstubs_db_min.py +331 -331
  61. stubber/board/createstubs_db_mpy.mpy +0 -0
  62. stubber/board/createstubs_lvgl.py +741 -741
  63. stubber/board/createstubs_lvgl_min.py +741 -741
  64. stubber/board/createstubs_mem.py +766 -766
  65. stubber/board/createstubs_mem_min.py +306 -306
  66. stubber/board/createstubs_mem_mpy.mpy +0 -0
  67. stubber/board/createstubs_min.py +294 -294
  68. stubber/board/createstubs_mpy.mpy +0 -0
  69. stubber/board/fw_info.py +141 -141
  70. stubber/board/info.py +183 -183
  71. stubber/board/main.py +19 -19
  72. stubber/board/modulelist.txt +247 -247
  73. stubber/board/pyrightconfig.json +34 -34
  74. stubber/bulk/mcu_stubber.py +454 -454
  75. stubber/codemod/_partials/__init__.py +48 -48
  76. stubber/codemod/_partials/db_main.py +147 -147
  77. stubber/codemod/_partials/lvgl_main.py +77 -77
  78. stubber/codemod/_partials/modules_reader.py +80 -80
  79. stubber/codemod/add_comment.py +53 -53
  80. stubber/codemod/add_method.py +65 -65
  81. stubber/codemod/board.py +317 -317
  82. stubber/codemod/enrich.py +145 -145
  83. stubber/codemod/merge_docstub.py +284 -284
  84. stubber/codemod/modify_list.py +54 -54
  85. stubber/codemod/utils.py +57 -57
  86. stubber/commands/build_cmd.py +94 -94
  87. stubber/commands/cli.py +55 -51
  88. stubber/commands/clone_cmd.py +77 -66
  89. stubber/commands/config_cmd.py +29 -29
  90. stubber/commands/enrich_folder_cmd.py +71 -70
  91. stubber/commands/get_core_cmd.py +71 -69
  92. stubber/commands/get_docstubs_cmd.py +89 -87
  93. stubber/commands/get_frozen_cmd.py +114 -112
  94. stubber/commands/get_mcu_cmd.py +61 -56
  95. stubber/commands/merge_cmd.py +67 -66
  96. stubber/commands/publish_cmd.py +119 -119
  97. stubber/commands/stub_cmd.py +31 -30
  98. stubber/commands/switch_cmd.py +62 -54
  99. stubber/commands/variants_cmd.py +49 -48
  100. stubber/cst_transformer.py +178 -178
  101. stubber/data/board_info.csv +193 -193
  102. stubber/data/board_info.json +1729 -1729
  103. stubber/data/micropython_tags.csv +15 -15
  104. stubber/data/requirements-core-micropython.txt +38 -38
  105. stubber/data/requirements-core-pycopy.txt +39 -39
  106. stubber/downloader.py +36 -36
  107. stubber/freeze/common.py +68 -68
  108. stubber/freeze/freeze_folder.py +69 -69
  109. stubber/freeze/freeze_manifest_2.py +113 -113
  110. stubber/freeze/get_frozen.py +127 -127
  111. stubber/get_cpython.py +101 -101
  112. stubber/get_lobo.py +59 -59
  113. stubber/minify.py +418 -418
  114. stubber/publish/bump.py +86 -86
  115. stubber/publish/candidates.py +262 -262
  116. stubber/publish/database.py +18 -18
  117. stubber/publish/defaults.py +45 -45
  118. stubber/publish/enums.py +24 -24
  119. stubber/publish/helpers.py +29 -29
  120. stubber/publish/merge_docstubs.py +130 -130
  121. stubber/publish/missing_class_methods.py +49 -49
  122. stubber/publish/package.py +146 -146
  123. stubber/publish/pathnames.py +51 -51
  124. stubber/publish/publish.py +120 -120
  125. stubber/publish/pypi.py +38 -38
  126. stubber/publish/stubpackage.py +1029 -1029
  127. stubber/rst/__init__.py +9 -9
  128. stubber/rst/classsort.py +77 -77
  129. stubber/rst/lookup.py +530 -530
  130. stubber/rst/output_dict.py +401 -401
  131. stubber/rst/reader.py +822 -822
  132. stubber/rst/report_return.py +69 -69
  133. stubber/rst/rst_utils.py +540 -540
  134. stubber/stubber.py +38 -38
  135. stubber/stubs_from_docs.py +90 -90
  136. stubber/tools/manifestfile.py +655 -610
  137. stubber/tools/readme.md +7 -6
  138. stubber/update_fallback.py +117 -117
  139. stubber/update_module_list.py +123 -123
  140. stubber/utils/__init__.py +5 -5
  141. stubber/utils/config.py +127 -127
  142. stubber/utils/makeversionhdr.py +54 -54
  143. stubber/utils/manifest.py +92 -92
  144. stubber/utils/post.py +79 -79
  145. stubber/utils/repos.py +157 -154
  146. stubber/utils/stubmaker.py +139 -139
  147. stubber/utils/typed_config_toml.py +77 -77
  148. stubber/utils/versions.py +128 -120
  149. stubber/variants.py +106 -106
  150. micropython_stubber-1.20.5.dist-info/RECORD +0 -159
  151. {micropython_stubber-1.20.5.dist-info → micropython_stubber-1.23.0.dist-info}/WHEEL +0 -0
  152. {micropython_stubber-1.20.5.dist-info → micropython_stubber-1.23.0.dist-info}/entry_points.txt +0 -0
@@ -1,825 +1,825 @@
1
- """
2
- Create stubs for (all) modules on a MicroPython board.
3
-
4
- This variant of the createstubs.py script is optimized for use on very-low-memory devices.
5
- Note: this version has undergone limited testing.
6
-
7
- 1) reads the list of modules from a text file `modulelist.txt` that should be uploaded to the device.
8
- 2) stored the already processed modules in a text file `modulelist.done`
9
- 3) process the modules in the database:
10
- - stub the module
11
- - update the modulelist.done file
12
- - reboots the device if it runs out of memory
13
- 4) creates the modules.json
14
-
15
- If that cannot be found then only a single module (micropython) is stubbed.
16
- In order to run this on low-memory devices two additional steps are recommended:
17
- - minification, using python-minifierto reduce overall size, and remove logging overhead.
18
- - cross compilation, using mpy-cross, to avoid the compilation step on the micropython device
19
-
20
-
21
- This variant was generated from createstubs.py by micropython-stubber v1.20.5
22
- """
23
-
24
- # Copyright (c) 2019-2024 Jos Verlinde
25
-
26
- import gc
27
- import os
28
- import sys
29
- from time import sleep
30
-
31
- try:
32
- from ujson import dumps
33
- except:
34
- from json import dumps
35
-
36
- try:
37
- from machine import reset # type: ignore
38
- except ImportError:
39
- pass
40
-
41
- try:
42
- from collections import OrderedDict
43
- except ImportError:
44
- from ucollections import OrderedDict # type: ignore
45
-
46
- __version__ = "v1.20.5"
47
- ENOENT = 2
48
- _MAX_CLASS_LEVEL = 2 # Max class nesting
49
- LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."]
50
-
51
-
52
- # our own logging module to avoid dependency on and interfering with logging module
53
- class logging:
54
- # DEBUG = 10
55
- INFO = 20
56
- WARNING = 30
57
- ERROR = 40
58
- level = INFO
59
- prnt = print
60
-
61
- @staticmethod
62
- def getLogger(name):
63
- return logging()
64
-
65
- @classmethod
66
- def basicConfig(cls, level):
67
- cls.level = level
68
-
69
- # def debug(self, msg):
70
- # if self.level <= logging.DEBUG:
71
- # self.prnt("DEBUG :", msg)
72
-
73
- def info(self, msg):
74
- if self.level <= logging.INFO:
75
- self.prnt("INFO :", msg)
76
-
77
- def warning(self, msg):
78
- if self.level <= logging.WARNING:
79
- self.prnt("WARN :", msg)
80
-
81
- def error(self, msg):
82
- if self.level <= logging.ERROR:
83
- self.prnt("ERROR :", msg)
84
-
85
-
86
- log = logging.getLogger("stubber")
87
- logging.basicConfig(level=logging.INFO)
88
- # logging.basicConfig(level=logging.DEBUG)
89
-
90
-
91
- class Stubber:
92
- "Generate stubs for modules in firmware"
93
-
94
- def __init__(self, path: str = None, firmware_id: str = None): # type: ignore
95
- try:
96
- if os.uname().release == "1.13.0" and os.uname().version < "v1.13-103": # type: ignore
97
- raise NotImplementedError("MicroPython 1.13.0 cannot be stubbed")
98
- except AttributeError:
99
- pass # Allow testing on CPython 3.11
100
- self.info = _info()
101
- log.info("Port: {}".format(self.info["port"]))
102
- log.info("Board: {}".format(self.info["board"]))
103
- gc.collect()
104
- if firmware_id:
105
- self._fwid = firmware_id.lower()
106
- else:
107
- if self.info["family"] == "micropython":
108
- self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-")
109
- else:
110
- self._fwid = "{family}-v{version}-{port}".format(**self.info)
111
- self._start_free = gc.mem_free() # type: ignore
112
-
113
- if path:
114
- if path.endswith("/"):
115
- path = path[:-1]
116
- else:
117
- path = get_root()
118
-
119
- self.path = "{}/stubs/{}".format(path, self.flat_fwid).replace("//", "/")
120
- # log.debug(self.path)
121
- try:
122
- ensure_folder(path + "/")
123
- except OSError:
124
- log.error("error creating stub folder {}".format(path))
125
- self.problematic = [
126
- "upip",
127
- "upysh",
128
- "webrepl_setup",
129
- "http_client",
130
- "http_client_ssl",
131
- "http_server",
132
- "http_server_ssl",
133
- ]
134
- self.excluded = [
135
- "webrepl",
136
- "_webrepl",
137
- "port_diag",
138
- "example_sub_led.py",
139
- "example_pub_button.py",
140
- ]
141
- # there is no option to discover modules from micropython, list is read from an external file.
142
- self.modules = [] # type: list[str]
143
- self._json_name = None
144
- self._json_first = False
145
-
146
- def get_obj_attributes(self, item_instance: object):
147
- "extract information of the objects members and attributes"
148
- # name_, repr_(value), type as text, item_instance
149
- _result = []
150
- _errors = []
151
- # log.debug("get attributes {} {}".format(repr(item_instance), item_instance))
152
- for name in dir(item_instance):
153
- if name.startswith("__") and not name in self.modules:
154
- continue
155
- # log.debug("get attribute {}".format(name))
156
- try:
157
- val = getattr(item_instance, name)
158
- # name , item_repr(value) , type as text, item_instance, order
159
- # log.debug("attribute {}:{}".format(name, val))
160
- try:
161
- type_text = repr(type(val)).split("'")[1]
162
- except IndexError:
163
- type_text = ""
164
- if type_text in {"int", "float", "str", "bool", "tuple", "list", "dict"}:
165
- order = 1
166
- elif type_text in {"function", "method"}:
167
- order = 2
168
- elif type_text in ("class"):
169
- order = 3
170
- else:
171
- order = 4
172
- _result.append((name, repr(val), repr(type(val)), val, order))
173
- except AttributeError as e:
174
- _errors.append("Couldn't get attribute '{}' from object '{}', Err: {}".format(name, item_instance, e))
175
- except MemoryError as e:
176
- print("MemoryError: {}".format(e))
177
- sleep(1)
178
- reset()
179
-
180
- # remove internal __
181
- # _result = sorted([i for i in _result if not (i[0].startswith("_"))], key=lambda x: x[4])
182
- _result = sorted([i for i in _result if not (i[0].startswith("__"))], key=lambda x: x[4])
183
- gc.collect()
184
- return _result, _errors
185
-
186
- def add_modules(self, modules):
187
- "Add additional modules to be exported"
188
- self.modules = sorted(set(self.modules) | set(modules))
189
-
190
- def create_all_stubs(self):
191
- "Create stubs for all configured modules"
192
- log.info("Start micropython-stubber {} on {}".format(__version__, self._fwid))
193
- self.report_start()
194
- gc.collect()
195
- for module_name in self.modules:
196
- self.create_one_stub(module_name)
197
- self.report_end()
198
- log.info("Finally done")
199
-
200
- def create_one_stub(self, module_name: str):
201
- if module_name in self.problematic:
202
- log.warning("Skip module: {:<25} : Known problematic".format(module_name))
203
- return False
204
- if module_name in self.excluded:
205
- log.warning("Skip module: {:<25} : Excluded".format(module_name))
206
- return False
207
-
208
- file_name = "{}/{}.pyi".format(self.path, module_name.replace(".", "/"))
209
- gc.collect()
210
- result = False
211
- try:
212
- result = self.create_module_stub(module_name, file_name)
213
- except OSError:
214
- return False
215
- gc.collect()
216
- return result
217
-
218
- def create_module_stub(self, module_name: str, file_name: str = None) -> bool: # type: ignore
219
- """Create a Stub of a single python module
220
-
221
- Args:
222
- - module_name (str): name of the module to document. This module will be imported.
223
- - file_name (Optional[str]): the 'path/filename.pyi' to write to. If omitted will be created based on the module name.
224
- """
225
- if file_name is None:
226
- fname = module_name.replace(".", "_") + ".pyi"
227
- file_name = self.path + "/" + fname
228
- else:
229
- fname = file_name.split("/")[-1]
230
-
231
- if "/" in module_name:
232
- # for nested modules
233
- module_name = module_name.replace("/", ".")
234
-
235
- # import the module (as new_module) to examine it
236
- new_module = None
237
- try:
238
- new_module = __import__(module_name, None, None, ("*"))
239
- m1 = gc.mem_free() # type: ignore
240
- log.info("Stub module: {:<25} to file: {:<70} mem:{:>5}".format(module_name, fname, m1))
241
-
242
- except ImportError:
243
- # log.debug("Skip module: {:<25} {:<79}".format(module_name, "Module not found."))
244
- return False
245
-
246
- # Start a new file
247
- ensure_folder(file_name)
248
- with open(file_name, "w") as fp:
249
- info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}")
250
- s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__)
251
- fp.write(s)
252
- fp.write("from __future__ import annotations\nfrom typing import Any, Generator\nfrom _typeshed import Incomplete\n\n")
253
- self.write_object_stub(fp, new_module, module_name, "")
254
-
255
- self.report_add(module_name, file_name)
256
-
257
- if module_name not in {"os", "sys", "logging", "gc"}:
258
- # try to unload the module unless we use it
259
- try:
260
- del new_module
261
- except (OSError, KeyError): # lgtm [py/unreachable-statement]
262
- log.warning("could not del new_module")
263
- # do not try to delete from sys.modules - most times it does not work anyway
264
- gc.collect()
265
- return True
266
-
267
- def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, in_class: int = 0):
268
- "Write a module/object stub to an open file. Can be called recursive."
269
- gc.collect()
270
- if object_expr in self.problematic:
271
- log.warning("SKIPPING problematic module:{}".format(object_expr))
272
- return
273
-
274
- # # log.debug("DUMP : {}".format(object_expr))
275
- items, errors = self.get_obj_attributes(object_expr)
276
-
277
- if errors:
278
- log.error(errors)
279
-
280
- for item_name, item_repr, item_type_txt, item_instance, _ in items:
281
- # name_, repr_(value), type as text, item_instance, order
282
- if item_name in ["classmethod", "staticmethod", "BaseException", "Exception"]:
283
- # do not create stubs for these primitives
284
- continue
285
- if item_name[0].isdigit():
286
- log.warning("NameError: invalid name {}".format(item_name))
287
- continue
288
- # Class expansion only on first 3 levels (bit of a hack)
289
- if (
290
- item_type_txt == "<class 'type'>"
291
- and len(indent) <= _MAX_CLASS_LEVEL * 4
292
- # and not obj_name.endswith(".Pin")
293
- # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms
294
- ):
295
- # log.debug("{0}class {1}:".format(indent, item_name))
296
- superclass = ""
297
- is_exception = (
298
- item_name.endswith("Exception")
299
- or item_name.endswith("Error")
300
- or item_name
301
- in [
302
- "KeyboardInterrupt",
303
- "StopIteration",
304
- "SystemExit",
305
- ]
306
- )
307
- if is_exception:
308
- superclass = "Exception"
309
- s = "\n{}class {}({}):\n".format(indent, item_name, superclass)
310
- # s += indent + " ''\n"
311
- if is_exception:
312
- s += indent + " ...\n"
313
- fp.write(s)
314
- continue
315
- # write classdef
316
- fp.write(s)
317
- # first write the class literals and methods
318
- # log.debug("# recursion over class {0}".format(item_name))
319
- self.write_object_stub(
320
- fp,
321
- item_instance,
322
- "{0}.{1}".format(obj_name, item_name),
323
- indent + " ",
324
- in_class + 1,
325
- )
326
- # end with the __init__ method to make sure that the literals are defined
327
- # Add __init__
328
- s = indent + " def __init__(self, *argv, **kwargs) -> None:\n"
329
- s += indent + " ...\n\n"
330
- fp.write(s)
331
- elif any(word in item_type_txt for word in ["method", "function", "closure"]):
332
- # log.debug("# def {1} function/method/closure, type = '{0}'".format(item_type_txt, item_name))
333
- # module Function or class method
334
- # will accept any number of params
335
- # return type Any/Incomplete
336
- ret = "Incomplete"
337
- first = ""
338
- # Self parameter only on class methods/functions
339
- if in_class > 0:
340
- first = "self, "
341
- # class method - add function decoration
342
- if "bound_method" in item_type_txt or "bound_method" in item_repr:
343
- s = "{}@classmethod\n".format(indent) + "{}def {}(cls, *args, **kwargs) -> {}:\n".format(indent, item_name, ret)
344
- else:
345
- s = "{}def {}({}*args, **kwargs) -> {}:\n".format(indent, item_name, first, ret)
346
- s += indent + " ...\n\n"
347
- fp.write(s)
348
- # log.debug("\n" + s)
349
- elif item_type_txt == "<class 'module'>":
350
- # Skip imported modules
351
- # fp.write("# import {}\n".format(item_name))
352
- pass
353
-
354
- elif item_type_txt.startswith("<class '"):
355
- t = item_type_txt[8:-2]
356
- s = ""
357
-
358
- if t in ("str", "int", "float", "bool", "bytearray", "bytes"):
359
- # known type: use actual value
360
- # s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, item_repr, t)
361
- s = "{0}{1}: {3} = {2}\n".format(indent, item_name, item_repr, t)
362
- elif t in ("dict", "list", "tuple"):
363
- # dict, list , tuple: use empty value
364
- ev = {"dict": "{}", "list": "[]", "tuple": "()"}
365
- # s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t)
366
- s = "{0}{1}: {3} = {2}\n".format(indent, item_name, ev[t], t)
367
- else:
368
- # something else
369
- if t in ("object", "set", "frozenset", "Pin", "generator"): # "FileIO"
370
- # https://docs.python.org/3/tutorial/classes.html#item_instance-objects
371
- # use these types for the attribute
372
- if t == "generator":
373
- t = "Generator"
374
- s = "{0}{1}: {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
375
- else:
376
- # Requires Python 3.6 syntax, which is OK for the stubs/pyi
377
- t = "Incomplete"
378
- if " at " in item_repr:
379
- item_repr = item_repr.split(" at ")[0] + " at ...>"
380
- if " at " in item_repr:
381
- item_repr = item_repr.split(" at ")[0] + " at ...>"
382
- s = "{0}{1}: {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
383
- fp.write(s)
384
- # log.debug("\n" + s)
385
- else:
386
- # keep only the name
387
- # log.debug("# all other, type = '{0}'".format(item_type_txt))
388
- fp.write("# all other, type = '{0}'\n".format(item_type_txt))
389
-
390
- fp.write(indent + item_name + " # type: Incomplete\n")
391
-
392
- # del items
393
- # del errors
394
- # try:
395
- # del item_name, item_repr, item_type_txt, item_instance # type: ignore
396
- # except (OSError, KeyError, NameError):
397
- # pass
398
-
399
- @property
400
- def flat_fwid(self):
401
- "Turn _fwid from 'v1.2.3' into '1_2_3' to be used in filename"
402
- s = self._fwid
403
- # path name restrictions
404
- chars = " .()/\\:$"
405
- for c in chars:
406
- s = s.replace(c, "_")
407
- return s
408
-
409
- def clean(self, path: str = None): # type: ignore
410
- "Remove all files from the stub folder"
411
- if path is None:
412
- path = self.path
413
- log.info("Clean/remove files in folder: {}".format(path))
414
- try:
415
- os.stat(path) # TEMP workaround mpremote listdir bug -
416
- items = os.listdir(path)
417
- except (OSError, AttributeError):
418
- # os.listdir fails on unix
419
- return
420
- for fn in items:
421
- item = "{}/{}".format(path, fn)
422
- try:
423
- os.remove(item)
424
- except OSError:
425
- try: # folder
426
- self.clean(item)
427
- os.rmdir(item)
428
- except OSError:
429
- pass
430
-
431
- def report_start(self, filename: str = "modules.json"):
432
- """Start a report of the modules that have been stubbed
433
- "create json with list of exported modules"""
434
- self._json_name = "{}/{}".format(self.path, filename)
435
- self._json_first = True
436
- ensure_folder(self._json_name)
437
- log.info("Report file: {}".format(self._json_name))
438
- gc.collect()
439
- try:
440
- # write json by node to reduce memory requirements
441
- with open(self._json_name, "w") as f:
442
- f.write("{")
443
- f.write(dumps({"firmware": self.info})[1:-1])
444
- f.write(",\n")
445
- f.write(dumps({"stubber": {"version": __version__}, "stubtype": "firmware"})[1:-1])
446
- f.write(",\n")
447
- f.write('"modules" :[\n')
448
-
449
- except OSError as e:
450
- log.error("Failed to create the report.")
451
- self._json_name = None
452
- raise e
453
-
454
- def report_add(self, module_name: str, stub_file: str):
455
- "Add a module to the report"
456
- # write json by node to reduce memory requirements
457
- if not self._json_name:
458
- raise Exception("No report file")
459
- try:
460
- with open(self._json_name, "a") as f:
461
- if not self._json_first:
462
- f.write(",\n")
463
- else:
464
- self._json_first = False
465
- line = '{{"module": "{}", "file": "{}"}}'.format(module_name, stub_file.replace("\\", "/"))
466
- f.write(line)
467
-
468
- except OSError:
469
- log.error("Failed to create the report.")
470
-
471
- def report_end(self):
472
- if not self._json_name:
473
- raise Exception("No report file")
474
- with open(self._json_name, "a") as f:
475
- f.write("\n]}")
476
- # is used as sucess indicator
477
- log.info("Path: {}".format(self.path))
478
-
479
-
480
- def ensure_folder(path: str):
481
- "Create nested folders if needed"
482
- i = start = 0
483
- while i != -1:
484
- i = path.find("/", start)
485
- if i != -1:
486
- p = path[0] if i == 0 else path[:i]
487
- # p = partial folder
488
- try:
489
- _ = os.stat(p)
490
- except OSError as e:
491
- # folder does not exist
492
- if e.args[0] == ENOENT:
493
- try:
494
- os.mkdir(p)
495
- except OSError as e2:
496
- log.error("failed to create folder {}".format(p))
497
- raise e2
498
- # next level deep
499
- start = i + 1
500
-
501
-
502
- def _build(s):
503
- # extract build from sys.version or os.uname().version if available
504
- # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f'
505
- # sys.implementation.version: 'v1.13-103-gb137d064e'
506
- if not s:
507
- return ""
508
- s = s.split(" on ", 1)[0] if " on " in s else s
509
- if s.startswith("v"):
510
- if not "-" in s:
511
- return ""
512
- b = s.split("-")[1]
513
- return b
514
- if not "-preview" in s:
515
- return ""
516
- b = s.split("-preview")[1].split(".")[1]
517
- return b
518
-
519
-
520
- def _info(): # type:() -> dict[str, str]
521
- try:
522
- fam = sys.implementation[0] # type: ignore
523
- except TypeError:
524
- # testing on CPython 3.11
525
- fam = sys.implementation.name
526
-
527
- info = OrderedDict(
528
- {
529
- "family": fam,
530
- "version": "",
531
- "build": "",
532
- "ver": "",
533
- "port": sys.platform, # port: esp32 / win32 / linux / stm32
534
- "board": "UNKNOWN",
535
- "cpu": "",
536
- "mpy": "",
537
- "arch": "",
538
- }
539
- )
540
- # change port names to be consistent with the repo
541
- if info["port"].startswith("pyb"):
542
- info["port"] = "stm32"
543
- elif info["port"] == "win32":
544
- info["port"] = "windows"
545
- elif info["port"] == "linux":
546
- info["port"] = "unix"
547
- try:
548
- info["version"] = version_str(sys.implementation.version) # type: ignore
549
- except AttributeError:
550
- pass
551
- try:
552
- _machine = sys.implementation._machine if "_machine" in dir(sys.implementation) else os.uname().machine # type: ignore
553
- # info["board"] = "with".join(_machine.split("with")[:-1]).strip()
554
- info["board"] = _machine
555
- info["cpu"] = _machine.split("with")[-1].strip()
556
- info["mpy"] = (
557
- sys.implementation._mpy # type: ignore
558
- if "_mpy" in dir(sys.implementation)
559
- else sys.implementation.mpy if "mpy" in dir(sys.implementation) else "" # type: ignore
560
- )
561
- except (AttributeError, IndexError):
562
- pass
563
- info["board"] = get_boardname()
564
-
565
- try:
566
- if "uname" in dir(os): # old
567
- # extract build from uname().version if available
568
- info["build"] = _build(os.uname()[3]) # type: ignore
569
- if not info["build"]:
570
- # extract build from uname().release if available
571
- info["build"] = _build(os.uname()[2]) # type: ignore
572
- elif "version" in dir(sys): # new
573
- # extract build from sys.version if available
574
- info["build"] = _build(sys.version)
575
- except (AttributeError, IndexError, TypeError):
576
- pass
577
- # avoid build hashes
578
- # if info["build"] and len(info["build"]) > 5:
579
- # info["build"] = ""
580
-
581
- if info["version"] == "" and sys.platform not in ("unix", "win32"):
582
- try:
583
- u = os.uname() # type: ignore
584
- info["version"] = u.release
585
- except (IndexError, AttributeError, TypeError):
586
- pass
587
- # detect families
588
- for fam_name, mod_name, mod_thing in [
589
- ("pycopy", "pycopy", "const"),
590
- ("pycom", "pycom", "FAT"),
591
- ("ev3-pybricks", "pybricks.hubs", "EV3Brick"),
592
- ]:
593
- try:
594
- _t = __import__(mod_name, None, None, (mod_thing))
595
- info["family"] = fam_name
596
- del _t
597
- break
598
- except (ImportError, KeyError):
599
- pass
600
-
601
- if info["family"] == "ev3-pybricks":
602
- info["release"] = "2.0.0"
603
-
604
- if info["family"] == "micropython":
605
- info["version"]
606
- if (
607
- info["version"]
608
- and info["version"].endswith(".0")
609
- and info["version"] >= "1.10.0" # versions from 1.10.0 to 1.20.5 do not have a micro .0
610
- and info["version"] <= "1.19.9"
611
- ):
612
- # versions from 1.10.0 to 1.20.5 do not have a micro .0
613
- info["version"] = info["version"][:-2]
614
-
615
- # spell-checker: disable
616
- if "mpy" in info and info["mpy"]: # mpy on some v1.11+ builds
617
- sys_mpy = int(info["mpy"])
618
- # .mpy architecture
619
- arch = [
620
- None,
621
- "x86",
622
- "x64",
623
- "armv6",
624
- "armv6m",
625
- "armv7m",
626
- "armv7em",
627
- "armv7emsp",
628
- "armv7emdp",
629
- "xtensa",
630
- "xtensawin",
631
- ][sys_mpy >> 10]
632
- if arch:
633
- info["arch"] = arch
634
- # .mpy version.minor
635
- info["mpy"] = "v{}.{}".format(sys_mpy & 0xFF, sys_mpy >> 8 & 3)
636
- if info["build"] and not info["version"].endswith("-preview"):
637
- info["version"] = info["version"] + "-preview"
638
- # simple to use version[-build] string
639
- info["ver"] = f"{info['version']}-{info['build']}" if info["build"] else f"{info['version']}"
640
-
641
- return info
642
-
643
-
644
- def version_str(version: tuple): # -> str:
645
- v_str = ".".join([str(n) for n in version[:3]])
646
- if len(version) > 3 and version[3]:
647
- v_str += "-" + version[3]
648
- return v_str
649
-
650
-
651
- def get_boardname() -> str:
652
- "Read the board name from the boardname.py file that may have been created upfront"
653
- try:
654
- from boardname import BOARDNAME # type: ignore
655
-
656
- log.info("Found BOARDNAME: {}".format(BOARDNAME))
657
- except ImportError:
658
- log.warning("BOARDNAME not found")
659
- BOARDNAME = ""
660
- return BOARDNAME
661
-
662
-
663
- def get_root() -> str: # sourcery skip: use-assigned-variable
664
- "Determine the root folder of the device"
665
- try:
666
- c = os.getcwd()
667
- except (OSError, AttributeError):
668
- # unix port
669
- c = "."
670
- r = c
671
- for r in [c, "/sd", "/flash", "/", "."]:
672
- try:
673
- _ = os.stat(r)
674
- break
675
- except OSError:
676
- continue
677
- return r
678
-
679
-
680
- def file_exists(filename: str):
681
- try:
682
- if os.stat(filename)[0] >> 14:
683
- return True
684
- return False
685
- except OSError:
686
- return False
687
-
688
-
689
- def show_help():
690
- print("-p, --path path to store the stubs in, defaults to '.'")
691
- sys.exit(1)
692
-
693
-
694
- def read_path() -> str:
695
- "get --path from cmdline. [unix/win]"
696
- path = ""
697
- if len(sys.argv) == 3:
698
- cmd = (sys.argv[1]).lower()
699
- if cmd in ("--path", "-p"):
700
- path = sys.argv[2]
701
- else:
702
- show_help()
703
- elif len(sys.argv) == 2:
704
- show_help()
705
- return path
706
-
707
-
708
- def is_micropython() -> bool:
709
- "runtime test to determine full or micropython"
710
- # pylint: disable=unused-variable,eval-used
711
- try:
712
- # either test should fail on micropython
713
-
714
- # b) https://docs.micropython.org/en/latest/genrst/builtin_types.html#bytes-with-keywords-not-implemented
715
- # Micropython: NotImplementedError
716
- b = bytes("abc", encoding="utf8") # type: ignore # lgtm [py/unused-local-variable]
717
-
718
- # c) https://docs.micropython.org/en/latest/genrst/core_language.html#function-objects-do-not-have-the-module-attribute
719
- # Micropython: AttributeError
720
- c = is_micropython.__module__ # type: ignore # lgtm [py/unused-local-variable]
721
- return False
722
- except (NotImplementedError, AttributeError):
723
- return True
724
-
725
-
726
- SKIP_FILE = "modulelist.done"
727
-
728
-
729
- def get_modules(skip=0):
730
- # new
731
- for p in LIBS:
732
- fname = p + "/modulelist.txt"
733
- if not file_exists(fname):
734
- continue
735
- try:
736
- with open(fname) as f:
737
- i = 0
738
- while True:
739
- line = f.readline().strip()
740
- if not line:
741
- break
742
- if len(line) > 0 and line[0] == "#":
743
- continue
744
- i += 1
745
- if i < skip:
746
- continue
747
- yield line
748
- break
749
- except OSError:
750
- pass
751
-
752
-
753
- def write_skip(done):
754
- # write count of modules already processed to file
755
- with open(SKIP_FILE, "w") as f:
756
- f.write(str(done) + "\n")
757
-
758
-
759
- def read_skip():
760
- # read count of modules already processed from file
761
- done = 0
762
- try:
763
- with open(SKIP_FILE) as f:
764
- done = int(f.readline().strip())
765
- except OSError:
766
- pass
767
- return done
768
-
769
-
770
- def main():
771
- import machine # type: ignore
772
-
773
- was_running = file_exists(SKIP_FILE)
774
- if was_running:
775
- log.info("Continue from last run")
776
- else:
777
- log.info("Starting new run")
778
- # try:
779
- # f = open("modulelist.done", "r+b")
780
- # was_running = True
781
- # print("Continue from last run")
782
- # except OSError:
783
- # f = open("modulelist.done", "w+b")
784
- # was_running = False
785
- stubber = Stubber(path=read_path())
786
-
787
- # f_name = "{}/{}".format(stubber.path, "modules.json")
788
- skip = 0
789
- if not was_running:
790
- # Only clean folder if this is a first run
791
- stubber.clean()
792
- stubber.report_start("modules.json")
793
- else:
794
- skip = read_skip()
795
- stubber._json_name = "{}/{}".format(stubber.path, "modules.json")
796
-
797
- for modulename in get_modules(skip):
798
- # ------------------------------------
799
- # do epic shit
800
- # but sometimes things fail / run out of memory and reboot
801
- try:
802
- stubber.create_one_stub(modulename)
803
- except MemoryError:
804
- # RESET AND HOPE THAT IN THE NEXT CYCLE WE PROGRESS FURTHER
805
- machine.reset()
806
- # -------------------------------------
807
- gc.collect()
808
- # modules_done[modulename] = str(stubber._report[-1] if ok else "failed")
809
- # with open("modulelist.done", "a") as f:
810
- # f.write("{}={}\n".format(modulename, "ok" if ok else "failed"))
811
- skip += 1
812
- write_skip(skip)
813
-
814
- print("All modules have been processed, Finalizing report")
815
- stubber.report_end()
816
-
817
-
818
- if __name__ == "__main__" or is_micropython():
819
- if not file_exists("no_auto_stubber.txt"):
820
- try:
821
- gc.threshold(4 * 1024) # type: ignore
822
- gc.enable()
823
- except BaseException:
824
- pass
825
- main()
1
+ """
2
+ Create stubs for (all) modules on a MicroPython board.
3
+
4
+ This variant of the createstubs.py script is optimized for use on very-low-memory devices.
5
+ Note: this version has undergone limited testing.
6
+
7
+ 1) reads the list of modules from a text file `modulelist.txt` that should be uploaded to the device.
8
+ 2) stored the already processed modules in a text file `modulelist.done`
9
+ 3) process the modules in the database:
10
+ - stub the module
11
+ - update the modulelist.done file
12
+ - reboots the device if it runs out of memory
13
+ 4) creates the modules.json
14
+
15
+ If that cannot be found then only a single module (micropython) is stubbed.
16
+ In order to run this on low-memory devices two additional steps are recommended:
17
+ - minification, using python-minifierto reduce overall size, and remove logging overhead.
18
+ - cross compilation, using mpy-cross, to avoid the compilation step on the micropython device
19
+
20
+
21
+ This variant was generated from createstubs.py by micropython-stubber v1.23.0
22
+ """
23
+
24
+ # Copyright (c) 2019-2024 Jos Verlinde
25
+
26
+ import gc
27
+ import os
28
+ import sys
29
+ from time import sleep
30
+
31
+ try:
32
+ from ujson import dumps
33
+ except:
34
+ from json import dumps
35
+
36
+ try:
37
+ from machine import reset # type: ignore
38
+ except ImportError:
39
+ pass
40
+
41
+ try:
42
+ from collections import OrderedDict
43
+ except ImportError:
44
+ from ucollections import OrderedDict # type: ignore
45
+
46
+ __version__ = "v1.23.0"
47
+ ENOENT = 2
48
+ _MAX_CLASS_LEVEL = 2 # Max class nesting
49
+ LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."]
50
+
51
+
52
+ # our own logging module to avoid dependency on and interfering with logging module
53
+ class logging:
54
+ # DEBUG = 10
55
+ INFO = 20
56
+ WARNING = 30
57
+ ERROR = 40
58
+ level = INFO
59
+ prnt = print
60
+
61
+ @staticmethod
62
+ def getLogger(name):
63
+ return logging()
64
+
65
+ @classmethod
66
+ def basicConfig(cls, level):
67
+ cls.level = level
68
+
69
+ # def debug(self, msg):
70
+ # if self.level <= logging.DEBUG:
71
+ # self.prnt("DEBUG :", msg)
72
+
73
+ def info(self, msg):
74
+ if self.level <= logging.INFO:
75
+ self.prnt("INFO :", msg)
76
+
77
+ def warning(self, msg):
78
+ if self.level <= logging.WARNING:
79
+ self.prnt("WARN :", msg)
80
+
81
+ def error(self, msg):
82
+ if self.level <= logging.ERROR:
83
+ self.prnt("ERROR :", msg)
84
+
85
+
86
+ log = logging.getLogger("stubber")
87
+ logging.basicConfig(level=logging.INFO)
88
+ # logging.basicConfig(level=logging.DEBUG)
89
+
90
+
91
+ class Stubber:
92
+ "Generate stubs for modules in firmware"
93
+
94
+ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore
95
+ try:
96
+ if os.uname().release == "1.13.0" and os.uname().version < "v1.13-103": # type: ignore
97
+ raise NotImplementedError("MicroPython 1.13.0 cannot be stubbed")
98
+ except AttributeError:
99
+ pass # Allow testing on CPython 3.11
100
+ self.info = _info()
101
+ log.info("Port: {}".format(self.info["port"]))
102
+ log.info("Board: {}".format(self.info["board"]))
103
+ gc.collect()
104
+ if firmware_id:
105
+ self._fwid = firmware_id.lower()
106
+ else:
107
+ if self.info["family"] == "micropython":
108
+ self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-")
109
+ else:
110
+ self._fwid = "{family}-v{version}-{port}".format(**self.info)
111
+ self._start_free = gc.mem_free() # type: ignore
112
+
113
+ if path:
114
+ if path.endswith("/"):
115
+ path = path[:-1]
116
+ else:
117
+ path = get_root()
118
+
119
+ self.path = "{}/stubs/{}".format(path, self.flat_fwid).replace("//", "/")
120
+ # log.debug(self.path)
121
+ try:
122
+ ensure_folder(path + "/")
123
+ except OSError:
124
+ log.error("error creating stub folder {}".format(path))
125
+ self.problematic = [
126
+ "upip",
127
+ "upysh",
128
+ "webrepl_setup",
129
+ "http_client",
130
+ "http_client_ssl",
131
+ "http_server",
132
+ "http_server_ssl",
133
+ ]
134
+ self.excluded = [
135
+ "webrepl",
136
+ "_webrepl",
137
+ "port_diag",
138
+ "example_sub_led.py",
139
+ "example_pub_button.py",
140
+ ]
141
+ # there is no option to discover modules from micropython, list is read from an external file.
142
+ self.modules = [] # type: list[str]
143
+ self._json_name = None
144
+ self._json_first = False
145
+
146
+ def get_obj_attributes(self, item_instance: object):
147
+ "extract information of the objects members and attributes"
148
+ # name_, repr_(value), type as text, item_instance
149
+ _result = []
150
+ _errors = []
151
+ # log.debug("get attributes {} {}".format(repr(item_instance), item_instance))
152
+ for name in dir(item_instance):
153
+ if name.startswith("__") and not name in self.modules:
154
+ continue
155
+ # log.debug("get attribute {}".format(name))
156
+ try:
157
+ val = getattr(item_instance, name)
158
+ # name , item_repr(value) , type as text, item_instance, order
159
+ # log.debug("attribute {}:{}".format(name, val))
160
+ try:
161
+ type_text = repr(type(val)).split("'")[1]
162
+ except IndexError:
163
+ type_text = ""
164
+ if type_text in {"int", "float", "str", "bool", "tuple", "list", "dict"}:
165
+ order = 1
166
+ elif type_text in {"function", "method"}:
167
+ order = 2
168
+ elif type_text in ("class"):
169
+ order = 3
170
+ else:
171
+ order = 4
172
+ _result.append((name, repr(val), repr(type(val)), val, order))
173
+ except AttributeError as e:
174
+ _errors.append("Couldn't get attribute '{}' from object '{}', Err: {}".format(name, item_instance, e))
175
+ except MemoryError as e:
176
+ print("MemoryError: {}".format(e))
177
+ sleep(1)
178
+ reset()
179
+
180
+ # remove internal __
181
+ # _result = sorted([i for i in _result if not (i[0].startswith("_"))], key=lambda x: x[4])
182
+ _result = sorted([i for i in _result if not (i[0].startswith("__"))], key=lambda x: x[4])
183
+ gc.collect()
184
+ return _result, _errors
185
+
186
+ def add_modules(self, modules):
187
+ "Add additional modules to be exported"
188
+ self.modules = sorted(set(self.modules) | set(modules))
189
+
190
+ def create_all_stubs(self):
191
+ "Create stubs for all configured modules"
192
+ log.info("Start micropython-stubber {} on {}".format(__version__, self._fwid))
193
+ self.report_start()
194
+ gc.collect()
195
+ for module_name in self.modules:
196
+ self.create_one_stub(module_name)
197
+ self.report_end()
198
+ log.info("Finally done")
199
+
200
+ def create_one_stub(self, module_name: str):
201
+ if module_name in self.problematic:
202
+ log.warning("Skip module: {:<25} : Known problematic".format(module_name))
203
+ return False
204
+ if module_name in self.excluded:
205
+ log.warning("Skip module: {:<25} : Excluded".format(module_name))
206
+ return False
207
+
208
+ file_name = "{}/{}.pyi".format(self.path, module_name.replace(".", "/"))
209
+ gc.collect()
210
+ result = False
211
+ try:
212
+ result = self.create_module_stub(module_name, file_name)
213
+ except OSError:
214
+ return False
215
+ gc.collect()
216
+ return result
217
+
218
+ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: # type: ignore
219
+ """Create a Stub of a single python module
220
+
221
+ Args:
222
+ - module_name (str): name of the module to document. This module will be imported.
223
+ - file_name (Optional[str]): the 'path/filename.pyi' to write to. If omitted will be created based on the module name.
224
+ """
225
+ if file_name is None:
226
+ fname = module_name.replace(".", "_") + ".pyi"
227
+ file_name = self.path + "/" + fname
228
+ else:
229
+ fname = file_name.split("/")[-1]
230
+
231
+ if "/" in module_name:
232
+ # for nested modules
233
+ module_name = module_name.replace("/", ".")
234
+
235
+ # import the module (as new_module) to examine it
236
+ new_module = None
237
+ try:
238
+ new_module = __import__(module_name, None, None, ("*"))
239
+ m1 = gc.mem_free() # type: ignore
240
+ log.info("Stub module: {:<25} to file: {:<70} mem:{:>5}".format(module_name, fname, m1))
241
+
242
+ except ImportError:
243
+ # log.debug("Skip module: {:<25} {:<79}".format(module_name, "Module not found."))
244
+ return False
245
+
246
+ # Start a new file
247
+ ensure_folder(file_name)
248
+ with open(file_name, "w") as fp:
249
+ info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}")
250
+ s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__)
251
+ fp.write(s)
252
+ fp.write("from __future__ import annotations\nfrom typing import Any, Generator\nfrom _typeshed import Incomplete\n\n")
253
+ self.write_object_stub(fp, new_module, module_name, "")
254
+
255
+ self.report_add(module_name, file_name)
256
+
257
+ if module_name not in {"os", "sys", "logging", "gc"}:
258
+ # try to unload the module unless we use it
259
+ try:
260
+ del new_module
261
+ except (OSError, KeyError): # lgtm [py/unreachable-statement]
262
+ log.warning("could not del new_module")
263
+ # do not try to delete from sys.modules - most times it does not work anyway
264
+ gc.collect()
265
+ return True
266
+
267
+ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, in_class: int = 0):
268
+ "Write a module/object stub to an open file. Can be called recursive."
269
+ gc.collect()
270
+ if object_expr in self.problematic:
271
+ log.warning("SKIPPING problematic module:{}".format(object_expr))
272
+ return
273
+
274
+ # # log.debug("DUMP : {}".format(object_expr))
275
+ items, errors = self.get_obj_attributes(object_expr)
276
+
277
+ if errors:
278
+ log.error(errors)
279
+
280
+ for item_name, item_repr, item_type_txt, item_instance, _ in items:
281
+ # name_, repr_(value), type as text, item_instance, order
282
+ if item_name in ["classmethod", "staticmethod", "BaseException", "Exception"]:
283
+ # do not create stubs for these primitives
284
+ continue
285
+ if item_name[0].isdigit():
286
+ log.warning("NameError: invalid name {}".format(item_name))
287
+ continue
288
+ # Class expansion only on first 3 levels (bit of a hack)
289
+ if (
290
+ item_type_txt == "<class 'type'>"
291
+ and len(indent) <= _MAX_CLASS_LEVEL * 4
292
+ # and not obj_name.endswith(".Pin")
293
+ # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms
294
+ ):
295
+ # log.debug("{0}class {1}:".format(indent, item_name))
296
+ superclass = ""
297
+ is_exception = (
298
+ item_name.endswith("Exception")
299
+ or item_name.endswith("Error")
300
+ or item_name
301
+ in [
302
+ "KeyboardInterrupt",
303
+ "StopIteration",
304
+ "SystemExit",
305
+ ]
306
+ )
307
+ if is_exception:
308
+ superclass = "Exception"
309
+ s = "\n{}class {}({}):\n".format(indent, item_name, superclass)
310
+ # s += indent + " ''\n"
311
+ if is_exception:
312
+ s += indent + " ...\n"
313
+ fp.write(s)
314
+ continue
315
+ # write classdef
316
+ fp.write(s)
317
+ # first write the class literals and methods
318
+ # log.debug("# recursion over class {0}".format(item_name))
319
+ self.write_object_stub(
320
+ fp,
321
+ item_instance,
322
+ "{0}.{1}".format(obj_name, item_name),
323
+ indent + " ",
324
+ in_class + 1,
325
+ )
326
+ # end with the __init__ method to make sure that the literals are defined
327
+ # Add __init__
328
+ s = indent + " def __init__(self, *argv, **kwargs) -> None:\n"
329
+ s += indent + " ...\n\n"
330
+ fp.write(s)
331
+ elif any(word in item_type_txt for word in ["method", "function", "closure"]):
332
+ # log.debug("# def {1} function/method/closure, type = '{0}'".format(item_type_txt, item_name))
333
+ # module Function or class method
334
+ # will accept any number of params
335
+ # return type Any/Incomplete
336
+ ret = "Incomplete"
337
+ first = ""
338
+ # Self parameter only on class methods/functions
339
+ if in_class > 0:
340
+ first = "self, "
341
+ # class method - add function decoration
342
+ if "bound_method" in item_type_txt or "bound_method" in item_repr:
343
+ s = "{}@classmethod\n".format(indent) + "{}def {}(cls, *args, **kwargs) -> {}:\n".format(indent, item_name, ret)
344
+ else:
345
+ s = "{}def {}({}*args, **kwargs) -> {}:\n".format(indent, item_name, first, ret)
346
+ s += indent + " ...\n\n"
347
+ fp.write(s)
348
+ # log.debug("\n" + s)
349
+ elif item_type_txt == "<class 'module'>":
350
+ # Skip imported modules
351
+ # fp.write("# import {}\n".format(item_name))
352
+ pass
353
+
354
+ elif item_type_txt.startswith("<class '"):
355
+ t = item_type_txt[8:-2]
356
+ s = ""
357
+
358
+ if t in ("str", "int", "float", "bool", "bytearray", "bytes"):
359
+ # known type: use actual value
360
+ # s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, item_repr, t)
361
+ s = "{0}{1}: {3} = {2}\n".format(indent, item_name, item_repr, t)
362
+ elif t in ("dict", "list", "tuple"):
363
+ # dict, list , tuple: use empty value
364
+ ev = {"dict": "{}", "list": "[]", "tuple": "()"}
365
+ # s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t)
366
+ s = "{0}{1}: {3} = {2}\n".format(indent, item_name, ev[t], t)
367
+ else:
368
+ # something else
369
+ if t in ("object", "set", "frozenset", "Pin", "generator"): # "FileIO"
370
+ # https://docs.python.org/3/tutorial/classes.html#item_instance-objects
371
+ # use these types for the attribute
372
+ if t == "generator":
373
+ t = "Generator"
374
+ s = "{0}{1}: {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
375
+ else:
376
+ # Requires Python 3.6 syntax, which is OK for the stubs/pyi
377
+ t = "Incomplete"
378
+ if " at " in item_repr:
379
+ item_repr = item_repr.split(" at ")[0] + " at ...>"
380
+ if " at " in item_repr:
381
+ item_repr = item_repr.split(" at ")[0] + " at ...>"
382
+ s = "{0}{1}: {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
383
+ fp.write(s)
384
+ # log.debug("\n" + s)
385
+ else:
386
+ # keep only the name
387
+ # log.debug("# all other, type = '{0}'".format(item_type_txt))
388
+ fp.write("# all other, type = '{0}'\n".format(item_type_txt))
389
+
390
+ fp.write(indent + item_name + " # type: Incomplete\n")
391
+
392
+ # del items
393
+ # del errors
394
+ # try:
395
+ # del item_name, item_repr, item_type_txt, item_instance # type: ignore
396
+ # except (OSError, KeyError, NameError):
397
+ # pass
398
+
399
+ @property
400
+ def flat_fwid(self):
401
+ "Turn _fwid from 'v1.2.3' into '1_2_3' to be used in filename"
402
+ s = self._fwid
403
+ # path name restrictions
404
+ chars = " .()/\\:$"
405
+ for c in chars:
406
+ s = s.replace(c, "_")
407
+ return s
408
+
409
+ def clean(self, path: str = None): # type: ignore
410
+ "Remove all files from the stub folder"
411
+ if path is None:
412
+ path = self.path
413
+ log.info("Clean/remove files in folder: {}".format(path))
414
+ try:
415
+ os.stat(path) # TEMP workaround mpremote listdir bug -
416
+ items = os.listdir(path)
417
+ except (OSError, AttributeError):
418
+ # os.listdir fails on unix
419
+ return
420
+ for fn in items:
421
+ item = "{}/{}".format(path, fn)
422
+ try:
423
+ os.remove(item)
424
+ except OSError:
425
+ try: # folder
426
+ self.clean(item)
427
+ os.rmdir(item)
428
+ except OSError:
429
+ pass
430
+
431
+ def report_start(self, filename: str = "modules.json"):
432
+ """Start a report of the modules that have been stubbed
433
+ "create json with list of exported modules"""
434
+ self._json_name = "{}/{}".format(self.path, filename)
435
+ self._json_first = True
436
+ ensure_folder(self._json_name)
437
+ log.info("Report file: {}".format(self._json_name))
438
+ gc.collect()
439
+ try:
440
+ # write json by node to reduce memory requirements
441
+ with open(self._json_name, "w") as f:
442
+ f.write("{")
443
+ f.write(dumps({"firmware": self.info})[1:-1])
444
+ f.write(",\n")
445
+ f.write(dumps({"stubber": {"version": __version__}, "stubtype": "firmware"})[1:-1])
446
+ f.write(",\n")
447
+ f.write('"modules" :[\n')
448
+
449
+ except OSError as e:
450
+ log.error("Failed to create the report.")
451
+ self._json_name = None
452
+ raise e
453
+
454
+ def report_add(self, module_name: str, stub_file: str):
455
+ "Add a module to the report"
456
+ # write json by node to reduce memory requirements
457
+ if not self._json_name:
458
+ raise Exception("No report file")
459
+ try:
460
+ with open(self._json_name, "a") as f:
461
+ if not self._json_first:
462
+ f.write(",\n")
463
+ else:
464
+ self._json_first = False
465
+ line = '{{"module": "{}", "file": "{}"}}'.format(module_name, stub_file.replace("\\", "/"))
466
+ f.write(line)
467
+
468
+ except OSError:
469
+ log.error("Failed to create the report.")
470
+
471
+ def report_end(self):
472
+ if not self._json_name:
473
+ raise Exception("No report file")
474
+ with open(self._json_name, "a") as f:
475
+ f.write("\n]}")
476
+ # is used as sucess indicator
477
+ log.info("Path: {}".format(self.path))
478
+
479
+
480
+ def ensure_folder(path: str):
481
+ "Create nested folders if needed"
482
+ i = start = 0
483
+ while i != -1:
484
+ i = path.find("/", start)
485
+ if i != -1:
486
+ p = path[0] if i == 0 else path[:i]
487
+ # p = partial folder
488
+ try:
489
+ _ = os.stat(p)
490
+ except OSError as e:
491
+ # folder does not exist
492
+ if e.args[0] == ENOENT:
493
+ try:
494
+ os.mkdir(p)
495
+ except OSError as e2:
496
+ log.error("failed to create folder {}".format(p))
497
+ raise e2
498
+ # next level deep
499
+ start = i + 1
500
+
501
+
502
+ def _build(s):
503
+ # extract build from sys.version or os.uname().version if available
504
+ # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f'
505
+ # sys.implementation.version: 'v1.13-103-gb137d064e'
506
+ if not s:
507
+ return ""
508
+ s = s.split(" on ", 1)[0] if " on " in s else s
509
+ if s.startswith("v"):
510
+ if not "-" in s:
511
+ return ""
512
+ b = s.split("-")[1]
513
+ return b
514
+ if not "-preview" in s:
515
+ return ""
516
+ b = s.split("-preview")[1].split(".")[1]
517
+ return b
518
+
519
+
520
+ def _info(): # type:() -> dict[str, str]
521
+ try:
522
+ fam = sys.implementation[0] # type: ignore
523
+ except TypeError:
524
+ # testing on CPython 3.11
525
+ fam = sys.implementation.name
526
+
527
+ info = OrderedDict(
528
+ {
529
+ "family": fam,
530
+ "version": "",
531
+ "build": "",
532
+ "ver": "",
533
+ "port": sys.platform, # port: esp32 / win32 / linux / stm32
534
+ "board": "UNKNOWN",
535
+ "cpu": "",
536
+ "mpy": "",
537
+ "arch": "",
538
+ }
539
+ )
540
+ # change port names to be consistent with the repo
541
+ if info["port"].startswith("pyb"):
542
+ info["port"] = "stm32"
543
+ elif info["port"] == "win32":
544
+ info["port"] = "windows"
545
+ elif info["port"] == "linux":
546
+ info["port"] = "unix"
547
+ try:
548
+ info["version"] = version_str(sys.implementation.version) # type: ignore
549
+ except AttributeError:
550
+ pass
551
+ try:
552
+ _machine = sys.implementation._machine if "_machine" in dir(sys.implementation) else os.uname().machine # type: ignore
553
+ # info["board"] = "with".join(_machine.split("with")[:-1]).strip()
554
+ info["board"] = _machine
555
+ info["cpu"] = _machine.split("with")[-1].strip()
556
+ info["mpy"] = (
557
+ sys.implementation._mpy # type: ignore
558
+ if "_mpy" in dir(sys.implementation)
559
+ else sys.implementation.mpy if "mpy" in dir(sys.implementation) else "" # type: ignore
560
+ )
561
+ except (AttributeError, IndexError):
562
+ pass
563
+ info["board"] = get_boardname()
564
+
565
+ try:
566
+ if "uname" in dir(os): # old
567
+ # extract build from uname().version if available
568
+ info["build"] = _build(os.uname()[3]) # type: ignore
569
+ if not info["build"]:
570
+ # extract build from uname().release if available
571
+ info["build"] = _build(os.uname()[2]) # type: ignore
572
+ elif "version" in dir(sys): # new
573
+ # extract build from sys.version if available
574
+ info["build"] = _build(sys.version)
575
+ except (AttributeError, IndexError, TypeError):
576
+ pass
577
+ # avoid build hashes
578
+ # if info["build"] and len(info["build"]) > 5:
579
+ # info["build"] = ""
580
+
581
+ if info["version"] == "" and sys.platform not in ("unix", "win32"):
582
+ try:
583
+ u = os.uname() # type: ignore
584
+ info["version"] = u.release
585
+ except (IndexError, AttributeError, TypeError):
586
+ pass
587
+ # detect families
588
+ for fam_name, mod_name, mod_thing in [
589
+ ("pycopy", "pycopy", "const"),
590
+ ("pycom", "pycom", "FAT"),
591
+ ("ev3-pybricks", "pybricks.hubs", "EV3Brick"),
592
+ ]:
593
+ try:
594
+ _t = __import__(mod_name, None, None, (mod_thing))
595
+ info["family"] = fam_name
596
+ del _t
597
+ break
598
+ except (ImportError, KeyError):
599
+ pass
600
+
601
+ if info["family"] == "ev3-pybricks":
602
+ info["release"] = "2.0.0"
603
+
604
+ if info["family"] == "micropython":
605
+ info["version"]
606
+ if (
607
+ info["version"]
608
+ and info["version"].endswith(".0")
609
+ and info["version"] >= "1.10.0" # versions from 1.10.0 to 1.23.0 do not have a micro .0
610
+ and info["version"] <= "1.19.9"
611
+ ):
612
+ # versions from 1.10.0 to 1.23.0 do not have a micro .0
613
+ info["version"] = info["version"][:-2]
614
+
615
+ # spell-checker: disable
616
+ if "mpy" in info and info["mpy"]: # mpy on some v1.11+ builds
617
+ sys_mpy = int(info["mpy"])
618
+ # .mpy architecture
619
+ arch = [
620
+ None,
621
+ "x86",
622
+ "x64",
623
+ "armv6",
624
+ "armv6m",
625
+ "armv7m",
626
+ "armv7em",
627
+ "armv7emsp",
628
+ "armv7emdp",
629
+ "xtensa",
630
+ "xtensawin",
631
+ ][sys_mpy >> 10]
632
+ if arch:
633
+ info["arch"] = arch
634
+ # .mpy version.minor
635
+ info["mpy"] = "v{}.{}".format(sys_mpy & 0xFF, sys_mpy >> 8 & 3)
636
+ if info["build"] and not info["version"].endswith("-preview"):
637
+ info["version"] = info["version"] + "-preview"
638
+ # simple to use version[-build] string
639
+ info["ver"] = f"{info['version']}-{info['build']}" if info["build"] else f"{info['version']}"
640
+
641
+ return info
642
+
643
+
644
+ def version_str(version: tuple): # -> str:
645
+ v_str = ".".join([str(n) for n in version[:3]])
646
+ if len(version) > 3 and version[3]:
647
+ v_str += "-" + version[3]
648
+ return v_str
649
+
650
+
651
+ def get_boardname() -> str:
652
+ "Read the board name from the boardname.py file that may have been created upfront"
653
+ try:
654
+ from boardname import BOARDNAME # type: ignore
655
+
656
+ log.info("Found BOARDNAME: {}".format(BOARDNAME))
657
+ except ImportError:
658
+ log.warning("BOARDNAME not found")
659
+ BOARDNAME = ""
660
+ return BOARDNAME
661
+
662
+
663
+ def get_root() -> str: # sourcery skip: use-assigned-variable
664
+ "Determine the root folder of the device"
665
+ try:
666
+ c = os.getcwd()
667
+ except (OSError, AttributeError):
668
+ # unix port
669
+ c = "."
670
+ r = c
671
+ for r in [c, "/sd", "/flash", "/", "."]:
672
+ try:
673
+ _ = os.stat(r)
674
+ break
675
+ except OSError:
676
+ continue
677
+ return r
678
+
679
+
680
+ def file_exists(filename: str):
681
+ try:
682
+ if os.stat(filename)[0] >> 14:
683
+ return True
684
+ return False
685
+ except OSError:
686
+ return False
687
+
688
+
689
+ def show_help():
690
+ print("-p, --path path to store the stubs in, defaults to '.'")
691
+ sys.exit(1)
692
+
693
+
694
+ def read_path() -> str:
695
+ "get --path from cmdline. [unix/win]"
696
+ path = ""
697
+ if len(sys.argv) == 3:
698
+ cmd = (sys.argv[1]).lower()
699
+ if cmd in ("--path", "-p"):
700
+ path = sys.argv[2]
701
+ else:
702
+ show_help()
703
+ elif len(sys.argv) == 2:
704
+ show_help()
705
+ return path
706
+
707
+
708
+ def is_micropython() -> bool:
709
+ "runtime test to determine full or micropython"
710
+ # pylint: disable=unused-variable,eval-used
711
+ try:
712
+ # either test should fail on micropython
713
+
714
+ # b) https://docs.micropython.org/en/latest/genrst/builtin_types.html#bytes-with-keywords-not-implemented
715
+ # Micropython: NotImplementedError
716
+ b = bytes("abc", encoding="utf8") # type: ignore # lgtm [py/unused-local-variable]
717
+
718
+ # c) https://docs.micropython.org/en/latest/genrst/core_language.html#function-objects-do-not-have-the-module-attribute
719
+ # Micropython: AttributeError
720
+ c = is_micropython.__module__ # type: ignore # lgtm [py/unused-local-variable]
721
+ return False
722
+ except (NotImplementedError, AttributeError):
723
+ return True
724
+
725
+
726
+ SKIP_FILE = "modulelist.done"
727
+
728
+
729
+ def get_modules(skip=0):
730
+ # new
731
+ for p in LIBS:
732
+ fname = p + "/modulelist.txt"
733
+ if not file_exists(fname):
734
+ continue
735
+ try:
736
+ with open(fname) as f:
737
+ i = 0
738
+ while True:
739
+ line = f.readline().strip()
740
+ if not line:
741
+ break
742
+ if len(line) > 0 and line[0] == "#":
743
+ continue
744
+ i += 1
745
+ if i < skip:
746
+ continue
747
+ yield line
748
+ break
749
+ except OSError:
750
+ pass
751
+
752
+
753
+ def write_skip(done):
754
+ # write count of modules already processed to file
755
+ with open(SKIP_FILE, "w") as f:
756
+ f.write(str(done) + "\n")
757
+
758
+
759
+ def read_skip():
760
+ # read count of modules already processed from file
761
+ done = 0
762
+ try:
763
+ with open(SKIP_FILE) as f:
764
+ done = int(f.readline().strip())
765
+ except OSError:
766
+ pass
767
+ return done
768
+
769
+
770
+ def main():
771
+ import machine # type: ignore
772
+
773
+ was_running = file_exists(SKIP_FILE)
774
+ if was_running:
775
+ log.info("Continue from last run")
776
+ else:
777
+ log.info("Starting new run")
778
+ # try:
779
+ # f = open("modulelist.done", "r+b")
780
+ # was_running = True
781
+ # print("Continue from last run")
782
+ # except OSError:
783
+ # f = open("modulelist.done", "w+b")
784
+ # was_running = False
785
+ stubber = Stubber(path=read_path())
786
+
787
+ # f_name = "{}/{}".format(stubber.path, "modules.json")
788
+ skip = 0
789
+ if not was_running:
790
+ # Only clean folder if this is a first run
791
+ stubber.clean()
792
+ stubber.report_start("modules.json")
793
+ else:
794
+ skip = read_skip()
795
+ stubber._json_name = "{}/{}".format(stubber.path, "modules.json")
796
+
797
+ for modulename in get_modules(skip):
798
+ # ------------------------------------
799
+ # do epic shit
800
+ # but sometimes things fail / run out of memory and reboot
801
+ try:
802
+ stubber.create_one_stub(modulename)
803
+ except MemoryError:
804
+ # RESET AND HOPE THAT IN THE NEXT CYCLE WE PROGRESS FURTHER
805
+ machine.reset()
806
+ # -------------------------------------
807
+ gc.collect()
808
+ # modules_done[modulename] = str(stubber._report[-1] if ok else "failed")
809
+ # with open("modulelist.done", "a") as f:
810
+ # f.write("{}={}\n".format(modulename, "ok" if ok else "failed"))
811
+ skip += 1
812
+ write_skip(skip)
813
+
814
+ print("All modules have been processed, Finalizing report")
815
+ stubber.report_end()
816
+
817
+
818
+ if __name__ == "__main__" or is_micropython():
819
+ if not file_exists("no_auto_stubber.txt"):
820
+ try:
821
+ gc.threshold(4 * 1024) # type: ignore
822
+ gc.enable()
823
+ except BaseException:
824
+ pass
825
+ main()