micropython-stubber 1.14.1__py3-none-any.whl → 1.15.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- micropython_stubber-1.15.1.dist-info/METADATA +244 -0
- {micropython_stubber-1.14.1.dist-info → micropython_stubber-1.15.1.dist-info}/RECORD +49 -46
- stubber/__init__.py +1 -1
- stubber/basicgit.py +27 -14
- stubber/board/createstubs.py +34 -36
- stubber/board/createstubs_db.py +35 -35
- stubber/board/createstubs_db_min.py +195 -193
- stubber/board/createstubs_db_mpy.mpy +0 -0
- stubber/board/createstubs_info.py +73 -42
- stubber/board/createstubs_lvgl.py +35 -35
- stubber/board/createstubs_lvgl_min.py +88 -87
- stubber/board/createstubs_lvgl_mpy.mpy +0 -0
- stubber/board/createstubs_mem.py +44 -40
- stubber/board/createstubs_mem_min.py +179 -174
- stubber/board/createstubs_mem_mpy.mpy +0 -0
- stubber/board/createstubs_min.py +75 -74
- stubber/board/createstubs_mpy.mpy +0 -0
- stubber/board/info.py +183 -0
- stubber/codemod/enrich.py +28 -16
- stubber/commands/build_cmd.py +3 -3
- stubber/commands/get_core_cmd.py +17 -5
- stubber/commands/get_docstubs_cmd.py +23 -8
- stubber/commands/get_frozen_cmd.py +62 -9
- stubber/commands/get_lobo_cmd.py +13 -3
- stubber/commands/merge_cmd.py +7 -4
- stubber/commands/publish_cmd.py +5 -4
- stubber/commands/stub_cmd.py +2 -1
- stubber/commands/variants_cmd.py +0 -1
- stubber/freeze/common.py +2 -2
- stubber/freeze/freeze_folder.py +1 -1
- stubber/freeze/get_frozen.py +1 -1
- stubber/minify.py +43 -28
- stubber/publish/bump.py +1 -1
- stubber/publish/candidates.py +61 -17
- stubber/publish/defaults.py +44 -0
- stubber/publish/merge_docstubs.py +19 -56
- stubber/publish/package.py +44 -37
- stubber/publish/pathnames.py +51 -0
- stubber/publish/publish.py +5 -4
- stubber/publish/stubpacker.py +55 -33
- stubber/rst/lookup.py +7 -16
- stubber/rst/reader.py +39 -8
- stubber/stubs_from_docs.py +5 -8
- stubber/utils/post.py +34 -40
- stubber/utils/repos.py +32 -17
- stubber/utils/stubmaker.py +22 -14
- micropython_stubber-1.14.1.dist-info/METADATA +0 -217
- {micropython_stubber-1.14.1.dist-info → micropython_stubber-1.15.1.dist-info}/LICENSE +0 -0
- {micropython_stubber-1.14.1.dist-info → micropython_stubber-1.15.1.dist-info}/WHEEL +0 -0
- {micropython_stubber-1.14.1.dist-info → micropython_stubber-1.15.1.dist-info}/entry_points.txt +0 -0
stubber/publish/stubpacker.py
CHANGED
@@ -22,6 +22,7 @@ from packaging.version import Version, parse
|
|
22
22
|
from pysondb import PysonDB
|
23
23
|
|
24
24
|
from stubber.publish.bump import bump_version
|
25
|
+
from stubber.publish.defaults import GENERIC_U, default_board
|
25
26
|
from stubber.publish.enums import StubSource
|
26
27
|
from stubber.publish.pypi import Version, get_pypi_versions
|
27
28
|
from stubber.utils.config import CONFIG
|
@@ -36,7 +37,6 @@ STUB_SKIPPER = {
|
|
36
37
|
StubSource.FIRMWARE: ["builtins"],
|
37
38
|
StubSource.DOC: [],
|
38
39
|
StubSource.CORE: [],
|
39
|
-
|
40
40
|
}
|
41
41
|
|
42
42
|
|
@@ -49,6 +49,7 @@ class StubPackage:
|
|
49
49
|
- package_path - the path to the folder where the package info will be stored ('./publish').
|
50
50
|
- pkg_version - the version of the package as used on PyPi (semver). Is stored directly in the `pyproject.toml` file
|
51
51
|
- pyproject - the contents of the `pyproject.toml` file
|
52
|
+
|
52
53
|
methods:
|
53
54
|
- from_json - load the package from json
|
54
55
|
- to_json - return the package as json
|
@@ -66,6 +67,9 @@ class StubPackage:
|
|
66
67
|
def __init__(
|
67
68
|
self,
|
68
69
|
package_name: str,
|
70
|
+
port: str,
|
71
|
+
*,
|
72
|
+
board: str = GENERIC_U,
|
69
73
|
version: str = "0.0.1",
|
70
74
|
description: str = "MicroPython stubs",
|
71
75
|
stubs: Optional[StubSources] = None,
|
@@ -89,9 +93,10 @@ class StubPackage:
|
|
89
93
|
STUB_PATH - root-relative path to the folder where the stubs are stored ('./stubs').
|
90
94
|
|
91
95
|
"""
|
96
|
+
self.port = port
|
97
|
+
self.board = board
|
92
98
|
if json_data is not None:
|
93
99
|
self.from_dict(json_data)
|
94
|
-
|
95
100
|
else:
|
96
101
|
# store essentials
|
97
102
|
# self.package_path = package_path
|
@@ -163,19 +168,19 @@ class StubPackage:
|
|
163
168
|
except FileNotFoundError as e:
|
164
169
|
raise FileNotFoundError(f"pyproject.toml file not found at {_toml}") from e
|
165
170
|
|
166
|
-
def
|
171
|
+
def next_pkg_version(self, production: bool) -> str:
|
167
172
|
"""Get the next version for the package"""
|
168
|
-
|
169
|
-
self.get_prerelease_package_version(production)
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
+
if self.mpy_version == "latest":
|
174
|
+
next_ver = self.get_prerelease_package_version(production)
|
175
|
+
else:
|
176
|
+
next_ver = self.get_next_package_version(production)
|
177
|
+
return next_ver
|
173
178
|
|
174
179
|
def get_prerelease_package_version(self, production: bool = False) -> str:
|
175
180
|
"""Get the next prerelease version for the package."""
|
176
181
|
rc = 1
|
177
182
|
if describe := get_git_describe(CONFIG.mpy_path.as_posix()):
|
178
|
-
# use versiontag and the
|
183
|
+
# use versiontag and the number of commits since the last tag
|
179
184
|
# "v1.19.1-841-g3446"
|
180
185
|
# 'v1.22.0-preview-19-g8eb7721b4'
|
181
186
|
parts = describe.split("-", 3)
|
@@ -189,7 +194,8 @@ class StubPackage:
|
|
189
194
|
base = Version(ver)
|
190
195
|
return str(bump_version(base, rc=rc))
|
191
196
|
else:
|
192
|
-
|
197
|
+
return "99.99.99post99"
|
198
|
+
# raise ValueError("cannot determine next version number micropython")
|
193
199
|
|
194
200
|
def get_next_package_version(self, prod: bool = False, rc=False) -> str:
|
195
201
|
"""Get the next version for the package."""
|
@@ -233,7 +239,7 @@ class StubPackage:
|
|
233
239
|
def to_dict(self) -> dict:
|
234
240
|
"""return the package as a dict to store in the jsondb
|
235
241
|
|
236
|
-
need to simplify some of the Objects to allow
|
242
|
+
need to simplify some of the Objects to allow serialization to json
|
237
243
|
- the paths to posix paths
|
238
244
|
- the version (semver) to a string
|
239
245
|
- toml file to list of lines
|
@@ -288,16 +294,15 @@ class StubPackage:
|
|
288
294
|
self.create_readme()
|
289
295
|
self.create_license()
|
290
296
|
|
291
|
-
|
292
|
-
def update_sources(stub_sources: StubSources) -> StubSources:
|
297
|
+
def update_sources(self) -> StubSources:
|
293
298
|
"""
|
294
299
|
Update the stub sources to:
|
295
300
|
- FIRMWARE: prefer -merged stubs over bare firmware stubs
|
296
301
|
- FROZEN: fallback to use the GENERIC folder for the frozen sources if no board specific folder exists
|
297
302
|
"""
|
298
303
|
updated_sources = []
|
299
|
-
for stub_type, fw_path in stub_sources:
|
300
|
-
# prefer -merged stubs over bare
|
304
|
+
for stub_type, fw_path in self.stub_sources:
|
305
|
+
# prefer -merged stubs over bare firmware stubs
|
301
306
|
if stub_type == StubSource.FIRMWARE:
|
302
307
|
# Check if -merged folder exists and use that instead
|
303
308
|
if fw_path.name.endswith("-merged"):
|
@@ -313,7 +318,20 @@ class StubPackage:
|
|
313
318
|
if (CONFIG.stub_path / fw_path).exists():
|
314
319
|
updated_sources.append((stub_type, fw_path))
|
315
320
|
else:
|
316
|
-
|
321
|
+
if fw_path.with_name("GENERIC").exists():
|
322
|
+
updated_sources.append((stub_type, fw_path.with_name("GENERIC")))
|
323
|
+
elif stub_type == StubSource.MERGED:
|
324
|
+
# Use the default board folder instead of the GENERIC board folder (if it exists)
|
325
|
+
if self.board.upper() == GENERIC_U:
|
326
|
+
family = fw_path.name.split("-")[0]
|
327
|
+
# TODO: use function the get the path
|
328
|
+
default_path = Path(
|
329
|
+
f"{family}-{clean_version(self.mpy_version, flat=True)}-{self.port}-{default_board(self.port, self.mpy_version)}-merged"
|
330
|
+
)
|
331
|
+
if (CONFIG.stub_path / default_path).exists():
|
332
|
+
fw_path = default_path
|
333
|
+
updated_sources.append((stub_type, fw_path))
|
334
|
+
# ---------
|
317
335
|
else:
|
318
336
|
updated_sources.append((stub_type, fw_path))
|
319
337
|
return updated_sources
|
@@ -329,7 +347,7 @@ class StubPackage:
|
|
329
347
|
"""
|
330
348
|
try:
|
331
349
|
# update to -merged and fallback to GENERIC
|
332
|
-
self.stub_sources = self.update_sources(
|
350
|
+
self.stub_sources = self.update_sources()
|
333
351
|
# Check if all stub source folders exist
|
334
352
|
for stub_type, src_path in self.stub_sources:
|
335
353
|
if not (CONFIG.stub_path / src_path).exists():
|
@@ -459,6 +477,7 @@ class StubPackage:
|
|
459
477
|
try:
|
460
478
|
with open(CONFIG.template_path / "pyproject.toml", "rb") as f:
|
461
479
|
_pyproject = tomllib.load(f)
|
480
|
+
# note: can be 'latest' which is not semver
|
462
481
|
_pyproject["tool"]["poetry"]["version"] = self.mpy_version
|
463
482
|
except FileNotFoundError as e:
|
464
483
|
log.error(f"Could not find template pyproject.toml file {e}")
|
@@ -571,7 +590,7 @@ class StubPackage:
|
|
571
590
|
return False
|
572
591
|
# todo: call poetry directly to improve error handling
|
573
592
|
try:
|
574
|
-
log.
|
593
|
+
log.debug(f"poetry {parameters} starting")
|
575
594
|
subprocess.run(
|
576
595
|
["poetry"] + parameters,
|
577
596
|
cwd=self.package_path,
|
@@ -581,7 +600,7 @@ class StubPackage:
|
|
581
600
|
universal_newlines=True,
|
582
601
|
)
|
583
602
|
log.trace(f"poetry {parameters} completed")
|
584
|
-
except (NotADirectoryError, FileNotFoundError) as e: # pragma: no cover
|
603
|
+
except (NotADirectoryError, FileNotFoundError) as e: # pragma: no cover # InvalidVersion
|
585
604
|
log.error("Exception on process, {}".format(e))
|
586
605
|
return False
|
587
606
|
except subprocess.CalledProcessError as e: # pragma: no cover
|
@@ -638,24 +657,24 @@ class StubPackage:
|
|
638
657
|
Check if (all) the packages sources exist.
|
639
658
|
"""
|
640
659
|
ok = True
|
641
|
-
for
|
642
|
-
if (CONFIG.stub_path /
|
660
|
+
for stub_type, src_path in self.update_sources():
|
661
|
+
if (CONFIG.stub_path / src_path).exists():
|
643
662
|
continue
|
644
|
-
if
|
663
|
+
if stub_type == StubSource.FROZEN:
|
645
664
|
# not a blocking issue if there are no frozen stubs, perhaps this port/board does not have any
|
646
665
|
continue
|
647
666
|
# todo: below is a workaround for different types, but where is the source of this difference coming from?
|
648
667
|
msg = (
|
649
|
-
f"{self.package_name}: source '{
|
650
|
-
if isinstance(
|
651
|
-
else f"{self.package_name}: source '{
|
668
|
+
f"{self.package_name}: source '{stub_type.value}' not found: {CONFIG.stub_path / src_path}"
|
669
|
+
if isinstance(stub_type, StubSource) # type: ignore
|
670
|
+
else f"{self.package_name}: source '{stub_type}' not found: {CONFIG.stub_path / src_path}"
|
652
671
|
)
|
653
672
|
self.status["error"] = msg
|
654
673
|
log.debug(msg)
|
655
674
|
ok = False
|
656
675
|
return ok
|
657
676
|
|
658
|
-
def update_package(self) -> bool:
|
677
|
+
def update_package(self, production: bool) -> bool:
|
659
678
|
"""Update the package .pyi files, if all the sources are available"""
|
660
679
|
log.info(f"- Update {self.package_path.name}")
|
661
680
|
log.trace(f"{self.package_path.as_posix()}")
|
@@ -673,12 +692,13 @@ class StubPackage:
|
|
673
692
|
try:
|
674
693
|
self.update_package_files()
|
675
694
|
self.update_included_stubs()
|
676
|
-
|
695
|
+
# for a new package the version could be 'latest', which is not a valid semver, so update
|
696
|
+
self.pkg_version = self.next_pkg_version(production)
|
697
|
+
return self.check()
|
677
698
|
except Exception as e: # pragma: no cover
|
678
699
|
log.error(f"{self.package_name}: {e}")
|
679
700
|
self.status["error"] = str(e)
|
680
701
|
return False
|
681
|
-
return True
|
682
702
|
|
683
703
|
def build(
|
684
704
|
self,
|
@@ -699,10 +719,12 @@ class StubPackage:
|
|
699
719
|
"""
|
700
720
|
log.info(f"Build: {self.package_path.name}")
|
701
721
|
|
702
|
-
ok = self.update_package()
|
722
|
+
ok = self.update_package(production)
|
703
723
|
self.status["version"] = self.pkg_version
|
704
724
|
if not ok:
|
705
|
-
log.info(f"{self.package_name}: skip - Could not update package")
|
725
|
+
log.info(f"{self.package_name}: skip - Could not build/update package")
|
726
|
+
if not self.status["error"]:
|
727
|
+
self.status["error"] = "Could not build/update package"
|
706
728
|
return False
|
707
729
|
# If there are changes to the package, then publish it
|
708
730
|
if self.is_changed():
|
@@ -714,7 +736,7 @@ class StubPackage:
|
|
714
736
|
if self.is_changed() or force:
|
715
737
|
# Build the distribution files
|
716
738
|
old_ver = self.pkg_version
|
717
|
-
self.pkg_version = self.
|
739
|
+
self.pkg_version = self.next_pkg_version(production)
|
718
740
|
self.status["version"] = self.pkg_version
|
719
741
|
# to get the next version
|
720
742
|
log.debug(
|
@@ -768,10 +790,10 @@ class StubPackage:
|
|
768
790
|
log.debug(f"{self.package_name}: skip publishing")
|
769
791
|
return False
|
770
792
|
|
771
|
-
self.
|
793
|
+
self.next_pkg_version(production=production)
|
772
794
|
# Publish the package to PyPi, Test-PyPi or Github
|
773
795
|
if self.is_changed() or force:
|
774
|
-
if self.mpy_version == "latest":
|
796
|
+
if self.mpy_version == "latest" and production and not force:
|
775
797
|
log.warning(
|
776
798
|
"version: `latest` package will only be available on Github, and not published to PyPi."
|
777
799
|
)
|
stubber/rst/lookup.py
CHANGED
@@ -16,29 +16,15 @@ __all__ = [
|
|
16
16
|
"RST_DOC_FIXES",
|
17
17
|
"DOCSTUB_SKIP",
|
18
18
|
"U_MODULES",
|
19
|
-
"DEFAULT_BOARDS",
|
20
|
-
# "FORCE_NON_DETECED",
|
21
19
|
]
|
22
20
|
|
23
21
|
# all possible Types needed for the stubs - exxess types should be removed later , and otherwise won't do much harm
|
24
22
|
TYPING_IMPORT: List[str] = [
|
25
|
-
"from typing import IO, Any, Callable, Coroutine, Dict, Generator, Iterator, List, NoReturn, Optional, Tuple, Union, NamedTuple, TypeVar
|
23
|
+
"from typing import IO, Any, Callable, Coroutine, Dict, Generator, Iterator, List, NoReturn, Optional, Tuple, Union, NamedTuple, TypeVar",
|
26
24
|
"from _typeshed import Incomplete",
|
27
25
|
]
|
28
26
|
|
29
27
|
|
30
|
-
# The default board for the ports modules documented with base name only
|
31
|
-
# TODO: these are not yest used in the stubber logic
|
32
|
-
# ESP32-GENERIC is curently hardcoded
|
33
|
-
DEFAULT_BOARDS: Dict[str, List[str]] = {
|
34
|
-
"stm32": ["stm32-PYBV11"],
|
35
|
-
"esp32": ["esp32-GENERIC", "esp32"],
|
36
|
-
"esp8266": ["esp8266-GENERIC", "esp8266"],
|
37
|
-
"rp2": ["rp2-PICO", "rp2-PICO_W"],
|
38
|
-
"samd": ["samd-SEEED_WIO_TERMINAL"],
|
39
|
-
}
|
40
|
-
|
41
|
-
|
42
28
|
@dataclass
|
43
29
|
class Fix:
|
44
30
|
"""A fix for a parameter or return type in the documentation that is needed to render it to a valid type annotation
|
@@ -142,6 +128,9 @@ LOOKUP_LIST = {
|
|
142
128
|
"machine.Signal.value": ("int", 0.95),
|
143
129
|
"machine.soft_reset": ("NoReturn", 0.95), # never returns
|
144
130
|
"machine.UART.irq": ("Incomplete", 0.95), # no IRQ type defined
|
131
|
+
"machine.UART.write": ("Union[int,None]", 0.95),
|
132
|
+
"machine.UART.readinto": ("Union[int,None]", 0.95),
|
133
|
+
"machine.UART.readline": ("Union[str,None]", 0.95),
|
145
134
|
"math.isnan": ("bool", 0.95),
|
146
135
|
"micropython.opt_level": ("Incomplete", 0.95), # Not clear in docstring
|
147
136
|
# since 1.19 const can also be string , bytes or tuple
|
@@ -180,7 +169,7 @@ LOOKUP_LIST = {
|
|
180
169
|
"_onewire.writebit": ("None", 0.95),
|
181
170
|
"_onewire.crc8": ("int", 0.95),
|
182
171
|
# espnow
|
183
|
-
"espnow.ESPNow.recv": ("List", 0.95), # list / ? tuple of bytestrings
|
172
|
+
"espnow.ESPNow.recv": ("Union[List, Tuple[None,None]]", 0.95), # list / ? tuple of bytestrings
|
184
173
|
}
|
185
174
|
|
186
175
|
|
@@ -526,6 +515,8 @@ CHILD_PARENT_CLASS = {
|
|
526
515
|
# ESPNow
|
527
516
|
"ESPNow": "ESPNowBase,Iterator",
|
528
517
|
"AIOESPNow": "ESPNow",
|
518
|
+
# array
|
519
|
+
"array": "List",
|
529
520
|
}
|
530
521
|
|
531
522
|
|
stubber/rst/reader.py
CHANGED
@@ -219,14 +219,43 @@ class RSTReader(FileReadWriter):
|
|
219
219
|
|
220
220
|
# return _l.startswith("..") and not any(_l.startswith(a) for a in self.docstring_anchors)
|
221
221
|
|
222
|
-
@property
|
223
|
-
def at_heading(self) -> bool:
|
222
|
+
# @property
|
223
|
+
def at_heading(self, large=False) -> bool:
|
224
224
|
"stop at heading"
|
225
|
-
|
225
|
+
u_line = self.rst_text[min(self.line_no + 1, self.max_line - 1)].rstrip()
|
226
226
|
# Heading ---, ==, ~~~
|
227
|
-
|
227
|
+
underlined = (
|
228
|
+
u_line.startswith("---") or u_line.startswith("===") or u_line.startswith("~~~")
|
229
|
+
)
|
230
|
+
if underlined and self.line_no > 0:
|
231
|
+
# check if previous line is a heading
|
232
|
+
line = self.rst_text[self.line_no].strip()
|
233
|
+
if line:
|
234
|
+
# module docstrings can be a bit larger than normal
|
235
|
+
if not large and len(line) == len(u_line):
|
236
|
+
# heading is same length as underlined
|
237
|
+
# for most docstrings that is a sensible boundary
|
238
|
+
return True
|
239
|
+
line = line.split()[0]
|
240
|
+
# stopwords in headings
|
241
|
+
return line.lower() in [
|
242
|
+
"classes",
|
243
|
+
"functions",
|
244
|
+
"methods",
|
245
|
+
"constants",
|
246
|
+
"exceptions",
|
247
|
+
"constructors",
|
248
|
+
"class",
|
249
|
+
"common",
|
250
|
+
"general",
|
251
|
+
# below are tuning based on module level docstrings
|
252
|
+
"time",
|
253
|
+
"pio",
|
254
|
+
"memory",
|
255
|
+
]
|
256
|
+
return False
|
228
257
|
|
229
|
-
def read_docstring(self) -> List[str]:
|
258
|
+
def read_docstring(self, large: bool = False) -> List[str]:
|
230
259
|
"""Read a textblock that will be used as a docstring, or used to process a toc tree
|
231
260
|
The textblock is terminated at the following RST line structures/tags
|
232
261
|
.. <anchor>
|
@@ -245,7 +274,7 @@ class RSTReader(FileReadWriter):
|
|
245
274
|
while (
|
246
275
|
self.line_no < len(self.rst_text)
|
247
276
|
and not self.at_anchor # stop at next anchor ( however .. note: and a few other anchors should be added)
|
248
|
-
and not self.at_heading # stop at next heading
|
277
|
+
and not self.at_heading(large) # stop at next heading
|
249
278
|
):
|
250
279
|
line = self.rst_text[self.line_no]
|
251
280
|
block.append(line.rstrip())
|
@@ -471,7 +500,7 @@ class RSTParser(RSTReader):
|
|
471
500
|
self.current_module = module_name
|
472
501
|
self.current_function = self.current_class = ""
|
473
502
|
# get module docstring
|
474
|
-
docstr = self.read_docstring()
|
503
|
+
docstr = self.read_docstring(large=True)
|
475
504
|
|
476
505
|
if len(docstr) > 0:
|
477
506
|
# Add link to online documentation
|
@@ -479,7 +508,9 @@ class RSTParser(RSTReader):
|
|
479
508
|
if "nightly" in self.source_tag:
|
480
509
|
version = "latest"
|
481
510
|
else:
|
482
|
-
version = self.source_tag.replace(
|
511
|
+
version = self.source_tag.replace(
|
512
|
+
"_", "."
|
513
|
+
) # TODO Use clean_version(self.source_tag)
|
483
514
|
docstr[
|
484
515
|
0
|
485
516
|
] = f"{docstr[0]}.\n\nMicroPython module: https://docs.micropython.org/en/{version}/library/{module_name}.html"
|
stubber/stubs_from_docs.py
CHANGED
@@ -9,13 +9,8 @@ from typing import List, Optional
|
|
9
9
|
|
10
10
|
from loguru import logger as log
|
11
11
|
|
12
|
-
from stubber.utils.post import run_autoflake, run_black
|
13
|
-
|
14
12
|
from stubber import utils
|
15
|
-
from stubber.rst import
|
16
|
-
DOCSTUB_SKIP,
|
17
|
-
U_MODULES,
|
18
|
-
)
|
13
|
+
from stubber.rst import DOCSTUB_SKIP, U_MODULES
|
19
14
|
from stubber.rst.reader import RSTWriter
|
20
15
|
|
21
16
|
|
@@ -26,6 +21,7 @@ def generate_from_rst(
|
|
26
21
|
release: Optional[str] = None,
|
27
22
|
pattern: str = "*.rst",
|
28
23
|
suffix: str = ".py",
|
24
|
+
black: bool = True,
|
29
25
|
) -> int:
|
30
26
|
# sourcery skip: remove-redundant-exception, simplify-single-exception-tuple
|
31
27
|
if not dst_path.exists():
|
@@ -44,8 +40,9 @@ def generate_from_rst(
|
|
44
40
|
clean_destination(dst_path)
|
45
41
|
make_docstubs(dst_path, v_tag, release, suffix, files)
|
46
42
|
|
47
|
-
|
48
|
-
|
43
|
+
log.info("::group:: start post processing of retrieved stubs")
|
44
|
+
# do not run stubgen
|
45
|
+
utils.do_post_processing([dst_path], stubgen=False, black=black, autoflake=True)
|
49
46
|
|
50
47
|
# Generate a module manifest for the docstubs
|
51
48
|
utils.make_manifest(
|
stubber/utils/post.py
CHANGED
@@ -3,21 +3,22 @@ import subprocess
|
|
3
3
|
from pathlib import Path
|
4
4
|
from typing import List
|
5
5
|
|
6
|
+
import autoflake
|
6
7
|
from loguru import logger as log
|
7
8
|
|
8
9
|
from .stubmaker import generate_pyi_files
|
9
10
|
|
10
|
-
# # log = logging.getLogger(__name__)
|
11
11
|
|
12
|
-
|
13
|
-
def do_post_processing(stub_paths: List[Path], pyi: bool, black: bool):
|
12
|
+
def do_post_processing(stub_paths: List[Path], stubgen: bool, black: bool, autoflake: bool):
|
14
13
|
"Common post processing"
|
15
14
|
for path in stub_paths:
|
16
|
-
if
|
15
|
+
if stubgen:
|
17
16
|
log.debug("Generate type hint files (pyi) in folder: {}".format(path))
|
18
17
|
generate_pyi_files(path)
|
19
18
|
if black:
|
20
19
|
run_black(path)
|
20
|
+
if autoflake:
|
21
|
+
run_autoflake(path, process_pyi=True)
|
21
22
|
|
22
23
|
|
23
24
|
def run_black(path: Path, capture_output: bool = False):
|
@@ -42,41 +43,34 @@ def run_autoflake(path: Path, capture_output: bool = False, process_pyi: bool =
|
|
42
43
|
needs to be run BEFORE black otherwise it does not recognize long import from`s.
|
43
44
|
note: is run file-by-file to include processing .pyi files
|
44
45
|
"""
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
path.as_posix(),
|
53
|
-
"-v",
|
54
|
-
"-v", # show some feedback
|
55
|
-
]
|
56
|
-
log.debug("Running autoflake on: {}".format(path))
|
57
|
-
# subprocess.run(cmd, capture_output=log.level >= logging.INFO)
|
58
|
-
result = subprocess.run(cmd, capture_output=capture_output)
|
59
|
-
if result.returncode != 0: # pragma: no cover
|
60
|
-
log.warning(f"autoflake failed on: {path}")
|
61
|
-
ret = result.returncode
|
62
|
-
|
46
|
+
if not path.exists():
|
47
|
+
log.warning(f"Path does not exist: {path}")
|
48
|
+
return -1
|
49
|
+
log.info(f"Running autoflake on: {path}")
|
50
|
+
# create a list of files to be formatted
|
51
|
+
files: List[str] = []
|
52
|
+
files.extend([str(f) for f in path.rglob("*.py")])
|
63
53
|
if process_pyi:
|
64
|
-
for
|
65
|
-
cmd = [
|
66
|
-
"autoflake",
|
67
|
-
"-r",
|
68
|
-
"--in-place",
|
69
|
-
# "--remove-all-unused-imports",
|
70
|
-
# "--ignore-init-module-imports",
|
71
|
-
file.as_posix(),
|
72
|
-
"-v",
|
73
|
-
"-v", # show some feedback
|
74
|
-
]
|
75
|
-
log.debug("Running autoflake on: {}".format(path))
|
76
|
-
# subprocess.run(cmd, capture_output=log.level >= logging.INFO)
|
77
|
-
result = subprocess.run(cmd, capture_output=capture_output)
|
78
|
-
if result.returncode != 0:
|
79
|
-
log.warning(f"autoflake failed on: {file}")
|
80
|
-
ret = result.returncode
|
54
|
+
files.extend([str(f) for f in path.rglob("*.pyi")])
|
81
55
|
|
82
|
-
|
56
|
+
# build an argument list
|
57
|
+
autoflake_args = {
|
58
|
+
"write_to_stdout": False, # print changed text to stdout
|
59
|
+
"in_place": True, # make changes to files instead of printing diffs
|
60
|
+
"remove_all_unused_imports": False,
|
61
|
+
"ignore_init_module_imports": False, # exclude __init__.py when removing unused imports
|
62
|
+
"expand_star_imports": False,
|
63
|
+
"remove_duplicate_keys": False,
|
64
|
+
"remove_unused_variables": False, # remove all unused imports (not just those from the standard library)
|
65
|
+
"remove_rhs_for_unused_variables": False,
|
66
|
+
"ignore_pass_statements": False, # remove pass when superfluous
|
67
|
+
"ignore_pass_after_docstring": False, # ignore pass statements after a newline ending on '"""'
|
68
|
+
"check": False, # return error code if changes are needed
|
69
|
+
"check_diff": False,
|
70
|
+
"quiet": False,
|
71
|
+
}
|
72
|
+
# format the files
|
73
|
+
exit_status = 0
|
74
|
+
for name in files:
|
75
|
+
log.debug(f"Running autoflake on: {name}")
|
76
|
+
exit_status |= autoflake.fix_file(name, args=autoflake_args)
|
stubber/utils/repos.py
CHANGED
@@ -39,7 +39,7 @@ def switch(tag: str, *, mpy_path: Path, mpy_lib_path: Path):
|
|
39
39
|
git.switch_branch(repo=mpy_path, branch="master")
|
40
40
|
else:
|
41
41
|
git.checkout_tag(repo=mpy_path, tag=tag)
|
42
|
-
match_lib_with_mpy(version_tag=tag, lib_path=mpy_lib_path)
|
42
|
+
match_lib_with_mpy(version_tag=tag, mpy_path=mpy_path, lib_path=mpy_lib_path)
|
43
43
|
|
44
44
|
|
45
45
|
def read_micropython_lib_commits(filename: str = "data/micropython_tags.csv"):
|
@@ -62,30 +62,39 @@ def read_micropython_lib_commits(filename: str = "data/micropython_tags.csv"):
|
|
62
62
|
reader = csv.DictReader(ntf.file, skipinitialspace=True) # dialect="excel",
|
63
63
|
rows = list(reader)
|
64
64
|
# create a dict version --> commit_hash
|
65
|
-
version_commit = {
|
65
|
+
version_commit = {
|
66
|
+
row["version"].split("/")[-1]: row["lib_commit_hash"]
|
67
|
+
for row in rows
|
68
|
+
if row["version"].startswith("refs/tags/")
|
69
|
+
}
|
66
70
|
# add default
|
67
71
|
version_commit = defaultdict(lambda: "master", version_commit)
|
68
72
|
return version_commit
|
69
73
|
|
70
74
|
|
71
|
-
def match_lib_with_mpy(version_tag: str, lib_path: Path):
|
75
|
+
def match_lib_with_mpy(version_tag: str, mpy_path: Path, lib_path: Path) -> bool:
|
72
76
|
micropython_lib_commits = read_micropython_lib_commits()
|
73
77
|
# Make sure that the correct micropython-lib release is checked out
|
74
78
|
# check if micropython-lib has matching tags
|
75
79
|
if version_tag == "latest":
|
76
|
-
|
80
|
+
# micropython-lib is now a submodule
|
81
|
+
result = git.checkout_commit("master", lib_path)
|
82
|
+
if not result:
|
83
|
+
log.error("Could not checkout micropython-lib @master")
|
84
|
+
return False
|
85
|
+
|
86
|
+
return git.sync_submodules(mpy_path)
|
77
87
|
elif Version(version_tag) >= Version("v1.20.0"):
|
78
|
-
#
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
# git submodule foreach --recursive git clean -xfd
|
85
|
-
# use the micropython_tags.csv to find the correct commit hash
|
86
|
-
return git.checkout_tag(version_tag, lib_path)
|
88
|
+
# micropython-lib is now a submodule
|
89
|
+
result = git.checkout_tag(version_tag, lib_path)
|
90
|
+
if not result:
|
91
|
+
log.error("Could not checkout micropython-lib @master")
|
92
|
+
return False
|
93
|
+
return git.sync_submodules(mpy_path)
|
87
94
|
else:
|
88
|
-
log.info(
|
95
|
+
log.info(
|
96
|
+
f"Matching repo's: Micropython {version_tag} needs micropython-lib:{micropython_lib_commits[version_tag]}"
|
97
|
+
)
|
89
98
|
return git.checkout_commit(micropython_lib_commits[version_tag], lib_path)
|
90
99
|
|
91
100
|
|
@@ -107,10 +116,16 @@ def fetch_repos(tag: str, mpy_path: Path, mpy_lib_path: Path):
|
|
107
116
|
git.switch_branch(repo=mpy_path, branch="master")
|
108
117
|
else:
|
109
118
|
git.checkout_tag(repo=mpy_path, tag=tag)
|
110
|
-
result = match_lib_with_mpy(version_tag=tag, lib_path=mpy_lib_path)
|
119
|
+
result = match_lib_with_mpy(version_tag=tag, mpy_path=mpy_path, lib_path=mpy_lib_path)
|
111
120
|
|
112
|
-
log.info(f"{mpy_path} {git.get_local_tag(mpy_path)}")
|
113
|
-
log.info(f"{mpy_lib_path} {git.get_local_tag(mpy_lib_path)}")
|
121
|
+
log.info(f"{str(mpy_path):<40} {git.get_local_tag(mpy_path)}")
|
122
|
+
log.info(f"{str(mpy_lib_path):<40} {git.get_local_tag(mpy_lib_path)}")
|
123
|
+
try:
|
124
|
+
sub_mod_path = mpy_path / "lib/micropython-lib"
|
125
|
+
if (sub_mod_path / ".git").exists():
|
126
|
+
log.info(f"{str(sub_mod_path):<40} {git.get_local_tag(sub_mod_path)}")
|
127
|
+
except Exception:
|
128
|
+
pass
|
114
129
|
return result
|
115
130
|
|
116
131
|
|