micropython-stubber 1.16.2__py3-none-any.whl → 1.17.0__py3-none-any.whl

stubber/board/fw_info.py

@@ -0,0 +1,135 @@
+# %%micropython
+import os
+import sys
+
+
+def _build(s):
+    # extract a build nr from a string
+    if not s:
+        return ""
+    if " on " in s:
+        s = s.split(" on ", 1)[0]
+    return s.split("-")[1] if "-" in s else ""
+
+
+def _version_str(version: tuple):  #  -> str:
+    v_str = ".".join([str(n) for n in version[:3]])
+    if len(version) > 3 and version[3]:
+        v_str += "-" + version[3]
+    return v_str
+
+
+def _info():  # type:() -> dict[str, str]
+    # sourcery skip: use-contextlib-suppress, use-fstring-for-formatting, use-named-expression
+    info = dict(
+        {
+            "family": sys.implementation.name,
+            "version": "",
+            "build": "",
+            "ver": "",
+            "port": "stm32" if sys.platform.startswith("pyb") else sys.platform,  # port: esp32 / win32 / linux / stm32
+            "board": "GENERIC",
+            "cpu": "",
+            "mpy": "",
+            "arch": "",
+        }
+    )
+    try:
+        info["version"] = _version_str(sys.implementation.version)
+    except AttributeError:
+        pass
+    try:
+        machine = sys.implementation._machine if "_machine" in dir(sys.implementation) else os.uname().machine
+        info["board"] = machine.strip()
+        info["cpu"] = machine.split("with")[-1].strip() if "with" in machine else ""
+        info["mpy"] = (
+            sys.implementation._mpy
+            if "_mpy" in dir(sys.implementation)
+            else sys.implementation.mpy
+            if "mpy" in dir(sys.implementation)
+            else ""
+        )
+    except (AttributeError, IndexError):
+        pass
+
+    try:
+        # extract build from uname().version if available
+        info["build"] = _build(os.uname()[3])
+        if not info["build"]:
+            # extract build from uname().release if available
+            info["build"] = _build(os.uname()[2])
+        if not info["build"] and ";" in sys.version:
+            # extract build from uname().release if available
+            info["build"] = _build(sys.version.split(";")[1])
+    except (AttributeError, IndexError):
+        pass
+    # avoid  build hashes
+    if info["build"] and len(info["build"]) > 5:
+        info["build"] = ""
+
+    if info["version"] == "" and sys.platform not in ("unix", "win32"):
+        try:
+            u = os.uname()
+            info["version"] = u.release
+        except (IndexError, AttributeError, TypeError):
+            pass
+    # detect families
+    for fam_name, mod_name, mod_thing in [
+        ("pycopy", "pycopy", "const"),
+        ("pycom", "pycom", "FAT"),
+        ("ev3-pybricks", "pybricks.hubs", "EV3Brick"),
+    ]:
+        try:
+            _t = __import__(mod_name, None, None, (mod_thing))
+            info["family"] = fam_name
+            del _t
+            break
+        except (ImportError, KeyError):
+            pass
+
+    if info["family"] == "ev3-pybricks":
+        info["release"] = "2.0.0"
+
+    if info["family"] == "micropython":
+        if (
+            info["version"]
+            and info["version"].endswith(".0")
+            and info["version"] >= "1.10.0"  # versions from 1.10.0 to 1.20.0 do not have a micro .0
+            and info["version"] <= "1.19.9"
+        ):
+            # drop the .0 for newer releases
+            info["version"] = info["version"][:-2]
+
+    # spell-checker: disable
+    if "mpy" in info and info["mpy"]:  # mpy on some v1.11+ builds
+        sys_mpy = int(info["mpy"])
+        # .mpy architecture
+        arch = [
+            None,
+            "x86",
+            "x64",
+            "armv6",
+            "armv6m",
+            "armv7m",
+            "armv7em",
+            "armv7emsp",
+            "armv7emdp",
+            "xtensa",
+            "xtensawin",
+        ][sys_mpy >> 10]
+        if arch:
+            info["arch"] = arch
+        # .mpy version.minor
+        info["mpy"] = "v{}.{}".format(sys_mpy & 0xFF, sys_mpy >> 8 & 3)
+    # simple to use version[-build] string avoiding f-strings for backward compat
+    info["ver"] = (
+        "v{version}-{build}".format(version=info["version"], build=info["build"])
+        if info["build"]
+        else "v{version}".format(version=info["version"])
+    )
+
+    return info
+
+
+print(_info())
+# del _info, _build, _version_str

stubber/board/modulelist.txt

@@ -1,5 +1,4 @@
-# list of modules to stub.
-# used by the memory optimised createstubs_mem.py
+# list of modules to stub used by the memory optimised createstubs_mem.py
 WM8960
 _OTA
 _asyncio

stubber/codemod/_partials/__init__.py

@@ -15,7 +15,7 @@ def _read_partial(path: Path) -> Iterator[str]:
     Read a partial from the file at `path`
     and yield only the lines between the ###PARTIAL### and ###PARTIALEND### markers
     """
-    lines = deque(path.read_text().splitlines(keepends=True))
+    lines = deque(path.read_text(encoding="utf-8").splitlines(keepends=True))
     _start = False
     _end = False
     # todo: allow processing of files that do not have the markers

stubber/codemod/_partials/db_main.py

@@ -3,20 +3,30 @@ This file contains the `def main()` funcion for the db variant of createstubs.py
 - type_check_only is used to avoid circular imports
 The partial is enclosed in ###PARTIAL### and ###PARTIALEND### markers
 """
+# sourcery skip: require-parameter-annotation, for-append-to-extend, use-named-expression
 
 from io import TextIOWrapper
 from typing import TYPE_CHECKING, List, type_check_only
 
-# sourcery skip: require-parameter-annotation
 if TYPE_CHECKING:
+    import gc
     import logging
     import sys
 
-    @type_check_only
+    class logging:
+        def getLogger(self, name: str) -> "logging":
+            ...
+
+        def info(self, msg: str) -> None:
+            ...
+
+    log = logging()
+
     class Stubber:
         path: str
         _report: List[str]
         modules = []
+        _json_name: str
 
         def __init__(self, path: str = "", firmware_id: str = "") -> None:
             ...
@@ -27,114 +37,122 @@ if TYPE_CHECKING:
         def create_one_stub(self, modulename: str) -> bool:
             ...
 
-        def report(self, filename: str = "modules.json"):
+        def report_start(self, filename: str = "modules.json"):
             ...
 
-        def write_json_header(self, f: TextIOWrapper):
-            ...
-
-        def write_json_node(self, f: TextIOWrapper, n, first=False):
-            ...
-
-        def write_json_end(self, f):
+        def report_end(self):
             ...
 
         def create_all_stubs(self):
             ...
 
-    @type_check_only
     def read_path() -> str:
         ...
 
-    @type_check_only
     class _gc:
         def collect(self) -> None:
             ...
 
     gc: _gc
-    _log = logging.getLogger("stubber")
+    log = logging.getLogger("stubber")
+
+    def file_exists(filename: str) -> bool:
+        ...
 
     LIBS = [".", "lib"]
 
 
 ###PARTIAL###
-def main():
-    import machine  # type: ignore
+SKIP_FILE = "modulelist.done"
+
+
+def get_modules(skip=0):
+    # new
+    for p in LIBS:
+        fname = p + "/modulelist.txt"
+        if not file_exists(fname):
+            continue
+        try:
+            with open(fname) as f:
+                i = 0
+                while True:
+                    line = f.readline().strip()
+                    if not line:
+                        break
+                    if len(line) > 0 and line[0] == "#":
+                        continue
+                    i += 1
+                    if i < skip:
+                        continue
+                    yield line
+                break
+        except OSError:
+            pass
 
+
+def write_skip(done):
+    # write count of modules already processed to file
+    with open(SKIP_FILE, "w") as f:
+        f.write(str(done) + "\n")
+
+
+def read_skip():
+    # read count of modules already processed from file
+    done = 0
     try:
-        f = open("modulelist.done", "r+b")
-        was_running = True
-        print("Opened existing db")
+        with open(SKIP_FILE) as f:
+            done = int(f.readline().strip())
     except OSError:
-        f = open("modulelist.done", "w+b")
-        print("created new db")
-        was_running = False
+        pass
+    return done
+
+
+def main():
+    import machine  # type: ignore
+
+    was_running = file_exists(SKIP_FILE)
+    if was_running:
+        log.info("Continue from last run")
+    else:
+        log.info("Starting new run")
+    # try:
+    #     f = open("modulelist.done", "r+b")
+    #     was_running = True
+    #     print("Continue from last run")
+    # except OSError:
+    #     f = open("modulelist.done", "w+b")
+    #     was_running = False
     stubber = Stubber(path=read_path())
 
     # f_name = "{}/{}".format(stubber.path, "modules.json")
+    skip = 0
     if not was_running:
         # Only clean folder if this is a first run
         stubber.clean()
-    # get list of modules to process
-    get_modulelist(stubber)
-    # remove the ones that are already done
-    modules_done = {}  # type: dict[str, str]
-    try:
-        with open("modulelist.done") as f:
-            # not optimal , but works on mpremote and esp8266
-            for line in f.read().split("\n"):
-                line = line.strip()
-                gc.collect()
-                if len(line) > 0:
-                    key, value = line.split("=", 1)
-                    modules_done[key] = value
-    except (OSError, SyntaxError):
-        pass
-    gc.collect()
-    # see if we can continue from where we left off
-    modules = [m for m in stubber.modules if m not in modules_done.keys()]
-    gc.collect()
-    for modulename in modules:
+        stubber.report_start("modules.json")
+    else:
+        skip = read_skip()
+        stubber._json_name = "{}/{}".format(stubber.path, "modules.json")
+
+    for modulename in get_modules(skip):
         # ------------------------------------
         # do epic shit
         # but sometimes things fail / run out of memory and reboot
-        ok = False
         try:
-            ok = stubber.create_one_stub(modulename)
+            stubber.create_one_stub(modulename)
         except MemoryError:
             # RESET AND HOPE THAT IN THE NEXT CYCLE WE PROGRESS FURTHER
             machine.reset()
         # -------------------------------------
         gc.collect()
-        modules_done[modulename] = str(stubber._report[-1] if ok else "failed")
-        with open("modulelist.done", "a") as f:
-            f.write("{}={}\n".format(modulename, "ok" if ok else "failed"))
-
-    # Finished processing - load all the results , and remove the failed ones
-    if modules_done:
-        # stubber.write_json_end(mod_fp)
-        stubber._report = [v for _, v in modules_done.items() if v != "failed"]
-        stubber.report()
-
-
-def get_modulelist(stubber):
-    stubber.modules = []  # avoid duplicates
-    for p in LIBS:
-        try:
-            with open(p + "/modulelist.txt") as f:
-                print("DEBUG: list of modules: " + p + "/modulelist.txt")
-                for line in f.read().split("\n"):
-                    line = line.strip()
-                    if len(line) > 0 and line[0] != "#":
-                        stubber.modules.append(line)
-                gc.collect()
-                break
-        except OSError:
-            pass
-    if not stubber.modules:
-        stubber.modules = ["micropython"]
-        _log.warn("Could not find modulelist.txt, using default modules")
-    gc.collect()
+        # modules_done[modulename] = str(stubber._report[-1] if ok else "failed")
+        # with open("modulelist.done", "a") as f:
+        #     f.write("{}={}\n".format(modulename, "ok" if ok else "failed"))
+        skip += 1
+        write_skip(skip)
+
+    print("All modules have been processed, Finalizing report")
+    stubber.report_end()
 
 
 ###PARTIALEND###

stubber/codemod/_partials/modules_reader.py

@@ -30,6 +30,9 @@ if TYPE_CHECKING:
     def read_path() -> str:
         ...
 
+    def file_exists(f: str) -> bool:
+        ...
+
     @type_check_only
     class _gc:
         def collect(self) -> None:
@@ -41,28 +44,37 @@ if TYPE_CHECKING:
     stubber: Stubber = None  # type: ignore
     LIBS: List[str] = [".", "lib"]
 
+
 # sourcery skip: use-named-expression
 ###PARTIAL###
 # Read stubs from modulelist in the current folder or in /libs
 # fall back to default modules
-stubber.modules = []  # avoid duplicates
-for p in LIBS:
-    try:
-        _1 = gc.mem_free()  # type: ignore
-        with open(p + "/modulelist.txt") as f:
-            print("Debug: List of modules: " + p + "/modulelist.txt")
-            line = f.readline()
-            while line:
-                line = line.strip()
+def get_modulelist(stubber):
+    # new
+    gc.collect()
+    stubber.modules = []  # avoid duplicates
+    for p in LIBS:
+        fname = p + "/modulelist.txt"
+        if not file_exists(fname):
+            continue
+        with open(fname) as f:
+            # print("DEBUG: list of modules: " + p + "/modulelist.txt")
+            while True:
+                line = f.readline().strip()
+                if not line:
+                    break
                 if len(line) > 0 and line[0] != "#":
                     stubber.modules.append(line)
-                line = f.readline()
-            gc.collect()  # type: ignore
-            print("Debug: Used memory to load modulelist.txt: " + str(_1 - gc.mem_free()) + " bytes")  # type: ignore
+            gc.collect()
+            print("BREAK")
             break
-    except Exception:
-        pass
-if not stubber.modules:
-    stubber.modules = ["micropython"]
-    print("Could not find modulelist.txt, using default modules")
+
+    if not stubber.modules:
+        stubber.modules = ["micropython"]
+        # _log.warn("Could not find modulelist.txt, using default modules")
+    gc.collect()
+
+
+stubber.modules = []  # avoid duplicates
+get_modulelist(stubber)
 ###PARTIALEND###

stubber/codemod/board.py

@@ -12,10 +12,10 @@ from libcst import matchers as m
 from libcst.codemod._codemod import Codemod  # type: ignore
 from packaging.version import Version
 
+from stubber import __version__
 from stubber.codemod._partials import Partial
 from stubber.codemod.modify_list import ListChangeSet, ModifyListElements
 from stubber.cst_transformer import update_module_docstr
-from stubber import __version__
 
 # matches on `stubber = Stubber()`
 _STUBBER_MATCHER = m.Assign(
@@ -146,9 +146,7 @@ class ModuleDocCodemod(codemod.Codemod):
     def __init__(self, context: codemod.CodemodContext, module_doc: str):
         super().__init__(context)
         if module_doc.endswith('"""\n'):
-            generated = (
-                f'\nThis variant was generated from createstubs.py by micropython-stubber v{Version(__version__).base_version}\n"""\n'
-            )
+            generated = f'\nThis variant was generated from createstubs.py by micropython-stubber v{Version(__version__).base_version}\n"""\n'
             module_doc = module_doc[:-4] + generated
         self.module_doc = module_doc
 

stubber/codemod/enrich.py

@@ -74,7 +74,7 @@ def enrich_file(
 
     log.debug(f"Merge {target_path} from {docstub_file}")
     # read source file
-    old_code = target_path.read_text()
+    old_code = target_path.read_text(encoding="utf-8")
 
     codemod_instance = merge_docstub.MergeCommand(context, docstub_file=docstub_file)
     if not (

stubber/commands/build_cmd.py

@@ -22,7 +22,7 @@ from stubber.utils.config import CONFIG
     multiple=True,
     default=[CONFIG.stable_version],
     show_default=True,
-    help="multiple: ",
+    help="MicroPython version to build, or 'preview' for the latest preview version",
 )
 @click.option(
     "--port",
@@ -72,9 +72,11 @@ def cli_build(
     ports = list(ports)
     boards = list(boards)
 
-    if len(versions) > 1 :
-        raise NotImplementedError("Multiple versions are not supported yet\n See https://github.com/Josverl/micropython-stubber/issues/487")
-    
+    if len(versions) > 1:
+        raise NotImplementedError(
+            "Multiple versions are not supported yet\n See https://github.com/Josverl/micropython-stubber/issues/487"
+        )
+
     # db = get_database(publish_path=CONFIG.publish_path, production=production)
     log.info(f"Build {family} {versions} {ports} {boards}")
 

stubber/commands/get_docstubs_cmd.py

@@ -39,9 +39,7 @@ from .cli import stubber_cli
     show_default=True,
 )
 #  @click.option("--family", "-f", "basename", default="micropython", help="Micropython family.", show_default=True)
-@click.option(
-    "--version", "--tag", default="", type=str, help="Version number to use. [default: Git tag]"
-)
+@click.option("--version", "--tag", default="", type=str, help="Version number to use. [default: Git tag]")
 @click.option("--black/--no-black", "-b/-nb", default=True, help="Run black", show_default=True)
 @click.pass_context
 def cli_docstubs(
@@ -72,16 +70,13 @@ def cli_docstubs(
         result = fetch_repos(version, CONFIG.mpy_path, CONFIG.mpy_lib_path)
         if not result:
             return -1
+    # get the current checked out version
+    version = utils.checkedout_version(CONFIG.mpy_path)
 
-    v_tag = git.get_local_tag(rst_path.as_posix())
-    if not v_tag:
-        # if we can't find a tag , bail
-        raise ValueError("No valid Tag found")
-    v_tag = utils.clean_version(v_tag, flat=True, drop_v=False)
     release = git.get_local_tag(rst_path.as_posix(), abbreviate=False) or ""
 
-    dst_path = Path(target) / f"{basename}-{v_tag}-docstubs"
+    dst_path = Path(target) / f"{basename}-{version}-docstubs"
 
-    log.info(f"Get docstubs for MicroPython {utils.clean_version(v_tag, drop_v=False)}")
-    generate_from_rst(rst_path, dst_path, v_tag, release=release, suffix=".pyi",black=black)
+    log.info(f"Get docstubs for MicroPython {utils.clean_version(version, drop_v=False)}")
+    generate_from_rst(rst_path, dst_path, version, release=release, suffix=".pyi", black=black)
     log.info("::group:: Done")

stubber/commands/get_frozen_cmd.py

@@ -36,6 +36,7 @@ from .cli import stubber_cli
     default="",
     # default=[CONFIG.stable_version],
     show_default=True,
+    help="The version of MicroPython to get the frozen modules for. Use 'preview' to get the latest version from the micropython repo",
 )
 @click.option(
     "--stubgen/--no-stubgen",
@@ -78,27 +79,21 @@ def cli_get_frozen(
     else:
         version = utils.clean_version(git.get_local_tag(CONFIG.mpy_path.as_posix()) or "0.0")
     if not version:
-        log.warning(
-            "Unable to find the micropython repo in folder : {}".format(CONFIG.mpy_path.as_posix())
-        )
+        log.warning("Unable to find the micropython repo in folder : {}".format(CONFIG.mpy_path.as_posix()))
 
     log.info("MicroPython version : {}".format(version))
     # folder/{family}-{version}-frozen
     family = "micropython"
-    stub_path = Path(stub_folder) / f"{family}-{utils.clean_version(version, flat=True)}-frozen"
+
+    stub_path = freeze_any(version=version, mpy_path=CONFIG.mpy_path, mpy_lib_path=CONFIG.mpy_lib_path)
     stub_paths.append(stub_path)
-    freeze_any(
-        stub_path, version=version, mpy_path=CONFIG.mpy_path, mpy_lib_path=CONFIG.mpy_lib_path
-    )
     # Also enrich the frozen modules from the doc stubs if available
 
     # first create .pyi files so they can be enriched
     utils.do_post_processing(stub_paths, stubgen=stubgen, black=False, autoflake=False)
     family = "micropython"
-    docstubs_path = (
-        Path(CONFIG.stub_path)
-        / f"{family}-{utils.clean_version(version, drop_v=False, flat=True)}-docstubs"
-    )
+    _version = utils.clean_version(version, drop_v=False, flat=True)
+    docstubs_path = Path(CONFIG.stub_path) / f"{family}-{_version}-docstubs"
     if docstubs_path.exists():
         log.info(f"Enriching {str(stub_path)} with {docstubs_path}")
         if merged := enrich_folder(

stubber/commands/switch_cmd.py

@@ -11,6 +11,7 @@ import click
 import stubber.basicgit as git
 from stubber.utils.config import CONFIG
 from stubber.utils.repos import fetch_repos, repo_paths
+from stubber.utils.versions import SET_PREVIEW, V_PREVIEW
 
 from .cli import stubber_cli
 
@@ -22,10 +23,10 @@ from .cli import stubber_cli
 # get version list from Git tags in the repo that is provided on the command line
 
 try:
-    VERSION_LIST = git.get_tags("micropython/micropython", minver="v1.9.3") + ["latest"]
+    VERSION_LIST = git.get_tags("micropython/micropython", minver="v1.9.3") + [V_PREVIEW, "latest"]
 except Exception:
     # offline fallback
-    VERSION_LIST = ["v1.91.1", "v1.20.0", "latest"]
+    VERSION_LIST = ["v1.91.1", "v1.20.1", "v1.21.0", "v1.22.1", "preview"]
 
 
 @stubber_cli.command(name="switch")
@@ -46,7 +47,8 @@ def cli_switch(path: Union[str, Path], tag: Optional[str] = None):
         mpy_path, mpy_lib_path = repo_paths(Path(path))
     except Exception:
         return -1
-    if not tag:
-        tag = "latest"
+    if not tag or tag in SET_PREVIEW:
+        tag = V_PREVIEW
+
     result = fetch_repos(tag, mpy_path, mpy_lib_path)
     return -1 if result else 0