micropython-stubber 1.23.2__py3-none-any.whl → 1.24.0__py3-none-any.whl

stubber/commands/variants_cmd.py

@@ -4,11 +4,11 @@ from pathlib import Path
 
 import rich_click as click
 from mpflash.logger import log
+
+import stubber
+from stubber.commands.cli import stubber_cli
 from stubber.utils.config import CONFIG
 from stubber.variants import create_variants
-import stubber
-
-from .cli import stubber_cli
 
 
 @click.option(

stubber/cst_transformer.py

@@ -1,7 +1,7 @@
 """helper functions for stub transformations"""
 
 # sourcery skip: snake-case-functions
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
 import libcst as cst
@@ -9,14 +9,27 @@ import libcst as cst
 
 @dataclass
 class TypeInfo:
-    "contains the functiondefs and classdefs info read from the stubs source"
+    "contains the functionDefs and classDefs info read from the stubs source"
     name: str
     decorators: Sequence[cst.Decorator]
     params: Optional[cst.Parameters] = None
     returns: Optional[cst.Annotation] = None
     docstr_node: Optional[cst.SimpleStatementLine] = None
     def_node: Optional[Union[cst.FunctionDef, cst.ClassDef]] = None
-    def_type: str = "?"  # funcdef or classdef or module
+    def_type: str = "?"  # funcDef or classDef or module
+
+
+@dataclass
+class AnnoValue:
+    "The different values for the annotations"
+    docstring: Optional[str] = ""  # strings
+    "Module docstring or function/method docstring"
+    docstring_node: Optional[cst.SimpleStatementLine] = None
+    "the docstring node for a function method to reuse with overloads"
+    type_info: Optional[TypeInfo] = None  # simple type
+    "function/method or class definition read from the docstub source"
+    overloads: List[TypeInfo] = field(default_factory=list)
+    "function / method overloads read from the docstub source"
 
 
 class TransformError(Exception):
@@ -27,7 +40,7 @@ class TransformError(Exception):
 
 
 MODULE_KEY = ("__module",)
-MODDOC_KEY = ("__module_docstring",)
+MOD_DOCSTR_KEY = ("__module_docstring",)
 
 # debug helper
 _m = cst.parse_module("")
@@ -44,17 +57,18 @@ class StubTypingCollector(cst.CSTVisitor):
         # store the annotations
         self.annotations: Dict[
             Tuple[str, ...],  # key: tuple of canonical class/function name
-            Union[TypeInfo, str],
+            AnnoValue,  # The TypeInfo or list of TypeInfo
         ] = {}
-        self.comments :List[str] = []
+        self.comments: List[str] = []
 
     # ------------------------------------------------------------
     def visit_Module(self, node: cst.Module) -> bool:
         """Store the module docstring"""
         docstr = node.get_docstring()
         if docstr:
-            self.annotations[MODULE_KEY] = docstr
+            self.annotations[MODULE_KEY] = AnnoValue(docstring=docstr)
         return True
+
     def visit_Comment(self, node: cst.Comment) -> None:
         """
         connect comments from the source
@@ -82,7 +96,7 @@ class StubTypingCollector(cst.CSTVisitor):
             def_type="classdef",
             def_node=node,
         )
-        self.annotations[tuple(self.stack)] = ti
+        self.annotations[tuple(self.stack)] = AnnoValue(type_info=ti)
 
     def leave_ClassDef(self, original_node: cst.ClassDef) -> None:
         """remove the class name from the stack"""
@@ -105,7 +119,14 @@ class StubTypingCollector(cst.CSTVisitor):
             def_type="funcdef",
             def_node=node,
         )
-        self.annotations[tuple(self.stack)] = ti
+        key = tuple(self.stack)
+        if not key in self.annotations:
+            # store the first function/method signature
+            self.annotations[key] = AnnoValue(type_info=ti)
+
+        if any(dec.decorator.value == "overload" for dec in node.decorators):  # type: ignore
+            # and store the overloads
+            self.annotations[key].overloads.append(ti)
 
     def update_append_first_node(self, node):
         """Store the function/method docstring or function/method sig"""
@@ -123,17 +144,31 @@ class StubTypingCollector(cst.CSTVisitor):
 
 def update_def_docstr(
     dest_node: Union[cst.FunctionDef, cst.ClassDef],
-    src_comment: Optional[cst.SimpleStatementLine],
+    src_docstr: Optional[Union[cst.SimpleStatementLine, str]] = None,
     src_node=None,
 ) -> Any:
     """
     Update the docstring of a function/method or class
+    The supplied `src_docstr` can be a string or a SimpleStatementLine
 
-    for functiondefs ending in an ellipsis, the entire body needs to be replaced.
-    in this case the src_body is mandatory.
+    for function defs ending in an ellipsis, the entire body needs to be replaced.
+    in this case `src_node` is required.
     """
-    if not src_comment:
+    if not src_docstr:
         return dest_node
+    if isinstance(src_docstr, str):
+        if not src_docstr[0] in ('"', "'"):
+            src_docstr = f'"""{src_docstr}"""'
+        # convert the string to a SimpleStatementLine
+        src_docstr = cst.SimpleStatementLine(
+            body=[
+                cst.Expr(
+                    value=cst.SimpleString(
+                        value=src_docstr,
+                    ),
+                ),
+            ]
+        )
 
     # function def on a single line ending with an ellipsis (...)
     if isinstance(dest_node.body, cst.SimpleStatementSuite):
@@ -144,13 +179,13 @@ def update_def_docstr(
         raise TransformError("Expected Def with Indented body")
 
     # classdef of functiondef with an indented body
-    # need some funcky casting to avoid issues with changing the body
-    # note : indented body is nested : body.body
+    # need some funky casting to avoid issues with changing the body
+    # note : indented body is nested : IndentedBlock.body.body
     if dest_node.get_docstring() is None:
         # append the new docstring and append the function body
-        body = tuple([src_comment] + list(dest_node.body.body))
+        body = tuple([src_docstr] + list(dest_node.body.body))
     else:
-        body = tuple([src_comment] + list(dest_node.body.body[1:]))
+        body = tuple([src_docstr] + list(dest_node.body.body[1:]))
     body_2 = dest_node.body.with_changes(body=body)
 
     return dest_node.with_changes(body=body_2)

stubber/freeze/common.py

@@ -59,14 +59,30 @@ def apply_frozen_module_fixes(freeze_path: Path, mpy_path: Path):
         with open(freeze_path / "umqtt" / "__init__.py", "a") as f:
             f.write("")
 
-    # NOTE: FIX 2 compensate for expicitly omited task.py from freeze manifest
-    # this is normally implemented as a C module, let's use the .py version to generate a stub for this
-    if (freeze_path / "uasyncio").exists() and not (freeze_path / "uasyncio" / "task.py").exists():
-        # copy task.py from micropython\extmod\uasyncio\task.py to stub_folder
-        log.debug("add missing : uasyncio/task.py")
-        task_py = mpy_path / "extmod" / "uasyncio" / "task.py"
-        try:
-            shutil.copy(str(task_py), str(freeze_path / "uasyncio"))
-        except OSError as er:
-            log.warning(f"error copying {task_py} : {er}")
-            # try to continue
+    # # NOTE: FIX 2 compensate for expicitly omited task.py from freeze manifest
+    # # this is normally implemented as a C module, let's use the .py version to generate a stub for this
+    # if (freeze_path / "asyncio").exists() and not (freeze_path / "asyncio" / "task.py").exists():
+    #     # copy task.py from micropython\extmod\asyncio\task.py to stub_folder
+    #     log.debug("add missing : asyncio/task.py")
+    #     task_py = mpy_path / "extmod" / "asyncio" / "task.py"
+    #     try:
+    #         shutil.copy(str(task_py), str(freeze_path / "asyncio"))
+    #     except OSError as er:
+    #         log.warning(f"error copying {task_py} : {er}")
+    #         # try to continue
+
+    RM_FREEZE = ["asyncio", "uasyncio.py", "rp2.py"]
+    removed = []
+    for mod in RM_FREEZE:
+        module = freeze_path / mod
+        if module.exists() and module.is_dir():
+            removed.append(mod)
+            log.debug(f"remove {mod}")
+            shutil.rmtree(freeze_path / mod)
+        elif module.exists() and module.is_file():
+            log.debug(f"remove {mod}")
+            module.unlink()
+    if removed:
+        with open(freeze_path / "removed.txt", "w") as f:
+            f.write("modules removed to avoid incorrect merge effects:\n")
+            f.write("\n".join(removed))

stubber/freeze/freeze_manifest_2.py

@@ -43,13 +43,20 @@ def make_path_vars(
         raise ValueError("board path not found")
 
     # VARS must be absolute paths
-    return {
+    vars = {
         "MPY_DIR": mpy_path.absolute().as_posix(),
         "MPY_LIB_DIR": mpy_lib_path.absolute().as_posix(),
         "PORT_DIR": port_path.absolute().as_posix(),
         "BOARD_DIR": board_path.absolute().as_posix(),
     }
 
+    if board and "ARDUINO" in board:
+        log.warning(f"HACK- Adding [TOP]/lib/arduino-lib to paths: {board}")
+        # see micropython/ports/renesas-ra/boards/ARDUINO_PORTENTA_C33/mpconfigboard.mk
+        vars["ARDUINO_LIB_DIR"] = (mpy_path / "lib/arduino-lib").absolute().as_posix()
+
+    return vars
+
 
 def freeze_one_manifest_2(
     manifest: Path, frozen_stub_path: Path, mpy_path: Path, mpy_lib_path: Path, version: str

stubber/freeze/get_frozen.py

@@ -19,13 +19,14 @@ The all_stubs folder should be mapped/symlinked to the micropython_stubs/stubs r
 # - 1.11 and older - include content of /port/modules folder if it exists
 import os
 import shutil  # start moving from os & glob to pathlib
+import subprocess
 from pathlib import Path
 from typing import List, Optional
 
 from mpflash.logger import log
+from mpflash.versions import SET_PREVIEW, V_PREVIEW
 from packaging.version import Version
 
-from mpflash.versions import SET_PREVIEW, V_PREVIEW
 from stubber import utils
 from stubber.freeze.freeze_folder import freeze_folders  # Micropython < v1.12
 from stubber.freeze.freeze_manifest_2 import freeze_one_manifest_2
@@ -59,6 +60,8 @@ def add_comment_to_path(path: Path, comment: str) -> None:
     pass
 
 
+
+
 def freeze_any(
     stub_folder: Optional[Path] = None,
     version: str = V_PREVIEW,

stubber/merge_config.py

@@ -0,0 +1,111 @@
+"""
+Merge configuration for the stubber.
+defines constants and util functions to copy, update or remove type modules
+"""
+
+import shutil
+from pathlib import Path
+from typing import Final, List
+
+from mpflash.logger import log
+from stubber.rst.lookup import U_MODULES
+
+EXT: Final = [".pyi", ".py", ""]
+CP_REFERENCE_TO_DOCSTUB: Final = [
+    "asyncio",
+    "rp2/PIOASMEmit.pyi",
+    "rp2/asm_pio.pyi",
+]
+"Modules to copy from reference modules to the docstubs"
+
+
+STDLIB_MODULES: Final = [
+    "collections",
+    "io",
+    "builtins",
+    "asyncio",
+    "sys",
+    # "os",  # TODO # Do not remove `os` to allow better typing by mypy for the `os` module
+    # "ssl",  # TODO
+]
+"""Modules that should be in /stdlib"""
+# and should not be in the individual packes as that causes duplication
+
+RM_MERGED: Final = (
+    [
+        "sys",  # use auto patched version from mpy_stdlib
+        "asyncio",  # use manually patched version from  mpy_stdlib
+        "_asyncio",  # ditto
+        "uasyncio",  # ditto
+        "_rp2",  # Leave out for now , to avoid conflicts with the rp2 module
+        "pycopy_imphook",  #  pycopy only: not needed in the merged stubs
+        # "os",
+    ]
+    + STDLIB_MODULES
+    + [f"u{mod}" for mod in U_MODULES]
+)
+"Modules to remove from merged stubs, U_MODULES will be recreated later"
+
+
+def copy_type_modules(source_folder: Path, target_folder: Path, CP_REFERENCE_MODULES: List[str]):
+    log.info("Adding additional type modules:")
+    for addition in CP_REFERENCE_MODULES:
+        src = source_folder / addition
+        if src.exists():
+            if src.is_dir():
+                target = target_folder / addition
+                log.debug(f" - add {target}")
+                shutil.copytree(src, target, dirs_exist_ok=True)
+            else:
+                target = target_folder / addition
+                log.debug(f" - add {target}")
+                target.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(src, target)
+
+
+def recreate_umodules(target_folder: Path):
+    log.info("create umodules to refer to modules in the merged stubs")
+    # Just `create an import * from module` in the umodule.pyi
+    for name in U_MODULES:
+        # delete complex or simple umodule
+        uname = target_folder / f"u{name}"
+        try:
+            if uname.exists():
+                if uname.is_dir():
+                    log.debug(f" - remove {uname}")
+                    shutil.rmtree(uname)
+                else:
+                    log.debug(f" - remove {uname}")
+                    uname.unlink()
+            else:
+                uname = uname.with_suffix(".pyi")
+                if uname.exists():
+                    log.debug(f" - remove {uname}")
+                    uname.unlink()
+        except OSError as e:
+            log.error(f"Error removing {uname}: {e}")
+            continue
+
+        uname = target_folder / f"u{name}.pyi"
+        with uname.open("w") as f:
+            f.write(f"# This umodule is a MicroPython reference to {name}\n")
+            f.write(f"from {name} import *\n")
+        log.debug(f" - recreated {uname.name}")
+
+
+def remove_modules(target_folder: Path, RM_MODULES: List[str]):
+    log.info("Removing modules from the merged stubs")
+
+    for name in RM_MODULES:
+        for ext in EXT:
+            target = target_folder / f"{name}{ext}"
+            if target.exists():
+                try:
+                    if target.is_dir():
+                        log.debug(f" - remove {target}")
+                        shutil.rmtree(target)
+                    else:
+                        log.debug(f" - remove {target}")
+                        target.unlink()
+                finally:
+                    log.debug(f" - remove {target}")

stubber/minify.py

@@ -17,7 +17,6 @@ except ImportError:
     python_minifier = None
 
 from mpflash.logger import log
-
 from mpflash.versions import SET_PREVIEW, V_PREVIEW
 
 # Type Aliases for minify
@@ -372,7 +371,7 @@ def cross_compile(
         _target = get_temp_file(suffix=".mpy")
     result = pipx_mpy_cross(version, source_file, _target)
     if result.stderr and "No matching distribution found for mpy-cross==" in result.stderr:
-        log.warning(f"mpy-cross=={version} not found, using default version.")
+        log.warning(f"mpy-cross=={version} not found, using most current version.")
         result = pipx_mpy_cross(V_PREVIEW, source_file, _target)
 
     if result.returncode == 0:

stubber/publish/database.py

@@ -1,18 +1,59 @@
-"""basic interface to the json database"""
+"""Interface to the database which stores the package information"""
 
+import sqlite3
 from pathlib import Path
-from typing import Union
 
-from pysondb import PysonDB
 
+def get_database(db_path: Path, production: bool = False) -> sqlite3.Connection:
+    """
+    Open the sqlite database at the given path.
+
+    The database should be located in a subfolder `/data` of the root path.
+    The database name is determined by the production flag as `all_packages[_test].db`
+    """
+    db_path = Path(db_path)
+    if db_path.stem == "publish":
+        db_path = db_path / ".." / "data"  # go up one level to find the database
+
+    db_path = db_path / f"all_packages{'' if production else '_test'}.db"
+    if db_path.exists():
+        conn = sqlite3.connect(db_path)
 
-def get_database(publish_path: Union[Path, str], production: bool = False) -> PysonDB:
+    else:
+        print(FileNotFoundError(f"Database file not found in path: {db_path}"))
+        conn = create_database(db_path)
+
+    conn.row_factory = sqlite3.Row  # return rows as dicts
+    return conn
+
+
+def create_database(db_path: Path) -> sqlite3.Connection:
     """
-    Open the json database at the given path.
+    Create a new database at the given path.
 
-    The database should be located in a subfolder `/publish` of the root path.
-    The database name is determined by the production flag as `package_data[_test].jsondb`
+    The database should be located in a subfolder `/data` of the root path.
+    """
+    db_path = Path(db_path)
+    if not db_path.parent.exists():
+        db_path.parent.mkdir(parents=True)
+    conn = sqlite3.connect(db_path)
+    SCHEMA = """
+    CREATE TABLE "packages" (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT,
+            description TEXT,
+            mpy_version TEXT,
+            pkg_version TEXT,
+            publish BOOLEAN,
+            stub_sources TEXT, -- json string
+            path TEXT,
+            hash TEXT,
+            stub_hash TEXT,
+            port TEXT DEFAULT "",
+            board TEXT DEFAULT "",
+            variant TEXT DEFAULT ""
+        )
     """
-    publish_path = Path(publish_path)
-    db_path = publish_path / f"package_data{'' if production else '_test'}.jsondb"
-    return PysonDB(db_path.as_posix())
+    conn.execute(SCHEMA)
+    conn.commit
+    return conn

stubber/publish/merge_docstubs.py

@@ -9,9 +9,9 @@ from typing import List, Optional, Union
 from mpflash.logger import log
 
 from stubber.codemod.enrich import enrich_folder
+from stubber.merge_config import RM_MERGED, recreate_umodules, remove_modules
 from stubber.publish.candidates import board_candidates, filter_list
 from stubber.publish.defaults import GENERIC, GENERIC_L, default_board
-from stubber.publish.missing_class_methods import add_machine_pin_call
 from stubber.publish.pathnames import get_base, get_board_path, get_merged_path
 from stubber.utils.config import CONFIG
 
@@ -63,14 +63,11 @@ def merge_all_docstubs(
             log.warning(f"No docstubs found for {candidate['version']}")
             continue
         if not board_path.exists():
-            log.info(f"skipping {merged_path.name}, no MCU stubs found in {board_path}")
+            log.debug(f"skipping {merged_path.name}, no MCU stubs found in {board_path}")
             continue
         log.info(f"Merge {candidate['version']} docstubs with boardstubs to {merged_path.name}")
         try:
             result = copy_and_merge_docstubs(board_path, merged_path, doc_path)
-            # Add methods from docstubs to the MCU stubs that do not exist in the MCU stubs
-            # Add the __call__ method to the machine.Pin and pyb.Pin class
-            add_machine_pin_call(merged_path, candidate["version"])
         except Exception as e:
             log.error(f"Error parsing {candidate['version']} docstubs: {e}")
             continue
@@ -83,9 +80,10 @@ def merge_all_docstubs(
 def copy_and_merge_docstubs(fw_path: Path, dest_path: Path, docstub_path: Path):
     """
     Parameters:
-        fw_path: Path to MCU stubs (absolute path)
+        fw_path: Path to the source MCU stubs (absolute path)
         dest_path: Path to destination (absolute path)
-        mpy_version: micropython version ('1.18')
+        docstub_path: Path to docstubs
+
 
     Copy files from the firmware stub folders to the merged
     - 1 - Copy all MCU stubs to the package folder
@@ -118,19 +116,38 @@ def copy_and_merge_docstubs(fw_path: Path, dest_path: Path, docstub_path: Path):
                 if (dest_path / f.name).with_suffix(suffix).exists():
                     (dest_path / f.name).with_suffix(suffix).unlink()
 
-    # delete builtins.pyi in the package folder
-    for name in [
-        "builtins",  # creates conflicts, better removed
-        "pycopy_imphook",  # is not intended to be used directly, and has an unresolved subclass
-    ]:
-        for suffix in [".py", ".pyi"]:
-            if (dest_path / name).with_suffix(suffix).exists():  # type: ignore
-                (dest_path / name).with_suffix(suffix).unlink()  # type: ignore
+    # remove unwanted modules
+    remove_modules(dest_path, RM_MERGED)
+    # fixup the umodules
+    recreate_umodules(dest_path)
 
     # 2 - Enrich the MCU stubs with the document stubs
-    result = enrich_folder(dest_path, docstub_path=docstub_path, write_back=True)
+    result = enrich_folder(source_folder=docstub_path, target_folder=dest_path, write_back=True)
+
+    refactor_rp2_module(dest_path)
 
     # copy the docstubs manifest.json file to the package folder
     if (docstub_path / "modules.json").exists():
         shutil.copy(docstub_path / "modules.json", dest_path / "doc_stubs.json")
     return result
+
+
+def refactor_rp2_module(dest_path: Path):
+    """refactor the rp2 module to allow for submodules"""
+    rp2_file = dest_path / "rp2.pyi"
+    if not rp2_file.exists():
+        # not a rp2
+        return
+
+    log.info(f"refactor rps module stub")
+    rp2_folder = dest_path / "rp2"
+    rp2_folder.mkdir(exist_ok=True)
+    if not (rp2_folder / "__init__.pyi").exists():
+        # do not overwrite docstubs __init__.pyi
+        rp2_file.rename(rp2_folder / "__init__.pyi")
+    # copy the asm_pio.pyi file from the reference folder
+    for submod in ["rp2/asm_pio.pyi"]:
+        file = CONFIG.mpy_stubs_path / "micropython-reference" / submod
+        if file.exists():
+            shutil.copy(file, rp2_folder / file.name)
+            log.info(f" - add rp2/{ file.name}")

stubber/publish/package.py

@@ -3,15 +3,14 @@ prepare a set of stub files for publishing to PyPi
 
 """
 
+import sqlite3
 import sys
 from pathlib import Path
 from typing import Dict, Union
 
 from mpflash.logger import log
-from packaging.version import parse
-from pysondb import PysonDB
-
 from mpflash.versions import clean_version
+
 from stubber.publish.defaults import GENERIC, GENERIC_L, default_board
 from stubber.publish.enums import StubSource
 from stubber.publish.stubpackage import StubPackage, StubSources
@@ -33,7 +32,7 @@ def package_name(*, port: str = "", board: str = "", family: str = "micropython"
 
 
 def get_package(
-    db: PysonDB,
+    db_conn: sqlite3.Connection,
     *,
     version: str,
     port: str,
@@ -44,13 +43,20 @@ def get_package(
     pkg_name = package_name(port=port, board=board, family=family)
     version = clean_version(version, drop_v=True)
     if package_info := get_package_info(
-        db,
+        db_conn,
         CONFIG.publish_path,
         pkg_name=pkg_name,
         mpy_version=version,
     ):
         # create package from the information retrieved from the database
-        return StubPackage(pkg_name, port, board=board, version=version, json_data=package_info)
+        p_db =  StubPackage(
+            pkg_name,
+            port,
+            board=board,
+            version=version,
+            json_data=package_info,
+        )
+        return p_db
 
     log.debug(f"No package found for {pkg_name} in database, creating new package")
     return create_package(
@@ -63,29 +69,37 @@ def get_package(
 
 
 def get_package_info(
-    db: PysonDB, pub_path: Path, *, pkg_name: str, mpy_version: str
+    db_conn: sqlite3.Connection,
+    pub_path: Path,
+    *,
+    pkg_name: str,
+    mpy_version: str,
 ) -> Union[Dict, None]:
     """
     get a package's record from the json db if it can be found
-    matches om the package name and version
+    matches on the package name and version
         pkg_name: package name (micropython-esp32-stubs)
         mpy_version: micropython/firmware version (1.18)
     """
     # find in the database
-    recs = db.get_by_query(
-        query=lambda x: x["mpy_version"] == mpy_version and x["name"] == pkg_name
+
+    cursor = db_conn.cursor()
+    cursor.execute(
+        """
+        SELECT * FROM packages 
+        WHERE name = ? AND mpy_version LIKE ?
+        ORDER BY pkg_version DESC
+    """,
+        (pkg_name, f"{mpy_version}%"),
     )
-    # dict to list
-    recs = [{"id": key, "data": recs[key]} for key in recs]
-    # sort
-    packages = sorted(recs, key=lambda x: parse(x["data"]["pkg_version"]))
+    packages = [dict(row) for row in cursor.fetchall()]
+
+    if packages:
+        pkg_from_db = packages[0]
 
-    if len(packages) > 0:
-        pkg_from_db = packages[-1]["data"]
         log.debug(f"Found latest {pkg_name} == {pkg_from_db['pkg_version']}")
         return pkg_from_db
-    else:
-        return None
+    return None
 
 
 def create_package(