micropython-stubber 1.24.1__py3-none-any.whl → 1.24.4__py3-none-any.whl

stubber/commands/get_docstubs_cmd.py

@@ -5,15 +5,17 @@ get-docstubs
 
 from pathlib import Path
 from typing import Optional
+
+import rich_click as click
 from packaging.version import Version
 
 import mpflash.basicgit as git
-import rich_click as click
+import stubber.utils as utils
 from mpflash.logger import log
-
 from stubber.codemod.enrich import enrich_folder
-import stubber.utils as utils
 from stubber.commands.cli import stubber_cli
+from stubber.merge_config import copy_type_modules
+from stubber.modcat import CP_REFERENCE_TO_DOCSTUB
 from stubber.stubs_from_docs import generate_from_rst
 from stubber.utils.config import CONFIG
 from stubber.utils.repos import fetch_repos
@@ -65,9 +67,25 @@ from stubber.utils.repos import fetch_repos
     "--enrich",
     is_flag=True,
     default=False,
-    help="Enrich with type information from micropython-reference",
+    help="Enrich with type information from reference/micropython",
     show_default=True,
 )
+# @click.option(
+#     "--copy-params",
+#     "copy_params",
+#     default=False,
+#     help="Copy the function/method parameters",
+#     show_default=True,
+#     is_flag=True,
+# )
+# @click.option(
+#     "--copy-docstr",
+#     "copy_docstr",
+#     default=False,
+#     help="Copy the docstrings",
+#     show_default=False,
+#     is_flag=True,
+# )
 @click.pass_context
 def cli_docstubs(
     ctx: click.Context,
@@ -126,19 +144,19 @@ def cli_docstubs(
 
     if enrich:
         if Version(version) < Version("1.24"):
-            log.warning(f"Enriching is not supported for version {version}")
+            log.warning(f"Enriching is only supported for version v1.24+, not {version}")
         else:
-            # !stubber enrich --params-only --source {reference} --dest {docstubs}
-            reference_path = CONFIG.stub_path.parent / "micropython-reference"
-            log.info(f"Enriching from {reference_path}")
+            reference_path = CONFIG.stub_path.parent / "reference/micropython"
             _ = enrich_folder(
                 reference_path,
                 dst_path,
                 show_diff=False,
                 write_back=True,
                 require_docstub=False,
-                params_only=True,
+                copy_params=True,
+                copy_docstr=False,
             )
+            copy_type_modules(reference_path, dst_path, CP_REFERENCE_TO_DOCSTUB)
             log.info("::group:: start post processing of retrieved stubs")
             # do not run stubgen
             utils.do_post_processing([dst_path], stubgen=False, black=black, autoflake=autoflake)

stubber/commands/get_frozen_cmd.py

@@ -109,6 +109,7 @@ def cli_get_frozen(
                 show_diff=False,
                 write_back=True,
                 require_docstub=False,
+                copy_docstr=True,
                 ext = ".pyi",
             ):
                 log.info(f" > Enriched {merged} frozen modules.")

stubber/commands/merge_cmd.py

@@ -5,8 +5,8 @@ enrich machinestubs with docstubs
 from typing import List, Union
 
 import rich_click as click
-from mpflash.logger import log
 
+from mpflash.logger import log
 from stubber.commands.cli import stubber_cli
 from stubber.publish.merge_docstubs import merge_all_docstubs
 from stubber.publish.package import GENERIC_L
@@ -60,6 +60,4 @@ def cli_merge_docstubs(
         versions = list(versions)
     # single version should be a string
     log.info(f"Merge docstubs for {family} {versions}")
-    _ = merge_all_docstubs(
-        versions=versions, family=family, boards=boards, ports=ports, mpy_path=CONFIG.mpy_path
-    )
+    _ = merge_all_docstubs(versions=versions, family=family, boards=boards, ports=ports)

stubber/merge_config.py

@@ -1,6 +1,6 @@
 """
 Merge configuration for the stubber.
-defines constants and util functions to copy, update or remove type modules
+util functions to copy, update or remove type modules
 """
 
 import shutil
@@ -8,43 +8,11 @@ from pathlib import Path
 from typing import Final, List
 
 from mpflash.logger import log
-from stubber.rst.lookup import U_MODULES
 
-EXT: Final = [".pyi", ".py", ""]
-CP_REFERENCE_TO_DOCSTUB: Final = [
-    "asyncio",
-    "rp2/PIOASMEmit.pyi",
-    "rp2/asm_pio.pyi",
-]
-"Modules to copy from reference modules to the docstubs"
-
-
-STDLIB_MODULES: Final = [
-    "collections",
-    "io",
-    "builtins",
-    "asyncio",
-    "sys",
-    # "os",  # TODO # Do not remove `os` to allow better typing by mypy for the `os` module
-    # "ssl",  # TODO
-]
-"""Modules that should be in /stdlib"""
-# and should not be in the individual packes as that causes duplication
+from stubber.modcat import (CP_REFERENCE_TO_DOCSTUB, RM_MERGED,
+                            STDLIB_ONLY_MODULES, U_MODULES)
 
-RM_MERGED: Final = (
-    [
-        "sys",  # use auto patched version from mpy_stdlib
-        "asyncio",  # use manually patched version from  mpy_stdlib
-        "_asyncio",  # ditto
-        "uasyncio",  # ditto
-        "_rp2",  # Leave out for now , to avoid conflicts with the rp2 module
-        "pycopy_imphook",  #  pycopy only: not needed in the merged stubs
-        # "os",
-    ]
-    + STDLIB_MODULES
-    + [f"u{mod}" for mod in U_MODULES]
-)
-"Modules to remove from merged stubs, U_MODULES will be recreated later"
+EXT: Final = [".pyi", ".py", ""]
 
 
 def copy_type_modules(source_folder: Path, target_folder: Path, CP_REFERENCE_MODULES: List[str]):
@@ -109,3 +77,4 @@ def remove_modules(target_folder: Path, RM_MODULES: List[str]):
                         target.unlink()
                 finally:
                     log.debug(f" - remove {target}")
+

stubber/minify.py

@@ -370,8 +370,8 @@ def cross_compile(
         # target must be a Path object
         _target = get_temp_file(suffix=".mpy")
     result = pipx_mpy_cross(version, source_file, _target)
-    if result.stderr and "No matching distribution found for mpy-cross==" in result.stderr:
-        log.warning(f"mpy-cross=={version} not found, using most current version.")
+    if result.stderr and "No matching distribution found for mpy-cross~=" in result.stderr:
+        log.warning(f"mpy-cross~={version} not found, using most current version.")
         result = pipx_mpy_cross(V_PREVIEW, source_file, _target)
 
     if result.returncode == 0:
@@ -395,7 +395,7 @@ def pipx_mpy_cross(version: str, source_file, _target):
     if version in SET_PREVIEW:
         version = ""
     if version:
-        version = "==" + version
+        version = "~=" + version.lstrip("v")
 
     cmd = ["pipx", "run", f"mpy-cross{version}"] if version else ["pipx", "run", "mpy-cross"]
     # Add params

stubber/modcat.py

@@ -0,0 +1,118 @@
+"""
+Used to define where stub modules should and should not be copied to, or merged to 
+
+This is shared between stubber, and external build scripts.
+"""
+from __future__ import annotations
+
+from typing import Final
+
+from stubber.publish.enums import StubSource
+
+########################################################
+STDLIB_ONLY_MODULES = [
+    "abc",
+    "array",
+    "collections",
+    "io",
+    "builtins",
+    "asyncio",
+    "sys",
+    "types",
+    "ssl",
+    "os", 
+    # Experiment
+    "json",
+    "struct"
+    # "socket", # should be ins tdlib - but is only available on networked boards 
+    # time # should be in stdlib - but has implementation differences per port)
+]
+"""Modules that should only be in stdlib, and not in the individual packages"""
+# and should not be in the individual packes as that causes duplication
+
+
+########################################################
+# stubber.publish.stubpacker                           #
+########################################################
+# Indicates which stubs will NOT be copied from the stub sources
+# and shared with multiple modules and the sdtbli build.py that is in the stubs repo
+STUBS_COPY_FILTER = {
+    StubSource.FROZEN: [
+        "espnow",  # merged stubs + documentation of the espnow module is better than the info in the frozen stubs
+        # "collections",  # must be in stdlib
+        # "types",  # must be in stdlib
+        # "abc",  # must be in stdlib
+        "time",  # used from merged ( should be in stdlib - but has implementation differences per port)
+        # "io",  # must be in stdlib
+    ] + STDLIB_ONLY_MODULES,
+    StubSource.FIRMWARE: [
+        "builtins",
+        # "collections",  # collections must be in stdlib
+    ] + STDLIB_ONLY_MODULES,
+    StubSource.MERGED: STDLIB_ONLY_MODULES,
+}
+
+# these modules will be replaced by a simple import statement to import from stdlib
+STDLIB_UMODULES = ["ucollections"]
+
+########################################################
+# stubber.rst...                                       #
+########################################################
+U_MODULES = [
+    "array",
+    "asyncio",
+    "binascii",
+    "bluetooth",
+    "errno",
+    "io",
+    "json",
+    "machine",
+    "os",
+    "platform",
+    "select",
+    "ssl",
+    "struct",
+    "socket",
+    "sys",
+    "time",
+    "zlib",
+]
+"""
+List of modules that are documented with the base name only, 
+but can also be imported with a `u` prefix
+"""
+
+########################################################
+# stubber.merge_config                                 #
+# defines constants to copy, update, remove type stubs #
+########################################################
+
+CP_REFERENCE_TO_DOCSTUB: Final = [
+    "asyncio", # just for documentation 
+    # stdlib stubs are using the version from stdlib/asyncio
+    "rp2/PIOASMEmit.pyi",
+    "rp2/asm_pio.pyi",
+]
+"Modules to copy from reference modules to the docstubs"
+
+
+
+
+RM_MERGED = (
+    [
+        "sys",  # use auto patched version from mpy_stdlib
+        "asyncio",  # use manually patched version from  mpy_stdlib
+        "_asyncio",  # ditto
+        "uasyncio",  # ditto
+        "_rp2",  # Leave out for now , to avoid conflicts with the rp2 module
+        "pycopy_imphook",  #  pycopy only: not needed in the merged stubs
+        # "os",
+        "types", # defined in webassembly pyscript - shadows stdlib
+        "abc", # defined in webassembly pyscript - shadows stdlib
+        # "uos", # ???? problems with mypy & webassembly stubs 
+    ]
+    + STDLIB_ONLY_MODULES
+    + [f"u{mod}" for mod in U_MODULES]
+)
+"Modules to remove from merged stubs, U_MODULES will be recreated later"
+

stubber/publish/merge_docstubs.py

@@ -7,7 +7,6 @@ from pathlib import Path
 from typing import List, Optional, Union
 
 from mpflash.logger import log
-
 from stubber.codemod.enrich import enrich_folder
 from stubber.merge_config import RM_MERGED, recreate_umodules, remove_modules
 from stubber.publish.candidates import board_candidates, filter_list
@@ -21,8 +20,8 @@ def merge_all_docstubs(
     family: str = "micropython",
     ports: Optional[Union[List[str], str]] = None,
     boards: Optional[Union[List[str], str]] = None,
-    *,
-    mpy_path: Path = CONFIG.mpy_path,
+    # *,
+    # mpy_path: Path = CONFIG.mpy_path,
 ):
     """merge docstubs and MCU stubs to merged stubs"""
     if versions is None:
@@ -67,7 +66,19 @@ def merge_all_docstubs(
             continue
         log.info(f"Merge {candidate['version']} docstubs with boardstubs to {merged_path.name}")
         try:
+            # TODO : webassembly: Need to merge from reference/pyscript as well 
             result = copy_and_merge_docstubs(board_path, merged_path, doc_path)
+            if candidate["port"] == "webassembly":
+                # TODO : webassembly: Need to merge from reference/pyscript as well 
+                # use enrich_folder to merge the docstubs
+                enrich_folder(
+                    source_folder=CONFIG.mpy_stubs_path / "reference/pyscript",
+                    target_folder=merged_path,
+                    write_back=True,
+                    copy_params=True,
+                    copy_docstr=True,
+                )
+                pass
         except Exception as e:
             log.error(f"Error parsing {candidate['version']} docstubs: {e}")
             continue
@@ -122,7 +133,13 @@ def copy_and_merge_docstubs(fw_path: Path, dest_path: Path, docstub_path: Path):
     recreate_umodules(dest_path)
 
     # 2 - Enrich the MCU stubs with the document stubs
-    result = enrich_folder(source_folder=docstub_path, target_folder=dest_path, write_back=True)
+    result = enrich_folder(
+        source_folder=docstub_path,
+        target_folder=dest_path,
+        write_back=True,
+        copy_params=True,
+        copy_docstr=True,
+    )
 
     refactor_rp2_module(dest_path)
 
@@ -147,7 +164,7 @@ def refactor_rp2_module(dest_path: Path):
         rp2_file.rename(rp2_folder / "__init__.pyi")
     # copy the asm_pio.pyi file from the reference folder
     for submod in ["rp2/asm_pio.pyi"]:
-        file = CONFIG.mpy_stubs_path / "micropython-reference" / submod
+        file = CONFIG.mpy_stubs_path / "reference/micropython" / submod
         if file.exists():
             shutil.copy(file, rp2_folder / file.name)
             log.info(f" - add rp2/{ file.name}")

stubber/publish/stubpackage.py

@@ -1,4 +1,12 @@
-"""Create a stub-only package for a specific version of micropython"""
+"""
+Create a stub-only package for a specific 
+- version 
+- port 
+- board 
+of micropython
+
+
+"""
 
 import hashlib
 import json
@@ -31,27 +39,11 @@ from stubber.publish.defaults import GENERIC_U, default_board
 from stubber.publish.enums import StubSource
 from stubber.publish.pypi import Version, get_pypi_versions
 from stubber.utils.config import CONFIG
+from stubber.modcat import STDLIB_UMODULES, STUBS_COPY_FILTER
 
 Status = NewType("Status", Dict[str, Union[str, None]])
 StubSources = List[Tuple[StubSource, Path]]
 
-# indicates which stubs will be skipped when copying for these stub sources
-STUBS_COPY_FILTER = {
-    StubSource.FROZEN: [
-        "espnow",  # merged stubs + documentation of the espnow module is better than the info in the forzen stubs
-    ],
-    StubSource.FIRMWARE: [
-        "builtins",
-        "collections",  # collections must be in stdlib
-    ],
-    StubSource.MERGED: [
-        "collections",  # collections must be in stdlib
-    ],
-}
-
-# these modules will be replaced by a simple import statement to import from stdlib
-STDLIB_UMODULES = ["ucollections"]
-
 
 class VersionedPackage:
     """
@@ -274,8 +266,12 @@ class Builder(VersionedPackage):
         # make sure parent folder exists
         _toml = self.toml_path
         (_toml).parent.mkdir(parents=True, exist_ok=True)
-        with open(_toml, "wb") as output:
-            tomli_w.dump(pyproject, output)
+        try: 
+            with open(_toml, "wb") as output:
+                tomli_w.dump(pyproject, output)
+        except OSError as e:
+            log.error(e)
+            log.debug(repr(_toml))
 
     # -----------------------------------------------
     def create_update_pyproject_toml(self) -> None:
@@ -359,10 +355,14 @@ class Builder(VersionedPackage):
 
     def copy_folder(self, stub_type: StubSource, src_path: Path):
         Path(self.package_path).mkdir(parents=True, exist_ok=True)
-        for item in (CONFIG.stub_path / src_path).rglob("*"):
+        for item in (CONFIG.stub_path / src_path).rglob("*.pyi"):
             if item.is_file():
                 # filter the 'poorly' decorated files
-                if stub_type in STUBS_COPY_FILTER and item.stem in STUBS_COPY_FILTER[stub_type]:
+                if stub_type in STUBS_COPY_FILTER and (
+                        item.stem in STUBS_COPY_FILTER[stub_type] or 
+                        item.parent.name in STUBS_COPY_FILTER[stub_type]
+                    ):
+                    log.trace(f"Skipping {item.name}")
                     continue
 
                 target = Path(self.package_path) / item.relative_to(CONFIG.stub_path / src_path)
@@ -631,8 +631,10 @@ class PoetryBuilder(Builder):
         try:
             with open(_toml, "rb") as f:
                 pyproject = tomllib.load(f)
-            # pyproject["tool"]["poetry"]["version"] = version
-            pyproject["project"]["version"] = version
+            if "project"  in pyproject:
+                pyproject["project"]["version"] = version
+            else:
+                pyproject["tool"]["poetry"]["version"] = version
             # update the version in the toml file
             with open(_toml, "wb") as output:
                 tomli_w.dump(pyproject, output)
@@ -742,10 +744,13 @@ class PoetryBuilder(Builder):
                 raise (e)
 
         # update the name , version and description of the package
-        # _pyproject["tool"]["poetry"]["name"] = self.package_name
-        # _pyproject["tool"]["poetry"]["description"] = self.description
-        _pyproject["project"]["name"] = self.package_name
-        _pyproject["project"]["description"] = self.description
+        if 'project' in _pyproject:
+            _pyproject["project"]["name"] = self.package_name
+            _pyproject["project"]["description"] = self.description
+        else: 
+            _pyproject["tool"]["poetry"]["name"] = self.package_name
+            _pyproject["tool"]["poetry"]["description"] = self.description
+
         # write out the pyproject.toml file
         self.pyproject = _pyproject
 

stubber/rst/lookup.py

@@ -16,7 +16,6 @@ __all__ = [
     "MODULE_GLUE",
     "RST_DOC_FIXES",
     "DOCSTUB_SKIP",
-    "U_MODULES",
 ]
 
 # all possible Types needed for the stubs - excess types should be removed later , and otherwise won't do much harm
@@ -52,25 +51,6 @@ class Fix:
     "the from_ string is a regular expression"
 
 
-U_MODULES = [
-    "array",
-    "asyncio",
-    "binascii",
-    "io",
-    "json",
-    "machine",
-    "os",
-    "select",
-    "ssl",
-    "struct",
-    "socket",
-    "time",
-    "zlib",
-]
-"""
-List of modules that are documented with the base name only, 
-but can also be imported with a `u` prefix
-"""
 
 # This table is used to try to correct the errors in the documentation,
 #  or adapt the human readable documentation to machine readable.
@@ -101,6 +81,9 @@ RST_DOC_FIXES: List[Tuple[str, str]] = [
         ".. method:: AIOESPNow._aiter__() / async AIOESPNow.__anext__()",
         ".. method:: AIOESPNow._aiter__()\n            async AIOESPNow.__anext__()",
     ),
+    # appended to in ssl.constant name  - ssl.PROTOCOL_DTLS_CLIENT(when DTLS support is enabled)
+    # Ugly hack to fix the documentation
+    ( '(when DTLS support is enabled)', " : Incomplete # (when DTLS support is enabled)")
 ]
 
 
@@ -154,8 +137,8 @@ LOOKUP_LIST = {
     "uio.open": ("IO", 0.95),  #  Open a file.
     "uos.listdir": ("List[Incomplete]", 0.95),
     "os.uname": ("uname_result", 0.95),
-    # undocumented CPython class ssl.SSLSocket
-    # TODO: include  ssl.SSLSocket from stdlib / mpy_typeshed, currently Incomplete
+    # ssl.SSLSocket is defined in reference/micropython/ssl
+    
     "ssl.wrap_socket": ("SSLSocket", 0.95),
     "ussl.wrap_socket": ("SSLSocket", 0.95),
     #
@@ -367,7 +350,7 @@ MODULE_GLUE = {
     "ssl": [
         "from typing_extensions import TypeAlias",
         "from _mpy_shed import StrOrBytesPath",
-        "SSLSocket : TypeAlias = Incomplete",
+        # SSLSocket is part of the refence stubs
     ],
     "struct": ANY_BUF,
     "time": [

stubber/rst/reader.py

@@ -69,19 +69,10 @@ from typing import List, Optional, Tuple
 
 from mpflash.logger import log
 from mpflash.versions import V_PREVIEW
-
-from stubber.rst import (
-    CHILD_PARENT_CLASS,
-    MODULE_GLUE,
-    PARAM_FIXES,
-    PARAM_RE_FIXES,
-    RST_DOC_FIXES,
-    TYPING_IMPORT,
-    ClassSourceDict,
-    FunctionSourceDict,
-    ModuleSourceDict,
-    return_type_from_context,
-)
+from stubber.rst import (CHILD_PARENT_CLASS, MODULE_GLUE, PARAM_FIXES,
+                         PARAM_RE_FIXES, RST_DOC_FIXES, TYPING_IMPORT,
+                         ClassSourceDict, FunctionSourceDict, ModuleSourceDict,
+                         return_type_from_context)
 from stubber.rst.lookup import Fix
 from stubber.utils.config import CONFIG
 
@@ -782,6 +773,10 @@ class RSTParser(RSTReader):
         self.line_no += counter - 1
         # clean up before returning
         names = [n.strip() for n in names if n.strip() != "etc."]  # remove etc.
+        # Ugly one-off hack 
+        # to remove the '(when DTLS support is enabled)' from the ssl constants
+        # names = [n.replace('(when DTLS support is enabled)', '') for n in names]  
+        
         return names
 
     def parse_data(self):

stubber/stubs_from_docs.py

@@ -10,7 +10,8 @@ from typing import List, Optional
 from mpflash.logger import log
 
 from stubber import utils
-from stubber.rst import DOCSTUB_SKIP, U_MODULES
+from stubber.rst import DOCSTUB_SKIP
+from stubber.modcat import  U_MODULES
 from stubber.rst.reader import RSTWriter
 
 

stubber/tools/manifestfile.py

@@ -29,6 +29,7 @@ from __future__ import print_function
 import contextlib
 import os
 import sys
+import glob
 import tempfile
 from collections import namedtuple
 
@@ -400,7 +401,7 @@ class ManifestFile:
                 self._metadata.pop()
 
     def _require_from_path(self, library_path, name, version, extra_kwargs):
-        for root, dirnames, filenames in os.walk(library_path): # type: ignore
+        for root, dirnames, filenames in os.walk(library_path):
             if os.path.basename(root) == name and "manifest.py" in filenames:
                 self.include(root, is_require=True, **extra_kwargs)
                 return True

stubber/{cst_transformer.py → typing_collector.py}

@@ -5,6 +5,7 @@ from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
 import libcst as cst
+from libcst import matchers as m
 
 
 @dataclass
@@ -43,12 +44,33 @@ MODULE_KEY = ("__module",)
 MOD_DOCSTR_KEY = ("__module_docstring",)
 
 # debug helper
-_m = cst.parse_module("")
+_code = cst.parse_module("").code_for_node
+
+
+def is_property(node: cst.FunctionDef) -> bool:
+    """check if the function is a property"""
+    return any(m.matches(dec, m.Decorator(decorator=m.Name(value="property"))) for dec in node.decorators)
+
+
+# FIXME : this is a clumsy way - but I cant find a working matcher for this
+def is_setter(node: cst.FunctionDef) -> bool:
+    """check if the function is a setter"""
+    return any(d for d in _code(node).split("\n") if d.startswith("@") and d.endswith(".setter"))
+
+
+def is_getter(node: cst.FunctionDef) -> bool:
+    """check if the function is a getter"""
+    return any(d for d in _code(node).split("\n") if d.startswith("@") and d.endswith(".getter"))
+
+
+def is_deleter(node: cst.FunctionDef) -> bool:
+    """check if the function is a deleter"""
+    return any(d for d in _code(node).split("\n") if d.startswith("@") and d.endswith(".deleter"))
 
 
 class StubTypingCollector(cst.CSTVisitor):
     """
-    Collect the function/method and class definitions from the stubs source
+    Collect the function/method and class definitions from ta rich .py or .pyi source
     """
 
     def __init__(self):
@@ -119,12 +141,22 @@ class StubTypingCollector(cst.CSTVisitor):
             def_type="funcdef",
             def_node=node,
         )
-        key = tuple(self.stack)
+        if is_getter(node):
+            extra = ["getter"]
+        elif is_setter(node):
+            extra = ["setter"]
+        elif is_deleter(node):
+            extra = ["deleter"]
+        else:
+            extra = []
+        key = tuple(self.stack + extra)
+
         if not key in self.annotations:
             # store the first function/method signature
             self.annotations[key] = AnnoValue(type_info=ti)
 
-        if any(dec.decorator.value == "overload" for dec in node.decorators):  # type: ignore
+
+        if any(m.matches(dec , m.Decorator(decorator=m.Name("overload"))) for dec in node.decorators):
             # and store the overloads
             self.annotations[key].overloads.append(ti)