micropython-stubber 1.23.3__py3-none-any.whl → 1.24.1__py3-none-any.whl

stubber/codemod/visitors/type_helpers.py

@@ -0,0 +1,182 @@
+"""
+Gather all TypeVar and TypeAlias assignments in a module.
+"""
+
+from typing import List, Sequence, Tuple, Union
+
+import libcst
+import libcst.matchers as m
+from libcst import SimpleStatementLine
+from libcst.codemod._context import CodemodContext
+from libcst.codemod._visitor import ContextAwareTransformer, ContextAwareVisitor
+from libcst.codemod.visitors._add_imports import _skip_first
+from typing_extensions import TypeAlias
+
+from mpflash.logger import log
+
+TypeHelper: TypeAlias = Union[libcst.Assign, libcst.AnnAssign]
+TypeHelpers: TypeAlias = List[TypeHelper]
+Statement: TypeAlias = Union[libcst.SimpleStatementLine, libcst.BaseCompoundStatement]
+
+_mod = libcst.parse_module("")  # Debugging aid : _mod.code_for_node(node)
+_code = _mod.code_for_node  #     Debugging aid : _code(node)
+
+
+def is_TypeAlias(statement) -> bool:
+    "Annotated Assign -  Foo:TypeAlias = ..."
+    if m.matches(statement, m.SimpleStatementLine()):
+        statement = statement.body[0]
+    return m.matches(
+        statement,
+        m.AnnAssign(annotation=m.Annotation(annotation=m.Name(value="TypeAlias"))),
+    )
+
+
+def is_TypeVar(statement):
+    "Assing - Foo = Typevar(...)"
+    if m.matches(statement, m.SimpleStatementLine()):
+        statement = statement.body[0]
+    return m.matches(
+        statement,
+        m.Assign(value=m.Call(func=m.Name(value="TypeVar"))),
+    )
+
+
+def is_ParamSpec(statement):
+    "Assign - Foo = ParamSpec(...)"
+    if m.matches(statement, m.SimpleStatementLine()):
+        statement = statement.body[0]
+    return m.matches(
+        statement,
+        m.Assign(value=m.Call(func=m.Name(value="ParamSpec"))),
+    )
+
+
+def is_import(statement):
+    "import - import foo"
+    return m.matches(statement, m.SimpleStatementLine(body=[m.ImportFrom() | m.Import()]))
+
+
+class GatherTypeHelpers(ContextAwareVisitor):
+    """
+    A class for tracking visited TypeVars and TypeAliases.
+    """
+
+    def __init__(self, context: CodemodContext) -> None:
+        super().__init__(context)
+        # Track all of the TypeVar, TypeAlias and Paramspec assignments found
+        self.all_typehelpers: TypeHelpers = []
+
+    def visit_Assign(self, node: libcst.Assign) -> None:
+        """
+        Find all TypeVar assignments in the module.
+        format: T = TypeVar("T", int, float, str, bytes, Tuple)
+        """
+        # is this a TypeVar assignment?
+        # needs to be more robust
+        if is_TypeVar(node) or is_ParamSpec(node):
+            self.all_typehelpers.append(node)
+
+    def visit_AnnAssign(self, node: libcst.AnnAssign) -> None:
+        """ "
+        Find all TypeAlias assignments in the module.
+        format: T: TypeAlias = str
+        """
+        if is_TypeAlias(node):
+            self.all_typehelpers.append(node)
+
+
+class AddTypeHelpers(ContextAwareTransformer):
+    """
+    Visitor loosly based on AddImportsVisitor
+    """
+
+    CONTEXT_KEY = "AddTypeHelpers"
+
+    def __init__(self, context: CodemodContext) -> None:
+        super().__init__(context)
+        self.new_typehelpers: TypeHelpers = context.scratch.get(self.CONTEXT_KEY, [])
+        self.all_typehelpers: TypeHelpers = []
+
+    @classmethod
+    def add_typevar(cls, context: CodemodContext, node: libcst.Assign):
+        new_typehelpers = context.scratch.get(cls.CONTEXT_KEY, [])
+        new_typehelpers.append(node)
+        context.scratch[cls.CONTEXT_KEY] = new_typehelpers
+        # add the typevar to the module
+
+    def leave_Module(
+        self,
+        original_node: libcst.Module,
+        updated_node: libcst.Module,
+    ) -> libcst.Module:
+        if not self.new_typehelpers:
+            return updated_node
+
+        # split the module into 3 parts
+        # before and after the insertions point , and a list of the TV and TA statements
+        (
+            statements_before,
+            helper_statements,
+            statements_after,
+        ) = self._split_module(original_node, updated_node)
+
+        # simpler to compare as text than to compare the nodes -
+        existing_targets = [
+            helper.body[0].targets[0].target.value  # type: ignore
+            for helper in helper_statements
+            if is_TypeVar(helper) or is_ParamSpec(helper)
+        ] + [
+            helper.body[0].target.value  # type: ignore
+            for helper in helper_statements
+            if is_TypeAlias(helper)
+        ]
+        statements_to_add = []
+        for new_typehelper in self.new_typehelpers:
+            if isinstance(new_typehelper, libcst.AnnAssign):
+                if new_typehelper.target.value not in existing_targets:  # type: ignore
+                    statements_to_add.append(SimpleStatementLine(body=[new_typehelper]))
+            elif isinstance(new_typehelper, libcst.Assign):
+                if new_typehelper.targets[0].target.value not in existing_targets:  # type: ignore
+                    statements_to_add.append(SimpleStatementLine(body=[new_typehelper]))
+
+        body = (
+            *statements_before,
+            *statements_to_add,
+            *statements_after,
+        )
+
+        return updated_node.with_changes(body=body)
+
+    def _split_module(
+        self,
+        orig_module: libcst.Module,
+        updated_module: libcst.Module,
+    ) -> Tuple[Sequence[Statement], Sequence[libcst.SimpleStatementLine], Sequence[Statement]]:
+        """
+        Split the module into 3 parts:
+        - before any TypeAlias, TypeVar or ParamSpec statements
+        - the TypeAlias and TypeVar statements
+        - the rest of the module after the TypeAlias and TypeVar statements
+        """
+        last_import = first_typehelper = last_typehelper = -1
+        start = 0
+        if _skip_first(orig_module):
+            start = 1
+
+        for i, statement in enumerate(orig_module.body[start:]):
+            if is_import(statement):
+                last_import = i + start
+                continue
+            if is_TypeVar(statement) or is_TypeAlias(statement) or is_ParamSpec(statement):
+                if first_typehelper == -1:
+                    first_typehelper = i + start
+                last_typehelper = i + start
+        # insert as high as possible, but after the last import and last TypeVar/TypeAlias statement
+        insert_after = max(start, last_import + 1, last_typehelper + 1)
+        assert insert_after != -1, "insert_after must be != -1"
+        #
+        before = list(updated_module.body[:insert_after])
+        after = list(updated_module.body[insert_after:])
+        helper_statements: Sequence[libcst.SimpleStatementLine] = list(updated_module.body[first_typehelper : last_typehelper + 1])  # type: ignore
+        return (before, helper_statements, after)

stubber/commands/build_cmd.py

@@ -4,7 +4,8 @@ from typing import List, Union
 
 import rich_click as click
 from mpflash.logger import log
-from tabulate import tabulate
+from rich.console import Console
+from rich.table import Table
 
 from stubber.commands.cli import stubber_cli
 from stubber.publish.defaults import GENERIC_U
@@ -77,7 +78,6 @@ def cli_build(
             "Multiple versions are not supported yet\n See https://github.com/Josverl/micropython-stubber/issues/487"
         )
 
-    # db = get_database(publish_path=CONFIG.publish_path, production=production)
     log.info(f"Build {family} {versions} {ports} {boards}")
 
     results = build_multiple(
@@ -91,4 +91,17 @@ def cli_build(
     )
     # log the number of results with no error
     log.info(f"Built {len([r for r in results if not r['error']])} stub packages")
-    print(tabulate(results, headers="keys"))
+    console = Console()
+    if not results:
+        console.print("No results to publish")
+        return
+    table = Table(title="Build Results", show_header=True, header_style="bold magenta")
+
+    for key in results[0].keys():
+        table.add_column(key)
+
+    for result in results:
+        if result["result"] != "-":
+            table.add_row(*[str(result[key]) for key in result.keys()])
+
+    console.print(table)

stubber/commands/clone_cmd.py

@@ -7,12 +7,12 @@ import os
 from pathlib import Path
 from typing import List, Tuple, Union
 
-import rich_click as click
 import mpflash.basicgit as git
+import rich_click as click
 from mpflash.logger import log
-from stubber.utils.config import CONFIG
 
-from .cli import stubber_cli
+from stubber.commands.cli import stubber_cli
+from stubber.utils.config import CONFIG
 
 ##########################################################################################
 # log = logging.getLogger("stubber")

stubber/commands/config_cmd.py

@@ -4,10 +4,9 @@
 
 from mpflash.logger import log
 
+from stubber.commands.cli import stubber_cli
 from stubber.utils.config import CONFIG
 
-from .cli import stubber_cli
-
 
 @stubber_cli.command(name="show-config")
 def cli_config():
@@ -27,3 +26,6 @@ def cli_config():
     log.info(f"CONFIG.stub_path           {CONFIG.stub_path}")
     log.info(f"CONFIG.publish_path        {CONFIG.publish_path}")
     log.info(f"CONFIG.template_path       {CONFIG.template_path}")
+    log.info(f"CONFIG.all_versions        {CONFIG.all_versions}")
+    log.info(f"CONFIG.stable_version      {CONFIG.stable_version}")
+    log.info(f"CONFIG.preview_version     {CONFIG.preview_version}")

stubber/commands/enrich_folder_cmd.py

@@ -1,5 +1,5 @@
 """
-enrich machinestubs with docstubs
+Enrich mcu/firmware stubs with information from the docstubs
 """
 
 from pathlib import Path
@@ -9,28 +9,30 @@ import rich_click as click
 from mpflash.logger import log
 
 from stubber.codemod.enrich import enrich_folder
+from stubber.commands.cli import stubber_cli
 from stubber.utils.config import CONFIG
 
-from .cli import stubber_cli
-
 
 @stubber_cli.command(name="enrich")
 @click.option(
+    "--dest",
+    "--destination",
+    "-d",
     "--stubs",
-    "-s",
-    "stubs_folder",
+    "dest_folder",
     default=CONFIG.stub_path.as_posix(),
     type=click.Path(exists=True, file_okay=True, dir_okay=True),
-    help="File or folder containing the MCU stubs to be updated",
+    help="The destination file or folder containing the stubs stubs to be updated",
     show_default=True,
 )
 @click.option(
+    "--source",
+    "-s",
     "--docstubs",
-    "-ds",
-    "docstubs_folder",
+    "source_folder",
     default=CONFIG.stub_path.as_posix(),
     type=click.Path(exists=True, file_okay=True, dir_okay=True),
-    help="File or folder containing the docstubs to be applied",
+    help="The source file or folder containing the docstubs to be read",
     show_default=True,
 )
 @click.option("--diff", default=False, help="Show diff", show_default=True, is_flag=True)
@@ -42,30 +44,40 @@ from .cli import stubber_cli
     is_flag=True,
 )
 @click.option(
-    "--package-name",
-    "-p",
-    "package_name",
-    default="",
-    help="Package name to be enriched (Optional)",
+    "--params-only",
+    "params_only",
+    default=False,
+    help="Copy only the parameters, not the docstrings (unless the docstring is missing)",
     show_default=True,
+    is_flag=True,
 )
+# @click.option(
+#     "--package-name",
+#     "-p",
+#     "package_name",
+#     default="",
+#     help="Package name to be enriched (Optional)",
+#     show_default=True,
+# )
 def cli_enrich_folder(
-    stubs_folder: Union[str, Path],
-    docstubs_folder: Union[str, Path],
+    dest_folder: Union[str, Path],
+    source_folder: Union[str, Path],
     diff: bool = False,
     dry_run: bool = False,
-    package_name: str = "",
+    params_only: bool = True,
+    # package_name: str = "",
 ):
     """
     Enrich the stubs in stub_folder with the docstubs in docstubs_folder.
     """
     write_back = not dry_run
-    log.info(f"Enriching {stubs_folder} with {docstubs_folder}")
+    log.info(f"Enriching {dest_folder} with {source_folder}")
     _ = enrich_folder(
-        Path(stubs_folder),
-        Path(docstubs_folder),
+        Path(source_folder),
+        Path(dest_folder),
         show_diff=diff,
         write_back=write_back,
         require_docstub=False,
-        package_name=package_name,
+        # package_name=package_name,
+        params_only=params_only,
     )

stubber/commands/get_core_cmd.py

@@ -12,10 +12,9 @@ from mpflash.logger import log
 
 import stubber.get_cpython as get_cpython
 import stubber.utils as utils
+from stubber.commands.cli import stubber_cli
 from stubber.utils.config import CONFIG
 
-from .cli import stubber_cli
-
 ##########################################################################################
 # log = logging.getLogger("stubber")
 #########################################################################################

stubber/commands/get_docstubs_cmd.py

@@ -5,18 +5,19 @@ get-docstubs
 
 from pathlib import Path
 from typing import Optional
+from packaging.version import Version
 
+import mpflash.basicgit as git
 import rich_click as click
 from mpflash.logger import log
 
-import mpflash.basicgit as git
+from stubber.codemod.enrich import enrich_folder
 import stubber.utils as utils
+from stubber.commands.cli import stubber_cli
 from stubber.stubs_from_docs import generate_from_rst
 from stubber.utils.config import CONFIG
 from stubber.utils.repos import fetch_repos
 
-from .cli import stubber_cli
-
 ##########################################################################################
 # log = logging.getLogger("stubber")
 #########################################################################################
@@ -47,14 +48,37 @@ from .cli import stubber_cli
     "--version", "--tag", default="", type=str, help="Version number to use. [default: Git tag]"
 )
 @click.option("--black/--no-black", "-b/-nb", default=True, help="Run black", show_default=True)
+@click.option(
+    "--autoflake/--no-autoflake",
+    default=True,
+    help="Run autoflake to clean imports",
+    show_default=True,
+)
+@click.option(
+    "--clean-rst/--no-clean-rst",
+    "-b/-nb",
+    default=True,
+    help="remove .rST constructs from the docstrings",
+    show_default=True,
+)
+@click.option(
+    "--enrich",
+    is_flag=True,
+    default=False,
+    help="Enrich with type information from micropython-reference",
+    show_default=True,
+)
 @click.pass_context
 def cli_docstubs(
     ctx: click.Context,
     path: Optional[str] = None,
     target: Optional[str] = None,
     black: bool = True,
+    autoflake: bool = True,
+    clean_rst: bool = True,
     basename: Optional[str] = None,
     version: str = "",
+    enrich: bool = False,
 ):
     """
     Build stubs from documentation.
@@ -80,13 +104,43 @@ def cli_docstubs(
         result = fetch_repos(version, CONFIG.mpy_path, CONFIG.mpy_lib_path)
         if not result:
             return -1
-    # get the current checked out version
-    version = utils.checkedout_version(CONFIG.mpy_path)
+    else:
+        # get the current checked out version
+        version = utils.checkedout_version(CONFIG.mpy_path)
 
     release = git.get_local_tag(rst_path.as_posix(), abbreviate=False) or ""
 
     dst_path = Path(target) / f"{basename}-{utils.clean_version(version, flat=True)}-docstubs"
 
     log.info(f"Get docstubs for MicroPython {utils.clean_version(version, drop_v=False)}")
-    generate_from_rst(rst_path, dst_path, version, release=release, suffix=".pyi", black=black)
+    generate_from_rst(
+        rst_path,
+        dst_path,
+        version,
+        release=release,
+        suffix=".pyi",
+        black=black,
+        autoflake=autoflake,
+        clean_rst=clean_rst,
+    )
+
+    if enrich:
+        if Version(version) < Version("1.24"):
+            log.warning(f"Enriching is not supported for version {version}")
+        else:
+            # !stubber enrich --params-only --source {reference} --dest {docstubs}
+            reference_path = CONFIG.stub_path.parent / "micropython-reference"
+            log.info(f"Enriching from {reference_path}")
+            _ = enrich_folder(
+                reference_path,
+                dst_path,
+                show_diff=False,
+                write_back=True,
+                require_docstub=False,
+                params_only=True,
+            )
+            log.info("::group:: start post processing of retrieved stubs")
+            # do not run stubgen
+            utils.do_post_processing([dst_path], stubgen=False, black=black, autoflake=autoflake)
+
     log.info("::group:: Done")

stubber/commands/get_frozen_cmd.py

@@ -7,16 +7,15 @@ from pathlib import Path
 from typing import List, Optional
 
 import rich_click as click
-from mpflash.logger import log
 
 import stubber.utils as utils
+from mpflash.logger import log
 from stubber.codemod.enrich import enrich_folder
+from stubber.commands.cli import stubber_cli
 from stubber.freeze.get_frozen import freeze_any
 from stubber.utils.config import CONFIG
 from stubber.utils.repos import fetch_repos
 
-from .cli import stubber_cli
-
 ##########################################################################################
 
 
@@ -96,19 +95,23 @@ def cli_get_frozen(
 
     # first create .pyi files so they can be enriched
     utils.do_post_processing(stub_paths, stubgen=stubgen, black=False, autoflake=False)
+
     family = "micropython"
     _version = utils.clean_version(version, drop_v=False, flat=True)
     docstubs_path = Path(CONFIG.stub_path) / f"{family}-{_version}-docstubs"
     if docstubs_path.exists():
-        log.info(f"Enriching {str(stub_path)} with {docstubs_path}")
-        if merged := enrich_folder(
-            stub_path,
-            docstubs_path,
-            show_diff=False,
-            write_back=True,
-            require_docstub=False,
-        ):
-            log.info(f"Enriched {merged} frozen modules from docstubs")
+        board_folders = [f.parent for f in list(stub_path.glob("**/modules.json"))]
+        for folder in board_folders:
+            log.info(f"Enriching {str(stub_path)} with {docstubs_path}")
+            if merged := enrich_folder(
+                docstubs_path,
+                folder,
+                show_diff=False,
+                write_back=True,
+                require_docstub=False,
+                ext = ".pyi",
+            ):
+                log.info(f" > Enriched {merged} frozen modules.")
     else:
         log.info(f"No docstubs found at {docstubs_path}")
 

stubber/commands/get_mcu_cmd.py

@@ -6,21 +6,21 @@
 
 
 from typing import List
+
 import rich_click as click
 from mpflash.logger import log
 
 from stubber.bulk.mcu_stubber import stub_connected_mcus
+from stubber.commands.cli import stubber_cli
 from stubber.utils.config import CONFIG
 
-from .cli import stubber_cli
-
 ##########################################################################################
 # log = logging.getLogger("stubber")
 #########################################################################################
 
 
 @stubber_cli.command(
-    name="get-mcu-stubs",
+    name="mcu-stubs",
     aliases=["get-mcu-stubs", "mcu-stubs", "mcu"],
 )
 @click.option(

stubber/commands/merge_cmd.py

@@ -7,12 +7,11 @@ from typing import List, Union
 import rich_click as click
 from mpflash.logger import log
 
+from stubber.commands.cli import stubber_cli
 from stubber.publish.merge_docstubs import merge_all_docstubs
 from stubber.publish.package import GENERIC_L
 from stubber.utils.config import CONFIG
 
-from .cli import stubber_cli
-
 
 @stubber_cli.command(name="merge")
 @click.option("--family", default="micropython", type=str, show_default=True)

stubber/commands/publish_cmd.py

@@ -6,7 +6,8 @@ from typing import List, Union
 
 import rich_click as click
 from mpflash.logger import log
-from tabulate import tabulate
+from rich.console import Console
+from rich.table import Table
 
 from stubber.commands.cli import stubber_cli
 from stubber.publish.defaults import GENERIC_U
@@ -81,7 +82,7 @@ def cli_publish(
     versions: Union[str, List[str]],
     ports: Union[str, List[str]],
     boards: Union[str, List[str]],
-    production: bool = True,
+    production: bool = False,
     build: bool = False,
     force: bool = False,
     dry_run: bool = False,
@@ -100,7 +101,6 @@ def cli_publish(
             "Multiple versions are not supported yet\n See https://github.com/Josverl/micropython-stubber/issues/487"
         )
 
-    # db = get_database(publish_path=CONFIG.publish_path, production=production)
     destination = "pypi" if production else "test-pypi"
     log.info(f"Publish {family} {versions} {ports} {boards} to {destination}")
 
@@ -115,4 +115,19 @@ def cli_publish(
         dry_run=dry_run,
         clean=clean,
     )
-    log.info(tabulate(results, headers="keys"))
+    console = Console()
+
+    if not results:
+        console.print("No results to publish")
+        return
+
+    table = Table(show_header=True, header_style="bold magenta", title="Publish Results")
+    headers = results[0].keys()
+    for header in headers:
+        table.add_column(header)
+
+    for result in results:
+        if result["result"] != "-":
+            table.add_row(*[str(result[header]) for header in headers])
+
+    console.print(table)

stubber/commands/stub_cmd.py

@@ -4,16 +4,16 @@
 # stub
 ##########################################################################################
 
-from mpflash.logger import log
 from pathlib import Path
 from typing import Union
 
 import rich_click as click
+from mpflash.logger import log
+
+from stubber.commands.cli import stubber_cli
 from stubber.utils import generate_pyi_files
 from stubber.utils.post import do_post_processing
 
-from .cli import stubber_cli
-
 ##########################################################################################
 # log = logging.getLogger("stubber")
 #########################################################################################

stubber/commands/switch_cmd.py

@@ -6,13 +6,11 @@ from pathlib import Path
 from typing import Optional, Union
 
 import rich_click as click
+from mpflash.versions import SET_PREVIEW, V_PREVIEW, micropython_versions
 
-import mpflash.basicgit as git
+from stubber.commands.cli import stubber_cli
 from stubber.utils.config import CONFIG
 from stubber.utils.repos import fetch_repos, repo_paths
-from mpflash.versions import SET_PREVIEW, V_PREVIEW
-
-from .cli import stubber_cli
 
 ##########################################################################################
 # log = logging.getLogger("stubber")
@@ -22,7 +20,7 @@ from .cli import stubber_cli
 # get version list from Git tags in the repo that is provided on the command line
 
 try:
-    VERSION_LIST = git.get_tags("micropython/micropython", minver="v1.9.3") + [
+    VERSION_LIST = micropython_versions(minver="v1.9.3") + [
         V_PREVIEW,
         "latest",
         "stable",

stubber/commands/variants_cmd.py

@@ -4,11 +4,11 @@ from pathlib import Path
 
 import rich_click as click
 from mpflash.logger import log
+
+import stubber
+from stubber.commands.cli import stubber_cli
 from stubber.utils.config import CONFIG
 from stubber.variants import create_variants
-import stubber
-
-from .cli import stubber_cli
 
 
 @click.option(