micropython-stubber 1.16.3__py3-none-any.whl → 1.17.0__py3-none-any.whl

stubber/board/createstubs.py

@@ -4,7 +4,6 @@ Create stubs for (all) modules on a MicroPython board
 # Copyright (c) 2019-2023 Jos Verlinde
 
 import gc
-import logging
 import os
 import sys
 from time import sleep
@@ -24,11 +23,49 @@ try:
 except ImportError:
     from ucollections import OrderedDict  # type: ignore
 
-
-__version__ = "v1.16.3"
+__version__ = "v1.17.0"
 ENOENT = 2
 _MAX_CLASS_LEVEL = 2  # Max class nesting
-LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"]
+LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."]
+
+
+# our own logging module to avoid dependency on and interfering with logging module
+class logging:
+    # DEBUG = 10
+    INFO = 20
+    WARNING = 30
+    ERROR = 40
+    level = INFO
+    prnt = print
+
+    @staticmethod
+    def getLogger(name):
+        return logging()
+
+    @classmethod
+    def basicConfig(cls, level):
+        cls.level = level
+
+    # def debug(self, msg):
+    #     if self.level <= logging.DEBUG:
+    #         self.prnt("DEBUG :", msg)
+
+    def info(self, msg):
+        if self.level <= logging.INFO:
+            self.prnt("INFO  :", msg)
+
+    def warning(self, msg):
+        if self.level <= logging.WARNING:
+            self.prnt("WARN  :", msg)
+
+    def error(self, msg):
+        if self.level <= logging.ERROR:
+            self.prnt("ERROR :", msg)
+
+
+log = logging.getLogger("stubber")
+logging.basicConfig(level=logging.INFO)
+# logging.basicConfig(level=logging.DEBUG)
 
 
 class Stubber:
@@ -40,11 +77,9 @@ class Stubber:
                 raise NotImplementedError("MicroPython 1.13.0 cannot be stubbed")
         except AttributeError:
             pass
-        self.log = logging.getLogger("stubber")
-        self._report = []  # type: list[str]
         self.info = _info()
-        self.log.info("Port: {}".format(self.info["port"]))
-        self.log.info("Board: {}".format(self.info["board"]))
+        log.info("Port: {}".format(self.info["port"]))
+        log.info("Board: {}".format(self.info["board"]))
         gc.collect()
         if firmware_id:
             self._fwid = firmware_id.lower()
@@ -62,11 +97,11 @@ class Stubber:
             path = get_root()
 
         self.path = "{}/stubs/{}".format(path, self.flat_fwid).replace("//", "/")
-        self.log.debug(self.path)
+        # log.debug(self.path)
         try:
             ensure_folder(path + "/")
         except OSError:
-            self.log.error("error creating stub folder {}".format(path))
+            log.error("error creating stub folder {}".format(path))
         self.problematic = [
             "upip",
             "upysh",
@@ -85,21 +120,23 @@ class Stubber:
         ]
         # there is no option to discover modules from micropython, list is read from an external file.
         self.modules = []  # type: list[str]
+        self._json_name = None
+        self._json_first = False
 
     def get_obj_attributes(self, item_instance: object):
         "extract information of the objects members and attributes"
         # name_, repr_(value), type as text, item_instance
         _result = []
         _errors = []
-        self.log.debug("get attributes {} {}".format(repr(item_instance), item_instance))
+        # log.debug("get attributes {} {}".format(repr(item_instance), item_instance))
         for name in dir(item_instance):
-            if name.startswith("_") and not name in self.modules:
+            if name.startswith("__") and not name in self.modules:
                 continue
-            self.log.debug("get attribute {}".format(name))
+            # log.debug("get attribute {}".format(name))
             try:
                 val = getattr(item_instance, name)
                 # name , item_repr(value) , type as text, item_instance, order
-                self.log.debug("attribute {}:{}".format(name, val))
+                # log.debug("attribute {}:{}".format(name, val))
                 try:
                     type_text = repr(type(val)).split("'")[1]
                 except IndexError:
@@ -132,21 +169,23 @@ class Stubber:
 
     def create_all_stubs(self):
         "Create stubs for all configured modules"
-        self.log.info("Start micropython-stubber v{} on {}".format(__version__, self._fwid))
+        log.info("Start micropython-stubber {} on {}".format(__version__, self._fwid))
+        self.report_start()
         gc.collect()
         for module_name in self.modules:
             self.create_one_stub(module_name)
-        self.log.info("Finally done")
+        self.report_end()
+        log.info("Finally done")
 
     def create_one_stub(self, module_name: str):
         if module_name in self.problematic:
-            self.log.warning("Skip module: {:<25}        : Known problematic".format(module_name))
+            log.warning("Skip module: {:<25}        : Known problematic".format(module_name))
             return False
         if module_name in self.excluded:
-            self.log.warning("Skip module: {:<25}        : Excluded".format(module_name))
+            log.warning("Skip module: {:<25}        : Excluded".format(module_name))
             return False
 
-        file_name = "{}/{}.py".format(self.path, module_name.replace(".", "/"))
+        file_name = "{}/{}.pyi".format(self.path, module_name.replace(".", "/"))
         gc.collect()
         result = False
         try:
@@ -161,10 +200,10 @@ class Stubber:
 
         Args:
         - module_name (str): name of the module to document. This module will be imported.
-        - file_name (Optional[str]): the 'path/filename.py' to write to. If omitted will be created based on the module name.
+        - file_name (Optional[str]): the 'path/filename.pyi' to write to. If omitted will be created based on the module name.
         """
         if file_name is None:
-            fname = module_name.replace(".", "_") + ".py"
+            fname = module_name.replace(".", "_") + ".pyi"
             file_name = self.path + "/" + fname
         else:
             fname = file_name.split("/")[-1]
@@ -178,10 +217,10 @@ class Stubber:
         try:
             new_module = __import__(module_name, None, None, ("*"))
             m1 = gc.mem_free()  # type: ignore
-            self.log.info("Stub module: {:<25} to file: {:<70} mem:{:>5}".format(module_name, fname, m1))
+            log.info("Stub module: {:<25} to file: {:<70} mem:{:>5}".format(module_name, fname, m1))
 
         except ImportError:
-            self.log.warning("Skip module: {:<25} {:<79}".format(module_name, "Module not found."))
+            # log.debug("Skip module: {:<25} {:<79}".format(module_name, "Module not found."))
             return False
 
         # Start a new file
@@ -193,22 +232,20 @@ class Stubber:
                 module_name, self._fwid, info_, __version__
             )
             fp.write(s)
-            fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n")
+            fp.write(
+                "from __future__ import annotations\nfrom typing import Any, Generator\nfrom _typeshed import Incomplete\n\n"
+            )
             self.write_object_stub(fp, new_module, module_name, "")
 
-        self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/")))
+        self.report_add(module_name, file_name)
 
         if module_name not in {"os", "sys", "logging", "gc"}:
             # try to unload the module unless we use it
             try:
                 del new_module
             except (OSError, KeyError):  # lgtm [py/unreachable-statement]
-                self.log.warning("could not del new_module")
-            # lets not try - most times it does not work anyway
-            # try:
-            #     del sys.modules[module_name]
-            # except KeyError:
-            #     self.log.warning("could not del sys.modules[{}]".format(module_name))
+                log.warning("could not del new_module")
+            # do not try to delete from sys.modules - most times it does not work anyway
         gc.collect()
         return True
 
@@ -216,14 +253,14 @@ class Stubber:
         "Write a module/object stub to an open file. Can be called recursive."
         gc.collect()
         if object_expr in self.problematic:
-            self.log.warning("SKIPPING problematic module:{}".format(object_expr))
+            log.warning("SKIPPING problematic module:{}".format(object_expr))
             return
 
-        # self.log.debug("DUMP    : {}".format(object_expr))
+        # # log.debug("DUMP    : {}".format(object_expr))
         items, errors = self.get_obj_attributes(object_expr)
 
         if errors:
-            self.log.error(errors)
+            log.error(errors)
 
         for item_name, item_repr, item_type_txt, item_instance, _ in items:
             # name_, repr_(value), type as text, item_instance, order
@@ -231,7 +268,7 @@ class Stubber:
                 # do not create stubs for these primitives
                 continue
             if item_name[0].isdigit():
-                self.log.warning("NameError: invalid name {}".format(item_name))
+                log.warning("NameError: invalid name {}".format(item_name))
                 continue
             # Class expansion only on first 3 levels (bit of a hack)
             if (
@@ -240,7 +277,7 @@ class Stubber:
                 # and not obj_name.endswith(".Pin")
                 # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms
             ):
-                self.log.info("{0}class {1}:".format(indent, item_name))
+                # log.debug("{0}class {1}:".format(indent, item_name))
                 superclass = ""
                 is_exception = (
                     item_name.endswith("Exception")
@@ -263,7 +300,7 @@ class Stubber:
                 # write classdef
                 fp.write(s)
                 # first write the class literals and methods
-                self.log.debug("# recursion over class {0}".format(item_name))
+                # log.debug("# recursion over class {0}".format(item_name))
                 self.write_object_stub(
                     fp,
                     item_instance,
@@ -277,7 +314,7 @@ class Stubber:
                 s += indent + "        ...\n\n"
                 fp.write(s)
             elif any(word in item_type_txt for word in ["method", "function", "closure"]):
-                self.log.debug("# def {1} function/method/closure, type = '{0}'".format(item_type_txt, item_name))
+                # log.debug("# def {1} function/method/closure, type = '{0}'".format(item_type_txt, item_name))
                 # module Function or class method
                 # will accept any number of params
                 # return type Any/Incomplete
@@ -295,7 +332,7 @@ class Stubber:
                     s = "{}def {}({}*args, **kwargs) -> {}:\n".format(indent, item_name, first, ret)
                 s += indent + "    ...\n\n"
                 fp.write(s)
-                self.log.debug("\n" + s)
+                # log.debug("\n" + s)
             elif item_type_txt == "<class 'module'>":
                 # Skip imported modules
                 # fp.write("# import {}\n".format(item_name))
@@ -305,28 +342,32 @@ class Stubber:
                 t = item_type_txt[8:-2]
                 s = ""
 
-                if t in ["str", "int", "float", "bool", "bytearray", "bytes"]:
+                if t in ("str", "int", "float", "bool", "bytearray", "bytes"):
                     # known type: use actual value
-                    s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, item_repr, t)
-                elif t in ["dict", "list", "tuple"]:
+                    # s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, item_repr, t)
+                    s = "{0}{1}: {3} = {2}\n".format(indent, item_name, item_repr, t)
+                elif t in ("dict", "list", "tuple"):
                     # dict, list , tuple: use empty value
                     ev = {"dict": "{}", "list": "[]", "tuple": "()"}
-                    s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t)
+                    # s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t)
+                    s = "{0}{1}: {3} = {2}\n".format(indent, item_name, ev[t], t)
                 else:
                     # something else
-                    if t in ["object", "set", "frozenset", "Pin", "FileIO"]:
+                    if t in ("object", "set", "frozenset", "Pin", "generator"):  # "FileIO"
                         # https://docs.python.org/3/tutorial/classes.html#item_instance-objects
-                        #  use these types for the attribute
-                        s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
+                        # use these types for the attribute
+                        if t == "generator":
+                            t = "Generator"
+                        s = "{0}{1}: {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
                     else:
                         # Requires Python 3.6 syntax, which is OK for the stubs/pyi
                         t = "Incomplete"
-                        s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
+                        s = "{0}{1}: {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
                 fp.write(s)
-                self.log.debug("\n" + s)
+                # log.debug("\n" + s)
             else:
                 # keep only the name
-                self.log.debug("# all other, type = '{0}'".format(item_type_txt))
+                # log.debug("# all other, type = '{0}'".format(item_type_txt))
                 fp.write("# all other, type = '{0}'\n".format(item_type_txt))
 
                 fp.write(indent + item_name + " # type: Incomplete\n")
@@ -352,7 +393,7 @@ class Stubber:
         "Remove all files from the stub folder"
         if path is None:
             path = self.path
-        self.log.info("Clean/remove files in folder: {}".format(path))
+        log.info("Clean/remove files in folder: {}".format(path))
         try:
             os.stat(path)  # TEMP workaround mpremote listdir bug -
             items = os.listdir(path)
@@ -370,43 +411,53 @@ class Stubber:
                 except OSError:
                     pass
 
-    def report(self, filename: str = "modules.json"):
-        "create json with list of exported modules"
-        self.log.info(
-            "Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)
-        )
-        f_name = "{}/{}".format(self.path, filename)
-        self.log.info("Report file: {}".format(f_name))
+    def report_start(self, filename: str = "modules.json"):
+        """Start a report of the modules that have been stubbed
+        "create json with list of exported modules"""
+        self._json_name = "{}/{}".format(self.path, filename)
+        self._json_first = True
+        ensure_folder(self._json_name)
+        log.info("Report file: {}".format(self._json_name))
         gc.collect()
         try:
             # write json by node to reduce memory requirements
-            with open(f_name, "w") as f:
-                self.write_json_header(f)
-                first = True
-                for n in self._report:
-                    self.write_json_node(f, n, first)
-                    first = False
-                self.write_json_end(f)
-            used = self._start_free - gc.mem_free()  # type: ignore
-            self.log.info("Memory used: {0} Kb".format(used // 1024))
-        except OSError:
-            self.log.error("Failed to create the report.")
-
-    def write_json_header(self, f):
-        f.write("{")
-        f.write(dumps({"firmware": self.info})[1:-1])
-        f.write(",\n")
-        f.write(dumps({"stubber": {"version": __version__}, "stubtype": "firmware"})[1:-1])
-        f.write(",\n")
-        f.write('"modules" :[\n')
+            with open(self._json_name, "w") as f:
+                f.write("{")
+                f.write(dumps({"firmware": self.info})[1:-1])
+                f.write(",\n")
+                f.write(dumps({"stubber": {"version": __version__}, "stubtype": "firmware"})[1:-1])
+                f.write(",\n")
+                f.write('"modules" :[\n')
+
+        except OSError as e:
+            log.error("Failed to create the report.")
+            self._json_name = None
+            raise e
+
+    def report_add(self, module_name: str, stub_file: str):
+        "Add a module to the report"
+        # write json by node to reduce memory requirements
+        if not self._json_name:
+            raise Exception("No report file")
+        try:
+            with open(self._json_name, "a") as f:
+                if not self._json_first:
+                    f.write(",\n")
+                else:
+                    self._json_first = False
+                line = '{{"module": "{}", "file": "{}"}}'.format(module_name, stub_file.replace("\\", "/"))
+                f.write(line)
 
-    def write_json_node(self, f, n, first):
-        if not first:
-            f.write(",\n")
-        f.write(n)
+        except OSError:
+            log.error("Failed to create the report.")
 
-    def write_json_end(self, f):
-        f.write("\n]}")
+    def report_end(self):
+        if not self._json_name:
+            raise Exception("No report file")
+        with open(self._json_name, "a") as f:
+            f.write("\n]}")
+        # is used as sucess indicator
+        log.info("Path: {}".format(self.path))
 
 
 def ensure_folder(path: str):
@@ -452,7 +503,7 @@ def _build(s):
 def _info():  # type:() -> dict[str, str]
     info = OrderedDict(
         {
-            "family": sys.implementation.name,
+            "family": sys.implementation.name,  # type: ignore
             "version": "",
             "build": "",
             "ver": "",
@@ -482,9 +533,9 @@ def _info():  # type:() -> dict[str, str]
         info["board"] = _machine
         info["cpu"] = _machine.split("with")[-1].strip()
         info["mpy"] = (
-            sys.implementation._mpy
+            sys.implementation._mpy  # type: ignore
             if "_mpy" in dir(sys.implementation)
-            else sys.implementation.mpy
+            else sys.implementation.mpy  # type: ignore
             if "mpy" in dir(sys.implementation)
             else ""
         )
@@ -582,9 +633,10 @@ def get_boardname() -> str:
     "Read the board name from the boardname.py file that may have been created upfront"
     try:
         from boardname import BOARDNAME  # type: ignore
+
         log.info("Found BOARDNAME: {}".format(BOARDNAME))
     except ImportError:
-        log.info("BOARDNAME not found")
+        log.warning("BOARDNAME not found")
         BOARDNAME = ""
     return BOARDNAME
 
@@ -639,10 +691,6 @@ def is_micropython() -> bool:
     # pylint: disable=unused-variable,eval-used
     try:
         # either test should fail on micropython
-        # a) https://docs.micropython.org/en/latest/genrst/syntax.html#spaces
-        # Micropython : SyntaxError
-        # a = eval("1and 0")  # lgtm [py/unused-local-variable]
-        # Eval blocks some minification aspects
 
         # b) https://docs.micropython.org/en/latest/genrst/builtin_types.html#bytes-with-keywords-not-implemented
         # Micropython: NotImplementedError
@@ -907,16 +955,9 @@ def main():
     gc.collect()
 
     stubber.create_all_stubs()
-    stubber.report()
 
 
 if __name__ == "__main__" or is_micropython():
-    try:
-        log = logging.getLogger("stubber")
-        logging.basicConfig(level=logging.INFO)
-        # logging.basicConfig(level=logging.DEBUG)
-    except NameError:
-        pass
     if not file_exists("no_auto_stubber.txt"):
         try:
             gc.threshold(4 * 1024)  # type: ignore