micropython-stubber 1.16.3__py3-none-any.whl → 1.17.0__py3-none-any.whl

stubber/board/createstubs_mem.py

@@ -9,12 +9,11 @@
     - cross compilation, using mpy-cross, 
       to avoid the compilation step on the micropython device 
 
-This variant was generated from createstubs.py by micropython-stubber v1.16.3
+This variant was generated from createstubs.py by micropython-stubber v1.17.0
 """
 # Copyright (c) 2019-2023 Jos Verlinde
 
 import gc
-import logging
 import os
 import sys
 from time import sleep
@@ -34,26 +33,49 @@ try:
 except ImportError:
     from ucollections import OrderedDict  # type: ignore
 
-try:
-    from nope_machine import WDT
+__version__ = "v1.17.0"
+ENOENT = 2
+_MAX_CLASS_LEVEL = 2  # Max class nesting
+LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."]
 
-    wdt = WDT()
 
-except ImportError:
+# our own logging module to avoid dependency on and interfering with logging module
+class logging:
+    # DEBUG = 10
+    INFO = 20
+    WARNING = 30
+    ERROR = 40
+    level = INFO
+    prnt = print
 
-    class _WDT:
-        def feed(self):
-            pass
+    @staticmethod
+    def getLogger(name):
+        return logging()
 
-    wdt = _WDT()
+    @classmethod
+    def basicConfig(cls, level):
+        cls.level = level
 
+    # def debug(self, msg):
+    #     if self.level <= logging.DEBUG:
+    #         self.prnt("DEBUG :", msg)
 
-wdt.feed()
+    def info(self, msg):
+        if self.level <= logging.INFO:
+            self.prnt("INFO  :", msg)
 
-__version__ = "v1.16.3"
-ENOENT = 2
-_MAX_CLASS_LEVEL = 2  # Max class nesting
-LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"]
+    def warning(self, msg):
+        if self.level <= logging.WARNING:
+            self.prnt("WARN  :", msg)
+
+    def error(self, msg):
+        if self.level <= logging.ERROR:
+            self.prnt("ERROR :", msg)
+
+
+log = logging.getLogger("stubber")
+logging.basicConfig(level=logging.INFO)
+# logging.basicConfig(level=logging.DEBUG)
 
 
 class Stubber:
@@ -65,13 +87,10 @@ class Stubber:
                 raise NotImplementedError("MicroPython 1.13.0 cannot be stubbed")
         except AttributeError:
             pass
-        self.log = logging.getLogger("stubber")
-        self._report = []  # type: list[str]
         self.info = _info()
-        self.log.info("Port: {}".format(self.info["port"]))
-        self.log.info("Board: {}".format(self.info["board"]))
+        log.info("Port: {}".format(self.info["port"]))
+        log.info("Board: {}".format(self.info["board"]))
         gc.collect()
-        wdt.feed()
         if firmware_id:
             self._fwid = firmware_id.lower()
         else:
@@ -88,11 +107,11 @@ class Stubber:
             path = get_root()
 
         self.path = "{}/stubs/{}".format(path, self.flat_fwid).replace("//", "/")
-        self.log.debug(self.path)
+        # log.debug(self.path)
         try:
             ensure_folder(path + "/")
         except OSError:
-            self.log.error("error creating stub folder {}".format(path))
+            log.error("error creating stub folder {}".format(path))
         self.problematic = [
             "upip",
             "upysh",
@@ -111,21 +130,23 @@ class Stubber:
         ]
         # there is no option to discover modules from micropython, list is read from an external file.
         self.modules = []  # type: list[str]
+        self._json_name = None
+        self._json_first = False
 
     def get_obj_attributes(self, item_instance: object):
         "extract information of the objects members and attributes"
         # name_, repr_(value), type as text, item_instance
         _result = []
         _errors = []
-        self.log.debug("get attributes {} {}".format(repr(item_instance), item_instance))
+        # log.debug("get attributes {} {}".format(repr(item_instance), item_instance))
         for name in dir(item_instance):
-            if name.startswith("_") and not name in self.modules:
+            if name.startswith("__") and not name in self.modules:
                 continue
-            self.log.debug("get attribute {}".format(name))
+            # log.debug("get attribute {}".format(name))
             try:
                 val = getattr(item_instance, name)
                 # name , item_repr(value) , type as text, item_instance, order
-                self.log.debug("attribute {}:{}".format(name, val))
+                # log.debug("attribute {}:{}".format(name, val))
                 try:
                     type_text = repr(type(val)).split("'")[1]
                 except IndexError:
@@ -158,22 +179,23 @@ class Stubber:
 
     def create_all_stubs(self):
         "Create stubs for all configured modules"
-        self.log.info("Start micropython-stubber v{} on {}".format(__version__, self._fwid))
+        log.info("Start micropython-stubber {} on {}".format(__version__, self._fwid))
+        self.report_start()
         gc.collect()
         for module_name in self.modules:
             self.create_one_stub(module_name)
-        self.log.info("Finally done")
+        self.report_end()
+        log.info("Finally done")
 
     def create_one_stub(self, module_name: str):
-        wdt.feed()
         if module_name in self.problematic:
-            self.log.warning("Skip module: {:<25}        : Known problematic".format(module_name))
+            log.warning("Skip module: {:<25}        : Known problematic".format(module_name))
             return False
         if module_name in self.excluded:
-            self.log.warning("Skip module: {:<25}        : Excluded".format(module_name))
+            log.warning("Skip module: {:<25}        : Excluded".format(module_name))
             return False
 
-        file_name = "{}/{}.py".format(self.path, module_name.replace(".", "/"))
+        file_name = "{}/{}.pyi".format(self.path, module_name.replace(".", "/"))
         gc.collect()
         result = False
         try:
@@ -188,10 +210,10 @@ class Stubber:
 
         Args:
         - module_name (str): name of the module to document. This module will be imported.
-        - file_name (Optional[str]): the 'path/filename.py' to write to. If omitted will be created based on the module name.
+        - file_name (Optional[str]): the 'path/filename.pyi' to write to. If omitted will be created based on the module name.
         """
         if file_name is None:
-            fname = module_name.replace(".", "_") + ".py"
+            fname = module_name.replace(".", "_") + ".pyi"
             file_name = self.path + "/" + fname
         else:
             fname = file_name.split("/")[-1]
@@ -205,10 +227,10 @@ class Stubber:
         try:
             new_module = __import__(module_name, None, None, ("*"))
             m1 = gc.mem_free()  # type: ignore
-            self.log.info("Stub module: {:<25} to file: {:<70} mem:{:>5}".format(module_name, fname, m1))
+            log.info("Stub module: {:<25} to file: {:<70} mem:{:>5}".format(module_name, fname, m1))
 
         except ImportError:
-            self.log.warning("Skip module: {:<25} {:<79}".format(module_name, "Module not found."))
+            # log.debug("Skip module: {:<25} {:<79}".format(module_name, "Module not found."))
             return False
 
         # Start a new file
@@ -218,22 +240,18 @@ class Stubber:
             info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}")
             s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__)
             fp.write(s)
-            fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n")
+            fp.write("from __future__ import annotations\nfrom typing import Any, Generator\nfrom _typeshed import Incomplete\n\n")
             self.write_object_stub(fp, new_module, module_name, "")
 
-        self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/")))
+        self.report_add(module_name, file_name)
 
         if module_name not in {"os", "sys", "logging", "gc"}:
             # try to unload the module unless we use it
             try:
                 del new_module
             except (OSError, KeyError):  # lgtm [py/unreachable-statement]
-                self.log.warning("could not del new_module")
-            # lets not try - most times it does not work anyway
-            # try:
-            #     del sys.modules[module_name]
-            # except KeyError:
-            #     self.log.warning("could not del sys.modules[{}]".format(module_name))
+                log.warning("could not del new_module")
+            # do not try to delete from sys.modules - most times it does not work anyway
         gc.collect()
         return True
 
@@ -241,14 +259,14 @@ class Stubber:
         "Write a module/object stub to an open file. Can be called recursive."
         gc.collect()
         if object_expr in self.problematic:
-            self.log.warning("SKIPPING problematic module:{}".format(object_expr))
+            log.warning("SKIPPING problematic module:{}".format(object_expr))
             return
 
-        # self.log.debug("DUMP    : {}".format(object_expr))
+        # # log.debug("DUMP    : {}".format(object_expr))
         items, errors = self.get_obj_attributes(object_expr)
 
         if errors:
-            self.log.error(errors)
+            log.error(errors)
 
         for item_name, item_repr, item_type_txt, item_instance, _ in items:
             # name_, repr_(value), type as text, item_instance, order
@@ -256,7 +274,7 @@ class Stubber:
                 # do not create stubs for these primitives
                 continue
             if item_name[0].isdigit():
-                self.log.warning("NameError: invalid name {}".format(item_name))
+                log.warning("NameError: invalid name {}".format(item_name))
                 continue
             # Class expansion only on first 3 levels (bit of a hack)
             if (
@@ -265,7 +283,7 @@ class Stubber:
                 # and not obj_name.endswith(".Pin")
                 # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms
             ):
-                self.log.info("{0}class {1}:".format(indent, item_name))
+                # log.debug("{0}class {1}:".format(indent, item_name))
                 superclass = ""
                 is_exception = (
                     item_name.endswith("Exception")
@@ -288,7 +306,7 @@ class Stubber:
                 # write classdef
                 fp.write(s)
                 # first write the class literals and methods
-                self.log.debug("# recursion over class {0}".format(item_name))
+                # log.debug("# recursion over class {0}".format(item_name))
                 self.write_object_stub(
                     fp,
                     item_instance,
@@ -302,7 +320,7 @@ class Stubber:
                 s += indent + "        ...\n\n"
                 fp.write(s)
             elif any(word in item_type_txt for word in ["method", "function", "closure"]):
-                self.log.debug("# def {1} function/method/closure, type = '{0}'".format(item_type_txt, item_name))
+                # log.debug("# def {1} function/method/closure, type = '{0}'".format(item_type_txt, item_name))
                 # module Function or class method
                 # will accept any number of params
                 # return type Any/Incomplete
@@ -318,7 +336,7 @@ class Stubber:
                     s = "{}def {}({}*args, **kwargs) -> {}:\n".format(indent, item_name, first, ret)
                 s += indent + "    ...\n\n"
                 fp.write(s)
-                self.log.debug("\n" + s)
+                # log.debug("\n" + s)
             elif item_type_txt == "<class 'module'>":
                 # Skip imported modules
                 # fp.write("# import {}\n".format(item_name))
@@ -328,28 +346,32 @@ class Stubber:
                 t = item_type_txt[8:-2]
                 s = ""
 
-                if t in ["str", "int", "float", "bool", "bytearray", "bytes"]:
+                if t in ("str", "int", "float", "bool", "bytearray", "bytes"):
                     # known type: use actual value
-                    s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, item_repr, t)
-                elif t in ["dict", "list", "tuple"]:
+                    # s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, item_repr, t)
+                    s = "{0}{1}: {3} = {2}\n".format(indent, item_name, item_repr, t)
+                elif t in ("dict", "list", "tuple"):
                     # dict, list , tuple: use empty value
                     ev = {"dict": "{}", "list": "[]", "tuple": "()"}
-                    s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t)
+                    # s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t)
+                    s = "{0}{1}: {3} = {2}\n".format(indent, item_name, ev[t], t)
                 else:
                     # something else
-                    if t in ["object", "set", "frozenset", "Pin", "FileIO"]:
+                    if t in ("object", "set", "frozenset", "Pin", "generator"):  # "FileIO"
                         # https://docs.python.org/3/tutorial/classes.html#item_instance-objects
-                        #  use these types for the attribute
-                        s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
+                        # use these types for the attribute
+                        if t == "generator":
+                            t = "Generator"
+                        s = "{0}{1}: {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
                     else:
                         # Requires Python 3.6 syntax, which is OK for the stubs/pyi
                         t = "Incomplete"
-                        s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
+                        s = "{0}{1}: {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
                 fp.write(s)
-                self.log.debug("\n" + s)
+                # log.debug("\n" + s)
             else:
                 # keep only the name
-                self.log.debug("# all other, type = '{0}'".format(item_type_txt))
+                # log.debug("# all other, type = '{0}'".format(item_type_txt))
                 fp.write("# all other, type = '{0}'\n".format(item_type_txt))
 
                 fp.write(indent + item_name + " # type: Incomplete\n")
@@ -373,10 +395,9 @@ class Stubber:
 
     def clean(self, path: str = None):  # type: ignore
         "Remove all files from the stub folder"
-        wdt.feed()
         if path is None:
             path = self.path
-        self.log.info("Clean/remove files in folder: {}".format(path))
+        log.info("Clean/remove files in folder: {}".format(path))
         try:
             os.stat(path)  # TEMP workaround mpremote listdir bug -
             items = os.listdir(path)
@@ -394,42 +415,53 @@ class Stubber:
                 except OSError:
                     pass
 
-    def report(self, filename: str = "modules.json"):
-        "create json with list of exported modules"
-        wdt.feed()
-        self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path))
-        f_name = "{}/{}".format(self.path, filename)
-        self.log.info("Report file: {}".format(f_name))
+    def report_start(self, filename: str = "modules.json"):
+        """Start a report of the modules that have been stubbed
+        "create json with list of exported modules"""
+        self._json_name = "{}/{}".format(self.path, filename)
+        self._json_first = True
+        ensure_folder(self._json_name)
+        log.info("Report file: {}".format(self._json_name))
         gc.collect()
         try:
             # write json by node to reduce memory requirements
-            with open(f_name, "w") as f:
-                self.write_json_header(f)
-                first = True
-                for n in self._report:
-                    self.write_json_node(f, n, first)
-                    first = False
-                self.write_json_end(f)
-            used = self._start_free - gc.mem_free()  # type: ignore
-            self.log.info("Memory used: {0} Kb".format(used // 1024))
-        except OSError:
-            self.log.error("Failed to create the report.")
-
-    def write_json_header(self, f):
-        f.write("{")
-        f.write(dumps({"firmware": self.info})[1:-1])
-        f.write(",\n")
-        f.write(dumps({"stubber": {"version": __version__}, "stubtype": "firmware"})[1:-1])
-        f.write(",\n")
-        f.write('"modules" :[\n')
+            with open(self._json_name, "w") as f:
+                f.write("{")
+                f.write(dumps({"firmware": self.info})[1:-1])
+                f.write(",\n")
+                f.write(dumps({"stubber": {"version": __version__}, "stubtype": "firmware"})[1:-1])
+                f.write(",\n")
+                f.write('"modules" :[\n')
+
+        except OSError as e:
+            log.error("Failed to create the report.")
+            self._json_name = None
+            raise e
+
+    def report_add(self, module_name: str, stub_file: str):
+        "Add a module to the report"
+        # write json by node to reduce memory requirements
+        if not self._json_name:
+            raise Exception("No report file")
+        try:
+            with open(self._json_name, "a") as f:
+                if not self._json_first:
+                    f.write(",\n")
+                else:
+                    self._json_first = False
+                line = '{{"module": "{}", "file": "{}"}}'.format(module_name, stub_file.replace("\\", "/"))
+                f.write(line)
 
-    def write_json_node(self, f, n, first):
-        if not first:
-            f.write(",\n")
-        f.write(n)
+        except OSError:
+            log.error("Failed to create the report.")
 
-    def write_json_end(self, f):
-        f.write("\n]}")
+    def report_end(self):
+        if not self._json_name:
+            raise Exception("No report file")
+        with open(self._json_name, "a") as f:
+            f.write("\n]}")
+        # is used as sucess indicator
+        log.info("Path: {}".format(self.path))
 
 
 def ensure_folder(path: str):
@@ -475,7 +507,7 @@ def _build(s):
 def _info():  # type:() -> dict[str, str]
     info = OrderedDict(
         {
-            "family": sys.implementation.name,
+            "family": sys.implementation.name,  # type: ignore
             "version": "",
             "build": "",
             "ver": "",
@@ -503,9 +535,9 @@ def _info():  # type:() -> dict[str, str]
         info["board"] = _machine
         info["cpu"] = _machine.split("with")[-1].strip()
         info["mpy"] = (
-            sys.implementation._mpy
+            sys.implementation._mpy  # type: ignore
             if "_mpy" in dir(sys.implementation)
-            else sys.implementation.mpy
+            else sys.implementation.mpy  # type: ignore
             if "mpy" in dir(sys.implementation)
             else ""
         )
@@ -599,79 +631,16 @@ def version_str(version: tuple):  #  -> str:
     return v_str
 
 
-def read_boardname(info, desc: str = ""):
-    info["board"] = info["board"].replace(" ", "_")
-    found = False
-    for filename in [d + "/board_name.txt" for d in LIBS]:
-        wdt.feed()
-        # print("look up the board name in the file", filename)
-        if file_exists(filename):
-            with open(filename, "r") as file:
-                data = file.read()
-            if data:
-                info["board"] = data.strip()
-                found = True
-                break
-    if not found:
-        print("Board not found, guessing board name")
-        descr = ""
-        # descr = desc or info["board"].strip()
-        # if "with " + info["cpu"].upper() in descr:
-        #     # remove the with cpu part
-        #     descr = descr.split("with " + info["cpu"].upper())[0].strip()
-        info["board"] = descr
-
-
-# def read_boardname(info, desc: str = ""):
-#         wdt.feed()
-#         # print("look up the board name in the file", filename)
-#         if file_exists(filename):
-#             descr = desc or info["board"].strip()
-#             pos = descr.rfind(" with")
-#             if pos != -1:
-#                 short_descr = descr[:pos].strip()
-#             else:
-#                 short_descr = ""
-#             print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr))
-#             if find_board(info, descr, filename, short_descr):
-#                 found = True
-#                 break
-#     if not found:
-#         print("Board not found, guessing board name")
-#         descr = desc or info["board"].strip()
-#         if "with " + info["cpu"].upper() in descr:
-#             # remove the with cpu part
-#             descr = descr.split("with " + info["cpu"].upper())[0].strip()
-#         info["board"] = descr
-#     info["board"] = info["board"].replace(" ", "_")
-#     gc.collect()
-
-
-# def find_board(info: dict, descr: str, filename: str, short_descr: str):
-#     "Find the board in the provided board_info.csv file"
-#     short_hit = ""
-#     with open(filename, "r") as file:
-#         # ugly code to make testable in python and micropython
-#         # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file)
-#         while 1:
-#             line = file.readline()
-#             if not line:
-#                 break
-#             descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip()
-#             if descr_ == descr:
-#                 info["board"] = board_
-#                 return True
-#             elif short_descr and descr_ == short_descr:
-#                 if "with" in short_descr:
-#                     # Good enough - no need to trawl the entire file
-#                     info["board"] = board_
-#                     return True
-#                 # good enough if not found in the rest of the file (but slow)
-#                 short_hit = board_
-#     if short_hit:
-#         info["board"] = short_hit
-#         return True
-#     return False
+def get_boardname() -> str:
+    "Read the board name from the boardname.py file that may have been created upfront"
+    try:
+        from boardname import BOARDNAME  # type: ignore
+
+        log.info("Found BOARDNAME: {}".format(BOARDNAME))
+    except ImportError:
+        log.warning("BOARDNAME not found")
+        BOARDNAME = ""
+    return BOARDNAME
 
 
 def get_root() -> str:  # sourcery skip: use-assigned-variable
@@ -724,10 +693,6 @@ def is_micropython() -> bool:
     # pylint: disable=unused-variable,eval-used
     try:
         # either test should fail on micropython
-        # a) https://docs.micropython.org/en/latest/genrst/syntax.html#spaces
-        # Micropython : SyntaxError
-        # a = eval("1and 0")  # lgtm [py/unused-local-variable]
-        # Eval blocks some minification aspects
 
         # b) https://docs.micropython.org/en/latest/genrst/builtin_types.html#bytes-with-keywords-not-implemented
         # Micropython: NotImplementedError
@@ -749,40 +714,40 @@ def main():
     stubber.clean()
     # Read stubs from modulelist in the current folder or in /libs
     # fall back to default modules
-    stubber.modules = []  # avoid duplicates
-    for p in LIBS:
-        try:
-            _1 = gc.mem_free()  # type: ignore
-            with open(p + "/modulelist.txt") as f:
-                print("Debug: List of modules: " + p + "/modulelist.txt")
-                line = f.readline()
-                while line:
-                    line = line.strip()
+    def get_modulelist(stubber):
+        # new
+        gc.collect()
+        stubber.modules = []  # avoid duplicates
+        for p in LIBS:
+            fname = p + "/modulelist.txt"
+            if not file_exists(fname):
+                continue
+            with open(fname) as f:
+                # print("DEBUG: list of modules: " + p + "/modulelist.txt")
+                while True:
+                    line = f.readline().strip()
+                    if not line:
+                        break
                     if len(line) > 0 and line[0] != "#":
                         stubber.modules.append(line)
-                    line = f.readline()
-                gc.collect()  # type: ignore
-                print("Debug: Used memory to load modulelist.txt: " + str(_1 - gc.mem_free()) + " bytes")  # type: ignore
+                gc.collect()
+                print("BREAK")
                 break
-        except Exception:
-            pass
-    if not stubber.modules:
-        stubber.modules = ["micropython"]
-        print("Could not find modulelist.txt, using default modules")
+
+        if not stubber.modules:
+            stubber.modules = ["micropython"]
+            # _log.warn("Could not find modulelist.txt, using default modules")
+        gc.collect()
+
+    stubber.modules = []  # avoid duplicates
+    get_modulelist(stubber)
 
     gc.collect()
 
     stubber.create_all_stubs()
-    stubber.report()
 
 
 if __name__ == "__main__" or is_micropython():
-    try:
-        log = logging.getLogger("stubber")
-        logging.basicConfig(level=logging.INFO)
-        # logging.basicConfig(level=logging.DEBUG)
-    except NameError:
-        pass
     if not file_exists("no_auto_stubber.txt"):
         try:
             gc.threshold(4 * 1024)  # type: ignore