micropython-stubber 1.23.3__py3-none-any.whl → 1.24.1__py3-none-any.whl

stubber/publish/publish.py

@@ -7,8 +7,8 @@ prepare a set of stub files for publishing to PyPi
 from typing import Any, Dict, List, Optional, Union
 
 from mpflash.logger import log
-
 from mpflash.versions import V_PREVIEW
+
 from stubber.publish.candidates import board_candidates, filter_list
 from stubber.publish.database import get_database
 from stubber.publish.defaults import GENERIC_U
@@ -33,7 +33,7 @@ def build_multiple(
     ports = ports or ["all"]
     boards = boards or [GENERIC_U]
 
-    db = get_database(CONFIG.publish_path, production=production)
+    db_conn = get_database(CONFIG.publish_path, production=production)
     results: List[Dict[str, Any]] = []
     worklist = build_worklist(family, versions, ports, boards)
     if len(worklist) == 0:
@@ -42,7 +42,7 @@ def build_multiple(
     log.info(f"checking {len(worklist)} possible board candidates")
 
     for todo in worklist:
-        if package := get_package(db, **todo):
+        if package := get_package(db_conn, **todo):
             package.build_distribution(force=force, production=production)
             results.append(package.status)
         else:
@@ -69,18 +69,18 @@ def publish_multiple(
     ports = ports or ["all"]
     boards = boards or [GENERIC_U]
 
-    db = get_database(CONFIG.publish_path, production=production)
+    db_conn = get_database(CONFIG.publish_path, production=production)
     results = []
     worklist = build_worklist(family, versions, ports, boards)
 
     if len(worklist) == 0:
-        log.error("Could not find any packages than can be published.")
+        log.error("Could not find any packages that can be published.")
         return results
 
     for todo in worklist:
-        if package := get_package(db, **todo):
+        if package := get_package(db_conn, **todo):
             package.publish_distribution_ifchanged(
-                db=db,
+                db_conn=db_conn,
                 clean=clean,
                 force=force,
                 build=build,
@@ -113,7 +113,7 @@ def build_worklist(
     worklist = filter_list(worklist, ports, boards)
 
     for b in boards:
-        if b == "auto":
+        if b in ["auto", "all", "*"]:
             continue
         if not any(i for i in worklist if i["board"].lower() == b.lower()):
             log.warning(f"Could not find any package candidate for board {b}")

stubber/publish/stubpackage.py

@@ -3,29 +3,29 @@
 import hashlib
 import json
 import shutil
+import sqlite3
 import subprocess
-from pathlib import Path
 import sys
+from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 import tenacity
-
 from mpflash.basicgit import get_git_describe
+
 from stubber.publish.helpers import get_module_docstring
 
 if sys.version_info >= (3, 11):
-    import tomllib # type: ignore
+    import tomllib  # type: ignore
 else:
-    import tomli as tomllib # type: ignore
+    import tomli as tomllib  # type: ignore
 
 from typing import NewType
 
 import tomli_w
 from mpflash.logger import log
+from mpflash.versions import SET_PREVIEW, V_PREVIEW, clean_version
 from packaging.version import Version, parse
-from pysondb import PysonDB
 
-from mpflash.versions import SET_PREVIEW, V_PREVIEW, clean_version
 from stubber.publish.bump import bump_version
 from stubber.publish.defaults import GENERIC_U, default_board
 from stubber.publish.enums import StubSource
@@ -53,7 +53,7 @@ STUBS_COPY_FILTER = {
 STDLIB_UMODULES = ["ucollections"]
 
 
-class VersionedPackage(object):
+class VersionedPackage:
     """
     Represents a versioned package.
 
@@ -82,6 +82,7 @@ class VersionedPackage(object):
         return f"{self.package_name}=={self.mpy_version}"
 
     def __repr__(self) -> str:
+        # todo: use pkg_version to allow for comparing different versions of the same package
         return f"{self.package_name}=={self.mpy_version}"
 
     def __eq__(self, o: object) -> bool:
@@ -215,6 +216,7 @@ class Builder(VersionedPackage):
         mpy_version: str = "0.0.1",
         port: str,
         board: str = GENERIC_U,
+        variant: Optional[str] = None,
         description: str = "MicroPython stubs",
         stubs: Optional[StubSources] = None,
         # json_data: Optional[Dict[str, Any]] = None,
@@ -225,6 +227,7 @@ class Builder(VersionedPackage):
         self.mpy_version = mpy_version
         self.port = port
         self.board = board
+        self.variant = variant or ""
         self.description = description
         self.stub_sources = stubs or []
         self.hash = None  # intial hash
@@ -252,8 +255,12 @@ class Builder(VersionedPackage):
         pyproject = None
         _toml = self.toml_path
         if (_toml).exists():
-            with open(_toml, "rb") as f:
-                pyproject = tomllib.load(f)
+            log.debug(f"Load pyproject from {_toml}")
+            try:
+                with open(_toml, "rb") as f:
+                    pyproject = tomllib.load(f)
+            except tomllib.TOMLDecodeError as e:
+                log.error(f"Could not load pyproject.toml file {e}")
         return pyproject
 
     @pyproject.setter
@@ -340,7 +347,7 @@ class Builder(VersionedPackage):
     def update_umodules(self):
         """
         Replace the STDLIB umodules with a simple import statement
-        in order to allow the typecheckers to resove the stdlib modules in the usual stdlib location.
+        in order to allow the typecheckers to resovle the stdlib modules in the usual stdlib location.
         """
         for f in self.package_path.rglob("*.pyi"):
             if f.stem in STDLIB_UMODULES:
@@ -386,10 +393,12 @@ class Builder(VersionedPackage):
             json.dump(self.to_dict(), f, indent=4)
 
     def to_dict(self) -> dict:
-        """return the package as a dict to store in the jsondb
+        """return the package as a Dict[str,str] to store in the db
 
         need to simplify some of the Objects to allow serialization to json
-        - the paths to posix paths
+        - stub_sources
+            List[Tuple[StubSource, Path]] -> List[List[str, str]]  # convert
+            the paths to posix paths
         - the version (semver) to a string
         - toml file to list of lines
 
@@ -400,14 +409,19 @@ class Builder(VersionedPackage):
             "publish": self._publish,
             "pkg_version": str(self.pkg_version),
             "path": self.package_path.name,  # only store the folder name , as it is relative to the publish folder
-            "stub_sources": [(name, Path(path).as_posix()) for (name, path) in self.stub_sources],
+            "stub_sources": json.dumps(
+                [(name, Path(path).as_posix()) for (name, path) in self.stub_sources]
+            ),
             "description": self.description,
             "hash": self.hash,
             "stub_hash": self.stub_hash,
+            "port": self.port,
+            "board": self.board,
+            "variant": "",  # TODO: get the variant
         }
 
     def from_dict(self, json_data: Dict) -> None:
-        """load the package from a dict (from the jsondb)"""
+        """load the package from a dict (from the db)"""
         self.package_name = json_data["name"]
         # self.package_path = Path(json_data["path"])
         self.description = json_data["description"]
@@ -415,6 +429,10 @@ class Builder(VersionedPackage):
         self._publish = json_data["publish"]
         self.hash = json_data["hash"]
         self.stub_hash = json_data["stub_hash"]
+        self.port = json_data["port"]
+        self.board = json_data["board"]
+        self.variant = json_data["variant"]
+
         # create folder
         if not self.package_path.exists():
             self.package_path.mkdir(parents=True, exist_ok=True)
@@ -423,7 +441,7 @@ class Builder(VersionedPackage):
         # set pkg version after creating the toml file
         self.pkg_version = json_data["pkg_version"]
         self.stub_sources = []
-        for name, path in json_data["stub_sources"]:
+        for name, path in json.loads(json_data["stub_sources"]):
             if path.startswith("stubs/"):
                 path = path.replace("stubs/", "")
             self.stub_sources.append((name, Path(path)))
@@ -529,13 +547,13 @@ class Builder(VersionedPackage):
         with open(self.package_path / "README.md", "w") as f:
             f.write(f"# {self.package_name}\n\n")
             f.write(TEMPLATE_README)
-            f.write(f"Included stubs:\n")
+            f.write("Included stubs:\n")
             for name, folder in self.stub_sources:
                 f.write(f"* {name} from `stubs/{Path(folder).as_posix()}`\n")
 
-            f.write(f"\n\n")
-            f.write(f"origin | Family | Port | Board | Version\n")
-            f.write(f"-------|--------|------|-------|--------\n")
+            f.write("\n\n")
+            f.write("origin | Family | Port | Board | Version\n")
+            f.write("-------|--------|------|-------|--------\n")
             try:
                 f.write(
                     f"Firmware | {firmware_stubs['firmware']['family']} | {firmware_stubs['firmware']['port']} | {firmware_stubs['firmware']['machine']} | {clean_version(firmware_stubs['firmware']['version'])} \n"
@@ -592,7 +610,14 @@ class PoetryBuilder(Builder):
             return self.mpy_version
         with open(_toml, "rb") as f:
             pyproject = tomllib.load(f)
-        ver = pyproject["tool"]["poetry"]["version"]
+        # read the version new pyproject format / old format /  self.mpy_version
+        try:
+            ver = pyproject["project"]["version"]
+        except KeyError:
+            try:
+                ver = pyproject["tool"]["poetry"]["version"]
+            except KeyError:
+                ver = self.mpy_version
         return str(parse(ver)) if ver not in SET_PREVIEW else ver
 
     @pkg_version.setter
@@ -606,7 +631,8 @@ class PoetryBuilder(Builder):
         try:
             with open(_toml, "rb") as f:
                 pyproject = tomllib.load(f)
-            pyproject["tool"]["poetry"]["version"] = version
+            # pyproject["tool"]["poetry"]["version"] = version
+            pyproject["project"]["version"] = version
             # update the version in the toml file
             with open(_toml, "wb") as output:
                 tomli_w.dump(pyproject, output)
@@ -694,7 +720,14 @@ class PoetryBuilder(Builder):
             # update the dependencies section by reading these from the template file
             with open(CONFIG.template_path / "pyproject.toml", "rb") as f:
                 tpl = tomllib.load(f)
-            _pyproject["tool"]["poetry"]["dependencies"] = tpl["tool"]["poetry"]["dependencies"]
+
+            # copy new / poetry style dependencies
+            for section in ["dependencies"]:
+                for key in ["tool.poetry", "project"]:
+                    try:
+                        _pyproject[key][section] = tpl[key][section]
+                    except KeyError:
+                        pass
 
         else:
             # read the template pyproject.toml file from the template folder
@@ -702,14 +735,17 @@ class PoetryBuilder(Builder):
                 with open(CONFIG.template_path / "pyproject.toml", "rb") as f:
                     _pyproject = tomllib.load(f)
                 # note: can be 'latest' which is not semver
-                _pyproject["tool"]["poetry"]["version"] = self.mpy_version
+                # _pyproject["tool"]["poetry"]["version"] = self.mpy_version
+                _pyproject["project"]["version"] = self.mpy_version
             except FileNotFoundError as e:
                 log.error(f"Could not find template pyproject.toml file {e}")
                 raise (e)
 
         # update the name , version and description of the package
-        _pyproject["tool"]["poetry"]["name"] = self.package_name
-        _pyproject["tool"]["poetry"]["description"] = self.description
+        # _pyproject["tool"]["poetry"]["name"] = self.package_name
+        # _pyproject["tool"]["poetry"]["description"] = self.description
+        _pyproject["project"]["name"] = self.package_name
+        _pyproject["project"]["description"] = self.description
         # write out the pyproject.toml file
         self.pyproject = _pyproject
 
@@ -779,6 +815,14 @@ class StubPackage(PoetryBuilder):
             STUB_PATH - root-relative path to the folder where the stubs are stored ('./stubs').
 
         """
+        super().__init__(
+            package_name=package_name,
+            mpy_version=clean_version(version, drop_v=True),  # Initial version
+            port=port,
+            board=board,
+            description=description,
+            stubs=stubs or [],
+        )
         self.port = port
         self.board = board
         if json_data is not None:
@@ -787,14 +831,14 @@ class StubPackage(PoetryBuilder):
             # store essentials
             self.package_name = package_name
             self.description = description
-            self.mpy_version = clean_version(version, drop_v=True)  # Initial version
+            # self.mpy_version = clean_version(version, drop_v=True)  # Initial version
 
             self.create_update_pyproject_toml()
 
-            self.stub_sources: StubSources = []
-            # save the stub sources
-            if stubs:
-                self.stub_sources = stubs
+            # self.stub_sources: StubSources = []
+            # # save the stub sources
+            # if stubs:
+            #     self.stub_sources = stubs
 
         self.status: Status = Status(
             {
@@ -805,14 +849,6 @@ class StubPackage(PoetryBuilder):
                 "path": self.package_path.as_posix(),
             }
         )
-        super().__init__(
-            package_name=package_name,
-            mpy_version=self.mpy_version,
-            port=port,
-            board=board,
-            description=description,
-            stubs=self.stub_sources,
-        )
 
     def update_sources(self) -> StubSources:
         """
@@ -838,8 +874,9 @@ class StubPackage(PoetryBuilder):
                 # use if folder exists , else use GENERIC folder
                 if (CONFIG.stub_path / fw_path).exists():
                     updated_sources.append((stub_type, fw_path))
-                elif fw_path.with_name("GENERIC").exists():
+                elif (CONFIG.stub_path / fw_path).with_name("GENERIC").exists():
                     updated_sources.append((stub_type, fw_path.with_name("GENERIC")))
+
             elif stub_type == StubSource.MERGED:
                 # Use the default board folder instead of the GENERIC board folder (if it exists)
                 if self.board.upper() == GENERIC_U:
@@ -857,7 +894,7 @@ class StubPackage(PoetryBuilder):
 
     def update_distribution(self, production: bool) -> bool:
         """Update the package .pyi files, if all the sources are available"""
-        log.info(f"- Update {self.package_path.name}")
+        log.debug(f"- Update {self.package_path.name}")
         log.trace(f"{self.package_path.as_posix()}")
 
         # check if the sources exist
@@ -866,7 +903,7 @@ class StubPackage(PoetryBuilder):
             log.debug(
                 f"{self.package_name}: skipping as one or more source stub folders are missing"
             )
-            self.status["error"] = "Skipped, stub folder(s) missing"
+            self.status["error"] = "Skipped, No stubs found."
             shutil.rmtree(self.package_path.as_posix())
             self._publish = False  # type: ignore
             return False
@@ -900,15 +937,15 @@ class StubPackage(PoetryBuilder):
         :param force: BUILD even if no changes
         :return: True if the package was built
         """
-        log.info(f"Build: {self.package_path.name}")
 
         ok = self.update_distribution(production)
         self.status["version"] = self.pkg_version
         if not ok:
-            log.info(f"{self.package_name}: skip - Could not build/update package")
+            log.debug(f"{self.package_name}: skip - Could not build/update package")
             if not self.status["error"]:
                 self.status["error"] = "Could not build/update package"
             return False
+        log.info(f"Build: {self.package_path.name}")
 
         # If there are changes to the package, then publish it
         if self.is_changed() or force:
@@ -939,7 +976,7 @@ class StubPackage(PoetryBuilder):
 
     def publish_distribution_ifchanged(
         self,
-        db: PysonDB,
+        db_conn: sqlite3.Connection,
         *,
         production: bool,  # PyPI or Test-PyPi
         build=False,  #
@@ -984,9 +1021,9 @@ class StubPackage(PoetryBuilder):
                 )
                 self.status["result"] = "Published to GitHub"
             else:
-                return self.publish_distribution(dry_run, production, db)
+                return self.publish_distribution(dry_run, production, db_conn)
         elif force:
-            return self.publish_distribution(dry_run, production, db)
+            return self.publish_distribution(dry_run, production, db_conn)
         else:
             log.info(f"No changes to package : {self.package_name} {self.pkg_version}")
 
@@ -994,7 +1031,12 @@ class StubPackage(PoetryBuilder):
             self.clean()
         return True
 
-    def publish_distribution(self, dry_run, production, db):
+    def publish_distribution(
+        self,
+        dry_run: bool,
+        production: bool,
+        db_conn: sqlite3.Connection,
+    ) -> bool:
         """
         Publishes the package to PyPi or Test-PyPi.
 
@@ -1023,9 +1065,34 @@ class StubPackage(PoetryBuilder):
         if dry_run:
             log.warning(f"{self.package_name}: Dry run, not saving to database")
         else:
-            # get the package state and add it to the database
-            db.add(self.to_dict())
-            db.commit()
+            cursor = db_conn.cursor()
+
+            d = self.to_dict()
+
+            cursor.execute(
+                """
+                INSERT INTO packages (name, description, mpy_version, pkg_version, publish, stub_sources, path, hash, stub_hash, port, board, variant)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    d["name"],
+                    d["description"],
+                    d["mpy_version"],
+                    d["pkg_version"],
+                    d["publish"],
+                    d["stub_sources"],
+                    d["path"],
+                    d["hash"],
+                    d["stub_hash"],
+                    d["port"],
+                    d["board"],
+                    d["variant"],
+                ),
+            )
+
+            # # get the package state and add it to the database
+            # db_conn.add(self.to_dict())
+            db_conn.commit()
         return True
 
     def are_package_sources_available(self) -> bool: