idf-build-apps 2.0.0rc0__tar.gz → 2.0.0rc1__tar.gz

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/.github/workflows/check-pre-commit.yml

@@ -11,4 +11,8 @@ jobs:
         with:
           fetch-depth: 0
       - uses: actions/setup-python@v5
+      - id: changed-files
+        uses: tj-actions/changed-files@v41
       - uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }}

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/.github/workflows/test-build-idf-apps.yml

@@ -28,7 +28,7 @@ jobs:
           pip install flit
           flit build
       - name: Upload built python packages
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: wheel
           path: dist/idf_build_apps-*.whl
@@ -38,13 +38,13 @@ jobs:
     needs: build-python-packages
     strategy:
       matrix:
-        idf-branch: [ release-v5.0, release-v5.1 ]
+        idf-branch: [ release-v5.0, release-v5.1, release-v5.2 ]
     runs-on: ubuntu-latest
     container:
       image: espressif/idf:${{ matrix.idf-branch }}
     steps:
       - name: Download wheel
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: wheel
       - name: Build the Apps

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/CHANGELOG.md

@@ -2,6 +2,26 @@
 
 All notable changes to this project will be documented in this file.
 
+## v2.0.0rc1 (2024-01-05)
+
+### BREAKING CHANGE
+
+-  - Turn `app.build()` arguments to kwargs
+
+### Feat
+
+- support custom `_pre_build`, `_post_build` in App instances
+- add `json_to_app` method with support custom classes
+
+### Fix
+
+- remove os-specific `os.mknod`
+- sort App instance correctly
+- modify yaml dict shared by yaml anchors
+- make build_comment a `dump_only` field
+- improve error message when env var IDF_PATH not set
+- search sdkconfig path
+
 ## v2.0.0rc0 (2023-12-18)
 
 ### Refactor
@@ -106,6 +126,12 @@ All notable changes to this project will be documented in this file.
 - migrate `App` class to pydantic model
 - update dependencies and do code upgrade to python 3.7
 
+## v1.1.4 (2023-12-29)
+
+### Fix
+
+- stop modifying yaml dict shared by yaml anchors
+
 ## v1.1.3 (2023-11-13)
 
 ### Fix

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: idf-build-apps
-Version: 2.0.0rc0
+Version: 2.0.0rc1
 Summary: Tools for building ESP-IDF related apps.
 Author-email: Fu Hanxi <fuhanxi@espressif.com>
 Requires-Python: >=3.7

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/docs/index.rst

@@ -17,5 +17,6 @@ Welcome to idf-build-apps's documentation!
    config_file
    cli
    api/modules
+   migration/1.x_to_2.x
    CHANGELOG
    CONTRIBUTING

idf_build_apps-2.0.0rc1/docs/migration/1.x_to_2.x.md

@@ -0,0 +1,135 @@
+# Migration From 1.x to 2.x
+
+There are a few breaking changes in 2.x. This document will help you migrate your code from 1.x to 2.x.
+
+## Python Version Support
+
+idf-build-apps 1.x supports Python 2.7 and Python 3.4 or newer. idf-build-apps 2.x only supports Python 3.7 or newer.
+
+## Logging Related Changes
+
+In 2.x, we're following the standard Python logging convention.
+
+Before:
+
+```python
+from idf_build_apps import LOGGER
+```
+
+After:
+
+```python
+import logging
+idf_build_apps_logger = logging.getLogger('idf_build_apps')
+```
+
+## Normal Arguments to Keyword-only Arguments
+
+In 2.x, we move some arguments from normal arguments to keyword-only arguments. This is to make the API more consistent and easier to use.
+
+To understand the difference between these terms better, here's a quick summary:
+- "positonal-only argument" means the argument is a positional-only argument. (python 3.8+ only)
+- "keyword-only argument" means the argument is a keyword-only argument.
+- "normal argument" means the argument is not a positional-only argument, nor a keyword-only argument.
+
+For example, in the following function:
+
+```python
+def foo(a, /, b, c, *, d=1, e=2, f=3):
+    pass
+```
+
+- "a" is a positional-only argument.
+- "b" and "c" are normal arguments.
+- "d", "e", and "f" are keyword-only arguments.
+
+The following calls are valid:
+
+```python
+foo(1, 2, 3, d=4, e=5, f=6)
+foo(1, 2, c=3, d=4, e=5, f=6)
+foo(1, b=2, c=3, d=4, e=5, f=6)
+```
+
+The following calls are invalid:
+
+```python
+foo(1, 2, 3, 4, 5, 6)
+foo(1, b=2, 3, d=4, e=5, f=6)
+foo(a=1, b=2, c=3, d=4, e=5, f=6)
+```
+
+### `App.__init__()`
+
+The `__init__` function of `App` class, and all its sub-classes, like `CMakeApp`, and `MakeApp`, now takes only `app_dir`, and `target` as normal arguments. All the rest of the arguments are keyword-only arguments.
+
+Before:
+
+```python
+app = App('foo', 'esp32', 'sdkconfig.ci', 'default')
+```
+
+After:
+
+```python
+app = App('foo', 'esp32', sdkconfig_path='sdkconfig.ci', config_name='default')
+```
+
+or all in keyword-only arguments:
+
+```python
+app = App(app_dir='foo', target='esp32', sdkconfig_path='sdkconfig.ci', config_name='default')
+```
+
+### `App.build()`
+
+The `build` function of `App` class, and all its sub-classes, like `CMakeApp`, and `MakeApp`, now takes all arguments as keyword-only arguments.
+
+### `find_apps()`
+
+The `find_apps` function now takes only `paths` and `target` as normal arguments. All the rest of the arguments are keyword-only arguments.
+
+### `build_apps()`
+
+The `build_apps` function now takes only `apps` as normal argument. All the rest of the arguments are keyword-only arguments.
+
+## Function Signature Changes
+
+In 2.x, we change the signature of some functions to make them more intuitive and self-explanatory.
+
+### `find_apps()`
+
+- `build_log_path` is renamed to `build_log_filename`. The file will be generated under `build_dir` if specified.
+- `size_json_path` is renamed to `size_json_filename`. The file will be generated under `build_dir` if specified.
+
+## CLI Changes
+
+In 2.x, we change the separator for some options to better differentiate them from `None` and empty list.
+
+- `--modified-components`
+- `--modified-files`
+- `--ignore-app-dependencies-filepatterns`
+
+Before:
+
+```shell
+idf-build-apps build -p foo -t esp32 --modified-components foo bar --modified-files foo bar --ignore-app-dependencies-filepatterns foo bar
+```
+
+After:
+
+```shell
+idf-build-apps build -p foo -t esp32 --modified-components 'foo;bar' --modified-files 'foo;bar' --ignore-app-dependencies-filepatterns 'foo;bar'
+```
+
+passing `''` to specify it as `None`
+
+```shell
+idf-build-apps build -p foo -t esp32 --modified-components ''
+```
+
+or passing `';'` to specify it as an empty list
+
+```shell
+idf-build-apps build -p foo -t esp32 --modified-components ';'
+```

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/idf_build_apps/__init__.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 """
@@ -8,7 +8,7 @@ Tools for building ESP-IDF related apps.
 # ruff: noqa: E402
 # avoid circular imports
 
-__version__ = '2.0.0rc0'
+__version__ = '2.0.0rc1'
 
 from .session_args import (
     SessionArgs,
@@ -28,6 +28,7 @@ from .log import (
 from .main import (
     build_apps,
     find_apps,
+    json_to_app,
 )
 
 __all__ = [
@@ -37,6 +38,6 @@ __all__ = [
     'MakeApp',
     'find_apps',
     'build_apps',
+    'json_to_app',
     'setup_logging',
-    'SESSION_ARGS',
 ]

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/idf_build_apps/app.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import functools
@@ -8,11 +8,7 @@ import os
 import re
 import shutil
 import sys
-import tempfile
 import typing as t
-from copy import (
-    deepcopy,
-)
 from datetime import (
     datetime,
 )
@@ -28,10 +24,9 @@ from pydantic import (
     computed_field,
 )
 
-from idf_build_apps import (
+from . import (
     SESSION_ARGS,
 )
-
 from .build_apps_args import (
     BuildAppsArgs,
 )
@@ -132,12 +127,16 @@ class App(BaseModel):
 
     # build status related
     build_status: BuildStatus = BuildStatus.UNKNOWN
+    build_comment: t.Optional[str] = None
 
-    _build_comment: t.Optional[str] = None
     _build_stage: t.Optional[BuildStage] = None
     _build_duration: float = 0
     _build_timestamp: t.Optional[datetime] = None
 
+    __EQ_IGNORE_FIELDS__ = [
+        'build_comment',
+    ]
+
     def __init__(
         self,
         app_dir: str,
@@ -164,6 +163,7 @@ class App(BaseModel):
 
         self._build_log_filename = build_log_filename
         self._size_json_filename = size_json_filename
+        self._is_build_log_path_temp = not bool(build_log_filename)
 
         # pass all parameters to initialize hook method
         kwargs.update(
@@ -297,25 +297,18 @@ class App(BaseModel):
 
         return os.path.join(self.work_dir, self.build_dir)
 
-    @property
-    def build_comment(self) -> str:
-        return self._build_comment or ''
-
-    @build_comment.setter
-    def build_comment(self, value: str) -> None:
-        self._build_comment = value
-
     @computed_field  # type: ignore
     @property
     def build_log_filename(self) -> t.Optional[str]:
         return self._expand(self._build_log_filename)
 
     @property
-    def build_log_path(self) -> t.Optional[str]:
+    def build_log_path(self) -> str:
         if self.build_log_filename:
             return os.path.join(self.build_path, self.build_log_filename)
 
-        return None
+        # use a temp file if build log path is not specified
+        return os.path.join(self.build_path, f'.temp.build.{hash(self)}.log')
 
     @computed_field  # type: ignore
     @property
@@ -340,17 +333,21 @@ class App(BaseModel):
         real_sdkconfig_files: t.List[str] = []
         sdkconfig_files_defined_target: t.Optional[str] = None
 
+        # put the expanded variable files in a temporary directory
+        # will remove if the content is the same as the original one
         expanded_dir = os.path.join(self.work_dir, 'expanded_sdkconfig_files', os.path.basename(self.build_dir))
         if not os.path.isdir(expanded_dir):
             os.makedirs(expanded_dir)
 
         for f in self.sdkconfig_defaults_candidates + ([self.sdkconfig_path] if self.sdkconfig_path else []):
-            if not os.path.isabs(f):
-                f = os.path.join(self.work_dir, f)
-
+            # use filepath if abs/rel already point to itself
             if not os.path.isfile(f):
-                self._logger.debug('sdkconfig file %s not exists, skipping...', f)
-                continue
+                # find it in the app_dir
+                self._logger.debug('sdkconfig file %s not found, checking under app_dir...', f)
+                f = os.path.join(self.app_dir, f)
+                if not os.path.isfile(f):
+                    self._logger.debug('sdkconfig file %s not found, skipping...', f)
+                    continue
 
             expanded_fp = os.path.join(expanded_dir, os.path.basename(f))
             with open(f) as fr:
@@ -469,14 +466,7 @@ class App(BaseModel):
 
         return wrapper
 
-    @record_build_duration  # type: ignore
-    def build(
-        self,
-        manifest_rootpath: t.Optional[str] = None,
-        modified_components: t.Union[t.List[str], str, None] = None,
-        modified_files: t.Union[t.List[str], str, None] = None,
-        check_app_dependencies: bool = False,
-    ) -> None:
+    def _pre_build(self) -> None:
         if self.dry_run:
             self._build_stage = BuildStage.DRY_RUN
         else:
@@ -518,26 +508,32 @@ class App(BaseModel):
             if not self.dry_run:
                 os.unlink(sdkconfig_file)
 
-        if self.build_log_path:
-            self._logger.info('Writing build log to %s', self.build_log_path)
+        if os.path.isfile(self.build_log_path):
+            self._logger.debug('Removed existing build log file: %s', self.build_log_path)
+            if not self.dry_run:
+                os.unlink(self.build_log_path)
+        elif not self.dry_run:
+            os.makedirs(os.path.dirname(self.build_log_path), exist_ok=True)
+        self._logger.info('Writing build log to %s', self.build_log_path)
 
         if self.dry_run:
             self.build_status = BuildStatus.SKIPPED
             self.build_comment = 'dry run'
             return
 
-        if self.build_log_path:
-            logfile: t.IO[str] = open(self.build_log_path, 'w')
-            keep_logfile = True
-        else:
-            # delete manually later, used for tracking debugging info
-            logfile = tempfile.NamedTemporaryFile('w', delete=False)
-            keep_logfile = False
+    @record_build_duration  # type: ignore
+    def build(
+        self,
+        *,
+        manifest_rootpath: t.Optional[str] = None,
+        modified_components: t.Union[t.List[str], str, None] = None,
+        modified_files: t.Union[t.List[str], str, None] = None,
+        check_app_dependencies: bool = False,
+    ) -> None:
+        self._pre_build()
 
-        self._build_stage = BuildStage.BUILD
         try:
             self._build(
-                logfile=logfile,
                 manifest_rootpath=manifest_rootpath,
                 modified_components=to_list(modified_components),
                 modified_files=to_list(modified_files),
@@ -546,12 +542,17 @@ class App(BaseModel):
         except BuildError as e:
             self.build_status = BuildStatus.FAILED
             self.build_comment = str(e)
-        finally:
-            logfile.close()
 
+        self._post_build()
+
+    def _post_build(self) -> None:
         self._build_stage = BuildStage.POST_BUILD
+
+        if not os.path.isfile(self.build_log_path):
+            return
+
         has_unignored_warning = False
-        with open(logfile.name) as fr:
+        with open(self.build_log_path) as fr:
             lines = [line.rstrip() for line in fr.readlines() if line.rstrip()]
             for line in lines:
                 is_error_or_warning, ignored = self.is_error_or_warning(line)
@@ -567,15 +568,14 @@ class App(BaseModel):
             self._logger.error(
                 'Last %s lines from the build log "%s":',
                 self.LOG_DEBUG_LINES,
-                logfile.name,
+                self.build_log_path,
             )
             for line in lines[-self.LOG_DEBUG_LINES :]:
                 self._logger.error('%s', line)
 
-        # remove the log file if not specified and build succeeded
-        if not keep_logfile and self.build_status == BuildStatus.SUCCESS:
-            os.unlink(logfile.name)
-            self._logger.debug('Removed temporary build log file: %s', logfile.name)
+        if self._is_build_log_path_temp and self.build_status == BuildStatus.SUCCESS:
+            os.unlink(self.build_log_path)
+            self._logger.debug('Removed success build temporary log file: %s', self.build_log_path)
 
         # Generate Size Files
         if self.build_status == BuildStatus.SUCCESS:
@@ -586,8 +586,7 @@ class App(BaseModel):
             exclude_list = []
             if self.size_json_path:
                 exclude_list.append(os.path.basename(self.size_json_path))
-            if self.build_log_path:
-                exclude_list.append(os.path.basename(self.build_log_path))
+            exclude_list.append(os.path.basename(self.build_log_path))
 
             rmdir(
                 self.build_path,
@@ -604,13 +603,13 @@ class App(BaseModel):
 
     def _build(
         self,
-        logfile: t.IO[str],
+        *,
         manifest_rootpath: t.Optional[str] = None,
         modified_components: t.Optional[t.List[str]] = None,
         modified_files: t.Optional[t.List[str]] = None,
         check_app_dependencies: bool = False,
     ) -> None:
-        pass
+        self._build_stage = BuildStage.BUILD
 
     def _write_size_json(self) -> None:
         if not self.size_json_path:
@@ -779,12 +778,19 @@ class MakeApp(App):
 
     def _build(
         self,
-        logfile: t.IO[str],
+        *,
         manifest_rootpath: t.Optional[str] = None,
         modified_components: t.Optional[t.List[str]] = None,
         modified_files: t.Optional[t.List[str]] = None,
         check_app_dependencies: bool = False,
     ) -> None:
+        super()._build(
+            manifest_rootpath=manifest_rootpath,
+            modified_components=modified_components,
+            modified_files=modified_files,
+            check_app_dependencies=check_app_dependencies,
+        )
+
         # additional env variables
         additional_env_dict = {
             'IDF_TARGET': self.target,
@@ -801,8 +807,8 @@ class MakeApp(App):
         for cmd in commands:
             subprocess_run(
                 cmd,
-                log_terminal=False if self.build_log_path else True,
-                log_fs=logfile,
+                log_terminal=self._is_build_log_path_temp,
+                log_fs=self.build_log_path,
                 check=True,
                 additional_env_dict=additional_env_dict,
                 cwd=self.work_dir,
@@ -846,12 +852,19 @@ class CMakeApp(App):
 
     def _build(
         self,
-        logfile: t.IO[str],
+        *,
         manifest_rootpath: t.Optional[str] = None,
         modified_components: t.Optional[t.List[str]] = None,
         modified_files: t.Optional[t.List[str]] = None,
         check_app_dependencies: bool = False,
     ) -> None:
+        super()._build(
+            manifest_rootpath=manifest_rootpath,
+            modified_components=modified_components,
+            modified_files=modified_files,
+            check_app_dependencies=check_app_dependencies,
+        )
+
         if not self._checked_should_build:
             self._check_should_build(
                 manifest_rootpath=manifest_rootpath,
@@ -883,8 +896,8 @@ class CMakeApp(App):
         if modified_components is not None and check_app_dependencies and self.build_status == BuildStatus.UNKNOWN:
             subprocess_run(
                 common_args + ['reconfigure'],
-                log_terminal=False if self.build_log_path else True,
-                log_fs=logfile,
+                log_terminal=self._is_build_log_path_temp,
+                log_fs=self.build_log_path,
                 check=True,
                 additional_env_dict=additional_env_dict,
             )
@@ -905,23 +918,22 @@ class CMakeApp(App):
             return
 
         # idf.py build
-        build_args = deepcopy(common_args)
         if self.cmake_vars:
             for key, val in self.cmake_vars.items():
-                build_args.append(f'-D{key}={val}')
+                common_args.append(f'-D{key}={val}')
             if 'TEST_EXCLUDE_COMPONENTS' in self.cmake_vars and 'TEST_COMPONENTS' not in self.cmake_vars:
-                build_args.append('-DTESTS_ALL=1')
+                common_args.append('-DTESTS_ALL=1')
             if 'CONFIG_APP_BUILD_BOOTLOADER' in self.cmake_vars:
                 # In case if secure_boot is enabled then for bootloader build need to add `bootloader` cmd
-                build_args.append('bootloader')
-        build_args.append('build')
+                common_args.append('bootloader')
+        common_args.append('build')
         if self.verbose:
-            build_args.append('-v')
+            common_args.append('-v')
 
         subprocess_run(
-            build_args,
-            log_terminal=False if self.build_log_path else True,
-            log_fs=logfile,
+            common_args,
+            log_terminal=self._is_build_log_path_temp,
+            log_fs=self.build_log_path,
             check=True,
             additional_env_dict=additional_env_dict,
         )

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/idf_build_apps/constants.py

@@ -24,9 +24,9 @@ if _BUILDING_DOCS:
 if _BUILDING_DOCS:
     _idf_env = tempfile.gettempdir()
 else:
-    _idf_env = os.getenv('IDF_PATH', '')
-if not os.path.isdir(_idf_env):
-    raise ValueError(f'Invalid value for IDF_PATH: {_idf_env}')
+    _idf_env = os.getenv('IDF_PATH') or ''
+    if not _idf_env:
+        raise SystemExit('environment variable IDF_PATH must be set')
 
 
 IDF_PATH = os.path.abspath(_idf_env)

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/idf_build_apps/main.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
@@ -11,11 +11,17 @@ import sys
 import textwrap
 import typing as t
 
+from pydantic import (
+    Field,
+    create_model,
+)
+
 from . import (
     SESSION_ARGS,
 )
 from .app import (
     App,
+    AppDeserializer,
     CMakeApp,
     MakeApp,
 )
@@ -293,7 +299,6 @@ def build_apps(
         if f and os.path.isfile(f):
             os.remove(f)
             LOGGER.debug('Remove existing collect file %s', f)
-            os.mknod(f)
 
     exit_code = 0
     for i, app in enumerate(apps):
@@ -742,7 +747,7 @@ def main():
             os.makedirs(os.path.dirname(os.path.realpath(args.output)), exist_ok=True)
             with open(args.output, 'w') as fw:
                 for app in apps:
-                    fw.write(app.model_dump_json() + '\n')
+                    fw.write(app.to_json() + '\n')
         else:
             for app in apps:
                 print(app)
@@ -791,3 +796,37 @@ def main():
             print(f'  {app}')
 
     sys.exit(res)
+
+
+def json_to_app(json_str: str, extra_classes: t.Optional[t.List[t.Type[App]]] = None) -> App:
+    """
+    Deserialize json string to App object
+
+    .. note::
+
+        You can pass extra_cls to support custom App class. A custom App class must be a subclass of App, and have a
+        different value of `build_system`. For example, a custom CMake app
+
+        >>> class CustomApp(CMakeApp):
+        >>>    build_system: Literal['custom_cmake'] = 'custom_cmake'
+
+        Then you can pass the CustomApp class to the `extra_cls` argument
+
+        >>> json_str = CustomApp('.', 'esp32').to_json()
+        >>> json_to_app(json_str, extra_classes=[CustomApp])
+
+    :param json_str: json string
+    :param extra_classes: extra App class
+    :return: App object
+    """
+    types = [App, CMakeApp, MakeApp]
+    if extra_classes:
+        types.extend(extra_classes)
+
+    custom_deserializer = create_model(
+        '_CustomDeserializer',
+        app=(t.Union[tuple(types)], Field(discriminator='build_system')),
+        __base__=AppDeserializer,
+    )
+
+    return custom_deserializer.from_json(json_str)

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/idf_build_apps/manifest/manifest.py

@@ -58,15 +58,17 @@ class FolderRule:
     ) -> None:
         self.folder = os.path.abspath(folder)
 
-        for group in [enable, disable, disable_test]:
-            if group:
-                for d in group:
-                    d['stmt'] = d['if']  # avoid keyword `if`
-                    del d['if']
-
-        self.enable = [IfClause(**clause) for clause in enable] if enable else []
-        self.disable = [IfClause(**clause) for clause in disable] if disable else []
-        self.disable_test = [IfClause(**clause) for clause in disable_test] if disable_test else []
+        def _clause_to_if_clause(clause: t.Dict[str, t.Any]) -> IfClause:
+            _kwargs = {'stmt': clause['if']}
+            if 'temporary' in clause:
+                _kwargs['temporary'] = clause['temporary']
+            if 'reason' in clause:
+                _kwargs['reason'] = clause['reason']
+            return IfClause(**_kwargs)
+
+        self.enable = [_clause_to_if_clause(clause) for clause in enable] if enable else []
+        self.disable = [_clause_to_if_clause(clause) for clause in disable] if disable else []
+        self.disable_test = [_clause_to_if_clause(clause) for clause in disable_test] if disable_test else []
         self.depends_components = depends_components or []
         self.depends_filepatterns = depends_filepatterns or []
 

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/idf_build_apps/session_args.py

@@ -1,10 +1,13 @@
 # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
+import logging
 import os
 import re
 import typing as t
 
+LOGGER = logging.getLogger(__name__)
+
 
 class SessionArgs:
     workdir: str = os.getcwd()
@@ -35,27 +38,32 @@ class SessionArgs:
         self.override_sdkconfig_file_path = override_sdkconfig_merged_file
 
     def _get_override_sdkconfig_files_items(self, override_sdkconfig_files: t.Tuple[str]) -> t.Dict:
-        dct = {}
+        d = {}
         for f in override_sdkconfig_files:
-            if not os.path.isabs(f):
-                f = os.path.join(self.workdir, f)
+            # use filepath if abs/rel already point to itself
             if not os.path.isfile(f):
-                continue
+                # find it in the workdir
+                LOGGER.debug('override sdkconfig file %s not found, checking under app_dir...', f)
+                f = os.path.join(self.workdir, f)
+                if not os.path.isfile(f):
+                    LOGGER.debug('override sdkconfig file %s not found, skipping...', f)
+                    continue
+
             with open(f) as fr:
                 for line in fr:
                     m = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$").match(line)
                     if not m:
                         continue
-                    dct[m.group(1)] = m.group(2)
-        return dct
+                    d[m.group(1)] = m.group(2)
+        return d
 
     def _get_override_sdkconfig_items(self, override_sdkconfig_items: t.Tuple[str]) -> t.Dict:
-        dct = {}
+        d = {}
         for line in override_sdkconfig_items:
             m = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$").match(line)
             if m:
-                dct[m.group(1)] = m.group(2)
-        return dct
+                d[m.group(1)] = m.group(2)
+        return d
 
     def _create_override_sdkconfig_merged_file(self, override_sdkconfig_merged_items) -> t.Optional[str]:
         if not override_sdkconfig_merged_items:

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/idf_build_apps/utils.py

@@ -1,7 +1,8 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import fnmatch
+import functools
 import glob
 import logging
 import os
@@ -134,7 +135,7 @@ def find_first_match(pattern: str, path: str) -> t.Optional[str]:
 def subprocess_run(
     cmd: t.List[str],
     log_terminal: bool = True,
-    log_fs: t.Optional[t.IO[str]] = None,
+    log_fs: t.Union[t.IO[str], str, None] = None,
     check: bool = False,
     additional_env_dict: t.Optional[t.Dict[str, str]] = None,
     **kwargs,
@@ -157,16 +158,26 @@ def subprocess_run(
         subprocess_env.update(additional_env_dict)
 
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=subprocess_env, **kwargs)
-    if p.stdout:
-        for line in p.stdout:
-            if isinstance(line, bytes):
-                line = line.decode('utf-8')
 
-            if log_terminal:
-                sys.stdout.write(line)
+    def _log_stdout(fs: t.Optional[t.IO[str]] = None):
+        if p.stdout:
+            for line in p.stdout:
+                if isinstance(line, bytes):
+                    line = line.decode('utf-8')
+
+                if log_terminal:
+                    sys.stdout.write(line)
+
+                if fs:
+                    fs.write(line)
 
-            if log_fs:
-                log_fs.write(line)
+    if p.stdout:
+        if log_fs:
+            if isinstance(log_fs, str):
+                with open(log_fs, 'a') as fa:
+                    _log_stdout(fa)
+            else:
+                _log_stdout(log_fs)
 
     returncode = p.wait()
     if check and returncode != 0:
@@ -305,25 +316,43 @@ def files_matches_patterns(
     return False
 
 
+@functools.total_ordering
 class BaseModel(_BaseModel):
     """
     BaseModel that is hashable
     """
 
+    __EQ_IGNORE_FIELDS__: t.List[str] = []
+
     def __lt__(self, other: t.Any) -> bool:
         if isinstance(other, self.__class__):
             for k in self.model_dump():
-                if getattr(self, k) != getattr(other, k):
-                    return getattr(self, k) < getattr(other, k)
-                else:
+                if k in self.__EQ_IGNORE_FIELDS__:
                     continue
 
+                self_attr = getattr(self, k, '') or ''
+                other_attr = getattr(other, k, '') or ''
+
+                if self_attr != other_attr:
+                    return self_attr < other_attr
+
+                continue
+
+            return False
+
         return NotImplemented
 
     def __eq__(self, other: t.Any) -> bool:
         if isinstance(other, self.__class__):
             # we only care the public attributes
-            return self.model_dump() == other.model_dump()
+            self_model_dump = self.model_dump()
+            other_model_dump = other.model_dump()
+
+            for _field in self.__EQ_IGNORE_FIELDS__:
+                self_model_dump.pop(_field, None)
+                other_model_dump.pop(_field, None)
+
+            return self_model_dump == other_model_dump
 
         return NotImplemented
 

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/pyproject.toml

@@ -60,7 +60,7 @@ idf-build-apps = "idf_build_apps:main.main"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "2.0.0rc0"
+version = "2.0.0rc1"
 tag_format = "v$version"
 version_files = [
     "idf_build_apps/__init__.py",

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/setup.py

@@ -27,7 +27,7 @@ entry_points = \
 {'console_scripts': ['idf-build-apps = idf_build_apps:main.main']}
 
 setup(name='idf-build-apps',
-      version='2.0.0rc0',
+      version='2.0.0rc1',
       description='Tools for building ESP-IDF related apps.',
       author=None,
       author_email='Fu Hanxi <fuhanxi@espressif.com>',

idf_build_apps-2.0.0rc1/tests/test_app.py

@@ -0,0 +1,95 @@
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+
+import pytest
+from pydantic import (
+    ValidationError,
+)
+
+from idf_build_apps import (
+    AppDeserializer,
+    CMakeApp,
+    MakeApp,
+)
+from idf_build_apps.main import (
+    json_to_app,
+)
+
+if sys.version_info < (3, 8):
+    from typing_extensions import (
+        Literal,
+    )
+else:
+    from typing import (
+        Literal,
+    )
+
+
+def test_serialization():
+    a = CMakeApp('foo', 'bar')
+    a_s = a.to_json()
+
+    b = CMakeApp.model_validate_json(a_s)
+    assert a == b
+
+
+def test_deserialization(tmp_path):
+    a = CMakeApp('foo', 'bar', size_json_filename='size.json')
+    b = MakeApp('foo', 'bar', build_log_filename='build.log')
+
+    assert a != b
+
+    with open(tmp_path / 'test.txt', 'w') as fw:
+        fw.write(a.to_json() + '\n')
+        fw.write(b.to_json() + '\n')
+
+    with open(tmp_path / 'test.txt') as fr:
+        a_s = AppDeserializer.from_json(fr.readline())
+        b_s = AppDeserializer.from_json(fr.readline())
+
+    assert a == a_s
+    assert b == b_s
+
+
+def test_app_sorting():
+    a = CMakeApp('foo', 'esp32')
+    b = MakeApp('foo', 'esp32')
+
+    c = CMakeApp('foo', 'esp32', size_json_filename='size.json')
+    d = CMakeApp('foo', 'esp32s2')
+    e = CMakeApp('foo', 'esp32s2', build_comment='build_comment')
+
+    with pytest.raises(TypeError, match="'<' not supported between instances of 'CMakeApp' and 'MakeApp'"):
+        assert a < b
+
+    assert a < c < d
+    assert d > c > a
+
+    # __EQ_IGNORE_FIELDS__
+    assert d == e
+    assert not d < e
+    assert not d > e
+
+
+def test_app_deserializer():
+    a = CMakeApp('foo', 'esp32')
+    b = MakeApp('foo', 'esp32')
+
+    class CustomApp(CMakeApp):
+        build_system: Literal['custom'] = 'custom'  # type: ignore
+
+    c = CustomApp('foo', 'esp32')
+
+    assert json_to_app(a.to_json()) == a
+    assert json_to_app(b.to_json()) == b
+
+    with pytest.raises(
+        ValidationError,
+        match="Input tag 'custom' found using 'build_system' does not match "
+        "any of the expected tags: 'unknown', 'cmake', 'make'",
+    ):
+        assert json_to_app(c.to_json()) == c
+
+    assert json_to_app(c.to_json(), extra_classes=[CustomApp]) == c

{idf_build_apps-2.0.0rc0 → idf_build_apps-2.0.0rc1}/tests/test_manifest.py

@@ -21,7 +21,7 @@ from idf_build_apps.utils import (
 )
 
 
-def test_manifest(tmpdir, recwarn):
+def test_manifest(tmpdir, recwarn, monkeypatch):
     yaml_file = tmpdir / 'test.yml'
     yaml_file.write_text(
         """
@@ -54,11 +54,38 @@ test2:
     assert manifest.enable_build_targets('test2') == ['linux']
     assert manifest.enable_test_targets('test2') == ['linux']
 
-    Manifest.CHECK_MANIFEST_RULES = True
+    monkeypatch.setattr(idf_build_apps.manifest.manifest.Manifest, 'CHECK_MANIFEST_RULES', True)
     with pytest.raises(InvalidManifest, match=msg_fmt.format(os.path.join(tmpdir, 'test1'), yaml_file)):
         Manifest.from_file(yaml_file)
 
 
+def test_manifest_with_anchor(tmpdir, monkeypatch):
+    yaml_file = tmpdir / 'test.yml'
+    yaml_file.write_text(
+        """
+.base: &base
+  depends_components:
+    - a
+
+foo: &foo
+  <<: *base
+  disable:
+    - if: IDF_TARGET == "esp32"
+
+bar:
+  <<: *foo
+""",
+        encoding='utf8',
+    )
+
+    monkeypatch.setattr(idf_build_apps.manifest.manifest.FolderRule, 'DEFAULT_BUILD_TARGETS', ['esp32'])
+
+    with pytest.warns(UserWarning, match='Folder ".+" does not exist. Please check your manifest file'):
+        manifest = Manifest.from_file(yaml_file)
+
+    assert manifest.enable_build_targets('bar') == []
+
+
 class TestIfParser:
     def test_idf_version(self, monkeypatch):
         monkeypatch.setattr(idf_build_apps.manifest.if_parser, 'IDF_VERSION', Version('5.9.0'))

idf_build_apps-2.0.0rc0/tests/test_app.py

@@ -1,34 +0,0 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-
-from idf_build_apps import (
-    AppDeserializer,
-    CMakeApp,
-    MakeApp,
-)
-
-
-def test_serialization():
-    a = CMakeApp('foo', 'bar')
-    a_s = a.to_json()
-
-    b = CMakeApp.model_validate_json(a_s)
-    assert a == b
-
-
-def test_deserialization(tmp_path):
-    a = CMakeApp('foo', 'bar', size_json_filename='size.json')
-    b = MakeApp('foo', 'bar', build_log_filename='build.log')
-
-    assert a != b
-
-    with open(tmp_path / 'test.txt', 'w') as fw:
-        fw.write(a.to_json() + '\n')
-        fw.write(b.to_json() + '\n')
-
-    with open(tmp_path / 'test.txt') as fr:
-        a_s = AppDeserializer.from_json(fr.readline())
-        b_s = AppDeserializer.from_json(fr.readline())
-
-    assert a == a_s
-    assert b == b_s