aiaccel 2025.4__tar.gz

aiaccel-2025.4/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

aiaccel-2025.4/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

aiaccel-2025.4/.github/workflows/ci.yaml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  push:
+    branches: ["main", "develop/*"]
+  pull_request:
+
+jobs:
+  ci:
+    name: CI
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ['ubuntu-22.04']
+        python-version: ['3.10']
+        test_target: ['config', 'hpo', 'torch']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: pyproject.toml
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -e .[dev,github-actions]
+      - name: Run pytest
+        run: pytest -v --cov=aiaccel/${{ strategy.target }} --cov-branch --cov-report=term-missing tests/${{ strategy.target }}
+

aiaccel-2025.4/.github/workflows/gh-pages.yaml

@@ -0,0 +1,50 @@
+name: GitHub Pages
+
+on:
+  push:
+    branches: ["main", "develop/*"]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: GitHub Pages (Build)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[dev,github-actions]
+      - name: Sphinx build
+        run: |
+          sphinx-apidoc --maxdepth 2 -f -o ./docs/source/api_reference/ ./aiaccel/
+          cd docs
+          make html
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/build/html/
+
+  deploy:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    name: GitHub Pages (Deploy)
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

aiaccel-2025.4/.github/workflows/lint.yaml

@@ -0,0 +1,36 @@
+name: Lint
+
+on:
+  push:
+    branches: ["main", "develop/*"]
+  pull_request:
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ['ubuntu-22.04']
+        python-version: ['3.10']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: pyproject.toml
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install .[dev,github-actions]
+      - name: Perform ruff
+        run: |
+          ruff check
+          ruff format --check
+      - name: Perform mypy
+        run: |
+          mypy --config-file mypy.ini .
+      - name: Perform docstrfmt
+        run: |
+          docstrfmt --check docs/source/

aiaccel-2025.4/.gitignore

@@ -0,0 +1,134 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+# lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# application workspace
+/work/
+work_aiaccel/
+
+# vscode
+.vscode
+*.code-workspace
+
+# document
+docs/build/html
+docs/source/api_reference/
+!docs/source/api_reference/index.rst
+!docs/source/api_reference/torch.rst
+!docs/source/api_reference/hpo.rst
+
+# IntelliJ
+.idea
+
+# Docker
+.devcontainer
+Dockerfile
+
+# ruff
+.ruff_cache
+
+# example
+examples/hpo/samplers/coco/exdata
+examples/hpo/samplers/coco/*/optuna_csv
+examples/hpo/samplers/coco/*/step_csv

aiaccel-2025.4/.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.5
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.9.0
+    hooks:
+      - id: mypy
+        language: system
+        args: [--config-file, mypy.ini]
+  - repo: local
+    hooks:
+      - id: pytest
+        name: pytest
+        entry: pytest -s -v -x --cov=aiaccel/ tests/
+        stages: [push]
+        language: system
+        pass_filenames: false
+        always_run: true
+  - repo: https://github.com/LilSpazJoekp/docstrfmt
+    rev: v1.10.0
+    hooks:
+      - id: docstrfmt
+        args: [--check, docs/source/]
+        types_or: [rst]
+        exclude: docs/source/api_reference/generated

aiaccel-2025.4/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 National Institute of Advanced Industrial Science and Technology (AIST), Japan, All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

aiaccel-2025.4/MANIFEST.in

@@ -0,0 +1,2 @@
+include aiaccel/hpo/apps/config/*.yaml
+include aiaccel/torch/apps/config/*.yaml

aiaccel-2025.4/PKG-INFO

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: aiaccel
+Version: 2025.4
+Summary: AIST Toolkit for Accelerating Machine Learning Research
+Author-email: AIST <onishi-masaki@aist.go.jp>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Classifier: License :: OSI Approved :: MIT License
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: optuna>=3.4.0
+Requires-Dist: omegaconf
+Requires-Dist: hydra-core
+Requires-Dist: colorama
+Requires-Dist: lightning>=2.2.1
+Requires-Dist: torch>=2.2.0
+Requires-Dist: h5py
+Requires-Dist: dask-jobqueue
+Requires-Dist: rich
+Requires-Dist: mypy ; extra == "dev"
+Requires-Dist: myst-parser ; extra == "dev"
+Requires-Dist: pre-commit ; extra == "dev"
+Requires-Dist: pytest ; extra == "dev"
+Requires-Dist: pytest-cov ; extra == "dev"
+Requires-Dist: pytest-mock ; extra == "dev"
+Requires-Dist: pytest-subprocess ; extra == "dev"
+Requires-Dist: ruff ; extra == "dev"
+Requires-Dist: sphinx ; extra == "dev"
+Requires-Dist: sphinxcontrib-jquery ; extra == "dev"
+Requires-Dist: sphinx-intl ; extra == "dev"
+Requires-Dist: sphinx-fontawesome ; extra == "dev"
+Requires-Dist: sphinx-rtd-theme ; extra == "dev"
+Requires-Dist: sphinx-copybutton ; extra == "dev"
+Requires-Dist: pydata-sphinx-theme ; extra == "dev"
+Requires-Dist: sphinx_design ; extra == "dev"
+Requires-Dist: types-colorama ; extra == "dev"
+Requires-Dist: types-PyYAML ; extra == "dev"
+Requires-Dist: undecorated ; extra == "dev"
+Requires-Dist: pandas ; extra == "dev"
+Requires-Dist: pandas-stubs ; extra == "dev"
+Requires-Dist: matplotlib ; extra == "dev"
+Requires-Dist: docstrfmt ; extra == "dev"
+Requires-Dist: pytest-github-actions-annotate-failures ; extra == "github-actions"
+Project-URL: documentation, https://aistairc.github.io/aiaccel/
+Project-URL: repository, https://github.com/aistairc/aiaccel
+Provides-Extra: dev
+Provides-Extra: github-actions
+
+<div align="center"><img src="https://raw.githubusercontent.com/aistairc/aiaccel/master/docs/image/logo_aiaccel.png" width="600"/></div>
+<div align="center">
+    <img src="https://img.shields.io/github/license/aistairc/aiaccel.svg" />
+    <img src="https://img.shields.io/badge/Python-3.10-blue" />
+    <a href="https://aistairc.github.io/aiaccel">
+        <img src="https://github.com/aistairc/aiaccel/actions/workflows/gh-pages.yaml/badge.svg" />
+    </a>
+    <img src="https://github.com/aistairc/aiaccel/actions/workflows/ci.yaml/badge.svg" />
+</div>
+
+# AIST Toolkit for Accelerating Machine Learning Research
+
+* **Research-Oriented**: designed to accelerate your research cycles written in Python
+* **HPC Optimized**: intended to use in HPC clusters, including [AI Bridging Cloud Infrastructure (ABCI)](https://abci.ai/)
+* **Highly Modular**: designed to let you pick up any part of aiaccel for your research project
+
+# Key Features
+* [PyTorch/Lightning Toolkit](https://aistairc.github.io/aiaccel/api_reference/torch.html): training toolkit for HPC clusters.
+* [Hyperparameter Optimization (HPO)](https://aistairc.github.io/aiaccel/api_reference/hpo.html): ready-to-use HPO algorithms/tools.
+* [OmegaConf Utilities](https://aistairc.github.io/aiaccel/api_reference/config.html): OmegaConf-based config utilities.
+
+
+# Installation
+```bash
+pip install aiaccel
+```
+       
+# Acknowledgement
+* Part of this software was developed in a project commissioned by the New Energy and Industrial Technology Development Organization (NEDO).
+* Part of this software was developed by using ABCI 3.0 provided by AIST and AIST Solutions.
+* Part of this software was developed by using the TSUBAME4.0 supercomputer at Institute of Science Tokyo.
+

aiaccel-2025.4/README.md

@@ -0,0 +1,31 @@
+<div align="center"><img src="https://raw.githubusercontent.com/aistairc/aiaccel/master/docs/image/logo_aiaccel.png" width="600"/></div>
+<div align="center">
+    <img src="https://img.shields.io/github/license/aistairc/aiaccel.svg" />
+    <img src="https://img.shields.io/badge/Python-3.10-blue" />
+    <a href="https://aistairc.github.io/aiaccel">
+        <img src="https://github.com/aistairc/aiaccel/actions/workflows/gh-pages.yaml/badge.svg" />
+    </a>
+    <img src="https://github.com/aistairc/aiaccel/actions/workflows/ci.yaml/badge.svg" />
+</div>
+
+# AIST Toolkit for Accelerating Machine Learning Research
+
+* **Research-Oriented**: designed to accelerate your research cycles written in Python
+* **HPC Optimized**: intended to use in HPC clusters, including [AI Bridging Cloud Infrastructure (ABCI)](https://abci.ai/)
+* **Highly Modular**: designed to let you pick up any part of aiaccel for your research project
+
+# Key Features
+* [PyTorch/Lightning Toolkit](https://aistairc.github.io/aiaccel/api_reference/torch.html): training toolkit for HPC clusters.
+* [Hyperparameter Optimization (HPO)](https://aistairc.github.io/aiaccel/api_reference/hpo.html): ready-to-use HPO algorithms/tools.
+* [OmegaConf Utilities](https://aistairc.github.io/aiaccel/api_reference/config.html): OmegaConf-based config utilities.
+
+
+# Installation
+```bash
+pip install aiaccel
+```
+       
+# Acknowledgement
+* Part of this software was developed in a project commissioned by the New Energy and Industrial Technology Development Organization (NEDO).
+* Part of this software was developed by using ABCI 3.0 provided by AIST and AIST Solutions.
+* Part of this software was developed by using the TSUBAME4.0 supercomputer at Institute of Science Tokyo.

aiaccel-2025.4/aiaccel/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+__version__ = version(__package__)

aiaccel-2025.4/aiaccel/config/__init__.py

@@ -0,0 +1,19 @@
+from aiaccel.config.config import (
+    load_config,
+    overwrite_omegaconf_dumper,
+    pathlib2str_config,
+    print_config,
+    resolve_inherit,
+)
+from aiaccel.config.git import PackageGitStatus, collect_git_status_from_config, print_git_status
+
+__all__ = [
+    "load_config",
+    "overwrite_omegaconf_dumper",
+    "pathlib2str_config",
+    "print_config",
+    "resolve_inherit",
+    "PackageGitStatus",
+    "collect_git_status_from_config",
+    "print_git_status",
+]

aiaccel-2025.4/aiaccel/config/config.py

@@ -0,0 +1,181 @@
+from typing import Any
+
+import copy
+from copy import deepcopy
+from pathlib import Path
+import re
+
+from colorama import Fore
+from omegaconf import DictConfig, ListConfig
+from omegaconf import OmegaConf as oc  # noqa:N813
+from omegaconf._utils import OmegaConfDumper
+
+from yaml import Node
+from yaml.resolver import BaseResolver
+
+
+def overwrite_omegaconf_dumper(mode: str = "|") -> None:
+    """
+    Overwrites the default string representation in OmegaConf's YAML dumper.
+
+    This function modifies the `OmegaConfDumper` to represent multi-line strings
+    using the specified style (`mode`). By default, it uses the `|` block style
+    for multi-line strings. Single-line strings remain unchanged.
+
+    Args:
+        mode (str, optional): The YAML style character for multi-line strings.
+                              Defaults to "|".
+    """
+
+    def str_representer(dumper: OmegaConfDumper, data: str) -> Node:
+        return dumper.represent_scalar(
+            BaseResolver.DEFAULT_SCALAR_TAG, data, style=mode if len(data.splitlines()) > 1 else None
+        )
+
+    OmegaConfDumper.add_representer(str, str_representer)
+    OmegaConfDumper.str_representer_added = True
+
+
+def resolve_inherit(config: DictConfig | ListConfig) -> DictConfig | ListConfig:
+    """Resolve _inherit_ in config
+
+    Merge the dict in ``_inherit_`` into a dict of the same hierarchy.
+    ``_inherit_`` is specified by omegaconf interpolation
+
+    Args:
+        config (DictConfig | ListConfig): The configuration loaded by load_config
+
+    Returns:
+        DictConfig | ListConfig: The configuration without ``_inherit_``
+    """
+    while isinstance(config, DictConfig) and "_inherit_" in config:
+        # resolve _inherit_
+        inherit_configs = config["_inherit_"]
+        if not isinstance(inherit_configs, ListConfig):
+            inherit_configs = [inherit_configs]
+
+        config.pop("_inherit_")
+
+        for inherit_config in inherit_configs:
+            if isinstance(inherit_config, DictConfig):
+                config = oc.merge(inherit_config, config)
+
+    # load child DictConfig to resolve child _inherit_
+    if isinstance(config, DictConfig):
+        dst_config_dict = copy.deepcopy(config)
+        for key in config:
+            dst_config_dict[key] = resolve_inherit(config[key])
+        config = dst_config_dict
+    elif isinstance(config, ListConfig):
+        dst_config_list = copy.deepcopy(config)
+        for key in range(len(config)):
+            dst_config_list[key] = resolve_inherit(config[key])
+        config = dst_config_list
+
+    return config
+
+
+def load_config(
+    config_filename: str | Path,
+    parent_config: dict[str, Any] | DictConfig | ListConfig | None = None,
+) -> DictConfig | ListConfig:
+    """Load YAML configuration
+
+    When the user specifies ``_base_``, the specified YAML file is loaded as the base,
+    and the original configuration is merged with the base config.
+    If the configuration specified in ``_base_`` also contains ``_base_``, the process is handled recursively.
+
+    Additionally, if `bootstrap_config` is provided, it is merged with the final
+    configuration to ensure any default values or overrides are applied.
+
+    Args:
+        config (Path): Path to the configuration
+        parent_config (dict[str, Any] | DictConfig | ListConfig | None):
+            A configuration that is merged to the loaded configuration.
+            This is intended to define default config paths (e.g., working_directory) dynamically.
+
+    Returns:
+        merge_user_config (DictConfig): The merged configuration of the base config and the original config
+        user_config(DictConfig | ListConfig) : The configuration without ``_base_``
+
+    """
+
+    if not isinstance(config_filename, Path):
+        config_filename = Path(config_filename)
+
+    if not config_filename.is_absolute():
+        config_filename = Path.cwd() / config_filename
+
+    if parent_config is None:
+        parent_config = {}
+
+    config = oc.merge(oc.load(config_filename), parent_config)
+
+    if isinstance(config, DictConfig) and "_base_" in config:
+        # process _base_
+        base_paths = config["_base_"]
+        if not isinstance(base_paths, ListConfig):
+            base_paths = [base_paths]
+
+        config.pop("_base_")
+        for base_path in map(Path, base_paths):
+            if not base_path.is_absolute():
+                base_path = config_filename.parent / base_path
+
+            config = load_config(base_path, config)
+
+    return config
+
+
+def print_config(config: ListConfig | DictConfig, line_length: int = 80) -> None:
+    """
+    Print the given configuration with syntax highlighting.
+
+    This function converts `pathlib.Path` objects to strings before printing,
+    ensuring that the output YAML format remains valid. It also highlights
+    configuration keys in yellow for better readability.
+
+    Args:
+        config (ListConfig | DictConfig): The configuration to print.
+        line_length (int, optional): The width of the separator line (default: 80).
+
+    """
+
+    config = pathlib2str_config(config)  # https://github.com/omry/omegaconf/issues/82
+
+    print("=" * line_length)
+    for line in oc.to_yaml(config).splitlines():
+        print(re.sub(r"(\s*)(\w+):", rf"\1{Fore.YELLOW}\2{Fore.RESET}:", line, count=1))
+    print("=" * line_length)
+
+
+def pathlib2str_config(config: ListConfig | DictConfig) -> ListConfig | DictConfig:
+    """
+    Convert `pathlib.Path` objects in the configuration to strings.
+
+    This function recursively traverses the configuration and replaces all `pathlib.Path`
+    objects with their string representations. This is useful for saving the configuration
+    in a YAML file, as YAML does not support `Path` objects.
+
+    Args:
+        config (ListConfig | DictConfig): The configuration to convert.
+
+    Returns:
+        ListConfig | DictConfig: The modified configuration with `Path` objects replaced by strings.
+
+    """
+
+    def _inner_fn(config: ListConfig | DictConfig) -> ListConfig | DictConfig:
+        if isinstance(config, ListConfig):
+            for ii in range(len(config)):
+                config[ii] = _inner_fn(config[ii])
+        elif isinstance(config, DictConfig):
+            for k, v in config.items():
+                if isinstance(v, ListConfig | DictConfig):
+                    config[k] = _inner_fn(v)
+                elif isinstance(v, Path):
+                    config[k] = str(v)
+
+        return config
+
+    return _inner_fn(deepcopy(config))