pytorch_optimizer 3.11.0__tar.gz → 4.0.0__tar.gz

{pytorch_optimizer-3.11.0 → pytorch_optimizer-4.0.0}/.github/ISSUE_TEMPLATE/bug_report.md

@@ -14,8 +14,8 @@ A clear and concise description of what the bug is.
 ## To Reproduce
 
 * OS : (e.g. Linux, Windows, MacOS)
-* PyTorch version : (e.g. 2.0.1, 1.13, >=1.8, <1.10)
-* Python version :  (e.g. 3.8, 3.11)
+* PyTorch version : (e.g. 2.1.0, 2.13.0)
+* Python version :  (e.g. 3.10, 3.12)
 * pytorch-optimizer version : (e.g. 3.3.0)
 * reproducible codes : please share your reproducible codes, scripts, or links. If sharing the code is complicated, you can manually write minimal code to reproduce bugs!
 

{pytorch_optimizer-3.11.0 → pytorch_optimizer-4.0.0}/.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
+  workflow_dispatch:
   workflow_call:
     inputs:
       ref:
@@ -64,9 +65,8 @@ jobs:
       - name: Install dependencies
         run: uv sync --frozen --no-editable
 
-      - uses: jakebailey/pyright-action@8ec14b5cfe41f26e5f41686a31eb6012758217ef # v3.0.2
-        with:
-          version: 1.1.414
+      - name: Run type check
+        run: uv run --frozen ty check
 
   docs:
     name: Documentation
@@ -89,7 +89,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15']
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15']
 
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

{pytorch_optimizer-3.11.0 → pytorch_optimizer-4.0.0}/.github/workflows/pr_automerge.yml

@@ -5,7 +5,9 @@ on:
     types:
       - opened
       - synchronize
-  check_suite:
+      - labeled
+  workflow_run:
+    workflows: [CI]
     types:
       - completed
 
@@ -19,13 +21,15 @@ jobs:
 
     steps:
       - id: automerge
-        if: ${{ contains(github.event.pull_request.labels.*.name, 'automerge') }}
+        if: >-
+          contains(github.event.pull_request.labels.*.name, 'automerge') ||
+          (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
         name: automerge
         uses: pascalgn/automerge-action@7961b8b5eec56cc088c140b56d864285eabd3f67 # v0.16.4
         env:
           MERGE_LABELS: "automerge"
           MERGE_METHOD_LABEL_REQUIRED: "false"
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          GITHUB_TOKEN: ${{ github.token }}
           MERGE_METHOD: "squash"
           MERGE_REQUIRED_APPROVALS: "0"
           MERGE_DELETE_BRANCH: "true"

{pytorch_optimizer-3.11.0 → pytorch_optimizer-4.0.0}/.github/workflows/publish.yml

@@ -42,21 +42,23 @@ jobs:
       skip_coverage_upload: true
 
   prepare:
-    name: Prepare Version
+    name: Prepare Release
     needs: validate
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      pull-requests: write
     outputs:
       tag: ${{ steps.version.outputs.tag }}
       ref: ${{ steps.commit.outputs.ref }}
+      notes: ${{ steps.notes.outputs.body }}
     steps:
       - name: Checkout release ref
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.sha }}
           fetch-depth: 0
-          token: ${{ secrets.GH_TOKEN }}
+          token: ${{ github.token }}
 
       - name: Setup uv
         uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
@@ -78,6 +80,10 @@ jobs:
           if [[ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ]]; then
             version="$(uv version --bump "$BUMP" --no-sync --short)"
             tag="v$version"
+            if git rev-parse --verify --quiet "refs/tags/$tag"; then
+              echo "Tag $tag already exists" >&2
+              exit 1
+            fi
           else
             tag="$GITHUB_REF_NAME"
             if [[ "$tag" != "v$current" ]]; then
@@ -87,23 +93,81 @@ jobs:
           fi
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
 
-      - name: Commit and push version bump
-        id: commit
+      - name: Generate release notes and update changelogs
+        id: notes
         env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
           RELEASE_TAG: ${{ steps.version.outputs.tag }}
         run: |
-          if [[ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ]]; then
-            if git rev-parse --verify --quiet "refs/tags/$RELEASE_TAG"; then
-              echo "Tag $RELEASE_TAG already exists" >&2
-              exit 1
+          gh api --method POST "repos/$GH_REPO/releases/generate-notes" \
+            -f tag_name="$RELEASE_TAG" -f target_commitish="$GITHUB_SHA" \
+            --jq .body > "$RUNNER_TEMP/release-notes.md"
+          uv run --no-project python scripts/update_changelog.py "$RELEASE_TAG" "$RUNNER_TEMP/release-notes.md"
+
+          delimiter="release_notes_${RANDOM}_${RANDOM}_${RANDOM}"
+          {
+            echo "body<<$delimiter"
+            cat "$RUNNER_TEMP/release-notes.md"
+            echo "$delimiter"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Create release preparation pull request
+        id: release_pr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          branch: release/prepare-${{ steps.version.outputs.tag }}
+          commit-message: 'build: prepare release ${{ steps.version.outputs.tag }}'
+          title: 'build: prepare release ${{ steps.version.outputs.tag }}'
+          body: >-
+            Prepare the package version, lockfile, and generated changelogs for ${{ steps.version.outputs.tag }}.
+            This pull request must merge before package publishing and GitHub release creation.
+          base: main
+          add-paths: |
+            pyproject.toml
+            uv.lock
+            CHANGELOG.md
+            docs/changelogs/${{ steps.version.outputs.tag }}.md
+            docs/changelogs/index.md
+          delete-branch: true
+          token: ${{ github.token }}
+
+      - name: Merge release preparation pull request and resolve release commit
+        id: commit
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ steps.release_pr.outputs.pull-request-number }}
+          PR_HEAD_SHA: ${{ steps.release_pr.outputs.pull-request-head-sha }}
+        run: |
+          if [[ -z "$PR_NUMBER" || -z "$PR_HEAD_SHA" ]]; then
+            echo 'No release preparation pull request was created' >&2
+            exit 1
+          fi
+          for attempt in {1..10}; do
+            mergeable="$(gh pr view "$PR_NUMBER" --json mergeable --jq .mergeable)"
+            if [[ "$mergeable" != 'UNKNOWN' ]]; then
+              break
             fi
-            git config user.name 'github-actions[bot]'
-            git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
-            git add pyproject.toml uv.lock
-            git commit -m "build: prepare version ${RELEASE_TAG#v} [skip ci]"
-            git push origin HEAD:main
+            sleep 3
+          done
+          if [[ "$mergeable" == 'UNKNOWN' ]]; then
+            echo 'GitHub has not finished calculating PR mergeability' >&2
+            exit 1
           fi
-          echo "ref=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          gh pr merge "$PR_NUMBER" --squash --delete-branch --match-head-commit "$PR_HEAD_SHA"
+          merged_ref="$(gh pr view "$PR_NUMBER" --json state,mergeCommit \
+            --jq 'if .state == "MERGED" then .mergeCommit.oid else empty end')"
+          if [[ -z "$merged_ref" ]]; then
+            echo 'The release preparation pull request must be merged before publishing' >&2
+            exit 1
+          fi
+          if [[ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ]]; then
+            ref="$merged_ref"
+          else
+            ref="$GITHUB_SHA"
+          fi
+          echo "ref=$ref" >> "$GITHUB_OUTPUT"
 
   deploy:
     name: Publish Package
@@ -111,6 +175,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      id-token: write
     steps:
       - name: Checkout release commit
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -145,9 +210,7 @@ jobs:
           fi
 
       - name: Publish package to PyPI
-        env:
-          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        run: uv publish --check-url https://pypi.org/simple
+        run: uv publish --trusted-publishing always --check-url https://pypi.org/simple
 
   release:
     name: Create Release
@@ -156,59 +219,13 @@ jobs:
     permissions:
       contents: write
     steps:
-      - name: Generate release notes and create release
+      - name: Create release with the prepared notes
         uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
         with:
-          generate_release_notes: true
+          body: ${{ needs.prepare.outputs.notes }}
           draft: false
           prerelease: false
           name: pytorch-optimizer ${{ needs.prepare.outputs.tag }}
           tag_name: ${{ needs.prepare.outputs.tag }}
           target_commitish: ${{ needs.prepare.outputs.ref }}
-          token: ${{ secrets.GH_TOKEN }}
-
-  changelog:
-    name: Update Changelogs
-    runs-on: ubuntu-latest
-    needs: [prepare, release]
-    permissions:
-      contents: write
-      pull-requests: write
-    steps:
-      - name: Checkout main
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          ref: main
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: '3.12'
-
-      - name: Update changelogs from the published release
-        env:
-          GH_TOKEN: ${{ github.token }}
-          GH_REPO: ${{ github.repository }}
-          RELEASE_TAG: ${{ needs.prepare.outputs.tag }}
-        run: |
-          gh release view "$RELEASE_TAG" --json body --jq .body > "$RUNNER_TEMP/release-notes.md"
-          python scripts/update_changelog.py "$RELEASE_TAG" "$RUNNER_TEMP/release-notes.md"
-
-      - name: Create changelog pull request
-        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
-        with:
-          branch: docs/changelog-update-${{ needs.prepare.outputs.tag }}
-          commit-message: 'docs: update changelogs for ${{ needs.prepare.outputs.tag }}'
-          title: 'docs: update changelogs for ${{ needs.prepare.outputs.tag }}'
-          body: >-
-            Sync the generated release notes from
-            https://github.com/${{ github.repository }}/releases/tag/${{ needs.prepare.outputs.tag }}
-            into CHANGELOG.md and the documentation.
-          base: main
-          add-paths: |
-            CHANGELOG.md
-            docs/changelogs/${{ needs.prepare.outputs.tag }}.md
-            docs/changelogs/index.md
-          delete-branch: true
-          labels: automerge
-          token: ${{ secrets.GH_TOKEN }}
+          token: ${{ github.token }}

pytorch_optimizer-4.0.0/AGENTS.md

@@ -0,0 +1,110 @@
+# Agent Instructions
+
+## Development workflow
+
+- Use `uv` for dependency resolution and execution. Prefer `uv run just <recipe>` to use the versions in `uv.lock`.
+- The repository `justfile` is cross-platform and works on Windows, macOS, and Linux/WSL.
+- Run the relevant checks after changes:
+  - `uv run just format`
+  - `uv run just check`
+  - `uv run just test`
+- Cover new and changed implementation code at 100%. Check coverage with `uv run coverage report -m`.
+- Use `uv run coverage report -m` to find uncovered lines, then add focused cases in the appropriate test module.
+- Run focused optimizer tests with `uv run pytest tests/optimizers/test_muon.py -sv -vv`.
+- Select a training recipe with `uv run pytest tests/test_optimizers.py::TestOptimizerTraining -k adammini -sv -vv`.
+- Use `uv run just docs` to serve the Zensical documentation. Build it with strict validation using
+  `uv run just docs-build`.
+- Run `uv run just update-docs` after changing public exports to regenerate the API reference.
+
+## Architecture
+
+- `pytorch_optimizer/base/` contains `BaseOptimizer`, scheduler bases, shared types, and exceptions.
+- `pytorch_optimizer/optimizer/` contains optimizers, wrappers, and shared gradient and update utilities.
+- Put schedulers in `pytorch_optimizer/lr_scheduler/` and loss functions in `pytorch_optimizer/loss/`.
+- Export public components from `pytorch_optimizer/__init__.py`. Use `load_optimizer()` to load a class by name and
+  `create_optimizer()` to configure an optimizer with common options and wrappers.
+- Optional integrations include `bitsandbytes`, `q-galore-torch`, and `torchao`.
+- `tests/test_optimizers.py` runs shared training and interface tests using `OPTIMIZER_RECIPES` from
+  `tests/recipes.py`. Focused optimizer and algorithm-helper cases live in `tests/optimizers/test_<module>.py`.
+  Parameter validation, variants, wrappers, losses, and schedulers have separate shared test modules.
+- `tests/fixtures.py` owns the shared training model and parameter builders. `tests/utils.py` supplies optimizer
+  construction and training helpers; `tests/conftest.py` supplies pytest fixtures, including training data.
+- `tests/optimizer_cases.py` derives constructor options and model-based APIs from signatures, and sparse/complex
+  support from cached CPU step probes. Explicit exclusions cover specialized protocols and numerical limitations.
+- `zensical.toml` owns documentation navigation and theme settings. Keep the home page separate from `README.md`.
+
+## Code style
+
+- Follow nearby repository code and tests before introducing new patterns.
+- Add comments only when code is difficult to understand; avoid redundant, meaningless, or excessive comments.
+- Use blank lines to separate logical steps and improve readability.
+- Do not add introductory comments or docstrings to scripts.
+- Use a maximum line length of 119 characters and single-quoted strings.
+- Use the repository's existing typing style. Do not add `from __future__ import annotations` or `TYPE_CHECKING`
+  imports unless the surrounding code requires them.
+- Optimizers inherit from `BaseOptimizer`, implement `init_group()` and `step()`, and reuse its validation and update
+  helpers where applicable.
+- All optimizer implementations must follow the algorithms and update rules in their original papers.
+- Follow Google-style docstrings. Reuse helpers such as `apply_weight_decay()`, `apply_cautious()`, `debias()`,
+  `validate_learning_rate()`, `validate_betas()`, and `validate_range()` before adding update or validation logic.
+- Keep implementations focused; remove redundant compatibility layers and abstractions.
+
+## Testing
+
+- For optimizer changes, add or update minimal recipes in `OPTIMIZER_RECIPES`, including affected variants. Each
+  recipe contains an optimizer name, options, and an iteration count. Use the shared runner for model-based
+  optimizers when their update protocol fits; avoid separate training or smoke tests that a recipe can replace.
+- Add focused cases for behavior the training recipes do not verify, such as numerical updates, checkpoint
+  restoration, validation, wrappers, and edge cases. Follow nearby patterns and preserve 100% implementation coverage.
+- Give focused cases an observable assertion. Compare numerical results with `torch.testing.assert_close()` and
+  check expected exceptions with `pytest.raises()`. Use stable inputs and a trusted reference for iterative algorithms.
+- Reuse `TrainingModel`, `build_model()`, and parameter builders from `tests/fixtures.py`. Use a parameter when a test
+  only calls optimizer methods, and create additional shapes or model features only when the behavior requires them.
+- Use `tests.utils.build_optimizer()` for ordinary optimizer setup. Use `load_optimizer()` for constructor
+  validation or APIs that accept an optimizer class, and `create_optimizer()` when testing that public factory.
+- Use pytest classes to group related behavior where useful. Parametrize only applicable combinations and run
+  shared interface checks once per optimizer rather than once per recipe.
+- Reuse capability detection from `tests/optimizer_cases.py` instead of maintaining optimizer name lists. Keep
+  protocol-specific exclusions explicit; probes must preserve random state and propagate unexpected failures.
+- Keep variant, scheduler, and loss case data beside their tests. Copy recipe options before attaching callbacks or
+  parameter groups, and avoid retaining model instances or optimizer state in shared case data.
+- Keep tensors and training iteration counts small while retaining meaningful assertions. Remove redundant cases
+  without losing behavior checks or coverage, and avoid session-wide performance overrides.
+
+## Adding an optimizer
+
+- Add the implementation under `pytorch_optimizer/optimizer/`.
+- Export it from the optimizer package and register it in `OPTIMIZER_LIST` so it is available through `OPTIMIZERS`
+  and `load_optimizer()`.
+- Add a training recipe and focused cases following the testing guidance above.
+- Update the relevant documentation and algorithm table in `README.md`.
+- Write clear PR titles and descriptions; the release workflow generates notes from merged PRs and syncs them to
+  `CHANGELOG.md`, `docs/changelogs/<tag>.md`, and the changelog index through an automated PR.
+- Register new loss functions and schedulers in their package exports and add them to the corresponding tests and
+  README tables.
+- Leave the root `CHANGELOG.md` to the release automation.
+
+## Commits and pull requests
+
+Start commit subjects with one of these prefixes:
+
+| Prefix | Use for |
+| --- | --- |
+| `feat:` | Add a feature or public API. |
+| `fix:` | Correct a bug or unintended behavior. |
+| `perf:` | Improve execution speed or resource use. |
+| `style:` | Change code formatting without changing behavior. |
+| `refactor:` | Restructure code without changing public behavior. |
+| `docs:` | Add, revise, or reorganize documentation. |
+| `chore:` | Maintain development tools, automation, or repository housekeeping. |
+| `build:` | Change build, packaging, or dependency configuration. |
+| `update:` | Refresh project content or metadata when no more specific prefix applies. |
+
+Keep the text after the prefix imperative, concise, and specific to the committed change.
+Do not use commit prefixes outside this list.
+
+- Write concise, specific PR titles that describe the actual change. Use a title tag only when it adds useful context
+  and accurately matches the work; do not derive it mechanically from the commit prefix.
+- Feature PR titles follow the existing format, such as `[Feature] Implement Magma optimizer`.
+- Keep commits focused and verify the branch and working tree before pushing. Use `--force-with-lease` when you need
+  to rewrite a pushed branch.

{pytorch_optimizer-3.11.0 → pytorch_optimizer-4.0.0}/CHANGELOG.md

@@ -1,3 +1,103 @@
+# v4.0.0
+
+## What's Changed
+* docs: update changelogs for v3.11.0 by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/588
+* [Fix] Trigger automerge when label is added by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/590
+* [Fix] un-swap false positives and false negatives in SoftF1Loss by @plox-sumit in https://github.com/kozistr/pytorch_optimizer/pull/593
+* [Feature] Implement NorMuon optimizer by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/589
+* [Fix] Support strided inputs in Lovasz hinge loss by @AHMETHAKANBEZIR1 in https://github.com/kozistr/pytorch_optimizer/pull/597
+* [Fix] Prevent Tversky overflow on large half-precision masks by @AHMETHAKANBEZIR1 in https://github.com/kozistr/pytorch_optimizer/pull/599
+* [Feature] Support reduction modes in `FocalCosineLoss` by @AHMETHAKANBEZIR1 in https://github.com/kozistr/pytorch_optimizer/pull/595
+* Track the GRAMS first moment and debias its denominator by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/537
+* [Fix] put the Schedule-Free RAdam bias correction under the square root by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/545
+* [Fix] Correct DiffGrad, AdaFactor, SPlus, and SOAP updates by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/601
+* [Feature] Implement SaRA optimizer by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/602
+* [Build] Require Python 3.10 / PyTorch 2.1 and use lerp for moving averages by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/603
+* [Feature] Compile optimizer steps, batch AdEMAMix, and adopt Trusted Publishing by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/604
+* [Refactor] Modernize type hints and centralize package exports by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/605
+* Standardize and correct API docstrings by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/606
+* Fix optimizer updates, checkpoint precision, and memory overhead by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/607
+* Consolidate test fixtures and streamline optimizer tests by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/608
+* Fix pytest xfails and warnings and replace Pyright with ty by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/609
+* Fix optimizer updates, checkpoints, losses, and schedulers by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/610
+* Show Python 3.10+ in the README badge by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/611
+* Merge release changelog PRs using GITHUB_TOKEN by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/612
+* Fix release version preparation on protected main by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/613
+* Prepare versions and changelogs in one PR before publishing by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/614
+
+## New Contributors
+* @plox-sumit made their first contribution in https://github.com/kozistr/pytorch_optimizer/pull/593
+
+**Full Changelog**: https://github.com/kozistr/pytorch_optimizer/compare/v3.11.0...v4.0.0
+
+# v3.11.0
+
+## What's Changed
+* [Docs] Prepend changelog for v3.10.1 by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/509
+* [Feature] Implement `Magma` optimizer by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/514
+* [Fix] Keep `SignSGD` momentum buffer across steps by @11NOel11 in https://github.com/kozistr/pytorch_optimizer/pull/516
+* [Fix] Apply weight decay in `SignSGD` by @11NOel11 in https://github.com/kozistr/pytorch_optimizer/pull/518
+* [Fix] Match `foreach` and per-parameter rounding in `SignSGD` and `Tiger` by @11NOel11 in https://github.com/kozistr/pytorch_optimizer/pull/520
+* Fix AdaSmooth NaN on zero-initialized parameters by @winklemad in https://github.com/kozistr/pytorch_optimizer/pull/522
+* Fix AdaShift NaN on the first update with keep_num > 1 by @winklemad in https://github.com/kozistr/pytorch_optimizer/pull/523
+* Fix A2Grad diverging to NaN from an inverted running-mean weight by @winklemad in https://github.com/kozistr/pytorch_optimizer/pull/524
+* Fix M-SVAG NaN from debiasing moments by beta**step instead of 1 - beta**step by @winklemad in https://github.com/kozistr/pytorch_optimizer/pull/525
+* [Fix] `rho` calculation for `M-SVAG` optimizer by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/529
+* docs: migrate to Zensical and automate changelogs by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/532
+* Raise Shampoo factors to -1/(2k) by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/530
+* Do not start an AdamMini collective without a process group by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/534
+* Keep MARS last_grad stable across the next backward by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/533
+* Divide the DAdaptAdam distance estimate by sk_l1 by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/538
+* Store the first PID gradient for the next derivative by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/539
+* [Fix] scale StableAdamW after the second moment is updated by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/544
+* [Fix] advance the Prodigy step counter once per update by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/546
+* [Fix] keep LaProp's centered second moment out of the stored average by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/547
+* [Fix] bias-correct only the fast AdEMAMix moment by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/549
+* [Fix] lift Alice subspace vectors back to the parameter shape by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/550
+* Keep ApolloDQN stable weight decay inside one parameter by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/535
+* [Fix] lower DAdaptAdam test learning rate by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/552
+* Fix MADGRAD adding eps to the learning rate by @winklemad in https://github.com/kozistr/pytorch_optimizer/pull/526
+* Apply the documented `maximize` option in optimizers that ignored it by @winklemad in https://github.com/kozistr/pytorch_optimizer/pull/527
+* Fix Adai ignoring `maximize` (gradient negated twice) by @winklemad in https://github.com/kozistr/pytorch_optimizer/pull/528
+* [Fix] Match FAdam adaptive epsilon to the paper by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/557
+* [Fix] Correct Schedule-Free wrapper state and AdamMini GQA updates by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/558
+* [Fix] Apply MARS 1D weight decay by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/559
+* [Fix] Respect FlashAdamW learning-rate decay mode by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/561
+* [Fix] Handle mixed dtypes in StableAdamW foreach updates by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/560
+* [Fix] Honor maximize in StableAdamW scalar updates by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/562
+* [Fix] Honor maximize in LOMO updates by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/563
+* [Fix] Initialize LARS momentum from zero by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/564
+* [Fix] Include decay in the LARS trust ratio by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/565
+* [Fix] Initialize GrokFast EMA for maximize by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/566
+* [Fix] Use beta0 in Ranger21 positive-negative momentum by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/568
+* [Fix] Reject invalid schedule-free momentum values by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/567
+* [Fix] Preserve Ranger21 second-moment maximum by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/569
+* Move docs requirements file under docs/ by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/570
+* Honor Ranger21 gradient processing options by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/571
+* Clarify pull request title guidance by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/572
+* Apply coupled LAMB decay within the trust ratio by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/573
+* Match NovoGrad's first update and step size to the paper by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/574
+* [Fix] Correct Hessian sampling and optimizer edge cases by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/575
+* [Fix] Preserve base optimizer state in SAM checkpoints by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/576
+* [Fix] Remap TRAC references when loading checkpoints by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/579
+* [Fix] Project OrthoGrad gradients after closure evaluation by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/583
+* [Fix] Keep Lookahead slow momentum in separate storage by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/582
+* [Fix] Honor zero weight decay in create_optimizer by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/580
+* [Fix] Apply StableAdamW coupled decay before moment updates by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/581
+* [Fix] Apply BCE focal class weights and elementwise reduction by @AHMETHAKANBEZIR1 in https://github.com/kozistr/pytorch_optimizer/pull/585
+* [Fix] Stabilize OrthoGrad and FTRL GPU training by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/586
+* [Build] Add manual release workflow by @kozistr in https://github.com/kozistr/pytorch_optimizer/pull/587
+* Accumulate the AdaNorm norm average and guard zero gradients by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/536
+* [Fix] take the Ano second-moment sign before the decay by @shaneraphel in https://github.com/kozistr/pytorch_optimizer/pull/553
+
+## New Contributors
+* @11NOel11 made their first contribution in https://github.com/kozistr/pytorch_optimizer/pull/516
+* @winklemad made their first contribution in https://github.com/kozistr/pytorch_optimizer/pull/522
+* @shaneraphel made their first contribution in https://github.com/kozistr/pytorch_optimizer/pull/530
+* @AHMETHAKANBEZIR1 made their first contribution in https://github.com/kozistr/pytorch_optimizer/pull/585
+
+**Full Changelog**: https://github.com/kozistr/pytorch_optimizer/compare/v3.10.1...v3.11.0
+
 # v3.10.1
 
 ## Change Log

pytorch_optimizer-4.0.0/CONTRIBUTING.md

@@ -0,0 +1,85 @@
+# Contributing
+
+You can contribute code, tests, or documentation. Follow the existing implementations and keep your PR focused on
+the problem you are solving.
+
+## Setup and checks
+
+Install [uv](https://docs.astral.sh/uv/), clone the repository, and install its development dependencies:
+
+```shell
+uv sync
+```
+
+Before submitting a pull request, run:
+
+```shell
+uv run just format
+uv run just check
+uv run just test
+uv run coverage report -m
+```
+
+Cover new and changed implementation code at **100%**. Run tests on CPU by default. For GPU testing, use
+`python -m pytest --device=cuda` with a CUDA-enabled PyTorch interpreter.
+
+## Code
+
+- Use single-quoted strings and a maximum line length of 119 characters. Separate logical steps with blank lines.
+- Follow the original paper when implementing an optimizer and reuse existing `BaseOptimizer` helpers.
+- Inherit from `BaseOptimizer` when adding an optimizer and register it in `OPTIMIZER_LIST`. Register new components
+  in the appropriate package exports and update their README tables.
+
+## Tests
+
+Add `(optimizer_name, options, iterations)` to `OPTIMIZER_RECIPES` in [tests/recipes.py](tests/recipes.py)
+for optimizer convergence tests. Add focused cases in `tests/optimizers/test_<module>.py` for numerical updates,
+checkpoint restoration, or edge cases that the training recipes do not verify. Keep shared validation, variant,
+wrapper, loss, and scheduler cases in their existing `tests/test_*.py` modules.
+
+Reuse the model and parameter builders in [tests/fixtures.py](tests/fixtures.py) and helpers in
+[tests/utils.py](tests/utils.py). Use a parameter for tests that call optimizer methods; use the shared model for
+forward passes or model-based APIs. Group related cases in pytest classes where they share setup.
+
+Use small, deterministic inputs and compare results with known values or a trusted reference. A focused update test
+can use a parameter:
+
+```python
+import torch
+
+from tests.fixtures import make_parameter
+from tests.utils import build_optimizer
+
+
+def test_sgd_update():
+    param = make_parameter((2,), grad=1.0)
+    optimizer = build_optimizer('sgd', [param], lr=0.1)
+
+    optimizer.step()
+
+    torch.testing.assert_close(param, torch.tensor([-0.1, -0.1]))
+```
+
+Use `load_optimizer()` for constructor validation and `create_optimizer()` to test its public API. Check expected
+exceptions with `pytest.raises`, and parametrize the options each optimizer supports. Check existing cases before
+adding a test to avoid duplicating their assertions.
+
+## Documentation
+
+- Write API docstrings in Google style alongside the implementation.
+- Write documentation pages in `docs/`, with usage examples in [docs/getting-started.md](docs/getting-started.md).
+  Update `README.md` when adding components or changing public usage.
+- Run `uv run just docs-build` and resolve build errors and warnings before submitting changes to the site.
+
+## Pull requests
+
+Describe the problem, the resulting behavior, and how you tested the change. Include the paper or reference
+implementation for new optimizers and explain any algorithm differences. Report checks you could not run.
+
+Start commit subjects with `feat:`, `fix:`, `perf:`, `style:`, `refactor:`, `docs:`, `chore:`, `build:`, or `update:`.
+Use concise, specific commit subjects and PR titles.
+
+If you use a coding agent or LLM, review its output and make sure you understand the implementation and tests.
+You are responsible for the changes you submit.
+
+For questions, open an issue or discussion, or ask in your pull request.