AdamWClip 0.1.0__tar.gz

adamwclip-0.1.0/AdamWClip/__init__.py

@@ -0,0 +1,3 @@
+from .optimizer import AdamWClip as AdamWClip
+
+__all__ = ["AdamWClip"]

adamwclip-0.1.0/AdamWClip/optimizer.py

@@ -0,0 +1,83 @@
+import torch
+from torch.optim.optimizer import Optimizer
+
+class AdamWClip(Optimizer):
+	"""
+	AdamW optimizer with adaptive gradient clipping
+	"""
+
+	def __init__(self, params, lr=0.001, betas=(0.9,0.999), eps=1e-8, weight_decay=0.01, clip_grad_adapt = 3, clip_grad_min=0.01, clip_grad_warm_up=10):
+		"""
+		params, lr, betas, eps, weight_decay is identical to AdamW optimizer
+		On top of that, we have the following parameters:
+		:clip_grad_adapt:   adaptive gradient clipping threshold in terms of standard deviations of the clipped gradient distribution (default: 3)
+		:clip_grad_min:     minimum value for the adaptive gradient clipping threshold (default: 0.01)
+		:clip_grad_warm_up: Number of initial update steps without gradient clipping to obtain reasonable gradient statistics at the beginning (default: 10)
+		"""
+		beta_1, beta_2 = betas
+		defaults = dict(lr=lr, beta_1=beta_1, beta_2=beta_2,weight_decay=weight_decay, clip_grad_adapt = clip_grad_adapt,clip_grad_min=clip_grad_min)
+		self.iteration = 0
+		self.clip_grad_warm_up = clip_grad_warm_up
+		self.eps = eps
+		super(AdamWClip, self).__init__(params, defaults)
+
+	def step(self, closure=None, end=False):
+		"""Performs a single optimization step."""
+		loss = None
+		if closure is not None:
+			loss = closure()
+
+		self.iteration += 1
+
+		for group in self.param_groups:
+			lr = group['lr']
+			beta_1 = group['beta_1']
+			beta_2 = group['beta_2']
+			weight_decay = group['weight_decay']
+			clip_grad_adapt = group['clip_grad_adapt']
+			clip_grad_min = group['clip_grad_min']
+
+			for p in group['params']:
+				if p.grad is None:
+					continue
+				theta = p.data
+				grad = p.grad.data
+				if weight_decay != 0:
+					theta.add_(-weight_decay*lr*theta)
+
+				# Momentum part
+				param_state = self.state[p]
+
+				# Buffers:
+				if 'sum_grad' not in param_state:
+					# 1. momentum
+					sum_grad = param_state['sum_grad'] = torch.zeros_like(grad)
+					# 2. momentum
+					sum_grad_grad = param_state['sum_grad_grad'] = torch.zeros_like(grad)
+				else:
+					# 1. momentum
+					sum_grad = param_state['sum_grad']
+					# 2. momentum
+					sum_grad_grad = param_state['sum_grad_grad']
+
+				if clip_grad_adapt is not None and self.iteration > self.clip_grad_warm_up:
+					E_grad_grad = sum_grad_grad / (1-beta_2**self.iteration)
+					clamp_adapt = clip_grad_adapt*torch.sqrt(E_grad_grad).clamp_(min=clip_grad_min)
+					grad.clamp_(-clamp_adapt,clamp_adapt)
+
+				# 1. momentum
+				sum_grad.mul_(beta_1).add_(grad*(1-beta_1))
+
+				# 2. momentum
+				sum_grad_grad.mul_(beta_2).add_(grad*grad*(1-beta_2))
+
+				# bias correction
+				E_grad = sum_grad / (1-beta_1**self.iteration)
+				E_grad_grad = sum_grad_grad / (1-beta_2**self.iteration) # could be optimized
+
+				step = lr*E_grad/(torch.sqrt(E_grad_grad)+self.eps) # could be optimized
+
+				# apply update step
+				theta.add_(-step)
+
+		return loss

adamwclip-0.1.0/AdamWClip.egg-info/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: AdamWClip
+Version: 0.1.0
+Summary: AdamWClip is an optimizer that extends AdamW with adaptive gradient clipping.
+Home-page: https://github.com/wandeln/AdamWClip
+Author: Nils Wandel
+Author-email: wandeln@cs.uni-bonn.de
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: torch
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# AdamWClip: AdamW with adaptive gradient clipping
+
+AdamWClip is an optimizer that extends AdamW with adaptive gradient clipping. 
+This way, the gradient clipping thresholds automatically adapt to the gradient statistics and become equivariant with respect to scaling the gradients.
+This makes finding suitable clipping thresholds much easier (usually, the default thresholds are good to go).
+Furthermore, by directly utilizing the internal state variables of Adam, AdamWClip implements adaptive gradient clipping without an additional memory footprint.
+
+## Useage
+
+To use AdamWClip in your pytorch project, simply run the following:
+
+```python
+%pip install AdamWClip
+from AdamWClip import AdamWClip
+...
+optimizer = AdamWClip(model.parameters(),*args)
+```
+
+On top of the standard parameters from AdamW, AdamWClip offers the following additional parameters:  
+	
+	- clip_grad_adapt: adaptive gradient clipping threshold in terms of standard deviations of the clipped gradient distribution (default: 3)
+	- clip_grad_min: minimum value for the adaptive gradient clipping threshold (default: 0.01)
+	- clip_grad_warm_up: Number of initial update steps without gradient clipping to obtain reasonable gradient statistics at the beginning (default: 10)
+
+In most instances, the default values should be fine.
+
+If this optimizer becomes useful to you, please consider citing this repository :)

adamwclip-0.1.0/AdamWClip.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+README.md
+pyproject.toml
+setup.py
+AdamWClip/__init__.py
+AdamWClip/optimizer.py
+AdamWClip.egg-info/PKG-INFO
+AdamWClip.egg-info/SOURCES.txt
+AdamWClip.egg-info/dependency_links.txt
+AdamWClip.egg-info/requires.txt
+AdamWClip.egg-info/top_level.txt

adamwclip-0.1.0/AdamWClip.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

adamwclip-0.1.0/AdamWClip.egg-info/requires.txt

@@ -0,0 +1 @@
+torch

adamwclip-0.1.0/AdamWClip.egg-info/top_level.txt

@@ -0,0 +1 @@
+AdamWClip

adamwclip-0.1.0/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: AdamWClip
+Version: 0.1.0
+Summary: AdamWClip is an optimizer that extends AdamW with adaptive gradient clipping.
+Home-page: https://github.com/wandeln/AdamWClip
+Author: Nils Wandel
+Author-email: wandeln@cs.uni-bonn.de
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: torch
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# AdamWClip: AdamW with adaptive gradient clipping
+
+AdamWClip is an optimizer that extends AdamW with adaptive gradient clipping. 
+This way, the gradient clipping thresholds automatically adapt to the gradient statistics and become equivariant with respect to scaling the gradients.
+This makes finding suitable clipping thresholds much easier (usually, the default thresholds are good to go).
+Furthermore, by directly utilizing the internal state variables of Adam, AdamWClip implements adaptive gradient clipping without an additional memory footprint.
+
+## Useage
+
+To use AdamWClip in your pytorch project, simply run the following:
+
+```python
+%pip install AdamWClip
+from AdamWClip import AdamWClip
+...
+optimizer = AdamWClip(model.parameters(),*args)
+```
+
+On top of the standard parameters from AdamW, AdamWClip offers the following additional parameters:  
+	
+	- clip_grad_adapt: adaptive gradient clipping threshold in terms of standard deviations of the clipped gradient distribution (default: 3)
+	- clip_grad_min: minimum value for the adaptive gradient clipping threshold (default: 0.01)
+	- clip_grad_warm_up: Number of initial update steps without gradient clipping to obtain reasonable gradient statistics at the beginning (default: 10)
+
+In most instances, the default values should be fine.
+
+If this optimizer becomes useful to you, please consider citing this repository :)

adamwclip-0.1.0/README.md

@@ -0,0 +1,27 @@
+# AdamWClip: AdamW with adaptive gradient clipping
+
+AdamWClip is an optimizer that extends AdamW with adaptive gradient clipping. 
+This way, the gradient clipping thresholds automatically adapt to the gradient statistics and become equivariant with respect to scaling the gradients.
+This makes finding suitable clipping thresholds much easier (usually, the default thresholds are good to go).
+Furthermore, by directly utilizing the internal state variables of Adam, AdamWClip implements adaptive gradient clipping without an additional memory footprint.
+
+## Useage
+
+To use AdamWClip in your pytorch project, simply run the following:
+
+```python
+%pip install AdamWClip
+from AdamWClip import AdamWClip
+...
+optimizer = AdamWClip(model.parameters(),*args)
+```
+
+On top of the standard parameters from AdamW, AdamWClip offers the following additional parameters:  
+	
+	- clip_grad_adapt: adaptive gradient clipping threshold in terms of standard deviations of the clipped gradient distribution (default: 3)
+	- clip_grad_min: minimum value for the adaptive gradient clipping threshold (default: 0.01)
+	- clip_grad_warm_up: Number of initial update steps without gradient clipping to obtain reasonable gradient statistics at the beginning (default: 10)
+
+In most instances, the default values should be fine.
+
+If this optimizer becomes useful to you, please consider citing this repository :)

adamwclip-0.1.0/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"

adamwclip-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

adamwclip-0.1.0/setup.py

@@ -0,0 +1,23 @@
+from setuptools import setup, find_packages
+
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
+setup(
+    name="AdamWClip",
+    version="0.1.0",
+    packages=find_packages(),
+    install_requires=["torch"],
+    author="Nils Wandel",
+    author_email="wandeln@cs.uni-bonn.de",
+    description="AdamWClip is an optimizer that extends AdamW with adaptive gradient clipping.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/wandeln/AdamWClip",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.8",
+)