scalar-autograd 0.1.6 → 0.1.8

package/dist/Optimizers.d.ts

@@ -16,6 +16,7 @@ export declare abstract class Optimizer {
      * Performs a parameter update step.
      */
     abstract step(): void;
+    abstract resetStateFor(trainable: Value): void;
     /**
      * Sets grads of all trainables to zero.
      */
@@ -53,6 +54,7 @@ export declare class SGD extends Optimizer {
      * Performs a parameter update using standard SGD.
      */
     step(): void;
+    resetStateFor(trainable: Value): void;
 }
 /**
  * Adam and AdamW optimizer parameters.
@@ -88,6 +90,7 @@ export declare class Adam extends Optimizer {
      * Performs a parameter update using Adam optimization.
      */
     step(): void;
+    resetStateFor(trainable: Value): void;
 }
 /**
  * AdamW optimizer, supports decoupled weight decay and gradient clipping (same options as Adam).
@@ -111,4 +114,5 @@ export declare class AdamW extends Optimizer {
      * Performs a parameter update using AdamW optimization (decoupled weight decay).
      */
     step(): void;
+    resetStateFor(trainable: Value): void;
 }

package/dist/Optimizers.js

@@ -64,6 +64,8 @@ class SGD extends Optimizer {
             v.data -= this.learningRate * v.grad;
         }
     }
+    resetStateFor(trainable) {
+    }
 }
 exports.SGD = SGD;
 /**
@@ -119,6 +121,10 @@ class Adam extends Optimizer {
             this.v.set(v, vVal);
         }
     }
+    resetStateFor(trainable) {
+        this.m.set(trainable, 0);
+        this.v.set(trainable, 0);
+    }
 }
 exports.Adam = Adam;
 /**
@@ -173,5 +179,9 @@ class AdamW extends Optimizer {
             this.v.set(v, vVal);
         }
     }
+    resetStateFor(trainable) {
+        this.m.set(trainable, 0);
+        this.v.set(trainable, 0);
+    }
 }
 exports.AdamW = AdamW;

package/dist/Value.d.ts

@@ -200,6 +200,11 @@ export declare class Value {
      * @returns New Value which is the negation.
      */
     neg(): Value;
+    /**
+     * Returns sign(this).
+     * @returns New Value with sign.
+     */
+    sign(): Value;
     /**
      * Returns the sum of the given Values.
      * @param vals Array of Value objects

package/dist/Value.js

@@ -289,6 +289,13 @@ class Value {
     neg() {
         return ValueArithmetic_1.ValueArithmetic.neg(this);
     }
+    /**
+     * Returns sign(this).
+     * @returns New Value with sign.
+     */
+    sign() {
+        return ValueArithmetic_1.ValueArithmetic.sign(this);
+    }
     /**
      * Returns the sum of the given Values.
      * @param vals Array of Value objects

package/dist/Value.spec.js

@@ -230,6 +230,15 @@ describe('Value unary and binary operators: trigs, relu, abs, exp/log, min/max',
     it('numerical gradient: log', () => testUnaryGrad('log', x => x.log(), x => 1 / x, 1.5));
     it('numerical gradient: tanh', () => testUnaryGrad('tanh', x => x.tanh(), x => 1 - Math.tanh(x) ** 2, 0.9));
     it('numerical gradient: sigmoid', () => testUnaryGrad('sigmoid', x => x.sigmoid(), x => { const s = 1 / (1 + Math.exp(-x)); return s * (1 - s); }, 0.7));
+    it('numerical gradient: sign', () => testUnaryGrad('sign', x => x.sign(), x => 0, 2.0));
+    it('numerical gradient: sign negative', () => testUnaryGrad('sign', x => x.sign(), x => 0, -2.0));
+    it('gradient of sign(0) is 0', () => {
+        const x = new Value_1.Value(0.0, "x", true);
+        const y = x.sign();
+        expect(y.data).toBe(0); // sign(0) should be 0
+        y.backward();
+        expect(x.grad).toBe(0); // Analytical gradient for sign(0) is implemented as 0
+    });
     // Numerical vs analytic gradient checks for binary operators
     it('numerical gradient: add', () => testBinaryGrad('add', (a, b) => a.add(b), (a, b) => 1, (a, b) => 1, 1.3, -2.1));
     it('numerical gradient: sub', () => testBinaryGrad('sub', (a, b) => a.sub(b), (a, b) => 1, (a, b) => -1, 5.2, -1.2));

package/dist/ValueArithmetic.d.ts

@@ -23,4 +23,5 @@ export declare class ValueArithmetic {
     static sum(vals: Value[]): Value;
     static mean(vals: Value[]): Value;
     static neg(a: Value): Value;
+    static sign(a: Value): Value;
 }

package/dist/ValueArithmetic.js

@@ -176,5 +176,15 @@ class ValueArithmetic {
                 a.grad -= out.grad;
         }, `(-${a.label})`);
     }
+    static sign(a) {
+        const s = Math.sign(a.data);
+        return Value_1.Value.make(s, a, null, (out) => () => {
+            // The derivative of sign(x) is 0 for x != 0.
+            // At x = 0, the derivative is undefined (Dirac delta), but for practical purposes in ML,
+            // we can define it as 0.
+            if (a.requiresGrad)
+                a.grad += 0 * out.grad;
+        }, `sign(${a.label})`);
+    }
 }
 exports.ValueArithmetic = ValueArithmetic;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scalar-autograd",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "Scalar-based reverse-mode automatic differentiation in TypeScript.",
   "main": "dist/Value.js",
   "types": "dist/Value.d.ts",