scalar-autograd 0.1.3 → 0.1.5

package/dist/Losses.js

@@ -0,0 +1,145 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Losses = void 0;
+const Value_1 = require("./Value");
+const V_1 = require("./V");
+/**
+ * Throws an error if outputs and targets length do not match.
+ * @param outputs Array of output Values.
+ * @param targets Array of target Values.
+ */
+function checkLengthMatch(outputs, targets) {
+    if (outputs.length !== targets.length) {
+        throw new Error('Outputs and targets must have the same length');
+    }
+}
+class Losses {
+    /**
+     * Computes mean squared error (MSE) loss between outputs and targets.
+     * @param outputs Array of Value predictions.
+     * @param targets Array of Value targets.
+     * @returns Mean squared error as a Value.
+     */
+    static mse(outputs, targets) {
+        checkLengthMatch(outputs, targets);
+        if (!Array.isArray(outputs) || !Array.isArray(targets))
+            throw new TypeError('mse expects Value[] for both arguments.');
+        if (!outputs.length)
+            return new Value_1.Value(0);
+        const diffs = outputs.map((out, i) => out.sub(targets[i]).square());
+        return Value_1.Value.mean(diffs);
+    }
+    /**
+     * Computes mean absolute error (MAE) loss between outputs and targets.
+     * @param outputs Array of Value predictions.
+     * @param targets Array of Value targets.
+     * @returns Mean absolute error as a Value.
+     */
+    static mae(outputs, targets) {
+        checkLengthMatch(outputs, targets);
+        if (!Array.isArray(outputs) || !Array.isArray(targets))
+            throw new TypeError('mae expects Value[] for both arguments.');
+        if (!outputs.length)
+            return new Value_1.Value(0);
+        const diffs = outputs.map((out, i) => out.sub(targets[i]).abs());
+        return Value_1.Value.mean(diffs);
+    }
+    static EPS = 1e-12;
+    /**
+     * Computes binary cross-entropy loss between predicted outputs and targets (after sigmoid).
+     * @param outputs Array of Value predictions (expected in (0,1)).
+     * @param targets Array of Value targets (typically 0 or 1).
+     * @returns Binary cross-entropy loss as a Value.
+     */
+    static binaryCrossEntropy(outputs, targets) {
+        checkLengthMatch(outputs, targets);
+        if (!Array.isArray(outputs) || !Array.isArray(targets))
+            throw new TypeError('binaryCrossEntropy expects Value[] for both arguments.');
+        if (!outputs.length)
+            return new Value_1.Value(0);
+        const eps = Losses.EPS;
+        const one = new Value_1.Value(1);
+        const losses = outputs.map((out, i) => {
+            const t = targets[i];
+            const outClamped = out.clamp(eps, 1 - eps); // sigmoid should output (0,1)
+            return t.mul(outClamped.log()).add(one.sub(t).mul(one.sub(outClamped).log()));
+        });
+        return Value_1.Value.mean(losses).mul(-1);
+    }
+    /**
+     * Computes categorical cross-entropy loss between outputs (logits) and integer target classes.
+     * @param outputs Array of Value logits for each class.
+     * @param targets Array of integer class indices (0-based, one per sample).
+     * @returns Categorical cross-entropy loss as a Value.
+     */
+    static categoricalCrossEntropy(outputs, targets) {
+        // targets: integer encoded class indices
+        if (!Array.isArray(outputs) || !Array.isArray(targets))
+            throw new TypeError('categoricalCrossEntropy expects Value[] and number[].');
+        if (!outputs.length || !targets.length)
+            return new Value_1.Value(0);
+        if (targets.some(t => typeof t !== 'number' || !isFinite(t) || t < 0 || t >= outputs.length || Math.floor(t) !== t)) {
+            throw new Error('Target indices must be valid integers in [0, outputs.length)');
+        }
+        const eps = Losses.EPS;
+        const maxLogit = outputs.reduce((a, b) => a.data > b.data ? a : b);
+        const exps = outputs.map(out => out.sub(maxLogit).exp());
+        const sumExp = Value_1.Value.sum(exps).add(eps);
+        const softmax = exps.map(e => e.div(sumExp));
+        const tIndices = targets.map((t, i) => softmax[t]);
+        return Value_1.Value.mean(tIndices.map(sm => sm.add(eps).log().mul(-1)));
+    }
+    /**
+     * Computes Huber loss between outputs and targets.
+     * Combines quadratic loss for small residuals and linear loss for large residuals.
+     * @param outputs Array of Value predictions.
+     * @param targets Array of Value targets.
+     * @param delta Threshold at which to switch from quadratic to linear (default: 1.0).
+     * @returns Huber loss as a Value.
+     */
+    static huber(outputs, targets, delta = 1.0) {
+        checkLengthMatch(outputs, targets);
+        if (!Array.isArray(outputs) || !Array.isArray(targets))
+            throw new TypeError('huber expects Value[] for both arguments.');
+        if (!outputs.length)
+            return new Value_1.Value(0);
+        const deltaValue = new Value_1.Value(delta);
+        const half = new Value_1.Value(0.5);
+        const losses = outputs.map((out, i) => {
+            const residual = V_1.V.abs(V_1.V.sub(out, targets[i]));
+            const condition = V_1.V.lt(residual, deltaValue);
+            const quadraticLoss = V_1.V.mul(half, V_1.V.square(residual));
+            const linearLoss = V_1.V.mul(deltaValue, V_1.V.sub(residual, V_1.V.mul(half, deltaValue)));
+            return V_1.V.ifThenElse(condition, quadraticLoss, linearLoss);
+        });
+        return V_1.V.mean(losses);
+    }
+    /**
+     * Computes Tukey loss between outputs and targets.
+     * This robust loss function saturates for large residuals.
+     *
+     * @param outputs Array of Value predictions.
+     * @param targets Array of Value targets.
+     * @param c Threshold constant (typically 4.685).
+     * @returns Tukey loss as a Value.
+     */
+    static tukey(outputs, targets, c = 4.685) {
+        checkLengthMatch(outputs, targets);
+        const c2_over_6 = (c * c) / 6;
+        const cValue = V_1.V.C(c);
+        const c2_over_6_Value = V_1.V.C(c2_over_6);
+        const losses = outputs.map((out, i) => {
+            const diff = V_1.V.abs(V_1.V.sub(out, targets[i]));
+            const inlier = V_1.V.lte(diff, cValue);
+            const rc = V_1.V.div(diff, cValue);
+            const rc2 = V_1.V.square(rc);
+            const oneMinusRC2 = V_1.V.sub(1, rc2);
+            const inner = V_1.V.pow(oneMinusRC2, 3);
+            const inlierLoss = V_1.V.mul(c2_over_6_Value, V_1.V.sub(1, inner));
+            const loss = V_1.V.ifThenElse(inlier, inlierLoss, c2_over_6_Value);
+            return loss;
+        });
+        return V_1.V.mean(losses);
+    }
+}
+exports.Losses = Losses;

package/dist/Losses.spec.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const Value_1 = require("./Value");
+const Losses_1 = require("./Losses");
+describe("Losses", () => {
+    it("mse computes value and gradients correctly", () => {
+        const x = new Value_1.Value(2, "x", true);
+        const y = new Value_1.Value(3, "y", true);
+        const tx = new Value_1.Value(5, "tx");
+        const ty = new Value_1.Value(1, "ty");
+        const loss = Losses_1.Losses.mse([x, y], [tx, ty]); // (1/2)*((2-5)^2 + (3-1)^2) = (1/2)*(9+4) = 6.5
+        expect(loss.data).toBeCloseTo(6.5);
+        loss.backward();
+        expect(x.grad).toBeCloseTo(-3);
+        expect(y.grad).toBeCloseTo(2);
+    });
+    it("mae computes value and gradients correctly", () => {
+        const x = new Value_1.Value(2, "x", true);
+        const y = new Value_1.Value(-3, "y", true);
+        const tx = new Value_1.Value(5, "tx");
+        const ty = new Value_1.Value(2, "ty");
+        const loss = Losses_1.Losses.mae([x, y], [tx, ty]); // (1/2)*(abs(2-5)+abs(-3-2)) = (1/2)*(3+5)=4
+        expect(loss.data).toBeCloseTo(4);
+        loss.backward();
+        expect(x.grad).toBeCloseTo(-0.5);
+        expect(y.grad).toBeCloseTo(-0.5);
+    });
+    it("binaryCrossEntropy computes value and gradients correctly for easy case", () => {
+        const out = new Value_1.Value(0.9, "out", true);
+        const target = new Value_1.Value(1, "target");
+        const loss = Losses_1.Losses.binaryCrossEntropy([out], [target]);
+        expect(loss.data).toBeCloseTo(-Math.log(0.9));
+        loss.backward();
+        expect(out.grad).toBeCloseTo(-1 / 0.9, 4);
+    });
+    it("categoricalCrossEntropy computes value and gradients (softmax+NLL)", () => {
+        // logits: [2, 1, 0], true = 0
+        const a = new Value_1.Value(2, "a", true);
+        const b = new Value_1.Value(1, "b", true);
+        const c = new Value_1.Value(0, "c", true);
+        const targets = [0];
+        const loss = Losses_1.Losses.categoricalCrossEntropy([a, b, c], targets);
+        const softmax = [
+            Math.exp(2) / (Math.exp(2) + Math.exp(1) + Math.exp(0)),
+            Math.exp(1) / (Math.exp(2) + Math.exp(1) + Math.exp(0)),
+            Math.exp(0) / (Math.exp(2) + Math.exp(1) + Math.exp(0))
+        ];
+        expect(loss.data).toBeCloseTo(-Math.log(softmax[0]), 4);
+        loss.backward();
+        expect(a.grad).toBeCloseTo(softmax[0] - 1, 4);
+        expect(b.grad).toBeCloseTo(softmax[1], 4);
+        expect(c.grad).toBeCloseTo(softmax[2], 4);
+    });
+});

package/dist/Optimizers.edge-cases.spec.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const Value_1 = require("./Value");
+const Optimizers_1 = require("./Optimizers");
+describe('Optimizer edge cases', () => {
+    it('handles empty parameter list', () => {
+        const opt = new Optimizers_1.SGD([], { learningRate: 0.1 });
+        expect(() => opt.step()).not.toThrow();
+    });
+    it('filters out non-trainable parameters', () => {
+        const x = new Value_1.Value(1, 'x', true);
+        const y = new Value_1.Value(2, 'y', false);
+        const opt = new Optimizers_1.SGD([x, y], { learningRate: 0.1 });
+        x.grad = 1;
+        y.grad = 1;
+        opt.step();
+        expect(x.data).toBe(0.9);
+        expect(y.data).toBe(2); // unchanged
+    });
+    it('Adam handles zero gradients correctly', () => {
+        const x = new Value_1.Value(1, 'x', true);
+        const opt = new Optimizers_1.Adam([x], { learningRate: 0.1 });
+        x.grad = 0;
+        for (let i = 0; i < 10; i++) {
+            opt.step();
+        }
+        expect(x.data).toBe(1); // unchanged
+    });
+});

package/dist/Optimizers.js

@@ -0,0 +1,177 @@
+"use strict";
+// Optimizers.ts
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AdamW = exports.Adam = exports.SGD = exports.Optimizer = void 0;
+/**
+ * Abstract base class for all optimizers.
+ * Ensures only requiresGrad parameters are optimized.
+ */
+class Optimizer {
+    trainables;
+    learningRate;
+    /**
+     * Constructs an Optimizer.
+     * @param trainables Array of Value parameters to optimize.
+     * @param learningRate Learning rate for updates.
+     */
+    constructor(trainables, learningRate) {
+        this.trainables = trainables.filter(v => v.requiresGrad);
+        this.learningRate = learningRate;
+    }
+    /**
+     * Sets grads of all trainables to zero.
+     */
+    zeroGrad() {
+        for (const v of this.trainables)
+            v.grad = 0;
+    }
+    /**
+     * Clips global norm of gradients as regularization.
+     * @param maxNorm Maximum allowed norm for gradients.
+     */
+    clipGradients(maxNorm) {
+        const totalNorm = Math.sqrt(this.trainables.reduce((sum, v) => sum + v.grad * v.grad, 0));
+        if (totalNorm > maxNorm) {
+            const scale = maxNorm / (totalNorm + 1e-6);
+            for (const v of this.trainables)
+                v.grad *= scale;
+        }
+    }
+}
+exports.Optimizer = Optimizer;
+/**
+ * Stochastic Gradient Descent (SGD) optimizer. Accepts weightDecay and gradientClip for API consistency (ignored).
+ */
+class SGD extends Optimizer {
+    weightDecay;
+    gradientClip;
+    /**
+     * Constructs an SGD optimizer.
+     * @param trainables Array of Value parameters to optimize.
+     * @param opts Optional parameters (learningRate, weightDecay, gradientClip).
+     */
+    constructor(trainables, opts = {}) {
+        super(trainables, opts.learningRate ?? 1e-2);
+        this.weightDecay = opts.weightDecay ?? 0;
+        this.gradientClip = opts.gradientClip ?? 0;
+    }
+    /**
+     * Performs a parameter update using standard SGD.
+     */
+    step() {
+        // Intentionally ignoring weightDecay/gradientClip for SGD
+        for (const v of this.trainables) {
+            v.data -= this.learningRate * v.grad;
+        }
+    }
+}
+exports.SGD = SGD;
+/**
+ * Adam optimizer, supports decoupled weight decay and gradient clipping.
+ */
+class Adam extends Optimizer {
+    beta1;
+    beta2;
+    epsilon;
+    weightDecay;
+    gradientClip;
+    m = new Map();
+    v = new Map();
+    stepCount = 0;
+    /**
+     * Constructs an Adam optimizer.
+     * @param trainables Array of Value parameters to optimize.
+     * @param opts Optional parameters (learningRate, weightDecay, gradientClip, beta1, beta2, epsilon).
+     */
+    constructor(trainables, opts = {}) {
+        super(trainables, opts.learningRate ?? 0.001);
+        this.beta1 = opts.beta1 ?? 0.9;
+        this.beta2 = opts.beta2 ?? 0.999;
+        this.epsilon = opts.epsilon ?? 1e-8;
+        this.weightDecay = opts.weightDecay ?? 0;
+        this.gradientClip = opts.gradientClip ?? 0;
+        for (const v of this.trainables) {
+            this.m.set(v, 0);
+            this.v.set(v, 0);
+        }
+    }
+    /**
+     * Performs a parameter update using Adam optimization.
+     */
+    step() {
+        this.stepCount++;
+        for (const v of this.trainables) {
+            let grad = v.grad;
+            if (this.weightDecay > 0)
+                grad += this.weightDecay * v.data;
+            let m = this.m.get(v);
+            let vVal = this.v.get(v);
+            m = this.beta1 * m + (1 - this.beta1) * grad;
+            vVal = this.beta2 * vVal + (1 - this.beta2) * grad * grad;
+            const mHat = m / (1 - Math.pow(this.beta1, this.stepCount));
+            const vHat = vVal / (1 - Math.pow(this.beta2, this.stepCount));
+            let update = mHat / (Math.sqrt(vHat) + this.epsilon);
+            if (this.gradientClip > 0) {
+                update = Math.max(-this.gradientClip, Math.min(update, this.gradientClip));
+            }
+            v.data -= this.learningRate * update;
+            this.m.set(v, m);
+            this.v.set(v, vVal);
+        }
+    }
+}
+exports.Adam = Adam;
+/**
+ * AdamW optimizer, supports decoupled weight decay and gradient clipping (same options as Adam).
+ */
+class AdamW extends Optimizer {
+    beta1;
+    beta2;
+    epsilon;
+    weightDecay;
+    gradientClip;
+    m = new Map();
+    v = new Map();
+    stepCount = 0;
+    /**
+     * Constructs an AdamW optimizer.
+     * @param trainables Array of Value parameters to optimize.
+     * @param opts Optional parameters (learningRate, weightDecay, gradientClip, beta1, beta2, epsilon).
+     */
+    constructor(trainables, opts = {}) {
+        super(trainables, opts.learningRate ?? 0.001);
+        this.beta1 = opts.beta1 ?? 0.9;
+        this.beta2 = opts.beta2 ?? 0.999;
+        this.epsilon = opts.epsilon ?? 1e-8;
+        this.weightDecay = opts.weightDecay ?? 0.01;
+        this.gradientClip = opts.gradientClip ?? 0;
+        for (const v of this.trainables) {
+            this.m.set(v, 0);
+            this.v.set(v, 0);
+        }
+    }
+    /**
+     * Performs a parameter update using AdamW optimization (decoupled weight decay).
+     */
+    step() {
+        this.stepCount++;
+        for (const v of this.trainables) {
+            let grad = v.grad;
+            let m = this.m.get(v);
+            let vVal = this.v.get(v);
+            m = this.beta1 * m + (1 - this.beta1) * grad;
+            vVal = this.beta2 * vVal + (1 - this.beta2) * grad * grad;
+            const mHat = m / (1 - Math.pow(this.beta1, this.stepCount));
+            const vHat = vVal / (1 - Math.pow(this.beta2, this.stepCount));
+            let update = mHat / (Math.sqrt(vHat) + this.epsilon);
+            if (this.gradientClip > 0) {
+                update = Math.max(-this.gradientClip, Math.min(update, this.gradientClip));
+            }
+            // Weight decay is decoupled as in AdamW paper:
+            v.data -= this.learningRate * update + this.learningRate * this.weightDecay * v.data;
+            this.m.set(v, m);
+            this.v.set(v, vVal);
+        }
+    }
+}
+exports.AdamW = AdamW;

package/dist/Optimizers.spec.js

@@ -0,0 +1,56 @@
+"use strict";
+// Optimizers.spec.ts
+Object.defineProperty(exports, "__esModule", { value: true });
+const Value_1 = require("./Value");
+const Optimizers_1 = require("./Optimizers");
+function createLoss(x) {
+    // Loss = (x - 5)^2
+    const target = new Value_1.Value(5);
+    const diff = x.sub(target);
+    return diff.mul(diff);
+}
+describe("Optimizers", () => {
+    it("SGD minimizes simple quadratic loss", () => {
+        const x = new Value_1.Value(0, "x", true);
+        const opt = new Optimizers_1.SGD([x], { learningRate: 0.1 });
+        let lossVal;
+        for (let i = 0; i < 100; i++) {
+            const loss = createLoss(x);
+            lossVal = loss.data;
+            if (lossVal < 1e-6)
+                break;
+            opt.zeroGrad();
+            loss.backward();
+            opt.step();
+        }
+        // AdamW does true weight decay, so the final x is slightly under target
+        expect(x.data).toBeLessThan(5.0);
+        expect(x.data).toBeGreaterThan(4.8);
+    });
+    it("Adam minimizes simple quadratic loss", () => {
+        const x = new Value_1.Value(0, "x", true);
+        const opt = new Optimizers_1.Adam([x], { learningRate: 0.1 });
+        let lossVal;
+        for (let i = 0; i < 100; i++) {
+            const loss = createLoss(x);
+            lossVal = loss.data;
+            if (lossVal < 1e-6)
+                break;
+            opt.zeroGrad();
+            loss.backward();
+            opt.step();
+        }
+        expect(x.data).toBeCloseTo(5.0, 1);
+    });
+    it("AdamW minimizes simple quadratic loss with weight decay", () => {
+        const x = new Value_1.Value(0, "x", true);
+        const opt = new Optimizers_1.AdamW([x], { learningRate: 0.1, beta1: 0.9, beta2: 0.999, weightDecay: 0.005 });
+        for (let i = 0; i < 100; i++) {
+            const loss = createLoss(x);
+            opt.zeroGrad();
+            loss.backward();
+            opt.step();
+        }
+        expect(x.data).toBeCloseTo(5.0, 1);
+    });
+});

package/dist/Value.edge-cases.spec.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const Value_1 = require("./Value");
+// Edge cases and error handling
+describe('Value edge cases and error handling', () => {
+    it('throws on invalid numeric inputs', () => {
+        expect(() => new Value_1.Value(NaN)).toThrow();
+        expect(() => new Value_1.Value(Infinity)).toThrow();
+        expect(() => new Value_1.Value(-Infinity)).toThrow();
+    });
+    it('handles gradient accumulation correctly', () => {
+        const x = new Value_1.Value(2, 'x', true);
+        const y = x.mul(3);
+        const z = x.mul(4);
+        const out = y.add(z);
+        out.backward();
+        expect(x.grad).toBe(7); // 3 + 4
+    });
+    it('handles repeated use of same value in expression', () => {
+        const x = new Value_1.Value(3, 'x', true);
+        const y = x.mul(x).mul(x); // x^3
+        y.backward();
+        expect(x.grad).toBeCloseTo(27); // 3*x^2 = 27
+    });
+    it('throws on division by zero', () => {
+        const a = new Value_1.Value(1);
+        const b = new Value_1.Value(0);
+        expect(() => a.div(b)).toThrow();
+    });
+    it('throws on log of negative number', () => {
+        const x = new Value_1.Value(-1);
+        expect(() => x.log()).toThrow();
+    });
+    it('throws on negative base with fractional exponent', () => {
+        const x = new Value_1.Value(-2);
+        expect(() => x.pow(0.5)).toThrow();
+    });
+});
+// Complex expressions
+describe('Complex mathematical expressions', () => {
+    it('computes gradient of complex expression', () => {
+        const x = new Value_1.Value(0.5, 'x', true);
+        const y = x.sin().mul(x.cos()).add(x.exp());
+        y.backward();
+        const expected = Math.cos(0.5) ** 2 - Math.sin(0.5) ** 2 + Math.exp(0.5);
+        expect(x.grad).toBeCloseTo(expected, 4);
+    });
+    it('handles nested activation functions', () => {
+        const x = new Value_1.Value(0.5, 'x', true);
+        const y = x.tanh().sigmoid().relu();
+        y.backward();
+        expect(x.grad).toBeGreaterThan(0);
+    });
+});

package/dist/Value.grad-flow.spec.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const Value_1 = require("./Value");
+describe('Gradient flow control', () => {
+    it('stops gradient at non-requiresGrad nodes', () => {
+        const x = new Value_1.Value(2, 'x', true);
+        const y = new Value_1.Value(3, 'y', false);
+        const z = new Value_1.Value(4, 'z', true);
+        const out = x.mul(y).add(z);
+        out.backward();
+        expect(x.grad).toBe(3);
+        expect(y.grad).toBe(0);
+        expect(z.grad).toBe(1);
+    });
+    it('handles detached computation graphs', () => {
+        const x = new Value_1.Value(2, 'x', true);
+        const y = x.mul(3);
+        const z = new Value_1.Value(y.data, 'z', true); // detached
+        const out = z.mul(4);
+        out.backward();
+        expect(z.grad).toBe(4);
+        expect(x.grad).toBe(0); // no gradient flows to x
+    });
+});