scalar-autograd 0.1.4 → 0.1.5

package/dist/Value.js

@@ -0,0 +1,424 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Value = exports.Losses = exports.AdamW = exports.Adam = exports.SGD = exports.Optimizer = exports.V = void 0;
+var V_1 = require("./V");
+Object.defineProperty(exports, "V", { enumerable: true, get: function () { return V_1.V; } });
+var Optimizers_1 = require("./Optimizers");
+Object.defineProperty(exports, "Optimizer", { enumerable: true, get: function () { return Optimizers_1.Optimizer; } });
+Object.defineProperty(exports, "SGD", { enumerable: true, get: function () { return Optimizers_1.SGD; } });
+Object.defineProperty(exports, "Adam", { enumerable: true, get: function () { return Optimizers_1.Adam; } });
+Object.defineProperty(exports, "AdamW", { enumerable: true, get: function () { return Optimizers_1.AdamW; } });
+var Losses_1 = require("./Losses");
+Object.defineProperty(exports, "Losses", { enumerable: true, get: function () { return Losses_1.Losses; } });
+const EPS = 1e-12;
+const ValueTrig_1 = require("./ValueTrig");
+const ValueActivation_1 = require("./ValueActivation");
+const ValueArithmetic_1 = require("./ValueArithmetic");
+const ValueComparison_1 = require("./ValueComparison");
+class Value {
+    static no_grad_mode = false;
+    data;
+    grad = 0;
+    requiresGrad;
+    backwardFn = () => { };
+    prev = [];
+    label;
+    constructor(data, label = "", requiresGrad = false) {
+        if (typeof data !== 'number' || Number.isNaN(data) || !Number.isFinite(data)) {
+            throw new Error(`Invalid number passed to Value: ${data}`);
+        }
+        this.data = data;
+        this.label = label;
+        this.requiresGrad = requiresGrad;
+    }
+    static ensureValue(x) {
+        return typeof x === 'number' ? new Value(x) : x;
+    }
+    /**
+     * Returns sin(this).
+     * @returns New Value with sin.
+     */
+    sin() {
+        return ValueTrig_1.ValueTrig.sin(this);
+    }
+    /**
+     * Returns cos(this).
+     * @returns New Value with cos.
+     */
+    cos() {
+        return ValueTrig_1.ValueTrig.cos(this);
+    }
+    /**
+     * Returns tan(this).
+     * @returns New Value with tan.
+     */
+    tan() {
+        return ValueTrig_1.ValueTrig.tan(this);
+    }
+    /**
+     * Returns asin(this).
+     * @returns New Value with asin.
+     */
+    asin() {
+        return ValueTrig_1.ValueTrig.asin(this);
+    }
+    /**
+     * Returns acos(this).
+     * @returns New Value with acos.
+     */
+    acos() {
+        return ValueTrig_1.ValueTrig.acos(this);
+    }
+    /**
+     * Returns atan(this).
+     * @returns New Value with atan.
+     */
+    atan() {
+        return ValueTrig_1.ValueTrig.atan(this);
+    }
+    /**
+     * Returns relu(this).
+     * @returns New Value with relu.
+     */
+    relu() {
+        return ValueActivation_1.ValueActivation.relu(this);
+    }
+    /**
+     * Returns abs(this).
+     * @returns New Value with abs.
+     */
+    abs() {
+        return ValueArithmetic_1.ValueArithmetic.abs(this);
+    }
+    /**
+     * Returns exp(this).
+     * @returns New Value with exp.
+     */
+    exp() {
+        return ValueArithmetic_1.ValueArithmetic.exp(this);
+    }
+    /**
+     * Returns log(this).
+     * @returns New Value with log.
+     */
+    log() {
+        return ValueArithmetic_1.ValueArithmetic.log(this, EPS);
+    }
+    /**
+     * Returns min(this, other).
+     * @param other Value to compare
+     * @returns New Value with min.
+     */
+    min(other) {
+        return ValueArithmetic_1.ValueArithmetic.min(this, other);
+    }
+    /**
+     * Returns max(this, other).
+     * @param other Value to compare
+     * @returns New Value with max.
+     */
+    max(other) {
+        return ValueArithmetic_1.ValueArithmetic.max(this, other);
+    }
+    /**
+     * Adds this and other.
+     * @param other Value or number to add
+     * @returns New Value with sum.
+     */
+    add(other) {
+        return ValueArithmetic_1.ValueArithmetic.add(this, Value.ensureValue(other));
+    }
+    /**
+     * Multiplies this and other.
+     * @param other Value or number to multiply
+     * @returns New Value with product.
+     */
+    mul(other) {
+        return ValueArithmetic_1.ValueArithmetic.mul(this, Value.ensureValue(other));
+    }
+    /**
+     * Subtracts other from this.
+     * @param other Value or number to subtract
+     * @returns New Value with difference.
+     */
+    sub(other) {
+        return ValueArithmetic_1.ValueArithmetic.sub(this, Value.ensureValue(other));
+    }
+    /**
+     * Divides this by other.
+     * @param other Value or number divisor
+     * @returns New Value with quotient.
+     */
+    div(other) {
+        return ValueArithmetic_1.ValueArithmetic.div(this, Value.ensureValue(other), EPS);
+    }
+    /**
+     * Raises this to the power exp.
+     * @param exp Exponent
+     * @returns New Value with pow(this, exp)
+     */
+    pow(exp) {
+        return ValueArithmetic_1.ValueArithmetic.pow(this, exp);
+    }
+    /**
+     * Raises this to a dynamic Value (other).
+     * @param other Exponent Value or number
+     * @returns New Value with pow(this, other)
+     */
+    powValue(other) {
+        return ValueArithmetic_1.ValueArithmetic.powValue(this, Value.ensureValue(other), EPS);
+    }
+    /**
+     * Returns this modulo other.
+     * @param other Divisor Value
+     * @returns New Value with modulo.
+     */
+    mod(other) {
+        return ValueArithmetic_1.ValueArithmetic.mod(this, other);
+    }
+    /**
+     * Returns Value indicating if this equals other.
+     * @param other Value to compare
+     * @returns New Value (1 if equal, else 0)
+     */
+    eq(other) {
+        return ValueComparison_1.ValueComparison.eq(this, other);
+    }
+    /**
+     * Returns Value indicating if this not equals other.
+     * @param other Value to compare
+     * @returns New Value (1 if not equal, else 0)
+     */
+    neq(other) {
+        return ValueComparison_1.ValueComparison.neq(this, other);
+    }
+    /**
+     * Returns Value indicating if this greater than other.
+     * @param other Value to compare
+     * @returns New Value (1 if true, else 0)
+     */
+    gt(other) {
+        return ValueComparison_1.ValueComparison.gt(this, other);
+    }
+    /**
+     * Returns Value indicating if this less than other.
+     * @param other Value to compare
+     * @returns New Value (1 if true, else 0)
+     */
+    lt(other) {
+        return ValueComparison_1.ValueComparison.lt(this, other);
+    }
+    /**
+     * Returns Value indicating if this greater than or equal to other.
+     * @param other Value to compare
+     * @returns New Value (1 if true, else 0)
+     */
+    gte(other) {
+        return ValueComparison_1.ValueComparison.gte(this, other);
+    }
+    /**
+     * Returns Value indicating if this less than or equal to other.
+     * @param other Value to compare
+     * @returns New Value (1 if true, else 0)
+     */
+    lte(other) {
+        return ValueComparison_1.ValueComparison.lte(this, other);
+    }
+    /**
+     * Returns softplus(this).
+     * @returns New Value with softplus.
+     */
+    softplus() {
+        return ValueActivation_1.ValueActivation.softplus(this);
+    }
+    /**
+     * Returns the floor of this Value.
+     * @returns New Value with floor(data).
+     */
+    floor() {
+        return ValueArithmetic_1.ValueArithmetic.floor(this);
+    }
+    /**
+     * Returns the ceiling of this Value.
+     * @returns New Value with ceil(data).
+     */
+    ceil() {
+        return ValueArithmetic_1.ValueArithmetic.ceil(this);
+    }
+    /**
+     * Returns the rounded value of this Value.
+     * @returns New Value with rounded data.
+     */
+    round() {
+        return ValueArithmetic_1.ValueArithmetic.round(this);
+    }
+    /**
+     * Returns the square of this Value.
+     * @returns New Value with squared data.
+     */
+    square() {
+        return ValueArithmetic_1.ValueArithmetic.square(this);
+    }
+    /**
+     * Returns the cube of this Value.
+     * @returns New Value with cubed data.
+     */
+    cube() {
+        return ValueArithmetic_1.ValueArithmetic.cube(this);
+    }
+    /**
+     * Returns the reciprocal (1/x) of this Value.
+     * @returns New Value with reciprocal.
+     */
+    reciprocal() {
+        return ValueArithmetic_1.ValueArithmetic.reciprocal(this, EPS);
+    }
+    /**
+     * Clamps this between min and max.
+     * @param min Minimum value
+     * @param max Maximum value
+     * @returns New clamped Value
+     */
+    clamp(min, max) {
+        return ValueArithmetic_1.ValueArithmetic.clamp(this, min, max);
+    }
+    /**
+     * Returns the negation (-this) Value.
+     * @returns New Value which is the negation.
+     */
+    neg() {
+        return ValueArithmetic_1.ValueArithmetic.neg(this);
+    }
+    /**
+     * Returns the sum of the given Values.
+     * @param vals Array of Value objects
+     * @returns New Value holding their sum.
+     */
+    static sum(vals) {
+        return ValueArithmetic_1.ValueArithmetic.sum(vals);
+    }
+    /**
+     * Returns the mean of the given Values.
+     * @param vals Array of Value objects
+     * @returns New Value holding their mean.
+     */
+    static mean(vals) {
+        return ValueArithmetic_1.ValueArithmetic.mean(vals);
+    }
+    /**
+     * Returns tanh(this).
+     * @returns New Value with tanh.
+     */
+    tanh() {
+        return ValueActivation_1.ValueActivation.tanh(this);
+    }
+    /**
+     * Returns sigmoid(this).
+     * @returns New Value with sigmoid.
+     */
+    sigmoid() {
+        return ValueActivation_1.ValueActivation.sigmoid(this);
+    }
+    /**
+     * Performs a reverse-mode autodiff backward pass from this Value.
+     * @param zeroGrad If true, zeroes all grads in the graph before backward
+     */
+    backward(zeroGrad = false) {
+        // Only allow backward on scalars (not arrays), i.e. single value outputs
+        // (output shape check is redundant for this codebase, but keep to scalar-by-convention)
+        if (zeroGrad)
+            Value.zeroGradTree(this);
+        const topo = [];
+        const visited = new Set();
+        const buildTopo = (v) => {
+            if (!visited.has(v)) {
+                visited.add(v);
+                for (const child of v.prev) {
+                    buildTopo(child);
+                }
+                topo.push(v);
+            }
+        };
+        buildTopo(this);
+        this.grad = 1;
+        for (let i = topo.length - 1; i >= 0; i--) {
+            if (topo[i].requiresGrad) {
+                topo[i].backwardFn();
+            }
+        }
+    }
+    /**
+     * Sets all grad fields in the computation tree (from root) to 0.
+     * @param root Value to zero tree from
+     */
+    static zeroGradTree(root) {
+        const visited = new Set();
+        const visit = (v) => {
+            if (!visited.has(v)) {
+                visited.add(v);
+                v.grad = 0;
+                for (const child of v.prev)
+                    visit(child);
+            }
+        };
+        visit(root);
+    }
+    /**
+     * Sets all grad fields in all supplied trees to 0.
+     * @param vals Values whose trees to zero
+     */
+    static zeroGradAll(vals) {
+        const visited = new Set();
+        for (const v of vals) {
+            const visit = (u) => {
+                if (!visited.has(u)) {
+                    visited.add(u);
+                    u.grad = 0;
+                    for (const child of u.prev)
+                        visit(child);
+                }
+            };
+            visit(v);
+        }
+    }
+    /**
+     * Internal helper to construct a Value with correct backward fn and grads.
+     * @param data Output value data
+     * @param left Left operand Value
+     * @param right Right operand Value or null
+     * @param backwardFnBuilder Function to create backward closure
+     * @param label Node label for debugging
+     * @returns New Value node
+     */
+    static make(data, left, right, backwardFnBuilder, label) {
+        const requiresGrad = !Value.no_grad_mode && [left, right].filter(Boolean).some(v => v.requiresGrad);
+        const out = new Value(data, label, requiresGrad);
+        out.prev = Value.no_grad_mode ? [] : [left, right].filter(Boolean);
+        if (requiresGrad) {
+            out.backwardFn = backwardFnBuilder(out);
+        }
+        return out;
+    }
+    /**
+     * Returns string representation for debugging.
+     * @returns String summary of Value
+     */
+    toString() {
+        return `Value(data=${this.data.toFixed(4)}, grad=${this.grad.toFixed(4)}, label=${this.label})`;
+    }
+    /**
+     * Temporarily disables gradient tracking within the callback scope, like torch.no_grad().
+     * Restores the previous state after running fn.
+     */
+    static withNoGrad(fn) {
+        const prev = Value.no_grad_mode;
+        Value.no_grad_mode = true;
+        try {
+            return fn();
+        }
+        finally {
+            Value.no_grad_mode = prev;
+        }
+    }
+}
+exports.Value = Value;

package/dist/Value.losses-edge-cases.spec.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const Value_1 = require("./Value");
+const Losses_1 = require("./Losses");
+describe('Loss function edge cases', () => {
+    it('handles empty arrays', () => {
+        expect(Losses_1.Losses.mse([], []).data).toBe(0);
+        expect(Losses_1.Losses.mae([], []).data).toBe(0);
+        expect(Losses_1.Losses.binaryCrossEntropy([], []).data).toBe(0);
+        expect(Losses_1.Losses.categoricalCrossEntropy([], []).data).toBe(0);
+    });
+    it('throws on mismatched lengths', () => {
+        const a = [new Value_1.Value(1)];
+        const b = [new Value_1.Value(1), new Value_1.Value(2)];
+        expect(() => Losses_1.Losses.mse(a, b)).toThrow();
+    });
+    it('handles extreme values in binary cross entropy', () => {
+        const out = new Value_1.Value(0.999999, 'out', true);
+        const target = new Value_1.Value(1);
+        const loss = Losses_1.Losses.binaryCrossEntropy([out], [target]);
+        expect(loss.data).toBeGreaterThan(0);
+        expect(loss.data).toBeLessThan(0.1);
+    });
+    it('throws on invalid class indices in categorical cross entropy', () => {
+        const outputs = [new Value_1.Value(1), new Value_1.Value(2)];
+        expect(() => Losses_1.Losses.categoricalCrossEntropy(outputs, [2])).toThrow();
+        expect(() => Losses_1.Losses.categoricalCrossEntropy(outputs, [-1])).toThrow();
+        expect(() => Losses_1.Losses.categoricalCrossEntropy(outputs, [1.5])).toThrow();
+    });
+});

package/dist/Value.memory.spec.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const Value_1 = require("./Value");
+describe('Memory management', () => {
+    it('handles large computation graphs', () => {
+        let x = new Value_1.Value(1, 'x', true);
+        for (let i = 0; i < 100; i++) {
+            x = x.add(1).mul(1.01);
+        }
+        expect(() => x.backward()).not.toThrow();
+    });
+    it('zeroGradAll handles multiple disconnected graphs', () => {
+        const x1 = new Value_1.Value(1, 'x1', true);
+        const y1 = x1.mul(2);
+        const x2 = new Value_1.Value(2, 'x2', true);
+        const y2 = x2.mul(3);
+        y1.backward();
+        y2.backward();
+        Value_1.Value.zeroGradAll([y1, y2]);
+        expect(x1.grad).toBe(0);
+        expect(x2.grad).toBe(0);
+    });
+});

package/dist/Value.nn.spec.js

@@ -0,0 +1,111 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const Value_1 = require("./Value");
+const Optimizers_1 = require("./Optimizers");
+const Losses_1 = require("./Losses");
+describe("can train scalar neural networks on minimal problems", () => {
+    it("1. learns linear regression (y = 2x + 3) with SGD", () => {
+        let w = new Value_1.Value(Math.random(), "w", true);
+        let b = new Value_1.Value(Math.random(), "b", true);
+        const examples = [
+            { x: 1, y: 5 },
+            { x: 2, y: 7 },
+            { x: 3, y: 9 },
+        ];
+        const opt = new Optimizers_1.SGD([w, b], { learningRate: 0.1 });
+        for (let epoch = 0; epoch < 300; ++epoch) {
+            let preds = [];
+            let targets = [];
+            for (const ex of examples) {
+                const x = new Value_1.Value(ex.x, "x");
+                const pred = w.mul(x).add(b);
+                preds.push(pred);
+                targets.push(new Value_1.Value(ex.y));
+            }
+            let loss = Losses_1.Losses.mse(preds, targets);
+            if (loss.data < 1e-4)
+                break;
+            w.grad = 0;
+            b.grad = 0;
+            loss.backward();
+            opt.step();
+        }
+        expect(w.data).toBeCloseTo(2, 1);
+        expect(b.data).toBeCloseTo(3, 1);
+    });
+    it("2. learns quadratic fit (y = x^2) with SGD", () => {
+        let a = new Value_1.Value(Math.random(), "a", true);
+        let b = new Value_1.Value(Math.random(), "b", true);
+        let c = new Value_1.Value(Math.random(), "c", true);
+        const examples = [
+            { x: -1, y: 1 },
+            { x: 0, y: 0 },
+            { x: 2, y: 4 },
+            { x: 3, y: 9 },
+        ];
+        const opt = new Optimizers_1.SGD([a, b, c], { learningRate: 0.01 });
+        for (let epoch = 0; epoch < 400; ++epoch) {
+            let preds = [];
+            let targets = [];
+            for (const ex of examples) {
+                const x = new Value_1.Value(ex.x);
+                const pred = a.mul(x.square()).add(b.mul(x)).add(c);
+                preds.push(pred);
+                targets.push(new Value_1.Value(ex.y));
+            }
+            let loss = Losses_1.Losses.mse(preds, targets);
+            if (loss.data < 1e-4)
+                break;
+            a.grad = 0;
+            b.grad = 0;
+            c.grad = 0;
+            loss.backward();
+            opt.step();
+        }
+        expect(a.data).toBeCloseTo(1, 1);
+        expect(Math.abs(b.data)).toBeLessThan(0.5);
+        expect(Math.abs(c.data)).toBeLessThan(0.5);
+    });
+    /*
+    // This is hard to get to work reliably, I believe it's a difficult problem to solve!?
+    it("3. learns XOR with tiny MLP (2-2-1) with SGD", () => {
+      function mlp(x1: Value, x2: Value, params: Value[]): Value {
+        const [w1, w2, w3, w4, b1, b2, v1, v2, c] = params;
+        const h1 = w1.mul(x1).add(w2.mul(x2)).add(b1).tanh();
+        const h2 = w3.mul(x1).add(w4.mul(x2)).add(b2).tanh();
+        return v1.mul(h1).add(v2.mul(h2)).add(c).sigmoid();
+      }
+      let params = Array.from({ length: 9 }, (_, i) => new Value(Math.random() - 0.5, "p" + i, true));
+      const data = [
+        { x: [0, 0], y: 0 },
+        { x: [0, 1], y: 1 },
+        { x: [1, 0], y: 1 },
+        { x: [1, 1], y: 0 },
+      ];
+      const opt = new SGD(params, { learningRate: 0.01 });
+      for (let epoch = 0; epoch < 5000; ++epoch) {
+        let preds: Value[] = [];
+        let targets: Value[] = [];
+        for (const ex of data) {
+          const x1 = new Value(ex.x[0]);
+          const x2 = new Value(ex.x[1]);
+          const pred = mlp(x1, x2, params);
+          preds.push(pred);
+          targets.push(new Value(ex.y));
+        }
+        let loss = binaryCrossEntropy(preds, targets);
+        if (loss.data < 1e-3) break;
+        for (const p of params) p.grad = 0;
+        loss.backward();
+        opt.step();
+      }
+      const out00 = mlp(new Value(0), new Value(0), params).data;
+      const out01 = mlp(new Value(0), new Value(1), params).data;
+      const out10 = mlp(new Value(1), new Value(0), params).data;
+      const out11 = mlp(new Value(1), new Value(1), params).data;
+      expect((out00 < 0.4 || out00 > 0.6)).toBe(true);
+      expect(out01).toBeGreaterThan(0.6);
+      expect(out10).toBeGreaterThan(0.6);
+      expect(out11).toBeLessThan(0.4);
+    });*/
+});