turbo-stream 0.0.1

package/README.md

@@ -0,0 +1,26 @@
+# Turbo Stream
+
+A streaming data transport format that aims to support built-in features such as Promises, Dates, RegExps, Maps, Sets and more.
+
+## Shout Out!
+
+Shout out to Rich Harris and his https://github.com/rich-harris/devalue project. Devalue has heavily influenced this project and portions
+of the code have been directly lifted from it. I highly recommend checking it out if you need something more cusomizable or without streaming support.
+
+## Installation
+
+```bash
+npm install turbo-stream
+```
+
+## Usage
+
+```js
+import { decode, encode } from "turbo-stream";
+
+const encodedStream = encode(Promise.resolve(42));
+const decoded = await decode(encodedStream);
+console.log(decoded.value); // a Promise
+console.log(await decoded.value); // 42
+await decoded.done; // wait for the stream to finish
+```

package/dist/constants.d.ts

@@ -0,0 +1,14 @@
+export declare const UNDEFINED = -1;
+export declare const HOLE = -1;
+export declare const NAN = -2;
+export declare const POSITIVE_INFINITY = -3;
+export declare const NEGATIVE_INFINITY = -4;
+export declare const NEGATIVE_ZERO = -5;
+export declare const TYPE_BIGINT = "B";
+export declare const TYPE_DATE = "D";
+export declare const TYPE_MAP = "M";
+export declare const TYPE_SET = "S";
+export declare const TYPE_REGEXP = "R";
+export declare const TYPE_SYMBOL = "Y";
+export declare const TYPE_NULL_OBJECT = "N";
+export declare const TYPE_PROMISE = "P";

package/dist/constants.js

@@ -0,0 +1,14 @@
+export const UNDEFINED = -1;
+export const HOLE = -1;
+export const NAN = -2;
+export const POSITIVE_INFINITY = -3;
+export const NEGATIVE_INFINITY = -4;
+export const NEGATIVE_ZERO = -5;
+export const TYPE_BIGINT = "B";
+export const TYPE_DATE = "D";
+export const TYPE_MAP = "M";
+export const TYPE_SET = "S";
+export const TYPE_REGEXP = "R";
+export const TYPE_SYMBOL = "Y";
+export const TYPE_NULL_OBJECT = "N";
+export const TYPE_PROMISE = "P";

package/dist/decode.d.ts

@@ -0,0 +1,7 @@
+import { Deferred } from "./deferred.js";
+export interface ThisDecode {
+    values: unknown[];
+    hydrated: unknown[];
+    deferred: Record<number, Deferred<unknown>>;
+}
+export declare function unflatten(this: ThisDecode, parsed: unknown): unknown;

package/dist/decode.js

@@ -0,0 +1,91 @@
+import { HOLE, NAN, NEGATIVE_INFINITY, NEGATIVE_ZERO, POSITIVE_INFINITY, UNDEFINED, TYPE_BIGINT, TYPE_DATE, TYPE_MAP, TYPE_REGEXP, TYPE_SET, TYPE_SYMBOL, TYPE_PROMISE, } from "./constants.js";
+import { Deferred } from "./deferred.js";
+export function unflatten(parsed) {
+    if (typeof parsed === "number")
+        return hydrate.call(this, parsed, true);
+    if (!Array.isArray(parsed) || parsed.length === 0) {
+        throw new Error("Invalid input");
+    }
+    const startIndex = this.values.length;
+    this.values.push(...parsed);
+    this.hydrated.length = this.values.length;
+    return hydrate.call(this, startIndex);
+}
+function hydrate(index, standalone) {
+    if (index === UNDEFINED)
+        return undefined;
+    if (index === NAN)
+        return NaN;
+    if (index === POSITIVE_INFINITY)
+        return Infinity;
+    if (index === NEGATIVE_INFINITY)
+        return -Infinity;
+    if (index === NEGATIVE_ZERO)
+        return -0;
+    if (standalone)
+        throw new Error(`Invalid input`);
+    if (index in this.hydrated)
+        return this.hydrated[index];
+    const value = this.values[index];
+    if (!value || typeof value !== "object") {
+        this.hydrated[index] = value;
+    }
+    else if (Array.isArray(value)) {
+        if (typeof value[0] === "string") {
+            switch (value[0]) {
+                case TYPE_DATE:
+                    this.hydrated[index] = new Date(value[1]);
+                    break;
+                case TYPE_BIGINT:
+                    this.hydrated[index] = BigInt(value[1]);
+                    break;
+                case TYPE_REGEXP:
+                    this.hydrated[index] = new RegExp(value[1], value[2]);
+                    break;
+                case TYPE_SYMBOL:
+                    this.hydrated[index] = Symbol.for(value[1]);
+                    break;
+                case TYPE_SET:
+                    const set = new Set();
+                    this.hydrated[index] = set;
+                    for (let i = 1; i < value.length; i += 1) {
+                        set.add(hydrate.call(this, value[i]));
+                    }
+                    break;
+                case TYPE_MAP:
+                    const map = new Map();
+                    this.hydrated[index] = map;
+                    for (let i = 1; i < value.length; i += 2) {
+                        map.set(hydrate.call(this, value[i]), hydrate.call(this, value[i + 1]));
+                    }
+                    break;
+                case TYPE_PROMISE:
+                    const deferred = new Deferred();
+                    this.deferred[value[1]] = deferred;
+                    this.hydrated[index] = deferred.promise;
+                    break;
+                default:
+                    throw new Error(`Invalid input`);
+            }
+        }
+        else {
+            const array = new Array(value.length);
+            this.hydrated[index] = array;
+            for (let i = 0; i < value.length; i += 1) {
+                const n = value[i];
+                if (n === HOLE)
+                    continue;
+                array[i] = hydrate.call(this, n);
+            }
+        }
+    }
+    else {
+        const object = {};
+        this.hydrated[index] = object;
+        for (const key in value) {
+            const n = value[key];
+            object[key] = hydrate.call(this, n);
+        }
+    }
+    return this.hydrated[index];
+}

package/dist/deferred.d.ts

@@ -0,0 +1,6 @@
+export declare class Deferred<R> {
+    promise: Promise<R>;
+    resolve: (value: R | PromiseLike<R>) => void;
+    reject: (reason?: unknown) => void;
+    constructor();
+}

package/dist/deferred.js

@@ -0,0 +1,11 @@
+export class Deferred {
+    promise;
+    resolve;
+    reject;
+    constructor() {
+        this.promise = new Promise((resolve, reject) => {
+            this.resolve = resolve;
+            this.reject = reject;
+        });
+    }
+}

package/dist/encode.d.ts

@@ -0,0 +1,7 @@
+export interface ThisEncode {
+    index: number;
+    indicies: Map<unknown, number>;
+    stringified: string[];
+    deferred: [number, Promise<unknown>][];
+}
+export declare function flatten(this: ThisEncode, input: unknown): number;

package/dist/encode.js

@@ -0,0 +1,115 @@
+import { HOLE, NAN, NEGATIVE_INFINITY, NEGATIVE_ZERO, POSITIVE_INFINITY, UNDEFINED, TYPE_BIGINT, TYPE_DATE, TYPE_MAP, TYPE_REGEXP, TYPE_SET, TYPE_SYMBOL, TYPE_PROMISE, } from "./constants.js";
+export function flatten(input) {
+    if (this.indicies.has(input)) {
+        return this.indicies.get(input);
+    }
+    if (input === undefined)
+        return UNDEFINED;
+    if (Number.isNaN(input))
+        return NAN;
+    if (input === Infinity)
+        return POSITIVE_INFINITY;
+    if (input === -Infinity)
+        return NEGATIVE_INFINITY;
+    if (input === 0 && 1 / input < 0)
+        return NEGATIVE_ZERO;
+    const index = this.index++;
+    this.indicies.set(input, index);
+    stringify.call(this, input, index);
+    return index;
+}
+function stringify(input, index) {
+    switch (typeof input) {
+        case "boolean":
+        case "number":
+        case "string":
+            this.stringified[index] = JSON.stringify(input);
+            break;
+        case "bigint":
+            this.stringified[index] = `["${TYPE_BIGINT}","${input}"]`;
+            break;
+        case "symbol":
+            const keyFor = Symbol.keyFor(input);
+            if (!keyFor)
+                throw new Error("Cannot encode symbol unless created with Symbol.for()");
+            this.stringified[index] = `["${TYPE_SYMBOL}",${JSON.stringify(keyFor)}]`;
+            break;
+        case "object":
+            if (!input) {
+                this.stringified[index] = "null";
+                break;
+            }
+            if (Array.isArray(input)) {
+                let result = "[";
+                for (let i = 0; i < input.length; i++) {
+                    if (i > 0)
+                        result += ",";
+                    if (i in input) {
+                        result += flatten.call(this, input[i]);
+                    }
+                    else {
+                        result += HOLE;
+                    }
+                }
+                this.stringified[index] = result + "]";
+                break;
+            }
+            if (input instanceof Date) {
+                this.stringified[index] = `["${TYPE_DATE}",${input.getTime()}]`;
+                break;
+            }
+            if (input instanceof RegExp) {
+                this.stringified[index] = `["${TYPE_REGEXP}",${JSON.stringify(input.source)},${JSON.stringify(input.flags)}]`;
+                break;
+            }
+            if (input instanceof Set) {
+                let result = `["${TYPE_SET}"`;
+                for (const value of input) {
+                    result += "," + flatten.call(this, value);
+                }
+                this.stringified[index] = result + "]";
+                break;
+            }
+            if (input instanceof Map) {
+                let result = `["${TYPE_MAP}"`;
+                for (const [key, value] of input) {
+                    result += "," + flatten.call(this, key);
+                    result += "," + flatten.call(this, value);
+                }
+                this.stringified[index] = result + "]";
+                break;
+            }
+            if (input instanceof Promise) {
+                this.stringified[index] = `["${TYPE_PROMISE}",${index}]`;
+                this.deferred.push([index, input]);
+                break;
+            }
+            if (!isPlainObject(input)) {
+                console.log(input);
+                throw new Error("Cannot encode object with prototype");
+            }
+            let result = "{";
+            let sep = false;
+            for (const key in input) {
+                if (sep)
+                    result += ",";
+                sep = true;
+                result += JSON.stringify(key) + ":" + flatten.call(this, input[key]);
+            }
+            this.stringified[index] = result + "}";
+            break;
+        case "function":
+            throw new Error("Cannot encode function");
+        case "undefined":
+            throw new Error("This should never happen");
+    }
+}
+const objectProtoNames = Object.getOwnPropertyNames(Object.prototype)
+    .sort()
+    .join("\0");
+function isPlainObject(thing) {
+    const proto = Object.getPrototypeOf(thing);
+    return (proto === Object.prototype ||
+        proto === null ||
+        Object.getOwnPropertyNames(proto).sort().join("\0") === objectProtoNames);
+}

package/dist/turbo-stream.d.ts

@@ -0,0 +1,5 @@
+export declare function decode<T = unknown>(input: ReadableStream<Uint8Array>): Promise<{
+    value: T;
+    done: Promise<void>;
+}>;
+export declare function encode(input: unknown): ReadableStream<Uint8Array>;

package/dist/turbo-stream.js

@@ -0,0 +1,121 @@
+import { TYPE_PROMISE } from "./constants.js";
+import { unflatten } from "./decode.js";
+import { Deferred } from "./deferred.js";
+import { flatten } from "./encode.js";
+export async function decode(input) {
+    const decoder = new Decoder(input);
+    return decoder.decode();
+}
+class Decoder {
+    reader;
+    decoder;
+    constructor(input) {
+        this.reader = input.getReader();
+        this.decoder = {
+            deferred: {},
+            hydrated: [],
+            values: [],
+        };
+    }
+    async decode() {
+        const iterator = makeTextFileLineIterator(this.reader);
+        const read = await iterator.next();
+        if (!read.value || read.done)
+            throw new Error("Invalid input");
+        const decoded = unflatten.call(this.decoder, JSON.parse(read.value));
+        const done = (async () => {
+            for await (const line of iterator) {
+                let type = line[0];
+                switch (type) {
+                    case TYPE_PROMISE:
+                        const colonIndex = line.indexOf(":");
+                        const deferredId = Number(line.slice(1, colonIndex));
+                        const lineData = line.slice(colonIndex + 1);
+                        const deferredResult = unflatten.call(this.decoder, JSON.parse(lineData));
+                        this.decoder.deferred[deferredId].resolve(deferredResult);
+                        break;
+                    default:
+                        throw new Error("Invalid input");
+                }
+            }
+        })();
+        return { value: decoded, done };
+    }
+}
+export function encode(input) {
+    return new ReadableStream({
+        async start(controller) {
+            const textEncoder = new TextEncoder();
+            const encoder = {
+                index: 0,
+                indicies: new Map(),
+                stringified: [],
+                deferred: [],
+            };
+            const id = flatten.call(encoder, input);
+            const encoded = id < 0 ? String(id) : "[" + encoder.stringified.join(",") + "]";
+            controller.enqueue(textEncoder.encode(encoded + "\n"));
+            let activeDeferred = 0;
+            const done = new Deferred();
+            let alreadyDone = false;
+            if (encoder.deferred.length === 0) {
+                alreadyDone = true;
+                done.resolve();
+            }
+            else {
+                for (const [promiseId, promise] of encoder.deferred) {
+                    activeDeferred++;
+                    promise
+                        .then((value) => {
+                        const id = flatten.call(encoder, value);
+                        const encoded = id < 0
+                            ? String(id)
+                            : "[" + encoder.stringified.slice(id).join(",") + "]";
+                        controller.enqueue(textEncoder.encode(`${TYPE_PROMISE}${promiseId}:` + encoded + "\n"));
+                        activeDeferred--;
+                        if (activeDeferred === 0) {
+                            alreadyDone = true;
+                            done.resolve();
+                        }
+                    })
+                        .catch((reason) => {
+                        if (alreadyDone)
+                            return;
+                        alreadyDone = true;
+                        done.reject(reason);
+                    });
+                }
+            }
+            await done.promise;
+            controller.close();
+        },
+    });
+}
+async function* makeTextFileLineIterator(reader) {
+    const decoder = new TextDecoder();
+    let read = await reader.read();
+    let chunk = read.value ? decoder.decode(read.value, { stream: true }) : "";
+    let re = /\r\n|\n|\r/gm;
+    let startIndex = 0;
+    for (;;) {
+        let result = re.exec(chunk);
+        if (!result) {
+            if (read.done) {
+                break;
+            }
+            let remainder = chunk.slice(startIndex);
+            read = await reader.read();
+            chunk =
+                remainder +
+                    (read.value ? decoder.decode(read.value, { stream: true }) : "");
+            startIndex = re.lastIndex = 0;
+            continue;
+        }
+        yield chunk.substring(startIndex, result.index);
+        startIndex = re.lastIndex;
+    }
+    if (startIndex < chunk.length) {
+        // last line didn't end in a newline char
+        yield chunk.slice(startIndex);
+    }
+}

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "turbo-stream",
+  "version": "0.0.1",
+  "description": "A streaming data transport format that aims to support built-in features such as Promises, Dates, RegExps, Maps, Sets and more.",
+  "type": "module",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/turbo-stream.d.ts",
+      "default": "./dist/turbo-stream.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/node": "^20.8.7",
+    "expect": "^29.7.0",
+    "tsm": "^2.3.0",
+    "typescript": "^5.2.2"
+  },
+  "scripts": {
+    "build": "tsc --outDir dist --project tsconfig.lib.json",
+    "test": "node --no-warnings --loader tsm --enable-source-maps --test-reporter tap --test src/*.spec.ts",
+    "test-typecheck": "tsc --noEmit --project tsconfig.spec.json"
+  }
+}