turbo-stream 0.0.1 → 0.0.2

package/README.md

@@ -24,3 +24,5 @@ console.log(decoded.value); // a Promise
 console.log(await decoded.value); // 42
 await decoded.done; // wait for the stream to finish
 ```
+
+Stackblitz: https://stackblitz.com/edit/stackblitz-starters-2wm7dh?file=index.js

package/dist/flatten.d.ts

@@ -0,0 +1,2 @@
+import { type ThisEncode } from "./utils.js";
+export declare function flatten(this: ThisEncode, input: unknown): number;

package/dist/{encode.js → flatten.js}

@@ -1,4 +1,4 @@
-import { HOLE, NAN, NEGATIVE_INFINITY, NEGATIVE_ZERO, POSITIVE_INFINITY, UNDEFINED, TYPE_BIGINT, TYPE_DATE, TYPE_MAP, TYPE_REGEXP, TYPE_SET, TYPE_SYMBOL, TYPE_PROMISE, } from "./constants.js";
+import { HOLE, NAN, NEGATIVE_INFINITY, NEGATIVE_ZERO, POSITIVE_INFINITY, TYPE_BIGINT, TYPE_DATE, TYPE_MAP, TYPE_PROMISE, TYPE_REGEXP, TYPE_SET, TYPE_SYMBOL, UNDEFINED, } from "./utils.js";
 export function flatten(input) {
     if (this.indicies.has(input)) {
         return this.indicies.get(input);
@@ -81,11 +81,10 @@ function stringify(input, index) {
             }
             if (input instanceof Promise) {
                 this.stringified[index] = `["${TYPE_PROMISE}",${index}]`;
-                this.deferred.push([index, input]);
+                this.deferred[index] = input;
                 break;
             }
             if (!isPlainObject(input)) {
-                console.log(input);
                 throw new Error("Cannot encode object with prototype");
             }
             let result = "{";

package/dist/turbo-stream.d.ts

@@ -1,5 +1,5 @@
-export declare function decode<T = unknown>(input: ReadableStream<Uint8Array>): Promise<{
-    value: T;
-    done: Promise<void>;
+export declare function decode(readable: ReadableStream<Uint8Array>): Promise<{
+    done: Promise<undefined>;
+    value: unknown;
 }>;
 export declare function encode(input: unknown): ReadableStream<Uint8Array>;

package/dist/turbo-stream.js

@@ -1,121 +1,144 @@
-import { TYPE_PROMISE } from "./constants.js";
-import { unflatten } from "./decode.js";
-import { Deferred } from "./deferred.js";
-import { flatten } from "./encode.js";
-export async function decode(input) {
-    const decoder = new Decoder(input);
-    return decoder.decode();
+import { flatten } from "./flatten.js";
+import { unflatten } from "./unflatten.js";
+import { createLineSplittingTransform, Deferred, TYPE_PROMISE, } from "./utils.js";
+export async function decode(readable) {
+    const done = new Deferred();
+    const reader = readable
+        .pipeThrough(createLineSplittingTransform())
+        .getReader();
+    const decoder = {
+        values: [],
+        hydrated: [],
+        deferred: {},
+    };
+    const decoded = await decodeInitial.call(decoder, reader);
+    let donePromise = done.promise;
+    if (decoded.done) {
+        done.resolve();
+    }
+    else {
+        donePromise = decodeDeferred
+            .call(decoder, reader)
+            .then(done.resolve)
+            .catch((reason) => {
+            for (const deferred of Object.values(decoder.deferred)) {
+                deferred.reject(reason);
+            }
+            done.reject(reason);
+        });
+    }
+    return {
+        done: donePromise.then(() => reader.closed),
+        value: decoded.value,
+    };
 }
-class Decoder {
-    reader;
-    decoder;
-    constructor(input) {
-        this.reader = input.getReader();
-        this.decoder = {
-            deferred: {},
-            hydrated: [],
-            values: [],
-        };
+class SyntaxError extends Error {
+    name = "SyntaxError";
+    constructor(message) {
+        super(message ?? `Invalid input`);
+    }
+}
+async function decodeInitial(reader) {
+    const read = await reader.read();
+    if (!read.value) {
+        throw new SyntaxError();
+    }
+    let line;
+    try {
+        line = JSON.parse(read.value);
+    }
+    catch (reason) {
+        throw new SyntaxError();
     }
-    async decode() {
-        const iterator = makeTextFileLineIterator(this.reader);
-        const read = await iterator.next();
-        if (!read.value || read.done)
-            throw new Error("Invalid input");
-        const decoded = unflatten.call(this.decoder, JSON.parse(read.value));
-        const done = (async () => {
-            for await (const line of iterator) {
-                let type = line[0];
-                switch (type) {
-                    case TYPE_PROMISE:
-                        const colonIndex = line.indexOf(":");
-                        const deferredId = Number(line.slice(1, colonIndex));
-                        const lineData = line.slice(colonIndex + 1);
-                        const deferredResult = unflatten.call(this.decoder, JSON.parse(lineData));
-                        this.decoder.deferred[deferredId].resolve(deferredResult);
-                        break;
-                    default:
-                        throw new Error("Invalid input");
+    return {
+        done: read.done,
+        value: unflatten.call(this, line),
+    };
+}
+async function decodeDeferred(reader) {
+    let read = await reader.read();
+    while (!read.done) {
+        if (!read.value)
+            continue;
+        const line = read.value;
+        switch (line[0]) {
+            case TYPE_PROMISE:
+                const colonIndex = line.indexOf(":");
+                const deferredId = Number(line.slice(1, colonIndex));
+                const deferred = this.deferred[deferredId];
+                if (!deferred) {
+                    throw new Error(`Deferred ID ${deferredId} not found in stream`);
                 }
-            }
-        })();
-        return { value: decoded, done };
+                const lineData = line.slice(colonIndex + 1);
+                let jsonLine;
+                try {
+                    jsonLine = JSON.parse(lineData);
+                }
+                catch (reason) {
+                    throw new SyntaxError();
+                }
+                const value = unflatten.call(this, jsonLine);
+                deferred.resolve(value);
+                break;
+            // case TYPE_PROMISE_ERROR:
+            //   // TODO: transport promise rejections
+            //   break;
+            default:
+                throw new SyntaxError();
+        }
+        read = await reader.read();
     }
 }
 export function encode(input) {
-    return new ReadableStream({
+    const encoder = {
+        deferred: {},
+        index: 0,
+        indicies: new Map(),
+        stringified: [],
+    };
+    const textEncoder = new TextEncoder();
+    let lastSentIndex = 0;
+    const readable = new ReadableStream({
         async start(controller) {
-            const textEncoder = new TextEncoder();
-            const encoder = {
-                index: 0,
-                indicies: new Map(),
-                stringified: [],
-                deferred: [],
-            };
-            const id = flatten.call(encoder, input);
-            const encoded = id < 0 ? String(id) : "[" + encoder.stringified.join(",") + "]";
-            controller.enqueue(textEncoder.encode(encoded + "\n"));
-            let activeDeferred = 0;
-            const done = new Deferred();
-            let alreadyDone = false;
-            if (encoder.deferred.length === 0) {
-                alreadyDone = true;
-                done.resolve();
+            const id = flatten.call(encoder, await input);
+            if (id < 0) {
+                controller.enqueue(textEncoder.encode(`${id}\n`));
             }
             else {
-                for (const [promiseId, promise] of encoder.deferred) {
-                    activeDeferred++;
-                    promise
-                        .then((value) => {
-                        const id = flatten.call(encoder, value);
-                        const encoded = id < 0
-                            ? String(id)
-                            : "[" + encoder.stringified.slice(id).join(",") + "]";
-                        controller.enqueue(textEncoder.encode(`${TYPE_PROMISE}${promiseId}:` + encoded + "\n"));
-                        activeDeferred--;
-                        if (activeDeferred === 0) {
-                            alreadyDone = true;
-                            done.resolve();
+                controller.enqueue(textEncoder.encode(`[${encoder.stringified.join(",")}]\n`));
+                lastSentIndex = encoder.stringified.length - 1;
+            }
+            const seenPromises = new WeakSet();
+            while (Object.keys(encoder.deferred).length > 0) {
+                for (const [deferredId, deferred] of Object.entries(encoder.deferred)) {
+                    if (seenPromises.has(deferred))
+                        continue;
+                    seenPromises.add((encoder.deferred[Number(deferredId)] = deferred
+                        .then((resolved) => {
+                        const id = flatten.call(encoder, resolved);
+                        if (id < 0) {
+                            controller.enqueue(textEncoder.encode(`${TYPE_PROMISE}${deferredId}:${id}\n`));
                         }
+                        else {
+                            const values = encoder.stringified
+                                .slice(lastSentIndex + 1)
+                                .join(",");
+                            controller.enqueue(textEncoder.encode(`${TYPE_PROMISE}${deferredId}:[${values}]\n`));
+                            lastSentIndex = encoder.stringified.length - 1;
+                        }
+                    }, (reason) => {
+                        // TODO: Encode and send errors
+                        throw reason;
                     })
-                        .catch((reason) => {
-                        if (alreadyDone)
-                            return;
-                        alreadyDone = true;
-                        done.reject(reason);
-                    });
+                        .finally(() => {
+                        delete encoder.deferred[Number(deferredId)];
+                    })));
                 }
+                await Promise.race(Object.values(encoder.deferred));
             }
-            await done.promise;
+            await Promise.all(Object.values(encoder.deferred));
             controller.close();
         },
     });
-}
-async function* makeTextFileLineIterator(reader) {
-    const decoder = new TextDecoder();
-    let read = await reader.read();
-    let chunk = read.value ? decoder.decode(read.value, { stream: true }) : "";
-    let re = /\r\n|\n|\r/gm;
-    let startIndex = 0;
-    for (;;) {
-        let result = re.exec(chunk);
-        if (!result) {
-            if (read.done) {
-                break;
-            }
-            let remainder = chunk.slice(startIndex);
-            read = await reader.read();
-            chunk =
-                remainder +
-                    (read.value ? decoder.decode(read.value, { stream: true }) : "");
-            startIndex = re.lastIndex = 0;
-            continue;
-        }
-        yield chunk.substring(startIndex, result.index);
-        startIndex = re.lastIndex;
-    }
-    if (startIndex < chunk.length) {
-        // last line didn't end in a newline char
-        yield chunk.slice(startIndex);
-    }
+    return readable;
 }

package/dist/unflatten.d.ts

@@ -0,0 +1,2 @@
+import { type ThisDecode } from "./utils.js";
+export declare function unflatten(this: ThisDecode, parsed: unknown): unknown;

package/dist/{decode.js → unflatten.js}

@@ -1,5 +1,4 @@
-import { HOLE, NAN, NEGATIVE_INFINITY, NEGATIVE_ZERO, POSITIVE_INFINITY, UNDEFINED, TYPE_BIGINT, TYPE_DATE, TYPE_MAP, TYPE_REGEXP, TYPE_SET, TYPE_SYMBOL, TYPE_PROMISE, } from "./constants.js";
-import { Deferred } from "./deferred.js";
+import { Deferred, HOLE, NAN, NEGATIVE_INFINITY, NEGATIVE_ZERO, POSITIVE_INFINITY, TYPE_BIGINT, TYPE_DATE, TYPE_MAP, TYPE_PROMISE, TYPE_REGEXP, TYPE_SET, TYPE_SYMBOL, UNDEFINED, } from "./utils.js";
 export function unflatten(parsed) {
     if (typeof parsed === "number")
         return hydrate.call(this, parsed, true);
@@ -60,9 +59,14 @@ function hydrate(index, standalone) {
                     }
                     break;
                 case TYPE_PROMISE:
-                    const deferred = new Deferred();
-                    this.deferred[value[1]] = deferred;
-                    this.hydrated[index] = deferred.promise;
+                    if (this.hydrated[value[1]]) {
+                        this.hydrated[index] = this.hydrated[value[1]];
+                    }
+                    else {
+                        const deferred = new Deferred();
+                        this.deferred[value[1]] = deferred;
+                        this.hydrated[index] = deferred.promise;
+                    }
                     break;
                 default:
                     throw new Error(`Invalid input`);

package/dist/{constants.d.ts → utils.d.ts}

@@ -12,3 +12,21 @@ export declare const TYPE_REGEXP = "R";
 export declare const TYPE_SYMBOL = "Y";
 export declare const TYPE_NULL_OBJECT = "N";
 export declare const TYPE_PROMISE = "P";
+export interface ThisDecode {
+    values: unknown[];
+    hydrated: unknown[];
+    deferred: Record<number, Deferred<unknown>>;
+}
+export interface ThisEncode {
+    index: number;
+    indicies: Map<unknown, number>;
+    stringified: string[];
+    deferred: Record<number, Promise<unknown>>;
+}
+export declare class Deferred<T = unknown> {
+    promise: Promise<T>;
+    resolve: (value: T) => void;
+    reject: (reason: unknown) => void;
+    constructor();
+}
+export declare function createLineSplittingTransform(): TransformStream<any, any>;

package/dist/utils.js

@@ -0,0 +1,46 @@
+export const UNDEFINED = -1;
+export const HOLE = -1;
+export const NAN = -2;
+export const POSITIVE_INFINITY = -3;
+export const NEGATIVE_INFINITY = -4;
+export const NEGATIVE_ZERO = -5;
+export const TYPE_BIGINT = "B";
+export const TYPE_DATE = "D";
+export const TYPE_MAP = "M";
+export const TYPE_SET = "S";
+export const TYPE_REGEXP = "R";
+export const TYPE_SYMBOL = "Y";
+export const TYPE_NULL_OBJECT = "N";
+export const TYPE_PROMISE = "P";
+export class Deferred {
+    promise;
+    resolve;
+    reject;
+    constructor() {
+        this.promise = new Promise((resolve, reject) => {
+            this.resolve = resolve;
+            this.reject = reject;
+        });
+    }
+}
+export function createLineSplittingTransform() {
+    let decoder = new TextDecoder();
+    let leftover = "";
+    return new TransformStream({
+        transform(chunk, controller) {
+            let str = decoder.decode(chunk, { stream: true });
+            let parts = (leftover + str).split("\n");
+            // The last part might be a partial line, so keep it for the next chunk.
+            leftover = parts.pop() || "";
+            for (const part of parts) {
+                controller.enqueue(part);
+            }
+        },
+        flush(controller) {
+            // If there's any leftover data, enqueue it before closing.
+            if (leftover) {
+                controller.enqueue(leftover);
+            }
+        },
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turbo-stream",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "A streaming data transport format that aims to support built-in features such as Promises, Dates, RegExps, Maps, Sets and more.",
   "type": "module",
   "files": [
@@ -14,6 +14,11 @@
     },
     "./package.json": "./package.json"
   },
+  "scripts": {
+    "build": "tsc --outDir dist --project tsconfig.lib.json",
+    "test": "node --no-warnings --loader tsm --enable-source-maps --test-reporter tap --test src/*.spec.ts",
+    "test-typecheck": "tsc --noEmit --project tsconfig.spec.json"
+  },
   "keywords": [],
   "author": "",
   "license": "ISC",
@@ -22,10 +27,5 @@
     "expect": "^29.7.0",
     "tsm": "^2.3.0",
     "typescript": "^5.2.2"
-  },
-  "scripts": {
-    "build": "tsc --outDir dist --project tsconfig.lib.json",
-    "test": "node --no-warnings --loader tsm --enable-source-maps --test-reporter tap --test src/*.spec.ts",
-    "test-typecheck": "tsc --noEmit --project tsconfig.spec.json"
   }
-}
+}

package/dist/constants.js

@@ -1,14 +0,0 @@
-export const UNDEFINED = -1;
-export const HOLE = -1;
-export const NAN = -2;
-export const POSITIVE_INFINITY = -3;
-export const NEGATIVE_INFINITY = -4;
-export const NEGATIVE_ZERO = -5;
-export const TYPE_BIGINT = "B";
-export const TYPE_DATE = "D";
-export const TYPE_MAP = "M";
-export const TYPE_SET = "S";
-export const TYPE_REGEXP = "R";
-export const TYPE_SYMBOL = "Y";
-export const TYPE_NULL_OBJECT = "N";
-export const TYPE_PROMISE = "P";

package/dist/decode.d.ts

@@ -1,7 +0,0 @@
-import { Deferred } from "./deferred.js";
-export interface ThisDecode {
-    values: unknown[];
-    hydrated: unknown[];
-    deferred: Record<number, Deferred<unknown>>;
-}
-export declare function unflatten(this: ThisDecode, parsed: unknown): unknown;

package/dist/deferred.d.ts

@@ -1,6 +0,0 @@
-export declare class Deferred<R> {
-    promise: Promise<R>;
-    resolve: (value: R | PromiseLike<R>) => void;
-    reject: (reason?: unknown) => void;
-    constructor();
-}

package/dist/deferred.js

@@ -1,11 +0,0 @@
-export class Deferred {
-    promise;
-    resolve;
-    reject;
-    constructor() {
-        this.promise = new Promise((resolve, reject) => {
-            this.resolve = resolve;
-            this.reject = reject;
-        });
-    }
-}

package/dist/encode.d.ts

@@ -1,7 +0,0 @@
-export interface ThisEncode {
-    index: number;
-    indicies: Map<unknown, number>;
-    stringified: string[];
-    deferred: [number, Promise<unknown>][];
-}
-export declare function flatten(this: ThisEncode, input: unknown): number;