typenative 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/node/assert.js +2 -2
- package/bin/node/child_process.js +66 -8
- package/bin/node/fs.js +1012 -4
- package/bin/node/histogram.js +1991 -0
- package/bin/node/process.js +14 -2
- package/bin/node/sqlite.js +1488 -5
- package/bin/node/stream.js +2 -1
- package/bin/node/system_extra.js +3 -270
- package/bin/node/test.js +39 -1
- package/bin/node/test_report.js +47 -1
- package/bin/node/test_run.js +57 -2
- package/bin/node/util.js +8 -5
- package/bin/node/webassembly.js +20 -5
- package/bin/node/webview.js +3257 -244
- package/bin/node/zlib.js +118 -14
- package/bin/node/zlib_iter.js +585 -0
- package/bin/transpiler.js +1370 -258
- package/bin/version.js +1 -1
- package/govendor/software.sslmate.com/src/go-pkcs12/.gitattributes +10 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/LICENSE +28 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/README.md +31 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/bmp-string.go +63 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/crypto.go +331 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/errors.go +23 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/go.mod +3 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/internal/rc2/rc2.go +271 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/mac.go +202 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/pbkdf.go +184 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/pkcs12.go +1000 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/safebags.go +107 -0
- package/package.json +10 -1
- package/types/tsconfig.json +17 -0
- package/types/typenative-npm.d.ts +492 -12
- package/types/typenative.d.ts +88 -2
- package/.gitattributes +0 -1
- package/.github/workflows/ci.yml +0 -68
- package/.github/workflows/npm-publish-github-packages.yml +0 -88
- package/CHANGELOG.md +0 -1746
- package/CONTRIBUTION.md +0 -133
- package/conformance.md +0 -1497
- package/examples/README.md +0 -114
- package/examples/angular-templates/README.md +0 -141
- package/examples/angular-templates/bench.mjs +0 -173
- package/examples/angular-templates/dom-type-globals.d.ts +0 -21
- package/examples/angular-templates/dom-type-stubs.ts +0 -21
- package/examples/angular-templates/main.ts +0 -135
- package/examples/angular-templates/parity.mjs +0 -157
- package/examples/angular-templates/run-node.mjs +0 -69
- package/examples/angular-templates/vendor/angular/compiler/LICENSE +0 -21
- package/examples/desktop-notes/README.md +0 -106
- package/examples/desktop-notes/bench.mjs +0 -125
- package/examples/desktop-notes/main.ts +0 -131
- package/examples/desktop-notes/package-lock.json +0 -726
- package/examples/desktop-notes/package.json +0 -9
- package/examples/desktop-notes/ui.ts +0 -176
- package/examples/dupe-finder/README.md +0 -78
- package/examples/dupe-finder/bench.mjs +0 -120
- package/examples/dupe-finder/main.ts +0 -87
- package/examples/fibonacci/README.md +0 -89
- package/examples/fibonacci/bench.mjs +0 -87
- package/examples/fibonacci/main.ts +0 -21
- package/examples/hello-world/README.md +0 -62
- package/examples/hello-world/bench.mjs +0 -66
- package/examples/hello-world/main.ts +0 -7
- package/examples/json-server/README.md +0 -132
- package/examples/json-server/bench.mjs +0 -263
- package/examples/json-server/main.ts +0 -67
- package/examples/mandelbrot-png/README.md +0 -63
- package/examples/mandelbrot-png/bench.mjs +0 -95
- package/examples/mandelbrot-png/main.ts +0 -98
- package/examples/mandelbrot-png/preview.png +0 -0
- package/examples/marked-cli/README.md +0 -174
- package/examples/marked-cli/bench.mjs +0 -178
- package/examples/marked-cli/main.ts +0 -410
- package/examples/marked-cli/parity.mjs +0 -185
- package/examples/marked-cli/vendor/marked/LICENSE +0 -44
- package/examples/marked-cli/vendor/marked/VENDOR.md +0 -26
- package/examples/openai-chat/README.md +0 -148
- package/examples/openai-chat/bench.mjs +0 -175
- package/examples/openai-chat/main.ts +0 -488
- package/examples/react-compiler/README.md +0 -185
- package/examples/react-compiler/bench.mjs +0 -181
- package/examples/react-compiler/main.ts +0 -324
- package/examples/react-compiler/parity.mjs +0 -362
- package/examples/react-compiler/run-node.mjs +0 -90
- package/examples/react-compiler/vendor/babel/LICENSE +0 -22
- package/examples/react-compiler/vendor/babel/helper-globals/data/builtin-lower.json +0 -15
- package/examples/react-compiler/vendor/babel/helper-globals/data/builtin-upper.json +0 -51
- package/examples/react-compiler/vendor/charcodes/package.json +0 -25
- package/examples/react-compiler/vendor/invariant/LICENSE +0 -21
- package/examples/react-compiler/vendor/invariant/invariant.cjs +0 -51
- package/examples/react-compiler/vendor/invariant/invariant.d.cts +0 -11
- package/examples/react-compiler/vendor/invariant/types/LICENSE +0 -21
- package/examples/react-compiler/vendor/jridgewell/gen-mapping/LICENSE +0 -19
- package/examples/react-compiler/vendor/jridgewell/resolve-uri/LICENSE +0 -19
- package/examples/react-compiler/vendor/jridgewell/sourcemap-codec/LICENSE +0 -19
- package/examples/react-compiler/vendor/jridgewell/trace-mapping/LICENSE +0 -19
- package/examples/react-compiler/vendor/js-tokens/LICENSE +0 -21
- package/examples/react-compiler/vendor/js-tokens/index.cjs +0 -23
- package/examples/react-compiler/vendor/js-tokens/index.d.cts +0 -13
- package/examples/react-compiler/vendor/jsesc/LICENSE-MIT.txt +0 -20
- package/examples/react-compiler/vendor/jsesc/jsesc.cjs +0 -337
- package/examples/react-compiler/vendor/jsesc/jsesc.d.cts +0 -384
- package/examples/react-compiler/vendor/jsesc/types/LICENSE +0 -21
- package/examples/react-compiler/vendor/picocolors/LICENSE +0 -15
- package/examples/react-compiler/vendor/picocolors/picocolors.cjs +0 -75
- package/examples/react-compiler/vendor/picocolors/picocolors.d.cts +0 -5
- package/examples/react-compiler/vendor/picocolors/types.d.ts +0 -51
- package/examples/react-compiler/vendor/react-compiler/LICENSE +0 -21
- package/examples/react-compiler/vendor/stubs/babel-core.ts +0 -31
- package/examples/react-compiler/vendor/stubs/debug.ts +0 -18
- package/examples/react-compiler/vendor/stubs/js-tokens.d.ts +0 -13
- package/examples/react-compiler/vendor/stubs/pretty-format.ts +0 -18
- package/examples/react-compiler/vendor/zod/LICENSE +0 -21
- package/examples/react-compiler/vendor/zod-validation-error/LICENSE +0 -9
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/NonEmptyArray.ts +0 -5
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/joinPath.ts +0 -49
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/joinValues.ts +0 -48
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/stringify.ts +0 -50
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/titleCase.ts +0 -6
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/MessageBuilder.ts +0 -126
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/ValidationError.ts +0 -39
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/custom.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/errorMap.ts +0 -78
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/index.ts +0 -2
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidElement.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidKey.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidStringFormat.ts +0 -131
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidType.ts +0 -54
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidUnion.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidValue.ts +0 -40
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/notMultipleOf.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/tooBig.ts +0 -76
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/tooSmall.ts +0 -76
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/types.ts +0 -27
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/unrecognizedKeys.ts +0 -27
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromError.ts +0 -13
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromZodError.ts +0 -61
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromZodIssue.ts +0 -38
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/index.ts +0 -20
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isValidationError.ts +0 -5
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isValidationErrorLike.ts +0 -8
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isZodErrorLike.ts +0 -11
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/toValidationError.ts +0 -20
- package/examples/react-compiler/vendor.mjs +0 -688
- package/examples/todo-cli/README.md +0 -69
- package/examples/todo-cli/bench.mjs +0 -89
- package/examples/todo-cli/main.ts +0 -92
- package/examples/yaml-cli/README.md +0 -152
- package/examples/yaml-cli/bench.mjs +0 -265
- package/examples/yaml-cli/main.ts +0 -251
- package/examples/yaml-cli/parity.mjs +0 -156
- package/examples/yaml-cli/vendor/yaml/LICENSE +0 -13
- package/examples/yaml-cli/vendor/yaml/VENDOR.md +0 -25
- package/performance/README.md +0 -248
- package/performance/any-dynamic.ts +0 -21
- package/performance/any-record-view.ts +0 -20
- package/performance/arr-for-of.ts +0 -29
- package/performance/arr-hof.ts +0 -21
- package/performance/arr-push-index.ts +0 -15
- package/performance/arr-push-local.ts +0 -39
- package/performance/arr-slice-concat.ts +0 -24
- package/performance/arr-sort.ts +0 -21
- package/performance/async-await.ts +0 -24
- package/performance/async-promise-all.ts +0 -38
- package/performance/bigint-arith.ts +0 -36
- package/performance/buf-encode.ts +0 -29
- package/performance/class-accessors.ts +0 -48
- package/performance/class-dispatch.ts +0 -52
- package/performance/date-format.ts +0 -18
- package/performance/events-emit.ts +0 -27
- package/performance/exc-throw-rare.ts +0 -25
- package/performance/exc-try-catch.ts +0 -26
- package/performance/fn-closures.ts +0 -29
- package/performance/fn-recursion-tree.ts +0 -31
- package/performance/fn-rest-spread.ts +0 -43
- package/performance/gen-for-of.ts +0 -46
- package/performance/gen-next.ts +0 -32
- package/performance/iter-helpers.ts +0 -38
- package/performance/json-any.ts +0 -30
- package/performance/json-roundtrip.ts +0 -27
- package/performance/map-delete.ts +0 -24
- package/performance/map-iterate.ts +0 -26
- package/performance/map-number-keys.ts +0 -37
- package/performance/map-string-keys.ts +0 -22
- package/performance/num-arith-loop.ts +0 -14
- package/performance/num-bitwise.ts +0 -21
- package/performance/num-hash-imul.ts +0 -36
- package/performance/num-mod-int.ts +0 -16
- package/performance/num-to-string.ts +0 -16
- package/performance/obj-alloc.ts +0 -32
- package/performance/obj-keys-entries.ts +0 -44
- package/performance/obj-optional-chain.ts +0 -42
- package/performance/obj-record-dict.ts +0 -24
- package/performance/obj-spread-destructure.ts +0 -40
- package/performance/regex-exec.ts +0 -29
- package/performance/regex-replace-fn.ts +0 -24
- package/performance/regex-site.ts +0 -28
- package/performance/set-ops.ts +0 -20
- package/performance/str-char-scan.ts +0 -35
- package/performance/str-concat-plus.ts +0 -18
- package/performance/str-concat.ts +0 -24
- package/performance/str-methods.ts +0 -24
- package/performance/str-short-scan.ts +0 -26
- package/performance/str-split-join.ts +0 -23
- package/performance/str-switch-compare.ts +0 -50
- package/performance/str-template.ts +0 -16
- package/performance/str-unicode.ts +0 -25
- package/performance/typed-array.ts +0 -30
- package/performance/union-switch.ts +0 -57
- package/performance/union-typeof.ts +0 -40
- package/performance/wasm-call.ts +0 -63
- package/tsparser/check.go +0 -158
- package/tsparser/go.mod +0 -14
- package/tsparser/go.sum +0 -22
- package/tsparser/main.go +0 -363
- package/tsparser/tsparser.exe +0 -4
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
# OpenAI-compatible chat CLI
|
|
2
|
-
|
|
3
|
-
A terminal chat client for any OpenAI-compatible API. Answers **stream into
|
|
4
|
-
the terminal token by token** while the model is still writing them. It works
|
|
5
|
-
with the free OpenRouter API out of the box — no payment, just a key from
|
|
6
|
-
[openrouter.ai/keys](https://openrouter.ai/keys) — and with OpenAI, Ollama,
|
|
7
|
-
LM Studio, vLLM, or any other server that speaks `/v1/chat/completions`.
|
|
8
|
-
|
|
9
|
-
Demonstrates: WHATWG `fetch` with a **streaming response body**
|
|
10
|
-
(`response.body.getReader()` read in a loop), hand-parsed server-sent events
|
|
11
|
-
(`data:` lines, comments, `[DONE]`), `TextDecoder` with `{ stream: true }`,
|
|
12
|
-
`AbortController` + `setTimeout` as a request timeout, `node:readline/promises`
|
|
13
|
-
driven by `for await`, `JSON.parse` on each chunk, `process.env` / `argv`
|
|
14
|
-
handling, and writing deltas to `process.stdout` the moment they arrive — all
|
|
15
|
-
in one dependency-free file that runs unchanged under Node and as a compiled
|
|
16
|
-
binary.
|
|
17
|
-
|
|
18
|
-
## Usage
|
|
19
|
-
|
|
20
|
-
```sh
|
|
21
|
-
openai-chat # interactive multi-turn REPL
|
|
22
|
-
openai-chat "Explain SSE in one sentence" # one question, streamed answer, exit
|
|
23
|
-
openai-chat --model google/gemma-4-31b-it:free
|
|
24
|
-
openai-chat --system "Answer in as few words as possible"
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
In the REPL each message sees the whole conversation. Slash commands:
|
|
28
|
-
`/exit` (or Ctrl+D) quits, `/clear` resets the conversation,
|
|
29
|
-
`/system <text>` sets or replaces the system prompt. Ctrl+C aborts the answer
|
|
30
|
-
in flight without killing the session.
|
|
31
|
-
|
|
32
|
-
| Option | Meaning | Default |
|
|
33
|
-
|--------|---------|---------|
|
|
34
|
-
| `--model <id>` | Model id | provider default (`google/gemma-4-31b-it:free` on OpenRouter, `gpt-4o-mini` on OpenAI) |
|
|
35
|
-
| `--base-url <url>` | API root | `https://openrouter.ai/api/v1`, or `https://api.openai.com/v1` when only `OPENAI_API_KEY` is set |
|
|
36
|
-
| `--api-key <key>` | API key | `OPENROUTER_API_KEY`, else `OPENAI_API_KEY` |
|
|
37
|
-
| `--system <prompt>` | System prompt | none |
|
|
38
|
-
| `--max-tokens <n>` | Cap on generated tokens | provider default |
|
|
39
|
-
|
|
40
|
-
Other OpenAI-compatible servers, no account needed:
|
|
41
|
-
|
|
42
|
-
```sh
|
|
43
|
-
ollama serve # then:
|
|
44
|
-
openai-chat --base-url http://127.0.0.1:11434/v1 --api-key ollama --model llama3.2
|
|
45
|
-
```
|
|
46
|
-
|
|
47
|
-
A one-shot run keeps stdout clean — only the answer goes to stdout, the
|
|
48
|
-
token/time stats go to stderr — so piping works:
|
|
49
|
-
|
|
50
|
-
```sh
|
|
51
|
-
openai-chat "write a haiku about go channels" | tee haiku.txt
|
|
52
|
-
```
|
|
53
|
-
|
|
54
|
-
## Run it
|
|
55
|
-
|
|
56
|
-
Get a free OpenRouter key, then:
|
|
57
|
-
|
|
58
|
-
```sh
|
|
59
|
-
# sh
|
|
60
|
-
export OPENROUTER_API_KEY=sk-or-...
|
|
61
|
-
node main.ts "hello!" # under Node.js 24+
|
|
62
|
-
```
|
|
63
|
-
|
|
64
|
-
```sh
|
|
65
|
-
# with TypeNative, build one binary and run it
|
|
66
|
-
npx typenative --source main.ts --output dist/openai-chat.exe --release
|
|
67
|
-
./dist/openai-chat.exe
|
|
68
|
-
```
|
|
69
|
-
|
|
70
|
-
From a checkout of this repository:
|
|
71
|
-
|
|
72
|
-
```sh
|
|
73
|
-
node bin/cli.js --source examples/openai-chat/main.ts --output dist/openai-chat.exe
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
## How the streaming works
|
|
77
|
-
|
|
78
|
-
`streamChat` POSTs the conversation with `stream: true` and
|
|
79
|
-
`stream_options: { include_usage: true }`. The response body is a
|
|
80
|
-
server-sent-event stream, read incrementally:
|
|
81
|
-
|
|
82
|
-
```ts
|
|
83
|
-
const reader = response.body.getReader();
|
|
84
|
-
const decoder = new TextDecoder();
|
|
85
|
-
while (!finished) {
|
|
86
|
-
const { value, done } = await reader.read();
|
|
87
|
-
if (done) break;
|
|
88
|
-
buffer += decoder.decode(value, { stream: true }); // don't split UTF-8 mid-character
|
|
89
|
-
// ... peel off complete lines, collect `data:` lines, dispatch on blank lines
|
|
90
|
-
}
|
|
91
|
-
```
|
|
92
|
-
|
|
93
|
-
One SSE event is one or more `data:` lines ended by a blank line; the line
|
|
94
|
-
`data: [DONE]` ends the stream, and `: OPENROUTER PROCESSING` comment lines
|
|
95
|
-
(the provider's keep-alive) are ignored. Each event's JSON carries a
|
|
96
|
-
`choices[0].delta.content` fragment, which is printed as it arrives. A
|
|
97
|
-
malformed frame is skipped, not fatal. Errors surface the provider's own
|
|
98
|
-
message (`HTTP 401: No auth credentials found`), connection failures get a
|
|
99
|
-
friendly hint, and every request is bounded by an `AbortController` timeout.
|
|
100
|
-
|
|
101
|
-
## Benchmark: Node.js vs TypeNative
|
|
102
|
-
|
|
103
|
-
Reproduce with `node bench.mjs [runs=30]`. A real API call spends its time
|
|
104
|
-
waiting on the model, which would hide any client-side difference, so the
|
|
105
|
-
bench stands up a local mock API (plain `node:http`) that answers
|
|
106
|
-
`POST /v1/chat/completions` with a deterministic 46-chunk event stream sent
|
|
107
|
-
with pauses between chunks — the same shape OpenRouter and OpenAI return.
|
|
108
|
-
First both runtimes answer the same question once and their stdout is
|
|
109
|
-
compared byte for byte (**parity: identical, and equal to the streamed
|
|
110
|
-
answer**). Then it times full process runs (spawn to exit) of the one-shot
|
|
111
|
-
client, which covers startup, the HTTP round trip, and parsing the stream as
|
|
112
|
-
it arrives.
|
|
113
|
-
|
|
114
|
-
**Environment:** Windows 11 Pro, AMD Ryzen 5 7600, Node v26.13.0, Go 1.27.4,
|
|
115
|
-
TypeNative 0.7.0, default (non-`--release`) build.
|
|
116
|
-
|
|
117
|
-
| Runtime | min | median | mean | max |
|
|
118
|
-
|------------|-----------|-----------|-----------|-----------|
|
|
119
|
-
| Node.js | 757.81 ms | 766.16 ms | 769.18 ms | 779.88 ms |
|
|
120
|
-
| TypeNative | 696.02 ms | 700.48 ms | 700.57 ms | 711.66 ms |
|
|
121
|
-
|
|
122
|
-
**TypeNative is 1.1x faster per answer** (median). The ratio looks small
|
|
123
|
-
because both runtimes spend most of each run waiting for the mock's paced
|
|
124
|
-
chunks — that part is identical by construction. The client-side share
|
|
125
|
-
(startup, connect, first byte) is where they differ: about 65 ms of the
|
|
126
|
-
median run, mostly Node's ~45 ms startup. For an interactive chat session
|
|
127
|
-
the numbers that matter are the first token and memory, and the binary starts
|
|
128
|
-
about 9x faster and holds a fraction of Node's idle memory.
|
|
129
|
-
|
|
130
|
-
## Notes for your own streaming clients
|
|
131
|
-
|
|
132
|
-
Two TypeScript patterns that look natural but hit transpiler sharp edges
|
|
133
|
-
while writing this example (both are avoided above; TypeNative 0.4.1):
|
|
134
|
-
|
|
135
|
-
- `continue`/`break` out of a loop from inside a `catch` block does not
|
|
136
|
-
compile (`continue is not in a loop`) — the catch is emitted as its own Go
|
|
137
|
-
function. Set a flag in the catch and act on it after.
|
|
138
|
-
- `return` inside a `catch` block sets the return value but **falls through**
|
|
139
|
-
to the statements after the try/catch, unlike Node. Assign a result
|
|
140
|
-
variable instead:
|
|
141
|
-
|
|
142
|
-
```ts
|
|
143
|
-
try { throw new Error('boom'); } catch { return 42; }
|
|
144
|
-
console.error('reached in Go, skipped in Node');
|
|
145
|
-
```
|
|
146
|
-
|
|
147
|
-
Streaming itself needs no special support: `fetch`, reader loops, async
|
|
148
|
-
iteration over readline, and `AbortController` all behave as under Node.
|
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
// One-shot latency benchmark: Node.js vs TypeNative for the chat client.
|
|
2
|
-
//
|
|
3
|
-
// A chat request spends most of its time waiting on the model, so this bench
|
|
4
|
-
// measures what a client actually adds: process startup, opening the HTTPS
|
|
5
|
-
// connection, and parsing the SSE stream as it arrives. A local mock server
|
|
6
|
-
// (plain node:http) stands in for the API: it answers POST
|
|
7
|
-
// /v1/chat/completions with a deterministic event stream of ~45 chunks sent
|
|
8
|
-
// with pauses between them, exactly the shape OpenRouter and OpenAI return.
|
|
9
|
-
//
|
|
10
|
-
// First both runtimes answer the same question once and their stdout is
|
|
11
|
-
// compared byte for byte (parity). Then N full process runs (spawn to exit)
|
|
12
|
-
// of the one-shot client are timed against the mock.
|
|
13
|
-
//
|
|
14
|
-
// Usage: node bench.mjs [runs=30]
|
|
15
|
-
|
|
16
|
-
import { spawn, spawnSync } from 'node:child_process';
|
|
17
|
-
import http from 'node:http';
|
|
18
|
-
import fs from 'node:fs';
|
|
19
|
-
import path from 'node:path';
|
|
20
|
-
import { fileURLToPath } from 'node:url';
|
|
21
|
-
|
|
22
|
-
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
23
|
-
const repoRoot = path.resolve(here, '..', '..');
|
|
24
|
-
const cli = path.join(repoRoot, 'bin', 'cli.js');
|
|
25
|
-
const runs = Number(process.argv[2] ?? '30') || 30;
|
|
26
|
-
const exeName = process.platform === 'win32' ? 'openai-chat.exe' : 'openai-chat';
|
|
27
|
-
const exePath = path.join(here, 'dist', exeName);
|
|
28
|
-
|
|
29
|
-
// The deterministic answer the mock streams, one chunk per word
|
|
30
|
-
const words = [
|
|
31
|
-
'Server-sent', 'events', 'are', 'just', 'a', 'long', 'HTTP', 'response:', 'the', 'server',
|
|
32
|
-
'keeps', 'the', 'connection', 'open', 'and', 'writes', '"data:"', 'lines', 'separated', 'by',
|
|
33
|
-
'blank', 'lines.', 'A', 'chat', 'client', 'reads', 'each', 'event', 'as', 'it', 'arrives,',
|
|
34
|
-
'pulls', 'the', 'text', 'delta', 'out', 'of', 'its', 'JSON,', 'and', 'prints', 'it', 'immediately.',
|
|
35
|
-
];
|
|
36
|
-
const expectedAnswer = words.join(' ');
|
|
37
|
-
const expectedUsage = { prompt_tokens: 11, completion_tokens: words.length, total_tokens: 11 + words.length };
|
|
38
|
-
|
|
39
|
-
const chunkJson = (delta, extra = {}) =>
|
|
40
|
-
JSON.stringify({
|
|
41
|
-
id: 'chatcmpl-bench',
|
|
42
|
-
object: 'chat.completion.chunk',
|
|
43
|
-
created: 1700000000,
|
|
44
|
-
model: 'bench-model',
|
|
45
|
-
choices: [{ index: 0, delta, finish_reason: null }],
|
|
46
|
-
...extra,
|
|
47
|
-
});
|
|
48
|
-
|
|
49
|
-
function startMockServer() {
|
|
50
|
-
const server = http.createServer((req, res) => {
|
|
51
|
-
if (req.method !== 'POST' || !req.url.endsWith('/chat/completions')) {
|
|
52
|
-
res.writeHead(404, { 'content-type': 'application/json' });
|
|
53
|
-
res.end(JSON.stringify({ error: { message: 'not found' } }));
|
|
54
|
-
return;
|
|
55
|
-
}
|
|
56
|
-
if (req.headers.authorization !== 'Bearer bench-key') {
|
|
57
|
-
res.writeHead(401, { 'content-type': 'application/json' });
|
|
58
|
-
res.end(JSON.stringify({ error: { message: 'Invalid API key' } }));
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
req.resume();
|
|
62
|
-
res.writeHead(200, { 'content-type': 'text/event-stream' });
|
|
63
|
-
// The keep-alive comment real providers send while the model warms up
|
|
64
|
-
res.write(': OPENROUTER PROCESSING\n\n');
|
|
65
|
-
const events = [
|
|
66
|
-
`data: ${chunkJson({ role: 'assistant' })}\n\n`,
|
|
67
|
-
...words.map((word, i) => `data: ${chunkJson({ content: word + (i < words.length - 1 ? ' ' : '') })}\n\n`),
|
|
68
|
-
`data: ${chunkJson({}, { choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: expectedUsage })}\n\n`,
|
|
69
|
-
'data: [DONE]\n\n',
|
|
70
|
-
];
|
|
71
|
-
let i = 0;
|
|
72
|
-
const tick = () => {
|
|
73
|
-
if (i < events.length) {
|
|
74
|
-
res.write(events[i++]);
|
|
75
|
-
setTimeout(tick, 4);
|
|
76
|
-
} else {
|
|
77
|
-
res.end();
|
|
78
|
-
}
|
|
79
|
-
};
|
|
80
|
-
tick();
|
|
81
|
-
});
|
|
82
|
-
return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server)));
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
function stats(samples) {
|
|
86
|
-
const sorted = [...samples].sort((a, b) => a - b);
|
|
87
|
-
const mean = samples.reduce((sum, x) => sum + x, 0) / samples.length;
|
|
88
|
-
return { min: sorted[0], median: sorted[Math.floor(sorted.length / 2)], mean, max: sorted[sorted.length - 1] };
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function fmt(ms) {
|
|
92
|
-
return `${ms.toFixed(2)} ms`.padStart(11);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// Async spawn on purpose: the mock API lives in this process's event loop,
|
|
96
|
-
// and spawnSync would block the loop so the server could never answer.
|
|
97
|
-
function oneShot(command, args) {
|
|
98
|
-
return new Promise((resolve) => {
|
|
99
|
-
const child = spawn(command, args, {
|
|
100
|
-
cwd: here,
|
|
101
|
-
stdio: ['ignore', 'pipe', 'pipe'],
|
|
102
|
-
env: { ...process.env, OPENROUTER_API_KEY: '', OPENAI_API_KEY: '' },
|
|
103
|
-
});
|
|
104
|
-
let stdout = '';
|
|
105
|
-
let stderr = '';
|
|
106
|
-
child.stdout.on('data', (chunk) => {
|
|
107
|
-
stdout += chunk;
|
|
108
|
-
});
|
|
109
|
-
child.stderr.on('data', (chunk) => {
|
|
110
|
-
stderr += chunk;
|
|
111
|
-
});
|
|
112
|
-
child.on('exit', (code) => resolve({ status: code, stdout, stderr }));
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// 1. Compile the client to a native binary.
|
|
117
|
-
fs.mkdirSync(path.join(here, 'dist'), { recursive: true });
|
|
118
|
-
console.log('Compiling main.ts to a native binary with TypeNative...');
|
|
119
|
-
const build = spawnSync(process.execPath, [cli, '--source', 'main.ts', '--output', path.join('dist', exeName)], {
|
|
120
|
-
cwd: here,
|
|
121
|
-
stdio: 'inherit',
|
|
122
|
-
});
|
|
123
|
-
if (build.status !== 0) process.exit(build.status ?? 1);
|
|
124
|
-
|
|
125
|
-
// 2. Start the mock API and check both runtimes produce identical output.
|
|
126
|
-
const server = await startMockServer();
|
|
127
|
-
const port = server.address().port;
|
|
128
|
-
const mockBase = `http://127.0.0.1:${port}/v1`;
|
|
129
|
-
const question = 'What are server-sent events?';
|
|
130
|
-
const args = ['--api-key', 'bench-key', '--base-url', mockBase, question];
|
|
131
|
-
|
|
132
|
-
try {
|
|
133
|
-
console.log('\nChecking parity (same question, byte-for-byte stdout)...');
|
|
134
|
-
const nodeRun = await oneShot(process.execPath, ['main.ts', ...args]);
|
|
135
|
-
const nativeRun = await oneShot(exePath, args);
|
|
136
|
-
for (const [name, run] of [['Node.js', nodeRun], ['TypeNative', nativeRun]]) {
|
|
137
|
-
if (run.status !== 0) {
|
|
138
|
-
console.error(`${name} run failed (exit ${run.status}):\n${run.stderr}`);
|
|
139
|
-
process.exit(1);
|
|
140
|
-
}
|
|
141
|
-
if (run.stdout !== `${expectedAnswer}\n`) {
|
|
142
|
-
console.error(`${name} stdout does not match the expected answer:\n"${run.stdout}"`);
|
|
143
|
-
process.exit(1);
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
console.log('parity: Node.js and TypeNative stdout identical, matches the streamed answer.');
|
|
147
|
-
|
|
148
|
-
// 3. Time full one-shot runs (spawn -> exit) under both runtimes.
|
|
149
|
-
console.log(`\nTiming ${runs} full one-shot runs of each runtime...\n`);
|
|
150
|
-
const timeRuns = async (command, argv) => {
|
|
151
|
-
const samples = [];
|
|
152
|
-
for (let i = 0; i < runs + 2; i++) {
|
|
153
|
-
const t0 = performance.now();
|
|
154
|
-
const result = await oneShot(command, argv);
|
|
155
|
-
const elapsed = performance.now() - t0;
|
|
156
|
-
if (result.status !== 0) {
|
|
157
|
-
console.error(`run failed: ${command} ${argv.join(' ')}\n${result.stderr}`);
|
|
158
|
-
process.exit(1);
|
|
159
|
-
}
|
|
160
|
-
if (i >= 2) samples.push(elapsed); // first two are warmup
|
|
161
|
-
}
|
|
162
|
-
return stats(samples);
|
|
163
|
-
};
|
|
164
|
-
const nodeStats = await timeRuns(process.execPath, ['main.ts', ...args]);
|
|
165
|
-
const nativeStats = await timeRuns(exePath, args);
|
|
166
|
-
|
|
167
|
-
console.log(`\`openai-chat "<question>"\` against a local SSE server, ${runs} runs (full process: spawn -> exit):\n`);
|
|
168
|
-
console.log('| Runtime | min | median | mean | max |');
|
|
169
|
-
console.log('|------------|-------------|-------------|-------------|-------------|');
|
|
170
|
-
console.log(`| Node.js |${fmt(nodeStats.min)} |${fmt(nodeStats.median)} |${fmt(nodeStats.mean)} |${fmt(nodeStats.max)} |`);
|
|
171
|
-
console.log(`| TypeNative |${fmt(nativeStats.min)} |${fmt(nativeStats.median)} |${fmt(nativeStats.mean)} |${fmt(nativeStats.max)} |`);
|
|
172
|
-
console.log(`\nTypeNative is ${(nodeStats.median / nativeStats.median).toFixed(1)}x faster per answer (median).\n`);
|
|
173
|
-
} finally {
|
|
174
|
-
server.close();
|
|
175
|
-
}
|