typenative 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/.github/workflows/ci.yml +2 -0
  2. package/CHANGELOG.md +372 -0
  3. package/CONTRIBUTION.md +1 -1
  4. package/README.md +62 -22
  5. package/ROUND2-HANDOFF.md +200 -0
  6. package/bin/cli.js +2 -0
  7. package/bin/go/bridge.js +74 -14
  8. package/bin/go/convert.js +48 -0
  9. package/bin/go/metadata.js +13 -4
  10. package/bin/go/overrides.js +4 -6
  11. package/bin/go/stdlib.generated.js +2 -0
  12. package/bin/index.js +130 -2
  13. package/bin/main.js +2 -0
  14. package/bin/node/assert.js +2 -2
  15. package/bin/node/async_hooks.js +1 -1
  16. package/bin/node/buffer.js +16 -5
  17. package/bin/node/child_process.js +57 -9
  18. package/bin/node/cluster.js +44 -6
  19. package/bin/node/core.js +1 -1
  20. package/bin/node/crypto.js +205 -4
  21. package/bin/node/crypto_keys.js +21 -21
  22. package/bin/node/dgram.js +231 -7
  23. package/bin/node/diagnostics_channel.js +3 -3
  24. package/bin/node/dns.js +472 -9
  25. package/bin/node/errno_tables.js +445 -0
  26. package/bin/node/events.js +3 -3
  27. package/bin/node/fetch.js +2 -2
  28. package/bin/node/fs.js +24 -17
  29. package/bin/node/fs_watch.js +5 -5
  30. package/bin/node/http.js +131 -33
  31. package/bin/node/http2.js +7 -4
  32. package/bin/node/module_v8.js +142 -0
  33. package/bin/node/net.js +17 -14
  34. package/bin/node/path.js +17 -3
  35. package/bin/node/process.js +274 -9
  36. package/bin/node/readline.js +272 -1
  37. package/bin/node/registry.js +5 -1
  38. package/bin/node/sqlite.js +65 -17
  39. package/bin/node/stream.js +220 -22
  40. package/bin/node/stream_web.js +10 -10
  41. package/bin/node/system_extra.js +3 -3
  42. package/bin/node/test.js +834 -234
  43. package/bin/node/test_coverage.js +187 -0
  44. package/bin/node/test_report.js +1392 -0
  45. package/bin/node/test_run.js +575 -0
  46. package/bin/node/timers.js +4 -4
  47. package/bin/node/tls.js +5 -5
  48. package/bin/node/url.js +23 -5
  49. package/bin/node/util.js +49 -8
  50. package/bin/node/util_extra.js +13 -37
  51. package/bin/node/webview.js +654 -0
  52. package/bin/node/worker_threads.js +80 -20
  53. package/bin/node/zlib.js +3 -3
  54. package/bin/parse-node.js +4 -0
  55. package/bin/transpiler.js +14457 -1723
  56. package/bin/typecheck-node.js +1 -0
  57. package/bin/version.js +1 -1
  58. package/conformance.md +1153 -884
  59. package/examples/README.md +47 -10
  60. package/examples/desktop-notes/README.md +108 -0
  61. package/examples/desktop-notes/bench.mjs +125 -0
  62. package/examples/desktop-notes/main.ts +131 -0
  63. package/examples/desktop-notes/package-lock.json +726 -0
  64. package/examples/desktop-notes/package.json +9 -0
  65. package/examples/desktop-notes/ui.ts +176 -0
  66. package/examples/dupe-finder/README.md +76 -0
  67. package/examples/dupe-finder/bench.mjs +120 -0
  68. package/examples/dupe-finder/main.ts +87 -0
  69. package/examples/json-server/README.md +31 -29
  70. package/examples/json-server/bench.mjs +130 -38
  71. package/examples/mandelbrot-png/README.md +65 -0
  72. package/examples/mandelbrot-png/bench.mjs +95 -0
  73. package/examples/mandelbrot-png/main.ts +98 -0
  74. package/examples/mandelbrot-png/preview.png +0 -0
  75. package/examples/marked-cli/README.md +159 -0
  76. package/examples/marked-cli/bench.mjs +178 -0
  77. package/examples/marked-cli/main.ts +292 -0
  78. package/examples/marked-cli/parity.mjs +185 -0
  79. package/examples/marked-cli/vendor/marked/LICENSE +44 -0
  80. package/examples/marked-cli/vendor/marked/VENDOR.md +26 -0
  81. package/examples/todo-cli/README.md +69 -0
  82. package/examples/todo-cli/bench.mjs +89 -0
  83. package/examples/todo-cli/main.ts +92 -0
  84. package/examples/yaml-cli/README.md +136 -0
  85. package/examples/yaml-cli/bench.mjs +265 -0
  86. package/examples/yaml-cli/main.ts +29 -0
  87. package/examples/yaml-cli/parity.mjs +156 -0
  88. package/examples/yaml-cli/vendor/yaml/LICENSE +13 -0
  89. package/examples/yaml-cli/vendor/yaml/VENDOR.md +25 -0
  90. package/govendor/github.com/jchv/go-webview2/LICENSE +22 -0
  91. package/govendor/github.com/jchv/go-webview2/README.md +21 -0
  92. package/govendor/github.com/jchv/go-webview2/common.go +84 -0
  93. package/govendor/github.com/jchv/go-webview2/go.mod +3 -0
  94. package/govendor/github.com/jchv/go-webview2/internal/w32/w32.go +194 -0
  95. package/govendor/github.com/jchv/go-webview2/internal/w32/w32_386.go +14 -0
  96. package/govendor/github.com/jchv/go-webview2/internal/w32/w32_64bit.go +15 -0
  97. package/govendor/github.com/jchv/go-webview2/pkg/edge/COREWEBVIEW2_COLOR.go +8 -0
  98. package/govendor/github.com/jchv/go-webview2/pkg/edge/COREWEBVIEW2_HOST_RESOURCE_ACCESS_KIND.go +9 -0
  99. package/govendor/github.com/jchv/go-webview2/pkg/edge/COREWEBVIEW2_KEY_EVENT_KIND.go +10 -0
  100. package/govendor/github.com/jchv/go-webview2/pkg/edge/COREWEBVIEW2_MOVE_FOCUS_REASON.go +9 -0
  101. package/govendor/github.com/jchv/go-webview2/pkg/edge/COREWEBVIEW2_PHYSICAL_KEY_STATUS.go +10 -0
  102. package/govendor/github.com/jchv/go-webview2/pkg/edge/COREWEBVIEW2_WEB_RESOURCE_CONTEXT.go +23 -0
  103. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2AcceleratorKeyPressedEventArgs.go +78 -0
  104. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2AcceleratorKeyPressedEventHandler.go +52 -0
  105. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2Controller.go +132 -0
  106. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2Controller2.go +74 -0
  107. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2CreateCoreWebView2ControllerCompletedHandler.go +48 -0
  108. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2NavigationCompletedEventArgs.go +17 -0
  109. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2NavigationCompletedEventHandler.go +48 -0
  110. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2Settings.go +270 -0
  111. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2WebResourceRequest.go +43 -0
  112. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2WebResourceRequestedEventArgs.go +51 -0
  113. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2WebResourceRequestedEventHandler.go +48 -0
  114. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2WebResourceResponse.go +21 -0
  115. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2_2.go +21 -0
  116. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebView2_3.go +60 -0
  117. package/govendor/github.com/jchv/go-webview2/pkg/edge/ICoreWebViewSettings.go +396 -0
  118. package/govendor/github.com/jchv/go-webview2/pkg/edge/chromium.go +361 -0
  119. package/govendor/github.com/jchv/go-webview2/pkg/edge/chromium_386.go +24 -0
  120. package/govendor/github.com/jchv/go-webview2/pkg/edge/chromium_amd64.go +22 -0
  121. package/govendor/github.com/jchv/go-webview2/pkg/edge/chromium_arm64.go +26 -0
  122. package/govendor/github.com/jchv/go-webview2/pkg/edge/comproc.go +13 -0
  123. package/govendor/github.com/jchv/go-webview2/pkg/edge/comproc_go17.go +51 -0
  124. package/govendor/github.com/jchv/go-webview2/pkg/edge/comproc_go18.go +16 -0
  125. package/govendor/github.com/jchv/go-webview2/pkg/edge/corewebview2.go +411 -0
  126. package/govendor/github.com/jchv/go-webview2/pkg/edge/guid.go +223 -0
  127. package/govendor/github.com/jchv/go-webview2/webview.go +482 -0
  128. package/govendor/github.com/jchv/go-webview2/webviewloader/README.md +3 -0
  129. package/govendor/github.com/jchv/go-webview2/webviewloader/module.go +153 -0
  130. package/govendor/github.com/jchv/go-webview2/webviewloader/module_386.go +6 -0
  131. package/govendor/github.com/jchv/go-webview2/webviewloader/module_amd64.go +6 -0
  132. package/govendor/github.com/jchv/go-webview2/webviewloader/module_arm64.go +6 -0
  133. package/govendor/github.com/jchv/go-webview2/webviewloader/sdk/arm64/WebView2Loader.dll +0 -0
  134. package/govendor/github.com/jchv/go-webview2/webviewloader/sdk/x64/WebView2Loader.dll +0 -0
  135. package/govendor/github.com/jchv/go-webview2/webviewloader/sdk/x86/WebView2Loader.dll +0 -0
  136. package/govendor/github.com/jchv/go-winloader/LICENSE.md +7 -0
  137. package/govendor/github.com/jchv/go-winloader/README.md +124 -0
  138. package/govendor/github.com/jchv/go-winloader/common.go +11 -0
  139. package/govendor/github.com/jchv/go-winloader/go.mod +3 -0
  140. package/govendor/github.com/jchv/go-winloader/internal/loader/loader.go +25 -0
  141. package/govendor/github.com/jchv/go-winloader/internal/loader/machine.go +53 -0
  142. package/govendor/github.com/jchv/go-winloader/internal/memloader/cache.go +42 -0
  143. package/govendor/github.com/jchv/go-winloader/internal/memloader/loader.go +253 -0
  144. package/govendor/github.com/jchv/go-winloader/internal/pe/export.go +76 -0
  145. package/govendor/github.com/jchv/go-winloader/internal/pe/format.go +522 -0
  146. package/govendor/github.com/jchv/go-winloader/internal/pe/import.go +106 -0
  147. package/govendor/github.com/jchv/go-winloader/internal/pe/module.go +91 -0
  148. package/govendor/github.com/jchv/go-winloader/internal/pe/reloc.go +177 -0
  149. package/govendor/github.com/jchv/go-winloader/internal/pe/util.go +40 -0
  150. package/govendor/github.com/jchv/go-winloader/internal/vmem/common.go +55 -0
  151. package/govendor/github.com/jchv/go-winloader/internal/vmem/memory_other.go +71 -0
  152. package/govendor/github.com/jchv/go-winloader/internal/vmem/memory_windows.go +179 -0
  153. package/govendor/github.com/jchv/go-winloader/internal/winloader/hacks_other.go +17 -0
  154. package/govendor/github.com/jchv/go-winloader/internal/winloader/hacks_windows.go +73 -0
  155. package/govendor/github.com/jchv/go-winloader/internal/winloader/loader_windows.go +64 -0
  156. package/govendor/github.com/jchv/go-winloader/internal/winloader/machine_windows.go +32 -0
  157. package/govendor/github.com/jchv/go-winloader/internal/winloader/nativearch_386.go +9 -0
  158. package/govendor/github.com/jchv/go-winloader/internal/winloader/nativearch_amd64.go +9 -0
  159. package/govendor/github.com/jchv/go-winloader/internal/winloader/nativearch_arm64.go +9 -0
  160. package/govendor/github.com/jchv/go-winloader/loader_other.go +24 -0
  161. package/govendor/github.com/jchv/go-winloader/loader_windows.go +33 -0
  162. package/govendor/manifest.txt +3 -1
  163. package/package.json +1 -1
  164. package/tsparser/main.go +4 -0
  165. package/types/typenative-go.d.ts +23 -0
  166. package/types/typenative-npm.d.ts +359 -7
  167. package/types/typenative.d.ts +283 -42
@@ -0,0 +1,178 @@
1
+ // Benchmark: Node.js vs TypeNative for the marked CLI, both rendering
2
+ // markdown to HTML (`marked < in.md`).
3
+ //
4
+ // Compiles main.ts to a native binary with the TypeNative CLI from this repo
5
+ // checkout (../../bin/cli.js), then measures:
6
+ // (a) per-call latency: complete process runs (spawn to exit) on marked's
7
+ // own README.md, the kind of file a docs build or a hook renders once;
8
+ // (b) throughput: one run on about 10 MB of generated markdown (headings,
9
+ // paragraphs with emphasis, links and code spans, lists, tables, code
10
+ // fences and block quotes).
11
+ // Runs alternate Node and native, so drift on the machine hits both alike.
12
+ // Both runtimes' HTML must be identical, or the benchmark stops.
13
+ //
14
+ // Usage: node bench.mjs [runs=30] [throughputRuns=5] [megabytes=10]
15
+
16
+ import { spawnSync } from 'node:child_process';
17
+ import fs from 'node:fs';
18
+ import path from 'node:path';
19
+ import { fileURLToPath } from 'node:url';
20
+
21
+ const here = path.dirname(fileURLToPath(import.meta.url));
22
+ const repoRoot = path.resolve(here, '..', '..');
23
+ const cli = path.join(repoRoot, 'bin', 'cli.js');
24
+ const cache = path.join(here, '.cache');
25
+ const upstream = path.join(cache, 'marked-src');
26
+ const exePath = path.join(cache, process.platform === 'win32' ? 'marked.exe' : 'marked');
27
+ const runs = Number(process.argv[2] ?? '30') || 30;
28
+ const throughputRuns = Number(process.argv[3] ?? '5') || 5;
29
+ const megabytes = Number(process.argv[4] ?? '10') || 10;
30
+
31
+ // marked's README.md at the vendored tag (parity.mjs fetches the same checkout)
32
+ function readme() {
33
+ const file = path.join(upstream, 'README.md');
34
+ if (!fs.existsSync(file)) {
35
+ fs.mkdirSync(cache, { recursive: true });
36
+ console.log('Fetching marked v18.1.2 ...');
37
+ const clone = spawnSync('git', ['clone', '--quiet', '--depth', '1', '--branch', 'v18.1.2', '--filter=blob:none', '--sparse', 'https://github.com/markedjs/marked.git', upstream], { stdio: 'inherit' });
38
+ if (clone.status !== 0) process.exit(1);
39
+ spawnSync('git', ['-C', upstream, 'sparse-checkout', 'set', 'test/specs'], { stdio: 'inherit' });
40
+ }
41
+ return fs.readFileSync(file);
42
+ }
43
+
44
+ // Deterministic pseudo-random numbers (a fixed-seed LCG): the same document
45
+ // on every run and every machine
46
+ let seed = 42;
47
+ function rand(n) {
48
+ seed = (seed * 1103515245 + 12345) % 2147483648;
49
+ return Math.floor(seed / 65536) % n;
50
+ }
51
+
52
+ const WORDS = ['markdown', 'parser', 'native', 'binary', 'token', 'lexer', 'render', 'inline', 'block', 'heading',
53
+ 'paragraph', 'emphasis', 'table', 'fence', 'quote', 'list', 'link', 'image', 'escape', 'entity', 'spec', 'html'];
54
+
55
+ function word() {
56
+ return WORDS[rand(WORDS.length)];
57
+ }
58
+
59
+ function sentence() {
60
+ const parts = [];
61
+ const n = 6 + rand(14);
62
+ for (let i = 0; i < n; i++) {
63
+ const w = word();
64
+ switch (rand(12)) {
65
+ case 0: parts.push(`*${w}*`); break;
66
+ case 1: parts.push(`**${w}**`); break;
67
+ case 2: parts.push(`\`${w}()\``); break;
68
+ case 3: parts.push(`[${w}](https://example.com/${w}/${rand(1000)} "${w}")`); break;
69
+ case 4: parts.push(`_${w}_`); break;
70
+ default: parts.push(w);
71
+ }
72
+ }
73
+ return parts.join(' ') + '.';
74
+ }
75
+
76
+ function section(i) {
77
+ const lines = [`${'#'.repeat(1 + rand(3))} ${word()} ${word()} ${i}`, ''];
78
+ for (let p = 0; p < 2 + rand(3); p++) lines.push(`${sentence()} ${sentence()}\n${sentence()}`, '');
79
+ switch (rand(5)) {
80
+ case 0:
81
+ for (let k = 0; k < 3 + rand(5); k++) lines.push(`- ${sentence()}`);
82
+ lines.push('');
83
+ break;
84
+ case 1:
85
+ for (let k = 1; k <= 3 + rand(4); k++) lines.push(`${k}. ${sentence()}`);
86
+ lines.push('');
87
+ break;
88
+ case 2:
89
+ lines.push(`| ${word()} | ${word()} | ${word()} |`, '| --- | :---: | ---: |');
90
+ for (let k = 0; k < 3 + rand(6); k++) lines.push(`| ${word()} | ${rand(1000)} | *${word()}* |`);
91
+ lines.push('');
92
+ break;
93
+ case 3:
94
+ lines.push('```js', `function ${word()}(${word()}) {`, ` return ${rand(100)} < 3 && "${word()}";`, '}', '```', '');
95
+ break;
96
+ default:
97
+ lines.push(`> ${sentence()}`, `> ${sentence()}`, '');
98
+ }
99
+ return lines.join('\n');
100
+ }
101
+
102
+ function document(bytes) {
103
+ const out = [];
104
+ let size = 0;
105
+ for (let i = 0; size < bytes; i++) {
106
+ const s = section(i);
107
+ out.push(s);
108
+ size += s.length + 1;
109
+ }
110
+ return out.join('\n');
111
+ }
112
+
113
+ function stats(samples) {
114
+ const sorted = [...samples].sort((a, b) => a - b);
115
+ const mean = samples.reduce((sum, x) => sum + x, 0) / samples.length;
116
+ return { min: sorted[0], median: sorted[Math.floor(sorted.length / 2)], mean, max: sorted[sorted.length - 1] };
117
+ }
118
+
119
+ function fmt(ms) {
120
+ return `${ms.toFixed(2)} ms`.padStart(13);
121
+ }
122
+
123
+ function runOnce(command, args, input) {
124
+ const t0 = performance.now();
125
+ const result = spawnSync(command, args, { cwd: here, input, maxBuffer: 1 << 30 });
126
+ const elapsed = performance.now() - t0;
127
+ if (result.status !== 0) {
128
+ console.error(`run failed: ${command} ${args.join(' ')}\n${result.stderr}`);
129
+ process.exit(1);
130
+ }
131
+ return { elapsed, stdout: result.stdout };
132
+ }
133
+
134
+ // Alternating Node and native runs; the two outputs must match
135
+ function timeBoth(input, count) {
136
+ const node = [];
137
+ const native = [];
138
+ for (let i = 0; i < count; i++) {
139
+ const a = runOnce(process.execPath, ['main.ts'], input);
140
+ const b = runOnce(exePath, [], input);
141
+ if (!a.stdout.equals(b.stdout)) {
142
+ console.error('Node and native HTML differ');
143
+ process.exit(1);
144
+ }
145
+ node.push(a.elapsed);
146
+ native.push(b.elapsed);
147
+ }
148
+ return [stats(node), stats(native)];
149
+ }
150
+
151
+ function table(nodeStats, nativeStats, extra = () => '') {
152
+ console.log(`| Runtime | min | median | mean | max |${extra('head')}`);
153
+ console.log(`|------------|--------------|--------------|--------------|--------------|${extra('rule')}`);
154
+ console.log(`| Node.js |${fmt(nodeStats.min)} |${fmt(nodeStats.median)} |${fmt(nodeStats.mean)} |${fmt(nodeStats.max)} |${extra(nodeStats)}`);
155
+ console.log(`| TypeNative |${fmt(nativeStats.min)} |${fmt(nativeStats.median)} |${fmt(nativeStats.mean)} |${fmt(nativeStats.max)} |${extra(nativeStats)}`);
156
+ }
157
+
158
+ // 1. Compile the CLI to a native binary.
159
+ fs.mkdirSync(cache, { recursive: true });
160
+ const small = readme();
161
+ console.log('Compiling main.ts to a native binary with TypeNative...');
162
+ const build = spawnSync(process.execPath, [cli, '--source', 'main.ts', '--output', exePath], { cwd: here, stdio: 'inherit' });
163
+ if (build.status !== 0) process.exit(build.status ?? 1);
164
+
165
+ // 2. Per-call latency on marked's README.md.
166
+ console.log(`\n(a) \`marked < README.md\` (${small.length} bytes), ${runs} runs each (full process: spawn -> exit):\n`);
167
+ const [nodeCall, nativeCall] = timeBoth(small, runs);
168
+ table(nodeCall, nativeCall);
169
+ console.log(`\nTypeNative is ${(nodeCall.median / nativeCall.median).toFixed(1)}x faster per call (median).`);
170
+
171
+ // 3. Throughput on generated markdown.
172
+ const big = Buffer.from(document(megabytes * 1024 * 1024));
173
+ const mb = big.length / (1024 * 1024);
174
+ console.log(`\n(b) \`marked < generated.md\` (${mb.toFixed(1)} MB, ${big.toString().split('\n').length} lines), ${throughputRuns} runs each:\n`);
175
+ const [nodeBig, nativeBig] = timeBoth(big, throughputRuns);
176
+ table(nodeBig, nativeBig, (s) => (s === 'head' ? ' MB/s (median) |' : s === 'rule' ? '---------------|' : ` ${(mb / (s.median / 1000)).toFixed(2).padStart(13)} |`));
177
+ const ratio = nodeBig.median / nativeBig.median;
178
+ console.log(`\nTypeNative is ${ratio >= 1 ? `${ratio.toFixed(1)}x faster` : `${(1 / ratio).toFixed(1)}x slower`} on throughput (median).`);
@@ -0,0 +1,292 @@
1
+ // The `marked` package's own command-line tool, compiled from its unmodified
2
+ // upstream TypeScript (vendor/marked/src, see vendor/marked/VENDOR.md). This
3
+ // file is a TypeScript port of upstream's bin/main.js: it parses the same
4
+ // options, reads markdown from a string, a file or stdin, and prints the HTML
5
+ // (or the lexer's tokens as JSON).
6
+ //
7
+ // marked < in.md render markdown to HTML
8
+ // marked -s '*hi*' render a string
9
+ // marked -i in.md -o out.html read a file, write a file
10
+ // marked --tokens < in.md print the token list as JSON
11
+ // marked --no-gfm --pedantic any boolean option of marked.defaults
12
+ //
13
+ // Runs as-is under Node.js (`node main.ts`) and compiles with TypeNative.
14
+ //
15
+ // Differences from bin/main.js: --help prints the option summary below
16
+ // instead of opening the man page, --version prints the vendored version
17
+ // instead of reading package.json, a config file (-c, ~/.marked.json) must be
18
+ // JSON (a JavaScript config is a module Node would import and run, which a
19
+ // native binary can't), and the HTML is flushed before the process exits
20
+ // (bin/main.js exits right after writing it, cutting output to a pipe off at
21
+ // 64 KB).
22
+ import { promises } from 'node:fs';
23
+ import { resolve } from 'node:path';
24
+ import { homedir } from 'node:os';
25
+ import { marked } from './vendor/marked/src/marked.ts';
26
+ import type { MarkedExtension, MarkedOptions } from './vendor/marked/src/marked.ts';
27
+
28
+ const { access, readFile, writeFile } = promises;
29
+
30
+ const VERSION = '18.1.2';
31
+
32
+ const HELP = `marked - a markdown parser
33
+
34
+ Usage: marked [options] [file]
35
+
36
+ -o, --output <file> write the HTML to <file> instead of stdout
37
+ -i, --input <file> read the markdown from <file>
38
+ -s, --string <string> parse <string> as markdown
39
+ -t, --tokens print the token list as JSON instead of HTML
40
+ -c, --config <file> load options and extensions from a JSON file
41
+ -n, --no-clobber don't overwrite the --output file
42
+ -h, --help print this help
43
+ -v, --version print the version
44
+
45
+ --<option> / --no-<option>
46
+ set or clear any option of marked.defaults,
47
+ e.g. --gfm, --no-gfm, --breaks, --pedantic
48
+
49
+ With no file, --input or --string, the markdown is read from stdin.
50
+ `;
51
+
52
+ interface CliError {
53
+ code?: string;
54
+ path?: string;
55
+ message: string;
56
+ }
57
+
58
+ export async function main() {
59
+ async function help() {
60
+ console.log(HELP);
61
+ }
62
+
63
+ async function version() {
64
+ console.log(VERSION);
65
+ }
66
+
67
+ async function start(argv: string[]) {
68
+ const files: string[] = [];
69
+ const options: Record<string, unknown> = {};
70
+ let input: string | undefined;
71
+ let output: string | undefined;
72
+ let string: string | undefined;
73
+ let arg: string;
74
+ let tokens = false;
75
+ let config: string | undefined;
76
+ let opt: string;
77
+ let noclobber = false;
78
+
79
+ function getArg(): string {
80
+ let arg = argv.shift() as string;
81
+
82
+ if (arg.indexOf('--') === 0) {
83
+ // e.g. --opt
84
+ const parts = arg.split('=');
85
+ if (parts.length > 1) {
86
+ // e.g. --opt=val
87
+ argv.unshift(parts.slice(1).join('='));
88
+ }
89
+ arg = parts[0];
90
+ } else if (arg[0] === '-') {
91
+ if (arg.length > 2) {
92
+ // e.g. -abc
93
+ argv = arg.substring(1).split('').map(function(ch) {
94
+ return '-' + ch;
95
+ }).concat(argv);
96
+ arg = argv.shift() as string;
97
+ } else {
98
+ // e.g. -a
99
+ }
100
+ } else {
101
+ // e.g. foo
102
+ }
103
+
104
+ return arg;
105
+ }
106
+
107
+ while (argv.length) {
108
+ arg = getArg();
109
+ switch (arg) {
110
+ case '-o':
111
+ case '--output':
112
+ output = argv.shift();
113
+ break;
114
+ case '-i':
115
+ case '--input':
116
+ input = argv.shift();
117
+ break;
118
+ case '-s':
119
+ case '--string':
120
+ string = argv.shift();
121
+ break;
122
+ case '-t':
123
+ case '--tokens':
124
+ tokens = true;
125
+ break;
126
+ case '-c':
127
+ case '--config':
128
+ config = argv.shift();
129
+ break;
130
+ case '-n':
131
+ case '--no-clobber':
132
+ noclobber = true;
133
+ break;
134
+ case '-h':
135
+ case '--help':
136
+ return await help();
137
+ case '-v':
138
+ case '--version':
139
+ return await version();
140
+ default:
141
+ if (arg.indexOf('--') === 0) {
142
+ opt = camelize(arg.replace(/^--(no-)?/, ''));
143
+ const defaults = marked.defaults as Record<string, unknown>;
144
+ if (!(opt in defaults)) {
145
+ continue;
146
+ }
147
+ if (arg.indexOf('--no-') === 0) {
148
+ options[opt] = typeof defaults[opt] !== 'boolean'
149
+ ? null
150
+ : false;
151
+ } else {
152
+ options[opt] = typeof defaults[opt] !== 'boolean'
153
+ ? argv.shift()
154
+ : true;
155
+ }
156
+ } else {
157
+ files.push(arg);
158
+ }
159
+ break;
160
+ }
161
+ }
162
+
163
+ async function getData(): Promise<string> {
164
+ if (string !== undefined) {
165
+ return string;
166
+ }
167
+ if (input) {
168
+ return await readFile(input, 'utf8');
169
+ }
170
+ if (files.length > 0) {
171
+ return await readFile(files.pop() as string, 'utf8');
172
+ }
173
+ return await getStdin();
174
+ }
175
+
176
+ function resolveFile(file: string): string {
177
+ return resolve(file.replace(/^~/, () => homedir()));
178
+ }
179
+
180
+ async function fileExists(file: string): Promise<boolean> {
181
+ try {
182
+ await access(resolveFile(file));
183
+ return true;
184
+ } catch {
185
+ return false;
186
+ }
187
+ }
188
+
189
+ async function runConfig(file: string) {
190
+ const configFile = resolveFile(file);
191
+ if (!configFile.endsWith('.json')) {
192
+ throw Error(`Cannot load config file '${file}': only JSON config files are supported`);
193
+ }
194
+ let markedConfig = JSON.parse(await readFile(configFile, 'utf8'));
195
+ if (markedConfig.default) {
196
+ markedConfig = markedConfig.default;
197
+ }
198
+ marked.use(markedConfig as MarkedExtension);
199
+ }
200
+
201
+ const data = await getData();
202
+
203
+ if (config) {
204
+ if (!await fileExists(config)) {
205
+ throw Error(`Cannot load config file '${config}'`);
206
+ }
207
+
208
+ await runConfig(config);
209
+ } else {
210
+ const defaultConfig = [
211
+ '~/.marked.json',
212
+ '~/.marked.js',
213
+ '~/.marked/index.js',
214
+ ];
215
+
216
+ for (const configFile of defaultConfig) {
217
+ if (await fileExists(configFile)) {
218
+ await runConfig(configFile);
219
+ break;
220
+ }
221
+ }
222
+ }
223
+
224
+ const html = tokens
225
+ ? JSON.stringify(marked.lexer(data, options as MarkedOptions), null, 2)
226
+ : await marked.parse(data, options as MarkedOptions);
227
+
228
+ if (output) {
229
+ try {
230
+ return await writeFile(output, html, { flag: noclobber ? 'wx' : 'w' });
231
+ } catch (err) {
232
+ if (noclobber && (err as CliError).code === 'EEXIST') {
233
+ throw Error('marked: output file \'' + output + '\' already exists, disable the \'-n\' / \'--no-clobber\' flag to overwrite\n');
234
+ }
235
+ throw err;
236
+ }
237
+ }
238
+
239
+ // (flushed before main() exits the process: bin/main.js exits right after
240
+ // the write, which cuts output to a pipe off at 64 KB)
241
+ await new Promise<void>((resolve) => {
242
+ process.stdout.write(html + '\n', () => resolve());
243
+ });
244
+ }
245
+
246
+ /**
247
+ * Helpers
248
+ */
249
+ function getStdin(): Promise<string> {
250
+ return new Promise((resolve, reject) => {
251
+ const stdin = process.stdin;
252
+ let buff = '';
253
+
254
+ stdin.setEncoding('utf8');
255
+
256
+ stdin.on('data', function(data: string) {
257
+ buff += data;
258
+ });
259
+
260
+ stdin.on('error', function(err: Error) {
261
+ reject(err);
262
+ });
263
+
264
+ stdin.on('end', function() {
265
+ resolve(buff);
266
+ });
267
+
268
+ stdin.resume();
269
+ });
270
+ }
271
+
272
+ function camelize(text: string): string {
273
+ return text.replace(/(\w)-(\w)/g, function(_, a: string, b: string) {
274
+ return a + b.toUpperCase();
275
+ });
276
+ }
277
+
278
+ try {
279
+ await start(process.argv.slice(2));
280
+ process.exit(0);
281
+ } catch (err) {
282
+ const e = err as CliError;
283
+ if (e.code === 'ENOENT') {
284
+ process.stderr.write('marked: ' + e.path + ': No such file or directory');
285
+ } else {
286
+ process.stderr.write(e.message);
287
+ }
288
+ return process.exit(1);
289
+ }
290
+ }
291
+
292
+ main();
@@ -0,0 +1,185 @@
1
+ // Parity check: Node.js vs the TypeNative build of the marked CLI.
2
+ //
3
+ // Fetches marked's own spec suites at the vendored tag (v18.1.2) into .cache/,
4
+ // compiles main.ts to a native binary with the TypeNative CLI from this repo
5
+ // checkout, then feeds every case's markdown on stdin to both, in each mode:
6
+ // default (GFM), --no-gfm and --tokens (the lexer's token list as JSON). A
7
+ // case's front matter (gfm / pedantic / breaks) becomes CLI flags, as marked's
8
+ // test runner turns it into options. stdout, stderr and the exit code must
9
+ // match byte for byte (stack frames aside: see withoutStackFrames). Failures
10
+ // are written to .cache/failures/.
11
+ //
12
+ // Usage: node parity.mjs [filter] (filter: a case id substring or a mode)
13
+
14
+ import { spawn, spawnSync } from 'node:child_process';
15
+ import fs from 'node:fs';
16
+ import os from 'node:os';
17
+ import path from 'node:path';
18
+ import vm from 'node:vm';
19
+ import { fileURLToPath } from 'node:url';
20
+
21
+ const here = path.dirname(fileURLToPath(import.meta.url));
22
+ const repoRoot = path.resolve(here, '..', '..');
23
+ const cli = path.join(repoRoot, 'bin', 'cli.js');
24
+ const cache = path.join(here, '.cache');
25
+ const upstream = path.join(cache, 'marked-src');
26
+ const specs = path.join(upstream, 'test', 'specs');
27
+ const failuresDir = path.join(cache, 'failures');
28
+ const exePath = path.join(cache, process.platform === 'win32' ? 'marked.exe' : 'marked');
29
+ const filter = process.argv[2] ?? '';
30
+ const MODES = [
31
+ { name: 'default', args: [] },
32
+ { name: 'no-gfm', args: ['--no-gfm'] },
33
+ { name: 'tokens', args: ['--tokens'] }
34
+ ];
35
+
36
+ function run(command, args, options = {}) {
37
+ const result = spawnSync(command, args, { stdio: 'inherit', ...options });
38
+ if (result.status !== 0) {
39
+ console.error(`${command} ${args.join(' ')} failed`);
40
+ process.exit(1);
41
+ }
42
+ }
43
+
44
+ // The corpus: marked's spec suites (test/specs) at the vendored tag
45
+ function fetchSpecs() {
46
+ if (fs.existsSync(path.join(specs, 'commonmark'))) return;
47
+ fs.mkdirSync(cache, { recursive: true });
48
+ console.log('Fetching marked v18.1.2 test/specs ...');
49
+ run('git', ['clone', '--quiet', '--depth', '1', '--branch', 'v18.1.2', '--filter=blob:none', '--sparse', 'https://github.com/markedjs/marked.git', upstream]);
50
+ run('git', ['-C', upstream, 'sparse-checkout', 'set', 'test/specs']);
51
+ }
52
+
53
+ // Front matter (---\nkey: value\n---\n) as CLI flags; the rest is the markdown
54
+ function frontMatter(text) {
55
+ const match = /^---\r?\n([\s\S]*?)\r?\n---\r?\n/.exec(text);
56
+ if (!match) return { markdown: text, args: [] };
57
+ const args = [];
58
+ for (const line of match[1].split(/\r?\n/)) {
59
+ const [key, value] = line.split(':').map((s) => s.trim());
60
+ if (!['gfm', 'pedantic', 'breaks'].includes(key)) continue;
61
+ args.push(value === 'false' ? `--no-${key}` : `--${key}`);
62
+ }
63
+ return { markdown: text.slice(match[0].length), args };
64
+ }
65
+
66
+ function listInputs() {
67
+ const inputs = [];
68
+ for (const [dir, file] of [['commonmark', 'commonmark.0.31.2.json'], ['gfm', 'gfm.0.29.json']]) {
69
+ for (const c of JSON.parse(fs.readFileSync(path.join(specs, dir, file), 'utf8'))) {
70
+ inputs.push({ id: `${dir}-${c.example}`, markdown: c.markdown, args: [] });
71
+ }
72
+ }
73
+ for (const dir of ['new', 'original', 'redos']) {
74
+ for (const name of fs.readdirSync(path.join(specs, dir)).sort()) {
75
+ const file = path.join(specs, dir, name);
76
+ if (name.endsWith('.md')) {
77
+ inputs.push({ id: `${dir}-${name.slice(0, -3)}`, ...frontMatter(fs.readFileSync(file, 'utf8')) });
78
+ } else if (name.endsWith('.cjs')) {
79
+ // module.exports = { markdown: `…`, html: `…` }: evaluated with
80
+ // nothing but a module object in scope
81
+ const sandbox = { module: { exports: {} } };
82
+ vm.runInNewContext(fs.readFileSync(file, 'utf8'), sandbox, { filename: file, timeout: 5000 });
83
+ const { markdown } = sandbox.module.exports;
84
+ inputs.push({ id: `${dir}-${name.slice(0, -4)}`, markdown: String(markdown), args: [] });
85
+ }
86
+ }
87
+ }
88
+ return inputs;
89
+ }
90
+
91
+ function exec(command, args, input) {
92
+ return new Promise((resolve) => {
93
+ const p = spawn(command, args, { cwd: here, stdio: ['pipe', 'pipe', 'pipe'] });
94
+ const stdout = [];
95
+ const stderr = [];
96
+ p.stdout.on('data', (d) => stdout.push(d));
97
+ p.stderr.on('data', (d) => stderr.push(d));
98
+ const timer = setTimeout(() => p.kill('SIGKILL'), 10000);
99
+ p.on('close', (code, signal) => {
100
+ clearTimeout(timer);
101
+ resolve({ code: signal ? signal : code, stdout: Buffer.concat(stdout), stderr: Buffer.concat(stderr) });
102
+ });
103
+ p.stdin.on('error', () => {});
104
+ p.stdin.end(input);
105
+ });
106
+ }
107
+
108
+ // Node prints an error's stack (" at …" lines, the frames of the JS call
109
+ // stack); a native binary has no JS frames to show. The frames are dropped
110
+ // (an inspected error's own properties, which follow the last frame as " {",
111
+ // stay where they are); warnings' process ids differ too
112
+ function withoutStackFrames(buffer) {
113
+ const out = [];
114
+ for (const line of buffer.toString().split('\n')) {
115
+ if (/^ {4}at /.test(line)) {
116
+ if (line.endsWith(' {')) out[out.length - 1] += ' {';
117
+ continue;
118
+ }
119
+ // (and process warnings name the process id)
120
+ out.push(line.replace(/^\(node:\d+\) /, '(node) '));
121
+ }
122
+ return Buffer.from(out.join('\n'));
123
+ }
124
+
125
+ function sameOutput(a, b) {
126
+ return a.code === b.code && a.stdout.equals(b.stdout) && withoutStackFrames(a.stderr).equals(withoutStackFrames(b.stderr));
127
+ }
128
+
129
+ function report(failure) {
130
+ const { id, mode, input, args, node, native } = failure;
131
+ const text = [
132
+ `# ${id} (${mode}: ${args.join(' ')})`,
133
+ '## input',
134
+ input.toString(),
135
+ `## node (exit ${node.code})`,
136
+ '### stdout', node.stdout.toString(), '### stderr', node.stderr.toString(),
137
+ `## native (exit ${native.code})`,
138
+ '### stdout', native.stdout.toString(), '### stderr', native.stderr.toString()
139
+ ].join('\n');
140
+ fs.writeFileSync(path.join(failuresDir, `${id}-${mode}.txt`), text);
141
+ }
142
+
143
+ fetchSpecs();
144
+ console.log('Compiling main.ts ...');
145
+ run(process.execPath, [cli, '--source', path.join(here, 'main.ts'), '--output', exePath], { cwd: here });
146
+
147
+ const cases = [];
148
+ for (const input of listInputs()) {
149
+ for (const mode of MODES) {
150
+ if (filter && !input.id.includes(filter) && mode.name !== filter) continue;
151
+ cases.push({ ...input, mode, allArgs: [...input.args, ...mode.args] });
152
+ }
153
+ }
154
+ fs.rmSync(failuresDir, { recursive: true, force: true });
155
+ fs.mkdirSync(failuresDir, { recursive: true });
156
+
157
+ let next = 0;
158
+ let passed = 0;
159
+ const failed = [];
160
+ async function worker() {
161
+ while (next < cases.length) {
162
+ const c = cases[next++];
163
+ const input = Buffer.from(c.markdown);
164
+ const [node, native] = await Promise.all([
165
+ exec(process.execPath, [path.join(here, 'main.ts'), ...c.allArgs], input),
166
+ exec(exePath, c.allArgs, input)
167
+ ]);
168
+ if (sameOutput(node, native)) passed++;
169
+ else {
170
+ failed.push(`${c.id} (${c.mode.name})`);
171
+ report({ id: c.id, mode: c.mode.name, input, args: c.allArgs, node, native });
172
+ }
173
+ }
174
+ }
175
+ const workers = Math.max(2, Math.min(16, os.availableParallelism()));
176
+ await Promise.all(Array.from({ length: workers }, () => worker()));
177
+
178
+ const rate = cases.length ? ((100 * passed) / cases.length).toFixed(2) : '100.00';
179
+ console.log(`\n${passed}/${cases.length} cases match (${rate}%)`);
180
+ if (failed.length > 0) {
181
+ console.log(`${failed.length} failures (details in .cache/failures/):`);
182
+ for (const f of failed.slice(0, 40)) console.log(` ${f}`);
183
+ if (failed.length > 40) console.log(` … and ${failed.length - 40} more`);
184
+ process.exitCode = 1;
185
+ }