tjs-lang 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +15 -5
- package/CONTEXT.md +4 -0
- package/demo/docs.json +12 -690
- package/demo/examples.test.ts +6 -2
- package/demo/src/playground-shared.ts +48 -16
- package/demo/src/playground-test-results.test.ts +112 -0
- package/demo/src/tjs-playground.ts +11 -5
- package/dist/eslint.config.d.ts +2 -0
- package/dist/index.js +146 -141
- package/dist/index.js.map +4 -4
- package/dist/src/lang/core.d.ts +1 -1
- package/dist/src/lang/dialect.d.ts +35 -0
- package/dist/src/lang/emitters/ast.d.ts +1 -1
- package/dist/src/lang/emitters/js-tests.d.ts +7 -0
- package/dist/src/lang/emitters/js-wasm.d.ts +5 -1
- package/dist/src/lang/emitters/js.d.ts +21 -0
- package/dist/src/lang/index.d.ts +3 -1
- package/dist/src/lang/module-loader.d.ts +125 -0
- package/dist/src/lang/parser-transforms.d.ts +79 -0
- package/dist/src/lang/parser-types.d.ts +50 -0
- package/dist/src/lang/parser.d.ts +18 -0
- package/dist/src/lang/transpiler.d.ts +1 -0
- package/dist/src/lang/types.d.ts +18 -0
- package/dist/src/lang/wasm.d.ts +67 -1
- package/dist/src/vm/runtime.d.ts +18 -0
- package/dist/src/vm/vm.d.ts +15 -1
- package/dist/tjs-batteries.js +2 -2
- package/dist/tjs-batteries.js.map +2 -2
- package/dist/tjs-eval.js +43 -41
- package/dist/tjs-eval.js.map +3 -3
- package/dist/tjs-from-ts.js +2 -2
- package/dist/tjs-from-ts.js.map +2 -2
- package/dist/tjs-lang.js +107 -104
- package/dist/tjs-lang.js.map +4 -4
- package/dist/tjs-vm.js +53 -51
- package/dist/tjs-vm.js.map +3 -3
- package/docs/README.md +2 -0
- package/docs/lm-studio-setup.md +143 -0
- package/docs/universal-endpoint.md +122 -0
- package/llms.txt +9 -2
- package/package.json +24 -4
- package/src/batteries/audit.ts +6 -5
- package/src/batteries/llm.ts +8 -3
- package/src/builder.ts +0 -3
- package/src/cli/commands/check.ts +3 -2
- package/src/cli/commands/emit.ts +4 -2
- package/src/cli/commands/run.ts +6 -2
- package/src/cli/commands/test.ts +1 -1
- package/src/cli/commands/types.ts +2 -2
- package/src/lang/codegen.test.ts +4 -1
- package/src/lang/core.ts +6 -4
- package/src/lang/dialect.test.ts +63 -0
- package/src/lang/dialect.ts +50 -0
- package/src/lang/docs.test.ts +0 -1
- package/src/lang/emitters/ast.ts +145 -2
- package/src/lang/emitters/from-ts.ts +1 -1
- package/src/lang/emitters/js-tests.ts +46 -37
- package/src/lang/emitters/js.ts +19 -3
- package/src/lang/features.test.ts +6 -5
- package/src/lang/index.ts +18 -5
- package/src/lang/linter.ts +1 -1
- package/src/lang/module-loader.test.ts +9 -5
- package/src/lang/module-loader.ts +4 -5
- package/src/lang/parser-params.ts +1 -1
- package/src/lang/parser-transforms.ts +12 -18
- package/src/lang/parser-types.ts +17 -0
- package/src/lang/parser.test.ts +12 -6
- package/src/lang/parser.ts +113 -3
- package/src/lang/perf.test.ts +10 -4
- package/src/lang/runtime.ts +0 -1
- package/src/lang/subset-invariant.test.ts +90 -0
- package/src/lang/transpiler.ts +8 -0
- package/src/lang/types.ts +18 -0
- package/src/lang/wasm.test.ts +22 -16
- package/src/linalg/linalg.test.ts +16 -4
- package/src/linalg/vector-search.bench.test.ts +31 -10
- package/src/types/Type.ts +6 -6
- package/src/use-cases/asymmetric-client-server.test.ts +0 -2
- package/src/use-cases/batteries.test.ts +4 -0
- package/src/use-cases/client-server.test.ts +1 -1
- package/src/use-cases/local-helpers.test.ts +219 -0
- package/src/use-cases/timeout-overrides.test.ts +169 -0
- package/src/use-cases/unbundled-imports.test.ts +0 -1
- package/src/vm/atoms/batteries.ts +17 -3
- package/src/vm/runtime.ts +92 -7
- package/src/vm/vm.ts +50 -10
package/docs/README.md
CHANGED
|
@@ -20,9 +20,11 @@ ASTs that run sandboxed with fuel (gas) limits.
|
|
|
20
20
|
| Document | Description |
|
|
21
21
|
|----------|-------------|
|
|
22
22
|
| [Technical Context](https://github.com/tonioloewald/tjs-lang/blob/main/CONTEXT.md) | Architecture deep dive, design decisions |
|
|
23
|
+
| [The Universal Endpoint](https://github.com/tonioloewald/tjs-lang/blob/main/docs/universal-endpoint.md) | Isomorphic atoms: the same agent + one security model running in the data center and the browser client |
|
|
23
24
|
| [WASM Quick Start](https://github.com/tonioloewald/tjs-lang/blob/main/docs/WASM-QUICKSTART.md) | Build WASM-accelerated libraries with zero toolchain setup |
|
|
24
25
|
| [Design Patterns](https://github.com/tonioloewald/tjs-lang/blob/main/guides/patterns.md) | Common patterns in TJS and AJS |
|
|
25
26
|
| [Schema Validation](https://github.com/tonioloewald/tjs-lang/blob/main/guides/tosijs-schema.md) | Working with tosijs-schema |
|
|
27
|
+
| [LM Studio Setup](https://github.com/tonioloewald/tjs-lang/blob/main/docs/lm-studio-setup.md) | Running the LLM test suite locally — install, model loading, leaked-VRAM/CORS gotchas |
|
|
26
28
|
|
|
27
29
|
## Examples
|
|
28
30
|
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# Running the LLM test suite (LM Studio setup)
|
|
2
|
+
|
|
3
|
+
The LLM-dependent tests (`bun test` without `SKIP_LLM_TESTS`) talk to a local
|
|
4
|
+
**LM Studio** OpenAI-compatible server. This doc captures the setup and the
|
|
5
|
+
pain points hit getting it working on Linux — read the troubleshooting section
|
|
6
|
+
first if a model won't load or LLM tests fail fast.
|
|
7
|
+
|
|
8
|
+
## What has to be true
|
|
9
|
+
|
|
10
|
+
- LM Studio server running at **`http://localhost:1234/v1`** — this is hardcoded
|
|
11
|
+
(`src/batteries/models.ts` → `DEFAULT_BASE_URL`), **no env override**. Any
|
|
12
|
+
OpenAI-compatible server on 1234 works (Ollama bound to 1234 is a drop-in).
|
|
13
|
+
- **One chat LLM and one embedding model loaded** — the model audit
|
|
14
|
+
(`src/batteries/audit.ts`) needs both; it auto-detects a structured-output LLM
|
|
15
|
+
from the chat model.
|
|
16
|
+
- **CORS enabled** in LM Studio (see below) — required for the playground-example
|
|
17
|
+
tests, which run under happy-dom and send a CORS preflight.
|
|
18
|
+
|
|
19
|
+
## Quick start (happy path)
|
|
20
|
+
|
|
21
|
+
1. Install LM Studio, **update the runtimes** (see Install).
|
|
22
|
+
2. Load a chat LLM + an embedding model (e.g. `nomic-embed-text-v1.5`).
|
|
23
|
+
3. Developer → **Start Server** (port 1234) → **Enable CORS**.
|
|
24
|
+
4. Warm a clean audit cache, then run the full suite:
|
|
25
|
+
```bash
|
|
26
|
+
rm -f .models.cache.json
|
|
27
|
+
bun run test:llm # warms + writes one clean cache, in isolation
|
|
28
|
+
bun test # full suite reuses that cache (avoids the race below)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Sanity check the server (note **127.0.0.1**, not `localhost` — see IPv6 note):
|
|
32
|
+
```bash
|
|
33
|
+
curl -s http://127.0.0.1:1234/v1/models | grep -oE '"id":"[^"]*"'
|
|
34
|
+
~/.lmstudio/bin/lms ps # what's loaded + on GPU or CPU
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Install (Linux)
|
|
38
|
+
|
|
39
|
+
- **AppImage only** — no `.deb`/apt/snap. On the Ubuntu 24.04 / Mint 22 base,
|
|
40
|
+
install `libfuse2t64` first, then `chmod +x` and run.
|
|
41
|
+
- Requires **x86-64 + AVX2** (`grep -w avx2 /proc/cpuinfo`).
|
|
42
|
+
- **Update the inference runtimes after installing.** A fresh install ships with
|
|
43
|
+
old engines; recent model architectures (Gemma 3/3n/QAT, etc.) won't load until
|
|
44
|
+
you update. LM Studio → Settings → **Runtime** → update all. (A fresh install
|
|
45
|
+
needed many updates.)
|
|
46
|
+
- The CLI lives at `~/.lmstudio/bin/lms` (`lms bootstrap` adds it to PATH).
|
|
47
|
+
|
|
48
|
+
## Enable CORS (needed for playground-example tests)
|
|
49
|
+
|
|
50
|
+
`demo/examples.test.ts` runs under happy-dom (a browser-like env) and issues a
|
|
51
|
+
CORS preflight `OPTIONS` to `localhost:1234`. Without CORS enabled LM Studio
|
|
52
|
+
answers `400` and those ~8 tests fail with **"Cross-Origin Request Blocked"**.
|
|
53
|
+
Toggle **Enable CORS** in the LM Studio server settings. (Production doesn't hit
|
|
54
|
+
this — the playground proxies LLM calls through the `/run` cloud function.)
|
|
55
|
+
|
|
56
|
+
## Troubleshooting
|
|
57
|
+
|
|
58
|
+
### "Error loading model. (Exit code: null)" — crash partway through loading
|
|
59
|
+
|
|
60
|
+
The loader process died, almost always **out of VRAM** (it crashes mid-offload —
|
|
61
|
+
e.g. at ~53% of a 5–6.5 GB model) or, rarely, an architecture the runtime can't
|
|
62
|
+
build. Diagnose:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
nvidia-smi --query-gpu=memory.used,memory.free --format=csv,noheader
|
|
66
|
+
nvidia-smi --query-compute-apps=pid,used_memory,process_name --format=csv
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Two fixes:
|
|
70
|
+
|
|
71
|
+
- **Free VRAM** (see stray-worker note) so the model fits, or
|
|
72
|
+
- **Force CPU load** — slower but reliable, fits in system RAM:
|
|
73
|
+
```bash
|
|
74
|
+
~/.lmstudio/bin/lms load <model-id> --gpu off
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
The recent **gemma-4 / gemma-3n** models are multimodal (note the `mmproj-*.gguf`
|
|
78
|
+
projector files) and memory-heavy. A 12 GB card runs them fine **once VRAM is
|
|
79
|
+
actually free**. Blackwell GPUs (RTX 50xx) need a recent driver — 595+/CUDA 13
|
|
80
|
+
works.
|
|
81
|
+
|
|
82
|
+
### Leaked VRAM / killing the stray `node` worker — THE big one
|
|
83
|
+
|
|
84
|
+
LM Studio's **embedding worker** (`~/.lmstudio/.internal/utils/node` running
|
|
85
|
+
`embeddingworker.js`) can leak VRAM — observed holding **6.5 GB** while only
|
|
86
|
+
~700 MB of embedding models were loaded — and it is a **detached child process
|
|
87
|
+
that survives app restarts.** Closing/reopening the LM Studio window does *not*
|
|
88
|
+
reap it, so VRAM stays pinned and every subsequent GPU load crashes. Failed
|
|
89
|
+
GPU loads compound the leak (partial offloads never released).
|
|
90
|
+
|
|
91
|
+
Reclaim it by killing the workers directly, then verify before relaunching:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
pkill -f 'LM-Studio/lm-studio'
|
|
95
|
+
pkill -f '.lmstudio/.internal/utils/node' # the stuck embedding/resource workers
|
|
96
|
+
sleep 2
|
|
97
|
+
nvidia-smi --query-gpu=memory.used,memory.free --format=csv,noheader # should drop to ~1 GB used
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Then relaunch LM Studio fresh, Start Server, reload both models. If "restarting"
|
|
101
|
+
didn't help, you didn't kill these — check `nvidia-smi --query-compute-apps`.
|
|
102
|
+
|
|
103
|
+
### `curl localhost:1234` fails but the server is up
|
|
104
|
+
|
|
105
|
+
`localhost` may resolve to IPv6 `::1` while LM Studio listens on IPv4
|
|
106
|
+
`0.0.0.0:1234` → connection refused. Use **`127.0.0.1`** for manual probes.
|
|
107
|
+
(`ss -ltn | grep 1234` confirms it's listening regardless.)
|
|
108
|
+
|
|
109
|
+
### LLM tests fail *fast* (single-digit ms) with "No LLM/embedding available"
|
|
110
|
+
|
|
111
|
+
The audit classified no usable model and the result is cached. Causes:
|
|
112
|
+
- Models weren't loaded/ready when the audit ran (it caches the bad result for 24h).
|
|
113
|
+
- A model is listed in `/v1/models` (downloaded) but fails to load (see above).
|
|
114
|
+
|
|
115
|
+
Fix: make sure both model types are loaded and inferring, then **clear the cache**:
|
|
116
|
+
```bash
|
|
117
|
+
rm -f .models.cache.json
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
### Audit misclassifies models on a parallel run
|
|
121
|
+
|
|
122
|
+
`.models.cache.json` (cwd, 24h TTL) is shared, and many test files call `audit()`
|
|
123
|
+
concurrently. Clearing the cache right before a full parallel `bun test` makes
|
|
124
|
+
several audits probe **at once**, and under that load classifications come back
|
|
125
|
+
scrambled (embedding models tagged `LLM`, etc.). **Work around it** by writing one
|
|
126
|
+
clean cache in isolation first (`bun run test:llm`) and *not* clearing it before
|
|
127
|
+
the full run. (Proper fix is tracked in TODO — serialize the audit / isolate the
|
|
128
|
+
cache.)
|
|
129
|
+
|
|
130
|
+
### 3 `TJS Performance > CLI cold start` failures
|
|
131
|
+
|
|
132
|
+
Known, not LLM-related: `perf.test.ts` asserts cold start `< 200ms`; a loaded or
|
|
133
|
+
slower box measures ~210ms. Ignore, or relax the threshold.
|
|
134
|
+
|
|
135
|
+
## Command reference
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
~/.lmstudio/bin/lms ps # loaded models, device (GPU/CPU/Local), TTL
|
|
139
|
+
~/.lmstudio/bin/lms server status # is the OpenAI endpoint up + which port
|
|
140
|
+
~/.lmstudio/bin/lms load <id> --gpu off # CPU-only load (off | max | 0..1)
|
|
141
|
+
nvidia-smi --query-compute-apps=pid,used_memory,process_name --format=csv
|
|
142
|
+
curl -s http://127.0.0.1:1234/v1/models | grep -oE '"id":"[^"]*"'
|
|
143
|
+
```
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# The universal endpoint: isomorphic atoms, one security model
|
|
2
|
+
|
|
3
|
+
> An emergent win. The VM was built to run *untrusted* agent code safely —
|
|
4
|
+
> fuel-metered, capability-sandboxed, monadic errors instead of exceptions.
|
|
5
|
+
> Isomorphism was never a goal. But because of how those pieces fit, the same
|
|
6
|
+
> agent program runs **unchanged in the data center and in the browser client**,
|
|
7
|
+
> with one security model spanning both — and that turned out to be the most
|
|
8
|
+
> valuable property in production.
|
|
9
|
+
|
|
10
|
+
## What it buys you
|
|
11
|
+
|
|
12
|
+
In production we ran a **single agent endpoint that exists in two places at
|
|
13
|
+
once**: in the data center, and locally inside the web client. Tools (atoms)
|
|
14
|
+
come in matched pairs — a client-side implementation that can satisfy a request
|
|
15
|
+
from data already loaded in the browser, and a server-side implementation that
|
|
16
|
+
hits the datastore. The agent program is identical on both sides and doesn't
|
|
17
|
+
know which one it's talking to.
|
|
18
|
+
|
|
19
|
+
Concretely: both endpoints expose a `getRecords` atom.
|
|
20
|
+
|
|
21
|
+
- **Client `getRecords`** — looks for the records already loaded in the client.
|
|
22
|
+
If they're there, it returns them with **zero network round-trips**. If they're
|
|
23
|
+
not, it transparently delegates to the server-side `getRecords`.
|
|
24
|
+
- **Server `getRecords`** — queries the datastore.
|
|
25
|
+
|
|
26
|
+
Same name, same input/output schema, same authorization envelope. A program
|
|
27
|
+
written against `getRecords` runs locally-first and falls back to the server only
|
|
28
|
+
on a genuine miss — and you never wrote that fallback logic into the program. It
|
|
29
|
+
lives in the atom.
|
|
30
|
+
|
|
31
|
+
The three wins that fell out:
|
|
32
|
+
|
|
33
|
+
1. **One security model, front to back.** The same capability/fuel/RBAC envelope
|
|
34
|
+
is enforced in the browser and on the server — not duplicated, not "the client
|
|
35
|
+
asks nicely and the server re-checks." The *same rules* run in both places.
|
|
36
|
+
2. **Minimized round-tripping / data locality.** Code travels to the data.
|
|
37
|
+
Reads that hit already-loaded client state never leave the browser; only real
|
|
38
|
+
misses go to the server.
|
|
39
|
+
3. **Parallel tools.** The full toolset is available on both sides, so an
|
|
40
|
+
orchestration can run partly local and partly remote, transparently.
|
|
41
|
+
|
|
42
|
+
## Why it falls out of the architecture (rather than being bolted on)
|
|
43
|
+
|
|
44
|
+
Three properties of the AJS/AgentVM design make this work, and none of them were
|
|
45
|
+
added for isomorphism — they were there for *safe sandboxed execution*:
|
|
46
|
+
|
|
47
|
+
**1. Code is data.** An agent is a JSON AST (`ajs\`...\``/`transpile()`), not
|
|
48
|
+
compiled JavaScript. The same program serializes and runs anywhere a JS
|
|
49
|
+
`AgentVM` exists — a Node server, a Cloud Function, or a browser tab. "Code
|
|
50
|
+
travels to data" stops being a slogan and becomes a deployment option.
|
|
51
|
+
|
|
52
|
+
**2. The VM is environment-agnostic; all IO is injected.** `AgentVM` has zero IO
|
|
53
|
+
by default. Every side effect arrives through `ctx.capabilities` (`fetch`,
|
|
54
|
+
`store`, `llm`, …) passed to `run()`. So the *same* agent gets browser-flavored
|
|
55
|
+
capabilities in the client and data-center capabilities on the server — the
|
|
56
|
+
program is unaware of the difference. (See `src/vm/runtime.ts` — atoms only ever
|
|
57
|
+
reach the outside world via `ctx.capabilities`.)
|
|
58
|
+
|
|
59
|
+
**3. Atoms are the seam.** `new AgentVM(customAtoms)` merges environment-specific
|
|
60
|
+
implementations over the core set:
|
|
61
|
+
|
|
62
|
+
```ts
|
|
63
|
+
// src/vm/vm.ts
|
|
64
|
+
constructor(customAtoms = {}) {
|
|
65
|
+
this.atoms = { ...coreAtoms, ...customAtoms }
|
|
66
|
+
}
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The atom *contract* (op name, input/output schema) is identical across
|
|
70
|
+
environments; the *implementation* differs. The client VM is built with a
|
|
71
|
+
client `getRecords`; the server VM with a server `getRecords`. Swapping the
|
|
72
|
+
implementation per environment is the whole trick — and it's just object spread.
|
|
73
|
+
|
|
74
|
+
```ts
|
|
75
|
+
// client
|
|
76
|
+
const clientVM = new AgentVM({
|
|
77
|
+
getRecords: defineAtom('getRecords', In, Out, async (input, ctx) => {
|
|
78
|
+
const local = findLoaded(input, ctx) // already in the client?
|
|
79
|
+
if (local) return local // → no round-trip
|
|
80
|
+
return ctx.capabilities.fetch(SERVER_ENDPOINT, …) // → fall back to server
|
|
81
|
+
}),
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
// server
|
|
85
|
+
const serverVM = new AgentVM({
|
|
86
|
+
getRecords: defineAtom('getRecords', In, Out, async (input, ctx) =>
|
|
87
|
+
ctx.capabilities.store.query(input) // hit the datastore
|
|
88
|
+
),
|
|
89
|
+
})
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Why the single security model is the real prize
|
|
93
|
+
|
|
94
|
+
Normally the client and server each reimplement data access *and* authorization,
|
|
95
|
+
kept in sync by hand, and the client can't safely share the server's auth logic —
|
|
96
|
+
so the server re-validates everything and you maintain two security surfaces.
|
|
97
|
+
|
|
98
|
+
Here there's one. Authorization is the injected-capability boundary plus the
|
|
99
|
+
fuel budget plus RBAC — and the RBAC **rules themselves are TJS** (`src/rbac/`,
|
|
100
|
+
`rules.tjs`), i.e. portable, serializable data, not compiled server code. So the
|
|
101
|
+
exact same rule set evaluates in the browser and in the data center. A request
|
|
102
|
+
that's denied locally is denied for the same reason it'd be denied server-side,
|
|
103
|
+
because it's *the same rule running*. Fuel metering and monadic errors (no
|
|
104
|
+
thrown exceptions to exploit) apply identically in both runtimes.
|
|
105
|
+
|
|
106
|
+
Front-to-back, the security model isn't *mirrored* — it's *singular*.
|
|
107
|
+
|
|
108
|
+
## When to reach for it
|
|
109
|
+
|
|
110
|
+
- A rich client that already holds a working set of data, where most reads can be
|
|
111
|
+
served locally and only misses need the server.
|
|
112
|
+
- A system where you want one authorization story rather than a client SDK's
|
|
113
|
+
rules and a server API's rules drifting apart.
|
|
114
|
+
- Agent/tool orchestration that should run wherever the data is cheapest to
|
|
115
|
+
reach, without rewriting the program per environment.
|
|
116
|
+
|
|
117
|
+
## See also
|
|
118
|
+
|
|
119
|
+
- `CONTEXT.md` — architecture deep dive
|
|
120
|
+
- AJS pillar in `CLAUDE.md` — "Code travels to data"
|
|
121
|
+
- `src/vm/vm.ts`, `src/vm/runtime.ts` — the VM and the capability boundary
|
|
122
|
+
- `src/rbac/` — portable (TJS) security rules
|
package/llms.txt
CHANGED
|
@@ -8,6 +8,7 @@ This file is a navigation index for AI agents. It does not contain the docs them
|
|
|
8
8
|
|
|
9
9
|
- [CLAUDE-TJS-SYNTAX.md](CLAUDE-TJS-SYNTAX.md) — TJS syntax reference. Critical: `function foo(x: 'default')` is NOT a TypeScript string-literal type. The colon value is an **example**; `'default'` widens to `string`. LLMs get this wrong constantly.
|
|
10
10
|
- [CLAUDE.md](CLAUDE.md) — project overview, common commands, architecture, security model, and key APIs.
|
|
11
|
+
- [PRINCIPLES.md](PRINCIPLES.md) — **non-negotiable language invariants**: options-off TJS ⊇ JS, and TJS ⊇ AJS. A richer layer may do *more* with the same source but must never make subset-legal code illegal (e.g. un-runnable signature tests are inconclusive, not errors). The opt-in unit is the **dialect**: `tjs(src, { dialect: 'js' })` preserves plain-JS semantics, `'tjs'` (default) is native; file tools map extension→dialect via `dialectForFilename`/`sourceKindForFilename` from `tjs-lang/lang`. TypeScript routes through `fromTS` (`tjs-lang/lang/from-ts`) so the lean lang bundle stays TS-free — the full `js | ts | tjs` routing recipe is in PRINCIPLES.md. A violation is a bug.
|
|
11
12
|
|
|
12
13
|
## Language guides
|
|
13
14
|
|
|
@@ -27,7 +28,9 @@ This file is a navigation index for AI agents. It does not contain the docs them
|
|
|
27
28
|
- [CONTEXT.md](CONTEXT.md) — architecture deep dive (two-layer design, atom system, value resolution).
|
|
28
29
|
- [docs/function-predicate-design.md](docs/function-predicate-design.md) — Type / Generic / FunctionPredicate design.
|
|
29
30
|
- [docs/generic-dts-design.md](docs/generic-dts-design.md) — `.d.ts` generation from TJS.
|
|
30
|
-
- [docs/WASM-QUICKSTART.md](docs/WASM-QUICKSTART.md) — inline WASM, SIMD, `wasmBuffer()
|
|
31
|
+
- [docs/WASM-QUICKSTART.md](docs/WASM-QUICKSTART.md) — inline WASM, SIMD, `wasmBuffer()` (5-minute intro).
|
|
32
|
+
- [DOCS-WASM.md](DOCS-WASM.md) — canonical WASM reference: both inline `wasm{}` blocks and top-level `wasm function` declarations, memory model, cross-file composition (Phase 3), boundary distribution form (Phase 4), tjs-lang/linalg, current limitations.
|
|
33
|
+
- [wasm-library-plan.md](wasm-library-plan.md) — cross-file WASM library design + phased implementation history. Shipped in v0.8.0; see plan for what's deferred.
|
|
31
34
|
- [docs/native-engine-integration.md](docs/native-engine-integration.md) — embedding into a native runtime.
|
|
32
35
|
|
|
33
36
|
## Examples and live code
|
|
@@ -44,6 +47,7 @@ This file is a navigation index for AI agents. It does not contain the docs them
|
|
|
44
47
|
- `tjs-lang/vm` → `src/vm/index.ts` — VM and atoms.
|
|
45
48
|
- `tjs-lang/eval` → `src/lang/eval.ts` — `Eval`, `SafeFunction`.
|
|
46
49
|
- `tjs-lang/batteries` → `src/batteries/index.ts` — LM Studio integration (lazy, local-only).
|
|
50
|
+
- `tjs-lang/linalg` → `src/linalg/index.tjs` — SIMD linear-algebra kernels (`dot`, `norm_sq`, `dot_at`, `norm_sq_at`).
|
|
47
51
|
|
|
48
52
|
## Source map (for code-reading agents)
|
|
49
53
|
|
|
@@ -55,7 +59,10 @@ This file is a navigation index for AI agents. It does not contain the docs them
|
|
|
55
59
|
- `src/lang/runtime.ts` — TJS runtime: monadic errors, `checkType`, `createRuntime`.
|
|
56
60
|
- `src/lang/inference.ts` — infer `TypeDescriptor` from example values.
|
|
57
61
|
- `src/lang/json-schema.ts` — JSON Schema from `TypeDescriptor`.
|
|
58
|
-
- `src/lang/wasm.ts` — WASM bytecode emission.
|
|
62
|
+
- `src/lang/wasm.ts` — WASM bytecode emission. Multi-function module composition; wasm-to-wasm `call <index>` resolution; per-function index map for cross-call resolution (v0.8.0).
|
|
63
|
+
- `src/lang/emitters/js-wasm.ts` — JS bootstrap emitter for compiled wasm modules (one `WebAssembly.compile` per file).
|
|
64
|
+
- `src/lang/module-loader.ts` — transpile-time `.tjs`/`.ts`/`.js` module loader; used by cross-file `wasm function` composition (v0.8.0).
|
|
65
|
+
- `src/linalg/index.tjs` — `tjs-lang/linalg` stdlib source (v0.8.0).
|
|
59
66
|
- `src/lang/linter.ts` — static analysis (unused vars, unreachable code).
|
|
60
67
|
- `src/vm/runtime.ts` — atom implementations, expression sandboxing, fuel charging (security-critical, ~3000 lines).
|
|
61
68
|
- `src/vm/vm.ts` — `AgentVM` class.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tjs-lang",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.2",
|
|
4
4
|
"description": "Type-safe JavaScript dialect with runtime validation, sandboxed VM execution, and AI agent orchestration. Transpiles TypeScript to validated JS with fuel-metered execution for untrusted code.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -22,6 +22,25 @@
|
|
|
22
22
|
"license": "Apache-2.0",
|
|
23
23
|
"main": "./dist/index.js",
|
|
24
24
|
"types": "./dist/src/index.d.ts",
|
|
25
|
+
"typesVersions": {
|
|
26
|
+
"*": {
|
|
27
|
+
"eval": [
|
|
28
|
+
"./dist/src/lang/eval.d.ts"
|
|
29
|
+
],
|
|
30
|
+
"lang": [
|
|
31
|
+
"./dist/src/lang/transpiler.d.ts"
|
|
32
|
+
],
|
|
33
|
+
"lang/from-ts": [
|
|
34
|
+
"./dist/src/lang/emitters/from-ts.d.ts"
|
|
35
|
+
],
|
|
36
|
+
"vm": [
|
|
37
|
+
"./dist/src/vm/index.d.ts"
|
|
38
|
+
],
|
|
39
|
+
"batteries": [
|
|
40
|
+
"./dist/src/batteries/index.d.ts"
|
|
41
|
+
]
|
|
42
|
+
}
|
|
43
|
+
},
|
|
25
44
|
"exports": {
|
|
26
45
|
".": {
|
|
27
46
|
"bun": "./src/index.ts",
|
|
@@ -96,24 +115,25 @@
|
|
|
96
115
|
"@codemirror/state": "^6.5.3",
|
|
97
116
|
"@codemirror/theme-one-dark": "^6.1.3",
|
|
98
117
|
"@codemirror/view": "^6.39.9",
|
|
118
|
+
"@eslint/js": "^10.0.1",
|
|
99
119
|
"@happy-dom/global-registrator": "^20.1.0",
|
|
100
120
|
"@types/bun": "latest",
|
|
101
121
|
"@types/jsdom": "^21.1.7",
|
|
102
|
-
"@typescript-eslint/eslint-plugin": "^5.62.0",
|
|
103
|
-
"@typescript-eslint/parser": "^5.62.0",
|
|
104
122
|
"acorn-walk": "^8.3.4",
|
|
105
123
|
"chokidar": "^4.0.3",
|
|
106
124
|
"codemirror": "^6.0.2",
|
|
107
125
|
"esbuild": "^0.28.0",
|
|
108
|
-
"eslint": "^
|
|
126
|
+
"eslint": "^10.4.1",
|
|
109
127
|
"firebase": "^10.12.0",
|
|
110
128
|
"firebase-admin": "^13.6.0",
|
|
111
129
|
"firebase-functions": "^7.0.5",
|
|
130
|
+
"globals": "^17.6.0",
|
|
112
131
|
"marked": "^9.1.6",
|
|
113
132
|
"prettier": "^2.8.8",
|
|
114
133
|
"tosijs": "^1.5.6",
|
|
115
134
|
"tosijs-ui": "^1.4.7",
|
|
116
135
|
"typescript": "^5.6.2",
|
|
136
|
+
"typescript-eslint": "^8.61.0",
|
|
117
137
|
"valibot": "^0.36.0",
|
|
118
138
|
"vitest": "^2.0.5"
|
|
119
139
|
},
|
package/src/batteries/audit.ts
CHANGED
|
@@ -190,9 +190,10 @@ async function checkEmbedding(
|
|
|
190
190
|
}
|
|
191
191
|
}
|
|
192
192
|
|
|
193
|
-
//
|
|
193
|
+
// 32x32 solid-red PNG (base64). NOT 1x1 — degenerate sizes are rejected by
|
|
194
|
+
// real vision preprocessors (e.g. gemma: "Cannot handle this data type (1,1,1)").
|
|
194
195
|
const TINY_TEST_IMAGE =
|
|
195
|
-
'data:image/png;base64,
|
|
196
|
+
'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAMUlEQVR4nGO4o6FBU8QwasFoEI2motF8MFpUjJamo/XBaJU52qoYbReNNh01hkg+AACGobA9N+tfoAAAAABJRU5ErkJggg=='
|
|
196
197
|
|
|
197
198
|
async function checkVision(baseUrl: string, modelId: string): Promise<boolean> {
|
|
198
199
|
try {
|
|
@@ -225,13 +226,13 @@ export async function auditModels(baseUrl: string): Promise<ModelAudit[]> {
|
|
|
225
226
|
const cachedData = await readCache(baseUrl)
|
|
226
227
|
|
|
227
228
|
// 2. Get current model list from server
|
|
228
|
-
let serverModelIds: string[]
|
|
229
|
+
let serverModelIds: string[]
|
|
229
230
|
try {
|
|
230
231
|
const res = await fetch(`${baseUrl}/models`)
|
|
231
232
|
if (!res.ok) throw new Error('Could not connect')
|
|
232
233
|
const data = (await res.json()) as { data: { id: string }[] }
|
|
233
234
|
serverModelIds = data.data.map((m) => m.id).sort()
|
|
234
|
-
} catch
|
|
235
|
+
} catch {
|
|
235
236
|
// If we have cache and server is unavailable, use cache
|
|
236
237
|
if (cachedData) {
|
|
237
238
|
console.log('⚠️ LM Studio unavailable, using cached model audit.')
|
|
@@ -270,7 +271,7 @@ export async function auditModels(baseUrl: string): Promise<ModelAudit[]> {
|
|
|
270
271
|
let type: ModelAudit['type'] = 'Unknown'
|
|
271
272
|
let structured = false
|
|
272
273
|
let vision = false
|
|
273
|
-
let statusMsg
|
|
274
|
+
let statusMsg: string
|
|
274
275
|
let dimension: number | undefined = undefined
|
|
275
276
|
|
|
276
277
|
const isLLM = await checkLLM(baseUrl, model.id)
|
package/src/batteries/llm.ts
CHANGED
|
@@ -73,7 +73,10 @@ export function getLLMCapability(
|
|
|
73
73
|
model: model.id,
|
|
74
74
|
messages,
|
|
75
75
|
temperature: 0.7,
|
|
76
|
-
tools
|
|
76
|
+
// Only include tools when non-empty: an empty tools array makes
|
|
77
|
+
// LM Studio apply a tool-call grammar that can't be combined with
|
|
78
|
+
// a structured-output response_format ("lazy grammar" 400).
|
|
79
|
+
...(tools && tools.length > 0 ? { tools } : {}),
|
|
77
80
|
response_format: responseFormat,
|
|
78
81
|
}),
|
|
79
82
|
})
|
|
@@ -89,7 +92,8 @@ export function getLLMCapability(
|
|
|
89
92
|
} catch (e: any) {
|
|
90
93
|
if (e.cause?.code === 'ECONNREFUSED') {
|
|
91
94
|
throw new Error(
|
|
92
|
-
'No LLM provider configured. Please start LM Studio or provide an API key.'
|
|
95
|
+
'No LLM provider configured. Please start LM Studio or provide an API key.',
|
|
96
|
+
{ cause: e }
|
|
93
97
|
)
|
|
94
98
|
}
|
|
95
99
|
throw e
|
|
@@ -117,7 +121,8 @@ export function getLLMCapability(
|
|
|
117
121
|
} catch (e: any) {
|
|
118
122
|
if (e.cause?.code === 'ECONNREFUSED') {
|
|
119
123
|
throw new Error(
|
|
120
|
-
'No LLM provider configured. Please start LM Studio or provide an API key.'
|
|
124
|
+
'No LLM provider configured. Please start LM Studio or provide an API key.',
|
|
125
|
+
{ cause: e }
|
|
121
126
|
)
|
|
122
127
|
}
|
|
123
128
|
throw e
|
package/src/builder.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { coreAtoms, type Atom, type OpCode, type ExprNode } from './runtime'
|
|
2
2
|
|
|
3
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
4
3
|
type _AtomMap = typeof coreAtoms
|
|
5
4
|
|
|
6
5
|
// --- Condition String Parser ---
|
|
@@ -358,7 +357,6 @@ export interface ArgRef {
|
|
|
358
357
|
// --- Typed Builder ---
|
|
359
358
|
|
|
360
359
|
// Helper to extract input type from Atom definition
|
|
361
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
362
360
|
type AtomInput<T> = T extends Atom<infer I, any> ? I : never
|
|
363
361
|
|
|
364
362
|
// The Builder instance type with dynamic methods inferred from AtomMap
|
|
@@ -523,7 +521,6 @@ export class TypedBuilder<M extends Record<string, Atom<any, any>>> {
|
|
|
523
521
|
return(schema: any): BuilderType<M> {
|
|
524
522
|
const atom = this.atoms['return']
|
|
525
523
|
if (!atom) throw new Error("Atom 'return' not found")
|
|
526
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
527
524
|
const _schema = schema.schema ?? schema
|
|
528
525
|
return this.add(atom.create({ schema: _schema }))
|
|
529
526
|
}
|
|
@@ -3,13 +3,14 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { readFileSync } from 'fs'
|
|
6
|
-
import { tjs } from '../../lang'
|
|
6
|
+
import { tjs, dialectForFilename } from '../../lang'
|
|
7
7
|
|
|
8
8
|
export async function check(file: string): Promise<void> {
|
|
9
9
|
const source = readFileSync(file, 'utf-8')
|
|
10
10
|
|
|
11
11
|
try {
|
|
12
|
-
|
|
12
|
+
// `.js`/`.mjs` ⇒ plain-JS semantics preserved; `.tjs` ⇒ native modes.
|
|
13
|
+
const result = tjs(source, { dialect: dialectForFilename(file) })
|
|
13
14
|
|
|
14
15
|
// Report function info from types
|
|
15
16
|
if (result.types && Object.keys(result.types).length > 0) {
|
package/src/cli/commands/emit.ts
CHANGED
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
existsSync,
|
|
22
22
|
} from 'fs'
|
|
23
23
|
import { join, basename, dirname, extname } from 'path'
|
|
24
|
-
import { tjs } from '../../lang'
|
|
24
|
+
import { tjs, dialectForFilename } from '../../lang'
|
|
25
25
|
import { generateDocs } from '../../lang/docs'
|
|
26
26
|
import { generateDTS } from '../../lang/emitters/dts'
|
|
27
27
|
|
|
@@ -84,9 +84,11 @@ async function emitFile(
|
|
|
84
84
|
const filename = basename(inputPath)
|
|
85
85
|
|
|
86
86
|
try {
|
|
87
|
-
// Use 'report' mode to get test results without throwing
|
|
87
|
+
// Use 'report' mode to get test results without throwing.
|
|
88
|
+
// `.js`/`.mjs` ⇒ plain-JS semantics preserved; `.tjs` ⇒ native modes.
|
|
88
89
|
const result = tjs(source, {
|
|
89
90
|
filename,
|
|
91
|
+
dialect: dialectForFilename(inputPath),
|
|
90
92
|
debug: options.debug,
|
|
91
93
|
runTests: 'report',
|
|
92
94
|
})
|
package/src/cli/commands/run.ts
CHANGED
|
@@ -8,15 +8,19 @@ import { readFileSync } from 'fs'
|
|
|
8
8
|
import { resolve } from 'path'
|
|
9
9
|
import { preprocess } from '../../lang/parser'
|
|
10
10
|
import { transpileToJS } from '../../lang/emitters/js'
|
|
11
|
+
import { dialectForFilename } from '../../lang/dialect'
|
|
11
12
|
import * as runtime from '../../lang/runtime'
|
|
12
13
|
|
|
13
14
|
export async function run(file: string): Promise<void> {
|
|
14
15
|
const absolutePath = resolve(file)
|
|
15
16
|
const source = readFileSync(absolutePath, 'utf-8')
|
|
16
17
|
|
|
18
|
+
// `.js`/`.mjs` ⇒ plain-JS semantics preserved; `.tjs` ⇒ native modes.
|
|
19
|
+
const dialect = dialectForFilename(file)
|
|
20
|
+
|
|
17
21
|
try {
|
|
18
22
|
// Preprocess: transforms Type, Generic, Union declarations, runs tests
|
|
19
|
-
const preprocessed = preprocess(source)
|
|
23
|
+
const preprocessed = preprocess(source, { dialect })
|
|
20
24
|
|
|
21
25
|
if (preprocessed.testErrors.length > 0) {
|
|
22
26
|
console.error('Test failures:')
|
|
@@ -27,7 +31,7 @@ export async function run(file: string): Promise<void> {
|
|
|
27
31
|
}
|
|
28
32
|
|
|
29
33
|
// Transpile to JS
|
|
30
|
-
const result = transpileToJS(preprocessed.source)
|
|
34
|
+
const result = transpileToJS(preprocessed.source, { dialect })
|
|
31
35
|
|
|
32
36
|
if (result.warnings && result.warnings.length > 0) {
|
|
33
37
|
for (const warning of result.warnings) {
|
package/src/cli/commands/test.ts
CHANGED
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { readFileSync } from 'fs'
|
|
6
|
-
import { tjs } from '../../lang'
|
|
6
|
+
import { tjs, dialectForFilename } from '../../lang'
|
|
7
7
|
|
|
8
8
|
export async function types(file: string): Promise<void> {
|
|
9
9
|
const source = readFileSync(file, 'utf-8')
|
|
10
10
|
|
|
11
|
-
const result = tjs(source)
|
|
11
|
+
const result = tjs(source, { dialect: dialectForFilename(file) })
|
|
12
12
|
|
|
13
13
|
// Output the type information as JSON
|
|
14
14
|
const typeInfo = {
|
package/src/lang/codegen.test.ts
CHANGED
|
@@ -1543,8 +1543,11 @@ function map(arr: [''], counter = strLength): [0] { return arr.map(counter) }`)
|
|
|
1543
1543
|
)
|
|
1544
1544
|
const sig = testResults?.find((t) => t.isSignatureTest)
|
|
1545
1545
|
expect(sig).toBeDefined()
|
|
1546
|
+
// The module couldn't execute (runtime ReferenceError on `x`), so the
|
|
1547
|
+
// test couldn't run → inconclusive, never a build-blocking failure.
|
|
1546
1548
|
expect(sig?.passed).toBe(false)
|
|
1547
|
-
expect(sig?.
|
|
1549
|
+
expect(sig?.inconclusive).toBe(true)
|
|
1550
|
+
expect(sig?.error).toContain('could not be executed')
|
|
1548
1551
|
// Critical: no line attribution → editor won't mark a misleading line
|
|
1549
1552
|
expect(sig?.line).toBeUndefined()
|
|
1550
1553
|
})
|
package/src/lang/core.ts
CHANGED
|
@@ -12,7 +12,7 @@ import type {
|
|
|
12
12
|
FunctionSignature,
|
|
13
13
|
TypeDescriptor,
|
|
14
14
|
} from './types'
|
|
15
|
-
import { parse,
|
|
15
|
+
import { parse, extractFunctions } from './parser'
|
|
16
16
|
import { transformFunction } from './emitters/ast'
|
|
17
17
|
import {
|
|
18
18
|
transpileToJS,
|
|
@@ -26,6 +26,7 @@ export {
|
|
|
26
26
|
preprocess,
|
|
27
27
|
extractTDoc,
|
|
28
28
|
validateSingleFunction,
|
|
29
|
+
extractFunctions,
|
|
29
30
|
} from './parser'
|
|
30
31
|
export { transformFunction } from './emitters/ast'
|
|
31
32
|
|
|
@@ -47,14 +48,15 @@ export function transpile(
|
|
|
47
48
|
vmTarget: true,
|
|
48
49
|
})
|
|
49
50
|
|
|
50
|
-
const
|
|
51
|
+
const { entry, helpers } = extractFunctions(program, options.filename)
|
|
51
52
|
|
|
52
53
|
const { ast, signature, warnings } = transformFunction(
|
|
53
|
-
|
|
54
|
+
entry,
|
|
54
55
|
originalSource,
|
|
55
56
|
returnType,
|
|
56
57
|
options,
|
|
57
|
-
requiredParams
|
|
58
|
+
requiredParams,
|
|
59
|
+
helpers.size > 0 ? helpers : undefined
|
|
58
60
|
)
|
|
59
61
|
|
|
60
62
|
return {
|