virtualmatter 0.6.1 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -6
- package/dist/agents-md.generated.js +1 -1
- package/dist/client.js +37 -0
- package/dist/files.js +139 -18
- package/dist/index.js +35 -7
- package/dist/mcp.js +19 -6
- package/dist/pull.js +17 -8
- package/dist/state.js +77 -2
- package/dist/sync.js +265 -45
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -38,9 +38,16 @@ npx virtualmatter pull https://make.virtualmatter.ai/edit/my-world-<framing-id>
|
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
While `sync` runs, saving a Lua script in your editor deploys it - scripts
|
|
41
|
-
hot-reload in the engine.
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
hot-reload in the engine. Sync never overwrites work you have not synced: if
|
|
42
|
+
a file changed both here and in the live session (while sync ran, or while it
|
|
43
|
+
was stopped), your copy stays put and the remote version lands next to it as
|
|
44
|
+
`<name>.remote-conflict`. Merge, save, and it pushes. Edits made while sync
|
|
45
|
+
was stopped go out when it starts again.
|
|
46
|
+
|
|
47
|
+
Large voxel and terrain files (several GB is normal) are streamed to disk,
|
|
48
|
+
never loaded into memory. Sync does not push a file over 8 MB, the most the
|
|
49
|
+
world accepts in one write. It warns once and leaves that file as it is on
|
|
50
|
+
both sides.
|
|
44
51
|
|
|
45
52
|
`make`, `pull` and `sync` print the world's editor link ending in `?chat=closed`.
|
|
46
53
|
Open it in any browser to watch changes land live - no install needed.
|
|
@@ -56,10 +63,12 @@ Poke at the running engine from another terminal:
|
|
|
56
63
|
|
|
57
64
|
```bash
|
|
58
65
|
npx virtualmatter run-lua --code "return Server.GetInfo()"
|
|
66
|
+
npx virtualmatter run-lua --target client --code "return Client:GetID()" # UI, input, camera
|
|
59
67
|
npx virtualmatter errors
|
|
60
68
|
npx virtualmatter screenshot -o shot.png # default overview
|
|
61
69
|
npx virtualmatter screenshot --target "Player" -o p.png # frame one object
|
|
62
70
|
npx virtualmatter screenshot --at 0,30,40 --rot 0,-35,0 # exact camera
|
|
71
|
+
npx virtualmatter screenshot --client 4 -o ui.png # what client 4 sees, UI included
|
|
63
72
|
# --rot is yaw,pitch,roll
|
|
64
73
|
```
|
|
65
74
|
|
|
@@ -117,9 +126,9 @@ deploying it.
|
|
|
117
126
|
| `pull [target] [dir]` | Mirror a world's file tree. `target` is any link to the world (or your website) or a framing id, and always resolves to the world's editor; omitted, it uses this folder's world or your only one. |
|
|
118
127
|
| `sync [dir]` | Watch + two-way sync with the live session. Ctrl-C to stop. |
|
|
119
128
|
| `open [target]` (`client`) | Optional: open a world in the native desktop client, downloading and signing it in on first use. `--install-only`, `--print` (just the download URL), `--force`. |
|
|
120
|
-
| `run-lua [dir] --code "<lua>" [--target server\|client]` | Execute Lua in the running engine. |
|
|
121
|
-
| `errors [dir]` | Recent engine errors. |
|
|
122
|
-
| `screenshot [dir] [-o out.png]` | Capture the engine's
|
|
129
|
+
| `run-lua [dir] --code "<lua>" [--target server\|client] [--client <id>]` | Execute Lua in the running engine. A client run uses the only connected client unless `--client` names one (list them with `--code "return Server:GetClients()"`). Exit code 3 means the engine did not answer in time and may still be running the code: check its effect before sending code that changes the world again. It is never retried for you. |
|
|
130
|
+
| `errors [dir]` | Recent engine errors. Retried twice when the engine is busy or briefly unreachable, since it only reads. |
|
|
131
|
+
| `screenshot [dir] [-o out.png] [--client <id>]` | Capture the engine's view; `--client` captures through a connected client, UI included. |
|
|
123
132
|
| `mcp [dir] [--framing <id>]` | Run the stdio MCP server. |
|
|
124
133
|
| `login` / `logout` / `whoami` | Device-code sign-in; every other command signs you in automatically when needed. Tokens live in `~/.config/virtualmatter/credentials.json` (mode 0600) and refresh automatically. |
|
|
125
134
|
|
|
@@ -3,4 +3,4 @@
|
|
|
3
3
|
// frontend/content/agent-briefing.md and AGENTS.source.md. The CLI writes
|
|
4
4
|
// this into a world folder only when it cannot fetch the live copy. After
|
|
5
5
|
// changing either source, run: node frontend/scripts/sync-cli-agents-md.mjs
|
|
6
|
-
export const AGENTS_MD_FALLBACK = "# AGENTS.md - working with a Virtual Matter project\n\nThis file tells a coding agent how to think about a Virtual Matter project. The\n`virtualmatter` CLI drops a copy into every local project mirror it creates\n(`virtualmatter pull`); the canonical copy lives at\nhttps://make.virtualmatter.ai/AGENTS.md.\n\n## The rule that comes before everything else\n\n**Build through the CLI. Never through the website's agent.** If you were asked to\nbuild something in Virtual Matter, handed the domain virtualmatter.ai, or told to run\n`npx virtualmatter make`, the CLI is the path - always, from the first step. Do not\nopen virtualmatter.ai or make.virtualmatter.ai in a browser or under computer use in\norder to reach the built-in agent, and do not type prompts into the agent chat in the\n/edit view. Driving that agent puts two models in a relay where one would do: you spend\ntokens describing the work, it spends its own doing it, and what comes back is chat\nrather than files you can read. It is slower, it costs roughly twice the inference, it\nbills that inference to Virtual Matter's platform credits rather than the subscription\nyou already run on, an anonymous browser session cannot be steered after its first\nturn, and a rendered page gives you no engine errors. Everything that agent can do, you\ndo yourself with the commands below.\n\n**The rule holds whoever is signed in.** Being logged in, having credits to spare, or\nworking in an admin account with no limit at all is not an exemption, and neither is\nbeing told the credits do not matter. What is wasteful is the relay itself - two models\ndoing one model's job - not the bill.\n\n**Playtesting is the exception, and it is encouraged.** Once something is built, open\nthe world and try it: the editor link ending in `?chat=closed`, which `make`, `pull` and\n`sync` print, shows your latest changes live in any browser with no install, with\ncomputer use if that is how you drive a window. The native client\n(`npx virtualmatter open`) is optional, for more memory and frame rate. Looking at your\nwork, walking around in it, and playing it is the point. The rule is about who writes\nthe world, not about who is allowed to see it.\n\n**Every link resolves to the editor, and the editor is the only place you build.** Hand\nthe CLI or MCP server any link to a world - its editor, a play or share link, the\nproject page, an embed's iframe src, or the maker's own website - and it works on that\nworld's editor (its make framing). Play servers show the last published version and are\noverwritten on every publish, so work never goes there, and their /play and /g links\nshow that published copy rather than your latest edits.\n\n## The mental model\n\n- The world itself runs on Virtual Matter servers, not on the maker's machine. There is\n one authoritative server simulation per world session, and any number of connected\n clients (browser WASM or the native client).\n- The project's content is SDK Lua scripts plus assets (voxel data, prefabs, images,\n sounds) under a `Montage/` tree. Virtual Matter can publish that tree to a GitHub repo\n in the maker's account for versioning and remixing; pushing to that repo does not feed\n back into the running world today - live edits happen in the session.\n- Changes hot-reload. Saving a `.lua` file in the live session's Montage tree re-runs\n the module and its `Start()` on a fresh instance inside the running world - no restart,\n no build step.\n- Prefabs are reusable scene-graph snippets: JSON files with a `.prefab` extension in the\n Montage folder, instantiated at runtime via `Server:InsertPrefab(asset)`.\n- There is no local runtime - the world cannot run on the maker's machine. The\n `virtualmatter` CLI (npm) is how you reach it from a local harness, and it needs no\n setup: the first command that needs an account signs you in with a device code.\n `npx virtualmatter make \"My world\"` makes a world and mirrors its Montage tree\n into `./my-world`; `npx virtualmatter pull <any Virtual Matter link>` mirrors an\n existing one (/edit, /play, /g, /p, /projects links all work, with or without a\n readable slug); `npx virtualmatter list` shows your worlds; `npx virtualmatter sync`\n live-pushes saves into the running session (which hot-reloads them);\n `npx virtualmatter run-lua` / `errors` / `screenshot` drive the engine;\n `npx virtualmatter open` optionally opens the world in the native desktop client,\n downloaded and signed in on first use; and `npx virtualmatter mcp` serves\n all of it over MCP. A mirrored folder carries `.mcp.json` and `.cursor/mcp.json`\n (so Claude Code and Cursor register the MCP server on their own), this file, and a\n `CLAUDE.md` that imports it. Folders from virtualmatter 0.5.0 on also carry\n `.codex/config.toml`, which Codex loads once you trust the folder; with an older\n CLI, or if Codex does not list the server, register it once with\n `codex mcp add virtualmatter -- npx -y virtualmatter mcp`. In Claude Code outside a\n mirrored folder, use `claude mcp add virtualmatter -- npx -y virtualmatter mcp`. The\n server works before a world\n is selected (`list_projects`, `create_project`, `select_project`), then exposes\n `list_files`, `read_file`, `write_file`, `run_lua`, `get_engine_errors`,\n `capture_screenshot`, `open_native_client`, and `world_info`.\n- Work through the CLI, not the website - see the rule above. Prompting the built-in\n agent to do your building is the one thing to avoid; opening the world to play it is\n not.\n- Seeing your work: `virtualmatter screenshot` shoots a default overview of the world\n origin, `--target <object>` frames one object by name or id, and `--at x,y,z --rot\n yaw,pitch,roll` places the camera exactly. Y is up and -Z is forward, and the\n rotation really is yaw first: yaw 0 faces -Z, pitch -90 looks straight down, pitch 0\n is the horizon. The MCP `capture_screenshot`\n tool takes the same arguments. Verify a change with a screenshot plus `errors` rather\n than assuming a write worked.\n- The mirrored folder's AGENTS.md is this guide followed by the engine SDK's own agent guide\n (the AGENTS.md that lives in every Montage tree and inside the desktop client's\n Data/Sdk/Montage/). The engine guide assumes an in-session agent driving the engine\n through `atomo`; the merged file maps each `atomo` step to the CLI or MCP equivalent, and\n the `Skills/*.md` references it points at are in the folder. The SDK's own tooling\n (`atomo`, `vm_auth.py`, the agent-log hooks) is not mirrored: it only works inside a\n session. Sandboxed agents need network access for every CLI command.\n\n## Reference\n\nThe same facts as terse lists, for quick lookup.\n\n### Project model\n- A project (\"montage\") is one voxel world hosted with Virtual Matter.\n- A project has framings: \"make\" framings (editing sessions) and \"play\"\n framings (public play sessions). A framing is a running session slot\n on a voxel host.\n- Project content = SDK Lua scripts + assets under a Montage/ tree.\n Edits made in the live session hot-reload into the running world.\n The tree can be published to a maker-owned GitHub repo.\n\n### URL shapes (make.virtualmatter.ai)\n- /new builder for people: create a world from a prompt\n (also at https://virtualmatter.ai/); coding agents\n use npx virtualmatter make instead\n- /edit/<framing-id> maker session: live world + agent chat. The chat is\n for people; a coding agent builds with the CLI and\n never prompts it\n (also /edit/<slug>-<framing-id>; the CLI accepts both)\n- /play/<framing-id> play session for a specific framing\n- /g/<id> share URL resolver: 302s to a live session\n\n### APIs (unauthenticated)\n- GET https://make.virtualmatter.ai/api/v1/public/native-clients\n JSON: { iteration, clients: [{ platform, kind, variant, url, filename,\n file_size, branch, commit, match }] } - per-platform native client\n installers (kind \"download\") or store links (kind \"store\", iOS).\n Windows and Linux list two downloads each: variant \"installer\" /\n \"flatpak\" for people, and \"portable\" / \"tarball\", the file tree the\n CLI installs.\n\n### CLI + MCP (npm package \"virtualmatter\", Node >= 20)\n- npx virtualmatter make \"<name>\" make a world, mirror it into ./<slug>\n- npx virtualmatter pull <link> mirror an existing world (any link shape)\n- npx virtualmatter list your worlds with framing ids + URLs\n- npx virtualmatter sync live-push saves into the running world\n- npx virtualmatter open optional: open it in the native client\n- npx virtualmatter mcp stdio MCP server (list_projects,\n create_project, select_project, list_files, read_file, write_file,\n run_lua, get_engine_errors, capture_screenshot, open_native_client,\n world_info)\n- Sign-in happens on first use (device code); no separate login step.\n- These commands are the whole build path. Never prompt the built-in\n agent on the site to do the work; a browser or the native client is\n for playtesting what you built.\n- Any link to a world (editor, play or share link, project page, embed,\n or the maker's own website) resolves to its editor, the only place\n the CLI and MCP server work. Play servers show the last published\n version and are overwritten on publish.\n- Watch changes land live at the editor link ending in ?chat=closed,\n in any browser, no install. The native client is optional.\n- A mirrored folder carries .mcp.json, .cursor/mcp.json, AGENTS.md and\n CLAUDE.md, so Claude Code and Cursor register the server on their own.\n- Codex: folders from 0.5.0 on carry .codex/config.toml, loaded once the\n folder is trusted; otherwise register the server once:\n codex mcp add virtualmatter -- npx -y virtualmatter mcp\n\n## SDK Lua cheat sheet\n\nThe scripting language is Lua 5.4, sandboxed: `os`, `io`, `require`, `package`,\n`dofile`, `loadfile`, `loadstring` are nil. `math`, `string`, `table`, `coroutine`\nremain. No `os.time` - use `Time.time` (sim time), `Time.frame`, or\n`AE:GetDebugTime()` (wall clock).\n\n### Script shape\n\nEvery persistent behavior is a `.lua` file in the Montage tree returning a `self` table:\n\n```lua\nlocal self = {}\nfunction self:Start() end\nfunction self:Update(deltaTime) end\nreturn self\n```\n\nAttach to an object with `obj:AddScript(\"My Folder/Example.lua\", sync)` - the path is\nrelative to the Montage root; `sync = true` replicates the script to clients.\n`obj:FindScript(\"Example\")` returns the live instance.\n\n### Server / client split\n\nThe same script runs on the server and (when synced) on every client. Branch with\n`self.onServer` / `self.onClient`:\n\n```lua\nfunction self:Update(dt)\n if self.onClient then return end -- voxel edits are server-only\n -- authoritative logic here\nend\n```\n\nNothing replicates automatically: set `syncToClients = true` on the script or\nVoxelData component, and use `util:makeNetworkedTable(self, { hp = 100 })` for\nproperties that should sync (server writes, clients read, deltas only).\n`obj.pos` / `obj.rot` do not auto-sync - replicate them yourself.\n\n### RPC\n\n```lua\nself:RPC(\"serverDoSomething\", pos, dir) -- on a client: goes to the server\nfunction self:serverDoSomething(pos, dir, clientID) -- clientID auto-appended\n assert(self.onServer)\nend\n```\n\nOn the server, `self:RPC(...)` fans out to all clients. Reliable, FIFO per direction.\nRequires `self.component.syncToClients = true`.\n\n### Scene and objects (server)\n\n```lua\nlocal ob = Scene:CreateObject(\"Name\")\nob.save = true\nob:AddScript(\"Path/Script.lua\")\nScene:GetObjectByName(\"Name\")\nScene:CloneObject(ob) -- prefer over rebuilding\nob.active = false -- prefer over destroy\nob:AddTag(\"Enemy\"); FindObjectsWithTag(\"Enemy\") -- tags are runtime-only state\n```\n\n### Voxel editing (server only, async)\n\n```lua\nVox:Add(Sphere(pos, 2)):Color(1, 0, 0):Run()\nVox:Add(Box(Vec3(0, -1, 0), Vec3(20, 2, 20))):ForceStatic():Run()\n```\n\nShapes: `Box(center, fullSize)`, `Sphere(center, r)`, `Capsule(p1, p2, r)`,\n`Cylinder(p1, p2, r1[, r2])`. `Run()` is async - chain `:OnFinished(fn)` before\n`:Run()` to read post-commit state. Big edits stall the frame for every client;\nkeep in-game edits small and infrequent.\n\n### Input (event-driven, client input reaches the server)\n\n```lua\nself.component.syncToClients = true\nevents.keyDown.addListener(self, function(key, from) self.keys[key] = true end)\nevents.keyUp.addListener(self, function(key, from) self.keys[key] = false end)\n```\n\nPlayer-driven movement must be client-predicted - never gate the player's own\nfeedback on the server round-trip.\n\n### Physics\n\n```lua\nlocal rb = obj:AddComponent(\"RigidBody\")\nrb:AddImpulse(Vec3(0, 50, 0), obj.pos)\nrb.velocity; rb.mass; rb.gravityScale\n```\n\n### Time and diagnostics\n\n```lua\nTime.dt; Time.time; Time.frame; Time.timeScale\nAE:GetLogValue(\"Raycasts\") -- engine counters, e.g. raycasts this frame\nAE:GetAssets() -- list all assets (APIs take Assets, not paths)\n```\n\n### UI (client-side HUD)\n\nBuild screen UI with the MUI builder: `UI:AddPanel|AddButton|AddLabel|AddSlider|...`\nchained with `:Set{...}`; flexbox-like row-wrap layout.\n\n## Conventions\n\n- Coordinates: Y is up, -Z is forward, X is right.\n- Script files are named in capital case with spaces (`Character Controller.lua`).\n- Check whether a script is already attached before adding it - duplicates run twice.\n- The server is for authority (world edits, spawns, state); clients are for feel\n (prediction, FX, UI).\n\n## Embed or build a world for an existing website\n\nFor a shared project link, fetch `/api/v1/public/embed?target=<encoded-link>`\nand follow `/embed-guide.md`. Use the returned canonical player URL, not the\nmaker's external website URL. Keep the existing website and its design.\n\nFor account discovery or prompt-based creation, connect the hosted MCP at\n`https://make.virtualmatter.ai/api/v1/mcp` using OAuth. Users can sign up in\nthe connection flow. Tools: `list_projects`, `get_embed`, `create_project`\n(with a prompt and stable request_id), `get_build_status`. VM's agent builds\nusing VM credits. Review and share the private world before embedding.\nThe local CLI exposes `embed`, `build --prompt --request-id`, and `build-status`.\n\n## Local conversation history\n\nAfter pulling a project, run `virtualmatter agent-logs setup` once to enable\nfuture chat history capture for local harnesses. It adds Codex, Claude Code,\nCursor, and (when installed) Hermes hooks without replacing existing hooks.\nKeep `virtualmatter sync` or `virtualmatter agent-logs watch` running for retry\nand delayed transcript capture. Follow each harness's normal hook review and\nrestart flow. Use `agent-logs status` to inspect pending uploads and\n`agent-logs disable` to stop capture.\n\nOther harnesses can send portable public event JSONL with\n`virtualmatter agent-logs upload <file> --harness <name> --session <id>`, or use\nMCP `upload_agent_logs`. Supply stable event IDs for retries; omit private\nreasoning, system/developer prompts, credentials, and binary media.\n";
|
|
6
|
+
export const AGENTS_MD_FALLBACK = "# AGENTS.md - working with a Virtual Matter project\n\nThis file tells a coding agent how to think about a Virtual Matter project. The\n`virtualmatter` CLI drops a copy into every local project mirror it creates\n(`virtualmatter pull`); the canonical copy lives at\nhttps://make.virtualmatter.ai/AGENTS.md.\n\n## The rule that comes before everything else\n\n**Build through the CLI. Never through the website's agent.** If you were asked to\nbuild something in Virtual Matter, handed the domain virtualmatter.ai, or told to run\n`npx virtualmatter make`, the CLI is the path - always, from the first step. Do not\nopen virtualmatter.ai or make.virtualmatter.ai in a browser or under computer use in\norder to reach the built-in agent, and do not type prompts into the agent chat in the\n/edit view. Driving that agent puts two models in a relay where one would do: you spend\ntokens describing the work, it spends its own doing it, and what comes back is chat\nrather than files you can read. It is slower, it costs roughly twice the inference, it\nbills that inference to Virtual Matter's platform credits rather than the subscription\nyou already run on, an anonymous browser session cannot be steered after its first\nturn, and a rendered page gives you no engine errors. Everything that agent can do, you\ndo yourself with the commands below.\n\n**The rule holds whoever is signed in.** Being logged in, having credits to spare, or\nworking in an admin account with no limit at all is not an exemption, and neither is\nbeing told the credits do not matter. What is wasteful is the relay itself - two models\ndoing one model's job - not the bill.\n\n**Playtesting is the exception, and it is encouraged.** Once something is built, open\nthe world and try it: the editor link ending in `?chat=closed`, which `make`, `pull` and\n`sync` print, shows your latest changes live in any browser with no install, with\ncomputer use if that is how you drive a window. The native client\n(`npx virtualmatter open`) is optional, for more memory and frame rate. Looking at your\nwork, walking around in it, and playing it is the point. The rule is about who writes\nthe world, not about who is allowed to see it.\n\n**Every link resolves to the editor, and the editor is the only place you build.** Hand\nthe CLI or MCP server any link to a world - its editor, a play or share link, the\nproject page, an embed's iframe src, or the maker's own website - and it works on that\nworld's editor (its make framing). Play servers show the last published version and are\noverwritten on every publish, so work never goes there, and their /play and /g links\nshow that published copy rather than your latest edits.\n\n## The mental model\n\n- The world itself runs on Virtual Matter servers, not on the maker's machine. There is\n one authoritative server simulation per world session, and any number of connected\n clients (browser WASM or the native client).\n- The project's content is SDK Lua scripts plus assets (voxel data, prefabs, images,\n sounds) under a `Montage/` tree. Virtual Matter can publish that tree to a GitHub repo\n in the maker's account for versioning and remixing; pushing to that repo does not feed\n back into the running world today - live edits happen in the session.\n- Changes hot-reload. Saving a `.lua` file in the live session's Montage tree re-runs\n the module and its `Start()` on a fresh instance inside the running world - no restart,\n no build step.\n- Prefabs are reusable scene-graph snippets: JSON files with a `.prefab` extension in the\n Montage folder, instantiated at runtime via `Server:InsertPrefab(asset)`.\n- There is no local runtime - the world cannot run on the maker's machine. The\n `virtualmatter` CLI (npm) is how you reach it from a local harness, and it needs no\n setup: the first command that needs an account signs you in with a device code.\n `npx virtualmatter make \"My world\"` makes a world and mirrors its Montage tree\n into `./my-world`; `npx virtualmatter pull <any Virtual Matter link>` mirrors an\n existing one (/edit, /play, /g, /p, /projects links all work, with or without a\n readable slug); `npx virtualmatter list` shows your worlds; `npx virtualmatter sync`\n live-pushes saves into the running session (which hot-reloads them);\n `npx virtualmatter run-lua` / `errors` / `screenshot` drive the engine;\n `npx virtualmatter open` optionally opens the world in the native desktop client,\n downloaded and signed in on first use; and `npx virtualmatter mcp` serves\n all of it over MCP. A mirrored folder carries `.mcp.json` and `.cursor/mcp.json`\n (so Claude Code and Cursor register the MCP server on their own), this file, and a\n `CLAUDE.md` that imports it. Folders from virtualmatter 0.5.0 on also carry\n `.codex/config.toml`, which Codex loads once you trust the folder; with an older\n CLI, or if Codex does not list the server, register it once with\n `codex mcp add virtualmatter -- npx -y virtualmatter mcp`. In Claude Code outside a\n mirrored folder, use `claude mcp add virtualmatter -- npx -y virtualmatter mcp`. The\n server works before a world\n is selected (`list_projects`, `create_project`, `select_project`), then exposes\n `list_files`, `read_file`, `write_file`, `run_lua`, `get_engine_errors`,\n `capture_screenshot`, `open_native_client`, and `world_info`.\n- Work through the CLI, not the website - see the rule above. Prompting the built-in\n agent to do your building is the one thing to avoid; opening the world to play it is\n not.\n- Seeing your work: `virtualmatter screenshot` shoots a default overview of the world\n origin, `--target <object>` frames one object by name or id, and `--at x,y,z --rot\n yaw,pitch,roll` places the camera exactly. Y is up and -Z is forward, and the\n rotation really is yaw first: yaw 0 faces -Z, pitch -90 looks straight down, pitch 0\n is the horizon. The MCP `capture_screenshot`\n tool takes the same arguments. Verify a change with a screenshot plus `errors` rather\n than assuming a write worked.\n- A `run-lua` that fails with HTTP 504 (`run-lua timed out`, exit code 3 from CLI\n 0.6.4 on; the MCP `run_lua` tool returns the same text) reached the engine, which\n may still be running it. Do not send code that changes the world again: check its\n effect first, by reading back what it sets or taking a screenshot. Any other\n failure never reached the engine and is safe to retry.\n- The mirrored folder's AGENTS.md is this guide followed by the engine SDK's own agent guide\n (the AGENTS.md that lives in every Montage tree and inside the desktop client's\n Data/Sdk/Montage/). The engine guide assumes an in-session agent driving the engine\n through `atomo`; the merged file maps each `atomo` step to the CLI or MCP equivalent, and\n the `Skills/*.md` references it points at are in the folder. The SDK's own tooling\n (`atomo`, `vm_auth.py`, the agent-log hooks) is not mirrored: it only works inside a\n session. Sandboxed agents need network access for every CLI command.\n\n## Reference\n\nThe same facts as terse lists, for quick lookup.\n\n### Project model\n- A project (\"montage\") is one voxel world hosted with Virtual Matter.\n- A project has framings: \"make\" framings (editing sessions) and \"play\"\n framings (public play sessions). A framing is a running session slot\n on a voxel host.\n- Project content = SDK Lua scripts + assets under a Montage/ tree.\n Edits made in the live session hot-reload into the running world.\n The tree can be published to a maker-owned GitHub repo.\n\n### URL shapes (make.virtualmatter.ai)\n- /new builder for people: create a world from a prompt\n (also at https://virtualmatter.ai/); coding agents\n use npx virtualmatter make instead\n- /edit/<framing-id> maker session: live world + agent chat. The chat is\n for people; a coding agent builds with the CLI and\n never prompts it\n (also /edit/<slug>-<framing-id>; the CLI accepts both)\n- /play/<framing-id> play session for a specific framing\n- /g/<id> share URL resolver: 302s to a live session\n\n### APIs (unauthenticated)\n- GET https://make.virtualmatter.ai/api/v1/public/native-clients\n JSON: { iteration, clients: [{ platform, kind, variant, url, filename,\n file_size, branch, commit, match }] } - per-platform native client\n installers (kind \"download\") or store links (kind \"store\", iOS).\n Windows and Linux list two downloads each: variant \"installer\" /\n \"flatpak\" for people, and \"portable\" / \"tarball\", the file tree the\n CLI installs.\n\n### CLI + MCP (npm package \"virtualmatter\", Node >= 20)\n- npx virtualmatter make \"<name>\" make a world, mirror it into ./<slug>\n- npx virtualmatter pull <link> mirror an existing world (any link shape)\n- npx virtualmatter list your worlds with framing ids + URLs\n- npx virtualmatter sync live-push saves into the running world\n- npx virtualmatter open optional: open it in the native client\n- npx virtualmatter mcp stdio MCP server (list_projects,\n create_project, select_project, list_files, read_file, write_file,\n run_lua, get_engine_errors, capture_screenshot, open_native_client,\n world_info)\n- Sign-in happens on first use (device code); no separate login step.\n- These commands are the whole build path. Never prompt the built-in\n agent on the site to do the work; a browser or the native client is\n for playtesting what you built.\n- Any link to a world (editor, play or share link, project page, embed,\n or the maker's own website) resolves to its editor, the only place\n the CLI and MCP server work. Play servers show the last published\n version and are overwritten on publish.\n- Watch changes land live at the editor link ending in ?chat=closed,\n in any browser, no install. The native client is optional.\n- A mirrored folder carries .mcp.json, .cursor/mcp.json, AGENTS.md and\n CLAUDE.md, so Claude Code and Cursor register the server on their own.\n- Codex: folders from 0.5.0 on carry .codex/config.toml, loaded once the\n folder is trusted; otherwise register the server once:\n codex mcp add virtualmatter -- npx -y virtualmatter mcp\n\n## SDK Lua cheat sheet\n\nThe scripting language is Lua 5.4, sandboxed: `os`, `io`, `require`, `package`,\n`dofile`, `loadfile`, `loadstring` are nil. `math`, `string`, `table`, `coroutine`\nremain. No `os.time` - use `Time.time` (sim time), `Time.frame`, or\n`AE:GetDebugTime()` (wall clock).\n\n### Script shape\n\nEvery persistent behavior is a `.lua` file in the Montage tree returning a `self` table:\n\n```lua\nlocal self = {}\nfunction self:Start() end\nfunction self:Update(deltaTime) end\nreturn self\n```\n\nAttach to an object with `obj:AddScript(\"My Folder/Example.lua\", sync)` - the path is\nrelative to the Montage root; `sync = true` replicates the script to clients.\n`obj:FindScript(\"Example\")` returns the live instance.\n\n### Server / client split\n\nThe same script runs on the server and (when synced) on every client. Branch with\n`self.onServer` / `self.onClient`:\n\n```lua\nfunction self:Update(dt)\n if self.onClient then return end -- voxel edits are server-only\n -- authoritative logic here\nend\n```\n\nNothing replicates automatically: set `syncToClients = true` on the script or\nVoxelData component, and use `util:makeNetworkedTable(self, { hp = 100 })` for\nproperties that should sync (server writes, clients read, deltas only).\n`obj.pos` / `obj.rot` do not auto-sync - replicate them yourself.\n\n### RPC\n\n```lua\nself:RPC(\"serverDoSomething\", pos, dir) -- on a client: goes to the server\nfunction self:serverDoSomething(pos, dir, clientID) -- clientID auto-appended\n assert(self.onServer)\nend\n```\n\nOn the server, `self:RPC(...)` fans out to all clients. Reliable, FIFO per direction.\nRequires `self.component.syncToClients = true`.\n\n### Scene and objects (server)\n\n```lua\nlocal ob = Scene:CreateObject(\"Name\")\nob.save = true\nob:AddScript(\"Path/Script.lua\")\nScene:GetObjectByName(\"Name\")\nScene:CloneObject(ob) -- prefer over rebuilding\nob.active = false -- prefer over destroy\nob:AddTag(\"Enemy\"); FindObjectsWithTag(\"Enemy\") -- tags are runtime-only state\n```\n\n### Voxel editing (server only, async)\n\n```lua\nVox:Add(Sphere(pos, 2)):Color(1, 0, 0):Run()\nVox:Add(Box(Vec3(0, -1, 0), Vec3(20, 2, 20))):ForceStatic():Run()\n```\n\nShapes: `Box(center, fullSize)`, `Sphere(center, r)`, `Capsule(p1, p2, r)`,\n`Cylinder(p1, p2, r1[, r2])`. `Run()` is async - chain `:OnFinished(fn)` before\n`:Run()` to read post-commit state. Big edits stall the frame for every client;\nkeep in-game edits small and infrequent.\n\n### Input (event-driven, client input reaches the server)\n\n```lua\nself.component.syncToClients = true\nevents.keyDown.addListener(self, function(key, from) self.keys[key] = true end)\nevents.keyUp.addListener(self, function(key, from) self.keys[key] = false end)\n```\n\nPlayer-driven movement must be client-predicted - never gate the player's own\nfeedback on the server round-trip.\n\n### Physics\n\n```lua\nlocal rb = obj:AddComponent(\"RigidBody\")\nrb:AddImpulse(Vec3(0, 50, 0), obj.pos)\nrb.velocity; rb.mass; rb.gravityScale\n```\n\n### Time and diagnostics\n\n```lua\nTime.dt; Time.time; Time.frame; Time.timeScale\nAE:GetLogValue(\"Raycasts\") -- engine counters, e.g. raycasts this frame\nAE:GetAssets() -- list all assets (APIs take Assets, not paths)\n```\n\n### UI (client-side HUD)\n\nBuild screen UI with the MUI builder: `UI:AddPanel|AddButton|AddLabel|AddSlider|...`\nchained with `:Set{...}`; flexbox-like row-wrap layout.\n\n## Conventions\n\n- Coordinates: Y is up, -Z is forward, X is right.\n- Script files are named in capital case with spaces (`Character Controller.lua`).\n- Check whether a script is already attached before adding it - duplicates run twice.\n- The server is for authority (world edits, spawns, state); clients are for feel\n (prediction, FX, UI).\n\n## Embed or build a world for an existing website\n\nFor a shared project link, fetch `/api/v1/public/embed?target=<encoded-link>`\nand follow `/embed-guide.md`. Use the returned canonical player URL, not the\nmaker's external website URL. Keep the existing website and its design.\n\nFor account discovery or prompt-based creation, connect the hosted MCP at\n`https://make.virtualmatter.ai/api/v1/mcp` using OAuth. Users can sign up in\nthe connection flow. Tools: `list_projects`, `get_embed`, `create_project`\n(with a prompt and stable request_id), `get_build_status`. VM's agent builds\nusing VM credits. Review and share the private world before embedding.\nThe local CLI exposes `embed`, `build --prompt --request-id`, and `build-status`.\n\n## Local conversation history\n\nCapturing this harness's conversation into the world's Agent Logs is the maker's\ndecision, not yours. Do not run `virtualmatter agent-logs setup`, and do not upload\nlogs with `agent-logs upload` or MCP `upload_agent_logs`, unless the maker asks for it\nin their own words. `virtualmatter agent-logs status` shows whether capture is on;\n`virtualmatter agent-logs disable` turns it off.\n";
|
package/dist/client.js
CHANGED
|
@@ -131,6 +131,38 @@ async function download(url, dest, onProgress, fetchFn = fetch) {
|
|
|
131
131
|
}
|
|
132
132
|
fs.renameSync(tmp, dest);
|
|
133
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* The path of `lsregister`, the tool that edits macOS's database of known apps.
|
|
136
|
+
* Not on any PATH - it lives inside the LaunchServices framework.
|
|
137
|
+
*/
|
|
138
|
+
const LSREGISTER = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister";
|
|
139
|
+
/**
|
|
140
|
+
* Take this cached client OUT of macOS's app database.
|
|
141
|
+
*
|
|
142
|
+
* The cache holds a real .app bundle, and macOS registers every bundle it
|
|
143
|
+
* sees. Every copy carries the same bundle id, and the "which app opens
|
|
144
|
+
* virtualmatter:// links" preference is stored per bundle ID, not per path -
|
|
145
|
+
* so macOS picks among the copies by its own rules, and a cached build from
|
|
146
|
+
* last week can win over the one in /Applications. That is not hypothetical:
|
|
147
|
+
* a world link opened a nine-day-old cached client, which then refused to
|
|
148
|
+
* connect to its own world because the client predated a compatibility fix on
|
|
149
|
+
* that same release branch.
|
|
150
|
+
*
|
|
151
|
+
* The CLI launches this copy by path, so it never needs to be registered.
|
|
152
|
+
* Unregistering it leaves the user's installed app as the only candidate.
|
|
153
|
+
* Best-effort by design: this is tidiness, not correctness, and any failure
|
|
154
|
+
* (another OS, a future layout, a locked database) must not break a launch.
|
|
155
|
+
*/
|
|
156
|
+
export function dropFromLaunchServices(appPath, platform = detectPlatform(), runner = spawnSync) {
|
|
157
|
+
if (platform !== "macos" || !appPath.endsWith(".app"))
|
|
158
|
+
return;
|
|
159
|
+
try {
|
|
160
|
+
runner(LSREGISTER, ["-u", appPath], { stdio: "ignore" });
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
/* tidiness only */
|
|
164
|
+
}
|
|
165
|
+
}
|
|
134
166
|
function run(cmd, args) {
|
|
135
167
|
const res = spawnSync(cmd, args, { stdio: ["ignore", "ignore", "pipe"] });
|
|
136
168
|
if (res.error)
|
|
@@ -317,6 +349,8 @@ export async function ensureClientInstalled(opts = {}) {
|
|
|
317
349
|
}
|
|
318
350
|
}
|
|
319
351
|
const installed = { platform, dir, entrypoint, build };
|
|
352
|
+
// Keep the cache out of the OS's app database (see dropFromLaunchServices).
|
|
353
|
+
dropFromLaunchServices(entrypoint, platform);
|
|
320
354
|
fs.writeFileSync(path.join(dir, MARKER), JSON.stringify(installed, null, 2) + "\n");
|
|
321
355
|
const pruned = pruneOtherBuilds(dir);
|
|
322
356
|
if (pruned.length)
|
|
@@ -350,6 +384,9 @@ export function launchClient(installed, montageUrl) {
|
|
|
350
384
|
console.error(`Could not launch the client: ${err.message}`);
|
|
351
385
|
});
|
|
352
386
|
child.unref();
|
|
387
|
+
// Launching re-registers the bundle, so drop it again: the installed app in
|
|
388
|
+
// /Applications should stay the one that opens world links.
|
|
389
|
+
dropFromLaunchServices(installed.entrypoint, installed.platform);
|
|
353
390
|
}
|
|
354
391
|
// ---------------------------------------------------------------- sign-in hand-over
|
|
355
392
|
/**
|
package/dist/files.js
CHANGED
|
@@ -2,7 +2,25 @@
|
|
|
2
2
|
* Client for the session file + engine API served at
|
|
3
3
|
* https://<voxel_host>/s/<session_id>/api/montage/<framing_id>/...
|
|
4
4
|
*/
|
|
5
|
+
import crypto from "node:crypto";
|
|
6
|
+
import fs from "node:fs";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { Readable, Transform } from "node:stream";
|
|
9
|
+
import { pipeline } from "node:stream/promises";
|
|
5
10
|
import { authorizedFetch } from "./api.js";
|
|
11
|
+
/**
|
|
12
|
+
* The session API's per-write cap (session-agent server/api/files.py
|
|
13
|
+
* MAX_WRITE_BYTES). A bigger PUT is answered 413, so sync skips such files
|
|
14
|
+
* up front instead of reading gigabytes of voxel data just to be refused.
|
|
15
|
+
*/
|
|
16
|
+
export const MAX_WRITE_BYTES = 8 * 1024 * 1024;
|
|
17
|
+
/**
|
|
18
|
+
* Largest body `get()` reads into memory. Node refuses a Buffer past
|
|
19
|
+
* 2^31-1 bytes, and worlds do carry multi-GB voxel/terrain files - Rooftop
|
|
20
|
+
* Rumble's 2.7 GB one crashed every pull. Anything that writes a file to
|
|
21
|
+
* disk uses `download()`, which streams and has no limit.
|
|
22
|
+
*/
|
|
23
|
+
export const MAX_BUFFERED_BYTES = 256 * 1024 * 1024;
|
|
6
24
|
/**
|
|
7
25
|
* The reason an engine call failed, read out of the response body.
|
|
8
26
|
*
|
|
@@ -12,23 +30,42 @@ import { authorizedFetch } from "./api.js";
|
|
|
12
30
|
* browser instead. Falls back to the status when the body says nothing.
|
|
13
31
|
*/
|
|
14
32
|
export async function engineErrorMessage(res, what) {
|
|
15
|
-
|
|
33
|
+
const detail = await errorDetail(res);
|
|
34
|
+
return `${what} failed: HTTP ${res.status}${detail ? ` - ${detail}` : ""}`;
|
|
35
|
+
}
|
|
36
|
+
async function errorDetail(res) {
|
|
16
37
|
try {
|
|
17
38
|
const body = (await res.json());
|
|
18
39
|
const d = body.detail;
|
|
19
40
|
if (typeof d === "string")
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
41
|
+
return d;
|
|
42
|
+
if (d && typeof d === "object" && typeof d.message === "string") {
|
|
43
|
+
return d.message;
|
|
23
44
|
}
|
|
24
|
-
|
|
25
|
-
detail = JSON.stringify(d);
|
|
45
|
+
return d ? JSON.stringify(d) : "";
|
|
26
46
|
}
|
|
27
47
|
catch {
|
|
28
|
-
|
|
48
|
+
return ""; // non-JSON body, e.g. a proxy's own 504 page
|
|
29
49
|
}
|
|
30
|
-
return `${what} failed: HTTP ${res.status}${detail ? ` - ${detail}` : ""}`;
|
|
31
50
|
}
|
|
51
|
+
/**
|
|
52
|
+
* The engine did not answer an engine call in time (HTTP 504). The session
|
|
53
|
+
* relay answers 504 `engine_timeout` only when the call reached the engine,
|
|
54
|
+
* so for run-lua the snippet may still be running: resending it can run a
|
|
55
|
+
* mutation twice. Nothing retries it, and the CLI exits with
|
|
56
|
+
* ENGINE_TIMEOUT_EXIT_CODE so a script can tell it from a plain failure.
|
|
57
|
+
*/
|
|
58
|
+
export class EngineTimeoutError extends Error {
|
|
59
|
+
constructor(message) {
|
|
60
|
+
super(message);
|
|
61
|
+
this.name = "EngineTimeoutError";
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
export const ENGINE_TIMEOUT_EXIT_CODE = 3;
|
|
65
|
+
const RUN_LUA_TIMEOUT = "The engine did not answer in time; it may still be running this Lua. Do not resend code that changes the world: check its effect first.";
|
|
66
|
+
/** How long `engineErrors` waits before each retry of a 503/504; it only reads. */
|
|
67
|
+
const READ_RETRY_DELAYS_MS = [1_000, 3_000];
|
|
68
|
+
const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
32
69
|
export class ConflictError extends Error {
|
|
33
70
|
filePath;
|
|
34
71
|
constructor(filePath) {
|
|
@@ -37,6 +74,10 @@ export class ConflictError extends Error {
|
|
|
37
74
|
this.name = "ConflictError";
|
|
38
75
|
}
|
|
39
76
|
}
|
|
77
|
+
function tooBigToBuffer(filePath, size, limit) {
|
|
78
|
+
const mb = (n) => `${Math.round(n / (1024 * 1024))} MB`;
|
|
79
|
+
return `${filePath} is ${mb(size)}, too large to read into memory (limit ${mb(limit)}). It is in the folder \`virtualmatter pull\` mirrored - read it from disk.`;
|
|
80
|
+
}
|
|
40
81
|
function encodePath(p) {
|
|
41
82
|
return p.split("/").map(encodeURIComponent).join("/");
|
|
42
83
|
}
|
|
@@ -58,14 +99,49 @@ export function normalizeEtag(v) {
|
|
|
58
99
|
s = s.slice(1, -1);
|
|
59
100
|
return s;
|
|
60
101
|
}
|
|
102
|
+
/**
|
|
103
|
+
* Write a response body to `destAbs` without holding it in memory, hashing
|
|
104
|
+
* it on the way. Lands in a temp file beside the target and is renamed into
|
|
105
|
+
* place, so the target is either the old file or the whole new one - never a
|
|
106
|
+
* truncated half that a later sync would push back as an edit.
|
|
107
|
+
*/
|
|
108
|
+
export async function streamToFile(res, destAbs) {
|
|
109
|
+
fs.mkdirSync(path.dirname(destAbs), { recursive: true });
|
|
110
|
+
const tmp = `${destAbs}.vmdownload-${process.pid}-${crypto.randomBytes(4).toString("hex")}`;
|
|
111
|
+
const hasher = crypto.createHash("sha256");
|
|
112
|
+
let size = 0;
|
|
113
|
+
const tap = new Transform({
|
|
114
|
+
transform(chunk, _enc, cb) {
|
|
115
|
+
hasher.update(chunk);
|
|
116
|
+
size += chunk.length;
|
|
117
|
+
cb(null, chunk);
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
try {
|
|
121
|
+
const source = res.body
|
|
122
|
+
? Readable.fromWeb(res.body)
|
|
123
|
+
: Readable.from([]);
|
|
124
|
+
await pipeline(source, tap, fs.createWriteStream(tmp));
|
|
125
|
+
fs.renameSync(tmp, destAbs);
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
fs.rmSync(tmp, { force: true });
|
|
129
|
+
throw err;
|
|
130
|
+
}
|
|
131
|
+
return { hash: hasher.digest("hex"), size };
|
|
132
|
+
}
|
|
61
133
|
export class FilesClient {
|
|
62
134
|
sessionBase;
|
|
63
135
|
framingId;
|
|
64
136
|
fetchFn;
|
|
65
|
-
|
|
137
|
+
maxBufferedBytes;
|
|
138
|
+
sleep;
|
|
139
|
+
constructor(sessionBase, framingId, fetchFn = fetch, maxBufferedBytes = MAX_BUFFERED_BYTES, sleep = defaultSleep) {
|
|
66
140
|
this.sessionBase = sessionBase;
|
|
67
141
|
this.framingId = framingId;
|
|
68
142
|
this.fetchFn = fetchFn;
|
|
143
|
+
this.maxBufferedBytes = maxBufferedBytes;
|
|
144
|
+
this.sleep = sleep;
|
|
69
145
|
}
|
|
70
146
|
get apiRoot() {
|
|
71
147
|
return `${this.sessionBase}/api/montage/${encodeURIComponent(this.framingId)}`;
|
|
@@ -87,8 +163,30 @@ export class FilesClient {
|
|
|
87
163
|
const res = await authorizedFetch(`${this.apiRoot}/files/content/${encodePath(filePath)}`, {}, this.fetchFn);
|
|
88
164
|
if (!res.ok)
|
|
89
165
|
throw new Error(`Downloading ${filePath} failed: HTTP ${res.status}`);
|
|
90
|
-
const
|
|
91
|
-
|
|
166
|
+
const declared = Number(res.headers.get("Content-Length"));
|
|
167
|
+
if (Number.isFinite(declared) && declared > this.maxBufferedBytes) {
|
|
168
|
+
await res.body?.cancel();
|
|
169
|
+
throw new Error(tooBigToBuffer(filePath, declared, this.maxBufferedBytes));
|
|
170
|
+
}
|
|
171
|
+
// No (or a lying) Content-Length: count as it arrives and stop at the limit.
|
|
172
|
+
const chunks = [];
|
|
173
|
+
let size = 0;
|
|
174
|
+
if (res.body) {
|
|
175
|
+
for await (const chunk of Readable.fromWeb(res.body)) {
|
|
176
|
+
size += chunk.length;
|
|
177
|
+
if (size > this.maxBufferedBytes)
|
|
178
|
+
throw new Error(tooBigToBuffer(filePath, size, this.maxBufferedBytes));
|
|
179
|
+
chunks.push(chunk);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return { bytes: Buffer.concat(chunks, size), etag: normalizeEtag(res.headers.get("ETag")) };
|
|
183
|
+
}
|
|
184
|
+
async download(filePath, destAbs) {
|
|
185
|
+
const res = await authorizedFetch(`${this.apiRoot}/files/content/${encodePath(filePath)}`, {}, this.fetchFn);
|
|
186
|
+
if (!res.ok)
|
|
187
|
+
throw new Error(`Downloading ${filePath} failed: HTTP ${res.status}`);
|
|
188
|
+
const { hash, size } = await streamToFile(res, destAbs);
|
|
189
|
+
return { etag: normalizeEtag(res.headers.get("ETag")), hash, size };
|
|
92
190
|
}
|
|
93
191
|
async put(filePath, body, opts) {
|
|
94
192
|
const headers = { "Content-Type": "application/octet-stream" };
|
|
@@ -112,12 +210,20 @@ export class FilesClient {
|
|
|
112
210
|
throw new Error(`Deleting ${filePath} failed: HTTP ${res.status}`);
|
|
113
211
|
}
|
|
114
212
|
}
|
|
115
|
-
async runLua(code, target) {
|
|
213
|
+
async runLua(code, target, clientId) {
|
|
214
|
+
const payload = { code, target };
|
|
215
|
+
if (target === "client" && clientId !== undefined)
|
|
216
|
+
payload.client_id = clientId;
|
|
116
217
|
const res = await authorizedFetch(`${this.apiRoot}/engine/run-lua`, {
|
|
117
218
|
method: "POST",
|
|
118
219
|
headers: { "Content-Type": "application/json" },
|
|
119
|
-
body: JSON.stringify(
|
|
220
|
+
body: JSON.stringify(payload),
|
|
120
221
|
}, this.fetchFn);
|
|
222
|
+
if (res.status === 504) {
|
|
223
|
+
// Never retried: the engine may still be running the snippet.
|
|
224
|
+
const detail = (await errorDetail(res)) || RUN_LUA_TIMEOUT;
|
|
225
|
+
throw new EngineTimeoutError(`run-lua timed out: HTTP 504 - ${detail}`);
|
|
226
|
+
}
|
|
121
227
|
if (!res.ok)
|
|
122
228
|
throw new Error(await engineErrorMessage(res, "run-lua"));
|
|
123
229
|
// Server envelope: {output: <text>} - the engine bridge's text
|
|
@@ -126,11 +232,24 @@ export class FilesClient {
|
|
|
126
232
|
return body.output;
|
|
127
233
|
}
|
|
128
234
|
async engineErrors() {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
235
|
+
// Reading the error buffer changes nothing, so a busy (504) or briefly
|
|
236
|
+
// unreachable (503) engine is worth another try.
|
|
237
|
+
for (let attempt = 0;; attempt++) {
|
|
238
|
+
const res = await authorizedFetch(`${this.apiRoot}/engine/errors`, {}, this.fetchFn);
|
|
239
|
+
if (res.ok) {
|
|
240
|
+
const body = (await res.json());
|
|
241
|
+
return body.output;
|
|
242
|
+
}
|
|
243
|
+
const retryable = res.status === 503 || res.status === 504;
|
|
244
|
+
const delay = READ_RETRY_DELAYS_MS[attempt];
|
|
245
|
+
if (retryable && delay !== undefined) {
|
|
246
|
+
await res.body?.cancel();
|
|
247
|
+
await this.sleep(delay);
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
const message = await engineErrorMessage(res, "Fetching engine errors");
|
|
251
|
+
throw res.status === 504 ? new EngineTimeoutError(message) : new Error(message);
|
|
252
|
+
}
|
|
134
253
|
}
|
|
135
254
|
async screenshot(opts = {}) {
|
|
136
255
|
const body = {};
|
|
@@ -151,6 +270,8 @@ export class FilesClient {
|
|
|
151
270
|
camera_rot_z: rz,
|
|
152
271
|
});
|
|
153
272
|
}
|
|
273
|
+
if (opts.clientId !== undefined)
|
|
274
|
+
body.client_id = opts.clientId;
|
|
154
275
|
const res = await authorizedFetch(`${this.apiRoot}/engine/screenshot`, {
|
|
155
276
|
method: "POST",
|
|
156
277
|
headers: { "Content-Type": "application/json" },
|
package/dist/index.js
CHANGED
|
@@ -9,7 +9,7 @@ import { NotLoggedInError, clearCredentials, loadCredentials, pollForToken, save
|
|
|
9
9
|
import { openInBrowser } from "./browser.js";
|
|
10
10
|
import { detectPlatform, ensureClientInstalled, launchClient, seedNativeAuth } from "./client.js";
|
|
11
11
|
import { apiBase, packageVersion } from "./config.js";
|
|
12
|
-
import { FilesClient } from "./files.js";
|
|
12
|
+
import { ENGINE_TIMEOUT_EXIT_CODE, EngineTimeoutError, FilesClient } from "./files.js";
|
|
13
13
|
import { runMcpServer } from "./mcp.js";
|
|
14
14
|
import { pullCommand } from "./pull.js";
|
|
15
15
|
import { NeedsChoiceError, formatProjectList, makeFramingOf, resolveImplicitTarget, resolveTarget, } from "./resolve.js";
|
|
@@ -42,9 +42,15 @@ program
|
|
|
42
42
|
"install. The native client (`virtualmatter open`) is optional, for more memory",
|
|
43
43
|
"and frame rate.",
|
|
44
44
|
].join("\n"));
|
|
45
|
-
function fail(message) {
|
|
45
|
+
function fail(message, code = 1) {
|
|
46
46
|
console.error(message);
|
|
47
|
-
process.exit(
|
|
47
|
+
process.exit(code);
|
|
48
|
+
}
|
|
49
|
+
/** Exit with the failure's own code: an engine timeout is not a plain failure. */
|
|
50
|
+
function failWith(err) {
|
|
51
|
+
if (err instanceof EngineTimeoutError)
|
|
52
|
+
fail(err.message, ENGINE_TIMEOUT_EXIT_CODE);
|
|
53
|
+
fail(err instanceof Error ? err.message : String(err));
|
|
48
54
|
}
|
|
49
55
|
/** The device-code sign-in, shared by `login` and the automatic first-use sign-in. */
|
|
50
56
|
async function signIn() {
|
|
@@ -76,10 +82,10 @@ async function run(fn) {
|
|
|
76
82
|
return;
|
|
77
83
|
}
|
|
78
84
|
catch (again) {
|
|
79
|
-
|
|
85
|
+
failWith(again);
|
|
80
86
|
}
|
|
81
87
|
}
|
|
82
|
-
|
|
88
|
+
failWith(err);
|
|
83
89
|
}
|
|
84
90
|
}
|
|
85
91
|
/** Resolve a session-scoped FilesClient from a synced dir's state file. */
|
|
@@ -115,6 +121,14 @@ function slugFor(name) {
|
|
|
115
121
|
}
|
|
116
122
|
function reportPull(result, dest, watchUrl) {
|
|
117
123
|
console.log(`Pulled ${result.fileCount} files into ${dest}.`);
|
|
124
|
+
if (result.failed.length > 0) {
|
|
125
|
+
// Reported, not fatal: the folder and its state file are usable, and the
|
|
126
|
+
// first `sync` retries whatever is missing.
|
|
127
|
+
console.error(`${result.failed.length} file(s) could not be downloaded (\`virtualmatter sync\` will retry them):`);
|
|
128
|
+
for (const f of result.failed)
|
|
129
|
+
console.error(` ${f.path}: ${f.error}`);
|
|
130
|
+
process.exitCode = 1;
|
|
131
|
+
}
|
|
118
132
|
const files = result.agentFiles;
|
|
119
133
|
if (files && files.written.length > 0) {
|
|
120
134
|
console.log(`Wrote ${files.written.join(", ")} so Claude Code, Codex, and Cursor find their way around.`);
|
|
@@ -296,16 +310,21 @@ program
|
|
|
296
310
|
}));
|
|
297
311
|
program
|
|
298
312
|
.command("run-lua")
|
|
299
|
-
.description("Execute Lua in the running engine")
|
|
313
|
+
.description("Execute Lua in the running engine. Exit code 3 means the engine did not answer in time and may still be running the code: check its effect before sending it again")
|
|
300
314
|
.argument("[dir]", "a synced folder", ".")
|
|
301
315
|
.requiredOption("--code <lua>", "Lua source to execute")
|
|
302
316
|
.option("--target <target>", "server or client", "server")
|
|
317
|
+
.option("--client <id>", "client id for --target client (default: the only connected client; list them with --code 'return Server:GetClients()')")
|
|
303
318
|
.action((dir, opts) => run(async () => {
|
|
304
319
|
if (opts.target !== "server" && opts.target !== "client") {
|
|
305
320
|
fail("--target must be server or client");
|
|
306
321
|
}
|
|
322
|
+
const clientId = opts.client === undefined ? undefined : clientIdFlag(opts.client);
|
|
323
|
+
if (clientId !== undefined && opts.target !== "client") {
|
|
324
|
+
fail("--client needs --target client");
|
|
325
|
+
}
|
|
307
326
|
const { client } = await clientForDir(path.resolve(dir));
|
|
308
|
-
const result = await client.runLua(opts.code, opts.target);
|
|
327
|
+
const result = await client.runLua(opts.code, opts.target, clientId);
|
|
309
328
|
console.log(JSON.stringify({ result }, null, 2));
|
|
310
329
|
}));
|
|
311
330
|
program
|
|
@@ -316,6 +335,13 @@ program
|
|
|
316
335
|
const { client } = await clientForDir(path.resolve(dir));
|
|
317
336
|
console.log(JSON.stringify(await client.engineErrors(), null, 2));
|
|
318
337
|
}));
|
|
338
|
+
/** Parse a --client flag: a positive integer client id. */
|
|
339
|
+
function clientIdFlag(value) {
|
|
340
|
+
const n = Number(value);
|
|
341
|
+
if (!Number.isInteger(n) || n < 1)
|
|
342
|
+
fail(`--client takes a client id (a positive integer), got "${value}"`);
|
|
343
|
+
return n;
|
|
344
|
+
}
|
|
319
345
|
/** Parse an "x,y,z" flag into a numeric triple. */
|
|
320
346
|
function triple(value, flag) {
|
|
321
347
|
const parts = value.split(",").map((p) => Number(p.trim()));
|
|
@@ -333,6 +359,7 @@ program
|
|
|
333
359
|
.option("--rot <yaw,pitch,roll>", "camera rotation in degrees, yaw first (default: 0,-45,0)")
|
|
334
360
|
.option("--target <id-or-name>", "frame this object instead of using a camera pose")
|
|
335
361
|
.option("--distance <meters>", "framing distance for --target (default: from the object's bounds)")
|
|
362
|
+
.option("--client <id>", "capture through this connected client, UI included; with no --at/--rot/--target it is exactly what that client sees")
|
|
336
363
|
.action((dir, opts) => run(async () => {
|
|
337
364
|
const { client } = await clientForDir(path.resolve(dir));
|
|
338
365
|
const png = await client.screenshot({
|
|
@@ -340,6 +367,7 @@ program
|
|
|
340
367
|
rot: opts.rot ? triple(opts.rot, "--rot") : undefined,
|
|
341
368
|
target: opts.target,
|
|
342
369
|
distance: opts.distance === undefined ? undefined : Number(opts.distance),
|
|
370
|
+
clientId: opts.client === undefined ? undefined : clientIdFlag(opts.client),
|
|
343
371
|
});
|
|
344
372
|
fs.writeFileSync(opts.out, png);
|
|
345
373
|
console.log(`Wrote ${opts.out} (${png.length} bytes).`);
|
package/dist/mcp.js
CHANGED
|
@@ -195,7 +195,7 @@ export async function runMcpServer(dir, framingFlag) {
|
|
|
195
195
|
let pulled = null;
|
|
196
196
|
if (pull_to) {
|
|
197
197
|
const result = await pullCommand(framingId, pull_to);
|
|
198
|
-
pulled = { dir: result.dir, file_count: result.fileCount, agent_files: result.agentFiles };
|
|
198
|
+
pulled = { dir: result.dir, file_count: result.fileCount, failed: result.failed, agent_files: result.agentFiles };
|
|
199
199
|
}
|
|
200
200
|
return textResult(JSON.stringify({ ...projectSummary(project), selected: true, pulled }, null, 2));
|
|
201
201
|
});
|
|
@@ -211,7 +211,7 @@ export async function runMcpServer(dir, framingFlag) {
|
|
|
211
211
|
let pulled = null;
|
|
212
212
|
if (pull_to) {
|
|
213
213
|
const result = await pullCommand(resolved.framingId, pull_to);
|
|
214
|
-
pulled = { dir: result.dir, file_count: result.fileCount, agent_files: result.agentFiles };
|
|
214
|
+
pulled = { dir: result.dir, file_count: result.fileCount, failed: result.failed, agent_files: result.agentFiles };
|
|
215
215
|
}
|
|
216
216
|
return textResult(JSON.stringify({
|
|
217
217
|
framing_id: resolved.framingId,
|
|
@@ -249,17 +249,23 @@ export async function runMcpServer(dir, framingFlag) {
|
|
|
249
249
|
return textResult(JSON.stringify({ ok: true, path: filePath, etag }));
|
|
250
250
|
});
|
|
251
251
|
server.registerTool("run_lua", {
|
|
252
|
-
description: "Execute a Lua snippet in the running Virtual Matter engine and return its result. target \"server\" (default) runs in the server-side Lua state where game logic lives; \"client\" runs in
|
|
252
|
+
description: "Execute a Lua snippet in the running Virtual Matter engine and return its result. target \"server\" (default) runs in the server-side Lua state where game logic lives; \"client\" runs in a connected client (someone must have the world open) - that is where UI, input and the camera live. With several clients connected, pass client_id; list them with target server and code 'return Server:GetClients()'. Use this to inspect live state, call engine APIs, or nudge the scene without editing files. If it fails with \"run-lua timed out\", the engine may still be running the snippet: check its effect before sending code that changes the world again.",
|
|
253
253
|
inputSchema: {
|
|
254
254
|
code: z.string().describe("Lua source to execute"),
|
|
255
255
|
target: z
|
|
256
256
|
.enum(["server", "client"])
|
|
257
257
|
.optional()
|
|
258
258
|
.describe("Which Lua state to run in (default: server)"),
|
|
259
|
+
client_id: z
|
|
260
|
+
.number()
|
|
261
|
+
.int()
|
|
262
|
+
.positive()
|
|
263
|
+
.optional()
|
|
264
|
+
.describe("For target client: which connected client (default: the only one connected)"),
|
|
259
265
|
},
|
|
260
|
-
}, async ({ code, target }) => {
|
|
266
|
+
}, async ({ code, target, client_id }) => {
|
|
261
267
|
const client = await ctx.getClient();
|
|
262
|
-
const result = await client.runLua(code, target ?? "server");
|
|
268
|
+
const result = await client.runLua(code, target ?? "server", client_id);
|
|
263
269
|
return textResult(JSON.stringify({ result }, null, 2));
|
|
264
270
|
});
|
|
265
271
|
server.registerTool("get_engine_errors", {
|
|
@@ -286,13 +292,20 @@ export async function runMcpServer(dir, framingFlag) {
|
|
|
286
292
|
.length(3)
|
|
287
293
|
.optional()
|
|
288
294
|
.describe("Camera rotation in degrees as [yaw, pitch, roll] - that order. Yaw 0 faces -Z; pitch -90 looks straight down, 0 is the horizon. Default [0, -45, 0] looks down at the origin."),
|
|
295
|
+
client_id: z
|
|
296
|
+
.number()
|
|
297
|
+
.int()
|
|
298
|
+
.positive()
|
|
299
|
+
.optional()
|
|
300
|
+
.describe("Capture through this connected client, UI (HUD, menus) included - a server capture never shows UI. With no target/at/rot it is exactly what that client sees."),
|
|
289
301
|
},
|
|
290
|
-
}, async ({ target, at, rot }) => {
|
|
302
|
+
}, async ({ target, at, rot, client_id }) => {
|
|
291
303
|
const client = await ctx.getClient();
|
|
292
304
|
const png = await client.screenshot({
|
|
293
305
|
target,
|
|
294
306
|
at: at,
|
|
295
307
|
rot: rot,
|
|
308
|
+
clientId: client_id,
|
|
296
309
|
});
|
|
297
310
|
return {
|
|
298
311
|
content: [
|
package/dist/pull.js
CHANGED
|
@@ -5,7 +5,7 @@ import { resolveEditSession, sessionBaseUrl } from "./api.js";
|
|
|
5
5
|
import { composeAgentsMd, fetchAgentsMd, writeAgentFiles } from "./agentfiles.js";
|
|
6
6
|
import { FilesClient } from "./files.js";
|
|
7
7
|
import { isIgnoredPath } from "./ignore.js";
|
|
8
|
-
import { saveState } from "./state.js";
|
|
8
|
+
import { adoptLocal, saveState } from "./state.js";
|
|
9
9
|
export { fetchAgentsMd } from "./agentfiles.js";
|
|
10
10
|
export { AGENTS_MD_FALLBACK } from "./agents-md.generated.js";
|
|
11
11
|
/** Download workers per pull. The tree is ~1000 small files; sequential
|
|
@@ -50,8 +50,9 @@ export async function fetchEngineAgentsMd(client) {
|
|
|
50
50
|
export async function pullTree(client, dir, framingId) {
|
|
51
51
|
fs.mkdirSync(dir, { recursive: true });
|
|
52
52
|
const remote = (await client.list()).filter((f) => !isIgnoredPath(f.path));
|
|
53
|
-
const state = { framing_id: framingId, etags: {} };
|
|
53
|
+
const state = { framing_id: framingId, etags: {}, local: {} };
|
|
54
54
|
const progress = makeProgress(remote.length);
|
|
55
|
+
const failed = [];
|
|
55
56
|
let next = 0;
|
|
56
57
|
const worker = async () => {
|
|
57
58
|
for (;;) {
|
|
@@ -59,18 +60,26 @@ export async function pullTree(client, dir, framingId) {
|
|
|
59
60
|
if (index >= remote.length)
|
|
60
61
|
return;
|
|
61
62
|
const file = remote[index];
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
try {
|
|
64
|
+
// streamed to disk: a multi-GB voxel file must not pass through a Buffer
|
|
65
|
+
const { etag, hash } = await client.download(file.path, path.join(dir, file.path));
|
|
66
|
+
state.etags[file.path] = etag ?? file.etag;
|
|
67
|
+
// what the disk holds now, so the first sync can tell local edits apart
|
|
68
|
+
adoptLocal(state, dir, file.path, hash);
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
// One bad file must not cost the other thousand, nor the state file:
|
|
72
|
+
// without it the folder cannot sync at all. A file left out here has
|
|
73
|
+
// no etag, so the first `sync` pulls it again.
|
|
74
|
+
failed.push({ path: file.path, error: err instanceof Error ? err.message : String(err) });
|
|
75
|
+
}
|
|
67
76
|
progress.tick(file.path);
|
|
68
77
|
}
|
|
69
78
|
};
|
|
70
79
|
await Promise.all(Array.from({ length: Math.min(PULL_CONCURRENCY, remote.length) }, () => worker()));
|
|
71
80
|
progress.done();
|
|
72
81
|
saveState(dir, state);
|
|
73
|
-
return { dir, framingId, fileCount: remote.length };
|
|
82
|
+
return { dir, framingId, fileCount: remote.length - failed.length, failed };
|
|
74
83
|
}
|
|
75
84
|
export async function pullCommand(framingId, dir) {
|
|
76
85
|
console.log(`Resolving session for framing ${framingId} (this can cold-start one) ...`);
|
package/dist/state.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
/** The .virtualmatter.json state file: framing id + per-file
|
|
1
|
+
/** The .virtualmatter.json state file: framing id + per-file sync bookkeeping. */
|
|
2
|
+
import crypto from "node:crypto";
|
|
2
3
|
import fs from "node:fs";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { STATE_FILE } from "./config.js";
|
|
@@ -11,7 +12,7 @@ export function loadState(dir) {
|
|
|
11
12
|
const parsed = JSON.parse(raw);
|
|
12
13
|
if (typeof parsed.framing_id !== "string")
|
|
13
14
|
return null;
|
|
14
|
-
return { framing_id: parsed.framing_id, etags: parsed.etags ?? {} };
|
|
15
|
+
return { framing_id: parsed.framing_id, etags: parsed.etags ?? {}, local: parsed.local ?? {} };
|
|
15
16
|
}
|
|
16
17
|
catch {
|
|
17
18
|
return null;
|
|
@@ -30,3 +31,77 @@ export function requireState(dir) {
|
|
|
30
31
|
}
|
|
31
32
|
return state;
|
|
32
33
|
}
|
|
34
|
+
export function sha256Hex(bytes) {
|
|
35
|
+
return crypto.createHash("sha256").update(bytes).digest("hex");
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* sha256 of a file read in chunks. `readFileSync` cannot load a file past
|
|
39
|
+
* 2 GiB at all, and multi-GB voxel files are normal in a world.
|
|
40
|
+
*/
|
|
41
|
+
export function hashFileSync(abs) {
|
|
42
|
+
const hasher = crypto.createHash("sha256");
|
|
43
|
+
const buf = Buffer.allocUnsafe(1024 * 1024);
|
|
44
|
+
const fd = fs.openSync(abs, "r");
|
|
45
|
+
try {
|
|
46
|
+
for (;;) {
|
|
47
|
+
const n = fs.readSync(fd, buf, 0, buf.length, null);
|
|
48
|
+
if (n === 0)
|
|
49
|
+
break;
|
|
50
|
+
hasher.update(buf.subarray(0, n));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
fs.closeSync(fd);
|
|
55
|
+
}
|
|
56
|
+
return hasher.digest("hex");
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Record a file whose content hash is already known to match its last sync,
|
|
60
|
+
* from a stat alone. How a state file from before `local` records existed
|
|
61
|
+
* gets upgraded as files are first checked, so the next check is stat-only.
|
|
62
|
+
*/
|
|
63
|
+
export function adoptLocal(state, dir, relPath, hash) {
|
|
64
|
+
const st = fs.statSync(path.join(dir, relPath));
|
|
65
|
+
state.local ??= {};
|
|
66
|
+
state.local[relPath] = { hash, size: st.size, mtime_ms: st.mtimeMs };
|
|
67
|
+
}
|
|
68
|
+
/** Record `bytes` (just written to or read from `relPath`) as the synced local copy. */
|
|
69
|
+
export function recordLocal(state, dir, relPath, bytes) {
|
|
70
|
+
const st = fs.statSync(path.join(dir, relPath));
|
|
71
|
+
state.local ??= {};
|
|
72
|
+
state.local[relPath] = { hash: sha256Hex(bytes), size: st.size, mtime_ms: st.mtimeMs };
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The current content hash of a local file, or null if it does not exist.
|
|
76
|
+
* Reuses the recorded hash when size and mtime are unchanged, so a start
|
|
77
|
+
* over a big world does not re-read every voxel file.
|
|
78
|
+
*/
|
|
79
|
+
export function localHash(state, dir, relPath) {
|
|
80
|
+
const abs = path.join(dir, relPath);
|
|
81
|
+
let st;
|
|
82
|
+
try {
|
|
83
|
+
st = fs.statSync(abs);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
if (!st.isFile())
|
|
89
|
+
return null;
|
|
90
|
+
const rec = state.local?.[relPath];
|
|
91
|
+
if (rec && rec.size === st.size && rec.mtime_ms === st.mtimeMs)
|
|
92
|
+
return rec.hash;
|
|
93
|
+
return hashFileSync(abs);
|
|
94
|
+
}
|
|
95
|
+
const SHA256_HEX = /^[0-9a-f]{64}$/;
|
|
96
|
+
/**
|
|
97
|
+
* The content hash of `relPath` as of its last sync, or undefined when
|
|
98
|
+
* unknown. Falls back to the etag for state files written before `local`
|
|
99
|
+
* existed - the session API's etags ARE the sha256 of the content.
|
|
100
|
+
*/
|
|
101
|
+
export function syncedHash(state, relPath) {
|
|
102
|
+
const rec = state.local?.[relPath];
|
|
103
|
+
if (rec)
|
|
104
|
+
return rec.hash;
|
|
105
|
+
const etag = state.etags[relPath];
|
|
106
|
+
return etag !== undefined && SHA256_HEX.test(etag) ? etag : undefined;
|
|
107
|
+
}
|
package/dist/sync.js
CHANGED
|
@@ -1,21 +1,29 @@
|
|
|
1
1
|
import { startLogSync } from "./agent-logs.js";
|
|
2
2
|
/**
|
|
3
|
-
* The sync hot loop:
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
3
|
+
* The sync hot loop: reconcile both sides at start, watch the local tree and
|
|
4
|
+
* push edits with stored etags, poll the remote listing and pull files whose
|
|
5
|
+
* etag changed remotely.
|
|
6
|
+
*
|
|
7
|
+
* NEVER OVERWRITE UNSYNCED WORK. Every decision compares each side against
|
|
8
|
+
* what it looked like at the last sync - the server's etag, and the local
|
|
9
|
+
* record (content hash) in the state file - so a side that moved while the
|
|
10
|
+
* other did not is copied across, and a file BOTH sides changed is a
|
|
11
|
+
* conflict: the local file stays exactly as it is, the remote version lands
|
|
12
|
+
* next to it as <name>.remote-conflict, and nothing is pushed until you save.
|
|
13
|
+
* Conflicts never kill the loop.
|
|
7
14
|
*/
|
|
8
15
|
import fs from "node:fs";
|
|
9
16
|
import path from "node:path";
|
|
10
|
-
import { ConflictError } from "./files.js";
|
|
17
|
+
import { ConflictError, MAX_WRITE_BYTES } from "./files.js";
|
|
11
18
|
import { isIgnoredPath } from "./ignore.js";
|
|
12
|
-
import { loadState, saveState } from "./state.js";
|
|
19
|
+
import { loadState, adoptLocal, localHash, recordLocal, saveState, sha256Hex, syncedHash, } from "./state.js";
|
|
13
20
|
const consoleLogger = {
|
|
14
21
|
info: (m) => console.log(m),
|
|
15
22
|
warn: (m) => console.warn(m),
|
|
16
23
|
};
|
|
17
24
|
/** How long after our own local write of a pulled file we ignore watcher events for it. */
|
|
18
25
|
const SELF_WRITE_GRACE_MS = 2000;
|
|
26
|
+
export const CONFLICT_SUFFIX = ".remote-conflict";
|
|
19
27
|
export class SyncEngine {
|
|
20
28
|
client;
|
|
21
29
|
dir;
|
|
@@ -24,12 +32,17 @@ export class SyncEngine {
|
|
|
24
32
|
state;
|
|
25
33
|
/** relPath -> timestamp of a write WE made (pull), so the watcher skips it. */
|
|
26
34
|
selfWrites = new Map();
|
|
35
|
+
/** State changed without being written yet (see flush). */
|
|
36
|
+
dirty = false;
|
|
37
|
+
/** relPath -> "size:mtime" of an oversized file already warned about, so the scan does not repeat it every few seconds. */
|
|
38
|
+
tooBigWarned = new Map();
|
|
27
39
|
constructor(client, dir, state, opts = {}) {
|
|
28
40
|
this.client = client;
|
|
29
41
|
this.dir = dir;
|
|
30
42
|
this.log = opts.logger ?? consoleLogger;
|
|
31
43
|
this.now = opts.now ?? Date.now;
|
|
32
44
|
this.state = state;
|
|
45
|
+
this.state.local ??= {};
|
|
33
46
|
}
|
|
34
47
|
get framingId() {
|
|
35
48
|
return this.state.framing_id;
|
|
@@ -53,46 +66,170 @@ export class SyncEngine {
|
|
|
53
66
|
}
|
|
54
67
|
return true;
|
|
55
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Has the local copy changed since its last sync? `true` also when the
|
|
71
|
+
* base is unknown but the file exists - "maybe" must be treated as "yes",
|
|
72
|
+
* or an unknown base is exactly how local work gets pulled over.
|
|
73
|
+
*/
|
|
74
|
+
localChanged(relPath) {
|
|
75
|
+
const now = localHash(this.state, this.dir, relPath);
|
|
76
|
+
if (now === null)
|
|
77
|
+
return false;
|
|
78
|
+
const base = syncedHash(this.state, relPath);
|
|
79
|
+
if (base !== undefined && now === base) {
|
|
80
|
+
const rec = this.state.local?.[relPath];
|
|
81
|
+
if (!rec || rec.hash !== now) {
|
|
82
|
+
// Unchanged, but only known so by reading it (no record yet, or
|
|
83
|
+
// the mtime moved with the content the same). Record it so the
|
|
84
|
+
// next check is a stat - the backstop scan runs every few seconds.
|
|
85
|
+
adoptLocal(this.state, this.dir, relPath, now);
|
|
86
|
+
this.dirty = true;
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
const st = fs.statSync(this.abs(relPath));
|
|
90
|
+
if (st.mtimeMs !== rec.mtime_ms || st.size !== rec.size) {
|
|
91
|
+
adoptLocal(this.state, this.dir, relPath, now);
|
|
92
|
+
this.dirty = true;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
return true;
|
|
98
|
+
}
|
|
99
|
+
/** Record changes made while checking files (adopted records) - batched, one write per pass. */
|
|
100
|
+
flush() {
|
|
101
|
+
if (!this.dirty)
|
|
102
|
+
return;
|
|
103
|
+
this.dirty = false;
|
|
104
|
+
this.persist();
|
|
105
|
+
}
|
|
56
106
|
async pullFile(relPath, etag) {
|
|
57
|
-
const { bytes, etag: gotEtag } = await this.client.get(relPath);
|
|
58
|
-
const absPath = this.abs(relPath);
|
|
59
|
-
fs.mkdirSync(path.dirname(absPath), { recursive: true });
|
|
60
107
|
this.recordSelfWrite(relPath);
|
|
61
|
-
|
|
108
|
+
// streamed: voxel and terrain files run to several GB
|
|
109
|
+
const { etag: gotEtag, hash } = await this.client.download(relPath, this.abs(relPath));
|
|
110
|
+
this.recordSelfWrite(relPath); // the rename is the write the watcher sees; restart the grace from it
|
|
62
111
|
this.state.etags[relPath] = gotEtag ?? etag;
|
|
112
|
+
adoptLocal(this.state, this.dir, relPath, hash);
|
|
63
113
|
this.persist();
|
|
64
114
|
}
|
|
65
115
|
/**
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
116
|
+
* Both sides changed: keep the local file untouched, put the remote copy
|
|
117
|
+
* beside it, and point the stored etag at the remote version so that the
|
|
118
|
+
* next deliberate save of the local file is what wins. The local record is
|
|
119
|
+
* left alone, so the file still reads as locally changed.
|
|
120
|
+
*/
|
|
121
|
+
async writeConflict(relPath, remoteEtag) {
|
|
122
|
+
try {
|
|
123
|
+
const conflictPath = this.abs(relPath) + CONFLICT_SUFFIX;
|
|
124
|
+
const { etag } = await this.client.download(relPath, conflictPath);
|
|
125
|
+
const e = etag ?? remoteEtag;
|
|
126
|
+
if (e)
|
|
127
|
+
this.state.etags[relPath] = e;
|
|
128
|
+
this.persist();
|
|
129
|
+
this.log.warn(`conflict on ${relPath}: it changed both here and in the world. Your file is untouched; ` +
|
|
130
|
+
`the world's version is saved as ${relPath}${CONFLICT_SUFFIX} - merge, save, and it will push.`);
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
this.log.warn(`conflict on ${relPath}, and fetching the remote version failed: ${String(err)}`);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* A conflict the user has not resolved yet: the sidecar is still there and
|
|
138
|
+
* the local file has not been saved since it was written. Such a file is
|
|
139
|
+
* never pushed automatically - only a save does that.
|
|
140
|
+
*/
|
|
141
|
+
unresolvedConflict(relPath) {
|
|
142
|
+
try {
|
|
143
|
+
const side = fs.statSync(this.abs(relPath) + CONFLICT_SUFFIX);
|
|
144
|
+
const mine = fs.statSync(this.abs(relPath));
|
|
145
|
+
return side.mtimeMs >= mine.mtimeMs;
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Initial reconcile, per file, against the last sync:
|
|
153
|
+
*
|
|
154
|
+
* local unchanged, remote changed -> pull
|
|
155
|
+
* local changed, remote unchanged -> push
|
|
156
|
+
* both changed (or both new, and different) -> conflict (local kept)
|
|
157
|
+
* local only, never synced -> push (create)
|
|
158
|
+
* remote only -> pull
|
|
159
|
+
*
|
|
160
|
+
* A file deleted locally while sync was off is pulled back rather than
|
|
161
|
+
* deleted remotely: a start never destroys anything on either side.
|
|
69
162
|
*/
|
|
70
163
|
async initialSync() {
|
|
71
164
|
const remote = await this.client.list();
|
|
72
165
|
let pulled = 0;
|
|
73
166
|
let pushed = 0;
|
|
74
|
-
|
|
167
|
+
let conflicts = 0;
|
|
168
|
+
const remoteByPath = new Map();
|
|
75
169
|
for (const file of remote) {
|
|
76
170
|
if (isIgnoredPath(file.path))
|
|
77
171
|
continue;
|
|
78
|
-
|
|
172
|
+
remoteByPath.set(file.path, file);
|
|
79
173
|
const known = this.state.etags[file.path];
|
|
80
|
-
const
|
|
81
|
-
if (
|
|
174
|
+
const current = localHash(this.state, this.dir, file.path);
|
|
175
|
+
if (current === null) {
|
|
82
176
|
await this.pullFile(file.path, file.etag);
|
|
83
177
|
pulled++;
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (current === file.etag) {
|
|
181
|
+
// same content on both sides (etags are content hashes): just
|
|
182
|
+
// bring the bookkeeping up to date
|
|
183
|
+
const rec = this.state.local?.[file.path];
|
|
184
|
+
if (known !== file.etag || !rec || rec.hash !== current) {
|
|
185
|
+
this.state.etags[file.path] = file.etag;
|
|
186
|
+
adoptLocal(this.state, this.dir, file.path, current);
|
|
187
|
+
this.dirty = true;
|
|
188
|
+
}
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
const remoteChanged = known !== file.etag;
|
|
192
|
+
const localChanged = this.localChanged(file.path);
|
|
193
|
+
if (!localChanged && remoteChanged) {
|
|
194
|
+
await this.pullFile(file.path, file.etag);
|
|
195
|
+
this.log.info(`pulled ${file.path}`);
|
|
196
|
+
pulled++;
|
|
197
|
+
}
|
|
198
|
+
else if (localChanged && !remoteChanged) {
|
|
199
|
+
if (this.unresolvedConflict(file.path)) {
|
|
200
|
+
this.log.warn(`${file.path} still has an unresolved ${file.path}${CONFLICT_SUFFIX} - not pushing it. ` +
|
|
201
|
+
`Merge, save the file (or delete the ${CONFLICT_SUFFIX} file), and it will push.`);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
if (await this.pushFile(file.path))
|
|
205
|
+
pushed++;
|
|
206
|
+
}
|
|
207
|
+
else if (localChanged && remoteChanged) {
|
|
208
|
+
await this.writeConflict(file.path, file.etag);
|
|
209
|
+
conflicts++;
|
|
84
210
|
}
|
|
85
211
|
}
|
|
86
|
-
// Local files the remote has never seen get created remotely.
|
|
87
212
|
for (const relPath of this.walkLocal()) {
|
|
88
|
-
if (
|
|
213
|
+
if (remoteByPath.has(relPath))
|
|
89
214
|
continue;
|
|
90
|
-
if (this.state.etags[relPath]
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
215
|
+
if (this.state.etags[relPath] === undefined) {
|
|
216
|
+
// never synced: new here, create it remotely
|
|
217
|
+
if (await this.pushFile(relPath, { createOnly: true }))
|
|
218
|
+
pushed++;
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
// Synced once, gone remotely. Untouched here: leave it (a remote
|
|
222
|
+
// delete is not ours to undo). Edited here since: that is work, so it
|
|
223
|
+
// is re-created rather than stranded.
|
|
224
|
+
if (this.localChanged(relPath)) {
|
|
225
|
+
this.log.warn(`${relPath} was deleted in the world but edited here - re-creating it.`);
|
|
226
|
+
delete this.state.etags[relPath];
|
|
227
|
+
if (await this.pushFile(relPath, { createOnly: true }))
|
|
228
|
+
pushed++;
|
|
229
|
+
}
|
|
94
230
|
}
|
|
95
|
-
|
|
231
|
+
this.flush();
|
|
232
|
+
return { pulled, pushed, conflicts };
|
|
96
233
|
}
|
|
97
234
|
*walkLocal(rel = "") {
|
|
98
235
|
const absDir = path.join(this.dir, rel);
|
|
@@ -113,41 +250,58 @@ export class SyncEngine {
|
|
|
113
250
|
yield relPath;
|
|
114
251
|
}
|
|
115
252
|
}
|
|
253
|
+
/** Push the local file. Returns false when there was nothing to send or it became a conflict. */
|
|
116
254
|
async pushFile(relPath, opts) {
|
|
255
|
+
if (this.tooBigToPush(relPath))
|
|
256
|
+
return false;
|
|
117
257
|
const body = fs.readFileSync(this.abs(relPath));
|
|
118
258
|
const known = this.state.etags[relPath];
|
|
259
|
+
// unchanged since the last sync: nothing to send (a touch, or the
|
|
260
|
+
// watcher and the local scan both reporting the same save)
|
|
261
|
+
if (known !== undefined && syncedHash(this.state, relPath) === sha256Hex(body))
|
|
262
|
+
return false;
|
|
119
263
|
const createOnly = opts?.createOnly ?? known === undefined;
|
|
120
264
|
try {
|
|
121
265
|
const etag = await this.client.put(relPath, body, createOnly ? { createOnly: true } : { etag: known });
|
|
122
266
|
this.state.etags[relPath] = etag;
|
|
267
|
+
recordLocal(this.state, this.dir, relPath, body);
|
|
123
268
|
this.persist();
|
|
124
269
|
this.log.info(`pushed ${relPath}`);
|
|
270
|
+
return true;
|
|
125
271
|
}
|
|
126
272
|
catch (err) {
|
|
127
273
|
if (err instanceof ConflictError) {
|
|
128
|
-
await this.
|
|
129
|
-
return;
|
|
274
|
+
await this.writeConflict(relPath);
|
|
275
|
+
return false;
|
|
130
276
|
}
|
|
131
277
|
throw err;
|
|
132
278
|
}
|
|
133
279
|
}
|
|
134
|
-
/**
|
|
135
|
-
|
|
280
|
+
/**
|
|
281
|
+
* Over the session API's write cap: the server would answer 413, so do not
|
|
282
|
+
* read the file at all (a multi-GB read would also exhaust memory). Warned
|
|
283
|
+
* once per version of the file. The file stays as it is on both sides.
|
|
284
|
+
*/
|
|
285
|
+
tooBigToPush(relPath) {
|
|
286
|
+
let st;
|
|
136
287
|
try {
|
|
137
|
-
|
|
138
|
-
const conflictPath = this.abs(relPath) + ".remote-conflict";
|
|
139
|
-
fs.mkdirSync(path.dirname(conflictPath), { recursive: true });
|
|
140
|
-
fs.writeFileSync(conflictPath, bytes);
|
|
141
|
-
if (etag)
|
|
142
|
-
this.state.etags[relPath] = etag;
|
|
143
|
-
// Deliberately NOT overwriting the local file - the user resolves.
|
|
144
|
-
this.persist();
|
|
145
|
-
this.log.warn(`conflict on ${relPath}: the remote copy changed while you edited it. ` +
|
|
146
|
-
`Remote version saved as ${relPath}.remote-conflict - merge, save, and it will push.`);
|
|
288
|
+
st = fs.statSync(this.abs(relPath));
|
|
147
289
|
}
|
|
148
|
-
catch
|
|
149
|
-
|
|
290
|
+
catch {
|
|
291
|
+
return false;
|
|
150
292
|
}
|
|
293
|
+
if (st.size <= MAX_WRITE_BYTES) {
|
|
294
|
+
this.tooBigWarned.delete(relPath);
|
|
295
|
+
return false;
|
|
296
|
+
}
|
|
297
|
+
const key = `${st.size}:${st.mtimeMs}`;
|
|
298
|
+
if (this.tooBigWarned.get(relPath) !== key) {
|
|
299
|
+
this.tooBigWarned.set(relPath, key);
|
|
300
|
+
const mb = (n) => `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
|
301
|
+
this.log.warn(`not pushing ${relPath} (${mb(st.size)}): the world accepts at most ${mb(MAX_WRITE_BYTES)} per file. ` +
|
|
302
|
+
`Edit large voxel/terrain data in the engine instead; it is left unchanged in the world.`);
|
|
303
|
+
}
|
|
304
|
+
return true;
|
|
151
305
|
}
|
|
152
306
|
/** Watcher callback for a local add/change. */
|
|
153
307
|
async handleLocalChange(relPath) {
|
|
@@ -178,12 +332,13 @@ export class SyncEngine {
|
|
|
178
332
|
try {
|
|
179
333
|
await this.client.delete(norm, known);
|
|
180
334
|
delete this.state.etags[norm];
|
|
335
|
+
delete this.state.local?.[norm];
|
|
181
336
|
this.persist();
|
|
182
337
|
this.log.info(`deleted ${norm} remotely`);
|
|
183
338
|
}
|
|
184
339
|
catch (err) {
|
|
185
340
|
if (err instanceof ConflictError) {
|
|
186
|
-
await this.
|
|
341
|
+
await this.writeConflict(norm);
|
|
187
342
|
return;
|
|
188
343
|
}
|
|
189
344
|
this.log.warn(`remote delete of ${norm} failed: ${String(err)}`);
|
|
@@ -192,7 +347,8 @@ export class SyncEngine {
|
|
|
192
347
|
/**
|
|
193
348
|
* One remote poll tick: pull files whose remote etag differs from our
|
|
194
349
|
* bookkeeping. Files we just pushed match by etag, so they are skipped
|
|
195
|
-
* naturally.
|
|
350
|
+
* naturally. A file with local changes that have not gone out yet is a
|
|
351
|
+
* conflict, never a pull. Returns the pulled paths.
|
|
196
352
|
*/
|
|
197
353
|
async pollOnce() {
|
|
198
354
|
const remote = await this.client.list();
|
|
@@ -200,23 +356,81 @@ export class SyncEngine {
|
|
|
200
356
|
for (const file of remote) {
|
|
201
357
|
if (isIgnoredPath(file.path))
|
|
202
358
|
continue;
|
|
203
|
-
|
|
359
|
+
const exists = fs.existsSync(this.abs(file.path));
|
|
360
|
+
if (this.state.etags[file.path] === file.etag && exists)
|
|
361
|
+
continue;
|
|
362
|
+
if (!exists && this.state.etags[file.path] === file.etag) {
|
|
363
|
+
// Deleted here, unchanged in the world since the last sync: a delete
|
|
364
|
+
// the watcher has not delivered yet, not a file to restore. Pulling it
|
|
365
|
+
// back also marked it as our own write, so the late unlink event was
|
|
366
|
+
// then ignored and the file came back for good.
|
|
367
|
+
await this.handleLocalDelete(file.path);
|
|
204
368
|
continue;
|
|
369
|
+
}
|
|
370
|
+
if (exists) {
|
|
371
|
+
const current = localHash(this.state, this.dir, file.path);
|
|
372
|
+
if (current === file.etag) {
|
|
373
|
+
this.state.etags[file.path] = file.etag;
|
|
374
|
+
adoptLocal(this.state, this.dir, file.path, current);
|
|
375
|
+
this.persist();
|
|
376
|
+
continue;
|
|
377
|
+
}
|
|
378
|
+
if (this.localChanged(file.path)) {
|
|
379
|
+
// writeConflict points the etag at this remote version, so the
|
|
380
|
+
// check at the top of the loop keeps it from firing again
|
|
381
|
+
await this.writeConflict(file.path, file.etag);
|
|
382
|
+
continue;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
205
385
|
await this.pullFile(file.path, file.etag);
|
|
206
386
|
pulled.push(file.path);
|
|
207
387
|
this.log.info(`pulled ${file.path}`);
|
|
208
388
|
}
|
|
209
389
|
return pulled;
|
|
210
390
|
}
|
|
391
|
+
/**
|
|
392
|
+
* Backstop for the watcher: push every local file that changed since its
|
|
393
|
+
* last sync, and create new ones. File-system events get lost (editors
|
|
394
|
+
* that write in place, network drives, a burst while the queue is busy);
|
|
395
|
+
* a stat walk every few seconds cannot miss a save. Unchanged files cost a
|
|
396
|
+
* stat and no read. Returns the pushed paths.
|
|
397
|
+
*/
|
|
398
|
+
async scanLocal() {
|
|
399
|
+
const pushed = [];
|
|
400
|
+
for (const relPath of this.walkLocal()) {
|
|
401
|
+
if (this.isSelfWrite(relPath))
|
|
402
|
+
continue;
|
|
403
|
+
const known = this.state.etags[relPath];
|
|
404
|
+
if (known !== undefined && !this.localChanged(relPath))
|
|
405
|
+
continue;
|
|
406
|
+
if (known !== undefined && this.unresolvedConflict(relPath))
|
|
407
|
+
continue;
|
|
408
|
+
try {
|
|
409
|
+
if (await this.pushFile(relPath))
|
|
410
|
+
pushed.push(relPath);
|
|
411
|
+
}
|
|
412
|
+
catch (err) {
|
|
413
|
+
this.log.warn(`push of ${relPath} failed: ${String(err)}`);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
this.flush();
|
|
417
|
+
return pushed;
|
|
418
|
+
}
|
|
211
419
|
}
|
|
212
420
|
/** Run the full watch loop until SIGINT. Not unit-tested directly; the decisions above are. */
|
|
213
421
|
export async function runSyncLoop(engine, dir) {
|
|
214
422
|
const { default: chokidar } = await import("chokidar");
|
|
215
|
-
const { pulled, pushed } = await engine.initialSync();
|
|
216
|
-
console.log(`initial sync: ${pulled} pulled, ${pushed} pushed
|
|
423
|
+
const { pulled, pushed, conflicts } = await engine.initialSync();
|
|
424
|
+
console.log(`initial sync: ${pulled} pulled, ${pushed} pushed` +
|
|
425
|
+
(conflicts ? `, ${conflicts} conflict${conflicts === 1 ? "" : "s"} (see above)` : "") +
|
|
426
|
+
`. Watching ${dir} ...`);
|
|
217
427
|
const stopLogs = startLogSync(dir);
|
|
218
428
|
const watcher = chokidar.watch(dir, {
|
|
219
429
|
ignoreInitial: true,
|
|
430
|
+
// Wait for a write to settle before reading it. Without this an editor
|
|
431
|
+
// that rewrites a file in place fired "change" mid-write and the sync
|
|
432
|
+
// pushed a truncated file.
|
|
433
|
+
awaitWriteFinish: { stabilityThreshold: 250, pollInterval: 50 },
|
|
220
434
|
ignored: (p) => {
|
|
221
435
|
const rel = path.relative(dir, p).replace(/\\/g, "/");
|
|
222
436
|
return rel.length > 0 && isIgnoredPath(rel);
|
|
@@ -231,6 +445,12 @@ export async function runSyncLoop(engine, dir) {
|
|
|
231
445
|
watcher.on("unlink", (p) => enqueue(() => engine.handleLocalDelete(path.relative(dir, p))));
|
|
232
446
|
const interval = setInterval(() => {
|
|
233
447
|
enqueue(async () => {
|
|
448
|
+
try {
|
|
449
|
+
await engine.scanLocal();
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
console.warn(`local scan failed: ${String(err)}`);
|
|
453
|
+
}
|
|
234
454
|
try {
|
|
235
455
|
await engine.pollOnce();
|
|
236
456
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "virtualmatter",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.4",
|
|
4
4
|
"description": "CLI + MCP server for building with Virtual Matter - list and create worlds, sync their files, run Lua, capture screenshots, open the native client, and wire coding agents into a live session.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|