ravensight-playtest 0.5.2 → 0.7.0

package/addons/ravensight_simulate/simulate.gd

@@ -0,0 +1,360 @@
+extends Node
+## Ravensight simulate addon.
+##
+## Speaks the Simulation Runner protocol (simulation.start, simulation.step, one
+## JSON object per line) on stdin and stdout, so a Godot game can be played with
+## no window, no network and no model, on a hosted runner or on this machine.
+##
+## The CLI injects this into a throwaway copy of the project at export time, so
+## it is never in the customer's project. It does nothing unless the game was
+## started with `-- --ravensight-simulate` AND the build carries the setting
+## below, and it then blocks the main thread between steps: the game can only
+## move when the runner asks it to.
+##
+## Determinism: start the engine with `--fixed-fps 60`. Every frame is then one
+## physics tick and one process pass with a fixed delta. The seed is taken in
+## this node's _ready, before the first scene exists, so a scene's first randi()
+## already comes from it.
+##
+## Stdout: the protocol owns it. Every print() the game makes is off for the
+## whole process (Engine.print_to_stdout), and this node turns printing on only
+## for the length of one answer. That needs the project setting
+## application/run/flush_stdout_on_print, or a pipe would hold the answer back
+## forever; the CLI writes it into the throwaway copy.
+
+const FLAG := "--ravensight-simulate"
+const STATE_GROUP := "ravensight_state"
+const TICKS_PER_STEP := 15
+const BOOT_TICKS := 2
+const MAX_NODES := 4000
+const MAX_DIGEST_SUBTREES := 40
+const MAX_TEXT_CHARS := 120
+const MAX_ACTIONS := 64  # The protocol allows 256; nothing generic needs more.
+const MAX_METRICS := 32
+const WAIT_ACTION := "wait"
+const BUTTON_PREFIX := "button:"
+const MENU_ACTIONS := ["ui_accept", "ui_up", "ui_down", "ui_left", "ui_right"]
+const STEP_CAP_OUTCOME := "step_cap"
+const QUIT_OUTCOME := "game_quit"
+const PROTOCOL_VERSION := 1
+
+var _enabled := false
+var _request_id: Variant = null
+var _ticks_left := 0
+var _held: Array[String] = []
+var _offered: Array[String] = []
+var _steps := 0
+var _max_steps := 1
+
+
+func _enter_tree() -> void:
+	if FLAG not in OS.get_cmdline_user_args():
+		return
+	_enabled = true
+	Engine.print_to_stdout = false
+	process_mode = Node.PROCESS_MODE_ALWAYS
+	process_priority = -2147483648
+
+
+func _ready() -> void:
+	if not _enabled:
+		set_process(false)
+		queue_free()
+		return
+	# Before the first scene, so the scene sees the seed.
+	var message := _read_message()
+	if message.get("method", "") != "simulation.start":
+		_die()
+		return
+	var params: Dictionary = message.get("params", {})
+	_request_id = message.get("id")
+	_max_steps = maxi(1, int(params.get("max_steps", 1)))
+	seed(str(params.get("seed", "")).hash())
+	_ticks_left = BOOT_TICKS
+
+
+## The game quit in the middle of a step (a Quit button, a clean game over). That
+## is an ending and not a crash, so the step gets a terminal answer on the way out.
+func _exit_tree() -> void:
+	if not _enabled or _request_id == null:
+		return
+	_send({"done": true, "state_hash": ("quit:%d" % _steps).sha256_text().substr(0, 32), "actions": [], "metrics": {}, "outcome": QUIT_OUTCOME})
+
+
+func _process(_delta: float) -> void:
+	if not _enabled:
+		return
+	if _ticks_left > 0:
+		_ticks_left -= 1
+		if _ticks_left > 0:
+			return
+		_release_held()
+		_answer()
+	var message := _read_message()
+	if message.get("method", "") != "simulation.step":
+		_die()
+		return
+	_request_id = message.get("id")
+	var action := str((message.get("params", {}) as Dictionary).get("action", ""))
+	if action not in _offered:
+		_die()
+		return
+	_steps += 1
+	if action.begins_with(BUTTON_PREFIX):
+		var button: BaseButton = _buttons().get(action)
+		if button != null:
+			button.pressed.emit()
+	elif action != WAIT_ACTION:
+		Input.action_press(action)
+		_held.append(action)
+		var input := InputEventAction.new()
+		input.action = action
+		input.pressed = true
+		Input.parse_input_event(input)
+	_ticks_left = TICKS_PER_STEP
+
+
+func _release_held() -> void:
+	for action in _held:
+		Input.action_release(action)
+		var input := InputEventAction.new()
+		input.action = action
+		input.pressed = false
+		Input.parse_input_event(input)
+	_held.clear()
+
+
+## Blocks until the runner writes a line. An empty read is the runner gone.
+func _read_message() -> Dictionary:
+	var line := OS.read_string_from_stdin().strip_edges()
+	if line.is_empty():
+		_die()
+		return {}
+	var parsed: Variant = JSON.parse_string(line)
+	if typeof(parsed) != TYPE_DICTIONARY:
+		_die()
+		return {}
+	return parsed
+
+
+## Never a clean quit in the middle of a request: the runner must see this as a
+## game that stopped answering, not as an answer.
+func _die() -> void:
+	OS.kill(OS.get_process_id())
+
+
+func _answer() -> void:
+	var states := _game_states()
+	var flat := _flat(states)
+	var done := bool(flat.get("simulation_done", false))
+	var outcome: Variant = flat.get("simulation_outcome")
+	if not done and _steps >= _max_steps:
+		done = true
+		outcome = STEP_CAP_OUTCOME
+	_offered = [] if done else _action_ids()
+	var result := {
+		"done": done,
+		"state_hash": _state_hash(states),
+		"actions": _offered.map(func(id: String) -> Dictionary: return {"id": id}),
+		"metrics": _metrics(flat),
+	}
+	if done and outcome != null:
+		result["outcome"] = str(outcome).substr(0, 80)
+	_send(result)
+
+
+func _send(result: Dictionary) -> void:
+	var text := JSON.stringify({"jsonrpc": "2.0", "id": _request_id, "result": result}, "", false)
+	_request_id = null
+	Engine.print_to_stdout = true
+	print(text)
+	Engine.print_to_stdout = false
+
+
+## What can be done now: wait, the game's own input actions, the menu keys, and
+## one action per button on screen, so a generic policy can get past a title
+## screen. Sorted, so the list does not depend on declaration order.
+func _action_ids() -> Array[String]:
+	var ids: Array[String] = [WAIT_ACTION]
+	for action in InputMap.get_actions():
+		var id := str(action)
+		if id in MENU_ACTIONS or not (id.begins_with("ui_") or id.begins_with("spatial_editor") or id.begins_with("editor")):
+			ids.append(id)
+	for id in _buttons().keys():
+		ids.append(str(id))
+	ids.sort()
+	return ids.slice(0, MAX_ACTIONS)
+
+
+## Visible, enabled buttons in the current scene, keyed by their action id. A
+## second button with the same text is not offered: the id must name one thing.
+func _buttons() -> Dictionary:
+	var out := {}
+	var scene := get_tree().current_scene
+	if scene != null:
+		_collect_buttons(scene, out)
+	return out
+
+
+func _collect_buttons(node: Node, out: Dictionary) -> void:
+	if node is BaseButton:
+		var button := node as BaseButton
+		var label := _text_of(button).strip_edges().substr(0, 100)
+		var id := BUTTON_PREFIX + label
+		if button.is_visible_in_tree() and not button.disabled and not label.is_empty() and not out.has(id):
+			out[id] = button
+	for child in node.get_children():
+		_collect_buttons(child, out)
+
+
+func _game_states() -> Dictionary:
+	var states := {}
+	for node in get_tree().get_nodes_in_group(STATE_GROUP):
+		if not node.has_method("_ravensight_state"):
+			continue
+		var raw: Variant = node.call("_ravensight_state")
+		if typeof(raw) == TYPE_DICTIONARY:
+			states[str(node.get_path())] = raw
+	return states
+
+
+func _flat(states: Dictionary) -> Dictionary:
+	var out := {}
+	var paths := states.keys()
+	paths.sort()
+	for path in paths:
+		out.merge(states[path], true)
+	return out
+
+
+## Numbers and booleans only, sorted and capped: a metric is an aggregate the
+## server may keep, and a string could be anything.
+func _metrics(flat: Dictionary) -> Dictionary:
+	var keys := flat.keys()
+	keys.sort()
+	var out := {}
+	for key in keys:
+		if out.size() >= MAX_METRICS:
+			break
+		var value: Variant = flat[key]
+		var name_text := str(key)
+		if name_text.is_empty() or name_text.length() > 80 or name_text.begins_with("simulation_"):
+			continue
+		if typeof(value) == TYPE_INT or typeof(value) == TYPE_FLOAT:
+			if is_finite(float(value)) and absf(float(value)) <= 1e15:
+				out[name_text] = value
+		elif typeof(value) == TYPE_BOOL:
+			out[name_text] = 1 if value else 0
+	return out
+
+
+# The digest below is the driver addon's (addons/ravensight_driver/driver.gd), so
+# hosted and local read a game the same way. src/simulation/godotPack.test.js
+# fails when the two drift.
+
+func _state_hash(states: Dictionary) -> String:
+	var scene := get_tree().current_scene
+	var budget := {"left": MAX_NODES, "truncated": false, "count": 0}
+	var parts := PackedStringArray()
+	if scene != null:
+		parts.append(_digest_fields(scene, 0))
+		var index := 0
+		for child in scene.get_children():
+			if child == self:
+				index += 1
+				continue
+			_digest_node(child, index, parts, budget)
+			index += 1
+	parts.append(JSON.stringify(_jsonify(states, 0), "", true))
+	return "".join(parts).sha256_text().substr(0, 32)
+
+
+func _jsonify(value: Variant, depth: int) -> Variant:
+	if depth > 8:
+		return null
+	match typeof(value):
+		TYPE_DICTIONARY:
+			var out := {}
+			for key in (value as Dictionary).keys():
+				out[str(key)] = _jsonify(value[key], depth + 1)
+			return out
+		TYPE_ARRAY, TYPE_PACKED_STRING_ARRAY, TYPE_PACKED_INT32_ARRAY, TYPE_PACKED_FLOAT32_ARRAY:
+			var list := []
+			for item in value:
+				list.append(_jsonify(item, depth + 1))
+			return list
+		TYPE_NIL, TYPE_BOOL, TYPE_INT, TYPE_STRING:
+			return value
+		TYPE_FLOAT:
+			return snappedf(value, 0.001)
+		_:
+			return str(value)
+
+
+# BEGIN digest shared with driver.gd
+func _digest_node(node: Node, index: int, parts: PackedStringArray, budget: Dictionary) -> void:
+	if int(budget["left"]) <= 0:
+		budget["truncated"] = true
+		return
+	budget["left"] = int(budget["left"]) - 1
+	budget["count"] = int(budget["count"]) + 1
+	parts.append(_digest_fields(node, index))
+	var child_index := 0
+	for child in node.get_children():
+		_digest_node(child, child_index, parts, budget)
+		child_index += 1
+
+
+func _digest_name(node: Node, index: int) -> String:
+	var node_name := str(node.name)
+	return "#%d" % index if node_name.begins_with("@") else node_name
+
+
+func _digest_fields(node: Node, index: int) -> String:
+	var fields := "%s|%s" % [node.get_class(), _digest_name(node, index)]
+	if node is CanvasItem:
+		fields += "|v%d" % int((node as CanvasItem).is_visible_in_tree())
+	if node is Node2D:
+		var n2 := node as Node2D
+		fields += "|%.3f,%.3f,%.3f" % [n2.global_position.x, n2.global_position.y, n2.global_rotation]
+	elif node is Node3D:
+		var n3 := node as Node3D
+		var p3 := n3.global_position
+		var a3 := n3.global_rotation
+		fields += "|v%d|%.3f,%.3f,%.3f|%.3f,%.3f,%.3f" % [int(n3.is_visible_in_tree()), p3.x, p3.y, p3.z, a3.x, a3.y, a3.z]
+	elif node is Control:
+		var rect := (node as Control).get_global_rect()
+		fields += "|%.3f,%.3f,%.3f,%.3f" % [rect.position.x, rect.position.y, rect.size.x, rect.size.y]
+	if node is CharacterBody2D:
+		var v2 := (node as CharacterBody2D).velocity
+		fields += "|%.3f,%.3f" % [v2.x, v2.y]
+	elif node is CharacterBody3D:
+		var v3 := (node as CharacterBody3D).velocity
+		fields += "|%.3f,%.3f,%.3f" % [v3.x, v3.y, v3.z]
+	elif node is RigidBody2D:
+		var r2 := (node as RigidBody2D).linear_velocity
+		fields += "|%.3f,%.3f" % [r2.x, r2.y]
+	elif node is RigidBody3D:
+		var r3 := (node as RigidBody3D).linear_velocity
+		fields += "|%.3f,%.3f,%.3f" % [r3.x, r3.y, r3.z]
+	var text := _text_of(node)
+	if not text.is_empty():
+		fields += "|t:" + text.substr(0, MAX_TEXT_CHARS)
+	if node is BaseButton:
+		fields += "|d%d" % int((node as BaseButton).disabled)
+	return fields + ";"
+# END digest shared with driver.gd
+
+
+func _text_of(node: Node) -> String:
+	if node is RichTextLabel:
+		return (node as RichTextLabel).get_parsed_text()
+	if node is Label:
+		return (node as Label).text
+	if node is Button:
+		return (node as Button).text
+	if node is LineEdit:
+		return (node as LineEdit).text
+	if node is TextEdit:
+		return (node as TextEdit).text
+	return ""

package/docs/godot-driver.md

@@ -8,8 +8,8 @@ by launching the developer's own Godot on their own machine. It has two halves:
 - `src/run/drivers/godot.js` (Node): starts Godot, speaks that protocol, turns
   the scene tree into an observation, and writes screenshots.
 
-Spec 07 is the contract for both halves. This document is how the two sides fit
-together and what the driver is and is not able to do.
+This document is how the two sides fit together and what the driver is and is
+not able to do.
 
 ## The driver never reaches players
 
@@ -108,8 +108,9 @@ a project cannot be playtested.
 <godot> --path <temp copy> --resolution 1280x720 -- --ravensight-driver --ravensight-driver-port=<port>
 ```
 
-The binary is found the way spec 17 describes: `--godot-path`, then `$GODOT`,
-then `PATH` (`godot`, `godot4`), then the platform's usual install locations.
+The binary is found in this order: `--godot-path` (accepted by `check`, `run`,
+`fix` and `resume`), then `$RAVENSIGHT_GODOT`, `$GODOT` and `$GODOT_PATH`, then
+`PATH` (`godot`, `godot4`), then the platform's usual install locations.
 When the project declares an engine version in `config/features`, a binary with
 that major.minor wins over a newer one. Anything below 4.2 is refused.
 
@@ -130,7 +131,7 @@ Only those, plus a fixed allowlist of variables a process needs to start at all
 `LIBGL_ALWAYS_SOFTWARE` set that the Xvfb path needs, the Windows equivalents),
 reach Godot. The environment is **not** inherited wholesale: the game is the
 customer's own code running on the developer's machine, and it has no business
-seeing `ANTHROPIC_API_KEY`, `RAVENSIGHT_TOKEN`, cloud credentials or CI secrets.
+seeing a model provider's key, `RAVENSIGHT_TOKEN`, cloud credentials or CI secrets.
 `ENV_ALLOWLIST` is the list, and `extraEnv` is the deliberate escape hatch for a
 caller that needs one more variable through.
 
@@ -169,12 +170,12 @@ plus the two optional extras spec 03 lists, `consoleErrors()` and
 
 | Action | Driver calls |
 |---|---|
-| `key` | `key` press, `wait_frames{1}`, `key` release |
+| `key` | `key` press for the key and any `keys` held with it, then `wait_frames{1}` for a tap or `holdMs` of game time on the clock (at most 5 s), then a release of each key in reverse order |
 | `type` | one press and release per character, each carrying its unicode, then Enter when `submit` |
-| `click` with a string target | `click{node_path}` (the center of the Control's global rect) |
+| `click` with a string target | `click{node_path}` (the center of a Control's global rect, or where the active camera draws a Node3D; a Node3D behind the camera is not clickable) |
 | `click` with coordinates | `click{x,y}` |
 | `press_action` | `press_action{action, duration_ms}` |
-| `wait` | `wait_frames`, milliseconds converted at 60 fps |
+| `wait` | the milliseconds on the clock, at most 10 s |
 | `set_time_scale` | `set_time_scale` |
 | `quit` | `quit` |
 
@@ -194,6 +195,52 @@ A driver level error is returned, not thrown: an undefined input action comes
 back as `{ ok: false, code: -32012, data: { available_actions } }` so the
 persona can pick a real one instead of the run ending.
 
+## Holding keys
+
+A tap is one frame, which in a game that moves a character by `speed * delta`
+is a pixel or two. Holds and waits are timed on the clock, divided by the game's
+time scale, not counted in frames: a frame count depends on the display, and on
+a 120 Hz Mac 120 frames was one game second. The persona's `key` tool therefore takes `hold_ms` (up to
+5000) and `with_keys` (up to two more keys held at the same time, for a
+diagonal). Every key that went down is released, in reverse order, even when a
+later key was refused, so nothing stays held for the rest of the run. A driver
+that cannot hold keys refuses a hold as a driver error rather than tapping.
+
+## The game is paused between actions
+
+A model takes seconds to decide, and a game with a clock keeps running while it
+does. So after launch the driver sets `time_scale` to 0, and around each action
+it restores the scale the game was running at, performs the action (a held key
+holds for that much game time), waits the settle frames, and freezes the clock
+again. Observations and screenshots are taken with the clock stopped. A game
+that sets its own `time_scale` keeps it: the scale read back when the clock is
+frozen is the one restored next. `run --realtime` turns this off for a game
+that has to keep running in real time.
+
+`time_scale` 0 stops what reads `delta` or a timer. A game that moves on wall
+clock time (`Time.get_ticks_msec()`) is not paused by it.
+
+## Importing a fresh clone
+
+Godot resolves `class_name` through `.godot/global_script_class_cache.cfg`,
+which is gitignored. A fresh clone, a CI checkout or a git worktree has none,
+and every script that names a global class fails to parse. Before launching,
+the driver checks the developer's directory (read only): no `.godot/`, a
+`class_name` with no class cache, or a script newer than the cache, and it runs
+`godot --headless --path <temp copy> --import`, bounded at 3 minutes, on the
+copy. Its output goes to the run's `godot.log`. A failed import is logged and
+the launch goes ahead, so the boot check below decides.
+
+## A game that never boots fails the launch
+
+After `ping`, the driver lets the game run ten frames, then reads its output.
+A `Parse Error:`, a `Failed to load script ... with error`, a main scene that
+failed to load, or no current scene at all fails the launch with an error that
+starts `game_boot_failed:`. The run is completed as `failed` with that reason,
+so it is refunded as a failed run, and a fix verify whose game did not boot is
+reported "not verified, the game did not boot", never fixed. An ordinary
+runtime `ERROR:` line does not fail the boot.
+
 ## Observations cite files
 
 Every node in a `get_state` snapshot carries its `path`, and also its `scene`
@@ -308,3 +355,18 @@ Godot installed.
 
 The fixture is never written to: every case copies it to a temp directory first,
 so `--import` and `--export-pack` leave no `.godot/` or `out/` behind.
+
+## Lockstep
+
+`RAVENSIGHT_DRIVER_LOCKSTEP=1` plus `RAVENSIGHT_DRIVER_SEED` put the addon in the
+deterministic mode the CLI uses for every persona run unless `--realtime` is set. See
+[REPLAY.md](REPLAY.md). Two requests only work there or matter there:
+
+- `seed{value}` reseeds the global generator and answers `{ seed, lockstep, tick, ticks_per_second }`.
+- `step{events, ticks, digest}` applies `events` (`{type: "key", keycode, pressed, unicode}`,
+  `{type: "action", action, pressed}`, `{type: "click", x, y | node_path, button, double}`),
+  runs exactly `ticks` ticks (1 to 3600), and answers `{ tick, frame, digest }`. Every event is
+  checked first, so a refused step changes nothing. Outside lockstep it answers `-32030`.
+
+`digest` answers the same shape without moving anything. `ping` also reports
+`lockstep`, `tick`, `seed`, `ticks_per_second` and `user_dir`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ravensight-playtest",
-  "version": "0.5.2",
+  "version": "0.7.0",
   "description": "Local AI playtests and seeded game simulations with reports in Ravensight.",
   "license": "MIT",
   "homepage": "https://ravensight.io/docs/#playtest",
@@ -27,6 +27,7 @@
     "bin",
     "src",
     "addons",
+    "templates",
     "docs/ADAPTERS.md",
     "docs/NATIVE_RUNNER.md",
     "docs/godot-driver.md",
@@ -45,8 +46,8 @@
     "./api": "./src/api/index.js"
   },
   "scripts": {
-    "test": "node --test --test-reporter=spec 'src/**/!(godot-real|web.e2e).test.js'",
-    "test:godot": "node --test --test-reporter=spec src/run/drivers/godot-real.test.js",
+    "test": "node --test --test-reporter=spec 'src/**/!(godot-real|web.e2e|scaffold.godot|template3d.godot).test.js'",
+    "test:godot": "node --test --test-reporter=spec src/run/drivers/godot-real.test.js src/create/scaffold.godot.test.js src/create/template3d.godot.test.js",
     "test:e2e": "node --test --test-reporter=spec src/run/drivers/web.e2e.test.js",
     "lint": "node scripts/no-dashes.mjs"
   },

package/src/api/README.md

@@ -128,6 +128,7 @@ All game-scoped, all under `/api/v1/games/:gameId/playtest`.
 | `api.jobs.patch(gameId, jobId, { state, reason })` | `PATCH /jobs/:jobId` -> `{ job }` |
 | `api.jobs.cancel(gameId, jobId)` | `POST /jobs/:jobId/cancel` |
 | `api.jobs.finish(gameId, jobId, { state, reason })` | `POST /jobs/:jobId/finish` |
+| `api.jobs.labsOutcomePr(gameId, jobId, { pr })` | `PUT /jobs/:jobId/labs-outcome/pr` |
 | `api.jobs.remove(gameId, jobId)` | `DELETE /jobs/:jobId` |
 | `api.jobs.uploads(gameId, jobId, { files })` | `POST /jobs/:jobId/uploads`, job-level keys |
 | `api.jobs.capabilityReport(gameId, jobId, body)` | `POST /jobs/:jobId/capability-report` |
@@ -166,7 +167,9 @@ Refusals worth handling by code:
 | Status | `error` | What it means |
 |---|---|---|
 | 400 | `idempotency_key_required`, `invalid_job` | fix the request |
-| 402 | `insufficient_balance` | carries `estimate_cents`, `balance_cents`, `shortfall_cents`, `topup_url` |
+| 402 | `funding_required` | a Labs job on an account below $10 of lifetime top-ups; carries `topup_url`. Fix v2 falls back to an ordinary fix |
+| 402 | `insufficient_balance` | carries `estimate_cents`, `balance_cents`, `shortfall_cents`, `topup_url`, and on a Labs job `minimum_balance_cents`, which the shortfall is measured against; the two balance figures are null for a role without billing access |
+| 409 | `settle_contended` | the job's price moved while it was being settled, nothing happened, retry after `Retry-After` |
 | 409 | `price_changed` | carries the new `estimate_cents` and `breakdown`; re-confirm |
 | 409 | `brief_required` | finish the brief first |
 | 409 | `cli_outdated` | carries `min_version` |