peachsim 0.1.527

package/cli/commands/maestro-generate.ts

@@ -0,0 +1,375 @@
+// plain-language Maestro authoring for `rnx maestro generate`.
+
+import { mkdtempSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import yaml from 'yaml'
+import { authHeaderValue, resolveCliAuth } from '../auth'
+import { registerRun, stepSummaryFromTrace } from '../run-registry'
+import { createBridgeFromParsed, parseBridgeCliArgs } from '../ws-bridge'
+import {
+  findUnexpectedFlowArgs,
+  FLOW_ARG_VALUE_FLAGS,
+  FLOW_BRIDGE_ARG_OPTIONS,
+  getLastFlowPreviewUploadResult,
+  getLastFlowTraceSteps,
+  runFlowPlayback,
+  type UnexpectedFlowArg,
+} from './flow'
+import { inspectDescribe } from './inspect/core'
+import { resolveDefaultUploadOrigin } from './upload'
+
+const DEFAULT_TEST_MODEL = process.env.RNX_TEST_MODEL || 'deepseek-v4-flash'
+const DESCRIBE_CONTEXT_LIMIT = 18_000
+const LLM_TIMEOUT_MS = 180_000
+
+function flagValue(args: string[], name: string): string | undefined {
+  const index = args.indexOf(name)
+  return index >= 0 ? args[index + 1] : undefined
+}
+
+function flagValueAny(args: string[], names: string[]): string | undefined {
+  for (const name of names) {
+    const value = flagValue(args, name)
+    if (value) return value
+  }
+  return undefined
+}
+
+// generate is a front-end for runFlowPlayback: it inspects the sim, asks the
+// model for a flow, then hands the argv down. these are the only flags it reads
+// itself; everything else belongs to the runner and is forwarded verbatim. this
+// file used to keep its own allowlist of what to forward, which silently
+// dropped --proof, --record, --profile, --screenshots, --out and --base-url
+// while `maestro test` honoured all six.
+const GENERATE_VALUE_FLAGS = ['--origin', '--preview-origin', '--public-origin']
+
+const VALUE_FLAGS = new Set([...GENERATE_VALUE_FLAGS, ...FLOW_ARG_VALUE_FLAGS])
+
+export function findPrompt(args: string[]): { prompt: string; index: number } | null {
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i]
+    if (!arg) continue
+    if (VALUE_FLAGS.has(arg)) {
+      i += 1
+      continue
+    }
+    if (arg.startsWith('-')) continue
+    return { prompt: arg, index: i }
+  }
+  return null
+}
+
+function stripPromptForBridge(args: string[], promptIndex: number): string[] {
+  return args.filter((_, index) => index !== promptIndex)
+}
+
+async function describeCurrentSim(args: string[], promptIndex: number): Promise<string> {
+  const bridgeArgs = stripPromptForBridge(args, promptIndex)
+  const parsedBridgeArgs = parseBridgeCliArgs(bridgeArgs, {
+    stripBooleanFlags: FLOW_BRIDGE_ARG_OPTIONS.stripBooleanFlags,
+    stripValueFlags: [
+      ...GENERATE_VALUE_FLAGS,
+      ...FLOW_BRIDGE_ARG_OPTIONS.stripValueFlags,
+    ],
+  })
+  const bridge = createBridgeFromParsed(parsedBridgeArgs)
+  try {
+    const describe = await inspectDescribe(bridge, {
+      describe: true,
+      verbose: false,
+      filter: '',
+      compact: true,
+      hideXy: true,
+    })
+    const parts = [
+      typeof describe.nodeCount === 'number' ? `nodes: ${describe.nodeCount}` : '',
+      describe.tree || '',
+    ].filter(Boolean)
+    const text = parts.join('\n')
+    return text.length > DESCRIBE_CONTEXT_LIMIT
+      ? text.slice(0, DESCRIBE_CONTEXT_LIMIT) + '\n[tree truncated]'
+      : text
+  } finally {
+    bridge.close()
+  }
+}
+
+const RNX_TEST_SYSTEM_PROMPT = `
+You write Maestro-compatible rnx YAML flows from a user's plain-language test goal.
+Return only JSON, no markdown, with this exact shape:
+{"summary":"short human-readable result goal","steps":[{"waitFor":{"text":"...","timeout":10000}}]}
+
+Use only these flow commands:
+- waitFor: { "text"?: string, "id"?: string, "timeout"?: number }
+- assertVisible: string or { "text"?: string, "id"?: string }
+- assertNotVisible: string or { "text"?: string, "id"?: string }
+- tapOn: string or { "text"?: string, "id"?: string, "index"?: number }
+- inputText: string
+- pressKey: string
+- hideKeyboard: true
+- swipe: { "direction": "UP" | "DOWN" | "LEFT" | "RIGHT", "duration"?: number }
+- waitForAnimationToEnd: true
+- takeScreenshot: "label"
+
+Prefer ids/testIDs from the tree. Prefer visible text when no id exists.
+Do not invent app-specific data that is not visible. Keep the flow short.
+End with an assertion or wait that proves the user's goal.
+`.trim()
+
+function buildPromptRequest(prompt: string, describe: string): string {
+  return `
+User goal:
+${prompt}
+
+Current rnx visible tree:
+${describe || '[no tree available]'}
+
+Generate the shortest replayable flow that verifies the goal from this current state.
+`.trim()
+}
+
+async function generateFlowJson(args: {
+  prompt: string
+  describe: string
+  origin: string
+  authHeader: string
+}): Promise<string> {
+  const res = await fetch(`${args.origin.replace(/\/$/, '')}/api/llm`, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      authorization: args.authHeader,
+    },
+    body: JSON.stringify({
+      modelId: DEFAULT_TEST_MODEL,
+      context: {
+        systemPrompt: RNX_TEST_SYSTEM_PROMPT,
+        messages: [
+          {
+            role: 'user',
+            content: buildPromptRequest(args.prompt, args.describe),
+            timestamp: Date.now(),
+          },
+        ],
+      },
+      options: { maxTokens: 1800, temperature: 0.2 },
+      threadId: null,
+    }),
+    signal: AbortSignal.timeout(LLM_TIMEOUT_MS),
+  })
+  if (!res.ok || !res.body) {
+    const text = await res.text().catch(() => '')
+    throw new Error(`/api/llm ${res.status}${text ? `: ${text}` : ''}`)
+  }
+
+  const reader = res.body.getReader()
+  const decoder = new TextDecoder()
+  const deltas: string[] = []
+  let buffer = ''
+  // /api/llm speaks pi-agent-core's proxy event protocol. text events carry
+  // visible deltas; transport diagnostics and keepalives are ignored here.
+  const handleLine = (line: string) => {
+    const trimmed = line.trim()
+    if (!trimmed) return
+    const event = JSON.parse(trimmed) as {
+      type?: string
+      delta?: string
+      errorMessage?: string
+    }
+    if (event.type === 'text_delta' && event.delta) {
+      deltas.push(event.delta)
+    } else if (event.type === 'error') {
+      throw new Error(event.errorMessage || 'llm error')
+    }
+  }
+
+  for (;;) {
+    const { value, done } = await reader.read()
+    if (done) break
+    buffer += decoder.decode(value, { stream: true })
+    let newline: number
+    while ((newline = buffer.indexOf('\n')) >= 0) {
+      handleLine(buffer.slice(0, newline))
+      buffer = buffer.slice(newline + 1)
+    }
+  }
+  buffer += decoder.decode()
+  if (buffer.trim()) handleLine(buffer)
+
+  const text = deltas.join('').trim()
+  if (!text) throw new Error('llm returned an empty test flow')
+  return text
+}
+
+function extractJson(text: string): unknown {
+  const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i)
+  const candidate = fenced
+    ? fenced[1]
+    : text.slice(text.indexOf('{'), text.lastIndexOf('}') + 1)
+  return JSON.parse(candidate)
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value && typeof value === 'object' && !Array.isArray(value))
+}
+
+function parseGeneratedFlow(text: string): {
+  summary: string
+  steps: Record<string, unknown>[]
+  yamlText: string
+} {
+  const parsed = extractJson(text)
+  if (!isRecord(parsed)) throw new Error('llm did not return a JSON object')
+  const summary =
+    typeof parsed.summary === 'string' && parsed.summary.trim()
+      ? parsed.summary.trim().slice(0, 500)
+      : 'rnx generated test'
+  const rawSteps = parsed.steps
+  if (!Array.isArray(rawSteps) || rawSteps.length === 0) {
+    throw new Error('llm returned no flow steps')
+  }
+  const steps = rawSteps.map((step, index) => {
+    if (!isRecord(step)) throw new Error(`flow step ${index + 1} is not an object`)
+    return step
+  })
+  return { summary, steps, yamlText: yaml.stringify(steps) }
+}
+
+export function buildFlowArgs(
+  args: string[],
+  promptIndex: number,
+  flowPath: string,
+  origin: string,
+) {
+  const out = [
+    flowPath,
+    '--preview',
+    '--preview-origin',
+    origin,
+    '--billing-kind',
+    'test_run',
+  ]
+  for (let i = 0; i < args.length; i++) {
+    if (i === promptIndex) continue
+    const arg = args[i]
+    if (!arg) continue
+    if (arg === '--origin' || arg === '--preview-origin') {
+      i += 1
+      continue
+    }
+    if (arg === '--public-origin') {
+      const value = args[i + 1]
+      if (value) out.push('--preview-public-origin', value)
+      i += 1
+      continue
+    }
+    out.push(arg)
+    if (VALUE_FLAGS.has(arg)) {
+      const value = args[i + 1]
+      if (value !== undefined) out.push(value)
+      i += 1
+    }
+  }
+  return out
+}
+
+// an argument nothing reads is an argument that silently changes what the run
+// does, and generate hands its argv to runFlowPlayback, which is the end of the
+// line for argv. so whatever survives buildFlowArgs is checked against the
+// runner's own vocabulary, before the sim inspection and the model call rather
+// than after them.
+export function findUnexpectedGenerateArgs(
+  args: string[],
+  promptIndex: number,
+): UnexpectedFlowArg[] {
+  const problems: UnexpectedFlowArg[] = []
+  for (let i = 0; i < args.length; i++) {
+    if (i === promptIndex) continue
+    const arg = args[i]
+    if (!arg) continue
+    // generate writes --billing-kind onto the flow argv itself and getFlag
+    // takes the first occurrence, so a caller's value loses to it in silence.
+    if (arg === '--billing-kind') {
+      problems.push({ arg, message: `${arg} is set by rnx maestro generate` })
+      i += 1
+      continue
+    }
+    if (!GENERATE_VALUE_FLAGS.includes(arg)) continue
+    const value = args[i + 1]
+    if (value === undefined || value.startsWith('-')) {
+      problems.push({ arg, message: `${arg} expects a value` })
+    }
+    i += 1
+  }
+  return [
+    ...problems,
+    ...findUnexpectedFlowArgs(buildFlowArgs(args, promptIndex, '<flow>', '<origin>')),
+  ]
+}
+
+export async function runMaestroGenerate(args: string[]): Promise<number> {
+  const promptArg = findPrompt(args)
+  if (!promptArg) {
+    console.error('  usage: rnx maestro generate "test goal"')
+    return 1
+  }
+  const { prompt, index: promptIndex } = promptArg
+  const unexpected = findUnexpectedGenerateArgs(args, promptIndex)
+  if (unexpected.length > 0) {
+    for (const problem of unexpected) {
+      console.error(`  error: ${problem.message}`)
+    }
+    console.error('\n  run `rnx maestro --help` for the flags this command accepts')
+    return 1
+  }
+  const cliAuth = resolveCliAuth()
+  if (!cliAuth || cliAuth.kind === 'github') {
+    console.error('  rnx maestro generate "<goal>" needs a login or RNX_API_KEY.')
+    console.error('  API keys must include preview_upload and llm scopes.')
+    return 1
+  }
+  const authHeader = authHeaderValue(cliAuth)
+  const origin = await resolveDefaultUploadOrigin(
+    flagValueAny(args, ['--origin', '--preview-origin']),
+  )
+  console.log('  inspecting current sim…')
+  const describe = await describeCurrentSim(args, promptIndex)
+  console.log('  generating Maestro flow…')
+  const generated = await generateFlowJson({ prompt, describe, origin, authHeader })
+  const flow = parseGeneratedFlow(generated)
+  const tempDir = mkdtempSync(join(tmpdir(), 'rnx-maestro-generate-'))
+  const flowPath = join(tempDir, 'flow.yaml')
+  writeFileSync(flowPath, flow.yamlText)
+  console.log(`  flow: ${flowPath}`)
+
+  const startedAt = Date.now()
+  const flowArgs = buildFlowArgs(args, promptIndex, flowPath, origin)
+  const exitCode = await runFlowPlayback(flowArgs)
+  const upload = getLastFlowPreviewUploadResult()
+  const traceSteps = getLastFlowTraceSteps()
+  const failedStep = traceSteps.find((step) => step.status === 'failure')
+  const status = exitCode === 0 ? 'passed' : 'failed'
+  const run = await registerRun({
+    origin,
+    kind: 'maestro',
+    prompt,
+    summary: flow.summary,
+    status,
+    failureMessage:
+      status === 'failed' ? (failedStep?.error ?? 'flow playback failed') : null,
+    previewShareId: upload?.shareId ?? null,
+    owner: flagValue(args, '--owner') ?? null,
+    repo: flagValue(args, '--repo') ?? null,
+    durationMs: Date.now() - startedAt,
+    flowYamlSizeBytes: Buffer.byteLength(flow.yamlText, 'utf8'),
+    ...stepSummaryFromTrace(traceSteps),
+    auth: cliAuth,
+  })
+
+  console.log(`\n  result: ${status}`)
+  if (run) console.log(`  run: ${run.id}`)
+  if (upload?.previewUrl) console.log(`  preview: ${upload.previewUrl}`)
+  if (run?.traceUrl) console.log(`  replay: ${run.traceUrl}`)
+  return exitCode
+}