tulingcode 0.1.0

package/src/tool/shell-tokenize.ts

@@ -0,0 +1,346 @@
+import { Effect } from "effect"
+import { parse as shellQuoteParse } from "shell-quote"
+
+export interface Argv {
+  line: number
+  tokens: string[]
+}
+
+export interface ParseError {
+  kind: "unclosed-quote" | "unsupported-operator" | "unclosed-heredoc" | "internal"
+  line: number
+  detail: string
+}
+
+// Null-byte delimited placeholder that shell-quote passes through verbatim.
+function heredocPlaceholder(n: number) {
+  return `\x00HD${n}\x00`
+}
+
+type HeredocResult =
+  | { ok: true; stripped: string; bodies: string[] }
+  | { ok: false; error: ParseError }
+
+function extractHeredocs(script: string): HeredocResult {
+  const bodies: string[] = []
+  let out = ""
+  let quote: '"' | "'" | null = null
+  let i = 0
+  let line = 1
+
+  while (i < script.length) {
+    const ch = script[i]
+
+    // --- Inside a quoted string: pass through verbatim ---
+    if (quote) {
+      if (ch === "\\" && quote === '"' && i + 1 < script.length) {
+        out += ch + script[i + 1]
+        if (script[i + 1] === "\n") line++
+        i += 2
+        continue
+      }
+      if (ch === quote) {
+        quote = null
+        out += ch
+        i++
+        continue
+      }
+      if (ch === "\n") line++
+      out += ch
+      i++
+      continue
+    }
+
+    // --- At top level ---
+    if (ch === '"' || ch === "'") {
+      quote = ch
+      out += ch
+      i++
+      continue
+    }
+
+    // Check for `<<` at top level (but not `<<<` herestring)
+    if (ch === "<" && script[i + 1] === "<") {
+      // `<<<` herestring: pass all three chars through so the existing operator
+      // check rejects them.
+      if (script[i + 2] === "<") {
+        out += "<<< "
+        i += 3
+        // Skip to newline so the rest of the line goes through normally
+        while (i < script.length && script[i] !== "\n") i++
+        continue
+      }
+
+      // Try to read a marker name starting at i+2
+      let j = i + 2
+      const markerStart = j
+      // Marker must start with letter or underscore
+      if (j < script.length && /[A-Za-z_]/.test(script[j])) {
+        while (j < script.length && /[A-Za-z0-9_]/.test(script[j])) j++
+        const marker = script.slice(markerStart, j)
+
+        // After marker: only optional whitespace, then newline (or EOF)
+        let k = j
+        while (k < script.length && (script[k] === " " || script[k] === "\t")) k++
+
+        if (k < script.length && script[k] !== "\n") {
+          // Non-whitespace after <<MARKER on the same line
+          return {
+            ok: false,
+            error: {
+              kind: "unsupported-operator",
+              line,
+              detail: "tokens after <<MARKER on the same line are not supported",
+            },
+          }
+        }
+
+        // Valid heredoc open: emit placeholder on the same line in output.
+        // Body lines and the closing marker are replaced with empty lines so
+        // that splitTopLevelLines sees the correct original line numbers.
+        const bodyIndex = bodies.length
+        const openLine = line
+        out += heredocPlaceholder(bodyIndex) + "\n"
+        // Advance past <<MARKER + optional whitespace + \n
+        i = k + 1
+        line++
+
+        // Collect body lines until we see a line that trims to exactly `marker`
+        const bodyLines: string[] = []
+        let closed = false
+        while (i < script.length) {
+          // Read one line (up to but not including its \n, or EOF)
+          let lineEnd = i
+          while (lineEnd < script.length && script[lineEnd] !== "\n") lineEnd++
+          const bodyLine = script.slice(i, lineEnd)
+          // Emit a blank line to preserve original line numbering
+          out += "\n"
+          if (bodyLine.trim() === marker) {
+            // Closing line — consume it (and trailing \n if present)
+            i = lineEnd + (lineEnd < script.length ? 1 : 0)
+            line++
+            closed = true
+            break
+          }
+          bodyLines.push(bodyLine)
+          i = lineEnd + (lineEnd < script.length ? 1 : 0)
+          line++
+        }
+
+        if (!closed) {
+          return {
+            ok: false,
+            error: {
+              kind: "unclosed-heredoc",
+              line: openLine,
+              detail: `unclosed heredoc <<${marker}`,
+            },
+          }
+        }
+
+        bodies.push(bodyLines.join("\n"))
+        continue
+      }
+
+      // `<<` without a valid marker — pass through so the existing operator
+      // check rejects it.
+      out += ch
+      i++
+      continue
+    }
+
+    out += ch
+    if (ch === "\n") line++
+    i++
+  }
+
+  return { ok: true, stripped: out, bodies }
+}
+
+// Regex matching a heredoc placeholder token produced by extractHeredocs.
+const HD_RE = /^\x00HD(\d+)\x00$/
+
+// POSIX-style comment pre-pass: operates on the post-heredoc-extraction text so
+// heredoc bodies are already removed. Drops word-boundary `#` to end-of-line,
+// escapes mid-token `#` so shell-quote treats it as a literal character.
+function preprocessComments(input: string): string {
+  let out = ""
+  let i = 0
+  let quote: '"' | "'" | null = null
+  let prevWasBoundary = true // start of input is a word boundary
+  while (i < input.length) {
+    const ch = input[i]
+    if (quote) {
+      out += ch
+      if (ch === "\\" && quote === '"' && i + 1 < input.length) {
+        out += input[i + 1]
+        i += 2
+        continue
+      }
+      if (ch === quote) quote = null
+      i++
+      prevWasBoundary = false
+      continue
+    }
+    if (ch === '"' || ch === "'") {
+      quote = ch
+      out += ch
+      i++
+      prevWasBoundary = false
+      continue
+    }
+    if (ch === "\\") {
+      if (i + 1 < input.length && input[i + 1] === "#") {
+        // Explicit \# escape: pass through verbatim so shell-quote produces literal #
+        out += "\\#"
+        i += 2
+        prevWasBoundary = false
+        continue
+      }
+      // Other backslash: pass through
+      out += ch
+      i++
+      prevWasBoundary = false
+      continue
+    }
+    if (ch === "#") {
+      if (prevWasBoundary) {
+        // word-boundary # → real comment, drop to end of line (don't consume the \n)
+        while (i < input.length && input[i] !== "\n") i++
+        continue
+      }
+      // mid-token # → escape so shell-quote treats as literal
+      out += "\\#"
+      i++
+      prevWasBoundary = false
+      continue
+    }
+    out += ch
+    prevWasBoundary = ch === "\n" || /\s/.test(ch)
+    i++
+  }
+  return out
+}
+
+export function tokenize(script: string): Effect.Effect<Argv[], ParseError> {
+  return Effect.suspend(() => {
+    if (script.trim() === "") return Effect.succeed([] as Argv[])
+
+    const heredocResult = extractHeredocs(script)
+    if (!heredocResult.ok) return Effect.fail(heredocResult.error)
+    const { stripped, bodies } = heredocResult
+
+    const segments = splitTopLevelLines(preprocessComments(stripped)).filter((seg) => seg.text.trim() !== "")
+    const out: Argv[] = []
+    for (const seg of segments) {
+      const unclosed = scanUnclosedQuote(seg.text)
+      if (unclosed) {
+        return Effect.fail<ParseError>({
+          kind: "unclosed-quote",
+          line: seg.line,
+          detail: `unclosed ${unclosed}-quoted string`,
+        })
+      }
+      const segTokens = shellQuoteParse(seg.text, (name: string) => "$" + name, { escape: "\\" })
+      const stringTokens: string[] = []
+      for (const t of segTokens) {
+        if (typeof t === "string") {
+          const match = HD_RE.exec(t)
+          if (match) {
+            stringTokens.push(bodies[parseInt(match[1], 10)])
+            continue
+          }
+          stringTokens.push(t)
+          continue
+        }
+        if (typeof t === "object" && t !== null && "op" in t) {
+          const tok = t as { op: string; pattern?: string }
+          const detail =
+            tok.op === "glob" && tok.pattern != null
+              ? `unsupported glob pattern: ${tok.pattern}`
+              : `unsupported shell operator: ${tok.op}`
+          return Effect.fail<ParseError>({
+            kind: "unsupported-operator",
+            line: seg.line,
+            detail,
+          })
+        }
+      }
+      if (stringTokens.length > 0) out.push({ line: seg.line, tokens: stringTokens })
+    }
+    return Effect.succeed(out)
+  })
+}
+
+function scanUnclosedQuote(segment: string): '"' | "'" | null {
+  let quote: '"' | "'" | null = null
+  for (let i = 0; i < segment.length; i++) {
+    const ch = segment[i]
+    if (quote) {
+      if (ch === "\\" && quote === '"' && i + 1 < segment.length) {
+        i++
+        continue
+      }
+      if (ch === quote) quote = null
+      continue
+    }
+    if (ch === '"' || ch === "'") quote = ch
+  }
+  return quote
+}
+
+// Split the script into segments separated by line breaks that are NOT inside
+// quoted strings. Returns each segment's starting line (1-indexed) and text.
+function splitTopLevelLines(script: string): Array<{ line: number; text: string }> {
+  const segments: Array<{ line: number; text: string }> = []
+  let buf = ""
+  let segStart = 1
+  let line = 1
+  let quote: '"' | "'" | null = null
+  let i = 0
+  while (i < script.length) {
+    const ch = script[i]
+    if (quote) {
+      if (ch === "\\" && quote === '"' && i + 1 < script.length) {
+        buf += ch + script[i + 1]
+        if (script[i + 1] === "\n") line++
+        i += 2
+        continue
+      }
+      if (ch === quote) {
+        quote = null
+        buf += ch
+        i++
+        continue
+      }
+      if (ch === "\n") line++
+      buf += ch
+      i++
+      continue
+    }
+    if (ch === '"' || ch === "'") {
+      quote = ch
+      buf += ch
+      i++
+      continue
+    }
+    if (ch === "\\" && i + 1 < script.length && script[i + 1] === "\n") {
+      // POSIX line continuation: \<LF> disappears, next physical line continues this command
+      line++
+      i += 2
+      continue
+    }
+    if (ch === "\n") {
+      segments.push({ line: segStart, text: buf })
+      buf = ""
+      line++
+      segStart = line
+      i++
+      continue
+    }
+    buf += ch
+    i++
+  }
+  if (buf.length > 0) segments.push({ line: segStart, text: buf })
+  return segments
+}

package/src/tool/shell-wrap.ts

@@ -0,0 +1,190 @@
+import { Cause, Effect } from "effect"
+import z from "zod"
+import * as Tool from "./tool"
+
+export const shellInputSchema = z.object({
+  script: z.string().min(1).describe(
+    [
+      "Multi-line shell-style script. Each non-blank line is one command; commands run sequentially and stop on first failure.",
+      'Quoting: "..." preserves literal newlines and processes \\" \\\\ escapes; \'...\' is verbatim.',
+      "<<EOF heredoc bodies are fully verbatim (no escape, no $vars).",
+      "# starts a line comment to end-of-line (quoted # is literal).",
+      "Variables ($VAR, ${VAR}) are preserved as literal text — no expansion.",
+      "See the tool description for the verb table.",
+    ].join(" "),
+  ),
+})
+
+type ShellInput = z.infer<typeof shellInputSchema>
+
+export function shellWrap<P extends z.ZodType, M extends Tool.Metadata>(
+  def: Tool.Def<P, M>,
+): Tool.Def<typeof shellInputSchema, Tool.Metadata> {
+  if (!def.shell) {
+    throw new Error(`shellWrap called on tool '${def.id}' that has no shell field`)
+  }
+  const shell = def.shell
+  return {
+    id: def.id,
+    description: shell.description,
+    parameters: shellInputSchema,
+    execute: (args: ShellInput, ctx) =>
+      Effect.gen(function* () {
+        // Guard: a model in shell mode may emit a JSON-shape call (e.g.
+        // {description, prompt, subagent_type}) with NO `script` field. Without
+        // this, shell.parse(undefined) → tokenize's script.trim() throws a
+        // minified TypeError. Return a structured, teaching error instead so the
+        // model can self-correct on the next turn.
+        if (typeof args.script !== "string" || args.script.trim() === "") {
+          // Shell mode received no usable `script`. Try tool-specific recovery:
+          // a model may emit the tool's JSON-shape args directly (e.g. actor's
+          // Task-prior {description,prompt,subagent_type}). If recover lifts it to
+          // the parsed shape, route to def.execute (which re-validates via zod).
+          const recovered = shell.recover?.(args as unknown)
+          if (recovered !== undefined) {
+            const op = operationLabel(recovered)
+            const exit = yield* Effect.exit(def.execute(recovered, ctx as Tool.Context))
+            if (exit._tag === "Failure") {
+              return {
+                title: `${def.id}: invalid arguments`,
+                output: formatFailedCommandNoVerb(jsonTeachingBody(def.id, describeFailure(exit.cause))),
+                metadata: { commands: 0, success: 0 } as Tool.Metadata,
+              }
+            }
+            // Success is silent — NO format hint. If JSON-shape works, let the
+            // session keep using it (actor fails shell-syntax on first try).
+            // Output is the tool's own (not wrapped in <command index=...>): a
+            // recovered call is a single op, so the per-command envelope is noise.
+            return {
+              title: `${def.id}: ${op}`,
+              output: exit.value.output,
+              metadata: { ...(exit.value.metadata as Tool.Metadata), commands: 1, success: 1 } as Tool.Metadata,
+            }
+          }
+          const o = (typeof args === "object" && args ? args : {}) as Record<string, unknown>
+          const body = "script" in o ? shellTeachingBody(def.id) : jsonTeachingBody(def.id)
+          return {
+            title: `${def.id}: missing script`,
+            output: formatFailedCommandNoVerb(body),
+            metadata: { commands: 0, success: 0 } as Tool.Metadata,
+          }
+        }
+        const parseExit = yield* Effect.exit(shell.parse(args.script))
+        if (parseExit._tag === "Failure") {
+          const err = Cause.squash(parseExit.cause)
+          const body = formatParseError(def.id, err)
+          return {
+            title: `${def.id}: parse error`,
+            output: formatFailedCommandNoVerb(body),
+            metadata: { commands: 0, success: 0 } as Tool.Metadata,
+          }
+        }
+        const parsedList = parseExit.value
+        if (parsedList.length === 0) {
+          return {
+            title: `${def.id}: empty script`,
+            output: formatFailedCommandNoVerb(`${def.id}: no commands found in script`),
+            metadata: { commands: 0, success: 0 } as Tool.Metadata,
+          }
+        }
+        const blocks: string[] = []
+        let lastMetadata: Tool.Metadata = {}
+        let success = 0
+        for (let i = 0; i < parsedList.length; i++) {
+          const parsed = parsedList[i]
+          // Convention: every shell-style tool's parameters discriminator is named
+          // `operation`. It may be a flat string (e.g. actor: `{ operation: "run" }`)
+          // or a nested object whose own discriminator is `action` (e.g. task:
+          // `{ operation: { action: "create" } }` — see the `.meta` comment in task.ts).
+          // Derive a string label for the output XML attribute from either shape.
+          const operation = operationLabel(parsed)
+          const exit = yield* Effect.exit(def.execute(parsed, ctx as Tool.Context))
+          if (exit._tag === "Failure") {
+            blocks.push(formatFailedCommand(i + 1, operation, describeFailure(exit.cause)))
+            if (i + 1 < parsedList.length) {
+              blocks.push(`<not-executed>commands #${i + 2}..#${parsedList.length}</not-executed>`)
+            }
+            return {
+              title: `${def.id}: command #${i + 1} failed`,
+              output: blocks.join("\n"),
+              metadata: { commands: parsedList.length, success } as Tool.Metadata,
+            }
+          }
+          success++
+          lastMetadata = exit.value.metadata as Tool.Metadata
+          blocks.push(formatOkCommand(i + 1, operation, exit.value.output))
+        }
+        return {
+          title: `${def.id}: ${parsedList.length} command(s)`,
+          output: blocks.join("\n"),
+          metadata: { ...lastMetadata, commands: parsedList.length, success } as Tool.Metadata,
+        }
+      }),
+  }
+}
+
+function formatOkCommand(index: number, operation: string, body: string): string {
+  return `<command index="${index}" operation="${escapeAttr(operation)}">\n${body}\n</command>`
+}
+
+function formatFailedCommand(index: number, operation: string, body: string): string {
+  return `<command index="${index}" operation="${escapeAttr(operation)}" failed="true">\n${body}\n</command>`
+}
+
+// Derives the string label for the `operation` XML attribute from a parsed
+// command, tolerating both discriminator shapes:
+//   - flat:   { operation: "run" }                 → "run"
+//   - nested: { operation: { action: "create" } }  → "create"
+// Falls back to "?" so the attribute is never undefined / "[object Object]".
+function operationLabel(parsed: unknown): string {
+  const op = (parsed as { operation?: unknown } | null | undefined)?.operation
+  if (typeof op === "string") return op
+  if (op && typeof op === "object" && typeof (op as { action?: unknown }).action === "string") {
+    return (op as { action: string }).action
+  }
+  return "?"
+}
+
+function escapeAttr(s: string): string {
+  return String(s ?? "")
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+}
+
+function describeFailure(cause: Cause.Cause<unknown>): string {
+  const squashed = Cause.squash(cause)
+  if (squashed instanceof Error) return squashed.message
+  return String(squashed)
+}
+
+function formatFailedCommandNoVerb(body: string): string {
+  return `<command failed="true">\n${body}\n</command>`
+}
+
+function shellTeachingBody(toolId: string): string {
+  return [
+    `${toolId}: this tool takes a single \`script\` string (shell-style), not JSON fields.`,
+    `Put the command in \`script\`, e.g.:  ${toolId} <verb> ...`,
+    `See the tool description for the verb list and examples.`,
+  ].join("\n")
+}
+
+function jsonTeachingBody(toolId: string, detail?: string): string {
+  return [
+    `${toolId}: could not run the call.`,
+    `Pass the operation as JSON, e.g.:  {"operation":{"action":"<verb>", ...}}`,
+    ...(detail ? [`detail: ${detail}`] : []),
+  ].join("\n")
+}
+
+function formatParseError(toolId: string, err: unknown): string {
+  if (err && typeof err === "object" && "kind" in err) {
+    const e = err as { kind: string; line?: number; detail?: string }
+    const line = e.line ?? "?"
+    return `${toolId}: parse error at line ${line}\n  ${e.detail ?? e.kind}`
+  }
+  if (err instanceof Error) return `${toolId}: parse error\n  ${err.message}`
+  return `${toolId}: parse error\n  ${String(err)}`
+}

package/src/tool/skill.ts

@@ -0,0 +1,76 @@
+import path from "path"
+import { pathToFileURL } from "url"
+import z from "zod"
+import { Effect } from "effect"
+import * as Stream from "effect/Stream"
+import { Ripgrep } from "../file/ripgrep"
+import { Skill } from "../skill"
+import * as Tool from "./tool"
+import DESCRIPTION from "./skill.txt"
+
+const Parameters = z.object({
+  name: z.string().describe("The name of the skill from available_skills"),
+})
+
+export const SkillTool = Tool.define(
+  "skill",
+  Effect.gen(function* () {
+    const skill = yield* Skill.Service
+    const rg = yield* Ripgrep.Service
+
+    return {
+      description: DESCRIPTION,
+      parameters: Parameters,
+      execute: (params: z.infer<typeof Parameters>, ctx: Tool.Context) =>
+        Effect.gen(function* () {
+          const info = yield* skill.get(params.name)
+          if (!info) {
+            const all = yield* skill.all()
+            const available = all.map((item) => item.name).join(", ")
+            throw new Error(`Skill "${params.name}" not found. Available skills: ${available || "none"}`)
+          }
+
+          yield* ctx.ask({
+            permission: "skill",
+            patterns: [params.name],
+            always: [params.name],
+            metadata: {},
+          })
+
+          const dir = path.dirname(info.location)
+          const base = pathToFileURL(dir).href
+          const limit = 10
+          const files = yield* rg.files({ cwd: dir, follow: false, hidden: true, signal: ctx.abort }).pipe(
+            Stream.filter((file) => !file.includes("SKILL.md")),
+            Stream.map((file) => path.resolve(dir, file)),
+            Stream.take(limit),
+            Stream.runCollect,
+            Effect.map((chunk) => [...chunk].map((file) => `<file>${file}</file>`).join("\n")),
+          )
+
+          return {
+            title: `Loaded skill: ${info.name}`,
+            output: [
+              `<skill_content name="${info.name}">`,
+              `# Skill: ${info.name}`,
+              "",
+              info.content.trim(),
+              "",
+              `Base directory for this skill: ${base}`,
+              "Relative paths in this skill (e.g., scripts/, reference/) are relative to this base directory.",
+              "Note: file list is sampled.",
+              "",
+              "<skill_files>",
+              files,
+              "</skill_files>",
+              "</skill_content>",
+            ].join("\n"),
+            metadata: {
+              name: info.name,
+              dir,
+            },
+          }
+        }).pipe(Effect.orDie),
+    }
+  }),
+)

package/src/tool/skill.txt

@@ -0,0 +1,5 @@
+Load a specialized skill when the task at hand matches one of the skills listed in the system prompt.
+
+Use this tool to inject the skill's instructions and resources into current conversation. The output may contain detailed workflow guidance as well as references to scripts, files, etc in the same directory as the skill.
+
+The skill name must match one of the skills listed in your system prompt.

package/src/tool/task.shell.txt

@@ -0,0 +1,57 @@
+Persistent work-item tool (shell form). Tasks form a tree (T1, T2, ...;
+subtasks T1.1, T1.2, ...) with lifecycle:
+open ⇄ in_progress, either → blocked → open, either → done | abandoned.
+
+# Script structure
+#   - Multi-line: one command per non-blank line, set-e on first failure
+#   - "double" quotes preserve literal LF and process \" \\ escapes
+#   - 'single' quotes are verbatim
+#   - # starts a line comment to end-of-line (quoted # is literal text)
+
+# create:
+    task create "Implement auth"
+    # subtask: pass the parent's TID via --parent; the subtask's id nests under it (e.g. T1.1)
+    task create "Email/password login" --parent T1
+
+# multi-line summary — heredoc body is verbatim (no escape needed for " \ $ #):
+    task create <<EOF
+    Implement auth
+    - email/password
+    - oauth (google, github)
+    - regex /\b[A-Z]+_TOKEN\b/ for env scan
+    EOF
+
+# inspect:
+    task list [<status>] [--include-terminal] [--include-archived]   # default: open+in_progress+blocked
+    task list open                     # filter by status
+    task get T1
+
+# state-machine transitions:
+    task start T1 [--reason "<note>"]  # mark in_progress (working on it now)
+    task block T1 "waiting on spec"
+    task unblock T1 "spec resolved"
+    task done T1 "all tests pass"
+    task abandon T1 "out of scope"
+
+# rename:
+    task rename T1 "Updated title"
+
+# every verb accepts --session <id> to act on another session (default: current).
+
+# multi-command (one per line, set-e on first failure):
+    task create "Lexer"
+    task create "Parser"
+    task create "Type checker"
+
+Status values for `task list <status>`: open | in_progress | blocked | done | abandoned
+
+# Removed verbs (use the listed equivalents instead):
+#   progress       — progress is maintained by the checkpoint writer in the
+#                    session-level progress.md; you do not journal events
+#   approve        — proposed state was removed; new tasks start in 'open'
+#   revise         — task is rename-only; long descriptions belong in §1
+#                    Active intent or main progress.md sections
+#   create_sub     — use 'create' with parent_id
+
+# Don't repeat the task tree in your reply — the UI already shows it.
+# Just summarize what you're working on or what changed.