saeeol 1.3.0 → 1.3.1

package/test/saeeol/tool-encoding.test.ts

@@ -0,0 +1,455 @@
+// Integration tests verifying that the agent file tools (read, write, edit,
+// apply_patch) detect and preserve the original encoding of files on disk.
+// Tests exercise the real tool pipeline rather than the Encoding helper
+// directly so we validate end-to-end behaviour.
+
+import { afterEach, describe, expect } from "bun:test"
+import { Effect, Layer } from "effect"
+import path from "path"
+import fs from "fs/promises"
+import iconv from "iconv-lite"
+import { Agent } from "../../src/agent/agent"
+import { AppFileSystem } from "@saeeol/core/filesystem"
+import { ApplyPatchTool } from "../../src/tool/apply_patch"
+import { Bus } from "../../src/bus"
+import * as CrossSpawnSpawner from "@saeeol/core/cross-spawn-spawner"
+import { EditTool } from "../../src/tool/edit"
+import { Format } from "../../src/format"
+import { Instance } from "../../src/project/instance"
+import { Instruction } from "../../src/session/instruction"
+import { LSP } from "../../src/lsp/lsp"
+import { MessageID, SessionID } from "../../src/session/schema"
+import { ReadTool } from "../../src/tool/read"
+import * as Tool from "../../src/tool/tool"
+import { Truncate } from "../../src/tool/truncate"
+import { WriteTool } from "../../src/tool/write"
+import { disposeAllInstances, provideTmpdirInstance } from "../fixture/fixture"
+import { testEffect } from "../lib/effect"
+
+const ctx = {
+  sessionID: SessionID.make("ses_test-encoding"),
+  messageID: MessageID.make(""),
+  callID: "",
+  agent: "build",
+  abort: AbortSignal.any([]),
+  messages: [],
+  metadata: () => Effect.void,
+  ask: () => Effect.void,
+}
+
+afterEach(async () => {
+  await disposeAllInstances()
+})
+
+const it = testEffect(
+  Layer.mergeAll(
+    Agent.defaultLayer,
+    AppFileSystem.defaultLayer,
+    CrossSpawnSpawner.defaultLayer,
+    Instruction.defaultLayer,
+    LSP.defaultLayer,
+    Bus.layer,
+    Format.defaultLayer,
+    Truncate.defaultLayer,
+  ),
+)
+
+const runRead = (args: Tool.InferParameters<typeof ReadTool>) =>
+  Effect.gen(function* () {
+    const info = yield* ReadTool
+    const tool = yield* info.init()
+    return yield* tool.execute(args, ctx)
+  })
+
+const runWrite = (args: Tool.InferParameters<typeof WriteTool>) =>
+  Effect.gen(function* () {
+    const info = yield* WriteTool
+    const tool = yield* info.init()
+    return yield* tool.execute(args, ctx)
+  })
+
+const runEdit = (args: Tool.InferParameters<typeof EditTool>) =>
+  Effect.gen(function* () {
+    const info = yield* EditTool
+    const tool = yield* info.init()
+    return yield* tool.execute(args, ctx)
+  })
+
+const runPatch = (args: Tool.InferParameters<typeof ApplyPatchTool>) =>
+  Effect.gen(function* () {
+    const info = yield* ApplyPatchTool
+    const tool = yield* info.init()
+    return yield* tool.execute(args, ctx)
+  })
+
+// FileTime was removed upstream; edit/write no longer require a prior read.
+const markRead = (_filepath: string) => Effect.void
+
+// iconv-lite's UTF codecs don't emit BOMs, but this codebase supports
+// "UTF-X with BOM" as a distinct variant. Prepend one here for fixture files
+// that are meant to have one.
+const UTF8_BOM = "utf-8-bom"
+const encodeBytes = (text: string, encoding: string): Buffer => {
+  if (encoding === UTF8_BOM) return Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), iconv.encode(text, "utf-8")])
+  const lower = encoding.toLowerCase()
+  if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)])
+  if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)])
+  if (lower === "utf-32le") return Buffer.concat([Buffer.from([0xff, 0xfe, 0x00, 0x00]), iconv.encode(text, encoding)])
+  if (lower === "utf-32be") return Buffer.concat([Buffer.from([0x00, 0x00, 0xfe, 0xff]), iconv.encode(text, encoding)])
+  return iconv.encode(text, encoding)
+}
+
+// Create a file with the given encoding by writing raw bytes.
+const putEncoded = (filepath: string, text: string, encoding: string) =>
+  Effect.promise(async () => {
+    await fs.mkdir(path.dirname(filepath), { recursive: true })
+    await fs.writeFile(filepath, encodeBytes(text, encoding))
+  })
+
+const loadDecoded = (filepath: string, encoding: string) =>
+  Effect.promise(async () => {
+    const bytes = await fs.readFile(filepath)
+    if (encoding === UTF8_BOM) {
+      const stripped = bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
+      return iconv.decode(stripped ? bytes.subarray(3) : bytes, "utf-8")
+    }
+    return iconv.decode(bytes, encoding)
+  })
+
+const loadBytes = (filepath: string) => Effect.promise(() => fs.readFile(filepath))
+
+// Sample phrases chosen to exercise each encoding's characteristic byte patterns.
+const samples = {
+  utf8: "Hello, world! — £100",
+  shiftJis: "こんにちは、世界！日本語のテストです。",
+  eucJp: "日本語のEUC-JPテスト文字列です。",
+  gb2312: "你好，世界！这是简体中文测试。",
+  big5: "你好，世界！這是繁體中文測試。",
+  eucKr: "안녕하세요, 세계! 한국어 테스트입니다.",
+  windows1251: "Привет, мир! Это тест кириллицы.",
+  koi8r: "Привет, мир! КОИ-8 Р тест.",
+}
+
+describe("tool encoding preservation", () => {
+  describe("ReadTool decodes files with non-UTF-8 encodings", () => {
+    const cases: Array<[string, string, string]> = [
+      ["UTF-8", "utf-8", samples.utf8],
+      ["UTF-8 with BOM", UTF8_BOM, samples.utf8],
+      ["UTF-16 LE with BOM", "utf-16le", samples.utf8],
+      ["UTF-16 BE with BOM", "utf-16be", samples.utf8],
+      ["UTF-32 LE with BOM", "utf-32le", samples.utf8],
+      ["UTF-32 BE with BOM", "utf-32be", samples.utf8],
+      ["Shift_JIS", "Shift_JIS", samples.shiftJis],
+      ["EUC-JP", "euc-jp", samples.eucJp],
+      ["GB2312", "gb2312", samples.gb2312],
+      ["Big5", "big5", samples.big5],
+      ["EUC-KR", "euc-kr", samples.eucKr],
+      ["Windows-1251", "windows-1251", samples.windows1251],
+      ["KOI8-R", "koi8-r", samples.koi8r],
+    ]
+
+    for (const [label, encoding, text] of cases) {
+      it.live(`decodes ${label} content for the model`, () =>
+        provideEncoded(encoding, text, (filepath) =>
+          Effect.gen(function* () {
+            const result = yield* runRead({ filePath: filepath })
+            expect(result.output).toContain(text)
+          }),
+        ),
+      )
+    }
+  })
+
+  describe("ReadTool does not flag non-Latin text files as binary", () => {
+    it.live("accepts Shift_JIS", () =>
+      provideEncoded("Shift_JIS", samples.shiftJis, (filepath) =>
+        Effect.gen(function* () {
+          const result = yield* runRead({ filePath: filepath })
+          expect(result.output).toContain(samples.shiftJis)
+        }),
+      ),
+    )
+
+    it.live("accepts UTF-16 LE with BOM (contains NUL bytes)", () =>
+      provideEncoded("utf-16le", samples.utf8, (filepath) =>
+        Effect.gen(function* () {
+          const result = yield* runRead({ filePath: filepath })
+          expect(result.output).toContain(samples.utf8)
+        }),
+      ),
+    )
+
+    it.live("accepts UTF-32 LE with BOM (3 of every 4 bytes are NUL)", () =>
+      provideEncoded("utf-32le", samples.utf8, (filepath) =>
+        Effect.gen(function* () {
+          const result = yield* runRead({ filePath: filepath })
+          expect(result.output).toContain(samples.utf8)
+        }),
+      ),
+    )
+  })
+
+  describe("WriteTool preserves existing file encoding when overwriting", () => {
+    const cases: Array<[string, string, string]> = [
+      ["UTF-8 with BOM", UTF8_BOM, samples.utf8],
+      ["Shift_JIS", "Shift_JIS", samples.shiftJis],
+      ["GB2312", "gb2312", samples.gb2312],
+      ["Windows-1251", "windows-1251", samples.windows1251],
+      ["UTF-16 LE", "utf-16le", samples.utf8],
+      ["UTF-32 LE", "utf-32le", samples.utf8],
+      ["UTF-32 BE", "utf-32be", samples.utf8],
+    ]
+
+    for (const [label, encoding, original] of cases) {
+      it.live(`preserves ${label} encoding on overwrite`, () =>
+        provideTmpdirInstance((dir) =>
+          Effect.gen(function* () {
+            const filepath = path.join(dir, "file.txt")
+            yield* putEncoded(filepath, original, encoding)
+            yield* markRead(filepath)
+
+            const replacement = original + " updated"
+            yield* runWrite({ filePath: filepath, content: replacement })
+
+            const decoded = yield* loadDecoded(filepath, encoding)
+            expect(decoded).toBe(replacement)
+
+            // Bytes should still match the original encoding (and differ from UTF-8).
+            const bytes = yield* loadBytes(filepath)
+            expect(bytes.equals(encodeBytes(replacement, encoding))).toBe(true)
+          }),
+        ),
+      )
+    }
+
+    it.live("defaults new files to UTF-8", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const filepath = path.join(dir, "new.txt")
+          yield* runWrite({ filePath: filepath, content: samples.utf8 })
+
+          const bytes = yield* loadBytes(filepath)
+          expect(bytes.equals(Buffer.from(samples.utf8, "utf-8"))).toBe(true)
+        }),
+      ),
+    )
+
+    // Guard against double-BOM regressions: if the model ever hands back content
+    // that already starts with U+FEFF (e.g. by round-tripping literal bytes),
+    // writing it to a BOM-encoded file must still produce exactly one BOM.
+    const bomCases: Array<[string, string, Buffer]> = [
+      ["UTF-8 with BOM", UTF8_BOM, Buffer.from([0xef, 0xbb, 0xbf])],
+      ["UTF-16 LE", "utf-16le", Buffer.from([0xff, 0xfe])],
+      ["UTF-16 BE", "utf-16be", Buffer.from([0xfe, 0xff])],
+      ["UTF-32 LE", "utf-32le", Buffer.from([0xff, 0xfe, 0x00, 0x00])],
+      ["UTF-32 BE", "utf-32be", Buffer.from([0x00, 0x00, 0xfe, 0xff])],
+    ]
+    for (const [label, encoding, bom] of bomCases) {
+      it.live(`does not emit a double BOM for ${label} when content starts with U+FEFF`, () =>
+        provideTmpdirInstance((dir) =>
+          Effect.gen(function* () {
+            const filepath = path.join(dir, "file.txt")
+            yield* putEncoded(filepath, "hello", encoding)
+            yield* markRead(filepath)
+
+            yield* runWrite({ filePath: filepath, content: "\uFEFFgoodbye" })
+
+            const bytes = yield* loadBytes(filepath)
+            // Exactly one BOM prefix, immediately followed by encoded payload.
+            expect(bytes.subarray(0, bom.length).equals(bom)).toBe(true)
+            expect(bytes.subarray(bom.length, bom.length * 2).equals(bom)).toBe(false)
+            const decoded = yield* loadDecoded(filepath, encoding)
+            expect(decoded).toBe("goodbye")
+          }),
+        ),
+      )
+    }
+  })
+
+  describe("EditTool preserves existing file encoding across edits", () => {
+    const cases: Array<[string, string, string, string, string]> = [
+      ["UTF-8 with BOM", UTF8_BOM, samples.utf8 + "\n second line", "world", "earth"],
+      ["Shift_JIS", "Shift_JIS", samples.shiftJis, "日本語", "ニホンゴ"],
+      ["GB2312", "gb2312", samples.gb2312, "简体中文", "中文简体"],
+      ["Windows-1251", "windows-1251", samples.windows1251, "мир", "планета"],
+      ["UTF-16 LE", "utf-16le", samples.utf8 + "\n second line", "world", "earth"],
+      ["UTF-32 LE", "utf-32le", samples.utf8 + "\n second line", "world", "earth"],
+    ]
+
+    for (const [label, encoding, original, oldString, newString] of cases) {
+      it.live(`preserves ${label} through edit`, () =>
+        provideTmpdirInstance((dir) =>
+          Effect.gen(function* () {
+            const filepath = path.join(dir, "doc.txt")
+            yield* putEncoded(filepath, original, encoding)
+            yield* markRead(filepath)
+
+            yield* runEdit({ filePath: filepath, oldString, newString })
+
+            const decoded = yield* loadDecoded(filepath, encoding)
+            const expected = original.replace(oldString, newString)
+            expect(decoded).toBe(expected)
+
+            const bytes = yield* loadBytes(filepath)
+            expect(bytes.equals(encodeBytes(expected, encoding))).toBe(true)
+          }),
+        ),
+      )
+    }
+  })
+
+  describe("ApplyPatchTool preserves encoding", () => {
+    it.live("preserves Shift_JIS through an update hunk", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const filepath = path.join(dir, "doc.txt")
+          const replacement = "日本語"
+          const original = "line1\n" + samples.shiftJis + "\nline3\n"
+          const expected = original.replace(samples.shiftJis, replacement)
+          yield* putEncoded(filepath, original, "Shift_JIS")
+
+          const patch = [
+            "*** Begin Patch",
+            "*** Update File: doc.txt",
+            "@@",
+            " line1",
+            "-" + samples.shiftJis,
+            "+" + replacement,
+            " line3",
+            "*** End Patch",
+          ].join("\n")
+
+          yield* runPatch({ patchText: patch })
+
+          const decoded = yield* loadDecoded(filepath, "Shift_JIS")
+          expect(decoded).toBe(expected)
+
+          // Bytes must still be Shift_JIS, not silently promoted to UTF-8.
+          const bytes = yield* loadBytes(filepath)
+          expect(bytes.equals(encodeBytes(expected, "Shift_JIS"))).toBe(true)
+        }),
+      ),
+    )
+
+    // Regression guard: the diff and additions/deletions counts surfaced to the
+    // user (and to the permission prompt) are derived from the pre-patch read
+    // of the file. A previous version reused a hard-coded UTF-8 decoder for
+    // that read, producing mojibake for any non-UTF-8 file. The bytes ended up
+    // correct because the patch helper does its own encoding-aware read, so
+    // tests that only checked final file bytes (above) missed the bug.
+    it.live("returns a non-mojibake diff for a Shift_JIS update", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const filepath = path.join(dir, "doc.txt")
+          const replacement = "日本語"
+          const original = "line1\n" + samples.shiftJis + "\nline3\n"
+          yield* putEncoded(filepath, original, "Shift_JIS")
+
+          const patch = [
+            "*** Begin Patch",
+            "*** Update File: doc.txt",
+            "@@",
+            " line1",
+            "-" + samples.shiftJis,
+            "+" + replacement,
+            " line3",
+            "*** End Patch",
+          ].join("\n")
+
+          const result = (yield* runPatch({ patchText: patch })) as {
+            metadata: {
+              diff: string
+              files: Array<{ additions: number; deletions: number }>
+            }
+          }
+
+          // The diff must contain the real decoded old/new lines, not a UTF-8
+          // misread of the Shift_JIS bytes (which would surface as U+FFFD).
+          expect(result.metadata.diff).toContain(samples.shiftJis)
+          expect(result.metadata.diff).toContain(replacement)
+          expect(result.metadata.diff).not.toContain("\uFFFD")
+
+          // Per-file stats are derived from the same diff, so a mojibake read
+          // would inflate both additions and deletions.
+          expect(result.metadata.files).toHaveLength(1)
+          expect(result.metadata.files[0].additions).toBe(1)
+          expect(result.metadata.files[0].deletions).toBe(1)
+        }),
+      ),
+    )
+
+    it.live("new files added via apply_patch are UTF-8", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const patch = ["*** Begin Patch", "*** Add File: new.txt", "+hello world", "*** End Patch"].join("\n")
+          yield* runPatch({ patchText: patch })
+          const bytes = yield* loadBytes(path.join(dir, "new.txt"))
+          expect(bytes.equals(Buffer.from("hello world\n", "utf-8"))).toBe(true)
+        }),
+      ),
+    )
+
+    // Deletes exercise a code path in patch/index.ts that doesn't write bytes
+    // back — verify it still works when the target file is non-UTF-8, because
+    // the deletion code has to decode the old contents to confirm match.
+    it.live("deletes a Windows-1251 file without UTF-8 corruption errors", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const filepath = path.join(dir, "legacy.txt")
+          yield* putEncoded(filepath, samples.windows1251, "windows-1251")
+          const patch = ["*** Begin Patch", "*** Delete File: legacy.txt", "*** End Patch"].join("\n")
+          yield* runPatch({ patchText: patch })
+          const exists = yield* Effect.promise(() =>
+            fs
+              .access(filepath)
+              .then(() => true)
+              .catch(() => false),
+          )
+          expect(exists).toBe(false)
+        }),
+      ),
+    )
+  })
+
+  // EditTool's replaceAll path rewrites the entire buffer and re-encodes it
+  // in one shot — regression guard that re-encoding a multi-occurrence edit in
+  // a legacy encoding yields byte-exact output.
+  describe("EditTool replaceAll preserves non-UTF-8 encoding", () => {
+    it.live("replaces every occurrence in Shift_JIS", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const filepath = path.join(dir, "doc.txt")
+          // Pad with additional Shift_JIS text so chardet has enough bytes
+          // to confidently identify the encoding.
+          const pad = samples.shiftJis + "\n"
+          const original = pad + "日本語\n日本語\n日本語\n" + pad
+          yield* putEncoded(filepath, original, "Shift_JIS")
+          yield* markRead(filepath)
+
+          yield* runEdit({ filePath: filepath, oldString: "日本語", newString: "ニホンゴ", replaceAll: true })
+
+          const expected =
+            pad.replaceAll("日本語", "ニホンゴ") +
+            "ニホンゴ\nニホンゴ\nニホンゴ\n" +
+            pad.replaceAll("日本語", "ニホンゴ")
+          const decoded = yield* loadDecoded(filepath, "Shift_JIS")
+          expect(decoded).toBe(expected)
+          const bytes = yield* loadBytes(filepath)
+          expect(bytes.equals(encodeBytes(expected, "Shift_JIS"))).toBe(true)
+        }),
+      ),
+    )
+  })
+})
+
+// Shared helper to set up a temp instance with an encoded file at `file.txt`.
+function provideEncoded<A, E, R>(encoding: string, text: string, body: (filepath: string) => Effect.Effect<A, E, R>) {
+  return provideTmpdirInstance((dir) =>
+    Effect.gen(function* () {
+      const filepath = path.join(dir, "file.txt")
+      yield* putEncoded(filepath, text, encoding)
+      yield* markRead(filepath)
+      return yield* body(filepath)
+    }),
+  )
+}

package/test/saeeol/tool-registry-indexing-import-failure.test.ts

@@ -0,0 +1,49 @@
+import { afterEach, describe, expect, mock, spyOn } from "bun:test"
+import { Effect, Layer } from "effect"
+import * as Log from "@saeeol/core/util/log"
+import { Instance } from "../../src/project/instance"
+import { disposeAllInstances, provideTmpdirInstance } from "../fixture/fixture"
+import * as CrossSpawnSpawner from "@saeeol/core/cross-spawn-spawner"
+import { testEffect } from "../lib/effect"
+
+const err = new Error("indexing import failed")
+
+mock.module("@/saeeol/indexing", () => {
+  throw err
+})
+
+const { ToolRegistry } = await import("../../src/tool/registry")
+
+const node = CrossSpawnSpawner.defaultLayer
+const it = testEffect(Layer.mergeAll(ToolRegistry.defaultLayer, node))
+
+afterEach(async () => {
+  await disposeAllInstances()
+})
+
+describe("saeeol tool registry indexing import failure", () => {
+  it.live("keeps non-indexing tools when the indexing module cannot load", () =>
+    provideTmpdirInstance(
+      () =>
+        Effect.gen(function* () {
+          const logger = Log.create({ service: "saeeol-tool-registry" })
+          const warn = spyOn(logger, "warn").mockImplementation(() => {})
+
+          try {
+            const registry = yield* ToolRegistry.Service
+            const ids = yield* registry.ids()
+
+            expect(ids).not.toContain("semantic_search")
+            expect(ids).toContain("question")
+            expect(ids).toContain("read")
+            expect(ids).toContain("suggest")
+            expect(warn.mock.calls[0]?.[0]).toBe("semantic search unavailable")
+            expect(warn.mock.calls[0]?.[1]?.err).toBeDefined()
+          } finally {
+            warn.mockRestore()
+          }
+        }),
+      { git: true },
+    ),
+  )
+})