tulingcode 0.1.0

package/src/format/index.ts

@@ -0,0 +1,203 @@
+import { Effect, Layer, Context } from "effect"
+import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"
+import * as CrossSpawnSpawner from "@/effect/cross-spawn-spawner"
+import { InstanceState } from "@/effect"
+import path from "path"
+import { mergeDeep } from "remeda"
+import z from "zod"
+import { Config } from "../config"
+import { Log } from "../util"
+import * as Formatter from "./formatter"
+
+const log = Log.create({ service: "format" })
+
+export const Status = z
+  .object({
+    name: z.string(),
+    extensions: z.string().array(),
+    enabled: z.boolean(),
+  })
+  .meta({
+    ref: "FormatterStatus",
+  })
+export type Status = z.infer<typeof Status>
+
+export interface Interface {
+  readonly init: () => Effect.Effect<void>
+  readonly status: () => Effect.Effect<Status[]>
+  readonly file: (filepath: string) => Effect.Effect<void>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@tulingcode/Format") {}
+
+export const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const config = yield* Config.Service
+    const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
+
+    const state = yield* InstanceState.make(
+      Effect.fn("Format.state")(function* (ctx) {
+        const commands: Record<string, string[] | false> = {}
+        const formatters: Record<string, Formatter.Info> = {}
+
+        async function getCommand(item: Formatter.Info) {
+          let cmd = commands[item.name]
+          if (cmd === false || cmd === undefined) {
+            cmd = await item.enabled(ctx)
+            commands[item.name] = cmd
+          }
+          return cmd
+        }
+
+        async function isEnabled(item: Formatter.Info) {
+          const cmd = await getCommand(item)
+          return cmd !== false
+        }
+
+        async function getFormatter(ext: string) {
+          const matching = Object.values(formatters).filter((item) => item.extensions.includes(ext))
+          const checks = await Promise.all(
+            matching.map(async (item) => {
+              log.info("checking", { name: item.name, ext })
+              const cmd = await getCommand(item)
+              if (cmd) {
+                log.info("enabled", { name: item.name, ext })
+              }
+              return {
+                item,
+                cmd,
+              }
+            }),
+          )
+          return checks.filter((x) => x.cmd).map((x) => ({ item: x.item, cmd: x.cmd! }))
+        }
+
+        function formatFile(filepath: string) {
+          return Effect.gen(function* () {
+            log.info("formatting", { file: filepath })
+            const ext = path.extname(filepath)
+
+            for (const { item, cmd } of yield* Effect.promise(() => getFormatter(ext))) {
+              if (cmd === false) continue
+              log.info("running", { command: cmd })
+              const replaced = cmd.map((x) => x.replace("$FILE", filepath))
+              const dir = yield* InstanceState.directory
+              const code = yield* spawner
+                .spawn(
+                  ChildProcess.make(replaced[0]!, replaced.slice(1), {
+                    cwd: dir,
+                    env: item.environment,
+                    extendEnv: true,
+                  }),
+                )
+                .pipe(
+                  Effect.flatMap((handle) => handle.exitCode),
+                  Effect.scoped,
+                  Effect.catch(() =>
+                    Effect.sync(() => {
+                      log.error("failed to format file", {
+                        error: "spawn failed",
+                        command: cmd,
+                        ...item.environment,
+                        file: filepath,
+                      })
+                      return ChildProcessSpawner.ExitCode(1)
+                    }),
+                  ),
+                )
+              if (code !== 0) {
+                log.error("failed", {
+                  command: cmd,
+                  ...item.environment,
+                })
+              }
+            }
+          })
+        }
+
+        const cfg = yield* config.get()
+
+        if (!cfg.formatter) {
+          log.info("all formatters are disabled")
+          log.info("init")
+          return {
+            formatters,
+            isEnabled,
+            formatFile,
+          }
+        }
+
+        for (const item of Object.values(Formatter)) {
+          formatters[item.name] = item
+        }
+
+        if (cfg.formatter !== true) {
+          for (const [name, item] of Object.entries(cfg.formatter)) {
+            const builtIn = Formatter[name as keyof typeof Formatter]
+
+            // Ruff and uv are both the same formatter, so disabling either should disable both.
+            if (["ruff", "uv"].includes(name) && (cfg.formatter.ruff?.disabled || cfg.formatter.uv?.disabled)) {
+              // TODO combine formatters so shared backends like Ruff/uv don't need linked disable handling here.
+              delete formatters.ruff
+              delete formatters.uv
+              continue
+            }
+            if (item.disabled) {
+              delete formatters[name]
+              continue
+            }
+            const info = mergeDeep(builtIn ?? { extensions: [] }, item)
+
+            formatters[name] = {
+              ...info,
+              name,
+              extensions: info.extensions ?? [],
+              enabled: builtIn && !info.command ? builtIn.enabled : async (_context) => info.command ?? false,
+            }
+          }
+        }
+
+        log.info("init")
+
+        return {
+          formatters,
+          isEnabled,
+          formatFile,
+        }
+      }),
+    )
+
+    const init = Effect.fn("Format.init")(function* () {
+      yield* InstanceState.get(state)
+    })
+
+    const status = Effect.fn("Format.status")(function* () {
+      const { formatters, isEnabled } = yield* InstanceState.get(state)
+      const result: Status[] = []
+      for (const formatter of Object.values(formatters)) {
+        const isOn = yield* Effect.promise(() => isEnabled(formatter))
+        result.push({
+          name: formatter.name,
+          extensions: formatter.extensions,
+          enabled: isOn,
+        })
+      }
+      return result
+    })
+
+    const file = Effect.fn("Format.file")(function* (filepath: string) {
+      const { formatFile } = yield* InstanceState.get(state)
+      yield* formatFile(filepath)
+    })
+
+    return Service.of({ init, status, file })
+  }),
+)
+
+export const defaultLayer = layer.pipe(
+  Layer.provide(Config.defaultLayer),
+  Layer.provide(CrossSpawnSpawner.defaultLayer),
+)
+
+export * as Format from "."

package/src/git/index.ts

@@ -0,0 +1,260 @@
+import * as CrossSpawnSpawner from "@/effect/cross-spawn-spawner"
+import { Effect, Layer, Context, Stream } from "effect"
+import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"
+
+const cfg = [
+  "--no-optional-locks",
+  "-c",
+  "core.autocrlf=false",
+  "-c",
+  "core.fsmonitor=false",
+  "-c",
+  "core.longpaths=true",
+  "-c",
+  "core.symlinks=true",
+  "-c",
+  "core.quotepath=false",
+] as const
+
+const out = (result: { text(): string }) => result.text().trim()
+const nuls = (text: string) => text.split("\0").filter(Boolean)
+const fail = (err: unknown) =>
+  ({
+    exitCode: 1,
+    text: () => "",
+    stdout: Buffer.alloc(0),
+    stderr: Buffer.from(err instanceof Error ? err.message : String(err)),
+  }) satisfies Result
+
+export type Kind = "added" | "deleted" | "modified"
+
+export type Base = {
+  readonly name: string
+  readonly ref: string
+}
+
+export type Item = {
+  readonly file: string
+  readonly code: string
+  readonly status: Kind
+}
+
+export type Stat = {
+  readonly file: string
+  readonly additions: number
+  readonly deletions: number
+}
+
+export interface Result {
+  readonly exitCode: number
+  readonly text: () => string
+  readonly stdout: Buffer
+  readonly stderr: Buffer
+}
+
+export interface Options {
+  readonly cwd: string
+  readonly env?: Record<string, string>
+}
+
+export interface Interface {
+  readonly run: (args: string[], opts: Options) => Effect.Effect<Result>
+  readonly branch: (cwd: string) => Effect.Effect<string | undefined>
+  readonly prefix: (cwd: string) => Effect.Effect<string>
+  readonly defaultBranch: (cwd: string) => Effect.Effect<Base | undefined>
+  readonly hasHead: (cwd: string) => Effect.Effect<boolean>
+  readonly mergeBase: (cwd: string, base: string, head?: string) => Effect.Effect<string | undefined>
+  readonly show: (cwd: string, ref: string, file: string, prefix?: string) => Effect.Effect<string>
+  readonly status: (cwd: string) => Effect.Effect<Item[]>
+  readonly diff: (cwd: string, ref: string) => Effect.Effect<Item[]>
+  readonly stats: (cwd: string, ref: string) => Effect.Effect<Stat[]>
+}
+
+const kind = (code: string): Kind => {
+  if (code === "??") return "added"
+  if (code.includes("U")) return "modified"
+  if (code.includes("A") && !code.includes("D")) return "added"
+  if (code.includes("D") && !code.includes("A")) return "deleted"
+  return "modified"
+}
+
+export class Service extends Context.Service<Service, Interface>()("@tulingcode/Git") {}
+
+export const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
+
+    const run = Effect.fn("Git.run")(
+      function* (args: string[], opts: Options) {
+        const proc = ChildProcess.make("git", [...cfg, ...args], {
+          cwd: opts.cwd,
+          env: opts.env,
+          extendEnv: true,
+          stdin: "ignore",
+          stdout: "pipe",
+          stderr: "pipe",
+        })
+        const handle = yield* spawner.spawn(proc)
+        const [stdout, stderr] = yield* Effect.all(
+          [Stream.mkString(Stream.decodeText(handle.stdout)), Stream.mkString(Stream.decodeText(handle.stderr))],
+          { concurrency: 2 },
+        )
+        return {
+          exitCode: yield* handle.exitCode,
+          text: () => stdout,
+          stdout: Buffer.from(stdout),
+          stderr: Buffer.from(stderr),
+        } satisfies Result
+      },
+      Effect.scoped,
+      Effect.catch((err) => Effect.succeed(fail(err))),
+    )
+
+    const text = Effect.fn("Git.text")(function* (args: string[], opts: Options) {
+      return (yield* run(args, opts)).text()
+    })
+
+    const lines = Effect.fn("Git.lines")(function* (args: string[], opts: Options) {
+      return (yield* text(args, opts))
+        .split(/\r?\n/)
+        .map((item) => item.trim())
+        .filter(Boolean)
+    })
+
+    const refs = Effect.fnUntraced(function* (cwd: string) {
+      return yield* lines(["for-each-ref", "--format=%(refname:short)", "refs/heads"], { cwd })
+    })
+
+    const configured = Effect.fnUntraced(function* (cwd: string, list: string[]) {
+      const result = yield* run(["config", "init.defaultBranch"], { cwd })
+      const name = out(result)
+      if (!name || !list.includes(name)) return
+      return { name, ref: name } satisfies Base
+    })
+
+    const primary = Effect.fnUntraced(function* (cwd: string) {
+      const list = yield* lines(["remote"], { cwd })
+      if (list.includes("origin")) return "origin"
+      if (list.length === 1) return list[0]
+      if (list.includes("upstream")) return "upstream"
+      return list[0]
+    })
+
+    const branch = Effect.fn("Git.branch")(function* (cwd: string) {
+      const result = yield* run(["symbolic-ref", "--quiet", "--short", "HEAD"], { cwd })
+      if (result.exitCode !== 0) return
+      const text = out(result)
+      return text || undefined
+    })
+
+    const prefix = Effect.fn("Git.prefix")(function* (cwd: string) {
+      const result = yield* run(["rev-parse", "--show-prefix"], { cwd })
+      if (result.exitCode !== 0) return ""
+      return out(result)
+    })
+
+    const defaultBranch = Effect.fn("Git.defaultBranch")(function* (cwd: string) {
+      const remote = yield* primary(cwd)
+      if (remote) {
+        const head = yield* run(["symbolic-ref", `refs/remotes/${remote}/HEAD`], { cwd })
+        if (head.exitCode === 0) {
+          const ref = out(head).replace(/^refs\/remotes\//, "")
+          const name = ref.startsWith(`${remote}/`) ? ref.slice(`${remote}/`.length) : ""
+          if (name) return { name, ref } satisfies Base
+        }
+      }
+
+      const list = yield* refs(cwd)
+      const next = yield* configured(cwd, list)
+      if (next) return next
+      if (list.includes("main")) return { name: "main", ref: "main" } satisfies Base
+      if (list.includes("master")) return { name: "master", ref: "master" } satisfies Base
+    })
+
+    const hasHead = Effect.fn("Git.hasHead")(function* (cwd: string) {
+      const result = yield* run(["rev-parse", "--verify", "HEAD"], { cwd })
+      return result.exitCode === 0
+    })
+
+    const mergeBase = Effect.fn("Git.mergeBase")(function* (cwd: string, base: string, head = "HEAD") {
+      const result = yield* run(["merge-base", base, head], { cwd })
+      if (result.exitCode !== 0) return
+      const text = out(result)
+      return text || undefined
+    })
+
+    const show = Effect.fn("Git.show")(function* (cwd: string, ref: string, file: string, prefix = "") {
+      const target = prefix ? `${prefix}${file}` : file
+      const result = yield* run(["show", `${ref}:${target}`], { cwd })
+      if (result.exitCode !== 0) return ""
+      if (result.stdout.includes(0)) return ""
+      return result.text()
+    })
+
+    const status = Effect.fn("Git.status")(function* (cwd: string) {
+      return nuls(
+        yield* text(["status", "--porcelain=v1", "--untracked-files=all", "--no-renames", "-z", "--", "."], {
+          cwd,
+        }),
+      ).flatMap((item) => {
+        const file = item.slice(3)
+        if (!file) return []
+        const code = item.slice(0, 2)
+        return [{ file, code, status: kind(code) } satisfies Item]
+      })
+    })
+
+    const diff = Effect.fn("Git.diff")(function* (cwd: string, ref: string) {
+      const list = nuls(
+        yield* text(["diff", "--no-ext-diff", "--no-renames", "--name-status", "-z", ref, "--", "."], { cwd }),
+      )
+      return list.flatMap((code, idx) => {
+        if (idx % 2 !== 0) return []
+        const file = list[idx + 1]
+        if (!code || !file) return []
+        return [{ file, code, status: kind(code) } satisfies Item]
+      })
+    })
+
+    const stats = Effect.fn("Git.stats")(function* (cwd: string, ref: string) {
+      return nuls(
+        yield* text(["diff", "--no-ext-diff", "--no-renames", "--numstat", "-z", ref, "--", "."], { cwd }),
+      ).flatMap((item) => {
+        const a = item.indexOf("\t")
+        const b = item.indexOf("\t", a + 1)
+        if (a === -1 || b === -1) return []
+        const file = item.slice(b + 1)
+        if (!file) return []
+        const adds = item.slice(0, a)
+        const dels = item.slice(a + 1, b)
+        const additions = adds === "-" ? 0 : Number.parseInt(adds || "0", 10)
+        const deletions = dels === "-" ? 0 : Number.parseInt(dels || "0", 10)
+        return [
+          {
+            file,
+            additions: Number.isFinite(additions) ? additions : 0,
+            deletions: Number.isFinite(deletions) ? deletions : 0,
+          } satisfies Stat,
+        ]
+      })
+    })
+
+    return Service.of({
+      run,
+      branch,
+      prefix,
+      defaultBranch,
+      hasHead,
+      mergeBase,
+      show,
+      status,
+      diff,
+      stats,
+    })
+  }),
+)
+
+export const defaultLayer = layer.pipe(Layer.provide(CrossSpawnSpawner.defaultLayer))
+
+export * as Git from "."

package/src/global/index.ts

@@ -0,0 +1,54 @@
+import fs from "fs/promises"
+import path from "path"
+import os from "os"
+import { Filesystem } from "../util"
+import { Flock } from "@tuling-ai/shared/util/flock"
+import { resolvetulingcodeHome } from "@tuling-ai/shared/global"
+
+const { data, cache, config, state } = resolvetulingcodeHome()
+
+export const Path = {
+  // HOME/USERPROFILE read directly because Bun caches os.homedir() at startup.
+  // Tests set these env vars to isolate from the developer's real home.
+  get home() {
+    return process.env.HOME || process.env.USERPROFILE || os.homedir()
+  },
+  data,
+  bin: path.join(cache, "bin"),
+  log: path.join(data, "log"),
+  cache,
+  config,
+  state,
+}
+
+// Initialize Flock with global state path
+Flock.setGlobal({ state })
+
+await Promise.all([
+  fs.mkdir(Path.data, { recursive: true }),
+  fs.mkdir(Path.config, { recursive: true }),
+  fs.mkdir(Path.state, { recursive: true }),
+  fs.mkdir(Path.log, { recursive: true }),
+  fs.mkdir(Path.bin, { recursive: true }),
+])
+
+const CACHE_VERSION = "21"
+
+const version = await Filesystem.readText(path.join(Path.cache, "version")).catch(() => "0")
+
+if (version !== CACHE_VERSION) {
+  try {
+    const contents = await fs.readdir(Path.cache)
+    await Promise.all(
+      contents.map((item) =>
+        fs.rm(path.join(Path.cache, item), {
+          recursive: true,
+          force: true,
+        }),
+      ),
+    )
+  } catch {}
+  await Filesystem.write(path.join(Path.cache, "version"), CACHE_VERSION)
+}
+
+export * as Global from "."

package/src/history/backfill.ts

@@ -0,0 +1,162 @@
+import { Context, Effect, Layer } from "effect"
+import { and, asc, desc, eq, gt, sql } from "drizzle-orm"
+import { Database } from "../storage"
+import { Config } from "../config"
+import { PartTable, SessionTable } from "../session/session.sql"
+import { HistoryFtsTable } from "./fts.sql"
+import { extract, DEFAULT_KINDS, type Kind } from "./extract"
+import { makeResolver, type Resolver } from "./resolve"
+import { Log } from "../util"
+import type { MessageV2 } from "../session/message-v2"
+
+const log = Log.create({ service: "history.backfill" })
+
+const BATCH = 500
+
+/**
+ * Walk PartTable newest-session-first with the given enabled kinds.
+ * Idempotent — re-running skips already-indexed parts via NOT EXISTS.
+ * Exposed for tests so callers can pass a pre-resolved enabled set.
+ */
+export function backfillAll(enabled: ReadonlySet<Kind> = new Set(DEFAULT_KINDS)) {
+  return Effect.gen(function* () {
+    if (enabled.size === 0) return
+
+    const resolver = makeResolver()
+    const sessions = Database.use((db) =>
+      db
+        .select({ id: SessionTable.id, project_id: SessionTable.project_id })
+        .from(SessionTable)
+        .orderBy(desc(SessionTable.time_updated))
+        .all(),
+    )
+
+    for (const session of sessions) {
+      yield* scanSession(session, resolver, enabled).pipe(
+        Effect.catchCause((cause) =>
+          Effect.sync(() => log.warn("session scan failed", { session: session.id, cause: String(cause) })),
+        ),
+      )
+      yield* Effect.sleep("50 millis")
+    }
+    log.info("backfill complete", { sessions: sessions.length })
+  })
+}
+
+function scanSession(
+  session: { id: string; project_id: string },
+  resolver: Resolver,
+  enabled: ReadonlySet<Kind>,
+) {
+  return Effect.gen(function* () {
+    let cursor = ""
+    while (true) {
+      const parts = Database.use((db) =>
+        db
+          .select()
+          .from(PartTable)
+          .where(
+            and(
+              eq(PartTable.session_id, session.id as any),
+              gt(PartTable.id, cursor as any),
+              sql`NOT EXISTS (SELECT 1 FROM history_fts WHERE history_fts.part_id = ${PartTable.id})`,
+            ),
+          )
+          .orderBy(asc(PartTable.id))
+          .limit(BATCH)
+          .all(),
+      )
+      if (parts.length === 0) return
+
+      yield* writeBatch(parts, session.project_id, resolver, enabled)
+      cursor = parts[parts.length - 1]!.id
+      yield* Effect.sleep("10 millis")
+    }
+  })
+}
+
+function writeBatch(
+  parts: Array<{ id: string; session_id: string; message_id: string; data: unknown; time_created: number }>,
+  projectID: string,
+  resolver: Resolver,
+  enabled: ReadonlySet<Kind>,
+) {
+  return Effect.gen(function* () {
+    type ToWrite = {
+      part: (typeof parts)[number]
+      kind: Kind
+      body: string
+      tool_name: string | null
+      time: number
+    }
+    const writes: ToWrite[] = []
+    for (const p of parts) {
+      const role = yield* resolver.role(p.message_id)
+      // Reconstruct the MessageV2.Part shape that extract() expects.
+      // PartTable.data stores everything except id/sessionID/messageID.
+      const fullPart = {
+        id: p.id,
+        sessionID: p.session_id,
+        messageID: p.message_id,
+        ...(p.data as object),
+      } as MessageV2.Part
+      const extracted = extract(fullPart, role, enabled)
+      if (!extracted) continue
+      writes.push({
+        part: p,
+        kind: extracted.kind,
+        body: extracted.body,
+        tool_name: extracted.tool_name,
+        time: p.time_created,
+      })
+    }
+    if (writes.length === 0) return
+    Database.transaction((tx) => {
+      for (const w of writes) {
+        tx.insert(HistoryFtsTable)
+          .values({
+            part_id: w.part.id,
+            session_id: w.part.session_id,
+            message_id: w.part.message_id,
+            project_id: projectID,
+            kind: w.kind,
+            tool_name: w.tool_name,
+            body: w.body,
+            time_created: w.time,
+          })
+          .onConflictDoUpdate({
+            target: HistoryFtsTable.part_id,
+            set: { kind: w.kind, tool_name: w.tool_name, body: w.body, time_created: w.time },
+          })
+          .run()
+      }
+    })
+  })
+}
+
+export interface Interface {
+  readonly init: () => Effect.Effect<void>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@tulingcode/History.Backfill") {}
+
+export const layer: Layer.Layer<Service, never, Config.Service> = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const cfg = yield* Config.Service
+    return Service.of({
+      init: Effect.fn("History.Backfill.init")(function* () {
+        const config = yield* cfg.get()
+        const kinds = config.history?.kinds ?? DEFAULT_KINDS
+        const enabled = new Set<Kind>(kinds as readonly Kind[])
+        // Fire-and-forget: do not block bootstrap on the potentially long scan.
+        yield* backfillAll(enabled).pipe(
+          Effect.catchCause((cause) =>
+            Effect.sync(() => log.warn("backfill aborted", { cause: String(cause) })),
+          ),
+          Effect.forkDetach,
+        )
+      }),
+    })
+  }),
+)