tulingcode 0.1.0

package/src/sql.d.ts

@@ -0,0 +1,4 @@
+declare module "*.sql" {
+  const content: string
+  export default content
+}

package/src/storage/db.bun.ts

@@ -0,0 +1,8 @@
+import { Database } from "bun:sqlite"
+import { drizzle } from "drizzle-orm/bun-sqlite"
+
+export function init(path: string) {
+  const sqlite = new Database(path, { create: true })
+  const db = drizzle({ client: sqlite })
+  return db
+}

package/src/storage/db.node.ts

@@ -0,0 +1,8 @@
+import { DatabaseSync } from "node:sqlite"
+import { drizzle } from "drizzle-orm/node-sqlite"
+
+export function init(path: string) {
+  const sqlite = new DatabaseSync(path)
+  const db = drizzle({ client: sqlite })
+  return db
+}

package/src/storage/db.ts

@@ -0,0 +1,172 @@
+import { type SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"
+import { migrate } from "drizzle-orm/bun-sqlite/migrator"
+import { type SQLiteTransaction } from "drizzle-orm/sqlite-core"
+export * from "drizzle-orm"
+import { LocalContext } from "../util"
+import { lazy } from "../util/lazy"
+import { Global } from "../global"
+import { Log } from "../util"
+import { NamedError } from "@tuling-ai/shared/util/error"
+import z from "zod"
+import path from "path"
+import { readFileSync, readdirSync, existsSync } from "fs"
+import { Flag } from "../flag/flag"
+import { InstallationChannel } from "../installation/version"
+import { InstanceState } from "@/effect"
+import { iife } from "@/util/iife"
+import { init } from "#db"
+
+declare const OPENCODE_MIGRATIONS: { sql: string; timestamp: number; name: string }[] | undefined
+
+export const NotFoundError = NamedError.create(
+  "NotFoundError",
+  z.object({
+    message: z.string(),
+  }),
+)
+
+const log = Log.create({ service: "db" })
+
+export function getChannelPath() {
+  if (["latest", "beta", "prod"].includes(InstallationChannel) || Flag.tulingcode_DISABLE_CHANNEL_DB)
+    return path.join(Global.Path.data, "tulingcode.db")
+  const safe = InstallationChannel.replace(/[^a-zA-Z0-9._-]/g, "-")
+  return path.join(Global.Path.data, `tulingcode-${safe}.db`)
+}
+
+export const Path = iife(() => {
+  if (Flag.tulingcode_DB) {
+    if (Flag.tulingcode_DB === ":memory:" || path.isAbsolute(Flag.tulingcode_DB)) return Flag.tulingcode_DB
+    return path.join(Global.Path.data, Flag.tulingcode_DB)
+  }
+  return getChannelPath()
+})
+
+export type Transaction = SQLiteTransaction<"sync", void>
+
+type Client = SQLiteBunDatabase
+
+type Journal = { sql: string; timestamp: number; name: string }[]
+
+function time(tag: string) {
+  const match = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/.exec(tag)
+  if (!match) return 0
+  return Date.UTC(
+    Number(match[1]),
+    Number(match[2]) - 1,
+    Number(match[3]),
+    Number(match[4]),
+    Number(match[5]),
+    Number(match[6]),
+  )
+}
+
+function migrations(dir: string): Journal {
+  const dirs = readdirSync(dir, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+
+  const sql = dirs
+    .map((name) => {
+      const file = path.join(dir, name, "migration.sql")
+      if (!existsSync(file)) return
+      return {
+        sql: readFileSync(file, "utf-8"),
+        timestamp: time(name),
+        name,
+      }
+    })
+    .filter(Boolean) as Journal
+
+  return sql.sort((a, b) => a.timestamp - b.timestamp)
+}
+
+export const Client = lazy(() => {
+  log.info("opening database", { path: Path })
+
+  const db = init(Path)
+
+  db.run("PRAGMA journal_mode = WAL")
+  db.run("PRAGMA synchronous = NORMAL")
+  db.run("PRAGMA busy_timeout = 5000")
+  db.run("PRAGMA cache_size = -64000")
+  db.run("PRAGMA foreign_keys = ON")
+  db.run("PRAGMA wal_checkpoint(PASSIVE)")
+
+  // Apply schema migrations
+  const entries =
+    typeof OPENCODE_MIGRATIONS !== "undefined"
+      ? OPENCODE_MIGRATIONS
+      : migrations(path.join(import.meta.dirname, "../../migration"))
+  if (entries.length > 0) {
+    log.info("applying migrations", {
+      count: entries.length,
+      mode: typeof OPENCODE_MIGRATIONS !== "undefined" ? "bundled" : "dev",
+    })
+    if (Flag.tulingcode_SKIP_MIGRATIONS) {
+      for (const item of entries) {
+        item.sql = "select 1;"
+      }
+    }
+    migrate(db, entries)
+  }
+
+  return db
+})
+
+export function close() {
+  Client().$client.close()
+  Client.reset()
+}
+
+export type TxOrDb = Transaction | Client
+
+const ctx = LocalContext.create<{
+  tx: TxOrDb
+  effects: (() => void | Promise<void>)[]
+}>("database")
+
+export function use<T>(callback: (trx: TxOrDb) => T): T {
+  try {
+    return callback(ctx.use().tx)
+  } catch (err) {
+    if (err instanceof LocalContext.NotFound) {
+      const effects: (() => void | Promise<void>)[] = []
+      const result = ctx.provide({ effects, tx: Client() }, () => callback(Client()))
+      for (const effect of effects) effect()
+      return result
+    }
+    throw err
+  }
+}
+
+export function effect(fn: () => any | Promise<any>) {
+  const bound = InstanceState.bind(fn)
+  try {
+    ctx.use().effects.push(bound)
+  } catch {
+    bound()
+  }
+}
+
+type NotPromise<T> = T extends Promise<any> ? never : T
+
+export function transaction<T>(
+  callback: (tx: TxOrDb) => NotPromise<T>,
+  options?: {
+    behavior?: "deferred" | "immediate" | "exclusive"
+  },
+): NotPromise<T> {
+  try {
+    return callback(ctx.use().tx)
+  } catch (err) {
+    if (err instanceof LocalContext.NotFound) {
+      const effects: (() => void | Promise<void>)[] = []
+      const txCallback = InstanceState.bind((tx: TxOrDb) => ctx.provide({ tx, effects }, () => callback(tx)))
+      const result = Client().transaction(txCallback, { behavior: options?.behavior })
+      for (const effect of effects) effect()
+      return result as NotPromise<T>
+    }
+    throw err
+  }
+}

package/src/storage/index.ts

@@ -0,0 +1,26 @@
+export * as JsonMigration from "./json-migration"
+export * as Database from "./db"
+export * as Storage from "./storage"
+export {
+  asc,
+  eq,
+  and,
+  or,
+  inArray,
+  desc,
+  not,
+  sql,
+  isNull,
+  isNotNull,
+  count,
+  like,
+  exists,
+  between,
+  gt,
+  gte,
+  lt,
+  lte,
+  ne,
+} from "drizzle-orm"
+export type { SQL } from "drizzle-orm"
+export { NotFoundError } from "./storage"

package/src/storage/json-migration.ts

@@ -0,0 +1,426 @@
+import type { SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"
+import type { NodeSQLiteDatabase } from "drizzle-orm/node-sqlite"
+import { Global } from "../global"
+import { Log } from "../util"
+import { ProjectTable } from "../project/project.sql"
+import { SessionTable, MessageTable, PartTable, TodoTable, PermissionTable } from "../session/session.sql"
+import { SessionShareTable } from "../share/share.sql"
+import path from "path"
+import { existsSync } from "fs"
+import { Filesystem } from "../util"
+import { Glob } from "@tuling-ai/shared/util/glob"
+
+const log = Log.create({ service: "json-migration" })
+
+export type Progress = {
+  current: number
+  total: number
+  label: string
+}
+
+type Options = {
+  progress?: (event: Progress) => void
+}
+
+export async function run(db: SQLiteBunDatabase<any, any> | NodeSQLiteDatabase<any, any>, options?: Options) {
+  const storageDir = path.join(Global.Path.data, "storage")
+
+  if (!existsSync(storageDir)) {
+    log.info("storage directory does not exist, skipping migration")
+    return {
+      projects: 0,
+      sessions: 0,
+      messages: 0,
+      parts: 0,
+      todos: 0,
+      permissions: 0,
+      shares: 0,
+      errors: [] as string[],
+    }
+  }
+
+  log.info("starting json to sqlite migration", { storageDir })
+  const start = performance.now()
+
+  // const db = drizzle({ client: sqlite })
+
+  // Optimize SQLite for bulk inserts
+  db.run("PRAGMA journal_mode = WAL")
+  db.run("PRAGMA synchronous = OFF")
+  db.run("PRAGMA cache_size = 10000")
+  db.run("PRAGMA temp_store = MEMORY")
+  const stats = {
+    projects: 0,
+    sessions: 0,
+    messages: 0,
+    parts: 0,
+    todos: 0,
+    permissions: 0,
+    shares: 0,
+    errors: [] as string[],
+  }
+  const orphans = {
+    sessions: 0,
+    todos: 0,
+    permissions: 0,
+    shares: 0,
+  }
+  const errs = stats.errors
+
+  const batchSize = 1000
+  const now = Date.now()
+
+  async function list(pattern: string) {
+    return Glob.scan(pattern, { cwd: storageDir, absolute: true })
+  }
+
+  async function read(files: string[], start: number, end: number) {
+    const count = end - start
+    // oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
+    const tasks = new Array(count)
+    for (let i = 0; i < count; i++) {
+      tasks[i] = Filesystem.readJson(files[start + i])
+    }
+    const results = await Promise.allSettled(tasks)
+    // oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
+    const items = new Array(count)
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]
+      if (result.status === "fulfilled") {
+        items[i] = result.value
+        continue
+      }
+      errs.push(`failed to read ${files[start + i]}: ${result.reason}`)
+    }
+    return items
+  }
+
+  function insert(values: unknown[], table: Parameters<typeof db.insert>[0], label: string) {
+    if (values.length === 0) return 0
+    try {
+      db.insert(table).values(values).onConflictDoNothing().run()
+      return values.length
+    } catch (e) {
+      errs.push(`failed to migrate ${label} batch: ${e}`)
+      return 0
+    }
+  }
+
+  // Pre-scan all files upfront to avoid repeated glob operations
+  log.info("scanning files...")
+  const [projectFiles, sessionFiles, messageFiles, partFiles, todoFiles, permFiles, shareFiles] = await Promise.all([
+    list("project/*.json"),
+    list("session/*/*.json"),
+    list("message/*/*.json"),
+    list("part/*/*.json"),
+    list("todo/*.json"),
+    list("permission/*.json"),
+    list("session_share/*.json"),
+  ])
+
+  log.info("file scan complete", {
+    projects: projectFiles.length,
+    sessions: sessionFiles.length,
+    messages: messageFiles.length,
+    parts: partFiles.length,
+    todos: todoFiles.length,
+    permissions: permFiles.length,
+    shares: shareFiles.length,
+  })
+
+  const total = Math.max(
+    1,
+    projectFiles.length +
+      sessionFiles.length +
+      messageFiles.length +
+      partFiles.length +
+      todoFiles.length +
+      permFiles.length +
+      shareFiles.length,
+  )
+  const progress = options?.progress
+  let current = 0
+  const step = (label: string, count: number) => {
+    current = Math.min(total, current + count)
+    progress?.({ current, total, label })
+  }
+
+  progress?.({ current, total, label: "starting" })
+
+  db.run("BEGIN TRANSACTION")
+
+  // Migrate projects first (no FK deps)
+  // Derive all IDs from file paths, not JSON content
+  const projectIds = new Set<string>()
+  const projectValues: unknown[] = []
+  for (let i = 0; i < projectFiles.length; i += batchSize) {
+    const end = Math.min(i + batchSize, projectFiles.length)
+    const batch = await read(projectFiles, i, end)
+    projectValues.length = 0
+    for (let j = 0; j < batch.length; j++) {
+      const data = batch[j]
+      if (!data) continue
+      const id = path.basename(projectFiles[i + j], ".json")
+      projectIds.add(id)
+      projectValues.push({
+        id,
+        worktree: data.worktree ?? "/",
+        vcs: data.vcs,
+        name: data.name ?? undefined,
+        icon_url: data.icon?.url,
+        icon_color: data.icon?.color,
+        time_created: data.time?.created ?? now,
+        time_updated: data.time?.updated ?? now,
+        time_initialized: data.time?.initialized,
+        sandboxes: data.sandboxes ?? [],
+        commands: data.commands,
+      })
+    }
+    stats.projects += insert(projectValues, ProjectTable, "project")
+    step("projects", end - i)
+  }
+  log.info("migrated projects", { count: stats.projects, duration: Math.round(performance.now() - start) })
+
+  // Migrate sessions (depends on projects)
+  // Derive all IDs from directory/file paths, not JSON content, since earlier
+  // migrations may have moved sessions to new directories without updating the JSON
+  const sessionProjects = sessionFiles.map((file) => path.basename(path.dirname(file)))
+  const sessionIds = new Set<string>()
+  const sessionValues: unknown[] = []
+  for (let i = 0; i < sessionFiles.length; i += batchSize) {
+    const end = Math.min(i + batchSize, sessionFiles.length)
+    const batch = await read(sessionFiles, i, end)
+    sessionValues.length = 0
+    for (let j = 0; j < batch.length; j++) {
+      const data = batch[j]
+      if (!data) continue
+      const id = path.basename(sessionFiles[i + j], ".json")
+      const projectID = sessionProjects[i + j]
+      if (!projectIds.has(projectID)) {
+        orphans.sessions++
+        continue
+      }
+      sessionIds.add(id)
+      sessionValues.push({
+        id,
+        project_id: projectID,
+        parent_id: data.parentID ?? null,
+        slug: data.slug ?? "",
+        directory: data.directory ?? "",
+        title: data.title ?? "",
+        version: data.version ?? "",
+        share_url: data.share?.url ?? null,
+        summary_additions: data.summary?.additions ?? null,
+        summary_deletions: data.summary?.deletions ?? null,
+        summary_files: data.summary?.files ?? null,
+        summary_diffs: data.summary?.diffs ?? null,
+        revert: data.revert ?? null,
+        permission: data.permission ?? null,
+        time_created: data.time?.created ?? now,
+        time_updated: data.time?.updated ?? now,
+        time_compacting: data.time?.compacting ?? null,
+        time_archived: data.time?.archived ?? null,
+      })
+    }
+    stats.sessions += insert(sessionValues, SessionTable, "session")
+    step("sessions", end - i)
+  }
+  log.info("migrated sessions", { count: stats.sessions })
+  if (orphans.sessions > 0) {
+    log.warn("skipped orphaned sessions", { count: orphans.sessions })
+  }
+
+  // Migrate messages using pre-scanned file map
+  const allMessageFiles = [] as string[]
+  const allMessageSessions = [] as string[]
+  const messageSessions = new Map<string, string>()
+  for (const file of messageFiles) {
+    const sessionID = path.basename(path.dirname(file))
+    if (!sessionIds.has(sessionID)) continue
+    allMessageFiles.push(file)
+    allMessageSessions.push(sessionID)
+  }
+
+  for (let i = 0; i < allMessageFiles.length; i += batchSize) {
+    const end = Math.min(i + batchSize, allMessageFiles.length)
+    const batch = await read(allMessageFiles, i, end)
+    // oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
+    const values = new Array(batch.length)
+    let count = 0
+    for (let j = 0; j < batch.length; j++) {
+      const data = batch[j]
+      if (!data) continue
+      const file = allMessageFiles[i + j]
+      const id = path.basename(file, ".json")
+      const sessionID = allMessageSessions[i + j]
+      messageSessions.set(id, sessionID)
+      const rest = data
+      delete rest.id
+      delete rest.sessionID
+      values[count++] = {
+        id,
+        session_id: sessionID,
+        time_created: data.time?.created ?? now,
+        time_updated: data.time?.updated ?? now,
+        data: rest,
+      }
+    }
+    values.length = count
+    stats.messages += insert(values, MessageTable, "message")
+    step("messages", end - i)
+  }
+  log.info("migrated messages", { count: stats.messages })
+
+  // Migrate parts using pre-scanned file map
+  for (let i = 0; i < partFiles.length; i += batchSize) {
+    const end = Math.min(i + batchSize, partFiles.length)
+    const batch = await read(partFiles, i, end)
+    // oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
+    const values = new Array(batch.length)
+    let count = 0
+    for (let j = 0; j < batch.length; j++) {
+      const data = batch[j]
+      if (!data) continue
+      const file = partFiles[i + j]
+      const id = path.basename(file, ".json")
+      const messageID = path.basename(path.dirname(file))
+      const sessionID = messageSessions.get(messageID)
+      if (!sessionID) {
+        errs.push(`part missing message session: ${file}`)
+        continue
+      }
+      if (!sessionIds.has(sessionID)) continue
+      const rest = data
+      delete rest.id
+      delete rest.messageID
+      delete rest.sessionID
+      values[count++] = {
+        id,
+        message_id: messageID,
+        session_id: sessionID,
+        time_created: data.time?.created ?? now,
+        time_updated: data.time?.updated ?? now,
+        data: rest,
+      }
+    }
+    values.length = count
+    stats.parts += insert(values, PartTable, "part")
+    step("parts", end - i)
+  }
+  log.info("migrated parts", { count: stats.parts })
+
+  // Migrate todos
+  const todoSessions = todoFiles.map((file) => path.basename(file, ".json"))
+  for (let i = 0; i < todoFiles.length; i += batchSize) {
+    const end = Math.min(i + batchSize, todoFiles.length)
+    const batch = await read(todoFiles, i, end)
+    const values: unknown[] = []
+    for (let j = 0; j < batch.length; j++) {
+      const data = batch[j]
+      if (!data) continue
+      const sessionID = todoSessions[i + j]
+      if (!sessionIds.has(sessionID)) {
+        orphans.todos++
+        continue
+      }
+      if (!Array.isArray(data)) {
+        errs.push(`todo not an array: ${todoFiles[i + j]}`)
+        continue
+      }
+      for (let position = 0; position < data.length; position++) {
+        const todo = data[position]
+        if (!todo?.content || !todo?.status) continue
+        values.push({
+          session_id: sessionID,
+          content: todo.content,
+          status: todo.status,
+          position,
+          time_created: now,
+          time_updated: now,
+        })
+      }
+    }
+    stats.todos += insert(values, TodoTable, "todo")
+    step("todos", end - i)
+  }
+  log.info("migrated todos", { count: stats.todos })
+  if (orphans.todos > 0) {
+    log.warn("skipped orphaned todos", { count: orphans.todos })
+  }
+
+  // Migrate permissions
+  const permProjects = permFiles.map((file) => path.basename(file, ".json"))
+  const permValues: unknown[] = []
+  for (let i = 0; i < permFiles.length; i += batchSize) {
+    const end = Math.min(i + batchSize, permFiles.length)
+    const batch = await read(permFiles, i, end)
+    permValues.length = 0
+    for (let j = 0; j < batch.length; j++) {
+      const data = batch[j]
+      if (!data) continue
+      const projectID = permProjects[i + j]
+      if (!projectIds.has(projectID)) {
+        orphans.permissions++
+        continue
+      }
+      permValues.push({ project_id: projectID, data })
+    }
+    stats.permissions += insert(permValues, PermissionTable, "permission")
+    step("permissions", end - i)
+  }
+  log.info("migrated permissions", { count: stats.permissions })
+  if (orphans.permissions > 0) {
+    log.warn("skipped orphaned permissions", { count: orphans.permissions })
+  }
+
+  // Migrate session shares
+  const shareSessions = shareFiles.map((file) => path.basename(file, ".json"))
+  const shareValues: unknown[] = []
+  for (let i = 0; i < shareFiles.length; i += batchSize) {
+    const end = Math.min(i + batchSize, shareFiles.length)
+    const batch = await read(shareFiles, i, end)
+    shareValues.length = 0
+    for (let j = 0; j < batch.length; j++) {
+      const data = batch[j]
+      if (!data) continue
+      const sessionID = shareSessions[i + j]
+      if (!sessionIds.has(sessionID)) {
+        orphans.shares++
+        continue
+      }
+      if (!data?.id || !data?.secret || !data?.url) {
+        errs.push(`session_share missing id/secret/url: ${shareFiles[i + j]}`)
+        continue
+      }
+      shareValues.push({ session_id: sessionID, id: data.id, secret: data.secret, url: data.url })
+    }
+    stats.shares += insert(shareValues, SessionShareTable, "session_share")
+    step("shares", end - i)
+  }
+  log.info("migrated session shares", { count: stats.shares })
+  if (orphans.shares > 0) {
+    log.warn("skipped orphaned session shares", { count: orphans.shares })
+  }
+
+  db.run("COMMIT")
+
+  log.info("json migration complete", {
+    projects: stats.projects,
+    sessions: stats.sessions,
+    messages: stats.messages,
+    parts: stats.parts,
+    todos: stats.todos,
+    permissions: stats.permissions,
+    shares: stats.shares,
+    errorCount: stats.errors.length,
+    duration: Math.round(performance.now() - start),
+  })
+
+  if (stats.errors.length > 0) {
+    log.warn("migration errors", { errors: stats.errors.slice(0, 20) })
+  }
+
+  progress?.({ current: total, total, label: "complete" })
+
+  return stats
+}

package/src/storage/schema.sql.ts

@@ -0,0 +1,10 @@
+import { integer } from "drizzle-orm/sqlite-core"
+
+export const Timestamps = {
+  time_created: integer()
+    .notNull()
+    .$default(() => Date.now()),
+  time_updated: integer()
+    .notNull()
+    .$onUpdate(() => Date.now()),
+}

package/src/storage/schema.ts

@@ -0,0 +1,7 @@
+export { AccountTable, AccountStateTable, ControlAccountTable } from "../account/account.sql"
+export { ProjectTable } from "../project/project.sql"
+export { SessionTable, MessageTable, PartTable, TodoTable, PermissionTable } from "../session/session.sql"
+export { SessionShareTable } from "../share/share.sql"
+export { WorkspaceTable } from "../control-plane/workspace.sql"
+export { WorkflowRunTable } from "../workflow/workflow.sql"
+export { HistoryFtsTable } from "../history/fts.sql"