saeeol 1.0.9 → 1.1.1

package/src/provider/loader-local.ts

@@ -0,0 +1,185 @@
+/** Local provider loaders — Ollama, LM Studio, vLLM, llama.cpp, text-generation-webui */
+
+import os from "os"
+import { Effect } from "effect"
+import { InstallationVersion } from "@saeeol/core/installation/version"
+import type { CustomDep, CustomLoader } from "./provider-types"
+import type { Info } from "./provider-schema"
+import { useLanguageModel } from "./bundled-providers"
+import { iife } from "@/util/iife"
+
+function ua(name: string): string {
+  return `saeeol/${InstallationVersion} ${name} (${os.platform()} ${os.release()}; ${os.arch()})`
+}
+
+export function localLoaders(dep: CustomDep): Record<string, CustomLoader> {
+  return {
+    ollama: Effect.fnUntraced(function* (_provider: Info) {
+      const env = yield* dep.env()
+      const endpoint = iife(() => {
+        return [
+          _provider.options?.baseURL,
+          env["OLLAMA_BASE_URL"],
+          env["OLLAMA_HOST"],
+        ].find((url) => typeof url === "string" && url.trim() !== "")
+      }) ?? "http://localhost:11434"
+
+      return {
+        autoload: true,
+        options: {
+          baseURL: endpoint,
+          headers: { "User-Agent": ua("ollama") },
+        },
+        vars(): Record<string, string> {
+          return { OLLAMA_BASE_URL: endpoint }
+        },
+        async getModel(sdk: any, modelID: string, _options?: Record<string, any>) {
+          if (useLanguageModel(sdk)) return sdk.languageModel(modelID)
+          return sdk.chat(modelID)
+        },
+        async discoverModels() {
+          try {
+            const res = await fetch(`${endpoint}/api/tags`, { signal: AbortSignal.timeout(5000) })
+            if (!res.ok) return {}
+            const data = (await res.json()) as { models?: Array<{ name: string }> }
+            const models: Record<string, any> = {}
+            for (const m of data.models ?? []) {
+              models[m.name] = {
+                id: m.name, name: m.name, providerID: "ollama",
+                api: { id: m.name, url: `${endpoint}/v1`, npm: "@ai-sdk/openai-compatible" },
+                capabilities: { temperature: true, reasoning: false, attachment: false, toolcall: true, input: { text: true, image: false, audio: false, video: false, pdf: false }, output: { text: true, image: false, audio: false, video: false, pdf: false }, interleaved: false },
+                cost: { input: 0, output: 0, cache: { read: 0, write: 0 } },
+                limit: { context: 32768, output: 4096 },
+                status: "active" as const, options: {}, headers: {}, release_date: "",
+              }
+            }
+            return models
+          } catch {
+            return {}
+          }
+        },
+      }
+    }),
+
+    lmstudio: Effect.fnUntraced(function* (_provider: Info) {
+      const env = yield* dep.env()
+      const endpoint = iife(() => {
+        return [
+          _provider.options?.baseURL,
+          env["LMSTUDIO_BASE_URL"],
+        ].find((url) => typeof url === "string" && url.trim() !== "")
+      }) ?? "http://localhost:1234/v1"
+
+      return {
+        autoload: true,
+        options: {
+          baseURL: endpoint,
+          headers: { "User-Agent": ua("lmstudio") },
+        },
+        vars(): Record<string, string> {
+          return { LMSTUDIO_BASE_URL: endpoint }
+        },
+        async getModel(sdk: any, modelID: string, _options?: Record<string, any>) {
+          if (useLanguageModel(sdk)) return sdk.languageModel(modelID)
+          return sdk.chat(modelID)
+        },
+        async discoverModels() {
+          try {
+            const res = await fetch(`${endpoint}/models`, { signal: AbortSignal.timeout(5000) })
+            if (!res.ok) return {}
+            const data = (await res.json()) as { data?: Array<{ id: string }> }
+            const models: Record<string, any> = {}
+            for (const m of data.data ?? []) {
+              models[m.id] = {
+                id: m.id, name: m.id, providerID: "lmstudio",
+                api: { id: m.id, url: endpoint, npm: "@ai-sdk/openai-compatible" },
+                capabilities: { temperature: true, reasoning: false, attachment: false, toolcall: true, input: { text: true, image: false, audio: false, video: false, pdf: false }, output: { text: true, image: false, audio: false, video: false, pdf: false }, interleaved: false },
+                cost: { input: 0, output: 0, cache: { read: 0, write: 0 } },
+                limit: { context: 32768, output: 4096 },
+                status: "active" as const, options: {}, headers: {}, release_date: "",
+              }
+            }
+            return models
+          } catch {
+            return {}
+          }
+        },
+      }
+    }),
+
+    vllm: Effect.fnUntraced(function* (_provider: Info) {
+      const env = yield* dep.env()
+      const endpoint = iife(() => {
+        return [
+          _provider.options?.baseURL,
+          env["VLLM_BASE_URL"],
+        ].find((url) => typeof url === "string" && url.trim() !== "")
+      }) ?? "http://localhost:8000/v1"
+
+      return {
+        autoload: true,
+        options: {
+          baseURL: endpoint,
+          headers: { "User-Agent": ua("vllm") },
+        },
+        vars(): Record<string, string> {
+          return { VLLM_BASE_URL: endpoint }
+        },
+        async getModel(sdk: any, modelID: string, _options?: Record<string, any>) {
+          if (useLanguageModel(sdk)) return sdk.languageModel(modelID)
+          return sdk.chat(modelID)
+        },
+      }
+    }),
+
+    "text-generation-webui": Effect.fnUntraced(function* (_provider: Info) {
+      const env = yield* dep.env()
+      const endpoint = iife(() => {
+        return [
+          _provider.options?.baseURL,
+          env["TEXT_GEN_WEBUI_BASE_URL"],
+        ].find((url) => typeof url === "string" && url.trim() !== "")
+      }) ?? "http://localhost:5000"
+
+      return {
+        autoload: true,
+        options: {
+          baseURL: endpoint,
+          headers: { "User-Agent": ua("text-generation-webui") },
+        },
+        vars(): Record<string, string> {
+          return { TEXT_GEN_WEBUI_BASE_URL: endpoint }
+        },
+        async getModel(sdk: any, modelID: string, _options?: Record<string, any>) {
+          if (useLanguageModel(sdk)) return sdk.languageModel(modelID)
+          return sdk.chat(modelID)
+        },
+      }
+    }),
+
+    "llama.cpp": Effect.fnUntraced(function* (_provider: Info) {
+      const env = yield* dep.env()
+      const endpoint = iife(() => {
+        return [
+          _provider.options?.baseURL,
+          env["LLAMA_CPP_BASE_URL"],
+        ].find((url) => typeof url === "string" && url.trim() !== "")
+      }) ?? "http://localhost:8080"
+
+      return {
+        autoload: true,
+        options: {
+          baseURL: endpoint,
+          headers: { "User-Agent": ua("llama.cpp") },
+        },
+        vars(): Record<string, string> {
+          return { LLAMA_CPP_BASE_URL: endpoint }
+        },
+        async getModel(sdk: any, modelID: string, _options?: Record<string, any>) {
+          if (useLanguageModel(sdk)) return sdk.languageModel(modelID)
+          return sdk.chat(modelID)
+        },
+      }
+    }),
+  }
+}

package/src/provider/local/embedder.ts

@@ -0,0 +1,220 @@
+/** 로컬 임베딩 서버 — Ollama 기반 임베딩 모델 실행 관리 */
+
+import { Effect } from "effect"
+import * as Log from "@saeeol/core/util/log"
+import { Process } from "@/util/process"
+import type { EmbeddingServer, EmbedderStatus, HardwareProfile, LLMBakeParams } from "@/ltm/types"
+import type { BackendType } from "@/provider/local/types"
+import * as GPU from "@/provider/local/gpu"
+import * as RAG from "@/provider/local/rag"
+import * as Hub from "@/provider/local/hub"
+import * as Manager from "@/provider/local/model-manager"
+import { iife } from "@/util/iife"
+import * as Bus from "@/bus"
+import { LTMEvent } from "@/ltm/events"
+
+const log = Log.create({ service: "local/embedder" })
+
+let server: EmbeddingServer | undefined
+
+// ── Ollama 상태 확인 ──
+
+async function isOllamaRunning(endpoint: string): Promise<boolean> {
+  try {
+    const res = await fetch(`${endpoint}/api/tags`, { signal: AbortSignal.timeout(3000) })
+    return res.ok
+  } catch {
+    return false
+  }
+}
+
+async function isModelLoaded(endpoint: string, model: string): Promise<boolean> {
+  try {
+    const res = await fetch(`${endpoint}/api/tags`, { signal: AbortSignal.timeout(3000) })
+    if (!res.ok) return false
+    const data = (await res.json()) as { models?: Array<{ name: string }> }
+    return (data.models ?? []).some((m) => m.name === model || m.name === `${model}:latest`)
+  } catch {
+    return false
+  }
+}
+
+/** Ollama에 임베딩 모델이 설치되어 있는지 확인 */
+async function isModelInstalled(endpoint: string, model: string): Promise<boolean> {
+  try {
+    const res = await fetch(`${endpoint}/api/tags`, { signal: AbortSignal.timeout(5000) })
+    if (!res.ok) return false
+    const data = (await res.json()) as { models?: Array<{ name: string }> }
+    return (data.models ?? []).some((m) =>
+      m.name === model || m.name === `${model}:latest` || m.name.replace(":latest", "") === model,
+    )
+  } catch {
+    return false
+  }
+}
+
+// ── Ollama에 모델 pull ──
+
+async function pullModel(endpoint: string, model: string): Promise<void> {
+  log.info("pulling embedding model", { model })
+  const ollamaModel = getOllamaModelName(model)
+
+  try {
+    const res = await fetch(`${endpoint}/api/pull`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ name: ollamaModel, stream: false }),
+      signal: AbortSignal.timeout(300_000), // 5분 타임아웃
+    })
+    if (!res.ok) {
+      const err = await res.text().catch(() => "unknown error")
+      throw new Error(`Ollama pull failed: ${res.status} ${err}`)
+    }
+    log.info("model pulled", { model: ollamaModel })
+  } catch (e) {
+    log.error("failed to pull model", { model: ollamaModel, error: e })
+    throw e
+  }
+}
+
+/** 내부 모델 ID → Ollama 모델명 매핑 */
+function getOllamaModelName(modelId: string): string {
+  const map: Record<string, string> = {
+    "bge-small-en": "nomic-embed-text", // Ollama에 가장 가벼운 임베딩
+    "bge-base-en": "nomic-embed-text",
+    "bge-large-en": "nomic-embed-text",
+    "bge-m3": "bge-m3",
+    "nomic-embed": "nomic-embed-text",
+    "all-minilm-l6": "all-minilm",
+    "gte-small": "nomic-embed-text",
+  }
+  return map[modelId] ?? modelId
+}
+
+// ── 임베딩 API 호출 ──
+
+async function embedViaOllama(endpoint: string, model: string, texts: string[]): Promise<number[][]> {
+  const ollamaModel = getOllamaModelName(model)
+  const res = await fetch(`${endpoint}/api/embed`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ model: ollamaModel, input: texts }),
+    signal: AbortSignal.timeout(30_000),
+  })
+
+  if (!res.ok) {
+    const err = await res.text().catch(() => "unknown error")
+    throw new Error(`Embedding failed: ${res.status} ${err}`)
+  }
+
+  const data = (await res.json()) as { embeddings?: number[][] }
+  if (!data.embeddings || !Array.isArray(data.embeddings)) {
+    throw new Error("Invalid embedding response")
+  }
+  return data.embeddings
+}
+
+// ── VRAM 추정 ──
+
+function estimateEmbeddingVRAM(modelId: string): number {
+  const model = RAG.EMBEDDING_MODELS.find((m) => m.id === modelId)
+  if (!model) return 200
+  return Math.ceil(model.sizeBytes * 1.2 / (1024 * 1024))
+}
+
+// ── 공개 API ──
+
+/** 임베딩 서버 시작 */
+export async function start(bake: LLMBakeParams): Promise<EmbeddingServer> {
+  if (server && server.status === "running") {
+    log.info("embedder already running", { model: server.model })
+    return server
+  }
+
+  const endpoint = iife(() => {
+    const env = process.env.OLLAMA_BASE_URL ?? process.env.OLLAMA_HOST
+    return env ?? "http://localhost:11434"
+  })
+
+  server = {
+    id: "local-embedding",
+    model: bake.embeddingModel,
+    status: "starting",
+    endpoint,
+    dimensions: bake.embeddingDimensions,
+    vramMB: estimateEmbeddingVRAM(bake.embeddingModel),
+  }
+
+  void Bus.publish(LTMEvent.EmbedderStatusChanged, { status: "starting", model: bake.embeddingModel })
+
+  // Ollama 실행 확인
+  const running = await isOllamaRunning(endpoint)
+  if (!running) {
+    log.warn("Ollama not running, embedding server unavailable", { endpoint })
+    server.status = "error"
+    void Bus.publish(LTMEvent.EmbedderStatusChanged, { status: "error", model: bake.embeddingModel })
+    return server
+  }
+
+  // 모델 설치 확인
+  const ollamaModel = getOllamaModelName(bake.embeddingModel)
+  const installed = await isModelInstalled(endpoint, ollamaModel)
+  if (!installed) {
+    log.info("model not installed, pulling", { model: ollamaModel })
+    await pullModel(endpoint, ollamaModel)
+  }
+
+  // 모델 로드 확인 (첫 호출 시 자동 로드됨)
+  const loaded = await isModelLoaded(endpoint, ollamaModel)
+  if (!loaded) {
+    // 웜업 호출 — 모델을 메모리에 로드
+    try {
+      await embedViaOllama(endpoint, bake.embeddingModel, ["warmup"])
+      log.info("model warmed up", { model: ollamaModel })
+    } catch (e) {
+      log.error("warmup failed", { model: ollamaModel, error: e })
+      server.status = "error"
+      void Bus.publish(LTMEvent.EmbedderStatusChanged, { status: "error", model: bake.embeddingModel })
+      return server
+    }
+  }
+
+  server.status = "running"
+  void Bus.publish(LTMEvent.EmbedderStatusChanged, { status: "running", model: bake.embeddingModel })
+  log.info("embedding server started", { model: bake.embeddingModel, dimensions: bake.embeddingDimensions })
+
+  return server
+}
+
+/** 임베딩 서버 중지 */
+export async function stop(): Promise<void> {
+  if (!server) return
+  server.status = "stopped"
+  void Bus.publish(LTMEvent.EmbedderStatusChanged, { status: "stopped" })
+  log.info("embedding server stopped")
+  server = undefined
+}
+
+/** 현재 상태 */
+export function status(): EmbeddingServer | undefined {
+  return server
+}
+
+/** 텍스트 배열 → 임베딩 벡터 */
+export async function embed(texts: string[]): Promise<number[][]> {
+  if (!server || server.status !== "running") {
+    throw new Error("Embedding server not running")
+  }
+  return embedViaOllama(server.endpoint, server.model, texts)
+}
+
+/** 단일 텍스트 → 임베딩 벡터 */
+export async function embedOne(text: string): Promise<number[]> {
+  const vectors = await embed([text])
+  return vectors[0]!
+}
+
+/** VRAM 사용량 (MB) */
+export function vramUsage(): number {
+  return server?.vramMB ?? 0
+}

package/src/provider/local/events.ts

@@ -0,0 +1,74 @@
+/** Local model events — TUI/webview subscription for progress/status updates */
+
+import { BusEvent } from "@/bus/bus-event"
+import { Schema } from "effect"
+
+export const LocalModelEvent = {
+  DownloadStarted: BusEvent.define(
+    "local.download.started",
+    Schema.Struct({
+      id: Schema.String,
+      repo: Schema.String,
+      filename: Schema.String,
+      totalBytes: Schema.Number,
+    }),
+  ),
+  DownloadProgress: BusEvent.define(
+    "local.download.progress",
+    Schema.Struct({
+      id: Schema.String,
+      downloaded: Schema.Number,
+      total: Schema.Number,
+      speedMBps: Schema.Finite,
+    }),
+  ),
+  DownloadCompleted: BusEvent.define(
+    "local.download.completed",
+    Schema.Struct({
+      id: Schema.String,
+      repo: Schema.String,
+      filename: Schema.String,
+      path: Schema.String,
+    }),
+  ),
+  DownloadFailed: BusEvent.define(
+    "local.download.failed",
+    Schema.Struct({
+      id: Schema.String,
+      repo: Schema.String,
+      error: Schema.String,
+    }),
+  ),
+  ModelStarted: BusEvent.define(
+    "local.model.started",
+    Schema.Struct({
+      id: Schema.String,
+      repo: Schema.String,
+      endpoint: Schema.String,
+      gpuIndex: Schema.optional(Schema.Number),
+    }),
+  ),
+  ModelStopped: BusEvent.define(
+    "local.model.stopped",
+    Schema.Struct({
+      id: Schema.String,
+      repo: Schema.String,
+    }),
+  ),
+  ModelError: BusEvent.define(
+    "local.model.error",
+    Schema.Struct({
+      id: Schema.String,
+      repo: Schema.String,
+      error: Schema.String,
+    }),
+  ),
+  GPUProfiled: BusEvent.define(
+    "local.gpu.profiled",
+    Schema.Struct({
+      gpuCount: Schema.Number,
+      totalVRAMMB: Schema.Number,
+      availableVRAMMB: Schema.Number,
+    }),
+  ),
+}

package/src/provider/local/gpu.ts

@@ -0,0 +1,93 @@
+/** GPU profiler — detect hardware capabilities for model scheduling */
+
+import { Effect } from "effect"
+import * as Log from "@saeeol/core/util/log"
+import { Process } from "@/util/process"
+import type { GPUInfo, GPUProfile } from "./types"
+
+const log = Log.create({ service: "local/gpu" })
+
+/** Parse nvidia-smi output into structured GPU info */
+function parseNvidiaSmi(stdout: string): GPUInfo[] {
+  const gpus: GPUInfo[] = []
+  // nvidia-smi --query-gpu=index,name,memory.total,memory.used,memory.free,compute_cap,driver_version,cuda_version --format=csv,noheader,nounits
+  for (const line of stdout.trim().split("\n")) {
+    const parts = line.split(",").map((s) => s.trim())
+    if (parts.length < 5) continue
+    gpus.push({
+      index: Number(parts[0]),
+      name: parts[1],
+      vramTotalMB: Number(parts[2]),
+      vramUsedMB: Number(parts[3]),
+      vramFreeMB: Number(parts[4]),
+      computeCapability: parts[5] || undefined,
+      driverVersion: parts[6] || undefined,
+      cudaVersion: parts[7] || undefined,
+    })
+  }
+  return gpus
+}
+
+/** Profile all available GPUs via nvidia-smi */
+export const profile: Effect.Effect<GPUProfile> = Effect.gen(function* () {
+  const result = yield* Effect.tryPromise(() =>
+    Process.run([
+      "nvidia-smi",
+      "--query-gpu=index,name,memory.total,memory.used,memory.free,compute_cap,driver_version,cuda_version",
+      "--format=csv,noheader,nounits",
+    ]),
+  ).pipe(Effect.catch(() => Effect.succeed({ code: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) } as Process.Result)))
+
+  if (result.code !== 0) {
+    log.info("no nvidia-smi found, GPU profiling unavailable")
+    return {
+      gpus: [],
+      totalVRAMMB: 0,
+      availableVRAMMB: 0,
+      cudaAvailable: false,
+    }
+  }
+
+  const gpus = parseNvidiaSmi(result.stdout.toString())
+  return {
+    gpus,
+    totalVRAMMB: gpus.reduce((sum, g) => sum + g.vramTotalMB, 0),
+    availableVRAMMB: gpus.reduce((sum, g) => sum + g.vramFreeMB, 0),
+    cudaAvailable: gpus.length > 0,
+  }
+})
+
+/** Estimate VRAM needed for a model (bytes) at a given quantization */
+export function estimateVRAM(modelBytes: number, quantization: string): number {
+  // Runtime overhead: ~20% for KV cache, context, etc.
+  // GGUF files are already quantized — model file size ≈ VRAM needed + 20%
+  const overhead = modelBytes * 0.2
+  return Math.ceil((modelBytes + overhead) / (1024 * 1024)) // return MB
+}
+
+/** Find the best GPU for a model that needs `vramMB` megabytes */
+export function findBestGPU(profile: GPUProfile, vramMB: number): GPUInfo | undefined {
+  // Prefer GPU with most free VRAM that can fit the model
+  const candidates = profile.gpus.filter((g) => g.vramFreeMB >= vramMB)
+  if (candidates.length === 0) return undefined
+  return candidates.sort((a, b) => b.vramFreeMB - a.vramFreeMB)[0]
+}
+
+/** Check if a model fits in available GPU memory */
+export function canFit(profile: GPUProfile, vramMB: number): boolean {
+  return profile.availableVRAMMB >= vramMB
+}
+
+/** Suggest quantization level based on available VRAM */
+export function suggestQuantization(
+  availableMB: number,
+  modelSizeBytes: Record<string, number>,
+): string | undefined {
+  // Try from highest quality to lowest
+  const order = ["fp16", "q8_0", "q6_k", "q5_k_m", "q5_k_s", "q4_k_m", "q4_k_s", "q3_k_m", "q3_k_s", "q2_k"]
+  for (const q of order) {
+    const size = modelSizeBytes[q]
+    if (size !== undefined && estimateVRAM(size, q) <= availableMB) return q
+  }
+  return undefined
+}