saeeol 1.0.9 → 1.1.1

package/src/provider/local/hub.ts

@@ -0,0 +1,174 @@
+/** HuggingFace Hub client — search, browse, download models */
+
+import { Effect } from "effect"
+import * as Log from "@saeeol/core/util/log"
+import { iife } from "@/util/iife"
+import type { HFModelSearch, HFModelInfo, HFSibling, ModelArtifact, Quantization, ModelFormat, RAGAsset, RAGAssetType } from "./types"
+
+const log = Log.create({ service: "local/hub" })
+const API = "https://huggingface.co/api"
+
+/** Search HuggingFace Hub for models */
+export async function search(query: string, opts?: { limit?: number; tags?: string[] }): Promise<HFModelSearch[]> {
+  const params = new URLSearchParams({
+    search: query,
+    limit: String(opts?.limit ?? 20),
+    ...(opts?.tags?.length ? { tags: opts.tags.join(",") } : {}),
+  })
+
+  const res = await fetch(`${API}/models?${params}`)
+  if (!res.ok) throw new Error(`HF search failed: ${res.status} ${res.statusText}`)
+
+  const items = (await res.json()) as any[]
+  return items.map((item) => ({
+    id: item.id ?? item.modelId ?? "",
+    name: item.id ?? "",
+    author: item.author ?? "",
+    downloads: item.downloads ?? 0,
+    likes: item.likes ?? 0,
+    tags: item.tags ?? [],
+    pipelineTag: item.pipeline_tag ?? undefined,
+    libraryName: item.library_name ?? undefined,
+  }))
+}
+
+/** Get detailed model info from HF Hub */
+export async function info(repo: string): Promise<HFModelInfo> {
+  const res = await fetch(`${API}/models/${repo}`)
+  if (!res.ok) throw new Error(`HF model info failed: ${res.status} ${res.statusText}`)
+
+  const data = (await res.json()) as any
+  return {
+    id: data.id ?? data.modelId ?? repo,
+    modelId: data.modelId ?? repo,
+    sha: data.sha ?? "",
+    siblings: (data.siblings ?? []) as HFSibling[],
+    tags: data.tags ?? [],
+    downloads: data.downloads ?? 0,
+    likes: data.likes ?? 0,
+    private: data.private ?? false,
+  }
+}
+
+/** List GGUF files in a repo */
+export function listGGUF(siblings: HFSibling[]): string[] {
+  return siblings
+    .map((s) => s.rfilename)
+    .filter((f) => f.endsWith(".gguf"))
+}
+
+/** Parse quantization from GGUF filename */
+export function parseQuant(filename: string): Quantization | undefined {
+  const lower = filename.toLowerCase()
+  const quants: Quantization[] = [
+    "q2_k", "q3_k_s", "q3_k_m", "q3_k_l",
+    "q4_0", "q4_1", "q4_k_s", "q4_k_m",
+    "q5_0", "q5_1", "q5_k_s", "q5_k_m",
+    "q6_k", "q8_0", "fp16", "bf16", "fp32",
+  ]
+  for (const q of quants) {
+    if (lower.includes(q)) return q
+  }
+  return undefined
+}
+
+/** Resolve download URL for a file in a repo */
+export function downloadURL(repo: string, filename: string): string {
+  return `https://huggingface.co/${repo}/resolve/main/${filename}`
+}
+
+/** Build a ModelArtifact from repo + sibling info */
+export function buildArtifact(
+  repo: string,
+  filename: string,
+  sizeBytes: number,
+  sha256?: string,
+): ModelArtifact {
+  const q = parseQuant(filename)
+  const ext = filename.split(".").pop() ?? ""
+  const formatMap: Record<string, ModelFormat> = {
+    gguf: "gguf",
+    safetensors: "safetensors",
+    bin: "pytorch",
+    pt: "pytorch",
+    onnx: "onnx",
+  }
+
+  return {
+    id: `${repo}/${filename}`,
+    repo,
+    filename,
+    format: formatMap[ext] ?? "pytorch",
+    quantization: q ?? "fp16",
+    sizeBytes,
+    sha256,
+  }
+}
+
+/** Download a file from HF Hub with progress callback */
+export async function download(
+  repo: string,
+  filename: string,
+  destPath: string,
+  opts?: { onProgress?: (downloaded: number, total: number) => void; signal?: AbortSignal },
+): Promise<void> {
+  const url = downloadURL(repo, filename)
+
+  const res = await fetch(url, { signal: opts?.signal })
+  if (!res.ok) throw new Error(`HF download failed: ${res.status} ${res.statusText}`)
+  if (!res.body) throw new Error("No response body")
+
+  const total = Number(res.headers.get("content-length") ?? "0")
+  const file = Bun.file(destPath)
+  const writer = file.writer()
+
+  let downloaded = 0
+  const reader = res.body.getReader()
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      writer.write(value)
+      downloaded += value.length
+      opts?.onProgress?.(downloaded, total)
+    }
+  } finally {
+    writer.end()
+  }
+
+  log.info("downloaded", { repo, filename, sizeMB: Math.round(downloaded / 1024 / 1024) })
+}
+
+/** Search for embedding/reranker models */
+export async function searchRAG(
+  type: RAGAssetType,
+  query?: string,
+  limit?: number,
+): Promise<RAGAsset[]> {
+  const tagMap: Record<RAGAssetType, string> = {
+    embedding: "sentence-transformers",
+    reranker: "reranker",
+    vectordb: "vector",
+  }
+
+  const params = new URLSearchParams({
+    limit: String(limit ?? 20),
+    tags: tagMap[type],
+    ...(query ? { search: query } : {}),
+  })
+
+  const res = await fetch(`${API}/models?${params}`)
+  if (!res.ok) return []
+
+  const items = (await res.json()) as any[]
+  return items.map((item) => ({
+    id: item.id ?? "",
+    name: item.id ?? "",
+    type,
+    repo: item.id ?? "",
+    format: "safetensors" as ModelFormat,
+    sizeBytes: 0,
+    dimensions: undefined,
+  }))
+}

package/src/provider/local/index.ts

@@ -0,0 +1,10 @@
+/** Local model orchestration — public API */
+
+export * from "./types"
+export * as GPU from "./gpu"
+export * as Hub from "./hub"
+export * as Manager from "./model-manager"
+export * as Orchestrator from "./orchestrator"
+export * as RAG from "./rag"
+export * as Embedder from "./embedder"
+export { LocalModelEvent } from "./events"

package/src/provider/local/model-manager.ts

@@ -0,0 +1,113 @@
+/** Local model manager — install, remove, list, configure models on disk */
+
+import path from "path"
+import { mkdir, rm, readdir, stat } from "fs/promises"
+import { Effect } from "effect"
+import * as Log from "@saeeol/core/util/log"
+import { Global } from "@saeeol/core/global"
+import { iife } from "@/util/iife"
+import type { ModelArtifact, BackendType } from "./types"
+import * as Hub from "./hub"
+
+const log = Log.create({ service: "local/manager" })
+
+/** Root directory for all local models */
+export function modelsDir(): string {
+  return path.join(Global.Path.data, "local-models")
+}
+
+/** Directory for a specific model artifact */
+export function modelDir(artifact: ModelArtifact): string {
+  return path.join(modelsDir(), artifact.repo.replace("/", "__"), artifact.filename)
+}
+
+/** Directory for a specific backend's data */
+export function backendDir(backend: BackendType): string {
+  return path.join(modelsDir(), "backends", backend)
+}
+
+/** Directory for RAG assets */
+export function ragDir(): string {
+  return path.join(modelsDir(), "rag")
+}
+
+/** Ensure a directory exists */
+async function ensure(dir: string): Promise<void> {
+  await mkdir(dir, { recursive: true })
+}
+
+/** Install a model from HuggingFace Hub */
+export async function install(
+  artifact: ModelArtifact,
+  opts?: { onProgress?: (downloaded: number, total: number) => void; signal?: AbortSignal },
+): Promise<string> {
+  const dest = modelDir(artifact)
+  await ensure(path.dirname(dest))
+
+  const fullPath = path.join(dest, artifact.filename)
+  const exists = await stat(fullPath).catch(() => undefined)
+  if (exists?.size === artifact.sizeBytes) {
+    log.info("model already installed", { repo: artifact.repo, filename: artifact.filename })
+    return fullPath
+  }
+
+  log.info("installing model", { repo: artifact.repo, filename: artifact.filename })
+  await Hub.download(artifact.repo, artifact.filename, fullPath, opts)
+  return fullPath
+}
+
+/** Remove a model from disk */
+export async function uninstall(artifact: ModelArtifact): Promise<void> {
+  const dir = modelDir(artifact)
+  await rm(dir, { recursive: true, force: true })
+  log.info("uninstalled model", { repo: artifact.repo, filename: artifact.filename })
+}
+
+/** List all installed models on disk */
+export async function list(): Promise<Array<{ repo: string; filename: string; path: string; sizeBytes: number }>> {
+  const root = modelsDir()
+  const result: Array<{ repo: string; filename: string; path: string; sizeBytes: number }> = []
+
+  const entries = await readdir(root, { withFileTypes: true }).catch(() => [] as import("fs").Dirent[])
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue
+    if (entry.name === "backends" || entry.name === "rag") continue
+
+    const repo = entry.name.replace("__", "/")
+    const repoDir = path.join(root, entry.name)
+    const files = await readdir(repoDir).catch(() => [] as string[])
+
+    for (const file of files) {
+      const filePath = path.join(repoDir, file)
+      const s = await stat(filePath).catch(() => null)
+      if (s?.isFile()) {
+        result.push({ repo, filename: file, path: filePath, sizeBytes: s.size })
+      }
+    }
+  }
+
+  return result
+}
+
+/** Install a RAG asset (embedding model, reranker, vectordb) */
+export async function installRAG(
+  asset: { repo: string; filename: string; sizeBytes: number },
+  opts?: { onProgress?: (downloaded: number, total: number) => void; signal?: AbortSignal },
+): Promise<string> {
+  const dir = path.join(ragDir(), asset.repo.replace("/", "__"))
+  await ensure(dir)
+
+  const fullPath = path.join(dir, asset.filename)
+  const exists = await stat(fullPath).catch(() => undefined)
+  if (exists?.size === asset.sizeBytes) return fullPath
+
+  await Hub.download(asset.repo, asset.filename, fullPath, opts)
+  return fullPath
+}
+
+/** Check if a model is installed */
+export async function isInstalled(artifact: ModelArtifact): Promise<boolean> {
+  const fullPath = path.join(modelDir(artifact), artifact.filename)
+  const s = await stat(fullPath).catch(() => undefined)
+  return s?.isFile() === true && s.size > 0
+}

package/src/provider/local/orchestrator.ts

@@ -0,0 +1,301 @@
+/** Model orchestrator — run multiple models simultaneously with GPU-aware scheduling */
+
+import { Effect } from "effect"
+import * as Log from "@saeeol/core/util/log"
+import { Process } from "@/util/process"
+import { Global } from "@saeeol/core/global"
+import type { ModelArtifact, ModelInstance, BackendType, BackendStatus, GPUProfile } from "./types"
+import * as GPU from "./gpu"
+import * as Manager from "./model-manager"
+import { which } from "@/util/which"
+import { iife } from "@/util/iife"
+import path from "path"
+import { mkdir } from "fs/promises"
+
+const log = Log.create({ service: "local/orchestrator" })
+
+const running = new Map<string, ModelInstance>()
+let nextPort = 11435 // Start after default Ollama port
+
+/** Detect which local backends are installed */
+export async function detectBackends(): Promise<BackendStatus[]> {
+  const backends: Array<{ type: BackendType; cmds: string[]; defaultPort: number }> = [
+    { type: "ollama", cmds: ["ollama"], defaultPort: 11434 },
+    { type: "lmstudio", cmds: ["lms"], defaultPort: 1234 },
+    { type: "vllm", cmds: ["vllm", "python -m vllm"], defaultPort: 8000 },
+    { type: "llama.cpp", cmds: ["llama-server", "llama-cpp-server"], defaultPort: 8080 },
+  ]
+
+  const results: BackendStatus[] = []
+  for (const b of backends) {
+    let cmd = ""
+    let available = false
+    for (const c of b.cmds) {
+      const found = await which(c.split(" ")[0])
+      if (found) { cmd = c; available = true; break }
+    }
+
+    results.push({
+      type: b.type,
+      available,
+      endpoint: `http://localhost:${b.defaultPort}`,
+      version: undefined,
+      loadedModels: [],
+    })
+  }
+
+  return results
+}
+
+/** Check if a backend server is responding */
+export async function isAlive(endpoint: string): Promise<boolean> {
+  try {
+    const res = await fetch(endpoint, { signal: AbortSignal.timeout(3000) })
+    return res.ok || res.status === 404 // 404 is fine, means server is up
+  } catch {
+    return false
+  }
+}
+
+/** Allocate a free port for a new model instance */
+function allocPort(): number {
+  return nextPort++
+}
+
+/** Load a model via Ollama */
+async function loadViaOllama(artifact: ModelArtifact, modelPath: string): Promise<ModelInstance> {
+  const port = 11434 // Ollama default
+
+  // Register model with Ollama if not already present
+  const modelfile = `FROM ${modelPath}`
+  const name = artifact.repo.replace("/", "-").toLowerCase()
+
+  const result = await Process.run(["ollama", "create", name, "-f", "-"])
+
+  log.info("ollama create result", { code: result.code })
+
+  return {
+    id: name,
+    artifact,
+    status: "running",
+    port,
+    endpoint: `http://localhost:${port}`,
+  }
+}
+
+/** Load a model via llama.cpp server */
+async function loadViaLlamaCpp(artifact: ModelArtifact, modelPath: string, gpuIndex?: number): Promise<ModelInstance> {
+  const port = allocPort()
+
+  const args = ["-m", modelPath, "--port", String(port), "-c", "4096", "-ngl", "99"]
+  if (gpuIndex !== undefined) args.push("--gpu", String(gpuIndex))
+
+  const proc = Bun.spawn(["llama-server", ...args], {
+    stdout: "pipe",
+    stderr: "pipe",
+    windowsHide: true,
+    detached: true,
+  })
+
+  // Wait for server to start
+  for (let i = 0; i < 30; i++) {
+    await new Promise((r) => setTimeout(r, 1000))
+    if (await isAlive(`http://localhost:${port}`)) break
+  }
+
+  const instance: ModelInstance = {
+    id: `llamacpp-${port}`,
+    artifact,
+    status: "running",
+    pid: proc.pid,
+    port,
+    gpuIndex,
+    endpoint: `http://localhost:${port}/v1`,
+  }
+
+  running.set(instance.id, instance)
+  return instance
+}
+
+/** Load a model via vLLM */
+async function loadViaVLLM(artifact: ModelArtifact, modelPath: string, gpuIndex?: number): Promise<ModelInstance> {
+  const port = allocPort()
+
+  const args = [
+    "serve", artifact.repo,
+    "--port", String(port),
+    "--served-model-name", artifact.repo,
+    "--trust-remote-code",
+  ]
+
+  const proc = Bun.spawn(["vllm", ...args], {
+    stdout: "pipe",
+    stderr: "pipe",
+    windowsHide: true,
+    detached: true,
+    env: {
+      ...process.env,
+      ...(gpuIndex !== undefined ? { CUDA_VISIBLE_DEVICES: String(gpuIndex) } : {}),
+    },
+  })
+
+  // vLLM takes longer to start
+  for (let i = 0; i < 120; i++) {
+    await new Promise((r) => setTimeout(r, 2000))
+    if (await isAlive(`http://localhost:${port}`)) break
+  }
+
+  const instance: ModelInstance = {
+    id: `vllm-${port}`,
+    artifact,
+    status: "running",
+    pid: proc.pid,
+    port,
+    gpuIndex,
+    endpoint: `http://localhost:${port}/v1`,
+  }
+
+  running.set(instance.id, instance)
+  return instance
+}
+
+/** Smart-load a model: pick the best backend and GPU automatically */
+export async function load(
+  artifact: ModelArtifact,
+  opts?: { backend?: BackendType; gpuIndex?: number },
+): Promise<ModelInstance> {
+  const gpuProfile = await Effect.runPromise(GPU.profile)
+  const vramNeeded = GPU.estimateVRAM(artifact.sizeBytes, artifact.quantization)
+
+  // Pick GPU
+  const gpuIndex = opts?.gpuIndex ?? (() => {
+    const gpu = GPU.findBestGPU(gpuProfile, vramNeeded)
+    return gpu?.index
+  })()
+
+  if (gpuIndex === undefined && gpuProfile.cudaAvailable) {
+    log.warn("insufficient VRAM, model may run on CPU", { needed: vramNeeded, available: gpuProfile.availableVRAMMB })
+  }
+
+  // Install if needed
+  const modelPath = await Manager.install(artifact)
+
+  // Pick backend
+  const backends = await detectBackends()
+  const preferred = opts?.backend ?? iife(() => {
+    if (artifact.format === "gguf") {
+      if (backends.find((b) => b.type === "llama.cpp")?.available) return "llama.cpp" as BackendType
+      if (backends.find((b) => b.type === "ollama")?.available) return "ollama" as BackendType
+    }
+    if (backends.find((b) => b.type === "vllm")?.available) return "vllm" as BackendType
+    if (backends.find((b) => b.type === "ollama")?.available) return "ollama" as BackendType
+    return "llama.cpp" as BackendType
+  })
+
+  log.info("loading model", { repo: artifact.repo, backend: preferred, gpuIndex })
+
+  // Load via chosen backend
+  const instance: ModelInstance = {
+    id: `${preferred}-${artifact.repo}`,
+    artifact,
+    status: "starting",
+    gpuIndex,
+    vramUsageMB: vramNeeded,
+  }
+
+  running.set(instance.id, { ...instance })
+
+  switch (preferred) {
+    case "ollama":
+      return loadViaOllama(artifact, modelPath)
+    case "llama.cpp":
+      return loadViaLlamaCpp(artifact, modelPath, gpuIndex)
+    case "vllm":
+      return loadViaVLLM(artifact, modelPath, gpuIndex)
+    default:
+      return loadViaLlamaCpp(artifact, modelPath, gpuIndex)
+  }
+}
+
+/** Unload a running model */
+export async function unload(instanceId: string): Promise<void> {
+  const inst = running.get(instanceId)
+  if (!inst) return
+
+  if (inst.pid) {
+    try { process.kill(inst.pid) } catch { /* already dead */ }
+  }
+
+  ;(inst as any).status = "stopped"
+  ;(inst as any).pid = undefined
+  running.delete(instanceId)
+  log.info("unloaded model", { id: instanceId })
+}
+
+/** List all running model instances */
+export function runningInstances(): ModelInstance[] {
+  return [...running.values()]
+}
+
+/** Get a running instance by its endpoint */
+export function findByEndpoint(url: string): ModelInstance | undefined {
+  return [...running.values()].find((i) => i.endpoint === url)
+}
+
+/** Load multiple models with GPU-aware scheduling */
+export async function loadBatch(
+  artifacts: ModelArtifact[],
+): Promise<ModelInstance[]> {
+  const gpuProfile = await Effect.runPromise(GPU.profile)
+  const results: ModelInstance[] = []
+
+  // Sort by size descending — load largest first to ensure they get GPU space
+  const sorted = [...artifacts].sort((a, b) => b.sizeBytes - a.sizeBytes)
+
+  // Track per-GPU allocation
+  const gpuAlloc = new Map<number, number>() // gpuIndex -> usedMB
+  for (const g of gpuProfile.gpus) gpuAlloc.set(g.index, g.vramTotalMB - g.vramFreeMB)
+
+  for (const artifact of sorted) {
+    const vramNeeded = GPU.estimateVRAM(artifact.sizeBytes, artifact.quantization)
+
+    // Find GPU with most remaining space
+    let bestGpu: number | undefined
+    let bestFree = 0
+    for (const [idx, used] of gpuAlloc) {
+      const gpu = gpuProfile.gpus.find((g) => g.index === idx)
+      if (!gpu) continue
+      const free = gpu.vramTotalMB - used
+      if (free >= vramNeeded && free > bestFree) {
+        bestGpu = idx
+        bestFree = free
+      }
+    }
+
+    if (bestGpu !== undefined) {
+      gpuAlloc.set(bestGpu, (gpuAlloc.get(bestGpu) ?? 0) + vramNeeded)
+    }
+
+    try {
+      const instance = await load(artifact, { gpuIndex: bestGpu })
+      results.push(instance)
+    } catch (e) {
+      log.error("failed to load model in batch", { repo: artifact.repo, error: e })
+      results.push({
+        id: `error-${artifact.repo}`,
+        artifact,
+        status: "error",
+        error: e instanceof Error ? e.message : String(e),
+      })
+    }
+  }
+
+  return results
+}
+
+/** Unload all running models */
+export async function unloadAll(): Promise<void> {
+  for (const id of [...running.keys()]) {
+    await unload(id)
+  }
+}