saeeol 1.4.1 → 1.4.3

package/src/provider/local/hub.ts

@@ -1,174 +0,0 @@
-/** HuggingFace Hub client — search, browse, download models */
-
-import { Effect } from "effect"
-import * as Log from "@saeeol/core/util/log"
-import { iife } from "@/util/iife"
-import type { HFModelSearch, HFModelInfo, HFSibling, ModelArtifact, Quantization, ModelFormat, RAGAsset, RAGAssetType } from "./types"
-
-const log = Log.create({ service: "local/hub" })
-const API = "https://huggingface.co/api"
-
-/** Search HuggingFace Hub for models */
-export async function search(query: string, opts?: { limit?: number; tags?: string[] }): Promise<HFModelSearch[]> {
-  const params = new URLSearchParams({
-    search: query,
-    limit: String(opts?.limit ?? 20),
-    ...(opts?.tags?.length ? { tags: opts.tags.join(",") } : {}),
-  })
-
-  const res = await fetch(`${API}/models?${params}`)
-  if (!res.ok) throw new Error(`HF search failed: ${res.status} ${res.statusText}`)
-
-  const items = (await res.json()) as any[]
-  return items.map((item) => ({
-    id: item.id ?? item.modelId ?? "",
-    name: item.id ?? "",
-    author: item.author ?? "",
-    downloads: item.downloads ?? 0,
-    likes: item.likes ?? 0,
-    tags: item.tags ?? [],
-    pipelineTag: item.pipeline_tag ?? undefined,
-    libraryName: item.library_name ?? undefined,
-  }))
-}
-
-/** Get detailed model info from HF Hub */
-export async function info(repo: string): Promise<HFModelInfo> {
-  const res = await fetch(`${API}/models/${repo}`)
-  if (!res.ok) throw new Error(`HF model info failed: ${res.status} ${res.statusText}`)
-
-  const data = (await res.json()) as any
-  return {
-    id: data.id ?? data.modelId ?? repo,
-    modelId: data.modelId ?? repo,
-    sha: data.sha ?? "",
-    siblings: (data.siblings ?? []) as HFSibling[],
-    tags: data.tags ?? [],
-    downloads: data.downloads ?? 0,
-    likes: data.likes ?? 0,
-    private: data.private ?? false,
-  }
-}
-
-/** List GGUF files in a repo */
-export function listGGUF(siblings: HFSibling[]): string[] {
-  return siblings
-    .map((s) => s.rfilename)
-    .filter((f) => f.endsWith(".gguf"))
-}
-
-/** Parse quantization from GGUF filename */
-export function parseQuant(filename: string): Quantization | undefined {
-  const lower = filename.toLowerCase()
-  const quants: Quantization[] = [
-    "q2_k", "q3_k_s", "q3_k_m", "q3_k_l",
-    "q4_0", "q4_1", "q4_k_s", "q4_k_m",
-    "q5_0", "q5_1", "q5_k_s", "q5_k_m",
-    "q6_k", "q8_0", "fp16", "bf16", "fp32",
-  ]
-  for (const q of quants) {
-    if (lower.includes(q)) return q
-  }
-  return undefined
-}
-
-/** Resolve download URL for a file in a repo */
-export function downloadURL(repo: string, filename: string): string {
-  return `https://huggingface.co/${repo}/resolve/main/${filename}`
-}
-
-/** Build a ModelArtifact from repo + sibling info */
-export function buildArtifact(
-  repo: string,
-  filename: string,
-  sizeBytes: number,
-  sha256?: string,
-): ModelArtifact {
-  const q = parseQuant(filename)
-  const ext = filename.split(".").pop() ?? ""
-  const formatMap: Record<string, ModelFormat> = {
-    gguf: "gguf",
-    safetensors: "safetensors",
-    bin: "pytorch",
-    pt: "pytorch",
-    onnx: "onnx",
-  }
-
-  return {
-    id: `${repo}/${filename}`,
-    repo,
-    filename,
-    format: formatMap[ext] ?? "pytorch",
-    quantization: q ?? "fp16",
-    sizeBytes,
-    sha256,
-  }
-}
-
-/** Download a file from HF Hub with progress callback */
-export async function download(
-  repo: string,
-  filename: string,
-  destPath: string,
-  opts?: { onProgress?: (downloaded: number, total: number) => void; signal?: AbortSignal },
-): Promise<void> {
-  const url = downloadURL(repo, filename)
-
-  const res = await fetch(url, { signal: opts?.signal })
-  if (!res.ok) throw new Error(`HF download failed: ${res.status} ${res.statusText}`)
-  if (!res.body) throw new Error("No response body")
-
-  const total = Number(res.headers.get("content-length") ?? "0")
-  const file = Bun.file(destPath)
-  const writer = file.writer()
-
-  let downloaded = 0
-  const reader = res.body.getReader()
-
-  try {
-    while (true) {
-      const { done, value } = await reader.read()
-      if (done) break
-      writer.write(value)
-      downloaded += value.length
-      opts?.onProgress?.(downloaded, total)
-    }
-  } finally {
-    writer.end()
-  }
-
-  log.info("downloaded", { repo, filename, sizeMB: Math.round(downloaded / 1024 / 1024) })
-}
-
-/** Search for embedding/reranker models */
-export async function searchRAG(
-  type: RAGAssetType,
-  query?: string,
-  limit?: number,
-): Promise<RAGAsset[]> {
-  const tagMap: Record<RAGAssetType, string> = {
-    embedding: "sentence-transformers",
-    reranker: "reranker",
-    vectordb: "vector",
-  }
-
-  const params = new URLSearchParams({
-    limit: String(limit ?? 20),
-    tags: tagMap[type],
-    ...(query ? { search: query } : {}),
-  })
-
-  const res = await fetch(`${API}/models?${params}`)
-  if (!res.ok) return []
-
-  const items = (await res.json()) as any[]
-  return items.map((item) => ({
-    id: item.id ?? "",
-    name: item.id ?? "",
-    type,
-    repo: item.id ?? "",
-    format: "safetensors" as ModelFormat,
-    sizeBytes: 0,
-    dimensions: undefined,
-  }))
-}

package/src/provider/local/index.ts

@@ -1,10 +0,0 @@
-/** Local model orchestration — public API */
-
-export * from "./types"
-export * as GPU from "./gpu"
-export * as Hub from "./hub"
-export * as Manager from "./model-manager"
-export * as Orchestrator from "./orchestrator"
-export * as RAG from "./rag"
-export * as Embedder from "./embedder"
-export { LocalModelEvent } from "./events"

package/src/provider/local/model-manager.ts

@@ -1,113 +0,0 @@
-/** Local model manager — install, remove, list, configure models on disk */
-
-import path from "path"
-import { mkdir, rm, readdir, stat } from "fs/promises"
-import { Effect } from "effect"
-import * as Log from "@saeeol/core/util/log"
-import { Global } from "@saeeol/core/global"
-import { iife } from "@/util/iife"
-import type { ModelArtifact, BackendType } from "./types"
-import * as Hub from "./hub"
-
-const log = Log.create({ service: "local/manager" })
-
-/** Root directory for all local models */
-export function modelsDir(): string {
-  return path.join(Global.Path.data, "local-models")
-}
-
-/** Directory for a specific model artifact */
-export function modelDir(artifact: ModelArtifact): string {
-  return path.join(modelsDir(), artifact.repo.replace("/", "__"), artifact.filename)
-}
-
-/** Directory for a specific backend's data */
-export function backendDir(backend: BackendType): string {
-  return path.join(modelsDir(), "backends", backend)
-}
-
-/** Directory for RAG assets */
-export function ragDir(): string {
-  return path.join(modelsDir(), "rag")
-}
-
-/** Ensure a directory exists */
-async function ensure(dir: string): Promise<void> {
-  await mkdir(dir, { recursive: true })
-}
-
-/** Install a model from HuggingFace Hub */
-export async function install(
-  artifact: ModelArtifact,
-  opts?: { onProgress?: (downloaded: number, total: number) => void; signal?: AbortSignal },
-): Promise<string> {
-  const dest = modelDir(artifact)
-  await ensure(path.dirname(dest))
-
-  const fullPath = path.join(dest, artifact.filename)
-  const exists = await stat(fullPath).catch(() => undefined)
-  if (exists?.size === artifact.sizeBytes) {
-    log.info("model already installed", { repo: artifact.repo, filename: artifact.filename })
-    return fullPath
-  }
-
-  log.info("installing model", { repo: artifact.repo, filename: artifact.filename })
-  await Hub.download(artifact.repo, artifact.filename, fullPath, opts)
-  return fullPath
-}
-
-/** Remove a model from disk */
-export async function uninstall(artifact: ModelArtifact): Promise<void> {
-  const dir = modelDir(artifact)
-  await rm(dir, { recursive: true, force: true })
-  log.info("uninstalled model", { repo: artifact.repo, filename: artifact.filename })
-}
-
-/** List all installed models on disk */
-export async function list(): Promise<Array<{ repo: string; filename: string; path: string; sizeBytes: number }>> {
-  const root = modelsDir()
-  const result: Array<{ repo: string; filename: string; path: string; sizeBytes: number }> = []
-
-  const entries = await readdir(root, { withFileTypes: true }).catch(() => [] as import("fs").Dirent[])
-  for (const entry of entries) {
-    if (!entry.isDirectory()) continue
-    if (entry.name === "backends" || entry.name === "rag") continue
-
-    const repo = entry.name.replace("__", "/")
-    const repoDir = path.join(root, entry.name)
-    const files = await readdir(repoDir).catch(() => [] as string[])
-
-    for (const file of files) {
-      const filePath = path.join(repoDir, file)
-      const s = await stat(filePath).catch(() => null)
-      if (s?.isFile()) {
-        result.push({ repo, filename: file, path: filePath, sizeBytes: s.size })
-      }
-    }
-  }
-
-  return result
-}
-
-/** Install a RAG asset (embedding model, reranker, vectordb) */
-export async function installRAG(
-  asset: { repo: string; filename: string; sizeBytes: number },
-  opts?: { onProgress?: (downloaded: number, total: number) => void; signal?: AbortSignal },
-): Promise<string> {
-  const dir = path.join(ragDir(), asset.repo.replace("/", "__"))
-  await ensure(dir)
-
-  const fullPath = path.join(dir, asset.filename)
-  const exists = await stat(fullPath).catch(() => undefined)
-  if (exists?.size === asset.sizeBytes) return fullPath
-
-  await Hub.download(asset.repo, asset.filename, fullPath, opts)
-  return fullPath
-}
-
-/** Check if a model is installed */
-export async function isInstalled(artifact: ModelArtifact): Promise<boolean> {
-  const fullPath = path.join(modelDir(artifact), artifact.filename)
-  const s = await stat(fullPath).catch(() => undefined)
-  return s?.isFile() === true && s.size > 0
-}

package/src/provider/local/orchestrator.ts

@@ -1,301 +0,0 @@
-/** Model orchestrator — run multiple models simultaneously with GPU-aware scheduling */
-
-import { Effect } from "effect"
-import * as Log from "@saeeol/core/util/log"
-import { Process } from "@/util/process"
-import { Global } from "@saeeol/core/global"
-import type { ModelArtifact, ModelInstance, BackendType, BackendStatus, GPUProfile } from "./types"
-import * as GPU from "./gpu"
-import * as Manager from "./model-manager"
-import { which } from "@/util/which"
-import { iife } from "@/util/iife"
-import path from "path"
-import { mkdir } from "fs/promises"
-
-const log = Log.create({ service: "local/orchestrator" })
-
-const running = new Map<string, ModelInstance>()
-let nextPort = 11435 // Start after default Ollama port
-
-/** Detect which local backends are installed */
-export async function detectBackends(): Promise<BackendStatus[]> {
-  const backends: Array<{ type: BackendType; cmds: string[]; defaultPort: number }> = [
-    { type: "ollama", cmds: ["ollama"], defaultPort: 11434 },
-    { type: "lmstudio", cmds: ["lms"], defaultPort: 1234 },
-    { type: "vllm", cmds: ["vllm", "python -m vllm"], defaultPort: 8000 },
-    { type: "llama.cpp", cmds: ["llama-server", "llama-cpp-server"], defaultPort: 8080 },
-  ]
-
-  const results: BackendStatus[] = []
-  for (const b of backends) {
-    let cmd = ""
-    let available = false
-    for (const c of b.cmds) {
-      const found = await which(c.split(" ")[0])
-      if (found) { cmd = c; available = true; break }
-    }
-
-    results.push({
-      type: b.type,
-      available,
-      endpoint: `http://localhost:${b.defaultPort}`,
-      version: undefined,
-      loadedModels: [],
-    })
-  }
-
-  return results
-}
-
-/** Check if a backend server is responding */
-export async function isAlive(endpoint: string): Promise<boolean> {
-  try {
-    const res = await fetch(endpoint, { signal: AbortSignal.timeout(3000) })
-    return res.ok || res.status === 404 // 404 is fine, means server is up
-  } catch {
-    return false
-  }
-}
-
-/** Allocate a free port for a new model instance */
-function allocPort(): number {
-  return nextPort++
-}
-
-/** Load a model via Ollama */
-async function loadViaOllama(artifact: ModelArtifact, modelPath: string): Promise<ModelInstance> {
-  const port = 11434 // Ollama default
-
-  // Register model with Ollama if not already present
-  const modelfile = `FROM ${modelPath}`
-  const name = artifact.repo.replace("/", "-").toLowerCase()
-
-  const result = await Process.run(["ollama", "create", name, "-f", "-"])
-
-  log.info("ollama create result", { code: result.code })
-
-  return {
-    id: name,
-    artifact,
-    status: "running",
-    port,
-    endpoint: `http://localhost:${port}`,
-  }
-}
-
-/** Load a model via llama.cpp server */
-async function loadViaLlamaCpp(artifact: ModelArtifact, modelPath: string, gpuIndex?: number): Promise<ModelInstance> {
-  const port = allocPort()
-
-  const args = ["-m", modelPath, "--port", String(port), "-c", "4096", "-ngl", "99"]
-  if (gpuIndex !== undefined) args.push("--gpu", String(gpuIndex))
-
-  const proc = Bun.spawn(["llama-server", ...args], {
-    stdout: "pipe",
-    stderr: "pipe",
-    windowsHide: true,
-    detached: true,
-  })
-
-  // Wait for server to start
-  for (let i = 0; i < 30; i++) {
-    await new Promise((r) => setTimeout(r, 1000))
-    if (await isAlive(`http://localhost:${port}`)) break
-  }
-
-  const instance: ModelInstance = {
-    id: `llamacpp-${port}`,
-    artifact,
-    status: "running",
-    pid: proc.pid,
-    port,
-    gpuIndex,
-    endpoint: `http://localhost:${port}/v1`,
-  }
-
-  running.set(instance.id, instance)
-  return instance
-}
-
-/** Load a model via vLLM */
-async function loadViaVLLM(artifact: ModelArtifact, modelPath: string, gpuIndex?: number): Promise<ModelInstance> {
-  const port = allocPort()
-
-  const args = [
-    "serve", artifact.repo,
-    "--port", String(port),
-    "--served-model-name", artifact.repo,
-    "--trust-remote-code",
-  ]
-
-  const proc = Bun.spawn(["vllm", ...args], {
-    stdout: "pipe",
-    stderr: "pipe",
-    windowsHide: true,
-    detached: true,
-    env: {
-      ...process.env,
-      ...(gpuIndex !== undefined ? { CUDA_VISIBLE_DEVICES: String(gpuIndex) } : {}),
-    },
-  })
-
-  // vLLM takes longer to start
-  for (let i = 0; i < 120; i++) {
-    await new Promise((r) => setTimeout(r, 2000))
-    if (await isAlive(`http://localhost:${port}`)) break
-  }
-
-  const instance: ModelInstance = {
-    id: `vllm-${port}`,
-    artifact,
-    status: "running",
-    pid: proc.pid,
-    port,
-    gpuIndex,
-    endpoint: `http://localhost:${port}/v1`,
-  }
-
-  running.set(instance.id, instance)
-  return instance
-}
-
-/** Smart-load a model: pick the best backend and GPU automatically */
-export async function load(
-  artifact: ModelArtifact,
-  opts?: { backend?: BackendType; gpuIndex?: number },
-): Promise<ModelInstance> {
-  const gpuProfile = await Effect.runPromise(GPU.profile)
-  const vramNeeded = GPU.estimateVRAM(artifact.sizeBytes, artifact.quantization)
-
-  // Pick GPU
-  const gpuIndex = opts?.gpuIndex ?? (() => {
-    const gpu = GPU.findBestGPU(gpuProfile, vramNeeded)
-    return gpu?.index
-  })()
-
-  if (gpuIndex === undefined && gpuProfile.cudaAvailable) {
-    log.warn("insufficient VRAM, model may run on CPU", { needed: vramNeeded, available: gpuProfile.availableVRAMMB })
-  }
-
-  // Install if needed
-  const modelPath = await Manager.install(artifact)
-
-  // Pick backend
-  const backends = await detectBackends()
-  const preferred = opts?.backend ?? iife(() => {
-    if (artifact.format === "gguf") {
-      if (backends.find((b) => b.type === "llama.cpp")?.available) return "llama.cpp" as BackendType
-      if (backends.find((b) => b.type === "ollama")?.available) return "ollama" as BackendType
-    }
-    if (backends.find((b) => b.type === "vllm")?.available) return "vllm" as BackendType
-    if (backends.find((b) => b.type === "ollama")?.available) return "ollama" as BackendType
-    return "llama.cpp" as BackendType
-  })
-
-  log.info("loading model", { repo: artifact.repo, backend: preferred, gpuIndex })
-
-  // Load via chosen backend
-  const instance: ModelInstance = {
-    id: `${preferred}-${artifact.repo}`,
-    artifact,
-    status: "starting",
-    gpuIndex,
-    vramUsageMB: vramNeeded,
-  }
-
-  running.set(instance.id, { ...instance })
-
-  switch (preferred) {
-    case "ollama":
-      return loadViaOllama(artifact, modelPath)
-    case "llama.cpp":
-      return loadViaLlamaCpp(artifact, modelPath, gpuIndex)
-    case "vllm":
-      return loadViaVLLM(artifact, modelPath, gpuIndex)
-    default:
-      return loadViaLlamaCpp(artifact, modelPath, gpuIndex)
-  }
-}
-
-/** Unload a running model */
-export async function unload(instanceId: string): Promise<void> {
-  const inst = running.get(instanceId)
-  if (!inst) return
-
-  if (inst.pid) {
-    try { process.kill(inst.pid) } catch { /* already dead */ }
-  }
-
-  ;(inst as any).status = "stopped"
-  ;(inst as any).pid = undefined
-  running.delete(instanceId)
-  log.info("unloaded model", { id: instanceId })
-}
-
-/** List all running model instances */
-export function runningInstances(): ModelInstance[] {
-  return [...running.values()]
-}
-
-/** Get a running instance by its endpoint */
-export function findByEndpoint(url: string): ModelInstance | undefined {
-  return [...running.values()].find((i) => i.endpoint === url)
-}
-
-/** Load multiple models with GPU-aware scheduling */
-export async function loadBatch(
-  artifacts: ModelArtifact[],
-): Promise<ModelInstance[]> {
-  const gpuProfile = await Effect.runPromise(GPU.profile)
-  const results: ModelInstance[] = []
-
-  // Sort by size descending — load largest first to ensure they get GPU space
-  const sorted = [...artifacts].sort((a, b) => b.sizeBytes - a.sizeBytes)
-
-  // Track per-GPU allocation
-  const gpuAlloc = new Map<number, number>() // gpuIndex -> usedMB
-  for (const g of gpuProfile.gpus) gpuAlloc.set(g.index, g.vramTotalMB - g.vramFreeMB)
-
-  for (const artifact of sorted) {
-    const vramNeeded = GPU.estimateVRAM(artifact.sizeBytes, artifact.quantization)
-
-    // Find GPU with most remaining space
-    let bestGpu: number | undefined
-    let bestFree = 0
-    for (const [idx, used] of gpuAlloc) {
-      const gpu = gpuProfile.gpus.find((g) => g.index === idx)
-      if (!gpu) continue
-      const free = gpu.vramTotalMB - used
-      if (free >= vramNeeded && free > bestFree) {
-        bestGpu = idx
-        bestFree = free
-      }
-    }
-
-    if (bestGpu !== undefined) {
-      gpuAlloc.set(bestGpu, (gpuAlloc.get(bestGpu) ?? 0) + vramNeeded)
-    }
-
-    try {
-      const instance = await load(artifact, { gpuIndex: bestGpu })
-      results.push(instance)
-    } catch (e) {
-      log.error("failed to load model in batch", { repo: artifact.repo, error: e })
-      results.push({
-        id: `error-${artifact.repo}`,
-        artifact,
-        status: "error",
-        error: e instanceof Error ? e.message : String(e),
-      })
-    }
-  }
-
-  return results
-}
-
-/** Unload all running models */
-export async function unloadAll(): Promise<void> {
-  for (const id of [...running.keys()]) {
-    await unload(id)
-  }
-}