vllm-i64 0.1.0

package/dist/index.js

@@ -0,0 +1,381 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  CacheEndpoint: () => CacheEndpoint,
+  ChatEndpoint: () => ChatEndpoint,
+  CompletionsEndpoint: () => CompletionsEndpoint,
+  HttpClient: () => HttpClient,
+  I64Client: () => I64Client,
+  LoRAEndpoint: () => LoRAEndpoint,
+  MonitorEndpoint: () => MonitorEndpoint,
+  RAGEndpoint: () => RAGEndpoint,
+  default: () => index_default
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/client.ts
+var HttpClient = class {
+  baseUrl;
+  apiKey;
+  timeout;
+  constructor(baseUrl = "http://localhost:8000", options = {}) {
+    this.baseUrl = baseUrl.replace(/\/+$/, "");
+    this.apiKey = options.apiKey;
+    this.timeout = options.timeoutMs ?? 12e4;
+  }
+  async fetch(path, init = {}) {
+    const headers = {
+      "Content-Type": "application/json",
+      ...init.headers
+    };
+    if (this.apiKey) {
+      headers["Authorization"] = `Bearer ${this.apiKey}`;
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeout);
+    try {
+      const res = await fetch(`${this.baseUrl}${path}`, {
+        ...init,
+        headers,
+        signal: controller.signal
+      });
+      if (!res.ok) {
+        const body = await res.text();
+        let msg;
+        try {
+          msg = JSON.parse(body)?.error?.message ?? body;
+        } catch {
+          msg = body;
+        }
+        throw new Error(`vllm-i64 ${res.status}: ${msg}`);
+      }
+      return res;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+  async get(path) {
+    const res = await this.fetch(path);
+    return res.json();
+  }
+  async post(path, body) {
+    const res = await this.fetch(path, {
+      method: "POST",
+      body: JSON.stringify(body)
+    });
+    return res.json();
+  }
+  // ---------------------------------------------------------------
+  // SSE streaming
+  // ---------------------------------------------------------------
+  async *readSSE(res) {
+    if (!res.body) return;
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() ?? "";
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (!trimmed.startsWith("data: ")) continue;
+          const payload = trimmed.slice(6);
+          if (payload === "[DONE]") return;
+          try {
+            const data = JSON.parse(payload);
+            const content = data.choices?.[0]?.delta?.content ?? data.choices?.[0]?.text ?? "";
+            if (content) yield content;
+          } catch {
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+  async *readSSERaw(res) {
+    if (!res.body) return;
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() ?? "";
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (!trimmed.startsWith("data: ")) continue;
+          const payload = trimmed.slice(6);
+          if (payload === "[DONE]") return;
+          try {
+            yield JSON.parse(payload);
+          } catch {
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+};
+
+// src/endpoints/chat.ts
+var ChatEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /**
+   * Chat completion (non-streaming).
+   *
+   * @example
+   * ```ts
+   * const res = await client.chat.create([{ role: "user", content: "Hi" }]);
+   * console.log(res.choices[0].message.content);
+   * ```
+   */
+  async create(messages, options = {}) {
+    return this.http.post("/v1/chat/completions", {
+      model: "default",
+      messages,
+      ...options,
+      stream: false
+    });
+  }
+  /**
+   * Streaming chat — yields content strings.
+   *
+   * @example
+   * ```ts
+   * for await (const chunk of client.chat.stream([{ role: "user", content: "Hi" }])) {
+   *   process.stdout.write(chunk);
+   * }
+   * ```
+   */
+  async *stream(messages, options = {}) {
+    const res = await this.http.fetch("/v1/chat/completions", {
+      method: "POST",
+      body: JSON.stringify({
+        model: "default",
+        messages,
+        ...options,
+        stream: true
+      })
+    });
+    yield* this.http.readSSE(res);
+  }
+  /**
+   * Streaming chat — yields raw SSE delta objects.
+   */
+  async *streamRaw(messages, options = {}) {
+    const res = await this.http.fetch("/v1/chat/completions", {
+      method: "POST",
+      body: JSON.stringify({
+        model: "default",
+        messages,
+        ...options,
+        stream: true
+      })
+    });
+    yield* this.http.readSSERaw(res);
+  }
+};
+
+// src/endpoints/completions.ts
+var CompletionsEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Text completion (non-streaming). */
+  async create(prompt, options = {}) {
+    return this.http.post("/v1/completions", {
+      model: "default",
+      prompt,
+      ...options,
+      stream: false
+    });
+  }
+  /** Streaming text completion — yields text chunks. */
+  async *stream(prompt, options = {}) {
+    const res = await this.http.fetch("/v1/completions", {
+      method: "POST",
+      body: JSON.stringify({
+        model: "default",
+        prompt,
+        ...options,
+        stream: true
+      })
+    });
+    yield* this.http.readSSE(res);
+  }
+  /** Submit multiple prompts at once. */
+  async batch(prompts, options = {}) {
+    return this.http.post("/v1/batch", { prompts, ...options });
+  }
+};
+
+// src/endpoints/cache.ts
+var CacheEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Get KV cache statistics. */
+  async stats() {
+    return this.http.get("/v1/cache/stats");
+  }
+  /** Purge prefix cache (admin). */
+  async purge() {
+    return this.http.post("/v1/cache/purge", {});
+  }
+};
+
+// src/endpoints/lora.ts
+var LoRAEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Load a LoRA adapter (admin). */
+  async load(params) {
+    return this.http.post("/v1/lora/load", params);
+  }
+  /** Unload a LoRA adapter (admin). */
+  async unload(adapter_id) {
+    return this.http.post("/v1/lora/unload", { adapter_id });
+  }
+  /** List loaded adapters. */
+  async list() {
+    return this.http.get("/v1/lora/list");
+  }
+};
+
+// src/endpoints/monitor.ts
+var MonitorEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Health check with engine stats. */
+  async health() {
+    return this.http.get("/health");
+  }
+  /** Check if server is reachable and ready. */
+  async isReady() {
+    try {
+      const h = await this.health();
+      return h.status === "ok" || h.status === "degraded";
+    } catch {
+      return false;
+    }
+  }
+  /** List available models. */
+  async models() {
+    return this.http.get("/v1/models");
+  }
+  /** Live monitoring snapshot (batch, KV, perf, GPU). */
+  async snapshot() {
+    return this.http.get("/v1/monitor");
+  }
+  /** Latency percentiles and request stats. */
+  async metrics() {
+    return this.http.get("/v1/metrics");
+  }
+  /** Expert routing distribution (MoE models). */
+  async experts() {
+    return this.http.get("/v1/experts");
+  }
+  /** Cancel a running request. */
+  async cancel(requestId) {
+    return this.http.post(`/v1/cancel/${requestId}`, {});
+  }
+};
+
+// src/endpoints/rag.ts
+var RAGEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Index text or a file. */
+  async index(params) {
+    return this.http.post("/v1/rag/index", params);
+  }
+  /** Search indexed documents. */
+  async search(query, k = 3) {
+    return this.http.post("/v1/rag/search", { query, k });
+  }
+  /** Get RAG index statistics. */
+  async stats() {
+    return this.http.get("/v1/rag/stats");
+  }
+};
+
+// src/index.ts
+var I64Client = class {
+  http;
+  /** Chat completions (streaming + non-streaming, tool_calls). */
+  chat;
+  /** Text completions (streaming + batch). */
+  completions;
+  /** KV cache management (stats, purge). */
+  cache;
+  /** LoRA adapter management (load, unload, list). */
+  lora;
+  /** Monitoring, health, metrics, expert routing. */
+  monitor;
+  /** RAG — index, search, stats. */
+  rag;
+  /**
+   * Create a vllm-i64 client.
+   *
+   * @param baseUrl - Server URL (default: http://localhost:8000)
+   * @param options - API key and timeout
+   */
+  constructor(baseUrl = "http://localhost:8000", options = {}) {
+    this.http = new HttpClient(baseUrl, options);
+    this.chat = new ChatEndpoint(this.http);
+    this.completions = new CompletionsEndpoint(this.http);
+    this.cache = new CacheEndpoint(this.http);
+    this.lora = new LoRAEndpoint(this.http);
+    this.monitor = new MonitorEndpoint(this.http);
+    this.rag = new RAGEndpoint(this.http);
+  }
+  /** Server base URL. */
+  get baseUrl() {
+    return this.http.baseUrl;
+  }
+};
+var index_default = I64Client;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  CacheEndpoint,
+  ChatEndpoint,
+  CompletionsEndpoint,
+  HttpClient,
+  I64Client,
+  LoRAEndpoint,
+  MonitorEndpoint,
+  RAGEndpoint
+});

package/dist/index.mjs

@@ -0,0 +1,347 @@
+// src/client.ts
+var HttpClient = class {
+  baseUrl;
+  apiKey;
+  timeout;
+  constructor(baseUrl = "http://localhost:8000", options = {}) {
+    this.baseUrl = baseUrl.replace(/\/+$/, "");
+    this.apiKey = options.apiKey;
+    this.timeout = options.timeoutMs ?? 12e4;
+  }
+  async fetch(path, init = {}) {
+    const headers = {
+      "Content-Type": "application/json",
+      ...init.headers
+    };
+    if (this.apiKey) {
+      headers["Authorization"] = `Bearer ${this.apiKey}`;
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeout);
+    try {
+      const res = await fetch(`${this.baseUrl}${path}`, {
+        ...init,
+        headers,
+        signal: controller.signal
+      });
+      if (!res.ok) {
+        const body = await res.text();
+        let msg;
+        try {
+          msg = JSON.parse(body)?.error?.message ?? body;
+        } catch {
+          msg = body;
+        }
+        throw new Error(`vllm-i64 ${res.status}: ${msg}`);
+      }
+      return res;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+  async get(path) {
+    const res = await this.fetch(path);
+    return res.json();
+  }
+  async post(path, body) {
+    const res = await this.fetch(path, {
+      method: "POST",
+      body: JSON.stringify(body)
+    });
+    return res.json();
+  }
+  // ---------------------------------------------------------------
+  // SSE streaming
+  // ---------------------------------------------------------------
+  async *readSSE(res) {
+    if (!res.body) return;
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() ?? "";
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (!trimmed.startsWith("data: ")) continue;
+          const payload = trimmed.slice(6);
+          if (payload === "[DONE]") return;
+          try {
+            const data = JSON.parse(payload);
+            const content = data.choices?.[0]?.delta?.content ?? data.choices?.[0]?.text ?? "";
+            if (content) yield content;
+          } catch {
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+  async *readSSERaw(res) {
+    if (!res.body) return;
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() ?? "";
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (!trimmed.startsWith("data: ")) continue;
+          const payload = trimmed.slice(6);
+          if (payload === "[DONE]") return;
+          try {
+            yield JSON.parse(payload);
+          } catch {
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+};
+
+// src/endpoints/chat.ts
+var ChatEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /**
+   * Chat completion (non-streaming).
+   *
+   * @example
+   * ```ts
+   * const res = await client.chat.create([{ role: "user", content: "Hi" }]);
+   * console.log(res.choices[0].message.content);
+   * ```
+   */
+  async create(messages, options = {}) {
+    return this.http.post("/v1/chat/completions", {
+      model: "default",
+      messages,
+      ...options,
+      stream: false
+    });
+  }
+  /**
+   * Streaming chat — yields content strings.
+   *
+   * @example
+   * ```ts
+   * for await (const chunk of client.chat.stream([{ role: "user", content: "Hi" }])) {
+   *   process.stdout.write(chunk);
+   * }
+   * ```
+   */
+  async *stream(messages, options = {}) {
+    const res = await this.http.fetch("/v1/chat/completions", {
+      method: "POST",
+      body: JSON.stringify({
+        model: "default",
+        messages,
+        ...options,
+        stream: true
+      })
+    });
+    yield* this.http.readSSE(res);
+  }
+  /**
+   * Streaming chat — yields raw SSE delta objects.
+   */
+  async *streamRaw(messages, options = {}) {
+    const res = await this.http.fetch("/v1/chat/completions", {
+      method: "POST",
+      body: JSON.stringify({
+        model: "default",
+        messages,
+        ...options,
+        stream: true
+      })
+    });
+    yield* this.http.readSSERaw(res);
+  }
+};
+
+// src/endpoints/completions.ts
+var CompletionsEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Text completion (non-streaming). */
+  async create(prompt, options = {}) {
+    return this.http.post("/v1/completions", {
+      model: "default",
+      prompt,
+      ...options,
+      stream: false
+    });
+  }
+  /** Streaming text completion — yields text chunks. */
+  async *stream(prompt, options = {}) {
+    const res = await this.http.fetch("/v1/completions", {
+      method: "POST",
+      body: JSON.stringify({
+        model: "default",
+        prompt,
+        ...options,
+        stream: true
+      })
+    });
+    yield* this.http.readSSE(res);
+  }
+  /** Submit multiple prompts at once. */
+  async batch(prompts, options = {}) {
+    return this.http.post("/v1/batch", { prompts, ...options });
+  }
+};
+
+// src/endpoints/cache.ts
+var CacheEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Get KV cache statistics. */
+  async stats() {
+    return this.http.get("/v1/cache/stats");
+  }
+  /** Purge prefix cache (admin). */
+  async purge() {
+    return this.http.post("/v1/cache/purge", {});
+  }
+};
+
+// src/endpoints/lora.ts
+var LoRAEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Load a LoRA adapter (admin). */
+  async load(params) {
+    return this.http.post("/v1/lora/load", params);
+  }
+  /** Unload a LoRA adapter (admin). */
+  async unload(adapter_id) {
+    return this.http.post("/v1/lora/unload", { adapter_id });
+  }
+  /** List loaded adapters. */
+  async list() {
+    return this.http.get("/v1/lora/list");
+  }
+};
+
+// src/endpoints/monitor.ts
+var MonitorEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Health check with engine stats. */
+  async health() {
+    return this.http.get("/health");
+  }
+  /** Check if server is reachable and ready. */
+  async isReady() {
+    try {
+      const h = await this.health();
+      return h.status === "ok" || h.status === "degraded";
+    } catch {
+      return false;
+    }
+  }
+  /** List available models. */
+  async models() {
+    return this.http.get("/v1/models");
+  }
+  /** Live monitoring snapshot (batch, KV, perf, GPU). */
+  async snapshot() {
+    return this.http.get("/v1/monitor");
+  }
+  /** Latency percentiles and request stats. */
+  async metrics() {
+    return this.http.get("/v1/metrics");
+  }
+  /** Expert routing distribution (MoE models). */
+  async experts() {
+    return this.http.get("/v1/experts");
+  }
+  /** Cancel a running request. */
+  async cancel(requestId) {
+    return this.http.post(`/v1/cancel/${requestId}`, {});
+  }
+};
+
+// src/endpoints/rag.ts
+var RAGEndpoint = class {
+  constructor(http) {
+    this.http = http;
+  }
+  /** Index text or a file. */
+  async index(params) {
+    return this.http.post("/v1/rag/index", params);
+  }
+  /** Search indexed documents. */
+  async search(query, k = 3) {
+    return this.http.post("/v1/rag/search", { query, k });
+  }
+  /** Get RAG index statistics. */
+  async stats() {
+    return this.http.get("/v1/rag/stats");
+  }
+};
+
+// src/index.ts
+var I64Client = class {
+  http;
+  /** Chat completions (streaming + non-streaming, tool_calls). */
+  chat;
+  /** Text completions (streaming + batch). */
+  completions;
+  /** KV cache management (stats, purge). */
+  cache;
+  /** LoRA adapter management (load, unload, list). */
+  lora;
+  /** Monitoring, health, metrics, expert routing. */
+  monitor;
+  /** RAG — index, search, stats. */
+  rag;
+  /**
+   * Create a vllm-i64 client.
+   *
+   * @param baseUrl - Server URL (default: http://localhost:8000)
+   * @param options - API key and timeout
+   */
+  constructor(baseUrl = "http://localhost:8000", options = {}) {
+    this.http = new HttpClient(baseUrl, options);
+    this.chat = new ChatEndpoint(this.http);
+    this.completions = new CompletionsEndpoint(this.http);
+    this.cache = new CacheEndpoint(this.http);
+    this.lora = new LoRAEndpoint(this.http);
+    this.monitor = new MonitorEndpoint(this.http);
+    this.rag = new RAGEndpoint(this.http);
+  }
+  /** Server base URL. */
+  get baseUrl() {
+    return this.http.baseUrl;
+  }
+};
+var index_default = I64Client;
+export {
+  CacheEndpoint,
+  ChatEndpoint,
+  CompletionsEndpoint,
+  HttpClient,
+  I64Client,
+  LoRAEndpoint,
+  MonitorEndpoint,
+  RAGEndpoint,
+  index_default as default
+};