pi-llama-cpp 0.1.0

package/LICENSE.md

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2026 Gabriel Sanhueza (https://github.com/gsanhueza)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,109 @@
+# pi-llama-cpp
+
+A [Pi Coding Agent](https://pi.dev/) extension that integrates with a running [llama.cpp server](https://github.com/ggml-org/llama.cpp) to provide live model browsing, loading, and switching directly from Pi.
+
+## Features
+
+- **Auto-detect models** — discovers all models available on your running llama.cpp server
+- **Live status indicators** — see which models are loaded, loading, failed, or unloaded with color-coded icons
+- **Load / unload / switch** — manage models directly from the Pi command palette
+- **Multi-model router support** — works with both single-model and multi-model llama.cpp server configurations
+- **Image model support** — detects multimodal models automatically
+- **Flexible URL resolution** — configures the server URL via project config, environment variable, or global settings
+
+## Installation
+
+This package is a Pi extension. Install it in your project:
+
+```bash
+pi install https://github.com/gsanhueza/pi-llama-cpp
+```
+
+## Configuration
+
+The extension resolves the llama.cpp server URL using the following priority order:
+
+1. **Per-project config** — `.pi/llama-server.json` in your project root:
+
+   ```json
+   {
+     "url": "http://127.0.0.1:8080"
+   }
+   ```
+
+2. **Environment variable** — `LLAMA_SERVER_URL`
+
+3. **Global settings** — `~/.pi/agent/settings.json`:
+
+   ```json
+   {
+     "llamaServerUrl": "http://127.0.0.1:8080"
+   }
+   ```
+
+4. **Default** — `http://127.0.0.1:8080`
+
+### API Key
+
+If your llama.cpp server requires authentication, use `/login` in Pi, select the "API key" option, and choose the `llama-server` provider.
+
+Alternatively, configure the API key in `~/.pi/agent/auth.json`:
+
+```json
+{
+  "llama-server": {
+    "type": "bearer",
+    "key": "your-api-key-here"
+  }
+}
+```
+
+## Usage
+
+### Prerequisites
+
+Make sure your llama.cpp server is running with the appropriate flags. For multi-model support (model router), start the server with:
+
+```bash
+llama-server --models-preset path/to/presets.ini
+```
+
+(You can use both `--fit-ctx` and `--ctx-size` in the preset — the extension checks both.)
+
+For single-model mode, a standard invocation works:
+
+```bash
+llama-server --model path/to/model.gguf --ctx-size 128000 ...
+```
+
+### Commands
+
+| Command   | Description                                                                                |
+| --------- | ------------------------------------------------------------------------------------------ |
+| `/models` | Browse llama-server models with live status. Select a model to load, switch, or unload it. |
+
+### Model Actions
+
+When browsing models via the `/models` command, you can:
+
+- **Load & switch** — Load an unloaded model and switch to it
+- **Switch model** — Switch to a model that is already loaded
+- **Unload** — Unload a loaded model to free memory
+
+### Model Selection Event
+
+When Pi switches models (e.g., via `model_select`), the extension automatically loads the selected model on the llama.cpp server. This keeps the server in sync with the active model in Pi.
+
+### Model Configuration
+
+Each model exposed to Pi includes the following defaults:
+
+- **`maxTokens`** — `16384` (maximum tokens per response)
+- **`reasoning`** — `true` (assumed, as llama.cpp's `/models` endpoint does not expose it)
+- **`cost`** — all zero (local model)
+
+## Dependencies
+
+| Dependency                      | Purpose                               |
+| ------------------------------- | ------------------------------------- |
+| `@mariozechner/pi-coding-agent` | Pi Coding Agent SDK (peer dependency) |

package/index.ts

@@ -0,0 +1,48 @@
+import type {
+  ExtensionAPI,
+  ExtensionCommandContext,
+} from "@mariozechner/pi-coding-agent";
+import { modelsCommandHandler } from "./src/handlers";
+import { isServerReady, listModels } from "./src/tools/retriever";
+import { resolveApiKey, resolveUrl } from "./src/tools/resolver";
+import { PROVIDER_NAME } from "./src/constants";
+import { onModelSelect } from "./src/events";
+
+export default async function (pi: ExtensionAPI) {
+  // Command registration
+  if (!(await isServerReady())) {
+    pi.registerCommand("models", {
+      description: `${PROVIDER_NAME} models (offline)`,
+      handler: async (
+        _: string,
+        ctx: ExtensionCommandContext,
+      ): Promise<void> => {
+        const url = await resolveUrl(ctx.cwd);
+        ctx.ui.notify(`${PROVIDER_NAME} unreachable at ${url}`, "error");
+      },
+    });
+
+    return;
+  }
+
+  const cwd = process.cwd();
+  const url = await resolveUrl(cwd);
+  const serverModels = await listModels();
+
+  pi.registerCommand("models", {
+    description: `Browse ${PROVIDER_NAME} models (live status)`,
+    handler: async (_: string, ctx: ExtensionCommandContext) =>
+      await modelsCommandHandler(ctx, pi, serverModels),
+  });
+
+  // Provider registration
+  pi.registerProvider(PROVIDER_NAME, {
+    baseUrl: `${url}/v1`,
+    api: "openai-completions",
+    apiKey: await resolveApiKey(),
+    models: await Promise.all(serverModels.map((m) => m.toProviderConfig())),
+  });
+
+  // Events registration
+  pi.on("model_select", onModelSelect);
+}

package/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "pi-llama-cpp",
+  "version": "0.1.0",
+  "description": "Pi extension for llama.cpp integration. Supports both router and single modes",
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "llama-cpp",
+    "llama.cpp"
+  ],
+  "peerDependencies": {
+    "@mariozechner/pi-coding-agent": "*"
+  },
+  "pi": {
+    "extensions": [
+      "./index"
+    ]
+  }
+}

package/src/constants.ts

@@ -0,0 +1,29 @@
+/**
+ * This provider's name
+ */
+export const PROVIDER_NAME = "llama-server";
+
+/**
+ * The default URL if the resolver couldn't find it
+ */
+export const DEFAULT_LLAMA_SERVER_URL = "http://127.0.0.1:8080";
+
+/**
+ * The default context if the server didn't expose it
+ */
+export const DEFAULT_CTX = 128000;
+
+/**
+ * Maximum number of tokens a model can generate in a single response
+ */
+export const MAX_TOKENS = 16384;
+
+/**
+ * Polling interval (ms) for checking model load status
+ */
+export const POLLING_INTERVAL = 500;
+
+/**
+ * Maximum time (ms) to wait for model loading before giving up
+ */
+export const POLLING_TIMEOUT = 60000;

package/src/enums/actions.ts

@@ -0,0 +1,7 @@
+/** The possible actions for the /models command */
+export enum Actions {
+  SWITCH = "Switch model",
+  LOAD = "Load & switch",
+  UNLOAD = "Unload",
+  CANCEL = "Cancel",
+}

package/src/enums/status.ts

@@ -0,0 +1,7 @@
+/** The possible statuses of llama-server models */
+export enum Status {
+  LOADED = "loaded",
+  LOADING = "loading",
+  FAILED = "failed",
+  UNLOADED = "unloaded",
+}

package/src/events.ts

@@ -0,0 +1,23 @@
+import { ExtensionContext } from "@mariozechner/pi-coding-agent";
+import { PROVIDER_NAME } from "./constants";
+import { listModels } from "./tools/retriever";
+import { ModelSelectEvent } from "./interfaces/IModelSelectEvent";
+
+/**
+ * Reacts to a new model event triggered by Pi
+ * @param event Model selection event
+ * @param ctx Pi context
+ */
+export const onModelSelect = async (
+  event: ModelSelectEvent,
+  ctx: ExtensionContext,
+) => {
+  if (event.model.provider !== PROVIDER_NAME) return;
+
+  const models = await listModels();
+  const model = models.find((m) => m.id === event.model.id);
+  if (!model) return;
+
+  ctx.ui.notify(`>> Loading ${model.id}...`, "info");
+  await model.load();
+};

package/src/handlers.ts

@@ -0,0 +1,91 @@
+import type {
+  ExtensionAPI,
+  ExtensionCommandContext,
+} from "@mariozechner/pi-coding-agent";
+import { Status } from "./enums/status";
+import { BaseModel } from "./models/baseModel";
+import { Actions } from "./enums/actions";
+import { PROVIDER_NAME } from "./constants";
+
+/**
+ * Defines a handler when llama-server is running
+ * @param ctx Pi context
+ * @returns The action and model, if detected
+ */
+const modelSelectionHandler = async (
+  ctx: ExtensionCommandContext,
+  models: BaseModel[],
+): Promise<{ action: Actions; model: BaseModel } | null> => {
+  // Setup the labels
+  const labels = await Promise.all(models.map((m) => m.getLabel()));
+
+  // Detect the selected model
+  const choice = await ctx.ui.select(`${PROVIDER_NAME} models:`, labels);
+  if (!choice) return null;
+
+  const idx = labels.indexOf(choice);
+  const model = models[idx];
+
+  // Define the actions that the user can do
+  const allActions = {
+    [Status.LOADED]: [Actions.UNLOAD, Actions.CANCEL],
+    [Status.LOADING]: [Actions.CANCEL],
+    [Status.FAILED]: [Actions.SWITCH, Actions.CANCEL],
+    [Status.UNLOADED]: [Actions.SWITCH, Actions.CANCEL],
+  };
+
+  const status = await model.getStatus();
+  const actions = allActions[status];
+
+  const action = (await ctx.ui.select(`${model.id}`, actions)) as Actions;
+  if (!action || action === Actions.CANCEL) return null;
+
+  // Send the selected action with the corresponding model
+  return { action, model };
+};
+
+/**
+ * Handles the /models command
+ * @param ctx The context used by Pi
+ * @param pi The Pi extension
+ */
+export const modelsCommandHandler = async (
+  ctx: ExtensionCommandContext,
+  pi: ExtensionAPI,
+  models: BaseModel[],
+): Promise<void> => {
+  const event = await modelSelectionHandler(ctx, models);
+  if (!event) return;
+
+  // Detect the model
+  const { action, model } = event;
+
+  // Execute the selected action
+  if (action === Actions.UNLOAD) {
+    await model.unload();
+    ctx.ui.notify(`Unloaded ${model.id}`, "info");
+  } else {
+    const status = await model.getStatus();
+    if (status === Status.LOADED) return;
+
+    ctx.ui.notify(`Loading ${model.id}...`, "info");
+
+    // Load the model without blocking the UI
+    const onSuccess = async () => {
+      const piModel = ctx.modelRegistry.find(PROVIDER_NAME, model.id);
+      if (!piModel) {
+        throw new Error(`Cannot find model ${model.id} in pi registry`);
+      }
+
+      await pi.setModel(piModel);
+      ctx.ui.notify(`Model ${model.id} ready`, "info");
+    };
+
+    const onFailure = (err: any) => {
+      const message = err instanceof Error ? err.message : String(err);
+      ctx.ui.notify(message, "error");
+    };
+
+    model.load().then(onSuccess).catch(onFailure);
+  }
+};

package/src/interfaces/IAuthFile.ts

@@ -0,0 +1,8 @@
+import { PROVIDER_NAME } from "../constants";
+
+export interface IAuthFile {
+  [PROVIDER_NAME]: {
+    type: string;
+    key: string;
+  };
+}

package/src/interfaces/IModelSelectEvent.ts

@@ -0,0 +1,3 @@
+export interface ModelSelectEvent {
+  model: { id: string; provider: string };
+}

package/src/interfaces/IRouterModel.ts

@@ -0,0 +1,9 @@
+export interface IRouterModel {
+  id: string;
+  aliases?: string[];
+  tags: string[];
+  object: string;
+  owned_by: string;
+  created: number;
+  status: { value: string; args: string[] };
+}

package/src/interfaces/ISingleModel.ts

@@ -0,0 +1,20 @@
+export interface ISingleModel {
+  name: string;
+  model: string;
+  modified_at: string;
+  size: string;
+  digest: string;
+  type: string;
+  description: string;
+  tags: string[];
+  capabilities: string[];
+  parameters: string;
+  details: {
+    parent_model: string;
+    format: string;
+    family: string;
+    families: string[];
+    parameter_size: string;
+    quantization_level: string;
+  };
+}

package/src/models/baseModel.ts

@@ -0,0 +1,111 @@
+import type { ProviderModelConfig } from "@mariozechner/pi-coding-agent";
+import { MAX_TOKENS, POLLING_INTERVAL, POLLING_TIMEOUT } from "../constants";
+import { Status } from "../enums/status";
+import { rpc } from "../tools/retriever";
+
+export abstract class BaseModel {
+  protected readonly statusMapper: Record<string, Status> = {
+    loaded: Status.LOADED,
+    loading: Status.LOADING,
+    failed: Status.FAILED,
+    unloaded: Status.UNLOADED,
+  };
+
+  protected readonly labelIcons: Record<Status, string> = {
+    [Status.LOADED]: "🟢",
+    [Status.LOADING]: "🟡",
+    [Status.FAILED]: "🔴",
+    [Status.UNLOADED]: "⚪",
+  };
+
+  abstract get id(): string;
+
+  abstract get name(): string;
+
+  get reasoning(): boolean {
+    // We don't have a way to detect this, so we'll fallback to true
+    return true;
+  }
+
+  /**
+   * Detects if the model can load images
+   */
+  abstract get capabilities(): ["text"] | ["image"];
+
+  /**
+   * Gets the load status of the model
+   */
+  abstract getStatus(): Promise<Status>;
+
+  /**
+   * Gets the context size of a particular model
+   */
+  abstract getContextSize(): Promise<number>;
+
+  /**
+   * Returns the corresponding label of our load status
+   */
+  async getLabel(): Promise<string> {
+    const status = await this.getStatus();
+    return `${this.labelIcons[status]} ${this.name}`;
+  }
+
+  /**
+   * Converts the llama-server model into a configuration object used by Pi
+   * @returns A Pi configuration object
+   */
+  async toProviderConfig(): Promise<ProviderModelConfig> {
+    const response = {
+      id: this.id,
+      name: this.name,
+      reasoning: this.reasoning,
+      input: this.capabilities,
+      contextWindow: await this.getContextSize(),
+      cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+      maxTokens: MAX_TOKENS,
+    };
+
+    return response;
+  }
+
+  /**
+   * Loads the model in llama-server
+   */
+  async load(): Promise<void> {
+    if ((await this.getStatus()) === Status.LOADED) return;
+
+    await rpc("/models/load", { model: this.id });
+    await this.pollStatus();
+  }
+
+  /**
+   * Unloads the model from llama-server
+   */
+
+  async unload(): Promise<void> {
+    await rpc("/models/unload", { model: this.id });
+  }
+
+  /**
+   * Polls llama-server to check when the model is loaded
+   */
+  async pollStatus(): Promise<void> {
+    const startTime = Date.now();
+
+    // Check loading status
+    try {
+      while ((await this.getStatus()) === Status.LOADING) {
+        // Force a timeout if we wasted too much time polling
+        if (Date.now() - startTime > POLLING_TIMEOUT) {
+          const message = `Model loading timed out after ${POLLING_TIMEOUT} ms: ${this.id}`;
+          throw new Error(message);
+        }
+
+        await new Promise((r) => setTimeout(r, POLLING_INTERVAL));
+      }
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      throw new Error(message);
+    }
+  }
+}

package/src/models/routerModel.ts

@@ -0,0 +1,65 @@
+import { IRouterModel } from "../interfaces/IRouterModel";
+import { DEFAULT_CTX } from "../constants";
+import { rpc } from "../tools/retriever";
+import { Status } from "../enums/status";
+import { BaseModel } from "./baseModel";
+
+export class RouterModel extends BaseModel {
+  constructor(private readonly model: IRouterModel) {
+    super();
+  }
+
+  get id(): string {
+    return this.model.id;
+  }
+
+  get name(): string {
+    return this.model.aliases?.[0] || this.model.id;
+  }
+
+  get capabilities(): ["text"] | ["image"] {
+    const hasImage = this.model.status.args?.includes("--mmproj") ?? false;
+    return hasImage ? ["image"] : ["text"];
+  }
+
+  async getStatus(): Promise<Status> {
+    const { data } = await rpc<{ data: IRouterModel[] }>("/models");
+    const model = data.find((m) => m.id === this.id);
+    if (!model) return Status.UNLOADED;
+
+    const response = this.statusMapper[model.status.value];
+    if (!response) return Status.UNLOADED;
+
+    return response;
+  }
+
+  async getContextSize(): Promise<number> {
+    let response = this.extractFrom("--ctx-size");
+    if (response) return response;
+
+    response = this.extractFrom("--fit-ctx");
+    if (response) return response;
+
+    return DEFAULT_CTX;
+  }
+
+  /**
+   * Extracts the value from a llama-server argument
+   * @param arg The argument
+   * @returns The value
+   */
+  private extractFrom(arg: string): number | null {
+    const args = this.model.status.args;
+    if (!args) return null;
+
+    const ctxIdx = args.indexOf(arg);
+
+    if (ctxIdx === -1) return null;
+    if (args.length <= ctxIdx + 1) return null;
+
+    const parsed = parseInt(args[ctxIdx + 1], 10);
+    if (!isNaN(parsed)) return parsed;
+
+    return null;
+  }
+}

package/src/models/singleModel.ts

@@ -0,0 +1,36 @@
+import { ISingleModel } from "../interfaces/ISingleModel";
+import { DEFAULT_CTX } from "../constants";
+import { rpc } from "../tools/retriever";
+import { Status } from "../enums/status";
+import { BaseModel } from "./baseModel";
+
+export class SingleModel extends BaseModel {
+  constructor(private readonly model: ISingleModel) {
+    super();
+  }
+
+  get id(): string {
+    return this.model.name;
+  }
+
+  get name(): string {
+    return this.model.name;
+  }
+
+  get capabilities(): ["text"] | ["image"] {
+    const hasImage = this.model.capabilities.includes("multimodal");
+    return hasImage ? ["image"] : ["text"];
+  }
+
+  async getStatus(): Promise<Status> {
+    // In single-mode, the extension will only work when the model is fully loaded
+    return Status.LOADED;
+  }
+
+  async getContextSize(): Promise<number> {
+    const slots = await rpc<{ n_ctx: number }[]>("/slots");
+    const [{ n_ctx }] = slots;
+
+    return n_ctx ?? DEFAULT_CTX;
+  }
+}

package/src/tools/resolver.ts

@@ -0,0 +1,138 @@
+import { DEFAULT_LLAMA_SERVER_URL, PROVIDER_NAME } from "../constants";
+import { access, readFile, constants } from "node:fs/promises";
+import { join } from "node:path";
+import { IAuthFile } from "../interfaces/IAuthFile";
+
+// The URL is detected once, to reuse forever
+let resolvedUrl: string | undefined;
+
+/**
+ * Detects if a particular file is present
+ * @param filePath The path
+ * @returns True if exists
+ */
+const fileExists = async (filePath: string): Promise<boolean> => {
+  try {
+    await access(filePath, constants.F_OK);
+    return true;
+  } catch (error) {
+    return false;
+  }
+};
+
+/**
+ * Reads the contents of a file as JSON
+ * @param filePath The path
+ * @returns The content as JSON
+ */
+const readContents = async <T>(filePath: string): Promise<T | null> => {
+  const raw = await readFile(filePath, "utf-8");
+
+  try {
+    const contents = JSON.parse(raw);
+    return contents;
+  } catch (err) {
+    return null;
+  }
+};
+
+/**
+ * Reads a string value from a JSON config file
+ * @param filePath Path to the JSON config file
+ * @param key Key to extract from the parsed JSON
+ * @returns The string value, or null if file/key missing or invalid
+ */
+const readConfigValue = async <T>(
+  filePath: string,
+  key: string,
+): Promise<string | null> => {
+  const cfg = await readContents<T>(filePath);
+  return (cfg as Record<string, any>)?.[key] || null;
+};
+
+/**
+ * Reads API key from Pi's auth file
+ * @returns The API key, as defined by the auth.json file
+ */
+export const resolveApiKey = async (): Promise<string> => {
+  const placeholder = "sk-placeholder";
+
+  const authPath = join(process.env.HOME || ".", ".pi", "agent", "auth.json");
+  if (!(await fileExists(authPath))) return placeholder;
+
+  const response = await readConfigValue<IAuthFile>(authPath, PROVIDER_NAME);
+  return response ?? placeholder;
+};
+
+/**
+ * Resolves the llama-server url by searching for it in the global settings.json file
+ * @returns The URL, if found.
+ */
+const resolveGlobalUrl = async (): Promise<string | null> => {
+  const globalPath = join(
+    process.env.HOME || ".",
+    ".pi",
+    "agent",
+    "settings.json",
+  );
+
+  if (!(await fileExists(globalPath))) return null;
+  return readConfigValue<Record<string, string>>(globalPath, "llamaServerUrl");
+};
+
+/**
+ * Resolves the llama-server url by searching for it in the project's .pi/llama-server.json file
+ * @param cwd The current working directory
+ * @returns The URL, if found.
+ */
+const resolveProjectUrl = async (cwd: string): Promise<string | null> => {
+  const projectPath = join(cwd, ".pi", "llama-server.json");
+
+  if (!(await fileExists(projectPath))) return null;
+  return readConfigValue<Record<string, string>>(projectPath, "url");
+};
+
+/**
+ * Resolves the llama-server url by searching for it in the environment
+ * @returns The URL, if found.
+ */
+const resolveEnvUrl = async (): Promise<string | null> => {
+  return process.env.LLAMA_SERVER_URL ?? null;
+};
+
+/**
+ * Tries all possible ways to retrieve the llama-server URL
+ * @param cwd The current working directory
+ * @returns The URL, or a default if not found
+ */
+const resolveUrlWithFallbacks = async (cwd: string): Promise<string> => {
+  // 1. per-project config
+  let response = await resolveProjectUrl(cwd);
+  if (response) return response;
+
+  // 2. env
+  response = await resolveEnvUrl();
+  if (response) return response;
+
+  // 3. global settings: ~/.pi/agent/settings.json
+  response = await resolveGlobalUrl();
+  if (response) return response;
+
+  // 4. default
+  return DEFAULT_LLAMA_SERVER_URL;
+};
+
+/**
+ * Resolves the URL where llama-server is running
+ * @param cwd The current working directory
+ * @returns The URL, or a default if not found
+ */
+export const resolveUrl = async (cwd: string): Promise<string> => {
+  if (resolvedUrl) return resolvedUrl;
+  const result = await resolveUrlWithFallbacks(cwd);
+
+  // Strip trailing slashes
+  resolvedUrl = result.replace(/\/+$/, "");
+
+  return resolvedUrl;
+};

package/src/tools/retriever.ts

@@ -0,0 +1,76 @@
+import { ISingleModel } from "../interfaces/ISingleModel";
+import { IRouterModel } from "../interfaces/IRouterModel";
+import { SingleModel } from "../models/singleModel";
+import { RouterModel } from "../models/routerModel";
+import { BaseModel } from "../models/baseModel";
+import { resolveApiKey, resolveUrl } from "./resolver";
+
+/**
+ * Detects if the server is ready
+ * @returns True if it's ready to work
+ */
+export const isServerReady = async (): Promise<boolean> => {
+  try {
+    const { status } = await rpc<{ status: string }>("/health");
+    return status === "ok";
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * Extracts the data of a fetch command
+ * @param endpoint The endpoint to fetch from
+ * @param body The body (optional)
+ * @returns Data from the fetch command
+ */
+export const rpc = async <T>(
+  endpoint: string,
+  body?: Record<string, unknown>,
+) => {
+  const base = await resolveUrl(process.cwd());
+  const url = `${base}${endpoint}`;
+
+  const data = {
+    method: body ? "POST" : "GET",
+    headers: body ? { "Content-Type": "application/json" } : undefined,
+    body: body ? JSON.stringify(body) : undefined,
+  };
+
+  const apiKey = await resolveApiKey();
+  const res = await fetch(url, {
+    ...data,
+    headers: {
+      ...data.headers,
+      ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
+    },
+  });
+
+  if (!res.ok) {
+    const text = await res.text();
+    throw new Error(`${res.status}: ${text}`);
+  }
+  return res.json() as T;
+};
+
+/**
+ * Retrieves a list of available models from llama-server
+ * @param base Base URL to use
+ * @returns The list of models
+ */
+export const listModels = async (): Promise<BaseModel[]> => {
+  const { models, data } = await rpc<{
+    models?: ISingleModel[];
+    data: IRouterModel[];
+  }>("/models");
+
+  if (models) {
+    return models.map((m) => new SingleModel(m));
+  }
+
+  const response = data
+    .map((m) => new RouterModel(m))
+    .sort((a, b) => (a.id > b.id ? 1 : a.id === b.id ? 0 : -1));
+
+  return response;
+};