threadshelf 1.2.0

package/dist/src/generation/filesystem-browser.js

@@ -0,0 +1,83 @@
+import { existsSync } from 'fs';
+import { readdir, stat } from 'fs/promises';
+import { homedir, platform } from 'os';
+import { dirname, parse, resolve } from 'path';
+import { ValidationError } from '../validation.js';
+const MAX_DIRECTORIES = 500;
+export const isLoopbackAddress = (address) => {
+    let normalized = String(address || '')
+        .trim()
+        .replace(/^"|"$/g, '')
+        .toLowerCase();
+    if (normalized.startsWith('['))
+        normalized = normalized.slice(1, normalized.indexOf(']'));
+    else
+        normalized = normalized.replace(/^(\d{1,3}(?:\.\d{1,3}){3}):\d+$/, '$1');
+    return normalized === '127.0.0.1' || normalized === '::1' || normalized === '::ffff:127.0.0.1';
+};
+const forwardedAddresses = (value) => (Array.isArray(value) ? value : [value ?? ''])
+    .flatMap((header) => header.split(','))
+    .map((entry) => entry.trim())
+    .filter(Boolean);
+export const isLoopbackRequest = (remoteAddress, forwardedFor, forwarded, requestHostname) => {
+    if (!isLoopbackAddress(remoteAddress))
+        return false;
+    if (requestHostname !== undefined &&
+        requestHostname.toLowerCase() !== 'localhost' &&
+        !isLoopbackAddress(requestHostname)) {
+        return false;
+    }
+    const proxyAddresses = forwardedAddresses(forwardedFor);
+    const forwardedEntries = forwardedAddresses(forwarded);
+    for (const entry of forwardedEntries) {
+        const matches = [...entry.matchAll(/(?:^|;)\s*for=("?)(\[[^\]]+\]|[^;\s"]+)\1/gi)];
+        if (matches.length === 0)
+            return false;
+        proxyAddresses.push(...matches.map((match) => match[2]).filter((address) => Boolean(address)));
+    }
+    return proxyAddresses.every(isLoopbackAddress);
+};
+const filesystemRoots = () => {
+    const home = resolve(homedir());
+    if (platform() !== 'win32') {
+        return [
+            { name: 'Home', path: home },
+            { name: 'Filesystem', path: '/' },
+        ];
+    }
+    const drives = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+        .split('')
+        .map((letter) => `${letter}:\\`)
+        .filter((path) => existsSync(path))
+        .map((path) => ({ name: path.slice(0, 2), path }));
+    return [
+        { name: 'Home', path: home },
+        ...drives.filter((drive) => drive.path !== parse(home).root),
+    ];
+};
+export const browseDirectories = async (requestedPath) => {
+    if (requestedPath !== undefined &&
+        (typeof requestedPath !== 'string' ||
+            requestedPath.length > 4096 ||
+            requestedPath.includes('\0'))) {
+        throw new ValidationError('Invalid directory path', { field: 'path' });
+    }
+    const path = resolve(requestedPath?.trim() || homedir());
+    const info = await stat(path).catch(() => null);
+    if (!info?.isDirectory()) {
+        throw new ValidationError('Directory does not exist or is not accessible', { field: 'path' });
+    }
+    const entries = await readdir(path, { withFileTypes: true });
+    const directories = entries
+        .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink())
+        .map((entry) => ({ name: entry.name, path: resolve(path, entry.name) }))
+        .sort((a, b) => a.name.localeCompare(b.name, undefined, { sensitivity: 'base' }));
+    const parent = dirname(path);
+    return {
+        path,
+        parent: parent === path ? undefined : parent,
+        roots: filesystemRoots(),
+        directories: directories.slice(0, MAX_DIRECTORIES),
+        truncated: directories.length > MAX_DIRECTORIES,
+    };
+};

package/dist/src/generation/gguf-metadata.js

@@ -0,0 +1,179 @@
+import { open, stat } from 'fs/promises';
+const GGUF_MAGIC = 0x46554747; // "GGUF" read as little-endian uint32
+const CHUNK_BYTES = 1024 * 1024;
+// Tokenizer vocabularies dominate the header. Anything past this is not a sane model.
+const MAX_HEADER_BYTES = 512 * 1024 * 1024;
+const MAX_STRING_BYTES = 64 * 1024 * 1024;
+const MAX_KV_COUNT = 100_000;
+const TYPE_STRING = 8;
+const TYPE_ARRAY = 9;
+const SCALAR_BYTES = {
+    0: 1,
+    1: 1,
+    2: 2,
+    3: 2,
+    4: 4,
+    5: 4,
+    6: 4,
+    7: 1,
+    10: 8,
+    11: 8,
+    12: 8,
+};
+const NUMERIC_SUFFIXES = ['.context_length', '.block_count', '.nextn_predict_layers'];
+class HeaderReader {
+    handle;
+    buffer = Buffer.alloc(0);
+    bufferStart = 0;
+    position = 0;
+    limit;
+    constructor(handle, size) {
+        this.handle = handle;
+        this.limit = Math.min(size, MAX_HEADER_BYTES);
+    }
+    ensureWithinLimit(length) {
+        if (!Number.isSafeInteger(length) || length < 0 || this.position + length > this.limit) {
+            throw new Error('GGUF header is truncated or exceeds the metadata limit');
+        }
+    }
+    async bytes(length) {
+        this.ensureWithinLimit(length);
+        const offset = this.position - this.bufferStart;
+        if (offset < 0 || offset + length > this.buffer.length) {
+            const chunk = Buffer.alloc(Math.min(Math.max(length, CHUNK_BYTES), this.limit - this.position));
+            const { bytesRead } = await this.handle.read(chunk, 0, chunk.length, this.position);
+            if (bytesRead < length)
+                throw new Error('GGUF header is truncated');
+            this.buffer = chunk.subarray(0, bytesRead);
+            this.bufferStart = this.position;
+        }
+        const start = this.position - this.bufferStart;
+        this.position += length;
+        return this.buffer.subarray(start, start + length);
+    }
+    skip(length) {
+        this.ensureWithinLimit(length);
+        this.position += length;
+    }
+    async u32() {
+        return (await this.bytes(4)).readUInt32LE(0);
+    }
+    async u64() {
+        const value = (await this.bytes(8)).readBigUInt64LE(0);
+        if (value > BigInt(Number.MAX_SAFE_INTEGER))
+            throw new Error('GGUF length is out of range');
+        return Number(value);
+    }
+    async string() {
+        const length = await this.u64();
+        if (length > MAX_STRING_BYTES)
+            throw new Error('GGUF string is too long');
+        return (await this.bytes(length)).toString('utf8');
+    }
+}
+const readScalar = async (reader, type) => {
+    const value = await reader.bytes(SCALAR_BYTES[type]);
+    switch (type) {
+        case 0:
+            return value.readUInt8(0);
+        case 1:
+            return value.readInt8(0);
+        case 2:
+            return value.readUInt16LE(0);
+        case 3:
+            return value.readInt16LE(0);
+        case 4:
+            return value.readUInt32LE(0);
+        case 5:
+            return value.readInt32LE(0);
+        case 6:
+            return value.readFloatLE(0);
+        case 7:
+            return value.readUInt8(0) !== 0;
+        case 10:
+            return Number(value.readBigUInt64LE(0));
+        case 11:
+            return Number(value.readBigInt64LE(0));
+        default:
+            return value.readDoubleLE(0);
+    }
+};
+const parseGgufHeader = async (path) => {
+    const handle = await open(path, 'r');
+    try {
+        const reader = new HeaderReader(handle, (await handle.stat()).size);
+        if ((await reader.u32()) !== GGUF_MAGIC)
+            return null;
+        // Version 1 used 32-bit lengths; every current llama.cpp GGUF is v2 or v3.
+        if ((await reader.u32()) < 2)
+            return null;
+        await reader.u64(); // tensor count
+        const kvCount = await reader.u64();
+        if (kvCount > MAX_KV_COUNT)
+            return null;
+        const strings = new Map();
+        const numbers = new Map();
+        for (let index = 0; index < kvCount; index += 1) {
+            const key = await reader.string();
+            const type = await reader.u32();
+            if (type === TYPE_STRING) {
+                if (key === 'general.architecture' || key === 'general.name') {
+                    strings.set(key, await reader.string());
+                }
+                else {
+                    reader.skip(await reader.u64());
+                }
+            }
+            else if (type === TYPE_ARRAY) {
+                const elementType = await reader.u32();
+                const count = await reader.u64();
+                if (elementType === TYPE_STRING) {
+                    for (let element = 0; element < count; element += 1)
+                        reader.skip(await reader.u64());
+                }
+                else {
+                    const width = SCALAR_BYTES[elementType];
+                    if (!width)
+                        return null;
+                    reader.skip(width * count);
+                }
+            }
+            else if (SCALAR_BYTES[type]) {
+                const value = await readScalar(reader, type);
+                if (typeof value === 'number' && NUMERIC_SUFFIXES.some((suffix) => key.endsWith(suffix))) {
+                    numbers.set(key, value);
+                }
+            }
+            else {
+                return null;
+            }
+        }
+        // Keys are namespaced by architecture, which may appear anywhere in the header.
+        const architecture = strings.get('general.architecture');
+        const numeric = (suffix) => architecture ? numbers.get(`${architecture}${suffix}`) : undefined;
+        return {
+            architecture,
+            name: strings.get('general.name'),
+            contextLength: numeric('.context_length'),
+            blockCount: numeric('.block_count'),
+            nextnPredictLayers: Math.max(0, numeric('.nextn_predict_layers') ?? 0),
+        };
+    }
+    finally {
+        await handle.close();
+    }
+};
+const metadataCache = new Map();
+/** Reads GGUF metadata once per file version. Unreadable or non-GGUF files yield null. */
+export const readGgufMetadata = async (path) => {
+    const info = await stat(path).catch(() => null);
+    if (!info?.isFile())
+        return null;
+    const key = `${path}\0${info.size}\0${info.mtimeMs}`;
+    let pending = metadataCache.get(key);
+    if (!pending) {
+        pending = parseGgufHeader(path).catch(() => null);
+        metadataCache.set(key, pending);
+    }
+    return pending;
+};

package/dist/src/generation/hardware.js

@@ -0,0 +1,87 @@
+import { freemem, totalmem } from 'os';
+import { findLlamaExecutables } from './llama-install.js';
+import { inspectLlamaDevices } from './llama-process.js';
+import { runCommandCapture } from './llama-install.js';
+/**
+ * Weights are not the whole story: the KV cache, compute buffers and the OS all
+ * want memory too. Judging fit against the raw device size would recommend
+ * models that load and then immediately thrash.
+ */
+const VRAM_HEADROOM = 0.85;
+const RAM_HEADROOM = 0.6;
+export const judgeFit = (sizeBytes, profile) => {
+    const budget = profile.modelBudgetBytes;
+    if (budget <= 0)
+        return 'too-large';
+    if (sizeBytes <= budget)
+        return 'fits';
+    // Above the accelerator budget a model still runs, just partly on CPU.
+    if (sizeBytes <= profile.totalRamBytes * RAM_HEADROOM)
+        return 'tight';
+    return 'too-large';
+};
+const parseNvidiaSmi = (output) => output
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line, index) => {
+    const [name, total, free] = line.split(',').map((part) => part.trim());
+    const totalMiB = Number(total);
+    const freeMiB = Number(free);
+    if (!name || !Number.isFinite(totalMiB))
+        return null;
+    return {
+        id: `CUDA${index}`,
+        name,
+        totalBytes: totalMiB * 1024 ** 2,
+        freeBytes: (Number.isFinite(freeMiB) ? freeMiB : totalMiB) * 1024 ** 2,
+    };
+})
+    .filter((device) => device !== null);
+export const inspectHardware = async () => {
+    const totalRamBytes = totalmem();
+    const freeRamBytes = freemem();
+    let devices = [];
+    let detectionSource = 'none';
+    // An installed llama-server reports exactly the devices it would itself use,
+    // which beats guessing from vendor tools.
+    const executable = (await findLlamaExecutables())[0];
+    if (executable) {
+        const inspection = await inspectLlamaDevices(executable).catch(() => null);
+        if (inspection?.devices.length) {
+            devices = inspection.devices;
+            detectionSource = 'llama.cpp';
+        }
+    }
+    if (devices.length === 0) {
+        const output = await runCommandCapture('nvidia-smi', [
+            '--query-gpu=name,memory.total,memory.free',
+            '--format=csv,noheader,nounits',
+        ]).catch(() => '');
+        const parsed = parseNvidiaSmi(output);
+        if (parsed.length > 0) {
+            devices = parsed;
+            detectionSource = 'nvidia-smi';
+        }
+    }
+    const largest = devices.reduce((best, device) => Math.max(best, device.freeBytes || device.totalBytes), 0);
+    const vramBudgetBytes = Math.floor(largest * VRAM_HEADROOM);
+    const modelBudgetBytes = vramBudgetBytes > 0 ? vramBudgetBytes : Math.floor(totalRamBytes * RAM_HEADROOM);
+    const suggestedVariant = process.platform === 'darwin'
+        ? 'cpu'
+        : detectionSource === 'nvidia-smi' ||
+            devices.some((device) => /^cuda/i.test(device.id) || /nvidia|geforce|rtx|quadro|tesla/i.test(device.name))
+            ? 'cuda'
+            : devices.length > 0
+                ? 'vulkan'
+                : 'cpu';
+    return {
+        devices,
+        detectionSource,
+        totalRamBytes,
+        freeRamBytes,
+        vramBudgetBytes,
+        modelBudgetBytes,
+        suggestedVariant,
+    };
+};