structformatter 0.1.2

package/dist/server.js

@@ -0,0 +1,218 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createServer = createServer;
+const fastify_1 = __importDefault(require("fastify"));
+const node_crypto_1 = require("node:crypto");
+const node_stream_1 = require("node:stream");
+const engine_1 = require("./enforce/engine");
+const providers_1 = require("./providers");
+const cache_1 = require("./validate/cache");
+const sse_1 = require("./stream/sse");
+function isRawCapableAdapter(v) {
+    return typeof v.chatCompletionsRaw === 'function';
+}
+function setProxyHeaders(reply, headers) {
+    const skip = new Set(['content-length', 'transfer-encoding', 'connection', 'keep-alive']);
+    headers.forEach((value, key) => {
+        if (skip.has(key.toLowerCase()))
+            return;
+        reply.header(key, value);
+    });
+}
+function toBoolHeader(v) {
+    if (typeof v !== 'string')
+        return false;
+    const s = v.trim().toLowerCase();
+    return s !== '' && s !== '0' && s !== 'false';
+}
+function parseMaxAttemptsHeader(v) {
+    if (typeof v !== 'string')
+        return null;
+    const n = Number(v);
+    if (!Number.isInteger(n))
+        return null;
+    return Math.max(1, Math.min(n, 10));
+}
+function openaiError(type, message, details) {
+    return { error: { type, message, details } };
+}
+function chatCompletionResponse(args) {
+    const base = {
+        id: `chatcmpl_${(0, node_crypto_1.randomUUID)().replace(/-/g, '')}`,
+        object: 'chat.completion',
+        created: Math.floor(Date.now() / 1000),
+        model: args.model,
+        choices: [
+            {
+                index: 0,
+                message: { role: 'assistant', content: args.content },
+                finish_reason: 'stop',
+            },
+        ],
+        usage: args.usage,
+    };
+    if (args.debug)
+        base.__debug = args.debug;
+    return base;
+}
+function createServer(config) {
+    const app = (0, fastify_1.default)({
+        logger: process.env.NODE_ENV === 'test'
+            ? false
+            : {
+                level: process.env.STRUCTFORMATTER_LOG_LEVEL ??
+                    process.env.STRUCTUREDFORMATTER_LOG_LEVEL ??
+                    'info',
+            },
+        bodyLimit: config.server.request_body_limit_mb * 1024 * 1024,
+        genReqId: () => (0, node_crypto_1.randomUUID)(),
+    });
+    const registry = (0, providers_1.createProviderRegistry)(config);
+    const validatorCache = new cache_1.ValidatorCache({ maxSize: config.enforcement.validator_cache_size });
+    app.get('/healthz', async () => ({ ok: true }));
+    app.get('/v1/models', async () => ({
+        object: 'list',
+        data: (0, providers_1.listModelIds)(config).map((id) => ({ id, object: 'model' })),
+    }));
+    app.post('/v1/chat/completions', async (req, reply) => {
+        const body = req.body;
+        if (!body || typeof body !== 'object') {
+            return reply.code(400).send(openaiError('invalid_request', 'request body must be a JSON object'));
+        }
+        const request = body;
+        if (typeof request.model !== 'string' || !request.model) {
+            return reply.code(400).send(openaiError('invalid_request', 'missing model'));
+        }
+        if (!Array.isArray(request.messages)) {
+            return reply.code(400).send(openaiError('invalid_request', 'messages must be an array'));
+        }
+        const requestId = String(req.id);
+        const debug = toBoolHeader(req.headers['x-sf-debug']);
+        const maxAttemptsOverride = parseMaxAttemptsHeader(req.headers['x-sf-max-attempts']);
+        let resolved;
+        try {
+            resolved = (0, providers_1.resolveModel)(request.model, config);
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            return reply.code(400).send(openaiError('invalid_request', msg));
+        }
+        let adapter;
+        try {
+            adapter = (0, providers_1.getAdapter)(registry, resolved.provider);
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            return reply.code(400).send(openaiError('invalid_request', msg));
+        }
+        const rf = request.response_format;
+        if (rf && rf.type === 'json_schema') {
+            if (request.stream) {
+                return reply
+                    .code(400)
+                    .send(openaiError('invalid_request', 'streaming not supported for schema-enforced requests'));
+            }
+            const schema = rf.json_schema?.schema;
+            if (!schema || typeof schema !== 'object') {
+                return reply
+                    .code(400)
+                    .send(openaiError('invalid_request', 'response_format.json_schema.schema must be an object'));
+            }
+            const schemaBytes = Buffer.byteLength(JSON.stringify(schema), 'utf8');
+            if (schemaBytes > config.enforcement.schema_max_bytes) {
+                return reply.code(400).send(openaiError('invalid_request', 'schema too large'));
+            }
+            const policy = {
+                maxAttempts: maxAttemptsOverride ?? config.enforcement.max_attempts,
+                timeoutMsPerAttempt: config.enforcement.timeout_ms_per_attempt,
+                enableJsonRepair: config.enforcement.enable_jsonrepair,
+                enableDeterministicFix: config.enforcement.enable_deterministic_fix,
+                enableTypeCoercion: config.enforcement.enable_type_coercion,
+            };
+            const result = await (0, engine_1.enforceJsonSchema)({
+                requestId,
+                adapter,
+                provider: resolved.provider,
+                baseUrl: config.providers[resolved.provider]?.base_url ?? '',
+                upstreamModel: resolved.upstreamModel,
+                originalRequest: request,
+                schema: schema,
+                policy,
+                validatorCache,
+                debug,
+            });
+            if (!result.ok) {
+                const status = result.error.type === 'upstream_error' ? 502 : 422;
+                const payload = { error: result.error };
+                if (debug) {
+                    payload.__debug = { request_id: requestId, attempts: result.attempts, traces: result.attemptTraces };
+                }
+                return reply.code(status).send(payload);
+            }
+            return reply.code(200).send(chatCompletionResponse({
+                model: request.model,
+                content: result.jsonText,
+                usage: result.usage,
+                debug: debug ? { request_id: requestId, attempts: result.attempts, traces: result.attemptTraces } : undefined,
+            }));
+        }
+        const upstreamReq = { ...request, model: resolved.upstreamModel };
+        const ctx = {
+            requestId,
+            timeoutMs: config.enforcement.timeout_ms_per_attempt,
+            provider: resolved.provider,
+            baseUrl: config.providers[resolved.provider]?.base_url ?? '',
+            upstreamModel: resolved.upstreamModel,
+        };
+        if (request.stream) {
+            if (!isRawCapableAdapter(adapter)) {
+                return reply
+                    .code(400)
+                    .send(openaiError('invalid_request', 'streaming not supported for this provider'));
+            }
+            try {
+                const raw = await adapter.chatCompletionsRaw(upstreamReq, ctx);
+                setProxyHeaders(reply, raw.response.headers);
+                const contentType = raw.response.headers.get('content-type') ?? '';
+                const isEventStream = contentType.toLowerCase().includes('text/event-stream');
+                const body = raw.response.body;
+                if (!body) {
+                    raw.cleanup();
+                    return reply.code(200).send();
+                }
+                const nodeStream = node_stream_1.Readable.fromWeb(body);
+                nodeStream.on('end', raw.cleanup);
+                nodeStream.on('error', raw.cleanup);
+                reply.raw.on('close', () => {
+                    raw.abort();
+                    raw.cleanup();
+                });
+                reply.raw.on('error', () => {
+                    raw.abort();
+                    raw.cleanup();
+                });
+                const outStream = isEventStream
+                    ? nodeStream.pipe((0, sse_1.createSseModelRewriteTransform)(request.model))
+                    : nodeStream;
+                return reply.code(raw.response.status).send(outStream);
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                return reply.code(502).send(openaiError('upstream_error', msg));
+            }
+        }
+        try {
+            const upstreamResp = await adapter.chatCompletions(upstreamReq, ctx);
+            upstreamResp.model = request.model;
+            return reply.code(200).send(upstreamResp);
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            return reply.code(502).send(openaiError('upstream_error', msg));
+        }
+    });
+    return app;
+}

package/dist/stream/sse.d.ts

@@ -0,0 +1,2 @@
+import { Transform } from 'node:stream';
+export declare function createSseModelRewriteTransform(requestedModel: string): Transform;

package/dist/stream/sse.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSseModelRewriteTransform = createSseModelRewriteTransform;
+const node_stream_1 = require("node:stream");
+function rewriteEvent(eventText, requestedModel) {
+    const newline = eventText.includes('\r\n') ? '\r\n' : '\n';
+    const lines = eventText.split(/\r?\n/);
+    const out = lines.map((line) => {
+        if (!line.startsWith('data:'))
+            return line;
+        const data = line.slice('data:'.length).trimStart();
+        if (!data)
+            return line;
+        if (data === '[DONE]')
+            return 'data: [DONE]';
+        try {
+            const obj = JSON.parse(data);
+            if (obj && typeof obj === 'object') {
+                const rec = obj;
+                if (typeof rec.model === 'string')
+                    rec.model = requestedModel;
+            }
+            return `data: ${JSON.stringify(obj)}`;
+        }
+        catch {
+            return line;
+        }
+    });
+    return out.join(newline);
+}
+function createSseModelRewriteTransform(requestedModel) {
+    let buffer = '';
+    return new node_stream_1.Transform({
+        transform(chunk, _enc, cb) {
+            buffer += chunk.toString('utf8');
+            while (true) {
+                const lf = buffer.indexOf('\n\n');
+                const crlf = buffer.indexOf('\r\n\r\n');
+                const hasLf = lf !== -1;
+                const hasCrlf = crlf !== -1;
+                if (!hasLf && !hasCrlf)
+                    break;
+                const idx = hasLf && hasCrlf ? Math.min(lf, crlf) : hasLf ? lf : crlf;
+                const sep = idx === crlf ? '\r\n\r\n' : '\n\n';
+                const eventText = buffer.slice(0, idx);
+                buffer = buffer.slice(idx + sep.length);
+                this.push(rewriteEvent(eventText, requestedModel) + sep);
+            }
+            cb();
+        },
+        flush(cb) {
+            if (buffer)
+                this.push(buffer);
+            cb();
+        },
+    });
+}

package/dist/types/internal.d.ts

@@ -0,0 +1,53 @@
+import type { OpenAIChatCompletionsRequest, OpenAIChatCompletionsResponse } from './openai';
+export interface ProviderCapabilities {
+    json_object?: boolean;
+    tools?: boolean;
+    strict_tools?: boolean;
+}
+export interface RequestContext {
+    requestId: string;
+    timeoutMs: number;
+    provider: string;
+    baseUrl: string;
+    upstreamModel: string;
+}
+export interface ProviderAdapter {
+    name: string;
+    supports: ProviderCapabilities;
+    chatCompletions(req: OpenAIChatCompletionsRequest, ctx: RequestContext): Promise<OpenAIChatCompletionsResponse>;
+}
+export interface EnforcementPolicy {
+    maxAttempts: number;
+    timeoutMsPerAttempt: number;
+    enableJsonRepair: boolean;
+    enableDeterministicFix: boolean;
+    enableTypeCoercion: boolean;
+}
+export interface StructuredResult {
+    ok: true;
+    json: unknown;
+    jsonText: string;
+    usage: {
+        prompt_tokens: number;
+        completion_tokens: number;
+        total_tokens: number;
+    };
+    attempts: number;
+    attemptTraces?: unknown[];
+}
+export interface StructuredError {
+    ok: false;
+    error: {
+        type: string;
+        message: string;
+        details?: Record<string, unknown>;
+    };
+    attempts: number;
+    attemptTraces?: unknown[];
+}
+export declare class StructFormatterError extends Error {
+    readonly type: string;
+    readonly statusCode: number;
+    readonly details?: Record<string, unknown>;
+    constructor(type: string, message: string, statusCode: number, details?: Record<string, unknown>);
+}

package/dist/types/internal.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StructFormatterError = void 0;
+class StructFormatterError extends Error {
+    type;
+    statusCode;
+    details;
+    constructor(type, message, statusCode, details) {
+        super(message);
+        this.type = type;
+        this.statusCode = statusCode;
+        this.details = details;
+    }
+}
+exports.StructFormatterError = StructFormatterError;

package/dist/types/openai.d.ts

@@ -0,0 +1,59 @@
+export type OpenAIRole = 'system' | 'user' | 'assistant' | 'tool';
+export interface OpenAIChatMessage {
+    role: OpenAIRole;
+    content?: string | null;
+    name?: string;
+    tool_call_id?: string;
+    tool_calls?: unknown;
+}
+export interface OpenAIResponseFormatJsonSchema {
+    type: 'json_schema';
+    json_schema: {
+        name: string;
+        strict?: boolean;
+        schema: Record<string, unknown>;
+    };
+}
+export interface OpenAIResponseFormatJsonObject {
+    type: 'json_object';
+}
+export type OpenAIResponseFormat = OpenAIResponseFormatJsonSchema | OpenAIResponseFormatJsonObject | {
+    type: string;
+    [k: string]: unknown;
+};
+export interface OpenAIChatCompletionsRequest {
+    model: string;
+    messages: OpenAIChatMessage[];
+    temperature?: number;
+    top_p?: number;
+    max_tokens?: number;
+    stream?: boolean;
+    response_format?: OpenAIResponseFormat;
+    tools?: unknown;
+    tool_choice?: unknown;
+    [k: string]: unknown;
+}
+export interface OpenAIUsage {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+    total_tokens?: number;
+    [k: string]: unknown;
+}
+export interface OpenAIChatCompletionsChoice {
+    index: number;
+    message: {
+        role: 'assistant';
+        content: string | null;
+        tool_calls?: unknown;
+    };
+    finish_reason: string | null;
+}
+export interface OpenAIChatCompletionsResponse {
+    id: string;
+    object: 'chat.completion';
+    created: number;
+    model: string;
+    choices: OpenAIChatCompletionsChoice[];
+    usage?: OpenAIUsage;
+    [k: string]: unknown;
+}

package/dist/types/openai.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/validate/ajv.d.ts

@@ -0,0 +1,5 @@
+import Ajv2019 from 'ajv/dist/2019';
+import type { AnySchemaObject } from 'ajv';
+export type AjvDraft = '2019-09' | '2020-12';
+export declare function createAjv(draft?: AjvDraft): Ajv2019;
+export type JsonSchema = AnySchemaObject;

package/dist/validate/ajv.js

@@ -0,0 +1,18 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createAjv = createAjv;
+const _2019_1 = __importDefault(require("ajv/dist/2019"));
+const _2020_1 = __importDefault(require("ajv/dist/2020"));
+function createAjv(draft = '2020-12') {
+    const AjvCtor = draft === '2019-09' ? _2019_1.default : _2020_1.default;
+    return new AjvCtor({
+        allErrors: true,
+        strict: false,
+        allowUnionTypes: true,
+        validateSchema: true,
+        messages: true,
+    });
+}

package/dist/validate/cache.d.ts

@@ -0,0 +1,11 @@
+import type { ValidateFunction } from 'ajv';
+import { type AjvDraft } from './ajv';
+export declare class ValidatorCache {
+    private readonly cache;
+    private readonly ajv;
+    constructor(opts: {
+        maxSize: number;
+        draft?: AjvDraft;
+    });
+    get(schema: unknown): ValidateFunction;
+}

package/dist/validate/cache.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValidatorCache = void 0;
+const node_crypto_1 = require("node:crypto");
+const ajv_1 = require("./ajv");
+function stableStringify(value) {
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value);
+    if (Array.isArray(value))
+        return `[${value.map(stableStringify).join(',')}]`;
+    const obj = value;
+    const keys = Object.keys(obj).sort();
+    return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`).join(',')}}`;
+}
+function schemaHash(schema) {
+    const s = stableStringify(schema);
+    return (0, node_crypto_1.createHash)('sha256').update(s).digest('hex');
+}
+class LruCache {
+    maxSize;
+    map = new Map();
+    constructor(maxSize) {
+        this.maxSize = maxSize;
+    }
+    get(key) {
+        const v = this.map.get(key);
+        if (v === undefined)
+            return undefined;
+        this.map.delete(key);
+        this.map.set(key, v);
+        return v;
+    }
+    set(key, value) {
+        if (this.map.has(key))
+            this.map.delete(key);
+        this.map.set(key, value);
+        while (this.map.size > this.maxSize) {
+            const oldest = this.map.keys().next().value;
+            if (!oldest)
+                break;
+            this.map.delete(oldest);
+        }
+    }
+}
+class ValidatorCache {
+    cache;
+    ajv;
+    constructor(opts) {
+        this.cache = new LruCache(opts.maxSize);
+        this.ajv = (0, ajv_1.createAjv)(opts.draft ?? '2020-12');
+    }
+    get(schema) {
+        const key = schemaHash(schema);
+        const existing = this.cache.get(key);
+        if (existing)
+            return existing;
+        const validate = this.ajv.compile(schema);
+        this.cache.set(key, validate);
+        return validate;
+    }
+}
+exports.ValidatorCache = ValidatorCache;

package/dist/validate/errors.d.ts

@@ -0,0 +1,8 @@
+import type { ErrorObject } from 'ajv';
+export interface ValidationErrorSummary {
+    path: string;
+    message: string;
+    keyword: string;
+    params?: unknown;
+}
+export declare function summarizeAjvErrors(errors: ErrorObject[] | null | undefined): ValidationErrorSummary[];

package/dist/validate/errors.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarizeAjvErrors = summarizeAjvErrors;
+function summarizeAjvErrors(errors) {
+    if (!errors?.length)
+        return [];
+    return errors.slice(0, 50).map((e) => ({
+        path: e.instancePath || '/',
+        message: e.message ?? 'validation error',
+        keyword: e.keyword,
+        params: e.params,
+    }));
+}

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "structformatter",
+  "version": "0.1.2",
+  "description": "OpenAI-compatible proxy that enforces JSON Schema structured outputs for upstream LLM APIs that do not support native constrained decoding.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/FrankQDWang/StructFormatter.git"
+  },
+  "homepage": "https://github.com/FrankQDWang/StructFormatter#readme",
+  "bugs": {
+    "url": "https://github.com/FrankQDWang/StructFormatter/issues"
+  },
+  "keywords": [
+    "openai",
+    "proxy",
+    "structured-outputs",
+    "json-schema",
+    "ajv",
+    "fastify"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "bin": {
+    "structformatter": "dist/cli.js"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "config.example.yaml",
+    "CHANGELOG.md",
+    "README.md",
+    "README.zh-CN.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "dev": "tsx watch src/cli.ts",
+    "build": "tsc -p tsconfig.build.json",
+    "start": "node dist/cli.js",
+    "lint": "eslint .",
+    "format": "prettier -w .",
+    "prepack": "npm run build",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "ajv": "^8.17.1",
+    "fastify": "^5.6.2",
+    "jsonrepair": "^3.13.1",
+    "undici": "^7.16.0",
+    "yaml": "^2.8.2",
+    "zod": "^4.2.1"
+  },
+  "devDependencies": {
+    "@types/node": "^25.0.3",
+    "@typescript-eslint/eslint-plugin": "^8.50.1",
+    "@typescript-eslint/parser": "^8.50.1",
+    "eslint": "^9.39.2",
+    "eslint-config-prettier": "^10.1.8",
+    "prettier": "^3.7.4",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.16"
+  }
+}