scrapex 1.0.0-alpha.1 → 1.0.0-beta.1

package/dist/llm/index.cjs

@@ -1,70 +1,99 @@
-const require_enhancer = require('../enhancer-oM4BhYYS.cjs');
+const require_parsers = require('../parsers-Bneuws8x.cjs');
+const require_enhancer = require('../enhancer-j0xqKDJm.cjs');
+let zod = require("zod");
 
-//#region src/llm/anthropic.ts
-const DEFAULT_MODEL$1 = "claude-3-5-haiku-20241022";
-const DEFAULT_MAX_TOKENS$1 = 1024;
+//#region src/llm/http.ts
 /**
-* Anthropic Claude provider
-*
-* Requires @anthropic-ai/sdk as a peer dependency.
-*
-* @example
-* ```ts
-* const provider = new AnthropicProvider({ apiKey: 'sk-...' });
-* const result = await scrape(url, { llm: provider, enhance: ['summarize'] });
-* ```
+* HTTP-based LLM Provider using native fetch.
+* Provides a unified interface for any REST-based LLM API.
 */
-var AnthropicProvider = class {
-	name = "anthropic";
-	client;
-	model;
-	constructor(config = {}) {
-		const apiKey = config.apiKey ?? process.env.ANTHROPIC_API_KEY;
-		if (!apiKey) throw new require_enhancer.ScrapeError("Anthropic API key required. Set ANTHROPIC_API_KEY env var or pass apiKey in config.", "LLM_ERROR");
-		this.model = config.model ?? DEFAULT_MODEL$1;
-		try {
-			const { Anthropic } = require("@anthropic-ai/sdk");
-			this.client = new Anthropic({
-				apiKey,
-				baseURL: config.baseUrl
+/**
+* HTTP-based LLM provider.
+* Works with any REST API using native fetch.
+*/
+var HttpLLMProvider = class extends require_enhancer.BaseHttpProvider {
+	name;
+	requestBuilder;
+	responseMapper;
+	jsonMode;
+	constructor(config) {
+		super(config);
+		this.name = "http-llm";
+		this.jsonMode = config.jsonMode ?? false;
+		this.requestBuilder = config.requestBuilder ?? ((prompt, opts) => {
+			const messages = [];
+			if (opts.systemPrompt) messages.push({
+				role: "system",
+				content: opts.systemPrompt
 			});
-		} catch {
-			throw new require_enhancer.ScrapeError("@anthropic-ai/sdk is required for Anthropic provider. Install with: npm install @anthropic-ai/sdk", "LLM_ERROR");
-		}
+			messages.push({
+				role: "user",
+				content: prompt
+			});
+			const request = {
+				model: this.model,
+				messages
+			};
+			if (opts.temperature !== void 0) request.temperature = opts.temperature;
+			if (opts.maxTokens !== void 0) request.max_tokens = opts.maxTokens;
+			return request;
+		});
+		this.responseMapper = config.responseMapper ?? ((response) => {
+			const resp = response;
+			if (Array.isArray(resp.choices) && resp.choices.length > 0) {
+				const choice = resp.choices[0];
+				if (choice.message?.content) return choice.message.content;
+			}
+			if (Array.isArray(resp.content)) {
+				const textBlock = resp.content.find((c) => c.type === "text");
+				if (textBlock?.text) return textBlock.text;
+			}
+			throw new require_enhancer.ScrapeError("Unable to parse LLM response. Provide a custom responseMapper.", "VALIDATION_ERROR");
+		});
 	}
+	/**
+	* Generate a text completion.
+	*/
 	async complete(prompt, options = {}) {
-		try {
-			const content = (await this.client.messages.create({
-				model: this.model,
-				max_tokens: options.maxTokens ?? DEFAULT_MAX_TOKENS$1,
-				messages: [{
-					role: "user",
-					content: prompt
-				}],
-				system: options.systemPrompt,
-				temperature: options.temperature
-			})).content[0];
-			if (content?.type === "text" && content.text) return content.text;
-			throw new require_enhancer.ScrapeError("Unexpected or empty response from Anthropic", "LLM_ERROR");
-		} catch (error) {
-			if (error instanceof require_enhancer.ScrapeError) throw error;
-			throw new require_enhancer.ScrapeError(`Anthropic API error: ${error instanceof Error ? error.message : String(error)}`, "LLM_ERROR", void 0, error instanceof Error ? error : void 0);
-		}
+		let body = this.requestBuilder(prompt, options);
+		if (this.jsonMode && typeof body === "object" && body !== null) body = {
+			...body,
+			response_format: { type: "json_object" }
+		};
+		const { data } = await this.fetch(this.baseUrl, { body });
+		const content = this.responseMapper(data);
+		if (!content) throw new require_enhancer.ScrapeError("Empty response from LLM", "LLM_ERROR");
+		return content;
 	}
+	/**
+	* Generate a structured JSON completion with Zod validation.
+	*/
 	async completeJSON(prompt, schema, options = {}) {
 		const jsonPrompt = `${prompt}
 
 Respond ONLY with valid JSON matching this schema:
-${JSON.stringify(zodToJsonSchema$1(schema), null, 2)}
+${JSON.stringify(zodToJsonSchema(schema), null, 2)}
 
 Do not include any explanation or markdown formatting. Just the JSON object.`;
-		const response = await this.complete(jsonPrompt, {
+		const useJsonMode = this.jsonMode;
+		let body = this.requestBuilder(jsonPrompt, {
 			...options,
 			systemPrompt: options.systemPrompt ?? "You are a helpful assistant that responds only with valid JSON."
 		});
+		if (useJsonMode && typeof body === "object" && body !== null) body = {
+			...body,
+			response_format: { type: "json_object" }
+		};
+		const { data } = await this.fetch(this.baseUrl, { body });
+		const content = this.responseMapper(data);
+		if (!content) throw new require_enhancer.ScrapeError("Empty response from LLM", "LLM_ERROR");
 		try {
-			const jsonMatch = response.match(/\{[\s\S]*\}/);
-			if (!jsonMatch) throw new Error("No JSON object found in response");
+			const trimmed = content.trim();
+			try {
+				return schema.parse(JSON.parse(trimmed));
+			} catch {}
+			const jsonMatch = content.match(/[[{][\s\S]*[\]}]/);
+			if (!jsonMatch) throw new Error("No JSON found in response");
 			const parsed = JSON.parse(jsonMatch[0]);
 			return schema.parse(parsed);
 		} catch (error) {
@@ -73,244 +102,233 @@ Do not include any explanation or markdown formatting. Just the JSON object.`;
 	}
 };
 /**
-* Convert a Zod schema to a simple JSON Schema representation
-* (simplified version for prompt engineering)
+* Convert a Zod schema to a JSON Schema representation.
+* Uses Zod's built-in toJSONSchema method (Zod 4+).
+* Used for prompting LLMs to return structured data.
 */
-function zodToJsonSchema$1(schema) {
+function zodToJsonSchema(schema) {
+	if (typeof zod.z.toJSONSchema === "function") {
+		const { $schema, ...rest } = zod.z.toJSONSchema(schema);
+		return rest;
+	}
 	const def = schema._def;
-	switch (def.typeName) {
-		case "ZodObject": {
+	switch (def.type) {
+		case "object": {
 			const shape = schema.shape;
 			const properties = {};
-			for (const [key, value] of Object.entries(shape)) properties[key] = zodToJsonSchema$1(value);
+			const required = [];
+			for (const [key, value] of Object.entries(shape)) {
+				properties[key] = zodToJsonSchema(value);
+				if (value._def.type !== "optional") required.push(key);
+			}
 			return {
 				type: "object",
-				properties
+				properties,
+				required
 			};
 		}
-		case "ZodArray": return {
+		case "array": return {
 			type: "array",
-			items: zodToJsonSchema$1(def.type)
-		};
-		case "ZodString": return { type: "string" };
-		case "ZodNumber": return { type: "number" };
-		case "ZodBoolean": return { type: "boolean" };
-		case "ZodEnum": return {
-			type: "string",
-			enum: def.values
+			items: zodToJsonSchema(def.element)
 		};
+		case "string": return { type: "string" };
+		case "number": return { type: "number" };
+		case "boolean": return { type: "boolean" };
+		case "enum": {
+			const enumDef = def;
+			return {
+				type: "string",
+				enum: Object.values(enumDef.entries)
+			};
+		}
+		case "optional": return zodToJsonSchema(def.innerType);
 		default: return { type: "string" };
 	}
 }
 
 //#endregion
-//#region src/llm/openai.ts
-const DEFAULT_MODEL = "gpt-4o-mini";
-const DEFAULT_MAX_TOKENS = 1024;
-const DEFAULT_BASE_URL = "https://api.openai.com/v1";
+//#region src/llm/presets.ts
 /**
-* OpenAI-compatible provider
-*
-* Works with:
-* - OpenAI API
-* - Ollama (http://localhost:11434/v1)
-* - LM Studio (http://localhost:1234/v1)
-* - LocalAI
-* - vLLM
-* - Any OpenAI-compatible API
-*
-* Requires `openai` as a peer dependency.
+* Preset factory functions for common LLM providers.
+* All presets use the HttpLLMProvider with appropriate configuration.
+*/
+/**
+* Create an OpenAI LLM provider.
 *
 * @example
 * ```ts
-* // OpenAI
-* const provider = new OpenAIProvider({ apiKey: 'sk-...' });
-*
-* // Ollama
-* const provider = new OpenAIProvider({
-*   baseUrl: 'http://localhost:11434/v1',
-*   model: 'llama3.2',
-*   apiKey: 'ollama' // Ollama doesn't require a real key
-* });
+* const provider = createOpenAI({ apiKey: 'sk-...' });
+* const result = await scrape(url, { llm: provider, enhance: ['summarize'] });
+* ```
+*/
+function createOpenAI(options) {
+	const apiKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
+	if (!apiKey) throw new Error("OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey option.");
+	return new HttpLLMProvider({
+		baseUrl: options?.baseUrl ?? "https://api.openai.com/v1/chat/completions",
+		model: options?.model ?? "gpt-4o-mini",
+		headers: { Authorization: `Bearer ${apiKey}` },
+		jsonMode: true
+	});
+}
+/**
+* Create an Anthropic Claude LLM provider.
 *
-* // LM Studio
-* const provider = new OpenAIProvider({
-*   baseUrl: 'http://localhost:1234/v1',
-*   model: 'local-model',
-*   apiKey: 'lm-studio'
-* });
+* @example
+* ```ts
+* const provider = createAnthropic({ apiKey: 'sk-...' });
+* const result = await scrape(url, { llm: provider, enhance: ['summarize'] });
 * ```
 */
-var OpenAIProvider = class {
-	name = "openai";
-	client;
-	model;
-	constructor(config = {}) {
-		const apiKey = config.apiKey ?? process.env.OPENAI_API_KEY;
-		const baseUrl = config.baseUrl ?? DEFAULT_BASE_URL;
-		if (!apiKey && baseUrl === DEFAULT_BASE_URL) throw new require_enhancer.ScrapeError("OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey in config.", "LLM_ERROR");
-		this.model = config.model ?? DEFAULT_MODEL;
-		try {
-			const { OpenAI } = require("openai");
-			this.client = new OpenAI({
-				apiKey: apiKey ?? "local",
-				baseURL: baseUrl
-			});
-		} catch {
-			throw new require_enhancer.ScrapeError("openai package is required for OpenAI provider. Install with: npm install openai", "LLM_ERROR");
-		}
-	}
-	async complete(prompt, options = {}) {
-		try {
-			const client = this.client;
-			const messages = [];
-			if (options.systemPrompt) messages.push({
-				role: "system",
-				content: options.systemPrompt
-			});
-			messages.push({
-				role: "user",
-				content: prompt
-			});
-			const content = (await client.chat.completions.create({
-				model: this.model,
-				max_tokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
-				messages,
-				temperature: options.temperature
-			})).choices[0]?.message?.content;
-			if (!content) throw new require_enhancer.ScrapeError("Empty response from OpenAI", "LLM_ERROR");
-			return content;
-		} catch (error) {
-			if (error instanceof require_enhancer.ScrapeError) throw error;
-			throw new require_enhancer.ScrapeError(`OpenAI API error: ${error instanceof Error ? error.message : String(error)}`, "LLM_ERROR", void 0, error instanceof Error ? error : void 0);
-		}
-	}
-	async completeJSON(prompt, schema, options = {}) {
-		const client = this.client;
-		try {
-			const messages = [{
-				role: "system",
-				content: options.systemPrompt ?? "You are a helpful assistant that extracts information from content."
-			}, {
+function createAnthropic(options) {
+	const apiKey = options?.apiKey ?? process.env.ANTHROPIC_API_KEY;
+	if (!apiKey) throw new Error("Anthropic API key required. Set ANTHROPIC_API_KEY env var or pass apiKey option.");
+	const model = options?.model ?? "claude-3-5-haiku-20241022";
+	return new HttpLLMProvider({
+		baseUrl: "https://api.anthropic.com/v1/messages",
+		model,
+		headers: {
+			"x-api-key": apiKey,
+			"anthropic-version": "2023-06-01"
+		},
+		requestBuilder: (prompt, opts) => ({
+			model,
+			max_tokens: opts.maxTokens ?? 1024,
+			messages: [{
 				role: "user",
 				content: prompt
-			}];
-			const content = (await client.chat.completions.create({
-				model: this.model,
-				max_tokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
-				messages,
-				temperature: options.temperature,
-				response_format: { type: "json_object" }
-			})).choices[0]?.message?.content;
-			if (!content) throw new require_enhancer.ScrapeError("Empty response from OpenAI", "LLM_ERROR");
-			const parsed = JSON.parse(content);
-			return schema.parse(parsed);
-		} catch (error) {
-			if (error instanceof require_enhancer.ScrapeError) throw error;
-			const jsonPrompt = `${prompt}
-
-Respond ONLY with valid JSON matching this schema:
-${JSON.stringify(zodToJsonSchema(schema), null, 2)}
-
-Do not include any explanation or markdown formatting. Just the JSON object.`;
-			const response = await this.complete(jsonPrompt, {
-				...options,
-				systemPrompt: "You respond only with valid JSON."
-			});
-			try {
-				const jsonMatch = response.match(/\{[\s\S]*\}/);
-				if (!jsonMatch) throw new Error("No JSON object found in response");
-				const parsed = JSON.parse(jsonMatch[0]);
-				return schema.parse(parsed);
-			} catch (parseError) {
-				throw new require_enhancer.ScrapeError(`Failed to parse LLM response as JSON: ${parseError instanceof Error ? parseError.message : String(parseError)}`, "VALIDATION_ERROR", void 0, parseError instanceof Error ? parseError : void 0);
-			}
-		}
-	}
-};
-/**
-* Convert a Zod schema to JSON Schema for structured outputs
-*/
-function zodToJsonSchema(schema) {
-	const def = schema._def;
-	switch (def.typeName) {
-		case "ZodObject": {
-			const shape = schema.shape;
-			const properties = {};
-			const required = [];
-			for (const [key, value] of Object.entries(shape)) {
-				properties[key] = zodToJsonSchema(value);
-				if (value._def.typeName !== "ZodOptional") required.push(key);
-			}
-			return {
-				type: "object",
-				properties,
-				required
-			};
-		}
-		case "ZodArray": return {
-			type: "array",
-			items: zodToJsonSchema(def.type)
-		};
-		case "ZodString": return { type: "string" };
-		case "ZodNumber": return { type: "number" };
-		case "ZodBoolean": return { type: "boolean" };
-		case "ZodEnum": return {
-			type: "string",
-			enum: def.values
-		};
-		case "ZodOptional": return zodToJsonSchema(def.innerType);
-		default: return { type: "string" };
-	}
+			}],
+			...opts.systemPrompt && { system: opts.systemPrompt },
+			...opts.temperature !== void 0 && { temperature: opts.temperature }
+		}),
+		responseMapper: (res) => res.content.find((item) => item.type === "text")?.text ?? ""
+	});
 }
 /**
-* Create an OpenAI provider with default settings
+* Create a Groq LLM provider.
+* Groq provides fast inference for open-source models.
+*
+* @example
+* ```ts
+* const provider = createGroq({ model: 'llama-3.1-70b-versatile' });
+* ```
 */
-function createOpenAI(config) {
-	return new OpenAIProvider(config);
+function createGroq(options) {
+	const apiKey = options?.apiKey ?? process.env.GROQ_API_KEY;
+	if (!apiKey) throw new Error("Groq API key required. Set GROQ_API_KEY env var or pass apiKey option.");
+	return new HttpLLMProvider({
+		baseUrl: "https://api.groq.com/openai/v1/chat/completions",
+		model: options?.model ?? "llama-3.1-70b-versatile",
+		headers: { Authorization: `Bearer ${apiKey}` },
+		jsonMode: true
+	});
 }
 /**
-* Create an Ollama provider
+* Create an Ollama LLM provider for local models.
 *
 * @example
 * ```ts
 * const provider = createOllama({ model: 'llama3.2' });
 * ```
 */
-function createOllama(config = { model: "llama3.2" }) {
-	return new OpenAIProvider({
-		baseUrl: `http://localhost:${config.port ?? 11434}/v1`,
-		model: config.model,
-		apiKey: "ollama"
+function createOllama(options) {
+	return new HttpLLMProvider({
+		baseUrl: options.baseUrl ?? "http://localhost:11434/v1/chat/completions",
+		model: options.model,
+		requireHttps: false,
+		allowPrivate: true
 	});
 }
 /**
-* Create an LM Studio provider
+* Create an LM Studio LLM provider for local models.
 *
 * @example
 * ```ts
 * const provider = createLMStudio({ model: 'local-model' });
 * ```
 */
-function createLMStudio(config = { model: "local-model" }) {
-	return new OpenAIProvider({
-		baseUrl: `http://localhost:${config.port ?? 1234}/v1`,
-		model: config.model,
-		apiKey: "lm-studio"
+function createLMStudio(options) {
+	return new HttpLLMProvider({
+		baseUrl: options.baseUrl ?? "http://localhost:1234/v1/chat/completions",
+		model: options.model,
+		requireHttps: false,
+		allowPrivate: true
+	});
+}
+/**
+* Create a Together AI LLM provider.
+*
+* @example
+* ```ts
+* const provider = createTogether({ model: 'meta-llama/Llama-3.2-3B-Instruct-Turbo' });
+* ```
+*/
+function createTogether(options) {
+	const apiKey = options?.apiKey ?? process.env.TOGETHER_API_KEY;
+	if (!apiKey) throw new Error("Together API key required. Set TOGETHER_API_KEY env var or pass apiKey option.");
+	return new HttpLLMProvider({
+		baseUrl: "https://api.together.xyz/v1/chat/completions",
+		model: options?.model ?? "meta-llama/Llama-3.2-3B-Instruct-Turbo",
+		headers: { Authorization: `Bearer ${apiKey}` },
+		jsonMode: true
+	});
+}
+/**
+* Create an OpenRouter LLM provider.
+* OpenRouter provides access to many models through a unified API.
+*
+* @example
+* ```ts
+* const provider = createOpenRouter({
+*   model: 'anthropic/claude-3.5-sonnet',
+* });
+* ```
+*/
+function createOpenRouter(options) {
+	const apiKey = options.apiKey ?? process.env.OPENROUTER_API_KEY;
+	if (!apiKey) throw new Error("OpenRouter API key required. Set OPENROUTER_API_KEY env var or pass apiKey option.");
+	const headers = { Authorization: `Bearer ${apiKey}` };
+	if (options.siteUrl) headers["HTTP-Referer"] = options.siteUrl;
+	if (options.siteName) headers["X-Title"] = options.siteName;
+	return new HttpLLMProvider({
+		baseUrl: "https://openrouter.ai/api/v1/chat/completions",
+		model: options.model,
+		headers
 	});
 }
+/**
+* Create a generic HTTP LLM provider.
+* Use this for any OpenAI-compatible API.
+*
+* @example
+* ```ts
+* const provider = createHttpLLM({
+*   baseUrl: 'https://my-api.com/v1/chat/completions',
+*   model: 'my-model',
+*   headers: { Authorization: 'Bearer ...' },
+* });
+* ```
+*/
+function createHttpLLM(config) {
+	return new HttpLLMProvider(config);
+}
 
 //#endregion
-exports.AnthropicProvider = AnthropicProvider;
 exports.ClassifySchema = require_enhancer.ClassifySchema;
 exports.EntitiesSchema = require_enhancer.EntitiesSchema;
-exports.OpenAIProvider = OpenAIProvider;
+exports.HttpLLMProvider = HttpLLMProvider;
 exports.SummarySchema = require_enhancer.SummarySchema;
 exports.TagsSchema = require_enhancer.TagsSchema;
 exports.ask = require_enhancer.ask;
+exports.createAnthropic = createAnthropic;
+exports.createGroq = createGroq;
+exports.createHttpLLM = createHttpLLM;
 exports.createLMStudio = createLMStudio;
 exports.createOllama = createOllama;
 exports.createOpenAI = createOpenAI;
+exports.createOpenRouter = createOpenRouter;
+exports.createTogether = createTogether;
 exports.enhance = require_enhancer.enhance;
 exports.extract = require_enhancer.extract;
+exports.zodToJsonSchema = zodToJsonSchema;
 //# sourceMappingURL=index.cjs.map