scrapex 0.5.2 → 1.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +392 -145
- package/dist/enhancer-Q6CSc1gA.mjs +220 -0
- package/dist/enhancer-Q6CSc1gA.mjs.map +1 -0
- package/dist/enhancer-oM4BhYYS.cjs +268 -0
- package/dist/enhancer-oM4BhYYS.cjs.map +1 -0
- package/dist/index.cjs +852 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +264 -0
- package/dist/index.d.cts.map +1 -0
- package/dist/index.d.mts +264 -0
- package/dist/index.d.mts.map +1 -0
- package/dist/index.mjs +798 -0
- package/dist/index.mjs.map +1 -0
- package/dist/llm/index.cjs +316 -0
- package/dist/llm/index.cjs.map +1 -0
- package/dist/llm/index.d.cts +211 -0
- package/dist/llm/index.d.cts.map +1 -0
- package/dist/llm/index.d.mts +211 -0
- package/dist/llm/index.d.mts.map +1 -0
- package/dist/llm/index.mjs +310 -0
- package/dist/llm/index.mjs.map +1 -0
- package/dist/parsers/index.cjs +200 -0
- package/dist/parsers/index.cjs.map +1 -0
- package/dist/parsers/index.d.cts +133 -0
- package/dist/parsers/index.d.cts.map +1 -0
- package/dist/parsers/index.d.mts +133 -0
- package/dist/parsers/index.d.mts.map +1 -0
- package/dist/parsers/index.mjs +192 -0
- package/dist/parsers/index.mjs.map +1 -0
- package/dist/types-CNQZVW36.d.mts +150 -0
- package/dist/types-CNQZVW36.d.mts.map +1 -0
- package/dist/types-D0HYR95H.d.cts +150 -0
- package/dist/types-D0HYR95H.d.cts.map +1 -0
- package/package.json +80 -100
- package/dist/index.d.ts +0 -45
- package/dist/index.js +0 -8
- package/dist/scrapex.cjs.development.js +0 -1128
- package/dist/scrapex.cjs.development.js.map +0 -1
- package/dist/scrapex.cjs.production.min.js +0 -2
- package/dist/scrapex.cjs.production.min.js.map +0 -1
- package/dist/scrapex.esm.js +0 -1120
- package/dist/scrapex.esm.js.map +0 -1
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { h as ScrapedData, r as EnhancementType, s as ExtractionSchema } from "../types-D0HYR95H.cjs";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
//#region src/llm/types.d.ts
|
|
5
|
+
/**
|
|
6
|
+
* LLM completion options
|
|
7
|
+
*/
|
|
8
|
+
interface CompletionOptions {
|
|
9
|
+
maxTokens?: number;
|
|
10
|
+
temperature?: number;
|
|
11
|
+
systemPrompt?: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* LLM Provider interface - implemented by all providers
|
|
15
|
+
*/
|
|
16
|
+
interface LLMProvider {
|
|
17
|
+
readonly name: string;
|
|
18
|
+
/**
|
|
19
|
+
* Generate a text completion
|
|
20
|
+
*/
|
|
21
|
+
complete(prompt: string, options?: CompletionOptions): Promise<string>;
|
|
22
|
+
/**
|
|
23
|
+
* Generate a structured JSON completion with Zod validation
|
|
24
|
+
*/
|
|
25
|
+
completeJSON<T>(prompt: string, schema: z.ZodType<T>, options?: CompletionOptions): Promise<T>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Provider configuration for Anthropic
|
|
29
|
+
*/
|
|
30
|
+
interface AnthropicConfig {
|
|
31
|
+
apiKey?: string;
|
|
32
|
+
model?: string;
|
|
33
|
+
baseUrl?: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Provider configuration for OpenAI-compatible APIs
|
|
37
|
+
* Works with: OpenAI, Ollama, LM Studio, LocalAI, vLLM, etc.
|
|
38
|
+
*/
|
|
39
|
+
interface OpenAICompatibleConfig {
|
|
40
|
+
apiKey?: string;
|
|
41
|
+
model?: string;
|
|
42
|
+
baseUrl?: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Enhancement result types
|
|
46
|
+
*/
|
|
47
|
+
interface SummaryResult {
|
|
48
|
+
summary: string;
|
|
49
|
+
}
|
|
50
|
+
interface TagsResult {
|
|
51
|
+
tags: string[];
|
|
52
|
+
}
|
|
53
|
+
interface EntitiesResult {
|
|
54
|
+
people: string[];
|
|
55
|
+
organizations: string[];
|
|
56
|
+
technologies: string[];
|
|
57
|
+
locations: string[];
|
|
58
|
+
concepts: string[];
|
|
59
|
+
}
|
|
60
|
+
interface ClassifyResult {
|
|
61
|
+
contentType: string;
|
|
62
|
+
confidence: number;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Zod schemas for LLM outputs
|
|
66
|
+
*/
|
|
67
|
+
declare const SummarySchema: z.ZodObject<{
|
|
68
|
+
summary: z.ZodString;
|
|
69
|
+
}, z.core.$strip>;
|
|
70
|
+
declare const TagsSchema: z.ZodObject<{
|
|
71
|
+
tags: z.ZodArray<z.ZodString>;
|
|
72
|
+
}, z.core.$strip>;
|
|
73
|
+
declare const EntitiesSchema: z.ZodObject<{
|
|
74
|
+
people: z.ZodArray<z.ZodString>;
|
|
75
|
+
organizations: z.ZodArray<z.ZodString>;
|
|
76
|
+
technologies: z.ZodArray<z.ZodString>;
|
|
77
|
+
locations: z.ZodArray<z.ZodString>;
|
|
78
|
+
concepts: z.ZodArray<z.ZodString>;
|
|
79
|
+
}, z.core.$strip>;
|
|
80
|
+
declare const ClassifySchema: z.ZodObject<{
|
|
81
|
+
contentType: z.ZodEnum<{
|
|
82
|
+
unknown: "unknown";
|
|
83
|
+
article: "article";
|
|
84
|
+
repo: "repo";
|
|
85
|
+
docs: "docs";
|
|
86
|
+
package: "package";
|
|
87
|
+
video: "video";
|
|
88
|
+
tool: "tool";
|
|
89
|
+
product: "product";
|
|
90
|
+
}>;
|
|
91
|
+
confidence: z.ZodNumber;
|
|
92
|
+
}, z.core.$strip>;
|
|
93
|
+
//#endregion
|
|
94
|
+
//#region src/llm/anthropic.d.ts
|
|
95
|
+
/**
|
|
96
|
+
* Anthropic Claude provider
|
|
97
|
+
*
|
|
98
|
+
* Requires @anthropic-ai/sdk as a peer dependency.
|
|
99
|
+
*
|
|
100
|
+
* @example
|
|
101
|
+
* ```ts
|
|
102
|
+
* const provider = new AnthropicProvider({ apiKey: 'sk-...' });
|
|
103
|
+
* const result = await scrape(url, { llm: provider, enhance: ['summarize'] });
|
|
104
|
+
* ```
|
|
105
|
+
*/
|
|
106
|
+
declare class AnthropicProvider implements LLMProvider {
|
|
107
|
+
readonly name = "anthropic";
|
|
108
|
+
private client;
|
|
109
|
+
private model;
|
|
110
|
+
constructor(config?: AnthropicConfig);
|
|
111
|
+
complete(prompt: string, options?: CompletionOptions): Promise<string>;
|
|
112
|
+
completeJSON<T>(prompt: string, schema: z.ZodType<T>, options?: CompletionOptions): Promise<T>;
|
|
113
|
+
}
|
|
114
|
+
//#endregion
|
|
115
|
+
//#region src/llm/enhancer.d.ts
|
|
116
|
+
/**
|
|
117
|
+
* Enhance scraped data with LLM-powered features
|
|
118
|
+
*/
|
|
119
|
+
declare function enhance(data: ScrapedData, provider: LLMProvider, types: EnhancementType[]): Promise<Partial<ScrapedData>>;
|
|
120
|
+
/**
|
|
121
|
+
* Options for the ask() function
|
|
122
|
+
*/
|
|
123
|
+
interface AskOptions {
|
|
124
|
+
/** Key to store the result under in custom field */
|
|
125
|
+
key?: string;
|
|
126
|
+
/** Schema for structured response */
|
|
127
|
+
schema?: ExtractionSchema;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Ask a custom question about the scraped content
|
|
131
|
+
* Results are stored in the `custom` field of ScrapedData
|
|
132
|
+
*/
|
|
133
|
+
declare function ask(data: ScrapedData, provider: LLMProvider, prompt: string, options?: AskOptions): Promise<Partial<ScrapedData>>;
|
|
134
|
+
/**
|
|
135
|
+
* Extract structured data using LLM and a custom schema
|
|
136
|
+
*/
|
|
137
|
+
declare function extract<T>(data: ScrapedData, provider: LLMProvider, schema: ExtractionSchema, promptTemplate?: string): Promise<T>;
|
|
138
|
+
//#endregion
|
|
139
|
+
//#region src/llm/openai.d.ts
|
|
140
|
+
/**
|
|
141
|
+
* OpenAI-compatible provider
|
|
142
|
+
*
|
|
143
|
+
* Works with:
|
|
144
|
+
* - OpenAI API
|
|
145
|
+
* - Ollama (http://localhost:11434/v1)
|
|
146
|
+
* - LM Studio (http://localhost:1234/v1)
|
|
147
|
+
* - LocalAI
|
|
148
|
+
* - vLLM
|
|
149
|
+
* - Any OpenAI-compatible API
|
|
150
|
+
*
|
|
151
|
+
* Requires `openai` as a peer dependency.
|
|
152
|
+
*
|
|
153
|
+
* @example
|
|
154
|
+
* ```ts
|
|
155
|
+
* // OpenAI
|
|
156
|
+
* const provider = new OpenAIProvider({ apiKey: 'sk-...' });
|
|
157
|
+
*
|
|
158
|
+
* // Ollama
|
|
159
|
+
* const provider = new OpenAIProvider({
|
|
160
|
+
* baseUrl: 'http://localhost:11434/v1',
|
|
161
|
+
* model: 'llama3.2',
|
|
162
|
+
* apiKey: 'ollama' // Ollama doesn't require a real key
|
|
163
|
+
* });
|
|
164
|
+
*
|
|
165
|
+
* // LM Studio
|
|
166
|
+
* const provider = new OpenAIProvider({
|
|
167
|
+
* baseUrl: 'http://localhost:1234/v1',
|
|
168
|
+
* model: 'local-model',
|
|
169
|
+
* apiKey: 'lm-studio'
|
|
170
|
+
* });
|
|
171
|
+
* ```
|
|
172
|
+
*/
|
|
173
|
+
declare class OpenAIProvider implements LLMProvider {
|
|
174
|
+
readonly name = "openai";
|
|
175
|
+
private client;
|
|
176
|
+
private model;
|
|
177
|
+
constructor(config?: OpenAICompatibleConfig);
|
|
178
|
+
complete(prompt: string, options?: CompletionOptions): Promise<string>;
|
|
179
|
+
completeJSON<T>(prompt: string, schema: z.ZodType<T>, options?: CompletionOptions): Promise<T>;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Create an OpenAI provider with default settings
|
|
183
|
+
*/
|
|
184
|
+
declare function createOpenAI(config?: OpenAICompatibleConfig): OpenAIProvider;
|
|
185
|
+
/**
|
|
186
|
+
* Create an Ollama provider
|
|
187
|
+
*
|
|
188
|
+
* @example
|
|
189
|
+
* ```ts
|
|
190
|
+
* const provider = createOllama({ model: 'llama3.2' });
|
|
191
|
+
* ```
|
|
192
|
+
*/
|
|
193
|
+
declare function createOllama(config?: {
|
|
194
|
+
model: string;
|
|
195
|
+
port?: number;
|
|
196
|
+
}): OpenAIProvider;
|
|
197
|
+
/**
|
|
198
|
+
* Create an LM Studio provider
|
|
199
|
+
*
|
|
200
|
+
* @example
|
|
201
|
+
* ```ts
|
|
202
|
+
* const provider = createLMStudio({ model: 'local-model' });
|
|
203
|
+
* ```
|
|
204
|
+
*/
|
|
205
|
+
declare function createLMStudio(config?: {
|
|
206
|
+
model: string;
|
|
207
|
+
port?: number;
|
|
208
|
+
}): OpenAIProvider;
|
|
209
|
+
//#endregion
|
|
210
|
+
export { type AnthropicConfig, AnthropicProvider, type AskOptions, type ClassifyResult, ClassifySchema, type CompletionOptions, type EntitiesResult, EntitiesSchema, type LLMProvider, type OpenAICompatibleConfig, OpenAIProvider, type SummaryResult, SummarySchema, type TagsResult, TagsSchema, ask, createLMStudio, createOllama, createOpenAI, enhance, extract };
|
|
211
|
+
//# sourceMappingURL=index.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.cts","names":[],"sources":["../../src/llm/types.ts","../../src/llm/anthropic.ts","../../src/llm/enhancer.ts","../../src/llm/openai.ts"],"sourcesContent":[],"mappings":";;;;;;;AAKiB,UAAA,iBAAA,CAAiB;EASjB,SAAA,CAAA,EAAA,MAAW;EAMS,WAAA,CAAA,EAAA,MAAA;EAAoB,YAAA,CAAA,EAAA,MAAA;;;;;AAK6B,UAXrE,WAAA,CAWqE;EAAO,SAAA,IAAA,EAAA,MAAA;EAM5E;AAUjB;AASA;EAIiB,QAAA,CAAA,MAAU,EAAA,MAAA,EAAA,OAAA,CAAA,EAlCU,iBAkCV,CAAA,EAlC8B,OAkC9B,CAAA,MAAA,CAAA;EAIV;AAQjB;AAQA;0CAjD0C,CAAA,CAAE,QAAQ,cAAc,oBAAoB,QAAQ;;;;AAqD9F;UA/CiB,eAAA;;;EA+CM,OAAA,CAAA,EAAA,MAAA;;AAIvB;;;;UAzCiB,sBAAA;;;;;;;;AAyCU,UAhCV,aAAA,CAgCU;EAAA,OAAA,EAAA,MAAA;AAQ3B;UApCiB,UAAA;;;AAoCU,UAhCV,cAAA,CAgCU;EAAA,MAAA,EAAA,MAAA,EAAA;;;;ECxEd,QAAA,EAAA,MAAA,EAAA;;AAgC6B,UDgBzB,cAAA,CChByB;EAAyB,WAAA,EAAA,MAAA;EA8B7C,UAAA,EAAA,MAAA;;;;;AA9DoB,cDwD7B,aCxD6B,EDwDhB,CAAA,CAAA,SCxDgB,CAAA;EAAW,OAAA,aAAA;;cD4DxC,YAAU,CAAA,CAAA;;AEjEvB,CAAA,eAA6B,CAAA;AACrB,cFoEK,cEpEL,EFoEmB,CAAA,CAAA,SEpEnB,CAAA;EACI,MAAA,YAAA,YAAA,CAAA;EACH,aAAA,YAAA,YAAA,CAAA;EACU,YAAA,YAAA,YAAA,CAAA;EAAR,SAAA,YAAA,YAAA,CAAA;EAAR,QAAA,YAAA,YAAA,CAAA;CAAO,eAAA,CAAA;AAoDO,cFqBJ,cEjBF,EFiBgB,CAAA,CAAA,SEjBA,CAAA;EAOL,WAAG,WAAA,CAAA;IACjB,OAAA,EAAA,SAAA;IACI,OAAA,EAAA,SAAA;IAEA,IAAA,EAAA,MAAA;IACO,IAAA,EAAA,MAAA;IAAR,OAAA,EAAA,SAAA;IAAR,KAAA,EAAA,OAAA;IAAO,IAAA,EAAA,MAAA;IA8CY,OAAO,EAAA,SAAA;EACrB,CAAA,CAAA;EACI,UAAA,aAAA;CACF,eAAA,CAAA;;;;;AFjIV;AASA;;;;;;;;AAW6F,cCPhF,iBAAA,YAA6B,WDOmD,CAAA;EAM5E,SAAA,IAAA,GAAA,WAAe;EAUf,QAAA,MAAA;EASA,QAAA,KAAA;EAIA,WAAA,CAAA,MAAU,CAAA,EC/BL,eD+BK;EAIV,QAAA,CAAA,MAAA,EAAc,MAAA,EAAA,OAAA,CAAA,ECRW,iBDQX,CAAA,ECRoC,ODQpC,CAAA,MAAA,CAAA;EAQd,YAAA,CAAA,CAAA,CAAA,CAAA,MAAc,EAAA,MAAA,EAAA,MAAA,ECcnB,CAAA,CAAE,ODdiB,CCcT,CDdS,CAAA,EAAA,OAAA,CAAA,ECelB,iBDfkB,CAAA,ECgB1B,ODhB0B,CCgBlB,CDhBkB,CAAA;AAQ/B;;;;;AArEA;AASiB,iBEDK,OAAA,CFCM,IAAA,EEApB,WFAoB,EAAA,QAAA,EEChB,WFDgB,EAAA,KAAA,EEEnB,eFFmB,EAAA,CAAA,EEGzB,OFHyB,CEGjB,OFHiB,CEGT,WFHS,CAAA,CAAA;;;;AAWgB,UE4C3B,UAAA,CF5C2B;EAAsB;EAA4B,GAAA,CAAA,EAAA,MAAA;EAAR;EAAO,MAAA,CAAA,EEgDlF,gBFhDkF;AAM7F;AAUA;AASA;AAIA;AAIA;AAQiB,iBEcK,GAAA,CFdS,IAAA,EEevB,WFfuB,EAAA,QAAA,EEgBnB,WFhBmB,EAAA,MAAA,EAAA,MAAA,EAAA,OAAA,CAAA,EEkBnB,UFlBmB,CAAA,EEmB5B,OFnB4B,CEmBpB,OFnBoB,CEmBZ,WFnBY,CAAA,CAAA;AAQ/B;;;AAA0B,iBEyDJ,OFzDI,CAAA,CAAA,CAAA,CAAA,IAAA,EE0DlB,WF1DkB,EAAA,QAAA,EE2Dd,WF3Dc,EAAA,MAAA,EE4DhB,gBF5DgB,EAAA,cAAA,CAAA,EAAA,MAAA,CAAA,EE8DvB,OF9DuB,CE8Df,CF9De,CAAA;;;;;AArE1B;AASA;;;;;;;;;AAiBA;AAUA;AASA;AAIA;AAIA;AAQA;AAQA;;;;;AAIA;;;;;;AAIA;;;;cGzCa,cAAA,YAA0B;;;;uBAKjB;qCA8BoB,oBAAyB;0CAoCvD,CAAA,CAAE,QAAQ,cACT,oBACR,QAAQ;;;;AHxBb;iBG6IgB,YAAA,UAAsB,yBAAyB;;;;;;;;AFrN/D;AAKsB,iBE4NN,YAAA,CF5NM,MAyDA,CAzDA,EAAA;EA2BoB,KAAA,EAAA,MAAA;EAAyB,IAAA,CAAA,EAAA,MAAA;CA8B7C,CAAA,EEqKnB,cFrKmB;;;;;;;;;iBEqLN,cAAA,ODtPJ;EAFU,KAAA,EAAA,MAAO;EACrB,IAAA,CAAA,EAAA,MAAA;CACI,CAAA,ECwPT,cDxPS"}
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { h as ScrapedData, r as EnhancementType, s as ExtractionSchema } from "../types-CNQZVW36.mjs";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
//#region src/llm/types.d.ts
|
|
5
|
+
/**
|
|
6
|
+
* LLM completion options
|
|
7
|
+
*/
|
|
8
|
+
interface CompletionOptions {
|
|
9
|
+
maxTokens?: number;
|
|
10
|
+
temperature?: number;
|
|
11
|
+
systemPrompt?: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* LLM Provider interface - implemented by all providers
|
|
15
|
+
*/
|
|
16
|
+
interface LLMProvider {
|
|
17
|
+
readonly name: string;
|
|
18
|
+
/**
|
|
19
|
+
* Generate a text completion
|
|
20
|
+
*/
|
|
21
|
+
complete(prompt: string, options?: CompletionOptions): Promise<string>;
|
|
22
|
+
/**
|
|
23
|
+
* Generate a structured JSON completion with Zod validation
|
|
24
|
+
*/
|
|
25
|
+
completeJSON<T>(prompt: string, schema: z.ZodType<T>, options?: CompletionOptions): Promise<T>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Provider configuration for Anthropic
|
|
29
|
+
*/
|
|
30
|
+
interface AnthropicConfig {
|
|
31
|
+
apiKey?: string;
|
|
32
|
+
model?: string;
|
|
33
|
+
baseUrl?: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Provider configuration for OpenAI-compatible APIs
|
|
37
|
+
* Works with: OpenAI, Ollama, LM Studio, LocalAI, vLLM, etc.
|
|
38
|
+
*/
|
|
39
|
+
interface OpenAICompatibleConfig {
|
|
40
|
+
apiKey?: string;
|
|
41
|
+
model?: string;
|
|
42
|
+
baseUrl?: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Enhancement result types
|
|
46
|
+
*/
|
|
47
|
+
interface SummaryResult {
|
|
48
|
+
summary: string;
|
|
49
|
+
}
|
|
50
|
+
interface TagsResult {
|
|
51
|
+
tags: string[];
|
|
52
|
+
}
|
|
53
|
+
interface EntitiesResult {
|
|
54
|
+
people: string[];
|
|
55
|
+
organizations: string[];
|
|
56
|
+
technologies: string[];
|
|
57
|
+
locations: string[];
|
|
58
|
+
concepts: string[];
|
|
59
|
+
}
|
|
60
|
+
interface ClassifyResult {
|
|
61
|
+
contentType: string;
|
|
62
|
+
confidence: number;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Zod schemas for LLM outputs
|
|
66
|
+
*/
|
|
67
|
+
declare const SummarySchema: z.ZodObject<{
|
|
68
|
+
summary: z.ZodString;
|
|
69
|
+
}, z.core.$strip>;
|
|
70
|
+
declare const TagsSchema: z.ZodObject<{
|
|
71
|
+
tags: z.ZodArray<z.ZodString>;
|
|
72
|
+
}, z.core.$strip>;
|
|
73
|
+
declare const EntitiesSchema: z.ZodObject<{
|
|
74
|
+
people: z.ZodArray<z.ZodString>;
|
|
75
|
+
organizations: z.ZodArray<z.ZodString>;
|
|
76
|
+
technologies: z.ZodArray<z.ZodString>;
|
|
77
|
+
locations: z.ZodArray<z.ZodString>;
|
|
78
|
+
concepts: z.ZodArray<z.ZodString>;
|
|
79
|
+
}, z.core.$strip>;
|
|
80
|
+
declare const ClassifySchema: z.ZodObject<{
|
|
81
|
+
contentType: z.ZodEnum<{
|
|
82
|
+
unknown: "unknown";
|
|
83
|
+
article: "article";
|
|
84
|
+
repo: "repo";
|
|
85
|
+
docs: "docs";
|
|
86
|
+
package: "package";
|
|
87
|
+
video: "video";
|
|
88
|
+
tool: "tool";
|
|
89
|
+
product: "product";
|
|
90
|
+
}>;
|
|
91
|
+
confidence: z.ZodNumber;
|
|
92
|
+
}, z.core.$strip>;
|
|
93
|
+
//#endregion
|
|
94
|
+
//#region src/llm/anthropic.d.ts
|
|
95
|
+
/**
|
|
96
|
+
* Anthropic Claude provider
|
|
97
|
+
*
|
|
98
|
+
* Requires @anthropic-ai/sdk as a peer dependency.
|
|
99
|
+
*
|
|
100
|
+
* @example
|
|
101
|
+
* ```ts
|
|
102
|
+
* const provider = new AnthropicProvider({ apiKey: 'sk-...' });
|
|
103
|
+
* const result = await scrape(url, { llm: provider, enhance: ['summarize'] });
|
|
104
|
+
* ```
|
|
105
|
+
*/
|
|
106
|
+
declare class AnthropicProvider implements LLMProvider {
|
|
107
|
+
readonly name = "anthropic";
|
|
108
|
+
private client;
|
|
109
|
+
private model;
|
|
110
|
+
constructor(config?: AnthropicConfig);
|
|
111
|
+
complete(prompt: string, options?: CompletionOptions): Promise<string>;
|
|
112
|
+
completeJSON<T>(prompt: string, schema: z.ZodType<T>, options?: CompletionOptions): Promise<T>;
|
|
113
|
+
}
|
|
114
|
+
//#endregion
|
|
115
|
+
//#region src/llm/enhancer.d.ts
|
|
116
|
+
/**
|
|
117
|
+
* Enhance scraped data with LLM-powered features
|
|
118
|
+
*/
|
|
119
|
+
declare function enhance(data: ScrapedData, provider: LLMProvider, types: EnhancementType[]): Promise<Partial<ScrapedData>>;
|
|
120
|
+
/**
|
|
121
|
+
* Options for the ask() function
|
|
122
|
+
*/
|
|
123
|
+
interface AskOptions {
|
|
124
|
+
/** Key to store the result under in custom field */
|
|
125
|
+
key?: string;
|
|
126
|
+
/** Schema for structured response */
|
|
127
|
+
schema?: ExtractionSchema;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Ask a custom question about the scraped content
|
|
131
|
+
* Results are stored in the `custom` field of ScrapedData
|
|
132
|
+
*/
|
|
133
|
+
declare function ask(data: ScrapedData, provider: LLMProvider, prompt: string, options?: AskOptions): Promise<Partial<ScrapedData>>;
|
|
134
|
+
/**
|
|
135
|
+
* Extract structured data using LLM and a custom schema
|
|
136
|
+
*/
|
|
137
|
+
declare function extract<T>(data: ScrapedData, provider: LLMProvider, schema: ExtractionSchema, promptTemplate?: string): Promise<T>;
|
|
138
|
+
//#endregion
|
|
139
|
+
//#region src/llm/openai.d.ts
|
|
140
|
+
/**
|
|
141
|
+
* OpenAI-compatible provider
|
|
142
|
+
*
|
|
143
|
+
* Works with:
|
|
144
|
+
* - OpenAI API
|
|
145
|
+
* - Ollama (http://localhost:11434/v1)
|
|
146
|
+
* - LM Studio (http://localhost:1234/v1)
|
|
147
|
+
* - LocalAI
|
|
148
|
+
* - vLLM
|
|
149
|
+
* - Any OpenAI-compatible API
|
|
150
|
+
*
|
|
151
|
+
* Requires `openai` as a peer dependency.
|
|
152
|
+
*
|
|
153
|
+
* @example
|
|
154
|
+
* ```ts
|
|
155
|
+
* // OpenAI
|
|
156
|
+
* const provider = new OpenAIProvider({ apiKey: 'sk-...' });
|
|
157
|
+
*
|
|
158
|
+
* // Ollama
|
|
159
|
+
* const provider = new OpenAIProvider({
|
|
160
|
+
* baseUrl: 'http://localhost:11434/v1',
|
|
161
|
+
* model: 'llama3.2',
|
|
162
|
+
* apiKey: 'ollama' // Ollama doesn't require a real key
|
|
163
|
+
* });
|
|
164
|
+
*
|
|
165
|
+
* // LM Studio
|
|
166
|
+
* const provider = new OpenAIProvider({
|
|
167
|
+
* baseUrl: 'http://localhost:1234/v1',
|
|
168
|
+
* model: 'local-model',
|
|
169
|
+
* apiKey: 'lm-studio'
|
|
170
|
+
* });
|
|
171
|
+
* ```
|
|
172
|
+
*/
|
|
173
|
+
declare class OpenAIProvider implements LLMProvider {
|
|
174
|
+
readonly name = "openai";
|
|
175
|
+
private client;
|
|
176
|
+
private model;
|
|
177
|
+
constructor(config?: OpenAICompatibleConfig);
|
|
178
|
+
complete(prompt: string, options?: CompletionOptions): Promise<string>;
|
|
179
|
+
completeJSON<T>(prompt: string, schema: z.ZodType<T>, options?: CompletionOptions): Promise<T>;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Create an OpenAI provider with default settings
|
|
183
|
+
*/
|
|
184
|
+
declare function createOpenAI(config?: OpenAICompatibleConfig): OpenAIProvider;
|
|
185
|
+
/**
|
|
186
|
+
* Create an Ollama provider
|
|
187
|
+
*
|
|
188
|
+
* @example
|
|
189
|
+
* ```ts
|
|
190
|
+
* const provider = createOllama({ model: 'llama3.2' });
|
|
191
|
+
* ```
|
|
192
|
+
*/
|
|
193
|
+
declare function createOllama(config?: {
|
|
194
|
+
model: string;
|
|
195
|
+
port?: number;
|
|
196
|
+
}): OpenAIProvider;
|
|
197
|
+
/**
|
|
198
|
+
* Create an LM Studio provider
|
|
199
|
+
*
|
|
200
|
+
* @example
|
|
201
|
+
* ```ts
|
|
202
|
+
* const provider = createLMStudio({ model: 'local-model' });
|
|
203
|
+
* ```
|
|
204
|
+
*/
|
|
205
|
+
declare function createLMStudio(config?: {
|
|
206
|
+
model: string;
|
|
207
|
+
port?: number;
|
|
208
|
+
}): OpenAIProvider;
|
|
209
|
+
//#endregion
|
|
210
|
+
export { type AnthropicConfig, AnthropicProvider, type AskOptions, type ClassifyResult, ClassifySchema, type CompletionOptions, type EntitiesResult, EntitiesSchema, type LLMProvider, type OpenAICompatibleConfig, OpenAIProvider, type SummaryResult, SummarySchema, type TagsResult, TagsSchema, ask, createLMStudio, createOllama, createOpenAI, enhance, extract };
|
|
211
|
+
//# sourceMappingURL=index.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":[],"sources":["../../src/llm/types.ts","../../src/llm/anthropic.ts","../../src/llm/enhancer.ts","../../src/llm/openai.ts"],"sourcesContent":[],"mappings":";;;;;;;AAKiB,UAAA,iBAAA,CAAiB;EASjB,SAAA,CAAA,EAAA,MAAW;EAMS,WAAA,CAAA,EAAA,MAAA;EAAoB,YAAA,CAAA,EAAA,MAAA;;;;;AAK6B,UAXrE,WAAA,CAWqE;EAAO,SAAA,IAAA,EAAA,MAAA;EAM5E;AAUjB;AASA;EAIiB,QAAA,CAAA,MAAU,EAAA,MAAA,EAAA,OAAA,CAAA,EAlCU,iBAkCV,CAAA,EAlC8B,OAkC9B,CAAA,MAAA,CAAA;EAIV;AAQjB;AAQA;0CAjD0C,CAAA,CAAE,QAAQ,cAAc,oBAAoB,QAAQ;;;;AAqD9F;UA/CiB,eAAA;;;EA+CM,OAAA,CAAA,EAAA,MAAA;;AAIvB;;;;UAzCiB,sBAAA;;;;;;;;AAyCU,UAhCV,aAAA,CAgCU;EAAA,OAAA,EAAA,MAAA;AAQ3B;UApCiB,UAAA;;;AAoCU,UAhCV,cAAA,CAgCU;EAAA,MAAA,EAAA,MAAA,EAAA;;;;ECxEd,QAAA,EAAA,MAAA,EAAA;;AAgC6B,UDgBzB,cAAA,CChByB;EAAyB,WAAA,EAAA,MAAA;EA8B7C,UAAA,EAAA,MAAA;;;;;AA9DoB,cDwD7B,aCxD6B,EDwDhB,CAAA,CAAA,SCxDgB,CAAA;EAAW,OAAA,aAAA;;cD4DxC,YAAU,CAAA,CAAA;;AEjEvB,CAAA,eAA6B,CAAA;AACrB,cFoEK,cEpEL,EFoEmB,CAAA,CAAA,SEpEnB,CAAA;EACI,MAAA,YAAA,YAAA,CAAA;EACH,aAAA,YAAA,YAAA,CAAA;EACU,YAAA,YAAA,YAAA,CAAA;EAAR,SAAA,YAAA,YAAA,CAAA;EAAR,QAAA,YAAA,YAAA,CAAA;CAAO,eAAA,CAAA;AAoDO,cFqBJ,cEjBF,EFiBgB,CAAA,CAAA,SEjBA,CAAA;EAOL,WAAG,WAAA,CAAA;IACjB,OAAA,EAAA,SAAA;IACI,OAAA,EAAA,SAAA;IAEA,IAAA,EAAA,MAAA;IACO,IAAA,EAAA,MAAA;IAAR,OAAA,EAAA,SAAA;IAAR,KAAA,EAAA,OAAA;IAAO,IAAA,EAAA,MAAA;IA8CY,OAAO,EAAA,SAAA;EACrB,CAAA,CAAA;EACI,UAAA,aAAA;CACF,eAAA,CAAA;;;;;AFjIV;AASA;;;;;;;;AAW6F,cCPhF,iBAAA,YAA6B,WDOmD,CAAA;EAM5E,SAAA,IAAA,GAAA,WAAe;EAUf,QAAA,MAAA;EASA,QAAA,KAAA;EAIA,WAAA,CAAA,MAAU,CAAA,EC/BL,eD+BK;EAIV,QAAA,CAAA,MAAA,EAAc,MAAA,EAAA,OAAA,CAAA,ECRW,iBDQX,CAAA,ECRoC,ODQpC,CAAA,MAAA,CAAA;EAQd,YAAA,CAAA,CAAA,CAAA,CAAA,MAAc,EAAA,MAAA,EAAA,MAAA,ECcnB,CAAA,CAAE,ODdiB,CCcT,CDdS,CAAA,EAAA,OAAA,CAAA,ECelB,iBDfkB,CAAA,ECgB1B,ODhB0B,CCgBlB,CDhBkB,CAAA;AAQ/B;;;;;AArEA;AASiB,iBEDK,OAAA,CFCM,IAAA,EEApB,WFAoB,EAAA,QAAA,EEChB,WFDgB,EAAA,KAAA,EEEnB,eFFmB,EAAA,CAAA,EEGzB,OFHyB,CEGjB,OFHiB,CEGT,WFHS,CAAA,CAAA;;;;AAWgB,UE4C3B,UAAA,CF5C2B;EAAsB;EAA4B,GAAA,CAAA,EAAA,MAAA;EAAR;EAAO,MAAA,CAAA,EEgDlF,gBFhDkF;AAM7F;AAUA;AASA;AAIA;AAIA;AAQiB,iBEcK,GAAA,CFdS,IAAA,EEevB,WFfuB,EAAA,QAAA,EEgBnB,WFhBmB,EAAA,MAAA,EAAA,MAAA,EAAA,OAAA,CAAA,EEkBnB,UFlBmB,CAAA,EEmB5B,OFnB4B,CEmBpB,OFnBoB,CEmBZ,WFnBY,CAAA,CAAA;AAQ/B;;;AAA0B,iBEyDJ,OFzDI,CAAA,CAAA,CAAA,CAAA,IAAA,EE0DlB,WF1DkB,EAAA,QAAA,EE2Dd,WF3Dc,EAAA,MAAA,EE4DhB,gBF5DgB,EAAA,cAAA,CAAA,EAAA,MAAA,CAAA,EE8DvB,OF9DuB,CE8Df,CF9De,CAAA;;;;;AArE1B;AASA;;;;;;;;;AAiBA;AAUA;AASA;AAIA;AAIA;AAQA;AAQA;;;;;AAIA;;;;;;AAIA;;;;cGzCa,cAAA,YAA0B;;;;uBAKjB;qCA8BoB,oBAAyB;0CAoCvD,CAAA,CAAE,QAAQ,cACT,oBACR,QAAQ;;;;AHxBb;iBG6IgB,YAAA,UAAsB,yBAAyB;;;;;;;;AFrN/D;AAKsB,iBE4NN,YAAA,CF5NM,MAyDA,CAzDA,EAAA;EA2BoB,KAAA,EAAA,MAAA;EAAyB,IAAA,CAAA,EAAA,MAAA;CA8B7C,CAAA,EEqKnB,cFrKmB;;;;;;;;;iBEqLN,cAAA,ODtPJ;EAFU,KAAA,EAAA,MAAO;EACrB,IAAA,CAAA,EAAA,MAAA;CACI,CAAA,ECwPT,cDxPS"}
|