webpeel 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +257 -0
- package/dist/cli.js.map +1 -1
- package/dist/core/branding.d.ts +55 -0
- package/dist/core/branding.d.ts.map +1 -0
- package/dist/core/branding.js +235 -0
- package/dist/core/branding.js.map +1 -0
- package/dist/core/change-tracking.d.ts +76 -0
- package/dist/core/change-tracking.d.ts.map +1 -0
- package/dist/core/change-tracking.js +267 -0
- package/dist/core/change-tracking.js.map +1 -0
- package/dist/core/map.d.ts +4 -0
- package/dist/core/map.d.ts.map +1 -1
- package/dist/core/map.js +60 -5
- package/dist/core/map.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +73 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +290 -0
- package/dist/mcp/server.js.map +1 -1
- package/dist/types.d.ts +21 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/core/map.js
CHANGED
|
@@ -6,19 +6,58 @@ import { discoverSitemap } from './sitemap.js';
|
|
|
6
6
|
import { peel } from '../index.js';
|
|
7
7
|
export async function mapDomain(startUrl, options = {}) {
|
|
8
8
|
const startTime = Date.now();
|
|
9
|
-
const { useSitemap = true, crawlHomepage = true, maxUrls = 5000, timeout = 10000, includePatterns = [], excludePatterns = [], } = options;
|
|
9
|
+
const { useSitemap = true, crawlHomepage = true, maxUrls = 5000, timeout = 10000, includePatterns = [], excludePatterns = [], search, contentTypeFilter = [], } = options;
|
|
10
10
|
const urlObj = new URL(startUrl);
|
|
11
11
|
const domain = urlObj.hostname;
|
|
12
|
-
const allUrls = new
|
|
12
|
+
const allUrls = new Map(); // URL -> relevance score
|
|
13
13
|
let sitemapUrls = [];
|
|
14
14
|
// Compile filter patterns
|
|
15
15
|
const includeRegexes = includePatterns.map(p => new RegExp(p));
|
|
16
16
|
const excludeRegexes = excludePatterns.map(p => new RegExp(p));
|
|
17
|
+
// Parse search terms
|
|
18
|
+
const searchTerms = search ? search.toLowerCase().split(/\s+/).filter(t => t.length > 0) : [];
|
|
19
|
+
/**
|
|
20
|
+
* Calculate relevance score for a URL based on search terms
|
|
21
|
+
* Scores based on matches in URL path and title/description if available
|
|
22
|
+
*/
|
|
23
|
+
function calculateRelevance(url, title, description) {
|
|
24
|
+
if (searchTerms.length === 0)
|
|
25
|
+
return 1; // No search = all equal
|
|
26
|
+
let score = 0;
|
|
27
|
+
const urlLower = url.toLowerCase();
|
|
28
|
+
const titleLower = (title || '').toLowerCase();
|
|
29
|
+
const descLower = (description || '').toLowerCase();
|
|
30
|
+
for (const term of searchTerms) {
|
|
31
|
+
// URL path matches (highest weight)
|
|
32
|
+
if (urlLower.includes(term))
|
|
33
|
+
score += 3;
|
|
34
|
+
// Title matches
|
|
35
|
+
if (titleLower.includes(term))
|
|
36
|
+
score += 2;
|
|
37
|
+
// Description matches
|
|
38
|
+
if (descLower.includes(term))
|
|
39
|
+
score += 1;
|
|
40
|
+
}
|
|
41
|
+
return score;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Check if URL should be included based on patterns and content type
|
|
45
|
+
*/
|
|
17
46
|
function shouldInclude(url) {
|
|
18
47
|
if (excludeRegexes.some(r => r.test(url)))
|
|
19
48
|
return false;
|
|
20
49
|
if (includeRegexes.length > 0 && !includeRegexes.some(r => r.test(url)))
|
|
21
50
|
return false;
|
|
51
|
+
// Content type filter (check file extension)
|
|
52
|
+
if (contentTypeFilter.length > 0) {
|
|
53
|
+
const ext = url.split('.').pop()?.toLowerCase() || '';
|
|
54
|
+
const hasMatch = contentTypeFilter.some(type => {
|
|
55
|
+
const typeExt = type.replace(/^\./, '').toLowerCase();
|
|
56
|
+
return ext === typeExt || url.toLowerCase().includes(`.${typeExt}`);
|
|
57
|
+
});
|
|
58
|
+
if (!hasMatch)
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
22
61
|
return true;
|
|
23
62
|
}
|
|
24
63
|
// Step 1: Sitemap discovery
|
|
@@ -29,7 +68,9 @@ export async function mapDomain(startUrl, options = {}) {
|
|
|
29
68
|
if (allUrls.size >= maxUrls)
|
|
30
69
|
break;
|
|
31
70
|
if (shouldInclude(entry.url)) {
|
|
32
|
-
|
|
71
|
+
// Sitemap entries don't have title/description, just score by URL
|
|
72
|
+
const score = calculateRelevance(entry.url);
|
|
73
|
+
allUrls.set(entry.url, score);
|
|
33
74
|
}
|
|
34
75
|
}
|
|
35
76
|
}
|
|
@@ -43,7 +84,10 @@ export async function mapDomain(startUrl, options = {}) {
|
|
|
43
84
|
try {
|
|
44
85
|
const linkUrl = new URL(link);
|
|
45
86
|
if (linkUrl.hostname === domain && shouldInclude(link)) {
|
|
46
|
-
|
|
87
|
+
const score = calculateRelevance(link);
|
|
88
|
+
if (!allUrls.has(link)) {
|
|
89
|
+
allUrls.set(link, score);
|
|
90
|
+
}
|
|
47
91
|
}
|
|
48
92
|
}
|
|
49
93
|
catch { /* skip invalid URLs */ }
|
|
@@ -51,8 +95,19 @@ export async function mapDomain(startUrl, options = {}) {
|
|
|
51
95
|
}
|
|
52
96
|
catch { /* skip homepage crawl errors */ }
|
|
53
97
|
}
|
|
98
|
+
// Sort URLs by relevance score (highest first), then alphabetically
|
|
99
|
+
const sortedUrls = Array.from(allUrls.entries())
|
|
100
|
+
.sort((a, b) => {
|
|
101
|
+
if (search) {
|
|
102
|
+
// Sort by score first if searching
|
|
103
|
+
if (b[1] !== a[1])
|
|
104
|
+
return b[1] - a[1];
|
|
105
|
+
}
|
|
106
|
+
return a[0].localeCompare(b[0]);
|
|
107
|
+
})
|
|
108
|
+
.map(([url]) => url);
|
|
54
109
|
return {
|
|
55
|
-
urls:
|
|
110
|
+
urls: sortedUrls,
|
|
56
111
|
sitemapUrls,
|
|
57
112
|
total: allUrls.size,
|
|
58
113
|
elapsed: Date.now() - startTime,
|
package/dist/core/map.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/core/map.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/core/map.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAgCnC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,UAAsB,EAAE;IACxE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,EACJ,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,IAAI,EACpB,OAAO,GAAG,IAAI,EACd,OAAO,GAAG,KAAK,EACf,eAAe,GAAG,EAAE,EACpB,eAAe,GAAG,EAAE,EACpB,MAAM,EACN,iBAAiB,GAAG,EAAE,GACvB,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,yBAAyB;IACpE,IAAI,WAAW,GAAa,EAAE,CAAC;IAE/B,0BAA0B;IAC1B,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,qBAAqB;IACrB,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9F;;;OAGG;IACH,SAAS,kBAAkB,CAAC,GAAW,EAAE,KAAc,EAAE,WAAoB;QAC3E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,wBAAwB;QAEhE,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC/C,MAAM,SAAS,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAEpD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,oCAAoC;YACpC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;YACxC,gBAAgB;YAChB,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;YAC1C,sBAAsB;YACtB,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,SAAS,aAAa,CAAC,GAAW;QAChC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAEtF,6CAA6C;QAC7C,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACtD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;gBACtD,OAAO,GAAG,KAAK,OAAO,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ;gBAAE,OAAO,KAAK,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4BAA4B;IAC5B,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACpE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO;gBAAE,MAAM;YACnC,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,kEAAkE;gBAClE,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,IAAI,aAAa,IAAI,OAAO,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO;oBAAE,MAAM;gBACnC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,IAAI,OAAO,CAAC,QAAQ,KAAK,MAAM,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvD,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;wBACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;4BACvB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;wBAC3B,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,gCAAgC,CAAC,CAAC;IAC9C,CAAC;IAED,oEAAoE;IACpE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;SAC7C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,MAAM,EAAE,CAAC;YACX,mCAAmC;YACnC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,WAAW;QACX,KAAK,EAAE,OAAO,CAAC,IAAI;QACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAChC,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -9,6 +9,9 @@ export * from './types.js';
|
|
|
9
9
|
export { crawl, type CrawlOptions, type CrawlResult, type CrawlProgress } from './core/crawler.js';
|
|
10
10
|
export { discoverSitemap, type SitemapUrl, type SitemapResult } from './core/sitemap.js';
|
|
11
11
|
export { mapDomain, type MapOptions, type MapResult } from './core/map.js';
|
|
12
|
+
export { extractBranding, type BrandingProfile } from './core/branding.js';
|
|
13
|
+
export { trackChange, getSnapshot, clearSnapshots, type ChangeResult, type Snapshot } from './core/change-tracking.js';
|
|
14
|
+
export { extractWithLLM } from './core/extract.js';
|
|
12
15
|
/**
|
|
13
16
|
* Fetch and extract content from a URL
|
|
14
17
|
*
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CA8RtF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -13,6 +13,9 @@ export * from './types.js';
|
|
|
13
13
|
export { crawl } from './core/crawler.js';
|
|
14
14
|
export { discoverSitemap } from './core/sitemap.js';
|
|
15
15
|
export { mapDomain } from './core/map.js';
|
|
16
|
+
export { extractBranding } from './core/branding.js';
|
|
17
|
+
export { trackChange, getSnapshot, clearSnapshots } from './core/change-tracking.js';
|
|
18
|
+
export { extractWithLLM } from './core/extract.js';
|
|
16
19
|
/**
|
|
17
20
|
* Fetch and extract content from a URL
|
|
18
21
|
*
|
|
@@ -49,6 +52,10 @@ export async function peel(url, options = {}) {
|
|
|
49
52
|
if (actions && actions.length > 0) {
|
|
50
53
|
render = true;
|
|
51
54
|
}
|
|
55
|
+
// If branding is requested, force render mode
|
|
56
|
+
if (options.branding) {
|
|
57
|
+
render = true;
|
|
58
|
+
}
|
|
52
59
|
try {
|
|
53
60
|
// Fetch the page
|
|
54
61
|
const fetchResult = await smartFetch(url, {
|
|
@@ -188,6 +195,69 @@ export async function peel(url, options = {}) {
|
|
|
188
195
|
const fingerprint = createHash('sha256').update(content).digest('hex').slice(0, 16);
|
|
189
196
|
// Convert screenshot buffer to base64 if present
|
|
190
197
|
const screenshotBase64 = fetchResult.screenshot?.toString('base64');
|
|
198
|
+
// Extract branding if requested (requires browser)
|
|
199
|
+
let brandingProfile;
|
|
200
|
+
if (options.branding && render) {
|
|
201
|
+
try {
|
|
202
|
+
// Import playwright and create a page for branding extraction
|
|
203
|
+
const { chromium } = await import('playwright-core');
|
|
204
|
+
const browser = await chromium.launch({ headless: true });
|
|
205
|
+
const page = await browser.newPage({
|
|
206
|
+
userAgent: userAgent || undefined,
|
|
207
|
+
});
|
|
208
|
+
// Navigate to the URL
|
|
209
|
+
await page.goto(fetchResult.url, {
|
|
210
|
+
waitUntil: 'domcontentloaded',
|
|
211
|
+
timeout: timeout || 30000,
|
|
212
|
+
});
|
|
213
|
+
// Wait if specified
|
|
214
|
+
if (wait > 0) {
|
|
215
|
+
await page.waitForTimeout(wait);
|
|
216
|
+
}
|
|
217
|
+
// Extract branding
|
|
218
|
+
const { extractBranding } = await import('./core/branding.js');
|
|
219
|
+
brandingProfile = await extractBranding(page);
|
|
220
|
+
// Clean up
|
|
221
|
+
await browser.close();
|
|
222
|
+
}
|
|
223
|
+
catch (error) {
|
|
224
|
+
console.error('Branding extraction failed:', error);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
// Track content changes if requested
|
|
228
|
+
let changeResult;
|
|
229
|
+
if (options.changeTracking) {
|
|
230
|
+
try {
|
|
231
|
+
const { trackChange } = await import('./core/change-tracking.js');
|
|
232
|
+
changeResult = await trackChange(fetchResult.url, content, fingerprint);
|
|
233
|
+
}
|
|
234
|
+
catch (error) {
|
|
235
|
+
console.error('Change tracking failed:', error);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
// Generate AI summary if requested
|
|
239
|
+
let summaryText;
|
|
240
|
+
if (options.summary && options.llm) {
|
|
241
|
+
try {
|
|
242
|
+
const { extractWithLLM } = await import('./core/extract.js');
|
|
243
|
+
const summaryPrompt = typeof options.summary === 'object' && options.summary.prompt
|
|
244
|
+
? options.summary.prompt
|
|
245
|
+
: 'Summarize the main points of this content in a concise paragraph.';
|
|
246
|
+
const maxLength = typeof options.summary === 'object' && options.summary.maxLength
|
|
247
|
+
? options.summary.maxLength
|
|
248
|
+
: 200;
|
|
249
|
+
const result = await extractWithLLM(content, {
|
|
250
|
+
prompt: `${summaryPrompt} Keep it under ${maxLength} words.`,
|
|
251
|
+
llmApiKey: options.llm.apiKey,
|
|
252
|
+
llmModel: options.llm.model,
|
|
253
|
+
llmBaseUrl: options.llm.baseUrl,
|
|
254
|
+
});
|
|
255
|
+
summaryText = result.summary || Object.values(result)[0];
|
|
256
|
+
}
|
|
257
|
+
catch (error) {
|
|
258
|
+
console.error('Summary generation failed:', error);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
191
261
|
return {
|
|
192
262
|
url: fetchResult.url,
|
|
193
263
|
title,
|
|
@@ -202,6 +272,9 @@ export async function peel(url, options = {}) {
|
|
|
202
272
|
quality,
|
|
203
273
|
fingerprint,
|
|
204
274
|
extracted,
|
|
275
|
+
branding: brandingProfile,
|
|
276
|
+
changeTracking: changeResult,
|
|
277
|
+
summary: summaryText,
|
|
205
278
|
};
|
|
206
279
|
}
|
|
207
280
|
catch (error) {
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3J,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGtD,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAA2D,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAuC,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAmC,MAAM,eAAe,CAAC;AAE3E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,EACF,MAAM,GAAG,KAAK,EACd,OAAO,GAAG,KAAK,EACf,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,UAAU,EACnB,OAAO,GAAG,KAAK,EACf,SAAS,EACT,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,EACR,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,GAAG,KAAK,EACX,OAAO,EACP,OAAO,EACP,SAAS,GACV,GAAG,OAAO,CAAC;IAEZ,8CAA8C;IAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,gDAAgD;IAChD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE;YACxC,YAAY,EAAE,MAAM;YACpB,OAAO;YACP,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,SAAS,EAAE,OAAO;YAClB,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACpH,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAEjK,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAEvG,IAAI,OAAe,CAAC;QACpB,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,QAAQ,GAAQ,EAAE,CAAC;QACvB,IAAI,KAAK,GAAa,EAAE,CAAC;QACzB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,MAAM,EAAE,CAAC;YACX,yBAAyB;YACzB,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;YAC5B,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;YAED,4CAA4C;YAC5C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YACpD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACnB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;YACzB,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YAE5C,kEAAkE;YAClE,IAAI,WAAW,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACtB,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,MAAM;oBACT,OAAO,GAAG,WAAW,CAAC;oBACtB,MAAM;gBACR,KAAK,MAAM;oBACT,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;oBAClC,MAAM;gBACR,KAAK,UAAU,CAAC;gBAChB;oBACE,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC/C,MAAM;YACV,CAAC;YAED,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,+BAA+B;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC5C,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC1C,KAAK,GAAG,eAAe,CAAC;gBAExB,uCAAuC;gBACvC,MAAM,QAAQ,GAAG,4BAA4B,CAAC;gBAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC5C,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,KAAK,GAAG,2BAA2B,CAAC;YACtC,CAAC;YACD,OAAO,GAAG,GAAG,CAAC,CAAC,qCAAqC;QACtD,CAAC;aAAM,IAAI,KAAK,EAAE,CAAC;YACjB,4CAA4C;YAC5C,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE1E,yBAAyB;gBACzB,MAAM,KAAK,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;gBAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,KAAK,GAAG,CAAC,CAAC,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,eAAe,CAAC;oBAC7E,MAAM,SAAS,GAAa,EAAE,CAAC;oBAC/B,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;wBACnB,MAAM,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;wBACrD,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACpG,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAC1F,SAAS,CAAC,IAAI,CAAC,MAAM,SAAS,KAAK,QAAQ,KAAK,QAAQ,EAAE,CAAC,CAAC;wBAC5D,IAAI,QAAQ;4BAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACrC,CAAC,CAAC,CAAC;oBACH,OAAO,GAAG,KAAK,KAAK,OAAO,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;oBAC3B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,cAAc,CAAC;gBACtD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,KAAK,GAAG,cAAc,CAAC;YACzB,CAAC;YACD,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,0CAA0C;YAC1C,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;YAC3B,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,eAAe,CAAC;YAE5D,+BAA+B;YAC/B,MAAM,QAAQ,GAAG,4BAA4B,CAAC;YAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC5C,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5B,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;QAED,uCAAuC;QACvC,IAAI,SAA0C,CAAC;QAC/C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,yBAAyB;gBACzB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAC7D,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC/C,kCAAkC;gBAClC,SAAS,GAAG,iBAAiB,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,GAAG,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACtD,CAAC;QAED,kDAAkD;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEpF,iDAAiD;QACjD,MAAM,gBAAgB,GAAG,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,OAAO;YACP,UAAU,EAAE,gBAAgB;YAC5B,WAAW,EAAE,YAAY;YACzB,OAAO;YACP,WAAW;YACX,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,MAAM,OAAO,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAc,EACd,UAAkD,EAAE;IAEpD,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IACjD,MAAM,OAAO,GAAoD,EAAE,CAAC;IAEpE,qBAAqB;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CACtC,CAAC;QAEF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACjC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;oBACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,eAAe;iBACjD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3J,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGtD,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAA2D,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAuC,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAmC,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAwB,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAoC,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,EACF,MAAM,GAAG,KAAK,EACd,OAAO,GAAG,KAAK,EACf,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,UAAU,EACnB,OAAO,GAAG,KAAK,EACf,SAAS,EACT,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,EACR,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,GAAG,KAAK,EACX,OAAO,EACP,OAAO,EACP,SAAS,GACV,GAAG,OAAO,CAAC;IAEZ,8CAA8C;IAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,gDAAgD;IAChD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,8CAA8C;IAC9C,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE;YACxC,YAAY,EAAE,MAAM;YACpB,OAAO;YACP,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,SAAS,EAAE,OAAO;YAClB,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACpH,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAEjK,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAEvG,IAAI,OAAe,CAAC;QACpB,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,QAAQ,GAAQ,EAAE,CAAC;QACvB,IAAI,KAAK,GAAa,EAAE,CAAC;QACzB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,MAAM,EAAE,CAAC;YACX,yBAAyB;YACzB,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;YAC5B,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;YAED,4CAA4C;YAC5C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YACpD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACnB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;YACzB,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YAE5C,kEAAkE;YAClE,IAAI,WAAW,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACtB,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,MAAM;oBACT,OAAO,GAAG,WAAW,CAAC;oBACtB,MAAM;gBACR,KAAK,MAAM;oBACT,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;oBAClC,MAAM;gBACR,KAAK,UAAU,CAAC;gBAChB;oBACE,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC/C,MAAM;YACV,CAAC;YAED,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,+BAA+B;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC5C,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC1C,KAAK,GAAG,eAAe,CAAC;gBAExB,uCAAuC;gBACvC,MAAM,QAAQ,GAAG,4BAA4B,CAAC;gBAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC5C,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,KAAK,GAAG,2BAA2B,CAAC;YACtC,CAAC;YACD,OAAO,GAAG,GAAG,CAAC,CAAC,qCAAqC;QACtD,CAAC;aAAM,IAAI,KAAK,EAAE,CAAC;YACjB,4CAA4C;YAC5C,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE1E,yBAAyB;gBACzB,MAAM,KAAK,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;gBAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,KAAK,GAAG,CAAC,CAAC,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,eAAe,CAAC;oBAC7E,MAAM,SAAS,GAAa,EAAE,CAAC;oBAC/B,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;wBACnB,MAAM,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;wBACrD,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACpG,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAC1F,SAAS,CAAC,IAAI,CAAC,MAAM,SAAS,KAAK,QAAQ,KAAK,QAAQ,EAAE,CAAC,CAAC;wBAC5D,IAAI,QAAQ;4BAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACrC,CAAC,CAAC,CAAC;oBACH,OAAO,GAAG,KAAK,KAAK,OAAO,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;oBAC3B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,cAAc,CAAC;gBACtD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,KAAK,GAAG,cAAc,CAAC;YACzB,CAAC;YACD,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,0CAA0C;YAC1C,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;YAC3B,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,eAAe,CAAC;YAE5D,+BAA+B;YAC/B,MAAM,QAAQ,GAAG,4BAA4B,CAAC;YAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC5C,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5B,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;QAED,uCAAuC;QACvC,IAAI,SAA0C,CAAC;QAC/C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,yBAAyB;gBACzB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAC7D,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC/C,kCAAkC;gBAClC,SAAS,GAAG,iBAAiB,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,GAAG,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACtD,CAAC;QAED,kDAAkD;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEpF,iDAAiD;QACjD,MAAM,gBAAgB,GAAG,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEpE,mDAAmD;QACnD,IAAI,eAAyE,CAAC;QAC9E,IAAI,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;oBACjC,SAAS,EAAE,SAAS,IAAI,SAAS;iBAClC,CAAC,CAAC;gBAEH,sBAAsB;gBACtB,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE;oBAC/B,SAAS,EAAE,kBAAkB;oBAC7B,OAAO,EAAE,OAAO,IAAI,KAAK;iBAC1B,CAAC,CAAC;gBAEH,oBAAoB;gBACpB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;oBACb,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;gBAED,mBAAmB;gBACnB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;gBAC/D,eAAe,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;gBAE9C,WAAW;gBACX,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,IAAI,YAA0E,CAAC;QAC/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;gBAClE,YAAY,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;YAC1E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAED,mCAAmC;QACnC,IAAI,WAA+B,CAAC;QACpC,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAC7D,MAAM,aAAa,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM;oBACjF,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM;oBACxB,CAAC,CAAC,mEAAmE,CAAC;gBACxE,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC,SAAS;oBAChF,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS;oBAC3B,CAAC,CAAC,GAAG,CAAC;gBAER,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE;oBAC3C,MAAM,EAAE,GAAG,aAAa,kBAAkB,SAAS,SAAS;oBAC5D,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM;oBAC7B,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK;oBAC3B,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO;iBAChC,CAAC,CAAC;gBACH,WAAW,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAW,CAAC;YACrE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAED,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,OAAO;YACP,UAAU,EAAE,gBAAgB;YAC5B,WAAW,EAAE,YAAY;YACzB,OAAO;YACP,WAAW;YACX,SAAS;YACT,QAAQ,EAAE,eAAe;YACzB,cAAc,EAAE,YAAY;YAC5B,OAAO,EAAE,WAAW;SACrB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,MAAM,OAAO,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAc,EACd,UAAkD,EAAE;IAEpD,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IACjD,MAAM,OAAO,GAAoD,EAAE,CAAC;IAEpE,qBAAqB;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CACtC,CAAC;QAEF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACjC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;oBACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,eAAe;iBACjD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
|
package/dist/mcp/server.js
CHANGED
|
@@ -20,6 +20,30 @@ try {
|
|
|
20
20
|
pkgVersion = pkg.version;
|
|
21
21
|
}
|
|
22
22
|
catch { /* fallback */ }
|
|
23
|
+
/**
|
|
24
|
+
* Helper function to extract colors from content
|
|
25
|
+
*/
|
|
26
|
+
function extractColorsFromContent(content) {
|
|
27
|
+
const colors = [];
|
|
28
|
+
const hexRegex = /#[0-9A-Fa-f]{6}|#[0-9A-Fa-f]{3}/g;
|
|
29
|
+
const matches = content.match(hexRegex);
|
|
30
|
+
if (matches) {
|
|
31
|
+
colors.push(...[...new Set(matches)].slice(0, 10));
|
|
32
|
+
}
|
|
33
|
+
return colors;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Helper function to extract font information from content
|
|
37
|
+
*/
|
|
38
|
+
function extractFontsFromContent(content) {
|
|
39
|
+
const fonts = [];
|
|
40
|
+
const fontRegex = /font-family:\s*([^;}"'\n]+)/gi;
|
|
41
|
+
let match;
|
|
42
|
+
while ((match = fontRegex.exec(content)) !== null) {
|
|
43
|
+
fonts.push(match[1].trim());
|
|
44
|
+
}
|
|
45
|
+
return [...new Set(fonts)].slice(0, 5);
|
|
46
|
+
}
|
|
23
47
|
const server = new Server({
|
|
24
48
|
name: 'webpeel',
|
|
25
49
|
version: pkgVersion,
|
|
@@ -415,6 +439,103 @@ const tools = [
|
|
|
415
439
|
required: ['url'],
|
|
416
440
|
},
|
|
417
441
|
},
|
|
442
|
+
{
|
|
443
|
+
name: 'webpeel_brand',
|
|
444
|
+
description: 'Extract branding and design system from a URL. Returns colors, fonts, typography, and visual identity elements.',
|
|
445
|
+
annotations: {
|
|
446
|
+
title: 'Extract Branding',
|
|
447
|
+
readOnlyHint: true,
|
|
448
|
+
destructiveHint: false,
|
|
449
|
+
idempotentHint: true,
|
|
450
|
+
openWorldHint: true,
|
|
451
|
+
},
|
|
452
|
+
inputSchema: {
|
|
453
|
+
type: 'object',
|
|
454
|
+
properties: {
|
|
455
|
+
url: {
|
|
456
|
+
type: 'string',
|
|
457
|
+
description: 'URL to extract branding from',
|
|
458
|
+
},
|
|
459
|
+
render: {
|
|
460
|
+
type: 'boolean',
|
|
461
|
+
description: 'Use browser rendering',
|
|
462
|
+
default: false,
|
|
463
|
+
},
|
|
464
|
+
},
|
|
465
|
+
required: ['url'],
|
|
466
|
+
},
|
|
467
|
+
},
|
|
468
|
+
{
|
|
469
|
+
name: 'webpeel_change_track',
|
|
470
|
+
description: 'Track changes on a URL by generating a content fingerprint. Use this to detect when a page has been updated.',
|
|
471
|
+
annotations: {
|
|
472
|
+
title: 'Track Page Changes',
|
|
473
|
+
readOnlyHint: true,
|
|
474
|
+
destructiveHint: false,
|
|
475
|
+
idempotentHint: true,
|
|
476
|
+
openWorldHint: true,
|
|
477
|
+
},
|
|
478
|
+
inputSchema: {
|
|
479
|
+
type: 'object',
|
|
480
|
+
properties: {
|
|
481
|
+
url: {
|
|
482
|
+
type: 'string',
|
|
483
|
+
description: 'URL to track for changes',
|
|
484
|
+
},
|
|
485
|
+
render: {
|
|
486
|
+
type: 'boolean',
|
|
487
|
+
description: 'Use browser rendering',
|
|
488
|
+
default: false,
|
|
489
|
+
},
|
|
490
|
+
},
|
|
491
|
+
required: ['url'],
|
|
492
|
+
},
|
|
493
|
+
},
|
|
494
|
+
{
|
|
495
|
+
name: 'webpeel_summarize',
|
|
496
|
+
description: 'Generate an AI-powered summary of a webpage using an LLM. Requires an OpenAI-compatible API key.',
|
|
497
|
+
annotations: {
|
|
498
|
+
title: 'Summarize Page',
|
|
499
|
+
readOnlyHint: true,
|
|
500
|
+
destructiveHint: false,
|
|
501
|
+
idempotentHint: true,
|
|
502
|
+
openWorldHint: true,
|
|
503
|
+
},
|
|
504
|
+
inputSchema: {
|
|
505
|
+
type: 'object',
|
|
506
|
+
properties: {
|
|
507
|
+
url: {
|
|
508
|
+
type: 'string',
|
|
509
|
+
description: 'URL to summarize',
|
|
510
|
+
},
|
|
511
|
+
llmApiKey: {
|
|
512
|
+
type: 'string',
|
|
513
|
+
description: 'API key for LLM (OpenAI-compatible)',
|
|
514
|
+
},
|
|
515
|
+
prompt: {
|
|
516
|
+
type: 'string',
|
|
517
|
+
description: 'Custom summary prompt (default: "Summarize this webpage in 2-3 sentences.")',
|
|
518
|
+
default: 'Summarize this webpage in 2-3 sentences.',
|
|
519
|
+
},
|
|
520
|
+
llmModel: {
|
|
521
|
+
type: 'string',
|
|
522
|
+
description: 'LLM model to use (default: gpt-4o-mini)',
|
|
523
|
+
default: 'gpt-4o-mini',
|
|
524
|
+
},
|
|
525
|
+
llmBaseUrl: {
|
|
526
|
+
type: 'string',
|
|
527
|
+
description: 'LLM API base URL (default: https://api.openai.com/v1)',
|
|
528
|
+
default: 'https://api.openai.com/v1',
|
|
529
|
+
},
|
|
530
|
+
render: {
|
|
531
|
+
type: 'boolean',
|
|
532
|
+
description: 'Use browser rendering',
|
|
533
|
+
default: false,
|
|
534
|
+
},
|
|
535
|
+
},
|
|
536
|
+
required: ['url', 'llmApiKey'],
|
|
537
|
+
},
|
|
538
|
+
},
|
|
418
539
|
];
|
|
419
540
|
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
420
541
|
tools,
|
|
@@ -794,6 +915,175 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
794
915
|
],
|
|
795
916
|
};
|
|
796
917
|
}
|
|
918
|
+
if (name === 'webpeel_brand') {
|
|
919
|
+
const { url, render } = args;
|
|
920
|
+
// SECURITY: Validate input parameters
|
|
921
|
+
if (!url || typeof url !== 'string') {
|
|
922
|
+
throw new Error('Invalid URL parameter');
|
|
923
|
+
}
|
|
924
|
+
if (url.length > 2048) {
|
|
925
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
926
|
+
}
|
|
927
|
+
const options = {
|
|
928
|
+
render: render || false,
|
|
929
|
+
extract: {
|
|
930
|
+
selectors: {
|
|
931
|
+
primaryColor: 'meta[name="theme-color"]',
|
|
932
|
+
title: 'title',
|
|
933
|
+
logo: 'img[class*="logo"], img[alt*="logo"]',
|
|
934
|
+
},
|
|
935
|
+
},
|
|
936
|
+
};
|
|
937
|
+
// SECURITY: Wrap in timeout (60 seconds max)
|
|
938
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
939
|
+
setTimeout(() => reject(new Error('Brand extraction timed out after 60s')), 60000);
|
|
940
|
+
});
|
|
941
|
+
const result = await Promise.race([
|
|
942
|
+
peel(url, options),
|
|
943
|
+
timeoutPromise,
|
|
944
|
+
]);
|
|
945
|
+
// Extract branding info
|
|
946
|
+
const branding = {
|
|
947
|
+
url: result.url,
|
|
948
|
+
title: result.title,
|
|
949
|
+
extracted: result.extracted,
|
|
950
|
+
metadata: result.metadata,
|
|
951
|
+
colors: extractColorsFromContent(result.content),
|
|
952
|
+
fonts: extractFontsFromContent(result.content),
|
|
953
|
+
};
|
|
954
|
+
// SECURITY: Handle JSON serialization errors
|
|
955
|
+
let resultText;
|
|
956
|
+
try {
|
|
957
|
+
resultText = JSON.stringify(branding, null, 2);
|
|
958
|
+
}
|
|
959
|
+
catch (jsonError) {
|
|
960
|
+
resultText = JSON.stringify({
|
|
961
|
+
error: 'serialization_error',
|
|
962
|
+
message: 'Failed to serialize branding result',
|
|
963
|
+
});
|
|
964
|
+
}
|
|
965
|
+
return {
|
|
966
|
+
content: [
|
|
967
|
+
{
|
|
968
|
+
type: 'text',
|
|
969
|
+
text: resultText,
|
|
970
|
+
},
|
|
971
|
+
],
|
|
972
|
+
};
|
|
973
|
+
}
|
|
974
|
+
if (name === 'webpeel_change_track') {
|
|
975
|
+
const { url, render } = args;
|
|
976
|
+
// SECURITY: Validate input parameters
|
|
977
|
+
if (!url || typeof url !== 'string') {
|
|
978
|
+
throw new Error('Invalid URL parameter');
|
|
979
|
+
}
|
|
980
|
+
if (url.length > 2048) {
|
|
981
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
982
|
+
}
|
|
983
|
+
const options = {
|
|
984
|
+
render: render || false,
|
|
985
|
+
};
|
|
986
|
+
// SECURITY: Wrap in timeout (60 seconds max)
|
|
987
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
988
|
+
setTimeout(() => reject(new Error('Change tracking timed out after 60s')), 60000);
|
|
989
|
+
});
|
|
990
|
+
const result = await Promise.race([
|
|
991
|
+
peel(url, options),
|
|
992
|
+
timeoutPromise,
|
|
993
|
+
]);
|
|
994
|
+
// Return tracking info
|
|
995
|
+
const trackingInfo = {
|
|
996
|
+
url: result.url,
|
|
997
|
+
title: result.title,
|
|
998
|
+
fingerprint: result.fingerprint,
|
|
999
|
+
tokens: result.tokens,
|
|
1000
|
+
contentType: result.contentType,
|
|
1001
|
+
lastChecked: new Date().toISOString(),
|
|
1002
|
+
};
|
|
1003
|
+
// SECURITY: Handle JSON serialization errors
|
|
1004
|
+
let resultText;
|
|
1005
|
+
try {
|
|
1006
|
+
resultText = JSON.stringify(trackingInfo, null, 2);
|
|
1007
|
+
}
|
|
1008
|
+
catch (jsonError) {
|
|
1009
|
+
resultText = JSON.stringify({
|
|
1010
|
+
error: 'serialization_error',
|
|
1011
|
+
message: 'Failed to serialize tracking result',
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
return {
|
|
1015
|
+
content: [
|
|
1016
|
+
{
|
|
1017
|
+
type: 'text',
|
|
1018
|
+
text: resultText,
|
|
1019
|
+
},
|
|
1020
|
+
],
|
|
1021
|
+
};
|
|
1022
|
+
}
|
|
1023
|
+
if (name === 'webpeel_summarize') {
|
|
1024
|
+
const { url, llmApiKey, prompt, llmModel, llmBaseUrl, render, } = args;
|
|
1025
|
+
// SECURITY: Validate input parameters
|
|
1026
|
+
if (!url || typeof url !== 'string') {
|
|
1027
|
+
throw new Error('Invalid URL parameter');
|
|
1028
|
+
}
|
|
1029
|
+
if (url.length > 2048) {
|
|
1030
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
1031
|
+
}
|
|
1032
|
+
if (!llmApiKey || typeof llmApiKey !== 'string') {
|
|
1033
|
+
throw new Error('Invalid llmApiKey parameter: must be a string');
|
|
1034
|
+
}
|
|
1035
|
+
if (prompt !== undefined && typeof prompt !== 'string') {
|
|
1036
|
+
throw new Error('Invalid prompt parameter: must be a string');
|
|
1037
|
+
}
|
|
1038
|
+
if (llmModel !== undefined && typeof llmModel !== 'string') {
|
|
1039
|
+
throw new Error('Invalid llmModel parameter: must be a string');
|
|
1040
|
+
}
|
|
1041
|
+
if (llmBaseUrl !== undefined && typeof llmBaseUrl !== 'string') {
|
|
1042
|
+
throw new Error('Invalid llmBaseUrl parameter: must be a string');
|
|
1043
|
+
}
|
|
1044
|
+
const options = {
|
|
1045
|
+
render: render || false,
|
|
1046
|
+
extract: {
|
|
1047
|
+
prompt: prompt || 'Summarize this webpage in 2-3 sentences.',
|
|
1048
|
+
llmApiKey,
|
|
1049
|
+
llmModel: llmModel || 'gpt-4o-mini',
|
|
1050
|
+
llmBaseUrl: llmBaseUrl || 'https://api.openai.com/v1',
|
|
1051
|
+
},
|
|
1052
|
+
};
|
|
1053
|
+
// SECURITY: Wrap in timeout (60 seconds max)
|
|
1054
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
1055
|
+
setTimeout(() => reject(new Error('Summarize operation timed out after 60s')), 60000);
|
|
1056
|
+
});
|
|
1057
|
+
const result = await Promise.race([
|
|
1058
|
+
peel(url, options),
|
|
1059
|
+
timeoutPromise,
|
|
1060
|
+
]);
|
|
1061
|
+
// Return summary
|
|
1062
|
+
const summaryResult = {
|
|
1063
|
+
url: result.url,
|
|
1064
|
+
title: result.title,
|
|
1065
|
+
summary: result.extracted,
|
|
1066
|
+
};
|
|
1067
|
+
// SECURITY: Handle JSON serialization errors
|
|
1068
|
+
let resultText;
|
|
1069
|
+
try {
|
|
1070
|
+
resultText = JSON.stringify(summaryResult, null, 2);
|
|
1071
|
+
}
|
|
1072
|
+
catch (jsonError) {
|
|
1073
|
+
resultText = JSON.stringify({
|
|
1074
|
+
error: 'serialization_error',
|
|
1075
|
+
message: 'Failed to serialize summary result',
|
|
1076
|
+
});
|
|
1077
|
+
}
|
|
1078
|
+
return {
|
|
1079
|
+
content: [
|
|
1080
|
+
{
|
|
1081
|
+
type: 'text',
|
|
1082
|
+
text: resultText,
|
|
1083
|
+
},
|
|
1084
|
+
],
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
797
1087
|
throw new Error(`Unknown tool: ${name}`);
|
|
798
1088
|
}
|
|
799
1089
|
catch (error) {
|