webpeel 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/core/map.js CHANGED
@@ -6,19 +6,58 @@ import { discoverSitemap } from './sitemap.js';
6
6
  import { peel } from '../index.js';
7
7
  export async function mapDomain(startUrl, options = {}) {
8
8
  const startTime = Date.now();
9
- const { useSitemap = true, crawlHomepage = true, maxUrls = 5000, timeout = 10000, includePatterns = [], excludePatterns = [], } = options;
9
+ const { useSitemap = true, crawlHomepage = true, maxUrls = 5000, timeout = 10000, includePatterns = [], excludePatterns = [], search, contentTypeFilter = [], } = options;
10
10
  const urlObj = new URL(startUrl);
11
11
  const domain = urlObj.hostname;
12
- const allUrls = new Set();
12
+ const allUrls = new Map(); // URL -> relevance score
13
13
  let sitemapUrls = [];
14
14
  // Compile filter patterns
15
15
  const includeRegexes = includePatterns.map(p => new RegExp(p));
16
16
  const excludeRegexes = excludePatterns.map(p => new RegExp(p));
17
+ // Parse search terms
18
+ const searchTerms = search ? search.toLowerCase().split(/\s+/).filter(t => t.length > 0) : [];
19
+ /**
20
+ * Calculate relevance score for a URL based on search terms
21
+ * Scores based on matches in URL path and title/description if available
22
+ */
23
+ function calculateRelevance(url, title, description) {
24
+ if (searchTerms.length === 0)
25
+ return 1; // No search = all equal
26
+ let score = 0;
27
+ const urlLower = url.toLowerCase();
28
+ const titleLower = (title || '').toLowerCase();
29
+ const descLower = (description || '').toLowerCase();
30
+ for (const term of searchTerms) {
31
+ // URL path matches (highest weight)
32
+ if (urlLower.includes(term))
33
+ score += 3;
34
+ // Title matches
35
+ if (titleLower.includes(term))
36
+ score += 2;
37
+ // Description matches
38
+ if (descLower.includes(term))
39
+ score += 1;
40
+ }
41
+ return score;
42
+ }
43
+ /**
44
+ * Check if URL should be included based on patterns and content type
45
+ */
17
46
  function shouldInclude(url) {
18
47
  if (excludeRegexes.some(r => r.test(url)))
19
48
  return false;
20
49
  if (includeRegexes.length > 0 && !includeRegexes.some(r => r.test(url)))
21
50
  return false;
51
+ // Content type filter (check file extension)
52
+ if (contentTypeFilter.length > 0) {
53
+ const ext = url.split('.').pop()?.toLowerCase() || '';
54
+ const hasMatch = contentTypeFilter.some(type => {
55
+ const typeExt = type.replace(/^\./, '').toLowerCase();
56
+ return ext === typeExt || url.toLowerCase().includes(`.${typeExt}`);
57
+ });
58
+ if (!hasMatch)
59
+ return false;
60
+ }
22
61
  return true;
23
62
  }
24
63
  // Step 1: Sitemap discovery
@@ -29,7 +68,9 @@ export async function mapDomain(startUrl, options = {}) {
29
68
  if (allUrls.size >= maxUrls)
30
69
  break;
31
70
  if (shouldInclude(entry.url)) {
32
- allUrls.add(entry.url);
71
+ // Sitemap entries don't have title/description, just score by URL
72
+ const score = calculateRelevance(entry.url);
73
+ allUrls.set(entry.url, score);
33
74
  }
34
75
  }
35
76
  }
@@ -43,7 +84,10 @@ export async function mapDomain(startUrl, options = {}) {
43
84
  try {
44
85
  const linkUrl = new URL(link);
45
86
  if (linkUrl.hostname === domain && shouldInclude(link)) {
46
- allUrls.add(link);
87
+ const score = calculateRelevance(link);
88
+ if (!allUrls.has(link)) {
89
+ allUrls.set(link, score);
90
+ }
47
91
  }
48
92
  }
49
93
  catch { /* skip invalid URLs */ }
@@ -51,8 +95,19 @@ export async function mapDomain(startUrl, options = {}) {
51
95
  }
52
96
  catch { /* skip homepage crawl errors */ }
53
97
  }
98
+ // Sort URLs by relevance score (highest first), then alphabetically
99
+ const sortedUrls = Array.from(allUrls.entries())
100
+ .sort((a, b) => {
101
+ if (search) {
102
+ // Sort by score first if searching
103
+ if (b[1] !== a[1])
104
+ return b[1] - a[1];
105
+ }
106
+ return a[0].localeCompare(b[0]);
107
+ })
108
+ .map(([url]) => url);
54
109
  return {
55
- urls: Array.from(allUrls).sort(),
110
+ urls: sortedUrls,
56
111
  sitemapUrls,
57
112
  total: allUrls.size,
58
113
  elapsed: Date.now() - startTime,
@@ -1 +1 @@
1
- {"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/core/map.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AA4BnC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,UAAsB,EAAE;IACxE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,EACJ,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,IAAI,EACpB,OAAO,GAAG,IAAI,EACd,OAAO,GAAG,KAAK,EACf,eAAe,GAAG,EAAE,EACpB,eAAe,GAAG,EAAE,GACrB,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,WAAW,GAAa,EAAE,CAAC;IAE/B,0BAA0B;IAC1B,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,SAAS,aAAa,CAAC,GAAW;QAChC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACtF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4BAA4B;IAC5B,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACpE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO;gBAAE,MAAM;YACnC,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,IAAI,aAAa,IAAI,OAAO,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO;oBAAE,MAAM;gBACnC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,IAAI,OAAO,CAAC,QAAQ,KAAK,MAAM,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,gCAAgC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QAChC,WAAW;QACX,KAAK,EAAE,OAAO,CAAC,IAAI;QACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAChC,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/core/map.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAgCnC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,UAAsB,EAAE;IACxE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,EACJ,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,IAAI,EACpB,OAAO,GAAG,IAAI,EACd,OAAO,GAAG,KAAK,EACf,eAAe,GAAG,EAAE,EACpB,eAAe,GAAG,EAAE,EACpB,MAAM,EACN,iBAAiB,GAAG,EAAE,GACvB,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,yBAAyB;IACpE,IAAI,WAAW,GAAa,EAAE,CAAC;IAE/B,0BAA0B;IAC1B,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,qBAAqB;IACrB,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9F;;;OAGG;IACH,SAAS,kBAAkB,CAAC,GAAW,EAAE,KAAc,EAAE,WAAoB;QAC3E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,wBAAwB;QAEhE,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC/C,MAAM,SAAS,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAEpD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,oCAAoC;YACpC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;YACxC,gBAAgB;YAChB,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;YAC1C,sBAAsB;YACtB,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,SAAS,aAAa,CAAC,GAAW;QAChC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAEtF,6CAA6C;QAC7C,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACtD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;gBACtD,OAAO,GAAG,KAAK,OAAO,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ;gBAAE,OAAO,KAAK,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4BAA4B;IAC5B,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACpE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO;gBAAE,MAAM;YACnC,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,kEAAkE;gBAClE,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,IAAI,aAAa,IAAI,OAAO,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO;oBAAE,MAAM;gBACnC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,IAAI,OAAO,CAAC,QAAQ,KAAK,MAAM,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvD,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;wBACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;4BACvB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;wBAC3B,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,gCAAgC,CAAC,CAAC;IAC9C,CAAC;IAED,oEAAoE;IACpE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;SAC7C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,MAAM,EAAE,CAAC;YACX,mCAAmC;YACnC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,WAAW;QACX,KAAK,EAAE,OAAO,CAAC,IAAI;QACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAChC,CAAC;AACJ,CAAC"}
package/dist/index.d.ts CHANGED
@@ -9,6 +9,9 @@ export * from './types.js';
9
9
  export { crawl, type CrawlOptions, type CrawlResult, type CrawlProgress } from './core/crawler.js';
10
10
  export { discoverSitemap, type SitemapUrl, type SitemapResult } from './core/sitemap.js';
11
11
  export { mapDomain, type MapOptions, type MapResult } from './core/map.js';
12
+ export { extractBranding, type BrandingProfile } from './core/branding.js';
13
+ export { trackChange, getSnapshot, clearSnapshots, type ChangeResult, type Snapshot } from './core/change-tracking.js';
14
+ export { extractWithLLM } from './core/extract.js';
12
15
  /**
13
16
  * Fetch and extract content from a URL
14
17
  *
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAE3E;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAkNtF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CA8RtF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
package/dist/index.js CHANGED
@@ -13,6 +13,9 @@ export * from './types.js';
13
13
  export { crawl } from './core/crawler.js';
14
14
  export { discoverSitemap } from './core/sitemap.js';
15
15
  export { mapDomain } from './core/map.js';
16
+ export { extractBranding } from './core/branding.js';
17
+ export { trackChange, getSnapshot, clearSnapshots } from './core/change-tracking.js';
18
+ export { extractWithLLM } from './core/extract.js';
16
19
  /**
17
20
  * Fetch and extract content from a URL
18
21
  *
@@ -49,6 +52,10 @@ export async function peel(url, options = {}) {
49
52
  if (actions && actions.length > 0) {
50
53
  render = true;
51
54
  }
55
+ // If branding is requested, force render mode
56
+ if (options.branding) {
57
+ render = true;
58
+ }
52
59
  try {
53
60
  // Fetch the page
54
61
  const fetchResult = await smartFetch(url, {
@@ -188,6 +195,69 @@ export async function peel(url, options = {}) {
188
195
  const fingerprint = createHash('sha256').update(content).digest('hex').slice(0, 16);
189
196
  // Convert screenshot buffer to base64 if present
190
197
  const screenshotBase64 = fetchResult.screenshot?.toString('base64');
198
+ // Extract branding if requested (requires browser)
199
+ let brandingProfile;
200
+ if (options.branding && render) {
201
+ try {
202
+ // Import playwright and create a page for branding extraction
203
+ const { chromium } = await import('playwright-core');
204
+ const browser = await chromium.launch({ headless: true });
205
+ const page = await browser.newPage({
206
+ userAgent: userAgent || undefined,
207
+ });
208
+ // Navigate to the URL
209
+ await page.goto(fetchResult.url, {
210
+ waitUntil: 'domcontentloaded',
211
+ timeout: timeout || 30000,
212
+ });
213
+ // Wait if specified
214
+ if (wait > 0) {
215
+ await page.waitForTimeout(wait);
216
+ }
217
+ // Extract branding
218
+ const { extractBranding } = await import('./core/branding.js');
219
+ brandingProfile = await extractBranding(page);
220
+ // Clean up
221
+ await browser.close();
222
+ }
223
+ catch (error) {
224
+ console.error('Branding extraction failed:', error);
225
+ }
226
+ }
227
+ // Track content changes if requested
228
+ let changeResult;
229
+ if (options.changeTracking) {
230
+ try {
231
+ const { trackChange } = await import('./core/change-tracking.js');
232
+ changeResult = await trackChange(fetchResult.url, content, fingerprint);
233
+ }
234
+ catch (error) {
235
+ console.error('Change tracking failed:', error);
236
+ }
237
+ }
238
+ // Generate AI summary if requested
239
+ let summaryText;
240
+ if (options.summary && options.llm) {
241
+ try {
242
+ const { extractWithLLM } = await import('./core/extract.js');
243
+ const summaryPrompt = typeof options.summary === 'object' && options.summary.prompt
244
+ ? options.summary.prompt
245
+ : 'Summarize the main points of this content in a concise paragraph.';
246
+ const maxLength = typeof options.summary === 'object' && options.summary.maxLength
247
+ ? options.summary.maxLength
248
+ : 200;
249
+ const result = await extractWithLLM(content, {
250
+ prompt: `${summaryPrompt} Keep it under ${maxLength} words.`,
251
+ llmApiKey: options.llm.apiKey,
252
+ llmModel: options.llm.model,
253
+ llmBaseUrl: options.llm.baseUrl,
254
+ });
255
+ summaryText = result.summary || Object.values(result)[0];
256
+ }
257
+ catch (error) {
258
+ console.error('Summary generation failed:', error);
259
+ }
260
+ }
191
261
  return {
192
262
  url: fetchResult.url,
193
263
  title,
@@ -202,6 +272,9 @@ export async function peel(url, options = {}) {
202
272
  quality,
203
273
  fingerprint,
204
274
  extracted,
275
+ branding: brandingProfile,
276
+ changeTracking: changeResult,
277
+ summary: summaryText,
205
278
  };
206
279
  }
207
280
  catch (error) {
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3J,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGtD,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAA2D,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAuC,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAmC,MAAM,eAAe,CAAC;AAE3E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,EACF,MAAM,GAAG,KAAK,EACd,OAAO,GAAG,KAAK,EACf,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,UAAU,EACnB,OAAO,GAAG,KAAK,EACf,SAAS,EACT,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,EACR,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,GAAG,KAAK,EACX,OAAO,EACP,OAAO,EACP,SAAS,GACV,GAAG,OAAO,CAAC;IAEZ,8CAA8C;IAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,gDAAgD;IAChD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE;YACxC,YAAY,EAAE,MAAM;YACpB,OAAO;YACP,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,SAAS,EAAE,OAAO;YAClB,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACpH,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAEjK,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAEvG,IAAI,OAAe,CAAC;QACpB,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,QAAQ,GAAQ,EAAE,CAAC;QACvB,IAAI,KAAK,GAAa,EAAE,CAAC;QACzB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,MAAM,EAAE,CAAC;YACX,yBAAyB;YACzB,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;YAC5B,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;YAED,4CAA4C;YAC5C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YACpD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACnB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;YACzB,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YAE5C,kEAAkE;YAClE,IAAI,WAAW,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACtB,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,MAAM;oBACT,OAAO,GAAG,WAAW,CAAC;oBACtB,MAAM;gBACR,KAAK,MAAM;oBACT,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;oBAClC,MAAM;gBACR,KAAK,UAAU,CAAC;gBAChB;oBACE,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC/C,MAAM;YACV,CAAC;YAED,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,+BAA+B;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC5C,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC1C,KAAK,GAAG,eAAe,CAAC;gBAExB,uCAAuC;gBACvC,MAAM,QAAQ,GAAG,4BAA4B,CAAC;gBAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC5C,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,KAAK,GAAG,2BAA2B,CAAC;YACtC,CAAC;YACD,OAAO,GAAG,GAAG,CAAC,CAAC,qCAAqC;QACtD,CAAC;aAAM,IAAI,KAAK,EAAE,CAAC;YACjB,4CAA4C;YAC5C,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE1E,yBAAyB;gBACzB,MAAM,KAAK,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;gBAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,KAAK,GAAG,CAAC,CAAC,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,eAAe,CAAC;oBAC7E,MAAM,SAAS,GAAa,EAAE,CAAC;oBAC/B,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;wBACnB,MAAM,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;wBACrD,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACpG,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAC1F,SAAS,CAAC,IAAI,CAAC,MAAM,SAAS,KAAK,QAAQ,KAAK,QAAQ,EAAE,CAAC,CAAC;wBAC5D,IAAI,QAAQ;4BAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACrC,CAAC,CAAC,CAAC;oBACH,OAAO,GAAG,KAAK,KAAK,OAAO,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;oBAC3B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,cAAc,CAAC;gBACtD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,KAAK,GAAG,cAAc,CAAC;YACzB,CAAC;YACD,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,0CAA0C;YAC1C,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;YAC3B,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,eAAe,CAAC;YAE5D,+BAA+B;YAC/B,MAAM,QAAQ,GAAG,4BAA4B,CAAC;YAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC5C,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5B,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;QAED,uCAAuC;QACvC,IAAI,SAA0C,CAAC;QAC/C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,yBAAyB;gBACzB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAC7D,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC/C,kCAAkC;gBAClC,SAAS,GAAG,iBAAiB,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,GAAG,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACtD,CAAC;QAED,kDAAkD;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEpF,iDAAiD;QACjD,MAAM,gBAAgB,GAAG,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,OAAO;YACP,UAAU,EAAE,gBAAgB;YAC5B,WAAW,EAAE,YAAY;YACzB,OAAO;YACP,WAAW;YACX,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,MAAM,OAAO,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAc,EACd,UAAkD,EAAE;IAEpD,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IACjD,MAAM,OAAO,GAAoD,EAAE,CAAC;IAEpE,qBAAqB;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CACtC,CAAC;QAEF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACjC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;oBACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,eAAe;iBACjD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3J,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAGtD,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAA2D,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAuC,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAmC,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAwB,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAoC,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,EACF,MAAM,GAAG,KAAK,EACd,OAAO,GAAG,KAAK,EACf,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,UAAU,EACnB,OAAO,GAAG,KAAK,EACf,SAAS,EACT,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,EACR,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,GAAG,KAAK,EACX,OAAO,EACP,OAAO,EACP,SAAS,GACV,GAAG,OAAO,CAAC;IAEZ,8CAA8C;IAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,gDAAgD;IAChD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,8CAA8C;IAC9C,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE;YACxC,YAAY,EAAE,MAAM;YACpB,OAAO;YACP,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,SAAS,EAAE,OAAO;YAClB,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACpH,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,WAAW,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAEjK,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAEvG,IAAI,OAAe,CAAC;QACpB,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,QAAQ,GAAQ,EAAE,CAAC;QACvB,IAAI,KAAK,GAAa,EAAE,CAAC;QACzB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,MAAM,EAAE,CAAC;YACX,yBAAyB;YACzB,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;YAC5B,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;YAED,4CAA4C;YAC5C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YACpD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACnB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;YACzB,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YAE5C,kEAAkE;YAClE,IAAI,WAAW,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACtB,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,MAAM;oBACT,OAAO,GAAG,WAAW,CAAC;oBACtB,MAAM;gBACR,KAAK,MAAM;oBACT,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;oBAClC,MAAM;gBACR,KAAK,UAAU,CAAC;gBAChB;oBACE,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC/C,MAAM;YACV,CAAC;YAED,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,+BAA+B;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC5C,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC1C,KAAK,GAAG,eAAe,CAAC;gBAExB,uCAAuC;gBACvC,MAAM,QAAQ,GAAG,4BAA4B,CAAC;gBAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC5C,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,KAAK,GAAG,2BAA2B,CAAC;YACtC,CAAC;YACD,OAAO,GAAG,GAAG,CAAC,CAAC,qCAAqC;QACtD,CAAC;aAAM,IAAI,KAAK,EAAE,CAAC;YACjB,4CAA4C;YAC5C,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE1E,yBAAyB;gBACzB,MAAM,KAAK,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;gBAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,KAAK,GAAG,CAAC,CAAC,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,eAAe,CAAC;oBAC7E,MAAM,SAAS,GAAa,EAAE,CAAC;oBAC/B,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;wBACnB,MAAM,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;wBACrD,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACpG,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAC1F,SAAS,CAAC,IAAI,CAAC,MAAM,SAAS,KAAK,QAAQ,KAAK,QAAQ,EAAE,CAAC,CAAC;wBAC5D,IAAI,QAAQ;4BAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACrC,CAAC,CAAC,CAAC;oBACH,OAAO,GAAG,KAAK,KAAK,OAAO,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;oBAC3B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,cAAc,CAAC;gBACtD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,KAAK,GAAG,cAAc,CAAC;YACzB,CAAC;YACD,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,0CAA0C;YAC1C,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;YAC3B,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,eAAe,CAAC;YAE5D,+BAA+B;YAC/B,MAAM,QAAQ,GAAG,4BAA4B,CAAC;YAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC5C,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5B,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;QAED,uCAAuC;QACvC,IAAI,SAA0C,CAAC;QAC/C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,yBAAyB;gBACzB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAC7D,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC/C,kCAAkC;gBAClC,SAAS,GAAG,iBAAiB,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,GAAG,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACtD,CAAC;QAED,kDAAkD;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEpF,iDAAiD;QACjD,MAAM,gBAAgB,GAAG,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEpE,mDAAmD;QACnD,IAAI,eAAyE,CAAC;QAC9E,IAAI,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;oBACjC,SAAS,EAAE,SAAS,IAAI,SAAS;iBAClC,CAAC,CAAC;gBAEH,sBAAsB;gBACtB,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE;oBAC/B,SAAS,EAAE,kBAAkB;oBAC7B,OAAO,EAAE,OAAO,IAAI,KAAK;iBAC1B,CAAC,CAAC;gBAEH,oBAAoB;gBACpB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;oBACb,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;gBAED,mBAAmB;gBACnB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;gBAC/D,eAAe,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;gBAE9C,WAAW;gBACX,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,IAAI,YAA0E,CAAC;QAC/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;gBAClE,YAAY,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;YAC1E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAED,mCAAmC;QACnC,IAAI,WAA+B,CAAC;QACpC,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAC7D,MAAM,aAAa,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM;oBACjF,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM;oBACxB,CAAC,CAAC,mEAAmE,CAAC;gBACxE,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC,SAAS;oBAChF,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS;oBAC3B,CAAC,CAAC,GAAG,CAAC;gBAER,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE;oBAC3C,MAAM,EAAE,GAAG,aAAa,kBAAkB,SAAS,SAAS;oBAC5D,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM;oBAC7B,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK;oBAC3B,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO;iBAChC,CAAC,CAAC;gBACH,WAAW,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAW,CAAC;YACrE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAED,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,OAAO;YACP,UAAU,EAAE,gBAAgB;YAC5B,WAAW,EAAE,YAAY;YACzB,OAAO;YACP,WAAW;YACX,SAAS;YACT,QAAQ,EAAE,eAAe;YACzB,cAAc,EAAE,YAAY;YAC5B,OAAO,EAAE,WAAW;SACrB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,MAAM,OAAO,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAc,EACd,UAAkD,EAAE;IAEpD,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IACjD,MAAM,OAAO,GAAoD,EAAE,CAAC;IAEpE,qBAAqB;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CACtC,CAAC;QAEF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACjC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;oBACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,eAAe;iBACjD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
@@ -20,6 +20,30 @@ try {
20
20
  pkgVersion = pkg.version;
21
21
  }
22
22
  catch { /* fallback */ }
23
+ /**
24
+ * Helper function to extract colors from content
25
+ */
26
+ function extractColorsFromContent(content) {
27
+ const colors = [];
28
+ const hexRegex = /#[0-9A-Fa-f]{6}|#[0-9A-Fa-f]{3}/g;
29
+ const matches = content.match(hexRegex);
30
+ if (matches) {
31
+ colors.push(...[...new Set(matches)].slice(0, 10));
32
+ }
33
+ return colors;
34
+ }
35
+ /**
36
+ * Helper function to extract font information from content
37
+ */
38
+ function extractFontsFromContent(content) {
39
+ const fonts = [];
40
+ const fontRegex = /font-family:\s*([^;}"'\n]+)/gi;
41
+ let match;
42
+ while ((match = fontRegex.exec(content)) !== null) {
43
+ fonts.push(match[1].trim());
44
+ }
45
+ return [...new Set(fonts)].slice(0, 5);
46
+ }
23
47
  const server = new Server({
24
48
  name: 'webpeel',
25
49
  version: pkgVersion,
@@ -415,6 +439,103 @@ const tools = [
415
439
  required: ['url'],
416
440
  },
417
441
  },
442
+ {
443
+ name: 'webpeel_brand',
444
+ description: 'Extract branding and design system from a URL. Returns colors, fonts, typography, and visual identity elements.',
445
+ annotations: {
446
+ title: 'Extract Branding',
447
+ readOnlyHint: true,
448
+ destructiveHint: false,
449
+ idempotentHint: true,
450
+ openWorldHint: true,
451
+ },
452
+ inputSchema: {
453
+ type: 'object',
454
+ properties: {
455
+ url: {
456
+ type: 'string',
457
+ description: 'URL to extract branding from',
458
+ },
459
+ render: {
460
+ type: 'boolean',
461
+ description: 'Use browser rendering',
462
+ default: false,
463
+ },
464
+ },
465
+ required: ['url'],
466
+ },
467
+ },
468
+ {
469
+ name: 'webpeel_change_track',
470
+ description: 'Track changes on a URL by generating a content fingerprint. Use this to detect when a page has been updated.',
471
+ annotations: {
472
+ title: 'Track Page Changes',
473
+ readOnlyHint: true,
474
+ destructiveHint: false,
475
+ idempotentHint: true,
476
+ openWorldHint: true,
477
+ },
478
+ inputSchema: {
479
+ type: 'object',
480
+ properties: {
481
+ url: {
482
+ type: 'string',
483
+ description: 'URL to track for changes',
484
+ },
485
+ render: {
486
+ type: 'boolean',
487
+ description: 'Use browser rendering',
488
+ default: false,
489
+ },
490
+ },
491
+ required: ['url'],
492
+ },
493
+ },
494
+ {
495
+ name: 'webpeel_summarize',
496
+ description: 'Generate an AI-powered summary of a webpage using an LLM. Requires an OpenAI-compatible API key.',
497
+ annotations: {
498
+ title: 'Summarize Page',
499
+ readOnlyHint: true,
500
+ destructiveHint: false,
501
+ idempotentHint: true,
502
+ openWorldHint: true,
503
+ },
504
+ inputSchema: {
505
+ type: 'object',
506
+ properties: {
507
+ url: {
508
+ type: 'string',
509
+ description: 'URL to summarize',
510
+ },
511
+ llmApiKey: {
512
+ type: 'string',
513
+ description: 'API key for LLM (OpenAI-compatible)',
514
+ },
515
+ prompt: {
516
+ type: 'string',
517
+ description: 'Custom summary prompt (default: "Summarize this webpage in 2-3 sentences.")',
518
+ default: 'Summarize this webpage in 2-3 sentences.',
519
+ },
520
+ llmModel: {
521
+ type: 'string',
522
+ description: 'LLM model to use (default: gpt-4o-mini)',
523
+ default: 'gpt-4o-mini',
524
+ },
525
+ llmBaseUrl: {
526
+ type: 'string',
527
+ description: 'LLM API base URL (default: https://api.openai.com/v1)',
528
+ default: 'https://api.openai.com/v1',
529
+ },
530
+ render: {
531
+ type: 'boolean',
532
+ description: 'Use browser rendering',
533
+ default: false,
534
+ },
535
+ },
536
+ required: ['url', 'llmApiKey'],
537
+ },
538
+ },
418
539
  ];
419
540
  server.setRequestHandler(ListToolsRequestSchema, async () => ({
420
541
  tools,
@@ -794,6 +915,175 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
794
915
  ],
795
916
  };
796
917
  }
918
+ if (name === 'webpeel_brand') {
919
+ const { url, render } = args;
920
+ // SECURITY: Validate input parameters
921
+ if (!url || typeof url !== 'string') {
922
+ throw new Error('Invalid URL parameter');
923
+ }
924
+ if (url.length > 2048) {
925
+ throw new Error('URL too long (max 2048 characters)');
926
+ }
927
+ const options = {
928
+ render: render || false,
929
+ extract: {
930
+ selectors: {
931
+ primaryColor: 'meta[name="theme-color"]',
932
+ title: 'title',
933
+ logo: 'img[class*="logo"], img[alt*="logo"]',
934
+ },
935
+ },
936
+ };
937
+ // SECURITY: Wrap in timeout (60 seconds max)
938
+ const timeoutPromise = new Promise((_, reject) => {
939
+ setTimeout(() => reject(new Error('Brand extraction timed out after 60s')), 60000);
940
+ });
941
+ const result = await Promise.race([
942
+ peel(url, options),
943
+ timeoutPromise,
944
+ ]);
945
+ // Extract branding info
946
+ const branding = {
947
+ url: result.url,
948
+ title: result.title,
949
+ extracted: result.extracted,
950
+ metadata: result.metadata,
951
+ colors: extractColorsFromContent(result.content),
952
+ fonts: extractFontsFromContent(result.content),
953
+ };
954
+ // SECURITY: Handle JSON serialization errors
955
+ let resultText;
956
+ try {
957
+ resultText = JSON.stringify(branding, null, 2);
958
+ }
959
+ catch (jsonError) {
960
+ resultText = JSON.stringify({
961
+ error: 'serialization_error',
962
+ message: 'Failed to serialize branding result',
963
+ });
964
+ }
965
+ return {
966
+ content: [
967
+ {
968
+ type: 'text',
969
+ text: resultText,
970
+ },
971
+ ],
972
+ };
973
+ }
974
+ if (name === 'webpeel_change_track') {
975
+ const { url, render } = args;
976
+ // SECURITY: Validate input parameters
977
+ if (!url || typeof url !== 'string') {
978
+ throw new Error('Invalid URL parameter');
979
+ }
980
+ if (url.length > 2048) {
981
+ throw new Error('URL too long (max 2048 characters)');
982
+ }
983
+ const options = {
984
+ render: render || false,
985
+ };
986
+ // SECURITY: Wrap in timeout (60 seconds max)
987
+ const timeoutPromise = new Promise((_, reject) => {
988
+ setTimeout(() => reject(new Error('Change tracking timed out after 60s')), 60000);
989
+ });
990
+ const result = await Promise.race([
991
+ peel(url, options),
992
+ timeoutPromise,
993
+ ]);
994
+ // Return tracking info
995
+ const trackingInfo = {
996
+ url: result.url,
997
+ title: result.title,
998
+ fingerprint: result.fingerprint,
999
+ tokens: result.tokens,
1000
+ contentType: result.contentType,
1001
+ lastChecked: new Date().toISOString(),
1002
+ };
1003
+ // SECURITY: Handle JSON serialization errors
1004
+ let resultText;
1005
+ try {
1006
+ resultText = JSON.stringify(trackingInfo, null, 2);
1007
+ }
1008
+ catch (jsonError) {
1009
+ resultText = JSON.stringify({
1010
+ error: 'serialization_error',
1011
+ message: 'Failed to serialize tracking result',
1012
+ });
1013
+ }
1014
+ return {
1015
+ content: [
1016
+ {
1017
+ type: 'text',
1018
+ text: resultText,
1019
+ },
1020
+ ],
1021
+ };
1022
+ }
1023
+ if (name === 'webpeel_summarize') {
1024
+ const { url, llmApiKey, prompt, llmModel, llmBaseUrl, render, } = args;
1025
+ // SECURITY: Validate input parameters
1026
+ if (!url || typeof url !== 'string') {
1027
+ throw new Error('Invalid URL parameter');
1028
+ }
1029
+ if (url.length > 2048) {
1030
+ throw new Error('URL too long (max 2048 characters)');
1031
+ }
1032
+ if (!llmApiKey || typeof llmApiKey !== 'string') {
1033
+ throw new Error('Invalid llmApiKey parameter: must be a string');
1034
+ }
1035
+ if (prompt !== undefined && typeof prompt !== 'string') {
1036
+ throw new Error('Invalid prompt parameter: must be a string');
1037
+ }
1038
+ if (llmModel !== undefined && typeof llmModel !== 'string') {
1039
+ throw new Error('Invalid llmModel parameter: must be a string');
1040
+ }
1041
+ if (llmBaseUrl !== undefined && typeof llmBaseUrl !== 'string') {
1042
+ throw new Error('Invalid llmBaseUrl parameter: must be a string');
1043
+ }
1044
+ const options = {
1045
+ render: render || false,
1046
+ extract: {
1047
+ prompt: prompt || 'Summarize this webpage in 2-3 sentences.',
1048
+ llmApiKey,
1049
+ llmModel: llmModel || 'gpt-4o-mini',
1050
+ llmBaseUrl: llmBaseUrl || 'https://api.openai.com/v1',
1051
+ },
1052
+ };
1053
+ // SECURITY: Wrap in timeout (60 seconds max)
1054
+ const timeoutPromise = new Promise((_, reject) => {
1055
+ setTimeout(() => reject(new Error('Summarize operation timed out after 60s')), 60000);
1056
+ });
1057
+ const result = await Promise.race([
1058
+ peel(url, options),
1059
+ timeoutPromise,
1060
+ ]);
1061
+ // Return summary
1062
+ const summaryResult = {
1063
+ url: result.url,
1064
+ title: result.title,
1065
+ summary: result.extracted,
1066
+ };
1067
+ // SECURITY: Handle JSON serialization errors
1068
+ let resultText;
1069
+ try {
1070
+ resultText = JSON.stringify(summaryResult, null, 2);
1071
+ }
1072
+ catch (jsonError) {
1073
+ resultText = JSON.stringify({
1074
+ error: 'serialization_error',
1075
+ message: 'Failed to serialize summary result',
1076
+ });
1077
+ }
1078
+ return {
1079
+ content: [
1080
+ {
1081
+ type: 'text',
1082
+ text: resultText,
1083
+ },
1084
+ ],
1085
+ };
1086
+ }
797
1087
  throw new Error(`Unknown tool: ${name}`);
798
1088
  }
799
1089
  catch (error) {