research-powerpack-mcp 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +486 -0
- package/dist/clients/reddit.d.ts +61 -0
- package/dist/clients/reddit.d.ts.map +1 -0
- package/dist/clients/reddit.js +179 -0
- package/dist/clients/reddit.js.map +1 -0
- package/dist/clients/research.d.ts +41 -0
- package/dist/clients/research.d.ts.map +1 -0
- package/dist/clients/research.js +77 -0
- package/dist/clients/research.js.map +1 -0
- package/dist/clients/scraper.d.ts +44 -0
- package/dist/clients/scraper.d.ts.map +1 -0
- package/dist/clients/scraper.js +171 -0
- package/dist/clients/scraper.js.map +1 -0
- package/dist/clients/search.d.ts +46 -0
- package/dist/clients/search.d.ts.map +1 -0
- package/dist/clients/search.js +91 -0
- package/dist/clients/search.js.map +1 -0
- package/dist/config/index.d.ts +59 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +100 -0
- package/dist/config/index.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +152 -0
- package/dist/index.js.map +1 -0
- package/dist/schemas/deep-research.d.ts +100 -0
- package/dist/schemas/deep-research.d.ts.map +1 -0
- package/dist/schemas/deep-research.js +57 -0
- package/dist/schemas/deep-research.js.map +1 -0
- package/dist/schemas/scrape-links.d.ts +38 -0
- package/dist/schemas/scrape-links.d.ts.map +1 -0
- package/dist/schemas/scrape-links.js +26 -0
- package/dist/schemas/scrape-links.js.map +1 -0
- package/dist/schemas/web-search.d.ts +24 -0
- package/dist/schemas/web-search.d.ts.map +1 -0
- package/dist/schemas/web-search.js +12 -0
- package/dist/schemas/web-search.js.map +1 -0
- package/dist/services/file-attachment.d.ts +30 -0
- package/dist/services/file-attachment.d.ts.map +1 -0
- package/dist/services/file-attachment.js +196 -0
- package/dist/services/file-attachment.js.map +1 -0
- package/dist/services/llm-processor.d.ts +19 -0
- package/dist/services/llm-processor.d.ts.map +1 -0
- package/dist/services/llm-processor.js +44 -0
- package/dist/services/llm-processor.js.map +1 -0
- package/dist/services/markdown-cleaner.d.ts +8 -0
- package/dist/services/markdown-cleaner.d.ts.map +1 -0
- package/dist/services/markdown-cleaner.js +56 -0
- package/dist/services/markdown-cleaner.js.map +1 -0
- package/dist/tools/definitions.d.ts +66 -0
- package/dist/tools/definitions.d.ts.map +1 -0
- package/dist/tools/definitions.js +125 -0
- package/dist/tools/definitions.js.map +1 -0
- package/dist/tools/reddit.d.ts +10 -0
- package/dist/tools/reddit.d.ts.map +1 -0
- package/dist/tools/reddit.js +105 -0
- package/dist/tools/reddit.js.map +1 -0
- package/dist/tools/research.d.ts +14 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +126 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/scrape.d.ts +14 -0
- package/dist/tools/scrape.d.ts.map +1 -0
- package/dist/tools/scrape.js +111 -0
- package/dist/tools/scrape.js.map +1 -0
- package/dist/tools/search.d.ts +14 -0
- package/dist/tools/search.d.ts.map +1 -0
- package/dist/tools/search.js +121 -0
- package/dist/tools/search.js.map +1 -0
- package/dist/utils/errors.d.ts +8 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +30 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/markdown-formatter.d.ts +5 -0
- package/dist/utils/markdown-formatter.d.ts.map +1 -0
- package/dist/utils/markdown-formatter.js +15 -0
- package/dist/utils/markdown-formatter.js.map +1 -0
- package/dist/utils/url-aggregator.d.ts +55 -0
- package/dist/utils/url-aggregator.d.ts.map +1 -0
- package/dist/utils/url-aggregator.js +246 -0
- package/dist/utils/url-aggregator.js.map +1 -0
- package/package.json +56 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web Search Tool Handler
|
|
3
|
+
*/
|
|
4
|
+
import { SearchClient } from '../clients/search.js';
|
|
5
|
+
import { aggregateAndRank, buildUrlLookup, lookupUrl, generateEnhancedOutput, markConsensus, } from '../utils/url-aggregator.js';
|
|
6
|
+
import { CTR_WEIGHTS } from '../config/index.js';
|
|
7
|
+
function getPositionScore(position) {
|
|
8
|
+
if (position >= 1 && position <= 10) {
|
|
9
|
+
return CTR_WEIGHTS[position] ?? 0;
|
|
10
|
+
}
|
|
11
|
+
return Math.max(0, 10 - (position - 10) * 0.5);
|
|
12
|
+
}
|
|
13
|
+
export async function handleWebSearch(params, options = {}) {
|
|
14
|
+
const { sessionId, logger } = options;
|
|
15
|
+
const startTime = Date.now();
|
|
16
|
+
try {
|
|
17
|
+
if (sessionId && logger) {
|
|
18
|
+
await logger('info', `Searching for ${params.keywords.length} keyword(s)`, sessionId);
|
|
19
|
+
}
|
|
20
|
+
const client = new SearchClient();
|
|
21
|
+
const response = await client.searchMultiple(params.keywords);
|
|
22
|
+
const aggregation = aggregateAndRank(response.searches, 5);
|
|
23
|
+
const urlLookup = buildUrlLookup(aggregation.rankedUrls);
|
|
24
|
+
const consensusUrls = aggregation.rankedUrls.filter(url => url.frequency >= aggregation.frequencyThreshold);
|
|
25
|
+
let markdown = '';
|
|
26
|
+
if (consensusUrls.length > 0) {
|
|
27
|
+
markdown += generateEnhancedOutput(consensusUrls, params.keywords, aggregation.totalUniqueUrls, aggregation.frequencyThreshold, aggregation.thresholdNote);
|
|
28
|
+
markdown += '\n---\n\n';
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
markdown += `## The Perfect Search Results (Aggregated from ${response.totalKeywords} Queries)\n\n`;
|
|
32
|
+
markdown += `> *No high-consensus URLs found across searches. Results may be highly diverse.*\n\n`;
|
|
33
|
+
markdown += `---\n\n`;
|
|
34
|
+
}
|
|
35
|
+
// Limit output based on number of queries to keep under ~20k tokens
|
|
36
|
+
const MAX_QUERIES_SHOWN = 15;
|
|
37
|
+
const MAX_RESULTS_PER_QUERY = response.totalKeywords > 10 ? 5 : 10;
|
|
38
|
+
const queriesToShow = response.searches.slice(0, MAX_QUERIES_SHOWN);
|
|
39
|
+
const queriesOmitted = response.searches.length - queriesToShow.length;
|
|
40
|
+
markdown += `## 📊 Full Search Results by Query`;
|
|
41
|
+
if (queriesOmitted > 0) {
|
|
42
|
+
markdown += ` (showing ${queriesToShow.length} of ${response.searches.length})`;
|
|
43
|
+
}
|
|
44
|
+
markdown += `\n\n`;
|
|
45
|
+
let totalResults = 0;
|
|
46
|
+
queriesToShow.forEach((search, index) => {
|
|
47
|
+
markdown += `### Query ${index + 1}: "${search.keyword}"\n\n`;
|
|
48
|
+
search.results.slice(0, MAX_RESULTS_PER_QUERY).forEach((result, resultIndex) => {
|
|
49
|
+
const position = resultIndex + 1;
|
|
50
|
+
const positionScore = getPositionScore(position);
|
|
51
|
+
const rankedUrl = lookupUrl(result.link, urlLookup);
|
|
52
|
+
const frequency = rankedUrl?.frequency ?? 1;
|
|
53
|
+
const consensusMark = markConsensus(frequency);
|
|
54
|
+
const consensusInfo = rankedUrl
|
|
55
|
+
? `${consensusMark} (${frequency} searches)`
|
|
56
|
+
: `${consensusMark} (1 search)`;
|
|
57
|
+
markdown += `${position}. **[${result.title}](${result.link})** — Position ${position} | Score: ${positionScore.toFixed(1)} | Consensus: ${consensusInfo}\n`;
|
|
58
|
+
if (result.snippet) {
|
|
59
|
+
let snippet = result.snippet;
|
|
60
|
+
if (snippet.length > 150) {
|
|
61
|
+
snippet = snippet.substring(0, 147) + '...';
|
|
62
|
+
}
|
|
63
|
+
if (result.date) {
|
|
64
|
+
markdown += ` - *${result.date}* — ${snippet}\n`;
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
markdown += ` - ${snippet}\n`;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
markdown += '\n';
|
|
71
|
+
totalResults++;
|
|
72
|
+
});
|
|
73
|
+
if (search.related && search.related.length > 0) {
|
|
74
|
+
const relatedSuggestions = search.related
|
|
75
|
+
.slice(0, 5)
|
|
76
|
+
.map((r) => `\`${r}\``)
|
|
77
|
+
.join(', ');
|
|
78
|
+
markdown += `*Related:* ${relatedSuggestions}\n\n`;
|
|
79
|
+
}
|
|
80
|
+
if (index < queriesToShow.length - 1) {
|
|
81
|
+
markdown += `---\n\n`;
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
if (queriesOmitted > 0) {
|
|
85
|
+
markdown += `\n---\n\n> *${queriesOmitted} additional queries not shown. Consensus URLs above include all ${response.searches.length} queries.*\n`;
|
|
86
|
+
}
|
|
87
|
+
const executionTime = Date.now() - startTime;
|
|
88
|
+
if (sessionId && logger) {
|
|
89
|
+
await logger('info', `Search completed: ${totalResults} results, ${aggregation.totalUniqueUrls} unique URLs, ${consensusUrls.length} consensus URLs in ${executionTime}ms`, sessionId);
|
|
90
|
+
}
|
|
91
|
+
const metadata = {
|
|
92
|
+
total_keywords: response.totalKeywords,
|
|
93
|
+
total_results: totalResults,
|
|
94
|
+
execution_time_ms: executionTime,
|
|
95
|
+
total_unique_urls: aggregation.totalUniqueUrls,
|
|
96
|
+
consensus_url_count: consensusUrls.length,
|
|
97
|
+
frequency_threshold: aggregation.frequencyThreshold,
|
|
98
|
+
};
|
|
99
|
+
return { content: markdown, structuredContent: { content: markdown, metadata } };
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
103
|
+
if (sessionId && logger) {
|
|
104
|
+
await logger('error', errorMessage, sessionId);
|
|
105
|
+
}
|
|
106
|
+
const executionTime = Date.now() - startTime;
|
|
107
|
+
const errorContent = `# ❌ Search Failed\n\n${errorMessage}\n\n**Tip:** Make sure SERPER_API_KEY is set in your environment variables.`;
|
|
108
|
+
return {
|
|
109
|
+
content: errorContent,
|
|
110
|
+
structuredContent: {
|
|
111
|
+
content: errorContent,
|
|
112
|
+
metadata: {
|
|
113
|
+
total_keywords: params.keywords.length,
|
|
114
|
+
total_results: 0,
|
|
115
|
+
execution_time_ms: executionTime,
|
|
116
|
+
},
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,sBAAsB,EACtB,aAAa,GACd,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAOjD,SAAS,gBAAgB,CAAC,QAAgB;IACxC,IAAI,QAAQ,IAAI,CAAC,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,OAAO,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAuB,EACvB,UAAuB,EAAE;IAEzB,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,CAAC;QACH,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,CAAC,MAAM,EAAE,iBAAiB,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,EAAE,SAAS,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,cAAc,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAEzD,MAAM,aAAa,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CACjD,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,IAAI,WAAW,CAAC,kBAAkB,CACvD,CAAC;QAEF,IAAI,QAAQ,GAAG,EAAE,CAAC;QAElB,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,QAAQ,IAAI,sBAAsB,CAChC,aAAa,EACb,MAAM,CAAC,QAAQ,EACf,WAAW,CAAC,eAAe,EAC3B,WAAW,CAAC,kBAAkB,EAC9B,WAAW,CAAC,aAAa,CAC1B,CAAC;YACF,QAAQ,IAAI,WAAW,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,QAAQ,IAAI,kDAAkD,QAAQ,CAAC,aAAa,eAAe,CAAC;YACpG,QAAQ,IAAI,sFAAsF,CAAC;YACnG,QAAQ,IAAI,SAAS,CAAC;QACxB,CAAC;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,EAAE,CAAC;QAC7B,MAAM,qBAAqB,GAAG,QAAQ,CAAC,aAAa,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;QACpE,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QAEvE,QAAQ,IAAI,oCAAoC,CAAC;QACjD,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,IAAI,aAAa,aAAa,CAAC,MAAM,OAAO,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;QAClF,CAAC;QACD,QAAQ,IAAI,MAAM,CAAC;QAEnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACtC,QAAQ,IAAI,aAAa,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,OAAO,OAAO,CAAC;YAE9D,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE;gBAC7E,MAAM,QAAQ,GAAG,WAAW,GAAG,CAAC,CAAC;gBACjC,MAAM,aAAa,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;gBAEjD,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBACpD,MAAM,SAAS,GAAG,SAAS,EAAE,SAAS,IAAI,CAAC,CAAC;gBAC5C,MAAM,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;gBAC/C,MAAM,aAAa,GAAG,SAAS;oBAC7B,CAAC,CAAC,GAAG,aAAa,KAAK,SAAS,YAAY;oBAC5C,CAAC,CAAC,GAAG,aAAa,aAAa,CAAC;gBAElC,QAAQ,IAAI,GAAG,QAAQ,QAAQ,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,IAAI,kBAAkB,QAAQ,aAAa,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,aAAa,IAAI,CAAC;gBAE7J,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,IAAI,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;oBAC7B,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;wBACzB,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC;oBAC9C,CAAC;oBAED,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;wBAChB,QAAQ,IAAI,SAAS,MAAM,CAAC,IAAI,OAAO,OAAO,IAAI,CAAC;oBACrD,CAAC;yBAAM,CAAC;wBACN,QAAQ,IAAI,QAAQ,OAAO,IAAI,CAAC;oBAClC,CAAC;gBACH,CAAC;gBAED,QAAQ,IAAI,IAAI,CAAC;gBACjB,YAAY,EAAE,CAAC;YACjB,CAAC,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChD,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO;qBACtC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;qBACX,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;qBAC9B,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEd,QAAQ,IAAI,cAAc,kBAAkB,MAAM,CAAC;YACrD,CAAC;YAED,IAAI,KAAK,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrC,QAAQ,IAAI,SAAS,CAAC;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,IAAI,eAAe,cAAc,mEAAmE,QAAQ,CAAC,QAAQ,CAAC,MAAM,cAAc,CAAC;QACrJ,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAE7C,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,CACV,MAAM,EACN,qBAAqB,YAAY,aAAa,WAAW,CAAC,eAAe,iBAAiB,aAAa,CAAC,MAAM,sBAAsB,aAAa,IAAI,EACrJ,SAAS,CACV,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG;YACf,cAAc,EAAE,QAAQ,CAAC,aAAa;YACtC,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,aAAa;YAChC,iBAAiB,EAAE,WAAW,CAAC,eAAe;YAC9C,mBAAmB,EAAE,aAAa,CAAC,MAAM;YACzC,mBAAmB,EAAE,WAAW,CAAC,kBAAkB;SACpD,CAAC;QAEF,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;IACnF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5E,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC7C,MAAM,YAAY,GAAG,wBAAwB,YAAY,6EAA6E,CAAC;QAEvI,OAAO;YACL,OAAO,EAAE,YAAY;YACrB,iBAAiB,EAAE;gBACjB,OAAO,EAAE,YAAY;gBACrB,QAAQ,EAAE;oBACR,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;oBACtC,aAAa,EAAE,CAAC;oBAChB,iBAAiB,EAAE,aAAa;iBACjC;aACF;SACF,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CA6BnF"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Simple error handling for API requests
|
|
3
|
+
*/
|
|
4
|
+
export function createSimpleError(error) {
|
|
5
|
+
const err = error;
|
|
6
|
+
// Missing API key
|
|
7
|
+
if (err.message?.includes('API_KEY')) {
|
|
8
|
+
return { message: 'OPENROUTER_API_KEY environment variable required', code: 'AUTH_ERROR' };
|
|
9
|
+
}
|
|
10
|
+
// HTTP errors
|
|
11
|
+
if (err.response?.status) {
|
|
12
|
+
const status = err.response.status;
|
|
13
|
+
switch (status) {
|
|
14
|
+
case 401:
|
|
15
|
+
return { message: 'Invalid API key', code: 'AUTH_ERROR' };
|
|
16
|
+
case 429:
|
|
17
|
+
return { message: 'Rate limit exceeded - try again later', code: 'RATE_LIMIT' };
|
|
18
|
+
case 403:
|
|
19
|
+
return { message: 'API quota exceeded', code: 'QUOTA_ERROR' };
|
|
20
|
+
default:
|
|
21
|
+
return { message: `API error: ${status}`, code: 'API_ERROR' };
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// Network/timeout errors
|
|
25
|
+
if (err.code === 'ECONNABORTED' || err.message?.includes('timeout')) {
|
|
26
|
+
return { message: 'Request timeout - research may take up to 30 minutes', code: 'TIMEOUT' };
|
|
27
|
+
}
|
|
28
|
+
return { message: err.message || 'Unknown error occurred', code: 'UNKNOWN_ERROR' };
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,MAAM,GAAG,GAAG,KAA4E,CAAC;IAEzF,kBAAkB;IAClB,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO,EAAE,OAAO,EAAE,kDAAkD,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;IAC7F,CAAC;IAED,cAAc;IACd,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;QACnC,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,GAAG;gBACN,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;YAC5D,KAAK,GAAG;gBACN,OAAO,EAAE,OAAO,EAAE,uCAAuC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;YAClF,KAAK,GAAG;gBACN,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;YAChE;gBACE,OAAO,EAAE,OAAO,EAAE,cAAc,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAClE,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,OAAO,EAAE,sDAAsD,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAC9F,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,wBAAwB,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;AACrF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-formatter.d.ts","sourceRoot":"","sources":["../../src/utils/markdown-formatter.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAYtD"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Markdown formatting utilities
|
|
3
|
+
*/
|
|
4
|
+
export function removeMetaTags(content) {
|
|
5
|
+
if (!content || typeof content !== 'string') {
|
|
6
|
+
return content;
|
|
7
|
+
}
|
|
8
|
+
const lines = content.split('\n');
|
|
9
|
+
const filteredLines = lines.filter(line => {
|
|
10
|
+
const trimmed = line.trim();
|
|
11
|
+
return !trimmed.startsWith('- Meta:') && !trimmed.startsWith('Meta:');
|
|
12
|
+
});
|
|
13
|
+
return filteredLines.join('\n');
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=markdown-formatter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-formatter.js","sourceRoot":"","sources":["../../src/utils/markdown-formatter.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* URL Aggregator Utility
|
|
3
|
+
* Aggregates search results across multiple queries, calculates CTR-weighted scores,
|
|
4
|
+
* and generates consensus-based rankings.
|
|
5
|
+
*/
|
|
6
|
+
import type { KeywordSearchResult } from '../clients/search.js';
|
|
7
|
+
/**
|
|
8
|
+
* Ranked URL with normalized score
|
|
9
|
+
*/
|
|
10
|
+
export interface RankedUrl {
|
|
11
|
+
url: string;
|
|
12
|
+
title: string;
|
|
13
|
+
snippet: string;
|
|
14
|
+
rank: number;
|
|
15
|
+
score: number;
|
|
16
|
+
frequency: number;
|
|
17
|
+
positions: number[];
|
|
18
|
+
queries: string[];
|
|
19
|
+
bestPosition: number;
|
|
20
|
+
isConsensus: boolean;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Aggregation result containing all processed data
|
|
24
|
+
*/
|
|
25
|
+
interface AggregationResult {
|
|
26
|
+
rankedUrls: RankedUrl[];
|
|
27
|
+
totalUniqueUrls: number;
|
|
28
|
+
totalQueries: number;
|
|
29
|
+
frequencyThreshold: number;
|
|
30
|
+
thresholdNote?: string;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Mark consensus status for a URL
|
|
34
|
+
* Returns "✓" if frequency >= 3, else "✗"
|
|
35
|
+
*/
|
|
36
|
+
export declare function markConsensus(frequency: number): string;
|
|
37
|
+
/**
|
|
38
|
+
* Generate enhanced narrative output for consensus URLs
|
|
39
|
+
*/
|
|
40
|
+
export declare function generateEnhancedOutput(rankedUrls: RankedUrl[], allKeywords: string[], totalUniqueUrls: number, frequencyThreshold: number, thresholdNote?: string): string;
|
|
41
|
+
/**
|
|
42
|
+
* Full aggregation pipeline with fallback thresholds
|
|
43
|
+
* Tries ≥3, falls back to ≥2, then ≥1 if needed
|
|
44
|
+
*/
|
|
45
|
+
export declare function aggregateAndRank(searches: KeywordSearchResult[], minConsensusUrls?: number): AggregationResult;
|
|
46
|
+
/**
|
|
47
|
+
* Build URL lookup map for quick consensus checking during result formatting
|
|
48
|
+
*/
|
|
49
|
+
export declare function buildUrlLookup(rankedUrls: RankedUrl[]): Map<string, RankedUrl>;
|
|
50
|
+
/**
|
|
51
|
+
* Look up a URL in the ranked results
|
|
52
|
+
*/
|
|
53
|
+
export declare function lookupUrl(url: string, lookup: Map<string, RankedUrl>): RankedUrl | undefined;
|
|
54
|
+
export {};
|
|
55
|
+
//# sourceMappingURL=url-aggregator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-aggregator.d.ts","sourceRoot":"","sources":["../../src/utils/url-aggregator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAgBhE;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,UAAU,iBAAiB;IACzB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAqHD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEvD;AAyBD;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,SAAS,EAAE,EACvB,WAAW,EAAE,MAAM,EAAE,EACrB,eAAe,EAAE,MAAM,EACvB,kBAAkB,EAAE,MAAM,EAC1B,aAAa,CAAC,EAAE,MAAM,GACrB,MAAM,CAiER;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,mBAAmB,EAAE,EAC/B,gBAAgB,GAAE,MAAU,GAC3B,iBAAiB,CA+BnB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAW9E;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,SAAS,GAAG,SAAS,CAG5F"}
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* URL Aggregator Utility
|
|
3
|
+
* Aggregates search results across multiple queries, calculates CTR-weighted scores,
|
|
4
|
+
* and generates consensus-based rankings.
|
|
5
|
+
*/
|
|
6
|
+
import { CTR_WEIGHTS } from '../config/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* Get CTR weight for a position (1-10)
|
|
9
|
+
* Positions beyond 10 get minimal weight
|
|
10
|
+
*/
|
|
11
|
+
function getCtrWeight(position) {
|
|
12
|
+
if (position >= 1 && position <= 10) {
|
|
13
|
+
return CTR_WEIGHTS[position] ?? 0;
|
|
14
|
+
}
|
|
15
|
+
// Positions beyond 10 get diminishing returns
|
|
16
|
+
return Math.max(0, 10 - (position - 10) * 0.5);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Aggregate results from multiple searches
|
|
20
|
+
* Flattens all results, deduplicates by URL, and tracks frequency/positions
|
|
21
|
+
*/
|
|
22
|
+
function aggregateResults(searches) {
|
|
23
|
+
const urlMap = new Map();
|
|
24
|
+
for (const search of searches) {
|
|
25
|
+
for (const result of search.results) {
|
|
26
|
+
const normalizedUrl = normalizeUrl(result.link);
|
|
27
|
+
const existing = urlMap.get(normalizedUrl);
|
|
28
|
+
if (existing) {
|
|
29
|
+
existing.frequency += 1;
|
|
30
|
+
existing.positions.push(result.position);
|
|
31
|
+
existing.queries.push(search.keyword);
|
|
32
|
+
existing.bestPosition = Math.min(existing.bestPosition, result.position);
|
|
33
|
+
existing.totalScore += getCtrWeight(result.position);
|
|
34
|
+
// Keep best title/snippet (from highest position)
|
|
35
|
+
if (result.position < existing.positions[0]) {
|
|
36
|
+
existing.title = result.title;
|
|
37
|
+
existing.snippet = result.snippet;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
urlMap.set(normalizedUrl, {
|
|
42
|
+
url: result.link,
|
|
43
|
+
title: result.title,
|
|
44
|
+
snippet: result.snippet,
|
|
45
|
+
frequency: 1,
|
|
46
|
+
positions: [result.position],
|
|
47
|
+
queries: [search.keyword],
|
|
48
|
+
bestPosition: result.position,
|
|
49
|
+
totalScore: getCtrWeight(result.position),
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return urlMap;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Normalize URL for deduplication
|
|
58
|
+
* Removes trailing slashes, www prefix, and normalizes protocol
|
|
59
|
+
*/
|
|
60
|
+
function normalizeUrl(url) {
|
|
61
|
+
try {
|
|
62
|
+
const parsed = new URL(url);
|
|
63
|
+
let host = parsed.hostname.replace(/^www\./, '');
|
|
64
|
+
let path = parsed.pathname.replace(/\/$/, '') || '/';
|
|
65
|
+
return `${host}${path}${parsed.search}`.toLowerCase();
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return url.toLowerCase().replace(/\/$/, '');
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Filter URLs by minimum frequency
|
|
73
|
+
* Returns URLs appearing in at least minFrequency searches
|
|
74
|
+
*/
|
|
75
|
+
function filterByFrequency(urlMap, minFrequency) {
|
|
76
|
+
const filtered = [];
|
|
77
|
+
for (const url of urlMap.values()) {
|
|
78
|
+
if (url.frequency >= minFrequency) {
|
|
79
|
+
filtered.push(url);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return filtered;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Calculate weighted scores and normalize to 100.0
|
|
86
|
+
* Returns sorted array with rank assignments
|
|
87
|
+
*/
|
|
88
|
+
function calculateWeightedScores(urls) {
|
|
89
|
+
if (urls.length === 0)
|
|
90
|
+
return [];
|
|
91
|
+
// Sort by total score descending
|
|
92
|
+
const sorted = [...urls].sort((a, b) => b.totalScore - a.totalScore);
|
|
93
|
+
// Find max score for normalization
|
|
94
|
+
const maxScore = sorted[0].totalScore;
|
|
95
|
+
// Map to ranked URLs with normalized scores
|
|
96
|
+
return sorted.map((url, index) => ({
|
|
97
|
+
url: url.url,
|
|
98
|
+
title: url.title,
|
|
99
|
+
snippet: url.snippet,
|
|
100
|
+
rank: index + 1,
|
|
101
|
+
score: maxScore > 0 ? (url.totalScore / maxScore) * 100 : 0,
|
|
102
|
+
frequency: url.frequency,
|
|
103
|
+
positions: url.positions,
|
|
104
|
+
queries: url.queries,
|
|
105
|
+
bestPosition: url.bestPosition,
|
|
106
|
+
isConsensus: url.frequency >= 3,
|
|
107
|
+
}));
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Mark consensus status for a URL
|
|
111
|
+
* Returns "✓" if frequency >= 3, else "✗"
|
|
112
|
+
*/
|
|
113
|
+
export function markConsensus(frequency) {
|
|
114
|
+
return frequency >= 3 ? '✓' : '✗';
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Generate justification for why a URL is ranked at its position
|
|
118
|
+
*/
|
|
119
|
+
function generateJustification(url, rank) {
|
|
120
|
+
const parts = [];
|
|
121
|
+
if (url.frequency >= 4) {
|
|
122
|
+
parts.push(`Appeared in ${url.frequency} different searches showing strong cross-query relevance`);
|
|
123
|
+
}
|
|
124
|
+
else if (url.frequency >= 3) {
|
|
125
|
+
parts.push(`Found across ${url.frequency} searches indicating solid topical coverage`);
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
parts.push(`Appeared in ${url.frequency} search${url.frequency > 1 ? 'es' : ''}`);
|
|
129
|
+
}
|
|
130
|
+
if (url.bestPosition === 1) {
|
|
131
|
+
parts.push('ranked #1 in at least one search');
|
|
132
|
+
}
|
|
133
|
+
else if (url.bestPosition <= 3) {
|
|
134
|
+
parts.push(`best position was top-3 (#${url.bestPosition})`);
|
|
135
|
+
}
|
|
136
|
+
return parts.join(', ') + '.';
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Generate enhanced narrative output for consensus URLs
|
|
140
|
+
*/
|
|
141
|
+
export function generateEnhancedOutput(rankedUrls, allKeywords, totalUniqueUrls, frequencyThreshold, thresholdNote) {
|
|
142
|
+
const lines = [];
|
|
143
|
+
// Header
|
|
144
|
+
lines.push(`## The Perfect Search Results (Aggregated from ${allKeywords.length} Queries)`);
|
|
145
|
+
lines.push('');
|
|
146
|
+
lines.push(`Based on ${allKeywords.length} distinct searches, we identified **${rankedUrls.length} high-consensus resources**. Here's what the data reveals:`);
|
|
147
|
+
lines.push('');
|
|
148
|
+
if (thresholdNote) {
|
|
149
|
+
lines.push(`> ${thresholdNote}`);
|
|
150
|
+
lines.push('');
|
|
151
|
+
}
|
|
152
|
+
// Top Consensus Resources
|
|
153
|
+
lines.push('### 🥇 Top Consensus Resources');
|
|
154
|
+
lines.push('');
|
|
155
|
+
for (const url of rankedUrls.slice(0, 20)) {
|
|
156
|
+
const highConsensus = url.frequency >= 4 ? ' ⭐ HIGHEST CONSENSUS' : '';
|
|
157
|
+
lines.push(`#### #${url.rank}: ${url.title} (Score: ${url.score.toFixed(1)})${highConsensus}`);
|
|
158
|
+
// Appeared in queries
|
|
159
|
+
const queriesList = url.queries.map(q => `"${q}"`).join(', ');
|
|
160
|
+
lines.push(`- **Appeared in:** ${url.frequency} queries (${queriesList})`);
|
|
161
|
+
// Best ranking
|
|
162
|
+
lines.push(`- **Best ranking:** Position ${url.bestPosition}`);
|
|
163
|
+
// Description (truncated snippet)
|
|
164
|
+
const description = url.snippet.length > 200
|
|
165
|
+
? url.snippet.substring(0, 197) + '...'
|
|
166
|
+
: url.snippet;
|
|
167
|
+
lines.push(`- **Description:** ${description}`);
|
|
168
|
+
// Justification
|
|
169
|
+
lines.push(`- **Why it's #${url.rank}:** ${generateJustification(url, url.rank)}`);
|
|
170
|
+
// URL
|
|
171
|
+
lines.push(`- **URL:** ${url.url}`);
|
|
172
|
+
lines.push('');
|
|
173
|
+
}
|
|
174
|
+
// Metadata section
|
|
175
|
+
lines.push('---');
|
|
176
|
+
lines.push('');
|
|
177
|
+
lines.push('### 📈 Metadata');
|
|
178
|
+
lines.push('');
|
|
179
|
+
lines.push(`- **Total Queries:** ${allKeywords.length} (${allKeywords.join(', ')})`);
|
|
180
|
+
// Sort all URLs by frequency for the unique URLs list
|
|
181
|
+
const sortedByFreq = [...rankedUrls].sort((a, b) => b.frequency - a.frequency);
|
|
182
|
+
const urlFreqList = sortedByFreq
|
|
183
|
+
.slice(0, 30)
|
|
184
|
+
.map(u => {
|
|
185
|
+
const shortUrl = u.url.length > 40 ? u.url.substring(0, 37) + '...' : u.url;
|
|
186
|
+
return `${shortUrl} (${u.frequency}x)`;
|
|
187
|
+
})
|
|
188
|
+
.join(', ');
|
|
189
|
+
lines.push(`- **Unique URLs Found:** ${totalUniqueUrls} — top by frequency: ${urlFreqList}`);
|
|
190
|
+
lines.push(`- **Consensus Threshold:** ≥${frequencyThreshold} appearances`);
|
|
191
|
+
lines.push('');
|
|
192
|
+
return lines.join('\n');
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Full aggregation pipeline with fallback thresholds
|
|
196
|
+
* Tries ≥3, falls back to ≥2, then ≥1 if needed
|
|
197
|
+
*/
|
|
198
|
+
export function aggregateAndRank(searches, minConsensusUrls = 5) {
|
|
199
|
+
const urlMap = aggregateResults(searches);
|
|
200
|
+
const totalUniqueUrls = urlMap.size;
|
|
201
|
+
const totalQueries = searches.length;
|
|
202
|
+
// Try thresholds in order: 3, 2, 1
|
|
203
|
+
const thresholds = [3, 2, 1];
|
|
204
|
+
let rankedUrls = [];
|
|
205
|
+
let usedThreshold = 3;
|
|
206
|
+
let thresholdNote;
|
|
207
|
+
for (const threshold of thresholds) {
|
|
208
|
+
const filtered = filterByFrequency(urlMap, threshold);
|
|
209
|
+
rankedUrls = calculateWeightedScores(filtered);
|
|
210
|
+
if (rankedUrls.length >= minConsensusUrls || threshold === 1) {
|
|
211
|
+
usedThreshold = threshold;
|
|
212
|
+
if (threshold < 3) {
|
|
213
|
+
thresholdNote = `Note: Frequency filter lowered to ≥${threshold} due to result diversity.`;
|
|
214
|
+
}
|
|
215
|
+
break;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return {
|
|
219
|
+
rankedUrls,
|
|
220
|
+
totalUniqueUrls,
|
|
221
|
+
totalQueries,
|
|
222
|
+
frequencyThreshold: usedThreshold,
|
|
223
|
+
thresholdNote,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Build URL lookup map for quick consensus checking during result formatting
|
|
228
|
+
*/
|
|
229
|
+
export function buildUrlLookup(rankedUrls) {
|
|
230
|
+
const lookup = new Map();
|
|
231
|
+
for (const url of rankedUrls) {
|
|
232
|
+
const normalized = normalizeUrl(url.url);
|
|
233
|
+
lookup.set(normalized, url);
|
|
234
|
+
// Also store original URL
|
|
235
|
+
lookup.set(url.url.toLowerCase(), url);
|
|
236
|
+
}
|
|
237
|
+
return lookup;
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Look up a URL in the ranked results
|
|
241
|
+
*/
|
|
242
|
+
export function lookupUrl(url, lookup) {
|
|
243
|
+
const normalized = normalizeUrl(url);
|
|
244
|
+
return lookup.get(normalized) || lookup.get(url.toLowerCase());
|
|
245
|
+
}
|
|
246
|
+
//# sourceMappingURL=url-aggregator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-aggregator.js","sourceRoot":"","sources":["../../src/utils/url-aggregator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA4CjD;;;GAGG;AACH,SAAS,YAAY,CAAC,QAAgB;IACpC,IAAI,QAAQ,IAAI,CAAC,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,OAAO,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IACD,8CAA8C;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;AACjD,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,QAA+B;IACvD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEhD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAE3C,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;gBACxB,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACzC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACtC,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACzE,QAAQ,CAAC,UAAU,IAAI,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACrD,kDAAkD;gBAClD,IAAI,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC5C,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;oBAC9B,QAAQ,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;gBACpC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE;oBACxB,GAAG,EAAE,MAAM,CAAC,IAAI;oBAChB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;oBAC5B,OAAO,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;oBACzB,YAAY,EAAE,MAAM,CAAC,QAAQ;oBAC7B,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC;iBAC1C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC;QACrD,OAAO,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CACxB,MAAkC,EAClC,YAAoB;IAEpB,MAAM,QAAQ,GAAoB,EAAE,CAAC;IAErC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,IAAI,GAAG,CAAC,SAAS,IAAI,YAAY,EAAE,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAAC,IAAqB;IACpD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,iCAAiC;IACjC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAErE,mCAAmC;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAEtC,4CAA4C;IAC5C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACjC,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,IAAI,EAAE,KAAK,GAAG,CAAC;QACf,KAAK,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3D,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,WAAW,EAAE,GAAG,CAAC,SAAS,IAAI,CAAC;KAChC,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,OAAO,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,GAAc,EAAE,IAAY;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,GAAG,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,SAAS,0DAA0D,CAAC,CAAC;IACrG,CAAC;SAAM,IAAI,GAAG,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,SAAS,6CAA6C,CAAC,CAAC;IACzF,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,SAAS,UAAU,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,IAAI,GAAG,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IACjD,CAAC;SAAM,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,6BAA6B,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CACpC,UAAuB,EACvB,WAAqB,EACrB,eAAuB,EACvB,kBAA0B,EAC1B,aAAsB;IAEtB,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,SAAS;IACT,KAAK,CAAC,IAAI,CAAC,kDAAkD,WAAW,CAAC,MAAM,WAAW,CAAC,CAAC;IAC5F,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,WAAW,CAAC,MAAM,uCAAuC,UAAU,CAAC,MAAM,4DAA4D,CAAC,CAAC;IAC/J,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,KAAK,aAAa,EAAE,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,0BAA0B;IAC1B,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QAC1C,MAAM,aAAa,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,YAAY,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;QAE/F,sBAAsB;QACtB,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,sBAAsB,GAAG,CAAC,SAAS,aAAa,WAAW,GAAG,CAAC,CAAC;QAE3E,eAAe;QACf,KAAK,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC;QAE/D,kCAAkC;QAClC,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG;YAC1C,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;YACvC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,sBAAsB,WAAW,EAAE,CAAC,CAAC;QAEhD,gBAAgB;QAChB,KAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,CAAC,IAAI,OAAO,qBAAqB,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEnF,MAAM;QACN,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,mBAAmB;IACnB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,WAAW,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAErF,sDAAsD;IACtD,MAAM,YAAY,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/E,MAAM,WAAW,GAAG,YAAY;SAC7B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE;QACP,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5E,OAAO,GAAG,QAAQ,KAAK,CAAC,CAAC,SAAS,IAAI,CAAC;IACzC,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,KAAK,CAAC,IAAI,CAAC,4BAA4B,eAAe,wBAAwB,WAAW,EAAE,CAAC,CAAC;IAC7F,KAAK,CAAC,IAAI,CAAC,+BAA+B,kBAAkB,cAAc,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,QAA+B,EAC/B,mBAA2B,CAAC;IAE5B,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC;IACpC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;IAErC,mCAAmC;IACnC,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7B,IAAI,UAAU,GAAgB,EAAE,CAAC;IACjC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,aAAiC,CAAC;IAEtC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACtD,UAAU,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAE/C,IAAI,UAAU,CAAC,MAAM,IAAI,gBAAgB,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YAC7D,aAAa,GAAG,SAAS,CAAC;YAC1B,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,aAAa,GAAG,sCAAsC,SAAS,2BAA2B,CAAC;YAC7F,CAAC;YACD,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO;QACL,UAAU;QACV,eAAe;QACf,YAAY;QACZ,kBAAkB,EAAE,aAAa;QACjC,aAAa;KACd,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,UAAuB;IACpD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;IAE5C,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAC5B,0BAA0B;QAC1B,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,MAA8B;IACnE,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;AACjE,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "research-powerpack-mcp",
|
|
3
|
+
"version": "3.0.0",
|
|
4
|
+
"description": "The ultimate research MCP toolkit: Reddit mining, web search with CTR aggregation, AI-powered deep research, and intelligent web scraping - all in one modular package",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"bin": {
|
|
8
|
+
"research-powerpack-mcp": "dist/index.js"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"dist"
|
|
12
|
+
],
|
|
13
|
+
"scripts": {
|
|
14
|
+
"build": "tsc",
|
|
15
|
+
"prepublishOnly": "npm run build",
|
|
16
|
+
"dev": "tsx src/index.ts",
|
|
17
|
+
"start": "node dist/index.js",
|
|
18
|
+
"typecheck": "tsc --noEmit"
|
|
19
|
+
},
|
|
20
|
+
"keywords": [
|
|
21
|
+
"mcp",
|
|
22
|
+
"research",
|
|
23
|
+
"reddit",
|
|
24
|
+
"serper",
|
|
25
|
+
"search",
|
|
26
|
+
"ai",
|
|
27
|
+
"model-context-protocol",
|
|
28
|
+
"deep-research",
|
|
29
|
+
"web-scraping",
|
|
30
|
+
"claude",
|
|
31
|
+
"anthropic",
|
|
32
|
+
"openrouter",
|
|
33
|
+
"scraping",
|
|
34
|
+
"research-powerpack"
|
|
35
|
+
],
|
|
36
|
+
"author": "Yiğit Konur <yigit35@gmail.com>",
|
|
37
|
+
"license": "MIT",
|
|
38
|
+
"repository": {
|
|
39
|
+
"type": "git",
|
|
40
|
+
"url": "https://github.com/yigitkonur/research-powerpack-mcp"
|
|
41
|
+
},
|
|
42
|
+
"dependencies": {
|
|
43
|
+
"@modelcontextprotocol/sdk": "^1.18.1",
|
|
44
|
+
"openai": "^4.77.0",
|
|
45
|
+
"zod": "^3.24.1",
|
|
46
|
+
"zod-to-json-schema": "^3.24.1"
|
|
47
|
+
},
|
|
48
|
+
"devDependencies": {
|
|
49
|
+
"@types/node": "^22.0.0",
|
|
50
|
+
"tsx": "^4.19.0",
|
|
51
|
+
"typescript": "^5.6.0"
|
|
52
|
+
},
|
|
53
|
+
"engines": {
|
|
54
|
+
"node": ">=20.0.0"
|
|
55
|
+
}
|
|
56
|
+
}
|