research-powerpack-mcp 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +486 -0
- package/dist/clients/reddit.d.ts +61 -0
- package/dist/clients/reddit.d.ts.map +1 -0
- package/dist/clients/reddit.js +179 -0
- package/dist/clients/reddit.js.map +1 -0
- package/dist/clients/research.d.ts +41 -0
- package/dist/clients/research.d.ts.map +1 -0
- package/dist/clients/research.js +77 -0
- package/dist/clients/research.js.map +1 -0
- package/dist/clients/scraper.d.ts +44 -0
- package/dist/clients/scraper.d.ts.map +1 -0
- package/dist/clients/scraper.js +171 -0
- package/dist/clients/scraper.js.map +1 -0
- package/dist/clients/search.d.ts +46 -0
- package/dist/clients/search.d.ts.map +1 -0
- package/dist/clients/search.js +91 -0
- package/dist/clients/search.js.map +1 -0
- package/dist/config/index.d.ts +59 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +100 -0
- package/dist/config/index.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +152 -0
- package/dist/index.js.map +1 -0
- package/dist/schemas/deep-research.d.ts +100 -0
- package/dist/schemas/deep-research.d.ts.map +1 -0
- package/dist/schemas/deep-research.js +57 -0
- package/dist/schemas/deep-research.js.map +1 -0
- package/dist/schemas/scrape-links.d.ts +38 -0
- package/dist/schemas/scrape-links.d.ts.map +1 -0
- package/dist/schemas/scrape-links.js +26 -0
- package/dist/schemas/scrape-links.js.map +1 -0
- package/dist/schemas/web-search.d.ts +24 -0
- package/dist/schemas/web-search.d.ts.map +1 -0
- package/dist/schemas/web-search.js +12 -0
- package/dist/schemas/web-search.js.map +1 -0
- package/dist/services/file-attachment.d.ts +30 -0
- package/dist/services/file-attachment.d.ts.map +1 -0
- package/dist/services/file-attachment.js +196 -0
- package/dist/services/file-attachment.js.map +1 -0
- package/dist/services/llm-processor.d.ts +19 -0
- package/dist/services/llm-processor.d.ts.map +1 -0
- package/dist/services/llm-processor.js +44 -0
- package/dist/services/llm-processor.js.map +1 -0
- package/dist/services/markdown-cleaner.d.ts +8 -0
- package/dist/services/markdown-cleaner.d.ts.map +1 -0
- package/dist/services/markdown-cleaner.js +56 -0
- package/dist/services/markdown-cleaner.js.map +1 -0
- package/dist/tools/definitions.d.ts +66 -0
- package/dist/tools/definitions.d.ts.map +1 -0
- package/dist/tools/definitions.js +125 -0
- package/dist/tools/definitions.js.map +1 -0
- package/dist/tools/reddit.d.ts +10 -0
- package/dist/tools/reddit.d.ts.map +1 -0
- package/dist/tools/reddit.js +105 -0
- package/dist/tools/reddit.js.map +1 -0
- package/dist/tools/research.d.ts +14 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +126 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/scrape.d.ts +14 -0
- package/dist/tools/scrape.d.ts.map +1 -0
- package/dist/tools/scrape.js +111 -0
- package/dist/tools/scrape.js.map +1 -0
- package/dist/tools/search.d.ts +14 -0
- package/dist/tools/search.d.ts.map +1 -0
- package/dist/tools/search.js +121 -0
- package/dist/tools/search.js.map +1 -0
- package/dist/utils/errors.d.ts +8 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +30 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/markdown-formatter.d.ts +5 -0
- package/dist/utils/markdown-formatter.d.ts.map +1 -0
- package/dist/utils/markdown-formatter.js +15 -0
- package/dist/utils/markdown-formatter.js.map +1 -0
- package/dist/utils/url-aggregator.d.ts +55 -0
- package/dist/utils/url-aggregator.d.ts.map +1 -0
- package/dist/utils/url-aggregator.js +246 -0
- package/dist/utils/url-aggregator.js.map +1 -0
- package/package.json +56 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
// Input schema for scrape_links tool
|
|
3
|
+
export const scrapeLinksParamsShape = {
|
|
4
|
+
urls: z
|
|
5
|
+
.array(z.string().url('Must be a valid HTTP or HTTPS URL'))
|
|
6
|
+
.min(3, 'Minimum 3 URLs required for efficient batch processing')
|
|
7
|
+
.max(50, 'Maximum 50 URLs allowed per request')
|
|
8
|
+
.describe('URLs to scrape (3-50). More URLs = broader coverage but fewer tokens per URL. 3 URLs: ~10K tokens each (deep); 50 URLs: ~640 tokens each (scan).'),
|
|
9
|
+
timeout: z
|
|
10
|
+
.number()
|
|
11
|
+
.min(5)
|
|
12
|
+
.max(120)
|
|
13
|
+
.default(30)
|
|
14
|
+
.describe('Timeout in seconds for each URL'),
|
|
15
|
+
use_llm: z
|
|
16
|
+
.boolean()
|
|
17
|
+
.default(false)
|
|
18
|
+
.describe('Enable AI processing for content extraction (requires OPENROUTER_API_KEY)'),
|
|
19
|
+
what_to_extract: z
|
|
20
|
+
.string()
|
|
21
|
+
.max(1000)
|
|
22
|
+
.optional()
|
|
23
|
+
.describe('Specific content extraction instructions for AI. Will be enhanced with conciseness suffix automatically.'),
|
|
24
|
+
};
|
|
25
|
+
export const scrapeLinksParamsSchema = z.object(scrapeLinksParamsShape);
|
|
26
|
+
//# sourceMappingURL=scrape-links.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrape-links.js","sourceRoot":"","sources":["../../src/schemas/scrape-links.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,qCAAqC;AACrC,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,IAAI,EAAE,CAAC;SACJ,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;SAC1D,GAAG,CAAC,CAAC,EAAE,wDAAwD,CAAC;SAChE,GAAG,CAAC,EAAE,EAAE,qCAAqC,CAAC;SAC9C,QAAQ,CAAC,kJAAkJ,CAAC;IAC/J,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,GAAG,CAAC;SACR,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CAAC,iCAAiC,CAAC;IAC9C,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2EAA2E,CAAC;IACxF,eAAe,EAAE,CAAC;SACf,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,0GAA0G,CAAC;CACxH,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const webSearchParamsShape: {
|
|
3
|
+
keywords: z.ZodArray<z.ZodString, "many">;
|
|
4
|
+
};
|
|
5
|
+
export declare const webSearchParamsSchema: z.ZodObject<{
|
|
6
|
+
keywords: z.ZodArray<z.ZodString, "many">;
|
|
7
|
+
}, "strip", z.ZodTypeAny, {
|
|
8
|
+
keywords: string[];
|
|
9
|
+
}, {
|
|
10
|
+
keywords: string[];
|
|
11
|
+
}>;
|
|
12
|
+
export type WebSearchParams = z.infer<typeof webSearchParamsSchema>;
|
|
13
|
+
export interface WebSearchOutput {
|
|
14
|
+
content: string;
|
|
15
|
+
metadata: {
|
|
16
|
+
total_keywords: number;
|
|
17
|
+
total_results: number;
|
|
18
|
+
execution_time_ms: number;
|
|
19
|
+
total_unique_urls?: number;
|
|
20
|
+
consensus_url_count?: number;
|
|
21
|
+
frequency_threshold?: number;
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=web-search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-search.d.ts","sourceRoot":"","sources":["../../src/schemas/web-search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,eAAO,MAAM,oBAAoB;;CAEhC,CAAC;AAEF,eAAO,MAAM,qBAAqB;;;;;;EAAiC,CAAC;AACpE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAGpE,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE;QACR,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;CACH"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
// Input schema for web_search tool (renamed from search_multiple)
|
|
3
|
+
const keywordsSchema = z
|
|
4
|
+
.array(z.string().min(1, 'Keyword cannot be empty').max(500, 'Keyword cannot exceed 500 characters'))
|
|
5
|
+
.min(1, 'At least one keyword is required')
|
|
6
|
+
.max(100, 'Cannot exceed 100 keywords per request')
|
|
7
|
+
.describe('Array of search keywords (1-100 keywords). Use specific, targeted keywords for best results.');
|
|
8
|
+
export const webSearchParamsShape = {
|
|
9
|
+
keywords: keywordsSchema,
|
|
10
|
+
};
|
|
11
|
+
export const webSearchParamsSchema = z.object(webSearchParamsShape);
|
|
12
|
+
//# sourceMappingURL=web-search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-search.js","sourceRoot":"","sources":["../../src/schemas/web-search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,kEAAkE;AAClE,MAAM,cAAc,GAAG,CAAC;KACrB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,sCAAsC,CAAC,CAAC;KACpG,GAAG,CAAC,CAAC,EAAE,kCAAkC,CAAC;KAC1C,GAAG,CAAC,GAAG,EAAE,wCAAwC,CAAC;KAClD,QAAQ,CAAC,8FAA8F,CAAC,CAAC;AAE5G,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,QAAQ,EAAE,cAAc;CACzB,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File attachment service for reading and formatting file contents
|
|
3
|
+
*/
|
|
4
|
+
interface FileAttachment {
|
|
5
|
+
path: string;
|
|
6
|
+
start_line?: number | undefined;
|
|
7
|
+
end_line?: number | undefined;
|
|
8
|
+
description?: string | undefined;
|
|
9
|
+
}
|
|
10
|
+
export declare class FileAttachmentService {
|
|
11
|
+
/**
|
|
12
|
+
* Format multiple file attachments into a markdown section
|
|
13
|
+
*/
|
|
14
|
+
formatAttachments(attachments: FileAttachment[]): Promise<string>;
|
|
15
|
+
/**
|
|
16
|
+
* Format a single file attachment
|
|
17
|
+
*/
|
|
18
|
+
private formatSingleFile;
|
|
19
|
+
/**
|
|
20
|
+
* Format code block with line numbers and smart truncation
|
|
21
|
+
*/
|
|
22
|
+
private formatCodeBlock;
|
|
23
|
+
/**
|
|
24
|
+
* Validate line range and return corrected values
|
|
25
|
+
*/
|
|
26
|
+
private validateLineRange;
|
|
27
|
+
private detectLanguage;
|
|
28
|
+
}
|
|
29
|
+
export {};
|
|
30
|
+
//# sourceMappingURL=file-attachment.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-attachment.d.ts","sourceRoot":"","sources":["../../src/services/file-attachment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC;AASD,qBAAa,qBAAqB;IAChC;;OAEG;IACG,iBAAiB,CAAC,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBvE;;OAEG;YACW,gBAAgB;IAoE9B;;OAEG;IACH,OAAO,CAAC,eAAe;IAiCvB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAoEzB,OAAO,CAAC,cAAc;CAcvB"}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File attachment service for reading and formatting file contents
|
|
3
|
+
*/
|
|
4
|
+
import { existsSync } from 'node:fs';
|
|
5
|
+
import { readFile } from 'node:fs/promises';
|
|
6
|
+
import { extname } from 'node:path';
|
|
7
|
+
export class FileAttachmentService {
|
|
8
|
+
/**
|
|
9
|
+
* Format multiple file attachments into a markdown section
|
|
10
|
+
*/
|
|
11
|
+
async formatAttachments(attachments) {
|
|
12
|
+
if (!attachments || attachments.length === 0) {
|
|
13
|
+
return '';
|
|
14
|
+
}
|
|
15
|
+
const results = await Promise.all(attachments.map((attachment) => this.formatSingleFile(attachment)));
|
|
16
|
+
// Build the attachments section
|
|
17
|
+
let output = '\n\n---\n\n# 📎 ATTACHED FILES\n\n';
|
|
18
|
+
output += `*${results.length} file${results.length > 1 ? 's' : ''} attached for context*\n\n`;
|
|
19
|
+
for (const result of results) {
|
|
20
|
+
output += result.content;
|
|
21
|
+
output += '\n\n';
|
|
22
|
+
}
|
|
23
|
+
return output;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Format a single file attachment
|
|
27
|
+
*/
|
|
28
|
+
async formatSingleFile(attachment) {
|
|
29
|
+
const { path, start_line, end_line, description } = attachment;
|
|
30
|
+
// Check if file exists
|
|
31
|
+
if (!existsSync(path)) {
|
|
32
|
+
return {
|
|
33
|
+
success: false,
|
|
34
|
+
path,
|
|
35
|
+
content: `## ❌ ${path}\n\n**FILE NOT FOUND**\n${description ? `\n*Description:* ${description}\n` : ''}`,
|
|
36
|
+
error: 'File not found',
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
try {
|
|
40
|
+
// Read file content
|
|
41
|
+
const content = await readFile(path, 'utf-8');
|
|
42
|
+
const lines = content.split('\n');
|
|
43
|
+
const language = this.detectLanguage(path);
|
|
44
|
+
// Validate line ranges
|
|
45
|
+
const validatedRange = this.validateLineRange(start_line, end_line, lines.length);
|
|
46
|
+
if (!validatedRange.valid) {
|
|
47
|
+
return {
|
|
48
|
+
success: false,
|
|
49
|
+
path,
|
|
50
|
+
content: `## ⚠️ ${path}\n\n**INVALID LINE RANGE**: ${validatedRange.error}\n${description ? `\n*Description:* ${description}\n` : ''}`,
|
|
51
|
+
error: validatedRange.error,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
// Extract relevant lines
|
|
55
|
+
const startIdx = validatedRange.start - 1;
|
|
56
|
+
const endIdx = validatedRange.end - 1;
|
|
57
|
+
const selectedLines = lines.slice(startIdx, endIdx + 1);
|
|
58
|
+
// Build formatted output
|
|
59
|
+
let formatted = `## 📄 ${path}\n\n`;
|
|
60
|
+
// Add metadata
|
|
61
|
+
const isPartial = start_line !== undefined || end_line !== undefined;
|
|
62
|
+
formatted += `**Language:** ${language} | `;
|
|
63
|
+
formatted += `**Lines:** ${isPartial ? `${validatedRange.start}-${validatedRange.end}` : lines.length} | `;
|
|
64
|
+
formatted += `**Size:** ${(content.length / 1024).toFixed(2)} KB\n`;
|
|
65
|
+
if (description) {
|
|
66
|
+
formatted += `\n*${description}*\n`;
|
|
67
|
+
}
|
|
68
|
+
formatted += '\n';
|
|
69
|
+
// Add file content with line numbers
|
|
70
|
+
formatted += this.formatCodeBlock(selectedLines, language, startIdx);
|
|
71
|
+
return {
|
|
72
|
+
success: true,
|
|
73
|
+
path,
|
|
74
|
+
content: formatted,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
return {
|
|
79
|
+
success: false,
|
|
80
|
+
path,
|
|
81
|
+
content: `## ❌ ${path}\n\n**ERROR READING FILE**: ${error instanceof Error ? error.message : String(error)}\n${description ? `\n*Description:* ${description}\n` : ''}`,
|
|
82
|
+
error: error instanceof Error ? error.message : String(error),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Format code block with line numbers and smart truncation
|
|
88
|
+
*/
|
|
89
|
+
formatCodeBlock(lines, language, startIdx) {
|
|
90
|
+
let output = `\`\`\`${language.toLowerCase()}\n`;
|
|
91
|
+
// Smart truncation for very large files (keep first 500 lines + last 100 lines)
|
|
92
|
+
if (lines.length > 600) {
|
|
93
|
+
// First 500 lines
|
|
94
|
+
const firstLines = lines.slice(0, 500);
|
|
95
|
+
firstLines.forEach((line, idx) => {
|
|
96
|
+
const lineNumber = startIdx + idx + 1;
|
|
97
|
+
output += `${lineNumber.toString().padStart(4, ' ')}: ${line}\n`;
|
|
98
|
+
});
|
|
99
|
+
// Truncation marker
|
|
100
|
+
output += `\n... [${lines.length - 600} lines truncated for brevity] ...\n\n`;
|
|
101
|
+
// Last 100 lines
|
|
102
|
+
const lastLines = lines.slice(-100);
|
|
103
|
+
lastLines.forEach((line, idx) => {
|
|
104
|
+
const lineNumber = startIdx + lines.length - 100 + idx + 1;
|
|
105
|
+
output += `${lineNumber.toString().padStart(4, ' ')}: ${line}\n`;
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
// Show all lines with numbers
|
|
110
|
+
lines.forEach((line, idx) => {
|
|
111
|
+
const lineNumber = startIdx + idx + 1;
|
|
112
|
+
output += `${lineNumber.toString().padStart(4, ' ')}: ${line}\n`;
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
output += '```';
|
|
116
|
+
return output;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Validate line range and return corrected values
|
|
120
|
+
*/
|
|
121
|
+
validateLineRange(start_line, end_line, totalLines) {
|
|
122
|
+
// No range specified - return full file
|
|
123
|
+
if (start_line === undefined && end_line === undefined) {
|
|
124
|
+
return { valid: true, start: 1, end: totalLines };
|
|
125
|
+
}
|
|
126
|
+
// Only start_line specified
|
|
127
|
+
if (start_line !== undefined && end_line === undefined) {
|
|
128
|
+
if (start_line < 1 || start_line > totalLines) {
|
|
129
|
+
return {
|
|
130
|
+
valid: false,
|
|
131
|
+
start: 1,
|
|
132
|
+
end: totalLines,
|
|
133
|
+
error: `start_line ${start_line} out of range (1-${totalLines})`,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
return { valid: true, start: start_line, end: totalLines };
|
|
137
|
+
}
|
|
138
|
+
// Only end_line specified
|
|
139
|
+
if (start_line === undefined && end_line !== undefined) {
|
|
140
|
+
if (end_line < 1 || end_line > totalLines) {
|
|
141
|
+
return {
|
|
142
|
+
valid: false,
|
|
143
|
+
start: 1,
|
|
144
|
+
end: totalLines,
|
|
145
|
+
error: `end_line ${end_line} out of range (1-${totalLines})`,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
return { valid: true, start: 1, end: end_line };
|
|
149
|
+
}
|
|
150
|
+
// Both specified
|
|
151
|
+
if (start_line !== undefined && end_line !== undefined) {
|
|
152
|
+
if (start_line < 1 || start_line > totalLines) {
|
|
153
|
+
return {
|
|
154
|
+
valid: false,
|
|
155
|
+
start: 1,
|
|
156
|
+
end: totalLines,
|
|
157
|
+
error: `start_line ${start_line} out of range (1-${totalLines})`,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
if (end_line < 1 || end_line > totalLines) {
|
|
161
|
+
return {
|
|
162
|
+
valid: false,
|
|
163
|
+
start: 1,
|
|
164
|
+
end: totalLines,
|
|
165
|
+
error: `end_line ${end_line} out of range (1-${totalLines})`,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
if (start_line > end_line) {
|
|
169
|
+
return {
|
|
170
|
+
valid: false,
|
|
171
|
+
start: 1,
|
|
172
|
+
end: totalLines,
|
|
173
|
+
error: `start_line ${start_line} cannot be greater than end_line ${end_line}`,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
return { valid: true, start: start_line, end: end_line };
|
|
177
|
+
}
|
|
178
|
+
return { valid: true, start: 1, end: totalLines };
|
|
179
|
+
}
|
|
180
|
+
detectLanguage(filePath) {
|
|
181
|
+
const ext = extname(filePath).toLowerCase();
|
|
182
|
+
const map = {
|
|
183
|
+
'.js': 'javascript', '.jsx': 'javascript', '.mjs': 'javascript',
|
|
184
|
+
'.ts': 'typescript', '.tsx': 'typescript',
|
|
185
|
+
'.py': 'python', '.go': 'go', '.rs': 'rust', '.rb': 'ruby',
|
|
186
|
+
'.java': 'java', '.c': 'c', '.cpp': 'cpp', '.h': 'c',
|
|
187
|
+
'.json': 'json', '.yaml': 'yaml', '.yml': 'yaml', '.toml': 'toml',
|
|
188
|
+
'.md': 'markdown', '.html': 'html', '.css': 'css', '.sql': 'sql',
|
|
189
|
+
'.sh': 'bash', '.xml': 'xml',
|
|
190
|
+
};
|
|
191
|
+
if (filePath.endsWith('Dockerfile'))
|
|
192
|
+
return 'dockerfile';
|
|
193
|
+
return map[ext] || 'text';
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=file-attachment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-attachment.js","sourceRoot":"","sources":["../../src/services/file-attachment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgBpC,MAAM,OAAO,qBAAqB;IAChC;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,WAA6B;QACnD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CACnE,CAAC;QAEF,gCAAgC;QAChC,IAAI,MAAM,GAAG,oCAAoC,CAAC;QAClD,MAAM,IAAI,IAAI,OAAO,CAAC,MAAM,QAAQ,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,4BAA4B,CAAC;QAE9F,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC;YACzB,MAAM,IAAI,MAAM,CAAC;QACnB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,UAA0B;QACvD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;QAE/D,uBAAuB;QACvB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI;gBACJ,OAAO,EAAE,QAAQ,IAAI,2BAA2B,WAAW,CAAC,CAAC,CAAC,oBAAoB,WAAW,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxG,KAAK,EAAE,gBAAgB;aACxB,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,oBAAoB;YACpB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAE3C,uBAAuB;YACvB,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAClF,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;gBAC1B,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,IAAI;oBACJ,OAAO,EAAE,SAAS,IAAI,+BAA+B,cAAc,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,oBAAoB,WAAW,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;oBACtI,KAAK,EAAE,cAAc,CAAC,KAAK;iBAC5B,CAAC;YACJ,CAAC;YAED,yBAAyB;YACzB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,GAAG,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC;YACtC,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;YAExD,yBAAyB;YACzB,IAAI,SAAS,GAAG,SAAS,IAAI,MAAM,CAAC;YAEpC,eAAe;YACf,MAAM,SAAS,GAAG,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,CAAC;YACrE,SAAS,IAAI,iBAAiB,QAAQ,KAAK,CAAC;YAC5C,SAAS,IAAI,cAAc,SAAS,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,KAAK,IAAI,cAAc,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAC3G,SAAS,IAAI,aAAa,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;YAEpE,IAAI,WAAW,EAAE,CAAC;gBAChB,SAAS,IAAI,MAAM,WAAW,KAAK,CAAC;YACtC,CAAC;YAED,SAAS,IAAI,IAAI,CAAC;YAElB,qCAAqC;YACrC,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAErE,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,IAAI;gBACJ,OAAO,EAAE,SAAS;aACnB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI;gBACJ,OAAO,EAAE,QAAQ,IAAI,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC,oBAAoB,WAAW,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACvK,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAe,EAAE,QAAgB,EAAE,QAAgB;QACzE,IAAI,MAAM,GAAG,SAAS,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;QAEjD,gFAAgF;QAChF,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACvB,kBAAkB;YAClB,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACvC,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;gBAC/B,MAAM,UAAU,GAAG,QAAQ,GAAG,GAAG,GAAG,CAAC,CAAC;gBACtC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC;YACnE,CAAC,CAAC,CAAC;YAEH,oBAAoB;YACpB,MAAM,IAAI,UAAU,KAAK,CAAC,MAAM,GAAG,GAAG,uCAAuC,CAAC;YAE9E,iBAAiB;YACjB,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YACpC,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;gBAC9B,MAAM,UAAU,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;gBAC3D,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC;YACnE,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,8BAA8B;YAC9B,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;gBAC1B,MAAM,UAAU,GAAG,QAAQ,GAAG,GAAG,GAAG,CAAC,CAAC;gBACtC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC;YACnE,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,KAAK,CAAC;QAChB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,iBAAiB,CACvB,UAA8B,EAC9B,QAA4B,EAC5B,UAAkB;QAElB,wCAAwC;QACxC,IAAI,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACvD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;QACpD,CAAC;QAED,4BAA4B;QAC5B,IAAI,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACvD,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,UAAU,EAAE,CAAC;gBAC9C,OAAO;oBACL,KAAK,EAAE,KAAK;oBACZ,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,UAAU;oBACf,KAAK,EAAE,cAAc,UAAU,oBAAoB,UAAU,GAAG;iBACjE,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;QAC7D,CAAC;QAED,0BAA0B;QAC1B,IAAI,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACvD,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAC1C,OAAO;oBACL,KAAK,EAAE,KAAK;oBACZ,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,UAAU;oBACf,KAAK,EAAE,YAAY,QAAQ,oBAAoB,UAAU,GAAG;iBAC7D,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC;QAClD,CAAC;QAED,iBAAiB;QACjB,IAAI,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACvD,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,UAAU,EAAE,CAAC;gBAC9C,OAAO;oBACL,KAAK,EAAE,KAAK;oBACZ,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,UAAU;oBACf,KAAK,EAAE,cAAc,UAAU,oBAAoB,UAAU,GAAG;iBACjE,CAAC;YACJ,CAAC;YACD,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAC1C,OAAO;oBACL,KAAK,EAAE,KAAK;oBACZ,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,UAAU;oBACf,KAAK,EAAE,YAAY,QAAQ,oBAAoB,UAAU,GAAG;iBAC7D,CAAC;YACJ,CAAC;YACD,IAAI,UAAU,GAAG,QAAQ,EAAE,CAAC;gBAC1B,OAAO;oBACL,KAAK,EAAE,KAAK;oBACZ,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,UAAU;oBACf,KAAK,EAAE,cAAc,UAAU,oCAAoC,QAAQ,EAAE;iBAC9E,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC;QAC3D,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;IACpD,CAAC;IAEO,cAAc,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,GAAG,GAA2B;YAClC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY;YAC/D,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY;YACzC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;YAC1D,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG;YACpD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;YACjE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK;YAChE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK;SAC7B,CAAC;QACF,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC;YAAE,OAAO,YAAY,CAAC;QACzD,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;IAC5B,CAAC;CACF"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM Processor for content extraction
|
|
3
|
+
* Uses OpenRouter via OPENROUTER_API_KEY for AI-powered content filtering
|
|
4
|
+
*/
|
|
5
|
+
import OpenAI from 'openai';
|
|
6
|
+
interface ProcessingConfig {
|
|
7
|
+
use_llm: boolean;
|
|
8
|
+
what_to_extract: string | undefined;
|
|
9
|
+
max_tokens?: number;
|
|
10
|
+
}
|
|
11
|
+
interface LLMResult {
|
|
12
|
+
content: string;
|
|
13
|
+
processed: boolean;
|
|
14
|
+
error?: string;
|
|
15
|
+
}
|
|
16
|
+
export declare function createLLMProcessor(): OpenAI | null;
|
|
17
|
+
export declare function processContentWithLLM(content: string, config: ProcessingConfig, processor?: OpenAI | null): Promise<LLMResult>;
|
|
18
|
+
export {};
|
|
19
|
+
//# sourceMappingURL=llm-processor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm-processor.d.ts","sourceRoot":"","sources":["../../src/services/llm-processor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,UAAU,gBAAgB;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,wBAAgB,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAWlD;AAED,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,gBAAgB,EACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GACxB,OAAO,CAAC,SAAS,CAAC,CAyBpB"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM Processor for content extraction
|
|
3
|
+
* Uses OpenRouter via OPENROUTER_API_KEY for AI-powered content filtering
|
|
4
|
+
*/
|
|
5
|
+
import OpenAI from 'openai';
|
|
6
|
+
import { RESEARCH, LLM_EXTRACTION, getCapabilities } from '../config/index.js';
|
|
7
|
+
let llmClient = null;
|
|
8
|
+
export function createLLMProcessor() {
|
|
9
|
+
if (!getCapabilities().llmExtraction)
|
|
10
|
+
return null;
|
|
11
|
+
if (!llmClient) {
|
|
12
|
+
llmClient = new OpenAI({
|
|
13
|
+
baseURL: RESEARCH.BASE_URL,
|
|
14
|
+
apiKey: RESEARCH.API_KEY,
|
|
15
|
+
timeout: 120000,
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
return llmClient;
|
|
19
|
+
}
|
|
20
|
+
export async function processContentWithLLM(content, config, processor) {
|
|
21
|
+
if (!config.use_llm || !processor || !content?.trim()) {
|
|
22
|
+
return { content, processed: false };
|
|
23
|
+
}
|
|
24
|
+
const prompt = config.what_to_extract
|
|
25
|
+
? `Extract and clean the following content. Focus on: ${config.what_to_extract}\n\nContent:\n${content}`
|
|
26
|
+
: `Clean and extract the main content from the following text, removing navigation, ads, and irrelevant elements:\n\n${content}`;
|
|
27
|
+
try {
|
|
28
|
+
const response = await processor.chat.completions.create({
|
|
29
|
+
model: LLM_EXTRACTION.MODEL,
|
|
30
|
+
messages: [{ role: 'user', content: prompt }],
|
|
31
|
+
max_tokens: config.max_tokens || LLM_EXTRACTION.MAX_TOKENS,
|
|
32
|
+
});
|
|
33
|
+
const result = response.choices?.[0]?.message?.content;
|
|
34
|
+
if (result) {
|
|
35
|
+
return { content: result, processed: true };
|
|
36
|
+
}
|
|
37
|
+
return { content, processed: false, error: 'No content in response' };
|
|
38
|
+
}
|
|
39
|
+
catch (err) {
|
|
40
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
41
|
+
return { content, processed: false, error };
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=llm-processor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm-processor.js","sourceRoot":"","sources":["../../src/services/llm-processor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAc/E,IAAI,SAAS,GAAkB,IAAI,CAAC;AAEpC,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,eAAe,EAAE,CAAC,aAAa;QAAE,OAAO,IAAI,CAAC;IAElD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,SAAS,GAAG,IAAI,MAAM,CAAC;YACrB,OAAO,EAAE,QAAQ,CAAC,QAAQ;YAC1B,MAAM,EAAE,QAAQ,CAAC,OAAO;YACxB,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAe,EACf,MAAwB,EACxB,SAAyB;IAEzB,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;QACtD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvC,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe;QACnC,CAAC,CAAC,sDAAsD,MAAM,CAAC,eAAe,iBAAiB,OAAO,EAAE;QACxG,CAAC,CAAC,qHAAqH,OAAO,EAAE,CAAC;IAEnI,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACvD,KAAK,EAAE,cAAc,CAAC,KAAK;YAC3B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;YAC7C,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,cAAc,CAAC,UAAU;SAC3D,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC9C,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;IACxE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/D,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC9C,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-cleaner.d.ts","sourceRoot":"","sources":["../../src/services/markdown-cleaner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,qBAAa,eAAe;IAC1B,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;CA+D5C"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Simple markdown cleaner service
|
|
3
|
+
* Converts HTML to markdown and cleans up scraped content
|
|
4
|
+
*/
|
|
5
|
+
export class MarkdownCleaner {
|
|
6
|
+
processContent(htmlContent) {
|
|
7
|
+
if (!htmlContent || typeof htmlContent !== 'string') {
|
|
8
|
+
return htmlContent;
|
|
9
|
+
}
|
|
10
|
+
// Basic HTML to markdown conversion
|
|
11
|
+
let content = htmlContent;
|
|
12
|
+
// Remove script and style tags
|
|
13
|
+
content = content.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
|
|
14
|
+
content = content.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
|
|
15
|
+
// Remove HTML comments
|
|
16
|
+
content = content.replace(/<!--[\s\S]*?-->/g, '');
|
|
17
|
+
// Basic tag cleanup (preserve structure if already markdown)
|
|
18
|
+
if (!content.includes('<')) {
|
|
19
|
+
return content;
|
|
20
|
+
}
|
|
21
|
+
// Simple conversions
|
|
22
|
+
content = content.replace(/<br\s*\/?>/gi, '\n');
|
|
23
|
+
content = content.replace(/<\/p>/gi, '\n\n');
|
|
24
|
+
content = content.replace(/<p[^>]*>/gi, '');
|
|
25
|
+
content = content.replace(/<\/div>/gi, '\n');
|
|
26
|
+
content = content.replace(/<div[^>]*>/gi, '');
|
|
27
|
+
// Headers
|
|
28
|
+
content = content.replace(/<h1[^>]*>(.*?)<\/h1>/gi, '# $1\n');
|
|
29
|
+
content = content.replace(/<h2[^>]*>(.*?)<\/h2>/gi, '## $1\n');
|
|
30
|
+
content = content.replace(/<h3[^>]*>(.*?)<\/h3>/gi, '### $1\n');
|
|
31
|
+
// Lists
|
|
32
|
+
content = content.replace(/<li[^>]*>(.*?)<\/li>/gi, '- $1\n');
|
|
33
|
+
content = content.replace(/<\/ul>/gi, '\n');
|
|
34
|
+
content = content.replace(/<ul[^>]*>/gi, '');
|
|
35
|
+
// Links
|
|
36
|
+
content = content.replace(/<a[^>]*href=["']([^"']*)["'][^>]*>(.*?)<\/a>/gi, '[$2]($1)');
|
|
37
|
+
// Strong/bold
|
|
38
|
+
content = content.replace(/<(strong|b)[^>]*>(.*?)<\/\1>/gi, '**$2**');
|
|
39
|
+
// Emphasis/italic
|
|
40
|
+
content = content.replace(/<(em|i)[^>]*>(.*?)<\/\1>/gi, '*$2*');
|
|
41
|
+
// Remove remaining HTML tags
|
|
42
|
+
content = content.replace(/<[^>]+>/g, '');
|
|
43
|
+
// Decode HTML entities
|
|
44
|
+
content = content.replace(/ /g, ' ');
|
|
45
|
+
content = content.replace(/"/g, '"');
|
|
46
|
+
content = content.replace(/'/g, "'");
|
|
47
|
+
content = content.replace(/</g, '<');
|
|
48
|
+
content = content.replace(/>/g, '>');
|
|
49
|
+
content = content.replace(/&/g, '&');
|
|
50
|
+
// Clean up whitespace
|
|
51
|
+
content = content.replace(/\n{3,}/g, '\n\n');
|
|
52
|
+
content = content.trim();
|
|
53
|
+
return content;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=markdown-cleaner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-cleaner.js","sourceRoot":"","sources":["../../src/services/markdown-cleaner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,OAAO,eAAe;IAC1B,cAAc,CAAC,WAAmB;QAChC,IAAI,CAAC,WAAW,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;YACpD,OAAO,WAAW,CAAC;QACrB,CAAC;QAED,oCAAoC;QACpC,IAAI,OAAO,GAAG,WAAW,CAAC;QAE1B,+BAA+B;QAC/B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,qDAAqD,EAAE,EAAE,CAAC,CAAC;QACrF,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,kDAAkD,EAAE,EAAE,CAAC,CAAC;QAElF,uBAAuB;QACvB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAElD,6DAA6D;QAC7D,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,qBAAqB;QACrB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QAChD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAC5C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC7C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAE9C,UAAU;QACV,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,wBAAwB,EAAE,QAAQ,CAAC,CAAC;QAC9D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,wBAAwB,EAAE,SAAS,CAAC,CAAC;QAC/D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,wBAAwB,EAAE,UAAU,CAAC,CAAC;QAEhE,QAAQ;QACR,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,wBAAwB,EAAE,QAAQ,CAAC,CAAC;QAC9D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC5C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAE7C,QAAQ;QACR,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,gDAAgD,EAAE,UAAU,CAAC,CAAC;QAExF,cAAc;QACd,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,gCAAgC,EAAE,QAAQ,CAAC,CAAC;QAEtE,kBAAkB;QAClB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;QAEhE,6BAA6B;QAC7B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAE1C,uBAAuB;QACvB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC1C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC1C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC1C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAEzC,sBAAsB;QACtB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAEzB,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MCP Tool Definitions
|
|
3
|
+
* Extracted from index.ts for cleaner separation
|
|
4
|
+
*/
|
|
5
|
+
export declare const TOOLS: ({
|
|
6
|
+
name: string;
|
|
7
|
+
description: string;
|
|
8
|
+
inputSchema: {
|
|
9
|
+
type: "object";
|
|
10
|
+
properties: {
|
|
11
|
+
queries: {
|
|
12
|
+
type: string;
|
|
13
|
+
items: {
|
|
14
|
+
type: string;
|
|
15
|
+
};
|
|
16
|
+
description: string;
|
|
17
|
+
};
|
|
18
|
+
date_after: {
|
|
19
|
+
type: string;
|
|
20
|
+
description: string;
|
|
21
|
+
};
|
|
22
|
+
urls?: undefined;
|
|
23
|
+
fetch_comments?: undefined;
|
|
24
|
+
max_comments?: undefined;
|
|
25
|
+
};
|
|
26
|
+
required: string[];
|
|
27
|
+
};
|
|
28
|
+
} | {
|
|
29
|
+
name: string;
|
|
30
|
+
description: string;
|
|
31
|
+
inputSchema: {
|
|
32
|
+
type: "object";
|
|
33
|
+
properties: {
|
|
34
|
+
urls: {
|
|
35
|
+
type: string;
|
|
36
|
+
items: {
|
|
37
|
+
type: string;
|
|
38
|
+
};
|
|
39
|
+
description: string;
|
|
40
|
+
};
|
|
41
|
+
fetch_comments: {
|
|
42
|
+
type: string;
|
|
43
|
+
description: string;
|
|
44
|
+
default: boolean;
|
|
45
|
+
};
|
|
46
|
+
max_comments: {
|
|
47
|
+
type: string;
|
|
48
|
+
description: string;
|
|
49
|
+
default: number;
|
|
50
|
+
};
|
|
51
|
+
queries?: undefined;
|
|
52
|
+
date_after?: undefined;
|
|
53
|
+
};
|
|
54
|
+
required: string[];
|
|
55
|
+
};
|
|
56
|
+
} | {
|
|
57
|
+
name: string;
|
|
58
|
+
description: string;
|
|
59
|
+
inputSchema: import("zod-to-json-schema").JsonSchema7Type & {
|
|
60
|
+
$schema?: string | undefined;
|
|
61
|
+
definitions?: {
|
|
62
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
63
|
+
} | undefined;
|
|
64
|
+
};
|
|
65
|
+
})[];
|
|
66
|
+
//# sourceMappingURL=definitions.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"definitions.d.ts","sourceRoot":"","sources":["../../src/tools/definitions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAsHjB,CAAC"}
|