paper-search-cli 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -9
- package/README.zh.md +11 -9
- package/dist/capabilities/body-snippet-search/handler.d.ts +9 -0
- package/dist/capabilities/body-snippet-search/handler.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/handler.js +17 -0
- package/dist/capabilities/body-snippet-search/handler.js.map +1 -0
- package/dist/capabilities/body-snippet-search/index.d.ts +7 -0
- package/dist/capabilities/body-snippet-search/index.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/index.js +6 -0
- package/dist/capabilities/body-snippet-search/index.js.map +1 -0
- package/dist/capabilities/body-snippet-search/schemas.d.ts +36 -0
- package/dist/capabilities/body-snippet-search/schemas.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/schemas.js +16 -0
- package/dist/capabilities/body-snippet-search/schemas.js.map +1 -0
- package/dist/capabilities/body-snippet-search/service.d.ts +4 -0
- package/dist/capabilities/body-snippet-search/service.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/service.js +4 -0
- package/dist/capabilities/body-snippet-search/service.js.map +1 -0
- package/dist/capabilities/body-snippet-search/tools.d.ts +187 -0
- package/dist/capabilities/body-snippet-search/tools.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/tools.js +49 -0
- package/dist/capabilities/body-snippet-search/tools.js.map +1 -0
- package/dist/capabilities/body-snippet-search/types.d.ts +14 -0
- package/dist/capabilities/body-snippet-search/types.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/types.js +2 -0
- package/dist/capabilities/body-snippet-search/types.js.map +1 -0
- package/dist/capabilities/citation-expansion/CitationService.d.ts +47 -0
- package/dist/capabilities/citation-expansion/CitationService.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/CitationService.js +243 -0
- package/dist/capabilities/citation-expansion/CitationService.js.map +1 -0
- package/dist/capabilities/citation-expansion/handler.d.ts +36 -0
- package/dist/capabilities/citation-expansion/handler.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/handler.js +44 -0
- package/dist/capabilities/citation-expansion/handler.js.map +1 -0
- package/dist/capabilities/citation-expansion/index.d.ts +6 -0
- package/dist/capabilities/citation-expansion/index.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/index.js +5 -0
- package/dist/capabilities/citation-expansion/index.js.map +1 -0
- package/dist/capabilities/citation-expansion/schemas.d.ts +28 -0
- package/dist/capabilities/citation-expansion/schemas.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/schemas.js +13 -0
- package/dist/capabilities/citation-expansion/schemas.js.map +1 -0
- package/dist/capabilities/citation-expansion/tools.d.ts +82 -0
- package/dist/capabilities/citation-expansion/tools.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/tools.js +48 -0
- package/dist/capabilities/citation-expansion/tools.js.map +1 -0
- package/dist/capabilities/citation-expansion/types.d.ts +22 -0
- package/dist/capabilities/citation-expansion/types.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/types.js +2 -0
- package/dist/capabilities/citation-expansion/types.js.map +1 -0
- package/dist/capabilities/journal-metrics/JournalMetricsService.d.ts +3 -0
- package/dist/capabilities/journal-metrics/JournalMetricsService.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/JournalMetricsService.js +142 -0
- package/dist/capabilities/journal-metrics/JournalMetricsService.js.map +1 -0
- package/dist/capabilities/journal-metrics/handler.d.ts +12 -0
- package/dist/capabilities/journal-metrics/handler.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/handler.js +38 -0
- package/dist/capabilities/journal-metrics/handler.js.map +1 -0
- package/dist/capabilities/journal-metrics/index.d.ts +6 -0
- package/dist/capabilities/journal-metrics/index.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/index.js +5 -0
- package/dist/capabilities/journal-metrics/index.js.map +1 -0
- package/dist/capabilities/journal-metrics/schemas.d.ts +28 -0
- package/dist/capabilities/journal-metrics/schemas.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/schemas.js +13 -0
- package/dist/capabilities/journal-metrics/schemas.js.map +1 -0
- package/dist/capabilities/journal-metrics/tools.d.ts +67 -0
- package/dist/capabilities/journal-metrics/tools.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/tools.js +27 -0
- package/dist/capabilities/journal-metrics/tools.js.map +1 -0
- package/dist/capabilities/journal-metrics/types.d.ts +36 -0
- package/dist/capabilities/journal-metrics/types.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/types.js +2 -0
- package/dist/capabilities/journal-metrics/types.js.map +1 -0
- package/dist/capabilities/metadata-search/MultiSourceSearchService.d.ts +9 -0
- package/dist/capabilities/metadata-search/MultiSourceSearchService.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/MultiSourceSearchService.js +78 -0
- package/dist/capabilities/metadata-search/MultiSourceSearchService.js.map +1 -0
- package/dist/capabilities/metadata-search/handler.d.ts +23 -0
- package/dist/capabilities/metadata-search/handler.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/handler.js +163 -0
- package/dist/capabilities/metadata-search/handler.js.map +1 -0
- package/dist/capabilities/metadata-search/index.d.ts +6 -0
- package/dist/capabilities/metadata-search/index.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/index.js +5 -0
- package/dist/capabilities/metadata-search/index.js.map +1 -0
- package/dist/capabilities/metadata-search/schemas.d.ts +83 -0
- package/dist/capabilities/metadata-search/schemas.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/schemas.js +81 -0
- package/dist/capabilities/metadata-search/schemas.js.map +1 -0
- package/dist/capabilities/metadata-search/tools.d.ts +178 -0
- package/dist/capabilities/metadata-search/tools.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/tools.js +75 -0
- package/dist/capabilities/metadata-search/tools.js.map +1 -0
- package/dist/capabilities/metadata-search/types.d.ts +13 -0
- package/dist/capabilities/metadata-search/types.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/types.js +2 -0
- package/dist/capabilities/metadata-search/types.js.map +1 -0
- package/dist/capabilities/pdf-discovery/DownloadTier.d.ts +22 -0
- package/dist/capabilities/pdf-discovery/DownloadTier.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/DownloadTier.js +2 -0
- package/dist/capabilities/pdf-discovery/DownloadTier.js.map +1 -0
- package/dist/capabilities/pdf-discovery/OpenAccessFallbackService.d.ts +10 -0
- package/dist/capabilities/pdf-discovery/OpenAccessFallbackService.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/OpenAccessFallbackService.js +53 -0
- package/dist/capabilities/pdf-discovery/OpenAccessFallbackService.js.map +1 -0
- package/dist/capabilities/pdf-discovery/handler.d.ts +19 -0
- package/dist/capabilities/pdf-discovery/handler.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/handler.js +56 -0
- package/dist/capabilities/pdf-discovery/handler.js.map +1 -0
- package/dist/capabilities/pdf-discovery/index.d.ts +7 -0
- package/dist/capabilities/pdf-discovery/index.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/index.js +5 -0
- package/dist/capabilities/pdf-discovery/index.js.map +1 -0
- package/dist/capabilities/pdf-discovery/schemas.d.ts +37 -0
- package/dist/capabilities/pdf-discovery/schemas.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/schemas.js +22 -0
- package/dist/capabilities/pdf-discovery/schemas.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/directPdfUrl.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/tiers/directPdfUrl.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/directPdfUrl.js +27 -0
- package/dist/capabilities/pdf-discovery/tiers/directPdfUrl.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/primary.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/tiers/primary.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/primary.js +21 -0
- package/dist/capabilities/pdf-discovery/tiers/primary.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/repositories.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/tiers/repositories.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/repositories.js +35 -0
- package/dist/capabilities/pdf-discovery/tiers/repositories.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/scihub.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/tiers/scihub.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/scihub.js +21 -0
- package/dist/capabilities/pdf-discovery/tiers/scihub.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/unpaywall.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/tiers/unpaywall.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/unpaywall.js +29 -0
- package/dist/capabilities/pdf-discovery/tiers/unpaywall.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tools.d.ts +114 -0
- package/dist/capabilities/pdf-discovery/tools.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tools.js +42 -0
- package/dist/capabilities/pdf-discovery/tools.js.map +1 -0
- package/dist/capabilities/pdf-discovery/types.d.ts +19 -0
- package/dist/capabilities/pdf-discovery/types.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/types.js +2 -0
- package/dist/capabilities/pdf-discovery/types.js.map +1 -0
- package/dist/cli.js +10 -6
- package/dist/cli.js.map +1 -1
- package/dist/core/capabilityProfile.d.ts +1 -17
- package/dist/core/capabilityProfile.d.ts.map +1 -1
- package/dist/core/capabilityProfile.js +1 -152
- package/dist/core/capabilityProfile.js.map +1 -1
- package/dist/core/diagnostics.d.ts +1 -42
- package/dist/core/diagnostics.d.ts.map +1 -1
- package/dist/core/diagnostics.js +1 -585
- package/dist/core/diagnostics.js.map +1 -1
- package/dist/core/handleToolCall.d.ts +1 -0
- package/dist/core/handleToolCall.d.ts.map +1 -1
- package/dist/core/handleToolCall.js +265 -476
- package/dist/core/handleToolCall.js.map +1 -1
- package/dist/core/httpPolicies.d.ts +2 -0
- package/dist/core/httpPolicies.d.ts.map +1 -0
- package/dist/core/httpPolicies.js +2 -0
- package/dist/core/httpPolicies.js.map +1 -0
- package/dist/core/liveSmoke.d.ts +1 -41
- package/dist/core/liveSmoke.d.ts.map +1 -1
- package/dist/core/liveSmoke.js +1 -225
- package/dist/core/liveSmoke.js.map +1 -1
- package/dist/core/platformFactories.d.ts +2 -0
- package/dist/core/platformFactories.d.ts.map +1 -0
- package/dist/core/platformFactories.js +2 -0
- package/dist/core/platformFactories.js.map +1 -0
- package/dist/core/platformMetadata.d.ts +1 -27
- package/dist/core/platformMetadata.d.ts.map +1 -1
- package/dist/core/platformMetadata.js +1 -257
- package/dist/core/platformMetadata.js.map +1 -1
- package/dist/core/schemas.d.ts +11 -177
- package/dist/core/schemas.d.ts.map +1 -1
- package/dist/core/schemas.js +14 -107
- package/dist/core/schemas.js.map +1 -1
- package/dist/core/searchers.d.ts +24 -24
- package/dist/core/searchers.d.ts.map +1 -1
- package/dist/core/searchers.js +14 -79
- package/dist/core/searchers.js.map +1 -1
- package/dist/core/textReports.d.ts +1 -20
- package/dist/core/textReports.d.ts.map +1 -1
- package/dist/core/textReports.js +1 -84
- package/dist/core/textReports.js.map +1 -1
- package/dist/core/tools.d.ts.map +1 -1
- package/dist/core/tools.js +15 -183
- package/dist/core/tools.js.map +1 -1
- package/dist/infrastructure/cache/RequestCache.d.ts +26 -0
- package/dist/infrastructure/cache/RequestCache.d.ts.map +1 -0
- package/dist/infrastructure/cache/RequestCache.js +66 -0
- package/dist/infrastructure/cache/RequestCache.js.map +1 -0
- package/dist/infrastructure/http/ErrorHandler.d.ts +99 -0
- package/dist/infrastructure/http/ErrorHandler.d.ts.map +1 -0
- package/dist/infrastructure/http/ErrorHandler.js +266 -0
- package/dist/infrastructure/http/ErrorHandler.js.map +1 -0
- package/dist/infrastructure/http/HttpClient.d.ts +31 -0
- package/dist/infrastructure/http/HttpClient.d.ts.map +1 -0
- package/dist/infrastructure/http/HttpClient.js +50 -0
- package/dist/infrastructure/http/HttpClient.js.map +1 -0
- package/dist/infrastructure/pdf/PDFExtractor.d.ts +34 -0
- package/dist/infrastructure/pdf/PDFExtractor.d.ts.map +1 -0
- package/dist/infrastructure/pdf/PDFExtractor.js +130 -0
- package/dist/infrastructure/pdf/PDFExtractor.js.map +1 -0
- package/dist/infrastructure/pdf/PdfDownload.d.ts +7 -0
- package/dist/infrastructure/pdf/PdfDownload.d.ts.map +1 -0
- package/dist/infrastructure/pdf/PdfDownload.js +175 -0
- package/dist/infrastructure/pdf/PdfDownload.js.map +1 -0
- package/dist/infrastructure/rate-limit/RateLimiter.d.ts +50 -0
- package/dist/infrastructure/rate-limit/RateLimiter.d.ts.map +1 -0
- package/dist/infrastructure/rate-limit/RateLimiter.js +128 -0
- package/dist/infrastructure/rate-limit/RateLimiter.js.map +1 -0
- package/dist/infrastructure/security/SecurityUtils.d.ts +80 -0
- package/dist/infrastructure/security/SecurityUtils.d.ts.map +1 -0
- package/dist/infrastructure/security/SecurityUtils.js +357 -0
- package/dist/infrastructure/security/SecurityUtils.js.map +1 -0
- package/dist/management/capability-profile/index.d.ts +18 -0
- package/dist/management/capability-profile/index.d.ts.map +1 -0
- package/dist/management/capability-profile/index.js +167 -0
- package/dist/management/capability-profile/index.js.map +1 -0
- package/dist/management/config/index.d.ts +2 -0
- package/dist/management/config/index.d.ts.map +1 -0
- package/dist/management/config/index.js +2 -0
- package/dist/management/config/index.js.map +1 -0
- package/dist/management/diagnostics/index.d.ts +43 -0
- package/dist/management/diagnostics/index.d.ts.map +1 -0
- package/dist/management/diagnostics/index.js +586 -0
- package/dist/management/diagnostics/index.js.map +1 -0
- package/dist/management/reports/index.d.ts +21 -0
- package/dist/management/reports/index.d.ts.map +1 -0
- package/dist/management/reports/index.js +85 -0
- package/dist/management/reports/index.js.map +1 -0
- package/dist/management/skills/index.d.ts +2 -0
- package/dist/management/skills/index.d.ts.map +1 -0
- package/dist/management/skills/index.js +2 -0
- package/dist/management/skills/index.js.map +1 -0
- package/dist/management/smoke/index.d.ts +42 -0
- package/dist/management/smoke/index.d.ts.map +1 -0
- package/dist/management/smoke/index.js +226 -0
- package/dist/management/smoke/index.js.map +1 -0
- package/dist/platforms/BioRxivSearcher.d.ts.map +1 -1
- package/dist/platforms/BioRxivSearcher.js +40 -21
- package/dist/platforms/BioRxivSearcher.js.map +1 -1
- package/dist/platforms/GoogleScholarSearcher.d.ts.map +1 -1
- package/dist/platforms/GoogleScholarSearcher.js +3 -2
- package/dist/platforms/GoogleScholarSearcher.js.map +1 -1
- package/dist/registry/aliases.d.ts +2 -0
- package/dist/registry/aliases.d.ts.map +1 -0
- package/dist/registry/aliases.js +2 -0
- package/dist/registry/aliases.js.map +1 -0
- package/dist/registry/httpPolicies.d.ts +3 -0
- package/dist/registry/httpPolicies.d.ts.map +1 -0
- package/dist/registry/httpPolicies.js +17 -0
- package/dist/registry/httpPolicies.js.map +1 -0
- package/dist/registry/platformFactories.d.ts +8 -0
- package/dist/registry/platformFactories.d.ts.map +1 -0
- package/dist/registry/platformFactories.js +55 -0
- package/dist/registry/platformFactories.js.map +1 -0
- package/dist/registry/platformMetadata.d.ts +39 -0
- package/dist/registry/platformMetadata.d.ts.map +1 -0
- package/dist/registry/platformMetadata.js +321 -0
- package/dist/registry/platformMetadata.js.map +1 -0
- package/dist/services/CitationService.d.ts +2 -65
- package/dist/services/CitationService.d.ts.map +1 -1
- package/dist/services/CitationService.js +1 -236
- package/dist/services/CitationService.js.map +1 -1
- package/dist/services/JournalMetricsService.d.ts +2 -37
- package/dist/services/JournalMetricsService.d.ts.map +1 -1
- package/dist/services/JournalMetricsService.js +1 -141
- package/dist/services/JournalMetricsService.js.map +1 -1
- package/dist/services/MultiSourceSearchService.d.ts +2 -18
- package/dist/services/MultiSourceSearchService.d.ts.map +1 -1
- package/dist/services/MultiSourceSearchService.js +1 -77
- package/dist/services/MultiSourceSearchService.js.map +1 -1
- package/dist/services/OpenAccessFallbackService.d.ts +3 -20
- package/dist/services/OpenAccessFallbackService.d.ts.map +1 -1
- package/dist/services/OpenAccessFallbackService.js +1 -123
- package/dist/services/OpenAccessFallbackService.js.map +1 -1
- package/dist/utils/ErrorHandler.d.ts +2 -98
- package/dist/utils/ErrorHandler.d.ts.map +1 -1
- package/dist/utils/ErrorHandler.js +2 -265
- package/dist/utils/ErrorHandler.js.map +1 -1
- package/dist/utils/HttpClient.d.ts +1 -5
- package/dist/utils/HttpClient.d.ts.map +1 -1
- package/dist/utils/HttpClient.js +1 -29
- package/dist/utils/HttpClient.js.map +1 -1
- package/dist/utils/PDFExtractor.d.ts +2 -33
- package/dist/utils/PDFExtractor.d.ts.map +1 -1
- package/dist/utils/PDFExtractor.js +2 -129
- package/dist/utils/PDFExtractor.js.map +1 -1
- package/dist/utils/PdfDownload.d.ts +1 -6
- package/dist/utils/PdfDownload.d.ts.map +1 -1
- package/dist/utils/PdfDownload.js +1 -141
- package/dist/utils/PdfDownload.js.map +1 -1
- package/dist/utils/RateLimiter.d.ts +1 -49
- package/dist/utils/RateLimiter.d.ts.map +1 -1
- package/dist/utils/RateLimiter.js +1 -120
- package/dist/utils/RateLimiter.js.map +1 -1
- package/dist/utils/RequestCache.d.ts +2 -25
- package/dist/utils/RequestCache.d.ts.map +1 -1
- package/dist/utils/RequestCache.js +2 -65
- package/dist/utils/RequestCache.js.map +1 -1
- package/dist/utils/SecurityUtils.d.ts +2 -79
- package/dist/utils/SecurityUtils.d.ts.map +1 -1
- package/dist/utils/SecurityUtils.js +2 -356
- package/dist/utils/SecurityUtils.js.map +1 -1
- package/package.json +1 -1
- package/skills/paper-search/SKILL.md +4 -2
- package/skills/paper-search/references/capability-routing.md +15 -2
- package/skills/paper-search/references/cli-contract.md +19 -0
- package/skills/paper-search/references/management-layer.md +1 -0
|
@@ -1,130 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import * as path from 'path';
|
|
4
|
-
// @ts-ignore - pdf-parse doesn't have type definitions
|
|
5
|
-
import pdf from 'pdf-parse';
|
|
6
|
-
import { logDebug, logWarn } from './Logger.js';
|
|
7
|
-
import { TIMEOUTS, USER_AGENT } from '../config/constants.js';
|
|
8
|
-
export class PDFExtractor {
|
|
9
|
-
/**
|
|
10
|
-
* Download PDF from URL and extract text
|
|
11
|
-
*/
|
|
12
|
-
async extractFromUrl(url, options = {}) {
|
|
13
|
-
try {
|
|
14
|
-
logDebug(`Downloading PDF from: ${url}`);
|
|
15
|
-
const response = await axios.get(url, {
|
|
16
|
-
responseType: 'arraybuffer',
|
|
17
|
-
timeout: TIMEOUTS.DOWNLOAD,
|
|
18
|
-
headers: {
|
|
19
|
-
'User-Agent': USER_AGENT,
|
|
20
|
-
'Accept': 'application/pdf'
|
|
21
|
-
}
|
|
22
|
-
});
|
|
23
|
-
if (response.status !== 200) {
|
|
24
|
-
throw new Error(`Failed to download PDF: HTTP ${response.status}`);
|
|
25
|
-
}
|
|
26
|
-
const buffer = Buffer.from(response.data);
|
|
27
|
-
return await this.extractFromBuffer(buffer, options);
|
|
28
|
-
}
|
|
29
|
-
catch (error) {
|
|
30
|
-
logWarn(`Error downloading PDF from ${url}:`, error.message);
|
|
31
|
-
throw error;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
/**
|
|
35
|
-
* Extract text from PDF file
|
|
36
|
-
*/
|
|
37
|
-
async extractFromFile(filePath, options = {}) {
|
|
38
|
-
try {
|
|
39
|
-
if (!fs.existsSync(filePath)) {
|
|
40
|
-
throw new Error(`PDF file not found: ${filePath}`);
|
|
41
|
-
}
|
|
42
|
-
logDebug(`Extracting text from PDF file: ${filePath}`);
|
|
43
|
-
const dataBuffer = fs.readFileSync(filePath);
|
|
44
|
-
return await this.extractFromBuffer(dataBuffer, options);
|
|
45
|
-
}
|
|
46
|
-
catch (error) {
|
|
47
|
-
logWarn(`Error extracting PDF from file ${filePath}:`, error.message);
|
|
48
|
-
throw error;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Extract text from PDF buffer
|
|
53
|
-
*/
|
|
54
|
-
async extractFromBuffer(buffer, options = {}) {
|
|
55
|
-
try {
|
|
56
|
-
const pdfOptions = {};
|
|
57
|
-
if (options.maxPages) {
|
|
58
|
-
pdfOptions.max = options.maxPages;
|
|
59
|
-
}
|
|
60
|
-
const data = await pdf(buffer, pdfOptions);
|
|
61
|
-
let text = data.text;
|
|
62
|
-
if (options.cleanText !== false) {
|
|
63
|
-
text = this.cleanText(text);
|
|
64
|
-
}
|
|
65
|
-
logDebug(`Extracted ${data.numpages} pages, ${text.length} characters`);
|
|
66
|
-
return {
|
|
67
|
-
text,
|
|
68
|
-
numPages: data.numpages,
|
|
69
|
-
info: data.info,
|
|
70
|
-
metadata: data.metadata
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
catch (error) {
|
|
74
|
-
logWarn('Error parsing PDF:', error.message);
|
|
75
|
-
throw error;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
/**
|
|
79
|
-
* Clean extracted text
|
|
80
|
-
*/
|
|
81
|
-
cleanText(text) {
|
|
82
|
-
return text
|
|
83
|
-
// Remove excessive whitespace
|
|
84
|
-
.replace(/\s+/g, ' ')
|
|
85
|
-
// Remove page numbers (common patterns)
|
|
86
|
-
.replace(/\n\s*\d+\s*\n/g, '\n')
|
|
87
|
-
// Remove headers/footers (lines with only numbers or short text)
|
|
88
|
-
.replace(/\n\s*[\w\s]{1,30}\s*\n/g, '\n')
|
|
89
|
-
// Normalize line breaks
|
|
90
|
-
.replace(/\n{3,}/g, '\n\n')
|
|
91
|
-
// Trim
|
|
92
|
-
.trim();
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* Download PDF and save to file
|
|
96
|
-
*/
|
|
97
|
-
async downloadPdf(url, savePath) {
|
|
98
|
-
try {
|
|
99
|
-
logDebug(`Downloading PDF from ${url} to ${savePath}`);
|
|
100
|
-
// Ensure directory exists
|
|
101
|
-
const dir = path.dirname(savePath);
|
|
102
|
-
if (!fs.existsSync(dir)) {
|
|
103
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
104
|
-
}
|
|
105
|
-
const response = await axios.get(url, {
|
|
106
|
-
responseType: 'stream',
|
|
107
|
-
timeout: TIMEOUTS.DOWNLOAD,
|
|
108
|
-
headers: {
|
|
109
|
-
'User-Agent': USER_AGENT,
|
|
110
|
-
'Accept': 'application/pdf'
|
|
111
|
-
}
|
|
112
|
-
});
|
|
113
|
-
const writer = fs.createWriteStream(savePath);
|
|
114
|
-
response.data.pipe(writer);
|
|
115
|
-
return new Promise((resolve, reject) => {
|
|
116
|
-
writer.on('finish', () => {
|
|
117
|
-
logDebug(`PDF saved to: ${savePath}`);
|
|
118
|
-
resolve(savePath);
|
|
119
|
-
});
|
|
120
|
-
writer.on('error', reject);
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
|
-
catch (error) {
|
|
124
|
-
logWarn(`Error downloading PDF:`, error.message);
|
|
125
|
-
throw error;
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
export default PDFExtractor;
|
|
1
|
+
export * from '../infrastructure/pdf/PDFExtractor.js';
|
|
2
|
+
export { default } from '../infrastructure/pdf/PDFExtractor.js';
|
|
130
3
|
//# sourceMappingURL=PDFExtractor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PDFExtractor.js","sourceRoot":"","sources":["../../src/utils/PDFExtractor.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"PDFExtractor.js","sourceRoot":"","sources":["../../src/utils/PDFExtractor.ts"],"names":[],"mappings":"AAAA,cAAc,uCAAuC,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,uCAAuC,CAAC"}
|
|
@@ -1,7 +1,2 @@
|
|
|
1
|
-
export
|
|
2
|
-
headers?: Record<string, string>;
|
|
3
|
-
}
|
|
4
|
-
export declare function safeFilename(value: string, fallback?: string): string;
|
|
5
|
-
export declare function isPdfBuffer(buffer: Buffer, contentType?: string): boolean;
|
|
6
|
-
export declare function downloadPdfFromUrl(pdfUrl: string, savePath: string, filenameHint: string, options?: PdfDownloadOptions): Promise<string>;
|
|
1
|
+
export * from '../infrastructure/pdf/PdfDownload.js';
|
|
7
2
|
//# sourceMappingURL=PdfDownload.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PdfDownload.d.ts","sourceRoot":"","sources":["../../src/utils/PdfDownload.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"PdfDownload.d.ts","sourceRoot":"","sources":["../../src/utils/PdfDownload.ts"],"names":[],"mappings":"AAAA,cAAc,sCAAsC,CAAC"}
|
|
@@ -1,142 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
import * as fs from 'fs';
|
|
3
|
-
import * as path from 'path';
|
|
4
|
-
import { Readable } from 'stream';
|
|
5
|
-
import { TIMEOUTS, USER_AGENT } from '../config/constants.js';
|
|
6
|
-
export function safeFilename(value, fallback = 'paper') {
|
|
7
|
-
const safe = value.replace(/[^a-zA-Z0-9._-]+/g, '_').replace(/^[_\-.]+|[_\-.]+$/g, '');
|
|
8
|
-
return (safe || fallback).slice(0, 120);
|
|
9
|
-
}
|
|
10
|
-
export function isPdfBuffer(buffer, contentType = '') {
|
|
11
|
-
return contentType.toLowerCase().includes('pdf') || buffer.subarray(0, 4).toString() === '%PDF';
|
|
12
|
-
}
|
|
13
|
-
async function getStreamSnippet(stream, maxBytes = 240) {
|
|
14
|
-
return new Promise((resolve) => {
|
|
15
|
-
let buffer = Buffer.alloc(0);
|
|
16
|
-
const onData = (chunk) => {
|
|
17
|
-
buffer = Buffer.concat([buffer, chunk]);
|
|
18
|
-
if (buffer.length >= maxBytes) {
|
|
19
|
-
cleanup();
|
|
20
|
-
stream.destroy?.();
|
|
21
|
-
}
|
|
22
|
-
};
|
|
23
|
-
const cleanup = () => {
|
|
24
|
-
if (typeof stream.off === 'function') {
|
|
25
|
-
stream.off('data', onData);
|
|
26
|
-
}
|
|
27
|
-
};
|
|
28
|
-
stream.on('data', onData);
|
|
29
|
-
stream.on('end', () => {
|
|
30
|
-
cleanup();
|
|
31
|
-
resolve(buffer.subarray(0, maxBytes).toString('utf8').replace(/\s+/g, ' ').trim());
|
|
32
|
-
});
|
|
33
|
-
stream.on('close', () => {
|
|
34
|
-
cleanup();
|
|
35
|
-
resolve(buffer.subarray(0, maxBytes).toString('utf8').replace(/\s+/g, ' ').trim());
|
|
36
|
-
});
|
|
37
|
-
stream.on('error', () => {
|
|
38
|
-
cleanup();
|
|
39
|
-
resolve(buffer.subarray(0, maxBytes).toString('utf8').replace(/\s+/g, ' ').trim());
|
|
40
|
-
});
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
export async function downloadPdfFromUrl(pdfUrl, savePath, filenameHint, options = {}) {
|
|
44
|
-
if (!pdfUrl) {
|
|
45
|
-
throw new Error('Missing PDF URL');
|
|
46
|
-
}
|
|
47
|
-
if (pdfUrl.startsWith('ftp://')) {
|
|
48
|
-
throw new Error(`FTP PDF links are not supported by the Node downloader: ${pdfUrl}`);
|
|
49
|
-
}
|
|
50
|
-
fs.mkdirSync(savePath, { recursive: true });
|
|
51
|
-
const outputPath = path.join(savePath, `${safeFilename(filenameHint)}.pdf`);
|
|
52
|
-
const response = await axios.get(pdfUrl, {
|
|
53
|
-
responseType: 'stream',
|
|
54
|
-
timeout: TIMEOUTS.DOWNLOAD,
|
|
55
|
-
maxRedirects: 5,
|
|
56
|
-
headers: {
|
|
57
|
-
'User-Agent': USER_AGENT,
|
|
58
|
-
Accept: 'application/pdf,*/*',
|
|
59
|
-
...(options.headers || {})
|
|
60
|
-
},
|
|
61
|
-
validateStatus: status => status < 500
|
|
62
|
-
});
|
|
63
|
-
if (response.status >= 400) {
|
|
64
|
-
let errStream = response.data;
|
|
65
|
-
if (!(errStream instanceof Readable) && !(errStream && typeof errStream.on === 'function')) {
|
|
66
|
-
errStream = Readable.from(Buffer.isBuffer(errStream) ? errStream : Buffer.from(errStream || ''));
|
|
67
|
-
}
|
|
68
|
-
const errorSnippet = await getStreamSnippet(errStream);
|
|
69
|
-
const cfHeader = response.headers['cf-mitigated'];
|
|
70
|
-
const challenge = !!cfHeader || /cloudflare|challenge|just a moment/i.test(errorSnippet);
|
|
71
|
-
const reason = challenge ? 'provider returned an HTML anti-bot challenge instead of a PDF' : 'provider refused the request';
|
|
72
|
-
throw new Error(`PDF download failed with HTTP ${response.status}: ${reason}`);
|
|
73
|
-
}
|
|
74
|
-
return new Promise((resolve, reject) => {
|
|
75
|
-
const writer = fs.createWriteStream(outputPath);
|
|
76
|
-
let reader = response.data;
|
|
77
|
-
if (!(reader instanceof Readable) && !(reader && typeof reader.on === 'function')) {
|
|
78
|
-
reader = Readable.from(Buffer.isBuffer(reader) ? reader : Buffer.from(reader || ''));
|
|
79
|
-
}
|
|
80
|
-
let checked = false;
|
|
81
|
-
let totalBytes = 0;
|
|
82
|
-
const contentLengthHeader = response.headers['content-length'];
|
|
83
|
-
const expectedLength = contentLengthHeader ? parseInt(String(contentLengthHeader), 10) : 0;
|
|
84
|
-
writer.on('error', (err) => {
|
|
85
|
-
if (typeof reader.destroy === 'function')
|
|
86
|
-
reader.destroy();
|
|
87
|
-
fs.unlink(outputPath, () => { });
|
|
88
|
-
reject(err);
|
|
89
|
-
});
|
|
90
|
-
reader.on('data', (chunk) => {
|
|
91
|
-
totalBytes += chunk.length;
|
|
92
|
-
if (!checked) {
|
|
93
|
-
checked = true;
|
|
94
|
-
const contentType = String(response.headers['content-type'] || '').toLowerCase();
|
|
95
|
-
const isPdf = contentType.includes('pdf') || chunk.subarray(0, 4).toString() === '%PDF';
|
|
96
|
-
if (!isPdf) {
|
|
97
|
-
const snippet = chunk.subarray(0, 240).toString('utf8').replace(/\s+/g, ' ').trim();
|
|
98
|
-
const cfHeader = response.headers['cf-mitigated'];
|
|
99
|
-
const challenge = !!cfHeader || /cloudflare|challenge|preparing to download|proof-of-work|<!doctype html/i.test(snippet);
|
|
100
|
-
const reason = challenge
|
|
101
|
-
? 'the provider returned an HTML challenge page instead of a PDF'
|
|
102
|
-
: `content-type was ${contentType || 'unknown'}`;
|
|
103
|
-
if (typeof reader.destroy === 'function')
|
|
104
|
-
reader.destroy();
|
|
105
|
-
writer.destroy();
|
|
106
|
-
fs.unlink(outputPath, () => { });
|
|
107
|
-
reject(new Error(`Resolved URL did not return a PDF (${reason}): ${pdfUrl}`));
|
|
108
|
-
return;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
if (!writer.write(chunk)) {
|
|
112
|
-
if (typeof reader.pause === 'function')
|
|
113
|
-
reader.pause();
|
|
114
|
-
}
|
|
115
|
-
});
|
|
116
|
-
writer.on('drain', () => {
|
|
117
|
-
if (typeof reader.resume === 'function')
|
|
118
|
-
reader.resume();
|
|
119
|
-
});
|
|
120
|
-
reader.on('end', () => {
|
|
121
|
-
writer.end();
|
|
122
|
-
});
|
|
123
|
-
reader.on('close', () => {
|
|
124
|
-
if (typeof reader.destroy === 'function')
|
|
125
|
-
reader.destroy();
|
|
126
|
-
});
|
|
127
|
-
writer.on('finish', () => {
|
|
128
|
-
if (expectedLength && totalBytes !== expectedLength) {
|
|
129
|
-
fs.unlink(outputPath, () => { });
|
|
130
|
-
reject(new Error(`PDF download incomplete: received ${totalBytes} bytes, expected ${expectedLength} bytes`));
|
|
131
|
-
return;
|
|
132
|
-
}
|
|
133
|
-
resolve(outputPath);
|
|
134
|
-
});
|
|
135
|
-
reader.on('error', (err) => {
|
|
136
|
-
writer.destroy();
|
|
137
|
-
fs.unlink(outputPath, () => { });
|
|
138
|
-
reject(err);
|
|
139
|
-
});
|
|
140
|
-
});
|
|
141
|
-
}
|
|
1
|
+
export * from '../infrastructure/pdf/PdfDownload.js';
|
|
142
2
|
//# sourceMappingURL=PdfDownload.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PdfDownload.js","sourceRoot":"","sources":["../../src/utils/PdfDownload.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"PdfDownload.js","sourceRoot":"","sources":["../../src/utils/PdfDownload.ts"],"names":[],"mappings":"AAAA,cAAc,sCAAsC,CAAC"}
|
|
@@ -1,50 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
* 请求速率限制器
|
|
3
|
-
* 用于控制API请求频率,遵守各平台的使用限制
|
|
4
|
-
*/
|
|
5
|
-
export interface RateLimiterOptions {
|
|
6
|
-
/** 每秒最大请求数 */
|
|
7
|
-
requestsPerSecond: number;
|
|
8
|
-
/** 突发请求容量 */
|
|
9
|
-
burstCapacity?: number;
|
|
10
|
-
/** 是否启用调试日志 */
|
|
11
|
-
debug?: boolean;
|
|
12
|
-
}
|
|
13
|
-
export declare class RateLimiter {
|
|
14
|
-
private readonly requestsPerSecond;
|
|
15
|
-
private readonly intervalMs;
|
|
16
|
-
private readonly burstCapacity;
|
|
17
|
-
private readonly debug;
|
|
18
|
-
private tokens;
|
|
19
|
-
private lastRefill;
|
|
20
|
-
private intervalHandle;
|
|
21
|
-
private readonly pendingRequests;
|
|
22
|
-
constructor(options: RateLimiterOptions);
|
|
23
|
-
/**
|
|
24
|
-
* 等待直到可以发送请求
|
|
25
|
-
*/
|
|
26
|
-
waitForPermission(): Promise<void>;
|
|
27
|
-
/**
|
|
28
|
-
* 补充令牌(令牌桶算法)
|
|
29
|
-
*/
|
|
30
|
-
private refillTokens;
|
|
31
|
-
/**
|
|
32
|
-
* 处理等待中的请求
|
|
33
|
-
*/
|
|
34
|
-
private processPendingRequests;
|
|
35
|
-
/**
|
|
36
|
-
* 获取当前状态
|
|
37
|
-
*/
|
|
38
|
-
getStatus(): {
|
|
39
|
-
availableTokens: number;
|
|
40
|
-
maxTokens: number;
|
|
41
|
-
requestsPerSecond: number;
|
|
42
|
-
pendingRequests: number;
|
|
43
|
-
};
|
|
44
|
-
/**
|
|
45
|
-
* 清理过期的等待请求(超过30秒)
|
|
46
|
-
*/
|
|
47
|
-
cleanup(): void;
|
|
48
|
-
dispose(): void;
|
|
49
|
-
}
|
|
1
|
+
export * from '../infrastructure/rate-limit/RateLimiter.js';
|
|
50
2
|
//# sourceMappingURL=RateLimiter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RateLimiter.d.ts","sourceRoot":"","sources":["../../src/utils/RateLimiter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"RateLimiter.d.ts","sourceRoot":"","sources":["../../src/utils/RateLimiter.ts"],"names":[],"mappings":"AAAA,cAAc,6CAA6C,CAAC"}
|
|
@@ -1,121 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
* 请求速率限制器
|
|
3
|
-
* 用于控制API请求频率,遵守各平台的使用限制
|
|
4
|
-
*/
|
|
5
|
-
import { logDebug } from './Logger.js';
|
|
6
|
-
export class RateLimiter {
|
|
7
|
-
requestsPerSecond;
|
|
8
|
-
intervalMs;
|
|
9
|
-
burstCapacity;
|
|
10
|
-
debug;
|
|
11
|
-
tokens;
|
|
12
|
-
lastRefill;
|
|
13
|
-
intervalHandle;
|
|
14
|
-
pendingRequests = [];
|
|
15
|
-
constructor(options) {
|
|
16
|
-
this.requestsPerSecond = options.requestsPerSecond;
|
|
17
|
-
this.intervalMs = 1000 / this.requestsPerSecond;
|
|
18
|
-
this.burstCapacity = options.burstCapacity || this.requestsPerSecond;
|
|
19
|
-
this.debug = options.debug || false;
|
|
20
|
-
this.tokens = this.burstCapacity;
|
|
21
|
-
this.lastRefill = Date.now();
|
|
22
|
-
// 定期处理等待中的请求
|
|
23
|
-
this.intervalHandle = setInterval(() => this.processPendingRequests(), Math.min(this.intervalMs, 100));
|
|
24
|
-
// Don't keep the process alive just because of the limiter interval.
|
|
25
|
-
this.intervalHandle.unref?.();
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* 等待直到可以发送请求
|
|
29
|
-
*/
|
|
30
|
-
async waitForPermission() {
|
|
31
|
-
this.refillTokens();
|
|
32
|
-
if (this.tokens > 0) {
|
|
33
|
-
this.tokens--;
|
|
34
|
-
if (this.debug) {
|
|
35
|
-
logDebug(`RateLimiter: Request allowed, ${this.tokens} tokens remaining`);
|
|
36
|
-
}
|
|
37
|
-
return Promise.resolve();
|
|
38
|
-
}
|
|
39
|
-
// 没有可用令牌,加入等待队列
|
|
40
|
-
return new Promise((resolve) => {
|
|
41
|
-
this.pendingRequests.push({
|
|
42
|
-
resolve,
|
|
43
|
-
timestamp: Date.now()
|
|
44
|
-
});
|
|
45
|
-
if (this.debug) {
|
|
46
|
-
logDebug(`RateLimiter: Request queued, ${this.pendingRequests.length} waiting`);
|
|
47
|
-
}
|
|
48
|
-
});
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* 补充令牌(令牌桶算法)
|
|
52
|
-
*/
|
|
53
|
-
refillTokens() {
|
|
54
|
-
const now = Date.now();
|
|
55
|
-
const timePassed = now - this.lastRefill;
|
|
56
|
-
if (timePassed >= this.intervalMs) {
|
|
57
|
-
const tokensToAdd = Math.floor(timePassed / this.intervalMs);
|
|
58
|
-
this.tokens = Math.min(this.burstCapacity, this.tokens + tokensToAdd);
|
|
59
|
-
this.lastRefill = now;
|
|
60
|
-
if (this.debug && tokensToAdd > 0) {
|
|
61
|
-
logDebug(`RateLimiter: Added ${tokensToAdd} tokens, total: ${this.tokens}`);
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
/**
|
|
66
|
-
* 处理等待中的请求
|
|
67
|
-
*/
|
|
68
|
-
processPendingRequests() {
|
|
69
|
-
this.refillTokens();
|
|
70
|
-
while (this.tokens > 0 && this.pendingRequests.length > 0) {
|
|
71
|
-
const request = this.pendingRequests.shift();
|
|
72
|
-
if (request) {
|
|
73
|
-
this.tokens--;
|
|
74
|
-
request.resolve();
|
|
75
|
-
if (this.debug) {
|
|
76
|
-
const waitTime = Date.now() - request.timestamp;
|
|
77
|
-
logDebug(`RateLimiter: Released waiting request (waited ${waitTime}ms), ${this.tokens} tokens remaining`);
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* 获取当前状态
|
|
84
|
-
*/
|
|
85
|
-
getStatus() {
|
|
86
|
-
this.refillTokens();
|
|
87
|
-
return {
|
|
88
|
-
availableTokens: this.tokens,
|
|
89
|
-
maxTokens: this.burstCapacity,
|
|
90
|
-
requestsPerSecond: this.requestsPerSecond,
|
|
91
|
-
pendingRequests: this.pendingRequests.length
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* 清理过期的等待请求(超过30秒)
|
|
96
|
-
*/
|
|
97
|
-
cleanup() {
|
|
98
|
-
const now = Date.now();
|
|
99
|
-
const timeoutMs = 30000; // 30秒超时
|
|
100
|
-
let removedCount = 0;
|
|
101
|
-
while (this.pendingRequests.length > 0) {
|
|
102
|
-
const first = this.pendingRequests[0];
|
|
103
|
-
if (now - first.timestamp > timeoutMs) {
|
|
104
|
-
this.pendingRequests.shift();
|
|
105
|
-
// 拒绝过期的请求
|
|
106
|
-
first.resolve(); // 或者可以reject,但这里选择允许继续
|
|
107
|
-
removedCount++;
|
|
108
|
-
}
|
|
109
|
-
else {
|
|
110
|
-
break;
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
if (this.debug && removedCount > 0) {
|
|
114
|
-
logDebug(`RateLimiter: Cleaned up ${removedCount} expired requests`);
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
dispose() {
|
|
118
|
-
clearInterval(this.intervalHandle);
|
|
119
|
-
}
|
|
120
|
-
}
|
|
1
|
+
export * from '../infrastructure/rate-limit/RateLimiter.js';
|
|
121
2
|
//# sourceMappingURL=RateLimiter.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RateLimiter.js","sourceRoot":"","sources":["../../src/utils/RateLimiter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"RateLimiter.js","sourceRoot":"","sources":["../../src/utils/RateLimiter.ts"],"names":[],"mappings":"AAAA,cAAc,6CAA6C,CAAC"}
|
|
@@ -1,26 +1,3 @@
|
|
|
1
|
-
export
|
|
2
|
-
|
|
3
|
-
ttlMs?: number;
|
|
4
|
-
}
|
|
5
|
-
export interface CacheStats {
|
|
6
|
-
size: number;
|
|
7
|
-
maxSize: number;
|
|
8
|
-
hits: number;
|
|
9
|
-
misses: number;
|
|
10
|
-
hitRate: number;
|
|
11
|
-
}
|
|
12
|
-
export declare class RequestCache<T = any> {
|
|
13
|
-
private cache;
|
|
14
|
-
private hits;
|
|
15
|
-
private misses;
|
|
16
|
-
constructor(options?: CacheOptions);
|
|
17
|
-
generateKey(platform: string, query: string, options?: Record<string, any>): string;
|
|
18
|
-
get(key: string): T | undefined;
|
|
19
|
-
set(key: string, value: T): void;
|
|
20
|
-
has(key: string): boolean;
|
|
21
|
-
delete(key: string): boolean;
|
|
22
|
-
clear(): void;
|
|
23
|
-
getStats(): CacheStats;
|
|
24
|
-
}
|
|
25
|
-
export default RequestCache;
|
|
1
|
+
export * from '../infrastructure/cache/RequestCache.js';
|
|
2
|
+
export { default } from '../infrastructure/cache/RequestCache.js';
|
|
26
3
|
//# sourceMappingURL=RequestCache.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RequestCache.d.ts","sourceRoot":"","sources":["../../src/utils/RequestCache.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"RequestCache.d.ts","sourceRoot":"","sources":["../../src/utils/RequestCache.ts"],"names":[],"mappings":"AAAA,cAAc,yCAAyC,CAAC;AACxD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC"}
|
|
@@ -1,66 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import { logDebug } from './Logger.js';
|
|
4
|
-
export class RequestCache {
|
|
5
|
-
cache;
|
|
6
|
-
hits = 0;
|
|
7
|
-
misses = 0;
|
|
8
|
-
constructor(options = {}) {
|
|
9
|
-
const maxSize = options.maxSize || 100;
|
|
10
|
-
const ttlMs = options.ttlMs || 3600000; // 1 hour default
|
|
11
|
-
this.cache = new LRUCache({
|
|
12
|
-
max: maxSize,
|
|
13
|
-
ttl: ttlMs,
|
|
14
|
-
updateAgeOnGet: true,
|
|
15
|
-
updateAgeOnHas: false
|
|
16
|
-
});
|
|
17
|
-
logDebug(`RequestCache initialized: maxSize=${maxSize}, ttl=${ttlMs}ms`);
|
|
18
|
-
}
|
|
19
|
-
generateKey(platform, query, options) {
|
|
20
|
-
const data = JSON.stringify({
|
|
21
|
-
platform,
|
|
22
|
-
query: query.toLowerCase().trim(),
|
|
23
|
-
options: options || {}
|
|
24
|
-
});
|
|
25
|
-
return createHash('sha256').update(data).digest('hex');
|
|
26
|
-
}
|
|
27
|
-
get(key) {
|
|
28
|
-
const value = this.cache.get(key);
|
|
29
|
-
if (value !== undefined) {
|
|
30
|
-
this.hits++;
|
|
31
|
-
logDebug(`Cache hit: ${key.substring(0, 8)}...`);
|
|
32
|
-
return value;
|
|
33
|
-
}
|
|
34
|
-
this.misses++;
|
|
35
|
-
logDebug(`Cache miss: ${key.substring(0, 8)}...`);
|
|
36
|
-
return undefined;
|
|
37
|
-
}
|
|
38
|
-
set(key, value) {
|
|
39
|
-
this.cache.set(key, value);
|
|
40
|
-
logDebug(`Cache set: ${key.substring(0, 8)}...`);
|
|
41
|
-
}
|
|
42
|
-
has(key) {
|
|
43
|
-
return this.cache.has(key);
|
|
44
|
-
}
|
|
45
|
-
delete(key) {
|
|
46
|
-
return this.cache.delete(key);
|
|
47
|
-
}
|
|
48
|
-
clear() {
|
|
49
|
-
this.cache.clear();
|
|
50
|
-
this.hits = 0;
|
|
51
|
-
this.misses = 0;
|
|
52
|
-
logDebug('Cache cleared');
|
|
53
|
-
}
|
|
54
|
-
getStats() {
|
|
55
|
-
const total = this.hits + this.misses;
|
|
56
|
-
return {
|
|
57
|
-
size: this.cache.size,
|
|
58
|
-
maxSize: this.cache.max,
|
|
59
|
-
hits: this.hits,
|
|
60
|
-
misses: this.misses,
|
|
61
|
-
hitRate: total > 0 ? this.hits / total : 0
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
export default RequestCache;
|
|
1
|
+
export * from '../infrastructure/cache/RequestCache.js';
|
|
2
|
+
export { default } from '../infrastructure/cache/RequestCache.js';
|
|
66
3
|
//# sourceMappingURL=RequestCache.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RequestCache.js","sourceRoot":"","sources":["../../src/utils/RequestCache.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"RequestCache.js","sourceRoot":"","sources":["../../src/utils/RequestCache.ts"],"names":[],"mappings":"AAAA,cAAc,yCAAyC,CAAC;AACxD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC"}
|