smart-web-mcp 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/korea/fine-dust.d.ts +7 -0
- package/dist/korea/fine-dust.js +49 -1
- package/dist/korea/fine-dust.js.map +1 -1
- package/dist/korea/law.d.ts +8 -0
- package/dist/korea/law.js +53 -3
- package/dist/korea/law.js.map +1 -1
- package/dist/korea/proxy-client.d.ts +1 -0
- package/dist/korea/proxy-client.js +103 -8
- package/dist/korea/proxy-client.js.map +1 -1
- package/dist/korea/public-restroom.d.ts +7 -4
- package/dist/korea/public-restroom.js +71 -12
- package/dist/korea/public-restroom.js.map +1 -1
- package/dist/korea/real-estate.d.ts +7 -0
- package/dist/korea/real-estate.js +72 -5
- package/dist/korea/real-estate.js.map +1 -1
- package/dist/korea/stock.js +49 -20
- package/dist/korea/stock.js.map +1 -1
- package/dist/korea/weather.js +5 -0
- package/dist/korea/weather.js.map +1 -1
- package/dist/mcp-server.js +1 -4
- package/dist/mcp-server.js.map +1 -1
- package/dist/smartcrawl.js +14 -0
- package/dist/smartcrawl.js.map +1 -1
- package/dist/smartfetch/providers/naver-blog.js +49 -7
- package/dist/smartfetch/providers/naver-blog.js.map +1 -1
- package/dist/smartfetch/providers/naver-cafe.js +26 -7
- package/dist/smartfetch/providers/naver-cafe.js.map +1 -1
- package/dist/smartfetch/providers/wikidocs.js +33 -11
- package/dist/smartfetch/providers/wikidocs.js.map +1 -1
- package/package.json +1 -1
|
@@ -8,6 +8,21 @@ function removeStyleDeclarationHelpers(html) {
|
|
|
8
8
|
function stripWikidocsContent(html) {
|
|
9
9
|
return stripTags(removeStyleDeclarationHelpers(removeNonContentBlocks(html))).trim();
|
|
10
10
|
}
|
|
11
|
+
function cleanText(value) {
|
|
12
|
+
return String(value || "").replace(/\s+/g, " ").trim();
|
|
13
|
+
}
|
|
14
|
+
function isWikidocsUnavailableShell(html, title, text) {
|
|
15
|
+
const normalizedTitle = cleanText(title);
|
|
16
|
+
const normalizedText = cleanText(text);
|
|
17
|
+
const hasErrorContainer = /class=["'][^"']*(?:error-container|error-message|not-found)[^"']*["']/i.test(html);
|
|
18
|
+
const titleLooksLikeShell = /^오류(?:\s|$|[-–—:])/i.test(normalizedTitle) || /^(?:error|not found|page not found)\b/i.test(normalizedTitle);
|
|
19
|
+
const koreanShellText = /페이지를 찾을 수 없습니다|삭제되었거나 비공개|존재하지 않는 페이지|요청하신 페이지를 찾을 수 없습니다|권한이 없습니다|접근 권한이 없습니다|비공개 (?:문서|페이지)|삭제된 (?:문서|페이지)/i.test(normalizedText);
|
|
20
|
+
const englishShellTitle = /(?:not found|page unavailable|page not found|private page|deleted page)/i.test(normalizedTitle);
|
|
21
|
+
return hasErrorContainer || englishShellTitle || (titleLooksLikeShell && koreanShellText);
|
|
22
|
+
}
|
|
23
|
+
function wikidocsUnavailableGuidance() {
|
|
24
|
+
return "Wikidocs page unavailable: the page rendered an error or unavailable shell instead of article content. Verify the URL, or retry with an authenticated/browser session if the page is private.";
|
|
25
|
+
}
|
|
11
26
|
/**
|
|
12
27
|
* Extract the main article body from wikidocs.net HTML.
|
|
13
28
|
*
|
|
@@ -20,8 +35,8 @@ function extractWikidocsBody(html) {
|
|
|
20
35
|
// Primary: the dedicated content div
|
|
21
36
|
const contentMatch = html.match(/<div[^>]+class=["'][^"']*page-content[^"']*["'][^>]*>([\s\S]*?)<\/div>\s*(?:<div[^>]+class=["'][^"']*(?:page-nav|footer|apprise)["']|$)/i);
|
|
22
37
|
if (contentMatch?.[1]) {
|
|
23
|
-
const text = stripWikidocsContent(contentMatch[1]);
|
|
24
|
-
if (text
|
|
38
|
+
const text = cleanText(stripWikidocsContent(contentMatch[1]));
|
|
39
|
+
if (text)
|
|
25
40
|
return text;
|
|
26
41
|
}
|
|
27
42
|
// Broader: extract from page-content with a greedy second-capture approach
|
|
@@ -46,8 +61,8 @@ function extractWikidocsBody(html) {
|
|
|
46
61
|
endIdx = match;
|
|
47
62
|
}
|
|
48
63
|
const content = afterDiv.substring(0, endIdx);
|
|
49
|
-
const text = stripWikidocsContent(content);
|
|
50
|
-
if (text
|
|
64
|
+
const text = cleanText(stripWikidocsContent(content));
|
|
65
|
+
if (text)
|
|
51
66
|
return text;
|
|
52
67
|
}
|
|
53
68
|
}
|
|
@@ -58,11 +73,11 @@ function extractWikidocsBody(html) {
|
|
|
58
73
|
const withoutSidebar = loadMatch[1]
|
|
59
74
|
.replace(/<div[^>]+class=["'][^"']*col-sm-3[^"']*["'][^>]*>[\s\S]*?<\/div>\s*<\/div>/gi, "")
|
|
60
75
|
.replace(/<div[^>]+class=["'][^"']*sidebar[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, "");
|
|
61
|
-
const text = stripWikidocsContent(withoutSidebar);
|
|
62
|
-
if (text
|
|
76
|
+
const text = cleanText(stripWikidocsContent(withoutSidebar));
|
|
77
|
+
if (text)
|
|
63
78
|
return text;
|
|
64
79
|
}
|
|
65
|
-
return
|
|
80
|
+
return "";
|
|
66
81
|
}
|
|
67
82
|
export const wikidocsProvider = {
|
|
68
83
|
id: "wikidocs",
|
|
@@ -73,14 +88,16 @@ export const wikidocsProvider = {
|
|
|
73
88
|
const title = extractMetaProperty(html, "og:title") || extractTitleFromHtml(html);
|
|
74
89
|
const description = extractMetaProperty(html, "og:description") || extractMetaDescription(html);
|
|
75
90
|
const body = extractWikidocsBody(html);
|
|
91
|
+
const unavailable = isWikidocsUnavailableShell(html, title, body);
|
|
92
|
+
const text = unavailable ? wikidocsUnavailableGuidance() : body;
|
|
76
93
|
return {
|
|
77
94
|
post: {
|
|
78
95
|
url: context.url,
|
|
79
96
|
canonical_url: base,
|
|
80
97
|
title,
|
|
81
98
|
description,
|
|
82
|
-
text:
|
|
83
|
-
status: body ? "ok" : "blocked_or_unavailable",
|
|
99
|
+
text: text.slice(0, 50000),
|
|
100
|
+
status: body && !unavailable ? "ok" : "blocked_or_unavailable",
|
|
84
101
|
kind: "reference_article",
|
|
85
102
|
},
|
|
86
103
|
thread: [],
|
|
@@ -90,8 +107,13 @@ export const wikidocsProvider = {
|
|
|
90
107
|
...extractAbsoluteAnchorHrefs(html, base),
|
|
91
108
|
...extractUrls(html),
|
|
92
109
|
]),
|
|
93
|
-
partial: !body,
|
|
94
|
-
errors:
|
|
110
|
+
partial: !body || unavailable,
|
|
111
|
+
errors: unavailable
|
|
112
|
+
? [
|
|
113
|
+
...context.errors,
|
|
114
|
+
{ category: "unavailable", code: "wikidocs_article_unavailable", message: wikidocsUnavailableGuidance() },
|
|
115
|
+
]
|
|
116
|
+
: context.errors,
|
|
95
117
|
method: context.active.method,
|
|
96
118
|
};
|
|
97
119
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wikidocs.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/wikidocs.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,0BAA0B,EAE1B,sBAAsB,EACtB,mBAAmB,EACnB,oBAAoB,EACpB,WAAW,EACX,SAAS,GACV,MAAM,iBAAiB,CAAA;AAGxB,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,sDAAsD,EAAE,GAAG,CAAC,CAAA;AAClF,CAAC;AAED,SAAS,6BAA6B,CAAC,IAAY;IACjD,OAAO,IAAI,CAAC,OAAO,CAAC,uGAAuG,EAAE,EAAE,CAAC,CAAA;AAClI,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,OAAO,SAAS,CAAC,6BAA6B,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AACtF,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,qCAAqC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,0IAA0I,CAAC,CAAA;IAC3K,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"wikidocs.js","sourceRoot":"","sources":["../../../src/smartfetch/providers/wikidocs.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,0BAA0B,EAE1B,sBAAsB,EACtB,mBAAmB,EACnB,oBAAoB,EACpB,WAAW,EACX,SAAS,GACV,MAAM,iBAAiB,CAAA;AAGxB,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,sDAAsD,EAAE,GAAG,CAAC,CAAA;AAClF,CAAC;AAED,SAAS,6BAA6B,CAAC,IAAY;IACjD,OAAO,IAAI,CAAC,OAAO,CAAC,uGAAuG,EAAE,EAAE,CAAC,CAAA;AAClI,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,OAAO,SAAS,CAAC,6BAA6B,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AACtF,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACxD,CAAC;AAED,SAAS,0BAA0B,CAAC,IAAY,EAAE,KAAa,EAAE,IAAY;IAC3E,MAAM,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;IACxC,MAAM,cAAc,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IACtC,MAAM,iBAAiB,GAAG,wEAAwE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7G,MAAM,mBAAmB,GAAG,qBAAqB,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,wCAAwC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IACzI,MAAM,eAAe,GAAG,+GAA+G,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAC5J,MAAM,iBAAiB,GAAG,0EAA0E,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IAE1H,OAAO,iBAAiB,IAAI,iBAAiB,IAAI,CAAC,mBAAmB,IAAI,eAAe,CAAC,CAAA;AAC3F,CAAC;AAED,SAAS,2BAA2B;IAClC,OAAO,+LAA+L,CAAA;AACxM,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,qCAAqC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,0IAA0I,CAAC,CAAA;IAC3K,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7D,IAAI,IAAI;YAAE,OAAO,IAAI,CAAA;IACvB,CAAC;IAED,2EAA2E;IAC3E,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IACrD,IAAI,gBAAgB,KAAK,CAAC,CAAC,EAAE,CAAC;QAC5B,kDAAkD;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAA;QAC3D,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,0EAA0E;YAC1E,6EAA6E;YAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;YACzC,uEAAuE;YACvE,MAAM,eAAe,GAAG;gBACtB,uEAAuE;gBACvE,UAAU;gBACV,yBAAyB;aAC1B,CAAA;YACD,IAAI,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;YAC5B,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;gBACtC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,GAAG,MAAM;oBAAE,MAAM,GAAG,KAAK,CAAA;YACpD,CAAC;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;YAC7C,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAA;YACrD,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAA;QACvB,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,0DAA0D,CAAC,CAAA;IACxF,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnB,oEAAoE;QACpE,MAAM,cAAc,GAAG,SAAS,CAAC,CAAC,CAAC;aAChC,OAAO,CAAC,8EAA8E,EAAE,EAAE,CAAC;aAC3F,OAAO,CAAC,mEAAmE,EAAE,EAAE,CAAC,CAAA;QACnF,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,cAAc,CAAC,CAAC,CAAA;QAC5D,IAAI,IAAI;YAAE,OAAO,IAAI,CAAA;IACvB,CAAC;IAED,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAuB;IAClD,EAAE,EAAE,UAAU;IACd,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,kBAAkB;IACxD,KAAK,CAAC,SAAS,CAAC,OAA0B;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAA;QACnC,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAA;QAE/C,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAA;QACjF,MAAM,WAAW,GAAG,mBAAmB,CAAC,IAAI,EAAE,gBAAgB,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC/F,MAAM,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAA;QACtC,MAAM,WAAW,GAAG,0BAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;QACjE,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;QAE/D,OAAO;YACL,IAAI,EAAE;gBACJ,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,aAAa,EAAE,IAAI;gBACnB,KAAK;gBACL,WAAW;gBACX,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;gBAC1B,MAAM,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,wBAAwB;gBAC9D,IAAI,EAAE,mBAAmB;aAC1B;YACD,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,EAAE;YACZ,cAAc,EAAE,UAAU,CAAC;gBACzB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK;gBACvB,GAAG,0BAA0B,CAAC,IAAI,EAAE,IAAI,CAAC;gBACzC,GAAG,WAAW,CAAC,IAAI,CAAC;aACrB,CAAC;YACF,OAAO,EAAE,CAAC,IAAI,IAAI,WAAW;YAC7B,MAAM,EAAE,WAAW;gBACjB,CAAC,CAAC;oBACE,GAAG,OAAO,CAAC,MAAM;oBACjB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,8BAA8B,EAAE,OAAO,EAAE,2BAA2B,EAAE,EAAE;iBAC1G;gBACH,CAAC,CAAC,OAAO,CAAC,MAAM;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;SAC9B,CAAA;IACH,CAAC;CACF,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "smart-web-mcp",
|
|
3
3
|
"mcpName": "io.github.jojo-labs/smart-web",
|
|
4
|
-
"version": "0.20.
|
|
4
|
+
"version": "0.20.1",
|
|
5
5
|
"packageManager": "npm@11.9.0",
|
|
6
6
|
"description": "Local MCP for web search, direct-URL fetch, compact normalized output, site crawling, and docs export.",
|
|
7
7
|
"homepage": "https://github.com/jojo-labs/smart-web",
|