webcrawl 1.4.2 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +5 -1
- package/dist/core/engine/chrome.d.ts +14 -0
- package/dist/core/engine/chrome.js +140 -0
- package/dist/core/engine/crawler.d.ts +6 -5
- package/dist/core/engine/crawler.js +51 -18
- package/dist/core/engine/fetcher.js +24 -5
- package/dist/core/engine/index.d.ts +1 -0
- package/dist/core/engine/index.js +1 -0
- package/dist/core/extract/index.d.ts +1 -0
- package/dist/core/extract/index.js +1 -0
- package/dist/core/extract/structured.d.ts +14 -0
- package/dist/core/extract/structured.js +144 -0
- package/dist/core/local-engine.d.ts +11 -1
- package/dist/core/local-engine.js +191 -116
- package/dist/core/ranker/lexical.js +1 -0
- package/dist/core/ranker/types.d.ts +1 -1
- package/dist/core/search/catalog.d.ts +2 -0
- package/dist/core/search/pipeline.js +24 -9
- package/dist/core/search/providers.d.ts +10 -0
- package/dist/core/search/providers.js +97 -2
- package/dist/format.js +5 -1
- package/dist/setup.js +5 -1
- package/package.json +3 -3
package/dist/format.js
CHANGED
|
@@ -49,7 +49,11 @@ export function formatSearch(res) {
|
|
|
49
49
|
})
|
|
50
50
|
.join("\n")}`);
|
|
51
51
|
}
|
|
52
|
-
|
|
52
|
+
if (parts.length)
|
|
53
|
+
return parts.join("\n\n");
|
|
54
|
+
if (res.success === false && str(res.error))
|
|
55
|
+
return `Error: ${str(res.error)}`;
|
|
56
|
+
return "No results.";
|
|
53
57
|
}
|
|
54
58
|
export function formatScrape(res) {
|
|
55
59
|
if (res.success === false && str(res.error)) {
|
package/dist/setup.js
CHANGED
|
@@ -147,18 +147,22 @@ export function runDoctor(env = process.env, effectiveMode = "local", apiKeySour
|
|
|
147
147
|
});
|
|
148
148
|
// 3. Local Chrome / Stealth Browser availability
|
|
149
149
|
const chromePaths = [
|
|
150
|
+
env.WEBCRAWL_CHROME_PATH,
|
|
151
|
+
env.INTERACT_CHROME_PATH,
|
|
150
152
|
env.CHROME_PATH,
|
|
151
153
|
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
152
154
|
"/usr/bin/google-chrome",
|
|
155
|
+
"/usr/bin/google-chrome-stable",
|
|
153
156
|
"/usr/bin/chromium",
|
|
154
157
|
"/usr/bin/chromium-browser",
|
|
158
|
+
"/snap/bin/chromium",
|
|
155
159
|
].filter(Boolean);
|
|
156
160
|
const foundChrome = chromePaths.find((p) => existsSync(p));
|
|
157
161
|
checks.push({
|
|
158
162
|
name: "Local stealth browser",
|
|
159
163
|
status: foundChrome ? "ok" : "optional",
|
|
160
164
|
detail: foundChrome
|
|
161
|
-
? `Found Chrome at ${foundChrome}`
|
|
165
|
+
? `Found Chrome at ${foundChrome} (used to render JS pages)`
|
|
162
166
|
: "No local Chrome binary detected; static/TLS fetch works out of the box, or run `webcrawl login` for cloud stealth browsers",
|
|
163
167
|
});
|
|
164
168
|
// 4. Search providers
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webcrawl",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.3",
|
|
4
4
|
"description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -127,8 +127,8 @@
|
|
|
127
127
|
"@ai-sdk/gateway": "^4.0.123",
|
|
128
128
|
"@firecrawl/pdf-inspector": "^1.25.2",
|
|
129
129
|
"@vercel/sandbox": "^1.0.0",
|
|
130
|
-
"@webcrawl/client": "^1.4.
|
|
131
|
-
"@webcrawl/curl-impersonate": "^1.4.
|
|
130
|
+
"@webcrawl/client": "^1.4.3",
|
|
131
|
+
"@webcrawl/curl-impersonate": "^1.4.3",
|
|
132
132
|
"ai": "^7.0.150",
|
|
133
133
|
"impit": "^0.14.7",
|
|
134
134
|
"mammoth": "^1.13.0",
|