pi-unsloth-webtools 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -11
- package/engines.ts +107 -45
- package/html-to-md.ts +22 -18
- package/index.ts +21 -11
- package/package.json +1 -1
- package/pdf.ts +92 -11
- package/web-access.ts +68 -50
- package/web-fetch.ts +382 -93
- package/web-search.ts +5 -4
package/web-search.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { checkUrlAccess, scopeSearchQuery, type WebsitePolicy } from "./web-access.ts";
|
|
2
|
+
import { collapseWhitespace } from "./html-to-md.ts";
|
|
2
3
|
import {
|
|
3
4
|
autoTextSearch,
|
|
4
5
|
EmptySweepError,
|
|
@@ -46,7 +47,7 @@ export async function webSearch(
|
|
|
46
47
|
query: string | undefined,
|
|
47
48
|
options: WebSearchOptions = {},
|
|
48
49
|
): Promise<string> {
|
|
49
|
-
const maxResults = options.maxResults ??
|
|
50
|
+
const maxResults = options.maxResults ?? MAX_RESULTS;
|
|
50
51
|
const timeoutMs = options.timeoutMs ?? SEARCH_TIMEOUT_MS;
|
|
51
52
|
const signal = options.signal;
|
|
52
53
|
const policy = options.websitePolicy ?? null;
|
|
@@ -92,9 +93,9 @@ export function searchFailureMessage(exc: unknown, timeoutMs = SEARCH_TIMEOUT_MS
|
|
|
92
93
|
|
|
93
94
|
export function formatSearchResults(results: SearchResult[]): string {
|
|
94
95
|
const parts = results.map((result) => {
|
|
95
|
-
const title = String(result.title ?? "")
|
|
96
|
-
const href = String(result.href ?? "")
|
|
97
|
-
const snippet = String(result.body ?? "")
|
|
96
|
+
const title = collapseWhitespace(String(result.title ?? ""));
|
|
97
|
+
const href = collapseWhitespace(String(result.href ?? ""));
|
|
98
|
+
const snippet = collapseWhitespace(String(result.body ?? ""));
|
|
98
99
|
return `Title: ${title}\nURL: ${href}\nSnippet: ${snippet}`;
|
|
99
100
|
});
|
|
100
101
|
const text = parts.join("\n\n---\n\n");
|