smart-web-mcp 0.43.8 → 0.44.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/README.md +6 -4
- package/dist/cli.js +3 -1
- package/dist/cli.js.map +1 -1
- package/dist/mcp-server.js +61 -11
- package/dist/mcp-server.js.map +1 -1
- package/dist/shared.d.ts +19 -2
- package/dist/shared.js.map +1 -1
- package/dist/smartfetch.d.ts +4 -1
- package/dist/smartfetch.js +16 -2
- package/dist/smartfetch.js.map +1 -1
- package/dist/smartsearch.d.ts +1 -0
- package/dist/smartsearch.js +120 -57
- package/dist/smartsearch.js.map +1 -1
- package/package.json +1 -1
- package/python/requirements-undetected-chromedriver.txt +1 -1
package/dist/smartsearch.js
CHANGED
|
@@ -109,6 +109,18 @@ export function projectSmartsearchOutput(response, preset = "compact") {
|
|
|
109
109
|
const fallbacks = budgetList(response.fallbacks_tried, policy.maxFallbacks);
|
|
110
110
|
recordListMetadata(metadata, "fallbacks_tried", fallbacks.metadata);
|
|
111
111
|
const projectedFallbacks = fallbacks.items.map((fallback, index) => projectText(fallback, policy.maxFallbackChars, `fallbacks_tried.${index}`));
|
|
112
|
+
const execution = response.execution ? (() => {
|
|
113
|
+
const attempts = budgetList(response.execution.attempts, policy.maxFallbacks);
|
|
114
|
+
recordListMetadata(metadata, "execution.attempts", attempts.metadata);
|
|
115
|
+
return {
|
|
116
|
+
...response.execution,
|
|
117
|
+
attempts: attempts.items.map((attempt, index) => ({
|
|
118
|
+
...attempt,
|
|
119
|
+
provider: projectText(attempt.provider, policy.maxEngineChars, `execution.attempts.${index}.provider`),
|
|
120
|
+
...(attempt.error === undefined ? {} : { error: projectText(attempt.error, policy.maxNoteChars, `execution.attempts.${index}.error`) }),
|
|
121
|
+
})),
|
|
122
|
+
};
|
|
123
|
+
})() : undefined;
|
|
112
124
|
const handoff = response.handoff ? (() => {
|
|
113
125
|
const items = budgetList(response.handoff.items, policy.maxHandoffItems);
|
|
114
126
|
recordListMetadata(metadata, "handoff.items", items.metadata);
|
|
@@ -157,6 +169,7 @@ export function projectSmartsearchOutput(response, preset = "compact") {
|
|
|
157
169
|
results: projectedResults,
|
|
158
170
|
fallbacks_tried: projectedFallbacks,
|
|
159
171
|
notes: projectedNotes,
|
|
172
|
+
...(execution ? { execution } : {}),
|
|
160
173
|
...(handoff ? { handoff } : {}),
|
|
161
174
|
...(assessment ? { assessment } : {}),
|
|
162
175
|
budget: {
|
|
@@ -166,6 +179,21 @@ export function projectSmartsearchOutput(response, preset = "compact") {
|
|
|
166
179
|
},
|
|
167
180
|
};
|
|
168
181
|
}
|
|
182
|
+
export function isSmartsearchExecutionFailure(response) {
|
|
183
|
+
return response.execution?.status === "failed";
|
|
184
|
+
}
|
|
185
|
+
function completedProviderSearch(results, note) {
|
|
186
|
+
return { ok: results.length > 0, status: "succeeded", results, note };
|
|
187
|
+
}
|
|
188
|
+
function failedProviderSearch(note) {
|
|
189
|
+
return { ok: false, status: "failed", results: [], note };
|
|
190
|
+
}
|
|
191
|
+
function smartsearchExecution(attempts) {
|
|
192
|
+
return {
|
|
193
|
+
status: attempts.some((attempt) => attempt.status === "succeeded") ? "succeeded" : "failed",
|
|
194
|
+
attempts,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
169
197
|
function localOnlyMode() {
|
|
170
198
|
return configEnabled("network.localOnly", false, { private: true });
|
|
171
199
|
}
|
|
@@ -429,7 +457,7 @@ async function searchVelog(query, numResults, timeoutMs) {
|
|
|
429
457
|
}),
|
|
430
458
|
});
|
|
431
459
|
if (!result.ok)
|
|
432
|
-
return
|
|
460
|
+
return failedProviderSearch(result.error || `Velog search error ${result.status}`);
|
|
433
461
|
try {
|
|
434
462
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
435
463
|
const raw = Array.isArray(data?.data?.searchPosts?.posts) ? data.data.searchPosts.posts : [];
|
|
@@ -439,11 +467,11 @@ async function searchVelog(query, numResults, timeoutMs) {
|
|
|
439
467
|
snippet: String(item?.short_description || "").trim(),
|
|
440
468
|
engine: "velog",
|
|
441
469
|
})).filter((item) => item.url)).slice(0, numResults);
|
|
442
|
-
return
|
|
470
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "Velog search returned no results");
|
|
443
471
|
}
|
|
444
472
|
catch (error) {
|
|
445
473
|
const message = error instanceof Error ? error.message : String(error);
|
|
446
|
-
return
|
|
474
|
+
return failedProviderSearch(`Velog search parse error: ${message}`);
|
|
447
475
|
}
|
|
448
476
|
}
|
|
449
477
|
async function searchStackExchange(query, numResults, timeoutMs) {
|
|
@@ -463,7 +491,7 @@ async function searchStackExchange(query, numResults, timeoutMs) {
|
|
|
463
491
|
},
|
|
464
492
|
});
|
|
465
493
|
if (!result.ok)
|
|
466
|
-
return
|
|
494
|
+
return failedProviderSearch(result.error || `StackExchange error ${result.status}`);
|
|
467
495
|
try {
|
|
468
496
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
469
497
|
const raw = Array.isArray(data?.items) ? data.items : [];
|
|
@@ -473,11 +501,11 @@ async function searchStackExchange(query, numResults, timeoutMs) {
|
|
|
473
501
|
snippet: String(item?.tags?.join(", ") || "").trim(),
|
|
474
502
|
engine: "stackexchange",
|
|
475
503
|
})).filter((item) => item.url)).slice(0, numResults);
|
|
476
|
-
return
|
|
504
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "StackExchange search returned no results");
|
|
477
505
|
}
|
|
478
506
|
catch (error) {
|
|
479
507
|
const message = error instanceof Error ? error.message : String(error);
|
|
480
|
-
return
|
|
508
|
+
return failedProviderSearch(`StackExchange parse error: ${message}`);
|
|
481
509
|
}
|
|
482
510
|
}
|
|
483
511
|
async function searchHackerNews(query, numResults, timeoutMs) {
|
|
@@ -489,7 +517,7 @@ async function searchHackerNews(query, numResults, timeoutMs) {
|
|
|
489
517
|
},
|
|
490
518
|
});
|
|
491
519
|
if (!result.ok)
|
|
492
|
-
return
|
|
520
|
+
return failedProviderSearch(result.error || `Hacker News error ${result.status}`);
|
|
493
521
|
try {
|
|
494
522
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
495
523
|
const raw = Array.isArray(data?.hits) ? data.hits : [];
|
|
@@ -514,11 +542,11 @@ async function searchHackerNews(query, numResults, timeoutMs) {
|
|
|
514
542
|
engine: "hackernews",
|
|
515
543
|
};
|
|
516
544
|
}).filter((item) => item.url)).slice(0, numResults);
|
|
517
|
-
return
|
|
545
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "Hacker News search returned no results");
|
|
518
546
|
}
|
|
519
547
|
catch (error) {
|
|
520
548
|
const message = error instanceof Error ? error.message : String(error);
|
|
521
|
-
return
|
|
549
|
+
return failedProviderSearch(`Hacker News parse error: ${message}`);
|
|
522
550
|
}
|
|
523
551
|
}
|
|
524
552
|
function redditSubredditScope(query) {
|
|
@@ -552,11 +580,12 @@ async function searchReddit(query, numResults, timeoutMs) {
|
|
|
552
580
|
const subreddit = redditSubredditScope(query);
|
|
553
581
|
const notes = [];
|
|
554
582
|
let collected = [];
|
|
583
|
+
let completed = false;
|
|
555
584
|
const collect = (results) => {
|
|
556
585
|
collected = uniqueResults([...collected, ...results]).slice(0, numResults);
|
|
557
586
|
return collected.length >= numResults;
|
|
558
587
|
};
|
|
559
|
-
const success = () => (
|
|
588
|
+
const success = () => completedProviderSearch(collected, "");
|
|
560
589
|
const keepScope = (results) => {
|
|
561
590
|
if (!subreddit)
|
|
562
591
|
return results;
|
|
@@ -581,6 +610,7 @@ async function searchReddit(query, numResults, timeoutMs) {
|
|
|
581
610
|
},
|
|
582
611
|
});
|
|
583
612
|
if (rss.ok) {
|
|
613
|
+
completed = true;
|
|
584
614
|
const feed = parseFeed(rss.text, rssUrl);
|
|
585
615
|
const results = keepScope(uniqueResults((feed?.entries || []).map((entry) => {
|
|
586
616
|
let compatibleUrl = "";
|
|
@@ -631,6 +661,7 @@ async function searchReddit(query, numResults, timeoutMs) {
|
|
|
631
661
|
else {
|
|
632
662
|
notes.push("Arctic Shift returned no matching Reddit posts");
|
|
633
663
|
}
|
|
664
|
+
completed = true;
|
|
634
665
|
}
|
|
635
666
|
catch (error) {
|
|
636
667
|
notes.push(`Arctic Shift parse error: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -659,6 +690,7 @@ async function searchReddit(query, numResults, timeoutMs) {
|
|
|
659
690
|
else {
|
|
660
691
|
notes.push("PullPush returned no matching historical Reddit posts");
|
|
661
692
|
}
|
|
693
|
+
completed = true;
|
|
662
694
|
}
|
|
663
695
|
catch (error) {
|
|
664
696
|
notes.push(`PullPush parse error: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -687,6 +719,7 @@ async function searchReddit(query, numResults, timeoutMs) {
|
|
|
687
719
|
else {
|
|
688
720
|
notes.push("Reddit JSON returned no matching posts");
|
|
689
721
|
}
|
|
722
|
+
completed = true;
|
|
690
723
|
}
|
|
691
724
|
catch (error) {
|
|
692
725
|
notes.push(`Reddit JSON parse error: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -697,7 +730,8 @@ async function searchReddit(query, numResults, timeoutMs) {
|
|
|
697
730
|
}
|
|
698
731
|
if (collected.length > 0)
|
|
699
732
|
return success();
|
|
700
|
-
|
|
733
|
+
const note = notes.filter(Boolean).join("; ") || "Reddit search returned no results";
|
|
734
|
+
return completed ? completedProviderSearch([], note) : failedProviderSearch(note);
|
|
701
735
|
}
|
|
702
736
|
async function searchGitHub(query, numResults, timeoutMs) {
|
|
703
737
|
const q = stripSiteOperators(query) || query;
|
|
@@ -708,7 +742,7 @@ async function searchGitHub(query, numResults, timeoutMs) {
|
|
|
708
742
|
},
|
|
709
743
|
});
|
|
710
744
|
if (!result.ok)
|
|
711
|
-
return
|
|
745
|
+
return failedProviderSearch(result.error || `GitHub error ${result.status}`);
|
|
712
746
|
try {
|
|
713
747
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
714
748
|
const raw = Array.isArray(data?.items) ? data.items : [];
|
|
@@ -722,17 +756,17 @@ async function searchGitHub(query, numResults, timeoutMs) {
|
|
|
722
756
|
].filter(Boolean).join(" · "),
|
|
723
757
|
engine: "github",
|
|
724
758
|
}))).slice(0, numResults);
|
|
725
|
-
return
|
|
759
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "GitHub search returned no results");
|
|
726
760
|
}
|
|
727
761
|
catch (error) {
|
|
728
762
|
const message = error instanceof Error ? error.message : String(error);
|
|
729
|
-
return
|
|
763
|
+
return failedProviderSearch(`GitHub parse error: ${message}`);
|
|
730
764
|
}
|
|
731
765
|
}
|
|
732
766
|
async function searchGitHubRepoDocs(query, numResults, timeoutMs) {
|
|
733
767
|
const repoScope = parseGitHubRepoScope(query);
|
|
734
768
|
if (!repoScope)
|
|
735
|
-
return
|
|
769
|
+
return failedProviderSearch("GitHub repository docs fallback skipped without owner/repo site scope");
|
|
736
770
|
const repoResult = await fetchText(`https://api.github.com/repos/${encodeURIComponent(repoScope.owner)}/${encodeURIComponent(repoScope.repo)}`, timeoutMs, {
|
|
737
771
|
headers: {
|
|
738
772
|
accept: "application/vnd.github+json",
|
|
@@ -740,7 +774,7 @@ async function searchGitHubRepoDocs(query, numResults, timeoutMs) {
|
|
|
740
774
|
},
|
|
741
775
|
});
|
|
742
776
|
if (!repoResult.ok)
|
|
743
|
-
return
|
|
777
|
+
return failedProviderSearch(repoResult.error || `GitHub repository metadata error ${repoResult.status}`);
|
|
744
778
|
let defaultBranch = "main";
|
|
745
779
|
try {
|
|
746
780
|
const repoData = repoResult.text ? JSON.parse(repoResult.text) : {};
|
|
@@ -756,7 +790,7 @@ async function searchGitHubRepoDocs(query, numResults, timeoutMs) {
|
|
|
756
790
|
},
|
|
757
791
|
});
|
|
758
792
|
if (!result.ok)
|
|
759
|
-
return
|
|
793
|
+
return failedProviderSearch(result.error || `GitHub repository docs error ${result.status}`);
|
|
760
794
|
try {
|
|
761
795
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
762
796
|
const raw = Array.isArray(data?.tree) ? data.tree : [];
|
|
@@ -782,11 +816,11 @@ async function searchGitHubRepoDocs(query, numResults, timeoutMs) {
|
|
|
782
816
|
snippet: `GitHub repository document · path:${item.path}`,
|
|
783
817
|
engine: "github_repo_docs",
|
|
784
818
|
}))).slice(0, numResults);
|
|
785
|
-
return
|
|
819
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "GitHub repository docs search returned no matching documents");
|
|
786
820
|
}
|
|
787
821
|
catch (error) {
|
|
788
822
|
const message = error instanceof Error ? error.message : String(error);
|
|
789
|
-
return
|
|
823
|
+
return failedProviderSearch(`GitHub repository docs parse error: ${message}`);
|
|
790
824
|
}
|
|
791
825
|
}
|
|
792
826
|
async function searchMdn(query, numResults, timeoutMs) {
|
|
@@ -798,7 +832,7 @@ async function searchMdn(query, numResults, timeoutMs) {
|
|
|
798
832
|
},
|
|
799
833
|
});
|
|
800
834
|
if (!result.ok)
|
|
801
|
-
return
|
|
835
|
+
return failedProviderSearch(result.error || `MDN error ${result.status}`);
|
|
802
836
|
try {
|
|
803
837
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
804
838
|
const raw = Array.isArray(data?.documents) ? data.documents : [];
|
|
@@ -811,11 +845,11 @@ async function searchMdn(query, numResults, timeoutMs) {
|
|
|
811
845
|
engine: "mdn",
|
|
812
846
|
};
|
|
813
847
|
}).filter((item) => item.url && item.url !== "https://developer.mozilla.org/")).slice(0, numResults);
|
|
814
|
-
return
|
|
848
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "MDN search returned no results");
|
|
815
849
|
}
|
|
816
850
|
catch (error) {
|
|
817
851
|
const message = error instanceof Error ? error.message : String(error);
|
|
818
|
-
return
|
|
852
|
+
return failedProviderSearch(`MDN parse error: ${message}`);
|
|
819
853
|
}
|
|
820
854
|
}
|
|
821
855
|
async function searchCrates(query, numResults, timeoutMs) {
|
|
@@ -827,7 +861,7 @@ async function searchCrates(query, numResults, timeoutMs) {
|
|
|
827
861
|
},
|
|
828
862
|
});
|
|
829
863
|
if (!result.ok)
|
|
830
|
-
return
|
|
864
|
+
return failedProviderSearch(result.error || `crates.io error ${result.status}`);
|
|
831
865
|
try {
|
|
832
866
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
833
867
|
const raw = Array.isArray(data?.crates) ? data.crates : [];
|
|
@@ -845,11 +879,11 @@ async function searchCrates(query, numResults, timeoutMs) {
|
|
|
845
879
|
engine: "crates",
|
|
846
880
|
};
|
|
847
881
|
}).filter((item) => item.url)).slice(0, numResults);
|
|
848
|
-
return
|
|
882
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "crates.io search returned no results");
|
|
849
883
|
}
|
|
850
884
|
catch (error) {
|
|
851
885
|
const message = error instanceof Error ? error.message : String(error);
|
|
852
|
-
return
|
|
886
|
+
return failedProviderSearch(`crates.io parse error: ${message}`);
|
|
853
887
|
}
|
|
854
888
|
}
|
|
855
889
|
async function searchNpmPackage(packageName, timeoutMs) {
|
|
@@ -860,7 +894,7 @@ async function searchNpmPackage(packageName, timeoutMs) {
|
|
|
860
894
|
},
|
|
861
895
|
});
|
|
862
896
|
if (!result.ok)
|
|
863
|
-
return
|
|
897
|
+
return failedProviderSearch(result.error || `npm package error ${result.status}`);
|
|
864
898
|
try {
|
|
865
899
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
866
900
|
const record = data;
|
|
@@ -877,11 +911,11 @@ async function searchNpmPackage(packageName, timeoutMs) {
|
|
|
877
911
|
].filter(Boolean).join(" · "),
|
|
878
912
|
engine: "npm",
|
|
879
913
|
}]);
|
|
880
|
-
return
|
|
914
|
+
return completedProviderSearch(results, "");
|
|
881
915
|
}
|
|
882
916
|
catch (error) {
|
|
883
917
|
const message = error instanceof Error ? error.message : String(error);
|
|
884
|
-
return
|
|
918
|
+
return failedProviderSearch(`npm package parse error: ${message}`);
|
|
885
919
|
}
|
|
886
920
|
}
|
|
887
921
|
async function searchNpm(query, numResults, timeoutMs) {
|
|
@@ -893,7 +927,7 @@ async function searchNpm(query, numResults, timeoutMs) {
|
|
|
893
927
|
},
|
|
894
928
|
});
|
|
895
929
|
if (!result.ok)
|
|
896
|
-
return
|
|
930
|
+
return failedProviderSearch(result.error || `npm error ${result.status}`);
|
|
897
931
|
try {
|
|
898
932
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
899
933
|
const raw = Array.isArray(data?.objects) ? data.objects : [];
|
|
@@ -911,11 +945,11 @@ async function searchNpm(query, numResults, timeoutMs) {
|
|
|
911
945
|
engine: "npm",
|
|
912
946
|
};
|
|
913
947
|
}).filter((item) => item.url)).slice(0, numResults);
|
|
914
|
-
return
|
|
948
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "npm search returned no results");
|
|
915
949
|
}
|
|
916
950
|
catch (error) {
|
|
917
951
|
const message = error instanceof Error ? error.message : String(error);
|
|
918
|
-
return
|
|
952
|
+
return failedProviderSearch(`npm parse error: ${message}`);
|
|
919
953
|
}
|
|
920
954
|
}
|
|
921
955
|
function braveHtmlEnabled() {
|
|
@@ -935,7 +969,7 @@ async function searchWikipedia(query, numResults, timeoutMs) {
|
|
|
935
969
|
},
|
|
936
970
|
});
|
|
937
971
|
if (!result.ok)
|
|
938
|
-
return
|
|
972
|
+
return failedProviderSearch(result.error || `Wikipedia error ${result.status}`);
|
|
939
973
|
try {
|
|
940
974
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
941
975
|
const raw = Array.isArray(data?.query?.search) ? data.query.search : [];
|
|
@@ -950,11 +984,11 @@ async function searchWikipedia(query, numResults, timeoutMs) {
|
|
|
950
984
|
engine: "wikipedia",
|
|
951
985
|
};
|
|
952
986
|
}).filter((item) => item.url)).slice(0, numResults);
|
|
953
|
-
return
|
|
987
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "Wikipedia search returned no results");
|
|
954
988
|
}
|
|
955
989
|
catch (error) {
|
|
956
990
|
const message = error instanceof Error ? error.message : String(error);
|
|
957
|
-
return
|
|
991
|
+
return failedProviderSearch(`Wikipedia parse error: ${message}`);
|
|
958
992
|
}
|
|
959
993
|
}
|
|
960
994
|
async function searchDuckDuckGo(query, numResults, timeoutMs) {
|
|
@@ -964,9 +998,9 @@ async function searchDuckDuckGo(query, numResults, timeoutMs) {
|
|
|
964
998
|
},
|
|
965
999
|
});
|
|
966
1000
|
if (!result.ok)
|
|
967
|
-
return
|
|
1001
|
+
return failedProviderSearch(result.error || `DuckDuckGo error ${result.status}`);
|
|
968
1002
|
if (/anomaly\.js|sending automated queries|captcha/i.test(result.text)) {
|
|
969
|
-
return
|
|
1003
|
+
return failedProviderSearch("DuckDuckGo blocked the automated HTML fallback");
|
|
970
1004
|
}
|
|
971
1005
|
const blocks = result.text.split(/<div class="result[^"]*"/i).slice(1);
|
|
972
1006
|
const results = uniqueResults(blocks.map((block) => {
|
|
@@ -981,7 +1015,7 @@ async function searchDuckDuckGo(query, numResults, timeoutMs) {
|
|
|
981
1015
|
engine: "duckduckgo",
|
|
982
1016
|
};
|
|
983
1017
|
}).filter((item) => item.url)).slice(0, numResults);
|
|
984
|
-
return
|
|
1018
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "DuckDuckGo returned no results");
|
|
985
1019
|
}
|
|
986
1020
|
async function searchBraveHtml(query, numResults, timeoutMs) {
|
|
987
1021
|
const result = await fetchText(`https://search.brave.com/search?q=${encodeURIComponent(query)}&source=web`, timeoutMs, {
|
|
@@ -991,7 +1025,7 @@ async function searchBraveHtml(query, numResults, timeoutMs) {
|
|
|
991
1025
|
},
|
|
992
1026
|
});
|
|
993
1027
|
if (!result.ok)
|
|
994
|
-
return
|
|
1028
|
+
return failedProviderSearch(result.error || `Brave HTML error ${result.status}`);
|
|
995
1029
|
const results = [];
|
|
996
1030
|
const regex = /<a href="(https?:\/\/[^"#]+)"[^>]*>[\s\S]*?<div class="title[^>]*"[^>]*>([\s\S]*?)<\/div>[\s\S]*?<\/a>[\s\S]*?<div class="generic-snippet[\s\S]*?<div class="content[^>]*"[^>]*>([\s\S]*?)<\/div>/gi;
|
|
997
1031
|
for (const match of result.text.matchAll(regex)) {
|
|
@@ -1005,7 +1039,7 @@ async function searchBraveHtml(query, numResults, timeoutMs) {
|
|
|
1005
1039
|
break;
|
|
1006
1040
|
}
|
|
1007
1041
|
const unique = uniqueResults(results);
|
|
1008
|
-
return
|
|
1042
|
+
return completedProviderSearch(unique, unique.length > 0 ? "" : "Brave HTML returned no results");
|
|
1009
1043
|
}
|
|
1010
1044
|
async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
|
|
1011
1045
|
const normalizedBase = baseUrl.replace(/\/+$/, "");
|
|
@@ -1016,7 +1050,7 @@ async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
|
|
|
1016
1050
|
},
|
|
1017
1051
|
});
|
|
1018
1052
|
if (!result.ok)
|
|
1019
|
-
return
|
|
1053
|
+
return failedProviderSearch(result.error || `SearXNG error ${result.status}`);
|
|
1020
1054
|
try {
|
|
1021
1055
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
1022
1056
|
const raw = Array.isArray(data?.results) ? data.results : [];
|
|
@@ -1026,11 +1060,11 @@ async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
|
|
|
1026
1060
|
snippet: String(item?.content || item?.description || item?.text || "").trim(),
|
|
1027
1061
|
engine: "searxng",
|
|
1028
1062
|
}))).slice(0, numResults);
|
|
1029
|
-
return
|
|
1063
|
+
return completedProviderSearch(results, results.length > 0 ? "" : "SearXNG returned no results");
|
|
1030
1064
|
}
|
|
1031
1065
|
catch (error) {
|
|
1032
1066
|
const message = error instanceof Error ? error.message : String(error);
|
|
1033
|
-
return
|
|
1067
|
+
return failedProviderSearch(`SearXNG parse error: ${message}`);
|
|
1034
1068
|
}
|
|
1035
1069
|
}
|
|
1036
1070
|
export async function runSmartsearch(options, runtime = {}) {
|
|
@@ -1042,6 +1076,15 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1042
1076
|
const searxngBaseUrl = (options.searxngBaseUrl || configText("search.searxngBaseUrl") || "").trim();
|
|
1043
1077
|
const fallbacks = [];
|
|
1044
1078
|
const notes = [];
|
|
1079
|
+
const attempts = [];
|
|
1080
|
+
const recordAttempt = (provider, outcome) => {
|
|
1081
|
+
attempts.push({
|
|
1082
|
+
provider,
|
|
1083
|
+
status: outcome.status,
|
|
1084
|
+
result_count: outcome.results.length,
|
|
1085
|
+
...(outcome.status === "failed" && outcome.note ? { error: outcome.note } : {}),
|
|
1086
|
+
});
|
|
1087
|
+
};
|
|
1045
1088
|
try {
|
|
1046
1089
|
const siteOperator = extractSiteOperator(query);
|
|
1047
1090
|
if (siteOperator === "velog.io") {
|
|
@@ -1052,8 +1095,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1052
1095
|
else {
|
|
1053
1096
|
const velog = await searchVelog(query, numResults, timeoutMs);
|
|
1054
1097
|
fallbacks.push("velog");
|
|
1098
|
+
recordAttempt("velog", velog);
|
|
1055
1099
|
if (velog.ok)
|
|
1056
|
-
return maybeRunCrawlHandoff({ engine: "velog", query, results: velog.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1100
|
+
return maybeRunCrawlHandoff({ engine: "velog", query, results: velog.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1057
1101
|
if (velog.note)
|
|
1058
1102
|
notes.push(velog.note);
|
|
1059
1103
|
}
|
|
@@ -1066,8 +1110,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1066
1110
|
else {
|
|
1067
1111
|
const stackexchange = await searchStackExchange(query, numResults, timeoutMs);
|
|
1068
1112
|
fallbacks.push("stackexchange");
|
|
1113
|
+
recordAttempt("stackexchange", stackexchange);
|
|
1069
1114
|
if (stackexchange.ok)
|
|
1070
|
-
return maybeRunCrawlHandoff({ engine: "stackexchange", query, results: stackexchange.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1115
|
+
return maybeRunCrawlHandoff({ engine: "stackexchange", query, results: stackexchange.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1071
1116
|
if (stackexchange.note)
|
|
1072
1117
|
notes.push(stackexchange.note);
|
|
1073
1118
|
}
|
|
@@ -1080,8 +1125,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1080
1125
|
else {
|
|
1081
1126
|
const hackernews = await searchHackerNews(query, numResults, timeoutMs);
|
|
1082
1127
|
fallbacks.push("hackernews");
|
|
1128
|
+
recordAttempt("hackernews", hackernews);
|
|
1083
1129
|
if (hackernews.ok)
|
|
1084
|
-
return maybeRunCrawlHandoff({ engine: "hackernews", query, results: hackernews.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1130
|
+
return maybeRunCrawlHandoff({ engine: "hackernews", query, results: hackernews.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1085
1131
|
if (hackernews.note)
|
|
1086
1132
|
notes.push(hackernews.note);
|
|
1087
1133
|
}
|
|
@@ -1094,8 +1140,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1094
1140
|
else {
|
|
1095
1141
|
const reddit = await searchReddit(query, numResults, timeoutMs);
|
|
1096
1142
|
fallbacks.push("reddit");
|
|
1143
|
+
recordAttempt("reddit", reddit);
|
|
1097
1144
|
if (reddit.ok)
|
|
1098
|
-
return maybeRunCrawlHandoff({ engine: "reddit", query, results: reddit.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1145
|
+
return maybeRunCrawlHandoff({ engine: "reddit", query, results: reddit.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1099
1146
|
if (reddit.note)
|
|
1100
1147
|
notes.push(reddit.note);
|
|
1101
1148
|
}
|
|
@@ -1108,8 +1155,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1108
1155
|
else if (parseGitHubRepoScope(query)) {
|
|
1109
1156
|
const githubRepoDocs = await searchGitHubRepoDocs(query, numResults, timeoutMs);
|
|
1110
1157
|
fallbacks.push("github_repo_docs");
|
|
1158
|
+
recordAttempt("github_repo_docs", githubRepoDocs);
|
|
1111
1159
|
if (githubRepoDocs.ok)
|
|
1112
|
-
return maybeRunCrawlHandoff({ engine: "github_repo_docs", query, results: githubRepoDocs.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1160
|
+
return maybeRunCrawlHandoff({ engine: "github_repo_docs", query, results: githubRepoDocs.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1113
1161
|
if (githubRepoDocs.note)
|
|
1114
1162
|
notes.push(githubRepoDocs.note);
|
|
1115
1163
|
}
|
|
@@ -1119,8 +1167,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1119
1167
|
else {
|
|
1120
1168
|
const github = await searchGitHub(query, numResults, timeoutMs);
|
|
1121
1169
|
fallbacks.push("github");
|
|
1170
|
+
recordAttempt("github", github);
|
|
1122
1171
|
if (github.ok)
|
|
1123
|
-
return maybeRunCrawlHandoff({ engine: "github", query, results: github.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1172
|
+
return maybeRunCrawlHandoff({ engine: "github", query, results: github.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1124
1173
|
if (github.note)
|
|
1125
1174
|
notes.push(github.note);
|
|
1126
1175
|
}
|
|
@@ -1136,8 +1185,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1136
1185
|
? await searchNpmPackage(exactPackage, timeoutMs)
|
|
1137
1186
|
: await searchNpm(query, numResults, timeoutMs);
|
|
1138
1187
|
fallbacks.push("npm");
|
|
1188
|
+
recordAttempt("npm", npm);
|
|
1139
1189
|
if (npm.ok)
|
|
1140
|
-
return maybeRunCrawlHandoff({ engine: "npm", query, results: npm.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1190
|
+
return maybeRunCrawlHandoff({ engine: "npm", query, results: npm.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1141
1191
|
if (npm.note)
|
|
1142
1192
|
notes.push(npm.note);
|
|
1143
1193
|
}
|
|
@@ -1150,8 +1200,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1150
1200
|
else {
|
|
1151
1201
|
const mdn = await searchMdn(query, numResults, timeoutMs);
|
|
1152
1202
|
fallbacks.push("mdn");
|
|
1203
|
+
recordAttempt("mdn", mdn);
|
|
1153
1204
|
if (mdn.ok)
|
|
1154
|
-
return maybeRunCrawlHandoff({ engine: "mdn", query, results: mdn.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1205
|
+
return maybeRunCrawlHandoff({ engine: "mdn", query, results: mdn.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1155
1206
|
if (mdn.note)
|
|
1156
1207
|
notes.push(mdn.note);
|
|
1157
1208
|
}
|
|
@@ -1164,8 +1215,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1164
1215
|
else {
|
|
1165
1216
|
const crates = await searchCrates(query, numResults, timeoutMs);
|
|
1166
1217
|
fallbacks.push("crates");
|
|
1218
|
+
recordAttempt("crates", crates);
|
|
1167
1219
|
if (crates.ok)
|
|
1168
|
-
return maybeRunCrawlHandoff({ engine: "crates", query, results: crates.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1220
|
+
return maybeRunCrawlHandoff({ engine: "crates", query, results: crates.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1169
1221
|
if (crates.note)
|
|
1170
1222
|
notes.push(crates.note);
|
|
1171
1223
|
}
|
|
@@ -1178,8 +1230,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1178
1230
|
else {
|
|
1179
1231
|
const wikipedia = await searchWikipedia(query, numResults, timeoutMs);
|
|
1180
1232
|
fallbacks.push("wikipedia");
|
|
1233
|
+
recordAttempt("wikipedia", wikipedia);
|
|
1181
1234
|
if (wikipedia.ok)
|
|
1182
|
-
return maybeRunCrawlHandoff({ engine: "wikipedia", query, results: wikipedia.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1235
|
+
return maybeRunCrawlHandoff({ engine: "wikipedia", query, results: wikipedia.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1183
1236
|
if (wikipedia.note)
|
|
1184
1237
|
notes.push(wikipedia.note);
|
|
1185
1238
|
}
|
|
@@ -1192,8 +1245,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1192
1245
|
else {
|
|
1193
1246
|
const searxng = await searchSearxng(searxngBaseUrl, query, numResults, timeoutMs);
|
|
1194
1247
|
fallbacks.push("searxng");
|
|
1248
|
+
recordAttempt("searxng", searxng);
|
|
1195
1249
|
if (searxng.ok)
|
|
1196
|
-
return maybeRunCrawlHandoff({ engine: "searxng", query, results: searxng.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1250
|
+
return maybeRunCrawlHandoff({ engine: "searxng", query, results: searxng.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1197
1251
|
if (searxng.note)
|
|
1198
1252
|
notes.push(searxng.note);
|
|
1199
1253
|
}
|
|
@@ -1204,8 +1258,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1204
1258
|
if (duckduckgoEnabled()) {
|
|
1205
1259
|
const ddg = await searchDuckDuckGo(query, numResults, timeoutMs);
|
|
1206
1260
|
fallbacks.push("duckduckgo");
|
|
1261
|
+
recordAttempt("duckduckgo", ddg);
|
|
1207
1262
|
if (ddg.ok)
|
|
1208
|
-
return maybeRunCrawlHandoff({ engine: "duckduckgo", query, results: ddg.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1263
|
+
return maybeRunCrawlHandoff({ engine: "duckduckgo", query, results: ddg.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1209
1264
|
if (ddg.note)
|
|
1210
1265
|
notes.push(ddg.note);
|
|
1211
1266
|
}
|
|
@@ -1215,19 +1270,27 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1215
1270
|
if (braveHtmlEnabled()) {
|
|
1216
1271
|
const braveHtml = await searchBraveHtml(query, numResults, timeoutMs);
|
|
1217
1272
|
fallbacks.push("brave_html");
|
|
1273
|
+
recordAttempt("brave_html", braveHtml);
|
|
1218
1274
|
if (braveHtml.ok)
|
|
1219
|
-
return maybeRunCrawlHandoff({ engine: "brave_html", query, results: braveHtml.results, fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1275
|
+
return maybeRunCrawlHandoff({ engine: "brave_html", query, results: braveHtml.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1220
1276
|
if (braveHtml.note)
|
|
1221
1277
|
notes.push(braveHtml.note);
|
|
1222
1278
|
}
|
|
1223
1279
|
else {
|
|
1224
1280
|
notes.push("Brave HTML disabled by configuration");
|
|
1225
1281
|
}
|
|
1226
|
-
return maybeRunCrawlHandoff({ engine: "none", query, results: [], fallbacks_tried: fallbacks, notes }, options, runtime);
|
|
1282
|
+
return maybeRunCrawlHandoff({ engine: "none", query, results: [], fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1227
1283
|
}
|
|
1228
1284
|
catch (error) {
|
|
1229
1285
|
const message = error instanceof Error ? error.message : String(error);
|
|
1230
|
-
return maybeRunCrawlHandoff({
|
|
1286
|
+
return maybeRunCrawlHandoff({
|
|
1287
|
+
engine: "error",
|
|
1288
|
+
query,
|
|
1289
|
+
results: [],
|
|
1290
|
+
fallbacks_tried: fallbacks,
|
|
1291
|
+
notes: [...notes, message],
|
|
1292
|
+
execution: { status: "failed", attempts: [...attempts, { provider: "smartsearch", status: "failed", result_count: 0, error: message }] },
|
|
1293
|
+
}, options, runtime);
|
|
1231
1294
|
}
|
|
1232
1295
|
}
|
|
1233
1296
|
export function renderSmartsearch(response) {
|