webctx 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +97 -0
- package/CONTRIBUTORS.md +126 -0
- package/LICENSE +202 -0
- package/Makefile +69 -0
- package/README.md +121 -0
- package/bin/webctx.js +28 -0
- package/cmd/webctx/main.go +11 -0
- package/docs/porting-status.md +173 -0
- package/go.mod +3 -0
- package/internal/app/app.go +139 -0
- package/internal/app/app_test.go +194 -0
- package/internal/app/github.go +2084 -0
- package/internal/app/github_acceptance_test.go +166 -0
- package/internal/app/github_actions.go +1008 -0
- package/internal/app/github_actions_test.go +484 -0
- package/internal/app/github_activity_deployments.go +503 -0
- package/internal/app/github_activity_deployments_test.go +321 -0
- package/internal/app/github_commits.go +1004 -0
- package/internal/app/github_commits_test.go +503 -0
- package/internal/app/github_discussions_gists.go +1156 -0
- package/internal/app/github_discussions_gists_test.go +425 -0
- package/internal/app/github_issues.go +1616 -0
- package/internal/app/github_issues_test.go +731 -0
- package/internal/app/github_packages_projects.go +429 -0
- package/internal/app/github_packages_projects_test.go +168 -0
- package/internal/app/github_phase9_overview_stress_test.go +178 -0
- package/internal/app/github_pull_views.go +1141 -0
- package/internal/app/github_pull_views_test.go +603 -0
- package/internal/app/github_pulls.go +1459 -0
- package/internal/app/github_pulls_test.go +715 -0
- package/internal/app/github_refs.go +621 -0
- package/internal/app/github_refs_test.go +260 -0
- package/internal/app/github_search_profiles.go +505 -0
- package/internal/app/github_search_profiles_test.go +352 -0
- package/internal/app/github_test.go +888 -0
- package/internal/app/scrape.go +264 -0
- package/internal/app/tools.go +670 -0
- package/internal/buildinfo/buildinfo.go +16 -0
- package/package.json +66 -0
- package/scripts/postinstall.js +137 -0
|
@@ -0,0 +1,670 @@
|
|
|
1
|
+
package app
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"encoding/json"
|
|
6
|
+
"errors"
|
|
7
|
+
"fmt"
|
|
8
|
+
"io"
|
|
9
|
+
"net/http"
|
|
10
|
+
"net/url"
|
|
11
|
+
"os"
|
|
12
|
+
"sort"
|
|
13
|
+
"strings"
|
|
14
|
+
"sync"
|
|
15
|
+
"time"
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
type SearchParams struct {
|
|
19
|
+
Query string
|
|
20
|
+
ExcludeDomains []string
|
|
21
|
+
IncludeKeyword string
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
type SearchDoc struct {
|
|
25
|
+
URL string
|
|
26
|
+
Title string
|
|
27
|
+
Overview string
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
type SearchResult struct {
|
|
31
|
+
Docs []SearchDoc
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type providerDocs struct {
|
|
35
|
+
Docs []SearchDoc
|
|
36
|
+
Provider string
|
|
37
|
+
Err error
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
type scoredURL struct {
|
|
41
|
+
Result SearchDoc
|
|
42
|
+
InsertionOrder int
|
|
43
|
+
TotalScore float64
|
|
44
|
+
DuplicateCount int
|
|
45
|
+
BestPosition int
|
|
46
|
+
FinalScore float64
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
func Search(params SearchParams) (string, error) {
|
|
50
|
+
defaultExcludedDomains := []string{"youtube.com", "vimeo.com", "dailymotion.com", "twitch.tv", "tiktok.com", "instagram.com", "facebook.com"}
|
|
51
|
+
allExcludedDomains := append(append([]string{}, defaultExcludedDomains...), params.ExcludeDomains...)
|
|
52
|
+
truncatedKeyword := truncateWords(params.IncludeKeyword, 5)
|
|
53
|
+
|
|
54
|
+
type namedSearch struct {
|
|
55
|
+
name string
|
|
56
|
+
fn func(context.Context) (SearchResult, error)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
searches := []namedSearch{}
|
|
60
|
+
if strings.TrimSpace(truncatedKeyword) != "" {
|
|
61
|
+
searches = append(searches, namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) {
|
|
62
|
+
return searchWithExa(ctx, params.Query, nil, truncatedKeyword)
|
|
63
|
+
}})
|
|
64
|
+
} else {
|
|
65
|
+
searches = append(searches,
|
|
66
|
+
namedSearch{name: "Brave", fn: func(ctx context.Context) (SearchResult, error) { return searchWithBrave(ctx, params.Query) }},
|
|
67
|
+
namedSearch{name: "Tavily", fn: func(ctx context.Context) (SearchResult, error) { return searchWithTavily(ctx, params.Query, nil) }},
|
|
68
|
+
namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) { return searchWithExa(ctx, params.Query, nil, "") }},
|
|
69
|
+
)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
results := make([]providerDocs, len(searches))
|
|
73
|
+
var wg sync.WaitGroup
|
|
74
|
+
for i, s := range searches {
|
|
75
|
+
wg.Add(1)
|
|
76
|
+
go func(i int, s namedSearch) {
|
|
77
|
+
defer wg.Done()
|
|
78
|
+
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
|
|
79
|
+
defer cancel()
|
|
80
|
+
res, err := s.fn(ctx)
|
|
81
|
+
if err != nil {
|
|
82
|
+
results[i] = providerDocs{Provider: s.name, Docs: []SearchDoc{}, Err: err}
|
|
83
|
+
return
|
|
84
|
+
}
|
|
85
|
+
results[i] = providerDocs{Provider: s.name, Docs: res.Docs}
|
|
86
|
+
}(i, s)
|
|
87
|
+
}
|
|
88
|
+
wg.Wait()
|
|
89
|
+
|
|
90
|
+
total := 0
|
|
91
|
+
for _, r := range results {
|
|
92
|
+
total += len(r.Docs)
|
|
93
|
+
}
|
|
94
|
+
if total == 0 {
|
|
95
|
+
missingKeys := uniqueMissingCredentialKeys(results)
|
|
96
|
+
if len(missingKeys) > 0 {
|
|
97
|
+
return "", missingCredentialsError("Error searching the web", missingKeys, "")
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
failures := make([]string, 0, len(results))
|
|
101
|
+
for _, r := range results {
|
|
102
|
+
if r.Err == nil {
|
|
103
|
+
continue
|
|
104
|
+
}
|
|
105
|
+
failures = append(failures, fmt.Sprintf("%s: %v", r.Provider, r.Err))
|
|
106
|
+
}
|
|
107
|
+
if len(failures) > 0 {
|
|
108
|
+
return "", fmt.Errorf("Error searching the web: all search providers failed (%s)", strings.Join(failures, "; "))
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return "", errors.New("Error searching the web: all search providers failed to return results")
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
filtered := filterExcludedDomains(results, allExcludedDomains)
|
|
115
|
+
ranked, _ := scoreAndRankResults(filtered)
|
|
116
|
+
if len(ranked) > 35 {
|
|
117
|
+
ranked = ranked[:35]
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
parts := []string{fmt.Sprintf("Total Results: %d\n", len(ranked))}
|
|
121
|
+
for _, doc := range ranked {
|
|
122
|
+
title := decodeHTML(doc.Title)
|
|
123
|
+
overview := decodeHTML(doc.Overview)
|
|
124
|
+
parts = append(parts, fmt.Sprintf("- [%s](%s)", title, doc.URL))
|
|
125
|
+
if overview != "" {
|
|
126
|
+
parts = append(parts, fmt.Sprintf(" - %s", overview))
|
|
127
|
+
}
|
|
128
|
+
parts = append(parts, "")
|
|
129
|
+
}
|
|
130
|
+
return strings.Join(parts, "\n"), nil
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
func ReadLink(rawURL string) (string, error) {
|
|
134
|
+
native := readGitHubNative(rawURL)
|
|
135
|
+
switch native.Outcome {
|
|
136
|
+
case GitHubNativeSuccess:
|
|
137
|
+
return native.Markdown, nil
|
|
138
|
+
case GitHubNativeFailure:
|
|
139
|
+
if shouldBestEffortCrawlGitHubPackage(rawURL, native.Err) {
|
|
140
|
+
title, markdown, crawlErr := scrapeLinkWithFirecrawl(rawURL)
|
|
141
|
+
if crawlErr == nil && usefulGitHubPackageCrawl(title, markdown) {
|
|
142
|
+
notice := "> **Best-effort GitHub Package page crawl:** GitHub's structured Package API was unavailable for the current credential, so webctx crawled the public GitHub page with Firecrawl instead. This output may be incomplete or contain GitHub UI noise."
|
|
143
|
+
return formatReadLink(title, rawURL, notice+"\n\n"+markdown), nil
|
|
144
|
+
}
|
|
145
|
+
if crawlErr == nil {
|
|
146
|
+
crawlErr = fmt.Errorf("Firecrawl did not return a recognizable public GitHub Package page")
|
|
147
|
+
}
|
|
148
|
+
return "", fmt.Errorf("%v Best-effort public GitHub Package crawl also failed: %v", native.Err, crawlErr)
|
|
149
|
+
}
|
|
150
|
+
return "", native.Err
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if ok, _ := checkMarkdownAvailable(rawURL); ok {
|
|
154
|
+
result, err := fetchMarkdownContent(rawURL)
|
|
155
|
+
if err == nil {
|
|
156
|
+
return formatReadLink(result.Title, result.URL, result.Markdown), nil
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
title, markdown, err := scrapeLinkWithFirecrawl(rawURL)
|
|
161
|
+
if err != nil {
|
|
162
|
+
return "", err
|
|
163
|
+
}
|
|
164
|
+
return formatReadLink(title, rawURL, markdown), nil
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
func shouldBestEffortCrawlGitHubPackage(rawURL string, nativeErr error) bool {
|
|
168
|
+
if strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY")) == "" {
|
|
169
|
+
return false
|
|
170
|
+
}
|
|
171
|
+
target := parseGitHubTarget(rawURL)
|
|
172
|
+
if target == nil || target.Kind != GitHubTargetPackage {
|
|
173
|
+
return false
|
|
174
|
+
}
|
|
175
|
+
ghErr, ok := nativeErr.(*GitHubError)
|
|
176
|
+
if !ok {
|
|
177
|
+
return false
|
|
178
|
+
}
|
|
179
|
+
return ghErr.Kind == GitHubErrorAuthentication || ghErr.Kind == GitHubErrorForbidden
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
func usefulGitHubPackageCrawl(title, markdown string) bool {
|
|
183
|
+
title = strings.ToLower(strings.TrimSpace(title))
|
|
184
|
+
markdown = strings.TrimSpace(markdown)
|
|
185
|
+
if markdown == "" || strings.Contains(title, "page not found") || strings.Contains(title, "404") {
|
|
186
|
+
return false
|
|
187
|
+
}
|
|
188
|
+
return strings.Contains(title, "package") && len(markdown) >= 100
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
func scrapeLinkWithFirecrawl(rawURL string) (string, string, error) {
|
|
192
|
+
apiKey := strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
|
|
193
|
+
if apiKey == "" {
|
|
194
|
+
return "", "", missingCredentialsError("Error reading web page", []string{"FIRECRAWL_API_KEY"}, "for non-.md URLs")
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
requestBody := map[string]any{
|
|
198
|
+
"url": rawURL,
|
|
199
|
+
"formats": []string{"markdown"},
|
|
200
|
+
"onlyMainContent": true,
|
|
201
|
+
"skipTlsVerification": true,
|
|
202
|
+
"blockAds": true,
|
|
203
|
+
"removeBase64Images": true,
|
|
204
|
+
"maxAge": 600000,
|
|
205
|
+
"excludeTags": []string{"script", "style", "meta", "noscript", "svg", "img", "nav", "footer", "header", "aside", ".advertisement", "#ad"},
|
|
206
|
+
}
|
|
207
|
+
if strings.HasSuffix(strings.ToLower(rawURL), ".pdf") {
|
|
208
|
+
requestBody["parsers"] = []string{"pdf"}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
data, err := getFirecrawlQueue().enqueue(rawURL, func() (map[string]any, error) {
|
|
212
|
+
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
213
|
+
defer cancel()
|
|
214
|
+
body, err := doJSONRequest(ctx, http.MethodPost, "https://api.firecrawl.dev/v2/scrape", map[string]string{
|
|
215
|
+
"Authorization": "Bearer " + apiKey,
|
|
216
|
+
"Content-Type": "application/json",
|
|
217
|
+
}, requestBody)
|
|
218
|
+
if err != nil {
|
|
219
|
+
return nil, err
|
|
220
|
+
}
|
|
221
|
+
var parsed map[string]any
|
|
222
|
+
if err := json.Unmarshal(body, &parsed); err != nil {
|
|
223
|
+
return nil, err
|
|
224
|
+
}
|
|
225
|
+
if success, _ := parsed["success"].(bool); !success {
|
|
226
|
+
return nil, fmt.Errorf("Scraping failed for %s: %v", rawURL, parsed["error"])
|
|
227
|
+
}
|
|
228
|
+
return parsed, nil
|
|
229
|
+
})
|
|
230
|
+
if err != nil {
|
|
231
|
+
return "", "", fmt.Errorf("Error reading web page: %v", err)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
dataMap, _ := data["data"].(map[string]any)
|
|
235
|
+
metadata, _ := dataMap["metadata"].(map[string]any)
|
|
236
|
+
title, _ := metadata["title"].(string)
|
|
237
|
+
markdown, _ := dataMap["markdown"].(string)
|
|
238
|
+
if markdown == "" {
|
|
239
|
+
markdown = "No content extracted"
|
|
240
|
+
}
|
|
241
|
+
return title, markdown, nil
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
func MapSite(rawURL string) (string, error) {
|
|
245
|
+
apiKey := strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
|
|
246
|
+
if apiKey == "" {
|
|
247
|
+
return "", missingCredentialsError("Error mapping website", []string{"FIRECRAWL_API_KEY"}, "")
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
requestBody := map[string]any{
|
|
251
|
+
"url": rawURL,
|
|
252
|
+
"sitemap": "include",
|
|
253
|
+
"includeSubdomains": true,
|
|
254
|
+
"ignoreQueryParameters": true,
|
|
255
|
+
"limit": 5000,
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
259
|
+
defer cancel()
|
|
260
|
+
body, err := doJSONRequest(ctx, http.MethodPost, "https://api.firecrawl.dev/v2/map", map[string]string{
|
|
261
|
+
"Authorization": "Bearer " + apiKey,
|
|
262
|
+
"Content-Type": "application/json",
|
|
263
|
+
}, requestBody)
|
|
264
|
+
if err != nil {
|
|
265
|
+
return "", fmt.Errorf("Error mapping website: %v", err)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
var parsed map[string]any
|
|
269
|
+
if err := json.Unmarshal(body, &parsed); err != nil {
|
|
270
|
+
return "", fmt.Errorf("Error mapping website: %v", err)
|
|
271
|
+
}
|
|
272
|
+
if success, _ := parsed["success"].(bool); !success {
|
|
273
|
+
return "", fmt.Errorf("Error mapping website: Mapping failed for %s: %v", rawURL, parsed["error"])
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
linksAny, _ := parsed["links"].([]any)
|
|
277
|
+
parts := []string{fmt.Sprintf("total urls found: %d", len(linksAny)), ""}
|
|
278
|
+
for _, item := range linksAny {
|
|
279
|
+
switch link := item.(type) {
|
|
280
|
+
case string:
|
|
281
|
+
parts = append(parts, "- "+link, "")
|
|
282
|
+
case map[string]any:
|
|
283
|
+
parts = append(parts, "- "+stringValue(link["url"]))
|
|
284
|
+
if v := stringValue(link["title"]); v != "" {
|
|
285
|
+
parts = append(parts, "- "+v)
|
|
286
|
+
}
|
|
287
|
+
if v := stringValue(link["description"]); v != "" {
|
|
288
|
+
parts = append(parts, "- "+v)
|
|
289
|
+
}
|
|
290
|
+
parts = append(parts, "")
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return strings.Join(parts, "\n"), nil
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
func formatReadLink(title, rawURL, markdown string) string {
|
|
297
|
+
parts := []string{}
|
|
298
|
+
if strings.TrimSpace(title) != "" {
|
|
299
|
+
parts = append(parts, "# "+title, "")
|
|
300
|
+
}
|
|
301
|
+
parts = append(parts, "**URL:** "+rawURL, "", markdown)
|
|
302
|
+
return strings.Join(parts, "\n")
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
func uniqueMissingCredentialKeys(results []providerDocs) []string {
|
|
306
|
+
keys := map[string]struct{}{}
|
|
307
|
+
for _, r := range results {
|
|
308
|
+
if r.Err == nil {
|
|
309
|
+
continue
|
|
310
|
+
}
|
|
311
|
+
msg := r.Err.Error()
|
|
312
|
+
if !strings.HasPrefix(msg, "missing ") {
|
|
313
|
+
continue
|
|
314
|
+
}
|
|
315
|
+
key := strings.TrimSpace(strings.TrimPrefix(msg, "missing "))
|
|
316
|
+
if key != "" {
|
|
317
|
+
keys[key] = struct{}{}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
out := make([]string, 0, len(keys))
|
|
322
|
+
for key := range keys {
|
|
323
|
+
out = append(out, key)
|
|
324
|
+
}
|
|
325
|
+
sort.Strings(out)
|
|
326
|
+
return out
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
func missingCredentialsError(prefix string, keys []string, detail string) error {
|
|
330
|
+
sort.Strings(keys)
|
|
331
|
+
keysText := strings.Join(keys, ", ")
|
|
332
|
+
suffix := ""
|
|
333
|
+
if strings.TrimSpace(detail) != "" {
|
|
334
|
+
suffix = " " + strings.TrimSpace(detail)
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
return fmt.Errorf(
|
|
338
|
+
"%s: missing %s%s. Set them as environment variables, place them in a .env.local next to the binary, or store them in macOS Keychain with service %q and account names matching the env vars",
|
|
339
|
+
prefix,
|
|
340
|
+
keysText,
|
|
341
|
+
suffix,
|
|
342
|
+
keychainServiceName,
|
|
343
|
+
)
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
func searchWithBrave(ctx context.Context, query string) (SearchResult, error) {
|
|
347
|
+
apiKey := strings.TrimSpace(os.Getenv("BRAVE_API_KEY"))
|
|
348
|
+
if apiKey == "" {
|
|
349
|
+
return SearchResult{}, errors.New("missing BRAVE_API_KEY")
|
|
350
|
+
}
|
|
351
|
+
params := url.Values{}
|
|
352
|
+
params.Set("q", query)
|
|
353
|
+
params.Set("text_decorations", "false")
|
|
354
|
+
params.Set("result_filter", "web")
|
|
355
|
+
params.Set("limit", "20")
|
|
356
|
+
body, err := doRawRequest(ctx, http.MethodGet, "https://api.search.brave.com/res/v1/web/search?"+params.Encode(), map[string]string{
|
|
357
|
+
"Accept": "application/json",
|
|
358
|
+
"Accept-Encoding": "gzip",
|
|
359
|
+
"x-subscription-token": apiKey,
|
|
360
|
+
}, nil)
|
|
361
|
+
if err != nil {
|
|
362
|
+
return SearchResult{}, err
|
|
363
|
+
}
|
|
364
|
+
var parsed struct {
|
|
365
|
+
Web struct {
|
|
366
|
+
Results []struct {
|
|
367
|
+
URL string `json:"url"`
|
|
368
|
+
Title string `json:"title"`
|
|
369
|
+
Description string `json:"description"`
|
|
370
|
+
} `json:"results"`
|
|
371
|
+
} `json:"web"`
|
|
372
|
+
}
|
|
373
|
+
if err := json.Unmarshal(body, &parsed); err != nil {
|
|
374
|
+
return SearchResult{}, err
|
|
375
|
+
}
|
|
376
|
+
docs := make([]SearchDoc, 0, min(20, len(parsed.Web.Results)))
|
|
377
|
+
for i, r := range parsed.Web.Results {
|
|
378
|
+
if i >= 20 {
|
|
379
|
+
break
|
|
380
|
+
}
|
|
381
|
+
docs = append(docs, SearchDoc{URL: r.URL, Title: r.Title, Overview: r.Description})
|
|
382
|
+
}
|
|
383
|
+
return SearchResult{Docs: docs}, nil
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
func searchWithTavily(ctx context.Context, query string, excludeDomains []string) (SearchResult, error) {
|
|
387
|
+
apiKey := strings.TrimSpace(os.Getenv("TAVILY_API_KEY"))
|
|
388
|
+
if apiKey == "" {
|
|
389
|
+
return SearchResult{}, errors.New("missing TAVILY_API_KEY")
|
|
390
|
+
}
|
|
391
|
+
requestBody := map[string]any{
|
|
392
|
+
"api_key": apiKey,
|
|
393
|
+
"query": query,
|
|
394
|
+
"max_results": 20,
|
|
395
|
+
}
|
|
396
|
+
if len(excludeDomains) > 0 {
|
|
397
|
+
requestBody["exclude_domains"] = excludeDomains
|
|
398
|
+
}
|
|
399
|
+
body, err := doJSONRequest(ctx, http.MethodPost, "https://api.tavily.com/search", map[string]string{
|
|
400
|
+
"Content-Type": "application/json",
|
|
401
|
+
"Accept": "application/json",
|
|
402
|
+
}, requestBody)
|
|
403
|
+
if err != nil {
|
|
404
|
+
return SearchResult{}, err
|
|
405
|
+
}
|
|
406
|
+
var parsed struct {
|
|
407
|
+
Results []struct {
|
|
408
|
+
URL string `json:"url"`
|
|
409
|
+
Title string `json:"title"`
|
|
410
|
+
Content string `json:"content"`
|
|
411
|
+
} `json:"results"`
|
|
412
|
+
}
|
|
413
|
+
if err := json.Unmarshal(body, &parsed); err != nil {
|
|
414
|
+
return SearchResult{}, err
|
|
415
|
+
}
|
|
416
|
+
docs := make([]SearchDoc, 0, min(20, len(parsed.Results)))
|
|
417
|
+
for i, r := range parsed.Results {
|
|
418
|
+
if i >= 20 {
|
|
419
|
+
break
|
|
420
|
+
}
|
|
421
|
+
docs = append(docs, SearchDoc{URL: r.URL, Title: r.Title, Overview: r.Content})
|
|
422
|
+
}
|
|
423
|
+
return SearchResult{Docs: docs}, nil
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
func searchWithExa(ctx context.Context, query string, excludeDomains []string, includeKeyword string) (SearchResult, error) {
|
|
427
|
+
apiKey := strings.TrimSpace(os.Getenv("EXA_API_KEY"))
|
|
428
|
+
if apiKey == "" {
|
|
429
|
+
return SearchResult{}, errors.New("missing EXA_API_KEY")
|
|
430
|
+
}
|
|
431
|
+
requestBody := map[string]any{
|
|
432
|
+
"query": query,
|
|
433
|
+
"type": "auto",
|
|
434
|
+
"numResults": 25,
|
|
435
|
+
"contents": map[string]any{
|
|
436
|
+
"livecrawl": "preferred",
|
|
437
|
+
},
|
|
438
|
+
}
|
|
439
|
+
if len(excludeDomains) > 0 {
|
|
440
|
+
requestBody["excludeDomains"] = excludeDomains
|
|
441
|
+
}
|
|
442
|
+
if strings.TrimSpace(includeKeyword) != "" {
|
|
443
|
+
requestBody["includeText"] = []string{includeKeyword}
|
|
444
|
+
}
|
|
445
|
+
body, err := doJSONRequest(ctx, http.MethodPost, "https://api.exa.ai/search", map[string]string{
|
|
446
|
+
"Accept": "application/json",
|
|
447
|
+
"Content-Type": "application/json",
|
|
448
|
+
"x-api-key": apiKey,
|
|
449
|
+
}, requestBody)
|
|
450
|
+
if err != nil {
|
|
451
|
+
return SearchResult{}, err
|
|
452
|
+
}
|
|
453
|
+
var parsed struct {
|
|
454
|
+
Results []struct {
|
|
455
|
+
URL string `json:"url"`
|
|
456
|
+
Title string `json:"title"`
|
|
457
|
+
Text string `json:"text"`
|
|
458
|
+
Summary string `json:"summary"`
|
|
459
|
+
} `json:"results"`
|
|
460
|
+
}
|
|
461
|
+
if err := json.Unmarshal(body, &parsed); err != nil {
|
|
462
|
+
return SearchResult{}, err
|
|
463
|
+
}
|
|
464
|
+
docs := make([]SearchDoc, 0, min(25, len(parsed.Results)))
|
|
465
|
+
for i, r := range parsed.Results {
|
|
466
|
+
if i >= 25 {
|
|
467
|
+
break
|
|
468
|
+
}
|
|
469
|
+
overview := r.Text
|
|
470
|
+
if overview == "" {
|
|
471
|
+
overview = r.Summary
|
|
472
|
+
}
|
|
473
|
+
docs = append(docs, SearchDoc{URL: r.URL, Title: r.Title, Overview: overview})
|
|
474
|
+
}
|
|
475
|
+
return SearchResult{Docs: docs}, nil
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
func scoreAndRankResults(providerResults []providerDocs) ([]SearchDoc, int) {
|
|
479
|
+
urlScores := map[string]*scoredURL{}
|
|
480
|
+
totalResultsBeforeDedup := 0
|
|
481
|
+
|
|
482
|
+
for _, providerResult := range providerResults {
|
|
483
|
+
totalResultsBeforeDedup += len(providerResult.Docs)
|
|
484
|
+
for idx, doc := range providerResult.Docs {
|
|
485
|
+
normalized := normalizeURL(doc.URL)
|
|
486
|
+
position := idx + 1
|
|
487
|
+
weightedScore := float64(getPositionPoints(position)) * getProviderWeight(providerResult.Provider)
|
|
488
|
+
if existing, ok := urlScores[normalized]; ok {
|
|
489
|
+
existing.TotalScore += weightedScore
|
|
490
|
+
existing.DuplicateCount++
|
|
491
|
+
if position < existing.BestPosition {
|
|
492
|
+
existing.BestPosition = position
|
|
493
|
+
existing.Result = doc
|
|
494
|
+
}
|
|
495
|
+
continue
|
|
496
|
+
}
|
|
497
|
+
urlScores[normalized] = &scoredURL{
|
|
498
|
+
Result: doc,
|
|
499
|
+
InsertionOrder: len(urlScores),
|
|
500
|
+
TotalScore: weightedScore,
|
|
501
|
+
DuplicateCount: 1,
|
|
502
|
+
BestPosition: position,
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
scored := make([]*scoredURL, 0, len(urlScores))
|
|
508
|
+
for _, item := range urlScores {
|
|
509
|
+
duplicateBonus := 3.0
|
|
510
|
+
if item.BestPosition <= 5 {
|
|
511
|
+
duplicateBonus = 5.0
|
|
512
|
+
}
|
|
513
|
+
duplicatePenalty := 0.0
|
|
514
|
+
if item.DuplicateCount > 3 {
|
|
515
|
+
duplicatePenalty = -2.0
|
|
516
|
+
}
|
|
517
|
+
item.FinalScore = item.TotalScore + float64(item.DuplicateCount-1)*duplicateBonus + duplicatePenalty
|
|
518
|
+
scored = append(scored, item)
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
sort.SliceStable(scored, func(i, j int) bool {
|
|
522
|
+
if scored[i].FinalScore == scored[j].FinalScore {
|
|
523
|
+
return scored[i].InsertionOrder < scored[j].InsertionOrder
|
|
524
|
+
}
|
|
525
|
+
return scored[i].FinalScore > scored[j].FinalScore
|
|
526
|
+
})
|
|
527
|
+
|
|
528
|
+
results := make([]SearchDoc, 0, len(scored))
|
|
529
|
+
for _, item := range scored {
|
|
530
|
+
results = append(results, item.Result)
|
|
531
|
+
}
|
|
532
|
+
return results, totalResultsBeforeDedup - len(scored)
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
func getPositionPoints(position int) int {
|
|
536
|
+
scores := map[int]int{1: 30, 2: 27, 3: 24, 4: 21, 5: 19, 6: 16, 7: 13, 8: 11, 9: 9, 10: 7, 11: 5, 12: 4, 13: 3, 14: 2}
|
|
537
|
+
if score, ok := scores[position]; ok {
|
|
538
|
+
return score
|
|
539
|
+
}
|
|
540
|
+
return 1
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
func getProviderWeight(provider string) float64 {
|
|
544
|
+
switch provider {
|
|
545
|
+
case "Ref":
|
|
546
|
+
return 1.25
|
|
547
|
+
case "Exa", "Tavily", "Brave":
|
|
548
|
+
return 1.0
|
|
549
|
+
default:
|
|
550
|
+
return 1.0
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
func filterExcludedDomains(providerResults []providerDocs, excludeDomains []string) []providerDocs {
|
|
555
|
+
if len(excludeDomains) == 0 {
|
|
556
|
+
return providerResults
|
|
557
|
+
}
|
|
558
|
+
normalizedExclude := make(map[string]struct{}, len(excludeDomains))
|
|
559
|
+
for _, domain := range excludeDomains {
|
|
560
|
+
normalizedExclude[strings.TrimPrefix(strings.ToLower(strings.TrimSpace(domain)), "www.")] = struct{}{}
|
|
561
|
+
}
|
|
562
|
+
filtered := make([]providerDocs, 0, len(providerResults))
|
|
563
|
+
for _, providerResult := range providerResults {
|
|
564
|
+
docs := make([]SearchDoc, 0, len(providerResult.Docs))
|
|
565
|
+
for _, doc := range providerResult.Docs {
|
|
566
|
+
if _, blocked := normalizedExclude[extractDomain(doc.URL)]; !blocked {
|
|
567
|
+
docs = append(docs, doc)
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
filtered = append(filtered, providerDocs{Provider: providerResult.Provider, Docs: docs})
|
|
571
|
+
}
|
|
572
|
+
return filtered
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
func normalizeURL(raw string) string {
|
|
576
|
+
parsed, err := url.Parse(raw)
|
|
577
|
+
if err != nil || parsed.Host == "" {
|
|
578
|
+
return strings.Split(strings.TrimSuffix(strings.ToLower(raw), "/"), "?")[0]
|
|
579
|
+
}
|
|
580
|
+
base := strings.ToLower(parsed.Scheme + "://" + parsed.Host + strings.TrimSuffix(parsed.EscapedPath(), "/"))
|
|
581
|
+
tracking := map[string]struct{}{"utm_source": {}, "utm_medium": {}, "utm_campaign": {}, "utm_term": {}, "utm_content": {}, "ref": {}, "fbclid": {}, "gclid": {}}
|
|
582
|
+
vals := url.Values{}
|
|
583
|
+
for key, vs := range parsed.Query() {
|
|
584
|
+
if _, skip := tracking[strings.ToLower(key)]; skip {
|
|
585
|
+
continue
|
|
586
|
+
}
|
|
587
|
+
for _, v := range vs {
|
|
588
|
+
vals.Add(key, v)
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
if encoded := vals.Encode(); encoded != "" {
|
|
592
|
+
return base + "?" + encoded
|
|
593
|
+
}
|
|
594
|
+
return base
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
func extractDomain(raw string) string {
|
|
598
|
+
parsed, err := url.Parse(raw)
|
|
599
|
+
if err == nil && parsed.Hostname() != "" {
|
|
600
|
+
return strings.TrimPrefix(strings.ToLower(parsed.Hostname()), "www.")
|
|
601
|
+
}
|
|
602
|
+
cleaned := strings.TrimPrefix(strings.TrimPrefix(strings.ToLower(raw), "https://"), "http://")
|
|
603
|
+
cleaned = strings.TrimPrefix(cleaned, "www.")
|
|
604
|
+
return strings.Split(strings.Split(cleaned, "/")[0], "?")[0]
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
func decodeHTML(text string) string {
|
|
608
|
+
replacer := strings.NewReplacer("&", "&", "<", "<", ">", ">", """, `"`, "'", "'", "'", "'", "'", "'")
|
|
609
|
+
return replacer.Replace(text)
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
func truncateWords(s string, maxWords int) string {
|
|
613
|
+
fields := strings.Fields(strings.TrimSpace(s))
|
|
614
|
+
if len(fields) <= maxWords {
|
|
615
|
+
return strings.Join(fields, " ")
|
|
616
|
+
}
|
|
617
|
+
return strings.Join(fields[:maxWords], " ")
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
func doJSONRequest(ctx context.Context, method, rawURL string, headers map[string]string, payload any) ([]byte, error) {
|
|
621
|
+
bodyBytes, err := json.Marshal(payload)
|
|
622
|
+
if err != nil {
|
|
623
|
+
return nil, err
|
|
624
|
+
}
|
|
625
|
+
return doRawRequest(ctx, method, rawURL, headers, bodyBytes)
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
func doRawRequest(ctx context.Context, method, rawURL string, headers map[string]string, body []byte) ([]byte, error) {
|
|
629
|
+
var reader io.Reader
|
|
630
|
+
if body != nil {
|
|
631
|
+
reader = strings.NewReader(string(body))
|
|
632
|
+
}
|
|
633
|
+
req, err := http.NewRequestWithContext(ctx, method, rawURL, reader)
|
|
634
|
+
if err != nil {
|
|
635
|
+
return nil, err
|
|
636
|
+
}
|
|
637
|
+
for k, v := range headers {
|
|
638
|
+
req.Header.Set(k, v)
|
|
639
|
+
}
|
|
640
|
+
resp, err := http.DefaultClient.Do(req)
|
|
641
|
+
if err != nil {
|
|
642
|
+
return nil, err
|
|
643
|
+
}
|
|
644
|
+
defer resp.Body.Close()
|
|
645
|
+
respBody, err := io.ReadAll(resp.Body)
|
|
646
|
+
if err != nil {
|
|
647
|
+
return nil, err
|
|
648
|
+
}
|
|
649
|
+
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
650
|
+
if len(respBody) > 0 {
|
|
651
|
+
return nil, fmt.Errorf("API request failed: %s - %s", resp.Status, strings.TrimSpace(string(respBody)))
|
|
652
|
+
}
|
|
653
|
+
return nil, fmt.Errorf("API request failed: %s", resp.Status)
|
|
654
|
+
}
|
|
655
|
+
return respBody, nil
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
func stringValue(v any) string {
|
|
659
|
+
if s, ok := v.(string); ok {
|
|
660
|
+
return s
|
|
661
|
+
}
|
|
662
|
+
return ""
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
func min(a, b int) int {
|
|
666
|
+
if a < b {
|
|
667
|
+
return a
|
|
668
|
+
}
|
|
669
|
+
return b
|
|
670
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
package buildinfo
|
|
2
|
+
|
|
3
|
+
import "strings"
|
|
4
|
+
|
|
5
|
+
const defaultVersion = "dev"
|
|
6
|
+
|
|
7
|
+
// Version is overridden at build time via linker flags.
|
|
8
|
+
var Version = defaultVersion
|
|
9
|
+
|
|
10
|
+
func CurrentVersion() string {
|
|
11
|
+
trimmed := strings.TrimSpace(Version)
|
|
12
|
+
if trimmed == "" {
|
|
13
|
+
return defaultVersion
|
|
14
|
+
}
|
|
15
|
+
return trimmed
|
|
16
|
+
}
|