webctx 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/AGENTS.md +97 -0
  2. package/CONTRIBUTORS.md +126 -0
  3. package/LICENSE +202 -0
  4. package/Makefile +69 -0
  5. package/README.md +121 -0
  6. package/bin/webctx.js +28 -0
  7. package/cmd/webctx/main.go +11 -0
  8. package/docs/porting-status.md +173 -0
  9. package/go.mod +3 -0
  10. package/internal/app/app.go +139 -0
  11. package/internal/app/app_test.go +194 -0
  12. package/internal/app/github.go +2084 -0
  13. package/internal/app/github_acceptance_test.go +166 -0
  14. package/internal/app/github_actions.go +1008 -0
  15. package/internal/app/github_actions_test.go +484 -0
  16. package/internal/app/github_activity_deployments.go +503 -0
  17. package/internal/app/github_activity_deployments_test.go +321 -0
  18. package/internal/app/github_commits.go +1004 -0
  19. package/internal/app/github_commits_test.go +503 -0
  20. package/internal/app/github_discussions_gists.go +1156 -0
  21. package/internal/app/github_discussions_gists_test.go +425 -0
  22. package/internal/app/github_issues.go +1616 -0
  23. package/internal/app/github_issues_test.go +731 -0
  24. package/internal/app/github_packages_projects.go +429 -0
  25. package/internal/app/github_packages_projects_test.go +168 -0
  26. package/internal/app/github_phase9_overview_stress_test.go +178 -0
  27. package/internal/app/github_pull_views.go +1141 -0
  28. package/internal/app/github_pull_views_test.go +603 -0
  29. package/internal/app/github_pulls.go +1459 -0
  30. package/internal/app/github_pulls_test.go +715 -0
  31. package/internal/app/github_refs.go +621 -0
  32. package/internal/app/github_refs_test.go +260 -0
  33. package/internal/app/github_search_profiles.go +505 -0
  34. package/internal/app/github_search_profiles_test.go +352 -0
  35. package/internal/app/github_test.go +888 -0
  36. package/internal/app/scrape.go +264 -0
  37. package/internal/app/tools.go +670 -0
  38. package/internal/buildinfo/buildinfo.go +16 -0
  39. package/package.json +66 -0
  40. package/scripts/postinstall.js +137 -0
@@ -0,0 +1,670 @@
1
+ package app
2
+
3
+ import (
4
+ "context"
5
+ "encoding/json"
6
+ "errors"
7
+ "fmt"
8
+ "io"
9
+ "net/http"
10
+ "net/url"
11
+ "os"
12
+ "sort"
13
+ "strings"
14
+ "sync"
15
+ "time"
16
+ )
17
+
18
+ type SearchParams struct {
19
+ Query string
20
+ ExcludeDomains []string
21
+ IncludeKeyword string
22
+ }
23
+
24
+ type SearchDoc struct {
25
+ URL string
26
+ Title string
27
+ Overview string
28
+ }
29
+
30
+ type SearchResult struct {
31
+ Docs []SearchDoc
32
+ }
33
+
34
+ type providerDocs struct {
35
+ Docs []SearchDoc
36
+ Provider string
37
+ Err error
38
+ }
39
+
40
+ type scoredURL struct {
41
+ Result SearchDoc
42
+ InsertionOrder int
43
+ TotalScore float64
44
+ DuplicateCount int
45
+ BestPosition int
46
+ FinalScore float64
47
+ }
48
+
49
+ func Search(params SearchParams) (string, error) {
50
+ defaultExcludedDomains := []string{"youtube.com", "vimeo.com", "dailymotion.com", "twitch.tv", "tiktok.com", "instagram.com", "facebook.com"}
51
+ allExcludedDomains := append(append([]string{}, defaultExcludedDomains...), params.ExcludeDomains...)
52
+ truncatedKeyword := truncateWords(params.IncludeKeyword, 5)
53
+
54
+ type namedSearch struct {
55
+ name string
56
+ fn func(context.Context) (SearchResult, error)
57
+ }
58
+
59
+ searches := []namedSearch{}
60
+ if strings.TrimSpace(truncatedKeyword) != "" {
61
+ searches = append(searches, namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) {
62
+ return searchWithExa(ctx, params.Query, nil, truncatedKeyword)
63
+ }})
64
+ } else {
65
+ searches = append(searches,
66
+ namedSearch{name: "Brave", fn: func(ctx context.Context) (SearchResult, error) { return searchWithBrave(ctx, params.Query) }},
67
+ namedSearch{name: "Tavily", fn: func(ctx context.Context) (SearchResult, error) { return searchWithTavily(ctx, params.Query, nil) }},
68
+ namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) { return searchWithExa(ctx, params.Query, nil, "") }},
69
+ )
70
+ }
71
+
72
+ results := make([]providerDocs, len(searches))
73
+ var wg sync.WaitGroup
74
+ for i, s := range searches {
75
+ wg.Add(1)
76
+ go func(i int, s namedSearch) {
77
+ defer wg.Done()
78
+ ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
79
+ defer cancel()
80
+ res, err := s.fn(ctx)
81
+ if err != nil {
82
+ results[i] = providerDocs{Provider: s.name, Docs: []SearchDoc{}, Err: err}
83
+ return
84
+ }
85
+ results[i] = providerDocs{Provider: s.name, Docs: res.Docs}
86
+ }(i, s)
87
+ }
88
+ wg.Wait()
89
+
90
+ total := 0
91
+ for _, r := range results {
92
+ total += len(r.Docs)
93
+ }
94
+ if total == 0 {
95
+ missingKeys := uniqueMissingCredentialKeys(results)
96
+ if len(missingKeys) > 0 {
97
+ return "", missingCredentialsError("Error searching the web", missingKeys, "")
98
+ }
99
+
100
+ failures := make([]string, 0, len(results))
101
+ for _, r := range results {
102
+ if r.Err == nil {
103
+ continue
104
+ }
105
+ failures = append(failures, fmt.Sprintf("%s: %v", r.Provider, r.Err))
106
+ }
107
+ if len(failures) > 0 {
108
+ return "", fmt.Errorf("Error searching the web: all search providers failed (%s)", strings.Join(failures, "; "))
109
+ }
110
+
111
+ return "", errors.New("Error searching the web: all search providers failed to return results")
112
+ }
113
+
114
+ filtered := filterExcludedDomains(results, allExcludedDomains)
115
+ ranked, _ := scoreAndRankResults(filtered)
116
+ if len(ranked) > 35 {
117
+ ranked = ranked[:35]
118
+ }
119
+
120
+ parts := []string{fmt.Sprintf("Total Results: %d\n", len(ranked))}
121
+ for _, doc := range ranked {
122
+ title := decodeHTML(doc.Title)
123
+ overview := decodeHTML(doc.Overview)
124
+ parts = append(parts, fmt.Sprintf("- [%s](%s)", title, doc.URL))
125
+ if overview != "" {
126
+ parts = append(parts, fmt.Sprintf(" - %s", overview))
127
+ }
128
+ parts = append(parts, "")
129
+ }
130
+ return strings.Join(parts, "\n"), nil
131
+ }
132
+
133
+ func ReadLink(rawURL string) (string, error) {
134
+ native := readGitHubNative(rawURL)
135
+ switch native.Outcome {
136
+ case GitHubNativeSuccess:
137
+ return native.Markdown, nil
138
+ case GitHubNativeFailure:
139
+ if shouldBestEffortCrawlGitHubPackage(rawURL, native.Err) {
140
+ title, markdown, crawlErr := scrapeLinkWithFirecrawl(rawURL)
141
+ if crawlErr == nil && usefulGitHubPackageCrawl(title, markdown) {
142
+ notice := "> **Best-effort GitHub Package page crawl:** GitHub's structured Package API was unavailable for the current credential, so webctx crawled the public GitHub page with Firecrawl instead. This output may be incomplete or contain GitHub UI noise."
143
+ return formatReadLink(title, rawURL, notice+"\n\n"+markdown), nil
144
+ }
145
+ if crawlErr == nil {
146
+ crawlErr = fmt.Errorf("Firecrawl did not return a recognizable public GitHub Package page")
147
+ }
148
+ return "", fmt.Errorf("%v Best-effort public GitHub Package crawl also failed: %v", native.Err, crawlErr)
149
+ }
150
+ return "", native.Err
151
+ }
152
+
153
+ if ok, _ := checkMarkdownAvailable(rawURL); ok {
154
+ result, err := fetchMarkdownContent(rawURL)
155
+ if err == nil {
156
+ return formatReadLink(result.Title, result.URL, result.Markdown), nil
157
+ }
158
+ }
159
+
160
+ title, markdown, err := scrapeLinkWithFirecrawl(rawURL)
161
+ if err != nil {
162
+ return "", err
163
+ }
164
+ return formatReadLink(title, rawURL, markdown), nil
165
+ }
166
+
167
+ func shouldBestEffortCrawlGitHubPackage(rawURL string, nativeErr error) bool {
168
+ if strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY")) == "" {
169
+ return false
170
+ }
171
+ target := parseGitHubTarget(rawURL)
172
+ if target == nil || target.Kind != GitHubTargetPackage {
173
+ return false
174
+ }
175
+ ghErr, ok := nativeErr.(*GitHubError)
176
+ if !ok {
177
+ return false
178
+ }
179
+ return ghErr.Kind == GitHubErrorAuthentication || ghErr.Kind == GitHubErrorForbidden
180
+ }
181
+
182
+ func usefulGitHubPackageCrawl(title, markdown string) bool {
183
+ title = strings.ToLower(strings.TrimSpace(title))
184
+ markdown = strings.TrimSpace(markdown)
185
+ if markdown == "" || strings.Contains(title, "page not found") || strings.Contains(title, "404") {
186
+ return false
187
+ }
188
+ return strings.Contains(title, "package") && len(markdown) >= 100
189
+ }
190
+
191
+ func scrapeLinkWithFirecrawl(rawURL string) (string, string, error) {
192
+ apiKey := strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
193
+ if apiKey == "" {
194
+ return "", "", missingCredentialsError("Error reading web page", []string{"FIRECRAWL_API_KEY"}, "for non-.md URLs")
195
+ }
196
+
197
+ requestBody := map[string]any{
198
+ "url": rawURL,
199
+ "formats": []string{"markdown"},
200
+ "onlyMainContent": true,
201
+ "skipTlsVerification": true,
202
+ "blockAds": true,
203
+ "removeBase64Images": true,
204
+ "maxAge": 600000,
205
+ "excludeTags": []string{"script", "style", "meta", "noscript", "svg", "img", "nav", "footer", "header", "aside", ".advertisement", "#ad"},
206
+ }
207
+ if strings.HasSuffix(strings.ToLower(rawURL), ".pdf") {
208
+ requestBody["parsers"] = []string{"pdf"}
209
+ }
210
+
211
+ data, err := getFirecrawlQueue().enqueue(rawURL, func() (map[string]any, error) {
212
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
213
+ defer cancel()
214
+ body, err := doJSONRequest(ctx, http.MethodPost, "https://api.firecrawl.dev/v2/scrape", map[string]string{
215
+ "Authorization": "Bearer " + apiKey,
216
+ "Content-Type": "application/json",
217
+ }, requestBody)
218
+ if err != nil {
219
+ return nil, err
220
+ }
221
+ var parsed map[string]any
222
+ if err := json.Unmarshal(body, &parsed); err != nil {
223
+ return nil, err
224
+ }
225
+ if success, _ := parsed["success"].(bool); !success {
226
+ return nil, fmt.Errorf("Scraping failed for %s: %v", rawURL, parsed["error"])
227
+ }
228
+ return parsed, nil
229
+ })
230
+ if err != nil {
231
+ return "", "", fmt.Errorf("Error reading web page: %v", err)
232
+ }
233
+
234
+ dataMap, _ := data["data"].(map[string]any)
235
+ metadata, _ := dataMap["metadata"].(map[string]any)
236
+ title, _ := metadata["title"].(string)
237
+ markdown, _ := dataMap["markdown"].(string)
238
+ if markdown == "" {
239
+ markdown = "No content extracted"
240
+ }
241
+ return title, markdown, nil
242
+ }
243
+
244
+ func MapSite(rawURL string) (string, error) {
245
+ apiKey := strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
246
+ if apiKey == "" {
247
+ return "", missingCredentialsError("Error mapping website", []string{"FIRECRAWL_API_KEY"}, "")
248
+ }
249
+
250
+ requestBody := map[string]any{
251
+ "url": rawURL,
252
+ "sitemap": "include",
253
+ "includeSubdomains": true,
254
+ "ignoreQueryParameters": true,
255
+ "limit": 5000,
256
+ }
257
+
258
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
259
+ defer cancel()
260
+ body, err := doJSONRequest(ctx, http.MethodPost, "https://api.firecrawl.dev/v2/map", map[string]string{
261
+ "Authorization": "Bearer " + apiKey,
262
+ "Content-Type": "application/json",
263
+ }, requestBody)
264
+ if err != nil {
265
+ return "", fmt.Errorf("Error mapping website: %v", err)
266
+ }
267
+
268
+ var parsed map[string]any
269
+ if err := json.Unmarshal(body, &parsed); err != nil {
270
+ return "", fmt.Errorf("Error mapping website: %v", err)
271
+ }
272
+ if success, _ := parsed["success"].(bool); !success {
273
+ return "", fmt.Errorf("Error mapping website: Mapping failed for %s: %v", rawURL, parsed["error"])
274
+ }
275
+
276
+ linksAny, _ := parsed["links"].([]any)
277
+ parts := []string{fmt.Sprintf("total urls found: %d", len(linksAny)), ""}
278
+ for _, item := range linksAny {
279
+ switch link := item.(type) {
280
+ case string:
281
+ parts = append(parts, "- "+link, "")
282
+ case map[string]any:
283
+ parts = append(parts, "- "+stringValue(link["url"]))
284
+ if v := stringValue(link["title"]); v != "" {
285
+ parts = append(parts, "- "+v)
286
+ }
287
+ if v := stringValue(link["description"]); v != "" {
288
+ parts = append(parts, "- "+v)
289
+ }
290
+ parts = append(parts, "")
291
+ }
292
+ }
293
+ return strings.Join(parts, "\n"), nil
294
+ }
295
+
296
+ func formatReadLink(title, rawURL, markdown string) string {
297
+ parts := []string{}
298
+ if strings.TrimSpace(title) != "" {
299
+ parts = append(parts, "# "+title, "")
300
+ }
301
+ parts = append(parts, "**URL:** "+rawURL, "", markdown)
302
+ return strings.Join(parts, "\n")
303
+ }
304
+
305
+ func uniqueMissingCredentialKeys(results []providerDocs) []string {
306
+ keys := map[string]struct{}{}
307
+ for _, r := range results {
308
+ if r.Err == nil {
309
+ continue
310
+ }
311
+ msg := r.Err.Error()
312
+ if !strings.HasPrefix(msg, "missing ") {
313
+ continue
314
+ }
315
+ key := strings.TrimSpace(strings.TrimPrefix(msg, "missing "))
316
+ if key != "" {
317
+ keys[key] = struct{}{}
318
+ }
319
+ }
320
+
321
+ out := make([]string, 0, len(keys))
322
+ for key := range keys {
323
+ out = append(out, key)
324
+ }
325
+ sort.Strings(out)
326
+ return out
327
+ }
328
+
329
+ func missingCredentialsError(prefix string, keys []string, detail string) error {
330
+ sort.Strings(keys)
331
+ keysText := strings.Join(keys, ", ")
332
+ suffix := ""
333
+ if strings.TrimSpace(detail) != "" {
334
+ suffix = " " + strings.TrimSpace(detail)
335
+ }
336
+
337
+ return fmt.Errorf(
338
+ "%s: missing %s%s. Set them as environment variables, place them in a .env.local next to the binary, or store them in macOS Keychain with service %q and account names matching the env vars",
339
+ prefix,
340
+ keysText,
341
+ suffix,
342
+ keychainServiceName,
343
+ )
344
+ }
345
+
346
+ func searchWithBrave(ctx context.Context, query string) (SearchResult, error) {
347
+ apiKey := strings.TrimSpace(os.Getenv("BRAVE_API_KEY"))
348
+ if apiKey == "" {
349
+ return SearchResult{}, errors.New("missing BRAVE_API_KEY")
350
+ }
351
+ params := url.Values{}
352
+ params.Set("q", query)
353
+ params.Set("text_decorations", "false")
354
+ params.Set("result_filter", "web")
355
+ params.Set("limit", "20")
356
+ body, err := doRawRequest(ctx, http.MethodGet, "https://api.search.brave.com/res/v1/web/search?"+params.Encode(), map[string]string{
357
+ "Accept": "application/json",
358
+ "Accept-Encoding": "gzip",
359
+ "x-subscription-token": apiKey,
360
+ }, nil)
361
+ if err != nil {
362
+ return SearchResult{}, err
363
+ }
364
+ var parsed struct {
365
+ Web struct {
366
+ Results []struct {
367
+ URL string `json:"url"`
368
+ Title string `json:"title"`
369
+ Description string `json:"description"`
370
+ } `json:"results"`
371
+ } `json:"web"`
372
+ }
373
+ if err := json.Unmarshal(body, &parsed); err != nil {
374
+ return SearchResult{}, err
375
+ }
376
+ docs := make([]SearchDoc, 0, min(20, len(parsed.Web.Results)))
377
+ for i, r := range parsed.Web.Results {
378
+ if i >= 20 {
379
+ break
380
+ }
381
+ docs = append(docs, SearchDoc{URL: r.URL, Title: r.Title, Overview: r.Description})
382
+ }
383
+ return SearchResult{Docs: docs}, nil
384
+ }
385
+
386
+ func searchWithTavily(ctx context.Context, query string, excludeDomains []string) (SearchResult, error) {
387
+ apiKey := strings.TrimSpace(os.Getenv("TAVILY_API_KEY"))
388
+ if apiKey == "" {
389
+ return SearchResult{}, errors.New("missing TAVILY_API_KEY")
390
+ }
391
+ requestBody := map[string]any{
392
+ "api_key": apiKey,
393
+ "query": query,
394
+ "max_results": 20,
395
+ }
396
+ if len(excludeDomains) > 0 {
397
+ requestBody["exclude_domains"] = excludeDomains
398
+ }
399
+ body, err := doJSONRequest(ctx, http.MethodPost, "https://api.tavily.com/search", map[string]string{
400
+ "Content-Type": "application/json",
401
+ "Accept": "application/json",
402
+ }, requestBody)
403
+ if err != nil {
404
+ return SearchResult{}, err
405
+ }
406
+ var parsed struct {
407
+ Results []struct {
408
+ URL string `json:"url"`
409
+ Title string `json:"title"`
410
+ Content string `json:"content"`
411
+ } `json:"results"`
412
+ }
413
+ if err := json.Unmarshal(body, &parsed); err != nil {
414
+ return SearchResult{}, err
415
+ }
416
+ docs := make([]SearchDoc, 0, min(20, len(parsed.Results)))
417
+ for i, r := range parsed.Results {
418
+ if i >= 20 {
419
+ break
420
+ }
421
+ docs = append(docs, SearchDoc{URL: r.URL, Title: r.Title, Overview: r.Content})
422
+ }
423
+ return SearchResult{Docs: docs}, nil
424
+ }
425
+
426
+ func searchWithExa(ctx context.Context, query string, excludeDomains []string, includeKeyword string) (SearchResult, error) {
427
+ apiKey := strings.TrimSpace(os.Getenv("EXA_API_KEY"))
428
+ if apiKey == "" {
429
+ return SearchResult{}, errors.New("missing EXA_API_KEY")
430
+ }
431
+ requestBody := map[string]any{
432
+ "query": query,
433
+ "type": "auto",
434
+ "numResults": 25,
435
+ "contents": map[string]any{
436
+ "livecrawl": "preferred",
437
+ },
438
+ }
439
+ if len(excludeDomains) > 0 {
440
+ requestBody["excludeDomains"] = excludeDomains
441
+ }
442
+ if strings.TrimSpace(includeKeyword) != "" {
443
+ requestBody["includeText"] = []string{includeKeyword}
444
+ }
445
+ body, err := doJSONRequest(ctx, http.MethodPost, "https://api.exa.ai/search", map[string]string{
446
+ "Accept": "application/json",
447
+ "Content-Type": "application/json",
448
+ "x-api-key": apiKey,
449
+ }, requestBody)
450
+ if err != nil {
451
+ return SearchResult{}, err
452
+ }
453
+ var parsed struct {
454
+ Results []struct {
455
+ URL string `json:"url"`
456
+ Title string `json:"title"`
457
+ Text string `json:"text"`
458
+ Summary string `json:"summary"`
459
+ } `json:"results"`
460
+ }
461
+ if err := json.Unmarshal(body, &parsed); err != nil {
462
+ return SearchResult{}, err
463
+ }
464
+ docs := make([]SearchDoc, 0, min(25, len(parsed.Results)))
465
+ for i, r := range parsed.Results {
466
+ if i >= 25 {
467
+ break
468
+ }
469
+ overview := r.Text
470
+ if overview == "" {
471
+ overview = r.Summary
472
+ }
473
+ docs = append(docs, SearchDoc{URL: r.URL, Title: r.Title, Overview: overview})
474
+ }
475
+ return SearchResult{Docs: docs}, nil
476
+ }
477
+
478
+ func scoreAndRankResults(providerResults []providerDocs) ([]SearchDoc, int) {
479
+ urlScores := map[string]*scoredURL{}
480
+ totalResultsBeforeDedup := 0
481
+
482
+ for _, providerResult := range providerResults {
483
+ totalResultsBeforeDedup += len(providerResult.Docs)
484
+ for idx, doc := range providerResult.Docs {
485
+ normalized := normalizeURL(doc.URL)
486
+ position := idx + 1
487
+ weightedScore := float64(getPositionPoints(position)) * getProviderWeight(providerResult.Provider)
488
+ if existing, ok := urlScores[normalized]; ok {
489
+ existing.TotalScore += weightedScore
490
+ existing.DuplicateCount++
491
+ if position < existing.BestPosition {
492
+ existing.BestPosition = position
493
+ existing.Result = doc
494
+ }
495
+ continue
496
+ }
497
+ urlScores[normalized] = &scoredURL{
498
+ Result: doc,
499
+ InsertionOrder: len(urlScores),
500
+ TotalScore: weightedScore,
501
+ DuplicateCount: 1,
502
+ BestPosition: position,
503
+ }
504
+ }
505
+ }
506
+
507
+ scored := make([]*scoredURL, 0, len(urlScores))
508
+ for _, item := range urlScores {
509
+ duplicateBonus := 3.0
510
+ if item.BestPosition <= 5 {
511
+ duplicateBonus = 5.0
512
+ }
513
+ duplicatePenalty := 0.0
514
+ if item.DuplicateCount > 3 {
515
+ duplicatePenalty = -2.0
516
+ }
517
+ item.FinalScore = item.TotalScore + float64(item.DuplicateCount-1)*duplicateBonus + duplicatePenalty
518
+ scored = append(scored, item)
519
+ }
520
+
521
+ sort.SliceStable(scored, func(i, j int) bool {
522
+ if scored[i].FinalScore == scored[j].FinalScore {
523
+ return scored[i].InsertionOrder < scored[j].InsertionOrder
524
+ }
525
+ return scored[i].FinalScore > scored[j].FinalScore
526
+ })
527
+
528
+ results := make([]SearchDoc, 0, len(scored))
529
+ for _, item := range scored {
530
+ results = append(results, item.Result)
531
+ }
532
+ return results, totalResultsBeforeDedup - len(scored)
533
+ }
534
+
535
+ func getPositionPoints(position int) int {
536
+ scores := map[int]int{1: 30, 2: 27, 3: 24, 4: 21, 5: 19, 6: 16, 7: 13, 8: 11, 9: 9, 10: 7, 11: 5, 12: 4, 13: 3, 14: 2}
537
+ if score, ok := scores[position]; ok {
538
+ return score
539
+ }
540
+ return 1
541
+ }
542
+
543
+ func getProviderWeight(provider string) float64 {
544
+ switch provider {
545
+ case "Ref":
546
+ return 1.25
547
+ case "Exa", "Tavily", "Brave":
548
+ return 1.0
549
+ default:
550
+ return 1.0
551
+ }
552
+ }
553
+
554
+ func filterExcludedDomains(providerResults []providerDocs, excludeDomains []string) []providerDocs {
555
+ if len(excludeDomains) == 0 {
556
+ return providerResults
557
+ }
558
+ normalizedExclude := make(map[string]struct{}, len(excludeDomains))
559
+ for _, domain := range excludeDomains {
560
+ normalizedExclude[strings.TrimPrefix(strings.ToLower(strings.TrimSpace(domain)), "www.")] = struct{}{}
561
+ }
562
+ filtered := make([]providerDocs, 0, len(providerResults))
563
+ for _, providerResult := range providerResults {
564
+ docs := make([]SearchDoc, 0, len(providerResult.Docs))
565
+ for _, doc := range providerResult.Docs {
566
+ if _, blocked := normalizedExclude[extractDomain(doc.URL)]; !blocked {
567
+ docs = append(docs, doc)
568
+ }
569
+ }
570
+ filtered = append(filtered, providerDocs{Provider: providerResult.Provider, Docs: docs})
571
+ }
572
+ return filtered
573
+ }
574
+
575
+ func normalizeURL(raw string) string {
576
+ parsed, err := url.Parse(raw)
577
+ if err != nil || parsed.Host == "" {
578
+ return strings.Split(strings.TrimSuffix(strings.ToLower(raw), "/"), "?")[0]
579
+ }
580
+ base := strings.ToLower(parsed.Scheme + "://" + parsed.Host + strings.TrimSuffix(parsed.EscapedPath(), "/"))
581
+ tracking := map[string]struct{}{"utm_source": {}, "utm_medium": {}, "utm_campaign": {}, "utm_term": {}, "utm_content": {}, "ref": {}, "fbclid": {}, "gclid": {}}
582
+ vals := url.Values{}
583
+ for key, vs := range parsed.Query() {
584
+ if _, skip := tracking[strings.ToLower(key)]; skip {
585
+ continue
586
+ }
587
+ for _, v := range vs {
588
+ vals.Add(key, v)
589
+ }
590
+ }
591
+ if encoded := vals.Encode(); encoded != "" {
592
+ return base + "?" + encoded
593
+ }
594
+ return base
595
+ }
596
+
597
+ func extractDomain(raw string) string {
598
+ parsed, err := url.Parse(raw)
599
+ if err == nil && parsed.Hostname() != "" {
600
+ return strings.TrimPrefix(strings.ToLower(parsed.Hostname()), "www.")
601
+ }
602
+ cleaned := strings.TrimPrefix(strings.TrimPrefix(strings.ToLower(raw), "https://"), "http://")
603
+ cleaned = strings.TrimPrefix(cleaned, "www.")
604
+ return strings.Split(strings.Split(cleaned, "/")[0], "?")[0]
605
+ }
606
+
607
+ func decodeHTML(text string) string {
608
+ replacer := strings.NewReplacer("&amp;", "&", "&lt;", "<", "&gt;", ">", "&quot;", `"`, "&#x27;", "'", "&#39;", "'", "&apos;", "'")
609
+ return replacer.Replace(text)
610
+ }
611
+
612
+ func truncateWords(s string, maxWords int) string {
613
+ fields := strings.Fields(strings.TrimSpace(s))
614
+ if len(fields) <= maxWords {
615
+ return strings.Join(fields, " ")
616
+ }
617
+ return strings.Join(fields[:maxWords], " ")
618
+ }
619
+
620
+ func doJSONRequest(ctx context.Context, method, rawURL string, headers map[string]string, payload any) ([]byte, error) {
621
+ bodyBytes, err := json.Marshal(payload)
622
+ if err != nil {
623
+ return nil, err
624
+ }
625
+ return doRawRequest(ctx, method, rawURL, headers, bodyBytes)
626
+ }
627
+
628
+ func doRawRequest(ctx context.Context, method, rawURL string, headers map[string]string, body []byte) ([]byte, error) {
629
+ var reader io.Reader
630
+ if body != nil {
631
+ reader = strings.NewReader(string(body))
632
+ }
633
+ req, err := http.NewRequestWithContext(ctx, method, rawURL, reader)
634
+ if err != nil {
635
+ return nil, err
636
+ }
637
+ for k, v := range headers {
638
+ req.Header.Set(k, v)
639
+ }
640
+ resp, err := http.DefaultClient.Do(req)
641
+ if err != nil {
642
+ return nil, err
643
+ }
644
+ defer resp.Body.Close()
645
+ respBody, err := io.ReadAll(resp.Body)
646
+ if err != nil {
647
+ return nil, err
648
+ }
649
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
650
+ if len(respBody) > 0 {
651
+ return nil, fmt.Errorf("API request failed: %s - %s", resp.Status, strings.TrimSpace(string(respBody)))
652
+ }
653
+ return nil, fmt.Errorf("API request failed: %s", resp.Status)
654
+ }
655
+ return respBody, nil
656
+ }
657
+
658
+ func stringValue(v any) string {
659
+ if s, ok := v.(string); ok {
660
+ return s
661
+ }
662
+ return ""
663
+ }
664
+
665
+ func min(a, b int) int {
666
+ if a < b {
667
+ return a
668
+ }
669
+ return b
670
+ }
@@ -0,0 +1,16 @@
1
+ package buildinfo
2
+
3
+ import "strings"
4
+
5
+ const defaultVersion = "dev"
6
+
7
+ // Version is overridden at build time via linker flags.
8
+ var Version = defaultVersion
9
+
10
+ func CurrentVersion() string {
11
+ trimmed := strings.TrimSpace(Version)
12
+ if trimmed == "" {
13
+ return defaultVersion
14
+ }
15
+ return trimmed
16
+ }