webctx 0.2.8 → 0.2.10

package/internal/app/search_keys_test.go

@@ -0,0 +1,227 @@
+package app
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestSearchKeyOrderDeduplicationAndMissingKeys(t *testing.T) {
+	t.Setenv("EXA_API_KEYS", ` ["new","new","old"] `)
+	t.Setenv("EXA_API_KEY", "old")
+	if got, want := searchKeysForProvider("exa"), []string{"new", "old"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("deduplication: got %v, want %v", got, want)
+	}
+	t.Setenv("EXA_API_KEYS", "new,another\n old\n")
+	if got, want := searchKeysForProvider("exa"), []string{"new", "another", "old"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("CSV/newline parsing: got %v, want %v", got, want)
+	}
+	t.Setenv("EXA_API_KEYS", `[invalid`)
+	t.Setenv("EXA_API_KEY", "")
+	if got := searchKeysForProvider("exa"); len(got) != 0 {
+		t.Fatalf("malformed JSON should not be used as credential: %v", got)
+	}
+	if got := searchKeysForProvider("unknown"); len(got) != 0 {
+		t.Fatalf("unknown provider should have no keys: %v", got)
+	}
+}
+
+func TestStatusReportsMultipleKeysWithoutRevealingThem(t *testing.T) {
+	t.Setenv("WEBCTX_CONFIG_FILE", filepath.Join(t.TempDir(), "webctx", "config.json"))
+	t.Setenv("EXA_API_KEYS", "extra-secret-a,extra-secret-b")
+	t.Setenv("EXA_API_KEY", "legacy-secret")
+	t.Setenv("BRAVE_API_KEY", "")
+	t.Setenv("BRAVE_API_KEYS", "")
+	t.Setenv("TAVILY_API_KEY", "")
+	t.Setenv("TAVILY_API_KEYS", "")
+	t.Setenv("FIRECRAWL_API_KEY", "")
+	var out, errOut bytes.Buffer
+	if code := Run([]string{"status"}, &out, &errOut); code != 0 {
+		t.Fatalf("status error: %s", errOut.String())
+	}
+	if !strings.Contains(out.String(), "exa     enabled   key configured (3 keys)") {
+		t.Fatalf("incorrect key count: %q", out.String())
+	}
+	for _, secret := range []string{"extra-secret-a", "extra-secret-b", "legacy-secret"} {
+		if strings.Contains(out.String()+errOut.String(), secret) {
+			t.Fatal("status exposed a configured API key")
+		}
+	}
+}
+
+func TestManagedSearchKeychainAddRemoveAndInputValidation(t *testing.T) {
+	oldLookup, oldStore := keychainLookup, keychainStore
+	t.Cleanup(func() { keychainLookup, keychainStore = oldLookup, oldStore })
+	managed := map[string]string{}
+	keychainLookup = func(account string) (string, error) { return managed[account], nil }
+	keychainStore = func(account, secret string) error { managed[account] = secret; return nil }
+
+	for _, candidate := range []string{"", "one two", "line1\nline2", strings.Repeat("z", 4097)} {
+		if _, err := addManagedSearchKey("exa", strings.NewReader(candidate)); err == nil {
+			t.Fatalf("invalid secret accepted")
+		}
+	}
+	if len(managed) != 0 {
+		t.Fatal("invalid keys must not modify Keychain")
+	}
+	if _, err := addManagedSearchKey("unknown", strings.NewReader("token")); err == nil {
+		t.Fatal("unknown provider accepted")
+	}
+	for _, pair := range []struct {
+		key   string
+		count int
+	}{{"older", 1}, {"newer", 2}, {"newer", 2}} {
+		count, err := addManagedSearchKey("exa", strings.NewReader(pair.key+"\n"))
+		if err != nil || count != pair.count {
+			t.Fatalf("add: count=%d error=%v", count, err)
+		}
+	}
+	list, err := readManagedKeys("exa")
+	if err != nil || !reflect.DeepEqual(list, []string{"newer", "older"}) {
+		t.Fatalf("priority order changed: %v %v", list, err)
+	}
+	if strings.Contains(managed["EXA_API_KEYS"], "[REDACTED]") {
+		t.Fatal("managed keys stored as placeholder instead of actual tokens")
+	}
+	if _, err := removeManagedSearchKey("exa", "3"); err == nil {
+		t.Fatal("out-of-range removal accepted")
+	}
+	if count, err := removeManagedSearchKey("exa", "1"); err != nil || count != 1 {
+		t.Fatalf("remove: count=%d error=%v", count, err)
+	}
+	if list, err := readManagedKeys("exa"); err != nil || !reflect.DeepEqual(list, []string{"older"}) {
+		t.Fatalf("wrong managed key removed: %v %v", list, err)
+	}
+	managed["EXA_API_KEYS"] = "{invalid"
+	if _, err := addManagedSearchKey("exa", strings.NewReader("another")); err == nil {
+		t.Fatal("malformed Keychain JSON silently overwritten")
+	}
+}
+
+func TestManagedSearchKeyCommandNeverPrintsSecrets(t *testing.T) {
+	oldLookup, oldStore, oldStdin := keychainLookup, keychainStore, os.Stdin
+	t.Cleanup(func() { keychainLookup, keychainStore, os.Stdin = oldLookup, oldStore, oldStdin })
+	keychainLookup = func(account string) (string, error) { return "", nil }
+	keychainStore = func(account, value string) error { return nil }
+	input, writer, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := io.WriteString(writer, "sensitive-api-token\n"); err != nil {
+		t.Fatal(err)
+	}
+	_ = writer.Close()
+	os.Stdin = input
+	var out, errOut bytes.Buffer
+	code := Run([]string{"keys", "add", "exa", "--stdin"}, &out, &errOut)
+	_ = input.Close()
+	if code != 0 || strings.Contains(out.String()+errOut.String(), "sensitive-api-token") {
+		t.Fatalf("unsafe CLI output or command failed: code=%d out=%q err=%q", code, out.String(), errOut.String())
+	}
+}
+
+// Exercise the real HTTP request construction and key-selection order for all
+// three providers, without making network requests or disclosing API keys.
+func TestSearchProviderFallbackAcrossKeys(t *testing.T) {
+	t.Setenv("BRAVE_API_KEY", "legacy-brave")
+	t.Setenv("TAVILY_API_KEY", "legacy-tavily")
+	t.Setenv("EXA_API_KEY", "legacy-exa")
+	for _, name := range []string{"BRAVE", "TAVILY", "EXA"} {
+		t.Setenv(name+"_API_KEYS", "bad-"+strings.ToLower(name)+",good-"+strings.ToLower(name))
+	}
+	previous := http.DefaultClient.Transport
+	requests := map[string][]string{}
+	http.DefaultClient.Transport = roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		provider, key := "", ""
+		switch req.URL.Host {
+		case "api.search.brave.com":
+			provider, key = "brave", req.Header.Get("x-subscription-token")
+		case "api.exa.ai":
+			provider, key = "exa", req.Header.Get("x-api-key")
+		case "api.tavily.com":
+			provider = "tavily"
+			var payload map[string]any
+			_ = json.NewDecoder(req.Body).Decode(&payload)
+			key, _ = payload["api_key"].(string)
+		default:
+			t.Errorf("unexpected host: %s", req.URL.Host)
+		}
+		requests[provider] = append(requests[provider], key)
+		if strings.HasPrefix(key, "bad-") {
+			code := http.StatusUnauthorized
+			switch provider {
+			case "brave":
+				return testHTTPResponse(req, 422, `{"error":{"code":"SUBSCRIPTION_TOKEN_INVALID"}}`, nil), nil
+			case "tavily":
+				code = 432
+			case "exa":
+				code = http.StatusPaymentRequired
+			}
+			return testHTTPResponse(req, code, `{"error":"credits exhausted"}`, nil), nil
+		}
+		if strings.HasPrefix(key, "good-") {
+			switch provider {
+			case "brave":
+				return testHTTPResponse(req, 200, `{"web":{"results":[{"url":"https://example.com/guide","title":"Brave result"}]}}`, nil), nil
+			case "tavily", "exa":
+				return testHTTPResponse(req, 200, `{"results":[{"url":"https://example.com/guide","title":"API result"}]}`, nil), nil
+			}
+		}
+		return testHTTPResponse(req, 500, "unexpected key", nil), nil
+	})
+	t.Cleanup(func() { http.DefaultClient.Transport = previous })
+
+	for _, test := range []struct {
+		provider string
+		call     func() (SearchResult, error)
+	}{
+		{"brave", func() (SearchResult, error) { return searchWithBrave(context.Background(), "test query") }},
+		{"tavily", func() (SearchResult, error) { return searchWithTavily(context.Background(), "test query", nil) }},
+		{"exa", func() (SearchResult, error) {
+			return searchWithExa(context.Background(), "test query", nil, "exact phrase")
+		}},
+	} {
+		results, err := test.call()
+		if err != nil || len(results.Docs) != 1 {
+			t.Fatalf("%s: search failed after key rotation: %v %+v", test.provider, err, results)
+		}
+		want := []string{"bad-" + test.provider, "good-" + test.provider}
+		if got := requests[test.provider]; !reflect.DeepEqual(got, want) {
+			t.Fatalf("%s: key rotation order got %v, want %v", test.provider, got, want)
+		}
+	}
+}
+
+func TestSearchKeyRetryDoesNotMaskBadInputOrLeakKeys(t *testing.T) {
+	t.Setenv("EXA_API_KEYS", "first-secret,second-secret")
+	t.Setenv("EXA_API_KEY", "")
+	previous := http.DefaultClient.Transport
+	count, code := 0, http.StatusBadRequest
+	http.DefaultClient.Transport = roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		count++
+		return testHTTPResponse(req, code, fmt.Sprintf(`{"error":"key %s failed"}`, req.Header.Get("x-api-key")), nil), nil
+	})
+	t.Cleanup(func() { http.DefaultClient.Transport = previous })
+	_, err := searchWithExa(context.Background(), "test", nil, "")
+	if err == nil || count != 1 || strings.Contains(err.Error(), "first-secret") {
+		t.Fatalf("400 should not retry or leak a credential: count=%d error=%v", count, err)
+	}
+	count, code = 0, http.StatusUnprocessableEntity
+	_, err = searchWithExa(context.Background(), "test", nil, "")
+	if err == nil || count != 1 {
+		t.Fatalf("non-credential 422 should not retry: count=%d error=%v", count, err)
+	}
+	count, code = 0, http.StatusTooManyRequests
+	_, err = searchWithExa(context.Background(), "test", nil, "")
+	if err == nil || count != 2 || strings.Contains(err.Error(), "first-secret") || strings.Contains(err.Error(), "second-secret") || !strings.Contains(err.Error(), "up to 2") {
+		t.Fatalf("exhausted-key error is unsafe or not actionable: count=%d error=%v", count, err)
+	}
+}

package/internal/app/tools.go

@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"os"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -37,6 +38,11 @@ type providerDocs struct {
 	Err      error
 }
 
+type namedSearch struct {
+	name string
+	fn   func(context.Context) (SearchResult, error)
+}
+
 type scoredURL struct {
 	Result         SearchDoc
 	InsertionOrder int
@@ -47,6 +53,13 @@ type scoredURL struct {
 }
 
 func Search(params SearchParams) (string, error) {
+	config, err := currentProviderConfig()
+	if err != nil {
+		return "", err
+	}
+	if !config.enabled("brave") && !config.enabled("tavily") && !config.enabled("exa") {
+		return "", errors.New("Error searching the web: all search providers are disabled; run 'webctx providers enable brave', 'webctx providers enable tavily', or 'webctx providers enable exa'")
+	}
 	defaultExcludedDomains := []string{"youtube.com", "vimeo.com", "dailymotion.com", "twitch.tv", "tiktok.com", "instagram.com", "facebook.com"}
 	allExcludedDomains := append(append([]string{}, defaultExcludedDomains...), params.ExcludeDomains...)
 	truncatedKeyword := truncateWords(params.IncludeKeyword, 5)
@@ -56,46 +69,44 @@ func Search(params SearchParams) (string, error) {
 	alexandriaDone := make(chan string, 1)
 	go func() { alexandriaDone <- discoverAlexandria(params.Query) }()
 
-	type namedSearch struct {
-		name string
-		fn   func(context.Context) (SearchResult, error)
-	}
-
 	searches := []namedSearch{}
+	keywordFallback := false
 	if strings.TrimSpace(truncatedKeyword) != "" {
-		searches = append(searches, namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) {
-			return searchWithExa(ctx, params.Query, nil, truncatedKeyword)
-		}})
+		if config.enabled("exa") && len(searchKeysForProvider("exa")) > 0 {
+			searches = append(searches, namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) {
+				return searchWithExa(ctx, params.Query, nil, truncatedKeyword)
+			}})
+		}
 	} else {
-		searches = append(searches,
-			namedSearch{name: "Brave", fn: func(ctx context.Context) (SearchResult, error) { return searchWithBrave(ctx, params.Query) }},
-			namedSearch{name: "Tavily", fn: func(ctx context.Context) (SearchResult, error) { return searchWithTavily(ctx, params.Query, nil) }},
-			namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) { return searchWithExa(ctx, params.Query, nil, "") }},
-		)
+		searches = append(searches, alternativeSearches(config, params.Query)...)
+		if config.enabled("exa") {
+			searches = append(searches, namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) {
+				return searchWithExa(ctx, params.Query, nil, "")
+			}})
+		}
 	}
 
-	results := make([]providerDocs, len(searches))
-	var wg sync.WaitGroup
-	for i, s := range searches {
-		wg.Add(1)
-		go func(i int, s namedSearch) {
-			defer wg.Done()
-			ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
-			defer cancel()
-			res, err := s.fn(ctx)
-			if err != nil {
-				results[i] = providerDocs{Provider: s.name, Docs: []SearchDoc{}, Err: err}
-				return
-			}
-			results[i] = providerDocs{Provider: s.name, Docs: res.Docs}
-		}(i, s)
+	results := runSearches(searches)
+	if truncatedKeyword != "" && (len(searches) == 0 || (len(results) == 1 && results[0].Err != nil)) {
+		// Exa's includeText filter has no equivalent in Brave or Tavily. When Exa
+		// is disabled, unconfigured, or fails, search for the quoted phrase instead
+		// of making keyword searches unusable. Disclose the weaker semantics.
+		fallbackQuery := params.Query + " " + strconv.Quote(truncatedKeyword)
+		fallbackSearches := alternativeSearches(config, fallbackQuery)
+		if len(fallbackSearches) > 0 {
+			results = append(results, runSearches(fallbackSearches)...)
+			keywordFallback = true
+		} else if len(searches) == 0 && config.enabled("exa") {
+			// Preserve the helpful missing-key error if Exa is the only enabled
+			// provider and it has no key.
+			results = append(results, providerDocs{Provider: "Exa", Err: errors.New("missing EXA_API_KEY")})
+		}
 	}
-	wg.Wait()
-
-	total := 0
-	for _, r := range results {
-		total += len(r.Docs)
+	if len(results) == 0 {
+		return "", errors.New("Error searching the web: no enabled search providers are available for this query; use 'webctx status' to inspect settings")
 	}
+
+	total := countSearchDocs(results)
 	if total == 0 {
 		missingKeys := uniqueMissingCredentialKeys(results)
 		if len(missingKeys) > 0 {
@@ -123,6 +134,9 @@ func Search(params SearchParams) (string, error) {
 	}
 
 	parts := []string{fmt.Sprintf("Total Results: %d\n", len(ranked))}
+	if keywordFallback {
+		parts = append(parts, "> **Keyword fallback:** Exa was disabled or unavailable. Brave/Tavily searched for the quoted phrase; this is not a strict page-text inclusion filter.", "")
+	}
 	// The ranked index remains available, but agents also get useful source
 	// context without having to issue several follow-up read-link calls.
 	if context := enrichSearch(params.Query, ranked); context != "" {
@@ -144,6 +158,46 @@ func Search(params SearchParams) (string, error) {
 	return strings.Join(parts, "\n"), nil
 }
 
+func alternativeSearches(config providerConfig, query string) []namedSearch {
+	searches := []namedSearch{}
+	if config.enabled("brave") {
+		searches = append(searches, namedSearch{name: "Brave", fn: func(ctx context.Context) (SearchResult, error) { return searchWithBrave(ctx, query) }})
+	}
+	if config.enabled("tavily") {
+		searches = append(searches, namedSearch{name: "Tavily", fn: func(ctx context.Context) (SearchResult, error) { return searchWithTavily(ctx, query, nil) }})
+	}
+	return searches
+}
+
+func countSearchDocs(results []providerDocs) int {
+	count := 0
+	for _, r := range results {
+		count += len(r.Docs)
+	}
+	return count
+}
+
+func runSearches(searches []namedSearch) []providerDocs {
+	results := make([]providerDocs, len(searches))
+	var wg sync.WaitGroup
+	for i, s := range searches {
+		wg.Add(1)
+		go func(i int, s namedSearch) {
+			defer wg.Done()
+			ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
+			defer cancel()
+			res, err := s.fn(ctx)
+			if err != nil {
+				results[i] = providerDocs{Provider: s.name, Docs: []SearchDoc{}, Err: err}
+				return
+			}
+			results[i] = providerDocs{Provider: s.name, Docs: res.Docs}
+		}(i, s)
+	}
+	wg.Wait()
+	return results
+}
+
 func ReadLink(rawURL string) (string, error) {
 	if err := validateSourceURL(rawURL); err != nil {
 		return "", err
@@ -327,20 +381,17 @@ func missingCredentialsError(prefix string, keys []string, detail string) error
 }
 
 func searchWithBrave(ctx context.Context, query string) (SearchResult, error) {
-	apiKey := strings.TrimSpace(os.Getenv("BRAVE_API_KEY"))
-	if apiKey == "" {
-		return SearchResult{}, errors.New("missing BRAVE_API_KEY")
-	}
 	params := url.Values{}
 	params.Set("q", query)
 	params.Set("text_decorations", "false")
 	params.Set("result_filter", "web")
 	params.Set("limit", "20")
-	body, err := doRawRequest(ctx, http.MethodGet, "https://api.search.brave.com/res/v1/web/search?"+params.Encode(), map[string]string{
-		"Accept":               "application/json",
-		"Accept-Encoding":      "gzip",
-		"x-subscription-token": apiKey,
-	}, nil)
+	body, err := useSearchKeys(ctx, "brave", func(apiKey string) ([]byte, error) {
+		return doRawRequest(ctx, http.MethodGet, "https://api.search.brave.com/res/v1/web/search?"+params.Encode(), map[string]string{
+			"Accept":               "application/json",
+			"x-subscription-token": apiKey,
+		}, nil)
+	})
 	if err != nil {
 		return SearchResult{}, err
 	}
@@ -367,22 +418,20 @@ func searchWithBrave(ctx context.Context, query string) (SearchResult, error) {
 }
 
 func searchWithTavily(ctx context.Context, query string, excludeDomains []string) (SearchResult, error) {
-	apiKey := strings.TrimSpace(os.Getenv("TAVILY_API_KEY"))
-	if apiKey == "" {
-		return SearchResult{}, errors.New("missing TAVILY_API_KEY")
-	}
 	requestBody := map[string]any{
-		"api_key":     apiKey,
 		"query":       query,
 		"max_results": 20,
 	}
 	if len(excludeDomains) > 0 {
 		requestBody["exclude_domains"] = excludeDomains
 	}
-	body, err := doJSONRequest(ctx, http.MethodPost, "https://api.tavily.com/search", map[string]string{
-		"Content-Type": "application/json",
-		"Accept":       "application/json",
-	}, requestBody)
+	body, err := useSearchKeys(ctx, "tavily", func(apiKey string) ([]byte, error) {
+		requestBody["api_key"] = apiKey
+		return doJSONRequest(ctx, http.MethodPost, "https://api.tavily.com/search", map[string]string{
+			"Content-Type": "application/json",
+			"Accept":       "application/json",
+		}, requestBody)
+	})
 	if err != nil {
 		return SearchResult{}, err
 	}
@@ -407,10 +456,6 @@ func searchWithTavily(ctx context.Context, query string, excludeDomains []string
 }
 
 func searchWithExa(ctx context.Context, query string, excludeDomains []string, includeKeyword string) (SearchResult, error) {
-	apiKey := strings.TrimSpace(os.Getenv("EXA_API_KEY"))
-	if apiKey == "" {
-		return SearchResult{}, errors.New("missing EXA_API_KEY")
-	}
 	requestBody := map[string]any{
 		"query":      query,
 		"type":       "auto",
@@ -425,11 +470,13 @@ func searchWithExa(ctx context.Context, query string, excludeDomains []string, i
 	if strings.TrimSpace(includeKeyword) != "" {
 		requestBody["includeText"] = []string{includeKeyword}
 	}
-	body, err := doJSONRequest(ctx, http.MethodPost, "https://api.exa.ai/search", map[string]string{
-		"Accept":       "application/json",
-		"Content-Type": "application/json",
-		"x-api-key":    apiKey,
-	}, requestBody)
+	body, err := useSearchKeys(ctx, "exa", func(apiKey string) ([]byte, error) {
+		return doJSONRequest(ctx, http.MethodPost, "https://api.exa.ai/search", map[string]string{
+			"Accept":       "application/json",
+			"Content-Type": "application/json",
+			"x-api-key":    apiKey,
+		}, requestBody)
+	})
 	if err != nil {
 		return SearchResult{}, err
 	}
@@ -631,13 +678,20 @@ func doRawRequest(ctx context.Context, method, rawURL string, headers map[string
 	}
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		if len(respBody) > 0 {
-			return nil, fmt.Errorf("API request failed: %s - %s", resp.Status, strings.TrimSpace(string(respBody)))
+			return nil, &httpStatusError{Code: resp.StatusCode, Message: fmt.Sprintf("API request failed: %s - %s", resp.Status, strings.TrimSpace(string(respBody)))}
 		}
-		return nil, fmt.Errorf("API request failed: %s", resp.Status)
+		return nil, &httpStatusError{Code: resp.StatusCode, Message: "API request failed: " + resp.Status}
 	}
 	return respBody, nil
 }
 
+type httpStatusError struct {
+	Code    int
+	Message string
+}
+
+func (e *httpStatusError) Error() string { return e.Message }
+
 func stringValue(v any) string {
 	if s, ok := v.(string); ok {
 		return s

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webctx",
-  "version": "0.2.8",
+  "version": "0.2.10",
   "description": "Go CLI for agent-friendly web research, structured-data discovery, and source-native querying",
   "license": "MIT",
   "author": "amxv",