webctx 0.2.8 → 0.2.10

package/internal/app/providers_test.go

@@ -0,0 +1,305 @@
+package app
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestProviderCommandsPersistAndReport(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "webctx", "config.json")
+	t.Setenv("WEBCTX_CONFIG_FILE", path)
+	t.Setenv("BRAVE_API_KEY", "test-brave-secret")
+	t.Setenv("TAVILY_API_KEY", "")
+	t.Setenv("EXA_API_KEY", "test-exa-secret")
+	t.Setenv("FIRECRAWL_API_KEY", "")
+
+	run := func(args ...string) (int, string, string) {
+		t.Helper()
+		var out, errOut bytes.Buffer
+		code := Run(args, &out, &errOut)
+		return code, out.String(), errOut.String()
+	}
+	if code, text, errText := run("status"); code != 0 || !strings.Contains(text, "exa     enabled") || !strings.Contains(text, "brave   enabled") || errText != "" {
+		t.Fatalf("default status: code=%d text=%q error=%q", code, text, errText)
+	}
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		t.Fatalf("status should not create config file, got error %v", err)
+	}
+	if code, text, errText := run("providers", "disable", "ExA"); code != 0 || !strings.Contains(text, "exa disabled") || errText != "" {
+		t.Fatalf("disable: code=%d text=%q error=%q", code, text, errText)
+	}
+	if code, text, _ := run("providers"); code != 0 || !strings.Contains(text, "exa     disabled") || !strings.Contains(text, "key configured") {
+		t.Fatalf("persisted status: code=%d text=%q", code, text)
+	} else if strings.Contains(text, "test-exa-secret") || strings.Contains(text, "test-brave-secret") {
+		t.Fatalf("status exposed a credential: %q", text)
+	}
+	content, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var config providerConfig
+	if err := json.Unmarshal(content, &config); err != nil || !strings.EqualFold(strings.Join(config.DisabledProviders, ","), "exa") {
+		t.Fatalf("settings not persisted: %s; error=%v", content, err)
+	}
+	if info, err := os.Stat(path); err != nil || info.Mode().Perm() != 0o600 {
+		t.Fatalf("config permissions: info=%v error=%v", info, err)
+	}
+	if code, text, errText := run("providers", "enable", "exa"); code != 0 || !strings.Contains(text, "exa enabled") || errText != "" {
+		t.Fatalf("enable: code=%d text=%q error=%q", code, text, errText)
+	}
+	if code, text, _ := run("status"); code != 0 || !strings.Contains(text, "exa     enabled") {
+		t.Fatalf("restored status: code=%d text=%q", code, text)
+	}
+}
+
+func TestProviderCommandsRejectInvalidInputAndMalformedConfig(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config.json")
+	t.Setenv("WEBCTX_CONFIG_FILE", path)
+	for _, args := range [][]string{{"providers", "disable", "firecrawl"}, {"providers", "disable", "unknown"}, {"providers", "disable"}, {"status", "enable", "exa"}} {
+		var out, errOut bytes.Buffer
+		if Run(args, &out, &errOut) == 0 || errOut.Len() == 0 {
+			t.Fatalf("invalid command %q succeeded: %q", args, out.String())
+		}
+	}
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		t.Fatalf("invalid command wrote config: %v", err)
+	}
+	if err := os.WriteFile(path, []byte("{invalid"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var out, errOut bytes.Buffer
+	if Run([]string{"status"}, &out, &errOut) == 0 || !strings.Contains(errOut.String(), "invalid provider settings") {
+		t.Fatalf("malformed config not reported: %q", errOut.String())
+	}
+	if Run([]string{"providers", "disable", "exa"}, &out, &errOut) == 0 {
+		t.Fatal("malformed config should not be overwritten")
+	}
+}
+
+type providerRequests struct {
+	mu        sync.Mutex
+	queries   map[string][]string
+	failedExa bool
+	emptyExa  bool
+}
+
+func mockSearchProviders(t *testing.T, exaFailure bool) *providerRequests {
+	t.Helper()
+	prev := http.DefaultClient.Transport
+	stats := &providerRequests{queries: make(map[string][]string), failedExa: exaFailure}
+	http.DefaultClient.Transport = roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		provider, query := "", ""
+		switch req.URL.Host {
+		case "api.search.brave.com":
+			provider, query = "brave", req.URL.Query().Get("q")
+		case "api.tavily.com", "api.exa.ai":
+			if req.URL.Host == "api.tavily.com" {
+				provider = "tavily"
+			} else {
+				provider = "exa"
+			}
+			data, _ := io.ReadAll(req.Body)
+			var payload map[string]any
+			_ = json.Unmarshal(data, &payload)
+			query, _ = payload["query"].(string)
+		case "provider-test.example":
+			return testHTTPResponse(req, http.StatusOK, "# Example documentation\n\nThis markdown page contains enough content for a test.", map[string]string{"Content-Type": "text/markdown"}), nil
+		default:
+			return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+		}
+		stats.mu.Lock()
+		stats.queries[provider] = append(stats.queries[provider], query)
+		stats.mu.Unlock()
+		switch provider {
+		case "brave":
+			return testHTTPResponse(req, http.StatusOK, `{"web":{"results":[{"url":"https://provider-test.example/guide.md","title":"Guide","description":"Documentation"}]}}`, nil), nil
+		case "tavily":
+			return testHTTPResponse(req, http.StatusOK, `{"results":[]}`, nil), nil
+		case "exa":
+			if stats.failedExa {
+				return testHTTPResponse(req, http.StatusPaymentRequired, `{"error":"NO_MORE_CREDITS"}`, nil), nil
+			}
+			if stats.emptyExa {
+				return testHTTPResponse(req, http.StatusOK, `{"results":[]}`, nil), nil
+			}
+			return testHTTPResponse(req, http.StatusOK, `{"results":[{"url":"https://provider-test.example/guide.md","title":"Guide","text":"Documentation"}]}`, nil), nil
+		}
+		return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+	})
+	t.Cleanup(func() { http.DefaultClient.Transport = prev })
+	return stats
+}
+
+func (p *providerRequests) received(provider string) []string {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return append([]string(nil), p.queries[provider]...)
+}
+
+func searchTestEnv(t *testing.T) {
+	t.Helper()
+	t.Setenv("WEBCTX_CONFIG_FILE", filepath.Join(t.TempDir(), "config.json"))
+	t.Setenv("BRAVE_API_KEY", "test-brave")
+	t.Setenv("TAVILY_API_KEY", "test-tavily")
+	t.Setenv("EXA_API_KEY", "test-exa")
+	t.Setenv("BRAVE_API_KEYS", "")
+	t.Setenv("TAVILY_API_KEYS", "")
+	t.Setenv("EXA_API_KEYS", "")
+	t.Setenv("FIRECRAWL_API_KEY", "")
+}
+
+func TestSearchDoesNotCallDisabledExa(t *testing.T) {
+	searchTestEnv(t)
+	stats := mockSearchProviders(t, false)
+	if _, err := setSearchProviderEnabled("exa", false); err != nil {
+		t.Fatal(err)
+	}
+	result, err := Search(SearchParams{Query: "runtime docs"})
+	if err != nil || !strings.Contains(result, "Total Results: 1") {
+		t.Fatalf("search without Exa failed: %v; %q", err, result)
+	}
+	if got := stats.received("exa"); len(got) != 0 {
+		t.Fatalf("Exa was contacted despite being disabled: %v", got)
+	}
+	if got := stats.received("brave"); len(got) != 1 || got[0] != "runtime docs" {
+		t.Fatalf("Brave was not used: %v", got)
+	}
+	if got := stats.received("tavily"); len(got) != 1 {
+		t.Fatalf("Tavily was not used: %v", got)
+	}
+}
+
+func TestSearchSupportsOnlyBraveEnabled(t *testing.T) {
+	searchTestEnv(t)
+	stats := mockSearchProviders(t, false)
+	for _, name := range []string{"exa", "tavily"} {
+		if _, err := setSearchProviderEnabled(name, false); err != nil {
+			t.Fatal(err)
+		}
+	}
+	result, err := Search(SearchParams{Query: "runtime docs"})
+	if err != nil || !strings.Contains(result, "Total Results: 1") {
+		t.Fatalf("single-provider search failed: %v; %q", err, result)
+	}
+	if len(stats.received("exa")) != 0 || len(stats.received("tavily")) != 0 || len(stats.received("brave")) != 1 {
+		t.Fatalf("unexpected requests: %#v", stats.queries)
+	}
+}
+
+func TestKeywordFallbackSkipsDisabledExa(t *testing.T) {
+	searchTestEnv(t)
+	stats := mockSearchProviders(t, false)
+	if _, err := setSearchProviderEnabled("exa", false); err != nil {
+		t.Fatal(err)
+	}
+	result, err := Search(SearchParams{Query: "drizzle orm", IncludeKeyword: "migration guide"})
+	if err != nil || !strings.Contains(result, "Keyword fallback") {
+		t.Fatalf("keyword fallback failed: %v; %q", err, result)
+	}
+	if len(stats.received("exa")) != 0 {
+		t.Fatal("disabled Exa received keyword request")
+	}
+	for _, provider := range []string{"brave", "tavily"} {
+		got := stats.received(provider)
+		if len(got) != 1 || !strings.Contains(got[0], `"migration guide"`) {
+			t.Fatalf("%s did not receive a quoted phrase: %v", provider, got)
+		}
+	}
+}
+
+func TestKeywordFallbackOnExaCreditExhaustion(t *testing.T) {
+	searchTestEnv(t)
+	stats := mockSearchProviders(t, true)
+	result, err := Search(SearchParams{Query: "drizzle orm", IncludeKeyword: "migration guide"})
+	if err != nil || !strings.Contains(result, "Keyword fallback") || !strings.Contains(result, "Total Results: 1") {
+		t.Fatalf("keyword fallback after Exa error failed: %v; %q", err, result)
+	}
+	if len(stats.received("exa")) != 1 || len(stats.received("brave")) != 1 {
+		t.Fatalf("unexpected provider request sequence: %#v", stats.queries)
+	}
+}
+
+func TestKeywordExaSuccessDoesNotContactOtherProviders(t *testing.T) {
+	searchTestEnv(t)
+	stats := mockSearchProviders(t, false)
+	result, err := Search(SearchParams{Query: "drizzle orm", IncludeKeyword: "migration guide"})
+	if err != nil || strings.Contains(result, "Keyword fallback") || !strings.Contains(result, "Total Results: 1") {
+		t.Fatalf("unexpected keyword results: %v; %q", err, result)
+	}
+	if len(stats.received("exa")) != 1 || len(stats.received("brave")) != 0 || len(stats.received("tavily")) != 0 {
+		t.Fatalf("unexpected provider requests: %#v", stats.queries)
+	}
+}
+
+func TestKeywordExaNoMatchesDoesNotBroadenSearch(t *testing.T) {
+	searchTestEnv(t)
+	stats := mockSearchProviders(t, false)
+	stats.emptyExa = true
+	_, err := Search(SearchParams{Query: "drizzle orm", IncludeKeyword: "migration guide"})
+	if err == nil || !strings.Contains(err.Error(), "failed to return results") {
+		t.Fatalf("unexpected result for zero Exa matches: %v", err)
+	}
+	if len(stats.received("exa")) != 1 || len(stats.received("brave")) != 0 || len(stats.received("tavily")) != 0 {
+		t.Fatalf("strict keyword search was broadened: %#v", stats.queries)
+	}
+}
+
+func TestSearchWithAllProvidersDisabledFailsFast(t *testing.T) {
+	searchTestEnv(t)
+	stats := mockSearchProviders(t, false)
+	for _, name := range []string{"brave", "tavily", "exa"} {
+		if _, err := setSearchProviderEnabled(name, false); err != nil {
+			t.Fatal(err)
+		}
+	}
+	_, err := Search(SearchParams{Query: "anything"})
+	if err == nil || !strings.Contains(err.Error(), "all search providers are disabled") {
+		t.Fatalf("unexpected error when all disabled: %v", err)
+	}
+	if len(stats.received("exa"))+len(stats.received("tavily"))+len(stats.received("brave")) != 0 {
+		t.Fatal("search contacted provider with all disabled")
+	}
+}
+
+func TestBraveSearchTransparentlyDecodesGzip(t *testing.T) {
+	t.Setenv("BRAVE_API_KEY", "test-brave")
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Accept-Encoding") != "gzip" {
+			t.Errorf("expected Go HTTP transport to request gzip, got %q", r.Header.Get("Accept-Encoding"))
+		}
+		w.Header().Set("Content-Encoding", "gzip")
+		w.Header().Set("Content-Type", "application/json")
+		writer := gzip.NewWriter(w)
+		_, _ = writer.Write([]byte(`{"web":{"results":[{"url":"https://example.com/guide","title":"Guide","description":"Documentation"}]}}`))
+		_ = writer.Close()
+	}))
+	defer server.Close()
+
+	originalTransport := http.DefaultClient.Transport
+	http.DefaultClient.Transport = roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.Header.Get("Accept-Encoding") != "" {
+			t.Errorf("Brave search forced Accept-Encoding instead of letting Go decode it: %q", req.Header.Get("Accept-Encoding"))
+		}
+		copy := req.Clone(req.Context())
+		copy.URL.Scheme = "http"
+		copy.URL.Host = strings.TrimPrefix(server.URL, "http://")
+		copy.Host = copy.URL.Host
+		return http.DefaultTransport.RoundTrip(copy)
+	})
+	t.Cleanup(func() { http.DefaultClient.Transport = originalTransport })
+
+	result, err := searchWithBrave(context.Background(), "Go docs")
+	if err != nil || len(result.Docs) != 1 || result.Docs[0].Title != "Guide" {
+		t.Fatalf("Brave gzip response not decoded: %v; %#v", err, result)
+	}
+}

package/internal/app/scrape.go

@@ -21,7 +21,7 @@ type MarkdownResult struct {
 const keychainServiceName = "webctx"
 
 var (
-	credentialEnvKeys = []string{"BRAVE_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "FIRECRAWL_API_KEY", "GH_TOKEN", "GITHUB_TOKEN"}
+	credentialEnvKeys = []string{"BRAVE_API_KEY", "BRAVE_API_KEYS", "TAVILY_API_KEY", "TAVILY_API_KEYS", "EXA_API_KEY", "EXA_API_KEYS", "FIRECRAWL_API_KEY", "GH_TOKEN", "GITHUB_TOKEN"}
 	getwdFunc         = os.Getwd
 	executableFunc    = os.Executable
 	keychainLookup    = lookupKeychainSecret

package/internal/app/search_keys.go

@@ -0,0 +1,233 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"os/exec"
+	"runtime"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// A provider's legacy singular key remains supported. Additional keys are
+// checked first so newly provisioned keys take precedence over exhausted ones.
+// Plural values can be comma/newline-separated or a JSON string array (the
+// latter is how the macOS Keychain stores them).
+func searchKeysForProvider(name string) []string {
+	name = strings.ToLower(strings.TrimSpace(name))
+	if !validSearchProvider(name) {
+		return nil
+	}
+	prefix := strings.ToUpper(name) + "_API_KEY"
+	keys := parseSearchKeyList(os.Getenv(prefix + "S"))
+	if key := strings.TrimSpace(os.Getenv(prefix)); key != "" {
+		keys = append(keys, key)
+	}
+	return uniqueSearchKeys(keys)
+}
+
+func parseSearchKeyList(raw string) []string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return nil
+	}
+	if strings.HasPrefix(raw, "[") {
+		var keys []string
+		if json.Unmarshal([]byte(raw), &keys) != nil {
+			return nil // fail closed rather than sending malformed credential data
+		}
+		return uniqueSearchKeys(keys)
+	}
+	return uniqueSearchKeys(strings.FieldsFunc(raw, func(r rune) bool {
+		return r == ',' || r == '\n' || r == '\r'
+	}))
+}
+
+func uniqueSearchKeys(keys []string) []string {
+	seen := make(map[string]bool, len(keys))
+	unique := make([]string, 0, len(keys))
+	for _, key := range keys {
+		key = strings.TrimSpace(key)
+		if key != "" && !seen[key] {
+			seen[key] = true
+			unique = append(unique, key)
+		}
+	}
+	return unique
+}
+
+// Retrying with another key is appropriate for auth, quota, rate-limit, and
+// server/transport failures. Request validation errors or malformed successful
+// responses won't improve with another credential and should not be retried.
+func retryWithNextSearchKey(err error) bool {
+	var status *httpStatusError
+	if errors.As(err, &status) {
+		code := status.Code
+		// Brave Search rejects invalid subscription tokens with HTTP 422,
+		// rather than 401. Only retry that exact provider error code: a
+		// different 422 can indicate a bad query that no key can fix.
+		if code == 422 && strings.Contains(status.Message, "SUBSCRIPTION_TOKEN_INVALID") {
+			return true
+		}
+		return code == 401 || code == 402 || code == 403 || code == 408 || code == 429 || code == 432 || code >= 500
+	}
+	var transport *url.Error
+	return errors.As(err, &transport)
+}
+
+func redactSearchError(err error, keys []string) error {
+	if err == nil {
+		return nil
+	}
+	message := err.Error()
+	for _, key := range keys {
+		if key != "" {
+			message = strings.ReplaceAll(message, key, "[REDACTED]")
+		}
+	}
+	return errors.New(message)
+}
+
+func useSearchKeys(ctx context.Context, name string, attempt func(string) ([]byte, error)) ([]byte, error) {
+	keys := searchKeysForProvider(name)
+	if len(keys) == 0 {
+		return nil, fmt.Errorf("missing %s_API_KEY", strings.ToUpper(name))
+	}
+	var lastErr error
+	for index, key := range keys {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		body, err := attempt(key)
+		if err == nil {
+			return body, nil
+		}
+		lastErr = redactSearchError(err, keys)
+		if index+1 == len(keys) || !retryWithNextSearchKey(err) {
+			break
+		}
+	}
+	if len(keys) == 1 {
+		return nil, lastErr
+	}
+	return nil, fmt.Errorf("%s: search failed after trying available API keys (up to %d): %w", name, len(keys), lastErr)
+}
+
+var keychainStore = storeKeychainSecret
+
+// Additional keys are stored in one macOS Keychain entry per provider, without
+// exposing secrets in argv, config.json, logs, or CLI output. The security CLI
+// supports a prompt from stdin when -w is the final argument. It prompts for
+// the password twice, even when updating an entry, so feed matching lines.
+func storeKeychainSecret(account, secret string) error {
+	if runtime.GOOS != "darwin" {
+		return errors.New("key management requires macOS Keychain; on other platforms set PROVIDER_API_KEYS in the environment")
+	}
+	cmd := exec.Command("security", "add-generic-password", "-U", "-s", keychainServiceName, "-a", account, "-w")
+	cmd.Stdin = strings.NewReader(secret + "\n" + secret + "\n")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("could not save API keys to macOS Keychain: %w", err)
+	}
+	return nil
+}
+
+func managedKeychainAccount(name string) (string, error) {
+	name = strings.ToLower(strings.TrimSpace(name))
+	if !validSearchProvider(name) {
+		return "", fmt.Errorf("unknown search provider %q (choose brave, tavily, or exa)", name)
+	}
+	return strings.ToUpper(name) + "_API_KEYS", nil
+}
+
+func readManagedKeys(name string) ([]string, error) {
+	account, err := managedKeychainAccount(name)
+	if err != nil {
+		return nil, err
+	}
+	raw, err := keychainLookup(account)
+	if err != nil {
+		return nil, fmt.Errorf("could not read API keys from macOS Keychain: %w", err)
+	}
+	if raw == "" {
+		return nil, nil
+	}
+	var keys []string
+	if err := json.Unmarshal([]byte(raw), &keys); err != nil {
+		return nil, fmt.Errorf("the stored %s key list is not valid JSON; no keys were changed", name)
+	}
+	return uniqueSearchKeys(keys), nil
+}
+
+func writeManagedKeys(name string, keys []string) error {
+	account, err := managedKeychainAccount(name)
+	if err != nil {
+		return err
+	}
+	serialized, err := json.Marshal(uniqueSearchKeys(keys))
+	if err != nil {
+		return err
+	}
+	return keychainStore(account, string(serialized))
+}
+
+func addManagedSearchKey(name string, from io.Reader) (int, error) {
+	if _, err := managedKeychainAccount(name); err != nil {
+		return 0, err
+	}
+	input, err := io.ReadAll(io.LimitReader(from, 4097))
+	if err != nil {
+		return 0, fmt.Errorf("read API key from stdin: %w", err)
+	}
+	key := strings.TrimSpace(string(input))
+	if key == "" || len(input) > 4096 || strings.ContainsAny(key, "\r\n\t ") {
+		return 0, errors.New("API key must be a single nonempty token no longer than 4096 bytes")
+	}
+	keys, err := readManagedKeys(name)
+	if err != nil {
+		return 0, err
+	}
+	if slices.Contains(keys, key) {
+		return len(keys), nil
+	}
+	// Newest keys take priority, so an old exhausted legacy key is never
+	// attempted before a recently added working key.
+	keys = append([]string{key}, keys...)
+	if err := writeManagedKeys(name, keys); err != nil {
+		return 0, err
+	}
+	return len(keys), nil
+}
+
+func removeManagedSearchKey(name, rawIndex string) (int, error) {
+	keys, err := readManagedKeys(name)
+	if err != nil {
+		return 0, err
+	}
+	index, err := strconv.Atoi(rawIndex)
+	if err != nil || index < 1 || index > len(keys) {
+		return 0, fmt.Errorf("key index must be between 1 and %d (managed Keychain keys only)", len(keys))
+	}
+	keys = slices.Delete(keys, index-1, index)
+	if err := writeManagedKeys(name, keys); err != nil {
+		return 0, err
+	}
+	return len(keys), nil
+}
+
+func printManagedSearchKeySlots(out io.Writer) error {
+	_, _ = fmt.Fprintln(out, "\nManaged macOS Keychain keys (numbered newest-first; indexes start at 1):")
+	for _, provider := range searchProviderKeys {
+		keys, err := readManagedKeys(provider.name)
+		if err != nil {
+			return err
+		}
+		_, _ = fmt.Fprintf(out, "  %-7s %d managed\n", provider.name, len(keys))
+	}
+	return nil
+}