webctx 0.2.9 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -0
- package/README.md +12 -1
- package/internal/app/app.go +38 -0
- package/internal/app/app_test.go +6 -0
- package/internal/app/providers.go +5 -1
- package/internal/app/providers_test.go +3 -0
- package/internal/app/scrape.go +1 -1
- package/internal/app/search_keys.go +233 -0
- package/internal/app/search_keys_test.go +227 -0
- package/internal/app/tools.go +30 -29
- package/package.json +1 -1
package/AGENTS.md
CHANGED
|
@@ -48,6 +48,7 @@ Preserve these commands unless the user explicitly asks to change them:
|
|
|
48
48
|
|
|
49
49
|
- `webctx search <query> [--exclude domain1,domain2] [--keyword phrase]`
|
|
50
50
|
- `webctx status` and `webctx providers enable|disable <brave|tavily|exa>`
|
|
51
|
+
- `webctx keys list|add|remove` for macOS Keychain-managed additional search keys; `BRAVE_API_KEYS`, `TAVILY_API_KEYS`, and `EXA_API_KEYS` also work via environment variables
|
|
51
52
|
- `webctx read-link <url>`
|
|
52
53
|
- `webctx map-site <url>`
|
|
53
54
|
- `webctx --version`
|
|
@@ -55,6 +56,7 @@ Preserve these commands unless the user explicitly asks to change them:
|
|
|
55
56
|
Behavioral expectations:
|
|
56
57
|
|
|
57
58
|
- `search` combines enabled Brave, Tavily, and Exa results, then re-ranks them with duplicate-aware scoring; an unavailable Exa can fall back to quoted-phrase Brave/Tavily queries for `--keyword` (not a strict text filter).
|
|
59
|
+
- Search key rotation tries plural credentials first, legacy singular credentials afterward; failover applies only to credential/quota/rate/server/transport failures, never bad requests. Keep Firecrawl paid operations outside this rotation.
|
|
58
60
|
- `read-link` keeps the current GitHub raw-content and `.md` fast paths. Firecrawl uses its built-in 30-minute cache, automatic and enhanced proxies, then browser extraction as a final fallback.
|
|
59
61
|
- `map-site` keeps the current Firecrawl map request settings.
|
|
60
62
|
- The CLI should remain agent-friendly and emit plain markdown/text output.
|
package/README.md
CHANGED
|
@@ -115,6 +115,17 @@ webctx providers enable brave
|
|
|
115
115
|
|
|
116
116
|
Provider settings persist across runs, independently of environment variables and Keychain credentials. The settings file lives in the OS user config directory (`webctx/config.json`); set `WEBCTX_CONFIG_FILE` to override its location. Disabled providers are never queried. `status` only reports key presence, **not** live credit balances or API health.
|
|
117
117
|
|
|
118
|
+
Multiple keys per provider are supported. Add extra keys on macOS without placing secrets in command arguments or plaintext config files:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
webctx keys list # count configured and managed keys (never prints them)
|
|
122
|
+
pbpaste | webctx keys add exa --stdin # securely store a copied new key in macOS Keychain
|
|
123
|
+
webctx keys remove exa 1 # remove the newest Keychain-managed Exa key
|
|
124
|
+
webctx providers enable exa
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Or supply `BRAVE_API_KEYS`, `TAVILY_API_KEYS`, or `EXA_API_KEYS` as comma-separated keys (or JSON arrays) in the environment or `.env.local`, alongside the corresponding singular `*_API_KEY`. Keys in the plural list are attempted first, then the legacy single key. On a key-specific authentication, quota/credit, rate-limit, or server error, the next key is tried automatically; invalid requests are not retried. Manage only keys/accounts you are authorized to use and honor each provider's usage restrictions. The Firecrawl/Alexandria integration is separate and **is not included in automatic key rotation**, especially for paid operations.
|
|
128
|
+
|
|
118
129
|
`--keyword` prefers Exa's strict include-text filter when available. If Exa is disabled, unconfigured, or fails, Brave/Tavily search for the quoted phrase instead; the output explicitly notes that this fallback cannot guarantee the phrase appears in the page text. Firecrawl is a separate reading/site-map/Alexandria integration, not one of these three search providers.
|
|
119
130
|
|
|
120
131
|
## Map a site
|
|
@@ -145,7 +156,7 @@ FIRECRAWL_API_KEY=...
|
|
|
145
156
|
GH_TOKEN=...
|
|
146
157
|
```
|
|
147
158
|
|
|
148
|
-
- `BRAVE_API_KEY`, `TAVILY_API_KEY`, and `EXA_API_KEY` power normal `search`. Configure any one or more, and optionally disable providers you no longer want to query.
|
|
159
|
+
- `BRAVE_API_KEY`, `TAVILY_API_KEY`, and `EXA_API_KEY` power normal `search`. The plural `*_API_KEYS` names supply additional fallback credentials. Configure any one or more, and optionally disable providers you no longer want to query.
|
|
149
160
|
- `FIRECRAWL_API_KEY` powers `map-site`, page-crawl fallbacks, and Alexandria discovery/inspection/paid execution.
|
|
150
161
|
- `GH_TOKEN` or `GITHUB_TOKEN` is optional. It increases GitHub capacity and unlocks reads such as blame, Discussions, some Actions job logs, private resources the token can access, and richer PR review state.
|
|
151
162
|
|
package/internal/app/app.go
CHANGED
|
@@ -27,6 +27,40 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
tool := args[0]
|
|
30
|
+
if tool == "keys" {
|
|
31
|
+
if len(args) == 2 && args[1] == "list" {
|
|
32
|
+
loadEnvLocal()
|
|
33
|
+
if err := printProviderStatus(stdout); err != nil {
|
|
34
|
+
_, _ = fmt.Fprintln(stderr, err)
|
|
35
|
+
return 1
|
|
36
|
+
}
|
|
37
|
+
if err := printManagedSearchKeySlots(stdout); err != nil {
|
|
38
|
+
_, _ = fmt.Fprintln(stderr, err)
|
|
39
|
+
return 1
|
|
40
|
+
}
|
|
41
|
+
return 0
|
|
42
|
+
}
|
|
43
|
+
if len(args) == 4 && args[1] == "add" && args[3] == "--stdin" {
|
|
44
|
+
count, err := addManagedSearchKey(args[2], os.Stdin)
|
|
45
|
+
if err != nil {
|
|
46
|
+
_, _ = fmt.Fprintln(stderr, err)
|
|
47
|
+
return 1
|
|
48
|
+
}
|
|
49
|
+
_, _ = fmt.Fprintf(stdout, "Saved an additional %s API key in macOS Keychain (%d managed key(s)).\n", strings.ToLower(args[2]), count)
|
|
50
|
+
return 0
|
|
51
|
+
}
|
|
52
|
+
if len(args) == 4 && args[1] == "remove" {
|
|
53
|
+
count, err := removeManagedSearchKey(args[2], args[3])
|
|
54
|
+
if err != nil {
|
|
55
|
+
_, _ = fmt.Fprintln(stderr, err)
|
|
56
|
+
return 1
|
|
57
|
+
}
|
|
58
|
+
_, _ = fmt.Fprintf(stdout, "Removed managed %s API key (%d managed key(s) remain).\n", strings.ToLower(args[2]), count)
|
|
59
|
+
return 0
|
|
60
|
+
}
|
|
61
|
+
_, _ = fmt.Fprintln(stderr, "Usage: webctx keys list | keys add <brave|tavily|exa> --stdin | keys remove <provider> <managed-key-index>")
|
|
62
|
+
return 1
|
|
63
|
+
}
|
|
30
64
|
isProviderTool := tool == "status" || tool == "providers" || tool == "provider"
|
|
31
65
|
isProviderStatus := (tool == "status" && len(args) == 1) ||
|
|
32
66
|
((tool == "providers" || tool == "provider") && (len(args) == 1 || (len(args) == 2 && args[1] == "status")))
|
|
@@ -228,6 +262,9 @@ Usage:
|
|
|
228
262
|
webctx search <query> [--exclude domain1,domain2] [--keyword phrase]
|
|
229
263
|
webctx status
|
|
230
264
|
webctx providers [status|enable <brave|tavily|exa>|disable <brave|tavily|exa>]
|
|
265
|
+
webctx keys list
|
|
266
|
+
webctx keys add <brave|tavily|exa> --stdin
|
|
267
|
+
webctx keys remove <brave|tavily|exa> <managed-key-index>
|
|
231
268
|
webctx read-link <url> [--question 'what to find']
|
|
232
269
|
webctx map-site <url> [--query topic] [--limit 30] [--offset 0] [--path-prefix /api] [--language en|all]
|
|
233
270
|
webctx research [question] [--view sources|groups|operations] [--sources IDs] [--include inputs,output,examples] [--limit N] [--offset N]
|
|
@@ -241,6 +278,7 @@ Examples:
|
|
|
241
278
|
webctx search "drizzle orm" --keyword "migration guide"
|
|
242
279
|
webctx providers disable exa
|
|
243
280
|
webctx providers enable exa
|
|
281
|
+
webctx keys add exa --stdin < /path/to/private/key.txt
|
|
244
282
|
webctx research "podcast conversations about AI agents"
|
|
245
283
|
webctx research --view sources --limit 5
|
|
246
284
|
webctx inspect particle/podcasts/episodes/search
|
package/internal/app/app_test.go
CHANGED
|
@@ -88,6 +88,9 @@ func TestSearchMissingCredentialsErrorIsHelpful(t *testing.T) {
|
|
|
88
88
|
t.Setenv("BRAVE_API_KEY", "")
|
|
89
89
|
t.Setenv("TAVILY_API_KEY", "")
|
|
90
90
|
t.Setenv("EXA_API_KEY", "")
|
|
91
|
+
t.Setenv("BRAVE_API_KEYS", "")
|
|
92
|
+
t.Setenv("TAVILY_API_KEYS", "")
|
|
93
|
+
t.Setenv("EXA_API_KEYS", "")
|
|
91
94
|
|
|
92
95
|
text, err := Search(SearchParams{Query: "openai api"})
|
|
93
96
|
if err == nil {
|
|
@@ -180,6 +183,9 @@ func TestLoadKeychainEnvLoadsMissingOnly(t *testing.T) {
|
|
|
180
183
|
t.Setenv("BRAVE_API_KEY", "already-set")
|
|
181
184
|
t.Setenv("TAVILY_API_KEY", "")
|
|
182
185
|
t.Setenv("EXA_API_KEY", "")
|
|
186
|
+
t.Setenv("BRAVE_API_KEYS", "")
|
|
187
|
+
t.Setenv("TAVILY_API_KEYS", "")
|
|
188
|
+
t.Setenv("EXA_API_KEYS", "")
|
|
183
189
|
t.Setenv("FIRECRAWL_API_KEY", "")
|
|
184
190
|
t.Setenv("GH_TOKEN", "")
|
|
185
191
|
t.Setenv("GITHUB_TOKEN", "")
|
|
@@ -145,8 +145,12 @@ func printProviderStatus(out io.Writer) error {
|
|
|
145
145
|
if !config.enabled(p.name) {
|
|
146
146
|
state = "disabled"
|
|
147
147
|
}
|
|
148
|
-
|
|
148
|
+
count := len(searchKeysForProvider(p.name))
|
|
149
|
+
if count > 0 {
|
|
149
150
|
key = "key configured"
|
|
151
|
+
if count > 1 {
|
|
152
|
+
key = fmt.Sprintf("key configured (%d keys)", count)
|
|
153
|
+
}
|
|
150
154
|
}
|
|
151
155
|
_, _ = fmt.Fprintf(out, " %-7s %-9s %s\n", p.name, state, key)
|
|
152
156
|
}
|
|
@@ -152,6 +152,9 @@ func searchTestEnv(t *testing.T) {
|
|
|
152
152
|
t.Setenv("BRAVE_API_KEY", "test-brave")
|
|
153
153
|
t.Setenv("TAVILY_API_KEY", "test-tavily")
|
|
154
154
|
t.Setenv("EXA_API_KEY", "test-exa")
|
|
155
|
+
t.Setenv("BRAVE_API_KEYS", "")
|
|
156
|
+
t.Setenv("TAVILY_API_KEYS", "")
|
|
157
|
+
t.Setenv("EXA_API_KEYS", "")
|
|
155
158
|
t.Setenv("FIRECRAWL_API_KEY", "")
|
|
156
159
|
}
|
|
157
160
|
|
package/internal/app/scrape.go
CHANGED
|
@@ -21,7 +21,7 @@ type MarkdownResult struct {
|
|
|
21
21
|
const keychainServiceName = "webctx"
|
|
22
22
|
|
|
23
23
|
var (
|
|
24
|
-
credentialEnvKeys = []string{"BRAVE_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "FIRECRAWL_API_KEY", "GH_TOKEN", "GITHUB_TOKEN"}
|
|
24
|
+
credentialEnvKeys = []string{"BRAVE_API_KEY", "BRAVE_API_KEYS", "TAVILY_API_KEY", "TAVILY_API_KEYS", "EXA_API_KEY", "EXA_API_KEYS", "FIRECRAWL_API_KEY", "GH_TOKEN", "GITHUB_TOKEN"}
|
|
25
25
|
getwdFunc = os.Getwd
|
|
26
26
|
executableFunc = os.Executable
|
|
27
27
|
keychainLookup = lookupKeychainSecret
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
package app
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"encoding/json"
|
|
6
|
+
"errors"
|
|
7
|
+
"fmt"
|
|
8
|
+
"io"
|
|
9
|
+
"net/url"
|
|
10
|
+
"os"
|
|
11
|
+
"os/exec"
|
|
12
|
+
"runtime"
|
|
13
|
+
"slices"
|
|
14
|
+
"strconv"
|
|
15
|
+
"strings"
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
// A provider's legacy singular key remains supported. Additional keys are
|
|
19
|
+
// checked first so newly provisioned keys take precedence over exhausted ones.
|
|
20
|
+
// Plural values can be comma/newline-separated or a JSON string array (the
|
|
21
|
+
// latter is how the macOS Keychain stores them).
|
|
22
|
+
func searchKeysForProvider(name string) []string {
|
|
23
|
+
name = strings.ToLower(strings.TrimSpace(name))
|
|
24
|
+
if !validSearchProvider(name) {
|
|
25
|
+
return nil
|
|
26
|
+
}
|
|
27
|
+
prefix := strings.ToUpper(name) + "_API_KEY"
|
|
28
|
+
keys := parseSearchKeyList(os.Getenv(prefix + "S"))
|
|
29
|
+
if key := strings.TrimSpace(os.Getenv(prefix)); key != "" {
|
|
30
|
+
keys = append(keys, key)
|
|
31
|
+
}
|
|
32
|
+
return uniqueSearchKeys(keys)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
func parseSearchKeyList(raw string) []string {
|
|
36
|
+
raw = strings.TrimSpace(raw)
|
|
37
|
+
if raw == "" {
|
|
38
|
+
return nil
|
|
39
|
+
}
|
|
40
|
+
if strings.HasPrefix(raw, "[") {
|
|
41
|
+
var keys []string
|
|
42
|
+
if json.Unmarshal([]byte(raw), &keys) != nil {
|
|
43
|
+
return nil // fail closed rather than sending malformed credential data
|
|
44
|
+
}
|
|
45
|
+
return uniqueSearchKeys(keys)
|
|
46
|
+
}
|
|
47
|
+
return uniqueSearchKeys(strings.FieldsFunc(raw, func(r rune) bool {
|
|
48
|
+
return r == ',' || r == '\n' || r == '\r'
|
|
49
|
+
}))
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
func uniqueSearchKeys(keys []string) []string {
|
|
53
|
+
seen := make(map[string]bool, len(keys))
|
|
54
|
+
unique := make([]string, 0, len(keys))
|
|
55
|
+
for _, key := range keys {
|
|
56
|
+
key = strings.TrimSpace(key)
|
|
57
|
+
if key != "" && !seen[key] {
|
|
58
|
+
seen[key] = true
|
|
59
|
+
unique = append(unique, key)
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return unique
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Retrying with another key is appropriate for auth, quota, rate-limit, and
|
|
66
|
+
// server/transport failures. Request validation errors or malformed successful
|
|
67
|
+
// responses won't improve with another credential and should not be retried.
|
|
68
|
+
func retryWithNextSearchKey(err error) bool {
|
|
69
|
+
var status *httpStatusError
|
|
70
|
+
if errors.As(err, &status) {
|
|
71
|
+
code := status.Code
|
|
72
|
+
// Brave Search rejects invalid subscription tokens with HTTP 422,
|
|
73
|
+
// rather than 401. Only retry that exact provider error code: a
|
|
74
|
+
// different 422 can indicate a bad query that no key can fix.
|
|
75
|
+
if code == 422 && strings.Contains(status.Message, "SUBSCRIPTION_TOKEN_INVALID") {
|
|
76
|
+
return true
|
|
77
|
+
}
|
|
78
|
+
return code == 401 || code == 402 || code == 403 || code == 408 || code == 429 || code == 432 || code >= 500
|
|
79
|
+
}
|
|
80
|
+
var transport *url.Error
|
|
81
|
+
return errors.As(err, &transport)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
func redactSearchError(err error, keys []string) error {
|
|
85
|
+
if err == nil {
|
|
86
|
+
return nil
|
|
87
|
+
}
|
|
88
|
+
message := err.Error()
|
|
89
|
+
for _, key := range keys {
|
|
90
|
+
if key != "" {
|
|
91
|
+
message = strings.ReplaceAll(message, key, "[REDACTED]")
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return errors.New(message)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
func useSearchKeys(ctx context.Context, name string, attempt func(string) ([]byte, error)) ([]byte, error) {
|
|
98
|
+
keys := searchKeysForProvider(name)
|
|
99
|
+
if len(keys) == 0 {
|
|
100
|
+
return nil, fmt.Errorf("missing %s_API_KEY", strings.ToUpper(name))
|
|
101
|
+
}
|
|
102
|
+
var lastErr error
|
|
103
|
+
for index, key := range keys {
|
|
104
|
+
if ctx.Err() != nil {
|
|
105
|
+
return nil, ctx.Err()
|
|
106
|
+
}
|
|
107
|
+
body, err := attempt(key)
|
|
108
|
+
if err == nil {
|
|
109
|
+
return body, nil
|
|
110
|
+
}
|
|
111
|
+
lastErr = redactSearchError(err, keys)
|
|
112
|
+
if index+1 == len(keys) || !retryWithNextSearchKey(err) {
|
|
113
|
+
break
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if len(keys) == 1 {
|
|
117
|
+
return nil, lastErr
|
|
118
|
+
}
|
|
119
|
+
return nil, fmt.Errorf("%s: search failed after trying available API keys (up to %d): %w", name, len(keys), lastErr)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
var keychainStore = storeKeychainSecret
|
|
123
|
+
|
|
124
|
+
// Additional keys are stored in one macOS Keychain entry per provider, without
|
|
125
|
+
// exposing secrets in argv, config.json, logs, or CLI output. The security CLI
|
|
126
|
+
// supports a prompt from stdin when -w is the final argument. It prompts for
|
|
127
|
+
// the password twice, even when updating an entry, so feed matching lines.
|
|
128
|
+
func storeKeychainSecret(account, secret string) error {
|
|
129
|
+
if runtime.GOOS != "darwin" {
|
|
130
|
+
return errors.New("key management requires macOS Keychain; on other platforms set PROVIDER_API_KEYS in the environment")
|
|
131
|
+
}
|
|
132
|
+
cmd := exec.Command("security", "add-generic-password", "-U", "-s", keychainServiceName, "-a", account, "-w")
|
|
133
|
+
cmd.Stdin = strings.NewReader(secret + "\n" + secret + "\n")
|
|
134
|
+
if err := cmd.Run(); err != nil {
|
|
135
|
+
return fmt.Errorf("could not save API keys to macOS Keychain: %w", err)
|
|
136
|
+
}
|
|
137
|
+
return nil
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
func managedKeychainAccount(name string) (string, error) {
|
|
141
|
+
name = strings.ToLower(strings.TrimSpace(name))
|
|
142
|
+
if !validSearchProvider(name) {
|
|
143
|
+
return "", fmt.Errorf("unknown search provider %q (choose brave, tavily, or exa)", name)
|
|
144
|
+
}
|
|
145
|
+
return strings.ToUpper(name) + "_API_KEYS", nil
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
func readManagedKeys(name string) ([]string, error) {
|
|
149
|
+
account, err := managedKeychainAccount(name)
|
|
150
|
+
if err != nil {
|
|
151
|
+
return nil, err
|
|
152
|
+
}
|
|
153
|
+
raw, err := keychainLookup(account)
|
|
154
|
+
if err != nil {
|
|
155
|
+
return nil, fmt.Errorf("could not read API keys from macOS Keychain: %w", err)
|
|
156
|
+
}
|
|
157
|
+
if raw == "" {
|
|
158
|
+
return nil, nil
|
|
159
|
+
}
|
|
160
|
+
var keys []string
|
|
161
|
+
if err := json.Unmarshal([]byte(raw), &keys); err != nil {
|
|
162
|
+
return nil, fmt.Errorf("the stored %s key list is not valid JSON; no keys were changed", name)
|
|
163
|
+
}
|
|
164
|
+
return uniqueSearchKeys(keys), nil
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
func writeManagedKeys(name string, keys []string) error {
|
|
168
|
+
account, err := managedKeychainAccount(name)
|
|
169
|
+
if err != nil {
|
|
170
|
+
return err
|
|
171
|
+
}
|
|
172
|
+
serialized, err := json.Marshal(uniqueSearchKeys(keys))
|
|
173
|
+
if err != nil {
|
|
174
|
+
return err
|
|
175
|
+
}
|
|
176
|
+
return keychainStore(account, string(serialized))
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
func addManagedSearchKey(name string, from io.Reader) (int, error) {
|
|
180
|
+
if _, err := managedKeychainAccount(name); err != nil {
|
|
181
|
+
return 0, err
|
|
182
|
+
}
|
|
183
|
+
input, err := io.ReadAll(io.LimitReader(from, 4097))
|
|
184
|
+
if err != nil {
|
|
185
|
+
return 0, fmt.Errorf("read API key from stdin: %w", err)
|
|
186
|
+
}
|
|
187
|
+
key := strings.TrimSpace(string(input))
|
|
188
|
+
if key == "" || len(input) > 4096 || strings.ContainsAny(key, "\r\n\t ") {
|
|
189
|
+
return 0, errors.New("API key must be a single nonempty token no longer than 4096 bytes")
|
|
190
|
+
}
|
|
191
|
+
keys, err := readManagedKeys(name)
|
|
192
|
+
if err != nil {
|
|
193
|
+
return 0, err
|
|
194
|
+
}
|
|
195
|
+
if slices.Contains(keys, key) {
|
|
196
|
+
return len(keys), nil
|
|
197
|
+
}
|
|
198
|
+
// Newest keys take priority, so an old exhausted legacy key is never
|
|
199
|
+
// attempted before a recently added working key.
|
|
200
|
+
keys = append([]string{key}, keys...)
|
|
201
|
+
if err := writeManagedKeys(name, keys); err != nil {
|
|
202
|
+
return 0, err
|
|
203
|
+
}
|
|
204
|
+
return len(keys), nil
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
func removeManagedSearchKey(name, rawIndex string) (int, error) {
|
|
208
|
+
keys, err := readManagedKeys(name)
|
|
209
|
+
if err != nil {
|
|
210
|
+
return 0, err
|
|
211
|
+
}
|
|
212
|
+
index, err := strconv.Atoi(rawIndex)
|
|
213
|
+
if err != nil || index < 1 || index > len(keys) {
|
|
214
|
+
return 0, fmt.Errorf("key index must be between 1 and %d (managed Keychain keys only)", len(keys))
|
|
215
|
+
}
|
|
216
|
+
keys = slices.Delete(keys, index-1, index)
|
|
217
|
+
if err := writeManagedKeys(name, keys); err != nil {
|
|
218
|
+
return 0, err
|
|
219
|
+
}
|
|
220
|
+
return len(keys), nil
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
func printManagedSearchKeySlots(out io.Writer) error {
|
|
224
|
+
_, _ = fmt.Fprintln(out, "\nManaged macOS Keychain keys (numbered newest-first; indexes start at 1):")
|
|
225
|
+
for _, provider := range searchProviderKeys {
|
|
226
|
+
keys, err := readManagedKeys(provider.name)
|
|
227
|
+
if err != nil {
|
|
228
|
+
return err
|
|
229
|
+
}
|
|
230
|
+
_, _ = fmt.Fprintf(out, " %-7s %d managed\n", provider.name, len(keys))
|
|
231
|
+
}
|
|
232
|
+
return nil
|
|
233
|
+
}
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
package app
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"bytes"
|
|
5
|
+
"context"
|
|
6
|
+
"encoding/json"
|
|
7
|
+
"fmt"
|
|
8
|
+
"io"
|
|
9
|
+
"net/http"
|
|
10
|
+
"os"
|
|
11
|
+
"path/filepath"
|
|
12
|
+
"reflect"
|
|
13
|
+
"strings"
|
|
14
|
+
"testing"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
func TestSearchKeyOrderDeduplicationAndMissingKeys(t *testing.T) {
|
|
18
|
+
t.Setenv("EXA_API_KEYS", ` ["new","new","old"] `)
|
|
19
|
+
t.Setenv("EXA_API_KEY", "old")
|
|
20
|
+
if got, want := searchKeysForProvider("exa"), []string{"new", "old"}; !reflect.DeepEqual(got, want) {
|
|
21
|
+
t.Fatalf("deduplication: got %v, want %v", got, want)
|
|
22
|
+
}
|
|
23
|
+
t.Setenv("EXA_API_KEYS", "new,another\n old\n")
|
|
24
|
+
if got, want := searchKeysForProvider("exa"), []string{"new", "another", "old"}; !reflect.DeepEqual(got, want) {
|
|
25
|
+
t.Fatalf("CSV/newline parsing: got %v, want %v", got, want)
|
|
26
|
+
}
|
|
27
|
+
t.Setenv("EXA_API_KEYS", `[invalid`)
|
|
28
|
+
t.Setenv("EXA_API_KEY", "")
|
|
29
|
+
if got := searchKeysForProvider("exa"); len(got) != 0 {
|
|
30
|
+
t.Fatalf("malformed JSON should not be used as credential: %v", got)
|
|
31
|
+
}
|
|
32
|
+
if got := searchKeysForProvider("unknown"); len(got) != 0 {
|
|
33
|
+
t.Fatalf("unknown provider should have no keys: %v", got)
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
func TestStatusReportsMultipleKeysWithoutRevealingThem(t *testing.T) {
|
|
38
|
+
t.Setenv("WEBCTX_CONFIG_FILE", filepath.Join(t.TempDir(), "webctx", "config.json"))
|
|
39
|
+
t.Setenv("EXA_API_KEYS", "extra-secret-a,extra-secret-b")
|
|
40
|
+
t.Setenv("EXA_API_KEY", "legacy-secret")
|
|
41
|
+
t.Setenv("BRAVE_API_KEY", "")
|
|
42
|
+
t.Setenv("BRAVE_API_KEYS", "")
|
|
43
|
+
t.Setenv("TAVILY_API_KEY", "")
|
|
44
|
+
t.Setenv("TAVILY_API_KEYS", "")
|
|
45
|
+
t.Setenv("FIRECRAWL_API_KEY", "")
|
|
46
|
+
var out, errOut bytes.Buffer
|
|
47
|
+
if code := Run([]string{"status"}, &out, &errOut); code != 0 {
|
|
48
|
+
t.Fatalf("status error: %s", errOut.String())
|
|
49
|
+
}
|
|
50
|
+
if !strings.Contains(out.String(), "exa enabled key configured (3 keys)") {
|
|
51
|
+
t.Fatalf("incorrect key count: %q", out.String())
|
|
52
|
+
}
|
|
53
|
+
for _, secret := range []string{"extra-secret-a", "extra-secret-b", "legacy-secret"} {
|
|
54
|
+
if strings.Contains(out.String()+errOut.String(), secret) {
|
|
55
|
+
t.Fatal("status exposed a configured API key")
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
func TestManagedSearchKeychainAddRemoveAndInputValidation(t *testing.T) {
|
|
61
|
+
oldLookup, oldStore := keychainLookup, keychainStore
|
|
62
|
+
t.Cleanup(func() { keychainLookup, keychainStore = oldLookup, oldStore })
|
|
63
|
+
managed := map[string]string{}
|
|
64
|
+
keychainLookup = func(account string) (string, error) { return managed[account], nil }
|
|
65
|
+
keychainStore = func(account, secret string) error { managed[account] = secret; return nil }
|
|
66
|
+
|
|
67
|
+
for _, candidate := range []string{"", "one two", "line1\nline2", strings.Repeat("z", 4097)} {
|
|
68
|
+
if _, err := addManagedSearchKey("exa", strings.NewReader(candidate)); err == nil {
|
|
69
|
+
t.Fatalf("invalid secret accepted")
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if len(managed) != 0 {
|
|
73
|
+
t.Fatal("invalid keys must not modify Keychain")
|
|
74
|
+
}
|
|
75
|
+
if _, err := addManagedSearchKey("unknown", strings.NewReader("token")); err == nil {
|
|
76
|
+
t.Fatal("unknown provider accepted")
|
|
77
|
+
}
|
|
78
|
+
for _, pair := range []struct {
|
|
79
|
+
key string
|
|
80
|
+
count int
|
|
81
|
+
}{{"older", 1}, {"newer", 2}, {"newer", 2}} {
|
|
82
|
+
count, err := addManagedSearchKey("exa", strings.NewReader(pair.key+"\n"))
|
|
83
|
+
if err != nil || count != pair.count {
|
|
84
|
+
t.Fatalf("add: count=%d error=%v", count, err)
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
list, err := readManagedKeys("exa")
|
|
88
|
+
if err != nil || !reflect.DeepEqual(list, []string{"newer", "older"}) {
|
|
89
|
+
t.Fatalf("priority order changed: %v %v", list, err)
|
|
90
|
+
}
|
|
91
|
+
if strings.Contains(managed["EXA_API_KEYS"], "[REDACTED]") {
|
|
92
|
+
t.Fatal("managed keys stored as placeholder instead of actual tokens")
|
|
93
|
+
}
|
|
94
|
+
if _, err := removeManagedSearchKey("exa", "3"); err == nil {
|
|
95
|
+
t.Fatal("out-of-range removal accepted")
|
|
96
|
+
}
|
|
97
|
+
if count, err := removeManagedSearchKey("exa", "1"); err != nil || count != 1 {
|
|
98
|
+
t.Fatalf("remove: count=%d error=%v", count, err)
|
|
99
|
+
}
|
|
100
|
+
if list, err := readManagedKeys("exa"); err != nil || !reflect.DeepEqual(list, []string{"older"}) {
|
|
101
|
+
t.Fatalf("wrong managed key removed: %v %v", list, err)
|
|
102
|
+
}
|
|
103
|
+
managed["EXA_API_KEYS"] = "{invalid"
|
|
104
|
+
if _, err := addManagedSearchKey("exa", strings.NewReader("another")); err == nil {
|
|
105
|
+
t.Fatal("malformed Keychain JSON silently overwritten")
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
func TestManagedSearchKeyCommandNeverPrintsSecrets(t *testing.T) {
|
|
110
|
+
oldLookup, oldStore, oldStdin := keychainLookup, keychainStore, os.Stdin
|
|
111
|
+
t.Cleanup(func() { keychainLookup, keychainStore, os.Stdin = oldLookup, oldStore, oldStdin })
|
|
112
|
+
keychainLookup = func(account string) (string, error) { return "", nil }
|
|
113
|
+
keychainStore = func(account, value string) error { return nil }
|
|
114
|
+
input, writer, err := os.Pipe()
|
|
115
|
+
if err != nil {
|
|
116
|
+
t.Fatal(err)
|
|
117
|
+
}
|
|
118
|
+
if _, err := io.WriteString(writer, "sensitive-api-token\n"); err != nil {
|
|
119
|
+
t.Fatal(err)
|
|
120
|
+
}
|
|
121
|
+
_ = writer.Close()
|
|
122
|
+
os.Stdin = input
|
|
123
|
+
var out, errOut bytes.Buffer
|
|
124
|
+
code := Run([]string{"keys", "add", "exa", "--stdin"}, &out, &errOut)
|
|
125
|
+
_ = input.Close()
|
|
126
|
+
if code != 0 || strings.Contains(out.String()+errOut.String(), "sensitive-api-token") {
|
|
127
|
+
t.Fatalf("unsafe CLI output or command failed: code=%d out=%q err=%q", code, out.String(), errOut.String())
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Exercise the real HTTP request construction and key-selection order for all
|
|
132
|
+
// three providers, without making network requests or disclosing API keys.
|
|
133
|
+
func TestSearchProviderFallbackAcrossKeys(t *testing.T) {
|
|
134
|
+
t.Setenv("BRAVE_API_KEY", "legacy-brave")
|
|
135
|
+
t.Setenv("TAVILY_API_KEY", "legacy-tavily")
|
|
136
|
+
t.Setenv("EXA_API_KEY", "legacy-exa")
|
|
137
|
+
for _, name := range []string{"BRAVE", "TAVILY", "EXA"} {
|
|
138
|
+
t.Setenv(name+"_API_KEYS", "bad-"+strings.ToLower(name)+",good-"+strings.ToLower(name))
|
|
139
|
+
}
|
|
140
|
+
previous := http.DefaultClient.Transport
|
|
141
|
+
requests := map[string][]string{}
|
|
142
|
+
http.DefaultClient.Transport = roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
143
|
+
provider, key := "", ""
|
|
144
|
+
switch req.URL.Host {
|
|
145
|
+
case "api.search.brave.com":
|
|
146
|
+
provider, key = "brave", req.Header.Get("x-subscription-token")
|
|
147
|
+
case "api.exa.ai":
|
|
148
|
+
provider, key = "exa", req.Header.Get("x-api-key")
|
|
149
|
+
case "api.tavily.com":
|
|
150
|
+
provider = "tavily"
|
|
151
|
+
var payload map[string]any
|
|
152
|
+
_ = json.NewDecoder(req.Body).Decode(&payload)
|
|
153
|
+
key, _ = payload["api_key"].(string)
|
|
154
|
+
default:
|
|
155
|
+
t.Errorf("unexpected host: %s", req.URL.Host)
|
|
156
|
+
}
|
|
157
|
+
requests[provider] = append(requests[provider], key)
|
|
158
|
+
if strings.HasPrefix(key, "bad-") {
|
|
159
|
+
code := http.StatusUnauthorized
|
|
160
|
+
switch provider {
|
|
161
|
+
case "brave":
|
|
162
|
+
return testHTTPResponse(req, 422, `{"error":{"code":"SUBSCRIPTION_TOKEN_INVALID"}}`, nil), nil
|
|
163
|
+
case "tavily":
|
|
164
|
+
code = 432
|
|
165
|
+
case "exa":
|
|
166
|
+
code = http.StatusPaymentRequired
|
|
167
|
+
}
|
|
168
|
+
return testHTTPResponse(req, code, `{"error":"credits exhausted"}`, nil), nil
|
|
169
|
+
}
|
|
170
|
+
if strings.HasPrefix(key, "good-") {
|
|
171
|
+
switch provider {
|
|
172
|
+
case "brave":
|
|
173
|
+
return testHTTPResponse(req, 200, `{"web":{"results":[{"url":"https://example.com/guide","title":"Brave result"}]}}`, nil), nil
|
|
174
|
+
case "tavily", "exa":
|
|
175
|
+
return testHTTPResponse(req, 200, `{"results":[{"url":"https://example.com/guide","title":"API result"}]}`, nil), nil
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return testHTTPResponse(req, 500, "unexpected key", nil), nil
|
|
179
|
+
})
|
|
180
|
+
t.Cleanup(func() { http.DefaultClient.Transport = previous })
|
|
181
|
+
|
|
182
|
+
for _, test := range []struct {
|
|
183
|
+
provider string
|
|
184
|
+
call func() (SearchResult, error)
|
|
185
|
+
}{
|
|
186
|
+
{"brave", func() (SearchResult, error) { return searchWithBrave(context.Background(), "test query") }},
|
|
187
|
+
{"tavily", func() (SearchResult, error) { return searchWithTavily(context.Background(), "test query", nil) }},
|
|
188
|
+
{"exa", func() (SearchResult, error) {
|
|
189
|
+
return searchWithExa(context.Background(), "test query", nil, "exact phrase")
|
|
190
|
+
}},
|
|
191
|
+
} {
|
|
192
|
+
results, err := test.call()
|
|
193
|
+
if err != nil || len(results.Docs) != 1 {
|
|
194
|
+
t.Fatalf("%s: search failed after key rotation: %v %+v", test.provider, err, results)
|
|
195
|
+
}
|
|
196
|
+
want := []string{"bad-" + test.provider, "good-" + test.provider}
|
|
197
|
+
if got := requests[test.provider]; !reflect.DeepEqual(got, want) {
|
|
198
|
+
t.Fatalf("%s: key rotation order got %v, want %v", test.provider, got, want)
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
func TestSearchKeyRetryDoesNotMaskBadInputOrLeakKeys(t *testing.T) {
|
|
204
|
+
t.Setenv("EXA_API_KEYS", "first-secret,second-secret")
|
|
205
|
+
t.Setenv("EXA_API_KEY", "")
|
|
206
|
+
previous := http.DefaultClient.Transport
|
|
207
|
+
count, code := 0, http.StatusBadRequest
|
|
208
|
+
http.DefaultClient.Transport = roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
209
|
+
count++
|
|
210
|
+
return testHTTPResponse(req, code, fmt.Sprintf(`{"error":"key %s failed"}`, req.Header.Get("x-api-key")), nil), nil
|
|
211
|
+
})
|
|
212
|
+
t.Cleanup(func() { http.DefaultClient.Transport = previous })
|
|
213
|
+
_, err := searchWithExa(context.Background(), "test", nil, "")
|
|
214
|
+
if err == nil || count != 1 || strings.Contains(err.Error(), "first-secret") {
|
|
215
|
+
t.Fatalf("400 should not retry or leak a credential: count=%d error=%v", count, err)
|
|
216
|
+
}
|
|
217
|
+
count, code = 0, http.StatusUnprocessableEntity
|
|
218
|
+
_, err = searchWithExa(context.Background(), "test", nil, "")
|
|
219
|
+
if err == nil || count != 1 {
|
|
220
|
+
t.Fatalf("non-credential 422 should not retry: count=%d error=%v", count, err)
|
|
221
|
+
}
|
|
222
|
+
count, code = 0, http.StatusTooManyRequests
|
|
223
|
+
_, err = searchWithExa(context.Background(), "test", nil, "")
|
|
224
|
+
if err == nil || count != 2 || strings.Contains(err.Error(), "first-secret") || strings.Contains(err.Error(), "second-secret") || !strings.Contains(err.Error(), "up to 2") {
|
|
225
|
+
t.Fatalf("exhausted-key error is unsafe or not actionable: count=%d error=%v", count, err)
|
|
226
|
+
}
|
|
227
|
+
}
|
package/internal/app/tools.go
CHANGED
|
@@ -72,7 +72,7 @@ func Search(params SearchParams) (string, error) {
|
|
|
72
72
|
searches := []namedSearch{}
|
|
73
73
|
keywordFallback := false
|
|
74
74
|
if strings.TrimSpace(truncatedKeyword) != "" {
|
|
75
|
-
if config.enabled("exa") &&
|
|
75
|
+
if config.enabled("exa") && len(searchKeysForProvider("exa")) > 0 {
|
|
76
76
|
searches = append(searches, namedSearch{name: "Exa", fn: func(ctx context.Context) (SearchResult, error) {
|
|
77
77
|
return searchWithExa(ctx, params.Query, nil, truncatedKeyword)
|
|
78
78
|
}})
|
|
@@ -381,19 +381,17 @@ func missingCredentialsError(prefix string, keys []string, detail string) error
|
|
|
381
381
|
}
|
|
382
382
|
|
|
383
383
|
func searchWithBrave(ctx context.Context, query string) (SearchResult, error) {
|
|
384
|
-
apiKey := strings.TrimSpace(os.Getenv("BRAVE_API_KEY"))
|
|
385
|
-
if apiKey == "" {
|
|
386
|
-
return SearchResult{}, errors.New("missing BRAVE_API_KEY")
|
|
387
|
-
}
|
|
388
384
|
params := url.Values{}
|
|
389
385
|
params.Set("q", query)
|
|
390
386
|
params.Set("text_decorations", "false")
|
|
391
387
|
params.Set("result_filter", "web")
|
|
392
388
|
params.Set("limit", "20")
|
|
393
|
-
body, err :=
|
|
394
|
-
"
|
|
395
|
-
|
|
396
|
-
|
|
389
|
+
body, err := useSearchKeys(ctx, "brave", func(apiKey string) ([]byte, error) {
|
|
390
|
+
return doRawRequest(ctx, http.MethodGet, "https://api.search.brave.com/res/v1/web/search?"+params.Encode(), map[string]string{
|
|
391
|
+
"Accept": "application/json",
|
|
392
|
+
"x-subscription-token": apiKey,
|
|
393
|
+
}, nil)
|
|
394
|
+
})
|
|
397
395
|
if err != nil {
|
|
398
396
|
return SearchResult{}, err
|
|
399
397
|
}
|
|
@@ -420,22 +418,20 @@ func searchWithBrave(ctx context.Context, query string) (SearchResult, error) {
|
|
|
420
418
|
}
|
|
421
419
|
|
|
422
420
|
func searchWithTavily(ctx context.Context, query string, excludeDomains []string) (SearchResult, error) {
|
|
423
|
-
apiKey := strings.TrimSpace(os.Getenv("TAVILY_API_KEY"))
|
|
424
|
-
if apiKey == "" {
|
|
425
|
-
return SearchResult{}, errors.New("missing TAVILY_API_KEY")
|
|
426
|
-
}
|
|
427
421
|
requestBody := map[string]any{
|
|
428
|
-
"api_key": apiKey,
|
|
429
422
|
"query": query,
|
|
430
423
|
"max_results": 20,
|
|
431
424
|
}
|
|
432
425
|
if len(excludeDomains) > 0 {
|
|
433
426
|
requestBody["exclude_domains"] = excludeDomains
|
|
434
427
|
}
|
|
435
|
-
body, err :=
|
|
436
|
-
"
|
|
437
|
-
"
|
|
438
|
-
|
|
428
|
+
body, err := useSearchKeys(ctx, "tavily", func(apiKey string) ([]byte, error) {
|
|
429
|
+
requestBody["api_key"] = apiKey
|
|
430
|
+
return doJSONRequest(ctx, http.MethodPost, "https://api.tavily.com/search", map[string]string{
|
|
431
|
+
"Content-Type": "application/json",
|
|
432
|
+
"Accept": "application/json",
|
|
433
|
+
}, requestBody)
|
|
434
|
+
})
|
|
439
435
|
if err != nil {
|
|
440
436
|
return SearchResult{}, err
|
|
441
437
|
}
|
|
@@ -460,10 +456,6 @@ func searchWithTavily(ctx context.Context, query string, excludeDomains []string
|
|
|
460
456
|
}
|
|
461
457
|
|
|
462
458
|
func searchWithExa(ctx context.Context, query string, excludeDomains []string, includeKeyword string) (SearchResult, error) {
|
|
463
|
-
apiKey := strings.TrimSpace(os.Getenv("EXA_API_KEY"))
|
|
464
|
-
if apiKey == "" {
|
|
465
|
-
return SearchResult{}, errors.New("missing EXA_API_KEY")
|
|
466
|
-
}
|
|
467
459
|
requestBody := map[string]any{
|
|
468
460
|
"query": query,
|
|
469
461
|
"type": "auto",
|
|
@@ -478,11 +470,13 @@ func searchWithExa(ctx context.Context, query string, excludeDomains []string, i
|
|
|
478
470
|
if strings.TrimSpace(includeKeyword) != "" {
|
|
479
471
|
requestBody["includeText"] = []string{includeKeyword}
|
|
480
472
|
}
|
|
481
|
-
body, err :=
|
|
482
|
-
"
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
473
|
+
body, err := useSearchKeys(ctx, "exa", func(apiKey string) ([]byte, error) {
|
|
474
|
+
return doJSONRequest(ctx, http.MethodPost, "https://api.exa.ai/search", map[string]string{
|
|
475
|
+
"Accept": "application/json",
|
|
476
|
+
"Content-Type": "application/json",
|
|
477
|
+
"x-api-key": apiKey,
|
|
478
|
+
}, requestBody)
|
|
479
|
+
})
|
|
486
480
|
if err != nil {
|
|
487
481
|
return SearchResult{}, err
|
|
488
482
|
}
|
|
@@ -684,13 +678,20 @@ func doRawRequest(ctx context.Context, method, rawURL string, headers map[string
|
|
|
684
678
|
}
|
|
685
679
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
686
680
|
if len(respBody) > 0 {
|
|
687
|
-
return nil, fmt.
|
|
681
|
+
return nil, &httpStatusError{Code: resp.StatusCode, Message: fmt.Sprintf("API request failed: %s - %s", resp.Status, strings.TrimSpace(string(respBody)))}
|
|
688
682
|
}
|
|
689
|
-
return nil,
|
|
683
|
+
return nil, &httpStatusError{Code: resp.StatusCode, Message: "API request failed: " + resp.Status}
|
|
690
684
|
}
|
|
691
685
|
return respBody, nil
|
|
692
686
|
}
|
|
693
687
|
|
|
688
|
+
type httpStatusError struct {
|
|
689
|
+
Code int
|
|
690
|
+
Message string
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
func (e *httpStatusError) Error() string { return e.Message }
|
|
694
|
+
|
|
694
695
|
func stringValue(v any) string {
|
|
695
696
|
if s, ok := v.(string); ok {
|
|
696
697
|
return s
|