webctx 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/internal/app/app.go +6 -10
- package/internal/app/data_client.go +5 -11
- package/internal/app/data_examples.go +112 -0
- package/internal/app/data_execute.go +35 -24
- package/internal/app/data_inspect.go +2 -4
- package/internal/app/data_research.go +18 -1
- package/internal/app/data_tools_test.go +122 -14
- package/internal/app/data_types.go +3 -4
- package/internal/app/research.go +20 -4
- package/internal/app/research_copyable.go +48 -0
- package/internal/app/research_copyable_test.go +57 -0
- package/package.json +1 -1
- package/pkg/origo/data_test.go +5 -5
- package/pkg/origo/server.go +89 -18
- package/pkg/origo/server_test.go +56 -4
- package/pkg/retrieval/data.go +1 -0
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ webctx read-link <url>
|
|
|
10
10
|
webctx map-site <url>
|
|
11
11
|
webctx research "podcast conversations about AI agents"
|
|
12
12
|
webctx inspect particle/podcasts/episodes/search
|
|
13
|
-
webctx execute particle/podcasts/episodes/search --inputs '{"semantic_search":"AI agents","limit":2}'
|
|
13
|
+
webctx execute particle/podcasts/episodes/search --inputs '{"semantic_search":"AI agents","limit":2}'
|
|
14
14
|
```
|
|
15
15
|
|
|
16
16
|
The output is plain text or markdown, so it is easy to hand to ChatGPT, Codex, Claude Code, a shell script, or another tool. Search now retrieves relevant source excerpts automatically instead of returning only URLs. For a specific URL, add an optional question to explore related documentation and surface exact code and API details.
|
|
@@ -37,10 +37,10 @@ webctx research "podcast conversations about AI agents" --limit 5
|
|
|
37
37
|
webctx research --view sources --limit 5
|
|
38
38
|
webctx research --mode catalogue --view operations --sources particle --limit 10
|
|
39
39
|
webctx inspect particle/podcasts/episodes/search
|
|
40
|
-
webctx execute particle/podcasts/episodes/search --inputs '{"semantic_search":"AI agents","limit":2}'
|
|
40
|
+
webctx execute particle/podcasts/episodes/search --inputs '{"semantic_search":"AI agents","limit":2}'
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
-
Browse [the structured-data guide](https://webctx.ashray.xyz/docs/data-sources) for filters, paging, batches, full upstream responses, validation, and credit handling. Discovery/inspection is free; `execute` uses paid Alexandria credits. The per
|
|
43
|
+
Browse [the structured-data guide](https://webctx.ashray.xyz/docs/data-sources) for filters, paging, batches, full upstream responses, validation, and credit handling. Discovery/inspection is free; `execute` uses paid Alexandria credits. The hosted Origo server enforces a 200-credit limit per request, with no user-facing budget controls. Estimated and actual credits are shown in responses. Third-party provider terms require separate human acceptance; Origo never accepts them automatically.
|
|
44
44
|
|
|
45
45
|
Origo deploys independently through the `origo-api` Vercel project on pushes to `main`. CLI releases and npm publishing are unchanged. See the [Origo MCP guide](https://webctx.ashray.xyz/docs/origo) for architecture, setup, and security notes.
|
|
46
46
|
|
package/internal/app/app.go
CHANGED
|
@@ -125,12 +125,8 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
|
|
125
125
|
return 0
|
|
126
126
|
case "execute":
|
|
127
127
|
request := ExecuteDataInput{Raw: flags["raw"] == "true", RequestID: flags["request-id"]}
|
|
128
|
-
if
|
|
129
|
-
|
|
130
|
-
if e != nil {
|
|
131
|
-
return cliDataError(stderr, dataError("invalid_budget", "--max-credits requires an integer."))
|
|
132
|
-
}
|
|
133
|
-
request.MaxCredits = n
|
|
128
|
+
if _, supplied := flags["max-credits"]; supplied {
|
|
129
|
+
return cliDataError(stderr, dataError("unsupported_flag", "Credit limits are managed by the server; remove --max-credits."))
|
|
134
130
|
}
|
|
135
131
|
if jsonCalls := flags["calls"]; jsonCalls != "" {
|
|
136
132
|
data, e := cliJSONString(jsonCalls)
|
|
@@ -153,7 +149,7 @@ func Run(args []string, stdout, stderr io.Writer) int {
|
|
|
153
149
|
}
|
|
154
150
|
request.Calls = []DataCall{{ID: input, Inputs: inputs}}
|
|
155
151
|
} else {
|
|
156
|
-
return cliDataError(stderr, dataError("missing_calls", "Usage: webctx execute <operation-id> --inputs JSON
|
|
152
|
+
return cliDataError(stderr, dataError("missing_calls", "Usage: webctx execute <operation-id> --inputs JSON or --calls JSON."))
|
|
157
153
|
}
|
|
158
154
|
result, e := ExecuteData(request)
|
|
159
155
|
if e != nil {
|
|
@@ -177,8 +173,8 @@ Usage:
|
|
|
177
173
|
webctx map-site <url>
|
|
178
174
|
webctx research [question] [--view sources|groups|operations] [--sources IDs] [--include inputs,output,examples] [--limit N] [--offset N]
|
|
179
175
|
webctx inspect <operation-or-source-id> [--raw]
|
|
180
|
-
webctx execute <operation-id> --inputs '{"field":"value"}' [--
|
|
181
|
-
webctx execute --calls '[{"id":"source/operation","inputs":{}}]'
|
|
176
|
+
webctx execute <operation-id> --inputs '{"field":"value"}' [--request-id ID]
|
|
177
|
+
webctx execute --calls '[{"id":"source/operation","inputs":{}}]'
|
|
182
178
|
|
|
183
179
|
Examples:
|
|
184
180
|
webctx search "next.js server components"
|
|
@@ -187,7 +183,7 @@ Examples:
|
|
|
187
183
|
webctx research "podcast conversations about AI agents"
|
|
188
184
|
webctx research --view sources --limit 5
|
|
189
185
|
webctx inspect particle/podcasts/episodes/search
|
|
190
|
-
webctx execute particle/podcasts/episodes/search --inputs '{"semantic_search":"AI agents","limit":2}'
|
|
186
|
+
webctx execute particle/podcasts/episodes/search --inputs '{"semantic_search":"AI agents","limit":2}'
|
|
191
187
|
webctx read-link https://docs.example.com/guide
|
|
192
188
|
webctx read-link https://docs.example.com/api --question "How do auth and pagination work?"
|
|
193
189
|
webctx map-site https://example.com`, version)
|
|
@@ -18,7 +18,10 @@ import (
|
|
|
18
18
|
)
|
|
19
19
|
|
|
20
20
|
const dataAPIBase = "https://api.firecrawl.dev/v2"
|
|
21
|
-
|
|
21
|
+
|
|
22
|
+
// The server, not a client-provided tool parameter, controls this preflight ceiling.
|
|
23
|
+
// A production override is configured on the Origo Vercel project.
|
|
24
|
+
const defaultDataCreditBudget = 200
|
|
22
25
|
const maxDataResponseBytes int64 = 12 << 20
|
|
23
26
|
|
|
24
27
|
type DataError struct {
|
|
@@ -43,21 +46,12 @@ func dataAPIKey() (string, error) {
|
|
|
43
46
|
|
|
44
47
|
func dataCreditCap() int {
|
|
45
48
|
value, err := strconv.Atoi(strings.TrimSpace(os.Getenv("WEBCTX_ALEXANDRIA_MAX_CREDITS")))
|
|
46
|
-
if err != nil || value < 1 || value >
|
|
49
|
+
if err != nil || value < 1 || value > defaultDataCreditBudget {
|
|
47
50
|
return defaultDataCreditBudget
|
|
48
51
|
}
|
|
49
52
|
return value
|
|
50
53
|
}
|
|
51
54
|
|
|
52
|
-
func paidDataEnabled() bool {
|
|
53
|
-
switch strings.ToLower(strings.TrimSpace(os.Getenv("WEBCTX_ALEXANDRIA_PAID"))) {
|
|
54
|
-
case "0", "false", "no", "off":
|
|
55
|
-
return false
|
|
56
|
-
default:
|
|
57
|
-
return true
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
55
|
var safeRequestID = regexp.MustCompile("^[A-Za-z0-9_-]{8,128}$")
|
|
62
56
|
|
|
63
57
|
func newRequestID() (string, error) {
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
package app
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"regexp"
|
|
5
|
+
"strings"
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
var backtickedIdentifier = regexp.MustCompile("`([A-Za-z][A-Za-z0-9_/-]{1,30})`")
|
|
9
|
+
|
|
10
|
+
// Upstream examples can legitimately be absent. Offer a clearly marked,
|
|
11
|
+
// locally validated sample input without claiming it was published by the
|
|
12
|
+
// provider or that it was executed.
|
|
13
|
+
func contractExamples(contract map[string]any) any {
|
|
14
|
+
if published := contract["examples"]; published != nil {
|
|
15
|
+
if len(listField(published)) > 0 {
|
|
16
|
+
return published
|
|
17
|
+
}
|
|
18
|
+
if len(mapField(published)) > 0 {
|
|
19
|
+
return published
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
input, needed := nativeInputTemplate(contract)
|
|
23
|
+
fieldInfo := map[string]map[string]any{}
|
|
24
|
+
for _, v := range listField(contract["options"]) {
|
|
25
|
+
if field := mapField(v); field != nil {
|
|
26
|
+
fieldInfo[stringField(field["name"])] = field
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
for _, name := range needed {
|
|
30
|
+
field := fieldInfo[name]
|
|
31
|
+
if value, ok := sampleFieldValue(name, field); ok {
|
|
32
|
+
input[name] = value
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
// An illustrative example should not accidentally fetch huge datasets
|
|
36
|
+
// when copied verbatim. Users may adjust the inputs freely.
|
|
37
|
+
for _, name := range []string{"limit", "per_page", "page_size", "max_results", "count"} {
|
|
38
|
+
if n, ok := numberField(input[name]); ok && n > 5 {
|
|
39
|
+
input[name] = 5
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if _, _, err := parseOperationID(stringField(contract["id"])); err != nil {
|
|
43
|
+
return nil
|
|
44
|
+
}
|
|
45
|
+
validation := validateDataInputs(contract, input)
|
|
46
|
+
example := map[string]any{
|
|
47
|
+
"kind": "generated_from_inspected_schema",
|
|
48
|
+
"note": "Illustrative Origo-generated inputs; not a published provider example or an executed query.",
|
|
49
|
+
"inputs": input,
|
|
50
|
+
"valid_input_shape": validation == nil,
|
|
51
|
+
}
|
|
52
|
+
if validation != nil {
|
|
53
|
+
example["note"] = "Illustrative input template; fill required fields and check source-specific constraints before execution."
|
|
54
|
+
example["needs_input"] = needed
|
|
55
|
+
} else {
|
|
56
|
+
example["next"] = map[string]any{"tool": "execute", "arguments": map[string]any{
|
|
57
|
+
"calls": []any{map[string]any{"id": contract["id"], "inputs": input}},
|
|
58
|
+
}}
|
|
59
|
+
}
|
|
60
|
+
return []any{example}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
func sampleFieldValue(name string, field map[string]any) (any, bool) {
|
|
64
|
+
if field == nil {
|
|
65
|
+
return nil, false
|
|
66
|
+
}
|
|
67
|
+
if values := listField(field["oneOf"]); len(values) > 0 {
|
|
68
|
+
for _, v := range values {
|
|
69
|
+
if v != nil && v != "" {
|
|
70
|
+
return v, true
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
if sample, ok := field["example"]; ok && sample != nil {
|
|
75
|
+
return sample, true
|
|
76
|
+
}
|
|
77
|
+
if samples := listField(field["examples"]); len(samples) > 0 && samples[0] != nil {
|
|
78
|
+
return samples[0], true
|
|
79
|
+
}
|
|
80
|
+
if v, ok := field["default"]; ok && v != nil {
|
|
81
|
+
return v, true
|
|
82
|
+
}
|
|
83
|
+
info := stringField(field["about"]) + " " + stringField(field["description"])
|
|
84
|
+
lower := strings.ToLower(name + " " + info)
|
|
85
|
+
switch {
|
|
86
|
+
case strings.Contains(name, "series_id") && strings.Contains(info, "CPIAUCSL"):
|
|
87
|
+
return "CPIAUCSL", true
|
|
88
|
+
case strings.Contains(name, "series_id") && strings.Contains(info, "UNRATE"):
|
|
89
|
+
return "UNRATE", true
|
|
90
|
+
case strings.Contains(name, "series_id") && strings.Contains(info, "GDP"):
|
|
91
|
+
return "GDP", true
|
|
92
|
+
case strings.Contains(lower, "semantic_search"):
|
|
93
|
+
return "AI agents", true
|
|
94
|
+
case name == "query" || name == "search" || name == "keyword_search":
|
|
95
|
+
return "example topic", true
|
|
96
|
+
case name == "url" || strings.HasSuffix(name, "_url"):
|
|
97
|
+
return "https://example.com", true
|
|
98
|
+
}
|
|
99
|
+
// Backticked descriptions sometimes contain sample codes or IDs.
|
|
100
|
+
if strings.Contains(lower, "id") || strings.Contains(lower, "code") {
|
|
101
|
+
for _, match := range backtickedIdentifier.FindAllStringSubmatch(info, 10) {
|
|
102
|
+
candidate := match[1]
|
|
103
|
+
if strings.Contains(candidate, "/") || strings.Contains(candidate, "_") {
|
|
104
|
+
continue
|
|
105
|
+
}
|
|
106
|
+
if len(candidate) >= 3 && !strings.EqualFold(candidate, "required") {
|
|
107
|
+
return candidate, true
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return nil, false
|
|
112
|
+
}
|
|
@@ -215,7 +215,7 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
|
|
|
215
215
|
return int(math.Ceil(estimated)), nil
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
-
func continuationForDataResult(id string, originalInputs map[string]any, data any
|
|
218
|
+
func continuationForDataResult(id string, originalInputs map[string]any, data any) map[string]any {
|
|
219
219
|
value := mapField(data)
|
|
220
220
|
if value == nil {
|
|
221
221
|
return nil
|
|
@@ -224,8 +224,7 @@ func continuationForDataResult(id string, originalInputs map[string]any, data an
|
|
|
224
224
|
if provider := stringField(next["provider"]); provider != "" {
|
|
225
225
|
if capability := stringField(next["capability"]); capability != "" {
|
|
226
226
|
return map[string]any{"tool": "execute", "arguments": map[string]any{
|
|
227
|
-
"calls":
|
|
228
|
-
"max_credits": maxCredits,
|
|
227
|
+
"calls": []any{map[string]any{"id": provider + "/" + capability, "inputs": mapField(next["options"])}},
|
|
229
228
|
}}
|
|
230
229
|
}
|
|
231
230
|
}
|
|
@@ -245,16 +244,14 @@ func continuationForDataResult(id string, originalInputs map[string]any, data an
|
|
|
245
244
|
next["cursor"] = cursor
|
|
246
245
|
}
|
|
247
246
|
return map[string]any{"tool": "execute", "arguments": map[string]any{
|
|
248
|
-
"calls":
|
|
249
|
-
"max_credits": maxCredits,
|
|
247
|
+
"calls": []any{map[string]any{"id": id, "inputs": next}},
|
|
250
248
|
}}
|
|
251
249
|
}
|
|
252
250
|
if page, ok := numberField(value["next_page"]); ok && page > 0 {
|
|
253
251
|
next := copyInputs(originalInputs)
|
|
254
252
|
next["page"] = int(page)
|
|
255
253
|
return map[string]any{"tool": "execute", "arguments": map[string]any{
|
|
256
|
-
"calls":
|
|
257
|
-
"max_credits": maxCredits,
|
|
254
|
+
"calls": []any{map[string]any{"id": id, "inputs": next}},
|
|
258
255
|
}}
|
|
259
256
|
}
|
|
260
257
|
if offset, ok := numberField(value["next_offset"]); ok && offset >= 0 {
|
|
@@ -263,8 +260,7 @@ func continuationForDataResult(id string, originalInputs map[string]any, data an
|
|
|
263
260
|
if offset > previous {
|
|
264
261
|
next["offset"] = int(offset)
|
|
265
262
|
return map[string]any{"tool": "execute", "arguments": map[string]any{
|
|
266
|
-
"calls":
|
|
267
|
-
"max_credits": maxCredits,
|
|
263
|
+
"calls": []any{map[string]any{"id": id, "inputs": next}},
|
|
268
264
|
}}
|
|
269
265
|
}
|
|
270
266
|
}
|
|
@@ -285,16 +281,7 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
285
281
|
if len(input.Calls) == 0 || len(input.Calls) > 10 {
|
|
286
282
|
return nil, dataError("invalid_calls", "Provide 1-10 operation calls with id and inputs.")
|
|
287
283
|
}
|
|
288
|
-
if !paidDataEnabled() {
|
|
289
|
-
return nil, dataError("paid_data_disabled", "Paid data queries are disabled by WEBCTX_ALEXANDRIA_PAID=off.")
|
|
290
|
-
}
|
|
291
284
|
cap := dataCreditCap()
|
|
292
|
-
if input.MaxCredits > 0 && input.MaxCredits < cap {
|
|
293
|
-
cap = input.MaxCredits
|
|
294
|
-
}
|
|
295
|
-
if input.MaxCredits < 0 || input.MaxCredits > dataCreditCap() {
|
|
296
|
-
return nil, dataError("invalid_budget", fmt.Sprintf("max_credits must be between 1 and the configured maximum of %d.", dataCreditCap()))
|
|
297
|
-
}
|
|
298
285
|
if input.RequestID != "" && !safeRequestID.MatchString(input.RequestID) {
|
|
299
286
|
return nil, dataError("invalid_request_id", "request_id must have 8-128 letters, digits, hyphens or underscores.")
|
|
300
287
|
}
|
|
@@ -334,9 +321,9 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
334
321
|
}
|
|
335
322
|
estimated += cost
|
|
336
323
|
if estimated > cap {
|
|
337
|
-
return nil, &DataError{Code: "
|
|
338
|
-
Message: fmt.Sprintf("
|
|
339
|
-
Details: map[string]any{"estimated_credits": estimated, "
|
|
324
|
+
return nil, &DataError{Code: "server_credit_limit_exceeded",
|
|
325
|
+
Message: fmt.Sprintf("This request is estimated at %d credits, above Origo's %d-credit server limit. Reduce the number of operations or requested records.", estimated, cap),
|
|
326
|
+
Details: map[string]any{"estimated_credits": estimated, "server_credit_limit": cap},
|
|
340
327
|
}
|
|
341
328
|
}
|
|
342
329
|
provider, capability, _ := parseOperationID(call.ID)
|
|
@@ -360,7 +347,7 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
360
347
|
if errorsAsData(err, &typed) {
|
|
361
348
|
typed.Details = map[string]any{"request_id": requestID,
|
|
362
349
|
"retry": map[string]any{"tool": "execute", "arguments": ExecuteDataInput{
|
|
363
|
-
Calls: input.Calls,
|
|
350
|
+
Calls: input.Calls, RequestID: requestID, Raw: input.Raw,
|
|
364
351
|
}},
|
|
365
352
|
}
|
|
366
353
|
}
|
|
@@ -384,6 +371,9 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
384
371
|
"data": entry["data"],
|
|
385
372
|
"credits_used": entry["creditsCost"],
|
|
386
373
|
}
|
|
374
|
+
if explanation := dataMeasurementNote(entry["data"]); explanation != "" {
|
|
375
|
+
item["measurement_note"] = explanation
|
|
376
|
+
}
|
|
387
377
|
if token := entry["alexandriaId"]; token != nil {
|
|
388
378
|
item["data_id"] = token
|
|
389
379
|
}
|
|
@@ -391,7 +381,7 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
391
381
|
item["error"] = errValue
|
|
392
382
|
}
|
|
393
383
|
if i < len(input.Calls) {
|
|
394
|
-
if next := continuationForDataResult(id, input.Calls[i].Inputs, entry["data"]
|
|
384
|
+
if next := continuationForDataResult(id, input.Calls[i].Inputs, entry["data"]); next != nil {
|
|
395
385
|
item["next"] = next
|
|
396
386
|
}
|
|
397
387
|
}
|
|
@@ -401,7 +391,7 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
401
391
|
"results": results,
|
|
402
392
|
"request_id": requestID,
|
|
403
393
|
"estimated_credits": estimated,
|
|
404
|
-
"pricing_note": "
|
|
394
|
+
"pricing_note": "Estimated credits are calculated before execution; the upstream provider reports actual charges. Origo enforces a server-managed per-request credit limit.",
|
|
405
395
|
"guidance": "Use next to fetch additional pages where available. Reuse request_id when retrying the same paid request, but generate a new ID for the next page.",
|
|
406
396
|
}
|
|
407
397
|
if cost := payload["creditsCost"]; cost != nil {
|
|
@@ -422,3 +412,24 @@ func errorsAsData(err error, into **DataError) bool {
|
|
|
422
412
|
}
|
|
423
413
|
return ok
|
|
424
414
|
}
|
|
415
|
+
|
|
416
|
+
// Interpret only well-defined provider unit codes, and distinguish CPI index
|
|
417
|
+
// levels from annual inflation rates without transforming source values.
|
|
418
|
+
func dataMeasurementNote(raw any) string {
|
|
419
|
+
data := mapField(raw)
|
|
420
|
+
if data == nil {
|
|
421
|
+
return ""
|
|
422
|
+
}
|
|
423
|
+
series := strings.ToUpper(stringField(data["series_id"]))
|
|
424
|
+
if !strings.HasPrefix(series, "CPI") {
|
|
425
|
+
return ""
|
|
426
|
+
}
|
|
427
|
+
switch strings.ToLower(stringField(data["units"])) {
|
|
428
|
+
case "lin", "":
|
|
429
|
+
return "CPI observations are price-index levels, not annual inflation percentages. Do not label them as inflation rates; a year-over-year percentage requires a 12-month comparison or an appropriate FRED transformation."
|
|
430
|
+
case "pc1":
|
|
431
|
+
return "FRED units=pc1 represents the percent change from one year ago, not an index level."
|
|
432
|
+
default:
|
|
433
|
+
return "CPI observations use the requested FRED units transformation; inspect the units field before interpreting values as percentages."
|
|
434
|
+
}
|
|
435
|
+
}
|
|
@@ -144,10 +144,8 @@ func InspectData(input InspectDataInput) (map[string]any, error) {
|
|
|
144
144
|
}
|
|
145
145
|
provider, _, _ := parseOperationID(id)
|
|
146
146
|
inputs, missing := nativeInputTemplate(item)
|
|
147
|
-
budget := dataCreditCap()
|
|
148
147
|
execArguments := map[string]any{
|
|
149
|
-
"calls":
|
|
150
|
-
"max_credits": budget,
|
|
148
|
+
"calls": []any{map[string]any{"id": id, "inputs": inputs}},
|
|
151
149
|
}
|
|
152
150
|
response := map[string]any{
|
|
153
151
|
"id": id,
|
|
@@ -161,7 +159,7 @@ func InspectData(input InspectDataInput) (map[string]any, error) {
|
|
|
161
159
|
},
|
|
162
160
|
"inputs": map[string]any{"fields": item["options"], "requires_one_of": item["requiresOneOf"]},
|
|
163
161
|
"output": item["response"],
|
|
164
|
-
"examples": item
|
|
162
|
+
"examples": contractExamples(item),
|
|
165
163
|
"input_template": inputs,
|
|
166
164
|
"next": map[string]any{"tool": "execute", "arguments": execArguments},
|
|
167
165
|
"guidance": "Copy next.arguments into execute, fill any blank required inputs and adjust values according to inputs.fields. Execute uses credits.",
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
package app
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import (
|
|
4
|
+
"sort"
|
|
5
|
+
"strings"
|
|
6
|
+
)
|
|
4
7
|
|
|
5
8
|
func researchView(view, query string) (string, error) {
|
|
6
9
|
switch strings.ToLower(strings.TrimSpace(view)) {
|
|
@@ -165,6 +168,20 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
|
|
|
165
168
|
}
|
|
166
169
|
data := mapField(raw["data"])
|
|
167
170
|
items = listField(data["tools"])
|
|
171
|
+
// Some upstream ranked matches arrive out of order despite including a
|
|
172
|
+
// similarity score. Present the highest-scoring candidate first, retaining
|
|
173
|
+
// upstream order for exact ties or missing scores.
|
|
174
|
+
sort.SliceStable(items, func(i, j int) bool {
|
|
175
|
+
a, aok := numberField(mapField(items[i])["similarity"])
|
|
176
|
+
b, bok := numberField(mapField(items[j])["similarity"])
|
|
177
|
+
if aok != bok {
|
|
178
|
+
return aok
|
|
179
|
+
}
|
|
180
|
+
if !aok {
|
|
181
|
+
return false
|
|
182
|
+
}
|
|
183
|
+
return a > b
|
|
184
|
+
})
|
|
168
185
|
// Ranked search has no native cursor. Expose the free, complete
|
|
169
186
|
// catalogue with matching semantic query as an optional expansion.
|
|
170
187
|
next = map[string]any{"tool": "research", "arguments": map[string]any{
|
|
@@ -59,6 +59,18 @@ func (d *dataTestTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
|
|
59
59
|
return testHTTPResponse(req, status, string(b), nil), nil
|
|
60
60
|
}
|
|
61
61
|
if req.URL.Path == "/v2/search" {
|
|
62
|
+
if strings.Contains(stringField(payload["query"]), "relevance ordering") {
|
|
63
|
+
low := dataTestContract()
|
|
64
|
+
low["id"] = "particle/podcasts/episodes/list"
|
|
65
|
+
low["similarity"] = 0.1
|
|
66
|
+
high := dataTestContract()
|
|
67
|
+
high["id"] = "particle/podcasts/episodes/search"
|
|
68
|
+
high["similarity"] = 0.97
|
|
69
|
+
middle := dataTestContract()
|
|
70
|
+
middle["id"] = "particle/podcasts/episode"
|
|
71
|
+
middle["similarity"] = 0.8
|
|
72
|
+
return write(200, map[string]any{"success": true, "data": map[string]any{"tools": []any{low, high, middle}}})
|
|
73
|
+
}
|
|
62
74
|
return write(200, map[string]any{"success": true, "data": map[string]any{
|
|
63
75
|
"tools": []any{dataTestContract()},
|
|
64
76
|
}})
|
|
@@ -115,6 +127,20 @@ func (d *dataTestTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
|
|
115
127
|
return write(400, map[string]any{"success": false, "error": "Bad test request."})
|
|
116
128
|
}
|
|
117
129
|
|
|
130
|
+
func TestResearchSortsMatchesByActualSimilarity(t *testing.T) {
|
|
131
|
+
installDataTestTransport(t)
|
|
132
|
+
results, err := ResearchData(ResearchDataInput{Query: "relevance ordering", Limit: 3})
|
|
133
|
+
if err != nil {
|
|
134
|
+
t.Fatal(err)
|
|
135
|
+
}
|
|
136
|
+
found := results["results"].([]map[string]any)
|
|
137
|
+
if found[0]["id"] != "particle/podcasts/episodes/search" ||
|
|
138
|
+
found[1]["id"] != "particle/podcasts/episode" ||
|
|
139
|
+
found[2]["id"] != "particle/podcasts/episodes/list" {
|
|
140
|
+
t.Fatalf("results were not ordered by reported relevance: %#v", found)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
118
144
|
func installDataTestTransport(t *testing.T) *dataTestTransport {
|
|
119
145
|
t.Helper()
|
|
120
146
|
old := http.DefaultClient
|
|
@@ -122,7 +148,6 @@ func installDataTestTransport(t *testing.T) *dataTestTransport {
|
|
|
122
148
|
d := &dataTestTransport{}
|
|
123
149
|
http.DefaultClient = &http.Client{Transport: d}
|
|
124
150
|
t.Setenv("FIRECRAWL_API_KEY", "test-key")
|
|
125
|
-
t.Setenv("WEBCTX_ALEXANDRIA_PAID", "true")
|
|
126
151
|
t.Setenv("WEBCTX_ALEXANDRIA_MAX_CREDITS", "100")
|
|
127
152
|
return d
|
|
128
153
|
}
|
|
@@ -155,6 +180,10 @@ func TestProgressiveDataDiscoveryAndExecution(t *testing.T) {
|
|
|
155
180
|
if len(listField(mapField(details["inputs"])["fields"])) != 4 {
|
|
156
181
|
t.Fatalf("missing upstream contract fields: %#v", details["inputs"])
|
|
157
182
|
}
|
|
183
|
+
generated := listField(details["examples"])
|
|
184
|
+
if len(generated) == 0 || mapField(generated[0])["valid_input_shape"] != true {
|
|
185
|
+
t.Fatalf("inspection must generate an explicitly labeled, schema-valid example: %#v", details["examples"])
|
|
186
|
+
}
|
|
158
187
|
continued := mapField(details["next"])
|
|
159
188
|
if continued["tool"] != "execute" {
|
|
160
189
|
t.Fatalf("no next execution: %#v", continued)
|
|
@@ -169,8 +198,7 @@ func TestProgressiveDataDiscoveryAndExecution(t *testing.T) {
|
|
|
169
198
|
t.Fatalf("invalid template: %#v", skeleton)
|
|
170
199
|
}
|
|
171
200
|
got, err := ExecuteData(ExecuteDataInput{
|
|
172
|
-
Calls:
|
|
173
|
-
MaxCredits: 20,
|
|
201
|
+
Calls: []DataCall{{ID: id, Inputs: map[string]any{"semantic_search": "AI agents", "limit": float64(2)}}},
|
|
174
202
|
})
|
|
175
203
|
if err != nil {
|
|
176
204
|
t.Fatal(err)
|
|
@@ -222,20 +250,18 @@ func TestCatalogueBrowsingReturnsExactPaginationCall(t *testing.T) {
|
|
|
222
250
|
}
|
|
223
251
|
}
|
|
224
252
|
|
|
225
|
-
func
|
|
253
|
+
func TestExecutionValidationAndServerLimitRejectBeforePaidCall(t *testing.T) {
|
|
226
254
|
mock := installDataTestTransport(t)
|
|
227
255
|
id := "particle/podcasts/episodes/search"
|
|
228
256
|
for _, testCase := range []struct {
|
|
229
|
-
inputs
|
|
230
|
-
|
|
231
|
-
code string
|
|
257
|
+
inputs map[string]any
|
|
258
|
+
code string
|
|
232
259
|
}{
|
|
233
|
-
{map[string]any{},
|
|
234
|
-
{map[string]any{"semantic_search": "AI agents", "limit": 101},
|
|
235
|
-
{map[string]any{"semantic_search": "AI agents", "unknown_field": 123},
|
|
236
|
-
{map[string]any{"semantic_search": "AI agents", "limit": 5}, 5, "credit_budget_exceeded"},
|
|
260
|
+
{map[string]any{}, "invalid_inputs"},
|
|
261
|
+
{map[string]any{"semantic_search": "AI agents", "limit": 101}, "invalid_inputs"},
|
|
262
|
+
{map[string]any{"semantic_search": "AI agents", "unknown_field": 123}, "invalid_inputs"},
|
|
237
263
|
} {
|
|
238
|
-
_, err := ExecuteData(ExecuteDataInput{Calls: []DataCall{{ID: id, Inputs: testCase.inputs}}
|
|
264
|
+
_, err := ExecuteData(ExecuteDataInput{Calls: []DataCall{{ID: id, Inputs: testCase.inputs}}})
|
|
239
265
|
typed, ok := err.(*DataError)
|
|
240
266
|
if !ok || typed.Code != testCase.code {
|
|
241
267
|
t.Fatalf("wanted %s, got %v", testCase.code, err)
|
|
@@ -279,7 +305,7 @@ func TestCLIResearchAndInspectJSON(t *testing.T) {
|
|
|
279
305
|
if code := Run([]string{"inspect", "particle/podcasts/episodes/search"}, &out, &errBuf); code != 0 {
|
|
280
306
|
t.Fatalf("CLI inspect failed: %s", errBuf.String())
|
|
281
307
|
}
|
|
282
|
-
if !strings.Contains(out.String(), "semantic_search") ||
|
|
308
|
+
if !strings.Contains(out.String(), "semantic_search") || strings.Contains(out.String(), "max_credits") {
|
|
283
309
|
t.Fatalf("CLI inspect omitted exact execute syntax: %s", out.String())
|
|
284
310
|
}
|
|
285
311
|
}
|
|
@@ -287,7 +313,7 @@ func TestCLIResearchAndInspectJSON(t *testing.T) {
|
|
|
287
313
|
func TestProviderNextOffsetCreatesCopyReadyContinuation(t *testing.T) {
|
|
288
314
|
id := "fred-stlouisfed-org/economic-data/series_observations"
|
|
289
315
|
inputs := map[string]any{"series_id": "CPIAUCSL", "limit": 2.0, "offset": 0.0}
|
|
290
|
-
next := continuationForDataResult(id, inputs, map[string]any{"next_offset": 2.0, "count": 956.0}
|
|
316
|
+
next := continuationForDataResult(id, inputs, map[string]any{"next_offset": 2.0, "count": 956.0})
|
|
291
317
|
if mapField(next)["tool"] != "execute" {
|
|
292
318
|
t.Fatalf("expected executable pagination: %+v", next)
|
|
293
319
|
}
|
|
@@ -334,3 +360,85 @@ func TestInspectOnlyOutputStillHasExecuteInputTemplate(t *testing.T) {
|
|
|
334
360
|
t.Fatalf("missing necessary next-call schema after include=output: %+v", details)
|
|
335
361
|
}
|
|
336
362
|
}
|
|
363
|
+
|
|
364
|
+
func TestServerManagedCreditCeilingCannotBeOverridden(t *testing.T) {
|
|
365
|
+
mock := installDataTestTransport(t)
|
|
366
|
+
// The ceiling defaults to 200 and an invalidly high environment value
|
|
367
|
+
// cannot silently widen it beyond 200.
|
|
368
|
+
t.Setenv("WEBCTX_ALEXANDRIA_MAX_CREDITS", "200")
|
|
369
|
+
if got := dataCreditCap(); got != 200 {
|
|
370
|
+
t.Fatalf("bad production cap: %d", got)
|
|
371
|
+
}
|
|
372
|
+
t.Setenv("WEBCTX_ALEXANDRIA_MAX_CREDITS", "2000")
|
|
373
|
+
if got := dataCreditCap(); got != 200 {
|
|
374
|
+
t.Fatalf("server cap exceeded 200: %d", got)
|
|
375
|
+
}
|
|
376
|
+
t.Setenv("WEBCTX_ALEXANDRIA_MAX_CREDITS", "120")
|
|
377
|
+
if got := dataCreditCap(); got != 120 {
|
|
378
|
+
t.Fatalf("server environment not respected: %d", got)
|
|
379
|
+
}
|
|
380
|
+
id := "particle/podcasts/episodes/search"
|
|
381
|
+
validInputs := map[string]any{"semantic_search": "AI agents"}
|
|
382
|
+
// 9*15=135: reject before ANY provider is queried.
|
|
383
|
+
batch := make([]DataCall, 9)
|
|
384
|
+
for i := range batch {
|
|
385
|
+
batch[i] = DataCall{ID: id, Inputs: validInputs}
|
|
386
|
+
}
|
|
387
|
+
_, err := ExecuteData(ExecuteDataInput{Calls: batch})
|
|
388
|
+
typed, ok := err.(*DataError)
|
|
389
|
+
if !ok || typed.Code != "server_credit_limit_exceeded" ||
|
|
390
|
+
mapField(typed.Details)["estimated_credits"] != 135 {
|
|
391
|
+
t.Fatalf("expected server-only limit, got %#v", err)
|
|
392
|
+
}
|
|
393
|
+
mock.mu.Lock()
|
|
394
|
+
defer mock.mu.Unlock()
|
|
395
|
+
if mock.executions != 0 {
|
|
396
|
+
t.Fatalf("server limit failed to prevent execution, calls=%d", mock.executions)
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
func TestClientCreditControlIsNotAvailableInCLI(t *testing.T) {
|
|
401
|
+
installDataTestTransport(t)
|
|
402
|
+
var stdout, stderr bytes.Buffer
|
|
403
|
+
code := Run([]string{"execute", "particle/podcasts/episodes/search", "--inputs", `{"semantic_search":"AI agents"}`, "--max-credits", "1"}, &stdout, &stderr)
|
|
404
|
+
if code != 1 || !strings.Contains(stderr.String(), `"code": "unsupported_flag"`) {
|
|
405
|
+
t.Fatalf("old client credit option should fail clearly, status=%d error=%s", code, stderr.String())
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
func TestCPIInterpretationDoesNotConfuseIndexWithInflation(t *testing.T) {
|
|
410
|
+
levels := dataMeasurementNote(map[string]any{"series_id": "CPIAUCSL", "units": "lin"})
|
|
411
|
+
if !strings.Contains(levels, "index levels") || !strings.Contains(levels, "not annual inflation percentages") {
|
|
412
|
+
t.Fatalf("CPI level explanation missing: %s", levels)
|
|
413
|
+
}
|
|
414
|
+
percentage := dataMeasurementNote(map[string]any{"series_id": "CPIAUCSL", "units": "pc1"})
|
|
415
|
+
if !strings.Contains(percentage, "percent change from one year ago") {
|
|
416
|
+
t.Fatalf("transformation note missing: %s", percentage)
|
|
417
|
+
}
|
|
418
|
+
if dataMeasurementNote(map[string]any{"series_id": "UNRATE", "units": "lin"}) != "" {
|
|
419
|
+
t.Fatal("unemployment must not get CPI-specific semantics")
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
func TestGeneratedFREDExampleUsesDocumentedSeries(t *testing.T) {
|
|
424
|
+
c := map[string]any{
|
|
425
|
+
"id": "fred-stlouisfed-org/economic-data/series_observations",
|
|
426
|
+
"options": []any{
|
|
427
|
+
map[string]any{"name": "series_id", "type": "string", "required": true, "about": "FRED series id, e.g. GDP, UNRATE and CPIAUCSL"},
|
|
428
|
+
map[string]any{"name": "limit", "type": "number", "default": 1000, "min": 1, "max": 10000},
|
|
429
|
+
},
|
|
430
|
+
}
|
|
431
|
+
examples := listField(contractExamples(c))
|
|
432
|
+
if len(examples) != 1 {
|
|
433
|
+
t.Fatalf("no fallback example: %#v", examples)
|
|
434
|
+
}
|
|
435
|
+
example := mapField(examples[0])
|
|
436
|
+
inputs := mapField(example["inputs"])
|
|
437
|
+
if example["kind"] != "generated_from_inspected_schema" || example["valid_input_shape"] != true ||
|
|
438
|
+
inputs["series_id"] != "CPIAUCSL" || inputs["limit"] != 5 {
|
|
439
|
+
t.Fatalf("unexpected generated FRED sample: %#v", examples)
|
|
440
|
+
}
|
|
441
|
+
if mapField(example["next"])["tool"] != "execute" {
|
|
442
|
+
t.Fatalf("generated example cannot be copied into execute: %#v", example)
|
|
443
|
+
}
|
|
444
|
+
}
|
|
@@ -33,8 +33,7 @@ type DataCall struct {
|
|
|
33
33
|
// ExecuteDataInput supports up to ten calls in a single provider request.
|
|
34
34
|
// It validates each call against the free catalogue before using credits.
|
|
35
35
|
type ExecuteDataInput struct {
|
|
36
|
-
Calls
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
Raw bool `json:"raw,omitempty" jsonschema:"Include the complete original provider response"`
|
|
36
|
+
Calls []DataCall `json:"calls" jsonschema:"One to ten operation calls; inspect each operation to discover exact input field names"`
|
|
37
|
+
RequestID string `json:"request_id,omitempty" jsonschema:"Optional stable idempotency identifier for retrying a paid request; reuse unchanged if retrying"`
|
|
38
|
+
Raw bool `json:"raw,omitempty" jsonschema:"Include the complete original provider response"`
|
|
40
39
|
}
|
package/internal/app/research.go
CHANGED
|
@@ -87,6 +87,7 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
|
|
|
87
87
|
if err != nil {
|
|
88
88
|
return "", err
|
|
89
89
|
}
|
|
90
|
+
primary = copyableCurlExamples(primary)
|
|
90
91
|
|
|
91
92
|
// Combine on-page links and authoritative Markdown index links instead of
|
|
92
93
|
// filling the entire budget from whichever links happen to appear first.
|
|
@@ -109,7 +110,11 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
|
|
|
109
110
|
"**Source:** " + rawURL,
|
|
110
111
|
"",
|
|
111
112
|
}
|
|
112
|
-
|
|
113
|
+
primaryBudget := focusedPrimaryBudget
|
|
114
|
+
if asksForAPIContracts(question) {
|
|
115
|
+
primaryBudget = 5300
|
|
116
|
+
}
|
|
117
|
+
primaryExcerpt := focusExcerpt(primary, question, primaryBudget)
|
|
113
118
|
parts = append(parts, primaryExcerpt)
|
|
114
119
|
successful := 1
|
|
115
120
|
relatedSuccessful := 0
|
|
@@ -124,7 +129,11 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
|
|
|
124
129
|
relatedSuccessful++
|
|
125
130
|
seenURLs[source.url] = true
|
|
126
131
|
allRetrieved += "\n" + source.content
|
|
127
|
-
|
|
132
|
+
relatedBudget := relatedExcerptBudget(source.url)
|
|
133
|
+
if asksForAPIContracts(question) && relatedBudget > 3000 {
|
|
134
|
+
relatedBudget = 3000
|
|
135
|
+
}
|
|
136
|
+
excerpt := focusExcerpt(copyableCurlExamples(source.content), question, relatedBudget)
|
|
128
137
|
outputEvidence += "\n" + excerpt
|
|
129
138
|
parts = append(parts, "", "## Related source", "**Source:** "+source.url, "", excerpt)
|
|
130
139
|
}
|
|
@@ -172,7 +181,11 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
|
|
|
172
181
|
successful++
|
|
173
182
|
relatedSuccessful++
|
|
174
183
|
allRetrieved += "\n" + source.content
|
|
175
|
-
|
|
184
|
+
followupBudget := relatedExcerptBudget(source.url)
|
|
185
|
+
if asksForAPIContracts(question) && followupBudget > 3000 {
|
|
186
|
+
followupBudget = 3000
|
|
187
|
+
}
|
|
188
|
+
excerpt := focusExcerpt(copyableCurlExamples(source.content), question, followupBudget)
|
|
176
189
|
outputEvidence += "\n" + excerpt
|
|
177
190
|
parts = append(parts, "", "## Supporting implementation reference", "**Source:** "+source.url, "", excerpt)
|
|
178
191
|
}
|
|
@@ -181,7 +194,10 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
|
|
|
181
194
|
if contract := <-openAPIDone; contract.Content != "" {
|
|
182
195
|
successful++
|
|
183
196
|
outputEvidence += "\n" + contract.Content
|
|
184
|
-
|
|
197
|
+
// Put exact REST contracts first so coding agents encounter the
|
|
198
|
+
// implementation-ready operations before lengthy source excerpts.
|
|
199
|
+
front := []string{"## REST API contract", "**Source:** " + contract.URL, "", contract.Content, ""}
|
|
200
|
+
parts = append(append(append([]string(nil), parts[:4]...), front...), parts[4:]...)
|
|
185
201
|
}
|
|
186
202
|
}
|
|
187
203
|
if structured := <-structuredDone; structured != "" {
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
package app
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/json"
|
|
5
|
+
"regexp"
|
|
6
|
+
"strings"
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
var trailingPayloadSlash = regexp.MustCompile("(?m)\\\\[ \\t]*(\\r?\\n)")
|
|
10
|
+
|
|
11
|
+
// Some source scrapers leave JavaScript-style line-continuation slashes
|
|
12
|
+
// inside shell-quoted JSON. Those slashes make the published cURL request
|
|
13
|
+
// uncopyable. Repair only payloads that are initially invalid JSON and become
|
|
14
|
+
// valid JSON after removing those continuation markers. Label every repair.
|
|
15
|
+
// The original source URL remains in the surrounding cited context.
|
|
16
|
+
func copyableCurlExamples(markdown string) string {
|
|
17
|
+
parts := strings.Split(markdown, backtickFence)
|
|
18
|
+
if len(parts) < 3 {
|
|
19
|
+
return markdown
|
|
20
|
+
}
|
|
21
|
+
for i := 1; i < len(parts); i += 2 {
|
|
22
|
+
code := parts[i]
|
|
23
|
+
if !strings.Contains(code, "curl ") {
|
|
24
|
+
continue
|
|
25
|
+
}
|
|
26
|
+
start := strings.Index(code, "-d '{")
|
|
27
|
+
if start < 0 {
|
|
28
|
+
continue
|
|
29
|
+
}
|
|
30
|
+
bodyStart := start + len("-d '")
|
|
31
|
+
endRel := strings.LastIndex(code[bodyStart:], "}'")
|
|
32
|
+
if endRel < 0 {
|
|
33
|
+
continue
|
|
34
|
+
}
|
|
35
|
+
bodyEnd := bodyStart + endRel + 1
|
|
36
|
+
original := code[bodyStart:bodyEnd]
|
|
37
|
+
if json.Valid([]byte(original)) {
|
|
38
|
+
continue
|
|
39
|
+
}
|
|
40
|
+
fixed := trailingPayloadSlash.ReplaceAllString(original, "$1")
|
|
41
|
+
if fixed == original || !json.Valid([]byte(fixed)) {
|
|
42
|
+
continue
|
|
43
|
+
}
|
|
44
|
+
parts[i] = code[:bodyStart] + fixed + code[bodyEnd:]
|
|
45
|
+
parts[i-1] += "\n\n*[Copy-ready correction: removed stray line-continuation backslashes inside the published JSON payload; see the cited URL for the original.]*\n"
|
|
46
|
+
}
|
|
47
|
+
return strings.Join(parts, backtickFence)
|
|
48
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
package app
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/json"
|
|
5
|
+
"strings"
|
|
6
|
+
"testing"
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
func TestMalformedAlexandriaJSONBecomesCopyable(t *testing.T) {
|
|
10
|
+
source := "# REST example\n" + backtickFence + "bash\n" +
|
|
11
|
+
"curl https://api.firecrawl.dev/v2/search \\\n" +
|
|
12
|
+
" -H 'Content-Type: application/json' \\\n" +
|
|
13
|
+
" -d '{\n \"query\": \"podcast conversations\",\n \"sources\": [\\\n \"web\",\\\n \"alexandria\"\\\n ],\n \"limit\": 2\n}'\n" +
|
|
14
|
+
backtickFence
|
|
15
|
+
got := copyableCurlExamples(source)
|
|
16
|
+
if !strings.Contains(got, "Copy-ready correction") {
|
|
17
|
+
t.Fatalf("lost truthful correction label: %s", got)
|
|
18
|
+
}
|
|
19
|
+
start := strings.Index(got, "-d '{")
|
|
20
|
+
if start < 0 {
|
|
21
|
+
t.Fatalf("lost source cURL: %s", got)
|
|
22
|
+
}
|
|
23
|
+
bodyStart := start + len("-d '")
|
|
24
|
+
bodyEnd := bodyStart + strings.LastIndex(got[bodyStart:], "}'") + 1
|
|
25
|
+
if !json.Valid([]byte(got[bodyStart:bodyEnd])) {
|
|
26
|
+
t.Fatalf("JSON request still uncopyable: %s", got[bodyStart:bodyEnd])
|
|
27
|
+
}
|
|
28
|
+
if !strings.Contains(got, " -H 'Content-Type: application/json' \\") {
|
|
29
|
+
t.Fatalf("source shell continuations should be preserved: %s", got)
|
|
30
|
+
}
|
|
31
|
+
if strings.Contains(got, "\"sources\": [\\") {
|
|
32
|
+
t.Fatalf("malformed JSON continuation not repaired: %s", got)
|
|
33
|
+
}
|
|
34
|
+
if copyableCurlExamples(got) != got {
|
|
35
|
+
t.Fatalf("correction should be idempotent")
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
func TestCorrectSourceExamplesAreNeverRewritten(t *testing.T) {
|
|
40
|
+
source := backtickFence + "bash\ncurl https://api.example.com/v2/search -d '{\"sources\":[\"alexandria\"]}'\n" + backtickFence
|
|
41
|
+
if got := copyableCurlExamples(source); got != source {
|
|
42
|
+
t.Fatalf("valid published snippet was modified: %s", got)
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
func TestRESTContractsLeadAndExcerptsStayConcise(t *testing.T) {
|
|
47
|
+
if focusedPrimaryBudget <= 5300 {
|
|
48
|
+
t.Fatal("REST excerpt budget should be narrower than normal focused context")
|
|
49
|
+
}
|
|
50
|
+
source := "# Official API docs\n" + strings.Repeat("Large unrelated introduction about navigation.\n", 700) +
|
|
51
|
+
"\n## REST API endpoint example\n" +
|
|
52
|
+
backtickFence + "bash\ncurl https://api.example.com/v2/search -d '{\"sources\":[\"alexandria\"]}'\n" + backtickFence
|
|
53
|
+
got := focusExcerpt(source, "What are the REST API endpoints and exact request examples?", 5300)
|
|
54
|
+
if len(got) > 6000 || !strings.Contains(got, "/v2/search") {
|
|
55
|
+
t.Fatalf("REST excerpt did not prioritize complete example under budget: len=%d", len(got))
|
|
56
|
+
}
|
|
57
|
+
}
|
package/package.json
CHANGED
package/pkg/origo/data_test.go
CHANGED
|
@@ -49,17 +49,18 @@ func TestDataToolErrorsAreActionableStructuredMCPResults(t *testing.T) {
|
|
|
49
49
|
{"research", map[string]any{"view": "invalid"}, "invalid_view"},
|
|
50
50
|
{"inspect", map[string]any{"id": "not-a-valid-operation/"}, "invalid_operation"},
|
|
51
51
|
{"execute", map[string]any{"calls": []any{}}, "invalid_calls"},
|
|
52
|
+
{"execute", map[string]any{"calls": []any{map[string]any{"id": "example/data/list"}}, "max_credits": 1}, "invalid_arguments"},
|
|
52
53
|
} {
|
|
53
54
|
response := toolResponse(t, tc.name, tc.input, 11)
|
|
54
|
-
if response["isError"]
|
|
55
|
-
t.Fatalf("%s should return
|
|
55
|
+
if response["isError"] == true {
|
|
56
|
+
t.Fatalf("%s should return normal structured response, not an MCP error: %#v", tc.name, response)
|
|
56
57
|
}
|
|
57
58
|
output, ok := response["structuredContent"].(map[string]any)
|
|
58
59
|
if !ok {
|
|
59
60
|
t.Fatalf("%s did not preserve structuredContent: %#v", tc.name, response)
|
|
60
61
|
}
|
|
61
62
|
errorResult, ok := output["error"].(map[string]any)
|
|
62
|
-
if !ok || errorResult["code"] != tc.code {
|
|
63
|
+
if !ok || errorResult["code"] != tc.code || output["ok"] != false {
|
|
63
64
|
t.Fatalf("%s error should be %s: %#v", tc.name, tc.code, response)
|
|
64
65
|
}
|
|
65
66
|
}
|
|
@@ -68,7 +69,6 @@ func TestDataToolErrorsAreActionableStructuredMCPResults(t *testing.T) {
|
|
|
68
69
|
func TestDataToolResearchInspectExecuteOverHTTP(t *testing.T) {
|
|
69
70
|
t.Setenv("ORIGO_API_KEY", "secret")
|
|
70
71
|
t.Setenv("FIRECRAWL_API_KEY", "test")
|
|
71
|
-
t.Setenv("WEBCTX_ALEXANDRIA_PAID", "true")
|
|
72
72
|
original := http.DefaultClient
|
|
73
73
|
t.Cleanup(func() { http.DefaultClient = original })
|
|
74
74
|
var paidCount int
|
|
@@ -126,7 +126,7 @@ func TestDataToolResearchInspectExecuteOverHTTP(t *testing.T) {
|
|
|
126
126
|
}
|
|
127
127
|
result := toolResponse(t, "execute", map[string]any{"calls": []any{
|
|
128
128
|
map[string]any{"id": "example/data/list", "inputs": map[string]any{"query": "forty two"}},
|
|
129
|
-
}
|
|
129
|
+
}}, 14)
|
|
130
130
|
final := result["structuredContent"].(map[string]any)
|
|
131
131
|
if result["isError"] == true || final["credits_used"] != float64(2) || paidCount != 1 {
|
|
132
132
|
t.Fatalf("paid query failed: %#v calls=%d", final, paidCount)
|
package/pkg/origo/server.go
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
package origo
|
|
4
4
|
|
|
5
5
|
import (
|
|
6
|
+
"bytes"
|
|
6
7
|
"context"
|
|
7
8
|
"crypto/sha256"
|
|
8
9
|
"crypto/subtle"
|
|
@@ -12,6 +13,7 @@ import (
|
|
|
12
13
|
"strings"
|
|
13
14
|
|
|
14
15
|
"github.com/amxv/webctx/pkg/retrieval"
|
|
16
|
+
"github.com/google/jsonschema-go/jsonschema"
|
|
15
17
|
"github.com/modelcontextprotocol/go-sdk/mcp"
|
|
16
18
|
)
|
|
17
19
|
|
|
@@ -50,42 +52,109 @@ func makeServer() *mcp.Server {
|
|
|
50
52
|
}
|
|
51
53
|
return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: result}}}, nil, nil
|
|
52
54
|
})
|
|
53
|
-
|
|
55
|
+
s.AddTool(&mcp.Tool{
|
|
54
56
|
Name: "research",
|
|
55
57
|
Title: "Find data sources",
|
|
56
58
|
Description: "Discover real-world information sources and their available operations. Search semantically, browse all sources, groups or operations, filter by websites/source IDs/categories, and paginate. Free; returns IDs and copy-ready inspect calls. Use read_link for regular documentation.",
|
|
57
59
|
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
})
|
|
62
|
-
mcp.AddTool(s, &mcp.Tool{
|
|
60
|
+
InputSchema: researchDataSchema(),
|
|
61
|
+
}, dataHandler(retrieval.Research))
|
|
62
|
+
s.AddTool(&mcp.Tool{
|
|
63
63
|
Name: "inspect",
|
|
64
64
|
Title: "Inspect a data source or operation",
|
|
65
65
|
Description: "Explore operations in a source, or inspect one operation's EXACT input types, required fields, constraints, output contract, pricing, and examples. Returns the next execute call with the exact input keys to fill. Free.",
|
|
66
66
|
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
})
|
|
71
|
-
mcp.AddTool(s, &mcp.Tool{
|
|
67
|
+
InputSchema: inspectDataSchema(),
|
|
68
|
+
}, dataHandler(retrieval.Inspect))
|
|
69
|
+
s.AddTool(&mcp.Tool{
|
|
72
70
|
Name: "execute",
|
|
73
71
|
Title: "Query a data source (uses credits)",
|
|
74
|
-
Description: "Execute 1-10 inspected structured-data operations using
|
|
75
|
-
Annotations: &mcp.ToolAnnotations{ReadOnlyHint:
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return dataMCPResult(result, err)
|
|
79
|
-
})
|
|
72
|
+
Description: "Execute 1-10 inspected structured-data operations using exact operation IDs and inputs. Alexandria credits are charged automatically; the server enforces its configured 200-credit per-request ceiling. Actual credits are reported. Supports pagination, safe request_id retries and provider-native JSON. Does not accept third-party terms.",
|
|
73
|
+
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
|
|
74
|
+
InputSchema: executeDataSchema(),
|
|
75
|
+
}, dataHandler(retrieval.Execute))
|
|
80
76
|
return s
|
|
81
77
|
}
|
|
82
78
|
|
|
79
|
+
// Keep client-facing types narrow and self-describing. The SDK's default
|
|
80
|
+
// reflection makes optional slices nullable and string enums unconstrained;
|
|
81
|
+
// the catalogue operations support neither ambiguity.
|
|
82
|
+
func researchDataSchema() *jsonschema.Schema {
|
|
83
|
+
schema, err := jsonschema.For[retrieval.ResearchInput](nil)
|
|
84
|
+
if err != nil {
|
|
85
|
+
panic(err)
|
|
86
|
+
}
|
|
87
|
+
schema.Properties["mode"].Enum = []any{"ranked", "catalogue"}
|
|
88
|
+
schema.Properties["view"].Enum = []any{"sources", "groups", "operations"}
|
|
89
|
+
for _, key := range []string{"mode", "view"} {
|
|
90
|
+
schema.Properties[key].Type = "string"
|
|
91
|
+
schema.Properties[key].Types = nil
|
|
92
|
+
}
|
|
93
|
+
for _, key := range []string{"urls", "sources", "categories", "groups", "operations", "include"} {
|
|
94
|
+
p := schema.Properties[key]
|
|
95
|
+
p.Type = "array"
|
|
96
|
+
p.Types = nil
|
|
97
|
+
}
|
|
98
|
+
schema.Properties["include"].Items.Enum = []any{"inputs", "output", "examples"}
|
|
99
|
+
return schema
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
func inspectDataSchema() *jsonschema.Schema {
|
|
103
|
+
schema, err := jsonschema.For[retrieval.InspectInput](nil)
|
|
104
|
+
if err != nil {
|
|
105
|
+
panic(err)
|
|
106
|
+
}
|
|
107
|
+
p := schema.Properties["include"]
|
|
108
|
+
p.Type = "array"
|
|
109
|
+
p.Types = nil
|
|
110
|
+
p.Items.Enum = []any{"inputs", "output", "examples"}
|
|
111
|
+
return schema
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
func executeDataSchema() *jsonschema.Schema {
|
|
115
|
+
schema, err := jsonschema.For[retrieval.ExecuteInput](nil)
|
|
116
|
+
if err != nil {
|
|
117
|
+
panic(err)
|
|
118
|
+
}
|
|
119
|
+
countMin, countMax := 1, 10
|
|
120
|
+
p := schema.Properties["calls"]
|
|
121
|
+
p.Type = "array"
|
|
122
|
+
p.Types = nil
|
|
123
|
+
p.MinItems = &countMin
|
|
124
|
+
p.MaxItems = &countMax
|
|
125
|
+
return schema
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Manual validation keeps data errors in successful MCP tool responses rather
|
|
129
|
+
// than surfacing them as client-side INVALID_ARGUMENT tool exceptions.
|
|
130
|
+
func dataHandler[In any](fn func(In) (map[string]any, error)) mcp.ToolHandler {
|
|
131
|
+
return func(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
|
132
|
+
var input In
|
|
133
|
+
args := req.Params.Arguments
|
|
134
|
+
if len(args) == 0 {
|
|
135
|
+
args = []byte("{}")
|
|
136
|
+
}
|
|
137
|
+
decoder := json.NewDecoder(bytes.NewReader(args))
|
|
138
|
+
decoder.DisallowUnknownFields()
|
|
139
|
+
if err := decoder.Decode(&input); err != nil {
|
|
140
|
+
response, _, e := dataMCPResult(nil, &retrieval.DataError{Code: "invalid_arguments", Message: err.Error()})
|
|
141
|
+
return response, e
|
|
142
|
+
}
|
|
143
|
+
result, err := fn(input)
|
|
144
|
+
response, _, e := dataMCPResult(result, err)
|
|
145
|
+
return response, e
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
83
149
|
func dataMCPResult(result map[string]any, err error) (*mcp.CallToolResult, any, error) {
|
|
84
150
|
failed := err != nil
|
|
85
151
|
if failed {
|
|
86
152
|
// Use the shared error serializer to preserve codes, corrective details
|
|
87
153
|
// and provider-term acceptance URLs as structured tool output.
|
|
88
154
|
result = retrieval.DataErrorResult(err)
|
|
155
|
+
result["ok"] = false
|
|
156
|
+
} else {
|
|
157
|
+
result["ok"] = true
|
|
89
158
|
}
|
|
90
159
|
bytes, marshalErr := json.MarshalIndent(result, "", " ")
|
|
91
160
|
if marshalErr != nil {
|
|
@@ -94,7 +163,9 @@ func dataMCPResult(result map[string]any, err error) (*mcp.CallToolResult, any,
|
|
|
94
163
|
return &mcp.CallToolResult{
|
|
95
164
|
Content: []mcp.Content{&mcp.TextContent{Text: string(bytes)}},
|
|
96
165
|
StructuredContent: result,
|
|
97
|
-
|
|
166
|
+
// Keep validation/provider failures in the structured result. Clients
|
|
167
|
+
// commonly convert isError=true to opaque INVALID_ARGUMENT exceptions.
|
|
168
|
+
IsError: false,
|
|
98
169
|
}, nil, nil
|
|
99
170
|
}
|
|
100
171
|
|
package/pkg/origo/server_test.go
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
package origo
|
|
2
2
|
|
|
3
3
|
import (
|
|
4
|
+
"encoding/json"
|
|
4
5
|
"net/http"
|
|
5
6
|
"net/http/httptest"
|
|
6
7
|
"strings"
|
|
@@ -22,6 +23,57 @@ func request(t *testing.T, key, body string) *httptest.ResponseRecorder {
|
|
|
22
23
|
return response
|
|
23
24
|
}
|
|
24
25
|
|
|
26
|
+
func TestDataSchemasExposeExactEnumsAndNonNullableCalls(t *testing.T) {
|
|
27
|
+
t.Setenv("ORIGO_API_KEY", "secret")
|
|
28
|
+
result := request(t, "secret", `{"jsonrpc":"2.0","id":12,"method":"tools/list","params":{}}`)
|
|
29
|
+
if result.Code != 200 {
|
|
30
|
+
t.Fatalf("tools/list failed: %s", result.Body.String())
|
|
31
|
+
}
|
|
32
|
+
var decoded struct {
|
|
33
|
+
Result struct {
|
|
34
|
+
Tools []struct {
|
|
35
|
+
Name string `json:"name"`
|
|
36
|
+
InputSchema map[string]any `json:"inputSchema"`
|
|
37
|
+
Annotations map[string]any `json:"annotations"`
|
|
38
|
+
} `json:"tools"`
|
|
39
|
+
} `json:"result"`
|
|
40
|
+
}
|
|
41
|
+
if err := json.Unmarshal(result.Body.Bytes(), &decoded); err != nil {
|
|
42
|
+
t.Fatal(err)
|
|
43
|
+
}
|
|
44
|
+
schemas := map[string]map[string]any{}
|
|
45
|
+
for _, tool := range decoded.Result.Tools {
|
|
46
|
+
if tool.Annotations["readOnlyHint"] != true {
|
|
47
|
+
t.Fatalf("%s was not marked read-only", tool.Name)
|
|
48
|
+
}
|
|
49
|
+
if _, exists := tool.Annotations["openWorldHint"]; exists {
|
|
50
|
+
t.Fatalf("%s should omit openWorldHint", tool.Name)
|
|
51
|
+
}
|
|
52
|
+
schemas[tool.Name] = tool.InputSchema
|
|
53
|
+
}
|
|
54
|
+
if len(schemas) != 5 {
|
|
55
|
+
t.Fatalf("expected five tools, got %d", len(schemas))
|
|
56
|
+
}
|
|
57
|
+
props := func(name string) map[string]any { return schemas[name]["properties"].(map[string]any) }
|
|
58
|
+
mode := props("research")["mode"].(map[string]any)
|
|
59
|
+
view := props("research")["view"].(map[string]any)
|
|
60
|
+
if mode["type"] != "string" || len(mode["enum"].([]any)) != 2 ||
|
|
61
|
+
view["type"] != "string" || len(view["enum"].([]any)) != 3 {
|
|
62
|
+
t.Fatalf("research discovery schemas are not typed: mode=%#v view=%#v", mode, view)
|
|
63
|
+
}
|
|
64
|
+
incl := props("research")["include"].(map[string]any)
|
|
65
|
+
if incl["type"] != "array" || len(incl["items"].(map[string]any)["enum"].([]any)) != 3 {
|
|
66
|
+
t.Fatalf("include does not constrain allowed options: %#v", incl)
|
|
67
|
+
}
|
|
68
|
+
calls := props("execute")["calls"].(map[string]any)
|
|
69
|
+
if _, present := props("execute")["max_credits"]; present {
|
|
70
|
+
t.Fatal("execute still exposes credit-budget control")
|
|
71
|
+
}
|
|
72
|
+
if calls["type"] != "array" || calls["minItems"] != float64(1) || calls["maxItems"] != float64(10) {
|
|
73
|
+
t.Fatalf("execute calls must be non-null and 1–10 items: %#v", calls)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
25
77
|
func TestServerRequiresConfiguredKey(t *testing.T) {
|
|
26
78
|
t.Setenv("ORIGO_API_KEY", "")
|
|
27
79
|
response := request(t, "somekey", `{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}`)
|
|
@@ -62,11 +114,11 @@ func TestFiveToolsWithProgressiveDataInterface(t *testing.T) {
|
|
|
62
114
|
if count := strings.Count(text, `"name":`); count != 5 {
|
|
63
115
|
t.Fatalf("expected exactly 5 tools, got %d: %s", count, text)
|
|
64
116
|
}
|
|
65
|
-
if count := strings.Count(text, `"readOnlyHint":true`); count !=
|
|
66
|
-
t.Fatalf("expected
|
|
117
|
+
if count := strings.Count(text, `"readOnlyHint":true`); count != 5 {
|
|
118
|
+
t.Fatalf("expected all five MCP tools to declare readOnlyHint=true, got %d: %s", count, text)
|
|
67
119
|
}
|
|
68
|
-
if
|
|
69
|
-
t.Fatalf("
|
|
120
|
+
if strings.Contains(text, `"readOnlyHint":false`) || strings.Contains(text, `"openWorldHint"`) || strings.Contains(text, `"destructiveHint"`) {
|
|
121
|
+
t.Fatalf("unexpected non-read-only or open-world/destructive annotations: %s", text)
|
|
70
122
|
}
|
|
71
123
|
}
|
|
72
124
|
|
package/pkg/retrieval/data.go
CHANGED
|
@@ -6,6 +6,7 @@ type ResearchInput = app.ResearchDataInput
|
|
|
6
6
|
type InspectInput = app.InspectDataInput
|
|
7
7
|
type ExecuteInput = app.ExecuteDataInput
|
|
8
8
|
type DataCall = app.DataCall
|
|
9
|
+
type DataError = app.DataError
|
|
9
10
|
|
|
10
11
|
func Research(input ResearchInput) (map[string]any, error) { return app.ResearchData(input) }
|
|
11
12
|
func Inspect(input InspectInput) (map[string]any, error) { return app.InspectData(input) }
|