webctx 0.2.5 → 0.2.7

package/internal/app/data_replay.go

@@ -0,0 +1,83 @@
+package app
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"os"
+	"sync"
+	"time"
+)
+
+type localPaidReplay struct {
+	fingerprint string
+	output      map[string]any
+	expires     time.Time
+}
+
+var paidReplayMemory = struct {
+	sync.Mutex
+	items map[string]localPaidReplay
+}{items: map[string]localPaidReplay{}}
+
+func localReplayKey(id string) string {
+	h := sha256.Sum256([]byte(os.Getenv("FIRECRAWL_API_KEY") + "\x00" + id))
+	return hex.EncodeToString(h[:])
+}
+func paidRequestFingerprint(calls []DataCall) string {
+	body, _ := json.Marshal(calls)
+	h := sha256.Sum256(body)
+	return hex.EncodeToString(h[:])
+}
+
+// This ledger is only a warm-worker replay optimization. It cannot claim to
+// verify charges or prevent replay across multiple Vercel instances; upstream
+// x-request-id idempotency remains authoritative in that situation.
+func cachedPaidReplay(requestID string, calls []DataCall) (map[string]any, error) {
+	key := localReplayKey(requestID)
+	fingerprint := paidRequestFingerprint(calls)
+	paidReplayMemory.Lock()
+	entry, ok := paidReplayMemory.items[key]
+	paidReplayMemory.Unlock()
+	if !ok || time.Now().After(entry.expires) {
+		return nil, nil
+	}
+	if entry.fingerprint != fingerprint {
+		return nil, &DataError{
+			Code:    "duplicate_request",
+			Message: "This request_id already completed with different inputs. Restore the original calls to replay, or omit request_id for a new query.",
+			Details: map[string]any{"recovery": "Never retry different inputs with the same ID. Use the original operation and input list for replay, or generate a fresh ID."},
+		}
+	}
+	raw, err := json.Marshal(entry.output)
+	if err != nil {
+		return nil, nil
+	}
+	var result map[string]any
+	if json.Unmarshal(raw, &result) != nil {
+		return nil, nil
+	}
+	result["replayed"] = true
+	result["replay_status"] = "served_from_local_cache"
+	result["original_credits_used"] = result["credits_used"]
+	result["credits_charged_this_request"] = 0
+	result["billing_note"] = "This exact replay was served from Origo's in-process cache without contacting Firecrawl, so Origo incurred no new upstream execution on this attempt."
+	return result, nil
+}
+
+func storePaidReplay(requestID string, calls []DataCall, output map[string]any) {
+	raw, err := json.Marshal(output)
+	if err != nil || len(raw) > 1<<20 {
+		return
+	}
+	key := localReplayKey(requestID)
+	paidReplayMemory.Lock()
+	defer paidReplayMemory.Unlock()
+	if len(paidReplayMemory.items) > 192 {
+		paidReplayMemory.items = map[string]localPaidReplay{}
+	}
+	paidReplayMemory.items[key] = localPaidReplay{
+		fingerprint: paidRequestFingerprint(calls), output: output,
+		expires: time.Now().Add(15 * time.Minute),
+	}
+}

package/internal/app/data_research.go

@@ -1,6 +1,10 @@
 package app
 
-import "strings"
+import (
+	"net/url"
+	"sort"
+	"strings"
+)
 
 func researchView(view, query string) (string, error) {
 	switch strings.ToLower(strings.TrimSpace(view)) {
@@ -57,7 +61,7 @@ func normalizeResearchInput(input ResearchDataInput) (ResearchDataInput, string,
 	if err != nil {
 		return input, "", nil, err
 	}
-	if input.Limit == 0 {
+	if input.Limit == 0 && !input.limitProvided {
 		input.Limit = 10
 	}
 	if input.Limit < 1 || input.Limit > 100 {
@@ -71,6 +75,16 @@ func normalizeResearchInput(input ResearchDataInput) (ResearchDataInput, string,
 			return input, "", nil, dataError("invalid_url", "Website filters require public HTTP(S) URLs.")
 		}
 	}
+	if len(input.URLs) > 0 && len(input.Sources) == 0 {
+		for _, target := range input.URLs {
+			u, _ := url.Parse(target)
+			host := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.")
+			if host != "" {
+				source := strings.ReplaceAll(host, ".", "-")
+				input.Sources = append(input.Sources, source)
+			}
+		}
+	}
 	return input, view, expand, nil
 }
 
@@ -79,7 +93,7 @@ func nativeBrowseOptions(input ResearchDataInput, view string, expand []string)
 	if input.Query != "" {
 		o["query"] = input.Query
 	}
-	if len(input.URLs) > 0 {
+	if len(input.URLs) > 0 && len(input.Sources) == 0 {
 		o["urls"] = input.URLs
 	}
 	if len(input.Sources) > 0 {
@@ -120,6 +134,9 @@ func nativeBrowseOptions(input ResearchDataInput, view string, expand []string)
 		}
 	}
 	if len(expand) > 0 {
+		if containsString(expand, "examples") && !containsString(expand, "options") {
+			expand = append(append([]string(nil), expand...), "options")
+		}
 		o["expand"] = expand
 	}
 	return o
@@ -142,9 +159,19 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 	}
 	ctx, cancel := defaultDataContext()
 	defer cancel()
+	explicitExact := len(input.Operations) > 0
+	// An exact operation ID is authoritative. A descriptive query helps
+	// communicate intent, but must not exclude a known operation.
+	note := ""
+	if explicitExact && input.Query != "" {
+		note = "Exact operation IDs take precedence over descriptive query terms; the query was not used as a restrictive upstream filter."
+		input.Query = ""
+	}
 	semantic := input.Mode != "catalogue" && input.Query != "" && view == "tools" && input.Offset == 0 &&
 		len(input.URLs) == 0 && len(input.Sources) == 0 && len(input.Categories) == 0 &&
 		len(input.Groups) == 0 && len(input.Operations) == 0
+	localFilteredRanking := view == "tools" && input.Query != "" &&
+		(len(input.URLs) > 0 || len(input.Sources) > 0 || len(input.Groups) > 0 || len(input.Categories) > 0)
 	var items []any
 	var total any
 	var next any
@@ -156,15 +183,25 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		// compact upstream matches may omit both price and operation name.
 		// We still return compact result cards unless include was requested.
 		detail := "full"
+		fetchLimit := input.Limit * 4
+		if fetchLimit < 20 {
+			fetchLimit = 20
+		}
+		if fetchLimit > 100 {
+			fetchLimit = 100
+		}
 		raw, err = doDataAPI(ctx, "/search", map[string]any{
 			"query": input.Query, "sources": []string{"alexandria"},
-			"limit": input.Limit, "toolDetail": detail,
+			"limit": fetchLimit, "toolDetail": detail,
 		}, "")
 		if err != nil {
 			return nil, err
 		}
 		data := mapField(raw["data"])
 		items = listField(data["tools"])
+		// Some upstream ranked matches arrive out of order despite including a
+		// similarity score. Present the highest-scoring candidate first, retaining
+		// upstream order for exact ties or missing scores.
 		// Ranked search has no native cursor. Expose the free, complete
 		// catalogue with matching semantic query as an optional expansion.
 		next = map[string]any{"tool": "research", "arguments": map[string]any{
@@ -172,6 +209,22 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		}, "note": "Browse all matching operations with pagination; ranked discovery itself has no cursor."}
 	} else {
 		options := nativeBrowseOptions(input, view, expand)
+		if localFilteredRanking {
+			// Provider-specified lexical filters can exclude operations whose
+			// descriptions describe the requested data indirectly. Fetch a
+			// catalogue slice from the actual source, then rank it ourselves.
+			delete(options, "query")
+			options["limit"] = 100
+			options["offset"] = 0
+			note = "The source/category filters select candidates first. The question ranks those candidates locally, rather than excluding matching source operations."
+			if input.Mode != "catalogue" {
+				mode = "ranked_filtered"
+			}
+		}
+		if explicitExact {
+			delete(options, "query")
+			delete(options, "urls")
+		}
 		var catalogue map[string]any
 		catalogue, raw, err = freeDataBrowse(ctx, options)
 		if err != nil {
@@ -183,6 +236,48 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 			next = continuationFromCatalogue(native)
 		}
 	}
+	if view == "tools" {
+		termsQuery := input.Query
+		for _, op := range input.Operations {
+			termsQuery += " " + op
+		}
+		if strings.TrimSpace(termsQuery) != "" {
+			sort.SliceStable(items, func(i, j int) bool {
+				return intentRelevance(mapField(items[i]), termsQuery) > intentRelevance(mapField(items[j]), termsQuery)
+			})
+		} else {
+			sort.SliceStable(items, func(i, j int) bool {
+				a, aok := numberField(mapField(items[i])["similarity"])
+				b, bok := numberField(mapField(items[j])["similarity"])
+				if aok != bok {
+					return aok
+				}
+				return a > b
+			})
+		}
+	}
+	if semantic && len(items) > input.Limit {
+		items = items[:input.Limit]
+	}
+	if localFilteredRanking {
+		count := len(items)
+		start := input.Offset
+		if start > count {
+			start = count
+		}
+		end := start + input.Limit
+		if end > count {
+			end = count
+		}
+		items = items[start:end]
+		if count > end {
+			next = map[string]any{"tool": "research", "arguments": ResearchDataInput{
+				Query: input.Query, Mode: input.Mode, URLs: input.URLs, Sources: input.Sources,
+				Categories: input.Categories, Groups: input.Groups, View: niceDataView(view),
+				Include: input.Include, Limit: input.Limit, Offset: end,
+			}}
+		}
+	}
 	results := make([]map[string]any, 0, len(items))
 	for _, value := range items {
 		entry := mapField(value)
@@ -206,6 +301,15 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		}
 		if cost, ok := entry["creditsCost"]; ok {
 			result["credits_per_call"] = cost
+			if listed, ok := numberField(cost); ok {
+				if listed > float64(dataCreditCap()) {
+					result["executable"] = false
+					result["availability"] = "exceeds_server_credit_limit"
+					result["server_credit_limit"] = dataCreditCap()
+				} else if view == "tools" {
+					result["executable"] = true
+				}
+			}
 		}
 		if cost, ok := entry["perRecord"]; ok {
 			result["per_record"] = cost
@@ -213,8 +317,14 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		if count, ok := entry["toolCount"]; ok {
 			result["operation_count"] = count
 		}
+		if input.Query != "" && view == "tools" {
+			result["relevance"] = intentRelevance(entry, input.Query)
+		}
 		if rank, ok := entry["similarity"]; ok {
-			result["relevance"] = rank
+			result["semantic_similarity"] = rank
+			if input.Query == "" {
+				result["relevance"] = rank
+			}
 		}
 		if len(expand) > 0 {
 			if selected := entry["options"]; selected != nil && containsString(expand, "options") {
@@ -226,8 +336,15 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 			if selected := entry["examples"]; selected != nil && containsString(expand, "examples") {
 				result["examples"] = selected
 			}
+			if containsString(expand, "examples") && result["examples"] == nil {
+				if entry["options"] != nil {
+					result["examples"] = contractExamples(entry)
+				}
+			}
 		}
-		if continuation := mapField(entry["next"]); continuation != nil {
+		if view == "tools" && id != "" {
+			result["next"] = map[string]any{"tool": "inspect", "arguments": map[string]any{"id": id}}
+		} else if continuation := mapField(entry["next"]); continuation != nil {
 			result["next"] = continuationFromCatalogue(continuation)
 		} else if id != "" {
 			result["next"] = map[string]any{"tool": "inspect", "arguments": map[string]any{"id": id}}
@@ -239,6 +356,12 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		"count": len(results), "cost_credits": 0,
 		"guidance": "Use inspect with an operation ID to see complete inputs, output, examples and the next execute call. To explore a source, inspect its source ID.",
 	}
+	if note != "" {
+		output["mode_note"] = note
+	}
+	if len(input.URLs) > 0 && len(input.Sources) > 0 {
+		output["resolved_url_sources"] = input.Sources
+	}
 	if total != nil {
 		output["total"] = total
 	}
@@ -251,6 +374,55 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 	return output, nil
 }
 
+// Rank operations by their ability to deliver the requested data, not just
+// semantic similarity. Keep the provider similarity separately for auditing.
+func intentRelevance(entry map[string]any, query string) float64 {
+	if entry == nil {
+		return 0
+	}
+	base, _ := numberField(entry["similarity"])
+	if query == "" {
+		return base
+	}
+	q := strings.ToLower(query)
+	id := strings.ToLower(stringField(entry["id"]))
+	name := strings.ToLower(stringField(entry["name"]))
+	desc := strings.ToLower(stringField(entry["description"]))
+	p := id + " " + name
+	searchIntent := strings.Contains(q, "search") || strings.Contains(q, "find") ||
+		strings.Contains(q, "papers") || strings.Contains(q, "podcast") ||
+		strings.Contains(q, "trials") || strings.Contains(q, "about")
+	historyIntent := strings.Contains(q, "historical") || strings.Contains(q, "history") ||
+		strings.Contains(q, "time series") || strings.Contains(q, "over time") ||
+		strings.Contains(q, "inflation")
+	if historyIntent && (strings.Contains(p, "observations") ||
+		strings.Contains(p, "history") || strings.Contains(p, "rates") ||
+		strings.Contains(p, "timeseries")) {
+		base += 0.23
+	}
+	if searchIntent && (strings.Contains(p, "/search") || strings.Contains(p, "/studies") ||
+		strings.Contains(p, "episode-search")) {
+		base += 0.20
+	}
+	if searchIntent && strings.Contains(p, "/list") && !strings.Contains(q, "list") {
+		base -= 0.04
+	}
+	for _, ancillary := range []string{"suggest", "related", "directory", "releases", "/source", "metadata", "browse"} {
+		if strings.Contains(p, ancillary) && !strings.Contains(q, strings.TrimLeft(ancillary, "/")) {
+			base -= 0.14
+		}
+	}
+	terms := queryTerms(query)
+	if len(terms) > 0 {
+		fit := termScore(name+" "+desc, terms)
+		if fit > 8 {
+			fit = 8
+		}
+		base += float64(fit) * 0.007
+	}
+	return float64(int(base*1000000+0.5)) / 1000000
+}
+
 func niceDataView(view string) string {
 	switch view {
 	case "providers":

package/internal/app/data_tools_test.go

@@ -59,6 +59,18 @@ func (d *dataTestTransport) RoundTrip(req *http.Request) (*http.Response, error)
 		return testHTTPResponse(req, status, string(b), nil), nil
 	}
 	if req.URL.Path == "/v2/search" {
+		if strings.Contains(stringField(payload["query"]), "relevance ordering") {
+			low := dataTestContract()
+			low["id"] = "particle/podcasts/episodes/list"
+			low["similarity"] = 0.1
+			high := dataTestContract()
+			high["id"] = "particle/podcasts/episodes/search"
+			high["similarity"] = 0.97
+			middle := dataTestContract()
+			middle["id"] = "particle/podcasts/episode"
+			middle["similarity"] = 0.8
+			return write(200, map[string]any{"success": true, "data": map[string]any{"tools": []any{low, high, middle}}})
+		}
 		return write(200, map[string]any{"success": true, "data": map[string]any{
 			"tools": []any{dataTestContract()},
 		}})
@@ -115,6 +127,20 @@ func (d *dataTestTransport) RoundTrip(req *http.Request) (*http.Response, error)
 	return write(400, map[string]any{"success": false, "error": "Bad test request."})
 }
 
+func TestResearchSortsMatchesByActualSimilarity(t *testing.T) {
+	installDataTestTransport(t)
+	results, err := ResearchData(ResearchDataInput{Query: "relevance ordering", Limit: 3})
+	if err != nil {
+		t.Fatal(err)
+	}
+	found := results["results"].([]map[string]any)
+	if found[0]["id"] != "particle/podcasts/episodes/search" ||
+		found[1]["id"] != "particle/podcasts/episode" ||
+		found[2]["id"] != "particle/podcasts/episodes/list" {
+		t.Fatalf("results were not ordered by reported relevance: %#v", found)
+	}
+}
+
 func installDataTestTransport(t *testing.T) *dataTestTransport {
 	t.Helper()
 	old := http.DefaultClient
@@ -122,7 +148,6 @@ func installDataTestTransport(t *testing.T) *dataTestTransport {
 	d := &dataTestTransport{}
 	http.DefaultClient = &http.Client{Transport: d}
 	t.Setenv("FIRECRAWL_API_KEY", "test-key")
-	t.Setenv("WEBCTX_ALEXANDRIA_PAID", "true")
 	t.Setenv("WEBCTX_ALEXANDRIA_MAX_CREDITS", "100")
 	return d
 }
@@ -155,6 +180,10 @@ func TestProgressiveDataDiscoveryAndExecution(t *testing.T) {
 	if len(listField(mapField(details["inputs"])["fields"])) != 4 {
 		t.Fatalf("missing upstream contract fields: %#v", details["inputs"])
 	}
+	generated := listField(details["examples"])
+	if len(generated) == 0 || mapField(generated[0])["valid_input_shape"] != true {
+		t.Fatalf("inspection must generate an explicitly labeled, schema-valid example: %#v", details["examples"])
+	}
 	continued := mapField(details["next"])
 	if continued["tool"] != "execute" {
 		t.Fatalf("no next execution: %#v", continued)
@@ -169,8 +198,7 @@ func TestProgressiveDataDiscoveryAndExecution(t *testing.T) {
 		t.Fatalf("invalid template: %#v", skeleton)
 	}
 	got, err := ExecuteData(ExecuteDataInput{
-		Calls:      []DataCall{{ID: id, Inputs: map[string]any{"semantic_search": "AI agents", "limit": float64(2)}}},
-		MaxCredits: 20,
+		Calls: []DataCall{{ID: id, Inputs: map[string]any{"semantic_search": "AI agents", "limit": float64(2)}}},
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -222,20 +250,18 @@ func TestCatalogueBrowsingReturnsExactPaginationCall(t *testing.T) {
 	}
 }
 
-func TestExecutionValidationAndBudgetRejectBeforePaidCall(t *testing.T) {
+func TestExecutionValidationAndServerLimitRejectBeforePaidCall(t *testing.T) {
 	mock := installDataTestTransport(t)
 	id := "particle/podcasts/episodes/search"
 	for _, testCase := range []struct {
-		inputs  map[string]any
-		credits int
-		code    string
+		inputs map[string]any
+		code   string
 	}{
-		{map[string]any{}, 20, "invalid_inputs"},
-		{map[string]any{"semantic_search": "AI agents", "limit": 101}, 20, "invalid_inputs"},
-		{map[string]any{"semantic_search": "AI agents", "unknown_field": 123}, 20, "invalid_inputs"},
-		{map[string]any{"semantic_search": "AI agents", "limit": 5}, 5, "credit_budget_exceeded"},
+		{map[string]any{}, "invalid_inputs"},
+		{map[string]any{"semantic_search": "AI agents", "limit": 101}, "invalid_inputs"},
+		{map[string]any{"semantic_search": "AI agents", "unknown_field": 123}, "invalid_inputs"},
 	} {
-		_, err := ExecuteData(ExecuteDataInput{Calls: []DataCall{{ID: id, Inputs: testCase.inputs}}, MaxCredits: testCase.credits})
+		_, err := ExecuteData(ExecuteDataInput{Calls: []DataCall{{ID: id, Inputs: testCase.inputs}}})
 		typed, ok := err.(*DataError)
 		if !ok || typed.Code != testCase.code {
 			t.Fatalf("wanted %s, got %v", testCase.code, err)
@@ -260,8 +286,8 @@ func TestExecuteHandlesProviderTermsWithoutAccepting(t *testing.T) {
 		mapField(typed.RequiresAction)["url"] != "https://firecrawl.dev/app/settings?tab=data-sources" {
 		t.Fatalf("terms condition lost: %+v", err)
 	}
-	if mapField(typed.Details)["request_id"] == nil {
-		t.Fatalf("lost retriable request identifier: %+v", typed)
+	if mapField(typed.Details)["retry"] != nil {
+		t.Fatalf("terms rejection must not recommend automatic paid retry: %+v", typed)
 	}
 }
 
@@ -279,7 +305,7 @@ func TestCLIResearchAndInspectJSON(t *testing.T) {
 	if code := Run([]string{"inspect", "particle/podcasts/episodes/search"}, &out, &errBuf); code != 0 {
 		t.Fatalf("CLI inspect failed: %s", errBuf.String())
 	}
-	if !strings.Contains(out.String(), "semantic_search") || !strings.Contains(out.String(), "max_credits") {
+	if !strings.Contains(out.String(), "semantic_search") || strings.Contains(out.String(), "max_credits") {
 		t.Fatalf("CLI inspect omitted exact execute syntax: %s", out.String())
 	}
 }
@@ -287,7 +313,7 @@ func TestCLIResearchAndInspectJSON(t *testing.T) {
 func TestProviderNextOffsetCreatesCopyReadyContinuation(t *testing.T) {
 	id := "fred-stlouisfed-org/economic-data/series_observations"
 	inputs := map[string]any{"series_id": "CPIAUCSL", "limit": 2.0, "offset": 0.0}
-	next := continuationForDataResult(id, inputs, map[string]any{"next_offset": 2.0, "count": 956.0}, 15)
+	next := continuationForDataResult(id, inputs, map[string]any{"next_offset": 2.0, "count": 956.0})
 	if mapField(next)["tool"] != "execute" {
 		t.Fatalf("expected executable pagination: %+v", next)
 	}
@@ -334,3 +360,85 @@ func TestInspectOnlyOutputStillHasExecuteInputTemplate(t *testing.T) {
 		t.Fatalf("missing necessary next-call schema after include=output: %+v", details)
 	}
 }
+
+func TestServerManagedCreditCeilingCannotBeOverridden(t *testing.T) {
+	mock := installDataTestTransport(t)
+	// The ceiling defaults to 200 and an invalidly high environment value
+	// cannot silently widen it beyond 200.
+	t.Setenv("WEBCTX_ALEXANDRIA_MAX_CREDITS", "200")
+	if got := dataCreditCap(); got != 200 {
+		t.Fatalf("bad production cap: %d", got)
+	}
+	t.Setenv("WEBCTX_ALEXANDRIA_MAX_CREDITS", "2000")
+	if got := dataCreditCap(); got != 200 {
+		t.Fatalf("server cap exceeded 200: %d", got)
+	}
+	t.Setenv("WEBCTX_ALEXANDRIA_MAX_CREDITS", "120")
+	if got := dataCreditCap(); got != 120 {
+		t.Fatalf("server environment not respected: %d", got)
+	}
+	id := "particle/podcasts/episodes/search"
+	validInputs := map[string]any{"semantic_search": "AI agents"}
+	// 9*15=135: reject before ANY provider is queried.
+	batch := make([]DataCall, 9)
+	for i := range batch {
+		batch[i] = DataCall{ID: id, Inputs: validInputs}
+	}
+	_, err := ExecuteData(ExecuteDataInput{Calls: batch})
+	typed, ok := err.(*DataError)
+	if !ok || typed.Code != "server_credit_limit_exceeded" ||
+		mapField(typed.Details)["estimated_credits"] != 135 {
+		t.Fatalf("expected server-only limit, got %#v", err)
+	}
+	mock.mu.Lock()
+	defer mock.mu.Unlock()
+	if mock.executions != 0 {
+		t.Fatalf("server limit failed to prevent execution, calls=%d", mock.executions)
+	}
+}
+
+func TestClientCreditControlIsNotAvailableInCLI(t *testing.T) {
+	installDataTestTransport(t)
+	var stdout, stderr bytes.Buffer
+	code := Run([]string{"execute", "particle/podcasts/episodes/search", "--inputs", `{"semantic_search":"AI agents"}`, "--max-credits", "1"}, &stdout, &stderr)
+	if code != 1 || !strings.Contains(stderr.String(), `"code": "unsupported_flag"`) {
+		t.Fatalf("old client credit option should fail clearly, status=%d error=%s", code, stderr.String())
+	}
+}
+
+func TestCPIInterpretationDoesNotConfuseIndexWithInflation(t *testing.T) {
+	levels := dataMeasurementNote(map[string]any{"series_id": "CPIAUCSL", "units": "lin"})
+	if !strings.Contains(levels, "index levels") || !strings.Contains(levels, "not annual inflation percentages") {
+		t.Fatalf("CPI level explanation missing: %s", levels)
+	}
+	percentage := dataMeasurementNote(map[string]any{"series_id": "CPIAUCSL", "units": "pc1"})
+	if !strings.Contains(percentage, "percent change from one year ago") {
+		t.Fatalf("transformation note missing: %s", percentage)
+	}
+	if dataMeasurementNote(map[string]any{"series_id": "UNRATE", "units": "lin"}) != "" {
+		t.Fatal("unemployment must not get CPI-specific semantics")
+	}
+}
+
+func TestGeneratedFREDExampleUsesDocumentedSeries(t *testing.T) {
+	c := map[string]any{
+		"id": "fred-stlouisfed-org/economic-data/series_observations",
+		"options": []any{
+			map[string]any{"name": "series_id", "type": "string", "required": true, "about": "FRED series id, e.g. GDP, UNRATE and CPIAUCSL"},
+			map[string]any{"name": "limit", "type": "number", "default": 1000, "min": 1, "max": 10000},
+		},
+	}
+	examples := listField(contractExamples(c))
+	if len(examples) != 1 {
+		t.Fatalf("no fallback example: %#v", examples)
+	}
+	example := mapField(examples[0])
+	inputs := mapField(example["inputs"])
+	if example["kind"] != "generated_from_inspected_schema" || example["valid_input_shape"] != true ||
+		inputs["series_id"] != "CPIAUCSL" || inputs["limit"] != 5 {
+		t.Fatalf("unexpected generated FRED sample: %#v", examples)
+	}
+	if mapField(example["next"])["tool"] != "execute" {
+		t.Fatalf("generated example cannot be copied into execute: %#v", example)
+	}
+}

package/internal/app/data_types.go

@@ -1,20 +1,43 @@
 package app
 
+import (
+	"bytes"
+	"encoding/json"
+)
+
 // ResearchData discovers data sources and their available operations. Search
 // and catalogue browsing are both free Firecrawl Alexandria operations.
 type ResearchDataInput struct {
-	Query      string   `json:"query,omitempty" jsonschema:"Optional natural-language description of the information needed"`
-	Mode       string   `json:"mode,omitempty" jsonschema:"ranked semantic discovery or catalogue browsing with filters and pagination; defaults to ranked for a query"`
-	URLs       []string `json:"urls,omitempty" jsonschema:"Filter by websites associated with a data source"`
-	Sources    []string `json:"sources,omitempty" jsonschema:"Filter by source IDs returned by research"`
-	Categories []string `json:"categories,omitempty" jsonschema:"Filter by catalogue category IDs"`
-	Groups     []string `json:"groups,omitempty" jsonschema:"Filter by catalogue group IDs"`
-	Operations []string `json:"operations,omitempty" jsonschema:"Filter by exact operation IDs"`
-	View       string   `json:"view,omitempty" jsonschema:"What to browse: sources, groups, or operations; default operations for a query, otherwise sources"`
-	Include    []string `json:"include,omitempty" jsonschema:"Additional contract details: inputs, output, examples. Default is a concise overview"`
-	Limit      int      `json:"limit,omitempty" jsonschema:"Maximum results in this page, 1-100; default 10"`
-	Offset     int      `json:"offset,omitempty" jsonschema:"Catalogue pagination offset, zero-based"`
-	Raw        bool     `json:"raw,omitempty" jsonschema:"Also include the complete original Firecrawl response"`
+	Query         string   `json:"query,omitempty" jsonschema:"Optional natural-language description of the information needed"`
+	Mode          string   `json:"mode,omitempty" jsonschema:"ranked semantic discovery or catalogue browsing with filters and pagination; defaults to ranked for a query"`
+	URLs          []string `json:"urls,omitempty" jsonschema:"Filter by websites associated with a data source"`
+	Sources       []string `json:"sources,omitempty" jsonschema:"Filter by source IDs returned by research"`
+	Categories    []string `json:"categories,omitempty" jsonschema:"Filter by catalogue category IDs"`
+	Groups        []string `json:"groups,omitempty" jsonschema:"Filter by catalogue group IDs"`
+	Operations    []string `json:"operations,omitempty" jsonschema:"Filter by exact operation IDs"`
+	View          string   `json:"view,omitempty" jsonschema:"What to browse: sources, groups, or operations; default operations for a query, otherwise sources"`
+	Include       []string `json:"include,omitempty" jsonschema:"Additional contract details: inputs, output, examples. Default is a concise overview"`
+	Limit         int      `json:"limit,omitempty" jsonschema:"Maximum results in this page, 1-100; default 10"`
+	Offset        int      `json:"offset,omitempty" jsonschema:"Catalogue pagination offset, zero-based"`
+	Raw           bool     `json:"raw,omitempty" jsonschema:"Also include the complete original Firecrawl response"`
+	limitProvided bool     `json:"-"`
+}
+
+func (input *ResearchDataInput) UnmarshalJSON(raw []byte) error {
+	type noMethods ResearchDataInput
+	var parsed noMethods
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&parsed); err != nil {
+		return err
+	}
+	var fields map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &fields); err != nil {
+		return err
+	}
+	*input = ResearchDataInput(parsed)
+	_, input.limitProvided = fields["limit"]
+	return nil
 }
 
 // InspectDataInput returns the exact contract for a selected operation, or
@@ -33,8 +56,7 @@ type DataCall struct {
 // ExecuteDataInput supports up to ten calls in a single provider request.
 // It validates each call against the free catalogue before using credits.
 type ExecuteDataInput struct {
-	Calls      []DataCall `json:"calls" jsonschema:"One to ten operation calls; inspect each operation to discover exact input field names"`
-	MaxCredits int        `json:"max_credits,omitempty" jsonschema:"Maximum predicted credits for this request; defaults to WEBCTX_ALEXANDRIA_MAX_CREDITS or 100"`
-	RequestID  string     `json:"request_id,omitempty" jsonschema:"Optional stable idempotency identifier for retrying a paid request; reuse unchanged if retrying"`
-	Raw        bool       `json:"raw,omitempty" jsonschema:"Include the complete original provider response"`
+	Calls     []DataCall `json:"calls" jsonschema:"One to ten operation calls; inspect each operation to discover exact input field names"`
+	RequestID string     `json:"request_id,omitempty" jsonschema:"Optional stable idempotency identifier for retrying a paid request; reuse unchanged if retrying"`
+	Raw       bool       `json:"raw,omitempty" jsonschema:"Include the complete original provider response"`
 }