webctx 0.2.7 → 0.2.8

package/internal/app/app.go

@@ -89,6 +89,7 @@ func Run(args []string, stdout, stderr io.Writer) int {
 			result, e := MapSitePage(MapSiteInput{
 				URL: input, Limit: limit, Offset: offset, Query: flags["query"],
 				PathPrefix: flags["path-prefix"], Language: flags["language"],
+				limitProvided: flags["limit"] != "",
 			})
 			if e != nil {
 				return cliDataError(stderr, e)

package/internal/app/data_audit_test.go

@@ -34,6 +34,16 @@ func TestAuditPerRecordBoundsFromActualOperations(t *testing.T) {
 				"options": []any{map[string]any{"name": "ids", "type": "array"}}},
 			map[string]any{"ids": []any{"A", "B"}}, 4,
 		},
+		{
+			map[string]any{"id": "fullenrich/contacts/mobile", "creditsCost": 175, "perRecord": true,
+				"response": map[string]any{"paginated": true, "about": "One record in results when a mobile number was found, an empty array when not."}},
+			map[string]any{"linkedin_url": "https://www.linkedin.com/in/example-test-person/"}, 175,
+		},
+		{
+			map[string]any{"id": "example/search", "creditsCost": 2, "perRecord": true,
+				"options": []any{map[string]any{"name": "k", "type": "integer"}, map[string]any{"name": "limit", "type": "integer"}}},
+			map[string]any{"limit": 1, "k": 120}, 240,
+		},
 	}
 	for _, tt := range tests {
 		got, err := expectedCallCredits(tt.contract, tt.inputs)
@@ -43,6 +53,92 @@ func TestAuditPerRecordBoundsFromActualOperations(t *testing.T) {
 	}
 }
 
+func TestFilteredCataloguePaginationBeyondFirstHundred(t *testing.T) {
+	previous := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = previous })
+	t.Setenv("FIRECRAWL_API_KEY", "test-filtered-pagination")
+	const total = 215
+	var upstreamOffsets []int
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Path != "/v2/scrape" {
+			return nil, fmt.Errorf("unexpected endpoint %s", req.URL.Path)
+		}
+		var body map[string]any
+		if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
+			return nil, err
+		}
+		options := mapField(mapField(body["alexandria"])["options"])
+		offset, _ := numberField(options["offset"])
+		upstreamOffsets = append(upstreamOffsets, int(offset))
+		start, end := int(offset), int(offset)+100
+		if end > total {
+			end = total
+		}
+		var items []any
+		for i := start; i < end; i++ {
+			items = append(items, map[string]any{
+				"id":   fmt.Sprintf("fred-stlouisfed-org/data/series_%03d", i),
+				"name": "Inflation series observations", "description": "Retrieve inflation observations",
+				"creditsCost": 5, "provider": "fred-stlouisfed-org",
+			})
+		}
+		encoded, _ := json.Marshal(map[string]any{"success": true, "data": map[string]any{
+			"alexandria": []any{map[string]any{"provider": "firecrawl", "capability": "find-tools",
+				"data": map[string]any{"items": items, "total": total}}}}})
+		return testHTTPResponse(req, 200, string(encoded), nil), nil
+	})}
+	input := ResearchDataInput{Query: "inflation series", Mode: "ranked", Sources: []string{"fred-stlouisfed-org"}, Limit: 10}
+	first, err := ResearchData(input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first["ranking_incomplete"] != true || first["ranking_scope"] != "catalogue_page" {
+		t.Fatalf("ranked first page should disclose scope: %+v", first)
+	}
+	seen := map[string]bool{}
+	pages := 0
+	next := first
+	for {
+		pages++
+		rows := next["results"].([]map[string]any)
+		if len(rows) == 0 {
+			t.Fatalf("empty page %d", pages)
+		}
+		for _, row := range rows {
+			id := row["id"].(string)
+			if seen[id] {
+				t.Fatalf("pagination repeated %s", id)
+			}
+			seen[id] = true
+		}
+		cont := mapField(next["next"])
+		if cont == nil {
+			break
+		}
+		if pages > 30 {
+			t.Fatal("pagination did not terminate")
+		}
+		args := cont["arguments"].(ResearchDataInput)
+		next, err = ResearchData(args)
+		if err != nil {
+			t.Fatalf("page %d: %v", pages, err)
+		}
+	}
+	if len(seen) != total || pages != 22 {
+		t.Fatalf("pagination missed items: got %d of %d over %d pages", len(seen), total, pages)
+	}
+	hasSecondBatch := false
+	for _, offset := range upstreamOffsets {
+		if offset == 100 {
+			hasSecondBatch = true
+			break
+		}
+	}
+	if !hasSecondBatch {
+		t.Fatalf("never fetched beyond the first 100 entries: %v", upstreamOffsets)
+	}
+}
+
 func TestAuditPaginationAdaptersPreserveFilters(t *testing.T) {
 	tests := []struct {
 		name   string
@@ -327,6 +423,46 @@ func TestAuditWorkerIdempotentReplayAndCollision(t *testing.T) {
 	}
 }
 
+func TestConcurrentSameIDHasExactlyOnePaidExecution(t *testing.T) {
+	mock := installDataTestTransport(t)
+	const workers = 12
+	id := "audit-parallel-replay-20261010"
+	input := ExecuteDataInput{RequestID: id, Calls: []DataCall{{
+		ID:     "particle/podcasts/episodes/search",
+		Inputs: map[string]any{"semantic_search": "AI agents", "limit": float64(2)},
+	}}}
+	start := make(chan struct{})
+	type result struct {
+		output map[string]any
+		err    error
+	}
+	out := make(chan result, workers)
+	for range workers {
+		go func() {
+			<-start
+			response, err := ExecuteData(input)
+			out <- result{response, err}
+		}()
+	}
+	close(start)
+	replays := 0
+	for range workers {
+		r := <-out
+		if r.err != nil {
+			t.Fatalf("same-ID concurrent request failed: %v", r.err)
+		}
+		if r.output["replayed"] == true {
+			replays++
+		}
+	}
+	mock.mu.Lock()
+	calls := mock.executions
+	mock.mu.Unlock()
+	if calls != 1 || replays != workers-1 {
+		t.Fatalf("concurrent same-ID calls executed %d provider requests; %d cached replays; wanted 1 and %d", calls, replays, workers-1)
+	}
+}
+
 func TestAuditMixedTermsBatchKeepsIndependentResultsAndCredits(t *testing.T) {
 	previous := http.DefaultClient
 	t.Cleanup(func() { http.DefaultClient = previous })
@@ -374,7 +510,7 @@ func TestAuditMixedTermsBatchKeepsIndependentResultsAndCredits(t *testing.T) {
 		{ID: "fred/data/search", Inputs: map[string]any{"query": "inflation"}},
 		{ID: "sec/data/search", Inputs: map[string]any{"query": "filings"}},
 	}
-	result, err := ExecuteData(ExecuteDataInput{Calls: calls})
+	result, err := ExecuteData(ExecuteDataInput{Calls: calls, RequestID: "audit-partial-replay-20261010"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -400,6 +536,11 @@ func TestAuditMixedTermsBatchKeepsIndependentResultsAndCredits(t *testing.T) {
 	if paid["particle"] != 1 || paid["fred"] != 1 || paid["sec"] != 1 {
 		t.Fatalf("provider calls were not independent: %+v", paid)
 	}
+	replayed, err := ExecuteData(ExecuteDataInput{Calls: calls, RequestID: "audit-partial-replay-20261010"})
+	if err != nil || replayed["replayed"] != true || paid["fred"] != 1 || paid["sec"] != 1 ||
+		replayed["credits_charged_this_request"] != 0 {
+		t.Fatalf("replay of partial batch re-executed charged successes: reply=%+v paid=%+v error=%v", replayed, paid, err)
+	}
 }
 
 func TestAuditHighPriceOperationCannotBeRecommendedAsRunnable(t *testing.T) {

package/internal/app/data_execute.go

@@ -210,14 +210,16 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
 		count := 0.0
 		for _, key := range []string{"limit", "k", "top_k", "topK", "per_page", "page_size", "max_results", "count", "size", "num_results", "max_records", "results_per_page"} {
 			if n, ok := numberField(inputs[key]); ok && n >= 1 {
-				count = n
-				break
+				// Providers occasionally accept several independent count
+				// fields. Use the largest explicit bound instead of
+				// underestimating charges from the first matching key.
+				count = math.Max(count, n)
 			}
 		}
 		// For bulk identifier lookups, a response may be charged per input.
 		for _, key := range []string{"ids", "domains", "urls", "companies", "people", "symbols"} {
-			if records := listField(inputs[key]); count == 0 && len(records) > 0 {
-				count = float64(len(records))
+			if records := listField(inputs[key]); len(records) > 0 {
+				count = math.Max(count, float64(len(records)))
 			}
 		}
 		if count == 0 {
@@ -226,7 +228,7 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
 				for _, key := range []string{"limit", "k", "top_k", "topK", "per_page", "page_size", "max_results", "count", "size", "num_results", "max_records", "results_per_page"} {
 					if stringField(f["name"]) == key {
 						if n, ok := numberField(f["default"]); ok && n >= 1 {
-							count = n
+							count = math.Max(count, n)
 						}
 					}
 				}
@@ -252,6 +254,20 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
 					count = 1
 				}
 			}
+			// FullEnrich's contact mobile enrichment is per one result even
+			// though its contract sets paginated=true. The response expressly
+			// states "One record in results", with empty/no-match free.
+			response := mapField(contract["response"])
+			description := strings.ToLower(stringField(response["about"]))
+			if count == 0 && strings.Contains(description, "one record in results") &&
+				(strings.HasSuffix(id, "/mobile") || strings.HasSuffix(id, "/lookup")) {
+				for _, key := range []string{"linkedin_url", "first_name", "domain", "professional_network_url"} {
+					if strings.TrimSpace(stringField(inputs[key])) != "" {
+						count = 1
+						break
+					}
+				}
+			}
 		}
 		if count == 0 {
 			return 0, dataError("unbounded_per_record_cost",
@@ -476,6 +492,8 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
 		}
 	}
 	if input.RequestID != "" {
+		unlock := lockPaidRequest(requestID)
+		defer unlock()
 		if cached, err := cachedPaidReplay(requestID, input.Calls); err != nil {
 			return nil, err
 		} else if cached != nil {
@@ -540,9 +558,10 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
 	if input.Raw {
 		output["raw"] = rawResponses
 	}
-	if failed == 0 {
-		storePaidReplay(requestID, input.Calls, output)
-	}
+	// Cache partial responses too. Retrying a mixed batch must not resubmit
+	// already successful paid operations just because a different provider
+	// rejected its terms.
+	storePaidReplay(requestID, input.Calls, output)
 	return output, nil
 }
 

package/internal/app/data_replay.go

@@ -20,6 +20,41 @@ var paidReplayMemory = struct {
 	items map[string]localPaidReplay
 }{items: map[string]localPaidReplay{}}
 
+// In a single warm worker, serialize requests with the same idempotency key.
+// Without this, two concurrent requests can both miss the replay cache and
+// execute before either writes the receipt.
+type paidReplayGate struct {
+	mu    sync.Mutex
+	users int
+}
+
+var paidReplayGates = struct {
+	sync.Mutex
+	items map[string]*paidReplayGate
+}{items: map[string]*paidReplayGate{}}
+
+func lockPaidRequest(id string) func() {
+	key := localReplayKey(id)
+	paidReplayGates.Lock()
+	gate := paidReplayGates.items[key]
+	if gate == nil {
+		gate = &paidReplayGate{}
+		paidReplayGates.items[key] = gate
+	}
+	gate.users++
+	paidReplayGates.Unlock()
+	gate.mu.Lock()
+	return func() {
+		gate.mu.Unlock()
+		paidReplayGates.Lock()
+		gate.users--
+		if gate.users == 0 {
+			delete(paidReplayGates.items, key)
+		}
+		paidReplayGates.Unlock()
+	}
+}
+
 func localReplayKey(id string) string {
 	h := sha256.Sum256([]byte(os.Getenv("FIRECRAWL_API_KEY") + "\x00" + id))
 	return hex.EncodeToString(h[:])

package/internal/app/data_research.go

@@ -70,6 +70,12 @@ func normalizeResearchInput(input ResearchDataInput) (ResearchDataInput, string,
 	if input.Offset < 0 || input.Offset > 100000 {
 		return input, "", nil, dataError("invalid_offset", "offset must be between 0 and 100000.")
 	}
+	if input.Mode == "ranked" && input.Offset > 0 && strings.TrimSpace(input.Query) != "" &&
+		len(input.Sources) == 0 && len(input.URLs) == 0 && len(input.Groups) == 0 &&
+		len(input.Categories) == 0 && len(input.Operations) == 0 {
+		return input, "", nil, dataError("ranked_search_not_pageable",
+			"Ranked semantic search has no native pagination cursor. Use mode=catalogue with offset, or supply a source/category filter to browse and rerank catalogue pages.")
+	}
 	for _, target := range input.URLs {
 		if err := validateSourceURL(target); err != nil {
 			return input, "", nil, dataError("invalid_url", "Website filters require public HTTP(S) URLs.")
@@ -212,11 +218,13 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		if localFilteredRanking {
 			// Provider-specified lexical filters can exclude operations whose
 			// descriptions describe the requested data indirectly. Fetch a
-			// catalogue slice from the actual source, then rank it ourselves.
+			// catalogue page from the actual source, then rank that page.
+			// Do not reset every continuation to offset zero: large sources
+			// can have hundreds of operations.
 			delete(options, "query")
 			options["limit"] = 100
-			options["offset"] = 0
-			note = "The source/category filters select candidates first. The question ranks those candidates locally, rather than excluding matching source operations."
+			options["offset"] = (input.Offset / 100) * 100
+			note = "Source/category filters select candidates first. The question ranks each catalogue page locally (up to 100 operations per page); this is not a global ranking across unvisited pages."
 			if input.Mode != "catalogue" {
 				mode = "ranked_filtered"
 			}
@@ -261,7 +269,7 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 	}
 	if localFilteredRanking {
 		count := len(items)
-		start := input.Offset
+		start := input.Offset % 100
 		if start > count {
 			start = count
 		}
@@ -270,12 +278,16 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 			end = count
 		}
 		items = items[start:end]
-		if count > end {
+		sourceTotal, hasTotal := numberField(total)
+		more := count > end || (hasTotal && sourceTotal > float64((input.Offset/100)*100+count))
+		if more {
 			next = map[string]any{"tool": "research", "arguments": ResearchDataInput{
 				Query: input.Query, Mode: input.Mode, URLs: input.URLs, Sources: input.Sources,
 				Categories: input.Categories, Groups: input.Groups, View: niceDataView(view),
-				Include: input.Include, Limit: input.Limit, Offset: end,
+				Include: input.Include, Limit: input.Limit, Offset: input.Offset + len(items),
 			}}
+		} else {
+			next = nil
 		}
 	}
 	results := make([]map[string]any, 0, len(items))
@@ -359,6 +371,12 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 	if note != "" {
 		output["mode_note"] = note
 	}
+	if localFilteredRanking {
+		output["ranking_scope"] = "catalogue_page"
+		if sourceTotal, ok := numberField(total); ok {
+			output["ranking_incomplete"] = sourceTotal > 100
+		}
+	}
 	if len(input.URLs) > 0 && len(input.Sources) > 0 {
 		output["resolved_url_sources"] = input.Sources
 	}

package/internal/app/site_map_page.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -20,12 +21,30 @@ import (
 // MapSiteInput retrieves an agent-sized page rather than dumping thousands
 // of URLs. An omitted language prefers English; "all" allows all locales.
 type MapSiteInput struct {
-	URL        string `json:"url" jsonschema:"Website URL to map; HTTP(S) only"`
-	Query      string `json:"query,omitempty" jsonschema:"Optional topic to filter and rank discovered pages"`
-	Limit      int    `json:"limit,omitempty" jsonschema:"Number of URLs to return, 1-100; defaults to 30"`
-	Offset     int    `json:"offset,omitempty" jsonschema:"Pagination offset, zero-based"`
-	PathPrefix string `json:"path_prefix,omitempty" jsonschema:"Optional URL path prefix, e.g. /api/"`
-	Language   string `json:"language,omitempty" jsonschema:"Locale code such as en or fr; defaults to English, use all for all languages"`
+	URL           string `json:"url" jsonschema:"Website URL to map; HTTP(S) only"`
+	Query         string `json:"query,omitempty" jsonschema:"Optional topic to filter and rank discovered pages"`
+	Limit         int    `json:"limit,omitempty" jsonschema:"Number of URLs to return, 1-100; defaults to 30"`
+	Offset        int    `json:"offset,omitempty" jsonschema:"Pagination offset, zero-based"`
+	PathPrefix    string `json:"path_prefix,omitempty" jsonschema:"Optional URL path prefix, e.g. /api/"`
+	Language      string `json:"language,omitempty" jsonschema:"Locale code such as en or fr; defaults to English, use all for all languages"`
+	limitProvided bool   `json:"-"`
+}
+
+func (input *MapSiteInput) UnmarshalJSON(raw []byte) error {
+	type fields MapSiteInput
+	var parsed fields
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&parsed); err != nil {
+		return err
+	}
+	var presence map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &presence); err != nil {
+		return err
+	}
+	*input = MapSiteInput(parsed)
+	_, input.limitProvided = presence["limit"]
+	return nil
 }
 
 type siteMapCacheValue struct {
@@ -139,7 +158,7 @@ func mapPageLocale(raw string) string {
 }
 
 func MapSitePage(input MapSiteInput) (map[string]any, error) {
-	if input.Limit == 0 {
+	if input.Limit == 0 && !input.limitProvided {
 		input.Limit = 30
 	}
 	if input.Limit < 1 || input.Limit > 100 {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webctx",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "description": "Go CLI for agent-friendly web research, structured-data discovery, and source-native querying",
   "license": "MIT",
   "author": "amxv",

package/pkg/origo/data_test.go

@@ -74,6 +74,7 @@ func TestAuditMCPRejectsZeroLimitAndInvalidSourceURLsWithoutExceptions(t *testin
 		code string
 	}{
 		{"research", map[string]any{"mode": "catalogue", "view": "sources", "limit": 0}, "invalid_limit"},
+		{"map_site", map[string]any{"url": "https://docs.firecrawl.dev", "limit": 0}, "invalid_limit"},
 		{"read_link", map[string]any{"url": "file:///etc/passwd", "question": "summarize"}, "invalid_url"},
 		{"map_site", map[string]any{"url": "file:///etc/passwd"}, "invalid_url"},
 	} {