webctx 0.2.6 → 0.2.8

package/internal/app/data_execute.go

@@ -1,6 +1,9 @@
 package app
 
 import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"math"
@@ -70,6 +73,26 @@ func validateDataInputs(contract map[string]any, inputs map[string]any) error {
 			issues = append(issues, fmt.Sprintf("field %q must be %s", name, stringField(field["type"])))
 			continue
 		}
+		if integerDataInputs[name] {
+			if n, ok := numberField(value); ok && n != math.Trunc(n) {
+				issues = append(issues, fmt.Sprintf("field %q requires an integer value", name))
+			}
+		}
+		for _, key := range []string{"requires", "dependsOn"} {
+			if deps := listField(field[key]); len(deps) > 0 {
+				for _, dep := range deps {
+					dependent := stringField(dep)
+					if dependent != "" && inputs[dependent] == nil {
+						issues = append(issues, fmt.Sprintf("field %q requires accompanying field %q", name, dependent))
+					}
+				}
+			}
+		}
+		for _, other := range listField(field["mutuallyExclusiveWith"]) {
+			if alternate := stringField(other); alternate != "" && inputs[alternate] != nil {
+				issues = append(issues, fmt.Sprintf("fields %q and %q cannot be combined", name, alternate))
+			}
+		}
 		if enum := listField(field["oneOf"]); len(enum) > 0 {
 			found := false
 			for _, allowed := range enum {
@@ -185,27 +208,70 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
 	if perRecord, _ := contract["perRecord"].(bool); perRecord {
 		// A per-record price has no hard upper bound without a record limit.
 		count := 0.0
-		for _, key := range []string{"limit", "per_page", "page_size", "max_results", "count"} {
+		for _, key := range []string{"limit", "k", "top_k", "topK", "per_page", "page_size", "max_results", "count", "size", "num_results", "max_records", "results_per_page"} {
 			if n, ok := numberField(inputs[key]); ok && n >= 1 {
-				count = n
-				break
+				// Providers occasionally accept several independent count
+				// fields. Use the largest explicit bound instead of
+				// underestimating charges from the first matching key.
+				count = math.Max(count, n)
+			}
+		}
+		// For bulk identifier lookups, a response may be charged per input.
+		for _, key := range []string{"ids", "domains", "urls", "companies", "people", "symbols"} {
+			if records := listField(inputs[key]); len(records) > 0 {
+				count = math.Max(count, float64(len(records)))
 			}
 		}
 		if count == 0 {
 			for _, v := range listField(contract["options"]) {
 				f := mapField(v)
-				for _, key := range []string{"limit", "per_page", "page_size", "max_results", "count"} {
+				for _, key := range []string{"limit", "k", "top_k", "topK", "per_page", "page_size", "max_results", "count", "size", "num_results", "max_records", "results_per_page"} {
 					if stringField(f["name"]) == key {
 						if n, ok := numberField(f["default"]); ok && n >= 1 {
-							count = n
+							count = math.Max(count, n)
 						}
 					}
 				}
 			}
 		}
+		if count == 0 {
+			// An exact, non-paginated lookup with an individual identifier has
+			// a single-record upper bound, even without a limit parameter.
+			// Only apply this to known singleton operations, never searches.
+			id := strings.ToLower(stringField(contract["id"]))
+			isLookup := strings.HasSuffix(id, "/lookup") || strings.HasSuffix(id, "/get") ||
+				strings.HasSuffix(id, "/detail") || strings.HasSuffix(id, "/details") ||
+				strings.HasSuffix(id, "/resolve") || strings.HasSuffix(id, "/profile")
+			hasIdentifier := false
+			for _, key := range []string{"domain", "id", "company_id", "person_id", "symbol", "url", "linkedin_url", "repo", "doi"} {
+				if s := stringField(inputs[key]); s != "" {
+					hasIdentifier = true
+					break
+				}
+			}
+			if isLookup && hasIdentifier {
+				if paginated, ok := mapField(contract["response"])["paginated"].(bool); !ok || !paginated {
+					count = 1
+				}
+			}
+			// FullEnrich's contact mobile enrichment is per one result even
+			// though its contract sets paginated=true. The response expressly
+			// states "One record in results", with empty/no-match free.
+			response := mapField(contract["response"])
+			description := strings.ToLower(stringField(response["about"]))
+			if count == 0 && strings.Contains(description, "one record in results") &&
+				(strings.HasSuffix(id, "/mobile") || strings.HasSuffix(id, "/lookup")) {
+				for _, key := range []string{"linkedin_url", "first_name", "domain", "professional_network_url"} {
+					if strings.TrimSpace(stringField(inputs[key])) != "" {
+						count = 1
+						break
+					}
+				}
+			}
+		}
 		if count == 0 {
 			return 0, dataError("unbounded_per_record_cost",
-				"This operation charges per record but exposes no bounded record count. Inspect it and supply an explicit limit before execution.")
+				"This operation charges per record, but the contract does not establish a finite record bound. Inspect its output and supply a supported count parameter or smaller input set. Origo cannot safely invent a limit field.")
 		}
 		estimated = cost * count
 	}
@@ -220,6 +286,9 @@ func continuationForDataResult(id string, originalInputs map[string]any, data an
 	if value == nil {
 		return nil
 	}
+	if next := providerContinuation(id, originalInputs, data); next != nil {
+		return next
+	}
 	if next := mapField(value["next"]); next != nil {
 		if provider := stringField(next["provider"]); provider != "" {
 			if capability := stringField(next["capability"]); capability != "" {
@@ -275,6 +344,94 @@ func copyInputs(original map[string]any) map[string]any {
 	return out
 }
 
+// A multi-operation request uses a distinct, deterministic provider request ID
+// for every operation. The upstream batch endpoint is not transactionally
+// isolated: one provider's terms rejection can abort unrelated operations.
+func operationRequestID(root string, index int, id string, count int) string {
+	if count == 1 {
+		return root
+	}
+	digest := sha256.Sum256([]byte(id))
+	if len(root) > 92 {
+		root = root[:92]
+	}
+	return fmt.Sprintf("%s-%d-%s", root, index, hex.EncodeToString(digest[:6]))
+}
+
+func executeOneDataOperation(ctx context.Context, call DataCall, requestID string, index int, includeRaw bool) (map[string]any, float64, error) {
+	provider, capability, err := parseOperationID(call.ID)
+	if err != nil {
+		return nil, 0, err
+	}
+	options := call.Inputs
+	if options == nil {
+		options = map[string]any{}
+	}
+	raw, err := doDataAPI(ctx, "/scrape", map[string]any{"alexandria": []any{
+		map[string]any{"provider": provider, "capability": capability, "options": options},
+	}}, requestID)
+	if err != nil {
+		return nil, 0, err
+	}
+	payload := mapField(raw["data"])
+	results := listField(payload["alexandria"])
+	if len(results) != 1 {
+		return nil, 0, dataError("invalid_upstream_response", "The provider did not return exactly one result for the requested operation.")
+	}
+	entry := mapField(results[0])
+	if entry == nil {
+		return nil, 0, dataError("invalid_upstream_response", "The provider returned an unreadable result.")
+	}
+	if providerError := entry["error"]; providerError != nil {
+		return nil, 0, &DataError{Code: "provider_operation_failed", Message: fmt.Sprint(providerError)}
+	}
+	credits, _ := numberField(entry["creditsCost"])
+	if total, ok := numberField(payload["creditsCost"]); ok {
+		credits = total
+	}
+	item := map[string]any{
+		"operation_index": index, "id": call.ID, "source_id": provider,
+		"ok": true, "data": entry["data"],
+		"credits_used": credits, "receipt_scope": "upstream_request",
+		"provider_request_id":  requestID,
+		"provider_credit_note": "Provider-native fields named credits may use different units. credits_used is Firecrawl Alexandria billing.",
+	}
+	if token := entry["alexandriaId"]; token != nil {
+		item["data_id"] = token
+		item["receipt_scope"] = "provider_response_or_batch"
+	}
+	if note := dataMeasurementNote(entry["data"]); note != "" {
+		item["measurement_note"] = note
+	}
+	if next := continuationForDataResult(call.ID, call.Inputs, entry["data"]); next != nil {
+		item["next"] = next
+	}
+	if extra := raw["scrape_id"]; extra != nil {
+		item["scrape_id"] = extra
+	}
+	if includeRaw {
+		item["raw"] = raw
+	}
+	return item, credits, nil
+}
+
+func operationErrorResult(call DataCall, index int, requestID string, err error) map[string]any {
+	typed, ok := err.(*DataError)
+	if !ok {
+		typed = &DataError{Code: "execution_failed", Message: err.Error()}
+	}
+	out := map[string]any{
+		"id": call.ID, "operation_index": index, "ok": false,
+		"provider_request_id": requestID, "error": typed,
+		"credits_used": nil, "billing_status": "unknown_if_provider_accepted_request",
+	}
+	if typed.Code == "THIRD_PARTY_DATA_TERMS_REQUIRED" {
+		out["billing_status"] = "blocked_pending_human_terms"
+		out["credits_used"] = 0
+	}
+	return out
+}
+
 // ExecuteData validates contracts and credit estimates before calling any
 // paid provider. The native API accepts at most ten executions per request.
 func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
@@ -306,7 +463,6 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
 	}
 	wg.Wait()
 	var estimated int
-	var calls []map[string]any
 	for i, call := range input.Calls {
 		if checked[i].Err != nil {
 			return nil, checked[i].Err
@@ -326,12 +482,6 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
 				Details: map[string]any{"estimated_credits": estimated, "server_credit_limit": cap},
 			}
 		}
-		provider, capability, _ := parseOperationID(call.ID)
-		options := call.Inputs
-		if options == nil {
-			options = map[string]any{}
-		}
-		calls = append(calls, map[string]any{"provider": provider, "capability": capability, "options": options})
 	}
 	requestID := input.RequestID
 	if requestID == "" {
@@ -341,67 +491,77 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
 			return nil, dataError("internal_error", "Could not generate a request identifier.")
 		}
 	}
-	response, err := doDataAPI(ctx, "/scrape", map[string]any{"alexandria": calls}, requestID)
-	if err != nil {
-		var typed *DataError
-		if errorsAsData(err, &typed) {
-			typed.Details = map[string]any{"request_id": requestID,
-				"retry": map[string]any{"tool": "execute", "arguments": ExecuteDataInput{
-					Calls: input.Calls, RequestID: requestID, Raw: input.Raw,
-				}},
-			}
+	if input.RequestID != "" {
+		unlock := lockPaidRequest(requestID)
+		defer unlock()
+		if cached, err := cachedPaidReplay(requestID, input.Calls); err != nil {
+			return nil, err
+		} else if cached != nil {
+			return cached, nil
 		}
-		return nil, err
-	}
-	payload := mapField(response["data"])
-	nativeResults := listField(payload["alexandria"])
-	if len(nativeResults) == 0 {
-		return nil, dataError("invalid_upstream_response", "Paid request returned no Alexandria execution results.")
 	}
-	results := make([]map[string]any, 0, len(nativeResults))
-	for i, value := range nativeResults {
-		entry := mapField(value)
-		if entry == nil {
-			continue
-		}
-		id := stringField(entry["provider"]) + "/" + stringField(entry["capability"])
-		item := map[string]any{
-			"id":           id,
-			"source_id":    entry["provider"],
-			"data":         entry["data"],
-			"credits_used": entry["creditsCost"],
-		}
-		if explanation := dataMeasurementNote(entry["data"]); explanation != "" {
-			item["measurement_note"] = explanation
-		}
-		if token := entry["alexandriaId"]; token != nil {
-			item["data_id"] = token
-		}
-		if errValue := entry["error"]; errValue != nil {
-			item["error"] = errValue
-		}
-		if i < len(input.Calls) {
-			if next := continuationForDataResult(id, input.Calls[i].Inputs, entry["data"]); next != nil {
-				item["next"] = next
+	results := make([]map[string]any, 0, len(input.Calls))
+	creditsUsed := 0.0
+	failed := 0
+	var rawResponses []any
+	for i, call := range input.Calls {
+		operationID := operationRequestID(requestID, i, call.ID, len(input.Calls))
+		item, charged, err := executeOneDataOperation(ctx, call, operationID, i, input.Raw)
+		if err != nil {
+			// Treat idempotency collisions and contract/terms failures as
+			// independent operation errors. Do not suggest replaying a
+			// conflicting payload with the same id.
+			var typed *DataError
+			if errorsAsData(err, &typed) {
+				if typed.Code != "duplicate_request" && typed.Code != "THIRD_PARTY_DATA_TERMS_REQUIRED" {
+					typed.Details = map[string]any{
+						"request_id": operationID,
+						"recovery":   "For an uncertain network failure, retry the exact same operation and request_id. For a different query or page, omit request_id.",
+					}
+				}
 			}
+			if len(input.Calls) == 1 {
+				return nil, err
+			}
+			failed++
+			results = append(results, operationErrorResult(call, i, operationID, err))
+			continue
 		}
+		creditsUsed += charged
 		results = append(results, item)
+		if input.Raw {
+			rawResponses = append(rawResponses, map[string]any{"operation_index": i, "provider_request_id": operationID, "result": item})
+		}
 	}
 	output := map[string]any{
-		"results":           results,
-		"request_id":        requestID,
-		"estimated_credits": estimated,
-		"pricing_note":      "Estimated credits are calculated before execution; the upstream provider reports actual charges. Origo enforces a server-managed per-request credit limit.",
-		"guidance":          "Use next to fetch additional pages where available. Reuse request_id when retrying the same paid request, but generate a new ID for the next page.",
-	}
-	if cost := payload["creditsCost"]; cost != nil {
-		output["credits_used"] = cost
-	} else if cost := response["creditsUsed"]; cost != nil {
-		output["credits_used"] = cost
+		"results": results, "partial": failed > 0, "failed_operations": failed,
+		"request_id":                   requestID,
+		"estimated_credits":            estimated,
+		"credits_used":                 creditsUsed,
+		"pricing_note":                 "credits_used totals reported Firecrawl charges from successful operations. Failed operations may have indeterminate billing; provider-native credits use their own units.",
+		"guidance":                     "Use each result.next to fetch its next page. Retry an identical operation only with its provider_request_id. A new query/page must use a new request ID.",
+		"replay_status":                "upstream_replay_unverified",
+		"replayed":                     false,
+		"credits_charged_this_request": nil,
+		"original_credits_used":        nil,
+		"billing_note":                 "Firecrawl idempotent replay may return the original credit amount; without a provider replay flag or durable accounting data, Origo cannot assert that a retry incurred a new charge.",
+	}
+	if failed > 0 {
+		output["status"] = "partial_success"
+	}
+	if failed == len(input.Calls) {
+		output["status"] = "all_failed"
+	}
+	if failed == 0 {
+		output["status"] = "success"
 	}
 	if input.Raw {
-		output["raw"] = response
+		output["raw"] = rawResponses
 	}
+	// Cache partial responses too. Retrying a mixed batch must not resubmit
+	// already successful paid operations just because a different provider
+	// rejected its terms.
+	storePaidReplay(requestID, input.Calls, output)
 	return output, nil
 }
 

package/internal/app/data_inspect.go

@@ -72,33 +72,6 @@ func nativeInputTemplate(contract map[string]any) (map[string]any, []string) {
 			seen[name] = true
 		}
 	}
-	for _, group := range listField(contract["requiresOneOf"]) {
-		var matched bool
-		for _, field := range listField(group) {
-			name := stringField(field)
-			if _, already := template[name]; already && template[name] != nil {
-				matched = true
-				break
-			}
-		}
-		if matched {
-			continue
-		}
-		// Show one candidate with a blank value rather than guessing user
-		// intent or silently filling provider-specific search terms.
-		for _, field := range listField(group) {
-			name := stringField(field)
-			if name == "" {
-				continue
-			}
-			template[name] = nil
-			if !seen[name] {
-				need = append(need, name)
-				seen[name] = true
-			}
-			break
-		}
-	}
 	return template, need
 }
 
@@ -125,10 +98,9 @@ func InspectData(input InspectDataInput) (map[string]any, error) {
 			return nil, dataError("invalid_source", "Use an exact source ID returned by research.")
 		}
 		include := input.Include
-		if len(include) == 0 {
-			include = []string{"inputs", "output", "examples"}
-		}
-		list, err := ResearchData(ResearchDataInput{Sources: []string{id}, View: "operations", Include: include, Limit: 25, Raw: input.Raw})
+		// Provider-level discovery is intentionally lightweight: inspect a
+		// particular operation for full field-by-field contracts.
+		list, err := ResearchData(ResearchDataInput{Sources: []string{id}, View: "operations", Include: include, Limit: 12, Raw: input.Raw})
 		if err != nil {
 			return nil, err
 		}
@@ -157,12 +129,23 @@ func InspectData(input InspectDataInput) (map[string]any, error) {
 			"credits_per_call": item["creditsCost"],
 			"per_record":       item["perRecord"],
 		},
-		"inputs":         map[string]any{"fields": item["options"], "requires_one_of": item["requiresOneOf"]},
-		"output":         item["response"],
-		"examples":       contractExamples(item),
-		"input_template": inputs,
-		"next":           map[string]any{"tool": "execute", "arguments": execArguments},
-		"guidance":       "Copy next.arguments into execute, fill any blank required inputs and adjust values according to inputs.fields. Execute uses credits.",
+		"inputs":                   map[string]any{"fields": item["options"], "normalized_fields": normalizedDataInputFields(item), "requires_one_of": item["requiresOneOf"]},
+		"output":                   item["response"],
+		"normalized_output_fields": normalizedDataOutputFields(item),
+		"examples":                 contractExamples(item),
+		"input_template":           inputs,
+		"fill_one_of":              item["requiresOneOf"],
+		"next":                     map[string]any{"tool": "execute", "arguments": execArguments},
+		"guidance":                 "Copy next.arguments into execute, fill any blank required inputs and adjust values according to inputs.fields. Execute uses credits.",
+	}
+	if cost, ok := numberField(item["creditsCost"]); ok && cost > float64(dataCreditCap()) {
+		response["availability"] = "exceeds_server_credit_limit"
+		response["executable"] = false
+		response["server_credit_limit"] = dataCreditCap()
+		delete(response, "next")
+		response["guidance"] = fmt.Sprintf("This operation costs %v credits per call and exceeds Origo's %d-credit request ceiling. It cannot execute under this server configuration.", cost, dataCreditCap())
+	} else {
+		response["executable"] = true
 	}
 	if len(missing) > 0 {
 		response["fill_before_execution"] = missing

package/internal/app/data_pagination.go

@@ -0,0 +1,126 @@
+package app
+
+import (
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+// providerContinuation uses native cursor and page metadata without
+// reconstructing provider URLs. All previously supplied filters remain
+// unchanged; only the relevant paging parameter is added/replaced.
+func providerContinuation(id string, original map[string]any, data any) map[string]any {
+	value := mapField(data)
+	if value == nil {
+		return nil
+	}
+	makeNext := func(key string, token any) map[string]any {
+		if key == "" || token == nil || token == "" {
+			return nil
+		}
+		next := copyInputs(original)
+		if previous, exists := next[key]; exists && previous == token {
+			return nil
+		}
+		next[key] = token
+		return map[string]any{"tool": "execute", "arguments": map[string]any{
+			"calls": []any{map[string]any{"id": id, "inputs": next}},
+		}}
+	}
+	for _, candidate := range []struct {
+		location map[string]any
+		names    []string
+		input    string
+	}{
+		{value, []string{"next_page_token", "nextPageToken", "page_token_next"}, "page_token"},
+		{mapField(value["pagination"]), []string{"next_page_token", "nextPageToken"}, "page_token"},
+		{value, []string{"continuation", "next_continuation"}, "continuation"},
+		{mapField(value["metadata"]), []string{"search_after"}, "search_after"},
+		{mapField(value["meta"]), []string{"search_after"}, "search_after"},
+		{value, []string{"search_after"}, "search_after"},
+		{mapField(value["pagination"]), []string{"next_cursor", "nextCursor"}, "cursor"},
+		{mapField(value["pageInfo"]), []string{"endCursor"}, "cursor"},
+	} {
+		for _, name := range candidate.names {
+			token := candidate.location[name]
+			if token == nil || token == "" {
+				continue
+			}
+			key := candidate.input
+			if key == "page_token" {
+				for _, alias := range []string{"page_token", "pageToken", "pageTokenNext"} {
+					if _, exists := original[alias]; exists {
+						key = alias
+						break
+					}
+				}
+			}
+			if key == "cursor" {
+				for _, alias := range []string{"cursor", "after", "after_cursor"} {
+					if _, exists := original[alias]; exists {
+						key = alias
+						break
+					}
+				}
+			}
+			if next := makeNext(key, token); next != nil {
+				return next
+			}
+		}
+	}
+	// Treasury and other API-backed providers expose the following page as
+	// a URL. We extract only recognized pagination parameters; we never
+	// fetch or execute that URL directly.
+	links := mapField(value["links"])
+	for _, source := range []any{links["next"], mapField(value["paging"])["next"]} {
+		target := stringField(source)
+		if target == "" {
+			continue
+		}
+		u, err := url.Parse(target)
+		if err != nil {
+			continue
+		}
+		q := u.Query()
+		for _, candidate := range []struct{ From, To string }{
+			{"page[number]", "page"}, {"pageNumber", "page"}, {"page", "page"},
+			{"page_number", "page_number"}, {"page[size]", "page_size"},
+			{"pageToken", "page_token"}, {"page_token", "page_token"},
+			{"offset", "offset"}, {"cursor", "cursor"},
+		} {
+			raw := q.Get(candidate.From)
+			if raw == "" {
+				continue
+			}
+			key := candidate.To
+			if candidate.From == "page[number]" {
+				for _, alias := range []string{"page_number", "page", "pageNumber"} {
+					if _, exists := original[alias]; exists {
+						key = alias
+						break
+					}
+				}
+			}
+			token := any(raw)
+			if n, e := strconv.Atoi(raw); e == nil {
+				token = n
+			}
+			if next := makeNext(key, token); next != nil {
+				return next
+			}
+		}
+	}
+	// Some providers only return has_more and a current page number.
+	if more, _ := value["has_more"].(bool); more {
+		for _, name := range []string{"page", "page_number", "pageNumber"} {
+			if n, ok := numberField(original[name]); ok && n >= 0 {
+				return makeNext(name, int(n)+1)
+			}
+		}
+	}
+	if link := stringField(value["next"]); link != "" &&
+		!strings.HasPrefix(strings.ToLower(link), "http") {
+		return makeNext("cursor", link)
+	}
+	return nil
+}