webctx 0.2.5 → 0.2.7

package/internal/app/data_client.go

@@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -14,18 +15,27 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
+
+	"golang.org/x/sync/singleflight"
 )
 
 const dataAPIBase = "https://api.firecrawl.dev/v2"
-const defaultDataCreditBudget = 100
+
+// The server, not a client-provided tool parameter, controls this preflight ceiling.
+// A production override is configured on the Origo Vercel project.
+const defaultDataCreditBudget = 200
 const maxDataResponseBytes int64 = 12 << 20
 
 type DataError struct {
-	Code           string         `json:"code"`
-	Message        string         `json:"message"`
-	RequiresAction map[string]any `json:"requires_action,omitempty"`
-	Details        any            `json:"details,omitempty"`
+	Code              string         `json:"code"`
+	Message           string         `json:"message"`
+	RequiresAction    map[string]any `json:"requires_action,omitempty"`
+	Details           any            `json:"details,omitempty"`
+	RetryAfterSeconds int            `json:"retry_after_seconds,omitempty"`
+	RetryAt           string         `json:"retry_at,omitempty"`
+	Retryable         bool           `json:"retryable,omitempty"`
 }
 
 func (e *DataError) Error() string { return e.Code + ": " + e.Message }
@@ -43,22 +53,108 @@ func dataAPIKey() (string, error) {
 
 func dataCreditCap() int {
 	value, err := strconv.Atoi(strings.TrimSpace(os.Getenv("WEBCTX_ALEXANDRIA_MAX_CREDITS")))
-	if err != nil || value < 1 || value > 100000 {
+	if err != nil || value < 1 || value > defaultDataCreditBudget {
 		return defaultDataCreditBudget
 	}
 	return value
 }
 
-func paidDataEnabled() bool {
-	switch strings.ToLower(strings.TrimSpace(os.Getenv("WEBCTX_ALEXANDRIA_PAID"))) {
-	case "0", "false", "no", "off":
-		return false
-	default:
-		return true
+var safeRequestID = regexp.MustCompile("^[A-Za-z0-9_-]{8,128}$")
+
+var freeDataCooldown struct {
+	sync.Mutex
+	until time.Time
+}
+
+type dataReadRetryKey struct{}
+
+var upstreamDataGate = make(chan struct{}, 3)
+
+type catalogueCacheEntry struct {
+	payload  map[string]any
+	response map[string]any
+	expires  time.Time
+}
+
+var catalogueCache = struct {
+	sync.Mutex
+	entries map[string]catalogueCacheEntry
+}{entries: map[string]catalogueCacheEntry{}}
+var catalogueFlight singleflight.Group
+
+func cacheableCatalogueKey(options map[string]any) string {
+	key, err := dataAPIKey()
+	if err != nil || strings.HasPrefix(key, "test") {
+		return ""
+	}
+	snapshot, err := json.Marshal(options)
+	if err != nil {
+		return ""
 	}
+	hash := sha256.Sum256(append(append([]byte(key), 0), snapshot...))
+	return hex.EncodeToString(hash[:])
 }
 
-var safeRequestID = regexp.MustCompile("^[A-Za-z0-9_-]{8,128}$")
+// Coordinate upstream read traffic per warm worker. Vercel has multiple
+// workers, so this is a local best-effort throttle, not a global quota.
+func dataReadCooldown(ctx context.Context) error {
+	freeDataCooldown.Lock()
+	until := freeDataCooldown.until
+	freeDataCooldown.Unlock()
+	if remaining := time.Until(until); remaining > 0 {
+		if remaining < 4*time.Second {
+			timer := time.NewTimer(remaining)
+			defer timer.Stop()
+			select {
+			case <-timer.C:
+				return nil
+			case <-ctx.Done():
+				return dataError("request_cancelled", "The provider read was cancelled while throttled.")
+			}
+		}
+		return rateLimitError(int(remaining.Seconds())+1, "The provider's request allowance is temporarily exhausted.", "local_cooldown")
+	}
+	return nil
+}
+
+func rateLimitError(seconds int, message, source string) *DataError {
+	if seconds < 1 {
+		seconds = 60
+	}
+	if seconds > 3600 {
+		seconds = 3600
+	}
+	return &DataError{
+		Code: "rate_limited", Message: message, Retryable: true, RetryAfterSeconds: seconds,
+		RetryAt: time.Now().Add(time.Duration(seconds) * time.Second).UTC().Format(time.RFC3339),
+		Details: map[string]any{"retry_after_source": source, "guidance": "Retry this read after the indicated delay. For paid calls, preserve the original request_id and exact inputs."},
+	}
+}
+
+func readRetryDelay(headers http.Header) (int, string) {
+	if raw := strings.TrimSpace(headers.Get("Retry-After")); raw != "" {
+		if n, e := strconv.Atoi(raw); e == nil && n > 0 {
+			return n, "retry-after-header"
+		}
+		if t, e := http.ParseTime(raw); e == nil {
+			seconds := int(time.Until(t).Seconds()) + 1
+			if seconds > 0 {
+				return seconds, "retry-after-header"
+			}
+		}
+	}
+	if raw := strings.TrimSpace(headers.Get("RateLimit-Reset")); raw != "" {
+		if n, e := strconv.ParseInt(raw, 10, 64); e == nil {
+			if n > 1e9 {
+				n = int64(time.Until(time.Unix(n, 0)).Seconds()) + 1
+			}
+			if n > 0 && n < 3601 {
+				return int(n), "ratelimit-reset-header"
+			}
+		}
+	}
+	return 60, "estimated_minute_window"
+}
 
 func newRequestID() (string, error) {
 	var bytes [16]byte
@@ -86,7 +182,18 @@ func doDataAPI(ctx context.Context, endpoint string, payload any, requestID stri
 	if requestID != "" {
 		req.Header.Set("x-request-id", requestID)
 	}
+	if requestID == "" {
+		if err := dataReadCooldown(ctx); err != nil {
+			return nil, err
+		}
+	}
+	select {
+	case upstreamDataGate <- struct{}{}:
+	case <-ctx.Done():
+		return nil, dataError("request_cancelled", "Timed out waiting for an upstream request slot.")
+	}
 	res, err := http.DefaultClient.Do(req)
+	<-upstreamDataGate
 	if err != nil {
 		return nil, &DataError{Code: "network_error", Message: "The provider request did not complete. For a paid call, retry only with the same request_id to prevent duplicate charges."}
 	}
@@ -100,6 +207,10 @@ func doDataAPI(ctx context.Context, endpoint string, payload any, requestID stri
 	}
 	var decoded map[string]any
 	if json.Unmarshal(raw, &decoded) != nil {
+		if res.StatusCode == http.StatusTooManyRequests {
+			seconds, source := readRetryDelay(res.Header)
+			return nil, rateLimitError(seconds, "The upstream data provider is rate limiting requests.", source)
+		}
 		return nil, dataError("invalid_upstream_response", fmt.Sprintf("Provider returned non-JSON HTTP %d.", res.StatusCode))
 	}
 	if res.StatusCode < 200 || res.StatusCode >= 300 || decoded["success"] == false {
@@ -117,7 +228,36 @@ func doDataAPI(ctx context.Context, endpoint string, payload any, requestID stri
 		if len(message) > 800 {
 			message = message[:800]
 		}
+		if res.StatusCode == http.StatusTooManyRequests || strings.EqualFold(code, "rate_limit_exceeded") || strings.EqualFold(code, "rate_limited") {
+			seconds, source := readRetryDelay(res.Header)
+			if requestID == "" {
+				freeDataCooldown.Lock()
+				next := time.Now().Add(time.Duration(seconds) * time.Second)
+				if next.After(freeDataCooldown.until) {
+					freeDataCooldown.until = next
+				}
+				freeDataCooldown.Unlock()
+			}
+			// A short retry is safe for free discovery. Never automatically
+			// repeat a paid execution: its actual charging state may be unknown.
+			attempt, _ := ctx.Value(dataReadRetryKey{}).(int)
+			if requestID == "" && seconds <= 2 && attempt == 0 {
+				timer := time.NewTimer(time.Duration(seconds) * time.Second)
+				defer timer.Stop()
+				select {
+				case <-timer.C:
+					return doDataAPI(context.WithValue(ctx, dataReadRetryKey{}, 1), endpoint, payload, requestID)
+				case <-ctx.Done():
+					return nil, rateLimitError(seconds, "The provider's allowance has not recovered yet.", source)
+				}
+			}
+			return nil, rateLimitError(seconds, "The upstream data provider is rate limiting requests.", source)
+		}
 		out := &DataError{Code: code, Message: message}
+		if strings.EqualFold(code, "duplicate_request") {
+			out.Message = "This request_id was already used with different inputs. Reuse the exact original request body to replay it, or omit request_id to start a new query."
+			out.Details = map[string]any{"guidance": "Do not retry the conflicting payload with the same request_id.", "recovery": "Restore the exact original calls for replay, or generate a fresh request_id for intentionally new calls."}
+		}
 		if action, ok := decoded["requiresAction"].(map[string]any); ok {
 			if target := stringValue(action["url"]); target != "" {
 				if u, e := url.Parse(target); e == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" {
@@ -154,6 +294,43 @@ func extractNativeData(response map[string]any) (map[string]any, error) {
 }
 
 func freeDataBrowse(ctx context.Context, options map[string]any) (map[string]any, map[string]any, error) {
+	cacheKey := cacheableCatalogueKey(options)
+	if cacheKey != "" {
+		catalogueCache.Lock()
+		cached, ok := catalogueCache.entries[cacheKey]
+		catalogueCache.Unlock()
+		if ok && time.Now().Before(cached.expires) {
+			return cached.payload, cached.response, nil
+		}
+		type cacheResult struct {
+			payload  map[string]any
+			response map[string]any
+		}
+		result, err, _ := catalogueFlight.Do(cacheKey, func() (any, error) {
+			data, raw, e := uncachedFreeDataBrowse(ctx, options)
+			if e != nil {
+				return nil, e
+			}
+			catalogueCache.Lock()
+			if len(catalogueCache.entries) > 256 {
+				catalogueCache.entries = map[string]catalogueCacheEntry{}
+			}
+			catalogueCache.entries[cacheKey] = catalogueCacheEntry{
+				payload: data, response: raw, expires: time.Now().Add(2 * time.Minute),
+			}
+			catalogueCache.Unlock()
+			return cacheResult{payload: data, response: raw}, nil
+		})
+		if err != nil {
+			return nil, nil, err
+		}
+		got := result.(cacheResult)
+		return got.payload, got.response, nil
+	}
+	return uncachedFreeDataBrowse(ctx, options)
+}
+
+func uncachedFreeDataBrowse(ctx context.Context, options map[string]any) (map[string]any, map[string]any, error) {
 	response, err := nativeDataCall(ctx, "firecrawl", "find-tools", options, "")
 	if err != nil {
 		return nil, nil, err
@@ -164,7 +341,12 @@ func freeDataBrowse(ctx context.Context, options map[string]any) (map[string]any
 	}
 	details, ok := item["data"].(map[string]any)
 	if !ok {
-		return nil, nil, dataError("invalid_upstream_response", "The catalogue returned no browsable entries.")
+		if item["data"] == nil && item["error"] == nil {
+			// Some Alexandria catalogues signal an unknown ID by returning
+			// a successful find-tools call with data=null.
+			return map[string]any{"items": []any{}, "total": 0}, response, nil
+		}
+		return nil, nil, dataError("invalid_upstream_response", "The catalogue returned an unreadable response. Retry discovery before selecting an operation.")
 	}
 	return details, response, nil
 }

package/internal/app/data_contract_constraints.go

@@ -0,0 +1,65 @@
+package app
+
+import (
+	"strings"
+)
+
+var integerDataInputs = map[string]bool{
+	"page": true, "page_number": true, "pageNumber": true,
+	"page_size": true, "per_page": true, "offset": true,
+	"limit": true, "k": true, "top_k": true, "topK": true,
+	"count": true, "size": true, "max_results": true,
+	"num_results": true, "max_records": true, "results_per_page": true,
+}
+
+// Return normalized, machine-readable hints alongside the untouched native
+// field contract. Do not overwrite the upstream schema's declared types.
+func normalizedDataInputFields(contract map[string]any) []map[string]any {
+	fields := make([]map[string]any, 0)
+	for _, v := range listField(contract["options"]) {
+		item := mapField(v)
+		if item == nil {
+			continue
+		}
+		name := stringField(item["name"])
+		if name == "" {
+			continue
+		}
+		normalized := map[string]any{"name": name, "type": item["type"]}
+		typ := strings.ToLower(stringField(item["type"]))
+		description := strings.ToLower(stringField(item["about"]))
+		if typ == "number" && (integerDataInputs[name] || strings.Contains(description, "integer")) {
+			normalized["type"] = "integer"
+			normalized["original_type"] = item["type"]
+		}
+		for _, key := range []string{"pattern", "required", "min", "max", "default", "oneOf", "nullable", "requires", "dependsOn", "mutuallyExclusiveWith"} {
+			if value, ok := item[key]; ok {
+				normalized[key] = value
+			}
+		}
+		fields = append(fields, normalized)
+	}
+	return fields
+}
+
+func normalizedDataOutputFields(contract map[string]any) []map[string]any {
+	response := mapField(contract["response"])
+	fields := make([]map[string]any, 0)
+	for _, v := range listField(response["fields"]) {
+		item := mapField(v)
+		if item == nil {
+			continue
+		}
+		normalized := map[string]any{"name": item["name"], "type": item["type"]}
+		about := strings.ToLower(stringField(item["about"]))
+		nullable, _ := item["nullable"].(bool)
+		if strings.Contains(about, "null") || strings.Contains(about, "missing value") {
+			nullable = true
+		}
+		if nullable {
+			normalized["nullable"] = true
+		}
+		fields = append(fields, normalized)
+	}
+	return fields
+}

package/internal/app/data_examples.go

@@ -0,0 +1,185 @@
+package app
+
+import (
+	"regexp"
+	"strings"
+)
+
+var backtickedIdentifier = regexp.MustCompile("`([A-Za-z][A-Za-z0-9_/-]{1,30})`")
+
+// Upstream examples can legitimately be absent. Offer a clearly marked,
+// locally validated sample input without claiming it was published by the
+// provider or that it was executed.
+func contractExamples(contract map[string]any) any {
+	if published := contract["examples"]; published != nil {
+		if len(listField(published)) > 0 {
+			return published
+		}
+		if len(mapField(published)) > 0 {
+			return published
+		}
+	}
+	input, needed := nativeInputTemplate(contract)
+	fieldInfo := map[string]map[string]any{}
+	for _, v := range listField(contract["options"]) {
+		if field := mapField(v); field != nil {
+			fieldInfo[stringField(field["name"])] = field
+		}
+	}
+	for _, name := range needed {
+		field := fieldInfo[name]
+		if value, ok := sampleFieldValue(name, field); ok {
+			input[name] = value
+		}
+	}
+	var chooseOne []any
+	for _, v := range listField(contract["requiresOneOf"]) {
+		alternatives := listField(v)
+		satisfied := false
+		for _, candidate := range alternatives {
+			name := stringField(candidate)
+			if input[name] != nil && input[name] != "" {
+				satisfied = true
+				break
+			}
+		}
+		if satisfied {
+			continue
+		}
+		for _, candidate := range alternatives {
+			name := stringField(candidate)
+			if value, ok := sampleFieldValue(name, fieldInfo[name]); ok {
+				input[name] = value
+				satisfied = true
+				break
+			}
+		}
+		if !satisfied {
+			chooseOne = append(chooseOne, alternatives)
+		}
+	}
+	// An illustrative example should not accidentally fetch huge datasets
+	// when copied verbatim. Users may adjust the inputs freely.
+	for _, name := range []string{"limit", "per_page", "page_size", "max_results", "count"} {
+		if n, ok := numberField(input[name]); ok && n > 5 {
+			input[name] = 5
+		}
+	}
+	if _, _, err := parseOperationID(stringField(contract["id"])); err != nil {
+		return nil
+	}
+	validation := validateDataInputs(contract, input)
+	practical := validation == nil
+	if len(input) == 0 && len(listField(contract["options"])) > 0 {
+		practical = false
+	}
+	// A purely structural URL or a placeholder never proves a real record
+	// exists. High-credit person-profile lookups must use a real user URL.
+	for name, value := range input {
+		lower := strings.ToLower(name)
+		s := stringField(value)
+		if strings.Contains(lower, "linkedin") && strings.Contains(s, "example") {
+			practical = false
+		}
+		if strings.Contains(strings.ToLower(s), "placeholder") ||
+			strings.Contains(strings.ToLower(s), "identifier") {
+			practical = false
+		}
+	}
+	id := strings.ToLower(stringField(contract["id"]))
+	if (strings.HasSuffix(id, "/get") || strings.HasSuffix(id, "/lookup") ||
+		strings.HasSuffix(id, "/detail")) && len(input) == 0 {
+		practical = false
+	}
+	example := map[string]any{
+		"kind":               "generated_from_inspected_schema",
+		"note":               "Illustrative Origo-generated inputs; not a published provider example or an executed query.",
+		"inputs":             input,
+		"structurally_valid": validation == nil,
+		"valid_input_shape":  validation == nil && practical,
+		"ready_to_execute":   practical,
+	}
+	if validation != nil || !practical {
+		example["note"] = "Illustrative input template; fill required fields and check source-specific constraints before execution."
+		example["needs_input"] = needed
+		if len(chooseOne) > 0 {
+			example["needs_one_of"] = chooseOne
+		}
+	} else {
+		example["next"] = map[string]any{"tool": "execute", "arguments": map[string]any{
+			"calls": []any{map[string]any{"id": contract["id"], "inputs": input}},
+		}}
+	}
+	return []any{example}
+}
+
+func sampleFieldValue(name string, field map[string]any) (any, bool) {
+	if field == nil {
+		return nil, false
+	}
+	if values := listField(field["oneOf"]); len(values) > 0 {
+		for _, v := range values {
+			if v != nil && v != "" {
+				return v, true
+			}
+		}
+	}
+	if sample, ok := field["example"]; ok && sample != nil {
+		if s := stringField(sample); s != "" &&
+			(strings.Contains(strings.ToLower(s), "placeholder") ||
+				strings.Contains(strings.ToLower(s), "identifier")) {
+			// Example-like placeholders are not executable source IDs.
+		} else {
+			return sample, true
+		}
+	}
+	if samples := listField(field["examples"]); len(samples) > 0 && samples[0] != nil {
+		return samples[0], true
+	}
+	if v, ok := field["default"]; ok && v != nil {
+		return v, true
+	}
+	info := stringField(field["about"]) + " " + stringField(field["description"])
+	lower := strings.ToLower(name + " " + info)
+	switch {
+	case name == "companyKey":
+		return "NASDAQ_AAPL", true
+	case name == "repo" || name == "repository":
+		return "openai/openai-python", true
+	case name == "symbol" || name == "ticker":
+		return "AAPL", true
+	case name == "from" || name == "base_currency":
+		return "USD", true
+	case name == "to" || name == "quote_currency":
+		return "EUR", true
+	case name == "domain":
+		return "github.com", true
+	case strings.Contains(lower, "linkedin") || strings.Contains(lower, "professional_network_url"):
+		return nil, false
+	case strings.Contains(name, "series_id") && strings.Contains(info, "CPIAUCSL"):
+		return "CPIAUCSL", true
+	case strings.Contains(name, "series_id") && strings.Contains(info, "UNRATE"):
+		return "UNRATE", true
+	case strings.Contains(name, "series_id") && strings.Contains(info, "GDP"):
+		return "GDP", true
+	case strings.Contains(lower, "semantic_search"):
+		return "AI agents", true
+	case name == "query" || name == "search" || name == "keyword_search":
+		return "example topic", true
+	case name == "url" || strings.HasSuffix(name, "_url"):
+		return "https://example.com", true
+	}
+	// Backticked descriptions sometimes contain sample codes or IDs.
+	if strings.Contains(lower, "id") || strings.Contains(lower, "code") {
+		for _, match := range backtickedIdentifier.FindAllStringSubmatch(info, 10) {
+			candidate := match[1]
+			if strings.Contains(candidate, "/") || strings.Contains(candidate, "_") {
+				continue
+			}
+			if len(candidate) >= 3 && !strings.EqualFold(candidate, "required") {
+				return candidate, true
+			}
+		}
+	}
+	return nil, false
+}