webctx 0.2.6 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,9 @@
1
1
  package app
2
2
 
3
3
  import (
4
+ "context"
5
+ "crypto/sha256"
6
+ "encoding/hex"
4
7
  "encoding/json"
5
8
  "fmt"
6
9
  "math"
@@ -70,6 +73,26 @@ func validateDataInputs(contract map[string]any, inputs map[string]any) error {
70
73
  issues = append(issues, fmt.Sprintf("field %q must be %s", name, stringField(field["type"])))
71
74
  continue
72
75
  }
76
+ if integerDataInputs[name] {
77
+ if n, ok := numberField(value); ok && n != math.Trunc(n) {
78
+ issues = append(issues, fmt.Sprintf("field %q requires an integer value", name))
79
+ }
80
+ }
81
+ for _, key := range []string{"requires", "dependsOn"} {
82
+ if deps := listField(field[key]); len(deps) > 0 {
83
+ for _, dep := range deps {
84
+ dependent := stringField(dep)
85
+ if dependent != "" && inputs[dependent] == nil {
86
+ issues = append(issues, fmt.Sprintf("field %q requires accompanying field %q", name, dependent))
87
+ }
88
+ }
89
+ }
90
+ }
91
+ for _, other := range listField(field["mutuallyExclusiveWith"]) {
92
+ if alternate := stringField(other); alternate != "" && inputs[alternate] != nil {
93
+ issues = append(issues, fmt.Sprintf("fields %q and %q cannot be combined", name, alternate))
94
+ }
95
+ }
73
96
  if enum := listField(field["oneOf"]); len(enum) > 0 {
74
97
  found := false
75
98
  for _, allowed := range enum {
@@ -185,16 +208,22 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
185
208
  if perRecord, _ := contract["perRecord"].(bool); perRecord {
186
209
  // A per-record price has no hard upper bound without a record limit.
187
210
  count := 0.0
188
- for _, key := range []string{"limit", "per_page", "page_size", "max_results", "count"} {
211
+ for _, key := range []string{"limit", "k", "top_k", "topK", "per_page", "page_size", "max_results", "count", "size", "num_results", "max_records", "results_per_page"} {
189
212
  if n, ok := numberField(inputs[key]); ok && n >= 1 {
190
213
  count = n
191
214
  break
192
215
  }
193
216
  }
217
+ // For bulk identifier lookups, a response may be charged per input.
218
+ for _, key := range []string{"ids", "domains", "urls", "companies", "people", "symbols"} {
219
+ if records := listField(inputs[key]); count == 0 && len(records) > 0 {
220
+ count = float64(len(records))
221
+ }
222
+ }
194
223
  if count == 0 {
195
224
  for _, v := range listField(contract["options"]) {
196
225
  f := mapField(v)
197
- for _, key := range []string{"limit", "per_page", "page_size", "max_results", "count"} {
226
+ for _, key := range []string{"limit", "k", "top_k", "topK", "per_page", "page_size", "max_results", "count", "size", "num_results", "max_records", "results_per_page"} {
198
227
  if stringField(f["name"]) == key {
199
228
  if n, ok := numberField(f["default"]); ok && n >= 1 {
200
229
  count = n
@@ -203,9 +232,30 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
203
232
  }
204
233
  }
205
234
  }
235
+ if count == 0 {
236
+ // An exact, non-paginated lookup with an individual identifier has
237
+ // a single-record upper bound, even without a limit parameter.
238
+ // Only apply this to known singleton operations, never searches.
239
+ id := strings.ToLower(stringField(contract["id"]))
240
+ isLookup := strings.HasSuffix(id, "/lookup") || strings.HasSuffix(id, "/get") ||
241
+ strings.HasSuffix(id, "/detail") || strings.HasSuffix(id, "/details") ||
242
+ strings.HasSuffix(id, "/resolve") || strings.HasSuffix(id, "/profile")
243
+ hasIdentifier := false
244
+ for _, key := range []string{"domain", "id", "company_id", "person_id", "symbol", "url", "linkedin_url", "repo", "doi"} {
245
+ if s := stringField(inputs[key]); s != "" {
246
+ hasIdentifier = true
247
+ break
248
+ }
249
+ }
250
+ if isLookup && hasIdentifier {
251
+ if paginated, ok := mapField(contract["response"])["paginated"].(bool); !ok || !paginated {
252
+ count = 1
253
+ }
254
+ }
255
+ }
206
256
  if count == 0 {
207
257
  return 0, dataError("unbounded_per_record_cost",
208
- "This operation charges per record but exposes no bounded record count. Inspect it and supply an explicit limit before execution.")
258
+ "This operation charges per record, but the contract does not establish a finite record bound. Inspect its output and supply a supported count parameter or smaller input set. Origo cannot safely invent a limit field.")
209
259
  }
210
260
  estimated = cost * count
211
261
  }
@@ -220,6 +270,9 @@ func continuationForDataResult(id string, originalInputs map[string]any, data an
220
270
  if value == nil {
221
271
  return nil
222
272
  }
273
+ if next := providerContinuation(id, originalInputs, data); next != nil {
274
+ return next
275
+ }
223
276
  if next := mapField(value["next"]); next != nil {
224
277
  if provider := stringField(next["provider"]); provider != "" {
225
278
  if capability := stringField(next["capability"]); capability != "" {
@@ -275,6 +328,94 @@ func copyInputs(original map[string]any) map[string]any {
275
328
  return out
276
329
  }
277
330
 
331
+ // A multi-operation request uses a distinct, deterministic provider request ID
332
+ // for every operation. The upstream batch endpoint is not transactionally
333
+ // isolated: one provider's terms rejection can abort unrelated operations.
334
+ func operationRequestID(root string, index int, id string, count int) string {
335
+ if count == 1 {
336
+ return root
337
+ }
338
+ digest := sha256.Sum256([]byte(id))
339
+ if len(root) > 92 {
340
+ root = root[:92]
341
+ }
342
+ return fmt.Sprintf("%s-%d-%s", root, index, hex.EncodeToString(digest[:6]))
343
+ }
344
+
345
+ func executeOneDataOperation(ctx context.Context, call DataCall, requestID string, index int, includeRaw bool) (map[string]any, float64, error) {
346
+ provider, capability, err := parseOperationID(call.ID)
347
+ if err != nil {
348
+ return nil, 0, err
349
+ }
350
+ options := call.Inputs
351
+ if options == nil {
352
+ options = map[string]any{}
353
+ }
354
+ raw, err := doDataAPI(ctx, "/scrape", map[string]any{"alexandria": []any{
355
+ map[string]any{"provider": provider, "capability": capability, "options": options},
356
+ }}, requestID)
357
+ if err != nil {
358
+ return nil, 0, err
359
+ }
360
+ payload := mapField(raw["data"])
361
+ results := listField(payload["alexandria"])
362
+ if len(results) != 1 {
363
+ return nil, 0, dataError("invalid_upstream_response", "The provider did not return exactly one result for the requested operation.")
364
+ }
365
+ entry := mapField(results[0])
366
+ if entry == nil {
367
+ return nil, 0, dataError("invalid_upstream_response", "The provider returned an unreadable result.")
368
+ }
369
+ if providerError := entry["error"]; providerError != nil {
370
+ return nil, 0, &DataError{Code: "provider_operation_failed", Message: fmt.Sprint(providerError)}
371
+ }
372
+ credits, _ := numberField(entry["creditsCost"])
373
+ if total, ok := numberField(payload["creditsCost"]); ok {
374
+ credits = total
375
+ }
376
+ item := map[string]any{
377
+ "operation_index": index, "id": call.ID, "source_id": provider,
378
+ "ok": true, "data": entry["data"],
379
+ "credits_used": credits, "receipt_scope": "upstream_request",
380
+ "provider_request_id": requestID,
381
+ "provider_credit_note": "Provider-native fields named credits may use different units. credits_used is Firecrawl Alexandria billing.",
382
+ }
383
+ if token := entry["alexandriaId"]; token != nil {
384
+ item["data_id"] = token
385
+ item["receipt_scope"] = "provider_response_or_batch"
386
+ }
387
+ if note := dataMeasurementNote(entry["data"]); note != "" {
388
+ item["measurement_note"] = note
389
+ }
390
+ if next := continuationForDataResult(call.ID, call.Inputs, entry["data"]); next != nil {
391
+ item["next"] = next
392
+ }
393
+ if extra := raw["scrape_id"]; extra != nil {
394
+ item["scrape_id"] = extra
395
+ }
396
+ if includeRaw {
397
+ item["raw"] = raw
398
+ }
399
+ return item, credits, nil
400
+ }
401
+
402
+ func operationErrorResult(call DataCall, index int, requestID string, err error) map[string]any {
403
+ typed, ok := err.(*DataError)
404
+ if !ok {
405
+ typed = &DataError{Code: "execution_failed", Message: err.Error()}
406
+ }
407
+ out := map[string]any{
408
+ "id": call.ID, "operation_index": index, "ok": false,
409
+ "provider_request_id": requestID, "error": typed,
410
+ "credits_used": nil, "billing_status": "unknown_if_provider_accepted_request",
411
+ }
412
+ if typed.Code == "THIRD_PARTY_DATA_TERMS_REQUIRED" {
413
+ out["billing_status"] = "blocked_pending_human_terms"
414
+ out["credits_used"] = 0
415
+ }
416
+ return out
417
+ }
418
+
278
419
  // ExecuteData validates contracts and credit estimates before calling any
279
420
  // paid provider. The native API accepts at most ten executions per request.
280
421
  func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
@@ -306,7 +447,6 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
306
447
  }
307
448
  wg.Wait()
308
449
  var estimated int
309
- var calls []map[string]any
310
450
  for i, call := range input.Calls {
311
451
  if checked[i].Err != nil {
312
452
  return nil, checked[i].Err
@@ -326,12 +466,6 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
326
466
  Details: map[string]any{"estimated_credits": estimated, "server_credit_limit": cap},
327
467
  }
328
468
  }
329
- provider, capability, _ := parseOperationID(call.ID)
330
- options := call.Inputs
331
- if options == nil {
332
- options = map[string]any{}
333
- }
334
- calls = append(calls, map[string]any{"provider": provider, "capability": capability, "options": options})
335
469
  }
336
470
  requestID := input.RequestID
337
471
  if requestID == "" {
@@ -341,66 +475,73 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
341
475
  return nil, dataError("internal_error", "Could not generate a request identifier.")
342
476
  }
343
477
  }
344
- response, err := doDataAPI(ctx, "/scrape", map[string]any{"alexandria": calls}, requestID)
345
- if err != nil {
346
- var typed *DataError
347
- if errorsAsData(err, &typed) {
348
- typed.Details = map[string]any{"request_id": requestID,
349
- "retry": map[string]any{"tool": "execute", "arguments": ExecuteDataInput{
350
- Calls: input.Calls, RequestID: requestID, Raw: input.Raw,
351
- }},
352
- }
478
+ if input.RequestID != "" {
479
+ if cached, err := cachedPaidReplay(requestID, input.Calls); err != nil {
480
+ return nil, err
481
+ } else if cached != nil {
482
+ return cached, nil
353
483
  }
354
- return nil, err
355
- }
356
- payload := mapField(response["data"])
357
- nativeResults := listField(payload["alexandria"])
358
- if len(nativeResults) == 0 {
359
- return nil, dataError("invalid_upstream_response", "Paid request returned no Alexandria execution results.")
360
484
  }
361
- results := make([]map[string]any, 0, len(nativeResults))
362
- for i, value := range nativeResults {
363
- entry := mapField(value)
364
- if entry == nil {
365
- continue
366
- }
367
- id := stringField(entry["provider"]) + "/" + stringField(entry["capability"])
368
- item := map[string]any{
369
- "id": id,
370
- "source_id": entry["provider"],
371
- "data": entry["data"],
372
- "credits_used": entry["creditsCost"],
373
- }
374
- if explanation := dataMeasurementNote(entry["data"]); explanation != "" {
375
- item["measurement_note"] = explanation
376
- }
377
- if token := entry["alexandriaId"]; token != nil {
378
- item["data_id"] = token
379
- }
380
- if errValue := entry["error"]; errValue != nil {
381
- item["error"] = errValue
382
- }
383
- if i < len(input.Calls) {
384
- if next := continuationForDataResult(id, input.Calls[i].Inputs, entry["data"]); next != nil {
385
- item["next"] = next
485
+ results := make([]map[string]any, 0, len(input.Calls))
486
+ creditsUsed := 0.0
487
+ failed := 0
488
+ var rawResponses []any
489
+ for i, call := range input.Calls {
490
+ operationID := operationRequestID(requestID, i, call.ID, len(input.Calls))
491
+ item, charged, err := executeOneDataOperation(ctx, call, operationID, i, input.Raw)
492
+ if err != nil {
493
+ // Treat idempotency collisions and contract/terms failures as
494
+ // independent operation errors. Do not suggest replaying a
495
+ // conflicting payload with the same id.
496
+ var typed *DataError
497
+ if errorsAsData(err, &typed) {
498
+ if typed.Code != "duplicate_request" && typed.Code != "THIRD_PARTY_DATA_TERMS_REQUIRED" {
499
+ typed.Details = map[string]any{
500
+ "request_id": operationID,
501
+ "recovery": "For an uncertain network failure, retry the exact same operation and request_id. For a different query or page, omit request_id.",
502
+ }
503
+ }
386
504
  }
505
+ if len(input.Calls) == 1 {
506
+ return nil, err
507
+ }
508
+ failed++
509
+ results = append(results, operationErrorResult(call, i, operationID, err))
510
+ continue
387
511
  }
512
+ creditsUsed += charged
388
513
  results = append(results, item)
514
+ if input.Raw {
515
+ rawResponses = append(rawResponses, map[string]any{"operation_index": i, "provider_request_id": operationID, "result": item})
516
+ }
389
517
  }
390
518
  output := map[string]any{
391
- "results": results,
392
- "request_id": requestID,
393
- "estimated_credits": estimated,
394
- "pricing_note": "Estimated credits are calculated before execution; the upstream provider reports actual charges. Origo enforces a server-managed per-request credit limit.",
395
- "guidance": "Use next to fetch additional pages where available. Reuse request_id when retrying the same paid request, but generate a new ID for the next page.",
396
- }
397
- if cost := payload["creditsCost"]; cost != nil {
398
- output["credits_used"] = cost
399
- } else if cost := response["creditsUsed"]; cost != nil {
400
- output["credits_used"] = cost
519
+ "results": results, "partial": failed > 0, "failed_operations": failed,
520
+ "request_id": requestID,
521
+ "estimated_credits": estimated,
522
+ "credits_used": creditsUsed,
523
+ "pricing_note": "credits_used totals reported Firecrawl charges from successful operations. Failed operations may have indeterminate billing; provider-native credits use their own units.",
524
+ "guidance": "Use each result.next to fetch its next page. Retry an identical operation only with its provider_request_id. A new query/page must use a new request ID.",
525
+ "replay_status": "upstream_replay_unverified",
526
+ "replayed": false,
527
+ "credits_charged_this_request": nil,
528
+ "original_credits_used": nil,
529
+ "billing_note": "Firecrawl idempotent replay may return the original credit amount; without a provider replay flag or durable accounting data, Origo cannot assert that a retry incurred a new charge.",
530
+ }
531
+ if failed > 0 {
532
+ output["status"] = "partial_success"
533
+ }
534
+ if failed == len(input.Calls) {
535
+ output["status"] = "all_failed"
536
+ }
537
+ if failed == 0 {
538
+ output["status"] = "success"
401
539
  }
402
540
  if input.Raw {
403
- output["raw"] = response
541
+ output["raw"] = rawResponses
542
+ }
543
+ if failed == 0 {
544
+ storePaidReplay(requestID, input.Calls, output)
404
545
  }
405
546
  return output, nil
406
547
  }
@@ -72,33 +72,6 @@ func nativeInputTemplate(contract map[string]any) (map[string]any, []string) {
72
72
  seen[name] = true
73
73
  }
74
74
  }
75
- for _, group := range listField(contract["requiresOneOf"]) {
76
- var matched bool
77
- for _, field := range listField(group) {
78
- name := stringField(field)
79
- if _, already := template[name]; already && template[name] != nil {
80
- matched = true
81
- break
82
- }
83
- }
84
- if matched {
85
- continue
86
- }
87
- // Show one candidate with a blank value rather than guessing user
88
- // intent or silently filling provider-specific search terms.
89
- for _, field := range listField(group) {
90
- name := stringField(field)
91
- if name == "" {
92
- continue
93
- }
94
- template[name] = nil
95
- if !seen[name] {
96
- need = append(need, name)
97
- seen[name] = true
98
- }
99
- break
100
- }
101
- }
102
75
  return template, need
103
76
  }
104
77
 
@@ -125,10 +98,9 @@ func InspectData(input InspectDataInput) (map[string]any, error) {
125
98
  return nil, dataError("invalid_source", "Use an exact source ID returned by research.")
126
99
  }
127
100
  include := input.Include
128
- if len(include) == 0 {
129
- include = []string{"inputs", "output", "examples"}
130
- }
131
- list, err := ResearchData(ResearchDataInput{Sources: []string{id}, View: "operations", Include: include, Limit: 25, Raw: input.Raw})
101
+ // Provider-level discovery is intentionally lightweight: inspect a
102
+ // particular operation for full field-by-field contracts.
103
+ list, err := ResearchData(ResearchDataInput{Sources: []string{id}, View: "operations", Include: include, Limit: 12, Raw: input.Raw})
132
104
  if err != nil {
133
105
  return nil, err
134
106
  }
@@ -157,12 +129,23 @@ func InspectData(input InspectDataInput) (map[string]any, error) {
157
129
  "credits_per_call": item["creditsCost"],
158
130
  "per_record": item["perRecord"],
159
131
  },
160
- "inputs": map[string]any{"fields": item["options"], "requires_one_of": item["requiresOneOf"]},
161
- "output": item["response"],
162
- "examples": contractExamples(item),
163
- "input_template": inputs,
164
- "next": map[string]any{"tool": "execute", "arguments": execArguments},
165
- "guidance": "Copy next.arguments into execute, fill any blank required inputs and adjust values according to inputs.fields. Execute uses credits.",
132
+ "inputs": map[string]any{"fields": item["options"], "normalized_fields": normalizedDataInputFields(item), "requires_one_of": item["requiresOneOf"]},
133
+ "output": item["response"],
134
+ "normalized_output_fields": normalizedDataOutputFields(item),
135
+ "examples": contractExamples(item),
136
+ "input_template": inputs,
137
+ "fill_one_of": item["requiresOneOf"],
138
+ "next": map[string]any{"tool": "execute", "arguments": execArguments},
139
+ "guidance": "Copy next.arguments into execute, fill any blank required inputs and adjust values according to inputs.fields. Execute uses credits.",
140
+ }
141
+ if cost, ok := numberField(item["creditsCost"]); ok && cost > float64(dataCreditCap()) {
142
+ response["availability"] = "exceeds_server_credit_limit"
143
+ response["executable"] = false
144
+ response["server_credit_limit"] = dataCreditCap()
145
+ delete(response, "next")
146
+ response["guidance"] = fmt.Sprintf("This operation costs %v credits per call and exceeds Origo's %d-credit request ceiling. It cannot execute under this server configuration.", cost, dataCreditCap())
147
+ } else {
148
+ response["executable"] = true
166
149
  }
167
150
  if len(missing) > 0 {
168
151
  response["fill_before_execution"] = missing
@@ -0,0 +1,126 @@
1
+ package app
2
+
3
+ import (
4
+ "net/url"
5
+ "strconv"
6
+ "strings"
7
+ )
8
+
9
+ // providerContinuation uses native cursor and page metadata without
10
+ // reconstructing provider URLs. All previously supplied filters remain
11
+ // unchanged; only the relevant paging parameter is added/replaced.
12
+ func providerContinuation(id string, original map[string]any, data any) map[string]any {
13
+ value := mapField(data)
14
+ if value == nil {
15
+ return nil
16
+ }
17
+ makeNext := func(key string, token any) map[string]any {
18
+ if key == "" || token == nil || token == "" {
19
+ return nil
20
+ }
21
+ next := copyInputs(original)
22
+ if previous, exists := next[key]; exists && previous == token {
23
+ return nil
24
+ }
25
+ next[key] = token
26
+ return map[string]any{"tool": "execute", "arguments": map[string]any{
27
+ "calls": []any{map[string]any{"id": id, "inputs": next}},
28
+ }}
29
+ }
30
+ for _, candidate := range []struct {
31
+ location map[string]any
32
+ names []string
33
+ input string
34
+ }{
35
+ {value, []string{"next_page_token", "nextPageToken", "page_token_next"}, "page_token"},
36
+ {mapField(value["pagination"]), []string{"next_page_token", "nextPageToken"}, "page_token"},
37
+ {value, []string{"continuation", "next_continuation"}, "continuation"},
38
+ {mapField(value["metadata"]), []string{"search_after"}, "search_after"},
39
+ {mapField(value["meta"]), []string{"search_after"}, "search_after"},
40
+ {value, []string{"search_after"}, "search_after"},
41
+ {mapField(value["pagination"]), []string{"next_cursor", "nextCursor"}, "cursor"},
42
+ {mapField(value["pageInfo"]), []string{"endCursor"}, "cursor"},
43
+ } {
44
+ for _, name := range candidate.names {
45
+ token := candidate.location[name]
46
+ if token == nil || token == "" {
47
+ continue
48
+ }
49
+ key := candidate.input
50
+ if key == "page_token" {
51
+ for _, alias := range []string{"page_token", "pageToken", "pageTokenNext"} {
52
+ if _, exists := original[alias]; exists {
53
+ key = alias
54
+ break
55
+ }
56
+ }
57
+ }
58
+ if key == "cursor" {
59
+ for _, alias := range []string{"cursor", "after", "after_cursor"} {
60
+ if _, exists := original[alias]; exists {
61
+ key = alias
62
+ break
63
+ }
64
+ }
65
+ }
66
+ if next := makeNext(key, token); next != nil {
67
+ return next
68
+ }
69
+ }
70
+ }
71
+ // Treasury and other API-backed providers expose the following page as
72
+ // a URL. We extract only recognized pagination parameters; we never
73
+ // fetch or execute that URL directly.
74
+ links := mapField(value["links"])
75
+ for _, source := range []any{links["next"], mapField(value["paging"])["next"]} {
76
+ target := stringField(source)
77
+ if target == "" {
78
+ continue
79
+ }
80
+ u, err := url.Parse(target)
81
+ if err != nil {
82
+ continue
83
+ }
84
+ q := u.Query()
85
+ for _, candidate := range []struct{ From, To string }{
86
+ {"page[number]", "page"}, {"pageNumber", "page"}, {"page", "page"},
87
+ {"page_number", "page_number"}, {"page[size]", "page_size"},
88
+ {"pageToken", "page_token"}, {"page_token", "page_token"},
89
+ {"offset", "offset"}, {"cursor", "cursor"},
90
+ } {
91
+ raw := q.Get(candidate.From)
92
+ if raw == "" {
93
+ continue
94
+ }
95
+ key := candidate.To
96
+ if candidate.From == "page[number]" {
97
+ for _, alias := range []string{"page_number", "page", "pageNumber"} {
98
+ if _, exists := original[alias]; exists {
99
+ key = alias
100
+ break
101
+ }
102
+ }
103
+ }
104
+ token := any(raw)
105
+ if n, e := strconv.Atoi(raw); e == nil {
106
+ token = n
107
+ }
108
+ if next := makeNext(key, token); next != nil {
109
+ return next
110
+ }
111
+ }
112
+ }
113
+ // Some providers only return has_more and a current page number.
114
+ if more, _ := value["has_more"].(bool); more {
115
+ for _, name := range []string{"page", "page_number", "pageNumber"} {
116
+ if n, ok := numberField(original[name]); ok && n >= 0 {
117
+ return makeNext(name, int(n)+1)
118
+ }
119
+ }
120
+ }
121
+ if link := stringField(value["next"]); link != "" &&
122
+ !strings.HasPrefix(strings.ToLower(link), "http") {
123
+ return makeNext("cursor", link)
124
+ }
125
+ return nil
126
+ }
@@ -0,0 +1,83 @@
1
+ package app
2
+
3
+ import (
4
+ "crypto/sha256"
5
+ "encoding/hex"
6
+ "encoding/json"
7
+ "os"
8
+ "sync"
9
+ "time"
10
+ )
11
+
12
+ type localPaidReplay struct {
13
+ fingerprint string
14
+ output map[string]any
15
+ expires time.Time
16
+ }
17
+
18
+ var paidReplayMemory = struct {
19
+ sync.Mutex
20
+ items map[string]localPaidReplay
21
+ }{items: map[string]localPaidReplay{}}
22
+
23
+ func localReplayKey(id string) string {
24
+ h := sha256.Sum256([]byte(os.Getenv("FIRECRAWL_API_KEY") + "\x00" + id))
25
+ return hex.EncodeToString(h[:])
26
+ }
27
+ func paidRequestFingerprint(calls []DataCall) string {
28
+ body, _ := json.Marshal(calls)
29
+ h := sha256.Sum256(body)
30
+ return hex.EncodeToString(h[:])
31
+ }
32
+
33
+ // This ledger is only a warm-worker replay optimization. It cannot claim to
34
+ // verify charges or prevent replay across multiple Vercel instances; upstream
35
+ // x-request-id idempotency remains authoritative in that situation.
36
+ func cachedPaidReplay(requestID string, calls []DataCall) (map[string]any, error) {
37
+ key := localReplayKey(requestID)
38
+ fingerprint := paidRequestFingerprint(calls)
39
+ paidReplayMemory.Lock()
40
+ entry, ok := paidReplayMemory.items[key]
41
+ paidReplayMemory.Unlock()
42
+ if !ok || time.Now().After(entry.expires) {
43
+ return nil, nil
44
+ }
45
+ if entry.fingerprint != fingerprint {
46
+ return nil, &DataError{
47
+ Code: "duplicate_request",
48
+ Message: "This request_id already completed with different inputs. Restore the original calls to replay, or omit request_id for a new query.",
49
+ Details: map[string]any{"recovery": "Never retry different inputs with the same ID. Use the original operation and input list for replay, or generate a fresh ID."},
50
+ }
51
+ }
52
+ raw, err := json.Marshal(entry.output)
53
+ if err != nil {
54
+ return nil, nil
55
+ }
56
+ var result map[string]any
57
+ if json.Unmarshal(raw, &result) != nil {
58
+ return nil, nil
59
+ }
60
+ result["replayed"] = true
61
+ result["replay_status"] = "served_from_local_cache"
62
+ result["original_credits_used"] = result["credits_used"]
63
+ result["credits_charged_this_request"] = 0
64
+ result["billing_note"] = "This exact replay was served from Origo's in-process cache without contacting Firecrawl, so Origo incurred no new upstream execution on this attempt."
65
+ return result, nil
66
+ }
67
+
68
+ func storePaidReplay(requestID string, calls []DataCall, output map[string]any) {
69
+ raw, err := json.Marshal(output)
70
+ if err != nil || len(raw) > 1<<20 {
71
+ return
72
+ }
73
+ key := localReplayKey(requestID)
74
+ paidReplayMemory.Lock()
75
+ defer paidReplayMemory.Unlock()
76
+ if len(paidReplayMemory.items) > 192 {
77
+ paidReplayMemory.items = map[string]localPaidReplay{}
78
+ }
79
+ paidReplayMemory.items[key] = localPaidReplay{
80
+ fingerprint: paidRequestFingerprint(calls), output: output,
81
+ expires: time.Now().Add(15 * time.Minute),
82
+ }
83
+ }