webctx 0.2.6 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/internal/app/app.go +28 -1
- package/internal/app/data_audit_test.go +433 -0
- package/internal/app/data_client.go +193 -5
- package/internal/app/data_contract_constraints.go +65 -0
- package/internal/app/data_examples.go +79 -6
- package/internal/app/data_execute.go +202 -61
- package/internal/app/data_inspect.go +20 -37
- package/internal/app/data_pagination.go +126 -0
- package/internal/app/data_replay.go +83 -0
- package/internal/app/data_research.go +171 -16
- package/internal/app/data_tools_test.go +2 -2
- package/internal/app/data_types.go +35 -12
- package/internal/app/research.go +4 -1
- package/internal/app/research_answers.go +30 -0
- package/internal/app/site_map_page.go +256 -0
- package/internal/app/tools.go +16 -46
- package/package.json +1 -1
- package/pkg/origo/data_test.go +26 -0
- package/pkg/origo/server.go +17 -10
- package/pkg/retrieval/data.go +2 -0
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
package app
|
|
2
2
|
|
|
3
3
|
import (
|
|
4
|
+
"context"
|
|
5
|
+
"crypto/sha256"
|
|
6
|
+
"encoding/hex"
|
|
4
7
|
"encoding/json"
|
|
5
8
|
"fmt"
|
|
6
9
|
"math"
|
|
@@ -70,6 +73,26 @@ func validateDataInputs(contract map[string]any, inputs map[string]any) error {
|
|
|
70
73
|
issues = append(issues, fmt.Sprintf("field %q must be %s", name, stringField(field["type"])))
|
|
71
74
|
continue
|
|
72
75
|
}
|
|
76
|
+
if integerDataInputs[name] {
|
|
77
|
+
if n, ok := numberField(value); ok && n != math.Trunc(n) {
|
|
78
|
+
issues = append(issues, fmt.Sprintf("field %q requires an integer value", name))
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
for _, key := range []string{"requires", "dependsOn"} {
|
|
82
|
+
if deps := listField(field[key]); len(deps) > 0 {
|
|
83
|
+
for _, dep := range deps {
|
|
84
|
+
dependent := stringField(dep)
|
|
85
|
+
if dependent != "" && inputs[dependent] == nil {
|
|
86
|
+
issues = append(issues, fmt.Sprintf("field %q requires accompanying field %q", name, dependent))
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
for _, other := range listField(field["mutuallyExclusiveWith"]) {
|
|
92
|
+
if alternate := stringField(other); alternate != "" && inputs[alternate] != nil {
|
|
93
|
+
issues = append(issues, fmt.Sprintf("fields %q and %q cannot be combined", name, alternate))
|
|
94
|
+
}
|
|
95
|
+
}
|
|
73
96
|
if enum := listField(field["oneOf"]); len(enum) > 0 {
|
|
74
97
|
found := false
|
|
75
98
|
for _, allowed := range enum {
|
|
@@ -185,16 +208,22 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
|
|
|
185
208
|
if perRecord, _ := contract["perRecord"].(bool); perRecord {
|
|
186
209
|
// A per-record price has no hard upper bound without a record limit.
|
|
187
210
|
count := 0.0
|
|
188
|
-
for _, key := range []string{"limit", "per_page", "page_size", "max_results", "count"} {
|
|
211
|
+
for _, key := range []string{"limit", "k", "top_k", "topK", "per_page", "page_size", "max_results", "count", "size", "num_results", "max_records", "results_per_page"} {
|
|
189
212
|
if n, ok := numberField(inputs[key]); ok && n >= 1 {
|
|
190
213
|
count = n
|
|
191
214
|
break
|
|
192
215
|
}
|
|
193
216
|
}
|
|
217
|
+
// For bulk identifier lookups, a response may be charged per input.
|
|
218
|
+
for _, key := range []string{"ids", "domains", "urls", "companies", "people", "symbols"} {
|
|
219
|
+
if records := listField(inputs[key]); count == 0 && len(records) > 0 {
|
|
220
|
+
count = float64(len(records))
|
|
221
|
+
}
|
|
222
|
+
}
|
|
194
223
|
if count == 0 {
|
|
195
224
|
for _, v := range listField(contract["options"]) {
|
|
196
225
|
f := mapField(v)
|
|
197
|
-
for _, key := range []string{"limit", "per_page", "page_size", "max_results", "count"} {
|
|
226
|
+
for _, key := range []string{"limit", "k", "top_k", "topK", "per_page", "page_size", "max_results", "count", "size", "num_results", "max_records", "results_per_page"} {
|
|
198
227
|
if stringField(f["name"]) == key {
|
|
199
228
|
if n, ok := numberField(f["default"]); ok && n >= 1 {
|
|
200
229
|
count = n
|
|
@@ -203,9 +232,30 @@ func expectedCallCredits(contract map[string]any, inputs map[string]any) (int, e
|
|
|
203
232
|
}
|
|
204
233
|
}
|
|
205
234
|
}
|
|
235
|
+
if count == 0 {
|
|
236
|
+
// An exact, non-paginated lookup with an individual identifier has
|
|
237
|
+
// a single-record upper bound, even without a limit parameter.
|
|
238
|
+
// Only apply this to known singleton operations, never searches.
|
|
239
|
+
id := strings.ToLower(stringField(contract["id"]))
|
|
240
|
+
isLookup := strings.HasSuffix(id, "/lookup") || strings.HasSuffix(id, "/get") ||
|
|
241
|
+
strings.HasSuffix(id, "/detail") || strings.HasSuffix(id, "/details") ||
|
|
242
|
+
strings.HasSuffix(id, "/resolve") || strings.HasSuffix(id, "/profile")
|
|
243
|
+
hasIdentifier := false
|
|
244
|
+
for _, key := range []string{"domain", "id", "company_id", "person_id", "symbol", "url", "linkedin_url", "repo", "doi"} {
|
|
245
|
+
if s := stringField(inputs[key]); s != "" {
|
|
246
|
+
hasIdentifier = true
|
|
247
|
+
break
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
if isLookup && hasIdentifier {
|
|
251
|
+
if paginated, ok := mapField(contract["response"])["paginated"].(bool); !ok || !paginated {
|
|
252
|
+
count = 1
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
206
256
|
if count == 0 {
|
|
207
257
|
return 0, dataError("unbounded_per_record_cost",
|
|
208
|
-
"This operation charges per record but
|
|
258
|
+
"This operation charges per record, but the contract does not establish a finite record bound. Inspect its output and supply a supported count parameter or smaller input set. Origo cannot safely invent a limit field.")
|
|
209
259
|
}
|
|
210
260
|
estimated = cost * count
|
|
211
261
|
}
|
|
@@ -220,6 +270,9 @@ func continuationForDataResult(id string, originalInputs map[string]any, data an
|
|
|
220
270
|
if value == nil {
|
|
221
271
|
return nil
|
|
222
272
|
}
|
|
273
|
+
if next := providerContinuation(id, originalInputs, data); next != nil {
|
|
274
|
+
return next
|
|
275
|
+
}
|
|
223
276
|
if next := mapField(value["next"]); next != nil {
|
|
224
277
|
if provider := stringField(next["provider"]); provider != "" {
|
|
225
278
|
if capability := stringField(next["capability"]); capability != "" {
|
|
@@ -275,6 +328,94 @@ func copyInputs(original map[string]any) map[string]any {
|
|
|
275
328
|
return out
|
|
276
329
|
}
|
|
277
330
|
|
|
331
|
+
// A multi-operation request uses a distinct, deterministic provider request ID
|
|
332
|
+
// for every operation. The upstream batch endpoint is not transactionally
|
|
333
|
+
// isolated: one provider's terms rejection can abort unrelated operations.
|
|
334
|
+
func operationRequestID(root string, index int, id string, count int) string {
|
|
335
|
+
if count == 1 {
|
|
336
|
+
return root
|
|
337
|
+
}
|
|
338
|
+
digest := sha256.Sum256([]byte(id))
|
|
339
|
+
if len(root) > 92 {
|
|
340
|
+
root = root[:92]
|
|
341
|
+
}
|
|
342
|
+
return fmt.Sprintf("%s-%d-%s", root, index, hex.EncodeToString(digest[:6]))
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
func executeOneDataOperation(ctx context.Context, call DataCall, requestID string, index int, includeRaw bool) (map[string]any, float64, error) {
|
|
346
|
+
provider, capability, err := parseOperationID(call.ID)
|
|
347
|
+
if err != nil {
|
|
348
|
+
return nil, 0, err
|
|
349
|
+
}
|
|
350
|
+
options := call.Inputs
|
|
351
|
+
if options == nil {
|
|
352
|
+
options = map[string]any{}
|
|
353
|
+
}
|
|
354
|
+
raw, err := doDataAPI(ctx, "/scrape", map[string]any{"alexandria": []any{
|
|
355
|
+
map[string]any{"provider": provider, "capability": capability, "options": options},
|
|
356
|
+
}}, requestID)
|
|
357
|
+
if err != nil {
|
|
358
|
+
return nil, 0, err
|
|
359
|
+
}
|
|
360
|
+
payload := mapField(raw["data"])
|
|
361
|
+
results := listField(payload["alexandria"])
|
|
362
|
+
if len(results) != 1 {
|
|
363
|
+
return nil, 0, dataError("invalid_upstream_response", "The provider did not return exactly one result for the requested operation.")
|
|
364
|
+
}
|
|
365
|
+
entry := mapField(results[0])
|
|
366
|
+
if entry == nil {
|
|
367
|
+
return nil, 0, dataError("invalid_upstream_response", "The provider returned an unreadable result.")
|
|
368
|
+
}
|
|
369
|
+
if providerError := entry["error"]; providerError != nil {
|
|
370
|
+
return nil, 0, &DataError{Code: "provider_operation_failed", Message: fmt.Sprint(providerError)}
|
|
371
|
+
}
|
|
372
|
+
credits, _ := numberField(entry["creditsCost"])
|
|
373
|
+
if total, ok := numberField(payload["creditsCost"]); ok {
|
|
374
|
+
credits = total
|
|
375
|
+
}
|
|
376
|
+
item := map[string]any{
|
|
377
|
+
"operation_index": index, "id": call.ID, "source_id": provider,
|
|
378
|
+
"ok": true, "data": entry["data"],
|
|
379
|
+
"credits_used": credits, "receipt_scope": "upstream_request",
|
|
380
|
+
"provider_request_id": requestID,
|
|
381
|
+
"provider_credit_note": "Provider-native fields named credits may use different units. credits_used is Firecrawl Alexandria billing.",
|
|
382
|
+
}
|
|
383
|
+
if token := entry["alexandriaId"]; token != nil {
|
|
384
|
+
item["data_id"] = token
|
|
385
|
+
item["receipt_scope"] = "provider_response_or_batch"
|
|
386
|
+
}
|
|
387
|
+
if note := dataMeasurementNote(entry["data"]); note != "" {
|
|
388
|
+
item["measurement_note"] = note
|
|
389
|
+
}
|
|
390
|
+
if next := continuationForDataResult(call.ID, call.Inputs, entry["data"]); next != nil {
|
|
391
|
+
item["next"] = next
|
|
392
|
+
}
|
|
393
|
+
if extra := raw["scrape_id"]; extra != nil {
|
|
394
|
+
item["scrape_id"] = extra
|
|
395
|
+
}
|
|
396
|
+
if includeRaw {
|
|
397
|
+
item["raw"] = raw
|
|
398
|
+
}
|
|
399
|
+
return item, credits, nil
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
func operationErrorResult(call DataCall, index int, requestID string, err error) map[string]any {
|
|
403
|
+
typed, ok := err.(*DataError)
|
|
404
|
+
if !ok {
|
|
405
|
+
typed = &DataError{Code: "execution_failed", Message: err.Error()}
|
|
406
|
+
}
|
|
407
|
+
out := map[string]any{
|
|
408
|
+
"id": call.ID, "operation_index": index, "ok": false,
|
|
409
|
+
"provider_request_id": requestID, "error": typed,
|
|
410
|
+
"credits_used": nil, "billing_status": "unknown_if_provider_accepted_request",
|
|
411
|
+
}
|
|
412
|
+
if typed.Code == "THIRD_PARTY_DATA_TERMS_REQUIRED" {
|
|
413
|
+
out["billing_status"] = "blocked_pending_human_terms"
|
|
414
|
+
out["credits_used"] = 0
|
|
415
|
+
}
|
|
416
|
+
return out
|
|
417
|
+
}
|
|
418
|
+
|
|
278
419
|
// ExecuteData validates contracts and credit estimates before calling any
|
|
279
420
|
// paid provider. The native API accepts at most ten executions per request.
|
|
280
421
|
func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
@@ -306,7 +447,6 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
306
447
|
}
|
|
307
448
|
wg.Wait()
|
|
308
449
|
var estimated int
|
|
309
|
-
var calls []map[string]any
|
|
310
450
|
for i, call := range input.Calls {
|
|
311
451
|
if checked[i].Err != nil {
|
|
312
452
|
return nil, checked[i].Err
|
|
@@ -326,12 +466,6 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
326
466
|
Details: map[string]any{"estimated_credits": estimated, "server_credit_limit": cap},
|
|
327
467
|
}
|
|
328
468
|
}
|
|
329
|
-
provider, capability, _ := parseOperationID(call.ID)
|
|
330
|
-
options := call.Inputs
|
|
331
|
-
if options == nil {
|
|
332
|
-
options = map[string]any{}
|
|
333
|
-
}
|
|
334
|
-
calls = append(calls, map[string]any{"provider": provider, "capability": capability, "options": options})
|
|
335
469
|
}
|
|
336
470
|
requestID := input.RequestID
|
|
337
471
|
if requestID == "" {
|
|
@@ -341,66 +475,73 @@ func ExecuteData(input ExecuteDataInput) (map[string]any, error) {
|
|
|
341
475
|
return nil, dataError("internal_error", "Could not generate a request identifier.")
|
|
342
476
|
}
|
|
343
477
|
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
if
|
|
348
|
-
|
|
349
|
-
"retry": map[string]any{"tool": "execute", "arguments": ExecuteDataInput{
|
|
350
|
-
Calls: input.Calls, RequestID: requestID, Raw: input.Raw,
|
|
351
|
-
}},
|
|
352
|
-
}
|
|
478
|
+
if input.RequestID != "" {
|
|
479
|
+
if cached, err := cachedPaidReplay(requestID, input.Calls); err != nil {
|
|
480
|
+
return nil, err
|
|
481
|
+
} else if cached != nil {
|
|
482
|
+
return cached, nil
|
|
353
483
|
}
|
|
354
|
-
return nil, err
|
|
355
|
-
}
|
|
356
|
-
payload := mapField(response["data"])
|
|
357
|
-
nativeResults := listField(payload["alexandria"])
|
|
358
|
-
if len(nativeResults) == 0 {
|
|
359
|
-
return nil, dataError("invalid_upstream_response", "Paid request returned no Alexandria execution results.")
|
|
360
484
|
}
|
|
361
|
-
results := make([]map[string]any, 0, len(
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
if errValue := entry["error"]; errValue != nil {
|
|
381
|
-
item["error"] = errValue
|
|
382
|
-
}
|
|
383
|
-
if i < len(input.Calls) {
|
|
384
|
-
if next := continuationForDataResult(id, input.Calls[i].Inputs, entry["data"]); next != nil {
|
|
385
|
-
item["next"] = next
|
|
485
|
+
results := make([]map[string]any, 0, len(input.Calls))
|
|
486
|
+
creditsUsed := 0.0
|
|
487
|
+
failed := 0
|
|
488
|
+
var rawResponses []any
|
|
489
|
+
for i, call := range input.Calls {
|
|
490
|
+
operationID := operationRequestID(requestID, i, call.ID, len(input.Calls))
|
|
491
|
+
item, charged, err := executeOneDataOperation(ctx, call, operationID, i, input.Raw)
|
|
492
|
+
if err != nil {
|
|
493
|
+
// Treat idempotency collisions and contract/terms failures as
|
|
494
|
+
// independent operation errors. Do not suggest replaying a
|
|
495
|
+
// conflicting payload with the same id.
|
|
496
|
+
var typed *DataError
|
|
497
|
+
if errorsAsData(err, &typed) {
|
|
498
|
+
if typed.Code != "duplicate_request" && typed.Code != "THIRD_PARTY_DATA_TERMS_REQUIRED" {
|
|
499
|
+
typed.Details = map[string]any{
|
|
500
|
+
"request_id": operationID,
|
|
501
|
+
"recovery": "For an uncertain network failure, retry the exact same operation and request_id. For a different query or page, omit request_id.",
|
|
502
|
+
}
|
|
503
|
+
}
|
|
386
504
|
}
|
|
505
|
+
if len(input.Calls) == 1 {
|
|
506
|
+
return nil, err
|
|
507
|
+
}
|
|
508
|
+
failed++
|
|
509
|
+
results = append(results, operationErrorResult(call, i, operationID, err))
|
|
510
|
+
continue
|
|
387
511
|
}
|
|
512
|
+
creditsUsed += charged
|
|
388
513
|
results = append(results, item)
|
|
514
|
+
if input.Raw {
|
|
515
|
+
rawResponses = append(rawResponses, map[string]any{"operation_index": i, "provider_request_id": operationID, "result": item})
|
|
516
|
+
}
|
|
389
517
|
}
|
|
390
518
|
output := map[string]any{
|
|
391
|
-
"results":
|
|
392
|
-
"request_id":
|
|
393
|
-
"estimated_credits":
|
|
394
|
-
"
|
|
395
|
-
"
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
519
|
+
"results": results, "partial": failed > 0, "failed_operations": failed,
|
|
520
|
+
"request_id": requestID,
|
|
521
|
+
"estimated_credits": estimated,
|
|
522
|
+
"credits_used": creditsUsed,
|
|
523
|
+
"pricing_note": "credits_used totals reported Firecrawl charges from successful operations. Failed operations may have indeterminate billing; provider-native credits use their own units.",
|
|
524
|
+
"guidance": "Use each result.next to fetch its next page. Retry an identical operation only with its provider_request_id. A new query/page must use a new request ID.",
|
|
525
|
+
"replay_status": "upstream_replay_unverified",
|
|
526
|
+
"replayed": false,
|
|
527
|
+
"credits_charged_this_request": nil,
|
|
528
|
+
"original_credits_used": nil,
|
|
529
|
+
"billing_note": "Firecrawl idempotent replay may return the original credit amount; without a provider replay flag or durable accounting data, Origo cannot assert that a retry incurred a new charge.",
|
|
530
|
+
}
|
|
531
|
+
if failed > 0 {
|
|
532
|
+
output["status"] = "partial_success"
|
|
533
|
+
}
|
|
534
|
+
if failed == len(input.Calls) {
|
|
535
|
+
output["status"] = "all_failed"
|
|
536
|
+
}
|
|
537
|
+
if failed == 0 {
|
|
538
|
+
output["status"] = "success"
|
|
401
539
|
}
|
|
402
540
|
if input.Raw {
|
|
403
|
-
output["raw"] =
|
|
541
|
+
output["raw"] = rawResponses
|
|
542
|
+
}
|
|
543
|
+
if failed == 0 {
|
|
544
|
+
storePaidReplay(requestID, input.Calls, output)
|
|
404
545
|
}
|
|
405
546
|
return output, nil
|
|
406
547
|
}
|
|
@@ -72,33 +72,6 @@ func nativeInputTemplate(contract map[string]any) (map[string]any, []string) {
|
|
|
72
72
|
seen[name] = true
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
|
-
for _, group := range listField(contract["requiresOneOf"]) {
|
|
76
|
-
var matched bool
|
|
77
|
-
for _, field := range listField(group) {
|
|
78
|
-
name := stringField(field)
|
|
79
|
-
if _, already := template[name]; already && template[name] != nil {
|
|
80
|
-
matched = true
|
|
81
|
-
break
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
if matched {
|
|
85
|
-
continue
|
|
86
|
-
}
|
|
87
|
-
// Show one candidate with a blank value rather than guessing user
|
|
88
|
-
// intent or silently filling provider-specific search terms.
|
|
89
|
-
for _, field := range listField(group) {
|
|
90
|
-
name := stringField(field)
|
|
91
|
-
if name == "" {
|
|
92
|
-
continue
|
|
93
|
-
}
|
|
94
|
-
template[name] = nil
|
|
95
|
-
if !seen[name] {
|
|
96
|
-
need = append(need, name)
|
|
97
|
-
seen[name] = true
|
|
98
|
-
}
|
|
99
|
-
break
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
75
|
return template, need
|
|
103
76
|
}
|
|
104
77
|
|
|
@@ -125,10 +98,9 @@ func InspectData(input InspectDataInput) (map[string]any, error) {
|
|
|
125
98
|
return nil, dataError("invalid_source", "Use an exact source ID returned by research.")
|
|
126
99
|
}
|
|
127
100
|
include := input.Include
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
}
|
|
131
|
-
list, err := ResearchData(ResearchDataInput{Sources: []string{id}, View: "operations", Include: include, Limit: 25, Raw: input.Raw})
|
|
101
|
+
// Provider-level discovery is intentionally lightweight: inspect a
|
|
102
|
+
// particular operation for full field-by-field contracts.
|
|
103
|
+
list, err := ResearchData(ResearchDataInput{Sources: []string{id}, View: "operations", Include: include, Limit: 12, Raw: input.Raw})
|
|
132
104
|
if err != nil {
|
|
133
105
|
return nil, err
|
|
134
106
|
}
|
|
@@ -157,12 +129,23 @@ func InspectData(input InspectDataInput) (map[string]any, error) {
|
|
|
157
129
|
"credits_per_call": item["creditsCost"],
|
|
158
130
|
"per_record": item["perRecord"],
|
|
159
131
|
},
|
|
160
|
-
"inputs":
|
|
161
|
-
"output":
|
|
162
|
-
"
|
|
163
|
-
"
|
|
164
|
-
"
|
|
165
|
-
"
|
|
132
|
+
"inputs": map[string]any{"fields": item["options"], "normalized_fields": normalizedDataInputFields(item), "requires_one_of": item["requiresOneOf"]},
|
|
133
|
+
"output": item["response"],
|
|
134
|
+
"normalized_output_fields": normalizedDataOutputFields(item),
|
|
135
|
+
"examples": contractExamples(item),
|
|
136
|
+
"input_template": inputs,
|
|
137
|
+
"fill_one_of": item["requiresOneOf"],
|
|
138
|
+
"next": map[string]any{"tool": "execute", "arguments": execArguments},
|
|
139
|
+
"guidance": "Copy next.arguments into execute, fill any blank required inputs and adjust values according to inputs.fields. Execute uses credits.",
|
|
140
|
+
}
|
|
141
|
+
if cost, ok := numberField(item["creditsCost"]); ok && cost > float64(dataCreditCap()) {
|
|
142
|
+
response["availability"] = "exceeds_server_credit_limit"
|
|
143
|
+
response["executable"] = false
|
|
144
|
+
response["server_credit_limit"] = dataCreditCap()
|
|
145
|
+
delete(response, "next")
|
|
146
|
+
response["guidance"] = fmt.Sprintf("This operation costs %v credits per call and exceeds Origo's %d-credit request ceiling. It cannot execute under this server configuration.", cost, dataCreditCap())
|
|
147
|
+
} else {
|
|
148
|
+
response["executable"] = true
|
|
166
149
|
}
|
|
167
150
|
if len(missing) > 0 {
|
|
168
151
|
response["fill_before_execution"] = missing
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
package app
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"net/url"
|
|
5
|
+
"strconv"
|
|
6
|
+
"strings"
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
// providerContinuation uses native cursor and page metadata without
|
|
10
|
+
// reconstructing provider URLs. All previously supplied filters remain
|
|
11
|
+
// unchanged; only the relevant paging parameter is added/replaced.
|
|
12
|
+
func providerContinuation(id string, original map[string]any, data any) map[string]any {
|
|
13
|
+
value := mapField(data)
|
|
14
|
+
if value == nil {
|
|
15
|
+
return nil
|
|
16
|
+
}
|
|
17
|
+
makeNext := func(key string, token any) map[string]any {
|
|
18
|
+
if key == "" || token == nil || token == "" {
|
|
19
|
+
return nil
|
|
20
|
+
}
|
|
21
|
+
next := copyInputs(original)
|
|
22
|
+
if previous, exists := next[key]; exists && previous == token {
|
|
23
|
+
return nil
|
|
24
|
+
}
|
|
25
|
+
next[key] = token
|
|
26
|
+
return map[string]any{"tool": "execute", "arguments": map[string]any{
|
|
27
|
+
"calls": []any{map[string]any{"id": id, "inputs": next}},
|
|
28
|
+
}}
|
|
29
|
+
}
|
|
30
|
+
for _, candidate := range []struct {
|
|
31
|
+
location map[string]any
|
|
32
|
+
names []string
|
|
33
|
+
input string
|
|
34
|
+
}{
|
|
35
|
+
{value, []string{"next_page_token", "nextPageToken", "page_token_next"}, "page_token"},
|
|
36
|
+
{mapField(value["pagination"]), []string{"next_page_token", "nextPageToken"}, "page_token"},
|
|
37
|
+
{value, []string{"continuation", "next_continuation"}, "continuation"},
|
|
38
|
+
{mapField(value["metadata"]), []string{"search_after"}, "search_after"},
|
|
39
|
+
{mapField(value["meta"]), []string{"search_after"}, "search_after"},
|
|
40
|
+
{value, []string{"search_after"}, "search_after"},
|
|
41
|
+
{mapField(value["pagination"]), []string{"next_cursor", "nextCursor"}, "cursor"},
|
|
42
|
+
{mapField(value["pageInfo"]), []string{"endCursor"}, "cursor"},
|
|
43
|
+
} {
|
|
44
|
+
for _, name := range candidate.names {
|
|
45
|
+
token := candidate.location[name]
|
|
46
|
+
if token == nil || token == "" {
|
|
47
|
+
continue
|
|
48
|
+
}
|
|
49
|
+
key := candidate.input
|
|
50
|
+
if key == "page_token" {
|
|
51
|
+
for _, alias := range []string{"page_token", "pageToken", "pageTokenNext"} {
|
|
52
|
+
if _, exists := original[alias]; exists {
|
|
53
|
+
key = alias
|
|
54
|
+
break
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if key == "cursor" {
|
|
59
|
+
for _, alias := range []string{"cursor", "after", "after_cursor"} {
|
|
60
|
+
if _, exists := original[alias]; exists {
|
|
61
|
+
key = alias
|
|
62
|
+
break
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
if next := makeNext(key, token); next != nil {
|
|
67
|
+
return next
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// Treasury and other API-backed providers expose the following page as
|
|
72
|
+
// a URL. We extract only recognized pagination parameters; we never
|
|
73
|
+
// fetch or execute that URL directly.
|
|
74
|
+
links := mapField(value["links"])
|
|
75
|
+
for _, source := range []any{links["next"], mapField(value["paging"])["next"]} {
|
|
76
|
+
target := stringField(source)
|
|
77
|
+
if target == "" {
|
|
78
|
+
continue
|
|
79
|
+
}
|
|
80
|
+
u, err := url.Parse(target)
|
|
81
|
+
if err != nil {
|
|
82
|
+
continue
|
|
83
|
+
}
|
|
84
|
+
q := u.Query()
|
|
85
|
+
for _, candidate := range []struct{ From, To string }{
|
|
86
|
+
{"page[number]", "page"}, {"pageNumber", "page"}, {"page", "page"},
|
|
87
|
+
{"page_number", "page_number"}, {"page[size]", "page_size"},
|
|
88
|
+
{"pageToken", "page_token"}, {"page_token", "page_token"},
|
|
89
|
+
{"offset", "offset"}, {"cursor", "cursor"},
|
|
90
|
+
} {
|
|
91
|
+
raw := q.Get(candidate.From)
|
|
92
|
+
if raw == "" {
|
|
93
|
+
continue
|
|
94
|
+
}
|
|
95
|
+
key := candidate.To
|
|
96
|
+
if candidate.From == "page[number]" {
|
|
97
|
+
for _, alias := range []string{"page_number", "page", "pageNumber"} {
|
|
98
|
+
if _, exists := original[alias]; exists {
|
|
99
|
+
key = alias
|
|
100
|
+
break
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
token := any(raw)
|
|
105
|
+
if n, e := strconv.Atoi(raw); e == nil {
|
|
106
|
+
token = n
|
|
107
|
+
}
|
|
108
|
+
if next := makeNext(key, token); next != nil {
|
|
109
|
+
return next
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// Some providers only return has_more and a current page number.
|
|
114
|
+
if more, _ := value["has_more"].(bool); more {
|
|
115
|
+
for _, name := range []string{"page", "page_number", "pageNumber"} {
|
|
116
|
+
if n, ok := numberField(original[name]); ok && n >= 0 {
|
|
117
|
+
return makeNext(name, int(n)+1)
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
if link := stringField(value["next"]); link != "" &&
|
|
122
|
+
!strings.HasPrefix(strings.ToLower(link), "http") {
|
|
123
|
+
return makeNext("cursor", link)
|
|
124
|
+
}
|
|
125
|
+
return nil
|
|
126
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
package app
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"crypto/sha256"
|
|
5
|
+
"encoding/hex"
|
|
6
|
+
"encoding/json"
|
|
7
|
+
"os"
|
|
8
|
+
"sync"
|
|
9
|
+
"time"
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
type localPaidReplay struct {
|
|
13
|
+
fingerprint string
|
|
14
|
+
output map[string]any
|
|
15
|
+
expires time.Time
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
var paidReplayMemory = struct {
|
|
19
|
+
sync.Mutex
|
|
20
|
+
items map[string]localPaidReplay
|
|
21
|
+
}{items: map[string]localPaidReplay{}}
|
|
22
|
+
|
|
23
|
+
func localReplayKey(id string) string {
|
|
24
|
+
h := sha256.Sum256([]byte(os.Getenv("FIRECRAWL_API_KEY") + "\x00" + id))
|
|
25
|
+
return hex.EncodeToString(h[:])
|
|
26
|
+
}
|
|
27
|
+
func paidRequestFingerprint(calls []DataCall) string {
|
|
28
|
+
body, _ := json.Marshal(calls)
|
|
29
|
+
h := sha256.Sum256(body)
|
|
30
|
+
return hex.EncodeToString(h[:])
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// This ledger is only a warm-worker replay optimization. It cannot claim to
|
|
34
|
+
// verify charges or prevent replay across multiple Vercel instances; upstream
|
|
35
|
+
// x-request-id idempotency remains authoritative in that situation.
|
|
36
|
+
func cachedPaidReplay(requestID string, calls []DataCall) (map[string]any, error) {
|
|
37
|
+
key := localReplayKey(requestID)
|
|
38
|
+
fingerprint := paidRequestFingerprint(calls)
|
|
39
|
+
paidReplayMemory.Lock()
|
|
40
|
+
entry, ok := paidReplayMemory.items[key]
|
|
41
|
+
paidReplayMemory.Unlock()
|
|
42
|
+
if !ok || time.Now().After(entry.expires) {
|
|
43
|
+
return nil, nil
|
|
44
|
+
}
|
|
45
|
+
if entry.fingerprint != fingerprint {
|
|
46
|
+
return nil, &DataError{
|
|
47
|
+
Code: "duplicate_request",
|
|
48
|
+
Message: "This request_id already completed with different inputs. Restore the original calls to replay, or omit request_id for a new query.",
|
|
49
|
+
Details: map[string]any{"recovery": "Never retry different inputs with the same ID. Use the original operation and input list for replay, or generate a fresh ID."},
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
raw, err := json.Marshal(entry.output)
|
|
53
|
+
if err != nil {
|
|
54
|
+
return nil, nil
|
|
55
|
+
}
|
|
56
|
+
var result map[string]any
|
|
57
|
+
if json.Unmarshal(raw, &result) != nil {
|
|
58
|
+
return nil, nil
|
|
59
|
+
}
|
|
60
|
+
result["replayed"] = true
|
|
61
|
+
result["replay_status"] = "served_from_local_cache"
|
|
62
|
+
result["original_credits_used"] = result["credits_used"]
|
|
63
|
+
result["credits_charged_this_request"] = 0
|
|
64
|
+
result["billing_note"] = "This exact replay was served from Origo's in-process cache without contacting Firecrawl, so Origo incurred no new upstream execution on this attempt."
|
|
65
|
+
return result, nil
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
func storePaidReplay(requestID string, calls []DataCall, output map[string]any) {
|
|
69
|
+
raw, err := json.Marshal(output)
|
|
70
|
+
if err != nil || len(raw) > 1<<20 {
|
|
71
|
+
return
|
|
72
|
+
}
|
|
73
|
+
key := localReplayKey(requestID)
|
|
74
|
+
paidReplayMemory.Lock()
|
|
75
|
+
defer paidReplayMemory.Unlock()
|
|
76
|
+
if len(paidReplayMemory.items) > 192 {
|
|
77
|
+
paidReplayMemory.items = map[string]localPaidReplay{}
|
|
78
|
+
}
|
|
79
|
+
paidReplayMemory.items[key] = localPaidReplay{
|
|
80
|
+
fingerprint: paidRequestFingerprint(calls), output: output,
|
|
81
|
+
expires: time.Now().Add(15 * time.Minute),
|
|
82
|
+
}
|
|
83
|
+
}
|