reqon-dsl 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +22 -0
- package/dist/ast/nodes.d.ts +83 -4
- package/dist/ast/nodes.js +14 -0
- package/dist/auth/circuit-breaker.js +7 -6
- package/dist/auth/rate-limiter.d.ts +4 -0
- package/dist/auth/rate-limiter.js +9 -16
- package/dist/cli.d.ts +13 -0
- package/dist/cli.js +84 -4
- package/dist/config/constants.d.ts +141 -0
- package/dist/config/constants.js +128 -0
- package/dist/config/index.d.ts +4 -0
- package/dist/config/index.js +4 -0
- package/dist/control/index.d.ts +2 -0
- package/dist/control/index.js +1 -0
- package/dist/control/server.d.ts +88 -0
- package/dist/control/server.js +238 -0
- package/dist/control/types.d.ts +55 -0
- package/dist/control/types.js +7 -0
- package/dist/debug/cli-debugger.d.ts +17 -0
- package/dist/debug/cli-debugger.js +180 -0
- package/dist/debug/controller.d.ts +94 -0
- package/dist/debug/controller.js +45 -0
- package/dist/debug/index.d.ts +6 -0
- package/dist/debug/index.js +5 -0
- package/dist/errors/index.d.ts +67 -0
- package/dist/errors/index.js +89 -1
- package/dist/execution/index.d.ts +1 -1
- package/dist/execution/state.d.ts +24 -0
- package/dist/index.d.ts +21 -1
- package/dist/index.js +33 -2
- package/dist/interpreter/context.d.ts +14 -0
- package/dist/interpreter/context.js +15 -0
- package/dist/interpreter/evaluator.d.ts +63 -1
- package/dist/interpreter/evaluator.js +186 -39
- package/dist/interpreter/executor.d.ts +70 -14
- package/dist/interpreter/executor.js +503 -174
- package/dist/interpreter/fetch-handler.d.ts +9 -0
- package/dist/interpreter/fetch-handler.js +133 -24
- package/dist/interpreter/http.d.ts +5 -0
- package/dist/interpreter/http.js +26 -12
- package/dist/interpreter/index.d.ts +3 -1
- package/dist/interpreter/index.js +2 -0
- package/dist/interpreter/pagination.d.ts +11 -2
- package/dist/interpreter/pagination.js +95 -31
- package/dist/interpreter/signals.d.ts +8 -0
- package/dist/interpreter/signals.js +12 -0
- package/dist/interpreter/source-manager.d.ts +75 -0
- package/dist/interpreter/source-manager.js +157 -0
- package/dist/interpreter/step-handlers/apply-handler.d.ts +29 -0
- package/dist/interpreter/step-handlers/apply-handler.js +79 -0
- package/dist/interpreter/step-handlers/for-handler.d.ts +13 -0
- package/dist/interpreter/step-handlers/for-handler.js +71 -4
- package/dist/interpreter/step-handlers/index.d.ts +4 -2
- package/dist/interpreter/step-handlers/index.js +4 -2
- package/dist/interpreter/step-handlers/match-handler.d.ts +9 -0
- package/dist/interpreter/step-handlers/match-handler.js +43 -16
- package/dist/interpreter/step-handlers/pause-handler.d.ts +52 -0
- package/dist/interpreter/step-handlers/pause-handler.js +87 -0
- package/dist/interpreter/step-handlers/store-handler.d.ts +11 -1
- package/dist/interpreter/step-handlers/store-handler.js +45 -13
- package/dist/interpreter/step-handlers/types.d.ts +3 -0
- package/dist/interpreter/step-handlers/validate-handler.d.ts +2 -1
- package/dist/interpreter/step-handlers/validate-handler.js +4 -2
- package/dist/interpreter/step-handlers/webhook-handler.d.ts +3 -0
- package/dist/interpreter/step-handlers/webhook-handler.js +18 -2
- package/dist/interpreter/store-manager.d.ts +46 -0
- package/dist/interpreter/store-manager.js +66 -0
- package/dist/lexer/index.d.ts +11 -4
- package/dist/lexer/index.js +11 -4
- package/dist/lexer/tokens.d.ts +17 -1
- package/dist/lexer/tokens.js +36 -0
- package/dist/mcp/index.d.ts +11 -0
- package/dist/mcp/index.js +11 -0
- package/dist/mcp/server.d.ts +17 -0
- package/dist/mcp/server.js +451 -0
- package/dist/oas/index.d.ts +2 -0
- package/dist/oas/index.js +1 -0
- package/dist/oas/mock-generator.d.ts +12 -0
- package/dist/oas/mock-generator.js +187 -0
- package/dist/observability/events.d.ts +244 -0
- package/dist/observability/events.js +90 -0
- package/dist/observability/index.d.ts +15 -0
- package/dist/observability/index.js +12 -0
- package/dist/observability/logger.d.ts +106 -0
- package/dist/observability/logger.js +259 -0
- package/dist/observability/otel.d.ts +135 -0
- package/dist/observability/otel.js +386 -0
- package/dist/parser/action-parser.d.ts +105 -0
- package/dist/parser/action-parser.js +645 -0
- package/dist/parser/expressions.d.ts +13 -0
- package/dist/parser/expressions.js +72 -2
- package/dist/parser/fetch-parser.d.ts +27 -0
- package/dist/parser/fetch-parser.js +269 -0
- package/dist/parser/index.d.ts +17 -0
- package/dist/parser/index.js +17 -0
- package/dist/parser/parser.d.ts +44 -46
- package/dist/parser/parser.js +122 -1070
- package/dist/parser/pipeline-parser.d.ts +12 -0
- package/dist/parser/pipeline-parser.js +52 -0
- package/dist/parser/schedule-parser.d.ts +7 -0
- package/dist/parser/schedule-parser.js +137 -0
- package/dist/parser/source-parser.d.ts +9 -0
- package/dist/parser/source-parser.js +151 -0
- package/dist/pause/index.d.ts +14 -0
- package/dist/pause/index.js +11 -0
- package/dist/pause/manager.d.ts +118 -0
- package/dist/pause/manager.js +245 -0
- package/dist/pause/state.d.ts +93 -0
- package/dist/pause/state.js +103 -0
- package/dist/pause/store.d.ts +61 -0
- package/dist/pause/store.js +156 -0
- package/dist/plugin.d.ts +9 -12
- package/dist/plugin.js +10 -13
- package/dist/stores/factory.d.ts +1 -1
- package/dist/stores/factory.js +3 -2
- package/dist/stores/file.d.ts +26 -0
- package/dist/stores/file.js +64 -10
- package/dist/stores/index.d.ts +16 -1
- package/dist/stores/index.js +16 -1
- package/dist/stores/memory.d.ts +4 -0
- package/dist/stores/memory.js +11 -0
- package/dist/stores/types.d.ts +17 -0
- package/dist/stores/types.js +12 -0
- package/dist/trace/index.d.ts +16 -0
- package/dist/trace/index.js +12 -0
- package/dist/trace/recorder.d.ts +71 -0
- package/dist/trace/recorder.js +144 -0
- package/dist/trace/replay.d.ts +132 -0
- package/dist/trace/replay.js +264 -0
- package/dist/trace/state.d.ts +102 -0
- package/dist/trace/state.js +86 -0
- package/dist/trace/store.d.ts +69 -0
- package/dist/trace/store.js +225 -0
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.js +1 -0
- package/dist/utils/type-guards.d.ts +58 -0
- package/dist/utils/type-guards.js +92 -0
- package/dist/webhook/server.js +7 -6
- package/package.json +55 -6
- package/.claude/settings.local.json +0 -31
- package/.claude/skills/api-integration.md +0 -125
- package/.claude/skills/database-schema.md +0 -51
- package/.claude/skills/dsl-design.md +0 -80
- package/.claude/skills/property-testing.md +0 -143
- package/.claude/skills/reqon/SKILL.md +0 -44
- package/.claude/skills/reqon/references/examples.md +0 -206
- package/.claude/skills/reqon/references/syntax.md +0 -263
- package/.claude/skills/vscode-extension.md +0 -113
- package/.github/dependabot.yml +0 -32
- package/.github/pull_request_template.md +0 -21
- package/.github/workflows/ci.yml +0 -174
- package/.github/workflows/release.yml +0 -73
- package/CLAUDE.md +0 -72
- package/CONTRIBUTING.md +0 -161
- package/TODO.md +0 -51
- package/dist/auth/auth.test.d.ts +0 -1
- package/dist/auth/auth.test.js +0 -255
- package/dist/errors/errors.test.d.ts +0 -1
- package/dist/errors/errors.test.js +0 -165
- package/dist/execution/execution.test.d.ts +0 -1
- package/dist/execution/execution.test.js +0 -246
- package/dist/integration.test.d.ts +0 -1
- package/dist/integration.test.js +0 -168
- package/dist/interpreter/evaluator.test.d.ts +0 -1
- package/dist/interpreter/evaluator.test.js +0 -512
- package/dist/interpreter/http.test.d.ts +0 -1
- package/dist/interpreter/http.test.js +0 -299
- package/dist/interpreter/progress.test.d.ts +0 -1
- package/dist/interpreter/progress.test.js +0 -216
- package/dist/interpreter/schema-matcher.test.d.ts +0 -1
- package/dist/interpreter/schema-matcher.test.js +0 -122
- package/dist/lexer/lexer.d.ts +0 -24
- package/dist/lexer/lexer.js +0 -264
- package/dist/lexer/lexer.test.d.ts +0 -1
- package/dist/lexer/lexer.test.js +0 -259
- package/dist/loader/loader.test.d.ts +0 -1
- package/dist/loader/loader.test.js +0 -287
- package/dist/oas/oas.test.d.ts +0 -1
- package/dist/oas/oas.test.js +0 -218
- package/dist/parser/expressions.test.d.ts +0 -1
- package/dist/parser/expressions.test.js +0 -378
- package/dist/parser/match.test.d.ts +0 -1
- package/dist/parser/match.test.js +0 -254
- package/dist/parser/parser.test.d.ts +0 -1
- package/dist/parser/parser.test.js +0 -333
- package/dist/parser/schedule.test.d.ts +0 -1
- package/dist/parser/schedule.test.js +0 -241
- package/dist/scheduler/cron-parser.test.d.ts +0 -1
- package/dist/scheduler/cron-parser.test.js +0 -188
- package/dist/stores/file.test.d.ts +0 -1
- package/dist/stores/file.test.js +0 -165
- package/dist/stores/memory.test.d.ts +0 -1
- package/dist/stores/memory.test.js +0 -157
- package/dist/stores/stores.test.d.ts +0 -1
- package/dist/stores/stores.test.js +0 -158
- package/dist/sync/sync.test.d.ts +0 -1
- package/dist/sync/sync.test.js +0 -221
- package/docusaurus/README.md +0 -41
- package/docusaurus/docs/advanced/execution-state.md +0 -283
- package/docusaurus/docs/advanced/extending-reqon.md +0 -388
- package/docusaurus/docs/advanced/multi-file-missions.md +0 -250
- package/docusaurus/docs/advanced/parallel-execution.md +0 -353
- package/docusaurus/docs/api-reference.md +0 -443
- package/docusaurus/docs/authentication/api-key.md +0 -339
- package/docusaurus/docs/authentication/basic.md +0 -276
- package/docusaurus/docs/authentication/bearer.md +0 -282
- package/docusaurus/docs/authentication/oauth2.md +0 -317
- package/docusaurus/docs/authentication/overview.md +0 -251
- package/docusaurus/docs/cli.md +0 -229
- package/docusaurus/docs/core-concepts/actions.md +0 -286
- package/docusaurus/docs/core-concepts/missions.md +0 -264
- package/docusaurus/docs/core-concepts/schemas.md +0 -353
- package/docusaurus/docs/core-concepts/sources.md +0 -339
- package/docusaurus/docs/core-concepts/stores.md +0 -332
- package/docusaurus/docs/dsl-syntax/expressions.md +0 -361
- package/docusaurus/docs/dsl-syntax/fetch.md +0 -293
- package/docusaurus/docs/dsl-syntax/for-loops.md +0 -324
- package/docusaurus/docs/dsl-syntax/map.md +0 -345
- package/docusaurus/docs/dsl-syntax/match.md +0 -387
- package/docusaurus/docs/dsl-syntax/pipelines.md +0 -397
- package/docusaurus/docs/dsl-syntax/validate.md +0 -401
- package/docusaurus/docs/error-handling/dead-letter-queues.md +0 -399
- package/docusaurus/docs/error-handling/flow-control.md +0 -337
- package/docusaurus/docs/error-handling/retry-strategies.md +0 -368
- package/docusaurus/docs/examples.md +0 -488
- package/docusaurus/docs/getting-started.md +0 -256
- package/docusaurus/docs/http/circuit-breaker.md +0 -401
- package/docusaurus/docs/http/incremental-sync.md +0 -394
- package/docusaurus/docs/http/pagination.md +0 -361
- package/docusaurus/docs/http/rate-limiting.md +0 -383
- package/docusaurus/docs/http/requests.md +0 -328
- package/docusaurus/docs/http/retry.md +0 -402
- package/docusaurus/docs/intro.md +0 -90
- package/docusaurus/docs/openapi/loading-specs.md +0 -305
- package/docusaurus/docs/openapi/operation-calls.md +0 -314
- package/docusaurus/docs/openapi/overview.md +0 -212
- package/docusaurus/docs/openapi/response-validation.md +0 -344
- package/docusaurus/docs/scheduling/cron.md +0 -305
- package/docusaurus/docs/scheduling/daemon-mode.md +0 -317
- package/docusaurus/docs/scheduling/intervals.md +0 -289
- package/docusaurus/docs/scheduling/overview.md +0 -231
- package/docusaurus/docs/stores/custom-adapters.md +0 -376
- package/docusaurus/docs/stores/file.md +0 -236
- package/docusaurus/docs/stores/memory.md +0 -193
- package/docusaurus/docs/stores/overview.md +0 -274
- package/docusaurus/docs/stores/postgrest.md +0 -316
- package/docusaurus/docusaurus.config.ts +0 -148
- package/docusaurus/package-lock.json +0 -18029
- package/docusaurus/package.json +0 -47
- package/docusaurus/sidebars.ts +0 -155
- package/docusaurus/src/components/HomepageFeatures/index.tsx +0 -105
- package/docusaurus/src/components/HomepageFeatures/styles.module.css +0 -12
- package/docusaurus/src/css/custom.css +0 -169
- package/docusaurus/src/pages/index.module.css +0 -48
- package/docusaurus/src/pages/index.tsx +0 -110
- package/docusaurus/src/pages/markdown-page.md +0 -7
- package/docusaurus/static/.nojekyll +0 -0
- package/docusaurus/static/img/docusaurus-social-card.jpg +0 -0
- package/docusaurus/static/img/docusaurus.png +0 -0
- package/docusaurus/static/img/favicon.ico +0 -0
- package/docusaurus/static/img/logo.svg +0 -10
- package/docusaurus/static/img/undraw_docusaurus_mountain.svg +0 -171
- package/docusaurus/static/img/undraw_docusaurus_react.svg +0 -170
- package/docusaurus/static/img/undraw_docusaurus_tree.svg +0 -40
- package/docusaurus/tsconfig.json +0 -8
- package/examples/README.md +0 -112
- package/examples/error-handling/README.md +0 -150
- package/examples/error-handling/payment-processor.vague +0 -287
- package/examples/github-sync/README.md +0 -74
- package/examples/github-sync/fetch-issues.vague +0 -47
- package/examples/github-sync/fetch-prs.vague +0 -40
- package/examples/github-sync/mission.vague +0 -101
- package/examples/github-sync/normalize.vague +0 -70
- package/examples/jsonplaceholder/README.md +0 -28
- package/examples/jsonplaceholder/posts.vague +0 -48
- package/examples/petstore/README.md +0 -35
- package/examples/petstore/openapi.yaml +0 -97
- package/examples/petstore/sync.vague +0 -52
- package/examples/temporal-comparison/README.md +0 -297
- package/examples/temporal-comparison/reconciliation.vague +0 -355
- package/examples/temporal-comparison/temporal/activities/index.ts +0 -8
- package/examples/temporal-comparison/temporal/activities/shipstation.ts +0 -225
- package/examples/temporal-comparison/temporal/activities/shopify.ts +0 -257
- package/examples/temporal-comparison/temporal/activities/storage.ts +0 -198
- package/examples/temporal-comparison/temporal/activities/stripe.ts +0 -169
- package/examples/temporal-comparison/temporal/activities/validation.ts +0 -205
- package/examples/temporal-comparison/temporal/client/schedule.ts +0 -218
- package/examples/temporal-comparison/temporal/config/retry.ts +0 -63
- package/examples/temporal-comparison/temporal/types/index.ts +0 -129
- package/examples/temporal-comparison/temporal/workers/main.ts +0 -130
- package/examples/temporal-comparison/temporal/workflows/orderReconciliation.ts +0 -262
- package/examples/xero/README.md +0 -88
- package/examples/xero/invoices.vague +0 -189
- package/src/api-integration.test.ts +0 -954
- package/src/ast/index.ts +0 -1
- package/src/ast/nodes.ts +0 -310
- package/src/auth/auth.test.ts +0 -326
- package/src/auth/circuit-breaker.test.ts +0 -390
- package/src/auth/circuit-breaker.ts +0 -379
- package/src/auth/credentials.test.ts +0 -273
- package/src/auth/credentials.ts +0 -246
- package/src/auth/index.ts +0 -40
- package/src/auth/oauth2-provider.ts +0 -177
- package/src/auth/rate-limiter.ts +0 -459
- package/src/auth/token-store.ts +0 -177
- package/src/auth/types.ts +0 -159
- package/src/benchmark/e2e.bench.ts +0 -288
- package/src/benchmark/evaluator.bench.ts +0 -331
- package/src/benchmark/fixtures.ts +0 -295
- package/src/benchmark/index.ts +0 -108
- package/src/benchmark/lexer.bench.ts +0 -69
- package/src/benchmark/parser.bench.ts +0 -103
- package/src/benchmark/resilience.bench.ts +0 -193
- package/src/benchmark/store.bench.ts +0 -147
- package/src/benchmark/utils.ts +0 -230
- package/src/cli.ts +0 -313
- package/src/errors/errors.test.ts +0 -234
- package/src/errors/index.ts +0 -223
- package/src/execution/execution.test.ts +0 -307
- package/src/execution/index.ts +0 -21
- package/src/execution/state.ts +0 -207
- package/src/execution/store.ts +0 -188
- package/src/index.ts +0 -169
- package/src/integration.test.ts +0 -192
- package/src/interpreter/context.ts +0 -57
- package/src/interpreter/evaluator.test.ts +0 -796
- package/src/interpreter/evaluator.ts +0 -245
- package/src/interpreter/executor.ts +0 -946
- package/src/interpreter/fetch-handler.ts +0 -302
- package/src/interpreter/http.test.ts +0 -423
- package/src/interpreter/http.ts +0 -308
- package/src/interpreter/index.ts +0 -32
- package/src/interpreter/pagination.ts +0 -207
- package/src/interpreter/progress.test.ts +0 -276
- package/src/interpreter/schema-matcher.test.ts +0 -160
- package/src/interpreter/schema-matcher.ts +0 -168
- package/src/interpreter/signals.ts +0 -73
- package/src/interpreter/step-handlers/for-handler.ts +0 -65
- package/src/interpreter/step-handlers/index.ts +0 -17
- package/src/interpreter/step-handlers/map-handler.ts +0 -24
- package/src/interpreter/step-handlers/match-handler.ts +0 -101
- package/src/interpreter/step-handlers/store-handler.ts +0 -78
- package/src/interpreter/step-handlers/types.ts +0 -17
- package/src/interpreter/step-handlers/validate-handler.ts +0 -30
- package/src/interpreter/step-handlers/webhook-handler.ts +0 -142
- package/src/lexer/index.ts +0 -18
- package/src/lexer/lexer.test.ts +0 -316
- package/src/lexer/tokens.ts +0 -179
- package/src/loader/index.ts +0 -288
- package/src/loader/loader.test.ts +0 -360
- package/src/oas/index.ts +0 -4
- package/src/oas/loader.ts +0 -126
- package/src/oas/oas.test.ts +0 -254
- package/src/oas/validator.ts +0 -299
- package/src/parser/base.ts +0 -124
- package/src/parser/expressions.test.ts +0 -525
- package/src/parser/expressions.ts +0 -314
- package/src/parser/index.ts +0 -3
- package/src/parser/match.test.ts +0 -296
- package/src/parser/parser.test.ts +0 -739
- package/src/parser/parser.ts +0 -1469
- package/src/parser/schedule.test.ts +0 -287
- package/src/parser/webhook.test.ts +0 -248
- package/src/plugin.ts +0 -83
- package/src/scheduler/cron-parser.test.ts +0 -236
- package/src/scheduler/cron-parser.ts +0 -236
- package/src/scheduler/index.ts +0 -10
- package/src/scheduler/scheduler.ts +0 -443
- package/src/scheduler/types.ts +0 -71
- package/src/stores/factory.ts +0 -104
- package/src/stores/file.test.ts +0 -276
- package/src/stores/file.ts +0 -211
- package/src/stores/index.ts +0 -6
- package/src/stores/memory.test.ts +0 -238
- package/src/stores/memory.ts +0 -63
- package/src/stores/postgrest.test.ts +0 -488
- package/src/stores/postgrest.ts +0 -263
- package/src/stores/stores.test.ts +0 -197
- package/src/stores/types.ts +0 -58
- package/src/sync/index.ts +0 -16
- package/src/sync/state.ts +0 -126
- package/src/sync/store.ts +0 -139
- package/src/sync/sync.test.ts +0 -271
- package/src/utils/async.ts +0 -10
- package/src/utils/file.ts +0 -106
- package/src/utils/index.ts +0 -14
- package/src/utils/logger.ts +0 -53
- package/src/utils/path.ts +0 -47
- package/src/webhook/index.ts +0 -15
- package/src/webhook/server.test.ts +0 -253
- package/src/webhook/server.ts +0 -389
- package/src/webhook/store.ts +0 -239
- package/src/webhook/types.ts +0 -93
- package/tsconfig.json +0 -17
- package/vitest.config.ts +0 -39
|
@@ -2,6 +2,7 @@ import type { FetchStep } from '../ast/nodes.js';
|
|
|
2
2
|
import type { ExecutionContext } from './context.js';
|
|
3
3
|
import type { OASSource } from '../oas/index.js';
|
|
4
4
|
import { type SyncStore } from '../sync/index.js';
|
|
5
|
+
import type { EventType } from '../observability/index.js';
|
|
5
6
|
export interface FetchHandlerDeps {
|
|
6
7
|
ctx: ExecutionContext;
|
|
7
8
|
oasSources: Map<string, OASSource>;
|
|
@@ -15,6 +16,8 @@ export interface FetchHandlerDeps {
|
|
|
15
16
|
executionId?: string;
|
|
16
17
|
dryRun?: boolean;
|
|
17
18
|
log: (message: string) => void;
|
|
19
|
+
/** Optional event emitter for observability */
|
|
20
|
+
emit?: <T>(type: EventType, payload: T) => void;
|
|
18
21
|
}
|
|
19
22
|
export interface FetchResult {
|
|
20
23
|
data: unknown;
|
|
@@ -31,6 +34,7 @@ export declare class FetchHandler {
|
|
|
31
34
|
* Execute a fetch step, handling OAS resolution, pagination, and sync checkpoints.
|
|
32
35
|
*/
|
|
33
36
|
execute(step: FetchStep): Promise<FetchResult>;
|
|
37
|
+
private isRetryableError;
|
|
34
38
|
/**
|
|
35
39
|
* Record a sync checkpoint after successful fetch.
|
|
36
40
|
*/
|
|
@@ -40,4 +44,9 @@ export declare class FetchHandler {
|
|
|
40
44
|
private executePaginated;
|
|
41
45
|
private countRecords;
|
|
42
46
|
private validateOASResponse;
|
|
47
|
+
/**
|
|
48
|
+
* Generate mock data for dry run mode.
|
|
49
|
+
* Uses OAS response schema if available, otherwise returns a simple placeholder.
|
|
50
|
+
*/
|
|
51
|
+
private generateDryRunMockData;
|
|
43
52
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { evaluate, interpolatePath } from './evaluator.js';
|
|
2
|
-
import { resolveOperation, getResponseSchema, validateResponse } from '../oas/index.js';
|
|
2
|
+
import { resolveOperation, getResponseSchema, validateResponse, generateMockData, } from '../oas/index.js';
|
|
3
3
|
import { generateCheckpointKey, formatSinceDate } from '../sync/index.js';
|
|
4
4
|
import { extractNestedValue } from '../utils/path.js';
|
|
5
5
|
import { createPaginationStrategy } from './pagination.js';
|
|
@@ -23,29 +23,80 @@ export class FetchHandler {
|
|
|
23
23
|
if (!client) {
|
|
24
24
|
throw new Error(`Source not found: ${resolved.sourceName}`);
|
|
25
25
|
}
|
|
26
|
-
// Resolve "since" query parameter for incremental sync
|
|
27
|
-
const { query: sinceQuery, checkpointKey } = await this.resolveSinceParams(step, resolved.sourceName, resolved.operationId, resolved.path);
|
|
26
|
+
// Resolve "since" query parameter and/or header for incremental sync
|
|
27
|
+
const { query: sinceQuery, headers: sinceHeaders, checkpointKey, } = await this.resolveSinceParams(step, resolved.sourceName, resolved.operationId, resolved.path);
|
|
28
|
+
// Emit fetch.start event
|
|
29
|
+
this.deps.emit?.('fetch.start', {
|
|
30
|
+
source: resolved.sourceName,
|
|
31
|
+
method: resolved.method,
|
|
32
|
+
path: resolved.path,
|
|
33
|
+
isOAS: !!resolved.operationId,
|
|
34
|
+
operationId: resolved.operationId,
|
|
35
|
+
hasPagination: !!step.paginate,
|
|
36
|
+
hasSince: !!step.since,
|
|
37
|
+
});
|
|
38
|
+
const _fetchStartTime = Date.now(); // Reserved for fetch duration tracking
|
|
28
39
|
if (this.deps.dryRun) {
|
|
40
|
+
const mockData = this.generateDryRunMockData(resolved.sourceName, resolved.operationId);
|
|
29
41
|
this.deps.log('(dry run - skipping actual request)');
|
|
30
|
-
return { data:
|
|
42
|
+
return { data: mockData, checkpointKey };
|
|
31
43
|
}
|
|
32
44
|
// Execute with or without pagination
|
|
33
45
|
let data;
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
46
|
+
let pagesFetched;
|
|
47
|
+
let statusCode = 200;
|
|
48
|
+
try {
|
|
49
|
+
if (step.paginate) {
|
|
50
|
+
const result = await this.executePaginated(step, client, resolved.path, resolved.method, resolved.sourceName, resolved.operationId, sinceQuery, sinceHeaders);
|
|
51
|
+
data = result;
|
|
52
|
+
pagesFetched = Array.isArray(result) ? undefined : 1; // Will be set by executePaginated
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
const response = await client.request({
|
|
56
|
+
method: resolved.method,
|
|
57
|
+
path: resolved.path,
|
|
58
|
+
query: Object.keys(sinceQuery).length > 0 ? sinceQuery : undefined,
|
|
59
|
+
headers: Object.keys(sinceHeaders).length > 0 ? sinceHeaders : undefined,
|
|
60
|
+
body: step.body ? evaluate(step.body, this.deps.ctx) : undefined,
|
|
61
|
+
}, step.retry);
|
|
62
|
+
await this.validateOASResponse(resolved.sourceName, resolved.operationId, response.data);
|
|
63
|
+
data = response.data;
|
|
64
|
+
statusCode = response.status ?? 200;
|
|
65
|
+
}
|
|
66
|
+
// Emit fetch.complete event
|
|
67
|
+
this.deps.emit?.('fetch.complete', {
|
|
68
|
+
source: resolved.sourceName,
|
|
39
69
|
method: resolved.method,
|
|
40
70
|
path: resolved.path,
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
71
|
+
statusCode,
|
|
72
|
+
recordCount: this.countRecords(data) ?? 0,
|
|
73
|
+
pagesFetched,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
// Emit fetch.error event
|
|
78
|
+
this.deps.emit?.('fetch.error', {
|
|
79
|
+
source: resolved.sourceName,
|
|
80
|
+
path: resolved.path,
|
|
81
|
+
error: error.message,
|
|
82
|
+
retryable: this.isRetryableError(error),
|
|
83
|
+
});
|
|
84
|
+
throw error;
|
|
46
85
|
}
|
|
47
86
|
return { data, checkpointKey };
|
|
48
87
|
}
|
|
88
|
+
isRetryableError(error) {
|
|
89
|
+
if (error instanceof Error) {
|
|
90
|
+
const message = error.message.toLowerCase();
|
|
91
|
+
return (message.includes('timeout') ||
|
|
92
|
+
message.includes('network') ||
|
|
93
|
+
message.includes('rate limit') ||
|
|
94
|
+
message.includes('429') ||
|
|
95
|
+
message.includes('503') ||
|
|
96
|
+
message.includes('502'));
|
|
97
|
+
}
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
49
100
|
/**
|
|
50
101
|
* Record a sync checkpoint after successful fetch.
|
|
51
102
|
*/
|
|
@@ -76,6 +127,13 @@ export class FetchHandler {
|
|
|
76
127
|
executionId: this.deps.executionId,
|
|
77
128
|
});
|
|
78
129
|
this.deps.log(`Recorded sync checkpoint: ${key} at ${syncedAt.toISOString()}`);
|
|
130
|
+
// Emit sync.checkpoint event
|
|
131
|
+
this.deps.emit?.('sync.checkpoint', {
|
|
132
|
+
checkpointKey: key,
|
|
133
|
+
lastSyncTime: syncedAt.toISOString(),
|
|
134
|
+
recordsFetched: recordCount ?? 0,
|
|
135
|
+
isIncremental: true,
|
|
136
|
+
});
|
|
79
137
|
}
|
|
80
138
|
resolveFetchTarget(step) {
|
|
81
139
|
if (step.operationRef) {
|
|
@@ -92,7 +150,15 @@ export class FetchHandler {
|
|
|
92
150
|
return { sourceName, method: operation.method, path, operationId };
|
|
93
151
|
}
|
|
94
152
|
// Traditional: explicit method + path
|
|
95
|
-
|
|
153
|
+
let sourceName = step.source;
|
|
154
|
+
if (!sourceName) {
|
|
155
|
+
// Use the first available source as default
|
|
156
|
+
const firstSource = this.deps.ctx.sources.keys().next();
|
|
157
|
+
if (firstSource.done) {
|
|
158
|
+
throw new Error('No sources defined. Add a source to your mission before making fetch requests.');
|
|
159
|
+
}
|
|
160
|
+
sourceName = firstSource.value;
|
|
161
|
+
}
|
|
96
162
|
const method = step.method;
|
|
97
163
|
let path;
|
|
98
164
|
if (step.path.type === 'Literal' && step.path.dataType === 'string') {
|
|
@@ -106,25 +172,40 @@ export class FetchHandler {
|
|
|
106
172
|
}
|
|
107
173
|
async resolveSinceParams(step, sourceName, operationId, path) {
|
|
108
174
|
const query = {};
|
|
175
|
+
const headers = {};
|
|
109
176
|
if (!step.since || !this.deps.syncStore) {
|
|
110
|
-
return { query };
|
|
177
|
+
return { query, headers };
|
|
111
178
|
}
|
|
112
179
|
const checkpointKey = step.since.key ?? generateCheckpointKey(sourceName, operationId, path);
|
|
113
180
|
if (step.since.type === 'lastSync') {
|
|
114
181
|
const lastSync = await this.deps.syncStore.getLastSync(checkpointKey);
|
|
115
|
-
const paramName = step.since.param ?? 'since';
|
|
116
182
|
const format = step.since.format ?? 'iso';
|
|
117
|
-
|
|
118
|
-
|
|
183
|
+
const formattedDate = formatSinceDate(lastSync, format);
|
|
184
|
+
if (step.since.header) {
|
|
185
|
+
// Send as header (e.g., If-Modified-Since)
|
|
186
|
+
headers[step.since.header] = formattedDate;
|
|
187
|
+
this.deps.log(`Incremental sync: header ${step.since.header}=${formattedDate} (key: ${checkpointKey})`);
|
|
188
|
+
}
|
|
189
|
+
else {
|
|
190
|
+
// Send as query parameter (default behavior)
|
|
191
|
+
const paramName = step.since.param ?? 'since';
|
|
192
|
+
query[paramName] = formattedDate;
|
|
193
|
+
this.deps.log(`Incremental sync: ${paramName}=${formattedDate} (key: ${checkpointKey})`);
|
|
194
|
+
}
|
|
119
195
|
}
|
|
120
196
|
else if (step.since.type === 'expression' && step.since.expression) {
|
|
121
197
|
const value = evaluate(step.since.expression, this.deps.ctx);
|
|
122
|
-
|
|
123
|
-
|
|
198
|
+
if (step.since.header) {
|
|
199
|
+
headers[step.since.header] = String(value);
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
const paramName = step.since.param ?? 'since';
|
|
203
|
+
query[paramName] = String(value);
|
|
204
|
+
}
|
|
124
205
|
}
|
|
125
|
-
return { query, checkpointKey };
|
|
206
|
+
return { query, headers, checkpointKey };
|
|
126
207
|
}
|
|
127
|
-
async executePaginated(step, client, basePath, method, sourceName, operationId, sinceQuery = {}) {
|
|
208
|
+
async executePaginated(step, client, basePath, method, sourceName, operationId, sinceQuery = {}, sinceHeaders = {}) {
|
|
128
209
|
const allResults = [];
|
|
129
210
|
const paginate = step.paginate;
|
|
130
211
|
const strategy = createPaginationStrategy(paginate);
|
|
@@ -137,8 +218,9 @@ export class FetchHandler {
|
|
|
137
218
|
// Build query with pagination params
|
|
138
219
|
const paginationQuery = strategy.buildQuery(ctx);
|
|
139
220
|
const query = { ...sinceQuery, ...paginationQuery };
|
|
221
|
+
const headers = Object.keys(sinceHeaders).length > 0 ? sinceHeaders : undefined;
|
|
140
222
|
this.deps.log(`Fetching page ${ctx.page + 1}...`);
|
|
141
|
-
const response = await client.request({ method, path: basePath, query }, step.retry);
|
|
223
|
+
const response = await client.request({ method, path: basePath, query, headers }, step.retry);
|
|
142
224
|
await this.validateOASResponse(sourceName, operationId, response.data);
|
|
143
225
|
// Temporarily set response for until condition evaluation
|
|
144
226
|
this.deps.ctx.response = response.data;
|
|
@@ -158,6 +240,14 @@ export class FetchHandler {
|
|
|
158
240
|
ctx.cursor = pageResult.nextCursor;
|
|
159
241
|
}
|
|
160
242
|
ctx.page++;
|
|
243
|
+
// Emit heartbeat after each page
|
|
244
|
+
this.deps.emit?.('fetch.heartbeat', {
|
|
245
|
+
source: sourceName,
|
|
246
|
+
path: basePath,
|
|
247
|
+
pagesProcessed: ctx.page,
|
|
248
|
+
itemsFetched: allResults.length,
|
|
249
|
+
hasMore,
|
|
250
|
+
});
|
|
161
251
|
// Safety limit
|
|
162
252
|
if (ctx.page >= MAX_PAGINATION_PAGES) {
|
|
163
253
|
this.deps.log(`Warning: pagination limit (${MAX_PAGINATION_PAGES}) reached`);
|
|
@@ -200,4 +290,23 @@ export class FetchHandler {
|
|
|
200
290
|
this.deps.log(`Response validation warnings for ${operationId}:\n${errorMessages}`);
|
|
201
291
|
}
|
|
202
292
|
}
|
|
293
|
+
/**
|
|
294
|
+
* Generate mock data for dry run mode.
|
|
295
|
+
* Uses OAS response schema if available, otherwise returns a simple placeholder.
|
|
296
|
+
*/
|
|
297
|
+
generateDryRunMockData(sourceName, operationId) {
|
|
298
|
+
// Try to get schema from OAS source
|
|
299
|
+
if (operationId) {
|
|
300
|
+
const oasSource = this.deps.oasSources.get(sourceName);
|
|
301
|
+
if (oasSource) {
|
|
302
|
+
const schema = getResponseSchema(oasSource, operationId);
|
|
303
|
+
if (schema) {
|
|
304
|
+
this.deps.log(`Generating mock data from OAS schema for ${operationId}`);
|
|
305
|
+
return generateMockData(schema);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
// Fallback to simple placeholder
|
|
310
|
+
return { _dryRun: true, _message: 'No OAS schema available for mock generation' };
|
|
311
|
+
}
|
|
203
312
|
}
|
|
@@ -30,6 +30,11 @@ export declare class HttpClient {
|
|
|
30
30
|
private config;
|
|
31
31
|
constructor(config: HttpClientConfig);
|
|
32
32
|
request<T = unknown>(req: HttpRequest, retry?: RetryConfig): Promise<HttpResponse<T>>;
|
|
33
|
+
/**
|
|
34
|
+
* Safely parse response body, handling non-JSON responses gracefully.
|
|
35
|
+
* Attempts JSON parsing first, providing helpful errors on failure.
|
|
36
|
+
*/
|
|
37
|
+
private parseResponseBody;
|
|
33
38
|
private buildUrl;
|
|
34
39
|
private buildHeaders;
|
|
35
40
|
private calculateDelay;
|
package/dist/interpreter/http.js
CHANGED
|
@@ -1,13 +1,8 @@
|
|
|
1
1
|
import { parseRateLimitHeaders } from '../auth/rate-limiter.js';
|
|
2
2
|
import { CircuitBreakerError } from '../auth/circuit-breaker.js';
|
|
3
3
|
import { sleep } from '../utils/async.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
MAX_ATTEMPTS: 3,
|
|
7
|
-
INITIAL_DELAY_MS: 1000,
|
|
8
|
-
MAX_DELAY_MS: 30000,
|
|
9
|
-
BACKOFF: 'exponential',
|
|
10
|
-
};
|
|
4
|
+
import { HTTP_RETRY_DEFAULTS } from '../config/index.js';
|
|
5
|
+
import { FetchError } from '../errors/index.js';
|
|
11
6
|
export class HttpClient {
|
|
12
7
|
config;
|
|
13
8
|
constructor(config) {
|
|
@@ -21,10 +16,10 @@ export class HttpClient {
|
|
|
21
16
|
headers,
|
|
22
17
|
body: req.body ? JSON.stringify(req.body) : undefined,
|
|
23
18
|
};
|
|
24
|
-
const maxAttempts = retry?.maxAttempts ??
|
|
25
|
-
const backoff = retry?.backoff ??
|
|
26
|
-
const initialDelay = retry?.initialDelay ??
|
|
27
|
-
const maxDelay = retry?.maxDelay ??
|
|
19
|
+
const maxAttempts = retry?.maxAttempts ?? HTTP_RETRY_DEFAULTS.MAX_ATTEMPTS;
|
|
20
|
+
const backoff = retry?.backoff ?? HTTP_RETRY_DEFAULTS.BACKOFF;
|
|
21
|
+
const initialDelay = retry?.initialDelay ?? HTTP_RETRY_DEFAULTS.INITIAL_DELAY_MS;
|
|
22
|
+
const maxDelay = retry?.maxDelay ?? HTTP_RETRY_DEFAULTS.MAX_DELAY_MS;
|
|
28
23
|
let lastError = null;
|
|
29
24
|
// Check circuit breaker before attempting requests
|
|
30
25
|
if (this.config.circuitBreaker && this.config.sourceName) {
|
|
@@ -88,7 +83,7 @@ export class HttpClient {
|
|
|
88
83
|
fetchOptions.headers = newHeaders;
|
|
89
84
|
continue;
|
|
90
85
|
}
|
|
91
|
-
const data = await response.
|
|
86
|
+
const data = await this.parseResponseBody(response, url, req.method);
|
|
92
87
|
// Record success in circuit breaker
|
|
93
88
|
if (this.config.circuitBreaker && this.config.sourceName && response.status < 500) {
|
|
94
89
|
this.config.circuitBreaker.recordSuccess(this.config.sourceName, req.path);
|
|
@@ -117,6 +112,25 @@ export class HttpClient {
|
|
|
117
112
|
}
|
|
118
113
|
throw lastError ?? new Error('Request failed after all retries');
|
|
119
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* Safely parse response body, handling non-JSON responses gracefully.
|
|
117
|
+
* Attempts JSON parsing first, providing helpful errors on failure.
|
|
118
|
+
*/
|
|
119
|
+
async parseResponseBody(response, url, method) {
|
|
120
|
+
const contentType = response.headers.get('content-type') ?? '';
|
|
121
|
+
// Always attempt JSON parsing - many APIs don't set content-type correctly
|
|
122
|
+
try {
|
|
123
|
+
return await response.json();
|
|
124
|
+
}
|
|
125
|
+
catch (parseError) {
|
|
126
|
+
// Provide context about content-type mismatch if applicable
|
|
127
|
+
const isJsonContentType = contentType.includes('application/json') || contentType === '';
|
|
128
|
+
const contentTypeHint = !isJsonContentType
|
|
129
|
+
? ` (content-type was '${contentType}')`
|
|
130
|
+
: '';
|
|
131
|
+
throw new FetchError(`Failed to parse JSON response${contentTypeHint}: ${parseError.message}`, { url, method, statusCode: response.status, cause: parseError });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
120
134
|
buildUrl(path, query) {
|
|
121
135
|
const base = this.config.baseUrl.replace(/\/$/, '');
|
|
122
136
|
const cleanPath = path.startsWith('/') ? path : `/${path}`;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
export { MissionExecutor, type ExecutionResult, type ExecutionError, type ExecutorConfig, type ProgressCallbacks, type ExecutionStartEvent, type ExecutionCompleteEvent, type StageStartEvent, type StageCompleteEvent, } from './executor.js';
|
|
1
|
+
export { MissionExecutor, type ExecutionResult, type ExecutionError, type ExecutorConfig, type ProgressCallbacks, type ExecutionStartEvent, type ExecutionCompleteEvent, type StageStartEvent, type StageCompleteEvent, type AuthConfig, } from './executor.js';
|
|
2
2
|
export { HttpClient, BearerAuthProvider, OAuth2AuthProvider, type HttpClientConfig, type AuthProvider } from './http.js';
|
|
3
|
+
export { SourceManager, type SourceManagerConfig, type SourceManagerDeps } from './source-manager.js';
|
|
4
|
+
export { StoreManager, type StoreManagerConfig } from './store-manager.js';
|
|
3
5
|
export { createContext, childContext, getVariable, setVariable, type ExecutionContext } from './context.js';
|
|
4
6
|
export { evaluate, evaluateToString, interpolatePath } from './evaluator.js';
|
|
5
7
|
export { FetchHandler, type FetchHandlerDeps, type FetchResult } from './fetch-handler.js';
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export { MissionExecutor, } from './executor.js';
|
|
2
2
|
export { HttpClient, BearerAuthProvider, OAuth2AuthProvider } from './http.js';
|
|
3
|
+
export { SourceManager } from './source-manager.js';
|
|
4
|
+
export { StoreManager } from './store-manager.js';
|
|
3
5
|
export { createContext, childContext, getVariable, setVariable } from './context.js';
|
|
4
6
|
export { evaluate, evaluateToString, interpolatePath } from './evaluator.js';
|
|
5
7
|
export { FetchHandler } from './fetch-handler.js';
|
|
@@ -23,8 +23,13 @@ export interface PaginationStrategy {
|
|
|
23
23
|
buildQuery(ctx: PaginationContext): Record<string, string>;
|
|
24
24
|
/** Extract results and determine if more pages exist */
|
|
25
25
|
extractResults(response: unknown, ctx: PaginationContext): PageResult;
|
|
26
|
+
/** Clear any cached state (for reuse across different responses) */
|
|
27
|
+
clearCache?(): void;
|
|
26
28
|
}
|
|
27
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* @deprecated Use strategy.clearCache() instead. Kept for backward compatibility.
|
|
31
|
+
* Note: This now only clears the global compatibility cache, not instance caches.
|
|
32
|
+
*/
|
|
28
33
|
export declare function clearPaginationCache(): void;
|
|
29
34
|
/**
|
|
30
35
|
* Offset-based pagination (e.g., offset=100, offset=200)
|
|
@@ -32,9 +37,11 @@ export declare function clearPaginationCache(): void;
|
|
|
32
37
|
export declare class OffsetPaginationStrategy implements PaginationStrategy {
|
|
33
38
|
private config;
|
|
34
39
|
private cacheKey;
|
|
40
|
+
private cache;
|
|
35
41
|
constructor(config: PaginationConfig);
|
|
36
42
|
buildQuery(ctx: PaginationContext): Record<string, string>;
|
|
37
43
|
extractResults(response: unknown, ctx: PaginationContext): PageResult;
|
|
44
|
+
clearCache(): void;
|
|
38
45
|
}
|
|
39
46
|
/**
|
|
40
47
|
* Page number pagination (e.g., page=1, page=2)
|
|
@@ -42,20 +49,22 @@ export declare class OffsetPaginationStrategy implements PaginationStrategy {
|
|
|
42
49
|
export declare class PageNumberPaginationStrategy implements PaginationStrategy {
|
|
43
50
|
private config;
|
|
44
51
|
private cacheKey;
|
|
52
|
+
private cache;
|
|
45
53
|
constructor(config: PaginationConfig);
|
|
46
54
|
buildQuery(ctx: PaginationContext): Record<string, string>;
|
|
47
55
|
extractResults(response: unknown, ctx: PaginationContext): PageResult;
|
|
56
|
+
clearCache(): void;
|
|
48
57
|
}
|
|
49
58
|
/**
|
|
50
59
|
* Cursor-based pagination (e.g., cursor=abc123)
|
|
51
60
|
*/
|
|
52
61
|
export declare class CursorPaginationStrategy implements PaginationStrategy {
|
|
53
62
|
private config;
|
|
54
|
-
private cacheKey;
|
|
55
63
|
private cachedArrayField;
|
|
56
64
|
constructor(config: PaginationConfig);
|
|
57
65
|
buildQuery(ctx: PaginationContext): Record<string, string>;
|
|
58
66
|
extractResults(response: unknown, ctx: PaginationContext): PageResult;
|
|
67
|
+
clearCache(): void;
|
|
59
68
|
}
|
|
60
69
|
/**
|
|
61
70
|
* Create the appropriate pagination strategy based on config
|
|
@@ -1,40 +1,91 @@
|
|
|
1
1
|
import { extractNestedValue } from '../utils/path.js';
|
|
2
|
-
/**
|
|
3
|
-
const
|
|
2
|
+
/** Default TTL for cache entries (5 minutes) */
|
|
3
|
+
const DEFAULT_CACHE_TTL_MS = 5 * 60 * 1000;
|
|
4
|
+
/** Maximum number of cache entries before cleanup */
|
|
5
|
+
const MAX_CACHE_ENTRIES = 100;
|
|
6
|
+
/**
|
|
7
|
+
* Instance-level cache for array field discovery.
|
|
8
|
+
* Each pagination strategy instance has its own cache to avoid global state pollution.
|
|
9
|
+
*/
|
|
10
|
+
class ArrayFieldCache {
|
|
11
|
+
cache = new Map();
|
|
12
|
+
ttlMs;
|
|
13
|
+
constructor(ttlMs = DEFAULT_CACHE_TTL_MS) {
|
|
14
|
+
this.ttlMs = ttlMs;
|
|
15
|
+
}
|
|
16
|
+
get(key) {
|
|
17
|
+
const entry = this.cache.get(key);
|
|
18
|
+
if (!entry) {
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
// Check if expired
|
|
22
|
+
if (Date.now() > entry.expiresAt) {
|
|
23
|
+
this.cache.delete(key);
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
return entry.value;
|
|
27
|
+
}
|
|
28
|
+
set(key, value) {
|
|
29
|
+
// Cleanup if cache is getting too large
|
|
30
|
+
if (this.cache.size >= MAX_CACHE_ENTRIES) {
|
|
31
|
+
this.cleanup();
|
|
32
|
+
}
|
|
33
|
+
this.cache.set(key, {
|
|
34
|
+
value,
|
|
35
|
+
expiresAt: Date.now() + this.ttlMs,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
clear() {
|
|
39
|
+
this.cache.clear();
|
|
40
|
+
}
|
|
41
|
+
cleanup() {
|
|
42
|
+
const now = Date.now();
|
|
43
|
+
for (const [key, entry] of this.cache) {
|
|
44
|
+
if (now > entry.expiresAt) {
|
|
45
|
+
this.cache.delete(key);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// If still too large, remove oldest half
|
|
49
|
+
if (this.cache.size >= MAX_CACHE_ENTRIES) {
|
|
50
|
+
const entries = Array.from(this.cache.entries());
|
|
51
|
+
entries.sort((a, b) => a[1].expiresAt - b[1].expiresAt);
|
|
52
|
+
const toRemove = entries.slice(0, Math.floor(entries.length / 2));
|
|
53
|
+
for (const [key] of toRemove) {
|
|
54
|
+
this.cache.delete(key);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
4
59
|
/**
|
|
5
60
|
* Extract items array from response and determine if more pages exist
|
|
6
61
|
* Shared utility for offset and page-based pagination strategies
|
|
7
|
-
* Caches the discovered array field for subsequent pages
|
|
8
62
|
*/
|
|
9
|
-
function extractItemsFromResponse(response, pageSize, cacheKey) {
|
|
63
|
+
function extractItemsFromResponse(response, pageSize, cacheKey, cache) {
|
|
10
64
|
if (!response || typeof response !== 'object') {
|
|
11
65
|
return { items: [], hasMore: false };
|
|
12
66
|
}
|
|
13
67
|
const data = response;
|
|
14
|
-
// Check cache first
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
if (cachedField
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
};
|
|
27
|
-
}
|
|
68
|
+
// Check cache first
|
|
69
|
+
const cachedField = cache.get(cacheKey);
|
|
70
|
+
if (cachedField !== undefined) {
|
|
71
|
+
if (cachedField === null) {
|
|
72
|
+
return { items: [], hasMore: false };
|
|
73
|
+
}
|
|
74
|
+
const items = data[cachedField];
|
|
75
|
+
if (Array.isArray(items)) {
|
|
76
|
+
return {
|
|
77
|
+
items,
|
|
78
|
+
hasMore: items.length >= pageSize,
|
|
79
|
+
};
|
|
28
80
|
}
|
|
81
|
+
// Cached field no longer valid, clear it
|
|
82
|
+
cache.clear();
|
|
29
83
|
}
|
|
30
84
|
// Search for array field
|
|
31
85
|
for (const key of Object.keys(data)) {
|
|
32
86
|
if (Array.isArray(data[key])) {
|
|
33
87
|
const items = data[key];
|
|
34
|
-
|
|
35
|
-
if (cacheKey) {
|
|
36
|
-
arrayFieldCache.set(cacheKey, key);
|
|
37
|
-
}
|
|
88
|
+
cache.set(cacheKey, key);
|
|
38
89
|
return {
|
|
39
90
|
items,
|
|
40
91
|
hasMore: items.length >= pageSize,
|
|
@@ -42,14 +93,16 @@ function extractItemsFromResponse(response, pageSize, cacheKey) {
|
|
|
42
93
|
}
|
|
43
94
|
}
|
|
44
95
|
// Cache negative result
|
|
45
|
-
|
|
46
|
-
arrayFieldCache.set(cacheKey, null);
|
|
47
|
-
}
|
|
96
|
+
cache.set(cacheKey, null);
|
|
48
97
|
return { items: [], hasMore: false };
|
|
49
98
|
}
|
|
50
|
-
/**
|
|
99
|
+
/**
|
|
100
|
+
* @deprecated Use strategy.clearCache() instead. Kept for backward compatibility.
|
|
101
|
+
* Note: This now only clears the global compatibility cache, not instance caches.
|
|
102
|
+
*/
|
|
51
103
|
export function clearPaginationCache() {
|
|
52
|
-
|
|
104
|
+
// No-op - caches are now instance-level
|
|
105
|
+
// Individual strategies should call clearCache() if needed
|
|
53
106
|
}
|
|
54
107
|
/**
|
|
55
108
|
* Offset-based pagination (e.g., offset=100, offset=200)
|
|
@@ -57,9 +110,11 @@ export function clearPaginationCache() {
|
|
|
57
110
|
export class OffsetPaginationStrategy {
|
|
58
111
|
config;
|
|
59
112
|
cacheKey;
|
|
113
|
+
cache;
|
|
60
114
|
constructor(config) {
|
|
61
115
|
this.config = config;
|
|
62
116
|
this.cacheKey = `offset:${config.param}`;
|
|
117
|
+
this.cache = new ArrayFieldCache();
|
|
63
118
|
}
|
|
64
119
|
buildQuery(ctx) {
|
|
65
120
|
return {
|
|
@@ -67,7 +122,10 @@ export class OffsetPaginationStrategy {
|
|
|
67
122
|
};
|
|
68
123
|
}
|
|
69
124
|
extractResults(response, ctx) {
|
|
70
|
-
return extractItemsFromResponse(response, ctx.pageSize, this.cacheKey);
|
|
125
|
+
return extractItemsFromResponse(response, ctx.pageSize, this.cacheKey, this.cache);
|
|
126
|
+
}
|
|
127
|
+
clearCache() {
|
|
128
|
+
this.cache.clear();
|
|
71
129
|
}
|
|
72
130
|
}
|
|
73
131
|
/**
|
|
@@ -76,9 +134,11 @@ export class OffsetPaginationStrategy {
|
|
|
76
134
|
export class PageNumberPaginationStrategy {
|
|
77
135
|
config;
|
|
78
136
|
cacheKey;
|
|
137
|
+
cache;
|
|
79
138
|
constructor(config) {
|
|
80
139
|
this.config = config;
|
|
81
140
|
this.cacheKey = `page:${config.param}`;
|
|
141
|
+
this.cache = new ArrayFieldCache();
|
|
82
142
|
}
|
|
83
143
|
buildQuery(ctx) {
|
|
84
144
|
return {
|
|
@@ -86,7 +146,10 @@ export class PageNumberPaginationStrategy {
|
|
|
86
146
|
};
|
|
87
147
|
}
|
|
88
148
|
extractResults(response, ctx) {
|
|
89
|
-
return extractItemsFromResponse(response, ctx.pageSize, this.cacheKey);
|
|
149
|
+
return extractItemsFromResponse(response, ctx.pageSize, this.cacheKey, this.cache);
|
|
150
|
+
}
|
|
151
|
+
clearCache() {
|
|
152
|
+
this.cache.clear();
|
|
90
153
|
}
|
|
91
154
|
}
|
|
92
155
|
/**
|
|
@@ -94,11 +157,9 @@ export class PageNumberPaginationStrategy {
|
|
|
94
157
|
*/
|
|
95
158
|
export class CursorPaginationStrategy {
|
|
96
159
|
config;
|
|
97
|
-
cacheKey;
|
|
98
160
|
cachedArrayField = null;
|
|
99
161
|
constructor(config) {
|
|
100
162
|
this.config = config;
|
|
101
|
-
this.cacheKey = `cursor:${config.param}`;
|
|
102
163
|
}
|
|
103
164
|
buildQuery(ctx) {
|
|
104
165
|
if (ctx.cursor) {
|
|
@@ -137,6 +198,9 @@ export class CursorPaginationStrategy {
|
|
|
137
198
|
nextCursor,
|
|
138
199
|
};
|
|
139
200
|
}
|
|
201
|
+
clearCache() {
|
|
202
|
+
this.cachedArrayField = null;
|
|
203
|
+
}
|
|
140
204
|
}
|
|
141
205
|
/**
|
|
142
206
|
* Create the appropriate pagination strategy based on config
|
|
@@ -55,3 +55,11 @@ export declare class NoMatchError extends Error {
|
|
|
55
55
|
export declare class AbortError extends Error {
|
|
56
56
|
constructor(message?: string);
|
|
57
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Signal thrown when execution is paused by external control or pause step
|
|
60
|
+
*/
|
|
61
|
+
export declare class PauseSignal extends Error {
|
|
62
|
+
/** Pause ID if created by a pause step */
|
|
63
|
+
pauseId?: string;
|
|
64
|
+
constructor(pauseId?: string);
|
|
65
|
+
}
|
|
@@ -71,3 +71,15 @@ export class AbortError extends Error {
|
|
|
71
71
|
this.name = 'AbortError';
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Signal thrown when execution is paused by external control or pause step
|
|
76
|
+
*/
|
|
77
|
+
export class PauseSignal extends Error {
|
|
78
|
+
/** Pause ID if created by a pause step */
|
|
79
|
+
pauseId;
|
|
80
|
+
constructor(pauseId) {
|
|
81
|
+
super(pauseId ? `Execution paused (${pauseId})` : 'Execution paused');
|
|
82
|
+
this.name = 'PauseSignal';
|
|
83
|
+
this.pauseId = pauseId;
|
|
84
|
+
}
|
|
85
|
+
}
|