scrapme-flow 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +126 -2
  3. package/dist/src/commands/commands.models.d.ts +41 -0
  4. package/dist/src/commands/commands.models.js +2 -0
  5. package/dist/src/commands/commands.models.js.map +1 -0
  6. package/dist/src/commands/save-document.command.d.ts +9 -0
  7. package/dist/src/commands/save-document.command.js +33 -0
  8. package/dist/src/commands/save-document.command.js.map +1 -0
  9. package/dist/src/commands/scrape-page.command.d.ts +20 -0
  10. package/dist/src/commands/scrape-page.command.js +93 -0
  11. package/dist/src/commands/scrape-page.command.js.map +1 -0
  12. package/dist/src/commands/send-mail.command.d.ts +11 -0
  13. package/dist/src/commands/send-mail.command.js +39 -0
  14. package/dist/src/commands/send-mail.command.js.map +1 -0
  15. package/dist/src/config/config-builder.d.ts +24 -0
  16. package/dist/src/config/config-builder.js +129 -0
  17. package/dist/src/config/config-builder.js.map +1 -0
  18. package/dist/src/config/config-loader.d.ts +10 -0
  19. package/dist/src/config/config-loader.js +79 -0
  20. package/dist/src/config/config-loader.js.map +1 -0
  21. package/dist/src/config/config.models.d.ts +144 -0
  22. package/dist/src/config/config.models.js +2 -0
  23. package/dist/src/config/config.models.js.map +1 -0
  24. package/dist/src/config/config.schema.d.ts +75 -0
  25. package/dist/src/config/config.schema.js +87 -0
  26. package/dist/src/config/config.schema.js.map +1 -0
  27. package/dist/src/config/dsn-check.d.ts +5 -0
  28. package/dist/src/config/dsn-check.js +12 -0
  29. package/dist/src/config/dsn-check.js.map +1 -0
  30. package/dist/src/di/container.d.ts +13 -0
  31. package/dist/src/di/container.js +67 -0
  32. package/dist/src/di/container.js.map +1 -0
  33. package/dist/src/di/types.d.ts +27 -0
  34. package/dist/src/di/types.js +28 -0
  35. package/dist/src/di/types.js.map +1 -0
  36. package/dist/src/flow/command-executor.d.ts +18 -0
  37. package/dist/src/flow/command-executor.js +87 -0
  38. package/dist/src/flow/command-executor.js.map +1 -0
  39. package/dist/src/flow/command-resolver.d.ts +11 -0
  40. package/dist/src/flow/command-resolver.js +35 -0
  41. package/dist/src/flow/command-resolver.js.map +1 -0
  42. package/dist/src/flow/flow-catalog.d.ts +12 -0
  43. package/dist/src/flow/flow-catalog.js +31 -0
  44. package/dist/src/flow/flow-catalog.js.map +1 -0
  45. package/dist/src/flow/flow-errors.d.ts +66 -0
  46. package/dist/src/flow/flow-errors.js +119 -0
  47. package/dist/src/flow/flow-errors.js.map +1 -0
  48. package/dist/src/flow/flow-job-processor.d.ts +33 -0
  49. package/dist/src/flow/flow-job-processor.js +208 -0
  50. package/dist/src/flow/flow-job-processor.js.map +1 -0
  51. package/dist/src/flow/flow-params.d.ts +9 -0
  52. package/dist/src/flow/flow-params.js +52 -0
  53. package/dist/src/flow/flow-params.js.map +1 -0
  54. package/dist/src/flow/flow-runner.d.ts +36 -0
  55. package/dist/src/flow/flow-runner.js +209 -0
  56. package/dist/src/flow/flow-runner.js.map +1 -0
  57. package/dist/src/flow/flow-scheduler.d.ts +24 -0
  58. package/dist/src/flow/flow-scheduler.js +96 -0
  59. package/dist/src/flow/flow-scheduler.js.map +1 -0
  60. package/dist/src/flow/flow-validator.d.ts +29 -0
  61. package/dist/src/flow/flow-validator.js +141 -0
  62. package/dist/src/flow/flow-validator.js.map +1 -0
  63. package/dist/src/flow/flow.d.ts +32 -0
  64. package/dist/src/flow/flow.js +50 -0
  65. package/dist/src/flow/flow.js.map +1 -0
  66. package/dist/src/flow/flow.models.d.ts +288 -0
  67. package/dist/src/flow/flow.models.js +38 -0
  68. package/dist/src/flow/flow.models.js.map +1 -0
  69. package/dist/src/flow/retry-policy.d.ts +9 -0
  70. package/dist/src/flow/retry-policy.js +15 -0
  71. package/dist/src/flow/retry-policy.js.map +1 -0
  72. package/dist/src/flow/state-path.d.ts +21 -0
  73. package/dist/src/flow/state-path.js +37 -0
  74. package/dist/src/flow/state-path.js.map +1 -0
  75. package/dist/src/flow/step-input-builder.d.ts +15 -0
  76. package/dist/src/flow/step-input-builder.js +63 -0
  77. package/dist/src/flow/step-input-builder.js.map +1 -0
  78. package/dist/src/flows/scrape.flow.d.ts +6 -0
  79. package/dist/src/flows/scrape.flow.js +31 -0
  80. package/dist/src/flows/scrape.flow.js.map +1 -0
  81. package/dist/src/index.d.ts +57 -0
  82. package/dist/src/index.js +50 -0
  83. package/dist/src/index.js.map +1 -0
  84. package/dist/src/logging/logger.d.ts +17 -0
  85. package/dist/src/logging/logger.js +55 -0
  86. package/dist/src/logging/logger.js.map +1 -0
  87. package/dist/src/logging/logger.models.d.ts +25 -0
  88. package/dist/src/logging/logger.models.js +7 -0
  89. package/dist/src/logging/logger.models.js.map +1 -0
  90. package/dist/src/mail/mail.models.d.ts +15 -0
  91. package/dist/src/mail/mail.models.js +2 -0
  92. package/dist/src/mail/mail.models.js.map +1 -0
  93. package/dist/src/mail/smtp-mailer.d.ts +13 -0
  94. package/dist/src/mail/smtp-mailer.js +70 -0
  95. package/dist/src/mail/smtp-mailer.js.map +1 -0
  96. package/dist/src/mail/unconfigured-mailer.d.ts +6 -0
  97. package/dist/src/mail/unconfigured-mailer.js +20 -0
  98. package/dist/src/mail/unconfigured-mailer.js.map +1 -0
  99. package/dist/src/models.d.ts +11 -0
  100. package/dist/src/models.js +10 -0
  101. package/dist/src/models.js.map +1 -0
  102. package/dist/src/monitor/monitor-api.d.ts +18 -0
  103. package/dist/src/monitor/monitor-api.js +62 -0
  104. package/dist/src/monitor/monitor-api.js.map +1 -0
  105. package/dist/src/monitor/monitor-server.d.ts +33 -0
  106. package/dist/src/monitor/monitor-server.js +142 -0
  107. package/dist/src/monitor/monitor-server.js.map +1 -0
  108. package/dist/src/monitor/monitor.models.d.ts +27 -0
  109. package/dist/src/monitor/monitor.models.js +2 -0
  110. package/dist/src/monitor/monitor.models.js.map +1 -0
  111. package/dist/src/queue/pg-boss-flow-queue.d.ts +22 -0
  112. package/dist/src/queue/pg-boss-flow-queue.js +102 -0
  113. package/dist/src/queue/pg-boss-flow-queue.js.map +1 -0
  114. package/dist/src/queue/queue.models.d.ts +38 -0
  115. package/dist/src/queue/queue.models.js +2 -0
  116. package/dist/src/queue/queue.models.js.map +1 -0
  117. package/dist/src/reporting/lighty-error-reporter.d.ts +24 -0
  118. package/dist/src/reporting/lighty-error-reporter.js +96 -0
  119. package/dist/src/reporting/lighty-error-reporter.js.map +1 -0
  120. package/dist/src/reporting/log-error-reporter.d.ts +12 -0
  121. package/dist/src/reporting/log-error-reporter.js +35 -0
  122. package/dist/src/reporting/log-error-reporter.js.map +1 -0
  123. package/dist/src/reporting/reporting.models.d.ts +27 -0
  124. package/dist/src/reporting/reporting.models.js +2 -0
  125. package/dist/src/reporting/reporting.models.js.map +1 -0
  126. package/dist/src/scrapme.d.ts +61 -0
  127. package/dist/src/scrapme.js +133 -0
  128. package/dist/src/scrapme.js.map +1 -0
  129. package/dist/src/scrapme.models.d.ts +18 -0
  130. package/dist/src/scrapme.models.js +2 -0
  131. package/dist/src/scrapme.models.js.map +1 -0
  132. package/dist/src/storage/couch-connection.d.ts +19 -0
  133. package/dist/src/storage/couch-connection.js +67 -0
  134. package/dist/src/storage/couch-connection.js.map +1 -0
  135. package/dist/src/storage/couch-database.d.ts +44 -0
  136. package/dist/src/storage/couch-database.js +118 -0
  137. package/dist/src/storage/couch-database.js.map +1 -0
  138. package/dist/src/storage/couch-document-store.d.ts +14 -0
  139. package/dist/src/storage/couch-document-store.js +36 -0
  140. package/dist/src/storage/couch-document-store.js.map +1 -0
  141. package/dist/src/storage/couch-error.d.ts +12 -0
  142. package/dist/src/storage/couch-error.js +19 -0
  143. package/dist/src/storage/couch-error.js.map +1 -0
  144. package/dist/src/storage/couch-flow-run-store.d.ts +24 -0
  145. package/dist/src/storage/couch-flow-run-store.js +66 -0
  146. package/dist/src/storage/couch-flow-run-store.js.map +1 -0
  147. package/dist/src/storage/couch-run-store.d.ts +15 -0
  148. package/dist/src/storage/couch-run-store.js +52 -0
  149. package/dist/src/storage/couch-run-store.js.map +1 -0
  150. package/dist/src/storage/run-doc-ids.d.ts +9 -0
  151. package/dist/src/storage/run-doc-ids.js +15 -0
  152. package/dist/src/storage/run-doc-ids.js.map +1 -0
  153. package/dist/src/storage/storage.models.d.ts +95 -0
  154. package/dist/src/storage/storage.models.js +5 -0
  155. package/dist/src/storage/storage.models.js.map +1 -0
  156. package/dist/src/storage/unconfigured-document-store.d.ts +8 -0
  157. package/dist/src/storage/unconfigured-document-store.js +26 -0
  158. package/dist/src/storage/unconfigured-document-store.js.map +1 -0
  159. package/dist/src/util/preview.d.ts +10 -0
  160. package/dist/src/util/preview.js +44 -0
  161. package/dist/src/util/preview.js.map +1 -0
  162. package/dist/src/util/text.d.ts +8 -0
  163. package/dist/src/util/text.js +16 -0
  164. package/dist/src/util/text.js.map +1 -0
  165. package/dist/src/util/timing.d.ts +10 -0
  166. package/dist/src/util/timing.js +59 -0
  167. package/dist/src/util/timing.js.map +1 -0
  168. package/dist/src/util/url.d.ts +5 -0
  169. package/dist/src/util/url.js +11 -0
  170. package/dist/src/util/url.js.map +1 -0
  171. package/package.json +75 -4
@@ -0,0 +1,141 @@
1
+ import { z } from 'zod';
2
+ import { FlowDefinitionError } from './flow-errors.js';
3
+ import { VALUE_TYPE, } from './flow.models.js';
4
+ import { StatePath } from './state-path.js';
5
+ /**
6
+ * Checks a flow config (a `*.flow.ts` file) and turns it into a FlowDefinition, or throws FlowDefinitionError saying
7
+ * where (`flow "hello": steps[0] "printed".input["text"]: …`). Shape with zod; then what zod can't see: unique keys
8
+ * (a step's key defaults to its command's name), `from` paths that start with a param or an EARLIER step, commands
9
+ * that have an `execute` method. Field functions are plain JS and are not looked into.
10
+ */
11
+ export class FlowValidator {
12
+ /**
13
+ * Step keys and param names become state properties and the first part of paths, so they must be plain
14
+ * identifiers. A regex is the clearest way to say that (it's a rule about characters, not about structure).
15
+ */
16
+ static IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_-]*$/;
17
+ static IDENTIFIER_RULE = 'letters, digits, _ and - only, not starting with a digit or -';
18
+ static identifier = z.string().regex(FlowValidator.IDENTIFIER, FlowValidator.IDENTIFIER_RULE);
19
+ static int = (min) => z.number().int().min(min);
20
+ static field = z.union([
21
+ z.strictObject({ from: z.string(), optional: z.boolean().optional() }),
22
+ z.strictObject({ value: z.unknown() }),
23
+ z.custom((v) => typeof v === 'function', 'a field is { from }, { value } or a function (state) => value'),
24
+ ]);
25
+ static schema = z.strictObject({
26
+ name: FlowValidator.identifier,
27
+ description: z.string().optional(),
28
+ tags: z.record(z.string(), z.string()).optional(),
29
+ retries: FlowValidator.int(0).optional(),
30
+ retryDelaySeconds: FlowValidator.int(0).optional(),
31
+ retryBackoff: z.boolean().optional(),
32
+ timeoutSeconds: FlowValidator.int(1).optional(),
33
+ params: z
34
+ .record(FlowValidator.identifier, z.strictObject({
35
+ required: z.boolean().optional(),
36
+ type: z.enum(VALUE_TYPE).optional(),
37
+ default: z.unknown().optional(),
38
+ description: z.string().optional(),
39
+ }))
40
+ .optional(),
41
+ steps: z
42
+ .array(z.strictObject({
43
+ key: FlowValidator.identifier.optional(),
44
+ command: z.custom((v) => FlowValidator.isCommand(v), 'must be a command class (with execute()) or a command object'),
45
+ inputFrom: z.string().optional(),
46
+ input: z.record(z.string(), FlowValidator.field).optional(),
47
+ description: z.string().optional(),
48
+ retries: FlowValidator.int(0).optional(),
49
+ timeoutMs: FlowValidator.int(1).optional(),
50
+ retryDelayMs: FlowValidator.int(0).optional(),
51
+ maxRetryDelayMs: FlowValidator.int(0).optional(),
52
+ }))
53
+ .min(1, 'a flow needs at least one step'),
54
+ });
55
+ validate(config) {
56
+ const label = typeof config?.name === 'string' && config.name ? config.name : 'flow';
57
+ const parsed = FlowValidator.schema.safeParse(config);
58
+ if (!parsed.success)
59
+ throw new FlowDefinitionError(z.prettifyError(parsed.error).split('\n').join(' '), label);
60
+ const params = Object.entries(config.params ?? {}).map(([name, p]) => this.param(label, name, p));
61
+ const known = new Set(params.map((p) => p.name));
62
+ const steps = config.steps.map((step, index) => {
63
+ const definition = this.step(label, index, step, known);
64
+ known.add(definition.key);
65
+ return definition;
66
+ });
67
+ const queue = Object.fromEntries(Object.entries({ retries: config.retries, retryDelaySeconds: config.retryDelaySeconds, retryBackoff: config.retryBackoff, timeoutSeconds: config.timeoutSeconds }).filter(([, v]) => v !== undefined));
68
+ return {
69
+ name: config.name,
70
+ queue,
71
+ tags: { ...config.tags },
72
+ params,
73
+ steps,
74
+ ...(config.description ? { description: config.description } : {}),
75
+ };
76
+ }
77
+ /** A class whose prototype has execute(), or an object with execute(). */
78
+ static isCommand(value) {
79
+ if (typeof value === 'function')
80
+ return typeof value.prototype?.execute === 'function';
81
+ return value !== null && typeof value === 'object' && typeof value.execute === 'function';
82
+ }
83
+ /** Class name, the name of a FunctionCommand, or the object's class name. */
84
+ static commandName(command) {
85
+ if (typeof command === 'function')
86
+ return command.name || 'AnonymousCommand';
87
+ const named = command.name;
88
+ return typeof named === 'string' && named ? named : (command.constructor?.name ?? 'Command');
89
+ }
90
+ param(label, name, p) {
91
+ const required = p.required ?? false;
92
+ if (required && p.default !== undefined)
93
+ throw new FlowDefinitionError(`params.${name}: required and a default don't go together`, label);
94
+ return { name, required, type: p.type ?? VALUE_TYPE.STRING, ...(p.default !== undefined ? { default: structuredClone(p.default) } : {}) };
95
+ }
96
+ step(label, index, step, known) {
97
+ const { key: givenKey, command, inputFrom, input, description, ...options } = step;
98
+ const commandName = FlowValidator.commandName(command);
99
+ const key = givenKey ?? commandName;
100
+ const where = `steps[${index}] "${key}"`;
101
+ if (!FlowValidator.IDENTIFIER.test(key)) {
102
+ throw new FlowDefinitionError(`${where}: the command's name can't be a key (${FlowValidator.IDENTIFIER_RULE}) - give the step a key`, label);
103
+ }
104
+ if (known.has(key)) {
105
+ throw new FlowDefinitionError(`${where}: the key is already used by a param or an earlier step${givenKey ? '' : ' (the same command twice? give the steps their own keys)'}`, label);
106
+ }
107
+ if (inputFrom !== undefined)
108
+ this.reference(label, `${where}.inputFrom`, inputFrom, known);
109
+ const fields = Object.entries(input ?? {}).map(([name, field]) => this.field(label, `${where}.input["${name}"]`, name, field, known));
110
+ const spec = inputFrom === undefined && input === undefined ? undefined : { ...(inputFrom !== undefined ? { from: inputFrom } : {}), fields };
111
+ return {
112
+ key,
113
+ command: command,
114
+ commandName,
115
+ options: Object.fromEntries(Object.entries(options).filter(([, v]) => v !== undefined)),
116
+ ...(spec ? { input: spec } : {}),
117
+ ...(description ? { description } : {}),
118
+ };
119
+ }
120
+ field(label, where, name, field, known) {
121
+ if (!StatePath.split(name))
122
+ throw new FlowDefinitionError(`${where}: the name must be a name or a dotted path (a.b)`, label);
123
+ if (typeof field === 'function')
124
+ return { name, compute: field };
125
+ if ('from' in field) {
126
+ this.reference(label, where, field.from, known);
127
+ return { name, from: field.from, optional: field.optional ?? false };
128
+ }
129
+ return { name, value: structuredClone(field.value) };
130
+ }
131
+ /** A path must start with a param or an earlier step's key. */
132
+ reference(label, where, path, known) {
133
+ const segments = StatePath.split(path);
134
+ if (!segments)
135
+ throw new FlowDefinitionError(`${where}: "${path}" is not a valid path (names joined with ".")`, label);
136
+ if (!known.has(segments[0])) {
137
+ throw new FlowDefinitionError(`${where}: "${path}" - "${segments[0]}" is neither a param nor an earlier step (known: ${[...known].join(', ') || 'nothing'})`, label);
138
+ }
139
+ }
140
+ }
141
+ //# sourceMappingURL=flow-validator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"flow-validator.js","sourceRoot":"","sources":["../../../src/flow/flow-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EACL,UAAU,GAYX,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IACxB;;;OAGG;IACK,MAAM,CAAU,UAAU,GAAG,2BAA2B,CAAC;IACzD,MAAM,CAAU,eAAe,GAAG,+DAA+D,CAAC;IAClG,MAAM,CAAU,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,UAAU,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IACvG,MAAM,CAAU,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjE,MAAM,CAAU,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QACtC,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;QACtE,CAAC,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACtC,CAAC,CAAC,MAAM,CAAgC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,UAAU,EAAE,+DAA+D,CAAC;KACzI,CAAC,CAAC;IAEK,MAAM,CAAU,MAAM,GAAG,CAAC,CAAC,YAAY,CAAC;QAC9C,IAAI,EAAE,aAAa,CAAC,UAAU;QAC9B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAClC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;QACjD,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QACxC,iBAAiB,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QACpC,cAAc,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC/C,MAAM,EAAE,CAAC;aACN,MAAM,CACL,aAAa,CAAC,UAAU,EACxB,CAAC,CAAC,YAAY,CAAC;YACb,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAChC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAE;YACnC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SACnC,CAAC,CACH;aACA,QAAQ,EAAE;QACb,KAAK,EAAE,CAAC;aACL,KAAK,CACJ,CAAC,CAAC,YAAY,CAAC;YACb,GAAG,EAAE,aAAa,CAAC,UAAU,CAAC,QAAQ,EAAE;YACxC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAa,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,8DAA8D,CAAC;YAChI,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAChC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;YAC3D,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAClC,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;YACxC,SAAS,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;YAC1C,YAAY,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;YAC7C,eAAe,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;SACjD,CAAC,CACH;aACA,GAAG,CAAC,CAAC,EAAE,gCAAgC,CAAC;KAC5C,CAAC,CAAC;IAEI,QAAQ,CAAC,MAAkB;QAChC,MAAM,KAAK,GAAG,OAAO,MAAM,EAAE,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QACrF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,mBAAmB,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QAE/G,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAClG,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACxD,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAC1B,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAC9B,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CACtM,CAAC;QACF,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,KAAK;YACL,IAAI,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE;YACxB,MAAM;YACN,KAAK;YACL,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnE,CAAC;IACJ,CAAC;IAED,0EAA0E;IACnE,MAAM,CAAC,SAAS,CAAC,KAAc;QACpC,IAAI,OAAO,KAAK,KAAK,UAAU;YAAE,OAAO,OAAQ,KAA+C,CAAC,SAAS,EAAE,OAAO,KAAK,UAAU,CAAC;QAClI,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAQ,KAA+B,CAAC,OAAO,KAAK,UAAU,CAAC;IACvH,CAAC;IAED,6EAA6E;IACtE,MAAM,CAAC,WAAW,CAAC,OAAmB;QAC3C,IAAI,OAAO,OAAO,KAAK,UAAU;YAAE,OAAO,OAAO,CAAC,IAAI,IAAI,kBAAkB,CAAC;QAC7E,MAAM,KAAK,GAAI,OAA8B,CAAC,IAAI,CAAC;QACnD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,IAAI,SAAS,CAAC,CAAC;IAC/F,CAAC;IAEO,KAAK,CAAC,KAAa,EAAE,IAAY,EAAE,CAAc;QACvD,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC;QACrC,IAAI,QAAQ,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;YAAE,MAAM,IAAI,mBAAmB,CAAC,UAAU,IAAI,4CAA4C,EAAE,KAAK,CAAC,CAAC;QAC1I,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAC5I,CAAC;IAEO,IAAI,CAAC,KAAa,EAAE,KAAa,EAAE,IAAgB,EAAE,KAA0B;QACrF,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC;QACnF,MAAM,WAAW,GAAG,aAAa,CAAC,WAAW,CAAC,OAAqB,CAAC,CAAC;QACrE,MAAM,GAAG,GAAG,QAAQ,IAAI,WAAW,CAAC;QACpC,MAAM,KAAK,GAAG,SAAS,KAAK,MAAM,GAAG,GAAG,CAAC;QACzC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,mBAAmB,CAAC,GAAG,KAAK,wCAAwC,aAAa,CAAC,eAAe,yBAAyB,EAAE,KAAK,CAAC,CAAC;QAC/I,CAAC;QACD,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,mBAAmB,CAAC,GAAG,KAAK,0DAA0D,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,0DAA0D,EAAE,EAAE,KAAK,CAAC,CAAC;QACvL,CAAC;QACD,IAAI,SAAS,KAAK,SAAS;YAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,KAAK,YAAY,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAC3F,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,KAAK,WAAW,IAAI,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QACtI,MAAM,IAAI,GAA0B,SAAS,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC;QACrK,OAAO;YACL,GAAG;YACH,OAAO,EAAE,OAAqB;YAC9B,WAAW;YACX,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;YACvF,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,KAAa,EAAE,KAAa,EAAE,IAAY,EAAE,KAAkB,EAAE,KAA0B;QACtG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,mBAAmB,CAAC,GAAG,KAAK,kDAAkD,EAAE,KAAK,CAAC,CAAC;QAC7H,IAAI,OAAO,KAAK,KAAK,UAAU;YAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACjE,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAChD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK,EAAE,CAAC;QACvE,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;IACvD,CAAC;IAED,+DAA+D;IACvD,SAAS,CAAC,KAAa,EAAE,KAAa,EAAE,IAAY,EAAE,KAA0B;QACtF,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,mBAAmB,CAAC,GAAG,KAAK,MAAM,IAAI,+CAA+C,EAAE,KAAK,CAAC,CAAC;QACvH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,mBAAmB,CAAC,GAAG,KAAK,MAAM,IAAI,QAAQ,QAAQ,CAAC,CAAC,CAAC,oDAAoD,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,CAAC;QACvK,CAAC;IACH,CAAC"}
@@ -0,0 +1,32 @@
1
+ import type { Command, CommandContext, CommandOptions, FlowConfig, FlowDefinition, FlowParam, FlowQueueOptions, StepDefinition } from './flow.models.js';
2
+ /**
3
+ * A flow: an ordered list of steps, each running one command. Write it in a `*.flow.ts` file and export it; the
4
+ * config is checked here (FlowValidator), so a broken flow throws FlowDefinitionError when the file is imported.
5
+ *
6
+ * export const helloFlow = new Flow({
7
+ * name: 'hello',
8
+ * params: { name: { default: 'World' } },
9
+ * steps: [{ command: PrintMessageCommand, input: { text: (s) => 'Hello, ' + s.name + '!' } }],
10
+ * });
11
+ *
12
+ * Queue a run of it with `scrapme.flow(helloFlow, { name: 'Ada' })`.
13
+ */
14
+ export declare class Flow implements FlowDefinition {
15
+ readonly name: string;
16
+ readonly queue: FlowQueueOptions;
17
+ readonly tags: Readonly<Record<string, string>>;
18
+ readonly params: readonly FlowParam[];
19
+ readonly steps: readonly StepDefinition[];
20
+ readonly description?: string;
21
+ constructor(config: FlowConfig);
22
+ /** "page (ScrapePageCommand) → report (ParseIncomeStatementCommand) → …" — for logs and CLIs. */
23
+ describe(): string;
24
+ }
25
+ /** A command made from a function — for small glue steps that need no dependencies. */
26
+ export declare class FunctionCommand<I, O> implements Command<I, O> {
27
+ readonly name: string;
28
+ private readonly fn;
29
+ readonly options: CommandOptions;
30
+ constructor(name: string, fn: (input: I, ctx: CommandContext) => Promise<O> | O, options?: CommandOptions);
31
+ execute(input: I, ctx: CommandContext): Promise<O>;
32
+ }
@@ -0,0 +1,50 @@
1
+ import { FlowValidator } from './flow-validator.js';
2
+ /**
3
+ * A flow: an ordered list of steps, each running one command. Write it in a `*.flow.ts` file and export it; the
4
+ * config is checked here (FlowValidator), so a broken flow throws FlowDefinitionError when the file is imported.
5
+ *
6
+ * export const helloFlow = new Flow({
7
+ * name: 'hello',
8
+ * params: { name: { default: 'World' } },
9
+ * steps: [{ command: PrintMessageCommand, input: { text: (s) => 'Hello, ' + s.name + '!' } }],
10
+ * });
11
+ *
12
+ * Queue a run of it with `scrapme.flow(helloFlow, { name: 'Ada' })`.
13
+ */
14
+ export class Flow {
15
+ name;
16
+ queue;
17
+ tags;
18
+ params;
19
+ steps;
20
+ description;
21
+ constructor(config) {
22
+ const definition = new FlowValidator().validate(config);
23
+ this.name = definition.name;
24
+ this.queue = Object.freeze({ ...definition.queue });
25
+ this.tags = Object.freeze({ ...definition.tags });
26
+ this.params = Object.freeze([...definition.params]);
27
+ this.steps = Object.freeze([...definition.steps]);
28
+ if (definition.description)
29
+ this.description = definition.description;
30
+ }
31
+ /** "page (ScrapePageCommand) → report (ParseIncomeStatementCommand) → …" — for logs and CLIs. */
32
+ describe() {
33
+ return this.steps.map((s) => `${s.key} (${s.commandName})`).join(' → ');
34
+ }
35
+ }
36
+ /** A command made from a function — for small glue steps that need no dependencies. */
37
+ export class FunctionCommand {
38
+ name;
39
+ fn;
40
+ options;
41
+ constructor(name, fn, options = {}) {
42
+ this.name = name;
43
+ this.fn = fn;
44
+ this.options = options;
45
+ }
46
+ async execute(input, ctx) {
47
+ return this.fn(input, ctx);
48
+ }
49
+ }
50
+ //# sourceMappingURL=flow.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"flow.js","sourceRoot":"","sources":["../../../src/flow/flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,IAAI;IACC,IAAI,CAAS;IACb,KAAK,CAAmB;IACxB,IAAI,CAAmC;IACvC,MAAM,CAAuB;IAC7B,KAAK,CAA4B;IACjC,WAAW,CAAU;IAErC,YAAmB,MAAkB;QACnC,MAAM,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACpD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,WAAW;YAAE,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;IACxE,CAAC;IAED,iGAAiG;IAC1F,QAAQ;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1E,CAAC;CACF;AAED,uFAAuF;AACvF,MAAM,OAAO,eAAe;IAER;IACC;IACD;IAHlB,YACkB,IAAY,EACX,EAAqD,EACtD,UAA0B,EAAE;QAF5B,SAAI,GAAJ,IAAI,CAAQ;QACX,OAAE,GAAF,EAAE,CAAmD;QACtD,YAAO,GAAP,OAAO,CAAqB;IAC3C,CAAC;IAEG,KAAK,CAAC,OAAO,CAAC,KAAQ,EAAE,GAAmB;QAChD,OAAO,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF"}
@@ -0,0 +1,288 @@
1
+ import type { Logger } from '../logging/logger.js';
2
+ /**
3
+ * Status of a run (stored in its run document). queued → running → succeeded | retrying (pg-boss runs the whole
4
+ * flow again later) | failed (no attempts left, or a permanent error).
5
+ */
6
+ export declare const RUN_STATUS: {
7
+ readonly QUEUED: "queued";
8
+ readonly RUNNING: "running";
9
+ readonly RETRYING: "retrying";
10
+ readonly SUCCEEDED: "succeeded";
11
+ readonly FAILED: "failed";
12
+ };
13
+ export type RunStatus = (typeof RUN_STATUS)[keyof typeof RUN_STATUS];
14
+ /** A run in one of these never changes again. */
15
+ export declare const FINAL_RUN_STATUSES: readonly RunStatus[];
16
+ /** Status of one step of a run. A failed step becomes running again when the flow is retried. */
17
+ export declare const STEP_STATUS: {
18
+ readonly PENDING: "pending";
19
+ readonly RUNNING: "running";
20
+ readonly SUCCEEDED: "succeeded";
21
+ readonly FAILED: "failed";
22
+ };
23
+ export type StepStatus = (typeof STEP_STATUS)[keyof typeof STEP_STATUS];
24
+ /** How one attempt of a step ended. */
25
+ export declare const ATTEMPT_OUTCOME: {
26
+ readonly SUCCEEDED: "succeeded";
27
+ readonly FAILED: "failed";
28
+ readonly TIMEOUT: "timeout";
29
+ };
30
+ export type AttemptOutcome = (typeof ATTEMPT_OUTCOME)[keyof typeof ATTEMPT_OUTCOME];
31
+ /** `code` of the errors scrapme itself throws (your commands may use their own codes too). */
32
+ export declare const ERROR_CODE: {
33
+ /** PermanentError without a more specific code. */
34
+ readonly PERMANENT: "PERMANENT";
35
+ /** A command got an input it can't work with. */
36
+ readonly BAD_INPUT: "BAD_INPUT";
37
+ /** A step input refers to a value the state doesn't have (e.g. a field missing in a previous output). */
38
+ readonly MISSING_VALUE: "MISSING_VALUE";
39
+ readonly COMMAND_TIMEOUT: "COMMAND_TIMEOUT";
40
+ readonly FLOW_CANCELLED: "FLOW_CANCELLED";
41
+ readonly FLOW_FAILED: "FLOW_FAILED";
42
+ /** The flow definition is invalid (shape, bad reference, duplicate key, not a command). */
43
+ readonly FLOW_DEFINITION: "FLOW_DEFINITION";
44
+ /** The run input doesn't match the flow's `<param>`s. */
45
+ readonly FLOW_INPUT: "FLOW_INPUT";
46
+ /** ScrapePageCommand: the URL is a file (PDF…), not an HTML page. */
47
+ readonly NOT_HTML: "NOT_HTML";
48
+ /** ScrapePageCommand: the target answered 4xx (allowHttpErrors not set). */
49
+ readonly HTTP_STATUS: "HTTP_STATUS";
50
+ /** SendMailCommand: the SMTP server refused the message (5xx). */
51
+ readonly MAIL_REJECTED: "MAIL_REJECTED";
52
+ /** A command needs a service that wasn't given to Scrapme.create (scrapAnything, couch, mail). */
53
+ readonly NOT_CONFIGURED: "NOT_CONFIGURED";
54
+ };
55
+ export type ErrorCode = (typeof ERROR_CODE)[keyof typeof ERROR_CODE];
56
+ /** Retry/timeout settings. Precedence: step > command.options > FLOW_* config defaults. */
57
+ export interface CommandOptions {
58
+ /** Retries after the first attempt (attempts = retries + 1). */
59
+ retries?: number;
60
+ /** Limit of one attempt in ms; on expiry ctx.signal aborts and the attempt fails with CommandTimeoutError. */
61
+ timeoutMs?: number;
62
+ /** First retry delay (doubled per retry, with jitter). */
63
+ retryDelayMs?: number;
64
+ maxRetryDelayMs?: number;
65
+ }
66
+ /** What a command gets besides its input. */
67
+ export interface CommandContext {
68
+ readonly runId: string;
69
+ readonly flow: string;
70
+ /** Key of the step in the flow (the name its output is stored under). */
71
+ readonly step: string;
72
+ /** Attempt of this step within the current flow attempt, 1-based. */
73
+ readonly attempt: number;
74
+ readonly maxAttempts: number;
75
+ /** Attempt of the whole flow (pg-boss job), 1-based: > 1 means pg-boss runs the flow again. */
76
+ readonly flowAttempt: number;
77
+ /** Aborted on timeout or when the run is cancelled — pass it on and stop early. */
78
+ readonly signal: AbortSignal;
79
+ /** Logger with run/flow/step/attempt fields already set. */
80
+ readonly logger: Logger;
81
+ /** Everything produced so far: the flow input plus the outputs of all previous steps, by step key. */
82
+ readonly state: Readonly<Record<string, unknown>>;
83
+ }
84
+ /**
85
+ * One unit of work. Implement it as an `@injectable()` class: dependencies come through the constructor
86
+ * (`@inject(SomeService)` — resolved from your app's container), the input is what the step's `input` builds
87
+ * (default: the whole state),
88
+ * the returned value is stored in the state under the step key.
89
+ */
90
+ export interface Command<I = unknown, O = unknown> {
91
+ /** Defaults for this command; the step can override them. */
92
+ readonly options?: CommandOptions;
93
+ execute(input: I, ctx: CommandContext): Promise<O>;
94
+ /** Override the default retry decision (see isRetryable in flow-errors). */
95
+ isRetryable?(error: unknown): boolean;
96
+ }
97
+ /** A command class (resolved from the DI container, so its constructor injections work). */
98
+ export type CommandClass<I = unknown, O = unknown> = abstract new (...args: never[]) => Command<I, O>;
99
+ /** What a step runs: a command class (resolved from the DI container) or a ready instance (e.g. new FunctionCommand(...)). */
100
+ export type CommandRef<I = unknown, O = unknown> = CommandClass<I, O> | Command<I, O>;
101
+ /** Kind of a param value (checked when a run starts). */
102
+ export declare const VALUE_TYPE: {
103
+ readonly STRING: "string";
104
+ readonly NUMBER: "number";
105
+ readonly BOOLEAN: "boolean";
106
+ readonly JSON: "json";
107
+ };
108
+ export type ValueType = (typeof VALUE_TYPE)[keyof typeof VALUE_TYPE];
109
+ /** The state as a field function sees it: the run input plus the outputs of the steps so far, by key (shape up to the flow). */
110
+ export type FlowState = Readonly<Record<string, any>>;
111
+ /** One property of a step input, as written in a flow file. */
112
+ export type FieldConfig =
113
+ /** A value from the state (dotted path: "page.finalUrl"); missing → MISSING_VALUE, unless optional. */
114
+ {
115
+ readonly from: string;
116
+ readonly optional?: boolean;
117
+ }
118
+ /** A literal. */
119
+ | {
120
+ readonly value: unknown;
121
+ }
122
+ /** Computed from the state in plain JS: `(s) => 'doc:' + s.report.symbol`. */
123
+ | ((state: FlowState) => unknown);
124
+ /** A value the caller passes into the run, as written in a flow file. */
125
+ export interface ParamConfig {
126
+ readonly required?: boolean;
127
+ /** Default `string`. */
128
+ readonly type?: ValueType;
129
+ readonly default?: unknown;
130
+ readonly description?: string;
131
+ }
132
+ /** One step, as written in a flow file. */
133
+ export interface StepConfig extends CommandOptions {
134
+ /** Name the output is stored under in the state. Default: the command's name (class name). */
135
+ readonly key?: string;
136
+ readonly command: CommandRef<never, unknown>;
137
+ /** The step input is (a copy of) this value of the state. Without it and without `input`: the whole state. */
138
+ readonly inputFrom?: string;
139
+ /** The step input is an object with these properties (names may be dotted paths: "headers.Accept-Language"). */
140
+ readonly input?: Readonly<Record<string, FieldConfig>>;
141
+ readonly description?: string;
142
+ }
143
+ /** How the whole flow runs on its pg-boss queue (each run = one job; a failed job runs the flow again from its first step). */
144
+ export interface FlowQueueOptions {
145
+ /** Flow attempts after the first one (pg-boss retryLimit). Default: the `queue.retries` option (3). */
146
+ readonly retries?: number;
147
+ /** Delay before a flow attempt is retried (pg-boss retryDelay). Default: the `queue.retryDelaySeconds` option. */
148
+ readonly retryDelaySeconds?: number;
149
+ /** Double the delay on every retry (pg-boss retryBackoff). Default: the `queue.retryBackoff` option. */
150
+ readonly retryBackoff?: boolean;
151
+ /** One flow attempt may run this long, then pg-boss fails (and retries) it (expireInSeconds). */
152
+ readonly timeoutSeconds?: number;
153
+ }
154
+ /**
155
+ * A flow as written in a `*.flow.ts` file: `export const myFlow = new Flow({ name, params, steps })`.
156
+ * JSON-like data plus the command classes (and plain functions where a value is computed).
157
+ */
158
+ export interface FlowConfig extends FlowQueueOptions {
159
+ /** Letters, digits, _ and -; the flow's pg-boss queue is named after it (queue prefix + name). */
160
+ readonly name: string;
161
+ readonly description?: string;
162
+ /** Stored with every run, sent as tags with error reports. */
163
+ readonly tags?: Readonly<Record<string, string>>;
164
+ readonly params?: Readonly<Record<string, ParamConfig>>;
165
+ readonly steps: readonly StepConfig[];
166
+ }
167
+ /** A param after validation. */
168
+ export interface FlowParam {
169
+ readonly name: string;
170
+ readonly required: boolean;
171
+ readonly type: ValueType;
172
+ readonly default?: unknown;
173
+ }
174
+ /** A field after validation (`name` may be a dotted path). */
175
+ export type InputField = {
176
+ readonly name: string;
177
+ readonly from: string;
178
+ readonly optional: boolean;
179
+ } | {
180
+ readonly name: string;
181
+ readonly value: unknown;
182
+ } | {
183
+ readonly name: string;
184
+ readonly compute: (state: FlowState) => unknown;
185
+ };
186
+ /** How a step builds its input; undefined on the step = the whole state. */
187
+ export interface InputSpec {
188
+ readonly from?: string;
189
+ readonly fields: readonly InputField[];
190
+ }
191
+ /** A step after validation. */
192
+ export interface StepDefinition {
193
+ readonly key: string;
194
+ readonly command: CommandRef;
195
+ /** Class name or FunctionCommand name — for logs, stored runs and reports. */
196
+ readonly commandName: string;
197
+ readonly options: CommandOptions;
198
+ readonly input?: InputSpec;
199
+ readonly description?: string;
200
+ }
201
+ /** A flow after validation (Flow). */
202
+ export interface FlowDefinition {
203
+ readonly name: string;
204
+ /** Only what the flow sets; the rest comes from Scrapme.create({ queue }). */
205
+ readonly queue: FlowQueueOptions;
206
+ readonly tags: Readonly<Record<string, string>>;
207
+ readonly params: readonly FlowParam[];
208
+ readonly steps: readonly StepDefinition[];
209
+ readonly description?: string;
210
+ }
211
+ export interface SerializedError {
212
+ name: string;
213
+ message: string;
214
+ code?: string;
215
+ retryable: boolean;
216
+ stack?: string;
217
+ cause?: SerializedError;
218
+ }
219
+ export interface AttemptReport {
220
+ /** Attempt of the step within its flow attempt. */
221
+ attempt: number;
222
+ flowAttempt: number;
223
+ startedAt: string;
224
+ durationMs: number;
225
+ outcome: AttemptOutcome;
226
+ error?: SerializedError;
227
+ /** Delay before the next attempt, when there is one. */
228
+ retryInMs?: number;
229
+ }
230
+ /** One step of a run, as stored in the run document and shown by the monitor. */
231
+ export interface StepReport {
232
+ key: string;
233
+ command: string;
234
+ status: StepStatus;
235
+ /** Every attempt, over all flow attempts (each says which flow attempt it belonged to). */
236
+ attempts: AttemptReport[];
237
+ startedAt?: string;
238
+ finishedAt?: string;
239
+ durationMs?: number;
240
+ /** The output, shortened (long strings cut at defaults.maxStoredChars). */
241
+ output?: unknown;
242
+ error?: SerializedError;
243
+ }
244
+ /** One attempt of the whole flow (one execution of its pg-boss job). */
245
+ export interface FlowAttemptReport {
246
+ attempt: number;
247
+ jobId: string;
248
+ startedAt: string;
249
+ finishedAt?: string;
250
+ outcome?: typeof ATTEMPT_OUTCOME.SUCCEEDED | typeof ATTEMPT_OUTCOME.FAILED;
251
+ error?: SerializedError;
252
+ }
253
+ export interface ScheduleOptions {
254
+ /** Your own id (e.g. an idempotency key); default random UUID. Scheduling the same id twice is refused. */
255
+ runId?: string;
256
+ /** Run later. */
257
+ startAfter?: Date;
258
+ /** Higher runs first. */
259
+ priority?: number;
260
+ }
261
+ export interface WaitOptions {
262
+ /** Give up after this many ms (the run goes on). Default 10 minutes. */
263
+ timeoutMs?: number;
264
+ /** Default 1 000. */
265
+ pollIntervalMs?: number;
266
+ }
267
+ /** Outcome of executing one step with retries (CommandExecutor). */
268
+ export type StepExecution = {
269
+ ok: true;
270
+ value: unknown;
271
+ attempts: AttemptReport[];
272
+ durationMs: number;
273
+ } | {
274
+ ok: false;
275
+ error: unknown;
276
+ attempts: AttemptReport[];
277
+ durationMs: number;
278
+ };
279
+ /** Called by CommandExecutor after every failed attempt (recorded in the run document). */
280
+ export type AttemptFailedListener = (report: AttemptReport, error: unknown, final: boolean) => Promise<void> | void;
281
+ /** Called by CommandExecutor before every attempt (the monitor shows the step as running). */
282
+ export type AttemptStartedListener = (attempt: number) => Promise<void> | void;
283
+ export interface ResolvedStepPolicy {
284
+ maxAttempts: number;
285
+ timeoutMs: number;
286
+ retryDelayMs: number;
287
+ maxRetryDelayMs: number;
288
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Status of a run (stored in its run document). queued → running → succeeded | retrying (pg-boss runs the whole
3
+ * flow again later) | failed (no attempts left, or a permanent error).
4
+ */
5
+ export const RUN_STATUS = { QUEUED: 'queued', RUNNING: 'running', RETRYING: 'retrying', SUCCEEDED: 'succeeded', FAILED: 'failed' };
6
+ /** A run in one of these never changes again. */
7
+ export const FINAL_RUN_STATUSES = [RUN_STATUS.SUCCEEDED, RUN_STATUS.FAILED];
8
+ /** Status of one step of a run. A failed step becomes running again when the flow is retried. */
9
+ export const STEP_STATUS = { PENDING: 'pending', RUNNING: 'running', SUCCEEDED: 'succeeded', FAILED: 'failed' };
10
+ /** How one attempt of a step ended. */
11
+ export const ATTEMPT_OUTCOME = { SUCCEEDED: 'succeeded', FAILED: 'failed', TIMEOUT: 'timeout' };
12
+ /** `code` of the errors scrapme itself throws (your commands may use their own codes too). */
13
+ export const ERROR_CODE = {
14
+ /** PermanentError without a more specific code. */
15
+ PERMANENT: 'PERMANENT',
16
+ /** A command got an input it can't work with. */
17
+ BAD_INPUT: 'BAD_INPUT',
18
+ /** A step input refers to a value the state doesn't have (e.g. a field missing in a previous output). */
19
+ MISSING_VALUE: 'MISSING_VALUE',
20
+ COMMAND_TIMEOUT: 'COMMAND_TIMEOUT',
21
+ FLOW_CANCELLED: 'FLOW_CANCELLED',
22
+ FLOW_FAILED: 'FLOW_FAILED',
23
+ /** The flow definition is invalid (shape, bad reference, duplicate key, not a command). */
24
+ FLOW_DEFINITION: 'FLOW_DEFINITION',
25
+ /** The run input doesn't match the flow's `<param>`s. */
26
+ FLOW_INPUT: 'FLOW_INPUT',
27
+ /** ScrapePageCommand: the URL is a file (PDF…), not an HTML page. */
28
+ NOT_HTML: 'NOT_HTML',
29
+ /** ScrapePageCommand: the target answered 4xx (allowHttpErrors not set). */
30
+ HTTP_STATUS: 'HTTP_STATUS',
31
+ /** SendMailCommand: the SMTP server refused the message (5xx). */
32
+ MAIL_REJECTED: 'MAIL_REJECTED',
33
+ /** A command needs a service that wasn't given to Scrapme.create (scrapAnything, couch, mail). */
34
+ NOT_CONFIGURED: 'NOT_CONFIGURED',
35
+ };
36
+ /** Kind of a param value (checked when a run starts). */
37
+ export const VALUE_TYPE = { STRING: 'string', NUMBER: 'number', BOOLEAN: 'boolean', JSON: 'json' };
38
+ //# sourceMappingURL=flow.models.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"flow.models.js","sourceRoot":"","sources":["../../../src/flow/flow.models.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAW,CAAC;AAE5I,iDAAiD;AACjD,MAAM,CAAC,MAAM,kBAAkB,GAAyB,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;AAElG,iGAAiG;AACjG,MAAM,CAAC,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAW,CAAC;AAGzH,uCAAuC;AACvC,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAW,CAAC;AAGzG,8FAA8F;AAC9F,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,mDAAmD;IACnD,SAAS,EAAE,WAAW;IACtB,iDAAiD;IACjD,SAAS,EAAE,WAAW;IACtB,yGAAyG;IACzG,aAAa,EAAE,eAAe;IAC9B,eAAe,EAAE,iBAAiB;IAClC,cAAc,EAAE,gBAAgB;IAChC,WAAW,EAAE,aAAa;IAC1B,2FAA2F;IAC3F,eAAe,EAAE,iBAAiB;IAClC,yDAAyD;IACzD,UAAU,EAAE,YAAY;IACxB,qEAAqE;IACrE,QAAQ,EAAE,UAAU;IACpB,4EAA4E;IAC5E,WAAW,EAAE,aAAa;IAC1B,kEAAkE;IAClE,aAAa,EAAE,eAAe;IAC9B,kGAAkG;IAClG,cAAc,EAAE,gBAAgB;CACxB,CAAC;AAqDX,yDAAyD;AACzD,MAAM,CAAC,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAW,CAAC"}
@@ -0,0 +1,9 @@
1
+ import type { ResolvedStepPolicy } from './flow.models.js';
2
+ /** Exponential backoff with "equal jitter": half fixed, half random — retries of many runs do not line up. */
3
+ export declare class RetryPolicy {
4
+ private readonly policy;
5
+ private readonly random;
6
+ constructor(policy: Pick<ResolvedStepPolicy, 'retryDelayMs' | 'maxRetryDelayMs'>, random?: () => number);
7
+ /** Delay before attempt `failedAttempt + 1` (failedAttempt is 1-based). */
8
+ delayAfter(failedAttempt: number): number;
9
+ }
@@ -0,0 +1,15 @@
1
+ /** Exponential backoff with "equal jitter": half fixed, half random — retries of many runs do not line up. */
2
+ export class RetryPolicy {
3
+ policy;
4
+ random;
5
+ constructor(policy, random = Math.random) {
6
+ this.policy = policy;
7
+ this.random = random;
8
+ }
9
+ /** Delay before attempt `failedAttempt + 1` (failedAttempt is 1-based). */
10
+ delayAfter(failedAttempt) {
11
+ const base = Math.min(this.policy.maxRetryDelayMs, this.policy.retryDelayMs * 2 ** (failedAttempt - 1));
12
+ return Math.round(base / 2 + (base / 2) * this.random());
13
+ }
14
+ }
15
+ //# sourceMappingURL=retry-policy.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retry-policy.js","sourceRoot":"","sources":["../../../src/flow/retry-policy.ts"],"names":[],"mappings":"AAEA,8GAA8G;AAC9G,MAAM,OAAO,WAAW;IAEH;IACA;IAFnB,YACmB,MAAoE,EACpE,SAAuB,IAAI,CAAC,MAAM;QADlC,WAAM,GAAN,MAAM,CAA8D;QACpE,WAAM,GAAN,MAAM,CAA4B;IAClD,CAAC;IAEJ,2EAA2E;IACpE,UAAU,CAAC,aAAqB;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC;QACxG,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;CACF"}
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Dotted paths into the flow state: `page.html`, `report.periods`. Only plain property access — no indexes or
3
+ * expressions; anything smarter belongs in a command.
4
+ */
5
+ export declare class StatePath {
6
+ static readonly SEPARATOR = ".";
7
+ /** "a.b.c" → ["a","b","c"]; null when a segment is empty ("a..b", ".a", ""). */
8
+ static split(path: string): string[] | null;
9
+ /** The value at `path`, or `{ found: false }` when some part of it is missing. */
10
+ static get(root: unknown, path: string): {
11
+ found: true;
12
+ value: unknown;
13
+ } | {
14
+ found: false;
15
+ };
16
+ /**
17
+ * A copy of `root` with `value` at `path`; objects on the way are copied (never mutated) or created.
18
+ * A non-object in the way is replaced by an object.
19
+ */
20
+ static set(root: Record<string, unknown>, path: string, value: unknown): Record<string, unknown>;
21
+ }