scrapme-flow 0.0.0-stage → 0.2.0

package/dist/src/config/config-loader.js

@@ -0,0 +1,79 @@
+import { z } from 'zod';
+import { ConfigSchema } from './config.schema.js';
+import { Urls } from '../util/url.js';
+export class ConfigError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ConfigError';
+    }
+}
+/** The only place that reads environment variables. Empty strings count as "not set" (default applies). */
+export class ConfigLoader {
+    static load(env = process.env) {
+        const cleaned = {};
+        for (const key of Object.keys(ConfigSchema.env.shape)) {
+            const value = env[key];
+            if (value !== undefined && value.trim() !== '')
+                cleaned[key] = value;
+        }
+        const parsed = ConfigSchema.env.safeParse(cleaned);
+        if (!parsed.success)
+            throw new ConfigError(`Invalid scrapme configuration:\n${z.prettifyError(parsed.error)}`);
+        return ConfigLoader.deepFreeze(ConfigLoader.toConfig(parsed.data));
+    }
+    static toConfig(e) {
+        return {
+            logLevel: e.LOG_LEVEL,
+            scrapAnything: {
+                url: e.SCRAP_ANYTHING_URL,
+                secret: e.SCRAP_ANYTHING_SECRET,
+                requestTimeoutMs: e.SCRAP_ANYTHING_REQUEST_TIMEOUT_MS,
+                pollIntervalMs: e.SCRAP_ANYTHING_POLL_INTERVAL_MS,
+                waitTimeoutMs: e.SCRAP_ANYTHING_WAIT_TIMEOUT_MS,
+            },
+            scrape: { stepTimeoutMs: e.SCRAPE_STEP_TIMEOUT_MS, stepRetries: e.SCRAPE_STEP_RETRIES },
+            couch: {
+                url: Urls.withoutTrailingSlash(e.COUCHDB_URL),
+                publicUrl: Urls.withoutTrailingSlash(e.COUCHDB_PUBLIC_URL || e.COUCHDB_URL),
+                user: e.COUCHDB_USER,
+                password: e.COUCHDB_PASSWORD,
+                db: e.COUCHDB_DB,
+                runsDb: e.COUCHDB_RUNS_DB,
+                timeoutMs: e.COUCHDB_TIMEOUT_MS,
+                retryLimit: e.COUCHDB_RETRY_LIMIT,
+            },
+            smtp: {
+                host: e.SMTP_HOST,
+                port: e.SMTP_PORT,
+                secure: e.SMTP_SECURE,
+                user: e.SMTP_USER,
+                password: e.SMTP_PASSWORD,
+                timeoutMs: e.SMTP_TIMEOUT_MS,
+            },
+            mail: { from: e.MAIL_FROM, to: e.MAIL_TO, uiUrl: e.MAIL_UI_URL },
+            lighty: {
+                dsn: e.LIGHTY_DSN,
+                environment: e.LIGHTY_ENVIRONMENT,
+                release: e.LIGHTY_RELEASE,
+                flushTimeoutMs: e.LIGHTY_FLUSH_TIMEOUT_MS,
+            },
+            flow: {
+                retries: e.FLOW_RETRIES,
+                timeoutMs: e.FLOW_TIMEOUT_MS,
+                retryDelayMs: e.FLOW_RETRY_DELAY_MS,
+                maxRetryDelayMs: e.FLOW_RETRY_MAX_DELAY_MS,
+                saveRuns: e.FLOW_SAVE_RUNS,
+                maxStoredChars: e.FLOW_MAX_STORED_CHARS,
+            },
+        };
+    }
+    static deepFreeze(value) {
+        if (value && typeof value === 'object') {
+            for (const v of Object.values(value))
+                ConfigLoader.deepFreeze(v);
+            Object.freeze(value);
+        }
+        return value;
+    }
+}
+//# sourceMappingURL=config-loader.js.map

package/dist/src/config/config-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../../../src/config/config-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,YAAY,EAAkB,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,MAAM,OAAO,WAAY,SAAQ,KAAK;IACpC,YAAmB,OAAe;QAChC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED,2GAA2G;AAC3G,MAAM,OAAO,YAAY;IAChB,MAAM,CAAC,IAAI,CAAC,MAA0C,OAAO,CAAC,GAAG;QACtE,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACvE,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,WAAW,CAAC,mCAAmC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/G,OAAO,YAAY,CAAC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,CAAC;IAEO,MAAM,CAAC,QAAQ,CAAC,CAAY;QAClC,OAAO;YACL,QAAQ,EAAE,CAAC,CAAC,SAAS;YACrB,aAAa,EAAE;gBACb,GAAG,EAAE,CAAC,CAAC,kBAAkB;gBACzB,MAAM,EAAE,CAAC,CAAC,qBAAqB;gBAC/B,gBAAgB,EAAE,CAAC,CAAC,iCAAiC;gBACrD,cAAc,EAAE,CAAC,CAAC,+BAA+B;gBACjD,aAAa,EAAE,CAAC,CAAC,8BAA8B;aAChD;YACD,MAAM,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,sBAAsB,EAAE,WAAW,EAAE,CAAC,CAAC,mBAAmB,EAAE;YACvF,KAAK,EAAE;gBACL,GAAG,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC;gBAC7C,SAAS,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC,WAAW,CAAC;gBAC3E,IAAI,EAAE,CAAC,CAAC,YAAY;gBACpB,QAAQ,EAAE,CAAC,CAAC,gBAAgB;gBAC5B,EAAE,EAAE,CAAC,CAAC,UAAU;gBAChB,MAAM,EAAE,CAAC,CAAC,eAAe;gBACzB,SAAS,EAAE,CAAC,CAAC,kBAAkB;gBAC/B,UAAU,EAAE,CAAC,CAAC,mBAAmB;aAClC;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE,CAAC,CAAC,SAAS;gBACjB,IAAI,EAAE,CAAC,CAAC,SAAS;gBACjB,MAAM,EAAE,CAAC,CAAC,WAAW;gBACrB,IAAI,EAAE,CAAC,CAAC,SAAS;gBACjB,QAAQ,EAAE,CAAC,CAAC,aAAa;gBACzB,SAAS,EAAE,CAAC,CAAC,eAAe;aAC7B;YACD,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,WAAW,EAAE;YAChE,MAAM,EAAE;gBACN,GAAG,EAAE,CAAC,CAAC,UAAU;gBACjB,WAAW,EAAE,CAAC,CAAC,kBAAkB;gBACjC,OAAO,EAAE,CAAC,CAAC,cAAc;gBACzB,cAAc,EAAE,CAAC,CAAC,uBAAuB;aAC1C;YACD,IAAI,EAAE;gBACJ,OAAO,EAAE,CAAC,CAAC,YAAY;gBACvB,SAAS,EAAE,CAAC,CAAC,eAAe;gBAC5B,YAAY,EAAE,CAAC,CAAC,mBAAmB;gBACnC,eAAe,EAAE,CAAC,CAAC,uBAAuB;gBAC1C,QAAQ,EAAE,CAAC,CAAC,cAAc;gBAC1B,cAAc,EAAE,CAAC,CAAC,qBAAqB;aACxC;SACF,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,UAAU,CAAI,KAAQ;QACnC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;gBAAE,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/src/config/config.models.d.ts

@@ -0,0 +1,144 @@
+import type { LogLevel } from '../logging/logger.models.js';
+/** Postgres for the pg-boss flow queue (required). */
+export interface PostgresOptions {
+    /** e.g. postgres://user:password@localhost:5434/scrapme */
+    connectionString: string;
+    /** pg-boss tables live in this schema. Default "scrapme". */
+    schema?: string;
+}
+/**
+ * The pg-boss queues and their workers: one queue per flow (prefix + flow name), created when the flow is first
+ * scheduled. Each flow can override retries / delay / timeout (FlowConfig).
+ */
+export interface QueueOptions {
+    /** Queue names are prefix + flow name. Default "scrapme_". */
+    prefix?: string;
+    /** Flow attempts after the first one (each attempt runs the flow from its first step). Default 3. */
+    retries?: number;
+    /** Default 30. */
+    retryDelaySeconds?: number;
+    /** Default true. */
+    retryBackoff?: boolean;
+    /** One flow attempt may run this long (pg-boss expireInSeconds). Default 3 600. */
+    timeoutSeconds?: number;
+    /** Runs of one flow at the same time in this process. Default 1. */
+    concurrency?: number;
+    /** How often idle workers look for jobs, in seconds (min 0.5). Default 2. */
+    pollingIntervalSeconds?: number;
+    /** Finished jobs are removed from Postgres after this (the runs stay in CouchDB). Default 7. */
+    retentionDays?: number;
+    /** stop() waits this long for running flows. Default 30 000. */
+    shutdownTimeoutMs?: number;
+}
+/** scrap-anything — needed by ScrapePageCommand. */
+export interface ScrapAnythingOptions {
+    /** e.g. http://localhost:3000 */
+    url: string;
+    /** API_SECRET of scrap-anything (Bearer for POST /jobs). */
+    apiSecret: string;
+    /** One HTTP request to the API. Default 30 000. */
+    requestTimeoutMs?: number;
+    /** How often a waiting step asks for the job status. Default 2 000. */
+    pollIntervalMs?: number;
+    /** How long one ScrapePage attempt waits for its job (the server retries for up to ~7 min). Default 600 000. */
+    waitTimeoutMs?: number;
+    /** Timeout of one ScrapePage attempt; must be > waitTimeoutMs. Default waitTimeoutMs + 60 000. */
+    stepTimeoutMs?: number;
+    /** New server jobs after a failed one (the server already retried). Default 1. */
+    stepRetries?: number;
+}
+/** CouchDB — the run documents and the results of SaveDocumentCommand (required). */
+export interface CouchOptions {
+    /** e.g. http://localhost:5984 */
+    url: string;
+    user: string;
+    password: string;
+    /** Database of SaveDocumentCommand. Default "scrapme". */
+    db?: string;
+    /** Database of runs and failed attempts. Default "scrapme_runs". */
+    runsDb?: string;
+    /** CouchDB as seen from a browser (Fauxton links). Default url. */
+    publicUrl?: string;
+    /** Default 15 000. */
+    timeoutMs?: number;
+    /** Default 2. */
+    retryLimit?: number;
+}
+/** SMTP — needed by SendMailCommand. */
+export interface MailOptions {
+    host: string;
+    /** Default 587 with secure=false, 465 with secure=true. */
+    port?: number;
+    /** Default false. */
+    secure?: boolean;
+    user?: string;
+    password?: string;
+    /** Default 10 000. */
+    timeoutMs?: number;
+    /** e.g. "scrapme <scrapme@example.com>" */
+    from: string;
+    /** Default recipients of SendMailCommand. */
+    to?: string[];
+}
+/** LightyErrors (sentry-clone): a step that finally fails is reported there. */
+export interface LightyOptions {
+    /** http(s)://<project-key>@<host>/<project> */
+    dsn: string;
+    /** Default "production". */
+    environment?: string;
+    /** Default "scrapme". */
+    release?: string;
+    /** Default 5 000. */
+    flushTimeoutMs?: number;
+}
+/** Defaults of every step (a step or the command itself can override them). Command retries run inside a flow attempt. */
+export interface StepDefaults {
+    /** Attempts = retries + 1. Default 3. */
+    retries?: number;
+    /** One attempt. Default 60 000. */
+    timeoutMs?: number;
+    /** First retry delay, doubled per retry, with jitter. Default 1 000. */
+    retryDelayMs?: number;
+    /** Default 30 000. */
+    maxRetryDelayMs?: number;
+    /** Longer strings (HTML!) are cut in stored inputs/outputs. Default 2 000. */
+    maxStoredChars?: number;
+}
+/**
+ * Everything scrapme connects to, given by the app (scrapme reads no environment variables). `postgres` and `couch`
+ * are required: flows run on a pg-boss queue and their progress is kept in CouchDB. The rest is needed only by the
+ * commands that use it: no `scrapAnything` = ScrapePageCommand fails, no `mail` = SendMailCommand fails (both with
+ * NOT_CONFIGURED); no `lighty` = final errors are only logged.
+ */
+export interface ServiceOptions {
+    postgres: PostgresOptions;
+    couch: CouchOptions;
+    queue?: QueueOptions;
+    scrapAnything?: ScrapAnythingOptions;
+    mail?: MailOptions;
+    lighty?: LightyOptions;
+    defaults?: StepDefaults;
+    /** Default "info"; "silent" turns logging off. */
+    logLevel?: LogLevel;
+}
+export type ScrapAnythingConfig = Required<ScrapAnythingOptions>;
+export type CouchConfig = Required<CouchOptions>;
+export type MailConfig = Required<Omit<MailOptions, 'user' | 'password'>> & {
+    user: string | null;
+    password: string | null;
+};
+export type LightyConfig = Required<LightyOptions>;
+export type StepDefaultsConfig = Required<StepDefaults>;
+export type PostgresConfig = Required<PostgresOptions>;
+export type QueueConfig = Required<QueueOptions>;
+/** Validated, frozen configuration (ConfigBuilder). A missing optional service is null. */
+export interface ScrapmeConfig {
+    logLevel: LogLevel;
+    flow: StepDefaultsConfig;
+    postgres: PostgresConfig;
+    queue: QueueConfig;
+    scrapAnything: ScrapAnythingConfig | null;
+    couch: CouchConfig;
+    mail: MailConfig | null;
+    lighty: LightyConfig | null;
+}

package/dist/src/config/config.models.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=config.models.js.map

package/dist/src/config/config.models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.models.js","sourceRoot":"","sources":["../../../src/config/config.models.ts"],"names":[],"mappings":""}

package/dist/src/config/config.schema.d.ts

@@ -0,0 +1,75 @@
+import { z } from 'zod';
+/** The zod schema of every setting, with its default. Documented in .env.example. */
+export declare class ConfigSchema {
+    /**
+     * CouchDB's own rule for database names (docs.couchdb.org, "PUT /{db}"). It is a rule about characters, so a
+     * regex is the clearest way to state it.
+     */
+    private static readonly COUCH_DB_NAME;
+    private static readonly TRUE_VALUES;
+    private static readonly FALSE_VALUES;
+    private static readonly LIST_SEPARATOR;
+    private static int;
+    private static str;
+    private static bool;
+    private static couchDbName;
+    private static list;
+    static readonly env: z.ZodObject<{
+        LOG_LEVEL: z.ZodDefault<z.ZodEnum<{
+            debug: "debug";
+            info: "info";
+            warn: "warn";
+            error: "error";
+            silent: "silent";
+        }>>;
+        SCRAP_ANYTHING_URL: z.ZodDefault<z.ZodURL>;
+        SCRAP_ANYTHING_SECRET: z.ZodDefault<z.ZodString>;
+        SCRAP_ANYTHING_REQUEST_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SCRAP_ANYTHING_POLL_INTERVAL_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SCRAP_ANYTHING_WAIT_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SCRAPE_STEP_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SCRAPE_STEP_RETRIES: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        COUCHDB_URL: z.ZodDefault<z.ZodURL>;
+        COUCHDB_PUBLIC_URL: z.ZodDefault<z.ZodString>;
+        COUCHDB_USER: z.ZodDefault<z.ZodString>;
+        COUCHDB_PASSWORD: z.ZodDefault<z.ZodString>;
+        COUCHDB_DB: z.ZodDefault<z.ZodString>;
+        COUCHDB_RUNS_DB: z.ZodDefault<z.ZodString>;
+        COUCHDB_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        COUCHDB_RETRY_LIMIT: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SMTP_HOST: z.ZodDefault<z.ZodString>;
+        SMTP_PORT: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SMTP_SECURE: z.ZodPipe<z.ZodDefault<z.ZodEnum<{
+            0: "0";
+            1: "1";
+            true: "true";
+            yes: "yes";
+            false: "false";
+            no: "no";
+        }>>, z.ZodTransform<boolean, "0" | "1" | "true" | "yes" | "false" | "no">>;
+        SMTP_USER: z.ZodDefault<z.ZodString>;
+        SMTP_PASSWORD: z.ZodDefault<z.ZodString>;
+        SMTP_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        MAIL_FROM: z.ZodDefault<z.ZodString>;
+        MAIL_TO: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<string[], string>>;
+        MAIL_UI_URL: z.ZodDefault<z.ZodString>;
+        LIGHTY_DSN: z.ZodDefault<z.ZodString>;
+        LIGHTY_ENVIRONMENT: z.ZodDefault<z.ZodString>;
+        LIGHTY_RELEASE: z.ZodDefault<z.ZodString>;
+        LIGHTY_FLUSH_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_RETRIES: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_RETRY_DELAY_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_RETRY_MAX_DELAY_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_SAVE_RUNS: z.ZodPipe<z.ZodDefault<z.ZodEnum<{
+            0: "0";
+            1: "1";
+            true: "true";
+            yes: "yes";
+            false: "false";
+            no: "no";
+        }>>, z.ZodTransform<boolean, "0" | "1" | "true" | "yes" | "false" | "no">>;
+        FLOW_MAX_STORED_CHARS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    }, z.core.$strip>;
+}
+export type ParsedEnv = z.infer<typeof ConfigSchema.env>;

package/dist/src/config/config.schema.js

@@ -0,0 +1,87 @@
+import { z } from 'zod';
+import { LOG_LEVEL, LOG_LEVELS } from '../logging/logger.models.js';
+import { DsnCheck } from './dsn-check.js';
+/** The zod schema of every setting, with its default. Documented in .env.example. */
+export class ConfigSchema {
+    /**
+     * CouchDB's own rule for database names (docs.couchdb.org, "PUT /{db}"). It is a rule about characters, so a
+     * regex is the clearest way to state it.
+     */
+    static COUCH_DB_NAME = /^[a-z][a-z0-9_$()+/-]*$/;
+    static TRUE_VALUES = ['true', '1', 'yes'];
+    static FALSE_VALUES = ['false', '0', 'no'];
+    static LIST_SEPARATOR = ',';
+    static int(def, min = 0) {
+        return z.coerce.number().int().min(min).default(def);
+    }
+    static str(def) {
+        return z.string().trim().default(def);
+    }
+    static bool(def) {
+        const C = ConfigSchema;
+        return z
+            .enum([...C.TRUE_VALUES, ...C.FALSE_VALUES])
+            .default(def ? C.TRUE_VALUES[0] : C.FALSE_VALUES[0])
+            .transform((v) => C.TRUE_VALUES.includes(v));
+    }
+    static couchDbName(def) {
+        return z.string().regex(ConfigSchema.COUCH_DB_NAME, 'CouchDB names: lowercase letters, digits and _$()+/-, starting with a letter').default(def);
+    }
+    static list(def) {
+        return z
+            .string()
+            .default(def)
+            .transform((v) => v.split(ConfigSchema.LIST_SEPARATOR).map((s) => s.trim()).filter(Boolean));
+    }
+    static env = z
+        .object({
+        LOG_LEVEL: z.enum(LOG_LEVELS).default(LOG_LEVEL.INFO),
+        SCRAP_ANYTHING_URL: z.url().default('http://localhost:3000'),
+        SCRAP_ANYTHING_SECRET: ConfigSchema.str(''),
+        SCRAP_ANYTHING_REQUEST_TIMEOUT_MS: ConfigSchema.int(30_000, 1),
+        SCRAP_ANYTHING_POLL_INTERVAL_MS: ConfigSchema.int(2_000, 1),
+        SCRAP_ANYTHING_WAIT_TIMEOUT_MS: ConfigSchema.int(600_000, 1),
+        SCRAPE_STEP_TIMEOUT_MS: ConfigSchema.int(660_000, 1),
+        SCRAPE_STEP_RETRIES: ConfigSchema.int(1),
+        COUCHDB_URL: z.url().default('http://localhost:5984'),
+        COUCHDB_PUBLIC_URL: ConfigSchema.str(''),
+        COUCHDB_USER: ConfigSchema.str('admin'),
+        COUCHDB_PASSWORD: ConfigSchema.str('admin'),
+        COUCHDB_DB: ConfigSchema.couchDbName('scrapme'),
+        COUCHDB_RUNS_DB: ConfigSchema.couchDbName('scrapme_runs'),
+        COUCHDB_TIMEOUT_MS: ConfigSchema.int(15_000, 1),
+        COUCHDB_RETRY_LIMIT: ConfigSchema.int(2),
+        SMTP_HOST: ConfigSchema.str('localhost'),
+        SMTP_PORT: ConfigSchema.int(1025, 1),
+        SMTP_SECURE: ConfigSchema.bool(false),
+        SMTP_USER: ConfigSchema.str(''),
+        SMTP_PASSWORD: ConfigSchema.str(''),
+        SMTP_TIMEOUT_MS: ConfigSchema.int(10_000, 1),
+        MAIL_FROM: ConfigSchema.str('scrapme <scrapme@localhost>'),
+        MAIL_TO: ConfigSchema.list('owner@localhost'),
+        MAIL_UI_URL: ConfigSchema.str('http://localhost:8025'),
+        LIGHTY_DSN: ConfigSchema.str(''),
+        LIGHTY_ENVIRONMENT: ConfigSchema.str('development'),
+        LIGHTY_RELEASE: ConfigSchema.str('scrapme@0.1.0'),
+        LIGHTY_FLUSH_TIMEOUT_MS: ConfigSchema.int(5_000, 1),
+        FLOW_RETRIES: ConfigSchema.int(2),
+        FLOW_TIMEOUT_MS: ConfigSchema.int(60_000, 1),
+        FLOW_RETRY_DELAY_MS: ConfigSchema.int(1_000),
+        FLOW_RETRY_MAX_DELAY_MS: ConfigSchema.int(30_000),
+        FLOW_SAVE_RUNS: ConfigSchema.bool(true),
+        FLOW_MAX_STORED_CHARS: ConfigSchema.int(2_000, 100),
+    })
+        .refine((e) => e.SCRAPE_STEP_TIMEOUT_MS > e.SCRAP_ANYTHING_WAIT_TIMEOUT_MS, {
+        message: 'SCRAPE_STEP_TIMEOUT_MS must be greater than SCRAP_ANYTHING_WAIT_TIMEOUT_MS (the step waits for the server job)',
+        path: ['SCRAPE_STEP_TIMEOUT_MS'],
+    })
+        .refine((e) => e.FLOW_RETRY_MAX_DELAY_MS >= e.FLOW_RETRY_DELAY_MS, {
+        message: 'FLOW_RETRY_MAX_DELAY_MS must be >= FLOW_RETRY_DELAY_MS',
+        path: ['FLOW_RETRY_MAX_DELAY_MS'],
+    })
+        .refine((e) => !e.LIGHTY_DSN || DsnCheck.isValid(e.LIGHTY_DSN), {
+        message: 'LIGHTY_DSN must look like https://<project-key>@<host>/<project>',
+        path: ['LIGHTY_DSN'],
+    });
+}
+//# sourceMappingURL=config.schema.js.map

package/dist/src/config/config.schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.schema.js","sourceRoot":"","sources":["../../../src/config/config.schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,qFAAqF;AACrF,MAAM,OAAO,YAAY;IACvB;;;OAGG;IACK,MAAM,CAAU,aAAa,GAAG,yBAAyB,CAAC;IAC1D,MAAM,CAAU,WAAW,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAU,CAAC;IAC5D,MAAM,CAAU,YAAY,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAU,CAAC;IAC7D,MAAM,CAAU,cAAc,GAAG,GAAG,CAAC;IAErC,MAAM,CAAC,GAAG,CAAC,GAAW,EAAE,GAAG,GAAG,CAAC;QACrC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACvD,CAAC;IAEO,MAAM,CAAC,GAAG,CAAC,GAAW;QAC5B,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAEO,MAAM,CAAC,IAAI,CAAC,GAAY;QAC9B,MAAM,CAAC,GAAG,YAAY,CAAC;QACvB,OAAO,CAAC;aACL,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;aAC3C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;aACnD,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,WAAiC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;IAEO,MAAM,CAAC,WAAW,CAAC,GAAW;QACpC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,aAAa,EAAE,8EAA8E,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnJ,CAAC;IAEO,MAAM,CAAC,IAAI,CAAC,GAAW;QAC7B,OAAO,CAAC;aACL,MAAM,EAAE;aACR,OAAO,CAAC,GAAG,CAAC;aACZ,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IACjG,CAAC;IAEM,MAAM,CAAU,GAAG,GAAG,CAAC;SAC3B,MAAM,CAAC;QACN,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;QAErD,kBAAkB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,uBAAuB,CAAC;QAC5D,qBAAqB,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,iCAAiC,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,+BAA+B,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3D,8BAA8B,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5D,sBAAsB,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACpD,mBAAmB,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAExC,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,uBAAuB,CAAC;QACrD,kBAAkB,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACxC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC;QACvC,gBAAgB,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC;QAC3C,UAAU,EAAE,YAAY,CAAC,WAAW,CAAC,SAAS,CAAC;QAC/C,eAAe,EAAE,YAAY,CAAC,WAAW,CAAC,cAAc,CAAC;QACzD,kBAAkB,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,mBAAmB,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAExC,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC;QACxC,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,WAAW,EAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;QACrC,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,aAAa,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,eAAe,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5C,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,6BAA6B,CAAC;QAC1D,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC;QAC7C,WAAW,EAAE,YAAY,CAAC,GAAG,CAAC,uBAAuB,CAAC;QAEtD,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QAChC,kBAAkB,EAAE,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC;QACnD,cAAc,EAAE,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC;QACjD,uBAAuB,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAEnD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QACjC,eAAe,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5C,mBAAmB,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;QAC5C,uBAAuB,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC;QACjD,cAAc,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;QACvC,qBAAqB,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;KACpD,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB,GAAG,CAAC,CAAC,8BAA8B,EAAE;QAC1E,OAAO,EAAE,gHAAgH;QACzH,IAAI,EAAE,CAAC,wBAAwB,CAAC;KACjC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,uBAAuB,IAAI,CAAC,CAAC,mBAAmB,EAAE;QACjE,OAAO,EAAE,wDAAwD;QACjE,IAAI,EAAE,CAAC,yBAAyB,CAAC;KAClC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE;QAC9D,OAAO,EAAE,kEAAkE;QAC3E,IAAI,EAAE,CAAC,YAAY,CAAC;KACrB,CAAC,CAAC"}

package/dist/src/config/dsn-check.d.ts

@@ -0,0 +1,5 @@
+/** LightyErrors DSN: `http(s)://<project-key>@<host>[:port]/<project>` — checked with URL, not a pattern. */
+export declare class DsnCheck {
+    private static readonly PROTOCOLS;
+    static isValid(dsn: string): boolean;
+}

package/dist/src/config/dsn-check.js

@@ -0,0 +1,12 @@
+/** LightyErrors DSN: `http(s)://<project-key>@<host>[:port]/<project>` — checked with URL, not a pattern. */
+export class DsnCheck {
+    static PROTOCOLS = new Set(['http:', 'https:']);
+    static isValid(dsn) {
+        if (!URL.canParse(dsn))
+            return false;
+        const url = new URL(dsn);
+        const project = url.pathname.split('/').filter(Boolean);
+        return DsnCheck.PROTOCOLS.has(url.protocol) && url.username !== '' && url.password === '' && project.length > 0;
+    }
+}
+//# sourceMappingURL=dsn-check.js.map

package/dist/src/config/dsn-check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dsn-check.js","sourceRoot":"","sources":["../../../src/config/dsn-check.ts"],"names":[],"mappings":"AAAA,6GAA6G;AAC7G,MAAM,OAAO,QAAQ;IACX,MAAM,CAAU,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAE1D,MAAM,CAAC,OAAO,CAAC,GAAW;QAC/B,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACxD,OAAO,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAClH,CAAC"}

package/dist/src/di/container.d.ts

@@ -0,0 +1,13 @@
+import { Container } from 'inversify';
+import type { ScrapmeConfig } from '../config/config.models.js';
+/**
+ * Composition root of scrapme's own services (singletons). The container's parent is the app's container (when
+ * given): `autobind` resolves any `@injectable()` command class by its class, and its `@inject(...)`s find the
+ * app's services through the parent — commands need no bindings. A missing optional service (mail, scrapAnything)
+ * gets an "unconfigured" stand-in, so only the command that needs it fails (NOT_CONFIGURED). Replace a scrapme
+ * service with `scrapme.container.rebindSync(TYPES.X)` before start() (tests use this for the queue and stores).
+ */
+export declare class ContainerBuilder {
+    private static readonly APP;
+    static build(config: ScrapmeConfig, parent?: Container): Container;
+}

package/dist/src/di/container.js

@@ -0,0 +1,67 @@
+import { Container } from 'inversify';
+import { ScrapAnythingClient } from 'scrap-anything-client';
+import { CommandExecutor } from '../flow/command-executor.js';
+import { CommandResolver } from '../flow/command-resolver.js';
+import { FlowJobProcessor } from '../flow/flow-job-processor.js';
+import { FlowParams } from '../flow/flow-params.js';
+import { FlowScheduler } from '../flow/flow-scheduler.js';
+import { StepInputBuilder } from '../flow/step-input-builder.js';
+import { Logger } from '../logging/logger.js';
+import { SmtpMailer } from '../mail/smtp-mailer.js';
+import { UnconfiguredMailer } from '../mail/unconfigured-mailer.js';
+import { PgBossFlowQueue } from '../queue/pg-boss-flow-queue.js';
+import { LightyErrorReporter } from '../reporting/lighty-error-reporter.js';
+import { LogErrorReporter } from '../reporting/log-error-reporter.js';
+import { CouchConnection } from '../storage/couch-connection.js';
+import { CouchDocumentStore } from '../storage/couch-document-store.js';
+import { CouchRunStore } from '../storage/couch-run-store.js';
+import { TYPES } from './types.js';
+/**
+ * Composition root of scrapme's own services (singletons). The container's parent is the app's container (when
+ * given): `autobind` resolves any `@injectable()` command class by its class, and its `@inject(...)`s find the
+ * app's services through the parent — commands need no bindings. A missing optional service (mail, scrapAnything)
+ * gets an "unconfigured" stand-in, so only the command that needs it fails (NOT_CONFIGURED). Replace a scrapme
+ * service with `scrapme.container.rebindSync(TYPES.X)` before start() (tests use this for the queue and stores).
+ */
+export class ContainerBuilder {
+    static APP = 'scrapme';
+    static build(config, parent) {
+        const c = new Container({ defaultScope: 'Singleton', autobind: true, ...(parent ? { parent } : {}) });
+        c.bind(TYPES.Config).toConstantValue(config);
+        c.bind(TYPES.PostgresConfig).toConstantValue(config.postgres);
+        c.bind(TYPES.QueueConfig).toConstantValue(config.queue);
+        c.bind(TYPES.CouchConfig).toConstantValue(config.couch);
+        c.bind(TYPES.Logger).toConstantValue(new Logger(config.logLevel, { app: ContainerBuilder.APP }));
+        c.bind(TYPES.Container).toConstantValue(c);
+        c.bind(TYPES.FlowQueue).to(PgBossFlowQueue);
+        c.bind(TYPES.FlowScheduler).to(FlowScheduler);
+        c.bind(TYPES.FlowJobProcessor).to(FlowJobProcessor);
+        c.bind(TYPES.CommandResolver).to(CommandResolver);
+        c.bind(TYPES.CommandExecutor).to(CommandExecutor);
+        c.bind(TYPES.StepInputBuilder).to(StepInputBuilder);
+        c.bind(TYPES.FlowParams).to(FlowParams);
+        c.bind(TYPES.CouchConnection).to(CouchConnection);
+        c.bind(TYPES.RunStore).to(CouchRunStore);
+        c.bind(TYPES.DocumentStore).to(CouchDocumentStore);
+        if (config.lighty) {
+            c.bind(TYPES.LightyConfig).toConstantValue(config.lighty);
+            c.bind(TYPES.ErrorReporter).to(LightyErrorReporter);
+        }
+        else {
+            c.bind(TYPES.ErrorReporter).to(LogErrorReporter);
+        }
+        if (config.mail) {
+            c.bind(TYPES.MailConfig).toConstantValue(config.mail);
+            c.bind(TYPES.Mailer).to(SmtpMailer);
+        }
+        else {
+            c.bind(TYPES.Mailer).to(UnconfiguredMailer);
+        }
+        const sa = config.scrapAnything;
+        if (sa) {
+            c.bind(TYPES.ScrapAnythingClient).toConstantValue(new ScrapAnythingClient({ baseUrl: sa.url, apiSecret: sa.apiSecret, requestTimeoutMs: sa.requestTimeoutMs, pollIntervalMs: sa.pollIntervalMs, waitTimeoutMs: sa.waitTimeoutMs }));
+        }
+        return c;
+    }
+}
+//# sourceMappingURL=container.js.map

package/dist/src/di/container.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"container.js","sourceRoot":"","sources":["../../../src/di/container.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAE9D,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC;;;;;;GAMG;AACH,MAAM,OAAO,gBAAgB;IACnB,MAAM,CAAU,GAAG,GAAG,SAAS,CAAC;IAEjC,MAAM,CAAC,KAAK,CAAC,MAAqB,EAAE,MAAkB;QAC3D,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAEtG,CAAC,CAAC,IAAI,CAAgB,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxD,CAAC,CAAC,IAAI,CAAS,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,gBAAgB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACzG,CAAC,CAAC,IAAI,CAAY,KAAK,CAAC,SAAS,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAEtD,CAAC,CAAC,IAAI,CAAY,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;QACvD,CAAC,CAAC,IAAI,CAAgB,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC;QAC7D,CAAC,CAAC,IAAI,CAAmB,KAAK,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC;QACtE,CAAC,CAAC,IAAI,CAAkB,KAAK,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC,CAAC,IAAI,CAAkB,KAAK,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC,CAAC,IAAI,CAAmB,KAAK,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC;QACtE,CAAC,CAAC,IAAI,CAAa,KAAK,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;QAEpD,CAAC,CAAC,IAAI,CAAiB,KAAK,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;QAClE,CAAC,CAAC,IAAI,CAAW,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC,CAAC,IAAI,CAAgB,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC;QAElE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,CAAC,CAAC,IAAI,CAAgB,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,CAAC,CAAC,IAAI,CAAgB,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACtD,CAAC,CAAC,IAAI,CAAS,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,CAAC,CAAC,IAAI,CAAS,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC;QAChC,IAAI,EAAE,EAAE,CAAC;YACP,CAAC,CAAC,IAAI,CAAsB,KAAK,CAAC,mBAAmB,CAAC,CAAC,eAAe,CACpE,IAAI,mBAAmB,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,gBAAgB,EAAE,EAAE,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE,CAAC,cAAc,EAAE,aAAa,EAAE,EAAE,CAAC,aAAa,EAAE,CAAC,CACjL,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,CAAC;IACX,CAAC"}

package/dist/src/di/types.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Inversify service identifiers of scrapme's own services. Commands are resolved by their class (autobind);
+ * your services live in your app's container (passed to Scrapme.create), which is the parent of scrapme's.
+ */
+export declare const TYPES: {
+    readonly Config: symbol;
+    readonly PostgresConfig: symbol;
+    readonly QueueConfig: symbol;
+    readonly CouchConfig: symbol;
+    readonly MailConfig: symbol;
+    readonly LightyConfig: symbol;
+    readonly Logger: symbol;
+    readonly Container: symbol;
+    readonly FlowScheduler: symbol;
+    readonly FlowJobProcessor: symbol;
+    readonly FlowQueue: symbol;
+    readonly RunStore: symbol;
+    readonly CommandResolver: symbol;
+    readonly CommandExecutor: symbol;
+    readonly StepInputBuilder: symbol;
+    readonly FlowParams: symbol;
+    readonly ErrorReporter: symbol;
+    readonly DocumentStore: symbol;
+    readonly CouchConnection: symbol;
+    readonly Mailer: symbol;
+    readonly ScrapAnythingClient: symbol;
+};

package/dist/src/di/types.js

@@ -0,0 +1,28 @@
+/**
+ * Inversify service identifiers of scrapme's own services. Commands are resolved by their class (autobind);
+ * your services live in your app's container (passed to Scrapme.create), which is the parent of scrapme's.
+ */
+export const TYPES = {
+    Config: Symbol.for('scrapme.Config'),
+    PostgresConfig: Symbol.for('scrapme.PostgresConfig'),
+    QueueConfig: Symbol.for('scrapme.QueueConfig'),
+    CouchConfig: Symbol.for('scrapme.CouchConfig'),
+    MailConfig: Symbol.for('scrapme.MailConfig'),
+    LightyConfig: Symbol.for('scrapme.LightyConfig'),
+    Logger: Symbol.for('scrapme.Logger'),
+    Container: Symbol.for('scrapme.Container'),
+    FlowScheduler: Symbol.for('scrapme.FlowScheduler'),
+    FlowJobProcessor: Symbol.for('scrapme.FlowJobProcessor'),
+    FlowQueue: Symbol.for('scrapme.FlowQueue'),
+    RunStore: Symbol.for('scrapme.RunStore'),
+    CommandResolver: Symbol.for('scrapme.CommandResolver'),
+    CommandExecutor: Symbol.for('scrapme.CommandExecutor'),
+    StepInputBuilder: Symbol.for('scrapme.StepInputBuilder'),
+    FlowParams: Symbol.for('scrapme.FlowParams'),
+    ErrorReporter: Symbol.for('scrapme.ErrorReporter'),
+    DocumentStore: Symbol.for('scrapme.DocumentStore'),
+    CouchConnection: Symbol.for('scrapme.CouchConnection'),
+    Mailer: Symbol.for('scrapme.Mailer'),
+    ScrapAnythingClient: Symbol.for('scrapme.ScrapAnythingClient'),
+};
+//# sourceMappingURL=types.js.map

package/dist/src/di/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/di/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC;IACpD,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC;IAC9C,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC;IAC9C,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAC5C,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAChD,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACpC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC1C,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC;IAClD,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAC;IACxD,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC1C,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC;IACxC,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,yBAAyB,CAAC;IACtD,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,yBAAyB,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAC;IACxD,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAC5C,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC;IAClD,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC;IAClD,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,yBAAyB,CAAC;IACtD,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACpC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,6BAA6B,CAAC;CACtD,CAAC"}

package/dist/src/flow/command-executor.d.ts

@@ -0,0 +1,18 @@
+import type { ScrapmeConfig } from '../config/config.models.js';
+import { type AttemptFailedListener, type AttemptStartedListener, type Command, type CommandContext, type ResolvedStepPolicy, type StepDefinition, type StepExecution } from './flow.models.js';
+/**
+ * Runs one step inside a flow attempt: attempts with a timeout each, retried with backoff while the error is
+ * retryable. Never throws. (The whole flow is retried by pg-boss; this is the per-command retry inside it.)
+ */
+export declare class CommandExecutor {
+    private readonly config;
+    constructor(config: ScrapmeConfig);
+    /** step settings > command.options > defaults (Scrapme.create({ defaults })). */
+    policy(step: StepDefinition, command: Command): ResolvedStepPolicy;
+    execute(step: StepDefinition, command: Command, input: unknown, base: Omit<CommandContext, 'attempt' | 'maxAttempts' | 'signal'>, runSignal: AbortSignal | undefined, listeners: {
+        started?: AttemptStartedListener;
+        failed: AttemptFailedListener;
+    }): Promise<StepExecution>;
+    private report;
+    private shouldRetry;
+}