scrapme-flow 0.0.0-stage → 0.1.0

package/dist/src/config/config-builder.js

@@ -0,0 +1,131 @@
+import { z } from 'zod';
+import { LOG_LEVEL, LOG_LEVELS } from '../logging/logger.models.js';
+import { Urls } from '../util/url.js';
+import { DsnCheck } from './dsn-check.js';
+/** The services given to Scrapme.create are invalid (thrown by Scrapme.create). */
+export class ConfigError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ConfigError';
+    }
+}
+/** Checks the services an app gives to Scrapme.create (zod), fills in defaults and freezes the result. */
+export class ConfigBuilder {
+    /**
+     * CouchDB's own rule for database names (docs.couchdb.org, "PUT /{db}"). It is a rule about characters, so a
+     * regex is the clearest way to state it.
+     */
+    static COUCH_DB_NAME = /^[a-z][a-z0-9_$()+/-]*$/;
+    static STEP_TIMEOUT_MARGIN_MS = 60_000;
+    static SMTP_PORT = 587;
+    static SMTPS_PORT = 465;
+    /** pg-boss schema names go into SQL as identifiers. */
+    static SQL_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
+    static QUEUE_PREFIX = 'scrapme_';
+    static ms = (min = 0) => z.number().int().min(min);
+    static dbName = z.string().regex(ConfigBuilder.COUCH_DB_NAME, 'CouchDB names: lowercase letters, digits and _$()+/-, starting with a letter');
+    static schema = z.strictObject({
+        logLevel: z.enum(LOG_LEVELS).default(LOG_LEVEL.INFO),
+        defaults: z
+            .strictObject({
+            retries: ConfigBuilder.ms().default(3),
+            timeoutMs: ConfigBuilder.ms(1).default(60_000),
+            retryDelayMs: ConfigBuilder.ms().default(1_000),
+            maxRetryDelayMs: ConfigBuilder.ms().default(30_000),
+            maxStoredChars: ConfigBuilder.ms(100).default(2_000),
+        })
+            .default({ retries: 3, timeoutMs: 60_000, retryDelayMs: 1_000, maxRetryDelayMs: 30_000, maxStoredChars: 2_000 })
+            .refine((d) => d.maxRetryDelayMs >= d.retryDelayMs, { message: 'maxRetryDelayMs must be >= retryDelayMs' }),
+        postgres: z.strictObject({
+            connectionString: z.string().refine((v) => URL.canParse(v), 'must be a postgres:// URL'),
+            schema: z.string().regex(ConfigBuilder.SQL_IDENTIFIER, 'letters, digits and _ only').default('scrapme'),
+        }),
+        queue: z
+            .strictObject({
+            prefix: z.string().regex(ConfigBuilder.SQL_IDENTIFIER, 'letters, digits and _ only').default(ConfigBuilder.QUEUE_PREFIX),
+            retries: ConfigBuilder.ms().default(3),
+            retryDelaySeconds: ConfigBuilder.ms().default(30),
+            retryBackoff: z.boolean().default(true),
+            timeoutSeconds: ConfigBuilder.ms(1).default(3_600),
+            concurrency: ConfigBuilder.ms(1).default(1),
+            // pg-boss: at least 0.5 s
+            pollingIntervalSeconds: z.number().min(0.5).default(2),
+            retentionDays: ConfigBuilder.ms(1).default(7),
+            shutdownTimeoutMs: ConfigBuilder.ms(0).default(30_000),
+        })
+            .default({ prefix: ConfigBuilder.QUEUE_PREFIX, retries: 3, retryDelaySeconds: 30, retryBackoff: true, timeoutSeconds: 3_600, concurrency: 1, pollingIntervalSeconds: 2, retentionDays: 7, shutdownTimeoutMs: 30_000 }),
+        monitor: z.strictObject({ port: ConfigBuilder.ms(0).max(65_535), host: z.string().default('127.0.0.1') }).optional(),
+        scrapAnything: z
+            .strictObject({
+            url: z.url(),
+            apiSecret: z.string().min(1, 'apiSecret is required (API_SECRET of scrap-anything)'),
+            requestTimeoutMs: ConfigBuilder.ms(1).default(30_000),
+            pollIntervalMs: ConfigBuilder.ms(1).default(2_000),
+            waitTimeoutMs: ConfigBuilder.ms(1).default(600_000),
+            stepTimeoutMs: ConfigBuilder.ms(1).optional(),
+            stepRetries: ConfigBuilder.ms().default(1),
+        })
+            .transform((s) => ({ ...s, url: Urls.withoutTrailingSlash(s.url), stepTimeoutMs: s.stepTimeoutMs ?? s.waitTimeoutMs + ConfigBuilder.STEP_TIMEOUT_MARGIN_MS }))
+            .refine((s) => s.stepTimeoutMs > s.waitTimeoutMs, { message: 'stepTimeoutMs must be greater than waitTimeoutMs (the step waits for the server job)' })
+            .optional(),
+        couch: z
+            .strictObject({
+            url: z.url(),
+            user: z.string(),
+            password: z.string(),
+            db: ConfigBuilder.dbName.default('scrapme'),
+            runsDb: ConfigBuilder.dbName.default('scrapme_runs'),
+            publicUrl: z.url().optional(),
+            timeoutMs: ConfigBuilder.ms(1).default(15_000),
+            retryLimit: ConfigBuilder.ms().default(2),
+        })
+            .transform((c) => ({ ...c, url: Urls.withoutTrailingSlash(c.url), publicUrl: Urls.withoutTrailingSlash(c.publicUrl ?? c.url) })),
+        mail: z
+            .strictObject({
+            host: z.string().min(1),
+            port: ConfigBuilder.ms(1).optional(),
+            secure: z.boolean().default(false),
+            user: z.string().optional(),
+            password: z.string().optional(),
+            timeoutMs: ConfigBuilder.ms(1).default(10_000),
+            from: z.string().min(1),
+            to: z.array(z.string().min(1)).default([]),
+        })
+            .transform((m) => ({ ...m, port: m.port ?? (m.secure ? ConfigBuilder.SMTPS_PORT : ConfigBuilder.SMTP_PORT), user: m.user ?? null, password: m.password ?? null }))
+            .optional(),
+        lighty: z
+            .strictObject({
+            dsn: z.string().refine((dsn) => DsnCheck.isValid(dsn), 'must look like https://<project-key>@<host>/<project>'),
+            environment: z.string().default('production'),
+            release: z.string().default('scrapme'),
+            flushTimeoutMs: ConfigBuilder.ms(1).default(5_000),
+        })
+            .optional(),
+    });
+    static build(options) {
+        const parsed = ConfigBuilder.schema.safeParse(options);
+        if (!parsed.success)
+            throw new ConfigError(`Invalid scrapme options:\n${z.prettifyError(parsed.error)}`);
+        const o = parsed.data;
+        return ConfigBuilder.deepFreeze({
+            logLevel: o.logLevel,
+            flow: o.defaults,
+            postgres: o.postgres,
+            queue: o.queue,
+            monitor: o.monitor ?? null,
+            scrapAnything: o.scrapAnything ?? null,
+            couch: o.couch,
+            mail: o.mail ?? null,
+            lighty: o.lighty ?? null,
+        });
+    }
+    static deepFreeze(value) {
+        if (value && typeof value === 'object') {
+            for (const v of Object.values(value))
+                ConfigBuilder.deepFreeze(v);
+            Object.freeze(value);
+        }
+        return value;
+    }
+}
+//# sourceMappingURL=config-builder.js.map

package/dist/src/config/config-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config-builder.js","sourceRoot":"","sources":["../../../src/config/config-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,mFAAmF;AACnF,MAAM,OAAO,WAAY,SAAQ,KAAK;IACpC,YAAmB,OAAe;QAChC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED,0GAA0G;AAC1G,MAAM,OAAO,aAAa;IACxB;;;OAGG;IACK,MAAM,CAAU,aAAa,GAAG,yBAAyB,CAAC;IAC1D,MAAM,CAAU,sBAAsB,GAAG,MAAM,CAAC;IAChD,MAAM,CAAU,SAAS,GAAG,GAAG,CAAC;IAChC,MAAM,CAAU,UAAU,GAAG,GAAG,CAAC;IACzC,uDAAuD;IAC/C,MAAM,CAAU,cAAc,GAAG,0BAA0B,CAAC;IAC5D,MAAM,CAAU,YAAY,GAAG,UAAU,CAAC;IAE1C,MAAM,CAAU,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5D,MAAM,CAAU,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,aAAa,EAAE,8EAA8E,CAAC,CAAC;IAEvJ,MAAM,CAAU,MAAM,GAAG,CAAC,CAAC,YAAY,CAAC;QAC9C,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;QACpD,QAAQ,EAAE,CAAC;aACR,YAAY,CAAC;YACZ,OAAO,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACtC,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9C,YAAY,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;YAC/C,eAAe,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;YACnD,cAAc,EAAE,aAAa,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;SACrD,CAAC;aACD,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;aAC/G,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,OAAO,EAAE,yCAAyC,EAAE,CAAC;QAC7G,QAAQ,EAAE,CAAC,CAAC,YAAY,CAAC;YACvB,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,2BAA2B,CAAC;YACxF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;SACxG,CAAC;QACF,KAAK,EAAE,CAAC;aACL,YAAY,CAAC;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,YAAY,CAAC;YACxH,OAAO,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACtC,iBAAiB,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;YACjD,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;YACvC,cAAc,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;YAClD,WAAW,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3C,0BAA0B;YAC1B,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACtD,aAAa,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7C,iBAAiB,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;SACvD,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,aAAa,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC;QACxN,OAAO,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;QACpH,aAAa,EAAE,CAAC;aACb,YAAY,CAAC;YACZ,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE;YACZ,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,sDAAsD,CAAC;YACpF,gBAAgB,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACrD,cAAc,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;YAClD,aAAa,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;YACnD,aAAa,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;YAC7C,WAAW,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;SAC3C,CAAC;aACD,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,aAAa,GAAG,aAAa,CAAC,sBAAsB,EAAE,CAAC,CAAC;aAC7J,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,sFAAsF,EAAE,CAAC;aACrJ,QAAQ,EAAE;QACb,KAAK,EAAE,CAAC;aACL,YAAY,CAAC;YACZ,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE;YACZ,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;YAChB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;YACpB,EAAE,EAAE,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;YAC3C,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC;YACpD,SAAS,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;YAC7B,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9C,UAAU,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;SAC1C,CAAC;aACD,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClI,IAAI,EAAE,CAAC;aACJ,YAAY,CAAC;YACZ,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,IAAI,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;YAClC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC/B,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;SAC3C,CAAC;aACD,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;aACjK,QAAQ,EAAE;QACb,MAAM,EAAE,CAAC;aACN,YAAY,CAAC;YACZ,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,uDAAuD,CAAC;YAC/G,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC;YAC7C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;YACtC,cAAc,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;SACnD,CAAC;aACD,QAAQ,EAAE;KACd,CAAC,CAAC;IAEI,MAAM,CAAC,KAAK,CAAC,OAAuB;QACzC,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,WAAW,CAAC,6BAA6B,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzG,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC;QACtB,OAAO,aAAa,CAAC,UAAU,CAAC;YAC9B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,IAAI,EAAE,CAAC,CAAC,QAAQ;YAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI;YAC1B,aAAa,EAAE,CAAC,CAAC,aAAa,IAAI,IAAI;YACtC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;SACzB,CAAC,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,UAAU,CAAI,KAAQ;QACnC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;gBAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC"}

package/dist/src/config/config-loader.d.ts

@@ -0,0 +1,10 @@
+import type { ScrapmeConfig } from './config.models.js';
+export declare class ConfigError extends Error {
+    constructor(message: string);
+}
+/** The only place that reads environment variables. Empty strings count as "not set" (default applies). */
+export declare class ConfigLoader {
+    static load(env?: Record<string, string | undefined>): ScrapmeConfig;
+    private static toConfig;
+    private static deepFreeze;
+}

package/dist/src/config/config-loader.js

@@ -0,0 +1,79 @@
+import { z } from 'zod';
+import { ConfigSchema } from './config.schema.js';
+import { Urls } from '../util/url.js';
+export class ConfigError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ConfigError';
+    }
+}
+/** The only place that reads environment variables. Empty strings count as "not set" (default applies). */
+export class ConfigLoader {
+    static load(env = process.env) {
+        const cleaned = {};
+        for (const key of Object.keys(ConfigSchema.env.shape)) {
+            const value = env[key];
+            if (value !== undefined && value.trim() !== '')
+                cleaned[key] = value;
+        }
+        const parsed = ConfigSchema.env.safeParse(cleaned);
+        if (!parsed.success)
+            throw new ConfigError(`Invalid scrapme configuration:\n${z.prettifyError(parsed.error)}`);
+        return ConfigLoader.deepFreeze(ConfigLoader.toConfig(parsed.data));
+    }
+    static toConfig(e) {
+        return {
+            logLevel: e.LOG_LEVEL,
+            scrapAnything: {
+                url: e.SCRAP_ANYTHING_URL,
+                secret: e.SCRAP_ANYTHING_SECRET,
+                requestTimeoutMs: e.SCRAP_ANYTHING_REQUEST_TIMEOUT_MS,
+                pollIntervalMs: e.SCRAP_ANYTHING_POLL_INTERVAL_MS,
+                waitTimeoutMs: e.SCRAP_ANYTHING_WAIT_TIMEOUT_MS,
+            },
+            scrape: { stepTimeoutMs: e.SCRAPE_STEP_TIMEOUT_MS, stepRetries: e.SCRAPE_STEP_RETRIES },
+            couch: {
+                url: Urls.withoutTrailingSlash(e.COUCHDB_URL),
+                publicUrl: Urls.withoutTrailingSlash(e.COUCHDB_PUBLIC_URL || e.COUCHDB_URL),
+                user: e.COUCHDB_USER,
+                password: e.COUCHDB_PASSWORD,
+                db: e.COUCHDB_DB,
+                runsDb: e.COUCHDB_RUNS_DB,
+                timeoutMs: e.COUCHDB_TIMEOUT_MS,
+                retryLimit: e.COUCHDB_RETRY_LIMIT,
+            },
+            smtp: {
+                host: e.SMTP_HOST,
+                port: e.SMTP_PORT,
+                secure: e.SMTP_SECURE,
+                user: e.SMTP_USER,
+                password: e.SMTP_PASSWORD,
+                timeoutMs: e.SMTP_TIMEOUT_MS,
+            },
+            mail: { from: e.MAIL_FROM, to: e.MAIL_TO, uiUrl: e.MAIL_UI_URL },
+            lighty: {
+                dsn: e.LIGHTY_DSN,
+                environment: e.LIGHTY_ENVIRONMENT,
+                release: e.LIGHTY_RELEASE,
+                flushTimeoutMs: e.LIGHTY_FLUSH_TIMEOUT_MS,
+            },
+            flow: {
+                retries: e.FLOW_RETRIES,
+                timeoutMs: e.FLOW_TIMEOUT_MS,
+                retryDelayMs: e.FLOW_RETRY_DELAY_MS,
+                maxRetryDelayMs: e.FLOW_RETRY_MAX_DELAY_MS,
+                saveRuns: e.FLOW_SAVE_RUNS,
+                maxStoredChars: e.FLOW_MAX_STORED_CHARS,
+            },
+        };
+    }
+    static deepFreeze(value) {
+        if (value && typeof value === 'object') {
+            for (const v of Object.values(value))
+                ConfigLoader.deepFreeze(v);
+            Object.freeze(value);
+        }
+        return value;
+    }
+}
+//# sourceMappingURL=config-loader.js.map

package/dist/src/config/config-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../../../src/config/config-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,YAAY,EAAkB,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,MAAM,OAAO,WAAY,SAAQ,KAAK;IACpC,YAAmB,OAAe;QAChC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED,2GAA2G;AAC3G,MAAM,OAAO,YAAY;IAChB,MAAM,CAAC,IAAI,CAAC,MAA0C,OAAO,CAAC,GAAG;QACtE,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACvE,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,WAAW,CAAC,mCAAmC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/G,OAAO,YAAY,CAAC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,CAAC;IAEO,MAAM,CAAC,QAAQ,CAAC,CAAY;QAClC,OAAO;YACL,QAAQ,EAAE,CAAC,CAAC,SAAS;YACrB,aAAa,EAAE;gBACb,GAAG,EAAE,CAAC,CAAC,kBAAkB;gBACzB,MAAM,EAAE,CAAC,CAAC,qBAAqB;gBAC/B,gBAAgB,EAAE,CAAC,CAAC,iCAAiC;gBACrD,cAAc,EAAE,CAAC,CAAC,+BAA+B;gBACjD,aAAa,EAAE,CAAC,CAAC,8BAA8B;aAChD;YACD,MAAM,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,sBAAsB,EAAE,WAAW,EAAE,CAAC,CAAC,mBAAmB,EAAE;YACvF,KAAK,EAAE;gBACL,GAAG,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC;gBAC7C,SAAS,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC,WAAW,CAAC;gBAC3E,IAAI,EAAE,CAAC,CAAC,YAAY;gBACpB,QAAQ,EAAE,CAAC,CAAC,gBAAgB;gBAC5B,EAAE,EAAE,CAAC,CAAC,UAAU;gBAChB,MAAM,EAAE,CAAC,CAAC,eAAe;gBACzB,SAAS,EAAE,CAAC,CAAC,kBAAkB;gBAC/B,UAAU,EAAE,CAAC,CAAC,mBAAmB;aAClC;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE,CAAC,CAAC,SAAS;gBACjB,IAAI,EAAE,CAAC,CAAC,SAAS;gBACjB,MAAM,EAAE,CAAC,CAAC,WAAW;gBACrB,IAAI,EAAE,CAAC,CAAC,SAAS;gBACjB,QAAQ,EAAE,CAAC,CAAC,aAAa;gBACzB,SAAS,EAAE,CAAC,CAAC,eAAe;aAC7B;YACD,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,WAAW,EAAE;YAChE,MAAM,EAAE;gBACN,GAAG,EAAE,CAAC,CAAC,UAAU;gBACjB,WAAW,EAAE,CAAC,CAAC,kBAAkB;gBACjC,OAAO,EAAE,CAAC,CAAC,cAAc;gBACzB,cAAc,EAAE,CAAC,CAAC,uBAAuB;aAC1C;YACD,IAAI,EAAE;gBACJ,OAAO,EAAE,CAAC,CAAC,YAAY;gBACvB,SAAS,EAAE,CAAC,CAAC,eAAe;gBAC5B,YAAY,EAAE,CAAC,CAAC,mBAAmB;gBACnC,eAAe,EAAE,CAAC,CAAC,uBAAuB;gBAC1C,QAAQ,EAAE,CAAC,CAAC,cAAc;gBAC1B,cAAc,EAAE,CAAC,CAAC,qBAAqB;aACxC;SACF,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,UAAU,CAAI,KAAQ;QACnC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;gBAAE,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/src/config/config.models.d.ts

@@ -0,0 +1,154 @@
+import type { LogLevel } from '../logging/logger.models.js';
+/** Postgres for the pg-boss flow queue (required). */
+export interface PostgresOptions {
+    /** e.g. postgres://user:password@localhost:5434/scrapme */
+    connectionString: string;
+    /** pg-boss tables live in this schema. Default "scrapme". */
+    schema?: string;
+}
+/**
+ * The pg-boss queues and their workers: one queue per flow (prefix + flow name), created when the flow is first
+ * scheduled. Each flow can override retries / delay / timeout (FlowConfig).
+ */
+export interface QueueOptions {
+    /** Queue names are prefix + flow name. Default "scrapme_". */
+    prefix?: string;
+    /** Flow attempts after the first one (each attempt runs the flow from its first step). Default 3. */
+    retries?: number;
+    /** Default 30. */
+    retryDelaySeconds?: number;
+    /** Default true. */
+    retryBackoff?: boolean;
+    /** One flow attempt may run this long (pg-boss expireInSeconds). Default 3 600. */
+    timeoutSeconds?: number;
+    /** Runs of one flow at the same time in this process. Default 1. */
+    concurrency?: number;
+    /** How often idle workers look for jobs, in seconds (min 0.5). Default 2. */
+    pollingIntervalSeconds?: number;
+    /** Finished jobs are removed from Postgres after this (the runs stay in CouchDB). Default 7. */
+    retentionDays?: number;
+    /** stop() waits this long for running flows. Default 30 000. */
+    shutdownTimeoutMs?: number;
+}
+/** The monitoring dashboard (Svelte UI + JSON API on /api). */
+export interface MonitorOptions {
+    /** e.g. 3100; 0 = any free port. */
+    port: number;
+    /** Default "127.0.0.1" (only this machine); "0.0.0.0" to open it to the network. */
+    host?: string;
+}
+/** scrap-anything — needed by ScrapePageCommand. */
+export interface ScrapAnythingOptions {
+    /** e.g. http://localhost:3000 */
+    url: string;
+    /** API_SECRET of scrap-anything (Bearer for POST /jobs). */
+    apiSecret: string;
+    /** One HTTP request to the API. Default 30 000. */
+    requestTimeoutMs?: number;
+    /** How often a waiting step asks for the job status. Default 2 000. */
+    pollIntervalMs?: number;
+    /** How long one ScrapePage attempt waits for its job (the server retries for up to ~7 min). Default 600 000. */
+    waitTimeoutMs?: number;
+    /** Timeout of one ScrapePage attempt; must be > waitTimeoutMs. Default waitTimeoutMs + 60 000. */
+    stepTimeoutMs?: number;
+    /** New server jobs after a failed one (the server already retried). Default 1. */
+    stepRetries?: number;
+}
+/** CouchDB — the run documents and the results of SaveDocumentCommand (required). */
+export interface CouchOptions {
+    /** e.g. http://localhost:5984 */
+    url: string;
+    user: string;
+    password: string;
+    /** Database of SaveDocumentCommand. Default "scrapme". */
+    db?: string;
+    /** Database of runs and failed attempts. Default "scrapme_runs". */
+    runsDb?: string;
+    /** CouchDB as seen from a browser (Fauxton links). Default url. */
+    publicUrl?: string;
+    /** Default 15 000. */
+    timeoutMs?: number;
+    /** Default 2. */
+    retryLimit?: number;
+}
+/** SMTP — needed by SendMailCommand. */
+export interface MailOptions {
+    host: string;
+    /** Default 587 with secure=false, 465 with secure=true. */
+    port?: number;
+    /** Default false. */
+    secure?: boolean;
+    user?: string;
+    password?: string;
+    /** Default 10 000. */
+    timeoutMs?: number;
+    /** e.g. "scrapme <scrapme@example.com>" */
+    from: string;
+    /** Default recipients of SendMailCommand. */
+    to?: string[];
+}
+/** LightyErrors (sentry-clone): a step that finally fails is reported there. */
+export interface LightyOptions {
+    /** http(s)://<project-key>@<host>/<project> */
+    dsn: string;
+    /** Default "production". */
+    environment?: string;
+    /** Default "scrapme". */
+    release?: string;
+    /** Default 5 000. */
+    flushTimeoutMs?: number;
+}
+/** Defaults of every step (a step or the command itself can override them). Command retries run inside a flow attempt. */
+export interface StepDefaults {
+    /** Attempts = retries + 1. Default 3. */
+    retries?: number;
+    /** One attempt. Default 60 000. */
+    timeoutMs?: number;
+    /** First retry delay, doubled per retry, with jitter. Default 1 000. */
+    retryDelayMs?: number;
+    /** Default 30 000. */
+    maxRetryDelayMs?: number;
+    /** Longer strings (HTML!) are cut in stored inputs/outputs. Default 2 000. */
+    maxStoredChars?: number;
+}
+/**
+ * Everything scrapme connects to, given by the app (scrapme reads no environment variables). `postgres` and `couch`
+ * are required: flows run on a pg-boss queue and their progress is kept in CouchDB. The rest is needed only by the
+ * commands that use it: no `scrapAnything` = ScrapePageCommand fails, no `mail` = SendMailCommand fails (both with
+ * NOT_CONFIGURED); no `lighty` = final errors are only logged.
+ */
+export interface ServiceOptions {
+    postgres: PostgresOptions;
+    couch: CouchOptions;
+    queue?: QueueOptions;
+    monitor?: MonitorOptions;
+    scrapAnything?: ScrapAnythingOptions;
+    mail?: MailOptions;
+    lighty?: LightyOptions;
+    defaults?: StepDefaults;
+    /** Default "info"; "silent" turns logging off. */
+    logLevel?: LogLevel;
+}
+export type ScrapAnythingConfig = Required<ScrapAnythingOptions>;
+export type CouchConfig = Required<CouchOptions>;
+export type MailConfig = Required<Omit<MailOptions, 'user' | 'password'>> & {
+    user: string | null;
+    password: string | null;
+};
+export type LightyConfig = Required<LightyOptions>;
+export type StepDefaultsConfig = Required<StepDefaults>;
+export type PostgresConfig = Required<PostgresOptions>;
+export type QueueConfig = Required<QueueOptions>;
+export type MonitorConfig = Required<MonitorOptions>;
+/** Validated, frozen configuration (ConfigBuilder). A missing optional service is null. */
+export interface ScrapmeConfig {
+    logLevel: LogLevel;
+    flow: StepDefaultsConfig;
+    postgres: PostgresConfig;
+    queue: QueueConfig;
+    monitor: MonitorConfig | null;
+    scrapAnything: ScrapAnythingConfig | null;
+    couch: CouchConfig;
+    mail: MailConfig | null;
+    lighty: LightyConfig | null;
+}

package/dist/src/config/config.models.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=config.models.js.map

package/dist/src/config/config.models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.models.js","sourceRoot":"","sources":["../../../src/config/config.models.ts"],"names":[],"mappings":""}

package/dist/src/config/config.schema.d.ts

@@ -0,0 +1,75 @@
+import { z } from 'zod';
+/** The zod schema of every setting, with its default. Documented in .env.example. */
+export declare class ConfigSchema {
+    /**
+     * CouchDB's own rule for database names (docs.couchdb.org, "PUT /{db}"). It is a rule about characters, so a
+     * regex is the clearest way to state it.
+     */
+    private static readonly COUCH_DB_NAME;
+    private static readonly TRUE_VALUES;
+    private static readonly FALSE_VALUES;
+    private static readonly LIST_SEPARATOR;
+    private static int;
+    private static str;
+    private static bool;
+    private static couchDbName;
+    private static list;
+    static readonly env: z.ZodObject<{
+        LOG_LEVEL: z.ZodDefault<z.ZodEnum<{
+            debug: "debug";
+            info: "info";
+            warn: "warn";
+            error: "error";
+            silent: "silent";
+        }>>;
+        SCRAP_ANYTHING_URL: z.ZodDefault<z.ZodURL>;
+        SCRAP_ANYTHING_SECRET: z.ZodDefault<z.ZodString>;
+        SCRAP_ANYTHING_REQUEST_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SCRAP_ANYTHING_POLL_INTERVAL_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SCRAP_ANYTHING_WAIT_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SCRAPE_STEP_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SCRAPE_STEP_RETRIES: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        COUCHDB_URL: z.ZodDefault<z.ZodURL>;
+        COUCHDB_PUBLIC_URL: z.ZodDefault<z.ZodString>;
+        COUCHDB_USER: z.ZodDefault<z.ZodString>;
+        COUCHDB_PASSWORD: z.ZodDefault<z.ZodString>;
+        COUCHDB_DB: z.ZodDefault<z.ZodString>;
+        COUCHDB_RUNS_DB: z.ZodDefault<z.ZodString>;
+        COUCHDB_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        COUCHDB_RETRY_LIMIT: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SMTP_HOST: z.ZodDefault<z.ZodString>;
+        SMTP_PORT: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        SMTP_SECURE: z.ZodPipe<z.ZodDefault<z.ZodEnum<{
+            0: "0";
+            1: "1";
+            true: "true";
+            yes: "yes";
+            false: "false";
+            no: "no";
+        }>>, z.ZodTransform<boolean, "0" | "1" | "true" | "yes" | "false" | "no">>;
+        SMTP_USER: z.ZodDefault<z.ZodString>;
+        SMTP_PASSWORD: z.ZodDefault<z.ZodString>;
+        SMTP_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        MAIL_FROM: z.ZodDefault<z.ZodString>;
+        MAIL_TO: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<string[], string>>;
+        MAIL_UI_URL: z.ZodDefault<z.ZodString>;
+        LIGHTY_DSN: z.ZodDefault<z.ZodString>;
+        LIGHTY_ENVIRONMENT: z.ZodDefault<z.ZodString>;
+        LIGHTY_RELEASE: z.ZodDefault<z.ZodString>;
+        LIGHTY_FLUSH_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_RETRIES: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_TIMEOUT_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_RETRY_DELAY_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_RETRY_MAX_DELAY_MS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+        FLOW_SAVE_RUNS: z.ZodPipe<z.ZodDefault<z.ZodEnum<{
+            0: "0";
+            1: "1";
+            true: "true";
+            yes: "yes";
+            false: "false";
+            no: "no";
+        }>>, z.ZodTransform<boolean, "0" | "1" | "true" | "yes" | "false" | "no">>;
+        FLOW_MAX_STORED_CHARS: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    }, z.core.$strip>;
+}
+export type ParsedEnv = z.infer<typeof ConfigSchema.env>;

package/dist/src/config/config.schema.js

@@ -0,0 +1,87 @@
+import { z } from 'zod';
+import { LOG_LEVEL, LOG_LEVELS } from '../logging/logger.models.js';
+import { DsnCheck } from './dsn-check.js';
+/** The zod schema of every setting, with its default. Documented in .env.example. */
+export class ConfigSchema {
+    /**
+     * CouchDB's own rule for database names (docs.couchdb.org, "PUT /{db}"). It is a rule about characters, so a
+     * regex is the clearest way to state it.
+     */
+    static COUCH_DB_NAME = /^[a-z][a-z0-9_$()+/-]*$/;
+    static TRUE_VALUES = ['true', '1', 'yes'];
+    static FALSE_VALUES = ['false', '0', 'no'];
+    static LIST_SEPARATOR = ',';
+    static int(def, min = 0) {
+        return z.coerce.number().int().min(min).default(def);
+    }
+    static str(def) {
+        return z.string().trim().default(def);
+    }
+    static bool(def) {
+        const C = ConfigSchema;
+        return z
+            .enum([...C.TRUE_VALUES, ...C.FALSE_VALUES])
+            .default(def ? C.TRUE_VALUES[0] : C.FALSE_VALUES[0])
+            .transform((v) => C.TRUE_VALUES.includes(v));
+    }
+    static couchDbName(def) {
+        return z.string().regex(ConfigSchema.COUCH_DB_NAME, 'CouchDB names: lowercase letters, digits and _$()+/-, starting with a letter').default(def);
+    }
+    static list(def) {
+        return z
+            .string()
+            .default(def)
+            .transform((v) => v.split(ConfigSchema.LIST_SEPARATOR).map((s) => s.trim()).filter(Boolean));
+    }
+    static env = z
+        .object({
+        LOG_LEVEL: z.enum(LOG_LEVELS).default(LOG_LEVEL.INFO),
+        SCRAP_ANYTHING_URL: z.url().default('http://localhost:3000'),
+        SCRAP_ANYTHING_SECRET: ConfigSchema.str(''),
+        SCRAP_ANYTHING_REQUEST_TIMEOUT_MS: ConfigSchema.int(30_000, 1),
+        SCRAP_ANYTHING_POLL_INTERVAL_MS: ConfigSchema.int(2_000, 1),
+        SCRAP_ANYTHING_WAIT_TIMEOUT_MS: ConfigSchema.int(600_000, 1),
+        SCRAPE_STEP_TIMEOUT_MS: ConfigSchema.int(660_000, 1),
+        SCRAPE_STEP_RETRIES: ConfigSchema.int(1),
+        COUCHDB_URL: z.url().default('http://localhost:5984'),
+        COUCHDB_PUBLIC_URL: ConfigSchema.str(''),
+        COUCHDB_USER: ConfigSchema.str('admin'),
+        COUCHDB_PASSWORD: ConfigSchema.str('admin'),
+        COUCHDB_DB: ConfigSchema.couchDbName('scrapme'),
+        COUCHDB_RUNS_DB: ConfigSchema.couchDbName('scrapme_runs'),
+        COUCHDB_TIMEOUT_MS: ConfigSchema.int(15_000, 1),
+        COUCHDB_RETRY_LIMIT: ConfigSchema.int(2),
+        SMTP_HOST: ConfigSchema.str('localhost'),
+        SMTP_PORT: ConfigSchema.int(1025, 1),
+        SMTP_SECURE: ConfigSchema.bool(false),
+        SMTP_USER: ConfigSchema.str(''),
+        SMTP_PASSWORD: ConfigSchema.str(''),
+        SMTP_TIMEOUT_MS: ConfigSchema.int(10_000, 1),
+        MAIL_FROM: ConfigSchema.str('scrapme <scrapme@localhost>'),
+        MAIL_TO: ConfigSchema.list('owner@localhost'),
+        MAIL_UI_URL: ConfigSchema.str('http://localhost:8025'),
+        LIGHTY_DSN: ConfigSchema.str(''),
+        LIGHTY_ENVIRONMENT: ConfigSchema.str('development'),
+        LIGHTY_RELEASE: ConfigSchema.str('scrapme@0.1.0'),
+        LIGHTY_FLUSH_TIMEOUT_MS: ConfigSchema.int(5_000, 1),
+        FLOW_RETRIES: ConfigSchema.int(2),
+        FLOW_TIMEOUT_MS: ConfigSchema.int(60_000, 1),
+        FLOW_RETRY_DELAY_MS: ConfigSchema.int(1_000),
+        FLOW_RETRY_MAX_DELAY_MS: ConfigSchema.int(30_000),
+        FLOW_SAVE_RUNS: ConfigSchema.bool(true),
+        FLOW_MAX_STORED_CHARS: ConfigSchema.int(2_000, 100),
+    })
+        .refine((e) => e.SCRAPE_STEP_TIMEOUT_MS > e.SCRAP_ANYTHING_WAIT_TIMEOUT_MS, {
+        message: 'SCRAPE_STEP_TIMEOUT_MS must be greater than SCRAP_ANYTHING_WAIT_TIMEOUT_MS (the step waits for the server job)',
+        path: ['SCRAPE_STEP_TIMEOUT_MS'],
+    })
+        .refine((e) => e.FLOW_RETRY_MAX_DELAY_MS >= e.FLOW_RETRY_DELAY_MS, {
+        message: 'FLOW_RETRY_MAX_DELAY_MS must be >= FLOW_RETRY_DELAY_MS',
+        path: ['FLOW_RETRY_MAX_DELAY_MS'],
+    })
+        .refine((e) => !e.LIGHTY_DSN || DsnCheck.isValid(e.LIGHTY_DSN), {
+        message: 'LIGHTY_DSN must look like https://<project-key>@<host>/<project>',
+        path: ['LIGHTY_DSN'],
+    });
+}
+//# sourceMappingURL=config.schema.js.map

package/dist/src/config/config.schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.schema.js","sourceRoot":"","sources":["../../../src/config/config.schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,qFAAqF;AACrF,MAAM,OAAO,YAAY;IACvB;;;OAGG;IACK,MAAM,CAAU,aAAa,GAAG,yBAAyB,CAAC;IAC1D,MAAM,CAAU,WAAW,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAU,CAAC;IAC5D,MAAM,CAAU,YAAY,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAU,CAAC;IAC7D,MAAM,CAAU,cAAc,GAAG,GAAG,CAAC;IAErC,MAAM,CAAC,GAAG,CAAC,GAAW,EAAE,GAAG,GAAG,CAAC;QACrC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACvD,CAAC;IAEO,MAAM,CAAC,GAAG,CAAC,GAAW;QAC5B,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAEO,MAAM,CAAC,IAAI,CAAC,GAAY;QAC9B,MAAM,CAAC,GAAG,YAAY,CAAC;QACvB,OAAO,CAAC;aACL,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;aAC3C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;aACnD,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,WAAiC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;IAEO,MAAM,CAAC,WAAW,CAAC,GAAW;QACpC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,aAAa,EAAE,8EAA8E,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnJ,CAAC;IAEO,MAAM,CAAC,IAAI,CAAC,GAAW;QAC7B,OAAO,CAAC;aACL,MAAM,EAAE;aACR,OAAO,CAAC,GAAG,CAAC;aACZ,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IACjG,CAAC;IAEM,MAAM,CAAU,GAAG,GAAG,CAAC;SAC3B,MAAM,CAAC;QACN,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;QAErD,kBAAkB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,uBAAuB,CAAC;QAC5D,qBAAqB,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,iCAAiC,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,+BAA+B,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3D,8BAA8B,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5D,sBAAsB,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACpD,mBAAmB,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAExC,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,uBAAuB,CAAC;QACrD,kBAAkB,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACxC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC;QACvC,gBAAgB,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC;QAC3C,UAAU,EAAE,YAAY,CAAC,WAAW,CAAC,SAAS,CAAC;QAC/C,eAAe,EAAE,YAAY,CAAC,WAAW,CAAC,cAAc,CAAC;QACzD,kBAAkB,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,mBAAmB,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAExC,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC;QACxC,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,WAAW,EAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;QACrC,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,aAAa,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,eAAe,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5C,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,6BAA6B,CAAC;QAC1D,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC;QAC7C,WAAW,EAAE,YAAY,CAAC,GAAG,CAAC,uBAAuB,CAAC;QAEtD,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QAChC,kBAAkB,EAAE,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC;QACnD,cAAc,EAAE,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC;QACjD,uBAAuB,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAEnD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QACjC,eAAe,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5C,mBAAmB,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;QAC5C,uBAAuB,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC;QACjD,cAAc,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;QACvC,qBAAqB,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;KACpD,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB,GAAG,CAAC,CAAC,8BAA8B,EAAE;QAC1E,OAAO,EAAE,gHAAgH;QACzH,IAAI,EAAE,CAAC,wBAAwB,CAAC;KACjC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,uBAAuB,IAAI,CAAC,CAAC,mBAAmB,EAAE;QACjE,OAAO,EAAE,wDAAwD;QACjE,IAAI,EAAE,CAAC,yBAAyB,CAAC;KAClC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE;QAC9D,OAAO,EAAE,kEAAkE;QAC3E,IAAI,EAAE,CAAC,YAAY,CAAC;KACrB,CAAC,CAAC"}

package/dist/src/config/dsn-check.d.ts

@@ -0,0 +1,5 @@
+/** LightyErrors DSN: `http(s)://<project-key>@<host>[:port]/<project>` — checked with URL, not a pattern. */
+export declare class DsnCheck {
+    private static readonly PROTOCOLS;
+    static isValid(dsn: string): boolean;
+}

package/dist/src/config/dsn-check.js

@@ -0,0 +1,12 @@
+/** LightyErrors DSN: `http(s)://<project-key>@<host>[:port]/<project>` — checked with URL, not a pattern. */
+export class DsnCheck {
+    static PROTOCOLS = new Set(['http:', 'https:']);
+    static isValid(dsn) {
+        if (!URL.canParse(dsn))
+            return false;
+        const url = new URL(dsn);
+        const project = url.pathname.split('/').filter(Boolean);
+        return DsnCheck.PROTOCOLS.has(url.protocol) && url.username !== '' && url.password === '' && project.length > 0;
+    }
+}
+//# sourceMappingURL=dsn-check.js.map

package/dist/src/config/dsn-check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dsn-check.js","sourceRoot":"","sources":["../../../src/config/dsn-check.ts"],"names":[],"mappings":"AAAA,6GAA6G;AAC7G,MAAM,OAAO,QAAQ;IACX,MAAM,CAAU,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAE1D,MAAM,CAAC,OAAO,CAAC,GAAW;QAC/B,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACxD,OAAO,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAClH,CAAC"}

package/dist/src/di/container.d.ts

@@ -0,0 +1,13 @@
+import { Container } from 'inversify';
+import type { ScrapmeConfig } from '../config/config.models.js';
+/**
+ * Composition root of scrapme's own services (singletons). The container's parent is the app's container (when
+ * given): `autobind` resolves any `@injectable()` command class by its class, and its `@inject(...)`s find the
+ * app's services through the parent — commands need no bindings. A missing optional service (mail, scrapAnything)
+ * gets an "unconfigured" stand-in, so only the command that needs it fails (NOT_CONFIGURED). Replace a scrapme
+ * service with `scrapme.container.rebindSync(TYPES.X)` before start() (tests use this for the queue and stores).
+ */
+export declare class ContainerBuilder {
+    private static readonly APP;
+    static build(config: ScrapmeConfig, parent?: Container): Container;
+}