scrapme-flow 0.0.0-stage → 0.2.0

package/dist/src/monitor/monitor-api.js

@@ -0,0 +1,62 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var MonitorApi_1;
+import { inject, injectable } from 'inversify';
+import { TYPES } from '../di/types.js';
+import { RUN_STATUS } from '../flow/flow.models.js';
+/** The data behind the dashboard: the run documents in CouchDB (the monitor only reads them). */
+let MonitorApi = class MonitorApi {
+    static { MonitorApi_1 = this; }
+    store;
+    couch;
+    static MAX_RUNS = 500;
+    constructor(store, couch) {
+        this.store = store;
+        this.couch = couch;
+    }
+    async runs(limit, filter = {}) {
+        const docs = await this.store.list(Math.min(Math.max(1, limit), MonitorApi_1.MAX_RUNS));
+        return docs.filter((d) => (!filter.flow || d.flow === filter.flow) && (!filter.status || d.status === filter.status)).map((d) => MonitorApi_1.summary(d));
+    }
+    async run(runId) {
+        return this.store.get(runId);
+    }
+    /** Counts per status and the flow names, over the newest MAX_RUNS runs. */
+    async overview() {
+        const docs = await this.store.list(MonitorApi_1.MAX_RUNS);
+        const counts = Object.fromEntries(Object.values(RUN_STATUS).map((s) => [s, 0]));
+        for (const d of docs)
+            counts[d.status] = (counts[d.status] ?? 0) + 1;
+        const flows = [...new Set(docs.map((d) => d.flow))].sort();
+        return { flows, counts, couchUrl: this.couch.publicUrl, runsDb: this.couch.runsDb };
+    }
+    static summary(d) {
+        return {
+            runId: d.runId,
+            flow: d.flow,
+            status: d.status,
+            currentStep: d.currentStep,
+            flowAttempt: d.flowAttempts.at(-1)?.attempt ?? 0,
+            maxFlowAttempts: d.maxFlowAttempts,
+            createdAt: d.createdAt,
+            finishedAt: d.finishedAt ?? null,
+            durationMs: d.durationMs ?? null,
+            error: d.error ? d.error.message : null,
+            steps: d.steps.map((s) => ({ key: s.key, status: s.status, attempts: s.attempts.length })),
+        };
+    }
+};
+MonitorApi = MonitorApi_1 = __decorate([
+    injectable(),
+    __param(0, inject(TYPES.RunStore)),
+    __param(1, inject(TYPES.CouchConfig))
+], MonitorApi);
+export { MonitorApi };
+//# sourceMappingURL=monitor-api.js.map

package/dist/src/monitor/monitor-api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"monitor-api.js","sourceRoot":"","sources":["../../../src/monitor/monitor-api.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,UAAU,EAAkB,MAAM,wBAAwB,CAAC;AAIpE,iGAAiG;AAE1F,IAAM,UAAU,GAAhB,MAAM,UAAU;;IAIsB;IACG;IAJvC,MAAM,CAAU,QAAQ,GAAG,GAAG,CAAC;IAEtC,YAC2C,KAAe,EACZ,KAAkB;QADrB,UAAK,GAAL,KAAK,CAAU;QACZ,UAAK,GAAL,KAAK,CAAa;IAC7D,CAAC;IAEG,KAAK,CAAC,IAAI,CAAC,KAAa,EAAE,SAA6C,EAAE;QAC9E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,YAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtF,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1J,CAAC;IAEM,KAAK,CAAC,GAAG,CAAC,KAAa;QAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,2EAA2E;IACpE,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAU,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAA8B,CAAC;QAC7G,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IACtF,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,CAAa;QACjC,OAAO;YACL,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,WAAW,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC;YAChD,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,IAAI;YAChC,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,IAAI;YAChC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;YACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;SAC3F,CAAC;IACJ,CAAC;;AAxCU,UAAU;IADtB,UAAU,EAAE;IAKR,WAAA,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IACtB,WAAA,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;GALjB,UAAU,CAyCtB"}

package/dist/src/monitor/monitor-server.d.ts

@@ -0,0 +1,33 @@
+import type { MonitorConfig } from '../config/config.models.js';
+import type { Logger } from '../logging/logger.js';
+import type { MonitorApi } from './monitor-api.js';
+/**
+ * The monitoring dashboard: the built Svelte app (dashboard/dist) and its JSON API, on node:http.
+ *   GET /api/overview · /api/runs?limit=&flow=&status= · /api/runs/:runId
+ * Read-only. Binds to 127.0.0.1 unless monitor.host says otherwise.
+ */
+export declare class MonitorServer {
+    private readonly config;
+    private readonly api;
+    private static readonly API;
+    private static readonly DEFAULT_LIMIT;
+    private static readonly INDEX;
+    private static readonly TYPES;
+    /** Where the built dashboard is: next to the sources (tsx) or next to dist/ (the published package). */
+    private static readonly ASSET_DIRS;
+    private server;
+    private assets;
+    private readonly logger;
+    constructor(config: MonitorConfig, api: MonitorApi, logger: Logger);
+    /** Returns the dashboard URL. */
+    start(): Promise<string>;
+    get url(): string;
+    stop(): Promise<void>;
+    private handle;
+    private handleApi;
+    /** Files of the built dashboard; anything else → index.html (the app routes itself). */
+    private serveAsset;
+    private static findAssets;
+    private static isFile;
+    private static json;
+}

package/dist/src/monitor/monitor-server.js

@@ -0,0 +1,142 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var MonitorServer_1;
+import { createReadStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
+import { createServer } from 'node:http';
+import { extname, join, normalize, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { inject, injectable } from 'inversify';
+import { TYPES } from '../di/types.js';
+/**
+ * The monitoring dashboard: the built Svelte app (dashboard/dist) and its JSON API, on node:http.
+ *   GET /api/overview · /api/runs?limit=&flow=&status= · /api/runs/:runId
+ * Read-only. Binds to 127.0.0.1 unless monitor.host says otherwise.
+ */
+let MonitorServer = class MonitorServer {
+    static { MonitorServer_1 = this; }
+    config;
+    api;
+    static API = 'api';
+    static DEFAULT_LIMIT = 100;
+    static INDEX = 'index.html';
+    static TYPES = {
+        '.html': 'text/html; charset=utf-8',
+        '.js': 'text/javascript; charset=utf-8',
+        '.css': 'text/css; charset=utf-8',
+        '.svg': 'image/svg+xml',
+        '.json': 'application/json',
+        '.ico': 'image/x-icon',
+    };
+    /** Where the built dashboard is: next to the sources (tsx) or next to dist/ (the published package). */
+    static ASSET_DIRS = [new URL('../../dashboard/dist/', import.meta.url), new URL('../../../dashboard/dist/', import.meta.url)];
+    server = null;
+    assets = null;
+    logger;
+    constructor(config, api, logger) {
+        this.config = config;
+        this.api = api;
+        this.logger = logger.child({ component: 'monitor' });
+    }
+    /** Returns the dashboard URL. */
+    async start() {
+        this.assets = await MonitorServer_1.findAssets();
+        const server = createServer((req, res) => void this.handle(req, res));
+        await new Promise((resolve, reject) => {
+            server.once('error', reject);
+            server.listen(this.config.port, this.config.host, () => resolve());
+        });
+        this.server = server;
+        const url = this.url;
+        this.logger.info('monitor ready', { url, dashboard: this.assets ? 'built' : 'not built - run npm run build:dashboard' });
+        return url;
+    }
+    get url() {
+        const address = this.server?.address();
+        return address ? `http://${this.config.host}:${address.port}` : '';
+    }
+    async stop() {
+        const server = this.server;
+        this.server = null;
+        if (server)
+            await new Promise((resolve) => server.close(() => resolve()));
+    }
+    async handle(req, res) {
+        const url = new URL(req.url ?? '/', 'http://monitor');
+        const [first, resource, id] = url.pathname.split('/').filter(Boolean);
+        try {
+            if (req.method !== 'GET')
+                return MonitorServer_1.json(res, 405, { error: 'read-only' });
+            if (first === MonitorServer_1.API)
+                return await this.handleApi(res, resource, id, url.searchParams);
+            return await this.serveAsset(res, url.pathname);
+        }
+        catch (error) {
+            this.logger.error('monitor request failed', { path: url.pathname, error });
+            MonitorServer_1.json(res, 500, { error: error.message });
+        }
+    }
+    async handleApi(res, resource, id, query) {
+        switch (resource) {
+            case 'overview':
+                return MonitorServer_1.json(res, 200, await this.api.overview());
+            case 'runs': {
+                if (id) {
+                    const run = await this.api.run(decodeURIComponent(id));
+                    return run ? MonitorServer_1.json(res, 200, run) : MonitorServer_1.json(res, 404, { error: 'run not found' });
+                }
+                const limit = Number(query.get('limit') ?? MonitorServer_1.DEFAULT_LIMIT);
+                const flow = query.get('flow') ?? undefined;
+                const status = query.get('status') ?? undefined;
+                return MonitorServer_1.json(res, 200, await this.api.runs(Number.isFinite(limit) ? limit : MonitorServer_1.DEFAULT_LIMIT, { ...(flow ? { flow } : {}), ...(status ? { status } : {}) }));
+            }
+            default:
+                return MonitorServer_1.json(res, 404, { error: 'unknown endpoint' });
+        }
+    }
+    /** Files of the built dashboard; anything else → index.html (the app routes itself). */
+    async serveAsset(res, path) {
+        if (!this.assets) {
+            res.writeHead(200, { 'content-type': MonitorServer_1.TYPES['.html'] }).end('<h1>scrapme monitor</h1><p>The dashboard is not built: run <code>npm run build:dashboard</code> in scrapme. The API works: <a href="/api/runs">/api/runs</a>.</p>');
+            return;
+        }
+        const wanted = normalize(join(this.assets, decodeURIComponent(path)));
+        const inside = wanted.startsWith(this.assets + sep) || wanted === this.assets;
+        const file = inside && (await MonitorServer_1.isFile(wanted)) ? wanted : join(this.assets, MonitorServer_1.INDEX);
+        res.writeHead(200, { 'content-type': MonitorServer_1.TYPES[extname(file)] ?? 'application/octet-stream' });
+        createReadStream(file).pipe(res);
+    }
+    static async findAssets() {
+        for (const candidate of MonitorServer_1.ASSET_DIRS) {
+            const path = fileURLToPath(candidate);
+            if (!(await MonitorServer_1.isFile(join(path, MonitorServer_1.INDEX))))
+                continue;
+            let dir = normalize(path);
+            while (dir.endsWith(sep))
+                dir = dir.slice(0, -1);
+            return dir;
+        }
+        return null;
+    }
+    static async isFile(path) {
+        return stat(path).then((s) => s.isFile()).catch(() => false);
+    }
+    static json(res, status, body) {
+        res.writeHead(status, { 'content-type': MonitorServer_1.TYPES['.json'], 'cache-control': 'no-store' }).end(JSON.stringify(body));
+    }
+};
+MonitorServer = MonitorServer_1 = __decorate([
+    injectable(),
+    __param(0, inject(TYPES.MonitorConfig)),
+    __param(1, inject(TYPES.MonitorApi)),
+    __param(2, inject(TYPES.Logger))
+], MonitorServer);
+export { MonitorServer };
+//# sourceMappingURL=monitor-server.js.map

package/dist/src/monitor/monitor-server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"monitor-server.js","sourceRoot":"","sources":["../../../src/monitor/monitor-server.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,YAAY,EAA0D,MAAM,WAAW,CAAC;AAEjG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC;;;;GAIG;AAEI,IAAM,aAAa,GAAnB,MAAM,aAAa;;IAoBwB;IACH;IApBrC,MAAM,CAAU,GAAG,GAAG,KAAK,CAAC;IAC5B,MAAM,CAAU,aAAa,GAAG,GAAG,CAAC;IACpC,MAAM,CAAU,KAAK,GAAG,YAAY,CAAC;IACrC,MAAM,CAAU,KAAK,GAAqC;QAChE,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,gCAAgC;QACvC,MAAM,EAAE,yBAAyB;QACjC,MAAM,EAAE,eAAe;QACvB,OAAO,EAAE,kBAAkB;QAC3B,MAAM,EAAE,cAAc;KACvB,CAAC;IACF,wGAAwG;IAChG,MAAM,CAAU,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC,uBAAuB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,0BAA0B,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEvI,MAAM,GAAkB,IAAI,CAAC;IAC7B,MAAM,GAAkB,IAAI,CAAC;IACpB,MAAM,CAAS;IAEhC,YACgD,MAAqB,EACxB,GAAe,EACpC,MAAc;QAFU,WAAM,GAAN,MAAM,CAAe;QACxB,QAAG,GAAH,GAAG,CAAY;QAG1D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,iCAAiC;IAC1B,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,MAAM,GAAG,MAAM,eAAa,CAAC,UAAU,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QACtE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yCAAyC,EAAE,CAAC,CAAC;QACzH,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAW,GAAG;QACZ,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,OAAO,EAAoC,CAAC;QACzE,OAAO,OAAO,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,MAAM;YAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAClF,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,GAAoB,EAAE,GAAmB;QAC5D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,gBAAgB,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK;gBAAE,OAAO,eAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;YACtF,IAAI,KAAK,KAAK,eAAa,CAAC,GAAG;gBAAE,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;YAClG,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3E,eAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAmB,EAAE,QAA4B,EAAE,EAAsB,EAAE,KAAsB;QACvH,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,UAAU;gBACb,OAAO,eAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,IAAI,EAAE,EAAE,CAAC;oBACP,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvD,OAAO,GAAG,CAAC,CAAC,CAAC,eAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,eAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;gBAC5G,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,eAAa,CAAC,aAAa,CAAC,CAAC;gBACxE,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;gBAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC;gBAChD,OAAO,eAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAa,CAAC,aAAa,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACvL,CAAC;YACD;gBACE,OAAO,eAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,wFAAwF;IAChF,KAAK,CAAC,UAAU,CAAC,GAAmB,EAAE,IAAY;QACxD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,eAAa,CAAC,KAAK,CAAC,OAAO,CAAE,EAAE,CAAC,CAAC,GAAG,CAAC,mKAAmK,CAAC,CAAC;YAC/O,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;QAC9E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,eAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAa,CAAC,KAAK,CAAC,CAAC;QAC9G,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,eAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,0BAA0B,EAAE,CAAC,CAAC;QACzG,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,UAAU;QAC7B,KAAK,MAAM,SAAS,IAAI,eAAa,CAAC,UAAU,EAAE,CAAC;YACjD,MAAM,IAAI,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,CAAC,CAAC,MAAM,eAAa,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,eAAa,CAAC,KAAK,CAAC,CAAC,CAAC;gBAAE,SAAS;YAC7E,IAAI,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAC1B,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACjD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAY;QACtC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,CAAC;IAEO,MAAM,CAAC,IAAI,CAAC,GAAmB,EAAE,MAAc,EAAE,IAAa;QACpE,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,eAAa,CAAC,KAAK,CAAC,OAAO,CAAE,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;IAClI,CAAC;;AAlHU,aAAa;IADzB,UAAU,EAAE;IAqBR,WAAA,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAC3B,WAAA,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;IACxB,WAAA,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;GAtBZ,aAAa,CAmHzB"}

package/dist/src/monitor/monitor.models.d.ts

@@ -0,0 +1,27 @@
+import type { RunStatus, StepStatus } from '../flow/flow.models.js';
+/** One row of the runs list (GET /api/runs). */
+export interface RunSummary {
+    runId: string;
+    flow: string;
+    status: RunStatus;
+    currentStep: string | null;
+    flowAttempt: number;
+    maxFlowAttempts: number;
+    createdAt: string;
+    finishedAt: string | null;
+    durationMs: number | null;
+    error: string | null;
+    steps: {
+        key: string;
+        status: StepStatus;
+        attempts: number;
+    }[];
+}
+/** GET /api/overview */
+export interface Overview {
+    /** Names of the flows that have runs. */
+    flows: string[];
+    counts: Record<RunStatus, number>;
+    couchUrl: string;
+    runsDb: string;
+}

package/dist/src/monitor/monitor.models.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=monitor.models.js.map

package/dist/src/monitor/monitor.models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"monitor.models.js","sourceRoot":"","sources":["../../../src/monitor/monitor.models.ts"],"names":[],"mappings":""}

package/dist/src/queue/pg-boss-flow-queue.d.ts

@@ -0,0 +1,22 @@
+import type { PostgresConfig, QueueConfig } from '../config/config.models.js';
+import type { Logger } from '../logging/logger.js';
+import type { FlowJobData, FlowJobHandler, FlowJobOptions, FlowQueue } from './queue.models.js';
+/**
+ * Flow jobs on pg-boss. pg-boss retries a failed attempt after retryDelay (with backoff) and expires an attempt
+ * after expireInSeconds; scrapme aborts the job's signal at the same moment, so the running command stops.
+ */
+export declare class PgBossFlowQueue implements FlowQueue {
+    private readonly postgres;
+    private readonly queue;
+    private static readonly MS;
+    private boss;
+    private readonly workers;
+    private readonly running;
+    private readonly logger;
+    constructor(postgres: PostgresConfig, queue: QueueConfig, logger: Logger);
+    start(): Promise<void>;
+    work(queue: string, concurrency: number, handler: FlowJobHandler): Promise<void>;
+    schedule(queue: string, data: FlowJobData, o: FlowJobOptions): Promise<string>;
+    stop(): Promise<void>;
+    private requireBoss;
+}

package/dist/src/queue/pg-boss-flow-queue.js

@@ -0,0 +1,102 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var PgBossFlowQueue_1;
+import { inject, injectable } from 'inversify';
+import PgBoss from 'pg-boss';
+import { TYPES } from '../di/types.js';
+/**
+ * Flow jobs on pg-boss. pg-boss retries a failed attempt after retryDelay (with backoff) and expires an attempt
+ * after expireInSeconds; scrapme aborts the job's signal at the same moment, so the running command stops.
+ */
+let PgBossFlowQueue = class PgBossFlowQueue {
+    static { PgBossFlowQueue_1 = this; }
+    postgres;
+    queue;
+    static MS = 1_000;
+    boss = null;
+    workers = [];
+    running = new Set();
+    logger;
+    constructor(postgres, queue, logger) {
+        this.postgres = postgres;
+        this.queue = queue;
+        this.logger = logger.child({ component: 'pg-boss' });
+    }
+    async start() {
+        if (this.boss)
+            return;
+        const boss = new PgBoss({ connectionString: this.postgres.connectionString, schema: this.postgres.schema, deleteAfterDays: this.queue.retentionDays });
+        boss.on('error', (error) => this.logger.error('pg-boss error', { error }));
+        await boss.start();
+        this.boss = boss;
+        this.logger.debug('connected', { schema: this.postgres.schema });
+    }
+    async work(queue, concurrency, handler) {
+        const boss = this.requireBoss();
+        if (!(await boss.getQueue(queue)))
+            await boss.createQueue(queue);
+        for (let i = 0; i < concurrency; i++) {
+            const id = await boss.work(queue, { batchSize: 1, pollingIntervalSeconds: this.queue.pollingIntervalSeconds, includeMetadata: true }, async ([job]) => {
+                if (!job)
+                    return;
+                // pg-boss expires the attempt after expireInSeconds; stop the commands at the same moment
+                const controller = new AbortController();
+                const timer = setTimeout(() => controller.abort(new Error('flow attempt expired after ' + job.expireInSeconds + ' s')), job.expireInSeconds * PgBossFlowQueue_1.MS);
+                this.running.add(controller);
+                try {
+                    await handler({ id: job.id, data: job.data, retryCount: job.retryCount, retryLimit: job.retryLimit, signal: controller.signal });
+                }
+                finally {
+                    clearTimeout(timer);
+                    this.running.delete(controller);
+                }
+            });
+            this.workers.push(id);
+        }
+    }
+    async schedule(queue, data, o) {
+        const id = await this.requireBoss().send(queue, data, {
+            id: o.id,
+            retryLimit: o.retryLimit,
+            retryDelay: o.retryDelaySeconds,
+            retryBackoff: o.retryBackoff,
+            expireInSeconds: o.expireInSeconds,
+            ...(o.startAfter ? { startAfter: o.startAfter } : {}),
+            ...(o.priority !== undefined ? { priority: o.priority } : {}),
+        });
+        if (!id)
+            throw new Error('run ' + o.id + ' is already queued');
+        return id;
+    }
+    async stop() {
+        const boss = this.boss;
+        if (!boss)
+            return;
+        this.boss = null;
+        await Promise.all(this.workers.map((id) => boss.offWork({ id }).catch(() => undefined)));
+        this.workers.length = 0;
+        for (const controller of this.running)
+            controller.abort(new Error('scrapme is stopping'));
+        await boss.stop({ graceful: true, timeout: this.queue.shutdownTimeoutMs });
+    }
+    requireBoss() {
+        if (!this.boss)
+            throw new Error('scrapme is not started - call await scrapme.start() first');
+        return this.boss;
+    }
+};
+PgBossFlowQueue = PgBossFlowQueue_1 = __decorate([
+    injectable(),
+    __param(0, inject(TYPES.PostgresConfig)),
+    __param(1, inject(TYPES.QueueConfig)),
+    __param(2, inject(TYPES.Logger))
+], PgBossFlowQueue);
+export { PgBossFlowQueue };
+//# sourceMappingURL=pg-boss-flow-queue.js.map

package/dist/src/queue/pg-boss-flow-queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pg-boss-flow-queue.js","sourceRoot":"","sources":["../../../src/queue/pg-boss-flow-queue.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC;;;GAGG;AAEI,IAAM,eAAe,GAArB,MAAM,eAAe;;IAQuB;IACH;IARtC,MAAM,CAAU,EAAE,GAAG,KAAK,CAAC;IAC3B,IAAI,GAAkB,IAAI,CAAC;IAClB,OAAO,GAAa,EAAE,CAAC;IACvB,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IACrC,MAAM,CAAS;IAEhC,YACiD,QAAwB,EAC3B,KAAkB,EACxC,MAAc;QAFW,aAAQ,GAAR,QAAQ,CAAgB;QAC3B,UAAK,GAAL,KAAK,CAAa;QAG9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACvD,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;QACtB,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;QACvJ,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAC3E,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,KAAa,EAAE,WAAmB,EAAE,OAAuB;QAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAc,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,sBAAsB,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE;gBACjK,IAAI,CAAC,GAAG;oBAAE,OAAO;gBACjB,0FAA0F;gBAC1F,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,6BAA6B,GAAG,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,eAAe,GAAG,iBAAe,CAAC,EAAE,CAAC,CAAC;gBAClK,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,OAAO,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;gBACnI,CAAC;wBAAS,CAAC;oBACT,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,IAAiB,EAAE,CAAiB;QACvE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE;YACpD,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,UAAU,EAAE,CAAC,CAAC,iBAAiB;YAC/B,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9D,CAAC,CAAC;QACH,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,GAAG,oBAAoB,CAAC,CAAC;QAC/D,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,OAAO;YAAE,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC1F,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAC7E,CAAC;IAEO,WAAW;QACjB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC7F,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;;AAxEU,eAAe;IAD3B,UAAU,EAAE;IASR,WAAA,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAC5B,WAAA,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;IACzB,WAAA,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;GAVZ,eAAe,CAyE3B"}

package/dist/src/queue/queue.models.d.ts

@@ -0,0 +1,38 @@
+/** What a flow job carries (pg-boss job data). The run itself lives in CouchDB, not in the job. */
+export interface FlowJobData {
+    runId: string;
+}
+/** One execution of a flow job, as the worker sees it. */
+export interface FlowJob {
+    id: string;
+    data: FlowJobData;
+    /** 0 on the first attempt (pg-boss retryCount). */
+    retryCount: number;
+    /** pg-boss retryLimit: attempts = retryLimit + 1. */
+    retryLimit: number;
+    /** Aborted when the attempt has run for its timeout (pg-boss expires it then) or the worker stops. */
+    signal: AbortSignal;
+}
+/** How one run is sent to its flow's queue. */
+export interface FlowJobOptions {
+    /** pg-boss job id = the run id, so one run is one job. */
+    id: string;
+    retryLimit: number;
+    retryDelaySeconds: number;
+    retryBackoff: boolean;
+    expireInSeconds: number;
+    startAfter?: Date;
+    priority?: number;
+}
+export type FlowJobHandler = (job: FlowJob) => Promise<void>;
+/**
+ * The flow queues: pg-boss (PgBossFlowQueue), an in-memory fake in tests. One queue per flow; retries, delays and
+ * expiry of the jobs are pg-boss's — scrapme only says how many and how long.
+ */
+export interface FlowQueue {
+    start(): Promise<void>;
+    /** Create the queue (no-op when it exists) and start `concurrency` workers on it in this process. */
+    work(queue: string, concurrency: number, handler: FlowJobHandler): Promise<void>;
+    schedule(queue: string, data: FlowJobData, options: FlowJobOptions): Promise<string>;
+    stop(): Promise<void>;
+}

package/dist/src/queue/queue.models.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=queue.models.js.map

package/dist/src/queue/queue.models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.models.js","sourceRoot":"","sources":["../../../src/queue/queue.models.ts"],"names":[],"mappings":""}

package/dist/src/reporting/lighty-error-reporter.d.ts

@@ -0,0 +1,24 @@
+import type { LightyConfig } from '../config/config.models.js';
+import type { Logger } from '../logging/logger.js';
+import { type LineLevel } from '../logging/logger.models.js';
+import type { ErrorReportContext, ErrorReporter } from './reporting.models.js';
+/**
+ * Reports to LightyErrors (sentry-clone) with lighty-errors-client. The client is created lazily and NOT
+ * installed: a library must not patch the host's console or process hooks — FlowRunner flushes after a failed run.
+ */
+export declare class LightyErrorReporter implements ErrorReporter {
+    private readonly config;
+    /** Our line levels → lighty-errors-client's (it says "warning"). */
+    private static readonly LEVELS;
+    private static readonly LIBRARY;
+    /** Level of the event for a step that finally failed. */
+    private static readonly REPORT_LEVEL;
+    private client;
+    private readonly logger;
+    constructor(config: LightyConfig, logger: Logger);
+    report(error: unknown, context: ErrorReportContext): string;
+    log(message: string, data?: Record<string, unknown>, level?: LineLevel): void;
+    flush(timeoutMs?: number): Promise<boolean>;
+    close(timeoutMs?: number): Promise<void>;
+    private lighty;
+}

package/dist/src/reporting/lighty-error-reporter.js

@@ -0,0 +1,96 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var LightyErrorReporter_1;
+import { inject, injectable } from 'inversify';
+import { LightyClient } from 'lighty-errors-client';
+import { TYPES } from '../di/types.js';
+import { LINE_LEVEL } from '../logging/logger.models.js';
+/**
+ * Reports to LightyErrors (sentry-clone) with lighty-errors-client. The client is created lazily and NOT
+ * installed: a library must not patch the host's console or process hooks — FlowRunner flushes after a failed run.
+ */
+let LightyErrorReporter = class LightyErrorReporter {
+    static { LightyErrorReporter_1 = this; }
+    config;
+    /** Our line levels → lighty-errors-client's (it says "warning"). */
+    static LEVELS = {
+        [LINE_LEVEL.DEBUG]: 'debug',
+        [LINE_LEVEL.INFO]: 'info',
+        [LINE_LEVEL.WARN]: 'warning',
+        [LINE_LEVEL.ERROR]: 'error',
+    };
+    static LIBRARY = 'scrapme';
+    /** Level of the event for a step that finally failed. */
+    static REPORT_LEVEL = 'error';
+    client = null;
+    logger;
+    constructor(config, logger) {
+        this.config = config;
+        this.logger = logger.child({ component: 'lighty' });
+    }
+    report(error, context) {
+        try {
+            return this.lighty().captureException(error, {
+                level: LightyErrorReporter_1.REPORT_LEVEL,
+                tags: { flow: context.flow, step: context.step, command: context.command, ...context.tags },
+                extra: {
+                    runId: context.runId,
+                    attempt: context.attempt,
+                    maxAttempts: context.maxAttempts,
+                    ...(context.input !== undefined ? { input: context.input } : {}),
+                    ...context.extra,
+                },
+            });
+        }
+        catch (err) {
+            this.logger.warn('could not report the error to LightyErrors', { error: err });
+            return '';
+        }
+    }
+    log(message, data, level = LINE_LEVEL.INFO) {
+        this.lighty().addLog(message, data, LightyErrorReporter_1.LEVELS[level]);
+    }
+    async flush(timeoutMs = this.config.flushTimeoutMs) {
+        if (!this.client)
+            return true;
+        const ok = await this.client.flush(timeoutMs);
+        if (!ok)
+            this.logger.warn('LightyErrors did not accept all events in time (they stay queued and are retried)', { timeoutMs });
+        return ok;
+    }
+    async close(timeoutMs = this.config.flushTimeoutMs) {
+        const client = this.client;
+        this.client = null;
+        if (client)
+            await client.close(timeoutMs);
+    }
+    lighty() {
+        if (!this.client) {
+            const l = this.config;
+            this.client = new LightyClient({
+                dsn: l.dsn,
+                environment: l.environment,
+                release: l.release,
+                captureConsole: false,
+                captureUncaught: false,
+                unhandledRejections: 'none',
+                extra: { library: LightyErrorReporter_1.LIBRARY },
+            });
+        }
+        return this.client;
+    }
+};
+LightyErrorReporter = LightyErrorReporter_1 = __decorate([
+    injectable(),
+    __param(0, inject(TYPES.LightyConfig)),
+    __param(1, inject(TYPES.Logger))
+], LightyErrorReporter);
+export { LightyErrorReporter };
+//# sourceMappingURL=lighty-error-reporter.js.map

package/dist/src/reporting/lighty-error-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lighty-error-reporter.js","sourceRoot":"","sources":["../../../src/reporting/lighty-error-reporter.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAmC,MAAM,sBAAsB,CAAC;AAErF,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,OAAO,EAAE,UAAU,EAAkB,MAAM,6BAA6B,CAAC;AAGzE;;;GAGG;AAEI,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;;IAeiB;IAd/C,oEAAoE;IAC5D,MAAM,CAAU,MAAM,GAAsC;QAClE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO;QAC3B,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM;QACzB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS;QAC5B,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO;KAC5B,CAAC;IACM,MAAM,CAAU,OAAO,GAAG,SAAS,CAAC;IAC5C,yDAAyD;IACjD,MAAM,CAAU,YAAY,GAAG,OAAgB,CAAC;IAChD,MAAM,GAAwB,IAAI,CAAC;IAC1B,MAAM,CAAS;IAEhC,YAC+C,MAAoB,EAC3C,MAAc;QADS,WAAM,GAAN,MAAM,CAAc;QAGjE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IACtD,CAAC;IAEM,MAAM,CAAC,KAAc,EAAE,OAA2B;QACvD,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE;gBAC3C,KAAK,EAAE,qBAAmB,CAAC,YAAY;gBACvC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE;gBAC3F,KAAK,EAAE;oBACL,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChE,GAAG,OAAO,CAAC,KAAK;iBACjB;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAC/E,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEM,GAAG,CAAC,OAAe,EAAE,IAA8B,EAAE,QAAmB,UAAU,CAAC,IAAI;QAC5F,IAAI,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,qBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACzE,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc;QACvD,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC9B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,EAAE;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mFAAmF,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9H,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,MAAM;YAAE,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAEO,MAAM;QACZ,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YACtB,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,CAAC;gBAC7B,GAAG,EAAE,CAAC,CAAC,GAAG;gBACV,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,cAAc,EAAE,KAAK;gBACrB,eAAe,EAAE,KAAK;gBACtB,mBAAmB,EAAE,MAAM;gBAC3B,KAAK,EAAE,EAAE,OAAO,EAAE,qBAAmB,CAAC,OAAO,EAAE;aAChD,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;;AAvEU,mBAAmB;IAD/B,UAAU,EAAE;IAgBR,WAAA,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;IAC1B,WAAA,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;GAhBZ,mBAAmB,CAwE/B"}

package/dist/src/reporting/log-error-reporter.d.ts

@@ -0,0 +1,12 @@
+import type { Logger } from '../logging/logger.js';
+import type { LineLevel } from '../logging/logger.models.js';
+import type { ErrorReportContext, ErrorReporter } from './reporting.models.js';
+/** Used when LIGHTY_DSN is empty: the final error of a run goes to the log (and CouchDB, via FlowRunStore). */
+export declare class LogErrorReporter implements ErrorReporter {
+    private readonly logger;
+    constructor(logger: Logger);
+    report(error: unknown, context: ErrorReportContext): string;
+    log(_message: string, _data?: Record<string, unknown>, _level?: LineLevel): void;
+    flush(): Promise<boolean>;
+    close(): Promise<void>;
+}