specrails-desktop 2.0.0

package/server/dist/telemetry-export.js

@@ -0,0 +1,317 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.redactSecrets = redactSecrets;
+exports.createDiagnosticZip = createDiagnosticZip;
+const fs_1 = __importDefault(require("fs"));
+const zlib_1 = __importDefault(require("zlib"));
+// ─── Minimal ZIP writer ───────────────────────────────────────────────────────
+// ZIP format is simple: local file headers + data + central directory + EOCD.
+// We write stored (no compression) entries for simplicity — the outer gzip on
+// the blob is already compressed; NDJSON text compresses fine at transfer layer.
+function crc32(buf) {
+    // Standard CRC-32 polynomial table
+    const table = crc32Table();
+    let crc = 0xffffffff;
+    for (let i = 0; i < buf.length; i++) {
+        crc = (crc >>> 8) ^ table[(crc ^ buf[i]) & 0xff];
+    }
+    return (crc ^ 0xffffffff) >>> 0;
+}
+let _crcTable = null;
+function crc32Table() {
+    if (_crcTable)
+        return _crcTable;
+    _crcTable = new Uint32Array(256);
+    for (let i = 0; i < 256; i++) {
+        let c = i;
+        for (let k = 0; k < 8; k++) {
+            c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1);
+        }
+        _crcTable[i] = c;
+    }
+    return _crcTable;
+}
+function buildZip(entries) {
+    const localHeaders = [];
+    const centralDirs = [];
+    let offset = 0;
+    for (const entry of entries) {
+        const nameBytes = Buffer.from(entry.name, 'utf-8');
+        const data = entry.data;
+        const crc = crc32(data);
+        const size = data.length;
+        // Local file header (30 bytes + name)
+        const local = Buffer.alloc(30 + nameBytes.length);
+        local.writeUInt32LE(0x04034b50, 0); // signature
+        local.writeUInt16LE(20, 4); // version needed
+        local.writeUInt16LE(0, 6); // flags
+        local.writeUInt16LE(0, 8); // compression = STORED
+        local.writeUInt16LE(0, 10); // mod time
+        local.writeUInt16LE(0, 12); // mod date
+        local.writeUInt32LE(crc, 14); // crc-32
+        local.writeUInt32LE(size, 18); // compressed size
+        local.writeUInt32LE(size, 22); // uncompressed size
+        local.writeUInt16LE(nameBytes.length, 26); // file name length
+        local.writeUInt16LE(0, 28); // extra field length
+        nameBytes.copy(local, 30);
+        // Central directory header (46 bytes + name)
+        const central = Buffer.alloc(46 + nameBytes.length);
+        central.writeUInt32LE(0x02014b50, 0); // signature
+        central.writeUInt16LE(20, 4); // version made by
+        central.writeUInt16LE(20, 6); // version needed
+        central.writeUInt16LE(0, 8); // flags
+        central.writeUInt16LE(0, 10); // compression = STORED
+        central.writeUInt16LE(0, 12); // mod time
+        central.writeUInt16LE(0, 14); // mod date
+        central.writeUInt32LE(crc, 16); // crc-32
+        central.writeUInt32LE(size, 20); // compressed size
+        central.writeUInt32LE(size, 24); // uncompressed size
+        central.writeUInt16LE(nameBytes.length, 28); // file name length
+        central.writeUInt16LE(0, 30); // extra field length
+        central.writeUInt16LE(0, 32); // file comment length
+        central.writeUInt16LE(0, 34); // disk number start
+        central.writeUInt16LE(0, 36); // int file attributes
+        central.writeUInt32LE(0, 38); // ext file attributes
+        central.writeUInt32LE(offset, 42); // relative offset of local header
+        nameBytes.copy(central, 46);
+        localHeaders.push(local, data);
+        centralDirs.push(central);
+        offset += local.length + size;
+    }
+    const centralDirBuf = Buffer.concat(centralDirs);
+    const centralDirSize = centralDirBuf.length;
+    const centralDirOffset = offset;
+    // End of central directory record
+    const eocd = Buffer.alloc(22);
+    eocd.writeUInt32LE(0x06054b50, 0); // signature
+    eocd.writeUInt16LE(0, 4); // disk number
+    eocd.writeUInt16LE(0, 6); // disk with CD
+    eocd.writeUInt16LE(entries.length, 8); // entries on this disk
+    eocd.writeUInt16LE(entries.length, 10); // total entries
+    eocd.writeUInt32LE(centralDirSize, 12); // central dir size
+    eocd.writeUInt32LE(centralDirOffset, 16); // central dir offset
+    eocd.writeUInt16LE(0, 20); // comment length
+    return Buffer.concat([...localHeaders, centralDirBuf, eocd]);
+}
+// ─── NDJSON reader ────────────────────────────────────────────────────────────
+function decompressNdjson(filePath) {
+    try {
+        const compressed = fs_1.default.readFileSync(filePath);
+        return zlib_1.default.gunzipSync(compressed).toString('utf-8');
+    }
+    catch {
+        return '';
+    }
+}
+function hasTruncationMarker(filePath) {
+    const content = decompressNdjson(filePath);
+    return content.includes('"logs_truncated"');
+}
+// ─── Summary markdown ─────────────────────────────────────────────────────────
+function buildSummaryFromRaw(filePath, job, truncated) {
+    const lines = [];
+    if (truncated) {
+        lines.push('> **truncated: true** — The 10 MB raw telemetry cap was hit during this run.');
+        lines.push('> Log payloads were dropped after the cap. Traces and metrics are complete.');
+        lines.push('');
+    }
+    lines.push(`# Diagnostic Summary — Job ${job.id}`);
+    lines.push('');
+    lines.push(`- **Command**: \`${job.command}\``);
+    lines.push(`- **Status**: ${job.status}`);
+    lines.push(`- **Started**: ${job.started_at}`);
+    lines.push(`- **Finished**: ${job.finished_at ?? 'N/A'}`);
+    if (job.duration_ms != null)
+        lines.push(`- **Duration**: ${job.duration_ms}ms`);
+    if (job.total_cost_usd != null)
+        lines.push(`- **Cost**: $${job.total_cost_usd.toFixed(4)}`);
+    if (job.tokens_in != null)
+        lines.push(`- **Tokens in**: ${job.tokens_in}`);
+    if (job.tokens_out != null)
+        lines.push(`- **Tokens out**: ${job.tokens_out}`);
+    lines.push('');
+    lines.push('Raw telemetry is available in `telemetry.ndjson`.');
+    return lines.join('\n');
+}
+function buildSummaryFromRows(summaries, job) {
+    const lines = [];
+    lines.push(`# Diagnostic Summary — Job ${job.id}`);
+    lines.push('');
+    lines.push(`- **Command**: \`${job.command}\``);
+    lines.push(`- **Status**: ${job.status}`);
+    lines.push(`- **Started**: ${job.started_at}`);
+    lines.push(`- **Finished**: ${job.finished_at ?? 'N/A'}`);
+    lines.push('');
+    lines.push('> Raw telemetry has been compacted (older than 7 days). Per-phase summary below.');
+    lines.push('');
+    lines.push('## Phase Summaries');
+    lines.push('');
+    for (const s of summaries) {
+        lines.push(`### Phase: ${s.phase}`);
+        if (s.durationMs != null)
+            lines.push(`- Duration: ${s.durationMs}ms`);
+        if (s.tokensInput != null)
+            lines.push(`- Tokens in: ${s.tokensInput}`);
+        if (s.tokensOutput != null)
+            lines.push(`- Tokens out: ${s.tokensOutput}`);
+        if (s.tokensCache != null)
+            lines.push(`- Cache tokens: ${s.tokensCache}`);
+        if (s.costUsd != null)
+            lines.push(`- Cost: $${s.costUsd.toFixed(4)}`);
+        if (s.apiErrors)
+            lines.push(`- API errors: ${s.apiErrors}`);
+        if (s.toolCalls) {
+            try {
+                const tc = JSON.parse(s.toolCalls);
+                lines.push(`- Tool calls: ${Object.entries(tc).map(([k, v]) => `${k}×${v}`).join(', ')}`);
+            }
+            catch {
+                lines.push(`- Tool calls: ${s.toolCalls}`);
+            }
+        }
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+// ─── Log extraction ────────────────────────────────────────────────────────────
+// App log output is never written to a flat file — it is streamed over
+// WebSocket and persisted per-event in the `events` table. Reconstruct a
+// readable logs.txt by formatting every event row in seq order.
+function buildLogsFromEvents(events) {
+    if (events.length === 0) {
+        return '(No events recorded for this job.)\n';
+    }
+    const lines = [];
+    for (const ev of events) {
+        const ts = ev.timestamp;
+        const src = ev.source ?? ev.event_type;
+        let text = ev.payload;
+        // Most log events carry { line: string }; other event types may carry
+        // structured JSON. Extract the line field when present, else stringify.
+        try {
+            const parsed = JSON.parse(ev.payload);
+            if (typeof parsed.line === 'string')
+                text = parsed.line;
+            else if (typeof parsed.message === 'string')
+                text = parsed.message;
+            else
+                text = JSON.stringify(parsed);
+        }
+        catch {
+            // Payload isn't JSON — use raw string as-is
+        }
+        lines.push(`[${ts}] [${src}] ${redactSecrets(text)}`);
+    }
+    return lines.join('\n') + '\n';
+}
+/**
+ * Best-effort redaction of obvious secrets from diagnostic log text (B52). The
+ * exported ZIP is meant to be shared (support, bug reports), but the CLI event
+ * log can echo `export ANTHROPIC_API_KEY=...`, Bearer tokens, or provider keys.
+ * This scrubs the common shapes without destroying the log's diagnostic value.
+ */
+function redactSecrets(text) {
+    return text
+        // KEY=VALUE / TOKEN: VALUE for anything ending in API_KEY/TOKEN/SECRET/PASSWORD/ACCESS_KEY
+        .replace(/\b([A-Za-z0-9_]*(?:API_KEY|TOKEN|SECRET|PASSWORD|ACCESS_KEY))(\s*[=:]\s*)(["']?)[^\s"']+\3/gi, '$1$2[REDACTED]')
+        // Provider key prefixes (Anthropic/OpenAI sk-..., GitHub gh*_...)
+        .replace(/\bsk-[A-Za-z0-9_-]{16,}/g, 'sk-[REDACTED]')
+        .replace(/\bgh[pousr]_[A-Za-z0-9]{16,}/g, 'gh_[REDACTED]')
+        // Authorization: Bearer <token>
+        .replace(/\b(Bearer)\s+[A-Za-z0-9._-]{12,}/gi, '$1 [REDACTED]');
+}
+async function createDiagnosticZip(res, opts) {
+    const { job, blob, summaries, events, profile, pluginSnapshotPath } = opts;
+    const entries = [];
+    // plugins.json — per-job plugin snapshot, if present.
+    let pluginsSummary = null;
+    if (pluginSnapshotPath && fs_1.default.existsSync(pluginSnapshotPath)) {
+        try {
+            const raw = fs_1.default.readFileSync(pluginSnapshotPath, 'utf8');
+            pluginsSummary = JSON.parse(raw);
+            entries.push({ name: 'plugins.json', data: Buffer.from(raw, 'utf-8') });
+        }
+        catch {
+            // ignore malformed snapshot — diagnostic still useful without it
+        }
+    }
+    // job-metadata.json
+    const metadata = {
+        id: job.id,
+        command: job.command,
+        status: job.status,
+        started_at: job.started_at,
+        finished_at: job.finished_at,
+        duration_ms: job.duration_ms,
+        total_cost_usd: job.total_cost_usd,
+        tokens_in: job.tokens_in,
+        tokens_out: job.tokens_out,
+        tokens_cache_read: job.tokens_cache_read,
+        tokens_cache_create: job.tokens_cache_create,
+        num_turns: job.num_turns,
+        model: job.model,
+        session_id: job.session_id,
+    };
+    entries.push({
+        name: 'job-metadata.json',
+        data: Buffer.from(JSON.stringify({ ...metadata, profile_name: profile?.name ?? null }, null, 2), 'utf-8'),
+    });
+    // profile.json — the snapshot the job ran under (if any)
+    if (profile && profile.json) {
+        entries.push({
+            name: 'profile.json',
+            data: Buffer.from(profile.json, 'utf-8'),
+        });
+    }
+    // telemetry.ndjson
+    let truncated = false;
+    if (blob.state === 'active' && blob.path && fs_1.default.existsSync(blob.path)) {
+        truncated = hasTruncationMarker(blob.path);
+        const ndjsonContent = decompressNdjson(blob.path);
+        entries.push({
+            name: 'telemetry.ndjson',
+            data: Buffer.from(ndjsonContent, 'utf-8'),
+        });
+    }
+    else {
+        // Compacted: include a header note
+        const header = '# Telemetry data has been compacted (raw blob older than 7 days).\n# See summary.md for per-phase aggregates.\n';
+        entries.push({
+            name: 'telemetry.ndjson',
+            data: Buffer.from(header, 'utf-8'),
+        });
+    }
+    // logs.txt — reconstructed from persisted events
+    const logsContent = buildLogsFromEvents(events);
+    entries.push({
+        name: 'logs.txt',
+        data: Buffer.from(logsContent, 'utf-8'),
+    });
+    // summary.md
+    let summaryContent;
+    if (blob.state === 'active' && blob.path) {
+        summaryContent = buildSummaryFromRaw(blob.path, job, truncated);
+    }
+    else {
+        summaryContent = buildSummaryFromRows(summaries, job);
+    }
+    if (pluginsSummary) {
+        const lines = ['', '## Plugins'];
+        if (pluginsSummary.active.length > 0) {
+            lines.push('', '### Active', ...pluginsSummary.active.map((p) => `- ${p.name}@${p.version}`));
+        }
+        if (pluginsSummary.degraded.length > 0) {
+            lines.push('', '### Degraded', ...pluginsSummary.degraded.map((p) => `- ${p.name} (${p.reason})`));
+        }
+        summaryContent += '\n' + lines.join('\n') + '\n';
+    }
+    entries.push({
+        name: 'summary.md',
+        data: Buffer.from(summaryContent, 'utf-8'),
+    });
+    const zipBuf = buildZip(entries);
+    res.end(zipBuf);
+}

package/server/dist/telemetry-receiver.js

@@ -0,0 +1,224 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTelemetryRouter = createTelemetryRouter;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const os_1 = __importDefault(require("os"));
+const zlib_1 = __importDefault(require("zlib"));
+const express_1 = require("express");
+const db_1 = require("./db");
+const db_2 = require("./db");
+// 10 MB uncompressed cap per job blob
+const BLOB_SIZE_CAP = 10 * 1024 * 1024;
+// B2: the soft cap above only dropped `logs`, so traces/metrics could grow a
+// per-job blob without bound (disk-fill DoS). This hard cap bounds the TOTAL
+// blob across ALL signals while still letting traces/metrics outlive logs.
+const BLOB_HARD_CAP = 50 * 1024 * 1024;
+// Bounded in-memory append queue per (projectId, jobId) key — protects the
+// Express event loop from a burst of OTLP payloads all trying to gzip-append
+// simultaneously. Drops events (with a warning) when the queue exceeds this.
+const QUEUE_CAP = 10_000;
+const _blobState = new Map();
+function getBlobState(key) {
+    let s = _blobState.get(key);
+    if (!s) {
+        s = { uncompressedSize: 0, truncationMarkerWritten: false, queue: [], writing: false };
+        _blobState.set(key, s);
+    }
+    return s;
+}
+function blobKey(projectId, jobId) {
+    return `${projectId}:${jobId}`;
+}
+// ─── Blob path ────────────────────────────────────────────────────────────────
+function telemetryDir(projectSlug) {
+    return path_1.default.join(os_1.default.homedir(), '.specrails', 'projects', projectSlug, 'telemetry');
+}
+function blobPath(projectSlug, jobId) {
+    return path_1.default.join(telemetryDir(projectSlug), `${jobId}.ndjson.gz`);
+}
+// ─── Extract resource attributes ─────────────────────────────────────────────
+function extractAttr(attributes, key) {
+    if (!attributes)
+        return undefined;
+    const attr = attributes.find((a) => a.key === key);
+    if (!attr)
+        return undefined;
+    const v = attr.value;
+    if (typeof v.stringValue === 'string')
+        return v.stringValue;
+    if (v.intValue != null)
+        return String(v.intValue);
+    return undefined;
+}
+function extractJobAndProject(body) {
+    if (typeof body !== 'object' || body === null)
+        return null;
+    // OTLP/JSON: resourceSpans | resourceMetrics | resourceLogs
+    const b = body;
+    const resourceItems = [
+        ...(Array.isArray(b.resourceSpans) ? b.resourceSpans : []),
+        ...(Array.isArray(b.resourceMetrics) ? b.resourceMetrics : []),
+        ...(Array.isArray(b.resourceLogs) ? b.resourceLogs : []),
+    ];
+    for (const rs of resourceItems) {
+        const attrs = rs.resource?.attributes;
+        const jobId = extractAttr(attrs, 'specrails.job_id');
+        const projectId = extractAttr(attrs, 'specrails.project_id');
+        if (jobId && projectId)
+            return { jobId, projectId };
+    }
+    return null;
+}
+// ─── Gzip append ─────────────────────────────────────────────────────────────
+function appendToGzip(filePath, line) {
+    return new Promise((resolve, reject) => {
+        const data = Buffer.from(line + '\n', 'utf-8');
+        // Append a gzip member to the file. gzip is a concatenated format:
+        // a reader decompresses member by member, giving correct NDJSON.
+        const gz = zlib_1.default.createGzip();
+        const chunks = [];
+        gz.on('data', (c) => chunks.push(c));
+        gz.on('end', () => {
+            const compressed = Buffer.concat(chunks);
+            fs_1.default.appendFile(filePath, compressed, (err) => {
+                if (err)
+                    reject(err);
+                else
+                    resolve();
+            });
+        });
+        gz.on('error', reject);
+        gz.end(data);
+    });
+}
+// ─── Enqueue write task ───────────────────────────────────────────────────────
+function enqueueWrite(state, task) {
+    if (state.queue.length >= QUEUE_CAP) {
+        // Drop with a warning — we don't want to stall the event loop
+        console.warn('[telemetry-receiver] append queue full — dropping telemetry event');
+        return;
+    }
+    const wrappedTask = () => {
+        task().then(drain, drain);
+    };
+    function drain() {
+        state.writing = false;
+        const next = state.queue.shift();
+        if (next) {
+            state.writing = true;
+            next();
+        }
+    }
+    state.queue.push(wrappedTask);
+    if (!state.writing) {
+        state.writing = true;
+        const next = state.queue.shift();
+        next();
+    }
+}
+// ─── Ingest handler ───────────────────────────────────────────────────────────
+async function handleIngest(signal, body, registry, res) {
+    const ids = extractJobAndProject(body);
+    if (!ids) {
+        res.status(400).json({ error: 'Missing specrails.job_id or specrails.project_id in resource.attributes' });
+        return;
+    }
+    const { jobId, projectId } = ids;
+    const projectCtx = registry.getContext(projectId);
+    if (!projectCtx) {
+        res.status(404).json({ error: 'Project not found' });
+        return;
+    }
+    const { db, project } = projectCtx;
+    const job = (0, db_1.getJob)(db, jobId);
+    if (!job) {
+        res.status(404).json({ error: 'Job not found' });
+        return;
+    }
+    const key = blobKey(projectId, jobId);
+    const state = getBlobState(key);
+    const filePath = blobPath(project.slug, jobId);
+    const now = Date.now();
+    // Cap enforcement: drop logs once the 10 MB soft cap is reached (B2: and drop
+    // ANY signal — traces/metrics included — once the 50 MB hard cap is reached, so
+    // a job streaming endless spans can't fill the disk).
+    if (state.uncompressedSize >= BLOB_HARD_CAP || (state.uncompressedSize >= BLOB_SIZE_CAP && signal === 'logs')) {
+        res.status(200).json({ ok: true, dropped: true });
+        return;
+    }
+    // Determine raw line size for cap tracking (uncompressed JSON length)
+    const payloadStr = JSON.stringify(body);
+    const lineObj = { signal, receivedAt: new Date().toISOString(), payload: body };
+    const lineStr = JSON.stringify(lineObj);
+    const lineSize = Buffer.byteLength(lineStr, 'utf-8');
+    // Check if this write will push us over the cap
+    const willExceedCap = state.uncompressedSize + lineSize > BLOB_SIZE_CAP;
+    const prevSize = state.uncompressedSize;
+    state.uncompressedSize += lineSize;
+    // Ensure directory and blob pointer row exist before we enqueue the write
+    const dir = telemetryDir(project.slug);
+    fs_1.default.mkdirSync(dir, { recursive: true });
+    const existingBlob = (0, db_2.getTelemetryBlob)(db, jobId);
+    if (!existingBlob) {
+        (0, db_2.upsertTelemetryBlob)(db, {
+            jobId,
+            path: filePath,
+            byteSize: 0,
+            startedAt: now,
+            endedAt: now,
+            state: 'active',
+        });
+    }
+    else {
+        (0, db_2.upsertTelemetryBlob)(db, {
+            ...existingBlob,
+            byteSize: state.uncompressedSize,
+            endedAt: now,
+        });
+    }
+    // Enqueue the actual file append
+    enqueueWrite(state, async () => {
+        if (willExceedCap && signal === 'logs' && !state.truncationMarkerWritten) {
+            // Write the truncation marker once, before dropping this log event
+            state.truncationMarkerWritten = true;
+            const marker = JSON.stringify({ signal: 'control', event: 'logs_truncated', at: new Date().toISOString() });
+            await appendToGzip(filePath, marker);
+        }
+        // Drop further logs after cap
+        if (prevSize >= BLOB_SIZE_CAP && signal === 'logs')
+            return;
+        await appendToGzip(filePath, lineStr);
+        // Update byteSize in DB after each write
+        (0, db_2.upsertTelemetryBlob)(db, {
+            jobId,
+            path: filePath,
+            byteSize: state.uncompressedSize,
+            startedAt: existingBlob?.startedAt ?? now,
+            endedAt: now,
+            state: 'active',
+        });
+    });
+    // Suppress unused-variable warning for payloadStr — it's used above for size calc
+    void payloadStr;
+    res.status(200).json({ ok: true });
+}
+// ─── Router factory ───────────────────────────────────────────────────────────
+function createTelemetryRouter(registry) {
+    const router = (0, express_1.Router)();
+    const handle = (signal) => (req, res) => {
+        handleIngest(signal, req.body, registry, res).catch((err) => {
+            console.error(`[telemetry-receiver] error handling ${signal}:`, err);
+            if (!res.headersSent) {
+                res.status(500).json({ error: 'Internal server error' });
+            }
+        });
+    };
+    router.post('/v1/traces', handle('traces'));
+    router.post('/v1/metrics', handle('metrics'));
+    router.post('/v1/logs', handle('logs'));
+    return router;
+}