tewip-ai 0.62.0 → 0.63.0

package/dist/reporters/store.js

@@ -1,5 +1,5 @@
 // JSONL persistence shared by the reporter, the history script and future layers.
-import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { openSync, readSync, closeSync, fstatSync, appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';
 import { dirname, isAbsolute, join, resolve } from 'node:path';
 export const PROJECT_ROOT = join(import.meta.dirname, '..');
 export const DATA_DIR = join(PROJECT_ROOT, 'data');
@@ -13,23 +13,68 @@ export function appendJsonl(file, rows) {
     mkdirSync(dirname(file), { recursive: true });
     appendFileSync(file, rows.map((row) => JSON.stringify(row)).join('\n') + '\n', 'utf8');
 }
-/** Reads a JSONL file; missing file yields []. Malformed lines are skipped with a warning. */
+/**
+ * Reads a JSONL file; missing file yields []. Malformed lines are skipped with a warning.
+ *
+ * Read in chunks rather than whole. `readFileSync(file, 'utf8')` builds one string, and a string in
+ * Node cannot exceed 0x1fffffe8 characters - about 512MB. These files only ever grow: the weekly
+ * shadow run appends every CI failure it harvests from a public project and keeps the lot, so that
+ * accuracy is measured against everything ever collected rather than one week of it. gutenberg's
+ * dataset crossed the limit and **every run since has died on this line**, for weeks, while the
+ * workflow's only symptom was a red tick nobody was looking at.
+ *
+ * A UTF-8 character can straddle a chunk boundary, so the last partial line is carried into the next
+ * read rather than decoded on its own - decoding a split character gives U+FFFD and a line that no
+ * longer parses.
+ */
 export function readJsonl(file) {
     if (!existsSync(file))
         return [];
     const rows = [];
-    readFileSync(file, 'utf8')
-        .split(/\r?\n/)
-        .forEach((line, i) => {
-        if (!line.trim())
-            return;
-        try {
-            rows.push(JSON.parse(line));
+    const handle = openSync(file, 'r');
+    try {
+        const size = fstatSync(handle).size;
+        const buffer = Buffer.allocUnsafe(8 * 1024 * 1024);
+        let carry = Buffer.alloc(0);
+        let line = 0;
+        let offset = 0;
+        while (offset < size) {
+            const read = readSync(handle, buffer, 0, buffer.length, offset);
+            if (read <= 0)
+                break;
+            offset += read;
+            const chunk = Buffer.concat([carry, buffer.subarray(0, read)]);
+            let from = 0;
+            for (let i = 0; i < chunk.length; i += 1) {
+                if (chunk[i] !== 0x0a)
+                    continue;
+                const text = chunk.subarray(from, i).toString('utf8').replace(/\r$/, '');
+                from = i + 1;
+                line += 1;
+                if (!text.trim())
+                    continue;
+                try {
+                    rows.push(JSON.parse(text));
+                }
+                catch {
+                    console.warn(`[triage] skipping malformed line ${line} in ${file}`);
+                }
+            }
+            carry = chunk.subarray(from);
         }
-        catch {
-            console.warn(`[triage] skipping malformed line ${i + 1} in ${file}`);
+        const last = carry.toString('utf8').replace(/\r$/, '');
+        if (last.trim()) {
+            try {
+                rows.push(JSON.parse(last));
+            }
+            catch {
+                console.warn(`[triage] skipping malformed line ${line + 1} in ${file}`);
+            }
         }
-    });
+    }
+    finally {
+        closeSync(handle);
+    }
     return rows;
 }
 export const readTriageRecords = (file = TRIAGE_FILE) => readJsonl(file);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.62.0",
+  "version": "0.63.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [