pierre-review 0.1.5 → 0.1.6

package/dist/config.js

@@ -58,6 +58,15 @@ export const config = {
     host: process.env.HOST ?? (isCloud ? '0.0.0.0' : '127.0.0.1'),
     dbPath: isCloud || isAbsolute(rawDbUrl) ? rawDbUrl : resolve(backendRoot, rawDbUrl),
     backfillDays: intFromEnv('BACKFILL_DAYS', 90),
+    // First sync runs in two phases: a fast "foreground" window (matching the
+    // default timeline range) so the board is usable in seconds, then the rest of
+    // backfillDays is fetched in the background. Two-phase only kicks in when
+    // backfillDays exceeds this.
+    foregroundSyncDays: intFromEnv('FOREGROUND_SYNC_DAYS', 14),
+    // How many commit-file REST fetches to keep in flight at once (one pool per
+    // page). These draw from the REST quota (disjoint from the GraphQL points
+    // pool), so a modest pool safely cuts the dominant sync stage.
+    commitFileConcurrency: intFromEnv('COMMIT_FILE_CONCURRENCY', 10),
     syncCron: process.env.SYNC_CRON ?? '*/5 * * * *',
     syncOverlapMinutes: intFromEnv('SYNC_OVERLAP_MINUTES', 20),
     stallThresholdDays: intFromEnv('STALL_THRESHOLD_DAYS', 3),

package/dist/sync/commit-files.js

@@ -7,7 +7,7 @@ const { commitFiles } = schema;
  * `commit_files` cache and filling misses via REST. SHAs are immutable, so the
  * cache never expires — re-syncs are free.
  */
-export async function ensureCommitFiles(owner, name, shas, token) {
+export async function ensureCommitFiles(owner, name, shas, token, concurrency = 10) {
     const result = new Map();
     const unique = [...new Set(shas)];
     if (unique.length === 0)
@@ -20,12 +20,13 @@ export async function ensureCommitFiles(owner, name, shas, token) {
         .execute();
     for (const row of cached)
         result.set(row.sha, row.paths);
-    // Fetch cache misses with bounded concurrency. These REST calls dominate
-    // sync latency on a PR that just got several commits addressing threads;
-    // running them serially blocks the whole page loop. SHAs are immutable and
-    // the cache is idempotent, so parallelism is safe.
+    // Fetch cache misses through one continuously-saturated worker pool. These
+    // REST calls dominate sync latency; the caller now hands us a whole page's
+    // worth of misses at once (rather than per-PR waves), so a fixed set of
+    // workers each pull the next SHA the instant they finish — keeping `concurrency`
+    // requests in flight the whole time. SHAs are immutable and the cache upsert is
+    // idempotent, so order doesn't matter.
     const missing = unique.filter((sha) => !result.has(sha));
-    const CONCURRENCY = 5;
     const fetchOne = async (sha) => {
         try {
             const commit = await ghRestGetFor(token, `/repos/${owner}/${name}/commits/${sha}`);
@@ -43,9 +44,14 @@ export async function ensureCommitFiles(owner, name, shas, token) {
             result.set(sha, []);
         }
     };
-    for (let i = 0; i < missing.length; i += CONCURRENCY) {
-        await Promise.all(missing.slice(i, i + CONCURRENCY).map(fetchOne));
-    }
+    let next = 0;
+    const worker = async () => {
+        while (next < missing.length) {
+            const sha = missing[next++];
+            await fetchOne(sha);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(Math.max(1, concurrency), missing.length) }, worker));
     return result;
 }
 //# sourceMappingURL=commit-files.js.map

package/dist/sync/sync-manager.js

@@ -142,16 +142,63 @@ export async function runSyncForRepo(repoId, log, opts = {}) {
         log.error(`sync ${repo.owner}/${repo.name}: no access token for account ${repo.accountId}: ${err instanceof Error ? err.message : err}`);
         return false;
     }
-    const task = syncRepo({
+    const common = {
         owner: repo.owner,
         name: repo.name,
         accountId: repo.accountId,
         token,
-        ...plan,
         log,
-        onProgress: (p) => setSyncProgress(repoId, { ...p, mode: plan.mode }),
+        commitFileConcurrency: config.commitFileConcurrency,
         shouldCancel: () => cancelRequested.has(repoId),
-    })
+    };
+    // Two-phase only for a first full backfill (never-synced, not a forced "deep"
+    // re-sync) when the backfill window is wider than the foreground window. A deep
+    // re-sync stays single-pass — its board is already populated, so there's no
+    // blank-board wait to shorten.
+    const twoPhase = !opts.forceFull && plan.mode === 'full' && config.backfillDays > config.foregroundSyncDays;
+    const runWalk = async () => {
+        if (!twoPhase) {
+            await syncRepo({
+                ...common,
+                mode: plan.mode,
+                since: plan.since,
+                commitState: true,
+                onProgress: (p) => setSyncProgress(repoId, { ...p, mode: plan.mode }),
+            });
+            return;
+        }
+        // Phase 1 — the fast foreground window (the default timeline range). Committed
+        // per-PR so the recent board is usable in seconds, but does NOT stamp
+        // syncState, so the repo stays an "initial backfill" until phase 2 finishes.
+        const foregroundSince = new Date(Date.now() - config.foregroundSyncDays * DAY_MS);
+        const p1 = await syncRepo({
+            ...common,
+            mode: 'full',
+            since: foregroundSince,
+            commitState: false,
+            onProgress: (p) => setSyncProgress(repoId, { ...p, mode: 'full', foregroundComplete: false }),
+        });
+        if (p1.cancelled)
+            return;
+        // Foreground done — flip the flag so the UI drops the user into the recent
+        // view, then continue the SAME cursor walk back to the full backfill window.
+        setSyncProgress(repoId, {
+            percent: 1,
+            prsProcessed: p1.prCount,
+            pages: p1.pages,
+            mode: 'full',
+            foregroundComplete: true,
+        });
+        await syncRepo({
+            ...common,
+            mode: 'full',
+            since: plan.since, // now − backfillDays
+            startCursor: p1.endCursor,
+            commitState: true,
+            onProgress: (p) => setSyncProgress(repoId, { ...p, mode: 'full', foregroundComplete: true }),
+        });
+    };
+    const task = runWalk()
         .catch((err) => {
         log.error(`background sync ${repo.owner}/${repo.name} failed: ${err instanceof Error ? err.message : err}`);
     })
@@ -193,6 +240,8 @@ export async function syncAllRepos(log) {
                 accountId: repo.accountId,
                 token,
                 ...plan,
+                commitState: true,
+                commitFileConcurrency: config.commitFileConcurrency,
                 log,
                 onProgress: (p) => setSyncProgress(r.id, { ...p, mode: plan.mode }),
                 shouldCancel: () => cancelRequested.has(r.id),

package/dist/sync/sync-repo.js

@@ -1,3 +1,4 @@
+import { performance } from 'node:perf_hooks';
 import { eq } from 'drizzle-orm';
 import { db, schema } from '../db/client.js';
 import { getGraphqlClientFor } from '../github/client.js';
@@ -15,15 +16,28 @@ function clamp01(n) {
 }
 export async function syncRepo(opts) {
     const { owner, name, accountId, mode, since, onProgress } = opts;
+    const commitState = opts.commitState ?? true;
     const log = opts.log ?? consoleLogger;
     const client = getGraphqlClientFor(opts.token);
     const resolver = createUserResolver();
-    let cursor = null;
+    let cursor = opts.startCursor ?? null;
+    // The `after` value used to fetch the page currently being processed. When we
+    // stop at the `since` cutoff mid-page, this (not the page's endCursor) is what a
+    // follow-on phase resumes from, so the cutoff page's older PRs aren't skipped.
+    let pageStartCursor = cursor;
     let repoId = null;
     let prCount = 0;
     let pages = 0;
     let totalCost = 0;
     let lastRemaining = 0;
+    // Per-stage wall-clock accumulators, so the final log attributes the 2-3 min:
+    // page fetch (network/GraphQL) vs commit-file REST fan-out vs DB persist. This
+    // is the baseline that tells us which stage to optimise next.
+    let graphqlMs = 0;
+    let commitFilesMs = 0;
+    let persistMs = 0;
+    const timingSummary = () => `graphql ${(graphqlMs / 1000).toFixed(1)}s / commit-files ` +
+        `${(commitFilesMs / 1000).toFixed(1)}s / persist ${(persistMs / 1000).toFixed(1)}s`;
     // Time-walked progress: PRs arrive newest-first and we stop at `since`, so the
     // span [since .. newest] is the work and the current PR's updatedAt marks how
     // far through it we are.
@@ -46,11 +60,14 @@ export async function syncRepo(opts) {
                 cancelled = true;
                 break;
             }
+            pageStartCursor = cursor;
+            const tPage = performance.now();
             const resp = await client(REPO_ACTIVITY_QUERY, {
                 owner,
                 name,
                 cursor,
             });
+            graphqlMs += performance.now() - tPage;
             pages += 1;
             totalCost += resp.rateLimit.cost;
             lastRemaining = resp.rateLimit.remaining;
@@ -63,6 +80,11 @@ export async function syncRepo(opts) {
                 repoId = await upsertRepo(owner, name, resp.repository.id, resp.repository.defaultBranchRef?.name ?? null, accountId);
             }
             const { nodes, pageInfo } = resp.repository.pullRequests;
+            // First select the in-window PRs on this page and gather every commit SHA
+            // whose changed files we need (commits that could plausibly have addressed
+            // an open thread, i.e. landed after its last comment).
+            const pagePrs = [];
+            const pageShas = [];
             for (const pr of nodes) {
                 if (opts.shouldCancel?.()) {
                     cancelled = true;
@@ -74,20 +96,36 @@ export async function syncRepo(opts) {
                     break;
                 }
                 newestMs ??= updatedMs;
-                // Only fetch changed-files for commits that could plausibly have
-                // addressed an open thread (after its last comment).
                 const unresolved = pr.reviewThreads.nodes.filter((t) => !t.isResolved && t.comments.nodes.length > 0);
-                let shas = [];
                 if (unresolved.length > 0) {
                     const threshold = Math.min(...unresolved.map((t) => Date.parse(t.comments.nodes.at(-1).createdAt)));
-                    shas = pr.commits.nodes
-                        .filter((c) => Date.parse(c.commit.committedDate) > threshold)
-                        .map((c) => c.commit.oid);
+                    for (const c of pr.commits.nodes) {
+                        if (Date.parse(c.commit.committedDate) > threshold)
+                            pageShas.push(c.commit.oid);
+                    }
+                }
+                pagePrs.push({ pr, updatedMs });
+            }
+            // Fetch the whole page's commit files in one saturated pool (replacing the
+            // old per-PR serial waves), then persist each in-window PR. persistPr only
+            // reads the SHAs its own commits need, so a page-wide superset map is fine.
+            // Skip entirely if a cancel arrived mid-gather — don't do network/DB work
+            // we're about to throw away.
+            if (!cancelled && pagePrs.length > 0) {
+                const tFiles = performance.now();
+                const commitFilesBySha = await ensureCommitFiles(owner, name, pageShas, opts.token, opts.commitFileConcurrency);
+                commitFilesMs += performance.now() - tFiles;
+                for (const { pr, updatedMs } of pagePrs) {
+                    if (opts.shouldCancel?.()) {
+                        cancelled = true;
+                        break;
+                    }
+                    const tPersist = performance.now();
+                    await persistPr(pr, repoId, resolver, commitFilesBySha, accountId);
+                    persistMs += performance.now() - tPersist;
+                    prCount += 1;
+                    reportProgress(updatedMs);
                 }
-                const commitFilesBySha = await ensureCommitFiles(owner, name, shas, opts.token);
-                await persistPr(pr, repoId, resolver, commitFilesBySha, accountId);
-                prCount += 1;
-                reportProgress(updatedMs);
             }
             cursor = pageInfo.endCursor;
             if (cancelled || stop || !pageInfo.hasNextPage)
@@ -98,13 +136,15 @@ export async function syncRepo(opts) {
         // safely delete it. Already-persisted PRs are harmless (idempotent) and get
         // cleaned up with the repo, or resumed on the next sync for an existing repo.
         if (cancelled) {
-            log.info(`sync ${owner}/${name} cancelled after ${prCount} PRs / ${pages} page(s)`);
+            log.info(`sync ${owner}/${name} cancelled after ${prCount} PRs / ${pages} page(s) — timing: ${timingSummary()}`);
             return {
                 repoId: repoId ?? -1,
                 prCount,
                 pages,
                 rateLimitRemaining: lastRemaining,
                 rateLimitCost: totalCost,
+                cancelled: true,
+                endCursor: pageStartCursor,
             };
         }
         // Reached the cutoff / last page — the walk is complete.
@@ -112,25 +152,37 @@ export async function syncRepo(opts) {
         if (repoId === null) {
             throw new Error(`Repository ${owner}/${name} returned no data`);
         }
-        const now = new Date();
-        const statePatch = mode === 'full'
-            ? { lastFullSyncAt: now, lastIncrementalSyncAt: now }
-            : { lastIncrementalSyncAt: now };
-        await db
-            .insert(syncState)
-            .values({ repoId, ...statePatch, lastSyncStatus: 'ok', lastSyncError: null })
-            .onConflictDoUpdate({
-            target: syncState.repoId,
-            set: { ...statePatch, lastSyncStatus: 'ok', lastSyncError: null },
-        })
-            .execute();
-        log.info(`sync ${owner}/${name} [${mode}] done: ${prCount} PRs over ${pages} page(s), cost ${totalCost}, ${lastRemaining} remaining`);
+        // commitState=false (a two-phase foreground pass) deliberately does NOT stamp
+        // the repo as synced — the authoritative timestamp is written by the deeper
+        // pass that follows, so planSync keeps treating the repo as not-yet-fully-
+        // synced (and the cancel endpoint as an initial backfill) until then.
+        if (commitState) {
+            const now = new Date();
+            const statePatch = mode === 'full'
+                ? { lastFullSyncAt: now, lastIncrementalSyncAt: now }
+                : { lastIncrementalSyncAt: now };
+            await db
+                .insert(syncState)
+                .values({ repoId, ...statePatch, lastSyncStatus: 'ok', lastSyncError: null })
+                .onConflictDoUpdate({
+                target: syncState.repoId,
+                set: { ...statePatch, lastSyncStatus: 'ok', lastSyncError: null },
+            })
+                .execute();
+        }
+        const phase = commitState ? mode : `${mode} foreground`;
+        log.info(`sync ${owner}/${name} [${phase}] done: ${prCount} PRs over ${pages} page(s), ` +
+            `cost ${totalCost}, ${lastRemaining} remaining — timing: ${timingSummary()}`);
+        // Resume point for a follow-on phase: re-fetch the cutoff page (so its older
+        // PRs aren't skipped) if we stopped there, else the final cursor.
         return {
             repoId,
             prCount,
             pages,
             rateLimitRemaining: lastRemaining,
             rateLimitCost: totalCost,
+            cancelled: false,
+            endCursor: stop ? pageStartCursor : cursor,
         };
     }
     catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pierre-review",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Dashboard for tracking your team's GitHub PR activity across repos — local (SQLite + gh) or self-hosted multi-tenant cloud (Postgres + GitHub App).",
   "type": "module",
   "author": "Alex Wakeman",