stratagate-dsh 0.2.63 → 0.2.65

package/dist/plugin.js

@@ -2752,6 +2752,7 @@ var StrataGate = class _StrataGate {
   summaryJobs = /* @__PURE__ */ new Map();
   elementProjectionJobs = /* @__PURE__ */ new Map();
   graphProjectionJobs = /* @__PURE__ */ new Map();
+  manualRetryRuns = /* @__PURE__ */ new Map();
   usageReceipts = /* @__PURE__ */ new Map();
   successfulModelResponses = [];
   ingestionReceipts = /* @__PURE__ */ new Map();
@@ -2997,34 +2998,90 @@ var StrataGate = class _StrataGate {
   listSummaryJobs() {
     return [...this.summaryJobs.values()];
   }
-  /** Give one terminally failed Block Summary job a fresh, user-requested retry budget. */
+  runManualRetry(key, operation) {
+    const existing = this.manualRetryRuns.get(key);
+    if (existing) return existing;
+    const run = operation().finally(() => {
+      if (this.manualRetryRuns.get(key) === run) this.manualRetryRuns.delete(key);
+    });
+    this.manualRetryRuns.set(key, run);
+    return run;
+  }
+  /** Give one failed Block Summary job a fresh, user-requested retry budget. */
   async retryBlockSummary(id) {
     const blockId = id.trim();
     if (!blockId) throw new TypeError("Block id must not be empty");
-    const block = this.blocks.find((candidate) => candidate.id === blockId);
-    if (!block) throw new Error(`Unknown block: ${blockId}`);
-    if (block.processingStatus !== "pending") throw new Error(`Block ${blockId} is already ready`);
-    await this.commitMutation(() => {
-      const job = this.summaryJobs.get(blockId);
-      if (!job) throw new Error(`Missing summary job for block: ${blockId}`);
-      if (job.status !== "failed" || job.nextRetryAt !== null) {
-        throw new Error(`Block ${blockId} Summary has not exhausted automatic retries`);
+    return this.runManualRetry(`summary:${blockId}`, async () => {
+      const block = this.blocks.find((candidate) => candidate.id === blockId);
+      if (!block) throw new Error(`Unknown block: ${blockId}`);
+      if (block.processingStatus !== "pending") throw new Error(`Block ${blockId} is already ready`);
+      await this.commitMutation(() => {
+        const job = this.summaryJobs.get(blockId);
+        if (!job) throw new Error(`Missing summary job for block: ${blockId}`);
+        if (job.status !== "failed") throw new Error(`Block ${blockId} Summary is ${job.status}, not failed`);
+        this.summaryJobs.set(blockId, {
+          ...job,
+          status: "pending",
+          attempts: 0,
+          lastError: null,
+          nextRetryAt: null,
+          updatedAt: toUtc8Iso(this.now())
+        });
+      });
+      const extractedEvents = await this.processBlock(block, { retryFailed: false });
+      const processed = this.blocks.find((candidate) => candidate.id === blockId);
+      const readyBlocks = processed?.processingStatus === "ready" ? [processed] : [];
+      return { sealedBlocks: [], readyBlocks, extractedEvents, projectedElements: [] };
+    });
+  }
+  /** Retry only Event extraction for one failed Block without rerunning its Summary. */
+  async retryEventExtraction(id) {
+    const blockId = id.trim();
+    if (!blockId) throw new TypeError("Block id must not be empty");
+    return this.runManualRetry(`extraction:${blockId}`, async () => {
+      const block = this.blocks.find((candidate) => candidate.id === blockId);
+      if (!block) throw new Error(`Unknown block: ${blockId}`);
+      if (block.processingStatus !== "pending" || block.shouldExtract !== true) {
+        throw new Error(`Block ${blockId} is not waiting for Event extraction`);
       }
-      this.summaryJobs.set(blockId, {
-        ...job,
-        status: "pending",
-        attempts: 0,
-        lastError: null,
-        nextRetryAt: null,
-        updatedAt: toUtc8Iso(this.now())
+      await this.commitMutation(() => {
+        const job = this.extractionJobs.get(blockId);
+        if (!job) throw new Error(`Missing extraction job for block: ${blockId}`);
+        if (job.status !== "failed") throw new Error(`Event extraction ${blockId} is ${job.status}, not failed`);
+        this.extractionJobs.set(blockId, {
+          ...job,
+          attempts: 0,
+          lastError: null,
+          nextRetryAt: null,
+          updatedAt: toUtc8Iso(this.now())
+        });
       });
+      return await this.extractEligibleBlock({ blockId, retryFailed: true }) ?? [];
+    });
+  }
+  /** Retry exactly one failed Graph projection batch. */
+  async retryGraphProjection(id) {
+    const jobId = id.trim();
+    if (!jobId) throw new TypeError("Graph projection id must not be empty");
+    return this.runManualRetry(`graph:${jobId}`, async () => {
+      if (!this.graphProjector) throw new Error("Graph projector is not configured");
+      await this.commitMutation(() => {
+        const job = this.requireGraphProjectionJob(jobId);
+        if (job.status !== "failed") throw new Error(`Graph projection ${jobId} is ${job.status}, not failed`);
+        job.status = "pending";
+        job.attempts = 0;
+        job.lastError = null;
+        job.updatedAt = toUtc8Iso(this.now());
+      });
+      const batch = await this.claimGraphProjection(jobId);
+      if (!batch) throw new Error(`Graph projection ${jobId} could not be claimed`);
+      try {
+        return await this.completeGraphProjection(jobId, await this.graphProjector(batch));
+      } catch (error) {
+        await this.failGraphProjection(jobId, error);
+        throw error;
+      }
     });
-    const extractedEvents = await this.processBlock(block, { retryFailed: false });
-    const processed = this.blocks.find((candidate) => candidate.id === blockId);
-    const readyBlocks = processed?.processingStatus === "ready" ? [processed] : [];
-    const projectedElements = await this.projectEligibleElements() ?? [];
-    await this.projectEligibleGraph();
-    return { sealedBlocks: [], readyBlocks, extractedEvents, projectedElements };
   }
   listElementProjectionJobs() {
     return [...this.elementProjectionJobs.values()];
@@ -3664,9 +3721,9 @@ var StrataGate = class _StrataGate {
       job.updatedAt = toUtc8Iso(this.now());
     });
   }
-  async claimNextGraphProjection() {
+  async claimGraphProjection(jobId) {
     return this.commitMutation(() => {
-      const job = [...this.graphProjectionJobs.values()].filter((candidate) => candidate.status === "pending" || candidate.status === "failed").sort((left, right) => right.priority - left.priority || left.createdAt.localeCompare(right.createdAt))[0];
+      const job = [...this.graphProjectionJobs.values()].filter((candidate) => (jobId === void 0 || candidate.id === jobId) && (candidate.status === "pending" || candidate.status === "failed")).sort((left, right) => right.priority - left.priority || left.createdAt.localeCompare(right.createdAt))[0];
       if (!job) return null;
       const events = job.sourceEventIds.flatMap((id) => this.events.find((event) => event.id === id) ?? []);
       if (events.length === 0) throw new Error(`Graph projection ${job.id} has no available source events`);
@@ -3692,6 +3749,9 @@ var StrataGate = class _StrataGate {
       };
     });
   }
+  async claimNextGraphProjection() {
+    return this.claimGraphProjection();
+  }
   async completeGraphProjection(jobId, result) {
     return this.commitMutation(() => {
       const job = this.requireGraphProjectionJob(jobId);
@@ -5716,6 +5776,7 @@ var StrataGateRuntime = class {
   pendingSurfaceSync = /* @__PURE__ */ new Map();
   adminSnapshotCache = /* @__PURE__ */ new Map();
   externalImportRuns = /* @__PURE__ */ new Map();
+  adminJobRetryRuns = /* @__PURE__ */ new Map();
   feedbackDrafts = /* @__PURE__ */ new Map();
   pendingFeedbackSuggestionSessions = /* @__PURE__ */ new Set();
   pendingTurns = /* @__PURE__ */ new Map();
@@ -6715,34 +6776,53 @@ var StrataGateRuntime = class {
     });
     return update;
   }
-  async adminRetryBlockSummary(namespace, id) {
+  async adminRetryJob(namespace, kind, id) {
     const key = namespace.trim();
-    const blockId = id.trim();
+    const jobId = id.trim();
     if (!key) throw new TypeError("StrataGate admin namespace must not be empty");
-    if (!blockId) throw new TypeError("Block id must not be empty");
+    if (!jobId) throw new TypeError("StrataGate job id must not be empty");
+    const retryKey = `${key}\0${kind}\0${jobId}`;
+    const existing = this.adminJobRetryRuns.get(retryKey);
+    if (existing) return existing;
     const update = this.settingsTail.catch(() => {
     }).then(async () => {
       await this.flush();
       const { memory, owned } = await this.openAdminMemory(key, { derivation: true });
       try {
         await this.refreshExternalImportMemory(key, memory);
-        const block = memory.listBlocks().find((candidate) => candidate.id === blockId);
-        if (!block) throw new Error(`Unknown block: ${blockId}`);
-        const threadId = block.threadId ?? `admin-summary-retry:${blockId}`;
-        const session = block.threadId ? this.knownSessions.get(block.threadId)?.deref() : void 0;
-        const resumed = session && this.namespaceFor(session) === key ? await this.models.run(session, () => memory.retryBlockSummary(blockId)) : await this.models.runDetached(threadId, () => memory.retryBlockSummary(blockId));
+        const graphJob = kind === "graph-projection" ? memory.listGraphProjectionJobs().find((candidate) => candidate.id === jobId) : void 0;
+        const blockId = kind === "graph-projection" ? graphJob?.sourceEventIds.flatMap((eventId) => memory.listEvents().find(({ id: eventCandidateId }) => eventCandidateId === eventId)?.sourceBlockId ?? [])[0] : jobId;
+        const block = blockId ? memory.listBlocks().find((candidate) => candidate.id === blockId) : void 0;
+        if (kind !== "graph-projection" && !block) throw new Error(`Unknown block: ${jobId}`);
+        if (kind === "graph-projection" && !graphJob) throw new Error(`Unknown graph projection: ${jobId}`);
+        const threadId = block?.threadId ?? `admin-job-retry:${kind}:${jobId}`;
+        const session = block?.threadId ? this.knownSessions.get(block.threadId)?.deref() : void 0;
+        const retry = async () => {
+          if (kind === "block-summary") return memory.retryBlockSummary(jobId);
+          if (kind === "event-extraction") return memory.retryEventExtraction(jobId);
+          return memory.retryGraphProjection(jobId);
+        };
+        const result = session && this.namespaceFor(session) === key ? await this.models.run(session, retry) : await this.models.runDetached(threadId, retry);
         await this.persistSuccessfulResponses(memory);
-        if (resumed.readyBlocks.some(({ id: readyId }) => readyId === blockId) && block.threadId) {
-          this.pendingSurfaceSync.set(`${key}\0${blockId}`, { namespace: key, blockId });
+        const currentBlock = blockId ? memory.listBlocks().find(({ id: candidateId }) => candidateId === blockId) : void 0;
+        if (kind !== "graph-projection" && currentBlock?.processingStatus === "ready" && currentBlock.threadId) {
+          this.pendingSurfaceSync.set(`${key}\0${currentBlock.id}`, { namespace: key, blockId: currentBlock.id });
           if (session && this.namespaceFor(session) === key) await this.syncPendingRetrySurface(session, memory);
         }
-        const summaryJob = memory.listSummaryJobs().find(({ blockId: candidateId }) => candidateId === blockId);
+        const job = kind === "block-summary" ? memory.listSummaryJobs().find(({ blockId: candidateId }) => candidateId === jobId) : kind === "event-extraction" ? memory.listExtractionJobs().find(({ blockId: candidateId }) => candidateId === jobId) : memory.listGraphProjectionJobs().find(({ id: candidateId }) => candidateId === jobId);
+        const succeeded = kind === "graph-projection" ? job?.status === "completed" : job?.status === "succeeded" || job?.status === "skipped";
+        if (!succeeded) throw new Error(job?.lastError || `${kind} retry did not complete`);
         return {
+          kind,
+          jobId,
           blockId,
-          processingStatus: block.processingStatus,
-          ready: block.processingStatus === "ready",
-          summaryJob: summaryJob ?? null,
-          surfaceUpdated: !this.pendingSurfaceSync.has(`${key}\0${blockId}`)
+          status: job?.status,
+          job: job ?? null,
+          ...kind === "block-summary" ? { summaryJob: job ?? null } : {},
+          result,
+          processingStatus: currentBlock?.processingStatus ?? null,
+          ready: currentBlock?.processingStatus === "ready",
+          surfaceUpdated: !blockId || !this.pendingSurfaceSync.has(`${key}\0${blockId}`)
         };
       } finally {
         if (owned) await memory.close();
@@ -6751,7 +6831,14 @@ var StrataGateRuntime = class {
     this.settingsTail = update.then(() => {
     }, () => {
     });
-    return update;
+    const tracked = update.finally(() => {
+      if (this.adminJobRetryRuns.get(retryKey) === tracked) this.adminJobRetryRuns.delete(retryKey);
+    });
+    this.adminJobRetryRuns.set(retryKey, tracked);
+    return tracked;
+  }
+  async adminRetryBlockSummary(namespace, id) {
+    return this.adminRetryJob(namespace, "block-summary", id);
   }
   async applyBlockDecayLambda(value) {
     await this.flush();
@@ -7623,7 +7710,7 @@ function clusterKnowledgeGraph(rawNodes, rawEdges) {
 }
 
 // src/web.ts
-var STRATAGATE_DSH_VERSION = "0.2.57";
+var STRATAGATE_DSH_VERSION = "0.2.65";
 var LEGACY_THREAD_ID = "__legacy__";
 var nodeRequire2 = createRequire2(import.meta.url);
 function installedPackageVersion(names) {
@@ -7766,40 +7853,80 @@ async function overview(runtime, cachedEntries) {
   const rows = [];
   for (const { namespace, snapshot } of entries) {
     if (!snapshot) continue;
-    const failedJobs = snapshot.summaryJobs.filter(({ status }) => status === "failed").length + snapshot.extractionJobs.filter(({ status }) => status === "failed").length + snapshot.graphProjectionJobs.filter(({ status }) => status === "failed").length;
-    const processingJobs = snapshot.summaryJobs.filter(({ status, nextRetryAt }) => status === "pending" || status === "running" || status === "failed" && nextRetryAt !== null).length + snapshot.extractionJobs.filter(({ status, nextRetryAt }) => status === "running" || status === "failed" && nextRetryAt !== null).length + snapshot.graphProjectionJobs.filter(({ status }) => status === "pending" || status === "running").length;
-    const failedJobDetails = [
-      ...snapshot.summaryJobs.filter(({ status }) => status === "failed").map((job) => {
-        const block = snapshot.blocks.find(({ id }) => id === job.blockId);
-        return {
-          id: job.blockId,
-          kind: "block-summary",
-          attempts: job.attempts,
-          nextRetryAt: job.nextRetryAt,
-          lastError: job.lastError?.slice(0, 500) ?? null,
-          lastErrorFull: job.lastError,
-          updatedAt: job.updatedAt,
-          threadId: block?.threadId ?? null,
-          sequence: block?.sequence ?? null,
-          turnRange: block ? [block.startTurn, block.endTurn] : null
-        };
-      }),
-      ...snapshot.extractionJobs.filter(({ status }) => status === "failed").map((job) => ({
+    const recovered = recoverSnapshotView(snapshot);
+    const blockDetails = (blockIds) => [...new Set(blockIds)].flatMap((blockId) => {
+      const displayBlocks = recovered.blocks.filter(({ source, threadId }) => source.id === blockId && threadId !== LEGACY_THREAD_ID);
+      if (displayBlocks.length > 0) {
+        return displayBlocks.map((block2) => ({
+          id: block2.id,
+          sourceId: block2.source.id,
+          sequence: block2.source.sequence,
+          title: block2.source.l0Title ?? null,
+          threadId: block2.threadId,
+          turnRange: block2.turnRange,
+          shouldExtract: block2.source.shouldExtract ?? null
+        }));
+      }
+      const block = snapshot.blocks.find(({ id }) => id === blockId);
+      return block ? [{
+        id: block.id,
+        sourceId: block.id,
+        sequence: block.sequence,
+        title: block.l0Title ?? null,
+        threadId: block.threadId ?? null,
+        turnRange: [block.startTurn, block.endTurn],
+        shouldExtract: block.shouldExtract ?? null
+      }] : [];
+    });
+    const describeBlockJob = (kind, job) => {
+      const details = blockDetails([job.blockId]);
+      return {
         id: job.blockId,
-        kind: "event-extraction",
+        kind,
+        status: job.status,
         attempts: job.attempts,
+        nextRetryAt: job.nextRetryAt,
         lastError: job.lastError?.slice(0, 500) ?? null,
         lastErrorFull: job.lastError,
-        updatedAt: job.updatedAt
-      })),
-      ...snapshot.graphProjectionJobs.filter(({ status }) => status === "failed").map((job) => ({
+        updatedAt: job.updatedAt,
+        threadId: details[0]?.threadId ?? null,
+        blockIds: [job.blockId],
+        threadIds: [...new Set(details.flatMap(({ threadId }) => threadId ?? []))],
+        blockDetails: details,
+        sequence: details[0]?.sequence ?? null,
+        turnRange: details[0]?.turnRange ?? null
+      };
+    };
+    const describeGraphJob = (job) => {
+      const blockIds = [...new Set(job.sourceEventIds.flatMap((eventId) => snapshot.events.find(({ id }) => id === eventId)?.sourceBlockId ?? []))];
+      const details = blockDetails(blockIds);
+      return {
         id: job.id,
         kind: "graph-projection",
+        status: job.status,
         attempts: job.attempts,
+        nextRetryAt: null,
         lastError: job.lastError?.slice(0, 500) ?? null,
         lastErrorFull: job.lastError,
-        updatedAt: job.updatedAt
-      }))
+        updatedAt: job.updatedAt,
+        blockIds,
+        threadIds: [...new Set(details.flatMap(({ threadId }) => threadId ?? []))],
+        blockDetails: details,
+        sequences: details.map(({ sequence }) => sequence),
+        sourceEventIds: job.sourceEventIds
+      };
+    };
+    const failedJobs = snapshot.summaryJobs.filter(({ status }) => status === "failed").length + snapshot.extractionJobs.filter(({ status }) => status === "failed").length + snapshot.graphProjectionJobs.filter(({ status }) => status === "failed").length;
+    const processingJobs = snapshot.summaryJobs.filter(({ status, nextRetryAt }) => status === "pending" || status === "running" || status === "failed" && nextRetryAt !== null).length + snapshot.extractionJobs.filter(({ status, nextRetryAt }) => status === "running" || status === "failed" && nextRetryAt !== null).length + snapshot.graphProjectionJobs.filter(({ status }) => status === "pending" || status === "running").length;
+    const failedJobDetails = [
+      ...snapshot.summaryJobs.filter(({ status }) => status === "failed").map((job) => describeBlockJob("block-summary", job)),
+      ...snapshot.extractionJobs.filter(({ status }) => status === "failed").map((job) => describeBlockJob("event-extraction", job)),
+      ...snapshot.graphProjectionJobs.filter(({ status }) => status === "failed").map(describeGraphJob)
+    ];
+    const processingJobDetails = [
+      ...snapshot.summaryJobs.filter(({ status, nextRetryAt }) => status === "pending" || status === "running" || status === "failed" && nextRetryAt !== null).map((job) => describeBlockJob("block-summary", job)),
+      ...snapshot.extractionJobs.filter(({ status, nextRetryAt }) => status === "running" || status === "failed" && nextRetryAt !== null).map((job) => describeBlockJob("event-extraction", job)),
+      ...snapshot.graphProjectionJobs.filter(({ status }) => status === "pending" || status === "running").map(describeGraphJob)
     ];
     const timestamps = [
       ...snapshot.blocks.map(({ createdAt }) => createdAt),
@@ -7834,6 +7961,7 @@ async function overview(runtime, cachedEntries) {
       failedJobs,
       processingJobs,
       failedJobDetails,
+      processingJobDetails,
       successfulModelResponses: snapshot.successfulModelResponses ?? [],
       lastActivityAt: timestamps.at(-1) ?? null
     });
@@ -8345,11 +8473,33 @@ async function retryBlockSummary(runtime, url) {
   const snapshot = await requiredSnapshot(runtime, namespace);
   const job = snapshot.summaryJobs.find(({ blockId: candidateId }) => candidateId === blockId);
   if (!job) throw new AdminHttpError(404, `Unknown Block Summary job: ${blockId}`);
-  if (job.status !== "failed" || job.nextRetryAt !== null) {
-    throw new AdminHttpError(409, "Block Summary is not in a terminal failed state");
-  }
+  if (job.status !== "failed") throw new AdminHttpError(409, `Block Summary is ${job.status}, not failed`);
   return runtime.adminRetryBlockSummary(namespace, blockId);
 }
+async function retryJob(runtime, url) {
+  const namespace = url.searchParams.get("namespace")?.trim() ?? "";
+  const kind = url.searchParams.get("kind")?.trim() ?? "";
+  const jobId = url.searchParams.get("jobId")?.trim() ?? "";
+  if (!namespace) throw new AdminHttpError(400, "namespace is required");
+  if (!["block-summary", "event-extraction", "graph-projection"].includes(kind)) {
+    throw new AdminHttpError(400, "kind must be block-summary, event-extraction, or graph-projection");
+  }
+  if (!jobId) throw new AdminHttpError(400, "jobId is required");
+  if (jobId.startsWith("virtual:")) throw new AdminHttpError(409, "Recovered legacy fragments cannot be retried");
+  const snapshot = await requiredSnapshot(runtime, namespace);
+  const job = kind === "block-summary" ? snapshot.summaryJobs.find(({ blockId }) => blockId === jobId) : kind === "event-extraction" ? snapshot.extractionJobs.find(({ blockId }) => blockId === jobId) : snapshot.graphProjectionJobs.find(({ id }) => id === jobId);
+  if (!job) throw new AdminHttpError(404, `Unknown ${kind} job: ${jobId}`);
+  if (job.status !== "failed") throw new AdminHttpError(409, `${kind} job is ${job.status}, not failed`);
+  try {
+    return await runtime.adminRetryJob(
+      namespace,
+      kind,
+      jobId
+    );
+  } catch (error) {
+    throw new AdminHttpError(422, error instanceof Error ? error.message : String(error));
+  }
+}
 function receiptSources(snapshot, receipt) {
   const events = snapshot.events.filter(({ id }) => receipt.eventIds.includes(id));
   const elements = snapshot.elements.filter(({ id }) => receipt.elementIds.includes(id));
@@ -8390,7 +8540,7 @@ async function dashboard(runtime, url, ifNoneMatch) {
   const selected = entries.find(({ namespace }) => namespace === requestedNamespace) ?? entries[0];
   const threadId = url.searchParams.get("threadId")?.trim() ?? "";
   const revisionKey = entries.map(({ namespace, revision }) => `${namespace}:${revision}`).join("|");
-  const etag = `"${createHash2("sha256").update(`${revisionKey}\0${selected?.namespace ?? ""}\0${threadId}`).digest("base64url").slice(0, 24)}"`;
+  const etag = `"${createHash2("sha256").update(`${STRATAGATE_DSH_VERSION}\0${revisionKey}\0${selected?.namespace ?? ""}\0${threadId}`).digest("base64url").slice(0, 24)}"`;
   if (ifNoneMatch.split(",").map((value) => value.trim()).includes(etag)) return { etag, notModified: true };
   const overviewValue = await overview(runtime, entries);
   if (!selected) {
@@ -8465,6 +8615,9 @@ async function handleAdminRequest(runtime, req, res) {
     } else if (path === "/api/stratagate/blocks/retry-summary") {
       if (req.method !== "POST") throw new AdminHttpError(405, "StrataGate Block Summary retry requires POST");
       sendJson(res, 200, await retryBlockSummary(runtime, url));
+    } else if (path === "/api/stratagate/jobs/retry") {
+      if (req.method !== "POST") throw new AdminHttpError(405, "StrataGate job retry requires POST");
+      sendJson(res, 200, await retryJob(runtime, url));
     } else if (path === "/api/stratagate/import") {
       if (req.method === "GET") {
         const operation = url.searchParams.get("operation");