projectkeeper 0.0.0-stage → 1.0.0

package/src/server/http.ts

@@ -0,0 +1,224 @@
+/**
+ * Minimal HTTP layer: JSON routes, server-sent events, static files. Bound to 127.0.0.1, and answering only requests
+ * made to that local address by the workbench's own pages or by a program on this machine (`refusal` below).
+ */
+import { createReadStream, existsSync, statSync } from 'node:fs';
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
+import { extname, join, normalize, resolve } from 'node:path';
+
+export interface RouteContext {
+  readonly req: IncomingMessage;
+  readonly res: ServerResponse;
+  readonly params: Record<string, string>;
+  readonly query: URLSearchParams;
+  readonly body: unknown;
+}
+export type Handler = (ctx: RouteContext) => unknown | Promise<unknown>;
+
+interface Route { method: string; pattern: RegExp; keys: string[]; handler: Handler }
+
+export class HttpError extends Error {
+  readonly status: number;
+  constructor(status: number, message: string) { super(message); this.status = status; }
+}
+
+const MIME: Record<string, string> = {
+  '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.mjs': 'text/javascript; charset=utf-8',
+  '.css': 'text/css; charset=utf-8', '.json': 'application/json; charset=utf-8', '.svg': 'image/svg+xml',
+  '.png': 'image/png', '.ico': 'image/x-icon', '.md': 'text/markdown; charset=utf-8', '.txt': 'text/plain; charset=utf-8',
+  '.woff2': 'font/woff2', '.map': 'application/json',
+};
+
+// ───────────────────────── who is answered ─────────────────────────
+//
+// Listening on 127.0.0.1 keeps other machines out. It does not keep out a web page open in this machine's browser: the
+// browser will send a request to a local port for any page that asks. Its same-origin rule hides the answer from that
+// page, but the request has been carried out by then, and a page whose own name is made to resolve to 127.0.0.1 ("DNS
+// rebinding") counts as the same origin and reads the answers too. So every request, to a route or to a file, is
+// checked before anything is done with it. There is no login: a program running on this machine as the user is
+// answered, as docs/privacy.md says. There is no setting that widens this.
+
+/** The names a browser on this machine can reach the workbench by. A name of the user's own (a hosts-file alias, a
+ *  proxy in front) is not among them: nothing tells such a name from one an outside page made up. */
+const LOCAL_NAMES = ['127.0.0.1', 'localhost', '[::1]'];
+
+/** `Host` as the workbench's own address is written for the port a request arrived on (a browser leaves `:80` out). */
+function localHosts(port: number | undefined): string[] {
+  return LOCAL_NAMES.flatMap((name) => (port === 80 ? [name, `${name}:80`] : [`${name}:${port}`]));
+}
+
+const one = (value: string | string[] | undefined): string | undefined => (Array.isArray(value) ? value[0] : value);
+
+/**
+ * Why a request is not answered, or null when it is. In order:
+ *
+ *  - `Host` has to be the workbench's own local address with the port the request arrived on. A page reaching this
+ *    port under a name of its own sends that name. HTTP/1.0 may send no `Host` at all: refused too.
+ *  - `Origin`, when a request carries one, has to be that same address: the page that made the request was served
+ *    from here. Browsers send it with every request another origin's script makes and with every POST; a program that
+ *    is not a browser (`pk`, a test, curl) sends none.
+ *  - `Sec-Fetch-Site`, which browsers add themselves and a page cannot set, has to be `same-origin` (a page of the
+ *    workbench) or `none` (the address typed, a bookmark, a link opened from another program). `same-site` is
+ *    another origin as well — another port of localhost — and is refused like `cross-site`, which covers what carries
+ *    no `Origin`: an image, a script or a frame another page points here. One thing from elsewhere is answered: a
+ *    link followed to a page of the workbench, in a tab of its own; that gives the other page nothing to read and
+ *    starts nothing. The API is never answered that way, and neither is a frame.
+ *  - A request with a body says it is `application/json`. A page can post to another origin without the browser
+ *    asking first only as a form or as plain text, so this refuses such a post even where the headers above are absent.
+ */
+export function refusal(req: IncomingMessage, pathname: string | null): { status: 403 | 415; message: string } | null {
+  const port = req.socket.localPort;
+  const notHere = { status: 403 as const, message: `The ProjectKeeper workbench answers only at its own local address, http://127.0.0.1:${port}/, and only to its own pages.` };
+  const host = one(req.headers.host)?.toLowerCase();
+  if (host === undefined || !localHosts(port).includes(host)) return notHere;
+  const origin = one(req.headers.origin);
+  if (origin !== undefined && origin.toLowerCase() !== `http://${host}`) return notHere;
+  const site = one(req.headers['sec-fetch-site'])?.toLowerCase();
+  if (site !== undefined && site !== 'same-origin' && site !== 'none') {
+    const method = (req.method ?? 'GET').toUpperCase();
+    const followedLink = (method === 'GET' || method === 'HEAD') && one(req.headers['sec-fetch-mode']) === 'navigate' && one(req.headers['sec-fetch-dest']) === 'document';
+    if (!followedLink || pathname === null || pathname === '/api' || pathname.startsWith('/api/')) return notHere;
+  }
+  const hasBody = Number(req.headers['content-length'] ?? 0) > 0 || req.headers['transfer-encoding'] !== undefined;
+  if (hasBody && (one(req.headers['content-type']) ?? '').split(';')[0]!.trim().toLowerCase() !== 'application/json') {
+    return { status: 415, message: 'A request with a body must be sent as application/json.' };
+  }
+  return null;
+}
+
+export class SseHub {
+  private readonly clients = new Set<ServerResponse>();
+
+  attach(res: ServerResponse): void {
+    res.writeHead(200, {
+      'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive',
+      'X-Accel-Buffering': 'no',
+    });
+    res.write(': connected\n\n');
+    this.clients.add(res);
+    const ping = setInterval(() => { if (!res.writableEnded) res.write(': ping\n\n'); }, 25_000);
+    ping.unref?.();
+    res.on('close', () => { clearInterval(ping); this.clients.delete(res); });
+  }
+
+  broadcast(event: string, data: unknown): void {
+    const payload = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+    for (const client of this.clients) {
+      if (!client.writableEnded) client.write(payload);
+    }
+  }
+
+  get size(): number { return this.clients.size; }
+}
+
+export class HttpApp {
+  private readonly routes: Route[] = [];
+  private readonly statics: { prefix: string; dir: string }[] = [];
+  readonly sse = new SseHub();
+  private server: Server | null = null;
+
+  route(method: string, path: string, handler: Handler): void {
+    const keys: string[] = [];
+    const pattern = new RegExp('^' + path.replace(/\/:([A-Za-z_]+)/g, (_m, key: string) => { keys.push(key); return '/([^/]+)'; }) + '/?$');
+    this.routes.push({ method, pattern, keys, handler });
+  }
+
+  static(prefix: string, dir: string): void {
+    this.statics.push({ prefix, dir: resolve(dir) });
+  }
+
+  private async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    // A request line that is no path (`GET //`) is not a URL; it is answered as that, after the check of who asks, and
+    // never thrown from here, where nothing would catch it and the process would end.
+    let url: URL | null = null;
+    try { url = new URL(req.url ?? '/', 'http://127.0.0.1'); } catch { /* said below */ }
+    const method = (req.method ?? 'GET').toUpperCase();
+    const refused = refusal(req, url?.pathname ?? null) ?? (url ? null : { status: 400, message: 'The request names no path.' });
+    if (refused || !url) {
+      // Said in plain text: the one who reads it is a person who opened the workbench under another name. Nothing of
+      // the request is repeated in it, and the connection is not kept for a body that was not read.
+      res.writeHead(refused?.status ?? 400, { 'Content-Type': 'text/plain; charset=utf-8', 'Cache-Control': 'no-store', Connection: 'close' });
+      res.end(`${refused?.message ?? 'The request names no path.'}\n`);
+      return;
+    }
+    try {
+      for (const route of this.routes) {
+        if (route.method !== method) continue;
+        const match = route.pattern.exec(url.pathname);
+        if (!match) continue;
+        const params: Record<string, string> = {};
+        route.keys.forEach((key, i) => { params[key] = decodeURIComponent(match[i + 1] ?? ''); });
+        const body = method === 'GET' || method === 'HEAD' ? undefined : await readBody(req);
+        const result = await route.handler({ req, res, params, query: url.searchParams, body });
+        if (res.writableEnded || res.headersSent) return;
+        if (result === undefined) { res.writeHead(204); res.end(); return; }
+        const json = JSON.stringify(result);
+        res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' });
+        res.end(json);
+        return;
+      }
+      if (method === 'GET' || method === 'HEAD') {
+        for (const s of this.statics) {
+          if (!url.pathname.startsWith(s.prefix)) continue;
+          const rel = decodeURIComponent(url.pathname.slice(s.prefix.length)) || 'index.html';
+          const file = normalize(join(s.dir, rel));
+          if (!file.startsWith(s.dir)) { res.writeHead(403); res.end(); return; }
+          const target = existsSync(file) && statSync(file).isDirectory() ? join(file, 'index.html') : file;
+          if (!existsSync(target) || !statSync(target).isFile()) continue;
+          res.writeHead(200, { 'Content-Type': MIME[extname(target).toLowerCase()] ?? 'application/octet-stream', 'Cache-Control': 'no-cache' });
+          if (method === 'HEAD') { res.end(); return; }
+          createReadStream(target).pipe(res);
+          return;
+        }
+      }
+      res.writeHead(404, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: `Not found: ${method} ${url.pathname}` }));
+    } catch (error) {
+      const status = error instanceof HttpError ? error.status : 500;
+      const message = error instanceof Error ? error.message : String(error);
+      if (!res.headersSent) {
+        res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
+        res.end(JSON.stringify({ error: message }));
+      } else if (!res.writableEnded) {
+        res.end();
+      }
+      if (status === 500) console.error(`[http] ${method} ${url.pathname}: ${error instanceof Error ? error.stack : String(error)}`);
+    }
+  }
+
+  /** Always on 127.0.0.1: the address is not a parameter, an option or a setting. */
+  listen(port: number): Promise<{ port: number; close: () => Promise<void> }> {
+    return new Promise((resolvePromise, reject) => {
+      const server = createServer((req, res) => { void this.handle(req, res); });
+      server.on('error', reject);
+      server.listen(port, '127.0.0.1', () => {
+        this.server = server;
+        const address = server.address();
+        const actualPort = typeof address === 'object' && address ? address.port : port;
+        resolvePromise({ port: actualPort, close: () => new Promise((r) => server.close(() => r())) });
+      });
+    });
+  }
+}
+
+function readBody(req: IncomingMessage): Promise<unknown> {
+  return new Promise((resolvePromise, reject) => {
+    const chunks: Buffer[] = [];
+    req.on('data', (c: Buffer) => chunks.push(c));
+    req.on('end', () => {
+      const text = Buffer.concat(chunks).toString('utf8');
+      if (!text.trim()) { resolvePromise(undefined); return; }
+      try { resolvePromise(JSON.parse(text)); } catch { reject(new HttpError(400, 'Body is not valid JSON')); }
+    });
+    req.on('error', reject);
+  });
+}
+
+export function requireString(value: unknown, name: string): string {
+  if (typeof value !== 'string' || !value.trim()) throw new HttpError(400, `${name} is required`);
+  return value.trim();
+}
+
+export function optionalString(value: unknown): string | null {
+  return typeof value === 'string' && value.trim() ? value.trim() : null;
+}

package/src/server/k-agent-api.ts

@@ -0,0 +1,56 @@
+/** Read-only increment K material for `pk get` and `pk options` (Spec §7.10). */
+import type { App } from './app.ts';
+import { HttpError, type HttpApp } from './http.ts';
+import { processView } from './k-views.ts';
+import { breakpointBrief, patchBrief, sendBackBrief, territoryBrief } from '../context/k-briefs.ts';
+import { redactCredentials } from '../sources/anchor.ts';
+import type { ProjectStore } from '../store/project-store.ts';
+
+function objectName(store: ProjectStore, id: string): string {
+  return store.threads.get(id)?.title ?? store.reference.get(id)?.name ?? store.nodes.get(id)?.label
+    ?? store.territories.get(id)?.name ?? store.patches.get(id)?.title ?? id;
+}
+
+const safe = (text: string) => redactCredentials(text).text;
+
+export function registerKAgentRoutes(http: HttpApp, app: App): void {
+  http.route('GET', '/api/projects/:id/k-briefs/:oid', ({ params }) => {
+    const project = app.project(params.id!);
+    const store = app.store(project.id);
+    const id = params.oid!;
+    const b = store.breakpoints.get(id);
+    if (b) {
+      const view = processView(store, project, app.kEngines).breakpoints.find((item) => item.id === id)!;
+      return { text: breakpointBrief(view, objectName(store, view.targetId)) };
+    }
+    const sb = store.sendbacks.get(id);
+    if (sb) {
+      const view = processView(store, project, app.kEngines).sendBacks.find((item) => item.id === id)!;
+      return { text: sendBackBrief(view, objectName(store, view.targetId)) };
+    }
+    const patch = store.patches.get(id);
+    if (patch) return { text: patchBrief(patch, (oid) => objectName(store, oid)) };
+    const territory = store.territories.get(id);
+    if (territory) {
+      const code = app.kEngines.ledger?.code(store, project) ?? null;
+      const view = code?.territories.find((item) => item.id === id);
+      const ledger = code && view ? { view, version: code.version } : null;
+      // CodeTerritory has no Occurred field. Use its first trace when kept, or the first observation retained in the
+      // saved record, and never claim `updatedAt` is a commit or a date in the code's history.
+      const firstSeen = (store.traceFor('territories', id, Number.MAX_SAFE_INTEGER)[0]?.at ?? territory.updatedAt).slice(0, 10) || 'unknown';
+      return { text: territoryBrief(territory, (oid) => objectName(store, oid), ledger, firstSeen) };
+    }
+    throw new HttpError(404, `No breakpoint, send-back, semantic patch or code territory has the id ${id}`);
+  });
+
+  http.route('GET', '/api/projects/:id/k-options', ({ params }) => {
+    const project = app.project(params.id!);
+    const store = app.store(project.id);
+    const view = processView(store, project, app.kEngines);
+    return {
+      breakpoints: view.breakpoints.filter((b) => b.lit).map((b) => ({ id: b.id, kind: b.kind, targetId: b.targetId, targetName: safe(objectName(store, b.targetId)) })),
+      sendBacks: view.sendBacks.filter((s) => s.lit).map((s) => ({ id: s.id, stage: s.stage, to: s.to, what: safe(s.what), targetId: s.targetId, targetName: safe(objectName(store, s.targetId)) })),
+      sixThings: view.sixThings.map((s) => ({ thing: s.thing, objects: s.objectIds.map((id) => ({ id, name: safe(objectName(store, id)) })) })),
+    };
+  });
+}

package/src/server/k-rounds-d99.ts

@@ -0,0 +1,91 @@
+/**
+ * A round run by the main agent (D99; Spec §3.3, §3.10, §6.9) as the Keeper view's round tree shows it: the main agent with
+ * the stage it is in and each stage's time, the lanes it sent — what each answers, the slots it writes, its status, what it
+ * read, its brief and its report — the coverage check with each account, and the missing steps the lanes looked for and
+ * the spot-check confirmed. Pure reads of the round record, its jobs, its documents and the breakpoints.
+ */
+import type { ProjectStore } from '../store/project-store.ts';
+import type { KeeperJob, Usage } from '../model/types.ts';
+import type { ClerkRound, RoundDoc } from '../model/k-types.ts';
+import type { RoundCoverageView, RoundDocRef, RoundLaneView, RoundMainView, RoundView } from '../model/views-k.ts';
+
+/** A round the main agent runs: it has a stage (null before orientation) or a stage log; a round from before D99 has neither. */
+export const isMainAgentRound = (r: ClerkRound): boolean => r.stage !== undefined || r.stageLog !== undefined;
+
+const docRef = (d: RoundDoc): RoundDocRef => ({ id: d.id, kind: d.kind, title: d.title, path: d.path, at: d.at });
+
+/** The question a lane answers, as its brief opens: the first lines that are not headings. */
+export function questionOf(markdown: string, lines = 3): string | null {
+  const text = markdown.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !/^#{1,6}\s/.test(l) && !/^[-*_]{3,}$/.test(l)).slice(0, lines);
+  return text.length ? text.join('\n') : null;
+}
+
+/**
+ * What jobs read, each material once, from their steps' recorded reads (`JobStep.reads`): a file as it stands, a version
+ * of a file at a commit, a commit, a session. A search reads nothing; a step recorded before reads were kept counts none.
+ */
+export function readsOf(jobs: Iterable<KeeperJob>): RoundLaneView['read'] {
+  const files = new Set<string>(), versions = new Set<string>(), commits = new Set<string>(), sessions = new Set<string>();
+  for (const j of jobs) for (const s of j.steps) for (const r of s.reads ?? []) {
+    if (r.session) sessions.add(r.session);
+    else if (r.path && r.rev) versions.add(`${r.path.toLowerCase()}@${r.rev.toLowerCase().slice(0, 12)}`);
+    else if (r.path) files.add(r.path.toLowerCase());
+    else if (r.rev) commits.add(r.rev.toLowerCase().slice(0, 12));
+  }
+  return { files: files.size, versions: versions.size, commits: commits.size, sessions: sessions.size };
+}
+
+/** A job and every job under it (the investigations a lane sent). */
+export function withSent(store: ProjectStore, job: KeeperJob): KeeperJob[] {
+  const kids = store.jobs.filter((c) => c.parentJobId === job.id);
+  return [job, ...kids.flatMap((k) => withSent(store, k))];
+}
+
+/** The parts of `RoundView` a main-agent round adds; `usageOf` sums a set of jobs as the round's usage does. */
+export function mainAgentParts(store: ProjectStore, r: ClerkRound, jobs: readonly KeeperJob[], usageOf: (jobs: readonly KeeperJob[]) => Usage): Pick<RoundView, 'main' | 'lanes' | 'coverage' | 'missing'> {
+  const docs = store.roundDocs.filter((d) => d.roundId === r.id);
+  const mainJob = jobs.filter((j) => j.step!.kind === 'main').sort((a, b) => a.queuedAt.localeCompare(b.queuedAt)).pop() ?? null;
+  const log = r.stageLog ?? [];
+  // D103: once handed over, the main agent is in no stage any more; its Handover document opens from the tree.
+  const handoverDoc = r.handover ? store.roundDocs.get(r.handover.docId) : undefined;
+  const main: RoundMainView = {
+    jobId: mainJob?.id ?? null, stage: r.stage ?? null,
+    stages: log.map((e, i) => ({ stage: e.stage, startedAt: e.startedAt, endedAt: e.endedAt, timing: e.timing, current: i === log.length - 1 && e.endedAt === null && e.stage === r.stage })),
+    ...(r.handover ? { handover: { at: r.handover.at, doc: handoverDoc ? docRef(handoverDoc) : null } } : {}),
+  };
+  // The lanes the round records, in the order sent; a lane job the record does not name yet (sent a moment ago) after them.
+  const recorded = r.lanes ?? [];
+  const laneJobs = jobs.filter((j) => j.step!.kind === 'lane');
+  const extra = laneJobs.filter((j) => !recorded.some((l) => l.jobId === j.id || l.name === j.step!.path));
+  const lanes: RoundLaneView[] = [
+    ...recorded.map((l) => ({ lane: l, job: laneJobs.find((j) => j.id === l.jobId) ?? store.jobs.get(l.jobId) ?? null })),
+    ...extra.map((j) => ({ lane: null, job: j })),
+  ].map(({ lane, job }) => {
+    const name = lane?.name ?? job!.step!.path ?? job!.scope.label;
+    const brief = (lane ? store.roundDocs.get(lane.briefDocId) : undefined) ?? docs.find((d) => d.kind === 'Brief' && d.path === name && d.jobId !== job?.id);
+    const report = (lane?.reportDocId ? store.roundDocs.get(lane.reportDocId) : undefined)
+      ?? docs.filter((d) => d.kind === 'Report' && (d.path === name || (job !== null && d.jobId === job.id))).sort((a, b) => a.at.localeCompare(b.at)).pop();
+    const sent = job ? withSent(store, job) : [];
+    return {
+      name, kind: lane?.kind ?? job!.step!.lane?.kind ?? 'topic', slots: lane?.slots ?? job!.step!.lane?.slots ?? [],
+      stage: lane?.stage ?? null, sentAt: lane?.sentAt ?? job?.queuedAt ?? null,
+      question: brief ? questionOf(brief.markdown) : null,
+      jobId: job?.id ?? lane?.jobId ?? null, status: job?.status ?? 'Queued', model: job?.model ?? null,
+      startedAt: job?.startedAt ?? null, endedAt: job?.endedAt ?? null, timing: job?.timing ?? null,
+      usage: sent.length ? usageOf(sent) : null,
+      brief: brief ? docRef(brief) : null, report: report ? docRef(report) : null,
+      read: readsOf(sent),
+    };
+  });
+  const c = r.coverage ?? null;
+  const coverage: RoundCoverageView | null = c ? { at: c.at, settled: c.settled, untouched: c.untouched ?? null, accounted: c.accounted } : null;
+  const bps = store.breakpoints.all();
+  const missing = { looked: bps.filter((b) => b.looked?.roundId === r.id).length, checked: bps.filter((b) => b.checked?.roundId === r.id).length };
+  return { main, lanes, coverage, missing };
+}
+
+/** The main job's own documents, less the briefs of its lanes: those stand under each lane. */
+export function mainDocsOf(docs: readonly RoundDocRef[], lanes: readonly RoundLaneView[]): RoundDocRef[] {
+  const briefs = new Set(lanes.flatMap((l) => (l.brief ? [l.brief.id] : [])));
+  return docs.filter((d) => !briefs.has(d.id));
+}