takos-control 1.0.0

package/src/application/services/execution/workflow-run-lifecycle.ts

@@ -0,0 +1,389 @@
+/**
+ * Workflow Engine – run lifecycle (start, cancel, finalize)
+ */
+
+import type { Workflow, Conclusion } from 'takos-actions-engine';
+import {
+  parseWorkflow,
+  validateWorkflow,
+  createExecutionPlan,
+} from 'takos-actions-engine';
+import { generateId, now } from '../../../shared/utils';
+import * as gitStore from '../git-smart';
+import { getDb, workflowRuns, workflowJobs, workflowSteps, workflows, workflowSecrets } from '../../../infra/db';
+import { eq, and, ne, max, inArray, count } from 'drizzle-orm';
+import { buildWorkflowDispatchEnv } from '../actions';
+import { BadRequestError, InternalError, NotFoundError, ValidationError } from 'takos-common/errors';
+import type { WorkflowJobDefinition, WorkflowJobQueueMessage } from '../../../shared/types';
+import type { WorkflowBucket, StartRunOptions, WorkflowRunRecord } from './workflow-engine-types';
+import type { D1Database, Queue } from '../../../shared/types/bindings.ts';
+import { toWorkflowJobDefinition, toRunRecord } from './workflow-engine-converters';
+import { WORKFLOW_QUEUE_MESSAGE_VERSION } from '../../../shared/types';
+import { logWarn } from '../../../shared/utils/logger';
+
+// ---------------------------------------------------------------------------
+// startRun
+// ---------------------------------------------------------------------------
+
+export async function startRun(
+  db: D1Database,
+  bucket: WorkflowBucket,
+  queue: Queue<WorkflowJobQueueMessage> | undefined,
+  options: StartRunOptions,
+): Promise<WorkflowRunRecord> {
+  const drizzle = getDb(db);
+  const { repoId, workflowPath, event, ref, sha, inputs, actorId } = options;
+
+  const commitSha = await gitStore.resolveRef(db, repoId, ref);
+  if (!commitSha) {
+    throw new NotFoundError(`Ref ${ref}`);
+  }
+
+  const commit = await gitStore.getCommitData(bucket, commitSha);
+  if (!commit) {
+    throw new NotFoundError(`Commit ${commitSha}`);
+  }
+
+  const blob = await gitStore.getBlobAtPath(bucket, commit.tree, workflowPath);
+  if (!blob) {
+    throw new NotFoundError(`Workflow file ${workflowPath}`);
+  }
+
+  const content = new TextDecoder().decode(blob);
+
+  const { workflow, diagnostics } = parseWorkflow(content);
+  const errors = diagnostics.filter((d) => d.severity === 'error');
+  if (errors.length > 0) {
+    throw new ValidationError(`Workflow parse errors: ${errors.map((e) => e.message).join(', ')}`);
+  }
+
+  const validation = validateWorkflow(workflow);
+  const validationErrors = validation.diagnostics.filter((d) => d.severity === 'error');
+  if (validationErrors.length > 0) {
+    throw new ValidationError(`Workflow validation errors: ${validationErrors.map((e) => e.message).join(', ')}`);
+  }
+
+  if (!checkTrigger(workflow, event)) {
+    throw new BadRequestError(`Workflow does not support event: ${event}`);
+  }
+
+  const lastRun = await drizzle.select({ maxRunNumber: max(workflowRuns.runNumber) })
+    .from(workflowRuns)
+    .where(and(eq(workflowRuns.repoId, repoId), eq(workflowRuns.workflowPath, workflowPath)))
+    .get();
+
+  const runNumber = (lastRun?.maxRunNumber || 0) + 1;
+  const timestamp = now();
+  const runId = generateId();
+
+  const workflowId = await resolveWorkflowId(db, repoId, workflowPath);
+
+  await drizzle.insert(workflowRuns)
+    .values({
+      id: runId,
+      repoId,
+      workflowId,
+      workflowPath,
+      event,
+      ref: `refs/heads/${ref}`,
+      sha: sha || commitSha,
+      actorAccountId: actorId,
+      status: 'queued',
+      queuedAt: timestamp,
+      inputs: inputs ? JSON.stringify(inputs) : null,
+      runNumber,
+      runAttempt: 1,
+      createdAt: timestamp,
+    })
+    .run();
+
+  const plan = createExecutionPlan(workflow);
+
+  const jobEntries = Object.entries(workflow.jobs).map(([jobKey, jobDef]) => ({
+    id: generateId(),
+    key: jobKey,
+    definition: jobDef,
+    name: jobDef.name || jobKey,
+  }));
+
+  await Promise.all(jobEntries.map(async (entry) => {
+    await drizzle.insert(workflowJobs)
+      .values({
+        id: entry.id,
+        runId,
+        jobKey: entry.key,
+        name: entry.name,
+        status: 'queued',
+        queuedAt: timestamp,
+        createdAt: timestamp,
+      })
+      .run();
+
+    await Promise.all(entry.definition.steps.map(async (step, i) => {
+      const stepId = generateId();
+      const stepName = step.name || step.uses || step.run?.slice(0, 50) || `Step ${i + 1}`;
+      await drizzle.insert(workflowSteps)
+        .values({
+          id: stepId,
+          jobId: entry.id,
+          number: i + 1,
+          name: stepName,
+          status: 'pending',
+          runCommand: step.run || null,
+          usesAction: step.uses || null,
+          createdAt: timestamp,
+        })
+        .run();
+      return Promise.resolve();
+    }));
+  }));
+
+  const jobRecords = jobEntries.map((entry) => ({
+    id: entry.id,
+    key: entry.key,
+    definition: entry.definition,
+  }));
+
+  if (queue) {
+    const secretIds = await getSecretIds(db, repoId);
+
+    const firstPhaseJobs = plan.phases[0] || [];
+
+    for (const jobKey of firstPhaseJobs) {
+      const jobRecord = jobRecords.find((j) => j.key === jobKey);
+      if (!jobRecord) continue;
+
+      const queuedJobDefinition: WorkflowJobDefinition = toWorkflowJobDefinition(jobRecord.definition);
+      const dispatchEnv = buildWorkflowDispatchEnv({
+        workflow,
+        workflowPath,
+        repoId,
+        runId,
+        ref,
+        sha: sha || commitSha,
+        jobKey,
+        jobId: jobRecord.id,
+        jobDefinition: jobRecord.definition,
+      });
+
+      await enqueueJob(queue, {
+        runId,
+        jobId: jobRecord.id,
+        repoId,
+        ref,
+        sha: sha || commitSha,
+        jobKey,
+        jobDefinition: queuedJobDefinition,
+        env: dispatchEnv,
+        secretIds,
+      });
+    }
+  }
+
+  const run = await drizzle.select()
+    .from(workflowRuns)
+    .where(eq(workflowRuns.id, runId))
+    .get();
+
+  if (!run) {
+    throw new InternalError(`Workflow run not found after creation: ${runId}`);
+  }
+
+  return toRunRecord(run);
+}
+
+// ---------------------------------------------------------------------------
+// cancelRun
+// ---------------------------------------------------------------------------
+
+export async function cancelRun(db: D1Database, runId: string): Promise<void> {
+  const drizzle = getDb(db);
+  const timestamp = now();
+
+  await drizzle.update(workflowRuns)
+    .set({
+      status: 'cancelled',
+      conclusion: 'cancelled',
+      completedAt: timestamp,
+    })
+    .where(
+      and(
+        eq(workflowRuns.id, runId),
+        inArray(workflowRuns.status, ['queued', 'in_progress', 'waiting']),
+      )
+    )
+    .run();
+
+  await drizzle.update(workflowJobs)
+    .set({
+      status: 'completed',
+      conclusion: 'cancelled',
+      completedAt: timestamp,
+    })
+    .where(
+      and(
+        eq(workflowJobs.runId, runId),
+        inArray(workflowJobs.status, ['queued', 'in_progress', 'waiting']),
+      )
+    )
+    .run();
+
+  const jobs = await drizzle.select({ id: workflowJobs.id })
+    .from(workflowJobs)
+    .where(eq(workflowJobs.runId, runId))
+    .all();
+
+  const jobIds = jobs.map(j => j.id);
+
+  if (jobIds.length > 0) {
+    await drizzle.update(workflowSteps)
+      .set({
+        status: 'completed',
+        conclusion: 'cancelled',
+        completedAt: timestamp,
+      })
+      .where(
+        and(
+          inArray(workflowSteps.jobId, jobIds),
+          inArray(workflowSteps.status, ['pending', 'in_progress']),
+        )
+      )
+      .run();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// finalizeRunIfComplete
+// ---------------------------------------------------------------------------
+
+/**
+ * If all jobs for a run are completed, determine and persist the run conclusion.
+ */
+export async function finalizeRunIfComplete(db: D1Database, runId: string): Promise<void> {
+  const drizzle = getDb(db);
+
+  const pendingResult = await drizzle.select({ count: count() })
+    .from(workflowJobs)
+    .where(and(eq(workflowJobs.runId, runId), ne(workflowJobs.status, 'completed')))
+    .get();
+  const pendingJobsCount = pendingResult?.count ?? 0;
+
+  if (pendingJobsCount > 0) return;
+
+  const failedResult = await drizzle.select({ count: count() })
+    .from(workflowJobs)
+    .where(and(eq(workflowJobs.runId, runId), eq(workflowJobs.conclusion, 'failure')))
+    .get();
+  const failedJobsCount = failedResult?.count ?? 0;
+
+  const cancelledResult = await drizzle.select({ count: count() })
+    .from(workflowJobs)
+    .where(and(eq(workflowJobs.runId, runId), eq(workflowJobs.conclusion, 'cancelled')))
+    .get();
+  const cancelledJobsCount = cancelledResult?.count ?? 0;
+
+  let conclusion: Conclusion = 'success';
+  if (failedJobsCount > 0) {
+    conclusion = 'failure';
+  } else if (cancelledJobsCount > 0) {
+    conclusion = 'cancelled';
+  }
+
+  await drizzle.update(workflowRuns)
+    .set({
+      status: 'completed',
+      conclusion,
+      completedAt: now(),
+    })
+    .where(eq(workflowRuns.id, runId))
+    .run();
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers used by startRun / cancelRun
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if the workflow supports the given event
+ */
+export function checkTrigger(workflow: Workflow, event: string): boolean {
+  const on = workflow.on;
+
+  if (typeof on === 'string') {
+    return on === event;
+  }
+
+  if (Array.isArray(on)) {
+    return on.includes(event);
+  }
+
+  if (on && typeof on === 'object') {
+    return event in on;
+  }
+
+  return false;
+}
+
+/**
+ * Resolve the workflow ID from the database, or return null if not found.
+ */
+export async function resolveWorkflowId(db: D1Database, repoId: string, workflowPath: string): Promise<string | null> {
+  const drizzle = getDb(db);
+  const existing = await drizzle.select({ id: workflows.id })
+    .from(workflows)
+    .where(and(eq(workflows.repoId, repoId), eq(workflows.path, workflowPath)))
+    .get();
+  return existing?.id ?? null;
+}
+
+/**
+ * Retrieve secret IDs for a repo.
+ */
+export async function getSecretIds(db: D1Database, repoId: string): Promise<string[]> {
+  const drizzle = getDb(db);
+  const secretRecords = await drizzle.select({ id: workflowSecrets.id })
+    .from(workflowSecrets)
+    .where(eq(workflowSecrets.repoId, repoId))
+    .all();
+  return secretRecords.map((s) => s.id);
+}
+
+/**
+ * Enqueue a job for execution
+ */
+export async function enqueueJob(
+  queue: Queue<WorkflowJobQueueMessage> | undefined,
+  options: {
+    runId: string;
+    jobId: string;
+    repoId: string;
+    ref: string;
+    sha: string;
+    jobKey: string;
+    jobDefinition: WorkflowJobDefinition;
+    env: Record<string, string>;
+    secretIds: string[];
+  },
+): Promise<void> {
+  if (!queue) {
+    logWarn('No queue configured, job will not be executed', { module: 'services/execution/workflow-engine' });
+    return;
+  }
+
+  const message: WorkflowJobQueueMessage = {
+    version: WORKFLOW_QUEUE_MESSAGE_VERSION,
+    type: 'job',
+    runId: options.runId,
+    jobId: options.jobId,
+    repoId: options.repoId,
+    ref: options.ref,
+    sha: options.sha,
+    jobKey: options.jobKey,
+    jobDefinition: options.jobDefinition,
+    env: options.env,
+    secretIds: options.secretIds,
+    timestamp: Date.now(),
+  };
+
+  await queue.send(message);
+}

package/src/application/services/execution/workflow-storage.ts

@@ -0,0 +1,83 @@
+/**
+ * Workflow Engine – storage operations (logs and artifacts)
+ */
+
+import { generateId, now } from '../../../shared/utils';
+import { getDb, workflowJobs, workflowArtifacts } from '../../../infra/db';
+import { eq } from 'drizzle-orm';
+import type { D1Database } from '../../../shared/types/bindings.ts';
+import type { WorkflowBucket } from './workflow-engine-types';
+
+// ---------------------------------------------------------------------------
+// storeJobLogs
+// ---------------------------------------------------------------------------
+
+export async function storeJobLogs(
+  db: D1Database,
+  bucket: WorkflowBucket,
+  jobId: string,
+  logs: string,
+): Promise<string> {
+  const drizzle = getDb(db);
+  const r2Key = `workflow-logs/${jobId}.txt`;
+
+  await bucket.put(r2Key, logs, {
+    httpMetadata: {
+      contentType: 'text/plain',
+    },
+  });
+
+  await drizzle.update(workflowJobs)
+    .set({ logsR2Key: r2Key })
+    .where(eq(workflowJobs.id, jobId))
+    .run();
+
+  return r2Key;
+}
+
+// ---------------------------------------------------------------------------
+// createArtifact
+// ---------------------------------------------------------------------------
+
+export async function createArtifact(
+  db: D1Database,
+  bucket: WorkflowBucket,
+  options: {
+    runId: string;
+    name: string;
+    data: ArrayBuffer | Uint8Array | string;
+    mimeType?: string;
+    expiresInDays?: number;
+  },
+): Promise<{ id: string; r2Key: string }> {
+  const drizzle = getDb(db);
+  const { runId, name, data, mimeType, expiresInDays = 30 } = options;
+
+  const artifactId = generateId();
+  const r2Key = `workflow-artifacts/${runId}/${artifactId}/${name}`;
+  const timestamp = now();
+  const expiresAt = new Date(Date.now() + expiresInDays * 24 * 60 * 60 * 1000).toISOString();
+
+  await bucket.put(r2Key, data, {
+    httpMetadata: {
+      contentType: mimeType || 'application/octet-stream',
+    },
+  });
+
+  const size = typeof data === 'string' ? new TextEncoder().encode(data).length : data.byteLength;
+
+  await drizzle.insert(workflowArtifacts)
+    .values({
+      id: artifactId,
+      runId,
+      name,
+      r2Key,
+      sizeBytes: size,
+      mimeType: mimeType ?? null,
+      expiresAt,
+      createdAt: timestamp,
+    })
+    .run();
+
+  return { id: artifactId, r2Key };
+}

package/src/application/services/git-smart/client/fetch-pack.ts

@@ -0,0 +1,167 @@
+/**
+ * Git Smart HTTP Client — fetch pack (clone/fetch).
+ *
+ * Speaks the client side of the git upload-pack protocol.
+ * This is the inverse of `handleUploadPack` in `smart-http/upload-pack.ts`:
+ * the server builds want/have negotiation and constructs a packfile response;
+ * this module *sends* want/have lines and *receives* the packfile.
+ *
+ * The returned packfile bytes can be passed directly to `readPackfileAsync`
+ * from `protocol/packfile-reader.ts` to unpack objects into R2.
+ */
+
+import { encodePktLine, flushPkt, parsePktLines, pktLineText } from '../protocol/pkt-line';
+import { concatBytes } from '../core/sha1';
+
+const TEXT_ENCODER = new TextEncoder();
+
+/** Maximum packfile size we accept (100 MB). */
+const MAX_FETCH_PACKFILE_BYTES = 100 * 1024 * 1024;
+
+/**
+ * Fetch a packfile from a remote git HTTP server.
+ *
+ * Sends a `POST <url>/git-upload-pack` request with want/have negotiation
+ * and returns the raw packfile bytes (starting with `PACK` signature).
+ *
+ * @param url       Base git URL (e.g. `https://github.com/owner/repo.git`)
+ * @param authHeader  Authorization header value, or null for public repos
+ * @param wants     SHA-1 hashes of refs we want
+ * @param haves     SHA-1 hashes of refs we already have locally (empty for initial clone)
+ * @param options   Optional fetch configuration
+ * @returns Raw packfile bytes suitable for `readPackfileAsync`
+ */
+export async function fetchPackFromRemote(
+  url: string,
+  authHeader: string | null,
+  wants: string[],
+  haves: string[],
+  options?: { maxPackfileBytes?: number },
+): Promise<Uint8Array> {
+  if (wants.length === 0) {
+    throw new Error('fetchPackFromRemote: at least one want SHA is required');
+  }
+
+  const maxBytes = options?.maxPackfileBytes ?? MAX_FETCH_PACKFILE_BYTES;
+  const normalizedUrl = url.replace(/\/$/, '');
+  const uploadPackUrl = `${normalizedUrl}/git-upload-pack`;
+
+  // Build request body: want lines, flush, have lines, done
+  const requestParts: Uint8Array[] = [];
+
+  // First want line includes capabilities
+  const capabilities = 'side-band-64k no-progress ofs-delta';
+  requestParts.push(encodePktLine(`want ${wants[0]} ${capabilities}\n`));
+
+  // Remaining want lines
+  for (let i = 1; i < wants.length; i++) {
+    requestParts.push(encodePktLine(`want ${wants[i]}\n`));
+  }
+
+  requestParts.push(flushPkt());
+
+  // Have lines
+  for (const have of haves) {
+    requestParts.push(encodePktLine(`have ${have}\n`));
+  }
+
+  // Done
+  requestParts.push(encodePktLine('done\n'));
+
+  const requestBody = concatBytes(...requestParts);
+
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/x-git-upload-pack-request',
+    'User-Agent': 'takos-git-client/1.0',
+  };
+  if (authHeader) {
+    headers['Authorization'] = authHeader;
+  }
+
+  const response = await fetch(uploadPackUrl, {
+    method: 'POST',
+    headers,
+    body: requestBody as unknown as BodyInit,
+  });
+
+  if (!response.ok) {
+    const body = await response.text().catch(() => '');
+    throw new Error(
+      `Failed to fetch pack from ${normalizedUrl}: HTTP ${response.status}${body ? ` — ${body.slice(0, 200)}` : ''}`,
+    );
+  }
+
+  const responseBuffer = new Uint8Array(await response.arrayBuffer());
+
+  if (responseBuffer.length > maxBytes) {
+    throw new Error(
+      `Packfile response too large: ${responseBuffer.length} bytes (limit: ${maxBytes})`,
+    );
+  }
+
+  // Parse the response — extract packfile from side-band-64k framing
+  return extractPackfileFromResponse(responseBuffer);
+}
+
+/**
+ * Extract raw packfile data from a git upload-pack response.
+ *
+ * The response format is:
+ *   - NAK/ACK pkt-lines
+ *   - Side-band-64k frames: channel 1 = pack data, channel 2 = progress, channel 3 = error
+ *   - Flush packet
+ *
+ * If the response does not use side-band framing (e.g., the server doesn't
+ * support it), the raw response after the NAK line is treated as packfile data.
+ */
+function extractPackfileFromResponse(data: Uint8Array): Uint8Array {
+  const lines = parsePktLines(data);
+  const packParts: Uint8Array[] = [];
+  let hasSideBand = false;
+
+  for (const line of lines) {
+    if (line.type !== 'data' || !line.data || line.data.length === 0) continue;
+
+    const text = pktLineText(line);
+
+    // Skip NAK/ACK lines
+    if (text === 'NAK' || text.startsWith('ACK ')) continue;
+
+    // Check for side-band framing (channel byte as first byte)
+    const channel = line.data[0];
+
+    if (channel === 1) {
+      // Pack data channel
+      hasSideBand = true;
+      packParts.push(line.data.subarray(1));
+    } else if (channel === 2) {
+      // Progress channel — skip
+      hasSideBand = true;
+    } else if (channel === 3) {
+      // Error channel
+      const errorMsg = new TextDecoder().decode(line.data.subarray(1));
+      throw new Error(`Remote error: ${errorMsg}`);
+    } else if (!hasSideBand) {
+      // No side-band — treat raw data as packfile
+      packParts.push(line.data);
+    }
+  }
+
+  if (packParts.length === 0) {
+    throw new Error('No packfile data received from remote');
+  }
+
+  const packfile = concatBytes(...packParts);
+
+  // Verify PACK signature
+  if (packfile.length < 12) {
+    throw new Error(`Packfile too small: ${packfile.length} bytes`);
+  }
+
+  const sig = String.fromCharCode(packfile[0], packfile[1], packfile[2], packfile[3]);
+  if (sig !== 'PACK') {
+    throw new Error(`Invalid packfile signature: "${sig}"`);
+  }
+
+  return packfile;
+}