pid1 0.0.0-dev.0

package/src/process-manager.ts

@@ -0,0 +1,632 @@
+import { dirname, join } from "node:path";
+import { watch, type FSWatcher } from "node:fs";
+import { loadConfigFile } from "./config.ts";
+import { loadEnvForProcess } from "./env.ts";
+import { spawnWithLogging, type SpawnedProcess } from "./exec.ts";
+import { globalLogger } from "./logger.ts";
+import { createServer, type ServerOptions } from "./api/server.ts";
+
+export type ProcessManagerOptions = {
+  cwd?: string;
+  configPath?: string;
+  server?: ServerOptions & { enabled?: boolean };
+};
+
+export type ProcessStatus = "stopped" | "running" | "errored";
+export type RestartPolicy = "never" | "always" | "on-failure";
+
+export type ProcessConfig = {
+  name: string;
+  command: string;
+  args: string[];
+  env: Record<string, string>;
+  cwd?: string;
+  envFile?: string | false;
+  restartOnEnvChange?: boolean;
+  restartPolicy?: RestartPolicy;
+  maxRestarts?: number;
+  restartMinDelayMs?: number;
+  restartMaxDelayMs?: number;
+  restartResetSeconds?: number;
+};
+
+export type ManagedProcess = {
+  id: number;
+  config: ProcessConfig;
+  status: ProcessStatus;
+  spawned: SpawnedProcess | null;
+  exitCode: number | null;
+  restarts: number;
+  restartAttempts: number;
+  restartTimer: NodeJS.Timeout | null;
+  stopRequested: boolean;
+  startedAt: Date | null;
+};
+
+const DEFAULT_STOP_TIMEOUT_MS = 10000;
+const DEFAULT_RESTART_RESET_SECONDS = 300;
+const DEFAULT_RESTART_MAX = 10;
+const DEFAULT_RESTART_MIN_DELAY_MS = 5000;
+const DEFAULT_RESTART_MAX_DELAY_MS = 60000;
+
+export type ProcessInfo = {
+  id: number;
+  name: string;
+  status: ProcessStatus;
+  pid: number | undefined;
+  exitCode: number | null;
+  restarts: number;
+  startedAt: Date | null;
+  command: string;
+  args: string[];
+};
+
+export class ProcessManager {
+  private logger = globalLogger.child("pm");
+  private cwd: string;
+  private configPath?: string;
+  private configDir?: string;
+  private globalEnvFile?: string | false;
+  private processes: Map<string, ManagedProcess>;
+  private processIds: Map<number, string>;
+  private nextProcessId = 0;
+  private signalHandlersSetup = false;
+  private serverOptions?: ServerOptions & { enabled?: boolean };
+  private httpServer?: ReturnType<ReturnType<typeof createServer>["start"]>;
+  private envWatcher: FSWatcher | null = null;
+  private envChangeTimers = new Map<string, NodeJS.Timeout>();
+  private envRestartTimers = new Map<string, NodeJS.Timeout>();
+  private restartOnEnvChangeGlobal = true;
+  private restartMinDelayMsGlobal = DEFAULT_RESTART_MIN_DELAY_MS;
+  private restartMaxDelayMsGlobal = DEFAULT_RESTART_MAX_DELAY_MS;
+  private currentTask: SpawnedProcess | null = null;
+  private currentTaskName: string | null = null;
+  private taskAbortController: AbortController | null = null;
+
+  constructor(options: ProcessManagerOptions = {}) {
+    this.cwd = options.cwd ?? process.cwd();
+    this.configPath = options.configPath;
+    this.processes = new Map();
+    this.processIds = new Map();
+    this.serverOptions = options.server;
+  }
+
+  async init(): Promise<void> {
+    const config = await loadConfigFile(this.configPath, this.cwd);
+
+    this.configDir = dirname(config.configPath);
+    this.globalEnvFile = config.envFile;
+    this.restartOnEnvChangeGlobal = config.restartOnEnvChange ?? true;
+    this.restartMinDelayMsGlobal =
+      config.restartMinDelayMs ?? DEFAULT_RESTART_MIN_DELAY_MS;
+    this.restartMaxDelayMsGlobal =
+      config.restartMaxDelayMs ?? DEFAULT_RESTART_MAX_DELAY_MS;
+
+    this.logger.info(
+      `Executing ${config.tasks.length} tasks and registering ${config.processes.length} processes`,
+    );
+
+    this.setupSignalHandlers();
+    this.setupEnvWatchers();
+    await this.runTasks(config.tasks);
+
+    for (const p of config.processes) {
+      this.register(p);
+    }
+  }
+
+  register(config: ProcessConfig): void {
+    if (this.processes.has(config.name)) {
+      throw new Error(`Process ${config.name} already registered`);
+    }
+
+    const id = this.nextProcessId++;
+    const managed: ManagedProcess = {
+      id,
+      config,
+      status: "stopped",
+      spawned: null,
+      exitCode: null,
+      restarts: 0,
+      restartAttempts: 0,
+      restartTimer: null,
+      stopRequested: false,
+      startedAt: null,
+    };
+
+    this.processes.set(config.name, managed);
+    this.processIds.set(id, config.name);
+    this.logger.info(`Registered process: ${config.name} (id: ${id})`);
+  }
+
+  list(): ProcessInfo[] {
+    return [...this.processes.values()].map((m) => ({
+      id: m.id,
+      name: m.config.name,
+      status: m.status,
+      pid: m.spawned?.proc.pid,
+      exitCode: m.exitCode,
+      restarts: m.restarts,
+      startedAt: m.startedAt,
+      command: m.config.command,
+      args: m.config.args,
+    }));
+  }
+
+  start(name: string): void {
+    const { name: resolvedName, managed } = this.resolveProcessIdentifier(name);
+
+    if (managed.status === "running") {
+      this.logger.warn(`Process ${resolvedName} is already running`);
+      return;
+    }
+
+    this.spawnProcess(managed);
+  }
+
+  startAll(): void {
+    this.setupSignalHandlers();
+
+    for (const [_, managed] of this.processes) {
+      if (managed.status !== "running") {
+        this.spawnProcess(managed);
+      }
+    }
+  }
+
+  serve(options?: ServerOptions & { enabled?: boolean }): void {
+    const serverOpts = { ...this.serverOptions, ...options };
+    if (serverOpts.enabled === false) return;
+    const { enabled: _enabled, ...serverOptions } = serverOpts;
+    const server = createServer(this, serverOptions);
+    this.httpServer = server.start();
+  }
+
+  async stop(name: string, timeoutMs = DEFAULT_STOP_TIMEOUT_MS): Promise<void> {
+    const { name: resolvedName, managed } = this.resolveProcessIdentifier(name);
+
+    managed.stopRequested = true;
+    this.clearRestartTimer(managed);
+
+    if (managed.status !== "running" || !managed.spawned) {
+      this.logger.warn(`Process ${resolvedName} is not running`);
+      return;
+    }
+
+    const { spawned } = managed;
+
+    this.logger.info(`Stopping process: ${resolvedName} (SIGTERM)`);
+    spawned.kill("SIGTERM");
+
+    const exitedGracefully = await Promise.race([
+      spawned.done.then(() => true).catch(() => true),
+      new Promise<false>((resolve) =>
+        setTimeout(() => resolve(false), timeoutMs),
+      ),
+    ]);
+
+    if (!exitedGracefully && managed.status === "running") {
+      this.logger.warn(
+        `Process ${resolvedName} did not stop gracefully, sending SIGKILL`,
+      );
+      spawned.kill("SIGKILL");
+      await spawned.done.catch(() => {});
+    }
+  }
+
+  async stopAll(): Promise<void> {
+    this.logger.info("Stopping all processes...");
+
+    for (const managed of this.processes.values()) {
+      managed.stopRequested = true;
+      this.clearRestartTimer(managed);
+    }
+
+    const stopPromises = [...this.processes.entries()]
+      .filter(([, m]) => m.status === "running")
+      .map(([name]) => this.stop(name));
+
+    await Promise.allSettled(stopPromises);
+  }
+
+  async restart(name: string): Promise<void> {
+    const { name: resolvedName, managed } = this.resolveProcessIdentifier(name);
+
+    this.logger.info(`Restarting process: ${resolvedName}`);
+
+    if (managed.status === "running") {
+      await this.stop(resolvedName);
+    }
+
+    managed.restarts++;
+    this.spawnProcess(managed);
+  }
+
+  async restartAll(): Promise<void> {
+    this.logger.info("Restarting all processes...");
+
+    for (const [name] of this.processes) {
+      await this.restart(name);
+    }
+  }
+
+  async delete(name: string): Promise<void> {
+    const { name: resolvedName, managed } = this.resolveProcessIdentifier(name);
+
+    managed.stopRequested = true;
+    this.clearRestartTimer(managed);
+
+    if (managed.status === "running") {
+      await this.stop(resolvedName);
+    }
+
+    this.processes.delete(resolvedName);
+    this.processIds.delete(managed.id);
+    this.logger.info(`Deleted process: ${resolvedName}`);
+  }
+
+  async deleteAll(): Promise<void> {
+    await this.stopAll();
+    this.processes.clear();
+    this.logger.info("Deleted all processes");
+  }
+
+  get(name: string): ProcessInfo | undefined {
+    let resolved: { name: string; managed: ManagedProcess };
+    try {
+      resolved = this.resolveProcessIdentifier(name);
+    } catch {
+      return undefined;
+    }
+
+    return {
+      id: resolved.managed.id,
+      name: resolved.managed.config.name,
+      status: resolved.managed.status,
+      pid: resolved.managed.spawned?.proc.pid,
+      exitCode: resolved.managed.exitCode,
+      restarts: resolved.managed.restarts,
+      startedAt: resolved.managed.startedAt,
+      command: resolved.managed.config.command,
+      args: resolved.managed.config.args,
+    };
+  }
+
+  async shutdown(): Promise<void> {
+    await this.stopActiveTask();
+    await this.stopAll();
+    this.closeEnvWatcher();
+    if (this.httpServer) {
+      this.httpServer.close();
+      this.logger.info("HTTP server closed");
+    }
+  }
+
+  private spawnProcess(managed: ManagedProcess): void {
+    const { config } = managed;
+
+    managed.stopRequested = false;
+    this.clearRestartTimer(managed);
+
+    const env = loadEnvForProcess({
+      configDir: this.configDir ?? this.cwd,
+      processName: config.name,
+      globalEnvFile: this.globalEnvFile,
+      processEnvFile: config.envFile,
+      configEnv: config.env,
+    });
+
+    const spawned = spawnWithLogging({
+      cwd: config.cwd ?? this.cwd,
+      logFile: join(this.cwd, "logs", "processes", `${config.name}.log`),
+      type: "process",
+      name: config.name,
+      command: config.command,
+      args: config.args,
+      env,
+    });
+
+    managed.spawned = spawned;
+    managed.status = "running";
+    managed.startedAt = new Date();
+    managed.exitCode = null;
+
+    this.logger.info(
+      `Started process: ${config.name} (pid: ${spawned.proc.pid})`,
+    );
+
+    const handleExit = (exitCode: number | null, error?: unknown) => {
+      managed.exitCode = exitCode;
+      managed.status =
+        exitCode === null || exitCode === undefined || exitCode === 0
+          ? "stopped"
+          : "errored";
+      const now = Date.now();
+      const startedAt = managed.startedAt?.getTime() ?? now;
+      const uptimeMs = Math.max(0, now - startedAt);
+      const resetMs =
+        (config.restartResetSeconds ?? DEFAULT_RESTART_RESET_SECONDS) * 1000;
+
+      if (uptimeMs >= resetMs) {
+        managed.restartAttempts = 0;
+      }
+
+      if (exitCode === null || exitCode === undefined) {
+        this.logger.info(`Process ${config.name} stopped`);
+      } else if (exitCode !== 0) {
+        this.logger.info(`Process ${config.name} exited with code ${exitCode}`);
+      } else {
+        this.logger.info(`Process ${config.name} exited successfully`);
+      }
+
+      if (error) {
+        this.logger.error(`Process ${config.name} failed with error: ${error}`);
+      }
+
+      const policy = config.restartPolicy ?? "on-failure";
+      const isFailure =
+        exitCode === null || exitCode === undefined || exitCode !== 0;
+      const shouldRestart =
+        !managed.stopRequested &&
+        (policy === "always" || (policy === "on-failure" && isFailure));
+
+      if (!shouldRestart) {
+        return;
+      }
+
+      const maxRestarts = config.maxRestarts ?? DEFAULT_RESTART_MAX;
+      if (managed.restartAttempts >= maxRestarts) {
+        managed.status = "errored";
+        this.logger.warn(
+          `Process ${config.name} reached restart limit (${maxRestarts}), not restarting`,
+        );
+        return;
+      }
+
+      managed.restartAttempts += 1;
+      managed.restarts += 1;
+      const minDelay = config.restartMinDelayMs ?? this.restartMinDelayMsGlobal;
+      const maxDelay = config.restartMaxDelayMs ?? this.restartMaxDelayMsGlobal;
+      const delay = Math.min(
+        maxDelay,
+        minDelay * Math.pow(2, managed.restartAttempts - 1),
+      );
+
+      this.logger.warn(
+        `Restarting process ${config.name} in ${delay}ms (attempt ${managed.restartAttempts}/${maxRestarts})`,
+      );
+
+      managed.restartTimer = setTimeout(() => {
+        managed.restartTimer = null;
+        if (!managed.stopRequested) {
+          this.spawnProcess(managed);
+        }
+      }, delay);
+    };
+
+    spawned.done
+      .then((exitCode) => handleExit(exitCode))
+      .catch((error) => {
+        const exitCode = spawned.proc.exitCode ?? 1;
+        handleExit(exitCode, error);
+      });
+  }
+
+  private async runTasks(tasks: Array<ProcessConfig>): Promise<void> {
+    for (const task of tasks) {
+      const env = loadEnvForProcess({
+        configDir: this.configDir ?? this.cwd,
+        processName: task.name,
+        globalEnvFile: this.globalEnvFile,
+        processEnvFile: task.envFile,
+        configEnv: task.env,
+      });
+
+      const taskAbortController = new AbortController();
+      this.taskAbortController = taskAbortController;
+
+      const spawned = spawnWithLogging({
+        cwd: task.cwd ?? this.cwd,
+        logFile: join(this.cwd, "logs", "tasks", `${task.name}.log`),
+        type: "task",
+        name: task.name,
+        command: task.command,
+        args: task.args,
+        env,
+        abortSignal: taskAbortController.signal,
+      });
+
+      this.currentTask = spawned;
+      this.currentTaskName = task.name;
+
+      let exitCode: number | null;
+      try {
+        exitCode = await spawned.done;
+      } finally {
+        this.currentTask = null;
+        this.currentTaskName = null;
+        this.taskAbortController = null;
+      }
+
+      if (exitCode !== 0) {
+        this.logger.error(
+          `Task ${task.name} failed with exit code ${exitCode}`,
+        );
+        throw new Error(`Task ${task.name} failed`);
+      }
+    }
+
+    if (tasks.length > 0) {
+      this.logger.info("All tasks completed");
+    }
+  }
+
+  private clearRestartTimer(managed: ManagedProcess): void {
+    if (managed.restartTimer) {
+      clearTimeout(managed.restartTimer);
+      managed.restartTimer = null;
+    }
+  }
+
+  private async stopActiveTask(
+    timeoutMs = DEFAULT_STOP_TIMEOUT_MS,
+  ): Promise<void> {
+    if (!this.currentTask) return;
+
+    const taskName = this.currentTaskName ?? "unknown";
+    this.logger.info(`Stopping task: ${taskName} (SIGTERM)`);
+    this.currentTask.kill("SIGTERM");
+
+    const exitedGracefully = await Promise.race([
+      this.currentTask.done.then(() => true).catch(() => true),
+      new Promise<false>((resolve) =>
+        setTimeout(() => resolve(false), timeoutMs),
+      ),
+    ]);
+
+    if (!exitedGracefully) {
+      this.logger.warn(
+        `Task ${taskName} did not stop gracefully, sending SIGKILL`,
+      );
+      this.currentTask.kill("SIGKILL");
+      await this.currentTask.done.catch(() => {});
+    }
+  }
+
+  private abortActiveTask(signal: NodeJS.Signals): void {
+    if (this.taskAbortController && !this.taskAbortController.signal.aborted) {
+      this.taskAbortController.abort();
+    }
+    if (this.currentTask) {
+      this.currentTask.kill(signal);
+    }
+  }
+
+  private setupSignalHandlers(): void {
+    if (this.signalHandlersSetup) return;
+    this.signalHandlersSetup = true;
+
+    const signals = ["SIGINT", "SIGTERM"] as const;
+    for (const signal of signals) {
+      process.on(signal, () => {
+        process.stdout.write("\r\x1b[K");
+        this.logger.info(`Received ${signal}, shutting down...`);
+        this.abortActiveTask(signal);
+        this.shutdown().then(() => process.exit(0));
+      });
+    }
+  }
+
+  private setupEnvWatchers(): void {
+    if (this.envWatcher) return;
+    if (!this.restartOnEnvChangeGlobal) return;
+
+    const watchDir = this.configDir ?? this.cwd;
+    this.envWatcher = watch(watchDir, (event, filename) => {
+      if (!filename) return;
+      const envFile = filename.toString();
+      if (!envFile.startsWith(".env")) return;
+      if (event !== "change" && event !== "rename") return;
+
+      if (envFile === ".env") {
+        this.scheduleEnvRestart("global", () => {
+          this.logger.info("Detected .env change, scheduling restarts");
+          for (const managed of this.processes.values()) {
+            this.scheduleEnvRestartForProcess(managed);
+          }
+        });
+        return;
+      }
+
+      if (!envFile.startsWith(".env.")) return;
+      const processName = envFile.slice(".env.".length);
+      if (!processName) return;
+
+      this.scheduleEnvRestart(processName, () => {
+        const managed = this.processes.get(processName);
+        if (!managed) return;
+        this.logger.info(
+          `Detected ${envFile} change, scheduling restart for ${processName}`,
+        );
+        this.scheduleEnvRestartForProcess(managed);
+      });
+    });
+  }
+
+  private closeEnvWatcher(): void {
+    if (this.envWatcher) {
+      this.envWatcher.close();
+      this.envWatcher = null;
+    }
+    for (const timer of this.envChangeTimers.values()) {
+      clearTimeout(timer);
+    }
+    this.envChangeTimers.clear();
+    for (const timer of this.envRestartTimers.values()) {
+      clearTimeout(timer);
+    }
+    this.envRestartTimers.clear();
+  }
+
+  private scheduleEnvRestart(key: string, restart: () => void): void {
+    const existing = this.envChangeTimers.get(key);
+    if (existing) clearTimeout(existing);
+    const timer = setTimeout(() => {
+      this.envChangeTimers.delete(key);
+      restart();
+    }, 250);
+    this.envChangeTimers.set(key, timer);
+  }
+
+  private scheduleEnvRestartForProcess(managed: ManagedProcess): void {
+    if (!this.restartOnEnvChangeGlobal) return;
+    if (managed.config.restartOnEnvChange === false) return;
+    if (managed.status !== "running") return;
+    const name = managed.config.name;
+    const existing = this.envRestartTimers.get(name);
+    if (existing) clearTimeout(existing);
+
+    const delayMs =
+      managed.config.restartMinDelayMs ?? this.restartMinDelayMsGlobal;
+
+    const timer = setTimeout(
+      () => {
+        this.envRestartTimers.delete(name);
+        this.restart(name).catch((error) => {
+          this.logger.error(
+            `Failed to restart ${name} after env change: ${error}`,
+          );
+        });
+      },
+      Math.max(0, delayMs),
+    );
+
+    this.envRestartTimers.set(name, timer);
+    this.logger.info(
+      `Restarting ${name} after env change in ${Math.max(0, delayMs)}ms`,
+    );
+  }
+
+  private resolveProcessIdentifier(nameOrId: string): {
+    name: string;
+    managed: ManagedProcess;
+  } {
+    const byName = this.processes.get(nameOrId);
+    if (byName) {
+      return { name: nameOrId, managed: byName };
+    }
+
+    if (/^\d+$/.test(nameOrId)) {
+      const id = Number(nameOrId);
+      const name = this.processIds.get(id);
+      if (name) {
+        const managed = this.processes.get(name);
+        if (managed) {
+          return { name, managed };
+        }
+      }
+    }
+
+    throw new Error(`Process ${nameOrId} not found`);
+  }
+}