stepwise-migrations 1.0.14 → 1.0.15

package/src/index.ts

@@ -2,26 +2,22 @@
 
 import yargs from "yargs";
 import {
-  dbAuditHistory,
+  applyMigration,
+  applyUndoMigration,
   dbConnect,
-  dbCreateHistoryTable,
+  dbCreateEventsTable,
   dbCreateSchema,
-  dbGetScript,
-  dbHistorySchemaExists,
-  dbMigrationHistory,
+  dbDropAll,
+  dbGetAppliedScript,
+  dbSchemaExists,
   dbTableExists,
 } from "./db";
+import { getUndoFilename, loadState } from "./state";
 import {
-  applyDownMigration,
-  applyMigration,
-  validateDownMigrationFiles,
-  validateMigrationFiles,
-} from "./migrate";
-import {
-  printMigrationHistory,
+  abortIfErrors,
+  exitIfNotInitialized,
   printMigrationHistoryAndUnappliedMigrations,
-  readDownMigrationFiles,
-  readMigrationFiles,
+  sliceFromFirstNull,
   usage,
   validateArgs,
 } from "./utils";
@@ -33,134 +29,173 @@ const main = async () => {
 
   const schema = argv.schema;
   const command = argv._[0];
+  const napply = argv.napply || Infinity;
+  const nundo = argv.nundo || 1;
+  const filePath = argv.path;
 
   const client = await dbConnect(argv);
-  const historySchemaExists = await dbHistorySchemaExists(client, schema);
+  const schemaExists = await dbSchemaExists(client, schema);
   const tableExists = await dbTableExists(client, schema);
 
   if (command === "migrate") {
-    const nUp = argv.nup || Infinity;
-    if (!historySchemaExists) {
+    if (!schemaExists) {
       await dbCreateSchema(client, schema);
     }
     if (!tableExists) {
-      await dbCreateHistoryTable(client, schema);
+      await dbCreateEventsTable(client, schema);
     }
 
-    const migrationHistory = await dbMigrationHistory(client, schema);
-    const migrationFiles = await readMigrationFiles(argv.path);
+    const state = await loadState(client, schema, argv.path);
 
-    validateMigrationFiles(migrationFiles, migrationHistory);
+    abortIfErrors(state);
 
-    if (migrationFiles.length === migrationHistory.length) {
+    if (
+      state.files.unappliedVersionedFiles.length === 0 &&
+      state.files.unappliedRepeatableFiles.length === 0
+    ) {
       console.log("All migrations are already applied");
       process.exit(0);
     }
 
-    const migrationsToApply = migrationFiles.slice(
-      migrationHistory.length,
-      migrationHistory.length + nUp
-    );
+    const migrationsToApply = [
+      ...state.files.unappliedVersionedFiles,
+      ...state.files.unappliedRepeatableFiles,
+    ].slice(0, napply);
 
-    for (const { filename, script } of migrationsToApply) {
-      await applyMigration(client, schema, filename, script);
+    for (const migration of migrationsToApply) {
+      await applyMigration(client, schema, migration);
     }
 
-    console.log(`All done! Applied ${migrationsToApply.length} migrations`);
+    console.log(
+      `All done! Applied ${migrationsToApply.length} migration${
+        migrationsToApply.length === 1 ? "" : "s"
+      }`
+    );
 
     printMigrationHistoryAndUnappliedMigrations(
-      await readMigrationFiles(argv.path),
-      await dbMigrationHistory(client, schema)
+      await loadState(client, schema, filePath)
     );
   } else if (command === "info") {
-    if (!historySchemaExists) {
+    if (!schemaExists) {
       console.log("Schema does not exist");
     }
 
     if (!tableExists) {
-      console.log("Migration history table does not exist");
+      console.log(
+        "Migration table has not been initialised. Run migrate to begin."
+      );
     }
 
-    if (historySchemaExists && tableExists) {
-      printMigrationHistory(await dbMigrationHistory(client, schema));
+    if (schemaExists && tableExists) {
+      printMigrationHistoryAndUnappliedMigrations(
+        await loadState(client, schema, filePath)
+      );
     }
-  } else if (command === "validate") {
-    if (!historySchemaExists) {
+  } else if (command === "status") {
+    if (!schemaExists) {
       console.log("Schema does not exist");
     }
 
     if (!tableExists) {
-      console.log("Migration history table does not exist");
+      console.log(
+        "Migration table has not been initialised. Run migrate to begin."
+      );
     }
 
-    if (historySchemaExists && tableExists) {
-      validateMigrationFiles(
-        await readMigrationFiles(argv.path),
-        await dbMigrationHistory(client, schema)
+    if (schemaExists && tableExists) {
+      printMigrationHistoryAndUnappliedMigrations(
+        await loadState(client, schema, filePath)
       );
     }
+  } else if (command === "validate") {
+    exitIfNotInitialized(schemaExists, tableExists);
+
+    const state = await loadState(client, schema, argv.path);
+    if (schemaExists && tableExists) {
+      abortIfErrors(state);
+    }
     console.log("Validation passed");
 
-    printMigrationHistoryAndUnappliedMigrations(
-      await readMigrationFiles(argv.path),
-      await dbMigrationHistory(client, schema)
-    );
+    printMigrationHistoryAndUnappliedMigrations(state);
   } else if (command === "drop") {
     process.stdout.write(
       `Dropping the tables, schema and migration history table... `
     );
-    await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`);
+    await dbDropAll(client, schema);
     console.log(`done!`);
-  } else if (command === "down") {
-    const nDown = argv.ndown || 1;
-
-    const migrationHistory = await dbMigrationHistory(client, schema);
-    validateMigrationFiles(
-      await readMigrationFiles(argv.path),
-      migrationHistory,
-      false
-    );
+  } else if (command === "undo") {
+    const state = await loadState(client, schema, filePath);
 
-    const reverseMigrationHistory = migrationHistory.reverse().slice(0, nDown);
-    const downMigrationFilesToApply = await readDownMigrationFiles(
-      argv.path,
-      reverseMigrationHistory
-    );
+    abortIfErrors(state);
 
-    validateDownMigrationFiles(
-      downMigrationFilesToApply,
-      reverseMigrationHistory
+    const reversedAppliedVersionedMigrations =
+      state.current.appliedVersionedMigrations.slice().reverse();
+
+    const undosToApplyAll = reversedAppliedVersionedMigrations.map(
+      (migration) =>
+        state.files.undoFiles.find(
+          (file) => file.filename === getUndoFilename(migration.filename)
+        )
     );
-    for (const { filename, script, upFilename } of downMigrationFilesToApply) {
-      await applyDownMigration(client, schema, filename, script, upFilename);
+
+    const undosToApply = sliceFromFirstNull(undosToApplyAll).slice(0, nundo);
+
+    if (undosToApply.length < nundo) {
+      console.error(
+        `Error: not enough sequential (from last) undo migrations to apply ${nundo} undos.`
+      );
+      process.exit(1);
+    }
+
+    console.log(undosToApply);
+    for (const { filename, script } of undosToApply) {
+      await applyUndoMigration(client, schema, filename, script);
     }
     console.log(
-      `All done! Applied ${downMigrationFilesToApply.length} down migration${
-        downMigrationFilesToApply.length === 1 ? "" : "s"
+      `All done! Performed ${undosToApply.length} undo migration${
+        undosToApply.length === 1 ? "" : "s"
       }`
     );
 
-    printMigrationHistoryAndUnappliedMigrations(
-      await readMigrationFiles(argv.path),
-      await dbMigrationHistory(client, schema)
-    );
+    printMigrationHistoryAndUnappliedMigrations(state);
   } else if (command === "audit") {
-    const auditHistory = await dbAuditHistory(client, schema);
-    console.log("Audit history:");
+    exitIfNotInitialized(schemaExists, tableExists);
+
+    const state = await loadState(client, schema, argv.path);
+    console.log("Event history:");
     console.table(
-      auditHistory.map((row) => ({
+      state.events.map((row) => ({
         id: row.id,
         type: row.type,
-        name: row.name,
+        filename: row.filename,
         applied_by: row.applied_by,
         applied_at: row.applied_at,
       }))
     );
-  } else if (command === "get-script") {
-    const script = await dbGetScript(client, schema, argv.filename);
-    console.log(script);
+  } else if (command === "get-applied-script") {
+    if (!schemaExists) {
+      console.log("Schema does not exist");
+      process.exit(1);
+    }
+
+    if (!tableExists) {
+      console.log(
+        "Migration table has not been initialised. Run migrate to begin."
+      );
+      process.exit(1);
+    }
+
+    const state = await loadState(client, schema, argv.path);
+    const script = await dbGetAppliedScript(state, argv.filename);
+    if (script) {
+      console.log(script);
+    } else {
+      console.error(
+        `Script for ${argv.filename} not found, use the audit command to check all applied migrations`
+      );
+    }
   } else {
-    console.error(`Invalid command: ${argv._[0]}`);
+    console.error(`Invalid command: ${command}`);
     console.log(usage);
     process.exit(1);
   }

package/src/state.ts

@@ -0,0 +1,166 @@
+import gitDiff from "git-diff";
+import path from "path";
+import { PoolClient } from "pg";
+import { dbEventHistory } from "./db";
+import { AppliedMigration, MigrationState } from "./types";
+import { readMigrationFiles } from "./utils";
+
+export const validateMigrationFiles = (state: MigrationState) => {
+  let errors: string[] = [];
+
+  if (state.files.allFiles.length === 0) {
+    return ["No migration files found"];
+  }
+
+  if (
+    state.current.appliedVersionedMigrations.length >
+    state.files.versionedFiles.length
+  ) {
+    errors.push(
+      "Migration history is in a bad state: more applied versioned migrations than files"
+    );
+  }
+
+  for (let i = 0; i < state.current.appliedVersionedMigrations.length; i++) {
+    const { filename, script } = state.files.versionedFiles[i];
+    if (state.current.appliedVersionedMigrations[i].filename !== filename) {
+      errors.push(
+        `Migration history is in a bad state: applied versioned migration ${state.current.appliedVersionedMigrations[i].filename} does not match file ${filename}`
+      );
+      break;
+    }
+    if (state.current.appliedVersionedMigrations[i].script !== script) {
+      errors.push(
+        `Migration history is in a bad state: applied versioned migration ${state.current.appliedVersionedMigrations[i].filename} has been modified.\n\n` +
+          gitDiff(state.current.appliedVersionedMigrations[i].script, script, {
+            color: true,
+            noHeaders: true,
+          })
+      );
+      break;
+    }
+  }
+
+  if (
+    state.current.appliedRepeatableMigrations.length >
+    state.files.repeatableFiles.length
+  ) {
+    errors.push(
+      "Migration history is in a bad state: more applied repeatable migrations than files"
+    );
+  }
+
+  for (let i = 0; i < state.current.appliedRepeatableMigrations.length; i++) {
+    const { filename } = state.files.repeatableFiles[i];
+    if (state.current.appliedRepeatableMigrations[i].filename !== filename) {
+      errors.push(
+        `Migration history is in a bad state: applied repeatable migration ${state.current.appliedRepeatableMigrations[i].filename} does not match file ${filename}`
+      );
+      break;
+    }
+  }
+
+  return errors;
+};
+
+export const loadState = async (
+  client: PoolClient,
+  schema: string,
+  migrationPath: string
+): Promise<MigrationState> => {
+  const events = await dbEventHistory(client, schema);
+  const {
+    appliedVersionedMigrations,
+    appliedRepeatableMigrations,
+    errors: appliedErrors,
+  } = eventsToApplied(events);
+  const { files: allFiles, errors: readFileErrors } = await readMigrationFiles(
+    path.join(process.cwd(), migrationPath),
+    appliedVersionedMigrations
+  );
+  const unappliedVersionedFiles = allFiles
+    .filter((file) => file.type === "versioned")
+    .filter(
+      (file) =>
+        !appliedVersionedMigrations.find(
+          (event) => event.filename === file.filename
+        )
+    );
+  const unappliedRepeatableFiles = allFiles
+    .filter((file) => file.type === "repeatable")
+    .filter(
+      (file) =>
+        !appliedRepeatableMigrations.find(
+          (event) =>
+            event.filename === file.filename && event.script === file.script
+        )
+    );
+
+  return {
+    schema,
+    current: {
+      appliedVersionedMigrations,
+      appliedRepeatableMigrations,
+    },
+    events,
+    files: {
+      allFiles,
+      unappliedVersionedFiles,
+      unappliedRepeatableFiles,
+      versionedFiles: allFiles.filter((file) => file.type === "versioned"),
+      undoFiles: allFiles.filter((file) => file.type === "undo"),
+      repeatableFiles: allFiles.filter((file) => file.type === "repeatable"),
+    },
+    errors: appliedErrors.concat(readFileErrors),
+  };
+};
+
+export const getUndoFilename = (filename: string) => {
+  return filename.replace(".sql", ".undo.sql");
+};
+
+export const eventsToApplied = (
+  events: MigrationState["events"]
+): {
+  errors: string[];
+  appliedVersionedMigrations: AppliedMigration[];
+  appliedRepeatableMigrations: AppliedMigration[];
+} => {
+  let errors: string[] = [];
+  let appliedVersionedMigrations: AppliedMigration[] = [];
+  let appliedRepeatableMigrations: AppliedMigration[] = [];
+
+  for (const event of events) {
+    if (event.type === "versioned") {
+      appliedVersionedMigrations.push(event);
+    } else if (event.type === "undo") {
+      if (appliedVersionedMigrations.length === 0) {
+        errors.push(
+          "Events table is in a bad state: undo event without a migration to undo"
+        );
+        break;
+      } else if (
+        getUndoFilename(
+          appliedVersionedMigrations[appliedVersionedMigrations.length - 1]
+            .filename
+        ) !== event.filename
+      ) {
+        errors.push(
+          "Events table is in a bad state: down migration does not match the most recently applied migration"
+        );
+        break;
+      } else {
+        appliedVersionedMigrations.pop();
+      }
+    } else if (event.type === "repeatable") {
+      appliedRepeatableMigrations.push(event);
+    } else {
+      errors.push(
+        `Events table is in a bad state: unknown event type ${event.type}`
+      );
+      break;
+    }
+  }
+
+  return { errors, appliedVersionedMigrations, appliedRepeatableMigrations };
+};

package/src/types.ts

@@ -1,9 +1,54 @@
-export interface MigrationRow {
-  id: string;
-  name: string;
+import { z } from "zod";
+
+export type MigrationType = "versioned" | "undo" | "repeatable";
+
+export interface MigrationFile {
+  type: MigrationType;
+  filename: string;
+  script: string;
+}
+
+export interface AppliedMigration {
+  id: number;
+  type: MigrationType;
+  filename: string;
   script: string;
   applied_by: string;
   applied_at: string;
 }
 
-export type AuditRow = MigrationRow & { type: "up" | "down" };
+export interface MigrationState {
+  schema: string;
+  current: {
+    appliedVersionedMigrations: AppliedMigration[];
+    appliedRepeatableMigrations: AppliedMigration[];
+  };
+  events: {
+    id: number;
+    type: MigrationType;
+    filename: string;
+    script: string;
+    applied_by: string;
+    applied_at: string;
+  }[];
+  files: {
+    allFiles: MigrationFile[];
+    unappliedVersionedFiles: MigrationFile[];
+    unappliedRepeatableFiles: MigrationFile[];
+    versionedFiles: MigrationFile[];
+    undoFiles: MigrationFile[];
+    repeatableFiles: MigrationFile[];
+  };
+  errors: string[];
+}
+
+const MigrationType = z.enum(["versioned", "undo", "repeatable"]);
+
+export const EventRow = z.object({
+  id: z.number(),
+  type: MigrationType,
+  filename: z.string(),
+  script: z.string(),
+  applied_by: z.string(),
+  applied_at: z.string(),
+});