skir 0.0.1

package/src/snapshotter.ts

@@ -0,0 +1,136 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { checkBackwardCompatibility } from "./compatibility_checker.js";
+import {
+  formatError,
+  makeRed,
+  renderBreakingChanges,
+  renderErrors,
+} from "./error_renderer.js";
+import { collectModules } from "./module_collector.js";
+import { ModuleSet } from "./module_set.js";
+
+export async function takeSnapshot(args: {
+  rootDir: string;
+  srcDir: string;
+  check: boolean;
+}): Promise<void> {
+  const newModuleSet = await collectModules(args.srcDir);
+  if (newModuleSet.errors.length) {
+    renderErrors(newModuleSet.errors);
+    process.exit(1);
+  }
+  const snapshotPath = join(args.rootDir, "skir-snapshot.json");
+  const oldModuleSet = await readLastSnapshot(join(args.rootDir, snapshotPath));
+  if (!(oldModuleSet instanceof ModuleSet)) {
+    console.error(makeRed(`Corrupted snapshot file: ${snapshotPath}`));
+    console.error(`Error: ${oldModuleSet.error.toString()}`);
+    console.log(
+      "If the snapshot file cannot be restored to a valid state, delete it and run again. " +
+        "Breaking changes from recent commits will not be detected, but a valid snapshot will be created for future comparisons.",
+    );
+    process.exit(1);
+  }
+  const breakingChanges = checkBackwardCompatibility({
+    before: oldModuleSet,
+    after: newModuleSet,
+  });
+  if (breakingChanges.length) {
+    renderBreakingChanges(breakingChanges, {
+      before: oldModuleSet,
+      after: newModuleSet,
+    });
+    process.exit(1);
+  }
+  const newSnapshot = makeSnapshot(newModuleSet, new Date());
+  if (sameModules(newSnapshot, makeSnapshot(oldModuleSet, new Date()))) {
+    console.log("No changes detected since last snapshot.");
+    return;
+  }
+  if (args.check) {
+    console.error(
+      makeRed(
+        `Modules have changed since the last snapshot. ` +
+          `Run the command without --check to take a new snapshot.`,
+      ),
+    );
+    process.exit(1);
+  }
+  await writeFile(snapshotPath, JSON.stringify(newSnapshot, null, 2), "utf-8");
+  console.log("Snapshot taken. No breaking changes detected.");
+}
+
+async function readLastSnapshot(snapshotPath: string): Promise<
+  | ModuleSet
+  | {
+      kind: "corrupted";
+      error: any;
+    }
+> {
+  let snapshot: Snapshot;
+  try {
+    const textContent = await readFile(snapshotPath, "utf-8");
+    snapshot = JSON.parse(textContent);
+  } catch (error) {
+    if (error instanceof SyntaxError) {
+      return {
+        kind: "corrupted",
+        error: error,
+      };
+    }
+    const isNotFoundError =
+      error instanceof Error && "code" in error && error.code === "ENOENT";
+    if (isNotFoundError) {
+      return ModuleSet.fromMap(new Map<string, string>());
+    } else {
+      // Rethrow I/O error
+      throw error;
+    }
+  }
+  const pathToSourceCode = new Map<string, string>();
+  try {
+    for (const [path, sourceCode] of Object.entries(snapshot.modules)) {
+      // + "" to ensure string type
+      pathToSourceCode.set(path + "", sourceCode + "");
+    }
+  } catch (error) {
+    return {
+      kind: "corrupted",
+      error: error,
+    };
+  }
+  const moduleSet = ModuleSet.fromMap(pathToSourceCode);
+  if (moduleSet.errors.length) {
+    const firstError = formatError(moduleSet.errors[0]!);
+    return {
+      kind: "corrupted",
+      error: new Error(`errors in modules; first error: ${firstError}`),
+    };
+  }
+  return moduleSet;
+}
+
+function makeSnapshot(moduleSet: ModuleSet, now: Date): Snapshot {
+  const modules: { [path: string]: string } = {};
+  for (const module of moduleSet.resolvedModules) {
+    modules[module.path] = module.sourceCode;
+  }
+  return {
+    lastChange: now.toISOString(),
+    modules: modules,
+  };
+}
+
+function sameModules(a: Snapshot, b: Snapshot): boolean {
+  return (
+    Object.keys(a.modules).length === Object.keys(b.modules).length ||
+    Object.entries(a.modules).every(([path, sourceCode]) => {
+      return sourceCode === b.modules[path];
+    })
+  );
+}
+
+interface Snapshot {
+  lastChange: string;
+  modules: { [path: string]: string };
+}

package/src/tokenizer.ts

@@ -0,0 +1,216 @@
+import { unquoteAndUnescape } from "./literals.js";
+import type { CodeLine, ErrorSink, Result, SkirError, Token } from "./types.js";
+
+/** Tokenizes the given module. */
+export function tokenizeModule(
+  code: string,
+  modulePath: string,
+  keepComments?: "keep-comments",
+): Result<Token[]> {
+  const tokens: Token[] = [];
+  const errors: SkirError[] = [];
+
+  const lines = new Lines(code, modulePath);
+
+  // Multiline comment:            \/\*([^*]|\*[^\/])*(\*\/)?
+  // Single-line comment:          \/\/[^\n\r]*
+  // Number:                       -?(0|[1-9][0-9]*)(\.[0-9]+)?\b
+  // Word:                         \b\w+\b
+  // Whitespaces:                  [ \n\r\t]+
+  // Symbol:                       \.\.|\{\||\|\}|[{}[\]()*.:=;|?,]
+  // Double-quote string literal:  "(\\(\n|\r|\n\r|.)|[^\\\"\n\r])*"?
+  // Single-quote string literal:  '(\\(\n|\r|\n\r|.)|[^\\\'\n\r])*'?
+  //
+  // To iterate on this regex, use https://regex101.com/.
+  const re =
+    /(\/\*([^*]|\*[^/])*(\*\/)?)|(\/\/[^\n\r]*)|(-?(0|[1-9][0-9]*)(\.[0-9]+)?\b)|\b(\w+)\b|([ \n\r\t]+)|(\.\.|\{\||\|\}|[{}[\]()*.:=;|?,])|("(\\(\n|\r|\n\r|.)|[^\\"\n\r])*"?)|('(\\(\n|\r|\n\r|.)|[^\\'\n\r])*'?)|($)/g;
+  // 1    2             3        4              5  6              7                8       9            10                                 11   13                             14   16                             17
+
+  let group: RegExpExecArray | null;
+  let expectedPosition = 0;
+  while ((group = re.exec(code)) !== null) {
+    const position = re.lastIndex - group[0].length;
+
+    // Check that all the regex matches are consecutive.
+    // If there is a gap between two matches, it means that we have an invalid
+    // sequence of characters.
+    if (position !== expectedPosition) {
+      const line = lines.advancePosition(expectedPosition);
+      const colNumber = expectedPosition - line.position;
+      errors.push({
+        token: {
+          text: code.substring(expectedPosition, position),
+          position: expectedPosition,
+          line: line,
+          colNumber: colNumber,
+        },
+        message: "Invalid sequence of characters",
+      });
+    }
+    expectedPosition = re.lastIndex;
+
+    const line = lines.advancePosition(position);
+    const colNumber = position - line.position;
+    const token = {
+      text: group[0],
+      position: position,
+      line: line,
+      colNumber: colNumber,
+    };
+
+    // Skip multiline comments.
+    if (group[1] !== undefined && !keepComments) {
+      // Make sure the multiline comment is terminated.
+      if (!group[1].endsWith("*/")) {
+        errors.push({
+          token: token,
+          message: "Unterminated multi-line comment",
+        });
+      }
+      continue;
+    }
+
+    // Skip single-line comments.
+    if (group[4] !== undefined && !keepComments) {
+      continue;
+    }
+
+    if (group[8] !== undefined) {
+      if (!validateWord(token, errors)) {
+        continue;
+      }
+    }
+
+    // Skip whitespaces.
+    if (group[9] !== undefined) continue;
+
+    // Validate string literals.
+    const stringLiteral = group[11] || group[14];
+    if (stringLiteral !== undefined) {
+      if (!stringLiteral.endsWith(stringLiteral[0]!)) {
+        errors.push({
+          token: token,
+          message: "Unterminated string literal",
+        });
+        continue;
+      }
+      const stringLiteralRe =
+        /^["'](\\(\r\n|u[0-9A-Fa-f]{4}|["'\\/bfnrt\n\r])|[^\\])*["']$/;
+      if (!stringLiteralRe.test(stringLiteral)) {
+        errors.push({
+          token: token,
+          message: "String literal contains invalid escape sequence",
+        });
+        continue;
+      }
+      // Check that the string doesn't contain lone surrogates.
+      // See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/isWellFormed
+      const unquoted = unquoteAndUnescape(stringLiteral);
+      try {
+        encodeURIComponent(unquoted);
+      } catch {
+        errors.push({
+          token: token,
+          message: "String literal contains lone surrogates",
+        });
+        continue;
+      }
+    }
+
+    tokens.push(token);
+
+    if (group[17] !== undefined) {
+      // The end of the string has been reached.
+      // The last token we added is an empty string.
+      break;
+    }
+  }
+
+  return { result: tokens, errors: errors };
+}
+
+function validateWord(token: Token, errors: ErrorSink): boolean {
+  if (/^[0-9]/.test(token.text)) {
+    errors.push({
+      token: token,
+      message: "Invalid number",
+    });
+    return false;
+  }
+
+  if (token.text.startsWith("_")) {
+    errors.push({
+      token: token,
+      message: "Identifier cannot start with _",
+    });
+    return false;
+  }
+  if (token.text.endsWith("_")) {
+    errors.push({
+      token: token,
+      message: "Identifier cannot end with _",
+    });
+    return false;
+  }
+  if (token.text.includes("__")) {
+    errors.push({
+      token: token,
+      message: "Identifier cannot contain __ sequence",
+    });
+    return false;
+  }
+  if (/_[0-9]/.test(token.text)) {
+    // The problem is we could end up with duplicate field names when converting
+    // to camel case. For example: "field0" and "field_0" would both be
+    // converted to "field0".
+    errors.push({
+      token: token,
+      message: "Digit cannot follow _",
+    });
+    return false;
+  }
+
+  return true;
+}
+
+class Lines {
+  /** Splits the given code into multiple lines. */
+  constructor(code: string, modulePath: string) {
+    const lines = this.lines;
+    let lineStart = 0;
+    let group: RegExpExecArray | null;
+    const re = /\n|\r|\n\r|$/g;
+    while ((group = re.exec(code)) !== null) {
+      const match = group[0];
+      const lineEnd = re.lastIndex - match.length;
+      const lineString = code.substring(lineStart, lineEnd);
+      lines.push({
+        lineNumber: lines.length,
+        line: lineString,
+        position: lineStart,
+        modulePath: modulePath,
+      });
+      // Record the position after the EOL or EOF.
+      lineStart = re.lastIndex;
+      if (match === "") {
+        break;
+      }
+    }
+  }
+
+  /**
+   * Returns the line at the given position, where the position is the char
+   * index within `code`.
+   * Requires the position to be >= the position given the last time the method
+   * was called.
+   */
+  advancePosition(position: number): CodeLine {
+    const lines = this.lines;
+    while (lines.length >= 2 && position >= this.lines[1]!.position) {
+      lines.shift();
+    }
+    return lines[0]!;
+  }
+
+  private readonly lines: CodeLine[] = [];
+}