skir 0.0.1 → 0.0.2

package/src/formatter.ts

@@ -1,274 +1,285 @@
-import type { Token } from "./types.js";
+import type { Token } from "skir-internal";
+import { ModuleTokens } from "./tokenizer.js";
 
-export function formatModule(tokens: readonly Token[]): string {
-  const sink = new CodeSink();
-  let inValue = false;
-  let indentDepth = 0;
-  const iterator = new TokenIterator(tokens);
+export interface FormattedModule {
+  readonly newSourceCode: string;
+  /// For VSCode extension: text edits to convert the original source code into
+  // the formatted source code.
+  readonly textEdits: readonly TextEdit[];
+}
 
-  const copyInlineComments = (): void => {
-    while (
-      iterator.hasNext() &&
-      isComment(iterator.peek().text) &&
-      iterator.peek().line.lineNumber === iterator.current.line.lineNumber
-    ) {
-      // Preserve comments before line break.
-      sink.write("  " + iterator.next().text);
-    }
-  };
+export interface TextEdit {
+  readonly oldStart: number;
+  readonly oldEnd: number;
+  readonly newText: string;
+}
 
-  const breakLine = (): void => {
-    copyInlineComments();
-    const { lastLineOnlyHasWhitespaces } = sink;
-    if (iterator.hasNext()) {
-      const current = iterator.current;
-      const next = iterator.peek();
-      if (next.line.lineNumber >= current.line.lineNumber + 2) {
-        // Preserve double line breaks.
-        sink.write("\n");
-      }
-    }
-    if (!lastLineOnlyHasWhitespaces) {
-      sink.write("\n" + "  ".repeat(indentDepth));
-    }
-  };
+/**
+ * Formats the given module and returns the new source code.
+ * Preserves token ordering.
+ */
+export function formatModule(moduleTokens: ModuleTokens): FormattedModule {
+  const tokens = moduleTokens.tokensWithComments;
 
-  const breakLineAndIndent = (): void => {
-    ++indentDepth;
-    breakLine();
+  const context: Context = {
+    context: null,
+    indentStack: [{ indent: "" }],
   };
 
-  const unindent = (): void => {
-    --indentDepth;
-    sink.removeWhitespaceSuffix("  ");
+  let newSourceCode = "";
+  const textEdits: TextEdit[] = [];
+
+  const appendToken: (t: Token) => void = (t: Token) => {
+    const newToken = normalizeToken(t.text);
+    if (newToken !== t.text) {
+      textEdits.push({
+        oldStart: t.position,
+        oldEnd: t.position + t.text.length,
+        newText: newToken,
+      });
+    }
+    newSourceCode += newToken;
   };
+  appendToken(tokens[0]!);
 
-  while (iterator.hasNext()) {
-    const token = iterator.next();
-    switch (token.text) {
-      case "as":
-      case "const":
-      case "enum":
-      case "import":
-      case "method":
-      case "struct":
-      case "*":
-      case ":": {
-        sink.write(token.text + " ");
-        break;
-      }
-      case "from": {
-        sink.write(" from ");
-        break;
-      }
-      case "removed": {
-        if (iterator.hasNext() && iterator.peek().text === ";") {
-          sink.write("removed");
-        } else {
-          sink.write("removed ");
-        }
-        break;
-      }
-      case "{": {
-        if (iterator.hasNext() && iterator.peek().text === "}") {
-          sink.write(inValue ? "{}" : " {}");
-          iterator.next();
-          if (!inValue) {
-            breakLine();
-          }
-        } else {
-          sink.write(inValue ? "{" : " {");
-          breakLineAndIndent();
-        }
-        break;
-      }
-      case "{|": {
-        if (iterator.hasNext() && iterator.peek().text === "|}") {
-          sink.write(inValue ? "{||}" : " {||}");
-          iterator.next();
-          if (!inValue) {
-            breakLine();
-          }
-        } else {
-          sink.write(inValue ? "{|" : " {|");
-          breakLineAndIndent();
-        }
-        break;
-      }
-      case "}": {
-        if (inValue) {
-          sink.maybeWriteTrailingComma();
-          breakLine();
-        }
-        unindent();
-        sink.write("}");
-        if (!inValue) {
-          breakLine();
-        }
-        break;
-      }
-      case "|}": {
-        if (inValue) {
-          sink.maybeWriteTrailingComma();
-          breakLine();
-        }
-        unindent();
-        sink.write("|}");
-        if (!inValue) {
-          breakLine();
-        }
-        break;
-      }
-      case "[": {
-        if (iterator.hasNext() && iterator.peek().text === "]") {
-          sink.write("[]");
-          iterator.next();
-        } else {
-          sink.write("[");
-          if (inValue) {
-            breakLineAndIndent();
-          }
-        }
-        break;
-      }
-      case "]": {
-        if (inValue) {
-          sink.maybeWriteTrailingComma();
-          breakLine();
-          unindent();
-        }
-        sink.write("]");
-        break;
-      }
-      case ";": {
-        sink.write(";");
-        inValue = false;
-        breakLine();
-        break;
-      }
-      case "=": {
-        inValue = true;
-        sink.write(" = ");
-        break;
-      }
-      case ",": {
-        if (inValue) {
-          sink.write(",");
-          breakLine();
-        } else {
-          sink.write(", ");
-        }
+  for (let i = 1; i < tokens.length; i++) {
+    const token = tokens[i - 1]!;
+    const next = tokens[i]!;
+
+    // Find the next non-comment token
+    let nextNonComment = next;
+    for (let j = i; j < tokens.length; j++) {
+      const token = tokens[j]!;
+      if (!isComment(token)) {
+        nextNonComment = token;
         break;
       }
-      default: {
-        if (isComment(token.text)) {
-          sink.writeComment(token.text);
-          breakLine();
-        } else if (token.text.startsWith("'")) {
-          const unescapedDoubleQuoteRegex = /(?:^|[^\\])(?:\\\\)*"/;
-          if (unescapedDoubleQuoteRegex.test(token.text)) {
-            sink.write(token.text);
-          } else {
-            // Switch to double quotes.
-            const unquoted = token.text.slice(1, -1);
-            sink.write(`"${unquoted}"`);
-          }
-        } else {
-          sink.write(token.text);
-        }
-      }
     }
-  }
 
-  const result = sink.code;
-  if (indentDepth !== 0) {
-    throw new Error(`result=${result}`);
+    // Determine the text to add after 'token' and before 'next': a possible
+    // trailing comma followed by whitespace.
+    let newSeparator = shouldAddTrailingComma(token, nextNonComment!, context)
+      ? ","
+      : "";
+    newSeparator += getWhitespaceAfterToken(
+      token,
+      next,
+      nextNonComment!,
+      context,
+    );
+    const topOfStack = context.indentStack.at(-1)!;
+    if (newSeparator.endsWith("\n")) {
+      newSeparator = newSeparator + topOfStack.indent;
+    }
+
+    const oldSeparator = moduleTokens.sourceCode.slice(
+      token.position + token.text.length,
+      next.position,
+    );
+    if (oldSeparator !== newSeparator) {
+      textEdits.push({
+        oldStart: token.position + token.text.length,
+        oldEnd: next.position,
+        newText: newSeparator,
+      });
+    }
+
+    newSourceCode += newSeparator;
+
+    appendToken(next);
   }
 
-  return result;
+  return {
+    newSourceCode: newSourceCode,
+    textEdits: textEdits,
+  };
 }
 
-class TokenIterator {
-  private nextIndex = 0;
+type Context = {
+  context:
+    | "const" // Between 'const' and '='
+    | "in-value" // After 'const', between '=' and ';'
+    | "removed" // Between 'removed' and ';'
+    | null;
+  readonly indentStack: IndentStackItem[];
+};
 
-  constructor(private readonly tokens: readonly Token[]) {}
+interface IndentStackItem {
+  indent: string;
+  // If true, the new indentation level is for the declaration of an inline
+  // record as a method request type:
+  //    method GetFoo(
+  //      struct {
+  //        ...
+  //      }
+  //    ): Foo;
+  inlineRecordInBracket?: true;
+}
 
-  next(): Token {
-    if (this.nextIndex < this.tokens.length) {
-      return this.tokens[this.nextIndex++]!;
-    }
-    throw new Error();
-  }
+function getWhitespaceAfterToken(
+  token: Token,
+  next: Token,
+  // If 'next' is a comment, the next non-comment token after 'next'.
+  // Otherwise, 'next' itself.
+  nextNonComment: Token,
+  context: Context,
+): "" | " " | "  " | "\n" | "\n\n" {
+  const topOfStack: () => IndentStackItem = () => context.indentStack.at(-1)!;
 
-  peek(): Token {
-    if (this.nextIndex < this.tokens.length) {
-      return this.tokens[this.nextIndex]!;
-    }
-    throw new Error();
+  const indentUnit = "  ";
+  if (
+    token.text === "{" ||
+    token.text === "{|" ||
+    (context.context === "in-value" && token.text === "[")
+  ) {
+    context.indentStack.push({
+      indent: topOfStack().indent + indentUnit,
+    });
+  } else if (
+    token.text === "(" &&
+    ["struct", "enum"].includes(nextNonComment.text)
+  ) {
+    context.indentStack.push({
+      indent: topOfStack().indent + indentUnit,
+      inlineRecordInBracket: true,
+    });
   }
 
-  hasNext(): boolean {
-    return this.nextIndex < this.tokens.length;
+  if (
+    next.text === "}" ||
+    next.text === "|}" ||
+    (context.context === "in-value" && next.text === "]") ||
+    (next.text === ")" && topOfStack().inlineRecordInBracket)
+  ) {
+    context.indentStack.pop();
   }
 
-  get current(): Token {
-    const index = this.nextIndex - 1;
-    if (index < this.tokens.length) {
-      return this.tokens[index]!;
+  if (isComment(token)) {
+    return oneOrTwoLineBreaks(token, next);
+  } else if (
+    token.text !== "{" &&
+    next.text === "}" &&
+    context.context !== "in-value"
+  ) {
+    return "\n";
+  } else if (isComment(next)) {
+    return token.line.lineNumber === next.line.lineNumber
+      ? "  "
+      : oneOrTwoLineBreaks(token, next);
+  } else if (next.text === "=") {
+    return " ";
+  } else if (
+    (token.text === "[" && next.text === "]") ||
+    (token.text === "{" && next.text === "}") ||
+    (token.text === "{|" && next.text === "|}")
+  ) {
+    return "";
+  } else if (["{", "{|"].includes(token.text)) {
+    return "\n";
+  } else if (token.text === "[") {
+    return context.context === "in-value" ? "\n" : "";
+  } else if (["*", ":"].includes(token.text)) {
+    return " ";
+  } else if (token.text === "(") {
+    return ["struct", "enum"].includes(next.text) ? "\n" : "";
+  } else if (token.text === ")") {
+    return next.text === "{" ? " " : "";
+  } else if (token.text === ";") {
+    context.context = null;
+    return oneOrTwoLineBreaks(token, next);
+  } else if (token.text === "}") {
+    return [",", ";"].includes(next.text)
+      ? ""
+      : oneOrTwoLineBreaks(token, next);
+  } else if (token.text === ",") {
+    return context.context === "removed" ? " " : "\n";
+  } else if (token.text === "=") {
+    if (context.context === "const") {
+      context.context = "in-value";
     }
-    throw new Error();
+    return " ";
+  } else if (token.text === "const") {
+    context.context = "const";
+    return " ";
+  } else if (token.text === "removed") {
+    context.context = "removed";
+    return next.text === ";" ? "" : " ";
+  } else if (
+    context.context === "in-value" &&
+    ["]", "}", "|}"].includes(next.text)
+  ) {
+    return "\n";
+  } else if (
+    /^[A-Za-z]/.test(token.text) &&
+    !["(", ":", ",", ";", "|", ".", ")", "]", "?"].includes(next.text)
+  ) {
+    return " ";
+  } else {
+    return "";
   }
 }
 
-function isComment(token: string): boolean {
-  return token.startsWith("//") || token.startsWith("/*");
+function shouldAddTrailingComma(
+  first: Token,
+  nextNonComment: Token,
+  context: Context,
+): boolean {
+  return (
+    context.context === "in-value" &&
+    ["]", "}", "|}"].includes(nextNonComment.text) &&
+    !["[", "{", "{|", ","].includes(first.text)
+  );
 }
 
-class CodeSink {
-  private _code: string = "";
-  // Position after the last non-whitespace character which is not part of a
-  // comment.
-  private endPosition = 0;
-
-  /** Writes a token possibly preceded or followed by whitespaces. */
-  write(text: string): void {
-    const trimmed = text.trim();
-    if (trimmed && !isComment(trimmed)) {
-      this.endPosition = trimmed.endsWith(",")
-        ? 0
-        : this.code.length + text.trimEnd().length;
-    }
-    this._code += text;
+function oneOrTwoLineBreaks(first: Token, second: Token): "\n" | "\n\n" {
+  const firstLineNumber =
+    first.line.lineNumber + first.text.split("\n").length - 1;
+  if (
+    firstLineNumber < second.line.lineNumber - 1 &&
+    (isComment(second) || /^[A-Za-z]/.test(second.text))
+  ) {
+    return "\n\n";
+  } else {
+    return "\n";
   }
+}
 
-  writeComment(text: string): void {
-    if (this.lastLineOnlyHasWhitespaces) {
-      this._code += text;
-    } else {
-      this._code = this.code.trimEnd() + "  " + text;
-    }
-  }
+function isComment(token: Token): boolean {
+  return token.text.startsWith("//") || token.text.startsWith("/*");
+}
 
-  maybeWriteTrailingComma(): void {
-    if (this.endPosition === 0) {
-      return;
+function normalizeToken(token: string): string {
+  if (token.startsWith("//")) {
+    // Make sure there is a space between the double slash and the comment text.
+    if (
+      token.startsWith("// ") ||
+      token.startsWith("/// ") ||
+      token === "//" ||
+      token === "///"
+    ) {
+      return token;
+    } else if (token.startsWith("///")) {
+      return "/// " + token.slice(3);
+    } else {
+      return "// " + token.slice(2);
     }
-    const { code } = this;
-    this._code =
-      code.slice(0, this.endPosition) + "," + code.slice(this.endPosition);
-  }
-
-  removeWhitespaceSuffix(suffix: string): void {
-    if (this.code.endsWith(suffix)) {
-      this._code = this.code.slice(0, -suffix.length);
+  } else if (token.startsWith("'")) {
+    // A single-quoted string
+    if (token.includes('"')) {
+      // Remove escape characters before single quotes.
+      return token.replace(/\\(?=(?:\\\\)*')/g, "");
+    } else {
+      // If the string does not contain double quotes, turn it into a
+      // double-quoted string for consistency
+      const content = token.slice(1, -1);
+      // Remove escape characters before double quotes.
+      return '"' + content.replace(/\\(?=(?:\\\\)*")/g, "") + '"';
     }
-  }
-
-  get lastLineOnlyHasWhitespaces(): boolean {
-    return /^$|\n\s*$/.test(this.code);
-  }
-
-  get code(): string {
-    return this._code;
+  } else if (token.startsWith('"')) {
+    // A double-quoted string
+    // Remove escape characters before double quotes.
+    return token.replace(/\\(?=(?:\\\\)*')/g, "");
+  } else {
+    return token;
   }
 }

package/src/index.ts

@@ -1,6 +0,0 @@
-export type * from "./types.js";
-
-export { capitalize, convertCase } from "./casing.js";
-export { encodeInt32 } from "./encoding.js";
-export { unquoteAndUnescape } from "./literals.js";
-export { simpleHash } from "./parser.js";

package/src/language_server.ts

@@ -1,9 +1,3 @@
-import * as yaml from "yaml";
-import { fromZodError } from "zod-validation-error";
-import { SkirConfig } from "./config.js";
-import { ModuleParser, ModuleSet } from "./module_set.js";
-import { parseModule } from "./parser.js";
-import { tokenizeModule } from "./tokenizer.js";
 import type {
   Module,
   MutableModule,
@@ -11,7 +5,13 @@ import type {
   RecordLocation,
   Result,
   SkirError,
-} from "./types.js";
+} from "skir-internal";
+import * as yaml from "yaml";
+import { fromZodError } from "zod-validation-error";
+import { SkirConfig } from "./config.js";
+import { ModuleParser, ModuleSet } from "./module_set.js";
+import { parseModule } from "./parser.js";
+import { tokenizeModule } from "./tokenizer.js";
 
 export class LanguageServerModuleSet {
   constructor(private readonly rootPath: string) {}
@@ -120,7 +120,7 @@ export class LanguageServerModuleSet {
           errors: tokens.errors,
         };
       } else {
-        astTree = parseModule(tokens.result, uri, content);
+        astTree = parseModule(tokens.result);
       }
     }
     return {

package/src/literals.ts

@@ -1,17 +1,8 @@
-import type { DenseJson, Primitive } from "./types.js";
-
-export function unquoteAndUnescape(stringLiteral: string): string {
-  const unquoted = stringLiteral.slice(1, -1);
-  const parts = unquoted.split("\\\\");
-  for (let i = 0; i < parts.length; ++i) {
-    const part = parts[i]!.replace(/\\(\r\n|\n|\r)/g, "\\n")
-      // Escape unescaped double quotes which can appear in a single-quoted
-      // string literal.
-      .replace(/(?<=^|[^\\])"/g, '\\"');
-    parts[i] = JSON.parse(`"${part}"`);
-  }
-  return parts.join("\\");
-}
+import {
+  unquoteAndUnescape,
+  type DenseJson,
+  type Primitive,
+} from "skir-internal";
 
 export function valueHasPrimitiveType(
   token: string,