skir 0.0.1 → 0.0.2

package/dist/parser.js

@@ -1,10 +1,13 @@
+import { convertCase, simpleHash } from "skir-internal";
 import * as casing from "./casing.js";
+import { parseDocComments } from "./doc_comment_parser.js";
 /** Runs syntactic analysis on a module. */
-export function parseModule(tokens, modulePath, sourceCode) {
+export function parseModule(moduleTokens) {
+    const { tokens, modulePath, sourceCode } = moduleTokens;
     const errors = [];
     const it = new TokenIterator(tokens, errors);
     const declarations = parseDeclarations(it, "module");
-    it.expectThenMove([""]);
+    it.expectThenNext([""]);
     // Create a mappinng from names to declarations, and check for duplicates.
     const nameToDeclaration = {};
     for (const declaration of declarations) {
@@ -20,7 +23,7 @@ export function parseModule(tokens, modulePath, sourceCode) {
             if (name in nameToDeclaration) {
                 errors.push({
                     token: nameToken,
-                    message: `Duplicate identifier "${name}"`,
+                    message: `Duplicate identifier '${name}'`,
                 });
             }
             else {
@@ -52,11 +55,19 @@ function parseDeclarations(it, parentNode) {
     const isEndToken = (t) => t === "" || (parentNode !== "module" && t === "}");
     // Returns true if the token may be the last token of a valid statement.
     const isLastToken = (t) => t === "}" || t === ";";
-    while (!isEndToken(it.peek())) {
+    while (!isEndToken(it.current)) {
         const startIndex = it.index;
         const declaration = parseDeclaration(it, parentNode);
         if (declaration !== null) {
             result.push(declaration);
+            if (declaration.kind === "method") {
+                if (declaration.inlineRequestRecord) {
+                    result.push(declaration.inlineRequestRecord);
+                }
+                if (declaration.inlineResponseRecord) {
+                    result.push(declaration.inlineResponseRecord);
+                }
+            }
             continue;
         }
         // We have an invalid statement. An error was already registered. Perhaps
@@ -67,16 +78,16 @@ function parseDeclarations(it, parentNode) {
         const noTokenWasConsumed = it.index === startIndex;
         if (noTokenWasConsumed) {
             it.next();
-            if (isLastToken(it.peekBack())) {
+            if (isLastToken(it.previous)) {
                 // For example: two semicolons in a row.
                 continue;
             }
         }
         if (noTokenWasConsumed ||
-            (it.peek() !== "" && !isLastToken(it.peekBack()))) {
+            (it.current !== "" && !isLastToken(it.previous))) {
             let nestedLevel = 0;
             while (true) {
-                const token = it.peek();
+                const token = it.current;
                 if (token === "") {
                     break;
                 }
@@ -96,6 +107,7 @@ function parseDeclarations(it, parentNode) {
     return result;
 }
 function parseDeclaration(it, parentNode) {
+    const doc = parseDoc(it);
     let recordType = "enum";
     const parentIsRoot = parentNode === "module";
     const expected = [
@@ -107,31 +119,32 @@ function parseDeclaration(it, parentNode) {
         /*5:*/ parentIsRoot ? "method" : null,
         /*6:*/ parentIsRoot ? "const" : null,
     ];
-    const match = it.expectThenMove(expected);
+    const match = it.expectThenNext(expected);
     switch (match.case) {
         case 0:
             recordType = "struct";
         // Falls through.
         case 1:
-            return parseRecord(it, recordType);
+            return parseRecord(it, recordType, doc);
         case 2:
             return parseRemoved(it, match.token);
         case 3:
-            return parseField(it, match.token, parentNode);
+            return parseField(it, match.token, doc, parentNode);
         case 4:
             return parseImport(it);
         case 5:
-            return parseMethod(it);
+            return parseMethod(it, doc);
         case 6:
-            return parseConstant(it);
+            return parseConstant(it, doc);
         default:
             return null;
     }
 }
 class RecordBuilder {
-    constructor(recordName, recordType, stableId, errors) {
+    constructor(recordName, recordType, doc, stableId, errors) {
         this.recordName = recordName;
         this.recordType = recordType;
+        this.doc = doc;
         this.stableId = stableId;
         this.errors = errors;
         this.nameToDeclaration = {};
@@ -195,7 +208,7 @@ class RecordBuilder {
             if (name in this.nameToDeclaration) {
                 this.errors.push({
                     token: nameToken,
-                    message: `Duplicate identifier "${name}"`,
+                    message: `Duplicate identifier '${name}'`,
                 });
                 return;
             }
@@ -256,11 +269,11 @@ class RecordBuilder {
             key: key,
             name: this.recordName,
             recordType: this.recordType,
+            doc: this.doc,
             nameToDeclaration: this.nameToDeclaration,
             declarations: Object.values(this.nameToDeclaration),
             fields: fields,
             nestedRecords: nestedRecords,
-            numbering: this.numbering,
             removedNumbers: this.removedNumbers.sort(),
             recordNumber: this.stableId,
             numSlots: numSlots,
@@ -268,38 +281,61 @@ class RecordBuilder {
         };
     }
 }
-function parseRecord(it, recordType) {
+function parseRecord(it, recordType, doc, inlineContext) {
     // A struct or an enum.
-    const nameMatch = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
-    if (nameMatch.case < 0) {
-        return null;
+    let nameToken;
+    if (inlineContext) {
+        const { originalName } = inlineContext;
+        let transformedName = convertCase(originalName.text, "UpperCamel");
+        if (inlineContext.context === "method-request") {
+            transformedName += "Request";
+        }
+        else if (inlineContext.context === "method-response") {
+            transformedName += "Response";
+        }
+        nameToken = {
+            ...originalName,
+            text: transformedName,
+        };
+    }
+    else {
+        // Read the name.
+        const nameMatch = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
+        if (nameMatch.case < 0) {
+            return null;
+        }
+        casing.validate(nameMatch.token, "UpperCamel", it.errors);
+        nameToken = nameMatch.token;
     }
-    casing.validate(nameMatch.token, "UpperCamel", it.errors);
     let stableId = null;
-    if (it.peek() === "(") {
+    if (it.current === "(") {
         it.next();
         stableId = parseUint32(it);
         if (stableId < 0) {
             return null;
         }
-        if (it.expectThenMove([")"]).case < 0) {
+        if (it.expectThenNext([")"]).case < 0) {
             return null;
         }
     }
-    if (it.expectThenMove(["{"]).case < 0) {
+    if (it.expectThenNext(["{"]).case < 0) {
         return null;
     }
     const declarations = parseDeclarations(it, recordType);
-    it.expectThenMove(["}"]);
-    const builder = new RecordBuilder(nameMatch.token, recordType, stableId, it.errors);
+    it.expectThenNext(["}"]);
+    const builder = new RecordBuilder(nameToken, recordType, doc, stableId, it.errors);
     for (const declaration of declarations) {
         builder.addDeclaration(declaration);
+        if (declaration.kind === "field" && declaration.inlineRecord) {
+            builder.addDeclaration(declaration.inlineRecord);
+        }
     }
     return builder.build();
 }
-function parseField(it, name, recordType) {
+function parseField(it, name, doc, recordType) {
     // May only be undefined if the type is an enum.
-    let type = undefined;
+    let type;
+    let inlineRecord;
     let number = -1;
     while (true) {
         const typeAllowed = type === undefined && number < 0;
@@ -310,10 +346,16 @@ function parseField(it, name, recordType) {
             /*1:*/ numberAllowed ? "=" : null,
             /*2:*/ endAllowed ? ";" : null,
         ];
-        const match = it.expectThenMove(expected);
+        const match = it.expectThenNext(expected);
         switch (match.case) {
             case 0: {
-                type = parseType(it);
+                const inlineContext = {
+                    context: "field",
+                    originalName: name,
+                };
+                const typeOrInlineRecord = parseTypeOrInlineRecord(it, inlineContext);
+                type = typeOrInlineRecord.type;
+                inlineRecord = typeOrInlineRecord.inlineRecord;
                 if (type === undefined) {
                     return null;
                 }
@@ -340,11 +382,13 @@ function parseField(it, name, recordType) {
                     kind: "field",
                     name: name,
                     number: number,
+                    doc: doc,
                     unresolvedType: type,
                     // Will be populated at a later stage.
                     type: undefined,
                     // Will be populated at a later stage.
                     isRecursive: false,
+                    inlineRecord: inlineRecord,
                 };
             }
             case -1:
@@ -363,8 +407,32 @@ const PRIMITIVE_TYPES = new Set([
     "string",
     "bytes",
 ]);
+function parseTypeOrInlineRecord(it, inlineContext) {
+    if (it.current === "struct" || it.current === "enum") {
+        const recordType = it.current;
+        it.next();
+        const inlineRecord = parseRecord(it, recordType, EMPTY_DOC, inlineContext);
+        const type = inlineRecord
+            ? {
+                kind: "record",
+                nameParts: [inlineRecord.name],
+                absolute: false,
+            }
+            : undefined;
+        return {
+            type: type,
+            inlineRecord: inlineRecord ? inlineRecord : undefined,
+        };
+    }
+    else {
+        return {
+            type: parseType(it),
+            inlineRecord: undefined,
+        };
+    }
+}
 function parseType(it) {
-    const match = it.expectThenMove([
+    const match = it.expectThenNext([
         /*0:*/ "[",
         /*1:*/ TOKEN_IS_IDENTIFIER,
         /*2:*/ ".",
@@ -397,7 +465,7 @@ function parseType(it) {
     if (value === undefined) {
         return undefined;
     }
-    if (it.peek() === "?") {
+    if (it.current === "?") {
         it.next();
         return { kind: "optional", other: value };
     }
@@ -413,7 +481,7 @@ function parseArrayType(it) {
     let key = undefined;
     while (true) {
         const keyAllowed = !key && item.kind === "record";
-        const match = it.expectThenMove([
+        const match = it.expectThenNext([
             /*0:*/ keyAllowed ? "|" : null,
             /*1:*/ "]",
         ]);
@@ -436,12 +504,12 @@ function parseArrayType(it) {
 function parseFieldPath(it, pipeToken) {
     const fieldNames = [];
     while (true) {
-        const match = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
+        const match = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
         if (match.case < 0) {
             return undefined;
         }
         fieldNames.push(match.token);
-        if (it.peek() === ".") {
+        if (it.current === ".") {
             it.next();
         }
         else {
@@ -463,7 +531,7 @@ function parseRecordRef(it, nameOrDot) {
     const absolute = nameOrDot.text === ".";
     const nameParts = [];
     if (nameOrDot.text === ".") {
-        const match = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
+        const match = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
         if (match.case < 0) {
             return undefined;
         }
@@ -472,9 +540,9 @@ function parseRecordRef(it, nameOrDot) {
     else {
         nameParts.push(nameOrDot);
     }
-    while (it.peek() === ".") {
+    while (it.current === ".") {
         it.next();
-        const match = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
+        const match = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
         if (match.case < 0) {
             return undefined;
         }
@@ -483,7 +551,7 @@ function parseRecordRef(it, nameOrDot) {
     return { kind: "record", nameParts: nameParts, absolute: absolute };
 }
 function parseUint32(it) {
-    const match = it.expectThenMove([TOKEN_IS_POSITIVE_INT]);
+    const match = it.expectThenNext([TOKEN_IS_POSITIVE_INT]);
     if (match.case < 0) {
         return -1;
     }
@@ -500,8 +568,8 @@ function parseUint32(it) {
         return -1;
     }
 }
-// Parses the "removed" declaration.
-// Assumes the current token is the token after "removed".
+// Parses the 'removed' declaration.
+// Assumes the current token is the token after 'removed'.
 function parseRemoved(it, removedToken) {
     const numbers = [];
     // The 5 states are:
@@ -521,7 +589,7 @@ function parseRemoved(it, removedToken) {
             /*2:*/ expect === "?" || expect === "," || expect === ".." ? ";" : null,
             /*3:*/ expect === ".." ? ".." : null,
         ];
-        const match = it.expectThenMove(expected);
+        const match = it.expectThenNext(expected);
         switch (match.case) {
             case 0: {
                 // A comma.
@@ -583,7 +651,7 @@ function parseRemoved(it, removedToken) {
     };
 }
 function parseImport(it) {
-    const tokenMatch = it.expectThenMove(["*", TOKEN_IS_IDENTIFIER]);
+    const tokenMatch = it.expectThenNext(["*", TOKEN_IS_IDENTIFIER]);
     switch (tokenMatch.case) {
         case 0:
             return parseImportAs(it);
@@ -594,20 +662,20 @@ function parseImport(it) {
     }
 }
 function parseImportAs(it) {
-    if (it.expectThenMove(["as"]).case < 0)
+    if (it.expectThenNext(["as"]).case < 0)
         return null;
-    const aliasMatch = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
+    const aliasMatch = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
     if (aliasMatch.case < 0) {
         return null;
     }
     casing.validate(aliasMatch.token, "lower_underscore", it.errors);
-    if (it.expectThenMove(["from"]).case < 0)
+    if (it.expectThenNext(["from"]).case < 0)
         return null;
-    const modulePathMatch = it.expectThenMove([TOKEN_IS_STRING_LITERAL]);
+    const modulePathMatch = it.expectThenNext([TOKEN_IS_STRING_LITERAL]);
     if (modulePathMatch.case < 0) {
         return null;
     }
-    it.expectThenMove([";"]);
+    it.expectThenNext([";"]);
     const modulePath = modulePathMatch.token;
     return {
         kind: "import-alias",
@@ -617,21 +685,21 @@ function parseImportAs(it) {
 }
 function parseImportGivenNames(firstName, it) {
     const importedNames = [firstName];
-    while (it.peek() === ",") {
+    while (it.current === ",") {
         it.next();
-        const nameMatch = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
+        const nameMatch = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
         if (nameMatch.case < 0) {
             return null;
         }
         importedNames.push(nameMatch.token);
     }
-    if (it.expectThenMove(["from"]).case < 0)
+    if (it.expectThenNext(["from"]).case < 0)
         return null;
-    const modulePathMatch = it.expectThenMove([TOKEN_IS_STRING_LITERAL]);
+    const modulePathMatch = it.expectThenNext([TOKEN_IS_STRING_LITERAL]);
     if (modulePathMatch.case < 0) {
         return null;
     }
-    it.expectThenMove([";"]);
+    it.expectThenNext([";"]);
     const modulePath = modulePathMatch.token;
     return {
         kind: "import",
@@ -639,34 +707,43 @@ function parseImportGivenNames(firstName, it) {
         modulePath,
     };
 }
-function parseMethod(it) {
-    const nameMatch = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
+function parseMethod(it, doc) {
+    const nameMatch = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
     if (nameMatch.case < 0) {
         return null;
     }
-    casing.validate(nameMatch.token, "UpperCamel", it.errors);
-    if (it.expectThenMove(["("]).case < 0) {
+    const name = nameMatch.token;
+    casing.validate(name, "UpperCamel", it.errors);
+    if (it.expectThenNext(["("]).case < 0) {
         return null;
     }
-    const requestType = parseType(it);
+    const requestTypeOrInlineRecord = parseTypeOrInlineRecord(it, {
+        context: "method-request",
+        originalName: name,
+    });
+    const requestType = requestTypeOrInlineRecord.type;
     if (!requestType) {
         return null;
     }
-    if (it.expectThenMove([")"]).case < 0 || it.expectThenMove([":"]).case < 0) {
+    if (it.expectThenNext([")"]).case < 0 || it.expectThenNext([":"]).case < 0) {
         return null;
     }
-    const responseType = parseType(it);
+    const responseTypeOrInlineRecord = parseTypeOrInlineRecord(it, {
+        context: "method-response",
+        originalName: name,
+    });
+    const responseType = responseTypeOrInlineRecord.type;
     if (!responseType) {
         return null;
     }
-    const explicitNumber = it.expectThenMove(["=", ";"]).case === 0;
+    const explicitNumber = it.expectThenNext(["=", ";"]).case === 0;
     let number;
     if (explicitNumber) {
         number = parseUint32(it);
         if (number < 0) {
             return null;
         }
-        it.expectThenMove([";"]);
+        it.expectThenNext([";"]);
     }
     else {
         const methodName = nameMatch.token.text;
@@ -676,6 +753,7 @@ function parseMethod(it) {
     return {
         kind: "method",
         name: nameMatch.token,
+        doc: doc,
         unresolvedRequestType: requestType,
         unresolvedResponseType: responseType,
         // Will be populated at a later stage.
@@ -684,32 +762,35 @@ function parseMethod(it) {
         responseType: undefined,
         number: number,
         hasExplicitNumber: explicitNumber,
+        inlineRequestRecord: requestTypeOrInlineRecord.inlineRecord,
+        inlineResponseRecord: responseTypeOrInlineRecord.inlineRecord,
     };
 }
-function parseConstant(it) {
-    const nameMatch = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
+function parseConstant(it, doc) {
+    const nameMatch = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
     if (nameMatch.case < 0) {
         return null;
     }
     casing.validate(nameMatch.token, "UPPER_UNDERSCORE", it.errors);
-    if (it.expectThenMove([":"]).case < 0) {
+    if (it.expectThenNext([":"]).case < 0) {
         return null;
     }
     const type = parseType(it);
     if (!type) {
         return null;
     }
-    if (it.expectThenMove(["="]).case < 0) {
+    if (it.expectThenNext(["="]).case < 0) {
         return null;
     }
     const value = parseValue(it);
     if (value === null) {
         return null;
     }
-    it.expectThenMove([";"]);
+    it.expectThenNext([";"]);
     return {
         kind: "constant",
         name: nameMatch.token,
+        doc: doc,
         unresolvedType: type,
         type: undefined,
         value: value,
@@ -727,7 +808,7 @@ function parseValue(it) {
         /*6:*/ TOKEN_IS_NUMBER,
         /*7:*/ TOKEN_IS_STRING_LITERAL,
     ];
-    const match = it.expectThenMove(expected);
+    const match = it.expectThenNext(expected);
     switch (match.case) {
         case 0:
         case 1: {
@@ -771,17 +852,17 @@ function parseObjectValue(it, partial) {
     const closingToken = partial ? "|}" : "}";
     const entries = {};
     while (true) {
-        if (it.peek() === closingToken) {
+        if (it.current === closingToken) {
             it.next();
             return entries;
         }
-        const fieldNameMatch = it.expectThenMove([TOKEN_IS_IDENTIFIER]);
+        const fieldNameMatch = it.expectThenNext([TOKEN_IS_IDENTIFIER]);
         if (fieldNameMatch.case < 0) {
             return null;
         }
         const fieldNameToken = fieldNameMatch.token;
         const fieldName = fieldNameMatch.token.text;
-        if (it.expectThenMove([":"]).case < 0) {
+        if (it.expectThenNext([":"]).case < 0) {
             return null;
         }
         const value = parseValue(it);
@@ -798,7 +879,7 @@ function parseObjectValue(it, partial) {
             name: fieldNameToken,
             value: value,
         };
-        const endMatch = it.expectThenMove([",", closingToken]);
+        const endMatch = it.expectThenNext([",", closingToken]);
         if (endMatch.case < 0) {
             return null;
         }
@@ -808,7 +889,7 @@ function parseObjectValue(it, partial) {
     }
 }
 function parseArrayValue(it) {
-    if (it.peek() === "]") {
+    if (it.current === "]") {
         it.next();
         return [];
     }
@@ -819,19 +900,32 @@ function parseArrayValue(it) {
             return null;
         }
         items.push(item);
-        const match = it.expectThenMove([",", "]"]);
+        const match = it.expectThenNext([",", "]"]);
         if (match.case < 0) {
             return null;
         }
         if (match.token.text === "]") {
             return items;
         }
-        if (it.peek() === "]") {
+        if (it.current === "]") {
             it.next();
             return items;
         }
     }
 }
+function parseDoc(it) {
+    const docComments = [];
+    while (it.current.startsWith("///")) {
+        docComments.push(it.currentToken);
+        it.next();
+    }
+    const result = parseDocComments(docComments);
+    result.errors.forEach((e) => it.errors.push(e));
+    return result.result;
+}
+const EMPTY_DOC = {
+    pieces: [],
+};
 class TokenPredicate {
 }
 class TokenIsIdentifier extends TokenPredicate {
@@ -883,8 +977,16 @@ class TokenIterator {
     //
     // If the current token matches any predicate, i.e. if the index is not -1,
     // moves to the next token before returning. Otherwise, registers an error.
-    expectThenMove(expected) {
-        const token = this.tokens[this.tokenIndex];
+    expectThenNext(expected) {
+        let token = this.tokens[this.tokenIndex];
+        while (token.text.startsWith("///")) {
+            this.errors.push({
+                token: token,
+                message: "Doc comments can only precede declarations",
+            });
+            ++this.tokenIndex;
+            token = this.tokens[this.tokenIndex];
+        }
         for (let i = 0; i < expected.length; ++i) {
             const e = expected[i];
             if (e === null) {
@@ -907,7 +1009,7 @@ class TokenIterator {
             if (e === null) {
                 continue;
             }
-            expectedParts.push(e instanceof TokenPredicate ? e.what() : `"${e}"`);
+            expectedParts.push(e instanceof TokenPredicate ? e.what() : `'${e}'`);
         }
         const expectedMsg = expectedParts.length === 1
             ? expectedParts[0]
@@ -921,10 +1023,13 @@ class TokenIterator {
             token: token,
         };
     }
-    peek() {
-        return this.tokens[this.tokenIndex].text;
+    get currentToken() {
+        return this.tokens[this.tokenIndex];
     }
-    peekBack() {
+    get current() {
+        return this.currentToken.text;
+    }
+    get previous() {
         return this.tokens[this.tokenIndex - 1].text;
     }
     next() {
@@ -934,18 +1039,6 @@ class TokenIterator {
         return this.tokenIndex;
     }
 }
-/** Returns a uint32 hash of the given string. */
-export function simpleHash(input) {
-    // From https://stackoverflow.com/questions/6122571/simple-non-secure-hash-function-for-javascript
-    let hash = 0;
-    for (let i = 0; i < input.length; i++) {
-        const char = input.charCodeAt(i);
-        hash = (hash << 5) - hash + char;
-        hash |= 0;
-    }
-    // Signed int32 to unsigned int32.
-    return hash >>> 0;
-}
 function collectModuleRecords(declarations) {
     const result = [];
     const collect = (declarations, ancestors) => {