skir 1.1.2 → 1.1.3

package/dist/module_set.js

@@ -3,48 +3,77 @@ import { unquoteAndUnescape, } from "skir-internal";
 import { isStringLiteral, literalValueToDenseJson, literalValueToIdentity, valueHasPrimitiveType, } from "./literals.js";
 import { parseModule } from "./parser.js";
 import { tokenizeModule } from "./tokenizer.js";
+/**
+ * Result of compiling a set of modules. Immutable.
+ *
+ * Support incremental compilation by accepting an optional cache of the previous
+ * module set. This cache can be used to avoid re-parsing and re-resolving
+ * modules that haven't changed since the last compilation.
+ */
 export class ModuleSet {
-    static create(fileReader, rootPath) {
-        return new ModuleSet(new DefaultModuleParser(fileReader, rootPath));
+    static compile(modulePathToContent, cache) {
+        return new ModuleSet(modulePathToContent, cache);
     }
-    static fromMap(map) {
-        const result = new ModuleSet(new MapBasedModuleParser(map));
-        for (const modulePath of map.keys()) {
-            result.parseAndResolve(modulePath);
+    constructor(modulePathToContent, cache) {
+        this.modulePathToContent = modulePathToContent;
+        this.moduleBundles = new Map();
+        this.registry = new DeclarationRegistry();
+        this.cache = cache
+            ? new Cache(modulePathToContent, cache.moduleBundles)
+            : undefined;
+        for (const modulePath of modulePathToContent.keys()) {
+            this.parseAndResolve(modulePath, new Set());
         }
-        return result;
-    }
-    constructor(moduleParser) {
-        this.moduleParser = moduleParser;
-        this.mutableModules = new Map();
-        this.mutableRecordMap = new Map();
-        this.mutableResolvedModules = [];
-        this.numberToRecord = new Map();
-        this.numberToMethod = new Map();
-        this.mutableErrors = [];
+        this.finalizationResult = this.finalize();
+        // So it can be garbage collected.
+        this.cache = undefined;
     }
     parseAndResolve(modulePath, inProgressSet) {
-        const inMap = this.mutableModules.get(modulePath);
+        const inMap = this.moduleBundles.get(modulePath);
         if (inMap !== undefined) {
             return inMap;
         }
-        const result = this.doParseAndResolve(modulePath, inProgressSet || new Set());
-        this.mutableModules.set(modulePath, result);
-        this.mutableErrors.push(...result.errors);
+        const result = this.doParseAndResolve(modulePath, inProgressSet);
+        if (result) {
+            this.moduleBundles.set(modulePath, result);
+        }
         return result;
     }
     /** Called by `parseAndResolve` when the module is not in the map already. */
     doParseAndResolve(modulePath, inProgressSet) {
-        const errors = [];
-        let module;
+        const moduleContent = this.modulePathToContent.get(modulePath);
+        if (moduleContent === undefined) {
+            return null;
+        }
+        let moduleTokens;
         {
-            const parseResult = this.moduleParser.parseModule(modulePath);
-            if (parseResult.result === null) {
-                return parseResult;
+            const moduleCacheResult = this.cache?.getModuleCacheResult(modulePath) ?? { kind: "no-cache" };
+            switch (moduleCacheResult.kind) {
+                case "no-cache": {
+                    moduleTokens = tokenizeModule(moduleContent, modulePath);
+                    break;
+                }
+                case "module-tokens": {
+                    moduleTokens = moduleCacheResult.tokens;
+                    break;
+                }
+                case "module-bundle": {
+                    this.registry.mergeFrom(moduleCacheResult.bundle.registry);
+                    return moduleCacheResult.bundle;
+                }
             }
+        }
+        let module;
+        const errors = [];
+        {
+            const parseResult = parseModule(moduleTokens.result, "strict");
             errors.push(...parseResult.errors);
             module = parseResult.result;
         }
+        const moduleBundle = new ModuleBundle(moduleTokens, {
+            result: module,
+            errors: errors,
+        });
         // Process all imports.
         const pathToImports = new Map();
         for (const declaration of module.declarations) {
@@ -76,14 +105,16 @@ export class ModuleSet {
             inProgressSet.add(modulePath);
             const otherModule = this.parseAndResolve(otherModulePath, inProgressSet);
             inProgressSet.delete(modulePath);
-            if (otherModule.result === null) {
+            if (otherModule === null) {
                 errors.push({
                     token: declaration.modulePath,
                     message: "Module not found",
+                    expectedNames: suggestModulePaths(unquoteAndUnescape(declaration.modulePath.text), modulePath, this.modulePathToContent),
                 });
             }
-            else if (otherModule.errors.length !== 0) {
-                const hasCircularDependency = otherModule.errors.some((e) => e.message === circularDependencyMessage);
+            else if (otherModule.tokens.errors.length !== 0 ||
+                otherModule.module.errors.length !== 0) {
+                const hasCircularDependency = otherModule.module.errors.some((e) => e.message === circularDependencyMessage);
                 if (hasCircularDependency) {
                     errors.push({
                         token: declaration.modulePath,
@@ -102,12 +133,12 @@ export class ModuleSet {
                 // Make sure that the symbols we are importing exist in the imported
                 // module and are not imported symbols themselves.
                 for (const importedName of declaration.importedNames) {
-                    const importedDeclaration = otherModule.result.nameToDeclaration[importedName.text];
+                    const importedDeclaration = otherModule.module.result.nameToDeclaration[importedName.text];
                     if (importedDeclaration === undefined) {
                         errors.push({
                             token: importedName,
                             message: "Not found",
-                            expectedNames: declarationsToExpectedNames(otherModule.result.nameToDeclaration, (d) => d.kind === "record"),
+                            expectedNames: declarationsToExpectedNames(otherModule.module.result.nameToDeclaration, (d) => d.kind === "record"),
                         });
                     }
                     else if (importedDeclaration.kind === "import") {
@@ -126,7 +157,7 @@ export class ModuleSet {
             }
         }
         const pathToImportedNames = module.pathToImportedNames;
-        for (const [path, imports] of pathToImports.entries()) {
+        for (const [path, imports] of pathToImports) {
             const importsNoAlias = imports.filter((i) => i.kind === "import");
             const importsWithAlias = imports.filter((i) => i.kind === "import-alias");
             if (importsNoAlias.length && importsWithAlias.length) {
@@ -167,41 +198,25 @@ export class ModuleSet {
                 };
             }
         }
-        const result = {
-            result: module,
-            errors: errors,
-        };
         if (errors.length) {
-            return result;
+            return moduleBundle;
         }
-        this.mutableResolvedModules.push(module);
         // We can't merge these 3 loops into a single one, each operation must run
         // after the last operation ran on the whole map.
         // Loop 1: merge the module records map into the cross-module record map.
         for (const record of module.records) {
             const { key } = record.record;
-            this.mutableRecordMap.set(key, record);
+            this.registry.recordMap.set(key, record);
+            moduleBundle.registry.recordMap.set(key, record);
             const { recordNumber } = record.record;
             if (recordNumber != null && !modulePath.startsWith("@")) {
-                const existing = this.numberToRecord.get(recordNumber);
-                if (existing === undefined) {
-                    this.numberToRecord.set(recordNumber, key);
-                }
-                else {
-                    const otherRecord = this.recordMap.get(existing);
-                    const otherRecordName = otherRecord.record.name.text;
-                    const otherModulePath = otherRecord.modulePath;
-                    errors.push({
-                        token: record.record.name,
-                        message: `Same number as ${otherRecordName} in ${otherModulePath}`,
-                    });
-                }
+                moduleBundle.registry.pushNumberRecord(recordNumber, key);
             }
         }
         // Loop 2: resolve every field type of every record in the module.
         // Store the result in the Field object.
         const usedImports = new Set();
-        const typeResolver = new TypeResolver(module, this.mutableModules, usedImports, errors);
+        const typeResolver = new TypeResolver(module, this.moduleBundles, usedImports, errors);
         for (const record of module.records) {
             this.storeResolvedFieldTypes(record, typeResolver);
         }
@@ -247,21 +262,8 @@ export class ModuleSet {
                     this.validateArrayKeys(responseType, errors);
                 }
             }
-            if (!modulePath.startsWith("@")) {
-                const { number } = method;
-                const existing = this.numberToMethod.get(number);
-                if (existing === undefined) {
-                    this.numberToMethod.set(number, method);
-                }
-                else {
-                    const otherMethodName = existing.name.text;
-                    const otherModulePath = existing.name.line.modulePath;
-                    errors.push({
-                        token: method.name,
-                        message: `Same number as ${otherMethodName} in ${otherModulePath}`,
-                    });
-                }
-            }
+            const { number } = method;
+            moduleBundle.registry.pushNumberMethod(number, method);
             // Resolve the references in the doc comments of the method.
             this.resolveDocReferences(method, module, errors);
         }
@@ -279,7 +281,8 @@ export class ModuleSet {
             this.resolveDocReferences(constant, module, errors);
         }
         ensureAllImportsAreUsed(module, usedImports, errors);
-        return result;
+        this.registry.mergeFrom(moduleBundle.registry);
+        return moduleBundle;
     }
     storeResolvedFieldTypes(record, typeResolver) {
         for (const field of record.record.fields) {
@@ -776,8 +779,8 @@ export class ModuleSet {
                         if (!resolvedModulePath) {
                             return false;
                         }
-                        const importedModule = this.mutableModules.get(resolvedModulePath);
-                        if (!importedModule?.result) {
+                        const importedModule = this.moduleBundles.get(resolvedModulePath);
+                        if (!importedModule) {
                             return false;
                         }
                         let newNameChain;
@@ -788,7 +791,7 @@ export class ModuleSet {
                             firstName.declaration = match;
                             newNameChain = nameParts.slice(1);
                         }
-                        return tryResolveReference(ref, newNameChain, importedModule.result);
+                        return tryResolveReference(ref, newNameChain, importedModule.module.result);
                     }
                     case "constant":
                     case "field":
@@ -801,7 +804,7 @@ export class ModuleSet {
                 }
             }
         };
-        const { recordMap } = this;
+        const { recordMap } = this.registry;
         // Build list of naming scopes to search, in order of priority.
         const scopes = [];
         const pushRecordAncestorsToScopes = (record) => {
@@ -867,43 +870,87 @@ export class ModuleSet {
             }
         }
     }
-    mergeFrom(other) {
-        for (const [key, value] of other.mutableModules.entries()) {
-            this.mutableModules.set(key, value);
+    finalize() {
+        const modules = new Map();
+        for (const [modulePath, moduleBundle] of this.moduleBundles) {
+            const { module, tokens } = moduleBundle;
+            const moduleErrors = [...tokens.errors, ...module.errors];
+            modules.set(modulePath, {
+                result: module.result,
+                errors: moduleErrors,
+            });
         }
-        for (const [key, value] of other.recordMap.entries()) {
-            this.mutableRecordMap.set(key, value);
+        // Look for duplicate method numbers.
+        for (const methods of this.registry.numberToMethods.values()) {
+            if (methods.length <= 1) {
+                continue;
+            }
+            const pushError = (method, other) => {
+                const modulePath = method.name.line.modulePath;
+                const moduleResult = modules.get(modulePath);
+                const otherMethodName = other.name.text;
+                const otherModulePath = other.name.line.modulePath;
+                moduleResult.errors.push({
+                    token: method.name,
+                    message: `Same number as ${otherMethodName} in ${otherModulePath}`,
+                });
+            };
+            pushError(methods[0], methods[1]);
+            for (let i = 1; i < methods.length; ++i) {
+                pushError(methods[i], methods[0]);
+            }
         }
-        this.mutableResolvedModules.push(...other.resolvedModules);
-        for (const [key, value] of other.numberToRecord.entries()) {
-            this.numberToRecord.set(key, value);
+        // Look for duplicate record numbers.
+        for (const records of this.registry.numberToRecords.values()) {
+            if (records.length <= 1) {
+                continue;
+            }
+            const pushError = (recordKey, otherKey) => {
+                const record = this.registry.recordMap.get(recordKey);
+                const other = this.registry.recordMap.get(otherKey);
+                const modulePath = record.record.name.line.modulePath;
+                const moduleResult = modules.get(modulePath);
+                const otherRecordName = other.record.name.text;
+                const otherModulePath = other.modulePath;
+                moduleResult.errors.push({
+                    token: record.record.name,
+                    message: `Same number as ${otherRecordName} in ${otherModulePath}`,
+                });
+            };
+            pushError(records[0], records[1]);
+            for (let i = 1; i < records.length; ++i) {
+                pushError(records[i], records[0]);
+            }
         }
-        for (const [key, value] of other.numberToMethod.entries()) {
-            this.numberToMethod.set(key, value);
+        // Aggregate errors across all modules.
+        const errors = [];
+        for (const moduleBundle of modules.values()) {
+            errors.push(...moduleBundle.errors);
         }
-        this.mutableErrors.push(...other.errors);
-    }
-    get modules() {
-        return this.mutableModules;
-    }
-    get recordMap() {
-        return this.mutableRecordMap;
-    }
-    get resolvedModules() {
-        return this.mutableResolvedModules;
+        return {
+            modules: modules,
+            errors: errors,
+        };
     }
+    // END PROPERTIES
     findRecordByNumber(recordNumber) {
-        const recordKey = this.numberToRecord.get(recordNumber);
-        if (recordKey === undefined) {
-            return undefined;
-        }
-        return this.recordMap.get(recordKey);
+        const { numberToRecords, recordMap } = this.registry;
+        const recordKeys = numberToRecords.get(recordNumber);
+        return recordKeys?.length === 1 ? recordMap.get(recordKeys[0]) : undefined;
     }
     findMethodByNumber(methodNumber) {
-        return this.numberToMethod.get(methodNumber);
+        const { numberToMethods } = this.registry;
+        const methods = numberToMethods.get(methodNumber);
+        return methods?.length === 1 ? methods[0] : undefined;
+    }
+    get recordMap() {
+        return this.registry.recordMap;
     }
     get errors() {
-        return this.mutableErrors;
+        return this.finalizationResult.errors;
+    }
+    get modules() {
+        return this.finalizationResult.modules;
     }
 }
 /**
@@ -984,9 +1031,9 @@ function validateKeyedItems(items, fieldPath, errors) {
     }
 }
 class TypeResolver {
-    constructor(module, modules, usedImports, errors) {
+    constructor(module, moduleBundles, usedImports, errors) {
         this.module = module;
-        this.modules = modules;
+        this.moduleBundles = moduleBundles;
         this.usedImports = usedImports;
         this.errors = errors;
     }
@@ -1023,7 +1070,7 @@ class TypeResolver {
         // reference, or the module if the record reference is absolute (starts with
         // a dot).
         let start;
-        const { errors, module, modules, usedImports } = this;
+        const { errors, module, moduleBundles: modules, usedImports } = this;
         if (recordOrigin !== "top-level") {
             if (!recordRef.absolute) {
                 // Traverse the chain of ancestors from most nested to top-level.
@@ -1079,12 +1126,12 @@ class TypeResolver {
                     return undefined;
                 }
                 const newModuleResult = modules.get(newModulePath);
-                if (newModuleResult === undefined || newModuleResult.result === null) {
+                if (!newModuleResult) {
                     // The module was not found or has errors: an error was already
                     // registered, no need to register a new one.
                     return undefined;
                 }
-                const newModule = newModuleResult.result;
+                const newModule = newModuleResult.module.result;
                 if (newIt.kind === "import") {
                     newIt = newModule.nameToDeclaration[name];
                     if (!newIt) {
@@ -1123,6 +1170,111 @@ class TypeResolver {
         };
     }
 }
+class Cache {
+    constructor(modulePathToNewContent, modulePathToOldBundle) {
+        this.modulePathToOldBundle = modulePathToOldBundle;
+        this.modulePathToCacheResult = new Map();
+        const unchangedModulePaths = new Set();
+        for (const [modulePath, newContent] of modulePathToNewContent) {
+            const oldBundle = modulePathToOldBundle.get(modulePath);
+            if (oldBundle?.tokens.result.sourceCode === newContent) {
+                unchangedModulePaths.add(modulePath);
+            }
+        }
+        const classify = (modulePath) => {
+            const oldBundle = modulePathToOldBundle.get(modulePath);
+            if (!oldBundle) {
+                return { kind: "no-cache" };
+            }
+            {
+                const inMap = this.modulePathToCacheResult.get(modulePath);
+                if (inMap) {
+                    return inMap;
+                }
+            }
+            let result;
+            const newContent = modulePathToNewContent.get(modulePath);
+            if (newContent === oldBundle.tokens.result.sourceCode) {
+                // Assume best case, may downgrade later.
+                this.modulePathToCacheResult.set(modulePath, {
+                    kind: "module-bundle",
+                    bundle: oldBundle,
+                });
+                const directDependencies = Object.keys(oldBundle.module.result?.pathToImportedNames);
+                result = directDependencies.every((dep) => classify(dep).kind === "module-bundle")
+                    ? { kind: "module-bundle", bundle: oldBundle }
+                    : { kind: "module-tokens", tokens: oldBundle.tokens };
+            }
+            else {
+                result = { kind: "no-cache" };
+            }
+            this.modulePathToCacheResult.set(modulePath, result);
+            return result;
+        };
+        for (const modulePath of modulePathToOldBundle.keys()) {
+            classify(modulePath);
+        }
+    }
+    getModuleCacheResult(modulePath) {
+        return this.modulePathToCacheResult.get(modulePath) ?? { kind: "no-cache" };
+    }
+}
+/** Registry of declarations possibly across multiple modules. */
+class DeclarationRegistry {
+    constructor() {
+        this.recordMap = new Map();
+        this.numberToRecords = new Map();
+        this.numberToMethods = new Map();
+    }
+    mergeFrom(other) {
+        for (const [key, value] of other.recordMap) {
+            this.recordMap.set(key, value);
+        }
+        for (const [number, value] of other.numberToRecords) {
+            let existing = this.numberToRecords.get(number);
+            if (!existing) {
+                existing = [];
+                this.numberToRecords.set(number, existing);
+            }
+            existing.push(...value);
+        }
+        for (const [number, value] of other.numberToMethods) {
+            let existing = this.numberToMethods.get(number);
+            if (!existing) {
+                existing = [];
+                this.numberToMethods.set(number, existing);
+            }
+            existing.push(...value);
+        }
+    }
+    pushNumberRecord(number, record) {
+        let value = this.numberToRecords.get(number);
+        if (!value) {
+            value = [];
+            this.numberToRecords.set(number, value);
+        }
+        value.push(record);
+    }
+    pushNumberMethod(number, method) {
+        let value = this.numberToMethods.get(number);
+        if (!value) {
+            value = [];
+            this.numberToMethods.set(number, value);
+        }
+        value.push(method);
+    }
+}
+class ModuleBundle {
+    constructor(tokens, module) {
+        this.tokens = tokens;
+        this.module = module;
+        /**
+         * Registry of declarations found in this module only.
+         * Will be merged into the "global" registry.
+         */
+        this.registry = new DeclarationRegistry();
+    }
+}
 function ensureAllImportsAreUsed(module, usedImports, errors) {
     for (const declaration of module.declarations) {
         if (declaration.kind === "import") {
@@ -1145,43 +1297,82 @@ function ensureAllImportsAreUsed(module, usedImports, errors) {
         }
     }
 }
-class ModuleParserBase {
-    parseModule(modulePath) {
-        const code = this.readSourceCode(modulePath);
-        if (code === undefined) {
-            return {
-                result: null,
-                errors: [],
-            };
-        }
-        const tokens = tokenizeModule(code, modulePath);
-        if (tokens.errors.length !== 0) {
-            return {
-                result: null,
-                errors: tokens.errors,
-            };
-        }
-        return parseModule(tokens.result, "strict");
+/**
+ * Returns suggested module paths for import auto-completion.
+ *
+ * Given the partial path the user has typed, returns suggestions from
+ * `modulePathToContent`. For paths where there is more after the matched
+ * segment, the suggestion is truncated at the next "/" and a trailing "/" is
+ * appended to signal that it is a directory, not a file.
+ *
+ * Handles both absolute paths (e.g. "bb/ee") and relative paths (e.g.
+ * "./other", "../sibling").
+ */
+function suggestModulePaths(typedPath, originModulePath, modulePathToContent) {
+    const isRelative = typedPath.startsWith("./") || typedPath.startsWith("../");
+    // Compute the absolute path prefix to match against all module paths.
+    let absolutePrefix;
+    if (isRelative) {
+        // Split at the last "/" so that the directory component (fully typed) can
+        // be resolved cleanly, while the tail (partial filename/dir prefix) is
+        // appended afterwards.
+        const lastSlash = typedPath.lastIndexOf("/");
+        const dirComponent = typedPath.slice(0, lastSlash + 1);
+        const filePrefix = typedPath.slice(lastSlash + 1);
+        // Append a dummy filename so Paths.join/normalize treat the directory
+        // component as a file path (avoids trailing-slash edge-cases), then strip
+        // it off again.
+        const dummy = dirComponent + "__dummy__";
+        const resolvedDummy = Paths.join(Paths.dirname(originModulePath), dummy.startsWith("./") ? dummy.slice(2) : dummy).replace(/\\/g, "/");
+        const absoluteDir = resolvedDummy.slice(0, resolvedDummy.length - "__dummy__".length);
+        absolutePrefix = absoluteDir + filePrefix;
     }
-}
-class DefaultModuleParser extends ModuleParserBase {
-    constructor(fileReader, rootPath) {
-        super();
-        this.fileReader = fileReader;
-        this.rootPath = rootPath;
+    else if (originModulePath.startsWith("@") && !typedPath.startsWith("@")) {
+        // Mirror the package-prefix logic from resolveModulePath.
+        absolutePrefix = extractPackagePrefix(originModulePath) + typedPath;
     }
-    readSourceCode(modulePath) {
-        return this.fileReader.readTextFile(Paths.join(this.rootPath, modulePath));
+    else {
+        absolutePrefix = typedPath;
     }
-}
-class MapBasedModuleParser extends ModuleParserBase {
-    constructor(moduleMap) {
-        super();
-        this.moduleMap = moduleMap;
+    absolutePrefix = absolutePrefix.replace(/\\/g, "/");
+    if (absolutePrefix.startsWith("../")) {
+        // Typed path escapes the root; no valid suggestions.
+        return [];
     }
-    readSourceCode(modulePath) {
-        return this.moduleMap.get(modulePath);
+    const originBaseDir = Paths.dirname(originModulePath).replace(/\\/g, "/");
+    const suggestions = new Set();
+    for (const path of modulePathToContent.keys()) {
+        if (!path.startsWith(absolutePrefix))
+            continue;
+        const remaining = path.slice(absolutePrefix.length);
+        const slashIndex = remaining.indexOf("/");
+        // If there is a sub-path after the matched prefix, collapse to the next
+        // directory segment and append "/". Otherwise use the full path.
+        const absoluteSuggestion = slashIndex >= 0
+            ? absolutePrefix + remaining.slice(0, slashIndex + 1)
+            : path;
+        if (isRelative) {
+            const isDir = absoluteSuggestion.endsWith("/");
+            const absPath = isDir
+                ? absoluteSuggestion.slice(0, -1)
+                : absoluteSuggestion;
+            let rel = Paths.relative(originBaseDir, absPath).replace(/\\/g, "/");
+            // Paths.relative returns "" when both paths are the same; normalise to
+            // "." so the subsequent prefix-check and directory-slash logic work
+            // correctly (avoids producing the invalid path ".//" for same-dir cases).
+            if (rel === "") {
+                rel = ".";
+            }
+            if (!rel.startsWith(".")) {
+                rel = "./" + rel;
+            }
+            suggestions.add(isDir ? rel + "/" : rel);
+        }
+        else {
+            suggestions.add(absoluteSuggestion);
+        }
     }
+    return [...suggestions].map((name) => ({ name }));
 }
 function resolveModulePath(pathToken, originModulePath, errors) {
     let modulePath = unquoteAndUnescape(pathToken.text);