ventojs 2.0.0-canary.0 → 2.0.0-canary.2

package/CHANGELOG.md

@@ -5,20 +5,46 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## 2.0.0 - Unreleased
-Vento 2.0 is now dependency-free and compatible with browsers without requiring a build process.
+Vento 2.0 is now dependency-free and compatible with browsers without a build step.
 
 ### Added
 - Build-less browser support.
 - `plugins/mod.ts` module to register all default plugins easily.
+- Support for precompiled templates.
+- New filesystem loader to use File System API.
+- Better errors reporting [#131], [#137]
+- `core/errors.ts` module to format errors.
 
 ### Changed
 - Renamed `src` directory to `core`.
 - Moved all loaders to the `loaders` root directory.
-- Implemented a different approach to resolve the variables without using `meriyah` to analyze the code. [#128]
+- Implemented a different approach to resolve the variables without using `meriyah` to analyze the code [#128].
+- The signature of `tag` plugins has changed:
+  ```diff
+  -- (env: Environment, code: string, output: string, tokens: Tokens[])
+  ++ (env: Environment, token: Token, output: string, tokens: Tokens[])
+  ```
+- The `compileTokens` function has changed. The third argument is a string with the closing tag and now it throws an error if its not found:
+  ```diff
+  -- env.compileTokens(tokens, tmpOutput, ["/code"]);
+  -- if (tokens.length && (tokens[0][0] !== "tag" || tokens[0][1] !== "/code")) {
+  --   throw new Error("missing closing tag");
+  -- }
+  ++ env.compileTokens(tokens, tmpOutput, "/code");
+  ```
 
 ### Removed
+- `runStringSync` function.
 - Deprecated option `useWith`.
 - All extenal dependencies (`meriyah`, `estree`, etc).
 - `bare.ts` file since now it's useless.
 
+### Fixed
+- Functions output when the autoescape is enabled [#95]
+- Improved escape filter performance [#134]
+
+[#95]: https://github.com/ventojs/vento/issues/95
 [#128]: https://github.com/ventojs/vento/issues/128
+[#131]: https://github.com/ventojs/vento/issues/131
+[#134]: https://github.com/ventojs/vento/issues/134
+[#137]: https://github.com/ventojs/vento/issues/137

package/core/environment.js

@@ -1,6 +1,6 @@
 import iterateTopLevel from "./js.js";
 import tokenize from "./tokenizer.js";
-import { TemplateError } from "./errors.js";
+import { createError, SourceError } from "./errors.js";
 export class Environment {
     cache = new Map();
     options;
@@ -9,6 +9,10 @@ export class Environment {
     filters = {};
     utils = {
         callMethod,
+        createError,
+        safeString(str) {
+            return new SafeString(str);
+        },
     };
     constructor(options) {
         this.options = options;
@@ -16,8 +20,8 @@ export class Environment {
     use(plugin) {
         plugin(this);
     }
-    async run(file, data, from) {
-        const template = await this.load(file, from);
+    async run(file, data, from, position) {
+        const template = await this.load(file, from, position);
         return await template(data);
     }
     async runString(source, data, file) {
@@ -33,16 +37,26 @@ export class Environment {
         const template = this.compile(source, file);
         return await template(data);
     }
-    runStringSync(source, data) {
-        const template = this.compile(source, "", {}, true);
-        return template(data);
-    }
-    compile(source, path, defaults, sync = false) {
+    compile(source, path, defaults) {
         if (typeof source !== "string") {
-            throw new Error(`The source code of "${path}" must be a string. Got ${typeof source}`);
+            throw new TypeError(`The source code of "${path}" must be a string. Got ${typeof source}`);
         }
         const tokens = this.tokenize(source, path);
-        let code = this.compileTokens(tokens).join("\n");
+        const lastToken = tokens.at(-1);
+        if (lastToken[0] != "string") {
+            throw new SourceError("Unclosed tag", lastToken[2], path, source);
+        }
+        let code = "";
+        try {
+            code = this.compileTokens(tokens).join("\n");
+        }
+        catch (error) {
+            if (error instanceof SourceError) {
+                error.file ??= path;
+                error.source ??= source;
+            }
+            throw error;
+        }
         const { dataVarname, autoDataVarname } = this.options;
         if (autoDataVarname) {
             const generator = iterateTopLevel(code);
@@ -57,24 +71,35 @@ export class Environment {
         `;
             }
         }
-        const constructor = new Function("__file", "__env", "__defaults", "__err", `return${sync ? "" : " async"} function (${dataVarname}) {
-        let __pos = 0;
         try {
-          ${dataVarname} = Object.assign({}, __defaults, ${dataVarname});
-          const __exports = { content: "" };
-          ${code}
-          return __exports;
-        } catch (cause) {
-          const template = ${sync ? "" : "await"} __env.cache.get(__file);
-          throw new __err(__file, template?.source, __pos, cause);
-        }
-      }
-      `);
-        const template = constructor(path, this, defaults, TemplateError);
-        template.file = path;
-        template.code = code;
-        template.source = source;
-        return template;
+            const constructor = new Function("__env", `return async function __template(${dataVarname}) {
+          let __pos=0;
+          try {
+            ${dataVarname} = Object.assign({}, __template.defaults, ${dataVarname});
+            const __exports = { content: "" };
+            ${code}
+            return __exports;
+          } catch (error) {
+            throw __env.utils.createError(error, __template, __pos);
+          }
+        }`);
+            const template = constructor(this);
+            template.path = path;
+            template.code = constructor.toString();
+            template.source = source;
+            template.defaults = defaults || {};
+            return template;
+        }
+        catch (error) {
+            if (error instanceof SyntaxError) {
+                throw createError(error, { source, code, path });
+            }
+            if (error instanceof SourceError) {
+                error.file ??= path;
+                error.source ??= source;
+            }
+            throw error;
+        }
     }
     tokenize(source, path) {
         let tokens = tokenize(source);
@@ -86,7 +111,15 @@ export class Environment {
         }
         return tokens;
     }
-    async load(file, from) {
+    async load(file, from, position) {
+        if (!file) {
+            throw position !== undefined
+                ? new SourceError(`Invalid template (${typeof file})`, position, from)
+                : new Error(`Invalid template (${typeof file})`);
+        }
+        if (typeof file !== "string") {
+            throw new TypeError(`The template filename must be a string. Got ${typeof file}`);
+        }
         const path = this.options.loader.resolve(from || "", file);
         let cached = this.cache.get(path);
         if (cached) {
@@ -97,17 +130,32 @@ export class Environment {
             .split("?")[0]
             .split("#")[0];
         cached = this.options.loader.load(cleanPath)
-            .then(({ source, data }) => this.compile(source, path, data));
+            .catch((error) => {
+            throw position !== undefined
+                ? new SourceError(`Error loading template: ${error.message}`, position, from)
+                : error;
+        })
+            .then((result) => {
+            if (typeof result === "function") {
+                return result(this);
+            }
+            const { source, data } = result;
+            return this.compile(source, path, data);
+        });
         this.cache.set(path, cached);
         return await cached;
     }
-    compileTokens(tokens, outputVar = "__exports.content", stopAt) {
+    compileTokens(tokens, outputVar = "__exports.content", closeToken) {
         const compiled = [];
+        let openToken;
         tokens: while (tokens.length > 0) {
-            if (stopAt && tokens[0][0] === "tag" && stopAt.includes(tokens[0][1])) {
-                break;
+            const token = tokens.shift();
+            const [type, code, position] = token;
+            openToken ??= token;
+            // We found the closing tag, so we stop compiling
+            if (closeToken && type === "tag" && closeToken === code) {
+                return compiled;
             }
-            const [type, code, pos] = tokens.shift();
             if (type === "comment") {
                 continue;
             }
@@ -118,9 +166,9 @@ export class Environment {
                 continue;
             }
             if (type === "tag") {
-                compiled.push(`__pos = ${pos};`);
+                compiled.push(`__pos=${position};`);
                 for (const tag of this.tags) {
-                    const compiledTag = tag(this, code, outputVar, tokens);
+                    const compiledTag = tag(this, token, outputVar, tokens);
                     if (typeof compiledTag === "string") {
                         compiled.push(compiledTag);
                         continue tokens;
@@ -131,17 +179,22 @@ export class Environment {
                 compiled.push(`${outputVar} += (${expression}) ?? "";`);
                 continue;
             }
-            throw new Error(`Unknown token type "${type}"`);
+            throw new SourceError(`Unknown token type "${type}"`, position);
+        }
+        // If we reach here, it means we have an open token that wasn't closed
+        if (closeToken) {
+            throw new SourceError(`Missing closing tag ("${closeToken}" tag is expected)`, openToken[2]);
         }
         return compiled;
     }
     compileFilters(tokens, output, autoescape = false) {
         let unescaped = false;
         while (tokens.length > 0 && tokens[0][0] === "filter") {
-            const [, code] = tokens.shift();
+            const token = tokens.shift();
+            const [, code, position] = token;
             const match = code.match(/^(await\s+)?([\w.]+)(?:\((.*)\))?$/);
             if (!match) {
-                throw new Error(`Invalid filter: ${code}`);
+                throw new SourceError(`Invalid filter: ${code}`, position);
             }
             const [_, isAsync, name, args] = match;
             if (!Object.hasOwn(this.filters, name)) {
@@ -154,7 +207,7 @@ export class Environment {
                 }
                 else {
                     // It's a prototype's method (e.g. `String.toUpperCase()`)
-                    output = `${isAsync ? "await " : ""}__env.utils.callMethod(${output}, "${name}", ${args ? args : ""})`;
+                    output = `${isAsync ? "await " : ""}__env.utils.callMethod(${position}, ${output}, "${name}", ${args ? args : ""})`;
                 }
             }
             else {
@@ -182,16 +235,19 @@ function isGlobal(name) {
         return Object.hasOwn(globalThis[obj], prop);
     }
 }
+function callMethod(position, 
 // deno-lint-ignore no-explicit-any
-function callMethod(thisObject, method, ...args) {
+thisObject, method, ...args) {
     if (thisObject === null || thisObject === undefined) {
         return thisObject;
     }
     if (typeof thisObject[method] === "function") {
         return thisObject[method](...args);
     }
-    throw new Error(`"${method}" is not a valid filter, global object or a method of a ${typeof thisObject} variable`);
+    throw new SourceError(`Method "${method}" is not a function of ${typeof thisObject} variable`, position);
 }
 function checkAsync(fn) {
     return fn.constructor?.name === "AsyncFunction";
 }
+export class SafeString extends String {
+}

package/core/errors.js

@@ -1,34 +1,263 @@
-class VentoBaseError extends Error {
-    name = this.constructor.name;
+export class VentoError extends Error {
 }
-export class TemplateError extends VentoBaseError {
-    path;
-    source;
+export class SourceError extends VentoError {
     position;
-    constructor(path = "<unknown>", source = "<empty file>", position = 0, cause) {
-        const { line, column, code } = errorLine(source, position);
-        super(`Error in template ${path}:${line}:${column}\n\n${code.trim()}\n\n`, { cause });
-        this.path = path;
-        this.source = source;
+    file;
+    source;
+    constructor(message, position, file, source) {
+        super(message);
+        this.name = "SourceError";
         this.position = position;
+        this.file = file;
+        this.source = source;
+    }
+    getContext() {
+        return {
+            type: this.name,
+            message: this.message,
+            position: this.position,
+            file: this.file,
+            source: this.source,
+        };
     }
 }
-/** Returns the number and code of the errored line */
-export function errorLine(source, position) {
-    let line = 1;
-    let column = 1;
-    for (let index = 0; index < position; index++) {
-        if (source[index] === "\n" ||
-            (source[index] === "\r" && source[index + 1] === "\n")) {
-            line++;
-            column = 1;
-            if (source[index] === "\r") {
-                index++;
+export class RuntimeError extends VentoError {
+    #context;
+    position;
+    constructor(error, context, position) {
+        super(error.message);
+        this.name = error.name || "JavaScriptError";
+        this.#context = context;
+        this.cause = error;
+        this.position = position;
+    }
+    async getContext() {
+        const { code, source, path } = this.#context;
+        // If we don't have the position, we cannot provide a context
+        // Try to get the context from a SyntaxError
+        if (this.position === undefined) {
+            try {
+                return (await getSyntaxErrorContext(this.cause, this.#context)) ??
+                    {
+                        type: this.name || "JavaScriptError",
+                        message: this.message,
+                        source,
+                        code,
+                        file: path,
+                    };
             }
+            catch {
+                return {
+                    type: this.name || "JavaScriptError",
+                    message: this.message,
+                    source,
+                    code,
+                    file: path,
+                };
+            }
+        }
+        // Capture the exact position of the error in the compiled code
+        for (const frame of getStackFrames(this.cause)) {
+            if (frame.file !== "<anonymous>") {
+                continue;
+            }
+            return {
+                type: this.name || "JavaScriptError",
+                message: this.message,
+                source,
+                position: this.position,
+                code,
+                line: frame.line,
+                column: frame.column,
+                file: path,
+            };
+        }
+        // As a fallback, return the error with the available context
+        return {
+            type: this.name || "JavaScriptError",
+            message: this.message,
+            source,
+            position: this.position,
+            code,
+            file: path,
+        };
+    }
+}
+/** Create or complete VentoError with extra info from the template */
+export function createError(error, context, position) {
+    if (error instanceof RuntimeError)
+        return error;
+    // If the error is a SourceError, we can fill the missing context information
+    if (error instanceof SourceError) {
+        error.file ??= context.path;
+        error.source ??= context.source;
+        error.position ??= position;
+        return error;
+    }
+    // JavaScript syntax errors can be parsed to get accurate position
+    return new RuntimeError(error, context, position);
+}
+const colors = {
+    number: (n) => `\x1b[33m${n}\x1b[39m`,
+    dim: (line) => `\x1b[2m${line}\x1b[22m`,
+    error: (msg) => `\x1b[31m${msg}\x1b[39m`,
+};
+const plain = {
+    number: (n) => n,
+    dim: (line) => line,
+    error: (msg) => msg,
+};
+const formats = {
+    colors,
+    plain,
+};
+/** Prints an error to the console in a formatted way. */
+export async function printError(error, format = plain) {
+    if (error instanceof VentoError) {
+        const context = await error.getContext();
+        const fmt = typeof format === "string" ? formats[format] || plain : format;
+        if (context) {
+            console.error(stringifyError(context, fmt));
+            return;
+        }
+    }
+    console.error(error);
+}
+/** Converts an error context into a formatted string representation. */
+export function stringifyError(context, format = plain) {
+    const { type, message, source, position, code, line, column, file } = context;
+    const output = [];
+    // Print error type and message
+    output.push(`${format.error(type)}: ${message}`);
+    // If we don't know the position, we cannot print the source code
+    if (position === undefined || source === undefined) {
+        if (file) {
+            output.push(format.dim(file));
         }
-        else {
-            column++;
+        return output.join("\n");
+    }
+    const sourceLines = codeToLines(source);
+    const [sourceLine, sourceColumn] = getSourceLineColumn(sourceLines, position);
+    // Print file location if available
+    if (file) {
+        output.push(format.dim(`${file}:${sourceLine}:${sourceColumn}`));
+    }
+    const pad = sourceLine.toString().length;
+    // Print the latest lines of the source code before the error
+    for (let line = Math.max(sourceLine - 3, 1); line <= sourceLine; line++) {
+        const sidebar = ` ${format.number(`${line}`.padStart(pad))} ${format.dim("|")} `;
+        output.push(sidebar + sourceLines[line - 1].trimEnd());
+    }
+    // If we don't have the compiled code, return the tag position
+    const indent = ` ${" ".repeat(pad)} ${format.dim("|")}`;
+    // If we don't have the compiled code, print the tag position
+    if (!code || line === undefined || column === undefined) {
+        output.push(`${indent} ${" ".repeat(sourceColumn - 1)}${format.error(`^ ${message}`)}`);
+        return output.join("\n");
+    }
+    // Print the compiled code with the error position
+    const codeLines = codeToLines(code);
+    output.push(`${indent} ${" ".repeat(sourceColumn - 1)}${format.error("^")}`);
+    output.push(`${indent} ${format.dim(codeLines[line - 1]?.trimEnd() || "")}`);
+    output.push(`${indent} ${" ".repeat(column)} ${format.error(`^ ${message}`)}`);
+    return output.join("\n");
+}
+/**
+ * Extracts the context from a SyntaxError
+ * It does not work on Node.js and Bun due to the lack of position information
+ * in the stack trace of a dynamic import error.
+ */
+async function getSyntaxErrorContext(error, context) {
+    const { source, code } = context;
+    const url = URL.createObjectURL(new Blob([code], { type: "application/javascript" }));
+    const err = await import(url).catch((e) => e);
+    URL.revokeObjectURL(url);
+    for (const frame of getStackFrames(err)) {
+        if (!frame.file.startsWith("blob:")) {
+            continue;
         }
+        return {
+            type: "SyntaxError",
+            message: error.message,
+            source,
+            position: searchPosition(frame, code) ?? 0,
+            code,
+            line: frame.line,
+            column: frame.column,
+            file: context.path,
+        };
     }
-    return { line, column, code: source.split("\n")[line - 1] };
+}
+const LINE_TERMINATOR = /(\r\n?|[\n\u2028\u2029])/;
+/** Convert the source code into an array of lines */
+function codeToLines(code) {
+    const doubleLines = code.split(LINE_TERMINATOR);
+    const lines = [];
+    for (let i = 0; i < doubleLines.length; i += 2) {
+        lines.push(`${doubleLines[i]}${doubleLines[i + 1] ?? ""}`);
+    }
+    return lines;
+}
+const POSITION_VARIABLE = /^__pos=(\d+);$/;
+/** Search the closest token to an error */
+function searchPosition(frame, code) {
+    const posLine = codeToLines(code)
+        .slice(0, frame.line - 1)
+        .findLast((line) => POSITION_VARIABLE.test(line.trim()));
+    if (posLine) {
+        return Number(posLine.trim().slice(6, -1));
+    }
+}
+/** Get the line and column number of a position in the code */
+function getSourceLineColumn(lines, position) {
+    if (position < 0) {
+        return [1, 1]; // Position is before the start of the source
+    }
+    let index = 0;
+    for (const [line, content] of lines.entries()) {
+        const length = content.length;
+        if (position < index + length) {
+            return [line + 1, position - index + 1];
+        }
+        index += content.length;
+    }
+    throw new Error(`Position ${position} is out of bounds for the provided source lines.`);
+}
+/** Returns every combination of file, line and column of an error stack */
+// deno-lint-ignore no-explicit-any
+function* getStackFrames(error) {
+    // Firefox specific
+    const { columnNumber, lineNumber, fileName } = error;
+    if (columnNumber !== undefined && lineNumber !== undefined && fileName) {
+        yield {
+            file: normalizeFile(fileName),
+            line: lineNumber,
+            column: columnNumber,
+        };
+    }
+    const { stack } = error;
+    if (!stack) {
+        return;
+    }
+    const matches = stack.matchAll(/([^(\s,]+):(\d+):(\d+)/g);
+    for (const match of matches) {
+        const [_, file, line, column] = match;
+        // Skip Node, Bun & Deno internal stack frames
+        if (file.startsWith("node:") || file.startsWith("ext:") || file === "native") {
+            continue;
+        }
+        yield {
+            file: normalizeFile(file),
+            line: Number(line),
+            column: Number(column),
+        };
+    }
+}
+function normalizeFile(file) {
+    if (!file)
+        return "<anonymous>";
+    // Firefox may return "Function" for anonymous functions
+    if (file === "Function")
+        return "<anonymous>";
+    return file;
 }

package/core/reserved.js

@@ -2,6 +2,7 @@ const variables = new Set([
     // Words reserved by Vento, used internally. In general, don't use variable
     // names starting with two underscores to be future-proof and avoid clashes.
     "__file",
+    "__template",
     "__env",
     "__defaults",
     "__err",

package/core/tokenizer.js

@@ -8,12 +8,12 @@ export default function tokenize(source) {
             const index = source.indexOf("{{");
             const code = index === -1 ? source : source.slice(0, index);
             tokens.push([type, code, position]);
-            if (index === -1) {
-                break;
-            }
             position += index;
             source = source.slice(index);
             type = source.startsWith("{{#") ? "comment" : "tag";
+            if (index === -1) {
+                break;
+            }
             continue;
         }
         if (type === "comment") {
@@ -42,9 +42,11 @@ export default function tokenize(source) {
                     return curr;
                 }
                 // Filters
-                tokens.push(["filter", code]);
+                tokens.push(["filter", code, position + prev]);
                 return curr;
             });
+            if (indexes[lastIndex] == Infinity)
+                return tokens;
             position += indexes[lastIndex];
             source = source.slice(indexes[lastIndex]);
             type = "string";
@@ -64,6 +66,9 @@ export default function tokenize(source) {
             continue;
         }
     }
+    if (type == "string") {
+        tokens.push([type, "", position]);
+    }
     return tokens;
 }
 /**
@@ -82,5 +87,6 @@ export function parseTag(source) {
         indexes.push(index + 2);
         return indexes;
     }
-    throw new Error("Unclosed tag");
+    indexes.push(Infinity);
+    return indexes;
 }

package/loaders/file.js

@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
 import process from "node:process";
 /**
  * Vento file loader for loading templates from the file system.
- * Used by JS runtimes like Node, Deno, and Bun.
+ * Used by Node-like runtimes (Node, Deno, Bun, ...)
  */
 export class FileLoader {
     #root;

package/loaders/filesystem.js

@@ -0,0 +1,36 @@
+import { join } from "./utils.js";
+/**
+ * Vento FileSystem API loader for loading templates.
+ * Used by browser environments.
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/File_System_API
+ */
+export class FileSystemLoader {
+    #handle;
+    constructor(handle) {
+        this.#handle = handle;
+    }
+    async load(file) {
+        const parts = file.split("/");
+        let currentHandle = this.#handle;
+        while (parts.length > 1) {
+            const part = parts.shift();
+            if (part) {
+                currentHandle = await currentHandle.getDirectoryHandle(part, {
+                    create: false,
+                });
+            }
+        }
+        const entry = await currentHandle.getFileHandle(parts[0], {
+            create: false,
+        });
+        const fileHandle = await entry.getFile();
+        const source = await fileHandle.text();
+        return { source };
+    }
+    resolve(from, file) {
+        if (file.startsWith(".")) {
+            return join(from, "..", file);
+        }
+        return join(file);
+    }
+}