ventojs 2.0.0-canary.0 → 2.0.0-canary.1

package/CHANGELOG.md

@@ -5,20 +5,45 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## 2.0.0 - Unreleased
-Vento 2.0 is now dependency-free and compatible with browsers without requiring a build process.
+Vento 2.0 is now dependency-free and compatible with browsers without a build step.
 
 ### Added
 - Build-less browser support.
 - `plugins/mod.ts` module to register all default plugins easily.
+- Support for precompiled templates.
+- New filesystem loader to use File System API.
+- Better errors reporting [#131], [#137]
 
 ### Changed
 - Renamed `src` directory to `core`.
 - Moved all loaders to the `loaders` root directory.
-- Implemented a different approach to resolve the variables without using `meriyah` to analyze the code. [#128]
+- Implemented a different approach to resolve the variables without using `meriyah` to analyze the code [#128].
+- The signature of `tag` plugins has changed:
+  ```diff
+  -- (env: Environment, code: string, output: string, tokens: Tokens[])
+  ++ (env: Environment, token: Token, output: string, tokens: Tokens[])
+  ```
+- The `compileTokens` function has changed. The third argument is a string with the closing tag and now it throws an error if its not found:
+  ```diff
+  -- env.compileTokens(tokens, tmpOutput, ["/code"]);
+  -- if (tokens.length && (tokens[0][0] !== "tag" || tokens[0][1] !== "/code")) {
+  --   throw new Error("missing closing tag");
+  -- }
+  ++ env.compileTokens(tokens, tmpOutput, "/code");
+  ```
 
 ### Removed
+- `runStringSync` function.
 - Deprecated option `useWith`.
 - All extenal dependencies (`meriyah`, `estree`, etc).
 - `bare.ts` file since now it's useless.
 
+### Fixed
+- Functions output when the autoescape is enabled [#95]
+- Improved escape filter performance [#134]
+
+[#95]: https://github.com/ventojs/vento/issues/95
 [#128]: https://github.com/ventojs/vento/issues/128
+[#131]: https://github.com/ventojs/vento/issues/131
+[#134]: https://github.com/ventojs/vento/issues/134
+[#137]: https://github.com/ventojs/vento/issues/137

package/core/environment.js

@@ -1,6 +1,6 @@
 import iterateTopLevel from "./js.js";
 import tokenize from "./tokenizer.js";
-import { TemplateError } from "./errors.js";
+import { createError, TokenError } from "./errors.js";
 export class Environment {
     cache = new Map();
     options;
@@ -9,6 +9,10 @@ export class Environment {
     filters = {};
     utils = {
         callMethod,
+        createError,
+        safeString(str) {
+            return new SafeString(str);
+        },
     };
     constructor(options) {
         this.options = options;
@@ -33,16 +37,30 @@ export class Environment {
         const template = this.compile(source, file);
         return await template(data);
     }
-    runStringSync(source, data) {
-        const template = this.compile(source, "", {}, true);
-        return template(data);
-    }
-    compile(source, path, defaults, sync = false) {
+    compile(source, path, defaults) {
         if (typeof source !== "string") {
-            throw new Error(`The source code of "${path}" must be a string. Got ${typeof source}`);
+            throw new TypeError(`The source code of "${path}" must be a string. Got ${typeof source}`);
+        }
+        const allTokens = this.tokenize(source, path);
+        const tokens = [...allTokens];
+        const lastToken = tokens.at(-1);
+        if (lastToken[0] != "string") {
+            throw new TokenError("Unclosed tag", lastToken, source, path);
+        }
+        let code = "";
+        try {
+            code = this.compileTokens(tokens).join("\n");
+        }
+        catch (error) {
+            if (!(error instanceof Error))
+                throw error;
+            throw createError(error, {
+                source,
+                code,
+                tokens: allTokens,
+                path,
+            });
         }
-        const tokens = this.tokenize(source, path);
-        let code = this.compileTokens(tokens).join("\n");
         const { dataVarname, autoDataVarname } = this.options;
         if (autoDataVarname) {
             const generator = iterateTopLevel(code);
@@ -57,24 +75,35 @@ export class Environment {
         `;
             }
         }
-        const constructor = new Function("__file", "__env", "__defaults", "__err", `return${sync ? "" : " async"} function (${dataVarname}) {
-        let __pos = 0;
         try {
-          ${dataVarname} = Object.assign({}, __defaults, ${dataVarname});
-          const __exports = { content: "" };
-          ${code}
-          return __exports;
-        } catch (cause) {
-          const template = ${sync ? "" : "await"} __env.cache.get(__file);
-          throw new __err(__file, template?.source, __pos, cause);
+            const constructor = new Function("__env", `return async function __template(${dataVarname}) {
+          try {
+            ${dataVarname} = Object.assign({}, __template.defaults, ${dataVarname});
+            const __exports = { content: "" };
+            ${code}
+            return __exports;
+          } catch (error) {
+            throw __env.utils.createError(error, __template);
+          }
+        }`);
+            const template = constructor(this);
+            template.path = path;
+            template.code = constructor.toString();
+            template.source = source;
+            template.tokens = allTokens;
+            template.defaults = defaults || {};
+            return template;
+        }
+        catch (error) {
+            if (!(error instanceof Error))
+                throw error;
+            throw createError(error, {
+                source,
+                code,
+                tokens: allTokens,
+                path,
+            });
         }
-      }
-      `);
-        const template = constructor(path, this, defaults, TemplateError);
-        template.file = path;
-        template.code = code;
-        template.source = source;
-        return template;
     }
     tokenize(source, path) {
         let tokens = tokenize(source);
@@ -97,17 +126,27 @@ export class Environment {
             .split("?")[0]
             .split("#")[0];
         cached = this.options.loader.load(cleanPath)
-            .then(({ source, data }) => this.compile(source, path, data));
+            .then((result) => {
+            if (typeof result === "function") {
+                return result(this);
+            }
+            const { source, data } = result;
+            return this.compile(source, path, data);
+        });
         this.cache.set(path, cached);
         return await cached;
     }
-    compileTokens(tokens, outputVar = "__exports.content", stopAt) {
+    compileTokens(tokens, outputVar = "__exports.content", closeToken) {
         const compiled = [];
+        let openToken;
         tokens: while (tokens.length > 0) {
-            if (stopAt && tokens[0][0] === "tag" && stopAt.includes(tokens[0][1])) {
-                break;
+            const token = tokens.shift();
+            const [type, code, pos] = token;
+            openToken ??= token;
+            // We found the closing tag, so we stop compiling
+            if (closeToken && type === "tag" && closeToken === code) {
+                return compiled;
             }
-            const [type, code, pos] = tokens.shift();
             if (type === "comment") {
                 continue;
             }
@@ -118,9 +157,9 @@ export class Environment {
                 continue;
             }
             if (type === "tag") {
-                compiled.push(`__pos = ${pos};`);
+                compiled.push(`/*__pos:${pos}*/`);
                 for (const tag of this.tags) {
-                    const compiledTag = tag(this, code, outputVar, tokens);
+                    const compiledTag = tag(this, token, outputVar, tokens);
                     if (typeof compiledTag === "string") {
                         compiled.push(compiledTag);
                         continue tokens;
@@ -131,17 +170,22 @@ export class Environment {
                 compiled.push(`${outputVar} += (${expression}) ?? "";`);
                 continue;
             }
-            throw new Error(`Unknown token type "${type}"`);
+            throw new TokenError(`Unknown token type "${type}"`, token);
+        }
+        // If we reach here, it means we have an open token that wasn't closed
+        if (closeToken) {
+            throw new TokenError(`Missing closing tag ("${closeToken}" tag is expected)`, openToken);
         }
         return compiled;
     }
     compileFilters(tokens, output, autoescape = false) {
         let unescaped = false;
         while (tokens.length > 0 && tokens[0][0] === "filter") {
-            const [, code] = tokens.shift();
+            const token = tokens.shift();
+            const [, code, position] = token;
             const match = code.match(/^(await\s+)?([\w.]+)(?:\((.*)\))?$/);
             if (!match) {
-                throw new Error(`Invalid filter: ${code}`);
+                throw new TokenError(`Invalid filter: ${code}`, token);
             }
             const [_, isAsync, name, args] = match;
             if (!Object.hasOwn(this.filters, name)) {
@@ -154,7 +198,7 @@ export class Environment {
                 }
                 else {
                     // It's a prototype's method (e.g. `String.toUpperCase()`)
-                    output = `${isAsync ? "await " : ""}__env.utils.callMethod(${output}, "${name}", ${args ? args : ""})`;
+                    output = `${isAsync ? "await " : ""}__env.utils.callMethod(${position}, ${output}, "${name}", ${args ? args : ""})`;
                 }
             }
             else {
@@ -182,16 +226,19 @@ function isGlobal(name) {
         return Object.hasOwn(globalThis[obj], prop);
     }
 }
+function callMethod(position, 
 // deno-lint-ignore no-explicit-any
-function callMethod(thisObject, method, ...args) {
+thisObject, method, ...args) {
     if (thisObject === null || thisObject === undefined) {
         return thisObject;
     }
     if (typeof thisObject[method] === "function") {
         return thisObject[method](...args);
     }
-    throw new Error(`"${method}" is not a valid filter, global object or a method of a ${typeof thisObject} variable`);
+    throw new TokenError(`Method "${method}" is not a function of ${typeof thisObject} variable`, position);
 }
 function checkAsync(fn) {
     return fn.constructor?.name === "AsyncFunction";
 }
+export class SafeString extends String {
+}

package/core/errors.js

@@ -1,20 +1,168 @@
-class VentoBaseError extends Error {
-    name = this.constructor.name;
+export class VentoError extends Error {
 }
-export class TemplateError extends VentoBaseError {
-    path;
+export class TokenError extends VentoError {
+    token;
     source;
-    position;
-    constructor(path = "<unknown>", source = "<empty file>", position = 0, cause) {
-        const { line, column, code } = errorLine(source, position);
-        super(`Error in template ${path}:${line}:${column}\n\n${code.trim()}\n\n`, { cause });
-        this.path = path;
+    file;
+    constructor(message, token, source, file) {
+        super(message);
+        this.name = "TokenError";
+        this.token = token;
         this.source = source;
-        this.position = position;
+        this.file = file;
     }
+    getContext() {
+        if (!this.source || this.token === undefined) {
+            return;
+        }
+        return {
+            type: this.name,
+            message: this.message,
+            source: this.source,
+            position: typeof this.token === "number" ? this.token : this.token[2],
+            file: this.file,
+        };
+    }
+}
+export class RuntimeError extends VentoError {
+    context;
+    constructor(error, context) {
+        super(error.message);
+        this.name = error.name || "JavaScriptError";
+        this.context = context;
+        this.cause = error;
+    }
+    getContext() {
+        if (this.cause instanceof SyntaxError) {
+            return parseSyntaxError(this.cause, this.context);
+        }
+        if (this.cause instanceof Error) {
+            return parseError(this.cause, this.context);
+        }
+    }
+}
+export function createError(error, context) {
+    if (error instanceof RuntimeError)
+        return error;
+    // If the error is a TokenError, we can enhance it with the context information
+    if (error instanceof TokenError) {
+        error.file ??= context.path;
+        error.source ??= context.source;
+        return error;
+    }
+    // JavaScript syntax errors can be parsed to get accurate position
+    return new RuntimeError(error, context);
+}
+export async function printError(error) {
+    if (error instanceof VentoError) {
+        const context = await error.getContext();
+        if (context) {
+            console.error(stringifyContext(context));
+            return;
+        }
+    }
+    console.error(error);
+}
+function parseError(error, context) {
+    const stackMatch = error.stack?.match(/<anonymous>:(\d+):(\d+)/);
+    if (!stackMatch)
+        return;
+    const row = Number(stackMatch[1]) - 1;
+    const col = Number(stackMatch[2]);
+    const position = getAccurateErrorPosition(row, col, context);
+    if (position == -1)
+        return;
+    return {
+        type: error.name || "JavaScriptError",
+        message: error.message,
+        source: context.source,
+        position,
+        file: context.path,
+    };
+}
+async function parseSyntaxError(error, context) {
+    const code = `()=>{${context.code}}`;
+    const dataUrl = "data:application/javascript;base64," + btoa(code);
+    const stack = await import(dataUrl).catch(({ stack }) => stack);
+    if (!stack)
+        return;
+    const stackMatch = stack?.match(/:(\d+):(\d+)$/m);
+    if (!stackMatch)
+        return;
+    const row = Number(stackMatch[1]) - 1;
+    const col = Number(stackMatch[2]);
+    const position = getAccurateErrorPosition(row, col, context);
+    if (position == -1)
+        return;
+    return {
+        type: "SyntaxError",
+        message: error.message,
+        source: context.source,
+        position,
+        file: context.path,
+    };
+}
+function getAccurateErrorPosition(row, col, context) {
+    const { code, tokens, source } = context;
+    if (!tokens)
+        return -1;
+    const linesAndDelims = code.split(/(\r\n?|[\n\u2028\u2029])/);
+    const linesAndDelimsUntilIssue = linesAndDelims.slice(0, row * 2);
+    const issueIndex = linesAndDelimsUntilIssue.join("").length + col;
+    const posLine = linesAndDelimsUntilIssue.findLast((line) => {
+        return /^\/\*__pos:(\d+)\*\/$/.test(line);
+    });
+    if (!posLine)
+        return -1;
+    const position = Number(posLine.slice(8, -2));
+    const token = tokens.findLast((token) => {
+        if (token[2] == undefined)
+            return false;
+        return token[2] <= position;
+    });
+    if (!token)
+        return -1;
+    const isJS = token[1].startsWith(">");
+    const tag = isJS ? token[1].slice(1).trimStart() : token[1];
+    const issueStartIndex = code.lastIndexOf(tag, issueIndex);
+    if (issueStartIndex == -1)
+        return -1;
+    const sourceIssueStartIndex = source.indexOf(tag, position);
+    return sourceIssueStartIndex + issueIndex - issueStartIndex - 1;
+}
+const terminal = {
+    number: (n) => `\x1b[33m${n}\x1b[39m`,
+    dim: (line) => `\x1b[2m${line}\x1b[22m`,
+    error: (msg) => `\x1b[31m${msg}\x1b[39m`,
+};
+const LINE_TERMINATOR = /\r\n?|[\n\u2028\u2029]/;
+export function stringifyContext(context, format = terminal) {
+    const { type, message, source, position, file } = context;
+    const sourceAfterIssue = source.slice(position);
+    const newlineMatch = sourceAfterIssue.match(LINE_TERMINATOR);
+    const endIndex = position + (newlineMatch?.index ?? sourceAfterIssue.length);
+    const lines = source.slice(0, endIndex).split(LINE_TERMINATOR);
+    const displayedLineEntries = [...lines.entries()].slice(-3);
+    const endLineIndex = lines.at(-1).length + position - endIndex;
+    const numberLength = (displayedLineEntries.at(-1)[0] + 1).toString().length;
+    const displayedCode = displayedLineEntries.map(([index, line]) => {
+        const number = `${index + 1}`.padStart(numberLength);
+        const sidebar = ` ${format.number(number)} ${format.dim("|")} `;
+        return sidebar + line;
+    }).join("\n");
+    const sidebarWidth = numberLength + 4;
+    const tooltipIndex = sidebarWidth + endLineIndex;
+    const tooltipIndent = " ".repeat(tooltipIndex);
+    const tooltip = tooltipIndent + format.error(`^ ${message}`);
+    const output = [];
+    output.push(`${format.error(type)}: ${message}`);
+    if (file) {
+        output.push(format.dim(getLocation(file, source, position)), "");
+    }
+    output.push(displayedCode, tooltip);
+    return output.join("\n");
 }
-/** Returns the number and code of the errored line */
-export function errorLine(source, position) {
+function getLocation(file, source, position) {
     let line = 1;
     let column = 1;
     for (let index = 0; index < position; index++) {
@@ -30,5 +178,5 @@ export function errorLine(source, position) {
             column++;
         }
     }
-    return { line, column, code: source.split("\n")[line - 1] };
+    return `${file}:${line}:${column}`;
 }

package/core/reserved.js

@@ -2,6 +2,7 @@ const variables = new Set([
     // Words reserved by Vento, used internally. In general, don't use variable
     // names starting with two underscores to be future-proof and avoid clashes.
     "__file",
+    "__template",
     "__env",
     "__defaults",
     "__err",

package/core/tokenizer.js

@@ -8,12 +8,12 @@ export default function tokenize(source) {
             const index = source.indexOf("{{");
             const code = index === -1 ? source : source.slice(0, index);
             tokens.push([type, code, position]);
-            if (index === -1) {
-                break;
-            }
             position += index;
             source = source.slice(index);
             type = source.startsWith("{{#") ? "comment" : "tag";
+            if (index === -1) {
+                break;
+            }
             continue;
         }
         if (type === "comment") {
@@ -42,9 +42,11 @@ export default function tokenize(source) {
                     return curr;
                 }
                 // Filters
-                tokens.push(["filter", code]);
+                tokens.push(["filter", code, position + prev]);
                 return curr;
             });
+            if (indexes[lastIndex] == Infinity)
+                return tokens;
             position += indexes[lastIndex];
             source = source.slice(indexes[lastIndex]);
             type = "string";
@@ -64,6 +66,9 @@ export default function tokenize(source) {
             continue;
         }
     }
+    if (type == "string") {
+        tokens.push([type, "", position]);
+    }
     return tokens;
 }
 /**
@@ -82,5 +87,6 @@ export function parseTag(source) {
         indexes.push(index + 2);
         return indexes;
     }
-    throw new Error("Unclosed tag");
+    indexes.push(Infinity);
+    return indexes;
 }

package/loaders/file.js

@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
 import process from "node:process";
 /**
  * Vento file loader for loading templates from the file system.
- * Used by JS runtimes like Node, Deno, and Bun.
+ * Used by Node-like runtimes (Node, Deno, Bun, ...)
  */
 export class FileLoader {
     #root;

package/loaders/filesystem.js

@@ -0,0 +1,36 @@
+import { join } from "./utils.js";
+/**
+ * Vento FileSystem API loader for loading templates.
+ * Used by browser environments.
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/File_System_API
+ */
+export class FileSystemLoader {
+    #handle;
+    constructor(handle) {
+        this.#handle = handle;
+    }
+    async load(file) {
+        const parts = file.split("/");
+        let currentHandle = this.#handle;
+        while (parts.length > 1) {
+            const part = parts.shift();
+            if (part) {
+                currentHandle = await currentHandle.getDirectoryHandle(part, {
+                    create: false,
+                });
+            }
+        }
+        const entry = await currentHandle.getFileHandle(parts[0], {
+            create: false,
+        });
+        const fileHandle = await entry.getFile();
+        const source = await fileHandle.text();
+        return { source };
+    }
+    resolve(from, file) {
+        if (file.startsWith(".")) {
+            return join(from, "..", file);
+        }
+        return join(file);
+    }
+}

package/loaders/memory.js

@@ -1,24 +1,24 @@
-import path from "node:path";
+import { join } from "./utils.js";
 /**
  * Vento loader for loading templates from an in-memory object.
  * Used for testing or in-memory operations.
  */
 export class MemoryLoader {
-    files = {};
+    files;
     constructor(files) {
-        this.files = files;
+        this.files = new Map(Object.entries(files));
     }
     load(file) {
-        if (!(file in this.files)) {
+        const source = this.files.get(file);
+        if (source === undefined) {
             throw new Error(`File not found: ${file}`);
         }
-        const source = this.files[file];
         return Promise.resolve({ source });
     }
     resolve(from, file) {
         if (file.startsWith(".")) {
-            return path.join(path.dirname(from), file).replace(/\\/g, "/");
+            return join(from, "..", file);
         }
-        return path.join("/", file).replace(/\\/g, "/");
+        return join(file);
     }
 }

package/loaders/module.js

@@ -0,0 +1,50 @@
+/**
+ * Vento loader for loading templates from a ES modules.
+ * Used to load precompiled templates.
+ */
+export class ModuleLoader {
+    #root;
+    constructor(root) {
+        this.#root = root;
+    }
+    async load(file) {
+        const url = new URL(join(this.#root.pathname, file + ".js"), this.#root);
+        const module = await import(url.toString());
+        return module.default;
+    }
+    resolve(from, file) {
+        if (file.startsWith(".")) {
+            return join("/", dirname(from), file);
+        }
+        return join("/", file);
+    }
+}
+function join(...parts) {
+    return parts.join("/").replace(/\/+/g, "/");
+}
+function dirname(path) {
+    const lastSlash = path.lastIndexOf("/");
+    return lastSlash === -1 ? "." : path.slice(0, lastSlash);
+}
+/**
+ * Exports a template as a string that can be used in an ES module.
+ * This is useful for precompiled templates.
+ */
+export function exportTemplate(template, options) {
+    if (!template.source) {
+        throw new Error("Template source is not defined");
+    }
+    const exportCode = `export default function (__env) {
+    ${template.toString()};
+
+    ${options?.source
+        ? `__template.path = ${JSON.stringify(template.path)};
+        __template.code = ${JSON.stringify(template.code)};
+        __template.source = ${JSON.stringify(template.source)};`
+        : ""}
+    __template.defaults = ${JSON.stringify(template.defaults || {})};
+
+    return __template;
+  }`;
+    return exportCode;
+}

package/loaders/url.js

@@ -1,3 +1,4 @@
+import { join } from "./utils.js";
 /**
  * Vento URL loader for loading templates from a URL.
  * Used by browser environments.
@@ -14,15 +15,8 @@ export class UrlLoader {
     }
     resolve(from, file) {
         if (file.startsWith(".")) {
-            return join("/", dirname(from), file);
+            return join(from, "..", file);
         }
-        return join("/", file);
+        return join(file);
     }
 }
-function join(...parts) {
-    return parts.join("/").replace(/\/+/g, "/");
-}
-function dirname(path) {
-    const lastSlash = path.lastIndexOf("/");
-    return lastSlash === -1 ? "." : path.slice(0, lastSlash);
-}

package/loaders/utils.js

@@ -0,0 +1,17 @@
+// Adapted from https://gist.github.com/creationix/7435851
+export function join(...paths) {
+    const parts = []
+        .concat(...paths.map((path) => path.split("/")))
+        .filter((part) => part && part !== ".");
+    const newParts = [];
+    for (const part of parts) {
+        if (part === "..") {
+            newParts.pop();
+        }
+        else {
+            newParts.push(part);
+        }
+    }
+    newParts.unshift(""); // Ensure always a leading slash
+    return newParts.join("/");
+}