ventojs 1.15.2 → 2.0.0-canary.1

package/CHANGELOG.md

@@ -0,0 +1,49 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## 2.0.0 - Unreleased
+Vento 2.0 is now dependency-free and compatible with browsers without a build step.
+
+### Added
+- Build-less browser support.
+- `plugins/mod.ts` module to register all default plugins easily.
+- Support for precompiled templates.
+- New filesystem loader to use File System API.
+- Better errors reporting [#131], [#137]
+
+### Changed
+- Renamed `src` directory to `core`.
+- Moved all loaders to the `loaders` root directory.
+- Implemented a different approach to resolve the variables without using `meriyah` to analyze the code [#128].
+- The signature of `tag` plugins has changed:
+  ```diff
+  -- (env: Environment, code: string, output: string, tokens: Tokens[])
+  ++ (env: Environment, token: Token, output: string, tokens: Tokens[])
+  ```
+- The `compileTokens` function has changed. The third argument is a string with the closing tag and now it throws an error if its not found:
+  ```diff
+  -- env.compileTokens(tokens, tmpOutput, ["/code"]);
+  -- if (tokens.length && (tokens[0][0] !== "tag" || tokens[0][1] !== "/code")) {
+  --   throw new Error("missing closing tag");
+  -- }
+  ++ env.compileTokens(tokens, tmpOutput, "/code");
+  ```
+
+### Removed
+- `runStringSync` function.
+- Deprecated option `useWith`.
+- All extenal dependencies (`meriyah`, `estree`, etc).
+- `bare.ts` file since now it's useless.
+
+### Fixed
+- Functions output when the autoescape is enabled [#95]
+- Improved escape filter performance [#134]
+
+[#95]: https://github.com/ventojs/vento/issues/95
+[#128]: https://github.com/ventojs/vento/issues/128
+[#131]: https://github.com/ventojs/vento/issues/131
+[#134]: https://github.com/ventojs/vento/issues/134
+[#137]: https://github.com/ventojs/vento/issues/137

package/README.md

@@ -20,6 +20,8 @@ Nunjucks, Liquid, Mustache, and EJS.
 ## Features
 
 - Minimal, fast runtime. 🔥
+- No dependencies.
+- Compatible with browsers and JS runtimes (Deno, Node, Bun, etc).
 - Ergonomic by design. All tags and outputs are written with `{{` and `}}`.
 - Write JavaScript anywhere. `{{ await user.getName() }}` is real JS executed at
   runtime.

package/{esm/src → core}/environment.js

@@ -1,8 +1,6 @@
-import "../_dnt.polyfills.js";
-import * as dntShim from "../_dnt.shims.js";
+import iterateTopLevel from "./js.js";
 import tokenize from "./tokenizer.js";
-import { transformTemplateCode } from "./transformer.js";
-import { TemplateError, TransformError } from "./errors.js";
+import { createError, TokenError } from "./errors.js";
 export class Environment {
     cache = new Map();
     options;
@@ -11,6 +9,10 @@ export class Environment {
     filters = {};
     utils = {
         callMethod,
+        createError,
+        safeString(str) {
+            return new SafeString(str);
+        },
     };
     constructor(options) {
         this.options = options;
@@ -35,46 +37,73 @@ export class Environment {
         const template = this.compile(source, file);
         return await template(data);
     }
-    runStringSync(source, data) {
-        const template = this.compile(source, "", {}, true);
-        return template(data);
-    }
-    compile(source, path, defaults, sync = false) {
+    compile(source, path, defaults) {
         if (typeof source !== "string") {
-            throw new Error(`The source code of "${path}" must be a string. Got ${typeof source}`);
+            throw new TypeError(`The source code of "${path}" must be a string. Got ${typeof source}`);
+        }
+        const allTokens = this.tokenize(source, path);
+        const tokens = [...allTokens];
+        const lastToken = tokens.at(-1);
+        if (lastToken[0] != "string") {
+            throw new TokenError("Unclosed tag", lastToken, source, path);
+        }
+        let code = "";
+        try {
+            code = this.compileTokens(tokens).join("\n");
+        }
+        catch (error) {
+            if (!(error instanceof Error))
+                throw error;
+            throw createError(error, {
+                source,
+                code,
+                tokens: allTokens,
+                path,
+            });
         }
-        const tokens = this.tokenize(source, path);
-        let code = this.compileTokens(tokens).join("\n");
         const { dataVarname, autoDataVarname } = this.options;
         if (autoDataVarname) {
-            try {
-                code = transformTemplateCode(code, dataVarname);
-            }
-            catch (cause) {
-                if (cause instanceof TransformError) {
-                    throw new TemplateError(path, source, cause.position, cause);
-                }
-                throw new Error(`Unknown error while transforming ${path}`, { cause });
+            const generator = iterateTopLevel(code);
+            const [, , variables] = generator.next().value;
+            while (!generator.next().done)
+                ;
+            variables.delete(dataVarname);
+            if (variables.size > 0) {
+                code = `
+          var {${[...variables].join(",")}} = ${dataVarname};
+          {\n${code}\n}
+        `;
             }
         }
-        const constructor = new Function("__file", "__env", "__defaults", "__err", `return${sync ? "" : " async"} function (${dataVarname}) {
-        let __pos = 0;
         try {
-          ${dataVarname} = Object.assign({}, __defaults, ${dataVarname});
-          const __exports = { content: "" };
-          ${code}
-          return __exports;
-        } catch (cause) {
-          const template = ${sync ? "" : "await"} __env.cache.get(__file);
-          throw new __err(__file, template?.source, __pos, cause);
+            const constructor = new Function("__env", `return async function __template(${dataVarname}) {
+          try {
+            ${dataVarname} = Object.assign({}, __template.defaults, ${dataVarname});
+            const __exports = { content: "" };
+            ${code}
+            return __exports;
+          } catch (error) {
+            throw __env.utils.createError(error, __template);
+          }
+        }`);
+            const template = constructor(this);
+            template.path = path;
+            template.code = constructor.toString();
+            template.source = source;
+            template.tokens = allTokens;
+            template.defaults = defaults || {};
+            return template;
+        }
+        catch (error) {
+            if (!(error instanceof Error))
+                throw error;
+            throw createError(error, {
+                source,
+                code,
+                tokens: allTokens,
+                path,
+            });
         }
-      }
-      `);
-        const template = constructor(path, this, defaults, TemplateError);
-        template.file = path;
-        template.code = code;
-        template.source = source;
-        return template;
     }
     tokenize(source, path) {
         let tokens = tokenize(source);
@@ -97,17 +126,27 @@ export class Environment {
             .split("?")[0]
             .split("#")[0];
         cached = this.options.loader.load(cleanPath)
-            .then(({ source, data }) => this.compile(source, path, data));
+            .then((result) => {
+            if (typeof result === "function") {
+                return result(this);
+            }
+            const { source, data } = result;
+            return this.compile(source, path, data);
+        });
         this.cache.set(path, cached);
         return await cached;
     }
-    compileTokens(tokens, outputVar = "__exports.content", stopAt) {
+    compileTokens(tokens, outputVar = "__exports.content", closeToken) {
         const compiled = [];
+        let openToken;
         tokens: while (tokens.length > 0) {
-            if (stopAt && tokens[0][0] === "tag" && stopAt.includes(tokens[0][1])) {
-                break;
+            const token = tokens.shift();
+            const [type, code, pos] = token;
+            openToken ??= token;
+            // We found the closing tag, so we stop compiling
+            if (closeToken && type === "tag" && closeToken === code) {
+                return compiled;
             }
-            const [type, code, pos] = tokens.shift();
             if (type === "comment") {
                 continue;
             }
@@ -118,9 +157,9 @@ export class Environment {
                 continue;
             }
             if (type === "tag") {
-                compiled.push(`__pos = ${pos};`);
+                compiled.push(`/*__pos:${pos}*/`);
                 for (const tag of this.tags) {
-                    const compiledTag = tag(this, code, outputVar, tokens);
+                    const compiledTag = tag(this, token, outputVar, tokens);
                     if (typeof compiledTag === "string") {
                         compiled.push(compiledTag);
                         continue tokens;
@@ -131,17 +170,22 @@ export class Environment {
                 compiled.push(`${outputVar} += (${expression}) ?? "";`);
                 continue;
             }
-            throw new Error(`Unknown token type "${type}"`);
+            throw new TokenError(`Unknown token type "${type}"`, token);
+        }
+        // If we reach here, it means we have an open token that wasn't closed
+        if (closeToken) {
+            throw new TokenError(`Missing closing tag ("${closeToken}" tag is expected)`, openToken);
         }
         return compiled;
     }
     compileFilters(tokens, output, autoescape = false) {
         let unescaped = false;
         while (tokens.length > 0 && tokens[0][0] === "filter") {
-            const [, code] = tokens.shift();
+            const token = tokens.shift();
+            const [, code, position] = token;
             const match = code.match(/^(await\s+)?([\w.]+)(?:\((.*)\))?$/);
             if (!match) {
-                throw new Error(`Invalid filter: ${code}`);
+                throw new TokenError(`Invalid filter: ${code}`, token);
             }
             const [_, isAsync, name, args] = match;
             if (!Object.hasOwn(this.filters, name)) {
@@ -154,7 +198,7 @@ export class Environment {
                 }
                 else {
                     // It's a prototype's method (e.g. `String.toUpperCase()`)
-                    output = `${isAsync ? "await " : ""}__env.utils.callMethod(${output}, "${name}", ${args ? args : ""})`;
+                    output = `${isAsync ? "await " : ""}__env.utils.callMethod(${position}, ${output}, "${name}", ${args ? args : ""})`;
                 }
             }
             else {
@@ -171,26 +215,30 @@ export class Environment {
     }
 }
 function isGlobal(name) {
-    // @ts-ignore TS doesn't know about globalThis
-    if (Object.hasOwn(dntShim.dntGlobalThis, name)) {
+    if (name == "name")
+        return false;
+    if (Object.hasOwn(globalThis, name)) {
         return true;
     }
     if (name.includes(".")) {
         const [obj, prop] = name.split(".");
         // @ts-ignore TS doesn't know about globalThis
-        return Object.hasOwn(dntShim.dntGlobalThis[obj], prop);
+        return Object.hasOwn(globalThis[obj], prop);
     }
 }
+function callMethod(position, 
 // deno-lint-ignore no-explicit-any
-function callMethod(thisObject, method, ...args) {
+thisObject, method, ...args) {
     if (thisObject === null || thisObject === undefined) {
         return thisObject;
     }
     if (typeof thisObject[method] === "function") {
         return thisObject[method](...args);
     }
-    throw new Error(`"${method}" is not a valid filter, global object or a method of a ${typeof thisObject} variable`);
+    throw new TokenError(`Method "${method}" is not a function of ${typeof thisObject} variable`, position);
 }
 function checkAsync(fn) {
     return fn.constructor?.name === "AsyncFunction";
 }
+export class SafeString extends String {
+}

package/core/errors.js

@@ -0,0 +1,182 @@
+export class VentoError extends Error {
+}
+export class TokenError extends VentoError {
+    token;
+    source;
+    file;
+    constructor(message, token, source, file) {
+        super(message);
+        this.name = "TokenError";
+        this.token = token;
+        this.source = source;
+        this.file = file;
+    }
+    getContext() {
+        if (!this.source || this.token === undefined) {
+            return;
+        }
+        return {
+            type: this.name,
+            message: this.message,
+            source: this.source,
+            position: typeof this.token === "number" ? this.token : this.token[2],
+            file: this.file,
+        };
+    }
+}
+export class RuntimeError extends VentoError {
+    context;
+    constructor(error, context) {
+        super(error.message);
+        this.name = error.name || "JavaScriptError";
+        this.context = context;
+        this.cause = error;
+    }
+    getContext() {
+        if (this.cause instanceof SyntaxError) {
+            return parseSyntaxError(this.cause, this.context);
+        }
+        if (this.cause instanceof Error) {
+            return parseError(this.cause, this.context);
+        }
+    }
+}
+export function createError(error, context) {
+    if (error instanceof RuntimeError)
+        return error;
+    // If the error is a TokenError, we can enhance it with the context information
+    if (error instanceof TokenError) {
+        error.file ??= context.path;
+        error.source ??= context.source;
+        return error;
+    }
+    // JavaScript syntax errors can be parsed to get accurate position
+    return new RuntimeError(error, context);
+}
+export async function printError(error) {
+    if (error instanceof VentoError) {
+        const context = await error.getContext();
+        if (context) {
+            console.error(stringifyContext(context));
+            return;
+        }
+    }
+    console.error(error);
+}
+function parseError(error, context) {
+    const stackMatch = error.stack?.match(/<anonymous>:(\d+):(\d+)/);
+    if (!stackMatch)
+        return;
+    const row = Number(stackMatch[1]) - 1;
+    const col = Number(stackMatch[2]);
+    const position = getAccurateErrorPosition(row, col, context);
+    if (position == -1)
+        return;
+    return {
+        type: error.name || "JavaScriptError",
+        message: error.message,
+        source: context.source,
+        position,
+        file: context.path,
+    };
+}
+async function parseSyntaxError(error, context) {
+    const code = `()=>{${context.code}}`;
+    const dataUrl = "data:application/javascript;base64," + btoa(code);
+    const stack = await import(dataUrl).catch(({ stack }) => stack);
+    if (!stack)
+        return;
+    const stackMatch = stack?.match(/:(\d+):(\d+)$/m);
+    if (!stackMatch)
+        return;
+    const row = Number(stackMatch[1]) - 1;
+    const col = Number(stackMatch[2]);
+    const position = getAccurateErrorPosition(row, col, context);
+    if (position == -1)
+        return;
+    return {
+        type: "SyntaxError",
+        message: error.message,
+        source: context.source,
+        position,
+        file: context.path,
+    };
+}
+function getAccurateErrorPosition(row, col, context) {
+    const { code, tokens, source } = context;
+    if (!tokens)
+        return -1;
+    const linesAndDelims = code.split(/(\r\n?|[\n\u2028\u2029])/);
+    const linesAndDelimsUntilIssue = linesAndDelims.slice(0, row * 2);
+    const issueIndex = linesAndDelimsUntilIssue.join("").length + col;
+    const posLine = linesAndDelimsUntilIssue.findLast((line) => {
+        return /^\/\*__pos:(\d+)\*\/$/.test(line);
+    });
+    if (!posLine)
+        return -1;
+    const position = Number(posLine.slice(8, -2));
+    const token = tokens.findLast((token) => {
+        if (token[2] == undefined)
+            return false;
+        return token[2] <= position;
+    });
+    if (!token)
+        return -1;
+    const isJS = token[1].startsWith(">");
+    const tag = isJS ? token[1].slice(1).trimStart() : token[1];
+    const issueStartIndex = code.lastIndexOf(tag, issueIndex);
+    if (issueStartIndex == -1)
+        return -1;
+    const sourceIssueStartIndex = source.indexOf(tag, position);
+    return sourceIssueStartIndex + issueIndex - issueStartIndex - 1;
+}
+const terminal = {
+    number: (n) => `\x1b[33m${n}\x1b[39m`,
+    dim: (line) => `\x1b[2m${line}\x1b[22m`,
+    error: (msg) => `\x1b[31m${msg}\x1b[39m`,
+};
+const LINE_TERMINATOR = /\r\n?|[\n\u2028\u2029]/;
+export function stringifyContext(context, format = terminal) {
+    const { type, message, source, position, file } = context;
+    const sourceAfterIssue = source.slice(position);
+    const newlineMatch = sourceAfterIssue.match(LINE_TERMINATOR);
+    const endIndex = position + (newlineMatch?.index ?? sourceAfterIssue.length);
+    const lines = source.slice(0, endIndex).split(LINE_TERMINATOR);
+    const displayedLineEntries = [...lines.entries()].slice(-3);
+    const endLineIndex = lines.at(-1).length + position - endIndex;
+    const numberLength = (displayedLineEntries.at(-1)[0] + 1).toString().length;
+    const displayedCode = displayedLineEntries.map(([index, line]) => {
+        const number = `${index + 1}`.padStart(numberLength);
+        const sidebar = ` ${format.number(number)} ${format.dim("|")} `;
+        return sidebar + line;
+    }).join("\n");
+    const sidebarWidth = numberLength + 4;
+    const tooltipIndex = sidebarWidth + endLineIndex;
+    const tooltipIndent = " ".repeat(tooltipIndex);
+    const tooltip = tooltipIndent + format.error(`^ ${message}`);
+    const output = [];
+    output.push(`${format.error(type)}: ${message}`);
+    if (file) {
+        output.push(format.dim(getLocation(file, source, position)), "");
+    }
+    output.push(displayedCode, tooltip);
+    return output.join("\n");
+}
+function getLocation(file, source, position) {
+    let line = 1;
+    let column = 1;
+    for (let index = 0; index < position; index++) {
+        if (source[index] === "\n" ||
+            (source[index] === "\r" && source[index + 1] === "\n")) {
+            line++;
+            column = 1;
+            if (source[index] === "\r") {
+                index++;
+            }
+        }
+        else {
+            column++;
+        }
+    }
+    return `${file}:${line}:${column}`;
+}

package/{esm/src → core}/js.js

@@ -1,17 +1,19 @@
-import "../_dnt.polyfills.js";
+import reserved from "./reserved.js";
 const TEMPLATE_PART = /[`}](?:\\?[^])*?(?:`|\${)/y;
 const REGEX_LITERAL_START = /(?<=[(=:,?&!]\s*)\//y;
-const STOPPING_POINT = /['"`{}[\]/|]/g;
+const STOPPING_POINT = /['"`{}[\]/|]|((?<!\.\??)\b[a-zA-Z_]\w+)/g;
 /**
- * This function iterates over the top-level scope of a JavaScript source code string.
- * It yields pairs of the index and the type of each top-level element found.
+ * This function iterates over the top-level scope of a JavaScript source code
+ * string. It yields pairs of the index and the type of each top-level element
+ * found.
  *
  * @example `{ foo: { bar: 1 } }` will yield:
- * - [0, "{"] for the opening brace
- * - [18, "}"] for the closing brace
- * - [18, ""] for the end of the string
+ * - [0, "{", Set[]]                for the first opening brace
+ * - [18, "}", Set['foo', 'bar']]   for the _second_ closing brace
+ * - [18, "", Set['foo', 'bar']]    for the end of the string
  */
 export default function* iterateTopLevel(source, start = 0) {
+    const variables = new Set();
     let cursor = start;
     let depth = -1;
     const brackets = [];
@@ -25,7 +27,13 @@ export default function* iterateTopLevel(source, start = 0) {
             break parsing;
         }
         cursor = match.index;
-        const [stop] = match;
+        const [stop, variable] = match;
+        if (variable) {
+            cursor += variable.length;
+            if (!reserved.has(variable))
+                variables.add(variable);
+            continue;
+        }
         // Check the type of the stopping point.
         switch (stop) {
             case "|": {
@@ -33,7 +41,7 @@ export default function* iterateTopLevel(source, start = 0) {
                 // It's a pipe `|>` in the top-level scope
                 if (depth < 0 && source[cursor] === ">") {
                     cursor++;
-                    yield [cursor - 2, "|>"];
+                    yield [cursor - 2, "|>", variables];
                 }
                 break;
             }
@@ -58,7 +66,7 @@ export default function* iterateTopLevel(source, start = 0) {
             case "{": {
                 // It's an opening brace: yield if it's in the top-level scope.
                 if (depth < 0)
-                    yield [cursor, "{"];
+                    yield [cursor, "{", variables];
                 cursor++;
                 // Handle `{}`
                 if (source[cursor] == "}")
@@ -71,7 +79,7 @@ export default function* iterateTopLevel(source, start = 0) {
             case "[": {
                 // It's an opening brace: yield if it's in the top-level scope.
                 if (depth < 0)
-                    yield [cursor, "["];
+                    yield [cursor, "[", variables];
                 cursor++;
                 // Handle `[]`
                 if (source[cursor] == "]")
@@ -87,7 +95,7 @@ export default function* iterateTopLevel(source, start = 0) {
                     depth--;
                 // Yield if it's in the top-level scope.
                 if (depth < 0)
-                    yield [cursor, "]"];
+                    yield [cursor, "]", variables];
                 cursor++;
                 break;
             }
@@ -97,13 +105,13 @@ export default function* iterateTopLevel(source, start = 0) {
                     depth--;
                     // Yield if it's in the top-level scope.
                     if (depth < 0)
-                        yield [cursor, "}"];
+                        yield [cursor, "}", variables];
                     cursor++;
                     break;
                 }
                 // If it doesn't match, but we're in the top-level scope, yield anyway.
                 if (depth < 0) {
-                    yield [cursor, "}"];
+                    yield [cursor, "}", variables];
                     cursor++;
                     break;
                 }
@@ -179,5 +187,5 @@ export default function* iterateTopLevel(source, start = 0) {
             }
         }
     }
-    return [max, ""];
+    return [max, "", variables];
 }

package/core/reserved.js

@@ -0,0 +1,64 @@
+const variables = new Set([
+    // Words reserved by Vento, used internally. In general, don't use variable
+    // names starting with two underscores to be future-proof and avoid clashes.
+    "__file",
+    "__template",
+    "__env",
+    "__defaults",
+    "__err",
+    "__exports",
+    "__pos",
+    "__tmp",
+    // JS reserved words, and some "dangerous" words like `let`, `async`, `of` or
+    // `undefined`, which aren't technically reserved but don't name your
+    // variables that.
+    "async",
+    "await",
+    "break",
+    "case",
+    "catch",
+    "class",
+    "const",
+    "continue",
+    "debugger",
+    "default",
+    "delete",
+    "do",
+    "else",
+    "enum",
+    "export",
+    "extends",
+    "false",
+    "finally",
+    "for",
+    "function",
+    "if",
+    "import",
+    "in",
+    "instanceof",
+    "let",
+    "new",
+    "null",
+    "of",
+    "return",
+    "super",
+    "switch",
+    "this",
+    "throw",
+    "true",
+    "try",
+    "typeof",
+    "undefined",
+    "var",
+    "void",
+    "while",
+    "with",
+    "yield",
+    // Variables that are already defined globally
+    ...Object.getOwnPropertyNames(globalThis),
+]);
+// Remove `name` from the reserved variables
+// because it's widely used in templates
+// and it can cause issues if it's reserved.
+variables.delete("name");
+export default variables;