ventojs 1.14.0 → 2.0.0-canary.0

package/CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## 2.0.0 - Unreleased
+Vento 2.0 is now dependency-free and compatible with browsers without requiring a build process.
+
+### Added
+- Build-less browser support.
+- `plugins/mod.ts` module to register all default plugins easily.
+
+### Changed
+- Renamed `src` directory to `core`.
+- Moved all loaders to the `loaders` root directory.
+- Implemented a different approach to resolve the variables without using `meriyah` to analyze the code. [#128]
+
+### Removed
+- Deprecated option `useWith`.
+- All extenal dependencies (`meriyah`, `estree`, etc).
+- `bare.ts` file since now it's useless.
+
+[#128]: https://github.com/ventojs/vento/issues/128

package/README.md

@@ -20,6 +20,8 @@ Nunjucks, Liquid, Mustache, and EJS.
 ## Features
 
 - Minimal, fast runtime. 🔥
+- No dependencies.
+- Compatible with browsers and JS runtimes (Deno, Node, Bun, etc).
 - Ergonomic by design. All tags and outputs are written with `{{` and `}}`.
 - Write JavaScript anywhere. `{{ await user.getName() }}` is real JS executed at
   runtime.

package/browser.js

@@ -0,0 +1,19 @@
+import { Environment } from "./core/environment.js";
+import { UrlLoader } from "./loaders/url.js";
+import defaultPlugins from "./plugins/mod.js";
+export default function (options) {
+    // Determine the loader based on the includes option
+    const loader = options.includes instanceof URL
+        ? new UrlLoader(options.includes)
+        : options.includes;
+    // Create a new Environment instance with the provided options
+    const env = new Environment({
+        loader,
+        dataVarname: options.dataVarname || "it",
+        autoescape: options.autoescape ?? false,
+        autoDataVarname: options.autoDataVarname ?? true,
+    });
+    // Register the default plugins
+    env.use(defaultPlugins());
+    return env;
+}

package/{esm/src → core}/environment.js

@@ -1,8 +1,6 @@
-import "../_dnt.polyfills.js";
-import * as dntShim from "../_dnt.shims.js";
+import iterateTopLevel from "./js.js";
 import tokenize from "./tokenizer.js";
-import { transformTemplateCode } from "./transformer.js";
-import { TemplateError, TransformError } from "./errors.js";
+import { TemplateError } from "./errors.js";
 export class Environment {
     cache = new Map();
     options;
@@ -26,7 +24,7 @@ export class Environment {
         if (file) {
             const cached = this.cache.get(file);
             if (cached) {
-                return await cached(data);
+                return (await cached)(data);
             }
             const template = this.compile(source, file);
             this.cache.set(file, template);
@@ -47,14 +45,16 @@ export class Environment {
         let code = this.compileTokens(tokens).join("\n");
         const { dataVarname, autoDataVarname } = this.options;
         if (autoDataVarname) {
-            try {
-                code = transformTemplateCode(code, dataVarname);
-            }
-            catch (cause) {
-                if (cause instanceof TransformError) {
-                    throw new TemplateError(path, source, cause.position, cause);
-                }
-                throw new Error(`Unknown error while transforming ${path}`, { cause });
+            const generator = iterateTopLevel(code);
+            const [, , variables] = generator.next().value;
+            while (!generator.next().done)
+                ;
+            variables.delete(dataVarname);
+            if (variables.size > 0) {
+                code = `
+          var {${[...variables].join(",")}} = ${dataVarname};
+          {\n${code}\n}
+        `;
             }
         }
         const constructor = new Function("__file", "__env", "__defaults", "__err", `return${sync ? "" : " async"} function (${dataVarname}) {
@@ -65,7 +65,7 @@ export class Environment {
           ${code}
           return __exports;
         } catch (cause) {
-          const template = __env.cache.get(__file);
+          const template = ${sync ? "" : "await"} __env.cache.get(__file);
           throw new __err(__file, template?.source, __pos, cause);
         }
       }
@@ -77,12 +77,7 @@ export class Environment {
         return template;
     }
     tokenize(source, path) {
-        const result = tokenize(source);
-        let { tokens } = result;
-        const { position, error } = result;
-        if (error) {
-            throw new TemplateError(path, source, position, error);
-        }
+        let tokens = tokenize(source);
         for (const tokenPreprocessor of this.tokenPreprocessors) {
             const result = tokenPreprocessor(this, tokens, path);
             if (result !== undefined) {
@@ -93,16 +88,18 @@ export class Environment {
     }
     async load(file, from) {
         const path = this.options.loader.resolve(from || "", file);
-        if (!this.cache.has(path)) {
-            // Remove query and hash params from path before loading
-            const cleanPath = path
-                .split("?")[0]
-                .split("#")[0];
-            const { source, data } = await this.options.loader.load(cleanPath);
-            const template = this.compile(source, path, data);
-            this.cache.set(path, template);
+        let cached = this.cache.get(path);
+        if (cached) {
+            return cached;
         }
-        return this.cache.get(path);
+        // Remove query and hash params from path before loading
+        const cleanPath = path
+            .split("?")[0]
+            .split("#")[0];
+        cached = this.options.loader.load(cleanPath)
+            .then(({ source, data }) => this.compile(source, path, data));
+        this.cache.set(path, cached);
+        return await cached;
     }
     compileTokens(tokens, outputVar = "__exports.content", stopAt) {
         const compiled = [];
@@ -174,14 +171,15 @@ export class Environment {
     }
 }
 function isGlobal(name) {
-    // @ts-ignore TS doesn't know about globalThis
-    if (Object.hasOwn(dntShim.dntGlobalThis, name)) {
+    if (name == "name")
+        return false;
+    if (Object.hasOwn(globalThis, name)) {
         return true;
     }
     if (name.includes(".")) {
         const [obj, prop] = name.split(".");
         // @ts-ignore TS doesn't know about globalThis
-        return Object.hasOwn(dntShim.dntGlobalThis[obj], prop);
+        return Object.hasOwn(globalThis[obj], prop);
     }
 }
 // deno-lint-ignore no-explicit-any

package/{esm/src → core}/errors.js

@@ -13,13 +13,6 @@ export class TemplateError extends VentoBaseError {
         this.position = position;
     }
 }
-export class TransformError extends VentoBaseError {
-    position;
-    constructor(message, position = 0, cause) {
-        super(message, { cause });
-        this.position = position;
-    }
-}
 /** Returns the number and code of the errored line */
 export function errorLine(source, position) {
     let line = 1;

package/core/js.js

@@ -0,0 +1,191 @@
+import reserved from "./reserved.js";
+const TEMPLATE_PART = /[`}](?:\\?[^])*?(?:`|\${)/y;
+const REGEX_LITERAL_START = /(?<=[(=:,?&!]\s*)\//y;
+const STOPPING_POINT = /['"`{}[\]/|]|((?<!\.\??)\b[a-zA-Z_]\w+)/g;
+/**
+ * This function iterates over the top-level scope of a JavaScript source code
+ * string. It yields pairs of the index and the type of each top-level element
+ * found.
+ *
+ * @example `{ foo: { bar: 1 } }` will yield:
+ * - [0, "{", Set[]]                for the first opening brace
+ * - [18, "}", Set['foo', 'bar']]   for the _second_ closing brace
+ * - [18, "", Set['foo', 'bar']]    for the end of the string
+ */
+export default function* iterateTopLevel(source, start = 0) {
+    const variables = new Set();
+    let cursor = start;
+    let depth = -1;
+    const brackets = [];
+    const max = source.length;
+    parsing: while (cursor < max) {
+        // Search for the next stopping point (e.g., a brace, quote, or regex).
+        STOPPING_POINT.lastIndex = cursor;
+        const match = STOPPING_POINT.exec(source);
+        // No stopping point found, stop parsing.
+        if (!match) {
+            break parsing;
+        }
+        cursor = match.index;
+        const [stop, variable] = match;
+        if (variable) {
+            cursor += variable.length;
+            if (!reserved.has(variable))
+                variables.add(variable);
+            continue;
+        }
+        // Check the type of the stopping point.
+        switch (stop) {
+            case "|": {
+                cursor++;
+                // It's a pipe `|>` in the top-level scope
+                if (depth < 0 && source[cursor] === ">") {
+                    cursor++;
+                    yield [cursor - 2, "|>", variables];
+                }
+                break;
+            }
+            case "'":
+            case `"`: {
+                // It's a quote or double-quote string: find the end.
+                let escapes = 0;
+                do {
+                    cursor = source.indexOf(stop, cursor + 1);
+                    if (cursor == -1) { // No closing quote found
+                        break parsing;
+                    }
+                    escapes = 0;
+                    // Handle escaped quotes
+                    while (source[cursor - 1 - escapes] == "\\") {
+                        escapes++;
+                    }
+                } while (escapes % 2 != 0);
+                cursor++;
+                break;
+            }
+            case "{": {
+                // It's an opening brace: yield if it's in the top-level scope.
+                if (depth < 0)
+                    yield [cursor, "{", variables];
+                cursor++;
+                // Handle `{}`
+                if (source[cursor] == "}")
+                    cursor++;
+                // Push the opening brace onto the stack.
+                else
+                    brackets[++depth] = "{";
+                break;
+            }
+            case "[": {
+                // It's an opening brace: yield if it's in the top-level scope.
+                if (depth < 0)
+                    yield [cursor, "[", variables];
+                cursor++;
+                // Handle `[]`
+                if (source[cursor] == "]")
+                    cursor++;
+                // Push the opening brace onto the stack.
+                else
+                    brackets[++depth] = "[";
+                break;
+            }
+            case "]": {
+                // Close the last opened bracket if it matches.
+                if (brackets[depth] == "[")
+                    depth--;
+                // Yield if it's in the top-level scope.
+                if (depth < 0)
+                    yield [cursor, "]", variables];
+                cursor++;
+                break;
+            }
+            case "}": {
+                // Close the last opened brace if it matches.
+                if (brackets[depth] == "{") {
+                    depth--;
+                    // Yield if it's in the top-level scope.
+                    if (depth < 0)
+                        yield [cursor, "}", variables];
+                    cursor++;
+                    break;
+                }
+                // If it doesn't match, but we're in the top-level scope, yield anyway.
+                if (depth < 0) {
+                    yield [cursor, "}", variables];
+                    cursor++;
+                    break;
+                }
+                // Break if we're not inside in a template literal.
+                // otherwise, continue parsing.
+                if (brackets[depth] != "`") {
+                    cursor++;
+                    break;
+                }
+                depth--;
+            } /* falls through */
+            case "`": {
+                // Search for template literal part or end.
+                TEMPLATE_PART.lastIndex = cursor;
+                const match = TEMPLATE_PART.exec(source);
+                // If we don't find anything, return end of the string.
+                if (!match)
+                    return [max, ""];
+                const [part] = match;
+                cursor += part.length;
+                // We found the end of the template literal
+                if (source[cursor - 1] == "`")
+                    break;
+                // Otherwise, we found a template literal part.
+                // Store the opening backtick in the stack.
+                brackets[++depth] = "`";
+                break;
+            }
+            case "/": {
+                // It's a line comment
+                if (source[cursor + 1] == "/") {
+                    cursor = source.indexOf("\n", cursor + 2);
+                    if (cursor == -1)
+                        break parsing;
+                    break;
+                }
+                // It's a block comment
+                if (source[cursor + 1] == "*") {
+                    cursor = source.indexOf("*/", cursor + 2);
+                    if (cursor == -1)
+                        break parsing;
+                    break;
+                }
+                // Check if it's a regex literal.
+                REGEX_LITERAL_START.lastIndex = cursor;
+                if (!REGEX_LITERAL_START.test(source)) {
+                    cursor++;
+                    break;
+                }
+                // It's a regex literal: find the end.
+                let inCharClass = false;
+                cursor++;
+                do {
+                    const character = source[cursor];
+                    cursor++;
+                    switch (character) {
+                        case "\\":
+                            cursor++;
+                            break;
+                        case "[":
+                            inCharClass = true;
+                            break;
+                        case "]":
+                            inCharClass = false;
+                            break;
+                        case "/":
+                            if (!inCharClass)
+                                continue parsing;
+                            break;
+                    }
+                } while (cursor < max);
+                break parsing;
+            }
+        }
+    }
+    return [max, "", variables];
+}

package/core/reserved.js

@@ -0,0 +1,63 @@
+const variables = new Set([
+    // Words reserved by Vento, used internally. In general, don't use variable
+    // names starting with two underscores to be future-proof and avoid clashes.
+    "__file",
+    "__env",
+    "__defaults",
+    "__err",
+    "__exports",
+    "__pos",
+    "__tmp",
+    // JS reserved words, and some "dangerous" words like `let`, `async`, `of` or
+    // `undefined`, which aren't technically reserved but don't name your
+    // variables that.
+    "async",
+    "await",
+    "break",
+    "case",
+    "catch",
+    "class",
+    "const",
+    "continue",
+    "debugger",
+    "default",
+    "delete",
+    "do",
+    "else",
+    "enum",
+    "export",
+    "extends",
+    "false",
+    "finally",
+    "for",
+    "function",
+    "if",
+    "import",
+    "in",
+    "instanceof",
+    "let",
+    "new",
+    "null",
+    "of",
+    "return",
+    "super",
+    "switch",
+    "this",
+    "throw",
+    "true",
+    "try",
+    "typeof",
+    "undefined",
+    "var",
+    "void",
+    "while",
+    "with",
+    "yield",
+    // Variables that are already defined globally
+    ...Object.getOwnPropertyNames(globalThis),
+]);
+// Remove `name` from the reserved variables
+// because it's widely used in templates
+// and it can cause issues if it's reserved.
+variables.delete("name");
+export default variables;

package/core/tokenizer.js

@@ -0,0 +1,86 @@
+import iterateTopLevel from "./js.js";
+export default function tokenize(source) {
+    const tokens = [];
+    let type = "string";
+    let position = 0;
+    while (source.length > 0) {
+        if (type === "string") {
+            const index = source.indexOf("{{");
+            const code = index === -1 ? source : source.slice(0, index);
+            tokens.push([type, code, position]);
+            if (index === -1) {
+                break;
+            }
+            position += index;
+            source = source.slice(index);
+            type = source.startsWith("{{#") ? "comment" : "tag";
+            continue;
+        }
+        if (type === "comment") {
+            source = source.slice(3);
+            const index = source.indexOf("#}}");
+            const comment = index === -1 ? source : source.slice(0, index);
+            tokens.push([type, comment, position]);
+            if (index === -1) {
+                break;
+            }
+            position += index + 3;
+            source = source.slice(index + 3);
+            type = "string";
+            continue;
+        }
+        if (type === "tag") {
+            const indexes = parseTag(source);
+            const lastIndex = indexes.length - 1;
+            let tag;
+            indexes.reduce((prev, curr, index) => {
+                const code = source.slice(prev, curr - 2);
+                // Tag
+                if (index === 1) {
+                    tag = [type, code, position];
+                    tokens.push(tag);
+                    return curr;
+                }
+                // Filters
+                tokens.push(["filter", code]);
+                return curr;
+            });
+            position += indexes[lastIndex];
+            source = source.slice(indexes[lastIndex]);
+            type = "string";
+            // Search the closing echo tag {{ /echo }}
+            if (tag?.[1].match(/^\-?\s*echo\s*\-?$/)) {
+                const end = /{{\-?\s*\/echo\s*\-?}}/.exec(source);
+                if (!end) {
+                    tokens.push(["string", source, position]);
+                    return tokens;
+                }
+                tokens.push(["string", source.slice(0, end.index), position]);
+                position += end.index;
+                tokens.push(["tag", end[0].slice(2, -2), position]);
+                position += end[0].length;
+                source = source.slice(end.index + end[0].length);
+            }
+            continue;
+        }
+    }
+    return tokens;
+}
+/**
+ * Parse a tag and return the indexes of the start and end brackets, and the filters between.
+ * For example: {{ tag |> filter1 |> filter2 }} => [2, 9, 20, 31]
+ */
+export function parseTag(source) {
+    const indexes = [2];
+    for (const [index, reason] of iterateTopLevel(source, 2)) {
+        if (reason == "|>") {
+            indexes.push(index + 2);
+            continue;
+        }
+        else if (!source.startsWith("}}", index))
+            continue;
+        indexes.push(index + 2);
+        return indexes;
+    }
+    throw new Error("Unclosed tag");
+}

package/highlightjs-vento.js

@@ -0,0 +1,24 @@
+/** @type LanguageFn */
+export default function (hljs) {
+  return {
+    name: "vento",
+    subLanguage: "xml",
+    contains: [
+      hljs.COMMENT("{{#", "#}}"),
+      {
+        begin: "{{[-]?",
+        end: "[-]?}}",
+        subLanguage: "javascript",
+        excludeBegin: true,
+        excludeEnd: true,
+      },
+      {
+        begin: "^---\n",
+        end: "\n---\n",
+        subLanguage: "yaml",
+        excludeBegin: true,
+        excludeEnd: true,
+      },
+    ],
+  };
+}

package/{esm/src/loader.node.js → loaders/file.js}

@@ -1,7 +1,10 @@
-import "../_dnt.polyfills.js";
 import path from "node:path";
 import fs from "node:fs/promises";
 import process from "node:process";
+/**
+ * Vento file loader for loading templates from the file system.
+ * Used by JS runtimes like Node, Deno, and Bun.
+ */
 export class FileLoader {
     #root;
     constructor(root = process.cwd()) {
@@ -9,7 +12,7 @@ export class FileLoader {
     }
     async load(file) {
         return {
-            source: await fs.readFile(file, { encoding: "utf-8" }),
+            source: await fs.readFile(file, "utf-8"),
         };
     }
     resolve(from, file) {

package/loaders/memory.js

@@ -0,0 +1,24 @@
+import path from "node:path";
+/**
+ * Vento loader for loading templates from an in-memory object.
+ * Used for testing or in-memory operations.
+ */
+export class MemoryLoader {
+    files = {};
+    constructor(files) {
+        this.files = files;
+    }
+    load(file) {
+        if (!(file in this.files)) {
+            throw new Error(`File not found: ${file}`);
+        }
+        const source = this.files[file];
+        return Promise.resolve({ source });
+    }
+    resolve(from, file) {
+        if (file.startsWith(".")) {
+            return path.join(path.dirname(from), file).replace(/\\/g, "/");
+        }
+        return path.join("/", file).replace(/\\/g, "/");
+    }
+}

package/loaders/url.js

@@ -0,0 +1,28 @@
+/**
+ * Vento URL loader for loading templates from a URL.
+ * Used by browser environments.
+ */
+export class UrlLoader {
+    #root;
+    constructor(root) {
+        this.#root = root;
+    }
+    async load(file) {
+        const url = new URL(join(this.#root.pathname, file), this.#root);
+        const source = await (await fetch(url)).text();
+        return { source };
+    }
+    resolve(from, file) {
+        if (file.startsWith(".")) {
+            return join("/", dirname(from), file);
+        }
+        return join("/", file);
+    }
+}
+function join(...parts) {
+    return parts.join("/").replace(/\/+/g, "/");
+}
+function dirname(path) {
+    const lastSlash = path.lastIndexOf("/");
+    return lastSlash === -1 ? "." : path.slice(0, lastSlash);
+}

package/{esm/bare.js → mod.js}

@@ -1,14 +1,19 @@
-import { Environment } from "./src/environment.js";
-import { FileLoader } from "./src/loader.node.js";
+import { Environment } from "./core/environment.js";
+import { FileLoader } from "./loaders/file.js";
+import defaultPlugins from "./plugins/mod.js";
 export default function (options = {}) {
+    // Determine the loader based on the includes option
     const loader = typeof options.includes === "object"
         ? options.includes
         : new FileLoader(options.includes);
+    // Create a new Environment instance with the provided options
     const env = new Environment({
         loader,
         dataVarname: options.dataVarname || "it",
         autoescape: options.autoescape ?? false,
         autoDataVarname: options.autoDataVarname ?? true,
     });
+    // Register the default plugins
+    env.use(defaultPlugins());
     return env;
 }