pure-dango 1.8.0 → 1.8.2

package/dist/lib/tokenizer/tokenizer.js

@@ -0,0 +1,298 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var tokenizer_exports = {};
+__export(tokenizer_exports, {
+  tokenizer: () => tokenizer
+});
+module.exports = __toCommonJS(tokenizer_exports);
+var import_errors = require("../../runtime/errors");
+var quote_crimes = __toESM(require("./funnies"), 1);
+const REGEX = new RegExp([
+  // strings
+  "(`(?:[^`\\\\]|\\\\.)*`)",
+  // backtick strings
+  "('(?:[^'\\\\]|\\\\.)*')",
+  // single quote strings
+  "(\u201C(?:[^\u201C\u201D\\\\]|\\\\.)*\u201D)",
+  // curly single quote strings
+  "(\u2018(?:[^\u2018\u2019\\\\]|\\\\.)*\u2019)",
+  // curly double quote strings
+  // multi-char operators
+  "\\?\\?",
+  "&&",
+  "\\|\\|",
+  "!=",
+  "<=",
+  ">=",
+  "==",
+  "--",
+  "\\+\\+",
+  "-=",
+  "\\+=",
+  "/=",
+  "\\*=",
+  "&=",
+  "\\|=",
+  "\\^=",
+  // bitwise assignment
+  "<<=",
+  ">>=",
+  ">>>=",
+  // shift assignment
+  ">>>",
+  ">>",
+  "<<",
+  // shifts (longest first)
+  "\\.\\..",
+  // spread
+  // number literals
+  "0b[01]+",
+  // binary
+  "0x[0-9a-fA-F]+",
+  // hex
+  "\\d+(\\.\\d+)?",
+  // float
+  // identifiers
+  "[\\p{L}_][\\p{L}\\d_]*",
+  // single char
+  "[+\\-*/=()&^%$#@!<>?:~,|]",
+  "[\\[\\]{}]",
+  "[\\n;]",
+  "\\."
+].join("|"), "gu");
+const keywordSet = /* @__PURE__ */ new Set(["new", "const", "if", "else", "while", "continue", "break", "for", "in", "of", "function", "return", "import", "class", "extends", "inst", "internal", "try", "catch", "finally", "do", "switch", "case", "default"]);
+const separatorSet = /* @__PURE__ */ new Set(["\n", ",", "	", ";"]);
+const operatorSet = /* @__PURE__ */ new Set([
+  "...",
+  "??",
+  "&&",
+  "||",
+  "{",
+  "}",
+  "[",
+  "]",
+  "!=",
+  "<=",
+  ">=",
+  "==",
+  "-=",
+  "+=",
+  "++",
+  "/=",
+  "*=",
+  "--",
+  "&=",
+  "|=",
+  "^=",
+  "<<=",
+  ">>=",
+  ">>>=",
+  ">>>",
+  ">>",
+  "<<",
+  "+",
+  "-",
+  "*",
+  "/",
+  "%",
+  "=",
+  "(",
+  ")",
+  "&",
+  "^",
+  "!",
+  "<",
+  ">",
+  "?",
+  ":",
+  "~",
+  ".",
+  "|"
+]);
+function getType(code) {
+  if (keywordSet.has(code))
+    return "Keyword";
+  if (separatorSet.has(code))
+    return "Separator";
+  if (operatorSet.has(code))
+    return "Operator";
+  if (/^["'`]/.test(code))
+    return "StringLiteral";
+  if (/^\d/.test(code))
+    return "Literal";
+  return "Identifier";
+}
+function tokenizer(code) {
+  const tokens = [];
+  const uid = Math.random().toString(36).slice(2);
+  const stringMeta = /* @__PURE__ */ new Map();
+  const out = [];
+  let row = 1;
+  let column = 1;
+  let i = 0;
+  let stringIndex = 0;
+  while (i < code.length) {
+    const character = code[i];
+    if (character === "#") {
+      while (i < code.length && code[i] !== "\n")
+        i++;
+      continue;
+    }
+    if (character === "/" && code[i + 1] === "*") {
+      while (i < code.length && !(code[i] === "*" && code[i + 1] === "/")) {
+        if (code[i] === "\n") {
+          row++;
+          column = 1;
+        }
+        i++;
+      }
+      i += 2;
+      continue;
+    }
+    if ((character === '"' || character === "\u201C" || character === "'" || character === "\u2018" || character === "`") && (i === 0 || !/[\p{L}\d]/u.test(code[i - 1]))) {
+      const quote = character === "\u201C" ? "\u201D" : character === "\u2018" ? "\u2019" : character;
+      const stringRow = row;
+      const stringColumn = column;
+      const start = i;
+      i++;
+      column++;
+      while (i < code.length && code[i] !== quote) {
+        const type = quote === "\u201D" ? "double" : "single";
+        const opening = type === "double" ? "\u201C" : "\u2018";
+        if (code[i] === "\u201C" && quote === "\u201D" || code[i] === "\u2018" && quote === "\u2019") {
+          const quoteCrimesMessage = quote_crimes.QuoteCrimesMessage(quote_crimes.incrementQuoteCrimes() - 1, opening) + " ";
+          throw new import_errors.Tokenizer(
+            quoteCrimesMessage + `Found: ${opening}...${opening}. Tip: Either match them or just use normal ${type} quotes`,
+            row,
+            column
+          );
+        }
+        if (code[i] === "\\") {
+          i += 2;
+          column += 2;
+        } else {
+          if (code[i] === "\n") {
+            row++;
+            column = 1;
+          } else
+            column++;
+          i++;
+        }
+      }
+      i++;
+      column++;
+      const raw = code.slice(start, i);
+      const placeholder = `__STRING${uid}_${stringIndex++}__`;
+      stringMeta.set(
+        placeholder,
+        {
+          value: raw.slice(1, -1).replace(/\\"/g, '"').replace(/\\'/g, "'").replace(/\\\\/g, "\\").replace(/\\n/g, "\n").replace(/\\t/g, "	").replace(/\\r/g, "\r"),
+          row: stringRow,
+          column: stringColumn
+        }
+      );
+      out.push(placeholder);
+      continue;
+    }
+    if (character === "\n") {
+      row++;
+      column = 1;
+    } else
+      column++;
+    out.push(character);
+    i++;
+  }
+  const processed = out.join("");
+  row = 1;
+  column = 1;
+  let commented = false;
+  const matches = processed.matchAll(REGEX);
+  for (const match of matches) {
+    const value = match[0];
+    if (value === "\n") {
+      row++;
+      column = 1;
+      commented = false;
+      continue;
+    }
+    if (value === "#") {
+      commented = true;
+      continue;
+    }
+    if (commented || value === "	") {
+      column += value.length;
+      continue;
+    }
+    if (value === ",") {
+      tokens.push(
+        {
+          type: "Separator",
+          value,
+          row,
+          column
+        }
+      );
+      column += value.length;
+      continue;
+    }
+    if (stringMeta.has(value)) {
+      const meta = stringMeta.get(value);
+      tokens.push(
+        {
+          type: "StringLiteral",
+          value: meta.value,
+          row: meta.row,
+          column: meta.column
+        }
+      );
+      column += value.length;
+      continue;
+    }
+    const lines = value.split("\n");
+    if (lines.length > 1) {
+      row += lines.length - 1;
+      column = lines[lines.length - 1].length + 1;
+    }
+    tokens.push(
+      {
+        type: getType(value),
+        value,
+        row,
+        column
+      }
+    );
+    if (lines.length === 1)
+      column += value.length;
+  }
+  return tokens;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  tokenizer
+});

package/package.json

@@ -1,17 +1,22 @@
 {
     "name": "pure-dango",
-    "version": "1.8.0",
+    "version": "1.8.2",
     "type": "module",
     "description": "A simple programming language built in JavaScript",
     "license": "MIT",
-    "files": ["src/core", "src/runtime"],
+    "files": [
+        "src/core",
+        "src/runtime",
+        "dist"
+    ],
     "bin": {
         "pure-dango": "src/index.ts"
     },
     "scripts": {
         "dev": "tsx src/index.ts",
         "prebuild": "node src/scripts/libs.cjs",
-        "build": "node src/scripts/build.js"
+        "build": "node src/scripts/build.js",
+        "build:lib": "esbuild src/core/tokenizer/tokenizer.ts src/core/parser/main.ts --bundle=false --format=cjs --outdir=dist/lib --platform=node"
     },
     "dependencies": {
         "gmp-wasm": "^1.3.2"
@@ -41,5 +46,11 @@
             "node22-linux-x64",
             "node22-linux-arm64"
         ]
+    },
+    "main": "./dist/PureDango.cjs",
+    "exports": {
+        ".": "./dist/PureDango.cjs",
+        "./src/core/tokenizer/tokenizer.js": "./dist/lib/tokenizer.js",
+        "./src/core/parser/main.js": "./dist/lib/main.js"
     }
-}
+}

package/src/index.ts

@@ -38,7 +38,7 @@ const
 const packageJson =
 {
     name:        typeof PACKAGE_NAME        !== "undefined" ? PACKAGE_NAME        : "pure-dango",
-    version:     typeof PACKAGE_VERSION     !== "undefined" ? PACKAGE_VERSION     : "1.8.0",
+    version:     typeof PACKAGE_VERSION     !== "undefined" ? PACKAGE_VERSION     : "1.8.1",
     description: typeof PACKAGE_DESCRIPTION !== "undefined" ? PACKAGE_DESCRIPTION : "A simple programming language built in JavaScript"
 };