ts-highlight 0.1.0 → 0.3.0

package/README.md

@@ -1 +1,63 @@
-# <p align='center'> **es-highlight** </p>
+# <p align='center'> **ts-highlight** </p>
+
+<div align='center'>
+
+[![CI](https://github.com/a-marigold/crumb/actions/workflows/ci.yaml/badge.svg)](https://github.com/a-marigold/ts-highlight/actions) [![npm](https://img.shields.io/npm/v/ts-highlight)](https://npmjs.com/package/ts-highlight)
+
+</div>
+
+<div align='center'>
+
+![marigolds](assets/marigolds.webp)
+
+</div>
+
+### Usage
+
+```typescript
+import { higlight, type HighlightCSSClasses } from 'ts-higlight';
+
+// styles of generated code are extensible
+const cssClasses: HiglightCSSClasses = {
+    pre: 'pre-class second-pre-class', // classes can be divided as if they are in HTML `class` attribute
+    code: 'code-class',
+    line: 'Example-module-css-module__m3Y3uq__line', // classes can be CSS module strings
+
+    token: 'token-class', // `token` class is the class of every token of generated code
+
+    // Every token has its own class for fine grained configuration
+    operator: 'operator-class',
+
+    identifier: 'identifier',
+    keyword: 'keyword',
+
+    bigintChar: 'bigint-class', // class of bigint character (`10n` - `n` is `bigintChar`)
+    stringLiteral: 'string',
+
+    whitespace: 'whitespace', // even whitespace has its own class
+
+
+    ...
+};
+
+highlight(`let a = 'hello';`, cssClasses);
+```
+
+Real output will be:
+
+```html
+<pre class="pre-class second-pre-class">
+    <code class="code-class">
+        <div class="Example-module-css-module__m3Y3uq__line">
+            <span class="token-class keyword">let</span>
+            <span class="token-class whitespace"> </span>
+            <span class="token-class identifier">a</span>
+            <span class="token-class whitespace"> </span>
+            <span class="token-class operator-class">=</span>
+            <span class="token-class whitespace"> </span>
+            <span class="token-class string">'hello'</span>
+            <span class="token-class operator">;</span>
+        </div>
+    </code>
+</pre>
+```

package/dist/index.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Type of CSS classes object.
+ *
+ * Classes are used in generated HTML.
+ *
+ * CSS modules also can be used
+ *
+ *  
+ *
+ * Default generated HTML structure:
+ * ```html
+ * <pre>
+ *   <code>
+ *     <div> ... </div> <!-- line -->
+ *
+ *     ...
+ *   </code>
+ * </pre>
+ * ```
+ *
+ * @example
+ * const CSSClasses: HighlightCSSClasses = {
+ *    token: 'my-token-classname',
+ *    comment: 'my-comment-classname',
+ *    ...
+ * };
+ */
+type HighlightCSSClasses = Partial<{
+    /**
+     *
+     *
+     * `pre` class is used for root wrapper element
+     */
+    pre: string;
+    /**
+     * `code` class is used for generated code text element
+     */
+    code: string;
+    /**
+     * `line` class is used for every line of generated code
+     */
+    line: string;
+    /**
+     *
+     */
+    /**
+     * `token` class is used for every token of code, even whitespace
+     */
+    token: string;
+    whitespace: string;
+    comment: string;
+    keyword: string;
+    instruction: string;
+    operator: string;
+    constantIdentifier: string;
+    mutableIdentifier: string;
+    numberLiteral: string;
+    stringLiteral: string;
+    booleanLiteral: string;
+    NaNLiteral: string;
+    bigintChar: string;
+}>;
+
+/**
+ *
+ * Transforms received `source` string to HTML with higlighted source code.
+ *
+ * Classes could be divided with space as if they are in default HTML `class` attribute.
+ *
+ * @param {string} source JavaScript or TypeScript source code to highlight.
+ * @param {HiglightCSSClasses} cssClasses object with CSS classes.
+ *
+ * @returns {string} string with HTML of highlighted code.
+ *
+ * @example
+ * ```typescript
+ * const cssClasses: HighlightCSSClasses = {
+ *   pre: 'pre-element-class second-class',
+ *   code: 'my-code-element-class',
+ *
+ *   line: 'line-class my-line third-class',
+ *   token: 'token-class',
+ *
+ *   operator: 'operator-class',
+ *
+ *   keyword: 'keyword-class',
+ *   stringLiteral: 'string-class',
+ *
+ *   ...
+ * }
+ *
+ * highlight('let a = "hello";', cssClasses);
+ * ```
+ * Output will be:
+ * ```html
+ * <pre class="pre-element-class">
+ *   <code class="code-element-class">
+ *     <div class="line-class"> ... </div> <!-- code line with generated tokens inside -->
+ *   </code>
+ * </pre>
+ * ```
+ */
+declare const highlight: (source: string, cssClasses: HighlightCSSClasses) => string;
+
+export { highlight };
+export type { HighlightCSSClasses };

package/dist/index.js

@@ -0,0 +1 @@
+const e=new Set(["=","+","-","*","/","%","~","^",".",":","|","&","?","!","<",">","{","}","[","]","(",")",";",","]),t=new Set(["==","!=","<=",">=","++","--","**","*=","/=","%=","^=","&=","|=","&&","||","??"]),n=new Set(["===","!==","**=","<<=",">>=",">>>","&&=","||=","??="]),o={NaN:"NaNLiteral",abstract:"Keyword",as:"Instruction",assert:"Instruction",asserts:"Instruction",async:"Instruction",await:"Instruction",break:"Instruction",catch:"Instruction",class:"Keyword",const:"Keyword",continue:"Instruction",debugger:"Keyword",declare:"Keyword",default:"Instruction",delete:"Keyword",do:"Instruction",enum:"Keyword",export:"Instruction",false:"BooleanLiteral",finally:"Instruction",for:"Instruction",function:"Keyword",implements:"Keyword",import:"Instruction",in:"Keyword",instanceof:"Keyword",interface:"Keyword",is:"Instruction",keyof:"Keyword",let:"Keyword",new:"Keyword",package:"Instruction",this:"Keyword",throw:"Instruction",true:"BooleanLiteral",try:"Instruction",type:"Keyword",typeof:"Keyword",var:"Keyword",void:"Keyword",while:"Instruction",with:"Instruction",yield:"Instruction"},r=/^[a-zA-Zа-яА-Я_$]$/,i=/^[a-zA-Zа-яА-Я_$0-9]$/,s=/^[0-9]$/,a='<span class="',l="</span>",c=(c,u)=>{const y=(a=>{const l=[],c=a.length;let u=0;e:for(;u<c;){if(" "===a[u]||"\t"===a[u]){const e=u;for(u++;u<c&&(" "===a[u]||"\t"===a[u]);)u++;l[l.length]={type:"WhiteSpace",value:a.slice(e,u),start:e,end:u};continue e}if("\n"===a[u]||"\r"===a[u]){const e=u;"\r"===a[u]&&u++,u++,l[l.length]={type:"LineDivision",value:"\n",start:e,end:u};continue e}if(r.test(a[u])){const e=u;for(u++;u<c&&i.test(a[u]);)u++;const t=a.slice(e,u);l[l.length]={type:o[t]??"Identifier",value:t,start:e,end:u};continue e}if("'"===a[u]||'"'===a[u]||"`"===a[u]){const e=u,t=a[u];for(u++;u<c&&a[u]!==t;)u++;u++,l[l.length]={type:"StringLiteral",value:a.slice(e,u),start:e,end:u};continue e}if(s.test(a[u])){const e=u;for(;u<c&&s.test(a[u]);)u++;l[l.length]={type:"NumberLiteral",value:a.slice(e,u),start:e,end:u};continue e}if("/"===a[u]){const e=u;if(u++,"/"===a[u]){for(u++;u<c&&"\r"!==a[u]&&"\n"!==a[u];)u++;"\r"===a[u]&&(u+=2),l[l.length]={type:"Comment",value:a.slice(e,u),start:e,end:u},l[l.length]={type:"LineDivision",value:"\n",start:e,end:u};continue e}if("*"===a[u]){u++;let t=e;for(;u<c&&("*"!==a[u]||"/"!==a[u+1]);)"\n"!==a[u]&&"\r"!==a[u]||(l[l.length]={type:"Comment",value:a.slice(t,u),start:e,end:u},l[l.length]={type:"LineDivision",value:"\n",start:u,end:u+1},"\r"===a[u]&&u++,u++,t=u),u++;u+=2,l[l.length]={type:"Comment",value:a.slice(t,u),start:e,end:u};continue e}}if(a[u]+a[u+1]+a[u+2]+a[u+3]===">>>="){const e=u;u+=4,l[l.length]={type:"Operator",value:a.slice(e,u),start:e,end:u};continue e}if(n.has(a[u]+a[u+1]+a[u+2])){const e=u;u+=3,l[l.length]={type:"Operator",value:a.slice(e,u),start:e,end:u};continue e}if(t.has(a[u]+a[u+1])){const e=u;u+=2,l[l.length]={type:"Operator",value:a.slice(e,u),start:e,end:u};continue e}if(e.has(a[u])){const e=u;u++,l[l.length]={type:"Operator",value:a.slice(e,u),start:e,end:u};continue e}u++}return l})(c),d=((e,t)=>{const n=e.length;let o='<pre class="'+t.pre+'"><code class="'+t.code+'"><div class="'+t.line+'">',r=!0,i=0;for(;i<n;){const s=e[i];if("WhiteSpace"===s.type){o+=a+t.token+" "+t.whitespace+'">';let e=0;for(;e<s.value.length;)o+=" ",e++;o+=l,i++;continue}"LineDivision"!==s.type?"Operator"!==s.type?"Identifier"!==s.type?"Keyword"!==s.type?"Instruction"!==s.type?"StringLiteral"!==s.type?"NumberLiteral"!==s.type?"BooleanLiteral"!==s.type?"NaNLiteral"!==s.type?("Comment"!==s.type||(o+=a+t.token+" "+t.comment+'">'+s.value+l),i++):(o+=a+t.token+" "+t.NaNLiteral+'">'+s.value+l,i++):(o+=a+t.token+" "+t.booleanLiteral+'">'+s.value+l,i++):(o+=a+t.token+" "+t.numberLiteral+'">'+s.value+l,i++):(o+=a+t.token+" "+t.stringLiteral+'">'+s.value+l,i++):(o+=a+t.token+" "+t.instruction+">"+s.value+l,i++):(o+=a+t.token+" "+t.keyword+'">'+s.value+l,i++):(o+=a+t.token+" "+t.mutableIdentifier+'">'+s.value+l,i++):(o+=a+t.token+" "+t.operator+'">'+s.value+l,i++):(o+="</div>",i!==n-1&&(o+='<div class="'+t.line+'">',r=!1),i++)}return r&&(o+="</div>"),o+="</code></pre>",o})(y,u);return d};export{c as highlight};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "ts-highlight",
-    "version": "0.1.0",
+    "version": "0.3.0",
     "module": "./dist/index.js",
     "type": "module",
     "private": false,
@@ -11,7 +11,7 @@
     },
     "repository": {
         "type": "github",
-        "url": "https://github.com/a-marigold/es-higрlight"
+        "url": "https://github.com/a-marigold/ts-highlight"
     },
     "scripts": {
         "lint": "biome check",
@@ -26,6 +26,9 @@
         "higlight",
         "syntax-higlight"
     ],
+    "files": [
+        "dist"
+    ],
     "devDependencies": {
         "@biomejs/biome": "^2.3.10",
         "@rollup/plugin-terser": "^0.4.4",

package/.github/workflows/ci.yaml

@@ -1,29 +0,0 @@
-name: ci
-
-on:
-  push:
-    branches: '**'
-
-jobs:
-  ci:
-    name: ci
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: setup bun
-      uses: oven-sh/setup-bun@v2
-    - name: install dependencies
-      run: bun install
-
-    - name: run linter
-      run: bun run lint
-
-    - name: run tests
-      run: bun test
-
-    - name: build
-
-      run: bun run build

package/biome.json

@@ -1,30 +0,0 @@
-{
-    "$schema": "https://biomejs.dev/schemas/2.3.10/schema.json",
-    "vcs": {
-        "enabled": true,
-        "clientKind": "git"
-    },
-
-    "files": {
-        "ignoreUnknown": false,
-        "includes": ["*", "!node_modules", "!dist"]
-    },
-
-    "formatter": {
-        "enabled": false
-    },
-    "linter": {
-        "enabled": true,
-        "rules": {
-            "recommended": true
-        }
-    },
-    "assist": {
-        "enabled": true,
-        "actions": {
-            "source": {
-                "organizeImports": "off"
-            }
-        }
-    }
-}

package/rollup.config.js

@@ -1,34 +0,0 @@
-import { defineConfig } from 'rollup';
-
-import typescript from '@rollup/plugin-typescript';
-import dts from 'rollup-plugin-dts';
-
-import terser from '@rollup/plugin-terser';
-
-export default defineConfig([
-    {
-        input: './src/index.ts',
-
-        external: ['./src/__tests__/**'],
-
-        output: {
-            file: './dist/index.js',
-            format: 'esm',
-        },
-        plugins: [typescript({ exclude: ['**__tests__/**'] }), terser()],
-    },
-
-    {
-        input: './src/index.ts',
-
-        external: ['./src/__tests_/**'],
-
-        output: {
-            file: './dist/index.d.ts',
-
-            format: 'esm',
-        },
-
-        plugins: [dts()],
-    },
-]);

package/src/__tests__/tokenizer.test.ts

@@ -1,59 +0,0 @@
-import { describe, it, expect } from 'bun:test';
-
-import { tokenize } from '../tokenizer';
-
-describe('tokenizer', () => {
-    it('should return a valid tokens array', () => {
-        const source = `
-        let num = 10;
-
-        
-        console.log(num++);
-        `;
-
-        const tokens = tokenize(source);
-
-        console.log(tokens);
-        expect(tokens.length).toBe(25);
-        expect(
-            tokens.some(
-                (token) => token.type === 'Operator' && token.value === '++'
-            )
-        ).toBe(true);
-    });
-
-    it('should group `space` or `tab` order to one `WhiteSpace` token', () => {
-        const tabSource = `\t\t\t\t\t\t`;
-        const spaceSource = '      ';
-
-        const tabTokens = tokenize(tabSource);
-        const spaceTokens = tokenize(spaceSource);
-
-        expect(tabTokens.length).toBe(1);
-        expect(tabTokens[0].type).toBe('WhiteSpace');
-        expect(tabTokens[0].value.length).toBe(tabSource.length);
-
-        expect(spaceTokens.length).toBe(1);
-        expect(spaceTokens[0].type).toBe('WhiteSpace');
-        expect(spaceTokens[0].value.length).toBe(spaceSource.length);
-    });
-
-    it('should handle CRLF and LF line feeds to `LineDivision` token', () => {
-        const CRLFSource = '\r\n\r\n\r\n\r\n\r\n\r\n';
-        const LFSource = '\n\n\n\n\n\n';
-
-        const CRLFTokens = tokenize(CRLFSource);
-        const LFTokens = tokenize(LFSource);
-
-        expect(CRLFTokens.length).toBe(CRLFSource.length / 2);
-        for (let i = CRLFTokens.length - 1; i > 0; i--) {
-            expect(CRLFTokens[i].type).toBe('LineDivision');
-        }
-
-        expect(LFTokens.length).toBe(LFSource.length);
-
-        for (let i = 0; i < CRLFTokens.length; i++) {
-            expect(LFTokens[i].type).toBe('LineDivision');
-        }
-    });
-});

package/src/generator/constants.ts

@@ -1,25 +0,0 @@
-/**
- * String with opened `span` HTML element with opened `class` attribute.
- *
- * There is double quote `"` in `class` attribute
- *
- * @example
- * ```typescript
- * OPENED_SPAN_WITH_CLASS + 'text-class' + '">' + 'lorem ipsum' + '</span>';
- * // Output: '<span class="text-class">lorem ipsum</span>'
- * ```
- */
-export const OPENED_SPAN_WITH_CLASS = '<span class="';
-
-/**
- *
- *  String with closed `span` HTML element.
- *
- *
- * @example
- * ```typescript
- * '<span class="' + 'text-class' + '">' + 'dolor' + CLOSED_SPAN;
- * // Output: <span class="text-class">dolor</span>
- * ```
- */
-export const CLOSED_SPAN = '</span>';

package/src/generator/generate.ts

@@ -1,199 +0,0 @@
-import { OPENED_SPAN_WITH_CLASS, CLOSED_SPAN } from './constants';
-
-import type { Token } from '../tokenizer';
-import type { HighlightCSSClasses } from './types';
-
-export const generate = (
-    tokens: Token[],
-    cssClasses: HighlightCSSClasses
-): string => {
-    const tokensLength = tokens.length;
-
-    let generated: string =
-        '<pre class="' +
-        cssClasses.pre +
-        '"><code class="' +
-        cssClasses.code +
-        '"><div class="' +
-        cssClasses.line +
-        '">';
-
-    /**
-     *
-     * Boolean flag that shows is the current line opened.
-     * Needed for closing opened line in the end of loop.
-     */
-    let isLineOpened: boolean = true;
-
-    let tokenPos = 0;
-    while (tokenPos < tokensLength) {
-        const currentToken = tokens[tokenPos];
-
-        if (currentToken.type === 'WhiteSpace') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.whitespace +
-                '">';
-
-            let charPos = 0;
-            while (charPos < currentToken.value.length) {
-                generated += ' ';
-                charPos++;
-            }
-
-            generated += CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        if (currentToken.type === 'LineDivision') {
-            generated += '</div>';
-
-            if (tokenPos !== tokensLength - 1) {
-                generated += '<div class="' + cssClasses.line + '">';
-                isLineOpened = false;
-            }
-
-            tokenPos++;
-
-            continue;
-        }
-
-        if (currentToken.type === 'Operator') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.operator +
-                '">' +
-                currentToken.value +
-                CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        if (currentToken.type === 'Identifier') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.mutableIdentifier +
-                '">' +
-                currentToken.value +
-                CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        if (currentToken.type === 'Keyword') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.keyword +
-                '">' +
-                currentToken.value +
-                CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        // literals
-        if (currentToken.type === 'StringLiteral') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.stringLiteral +
-                '">' +
-                currentToken.value +
-                CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        if (currentToken.type === 'NumberLiteral') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.numberLiteral +
-                '">' +
-                currentToken.value +
-                CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        if (currentToken.type === 'BooleanLiteral') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.booleanLiteral +
-                '">' +
-                currentToken.value +
-                CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        if (currentToken.type === 'NaNLiteral') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.NaNLiteral +
-                '">' +
-                currentToken.value +
-                CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        // comments
-        if (currentToken.type === 'Comment') {
-            generated +=
-                OPENED_SPAN_WITH_CLASS +
-                cssClasses.token +
-                ' ' +
-                cssClasses.comment +
-                '">' +
-                currentToken.value +
-                CLOSED_SPAN;
-
-            tokenPos++;
-
-            continue;
-        }
-
-        // fallback
-        tokenPos++;
-    }
-
-    if (isLineOpened) {
-        generated += '</div>';
-    }
-
-    generated += '</code></pre>';
-
-    return generated;
-};

package/src/generator/index.ts

@@ -1,2 +0,0 @@
-export { generate } from './generate';
-export type { HighlightCSSClasses } from './types';

package/src/generator/types.ts

@@ -1,73 +0,0 @@
-/**
- * Type of CSS classes object.
- *
- * Classes are used in generated HTML.
- *
- * CSS modules also can be used
- *
- *  
- *
- * Default generated HTML structure:
- * ```html
- * <pre>
- *   <code>
- *     <div> ... </div> <!-- line -->
- *
- *     ...
- *   </code>
- * </pre>
- * ```
- *
- * @example
- * const CSSClasses: HighlightCSSClasses = {
- *    token: 'my-token-classname',
- *    comment: 'my-comment-classname',
- *    ...
- * };
- */
-export type HighlightCSSClasses = Partial<{
-    /**
-     *
-     *
-     * `pre` class is used for root wrapper element
-     */
-    pre: string;
-
-    /**
-     * `code` class is used for generated code text element
-     */
-    code: string;
-
-    /**
-     * `line` class is used for every line of generated code
-     */
-    line: string;
-
-    /**
-     *
-     */
-
-    /**
-     * `token` class is used for every token of code, even whitespace
-     */
-    token: string;
-
-    // trivia
-    whitespace: string;
-    comment: string;
-
-    keyword: string;
-
-    operator: string;
-
-    // identifiers
-    constantIdentifier: string;
-    mutableIdentifier: string;
-
-    // literals
-    numberLiteral: string;
-    stringLiteral: string;
-    booleanLiteral: string;
-    NaNLiteral: string;
-    bigintChar: string;
-}>;

package/src/highlight/highlight.ts

@@ -1,54 +0,0 @@
-import { tokenize } from '../tokenizer';
-import { generate } from '../generator';
-import type { HighlightCSSClasses } from '../generator';
-
-/**
- *
- * Transforms received `source` string to HTML with higlighted source code.
- *
- * Classes could be divided with space as if they are in default HTML `class` attribute.
- *
- * @param {string} source JavaScript or TypeScript source code to highlight.
- * @param {HiglightCSSClasses} cssClasses object with CSS classes.
- *
- * @returns {string} string with HTML of highlighted code.
- *
- * @example
- * ```typescript
- * const cssClasses: HighlightCSSClasses = {
- *   pre: 'pre-element-class second-class',
- *   code: 'my-code-element-class',
- *
- *   line: 'line-class my-line third-class',
- *   token: 'token-class',
- *
- *   operator: 'operator-class',
- *
- *   keyword: 'keyword-class',
- *   stringLiteral: 'string-class',
- *
- *   ...
- * }
- *
- * highlight('let a = "hello";', cssClasses);
- * ```
- * Output will be:
- * ```html
- * <pre class="pre-element-class">
- *   <code class="code-element-class">
- *     <div class="line-class"> ... </div> <!-- code line with generated tokens inside -->
- *   </code>
- * </pre>
- * ```
- */
-export const highlight = (
-    source: string,
-
-    cssClasses: HighlightCSSClasses
-): string => {
-    const tokens = tokenize(source);
-
-    const generated = generate(tokens, cssClasses);
-
-    return generated;
-};

package/src/highlight/index.ts

@@ -1 +0,0 @@
-export { highlight } from './highlight';

package/src/index.ts

@@ -1,2 +0,0 @@
-export { highlight } from './highlight';
-export type { HighlightCSSClasses } from './generator';

package/src/tokenizer/constants.ts

@@ -1,152 +0,0 @@
-import type {
-    TokenSet,
-    SingleOperators,
-    DoubleOperators,
-    TripleOperators,
-    QuadrupleOperator,
-    Keywords,
-} from './types';
-
-// operators
-
-const singleOperatorsInit: SingleOperators = [
-    '=',
-    '+',
-    '-',
-    '*',
-    '/',
-    '%',
-    '~',
-    '^',
-    '.',
-    ':',
-    '|',
-    '&',
-    '?',
-    '!',
-    '<',
-    '>',
-    '{',
-    '}',
-    '[',
-    ']',
-    '(',
-    ')',
-    ';',
-    ',',
-];
-const doubleOperatorsInit: DoubleOperators = [
-    '==',
-    '!=',
-
-    '<=',
-    '>=',
-
-    '++',
-    '--',
-    '**',
-
-    '*=',
-    '/=',
-    '%=',
-
-    '^=',
-    '&=',
-    '|=',
-
-    '&&',
-    '||',
-    '??',
-];
-const tripleOperatorsInit: TripleOperators = [
-    '===',
-    '!==',
-
-    '**=',
-    '<<=',
-    '>>=',
-    '>>>',
-    '&&=',
-    '||=',
-    '??=',
-];
-
-/**
- * `Set` with all javascript one-symbol operators
- *
- * @example '='
- */
-export const singleOperators: TokenSet = new Set(singleOperatorsInit);
-
-/**
- * `Set` with all javascript two-symbol operators
- *
- * @example '++'
- */
-export const doubleOperators: TokenSet = new Set(doubleOperatorsInit);
-
-/**
- * `Set` with all javascript three-symbol operators
- *
- * @example '>>>'
- */
-export const tripleOperators: TokenSet = new Set(tripleOperatorsInit);
-
-/**
- * The javascript four-symbol operator.
- *
- * Always equals '>>>=' on December 18, 2025 ECMAScript Specification
- */
-export const quadrupleOperator: QuadrupleOperator = '>>>=';
-
-// keywords
-
-const keywordsInit: Keywords = [
-    'var',
-    'let',
-    'const',
-    'typeof',
-    'class',
-
-    'in',
-
-    'new',
-
-    'instanceof',
-    'function',
-
-    'void',
-    'delete',
-    'keyof',
-    'abstract',
-    'interface',
-    'enum',
-    'type',
-    'implements',
-];
-
-/**
- * `Set` with javascript and typescript keywords
- */
-export const keywords: TokenSet = new Set(keywordsInit);
-
-// regular expresions (RegExp)
-
-/**
- * RegExp that is used to match javascript identifier start symbol
- *
- */
-
-export const IDENTIFIER_START_REGEXP: RegExp = /^[a-zA-Zа-яА-Я_$]$/;
-
-/**
- * RegExp that is used to match javascript identifier symbols after the first symbol
- */
-
-export const IDENTIFIER_REGEXP: RegExp = /^[a-zA-Zа-яА-Я_$0-9]$/;
-
-/**
- *
- * RegExp that is used to match any number
- */
-export const NUMBER_REGEXP: RegExp = /^[0-9]$/;

package/src/tokenizer/index.ts

@@ -1,2 +0,0 @@
-export type { TokenType, Token } from './types';
-export { tokenize } from './tokenize';

package/src/tokenizer/tokenize.ts

@@ -1,302 +0,0 @@
-import {
-    IDENTIFIER_START_REGEXP,
-    IDENTIFIER_REGEXP,
-    NUMBER_REGEXP,
-    singleOperators,
-    doubleOperators,
-    tripleOperators,
-    quadrupleOperator,
-    keywords,
-} from './constants';
-
-import type { Token } from './types';
-
-/**
- *
- * *Tokenizer* or *Lexer* function.
- *
- *
- * Divides `source` to tokens.
- *
- * @param {string} source - javascript or typescript source code to tokenize.
- *
- * @returns {Token[]} array with tokens from `source`.
- */
-export const tokenize = (source: string): Token[] => {
-    const tokens: Token[] = [];
-
-    const sourceLength = source.length;
-
-    let pos = 0;
-    main: while (pos < sourceLength) {
-        if (source[pos] === ' ' || source[pos] === '\t') {
-            const startPos = pos;
-
-            pos++;
-
-            while (
-                pos < sourceLength &&
-                (source[pos] === ' ' || source[pos] === '\t')
-            ) {
-                pos++;
-            }
-
-            tokens[tokens.length] = {
-                type: 'WhiteSpace',
-
-                value: source.slice(startPos, pos),
-
-                start: startPos,
-
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (source[pos] === '\n' || source[pos] === '\r') {
-            const startPos = pos;
-
-            if (source[pos] === '\r') {
-                pos++;
-            }
-
-            pos++;
-
-            tokens[tokens.length] = {
-                type: 'LineDivision',
-
-                value: '\n',
-
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        // literals
-        if (IDENTIFIER_START_REGEXP.test(source[pos])) {
-            const startPos = pos;
-            pos++;
-
-            while (pos < sourceLength && IDENTIFIER_REGEXP.test(source[pos])) {
-                pos++;
-            }
-
-            const identifier = source.slice(startPos, pos);
-
-            tokens[tokens.length] = {
-                type: keywords.has(identifier) ? 'Keyword' : 'Identifier',
-                value: identifier,
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (source[pos] === "'" || source[pos] === '"' || source[pos] === '`') {
-            const startPos = pos;
-            const startQuote = source[pos];
-
-            pos++;
-
-            while (pos < sourceLength && source[pos] !== startQuote) {
-                pos++;
-            }
-
-            pos++;
-
-            tokens[tokens.length] = {
-                type: 'StringLiteral',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-        if (NUMBER_REGEXP.test(source[pos])) {
-            const startPos = pos;
-
-            while (pos < sourceLength && NUMBER_REGEXP.test(source[pos])) {
-                pos++;
-            }
-
-            tokens[tokens.length] = {
-                type: 'NumberLiteral',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        // comments
-        if (source[pos] === '/') {
-            const startPos = pos;
-
-            pos++;
-
-            if (source[pos] === '/') {
-                pos++;
-
-                while (
-                    pos < sourceLength &&
-                    source[pos] !== '\r' &&
-                    source[pos] !== '\n'
-                ) {
-                    pos++;
-                }
-
-                if (source[pos] === '\r') {
-                    pos += 2;
-                }
-
-                tokens[tokens.length] = {
-                    type: 'Comment',
-
-                    value: source.slice(startPos, pos),
-                    start: startPos,
-                    end: pos,
-                };
-
-                tokens[tokens.length] = {
-                    type: 'LineDivision',
-
-                    value: '\n',
-
-                    start: startPos,
-                    end: pos,
-                };
-
-                continue main;
-            }
-
-            if (source[pos] === '*') {
-                pos++;
-
-                let lastCommentStart = startPos;
-
-                while (
-                    pos < sourceLength &&
-                    !(source[pos] === '*' && source[pos + 1] === '/')
-                ) {
-                    if (source[pos] === '\n' || source[pos] === '\r') {
-                        tokens[tokens.length] = {
-                            type: 'Comment',
-
-                            value: source.slice(lastCommentStart, pos),
-
-                            start: startPos,
-                            end: pos,
-                        };
-                        tokens[tokens.length] = {
-                            type: 'LineDivision',
-
-                            value: '\n',
-
-                            start: pos,
-                            end: pos + 1,
-                        };
-
-                        if (source[pos] === '\r') {
-                            pos++;
-                        }
-
-                        pos++;
-                        lastCommentStart = pos;
-                    }
-
-                    pos++;
-                }
-
-                pos += 2;
-
-                tokens[tokens.length] = {
-                    type: 'Comment',
-                    value: source.slice(lastCommentStart, pos),
-                    start: startPos,
-                    end: pos,
-                };
-
-                continue main;
-            }
-        }
-
-        // operators
-        if (
-            source[pos] +
-                source[pos + 1] +
-                source[pos + 2] +
-                source[pos + 3] ===
-            quadrupleOperator
-        ) {
-            const startPos = pos;
-
-            pos += 4;
-
-            tokens[tokens.length] = {
-                type: 'Operator',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (
-            tripleOperators.has(source[pos] + source[pos + 1] + source[pos + 2])
-        ) {
-            const startPos = pos;
-
-            pos += 3;
-
-            tokens[tokens.length] = {
-                type: 'Operator',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (doubleOperators.has(source[pos] + source[pos + 1])) {
-            const startPos = pos;
-
-            pos += 2;
-
-            tokens[tokens.length] = {
-                type: 'Operator',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (singleOperators.has(source[pos])) {
-            const startPos = pos;
-
-            pos++;
-
-            tokens[tokens.length] = {
-                type: 'Operator',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        pos++;
-    }
-
-    return tokens;
-};

package/src/tokenizer/types.ts

@@ -1,150 +0,0 @@
-type LiteralTokenType =
-    | 'NumberLiteral'
-    | 'StringLiteral'
-    | 'BooleanLiteral'
-    | 'NaNLiteral';
-
-/**
- * Variety of `Token` types
- */
-export type TokenType =
-    | 'Identifier'
-    | 'Keyword'
-    | 'Operator'
-    | 'WhiteSpace'
-    | 'Comment'
-    | 'LineDivision'
-    | LiteralTokenType;
-
-/**
- *
- */
-export type Token = {
-    type: TokenType;
-    value: string;
-
-    /**
-     * Start position of token value in source code
-     */
-    start: number;
-
-    /**
-     * The end of token value in source code
-     */
-    end: number;
-};
-
-export type SingleOperators = [
-    '=',
-    '+',
-    '-',
-    '*',
-    '/',
-    '%',
-    '~',
-    '^',
-    '.',
-    ':',
-    '|',
-    '&',
-    '?',
-    '!',
-    '<',
-    '>',
-
-    '{',
-    '}',
-    '[',
-    ']',
-    '(',
-    ')',
-
-    ';',
-    ','
-];
-export type SingleOperator = SingleOperators[number];
-
-export type DoubleOperators = [
-    '==',
-
-    '!=',
-    '<=',
-    '>=',
-    '++',
-    '--',
-    '**',
-    '*=',
-    '/=',
-    '%=',
-    '^=',
-    '&=',
-    '|=',
-
-    '&&',
-    '||',
-    '??'
-];
-export type DoubleOperator = DoubleOperators[number];
-
-export type TripleOperators = [
-    '===',
-    '!==',
-    '**=',
-
-    '<<=',
-    '>>=',
-    '>>>',
-
-    '&&=',
-    '||=',
-    '??='
-];
-export type TripleOperator = TripleOperators[number];
-
-export type QuadrupleOperator = '>>>=';
-
-export type JSKeywords = [
-    'var',
-    'let',
-
-    'const',
-
-    'typeof',
-
-    'class',
-    'in',
-
-    'new',
-    'instanceof',
-
-    'function',
-    'void',
-    'delete'
-];
-export type JSKeyword = JSKeywords[number];
-export type TSKeywords = [
-    'keyof',
-    'abstract',
-    'interface',
-    'enum',
-    'type',
-    'implements'
-];
-export type TSKeyword = TSKeywords[number];
-
-export type Keywords = [...JSKeywords, ...TSKeywords];
-export type Keyword = Keywords[number];
-
-/**
- *
- * Type that contains `TokenType` values to be checked in tokenizer.
- *
- *
- * @example
- * ```typescript
- * const singleOperatorsInit: SingleOperators = ['=', '+', '-' ...];
- *
- * const singleOperators: TokenSet = new Set(singleOperatorsInit);
- * ```
- */
-export type TokenSet = Set<string>;

package/tsconfig.json

@@ -1,28 +0,0 @@
-{
-    "compilerOptions": {
-        "lib": ["ESNext"],
-        "target": "ESNext",
-        "module": "esnext",
-        "moduleDetection": "force",
-
-        "moduleResolution": "bundler",
-
-        "verbatimModuleSyntax": true,
-
-        "noEmit": true,
-
-        "strict": true,
-        "skipLibCheck": true,
-        "noFallthroughCasesInSwitch": true,
-        "noUncheckedIndexedAccess": false,
-        "noPropertyAccessFromIndexSignature": true,
-        "noImplicitOverride": true,
-
-        "noUnusedLocals": true,
-        "noUnusedParameters": true,
-
-        "outDir": "dist"
-    },
-
-    "exclude": ["node_modules", "dist"]
-}