ts-highlight 0.1.0

package/.github/workflows/ci.yaml

@@ -0,0 +1,29 @@
+name: ci
+
+on:
+  push:
+    branches: '**'
+
+jobs:
+  ci:
+    name: ci
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: setup bun
+      uses: oven-sh/setup-bun@v2
+    - name: install dependencies
+      run: bun install
+
+    - name: run linter
+      run: bun run lint
+
+    - name: run tests
+      run: bun test
+
+    - name: build
+
+      run: bun run build

package/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>

package/README.md

@@ -0,0 +1 @@
+# <p align='center'> **es-highlight** </p>

package/biome.json

@@ -0,0 +1,30 @@
+{
+    "$schema": "https://biomejs.dev/schemas/2.3.10/schema.json",
+    "vcs": {
+        "enabled": true,
+        "clientKind": "git"
+    },
+
+    "files": {
+        "ignoreUnknown": false,
+        "includes": ["*", "!node_modules", "!dist"]
+    },
+
+    "formatter": {
+        "enabled": false
+    },
+    "linter": {
+        "enabled": true,
+        "rules": {
+            "recommended": true
+        }
+    },
+    "assist": {
+        "enabled": true,
+        "actions": {
+            "source": {
+                "organizeImports": "off"
+            }
+        }
+    }
+}

package/package.json

@@ -0,0 +1,41 @@
+{
+    "name": "ts-highlight",
+    "version": "0.1.0",
+    "module": "./dist/index.js",
+    "type": "module",
+    "private": false,
+    "license": "UNLICENSED",
+    "author": {
+        "name": "marigold",
+        "url": "https://github.com/a-marigold"
+    },
+    "repository": {
+        "type": "github",
+        "url": "https://github.com/a-marigold/es-higрlight"
+    },
+    "scripts": {
+        "lint": "biome check",
+        "build": "rollup -c",
+        "prepublishOnly": "bun run build",
+        "pshPub": "bun publish --access public"
+    },
+    "main": "./dist/index.js",
+    "types": "./dist/index.d.ts",
+    "keywords": [
+        "es-higlight",
+        "higlight",
+        "syntax-higlight"
+    ],
+    "devDependencies": {
+        "@biomejs/biome": "^2.3.10",
+        "@rollup/plugin-terser": "^0.4.4",
+        "@rollup/plugin-typescript": "^12.3.0",
+        "@types/bun": "latest",
+        "rollup": "^4.54.0",
+        "rollup-plugin-dts": "^6.3.0",
+        "tslib": "^2.8.1"
+    },
+    "peerDependencies": {
+        "typescript": "^5"
+    }
+}

package/rollup.config.js

@@ -0,0 +1,34 @@
+import { defineConfig } from 'rollup';
+
+import typescript from '@rollup/plugin-typescript';
+import dts from 'rollup-plugin-dts';
+
+import terser from '@rollup/plugin-terser';
+
+export default defineConfig([
+    {
+        input: './src/index.ts',
+
+        external: ['./src/__tests__/**'],
+
+        output: {
+            file: './dist/index.js',
+            format: 'esm',
+        },
+        plugins: [typescript({ exclude: ['**__tests__/**'] }), terser()],
+    },
+
+    {
+        input: './src/index.ts',
+
+        external: ['./src/__tests_/**'],
+
+        output: {
+            file: './dist/index.d.ts',
+
+            format: 'esm',
+        },
+
+        plugins: [dts()],
+    },
+]);

package/src/__tests__/tokenizer.test.ts

@@ -0,0 +1,59 @@
+import { describe, it, expect } from 'bun:test';
+
+import { tokenize } from '../tokenizer';
+
+describe('tokenizer', () => {
+    it('should return a valid tokens array', () => {
+        const source = `
+        let num = 10;
+
+        
+        console.log(num++);
+        `;
+
+        const tokens = tokenize(source);
+
+        console.log(tokens);
+        expect(tokens.length).toBe(25);
+        expect(
+            tokens.some(
+                (token) => token.type === 'Operator' && token.value === '++'
+            )
+        ).toBe(true);
+    });
+
+    it('should group `space` or `tab` order to one `WhiteSpace` token', () => {
+        const tabSource = `\t\t\t\t\t\t`;
+        const spaceSource = '      ';
+
+        const tabTokens = tokenize(tabSource);
+        const spaceTokens = tokenize(spaceSource);
+
+        expect(tabTokens.length).toBe(1);
+        expect(tabTokens[0].type).toBe('WhiteSpace');
+        expect(tabTokens[0].value.length).toBe(tabSource.length);
+
+        expect(spaceTokens.length).toBe(1);
+        expect(spaceTokens[0].type).toBe('WhiteSpace');
+        expect(spaceTokens[0].value.length).toBe(spaceSource.length);
+    });
+
+    it('should handle CRLF and LF line feeds to `LineDivision` token', () => {
+        const CRLFSource = '\r\n\r\n\r\n\r\n\r\n\r\n';
+        const LFSource = '\n\n\n\n\n\n';
+
+        const CRLFTokens = tokenize(CRLFSource);
+        const LFTokens = tokenize(LFSource);
+
+        expect(CRLFTokens.length).toBe(CRLFSource.length / 2);
+        for (let i = CRLFTokens.length - 1; i > 0; i--) {
+            expect(CRLFTokens[i].type).toBe('LineDivision');
+        }
+
+        expect(LFTokens.length).toBe(LFSource.length);
+
+        for (let i = 0; i < CRLFTokens.length; i++) {
+            expect(LFTokens[i].type).toBe('LineDivision');
+        }
+    });
+});

package/src/generator/constants.ts

@@ -0,0 +1,25 @@
+/**
+ * String with opened `span` HTML element with opened `class` attribute.
+ *
+ * There is double quote `"` in `class` attribute
+ *
+ * @example
+ * ```typescript
+ * OPENED_SPAN_WITH_CLASS + 'text-class' + '">' + 'lorem ipsum' + '</span>';
+ * // Output: '<span class="text-class">lorem ipsum</span>'
+ * ```
+ */
+export const OPENED_SPAN_WITH_CLASS = '<span class="';
+
+/**
+ *
+ *  String with closed `span` HTML element.
+ *
+ *
+ * @example
+ * ```typescript
+ * '<span class="' + 'text-class' + '">' + 'dolor' + CLOSED_SPAN;
+ * // Output: <span class="text-class">dolor</span>
+ * ```
+ */
+export const CLOSED_SPAN = '</span>';

package/src/generator/generate.ts

@@ -0,0 +1,199 @@
+import { OPENED_SPAN_WITH_CLASS, CLOSED_SPAN } from './constants';
+
+import type { Token } from '../tokenizer';
+import type { HighlightCSSClasses } from './types';
+
+export const generate = (
+    tokens: Token[],
+    cssClasses: HighlightCSSClasses
+): string => {
+    const tokensLength = tokens.length;
+
+    let generated: string =
+        '<pre class="' +
+        cssClasses.pre +
+        '"><code class="' +
+        cssClasses.code +
+        '"><div class="' +
+        cssClasses.line +
+        '">';
+
+    /**
+     *
+     * Boolean flag that shows is the current line opened.
+     * Needed for closing opened line in the end of loop.
+     */
+    let isLineOpened: boolean = true;
+
+    let tokenPos = 0;
+    while (tokenPos < tokensLength) {
+        const currentToken = tokens[tokenPos];
+
+        if (currentToken.type === 'WhiteSpace') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.whitespace +
+                '">';
+
+            let charPos = 0;
+            while (charPos < currentToken.value.length) {
+                generated += ' ';
+                charPos++;
+            }
+
+            generated += CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        if (currentToken.type === 'LineDivision') {
+            generated += '</div>';
+
+            if (tokenPos !== tokensLength - 1) {
+                generated += '<div class="' + cssClasses.line + '">';
+                isLineOpened = false;
+            }
+
+            tokenPos++;
+
+            continue;
+        }
+
+        if (currentToken.type === 'Operator') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.operator +
+                '">' +
+                currentToken.value +
+                CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        if (currentToken.type === 'Identifier') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.mutableIdentifier +
+                '">' +
+                currentToken.value +
+                CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        if (currentToken.type === 'Keyword') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.keyword +
+                '">' +
+                currentToken.value +
+                CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        // literals
+        if (currentToken.type === 'StringLiteral') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.stringLiteral +
+                '">' +
+                currentToken.value +
+                CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        if (currentToken.type === 'NumberLiteral') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.numberLiteral +
+                '">' +
+                currentToken.value +
+                CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        if (currentToken.type === 'BooleanLiteral') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.booleanLiteral +
+                '">' +
+                currentToken.value +
+                CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        if (currentToken.type === 'NaNLiteral') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.NaNLiteral +
+                '">' +
+                currentToken.value +
+                CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        // comments
+        if (currentToken.type === 'Comment') {
+            generated +=
+                OPENED_SPAN_WITH_CLASS +
+                cssClasses.token +
+                ' ' +
+                cssClasses.comment +
+                '">' +
+                currentToken.value +
+                CLOSED_SPAN;
+
+            tokenPos++;
+
+            continue;
+        }
+
+        // fallback
+        tokenPos++;
+    }
+
+    if (isLineOpened) {
+        generated += '</div>';
+    }
+
+    generated += '</code></pre>';
+
+    return generated;
+};

package/src/generator/index.ts

@@ -0,0 +1,2 @@
+export { generate } from './generate';
+export type { HighlightCSSClasses } from './types';

package/src/generator/types.ts

@@ -0,0 +1,73 @@
+/**
+ * Type of CSS classes object.
+ *
+ * Classes are used in generated HTML.
+ *
+ * CSS modules also can be used
+ *
+ *  
+ *
+ * Default generated HTML structure:
+ * ```html
+ * <pre>
+ *   <code>
+ *     <div> ... </div> <!-- line -->
+ *
+ *     ...
+ *   </code>
+ * </pre>
+ * ```
+ *
+ * @example
+ * const CSSClasses: HighlightCSSClasses = {
+ *    token: 'my-token-classname',
+ *    comment: 'my-comment-classname',
+ *    ...
+ * };
+ */
+export type HighlightCSSClasses = Partial<{
+    /**
+     *
+     *
+     * `pre` class is used for root wrapper element
+     */
+    pre: string;
+
+    /**
+     * `code` class is used for generated code text element
+     */
+    code: string;
+
+    /**
+     * `line` class is used for every line of generated code
+     */
+    line: string;
+
+    /**
+     *
+     */
+
+    /**
+     * `token` class is used for every token of code, even whitespace
+     */
+    token: string;
+
+    // trivia
+    whitespace: string;
+    comment: string;
+
+    keyword: string;
+
+    operator: string;
+
+    // identifiers
+    constantIdentifier: string;
+    mutableIdentifier: string;
+
+    // literals
+    numberLiteral: string;
+    stringLiteral: string;
+    booleanLiteral: string;
+    NaNLiteral: string;
+    bigintChar: string;
+}>;

package/src/highlight/highlight.ts

@@ -0,0 +1,54 @@
+import { tokenize } from '../tokenizer';
+import { generate } from '../generator';
+import type { HighlightCSSClasses } from '../generator';
+
+/**
+ *
+ * Transforms received `source` string to HTML with higlighted source code.
+ *
+ * Classes could be divided with space as if they are in default HTML `class` attribute.
+ *
+ * @param {string} source JavaScript or TypeScript source code to highlight.
+ * @param {HiglightCSSClasses} cssClasses object with CSS classes.
+ *
+ * @returns {string} string with HTML of highlighted code.
+ *
+ * @example
+ * ```typescript
+ * const cssClasses: HighlightCSSClasses = {
+ *   pre: 'pre-element-class second-class',
+ *   code: 'my-code-element-class',
+ *
+ *   line: 'line-class my-line third-class',
+ *   token: 'token-class',
+ *
+ *   operator: 'operator-class',
+ *
+ *   keyword: 'keyword-class',
+ *   stringLiteral: 'string-class',
+ *
+ *   ...
+ * }
+ *
+ * highlight('let a = "hello";', cssClasses);
+ * ```
+ * Output will be:
+ * ```html
+ * <pre class="pre-element-class">
+ *   <code class="code-element-class">
+ *     <div class="line-class"> ... </div> <!-- code line with generated tokens inside -->
+ *   </code>
+ * </pre>
+ * ```
+ */
+export const highlight = (
+    source: string,
+
+    cssClasses: HighlightCSSClasses
+): string => {
+    const tokens = tokenize(source);
+
+    const generated = generate(tokens, cssClasses);
+
+    return generated;
+};

package/src/highlight/index.ts

@@ -0,0 +1 @@
+export { highlight } from './highlight';

package/src/index.ts

@@ -0,0 +1,2 @@
+export { highlight } from './highlight';
+export type { HighlightCSSClasses } from './generator';

package/src/tokenizer/constants.ts

@@ -0,0 +1,152 @@
+import type {
+    TokenSet,
+    SingleOperators,
+    DoubleOperators,
+    TripleOperators,
+    QuadrupleOperator,
+    Keywords,
+} from './types';
+
+// operators
+
+const singleOperatorsInit: SingleOperators = [
+    '=',
+    '+',
+    '-',
+    '*',
+    '/',
+    '%',
+    '~',
+    '^',
+    '.',
+    ':',
+    '|',
+    '&',
+    '?',
+    '!',
+    '<',
+    '>',
+    '{',
+    '}',
+    '[',
+    ']',
+    '(',
+    ')',
+    ';',
+    ',',
+];
+const doubleOperatorsInit: DoubleOperators = [
+    '==',
+    '!=',
+
+    '<=',
+    '>=',
+
+    '++',
+    '--',
+    '**',
+
+    '*=',
+    '/=',
+    '%=',
+
+    '^=',
+    '&=',
+    '|=',
+
+    '&&',
+    '||',
+    '??',
+];
+const tripleOperatorsInit: TripleOperators = [
+    '===',
+    '!==',
+
+    '**=',
+    '<<=',
+    '>>=',
+    '>>>',
+    '&&=',
+    '||=',
+    '??=',
+];
+
+/**
+ * `Set` with all javascript one-symbol operators
+ *
+ * @example '='
+ */
+export const singleOperators: TokenSet = new Set(singleOperatorsInit);
+
+/**
+ * `Set` with all javascript two-symbol operators
+ *
+ * @example '++'
+ */
+export const doubleOperators: TokenSet = new Set(doubleOperatorsInit);
+
+/**
+ * `Set` with all javascript three-symbol operators
+ *
+ * @example '>>>'
+ */
+export const tripleOperators: TokenSet = new Set(tripleOperatorsInit);
+
+/**
+ * The javascript four-symbol operator.
+ *
+ * Always equals '>>>=' on December 18, 2025 ECMAScript Specification
+ */
+export const quadrupleOperator: QuadrupleOperator = '>>>=';
+
+// keywords
+
+const keywordsInit: Keywords = [
+    'var',
+    'let',
+    'const',
+    'typeof',
+    'class',
+
+    'in',
+
+    'new',
+
+    'instanceof',
+    'function',
+
+    'void',
+    'delete',
+    'keyof',
+    'abstract',
+    'interface',
+    'enum',
+    'type',
+    'implements',
+];
+
+/**
+ * `Set` with javascript and typescript keywords
+ */
+export const keywords: TokenSet = new Set(keywordsInit);
+
+// regular expresions (RegExp)
+
+/**
+ * RegExp that is used to match javascript identifier start symbol
+ *
+ */
+
+export const IDENTIFIER_START_REGEXP: RegExp = /^[a-zA-Zа-яА-Я_$]$/;
+
+/**
+ * RegExp that is used to match javascript identifier symbols after the first symbol
+ */
+
+export const IDENTIFIER_REGEXP: RegExp = /^[a-zA-Zа-яА-Я_$0-9]$/;
+
+/**
+ *
+ * RegExp that is used to match any number
+ */
+export const NUMBER_REGEXP: RegExp = /^[0-9]$/;

package/src/tokenizer/index.ts

@@ -0,0 +1,2 @@
+export type { TokenType, Token } from './types';
+export { tokenize } from './tokenize';

package/src/tokenizer/tokenize.ts

@@ -0,0 +1,302 @@
+import {
+    IDENTIFIER_START_REGEXP,
+    IDENTIFIER_REGEXP,
+    NUMBER_REGEXP,
+    singleOperators,
+    doubleOperators,
+    tripleOperators,
+    quadrupleOperator,
+    keywords,
+} from './constants';
+
+import type { Token } from './types';
+
+/**
+ *
+ * *Tokenizer* or *Lexer* function.
+ *
+ *
+ * Divides `source` to tokens.
+ *
+ * @param {string} source - javascript or typescript source code to tokenize.
+ *
+ * @returns {Token[]} array with tokens from `source`.
+ */
+export const tokenize = (source: string): Token[] => {
+    const tokens: Token[] = [];
+
+    const sourceLength = source.length;
+
+    let pos = 0;
+    main: while (pos < sourceLength) {
+        if (source[pos] === ' ' || source[pos] === '\t') {
+            const startPos = pos;
+
+            pos++;
+
+            while (
+                pos < sourceLength &&
+                (source[pos] === ' ' || source[pos] === '\t')
+            ) {
+                pos++;
+            }
+
+            tokens[tokens.length] = {
+                type: 'WhiteSpace',
+
+                value: source.slice(startPos, pos),
+
+                start: startPos,
+
+                end: pos,
+            };
+
+            continue main;
+        }
+
+        if (source[pos] === '\n' || source[pos] === '\r') {
+            const startPos = pos;
+
+            if (source[pos] === '\r') {
+                pos++;
+            }
+
+            pos++;
+
+            tokens[tokens.length] = {
+                type: 'LineDivision',
+
+                value: '\n',
+
+                start: startPos,
+                end: pos,
+            };
+
+            continue main;
+        }
+
+        // literals
+        if (IDENTIFIER_START_REGEXP.test(source[pos])) {
+            const startPos = pos;
+            pos++;
+
+            while (pos < sourceLength && IDENTIFIER_REGEXP.test(source[pos])) {
+                pos++;
+            }
+
+            const identifier = source.slice(startPos, pos);
+
+            tokens[tokens.length] = {
+                type: keywords.has(identifier) ? 'Keyword' : 'Identifier',
+                value: identifier,
+                start: startPos,
+                end: pos,
+            };
+
+            continue main;
+        }
+
+        if (source[pos] === "'" || source[pos] === '"' || source[pos] === '`') {
+            const startPos = pos;
+            const startQuote = source[pos];
+
+            pos++;
+
+            while (pos < sourceLength && source[pos] !== startQuote) {
+                pos++;
+            }
+
+            pos++;
+
+            tokens[tokens.length] = {
+                type: 'StringLiteral',
+                value: source.slice(startPos, pos),
+                start: startPos,
+                end: pos,
+            };
+
+            continue main;
+        }
+        if (NUMBER_REGEXP.test(source[pos])) {
+            const startPos = pos;
+
+            while (pos < sourceLength && NUMBER_REGEXP.test(source[pos])) {
+                pos++;
+            }
+
+            tokens[tokens.length] = {
+                type: 'NumberLiteral',
+                value: source.slice(startPos, pos),
+                start: startPos,
+                end: pos,
+            };
+
+            continue main;
+        }
+
+        // comments
+        if (source[pos] === '/') {
+            const startPos = pos;
+
+            pos++;
+
+            if (source[pos] === '/') {
+                pos++;
+
+                while (
+                    pos < sourceLength &&
+                    source[pos] !== '\r' &&
+                    source[pos] !== '\n'
+                ) {
+                    pos++;
+                }
+
+                if (source[pos] === '\r') {
+                    pos += 2;
+                }
+
+                tokens[tokens.length] = {
+                    type: 'Comment',
+
+                    value: source.slice(startPos, pos),
+                    start: startPos,
+                    end: pos,
+                };
+
+                tokens[tokens.length] = {
+                    type: 'LineDivision',
+
+                    value: '\n',
+
+                    start: startPos,
+                    end: pos,
+                };
+
+                continue main;
+            }
+
+            if (source[pos] === '*') {
+                pos++;
+
+                let lastCommentStart = startPos;
+
+                while (
+                    pos < sourceLength &&
+                    !(source[pos] === '*' && source[pos + 1] === '/')
+                ) {
+                    if (source[pos] === '\n' || source[pos] === '\r') {
+                        tokens[tokens.length] = {
+                            type: 'Comment',
+
+                            value: source.slice(lastCommentStart, pos),
+
+                            start: startPos,
+                            end: pos,
+                        };
+                        tokens[tokens.length] = {
+                            type: 'LineDivision',
+
+                            value: '\n',
+
+                            start: pos,
+                            end: pos + 1,
+                        };
+
+                        if (source[pos] === '\r') {
+                            pos++;
+                        }
+
+                        pos++;
+                        lastCommentStart = pos;
+                    }
+
+                    pos++;
+                }
+
+                pos += 2;
+
+                tokens[tokens.length] = {
+                    type: 'Comment',
+                    value: source.slice(lastCommentStart, pos),
+                    start: startPos,
+                    end: pos,
+                };
+
+                continue main;
+            }
+        }
+
+        // operators
+        if (
+            source[pos] +
+                source[pos + 1] +
+                source[pos + 2] +
+                source[pos + 3] ===
+            quadrupleOperator
+        ) {
+            const startPos = pos;
+
+            pos += 4;
+
+            tokens[tokens.length] = {
+                type: 'Operator',
+                value: source.slice(startPos, pos),
+                start: startPos,
+                end: pos,
+            };
+
+            continue main;
+        }
+
+        if (
+            tripleOperators.has(source[pos] + source[pos + 1] + source[pos + 2])
+        ) {
+            const startPos = pos;
+
+            pos += 3;
+
+            tokens[tokens.length] = {
+                type: 'Operator',
+                value: source.slice(startPos, pos),
+                start: startPos,
+                end: pos,
+            };
+
+            continue main;
+        }
+
+        if (doubleOperators.has(source[pos] + source[pos + 1])) {
+            const startPos = pos;
+
+            pos += 2;
+
+            tokens[tokens.length] = {
+                type: 'Operator',
+                value: source.slice(startPos, pos),
+                start: startPos,
+                end: pos,
+            };
+
+            continue main;
+        }
+
+        if (singleOperators.has(source[pos])) {
+            const startPos = pos;
+
+            pos++;
+
+            tokens[tokens.length] = {
+                type: 'Operator',
+                value: source.slice(startPos, pos),
+                start: startPos,
+                end: pos,
+            };
+
+            continue main;
+        }
+
+        pos++;
+    }
+
+    return tokens;
+};

package/src/tokenizer/types.ts

@@ -0,0 +1,150 @@
+type LiteralTokenType =
+    | 'NumberLiteral'
+    | 'StringLiteral'
+    | 'BooleanLiteral'
+    | 'NaNLiteral';
+
+/**
+ * Variety of `Token` types
+ */
+export type TokenType =
+    | 'Identifier'
+    | 'Keyword'
+    | 'Operator'
+    | 'WhiteSpace'
+    | 'Comment'
+    | 'LineDivision'
+    | LiteralTokenType;
+
+/**
+ *
+ */
+export type Token = {
+    type: TokenType;
+    value: string;
+
+    /**
+     * Start position of token value in source code
+     */
+    start: number;
+
+    /**
+     * The end of token value in source code
+     */
+    end: number;
+};
+
+export type SingleOperators = [
+    '=',
+    '+',
+    '-',
+    '*',
+    '/',
+    '%',
+    '~',
+    '^',
+    '.',
+    ':',
+    '|',
+    '&',
+    '?',
+    '!',
+    '<',
+    '>',
+
+    '{',
+    '}',
+    '[',
+    ']',
+    '(',
+    ')',
+
+    ';',
+    ','
+];
+export type SingleOperator = SingleOperators[number];
+
+export type DoubleOperators = [
+    '==',
+
+    '!=',
+    '<=',
+    '>=',
+    '++',
+    '--',
+    '**',
+    '*=',
+    '/=',
+    '%=',
+    '^=',
+    '&=',
+    '|=',
+
+    '&&',
+    '||',
+    '??'
+];
+export type DoubleOperator = DoubleOperators[number];
+
+export type TripleOperators = [
+    '===',
+    '!==',
+    '**=',
+
+    '<<=',
+    '>>=',
+    '>>>',
+
+    '&&=',
+    '||=',
+    '??='
+];
+export type TripleOperator = TripleOperators[number];
+
+export type QuadrupleOperator = '>>>=';
+
+export type JSKeywords = [
+    'var',
+    'let',
+
+    'const',
+
+    'typeof',
+
+    'class',
+    'in',
+
+    'new',
+    'instanceof',
+
+    'function',
+    'void',
+    'delete'
+];
+export type JSKeyword = JSKeywords[number];
+export type TSKeywords = [
+    'keyof',
+    'abstract',
+    'interface',
+    'enum',
+    'type',
+    'implements'
+];
+export type TSKeyword = TSKeywords[number];
+
+export type Keywords = [...JSKeywords, ...TSKeywords];
+export type Keyword = Keywords[number];
+
+/**
+ *
+ * Type that contains `TokenType` values to be checked in tokenizer.
+ *
+ *
+ * @example
+ * ```typescript
+ * const singleOperatorsInit: SingleOperators = ['=', '+', '-' ...];
+ *
+ * const singleOperators: TokenSet = new Set(singleOperatorsInit);
+ * ```
+ */
+export type TokenSet = Set<string>;

package/tsconfig.json

@@ -0,0 +1,28 @@
+{
+    "compilerOptions": {
+        "lib": ["ESNext"],
+        "target": "ESNext",
+        "module": "esnext",
+        "moduleDetection": "force",
+
+        "moduleResolution": "bundler",
+
+        "verbatimModuleSyntax": true,
+
+        "noEmit": true,
+
+        "strict": true,
+        "skipLibCheck": true,
+        "noFallthroughCasesInSwitch": true,
+        "noUncheckedIndexedAccess": false,
+        "noPropertyAccessFromIndexSignature": true,
+        "noImplicitOverride": true,
+
+        "noUnusedLocals": true,
+        "noUnusedParameters": true,
+
+        "outDir": "dist"
+    },
+
+    "exclude": ["node_modules", "dist"]
+}