ts-highlight 0.2.0 → 0.4.0

package/src/tokenizer/constants.ts

@@ -1,188 +0,0 @@
-import type {
-    TokenSet,
-    SingleOperators,
-    DoubleOperators,
-    TripleOperators,
-    QuadrupleOperator,
-    IdentifierLikeMap,
-} from './types';
-
-// operators
-
-const singleOperatorsInit: SingleOperators = [
-    '=',
-    '+',
-    '-',
-    '*',
-    '/',
-    '%',
-    '~',
-    '^',
-    '.',
-    ':',
-    '|',
-    '&',
-    '?',
-    '!',
-    '<',
-    '>',
-    '{',
-    '}',
-    '[',
-    ']',
-    '(',
-    ')',
-    ';',
-    ',',
-];
-const doubleOperatorsInit: DoubleOperators = [
-    '==',
-    '!=',
-
-    '<=',
-    '>=',
-
-    '++',
-    '--',
-    '**',
-
-    '*=',
-    '/=',
-    '%=',
-
-    '^=',
-    '&=',
-    '|=',
-
-    '&&',
-    '||',
-    '??',
-];
-const tripleOperatorsInit: TripleOperators = [
-    '===',
-    '!==',
-
-    '**=',
-    '<<=',
-    '>>=',
-    '>>>',
-    '&&=',
-    '||=',
-    '??=',
-];
-
-/**
- * `Set` with all javascript one-symbol operators
- *
- * @example '='
- */
-export const singleOperators: TokenSet = new Set(singleOperatorsInit);
-
-/**
- * `Set` with all javascript two-symbol operators
- *
- * @example '++'
- */
-export const doubleOperators: TokenSet = new Set(doubleOperatorsInit);
-
-/**
- * `Set` with all javascript three-symbol operators
- *
- * @example '>>>'
- */
-export const tripleOperators: TokenSet = new Set(tripleOperatorsInit);
-
-/**
- * The javascript four-symbol operator.
- *
- * Always equals '>>>=' on December 18, 2025 ECMAScript Specification
- */
-export const quadrupleOperator: QuadrupleOperator = '>>>=';
-
-// identifier like token types
-/**
- * Object with Identifier like literals and their TokenType.
- *
- * Used to determine correct TokenType.
- *
- *
- *
- * @example
- * ```typescript
- * const unknownIdentifier = 'function';
- *
- * const tokenTypeOfUnknownIdentifier = identifierLikeMap[unknownIdentifier as IdentifierLike];
- *
- * // Output:
- *
- * 'Keyword';
- *
- * ```
- */
-export const identifierLikeMap: IdentifierLikeMap = {
-    NaN: 'NaNLiteral',
-    abstract: 'Keyword',
-    as: 'Instruction',
-    assert: 'Instruction',
-    asserts: 'Instruction',
-    async: 'Instruction',
-    await: 'Instruction',
-    break: 'Instruction',
-    catch: 'Instruction',
-    class: 'Keyword',
-    const: 'Keyword',
-    continue: 'Instruction',
-    debugger: 'Keyword',
-    declare: 'Keyword',
-    default: 'Instruction',
-    delete: 'Keyword',
-    do: 'Instruction',
-    enum: 'Keyword',
-    export: 'Instruction',
-    false: 'BooleanLiteral',
-    finally: 'Instruction',
-    for: 'Instruction',
-    function: 'Keyword',
-    implements: 'Keyword',
-    import: 'Instruction',
-    in: 'Keyword',
-    instanceof: 'Keyword',
-    interface: 'Keyword',
-    is: 'Instruction',
-    keyof: 'Keyword',
-    let: 'Keyword',
-    new: 'Keyword',
-    package: 'Instruction',
-    this: 'Keyword',
-    throw: 'Instruction',
-    true: 'BooleanLiteral',
-    try: 'Instruction',
-    type: 'Keyword',
-    typeof: 'Keyword',
-    var: 'Keyword',
-    void: 'Keyword',
-    while: 'Instruction',
-    with: 'Instruction',
-    yield: 'Instruction',
-};
-
-// regular expresions (RegExp)
-
-/**
- * RegExp that is used to match javascript identifier start symbol
- *
- */
-
-export const IDENTIFIER_START_REGEXP: RegExp = /^[a-zA-Zа-яА-Я_$]$/;
-
-/**
- * RegExp that is used to match javascript identifier symbols after the first symbol
- */
-
-export const IDENTIFIER_REGEXP: RegExp = /^[a-zA-Zа-яА-Я_$0-9]$/;
-
-/**
- *
- * RegExp that is used to match any number
- */
-export const NUMBER_REGEXP: RegExp = /^[0-9]$/;

package/src/tokenizer/index.ts

@@ -1,2 +0,0 @@
-export type { TokenType, Token } from './types';
-export { tokenize } from './tokenize';

package/src/tokenizer/tokenize.ts

@@ -1,304 +0,0 @@
-import {
-    IDENTIFIER_START_REGEXP,
-    IDENTIFIER_REGEXP,
-    NUMBER_REGEXP,
-    singleOperators,
-    doubleOperators,
-    tripleOperators,
-    quadrupleOperator,
-    identifierLikeMap,
-} from './constants';
-
-import type { Token, IdentifierLike } from './types';
-
-/**
- *
- * *Tokenizer* or *Lexer* function.
- *
- *
- * Divides `source` to tokens.
- *
- * @param {string} source - javascript or typescript source code to tokenize.
- *
- * @returns {Token[]} array with tokens from `source`.
- */
-export const tokenize = (source: string): Token[] => {
-    const tokens: Token[] = [];
-
-    const sourceLength = source.length;
-
-    let pos = 0;
-    main: while (pos < sourceLength) {
-        if (source[pos] === ' ' || source[pos] === '\t') {
-            const startPos = pos;
-
-            pos++;
-
-            while (
-                pos < sourceLength &&
-                (source[pos] === ' ' || source[pos] === '\t')
-            ) {
-                pos++;
-            }
-
-            tokens[tokens.length] = {
-                type: 'WhiteSpace',
-
-                value: source.slice(startPos, pos),
-
-                start: startPos,
-
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (source[pos] === '\n' || source[pos] === '\r') {
-            const startPos = pos;
-
-            if (source[pos] === '\r') {
-                pos++;
-            }
-
-            pos++;
-
-            tokens[tokens.length] = {
-                type: 'LineDivision',
-
-                value: '\n',
-
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        // literals
-        if (IDENTIFIER_START_REGEXP.test(source[pos])) {
-            const startPos = pos;
-            pos++;
-
-            while (pos < sourceLength && IDENTIFIER_REGEXP.test(source[pos])) {
-                pos++;
-            }
-
-            const identifier = source.slice(startPos, pos);
-
-            tokens[tokens.length] = {
-                type:
-                    identifierLikeMap[identifier as IdentifierLike] ??
-                    'Identifier',
-                value: identifier,
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (source[pos] === "'" || source[pos] === '"' || source[pos] === '`') {
-            const startPos = pos;
-            const startQuote = source[pos];
-
-            pos++;
-
-            while (pos < sourceLength && source[pos] !== startQuote) {
-                pos++;
-            }
-
-            pos++;
-
-            tokens[tokens.length] = {
-                type: 'StringLiteral',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-        if (NUMBER_REGEXP.test(source[pos])) {
-            const startPos = pos;
-
-            while (pos < sourceLength && NUMBER_REGEXP.test(source[pos])) {
-                pos++;
-            }
-
-            tokens[tokens.length] = {
-                type: 'NumberLiteral',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        // comments
-        if (source[pos] === '/') {
-            const startPos = pos;
-
-            pos++;
-
-            if (source[pos] === '/') {
-                pos++;
-
-                while (
-                    pos < sourceLength &&
-                    source[pos] !== '\r' &&
-                    source[pos] !== '\n'
-                ) {
-                    pos++;
-                }
-
-                if (source[pos] === '\r') {
-                    pos += 2;
-                }
-
-                tokens[tokens.length] = {
-                    type: 'Comment',
-
-                    value: source.slice(startPos, pos),
-                    start: startPos,
-                    end: pos,
-                };
-
-                tokens[tokens.length] = {
-                    type: 'LineDivision',
-
-                    value: '\n',
-
-                    start: startPos,
-                    end: pos,
-                };
-
-                continue main;
-            }
-
-            if (source[pos] === '*') {
-                pos++;
-
-                let lastCommentStart = startPos;
-
-                while (
-                    pos < sourceLength &&
-                    !(source[pos] === '*' && source[pos + 1] === '/')
-                ) {
-                    if (source[pos] === '\n' || source[pos] === '\r') {
-                        tokens[tokens.length] = {
-                            type: 'Comment',
-
-                            value: source.slice(lastCommentStart, pos),
-
-                            start: startPos,
-                            end: pos,
-                        };
-                        tokens[tokens.length] = {
-                            type: 'LineDivision',
-
-                            value: '\n',
-
-                            start: pos,
-                            end: pos + 1,
-                        };
-
-                        if (source[pos] === '\r') {
-                            pos++;
-                        }
-
-                        pos++;
-                        lastCommentStart = pos;
-                    }
-
-                    pos++;
-                }
-
-                pos += 2;
-
-                tokens[tokens.length] = {
-                    type: 'Comment',
-                    value: source.slice(lastCommentStart, pos),
-                    start: startPos,
-                    end: pos,
-                };
-
-                continue main;
-            }
-        }
-
-        // operators
-        if (
-            source[pos] +
-                source[pos + 1] +
-                source[pos + 2] +
-                source[pos + 3] ===
-            quadrupleOperator
-        ) {
-            const startPos = pos;
-
-            pos += 4;
-
-            tokens[tokens.length] = {
-                type: 'Operator',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (
-            tripleOperators.has(source[pos] + source[pos + 1] + source[pos + 2])
-        ) {
-            const startPos = pos;
-
-            pos += 3;
-
-            tokens[tokens.length] = {
-                type: 'Operator',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (doubleOperators.has(source[pos] + source[pos + 1])) {
-            const startPos = pos;
-
-            pos += 2;
-
-            tokens[tokens.length] = {
-                type: 'Operator',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        if (singleOperators.has(source[pos])) {
-            const startPos = pos;
-
-            pos++;
-
-            tokens[tokens.length] = {
-                type: 'Operator',
-                value: source.slice(startPos, pos),
-                start: startPos,
-                end: pos,
-            };
-
-            continue main;
-        }
-
-        pos++;
-    }
-
-    return tokens;
-};

package/src/tokenizer/types.ts

@@ -1,197 +0,0 @@
-type LiteralTokenType =
-    | 'NumberLiteral'
-    | 'StringLiteral'
-    | 'BooleanLiteral'
-    | 'NaNLiteral';
-
-/**
- * Variety of `Token` types
- */
-export type TokenType =
-    | 'Identifier'
-    | 'Keyword'
-    | 'Operator'
-    | 'WhiteSpace'
-    | 'Comment'
-    | 'LineDivision'
-    | 'Instruction'
-    | LiteralTokenType;
-
-/**
- *
- */
-export type Token = {
-    type: TokenType;
-    value: string;
-
-    /**
-     * Start position of token value in source code
-     */
-    start: number;
-
-    /**
-     * The end of token value in source code
-     */
-    end: number;
-};
-
-export type SingleOperators = [
-    '=',
-    '+',
-    '-',
-    '*',
-    '/',
-    '%',
-    '~',
-    '^',
-    '.',
-    ':',
-    '|',
-    '&',
-    '?',
-    '!',
-    '<',
-    '>',
-
-    '{',
-    '}',
-    '[',
-    ']',
-    '(',
-    ')',
-
-    ';',
-    ','
-];
-export type SingleOperator = SingleOperators[number];
-
-export type DoubleOperators = [
-    '==',
-
-    '!=',
-    '<=',
-    '>=',
-    '++',
-    '--',
-    '**',
-    '*=',
-    '/=',
-    '%=',
-    '^=',
-    '&=',
-    '|=',
-
-    '&&',
-    '||',
-    '??'
-];
-export type DoubleOperator = DoubleOperators[number];
-
-export type TripleOperators = [
-    '===',
-    '!==',
-    '**=',
-
-    '<<=',
-    '>>=',
-    '>>>',
-
-    '&&=',
-    '||=',
-    '??='
-];
-export type TripleOperator = TripleOperators[number];
-
-export type QuadrupleOperator = '>>>=';
-
-export type JSKeywords = [
-    'var',
-    'let',
-
-    'const',
-
-    'typeof',
-
-    'class',
-    'in',
-
-    'new',
-    'instanceof',
-
-    'function',
-    'void',
-    'delete',
-
-    'this',
-
-    'debugger'
-];
-export type JSKeyword = JSKeywords[number];
-export type TSKeywords = [
-    'keyof',
-    'abstract',
-    'interface',
-    'enum',
-    'type',
-    'implements',
-    'declare'
-];
-export type TSKeyword = TSKeywords[number];
-
-export type Keywords = [...JSKeywords, ...TSKeywords];
-export type Keyword = Keywords[number];
-
-type JSInstruction =
-    | 'for'
-    | 'do'
-    | 'while'
-    | 'continue'
-    | 'break'
-    | 'import'
-    | 'export'
-    | 'package'
-    | 'try'
-    | 'catch'
-    | 'finally'
-    | 'async'
-    | 'await'
-    | 'yield'
-    | 'with'
-    | 'assert'
-    | 'default'
-    | 'throw';
-
-type TSInstruction = 'as' | 'asserts' | 'is';
-export type Instruction = JSInstruction | TSInstruction;
-
-type Literal = 'true' | 'false' | 'NaN';
-
-/**
- * Token Types that are like `Identifier` TokenType
- */
-export type IdentifierLike = Keyword | Instruction | Literal;
-
-/**
- * Record with Token Types that are like `Identifier` TokenType.
- * Used to determine correct TokenType
- */
-export type IdentifierLikeMap = {
-    [K in IdentifierLike]: Extract<
-        TokenType,
-        'Keyword' | 'Instruction' | 'BooleanLiteral' | 'NaNLiteral'
-    >;
-};
-
-/**
- *
- * Type that contains `TokenType` values to be checked in tokenizer.
- *
- *
- * @example
- * ```typescript
- * const singleOperatorsInit: SingleOperators = ['=', '+', '-' ...];
- *
- * const singleOperators: TokenSet = new Set(singleOperatorsInit);
- * ```
- */
-export type TokenSet = Set<string>;

package/tsconfig.json

@@ -1,28 +0,0 @@
-{
-    "compilerOptions": {
-        "lib": ["ESNext"],
-        "target": "ESNext",
-        "module": "esnext",
-        "moduleDetection": "force",
-
-        "moduleResolution": "bundler",
-
-        "verbatimModuleSyntax": true,
-
-        "noEmit": true,
-
-        "strict": true,
-        "skipLibCheck": true,
-        "noFallthroughCasesInSwitch": true,
-        "noUncheckedIndexedAccess": false,
-        "noPropertyAccessFromIndexSignature": true,
-        "noImplicitOverride": true,
-
-        "noUnusedLocals": true,
-        "noUnusedParameters": true,
-
-        "outDir": "dist"
-    },
-
-    "exclude": ["node_modules", "dist"]
-}