reqon-dsl 0.2.0 → 0.3.0

package/dist/lexer/lexer.js

@@ -1,264 +0,0 @@
-import { TokenType } from 'vague-lang';
-import { REQON_KEYWORDS, ReqonTokenType } from './tokens.js';
-import { LexerError } from '../errors/index.js';
-export class ReqonLexer {
-    source;
-    pos = 0;
-    line = 1;
-    column = 1;
-    constructor(source) {
-        this.source = source;
-    }
-    tokenize() {
-        const tokens = [];
-        while (!this.isAtEnd()) {
-            const token = this.nextToken();
-            if (token) {
-                tokens.push(token);
-            }
-        }
-        tokens.push(this.makeToken(TokenType.EOF, ''));
-        return tokens;
-    }
-    nextToken() {
-        this.skipWhitespace();
-        if (this.isAtEnd())
-            return null;
-        const char = this.peek();
-        // Skip comments
-        if (char === '/' && this.peekNext() === '/') {
-            this.skipLineComment();
-            return null;
-        }
-        // Newlines (significant for statement termination)
-        if (char === '\n') {
-            const token = this.makeToken(TokenType.NEWLINE, '\n');
-            this.advance();
-            this.line++;
-            this.column = 1;
-            return token;
-        }
-        // String literals
-        if (char === '"')
-            return this.readString();
-        // Numbers
-        if (this.isDigit(char))
-            return this.readNumber();
-        // Identifiers and keywords
-        if (this.isAlpha(char))
-            return this.readIdentifier();
-        // Operators and delimiters
-        return this.readOperator();
-    }
-    readString() {
-        const startColumn = this.column;
-        this.advance(); // consume opening quote
-        let value = '';
-        while (!this.isAtEnd() && this.peek() !== '"') {
-            if (this.peek() === '\\') {
-                this.advance();
-                const escaped = this.advance();
-                switch (escaped) {
-                    case 'n':
-                        value += '\n';
-                        break;
-                    case 't':
-                        value += '\t';
-                        break;
-                    case '\\':
-                        value += '\\';
-                        break;
-                    case '"':
-                        value += '"';
-                        break;
-                    default:
-                        value += escaped;
-                }
-            }
-            else {
-                value += this.advance();
-            }
-        }
-        if (this.isAtEnd()) {
-            throw new LexerError('Unterminated string', { line: this.line, column: startColumn }, { source: this.source });
-        }
-        this.advance(); // consume closing quote
-        return {
-            type: TokenType.STRING,
-            value,
-            line: this.line,
-            column: startColumn,
-        };
-    }
-    readNumber() {
-        const startColumn = this.column;
-        let value = '';
-        while (this.isDigit(this.peek())) {
-            value += this.advance();
-        }
-        // Handle decimals
-        if (this.peek() === '.' && this.isDigit(this.peekNext())) {
-            value += this.advance(); // consume '.'
-            while (this.isDigit(this.peek())) {
-                value += this.advance();
-            }
-        }
-        // Handle underscores in numbers (e.g., 100_000)
-        value = value.replace(/_/g, '');
-        return {
-            type: TokenType.NUMBER,
-            value,
-            line: this.line,
-            column: startColumn,
-        };
-    }
-    readIdentifier() {
-        const startColumn = this.column;
-        let value = '';
-        while (this.isAlphaNumeric(this.peek()) || this.peek() === '_') {
-            value += this.advance();
-        }
-        // Check Reqon keywords first, then fall back to Vague keywords
-        const reqonType = REQON_KEYWORDS[value];
-        if (reqonType) {
-            return { type: reqonType, value, line: this.line, column: startColumn };
-        }
-        // Use Vague's lexer to check for Vague keywords
-        const vagueKeywords = {
-            let: TokenType.LET,
-            schema: TokenType.SCHEMA,
-            from: TokenType.FROM,
-            import: TokenType.IMPORT,
-            context: TokenType.CONTEXT,
-            distribution: TokenType.DISTRIBUTION,
-            dataset: TokenType.DATASET,
-            with: TokenType.WITH,
-            where: TokenType.WHERE,
-            when: TokenType.WHEN,
-            in: TokenType.IN,
-            per: TokenType.PER,
-            any: TokenType.ANY,
-            of: TokenType.OF,
-            match: TokenType.MATCH,
-            affects: TokenType.AFFECTS,
-            constraints: TokenType.CONSTRAINTS,
-            validate: TokenType.VALIDATE,
-            assume: TokenType.ASSUME,
-            then: TokenType.THEN,
-            refine: TokenType.REFINE,
-            if: TokenType.IF,
-            violating: TokenType.VIOLATING,
-            unique: TokenType.UNIQUE,
-            private: TokenType.PRIVATE,
-            and: TokenType.AND,
-            or: TokenType.OR,
-            not: TokenType.NOT,
-            null: TokenType.NULL,
-            true: TokenType.TRUE,
-            false: TokenType.FALSE,
-            int: TokenType.INT,
-            decimal: TokenType.DECIMAL,
-            date: TokenType.DATE,
-        };
-        const type = vagueKeywords[value] ?? TokenType.IDENTIFIER;
-        return { type, value, line: this.line, column: startColumn };
-    }
-    readOperator() {
-        const startColumn = this.column;
-        const char = this.advance();
-        // Check for -> (Reqon-specific)
-        if (char === '-' && this.peek() === '>') {
-            this.advance();
-            return {
-                type: ReqonTokenType.RIGHT_ARROW,
-                value: '->',
-                line: this.line,
-                column: startColumn,
-            };
-        }
-        const twoChar = char + this.peek();
-        const twoCharOps = {
-            '..': TokenType.DOTDOT,
-            '=>': TokenType.ARROW,
-            '==': TokenType.DOUBLE_EQUALS,
-            '+=': TokenType.PLUS_EQUALS,
-            '<=': TokenType.LTE,
-            '>=': TokenType.GTE,
-        };
-        if (twoCharOps[twoChar]) {
-            this.advance();
-            return { type: twoCharOps[twoChar], value: twoChar, line: this.line, column: startColumn };
-        }
-        const singleCharOps = {
-            '|': TokenType.PIPE,
-            '~': TokenType.TILDE,
-            ':': TokenType.COLON,
-            '=': TokenType.EQUALS,
-            '+': TokenType.PLUS,
-            '-': TokenType.MINUS,
-            '*': TokenType.STAR,
-            '/': TokenType.SLASH,
-            '.': TokenType.DOT,
-            '^': TokenType.CARET,
-            '%': TokenType.PERCENT,
-            '<': TokenType.LT,
-            '>': TokenType.GT,
-            '?': TokenType.QUESTION,
-            ',': TokenType.COMMA,
-            '(': TokenType.LPAREN,
-            ')': TokenType.RPAREN,
-            '{': TokenType.LBRACE,
-            '}': TokenType.RBRACE,
-            '[': TokenType.LBRACKET,
-            ']': TokenType.RBRACKET,
-        };
-        const type = singleCharOps[char];
-        if (type) {
-            return { type, value: char, line: this.line, column: startColumn };
-        }
-        throw new LexerError(`Unexpected character '${char}'`, { line: this.line, column: startColumn }, { source: this.source });
-    }
-    skipWhitespace() {
-        while (!this.isAtEnd()) {
-            const char = this.peek();
-            if (char === ' ' || char === '\t' || char === '\r') {
-                this.advance();
-            }
-            else {
-                break;
-            }
-        }
-    }
-    skipLineComment() {
-        while (!this.isAtEnd() && this.peek() !== '\n') {
-            this.advance();
-        }
-    }
-    makeToken(type, value) {
-        return { type, value, line: this.line, column: this.column };
-    }
-    peek() {
-        return this.source[this.pos] ?? '\0';
-    }
-    peekNext() {
-        return this.source[this.pos + 1] ?? '\0';
-    }
-    advance() {
-        const char = this.source[this.pos];
-        this.pos++;
-        this.column++;
-        return char;
-    }
-    isAtEnd() {
-        return this.pos >= this.source.length;
-    }
-    isDigit(char) {
-        return char >= '0' && char <= '9';
-    }
-    isAlpha(char) {
-        return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char === '_';
-    }
-    isAlphaNumeric(char) {
-        return this.isAlpha(char) || this.isDigit(char);
-    }
-}

package/dist/lexer/lexer.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/lexer/lexer.test.js

@@ -1,259 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { Lexer as ReqonLexer, TokenType } from './index.js';
-import { ReqonTokenType } from './tokens.js';
-describe('ReqonLexer', () => {
-    describe('string tokenization', () => {
-        it('tokenizes simple strings', () => {
-            const lexer = new ReqonLexer('"hello world"');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.STRING);
-            expect(tokens[0].value).toBe('hello world');
-        });
-        it('handles escape sequences', () => {
-            const lexer = new ReqonLexer('"line1\\nline2"');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.STRING);
-            expect(tokens[0].value).toBe('line1\nline2');
-        });
-        it('handles tab escape sequence', () => {
-            const lexer = new ReqonLexer('"col1\\tcol2"');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].value).toBe('col1\tcol2');
-        });
-        it('handles escaped quotes', () => {
-            const lexer = new ReqonLexer('"say \\"hello\\""');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].value).toBe('say "hello"');
-        });
-        it('handles escaped backslash', () => {
-            const lexer = new ReqonLexer('"path\\\\to\\\\file"');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].value).toBe('path\\to\\file');
-        });
-        it('throws on unterminated string', () => {
-            const lexer = new ReqonLexer('"unterminated');
-            expect(() => lexer.tokenize()).toThrow('Unterminated string');
-        });
-    });
-    describe('number tokenization', () => {
-        it('tokenizes integers', () => {
-            const lexer = new ReqonLexer('42');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.NUMBER);
-            expect(tokens[0].value).toBe('42');
-        });
-        it('tokenizes decimals', () => {
-            const lexer = new ReqonLexer('3.14159');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.NUMBER);
-            expect(tokens[0].value).toBe('3.14159');
-        });
-        it('tokenizes numbers with leading zero', () => {
-            const lexer = new ReqonLexer('0.5');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.NUMBER);
-            expect(tokens[0].value).toBe('0.5');
-        });
-    });
-    describe('keyword tokenization', () => {
-        it('recognizes Reqon keywords', () => {
-            const lexer = new ReqonLexer('mission action source store fetch run');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
-            expect(tokens[1].type).toBe(ReqonTokenType.ACTION);
-            expect(tokens[2].type).toBe(ReqonTokenType.SOURCE);
-            expect(tokens[3].type).toBe(ReqonTokenType.STORE);
-            expect(tokens[4].type).toBe(ReqonTokenType.FETCH);
-            expect(tokens[5].type).toBe(ReqonTokenType.RUN);
-        });
-        it('recognizes HTTP method keywords', () => {
-            const lexer = new ReqonLexer('GET POST PUT PATCH DELETE');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(ReqonTokenType.GET);
-            expect(tokens[1].type).toBe(ReqonTokenType.POST);
-            expect(tokens[2].type).toBe(ReqonTokenType.PUT);
-            expect(tokens[3].type).toBe(ReqonTokenType.PATCH);
-            expect(tokens[4].type).toBe(ReqonTokenType.DELETE);
-        });
-        it('recognizes auth type keywords', () => {
-            const lexer = new ReqonLexer('oauth2 bearer basic api_key');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(ReqonTokenType.OAUTH2);
-            expect(tokens[1].type).toBe(ReqonTokenType.BEARER);
-            expect(tokens[2].type).toBe(ReqonTokenType.BASIC);
-            expect(tokens[3].type).toBe(ReqonTokenType.API_KEY);
-        });
-        it('recognizes Vague keywords', () => {
-            const lexer = new ReqonLexer('match where and or not true false null');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.MATCH);
-            expect(tokens[1].type).toBe(TokenType.WHERE);
-            expect(tokens[2].type).toBe(TokenType.AND);
-            expect(tokens[3].type).toBe(TokenType.OR);
-            expect(tokens[4].type).toBe(TokenType.NOT);
-            expect(tokens[5].type).toBe(TokenType.TRUE);
-            expect(tokens[6].type).toBe(TokenType.FALSE);
-            expect(tokens[7].type).toBe(TokenType.NULL);
-        });
-        it('treats unknown words as identifiers', () => {
-            const lexer = new ReqonLexer('myVariable anotherOne');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.IDENTIFIER);
-            expect(tokens[0].value).toBe('myVariable');
-            expect(tokens[1].type).toBe(TokenType.IDENTIFIER);
-            expect(tokens[1].value).toBe('anotherOne');
-        });
-    });
-    describe('operator tokenization', () => {
-        it('recognizes right arrow operator', () => {
-            const lexer = new ReqonLexer('map item -> Output');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(ReqonTokenType.MAP);
-            expect(tokens[1].type).toBe(TokenType.IDENTIFIER);
-            expect(tokens[2].type).toBe(TokenType.RIGHT_ARROW);
-            expect(tokens[3].type).toBe(TokenType.IDENTIFIER);
-        });
-        it('recognizes two-character operators', () => {
-            const lexer = new ReqonLexer('== <= >= => ..');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.DOUBLE_EQUALS);
-            expect(tokens[1].type).toBe(TokenType.LTE);
-            expect(tokens[2].type).toBe(TokenType.GTE);
-            expect(tokens[3].type).toBe(TokenType.ARROW);
-            expect(tokens[4].type).toBe(TokenType.DOTDOT);
-        });
-        it('recognizes single-character operators', () => {
-            const lexer = new ReqonLexer('+ - * / : = < > . , ( ) { } [ ]');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(TokenType.PLUS);
-            expect(tokens[1].type).toBe(TokenType.MINUS);
-            expect(tokens[2].type).toBe(TokenType.STAR);
-            expect(tokens[3].type).toBe(TokenType.SLASH);
-            expect(tokens[4].type).toBe(TokenType.COLON);
-            expect(tokens[5].type).toBe(TokenType.EQUALS);
-            expect(tokens[6].type).toBe(TokenType.LT);
-            expect(tokens[7].type).toBe(TokenType.GT);
-            expect(tokens[8].type).toBe(TokenType.DOT);
-            expect(tokens[9].type).toBe(TokenType.COMMA);
-            expect(tokens[10].type).toBe(TokenType.LPAREN);
-            expect(tokens[11].type).toBe(TokenType.RPAREN);
-            expect(tokens[12].type).toBe(TokenType.LBRACE);
-            expect(tokens[13].type).toBe(TokenType.RBRACE);
-            expect(tokens[14].type).toBe(TokenType.LBRACKET);
-            expect(tokens[15].type).toBe(TokenType.RBRACKET);
-        });
-        it('throws on unexpected character', () => {
-            const lexer = new ReqonLexer('@');
-            expect(() => lexer.tokenize()).toThrow("Unexpected character '@'");
-        });
-    });
-    describe('comments', () => {
-        it('skips single-line comments', () => {
-            const lexer = new ReqonLexer('mission // this is a comment\nTestMission');
-            const tokens = lexer.tokenize();
-            // Should have: MISSION, NEWLINE, IDENTIFIER, EOF
-            expect(tokens.filter((t) => t.type !== TokenType.NEWLINE && t.type !== TokenType.EOF))
-                .toHaveLength(2);
-            expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
-            expect(tokens[2].type).toBe(TokenType.IDENTIFIER);
-            expect(tokens[2].value).toBe('TestMission');
-        });
-        it('handles comment at end of file', () => {
-            const lexer = new ReqonLexer('mission // end comment');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
-            expect(tokens[tokens.length - 1].type).toBe(TokenType.EOF);
-        });
-    });
-    describe('line and column tracking', () => {
-        it('tracks line numbers', () => {
-            const lexer = new ReqonLexer('mission\nTestMission\n{\n}');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].line).toBe(1); // mission
-            expect(tokens[2].line).toBe(2); // TestMission
-            expect(tokens[4].line).toBe(3); // {
-            expect(tokens[6].line).toBe(4); // }
-        });
-        it('tracks column numbers', () => {
-            const lexer = new ReqonLexer('mission TestMission');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].column).toBe(1); // mission starts at column 1
-            expect(tokens[1].column).toBe(9); // TestMission starts at column 9
-        });
-        it('resets column after newline', () => {
-            const lexer = new ReqonLexer('abc\nxyz');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].column).toBe(1); // abc
-            expect(tokens[2].column).toBe(1); // xyz
-        });
-    });
-    describe('complex tokenization', () => {
-        it('tokenizes a complete mission definition', () => {
-            const source = `
-        mission SyncUsers {
-          source API {
-            auth: bearer,
-            base: "https://api.example.com"
-          }
-
-          store users: memory("users")
-
-          action FetchUsers {
-            fetch GET "/users"
-            store response -> users {
-              key: .id
-            }
-          }
-
-          run FetchUsers
-        }
-      `;
-            const lexer = new ReqonLexer(source);
-            const tokens = lexer.tokenize();
-            // Verify it doesn't throw and produces tokens
-            expect(tokens.length).toBeGreaterThan(0);
-            expect(tokens[tokens.length - 1].type).toBe(TokenType.EOF);
-            // Check key tokens are present
-            const tokenTypes = tokens.map((t) => t.type);
-            expect(tokenTypes).toContain(ReqonTokenType.MISSION);
-            expect(tokenTypes).toContain(ReqonTokenType.SOURCE);
-            expect(tokenTypes).toContain(ReqonTokenType.STORE);
-            expect(tokenTypes).toContain(ReqonTokenType.ACTION);
-            expect(tokenTypes).toContain(ReqonTokenType.FETCH);
-            expect(tokenTypes).toContain(ReqonTokenType.GET);
-            expect(tokenTypes).toContain(ReqonTokenType.RUN);
-        });
-        it('tokenizes pagination options', () => {
-            const lexer = new ReqonLexer('paginate offset page cursor until retry');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(ReqonTokenType.PAGINATE);
-            expect(tokens[1].type).toBe(ReqonTokenType.OFFSET);
-            expect(tokens[2].type).toBe(ReqonTokenType.PAGE);
-            expect(tokens[3].type).toBe(ReqonTokenType.CURSOR);
-            expect(tokens[4].type).toBe(ReqonTokenType.UNTIL);
-            expect(tokens[5].type).toBe(ReqonTokenType.RETRY);
-        });
-        it('tokenizes store options', () => {
-            const lexer = new ReqonLexer('key partial upsert');
-            const tokens = lexer.tokenize();
-            expect(tokens[0].type).toBe(ReqonTokenType.KEY);
-            expect(tokens[1].type).toBe(ReqonTokenType.PARTIAL);
-            expect(tokens[2].type).toBe(ReqonTokenType.UPSERT);
-        });
-    });
-    describe('whitespace handling', () => {
-        it('skips spaces and tabs', () => {
-            const lexer = new ReqonLexer('   mission   \t  action   ');
-            const tokens = lexer.tokenize();
-            const nonEofTokens = tokens.filter((t) => t.type !== TokenType.EOF);
-            expect(nonEofTokens).toHaveLength(2);
-            expect(nonEofTokens[0].type).toBe(ReqonTokenType.MISSION);
-            expect(nonEofTokens[1].type).toBe(ReqonTokenType.ACTION);
-        });
-        it('tracks newlines as tokens', () => {
-            const lexer = new ReqonLexer('a\nb\nc');
-            const tokens = lexer.tokenize();
-            expect(tokens.filter((t) => t.type === TokenType.NEWLINE)).toHaveLength(2);
-        });
-    });
-});

package/dist/loader/loader.test.d.ts

@@ -1 +0,0 @@
-export {};