reqon-dsl 0.2.0

package/dist/lexer/lexer.js

@@ -0,0 +1,264 @@
+import { TokenType } from 'vague-lang';
+import { REQON_KEYWORDS, ReqonTokenType } from './tokens.js';
+import { LexerError } from '../errors/index.js';
+export class ReqonLexer {
+    source;
+    pos = 0;
+    line = 1;
+    column = 1;
+    constructor(source) {
+        this.source = source;
+    }
+    tokenize() {
+        const tokens = [];
+        while (!this.isAtEnd()) {
+            const token = this.nextToken();
+            if (token) {
+                tokens.push(token);
+            }
+        }
+        tokens.push(this.makeToken(TokenType.EOF, ''));
+        return tokens;
+    }
+    nextToken() {
+        this.skipWhitespace();
+        if (this.isAtEnd())
+            return null;
+        const char = this.peek();
+        // Skip comments
+        if (char === '/' && this.peekNext() === '/') {
+            this.skipLineComment();
+            return null;
+        }
+        // Newlines (significant for statement termination)
+        if (char === '\n') {
+            const token = this.makeToken(TokenType.NEWLINE, '\n');
+            this.advance();
+            this.line++;
+            this.column = 1;
+            return token;
+        }
+        // String literals
+        if (char === '"')
+            return this.readString();
+        // Numbers
+        if (this.isDigit(char))
+            return this.readNumber();
+        // Identifiers and keywords
+        if (this.isAlpha(char))
+            return this.readIdentifier();
+        // Operators and delimiters
+        return this.readOperator();
+    }
+    readString() {
+        const startColumn = this.column;
+        this.advance(); // consume opening quote
+        let value = '';
+        while (!this.isAtEnd() && this.peek() !== '"') {
+            if (this.peek() === '\\') {
+                this.advance();
+                const escaped = this.advance();
+                switch (escaped) {
+                    case 'n':
+                        value += '\n';
+                        break;
+                    case 't':
+                        value += '\t';
+                        break;
+                    case '\\':
+                        value += '\\';
+                        break;
+                    case '"':
+                        value += '"';
+                        break;
+                    default:
+                        value += escaped;
+                }
+            }
+            else {
+                value += this.advance();
+            }
+        }
+        if (this.isAtEnd()) {
+            throw new LexerError('Unterminated string', { line: this.line, column: startColumn }, { source: this.source });
+        }
+        this.advance(); // consume closing quote
+        return {
+            type: TokenType.STRING,
+            value,
+            line: this.line,
+            column: startColumn,
+        };
+    }
+    readNumber() {
+        const startColumn = this.column;
+        let value = '';
+        while (this.isDigit(this.peek())) {
+            value += this.advance();
+        }
+        // Handle decimals
+        if (this.peek() === '.' && this.isDigit(this.peekNext())) {
+            value += this.advance(); // consume '.'
+            while (this.isDigit(this.peek())) {
+                value += this.advance();
+            }
+        }
+        // Handle underscores in numbers (e.g., 100_000)
+        value = value.replace(/_/g, '');
+        return {
+            type: TokenType.NUMBER,
+            value,
+            line: this.line,
+            column: startColumn,
+        };
+    }
+    readIdentifier() {
+        const startColumn = this.column;
+        let value = '';
+        while (this.isAlphaNumeric(this.peek()) || this.peek() === '_') {
+            value += this.advance();
+        }
+        // Check Reqon keywords first, then fall back to Vague keywords
+        const reqonType = REQON_KEYWORDS[value];
+        if (reqonType) {
+            return { type: reqonType, value, line: this.line, column: startColumn };
+        }
+        // Use Vague's lexer to check for Vague keywords
+        const vagueKeywords = {
+            let: TokenType.LET,
+            schema: TokenType.SCHEMA,
+            from: TokenType.FROM,
+            import: TokenType.IMPORT,
+            context: TokenType.CONTEXT,
+            distribution: TokenType.DISTRIBUTION,
+            dataset: TokenType.DATASET,
+            with: TokenType.WITH,
+            where: TokenType.WHERE,
+            when: TokenType.WHEN,
+            in: TokenType.IN,
+            per: TokenType.PER,
+            any: TokenType.ANY,
+            of: TokenType.OF,
+            match: TokenType.MATCH,
+            affects: TokenType.AFFECTS,
+            constraints: TokenType.CONSTRAINTS,
+            validate: TokenType.VALIDATE,
+            assume: TokenType.ASSUME,
+            then: TokenType.THEN,
+            refine: TokenType.REFINE,
+            if: TokenType.IF,
+            violating: TokenType.VIOLATING,
+            unique: TokenType.UNIQUE,
+            private: TokenType.PRIVATE,
+            and: TokenType.AND,
+            or: TokenType.OR,
+            not: TokenType.NOT,
+            null: TokenType.NULL,
+            true: TokenType.TRUE,
+            false: TokenType.FALSE,
+            int: TokenType.INT,
+            decimal: TokenType.DECIMAL,
+            date: TokenType.DATE,
+        };
+        const type = vagueKeywords[value] ?? TokenType.IDENTIFIER;
+        return { type, value, line: this.line, column: startColumn };
+    }
+    readOperator() {
+        const startColumn = this.column;
+        const char = this.advance();
+        // Check for -> (Reqon-specific)
+        if (char === '-' && this.peek() === '>') {
+            this.advance();
+            return {
+                type: ReqonTokenType.RIGHT_ARROW,
+                value: '->',
+                line: this.line,
+                column: startColumn,
+            };
+        }
+        const twoChar = char + this.peek();
+        const twoCharOps = {
+            '..': TokenType.DOTDOT,
+            '=>': TokenType.ARROW,
+            '==': TokenType.DOUBLE_EQUALS,
+            '+=': TokenType.PLUS_EQUALS,
+            '<=': TokenType.LTE,
+            '>=': TokenType.GTE,
+        };
+        if (twoCharOps[twoChar]) {
+            this.advance();
+            return { type: twoCharOps[twoChar], value: twoChar, line: this.line, column: startColumn };
+        }
+        const singleCharOps = {
+            '|': TokenType.PIPE,
+            '~': TokenType.TILDE,
+            ':': TokenType.COLON,
+            '=': TokenType.EQUALS,
+            '+': TokenType.PLUS,
+            '-': TokenType.MINUS,
+            '*': TokenType.STAR,
+            '/': TokenType.SLASH,
+            '.': TokenType.DOT,
+            '^': TokenType.CARET,
+            '%': TokenType.PERCENT,
+            '<': TokenType.LT,
+            '>': TokenType.GT,
+            '?': TokenType.QUESTION,
+            ',': TokenType.COMMA,
+            '(': TokenType.LPAREN,
+            ')': TokenType.RPAREN,
+            '{': TokenType.LBRACE,
+            '}': TokenType.RBRACE,
+            '[': TokenType.LBRACKET,
+            ']': TokenType.RBRACKET,
+        };
+        const type = singleCharOps[char];
+        if (type) {
+            return { type, value: char, line: this.line, column: startColumn };
+        }
+        throw new LexerError(`Unexpected character '${char}'`, { line: this.line, column: startColumn }, { source: this.source });
+    }
+    skipWhitespace() {
+        while (!this.isAtEnd()) {
+            const char = this.peek();
+            if (char === ' ' || char === '\t' || char === '\r') {
+                this.advance();
+            }
+            else {
+                break;
+            }
+        }
+    }
+    skipLineComment() {
+        while (!this.isAtEnd() && this.peek() !== '\n') {
+            this.advance();
+        }
+    }
+    makeToken(type, value) {
+        return { type, value, line: this.line, column: this.column };
+    }
+    peek() {
+        return this.source[this.pos] ?? '\0';
+    }
+    peekNext() {
+        return this.source[this.pos + 1] ?? '\0';
+    }
+    advance() {
+        const char = this.source[this.pos];
+        this.pos++;
+        this.column++;
+        return char;
+    }
+    isAtEnd() {
+        return this.pos >= this.source.length;
+    }
+    isDigit(char) {
+        return char >= '0' && char <= '9';
+    }
+    isAlpha(char) {
+        return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char === '_';
+    }
+    isAlphaNumeric(char) {
+        return this.isAlpha(char) || this.isDigit(char);
+    }
+}

package/dist/lexer/lexer.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/lexer/lexer.test.js

@@ -0,0 +1,259 @@
+import { describe, it, expect } from 'vitest';
+import { Lexer as ReqonLexer, TokenType } from './index.js';
+import { ReqonTokenType } from './tokens.js';
+describe('ReqonLexer', () => {
+    describe('string tokenization', () => {
+        it('tokenizes simple strings', () => {
+            const lexer = new ReqonLexer('"hello world"');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.STRING);
+            expect(tokens[0].value).toBe('hello world');
+        });
+        it('handles escape sequences', () => {
+            const lexer = new ReqonLexer('"line1\\nline2"');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.STRING);
+            expect(tokens[0].value).toBe('line1\nline2');
+        });
+        it('handles tab escape sequence', () => {
+            const lexer = new ReqonLexer('"col1\\tcol2"');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].value).toBe('col1\tcol2');
+        });
+        it('handles escaped quotes', () => {
+            const lexer = new ReqonLexer('"say \\"hello\\""');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].value).toBe('say "hello"');
+        });
+        it('handles escaped backslash', () => {
+            const lexer = new ReqonLexer('"path\\\\to\\\\file"');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].value).toBe('path\\to\\file');
+        });
+        it('throws on unterminated string', () => {
+            const lexer = new ReqonLexer('"unterminated');
+            expect(() => lexer.tokenize()).toThrow('Unterminated string');
+        });
+    });
+    describe('number tokenization', () => {
+        it('tokenizes integers', () => {
+            const lexer = new ReqonLexer('42');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.NUMBER);
+            expect(tokens[0].value).toBe('42');
+        });
+        it('tokenizes decimals', () => {
+            const lexer = new ReqonLexer('3.14159');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.NUMBER);
+            expect(tokens[0].value).toBe('3.14159');
+        });
+        it('tokenizes numbers with leading zero', () => {
+            const lexer = new ReqonLexer('0.5');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.NUMBER);
+            expect(tokens[0].value).toBe('0.5');
+        });
+    });
+    describe('keyword tokenization', () => {
+        it('recognizes Reqon keywords', () => {
+            const lexer = new ReqonLexer('mission action source store fetch run');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
+            expect(tokens[1].type).toBe(ReqonTokenType.ACTION);
+            expect(tokens[2].type).toBe(ReqonTokenType.SOURCE);
+            expect(tokens[3].type).toBe(ReqonTokenType.STORE);
+            expect(tokens[4].type).toBe(ReqonTokenType.FETCH);
+            expect(tokens[5].type).toBe(ReqonTokenType.RUN);
+        });
+        it('recognizes HTTP method keywords', () => {
+            const lexer = new ReqonLexer('GET POST PUT PATCH DELETE');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(ReqonTokenType.GET);
+            expect(tokens[1].type).toBe(ReqonTokenType.POST);
+            expect(tokens[2].type).toBe(ReqonTokenType.PUT);
+            expect(tokens[3].type).toBe(ReqonTokenType.PATCH);
+            expect(tokens[4].type).toBe(ReqonTokenType.DELETE);
+        });
+        it('recognizes auth type keywords', () => {
+            const lexer = new ReqonLexer('oauth2 bearer basic api_key');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(ReqonTokenType.OAUTH2);
+            expect(tokens[1].type).toBe(ReqonTokenType.BEARER);
+            expect(tokens[2].type).toBe(ReqonTokenType.BASIC);
+            expect(tokens[3].type).toBe(ReqonTokenType.API_KEY);
+        });
+        it('recognizes Vague keywords', () => {
+            const lexer = new ReqonLexer('match where and or not true false null');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.MATCH);
+            expect(tokens[1].type).toBe(TokenType.WHERE);
+            expect(tokens[2].type).toBe(TokenType.AND);
+            expect(tokens[3].type).toBe(TokenType.OR);
+            expect(tokens[4].type).toBe(TokenType.NOT);
+            expect(tokens[5].type).toBe(TokenType.TRUE);
+            expect(tokens[6].type).toBe(TokenType.FALSE);
+            expect(tokens[7].type).toBe(TokenType.NULL);
+        });
+        it('treats unknown words as identifiers', () => {
+            const lexer = new ReqonLexer('myVariable anotherOne');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.IDENTIFIER);
+            expect(tokens[0].value).toBe('myVariable');
+            expect(tokens[1].type).toBe(TokenType.IDENTIFIER);
+            expect(tokens[1].value).toBe('anotherOne');
+        });
+    });
+    describe('operator tokenization', () => {
+        it('recognizes right arrow operator', () => {
+            const lexer = new ReqonLexer('map item -> Output');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(ReqonTokenType.MAP);
+            expect(tokens[1].type).toBe(TokenType.IDENTIFIER);
+            expect(tokens[2].type).toBe(TokenType.RIGHT_ARROW);
+            expect(tokens[3].type).toBe(TokenType.IDENTIFIER);
+        });
+        it('recognizes two-character operators', () => {
+            const lexer = new ReqonLexer('== <= >= => ..');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.DOUBLE_EQUALS);
+            expect(tokens[1].type).toBe(TokenType.LTE);
+            expect(tokens[2].type).toBe(TokenType.GTE);
+            expect(tokens[3].type).toBe(TokenType.ARROW);
+            expect(tokens[4].type).toBe(TokenType.DOTDOT);
+        });
+        it('recognizes single-character operators', () => {
+            const lexer = new ReqonLexer('+ - * / : = < > . , ( ) { } [ ]');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(TokenType.PLUS);
+            expect(tokens[1].type).toBe(TokenType.MINUS);
+            expect(tokens[2].type).toBe(TokenType.STAR);
+            expect(tokens[3].type).toBe(TokenType.SLASH);
+            expect(tokens[4].type).toBe(TokenType.COLON);
+            expect(tokens[5].type).toBe(TokenType.EQUALS);
+            expect(tokens[6].type).toBe(TokenType.LT);
+            expect(tokens[7].type).toBe(TokenType.GT);
+            expect(tokens[8].type).toBe(TokenType.DOT);
+            expect(tokens[9].type).toBe(TokenType.COMMA);
+            expect(tokens[10].type).toBe(TokenType.LPAREN);
+            expect(tokens[11].type).toBe(TokenType.RPAREN);
+            expect(tokens[12].type).toBe(TokenType.LBRACE);
+            expect(tokens[13].type).toBe(TokenType.RBRACE);
+            expect(tokens[14].type).toBe(TokenType.LBRACKET);
+            expect(tokens[15].type).toBe(TokenType.RBRACKET);
+        });
+        it('throws on unexpected character', () => {
+            const lexer = new ReqonLexer('@');
+            expect(() => lexer.tokenize()).toThrow("Unexpected character '@'");
+        });
+    });
+    describe('comments', () => {
+        it('skips single-line comments', () => {
+            const lexer = new ReqonLexer('mission // this is a comment\nTestMission');
+            const tokens = lexer.tokenize();
+            // Should have: MISSION, NEWLINE, IDENTIFIER, EOF
+            expect(tokens.filter((t) => t.type !== TokenType.NEWLINE && t.type !== TokenType.EOF))
+                .toHaveLength(2);
+            expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
+            expect(tokens[2].type).toBe(TokenType.IDENTIFIER);
+            expect(tokens[2].value).toBe('TestMission');
+        });
+        it('handles comment at end of file', () => {
+            const lexer = new ReqonLexer('mission // end comment');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
+            expect(tokens[tokens.length - 1].type).toBe(TokenType.EOF);
+        });
+    });
+    describe('line and column tracking', () => {
+        it('tracks line numbers', () => {
+            const lexer = new ReqonLexer('mission\nTestMission\n{\n}');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].line).toBe(1); // mission
+            expect(tokens[2].line).toBe(2); // TestMission
+            expect(tokens[4].line).toBe(3); // {
+            expect(tokens[6].line).toBe(4); // }
+        });
+        it('tracks column numbers', () => {
+            const lexer = new ReqonLexer('mission TestMission');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].column).toBe(1); // mission starts at column 1
+            expect(tokens[1].column).toBe(9); // TestMission starts at column 9
+        });
+        it('resets column after newline', () => {
+            const lexer = new ReqonLexer('abc\nxyz');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].column).toBe(1); // abc
+            expect(tokens[2].column).toBe(1); // xyz
+        });
+    });
+    describe('complex tokenization', () => {
+        it('tokenizes a complete mission definition', () => {
+            const source = `
+        mission SyncUsers {
+          source API {
+            auth: bearer,
+            base: "https://api.example.com"
+          }
+
+          store users: memory("users")
+
+          action FetchUsers {
+            fetch GET "/users"
+            store response -> users {
+              key: .id
+            }
+          }
+
+          run FetchUsers
+        }
+      `;
+            const lexer = new ReqonLexer(source);
+            const tokens = lexer.tokenize();
+            // Verify it doesn't throw and produces tokens
+            expect(tokens.length).toBeGreaterThan(0);
+            expect(tokens[tokens.length - 1].type).toBe(TokenType.EOF);
+            // Check key tokens are present
+            const tokenTypes = tokens.map((t) => t.type);
+            expect(tokenTypes).toContain(ReqonTokenType.MISSION);
+            expect(tokenTypes).toContain(ReqonTokenType.SOURCE);
+            expect(tokenTypes).toContain(ReqonTokenType.STORE);
+            expect(tokenTypes).toContain(ReqonTokenType.ACTION);
+            expect(tokenTypes).toContain(ReqonTokenType.FETCH);
+            expect(tokenTypes).toContain(ReqonTokenType.GET);
+            expect(tokenTypes).toContain(ReqonTokenType.RUN);
+        });
+        it('tokenizes pagination options', () => {
+            const lexer = new ReqonLexer('paginate offset page cursor until retry');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(ReqonTokenType.PAGINATE);
+            expect(tokens[1].type).toBe(ReqonTokenType.OFFSET);
+            expect(tokens[2].type).toBe(ReqonTokenType.PAGE);
+            expect(tokens[3].type).toBe(ReqonTokenType.CURSOR);
+            expect(tokens[4].type).toBe(ReqonTokenType.UNTIL);
+            expect(tokens[5].type).toBe(ReqonTokenType.RETRY);
+        });
+        it('tokenizes store options', () => {
+            const lexer = new ReqonLexer('key partial upsert');
+            const tokens = lexer.tokenize();
+            expect(tokens[0].type).toBe(ReqonTokenType.KEY);
+            expect(tokens[1].type).toBe(ReqonTokenType.PARTIAL);
+            expect(tokens[2].type).toBe(ReqonTokenType.UPSERT);
+        });
+    });
+    describe('whitespace handling', () => {
+        it('skips spaces and tabs', () => {
+            const lexer = new ReqonLexer('   mission   \t  action   ');
+            const tokens = lexer.tokenize();
+            const nonEofTokens = tokens.filter((t) => t.type !== TokenType.EOF);
+            expect(nonEofTokens).toHaveLength(2);
+            expect(nonEofTokens[0].type).toBe(ReqonTokenType.MISSION);
+            expect(nonEofTokens[1].type).toBe(ReqonTokenType.ACTION);
+        });
+        it('tracks newlines as tokens', () => {
+            const lexer = new ReqonLexer('a\nb\nc');
+            const tokens = lexer.tokenize();
+            expect(tokens.filter((t) => t.type === TokenType.NEWLINE)).toHaveLength(2);
+        });
+    });
+});

package/dist/lexer/tokens.d.ts

@@ -0,0 +1,69 @@
+import { TokenType as VagueTokenType } from 'vague-lang';
+export declare enum ReqonTokenType {
+    MISSION = "MISSION",
+    ACTION = "ACTION",
+    SOURCE = "SOURCE",
+    STORE = "STORE",
+    CALL = "CALL",
+    RUN = "RUN",
+    FOR = "FOR",
+    MAP = "MAP",
+    EACH = "EACH",
+    PAGINATE = "PAGINATE",
+    UNTIL = "UNTIL",
+    RETRY = "RETRY",
+    KEY = "KEY",
+    PARTIAL = "PARTIAL",
+    UPSERT = "UPSERT",
+    SINCE = "SINCE",
+    LAST_SYNC = "LAST_SYNC",
+    GET = "GET",
+    POST = "POST",
+    PUT = "PUT",
+    PATCH = "PATCH",
+    DELETE = "DELETE",
+    NOSQL = "NOSQL",
+    SQL = "SQL",
+    MEMORY = "MEMORY",
+    OAUTH2 = "OAUTH2",
+    BEARER = "BEARER",
+    BASIC = "BASIC",
+    API_KEY = "API_KEY",
+    NONE = "NONE",
+    OFFSET = "OFFSET",
+    CURSOR = "CURSOR",
+    PAGE = "PAGE",
+    NOT_EQUALS = "NOT_EQUALS",// !=
+    BANG = "BANG",// !
+    FROM = "FROM",
+    SCHEDULE = "SCHEDULE",
+    CRON = "CRON",
+    EVERY = "EVERY",
+    AT = "AT",
+    HOURS = "HOURS",
+    MINUTES = "MINUTES",
+    SECONDS = "SECONDS",
+    DAYS = "DAYS",
+    WEEKS = "WEEKS",
+    CONTINUE = "CONTINUE",
+    SKIP = "SKIP",
+    ABORT = "ABORT",
+    QUEUE = "QUEUE",
+    JUMP = "JUMP",
+    IS = "IS",
+    WAIT = "WAIT",
+    TIMEOUT = "TIMEOUT",
+    PATH = "PATH",
+    EXPECTED_EVENTS = "EXPECTED_EVENTS",
+    EVENT_FILTER = "EVENT_FILTER",
+    STORAGE = "STORAGE"
+}
+export type TokenType = VagueTokenType | ReqonTokenType;
+export { TokenType as VagueTokenType } from 'vague-lang';
+export declare const REQON_KEYWORDS: Record<string, ReqonTokenType>;
+export interface ReqonToken {
+    type: TokenType | string;
+    value: string;
+    line: number;
+    column: number;
+}