reqon-dsl 0.2.0

package/src/lexer/lexer.test.ts

@@ -0,0 +1,316 @@
+import { describe, it, expect } from 'vitest';
+import { Lexer as ReqonLexer, TokenType, type Token } from './index.js';
+import { ReqonTokenType } from './tokens.js';
+
+describe('ReqonLexer', () => {
+  describe('string tokenization', () => {
+    it('tokenizes simple strings', () => {
+      const lexer = new ReqonLexer('"hello world"');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.STRING);
+      expect(tokens[0].value).toBe('hello world');
+    });
+
+    it('handles escape sequences', () => {
+      const lexer = new ReqonLexer('"line1\\nline2"');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.STRING);
+      expect(tokens[0].value).toBe('line1\nline2');
+    });
+
+    it('handles tab escape sequence', () => {
+      const lexer = new ReqonLexer('"col1\\tcol2"');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].value).toBe('col1\tcol2');
+    });
+
+    it('handles escaped quotes', () => {
+      const lexer = new ReqonLexer('"say \\"hello\\""');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].value).toBe('say "hello"');
+    });
+
+    it('handles escaped backslash', () => {
+      const lexer = new ReqonLexer('"path\\\\to\\\\file"');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].value).toBe('path\\to\\file');
+    });
+
+    it('throws on unterminated string', () => {
+      const lexer = new ReqonLexer('"unterminated');
+
+      expect(() => lexer.tokenize()).toThrow('Unterminated string');
+    });
+  });
+
+  describe('number tokenization', () => {
+    it('tokenizes integers', () => {
+      const lexer = new ReqonLexer('42');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.NUMBER);
+      expect(tokens[0].value).toBe('42');
+    });
+
+    it('tokenizes decimals', () => {
+      const lexer = new ReqonLexer('3.14159');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.NUMBER);
+      expect(tokens[0].value).toBe('3.14159');
+    });
+
+    it('tokenizes numbers with leading zero', () => {
+      const lexer = new ReqonLexer('0.5');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.NUMBER);
+      expect(tokens[0].value).toBe('0.5');
+    });
+  });
+
+  describe('keyword tokenization', () => {
+    it('recognizes Reqon keywords', () => {
+      const lexer = new ReqonLexer('mission action source store call run');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
+      expect(tokens[1].type).toBe(ReqonTokenType.ACTION);
+      expect(tokens[2].type).toBe(ReqonTokenType.SOURCE);
+      expect(tokens[3].type).toBe(ReqonTokenType.STORE);
+      expect(tokens[4].type).toBe(ReqonTokenType.CALL);
+      expect(tokens[5].type).toBe(ReqonTokenType.RUN);
+    });
+
+    it('recognizes HTTP method keywords', () => {
+      const lexer = new ReqonLexer('GET POST PUT PATCH DELETE');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(ReqonTokenType.GET);
+      expect(tokens[1].type).toBe(ReqonTokenType.POST);
+      expect(tokens[2].type).toBe(ReqonTokenType.PUT);
+      expect(tokens[3].type).toBe(ReqonTokenType.PATCH);
+      expect(tokens[4].type).toBe(ReqonTokenType.DELETE);
+    });
+
+    it('recognizes auth type keywords', () => {
+      const lexer = new ReqonLexer('oauth2 bearer basic api_key');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(ReqonTokenType.OAUTH2);
+      expect(tokens[1].type).toBe(ReqonTokenType.BEARER);
+      expect(tokens[2].type).toBe(ReqonTokenType.BASIC);
+      expect(tokens[3].type).toBe(ReqonTokenType.API_KEY);
+    });
+
+    it('recognizes Vague keywords', () => {
+      const lexer = new ReqonLexer('match where and or not true false null');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.MATCH);
+      expect(tokens[1].type).toBe(TokenType.WHERE);
+      expect(tokens[2].type).toBe(TokenType.AND);
+      expect(tokens[3].type).toBe(TokenType.OR);
+      expect(tokens[4].type).toBe(TokenType.NOT);
+      expect(tokens[5].type).toBe(TokenType.TRUE);
+      expect(tokens[6].type).toBe(TokenType.FALSE);
+      expect(tokens[7].type).toBe(TokenType.NULL);
+    });
+
+    it('treats unknown words as identifiers', () => {
+      const lexer = new ReqonLexer('myVariable anotherOne');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.IDENTIFIER);
+      expect(tokens[0].value).toBe('myVariable');
+      expect(tokens[1].type).toBe(TokenType.IDENTIFIER);
+      expect(tokens[1].value).toBe('anotherOne');
+    });
+  });
+
+  describe('operator tokenization', () => {
+    it('recognizes right arrow operator', () => {
+      const lexer = new ReqonLexer('map item -> Output');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(ReqonTokenType.MAP);
+      expect(tokens[1].type).toBe(TokenType.IDENTIFIER);
+      expect(tokens[2].type).toBe(TokenType.RIGHT_ARROW);
+      expect(tokens[3].type).toBe(TokenType.IDENTIFIER);
+    });
+
+    it('recognizes two-character operators', () => {
+      const lexer = new ReqonLexer('== <= >= => ..');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.DOUBLE_EQUALS);
+      expect(tokens[1].type).toBe(TokenType.LTE);
+      expect(tokens[2].type).toBe(TokenType.GTE);
+      expect(tokens[3].type).toBe(TokenType.ARROW);
+      expect(tokens[4].type).toBe(TokenType.DOTDOT);
+    });
+
+    it('recognizes single-character operators', () => {
+      const lexer = new ReqonLexer('+ - * / : = < > . , ( ) { } [ ]');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(TokenType.PLUS);
+      expect(tokens[1].type).toBe(TokenType.MINUS);
+      expect(tokens[2].type).toBe(TokenType.STAR);
+      expect(tokens[3].type).toBe(TokenType.SLASH);
+      expect(tokens[4].type).toBe(TokenType.COLON);
+      expect(tokens[5].type).toBe(TokenType.EQUALS);
+      expect(tokens[6].type).toBe(TokenType.LT);
+      expect(tokens[7].type).toBe(TokenType.GT);
+      expect(tokens[8].type).toBe(TokenType.DOT);
+      expect(tokens[9].type).toBe(TokenType.COMMA);
+      expect(tokens[10].type).toBe(TokenType.LPAREN);
+      expect(tokens[11].type).toBe(TokenType.RPAREN);
+      expect(tokens[12].type).toBe(TokenType.LBRACE);
+      expect(tokens[13].type).toBe(TokenType.RBRACE);
+      expect(tokens[14].type).toBe(TokenType.LBRACKET);
+      expect(tokens[15].type).toBe(TokenType.RBRACKET);
+    });
+
+    it('throws on unexpected character', () => {
+      const lexer = new ReqonLexer('@');
+
+      expect(() => lexer.tokenize()).toThrow("Unexpected character '@'");
+    });
+  });
+
+  describe('comments', () => {
+    it('skips single-line comments', () => {
+      const lexer = new ReqonLexer('mission // this is a comment\nTestMission');
+      const tokens = lexer.tokenize();
+
+      // Should have: MISSION, NEWLINE, IDENTIFIER, EOF
+      expect(tokens.filter((t: Token) => t.type !== TokenType.NEWLINE && t.type !== TokenType.EOF))
+        .toHaveLength(2);
+      expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
+      expect(tokens[2].type).toBe(TokenType.IDENTIFIER);
+      expect(tokens[2].value).toBe('TestMission');
+    });
+
+    it('handles comment at end of file', () => {
+      const lexer = new ReqonLexer('mission // end comment');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(ReqonTokenType.MISSION);
+      expect(tokens[tokens.length - 1].type).toBe(TokenType.EOF);
+    });
+  });
+
+  describe('line and column tracking', () => {
+    it('tracks line numbers', () => {
+      const lexer = new ReqonLexer('mission\nTestMission\n{\n}');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].line).toBe(1); // mission
+      expect(tokens[2].line).toBe(2); // TestMission
+      expect(tokens[4].line).toBe(3); // {
+      expect(tokens[6].line).toBe(4); // }
+    });
+
+    it('tracks column numbers', () => {
+      const lexer = new ReqonLexer('mission TestMission');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].column).toBe(1); // mission starts at column 1
+      expect(tokens[1].column).toBe(9); // TestMission starts at column 9
+    });
+
+    it('resets column after newline', () => {
+      const lexer = new ReqonLexer('abc\nxyz');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].column).toBe(1); // abc
+      expect(tokens[2].column).toBe(1); // xyz
+    });
+  });
+
+  describe('complex tokenization', () => {
+    it('tokenizes a complete mission definition', () => {
+      const source = `
+        mission SyncUsers {
+          source API {
+            auth: bearer,
+            base: "https://api.example.com"
+          }
+
+          store users: memory("users")
+
+          action FetchUsers {
+            get "/users"
+            store response -> users {
+              key: .id
+            }
+          }
+
+          run FetchUsers
+        }
+      `;
+
+      const lexer = new ReqonLexer(source);
+      const tokens = lexer.tokenize();
+
+      // Verify it doesn't throw and produces tokens
+      expect(tokens.length).toBeGreaterThan(0);
+      expect(tokens[tokens.length - 1].type).toBe(TokenType.EOF);
+
+      // Check key tokens are present
+      const tokenTypes = tokens.map((t: Token) => t.type);
+      expect(tokenTypes).toContain(ReqonTokenType.MISSION);
+      expect(tokenTypes).toContain(ReqonTokenType.SOURCE);
+      expect(tokenTypes).toContain(ReqonTokenType.STORE);
+      expect(tokenTypes).toContain(ReqonTokenType.ACTION);
+      expect(tokenTypes).toContain(ReqonTokenType.GET);
+      expect(tokenTypes).toContain(ReqonTokenType.RUN);
+    });
+
+    it('tokenizes pagination options', () => {
+      const lexer = new ReqonLexer('paginate offset page cursor until retry');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(ReqonTokenType.PAGINATE);
+      expect(tokens[1].type).toBe(ReqonTokenType.OFFSET);
+      expect(tokens[2].type).toBe(ReqonTokenType.PAGE);
+      expect(tokens[3].type).toBe(ReqonTokenType.CURSOR);
+      expect(tokens[4].type).toBe(ReqonTokenType.UNTIL);
+      expect(tokens[5].type).toBe(ReqonTokenType.RETRY);
+    });
+
+    it('tokenizes store options', () => {
+      const lexer = new ReqonLexer('key partial upsert');
+      const tokens = lexer.tokenize();
+
+      expect(tokens[0].type).toBe(ReqonTokenType.KEY);
+      expect(tokens[1].type).toBe(ReqonTokenType.PARTIAL);
+      expect(tokens[2].type).toBe(ReqonTokenType.UPSERT);
+    });
+  });
+
+  describe('whitespace handling', () => {
+    it('skips spaces and tabs', () => {
+      const lexer = new ReqonLexer('   mission   \t  action   ');
+      const tokens = lexer.tokenize();
+
+      const nonEofTokens = tokens.filter((t: Token) => t.type !== TokenType.EOF);
+      expect(nonEofTokens).toHaveLength(2);
+      expect(nonEofTokens[0].type).toBe(ReqonTokenType.MISSION);
+      expect(nonEofTokens[1].type).toBe(ReqonTokenType.ACTION);
+    });
+
+    it('tracks newlines as tokens', () => {
+      const lexer = new ReqonLexer('a\nb\nc');
+      const tokens = lexer.tokenize();
+
+      expect(tokens.filter((t: Token) => t.type === TokenType.NEWLINE)).toHaveLength(2);
+    });
+  });
+});

package/src/lexer/tokens.ts

@@ -0,0 +1,179 @@
+import { TokenType as VagueTokenType } from 'vague-lang';
+
+// Extend Vague's TokenType with Reqon-specific tokens
+export enum ReqonTokenType {
+  // Reqon keywords
+  MISSION = 'MISSION',
+  ACTION = 'ACTION',
+  SOURCE = 'SOURCE',
+  STORE = 'STORE',
+  CALL = 'CALL',
+  RUN = 'RUN',
+  FOR = 'FOR',
+  MAP = 'MAP',
+  EACH = 'EACH',
+  PAGINATE = 'PAGINATE',
+  UNTIL = 'UNTIL',
+  RETRY = 'RETRY',
+  KEY = 'KEY',
+  PARTIAL = 'PARTIAL',
+  UPSERT = 'UPSERT',
+  SINCE = 'SINCE',
+  LAST_SYNC = 'LAST_SYNC',
+
+  // HTTP methods
+  GET = 'GET',
+  POST = 'POST',
+  PUT = 'PUT',
+  PATCH = 'PATCH',
+  DELETE = 'DELETE',
+
+  // Store types
+  NOSQL = 'NOSQL',
+  SQL = 'SQL',
+  MEMORY = 'MEMORY',
+
+  // Auth types
+  OAUTH2 = 'OAUTH2',
+  BEARER = 'BEARER',
+  BASIC = 'BASIC',
+  API_KEY = 'API_KEY',
+  NONE = 'NONE',
+
+  // Pagination types
+  OFFSET = 'OFFSET',
+  CURSOR = 'CURSOR',
+  PAGE = 'PAGE',
+
+  // Additional operators (RIGHT_ARROW is now in Vague)
+  NOT_EQUALS = 'NOT_EQUALS', // !=
+  BANG = 'BANG', // !
+  // OAS integration
+  FROM = 'FROM',
+
+  // Scheduling
+  SCHEDULE = 'SCHEDULE',
+  CRON = 'CRON',
+  EVERY = 'EVERY',
+  AT = 'AT',
+  HOURS = 'HOURS',
+  MINUTES = 'MINUTES',
+  SECONDS = 'SECONDS',
+  DAYS = 'DAYS',
+  WEEKS = 'WEEKS',
+
+  // Flow control directives
+  CONTINUE = 'CONTINUE',
+  SKIP = 'SKIP',
+  ABORT = 'ABORT',
+  QUEUE = 'QUEUE',
+  JUMP = 'JUMP',
+
+  // Type checking
+  IS = 'IS',
+
+  // Webhook support
+  WAIT = 'WAIT',
+  TIMEOUT = 'TIMEOUT',
+  PATH = 'PATH',
+  EXPECTED_EVENTS = 'EXPECTED_EVENTS',
+  EVENT_FILTER = 'EVENT_FILTER',
+  STORAGE = 'STORAGE',
+}
+
+// Combined token type
+export type TokenType = VagueTokenType | ReqonTokenType;
+
+// Re-export Vague's TokenType for convenience
+export { TokenType as VagueTokenType } from 'vague-lang';
+
+// Reqon-specific keywords
+export const REQON_KEYWORDS: Record<string, ReqonTokenType> = {
+  mission: ReqonTokenType.MISSION,
+  action: ReqonTokenType.ACTION,
+  source: ReqonTokenType.SOURCE,
+  store: ReqonTokenType.STORE,
+  call: ReqonTokenType.CALL,
+  run: ReqonTokenType.RUN,
+  for: ReqonTokenType.FOR,
+  map: ReqonTokenType.MAP,
+  each: ReqonTokenType.EACH,
+  paginate: ReqonTokenType.PAGINATE,
+  until: ReqonTokenType.UNTIL,
+  retry: ReqonTokenType.RETRY,
+  key: ReqonTokenType.KEY,
+  partial: ReqonTokenType.PARTIAL,
+  upsert: ReqonTokenType.UPSERT,
+  since: ReqonTokenType.SINCE,
+  lastSync: ReqonTokenType.LAST_SYNC,
+
+  // HTTP methods (both lowercase and uppercase)
+  get: ReqonTokenType.GET,
+  post: ReqonTokenType.POST,
+  put: ReqonTokenType.PUT,
+  patch: ReqonTokenType.PATCH,
+  delete: ReqonTokenType.DELETE,
+  GET: ReqonTokenType.GET,
+  POST: ReqonTokenType.POST,
+  PUT: ReqonTokenType.PUT,
+  PATCH: ReqonTokenType.PATCH,
+  DELETE: ReqonTokenType.DELETE,
+
+  // Store types
+  nosql: ReqonTokenType.NOSQL,
+  sql: ReqonTokenType.SQL,
+  memory: ReqonTokenType.MEMORY,
+
+  // Auth types
+  oauth2: ReqonTokenType.OAUTH2,
+  bearer: ReqonTokenType.BEARER,
+  basic: ReqonTokenType.BASIC,
+  api_key: ReqonTokenType.API_KEY,
+  none: ReqonTokenType.NONE,
+
+  // Pagination types
+  offset: ReqonTokenType.OFFSET,
+  cursor: ReqonTokenType.CURSOR,
+  page: ReqonTokenType.PAGE,
+
+  // OAS integration
+  from: ReqonTokenType.FROM,
+
+  // Scheduling
+  schedule: ReqonTokenType.SCHEDULE,
+  cron: ReqonTokenType.CRON,
+  every: ReqonTokenType.EVERY,
+  at: ReqonTokenType.AT,
+  hours: ReqonTokenType.HOURS,
+  minutes: ReqonTokenType.MINUTES,
+  seconds: ReqonTokenType.SECONDS,
+  days: ReqonTokenType.DAYS,
+  weeks: ReqonTokenType.WEEKS,
+
+  // Flow control directives
+  continue: ReqonTokenType.CONTINUE,
+  skip: ReqonTokenType.SKIP,
+  abort: ReqonTokenType.ABORT,
+  queue: ReqonTokenType.QUEUE,
+  jump: ReqonTokenType.JUMP,
+
+  // Type checking
+  is: ReqonTokenType.IS,
+
+  // Webhook support
+  wait: ReqonTokenType.WAIT,
+  timeout: ReqonTokenType.TIMEOUT,
+  path: ReqonTokenType.PATH,
+  expectedEvents: ReqonTokenType.EXPECTED_EVENTS,
+  eventFilter: ReqonTokenType.EVENT_FILTER,
+  storage: ReqonTokenType.STORAGE,
+};
+
+// ReqonToken is now an alias for Vague's Token
+// Token type can be a string (for plugin-registered keywords)
+export interface ReqonToken {
+  type: TokenType | string;
+  value: string;
+  line: number;
+  column: number;
+}