tree-sitter-lispex 0.1.0 → 0.1.1

package/src/scanner.c

@@ -1,4 +1,5 @@
 #include "tree_sitter/parser.h"
+#include "unicode.h"
 
 #include <stdbool.h>
 #include <stdint.h>
@@ -8,6 +9,8 @@
 enum TokenType {
   BLOCK_COMMENT,
   CHARACTER,
+  MALFORMED_CHARACTER,
+  CHARACTER_END,
   BOOLEAN,
   BYTE,
   INTEGER,
@@ -36,8 +39,11 @@ void tree_sitter_lispex_external_scanner_deserialize(void *payload,
 }
 
 static bool is_space(int32_t c) {
-  return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' ||
-         c == 0xFEFF;
+  // Rust char::is_whitespace (Unicode White_Space); U+FEFF is a constituent.
+  return (c >= 0x09 && c <= 0x0D) || c == 0x20 || c == 0x85 ||
+         c == 0xA0 || c == 0x1680 || (c >= 0x2000 && c <= 0x200A) ||
+         c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F ||
+         c == 0x3000;
 }
 
 static bool is_delimiter(int32_t c) {
@@ -46,11 +52,6 @@ static bool is_delimiter(int32_t c) {
          c == '\'' || c == '`' || c == ',';
 }
 
-static bool is_ascii_alphanumeric(int32_t c) {
-  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
-         (c >= '0' && c <= '9');
-}
-
 static bool is_hex(int32_t c) {
   return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
          (c >= 'A' && c <= 'F');
@@ -108,38 +109,32 @@ static bool scan_character_after_hash(TSLexer *lexer) {
 
   int32_t first = lexer->lookahead;
   lexer->advance(lexer, false);
-  if (!is_ascii_alphanumeric(first) || first >= 128) {
+  if (!is_unicode_alphanumeric(first)) {
     lexer->result_symbol = CHARACTER;
     return true;
   }
 
   char name[64];
   size_t length = 0;
-  name[length++] = (char)first;
+  bool ascii_only = first < 128;
+  bool hex = first == 'x' || first == 'X';
+  bool has_tail = false;
+  name[length++] = ascii_only ? (char)first : '\0';
   while (!is_delimiter(lexer->lookahead) && lexer->lookahead != '|') {
-    if (lexer->lookahead >= 128 || length + 1 >= sizeof(name)) {
-      return false;
+    has_tail = true;
+    ascii_only = ascii_only && lexer->lookahead < 128;
+    hex = hex && is_hex(lexer->lookahead);
+    // Only named forms need buffering. Consume even arbitrarily long malformed
+    // names and hexadecimal spellings as one token, without truncating it.
+    if (length + 1 < sizeof(name)) {
+      name[length++] = lexer->lookahead < 128 ? (char)lexer->lookahead : '\0';
     }
-    name[length++] = (char)lexer->lookahead;
     lexer->advance(lexer, false);
   }
   name[length] = '\0';
 
-  bool valid = length == 1;
-  if (!valid && (name[0] == 'x' || name[0] == 'X')) {
-    valid = length > 1;
-    for (size_t index = 1; index < length && valid; index++) {
-      valid = is_hex(name[index]);
-    }
-  }
-  if (!valid) {
-    valid = is_named_character(name);
-  }
-  if (!valid) {
-    return false;
-  }
-
-  lexer->result_symbol = CHARACTER;
+  bool valid = !has_tail || hex || (ascii_only && is_named_character(name));
+  lexer->result_symbol = valid ? CHARACTER : MALFORMED_CHARACTER;
   return true;
 }
 
@@ -329,7 +324,7 @@ static bool scan_atom(TSLexer *lexer, const bool *valid_symbols) {
     lexer->result_symbol = INTEGER;
     matched = true;
   } else if (valid_symbols[SYMBOL] && text[0] != '#' &&
-             !(ascii_only && starts_number_like(text)) &&
+             !starts_number_like(text) &&
              !(ascii_only && strcmp(text, ".") == 0)) {
     lexer->result_symbol = SYMBOL;
     matched = true;
@@ -349,7 +344,8 @@ bool tree_sitter_lispex_external_scanner_scan(void *payload, TSLexer *lexer,
     if (lexer->lookahead == '|' && valid_symbols[BLOCK_COMMENT]) {
       return scan_block_comment_after_hash(lexer);
     }
-    if (lexer->lookahead == '\\' && valid_symbols[CHARACTER]) {
+    if (lexer->lookahead == '\\' &&
+        (valid_symbols[CHARACTER] || valid_symbols[MALFORMED_CHARACTER])) {
       return scan_character_after_hash(lexer);
     }
     return scan_hash_token_after_hash(lexer, valid_symbols);