tree-sitter-beancount 2.1.3 → 2.3.0

package/src/scanner.c

@@ -0,0 +1,208 @@
+#include <stdio.h> // test
+#include <tree_sitter/parser.h>
+#include <wctype.h>
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+typedef struct {
+    uint32_t length;
+    uint32_t capacity;
+    int16_t *data;
+} vec;
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+#define VEC_RESIZE(vec, _cap)                                                  \
+    void *tmp = realloc((vec).data, (_cap) * sizeof((vec).data[0]));           \
+    (vec).data = tmp;                                                          \
+    (vec).capacity = (_cap);
+
+#define VEC_GROW(vec, _cap)                                                    \
+    if ((vec).capacity < (_cap)) {                                             \
+        VEC_RESIZE((vec), (_cap));                                             \
+    }
+
+#define VEC_PUSH(vec, el)                                                      \
+    if ((vec).capacity == (vec).length) {                                      \
+        VEC_RESIZE((vec), MAX(16, (vec).length * 2));                          \
+    }                                                                          \
+    (vec).data[(vec).length++] = (el);
+
+#define VEC_POP(vec) (vec).length--;
+
+#define VEC_NEW                                                                \
+    { .len = 0, .capacity = 0, .data = NULL }
+
+#define VEC_BACK(vec) ((vec).data[(vec).length - 1])
+
+#define VEC_FREE(vec)                                                          \
+    {                                                                          \
+        if ((vec).data != NULL)                                                \
+            free((vec).data);                                                  \
+    }
+
+#define VEC_CLEAR(vec) (vec).length = 0;
+
+enum TokenType {
+    SECTION,
+    SECTIONEND,
+    END_OF_FILE,
+};
+
+typedef struct {
+    vec indent_length_stack;
+    vec org_section_stack;
+} Scanner;
+
+unsigned serialize(Scanner *scanner, char *buffer) {
+    size_t i = 0;
+
+    size_t indent_count = scanner->indent_length_stack.length - 1;
+    if (indent_count > UINT8_MAX)
+        indent_count = UINT8_MAX;
+    buffer[i++] = (char)indent_count;
+
+    int iter = 1;
+    for (; iter < scanner->indent_length_stack.length
+           && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE;
+         ++iter) {
+        buffer[i++] = (char)scanner->indent_length_stack.data[iter];
+    }
+
+    size_t org_section_count = scanner->org_section_stack.length - 1;
+    if (org_section_count > UINT8_MAX)
+        org_section_count = UINT8_MAX;
+    buffer[i++] = (char)org_section_count;
+
+    iter = 1;
+    for (; iter < scanner->org_section_stack.length
+           && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE;
+         ++iter) {
+        buffer[i++] = (char)scanner->org_section_stack.data[iter];
+    }
+
+    return i;
+}
+
+void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
+    VEC_CLEAR(scanner->org_section_stack);
+    VEC_PUSH(scanner->org_section_stack, 0);
+    VEC_CLEAR(scanner->indent_length_stack);
+    VEC_PUSH(scanner->indent_length_stack, 0);
+
+    if (length == 0)
+        return;
+
+    size_t i = 0;
+
+    size_t indent_count = (unsigned char)buffer[i++];
+    for (; i <= indent_count; i++) {
+        VEC_PUSH(scanner->indent_length_stack, (unsigned char)buffer[i]);
+    }
+
+    size_t org_section_count = (unsigned char)buffer[i++];
+    for (; i < length; i++) {
+        VEC_PUSH(scanner->org_section_stack, (unsigned char)buffer[i]);
+    }
+}
+
+void advance(TSLexer *lexer) {
+    lexer->advance(lexer, false);
+}
+
+void skip(TSLexer *lexer) {
+    lexer->advance(lexer, true);
+}
+
+static bool in_error_recovery(const bool *valid_symbols) {
+    return (valid_symbols[SECTION] && valid_symbols[SECTIONEND]
+            && valid_symbols[END_OF_FILE]);
+}
+
+bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
+
+    if (in_error_recovery(valid_symbols))
+        return false;
+
+    // - Section ends
+    int16_t indent_length = 0;
+    lexer->mark_end(lexer);
+    for (;;) {
+        if (lexer->lookahead == ' ') {
+            indent_length++;
+        } else if (lexer->lookahead == '\t') {
+            indent_length += 8;
+        } else if (lexer->lookahead == '\0') {
+
+            if (valid_symbols[SECTIONEND]) {
+                lexer->result_symbol = SECTIONEND;
+            } else if (valid_symbols[END_OF_FILE]) {
+                lexer->result_symbol = END_OF_FILE;
+            } else {
+                return false;
+            }
+
+            return true;
+        } else {
+            break;
+        }
+        skip(lexer);
+    }
+
+    if (indent_length == 0 && lexer->lookahead == '*') {
+        lexer->mark_end(lexer);
+        int16_t stars = 1;
+        skip(lexer);
+        while (lexer->lookahead == '*') {
+            stars++;
+            skip(lexer);
+        }
+
+        if (valid_symbols[SECTIONEND] && iswspace(lexer->lookahead) && stars > 0
+            && stars <= VEC_BACK(scanner->org_section_stack)) {
+            VEC_POP(scanner->org_section_stack);
+            lexer->result_symbol = SECTIONEND;
+            return true;
+        } else if (valid_symbols[SECTION] && iswspace(lexer->lookahead)) {
+            VEC_PUSH(scanner->org_section_stack, stars);
+            lexer->result_symbol = SECTION;
+            return true;
+        }
+        return false;
+    }
+
+    return false; // default
+}
+
+void *tree_sitter_beancount_external_scanner_create() {
+    Scanner *scanner = (Scanner *)calloc(1, sizeof(Scanner));
+    deserialize(scanner, NULL, 0);
+    return scanner;
+}
+
+bool tree_sitter_beancount_external_scanner_scan(void *payload,
+                                                 TSLexer *lexer,
+                                                 const bool *valid_symbols) {
+    Scanner *scanner = (Scanner *)payload;
+    return scan(scanner, lexer, valid_symbols);
+}
+
+unsigned tree_sitter_beancount_external_scanner_serialize(void *payload,
+                                                          char *buffer) {
+    Scanner *scanner = (Scanner *)payload;
+    return serialize(scanner, buffer);
+}
+
+void tree_sitter_beancount_external_scanner_deserialize(void *payload,
+                                                        const char *buffer,
+                                                        unsigned length) {
+    Scanner *scanner = (Scanner *)payload;
+    deserialize(scanner, buffer, length);
+}
+
+void tree_sitter_beancount_external_scanner_destroy(void *payload) {
+    Scanner *scanner = (Scanner *)payload;
+    VEC_FREE(scanner->indent_length_stack);
+    VEC_FREE(scanner->org_section_stack);
+    free(scanner);
+}

package/src/tree_sitter/parser.h

@@ -13,9 +13,8 @@ extern "C" {
 #define ts_builtin_sym_end 0
 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
 
-typedef uint16_t TSStateId;
-
 #ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
 typedef struct TSLanguage TSLanguage;
@@ -130,9 +129,16 @@ struct TSLanguage {
  *  Lexer Macros
  */
 
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
 #define START_LEXER()           \
   bool result = false;          \
   bool skip = false;            \
+  UNUSED                        \
   bool eof = false;             \
   int32_t lookahead;            \
   goto start;                   \
@@ -166,7 +172,7 @@ struct TSLanguage {
  *  Parse Table Macros
  */
 
-#define SMALL_STATE(id) id - LARGE_STATE_COUNT
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
 
 #define STATE(id) id
 
@@ -176,7 +182,7 @@ struct TSLanguage {
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = state_value            \
+      .state = (state_value)          \
     }                                 \
   }}
 
@@ -184,7 +190,7 @@ struct TSLanguage {
   {{                                  \
     .shift = {                        \
       .type = TSParseActionTypeShift, \
-      .state = state_value,           \
+      .state = (state_value),         \
       .repetition = true              \
     }                                 \
   }}

package/src/scanner.cc

@@ -1,171 +0,0 @@
-#include <cwctype>
-#include <tree_sitter/parser.h>
-#include <vector>
-
-namespace {
-
-using std::iswspace;
-using std::vector;
-
-enum TokenType {
-    SECTION,
-    SECTIONEND,
-    END_OF_FILE,
-};
-
-struct Scanner {
-    vector<int16_t> indent_length_stack;
-    vector<int16_t> org_section_stack;
-
-    Scanner() {
-        deserialize(NULL, 0);
-    }
-
-    unsigned serialize(char *buffer) {
-        size_t i = 0;
-
-        size_t indent_count = indent_length_stack.size() - 1;
-        if (indent_count > UINT8_MAX)
-            indent_count = UINT8_MAX;
-        buffer[i++] = indent_count;
-
-        vector<int16_t>::iterator iter = indent_length_stack.begin() + 1,
-                                  end = indent_length_stack.end();
-
-        for (; iter != end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE;
-             ++iter) {
-            buffer[i++] = *iter;
-        }
-
-        iter = org_section_stack.begin() + 1;
-        end = org_section_stack.end();
-
-        for (; iter != end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE;
-             ++iter) {
-            buffer[i++] = *iter;
-        }
-
-        return i;
-    }
-
-    void deserialize(const char *buffer, unsigned length) {
-        org_section_stack.clear();
-        org_section_stack.push_back(0);
-        indent_length_stack.clear();
-        indent_length_stack.push_back(-1);
-
-        if (length == 0)
-            return;
-
-        size_t i = 0;
-
-        size_t indent_count = (uint8_t)buffer[i++];
-
-        for (; i <= indent_count; i++)
-            indent_length_stack.push_back(buffer[i]);
-        for (; i < length; i++)
-            org_section_stack.push_back(buffer[i]);
-    }
-
-    void advance(TSLexer *lexer) {
-        lexer->advance(lexer, false);
-    }
-
-    void skip(TSLexer *lexer) {
-        lexer->advance(lexer, true);
-    }
-
-    static bool in_error_recovery(const bool *valid_symbols) {
-        return (valid_symbols[SECTION] && valid_symbols[SECTIONEND]
-                && valid_symbols[END_OF_FILE]);
-    }
-
-    bool scan(TSLexer *lexer, const bool *valid_symbols) {
-
-        if (in_error_recovery(valid_symbols))
-            return false;
-
-        // - Section ends
-        int16_t indent_length = 0;
-        lexer->mark_end(lexer);
-        for (;;) {
-            if (lexer->lookahead == ' ') {
-                indent_length++;
-            } else if (lexer->lookahead == '\t') {
-                indent_length += 8;
-            } else if (lexer->lookahead == '\0') {
-
-                if (valid_symbols[SECTIONEND]) {
-                    lexer->result_symbol = SECTIONEND;
-                } else if (valid_symbols[END_OF_FILE]) {
-                    lexer->result_symbol = END_OF_FILE;
-                } else {
-                    return false;
-                }
-
-                return true;
-            } else {
-                break;
-            }
-            skip(lexer);
-        }
-
-        if (indent_length == 0 && lexer->lookahead == '*') {
-            lexer->mark_end(lexer);
-            int16_t stars = 1;
-            skip(lexer);
-            while (lexer->lookahead == '*') {
-                stars++;
-                skip(lexer);
-            }
-
-            if (valid_symbols[SECTIONEND] && iswspace(lexer->lookahead)
-                && stars > 0 && stars <= org_section_stack.back()) {
-                org_section_stack.pop_back();
-                lexer->result_symbol = SECTIONEND;
-                return true;
-            } else if (valid_symbols[SECTION] && iswspace(lexer->lookahead)) {
-                org_section_stack.push_back(stars);
-                lexer->result_symbol = SECTION;
-                return true;
-            }
-            return false;
-        }
-
-        return false; // default
-    }
-};
-
-} // namespace
-
-extern "C" {
-
-void *tree_sitter_beancount_external_scanner_create() {
-    return new Scanner();
-}
-
-bool tree_sitter_beancount_external_scanner_scan(void *payload,
-                                                 TSLexer *lexer,
-                                                 const bool *valid_symbols) {
-    Scanner *scanner = static_cast<Scanner *>(payload);
-    return scanner->scan(lexer, valid_symbols);
-}
-
-unsigned tree_sitter_beancount_external_scanner_serialize(void *payload,
-                                                          char *buffer) {
-    Scanner *scanner = static_cast<Scanner *>(payload);
-    return scanner->serialize(buffer);
-}
-
-void tree_sitter_beancount_external_scanner_deserialize(void *payload,
-                                                        const char *buffer,
-                                                        unsigned length) {
-    Scanner *scanner = static_cast<Scanner *>(payload);
-    scanner->deserialize(buffer, length);
-}
-
-void tree_sitter_beancount_external_scanner_destroy(void *payload) {
-    Scanner *scanner = static_cast<Scanner *>(payload);
-    delete scanner;
-}
-}