tree-sitter-beancount 2.0.0 → 2.1.2

package/src/scanner.cc

@@ -0,0 +1,163 @@
+#include <cwctype>
+#include <tree_sitter/parser.h>
+#include <vector>
+
+namespace {
+
+using std::iswspace;
+using std::vector;
+
+enum TokenType {
+    HL_STARS,
+    SECTIONEND,
+    END_OF_FILE,
+};
+
+struct Scanner {
+    vector<int16_t> indent_length_stack;
+    vector<int16_t> org_section_stack;
+
+    Scanner() {
+        deserialize(NULL, 0);
+    }
+
+    unsigned serialize(char *buffer) {
+        size_t i = 0;
+
+        size_t indent_count = indent_length_stack.size() - 1;
+        if (indent_count > UINT8_MAX)
+            indent_count = UINT8_MAX;
+        buffer[i++] = indent_count;
+
+        vector<int16_t>::iterator iter = indent_length_stack.begin() + 1,
+                                  end = indent_length_stack.end();
+
+        for (; iter != end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE;
+             ++iter) {
+            buffer[i++] = *iter;
+        }
+
+        iter = org_section_stack.begin() + 1;
+        end = org_section_stack.end();
+
+        for (; iter != end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE;
+             ++iter) {
+            buffer[i++] = *iter;
+        }
+
+        return i;
+    }
+
+    void deserialize(const char *buffer, unsigned length) {
+        org_section_stack.clear();
+        org_section_stack.push_back(0);
+        indent_length_stack.clear();
+        indent_length_stack.push_back(-1);
+
+        if (length == 0)
+            return;
+
+        size_t i = 0;
+
+        size_t indent_count = (uint8_t)buffer[i++];
+
+        for (; i <= indent_count; i++)
+            indent_length_stack.push_back(buffer[i]);
+        for (; i < length; i++)
+            org_section_stack.push_back(buffer[i]);
+    }
+
+    void advance(TSLexer *lexer) {
+        lexer->advance(lexer, false);
+    }
+
+    void skip(TSLexer *lexer) {
+        lexer->advance(lexer, true);
+    }
+
+    bool scan(TSLexer *lexer, const bool *valid_symbols) {
+
+        // - Section ends
+        int16_t indent_length = 0;
+        lexer->mark_end(lexer);
+        for (;;) {
+            if (lexer->lookahead == ' ') {
+                indent_length++;
+            } else if (lexer->lookahead == '\t') {
+                indent_length += 8;
+            } else if (lexer->lookahead == '\0') {
+
+                if (valid_symbols[SECTIONEND]) {
+                    lexer->result_symbol = SECTIONEND;
+                } else if (valid_symbols[END_OF_FILE]) {
+                    lexer->result_symbol = END_OF_FILE;
+                } else {
+                    return false;
+                }
+
+                return true;
+            } else {
+                break;
+            }
+            skip(lexer);
+        }
+
+        if (indent_length == 0 && lexer->lookahead == '*') {
+            lexer->mark_end(lexer);
+            int16_t stars = 1;
+            skip(lexer);
+            while (lexer->lookahead == '*') {
+                stars++;
+                skip(lexer);
+            }
+
+            if (valid_symbols[SECTIONEND] && iswspace(lexer->lookahead)
+                && stars > 0 && stars <= org_section_stack.back()) {
+                org_section_stack.pop_back();
+                lexer->result_symbol = SECTIONEND;
+                return true;
+            } else if (valid_symbols[HL_STARS] && iswspace(lexer->lookahead)) {
+                org_section_stack.push_back(stars);
+                lexer->result_symbol = HL_STARS;
+                return true;
+            }
+            return false;
+        }
+
+        return false; // default
+    }
+};
+
+} // namespace
+
+extern "C" {
+
+void *tree_sitter_beancount_external_scanner_create() {
+    return new Scanner();
+}
+
+bool tree_sitter_beancount_external_scanner_scan(void *payload,
+                                                 TSLexer *lexer,
+                                                 const bool *valid_symbols) {
+    Scanner *scanner = static_cast<Scanner *>(payload);
+    return scanner->scan(lexer, valid_symbols);
+}
+
+unsigned tree_sitter_beancount_external_scanner_serialize(void *payload,
+                                                          char *buffer) {
+    Scanner *scanner = static_cast<Scanner *>(payload);
+    return scanner->serialize(buffer);
+}
+
+void tree_sitter_beancount_external_scanner_deserialize(void *payload,
+                                                        const char *buffer,
+                                                        unsigned length) {
+    Scanner *scanner = static_cast<Scanner *>(payload);
+    scanner->deserialize(buffer, length);
+}
+
+void tree_sitter_beancount_external_scanner_destroy(void *payload) {
+    Scanner *scanner = static_cast<Scanner *>(payload);
+    delete scanner;
+}
+}

package/src/tree_sitter/parser.h

@@ -13,6 +13,8 @@ extern "C" {
 #define ts_builtin_sym_end 0
 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
 
+typedef uint16_t TSStateId;
+
 #ifndef TREE_SITTER_API_H_
 typedef uint16_t TSSymbol;
 typedef uint16_t TSFieldId;
@@ -30,11 +32,10 @@ typedef struct {
   uint16_t length;
 } TSFieldMapSlice;
 
-typedef uint16_t TSStateId;
-
 typedef struct {
-  bool visible : 1;
-  bool named : 1;
+  bool visible;
+  bool named;
+  bool supertype;
 } TSSymbolMetadata;
 
 typedef struct TSLexer TSLexer;
@@ -56,21 +57,21 @@ typedef enum {
   TSParseActionTypeRecover,
 } TSParseActionType;
 
-typedef struct {
-  union {
-    struct {
-      TSStateId state;
-      bool extra : 1;
-      bool repetition : 1;
-    } shift;
-    struct {
-      TSSymbol symbol;
-      int16_t dynamic_precedence;
-      uint8_t child_count;
-      uint8_t production_id;
-    } reduce;
-  } params;
-  TSParseActionType type : 4;
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
 } TSParseAction;
 
 typedef struct {
@@ -82,7 +83,7 @@ typedef union {
   TSParseAction action;
   struct {
     uint8_t count;
-    bool reusable : 1;
+    bool reusable;
   } entry;
 } TSParseActionEntry;
 
@@ -92,13 +93,24 @@ struct TSLanguage {
   uint32_t alias_count;
   uint32_t token_count;
   uint32_t external_token_count;
-  const char **symbol_names;
-  const TSSymbolMetadata *symbol_metadata;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
   const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
   const TSParseActionEntry *parse_actions;
-  const TSLexMode *lex_modes;
+  const char * const *symbol_names;
+  const char * const *field_names;
+  const TSFieldMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
   const TSSymbol *alias_sequences;
-  uint16_t max_alias_sequence_length;
+  const TSLexMode *lex_modes;
   bool (*lex_fn)(TSLexer *, TSStateId);
   bool (*keyword_lex_fn)(TSLexer *, TSStateId);
   TSSymbol keyword_capture_token;
@@ -111,14 +123,7 @@ struct TSLanguage {
     unsigned (*serialize)(void *, char *);
     void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
-  uint32_t field_count;
-  const TSFieldMapSlice *field_map_slices;
-  const TSFieldMapEntry *field_map_entries;
-  const char **field_names;
-  uint32_t large_state_count;
-  const uint16_t *small_parse_table;
-  const uint32_t *small_parse_table_map;
-  const TSSymbol *public_symbol_map;
+  const TSStateId *primary_state_ids;
 };
 
 /*
@@ -167,66 +172,50 @@ struct TSLanguage {
 
 #define ACTIONS(id) id
 
-#define SHIFT(state_value)                \
-  {                                       \
-    {                                     \
-      .params = {                         \
-        .shift = {                        \
-          .state = state_value            \
-        }                                 \
-      },                                  \
-      .type = TSParseActionTypeShift      \
-    }                                     \
-  }
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = state_value            \
+    }                                 \
+  }}
 
 #define SHIFT_REPEAT(state_value)     \
-  {                                   \
-    {                                 \
-      .params = {                     \
-        .shift = {                    \
-          .state = state_value,       \
-          .repetition = true          \
-        }                             \
-      },                              \
-      .type = TSParseActionTypeShift  \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = state_value,           \
+      .repetition = true              \
     }                                 \
-  }
-
-#define RECOVER()                        \
-  {                                      \
-    { .type = TSParseActionTypeRecover } \
-  }
+  }}
 
 #define SHIFT_EXTRA()                 \
-  {                                   \
-    {                                 \
-      .params = {                     \
-        .shift = {                    \
-          .extra = true               \
-        }                             \
-      },                              \
-      .type = TSParseActionTypeShift  \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
     }                                 \
-  }
+  }}
 
 #define REDUCE(symbol_val, child_count_val, ...) \
-  {                                              \
-    {                                            \
-      .params = {                                \
-        .reduce = {                              \
-          .symbol = symbol_val,                  \
-          .child_count = child_count_val,        \
-          __VA_ARGS__                            \
-        },                                       \
-      },                                         \
-      .type = TSParseActionTypeReduce            \
-    }                                            \
-  }
-
-#define ACCEPT_INPUT()                  \
-  {                                     \
-    { .type = TSParseActionTypeAccept } \
-  }
+  {{                                             \
+    .reduce = {                                  \
+      .type = TSParseActionTypeReduce,           \
+      .symbol = symbol_val,                      \
+      .child_count = child_count_val,            \
+      __VA_ARGS__                                \
+    },                                           \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
 
 #ifdef __cplusplus
 }

package/test/corpus/arithmetic.txt

@@ -11,9 +11,8 @@ expr add
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
@@ -28,7 +27,6 @@ expr add
                     (currency)
                 )
             )
-        )
     )
 )
 
@@ -45,9 +43,8 @@ expr sub
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
@@ -62,7 +59,6 @@ expr sub
                     (currency)
                 )
             )
-        )
     )
 )
 
@@ -79,24 +75,22 @@ expr mult
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
-                    (binary_number_expr (number) (asterick) (number))
+                    (binary_number_expr (number) (asterisk) (number))
                     (currency)
                 )
             )
             (posting
                 (account)
                 (incomplete_amount
-                    (binary_number_expr (number) (asterick) (number))
+                    (binary_number_expr (number) (asterisk) (number))
                     (currency)
                 )
             )
-        )
     )
 )
 
@@ -113,9 +107,8 @@ expr div
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
@@ -130,7 +123,6 @@ expr div
                     (currency)
                 )
             )
-        )
     )
 )
 
@@ -148,9 +140,8 @@ expr neg
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
@@ -172,7 +163,6 @@ expr neg
                     (currency)
                 )
             )
-        )
     )
 )
 
@@ -189,9 +179,8 @@ expr pos
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
@@ -206,7 +195,6 @@ expr pos
                     (currency)
                 )
             )
-        )
     )
 )
 
@@ -225,14 +213,13 @@ expr predence
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
                     (binary_number_expr
-                        (binary_number_expr (number) (asterick) (number))
+                        (binary_number_expr (number) (asterisk) (number))
                         (plus)
                         (number)
                     )
@@ -243,9 +230,9 @@ expr predence
                 (account)
                 (incomplete_amount
                     (binary_number_expr
-                        (binary_number_expr (number) (plus) (number))
-                        (asterick)
                         (number)
+                        (plus)
+                        (binary_number_expr (number) (asterisk) (number))
                     )
                     (currency)
                 )
@@ -254,9 +241,9 @@ expr predence
                 (account)
                 (incomplete_amount
                     (binary_number_expr
-                        (binary_number_expr (number) (plus) (minus) (number))
-                        (asterick)
                         (number)
+                        (plus)
+                        (binary_number_expr (unary_number_expr (minus) (number)) (asterisk) (number))
                     )
                     (currency)
                 )
@@ -265,14 +252,13 @@ expr predence
                 (account)
                 (incomplete_amount
                     (binary_number_expr
-                        (binary_number_expr (number) (plus) (minus) (number))
-                        (asterick)
+                        (binary_number_expr (number) (plus) (unary_number_expr (minus) (number)))
+                        (asterisk)
                         (number)
                     )
                     (currency)
                 )
             )
-        )
     )
 )
 
@@ -289,15 +275,14 @@ expr groups
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
                     (binary_number_expr
-                        (binary_number_expr (number) (plus) (minus) (number))
-                        (asterick)
+                        (binary_number_expr (number) (plus) (unary_number_expr (minus) (number)))
+                        (asterisk)
                         (number)
                     )
                     (currency)
@@ -308,13 +293,12 @@ expr groups
                 (incomplete_amount
                     (binary_number_expr
                         (number)
-                        (asterick)
-                        (binary_number_expr (number) (plus) (minus) (number))
+                        (asterisk)
+                        (binary_number_expr (number) (plus) (unary_number_expr (minus) (number)))
                     )
                     (currency)
                 )
             )
-        )
     )
 )
 
@@ -325,7 +309,8 @@ expr different places
 2013-05-18 * "Test"
     Assets:Something   -(3 * 4) HOOL {120.01 * 2.1 USD} @ 134.02 * 2.1 USD
     Assets:Something   1000000 USD ;; No balance checks.
-    2014-01-01 balance Assets:Something  3 * 4 * 120.01 * 2.1  USD
+
+2014-01-01 balance Assets:Something  3 * 4 * 120.01 * 2.1  USD
     number: -(5662.23 + 22.3)
 
 ---
@@ -333,23 +318,56 @@ expr different places
 (file
     (transaction
         (date)
-        (flag)
-        (txn_strings (string))
-        (postings
+        (txn)
+        (narration)
             (posting
                 (account)
                 (incomplete_amount
-                    (binary_number_expr (number) (number))
+                    (unary_number_expr (minus) (binary_number_expr (number) (asterisk) (number)))
                     (currency)
                 )
+                (cost_spec
+                        (cost_comp
+                            (compound_amount
+                                (binary_number_expr (number) (asterisk) (number))
+                                (currency)
+                            )
+                        )
+                )
+                (at)
+                (price_annotation
+                    (incomplete_amount
+                        (binary_number_expr (number) (asterisk) (number))
+                        (currency)
+                    )
+                )
             )
             (posting
                 (account)
-                (incomplete_amount
-                    (binary_number_expr (number) (number))
-                    (currency)
+                (incomplete_amount (number) (currency))
+                (comment)
+            )
+    )
+    (balance
+        (date)
+        (account)
+        (amount_tolerance
+            (binary_number_expr
+                (binary_number_expr
+                    (binary_number_expr (number) (asterisk) (number))
+                    (asterisk)
+                    (number)
                 )
+                (asterisk)
+                (number)
             )
+            (currency)
         )
+            (key_value
+                (key)
+                (value
+                    (unary_number_expr (minus) (binary_number_expr (number) (plus) (number)))
+                )
+            )
     )
 )