rapydscript-ng 0.7.24 → 0.8.1

package/tree-sitter/src/scanner.c

@@ -0,0 +1,294 @@
+#include "tree_sitter/parser.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+// External tokens produced by this scanner.
+enum TokenType {
+    NEWLINE,
+    INDENT,
+    DEDENT,
+    REGEX,
+};
+
+// A simple growable stack of indentation widths. The bottom of the stack is
+// always 0 (the module level indentation).
+typedef struct {
+    uint32_t size;
+    uint32_t capacity;
+    uint16_t *data;
+    // When `has_pending` is set, `pending_indent` records the indentation of a
+    // line whose NEWLINE has already been emitted but whose INDENT/DEDENT
+    // tokens still need to be reconciled against the stack. This lets us emit
+    // several DEDENT tokens for a single line without needing to re-read the
+    // (already consumed) newline.
+    uint16_t pending_indent;
+    uint8_t has_pending;
+} Scanner;
+
+static void stack_push(Scanner *s, uint16_t value) {
+    if (s->size >= s->capacity) {
+        uint32_t new_cap = s->capacity ? s->capacity * 2 : 8;
+        s->data = realloc(s->data, new_cap * sizeof(uint16_t));
+        s->capacity = new_cap;
+    }
+    s->data[s->size++] = value;
+}
+
+static uint16_t stack_top(Scanner *s) {
+    return s->size ? s->data[s->size - 1] : 0;
+}
+
+static void stack_pop(Scanner *s) {
+    // Never pop the base (module) level.
+    if (s->size > 1) {
+        s->size--;
+    }
+}
+
+void *tree_sitter_rapydscript_external_scanner_create(void) {
+    Scanner *s = calloc(1, sizeof(Scanner));
+    stack_push(s, 0);
+    return s;
+}
+
+void tree_sitter_rapydscript_external_scanner_destroy(void *payload) {
+    Scanner *s = (Scanner *)payload;
+    if (s->data) {
+        free(s->data);
+    }
+    free(s);
+}
+
+unsigned tree_sitter_rapydscript_external_scanner_serialize(void *payload, char *buffer) {
+    Scanner *s = (Scanner *)payload;
+    unsigned i = 0;
+    buffer[i++] = (char)(s->has_pending ? 1 : 0);
+    buffer[i++] = (char)(s->pending_indent & 0xFF);
+    buffer[i++] = (char)((s->pending_indent >> 8) & 0xFF);
+    for (uint32_t k = 0; k < s->size; k++) {
+        if (i + 2 > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
+            break;
+        }
+        uint16_t v = s->data[k];
+        buffer[i++] = (char)(v & 0xFF);
+        buffer[i++] = (char)((v >> 8) & 0xFF);
+    }
+    return i;
+}
+
+void tree_sitter_rapydscript_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
+    Scanner *s = (Scanner *)payload;
+    s->size = 0;
+    s->has_pending = 0;
+    s->pending_indent = 0;
+    unsigned i = 0;
+    if (length >= 3) {
+        s->has_pending = (uint8_t)buffer[i++];
+        s->pending_indent = (uint8_t)buffer[i] | ((uint16_t)(uint8_t)buffer[i + 1] << 8);
+        i += 2;
+    }
+    while (i + 2 <= length) {
+        uint16_t v = (uint8_t)buffer[i] | ((uint16_t)(uint8_t)buffer[i + 1] << 8);
+        i += 2;
+        stack_push(s, v);
+    }
+    if (s->size == 0) {
+        stack_push(s, 0);
+    }
+}
+
+static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
+
+static bool is_letter(int32_t c) {
+    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
+}
+
+// Reads a JavaScript style regular expression literal. Assumes the current
+// lookahead is the opening '/'. Handles both plain literals (/ab/gi) and the
+// RapydScript "verbose" form (///  ab  ///).
+static bool scan_regex(TSLexer *lexer) {
+    advance(lexer); // consume opening '/'
+
+    if (lexer->lookahead == '/') {
+        advance(lexer);
+        if (lexer->lookahead == '/') {
+            // verbose regex: read until the closing '///'
+            advance(lexer);
+            int slashes = 0;
+            while (lexer->lookahead != 0) {
+                if (lexer->lookahead == '\\') {
+                    advance(lexer);
+                    if (lexer->lookahead != 0) {
+                        advance(lexer);
+                    }
+                    slashes = 0;
+                    continue;
+                }
+                if (lexer->lookahead == '/') {
+                    slashes++;
+                    advance(lexer);
+                    if (slashes == 3) {
+                        break;
+                    }
+                } else {
+                    slashes = 0;
+                    advance(lexer);
+                }
+            }
+        }
+        // else: empty regex '//' -> nothing more to read in the body
+    } else {
+        // plain regex literal: read until an unescaped '/', respecting classes
+        bool in_class = false;
+        while (lexer->lookahead != 0 && lexer->lookahead != '\n') {
+            if (lexer->lookahead == '\\') {
+                advance(lexer);
+                if (lexer->lookahead != 0) {
+                    advance(lexer);
+                }
+                continue;
+            }
+            if (lexer->lookahead == '[') {
+                in_class = true;
+                advance(lexer);
+                continue;
+            }
+            if (lexer->lookahead == ']') {
+                in_class = false;
+                advance(lexer);
+                continue;
+            }
+            if (lexer->lookahead == '/' && !in_class) {
+                advance(lexer);
+                break;
+            }
+            advance(lexer);
+        }
+    }
+
+    // trailing flags (g, i, m, s, u, y, ...)
+    while (is_letter(lexer->lookahead)) {
+        advance(lexer);
+    }
+
+    lexer->mark_end(lexer);
+    lexer->result_symbol = REGEX;
+    return true;
+}
+
+bool tree_sitter_rapydscript_external_scanner_scan(void *payload, TSLexer *lexer,
+                                                   const bool *valid_symbols) {
+    Scanner *scanner = (Scanner *)payload;
+
+    // 1. Reconcile any pending indentation left over from a previously emitted
+    //    NEWLINE/DEDENT. This is how we emit several DEDENT tokens for a single
+    //    line without re-reading the (already consumed) newline. Only DEDENT is
+    //    reconciled here: a pending INDENT is never correct (a new block always
+    //    begins with a freshly scanned indent), and acting on a stale pending
+    //    value would corrupt the stack when the scanner is next consulted for
+    //    an unrelated block.
+    if (scanner->has_pending) {
+        uint16_t current = stack_top(scanner);
+        if (valid_symbols[DEDENT] && scanner->pending_indent < current) {
+            stack_pop(scanner);
+            lexer->result_symbol = DEDENT;
+            return true;
+        }
+        // Nothing left to reconcile (or the grammar does not want it here).
+        scanner->has_pending = 0;
+    }
+
+    // 2. Regular expression literals. Only valid where a primary expression may
+    //    begin, so '/' cannot be a division operator in that position.
+    if (valid_symbols[REGEX] && !valid_symbols[NEWLINE] && !valid_symbols[INDENT] &&
+        !valid_symbols[DEDENT]) {
+        // Skip surrounding whitespace, including newlines: a regex literal may
+        // appear on its own line, e.g. as an argument following a preceding
+        // multi-line one (`foo(\n def(): ...\n ,\n /re/\n)`).
+        while (lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
+               lexer->lookahead == '\n' || lexer->lookahead == '\r' ||
+               lexer->lookahead == '\f') {
+            skip(lexer);
+        }
+        if (lexer->lookahead == '/') {
+            return scan_regex(lexer);
+        }
+        return false;
+    }
+
+    // 3. Indentation / newline handling.
+    bool found_end_of_line = false;
+    uint32_t indent_length = 0;
+
+    for (;;) {
+        if (lexer->lookahead == '\n') {
+            found_end_of_line = true;
+            indent_length = 0;
+            skip(lexer);
+        } else if (lexer->lookahead == '\r' || lexer->lookahead == '\f' ||
+                   lexer->lookahead == 0x0b) {
+            indent_length = 0;
+            skip(lexer);
+        } else if (lexer->lookahead == ' ') {
+            indent_length++;
+            skip(lexer);
+        } else if (lexer->lookahead == '\t') {
+            indent_length += 8;
+            skip(lexer);
+        } else if (lexer->eof(lexer)) {
+            found_end_of_line = true;
+            indent_length = 0;
+            break;
+        } else {
+            break;
+        }
+    }
+
+    if (found_end_of_line) {
+        uint16_t current = stack_top(scanner);
+
+        // Leading-dot chaining: a line at the same indentation that begins with
+        // '.' binds to the previous logical line (e.g. jQuery-style method
+        // chains and `do: ... .while`). Suppress the newline so the '.' simply
+        // continues the preceding expression. When the indentation differs a
+        // real INDENT/DEDENT is still required (that is how `.while` closes a
+        // `do` block), so this only applies at matching indentation.
+        if (lexer->lookahead == '.' && indent_length == current &&
+            !valid_symbols[INDENT]) {
+            return false;
+        }
+
+        if (valid_symbols[INDENT] && indent_length > current) {
+            stack_push(scanner, (uint16_t)indent_length);
+            lexer->result_symbol = INDENT;
+            return true;
+        }
+
+        if (valid_symbols[DEDENT] && indent_length < current) {
+            stack_pop(scanner);
+            // There may be more levels to pop; remember for subsequent calls.
+            if (indent_length < stack_top(scanner)) {
+                scanner->pending_indent = (uint16_t)indent_length;
+                scanner->has_pending = 1;
+            }
+            lexer->result_symbol = DEDENT;
+            return true;
+        }
+
+        if (valid_symbols[NEWLINE]) {
+            // The newline (and the next line's indentation) has been consumed.
+            // Record that indentation so we can emit the DEDENT/INDENT tokens
+            // that follow, once the grammar reaches a state that accepts them.
+            if (indent_length != current) {
+                scanner->pending_indent = (uint16_t)indent_length;
+                scanner->has_pending = 1;
+            }
+            lexer->result_symbol = NEWLINE;
+            return true;
+        }
+    }
+
+    return false;
+}

package/tree-sitter/src/tree_sitter/alloc.h

@@ -0,0 +1,54 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+// Allow clients to override allocation functions
+#ifdef TREE_SITTER_REUSE_ALLOCATOR
+
+extern void *(*ts_current_malloc)(size_t size);
+extern void *(*ts_current_calloc)(size_t count, size_t size);
+extern void *(*ts_current_realloc)(void *ptr, size_t size);
+extern void (*ts_current_free)(void *ptr);
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

package/tree-sitter/src/tree_sitter/array.h

@@ -0,0 +1,330 @@
+#ifndef TREE_SITTER_ARRAY_H_
+#define TREE_SITTER_ARRAY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./alloc.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
+#define Array(T)       \
+  struct {             \
+    T *contents;       \
+    uint32_t size;     \
+    uint32_t capacity; \
+  }
+
+/// Initialize an array.
+#define array_init(self) \
+  ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
+
+/// Create an empty array.
+#define array_new() \
+  { NULL, 0, 0 }
+
+/// Get a pointer to the element at a given `index` in the array.
+#define array_get(self, _index) \
+  (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
+
+/// Get a pointer to the first element in the array.
+#define array_front(self) array_get(self, 0)
+
+/// Get a pointer to the last element in the array.
+#define array_back(self) array_get(self, (self)->size - 1)
+
+/// Clear the array, setting its size to zero. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_clear(self) ((self)->size = 0)
+
+/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
+/// less than the array's current capacity, this function has no effect.
+#define array_reserve(self, new_capacity)        \
+  ((self)->contents = _array__reserve(           \
+    (void *)(self)->contents, &(self)->capacity, \
+    array_elem_size(self), new_capacity)         \
+  )
+
+/// Free any memory allocated for this array. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_delete(self)                           \
+  do {                                               \
+    if ((self)->contents) ts_free((self)->contents); \
+    (self)->contents = NULL;                         \
+    (self)->size = 0;                                \
+    (self)->capacity = 0;                            \
+  } while (0)
+
+/// Push a new `element` onto the end of the array.
+#define array_push(self, element)                                 \
+  do {                                                            \
+    (self)->contents = _array__grow(                              \
+      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
+      1, array_elem_size(self)                                    \
+    );                                                            \
+   (self)->contents[(self)->size++] = (element);                  \
+  } while(0)
+
+/// Increase the array's size by `count` elements.
+/// New elements are zero-initialized.
+#define array_grow_by(self, count)                                               \
+  do {                                                                           \
+    if ((count) == 0) break;                                                     \
+    (self)->contents = _array__grow(                                             \
+      (self)->contents, (self)->size, &(self)->capacity,                         \
+      count, array_elem_size(self)                                               \
+    );                                                                           \
+    memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
+    (self)->size += (count);                                                     \
+  } while (0)
+
+/// Append all elements from one array to the end of another.
+#define array_push_all(self, other) \
+  array_extend((self), (other)->size, (other)->contents)
+
+/// Append `count` elements to the end of the array, reading their values from the
+/// `contents` pointer.
+#define array_extend(self, count, other_contents)                 \
+  (self)->contents = _array__splice(                              \
+    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
+    array_elem_size(self), (self)->size, 0, count, other_contents \
+  )
+
+/// Remove `old_count` elements from the array starting at the given `index`. At
+/// the same index, insert `new_count` new elements, reading their values from the
+/// `new_contents` pointer.
+#define array_splice(self, _index, old_count, new_count, new_contents) \
+  (self)->contents = _array__splice(                                   \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
+    array_elem_size(self), _index, old_count, new_count, new_contents  \
+  )
+
+/// Insert one `element` into the array at the given `index`.
+#define array_insert(self, _index, element)                     \
+  (self)->contents = _array__splice(                            \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
+    array_elem_size(self), _index, 0, 1, &(element)             \
+  )
+
+/// Remove one element from the array at the given `index`.
+#define array_erase(self, _index) \
+  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
+
+/// Pop the last element off the array, returning the element by value.
+#define array_pop(self) ((self)->contents[--(self)->size])
+
+/// Assign the contents of one array to another, reallocating if necessary.
+#define array_assign(self, other)                                   \
+  (self)->contents = _array__assign(                                \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
+    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
+  )
+
+/// Swap one array with another
+#define array_swap(self, other)                                     \
+  do {                                                              \
+    void *_array_swap_tmp = (void *)(self)->contents;               \
+    (self)->contents = (other)->contents;                           \
+    (other)->contents = _array_swap_tmp;                            \
+    _array__swap(&(self)->size, &(self)->capacity,                  \
+                 &(other)->size, &(other)->capacity);               \
+  } while (0)
+
+/// Get the size of the array contents
+#define array_elem_size(self) (sizeof *(self)->contents)
+
+/// Search a sorted array for a given `needle` value, using the given `compare`
+/// callback to determine the order.
+///
+/// If an existing element is found to be equal to `needle`, then the `index`
+/// out-parameter is set to the existing value's index, and the `exists`
+/// out-parameter is set to true. Otherwise, `index` is set to an index where
+/// `needle` should be inserted in order to preserve the sorting, and `exists`
+/// is set to false.
+#define array_search_sorted_with(self, compare, needle, _index, _exists) \
+  _array__search_sorted(self, 0, compare, , needle, _index, _exists)
+
+/// Search a sorted array for a given `needle` value, using integer comparisons
+/// of a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_with`.
+#define array_search_sorted_by(self, field, needle, _index, _exists) \
+  _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
+
+/// Insert a given `value` into a sorted array, using the given `compare`
+/// callback to determine the order.
+#define array_insert_sorted_with(self, compare, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+/// Insert a given `value` into a sorted array, using integer comparisons of
+/// a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_by`.
+#define array_insert_sorted_by(self, field, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+// Private
+
+// Pointers to individual `Array` fields (rather than the entire `Array` itself)
+// are passed to the various `_array__*` functions below to address strict aliasing
+// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
+//
+// The `Array` type itself was not altered as a solution in order to avoid breakage
+// with existing consumers (in particular, parsers with external scanners).
+
+/// This is not what you're looking for, see `array_erase`.
+static inline void _array__erase(void* self_contents, uint32_t *size,
+                                size_t element_size, uint32_t index) {
+  assert(index < *size);
+  char *contents = (char *)self_contents;
+  memmove(contents + index * element_size, contents + (index + 1) * element_size,
+          (*size - index - 1) * element_size);
+  (*size)--;
+}
+
+/// This is not what you're looking for, see `array_reserve`.
+static inline void *_array__reserve(void *contents, uint32_t *capacity,
+                                  size_t element_size, uint32_t new_capacity) {
+  void *new_contents = contents;
+  if (new_capacity > *capacity) {
+    if (contents) {
+      new_contents = ts_realloc(contents, new_capacity * element_size);
+    } else {
+      new_contents = ts_malloc(new_capacity * element_size);
+    }
+    *capacity = new_capacity;
+  }
+  return new_contents;
+}
+
+/// This is not what you're looking for, see `array_assign`.
+static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                                 const void *other_contents, uint32_t other_size, size_t element_size) {
+  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
+  *self_size = other_size;
+  memcpy(new_contents, other_contents, *self_size * element_size);
+  return new_contents;
+}
+
+/// This is not what you're looking for, see `array_swap`.
+static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
+                               uint32_t *other_size, uint32_t *other_capacity) {
+  uint32_t tmp_size = *self_size;
+  uint32_t tmp_capacity = *self_capacity;
+  *self_size = *other_size;
+  *self_capacity = *other_capacity;
+  *other_size = tmp_size;
+  *other_capacity = tmp_capacity;
+}
+
+/// This is not what you're looking for, see `array_push` or `array_grow_by`.
+static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
+                               uint32_t count, size_t element_size) {
+  void *new_contents = contents;
+  uint32_t new_size = size + count;
+  if (new_size > *capacity) {
+    uint32_t new_capacity = *capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    if (new_capacity < new_size) new_capacity = new_size;
+    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
+  }
+  return new_contents;
+}
+
+/// This is not what you're looking for, see `array_splice`.
+static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
+                                 size_t element_size,
+                                 uint32_t index, uint32_t old_count,
+                                 uint32_t new_count, const void *elements) {
+  uint32_t new_size = *size + new_count - old_count;
+  uint32_t old_end = index + old_count;
+  uint32_t new_end = index + new_count;
+  assert(old_end <= *size);
+
+  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
+
+  char *contents = (char *)new_contents;
+  if (*size > old_end) {
+    memmove(
+      contents + new_end * element_size,
+      contents + old_end * element_size,
+      (*size - old_end) * element_size
+    );
+  }
+  if (new_count > 0) {
+    if (elements) {
+      memcpy(
+        (contents + index * element_size),
+        elements,
+        new_count * element_size
+      );
+    } else {
+      memset(
+        (contents + index * element_size),
+        0,
+        new_count * element_size
+      );
+    }
+  }
+  *size += new_count - old_count;
+
+  return new_contents;
+}
+
+/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
+/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
+#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
+  do { \
+    *(_index) = start; \
+    *(_exists) = false; \
+    uint32_t size = (self)->size - *(_index); \
+    if (size == 0) break; \
+    int comparison; \
+    while (size > 1) { \
+      uint32_t half_size = size / 2; \
+      uint32_t mid_index = *(_index) + half_size; \
+      comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
+      if (comparison <= 0) *(_index) = mid_index; \
+      size -= half_size; \
+    } \
+    comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
+    if (comparison == 0) *(_exists) = true; \
+    else if (comparison < 0) *(_index) += 1; \
+  } while (0)
+
+/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
+/// parameter by reference in order to work with the generic sorting function above.
+#define _compare_int(a, b) ((int)*(a) - (int)(b))
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_ARRAY_H_