tree-sitter-lispex 0.1.0

package/src/scanner.c

@@ -0,0 +1,358 @@
+#include "tree_sitter/parser.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum TokenType {
+  BLOCK_COMMENT,
+  CHARACTER,
+  BOOLEAN,
+  BYTE,
+  INTEGER,
+  RATIONAL,
+  REAL,
+  SYMBOL,
+  FORBIDDEN_READER_EXTENSION,
+};
+
+void *tree_sitter_lispex_external_scanner_create(void) { return NULL; }
+void tree_sitter_lispex_external_scanner_destroy(void *payload) {
+  (void)payload;
+}
+unsigned tree_sitter_lispex_external_scanner_serialize(void *payload,
+                                                       char *buffer) {
+  (void)payload;
+  (void)buffer;
+  return 0;
+}
+void tree_sitter_lispex_external_scanner_deserialize(void *payload,
+                                                     const char *buffer,
+                                                     unsigned length) {
+  (void)payload;
+  (void)buffer;
+  (void)length;
+}
+
+static bool is_space(int32_t c) {
+  return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' ||
+         c == 0xFEFF;
+}
+
+static bool is_delimiter(int32_t c) {
+  return c == 0 || is_space(c) || c == '(' || c == ')' || c == '[' ||
+         c == ']' || c == '{' || c == '}' || c == '"' || c == ';' ||
+         c == '\'' || c == '`' || c == ',';
+}
+
+static bool is_ascii_alphanumeric(int32_t c) {
+  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+         (c >= '0' && c <= '9');
+}
+
+static bool is_hex(int32_t c) {
+  return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
+         (c >= 'A' && c <= 'F');
+}
+
+static bool is_named_character(const char *name) {
+  static const char *const names[] = {
+      "space",  "newline", "linefeed", "tab",    "return",
+      "null",   "nul",     "delete",   "rubout", "escape",
+      "esc",    "backspace", "alarm",  "page",
+  };
+  const size_t count = sizeof(names) / sizeof(names[0]);
+  for (size_t index = 0; index < count; index++) {
+    if (strcmp(name, names[index]) == 0) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool scan_block_comment_after_hash(TSLexer *lexer) {
+  lexer->advance(lexer, false);
+  unsigned depth = 1;
+  while (depth > 0) {
+    if (lexer->lookahead == 0) {
+      return false;
+    }
+    if (lexer->lookahead == '#') {
+      lexer->advance(lexer, false);
+      if (lexer->lookahead == '|') {
+        lexer->advance(lexer, false);
+        depth++;
+      }
+      continue;
+    }
+    if (lexer->lookahead == '|') {
+      lexer->advance(lexer, false);
+      if (lexer->lookahead == '#') {
+        lexer->advance(lexer, false);
+        depth--;
+      }
+      continue;
+    }
+    lexer->advance(lexer, false);
+  }
+  lexer->result_symbol = BLOCK_COMMENT;
+  return true;
+}
+
+static bool scan_character_after_hash(TSLexer *lexer) {
+  lexer->advance(lexer, false);
+  if (lexer->lookahead == 0) {
+    return false;
+  }
+
+  int32_t first = lexer->lookahead;
+  lexer->advance(lexer, false);
+  if (!is_ascii_alphanumeric(first) || first >= 128) {
+    lexer->result_symbol = CHARACTER;
+    return true;
+  }
+
+  char name[64];
+  size_t length = 0;
+  name[length++] = (char)first;
+  while (!is_delimiter(lexer->lookahead) && lexer->lookahead != '|') {
+    if (lexer->lookahead >= 128 || length + 1 >= sizeof(name)) {
+      return false;
+    }
+    name[length++] = (char)lexer->lookahead;
+    lexer->advance(lexer, false);
+  }
+  name[length] = '\0';
+
+  bool valid = length == 1;
+  if (!valid && (name[0] == 'x' || name[0] == 'X')) {
+    valid = length > 1;
+    for (size_t index = 1; index < length && valid; index++) {
+      valid = is_hex(name[index]);
+    }
+  }
+  if (!valid) {
+    valid = is_named_character(name);
+  }
+  if (!valid) {
+    return false;
+  }
+
+  lexer->result_symbol = CHARACTER;
+  return true;
+}
+
+static char *scan_ascii_token(TSLexer *lexer, bool *ascii_only,
+                              size_t *length_out) {
+  size_t capacity = 64;
+  size_t length = 0;
+  char *buffer = malloc(capacity);
+  if (buffer == NULL) {
+    return NULL;
+  }
+  *ascii_only = true;
+  while (!is_delimiter(lexer->lookahead)) {
+    if (lexer->lookahead >= 128) {
+      *ascii_only = false;
+    }
+    if (length + 1 >= capacity) {
+      capacity *= 2;
+      char *grown = realloc(buffer, capacity);
+      if (grown == NULL) {
+        free(buffer);
+        return NULL;
+      }
+      buffer = grown;
+    }
+    buffer[length++] =
+        lexer->lookahead < 128 ? (char)lexer->lookahead : '\0';
+    lexer->advance(lexer, false);
+  }
+  buffer[length] = '\0';
+  *length_out = length;
+  return buffer;
+}
+
+static bool unsigned_integer_shape(const char *text) {
+  if (text[0] == '\0') {
+    return false;
+  }
+  if (text[0] == '0') {
+    return text[1] == '\0';
+  }
+  if (text[0] < '1' || text[0] > '9') {
+    return false;
+  }
+  for (size_t index = 1; text[index] != '\0'; index++) {
+    if (text[index] < '0' || text[index] > '9') {
+      return false;
+    }
+  }
+  return true;
+}
+
+static bool integer_shape(const char *text) {
+  return text[0] == '-' ? unsigned_integer_shape(text + 1)
+                        : unsigned_integer_shape(text);
+}
+
+static bool rational_shape(char *text) {
+  char *slash = strchr(text, '/');
+  if (slash == NULL || strchr(slash + 1, '/') != NULL) {
+    return false;
+  }
+  *slash = '\0';
+  bool valid = integer_shape(text) && slash[1] >= '1' && slash[1] <= '9';
+  for (char *cursor = slash + 2; *cursor != '\0' && valid; cursor++) {
+    valid = *cursor >= '0' && *cursor <= '9';
+  }
+  *slash = '/';
+  return valid;
+}
+
+static bool consume_digits(const char **cursor) {
+  const char *start = *cursor;
+  while (**cursor >= '0' && **cursor <= '9') {
+    (*cursor)++;
+  }
+  return *cursor != start;
+}
+
+static bool real_shape(const char *text) {
+  const char *cursor = text;
+  if (*cursor == '-') {
+    cursor++;
+  }
+  const char *whole = cursor;
+  if (!consume_digits(&cursor)) {
+    return false;
+  }
+  size_t whole_length = (size_t)(cursor - whole);
+  if ((whole_length > 1 && whole[0] == '0') ||
+      (whole_length == 0)) {
+    return false;
+  }
+
+  bool fraction = false;
+  bool exponent = false;
+  if (*cursor == '.') {
+    fraction = true;
+    cursor++;
+    if (!consume_digits(&cursor)) {
+      return false;
+    }
+  }
+  if (*cursor == 'e' || *cursor == 'E') {
+    exponent = true;
+    cursor++;
+    if (*cursor == '+' || *cursor == '-') {
+      cursor++;
+    }
+    if (!consume_digits(&cursor)) {
+      return false;
+    }
+  }
+  return *cursor == '\0' && (fraction || exponent);
+}
+
+static bool byte_shape(const char *text) {
+  if (!unsigned_integer_shape(text)) {
+    return false;
+  }
+  unsigned value = 0;
+  for (const char *cursor = text; *cursor != '\0'; cursor++) {
+    value = value * 10 + (unsigned)(*cursor - '0');
+  }
+  return value <= 255;
+}
+
+static bool starts_number_like(const char *text) {
+  return (text[0] >= '0' && text[0] <= '9') ||
+         (text[0] == '-' && text[1] >= '0' && text[1] <= '9');
+}
+
+static bool scan_hash_token_after_hash(TSLexer *lexer,
+                                       const bool *valid_symbols) {
+  if (lexer->lookahead == ';' &&
+      valid_symbols[FORBIDDEN_READER_EXTENSION]) {
+    lexer->advance(lexer, false);
+    lexer->result_symbol = FORBIDDEN_READER_EXTENSION;
+    return true;
+  }
+  if (!valid_symbols[BOOLEAN] &&
+      !valid_symbols[FORBIDDEN_READER_EXTENSION]) {
+    return false;
+  }
+
+  bool ascii_only = true;
+  size_t length = 0;
+  char *tail = scan_ascii_token(lexer, &ascii_only, &length);
+  if (tail == NULL) {
+    return false;
+  }
+  bool matched = false;
+  if (valid_symbols[BOOLEAN] && ascii_only &&
+      (strcmp(tail, "t") == 0 || strcmp(tail, "true") == 0 ||
+       strcmp(tail, "f") == 0 || strcmp(tail, "false") == 0)) {
+    lexer->result_symbol = BOOLEAN;
+    matched = true;
+  } else if (valid_symbols[FORBIDDEN_READER_EXTENSION] && ascii_only &&
+             strcmp(tail, "lang") == 0) {
+    lexer->result_symbol = FORBIDDEN_READER_EXTENSION;
+    matched = true;
+  }
+  free(tail);
+  return matched;
+}
+
+static bool scan_atom(TSLexer *lexer, const bool *valid_symbols) {
+  bool ascii_only = true;
+  size_t length = 0;
+  char *text = scan_ascii_token(lexer, &ascii_only, &length);
+  if (text == NULL || length == 0) {
+    free(text);
+    return false;
+  }
+
+  bool matched = false;
+  if (valid_symbols[BYTE] && ascii_only && byte_shape(text)) {
+    lexer->result_symbol = BYTE;
+    matched = true;
+  } else if (valid_symbols[RATIONAL] && ascii_only && rational_shape(text)) {
+    lexer->result_symbol = RATIONAL;
+    matched = true;
+  } else if (valid_symbols[REAL] && ascii_only && real_shape(text)) {
+    lexer->result_symbol = REAL;
+    matched = true;
+  } else if (valid_symbols[INTEGER] && ascii_only && integer_shape(text)) {
+    lexer->result_symbol = INTEGER;
+    matched = true;
+  } else if (valid_symbols[SYMBOL] && text[0] != '#' &&
+             !(ascii_only && starts_number_like(text)) &&
+             !(ascii_only && strcmp(text, ".") == 0)) {
+    lexer->result_symbol = SYMBOL;
+    matched = true;
+  }
+  free(text);
+  return matched;
+}
+
+bool tree_sitter_lispex_external_scanner_scan(void *payload, TSLexer *lexer,
+                                              const bool *valid_symbols) {
+  (void)payload;
+  while (is_space(lexer->lookahead)) {
+    lexer->advance(lexer, true);
+  }
+  if (lexer->lookahead == '#') {
+    lexer->advance(lexer, false);
+    if (lexer->lookahead == '|' && valid_symbols[BLOCK_COMMENT]) {
+      return scan_block_comment_after_hash(lexer);
+    }
+    if (lexer->lookahead == '\\' && valid_symbols[CHARACTER]) {
+      return scan_character_after_hash(lexer);
+    }
+    return scan_hash_token_after_hash(lexer, valid_symbols);
+  }
+  return scan_atom(lexer, valid_symbols);
+}

package/src/tree_sitter/alloc.h

@@ -0,0 +1,54 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+// Allow clients to override allocation functions
+#ifdef TREE_SITTER_REUSE_ALLOCATOR
+
+extern void *(*ts_current_malloc)(size_t size);
+extern void *(*ts_current_calloc)(size_t count, size_t size);
+extern void *(*ts_current_realloc)(void *ptr, size_t size);
+extern void (*ts_current_free)(void *ptr);
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

package/src/tree_sitter/array.h

@@ -0,0 +1,291 @@
+#ifndef TREE_SITTER_ARRAY_H_
+#define TREE_SITTER_ARRAY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./alloc.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
+#define Array(T)       \
+  struct {             \
+    T *contents;       \
+    uint32_t size;     \
+    uint32_t capacity; \
+  }
+
+/// Initialize an array.
+#define array_init(self) \
+  ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
+
+/// Create an empty array.
+#define array_new() \
+  { NULL, 0, 0 }
+
+/// Get a pointer to the element at a given `index` in the array.
+#define array_get(self, _index) \
+  (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
+
+/// Get a pointer to the first element in the array.
+#define array_front(self) array_get(self, 0)
+
+/// Get a pointer to the last element in the array.
+#define array_back(self) array_get(self, (self)->size - 1)
+
+/// Clear the array, setting its size to zero. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_clear(self) ((self)->size = 0)
+
+/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
+/// less than the array's current capacity, this function has no effect.
+#define array_reserve(self, new_capacity) \
+  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+
+/// Free any memory allocated for this array. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_delete(self) _array__delete((Array *)(self))
+
+/// Push a new `element` onto the end of the array.
+#define array_push(self, element)                            \
+  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
+   (self)->contents[(self)->size++] = (element))
+
+/// Increase the array's size by `count` elements.
+/// New elements are zero-initialized.
+#define array_grow_by(self, count) \
+  do { \
+    if ((count) == 0) break; \
+    _array__grow((Array *)(self), count, array_elem_size(self)); \
+    memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
+    (self)->size += (count); \
+  } while (0)
+
+/// Append all elements from one array to the end of another.
+#define array_push_all(self, other)                                       \
+  array_extend((self), (other)->size, (other)->contents)
+
+/// Append `count` elements to the end of the array, reading their values from the
+/// `contents` pointer.
+#define array_extend(self, count, contents)                    \
+  _array__splice(                                               \
+    (Array *)(self), array_elem_size(self), (self)->size, \
+    0, count,  contents                                        \
+  )
+
+/// Remove `old_count` elements from the array starting at the given `index`. At
+/// the same index, insert `new_count` new elements, reading their values from the
+/// `new_contents` pointer.
+#define array_splice(self, _index, old_count, new_count, new_contents)  \
+  _array__splice(                                                       \
+    (Array *)(self), array_elem_size(self), _index,                \
+    old_count, new_count, new_contents                                 \
+  )
+
+/// Insert one `element` into the array at the given `index`.
+#define array_insert(self, _index, element) \
+  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+
+/// Remove one element from the array at the given `index`.
+#define array_erase(self, _index) \
+  _array__erase((Array *)(self), array_elem_size(self), _index)
+
+/// Pop the last element off the array, returning the element by value.
+#define array_pop(self) ((self)->contents[--(self)->size])
+
+/// Assign the contents of one array to another, reallocating if necessary.
+#define array_assign(self, other) \
+  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+
+/// Swap one array with another
+#define array_swap(self, other) \
+  _array__swap((Array *)(self), (Array *)(other))
+
+/// Get the size of the array contents
+#define array_elem_size(self) (sizeof *(self)->contents)
+
+/// Search a sorted array for a given `needle` value, using the given `compare`
+/// callback to determine the order.
+///
+/// If an existing element is found to be equal to `needle`, then the `index`
+/// out-parameter is set to the existing value's index, and the `exists`
+/// out-parameter is set to true. Otherwise, `index` is set to an index where
+/// `needle` should be inserted in order to preserve the sorting, and `exists`
+/// is set to false.
+#define array_search_sorted_with(self, compare, needle, _index, _exists) \
+  _array__search_sorted(self, 0, compare, , needle, _index, _exists)
+
+/// Search a sorted array for a given `needle` value, using integer comparisons
+/// of a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_with`.
+#define array_search_sorted_by(self, field, needle, _index, _exists) \
+  _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
+
+/// Insert a given `value` into a sorted array, using the given `compare`
+/// callback to determine the order.
+#define array_insert_sorted_with(self, compare, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+/// Insert a given `value` into a sorted array, using integer comparisons of
+/// a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_by`.
+#define array_insert_sorted_by(self, field, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+// Private
+
+typedef Array(void) Array;
+
+/// This is not what you're looking for, see `array_delete`.
+static inline void _array__delete(Array *self) {
+  if (self->contents) {
+    ts_free(self->contents);
+    self->contents = NULL;
+    self->size = 0;
+    self->capacity = 0;
+  }
+}
+
+/// This is not what you're looking for, see `array_erase`.
+static inline void _array__erase(Array *self, size_t element_size,
+                                uint32_t index) {
+  assert(index < self->size);
+  char *contents = (char *)self->contents;
+  memmove(contents + index * element_size, contents + (index + 1) * element_size,
+          (self->size - index - 1) * element_size);
+  self->size--;
+}
+
+/// This is not what you're looking for, see `array_reserve`.
+static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
+  if (new_capacity > self->capacity) {
+    if (self->contents) {
+      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+    } else {
+      self->contents = ts_malloc(new_capacity * element_size);
+    }
+    self->capacity = new_capacity;
+  }
+}
+
+/// This is not what you're looking for, see `array_assign`.
+static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
+  _array__reserve(self, element_size, other->size);
+  self->size = other->size;
+  memcpy(self->contents, other->contents, self->size * element_size);
+}
+
+/// This is not what you're looking for, see `array_swap`.
+static inline void _array__swap(Array *self, Array *other) {
+  Array swap = *other;
+  *other = *self;
+  *self = swap;
+}
+
+/// This is not what you're looking for, see `array_push` or `array_grow_by`.
+static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
+  if (new_size > self->capacity) {
+    uint32_t new_capacity = self->capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    if (new_capacity < new_size) new_capacity = new_size;
+    _array__reserve(self, element_size, new_capacity);
+  }
+}
+
+/// This is not what you're looking for, see `array_splice`.
+static inline void _array__splice(Array *self, size_t element_size,
+                                 uint32_t index, uint32_t old_count,
+                                 uint32_t new_count, const void *elements) {
+  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t old_end = index + old_count;
+  uint32_t new_end = index + new_count;
+  assert(old_end <= self->size);
+
+  _array__reserve(self, element_size, new_size);
+
+  char *contents = (char *)self->contents;
+  if (self->size > old_end) {
+    memmove(
+      contents + new_end * element_size,
+      contents + old_end * element_size,
+      (self->size - old_end) * element_size
+    );
+  }
+  if (new_count > 0) {
+    if (elements) {
+      memcpy(
+        (contents + index * element_size),
+        elements,
+        new_count * element_size
+      );
+    } else {
+      memset(
+        (contents + index * element_size),
+        0,
+        new_count * element_size
+      );
+    }
+  }
+  self->size += new_count - old_count;
+}
+
+/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
+/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
+#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
+  do { \
+    *(_index) = start; \
+    *(_exists) = false; \
+    uint32_t size = (self)->size - *(_index); \
+    if (size == 0) break; \
+    int comparison; \
+    while (size > 1) { \
+      uint32_t half_size = size / 2; \
+      uint32_t mid_index = *(_index) + half_size; \
+      comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
+      if (comparison <= 0) *(_index) = mid_index; \
+      size -= half_size; \
+    } \
+    comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
+    if (comparison == 0) *(_exists) = true; \
+    else if (comparison < 0) *(_index) += 1; \
+  } while (0)
+
+/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
+/// parameter by reference in order to work with the generic sorting function above.
+#define _compare_int(a, b) ((int)*(a) - (int)(b))
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_ARRAY_H_