semantifold 0.2.0 → 0.3.0

package/node_modules/semantifold-tree-sitter-legacy-internal/node_modules/tree-sitter-cpp/src/tree_sitter/parser.h

@@ -0,0 +1,266 @@
+#ifndef TREE_SITTER_PARSER_H_
+#define TREE_SITTER_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
+
+#ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
+typedef uint16_t TSSymbol;
+typedef uint16_t TSFieldId;
+typedef struct TSLanguage TSLanguage;
+#endif
+
+typedef struct {
+  TSFieldId field_id;
+  uint8_t child_index;
+  bool inherited;
+} TSFieldMapEntry;
+
+typedef struct {
+  uint16_t index;
+  uint16_t length;
+} TSFieldMapSlice;
+
+typedef struct {
+  bool visible;
+  bool named;
+  bool supertype;
+} TSSymbolMetadata;
+
+typedef struct TSLexer TSLexer;
+
+struct TSLexer {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+  void (*advance)(TSLexer *, bool);
+  void (*mark_end)(TSLexer *);
+  uint32_t (*get_column)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
+  void (*log)(const TSLexer *, const char *, ...);
+};
+
+typedef enum {
+  TSParseActionTypeShift,
+  TSParseActionTypeReduce,
+  TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
+} TSParseActionType;
+
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
+} TSParseAction;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+} TSLexMode;
+
+typedef union {
+  TSParseAction action;
+  struct {
+    uint8_t count;
+    bool reusable;
+  } entry;
+} TSParseActionEntry;
+
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
+struct TSLanguage {
+  uint32_t version;
+  uint32_t symbol_count;
+  uint32_t alias_count;
+  uint32_t token_count;
+  uint32_t external_token_count;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
+  const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSParseActionEntry *parse_actions;
+  const char * const *symbol_names;
+  const char * const *field_names;
+  const TSFieldMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
+  const TSSymbol *alias_sequences;
+  const TSLexMode *lex_modes;
+  bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
+  struct {
+    const bool *states;
+    const TSSymbol *symbol_map;
+    void *(*create)(void);
+    void (*destroy)(void *);
+    bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
+  } external_scanner;
+  const TSStateId *primary_state_ids;
+};
+
+static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
+/*
+ *  Lexer Macros
+ */
+
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
+#define START_LEXER()           \
+  bool result = false;          \
+  bool skip = false;            \
+  UNUSED                        \
+  bool eof = false;             \
+  int32_t lookahead;            \
+  goto start;                   \
+  next_state:                   \
+  lexer->advance(lexer, skip);  \
+  start:                        \
+  skip = false;                 \
+  lookahead = lexer->lookahead;
+
+#define ADVANCE(state_value) \
+  {                          \
+    state = state_value;     \
+    goto next_state;         \
+  }
+
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
+#define SKIP(state_value) \
+  {                       \
+    skip = true;          \
+    state = state_value;  \
+    goto next_state;      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
+
+#define END_STATE() return result;
+
+/*
+ *  Parse Table Macros
+ */
+
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
+
+#define STATE(id) id
+
+#define ACTIONS(id) id
+
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value)          \
+    }                                 \
+  }}
+
+#define SHIFT_REPEAT(state_value)     \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value),         \
+      .repetition = true              \
+    }                                 \
+  }}
+
+#define SHIFT_EXTRA()                 \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
+    }                                 \
+  }}
+
+#define REDUCE(symbol_name, children, precedence, prod_id) \
+  {{                                                       \
+    .reduce = {                                            \
+      .type = TSParseActionTypeReduce,                     \
+      .symbol = symbol_name,                               \
+      .child_count = children,                             \
+      .dynamic_precedence = precedence,                    \
+      .production_id = prod_id                             \
+    },                                                     \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSER_H_

package/node_modules/semantifold-tree-sitter-legacy-internal/node_modules/tree-sitter-cpp/tree-sitter.json

@@ -0,0 +1,55 @@
+{
+  "grammars": [
+    {
+      "name": "cpp",
+      "camelcase": "CPP",
+      "scope": "source.cpp",
+      "path": ".",
+      "file-types": [
+        "cc",
+        "cpp",
+        "cxx",
+        "hpp",
+        "hxx",
+        "h"
+      ],
+      "highlights": [
+        "node_modules/tree-sitter-c/queries/highlights.scm",
+        "queries/highlights.scm"
+      ],
+      "injections": "queries/injections.scm",
+      "tags": "queries/tags.scm",
+      "injection-regex": "^(cc|cpp)$"
+    }
+  ],
+  "metadata": {
+    "version": "0.23.4",
+    "license": "MIT",
+    "description": "C++ grammar for tree-sitter",
+    "authors": [
+      {
+        "name": "Max Brunsfeld",
+        "email": "maxbrunsfeld@gmail.com"
+      },
+      {
+        "name": "Amaan Qureshi",
+        "email": "amaanq12@gmail.com"
+      },
+      {
+        "name": "John Drouhard",
+        "email": "john@drouhard.dev"
+      }
+    ],
+    "links": {
+      "repository": "https://github.com/tree-sitter/tree-sitter-cpp"
+    }
+  },
+  "bindings": {
+    "c": true,
+    "go": true,
+    "node": true,
+    "python": true,
+    "rust": true,
+    "swift": true
+  }
+}

package/node_modules/semantifold-tree-sitter-legacy-internal/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "semantifold-tree-sitter-legacy-internal",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Private parser-neutral C and CPP snapshots using Semantifold's isolated legacy Tree-sitter runtime",
+  "type": "module",
+  "main": "./src/c.js",
+  "files": [
+    "src/c.js",
+    "LICENSE",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=24"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "tree-sitter": "0.21.1",
+    "tree-sitter-c": "0.23.2",
+    "tree-sitter-cpp": "0.23.4"
+  }
+}

package/node_modules/semantifold-tree-sitter-legacy-internal/src/c.js

@@ -0,0 +1,152 @@
+// @ts-check
+
+import Parser from "tree-sitter"
+import CLanguage from "tree-sitter-c"
+import CppLanguage from "tree-sitter-cpp"
+
+/** @typedef {Readonly<{row: number, column: number}>} CstPosition */
+/** @typedef {Readonly<{field: string | null, node: CstNode}>} CstChild */
+/**
+ * @typedef {Readonly<{
+ *   type: string,
+ *   named: boolean,
+ *   extra: boolean,
+ *   error: boolean,
+ *   missing: boolean,
+ *   hasError: boolean,
+ *   startIndex: number,
+ *   endIndex: number,
+ *   startPosition: CstPosition,
+ *   endPosition: CstPosition,
+ *   children: readonly CstChild[]
+ * }>} CstNode
+ */
+/**
+ * @typedef {Readonly<{
+ *   schema: "semantifold.parser-cst",
+ *   version: 1,
+ *   language: "c" | "cpp",
+ *   root: CstNode
+ * }>} CstSnapshot
+ */
+
+/**
+ * Parses the explicitly selected official grammar with the isolated legacy runtime.
+ * @param {string} source - Caller-owned source text.
+ * @param {"c" | "cpp"} [language] - Exact grammar identity; existing callers select C by default.
+ * @returns {CstSnapshot} Recursively frozen parser-neutral CST data.
+ */
+export function parseCst(source, language = "c") {
+  if (language != "c" && language != "cpp") throw new Error(`Unsupported private parser language: ${language}`)
+  const parser = new Parser()
+
+  parser.setLanguage(language == "cpp" ? CppLanguage : CLanguage)
+  const tree = parser.parse(source)
+  const convertIndex = createIndexConverter(source, tree.rootNode.endIndex)
+  const lineStarts = createLineStarts(source)
+  const root = snapshotNode(tree.rootNode, convertIndex, lineStarts)
+
+  parser.reset()
+  return Object.freeze({language, root, schema: "semantifold.parser-cst", version: 1})
+}
+
+/**
+ * @param {import("tree-sitter").SyntaxNode} node
+ * @param {(index: number) => number} convertIndex
+ * @param {readonly number[]} lineStarts
+ * @returns {CstNode}
+ */
+function snapshotNode(node, convertIndex, lineStarts) {
+  /** @type {CstChild[]} */
+  const children = []
+
+  for (let index = 0; index < node.childCount; index += 1) {
+    const child = node.child(index)
+
+    if (!child) throw new Error(`Legacy Tree-sitter returned no child at index ${index}`)
+    children.push(Object.freeze({
+      field: node.fieldNameForChild(index) ?? null,
+      node: snapshotNode(child, convertIndex, lineStarts)
+    }))
+  }
+  const startIndex = convertIndex(node.startIndex)
+  const endIndex = convertIndex(node.endIndex)
+
+  return Object.freeze({
+    children: Object.freeze(children),
+    endIndex,
+    endPosition: positionForIndex(endIndex, lineStarts),
+    error: node.isError,
+    extra: node.isExtra,
+    hasError: node.hasError,
+    missing: node.isMissing,
+    named: node.isNamed,
+    startIndex,
+    startPosition: positionForIndex(startIndex, lineStarts),
+    type: node.type
+  })
+}
+
+/**
+ * Normalizes either legacy UTF-8 byte offsets or UTF-16 offsets to UTF-16.
+ * @param {string} source
+ * @param {number} rootEndIndex
+ * @returns {(index: number) => number}
+ */
+function createIndexConverter(source, rootEndIndex) {
+  if (rootEndIndex == source.length) return (index) => index
+
+  const utf8Length = Buffer.byteLength(source, "utf8")
+
+  if (rootEndIndex != utf8Length) {
+    throw new Error(`Unsupported legacy Tree-sitter index extent ${rootEndIndex} for source length ${source.length}`)
+  }
+  /** @type {Map<number, number>} */
+  const byteToUtf16 = new Map([[0, 0]])
+  let byteIndex = 0
+
+  for (let utf16Index = 0; utf16Index < source.length;) {
+    const codePoint = source.codePointAt(utf16Index)
+
+    if (codePoint == null) throw new Error(`Cannot read source code point at index ${utf16Index}`)
+    const character = String.fromCodePoint(codePoint)
+
+    byteIndex += Buffer.byteLength(character, "utf8")
+    utf16Index += character.length
+    byteToUtf16.set(byteIndex, utf16Index)
+  }
+  return (index) => {
+    const converted = byteToUtf16.get(index)
+
+    if (converted == null) throw new Error(`Legacy Tree-sitter returned a non-boundary byte index ${index}`)
+    return converted
+  }
+}
+
+/** @param {string} source @returns {readonly number[]} */
+function createLineStarts(source) {
+  const starts = [0]
+
+  for (let index = 0; index < source.length; index += 1) {
+    if (source[index] == "\n") starts.push(index + 1)
+  }
+  return starts
+}
+
+/**
+ * @param {number} index
+ * @param {readonly number[]} lineStarts
+ * @returns {CstPosition}
+ */
+function positionForIndex(index, lineStarts) {
+  let low = 0
+  let high = lineStarts.length
+
+  while (low + 1 < high) {
+    const middle = Math.floor((low + high) / 2)
+
+    if (/** @type {number} */ (lineStarts[middle]) <= index) low = middle
+    else high = middle
+  }
+  return Object.freeze({column: index - /** @type {number} */ (lineStarts[low]), row: low})
+}

package/node_modules/tree-sitter/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 maxbrunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/node_modules/tree-sitter/README.md

@@ -0,0 +1,128 @@
+# Node Tree-sitter
+
+[![CI][ci]](https://github.com/tree-sitter/node-tree-sitter/actions/workflows/ci.yml)
+[![npm][npm]](https://npmjs.com/package/tree-sitter)
+[![docs][docs]](https://tree-sitter.github.io/node-tree-sitter)
+
+This module provides Node.js bindings to the [tree-sitter] parsing library.
+
+## Installation
+
+```sh
+npm install tree-sitter
+```
+
+## Basic Usage
+
+### Prerequisites
+
+First, you'll need a Tree-sitter grammar for the language you want to parse. There are many [existing grammars][grammars],
+such as [tree-sitter-javascript][javascript]. These grammars can typically be installed with a package manager like NPM,
+so long as the author has published them.
+
+```sh
+npm install tree-sitter-javascript
+```
+
+You can also develop a new grammar by using the [Tree-sitter CLI][cli] and following the [docs][ts docs].
+
+### Parsing Source Code
+
+Once you've got your grammar, create a parser with that grammar.
+
+```javascript
+const Parser = require('tree-sitter');
+const JavaScript = require('tree-sitter-javascript');
+
+const parser = new Parser();
+parser.setLanguage(JavaScript);
+```
+
+Then you can parse some source code,
+
+```javascript
+const sourceCode = 'let x = 1; console.log(x);';
+const tree = parser.parse(sourceCode);
+```
+
+and inspect the syntax tree.
+
+```javascript
+console.log(tree.rootNode.toString());
+
+// (program
+//   (lexical_declaration
+//     (variable_declarator (identifier) (number)))
+//   (expression_statement
+//     (call_expression
+//       (member_expression (identifier) (property_identifier))
+//       (arguments (identifier)))))
+
+const callExpression = tree.rootNode.child(1).firstChild;
+console.log(callExpression);
+
+// {
+//   type: 'call_expression',
+//   startPosition: {row: 0, column: 16},
+//   endPosition: {row: 0, column: 30},
+//   startIndex: 0,
+//   endIndex: 30
+// }
+```
+
+If your source code *changes*, you can update the syntax tree. This is much faster than the first parse.
+
+```javascript
+// In the code, we replaced 'let' with 'const'.
+// So, we set our old end index to 3, and our new end index to 5.
+// Note that the end index is exclusive.
+const newSourceCode = 'const x = 1; console.log(x);';
+//                        ^ ^
+// indices:               3 5
+// points:            (0,3) (0,5)
+
+tree.edit({
+  startIndex: 0,
+  oldEndIndex: 3,
+  newEndIndex: 5,
+  startPosition: {row: 0, column: 0},
+  oldEndPosition: {row: 0, column: 3},
+  newEndPosition: {row: 0, column: 5},
+});
+
+const newTree = parser.parse(newSourceCode, tree);
+```
+
+### Parsing Text From a Custom Data Structure
+
+If your text is stored in a data structure other than a single string, such as a rope or array, you can parse it by supplying
+a callback to `parse` instead of a string:
+
+```javascript
+const sourceLines = [
+  'let x = 1;',
+  'console.log(x);'
+];
+
+const tree = parser.parse((index, position) => {
+  let line = sourceLines[position.row];
+  if (line) {
+    return line.slice(position.column);
+  }
+});
+```
+
+### Further Reading
+
+It's recommended that you read the [Tree-sitter documentation][usage docs] on using parsers to get a higher-level overview
+of the API. Once you're comfortable with the basics, you can explore the [full API documentation](https://tree-sitter.github.io/node-tree-sitter),
+which should map closely to the C API, though there are some differences.
+
+[ci]: https://img.shields.io/github/actions/workflow/status/tree-sitter/node-tree-sitter/ci.yml?logo=github&label=CI
+[cli]: https://github.com/tree-sitter/tree-sitter/tree/master/cli
+[docs]: https://img.shields.io/badge/docs-website-blue
+[npm]: https://img.shields.io/npm/v/tree-sitter?logo=npm
+[grammars]: https://github.com/tree-sitter/tree-sitter/wiki/List-of-parsers
+[javascript]: http://github.com/tree-sitter/tree-sitter-javascript
+[ts docs]: https://tree-sitter.github.io/tree-sitter/creating-parsers
+[usage docs]: https://tree-sitter.github.io/tree-sitter/using-parsers