tree-sitter-sed 0.4.0 → 0.6.0

package/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.13)
 
 project(
   tree-sitter-sed
-  VERSION 0.4.0
+  VERSION 0.6.0
   DESCRIPTION "Tree-sitter grammars for POSIX sed."
   HOMEPAGE_URL "https://github.com/konomanoasa/tree-sitter-sed"
   LANGUAGES C

package/Makefile

@@ -1,6 +1,6 @@
 LANGUAGE_NAME := tree-sitter-sed
 HOMEPAGE_URL := https://github.com/konomanoasa/tree-sitter-sed
-VERSION := 0.4.0
+VERSION := 0.6.0
 DESCRIPTION := Tree-sitter grammars for POSIX sed.
 
 PREFIX ?= /usr/local

package/README.md

@@ -16,10 +16,18 @@ npm install tree-sitter-sed
 
 This repository contains the following two grammars.
 
-| Grammar   | Regexp | C function              |
-| --------- | ------ | ----------------------- |
-| `sed`     | BRE    | `tree_sitter_sed()`     |
-| `sed_ere` | ERE    | `tree_sitter_sed_ere()` |
+| Grammar   | Regexp | C function              | Rust constant  |
+| --------- | ------ | ----------------------- | -------------- |
+| `sed`     | BRE    | `tree_sitter_sed()`     | `LANGUAGE`     |
+| `sed_ere` | ERE    | `tree_sitter_sed_ere()` | `LANGUAGE_ERE` |
+
+## Development
+
+```sh
+npm install
+npm run parse -- script.sed
+npm run parse:ere -- script.sed
+```
 
 ## Specifications
 

package/common/grammar.js

@@ -551,29 +551,17 @@ function operandRules(mode) {
   }
 
   return {
-    _substitute_function_without_write: ($) =>
+    _substitute_function: ($) =>
       choice(
         seq(
           $._complete_substitute_function,
           optional(
-            field(
-              "flags",
-              alias($._substitution_flags_without_write, $.substitution_flags),
-            ),
+            field("flags", alias($._substitution_flags, $.substitution_flags)),
           ),
         ),
         $._incomplete_substitute_function,
       ),
 
-    _substitute_function_with_write: ($) =>
-      seq(
-        $._complete_substitute_function,
-        field(
-          "flags",
-          alias($._substitution_flags_with_write, $.substitution_flags),
-        ),
-      ),
-
     _complete_substitute_function: ($) =>
       seq(
         functionVerb($, "s"),
@@ -659,6 +647,12 @@ function operandRules(mode) {
     escaped_newline: ($) =>
       namedExternal($, $._replacement_escaped_newline, "escaped_newline_token"),
 
+    _substitution_flags: ($) =>
+      choice(
+        $._substitution_flags_without_write,
+        seq(optional($._substitution_flags_without_write), $.write_flag),
+      ),
+
     _substitution_flags_without_write: ($) =>
       choice(
         substitutionFlagChoice($),
@@ -667,9 +661,6 @@ function operandRules(mode) {
         ),
       ),
 
-    _substitution_flags_with_write: ($) =>
-      seq(optional($._substitution_flags_without_write), $.write_flag),
-
     occurrence_flag: () => /[[:digit:]]+/,
 
     global_flag: () => "g",
@@ -684,8 +675,7 @@ function operandRules(mode) {
         choice(
           seq(
             $._flag_after_write_marker,
-            issueField($, "flag_after_write_flag"),
-            repeat(postWriteSubstitutionFlagChoice($)),
+            repeat1(postWriteSubstitutionFlagChoice($)),
             $._blanks,
             choice(
               field("wfile", alias($._substitution_wfile, $.wfile)),
@@ -955,39 +945,19 @@ function editingCommandRules() {
 
     _line_terminated_editing_command: ($) =>
       seq(
-        choice(
-          $._line_terminated_regular_editing_command_body,
-          $._line_terminated_substitute_editing_command_body,
-        ),
+        addressedForms($, lineTerminated, "line_terminated"),
         optional(issueField($, "unexpected_command_text")),
       ),
 
     _recovered_line_terminated_editing_command: ($) =>
-      choice(
-        prec.right(
-          seq(
-            $._line_terminated_regular_editing_command_body,
-            optional(issueField($, "unexpected_command_text")),
-            issueField($, "forbidden_command_separator"),
-            optional($._blanks),
-          ),
-        ),
-        prec.right(
-          seq(
-            $._line_terminated_substitute_editing_command_body,
-            optional(issueField($, "unexpected_command_text")),
-            issueField($, "command_after_write_flag"),
-            optional($._blanks),
-          ),
+      prec.right(
+        seq(
+          $._line_terminated_editing_command,
+          issueField($, "forbidden_command_separator"),
+          optional($._blanks),
         ),
       ),
 
-    _line_terminated_regular_editing_command_body: ($) =>
-      addressedForms($, lineTerminated, "line_terminated"),
-
-    _line_terminated_substitute_editing_command_body: ($) =>
-      addressedFunction($, $._line_terminated_substitute_function),
-
     _recovered_editing_command: ($) =>
       choice(
         addressedFunction($, $._unknown_function),
@@ -1034,10 +1004,7 @@ function editingCommandRules() {
     _missing_function: ($) => missingAtEndOrBoundary($, "missing_function"),
 
     _chainable_substitute_function: ($) =>
-      alias($._substitute_function_without_write, $.substitute_function),
-
-    _line_terminated_substitute_function: ($) =>
-      alias($._substitute_function_with_write, $.substitute_function),
+      alias($._substitute_function, $.substitute_function),
 
     negation: ($) =>
       seq(
@@ -1250,16 +1217,6 @@ function issueDefinitions(mode) {
       outcome: "undefined_syntax",
       rule: ($) => $._translate_nonportable_escape,
     },
-    {
-      reason: "command_after_write_flag",
-      outcome: "undefined_syntax",
-      rule: () => token.immediate(";"),
-    },
-    {
-      reason: "flag_after_write_flag",
-      outcome: "undefined_syntax",
-      rule: ($) => postWriteSubstitutionFlagChoice($),
-    },
     {
       reason: "equivalence_class_range_start",
       outcome: "unspecified_syntax",

package/common/regex.js

@@ -64,24 +64,21 @@ function intervalExpression($, openingName, closingName) {
 function compoundBracketExpression(
   $,
   openingExternal,
-  openingName,
+  marker,
   contentField,
   content,
   closingExternal,
-  closingName,
 ) {
-  const closing = namedExternal($, closingExternal, closingName);
+  const closing = seq(alias(closingExternal, marker), "]");
   return seq(
-    field("opening", namedExternal($, openingExternal, openingName)),
+    alias(openingExternal, "["),
+    marker,
     choice(
       seq(
         field(contentField, content),
         choice(
-          field("closing", closing),
-          seq(
-            issueField($, "malformed_bracket_term"),
-            optional(field("closing", closing)),
-          ),
+          closing,
+          seq(issueField($, "malformed_bracket_term"), optional(closing)),
           issueField($, "incomplete_bracket_term"),
         ),
       ),
@@ -90,7 +87,7 @@ function compoundBracketExpression(
           issueField($, "malformed_bracket_term"),
           issueField($, "incomplete_bracket_term"),
         ),
-        optional(field("closing", closing)),
+        optional(closing),
       ),
     ),
   );
@@ -255,7 +252,7 @@ function bracketRules() {
       compoundBracketExpression(
         $,
         $._regex_open_dot,
-        "open_dot",
+        ".",
         "element",
         choice(
           namedExternal($, $._regex_coll_elem_single, "coll_elem_single"),
@@ -263,32 +260,29 @@ function bracketRules() {
           namedExternal($, $._regex_meta_char, "meta_char"),
         ),
         $._regex_dot_close,
-        "dot_close",
       ),
 
     equivalence_class: ($) =>
       compoundBracketExpression(
         $,
         $._regex_open_equal,
-        "open_equal",
+        "=",
         "element",
         choice(
           namedExternal($, $._regex_coll_elem_single, "coll_elem_single"),
           namedExternal($, $._regex_coll_elem_multi, "coll_elem_multi"),
         ),
         $._regex_equal_close,
-        "equal_close",
       ),
 
     character_class: ($) =>
       compoundBracketExpression(
         $,
         $._regex_open_colon,
-        "open_colon",
+        ":",
         "name",
         namedExternal($, $._regex_class_name, "class_name"),
         $._regex_colon_close,
-        "colon_close",
       ),
   };
 }
@@ -340,6 +334,14 @@ function breDuplicationSymbol($) {
   );
 }
 
+function misplacedBreDuplSymbol($, reason) {
+  return choice(
+    seq(issueField($, reason), field("operator", $.bre_dupl_symbol)),
+    issueField($, "malformed_interval"),
+    issueField($, "incomplete_interval"),
+  );
+}
+
 function breRules() {
   return {
     bre_extension_escape: ($) =>
@@ -483,24 +485,10 @@ function breRules() {
       intervalExpression($, "back_open_brace", "back_close_brace"),
 
     leading_bre_dupl_symbol: ($) =>
-      choice(
-        seq(
-          issueField($, "leading_duplication_symbol"),
-          field("operator", $.bre_dupl_symbol),
-        ),
-        issueField($, "malformed_interval"),
-        issueField($, "incomplete_interval"),
-      ),
+      misplacedBreDuplSymbol($, "leading_duplication_symbol"),
 
     adjacent_bre_dupl_symbol: ($) =>
-      choice(
-        seq(
-          issueField($, "adjacent_duplication_symbol"),
-          field("operator", $.bre_dupl_symbol),
-        ),
-        issueField($, "malformed_interval"),
-        issueField($, "incomplete_interval"),
-      ),
+      misplacedBreDuplSymbol($, "adjacent_duplication_symbol"),
   };
 }
 

package/common/scanner.h

@@ -1234,7 +1234,6 @@ static bool scan_regex_bracket_opener(
   if (!valid_symbols[candidate]) {
     return false;
   }
-  consume(lexer);
   begin_compound_bracket_element(state);
   finish_first_bracket_element(state);
   state->regex_bracket_term_state = term_state;
@@ -1386,11 +1385,10 @@ static bool scan_regex_bracket_term_close(
     return false;
   }
 
-  advance(lexer);
+  consume(lexer);
   if (lexer->lookahead != ']') {
     return false;
   }
-  consume(lexer);
   state->regex_bracket_term_state = REGEX_BRACKET_TERM_NONE;
   finish_compound_bracket_element(state);
   return emit_symbol(valid_symbols, candidate, symbol);
@@ -2851,16 +2849,26 @@ static bool scan_command_token(
   }
 
   if (at_command_boundary(lexer)) {
-    const TSSymbol first_marker = lexer->eof(lexer)
-      ? MISSING_FUNCTION_MARKER
-      : NONCONFORMING_MISSING_FUNCTION_MARKER;
+    static const TSSymbol boundary_markers[][2] = {
+      {MISSING_FUNCTION_MARKER, NONCONFORMING_MISSING_FUNCTION_MARKER},
+      {MISSING_LABEL_MARKER, NONCONFORMING_MISSING_LABEL_MARKER},
+      {MISSING_RFILE_MARKER, NONCONFORMING_MISSING_RFILE_MARKER},
+      {MISSING_WFILE_MARKER, NONCONFORMING_MISSING_WFILE_MARKER},
+    };
+    const unsigned variant = lexer->eof(lexer) ? 0 : 1;
     for (
-      TSSymbol offset = 0;
-      offset <= MISSING_WFILE_MARKER - MISSING_FUNCTION_MARKER;
-      offset++
+      unsigned index = 0;
+      index < sizeof(boundary_markers) / sizeof(boundary_markers[0]);
+      index++
     ) {
-      const TSSymbol marker = first_marker + offset;
-      if (emit_missing_marker(lexer, valid_symbols, marker, symbol)) {
+      if (
+        emit_missing_marker(
+          lexer,
+          valid_symbols,
+          boundary_markers[index][variant],
+          symbol
+        )
+      ) {
         return true;
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-sed",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Tree-sitter grammars for POSIX sed.",
   "license": "MIT",
   "repository": "git+https://github.com/konomanoasa/tree-sitter-sed.git",
@@ -27,13 +27,15 @@
     "check": "biome check . && node scripts/check-c.js && node scripts/check-generated.js && npm test",
     "format": "biome check --write . && node scripts/check-c.js --write",
     "generate": "npm run tree-sitter -- generate-all",
+    "parse": "npm run tree-sitter -- parse --scope source.sed",
+    "parse:ere": "npm run tree-sitter -- parse --scope source.sed.ere",
     "test": "npm run tree-sitter -- test-corpus --rebuild && node --test",
     "test:fuzz": "npm run tree-sitter -- fuzz-all --iterations 100 --edits 5",
     "tree-sitter": "node scripts/tree-sitter.js"
   },
   "devDependencies": {
     "@biomejs/biome": "2.5.11",
-    "tree-sitter-cli": "0.26.13"
+    "tree-sitter-cli": "0.27.0"
   },
   "allowScripts": {
     "tree-sitter-cli": true

package/queries/highlights.scm

@@ -65,11 +65,39 @@
   (back_open_parenthesis)
   (close_bracket_token)
   (closing_brace_token)
-  (colon_close)
   (open_bracket)
-  (open_colon)
 ] @punctuation.bracket
 
+(character_class
+  "[" @punctuation.bracket
+  .
+  ":" @punctuation.delimiter)
+
+(character_class
+  ":" @punctuation.delimiter
+  .
+  "]" @punctuation.bracket)
+
+(collating_symbol
+  "[" @punctuation.bracket
+  .
+  "." @punctuation.delimiter)
+
+(collating_symbol
+  "." @punctuation.delimiter
+  .
+  "]" @punctuation.bracket)
+
+(equivalence_class
+  "[" @punctuation.bracket
+  .
+  "=" @punctuation.delimiter)
+
+(equivalence_class
+  "=" @punctuation.delimiter
+  .
+  "]" @punctuation.bracket)
+
 [
   (nonmatching_list_operator)
   (range_operator)
@@ -85,11 +113,7 @@
   (coll_elem_multi)
   (coll_elem_single)
   (collating_element_token)
-  (dot_close)
-  (equal_close)
   (meta_char)
-  (open_dot)
-  (open_equal)
   (period_token)
 ] @character.special
 

package/sed_ere/queries/highlights.scm

@@ -63,13 +63,41 @@
   (close_bracket_token)
   (close_parenthesis_token)
   (closing_brace_token)
-  (colon_close)
   (open_brace)
   (open_bracket)
-  (open_colon)
   (open_parenthesis)
 ] @punctuation.bracket
 
+(character_class
+  "[" @punctuation.bracket
+  .
+  ":" @punctuation.delimiter)
+
+(character_class
+  ":" @punctuation.delimiter
+  .
+  "]" @punctuation.bracket)
+
+(collating_symbol
+  "[" @punctuation.bracket
+  .
+  "." @punctuation.delimiter)
+
+(collating_symbol
+  "." @punctuation.delimiter
+  .
+  "]" @punctuation.bracket)
+
+(equivalence_class
+  "[" @punctuation.bracket
+  .
+  "=" @punctuation.delimiter)
+
+(equivalence_class
+  "=" @punctuation.delimiter
+  .
+  "]" @punctuation.bracket)
+
 [
   (nonmatching_list_operator)
   (range_operator)
@@ -85,11 +113,7 @@
   (coll_elem_multi)
   (coll_elem_single)
   (collating_element_token)
-  (dot_close)
-  (equal_close)
   (meta_char)
-  (open_dot)
-  (open_equal)
   (period_token)
 ] @character.special