tree-sitter-sed 0.13.0 → 0.15.0

package/Cargo.toml

@@ -10,7 +10,7 @@ name = "tree-sitter-sed"
 publish = false
 readme = "README.md"
 repository = "https://github.com/konomanoasa/tree-sitter-sed"
-version = "0.13.0"
+version = "0.15.0"
 
 build = "bindings/rust/build.rs"
 include = [

package/bindings/rust/lib.rs

@@ -199,7 +199,7 @@ mod tests {
           9,
           1,
           6,
-          ["unexpected_command_text", "invalid_substitution_flag"],
+          ["invalid_substitution_flag"; 2],
         ),
       ] {
         for (invalid_byte, reason) in [0, 0xff].into_iter().zip(reasons) {

package/common/grammar.js

@@ -47,6 +47,18 @@ function lineOperand($, first, rest) {
   );
 }
 
+function operandAtoms($, literal, ...atoms) {
+  return repeat1(
+    choice(
+      literal,
+      seq($._escape_prefix, issueField($, "invalid_encoding")),
+      issueField($, "nul_character"),
+      issueField($, "invalid_encoding"),
+      ...atoms,
+    ),
+  );
+}
+
 function commentText($) {
   return choice(
     namedExternal($, $._comment_text, "comment_text"),
@@ -205,6 +217,14 @@ function editingCommand($, kind) {
   return alias($[`_${kind}_editing_command`], $.editing_command);
 }
 
+function newlineSeparator($) {
+  return alias("\n", $.command_separator);
+}
+
+function semicolonSeparator($) {
+  return alias(";", $.command_separator);
+}
+
 function commandListRules() {
   return {
     script: ($) => optional(alias($._script_command_list, $.command_list)),
@@ -215,9 +235,7 @@ function commandListRules() {
     _initial_suppressing_comment_command_list: ($) =>
       seq(
         alias($._initial_suppressing_comment_command, $.editing_command),
-        optional(
-          seq(alias("\n", $.command_separator), optional($._command_list)),
-        ),
+        optional(seq(newlineSeparator($), optional($._command_list))),
       ),
 
     _initial_suppressing_comment_command: ($) =>
@@ -248,116 +266,91 @@ function commandListRules() {
         repeat(commentText($)),
       ),
 
+    // Lists are built with repeat so that Tree-sitter balances them; explicit
+    // recursion nests hidden nodes as deep as the command count, which makes
+    // cursor traversal quadratic.
     _command_list: ($) =>
       choice(
-        $._command_sequence,
+        seq(
+          repeat1($._command_unit),
+          optional(choice($._command_item, $._blanks)),
+        ),
+        $._command_item,
         $._blanks,
-        seq($._terminated_line, optional($._command_list)),
       ),
 
-    _terminated_line: ($) =>
-      seq($._command_sequence, alias("\n", $.command_separator)),
-
-    _command_sequence: ($) =>
+    _command_unit: ($) =>
       choice(
         seq(
-          editingCommand($, "line_terminated"),
-          optional($._unmatched_closing_brace_tail),
-        ),
-        seq(
-          editingCommand($, "recovered_line_terminated"),
-          optional(choice($._command_sequence, $._blanks)),
+          $._separable_command_item,
+          choice(
+            semicolonSeparator($),
+            newlineSeparator($),
+            issueNode($, "missing_separator_before_unmatched_brace"),
+          ),
         ),
-        seq($._separable_command_item, optional($._command_sequence_tail)),
         seq(
           $._unmatched_closing_brace_item,
-          optional($._unmatched_closing_brace_tail_content),
+          choice(
+            semicolonSeparator($),
+            newlineSeparator($),
+            issueNode($, "missing_separator_before_unmatched_brace"),
+            issueNode($, "missing_separator_after_unmatched_brace"),
+          ),
         ),
-      ),
-
-    _command_sequence_tail: ($) =>
-      choice(
         seq(
-          alias(";", $.command_separator),
-          optional(choice($._command_sequence, $._blanks)),
+          editingCommand($, "line_terminated"),
+          choice(
+            newlineSeparator($),
+            issueNode($, "missing_separator_before_unmatched_brace"),
+          ),
         ),
-        $._unmatched_closing_brace_tail,
+        editingCommand($, "recovered_line_terminated"),
       ),
 
-    _unmatched_closing_brace_tail: ($) =>
-      seq(
-        issueNode($, "missing_separator_before_unmatched_brace"),
+    _command_item: ($) =>
+      choice(
+        $._separable_command_item,
         $._unmatched_closing_brace_item,
-        optional($._unmatched_closing_brace_tail_content),
+        editingCommand($, "line_terminated"),
       ),
 
     _unmatched_closing_brace_item: ($) =>
-      seq(issueNode($, "unmatched_closing_brace"), optional($._blanks)),
-
-    _unmatched_closing_brace_tail_content: ($) =>
-      choice(
-        $._command_sequence_tail,
-        seq(
-          issueNode($, "missing_separator_after_unmatched_brace"),
-          $._command_sequence,
-        ),
+      seq(
+        optional($._brace_leading_blanks),
+        issueNode($, "unmatched_closing_brace"),
+        optional($._blanks),
       ),
 
     _separable_command_item: ($) =>
-      choice($.empty_command, $._chainable_command_item),
-
-    _chainable_command_item: ($) =>
-      choice(editingCommand($, "chainable"), editingCommand($, "recovered")),
+      choice(
+        $.empty_command,
+        editingCommand($, "chainable"),
+        editingCommand($, "recovered"),
+      ),
 
-    _block_command_sequence: ($) =>
+    _block_commands: ($) =>
       choice(
-        editingCommand($, "line_terminated"),
+        repeat1($._block_unit),
         seq(
-          editingCommand($, "recovered_line_terminated"),
-          optional($._block_command_sequence),
-        ),
-        seq(
-          $._separable_command_item,
-          optional(
-            seq(
-              alias(";", $.command_separator),
-              optional($._block_command_sequence),
-            ),
-          ),
+          repeat($._block_unit),
+          optional($._block_item),
+          missingCommandSeparator($),
         ),
       ),
 
-    _block_command_sequence_before_close: ($) =>
+    _block_unit: ($) =>
       choice(
         seq(
           $._separable_command_item,
-          alias(";", $.command_separator),
-          optional($._block_command_sequence_before_close),
-        ),
-        seq(
-          editingCommand($, "recovered_line_terminated"),
-          optional($._block_command_sequence_before_close),
+          choice(semicolonSeparator($), newlineSeparator($)),
         ),
+        seq(editingCommand($, "line_terminated"), newlineSeparator($)),
+        editingCommand($, "recovered_line_terminated"),
       ),
 
-    _block_command_list: ($) =>
-      choice(
-        $._block_command_sequence_before_close,
-        seq($._terminated_block_line, optional($._block_command_list)),
-      ),
-
-    _block_commands: ($) =>
-      choice($._block_command_list, $._block_command_list_missing_separator),
-
-    _block_command_list_missing_separator: ($) =>
-      choice(
-        missingCommandSeparator($),
-        seq($._block_command_sequence, missingCommandSeparator($)),
-        seq($._terminated_block_line, $._block_command_list_missing_separator),
-      ),
-
-    _terminated_block_line: ($) =>
-      seq($._block_command_sequence, alias("\n", $.command_separator)),
+    _block_item: ($) =>
+      choice($._separable_command_item, editingCommand($, "line_terminated")),
 
     empty_command: ($) => seq(optional($._blanks), $._empty_command_marker),
 
@@ -429,12 +422,9 @@ function addressRules(mode) {
     _max2_excess_address_clause: ($) =>
       prec.dynamic(
         1,
-        choice(
-          seq($._double_address_clause, issueField($, "excess_address_unit2")),
-          seq(
-            $._max2_excess_address_clause,
-            issueField($, "excess_address_unit2"),
-          ),
+        seq(
+          $._double_address_clause,
+          repeat1(issueField($, "excess_address_unit2")),
         ),
       ),
 
@@ -444,29 +434,27 @@ function addressRules(mode) {
     _max1_excess_address_clause: ($) =>
       prec.dynamic(
         1,
-        choice(
-          seq(field("first", $.address), issueField($, "excess_address_unit1")),
-          seq(
+        seq(
+          choice(
+            field("first", $.address),
             issueField($, "omitted_first_address_unit1"),
-            issueField($, "excess_address_unit1"),
-          ),
-          seq(
-            $._max1_excess_address_clause,
-            issueField($, "excess_address_unit1"),
           ),
+          repeat1(issueField($, "excess_address_unit1")),
         ),
       ),
 
     _max0_address_clause: ($) =>
       prec.dynamic(
         1,
-        choice(
-          issueField($, "excess_address"),
-          seq(
-            issueField($, "omitted_first_address_unit0"),
-            issueField($, "excess_address_unit0"),
+        seq(
+          choice(
+            issueField($, "excess_address"),
+            seq(
+              issueField($, "omitted_first_address_unit0"),
+              issueField($, "excess_address_unit0"),
+            ),
           ),
-          seq($._max0_address_clause, issueField($, "excess_address_unit0")),
+          repeat(issueField($, "excess_address_unit0")),
         ),
       ),
 
@@ -567,21 +555,17 @@ function operandRules(mode) {
       ),
 
     replacement: ($) =>
-      repeat1(
-        choice(
-          $.replacement_literal,
-          seq($._escape_prefix, issueField($, "invalid_encoding")),
-          issueField($, "nul_character"),
-          issueField($, "invalid_encoding"),
-          $.matched_text_reference,
-          $.replacement_backreference,
-          $.replacement_escaped_delimiter,
-          issueField($, "replacement_ampersand_delimiter_escape"),
-          $.replacement_escape,
-          $.escaped_newline,
-          issueField($, "unspecified_replacement_escape"),
-          issueField($, "incomplete_replacement_escape"),
-        ),
+      operandAtoms(
+        $,
+        $.replacement_literal,
+        $.matched_text_reference,
+        $.replacement_backreference,
+        $.replacement_escaped_delimiter,
+        issueField($, "replacement_ampersand_delimiter_escape"),
+        $.replacement_escape,
+        $.escaped_newline,
+        issueField($, "unspecified_replacement_escape"),
+        issueField($, "incomplete_replacement_escape"),
       ),
 
     replacement_literal: ($) => $._replacement_literal,
@@ -599,16 +583,8 @@ function operandRules(mode) {
     _substitution_flags: ($) =>
       prec.right(
         choice(
-          $._substitution_flags_without_write,
-          seq(optional($._substitution_flags_without_write), $.write_flag),
-        ),
-      ),
-
-    _substitution_flags_without_write: ($) =>
-      choice(
-        substitutionFlagChoice($),
-        prec.left(
-          seq($._substitution_flags_without_write, substitutionFlagChoice($)),
+          repeat1(substitutionFlagChoice($)),
+          seq(repeat(substitutionFlagChoice($)), $.write_flag),
         ),
       ),
 
@@ -646,17 +622,13 @@ function operandRules(mode) {
       ),
 
     translation_string: ($) =>
-      repeat1(
-        choice(
-          $.translation_literal,
-          seq($._escape_prefix, issueField($, "invalid_encoding")),
-          issueField($, "nul_character"),
-          issueField($, "invalid_encoding"),
-          $.translation_escape,
-          $.translation_escaped_delimiter,
-          issueField($, "undefined_translation_escape"),
-          issueField($, "incomplete_translation_escape"),
-        ),
+      operandAtoms(
+        $,
+        $.translation_literal,
+        $.translation_escape,
+        $.translation_escaped_delimiter,
+        issueField($, "undefined_translation_escape"),
+        issueField($, "incomplete_translation_escape"),
       ),
 
     translation_literal: ($) => $._translate_literal,
@@ -675,16 +647,12 @@ function functionRules() {
   const rules = {
     text: ($) =>
       seq(
-        repeat1(
-          choice(
-            $.text_literal,
-            seq($._escape_prefix, issueField($, "invalid_encoding")),
-            issueField($, "nul_character"),
-            issueField($, "invalid_encoding"),
-            $.text_backslash_escape,
-            $.text_escaped_newline,
-            issueField($, "unspecified_text_escape"),
-          ),
+        operandAtoms(
+          $,
+          $.text_literal,
+          $.text_backslash_escape,
+          $.text_escaped_newline,
+          issueField($, "unspecified_text_escape"),
         ),
         $._text_tail,
       ),
@@ -742,6 +710,7 @@ function functionRules() {
   const formArguments = {
     block: ($) => [
       field("commands", alias($._block_commands, $.command_list)),
+      optional($._brace_leading_blanks),
       choice(
         field("closing", $.closing_brace),
         issueField($, "missing_closing_brace"),
@@ -935,6 +904,25 @@ function sedRules(mode) {
   };
 }
 
+const missingCommandDefinitions = [
+  { reason: "missing_function", atBoundary: true },
+  { reason: "missing_label", atBoundary: true },
+  { reason: "missing_rfile", atBoundary: true },
+  { reason: "missing_wfile", atBoundary: true },
+  { reason: "missing_text_introducer", atBoundary: true },
+  {
+    reason: "incomplete_text_introducer",
+    rule: ($) => $._text_incomplete_introducer,
+  },
+  { reason: "missing_text" },
+  { reason: "missing_closing_brace" },
+  { reason: "missing_opening_delimiter", atBoundary: true },
+];
+
+const boundaryMissingReasons = missingCommandDefinitions
+  .filter(({ atBoundary }) => atBoundary)
+  .map(({ reason }) => reason);
+
 function missingMarkerNames(mode) {
   return [
     "omitted_address",
@@ -956,12 +944,7 @@ function missingMarkerNames(mode) {
     "missing_opening_delimiter",
     "missing_separator_before_unmatched_brace",
     "missing_separator_after_unmatched_brace",
-    "nonconforming_missing_function",
-    "nonconforming_missing_label",
-    "nonconforming_missing_rfile",
-    "nonconforming_missing_wfile",
-    "nonconforming_missing_text_introducer",
-    "nonconforming_missing_opening_delimiter",
+    ...boundaryMissingReasons.map((reason) => `nonconforming_${reason}`),
     "missing_subexpression_placeholder",
     "incomplete_bracket_list",
     "incomplete_bracket_expression",
@@ -975,15 +958,6 @@ function issueDefinitions(mode) {
     return ($) => $[`_${reason}_marker`];
   }
 
-  const boundaryMissingReasons = [
-    "missing_function",
-    "missing_label",
-    "missing_rfile",
-    "missing_wfile",
-    "missing_text_introducer",
-    "missing_opening_delimiter",
-  ];
-
   return [
     {
       reason: "malformed_bracket_term",
@@ -1357,7 +1331,11 @@ function issueDefinitions(mode) {
       reason: "invalid_substitution_flag",
       outcome: "nonconforming_syntax",
       rule: ($) =>
-        choice($._invalid_substitution_flag, issueField($, "invalid_encoding")),
+        choice(
+          $._invalid_substitution_flag,
+          $._nul_character,
+          issueField($, "invalid_encoding"),
+        ),
     },
     {
       reason: "invalid_delimiter",
@@ -1403,51 +1381,11 @@ function issueDefinitions(mode) {
       outcome: "nonconforming_syntax",
       rule: missing(`nonconforming_${reason}`),
     })),
-    {
-      reason: "missing_function",
-      outcome: "incomplete_syntax",
-      rule: missing("missing_function"),
-    },
-    {
-      reason: "missing_label",
-      outcome: "incomplete_syntax",
-      rule: missing("missing_label"),
-    },
-    {
-      reason: "missing_rfile",
-      outcome: "incomplete_syntax",
-      rule: missing("missing_rfile"),
-    },
-    {
-      reason: "missing_wfile",
-      outcome: "incomplete_syntax",
-      rule: missing("missing_wfile"),
-    },
-    {
-      reason: "missing_text_introducer",
-      outcome: "incomplete_syntax",
-      rule: missing("missing_text_introducer"),
-    },
-    {
-      reason: "incomplete_text_introducer",
-      outcome: "incomplete_syntax",
-      rule: ($) => $._text_incomplete_introducer,
-    },
-    {
-      reason: "missing_text",
-      outcome: "incomplete_syntax",
-      rule: missing("missing_text"),
-    },
-    {
-      reason: "missing_closing_brace",
-      outcome: "incomplete_syntax",
-      rule: missing("missing_closing_brace"),
-    },
-    {
-      reason: "missing_opening_delimiter",
+    ...missingCommandDefinitions.map(({ reason, rule }) => ({
+      reason,
       outcome: "incomplete_syntax",
-      rule: missing("missing_opening_delimiter"),
-    },
+      rule: rule ?? missing(reason),
+    })),
     {
       reason: "missing_subexpression",
       outcome: "incomplete_syntax",
@@ -1651,6 +1589,7 @@ function externalTokens($, mode) {
     $._line_word,
     $._argument_separator,
     $._right_brace,
+    $._brace_leading_blanks,
     $._block_trailing_blanks,
     $._empty_command_marker,
     $._reserved_unknown_function_token,

package/common/regex.js

@@ -123,6 +123,33 @@ function bracketExpression($, ambiguous) {
   );
 }
 
+// Character and equivalence classes are not portable range endpoints; their
+// ranges keep the range_expression shape with the endpoint inside an issue.
+const nonportableRangeClasses = ["character_class", "equivalence_class"];
+
+function rangeOperator($) {
+  return field(
+    "operator",
+    namedExternal($, $._regex_bracket_hyphen, "range_operator"),
+  );
+}
+
+function nonportableRangeRules() {
+  const rules = {};
+  for (const kind of nonportableRangeClasses) {
+    rules[`_${kind}_start_range`] = ($) =>
+      seq(
+        field("start", issueNode($, `${kind}_range_start`)),
+        rangeOperator($),
+      );
+  }
+  for (const kind of nonportableRangeClasses) {
+    rules[`_${kind}_end_range`] = ($) =>
+      field("term", issueNode($, `${kind}_range_end`));
+  }
+  return rules;
+}
+
 function bracketRules() {
   return {
     bracket_expression: ($) => bracketExpression($, false),
@@ -193,8 +220,9 @@ function bracketRules() {
           "start",
           choice(
             $.start_range,
-            alias($._character_class_start_range, $.start_range),
-            alias($._equivalence_class_start_range, $.start_range),
+            ...nonportableRangeClasses.map((kind) =>
+              alias($[`_${kind}_start_range`], $.start_range),
+            ),
           ),
         ),
         choice(
@@ -202,8 +230,9 @@ function bracketRules() {
             "end",
             choice(
               $.end_range,
-              alias($._character_class_end_range, $.end_range),
-              alias($._equivalence_class_end_range, $.end_range),
+              ...nonportableRangeClasses.map((kind) =>
+                alias($[`_${kind}_end_range`], $.end_range),
+              ),
             ),
           ),
           field(
@@ -217,38 +246,9 @@ function bracketRules() {
         ),
       ),
 
-    _character_class_start_range: ($) =>
-      seq(
-        field("start", issueNode($, "character_class_range_start")),
-        field(
-          "operator",
-          namedExternal($, $._regex_bracket_hyphen, "range_operator"),
-        ),
-      ),
-
-    _equivalence_class_start_range: ($) =>
-      seq(
-        field("start", issueNode($, "equivalence_class_range_start")),
-        field(
-          "operator",
-          namedExternal($, $._regex_bracket_hyphen, "range_operator"),
-        ),
-      ),
-
-    _character_class_end_range: ($) =>
-      field("term", issueNode($, "character_class_range_end")),
-
-    _equivalence_class_end_range: ($) =>
-      field("term", issueNode($, "equivalence_class_range_end")),
+    ...nonportableRangeRules(),
 
-    start_range: ($) =>
-      seq(
-        field("start", $.end_range),
-        field(
-          "operator",
-          namedExternal($, $._regex_bracket_hyphen, "range_operator"),
-        ),
-      ),
+    start_range: ($) => seq(field("start", $.end_range), rangeOperator($)),
 
     end_range: ($) =>
       choice(
@@ -316,6 +316,20 @@ function regularExpressionEscape($) {
   );
 }
 
+function oneCharOrCollElem($, ...additional) {
+  return choice(
+    issueField($, "invalid_regular_expression_character"),
+    issueField($, "invalid_encoding"),
+    $.ordinary_character,
+    regularExpressionEscape($),
+    $.sed_newline_escape,
+    $.period,
+    $._bracket_expression,
+    issueField($, "special_delimiter_escape"),
+    ...additional,
+  );
+}
+
 function commonRegularExpressionRules() {
   return {
     dup_count: ($) => $._regex_dup_count,
@@ -432,15 +446,8 @@ function breRules() {
     backreference: ($) => $._regex_backreference,
 
     one_char_or_coll_elem_bre: ($) =>
-      choice(
-        issueField($, "invalid_regular_expression_character"),
-        issueField($, "invalid_encoding"),
-        $.ordinary_character,
-        regularExpressionEscape($),
-        $.sed_newline_escape,
-        $.period,
-        $._bracket_expression,
-        issueField($, "special_delimiter_escape"),
+      oneCharOrCollElem(
+        $,
         issueField($, "bre_vertical_line_escape"),
         issueField($, "bre_question_mark_escape"),
         issueField($, "bre_plus_escape"),
@@ -524,17 +531,7 @@ function ereRules() {
         field("operator", issueNode($, "leading_duplication_symbol")),
       ),
 
-    one_char_or_coll_elem_ere: ($) =>
-      choice(
-        issueField($, "invalid_regular_expression_character"),
-        issueField($, "invalid_encoding"),
-        $.ordinary_character,
-        regularExpressionEscape($),
-        $.sed_newline_escape,
-        $.period,
-        $._bracket_expression,
-        issueField($, "special_delimiter_escape"),
-      ),
+    one_char_or_coll_elem_ere: ($) => oneCharOrCollElem($),
 
     ere_dupl_symbol: ($) =>
       seq(