tree-sitter-sed 0.12.0 → 0.13.0

package/Cargo.toml

@@ -10,7 +10,7 @@ name = "tree-sitter-sed"
 publish = false
 readme = "README.md"
 repository = "https://github.com/konomanoasa/tree-sitter-sed"
-version = "0.12.0"
+version = "0.13.0"
 
 build = "bindings/rust/build.rs"
 include = [

package/bindings/rust/build.rs

@@ -10,8 +10,9 @@ fn compile_parser(
   let mut c_config = cc::Build::new();
   c_config.std("c17").include(source_dir);
 
-  #[cfg(target_env = "msvc")]
-  c_config.flag("-utf-8");
+  if c_config.get_compiler().is_like_msvc() {
+    c_config.flag("-utf-8");
+  }
 
   if let Some(headers) = wasm_headers {
     c_config.include(headers);

package/bindings/rust/lib.rs

@@ -426,27 +426,86 @@ mod tests {
     {
       let mut parser = tree_sitter::Parser::new();
       parser.set_language(&language.into()).unwrap();
-      for (name, source, start, row, column, normal_kind, allow_replacement) in [
+      for (name, source, start, row, column, normal_nodes, allow_replacement) in [
         (
           "initial comment text",
           &b"#xn\np\n"[..],
           1,
           0,
           1,
-          "comment_text",
+          ("comment_text", None),
+          true,
+        ),
+        (
+          "label definition",
+          &b":axb\np\n"[..],
+          2,
+          0,
+          2,
+          ("label_literal", Some("label")),
+          true,
+        ),
+        (
+          "branch label",
+          &b"b axb\np\n"[..],
+          3,
+          0,
+          3,
+          ("label_literal", Some("label")),
+          true,
+        ),
+        (
+          "test label",
+          &b"t axb\np\n"[..],
+          3,
+          0,
+          3,
+          ("label_literal", Some("label")),
+          true,
+        ),
+        (
+          "rfile",
+          &b"r axb\np\n"[..],
+          3,
+          0,
+          3,
+          ("file_literal", Some("rfile")),
+          true,
+        ),
+        (
+          "wfile",
+          &b"w axb\np\n"[..],
+          3,
+          0,
+          3,
+          ("file_literal", Some("wfile")),
+          true,
+        ),
+        (
+          "substitution wfile",
+          &b"s/a/b/w axb\np\n"[..],
+          9,
+          0,
+          9,
+          ("file_literal", Some("wfile")),
+          true,
+        ),
+        (
+          "text",
+          &b"a\\\naxb\np\n"[..],
+          4,
+          1,
+          1,
+          ("text_literal", None),
           true,
         ),
-        ("label", &b": axb\np\n"[..], 3, 0, 3, "label", true),
-        ("rfile", &b"r axb\np\n"[..], 3, 0, 3, "rfile", true),
-        ("wfile", &b"w axb\np\n"[..], 3, 0, 3, "wfile", true),
-        ("text", &b"a\\\naxb\np\n"[..], 4, 1, 1, "text_literal", true),
         (
           "replacement",
           &b"s/a/axb/\np\n"[..],
           5,
           0,
           5,
-          "replacement_literal",
+          ("replacement_literal", None),
           true,
         ),
         (
@@ -455,7 +514,7 @@ mod tests {
           3,
           0,
           3,
-          "translation_literal",
+          ("translation_literal", None),
           true,
         ),
         (
@@ -464,7 +523,7 @@ mod tests {
           7,
           0,
           7,
-          "translation_literal",
+          ("translation_literal", None),
           true,
         ),
         (
@@ -473,7 +532,7 @@ mod tests {
           5,
           1,
           2,
-          "text_backslash_escape",
+          ("text_backslash_escape", None),
           false,
         ),
         (
@@ -482,7 +541,7 @@ mod tests {
           6,
           0,
           6,
-          "replacement_escape",
+          ("replacement_escape", None),
           false,
         ),
         (
@@ -491,7 +550,7 @@ mod tests {
           4,
           0,
           4,
-          "translation_escape",
+          ("translation_escape", None),
           false,
         ),
         (
@@ -500,7 +559,7 @@ mod tests {
           8,
           0,
           8,
-          "translation_escape",
+          ("translation_escape", None),
           false,
         ),
         (
@@ -509,7 +568,7 @@ mod tests {
           2,
           0,
           2,
-          "ordinary_character",
+          ("ordinary_character", None),
           true,
         ),
         (
@@ -518,7 +577,7 @@ mod tests {
           3,
           0,
           3,
-          "collating_element",
+          ("collating_element", None),
           true,
         ),
         (
@@ -527,7 +586,7 @@ mod tests {
           5,
           0,
           5,
-          "coll_elem_multi",
+          ("coll_elem_multi", None),
           true,
         ),
         (
@@ -536,7 +595,7 @@ mod tests {
           5,
           0,
           5,
-          "coll_elem_multi",
+          ("coll_elem_multi", None),
           true,
         ),
         (
@@ -545,16 +604,25 @@ mod tests {
           5,
           0,
           5,
-          "class_name",
+          ("class_name", None),
           false,
         ),
       ] {
-        for invalid in [&b"\xff"[..], &b"\xff\xfe"[..]] {
+        let (normal_kind, owner_kind) = normal_nodes;
+        let mut invalid_inputs = vec![&b"\xff"[..], &b"\xff\xfe"[..]];
+        if owner_kind.is_some() {
+          invalid_inputs.extend([&b"\0"[..], &b"\xff\0\xfe"[..]]);
+        }
+        for invalid in invalid_inputs {
           let mut tree = parser.parse(source, None).unwrap();
           assert!(!tree.root_node().has_error(), "{mode}, {name}");
           assert!(issue_signatures(&tree).is_empty(), "{mode}, {name}");
           let mut old_length = 1;
-          let mut replacements = vec![invalid, &source[start..start + 1]];
+          let mut replacements = vec![invalid];
+          if owner_kind.is_some() {
+            replacements.push(&b""[..]);
+          }
+          replacements.push(&source[start..start + 1]);
           if allow_replacement {
             replacements.push("\u{fffd}".as_bytes());
           }
@@ -585,23 +653,83 @@ mod tests {
             };
             let expected: Vec<_> = ranges
               .iter()
-              .map(|range| {
-                (
-                  "invalid_syntax".to_owned(),
-                  "invalid_encoding".to_owned(),
-                  *range,
-                )
+              .zip(invalid)
+              .map(|(range, byte)| {
+                let (outcome, reason) = if *byte == 0 {
+                  ("nonconforming_syntax", "nul_character")
+                } else {
+                  ("invalid_syntax", "invalid_encoding")
+                };
+                (outcome.to_owned(), reason.to_owned(), *range)
               })
               .collect();
+            let encoding_ranges: Vec<_> = expected
+              .iter()
+              .filter(|(_, reason, _)| reason == "invalid_encoding")
+              .map(|(_, _, range)| *range)
+              .collect();
             let context =
               format!("{mode}, {name}, invalid {invalid:?}, step {step}");
             for parsed in [&fresh, &incremental] {
               assert!(!parsed.root_node().has_error(), "{context}");
               assert_eq!(issue_signatures(parsed), expected, "{context}");
-              assert_invalid_encoding_leaves(parsed, &ranges, &context);
+              assert_invalid_encoding_leaves(
+                parsed,
+                &encoding_ranges,
+                &context,
+              );
               let mut nodes = vec![parsed.root_node()];
               let mut repaired_leaf = false;
+              let mut owners = 0;
               while let Some(node) = nodes.pop() {
+                if Some(node.kind()) == owner_kind {
+                  owners += 1;
+                  let owner_range = Range {
+                    start_byte: start - 1,
+                    end_byte: start + replacement.len() + 1,
+                    start_point: Point::new(row, column - 1),
+                    end_point: Point::new(row, column + replacement.len() + 1),
+                  };
+                  assert_eq!(node.range(), owner_range, "{context}");
+                  let expected_children = if step == 0 {
+                    let mut children = vec![(
+                      normal_kind,
+                      Range {
+                        end_byte: start,
+                        end_point: Point::new(row, column),
+                        ..owner_range
+                      },
+                    )];
+                    children.extend(
+                      ranges.iter().map(|range| ("syntax_issue", *range)),
+                    );
+                    children.push((
+                      normal_kind,
+                      Range {
+                        start_byte: start + replacement.len(),
+                        start_point: Point::new(
+                          row,
+                          column + replacement.len(),
+                        ),
+                        ..owner_range
+                      },
+                    ));
+                    children
+                  } else {
+                    vec![(normal_kind, owner_range)]
+                  };
+                  let children: Vec<_> = (0..node.child_count())
+                    .map(|index| {
+                      let child = node.child(index).unwrap();
+                      assert!(child.is_named(), "{context}");
+                      if child.kind() == normal_kind {
+                        assert_eq!(child.child_count(), 0, "{context}");
+                      }
+                      (child.kind(), child.range())
+                    })
+                    .collect();
+                  assert_eq!(children, expected_children, "{context}");
+                }
                 if name == "initial comment text" {
                   assert_ne!(
                     node.kind(),
@@ -624,6 +752,9 @@ mod tests {
               if step > 0 {
                 assert!(repaired_leaf, "{context}: normal leaf must return");
               }
+              if owner_kind.is_some() {
+                assert_eq!(owners, 1, "{context}: operand must have one owner");
+              }
             }
             assert_eq!(
               incremental.root_node().to_sexp(),

package/common/grammar.js

@@ -32,13 +32,13 @@ function lineOperand($, first, rest) {
   return prec.right(
     seq(
       choice(
-        first,
+        namedExternal($, first, "file_literal"),
         issueField($, "nul_character"),
         issueField($, "invalid_encoding"),
       ),
       repeat(
         choice(
-          rest,
+          namedExternal($, rest, "file_literal"),
           issueField($, "nul_character"),
           issueField($, "invalid_encoding"),
         ),
@@ -718,7 +718,7 @@ function functionRules() {
       prec.right(
         repeat1(
           choice(
-            $._line_word,
+            namedExternal($, $._line_word, "label_literal"),
             issueField($, "nul_character"),
             issueField($, "invalid_encoding"),
           ),
@@ -746,7 +746,7 @@ function functionRules() {
         field("closing", $.closing_brace),
         issueField($, "missing_closing_brace"),
       ),
-      optional($._blanks),
+      optional($._block_trailing_blanks),
     ],
     text: ($) => [
       choice(
@@ -1651,6 +1651,7 @@ function externalTokens($, mode) {
     $._line_word,
     $._argument_separator,
     $._right_brace,
+    $._block_trailing_blanks,
     $._empty_command_marker,
     $._reserved_unknown_function_token,
     $._regex_incomplete_group,

package/common/scanner.h

@@ -126,6 +126,7 @@ enum TokenType {
   LINE_WORD,
   ARGUMENT_SEPARATOR,
   RIGHT_BRACE,
+  BLOCK_TRAILING_BLANKS,
   EMPTY_COMMAND_MARKER,
   RESERVED_UNKNOWN_FUNCTION_TOKEN,
   REGEX_INCOMPLETE_GROUP,
@@ -2344,6 +2345,13 @@ static bool scan_command_token(
   const bool *valid_symbols,
   TSSymbol *symbol
 ) {
+  if (valid_symbols[BLOCK_TRAILING_BLANKS] && is_blank(lexer->lookahead)) {
+    advance_past_blanks(lexer);
+    lexer->mark_end(lexer);
+    *symbol = BLOCK_TRAILING_BLANKS;
+    return true;
+  }
+
   if (
     valid_symbols[EMPTY_COMMAND_MARKER] &&
     (lexer->lookahead == ';' || lexer->lookahead == '\n')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-sed",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "type": "module",
   "description": "Tree-sitter grammars for POSIX sed.",
   "license": "MIT",

package/queries/highlights.scm

@@ -41,7 +41,7 @@
   (text_escaped_newline)
 ] @punctuation.special
 
-(label) @label
+(label_literal) @label
 
 (last_line_address) @constant.builtin
 
@@ -57,10 +57,7 @@
   (replacement_backreference)
 ] @string.special.symbol
 
-[
-  (rfile)
-  (wfile)
-] @string.special.path
+(file_literal) @string.special.path
 
 [
   (back_close_brace)

package/sed_ere/queries/highlights.scm

@@ -42,7 +42,7 @@
   (text_escaped_newline)
 ] @punctuation.special
 
-(label) @label
+(label_literal) @label
 
 (last_line_address) @constant.builtin
 
@@ -57,10 +57,7 @@
   (replacement_backreference)
 ] @string.special.symbol
 
-[
-  (rfile)
-  (wfile)
-] @string.special.path
+(file_literal) @string.special.path
 
 [
   (close_brace)

package/sed_ere/src/grammar.json

@@ -2910,8 +2910,13 @@
             "type": "CHOICE",
             "members": [
               {
-                "type": "SYMBOL",
-                "name": "_file_argument"
+                "type": "ALIAS",
+                "content": {
+                  "type": "SYMBOL",
+                  "name": "_file_argument"
+                },
+                "named": true,
+                "value": "file_literal"
               },
               {
                 "type": "FIELD",
@@ -2947,8 +2952,13 @@
               "type": "CHOICE",
               "members": [
                 {
-                  "type": "SYMBOL",
-                  "name": "_line_word"
+                  "type": "ALIAS",
+                  "content": {
+                    "type": "SYMBOL",
+                    "name": "_line_word"
+                  },
+                  "named": true,
+                  "value": "file_literal"
                 },
                 {
                   "type": "FIELD",
@@ -2992,8 +3002,13 @@
             "type": "CHOICE",
             "members": [
               {
-                "type": "SYMBOL",
-                "name": "_file_argument"
+                "type": "ALIAS",
+                "content": {
+                  "type": "SYMBOL",
+                  "name": "_file_argument"
+                },
+                "named": true,
+                "value": "file_literal"
               },
               {
                 "type": "FIELD",
@@ -3029,8 +3044,13 @@
               "type": "CHOICE",
               "members": [
                 {
-                  "type": "SYMBOL",
-                  "name": "_line_word"
+                  "type": "ALIAS",
+                  "content": {
+                    "type": "SYMBOL",
+                    "name": "_line_word"
+                  },
+                  "named": true,
+                  "value": "file_literal"
                 },
                 {
                   "type": "FIELD",
@@ -3074,8 +3094,13 @@
             "type": "CHOICE",
             "members": [
               {
-                "type": "SYMBOL",
-                "name": "_substitution_wfile_argument"
+                "type": "ALIAS",
+                "content": {
+                  "type": "SYMBOL",
+                  "name": "_substitution_wfile_argument"
+                },
+                "named": true,
+                "value": "file_literal"
               },
               {
                 "type": "FIELD",
@@ -3111,8 +3136,13 @@
               "type": "CHOICE",
               "members": [
                 {
-                  "type": "SYMBOL",
-                  "name": "_substitution_wfile_continuation"
+                  "type": "ALIAS",
+                  "content": {
+                    "type": "SYMBOL",
+                    "name": "_substitution_wfile_continuation"
+                  },
+                  "named": true,
+                  "value": "file_literal"
                 },
                 {
                   "type": "FIELD",
@@ -3155,8 +3185,13 @@
           "type": "CHOICE",
           "members": [
             {
-              "type": "SYMBOL",
-              "name": "_line_word"
+              "type": "ALIAS",
+              "content": {
+                "type": "SYMBOL",
+                "name": "_line_word"
+              },
+              "named": true,
+              "value": "label_literal"
             },
             {
               "type": "FIELD",
@@ -3519,7 +3554,7 @@
           "members": [
             {
               "type": "SYMBOL",
-              "name": "_blanks"
+              "name": "_block_trailing_blanks"
             },
             {
               "type": "BLANK"
@@ -10442,6 +10477,10 @@
       "type": "SYMBOL",
       "name": "_right_brace"
     },
+    {
+      "type": "SYMBOL",
+      "name": "_block_trailing_blanks"
+    },
     {
       "type": "SYMBOL",
       "name": "_empty_command_marker"

package/sed_ere/src/node-types.json

@@ -1739,6 +1739,16 @@
           }
         ]
       }
+    },
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "label_literal",
+          "named": true
+        }
+      ]
     }
   },
   {
@@ -2450,6 +2460,16 @@
           }
         ]
       }
+    },
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "file_literal",
+          "named": true
+        }
+      ]
     }
   },
   {
@@ -3183,6 +3203,16 @@
           }
         ]
       }
+    },
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "file_literal",
+          "named": true
+        }
+      ]
     }
   },
   {
@@ -3359,6 +3389,10 @@
     "type": "delimiter",
     "named": true
   },
+  {
+    "type": "file_literal",
+    "named": true
+  },
   {
     "type": "function_verb",
     "named": true
@@ -3371,6 +3405,10 @@
     "type": "interval_separator",
     "named": true
   },
+  {
+    "type": "label_literal",
+    "named": true
+  },
   {
     "type": "last_line_address",
     "named": true