tree-sitter-sed 0.2.0 → 0.4.0

package/common/scanner.h

@@ -1,5 +1,5 @@
-#ifndef TREE_SITTER_POSIX_SED_COMMON_SCANNER_H_
-#define TREE_SITTER_POSIX_SED_COMMON_SCANNER_H_
+#ifndef TREE_SITTER_SED_COMMON_SCANNER_H_
+#define TREE_SITTER_SED_COMMON_SCANNER_H_
 
 #ifndef SED_REGEX_EXTENDED
 #error "SED_REGEX_EXTENDED must be defined before including scanner.h"
@@ -42,7 +42,6 @@ enum TokenType {
   BRE_QUESTION_MARK_ESCAPE_MARKER,
   BRE_PLUS_ESCAPE_MARKER,
   REGEX_UNMATCHED_INTERVAL_CLOSE,
-  UNMATCHED_INTERVAL_CLOSE_MARKER,
 #endif
 #if SED_REGEX_EXTENDED
   REGEX_ALTERNATION_OPERATOR,
@@ -109,6 +108,7 @@ enum TokenType {
   TRANSLATE_UNTERMINATED_SOURCE,
   TRANSLATE_UNTERMINATED_DESTINATION,
   INVALID_SUBSTITUTION_FLAG,
+  FLAG_AFTER_WRITE_MARKER,
   TEXT_COMMAND_START,
   TEXT_LITERAL,
   TEXT_BACKSLASH_ESCAPE,
@@ -122,7 +122,7 @@ enum TokenType {
   SUBSTITUTION_WFILE_ARGUMENT,
   LINE_WORD,
   RIGHT_BRACE,
-  RESERVED_UNKNOWN_FUNCTION,
+  RESERVED_UNKNOWN_FUNCTION_TOKEN,
   REGEX_INCOMPLETE_GROUP,
   REGEX_INCOMPLETE_BRACKET_TERM,
   REGEX_INCOMPLETE_INTERVAL,
@@ -132,15 +132,15 @@ enum TokenType {
   REPLACEMENT_LINE_UNTERMINATED,
   TRANSLATE_LINE_UNTERMINATED_SOURCE,
   TRANSLATE_LINE_UNTERMINATED_DESTINATION,
+  BLANKS_AROUND_ADDRESS_SEPARATOR,
   OMITTED_ADDRESS_MARKER,
+  INCOMPLETE_OMITTED_ADDRESS_MARKER,
   OMITTED_FIRST_ADDRESS_MARKER,
   EMPTY_SUBEXPRESSION_MARKER,
   MISSING_SUBEXPRESSION_MARKER,
 #if SED_REGEX_EXTENDED
   EMPTY_ALTERNATIVE_MARKER,
 #endif
-  EXCESS_ADDRESSES_MARKER,
-  ADDITIONAL_ADDRESS_MARKER,
   MISSING_FUNCTION_MARKER,
   MISSING_LABEL_MARKER,
   MISSING_RFILE_MARKER,
@@ -149,9 +149,7 @@ enum TokenType {
   MISSING_TEXT_INTRODUCER_MARKER,
   MISSING_TEXT_MARKER,
   MISSING_COMMAND_SEPARATOR_MARKER,
-  BLANKS_AROUND_ADDRESS_SEPARATOR_MARKER,
   MISSING_ADDRESS_SEPARATOR_MARKER,
-  DUPLICATE_NEGATION_MARKER,
   MISSING_CLOSING_BRACE_MARKER,
   MISSING_OPENING_DELIMITER_MARKER,
   MISSING_SEPARATOR_BEFORE_UNMATCHED_BRACE_MARKER,
@@ -248,6 +246,17 @@ enum {
   SCANNER_SERIALIZED_STATE_SIZE = 17,
 };
 
+enum {
+  SERIALIZED_AT_BRANCH_START_FLAG = 1 << 0,
+  SERIALIZED_AFTER_ALTERNATION_FLAG = 1 << 1,
+  SERIALIZED_AFTER_ANCHOR_FLAG = 1 << 2,
+  SERIALIZED_TEXT_STATE_SHIFT = 3,
+  SERIALIZED_MODE_FLAGS_MAXIMUM = 31,
+  SERIALIZED_RANGE_PENDING_FLAG = 1 << 0,
+  SERIALIZED_DUPLICATION_STATE_SHIFT = 1,
+  SERIALIZED_BRACKET_FLAGS_MAXIMUM = 7,
+};
+
 static void reset_bracket_tracking(ScannerState *state) {
   state->regex_bracket_term_state = REGEX_BRACKET_TERM_NONE;
   state->regex_bracket_pending_element = REGEX_BRACKET_PENDING_NONE;
@@ -306,6 +315,10 @@ static uint16_t deserialize_uint16(const char *buffer, unsigned offset) {
     ((uint16_t)(unsigned char)buffer[offset + 1] << 8));
 }
 
+static bool is_regex_mode(enum ScannerMode mode) {
+  return mode == MODE_REGEX_ADDRESS || mode == MODE_SUBSTITUTE_PATTERN;
+}
+
 static unsigned sed_scanner_serialize(void *payload, char *buffer) {
   const ScannerState *state = payload;
   const uint32_t delimiter = (uint32_t)state->delimiter;
@@ -313,10 +326,11 @@ static unsigned sed_scanner_serialize(void *payload, char *buffer) {
   buffer[0] = SCANNER_SERIALIZATION_VERSION;
   buffer[1] = (char)state->mode;
   buffer[2] = (char)state->regex_state;
-  buffer[3] = (state->regex_at_branch_start ? 1 : 0) |
-    (state->regex_after_alternation ? 2 : 0) |
-    (state->regex_after_anchor ? 4 : 0) |
-    ((unsigned char)state->text_state << 3);
+  buffer[3] =
+    (state->regex_at_branch_start ? SERIALIZED_AT_BRANCH_START_FLAG : 0) |
+    (state->regex_after_alternation ? SERIALIZED_AFTER_ALTERNATION_FLAG : 0) |
+    (state->regex_after_anchor ? SERIALIZED_AFTER_ANCHOR_FLAG : 0) |
+    ((unsigned char)state->text_state << SERIALIZED_TEXT_STATE_SHIFT);
   buffer[4] = (char)(delimiter & UINT32_C(0xff));
   buffer[5] = (char)((delimiter >> 8) & UINT32_C(0xff));
   buffer[6] = (char)((delimiter >> 16) & UINT32_C(0xff));
@@ -328,11 +342,103 @@ static unsigned sed_scanner_serialize(void *payload, char *buffer) {
   buffer[13] = (char)state->regex_bracket_first_element;
   buffer[14] = (char)state->regex_bracket_last_element;
   buffer[15] = (char)state->regex_bracket_element_count;
-  buffer[16] = (state->regex_bracket_range_pending ? 1 : 0) |
-    ((unsigned char)state->regex_duplication_state << 1);
+  buffer[16] =
+    (state->regex_bracket_range_pending ? SERIALIZED_RANGE_PENDING_FLAG : 0) |
+    ((unsigned char)state->regex_duplication_state
+      << SERIALIZED_DUPLICATION_STATE_SHIFT);
   return SCANNER_SERIALIZED_STATE_SIZE;
 }
 
+static bool bracket_tracking_is_reset(const ScannerState *state) {
+  return state->regex_bracket_term_state ==
+    REGEX_BRACKET_TERM_NONE &&
+    state->regex_bracket_pending_element ==
+    REGEX_BRACKET_PENDING_NONE &&
+    state->regex_bracket_first_element ==
+    REGEX_BRACKET_PENDING_NONE &&
+    state->regex_bracket_last_element ==
+    REGEX_BRACKET_PENDING_NONE &&
+    state->regex_bracket_element_count ==
+    0 &&
+    !state->regex_bracket_range_pending;
+}
+
+static bool regex_position_is_reset(const ScannerState *state) {
+  return !state->regex_at_branch_start &&
+    state->regex_duplication_state ==
+    REGEX_DUPLICATION_NONE &&
+    !state->regex_after_alternation &&
+    !state->regex_after_anchor;
+}
+
+static bool deserialized_delimiter_is_valid(uint32_t delimiter) {
+  return delimiter !=
+    0 &&
+    delimiter <=
+    UINT32_C(0x10ffff) &&
+    !(delimiter >= UINT32_C(0xd800) && delimiter <= UINT32_C(0xdfff)) &&
+    delimiter !=
+    (uint32_t)'\\' &&
+    delimiter != (uint32_t)'\n';
+}
+
+static bool deserialized_state_is_valid(const ScannerState *candidate) {
+  if (candidate->mode == MODE_NONE) {
+    return false;
+  }
+
+  if (candidate->mode == MODE_TEXT) {
+    return candidate->delimiter ==
+      0 &&
+      candidate->regex_state ==
+      REGEX_OUTSIDE_BRACKET &&
+      regex_position_is_reset(candidate) &&
+      candidate->regex_group_depth ==
+      0 &&
+      candidate->regex_interval_state ==
+      REGEX_INTERVAL_NONE &&
+      bracket_tracking_is_reset(candidate);
+  }
+
+  if (!deserialized_delimiter_is_valid((uint32_t)candidate->delimiter)) {
+    return false;
+  }
+
+  if (!is_regex_mode(candidate->mode)) {
+    return candidate->regex_state ==
+      REGEX_OUTSIDE_BRACKET &&
+      regex_position_is_reset(candidate) &&
+      candidate->text_state ==
+      TEXT_EMPTY &&
+      candidate->regex_interval_state ==
+      REGEX_INTERVAL_NONE &&
+      bracket_tracking_is_reset(candidate) &&
+      candidate->regex_group_depth == 0;
+  }
+
+  if (candidate->text_state != TEXT_EMPTY) {
+    return false;
+  }
+
+  if (candidate->regex_state == REGEX_OUTSIDE_BRACKET) {
+    return bracket_tracking_is_reset(candidate);
+  }
+
+  if (!regex_position_is_reset(candidate)) {
+    return false;
+  }
+
+  if (
+    candidate->regex_state ==
+    REGEX_BRACKET_FIRST ||
+    candidate->regex_state == REGEX_BRACKET_AFTER_CARET
+  ) {
+    return bracket_tracking_is_reset(candidate);
+  }
+
+  return true;
+}
+
 static void
 sed_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
   ScannerState *state = payload;
@@ -346,256 +452,106 @@ sed_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
     return;
   }
 
-  const enum ScannerMode mode = (enum ScannerMode)(unsigned char)buffer[1];
-  const enum RegexState regex_state = (enum RegexState)(unsigned char)buffer[2];
   const unsigned char mode_flags = (unsigned char)buffer[3];
-  const uint32_t delimiter = (uint32_t)(unsigned char)buffer[4] |
-    ((uint32_t)(unsigned char)buffer[5] << 8) |
-    ((uint32_t)(unsigned char)buffer[6] << 16) |
-    ((uint32_t)(unsigned char)buffer[7] << 24);
-  const uint16_t regex_group_depth = deserialize_uint16(buffer, 8);
-  const enum RegexIntervalState regex_interval_state =
-    (enum RegexIntervalState)(unsigned char)buffer[10];
-  const enum RegexBracketTermState regex_bracket_term_state =
-    (enum RegexBracketTermState)(unsigned char)buffer[11];
-  const enum RegexBracketPendingElement regex_bracket_pending_element =
-    (enum RegexBracketPendingElement)(unsigned char)buffer[12];
-  const enum RegexBracketPendingElement regex_bracket_first_element =
-    (enum RegexBracketPendingElement)(unsigned char)buffer[13];
-  const enum RegexBracketPendingElement regex_bracket_last_element =
-    (enum RegexBracketPendingElement)(unsigned char)buffer[14];
-  const uint8_t regex_bracket_element_count =
-    (uint8_t)(unsigned char)buffer[15];
   const unsigned char regex_bracket_flags = (unsigned char)buffer[16];
-  const bool regex_bracket_range_pending = (regex_bracket_flags & 1U) != 0;
-  const enum RegexDuplicationState regex_duplication_state =
-    (enum RegexDuplicationState)((regex_bracket_flags >> 1) & 3U);
-
-  if (mode > MODE_TEXT) {
+  const ScannerState candidate = {
+    .delimiter = (int32_t)((uint32_t)(unsigned char)buffer[4] |
+      ((uint32_t)(unsigned char)buffer[5] << 8) |
+      ((uint32_t)(unsigned char)buffer[6] << 16) |
+      ((uint32_t)(unsigned char)buffer[7] << 24)),
+    .mode = (enum ScannerMode)(unsigned char)buffer[1],
+    .regex_state = (enum RegexState)(unsigned char)buffer[2],
+    .regex_at_branch_start =
+      (mode_flags & SERIALIZED_AT_BRANCH_START_FLAG) != 0,
+    .regex_duplication_state = (enum RegexDuplicationState)(
+      (regex_bracket_flags >> SERIALIZED_DUPLICATION_STATE_SHIFT) & 3U
+    ),
+    .regex_after_alternation =
+      (mode_flags & SERIALIZED_AFTER_ALTERNATION_FLAG) != 0,
+    .regex_after_anchor = (mode_flags & SERIALIZED_AFTER_ANCHOR_FLAG) != 0,
+    .text_state =
+      (enum TextState)((mode_flags >> SERIALIZED_TEXT_STATE_SHIFT) & 3U),
+    .regex_interval_state = (enum RegexIntervalState)(unsigned char)buffer[10],
+    .regex_bracket_term_state =
+      (enum RegexBracketTermState)(unsigned char)buffer[11],
+    .regex_bracket_pending_element =
+      (enum RegexBracketPendingElement)(unsigned char)buffer[12],
+    .regex_bracket_first_element =
+      (enum RegexBracketPendingElement)(unsigned char)buffer[13],
+    .regex_bracket_last_element =
+      (enum RegexBracketPendingElement)(unsigned char)buffer[14],
+    .regex_bracket_element_count = (uint8_t)(unsigned char)buffer[15],
+    .regex_bracket_range_pending =
+      (regex_bracket_flags & SERIALIZED_RANGE_PENDING_FLAG) != 0,
+    .regex_group_depth = deserialize_uint16(buffer, 8),
+  };
+
+  if (candidate.mode > MODE_TEXT) {
     return;
   }
 
-  if (regex_state > REGEX_BRACKET_BODY) {
+  if (candidate.regex_state > REGEX_BRACKET_BODY) {
     return;
   }
 
-  if ((mode_flags & (unsigned char)~31U) != 0) {
+  if ((mode_flags & (unsigned char)~SERIALIZED_MODE_FLAGS_MAXIMUM) != 0) {
     return;
   }
-  const bool regex_at_branch_start = (mode_flags & 1U) != 0;
-  const bool regex_after_alternation = (mode_flags & 2U) != 0;
-  const bool regex_after_anchor = (mode_flags & 4U) != 0;
-  const enum TextState text_state = (enum TextState)((mode_flags >> 3) & 3U);
-  if (text_state > TEXT_AFTER_ESCAPED_NEWLINE) {
+  if (candidate.text_state > TEXT_AFTER_ESCAPED_NEWLINE) {
     return;
   }
 #if !SED_REGEX_EXTENDED
-  if (regex_after_alternation) {
+  if (candidate.regex_after_alternation) {
     return;
   }
 #endif
 
-  if (regex_interval_state > REGEX_INTERVAL_AFTER_MAXIMUM) {
+  if (candidate.regex_interval_state > REGEX_INTERVAL_AFTER_MAXIMUM) {
     return;
   }
-  if (regex_bracket_term_state > REGEX_BRACKET_TERM_EQUAL) {
+  if (candidate.regex_bracket_term_state > REGEX_BRACKET_TERM_EQUAL) {
     return;
   }
   if (
-    regex_bracket_pending_element >
+    candidate.regex_bracket_pending_element >
     REGEX_BRACKET_PENDING_COLON ||
-    regex_bracket_first_element >
+    candidate.regex_bracket_first_element >
     REGEX_BRACKET_PENDING_COLON ||
-    regex_bracket_last_element >
+    candidate.regex_bracket_last_element >
     REGEX_BRACKET_PENDING_COLON ||
-    regex_bracket_element_count >
+    candidate.regex_bracket_element_count >
     3U ||
-    (regex_bracket_flags & (unsigned char)~7U) !=
+    (regex_bracket_flags & (unsigned char)~SERIALIZED_BRACKET_FLAGS_MAXIMUM) !=
     0 ||
 #if SED_REGEX_EXTENDED
-    regex_duplication_state > REGEX_AFTER_REPETITION_MODIFIER
+    candidate.regex_duplication_state > REGEX_AFTER_REPETITION_MODIFIER
 #else
-    regex_duplication_state > REGEX_AFTER_DUPLICATION_SYMBOL
+    candidate.regex_duplication_state > REGEX_AFTER_DUPLICATION_SYMBOL
 #endif
   ) {
     return;
   }
   if (
-    (regex_bracket_element_count ==
+    (candidate.regex_bracket_element_count ==
       0 &&
-      (regex_bracket_first_element !=
+      (candidate.regex_bracket_first_element !=
         REGEX_BRACKET_PENDING_NONE ||
-        regex_bracket_last_element != REGEX_BRACKET_PENDING_NONE)) ||
-    (regex_bracket_element_count >
+        candidate.regex_bracket_last_element != REGEX_BRACKET_PENDING_NONE)) ||
+    (candidate.regex_bracket_element_count >
       0 &&
-      (regex_bracket_first_element ==
+      (candidate.regex_bracket_first_element ==
         REGEX_BRACKET_PENDING_NONE ||
-        regex_bracket_last_element == REGEX_BRACKET_PENDING_NONE)) ||
-    (regex_bracket_pending_element !=
-      REGEX_BRACKET_PENDING_NONE &&
-      regex_bracket_range_pending)
-  ) {
-    return;
-  }
-
-  if (mode == MODE_NONE) {
-    return;
-  }
-
-  if (mode == MODE_TEXT) {
-    if (
-      delimiter ==
-      0 &&
-      regex_state ==
-      REGEX_OUTSIDE_BRACKET &&
-      !regex_at_branch_start &&
-      regex_duplication_state ==
-      REGEX_DUPLICATION_NONE &&
-      regex_group_depth ==
-      0 &&
-      !regex_after_anchor &&
-      regex_interval_state ==
-      REGEX_INTERVAL_NONE &&
-      !regex_after_alternation &&
-      regex_bracket_term_state ==
-      REGEX_BRACKET_TERM_NONE &&
-      regex_bracket_pending_element ==
-      REGEX_BRACKET_PENDING_NONE &&
-      regex_bracket_first_element ==
+        candidate.regex_bracket_last_element == REGEX_BRACKET_PENDING_NONE)) ||
+    (candidate.regex_bracket_pending_element !=
       REGEX_BRACKET_PENDING_NONE &&
-      regex_bracket_last_element ==
-      REGEX_BRACKET_PENDING_NONE &&
-      regex_bracket_element_count ==
-      0 &&
-      !regex_bracket_range_pending
-    ) {
-      state->mode = mode;
-      state->text_state = text_state;
-    }
-    return;
-  }
-
-  if (
-    delimiter ==
-    0 ||
-    delimiter >
-    UINT32_C(0x10ffff) ||
-    (delimiter >= UINT32_C(0xd800) && delimiter <= UINT32_C(0xdfff)) ||
-    delimiter ==
-    (uint32_t)'\\' ||
-    delimiter == (uint32_t)'\n'
-  ) {
-    return;
-  }
-
-  if (
-    mode !=
-    MODE_REGEX_ADDRESS &&
-    mode !=
-    MODE_SUBSTITUTE_PATTERN &&
-    (regex_state !=
-      REGEX_OUTSIDE_BRACKET ||
-      regex_at_branch_start ||
-      regex_duplication_state !=
-      REGEX_DUPLICATION_NONE ||
-      regex_after_alternation ||
-      regex_after_anchor ||
-      text_state !=
-      TEXT_EMPTY ||
-      regex_interval_state !=
-      REGEX_INTERVAL_NONE ||
-      regex_bracket_term_state !=
-      REGEX_BRACKET_TERM_NONE ||
-      regex_bracket_pending_element !=
-      REGEX_BRACKET_PENDING_NONE ||
-      regex_bracket_first_element !=
-      REGEX_BRACKET_PENDING_NONE ||
-      regex_bracket_last_element !=
-      REGEX_BRACKET_PENDING_NONE ||
-      regex_bracket_element_count !=
-      0 ||
-      regex_bracket_range_pending)
-  ) {
-    return;
-  }
-
-  if (
-    regex_state !=
-    REGEX_OUTSIDE_BRACKET &&
-    (regex_at_branch_start ||
-      regex_duplication_state !=
-      REGEX_DUPLICATION_NONE ||
-      regex_after_alternation ||
-      regex_after_anchor)
-  ) {
-    return;
-  }
-
-  if (
-    regex_state ==
-    REGEX_OUTSIDE_BRACKET &&
-    (regex_bracket_term_state !=
-      REGEX_BRACKET_TERM_NONE ||
-      regex_bracket_pending_element !=
-      REGEX_BRACKET_PENDING_NONE ||
-      regex_bracket_first_element !=
-      REGEX_BRACKET_PENDING_NONE ||
-      regex_bracket_last_element !=
-      REGEX_BRACKET_PENDING_NONE ||
-      regex_bracket_element_count !=
-      0 ||
-      regex_bracket_range_pending)
-  ) {
-    return;
-  }
-
-  if (
-    (regex_state ==
-      REGEX_BRACKET_FIRST ||
-      regex_state == REGEX_BRACKET_AFTER_CARET) &&
-    (regex_bracket_term_state !=
-      REGEX_BRACKET_TERM_NONE ||
-      regex_bracket_pending_element !=
-      REGEX_BRACKET_PENDING_NONE ||
-      regex_bracket_element_count !=
-      0 ||
-      regex_bracket_range_pending)
-  ) {
-    return;
-  }
-
-  if (
-    (mode == MODE_REGEX_ADDRESS || mode == MODE_SUBSTITUTE_PATTERN) &&
-    text_state != TEXT_EMPTY
+      candidate.regex_bracket_range_pending)
   ) {
     return;
   }
 
-  if (
-    mode !=
-    MODE_REGEX_ADDRESS &&
-    mode !=
-    MODE_SUBSTITUTE_PATTERN &&
-    regex_group_depth != 0
-  ) {
-    return;
+  if (deserialized_state_is_valid(&candidate)) {
+    *state = candidate;
   }
-
-  state->mode = mode;
-  state->delimiter = (int32_t)delimiter;
-  state->regex_state = regex_state;
-  state->regex_at_branch_start = regex_at_branch_start;
-  state->regex_duplication_state = regex_duplication_state;
-  state->regex_after_alternation = regex_after_alternation;
-  state->regex_after_anchor = regex_after_anchor;
-  state->regex_interval_state = regex_interval_state;
-  state->regex_bracket_term_state = regex_bracket_term_state;
-  state->regex_bracket_pending_element = regex_bracket_pending_element;
-  state->regex_bracket_first_element = regex_bracket_first_element;
-  state->regex_bracket_last_element = regex_bracket_last_element;
-  state->regex_bracket_element_count = regex_bracket_element_count;
-  state->regex_bracket_range_pending = regex_bracket_range_pending;
-  state->regex_group_depth = regex_group_depth;
 }
 
 static void advance(TSLexer *lexer) {
@@ -789,6 +745,42 @@ static bool scan_substitution_wfile_argument(TSLexer *lexer) {
   return true;
 }
 
+enum FlagAfterWriteScan {
+  FLAG_AFTER_WRITE_NONE,
+  FLAG_AFTER_WRITE_MATCH,
+  FLAG_AFTER_WRITE_FALLBACK,
+};
+
+static bool is_substitution_flag_character(int32_t character) {
+  if (is_digit(character)) {
+    return true;
+  }
+
+  switch (character) {
+  case 'g':
+  case 'i':
+  case 'p':
+  case 'w':
+    return true;
+  default:
+    return false;
+  }
+}
+
+static enum FlagAfterWriteScan scan_flag_after_write_marker(TSLexer *lexer) {
+  if (!is_substitution_flag_character(lexer->lookahead)) {
+    return FLAG_AFTER_WRITE_NONE;
+  }
+
+  lexer->mark_end(lexer);
+  do {
+    advance(lexer);
+  } while (is_substitution_flag_character(lexer->lookahead));
+
+  return is_blank(lexer->lookahead) ? FLAG_AFTER_WRITE_MATCH
+                                    : FLAG_AFTER_WRITE_FALLBACK;
+}
+
 static bool scan_right_brace(TSLexer *lexer, ScannerState *state) {
   if (state->mode != MODE_NONE) {
     return false;
@@ -883,8 +875,8 @@ static enum PostBlankRecoveryScan scan_post_blank_recovery(
 ) {
   if (
     !valid_symbols[OMITTED_ADDRESS_MARKER] &&
-    !valid_symbols[ADDITIONAL_ADDRESS_MARKER] &&
-    !valid_symbols[BLANKS_AROUND_ADDRESS_SEPARATOR_MARKER] &&
+    !valid_symbols[INCOMPLETE_OMITTED_ADDRESS_MARKER] &&
+    !valid_symbols[BLANKS_AROUND_ADDRESS_SEPARATOR] &&
     !valid_symbols[MISSING_ADDRESS_SEPARATOR_MARKER] &&
     !valid_symbols[MISSING_SEPARATOR_BEFORE_UNMATCHED_BRACE_MARKER] &&
     !valid_symbols[MISSING_COMMAND_SEPARATOR_MARKER] &&
@@ -896,22 +888,23 @@ static enum PostBlankRecoveryScan scan_post_blank_recovery(
   lexer->mark_end(lexer);
   advance_past_blanks(lexer);
 
-  if (valid_symbols[OMITTED_ADDRESS_MARKER] && !can_start_address(lexer)) {
-    *symbol = OMITTED_ADDRESS_MARKER;
-    return POST_BLANK_RECOVERY_TOKEN;
-  }
+  const bool after_separator = valid_symbols[OMITTED_ADDRESS_MARKER] ||
+    valid_symbols[INCOMPLETE_OMITTED_ADDRESS_MARKER];
+
   if (
-    valid_symbols[ADDITIONAL_ADDRESS_MARKER] &&
-    (lexer->lookahead == ',' || can_start_address(lexer))
+    valid_symbols[BLANKS_AROUND_ADDRESS_SEPARATOR] &&
+    (after_separator ? can_start_address(lexer) : lexer->lookahead == ',')
   ) {
-    *symbol = ADDITIONAL_ADDRESS_MARKER;
+    lexer->mark_end(lexer);
+    *symbol = BLANKS_AROUND_ADDRESS_SEPARATOR;
     return POST_BLANK_RECOVERY_TOKEN;
   }
-  if (
-    valid_symbols[BLANKS_AROUND_ADDRESS_SEPARATOR_MARKER] &&
-    lexer->lookahead == ','
-  ) {
-    *symbol = BLANKS_AROUND_ADDRESS_SEPARATOR_MARKER;
+  if (after_separator && !can_start_address(lexer)) {
+    *symbol = lexer->eof(lexer) ? INCOMPLETE_OMITTED_ADDRESS_MARKER
+                                : OMITTED_ADDRESS_MARKER;
+    if (!valid_symbols[*symbol]) {
+      return POST_BLANK_RECOVERY_FAILED;
+    }
     return POST_BLANK_RECOVERY_TOKEN;
   }
   if (
@@ -1021,10 +1014,6 @@ static bool scan_text_token(
   return true;
 }
 
-static bool is_regex_mode(enum ScannerMode mode) {
-  return mode == MODE_REGEX_ADDRESS || mode == MODE_SUBSTITUTE_PATTERN;
-}
-
 static bool regex_is_inside_bracket(const ScannerState *state) {
   return state->regex_state != REGEX_OUTSIDE_BRACKET;
 }
@@ -1125,44 +1114,36 @@ update_bracket_state_after_literal(ScannerState *state, int32_t character) {
 static bool
 regex_literal_boundary(const ScannerState *state, int32_t character) {
   if (state->regex_state == REGEX_OUTSIDE_BRACKET) {
-    return character ==
-      state->delimiter ||
-      character ==
-      '\\' ||
-      character ==
-      '(' ||
-      character ==
-      ')' ||
-      character ==
-      '[' ||
-      character ==
-      '*' ||
-      character ==
-      '+' ||
-      character ==
-      '?' ||
-      character ==
-      '|' ||
-      character ==
-      '{' ||
-      character ==
-      '.' ||
-      character ==
-      '^' ||
-      character == '$';
+    switch (character) {
+    case '\\':
+    case '(':
+    case ')':
+    case '[':
+    case '*':
+    case '+':
+    case '?':
+    case '|':
+    case '{':
+    case '.':
+    case '^':
+    case '$':
+      return true;
+    default:
+      return character == state->delimiter;
+    }
   }
 
-  if (
-    character ==
-    '[' ||
-    (state->regex_state == REGEX_BRACKET_FIRST && character == '^') ||
-    (state->regex_state == REGEX_BRACKET_BODY && character == '-') ||
-    (state->regex_state == REGEX_BRACKET_BODY && character == ']')
-  ) {
+  switch (character) {
+  case '[':
     return true;
+  case '^':
+    return state->regex_state == REGEX_BRACKET_FIRST;
+  case '-':
+  case ']':
+    return state->regex_state == REGEX_BRACKET_BODY;
+  default:
+    return false;
   }
-
-  return false;
 }
 
 static enum LiteralScanResult
@@ -1230,15 +1211,19 @@ static bool scan_regex_bracket_opener(
   lexer->mark_end(lexer);
 
   TSSymbol candidate;
+  enum RegexBracketTermState term_state;
   switch (lexer->lookahead) {
   case ':':
     candidate = REGEX_OPEN_COLON;
+    term_state = REGEX_BRACKET_TERM_COLON;
     break;
   case '.':
     candidate = REGEX_OPEN_DOT;
+    term_state = REGEX_BRACKET_TERM_DOT;
     break;
   case '=':
     candidate = REGEX_OPEN_EQUAL;
+    term_state = REGEX_BRACKET_TERM_EQUAL;
     break;
   default:
     begin_single_bracket_element(state, '[');
@@ -1252,10 +1237,7 @@ static bool scan_regex_bracket_opener(
   consume(lexer);
   begin_compound_bracket_element(state);
   finish_first_bracket_element(state);
-  state->regex_bracket_term_state = candidate == REGEX_OPEN_COLON
-    ? REGEX_BRACKET_TERM_COLON
-    : candidate == REGEX_OPEN_DOT ? REGEX_BRACKET_TERM_DOT
-                                  : REGEX_BRACKET_TERM_EQUAL;
+  state->regex_bracket_term_state = term_state;
   *symbol = candidate;
   return true;
 }
@@ -1277,6 +1259,18 @@ static bool bracket_meta_character(int32_t character) {
   return character == '^' || character == '-' || character == ']';
 }
 
+static int32_t
+bracket_term_close_marker(enum RegexBracketTermState term_state) {
+  switch (term_state) {
+  case REGEX_BRACKET_TERM_COLON:
+    return ':';
+  case REGEX_BRACKET_TERM_DOT:
+    return '.';
+  default:
+    return '=';
+  }
+}
+
 static TSSymbol regex_bracket_term_content_symbol(
   const ScannerState *state,
   uint8_t character_count,
@@ -1509,6 +1503,18 @@ static bool scan_regex_interval(
   return emit_symbol(valid_symbols, REGEX_INTERVAL_OPEN, symbol);
 }
 
+static bool emit_interval_issue(
+  TSLexer *lexer,
+  ScannerState *state,
+  TSSymbol candidate,
+  TSSymbol *symbol
+) {
+  lexer->mark_end(lexer);
+  state->regex_interval_state = REGEX_INTERVAL_NONE;
+  *symbol = candidate;
+  return true;
+}
+
 static bool
 scan_invalid_interval_remainder(TSLexer *lexer, ScannerState *state);
 
@@ -1604,9 +1610,7 @@ static enum DuplicationContextScan scan_regex_duplication_context_marker(
 #endif
 
   if (lexer->eof(lexer) && valid_symbols[REGEX_INCOMPLETE_INTERVAL]) {
-    lexer->mark_end(lexer);
-    state->regex_interval_state = REGEX_INTERVAL_NONE;
-    *symbol = REGEX_INCOMPLETE_INTERVAL;
+    emit_interval_issue(lexer, state, REGEX_INCOMPLETE_INTERVAL, symbol);
     return DUPLICATION_CONTEXT_TOKEN;
   }
 
@@ -1615,12 +1619,25 @@ static enum DuplicationContextScan scan_regex_duplication_context_marker(
     return DUPLICATION_CONTEXT_TOKEN;
   }
   scan_invalid_interval_remainder(lexer, state);
-  lexer->mark_end(lexer);
-  state->regex_interval_state = REGEX_INTERVAL_NONE;
-  *symbol = REGEX_INVALID_INTERVAL;
+  emit_interval_issue(lexer, state, REGEX_INVALID_INTERVAL, symbol);
   return DUPLICATION_CONTEXT_TOKEN;
 }
 
+static bool raw_regex_operator_character(int32_t character) {
+  switch (character) {
+  case '(':
+  case ')':
+  case '*':
+  case '+':
+  case '?':
+  case '|':
+  case '{':
+    return true;
+  default:
+    return false;
+  }
+}
+
 static bool scan_raw_regex_operator(
   TSLexer *lexer,
   ScannerState *state,
@@ -1689,39 +1706,42 @@ static bool scan_raw_regex_operator(
 }
 
 static bool regex_delimiter_is_special(int32_t character) {
+  switch (character) {
 #if SED_REGEX_EXTENDED
-  return character ==
-    '.' ||
-    character ==
-    '[' ||
-    character ==
-    '(' ||
-    character ==
-    ')' ||
-    character ==
-    '*' ||
-    character ==
-    '+' ||
-    character ==
-    '?' ||
-    character ==
-    '{' ||
-    character ==
-    '|' ||
-    character ==
-    '^' ||
-    character == '$';
-#else
-  return character ==
-    '.' ||
-    character ==
-    '[' ||
-    character ==
-    '*' ||
-    character ==
-    '^' ||
-    character == '$';
+  case '(':
+  case ')':
+  case '+':
+  case '?':
+  case '{':
+  case '|':
+#endif
+  case '.':
+  case '[':
+  case '*':
+  case '^':
+  case '$':
+    return true;
+  default:
+    return false;
+  }
+}
+
+static bool escape_quotes_regex_syntax(int32_t character) {
+  switch (character) {
+#if SED_REGEX_EXTENDED
+  case '}':
 #endif
+  case '.':
+  case '*':
+  case '^':
+  case '$':
+  case '[':
+  case '\\':
+  case ']':
+    return true;
+  default:
+    return false;
+  }
 }
 
 static bool scan_regex_escaped_delimiter(
@@ -1837,29 +1857,11 @@ static bool scan_regex_escape_after_backslash(
     return emit_symbol(valid_symbols, REGEX_NEWLINE_ESCAPE, symbol);
   }
 
-  const int32_t character = lexer->lookahead;
-  const bool quotes_regex_syntax = character ==
-    '.' ||
-    character ==
-    '*' ||
-    character ==
-    '^' ||
-    character ==
-    '$' ||
-    character ==
-    '[' ||
-    character ==
-    '\\' ||
-    character ==
-    ']' ||
-    (SED_REGEX_EXTENDED && character == '}');
-  if (!quotes_regex_syntax) {
-    consume(lexer);
-    return emit_symbol(valid_symbols, REGEX_NONPORTABLE_ESCAPE, symbol);
-  }
-
+  const TSSymbol candidate = escape_quotes_regex_syntax(lexer->lookahead)
+    ? REGEX_QUOTED_ESCAPE
+    : REGEX_NONPORTABLE_ESCAPE;
   consume(lexer);
-  return emit_symbol(valid_symbols, REGEX_QUOTED_ESCAPE, symbol);
+  return emit_symbol(valid_symbols, candidate, symbol);
 }
 
 #if !SED_REGEX_EXTENDED
@@ -2053,10 +2055,12 @@ static bool scan_regex_token(
       return true;
     }
     if (lexer->eof(lexer) && valid_symbols[REGEX_INCOMPLETE_INTERVAL]) {
-      lexer->mark_end(lexer);
-      state->regex_interval_state = REGEX_INTERVAL_NONE;
-      *symbol = REGEX_INCOMPLETE_INTERVAL;
-      return true;
+      return emit_interval_issue(
+        lexer,
+        state,
+        REGEX_INCOMPLETE_INTERVAL,
+        symbol
+      );
     }
     if (!valid_symbols[REGEX_INVALID_INTERVAL]) {
       return false;
@@ -2067,10 +2071,7 @@ static bool scan_regex_token(
       advance_past_escaped_character(lexer);
       scan_invalid_interval_remainder(lexer, state);
     }
-    lexer->mark_end(lexer);
-    state->regex_interval_state = REGEX_INTERVAL_NONE;
-    *symbol = REGEX_INVALID_INTERVAL;
-    return true;
+    return emit_interval_issue(lexer, state, REGEX_INVALID_INTERVAL, symbol);
   }
 #endif
 
@@ -2092,14 +2093,10 @@ static bool scan_regex_token(
       valid_symbols[REGEX_COLL_ELEM_MULTI] ||
       valid_symbols[REGEX_META_CHAR]
     ) {
-      const int32_t close_marker =
-        state->regex_bracket_term_state == REGEX_BRACKET_TERM_COLON ? ':'
-        : state->regex_bracket_term_state == REGEX_BRACKET_TERM_DOT ? '.'
-                                                                    : '=';
       return scan_regex_bracket_term_content(
         lexer,
         state,
-        close_marker,
+        bracket_term_close_marker(state->regex_bracket_term_state),
         valid_symbols,
         symbol
       );
@@ -2234,21 +2231,7 @@ static bool scan_regex_token(
       );
     }
 
-    if (
-      lexer->lookahead ==
-      '(' ||
-      lexer->lookahead ==
-      ')' ||
-      lexer->lookahead ==
-      '*' ||
-      lexer->lookahead ==
-      '+' ||
-      lexer->lookahead ==
-      '?' ||
-      lexer->lookahead ==
-      '|' ||
-      lexer->lookahead == '{'
-    ) {
+    if (raw_regex_operator_character(lexer->lookahead)) {
       return scan_raw_regex_operator(lexer, state, valid_symbols, symbol);
     }
 
@@ -2514,226 +2497,196 @@ static bool scan_translate_token(
   return false;
 }
 
-static bool sed_scanner_scan_impl(
-  ScannerState *state,
+static bool scan_regex_recovery_marker(
   TSLexer *lexer,
+  ScannerState *state,
   const bool *valid_symbols,
   TSSymbol *symbol
 ) {
-  if (valid_symbols[ERROR_SENTINEL]) {
-    return false;
+#if !SED_REGEX_EXTENDED
+  if (
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      BRE_VERTICAL_LINE_ESCAPE_MARKER,
+      symbol
+    ) ||
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      BRE_QUESTION_MARK_ESCAPE_MARKER,
+      symbol
+    ) ||
+    emit_missing_marker(lexer, valid_symbols, BRE_PLUS_ESCAPE_MARKER, symbol)
+  ) {
+    return true;
   }
+#endif
 
-  if (state->mode == MODE_TEXT) {
-    return scan_text_token(lexer, state, valid_symbols, symbol);
-  }
+  const bool at_line_boundary = lexer->eof(lexer) || lexer->lookahead == '\n';
+  const bool at_regex_end =
+    at_line_boundary || lexer->lookahead == state->delimiter;
 
-#if !SED_REGEX_EXTENDED
   if (
-    is_regex_mode(state->mode) &&
-    (emit_missing_marker(
-       lexer,
-       valid_symbols,
-       BRE_VERTICAL_LINE_ESCAPE_MARKER,
-       symbol
-     ) ||
-      emit_missing_marker(
-        lexer,
-        valid_symbols,
-        BRE_QUESTION_MARK_ESCAPE_MARKER,
-        symbol
-      ) ||
-      emit_missing_marker(
-        lexer,
-        valid_symbols,
-        BRE_PLUS_ESCAPE_MARKER,
-        symbol
-      ) ||
-      emit_missing_marker(
-        lexer,
-        valid_symbols,
-        UNMATCHED_INTERVAL_CLOSE_MARKER,
-        symbol
-      ))
-  ) {
-    return true;
-  }
-#endif
-
-  if (
-    is_regex_mode(state->mode) &&
     state->regex_state ==
     REGEX_OUTSIDE_BRACKET &&
     state->regex_group_depth >
     0 &&
     !state->regex_after_alternation &&
-    (lexer->eof(lexer) ||
-      lexer->lookahead ==
-      '\n' ||
-      lexer->lookahead == state->delimiter)
+    at_regex_end &&
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      lexer->eof(lexer) ? MISSING_SUBEXPRESSION_MARKER
+                        : MISSING_SUBEXPRESSION_PLACEHOLDER_MARKER,
+      symbol
+    )
   ) {
-    const TSSymbol candidate = lexer->eof(lexer)
-      ? MISSING_SUBEXPRESSION_MARKER
-      : MISSING_SUBEXPRESSION_PLACEHOLDER_MARKER;
-    if (valid_symbols[candidate]) {
-      lexer->mark_end(lexer);
-      *symbol = candidate;
-      return true;
-    }
+    return true;
   }
 
   if (
-    is_regex_mode(state->mode) &&
     (state->regex_state ==
       REGEX_BRACKET_FIRST ||
       state->regex_state == REGEX_BRACKET_AFTER_CARET) &&
-    (lexer->eof(lexer) || lexer->lookahead == '\n')
+    at_line_boundary &&
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      lexer->eof(lexer) ? INCOMPLETE_BRACKET_LIST_MARKER
+                        : MISSING_BRACKET_LIST_MARKER,
+      symbol
+    )
   ) {
-    const TSSymbol candidate = lexer->eof(lexer)
-      ? INCOMPLETE_BRACKET_LIST_MARKER
-      : MISSING_BRACKET_LIST_MARKER;
-    if (valid_symbols[candidate]) {
-      lexer->mark_end(lexer);
-      *symbol = candidate;
-      return true;
-    }
+    return true;
   }
 
   if (
-    is_regex_mode(state->mode) &&
     state->regex_bracket_term_state !=
     REGEX_BRACKET_TERM_NONE &&
-    (lexer->eof(lexer) || lexer->lookahead == '\n')
+    at_line_boundary &&
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      lexer->eof(lexer) ? REGEX_INCOMPLETE_BRACKET_TERM
+                        : REGEX_MALFORMED_BRACKET_TERM,
+      symbol
+    )
   ) {
-    const TSSymbol candidate = lexer->eof(lexer) ? REGEX_INCOMPLETE_BRACKET_TERM
-                                                 : REGEX_MALFORMED_BRACKET_TERM;
-    if (valid_symbols[candidate]) {
-      lexer->mark_end(lexer);
-      state->regex_bracket_term_state = REGEX_BRACKET_TERM_NONE;
-      finish_compound_bracket_element(state);
-      *symbol = candidate;
-      return true;
-    }
+    state->regex_bracket_term_state = REGEX_BRACKET_TERM_NONE;
+    finish_compound_bracket_element(state);
+    return true;
   }
 
   if (
-    is_regex_mode(state->mode) &&
     regex_is_inside_bracket(state) &&
-    (lexer->eof(lexer) || lexer->lookahead == '\n')
+    at_line_boundary &&
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      lexer->eof(lexer) ? INCOMPLETE_BRACKET_EXPRESSION_MARKER
+                        : UNCLOSED_BRACKET_EXPRESSION_MARKER,
+      symbol
+    )
   ) {
-    const TSSymbol candidate = lexer->eof(lexer)
-      ? INCOMPLETE_BRACKET_EXPRESSION_MARKER
-      : UNCLOSED_BRACKET_EXPRESSION_MARKER;
-    if (valid_symbols[candidate]) {
-      lexer->mark_end(lexer);
-      state->regex_state = REGEX_OUTSIDE_BRACKET;
-      reset_bracket_tracking(state);
-      *symbol = candidate;
-      return true;
-    }
+    state->regex_state = REGEX_OUTSIDE_BRACKET;
+    reset_bracket_tracking(state);
+    return true;
   }
 
 #if SED_REGEX_EXTENDED
   if (
-    is_regex_mode(state->mode) &&
     state->regex_state ==
     REGEX_OUTSIDE_BRACKET &&
     state->regex_after_alternation &&
-    (lexer->lookahead ==
-      state->delimiter ||
-      lexer->lookahead ==
-      '\n' ||
-      lexer->eof(lexer))
+    at_regex_end &&
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      lexer->eof(lexer) ? INCOMPLETE_ALTERNATIVE_MARKER
+                        : EMPTY_ALTERNATIVE_MARKER,
+      symbol
+    )
   ) {
-    const TSSymbol candidate = lexer->eof(lexer) ? INCOMPLETE_ALTERNATIVE_MARKER
-                                                 : EMPTY_ALTERNATIVE_MARKER;
-    if (valid_symbols[candidate]) {
-      lexer->mark_end(lexer);
-      state->regex_after_alternation = false;
-      *symbol = candidate;
-      return true;
-    }
+    state->regex_after_alternation = false;
+    return true;
   }
 #endif
 
   if (
-    is_regex_mode(state->mode) &&
     state->regex_interval_state !=
     REGEX_INTERVAL_NONE &&
-    (lexer->lookahead ==
-      state->delimiter ||
-      lexer->lookahead ==
-      '\n' ||
+    (at_regex_end
 #if !SED_REGEX_EXTENDED
-      (state->delimiter == '}' && lexer->lookahead == '\\') ||
+     || (state->delimiter == '}' && lexer->lookahead == '\\')
 #endif
-      lexer->eof(lexer))
+    ) &&
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      lexer->eof(lexer) ? REGEX_INCOMPLETE_INTERVAL : REGEX_INVALID_INTERVAL,
+      symbol
+    )
   ) {
-    const TSSymbol candidate =
-      lexer->eof(lexer) ? REGEX_INCOMPLETE_INTERVAL : REGEX_INVALID_INTERVAL;
-    if (valid_symbols[candidate]) {
-      lexer->mark_end(lexer);
-      state->regex_interval_state = REGEX_INTERVAL_NONE;
-      *symbol = candidate;
-      return true;
-    }
+    state->regex_interval_state = REGEX_INTERVAL_NONE;
+    return true;
   }
 
   if (
-    is_regex_mode(state->mode) &&
     state->regex_state ==
     REGEX_OUTSIDE_BRACKET &&
-    (lexer->lookahead == '\n' || lexer->eof(lexer))
+    state->regex_group_depth >
+    0 &&
+    at_line_boundary &&
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      lexer->eof(lexer) ? REGEX_INCOMPLETE_GROUP : REGEX_UNCLOSED_GROUP,
+      symbol
+    )
   ) {
-    if (state->regex_group_depth > 0) {
-      const TSSymbol candidate =
-        lexer->eof(lexer) ? REGEX_INCOMPLETE_GROUP : REGEX_UNCLOSED_GROUP;
-      if (valid_symbols[candidate]) {
-        lexer->mark_end(lexer);
-        state->regex_group_depth--;
-        *symbol = candidate;
-        return true;
-      }
-    }
+    state->regex_group_depth--;
+    return true;
   }
 
-  const bool delimiter_is_active = state->mode !=
-    MODE_NONE &&
-    state->mode !=
-    MODE_TEXT &&
-    lexer->lookahead ==
-    state->delimiter &&
-    (!is_regex_mode(state->mode) ||
-      state->regex_state == REGEX_OUTSIDE_BRACKET);
-  if (delimiter_is_active) {
-    if (state->regex_group_depth > 0 && valid_symbols[REGEX_UNCLOSED_GROUP]) {
-      state->regex_group_depth--;
-      *symbol = REGEX_UNCLOSED_GROUP;
-      return true;
-    }
-    return scan_active_mode_delimiter(lexer, state, valid_symbols, symbol);
-  }
+  return false;
+}
 
-  switch (state->mode) {
-  case MODE_REGEX_ADDRESS:
-  case MODE_SUBSTITUTE_PATTERN:
-    return scan_regex_token(lexer, state, valid_symbols, symbol);
-  case MODE_SUBSTITUTE_REPLACEMENT:
-    return scan_replacement_token(lexer, state, valid_symbols, symbol);
-  case MODE_TRANSLATE_SOURCE:
-  case MODE_TRANSLATE_DESTINATION:
-    return scan_translate_token(lexer, state, valid_symbols, symbol);
-  case MODE_TEXT:
-    return false;
-  case MODE_NONE:
-    break;
-  }
+static TSSymbol missing_text_introducer_symbol(const TSLexer *lexer) {
+  return lexer->eof(lexer) ? MISSING_TEXT_INTRODUCER_MARKER
+                           : NONCONFORMING_MISSING_TEXT_INTRODUCER_MARKER;
+}
 
+static bool scan_command_token(
+  TSLexer *lexer,
+  ScannerState *state,
+  const bool *valid_symbols,
+  TSSymbol *symbol
+) {
   if (valid_symbols[LINE_WORD] && scan_to_physical_line_end(lexer, false)) {
     *symbol = LINE_WORD;
     return true;
   }
 
+  if (valid_symbols[FLAG_AFTER_WRITE_MARKER]) {
+    const enum FlagAfterWriteScan scan = scan_flag_after_write_marker(lexer);
+    if (scan == FLAG_AFTER_WRITE_MATCH) {
+      *symbol = FLAG_AFTER_WRITE_MARKER;
+      return true;
+    }
+    if (
+      scan ==
+      FLAG_AFTER_WRITE_FALLBACK &&
+      valid_symbols[OMITTED_FILE_SEPARATOR_MARKER]
+    ) {
+      *symbol = OMITTED_FILE_SEPARATOR_MARKER;
+      return true;
+    }
+    if (scan == FLAG_AFTER_WRITE_FALLBACK) {
+      return false;
+    }
+  }
+
   if (
     valid_symbols[SUBSTITUTION_WFILE_ARGUMENT] &&
     scan_substitution_wfile_argument(lexer)
@@ -2750,15 +2703,7 @@ static bool sed_scanner_scan_impl(
   if (
     valid_symbols[INVALID_SUBSTITUTION_FLAG] &&
     !is_blank(lexer->lookahead) &&
-    !is_digit(lexer->lookahead) &&
-    lexer->lookahead !=
-    'g' &&
-    lexer->lookahead !=
-    'i' &&
-    lexer->lookahead !=
-    'p' &&
-    lexer->lookahead !=
-    'w' &&
+    !is_substitution_flag_character(lexer->lookahead) &&
     lexer->lookahead !=
     ';' &&
     lexer->lookahead !=
@@ -2777,10 +2722,11 @@ static bool sed_scanner_scan_impl(
       *symbol = TEXT_COMMAND_START;
       return true;
     }
-    const TSSymbol introducer_marker = lexer->eof(lexer)
-      ? MISSING_TEXT_INTRODUCER_MARKER
-      : NONCONFORMING_MISSING_TEXT_INTRODUCER_MARKER;
-    return emit_symbol(valid_symbols, introducer_marker, symbol);
+    return emit_symbol(
+      valid_symbols,
+      missing_text_introducer_symbol(lexer),
+      symbol
+    );
   }
 
   if (valid_symbols[DEFAULT_OUTPUT_SUPPRESSION] && lexer->lookahead == 'n') {
@@ -2794,11 +2740,23 @@ static bool sed_scanner_scan_impl(
     return true;
   }
 
+  // Preserve an address interpretation before reserved-function recovery.
   if (
-    valid_symbols[RESERVED_UNKNOWN_FUNCTION] &&
+    valid_symbols[MISSING_ADDRESS_SEPARATOR_MARKER] && can_start_address(lexer)
+  ) {
+    return emit_missing_marker(
+      lexer,
+      valid_symbols,
+      MISSING_ADDRESS_SEPARATOR_MARKER,
+      symbol
+    );
+  }
+
+  if (
+    valid_symbols[RESERVED_UNKNOWN_FUNCTION_TOKEN] &&
     scan_reserved_unknown_function(lexer)
   ) {
-    *symbol = RESERVED_UNKNOWN_FUNCTION;
+    *symbol = RESERVED_UNKNOWN_FUNCTION_TOKEN;
     return true;
   }
 
@@ -2839,6 +2797,9 @@ static bool sed_scanner_scan_impl(
   }
 
   if (
+    valid_symbols[OMITTED_FIRST_ADDRESS_MARKER] &&
+    lexer->lookahead ==
+    ',' &&
     emit_missing_marker(
       lexer,
       valid_symbols,
@@ -2850,54 +2811,20 @@ static bool sed_scanner_scan_impl(
   }
 
   if (
-    valid_symbols[OMITTED_ADDRESS_MARKER] &&
+    (valid_symbols[OMITTED_ADDRESS_MARKER] ||
+      valid_symbols[INCOMPLETE_OMITTED_ADDRESS_MARKER]) &&
     !can_start_address(lexer) &&
     !is_blank(lexer->lookahead)
   ) {
     return emit_missing_marker(
       lexer,
       valid_symbols,
-      OMITTED_ADDRESS_MARKER,
+      lexer->eof(lexer) ? INCOMPLETE_OMITTED_ADDRESS_MARKER
+                        : OMITTED_ADDRESS_MARKER,
       symbol
     );
   }
 
-  if (
-    valid_symbols[ADDITIONAL_ADDRESS_MARKER] &&
-    (lexer->lookahead == ',' || can_start_address(lexer))
-  ) {
-    return emit_missing_marker(
-      lexer,
-      valid_symbols,
-      ADDITIONAL_ADDRESS_MARKER,
-      symbol
-    );
-  }
-
-  if (
-    valid_symbols[MISSING_ADDRESS_SEPARATOR_MARKER] && can_start_address(lexer)
-  ) {
-    return emit_missing_marker(
-      lexer,
-      valid_symbols,
-      MISSING_ADDRESS_SEPARATOR_MARKER,
-      symbol
-    );
-  }
-
-  if (
-    valid_symbols[BLANKS_AROUND_ADDRESS_SEPARATOR_MARKER] &&
-    lexer->lookahead == ','
-  ) {
-    lexer->mark_end(lexer);
-    advance(lexer);
-    if (is_blank(lexer->lookahead)) {
-      *symbol = BLANKS_AROUND_ADDRESS_SEPARATOR_MARKER;
-      return true;
-    }
-    return false;
-  }
-
   if (is_blank(lexer->lookahead)) {
     switch (scan_post_blank_recovery(lexer, valid_symbols, symbol)) {
     case POST_BLANK_RECOVERY_TOKEN:
@@ -2909,18 +2836,6 @@ static bool sed_scanner_scan_impl(
     }
   }
 
-  if (
-    emit_missing_marker(lexer, valid_symbols, DUPLICATE_NEGATION_MARKER, symbol)
-  ) {
-    return true;
-  }
-
-  if (
-    emit_missing_marker(lexer, valid_symbols, EXCESS_ADDRESSES_MARKER, symbol)
-  ) {
-    return true;
-  }
-
   if (
     !at_command_boundary(lexer) &&
     !is_blank(lexer->lookahead) &&
@@ -2951,11 +2866,13 @@ static bool sed_scanner_scan_impl(
     }
   }
 
-  const TSSymbol missing_text_introducer = lexer->eof(lexer)
-    ? MISSING_TEXT_INTRODUCER_MARKER
-    : NONCONFORMING_MISSING_TEXT_INTRODUCER_MARKER;
   if (
-    emit_missing_marker(lexer, valid_symbols, missing_text_introducer, symbol)
+    emit_missing_marker(
+      lexer,
+      valid_symbols,
+      missing_text_introducer_symbol(lexer),
+      symbol
+    )
   ) {
     return true;
   }
@@ -3029,6 +2946,60 @@ static bool sed_scanner_scan_impl(
   return false;
 }
 
+static bool sed_scanner_scan_impl(
+  ScannerState *state,
+  TSLexer *lexer,
+  const bool *valid_symbols,
+  TSSymbol *symbol
+) {
+  if (valid_symbols[ERROR_SENTINEL]) {
+    return false;
+  }
+
+  if (state->mode == MODE_TEXT) {
+    return scan_text_token(lexer, state, valid_symbols, symbol);
+  }
+
+  if (
+    is_regex_mode(state->mode) &&
+    scan_regex_recovery_marker(lexer, state, valid_symbols, symbol)
+  ) {
+    return true;
+  }
+
+  const bool delimiter_is_active = state->mode !=
+    MODE_NONE &&
+    lexer->lookahead ==
+    state->delimiter &&
+    (!is_regex_mode(state->mode) ||
+      state->regex_state == REGEX_OUTSIDE_BRACKET);
+  if (delimiter_is_active) {
+    if (state->regex_group_depth > 0 && valid_symbols[REGEX_UNCLOSED_GROUP]) {
+      state->regex_group_depth--;
+      *symbol = REGEX_UNCLOSED_GROUP;
+      return true;
+    }
+    return scan_active_mode_delimiter(lexer, state, valid_symbols, symbol);
+  }
+
+  switch (state->mode) {
+  case MODE_REGEX_ADDRESS:
+  case MODE_SUBSTITUTE_PATTERN:
+    return scan_regex_token(lexer, state, valid_symbols, symbol);
+  case MODE_SUBSTITUTE_REPLACEMENT:
+    return scan_replacement_token(lexer, state, valid_symbols, symbol);
+  case MODE_TRANSLATE_SOURCE:
+  case MODE_TRANSLATE_DESTINATION:
+    return scan_translate_token(lexer, state, valid_symbols, symbol);
+  case MODE_TEXT:
+    return false;
+  case MODE_NONE:
+    break;
+  }
+
+  return scan_command_token(lexer, state, valid_symbols, symbol);
+}
+
 static void
 update_regex_position_after_symbol(ScannerState *state, TSSymbol symbol) {
 #if !SED_REGEX_EXTENDED
@@ -3037,9 +3008,7 @@ update_regex_position_after_symbol(ScannerState *state, TSSymbol symbol) {
     BRE_VERTICAL_LINE_ESCAPE_MARKER ||
     symbol ==
     BRE_QUESTION_MARK_ESCAPE_MARKER ||
-    symbol ==
-    BRE_PLUS_ESCAPE_MARKER ||
-    symbol == UNMATCHED_INTERVAL_CLOSE_MARKER
+    symbol == BRE_PLUS_ESCAPE_MARKER
   ) {
     return;
   }
@@ -3081,7 +3050,15 @@ update_regex_position_after_symbol(ScannerState *state, TSSymbol symbol) {
   }
 #endif
 
-  if (symbol == REGEX_BEGINNING_ANCHOR || symbol == REGEX_END_ANCHOR) {
+  bool is_anchor =
+    symbol == REGEX_BEGINNING_ANCHOR || symbol == REGEX_END_ANCHOR;
+#if !SED_REGEX_EXTENDED
+  is_anchor = is_anchor ||
+    symbol ==
+    REGEX_BRE_SUBEXPRESSION_CARET ||
+    symbol == REGEX_BRE_SUBEXPRESSION_DOLLAR;
+#endif
+  if (is_anchor) {
     set_regex_position(state, true, REGEX_DUPLICATION_NONE, false, true);
     return;
   }