robotcode-language-server 2.6.2__py3-none-any.whl → 2.8.0.dev548__py3-none-any.whl

robotcode/language_server/robotframework/parts/semantic_tokens.py

@@ -39,7 +39,6 @@ from robot.parsing.model.statements import (
     TestTemplate,
     Variable,
     VariablesImport,
-    WhileHeader,
 )
 from robot.utils.escaping import unescape
 
@@ -67,8 +66,14 @@ from robotcode.robot.diagnostics.library_doc import (
 )
 from robotcode.robot.diagnostics.model_helper import ModelHelper
 from robotcode.robot.diagnostics.namespace import Namespace
-from robotcode.robot.diagnostics.semantic_analyzer.enums import TokenKind
+from robotcode.robot.diagnostics.semantic_analyzer.enums import NodeKind, TokenKind, TokenModifier
 from robotcode.robot.diagnostics.semantic_analyzer.model import SemanticModel
+from robotcode.robot.diagnostics.semantic_analyzer.nodes import (
+    RunKeywordCallStatement,
+    SemanticStatement,
+    SemanticToken,
+)
+from robotcode.robot.diagnostics.semantic_analyzer.variable_tokenizer import VariableSite, build_variable_sub_tokens
 from robotcode.robot.utils import RF_VERSION
 from robotcode.robot.utils.ast import (
     cached_isinstance,
@@ -1034,46 +1039,51 @@ class SemanticTokenGenerator:
     creating and managing its own token mapper and keyword analyzer.
     """
 
-    # Mapping from SemanticModel TokenKind to LSP semantic token types.
-    # Used by collect_tokens_from_model() for the new model-based path.
-    _TOKEN_KIND_TO_SEM_TOKEN: Dict[TokenKind, Tuple[AnyTokenType, Optional[Set[AnyTokenModifier]]]] = {
+    # ------------------------------------------------------------------
+    # Model-based path (Tier 1): declarative rendering of the SemanticModel.
+    #
+    # After analysis every token carries its final render semantics (kind +
+    # modifiers), so rendering is: leaf descent -> static table lookup ->
+    # delta encoding, plus a small legacy-compat emission policy keyed only
+    # on TokenKind / NodeKind / TokenModifier (design D6/D7). No RF version
+    # checks, no tokenization, no statement-type semantics here.
+    # ------------------------------------------------------------------
+
+    _TOKEN_KIND_TO_SEM_TOKEN: ClassVar[Dict[TokenKind, Tuple[AnyTokenType, Optional[Set[AnyTokenModifier]]]]] = {
         TokenKind.KEYWORD: (RobotSemTokenTypes.KEYWORD, None),
+        TokenKind.KEYWORD_INNER: (RobotSemTokenTypes.KEYWORD_INNER, None),
         TokenKind.BDD_PREFIX: (RobotSemTokenTypes.BDD_PREFIX, None),
         TokenKind.NAMESPACE: (RobotSemTokenTypes.NAMESPACE, None),
+        # Variables render their name only (see `_render_variable_part`).
         TokenKind.VARIABLE: (RobotSemTokenTypes.VARIABLE, None),
         TokenKind.VARIABLE_NOT_FOUND: (RobotSemTokenTypes.VARIABLE, None),
-        TokenKind.VARIABLE_PREFIX: (RobotSemTokenTypes.VARIABLE_BEGIN, None),
-        TokenKind.VARIABLE_OPEN_BRACE: (RobotSemTokenTypes.VARIABLE_BEGIN, None),
-        TokenKind.VARIABLE_CLOSE_BRACE: (RobotSemTokenTypes.VARIABLE_END, None),
-        TokenKind.VARIABLE_BASE: (RobotSemTokenTypes.VARIABLE, None),
-        TokenKind.VARIABLE_EXTENDED: (RobotSemTokenTypes.VARIABLE, None),
-        TokenKind.VARIABLE_TYPE_SEPARATOR: (SemanticTokenTypes.OPERATOR, None),
-        TokenKind.VARIABLE_TYPE_HINT: (SemanticTokenTypes.TYPE, None),
-        TokenKind.VARIABLE_DEFAULT_SEPARATOR: (SemanticTokenTypes.OPERATOR, None),
+        TokenKind.VARIABLE_NAME: (RobotSemTokenTypes.VARIABLE, None),
         TokenKind.VARIABLE_DEFAULT_VALUE: (RobotSemTokenTypes.ARGUMENT, None),
-        TokenKind.VARIABLE_PATTERN_SEPARATOR: (SemanticTokenTypes.OPERATOR, None),
         TokenKind.VARIABLE_PATTERN: (RobotSemTokenTypes.ARGUMENT, None),
-        TokenKind.VARIABLE_ASSIGN_MARK: (RobotSemTokenTypes.VARIABLE, None),
-        TokenKind.VARIABLE_EXPRESSION_OPEN: (RobotSemTokenTypes.EXPRESSION_BEGIN, None),
-        TokenKind.VARIABLE_EXPRESSION_CLOSE: (RobotSemTokenTypes.EXPRESSION_END, None),
-        TokenKind.PYTHON_EXPRESSION: (RobotSemTokenTypes.VARIABLE_EXPRESSION, None),
-        TokenKind.PYTHON_VARIABLE_REF: (RobotSemTokenTypes.VARIABLE, None),
-        TokenKind.VARIABLE_INDEX: (RobotSemTokenTypes.VARIABLE, None),
-        TokenKind.VARIABLE_INDEX_OPEN: (RobotSemTokenTypes.VARIABLE_BEGIN, None),
-        TokenKind.VARIABLE_INDEX_CLOSE: (RobotSemTokenTypes.VARIABLE_END, None),
-        TokenKind.VARIABLE_INDEX_CONTENT: (RobotSemTokenTypes.VARIABLE, None),
         TokenKind.TEXT_FRAGMENT: (RobotSemTokenTypes.ARGUMENT, None),
         TokenKind.ARGUMENT: (RobotSemTokenTypes.ARGUMENT, None),
         TokenKind.NAMED_ARGUMENT_NAME: (RobotSemTokenTypes.NAMED_ARGUMENT, None),
         TokenKind.NAMED_ARGUMENT_VALUE: (RobotSemTokenTypes.ARGUMENT, None),
+        TokenKind.PARAMETER: (SemanticTokenTypes.PARAMETER, None),
         TokenKind.CONTROL_FLOW: (RobotSemTokenTypes.CONTROL_FLOW, None),
         TokenKind.CONDITION: (RobotSemTokenTypes.ARGUMENT, None),
+        TokenKind.FOR_SEPARATOR: (RobotSemTokenTypes.FOR_SEPARATOR, None),
+        TokenKind.VAR_MARKER: (RobotSemTokenTypes.VAR, None),
+        TokenKind.OPTION: (RobotSemTokenTypes.CONTROL_FLOW, None),
+        TokenKind.OPTION_VALUE: (RobotSemTokenTypes.CONTROL_FLOW, None),
         TokenKind.TEST_NAME: (RobotSemTokenTypes.TESTCASE_NAME, {SemanticTokenModifiers.DECLARATION}),
         TokenKind.KEYWORD_NAME: (RobotSemTokenTypes.KEYWORD_NAME, {SemanticTokenModifiers.DECLARATION}),
-        TokenKind.VARIABLE_NAME: (RobotSemTokenTypes.VARIABLE, {SemanticTokenModifiers.DECLARATION}),
         TokenKind.SETTING_NAME: (RobotSemTokenTypes.SETTING, None),
-        TokenKind.IMPORT_NAME: (RobotSemTokenTypes.SETTING_IMPORT, None),
+        TokenKind.SETTING_IMPORT: (RobotSemTokenTypes.SETTING_IMPORT, None),
+        TokenKind.IMPORT_NAME: (RobotSemTokenTypes.NAMESPACE, None),
+        TokenKind.OPERATOR: (SemanticTokenTypes.OPERATOR, None),
         TokenKind.HEADER: (RobotSemTokenTypes.HEADER, None),
+        TokenKind.HEADER_SETTINGS: (RobotSemTokenTypes.HEADER_SETTINGS, None),
+        TokenKind.HEADER_VARIABLE: (RobotSemTokenTypes.HEADER_VARIABLE, None),
+        TokenKind.HEADER_TESTCASE: (RobotSemTokenTypes.HEADER_TESTCASE, None),
+        TokenKind.HEADER_TASK: (RobotSemTokenTypes.HEADER_TASK, None),
+        TokenKind.HEADER_KEYWORD: (RobotSemTokenTypes.HEADER_KEYWORD, None),
+        TokenKind.HEADER_COMMENT: (RobotSemTokenTypes.HEADER_COMMENT, None),
         TokenKind.SEPARATOR: (RobotSemTokenTypes.SEPARATOR, None),
         TokenKind.CONTINUATION: (RobotSemTokenTypes.CONTINUATION, None),
         TokenKind.COMMENT: (SemanticTokenTypes.COMMENT, None),
@@ -1082,6 +1092,61 @@ class SemanticTokenGenerator:
         TokenKind.ERROR: (RobotSemTokenTypes.ERROR, None),
     }
 
+    _TOKEN_MODIFIER_TO_SEM: ClassVar[Dict[TokenModifier, AnyTokenModifier]] = {
+        TokenModifier.BUILTIN: RobotSemTokenModifiers.BUILTIN,
+        TokenModifier.EMBEDDED: RobotSemTokenModifiers.EMBEDDED,
+        TokenModifier.DECLARATION: SemanticTokenModifiers.DECLARATION,
+        TokenModifier.DOCUMENTATION: SemanticTokenModifiers.DOCUMENTATION,
+    }
+
+    # --- Legacy-compat emission policy (design D7) ---
+
+    # Whole options of control structures render atomically: one control-flow
+    # token per option, no descent into their sub-structure.
+    _ATOMIC_KINDS: ClassVar[FrozenSet[TokenKind]] = frozenset({TokenKind.OPTION, TokenKind.OPTION_VALUE})
+    # Variables and parameters render only their name (and type hint); the
+    # grammar shows prefix, braces, item access, `=`, patterns, defaults and
+    # inline Python.
+    _VARIABLE_KINDS: ClassVar[FrozenSet[TokenKind]] = frozenset(
+        {TokenKind.VARIABLE, TokenKind.VARIABLE_NOT_FOUND, TokenKind.VARIABLE_NAME, TokenKind.PARAMETER}
+    )
+    # Model-only kinds: carried for model consumers (inline values, debug
+    # variable extraction), never rendered — legacy highlights no bare-`$var`
+    # expression references.
+    _MODEL_ONLY_KINDS: ClassVar[FrozenSet[TokenKind]] = frozenset({TokenKind.PYTHON_VARIABLE_REF})
+    # Plain argument text is suppressed by the legacy path...
+    _ARGUMENT_TEXT_KINDS: ClassVar[FrozenSet[TokenKind]] = frozenset(
+        {
+            TokenKind.ARGUMENT,
+            TokenKind.TEXT_FRAGMENT,
+            TokenKind.TAG,
+            TokenKind.CONDITION,
+            TokenKind.NAMED_ARGUMENT_VALUE,
+            TokenKind.VARIABLE_DEFAULT_VALUE,
+            TokenKind.VARIABLE_PATTERN,
+        }
+    )
+    # ...except in template argument rows and metadata values...
+    # (TEMPLATE_KEYWORD: an unresolved template name keeps kind ARGUMENT and
+    # must still render, matching legacy.)
+    _YIELD_ARGUMENT_STMT_KINDS: ClassVar[FrozenSet[NodeKind]] = frozenset(
+        {NodeKind.TEMPLATE_DATA, NodeKind.SETTING_METADATA, NodeKind.TEMPLATE_KEYWORD}
+    )
+    # Comments only render on keyword-call / import statements (and inside
+    # invalid sections).
+    _COMMENT_STMT_KINDS: ClassVar[FrozenSet[NodeKind]] = frozenset(
+        {NodeKind.KEYWORD_CALL, NodeKind.SETUP, NodeKind.TEARDOWN, NodeKind.TEMPLATE_KEYWORD, NodeKind.IMPORT}
+    )
+    # Statements whose keyword name comes from a Token.NAME cell -- their
+    # BDD-prefix gap token renders as `argument` (the NAME cell's initial
+    # legacy mapping), not as a keyword.
+    _NAME_KEYWORD_STMT_KINDS: ClassVar[FrozenSet[NodeKind]] = frozenset(
+        {NodeKind.SETUP, NodeKind.TEARDOWN, NodeKind.TEMPLATE_KEYWORD}
+    )
+    _BDD_FOLLOW_KINDS: ClassVar[FrozenSet[TokenKind]] = frozenset(
+        {TokenKind.KEYWORD, TokenKind.KEYWORD_INNER, TokenKind.NAMESPACE, TokenKind.ARGUMENT}
+    )
+
     def __init__(self) -> None:
         """Initialize the generator with its own dependencies."""
         self.token_mapper = SemanticTokenMapper()
@@ -1098,8 +1163,9 @@ class SemanticTokenGenerator:
         """Collect semantic tokens from the pre-built SemanticModel.
 
         This is the Tier 1 model-based path, used when the semantic model
-        feature flag is enabled. It maps SemanticModel TokenKind values
-        to LSP semantic token types via _TOKEN_KIND_TO_SEM_TOKEN.
+        feature flag is enabled. The model carries final render semantics,
+        so this is a declarative mapping (see the class-level tables) plus
+        UTF-16 delta encoding.
         """
         data: List[int] = []
         last_line = 0
@@ -1109,30 +1175,21 @@ class SemanticTokenGenerator:
         for stmt in semantic_model.statements:
             check_current_task_canceled()
 
-            for token in stmt.tokens:
-                if token.length == 0:
+            for info in self._iter_model_sem_tokens(stmt, semantic_model):
+                if info.length == 0:
                     continue
 
+                token_line_0 = info.lineno - 1
                 # Range filtering
-                token_line_0 = token.line - 1
-                if range is not None:
-                    if token_line_0 < range.start.line:
-                        continue
-                    if token_line_0 > range.end.line:
-                        break
-
-                sem_info = self._TOKEN_KIND_TO_SEM_TOKEN.get(token.kind)
-                if sem_info is None:
+                if range is not None and (token_line_0 < range.start.line or token_line_0 > range.end.line):
                     continue
 
-                sem_type, sem_mods = sem_info
-
                 # Convert to UTF-16 positions
                 token_range = range_to_utf16(
                     lines,
                     Range(
-                        start=Position(line=token_line_0, character=token.col_offset),
-                        end=Position(line=token_line_0, character=token.col_offset + token.length),
+                        start=Position(line=token_line_0, character=info.col_offset),
+                        end=Position(line=token_line_0, character=info.col_offset + info.length),
                     ),
                 )
 
@@ -1154,11 +1211,244 @@ class SemanticTokenGenerator:
                 last_line = current_line
 
                 data.append(token_length)
-                data.append(token_types.index(sem_type))
-                data.append(reduce(operator.or_, [2 ** token_modifiers.index(e) for e in sem_mods]) if sem_mods else 0)
+                data.append(token_types.index(info.sem_token_type))
+                data.append(
+                    reduce(operator.or_, [2 ** token_modifiers.index(e) for e in info.sem_modifiers])
+                    if info.sem_modifiers
+                    else 0
+                )
 
         return SemanticTokens(data=data)
 
+    def _emit_model_token(self, token: SemanticToken) -> Optional[SemTokenInfo]:
+        """Map one model token through the static kind / modifier tables."""
+        sem_info = self._TOKEN_KIND_TO_SEM_TOKEN.get(token.kind)
+        if sem_info is None:
+            return None
+        sem_type, static_mods = sem_info
+        sem_mods: Optional[Set[AnyTokenModifier]] = set(static_mods) if static_mods else None
+        if token.modifiers:
+            mapped = {self._TOKEN_MODIFIER_TO_SEM[m] for m in token.modifiers if m in self._TOKEN_MODIFIER_TO_SEM}
+            if mapped:
+                sem_mods = (sem_mods or set()) | mapped
+        return SemTokenInfo(token.line, token.col_offset, token.length, sem_type, sem_mods)
+
+    def _iter_model_sem_tokens(
+        self, stmt: SemanticStatement, model: SemanticModel, is_inner: bool = False
+    ) -> Iterator[SemTokenInfo]:
+        """Yield SemTokenInfo for one statement, matching the legacy output.
+
+        `is_inner` marks a Run Keyword inner call (its BDD gap token renders
+        as keywordCallInner).
+        """
+        # Documentation statements emit only their continuation markers
+        # (legacy renders neither the setting name nor the body).
+        if stmt.kind is NodeKind.SETTING_DOCUMENTATION:
+            for token in stmt.tokens:
+                if token.kind is TokenKind.CONTINUATION:
+                    yield SemTokenInfo(token.line, token.col_offset, token.length, RobotSemTokenTypes.CONTINUATION)
+            return
+
+        section = model.enclosing_section(stmt)
+        emit_comments = stmt.kind in self._COMMENT_STMT_KINDS or (
+            section is not None and section.kind is NodeKind.INVALID_SECTION
+        )
+
+        if cached_isinstance(stmt, RunKeywordCallStatement):
+            kw_doc = stmt.keyword_doc
+            # Legacy only decomposes the hardcoded BuiltIn run keywords;
+            # register-only run keywords render flat.
+            if kw_doc is not None and kw_doc.is_any_run_keyword():
+                yield from self._iter_run_keyword_sem_tokens(stmt, model, emit_comments, is_inner)
+                return
+
+        yield_arguments = stmt.kind in self._YIELD_ARGUMENT_STMT_KINDS
+
+        prev: Optional[SemanticToken] = None
+        for token in stmt.tokens:
+            gap = self._bdd_gap_token(stmt, prev, token, is_inner)
+            if gap is not None:
+                yield gap
+            yield from self._render_model_token(token, emit_comments, yield_arguments)
+            prev = token
+
+    def _bdd_gap_token(
+        self,
+        stmt: SemanticStatement,
+        prev: Optional[SemanticToken],
+        token: SemanticToken,
+        is_inner: bool,
+    ) -> Optional[SemTokenInfo]:
+        """Legacy quirk: a BDD prefix (`Given`/`When`/...) emits a length-1
+        keyword-typed token for the separator character between the prefix
+        word and the keyword name. The model folds that space into the prefix
+        offset, so it is synthesized here from adjacent leaf kinds."""
+        if (
+            prev is None
+            or prev.kind is not TokenKind.BDD_PREFIX
+            or token.kind not in self._BDD_FOLLOW_KINDS
+            or token.col_offset <= prev.col_offset + prev.length
+        ):
+            return None
+        if is_inner:
+            sem_type: AnyTokenType = RobotSemTokenTypes.KEYWORD_INNER
+        elif stmt.kind in self._NAME_KEYWORD_STMT_KINDS:
+            sem_type = RobotSemTokenTypes.ARGUMENT
+        else:
+            sem_type = RobotSemTokenTypes.KEYWORD
+        gap_col = prev.col_offset + prev.length
+        return SemTokenInfo(prev.line, gap_col, token.col_offset - gap_col, sem_type)
+
+    def _render_model_token(
+        self,
+        token: SemanticToken,
+        emit_comments: bool,
+        yield_arguments: bool,
+    ) -> Iterator[SemTokenInfo]:
+        """Render one SemanticToken (recursively descending into sub-tokens)
+        through the static tables and the declarative emission policy."""
+        kind = token.kind
+        mods = token.modifiers
+
+        if kind in self._MODEL_ONLY_KINDS:
+            return
+
+        if kind in self._VARIABLE_KINDS and token.sub_tokens:
+            for sub in token.sub_tokens:
+                yield from self._render_variable_part(sub, token)
+            return
+
+        # Atomic kinds render whole; their sub-structure stays model-only.
+        if kind in self._ATOMIC_KINDS:
+            info = self._emit_model_token(token)
+            if info is not None:
+                yield info
+            return
+
+        # Legacy quirk: `[Documentation]` / `Metadata` setting names render
+        # as nothing.
+        if kind is TokenKind.SETTING_NAME and mods and TokenModifier.DOCUMENTATION in mods:
+            return
+
+        # Legacy quirk: an embedded keyword name that does not match its own
+        # pattern (parent carries EMBEDDED, no sub-tokens) renders as nothing.
+        if (
+            kind in (TokenKind.KEYWORD, TokenKind.KEYWORD_INNER)
+            and mods
+            and TokenModifier.EMBEDDED in mods
+            and not token.sub_tokens
+        ):
+            return
+
+        # Leaf descent: emit the leaves instead of the parent.
+        if token.sub_tokens:
+            for sub in token.sub_tokens:
+                yield from self._render_model_token(sub, emit_comments, yield_arguments)
+            return
+
+        # Whitespace / line-structure tokens are never rendered.
+        if kind is TokenKind.SEPARATOR or kind is TokenKind.EOL:
+            return
+
+        if kind is TokenKind.COMMENT:
+            if emit_comments:
+                yield SemTokenInfo(token.line, token.col_offset, token.length, SemanticTokenTypes.COMMENT)
+            return
+
+        # Plain argument text renders only in template rows / metadata values
+        # and for embedded-argument fragments.
+        if kind in self._ARGUMENT_TEXT_KINDS and not yield_arguments and not (mods and TokenModifier.EMBEDDED in mods):
+            return
+
+        info = self._emit_model_token(token)
+        if info is not None:
+            yield info
+
+    def _render_variable_part(self, token: SemanticToken, variable: SemanticToken) -> Iterator[SemTokenInfo]:
+        """Render one sub-token of `variable`: its name with the type and
+        modifiers of the variable, its type hint as `type`, nested variables as
+        variables of their own, and nothing else."""
+        kind = token.kind
+        if kind in self._VARIABLE_KINDS:
+            yield from self._render_model_token(token, emit_comments=False, yield_arguments=False)
+            return
+        if kind is TokenKind.PYTHON_EXPRESSION:
+            return
+        if token.sub_tokens:
+            for sub in token.sub_tokens:
+                yield from self._render_variable_part(sub, variable)
+            return
+        if kind is TokenKind.VARIABLE_BASE:
+            info = self._emit_model_token(variable)
+            if info is not None:
+                yield SemTokenInfo(token.line, token.col_offset, token.length, info.sem_token_type, info.sem_modifiers)
+        elif kind is TokenKind.VARIABLE_TYPE_HINT:
+            yield SemTokenInfo(token.line, token.col_offset, token.length, SemanticTokenTypes.TYPE)
+
+    def _iter_run_keyword_sem_tokens(
+        self,
+        stmt: "RunKeywordCallStatement",
+        model: SemanticModel,
+        emit_comments: bool,
+        is_inner: bool,
+    ) -> Iterator[SemTokenInfo]:
+        """Render a Run Keyword variant, position-merging inner calls with the
+        outer token stream. Cells covered by an inner call render from the
+        inner (KEYWORD_INNER) decomposition; the remaining outer tokens (outer
+        keyword, condition, ELSE/AND control-flow cells) render from the outer
+        list. The merge is purely structural."""
+        inner_infos: List[SemTokenInfo] = []
+        covered: List[Tuple[Tuple[int, int], Tuple[int, int]]] = []
+        for inner in stmt.inner_calls:
+            span = self._statement_span(inner)
+            if span is not None:
+                covered.append(span)
+            inner_infos.extend(self._iter_model_sem_tokens(inner, model, is_inner=True))
+
+        outer_infos: List[SemTokenInfo] = []
+        prev: Optional[SemanticToken] = None
+        for token in stmt.tokens:
+            pos = (token.line, token.col_offset)
+            if any(start <= pos < end for start, end in covered):
+                prev = token
+                continue
+            gap = self._bdd_gap_token(stmt, prev, token, is_inner)
+            if gap is not None:
+                outer_infos.append(gap)
+            outer_infos.extend(self._render_model_token(token, emit_comments, False))
+            prev = token
+
+        merged = inner_infos + outer_infos
+        merged.sort(key=lambda i: (i.lineno, i.col_offset))
+        yield from merged
+
+    @staticmethod
+    def _statement_span(
+        stmt: SemanticStatement,
+    ) -> Optional[Tuple[Tuple[int, int], Tuple[int, int]]]:
+        """Source span (start, end) as (line, col) tuples over all tokens of a
+        statement, recursing into nested Run Keyword inner calls."""
+        start: Optional[Tuple[int, int]] = None
+        end: Optional[Tuple[int, int]] = None
+
+        def visit(s: SemanticStatement) -> None:
+            nonlocal start, end
+            for token in s.tokens:
+                lo = (token.line, token.col_offset)
+                hi = (token.line, token.col_offset + token.length)
+                if start is None or lo < start:
+                    start = lo
+                if end is None or hi > end:
+                    end = hi
+            if cached_isinstance(s, RunKeywordCallStatement):
+                for inner in s.inner_calls:
+                    visit(inner)
+
+        visit(stmt)
+        if start is None or end is None:
+            return None
+        return start, end
+
     def _get_tokens_after(self, tokens: List[Token], target_token: Token) -> List[Token]:
         """Get all tokens after target token efficiently.
 
@@ -1194,6 +1484,7 @@ class SemanticTokenGenerator:
         col_offset: Optional[int] = None,
         length: Optional[int] = None,
         yield_arguments: bool = False,
+        site: VariableSite = VariableSite.USAGE,
     ) -> Iterator[SemTokenInfo]:
         """Generate semantic token information for Robot Framework tokens.
 
@@ -1205,6 +1496,7 @@ class SemanticTokenGenerator:
             col_offset: Optional column offset override
             length: Optional length override
             yield_arguments: Whether to yield argument tokens
+            site: Where a variable token stands
 
         Yields:
             SemTokenInfo: Semantic token information for LSP client
@@ -1393,37 +1685,8 @@ class SemanticTokenGenerator:
                         col_offset,
                         length,
                     )
-            elif token.type == Token.OPTION:
-                if (
-                    cached_isinstance(node, ExceptHeader) or cached_isinstance(node, WhileHeader)
-                ) and "=" in token.value:
-                    if col_offset is None:
-                        col_offset = token.col_offset
-
-                    name, value = token.value.split("=", 1)
-                    yield SemTokenInfo.from_token(
-                        token,
-                        RobotSemTokenTypes.VARIABLE,
-                        sem_mod,
-                        col_offset,
-                        len(name),
-                    )
-                    yield SemTokenInfo.from_token(
-                        token,
-                        SemanticTokenTypes.OPERATOR,
-                        sem_mod,
-                        col_offset + len(name),
-                        1,
-                    )
-                    yield SemTokenInfo.from_token(
-                        token,
-                        sem_type,
-                        sem_mod,
-                        col_offset + len(name) + 1,
-                        len(value),
-                    )
-                else:
-                    yield SemTokenInfo.from_token(token, sem_type, sem_mod, col_offset, length)
+            elif token.type in (Token.VARIABLE, Token.ASSIGN):
+                yield from self._generate_variable_sem_tokens(namespace, token, sem_type, sem_mod, site)
             elif (
                 token.type in Token.SETTING_TOKENS and token.value and token.value[0] == "[" and token.value[-1] == "]"
             ):
@@ -1471,33 +1734,37 @@ class SemanticTokenGenerator:
         if token.type in {Token.ARGUMENT, Token.TESTCASE_NAME, Token.KEYWORD_NAME} or (
             token.type == Token.NAME and cached_isinstance(node, VariablesImport, LibraryImport, ResourceImport)
         ):
-            if (
+            if cached_isinstance(node, Arguments):
+                # The name of an argument is a parameter; a default value continues as argument.
+                name, value = split_from_equals(token.value)
+                if name:
+                    yield from self._generate_variable_sem_tokens(
+                        namespace,
+                        Token(Token.VARIABLE, name, token.lineno, token.col_offset),
+                        SemanticTokenTypes.PARAMETER,
+                        None,
+                        VariableSite.DECLARATION,
+                    )
+                token = Token(
+                    token.type,
+                    value or "",
+                    token.lineno,
+                    token.col_offset + len(name) + (1 if value is not None else 0),
+                    token.error,
+                )
+            elif (
                 cached_isinstance(node, Variable) and token.type == Token.ARGUMENT and node.name and node.name[0] == "&"
-            ) or (cached_isinstance(node, Arguments)):
+            ):
                 name, value = split_from_equals(token.value)
                 if value is not None:
                     length = len(name)
 
                     yield SemTokenInfo.from_token(
-                        Token(
-                            ROBOT_NAMED_ARGUMENT if cached_isinstance(node, Variable) else SemanticTokenTypes.PARAMETER,
-                            name,
-                            token.lineno,
-                            token.col_offset,
-                        ),
-                        (
-                            RobotSemTokenTypes.NAMED_ARGUMENT
-                            if cached_isinstance(node, Variable)
-                            else SemanticTokenTypes.PARAMETER
-                        ),
+                        Token(ROBOT_NAMED_ARGUMENT, name, token.lineno, token.col_offset),
+                        RobotSemTokenTypes.NAMED_ARGUMENT,
                     )
                     yield SemTokenInfo.from_token(
-                        Token(
-                            ROBOT_OPERATOR,
-                            "=",
-                            token.lineno,
-                            token.col_offset + length,
-                        ),
+                        Token(ROBOT_OPERATOR, "=", token.lineno, token.col_offset + length),
                         SemanticTokenTypes.OPERATOR,
                     )
                     token = Token(
@@ -1507,36 +1774,70 @@ class SemanticTokenGenerator:
                         token.col_offset + length + 1,
                         token.error,
                     )
-                elif cached_isinstance(node, Arguments) and name:
-                    yield SemTokenInfo.from_token(
-                        Token(
-                            ROBOT_NAMED_ARGUMENT,
-                            name,
-                            token.lineno,
-                            token.col_offset,
-                        ),
-                        RobotSemTokenTypes.NAMED_ARGUMENT,
-                    )
-                    token = Token(
-                        token.type,
-                        "",
-                        token.lineno,
-                        token.col_offset + len(name),
-                        token.error,
-                    )
 
+            site = VariableSite.KEYWORD_NAME if token.type == Token.KEYWORD_NAME else VariableSite.USAGE
             for sub_token in ModelHelper.tokenize_variables(
                 token,
                 ignore_errors=True,
                 identifiers="$" if token.type == Token.KEYWORD_NAME else "$@&%",
             ):
-                for e in self.generate_sem_sub_tokens(namespace, builtin_library_doc, sub_token, node):
+                for e in self.generate_sem_sub_tokens(namespace, builtin_library_doc, sub_token, node, site=site):
                     yield e
 
         else:
-            for e in self.generate_sem_sub_tokens(namespace, builtin_library_doc, token, node):
+            # A whole VARIABLE or ASSIGN cell defines a variable.
+            site = (
+                VariableSite.UNTYPED_DECLARATION if cached_isinstance(node, ExceptHeader) else VariableSite.DECLARATION
+            )
+            for e in self.generate_sem_sub_tokens(namespace, builtin_library_doc, token, node, site=site):
                 yield e
 
+    def _generate_variable_sem_tokens(
+        self,
+        namespace: Namespace,
+        token: Token,
+        sem_type: AnyTokenType,
+        sem_mod: Optional[Set[AnyTokenModifier]],
+        site: VariableSite,
+    ) -> Iterator[SemTokenInfo]:
+        """A variable sends its name with `sem_type` and its type hint as `type`;
+        prefix, braces, item access, `=`, patterns, defaults and inline Python
+        are left to the grammar. The parts are split like the semantic model
+        splits them."""
+        position = Position(line=token.lineno - 1, character=token.col_offset)
+
+        def is_defined(name: str) -> bool:
+            return namespace.find_variable(name, position=position) is not None
+
+        sub_tokens = build_variable_sub_tokens(
+            token.value, token.lineno, token.col_offset, site=site, is_defined=is_defined
+        )
+        if not sub_tokens:
+            yield SemTokenInfo.from_token(token, sem_type, sem_mod)
+            return
+        yield from self._iter_variable_name_sem_tokens(sub_tokens, sem_type, sem_mod)
+
+    def _iter_variable_name_sem_tokens(
+        self,
+        sub_tokens: Sequence[SemanticToken],
+        sem_type: AnyTokenType,
+        sem_mod: Optional[Set[AnyTokenModifier]],
+    ) -> Iterator[SemTokenInfo]:
+        for sub in sub_tokens:
+            if sub.kind is TokenKind.VARIABLE:
+                if sub.sub_tokens:
+                    yield from self._iter_variable_name_sem_tokens(sub.sub_tokens, RobotSemTokenTypes.VARIABLE, None)
+                else:
+                    yield SemTokenInfo(sub.line, sub.col_offset, sub.length, RobotSemTokenTypes.VARIABLE)
+            elif sub.kind is TokenKind.PYTHON_EXPRESSION:
+                continue
+            elif sub.sub_tokens:
+                yield from self._iter_variable_name_sem_tokens(sub.sub_tokens, sem_type, sem_mod)
+            elif sub.kind is TokenKind.VARIABLE_BASE:
+                yield SemTokenInfo(sub.line, sub.col_offset, sub.length, sem_type, sem_mod)
+            elif sub.kind is TokenKind.VARIABLE_TYPE_HINT:
+                yield SemTokenInfo(sub.line, sub.col_offset, sub.length, SemanticTokenTypes.TYPE)
+
     def collect_tokens(
         self,
         document: TextDocument,