tree-sitter-ktav 0.2.1 → 0.5.0

package/README.md

@@ -171,7 +171,8 @@ rejects (mostly missing-whitespace-after-marker — § 6.10).
 
 ## License
 
-MIT — see [LICENSE](LICENSE).
+Dual-licensed under **MIT OR Apache-2.0** — see
+[LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE).
 
 ## Other Ktav repositories
 

package/grammar.js

@@ -1,24 +1,36 @@
 /**
  * Tree-sitter grammar for Ktav (כְּתָב) — the Written Configuration Format.
  *
- * Spec: https://github.com/ktav-lang/spec/blob/main/versions/0.1/spec.md
+ * Spec: https://github.com/ktav-lang/spec/blob/main/versions/0.5/spec.md
  *
  * Ktav is line-oriented. Every line is one of:
  *   - blank
- *   - a comment (`# ...`)
- *   - a key:value pair (with markers `:`, `::`, `:i`, `:f`)
+ *   - a comment (`## ...`)  ← NOTE: double-hash in 0.5.0; single `#` is content
+ *   - a key:value pair (with markers `:` or `::`)
  *   - a structural opener / closer for compounds (`{`, `}`, `[`, `]`,
  *     `(`, `((`, `)`, `))`)
- *   - an array item (inside an open `[` array)
+ *   - an array item (inside an open `[` array, or at the top level)
  *   - raw content of a multi-line string
  *
+ * Changes from 0.3.0 (spec 0.1.1) to 0.5.0:
+ *   - Comment marker changed from `#` to `##`. Single `#` is now a
+ *     content byte (allowed in keys and scalar values).
+ *   - Typed markers `:i` and `:f` removed. Only `:` and `::` remain.
+ *   - Inline compounds: `{key: value, ...}` and `[v1, v2, ...]` are
+ *     now valid as pair values or array items (inline_object /
+ *     inline_array rules).
+ *   - Number literals: hex (`0x`), octal (`0o`), binary (`0b`), decimal
+ *     with underscore separators, and floats with `.` or exponent.
+ *     These are captured as distinct node kinds for highlighting.
+ *   - Escape sequences inside inline scalars: `\\`, `\,`, `\}`, `\]`,
+ *     `\{`, `\[`, `\n`, `\r`.
+ *
  * Strategy:
  *   - Newlines are explicit (`_newline`) and structural openers/closers
  *     are tokens that include the trailing whitespace + newline so they
  *     can NEVER be confused with a scalar starting with the same byte.
- *   - The four pair separators (`:`, `::`, `:i`, `:f`) are recognized
- *     by the lexer with longest-match precedence (`::` > `:`, `:i`/`:f`
- *     > `:`).
+ *   - The two pair separators (`:`, `::`) are recognized by the lexer
+ *     with longest-match precedence (`::` > `:`).
  *   - The mandatory-whitespace-after-marker rule (§ 6.10) is enforced
  *     by an external scanner token `_marker_ws`, which is a zero-width
  *     assertion that only succeeds when the byte right after the
@@ -31,6 +43,8 @@
  *     a parse error.
  *   - Multi-line string content is captured as a sequence of opaque
  *     "raw lines" up to the matching terminator.
+ *   - Inline compound values are fully parsed; escape sequences inside
+ *     inline scalars are captured as `escape_sequence` nodes.
  *   - Indentation is not significant (matches the spec).
  */
 
@@ -55,6 +69,10 @@ module.exports = grammar({
   word: $ => $._key_segment,
 
   rules: {
+    // The top-level document is a sequence of lines. Per spec § 5.0.1
+    // the root may be either an Object or an Array. Tree-sitter accepts
+    // both kinds of line anywhere; semantic dispatch is left to the
+    // reference parser.
     source_file: $ => repeat($._line),
 
     // ---- Top-level lines ----
@@ -62,6 +80,7 @@ module.exports = grammar({
       $.comment,
       $.blank_line,
       $.object_pair,
+      $.top_array_item,
     ),
 
     blank_line: $ => $._newline,
@@ -69,26 +88,26 @@ module.exports = grammar({
     _newline: $ => /\r?\n/,
 
     // ---- Comment ----
-    comment: $ => seq(
-      '#',
-      optional(/[^\r\n]*/),
-      $._newline,
-    ),
+    //
+    // In spec 0.5.0 a comment starts with `##` (two hashes). A single
+    // `#` is ordinary content. The token captures the whole line
+    // including the trailing newline to beat `_top_scalar_text` at the
+    // lexer's longest-match step.
+    comment: $ => token(prec(1, /##[^\r\n]*\r?\n/)),
 
     // ---- Object pair ----
     //
     // After the separator, the external `_marker_ws` token asserts
-    // that the next byte is whitespace, CR, LF, or EOF. This enforces
-    // § 6.10 (mandatory whitespace after marker): `key:value` fails.
+    // that the next byte is whitespace, CR, LF, or EOF (§ 6.10).
     object_pair: $ => choice(
-      // After `::`, `:i`, `:f` the body is a literal/typed scalar — § 5.2:
-      // it is NOT dispatched through compound-opener / multi-line dispatch.
-      // Only `empty_value` (immediate newline) or `scalar` is allowed.
+      // After `::` the body is a literal scalar (NOT dispatched through
+      // compound-opener / multi-line dispatch). `raw_scalar` accepts any
+      // line content including `(`, `{`, `[`-starting text.
       seq(
         field('key', $.key),
-        field('separator', choice($.sep_raw, $.sep_int, $.sep_float)),
+        field('separator', $.sep_raw),
         $._marker_ws,
-        field('value', choice($.empty_value, $.scalar)),
+        field('value', choice($.empty_value, $.raw_scalar)),
       ),
       // After `:` the body goes through the full § 5.2 dispatch.
       seq(
@@ -102,27 +121,31 @@ module.exports = grammar({
       ),
     ),
 
-    // Tree-sitter prefers longest token match. To make sure `::` wins
-    // over `:`, `::` is given higher precedence; same for `:i`/`:f`.
+    // `::` wins over `:` via higher precedence.
     sep_raw:    $ => token(prec(3, '::')),
-    sep_int:    $ => token(prec(2, ':i')),
-    sep_float:  $ => token(prec(2, ':f')),
     sep_string: $ => token(prec(1, ':')),
 
     // ---- Keys ----
     key: $ => choice(
-      $._key_segment,
+      $._spaced_key,
       $.dotted_key,
     ),
 
+    // A key may contain internal whitespace (spec 0.5.0 § 4): the run of
+    // space-separated segments up to the separator is one key
+    // (`multi word key: value`). The inter-segment spaces are `extras`,
+    // so the `key` node still spans the whole text with no named children
+    // (renders as `(key)`, same as a single-segment key).
+    _spaced_key: $ => prec.left(repeat1($._key_segment)),
+
     dotted_key: $ => prec.left(seq(
       $._key_segment,
       repeat1(seq('.', $._key_segment)),
     )),
 
-    // Key segment: any chars except whitespace, "[", "]", "{", "}",
-    // ":", "#", ".".
-    _key_segment: $ => /[^\s\[\]\{\}:#.\r\n]+/,
+    // Key segment: any chars except whitespace, "[", "]", "{", "}", "(", ")",
+    // ":", ",", ".". Note: "#" is now allowed in keys (spec 0.5.0 § 4).
+    _key_segment: $ => /[^\s\[\]\{\}\(\):#,.\r\n]+/,
 
     // ---- Value line ----
     _value_line: $ => choice(
@@ -136,8 +159,14 @@ module.exports = grammar({
       $.empty_array,
       $.empty_paren,
       $.empty_double_paren,
+      // Inline compounds (new in 0.5.0).
+      $.inline_object,
+      $.inline_array,
       // Keywords (single token followed by newline).
       $.keyword,
+      // Number literals (distinct nodes for highlighting).
+      $.integer,
+      $.float,
       // Scalar — catch-all line content.
       $.scalar,
     ),
@@ -152,25 +181,12 @@ module.exports = grammar({
     empty_double_paren: $ => seq(token(prec(5, '(())')), $._newline),
 
     // ---- Multi-line compounds ----
-    //
-    // Openers are tokens that include the rest of the line up to and
-    // including the newline, so they cannot be confused with a scalar
-    // starting with the same byte. Closers, by contrast, are split
-    // into the bracket character(s) plus the external `_strict_eol`
-    // token; this lets the scanner reject pathological lines like
-    // `}   trailing_text\n` (§ 5.6.1 and the cleanliness rule for
-    // object/array closers).
     open_brace:    $ => token(prec(4, /\{[ \t]*\r?\n/)),
     close_brace:   $ => seq(token(prec(4, '}')),    $._strict_eol),
     open_bracket:  $ => token(prec(4, /\[[ \t]*\r?\n/)),
     close_bracket: $ => seq(token(prec(4, ']')),    $._strict_eol),
     open_paren:    $ => token(prec(4, /\([ \t]*\r?\n/)),
     open_dparen:   $ => token(prec(5, /\(\([ \t]*\r?\n/)),
-    // close_paren / close_dparen are emitted by the external scanner as
-    // `_stripped_close` / `_verbatim_close`. They are context-sensitive:
-    // inside `(...)` only `)` closes (a `))` line is content); inside
-    // `((...))` only `))` closes (a single `)` line is content). The
-    // scanner consults `valid_symbols` to decide which form to attempt.
     close_paren:   $ => $._stripped_close,
     close_dparen:  $ => $._verbatim_close,
 
@@ -186,30 +202,144 @@ module.exports = grammar({
       $.close_bracket,
     ),
 
-    // ---- Array items ----
+    // ---- Inline compounds (new in spec 0.5.0) ----
     //
-    // Marker-prefixed items must, like pair lines, have whitespace or
-    // EOL after the marker (§ 6.10).
+    // `{key: value, key2: value2}` and `[v1, v2, v3]` are valid as a
+    // value on the right-hand side of a pair or as an array item.
+    // Trailing comma is allowed. Nesting is supported.
+    //
+    // These are followed by a newline (they consume the rest of the line).
+    inline_object: $ => seq(
+      '{',
+      optional($._inline_pair_list),
+      '}',
+      $._newline,
+    ),
+
+    inline_array: $ => seq(
+      '[',
+      optional($._inline_item_list),
+      ']',
+      $._newline,
+    ),
+
+    _inline_pair_list: $ => seq(
+      $.inline_pair,
+      repeat(seq(',', $.inline_pair)),
+      optional(','),
+    ),
+
+    // The value is optional: `{x:, y: 1}` and `{empty:}` are valid —
+    // a separator immediately followed by `,` or `}` is an empty value
+    // (spec 0.5.0 § 5.8).
+    inline_pair: $ => seq(
+      field('key', $.key),
+      field('separator', choice($.sep_raw, $.sep_string)),
+      optional(field('value', $.inline_value)),
+    ),
+
+    _inline_item_list: $ => seq(
+      $.inline_value,
+      repeat(seq(',', $.inline_value)),
+      optional(','),
+    ),
+
+    // An inline value is either a nested inline compound, or an inline
+    // scalar (which may contain escape sequences).
+    inline_value: $ => choice(
+      $.nested_inline_object,
+      $.nested_inline_array,
+      $.inline_scalar,
+    ),
+
+    nested_inline_object: $ => seq(
+      '{',
+      optional($._inline_pair_list),
+      '}',
+    ),
+
+    nested_inline_array: $ => seq(
+      '[',
+      optional($._inline_item_list),
+      ']',
+    ),
+
+    // An inline scalar is terminated by an unescaped `,`, `}`, or `]`.
+    // It may contain escape sequences (§ 3.7).
+    //
+    // The first chunk must NOT begin with `{` or `[`: a value position
+    // that opens with `{`/`[` is a nested compound, not a scalar. After
+    // the first character those bytes are ordinary literal content
+    // (`hello{world`, `mid[bracket`). This head/rest split keeps
+    // `nested_inline_*` vs `inline_scalar` unambiguous without sacrificing
+    // mid-value literal braces (§ 5.8).
+    inline_scalar: $ => seq(
+      choice($.escape_sequence, $._inline_scalar_head),
+      repeat(choice($.escape_sequence, $._inline_scalar_text)),
+    ),
+
+    // Escape sequences recognised inside inline scalars (§ 3.7).
+    escape_sequence: $ => token(choice(
+      '\\\\',
+      '\\,',
+      '\\}',
+      '\\]',
+      '\\{',
+      '\\[',
+      '\\n',
+      '\\r',
+    )),
+
+    // Leading chunk of a scalar: first byte excludes whitespace and the
+    // openers `{`/`[` (so a value starting with an opener is a nested
+    // compound), plus the usual `\` `,` `}` `]` / CR / LF. Subsequent
+    // bytes allow `{`/`[` as literal content.
+    _inline_scalar_head: $ => token(/[^\s\\,\{\[\}\]\r\n][^\\,\}\]\r\n]*/),
+
+    // Continuation text after the head (or after an escape): any byte
+    // except `\`, `,`, the closers `}` `]`, and CR / LF. Open delimiters
+    // `{`/`[` are allowed here as literal content.
+    _inline_scalar_text: $ => token(/[^\\,\}\]\r\n]+/),
+
+    // ---- Array items ----
     array_item: $ => choice(
       seq(
         field('marker', $.sep_raw),
         $._marker_ws,
-        field('value', choice($.empty_value, $.scalar)),
-      ),
-      seq(
-        field('marker', $.sep_int),
-        $._marker_ws,
-        field('value', choice($.empty_value, $.scalar)),
+        field('value', choice($.empty_value, $.raw_scalar)),
       ),
+      // Plain value item — same set as object pair value.
+      field('value', $._value_line),
+    ),
+
+    // Top-level array item (§ 5.0.1).
+    top_array_item: $ => choice(
       seq(
-        field('marker', $.sep_float),
+        field('marker', $.sep_raw),
         $._marker_ws,
-        field('value', choice($.empty_value, $.scalar)),
+        field('value', choice($.empty_value, $.raw_scalar)),
       ),
-      // Plain value item — same set as object pair value.
-      field('value', $._value_line),
+      field('value', $.compound_object),
+      field('value', $.compound_array),
+      field('value', $.multiline_stripped),
+      field('value', $.multiline_verbatim),
+      field('value', $.empty_object),
+      field('value', $.empty_array),
+      field('value', $.empty_paren),
+      field('value', $.empty_double_paren),
+      field('value', $.inline_object),
+      field('value', $.inline_array),
+      field('value', $.keyword),
+      field('value', $.integer),
+      field('value', $.float),
+      field('value', $.top_scalar),
     ),
 
+    // `top_scalar` — bare-scalar at the document root. Forbids `:` so
+    // that pair-shaped lines always parse as `object_pair`.
+    top_scalar: $ => $._top_scalar_text,
+    _top_scalar_text: $ => token(/[^\s:\{\[\(\r\n][^:\r\n]*\r?\n/),
+
     // ---- Multi-line strings ----
     multiline_stripped: $ => seq(
       $.open_paren,
@@ -223,11 +353,6 @@ module.exports = grammar({
       $.close_dparen,
     ),
 
-    // A content line inside a multi-line string. Captured as one
-    // token. The closer tokens (`)`, `))` plus _strict_eol) win at
-    // the LR(1) boundary because they require their own line; any
-    // line whose content includes more than just the terminator
-    // falls through to this rule.
     multiline_content_line: $ => token(prec(-1, /[^\r\n]*\r?\n/)),
 
     // ---- Scalar (default value body, until end of line) ----
@@ -236,12 +361,55 @@ module.exports = grammar({
       $._newline,
     ),
 
+    // `raw_scalar` is used exclusively after `::` (raw marker). It accepts
+    // any non-empty line content, including `(`, `{`, `[`-starting text.
+    // Per spec § 5.2: after `::` the body is NEVER dispatched as a
+    // compound opener or multi-line string opener — it is always a literal
+    // String value. This is a separate rule (not `scalar`) because
+    // `scalar`'s underlying `_scalar_text` deliberately excludes those
+    // opening bytes to avoid lexer ambiguity in the non-raw value context.
+    raw_scalar: $ => seq(
+      $._raw_scalar_text,
+      $._newline,
+    ),
+    _raw_scalar_text: $ => /[^\s\r\n][^\r\n]*/,
+
     // Scalar text: any non-whitespace, non-newline content up to end
-    // of line. `#` is allowed (`color: #ff00ff` is a valid value);
-    // the `#` only opens a comment when it is the first non-whitespace
-    // char of a line — tree-sitter's context-aware lexer disambiguates
-    // because `comment` is only valid at a line-start parse state.
-    _scalar_text: $ => /[^\s\r\n][^\r\n]*/,
+    // of line. Both `#` and `##` are allowed as content bytes in 0.5.0.
+    // Lines starting with `{` or `[` are always handled by structural
+    // rules (compound_object, compound_array, empty_object, empty_array,
+    // inline_object, inline_array), so `_scalar_text` explicitly excludes
+    // those opening bytes at position 0 to avoid the greedy-token
+    // ambiguity. Lines starting with `(` are handled by multiline or
+    // empty-paren rules likewise.
+    _scalar_text: $ => /[^\s\{\[\(\r\n][^\r\n]*/,
+
+    // ---- Number literals ----
+    //
+    // Captured as distinct node kinds so syntax highlighters can colour
+    // them differently from plain strings.
+    //
+    // IMPORTANT: These are whole-line tokens (pattern + optional
+    // horizontal whitespace + newline). Using a whole-line token (like
+    // `comment` and `_top_scalar_text`) avoids the ambiguity where the
+    // partial integer token `1` wins over the scalar token `1:2:3` via
+    // prec — with a whole-line token the match for `1:2:3\n` is length
+    // 5 for scalar and no match for integer (`:` breaks the pattern),
+    // so scalar correctly wins on non-numeric lines.
+    //
+    // Float must be tested before integer because the float pattern
+    // (decimal point form) is a strict superset of the integer pattern
+    // prefix. Float is given prec(3) so it beats integer on `1.5\n`.
+    //
+    // Integer forms (§ 3.6): hex, octal, binary, decimal with underscores.
+    integer: $ => token(prec(2,
+      /[+-]?(0x[0-9a-fA-F]([_]?[0-9a-fA-F])*|0o[0-7]([_]?[0-7])*|0b[01]([_]?[01])*|[0-9]([_]?[0-9])*)[ \t]*\r?\n/
+    )),
+
+    // Float forms (§ 3.6): decimal-point form or exponent-only form.
+    float: $ => token(prec(3,
+      /([+-]?[0-9]([_]?[0-9])*\.[0-9]([_]?[0-9])*([eE][+-]?[0-9]([_]?[0-9])*)?|[+-]?[0-9]([_]?[0-9])*[eE][+-]?[0-9]([_]?[0-9])*)[ \t]*\r?\n/
+    )),
 
     // ---- Keywords ----
     keyword: $ => seq(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-ktav",
-  "version": "0.2.1",
+  "version": "0.5.0",
   "description": "Tree-sitter grammar for Ktav (כְּתָב) — the Written Configuration Format",
   "keywords": [
     "parser",
@@ -11,7 +11,7 @@
     "configuration"
   ],
   "author": "Marat K <phpcraftdream@gmail.com>",
-  "license": "MIT",
+  "license": "MIT OR Apache-2.0",
   "homepage": "https://github.com/ktav-lang/tree-sitter-ktav",
   "repository": {
     "type": "git",

package/queries/highlights.scm

@@ -14,8 +14,6 @@
 ; ---- Pair separators ----
 (sep_string) @punctuation.delimiter
 (sep_raw)    @punctuation.special
-(sep_int)    @punctuation.special
-(sep_float)  @punctuation.special
 
 ; ---- Compound brackets ----
 ; The structural openers / closers each form their own visible token
@@ -43,9 +41,12 @@
 (kw_true)  @constant.builtin.boolean
 (kw_false) @constant.builtin.boolean
 
+; ---- Number literals (new in spec 0.5.0) ----
+(integer) @number
+(float)   @number.float
+
 ; ---- String values ----
-; Plain scalar after `:` — usually a string. (Numeric coercion happens
-; only after `:i`/`:f`; see below.)
+; Plain scalar after `:` — a string.
 (object_pair
   separator: (sep_string)
   value: (scalar) @string)
@@ -53,33 +54,22 @@
 ; Raw string after `::`
 (object_pair
   separator: (sep_raw)
-  value: (scalar) @string.special)
-
-; Typed integer / float bodies
-(object_pair
-  separator: (sep_int)
-  value: (scalar) @number)
-
-(object_pair
-  separator: (sep_float)
-  value: (scalar) @number.float)
+  value: (raw_scalar) @string.special)
 
 ; ---- Array items ----
 (array_item
   marker: (sep_raw)
-  value: (scalar) @string.special)
-
-(array_item
-  marker: (sep_int)
-  value: (scalar) @number)
-
-(array_item
-  marker: (sep_float)
-  value: (scalar) @number.float)
+  value: (raw_scalar) @string.special)
 
 (array_item
   value: (scalar) @string)
 
+; ---- Inline compounds (new in spec 0.5.0) ----
+(inline_object) @punctuation.bracket
+(inline_array)  @punctuation.bracket
+(inline_scalar) @string
+(escape_sequence) @string.escape
+
 ; ---- Multi-line strings ----
 (multiline_stripped) @string
 (multiline_verbatim) @string