volaro 0.0.2 → 0.1.0-alpha.3

package/compiler/validator/vlcheck/diagnostics.py

@@ -0,0 +1,88 @@
+"""Diagnostics: severity, source position mapping, and human / JSON rendering.
+
+The JSON shape mirrors spec section 18 (`vl check --json`): a stable `code`, a
+`severity`, a `message`, a byte `range`, and an optional `fix`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class Severity(str, Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+
+
+class LineIndex:
+    """Maps byte offsets to 1-based (line, column). Built once per file."""
+
+    def __init__(self, text: str) -> None:
+        self.text = text
+        self._starts = [0]
+        for i, ch in enumerate(text):
+            if ch == "\n":
+                self._starts.append(i + 1)
+        self._lines = text.split("\n")
+
+    def locate(self, offset: int) -> tuple[int, int]:
+        offset = max(0, min(offset, len(self.text)))
+        lo, hi = 0, len(self._starts) - 1
+        while lo < hi:
+            mid = (lo + hi + 1) // 2
+            if self._starts[mid] <= offset:
+                lo = mid
+            else:
+                hi = mid - 1
+        return lo + 1, offset - self._starts[lo] + 1
+
+    def line_text(self, line_no: int) -> str:
+        if 1 <= line_no <= len(self._lines):
+            return self._lines[line_no - 1]
+        return ""
+
+
+@dataclass
+class Diagnostic:
+    code: str
+    severity: Severity
+    message: str
+    start: int
+    end: int
+    fix: str | None = None
+    file: str = "<input>"
+
+    def render_human(self, li: LineIndex) -> str:
+        line, col = li.locate(self.start)
+        src = li.line_text(line)
+        span = max(1, self.end - self.start)
+        # keep the caret run on the line
+        span = min(span, max(1, len(src) - (col - 1)))
+        gutter = f"{line:>4} "
+        pad = " " * len(gutter)
+        out = [
+            f"{self.severity.value}[{self.code}]: {self.message}",
+            f"{pad}┌─ {self.file}:{line}:{col}",
+            f"{pad}│",
+            f"{gutter}│ {src}",
+            f"{pad}│ {' ' * (col - 1)}{'^' * span}",
+        ]
+        if self.fix:
+            out.append(f"{pad}= fix: {self.fix}")
+        return "\n".join(out)
+
+    def to_json(self, li: LineIndex) -> dict:
+        line, col = li.locate(self.start)
+        obj = {
+            "code": self.code,
+            "severity": self.severity.value,
+            "message": self.message,
+            "file": self.file,
+            "range": {"start": self.start, "end": self.end},
+            "position": {"line": line, "col": col},
+        }
+        if self.fix:
+            obj["fix"] = {"description": self.fix}
+        return obj

package/compiler/validator/vlcheck/lexer.py

@@ -0,0 +1,343 @@
+"""Lexer: UTF-8 source -> token stream with synthesized NEWLINE / INDENT / DEDENT.
+
+Follows spec section 3:
+  - 2 spaces per indent level; tabs in leading whitespace are a hard error
+  - newline ends a statement; indent opens a block; dedent closes it
+  - inside an open bracket pair ( [ {  indentation produces no INDENT/DEDENT;
+    NEWLINE is still emitted, since spec 3.7 makes it an item separator there
+  - `#` line comment, `##` doc comment (both produce no tokens in this prototype)
+  - duration literals (10s, 500ms, 30d) are their own kind, distinct from int
+The lexer never fails hard: it records a Diagnostic and keeps going.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .diagnostics import Diagnostic, Severity
+
+# The 29 reserved words (spec section 3.5). Never identifiers, in any position.
+KEYWORDS = {
+    "and", "as", "auth", "break", "catch", "derive", "else", "err", "false",
+    "fn", "for", "if", "in", "let", "match", "mut", "nil", "not",
+    "on", "or", "par", "return", "skip", "spawn", "true", "try", "type",
+    "use", "while",
+}
+# The 18 contextual words (spec section 3.5). Keywords only as the head of a
+# declaration or a block that expects them; ordinary identifiers everywhere else
+# (a field named `state`, a function named `load`). Recognised by position in the
+# parser via `at_soft`; the lexer emits them as plain NAME tokens.
+SOFT_KEYWORDS = {
+    "api", "error", "errors", "index", "key", "list", "load", "model", "none",
+    "page", "pending", "prefix", "public", "role", "service", "session", "state",
+    "theme", "view", "recipe",
+}
+
+PRIMS = {"int", "float", "str", "bool", "bytes", "time", "dur", "uuid", "json"}
+
+DURATION_SUFFIXES = ("ms", "ns", "us", "s", "m", "h", "d")
+
+_OPS2 = ("..", "|>", "??", "->", "==", "!=", "<=", ">=", "+=", "-=", "*=", "/=")
+_OPS1 = set("+-*/%<>=(){}[].,:?!|")
+_OPEN = set("([{")
+_CLOSE = set(")]}")
+
+
+@dataclass
+class Token:
+    kind: str          # NAME TYPENAME PRIM KW OP INT FLOAT STRING DURATION BOOL NIL
+                       # NEWLINE INDENT DEDENT EOF
+    value: str
+    start: int
+    end: int
+    interps: list = field(default_factory=list)   # STRING only: [(start, end)] of {expr} spans
+
+
+class Lexer:
+    def __init__(self, src: str, filename: str = "<input>") -> None:
+        self.src = src
+        self.filename = filename
+        self.i = 0
+        self.n = len(src)
+        self.tokens: list[Token] = []
+        self.diags: list[Diagnostic] = []
+        self.indents = [0]
+        self.bracket_depth = 0
+        self.at_line_start = True
+
+    # -- helpers -----------------------------------------------------------
+    def _err(self, code: str, msg: str, s: int, e: int, fix: str | None = None) -> None:
+        self.diags.append(Diagnostic(code, Severity.ERROR, msg, s, e, fix, self.filename))
+
+    def _add(self, kind: str, value: str, s: int, e: int) -> Token:
+        t = Token(kind, value, s, e)
+        self.tokens.append(t)
+        return t
+
+    def _emit_newline(self, at: int) -> None:
+        if not self.tokens:
+            return
+        if self.tokens[-1].kind in ("NEWLINE", "INDENT", "DEDENT"):
+            return
+        self._add("NEWLINE", "", at, at)
+
+    # -- main ------------------------------------------------------------
+    def lex(self) -> list[Token]:
+        if self.src.startswith(""):
+            self._err("E-BOM", "byte-order mark at the start of a file is not allowed",
+                      0, 1, "save the file as UTF-8 with no signature")
+            self.i = 1
+
+        while self.i < self.n:
+            if self.at_line_start:
+                if self.bracket_depth == 0:
+                    self._handle_line_start()
+                    if self.i >= self.n:
+                        break
+                else:
+                    # Inside an open bracket, indentation produces no tokens (by
+                    # design — see the module docstring). Clear the flag here
+                    # rather than leaving it set: otherwise, if bracket_depth
+                    # drops back to 0 mid-line (a closing bracket is not the
+                    # last thing on its line, e.g. `)?`), this stale flag fires
+                    # on the next loop iteration at the post-bracket cursor
+                    # position — not real leading whitespace — and emits a
+                    # bogus DEDENT computed from width 0.
+                    self.at_line_start = False
+
+            c = self.src[self.i]
+            if c == "\n":
+                self._emit_newline(self.i)
+                self.i += 1
+                self.at_line_start = True
+                continue
+            if c in " \r":
+                self.i += 1
+                continue
+            if c == "#":
+                self._consume_comment()
+                continue
+            if c == '"':
+                self._lex_string()
+                continue
+            if c.isdigit():
+                self._lex_number()
+                continue
+            if c.isalpha() or c == "_":
+                self._lex_ident()
+                continue
+            if self._lex_operator():
+                continue
+            self._err("E-CHAR", f"unexpected character {c!r}", self.i, self.i + 1)
+            self.i += 1
+
+        # flush
+        self._emit_newline(self.n)
+        while len(self.indents) > 1:
+            self.indents.pop()
+            self._add("DEDENT", "", self.n, self.n)
+        self._add("EOF", "", self.n, self.n)
+        return self.tokens
+
+    # -- indentation ---------------------------------------------------------
+    def _handle_line_start(self) -> None:
+        self.at_line_start = False
+        j = self.i
+        width = 0
+        saw_tab = False
+        while j < self.n and self.src[j] in " \t":
+            if self.src[j] == "\t":
+                saw_tab = True
+                width += 2   # count as one level so recovery keeps a plausible shape
+            else:
+                width += 1
+            j += 1
+
+        # blank line or comment-only line: do not touch the indent stack
+        if j >= self.n or self.src[j] in "\n#":
+            self.i = j
+            return
+
+        if saw_tab:
+            self._err("E-TAB", "tabs are not allowed in indentation", self.i, j,
+                      "use two spaces per indent level")
+        if width % 2 != 0:
+            self._err("E-INDENT-ODD",
+                      f"indentation of {width} spaces is not a multiple of 2",
+                      self.i, j, "use two spaces per indent level")
+
+        self.i = j
+        top = self.indents[-1]
+        if width > top:
+            self.indents.append(width)
+            self._add("INDENT", "", self.i, self.i)
+        elif width < top:
+            while len(self.indents) > 1 and self.indents[-1] > width:
+                self.indents.pop()
+                self._add("DEDENT", "", self.i, self.i)
+            if self.indents[-1] != width:
+                self._err("E-INDENT", "dedent does not match any enclosing block",
+                          self.i, self.i + 1, "align this line with an enclosing block")
+                self.indents.append(width)  # recover
+
+    # -- tokens ------------------------------------------------------------
+    def _consume_comment(self) -> None:
+        while self.i < self.n and self.src[self.i] != "\n":
+            self.i += 1
+
+    def _lex_string(self) -> None:
+        s = self.i
+        src = self.src
+        if src[self.i:self.i + 3] == '"""':
+            self.i += 3
+            while self.i < self.n and src[self.i:self.i + 3] != '"""':
+                if src[self.i] == "\\":
+                    self._lex_escape()
+                else:
+                    self.i += 1
+            if src[self.i:self.i + 3] == '"""':
+                self.i += 3
+            else:
+                self._err("E-STRING", "unterminated multi-line string", s, self.n)
+            self._add("STRING", src[s:self.i], s, self.i)
+            return
+
+        self.i += 1  # opening quote
+        interps: list[tuple[int, int]] = []
+        while self.i < self.n:
+            c = src[self.i]
+            if c == "\\":
+                self._lex_escape()
+                continue
+            if c == "\n":
+                self._err("E-STRING", "unterminated string literal", s, self.i,
+                          'use """ for a multi-line string')
+                break
+            if c == '"':
+                self.i += 1
+                break
+            if c == "{":
+                self.i += 1
+                estart = self.i
+                depth = 1
+                while self.i < self.n and depth > 0:
+                    cc = src[self.i]
+                    if cc == "\n":
+                        break
+                    if cc == "{":
+                        depth += 1
+                    elif cc == "}":
+                        depth -= 1
+                        if depth == 0:
+                            break
+                    self.i += 1
+                interps.append((estart, self.i))
+                if self.i < self.n and src[self.i] == "}":
+                    self.i += 1
+                continue
+            self.i += 1
+        tok = self._add("STRING", src[s:self.i], s, self.i)
+        tok.interps = interps
+
+    def _lex_escape(self) -> None:
+        """Validate escapes before emission can silently change their meaning."""
+        start = self.i
+        self.i += 1
+        if self.i < self.n and self.src[self.i] in 'nt"\\{}':
+            self.i += 1
+            return
+        if self.src[self.i:self.i + 2] == "u{":
+            end = self.src.find("}", self.i + 2)
+            digits = self.src[self.i + 2:end] if end != -1 else ""
+            if (1 <= len(digits) <= 6
+                    and all(c in "0123456789abcdefABCDEF" for c in digits)):
+                value = int(digits, 16)
+                if value <= 0x10ffff and not 0xd800 <= value <= 0xdfff:
+                    self.i = end + 1
+                    return
+            self._err("E-STRING-ESCAPE", "invalid Unicode string escape", start,
+                      min(self.n, end + 1 if end != -1 else self.i + 1),
+                      r"use \u{HEX} with a Unicode scalar value (no surrogate code points)")
+            if end != -1 and not any(c in self.src[self.i:end] for c in '\n"'):
+                self.i = end + 1
+            return
+        self._err("E-STRING-ESCAPE", "unknown string escape", start,
+                  min(self.n, self.i + 1),
+                  r'double a literal backslash: "\\s" for regex \s; supported escapes: \n \t \" \\ \{ \} \u{HEX}')
+        # Keep a closing quote/newline available for normal string recovery.
+        if self.i < self.n and self.src[self.i] not in '\n"':
+            self.i += 1
+
+    def _lex_number(self) -> None:
+        s = self.i
+        src = self.src
+        if src[self.i] == "0" and self.i + 1 < self.n and src[self.i + 1] in "xXbB":
+            self.i += 2
+            while self.i < self.n and (src[self.i].isalnum() or src[self.i] == "_"):
+                self.i += 1
+            self._add("INT", src[s:self.i], s, self.i)
+            return
+
+        while self.i < self.n and (src[self.i].isdigit() or src[self.i] == "_"):
+            self.i += 1
+
+        is_float = False
+        if (self.i + 1 < self.n and src[self.i] == "." and src[self.i + 1].isdigit()):
+            is_float = True
+            self.i += 1
+            while self.i < self.n and (src[self.i].isdigit() or src[self.i] == "_"):
+                self.i += 1
+
+        if not is_float:
+            for suf in DURATION_SUFFIXES:
+                if src[self.i:self.i + len(suf)] == suf:
+                    after = self.i + len(suf)
+                    if after >= self.n or not (src[after].isalnum() or src[after] == "_"):
+                        self.i = after
+                        self._add("DURATION", src[s:self.i], s, self.i)
+                        return
+
+        self._add("FLOAT" if is_float else "INT", src[s:self.i], s, self.i)
+
+    def _lex_ident(self) -> None:
+        s = self.i
+        src = self.src
+        while self.i < self.n and (src[self.i].isalnum() or src[self.i] == "_"):
+            self.i += 1
+        word = src[s:self.i]
+        if word in ("true", "false"):
+            self._add("BOOL", word, s, self.i)
+        elif word == "nil":
+            self._add("NIL", word, s, self.i)
+        elif word in KEYWORDS:
+            self._add("KW", word, s, self.i)
+        elif word in PRIMS:
+            self._add("PRIM", word, s, self.i)
+        elif word[0].isupper():
+            self._add("TYPENAME", word, s, self.i)
+        else:
+            self._add("NAME", word, s, self.i)
+
+    def _lex_operator(self) -> bool:
+        src = self.src
+        if src[self.i:self.i + 3] == "..=":
+            self._push_op("..=", 3)
+            return True
+        two = src[self.i:self.i + 2]
+        if two in _OPS2:
+            self._push_op(two, 2)
+            return True
+        c = src[self.i]
+        if c in _OPS1:
+            if c in _OPEN:
+                self.bracket_depth += 1
+            elif c in _CLOSE:
+                self.bracket_depth = max(0, self.bracket_depth - 1)
+            self._push_op(c, 1)
+            return True
+        return False
+
+    def _push_op(self, text: str, length: int) -> None:
+        s = self.i
+        self.i += length
+        self._add("OP", text, s, self.i)