opencrbasic 1.0.0__py3-none-any.whl

opencrbasic/positions.py

@@ -0,0 +1,274 @@
+"""Source positions for editor tooling: compiler-reported lines and logical columns -> physical
+file ranges -> LSP positions. Read-only over the provenance the loader records (`source.Piece`,
+`source.LineSpan`, `source.Occurrence`); it changes no compiler output. See `docs/lsp-api.md`
+"Source positions".
+
+Three levels:
+
+- `reported_spans` / `line_span`: a compiler-reported (file, line) -> the physical line span(s) of
+  the logical line(s) reported there (`Program.line_map`).
+- `logical_ranges`: a column range of a `LogicalLine.text` -> physical (line, column) ranges,
+  through continuation joins and initializer pulls.
+- `lsp_position` / `lsp_range`: a physical column counted in decoded (Latin-1) characters, i.e. a
+  byte offset in the line, -> an LSP `Position` for the editor's file encoding and the negotiated
+  position encoding.
+"""
+
+import codecs
+from collections.abc import Mapping, Sequence
+from typing import NamedTuple
+
+from .source import LineSpan, LogicalLine, Occurrence, Piece
+
+__all__ = [
+    "LineSpan",
+    "Occurrence",
+    "Piece",
+    "PhysicalRange",
+    "reported_spans",
+    "line_span",
+    "logical_ranges",
+    "exact_range",
+    "physical_lines",
+    "lsp_position",
+    "lsp_range",
+    "decoded_position",
+    "POSITION_ENCODINGS",
+    "units",
+    "EditorBytes",
+    "text_lines",
+    "compiler_column",
+    "editor_character",
+]
+
+POSITION_ENCODINGS = ("utf-16", "utf-8", "utf-32")
+
+
+class EditorBytes(bytes):
+    """A document's compiler bytes (its editor `text` encoded with `errors="replace"`) that keep
+    the editor text they came from. Feature positions are mapped through that text when it is
+    present (`compiler_column` / `editor_character`), so a replaced character keeps its editor
+    width. `text` exists only in the process that encoded it: pickling (the analysis job sent to a
+    worker process) and copying give plain `bytes`, so a job carries each document's bytes once."""
+
+    text: str | None = None
+
+    def __new__(cls, data: bytes, text: str | None = None) -> EditorBytes:
+        out = super().__new__(cls, data)
+        out.text = text
+        return out
+
+    def __reduce__(self):
+        return bytes, (bytes(self),)
+
+
+def units(text: str, encoding: str) -> int:
+    """Length of `text` in position-encoding units: UTF-16 code units, UTF-8 bytes or code points."""
+    if encoding == "utf-16":
+        return len(text.encode("utf-16-le", errors="surrogatepass")) // 2
+    if encoding == "utf-8":
+        return len(text.encode("utf-8", errors="replace"))
+    if encoding == "utf-32":
+        return len(text)
+    raise ValueError(f"unsupported position encoding {encoding!r}")
+
+
+def text_lines(text: str) -> list[str]:
+    """Editor text split into physical lines exactly as `physical_lines` splits the bytes (CRLF
+    and LF), so line N here is line N of the compiler bytes."""
+    return text.replace("\r\n", "\n").split("\n")
+
+
+def compiler_column(line: str, character: int, *, encoding: str = "utf-16", file_encoding: str = "utf-8") -> int:
+    """Decoded (byte) column in the compiler bytes of editor `line` for `character` (position
+    encoding units). Each editor character is `len(ch.encode(file_encoding, "replace"))` bytes, so
+    this is exact for a replaced character too. A `character` inside an editor character snaps back
+    to its start; one past the line end clamps to the line's byte length."""
+    target = max(character, 0)
+    seen = col = 0
+    for ch in line:
+        step = units(ch, encoding)
+        if seen + step > target:
+            return col
+        seen += step
+        col += len(ch.encode(file_encoding, errors="replace"))
+        if seen == target:
+            return col
+    return col
+
+
+def editor_character(line: str, column: int, *, encoding: str = "utf-16", file_encoding: str = "utf-8") -> int:
+    """The inverse of `compiler_column`: decoded (byte) `column` of the compiler bytes of editor
+    `line` -> `character` in `encoding` units. A column inside an encoded character snaps back to
+    its start; one past the end clamps to the line's length in units."""
+    target = max(column, 0)
+    col = seen = 0
+    for ch in line:
+        size = len(ch.encode(file_encoding, errors="replace"))
+        if col + size > target:
+            return seen
+        col += size
+        seen += units(ch, encoding)
+        if col == target:
+            return seen
+    return seen
+
+
+LineMap = Mapping[str, Mapping[int, Sequence[LineSpan]]]
+
+
+class PhysicalRange(NamedTuple):
+    """`columns [start_col, end_col)` of physical line `line` (1-based) of `file`'s `occurrence`.
+    Columns are 0-based decoded (Latin-1) characters, i.e. byte offsets into the line."""
+
+    file: str
+    occurrence: int
+    line: int
+    start_col: int
+    end_col: int
+
+
+def reported_spans(line_map: LineMap | None, file: str, line: int) -> list[LineSpan]:
+    """Every physical span whose logical line the compiler reports at `line` of `file` (a display
+    name; the empty name is not resolved here). Several logical lines can share a reported line
+    (after initializer pulls) and a file loaded twice contributes one span per occurrence."""
+    if not line_map:
+        return []
+    by_line = line_map.get(file)
+    if by_line is None:
+        lowered = file.lower()
+        by_line = next((m for name, m in line_map.items() if name.lower() == lowered), None)
+    return list(by_line.get(line, ())) if by_line else []
+
+
+def line_span(line_map: LineMap | None, file: str, line: int) -> tuple[int, int] | None:
+    """The covering physical (first, last) line of `reported_spans`, or None when unknown."""
+    spans = reported_spans(line_map, file, line)
+    if not spans:
+        return None
+    return min(s.first for s in spans), max(s.last for s in spans)
+
+
+def logical_ranges(ll: LogicalLine, start: int, end: int) -> list[PhysicalRange]:
+    """Physical ranges for `ll.text[start:end]`, in order. A range that crosses a continuation
+    join, or a loader-inserted `" = "`/`" "`, comes back as several ranges (inserted characters
+    map to nothing); an empty `start == end` gives one empty range, or none when that offset is
+    inserted text. Characters with no piece are skipped."""
+    out: list[PhysicalRange] = []
+    for p in ll.pieces:
+        lo, hi = max(start, p.start), min(end, p.end)
+        if start == end:
+            if p.start <= start <= p.end:
+                return [PhysicalRange(p.file, p.occurrence, p.line, p.col + start - p.start, p.col + start - p.start)]
+            continue
+        if lo >= hi:
+            continue
+        piece = PhysicalRange(p.file, p.occurrence, p.line, p.col + lo - p.start, p.col + hi - p.start)
+        if out and out[-1][:3] == piece[:3] and out[-1].end_col == piece.start_col:
+            piece = piece._replace(start_col=out.pop().start_col)
+        out.append(piece)
+    return out
+
+
+def exact_range(ll: LogicalLine, start: int, end: int) -> PhysicalRange | None:
+    """`logical_ranges` when the whole of `ll.text[start:end]` is one contiguous physical slice
+    (a token from a single piece), else None."""
+    ranges = logical_ranges(ll, start, end)
+    if len(ranges) != 1:
+        return None
+    r = ranges[0]
+    return r if start == end or r.end_col - r.start_col == end - start else None
+
+
+def physical_lines(data: bytes) -> list[bytes]:
+    """`data` split into physical lines exactly as the loader does (CRLF and LF; a lone CR stays
+    inside its line, which an editor would split -- see the limits in `docs/lsp-api.md`)."""
+    return data.replace(b"\r\n", b"\n").split(b"\n")
+
+
+def lsp_position(
+    data: bytes | Sequence[bytes], line: int, col: int, *, encoding: str = "utf-16", editor_encoding: str = "utf-8"
+) -> tuple[int, int]:
+    """LSP `(line, character)` for decoded column `col` of physical `line` (1-based).
+
+    `data` is the file's bytes (or `physical_lines(data)`); `col` counts the compiler's Latin-1
+    characters, so it is a byte offset into that line. The prefix `line_bytes[:col]` is decoded with
+    `editor_encoding` (errors replaced; an incomplete trailing sequence is dropped, so a column
+    inside a multi-byte character snaps back to that character's start) and its length is counted
+    in `encoding` units: UTF-16 code units, UTF-8 bytes, or UTF-32 code points. Only ASCII-
+    compatible editor encodings are meaningful. A column past the line end is clamped to it."""
+    if encoding not in POSITION_ENCODINGS:
+        raise ValueError(f"unsupported position encoding {encoding!r}")
+    lines = physical_lines(data) if isinstance(data, (bytes, bytearray)) else data
+    if not 1 <= line <= len(lines):
+        raise IndexError(f"line {line} outside 1..{len(lines)}")
+    prefix = lines[line - 1][: max(col, 0)]
+    decoder = codecs.getincrementaldecoder(editor_encoding)(errors="replace")
+    text = decoder.decode(prefix, final=False)
+    if encoding == "utf-16":
+        return line - 1, len(text.encode("utf-16-le")) // 2
+    if encoding == "utf-8":
+        return line - 1, len(text.encode("utf-8", errors="replace"))
+    return line - 1, len(text)
+
+
+def lsp_range(
+    data: bytes | Sequence[bytes], r: PhysicalRange, *, encoding: str = "utf-16", editor_encoding: str = "utf-8"
+) -> tuple[tuple[int, int], tuple[int, int]]:
+    """`((line, character), (line, character))` for a `PhysicalRange` of the same file."""
+    lines = physical_lines(data) if isinstance(data, (bytes, bytearray)) else data
+    return (
+        lsp_position(lines, r.line, r.start_col, encoding=encoding, editor_encoding=editor_encoding),
+        lsp_position(lines, r.line, r.end_col, encoding=encoding, editor_encoding=editor_encoding),
+    )
+
+
+def _units(text: str, encoding: str) -> int:
+    if encoding == "utf-16":
+        return len(text.encode("utf-16-le")) // 2
+    if encoding == "utf-8":
+        return len(text.encode("utf-8", errors="replace"))
+    return len(text)
+
+
+def decoded_position(
+    data: bytes | Sequence[bytes],
+    line: int,
+    character: int,
+    *,
+    encoding: str = "utf-16",
+    editor_encoding: str = "utf-8",
+) -> tuple[int, int]:
+    """The inverse of `lsp_position`: LSP `(line, character)` (0-based line, `character` counted
+    in `encoding` units) -> `(line, col)` with `line` 1-based and `col` the decoded (Latin-1)
+    column, i.e. the byte offset into that physical line.
+
+    `character` on a character boundary gives the offset of that boundary, so
+    `decoded_position(data, *lsp_position(data, l, c)) == (l, c)` for every `c` on a boundary.
+    A `character` inside a character (the middle of a UTF-16 surrogate pair) snaps back to that
+    character's start, as `lsp_position` snaps a column; one past the line end is clamped to the
+    line length. A line outside the file raises `IndexError`; an unknown encoding `ValueError`."""
+    if encoding not in POSITION_ENCODINGS:
+        raise ValueError(f"unsupported position encoding {encoding!r}")
+    lines = physical_lines(data) if isinstance(data, (bytes, bytearray)) else data
+    if not 0 <= line < len(lines):
+        raise IndexError(f"line {line} outside 0..{len(lines) - 1}")
+    raw = lines[line]
+    target = max(character, 0)
+    if raw.isascii():
+        return line + 1, min(target, len(raw))
+    if target == 0:
+        return line + 1, 0
+    decoder = codecs.getincrementaldecoder(editor_encoding)(errors="replace")
+    units = start = 0
+    for col in range(len(raw)):
+        step = _units(decoder.decode(raw[col : col + 1], final=False), encoding)
+        if not step:
+            continue  # an incomplete multi-byte sequence so far
+        if units + step > target:
+            return line + 1, start  # inside a character: snap back to its start
+        units += step
+        start = col + 1
+        if units == target:
+            return line + 1, start
+    return line + 1, len(raw)

opencrbasic/preprocess.py

@@ -0,0 +1,269 @@
+"""`-p FILE` / `-i FILE`: CR6Comp's preprocessed-source output.
+
+Independent of the compiler's own compile pipeline (`source.py`/`parser.py`), which always
+comment-strips and fully expands `Include`/`IncludeSection` before symbol resolution: the exe's
+`-p`/`-i` text dump is a much shallower transformation of the raw file. Oracle-verified
+byte-for-byte on the repository's 16 programs and hand-written probes (`docs/cli.md`
+"Preprocessed output (-p / -i)"; this replaces that section's earlier, unverified draft, which
+wrongly assumed comments were stripped and includes expanded under `-p`):
+
+- comments are kept verbatim (trailing `'...` and whole-line) -- NOT stripped.
+- blank lines are kept.
+- a line continuation (an unquoted, uncommented `_` whose only preceding characters on the line
+  are whitespace) is resolved by deleting just the underscore and directly concatenating the
+  following physical line's raw text: no separator is inserted and the continuing line's own
+  leading whitespace is kept. A comment trailing the `_` on the same physical line is dropped
+  along with the marker (oracle-confirmed: unlike an ordinary trailing comment, it does not
+  survive into the output).
+- `#If`/`#ElseIf`/`#Else`/`#EndIf`/`#If_no_remove`/`#UnDef` directive lines are removed; only the
+  taken branch's own lines remain. Resolution uses the same Const environment as a real compile,
+  so a Const defined via an `IncludeSection()` -- itself left unexpanded in `-p`'s own output --
+  still resolves correctly (oracle-confirmed: `Main.CR6`'s `#If BatteryTempSensor > 0 Then` picks
+  the `#Else` branch because `BatteryTempSensor` comes from `StationConfig.crb`).
+- `-p`: a bare `Include(...)` or `IncludeSection(...)` statement is left as literal, unexpanded
+  text.
+- `-i`: both are expanded recursively, splicing in the target file's own preprocessed text in
+  place (oracle-confirmed on both forms).
+- output is CRLF throughout with one trailing CRLF, `latin-1`-decoded/encoded like `source.py`.
+
+This reuses `parser._Parser`'s directive-handling and Const-folding (`_handle_directive`,
+`_parse_const`, `_active`) so `#If`/Const resolution is the same oracle-verified engine the real
+front end uses for compilation, not a second implementation of CRBasic's operator precedence
+that could quietly drift from it. `parser.py` itself is not modified here, only called into.
+
+Not yet verified against the oracle (documented gaps, `docs/cli.md`):
+- a bare `Include(...)` nested inside an already-`-i`-expanded file (illegal for compilation,
+  `Include file within an Include file is illegal.`) falls back to literal text rather than
+  guessing at behaviour with no oracle sample.
+- a dangling continuation marker on a file's last physical line (nothing follows to join).
+"""
+
+from pathlib import Path
+
+from . import lexer, source
+from .diagnostics import Diagnostics
+from .parser import _DIRECTIVE_RE, _Parser, _split_text_segments, _TokStream
+from .profiles import Profile
+
+
+def _split_physical(raw: str) -> list[str]:
+    """Physical lines, without the extra empty trailing element `str.split("\\n")` produces for
+    input that (as is normal) ends with a line terminator -- unlike `source.py`'s own splitter,
+    which can leave that artifact in because its caller always skips blank logical lines anyway.
+    """
+    parts = raw.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+    if parts and parts[-1] == "":
+        parts.pop()
+    return parts
+
+
+def _is_continuation_raw(stripped: str) -> tuple[bool, str]:
+    """Like `source._is_continuation`, but keeps the whitespace before the `_` -- oracle-
+    confirmed the exe's own `-p`/`-i` join keeps it, where `source.py`'s own variant strips it
+    further (harmless there: it only feeds the lexer, which doesn't see whitespace).
+    """
+    t = stripped.rstrip(" \t")
+    if t.endswith("_") and (len(t) == 1 or t[-2] in " \t"):
+        return True, t[:-1]
+    return False, stripped
+
+
+class _Group:
+    """One continuation-joined logical line: `raw` is what `-p`/`-i` emits for it (verbatim
+    except for the continuation markers themselves); `eval` is the comment-stripped, single-
+    space-joined text used only to recognise directives/Const/Include, matching `source.py`'s own
+    joining rule so `#If`/Const evaluation agrees with a real compile.
+    """
+
+    __slots__ = ("raw", "eval", "line")
+
+    def __init__(self, raw: str, eval_text: str, line: int):
+        self.raw = raw
+        self.eval = eval_text
+        self.line = line
+
+
+def _read_groups(physical: list[str]) -> list[_Group]:
+    groups: list[_Group] = []
+    i = 0
+    n = len(physical)
+    while i < n:
+        raw_parts: list[str] = []
+        eval_text = source._strip_comment(physical[i])
+        while True:
+            cont, eval_prefix = source._is_continuation(eval_text)
+            if not cont or i + 1 >= n:
+                raw_parts.append(physical[i])
+                if cont:
+                    eval_text = eval_prefix
+                break
+            _, raw_prefix = _is_continuation_raw(source._strip_comment(physical[i]))
+            raw_parts.append(raw_prefix)
+            i += 1
+            eval_text = eval_prefix + " " + source._strip_comment(physical[i]).lstrip(" \t")
+        groups.append(_Group("".join(raw_parts), eval_text, i + 1))
+        i += 1
+    return groups
+
+
+def _fold_consts_and_check_endprog(eval_text: str, line: int, filename: str, parser_instance: _Parser) -> bool:
+    """Folds any `Const` segment into `parser_instance.env` and returns True the moment an
+    `EndProg` statement is seen. CR6Comp stops reading the file at the first top-level `EndProg`
+    (`parser.py`'s own docstring, oracle-confirmed there); the same is true of `-p`/`-i`'s own
+    text dump -- oracle-confirmed on the CHM corpus, several of whose examples concatenate a
+    second, unrelated program (or bare prose) after a first program's `EndProg`, none of which
+    appears in `-p`/`-i`'s output.
+    """
+    for seg in _split_text_segments(eval_text):
+        if not seg.strip():
+            continue
+        try:
+            tokens = lexer.tokenize(seg, line)
+        except lexer.LexError:
+            continue
+        real = [t for t in tokens if t.kind != "EOF"]
+        if not real or real[0].kind != "IDENT":
+            continue
+        if real[0].upper == "CONST":
+            try:
+                parser_instance._parse_const(_TokStream(tokens), line, filename)
+            except Exception:
+                pass
+        elif real[0].upper == "ENDPROG":
+            return True
+    return False
+
+
+def _walk_file(
+    groups: list[_Group],
+    filename: str,
+    section: str | None,
+    directory: Path | None,
+    parser_instance: _Parser,
+    expand_includes: bool,
+    in_include: bool,
+    seen: frozenset[str],
+    stop: dict | None = None,  # {"at": (file name lower, line), "hit": bool}
+) -> list[str]:
+    out: list[str] = []
+    for g in groups:
+        if parser_instance._done:
+            break
+        parser_instance.env.section = section
+        m = _DIRECTIVE_RE.match(g.eval)
+        if m:
+            kind = m.group(1).upper()
+            if kind == "IF_NO_REMOVE":
+                kind = "IF"
+            parser_instance._handle_directive(kind, m.group(2), g.line, filename)
+            continue
+        if not parser_instance._active():
+            continue
+        if stop is not None and stop["at"] == (filename.lower(), g.line):
+            # the reference stops preprocessing with the pass-1 error its message buffer cannot hold,
+            # at its first active occurrence (oracle, CR6, round 74, `docs/artifacts.md`)
+            out.append(g.raw)
+            parser_instance._done = stop["hit"] = True
+            break
+        if _fold_consts_and_check_endprog(g.eval, g.line, filename, parser_instance):
+            out.append(g.raw)
+            parser_instance._done = True
+            break
+        m_sec = source._INCLUDE_SECTION_RE.match(g.eval)
+        m_inc = None if m_sec else source._INCLUDE_RE.match(g.eval)
+        if (m_sec or m_inc) and not expand_includes:
+            # `-p` ends its output after an include whose file is missing (oracle, CR6, round 74,
+            # artifact matrix `fail_include`/`fail_section`)
+            spec = m_sec.group(2) if m_sec else m_inc.group(1) if m_inc else ""
+            out.append(g.raw)
+            target = source._resolve_include_path(spec, directory)
+            if target is None:
+                parser_instance._done = True
+                break
+            key = str(target.resolve())
+            if key not in seen and not (m_inc and in_include):
+                # not written, but its `Const`s still decide later `#If`s (`Main.CR6`'s
+                # `#If BatteryTempSensor > 0` with the Const in an included `StationConfig.crb`)
+                try:
+                    child = _read_groups(_split_physical(target.read_bytes().decode("latin-1")))
+                except OSError:
+                    continue
+                _walk_file(
+                    child,
+                    target.name,
+                    m_sec.group(1) if m_sec else None,
+                    target.parent,
+                    parser_instance,
+                    False,
+                    in_include or bool(m_inc),
+                    seen | {key},
+                    stop,
+                )
+                if stop is not None and stop["hit"]:
+                    break
+                parser_instance._done = False  # an `EndProg` in the include does not end this file
+            continue
+        if (m_sec or m_inc) and expand_includes:
+            if m_sec:
+                spec = m_sec.group(2)
+                new_section, new_in_include = m_sec.group(1), in_include
+            else:
+                if in_include:
+                    out.append(g.raw)
+                    continue
+                assert m_inc is not None  # (`m_sec or m_inc`)
+                spec = m_inc.group(1)
+                new_section, new_in_include = None, True
+            target = source._resolve_include_path(spec, directory)
+            if target is None:
+                out.append(g.raw)  # `-i` keeps a missing include as written (oracle, CR6, round 74)
+                continue
+            key = str(target.resolve())
+            if key in seen:
+                out.append(g.raw)
+                continue
+            try:
+                data = target.read_bytes()
+            except OSError:
+                continue
+            child_groups = _read_groups(_split_physical(data.decode("latin-1")))
+            out.extend(
+                _walk_file(
+                    child_groups,
+                    target.name,
+                    new_section,
+                    target.parent,
+                    parser_instance,
+                    expand_includes,
+                    new_in_include,
+                    seen | {key},
+                    stop,
+                )
+            )
+            if parser_instance._done:
+                break
+            continue
+        out.append(g.raw)
+    return out
+
+
+def preprocess(path: Path, profile: Profile, *, expand_includes: bool, stop: tuple[str, int] | None = None) -> bytes:
+    """The `-p` (`expand_includes=False`) or `-i` (`expand_includes=True`) output bytes for
+    `path`: CRLF, one trailing CRLF, `latin-1`. With `stop` (file name, line), the output ends with
+    that statement: where a dropped pass-1 error ends the reference compile."""
+    path = Path(path)
+    data = path.read_bytes()
+    groups = _read_groups(_split_physical(data.decode("latin-1")))
+    parser_instance = _Parser(profile, Diagnostics())
+    lines = _walk_file(
+        groups,
+        path.name,
+        None,
+        path.parent,
+        parser_instance,
+        expand_includes,
+        False,
+        frozenset({str(path.resolve())}),
+        None if stop is None else {"at": (stop[0].lower(), stop[1]), "hit": False},
+    )
+    return ("\r\n".join(lines) + "\r\n").encode("latin-1", "replace")