pyvexfile 2.0.0__py3-none-any.whl

pyvexfile/__init__.py

@@ -0,0 +1,35 @@
+"""pyvexfile: parser for VEX files that keeps the comments and layout of the original file.
+
+Usage
+-----
+
+from pyvexfile import Vex
+
+vex = Vex.from_file("experiment.vex")
+vex["STATION"]["Jb"]["$SITE"]                     # value of 'ref $SITE = ...;'
+vex["FREQ"]["4632MHz32x32MHz"].get_all("chan_def")
+vex["EXPER"]["N26C3"]["exper_name"] = "N26C4"
+problems = vex.verify()
+vex.to_file("new.vex")
+
+Unmodified parts of the file are written exactly as they were read, comments included.
+"""
+from importlib.metadata import PackageNotFoundError, version
+
+from .lexer import VexSyntaxError
+from .nodes import BlankLine, Block, Comment, Def, Literal, Node, Scan, Statement, Value, Vex
+from .observation import IF, Channel, Station
+from .parser import parse
+from .verify import Issue, Severity
+
+
+__all__: list[str] = [
+    "Vex", "Block", "Def", "Scan", "Statement", "Comment", "BlankLine", "Literal", "Node",
+    "Value", "Station", "Channel", "IF", "Issue", "Severity", "VexSyntaxError", "parse",
+    "__version__",
+]
+
+try:
+    __version__ = version("pyvexfile")
+except PackageNotFoundError:  # running from the source tree without installing it
+    __version__ = "unknown"

pyvexfile/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+
+sys.exit(main())

pyvexfile/cli.py

@@ -0,0 +1,282 @@
+"""Command line interface to inspect VEX files.
+
+    vex experiment.vex                 summary of the observation
+    vex experiment.vex --stations      list of stations
+    vex experiment.vex --if            IF setups of each mode
+    vex experiment.vex --scans         list of scans
+    vex experiment.vex --freq          frequencies observed by each station, with a plot
+    vex experiment.vex --verify        check the file (exit status 1 if it has errors)
+    vex experiment.vex --version       VEX version of the file
+
+Several options can be combined.
+"""
+import argparse
+import math
+import shutil
+import sys
+from collections import Counter
+from collections.abc import Callable, Sequence
+from datetime import timedelta
+from pathlib import Path
+
+from .lexer import VexSyntaxError
+from .nodes import Vex
+from .observation import IF, covered_ranges
+from .verify import Severity
+
+
+__all__: list[str] = ["main"]
+
+# Characters used in the frequency plot, depending on how much of each cell is covered
+_FULL, _PARTIAL, _EMPTY = "█", "▌", " "
+_MIN_PLOT_WIDTH = 20
+
+
+def _table(headers: Sequence[str], rows: Sequence[Sequence[object]], indent: str = "") -> str:
+    """Returns the rows as a text table with aligned columns."""
+    cells = [[str(cell) for cell in row] for row in rows]
+    widths = [max([len(header)] + [len(row[i]) for row in cells]) for i, header in enumerate(headers)]
+    lines = [headers, ["-" * width for width in widths]] + cells
+    return "\n".join(indent + "  ".join(cell.ljust(width) for cell, width in zip(line, widths)).rstrip()
+                     for line in lines)
+
+
+def _format_frequency(value: float) -> str:
+    return f"{value:.2f}".rstrip("0").rstrip(".")
+
+
+def _section(title: str, body: str) -> str:
+    return f"{title}\n{body}"
+
+
+def show_summary(vex: Vex) -> str:
+    scans = vex.scans
+    starts = [scan.start for scan in scans if scan.start is not None]
+    rows = [
+        ("Experiment", vex.experiment or "-"),
+        ("VEX version", vex.version or "-"),
+        ("Stations", f"{len(vex.stations)}: {' '.join(s.code for s in vex.stations)}"),
+        ("Modes", ", ".join(vex.modes) or "-"),
+        ("Sources", len(vex["SOURCE"]) if "SOURCE" in vex else 0),
+        ("Scans", len(scans)),
+    ]
+    if starts:
+        end = max(scan.start + timedelta(seconds=scan.duration or 0)
+                  for scan in scans if scan.start is not None)
+        rows.append(("Start (UTC)", f"{min(starts):%Y-%m-%d %H:%M:%S}"))
+        rows.append(("End (UTC)", f"{end:%Y-%m-%d %H:%M:%S}"))
+
+    width = max(len(name) for name, _ in rows)
+    return "\n".join(f"{name.ljust(width)}  {value}" for name, value in rows)
+
+
+def show_version(vex: Vex) -> str:
+    return f"VEX version: {vex.version or 'unknown (no VEX_rev statement)'}"
+
+
+def show_stations(vex: Vex) -> str:
+    rows = [(s.code, s.name, s.antenna or "-", s.das or "-") for s in vex.stations]
+    return _section(f"Stations ({len(rows)})", _table(["Code", "Name", "Antenna", "DAS"], rows))
+
+
+def show_scans(vex: Vex) -> str:
+    rows = []
+    for scan in vex.scans:
+        start = f"{scan.start:%Y-%m-%d %H:%M:%S}" if scan.start else "-"
+        duration = f"{scan.duration:.0f} s" if scan.duration is not None else "-"
+        rows.append((scan.name, start, duration, " ".join(scan.sources), scan.mode or "-",
+                     f"{len(scan.stations)}: {' '.join(scan.stations)}"))
+
+    headers = ["Scan", "Start (UTC)", "Duration", "Source", "Mode", "Stations"]
+    return _section(f"Scans ({len(rows)})", _table(headers, rows))
+
+
+def show_ifs(vex: Vex) -> str:
+    sections = []
+    for mode in vex.modes:
+        rows = []
+        for name, stations in vex.setups("IF", mode).items():
+            definition = vex["IF"].get(name) if "IF" in vex else None
+            ifs = IF.from_def(definition) if definition is not None else []
+            for i, an_if in enumerate(ifs or [None]):
+                setup = (name, " ".join(stations)) if i == 0 else ("", "")
+                if an_if is None:
+                    rows.append(setup + ("(missing in $IF)", "", "", "", ""))
+                else:
+                    rows.append(setup + (an_if.name, an_if.physical_name, an_if.polarization,
+                                         _format_frequency(an_if.lo), an_if.sideband))
+
+        headers = ["$IF setup", "Stations", "IF", "Physical", "Pol", "LO (MHz)", "Sideband"]
+        sections.append(_section(f"Mode {mode}", _table(headers, rows, indent="  ")))
+
+    return "\n\n".join(sections) if sections else "No modes defined in $MODE"
+
+
+def _tick_step(span: float, max_ticks: int) -> float:
+    """Returns a round step (1, 2 or 5 times a power of 10) giving at most max_ticks ticks in span."""
+    step = 10 ** math.floor(math.log10(span / max(max_ticks, 1))) if span > 0 else 1.0
+    while span / step > max_ticks:
+        for factor in (2, 2.5, 2):  # 1 -> 2 -> 5 -> 10
+            step *= factor
+            if span / step <= max_ticks:
+                break
+
+    return step
+
+
+def _ruler(low: float, high: float, width: int) -> tuple[str, str]:
+    """Returns the lines with the frequency labels and the ticks for a plot of the given width,
+    with ticks at round frequencies.
+    """
+    span = high - low
+    step = _tick_step(span, max(2, width // 12))
+    ticks = ["─"] * width
+    ticks[0], ticks[-1] = "├", "┤"
+    labels = [" "] * width
+    free_from = 0  # first column where a new label can start
+    value = math.ceil(low / step) * step
+    while value <= high + 1e-9:
+        position = round((value - low) / span * (width - 1)) if span > 0 else 0
+        if 0 < position < width - 1:
+            ticks[position] = "┼"
+
+        text = _format_frequency(value)
+        start = min(max(position - len(text) // 2, 0), width - len(text))
+        if start >= free_from:
+            labels[start:start + len(text)] = text
+            free_from = start + len(text) + 1
+
+        value += step
+
+    return "".join(labels).rstrip(), "".join(ticks)
+
+
+def _plot_row(ranges: list[tuple[float, float]], low: float, high: float, width: int) -> str:
+    cell = (high - low) / width
+    row = []
+    for i in range(width):
+        cell_low, cell_high = low + i * cell, low + (i + 1) * cell
+        covered = sum(max(0.0, min(high_, cell_high) - max(low_, cell_low)) for low_, high_ in ranges)
+        row.append(_FULL if covered >= 0.99 * cell else _PARTIAL if covered > 0 else _EMPTY)
+
+    return "".join(row)
+
+
+def show_frequencies(vex: Vex, width: int | None = None) -> str:
+    width = width or shutil.get_terminal_size((100, 24)).columns
+    sections = []
+    for mode in vex.modes:
+        stations = [code for codes in vex.setups("FREQ", mode).values() for code in codes]
+        stations = list(dict.fromkeys(stations))
+        channels = {code: vex.channels(mode, code) for code in stations}
+        rows = []
+        for code in stations:
+            setup = vex.setup("FREQ", mode, code)
+            chans = channels[code]
+            bandwidths = Counter(channel.bandwidth for channel in chans)
+            polarizations = sorted({c.polarization for c in chans if c.polarization}) or ["?"]
+            rows.append((
+                code,
+                setup.name if setup is not None else "(missing in $FREQ)",
+                ", ".join(f"{n}x{_format_frequency(bw)}" for bw, n in sorted(bandwidths.items())),
+                ", ".join(f"{_format_frequency(lo)}-{_format_frequency(hi)}"
+                          for lo, hi in covered_ranges(chans)),
+                " ".join(polarizations),
+            ))
+
+        headers = ["Station", "$FREQ setup", "Channels (MHz)", "Covered (MHz)", "Pol"]
+        body = [_table(headers, rows, indent="  ")]
+        all_channels = [channel for chans in channels.values() for channel in chans]
+        if all_channels:
+            low = min(channel.low for channel in all_channels)
+            high = max(channel.high for channel in all_channels)
+            label_width = max(len(code) for code in stations)
+            plot_width = max(_MIN_PLOT_WIDTH, width - label_width - 6)
+            labels, ticks = _ruler(low, high, plot_width)
+            pad = "  " + " " * label_width + " "
+            body.append("")
+            body.append(f"{pad} Frequency coverage (MHz)")
+            body.append(f"{pad} {labels}")
+            body.append(f"{pad} {ticks}")
+            for code in stations:
+                row = _plot_row(covered_ranges(channels[code]), low, high, plot_width)
+                body.append(f"  {code.ljust(label_width)} │{row}│")
+
+            body.append(f"{pad} {ticks}")
+
+        sections.append(_section(f"Mode {mode}", "\n".join(body)))
+
+    return "\n\n".join(sections) if sections else "No modes defined in $MODE"
+
+
+def show_verify(vex: Vex, path: Path, freq_overlap: bool = True) -> tuple[str, bool]:
+    """Returns the problems found in the file, and whether any of them is an error."""
+    issues = vex.verify(freq_overlap=freq_overlap)
+    if not issues:
+        return f"{path}: no problems found", False
+
+    lines = [f"{path}:{'' if issue.line is None else f'{issue.line}:'} {issue.severity}: {issue.message}"
+             for issue in issues]
+    n_errors = sum(issue.severity is Severity.ERROR for issue in issues)
+    n_warnings = len(issues) - n_errors
+    lines.append(f"{n_errors} error{'s' if n_errors != 1 else ''}, "
+                 f"{n_warnings} warning{'s' if n_warnings != 1 else ''}")
+    return "\n".join(lines), n_errors > 0
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="vex",
+        description="Inspect a VEX file. Without options, shows a summary of the observation.",
+    )
+    parser.add_argument("vexfile", type=Path, help="the VEX file")
+    parser.add_argument("--version", action="store_true", help="show the VEX version of the file")
+    parser.add_argument("--stations", action="store_true", help="list the stations")
+    parser.add_argument("--if", dest="ifs", action="store_true", help="list the IF setups of each mode")
+    parser.add_argument("--scans", action="store_true", help="list the scans")
+    parser.add_argument("--freq", action="store_true",
+                        help="show the frequencies observed by each station, with a plot")
+    parser.add_argument("--verify", action="store_true", help="check that the VEX file is correct")
+    parser.add_argument("--no-freq-overlap", dest="freq_overlap", action="store_false",
+                        help="with --verify, do not check that each station shares frequencies "
+                             "with the other stations in its scans")
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    try:
+        vex = Vex.from_file(args.vexfile)
+    except OSError as error:
+        print(f"vex: cannot read {args.vexfile}: {error.strerror or error}", file=sys.stderr)
+        return 1
+    except VexSyntaxError as error:
+        print(f"{args.vexfile}:{error.line}:{error.col}: syntax error: {error.message}", file=sys.stderr)
+        return 1
+
+    has_errors = False
+
+    def verification() -> str:
+        nonlocal has_errors
+        text, has_errors = show_verify(vex, args.vexfile, args.freq_overlap)
+        return text
+
+    views: list[tuple[bool, Callable[[], str]]] = [
+        (args.version, lambda: show_version(vex)),
+        (args.verify, verification),
+        (args.stations, lambda: show_stations(vex)),
+        (args.ifs, lambda: show_ifs(vex)),
+        (args.freq, lambda: show_frequencies(vex)),
+        (args.scans, lambda: show_scans(vex)),
+    ]
+    outputs = [view() for selected, view in views if selected] or [show_summary(vex)]
+    try:
+        print("\n\n".join(outputs))
+    except BrokenPipeError:  # e.g. when piped to 'head'
+        sys.stderr.close()
+
+    return 1 if has_errors else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pyvexfile/lexer.py

@@ -0,0 +1,228 @@
+"""Lexical analysis of VEX files.
+
+The tokenizer is lossless: joining the values of all the tokens produced by
+tokenize() gives back the input text exactly, including comments, whitespace
+and line breaks. This allows the parser to keep the comments and the original
+layout of a VEX file, so it can be written again without changes.
+
+Usage
+-----
+
+from pyvexfile.lexer import tokenize
+
+for token in tokenize(text):
+    print(token.type, repr(token.value), token.line, token.col)
+"""
+from collections.abc import Iterator
+from dataclasses import dataclass
+from enum import Enum, auto
+
+
+__all__: list[str] = ["TokenType", "Token", "VexSyntaxError", "tokenize", "TRIVIA"]
+
+
+class TokenType(Enum):
+    """Types of the tokens that can be found in a VEX file:
+    - WORD. Any sequence of characters not containing whitespace, ';', ':', '=', '$', '&' or '*'.
+    - STRING. A quoted string, including the quotes ("...").
+    - SEMICOLON, COLON, EQUALS, DOLLAR, AMPERSAND. The single-character punctuation.
+    - COMMENT. Everything from a '*' until the end of the line (the line break not included).
+    - WHITESPACE. A sequence of spaces and/or tabs.
+    - NEWLINE. A line break ('\\n' or '\\r\\n').
+    - LITERAL. A full literal block, from 'start_literal(...)' until 'end_literal(...)',
+      both included. The final ';' after 'end_literal(...)' is a separated SEMICOLON token.
+    """
+
+    WORD = auto()
+    STRING = auto()
+    SEMICOLON = auto()
+    COLON = auto()
+    EQUALS = auto()
+    DOLLAR = auto()
+    AMPERSAND = auto()
+    COMMENT = auto()
+    WHITESPACE = auto()
+    NEWLINE = auto()
+    LITERAL = auto()
+
+
+# Tokens that do not carry syntactic meaning, but must be kept to reproduce the file
+TRIVIA: frozenset[TokenType] = frozenset(
+    {TokenType.COMMENT, TokenType.WHITESPACE, TokenType.NEWLINE}
+)
+
+_PUNCTUATION: dict[str, TokenType] = {
+    ";": TokenType.SEMICOLON,
+    ":": TokenType.COLON,
+    "=": TokenType.EQUALS,
+    "$": TokenType.DOLLAR,
+    "&": TokenType.AMPERSAND,
+}
+_WHITESPACE = " \t\f\v"
+# Note that '"' is not a delimiter: it only opens a string at the start of a token,
+# so values like  28d48'04.938910"  are read as a single word.
+_WORD_DELIMITERS = frozenset(_WHITESPACE + "\r\n;:=$&*")
+_LITERAL_START = "start_literal"
+_LITERAL_END = "end_literal"
+
+
+class VexSyntaxError(ValueError):
+    """Error in the syntax of a VEX file, at the given line and column (both starting at 1)."""
+
+    def __init__(self, message: str, line: int, col: int):
+        super().__init__(f"line {line}, column {col}: {message}")
+        self.message = message
+        self.line = line
+        self.col = col
+
+
+@dataclass(frozen=True, slots=True)
+class Token:
+    """A token from a VEX file. 'offset' is the position of its first character in the
+    input text, and 'line' and 'col' its line and column numbers (starting at 1).
+    """
+
+    type: TokenType
+    value: str
+    line: int
+    col: int
+    offset: int
+
+
+def tokenize(text: str) -> Iterator[Token]:
+    """Splits the text of a VEX file into Tokens. Raises VexSyntaxError if a string or
+    a literal block is not terminated.
+    """
+    pos, line, col = 0, 1, 1
+    while pos < len(text):
+        token_type, end = _scan_token(text, pos, line, col)
+        value = text[pos:end]
+        yield Token(token_type, value, line, col, pos)
+        if (n_lines := value.count("\n")) > 0:
+            line += n_lines
+            col = len(value) - value.rfind("\n")
+        else:
+            col += len(value)
+
+        pos = end
+
+
+def _scan_token(text: str, pos: int, line: int, col: int) -> tuple[TokenType, int]:
+    """Identifies the token starting at text[pos]. Returns its type and the position
+    right after its last character.
+    """
+    char = text[pos]
+    if char == "\n":
+        return TokenType.NEWLINE, pos + 1
+
+    if char == "\r":
+        if text.startswith("\r\n", pos):
+            return TokenType.NEWLINE, pos + 2
+
+        return TokenType.WHITESPACE, pos + 1
+
+    if char in _WHITESPACE:
+        end = pos + 1
+        while end < len(text) and text[end] in _WHITESPACE:
+            end += 1
+
+        return TokenType.WHITESPACE, end
+
+    if char == "*":
+        end = text.find("\n", pos)
+        if end == -1:
+            end = len(text)
+        elif text[end - 1] == "\r":
+            end -= 1
+
+        return TokenType.COMMENT, end
+
+    if char == '"':
+        return TokenType.STRING, _scan_string(text, pos, line, col)
+
+    if char in _PUNCTUATION:
+        return _PUNCTUATION[char], pos + 1
+
+    if (literal_end := _scan_literal(text, pos, line, col)) is not None:
+        return TokenType.LITERAL, literal_end
+
+    end = pos + 1
+    while end < len(text) and text[end] not in _WORD_DELIMITERS:
+        end += 1
+
+    return TokenType.WORD, end
+
+
+def _scan_string(text: str, pos: int, line: int, col: int) -> int:
+    """Returns the position right after the closing quote of the string opened at text[pos].
+    A backslash escapes the following character. Strings cannot span multiple lines.
+    """
+    end = pos + 1
+    while end < len(text):
+        char = text[end]
+        if char == "\\":
+            end += 2
+        elif char == '"':
+            return end + 1
+        elif char in "\r\n":
+            break
+        else:
+            end += 1
+
+    raise VexSyntaxError("unterminated string", line, col)
+
+
+def _scan_literal(text: str, pos: int, line: int, col: int) -> int | None:
+    """If a literal block starts at text[pos], returns the position right after the
+    'end_literal(label)' that closes it. Otherwise returns None.
+
+    A literal block has the form:
+        start_literal(label);
+        ...any text, kept verbatim...
+        end_literal(label);
+    where the label is optional, but must be the same in both lines.
+    """
+    label_start = _open_parenthesis(text, pos, _LITERAL_START)
+    if label_start is None:
+        return None
+
+    label_end = text.find(")", label_start)
+    if label_end == -1:
+        raise VexSyntaxError(f"'{_LITERAL_START}(' without closing ')'", line, col)
+
+    label = text[label_start:label_end].strip()
+    search_from = label_end + 1
+    while (end_pos := text.find(_LITERAL_END, search_from)) != -1:
+        end_label_start = _open_parenthesis(text, end_pos, _LITERAL_END)
+        if end_label_start is not None:
+            end_label_end = text.find(")", end_label_start)
+            if end_label_end == -1:
+                break
+
+            if text[end_label_start:end_label_end].strip() == label:
+                return end_label_end + 1
+
+        search_from = end_pos + len(_LITERAL_END)
+
+    raise VexSyntaxError(
+        f"literal block '{_LITERAL_START}({label})' without matching '{_LITERAL_END}({label})'",
+        line,
+        col,
+    )
+
+
+def _open_parenthesis(text: str, pos: int, keyword: str) -> int | None:
+    """If text[pos:] starts with keyword followed by '(' (optionally with whitespace in between),
+    returns the position right after the '('. Otherwise returns None.
+    """
+    if not text.startswith(keyword, pos):
+        return None
+
+    after = pos + len(keyword)
+    while after < len(text) and text[after] in _WHITESPACE:
+        after += 1
+
+    if after < len(text) and text[after] == "(":
+        return after + 1
+
+    return None