oaknut-basic 12.10.1__tar.gz → 12.12.0__tar.gz

{oaknut_basic-12.10.1/src/oaknut_basic.egg-info → oaknut_basic-12.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oaknut-basic
-Version: 12.10.1
+Version: 12.12.0
 Summary: BBC BASIC tools: program tokeniser/de-tokeniser and PRINT#/INPUT# data-file reader/writer
 Author-email: Robert Smallshire <robert@smallshire.org.uk>
 License-Expression: MIT

{oaknut_basic-12.10.1 → oaknut_basic-12.12.0}/src/oaknut/basic/__init__.py

@@ -30,7 +30,9 @@ from oaknut.basic.datafile import (
     BbcBasicDataReader,
     BbcBasicDataWriter,
 )
+from oaknut.basic.detect import Detection, Verdict, detect
 from oaknut.basic.detokeniser import detokenise, detokenise_body
+from oaknut.basic.dialect import BASIC_II, BASIC_V, Dialect
 from oaknut.basic.exceptions import (
     AlreadyNumberedError,
     BASICError,
@@ -79,7 +81,7 @@ from oaknut.basic.tokens import (
     TOKEN_TO_KEYWORD,
 )
 
-__version__ = "12.10.1"
+__version__ = "12.12.0"
 
 # Canonical load addresses for BBC BASIC programs on each host.
 # Programs saved by *SAVE on a real machine use these by default.
@@ -87,6 +89,8 @@ BBC_BASIC_LOAD_ADDRESS = 0x1900
 ELECTRON_BASIC_LOAD_ADDRESS = 0x0E00
 
 __all__ = [
+    "BASIC_II",
+    "BASIC_V",
     "BBC_BASIC_LOAD_ADDRESS",
     "DEFAULT_LINE_NUMBER",
     "DEFAULT_LINE_STEP",
@@ -109,7 +113,9 @@ __all__ = [
     "BbcBasicDataWriter",
     "DataFileError",
     "DataFileTypeMismatchError",
+    "Detection",
     "DetokeniseError",
+    "Dialect",
     "Float5RangeError",
     "IntegerRangeError",
     "InvalidLineLengthError",
@@ -126,7 +132,9 @@ __all__ = [
     "TruncatedRecordError",
     "UnknownTagError",
     "UnnumberedLineError",
+    "Verdict",
     "decode_line_number",
+    "detect",
     "detokenise",
     "detokenise_body",
     "number_lines",

{oaknut_basic-12.10.1 → oaknut_basic-12.12.0}/src/oaknut/basic/cli.py

@@ -260,7 +260,16 @@ def tokenise(
     help='Text encoding for the OUTPUT source. Use "acorn" for the BBC '
     "character set with CR line endings (e.g. writing back to a disc image).",
 )
-def detokenise(input_stream, output_stream, encoding: str) -> None:
+@click.option(
+    "--dialect",
+    type=click.Choice(["ii", "2", "v", "5"]),
+    default="ii",
+    show_default=True,
+    help='BBC BASIC dialect, as a Roman or Arabic numeral. Use "v" (or "5") for '
+    "Archimedes / RISC OS programs, so the &C6/&C7/&C8 escape tokens "
+    "(CASE, SYS, ORIGIN, ...) decode correctly.",
+)
+def detokenise(input_stream, output_stream, encoding: str, dialect: str) -> None:
     """De-tokenise a stored BBC BASIC program into source text.
 
     Reads a tokenised program from INPUT and writes numbered source text
@@ -273,13 +282,72 @@ def detokenise(input_stream, output_stream, encoding: str) -> None:
     OUTPUT is written in --encoding (``utf-8`` with host-native ``LF`` line
     endings by default, for a host text file; pass ``acorn`` for the BBC
     character set with ``CR`` endings, e.g. writing back to a disc image).
+
+    Pass ``--dialect v`` (or ``--dialect 5``) for Archimedes / RISC OS
+    programs (BBC BASIC V), whose extended keywords use the &C6/&C7/&C8
+    two-byte escape tokens. The Roman (``ii``/``v``) and Arabic
+    (``2``/``5``) numerals are interchangeable.
     """
+    from oaknut.basic import BASIC_II, BASIC_V
     from oaknut.basic import detokenise as detokenise_program
 
-    listing = detokenise_program(input_stream.read())
+    selected_dialect = BASIC_V if dialect in ("v", "5") else BASIC_II
+    listing = detokenise_program(input_stream.read(), dialect=selected_dialect)
     output_stream.write(_listing_to_bytes(listing, encoding))
 
 
+@cli.command()
+@click.argument(
+    "inputs",
+    metavar="[FILE]...",
+    type=click.File("rb"),
+    nargs=-1,
+)
+@click.option(
+    "--quiet",
+    "-q",
+    is_flag=True,
+    help="Suppress per-file output; report the verdict through the exit code only.",
+)
+@click.pass_context
+def detect(ctx: click.Context, inputs, quiet: bool) -> None:
+    """Identify whether each FILE is a tokenised BBC BASIC program.
+
+    Classifies each FILE by its byte structure alone — the length-driven
+    line walk the BBC BASIC ROM performs to ``LIST`` a program — without
+    de-tokenising, so unlabelled files harvested from disc images can be
+    filtered. With no FILE, or ``-``, reads a single blob from standard
+    input ::
+
+        oaknut-basic detect *
+        disc get game.ssd MENU - | oaknut-basic detect
+
+    Each line reports a ``BASIC`` flag (blank when not), the verdict, the
+    path and the reason; non-fatal observations follow as indented notes.
+    The exit code is non-zero if any input is not a BASIC program, so it
+    composes as a shell filter. The verdict is one of ``basic`` (a clean
+    program), ``basic+`` (a program with data appended), ``maybe`` (begins
+    as BASIC, then the structure breaks) or ``not-basic``.
+    """
+    from oaknut.basic import detect as detect_blob
+
+    streams = inputs if inputs else (click.get_binary_stream("stdin"),)
+    all_basic = True
+    for stream in streams:
+        result = detect_blob(stream.read())
+        all_basic = all_basic and result.is_basic
+        if quiet:
+            continue
+        name = getattr(stream, "name", "<stdin>")
+        flag = "BASIC" if result.is_basic else "     "
+        click.echo(f"{flag} {result.verdict.value:9} {name}  -- {result.reason}")
+        for note in result.notes:
+            click.echo(f"          note: {note}")
+
+    if not all_basic:
+        ctx.exit(1)
+
+
 @cli.group()
 def data() -> None:
     """Read and write BBC BASIC data files (PRINT#/INPUT#/BPUT#/BGET#).

oaknut_basic-12.12.0/src/oaknut/basic/detect.py

@@ -0,0 +1,212 @@
+"""Identify tokenised BBC BASIC programs by structure alone.
+
+Files harvested from Acorn disc images (DFS ``.ssd``, ADFS ``.adl``)
+carry no filetype or extension, so a blob's nature has to be inferred
+from its bytes. A stored BBC BASIC program — BASIC I, II, IV and V all
+share the same line framing — is a run of lines::
+
+    &0D  <line-hi>  <line-lo>  <len>   <len-4 tokenised body bytes>
+
+repeated, then a terminator: ``&0D`` followed by a byte with its top bit
+set. BASIC writes ``&0D &FF``, but the ROM only tests the top bit, and
+programs with data appended sometimes tamper with the second byte, so
+``&0D &80``-``&FF`` all terminate — :func:`detect` matches the ROM, not
+the literal ``&FF``, and merely notes a non-``&FF`` terminator.
+
+The walk is length-driven: each line's own ``<len>`` byte says where the
+next ``&0D`` must be, so tokenised body bytes that happen to look like
+``&0D`` never trip the scan. It is exactly what the ROM does to ``LIST``
+a program, which makes it about as faithful a classifier as is possible
+without a full de-tokenise. Because the framing is identical across all
+BASIC versions, detection is purely structural and needs no
+:class:`~oaknut.basic.Dialect` — it sits well below the token table.
+
+Unlike :func:`~oaknut.basic.scan_program`, which raises
+:class:`~oaknut.basic.DetokeniseError` on malformed input, :func:`detect`
+is **total**: it classifies clean programs, program-plus-data, truncated
+fragments and arbitrary garbage into a :class:`Verdict` with evidence,
+and never raises. Classification is deliberately conservative and
+reason-bearing so a corpus filter can explain every accept and reject.
+"""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass, field
+
+from oaknut.basic.tokens import HEADER_LENGTH
+
+_CR = 0x0D
+
+# A stored line number is 0-32767 (see oaknut.basic.linenumber.MAX_LINE_NUMBER),
+# so its high byte is always <= &7F — top bit clear. The end-of-program
+# terminator's second byte has the top bit set (&FF, or a tampered &80-&FE).
+# Testing the top bit therefore tells a terminator from a line-number high
+# byte, exactly as the ROM's LIST loop does.
+_TERMINATOR_TOP_BIT = 0x80
+
+
+class Verdict(enum.Enum):
+    """How confident :func:`detect` is that a blob is tokenised BBC BASIC."""
+
+    #: A clean walk all the way to a proper terminator, no trailing bytes.
+    BASIC = "basic"
+    #: A valid program followed by extra bytes (data appended after the end).
+    BASIC_TRAILING = "basic+"
+    #: Begins as one or more well-formed lines, then the structure breaks.
+    MAYBE = "maybe"
+    #: Does not begin as a tokenised line at all.
+    NOT_BASIC = "not-basic"
+
+    @property
+    def is_basic(self) -> bool:
+        """True for the verdicts a corpus filter would extract as BASIC."""
+        return self in (Verdict.BASIC, Verdict.BASIC_TRAILING)
+
+
+@dataclass(frozen=True)
+class Detection:
+    """The outcome of inspecting one blob, with the evidence behind it.
+
+    Attributes:
+        verdict: The :class:`Verdict` reached.
+        reason: A human-readable explanation of the decision.
+        line_count: Well-formed lines walked before the terminator or break.
+        program_length: Bytes up to and including the terminator (0 if none
+            was reached).
+        trailing_length: Bytes after the terminator (non-zero only for
+            :attr:`Verdict.BASIC_TRAILING`).
+        first_line: The first line's number, or ``None`` if no line parsed.
+        last_line: The last line's number, or ``None`` if no line parsed.
+        ascending: Whether the line numbers were strictly ascending.
+        notes: Non-fatal observations (e.g. a non-``&FF`` terminator).
+    """
+
+    verdict: Verdict
+    reason: str
+    line_count: int = 0
+    program_length: int = 0
+    trailing_length: int = 0
+    first_line: int | None = None
+    last_line: int | None = None
+    ascending: bool = True
+    notes: tuple[str, ...] = field(default_factory=tuple)
+
+    @property
+    def is_basic(self) -> bool:
+        """True when the verdict is :attr:`Verdict.BASIC` or ``BASIC_TRAILING``."""
+        return self.verdict.is_basic
+
+
+def detect(data: bytes | bytearray) -> Detection:
+    """Classify *data* as tokenised BBC BASIC (or not) by walking its lines.
+
+    Args:
+        data: The raw bytes to classify.
+
+    Returns:
+        A :class:`Detection` carrying the :class:`Verdict` and the
+        evidence gathered during the walk. Always returns; never raises.
+    """
+    n = len(data)
+    if n < 2:
+        return Detection(Verdict.NOT_BASIC, "too short to be a BASIC line")
+    if data[0] != _CR:
+        return Detection(
+            Verdict.NOT_BASIC,
+            f"first byte is &{data[0]:02X}, not a &0D line marker",
+        )
+
+    pos = 0
+    line_count = 0
+    first_line: int | None = None
+    last_line: int | None = None
+    prev_line: int | None = None
+    ascending = True
+    notes: list[str] = []
+
+    def broke(reason: str) -> Detection:
+        """A broken walk is MAYBE if it parsed >=1 clean line, else NOT_BASIC.
+
+        One or more well-formed lines before the break means the blob
+        really does begin as a tokenised program (truncated, or BASIC
+        followed by data without a proper terminator); zero clean lines
+        means the leading ``&0D`` was a coincidence.
+        """
+        return Detection(
+            Verdict.MAYBE if line_count > 0 else Verdict.NOT_BASIC,
+            reason,
+            line_count=line_count,
+            first_line=first_line,
+            last_line=last_line,
+            ascending=ascending,
+            notes=tuple(notes),
+        )
+
+    while True:
+        # Every iteration must land on a line marker.
+        if pos >= n or data[pos] != _CR:
+            found = f"&{data[pos]:02X}" if pos < n else "end of data"
+            return broke(f"expected &0D at offset {pos}, found {found}")
+
+        # The byte after &0D is either a terminator (top bit set) or a
+        # line-number high byte.
+        if pos + 1 >= n:
+            return broke(f"truncated after &0D at offset {pos}")
+
+        marker = data[pos + 1]
+        if marker & _TERMINATOR_TOP_BIT:
+            # A terminator before any line parsed means the leading &0D was
+            # a coincidence (e.g. a View document opening &0D &80), not an
+            # empty program — real type-ins always have at least one line.
+            if line_count == 0:
+                return Detection(
+                    Verdict.NOT_BASIC,
+                    f"&0D &{marker:02X} at offset 0 — terminator with no lines, "
+                    "not a program",
+                )
+            program_length = pos + 2
+            trailing = n - program_length
+            if marker != 0xFF:
+                notes.append(
+                    f"terminator second byte &{marker:02X} (not &FF, top bit "
+                    "only) — data may follow"
+                )
+            verdict = Verdict.BASIC if trailing == 0 else Verdict.BASIC_TRAILING
+            reason = f"walked {line_count} line(s) to a terminator" + (
+                f", {trailing} trailing byte(s)" if trailing else ""
+            )
+            return Detection(
+                verdict=verdict,
+                reason=reason,
+                line_count=line_count,
+                program_length=program_length,
+                trailing_length=trailing,
+                first_line=first_line,
+                last_line=last_line,
+                ascending=ascending,
+                notes=tuple(notes),
+            )
+
+        # A real line: need the full 4-byte header.
+        if pos + HEADER_LENGTH > n:
+            return broke(f"truncated line header at offset {pos}")
+
+        line_no = (marker << 8) | data[pos + 2]
+        length = data[pos + 3]
+        if length < HEADER_LENGTH:
+            return broke(
+                f"line {line_no} at offset {pos} has length {length} "
+                f"(< {HEADER_LENGTH})"
+            )
+        if pos + length > n:
+            return broke(f"line {line_no} at offset {pos} runs {length} bytes past end of data")
+
+        if first_line is None:
+            first_line = line_no
+        if prev_line is not None and line_no <= prev_line:
+            ascending = False
+        prev_line = line_no
+        last_line = line_no
+        line_count += 1
+        pos += length

{oaknut_basic-12.10.1 → oaknut_basic-12.12.0}/src/oaknut/basic/detokeniser.py

@@ -26,15 +26,21 @@ including) the next one, so ``length = 4 + len(body)``.
 
 from __future__ import annotations
 
+from oaknut.basic.dialect import BASIC_II, Dialect
 from oaknut.basic.scanner import scan, scan_program
 
 
-def detokenise(data: bytes) -> str:
-    """De-tokenise a BBC BASIC II program into source text.
+def detokenise(data: bytes, *, dialect: Dialect = BASIC_II) -> str:
+    """De-tokenise a stored BBC BASIC program into source text.
 
     Args:
         data: A tokenised program, as stored on disc or in memory,
             terminated by the ``&0D &FF`` end marker.
+        dialect: The BBC BASIC variant whose token tables to use.
+            Defaults to :data:`~oaknut.basic.BASIC_II`; pass
+            :data:`~oaknut.basic.BASIC_V` for Archimedes / RISC OS
+            programs so the ``&C6``/``&C7``/``&C8`` escape tokens and the
+            re-purposed single-byte tokens decode correctly.
 
     Returns:
         The program as numbered source text, lines separated by ``"\\n"``
@@ -47,11 +53,11 @@ def detokenise(data: bytes) -> str:
     """
     return "".join(
         f"{record.line_number}{''.join(token.value for token in record.tokens)}\n"
-        for record in scan_program(data)
+        for record in scan_program(data, dialect=dialect)
     )
 
 
-def detokenise_body(data: bytes) -> str:
+def detokenise_body(data: bytes, *, dialect: Dialect = BASIC_II) -> str:
     """De-tokenise an unframed line body into text.
 
     Unlike :func:`detokenise`, this expects a line *body* — inline token
@@ -62,6 +68,9 @@ def detokenise_body(data: bytes) -> str:
 
     Args:
         data: A line body of inline token bytes.
+        dialect: The BBC BASIC variant whose token tables to use, passed
+            through to :func:`scan`. Defaults to
+            :data:`~oaknut.basic.BASIC_II`.
 
     Returns:
         The body as source text, using latin-1/code-point semantics so it
@@ -70,4 +79,4 @@ def detokenise_body(data: bytes) -> str:
     Raises:
         DetokeniseError: A ``&8D`` line-number reference is truncated.
     """
-    return "".join(token.value for token in scan(data))
+    return "".join(token.value for token in scan(data, dialect=dialect))

oaknut_basic-12.12.0/src/oaknut/basic/dialect.py

@@ -0,0 +1,164 @@
+"""BBC BASIC dialects for de-tokenisation.
+
+The :mod:`~oaknut.basic.tokens` table is **BBC BASIC II** — the BBC
+Micro's 8-bit language ROM. BBC BASIC V (the Archimedes / RISC OS ARM
+BASIC) keeps that table but extends it two ways, and de-tokenising a
+BASIC V program with the BASIC II table corrupts every extended keyword:
+
+- Three BASIC II command tokens — ``&C6``, ``&C7``, ``&C8`` (``AUTO``,
+  ``DELETE``, ``LOAD``) — become **two-byte escape prefixes**. The byte
+  that follows selects an extended keyword, with the second byte counting
+  from ``&8E`` in each table: ``&C6`` for the extended *functions*
+  (``SUM``, ``BEAT``), ``&C7`` for the extended *commands* (``APPEND`` …
+  ``RENUMBER`` … ``INSTALL``), and ``&C8`` for the extended *statements*
+  (``CASE``, ``CIRCLE``, ``ORIGIN``, ``SYS`` …).
+
+- Several single-byte slots are re-purposed. ``&7F`` becomes
+  ``OTHERWISE``, and ``&C9``-``&CE`` — ``LIST``/``NEW``/``OLD``/
+  ``RENUMBER``/``SAVE`` and an unused gap in BASIC II — become the block
+  keywords ``WHEN``/``OF``/``ENDCASE``/``ELSE``/``ENDIF``/``ENDWHILE``
+  (the command spellings move into the ``&C7`` escape table).
+
+A :class:`Dialect` bundles the single-byte token map with the escape
+tables, so the scanner and de-tokeniser can target either language with
+one parameter. :data:`BASIC_II` is the default everywhere — code that
+does not name a dialect keeps its existing BASIC II behaviour.
+
+The token values were cross-checked against two independent
+implementations: Justin Fletcher's de-tokeniser tables (the
+``gerph/riscos-basic-detokenise`` ``c/basic`` source) and Steve Fryatt's
+``tokenize`` (``src/parse.c``), which agree byte-for-byte.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+
+from oaknut.basic.tokens import TOKEN_TO_KEYWORD
+
+
+@dataclass(frozen=True)
+class Dialect:
+    """A BBC BASIC variant's token-to-keyword mapping.
+
+    Attributes:
+        name: Human-readable dialect name (e.g. ``"BBC BASIC V"``).
+        single_byte: Maps a single token byte to its keyword spelling.
+            Covers the whole one-byte keyword range, including the few
+            sub-``&80`` tokens such as ``&7F`` (``OTHERWISE``).
+        escape: Maps a two-byte escape *prefix* (``&C6``/``&C7``/``&C8``
+            in BASIC V) to a table from the *following* byte to its
+            extended keyword. Empty for dialects without escape tokens.
+    """
+
+    name: str
+    single_byte: Mapping[int, str]
+    escape: Mapping[int, Mapping[int, str]]
+
+
+# --- BBC BASIC II -----------------------------------------------------------
+
+#: BBC BASIC II — the BBC Micro's 8-bit language ROM. No escape tokens;
+#: the single-byte map is the canonical :data:`TOKEN_TO_KEYWORD` table.
+BASIC_II = Dialect(name="BBC BASIC II", single_byte=TOKEN_TO_KEYWORD, escape=MappingProxyType({}))
+
+
+# --- BBC BASIC V ------------------------------------------------------------
+
+# The extended-token tables, keyed by the byte that follows the prefix.
+# Each runs from &8E upward, in ROM order.
+_EXTENDED_FUNCTIONS = ("SUM", "BEAT")
+_EXTENDED_COMMANDS = (
+    "APPEND",
+    "AUTO",
+    "CRUNCH",
+    "DELETE",
+    "EDIT",
+    "HELP",
+    "LIST",
+    "LOAD",
+    "LVAR",
+    "NEW",
+    "OLD",
+    "RENUMBER",
+    "SAVE",
+    "TEXTLOAD",
+    "TEXTSAVE",
+    "TWIN",
+    "TWINO",
+    "INSTALL",
+)
+_EXTENDED_STATEMENTS = (
+    "CASE",
+    "CIRCLE",
+    "FILL",
+    "ORIGIN",
+    "POINT",
+    "RECTANGLE",
+    "SWAP",
+    "WHILE",
+    "WAIT",
+    "MOUSE",
+    "QUIT",
+    "SYS",
+    "INSTALL",
+    "LIBRARY",
+    "TINT",
+    "ELLIPSE",
+    "BEATS",
+    "TEMPO",
+    "VOICES",
+    "VOICE",
+    "STEREO",
+    "OVERLAY",
+)
+
+_EXTENDED_TOKEN_BASE = 0x8E
+
+
+def _extended_table(keywords: tuple[str, ...]) -> Mapping[int, str]:
+    """Build a {second byte -> keyword} table counting from ``&8E``."""
+    return MappingProxyType(
+        {_EXTENDED_TOKEN_BASE + offset: keyword for offset, keyword in enumerate(keywords)}
+    )
+
+
+# Single-byte tokens BASIC V re-purposes from BASIC II. &7F is new;
+# &C9-&CE displace the LIST/NEW/OLD/RENUMBER/SAVE command tokens (whose
+# spellings move into the &C7 escape table) and the unused &CE gap.
+_BASIC_V_SINGLE_BYTE_OVERRIDES = {
+    0x7F: "OTHERWISE",
+    0xC9: "WHEN",
+    0xCA: "OF",
+    0xCB: "ENDCASE",
+    0xCC: "ELSE",
+    0xCD: "ENDIF",
+    0xCE: "ENDWHILE",
+}
+
+# Start from BASIC II, drop the three bytes that are now escape prefixes
+# (so a bare prefix never resolves to AUTO/DELETE/LOAD), then apply the
+# re-purposed single-byte slots.
+_basic_v_single_byte = {
+    token: keyword
+    for token, keyword in TOKEN_TO_KEYWORD.items()
+    if token not in (0xC6, 0xC7, 0xC8)
+}
+_basic_v_single_byte.update(_BASIC_V_SINGLE_BYTE_OVERRIDES)
+
+#: BBC BASIC V — the Archimedes / RISC OS ARM BASIC. Adds the
+#: ``&C6``/``&C7``/``&C8`` escape tables and the re-purposed single-byte
+#: tokens described in this module's docstring.
+BASIC_V = Dialect(
+    name="BBC BASIC V",
+    single_byte=MappingProxyType(_basic_v_single_byte),
+    escape=MappingProxyType(
+        {
+            0xC6: _extended_table(_EXTENDED_FUNCTIONS),
+            0xC7: _extended_table(_EXTENDED_COMMANDS),
+            0xC8: _extended_table(_EXTENDED_STATEMENTS),
+        }
+    ),
+)

{oaknut_basic-12.10.1 → oaknut_basic-12.12.0}/src/oaknut/basic/scanner.py

@@ -26,13 +26,14 @@ import enum
 from collections.abc import Iterator
 from dataclasses import dataclass
 
+from oaknut.basic.dialect import BASIC_II, Dialect
 from oaknut.basic.exceptions import (
     InvalidLineLengthError,
     MissingLineMarkerError,
     TruncatedProgramError,
 )
 from oaknut.basic.linenumber import decode_line_number
-from oaknut.basic.tokens import HEADER_LENGTH, LINE_NUMBER_TOKEN, TOKEN_TO_KEYWORD
+from oaknut.basic.tokens import HEADER_LENGTH, LINE_NUMBER_TOKEN
 
 _CR = 0x0D
 _END_MARKER = 0xFF
@@ -90,7 +91,7 @@ class LineRecord:
     start: int
 
 
-def scan(data: bytes, *, base_offset: int = 0) -> Iterator[Token]:
+def scan(data: bytes, *, base_offset: int = 0, dialect: Dialect = BASIC_II) -> Iterator[Token]:
     """Scan an unframed line body into a stream of :class:`Token`.
 
     Args:
@@ -99,10 +100,17 @@ def scan(data: bytes, *, base_offset: int = 0) -> Iterator[Token]:
         base_offset: Added to every token's :attr:`~Token.start`, so
             offsets can be reported against a larger buffer (for example
             the line body's position within a whole program).
+        dialect: The BBC BASIC variant whose token tables to use.
+            Defaults to :data:`~oaknut.basic.BASIC_II`; pass
+            :data:`~oaknut.basic.BASIC_V` to resolve the
+            ``&C6``/``&C7``/``&C8`` two-byte escape tokens and the
+            re-purposed single-byte tokens of the Archimedes language.
 
     Yields:
         One :class:`Token` per keyword, literal run, string literal, and
-        ``&8D`` line-number reference, in source order.
+        ``&8D`` line-number reference, in source order. For a two-byte
+        escape keyword the token's :attr:`~Token.token` is the prefix
+        byte and its :attr:`~Token.start` is the prefix's offset.
 
     Raises:
         DetokeniseError: A ``&8D`` reference is truncated.
@@ -141,13 +149,20 @@ def scan(data: bytes, *, base_offset: int = 0) -> Iterator[Token]:
             yield Token(TokenKind.LINENUM, str(decode_line_number(payload)), base_offset + i)
             i += 4
             continue
-        if b >= 0x80 and b in TOKEN_TO_KEYWORD:
+        if b in dialect.escape and i + 1 < n and data[i + 1] in dialect.escape[b]:
             yield from flush()
-            yield Token(TokenKind.KEYWORD, TOKEN_TO_KEYWORD[b], base_offset + i, b)
+            keyword = dialect.escape[b][data[i + 1]]
+            yield Token(TokenKind.KEYWORD, keyword, base_offset + i, b)
+            i += 2
+            continue
+        if b in dialect.single_byte:
+            yield from flush()
+            yield Token(TokenKind.KEYWORD, dialect.single_byte[b], base_offset + i, b)
             i += 1
             continue
-        # A literal byte (< &80) or an unknown high byte (e.g. the &CE
-        # gap): coalesce into a TEXT run so the byte still round-trips.
+        # A literal byte, an unknown high byte, or an escape prefix whose
+        # following byte has no extended-token meaning: coalesce into a
+        # TEXT run so the byte still round-trips.
         if text_start < 0:
             text_start = i
         text_chars.append(chr(b))
@@ -155,12 +170,15 @@ def scan(data: bytes, *, base_offset: int = 0) -> Iterator[Token]:
     yield from flush()
 
 
-def scan_program(data: bytes) -> Iterator[LineRecord]:
+def scan_program(data: bytes, *, dialect: Dialect = BASIC_II) -> Iterator[LineRecord]:
     """Walk a ``&0D``-framed stored program, scanning each line body.
 
     Args:
         data: A tokenised program as stored on disc or in memory,
             terminated by the ``&0D &FF`` end marker.
+        dialect: The BBC BASIC variant whose token tables to use, passed
+            through to :func:`scan`. Defaults to
+            :data:`~oaknut.basic.BASIC_II`.
 
     Yields:
         One :class:`LineRecord` per line, in storage order, each carrying
@@ -187,6 +205,6 @@ def scan_program(data: bytes) -> Iterator[LineRecord]:
         if length < HEADER_LENGTH or i + length > n:
             raise InvalidLineLengthError(offset=i, length=length)
         body = data[i + HEADER_LENGTH : i + length]
-        tokens = tuple(scan(body, base_offset=i + HEADER_LENGTH))
+        tokens = tuple(scan(body, base_offset=i + HEADER_LENGTH, dialect=dialect))
         yield LineRecord(line_number=line_number, tokens=tokens, start=i)
         i += length

{oaknut_basic-12.10.1 → oaknut_basic-12.12.0/src/oaknut_basic.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oaknut-basic
-Version: 12.10.1
+Version: 12.12.0
 Summary: BBC BASIC tools: program tokeniser/de-tokeniser and PRINT#/INPUT# data-file reader/writer
 Author-email: Robert Smallshire <robert@smallshire.org.uk>
 License-Expression: MIT

{oaknut_basic-12.10.1 → oaknut_basic-12.12.0}/src/oaknut_basic.egg-info/SOURCES.txt

@@ -4,7 +4,9 @@ pyproject.toml
 src/oaknut/basic/__init__.py
 src/oaknut/basic/cli.py
 src/oaknut/basic/datafile.py
+src/oaknut/basic/detect.py
 src/oaknut/basic/detokeniser.py
+src/oaknut/basic/dialect.py
 src/oaknut/basic/exceptions.py
 src/oaknut/basic/float5.py
 src/oaknut/basic/linenumber.py
@@ -19,10 +21,12 @@ src/oaknut_basic.egg-info/entry_points.txt
 src/oaknut_basic.egg-info/requires.txt
 src/oaknut_basic.egg-info/top_level.txt
 tests/test_basic.py
+tests/test_basic_v.py
 tests/test_cli.py
 tests/test_crunch_rules.py
 tests/test_data_cli.py
 tests/test_datafile.py
+tests/test_detect.py
 tests/test_detokeniser.py
 tests/test_float5.py
 tests/test_keyword_coverage.py

oaknut_basic-12.12.0/tests/test_basic_v.py

@@ -0,0 +1,195 @@
+"""Tests for the BBC BASIC V (Archimedes/RISC OS) detokenising dialect.
+
+BASIC V re-uses three BASIC II command tokens — ``&C6``/``&C7``/``&C8`` —
+as two-byte *escape prefixes*: the following byte selects an extended
+keyword (``SUM``, ``RENUMBER``, ``CASE``/``SYS``/``ORIGIN`` ...). It also
+re-purposes several single-byte slots: ``&7F`` becomes ``OTHERWISE`` and
+``&C9``-``&CE`` — ``LIST``/``NEW``/``OLD``/``RENUMBER``/``SAVE`` in
+BASIC II — become the block keywords ``WHEN``/``OF``/``ENDCASE``/
+``ELSE``/``ENDIF``/``ENDWHILE``. The default dialect stays BASIC II, so
+the escape bytes still decode to ``AUTO``/``DELETE``/``LOAD`` unless the
+caller opts into :data:`BASIC_V`.
+"""
+
+import pytest
+from oaknut.basic import (
+    BASIC_II,
+    BASIC_V,
+    Dialect,
+    TokenKind,
+    detokenise,
+    detokenise_body,
+    scan,
+)
+
+
+def _program(*lines: tuple[int, bytes]) -> bytes:
+    """Frame (line_number, body) pairs into a tokenised program."""
+    out = bytearray()
+    for line_number, body in lines:
+        out += bytes((0x0D, (line_number >> 8) & 0xFF, line_number & 0xFF, 4 + len(body)))
+        out += body
+    out += b"\x0d\xff"
+    return bytes(out)
+
+
+class TestEscapeStatements:
+    """``&C8`` <byte> — extended statement tokens, second byte from &8E."""
+
+    @pytest.mark.parametrize(
+        ("second", "keyword"),
+        [
+            (0x8E, "CASE"),
+            (0x8F, "CIRCLE"),
+            (0x90, "FILL"),
+            (0x91, "ORIGIN"),
+            (0x92, "POINT"),
+            (0x93, "RECTANGLE"),
+            (0x94, "SWAP"),
+            (0x95, "WHILE"),
+            (0x96, "WAIT"),
+            (0x97, "MOUSE"),
+            (0x98, "QUIT"),
+            (0x99, "SYS"),
+            (0x9A, "INSTALL"),
+            (0x9B, "LIBRARY"),
+            (0x9C, "TINT"),
+            (0x9D, "ELLIPSE"),
+            (0x9E, "BEATS"),
+            (0x9F, "TEMPO"),
+            (0xA0, "VOICES"),
+            (0xA1, "VOICE"),
+            (0xA2, "STEREO"),
+            (0xA3, "OVERLAY"),
+        ],
+    )
+    def test_c8_escape_decodes_to_statement(self, second: int, keyword: str):
+        assert detokenise_body(bytes((0xC8, second)), dialect=BASIC_V) == keyword
+
+
+class TestEscapeCommands:
+    """``&C7`` <byte> — extended command tokens, second byte from &8E."""
+
+    @pytest.mark.parametrize(
+        ("second", "keyword"),
+        [
+            (0x8E, "APPEND"),
+            (0x8F, "AUTO"),
+            (0x90, "CRUNCH"),
+            (0x91, "DELETE"),
+            (0x92, "EDIT"),
+            (0x93, "HELP"),
+            (0x94, "LIST"),
+            (0x95, "LOAD"),
+            (0x96, "LVAR"),
+            (0x97, "NEW"),
+            (0x98, "OLD"),
+            (0x99, "RENUMBER"),
+            (0x9A, "SAVE"),
+            (0x9B, "TEXTLOAD"),
+            (0x9C, "TEXTSAVE"),
+            (0x9D, "TWIN"),
+            (0x9E, "TWINO"),
+            (0x9F, "INSTALL"),
+        ],
+    )
+    def test_c7_escape_decodes_to_command(self, second: int, keyword: str):
+        assert detokenise_body(bytes((0xC7, second)), dialect=BASIC_V) == keyword
+
+
+class TestEscapeFunctions:
+    """``&C6`` <byte> — the two extended function tokens."""
+
+    @pytest.mark.parametrize(("second", "keyword"), [(0x8E, "SUM"), (0x8F, "BEAT")])
+    def test_c6_escape_decodes_to_function(self, second: int, keyword: str):
+        assert detokenise_body(bytes((0xC6, second)), dialect=BASIC_V) == keyword
+
+
+class TestSingleByteDifferences:
+    """Single-byte tokens BASIC V re-purposes from BASIC II."""
+
+    @pytest.mark.parametrize(
+        ("token", "keyword"),
+        [
+            (0x7F, "OTHERWISE"),
+            (0xC9, "WHEN"),
+            (0xCA, "OF"),
+            (0xCB, "ENDCASE"),
+            (0xCC, "ELSE"),
+            (0xCD, "ENDIF"),
+            (0xCE, "ENDWHILE"),
+        ],
+    )
+    def test_single_byte_token_decodes_to_basic_v_keyword(self, token: int, keyword: str):
+        assert detokenise_body(bytes((token,)), dialect=BASIC_V) == keyword
+
+    def test_shared_single_byte_tokens_are_unchanged(self):
+        # Tokens common to both dialects still decode the same way.
+        assert detokenise_body(b"\xf1", dialect=BASIC_V) == "PRINT"
+        assert detokenise_body(b"\x8f", dialect=BASIC_V) == "PTR"
+
+
+class TestIssue44Reproduction:
+    def test_origin_statement_is_not_loadtime(self):
+        # Line 140 of "Herding (RR1)", Acorn User June 1991:
+        #   C8 91 " 640,512"  ->  ORIGIN 640,512   (was: LOADTIME 640,512)
+        body = bytes((0xC8, 0x91)) + b" 640,512"
+        assert detokenise_body(body, dialect=BASIC_V) == "ORIGIN 640,512"
+
+    def test_program_round_through_detokenise(self):
+        program = _program((140, bytes((0xC8, 0x91)) + b" 640,512"))
+        assert detokenise(program, dialect=BASIC_V) == "140ORIGIN 640,512\n"
+
+
+class TestDefaultDialectUnchanged:
+    """Without opting in, the escape bytes keep their BASIC II meaning."""
+
+    def test_c8_defaults_to_load_plus_token(self):
+        body = bytes((0xC8, 0x91)) + b" 640,512"
+        assert detokenise_body(body) == "LOADTIME 640,512"
+
+    def test_default_dialect_is_basic_ii(self):
+        body = bytes((0xC8, 0x91))
+        assert detokenise_body(body, dialect=BASIC_II) == detokenise_body(body)
+
+    def test_basic_v_c9_is_list_under_basic_ii(self):
+        assert detokenise_body(b"\xc9") == "LIST"
+
+
+class TestScanStream:
+    def test_escape_token_is_one_keyword_advancing_two_bytes(self):
+        body = bytes((0xC8, 0x99)) + b"X"  # SYS then a literal X
+        tokens = list(scan(body, dialect=BASIC_V))
+        assert tokens[0].kind is TokenKind.KEYWORD
+        assert tokens[0].value == "SYS"
+        assert tokens[0].start == 0
+        assert tokens[1].kind is TokenKind.TEXT
+        assert tokens[1].value == "X"
+        assert tokens[1].start == 2
+
+    def test_escape_byte_inside_string_stays_literal(self):
+        body = b'"' + bytes((0xC8, 0x91)) + b'"'
+        tokens = list(scan(body, dialect=BASIC_V))
+        assert len(tokens) == 1
+        assert tokens[0].kind is TokenKind.STRING
+        assert tokens[0].value == '"\xc8\x91"'
+
+    def test_unterminated_escape_prefix_stays_literal(self):
+        # A trailing &C8 with no following byte must not over-read.
+        tokens = list(scan(b"\xc8", dialect=BASIC_V))
+        assert tokens[0].kind is TokenKind.TEXT
+        assert tokens[0].value == "\xc8"
+
+
+class TestDialectObject:
+    def test_basic_v_has_a_name(self):
+        assert isinstance(BASIC_V, Dialect)
+        assert "V" in BASIC_V.name
+
+    def test_unknown_escape_second_byte_is_left_literal(self):
+        # &C8 followed by a byte with no extended-token meaning: emit the
+        # prefix as a literal rather than inventing a keyword or over-reading.
+        body = bytes((0xC8, 0x41))  # &41 = 'A', not an extended statement
+        tokens = list(scan(body, dialect=BASIC_V))
+        assert tokens[0].kind is TokenKind.TEXT
+        assert tokens[0].value.startswith("\xc8")

{oaknut_basic-12.10.1 → oaknut_basic-12.12.0}/tests/test_cli.py

@@ -9,6 +9,7 @@ rather than re-testing the numbering or tokenising logic itself.
 from pathlib import Path
 
 import oaknut.basic as basic
+import pytest
 from click.testing import CliRunner
 from oaknut.basic.cli import cli
 
@@ -230,6 +231,84 @@ class TestDetokeniseCommand:
         assert result.exit_code != 0
         assert "offset 0" in result.output
 
+    @pytest.mark.parametrize("dialect", ["v", "5"])
+    def test_dialect_v_decodes_escape_tokens(self, dialect: str):
+        # &C8 &91 is the BASIC V ORIGIN statement, not "LOADTIME".
+        # The Roman "v" and Arabic "5" numerals are interchangeable.
+        program = b"\x0d\x00\x8c\x0e\xc8\x91 640,512\x0d\xff"
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detokenise", "--dialect", dialect], input=program)
+        assert result.exit_code == 0
+        assert result.stdout_bytes == b"140ORIGIN 640,512\n"
+
+    @pytest.mark.parametrize("dialect", ["ii", "2"])
+    def test_dialect_ii_keeps_basic_ii_meaning(self, dialect: str):
+        program = b"\x0d\x00\x8c\x0e\xc8\x91 640,512\x0d\xff"
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detokenise", "--dialect", dialect], input=program)
+        assert result.exit_code == 0
+        assert result.stdout_bytes == b"140LOADTIME 640,512\n"
+
+    def test_default_dialect_is_basic_ii(self):
+        program = b"\x0d\x00\x8c\x0e\xc8\x91 640,512\x0d\xff"
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detokenise"], input=program)
+        assert result.exit_code == 0
+        assert result.stdout_bytes == b"140LOADTIME 640,512\n"
+
+
+class TestDetectCommand:
+    def test_stdin_basic_program_exits_zero(self):
+        program = basic.tokenise("10 PRINT")
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect"], input=program)
+        assert result.exit_code == 0
+        assert "BASIC" in result.output
+        assert "basic" in result.output
+
+    def test_stdin_non_basic_exits_nonzero(self):
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect"], input=b"just some text")
+        assert result.exit_code == 1
+        assert "not-basic" in result.output
+
+    def test_files_mixed_reports_each_and_exits_nonzero(self, tmp_path):
+        good = tmp_path / "PROG"
+        good.write_bytes(basic.tokenise("10 END"))
+        bad = tmp_path / "DATA"
+        bad.write_bytes(b"not a program")
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect", str(good), str(bad)])
+        assert result.exit_code == 1
+        assert str(good) in result.output
+        assert str(bad) in result.output
+        # One line per file.
+        assert result.output.count("PROG") == 1
+        assert result.output.count("DATA") == 1
+
+    def test_all_basic_files_exit_zero(self, tmp_path):
+        first = tmp_path / "A"
+        first.write_bytes(basic.tokenise("10 END"))
+        second = tmp_path / "B"
+        second.write_bytes(basic.tokenise("20 PRINT"))
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect", str(first), str(second)])
+        assert result.exit_code == 0
+
+    def test_quiet_suppresses_output_but_keeps_exit_code(self):
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect", "--quiet"], input=b"not basic")
+        assert result.exit_code == 1
+        assert result.output == ""
+
+    def test_non_ff_terminator_notes_are_shown(self):
+        program = bytearray(basic.tokenise("10 PRINT"))
+        program[-1] = 0x80  # top bit still set, so still a terminator
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect"], input=bytes(program))
+        assert result.exit_code == 0
+        assert "note:" in result.output
+
 
 class TestTokeniseDetokeniseRoundTrip:
     def test_cli_round_trip(self):

oaknut_basic-12.12.0/tests/test_detect.py

@@ -0,0 +1,152 @@
+"""Tests for the structural tokenised-BBC-BASIC detector.
+
+The detector walks the &0D-framed line structure the same way the ROM's
+``LIST`` does, but is *tolerant*: unlike :func:`scan_program`, which
+raises on malformed input, :func:`detect` classifies any blob — clean
+program, program-plus-data, truncated fragment, or arbitrary garbage —
+into a :class:`Verdict` with evidence, and never raises.
+"""
+
+import pytest
+from oaknut.basic import Detection, Verdict, detect, tokenise
+
+
+def _program(*lines: tuple[int, bytes]) -> bytes:
+    """Frame (line_number, body) pairs into a tokenised program."""
+    out = bytearray()
+    for line_number, body in lines:
+        out += bytes((0x0D, (line_number >> 8) & 0xFF, line_number & 0xFF, 4 + len(body)))
+        out += body
+    out += b"\x0d\xff"
+    return bytes(out)
+
+
+class TestClean:
+    def test_tokenised_program_is_basic(self):
+        program = tokenise("10 PRINT \"HELLO\"\n20 GOTO 10")
+        result = detect(program)
+        assert result.verdict is Verdict.BASIC
+        assert result.is_basic
+        assert result.line_count == 2
+        assert result.first_line == 10
+        assert result.last_line == 20
+        assert result.trailing_length == 0
+        assert result.program_length == len(program)
+        assert result.ascending
+
+    def test_reason_reports_the_line_count(self):
+        result = detect(tokenise("10 END"))
+        assert "1 line" in result.reason
+
+    def test_single_bodyless_line_is_basic(self):
+        # A line with only its 4-byte header (no body) is well-formed.
+        result = detect(_program((10, b"")))
+        assert result.verdict is Verdict.BASIC
+        assert result.line_count == 1
+
+
+class TestTrailing:
+    def test_program_with_appended_data_is_basic_trailing(self):
+        program = tokenise("10 PRINT")
+        blob = program + b"junk appended after the terminator"
+        result = detect(blob)
+        assert result.verdict is Verdict.BASIC_TRAILING
+        assert result.is_basic
+        assert result.trailing_length == len(b"junk appended after the terminator")
+        assert result.program_length == len(program)
+        assert "trailing" in result.reason
+
+
+class TestMaybe:
+    def test_truncated_mid_line_is_maybe(self):
+        # One clean line, then a second line header whose length runs off
+        # the end: it begins as BASIC but the structure breaks.
+        blob = _program((10, b"PRINT"))[:-2]  # drop the terminator
+        blob = blob + b"\x0d\x00\x14\x40"  # a line claiming 0x40 body bytes
+        result = detect(blob)
+        assert result.verdict is Verdict.MAYBE
+        assert not result.is_basic
+        assert result.line_count == 1
+
+    def test_first_line_length_too_small_after_one_good_line(self):
+        good = _program((10, b"AB"))[:-2]  # first line, no terminator
+        blob = good + b"\x0d\x00\x14\x03"  # second line header, length 3 (< 4)
+        result = detect(blob)
+        assert result.verdict is Verdict.MAYBE
+        assert "length 3" in result.reason
+
+
+class TestNotBasic:
+    def test_not_cr_led_is_not_basic(self):
+        result = detect(b"This is just some text.")
+        assert result.verdict is Verdict.NOT_BASIC
+        assert not result.is_basic
+        assert "not a &0D line marker" in result.reason
+
+    def test_leading_cr_terminator_with_no_lines_is_not_basic(self):
+        # A View document opening 0x0D 0x80: a terminator before any line.
+        result = detect(b"\x0d\x80the rest of a view document")
+        assert result.verdict is Verdict.NOT_BASIC
+        assert result.line_count == 0
+
+    def test_too_short_is_not_basic(self):
+        assert detect(b"").verdict is Verdict.NOT_BASIC
+        assert detect(b"\x0d").verdict is Verdict.NOT_BASIC
+
+    def test_first_header_malformed_is_not_basic(self):
+        # 0x0D, a line-number high byte, then it runs out: zero clean lines.
+        result = detect(b"\x0d\x00\x0a")  # truncated first header
+        assert result.verdict is Verdict.NOT_BASIC
+
+
+class TestTerminatorTolerance:
+    def test_non_ff_top_bit_terminator_is_basic_with_note(self):
+        program = bytearray(tokenise("10 PRINT"))
+        # Tamper with the terminator's second byte: top bit still set.
+        assert program[-1] == 0xFF
+        program[-1] = 0x80
+        result = detect(bytes(program))
+        assert result.verdict is Verdict.BASIC
+        assert result.is_basic
+        assert any("not &FF" in note for note in result.notes)
+
+    def test_ff_terminator_has_no_note(self):
+        assert detect(tokenise("10 PRINT")).notes == ()
+
+
+class TestEvidence:
+    def test_descending_line_numbers_flagged_not_ascending(self):
+        # Structurally valid, but line numbers go down: still BASIC.
+        blob = _program((20, b"A"), (10, b"B"))
+        result = detect(blob)
+        assert result.verdict is Verdict.BASIC
+        assert result.ascending is False
+        assert result.first_line == 20
+        assert result.last_line == 10
+
+    def test_notes_is_immutable_sequence(self):
+        # Evidence is a frozen result; notes must not be a mutable list a
+        # caller could accidentally extend.
+        result = detect(tokenise("10 END"))
+        assert isinstance(result, Detection)
+        with pytest.raises((AttributeError, TypeError)):
+            result.verdict = Verdict.NOT_BASIC  # frozen dataclass
+
+
+class TestNeverRaises:
+    @pytest.mark.parametrize(
+        "blob",
+        [
+            b"",
+            b"\x0d",
+            b"\x0d\x0d\x0d\x0d",
+            b"\x0d\x00\x0a\xff",  # header length byte 0xFF
+            b"\x0d\x00\x0a\x04",  # bodyless-length line then nothing
+            bytes(range(256)),
+            b"\x0d" + bytes(range(256)),
+        ],
+    )
+    def test_detect_is_total(self, blob: bytes):
+        # Must classify, never throw, whatever the input.
+        result = detect(blob)
+        assert isinstance(result.verdict, Verdict)