oaknut-basic 12.15.0__tar.gz → 12.16.0__tar.gz

{oaknut_basic-12.15.0 → oaknut_basic-12.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oaknut-basic
-Version: 12.15.0
+Version: 12.16.0
 Summary: BBC BASIC tools: program tokeniser/de-tokeniser and PRINT#/INPUT# data-file reader/writer
 Author-email: Robert Smallshire <robert@smallshire.org.uk>
 License-Expression: MIT

{oaknut_basic-12.15.0 → oaknut_basic-12.16.0}/src/oaknut/basic/__init__.py

@@ -67,7 +67,7 @@ from oaknut.basic.scanner import (
     scan,
     scan_program,
 )
-from oaknut.basic.tokeniser import tokenise
+from oaknut.basic.tokeniser import Crunch, tokenise
 from oaknut.basic.tokens import (
     FLAG_CONDITIONAL,
     FLAG_FN_PROC,
@@ -81,7 +81,7 @@ from oaknut.basic.tokens import (
     TOKEN_TO_KEYWORD,
 )
 
-__version__ = "12.15.0"
+__version__ = "12.16.0"
 
 # Canonical load addresses for BBC BASIC programs on each host.
 # Programs saved by *SAVE on a real machine use these by default.
@@ -111,6 +111,7 @@ __all__ = [
     "BbcBasicDataFileBase",
     "BbcBasicDataReader",
     "BbcBasicDataWriter",
+    "Crunch",
     "DataFileError",
     "DataFileTypeMismatchError",
     "Detection",

{oaknut_basic-12.15.0 → oaknut_basic-12.16.0}/src/oaknut/basic/cli.py

@@ -212,8 +212,23 @@ def _listing_to_bytes(listing: str, encoding: str) -> bytes:
     default=None,
     help="Auto-numbering increment. Defaults to 10 when only --start is given.",
 )
+@click.option(
+    "--crunch",
+    type=click.Choice(["rom", "greedy"]),
+    default="rom",
+    show_default=True,
+    help="Which tokeniser to emulate. \"rom\" is byte-exact to the BBC BASIC "
+    "ROM. Use \"greedy\" to reproduce the greedier third-party tokeniser "
+    "behind some early-1980s commercial programs, so their de-tokenised "
+    "source re-tokenises byte-identically.",
+)
 def tokenise(
-    input_stream, output_stream, encoding: str, start: int | None, step: int | None
+    input_stream,
+    output_stream,
+    encoding: str,
+    start: int | None,
+    step: int | None,
+    crunch: str,
 ) -> None:
     """Tokenise BBC BASIC source text into a stored program.
 
@@ -232,6 +247,11 @@ def tokenise(
     \b
         oaknut-basic tokenise --start 10 unnumbered.bas MENU
 
+    Pass ``--crunch greedy`` to reproduce a greedier third-party tokeniser
+    used by a class of early-1980s commercial programs; the default
+    ``rom`` is byte-exact to the BBC BASIC ROM. Use it only to regenerate
+    such a program byte-for-byte from its de-tokenised source.
+
     INPUT is read in --encoding (``utf-8`` by default, for source authored
     in a modern editor; pass ``acorn`` for the BBC character set, e.g.
     source taken straight off a disc image).
@@ -239,7 +259,7 @@ def tokenise(
     from oaknut.basic import tokenise as tokenise_source
 
     source = _source_to_code_points(input_stream.read(), encoding)
-    output_stream.write(tokenise_source(source, start=start, step=step))
+    output_stream.write(tokenise_source(source, start=start, step=step, crunch=crunch))
 
 
 @cli.command()

{oaknut_basic-12.15.0 → oaknut_basic-12.16.0}/src/oaknut/basic/tokeniser.py

@@ -27,6 +27,8 @@ the CLI does it at its I/O boundary.
 
 from __future__ import annotations
 
+from typing import Literal
+
 from oaknut.basic.exceptions import (
     AlreadyNumberedError,
     LineNumberOrderError,
@@ -59,6 +61,13 @@ from oaknut.basic.tokens import (
 _CR = 0x0D
 _END_MARKER = 0xFF
 
+# The crunch dialect. ``"rom"`` is byte-exact to the BBC BASIC ROM's
+# line-input crunch (the default). ``"greedy"`` reproduces a greedier
+# third-party tokeniser found in early-1980s commercial programs, whose
+# keyword recognition differs from the ROM in three localised ways (see
+# :func:`tokenise`); everything else is identical to the ROM crunch.
+Crunch = Literal["rom", "greedy"]
+
 
 def _split_source_lines(source: str) -> list[str]:
     return LINE_SEPARATOR_RE.split(source)
@@ -70,7 +79,13 @@ for _keyword, _token, _flags in KEYWORDS:
     _KEYWORDS_BY_FIRST.setdefault(_keyword[0], []).append((_keyword, _token, _flags))
 
 
-def tokenise(source: str, *, start: int | None = None, step: int | None = None) -> bytes:
+def tokenise(
+    source: str,
+    *,
+    start: int | None = None,
+    step: int | None = None,
+    crunch: Crunch = "rom",
+) -> bytes:
     """Tokenise BBC BASIC II source text into a stored program.
 
     Args:
@@ -82,6 +97,24 @@ def tokenise(source: str, *, start: int | None = None, step: int | None = None)
             numbers. Defaults to 10 when only *step* is given.
         step: Increment for auto-numbering. Defaults to 10 when only
             *start* is given.
+        crunch: Which tokeniser to emulate. ``"rom"`` (the default) is
+            byte-exact to the BBC BASIC ROM's line-input crunch.
+            ``"greedy"`` reproduces a greedier third-party tokeniser used
+            by a class of early-1980s commercial programs, so their
+            de-tokenised source re-tokenises byte-identically. It differs
+            from the ROM in three ways only:
+
+            1. A keyword interrupts a ``&`` hex constant, ending the hex
+               run where the keyword begins (``&FE60AND`` becomes ``&FE60``
+               then ``AND``, not the literal run ``&FE60A`` then ``NDROW``).
+            2. An ``FN``/``PROC`` name terminates at a following ``THEN``
+               or ``ELSE`` (the ``FLAG_START`` keywords), keeping the first
+               name character but ending the name there. Other embedded
+               keywords (``READ`` in ``PROCREADKP``) are left intact.
+            3. A conditional keyword before a name character is suppressed
+               only when that character does not itself begin a keyword, so
+               ``STOP`` before ``ELSE`` tokenises while ``NEW`` in
+               ``NEWKEY%`` stays literal.
 
     Returns:
         The tokenised program bytes, terminated by ``&0D &FF``.
@@ -118,7 +151,7 @@ def tokenise(source: str, *, start: int | None = None, step: int | None = None)
             raise LineNumberOrderError(line_index, line_number, previous_number, line)
         previous_number = line_number
 
-        body = _tokenise_body(body_text, line_index=line_index, line_text=line)
+        body = _tokenise_body(body_text, line_index=line_index, line_text=line, crunch=crunch)
         if len(body) > MAX_BODY_LENGTH:
             raise LineTooLongError(line_index, line_number, len(body), line)
 
@@ -174,13 +207,18 @@ def _is_hex_digit(c: str) -> bool:
     return ("0" <= c <= "9") or ("A" <= c <= "F")
 
 
-def _try_keyword(text: str, i: int) -> tuple[int, int, int] | None:
+def _try_keyword(text: str, i: int, *, crunch: Crunch) -> tuple[int, int, int] | None:
     """Match a keyword at ``text[i]``.
 
     Returns ``(token, consumed, flags)`` for the first full or
     ``.``-abbreviated match in ROM order, or ``None`` if the cursor does
     not begin a keyword (including a conditional keyword suppressed by a
     following name character).
+
+    Conditional suppression depends on *crunch*: the ``"rom"`` crunch
+    suppresses whenever a name character follows; ``"greedy"`` suppresses
+    only when that character does not itself begin a keyword, so a
+    conditional keyword is still recognised before ``ELSE`` and the like.
     """
     group = _KEYWORDS_BY_FIRST.get(text[i])
     if group is None:
@@ -209,14 +247,59 @@ def _try_keyword(text: str, i: int) -> tuple[int, int, int] | None:
         if abbreviated:
             return token, p + 1, flags
         # Full match. A conditional keyword is suppressed when a name
-        # character follows, so it stays part of an identifier.
+        # character follows, so it stays part of an identifier — unless
+        # the greedy crunch sees that character begin its own keyword.
         if flags & FLAG_CONDITIONAL and i + klen < n and _is_name_char(text[i + klen]):
-            return None
+            if crunch == "rom" or not _starts_keyword(text, i + klen):
+                return None
         return token, klen, flags
     return None
 
 
-def _tokenise_body(text: str, *, line_index: int, line_text: str) -> bytearray:
+def _starts_keyword(text: str, i: int) -> bool:
+    """True if a keyword (full or ``.``-abbreviated) begins at ``text[i]``.
+
+    A plain prefix test with no conditional suppression, used by the
+    greedy crunch to decide where a hex run or a suppressed conditional
+    keyword should yield to a fresh keyword match.
+    """
+    group = _KEYWORDS_BY_FIRST.get(text[i])
+    if group is None:
+        return False
+    n = len(text)
+    for keyword, _token, _flags in group:
+        for p, keyword_char in enumerate(keyword):
+            if i + p >= n:
+                break
+            source_char = text[i + p]
+            if source_char == keyword_char:
+                continue
+            if source_char == ".":
+                return True
+            break
+        else:
+            return True  # ran the whole keyword: a full match
+    return False
+
+
+def _starts_flag_start_keyword(text: str, i: int) -> bool:
+    """True if a ``FLAG_START`` keyword (``THEN``/``ELSE``/...) begins here.
+
+    A full-match-only test (no abbreviation): the greedy crunch breaks an
+    ``FN``/``PROC`` name at such a keyword.
+    """
+    group = _KEYWORDS_BY_FIRST.get(text[i])
+    if group is None:
+        return False
+    for keyword, _token, flags in group:
+        if flags & FLAG_START and text[i : i + len(keyword)] == keyword:
+            return True
+    return False
+
+
+def _tokenise_body(
+    text: str, *, line_index: int, line_text: str, crunch: Crunch = "rom"
+) -> bytearray:
     """Crunch one statement-text body into tokenised bytes."""
     out = bytearray()
     i = 0
@@ -250,6 +333,10 @@ def _tokenise_body(text: str, *, line_index: int, line_text: str) -> bytearray:
             out.append(ord("&"))
             i += 1
             while i < n and _is_hex_digit(text[i]):
+                # Greedy crunch (rule 1): a keyword beginning inside the run
+                # ends the hex constant, so `&FE60AND` -> `&FE60`, `AND`.
+                if crunch == "greedy" and _is_letter(text[i]) and _starts_keyword(text, i):
+                    break
                 out.append(ord(text[i]))
                 i += 1
             mid = True  # a value, but does not disarm (`&FF 1` encodes the 1)
@@ -304,7 +391,7 @@ def _tokenise_body(text: str, *, line_index: int, line_text: str) -> bytearray:
             continue
 
         if "A" <= c <= "W":  # potential keyword
-            match = _try_keyword(text, i)
+            match = _try_keyword(text, i, crunch=crunch)
             if match is not None:
                 token, consumed, flags = match
                 emit = token
@@ -328,6 +415,15 @@ def _tokenise_body(text: str, *, line_index: int, line_text: str) -> bytearray:
                 if flags & FLAG_FN_PROC:
                     name_start = i
                     while i < n and _is_name_char(text[i]):
+                        # Greedy crunch (rule 2): the name breaks at a
+                        # following THEN/ELSE, but the first char is always
+                        # kept (`PROCWTKEYELSE...` -> name `WTKEY`, `ELSE`).
+                        if (
+                            crunch == "greedy"
+                            and i > name_start
+                            and _starts_flag_start_keyword(text, i)
+                        ):
+                            break
                         out.append(ord(text[i]))
                         i += 1
                     if i > name_start:  # consumed an identifier -> read a name

{oaknut_basic-12.15.0 → oaknut_basic-12.16.0}/src/oaknut_basic.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oaknut-basic
-Version: 12.15.0
+Version: 12.16.0
 Summary: BBC BASIC tools: program tokeniser/de-tokeniser and PRINT#/INPUT# data-file reader/writer
 Author-email: Robert Smallshire <robert@smallshire.org.uk>
 License-Expression: MIT

{oaknut_basic-12.15.0 → oaknut_basic-12.16.0}/src/oaknut_basic.egg-info/SOURCES.txt

@@ -29,6 +29,7 @@ tests/test_datafile.py
 tests/test_detect.py
 tests/test_detokeniser.py
 tests/test_float5.py
+tests/test_greedy_crunch.py
 tests/test_keyword_coverage.py
 tests/test_linenumber.py
 tests/test_numbering.py

{oaknut_basic-12.15.0 → oaknut_basic-12.16.0}/tests/test_cli.py

@@ -252,6 +252,26 @@ class TestTokeniseCommand:
         assert result.exit_code != 0
         assert "no line number" in result.output
 
+    def test_default_crunch_is_rom(self):
+        runner = CliRunner()
+        result = runner.invoke(cli, ["tokenise"], input=b"10 A=?&FE60ANDROW%\n")
+        assert result.exit_code == 0
+        assert result.stdout_bytes == basic.tokenise("10 A=?&FE60ANDROW%\n", crunch="rom")
+
+    def test_greedy_crunch_selects_the_greedier_tokeniser(self):
+        runner = CliRunner()
+        source = b"10 A=?&FE60ANDROW%\n"
+        result = runner.invoke(cli, ["tokenise", "--crunch", "greedy"], input=source)
+        assert result.exit_code == 0
+        assert result.stdout_bytes == basic.tokenise(source.decode(), crunch="greedy")
+        # The greedy result genuinely differs from the ROM default.
+        assert result.stdout_bytes != basic.tokenise(source.decode(), crunch="rom")
+
+    def test_unknown_crunch_is_rejected(self):
+        runner = CliRunner()
+        result = runner.invoke(cli, ["tokenise", "--crunch", "nope"], input=b"10 END\n")
+        assert result.exit_code != 0
+
 
 class TestTokeniseEncoding:
     def test_default_decodes_host_utf8_pound_sign(self):

oaknut_basic-12.16.0/tests/test_greedy_crunch.py

@@ -0,0 +1,150 @@
+"""The greedy crunch mode (``tokenise(..., crunch="greedy")``).
+
+The BBC BASIC ROM's crunch is only one of the tokenisers that produced
+tokenised BBC BASIC in the wild. A class of early-1980s commercial
+programs was crunched by a *greedier* tool whose keyword recognition
+differs from the ROM in three specific ways, so their de-tokenised source
+does not re-tokenise byte-identically under the ROM-accurate default.
+
+``crunch="greedy"`` reproduces that tool. The default (``crunch="rom"``)
+stays byte-exact to the ROM, so the two modes are pinned side by side on
+the same source below.
+
+The three rule differences (see :func:`oaknut.basic.tokenise`):
+
+- **Rule 1** — a keyword interrupts a hex constant. The ROM copies every
+  ``0``-``9`` / ``A``-``F`` in a ``&`` run unconditionally; greedy stops
+  the run where a keyword begins (``&FE60ANDROW%`` -> ``&FE60``, ``AND``,
+  ``ROW%``).
+- **Rule 2** — an ``FN``/``PROC`` name terminates at a ``FLAG_START``
+  keyword (``THEN``/``ELSE``). The ROM's name-skip swallows every
+  alphanumeric; greedy keeps the first name character but breaks the name
+  at a following ``THEN``/``ELSE`` — and *only* those, so a function
+  keyword embedded in a name (``READ`` in ``PROCREADKP``) is left intact.
+- **Rule 3** — refined conditional suppression. The ROM keeps a
+  conditional keyword literal whenever a name character follows; greedy
+  does so only when that character does not itself begin a keyword, so
+  ``STOP`` before ``ELSE`` tokenises while ``NEW`` in ``NEWKEY%`` stays
+  literal.
+
+The two commercial programs under ``data/greedy`` (Voltmace / Custom
+Video Productions, 1983) validate the whole model: their own
+``detokenise`` output re-tokenises byte-for-byte under greedy.
+"""
+
+from pathlib import Path
+
+import pytest
+from oaknut.basic import detokenise, tokenise
+
+_DATA_DIRPATH = Path(__file__).parent / "data"
+_GREEDY_DIRPATH = _DATA_DIRPATH / "greedy"
+
+
+def _body(statement: str, *, crunch: str = "rom") -> bytes:
+    """Tokenise a single body (no leading space) and return its bytes."""
+    program = tokenise("10" + statement, crunch=crunch)
+    length = program[3]
+    return program[4:length]
+
+
+# (source after the line number, ROM body, greedy body). The ROM column is
+# the current default and must not change; the greedy column reproduces the
+# commercial files. Values transcribed from issue #48.
+_VECTORS = [
+    # Rule 1 — a keyword interrupts a hex constant.
+    (
+        "A=?&FE60ANDROW%",
+        bytes.fromhex("41 3d 3f 26 46 45 36 30 41 4e 44 52 4f 57 25"),
+        bytes.fromhex("41 3d 3f 26 46 45 36 30 80 52 4f 57 25"),
+    ),
+    # Rule 2 — an FN/PROC name breaks at a FLAG_START keyword (THEN/ELSE).
+    (
+        "IFA THENPROCWTKEYELSEPROCBKKEY",
+        bytes.fromhex(
+            "e7 41 20 8c f2 57 54 4b 45 59 45 4c 53 45 50 52 4f 43 42 4b 4b 45 59"
+        ),
+        bytes.fromhex("e7 41 20 8c f2 57 54 4b 45 59 8b f2 42 4b 4b 45 59"),
+    ),
+    # Rule 2 — a function keyword embedded in a name is not a break point.
+    (
+        "PROCREADKP",
+        bytes.fromhex("f2 52 45 41 44 4b 50"),
+        bytes.fromhex("f2 52 45 41 44 4b 50"),
+    ),
+    # Rule 3 — a conditional keyword before another keyword tokenises.
+    (
+        "IFA=1THENSTOPELSEGOTO90",
+        bytes.fromhex("e7 41 3d 31 8c 53 54 4f 50 45 4c 53 45 47 4f 54 4f 39 30"),
+        bytes.fromhex("e7 41 3d 31 8c fa 8b e5 8d 44 5a 40"),
+    ),
+    # Rule 3 — a conditional keyword before a plain name char stays literal.
+    (
+        "NEWKEY%=0",
+        bytes.fromhex("4e 45 57 4b 45 59 25 3d 30"),
+        bytes.fromhex("4e 45 57 4b 45 59 25 3d 30"),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    ("source", "rom_body", "greedy_body"),
+    _VECTORS,
+    ids=[v[0] for v in _VECTORS],
+)
+def test_rom_default_is_unchanged(source, rom_body, greedy_body):
+    assert _body(source, crunch="rom") == rom_body
+
+
+@pytest.mark.parametrize(
+    ("source", "rom_body", "greedy_body"),
+    _VECTORS,
+    ids=[v[0] for v in _VECTORS],
+)
+def test_greedy_reproduces_the_greedier_tool(source, rom_body, greedy_body):
+    assert _body(source, crunch="greedy") == greedy_body
+
+
+def test_default_crunch_is_rom():
+    # The default must remain ROM-accurate: an un-flagged call matches "rom".
+    source = "10 A=?&FE60ANDROW%"
+    assert tokenise(source) == tokenise(source, crunch="rom")
+
+
+def _program_lines(program: bytes) -> dict[int, bytes]:
+    """Split a stored program into ``{line_number: body_bytes}``."""
+    lines: dict[int, bytes] = {}
+    offset = 0
+    while offset + 3 < len(program):
+        if program[offset] != 0x0D or program[offset + 1] == 0xFF:
+            break
+        record_length = program[offset + 3]
+        number = (program[offset + 1] << 8) | program[offset + 2]
+        lines[number] = program[offset + 4 : offset + record_length]
+        offset += record_length
+    return lines
+
+
+class TestCommercialPrograms:
+    """The KEYPAD / JOYSTIK Voltmace programs round-trip under greedy."""
+
+    def test_keypad_round_trips_byte_identically(self):
+        program = (_GREEDY_DIRPATH / "keypad.tokens").read_bytes()
+        assert tokenise(detokenise(program), crunch="greedy") == program
+
+    def test_joystik_round_trips_except_the_encryption_boundary_line(self):
+        # JOYSTIK's line 1890 is the file's ROL-encryption boundary line, a
+        # decode artefact of that binary rather than a tokeniser difference;
+        # every other line re-tokenises byte-for-byte under greedy.
+        program = (_GREEDY_DIRPATH / "joystik.tokens").read_bytes()
+        retokenised = tokenise(detokenise(program), crunch="greedy")
+        original = _program_lines(program)
+        rebuilt = _program_lines(retokenised)
+        differing = [n for n in original if original[n] != rebuilt.get(n)]
+        assert differing == [1890]
+
+    def test_rom_crunch_does_not_round_trip_the_greedy_files(self):
+        # Guard the premise: the ROM-accurate default cannot regenerate a
+        # program the greedier tool produced.
+        program = (_GREEDY_DIRPATH / "keypad.tokens").read_bytes()
+        assert tokenise(detokenise(program), crunch="rom") != program