oaknut-basic 12.11.0__tar.gz → 12.13.1__tar.gz

{oaknut_basic-12.11.0 → oaknut_basic-12.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oaknut-basic
-Version: 12.11.0
+Version: 12.13.1
 Summary: BBC BASIC tools: program tokeniser/de-tokeniser and PRINT#/INPUT# data-file reader/writer
 Author-email: Robert Smallshire <robert@smallshire.org.uk>
 License-Expression: MIT

{oaknut_basic-12.11.0 → oaknut_basic-12.13.1}/src/oaknut/basic/__init__.py

@@ -30,6 +30,7 @@ from oaknut.basic.datafile import (
     BbcBasicDataReader,
     BbcBasicDataWriter,
 )
+from oaknut.basic.detect import Detection, Verdict, detect
 from oaknut.basic.detokeniser import detokenise, detokenise_body
 from oaknut.basic.dialect import BASIC_II, BASIC_V, Dialect
 from oaknut.basic.exceptions import (
@@ -80,7 +81,7 @@ from oaknut.basic.tokens import (
     TOKEN_TO_KEYWORD,
 )
 
-__version__ = "12.11.0"
+__version__ = "12.13.1"
 
 # Canonical load addresses for BBC BASIC programs on each host.
 # Programs saved by *SAVE on a real machine use these by default.
@@ -112,6 +113,7 @@ __all__ = [
     "BbcBasicDataWriter",
     "DataFileError",
     "DataFileTypeMismatchError",
+    "Detection",
     "DetokeniseError",
     "Dialect",
     "Float5RangeError",
@@ -130,7 +132,9 @@ __all__ = [
     "TruncatedRecordError",
     "UnknownTagError",
     "UnnumberedLineError",
+    "Verdict",
     "decode_line_number",
+    "detect",
     "detokenise",
     "detokenise_body",
     "number_lines",

{oaknut_basic-12.11.0 → oaknut_basic-12.13.1}/src/oaknut/basic/cli.py

@@ -296,6 +296,58 @@ def detokenise(input_stream, output_stream, encoding: str, dialect: str) -> None
     output_stream.write(_listing_to_bytes(listing, encoding))
 
 
+@cli.command()
+@click.argument(
+    "inputs",
+    metavar="[FILE]...",
+    type=click.File("rb"),
+    nargs=-1,
+)
+@click.option(
+    "--quiet",
+    "-q",
+    is_flag=True,
+    help="Suppress per-file output; report the verdict through the exit code only.",
+)
+@click.pass_context
+def detect(ctx: click.Context, inputs, quiet: bool) -> None:
+    """Identify whether each FILE is a tokenised BBC BASIC program.
+
+    Classifies each FILE by its byte structure alone — the length-driven
+    line walk the BBC BASIC ROM performs to ``LIST`` a program — without
+    de-tokenising, so unlabelled files harvested from disc images can be
+    filtered. With no FILE, or ``-``, reads a single blob from standard
+    input ::
+
+        oaknut-basic detect *
+        disc get game.ssd MENU - | oaknut-basic detect
+
+    Each line reports a ``BASIC`` flag (blank when not), the verdict, the
+    path and the reason; non-fatal observations follow as indented notes.
+    The exit code is non-zero if any input is not a BASIC program, so it
+    composes as a shell filter. The verdict is one of ``basic`` (a clean
+    program), ``basic+`` (a program with data appended), ``maybe`` (begins
+    as BASIC, then the structure breaks) or ``not-basic``.
+    """
+    from oaknut.basic import detect as detect_blob
+
+    streams = inputs if inputs else (click.get_binary_stream("stdin"),)
+    all_basic = True
+    for stream in streams:
+        result = detect_blob(stream.read())
+        all_basic = all_basic and result.is_basic
+        if quiet:
+            continue
+        name = getattr(stream, "name", "<stdin>")
+        flag = "BASIC" if result.is_basic else "     "
+        click.echo(f"{flag} {result.verdict.value:9} {name}  -- {result.reason}")
+        for note in result.notes:
+            click.echo(f"          note: {note}")
+
+    if not all_basic:
+        ctx.exit(1)
+
+
 @cli.group()
 def data() -> None:
     """Read and write BBC BASIC data files (PRINT#/INPUT#/BPUT#/BGET#).

oaknut_basic-12.13.1/src/oaknut/basic/detect.py

@@ -0,0 +1,212 @@
+"""Identify tokenised BBC BASIC programs by structure alone.
+
+Files harvested from Acorn disc images (DFS ``.ssd``, ADFS ``.adl``)
+carry no filetype or extension, so a blob's nature has to be inferred
+from its bytes. A stored BBC BASIC program — BASIC I, II, IV and V all
+share the same line framing — is a run of lines::
+
+    &0D  <line-hi>  <line-lo>  <len>   <len-4 tokenised body bytes>
+
+repeated, then a terminator: ``&0D`` followed by a byte with its top bit
+set. BASIC writes ``&0D &FF``, but the ROM only tests the top bit, and
+programs with data appended sometimes tamper with the second byte, so
+``&0D &80``-``&FF`` all terminate — :func:`detect` matches the ROM, not
+the literal ``&FF``, and merely notes a non-``&FF`` terminator.
+
+The walk is length-driven: each line's own ``<len>`` byte says where the
+next ``&0D`` must be, so tokenised body bytes that happen to look like
+``&0D`` never trip the scan. It is exactly what the ROM does to ``LIST``
+a program, which makes it about as faithful a classifier as is possible
+without a full de-tokenise. Because the framing is identical across all
+BASIC versions, detection is purely structural and needs no
+:class:`~oaknut.basic.Dialect` — it sits well below the token table.
+
+Unlike :func:`~oaknut.basic.scan_program`, which raises
+:class:`~oaknut.basic.DetokeniseError` on malformed input, :func:`detect`
+is **total**: it classifies clean programs, program-plus-data, truncated
+fragments and arbitrary garbage into a :class:`Verdict` with evidence,
+and never raises. Classification is deliberately conservative and
+reason-bearing so a corpus filter can explain every accept and reject.
+"""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass, field
+
+from oaknut.basic.tokens import HEADER_LENGTH
+
+_CR = 0x0D
+
+# A stored line number is 0-32767 (see oaknut.basic.linenumber.MAX_LINE_NUMBER),
+# so its high byte is always <= &7F — top bit clear. The end-of-program
+# terminator's second byte has the top bit set (&FF, or a tampered &80-&FE).
+# Testing the top bit therefore tells a terminator from a line-number high
+# byte, exactly as the ROM's LIST loop does.
+_TERMINATOR_TOP_BIT = 0x80
+
+
+class Verdict(enum.Enum):
+    """How confident :func:`detect` is that a blob is tokenised BBC BASIC."""
+
+    #: A clean walk all the way to a proper terminator, no trailing bytes.
+    BASIC = "basic"
+    #: A valid program followed by extra bytes (data appended after the end).
+    BASIC_TRAILING = "basic+"
+    #: Begins as one or more well-formed lines, then the structure breaks.
+    MAYBE = "maybe"
+    #: Does not begin as a tokenised line at all.
+    NOT_BASIC = "not-basic"
+
+    @property
+    def is_basic(self) -> bool:
+        """True for the verdicts a corpus filter would extract as BASIC."""
+        return self in (Verdict.BASIC, Verdict.BASIC_TRAILING)
+
+
+@dataclass(frozen=True)
+class Detection:
+    """The outcome of inspecting one blob, with the evidence behind it.
+
+    Attributes:
+        verdict: The :class:`Verdict` reached.
+        reason: A human-readable explanation of the decision.
+        line_count: Well-formed lines walked before the terminator or break.
+        program_length: Bytes up to and including the terminator (0 if none
+            was reached).
+        trailing_length: Bytes after the terminator (non-zero only for
+            :attr:`Verdict.BASIC_TRAILING`).
+        first_line: The first line's number, or ``None`` if no line parsed.
+        last_line: The last line's number, or ``None`` if no line parsed.
+        ascending: Whether the line numbers were strictly ascending.
+        notes: Non-fatal observations (e.g. a non-``&FF`` terminator).
+    """
+
+    verdict: Verdict
+    reason: str
+    line_count: int = 0
+    program_length: int = 0
+    trailing_length: int = 0
+    first_line: int | None = None
+    last_line: int | None = None
+    ascending: bool = True
+    notes: tuple[str, ...] = field(default_factory=tuple)
+
+    @property
+    def is_basic(self) -> bool:
+        """True when the verdict is :attr:`Verdict.BASIC` or ``BASIC_TRAILING``."""
+        return self.verdict.is_basic
+
+
+def detect(data: bytes | bytearray) -> Detection:
+    """Classify *data* as tokenised BBC BASIC (or not) by walking its lines.
+
+    Args:
+        data: The raw bytes to classify.
+
+    Returns:
+        A :class:`Detection` carrying the :class:`Verdict` and the
+        evidence gathered during the walk. Always returns; never raises.
+    """
+    n = len(data)
+    if n < 2:
+        return Detection(Verdict.NOT_BASIC, "too short to be a BASIC line")
+    if data[0] != _CR:
+        return Detection(
+            Verdict.NOT_BASIC,
+            f"first byte is &{data[0]:02X}, not a &0D line marker",
+        )
+
+    pos = 0
+    line_count = 0
+    first_line: int | None = None
+    last_line: int | None = None
+    prev_line: int | None = None
+    ascending = True
+    notes: list[str] = []
+
+    def broke(reason: str) -> Detection:
+        """A broken walk is MAYBE if it parsed >=1 clean line, else NOT_BASIC.
+
+        One or more well-formed lines before the break means the blob
+        really does begin as a tokenised program (truncated, or BASIC
+        followed by data without a proper terminator); zero clean lines
+        means the leading ``&0D`` was a coincidence.
+        """
+        return Detection(
+            Verdict.MAYBE if line_count > 0 else Verdict.NOT_BASIC,
+            reason,
+            line_count=line_count,
+            first_line=first_line,
+            last_line=last_line,
+            ascending=ascending,
+            notes=tuple(notes),
+        )
+
+    while True:
+        # Every iteration must land on a line marker.
+        if pos >= n or data[pos] != _CR:
+            found = f"&{data[pos]:02X}" if pos < n else "end of data"
+            return broke(f"expected &0D at offset {pos}, found {found}")
+
+        # The byte after &0D is either a terminator (top bit set) or a
+        # line-number high byte.
+        if pos + 1 >= n:
+            return broke(f"truncated after &0D at offset {pos}")
+
+        marker = data[pos + 1]
+        if marker & _TERMINATOR_TOP_BIT:
+            # A terminator before any line parsed means the leading &0D was
+            # a coincidence (e.g. a View document opening &0D &80), not an
+            # empty program — real type-ins always have at least one line.
+            if line_count == 0:
+                return Detection(
+                    Verdict.NOT_BASIC,
+                    f"&0D &{marker:02X} at offset 0 — terminator with no lines, "
+                    "not a program",
+                )
+            program_length = pos + 2
+            trailing = n - program_length
+            if marker != 0xFF:
+                notes.append(
+                    f"terminator second byte &{marker:02X} (not &FF, top bit "
+                    "only) — data may follow"
+                )
+            verdict = Verdict.BASIC if trailing == 0 else Verdict.BASIC_TRAILING
+            reason = f"walked {line_count} line(s) to a terminator" + (
+                f", {trailing} trailing byte(s)" if trailing else ""
+            )
+            return Detection(
+                verdict=verdict,
+                reason=reason,
+                line_count=line_count,
+                program_length=program_length,
+                trailing_length=trailing,
+                first_line=first_line,
+                last_line=last_line,
+                ascending=ascending,
+                notes=tuple(notes),
+            )
+
+        # A real line: need the full 4-byte header.
+        if pos + HEADER_LENGTH > n:
+            return broke(f"truncated line header at offset {pos}")
+
+        line_no = (marker << 8) | data[pos + 2]
+        length = data[pos + 3]
+        if length < HEADER_LENGTH:
+            return broke(
+                f"line {line_no} at offset {pos} has length {length} "
+                f"(< {HEADER_LENGTH})"
+            )
+        if pos + length > n:
+            return broke(f"line {line_no} at offset {pos} runs {length} bytes past end of data")
+
+        if first_line is None:
+            first_line = line_no
+        if prev_line is not None and line_no <= prev_line:
+            ascending = False
+        prev_line = line_no
+        last_line = line_no
+        line_count += 1
+        pos += length

{oaknut_basic-12.11.0 → oaknut_basic-12.13.1}/src/oaknut_basic.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oaknut-basic
-Version: 12.11.0
+Version: 12.13.1
 Summary: BBC BASIC tools: program tokeniser/de-tokeniser and PRINT#/INPUT# data-file reader/writer
 Author-email: Robert Smallshire <robert@smallshire.org.uk>
 License-Expression: MIT

{oaknut_basic-12.11.0 → oaknut_basic-12.13.1}/src/oaknut_basic.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ pyproject.toml
 src/oaknut/basic/__init__.py
 src/oaknut/basic/cli.py
 src/oaknut/basic/datafile.py
+src/oaknut/basic/detect.py
 src/oaknut/basic/detokeniser.py
 src/oaknut/basic/dialect.py
 src/oaknut/basic/exceptions.py
@@ -25,6 +26,7 @@ tests/test_cli.py
 tests/test_crunch_rules.py
 tests/test_data_cli.py
 tests/test_datafile.py
+tests/test_detect.py
 tests/test_detokeniser.py
 tests/test_float5.py
 tests/test_keyword_coverage.py

{oaknut_basic-12.11.0 → oaknut_basic-12.13.1}/tests/test_cli.py

@@ -257,6 +257,59 @@ class TestDetokeniseCommand:
         assert result.stdout_bytes == b"140LOADTIME 640,512\n"
 
 
+class TestDetectCommand:
+    def test_stdin_basic_program_exits_zero(self):
+        program = basic.tokenise("10 PRINT")
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect"], input=program)
+        assert result.exit_code == 0
+        assert "BASIC" in result.output
+        assert "basic" in result.output
+
+    def test_stdin_non_basic_exits_nonzero(self):
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect"], input=b"just some text")
+        assert result.exit_code == 1
+        assert "not-basic" in result.output
+
+    def test_files_mixed_reports_each_and_exits_nonzero(self, tmp_path):
+        good = tmp_path / "PROG"
+        good.write_bytes(basic.tokenise("10 END"))
+        bad = tmp_path / "DATA"
+        bad.write_bytes(b"not a program")
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect", str(good), str(bad)])
+        assert result.exit_code == 1
+        assert str(good) in result.output
+        assert str(bad) in result.output
+        # One line per file.
+        assert result.output.count("PROG") == 1
+        assert result.output.count("DATA") == 1
+
+    def test_all_basic_files_exit_zero(self, tmp_path):
+        first = tmp_path / "A"
+        first.write_bytes(basic.tokenise("10 END"))
+        second = tmp_path / "B"
+        second.write_bytes(basic.tokenise("20 PRINT"))
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect", str(first), str(second)])
+        assert result.exit_code == 0
+
+    def test_quiet_suppresses_output_but_keeps_exit_code(self):
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect", "--quiet"], input=b"not basic")
+        assert result.exit_code == 1
+        assert result.output == ""
+
+    def test_non_ff_terminator_notes_are_shown(self):
+        program = bytearray(basic.tokenise("10 PRINT"))
+        program[-1] = 0x80  # top bit still set, so still a terminator
+        runner = CliRunner()
+        result = runner.invoke(cli, ["detect"], input=bytes(program))
+        assert result.exit_code == 0
+        assert "note:" in result.output
+
+
 class TestTokeniseDetokeniseRoundTrip:
     def test_cli_round_trip(self):
         runner = CliRunner()

oaknut_basic-12.13.1/tests/test_detect.py

@@ -0,0 +1,152 @@
+"""Tests for the structural tokenised-BBC-BASIC detector.
+
+The detector walks the &0D-framed line structure the same way the ROM's
+``LIST`` does, but is *tolerant*: unlike :func:`scan_program`, which
+raises on malformed input, :func:`detect` classifies any blob — clean
+program, program-plus-data, truncated fragment, or arbitrary garbage —
+into a :class:`Verdict` with evidence, and never raises.
+"""
+
+import pytest
+from oaknut.basic import Detection, Verdict, detect, tokenise
+
+
+def _program(*lines: tuple[int, bytes]) -> bytes:
+    """Frame (line_number, body) pairs into a tokenised program."""
+    out = bytearray()
+    for line_number, body in lines:
+        out += bytes((0x0D, (line_number >> 8) & 0xFF, line_number & 0xFF, 4 + len(body)))
+        out += body
+    out += b"\x0d\xff"
+    return bytes(out)
+
+
+class TestClean:
+    def test_tokenised_program_is_basic(self):
+        program = tokenise("10 PRINT \"HELLO\"\n20 GOTO 10")
+        result = detect(program)
+        assert result.verdict is Verdict.BASIC
+        assert result.is_basic
+        assert result.line_count == 2
+        assert result.first_line == 10
+        assert result.last_line == 20
+        assert result.trailing_length == 0
+        assert result.program_length == len(program)
+        assert result.ascending
+
+    def test_reason_reports_the_line_count(self):
+        result = detect(tokenise("10 END"))
+        assert "1 line" in result.reason
+
+    def test_single_bodyless_line_is_basic(self):
+        # A line with only its 4-byte header (no body) is well-formed.
+        result = detect(_program((10, b"")))
+        assert result.verdict is Verdict.BASIC
+        assert result.line_count == 1
+
+
+class TestTrailing:
+    def test_program_with_appended_data_is_basic_trailing(self):
+        program = tokenise("10 PRINT")
+        blob = program + b"junk appended after the terminator"
+        result = detect(blob)
+        assert result.verdict is Verdict.BASIC_TRAILING
+        assert result.is_basic
+        assert result.trailing_length == len(b"junk appended after the terminator")
+        assert result.program_length == len(program)
+        assert "trailing" in result.reason
+
+
+class TestMaybe:
+    def test_truncated_mid_line_is_maybe(self):
+        # One clean line, then a second line header whose length runs off
+        # the end: it begins as BASIC but the structure breaks.
+        blob = _program((10, b"PRINT"))[:-2]  # drop the terminator
+        blob = blob + b"\x0d\x00\x14\x40"  # a line claiming 0x40 body bytes
+        result = detect(blob)
+        assert result.verdict is Verdict.MAYBE
+        assert not result.is_basic
+        assert result.line_count == 1
+
+    def test_first_line_length_too_small_after_one_good_line(self):
+        good = _program((10, b"AB"))[:-2]  # first line, no terminator
+        blob = good + b"\x0d\x00\x14\x03"  # second line header, length 3 (< 4)
+        result = detect(blob)
+        assert result.verdict is Verdict.MAYBE
+        assert "length 3" in result.reason
+
+
+class TestNotBasic:
+    def test_not_cr_led_is_not_basic(self):
+        result = detect(b"This is just some text.")
+        assert result.verdict is Verdict.NOT_BASIC
+        assert not result.is_basic
+        assert "not a &0D line marker" in result.reason
+
+    def test_leading_cr_terminator_with_no_lines_is_not_basic(self):
+        # A View document opening 0x0D 0x80: a terminator before any line.
+        result = detect(b"\x0d\x80the rest of a view document")
+        assert result.verdict is Verdict.NOT_BASIC
+        assert result.line_count == 0
+
+    def test_too_short_is_not_basic(self):
+        assert detect(b"").verdict is Verdict.NOT_BASIC
+        assert detect(b"\x0d").verdict is Verdict.NOT_BASIC
+
+    def test_first_header_malformed_is_not_basic(self):
+        # 0x0D, a line-number high byte, then it runs out: zero clean lines.
+        result = detect(b"\x0d\x00\x0a")  # truncated first header
+        assert result.verdict is Verdict.NOT_BASIC
+
+
+class TestTerminatorTolerance:
+    def test_non_ff_top_bit_terminator_is_basic_with_note(self):
+        program = bytearray(tokenise("10 PRINT"))
+        # Tamper with the terminator's second byte: top bit still set.
+        assert program[-1] == 0xFF
+        program[-1] = 0x80
+        result = detect(bytes(program))
+        assert result.verdict is Verdict.BASIC
+        assert result.is_basic
+        assert any("not &FF" in note for note in result.notes)
+
+    def test_ff_terminator_has_no_note(self):
+        assert detect(tokenise("10 PRINT")).notes == ()
+
+
+class TestEvidence:
+    def test_descending_line_numbers_flagged_not_ascending(self):
+        # Structurally valid, but line numbers go down: still BASIC.
+        blob = _program((20, b"A"), (10, b"B"))
+        result = detect(blob)
+        assert result.verdict is Verdict.BASIC
+        assert result.ascending is False
+        assert result.first_line == 20
+        assert result.last_line == 10
+
+    def test_notes_is_immutable_sequence(self):
+        # Evidence is a frozen result; notes must not be a mutable list a
+        # caller could accidentally extend.
+        result = detect(tokenise("10 END"))
+        assert isinstance(result, Detection)
+        with pytest.raises((AttributeError, TypeError)):
+            result.verdict = Verdict.NOT_BASIC  # frozen dataclass
+
+
+class TestNeverRaises:
+    @pytest.mark.parametrize(
+        "blob",
+        [
+            b"",
+            b"\x0d",
+            b"\x0d\x0d\x0d\x0d",
+            b"\x0d\x00\x0a\xff",  # header length byte 0xFF
+            b"\x0d\x00\x0a\x04",  # bodyless-length line then nothing
+            bytes(range(256)),
+            b"\x0d" + bytes(range(256)),
+        ],
+    )
+    def test_detect_is_total(self, blob: bytes):
+        # Must classify, never throw, whatever the input.
+        result = detect(blob)
+        assert isinstance(result.verdict, Verdict)