web-a2e 1.0.0

package/src/core/basic/basic_tokenizer.cpp

@@ -0,0 +1,286 @@
+/*
+ * basic_tokenizer.cpp - Applesoft BASIC tokenizer for direct memory insertion
+ *
+ * Written by
+ *  Mike Daley <michael_daley@icloud.com>
+ */
+
+#include "basic_tokenizer.hpp"
+#include "basic_tokens.hpp"
+#include <algorithm>
+#include <cctype>
+#include <cstring>
+#include <string>
+#include <vector>
+
+namespace a2e {
+
+// A keyword entry for greedy longest-match tokenization
+struct KeywordEntry {
+    const char* keyword;
+    uint8_t token;
+    size_t length;
+};
+
+// Build sorted keyword list (longest first) for greedy matching
+static std::vector<KeywordEntry> buildKeywordList() {
+    std::vector<KeywordEntry> list;
+    for (int i = 0; i < APPLESOFT_TOKEN_COUNT; i++) {
+        KeywordEntry e;
+        e.keyword = APPLESOFT_TOKENS[i];
+        e.token = static_cast<uint8_t>(0x80 + i);
+        e.length = strlen(APPLESOFT_TOKENS[i]);
+        list.push_back(e);
+    }
+    // Sort by length descending for greedy longest-match
+    std::sort(list.begin(), list.end(), [](const KeywordEntry& a, const KeywordEntry& b) {
+        return a.length > b.length;
+    });
+    return list;
+}
+
+// Parsed BASIC line
+struct BasicLine {
+    int lineNumber;
+    std::string content; // uppercase content after line number
+};
+
+// Parse source into lines, extracting line numbers
+static std::vector<BasicLine> parseSource(const char* source) {
+    std::vector<BasicLine> lines;
+    std::string src(source);
+
+    size_t pos = 0;
+    while (pos < src.size()) {
+        // Find end of line
+        size_t eol = src.find('\n', pos);
+        if (eol == std::string::npos) eol = src.size();
+
+        std::string rawLine = src.substr(pos, eol - pos);
+        pos = eol + 1;
+
+        // Trim leading/trailing whitespace
+        size_t start = rawLine.find_first_not_of(" \t\r");
+        if (start == std::string::npos) continue;
+        std::string trimmed = rawLine.substr(start);
+
+        // Extract line number
+        if (trimmed.empty() || !isdigit(static_cast<unsigned char>(trimmed[0]))) continue;
+
+        size_t numEnd = 0;
+        while (numEnd < trimmed.size() && isdigit(static_cast<unsigned char>(trimmed[numEnd]))) {
+            numEnd++;
+        }
+
+        int lineNum = std::stoi(trimmed.substr(0, numEnd));
+        if (lineNum < 0 || lineNum > 63999) continue;
+
+        // Get content after line number, skip leading spaces
+        std::string content = trimmed.substr(numEnd);
+        size_t contentStart = content.find_first_not_of(" \t");
+        if (contentStart != std::string::npos) {
+            content = content.substr(contentStart);
+        } else {
+            content = "";
+        }
+
+        // Skip lines with only a line number and no content
+        if (content.empty()) continue;
+
+        // Convert to uppercase, preserving case inside quoted strings
+        std::string upper;
+        upper.reserve(content.size());
+        bool inQuote = false;
+        for (char c : content) {
+            if (c == '"') {
+                inQuote = !inQuote;
+                upper += c;
+            } else if (inQuote) {
+                upper += c;
+            } else {
+                upper += static_cast<char>(toupper(static_cast<unsigned char>(c)));
+            }
+        }
+
+        lines.push_back({lineNum, upper});
+    }
+
+    // Sort by line number
+    std::sort(lines.begin(), lines.end(), [](const BasicLine& a, const BasicLine& b) {
+        return a.lineNumber < b.lineNumber;
+    });
+
+    return lines;
+}
+
+// Tokenize a single line's content
+static std::vector<uint8_t> tokenizeLine(const std::string& text,
+                                          const std::vector<KeywordEntry>& keywords) {
+    std::vector<uint8_t> bytes;
+    size_t i = 0;
+    bool inRem = false;
+    bool inData = false;
+    bool inQuote = false;
+
+    while (i < text.size()) {
+        char ch = text[i];
+
+        // Inside a quoted string - emit as-is until closing quote
+        if (inQuote) {
+            bytes.push_back(static_cast<uint8_t>(text[i]));
+            if (ch == '"') inQuote = false;
+            i++;
+            continue;
+        }
+
+        // After REM token - emit rest of line as raw ASCII
+        if (inRem) {
+            bytes.push_back(static_cast<uint8_t>(text[i]));
+            i++;
+            continue;
+        }
+
+        // After DATA token - emit as-is until colon
+        if (inData) {
+            if (ch == ':') {
+                inData = false;
+                // Fall through to normal processing for the colon
+            } else {
+                bytes.push_back(static_cast<uint8_t>(text[i]));
+                i++;
+                continue;
+            }
+        }
+
+        // Opening quote
+        if (ch == '"') {
+            inQuote = true;
+            bytes.push_back(static_cast<uint8_t>(text[i]));
+            i++;
+            continue;
+        }
+
+        // ? is shorthand for PRINT
+        if (ch == '?') {
+            bytes.push_back(0xBA); // PRINT token
+            i++;
+            continue;
+        }
+
+        // Skip spaces outside strings (Apple II tokenizer ignores spaces)
+        if (ch == ' ') {
+            i++;
+            continue;
+        }
+
+        // Try greedy longest-match against keywords
+        bool matched = false;
+        const char* remaining = text.c_str() + i;
+        size_t remainingLen = text.size() - i;
+
+        for (const auto& kw : keywords) {
+            if (kw.length <= remainingLen &&
+                strncmp(remaining, kw.keyword, kw.length) == 0) {
+                bytes.push_back(kw.token);
+                i += kw.length;
+                matched = true;
+
+                if (kw.token == 0xB2) { // REM
+                    inRem = true;
+                } else if (kw.token == 0x83) { // DATA
+                    inData = true;
+                }
+                break;
+            }
+        }
+
+        if (!matched) {
+            bytes.push_back(static_cast<uint8_t>(text[i]));
+            i++;
+        }
+    }
+
+    return bytes;
+}
+
+int loadBasicProgram(const char* source, MemReadFn readMem, MemWriteFn writeMem) {
+    if (!source) return -1;
+
+    auto lines = parseSource(source);
+    if (lines.empty()) return 0;
+
+    auto keywords = buildKeywordList();
+
+    constexpr uint16_t txttab = 0x0801;
+    uint16_t addr = txttab;
+
+    // First pass: tokenize all lines and compute layout
+    struct TokenizedLine {
+        int lineNumber;
+        std::vector<uint8_t> tokens;
+        uint16_t lineSize; // 2 (next-ptr) + 2 (line-num) + tokens + 1 (terminator)
+    };
+
+    std::vector<TokenizedLine> tokenizedLines;
+    for (const auto& line : lines) {
+        auto tokens = tokenizeLine(line.content, keywords);
+        uint16_t lineSize = static_cast<uint16_t>(2 + 2 + tokens.size() + 1);
+        tokenizedLines.push_back({line.lineNumber, std::move(tokens), lineSize});
+    }
+
+    // Second pass: write into memory
+    for (const auto& line : tokenizedLines) {
+        uint16_t nextAddr = addr + line.lineSize;
+
+        // Bounds check
+        if (nextAddr >= 0xC000) return -1;
+
+        // Next-pointer (little-endian)
+        writeMem(addr,     nextAddr & 0xFF);
+        writeMem(addr + 1, (nextAddr >> 8) & 0xFF);
+
+        // Line number (little-endian)
+        writeMem(addr + 2, line.lineNumber & 0xFF);
+        writeMem(addr + 3, (line.lineNumber >> 8) & 0xFF);
+
+        // Token bytes
+        for (size_t i = 0; i < line.tokens.size(); i++) {
+            writeMem(static_cast<uint16_t>(addr + 4 + i), line.tokens[i]);
+        }
+
+        // Line terminator
+        writeMem(static_cast<uint16_t>(addr + 4 + line.tokens.size()), 0x00);
+
+        addr = nextAddr;
+    }
+
+    // End-of-program marker
+    writeMem(addr,     0x00);
+    writeMem(addr + 1, 0x00);
+
+    uint16_t endAddr = addr + 2;
+
+    // Read MEMSIZE ($73/$74) from memory
+    uint16_t memsize = readMem(0x73) | (readMem(0x74) << 8);
+
+    // Set zero page pointers
+    auto writePtr = [&](uint16_t zpAddr, uint16_t value) {
+        writeMem(zpAddr,     value & 0xFF);
+        writeMem(zpAddr + 1, (value >> 8) & 0xFF);
+    };
+
+    writePtr(0x67, txttab);   // TXTTAB - start of program
+    writePtr(0x69, endAddr);  // VARTAB - start of variable space
+    writePtr(0x6B, endAddr);  // ARYTAB - start of array space
+    writePtr(0x6D, endAddr);  // STREND - end of numeric storage
+    writePtr(0x6F, memsize);  // FRETOP - end of string storage
+    writePtr(0xAF, endAddr);  // PRGEND - end of program
+
+    // Set interpreter state for direct mode
+    writePtr(0xB8, txttab - 1); // TXTPTR
+    writeMem(0x76, 0xFF);       // CURLIN+1 high byte = $FF (direct mode)
+
+    return static_cast<int>(tokenizedLines.size());
+}
+
+} // namespace a2e

package/src/core/basic/basic_tokenizer.hpp

@@ -0,0 +1,26 @@
+/*
+ * basic_tokenizer.hpp - Applesoft BASIC tokenizer for direct memory insertion
+ *
+ * Written by
+ *  Mike Daley <michael_daley@icloud.com>
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+#include <functional>
+
+namespace a2e {
+
+// Callback types for reading/writing emulator memory
+using MemReadFn = std::function<uint8_t(uint16_t)>;
+using MemWriteFn = std::function<void(uint16_t, uint8_t)>;
+
+// Tokenize BASIC source text and load it into emulator memory.
+// Writes the tokenized program at TXTTAB (0x0801), sets zero page pointers.
+// Uses callbacks to read/write memory (main RAM, bypassing ALTZP).
+// Returns the number of lines loaded, or -1 on error.
+int loadBasicProgram(const char* source, MemReadFn readMem, MemWriteFn writeMem);
+
+} // namespace a2e

package/src/core/basic/basic_tokens.hpp

@@ -0,0 +1,295 @@
+/*
+ * basic_tokens.hpp - Applesoft and Integer BASIC token tables and keyword helpers
+ *
+ * Written by
+ *  Mike Daley <michael_daley@icloud.com>
+ */
+
+#pragma once
+
+#include <cstring>
+
+namespace a2e {
+
+// Applesoft BASIC tokens - index = token byte - 0x80
+static constexpr const char* APPLESOFT_TOKENS[] = {
+  "END",      // $80
+  "FOR",      // $81
+  "NEXT",     // $82
+  "DATA",     // $83
+  "INPUT",    // $84
+  "DEL",      // $85
+  "DIM",      // $86
+  "READ",     // $87
+  "GR",       // $88
+  "TEXT",     // $89
+  "PR#",      // $8A
+  "IN#",      // $8B
+  "CALL",     // $8C
+  "PLOT",     // $8D
+  "HLIN",     // $8E
+  "VLIN",     // $8F
+  "HGR2",     // $90
+  "HGR",      // $91
+  "HCOLOR=",  // $92
+  "HPLOT",    // $93
+  "DRAW",     // $94
+  "XDRAW",    // $95
+  "HTAB",     // $96
+  "HOME",     // $97
+  "ROT=",     // $98
+  "SCALE=",   // $99
+  "SHLOAD",   // $9A
+  "TRACE",    // $9B
+  "NOTRACE",  // $9C
+  "NORMAL",   // $9D
+  "INVERSE",  // $9E
+  "FLASH",    // $9F
+  "COLOR=",   // $A0
+  "POP",      // $A1
+  "VTAB",     // $A2
+  "HIMEM:",   // $A3
+  "LOMEM:",   // $A4
+  "ONERR",    // $A5
+  "RESUME",   // $A6
+  "RECALL",   // $A7
+  "STORE",    // $A8
+  "SPEED=",   // $A9
+  "LET",      // $AA
+  "GOTO",     // $AB
+  "RUN",      // $AC
+  "IF",       // $AD
+  "RESTORE",  // $AE
+  "&",        // $AF
+  "GOSUB",    // $B0
+  "RETURN",   // $B1
+  "REM",      // $B2
+  "STOP",     // $B3
+  "ON",       // $B4
+  "WAIT",     // $B5
+  "LOAD",     // $B6
+  "SAVE",     // $B7
+  "DEF",      // $B8
+  "POKE",     // $B9
+  "PRINT",    // $BA
+  "CONT",     // $BB
+  "LIST",     // $BC
+  "CLEAR",    // $BD
+  "GET",      // $BE
+  "NEW",      // $BF
+  "TAB(",     // $C0
+  "TO",       // $C1
+  "FN",       // $C2
+  "SPC(",     // $C3
+  "THEN",     // $C4
+  "AT",       // $C5
+  "NOT",      // $C6
+  "STEP",     // $C7
+  "+",        // $C8
+  "-",        // $C9
+  "*",        // $CA
+  "/",        // $CB
+  "^",        // $CC
+  "AND",      // $CD
+  "OR",       // $CE
+  ">",        // $CF
+  "=",        // $D0
+  "<",        // $D1
+  "SGN",      // $D2
+  "INT",      // $D3
+  "ABS",      // $D4
+  "USR",      // $D5
+  "FRE",      // $D6
+  "SCRN(",    // $D7
+  "PDL",      // $D8
+  "POS",      // $D9
+  "SQR",      // $DA
+  "RND",      // $DB
+  "LOG",      // $DC
+  "EXP",      // $DD
+  "COS",      // $DE
+  "SIN",      // $DF
+  "TAN",      // $E0
+  "ATN",      // $E1
+  "PEEK",     // $E2
+  "LEN",      // $E3
+  "STR$",     // $E4
+  "VAL",      // $E5
+  "ASC",      // $E6
+  "CHR$",     // $E7
+  "LEFT$",    // $E8
+  "RIGHT$",   // $E9
+  "MID$",     // $EA
+};
+
+static constexpr int APPLESOFT_TOKEN_COUNT = sizeof(APPLESOFT_TOKENS) / sizeof(APPLESOFT_TOKENS[0]);
+
+// Integer BASIC tokens ($00-$7F)
+static constexpr const char* INTEGER_BASIC_TOKENS[] = {
+  " HIMEM: ",  // $00
+  nullptr,     // $01 (end of line)
+  "_",         // $02
+  ":",         // $03
+  " LOAD ",    // $04
+  " SAVE ",    // $05
+  " CON ",     // $06
+  " RUN ",     // $07
+  " RUN ",     // $08
+  " DEL ",     // $09
+  ",",         // $0A
+  " NEW ",     // $0B
+  " CLR ",     // $0C
+  " AUTO ",    // $0D
+  ",",         // $0E
+  " MAN ",     // $0F
+  " HIMEM: ",  // $10
+  " LOMEM: ",  // $11
+  "+",         // $12
+  "-",         // $13
+  "*",         // $14
+  "/",         // $15
+  "=",         // $16
+  "#",         // $17
+  ">=",        // $18
+  ">",         // $19
+  "<=",        // $1A
+  "<>",        // $1B
+  "<",         // $1C
+  " AND ",     // $1D
+  " OR ",      // $1E
+  " MOD ",     // $1F
+  "^",         // $20
+  "+",         // $21
+  "(",         // $22
+  ",",         // $23
+  " THEN ",    // $24
+  " THEN ",    // $25
+  ",",         // $26
+  ",",         // $27
+  "\"",        // $28 (start quote)
+  "\"",        // $29 (end quote)
+  "(",         // $2A
+  "!",         // $2B
+  "!",         // $2C
+  "(",         // $2D
+  "PEEK",      // $2E
+  "RND",       // $2F
+  "SGN",       // $30
+  "ABS",       // $31
+  "PDL",       // $32
+  "RNDX",      // $33
+  "(",         // $34
+  "+",         // $35
+  "-",         // $36
+  " NOT ",     // $37
+  "(",         // $38
+  "=",         // $39
+  "#",         // $3A
+  "LEN(",      // $3B
+  "ASC(",      // $3C
+  "SCRN(",     // $3D
+  ",",         // $3E
+  "(",         // $3F
+  "$",         // $40
+  "$",         // $41
+  "(",         // $42
+  ",",         // $43
+  ",",         // $44
+  ";",         // $45
+  ";",         // $46
+  ";",         // $47
+  ",",         // $48
+  ",",         // $49
+  ",",         // $4A
+  " TEXT ",    // $4B
+  " GR ",      // $4C
+  " CALL ",    // $4D
+  " DIM ",     // $4E
+  " DIM ",     // $4F
+  " TAB ",     // $50
+  " END ",     // $51
+  " INPUT ",   // $52
+  " INPUT ",   // $53
+  " INPUT ",   // $54
+  " FOR ",     // $55
+  "=",         // $56
+  " TO ",      // $57
+  " STEP ",    // $58
+  " NEXT ",    // $59
+  ",",         // $5A
+  " RETURN ",  // $5B
+  " GOSUB ",   // $5C
+  " REM ",     // $5D
+  " LET ",     // $5E
+  " GOTO ",    // $5F
+  " IF ",      // $60
+  " PRINT ",   // $61
+  " PRINT ",   // $62
+  " PRINT ",   // $63
+  " POKE ",    // $64
+  ",",         // $65
+  " COLOR= ",  // $66
+  " PLOT ",    // $67
+  ",",         // $68
+  " HLIN ",    // $69
+  ",",         // $6A
+  " AT ",      // $6B
+  " VLIN ",    // $6C
+  ",",         // $6D
+  " AT ",      // $6E
+  " VTAB ",    // $6F
+  "=",         // $70
+  "=",         // $71
+  ")",         // $72
+  ")",         // $73
+  " LIST ",    // $74
+  ",",         // $75
+  " LIST ",    // $76
+  " POP ",     // $77
+  " NODSP ",   // $78
+  " DSP ",     // $79
+  " NOTRACE ", // $7A
+  " DSP ",     // $7B
+  " DSP ",     // $7C
+  " TRACE ",   // $7D
+  " PR# ",     // $7E
+  " IN# ",     // $7F
+};
+
+static constexpr int INTEGER_BASIC_TOKEN_COUNT = 0x80;
+
+// Keywords that need space before them (Applesoft)
+inline bool needsSpaceBefore(const char* token) {
+  // Check common keywords
+  const char* keywords[] = {
+    "FOR", "NEXT", "DATA", "INPUT", "DIM", "READ", "GR", "TEXT",
+    "CALL", "PLOT", "HLIN", "VLIN", "HGR2", "HGR", "HPLOT",
+    "DRAW", "XDRAW", "HTAB", "HOME", "SHLOAD", "TRACE", "NOTRACE",
+    "NORMAL", "INVERSE", "FLASH", "POP", "VTAB", "ONERR", "RESUME",
+    "RECALL", "STORE", "LET", "GOTO", "RUN", "IF", "RESTORE",
+    "GOSUB", "RETURN", "REM", "STOP", "ON", "WAIT", "LOAD", "SAVE",
+    "DEF", "POKE", "PRINT", "CONT", "LIST", "CLEAR", "GET", "NEW",
+    "TO", "FN", "THEN", "AT", "NOT", "STEP", "AND", "OR", "END",
+  };
+  for (const char* kw : keywords) {
+    if (strcmp(token, kw) == 0) return true;
+  }
+  return false;
+}
+
+// Keywords that need space after them (Applesoft)
+inline bool needsSpaceAfter(const char* token) {
+  const char* keywords[] = {
+    "GOTO", "GOSUB", "THEN", "TO", "STEP", "AND", "OR", "NOT",
+    "IF", "ON", "LET", "FOR", "NEXT", "PRINT", "INPUT", "READ",
+    "DATA", "DIM", "DEF", "POKE", "CALL", "PLOT", "HLIN", "VLIN",
+    "HPLOT", "DRAW", "XDRAW", "HTAB", "VTAB", "ONERR", "WAIT",
+    "GET", "AT", "FN",
+  };
+  for (const char* kw : keywords) {
+    if (strcmp(token, kw) == 0) return true;
+  }
+  return false;
+}
+
+} // namespace a2e