Barmaja-PyArduino 2.0.0__py3-none-any.whl

Barmaja_PyArduino/toolchain.py

@@ -0,0 +1,506 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from .board import UNO, BoardProfile
+from .environment import ensure_runtime_dirs, env_seconds, find_normal_arduino_cli, find_pictoblox_arduino_cli, invalidate_cache, run_text
+from .errors import BoardDetectionError, HardwareError, InternalCompilerError, ToolchainError
+
+# Common USB-UART bridges used on Uno-compatible classroom boards.
+KNOWN_UNO_BRIDGES = {
+    ("1a86", "7523"): "CH340/CH341",
+    ("1a86", "5523"): "CH340/CH341",
+    ("10c4", "ea60"): "CP210x",
+    ("0403", "6001"): "FTDI",
+}
+GENUINE_UNO_IDS = {
+    ("2341", "0043"), ("2341", "0001"), ("2a03", "0043"), ("2a03", "0001"),
+}
+
+@dataclass(frozen=True)
+class DetectedBoard:
+    port: str
+    fqbn: str
+    name: str
+    confidence: str = "exact"  # exact / probable-clone
+    transport: str | None = None
+
+@dataclass(frozen=True)
+class BuildMemory:
+    flash_used: int | None = None
+    flash_total: int | None = None
+    sram_used: int | None = None
+    sram_total: int | None = None
+
+    @property
+    def flash_percent(self) -> float | None:
+        if self.flash_used is None or not self.flash_total:
+            return None
+        return self.flash_used * 100.0 / self.flash_total
+
+    @property
+    def sram_percent(self) -> float | None:
+        if self.sram_used is None or not self.sram_total:
+            return None
+        return self.sram_used * 100.0 / self.sram_total
+
+@dataclass(frozen=True)
+class CompileArtifact:
+    sketch_dir: Path
+    build_dir: Path
+    memory: BuildMemory
+    output: str
+
+
+def bundled_arduino_cli() -> str | None:
+    """The pinned Barmaja runtime installed by the Full Offline Pack, if present."""
+    dirs = ensure_runtime_dirs()
+    names = ("arduino-cli.exe", "arduino-cli") if os.name == "nt" else ("arduino-cli",)
+    for name in names:
+        p = dirs["root"] / "runtime" / name
+        if p.exists():
+            return str(p)
+    return None
+
+
+def find_arduino_cli() -> str:
+    """Select the Arduino CLI used for every Barmaja compile/upload.
+
+    Order: explicit BARMAJA_ARDUINO_CLI override, then the pinned Barmaja runtime
+    (Full Offline Pack), then an existing Arduino installation (Light Pack reuse),
+    then a PictoBlox-bundled CLI. The pinned runtime wins over an unrelated
+    Arduino IDE so a classroom laptop always compiles with the tested toolchain,
+    even for Windows accounts that never opened the Arduino IDE.
+    """
+    override = os.environ.get("BARMAJA_ARDUINO_CLI")
+    if override and Path(override).exists():
+        return str(Path(override).resolve())
+    bundled = bundled_arduino_cli()
+    if bundled:
+        return bundled
+    found = find_normal_arduino_cli()
+    if found:
+        return found
+    # PictoBlox is only reused when an actual Arduino CLI executable is discoverable.
+    # Capability/version validation is then performed by Smart Doctor.
+    pb = find_pictoblox_arduino_cli()
+    if pb:
+        return pb
+    raise ToolchainError(
+        "Arduino compilation environment was not found. Install Arduino support or use the Barmaja PyArduino Offline Pack."
+    )
+
+
+def _run(args: list[str], *, timeout: float = 30.0, cwd: str | Path | None = None):
+    return run_text(args, timeout=timeout, cwd=cwd)
+
+
+def _extract_ports(payload) -> list[dict]:
+    if isinstance(payload, dict):
+        for key in ("detected_ports", "ports"):
+            if isinstance(payload.get(key), list):
+                return payload[key]
+        if "port" in payload or "address" in payload:
+            return [payload]
+    if isinstance(payload, list):
+        return payload
+    return []
+
+
+def _hex(v) -> str | None:
+    if v is None:
+        return None
+    if isinstance(v, int):
+        return f"{v:04x}"
+    s = str(v).lower().replace("0x", "").strip()
+    return s.zfill(4) if re.fullmatch(r"[0-9a-f]+", s or "x") else None
+
+
+def _port_meta(item: dict) -> tuple[str | None, str | None, str | None, str | None]:
+    p = item.get("port", {}) if isinstance(item.get("port"), dict) else item
+    address = p.get("address") or item.get("address")
+    props = p.get("properties") or item.get("properties") or {}
+    vid = _hex(props.get("vid") or props.get("VID") or p.get("vid") or item.get("vid"))
+    pid = _hex(props.get("pid") or props.get("PID") or p.get("pid") or item.get("pid"))
+    protocol = p.get("protocol") or item.get("protocol")
+    return address, vid, pid, protocol
+
+
+def detect_board(profile: BoardProfile = UNO) -> DetectedBoard:
+    cli = find_arduino_cli()
+    result = _run([cli, "board", "list", "--format", "json"], timeout=12)
+    if result.returncode != 0:
+        invalidate_cache()
+        raise BoardDetectionError(
+            _translate_tool_failure(result.stderr or result.stdout, "Arduino CLI could not list connected boards."),
+            reason="list-failed",
+        )
+    try:
+        payload = json.loads(result.stdout or "{}")
+    except json.JSONDecodeError as e:
+        raise BoardDetectionError("Arduino CLI returned unreadable board information.", reason="unreadable") from e
+
+    # Arduino CLI may exit 0 even when a discovery helper failed to start.
+    # When no ports are available, do not misreport that as "board not connected".
+    if isinstance(payload, dict) and not _extract_ports(payload):
+        warnings = payload.get("warnings")
+        if isinstance(warnings, list):
+            warning_text = " | ".join(str(w) for w in warnings if w)
+        elif warnings:
+            warning_text = str(warnings)
+        else:
+            warning_text = ""
+        if warning_text and "discovery" in warning_text.lower():
+            invalidate_cache()
+            raise BoardDetectionError(
+                "Barmaja could not scan USB ports because the Arduino port scanner failed. "
+                "Run Smart Doctor or reinstall the Full Offline package.",
+                reason="list-failed",
+                details=warning_text,
+            )
+
+    exact: list[DetectedBoard] = []
+    fallback: list[DetectedBoard] = []
+    seen_ports: list[str] = []
+    for item in _extract_ports(payload):
+        if not isinstance(item, dict):
+            continue
+        address, vid, pid, protocol = _port_meta(item)
+        if address:
+            seen_ports.append(address)
+        boards = item.get("matching_boards") or item.get("boards") or []
+        for board in boards:
+            fqbn = board.get("fqbn")
+            name = board.get("name", "Arduino")
+            if address and fqbn == profile.fqbn:
+                exact.append(DetectedBoard(address, fqbn, name, "exact", protocol))
+        if address and vid and pid:
+            key = (vid, pid)
+            if key in GENUINE_UNO_IDS:
+                fallback.append(DetectedBoard(address, profile.fqbn, profile.name, "exact", protocol))
+            elif key in KNOWN_UNO_BRIDGES:
+                fallback.append(DetectedBoard(address, profile.fqbn, f"Uno-compatible via {KNOWN_UNO_BRIDGES[key]}", "probable-clone", protocol))
+
+    exact_unique = {b.port: b for b in exact}
+    if len(exact_unique) == 1:
+        return next(iter(exact_unique.values()))
+    if len(exact_unique) > 1:
+        ports = tuple(sorted(exact_unique))
+        raise BoardDetectionError(
+            f"More than one {profile.name} is connected: {', '.join(ports)}. Disconnect extra boards, or choose one with: pyarduino upload <file.py> --port {ports[0]}",
+            reason="multiple", ports=ports,
+        )
+
+    # Safe clone fallback only when exactly one known USB-UART candidate exists.
+    fallback_unique = {b.port: b for b in fallback}
+    if len(fallback_unique) == 1:
+        return next(iter(fallback_unique.values()))
+    if len(fallback_unique) > 1:
+        ports = ", ".join(sorted(fallback_unique))
+        choices = tuple(sorted(fallback_unique))
+        raise BoardDetectionError(
+            f"Several possible Uno-compatible serial devices were detected: {ports}. Barmaja will not guess which one to upload to. Choose one with: pyarduino upload <file.py> --port {choices[0]}",
+            reason="multiple", ports=choices,
+        )
+
+    if seen_ports:
+        ports = tuple(sorted(set(seen_ports)))
+        raise BoardDetectionError(
+            f"No {profile.name} was identified. Ports seen: {', '.join(ports)}.",
+            reason="not-found", ports=ports,
+        )
+    raise BoardDetectionError(
+        f"No {profile.name} was identified. Connect the board and make sure its USB serial driver is available.",
+        reason="not-found", ports=(),
+    )
+
+
+_KEEP_BUILD_DIRS = 4
+
+
+def _prune_old_builds(build_root: Path, keep: int = _KEEP_BUILD_DIRS) -> None:
+    """Keep only the newest build folders; each one holds megabytes of objects."""
+    try:
+        dirs = [d for d in build_root.iterdir() if d.is_dir()]
+        dirs.sort(key=lambda d: d.stat().st_mtime, reverse=True)
+        for old in dirs[keep:]:
+            shutil.rmtree(old, ignore_errors=True)
+    except OSError:
+        pass
+
+
+def create_sketch(cpp: str, work_root: str | Path | None = None) -> Path:
+    if work_root:
+        root = Path(work_root)
+    else:
+        build_root = ensure_runtime_dirs()["build"]
+        _prune_old_builds(build_root, keep=_KEEP_BUILD_DIRS - 1)
+        root = build_root / next(tempfile._get_candidate_names())
+    sketch_dir = root / "BarmajaGenerated"
+    sketch_dir.mkdir(parents=True, exist_ok=True)
+    (sketch_dir / "BarmajaGenerated.ino").write_text(cpp, encoding="utf-8")
+    return sketch_dir
+
+
+def _memory_from_output(text: str, profile: BoardProfile) -> BuildMemory:
+    flash_used = sram_used = None
+    flash_total, sram_total = profile.flash_bytes, profile.sram_bytes
+    # Arduino CLI English output (Barmaja forces ARDUINO_LOCALE=en):
+    # "Sketch uses 924 bytes (2%) of program storage space. Maximum is 32256 bytes."
+    # "Global variables use 9 bytes (0%) of dynamic memory, ... Maximum is 2048 bytes."
+    m = re.search(r"Sketch uses\s+([\d,]+)\s+bytes", text, re.I)
+    if m:
+        flash_used = int(m.group(1).replace(",", ""))
+        mx = re.search(r"program storage space\.\s*Maximum is\s+([\d,]+)", text, re.I)
+        if mx:
+            flash_total = int(mx.group(1).replace(",", ""))
+    m = re.search(r"Global variables use\s+([\d,]+)\s+bytes", text, re.I)
+    if m:
+        sram_used = int(m.group(1).replace(",", ""))
+        mx = re.search(r"dynamic memory.*?Maximum is\s+([\d,]+)", text, re.I | re.S)
+        if mx:
+            sram_total = int(mx.group(1).replace(",", ""))
+    return BuildMemory(flash_used, flash_total if flash_used is not None else None, sram_used, sram_total if sram_used is not None else None)
+
+
+_MEMORY_OVERFLOW_MARKERS = (
+    "not within region", "region `data' overflowed", "region `text' overflowed", "region 'data' overflowed",
+    "region 'text' overflowed", "will not fit in region", "sketch too big", "not enough memory",
+    "text section exceeds available space", "data section exceeds available space",
+)
+
+
+def _is_memory_overflow(text: str) -> bool:
+    low = (text or "").lower()
+    return any(marker in low for marker in _MEMORY_OVERFLOW_MARKERS)
+
+
+def _missing_header(text: str) -> str | None:
+    m = re.search(r"fatal error:\s*([\w./-]+\.h):\s*No such file", text or "", re.I)
+    return m.group(1) if m else None
+
+
+def _generated_code_errors(text: str) -> list[str]:
+    """Compiler errors located in the Barmaja-generated sketch itself."""
+    return [line.strip() for line in (text or "").splitlines()
+            if "BarmajaGenerated.ino" in line and re.search(r"\berror:", line)]
+
+
+def _translate_tool_failure(text: str, default: str) -> str:
+    low = (text or "").lower()
+    if _is_memory_overflow(text):
+        return "The program is too big for the Arduino Uno memory. Use shorter lists/strings or fewer libraries."
+    header = _missing_header(text)
+    if header:
+        return f"A required Arduino library is missing ({header}). Run 'pyarduino doctor' for the missing library list."
+    if "access is denied" in low or "permission denied" in low or "resource busy" in low or "can't open device" in low or "cannot open port" in low:
+        return "The Arduino port is busy or unavailable. Close Serial Monitor/PictoBlox/other Arduino tools and try again."
+    if "programmer is not responding" in low or "stk500_recv" in low or "not in sync" in low:
+        return "Arduino Uno did not respond during upload. Check the selected board, USB cable, and COM port."
+    if "no such file or directory" in low and ("avr-g" in low or "avrdude" in low or "ctags" in low):
+        return "The Arduino compiler environment is incomplete. Run 'pyarduino doctor' or use the Barmaja Offline Pack."
+    if "library" in low and ("not found" in low or "no such file" in low):
+        return "A required Arduino library is missing. Run 'pyarduino doctor' for the missing library list."
+    if "timed out" in low:
+        return "The Arduino tool timed out. Reconnect the board and try again."
+    compact = " ".join((text or "").strip().splitlines()[-3:])
+    return compact[:700] if compact else default
+
+
+def pinned_library_dirs() -> list[Path]:
+    """Pinned BARMAJA libraries from both root spellings (installed first)."""
+    from .environment import private_arduino_library_roots
+    found: dict[str, Path] = {}
+    for private in private_arduino_library_roots():
+        for path in sorted(private.iterdir(), key=lambda p: p.name.lower()):
+            if path.is_dir() and (path / "library.properties").exists():
+                found.setdefault(path.name.lower(), path)
+    return [found[key] for key in sorted(found)]
+
+
+def _private_library_args() -> list[str]:
+    """Pin each Barmaja library with --library (top priority in arduino-cli).
+
+    --libraries only adds a search folder; a same-named header in the user's
+    sketchbook (e.g. a random 'DHT' zip library) could still win. --library
+    makes the tested pinned copy authoritative for its headers.
+    """
+    args: list[str] = []
+    for lib in pinned_library_dirs():
+        args += ["--library", str(lib)]
+    # Device-pack adapters ship inside the Python package (or external pack folders).
+    from .device_packs import adapter_library_dirs
+    for lib in adapter_library_dirs():
+        args += ["--library", str(lib)]
+    # PRIVATE Stage-2 test only.  This opt-in path does not modify the frozen
+    # V1 runtime's arduino-libraries folder or the teacher installer.
+    dev_root = os.environ.get("BARMAJA_DEV_LIBRARIES", "").strip()
+    if dev_root:
+        candidate = Path(dev_root).expanduser()
+        if not candidate.is_dir():
+            raise ToolchainError(f"BARMAJA_DEV_LIBRARIES does not exist: {candidate}")
+        for lib in sorted(candidate.iterdir()):
+            if lib.is_dir() and (lib / "library.properties").is_file():
+                args += ["--library", str(lib)]
+    return args
+
+
+def _cli_config_args(cli: str | None = None) -> list[str]:
+    cfg = os.environ.get("BARMAJA_ARDUINO_CONFIG")
+    if cfg and Path(cfg).exists():
+        return ["--config-file", cfg]
+    bundled = ensure_runtime_dirs()["root"] / "runtime" / "arduino-cli.yaml"
+    if bundled.exists():
+        # Only force the bundled config for the bundled CLI. Existing Arduino
+        # installations keep their own configuration untouched.
+        if cli is None:
+            return ["--config-file", str(bundled)]
+        try:
+            cli_path = Path(cli).resolve()
+            runtime_root = (ensure_runtime_dirs()["root"] / "runtime").resolve()
+            if cli_path.parent == runtime_root:
+                return ["--config-file", str(bundled)]
+        except OSError:
+            pass
+    return []
+
+
+def compile_sketch(cpp: str, profile: BoardProfile = UNO, work_root: str | Path | None = None) -> CompileArtifact:
+    if os.environ.get("BARMAJA_SKIP_DOCTOR") != "1":
+        from .doctor import ensure_environment_ready
+        ensure_environment_ready()
+    cli = find_arduino_cli()
+    sketch = create_sketch(cpp, work_root)
+    build_dir = sketch.parent / "build"
+    build_dir.mkdir(parents=True, exist_ok=True)
+    args = [cli, *_cli_config_args(cli), "compile", "--fqbn", profile.fqbn, "--warnings", "all", *_private_library_args(), "--build-path", str(build_dir), str(sketch)]
+    try:
+        result = _run(args, timeout=env_seconds("BARMAJA_COMPILE_TIMEOUT", 180.0))
+    except ToolchainError:
+        # A timeout/start failure may mean the compiler/runtime is damaged. Never
+        # leave a READY cache behind after an external compile failure.
+        invalidate_cache()
+        raise
+    output = ((result.stdout or "") + "\n" + (result.stderr or "")).strip()
+    if result.returncode != 0:
+        if _is_memory_overflow(output):
+            # A real, student-fixable limit; the environment is fine.
+            raise HardwareError(
+                "The program is too big for the Arduino Uno memory (2048 bytes of RAM, 32256 bytes of program space). "
+                "Use shorter lists and strings, or fewer libraries at the same time.",
+                details=" ".join(output.splitlines()[-3:])[:700],
+            )
+        generated = _generated_code_errors(output)
+        if generated and not _missing_header(output):
+            # An accepted Python program must always produce valid C++. Do not
+            # blame (or re-diagnose) the student's Arduino environment.
+            raise InternalCompilerError(
+                "Barmaja PyArduino generated Arduino code that did not compile. This is a Barmaja bug, not a mistake in your program.",
+                details="\n".join(generated[:5])[:1500],
+            )
+        invalidate_cache()
+        raise ToolchainError("Arduino compilation failed. " + _translate_tool_failure(output, "Arduino compilation failed."))
+    memory = _memory_from_output(output, profile)
+    _check_real_memory(memory, profile)
+    return CompileArtifact(sketch, build_dir, memory, output)
+
+
+def _check_real_memory(memory: BuildMemory, profile: BoardProfile) -> None:
+    """Enforce the student SRAM safety limit on the real linker result."""
+    if memory.sram_used is None or not memory.sram_total:
+        return
+    ratio = memory.sram_used / memory.sram_total
+    if ratio >= profile.sram_error_ratio:
+        raise HardwareError(
+            f"The program uses {memory.sram_used} of {memory.sram_total} bytes of Arduino Uno RAM ({ratio:.0%}). "
+            f"Above {profile.sram_error_ratio:.0%} the board can crash or restart while running. "
+            "Use shorter lists and strings, or fewer libraries at the same time."
+        )
+
+
+def board_for_port(port: str, profile: BoardProfile = UNO) -> DetectedBoard:
+    """Manual fallback: the student/teacher named the COM port explicitly."""
+    port = (port or "").strip()
+    if not port:
+        raise BoardDetectionError("No serial port was given.")
+    return DetectedBoard(port, profile.fqbn, profile.name, "manual", "serial")
+
+
+def compile_existing_sketch(sketch_dir: str | Path, profile: BoardProfile = UNO) -> CompileArtifact:
+    """Compile an existing Arduino sketch directory with Barmaja's pinned toolchain.
+
+    LiveRun uses this for the bundled Telemetrix4Arduino server sketch. The
+    student's standalone compiler still uses ``compile_sketch()`` above.
+    """
+    if os.environ.get("BARMAJA_SKIP_DOCTOR") != "1":
+        from .doctor import ensure_environment_ready
+        ensure_environment_ready()
+
+    sketch = Path(sketch_dir).expanduser().resolve()
+    if not sketch.is_dir() or not any(sketch.glob("*.ino")):
+        raise ToolchainError(f"Arduino sketch folder is missing or invalid: {sketch}")
+
+    cli = find_arduino_cli()
+    build_root = ensure_runtime_dirs()["build"]
+    _prune_old_builds(build_root, keep=_KEEP_BUILD_DIRS - 1)
+    root = build_root / next(tempfile._get_candidate_names())
+    build_dir = root / "build"
+    build_dir.mkdir(parents=True, exist_ok=True)
+
+    args = [
+        cli, *_cli_config_args(cli), "compile", "--fqbn", profile.fqbn,
+        "--warnings", "all", *_private_library_args(),
+        "--build-path", str(build_dir), str(sketch),
+    ]
+    try:
+        result = _run(args, timeout=env_seconds("BARMAJA_COMPILE_TIMEOUT", 180.0))
+    except ToolchainError:
+        invalidate_cache()
+        raise
+
+    output = ((result.stdout or "") + "\n" + (result.stderr or "")).strip()
+    if result.returncode != 0:
+        invalidate_cache()
+        raise ToolchainError(
+            "Arduino firmware compilation failed. "
+            + _translate_tool_failure(output, "Arduino firmware compilation failed.")
+        )
+    memory = _memory_from_output(output, profile)
+    _check_real_memory(memory, profile)
+    return CompileArtifact(sketch, build_dir, memory, output)
+
+
+def upload_compiled(artifact: CompileArtifact, board: DetectedBoard, profile: BoardProfile = UNO) -> Path:
+    """Upload an already-compiled sketch without compiling it a second time."""
+    cli = find_arduino_cli()
+    target = board
+    if target.fqbn != profile.fqbn:
+        raise ToolchainError(f"Compile/upload board mismatch: expected {profile.fqbn}, got {target.fqbn}.")
+    args = [cli, *_cli_config_args(cli), "upload", "-p", target.port, "--fqbn", profile.fqbn, "--input-dir", str(artifact.build_dir), str(artifact.sketch_dir)]
+    try:
+        result = _run(args, timeout=env_seconds("BARMAJA_UPLOAD_TIMEOUT", 90.0))
+    except ToolchainError:
+        invalidate_cache()
+        raise
+    if result.returncode != 0:
+        # Upload failures are usually hardware/port problems; board/COM state is
+        # never trusted from the environment cache, and the cache is invalidated
+        # so the next run re-verifies the toolchain as well.
+        invalidate_cache()
+        details = ((result.stderr or "") + "\n" + (result.stdout or "")).strip()
+        raise ToolchainError("Arduino upload failed. " + _translate_tool_failure(details, "Arduino upload failed."))
+    return artifact.sketch_dir
+
+
+def upload_sketch(cpp: str, board: DetectedBoard | None = None, profile: BoardProfile = UNO) -> Path:
+    if board is None:
+        port = os.environ.get("BARMAJA_PORT")
+        board = board_for_port(port, profile) if port else detect_board(profile)
+    artifact = compile_sketch(cpp, profile)
+    return upload_compiled(artifact, board, profile)