fastled 1.3.30__py3-none-any.whl → 1.4.50__py3-none-any.whl

fastled/web_compile.py

@@ -1,35 +1,23 @@
-import _thread
 import io
-import json
 import os
 import shutil
 import tempfile
 import time
 import zipfile
-from concurrent.futures import Future, ThreadPoolExecutor, as_completed
-from dataclasses import dataclass
 from pathlib import Path
 
 import httpx
 
-from fastled.settings import SERVER_PORT
-from fastled.sketch import get_sketch_files
+from fastled.emoji_util import EMO, safe_print
+from fastled.find_good_connection import find_good_connection
+from fastled.interruptible_http import make_interruptible_post_request
+from fastled.settings import AUTH_TOKEN, SERVER_PORT
 from fastled.types import BuildMode, CompileResult
-from fastled.util import hash_file
+from fastled.zip_files import ZipResult, zip_files
 
 DEFAULT_HOST = "https://fastled.onrender.com"
 ENDPOINT_COMPILED_WASM = "compile/wasm"
-_TIMEOUT = 60 * 4  # 2 mins timeout
-_AUTH_TOKEN = "oBOT5jbsO4ztgrpNsQwlmFLIKB"
-ENABLE_EMBEDDED_DATA = True
-_EXECUTOR = ThreadPoolExecutor(max_workers=8)
-
-
-@dataclass
-class ConnectionResult:
-    host: str
-    success: bool
-    ipv4: bool
+_TIMEOUT = 60 * 4  # 4 mins timeout
 
 
 def _sanitize_host(host: str) -> str:
@@ -42,120 +30,36 @@ def _sanitize_host(host: str) -> str:
     return host if host.startswith("http://") else f"http://{host}"
 
 
-def _test_connection(host: str, use_ipv4: bool) -> ConnectionResult:
-    # Function static cache
-    host = _sanitize_host(host)
-    transport = httpx.HTTPTransport(local_address="0.0.0.0") if use_ipv4 else None
-    result: ConnectionResult | None = None
-    try:
-        with httpx.Client(
-            timeout=_TIMEOUT,
-            transport=transport,
-        ) as test_client:
-            test_response = test_client.get(
-                f"{host}/healthz", timeout=3, follow_redirects=True
-            )
-            result = ConnectionResult(host, test_response.status_code == 200, use_ipv4)
-    except KeyboardInterrupt:
-        _thread.interrupt_main()
-        result = ConnectionResult(host, False, use_ipv4)
-    except TimeoutError:
-        result = ConnectionResult(host, False, use_ipv4)
-    except Exception:
-        result = ConnectionResult(host, False, use_ipv4)
-    return result
-
-
-def _file_info(file_path: Path) -> str:
-    hash_txt = hash_file(file_path)
-    file_size = file_path.stat().st_size
-    json_str = json.dumps({"hash": hash_txt, "size": file_size})
-    return json_str
-
-
-@dataclass
-class ZipResult:
-    zip_bytes: bytes
-    zip_embedded_bytes: bytes | None
-    success: bool
-    error: str | None
-
-
-def zip_files(directory: Path, build_mode: BuildMode) -> ZipResult | Exception:
-    print("Zipping files...")
-    try:
-        files = get_sketch_files(directory)
-        if not files:
-            raise FileNotFoundError(f"No files found in {directory}")
-        for f in files:
-            print(f"Adding file: {f}")
-        # Create in-memory zip file
-        has_embedded_zip = False
-        zip_embedded_buffer = io.BytesIO()
-        zip_buffer = io.BytesIO()
-        with zipfile.ZipFile(
-            zip_embedded_buffer, "w", zipfile.ZIP_DEFLATED, compresslevel=9
-        ) as emebedded_zip_file:
-            with zipfile.ZipFile(
-                zip_buffer, "w", zipfile.ZIP_DEFLATED, compresslevel=9
-            ) as zip_file:
-                for file_path in files:
-                    if "fastled_js" in str(file_path):
-                        # These can be huge, don't send the output files back to the server!
-                        continue
-                    relative_path = file_path.relative_to(directory)
-                    achive_path = str(Path("wasm") / relative_path)
-                    if str(relative_path).startswith("data") and ENABLE_EMBEDDED_DATA:
-                        _file_info_str = _file_info(file_path)
-                        zip_file.writestr(
-                            achive_path + ".embedded.json", _file_info_str
-                        )
-                        emebedded_zip_file.write(file_path, relative_path)
-                        has_embedded_zip = True
-                    else:
-                        zip_file.write(file_path, achive_path)
-                # write build mode into the file as build.txt so that sketches are fingerprinted
-                # based on the build mode. Otherwise the same sketch with different build modes
-                # will have the same fingerprint.
-                zip_file.writestr(
-                    str(Path("wasm") / "build_mode.txt"), build_mode.value
-                )
-        result = ZipResult(
-            zip_bytes=zip_buffer.getvalue(),
-            zip_embedded_bytes=(
-                zip_embedded_buffer.getvalue() if has_embedded_zip else None
-            ),
-            success=True,
-            error=None,
-        )
-        return result
-    except Exception as e:
-        return e
-
-
-def find_good_connection(
-    urls: list[str], filter_out_bad=True, use_ipv6: bool = True
-) -> ConnectionResult | None:
-    futures: list[Future] = []
-    for url in urls:
-
-        f = _EXECUTOR.submit(_test_connection, url, use_ipv4=True)
-        futures.append(f)
-        if use_ipv6 and "localhost" not in url:
-            f_v6 = _EXECUTOR.submit(_test_connection, url, use_ipv4=False)
-            futures.append(f_v6)
+def _check_embedded_http_status(response_content: bytes) -> tuple[bool, int | None]:
+    """
+    Check if the response content has an embedded HTTP status at the end.
 
+    Returns:
+        tuple: (has_embedded_status, status_code)
+               has_embedded_status is True if an embedded status was found
+               status_code is the embedded status code or None if not found
+    """
     try:
-        # Return first successful result
-        for future in as_completed(futures):
-            result: ConnectionResult = future.result()
-            if result.success or not filter_out_bad:
-                return result
-    finally:
-        # Cancel any remaining futures
-        for future in futures:
-            future.cancel()
-    return None
+        # Convert bytes to string for parsing
+        content_str = response_content.decode("utf-8", errors="ignore")
+
+        # Look for HTTP_STATUS: at the end of the content
+        lines = content_str.strip().split("\n")
+        if lines:
+            last_line = lines[-1].strip()
+            if "HTTP_STATUS:" in last_line:
+                try:
+                    right = last_line.split("HTTP_STATUS:")[-1].strip()
+                    status_code = int(right)
+                    return True, status_code
+                except (ValueError, IndexError):
+                    # Malformed status line
+                    return True, None
+
+        return False, None
+    except Exception:
+        # If we can't parse the content, assume no embedded status
+        return False, None
 
 
 def _banner(msg: str) -> str:
@@ -186,7 +90,212 @@ def _banner(msg: str) -> str:
 
 
 def _print_banner(msg: str) -> None:
-    print(_banner(msg))
+    safe_print(_banner(msg))
+
+
+def _compile_libfastled(
+    host: str,
+    auth_token: str,
+    build_mode: BuildMode,
+) -> httpx.Response:
+    """Compile the FastLED library separately."""
+    host = _sanitize_host(host)
+    urls = [host]
+    domain = host.split("://")[-1]
+    if ":" not in domain:
+        urls.append(f"{host}:{SERVER_PORT}")
+
+    connection_result = find_good_connection(urls)
+    if connection_result is None:
+        raise ConnectionError(
+            "Connection failed to all endpoints for libfastled compilation"
+        )
+
+    ipv4_stmt = "IPv4" if connection_result.ipv4 else "IPv6"
+    transport = (
+        httpx.HTTPTransport(local_address="0.0.0.0") if connection_result.ipv4 else None
+    )
+
+    headers = {
+        "accept": "application/json",
+        "authorization": auth_token,
+        "build": build_mode.value.lower(),
+    }
+
+    url = f"{connection_result.host}/compile/libfastled"
+    print(f"Compiling libfastled on {url} via {ipv4_stmt}")
+
+    # Use interruptible HTTP request
+    response = make_interruptible_post_request(
+        url=url,
+        files={},  # No files for libfastled compilation
+        headers=headers,
+        transport=transport,
+        timeout=_TIMEOUT * 2,  # Give more time for library compilation
+        follow_redirects=False,
+    )
+
+    return response
+
+
+def _send_compile_request(
+    host: str,
+    zip_bytes: bytes,
+    auth_token: str,
+    build_mode: BuildMode,
+    profile: bool,
+    no_platformio: bool,
+) -> httpx.Response:
+    """Send the compile request to the server and return the response."""
+    host = _sanitize_host(host)
+    urls = [host]
+    domain = host.split("://")[-1]
+    if ":" not in domain:
+        urls.append(f"{host}:{SERVER_PORT}")
+
+    connection_result = find_good_connection(urls)
+    if connection_result is None:
+        raise ConnectionError("Connection failed to all endpoints")
+
+    ipv4_stmt = "IPv4" if connection_result.ipv4 else "IPv6"
+    transport = (
+        httpx.HTTPTransport(local_address="0.0.0.0") if connection_result.ipv4 else None
+    )
+
+    archive_size = len(zip_bytes)
+
+    headers = {
+        "accept": "application/json",
+        "authorization": auth_token,
+        "build": (
+            build_mode.value.lower() if build_mode else BuildMode.QUICK.value.lower()
+        ),
+        "profile": "true" if profile else "false",
+        "no-platformio": "true" if no_platformio else "false",
+        "allow-libcompile": "false",  # Always false since we handle it manually
+    }
+
+    url = f"{connection_result.host}/{ENDPOINT_COMPILED_WASM}"
+    print(f"Compiling sketch on {url} via {ipv4_stmt}. Zip size: {archive_size} bytes")
+    files = {"file": ("wasm.zip", zip_bytes, "application/x-zip-compressed")}
+
+    # Use interruptible HTTP request
+    response = make_interruptible_post_request(
+        url=url,
+        files=files,
+        headers=headers,
+        transport=transport,
+        timeout=_TIMEOUT,
+        follow_redirects=True,
+    )
+
+    return response
+
+
+def _process_compile_response(
+    response: httpx.Response,
+    zip_result: ZipResult,
+    start_time: float,
+    zip_time: float,
+    libfastled_time: float,
+    sketch_time: float,
+) -> CompileResult:
+    """Process the compile response and return the final result."""
+    if response.status_code != 200:
+        json_response = response.json()
+        detail = json_response.get("detail", "Could not compile")
+        return CompileResult(
+            success=False,
+            stdout=detail,
+            hash_value=None,
+            zip_bytes=b"",
+            zip_time=zip_time,
+            libfastled_time=libfastled_time,
+            sketch_time=sketch_time,
+            response_processing_time=0.0,  # No response processing in error case
+        )
+
+    print(f"Response status code: {response}")
+
+    # Time the response processing
+    response_processing_start = time.time()
+
+    # Create a temporary directory to extract the zip
+    with tempfile.TemporaryDirectory() as extract_dir:
+        extract_path = Path(extract_dir)
+
+        # Write the response content to a temporary zip file
+        temp_zip = extract_path / "response.zip"
+        temp_zip.write_bytes(response.content)
+
+        # Extract the zip
+        shutil.unpack_archive(temp_zip, extract_path, "zip")
+
+        if zip_result.zip_embedded_bytes:
+            # extract the embedded bytes, which were not sent to the server
+            temp_zip.write_bytes(zip_result.zip_embedded_bytes)
+            shutil.unpack_archive(temp_zip, extract_path, "zip")
+
+        # we don't need the temp zip anymore
+        temp_zip.unlink()
+
+        # Read stdout from out.txt if it exists
+        stdout_file = extract_path / "out.txt"
+        hash_file = extract_path / "hash.txt"
+        stdout = (
+            stdout_file.read_text(encoding="utf-8", errors="replace")
+            if stdout_file.exists()
+            else ""
+        )
+        hash_value = (
+            hash_file.read_text(encoding="utf-8", errors="replace")
+            if hash_file.exists()
+            else None
+        )
+
+        # now rezip the extracted files since we added the embedded json files
+        out_buffer = io.BytesIO()
+        with zipfile.ZipFile(
+            out_buffer, "w", zipfile.ZIP_DEFLATED, compresslevel=9
+        ) as out_zip:
+            for root, _, _files in os.walk(extract_path):
+                for file in _files:
+                    file_path = Path(root) / file
+                    relative_path = file_path.relative_to(extract_path)
+                    out_zip.write(file_path, relative_path)
+
+        response_processing_time = time.time() - response_processing_start
+        diff_time = time.time() - start_time
+
+        # Create detailed timing breakdown
+        unaccounted_time = diff_time - (
+            zip_time + libfastled_time + sketch_time + response_processing_time
+        )
+        msg = f"Compilation success, took {diff_time:.2f} seconds\n"
+        msg += f"     zip creation: {zip_time:.2f}\n"
+        if libfastled_time > 0:
+            msg += f"     libfastled: {libfastled_time:.2f}\n"
+        msg += f"     sketch compile + link: {sketch_time:.2f}\n"
+        msg += f"     response processing: {response_processing_time:.2f}\n"
+        if unaccounted_time > 0.01:  # Only show if significant
+            msg += f"     other overhead: {unaccounted_time:.2f}"
+
+        _print_banner(msg)
+        return CompileResult(
+            success=True,
+            stdout=stdout,
+            hash_value=hash_value,
+            zip_bytes=out_buffer.getvalue(),
+            zip_time=zip_time,
+            libfastled_time=libfastled_time,
+            sketch_time=sketch_time,
+            response_processing_time=response_processing_time,
+        )
+
+
+def _libcompiled_is_allowed(host: str) -> bool:
+    """Check if libcompiled is allowed for the given host."""
+    return "localhost" in host or "127.0.0.1" in host or "0.0.0.0" in host
 
 
 def web_compile(
@@ -195,6 +304,8 @@ def web_compile(
     auth_token: str | None = None,
     build_mode: BuildMode | None = None,
     profile: bool = False,
+    no_platformio: bool = False,
+    allow_libcompile: bool = True,
 ) -> CompileResult:
     start_time = time.time()
     if isinstance(directory, str):
@@ -202,124 +313,156 @@ def web_compile(
     host = _sanitize_host(host or DEFAULT_HOST)
     build_mode = build_mode or BuildMode.QUICK
     _print_banner(f"Compiling on {host}")
-    auth_token = auth_token or _AUTH_TOKEN
+    auth_token = auth_token or AUTH_TOKEN
     if not directory.exists():
         raise FileNotFoundError(f"Directory not found: {directory}")
-    zip_result = zip_files(directory, build_mode=build_mode)
+
+    if allow_libcompile and not _libcompiled_is_allowed(host):
+        print(
+            f"{EMO('🚫', '[ERROR]')} libcompile is not allowed for host {host}, disabling."
+        )
+        allow_libcompile = False
+
+    # Time the zip creation
+    zip_start_time = time.time()
+    zip_result: ZipResult | Exception = zip_files(directory, build_mode=build_mode)
+    zip_time = time.time() - zip_start_time
+
     if isinstance(zip_result, Exception):
         return CompileResult(
-            success=False, stdout=str(zip_result), hash_value=None, zip_bytes=b""
+            success=False,
+            stdout=str(zip_result),
+            hash_value=None,
+            zip_bytes=b"",
+            zip_time=zip_time,
+            libfastled_time=0.0,  # No libfastled compilation in zip error case
+            sketch_time=0.0,  # No sketch compilation in zip error case
+            response_processing_time=0.0,  # No response processing in zip error case
         )
     zip_bytes = zip_result.zip_bytes
-    archive_size = len(zip_bytes)
     print(f"Web compiling on {host}...")
+
+    # Track timing for each step
+    libfastled_time = 0.0
+    sketch_time = 0.0
+
     try:
-        host = _sanitize_host(host)
-        urls = [host]
-        domain = host.split("://")[-1]
-        if ":" not in domain:
-            urls.append(f"{host}:{SERVER_PORT}")
-
-        connection_result = find_good_connection(urls)
-        if connection_result is None:
-            _print_banner("Connection failed to all endpoints")
-            return CompileResult(
-                success=False,
-                stdout="Connection failed",
-                hash_value=None,
-                zip_bytes=b"",
-            )
-
-        ipv4_stmt = "IPv4" if connection_result.ipv4 else "IPv6"
-        transport = (
-            httpx.HTTPTransport(local_address="0.0.0.0")
-            if connection_result.ipv4
-            else None
+        # Step 1: Compile libfastled if requested
+        if allow_libcompile:
+            print("Step 1: Compiling libfastled...")
+            libfastled_start_time = time.time()
+            try:
+                libfastled_response = _compile_libfastled(host, auth_token, build_mode)
+
+                # Check HTTP response status first
+                if libfastled_response.status_code != 200:
+                    msg = f"Error: libfastled compilation failed with HTTP status {libfastled_response.status_code}"
+
+                    # Error out here, this is a critical error
+                    stdout = libfastled_response.content
+                    if stdout is not None:
+                        stdout = stdout.decode("utf-8", errors="replace") + "\n" + msg
+                    else:
+                        stdout = msg
+                    return CompileResult(
+                        success=False,
+                        stdout=stdout,
+                        hash_value=None,
+                        zip_bytes=b"",
+                        zip_time=zip_time,
+                        libfastled_time=libfastled_time,
+                        sketch_time=0.0,  # No sketch compilation when libfastled fails
+                        response_processing_time=0.0,  # No response processing when libfastled fails
+                    )
+                else:
+                    # Check for embedded HTTP status in response content
+                    has_embedded_status, embedded_status = _check_embedded_http_status(
+                        libfastled_response.content
+                    )
+                    if has_embedded_status:
+                        if embedded_status is not None and embedded_status != 200:
+                            msg = f"Warning: libfastled compilation failed with embedded status {embedded_status}"
+                            # this is a critical error
+                            stdout = libfastled_response.content
+                            if stdout is not None:
+                                stdout = (
+                                    stdout.decode("utf-8", errors="replace")
+                                    + "\n"
+                                    + msg
+                                )
+                            else:
+                                stdout = msg
+                            return CompileResult(
+                                success=False,
+                                stdout=stdout,
+                                hash_value=None,
+                                zip_bytes=b"",
+                                zip_time=zip_time,
+                                libfastled_time=libfastled_time,
+                                sketch_time=0.0,  # No sketch compilation when libfastled fails
+                                response_processing_time=0.0,  # No response processing when libfastled fails
+                            )
+                            # Continue with sketch compilation even if libfastled fails
+                        elif embedded_status is None:
+                            print(
+                                "Warning: libfastled compilation returned malformed embedded status"
+                            )
+                            # Continue with sketch compilation even if libfastled fails
+                        else:
+                            print("✅ libfastled compilation successful")
+                    else:
+                        print("✅ libfastled compilation successful")
+                libfastled_time = time.time() - libfastled_start_time
+            except Exception as e:
+                libfastled_time = time.time() - libfastled_start_time
+                print(f"Warning: libfastled compilation failed: {e}")
+                # Continue with sketch compilation even if libfastled fails
+        else:
+            print("Step 1 (skipped): Compiling libfastled")
+
+        # Step 2: Compile the sketch
+        print("Step 2: Compiling sketch...")
+        sketch_start_time = time.time()
+        response = _send_compile_request(
+            host,
+            zip_bytes,
+            auth_token,
+            build_mode,
+            profile,
+            no_platformio,
+        )
+        sketch_time = time.time() - sketch_start_time
+
+        return _process_compile_response(
+            response, zip_result, start_time, zip_time, libfastled_time, sketch_time
         )
-        with httpx.Client(
-            transport=transport,
-            timeout=_TIMEOUT,
-        ) as client:
-            headers = {
-                "accept": "application/json",
-                "authorization": auth_token,
-                "build": (
-                    build_mode.value.lower()
-                    if build_mode
-                    else BuildMode.QUICK.value.lower()
-                ),
-                "profile": "true" if profile else "false",
-            }
-
-            url = f"{connection_result.host}/{ENDPOINT_COMPILED_WASM}"
-            print(f"Compiling on {url} via {ipv4_stmt}. Zip size: {archive_size} bytes")
-            files = {"file": ("wasm.zip", zip_bytes, "application/x-zip-compressed")}
-            response = client.post(
-                url,
-                follow_redirects=True,
-                files=files,
-                headers=headers,
-                timeout=_TIMEOUT,
-            )
-
-            if response.status_code != 200:
-                json_response = response.json()
-                detail = json_response.get("detail", "Could not compile")
-                return CompileResult(
-                    success=False, stdout=detail, hash_value=None, zip_bytes=b""
-                )
-
-            print(f"Response status code: {response}")
-            # Create a temporary directory to extract the zip
-            with tempfile.TemporaryDirectory() as extract_dir:
-                extract_path = Path(extract_dir)
-
-                # Write the response content to a temporary zip file
-                temp_zip = extract_path / "response.zip"
-                temp_zip.write_bytes(response.content)
-
-                # Extract the zip
-                shutil.unpack_archive(temp_zip, extract_path, "zip")
-
-                if zip_result.zip_embedded_bytes:
-                    # extract the embedded bytes, which were not sent to the server
-                    temp_zip.write_bytes(zip_result.zip_embedded_bytes)
-                    shutil.unpack_archive(temp_zip, extract_path, "zip")
-
-                # we don't need the temp zip anymore
-                temp_zip.unlink()
-
-                # Read stdout from out.txt if it exists
-                stdout_file = extract_path / "out.txt"
-                hash_file = extract_path / "hash.txt"
-                stdout = stdout_file.read_text() if stdout_file.exists() else ""
-                hash_value = hash_file.read_text() if hash_file.exists() else None
-
-                # now rezip the extracted files since we added the embedded json files
-                out_buffer = io.BytesIO()
-                with zipfile.ZipFile(
-                    out_buffer, "w", zipfile.ZIP_DEFLATED, compresslevel=9
-                ) as out_zip:
-                    for root, _, _files in os.walk(extract_path):
-                        for file in _files:
-                            file_path = Path(root) / file
-                            relative_path = file_path.relative_to(extract_path)
-                            out_zip.write(file_path, relative_path)
-
-                diff_time = time.time() - start_time
-                msg = f"Compilation success, took {diff_time:.2f} seconds"
-                _print_banner(msg)
-                return CompileResult(
-                    success=True,
-                    stdout=stdout,
-                    hash_value=hash_value,
-                    zip_bytes=out_buffer.getvalue(),
-                )
+
     except KeyboardInterrupt:
         print("Keyboard interrupt")
         raise
+    except (ConnectionError, httpx.RemoteProtocolError, httpx.RequestError) as e:
+        # Handle connection and server disconnection issues
+        error_msg = f"Server connection error: {e}"
+        _print_banner(error_msg)
+        return CompileResult(
+            success=False,
+            stdout=error_msg,
+            hash_value=None,
+            zip_bytes=b"",
+            zip_time=zip_time,
+            libfastled_time=libfastled_time,
+            sketch_time=sketch_time,
+            response_processing_time=0.0,
+        )
     except httpx.HTTPError as e:
         print(f"Error: {e}")
         return CompileResult(
-            success=False, stdout=str(e), hash_value=None, zip_bytes=b""
+            success=False,
+            stdout=str(e),
+            hash_value=None,
+            zip_bytes=b"",
+            zip_time=zip_time,
+            libfastled_time=libfastled_time,
+            sketch_time=sketch_time,
+            response_processing_time=0.0,  # No response processing in HTTP error case
         )