From a3f21a218e297acd90e9d50e50d1673bffea6f32 Mon Sep 17 00:00:00 2001 From: David Montero Crespo Date: Fri, 8 May 2026 14:36:34 -0300 Subject: [PATCH] fix(esp32): trim flash image before serializing, pad on QEMU attach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #101 reproducer: an ESP32 sketch that pulls in Adafruit_SSD1306 + Adafruit_GFX produced a "No response from server. Is the backend running on port 8001?" error in the browser. The compile actually succeeded backend-side, but the JSON response carrying the firmware was ~5.5 MB of base64 — the ESP-IDF compiler builds a full 4 MB merged flash image (mostly 0xFF padding), encodes it whole, and ships it. In prod that response goes through nginx + Cloudflare, which buffer-fail or RST the connection on payloads that big — axios then lands in the "no response" branch with no HTTP status to surface. Fix: trim the trailing 0xFF padding before serializing, re-pad to a valid QEMU flash size (2/4/8/16 MB) just before mtd attach. Lossless: bytes after `last_used` in the merge are 0xFF by construction, so trim → pad reproduces the original image byte-for-byte. Numbers from the reproducer (Adafruit_SSD1306 + Adafruit_GFX, esp32:esp32:esp32 board): before: ~5.5 MB JSON response after: 539 KB JSON response (10× smaller) backend/app/services/espidf_compiler.py _merge_flash_image now tracks `last_used` across the three placed sections (bootloader / partitions / app) and writes only flash[:last_used] to merged_flash.bin. backend/app/services/esp32_flash_image.py (new) Shared `pad_to_flash_size(bytes) -> bytes` helper. Rounds up to the next valid QEMU flash size with a 4 MB minimum, matches the frontend's existing padToFlashSize logic in Esp32MicroPythonLoader. Raises ValueError on >16 MB inputs (would indicate a broken upstream merge, not anything user-recoverable). backend/app/services/esp32_lib_bridge.py backend/app/services/esp32_worker.py Both QEMU consumer paths (in-process and subprocess) call pad_to_flash_size right after base64.b64decode, before writing the tmp .bin that QEMU attaches with `-drive if=mtd,format=raw`. Verified: smoke test confirms trim → pad → original is byte-exact. Edge cases covered: small payloads pad up to the 4 MB minimum; firmwares >16 MB are rejected loudly. Closes #101 Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/esp32_flash_image.py | 44 +++++++++++++++++++++++ backend/app/services/esp32_lib_bridge.py | 6 +++- backend/app/services/esp32_worker.py | 6 +++- backend/app/services/espidf_compiler.py | 18 ++++++++-- 4 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 backend/app/services/esp32_flash_image.py diff --git a/backend/app/services/esp32_flash_image.py b/backend/app/services/esp32_flash_image.py new file mode 100644 index 00000000..24659521 --- /dev/null +++ b/backend/app/services/esp32_flash_image.py @@ -0,0 +1,44 @@ +"""Helpers for ESP32 flash images shared between the in-process bridge and +the subprocess worker. + +The ESP-IDF compiler in `espidf_compiler.py` builds a full 4 MB merged flash +image and then **trims the trailing 0xFF padding** before serializing the +binary into the JSON compile response (issue #101 — sending the full 4 MB +gave a ~5.5 MB base64 string that nginx / Cloudflare would buffer-fail with +"No response from server"). The frontend stores the trimmed bytes; this +module re-pads them on the receiving side just before QEMU attaches the +image as an MTD drive — QEMU rejects flash sizes that aren't a power-of-2 +megabyte (2 / 4 / 8 / 16 MB). + +This is a lossless round trip: bytes after `last_used` in the compiler's +merge are 0xFF by construction, so trim → pad reproduces the original +image byte-for-byte. +""" + +from __future__ import annotations + +# Sizes QEMU's esp32-picsimlab MTD layer accepts. ESP32 / S3 / C3 builds +# all default to 4 MB (CONFIG_ESPTOOLPY_FLASHSIZE_4MB). We only ever +# round UP — never down. +_VALID_FLASH_SIZES = [s * 1024 * 1024 for s in (2, 4, 8, 16)] +_MIN_FLASH_BYTES = 4 * 1024 * 1024 + + +def pad_to_flash_size(fw_bytes: bytes) -> bytes: + """Pad `fw_bytes` with 0xFF up to the next valid QEMU flash size. + + Returns the input unchanged when it's already at or above a valid size. + Raises `ValueError` if the firmware exceeds the largest size we accept + (16 MB) — at that point something is wrong with the upstream merge. + """ + target = next( + (s for s in _VALID_FLASH_SIZES if s >= max(len(fw_bytes), _MIN_FLASH_BYTES)), + None, + ) + if target is None: + raise ValueError( + f'ESP32 firmware too large for QEMU: {len(fw_bytes)} bytes (max 16 MB)' + ) + if len(fw_bytes) >= target: + return fw_bytes + return fw_bytes + b'\xff' * (target - len(fw_bytes)) diff --git a/backend/app/services/esp32_lib_bridge.py b/backend/app/services/esp32_lib_bridge.py index 73a96a1a..cd73365e 100644 --- a/backend/app/services/esp32_lib_bridge.py +++ b/backend/app/services/esp32_lib_bridge.py @@ -163,7 +163,11 @@ class Esp32LibBridge: def start(self, firmware_b64: str, machine: str = 'esp32-picsimlab') -> None: """Decode firmware, init QEMU, start event loop in daemon thread.""" - fw_bytes = base64.b64decode(firmware_b64) + from app.services.esp32_flash_image import pad_to_flash_size + # The compiler trims trailing 0xFF padding before serializing (issue + # #101 — full 4 MB images blew nginx buffers). Re-pad here so QEMU's + # MTD layer sees a valid power-of-2 flash size. + fw_bytes = pad_to_flash_size(base64.b64decode(firmware_b64)) tmp = tempfile.NamedTemporaryFile(suffix='.bin', delete=False) tmp.write(fw_bytes) tmp.close() diff --git a/backend/app/services/esp32_worker.py b/backend/app/services/esp32_worker.py index 918f7bec..d938f9b2 100644 --- a/backend/app/services/esp32_worker.py +++ b/backend/app/services/esp32_worker.py @@ -314,7 +314,11 @@ def main() -> None: # noqa: C901 (complexity OK for inline worker) # ── 3. Write firmware to a temp file ────────────────────────────────────── try: - fw_bytes = base64.b64decode(firmware_b64) + # The compiler trims trailing 0xFF padding before serializing (issue + # #101 — full 4 MB images blew nginx buffers). Re-pad here so QEMU's + # MTD layer sees a valid power-of-2 flash size. + from app.services.esp32_flash_image import pad_to_flash_size + fw_bytes = pad_to_flash_size(base64.b64decode(firmware_b64)) tmp = tempfile.NamedTemporaryFile(suffix='.bin', delete=False) tmp.write(fw_bytes) tmp.close() diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index af104e17..0633fe31 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -771,6 +771,7 @@ class ESPIDFCompiler: missing = [k for k, v in files_found.items() if not v] raise FileNotFoundError(f'Missing binaries for merge: {missing}') + last_used = 0 for offset, path in [ (bootloader_offset, bootloader), (0x8000, partitions), @@ -778,11 +779,24 @@ class ESPIDFCompiler: ]: data = path.read_bytes() flash[offset:offset + len(data)] = data + last_used = max(last_used, offset + len(data)) logger.info(f'[espidf] Placed {path.name} at 0x{offset:04X} ({len(data)} bytes)') + # Trim the trailing 0xFF padding before serializing. + # + # Keeping the full 4 MB flash image here gives a ~5.5 MB base64 JSON + # response that nginx / Cloudflare can choke on (issue #101 — user + # saw "No response from server"). The frontend stores the trimmed + # bytes and the backend pads back to the QEMU flash size at the + # bridge layer right before mtd attach. Lossless: bytes after + # last_used are 0xFF by construction, so re-padding restores the + # original image byte-for-byte. merged_path = build_dir / 'merged_flash.bin' - merged_path.write_bytes(bytes(flash)) - logger.info(f'[espidf] Merged flash image: {merged_path.stat().st_size} bytes') + merged_path.write_bytes(bytes(flash[:last_used])) + logger.info( + f'[espidf] Merged flash image (trimmed): {merged_path.stat().st_size} bytes ' + f'(would have been {FLASH_SIZE} bytes unpadded)' + ) return merged_path async def compile(self, files: list[dict], board_fqbn: str) -> dict: