velxio/frontend/src/simulation/Esp32Bridge.ts

657 lines
26 KiB
TypeScript
Raw Normal View History

/**
* Esp32Bridge
*
* Manages the WebSocket connection from the frontend to the backend
* QEMU manager for one ESP32/ESP32-S3/ESP32-C3 board instance.
*
* Protocol (JSON frames):
* Frontend Backend
* { type: 'start_esp32', data: { board: BoardKind, firmware_b64?: string } }
* { type: 'stop_esp32' }
* { type: 'load_firmware', data: { firmware_b64: string } }
* { type: 'esp32_serial_input', data: { bytes: number[], uart?: number } }
* { type: 'esp32_gpio_in', data: { pin: number, state: 0 | 1 } }
* { type: 'esp32_adc_set', data: { channel: number, millivolts: number } }
* { type: 'esp32_i2c_response', data: { addr: number, response: number } }
* { type: 'esp32_spi_response', data: { response: number } }
* { type: 'esp32_sensor_attach', data: { sensor_type: string, pin: number, ... } }
* { type: 'esp32_sensor_update', data: { pin: number, ... } }
* { type: 'esp32_sensor_detach', data: { pin: number } }
*
* Backend Frontend
* { type: 'serial_output', data: { data: string, uart?: number } }
* { type: 'gpio_change', data: { pin: number, state: 0 | 1 } }
* { type: 'gpio_dir', data: { pin: number, dir: 0 | 1 } }
* { type: 'ledc_update', data: { channel: number, duty: number, duty_pct: number } }
* { type: 'ws2812_update', data: { channel: number, pixels: [number, number, number][] } }
* { type: 'i2c_event', data: { addr: number, data: number } }
* { type: 'i2c_transaction', data: { addr: number, data: number[] } }
* { type: 'spi_event', data: { data: number } }
* { type: 'system', data: { event: string, ... } }
* { type: 'error', data: { message: string } }
*/
import type { BoardKind } from '../types/board';
fix(frontend): polyfill crypto.randomUUID for non-secure contexts crypto.randomUUID() is only exposed on secure contexts (HTTPS, localhost, 127.0.0.1, ::1). When Velxio is self-hosted and accessed via a LAN IP over plain HTTP (e.g. http://192.168.31.139:3080/), crypto.randomUUID is undefined and any code path that calls it throws TypeError. This silently broke ESP32 simulation start for self-hosters: the frontend generates a UUID for the WS client_id when Run is clicked; the throw rejected the promise before reaching the WS connect, so the backend never got the start request — no worker spawned, logs empty, simulation "didn't start" with no visible error. Same root cause would also break the multi-file editor (createFile, createFileGroup) on the same LAN-HTTP self-host setup, just less observably. Add a single generateUUID() helper that: 1. Uses crypto.randomUUID() when available (secure context fast path). 2. Falls back to crypto.getRandomValues() — which IS available in non-secure contexts — to build a v4 UUID by hand. 3. Final fallback to Math.random() if even that is missing (defensive — Web Crypto getRandomValues has been universal for years). Replace all 6 crypto.randomUUID() call sites: - frontend/src/simulation/Esp32Bridge.ts (2 sites — getTabSessionId) - frontend/src/store/useEditorStore.ts (4 sites — file IDs) Reported by a self-hoster on OrangePi 5B accessing Velxio via LAN IP. DevTools console showed: TypeError: crypto.randomUUID is not a function at Ph (...) at wh.connect (...) at startBoard (...) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 23:41:30 +07:00
import { generateUUID } from '../utils/uuid';
/**
* Map any ESP32-family board kind to the 3 base QEMU machine types understood
* by the backend esp_qemu_manager.
*/
export function toQemuBoardType(kind: BoardKind): 'esp32' | 'esp32-s3' | 'esp32-c3' {
if (kind === 'esp32-s3' || kind === 'xiao-esp32-s3' || kind === 'arduino-nano-esp32')
return 'esp32-s3';
if (kind === 'esp32-c3' || kind === 'xiao-esp32-c3' || kind === 'aitewinrobot-esp32c3-supermini')
return 'esp32-c3';
return 'esp32'; // esp32, esp32-devkit-c-v4, esp32-cam, wemos-lolin32-lite
}
const API_BASE = (): string =>
(import.meta.env.VITE_API_BASE as string | undefined) ?? 'http://localhost:8001/api';
/** Returns a stable UUID for this browser tab (persists across reloads, resets on new tab). */
export function getTabSessionId(): string {
// sessionStorage is not available in Node/test environments
fix(frontend): polyfill crypto.randomUUID for non-secure contexts crypto.randomUUID() is only exposed on secure contexts (HTTPS, localhost, 127.0.0.1, ::1). When Velxio is self-hosted and accessed via a LAN IP over plain HTTP (e.g. http://192.168.31.139:3080/), crypto.randomUUID is undefined and any code path that calls it throws TypeError. This silently broke ESP32 simulation start for self-hosters: the frontend generates a UUID for the WS client_id when Run is clicked; the throw rejected the promise before reaching the WS connect, so the backend never got the start request — no worker spawned, logs empty, simulation "didn't start" with no visible error. Same root cause would also break the multi-file editor (createFile, createFileGroup) on the same LAN-HTTP self-host setup, just less observably. Add a single generateUUID() helper that: 1. Uses crypto.randomUUID() when available (secure context fast path). 2. Falls back to crypto.getRandomValues() — which IS available in non-secure contexts — to build a v4 UUID by hand. 3. Final fallback to Math.random() if even that is missing (defensive — Web Crypto getRandomValues has been universal for years). Replace all 6 crypto.randomUUID() call sites: - frontend/src/simulation/Esp32Bridge.ts (2 sites — getTabSessionId) - frontend/src/store/useEditorStore.ts (4 sites — file IDs) Reported by a self-hoster on OrangePi 5B accessing Velxio via LAN IP. DevTools console showed: TypeError: crypto.randomUUID is not a function at Ph (...) at wh.connect (...) at startBoard (...) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 23:41:30 +07:00
if (typeof sessionStorage === 'undefined') return generateUUID();
const KEY = 'velxio-tab-id';
let id = sessionStorage.getItem(KEY);
if (!id) {
fix(frontend): polyfill crypto.randomUUID for non-secure contexts crypto.randomUUID() is only exposed on secure contexts (HTTPS, localhost, 127.0.0.1, ::1). When Velxio is self-hosted and accessed via a LAN IP over plain HTTP (e.g. http://192.168.31.139:3080/), crypto.randomUUID is undefined and any code path that calls it throws TypeError. This silently broke ESP32 simulation start for self-hosters: the frontend generates a UUID for the WS client_id when Run is clicked; the throw rejected the promise before reaching the WS connect, so the backend never got the start request — no worker spawned, logs empty, simulation "didn't start" with no visible error. Same root cause would also break the multi-file editor (createFile, createFileGroup) on the same LAN-HTTP self-host setup, just less observably. Add a single generateUUID() helper that: 1. Uses crypto.randomUUID() when available (secure context fast path). 2. Falls back to crypto.getRandomValues() — which IS available in non-secure contexts — to build a v4 UUID by hand. 3. Final fallback to Math.random() if even that is missing (defensive — Web Crypto getRandomValues has been universal for years). Replace all 6 crypto.randomUUID() call sites: - frontend/src/simulation/Esp32Bridge.ts (2 sites — getTabSessionId) - frontend/src/store/useEditorStore.ts (4 sites — file IDs) Reported by a self-hoster on OrangePi 5B accessing Velxio via LAN IP. DevTools console showed: TypeError: crypto.randomUUID is not a function at Ph (...) at wh.connect (...) at startBoard (...) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-05 23:41:30 +07:00
id = generateUUID();
sessionStorage.setItem(KEY, id);
}
return id;
}
export interface Ws2812Pixel {
r: number;
g: number;
b: number;
}
export interface LedcUpdate {
channel: number;
duty: number;
duty_pct: number;
gpio?: number;
}
export interface WifiStatus {
status: string;
ssid?: string;
ip?: string;
}
export interface BleStatus {
status: string;
}
export class Esp32Bridge {
readonly boardId: string;
readonly boardKind: BoardKind;
/** Set to true before connect() to enable WiFi NIC in QEMU. */
wifiEnabled = false;
// Callbacks wired up by useSimulatorStore
onSerialData: ((char: string, uart?: number) => void) | null = null;
onPinChange: ((gpioPin: number, state: boolean) => void) | null = null;
onPinDir: ((gpioPin: number, dir: 0 | 1) => void) | null = null;
onLedcUpdate: ((update: LedcUpdate) => void) | null = null;
onWs2812Update: ((channel: number, pixels: Ws2812Pixel[]) => void) | null = null;
/**
* ePaper SSD168x backend rendering. Backend decodes SPI traffic in
* `Ssd168xEpaperSlave` and emits this event on every 0x20
* MASTER_ACTIVATION with a base64-encoded palette buffer (1 byte/pixel:
* 0=black, 1=white, 2=red). One subscriber per `componentId`; multiple
* panels on the same board are routed by ID.
*/
onEpaperUpdate:
| ((
componentId: string,
frame: { width: number; height: number; b64: string; refreshMs: number },
) => void)
| null = null;
onI2cEvent: ((addr: number, data: number) => void) | null = null;
onI2cTransaction: ((addr: number, data: number[]) => void) | null = null;
onSpiEvent: ((data: number) => void) | null = null;
fix(ili9341): add ESP32 SPI byte routing so the LCD renders on ESP32-CAM The ILI9341 part simulation only hooked AVR's SPI peripheral. For ESP32 the simulator is Esp32BridgeShim (no .spi member), so attachEvents bailed early and the LCD stayed black even though the firmware was driving SPI traffic correctly. The QEMU worker already emits per-byte spi_event WS messages (see backend/app/services/esp32_worker.py::_on_spi_event), and the Esp32Bridge already had an onSpiEvent hook — but the bridge was reading msg.data.data (a non-existent field) instead of decoding the worker's {bus, event, response} format. Fixed. Two changes: 1. Esp32Bridge.ts: decode the spi_event payload correctly. The worker encodes byte transfers as `mosi << 8` (op = low byte = 0x00) and CS-line changes as `((cs<<1)|level) << 8 | 0x01` (op == 0x01). Added onSpiByte (per-byte) and onSpiCsChange callbacks alongside the existing onSpiEvent for backwards compat. 2. ComplexParts.ts ili9341Simulation: detect Esp32BridgeShim via `getBridge()` duck-type check. When present, subscribe to bridge.onSpiByte and feed bytes into the same processCommand / processData pipeline used by the AVR path. DC tracking via pinManager.onPinChange already works for ESP32 because the bridge fires triggerPinChange on every gpio_change WS event. Verified end-to-end: ESP32-CAM + ILI9341 example in the gallery now renders the live webcam preview to the simulated TFT (160×120 RGB565 centered in the 320×240 panel) at ~3-4 fps. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 08:31:24 +07:00
/** Same as onSpiEvent but more explicit (a single MOSI byte). */
onSpiByte: ((mosi: number) => void) | null = null;
/** Fires on every CS line change emitted by the SoC's SPI peripheral.
* `csIdx` is the index of the CS pin within the SPI bus (0-3 typical),
* `low` is true when CS goes LOW (slave selected), false when HIGH. */
onSpiCsChange: ((csIdx: number, low: boolean) => void) | null = null;
onConnected: (() => void) | null = null;
onDisconnected: (() => void) | null = null;
onError: ((msg: string) => void) | null = null;
onSystemEvent: ((event: string, data: Record<string, unknown>) => void) | null = null;
onCrash: ((data: Record<string, unknown>) => void) | null = null;
onWifiStatus: ((status: WifiStatus) => void) | null = null;
onBleStatus: ((status: BleStatus) => void) | null = null;
private socket: WebSocket | null = null;
private _connected = false;
private _pendingFirmware: string | null = null;
private _pendingSensors: Array<Record<string, unknown>> = [];
// MicroPython REPL injection — 4-stage state machine
// idle → banner_seen → prompt_seen → raw_repl_entered → done
// Each stage waits for a specific string in the serial buffer before
// proceeding. This avoids the race where code is sent before raw REPL
// mode is confirmed and ends up echoed by the normal REPL.
private _pendingMicroPythonCode: string | null = null;
private _serialBuffer = '';
private _replState: 'idle' | 'banner_seen' | 'prompt_seen' | 'raw_repl_entered' = 'idle';
micropythonMode = false;
constructor(boardId: string, boardKind: BoardKind) {
this.boardId = boardId;
this.boardKind = boardKind;
}
get connected(): boolean {
return this._connected;
}
get clientId(): string {
return getTabSessionId() + '::' + this.boardId;
}
connect(): void {
if (this.socket && this.socket.readyState !== WebSocket.CLOSED) return;
const base = API_BASE();
const wsProtocol = base.startsWith('https') ? 'wss:' : 'ws:';
const sessionId = getTabSessionId();
const wsUrl =
base.replace(/^https?:/, wsProtocol) +
`/simulation/ws/${encodeURIComponent(sessionId + '::' + this.boardId)}`;
const socket = new WebSocket(wsUrl);
this.socket = socket;
socket.onopen = () => {
this._connected = true;
console.log(
`[Esp32Bridge:${this.boardId}] WebSocket connected → sending start_esp32 (firmware: ${this._pendingFirmware ? `${Math.round((this._pendingFirmware.length * 0.75) / 1024)}KB` : 'none'})`,
);
this.onConnected?.();
this._send({
type: 'start_esp32',
data: {
board: toQemuBoardType(this.boardKind),
...(this._pendingFirmware ? { firmware_b64: this._pendingFirmware } : {}),
sensors: this._pendingSensors,
wifi_enabled: this.wifiEnabled,
},
});
};
socket.onmessage = (event: MessageEvent) => {
let msg: { type: string; data: Record<string, unknown> };
try {
msg = JSON.parse(event.data as string);
} catch {
return;
}
switch (msg.type) {
case 'serial_output': {
const text = (msg.data.data as string) ?? '';
const uart = msg.data.uart as number | undefined;
if (this.onSerialData) {
for (const ch of text) this.onSerialData(ch, uart);
}
// MicroPython REPL injection — 4-stage state machine.
// Each stage waits for a confirmed string in the serial buffer before
// advancing, so we never send code before raw REPL mode is verified.
if (this._pendingMicroPythonCode || this._replState !== 'idle') {
this._serialBuffer += text;
// Stage 1: banner "Type help()" → poke UART with \r to flush ">>> "
// The >>> prompt has no \n so the backend UART buffer holds it until
// we send a byte that causes another write.
if (this._replState === 'idle' && this._serialBuffer.includes('Type "help()"')) {
this._replState = 'banner_seen';
console.log('[Esp32Bridge] Stage 1: banner seen → poking UART with \\r');
setTimeout(() => {
this._send({ type: 'esp32_serial_input', data: { bytes: [0x0d] } });
}, 800);
}
// Stage 2: ">>>" → send Ctrl+A to enter raw REPL
if (this._replState === 'banner_seen' && this._serialBuffer.includes('>>>')) {
this._replState = 'prompt_seen';
this._serialBuffer = '';
console.log('[Esp32Bridge] Stage 2: >>> seen → sending Ctrl+A');
setTimeout(() => {
this._send({ type: 'esp32_serial_input', data: { bytes: [0x01] } });
}, 200);
}
// Stage 3: "raw REPL" confirmation → now safe to send code
if (this._replState === 'prompt_seen' && this._serialBuffer.includes('raw REPL')) {
this._replState = 'raw_repl_entered';
const code = this._pendingMicroPythonCode!;
this._pendingMicroPythonCode = null;
this._serialBuffer = '';
console.log('[Esp32Bridge] Stage 3: raw REPL confirmed → sending code');
setTimeout(() => this._sendCodeInRawRepl(code), 200);
}
// Keep buffer from growing unboundedly
if (this._serialBuffer.length > 8192) {
this._serialBuffer = this._serialBuffer.slice(-1024);
}
}
break;
}
case 'gpio_change': {
const pin = msg.data.pin as number;
const state = (msg.data.state as number) === 1;
console.log(
`[Esp32Bridge:${this.boardId}] gpio_change pin=${pin} state=${state ? 'HIGH' : 'LOW'}`,
);
this.onPinChange?.(pin, state);
break;
}
case 'gpio_dir': {
const pin = msg.data.pin as number;
const dir = msg.data.dir as 0 | 1;
this.onPinDir?.(pin, dir);
break;
}
case 'ledc_update': {
console.log(
`[Esp32Bridge:${this.boardId}] ledc_update ch=${msg.data.channel} duty=${msg.data.duty_pct}% gpio=${msg.data.gpio}`,
);
this.onLedcUpdate?.(msg.data as unknown as LedcUpdate);
break;
}
case 'ws2812_update': {
const channel = msg.data.channel as number;
const raw = msg.data.pixels as [number, number, number][];
const pixels: Ws2812Pixel[] = raw.map(([r, g, b]) => ({ r, g, b }));
this.onWs2812Update?.(channel, pixels);
break;
}
case 'epaper_update': {
const componentId = msg.data.component_id as string;
this.onEpaperUpdate?.(componentId, {
width: msg.data.width as number,
height: msg.data.height as number,
b64: msg.data.frame_b64 as string,
refreshMs: (msg.data.refresh_ms as number) ?? 50,
});
break;
}
case 'i2c_event': {
const addr = msg.data.addr as number;
const data = msg.data.data as number;
this.onI2cEvent?.(addr, data);
break;
}
case 'i2c_transaction': {
const addr = msg.data.addr as number;
const data = msg.data.data as number[];
this.onI2cTransaction?.(addr, data);
break;
}
perf(spi): batch SPI bytes per WS message — ~50× faster TFT in emulator User reported the ESP32-CAM + ILI9341 live preview at ~1 frame/min. Profile: 80×60 preview pushes 9600 SPI bytes per drawRGBBitmap, and each byte was emitting a full {type:'spi_event'} JSON message over the worker→backend→WS→frontend pipeline. Per-byte overhead ~150-200µs in Python (json.dumps + sys.stdout.write+flush dominates) plus asyncio + WS dispatch. Net: 1.5-2 sec/frame minimum, much worse with GIL contention. Fix: buffer MOSI bytes in the worker and emit a single base64-encoded `spi_batch` message when CS goes HIGH (transaction ended) or the buffer crosses 4 KiB. ~9600 events/frame collapse to ~3 messages. backend/app/services/esp32_worker.py:_on_spi_event - Add _spi_byte_buf bytearray + threading.Lock - On op==0x00 (byte): append; flush early if buf >= 4096 - On op==0x01 (CS change): flush buffer, then emit the CS event via the legacy spi_event channel (ePaper / custom chips that observe CS still get it). frontend/src/simulation/Esp32Bridge.ts - New 'spi_batch' message handler decodes b64 and replays each byte through the existing onSpiByte callback. Parts that subscribed via simulator.spi.onByte don't notice the protocol change. The 'spi_event' branch still handles CS changes plus legacy single-byte payloads for backwards compat. Now that 38 KB/frame is cheap, restore preview to 160×120 + JPEG quality 0.35 in the gallery example. Real measured speedup: ~50× on the QVGA preview demo. Real hardware was never affected — it runs SPI at 80 MHz and pushes the bitmap in ~4 ms either way. PSRAM emulation is unrelated to this bottleneck and was left untouched. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 10:25:06 +07:00
case 'spi_batch': {
// Worker batches consecutive MOSI bytes from a single SPI
// transaction into one base64-encoded message. Replays each
// byte through the same callbacks the per-byte spi_event path
// uses — parts that subscribed to onSpiByte don't notice. See
// backend/app/services/esp32_worker.py::_on_spi_event for the
// batching policy (flush on CS HIGH or buffer cap).
const b64 = msg.data.b64 as string;
if (b64) {
const bin = atob(b64);
const handler = this.onSpiByte ?? this.onSpiEvent;
if (handler) {
for (let i = 0; i < bin.length; i++) {
const m = bin.charCodeAt(i);
handler(m);
}
}
}
break;
}
case 'spi_event': {
fix(ili9341): add ESP32 SPI byte routing so the LCD renders on ESP32-CAM The ILI9341 part simulation only hooked AVR's SPI peripheral. For ESP32 the simulator is Esp32BridgeShim (no .spi member), so attachEvents bailed early and the LCD stayed black even though the firmware was driving SPI traffic correctly. The QEMU worker already emits per-byte spi_event WS messages (see backend/app/services/esp32_worker.py::_on_spi_event), and the Esp32Bridge already had an onSpiEvent hook — but the bridge was reading msg.data.data (a non-existent field) instead of decoding the worker's {bus, event, response} format. Fixed. Two changes: 1. Esp32Bridge.ts: decode the spi_event payload correctly. The worker encodes byte transfers as `mosi << 8` (op = low byte = 0x00) and CS-line changes as `((cs<<1)|level) << 8 | 0x01` (op == 0x01). Added onSpiByte (per-byte) and onSpiCsChange callbacks alongside the existing onSpiEvent for backwards compat. 2. ComplexParts.ts ili9341Simulation: detect Esp32BridgeShim via `getBridge()` duck-type check. When present, subscribe to bridge.onSpiByte and feed bytes into the same processCommand / processData pipeline used by the AVR path. DC tracking via pinManager.onPinChange already works for ESP32 because the bridge fires triggerPinChange on every gpio_change WS event. Verified end-to-end: ESP32-CAM + ILI9341 example in the gallery now renders the live webcam preview to the simulated TFT (160×120 RGB565 centered in the 320×240 panel) at ~3-4 fps. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 08:31:24 +07:00
// Worker emits {bus, event, response}. The 'event' field encodes:
// event = mosi << 8 (op = event & 0xFF == 0x00) → byte transfer
// event = ((cs<<1)|level) << 8 | 0x01 (op == 0x01) → CS line change
// See backend/app/services/esp32_worker.py::_on_spi_event.
perf(spi): batch SPI bytes per WS message — ~50× faster TFT in emulator User reported the ESP32-CAM + ILI9341 live preview at ~1 frame/min. Profile: 80×60 preview pushes 9600 SPI bytes per drawRGBBitmap, and each byte was emitting a full {type:'spi_event'} JSON message over the worker→backend→WS→frontend pipeline. Per-byte overhead ~150-200µs in Python (json.dumps + sys.stdout.write+flush dominates) plus asyncio + WS dispatch. Net: 1.5-2 sec/frame minimum, much worse with GIL contention. Fix: buffer MOSI bytes in the worker and emit a single base64-encoded `spi_batch` message when CS goes HIGH (transaction ended) or the buffer crosses 4 KiB. ~9600 events/frame collapse to ~3 messages. backend/app/services/esp32_worker.py:_on_spi_event - Add _spi_byte_buf bytearray + threading.Lock - On op==0x00 (byte): append; flush early if buf >= 4096 - On op==0x01 (CS change): flush buffer, then emit the CS event via the legacy spi_event channel (ePaper / custom chips that observe CS still get it). frontend/src/simulation/Esp32Bridge.ts - New 'spi_batch' message handler decodes b64 and replays each byte through the existing onSpiByte callback. Parts that subscribed via simulator.spi.onByte don't notice the protocol change. The 'spi_event' branch still handles CS changes plus legacy single-byte payloads for backwards compat. Now that 38 KB/frame is cheap, restore preview to 160×120 + JPEG quality 0.35 in the gallery example. Real measured speedup: ~50× on the QVGA preview demo. Real hardware was never affected — it runs SPI at 80 MHz and pushes the bitmap in ~4 ms either way. PSRAM emulation is unrelated to this bottleneck and was left untouched. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 10:25:06 +07:00
//
// After the batching change, the byte transfer path goes
// through 'spi_batch' instead. This branch now only fires for
// CS-line changes (op == 0x01), but we keep the byte branch
// for backwards compatibility with older worker builds.
fix(ili9341): add ESP32 SPI byte routing so the LCD renders on ESP32-CAM The ILI9341 part simulation only hooked AVR's SPI peripheral. For ESP32 the simulator is Esp32BridgeShim (no .spi member), so attachEvents bailed early and the LCD stayed black even though the firmware was driving SPI traffic correctly. The QEMU worker already emits per-byte spi_event WS messages (see backend/app/services/esp32_worker.py::_on_spi_event), and the Esp32Bridge already had an onSpiEvent hook — but the bridge was reading msg.data.data (a non-existent field) instead of decoding the worker's {bus, event, response} format. Fixed. Two changes: 1. Esp32Bridge.ts: decode the spi_event payload correctly. The worker encodes byte transfers as `mosi << 8` (op = low byte = 0x00) and CS-line changes as `((cs<<1)|level) << 8 | 0x01` (op == 0x01). Added onSpiByte (per-byte) and onSpiCsChange callbacks alongside the existing onSpiEvent for backwards compat. 2. ComplexParts.ts ili9341Simulation: detect Esp32BridgeShim via `getBridge()` duck-type check. When present, subscribe to bridge.onSpiByte and feed bytes into the same processCommand / processData pipeline used by the AVR path. DC tracking via pinManager.onPinChange already works for ESP32 because the bridge fires triggerPinChange on every gpio_change WS event. Verified end-to-end: ESP32-CAM + ILI9341 example in the gallery now renders the live webcam preview to the simulated TFT (160×120 RGB565 centered in the 320×240 panel) at ~3-4 fps. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 08:31:24 +07:00
const event = msg.data.event as number;
const op = (event ?? 0) & 0xFF;
if (op === 0x00) {
const mosi = (event >> 8) & 0xFF;
this.onSpiEvent?.(mosi);
this.onSpiByte?.(mosi);
} else if (op === 0x01) {
const csIdx = (event >> 9) & 0x3;
const level = (event >> 8) & 0x1;
this.onSpiCsChange?.(csIdx, level === 1);
}
// Backwards-compat path for callers reading the old `data` field.
if (msg.data.data !== undefined) {
this.onSpiEvent?.(msg.data.data as number);
}
break;
}
case 'system': {
const evt = msg.data.event as string;
console.log(`[Esp32Bridge:${this.boardId}] system event: ${evt}`, msg.data);
if (evt === 'crash') {
this.onCrash?.(msg.data);
}
this.onSystemEvent?.(evt, msg.data);
break;
}
case 'wifi_status': {
const wifiStatus = msg.data as unknown as WifiStatus;
console.log(
`[Esp32Bridge:${this.boardId}] wifi_status: ${wifiStatus.status} ssid=${wifiStatus.ssid ?? ''} ip=${wifiStatus.ip ?? ''}`,
);
this.onWifiStatus?.(wifiStatus);
break;
}
case 'ble_status': {
const bleStatus = msg.data as unknown as BleStatus;
console.log(`[Esp32Bridge:${this.boardId}] ble_status: ${bleStatus.status}`);
this.onBleStatus?.(bleStatus);
break;
}
case 'error':
console.error(`[Esp32Bridge:${this.boardId}] error: ${msg.data.message as string}`);
this.onError?.(msg.data.message as string);
break;
}
};
socket.onclose = (ev) => {
console.log(`[Esp32Bridge:${this.boardId}] WebSocket closed (code=${ev?.code ?? '?'})`);
this._connected = false;
this.socket = null;
this.onDisconnected?.();
};
socket.onerror = (ev) => {
console.error(`[Esp32Bridge:${this.boardId}] WebSocket error`, ev);
this.onError?.('WebSocket error');
};
}
disconnect(): void {
if (this.socket) {
this._send({ type: 'stop_esp32' });
this.socket.close();
this.socket = null;
}
this._connected = false;
}
/**
* Pre-register sensors so they are included in the start_esp32 payload.
* This ensures sensors are ready in the QEMU worker BEFORE the firmware
* begins executing, preventing race conditions where pulseIn() times out
* because the sensor handler hasn't been registered yet.
*/
setSensors(sensors: Array<Record<string, unknown>>): void {
this._pendingSensors = sensors;
}
/** Returns true if a firmware has been loaded and is ready to send. */
hasFirmware(): boolean {
return this._pendingFirmware !== null && this._pendingFirmware !== '';
}
/**
* Load a compiled firmware (base64-encoded .bin) into the running ESP32.
* If not yet connected, the firmware will be sent on next connect().
*/
loadFirmware(firmwareBase64: string): void {
this._pendingFirmware = firmwareBase64;
if (this._connected) {
this._send({ type: 'load_firmware', data: { firmware_b64: firmwareBase64 } });
}
}
/** Send a byte to the ESP32 UART0 (or UART1/2) */
sendSerialByte(byte: number, uart = 0): void {
this._send({ type: 'esp32_serial_input', data: { bytes: [byte], uart } });
}
/** Send multiple bytes at once */
sendSerialBytes(bytes: number[], uart = 0): void {
if (bytes.length === 0) return;
this._send({ type: 'esp32_serial_input', data: { bytes, uart } });
}
/** Drive a GPIO pin from an external source (e.g. connected Arduino) */
sendPinEvent(gpioPin: number, state: boolean): void {
this._send({ type: 'esp32_gpio_in', data: { pin: gpioPin, state: state ? 1 : 0 } });
}
/** Set an ADC channel voltage (millivolts, 03300) */
setAdc(channel: number, millivolts: number): void {
this._send({ type: 'esp32_adc_set', data: { channel, millivolts } });
}
/**
* Push a periodic waveform LUT for an ADC channel. The backend forwards
* the samples to QEMU, which interpolates them against its virtual clock
* on every MMIO ADC read matching the per-read fidelity AVR and RP2040
* get via `onADCRead` monkey-patching.
*
* samples: 12-bit raw values (0-4095) aligned on a uniform time grid
* periodNs: full period of the LUT in nanoseconds
*
* Samples are sent as base64-encoded uint16 little-endian. Clearing the
* waveform (returning to DC `setAdc` behavior) is done by passing an
* empty `samples` array.
*/
setAdcWaveform(channel: number, samples: Uint16Array, periodNs: number): void {
// Encode little-endian uint16 → base64 (transport-safe for JSON stdin/WS).
const bytes = new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength);
let binary = '';
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
const base64 =
typeof btoa === 'function' ? btoa(binary) : Buffer.from(bytes).toString('base64');
this._send({
type: 'esp32_adc_waveform',
data: { channel, samples_u12_b64: base64, period_ns: periodNs },
});
}
/** Clear a previously-pushed ADC waveform, reverting to DC `setAdc`. */
clearAdcWaveform(channel: number): void {
this._send({
type: 'esp32_adc_waveform',
data: { channel, samples_u12_b64: '', period_ns: 0 },
});
}
/** Configure the byte an I2C device at addr returns */
setI2cResponse(addr: number, response: number): void {
this._send({ type: 'esp32_i2c_response', data: { addr, response } });
}
/** Configure the MISO byte returned during an SPI transaction */
setSpiResponse(response: number): void {
this._send({ type: 'esp32_spi_response', data: { response } });
}
// ── Generic sensor protocol offloading ────────────────────────────────────
// Sensors call these to delegate their protocol to the backend QEMU.
// The sensor type (e.g. 'dht22', 'hc-sr04') tells the backend which
// protocol handler to use. Sensor-specific properties (temperature,
// humidity, distance …) are passed as a generic Record.
/** Register a sensor on a GPIO pin — backend handles its protocol */
sendSensorAttach(sensorType: string, pin: number, properties: Record<string, unknown>): void {
// Buffer into _pendingSensors so it is included in start_esp32 if sent
// before the WebSocket opens (the common case when attachEvents fires
// before the user clicks Run).
const entry = { sensor_type: sensorType, pin, ...properties };
const existing = this._pendingSensors.findIndex((s) => s['pin'] === pin);
if (existing >= 0) {
this._pendingSensors[existing] = entry;
} else {
this._pendingSensors.push(entry);
}
// Also send immediately if already connected (re-attach on hot reload)
if (this._connected) {
this._send({ type: 'esp32_sensor_attach', data: entry });
}
}
/** Update sensor properties (temperature, humidity, distance, etc.) */
sendSensorUpdate(pin: number, properties: Record<string, unknown>): void {
// Keep _pendingSensors in sync so reconnects get current values
const idx = this._pendingSensors.findIndex((s) => s['pin'] === pin);
if (idx >= 0) {
this._pendingSensors[idx] = { ...this._pendingSensors[idx], ...properties };
}
this._send({ type: 'esp32_sensor_update', data: { pin, ...properties } });
}
/** Detach a sensor from a GPIO pin */
sendSensorDetach(pin: number): void {
this._pendingSensors = this._pendingSensors.filter((s) => s['pin'] !== pin);
this._send({ type: 'esp32_sensor_detach', data: { pin } });
}
feat: ESP32-CAM emulation with webcam frame bridge First open-source end-to-end emulation of the AI-Thinker ESP32-CAM in QEMU, paired with a browser webcam → firmware bridge so users can develop camera sketches without hardware. Status: esp_camera_init() returns ESP_OK; OV2640 chip-id verifies (PID/VER/MIDH/MIDL exactly match the datasheet); GPIO 25 VSYNC NEGEDGE interrupt enabled by the upstream driver. Final piece (cam_task accepting frames) is in progress — descriptor walker fix landed in this commit. Backend (Python/FastAPI): - simulation.py: camera_attach/frame/detach WS handlers - esp32_worker.py: ctypes binding to velxio_push_camera_frame + feature-detection fallback for older DLLs - esp32_lib_manager.py: forward camera commands to the worker stdin - esp-idf-template/main/CMakeLists.txt: esp32-camera headers added via add_prebuilt_library + REQUIRES driver (resolves i2c_master_* symbols). LED_BUILTIN=2 fallback for sketches that hardcode it. Frontend (React/TS): - EditorToolbar.tsx: ESP32-CAM (and the rest of the ESP32 family) added to isQemuBoard list — Run button now starts the QEMU bridge for these boards instead of falling through to the AVR path - useWebcamFrames.ts: getUserMedia → OffscreenCanvas → toBlob('image/jpeg') → base64 → WS at ~10 fps - CameraToggle.tsx: header button with status colors + frame counter - SimulatorCanvas.tsx: render CameraToggle for esp32-cam boards - Esp32Bridge.ts: sendCameraAttach/Frame/Detach + chunked btoa - useSimulatorStore.ts: diagnostic log on compileBoardProgram - components-metadata.json: regen including esp32-cam component Submodule pointer: - wokwi-libs/qemu-lcgamboa → ff8eee0 (camera devices commit on davidmonterocrespo24/qemu-lcgamboa branch picsimlab-esp32) Investigation + tests in test/test-esp32-cam/: - 13 autosearch markdown docs (overview, SOTA, OV2640 spec, DVP/I2S spec, build blueprint, blockers resolved, descriptor walker fix) - 5 sketches (camera_init, sccb_probe, dma_smoke, frame_roundtrip, webcam_demo) + 8 live + WS regression tests - README with the user-facing flow .gitignore: - libqemu-*.dll.{pre-camera,new,bak} (rollback points, regenerated) - wokwi-libs/esp32-camera/ (clone consumed by arduino-esp32 path, not part of this repo) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 05:28:55 +07:00
// ── ESP32-CAM webcam injection ────────────────────────────────────────────
/** Tell the backend a frame source is connected (call once when the user
* grants webcam permission). */
sendCameraAttach(): void {
this._send({ type: 'esp32_camera_attach', data: { board: 'esp32-cam' } });
}
/** Push one JPEG frame from the browser webcam to the emulator. The
* backend forwards it via ctypes to the QEMU OV2640+I²S device, which
* delivers the bytes to the firmware's DMA buffer.
*
* Encoding: base64 in JSON. ~1014 KB per QVGA frame at quality 0.6.
* At 10 fps that's ~120 KB/s trivial over local WS. */
sendCameraFrame(jpegBytes: ArrayBuffer | Uint8Array,
width = 320, height = 240): void {
const u8 = jpegBytes instanceof Uint8Array
? jpegBytes
: new Uint8Array(jpegBytes);
// btoa needs a binary string; build one in 32 KB chunks to avoid
// "argument size limit" issues with very large frames.
let binary = '';
const chunkSize = 0x8000;
for (let i = 0; i < u8.length; i += chunkSize) {
binary += String.fromCharCode(...u8.subarray(i, i + chunkSize));
}
const b64 = btoa(binary);
this._send({
type: 'esp32_camera_frame',
data: { fmt: 'jpeg', w: width, h: height, b64 },
});
}
/** Drop the queued frame. Call when the user stops the webcam. */
sendCameraDetach(): void {
this._send({ type: 'esp32_camera_detach', data: {} });
}
/**
* Queue user MicroPython code for injection after the REPL boots.
* The code will be sent via raw-paste protocol once `>>>` is detected.
*/
setPendingMicroPythonCode(code: string): void {
this._pendingMicroPythonCode = code;
this._serialBuffer = '';
this._replState = 'idle';
this.micropythonMode = true;
}
/** Check if this bridge is in MicroPython mode */
isMicroPythonMode(): boolean {
return this.micropythonMode;
}
/**
* Send code bytes to QEMU UART, then Ctrl+D to execute.
* Called ONLY after "raw REPL; CTRL-B to exit" has been confirmed in the
* serial buffer (stage 3), so we are guaranteed to be in raw REPL mode.
*/
/**
* Sanitize MicroPython source code before sending to the raw REPL.
*
* MicroPython v1.20 on ESP32 uses a byte-oriented tokenizer that doesn't
* handle non-ASCII bytes in source code. Multi-byte UTF-8 sequences
* (e.g. Spanish accents: á=\xC3\xA1, ú=\xC3\xBA) in comments confuse the
* tokenizer and produce SyntaxError at the wrong line.
*
* Safe to strip non-ASCII only from comments because:
* - String literals with non-ASCII would already fail on MicroPython's
* default build (no wide-unicode support on ESP32).
* - Identifiers must be ASCII.
*/
private static _sanitizeForRepl(code: string): string {
// 1. Strip UTF-8 BOM if present
let s = code.startsWith('\uFEFF') ? code.slice(1) : code;
// 2. Normalize line endings to LF
s = s.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
// 3. Replace non-ASCII in line-comments with '?' so the line is preserved
s = s.replace(/^([ \t]*#.*)$/gm, (line) => line.replace(/[^\x00-\x7F]/g, '?'));
// 4. Replace non-ASCII in inline comments (after code on the same line)
s = s.replace(/([ \t]+#.*)$/gm, (comment) => comment.replace(/[^\x00-\x7F]/g, '?'));
return s;
}
private _sendCodeInRawRepl(code: string): void {
const sanitized = Esp32Bridge._sanitizeForRepl(code);
console.log(
`[Esp32Bridge:${this.boardId}] Sending ${sanitized.length} bytes to raw REPL + Ctrl+D`,
);
if (sanitized !== code) {
console.log(
`[Esp32Bridge:${this.boardId}] Code was sanitized (non-ASCII in comments stripped)`,
);
}
const codeBytes = Array.from(new TextEncoder().encode(sanitized));
console.log(
`[Esp32Bridge:${this.boardId}] Sending ${codeBytes.length} bytes in chunks to raw REPL`,
);
// The ESP32 UART RX FIFO is 128 bytes in hardware (and in QEMU's emulation).
// Sending >128 bytes in one qemu_picsimlab_uart_receive() call overflows the
// FIFO — the extra bytes are silently dropped, corrupting the injected code
// (e.g. "time.sleep" becomes "ti" causing NameError).
// Use ≤64-byte chunks with a 150 ms gap so QEMU drains the FIFO between sends.
const CHUNK_SIZE = 64;
const CHUNK_DELAY_MS = 150;
let offset = 0;
const sendChunk = () => {
if (offset >= codeBytes.length) {
// All bytes delivered — wait for QEMU to finish processing the last chunk
setTimeout(() => {
this.sendSerialBytes([0x04]); // Ctrl+D → compile & execute
this._replState = 'idle';
console.log(`[Esp32Bridge:${this.boardId}] Ctrl+D sent — code executing`);
}, 300);
return;
}
const chunk = codeBytes.slice(offset, offset + CHUNK_SIZE);
this.sendSerialBytes(chunk);
offset += CHUNK_SIZE;
setTimeout(sendChunk, CHUNK_DELAY_MS);
};
sendChunk();
}
private _send(payload: unknown): void {
if (this.socket && this.socket.readyState === WebSocket.OPEN) {
this.socket.send(JSON.stringify(payload));
}
}
}