fix(webcam): per-board frame byte budget - Sense frames fit the S3 fb

Real webcams hit the 23000-byte bound, but the S3 esp32-camera build
allocates width*height/5 = 15360 bytes for a QVGA JPEG frame buffer and
stops copying one 1 KiB DMA half-buffer short of it, so oversized frames
lost their tail - including the EOI marker cam_hal scans for (the NO-EOI
storm on staging; the small canvas-fake frames masked it). The bounded
encoder now takes a budget (14000 for the Sense, default 23000 for the
QEMU ESP32-CAM path unchanged), the downscale fallback re-runs its own
quality ladder, and a frame that cannot fit is skipped instead of sent
truncated.
This commit is contained in:
David Montero Crespo 2026-08-04 16:18:10 +02:00
parent 5a71023ac9
commit 57e04b60e8
3 changed files with 49 additions and 20 deletions

View File

@ -20,9 +20,12 @@ import { useSimulatorStore } from '../../store/useSimulatorStore';
interface CameraToggleProps {
boardId: string | null;
/** Per-board frame byte budget (see useWebcamFrames). Omit for the
* default QEMU ESP32-CAM bound. */
maxFrameBytes?: number;
}
export const CameraToggle: React.FC<CameraToggleProps> = ({ boardId }) => {
export const CameraToggle: React.FC<CameraToggleProps> = ({ boardId, maxFrameBytes }) => {
const {
status,
errorMessage,
@ -32,7 +35,7 @@ export const CameraToggle: React.FC<CameraToggleProps> = ({ boardId }) => {
lastDownscaled,
start,
stop,
} = useWebcamFrames();
} = useWebcamFrames(maxFrameBytes);
// Auto-start when the sketch RUNS. An ESP32-CAM sketch exists to capture, so
// waiting for a click on this toggle just reads as "the camera is broken":

View File

@ -2706,7 +2706,16 @@ export const SimulatorCanvas = ({ headerSlot }: SimulatorCanvasProps = {}) => {
Sense (OV2640-compatible over LCD_CAM). */}
{(activeBoard?.boardKind === 'esp32-cam' ||
activeBoard?.boardKind === 'xiao-esp32s3-sense') && (
<CameraToggle boardId={activeBoard.id} />
<CameraToggle
boardId={activeBoard.id}
// The S3 esp32-camera build allocates width*height/5 bytes
// for a QVGA JPEG frame (15360) and stops copying at
// fb_size - one 1 KiB DMA half-buffer: frames must stay
// under ~14336 or the EOI marker is truncated (NO-EOI).
maxFrameBytes={
activeBoard.boardKind === 'xiao-esp32s3-sense' ? 14000 : undefined
}
/>
)}
{/* Microphone stream toggle, for boards whose def declares an

View File

@ -130,17 +130,19 @@ function downscaleCanvas(
}
/** Encode the canvas to JPEG with progressively lower quality until
* the result fits in MAX_FRAME_BYTES. Falls back to a 240×180
* downscale if even quality 0.1 at full resolution is too large.
* Returns null only when the canvas is invalid or the browser
* refuses to encode at any quality (very rare). */
* the result fits in `maxBytes`. Falls back to a 240×180 downscale
* (with its own quality ladder) if full resolution cannot fit.
* Returns null when the canvas is invalid, the browser refuses to
* encode, or nothing fits the budget — a skipped frame is recoverable,
* an oversized one corrupts the guest's frame buffer (truncated EOI). */
async function encodeBoundedJpeg(
c: OffscreenCanvas | HTMLCanvasElement,
maxBytes: number,
): Promise<EncodedFrame | null> {
for (const q of QUALITY_LADDER) {
const blob = await canvasToJpeg(c, q);
if (!blob) return null;
if (blob.size <= MAX_FRAME_BYTES) {
if (blob.size <= maxBytes) {
return {
buf: await blob.arrayBuffer(),
bytes: blob.size,
@ -149,20 +151,32 @@ async function encodeBoundedJpeg(
};
}
}
// Worst case: HD/4K webcam, ultra-detailed scene, quality 0.1
// still overshoots even the 23 KiB cap. Downscale + medium quality.
// Worst case: HD/4K webcam, ultra-detailed scene — quality 0.1 at full
// resolution still overshoots the budget. Downscale and re-run the ladder.
const small = downscaleCanvas(c, FALLBACK_W, FALLBACK_H);
const blob = await canvasToJpeg(small, 0.4);
if (!blob) return null;
return {
buf: await blob.arrayBuffer(),
bytes: blob.size,
quality: 0.4,
downscaled: true,
};
for (const q of [0.4, 0.3, 0.2, 0.1]) {
const blob = await canvasToJpeg(small, q);
if (!blob) return null;
if (blob.size <= maxBytes) {
return {
buf: await blob.arrayBuffer(),
bytes: blob.size,
quality: q,
downscaled: true,
};
}
}
return null;
}
export function useWebcamFrames(): UseWebcamFramesResult {
export function useWebcamFrames(
/** Per-board frame byte budget. The default fits the QEMU ESP32-CAM's
* 32 KiB walker; boards whose camera driver allocates a smaller frame
* buffer (the S3's esp32-camera build: width*height/5 for JPEG) must
* pass a tighter bound or the JPEG tail — including the EOI marker the
* driver scans for — is truncated on copy (cam_hal "NO-EOI"). */
maxFrameBytes: number = MAX_FRAME_BYTES,
): UseWebcamFramesResult {
const [status, setStatus] = useState<WebcamStatus>('idle');
const [errorMessage, setErrorMessage] = useState<string | null>(null);
const [framesSent, setFramesSent] = useState(0);
@ -170,6 +184,9 @@ export function useWebcamFrames(): UseWebcamFramesResult {
const [lastQualityUsed, setLastQualityUsed] = useState(0.5);
const [lastDownscaled, setLastDownscaled] = useState(false);
const maxBytesRef = useRef(maxFrameBytes);
maxBytesRef.current = maxFrameBytes;
const videoRef = useRef<HTMLVideoElement | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const timerRef = useRef<number | null>(null);
@ -280,7 +297,7 @@ export function useWebcamFrames(): UseWebcamFramesResult {
// Use the bounded encoder so the JPEG always fits in the
// emulator's per-frame budget regardless of webcam hardware.
const encoded = await encodeBoundedJpeg(c);
const encoded = await encodeBoundedJpeg(c, maxBytesRef.current);
if (!encoded) return;
const id = boardIdRef.current;
if (!id) return;