fix(webcam): per-board frame byte budget - Sense frames fit the S3 fb
Real webcams hit the 23000-byte bound, but the S3 esp32-camera build allocates width*height/5 = 15360 bytes for a QVGA JPEG frame buffer and stops copying one 1 KiB DMA half-buffer short of it, so oversized frames lost their tail - including the EOI marker cam_hal scans for (the NO-EOI storm on staging; the small canvas-fake frames masked it). The bounded encoder now takes a budget (14000 for the Sense, default 23000 for the QEMU ESP32-CAM path unchanged), the downscale fallback re-runs its own quality ladder, and a frame that cannot fit is skipped instead of sent truncated.
This commit is contained in:
parent
5a71023ac9
commit
57e04b60e8
|
|
@ -20,9 +20,12 @@ import { useSimulatorStore } from '../../store/useSimulatorStore';
|
|||
|
||||
interface CameraToggleProps {
|
||||
boardId: string | null;
|
||||
/** Per-board frame byte budget (see useWebcamFrames). Omit for the
|
||||
* default QEMU ESP32-CAM bound. */
|
||||
maxFrameBytes?: number;
|
||||
}
|
||||
|
||||
export const CameraToggle: React.FC<CameraToggleProps> = ({ boardId }) => {
|
||||
export const CameraToggle: React.FC<CameraToggleProps> = ({ boardId, maxFrameBytes }) => {
|
||||
const {
|
||||
status,
|
||||
errorMessage,
|
||||
|
|
@ -32,7 +35,7 @@ export const CameraToggle: React.FC<CameraToggleProps> = ({ boardId }) => {
|
|||
lastDownscaled,
|
||||
start,
|
||||
stop,
|
||||
} = useWebcamFrames();
|
||||
} = useWebcamFrames(maxFrameBytes);
|
||||
|
||||
// Auto-start when the sketch RUNS. An ESP32-CAM sketch exists to capture, so
|
||||
// waiting for a click on this toggle just reads as "the camera is broken":
|
||||
|
|
|
|||
|
|
@ -2706,7 +2706,16 @@ export const SimulatorCanvas = ({ headerSlot }: SimulatorCanvasProps = {}) => {
|
|||
Sense (OV2640-compatible over LCD_CAM). */}
|
||||
{(activeBoard?.boardKind === 'esp32-cam' ||
|
||||
activeBoard?.boardKind === 'xiao-esp32s3-sense') && (
|
||||
<CameraToggle boardId={activeBoard.id} />
|
||||
<CameraToggle
|
||||
boardId={activeBoard.id}
|
||||
// The S3 esp32-camera build allocates width*height/5 bytes
|
||||
// for a QVGA JPEG frame (15360) and stops copying at
|
||||
// fb_size - one 1 KiB DMA half-buffer: frames must stay
|
||||
// under ~14336 or the EOI marker is truncated (NO-EOI).
|
||||
maxFrameBytes={
|
||||
activeBoard.boardKind === 'xiao-esp32s3-sense' ? 14000 : undefined
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Microphone stream toggle, for boards whose def declares an
|
||||
|
|
|
|||
|
|
@ -130,17 +130,19 @@ function downscaleCanvas(
|
|||
}
|
||||
|
||||
/** Encode the canvas to JPEG with progressively lower quality until
|
||||
* the result fits in MAX_FRAME_BYTES. Falls back to a 240×180
|
||||
* downscale if even quality 0.1 at full resolution is too large.
|
||||
* Returns null only when the canvas is invalid or the browser
|
||||
* refuses to encode at any quality (very rare). */
|
||||
* the result fits in `maxBytes`. Falls back to a 240×180 downscale
|
||||
* (with its own quality ladder) if full resolution cannot fit.
|
||||
* Returns null when the canvas is invalid, the browser refuses to
|
||||
* encode, or nothing fits the budget — a skipped frame is recoverable,
|
||||
* an oversized one corrupts the guest's frame buffer (truncated EOI). */
|
||||
async function encodeBoundedJpeg(
|
||||
c: OffscreenCanvas | HTMLCanvasElement,
|
||||
maxBytes: number,
|
||||
): Promise<EncodedFrame | null> {
|
||||
for (const q of QUALITY_LADDER) {
|
||||
const blob = await canvasToJpeg(c, q);
|
||||
if (!blob) return null;
|
||||
if (blob.size <= MAX_FRAME_BYTES) {
|
||||
if (blob.size <= maxBytes) {
|
||||
return {
|
||||
buf: await blob.arrayBuffer(),
|
||||
bytes: blob.size,
|
||||
|
|
@ -149,20 +151,32 @@ async function encodeBoundedJpeg(
|
|||
};
|
||||
}
|
||||
}
|
||||
// Worst case: HD/4K webcam, ultra-detailed scene, quality 0.1
|
||||
// still overshoots even the 23 KiB cap. Downscale + medium quality.
|
||||
// Worst case: HD/4K webcam, ultra-detailed scene — quality 0.1 at full
|
||||
// resolution still overshoots the budget. Downscale and re-run the ladder.
|
||||
const small = downscaleCanvas(c, FALLBACK_W, FALLBACK_H);
|
||||
const blob = await canvasToJpeg(small, 0.4);
|
||||
if (!blob) return null;
|
||||
return {
|
||||
buf: await blob.arrayBuffer(),
|
||||
bytes: blob.size,
|
||||
quality: 0.4,
|
||||
downscaled: true,
|
||||
};
|
||||
for (const q of [0.4, 0.3, 0.2, 0.1]) {
|
||||
const blob = await canvasToJpeg(small, q);
|
||||
if (!blob) return null;
|
||||
if (blob.size <= maxBytes) {
|
||||
return {
|
||||
buf: await blob.arrayBuffer(),
|
||||
bytes: blob.size,
|
||||
quality: q,
|
||||
downscaled: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function useWebcamFrames(): UseWebcamFramesResult {
|
||||
export function useWebcamFrames(
|
||||
/** Per-board frame byte budget. The default fits the QEMU ESP32-CAM's
|
||||
* 32 KiB walker; boards whose camera driver allocates a smaller frame
|
||||
* buffer (the S3's esp32-camera build: width*height/5 for JPEG) must
|
||||
* pass a tighter bound or the JPEG tail — including the EOI marker the
|
||||
* driver scans for — is truncated on copy (cam_hal "NO-EOI"). */
|
||||
maxFrameBytes: number = MAX_FRAME_BYTES,
|
||||
): UseWebcamFramesResult {
|
||||
const [status, setStatus] = useState<WebcamStatus>('idle');
|
||||
const [errorMessage, setErrorMessage] = useState<string | null>(null);
|
||||
const [framesSent, setFramesSent] = useState(0);
|
||||
|
|
@ -170,6 +184,9 @@ export function useWebcamFrames(): UseWebcamFramesResult {
|
|||
const [lastQualityUsed, setLastQualityUsed] = useState(0.5);
|
||||
const [lastDownscaled, setLastDownscaled] = useState(false);
|
||||
|
||||
const maxBytesRef = useRef(maxFrameBytes);
|
||||
maxBytesRef.current = maxFrameBytes;
|
||||
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const timerRef = useRef<number | null>(null);
|
||||
|
|
@ -280,7 +297,7 @@ export function useWebcamFrames(): UseWebcamFramesResult {
|
|||
|
||||
// Use the bounded encoder so the JPEG always fits in the
|
||||
// emulator's per-frame budget regardless of webcam hardware.
|
||||
const encoded = await encodeBoundedJpeg(c);
|
||||
const encoded = await encodeBoundedJpeg(c, maxBytesRef.current);
|
||||
if (!encoded) return;
|
||||
const id = boardIdRef.current;
|
||||
if (!id) return;
|
||||
|
|
|
|||
Loading…
Reference in New Issue