diff --git a/frontend/src/components/simulator/MicrophoneToggle.tsx b/frontend/src/components/simulator/MicrophoneToggle.tsx new file mode 100644 index 00000000..cc4f8ff0 --- /dev/null +++ b/frontend/src/components/simulator/MicrophoneToggle.tsx @@ -0,0 +1,100 @@ +/** + * Microphone button rendered on the canvas header for boards whose def + * declares builtInMicrophone (XIAO ESP32S3 Sense, M5Stack Cardputer ADV). + * + * Click -> asks for mic permission -> streams the computer's microphone into + * the board's emulated PDM/I2S mic through the bridge's setMicrophoneSource + * seam. While off, the bridge feeds its built-in 440 Hz test tone, so unlike + * the camera there is NO auto-start on run: sketches read a live level either + * way, and grabbing the user's microphone should take an explicit click. + * + * Visual states mirror CameraToggle: + * idle grey mic icon + * requesting orange (pulsing) — browser permission prompt visible + * streaming green + live level meter + * denied red, click-to-retry + * error red, hover for message + */ +import React from 'react'; +import { useMicrophoneStream } from '../../hooks/useMicrophoneStream'; + +interface MicrophoneToggleProps { + boardId: string | null; +} + +export const MicrophoneToggle: React.FC = ({ boardId }) => { + const { status, errorMessage, level, start, stop } = useMicrophoneStream(); + + const handleClick = () => { + if (!boardId) return; + if (status === 'streaming') { + stop(); + } else { + void start(boardId); + } + }; + + const isOn = status === 'streaming'; + const pct = Math.round(level * 100); + + const tooltip = + status === 'streaming' + ? `Streaming your microphone (level ${pct}%) — click to stop (falls back to the test tone)` + : status === 'requesting' + ? 'Asking for microphone permission…' + : status === 'denied' + ? 'Permission denied. Click to retry.' + : status === 'error' + ? `Microphone error: ${errorMessage ?? 'unknown'}` + : 'Use your microphone as the board mic (off = 440 Hz test tone)'; + + const color = + status === 'streaming' + ? '#3fb950' + : status === 'requesting' + ? '#f0883e' + : status === 'denied' || status === 'error' + ? '#f85149' + : '#ccc'; + + return ( + + ); +}; diff --git a/frontend/src/components/simulator/SimulatorCanvas.tsx b/frontend/src/components/simulator/SimulatorCanvas.tsx index 83b4ef17..f38b23fd 100644 --- a/frontend/src/components/simulator/SimulatorCanvas.tsx +++ b/frontend/src/components/simulator/SimulatorCanvas.tsx @@ -21,6 +21,7 @@ import { InstrumentComponent } from '../components-instruments/InstrumentCompone import { ComponentRegistry } from '../../services/ComponentRegistry'; import { getTabSessionId } from '../../simulation/Esp32Bridge'; import { CameraToggle } from './CameraToggle'; +import { MicrophoneToggle } from './MicrophoneToggle'; import { WireLayer } from './WireLayer'; import type { SegmentHandle, WaypointHandle, AlignmentGuide } from './WireLayer'; import { ElectricalOverlay } from '../analog-ui/ElectricalOverlay'; @@ -2708,6 +2709,16 @@ export const SimulatorCanvas = ({ headerSlot }: SimulatorCanvasProps = {}) => { )} + {/* Microphone stream toggle, for boards whose def declares an + on-board mic (their bridge implements setMicrophoneSource). + Off = the bridge's 440 Hz test tone keeps mic sketches + alive, so no auto-start: taking the user's microphone is + an explicit click. */} + {activeBoard && + getProBoard(activeBoard.boardKind)?.builtInMicrophone === true && ( + + )} + {/* WiFi status indicator + IoT-gateway launcher (ESP32 + Pico W) */} {activeBoard && (isEsp32Kind(activeBoard.boardKind) || activeBoard.boardKind === 'pi-pico-w') && diff --git a/frontend/src/hooks/useMicrophoneStream.ts b/frontend/src/hooks/useMicrophoneStream.ts new file mode 100644 index 00000000..483653e5 --- /dev/null +++ b/frontend/src/hooks/useMicrophoneStream.ts @@ -0,0 +1,193 @@ +/** + * Stream the computer's microphone into a board's emulated on-board mic. + * + * Lifecycle: + * - start(boardId): asks for mic permission, wires the capture chain. + * - stop(): releases the mic; the bridge falls back to its + * built-in 440 Hz test tone. + * - status: 'idle' | 'requesting' | 'streaming' | 'denied' | 'error'. + * + * The transport is the bridge's setMicrophoneSource(fn) seam: the engine's + * I2S RX fill pulls one signed 16-bit sample per call at 16 kHz. Boards whose + * bridge lacks the seam (QEMU path, non-audio bridges) are reported as an + * error instead of pretending to stream. + * + * Implementation notes: + * - Capture runs at the AudioContext's native rate (typically 48 kHz) and + * is decimated to 16 kHz by fractional stepping — the emulated PDM mic is + * a level-meter-grade source, not a hi-fi path. + * - The engine paces its pulls in SIMULATED cpu cycles, so during the boot + * boost it can consume up to 4x faster than real time. A 2 s ring buffer + * absorbs jitter; underflow reads as silence (0), overflow drops the + * oldest audio. Both are the honest behaviours for a live mic. + * - ScriptProcessorNode is deprecated but universal and needs no worklet + * module loading (CSP-proof). Its output buffer is never written, so + * nothing echoes to the speakers. + */ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { getEsp32Bridge, useSimulatorStore } from '../store/useSimulatorStore'; + +export type MicStatus = 'idle' | 'requesting' | 'streaming' | 'denied' | 'error'; + +export interface UseMicrophoneStreamResult { + status: MicStatus; + errorMessage: string | null; + /** Peak level of the last capture block, 0..1. Drives the live meter. */ + level: number; + start: (boardId: string) => Promise; + stop: () => void; +} + +/** The emulated microphone's sample rate (what the I2S RX path pulls at). */ +const MIC_HZ = 16_000; +/** Ring capacity: 2 s of audio at MIC_HZ. */ +const RING_CAPACITY = MIC_HZ * 2; + +interface MicBridge { + setMicrophoneSource?: (s: (() => number) | null) => void; +} + +export function useMicrophoneStream(): UseMicrophoneStreamResult { + const [status, setStatus] = useState('idle'); + const [errorMessage, setErrorMessage] = useState(null); + const [level, setLevel] = useState(0); + + const boardIdRef = useRef(null); + const streamRef = useRef(null); + const ctxRef = useRef(null); + const processorRef = useRef(null); + const sourceFnRef = useRef<(() => number) | null>(null); + + const stop = useCallback(() => { + if (boardIdRef.current) { + // null = back to the bridge's built-in test tone. + (getEsp32Bridge(boardIdRef.current) as MicBridge | undefined)?.setMicrophoneSource?.(null); + } + sourceFnRef.current = null; + processorRef.current?.disconnect(); + processorRef.current = null; + void ctxRef.current?.close().catch(() => {}); + ctxRef.current = null; + streamRef.current?.getTracks().forEach((t) => t.stop()); + streamRef.current = null; + boardIdRef.current = null; + setStatus('idle'); + setLevel(0); + }, []); + + const start = useCallback( + async (boardId: string) => { + setStatus('requesting'); + setErrorMessage(null); + + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch (err: unknown) { + const e = err as { name?: string; message?: string }; + if (e.name === 'NotAllowedError') { + setStatus('denied'); + setErrorMessage('Microphone permission denied'); + } else if (e.name === 'NotFoundError') { + setStatus('error'); + setErrorMessage('No microphone detected'); + } else { + setStatus('error'); + setErrorMessage(e.message ?? 'getUserMedia failed'); + } + return; + } + + const bridge = getEsp32Bridge(boardId) as MicBridge | undefined; + if (typeof bridge?.setMicrophoneSource !== 'function') { + stream.getTracks().forEach((t) => t.stop()); + setStatus('error'); + setErrorMessage('This board has no microphone input path'); + return; + } + + // 16 kHz signed 16-bit ring between the capture callback (writer) and + // the engine's per-sample pull (reader). + const ring = new Int16Array(RING_CAPACITY); + let readPos = 0; + let writePos = 0; + let count = 0; + + const Ctor = + window.AudioContext ?? + (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; + if (!Ctor) { + stream.getTracks().forEach((t) => t.stop()); + setStatus('error'); + setErrorMessage('AudioContext unavailable'); + return; + } + const ctx = new Ctor(); + // The toggle click is the user gesture that unlocks a suspended context. + void ctx.resume().catch(() => {}); + const micNode = ctx.createMediaStreamSource(stream); + const processor = ctx.createScriptProcessor(4096, 1, 1); + const step = ctx.sampleRate / MIC_HZ; + let frac = 0; + + processor.onaudioprocess = (e) => { + const input = e.inputBuffer.getChannelData(0); + let peak = 0; + for (let j = 0; j < input.length; j++) { + const a = input[j] < 0 ? -input[j] : input[j]; + if (a > peak) peak = a; + } + let i = frac; + for (; i < input.length; i += step) { + const v = Math.max(-1, Math.min(1, input[Math.floor(i)])); + ring[writePos] = (v * 32767) | 0; + writePos = (writePos + 1) % RING_CAPACITY; + if (count < RING_CAPACITY) { + count++; + } else { + readPos = (readPos + 1) % RING_CAPACITY; // overflow: drop oldest + } + } + frac = i - input.length; + setLevel(Math.round(peak * 100) / 100); + }; + micNode.connect(processor); + // ScriptProcessor only fires while routed to the destination; its + // output buffer stays zeroed, so nothing is audible. + processor.connect(ctx.destination); + + const sourceFn = () => { + if (count === 0) return 0; // starved: emulation pulling ahead of real time + const s = ring[readPos]; + readPos = (readPos + 1) % RING_CAPACITY; + count--; + return s; + }; + bridge.setMicrophoneSource(sourceFn); + + boardIdRef.current = boardId; + streamRef.current = stream; + ctxRef.current = ctx; + processorRef.current = processor; + sourceFnRef.current = sourceFn; + setStatus('streaming'); + }, + [], + ); + + // The pro DelegatingEsp32Bridge stashes the source across Run/Stop cycles, + // but a board-type change rebuilds the whole wrapper and loses it. Re-apply + // on every run start so the stream survives either path. + const running = useSimulatorStore((st) => st.running); + useEffect(() => { + if (!running || status !== 'streaming' || !boardIdRef.current || !sourceFnRef.current) return; + (getEsp32Bridge(boardIdRef.current) as MicBridge | undefined)?.setMicrophoneSource?.( + sourceFnRef.current, + ); + }, [running, status]); + + // Release the mic when the owner unmounts. + useEffect(() => stop, [stop]); + + return { status, errorMessage, level, start, stop }; +} diff --git a/frontend/src/lib/proBoardRegistry.ts b/frontend/src/lib/proBoardRegistry.ts index 64fc90f4..c6ae06c6 100644 --- a/frontend/src/lib/proBoardRegistry.ts +++ b/frontend/src/lib/proBoardRegistry.ts @@ -101,6 +101,10 @@ export interface ProBoardDef { /** Built-in microSD on a shared SPI bus: the CS pin the bridge must gate. * (A standalone SD card component still overrides this to un-gated.) */ builtInSdCsPin?: number; + /** Board carries an on-board microphone whose bridge implements + * setMicrophoneSource (I2S RX sample injection): shows the canvas-header + * Mic toggle that streams the computer's microphone into it. */ + builtInMicrophone?: boolean; /** Built-in peripheral attachment for a RUNNING board (LCD decoder onto the * element's own canvas, speaker, on-board button/keyboard event forwarding). * Called shortly after run start with the board's DOM element plus its