feat(mic): stream the computer's microphone into boards with an on-board mic
New canvas-header Mic toggle, shown for boards whose ProBoardDef declares builtInMicrophone (their bridge implements setMicrophoneSource). Capture runs at the AudioContext's native rate, is decimated to 16 kHz and ring- buffered (2 s, drop-oldest on overflow, silence on underflow - the engine paces pulls in simulated cycles, up to 4x real time during boot boost). No auto-start: with the toggle off the bridge's 440 Hz test tone keeps mic sketches alive, so taking the user's microphone stays an explicit click. The source is re-applied on every run start to survive bridge rebuilds.
This commit is contained in:
David Montero Crespo
parent
bd1ee89b5e
commit
e96dae794a
|
|
@ -0,0 +1,100 @@
|
|||
/**
|
||||
* Microphone button rendered on the canvas header for boards whose def
|
||||
* declares builtInMicrophone (XIAO ESP32S3 Sense, M5Stack Cardputer ADV).
|
||||
*
|
||||
* Click -> asks for mic permission -> streams the computer's microphone into
|
||||
* the board's emulated PDM/I2S mic through the bridge's setMicrophoneSource
|
||||
* seam. While off, the bridge feeds its built-in 440 Hz test tone, so unlike
|
||||
* the camera there is NO auto-start on run: sketches read a live level either
|
||||
* way, and grabbing the user's microphone should take an explicit click.
|
||||
*
|
||||
* Visual states mirror CameraToggle:
|
||||
* idle grey mic icon
|
||||
* requesting orange (pulsing) — browser permission prompt visible
|
||||
* streaming green + live level meter
|
||||
* denied red, click-to-retry
|
||||
* error red, hover for message
|
||||
*/
|
||||
import React from 'react';
|
||||
import { useMicrophoneStream } from '../../hooks/useMicrophoneStream';
|
||||
|
||||
interface MicrophoneToggleProps {
|
||||
boardId: string | null;
|
||||
}
|
||||
|
||||
export const MicrophoneToggle: React.FC<MicrophoneToggleProps> = ({ boardId }) => {
|
||||
const { status, errorMessage, level, start, stop } = useMicrophoneStream();
|
||||
|
||||
const handleClick = () => {
|
||||
if (!boardId) return;
|
||||
if (status === 'streaming') {
|
||||
stop();
|
||||
} else {
|
||||
void start(boardId);
|
||||
}
|
||||
};
|
||||
|
||||
const isOn = status === 'streaming';
|
||||
const pct = Math.round(level * 100);
|
||||
|
||||
const tooltip =
|
||||
status === 'streaming'
|
||||
? `Streaming your microphone (level ${pct}%) — click to stop (falls back to the test tone)`
|
||||
: status === 'requesting'
|
||||
? 'Asking for microphone permission…'
|
||||
: status === 'denied'
|
||||
? 'Permission denied. Click to retry.'
|
||||
: status === 'error'
|
||||
? `Microphone error: ${errorMessage ?? 'unknown'}`
|
||||
: 'Use your microphone as the board mic (off = 440 Hz test tone)';
|
||||
|
||||
const color =
|
||||
status === 'streaming'
|
||||
? '#3fb950'
|
||||
: status === 'requesting'
|
||||
? '#f0883e'
|
||||
: status === 'denied' || status === 'error'
|
||||
? '#f85149'
|
||||
: '#ccc';
|
||||
|
||||
return (
|
||||
<button
|
||||
onClick={handleClick}
|
||||
disabled={!boardId || status === 'requesting'}
|
||||
title={tooltip}
|
||||
style={{
|
||||
backgroundColor: isOn ? 'rgba(63,185,80,0.15)' : 'transparent',
|
||||
border: `1px solid ${
|
||||
isOn ? '#3fb950' : status === 'requesting' ? '#555' : 'transparent'
|
||||
}`,
|
||||
borderRadius: 4,
|
||||
padding: '4px 10px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 6,
|
||||
color,
|
||||
fontSize: 13,
|
||||
cursor: boardId ? 'pointer' : 'not-allowed',
|
||||
animation:
|
||||
status === 'requesting' ? 'velxio-pulse 1s ease-in-out infinite' : undefined,
|
||||
}}
|
||||
>
|
||||
<svg
|
||||
width="16"
|
||||
height="16"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
|
||||
<path d="M19 10v2a7 7 0 0 1-14 0v-2" />
|
||||
<line x1="12" y1="19" x2="12" y2="23" />
|
||||
<line x1="8" y1="23" x2="16" y2="23" />
|
||||
</svg>
|
||||
<span>{isOn ? `Mic ${pct}%` : 'Mic'}</span>
|
||||
</button>
|
||||
);
|
||||
};
|
||||
|
|
@ -21,6 +21,7 @@ import { InstrumentComponent } from '../components-instruments/InstrumentCompone
|
|||
import { ComponentRegistry } from '../../services/ComponentRegistry';
|
||||
import { getTabSessionId } from '../../simulation/Esp32Bridge';
|
||||
import { CameraToggle } from './CameraToggle';
|
||||
import { MicrophoneToggle } from './MicrophoneToggle';
|
||||
import { WireLayer } from './WireLayer';
|
||||
import type { SegmentHandle, WaypointHandle, AlignmentGuide } from './WireLayer';
|
||||
import { ElectricalOverlay } from '../analog-ui/ElectricalOverlay';
|
||||
|
|
@ -2708,6 +2709,16 @@ export const SimulatorCanvas = ({ headerSlot }: SimulatorCanvasProps = {}) => {
|
|||
<CameraToggle boardId={activeBoard.id} />
|
||||
)}
|
||||
|
||||
{/* Microphone stream toggle, for boards whose def declares an
|
||||
on-board mic (their bridge implements setMicrophoneSource).
|
||||
Off = the bridge's 440 Hz test tone keeps mic sketches
|
||||
alive, so no auto-start: taking the user's microphone is
|
||||
an explicit click. */}
|
||||
{activeBoard &&
|
||||
getProBoard(activeBoard.boardKind)?.builtInMicrophone === true && (
|
||||
<MicrophoneToggle boardId={activeBoard.id} />
|
||||
)}
|
||||
|
||||
{/* WiFi status indicator + IoT-gateway launcher (ESP32 + Pico W) */}
|
||||
{activeBoard &&
|
||||
(isEsp32Kind(activeBoard.boardKind) || activeBoard.boardKind === 'pi-pico-w') &&
|
||||
|
|
|
|||
|
|
@ -0,0 +1,193 @@
|
|||
/**
|
||||
* Stream the computer's microphone into a board's emulated on-board mic.
|
||||
*
|
||||
* Lifecycle:
|
||||
* - start(boardId): asks for mic permission, wires the capture chain.
|
||||
* - stop(): releases the mic; the bridge falls back to its
|
||||
* built-in 440 Hz test tone.
|
||||
* - status: 'idle' | 'requesting' | 'streaming' | 'denied' | 'error'.
|
||||
*
|
||||
* The transport is the bridge's setMicrophoneSource(fn) seam: the engine's
|
||||
* I2S RX fill pulls one signed 16-bit sample per call at 16 kHz. Boards whose
|
||||
* bridge lacks the seam (QEMU path, non-audio bridges) are reported as an
|
||||
* error instead of pretending to stream.
|
||||
*
|
||||
* Implementation notes:
|
||||
* - Capture runs at the AudioContext's native rate (typically 48 kHz) and
|
||||
* is decimated to 16 kHz by fractional stepping — the emulated PDM mic is
|
||||
* a level-meter-grade source, not a hi-fi path.
|
||||
* - The engine paces its pulls in SIMULATED cpu cycles, so during the boot
|
||||
* boost it can consume up to 4x faster than real time. A 2 s ring buffer
|
||||
* absorbs jitter; underflow reads as silence (0), overflow drops the
|
||||
* oldest audio. Both are the honest behaviours for a live mic.
|
||||
* - ScriptProcessorNode is deprecated but universal and needs no worklet
|
||||
* module loading (CSP-proof). Its output buffer is never written, so
|
||||
* nothing echoes to the speakers.
|
||||
*/
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { getEsp32Bridge, useSimulatorStore } from '../store/useSimulatorStore';
|
||||
|
||||
export type MicStatus = 'idle' | 'requesting' | 'streaming' | 'denied' | 'error';
|
||||
|
||||
export interface UseMicrophoneStreamResult {
|
||||
status: MicStatus;
|
||||
errorMessage: string | null;
|
||||
/** Peak level of the last capture block, 0..1. Drives the live meter. */
|
||||
level: number;
|
||||
start: (boardId: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
}
|
||||
|
||||
/** The emulated microphone's sample rate (what the I2S RX path pulls at). */
|
||||
const MIC_HZ = 16_000;
|
||||
/** Ring capacity: 2 s of audio at MIC_HZ. */
|
||||
const RING_CAPACITY = MIC_HZ * 2;
|
||||
|
||||
interface MicBridge {
|
||||
setMicrophoneSource?: (s: (() => number) | null) => void;
|
||||
}
|
||||
|
||||
export function useMicrophoneStream(): UseMicrophoneStreamResult {
|
||||
const [status, setStatus] = useState<MicStatus>('idle');
|
||||
const [errorMessage, setErrorMessage] = useState<string | null>(null);
|
||||
const [level, setLevel] = useState(0);
|
||||
|
||||
const boardIdRef = useRef<string | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const ctxRef = useRef<AudioContext | null>(null);
|
||||
const processorRef = useRef<ScriptProcessorNode | null>(null);
|
||||
const sourceFnRef = useRef<(() => number) | null>(null);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
if (boardIdRef.current) {
|
||||
// null = back to the bridge's built-in test tone.
|
||||
(getEsp32Bridge(boardIdRef.current) as MicBridge | undefined)?.setMicrophoneSource?.(null);
|
||||
}
|
||||
sourceFnRef.current = null;
|
||||
processorRef.current?.disconnect();
|
||||
processorRef.current = null;
|
||||
void ctxRef.current?.close().catch(() => {});
|
||||
ctxRef.current = null;
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
boardIdRef.current = null;
|
||||
setStatus('idle');
|
||||
setLevel(0);
|
||||
}, []);
|
||||
|
||||
const start = useCallback(
|
||||
async (boardId: string) => {
|
||||
setStatus('requesting');
|
||||
setErrorMessage(null);
|
||||
|
||||
let stream: MediaStream;
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
} catch (err: unknown) {
|
||||
const e = err as { name?: string; message?: string };
|
||||
if (e.name === 'NotAllowedError') {
|
||||
setStatus('denied');
|
||||
setErrorMessage('Microphone permission denied');
|
||||
} else if (e.name === 'NotFoundError') {
|
||||
setStatus('error');
|
||||
setErrorMessage('No microphone detected');
|
||||
} else {
|
||||
setStatus('error');
|
||||
setErrorMessage(e.message ?? 'getUserMedia failed');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const bridge = getEsp32Bridge(boardId) as MicBridge | undefined;
|
||||
if (typeof bridge?.setMicrophoneSource !== 'function') {
|
||||
stream.getTracks().forEach((t) => t.stop());
|
||||
setStatus('error');
|
||||
setErrorMessage('This board has no microphone input path');
|
||||
return;
|
||||
}
|
||||
|
||||
// 16 kHz signed 16-bit ring between the capture callback (writer) and
|
||||
// the engine's per-sample pull (reader).
|
||||
const ring = new Int16Array(RING_CAPACITY);
|
||||
let readPos = 0;
|
||||
let writePos = 0;
|
||||
let count = 0;
|
||||
|
||||
const Ctor =
|
||||
window.AudioContext ??
|
||||
(window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
|
||||
if (!Ctor) {
|
||||
stream.getTracks().forEach((t) => t.stop());
|
||||
setStatus('error');
|
||||
setErrorMessage('AudioContext unavailable');
|
||||
return;
|
||||
}
|
||||
const ctx = new Ctor();
|
||||
// The toggle click is the user gesture that unlocks a suspended context.
|
||||
void ctx.resume().catch(() => {});
|
||||
const micNode = ctx.createMediaStreamSource(stream);
|
||||
const processor = ctx.createScriptProcessor(4096, 1, 1);
|
||||
const step = ctx.sampleRate / MIC_HZ;
|
||||
let frac = 0;
|
||||
|
||||
processor.onaudioprocess = (e) => {
|
||||
const input = e.inputBuffer.getChannelData(0);
|
||||
let peak = 0;
|
||||
for (let j = 0; j < input.length; j++) {
|
||||
const a = input[j] < 0 ? -input[j] : input[j];
|
||||
if (a > peak) peak = a;
|
||||
}
|
||||
let i = frac;
|
||||
for (; i < input.length; i += step) {
|
||||
const v = Math.max(-1, Math.min(1, input[Math.floor(i)]));
|
||||
ring[writePos] = (v * 32767) | 0;
|
||||
writePos = (writePos + 1) % RING_CAPACITY;
|
||||
if (count < RING_CAPACITY) {
|
||||
count++;
|
||||
} else {
|
||||
readPos = (readPos + 1) % RING_CAPACITY; // overflow: drop oldest
|
||||
}
|
||||
}
|
||||
frac = i - input.length;
|
||||
setLevel(Math.round(peak * 100) / 100);
|
||||
};
|
||||
micNode.connect(processor);
|
||||
// ScriptProcessor only fires while routed to the destination; its
|
||||
// output buffer stays zeroed, so nothing is audible.
|
||||
processor.connect(ctx.destination);
|
||||
|
||||
const sourceFn = () => {
|
||||
if (count === 0) return 0; // starved: emulation pulling ahead of real time
|
||||
const s = ring[readPos];
|
||||
readPos = (readPos + 1) % RING_CAPACITY;
|
||||
count--;
|
||||
return s;
|
||||
};
|
||||
bridge.setMicrophoneSource(sourceFn);
|
||||
|
||||
boardIdRef.current = boardId;
|
||||
streamRef.current = stream;
|
||||
ctxRef.current = ctx;
|
||||
processorRef.current = processor;
|
||||
sourceFnRef.current = sourceFn;
|
||||
setStatus('streaming');
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
// The pro DelegatingEsp32Bridge stashes the source across Run/Stop cycles,
|
||||
// but a board-type change rebuilds the whole wrapper and loses it. Re-apply
|
||||
// on every run start so the stream survives either path.
|
||||
const running = useSimulatorStore((st) => st.running);
|
||||
useEffect(() => {
|
||||
if (!running || status !== 'streaming' || !boardIdRef.current || !sourceFnRef.current) return;
|
||||
(getEsp32Bridge(boardIdRef.current) as MicBridge | undefined)?.setMicrophoneSource?.(
|
||||
sourceFnRef.current,
|
||||
);
|
||||
}, [running, status]);
|
||||
|
||||
// Release the mic when the owner unmounts.
|
||||
useEffect(() => stop, [stop]);
|
||||
|
||||
return { status, errorMessage, level, start, stop };
|
||||
}
|
||||
|
|
@ -101,6 +101,10 @@ export interface ProBoardDef {
|
|||
/** Built-in microSD on a shared SPI bus: the CS pin the bridge must gate.
|
||||
* (A standalone SD card component still overrides this to un-gated.) */
|
||||
builtInSdCsPin?: number;
|
||||
/** Board carries an on-board microphone whose bridge implements
|
||||
* setMicrophoneSource (I2S RX sample injection): shows the canvas-header
|
||||
* Mic toggle that streams the computer's microphone into it. */
|
||||
builtInMicrophone?: boolean;
|
||||
/** Built-in peripheral attachment for a RUNNING board (LCD decoder onto the
|
||||
* element's own canvas, speaker, on-board button/keyboard event forwarding).
|
||||
* Called shortly after run start with the board's DOM element plus its
|
||||
|
|
|
|||
Loading…
Reference in New Issue