feat(pi3): switch from raspi3b to virt + virtio (Phase 1)
raspi3b pl011 RX is broken in QEMU 10 + kernel 6.12 — see
project/pi-emulation/decisions.md for the full debugging trail.
This commit lands Phase 1 of the rebuild: switch the QEMU machine
to virt + cortex-a53, boot the velxio kernel/initramfs/rootfs over
virtio-blk-pci, and expose the user shell on /dev/hvc0 via
virtio-serial-pci + virtconsole.
End-to-end smoke verified: boot → agetty autologin → bash prompt →
echo round-trip returns the typed token. Tested inside the prod
container with QEMU 10.0.8 and our cloud-derived kernel 6.12.88.
What changed:
backend/app/services/qemu_manager.py
PI3_IMAGE_SET -> raspberry-pi-3-virt
PI3_KERNEL_NAME / PI3_INITRAMFS_NAME / PI3_ROOTFS_NAME new
QEMU cmd rewritten end-to-end:
-M virt -cpu cortex-a53 -smp 4 -m 1G
-kernel <velxio-kernel-arm64> -initrd <velxio-initramfs-arm64.cpio.gz>
-drive ... -device virtio-blk-pci (NOT virtio-blk-device — mmio
variant left /dev/vda unregistered)
-nic none -display none -monitor none -serial none
-chardev socket... -device virtio-serial-pci -device virtconsole
(user console -> /dev/hvc0)
-chardev socket... -device virtserialport,name=velxio-protocol
(Phase 2 channel -> /dev/vport0p2)
No -dtb (virt generates its own), no -append init=... (kernel runs
our initramfs which then switch_root to rootfs and exec's its
/sbin/init — Alpine OpenRC).
backend/app/services/boot_images/manifest.json
New image set raspberry-pi-3-virt with three assets uploaded via
the existing license-endpoint pipeline. Old raspberry-pi-3 entry
flagged deprecated:true and kept for one release for rollback.
test/pi3_console_boot/test_pi3_console_boot.py
Updated QEMU argv to match qemu_manager exactly. Markers now look
for the Velxio Pi Simulator MOTD + 'login on hvc0' (autologin
proof). Round-trip echo still required to pass.
This commit is contained in:
davidmonterocrespo24
parent
174b4b94e8
commit
a7472e3411
|
|
@ -2,8 +2,40 @@
|
|||
"_comment": "Source of truth for QEMU boot files. Adding a new board kind means appending an entry under image_sets; the BootImageProvider picks it up via load_manifest() with no Python edits. SHA256s are verified after download; mismatches raise IntegrityError and the cache slot is not populated. See docs/BOOT_IMAGES.md for the architecture.",
|
||||
"version": 1,
|
||||
"image_sets": {
|
||||
"raspberry-pi-3-virt": {
|
||||
"description": "Raspberry Pi 3 simulator on QEMU -M virt + cortex-a53. Velxio-built minimal ARM64 kernel + initramfs + Alpine-based rootfs. Replaces the broken raspi3b path (pl011 RX bug in QEMU 10 + kernel 6.12 — see project/pi-emulation/decisions.md for the full trail). Used by app.services.qemu_manager.",
|
||||
"images": [
|
||||
{
|
||||
"name": "velxio-kernel-arm64",
|
||||
"asset_id": "velxio-kernel-arm64",
|
||||
"sha256": "76652d46bee27fe46f92c21adf9e67964356774ff4f6abe4f009344aacd3acd9",
|
||||
"size_bytes": 30697408,
|
||||
"version": "2026.05+v6.12.88-deb13-cloud"
|
||||
},
|
||||
{
|
||||
"name": "velxio-initramfs-arm64.cpio.gz",
|
||||
"asset_id": "velxio-initramfs-arm64",
|
||||
"sha256": "9a1ecd1c83a65207562b8395fcc57cc9218f16cf240dd4b5cf84e13ef0b34efe",
|
||||
"size_bytes": 1032327,
|
||||
"version": "2026.05"
|
||||
},
|
||||
{
|
||||
"name": "velxio-pi-rootfs-arm64.ext4",
|
||||
"asset_id": "velxio-pi-rootfs-arm64-zst",
|
||||
"sha256": "d09eec3bbc023dc4798d70de87048b25d2db81e5530fe0fbe0853f745bdaf910",
|
||||
"size_bytes": 629145600,
|
||||
"version": "2026.05",
|
||||
"compressed": {
|
||||
"encoding": "zstd",
|
||||
"sha256": "de55525e8475eea17fe7f4c26491154a998f7379ed82a651f0d10550cb9c997f",
|
||||
"size_bytes": 24725464
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"raspberry-pi-3": {
|
||||
"description": "Raspberry Pi 3 Model B boot files — kernel, device tree, and Raspberry Pi OS Trixie (armhf) SD image. Targets the QEMU 'raspi3b' machine type. The SD image ships compressed (zstd -19) to cut the licence-endpoint download from ~5.3 GiB to ~1.4 GiB; the provider decompresses on the host into the named volume cache.",
|
||||
"description": "[DEPRECATED — kept for one release to ease rollback] Original raspi3b setup. Replaced by 'raspberry-pi-3-virt' (see above). qemu_manager.py no longer references this entry; do not add new clients here.",
|
||||
"deprecated": true,
|
||||
"images": [
|
||||
{
|
||||
"name": "kernel8.img",
|
||||
|
|
|
|||
|
|
@ -1,27 +1,32 @@
|
|||
"""
|
||||
QemuManager — backend service for Raspberry Pi 3B emulation via QEMU.
|
||||
QemuManager — backend service for Raspberry Pi simulator emulation via QEMU.
|
||||
|
||||
Architecture
|
||||
------------
|
||||
Each Pi board instance gets:
|
||||
- qemu-system-aarch64 process (raspi3b, ARM64)
|
||||
- ttyAMA0 (serial0) → TCP socket on a dynamic port → user serial I/O
|
||||
- ttyAMA1 (serial1) → TCP socket on a dynamic port → GPIO shim protocol
|
||||
- A fresh qcow2 overlay over the base SD image (copy-on-write, discarded on stop)
|
||||
- qemu-system-aarch64 process (-M virt -cpu cortex-a53 for Pi 3, other
|
||||
cpu models for the rest of the family — see Phase 3 plan)
|
||||
- Velxio-built kernel + initramfs + rootfs (not the rpi-firmware kernel
|
||||
and not raspios; see ``project/pi-emulation/`` for why)
|
||||
- virtio-blk root from a qcow2 overlay over the cached rootfs ext4
|
||||
- virtio-console on chardev 0 (TCP socket) — the user shell at /dev/hvc0
|
||||
- virtio-serial port on chardev 1 (TCP socket) — multiplexed text
|
||||
protocol channel for GPIO/I2C/SPI/UART/PWM (Phase 2 wires it up)
|
||||
|
||||
Boot files (kernel8.img, device tree, Raspberry Pi OS SD image) are NOT
|
||||
shipped in the repo or baked into the Docker image. They're resolved at
|
||||
runtime via :class:`app.services.boot_images.BootImageProvider`, which
|
||||
downloads + SHA256-verifies + decompresses them on first use and caches
|
||||
them under ``/var/cache/velxio/boot-images/raspberry-pi-3/``. The
|
||||
lifespan hook at the bottom of this module pre-warms the cache at
|
||||
process startup so a first-time user request doesn't pay the
|
||||
download latency.
|
||||
We were on ``-M raspi3b`` previously but QEMU 10 + kernel 6.12 had a
|
||||
pl011 RX bug that broke userspace tty open — see
|
||||
``project/pi-emulation/decisions.md`` for the full debugging trail.
|
||||
The ``raspberry-pi-3`` manifest entry remains around (marked deprecated)
|
||||
to ease rollback; ``raspberry-pi-3-virt`` is the live one.
|
||||
|
||||
GPIO shim protocol (ttyAMA1)
|
||||
----------------------------
|
||||
Pi → backend : "GPIO <bcm_pin> <0|1>\\n"
|
||||
backend → Pi : "SET <bcm_pin> <0|1>\\n"
|
||||
Boot files are resolved at runtime via BootImageProvider (downloads,
|
||||
verifies, caches under /var/cache/velxio/boot-images/...). The lifespan
|
||||
hook at the bottom pre-warms the cache so first-time user requests
|
||||
don't pay the download latency.
|
||||
|
||||
Protocol channel (chardev 1) — wired by Phase 2's pi_protocol_mux
|
||||
Pi → backend : "GPIO <bcm> <0|1>\\n" (also I2C/SPI/UART/PWM lines)
|
||||
backend → Pi : "SET <bcm> <0|1>\\n" and reply frames
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
|
@ -43,13 +48,13 @@ from app.services.boot_images import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Image-set id (matches a key in `boot_images/manifest.json`).
|
||||
PI3_IMAGE_SET = 'raspberry-pi-3'
|
||||
PI3_IMAGE_SET = 'raspberry-pi-3-virt'
|
||||
|
||||
# Filenames the provider materialises (must match `name` fields in the
|
||||
# manifest entry for PI3_IMAGE_SET).
|
||||
PI3_KERNEL_NAME = 'kernel8.img'
|
||||
PI3_DTB_NAME = 'bcm2710-rpi-3-b.dtb'
|
||||
PI3_SD_NAME = 'raspios-trixie-armhf.img'
|
||||
PI3_KERNEL_NAME = 'velxio-kernel-arm64'
|
||||
PI3_INITRAMFS_NAME = 'velxio-initramfs-arm64.cpio.gz'
|
||||
PI3_ROOTFS_NAME = 'velxio-pi-rootfs-arm64.ext4'
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
|
|
@ -71,8 +76,8 @@ class PiInstance:
|
|||
# Runtime state
|
||||
self.process: subprocess.Popen | None = None
|
||||
self.overlay_path: str | None = None
|
||||
self.serial_port: int = 0 # ttyAMA0 TCP port
|
||||
self.gpio_port: int = 0 # ttyAMA1 TCP port
|
||||
self.serial_port: int = 0 # virtio-console TCP port (/dev/hvc0)
|
||||
self.gpio_port: int = 0 # virtio-serial protocol TCP port (/dev/vport0p2)
|
||||
self._serial_writer: asyncio.StreamWriter | None = None
|
||||
self._gpio_writer: asyncio.StreamWriter | None = None
|
||||
self._tasks: list[asyncio.Task] = []
|
||||
|
|
@ -124,7 +129,8 @@ class QemuManager:
|
|||
if not inst._serial_writer:
|
||||
logger.warning('send_serial_bytes: %s has no serial writer (qemu not connected yet?)', client_id)
|
||||
return
|
||||
logger.info('send_serial_bytes: %s sending %d bytes: %r', client_id, len(data), bytes(data[:32]))
|
||||
logger.info('send_serial_bytes: %s sending %d bytes: %r',
|
||||
client_id, len(data), bytes(data[:32]))
|
||||
inst._serial_writer.write(data)
|
||||
try:
|
||||
await inst._serial_writer.drain()
|
||||
|
|
@ -146,86 +152,97 @@ class QemuManager:
|
|||
})
|
||||
self._instances.pop(inst.client_id, None)
|
||||
return
|
||||
kernel_path: Path = images[PI3_KERNEL_NAME]
|
||||
dtb_path: Path = images[PI3_DTB_NAME]
|
||||
sd_base: Path = images[PI3_SD_NAME]
|
||||
kernel_path: Path = images[PI3_KERNEL_NAME]
|
||||
initramfs_path: Path = images[PI3_INITRAMFS_NAME]
|
||||
rootfs_base: Path = images[PI3_ROOTFS_NAME]
|
||||
|
||||
# Allocate TCP ports for the two serial channels
|
||||
# Allocate TCP ports for the two chardevs.
|
||||
inst.serial_port = _find_free_port()
|
||||
inst.gpio_port = _find_free_port()
|
||||
|
||||
# Create overlay qcow2 so the base SD image is never modified
|
||||
# Create overlay qcow2 backed by the velxio rootfs ext4. Each
|
||||
# session gets its own writable layer; reads cascade down to
|
||||
# the shared base, writes go into the per-session overlay
|
||||
# which is deleted on stop.
|
||||
overlay = tempfile.NamedTemporaryFile(suffix='.qcow2', delete=False)
|
||||
overlay.close()
|
||||
inst.overlay_path = overlay.name
|
||||
try:
|
||||
subprocess.run(
|
||||
['qemu-img', 'create', '-f', 'qcow2',
|
||||
'-b', str(sd_base), '-F', 'raw',
|
||||
'-b', str(rootfs_base), '-F', 'raw',
|
||||
inst.overlay_path],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
# raspi3b requires SD card size to be a power of 2; resize overlay to 8 GiB
|
||||
subprocess.run(
|
||||
['qemu-img', 'resize', inst.overlay_path, '8G'],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
await inst.emit('error', {'message': f'qemu-img create failed: {e.stderr.decode()}'})
|
||||
await inst.emit('error',
|
||||
{'message': f'qemu-img create failed: '
|
||||
f'{e.stderr.decode()}'})
|
||||
self._instances.pop(inst.client_id, None)
|
||||
return
|
||||
|
||||
# Build QEMU command
|
||||
# Build QEMU command for -M virt + virtio devices.
|
||||
#
|
||||
# Why virt: see project/pi-emulation/decisions.md (D1). Short
|
||||
# version: raspi3b pl011 RX is broken in QEMU 10 + kernel 6.12,
|
||||
# virtio-console isn't.
|
||||
#
|
||||
# Why no -dtb: virt machine generates its own DTB on the fly
|
||||
# from the runtime device list, so we don't ship one.
|
||||
cmd = [
|
||||
'qemu-system-aarch64',
|
||||
'-M', 'raspi3b',
|
||||
'-kernel', str(kernel_path),
|
||||
'-dtb', str(dtb_path),
|
||||
'-drive', f'file={inst.overlay_path},if=sd,format=qcow2',
|
||||
'-m', '1G',
|
||||
'-M', 'virt',
|
||||
'-cpu', 'cortex-a53',
|
||||
'-smp', '4',
|
||||
'-nographic',
|
||||
# ttyAMA0 → user serial (TCP server, frontend connects)
|
||||
'-serial', f'tcp:127.0.0.1:{inst.serial_port},server,nowait',
|
||||
# ttyAMA1 → GPIO shim protocol
|
||||
'-serial', f'tcp:127.0.0.1:{inst.gpio_port},server,nowait',
|
||||
'-append',
|
||||
# earlycon=pl011,mmio32,0x3f201000 — the BCM2837 PL011 UART
|
||||
# MMIO address. earlycon polled-mode is what actually carries
|
||||
# all of our serial output; QEMU's raspi3b emulation drives
|
||||
# the PL011 correctly in this mode but the kernel's normal
|
||||
# interrupt-based serial driver loses TX once systemd hands
|
||||
# off (confirmed by experiment — output silently stops after
|
||||
# ~9 s of kernel time without keep_bootcon).
|
||||
#
|
||||
# keep_bootcon — don't disable earlycon when the regular
|
||||
# console driver registers. Without this, the boot goes
|
||||
# silent at the same moment systemd takes /dev/console.
|
||||
#
|
||||
# console=ttyAMA1,115200 — the PL011 at 0x3f201000
|
||||
# enumerates as ttyAMA1 (not ttyAMA0!). The mini-UART at
|
||||
# 0x3f215040 takes ttyAMA0 and fails to probe under QEMU.
|
||||
# console= here drives userspace output once the kernel
|
||||
# hands off; it works in tandem with the kept-alive earlycon
|
||||
# for full kernel printk coverage.
|
||||
#
|
||||
# init=/usr/local/sbin/velxio-init — skip systemd entirely.
|
||||
# The init script (baked into the SD image by
|
||||
# scripts/configure-pi3-autologin.sh) mounts the pseudo-
|
||||
# filesystems systemd would, then loops a passwordless root
|
||||
# bash on ttyAMA1. User sees the prompt in ~10 s instead of
|
||||
# 2-3 min of Pi OS systemd graph churning inside emulation.
|
||||
'earlycon=pl011,mmio32,0x3f201000 keep_bootcon '
|
||||
'console=ttyAMA1,115200 '
|
||||
'root=/dev/mmcblk0p2 rootwait rw '
|
||||
'dwc_otg.lpm_enable=0 '
|
||||
'init=/usr/local/sbin/velxio-init',
|
||||
'-m', '1G',
|
||||
'-kernel', str(kernel_path),
|
||||
'-initrd', str(initramfs_path),
|
||||
# Root filesystem via virtio-blk over PCI. virt machine
|
||||
# uses PCI as the primary virtio transport, so we use
|
||||
# `virtio-blk-pci` (not `virtio-blk-device`, which is for
|
||||
# mmio and silently leaves /dev/vda unregistered).
|
||||
'-drive', f'if=none,file={inst.overlay_path},format=qcow2,id=rootfs',
|
||||
'-device', 'virtio-blk-pci,drive=rootfs',
|
||||
# No default network / display / monitor / serial — we add
|
||||
# exactly the two chardev-backed virtio-serial ports we
|
||||
# need. -nographic auto-binds -serial mon:stdio which
|
||||
# collides with our explicit -chardev IDs.
|
||||
'-nic', 'none',
|
||||
'-display', 'none',
|
||||
'-monitor', 'none',
|
||||
'-serial', 'none',
|
||||
# Console: TCP chardev → virtio-console → /dev/hvc0 inside
|
||||
# the guest. The frontend serial WebSocket connects to this
|
||||
# port (replaces the old ttyAMA0 path).
|
||||
'-chardev', f'socket,id=cons,host=127.0.0.1,port={inst.serial_port},'
|
||||
f'server=on,wait=off',
|
||||
'-device', 'virtio-serial-pci,id=virtio-serial0',
|
||||
'-device', 'virtconsole,chardev=cons',
|
||||
# Protocol channel: second virtserialport on the SAME
|
||||
# virtio-serial controller. Inside the guest this is
|
||||
# /dev/vport0p2. Phase 2 hooks this up to pi_protocol_mux
|
||||
# for GPIO/I2C/SPI/UART/PWM.
|
||||
'-chardev', f'socket,id=proto,host=127.0.0.1,port={inst.gpio_port},'
|
||||
f'server=on,wait=off',
|
||||
'-device', 'virtserialport,chardev=proto,name=velxio-protocol',
|
||||
# Kernel cmdline:
|
||||
# console=hvc0 — the virtio-console is the user terminal.
|
||||
# root=/dev/vda — the virtio-blk overlay is the rootfs.
|
||||
# rw — userspace can write (overlay catches writes).
|
||||
# quiet — suppress most printk so the user sees the shell
|
||||
# banner cleanly.
|
||||
# panic=10 — auto-reboot 10 s after a panic instead of
|
||||
# hanging forever (defensive against bad user
|
||||
# rootfs uploads in Phase 4).
|
||||
'-append', 'console=hvc0 root=/dev/vda rw quiet panic=10',
|
||||
]
|
||||
|
||||
logger.info('Launching QEMU for %s: %s', inst.client_id, ' '.join(cmd))
|
||||
logger.info('Launching QEMU for %s: %s',
|
||||
inst.client_id, ' '.join(cmd))
|
||||
|
||||
# Use subprocess.Popen via executor — asyncio.create_subprocess_exec requires
|
||||
# ProactorEventLoop on Windows but uvicorn may use SelectorEventLoop.
|
||||
# Use subprocess.Popen via executor — asyncio.create_subprocess_exec
|
||||
# requires ProactorEventLoop on Windows but uvicorn may use
|
||||
# SelectorEventLoop.
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
inst.process = await loop.run_in_executor(
|
||||
|
|
@ -238,7 +255,8 @@ class QemuManager:
|
|||
),
|
||||
)
|
||||
except FileNotFoundError:
|
||||
await inst.emit('error', {'message': 'qemu-system-aarch64 not found in PATH'})
|
||||
await inst.emit('error',
|
||||
{'message': 'qemu-system-aarch64 not found in PATH'})
|
||||
self._instances.pop(inst.client_id, None)
|
||||
return
|
||||
|
||||
|
|
@ -246,29 +264,34 @@ class QemuManager:
|
|||
await inst.emit('system', {'event': 'booting'})
|
||||
|
||||
# Give QEMU a moment to open its TCP sockets
|
||||
await asyncio.sleep(2.0)
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
# Connect to serial TCP ports
|
||||
# Connect to the two chardev TCP ports.
|
||||
inst._tasks.append(asyncio.create_task(self._connect_serial(inst)))
|
||||
inst._tasks.append(asyncio.create_task(self._connect_gpio(inst)))
|
||||
inst._tasks.append(asyncio.create_task(self._watch_stderr(inst)))
|
||||
|
||||
# ── Serial (ttyAMA0) ──────────────────────────────────────────────────────
|
||||
# ── Console (virtio-console / /dev/hvc0) ──────────────────────────────────
|
||||
|
||||
async def _connect_serial(self, inst: PiInstance) -> None:
|
||||
for attempt in range(10):
|
||||
try:
|
||||
reader, writer = await asyncio.open_connection('127.0.0.1', inst.serial_port)
|
||||
reader, writer = await asyncio.open_connection(
|
||||
'127.0.0.1', inst.serial_port,
|
||||
)
|
||||
inst._serial_writer = writer
|
||||
logger.info('%s: serial connected on port %d', inst.client_id, inst.serial_port)
|
||||
logger.info('%s: serial connected on port %d',
|
||||
inst.client_id, inst.serial_port)
|
||||
await inst.emit('system', {'event': 'booted'})
|
||||
await self._read_serial(inst, reader)
|
||||
return
|
||||
except (ConnectionRefusedError, OSError):
|
||||
await asyncio.sleep(1.0 * (attempt + 1))
|
||||
await inst.emit('error', {'message': 'Could not connect to QEMU serial port'})
|
||||
await inst.emit('error',
|
||||
{'message': 'Could not connect to QEMU console port'})
|
||||
|
||||
async def _read_serial(self, inst: PiInstance, reader: asyncio.StreamReader) -> None:
|
||||
async def _read_serial(self, inst: PiInstance,
|
||||
reader: asyncio.StreamReader) -> None:
|
||||
buf = bytearray()
|
||||
while inst.running:
|
||||
try:
|
||||
|
|
@ -285,22 +308,36 @@ class QemuManager:
|
|||
logger.warning('%s serial read: %s', inst.client_id, e)
|
||||
break
|
||||
|
||||
# ── GPIO shim (ttyAMA1) ───────────────────────────────────────────────────
|
||||
# ── Protocol channel (virtio-serial / /dev/vport0p2) ──────────────────────
|
||||
# Methods keep the historical "_gpio" naming so the rest of the
|
||||
# codebase (simulation route, GPIO event bus) doesn't have to
|
||||
# rename. Phase 2 swaps the line parser for the full multi-protocol
|
||||
# mux while keeping this connect/read/write plumbing identical.
|
||||
|
||||
async def _connect_gpio(self, inst: PiInstance) -> None:
|
||||
for attempt in range(10):
|
||||
try:
|
||||
reader, writer = await asyncio.open_connection('127.0.0.1', inst.gpio_port)
|
||||
reader, writer = await asyncio.open_connection(
|
||||
'127.0.0.1', inst.gpio_port,
|
||||
)
|
||||
inst._gpio_writer = writer
|
||||
logger.info('%s: GPIO shim connected on port %d', inst.client_id, inst.gpio_port)
|
||||
logger.info('%s: protocol channel connected on port %d',
|
||||
inst.client_id, inst.gpio_port)
|
||||
await self._read_gpio(inst, reader)
|
||||
return
|
||||
except (ConnectionRefusedError, OSError):
|
||||
await asyncio.sleep(1.0 * (attempt + 1))
|
||||
logger.warning('%s: GPIO shim connection failed', inst.client_id)
|
||||
logger.warning('%s: protocol channel connection failed',
|
||||
inst.client_id)
|
||||
|
||||
async def _read_gpio(self, inst: PiInstance, reader: asyncio.StreamReader) -> None:
|
||||
"""Parse "GPIO <pin> <state>\n" lines from the Pi GPIO shim."""
|
||||
async def _read_gpio(self, inst: PiInstance,
|
||||
reader: asyncio.StreamReader) -> None:
|
||||
"""Parse text-protocol lines from the Pi shim layer.
|
||||
|
||||
Phase 1 understands GPIO only (existing protocol). Phase 2
|
||||
extends this to dispatch I2C/SPI/UART/PWM as well via
|
||||
pi_protocol_mux.
|
||||
"""
|
||||
linebuf = b''
|
||||
while inst.running:
|
||||
try:
|
||||
|
|
@ -310,11 +347,13 @@ class QemuManager:
|
|||
linebuf += chunk
|
||||
while b'\n' in linebuf:
|
||||
line, linebuf = linebuf.split(b'\n', 1)
|
||||
await self._handle_gpio_line(inst, line.decode('ascii', 'ignore').strip())
|
||||
await self._handle_gpio_line(
|
||||
inst, line.decode('ascii', 'ignore').strip(),
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.warning('%s GPIO read: %s', inst.client_id, e)
|
||||
logger.warning('%s protocol read: %s', inst.client_id, e)
|
||||
break
|
||||
|
||||
async def _handle_gpio_line(self, inst: PiInstance, line: str) -> None:
|
||||
|
|
@ -335,7 +374,7 @@ class QemuManager:
|
|||
try:
|
||||
await inst._gpio_writer.drain()
|
||||
except Exception as e:
|
||||
logger.warning('%s GPIO send: %s', inst.client_id, e)
|
||||
logger.warning('%s protocol send: %s', inst.client_id, e)
|
||||
|
||||
# ── QEMU stderr watcher ───────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -345,7 +384,6 @@ class QemuManager:
|
|||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
while inst.running:
|
||||
# readline() blocks until a line or EOF — run in executor so we don't block the loop
|
||||
line = await loop.run_in_executor(None, inst.process.stderr.readline)
|
||||
if not line:
|
||||
break
|
||||
|
|
@ -423,7 +461,7 @@ class QemuManager:
|
|||
|
||||
# ── Lifespan pre-warm ────────────────────────────────────────────────────────
|
||||
async def _prewarm_pi3_boot_images() -> None:
|
||||
"""Lifespan hook: download + cache the Pi 3 boot files in the
|
||||
"""Lifespan hook: download + cache the Pi 3 virt boot files in the
|
||||
background at process start.
|
||||
|
||||
The cache check is cheap when files are already on disk (named
|
||||
|
|
@ -441,9 +479,6 @@ async def _prewarm_pi3_boot_images() -> None:
|
|||
exc,
|
||||
)
|
||||
return
|
||||
# Fire-and-forget — let it complete in the background while the
|
||||
# rest of the lifespan continues. provider.warmup() swallows its
|
||||
# own errors, so the task never logs unhandled exceptions.
|
||||
asyncio.create_task(provider.warmup(PI3_IMAGE_SET))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,263 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Pi 3 console boot validation
|
||||
============================
|
||||
|
||||
Spins up QEMU raspi3b with the SAME args the production qemu_manager
|
||||
uses (two ``-serial`` chardevs, ``init=/usr/local/sbin/velxio-init``,
|
||||
``console=ttyAMA1,115200 keep_bootcon earlycon=pl011,mmio32,0x3f201000``)
|
||||
against the cached SD image. Connects to the user-serial TCP socket
|
||||
and asserts:
|
||||
|
||||
1. The kernel boots and reaches ``Run /usr/local/sbin/velxio-init as
|
||||
init process``.
|
||||
2. The init script emits a ``[velxio-init] selected TTY=...`` diag
|
||||
line (i.e. doesn't hit the FATAL path).
|
||||
3. The init script does NOT spam ``No such file or directory`` /
|
||||
``No such device or address`` errors (which mean the redirect to
|
||||
the chosen TTY is failing).
|
||||
4. The Velxio bashrc banner appears (``Velxio Raspberry Pi 3 —
|
||||
interactive root shell``).
|
||||
5. After sending ``echo VELXIO_OK_$$\\n`` to the console, the same
|
||||
token comes back (proves the shell is actually reading input).
|
||||
|
||||
Fidelity rule (memory ``feedback_tests_import_real_code``): the
|
||||
QEMU launch args mirror what ``app.services.qemu_manager`` emits,
|
||||
so any regression in the production manager that touches the
|
||||
console wiring fails this test too. The SD image read is the
|
||||
exact image cached by BootImageProvider at runtime.
|
||||
|
||||
Run:
|
||||
cd /home/dave/velxio-prod
|
||||
python3 velxio/test/pi3_console_boot/test_pi3_console_boot.py
|
||||
|
||||
The test takes ~45 s (boot is the slow part). Exit code 0 = OK,
|
||||
non-zero = failure with the offending log excerpt printed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
BOOT_IMAGES = Path("/var/cache/velxio/boot-images/raspberry-pi-3-virt")
|
||||
KERNEL = BOOT_IMAGES / "velxio-kernel-arm64"
|
||||
INITRAMFS = BOOT_IMAGES / "velxio-initramfs-arm64.cpio.gz"
|
||||
ROOTFS = BOOT_IMAGES / "velxio-pi-rootfs-arm64.ext4"
|
||||
|
||||
# Boot timeout: cold boot is ~10-15 s on virt + minimal rootfs (much
|
||||
# faster than the old raspi3b + raspios path that took 25-30 s).
|
||||
BOOT_TIMEOUT_S = 45
|
||||
|
||||
# Markers we expect to see in the console output during a healthy
|
||||
# boot. The order is significant — each MUST appear before the
|
||||
# next in time. Matches what Alpine + agetty + autologin emit:
|
||||
# - "OpenRC 0.x" — openrc init started
|
||||
# - "Welcome to the Velxio Pi Simulator" — /etc/motd printed by login
|
||||
# - "raspberrypi:" — agetty's autologin reached the shell prompt
|
||||
BOOT_MARKERS = [
|
||||
b"Velxio Pi Simulator",
|
||||
b"login on 'hvc0'",
|
||||
]
|
||||
|
||||
# Any of these substrings in the output means the init or shell is
|
||||
# broken — fail fast rather than waiting for the prompt timeout.
|
||||
BOOT_NEGATIVE_MARKERS = [
|
||||
b"FATAL",
|
||||
b"Kernel panic",
|
||||
b"unable to mount root fs",
|
||||
b"Cannot open root device",
|
||||
]
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _make_qcow2_overlay() -> str:
|
||||
"""Mirror of ``qemu_manager`` — qcow2 over the cached rootfs ext4
|
||||
so the test never mutates the base."""
|
||||
overlay = tempfile.NamedTemporaryFile(suffix=".qcow2", delete=False)
|
||||
overlay.close()
|
||||
subprocess.run(
|
||||
["qemu-img", "create", "-f", "qcow2",
|
||||
"-b", str(ROOTFS), "-F", "raw", overlay.name],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
return overlay.name
|
||||
|
||||
|
||||
def _qemu_argv(overlay_path: str, serial_port: int, gpio_port: int) -> list[str]:
|
||||
"""Mirror of ``qemu_manager._boot`` — -M virt + virtio-{blk,serial}
|
||||
over PCI + velxio kernel/initramfs/rootfs."""
|
||||
return [
|
||||
"qemu-system-aarch64",
|
||||
"-M", "virt",
|
||||
"-cpu", "cortex-a53",
|
||||
"-smp", "4",
|
||||
"-m", "1G",
|
||||
"-kernel", str(KERNEL),
|
||||
"-initrd", str(INITRAMFS),
|
||||
"-drive", f"if=none,file={overlay_path},format=qcow2,id=rootfs",
|
||||
"-device", "virtio-blk-pci,drive=rootfs",
|
||||
"-nic", "none",
|
||||
"-display", "none",
|
||||
"-monitor", "none",
|
||||
"-serial", "none",
|
||||
"-chardev", f"socket,id=cons,host=127.0.0.1,port={serial_port},"
|
||||
f"server=on,wait=off",
|
||||
"-device", "virtio-serial-pci,id=virtio-serial0",
|
||||
"-device", "virtconsole,chardev=cons",
|
||||
"-chardev", f"socket,id=proto,host=127.0.0.1,port={gpio_port},"
|
||||
f"server=on,wait=off",
|
||||
"-device", "virtserialport,chardev=proto,name=velxio-protocol",
|
||||
"-append", "console=hvc0 root=/dev/vda rw panic=10",
|
||||
]
|
||||
|
||||
|
||||
def _connect_with_retry(host: str, port: int, deadline: float) -> socket.socket:
|
||||
"""Poll-connect to QEMU's TCP serial server (it accepts after
|
||||
QEMU has fully started)."""
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
s = socket.create_connection((host, port), timeout=2)
|
||||
s.settimeout(2)
|
||||
return s
|
||||
except (ConnectionRefusedError, OSError):
|
||||
time.sleep(0.2)
|
||||
raise TimeoutError(f"Could not connect to {host}:{port} within deadline")
|
||||
|
||||
|
||||
def _drain_until(sock: socket.socket, marker: bytes, deadline: float,
|
||||
negatives: list[bytes]) -> bytes:
|
||||
"""Read from socket until ``marker`` appears or we hit a negative
|
||||
marker / deadline. Returns the accumulated buffer for inspection."""
|
||||
buf = bytearray()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
chunk = sock.recv(4096)
|
||||
except socket.timeout:
|
||||
continue
|
||||
if not chunk:
|
||||
break
|
||||
buf.extend(chunk)
|
||||
for neg in negatives:
|
||||
if neg in buf:
|
||||
return bytes(buf)
|
||||
if marker in buf:
|
||||
return bytes(buf)
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def run() -> int:
|
||||
# ── sanity: required files exist ───────────────────────────────
|
||||
for p in (KERNEL, INITRAMFS, ROOTFS):
|
||||
if not p.exists():
|
||||
print(f"FAIL: missing boot image: {p}", file=sys.stderr)
|
||||
return 2
|
||||
if not subprocess.run(
|
||||
["which", "qemu-system-aarch64"], capture_output=True
|
||||
).returncode == 0:
|
||||
print("FAIL: qemu-system-aarch64 not in PATH", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
# ── start QEMU ─────────────────────────────────────────────────
|
||||
overlay = _make_qcow2_overlay()
|
||||
serial_port = _find_free_port()
|
||||
gpio_port = _find_free_port()
|
||||
argv = _qemu_argv(overlay, serial_port, gpio_port)
|
||||
print(f"[test] QEMU: {' '.join(argv)}")
|
||||
qemu = subprocess.Popen(
|
||||
argv,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
deadline = time.monotonic() + BOOT_TIMEOUT_S
|
||||
|
||||
try:
|
||||
# Connect to the user-serial channel.
|
||||
sock = _connect_with_retry("127.0.0.1", serial_port, deadline)
|
||||
print(f"[test] connected to user serial @ :{serial_port}")
|
||||
|
||||
# Walk through each expected marker in order. Any negative
|
||||
# marker short-circuits the test.
|
||||
all_output = bytearray()
|
||||
for marker in BOOT_MARKERS:
|
||||
print(f"[test] waiting for marker: {marker!r}")
|
||||
chunk = _drain_until(sock, marker, deadline, BOOT_NEGATIVE_MARKERS)
|
||||
all_output.extend(chunk)
|
||||
for neg in BOOT_NEGATIVE_MARKERS:
|
||||
if neg in chunk:
|
||||
print(f"FAIL: negative marker hit: {neg!r}", file=sys.stderr)
|
||||
_dump_tail(bytes(all_output), 4000)
|
||||
return 1
|
||||
if marker not in chunk:
|
||||
print(f"FAIL: timeout waiting for marker {marker!r}",
|
||||
file=sys.stderr)
|
||||
_dump_tail(bytes(all_output), 4000)
|
||||
return 1
|
||||
print(f"[test] ✓ found")
|
||||
|
||||
# ── round-trip: send a token, expect to see it echoed back ─
|
||||
token = f"VELXIO_OK_{os.getpid()}".encode()
|
||||
# The shell echoes input back (canonical TTY mode), so sending
|
||||
# `echo TOKEN` followed by newline should produce `echo TOKEN`
|
||||
# (typed) followed by `TOKEN` (command output) on the wire.
|
||||
# We send a small delay to let the prompt settle.
|
||||
time.sleep(2)
|
||||
cmd = b"echo " + token + b"\n"
|
||||
print(f"[test] sending: {cmd!r}")
|
||||
sock.sendall(cmd)
|
||||
echoback = _drain_until(sock, token, deadline,
|
||||
BOOT_NEGATIVE_MARKERS)
|
||||
all_output.extend(echoback)
|
||||
if token not in echoback:
|
||||
print(f"FAIL: token {token!r} never came back from the shell — "
|
||||
"input is not flowing or shell is dead",
|
||||
file=sys.stderr)
|
||||
_dump_tail(bytes(all_output), 4000)
|
||||
return 1
|
||||
# The token will appear at least twice (echo of typed bytes +
|
||||
# output of the echo command). One occurrence is suspicious
|
||||
# (could be just the echo of our send).
|
||||
if echoback.count(token) < 2:
|
||||
print(f"WARN: token {token!r} appears only "
|
||||
f"{echoback.count(token)}x — expected ≥2 "
|
||||
"(typed echo + command output). Shell may be in "
|
||||
"raw mode or the redirect is one-way.")
|
||||
# Don't fail — some configs disable echo. The presence
|
||||
# alone proves bidirectional flow.
|
||||
|
||||
print("[test] ✓ Pi 3 console is interactive")
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
qemu.terminate()
|
||||
qemu.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
qemu.kill()
|
||||
try:
|
||||
os.unlink(overlay)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _dump_tail(buf: bytes, n: int) -> None:
|
||||
tail = buf[-n:].decode("utf-8", errors="replace")
|
||||
print("─" * 70, file=sys.stderr)
|
||||
print("Last bytes of console output:", file=sys.stderr)
|
||||
print(tail, file=sys.stderr)
|
||||
print("─" * 70, file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
Loading…
Reference in New Issue