From a7472e3411a962639b4f3029a3a7d6c5277e9f9a Mon Sep 17 00:00:00 2001 From: davidmonterocrespo24 Date: Mon, 18 May 2026 05:36:02 +0200 Subject: [PATCH] feat(pi3): switch from raspi3b to virt + virtio (Phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit raspi3b pl011 RX is broken in QEMU 10 + kernel 6.12 — see project/pi-emulation/decisions.md for the full debugging trail. This commit lands Phase 1 of the rebuild: switch the QEMU machine to virt + cortex-a53, boot the velxio kernel/initramfs/rootfs over virtio-blk-pci, and expose the user shell on /dev/hvc0 via virtio-serial-pci + virtconsole. End-to-end smoke verified: boot → agetty autologin → bash prompt → echo round-trip returns the typed token. Tested inside the prod container with QEMU 10.0.8 and our cloud-derived kernel 6.12.88. What changed: backend/app/services/qemu_manager.py PI3_IMAGE_SET -> raspberry-pi-3-virt PI3_KERNEL_NAME / PI3_INITRAMFS_NAME / PI3_ROOTFS_NAME new QEMU cmd rewritten end-to-end: -M virt -cpu cortex-a53 -smp 4 -m 1G -kernel -initrd -drive ... -device virtio-blk-pci (NOT virtio-blk-device — mmio variant left /dev/vda unregistered) -nic none -display none -monitor none -serial none -chardev socket... -device virtio-serial-pci -device virtconsole (user console -> /dev/hvc0) -chardev socket... -device virtserialport,name=velxio-protocol (Phase 2 channel -> /dev/vport0p2) No -dtb (virt generates its own), no -append init=... (kernel runs our initramfs which then switch_root to rootfs and exec's its /sbin/init — Alpine OpenRC). backend/app/services/boot_images/manifest.json New image set raspberry-pi-3-virt with three assets uploaded via the existing license-endpoint pipeline. Old raspberry-pi-3 entry flagged deprecated:true and kept for one release for rollback. test/pi3_console_boot/test_pi3_console_boot.py Updated QEMU argv to match qemu_manager exactly. Markers now look for the Velxio Pi Simulator MOTD + 'login on hvc0' (autologin proof). Round-trip echo still required to pass. --- .../app/services/boot_images/manifest.json | 34 ++- backend/app/services/qemu_manager.py | 241 +++++++++------- .../pi3_console_boot/test_pi3_console_boot.py | 263 ++++++++++++++++++ 3 files changed, 434 insertions(+), 104 deletions(-) create mode 100644 test/pi3_console_boot/test_pi3_console_boot.py diff --git a/backend/app/services/boot_images/manifest.json b/backend/app/services/boot_images/manifest.json index 29a15e4d..340219dd 100644 --- a/backend/app/services/boot_images/manifest.json +++ b/backend/app/services/boot_images/manifest.json @@ -2,8 +2,40 @@ "_comment": "Source of truth for QEMU boot files. Adding a new board kind means appending an entry under image_sets; the BootImageProvider picks it up via load_manifest() with no Python edits. SHA256s are verified after download; mismatches raise IntegrityError and the cache slot is not populated. See docs/BOOT_IMAGES.md for the architecture.", "version": 1, "image_sets": { + "raspberry-pi-3-virt": { + "description": "Raspberry Pi 3 simulator on QEMU -M virt + cortex-a53. Velxio-built minimal ARM64 kernel + initramfs + Alpine-based rootfs. Replaces the broken raspi3b path (pl011 RX bug in QEMU 10 + kernel 6.12 — see project/pi-emulation/decisions.md for the full trail). Used by app.services.qemu_manager.", + "images": [ + { + "name": "velxio-kernel-arm64", + "asset_id": "velxio-kernel-arm64", + "sha256": "76652d46bee27fe46f92c21adf9e67964356774ff4f6abe4f009344aacd3acd9", + "size_bytes": 30697408, + "version": "2026.05+v6.12.88-deb13-cloud" + }, + { + "name": "velxio-initramfs-arm64.cpio.gz", + "asset_id": "velxio-initramfs-arm64", + "sha256": "9a1ecd1c83a65207562b8395fcc57cc9218f16cf240dd4b5cf84e13ef0b34efe", + "size_bytes": 1032327, + "version": "2026.05" + }, + { + "name": "velxio-pi-rootfs-arm64.ext4", + "asset_id": "velxio-pi-rootfs-arm64-zst", + "sha256": "d09eec3bbc023dc4798d70de87048b25d2db81e5530fe0fbe0853f745bdaf910", + "size_bytes": 629145600, + "version": "2026.05", + "compressed": { + "encoding": "zstd", + "sha256": "de55525e8475eea17fe7f4c26491154a998f7379ed82a651f0d10550cb9c997f", + "size_bytes": 24725464 + } + } + ] + }, "raspberry-pi-3": { - "description": "Raspberry Pi 3 Model B boot files — kernel, device tree, and Raspberry Pi OS Trixie (armhf) SD image. Targets the QEMU 'raspi3b' machine type. The SD image ships compressed (zstd -19) to cut the licence-endpoint download from ~5.3 GiB to ~1.4 GiB; the provider decompresses on the host into the named volume cache.", + "description": "[DEPRECATED — kept for one release to ease rollback] Original raspi3b setup. Replaced by 'raspberry-pi-3-virt' (see above). qemu_manager.py no longer references this entry; do not add new clients here.", + "deprecated": true, "images": [ { "name": "kernel8.img", diff --git a/backend/app/services/qemu_manager.py b/backend/app/services/qemu_manager.py index 9b9f7e00..112bcef0 100644 --- a/backend/app/services/qemu_manager.py +++ b/backend/app/services/qemu_manager.py @@ -1,27 +1,32 @@ """ -QemuManager — backend service for Raspberry Pi 3B emulation via QEMU. +QemuManager — backend service for Raspberry Pi simulator emulation via QEMU. Architecture ------------ Each Pi board instance gets: - - qemu-system-aarch64 process (raspi3b, ARM64) - - ttyAMA0 (serial0) → TCP socket on a dynamic port → user serial I/O - - ttyAMA1 (serial1) → TCP socket on a dynamic port → GPIO shim protocol - - A fresh qcow2 overlay over the base SD image (copy-on-write, discarded on stop) + - qemu-system-aarch64 process (-M virt -cpu cortex-a53 for Pi 3, other + cpu models for the rest of the family — see Phase 3 plan) + - Velxio-built kernel + initramfs + rootfs (not the rpi-firmware kernel + and not raspios; see ``project/pi-emulation/`` for why) + - virtio-blk root from a qcow2 overlay over the cached rootfs ext4 + - virtio-console on chardev 0 (TCP socket) — the user shell at /dev/hvc0 + - virtio-serial port on chardev 1 (TCP socket) — multiplexed text + protocol channel for GPIO/I2C/SPI/UART/PWM (Phase 2 wires it up) -Boot files (kernel8.img, device tree, Raspberry Pi OS SD image) are NOT -shipped in the repo or baked into the Docker image. They're resolved at -runtime via :class:`app.services.boot_images.BootImageProvider`, which -downloads + SHA256-verifies + decompresses them on first use and caches -them under ``/var/cache/velxio/boot-images/raspberry-pi-3/``. The -lifespan hook at the bottom of this module pre-warms the cache at -process startup so a first-time user request doesn't pay the -download latency. +We were on ``-M raspi3b`` previously but QEMU 10 + kernel 6.12 had a +pl011 RX bug that broke userspace tty open — see +``project/pi-emulation/decisions.md`` for the full debugging trail. +The ``raspberry-pi-3`` manifest entry remains around (marked deprecated) +to ease rollback; ``raspberry-pi-3-virt`` is the live one. -GPIO shim protocol (ttyAMA1) ----------------------------- - Pi → backend : "GPIO <0|1>\\n" - backend → Pi : "SET <0|1>\\n" +Boot files are resolved at runtime via BootImageProvider (downloads, +verifies, caches under /var/cache/velxio/boot-images/...). The lifespan +hook at the bottom pre-warms the cache so first-time user requests +don't pay the download latency. + +Protocol channel (chardev 1) — wired by Phase 2's pi_protocol_mux + Pi → backend : "GPIO <0|1>\\n" (also I2C/SPI/UART/PWM lines) + backend → Pi : "SET <0|1>\\n" and reply frames """ import asyncio @@ -43,13 +48,13 @@ from app.services.boot_images import ( logger = logging.getLogger(__name__) # Image-set id (matches a key in `boot_images/manifest.json`). -PI3_IMAGE_SET = 'raspberry-pi-3' +PI3_IMAGE_SET = 'raspberry-pi-3-virt' # Filenames the provider materialises (must match `name` fields in the # manifest entry for PI3_IMAGE_SET). -PI3_KERNEL_NAME = 'kernel8.img' -PI3_DTB_NAME = 'bcm2710-rpi-3-b.dtb' -PI3_SD_NAME = 'raspios-trixie-armhf.img' +PI3_KERNEL_NAME = 'velxio-kernel-arm64' +PI3_INITRAMFS_NAME = 'velxio-initramfs-arm64.cpio.gz' +PI3_ROOTFS_NAME = 'velxio-pi-rootfs-arm64.ext4' def _find_free_port() -> int: @@ -71,8 +76,8 @@ class PiInstance: # Runtime state self.process: subprocess.Popen | None = None self.overlay_path: str | None = None - self.serial_port: int = 0 # ttyAMA0 TCP port - self.gpio_port: int = 0 # ttyAMA1 TCP port + self.serial_port: int = 0 # virtio-console TCP port (/dev/hvc0) + self.gpio_port: int = 0 # virtio-serial protocol TCP port (/dev/vport0p2) self._serial_writer: asyncio.StreamWriter | None = None self._gpio_writer: asyncio.StreamWriter | None = None self._tasks: list[asyncio.Task] = [] @@ -124,7 +129,8 @@ class QemuManager: if not inst._serial_writer: logger.warning('send_serial_bytes: %s has no serial writer (qemu not connected yet?)', client_id) return - logger.info('send_serial_bytes: %s sending %d bytes: %r', client_id, len(data), bytes(data[:32])) + logger.info('send_serial_bytes: %s sending %d bytes: %r', + client_id, len(data), bytes(data[:32])) inst._serial_writer.write(data) try: await inst._serial_writer.drain() @@ -146,86 +152,97 @@ class QemuManager: }) self._instances.pop(inst.client_id, None) return - kernel_path: Path = images[PI3_KERNEL_NAME] - dtb_path: Path = images[PI3_DTB_NAME] - sd_base: Path = images[PI3_SD_NAME] + kernel_path: Path = images[PI3_KERNEL_NAME] + initramfs_path: Path = images[PI3_INITRAMFS_NAME] + rootfs_base: Path = images[PI3_ROOTFS_NAME] - # Allocate TCP ports for the two serial channels + # Allocate TCP ports for the two chardevs. inst.serial_port = _find_free_port() inst.gpio_port = _find_free_port() - # Create overlay qcow2 so the base SD image is never modified + # Create overlay qcow2 backed by the velxio rootfs ext4. Each + # session gets its own writable layer; reads cascade down to + # the shared base, writes go into the per-session overlay + # which is deleted on stop. overlay = tempfile.NamedTemporaryFile(suffix='.qcow2', delete=False) overlay.close() inst.overlay_path = overlay.name try: subprocess.run( ['qemu-img', 'create', '-f', 'qcow2', - '-b', str(sd_base), '-F', 'raw', + '-b', str(rootfs_base), '-F', 'raw', inst.overlay_path], check=True, capture_output=True, ) - # raspi3b requires SD card size to be a power of 2; resize overlay to 8 GiB - subprocess.run( - ['qemu-img', 'resize', inst.overlay_path, '8G'], - check=True, capture_output=True, - ) except subprocess.CalledProcessError as e: - await inst.emit('error', {'message': f'qemu-img create failed: {e.stderr.decode()}'}) + await inst.emit('error', + {'message': f'qemu-img create failed: ' + f'{e.stderr.decode()}'}) self._instances.pop(inst.client_id, None) return - # Build QEMU command + # Build QEMU command for -M virt + virtio devices. + # + # Why virt: see project/pi-emulation/decisions.md (D1). Short + # version: raspi3b pl011 RX is broken in QEMU 10 + kernel 6.12, + # virtio-console isn't. + # + # Why no -dtb: virt machine generates its own DTB on the fly + # from the runtime device list, so we don't ship one. cmd = [ 'qemu-system-aarch64', - '-M', 'raspi3b', - '-kernel', str(kernel_path), - '-dtb', str(dtb_path), - '-drive', f'file={inst.overlay_path},if=sd,format=qcow2', - '-m', '1G', + '-M', 'virt', + '-cpu', 'cortex-a53', '-smp', '4', - '-nographic', - # ttyAMA0 → user serial (TCP server, frontend connects) - '-serial', f'tcp:127.0.0.1:{inst.serial_port},server,nowait', - # ttyAMA1 → GPIO shim protocol - '-serial', f'tcp:127.0.0.1:{inst.gpio_port},server,nowait', - '-append', - # earlycon=pl011,mmio32,0x3f201000 — the BCM2837 PL011 UART - # MMIO address. earlycon polled-mode is what actually carries - # all of our serial output; QEMU's raspi3b emulation drives - # the PL011 correctly in this mode but the kernel's normal - # interrupt-based serial driver loses TX once systemd hands - # off (confirmed by experiment — output silently stops after - # ~9 s of kernel time without keep_bootcon). - # - # keep_bootcon — don't disable earlycon when the regular - # console driver registers. Without this, the boot goes - # silent at the same moment systemd takes /dev/console. - # - # console=ttyAMA1,115200 — the PL011 at 0x3f201000 - # enumerates as ttyAMA1 (not ttyAMA0!). The mini-UART at - # 0x3f215040 takes ttyAMA0 and fails to probe under QEMU. - # console= here drives userspace output once the kernel - # hands off; it works in tandem with the kept-alive earlycon - # for full kernel printk coverage. - # - # init=/usr/local/sbin/velxio-init — skip systemd entirely. - # The init script (baked into the SD image by - # scripts/configure-pi3-autologin.sh) mounts the pseudo- - # filesystems systemd would, then loops a passwordless root - # bash on ttyAMA1. User sees the prompt in ~10 s instead of - # 2-3 min of Pi OS systemd graph churning inside emulation. - 'earlycon=pl011,mmio32,0x3f201000 keep_bootcon ' - 'console=ttyAMA1,115200 ' - 'root=/dev/mmcblk0p2 rootwait rw ' - 'dwc_otg.lpm_enable=0 ' - 'init=/usr/local/sbin/velxio-init', + '-m', '1G', + '-kernel', str(kernel_path), + '-initrd', str(initramfs_path), + # Root filesystem via virtio-blk over PCI. virt machine + # uses PCI as the primary virtio transport, so we use + # `virtio-blk-pci` (not `virtio-blk-device`, which is for + # mmio and silently leaves /dev/vda unregistered). + '-drive', f'if=none,file={inst.overlay_path},format=qcow2,id=rootfs', + '-device', 'virtio-blk-pci,drive=rootfs', + # No default network / display / monitor / serial — we add + # exactly the two chardev-backed virtio-serial ports we + # need. -nographic auto-binds -serial mon:stdio which + # collides with our explicit -chardev IDs. + '-nic', 'none', + '-display', 'none', + '-monitor', 'none', + '-serial', 'none', + # Console: TCP chardev → virtio-console → /dev/hvc0 inside + # the guest. The frontend serial WebSocket connects to this + # port (replaces the old ttyAMA0 path). + '-chardev', f'socket,id=cons,host=127.0.0.1,port={inst.serial_port},' + f'server=on,wait=off', + '-device', 'virtio-serial-pci,id=virtio-serial0', + '-device', 'virtconsole,chardev=cons', + # Protocol channel: second virtserialport on the SAME + # virtio-serial controller. Inside the guest this is + # /dev/vport0p2. Phase 2 hooks this up to pi_protocol_mux + # for GPIO/I2C/SPI/UART/PWM. + '-chardev', f'socket,id=proto,host=127.0.0.1,port={inst.gpio_port},' + f'server=on,wait=off', + '-device', 'virtserialport,chardev=proto,name=velxio-protocol', + # Kernel cmdline: + # console=hvc0 — the virtio-console is the user terminal. + # root=/dev/vda — the virtio-blk overlay is the rootfs. + # rw — userspace can write (overlay catches writes). + # quiet — suppress most printk so the user sees the shell + # banner cleanly. + # panic=10 — auto-reboot 10 s after a panic instead of + # hanging forever (defensive against bad user + # rootfs uploads in Phase 4). + '-append', 'console=hvc0 root=/dev/vda rw quiet panic=10', ] - logger.info('Launching QEMU for %s: %s', inst.client_id, ' '.join(cmd)) + logger.info('Launching QEMU for %s: %s', + inst.client_id, ' '.join(cmd)) - # Use subprocess.Popen via executor — asyncio.create_subprocess_exec requires - # ProactorEventLoop on Windows but uvicorn may use SelectorEventLoop. + # Use subprocess.Popen via executor — asyncio.create_subprocess_exec + # requires ProactorEventLoop on Windows but uvicorn may use + # SelectorEventLoop. loop = asyncio.get_running_loop() try: inst.process = await loop.run_in_executor( @@ -238,7 +255,8 @@ class QemuManager: ), ) except FileNotFoundError: - await inst.emit('error', {'message': 'qemu-system-aarch64 not found in PATH'}) + await inst.emit('error', + {'message': 'qemu-system-aarch64 not found in PATH'}) self._instances.pop(inst.client_id, None) return @@ -246,29 +264,34 @@ class QemuManager: await inst.emit('system', {'event': 'booting'}) # Give QEMU a moment to open its TCP sockets - await asyncio.sleep(2.0) + await asyncio.sleep(1.0) - # Connect to serial TCP ports + # Connect to the two chardev TCP ports. inst._tasks.append(asyncio.create_task(self._connect_serial(inst))) inst._tasks.append(asyncio.create_task(self._connect_gpio(inst))) inst._tasks.append(asyncio.create_task(self._watch_stderr(inst))) - # ── Serial (ttyAMA0) ────────────────────────────────────────────────────── + # ── Console (virtio-console / /dev/hvc0) ────────────────────────────────── async def _connect_serial(self, inst: PiInstance) -> None: for attempt in range(10): try: - reader, writer = await asyncio.open_connection('127.0.0.1', inst.serial_port) + reader, writer = await asyncio.open_connection( + '127.0.0.1', inst.serial_port, + ) inst._serial_writer = writer - logger.info('%s: serial connected on port %d', inst.client_id, inst.serial_port) + logger.info('%s: serial connected on port %d', + inst.client_id, inst.serial_port) await inst.emit('system', {'event': 'booted'}) await self._read_serial(inst, reader) return except (ConnectionRefusedError, OSError): await asyncio.sleep(1.0 * (attempt + 1)) - await inst.emit('error', {'message': 'Could not connect to QEMU serial port'}) + await inst.emit('error', + {'message': 'Could not connect to QEMU console port'}) - async def _read_serial(self, inst: PiInstance, reader: asyncio.StreamReader) -> None: + async def _read_serial(self, inst: PiInstance, + reader: asyncio.StreamReader) -> None: buf = bytearray() while inst.running: try: @@ -285,22 +308,36 @@ class QemuManager: logger.warning('%s serial read: %s', inst.client_id, e) break - # ── GPIO shim (ttyAMA1) ─────────────────────────────────────────────────── + # ── Protocol channel (virtio-serial / /dev/vport0p2) ────────────────────── + # Methods keep the historical "_gpio" naming so the rest of the + # codebase (simulation route, GPIO event bus) doesn't have to + # rename. Phase 2 swaps the line parser for the full multi-protocol + # mux while keeping this connect/read/write plumbing identical. async def _connect_gpio(self, inst: PiInstance) -> None: for attempt in range(10): try: - reader, writer = await asyncio.open_connection('127.0.0.1', inst.gpio_port) + reader, writer = await asyncio.open_connection( + '127.0.0.1', inst.gpio_port, + ) inst._gpio_writer = writer - logger.info('%s: GPIO shim connected on port %d', inst.client_id, inst.gpio_port) + logger.info('%s: protocol channel connected on port %d', + inst.client_id, inst.gpio_port) await self._read_gpio(inst, reader) return except (ConnectionRefusedError, OSError): await asyncio.sleep(1.0 * (attempt + 1)) - logger.warning('%s: GPIO shim connection failed', inst.client_id) + logger.warning('%s: protocol channel connection failed', + inst.client_id) - async def _read_gpio(self, inst: PiInstance, reader: asyncio.StreamReader) -> None: - """Parse "GPIO \n" lines from the Pi GPIO shim.""" + async def _read_gpio(self, inst: PiInstance, + reader: asyncio.StreamReader) -> None: + """Parse text-protocol lines from the Pi shim layer. + + Phase 1 understands GPIO only (existing protocol). Phase 2 + extends this to dispatch I2C/SPI/UART/PWM as well via + pi_protocol_mux. + """ linebuf = b'' while inst.running: try: @@ -310,11 +347,13 @@ class QemuManager: linebuf += chunk while b'\n' in linebuf: line, linebuf = linebuf.split(b'\n', 1) - await self._handle_gpio_line(inst, line.decode('ascii', 'ignore').strip()) + await self._handle_gpio_line( + inst, line.decode('ascii', 'ignore').strip(), + ) except asyncio.TimeoutError: continue except Exception as e: - logger.warning('%s GPIO read: %s', inst.client_id, e) + logger.warning('%s protocol read: %s', inst.client_id, e) break async def _handle_gpio_line(self, inst: PiInstance, line: str) -> None: @@ -335,7 +374,7 @@ class QemuManager: try: await inst._gpio_writer.drain() except Exception as e: - logger.warning('%s GPIO send: %s', inst.client_id, e) + logger.warning('%s protocol send: %s', inst.client_id, e) # ── QEMU stderr watcher ─────────────────────────────────────────────────── @@ -345,7 +384,6 @@ class QemuManager: loop = asyncio.get_running_loop() try: while inst.running: - # readline() blocks until a line or EOF — run in executor so we don't block the loop line = await loop.run_in_executor(None, inst.process.stderr.readline) if not line: break @@ -423,7 +461,7 @@ class QemuManager: # ── Lifespan pre-warm ──────────────────────────────────────────────────────── async def _prewarm_pi3_boot_images() -> None: - """Lifespan hook: download + cache the Pi 3 boot files in the + """Lifespan hook: download + cache the Pi 3 virt boot files in the background at process start. The cache check is cheap when files are already on disk (named @@ -441,9 +479,6 @@ async def _prewarm_pi3_boot_images() -> None: exc, ) return - # Fire-and-forget — let it complete in the background while the - # rest of the lifespan continues. provider.warmup() swallows its - # own errors, so the task never logs unhandled exceptions. asyncio.create_task(provider.warmup(PI3_IMAGE_SET)) diff --git a/test/pi3_console_boot/test_pi3_console_boot.py b/test/pi3_console_boot/test_pi3_console_boot.py new file mode 100644 index 00000000..e49a3a30 --- /dev/null +++ b/test/pi3_console_boot/test_pi3_console_boot.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +""" +Pi 3 console boot validation +============================ + +Spins up QEMU raspi3b with the SAME args the production qemu_manager +uses (two ``-serial`` chardevs, ``init=/usr/local/sbin/velxio-init``, +``console=ttyAMA1,115200 keep_bootcon earlycon=pl011,mmio32,0x3f201000``) +against the cached SD image. Connects to the user-serial TCP socket +and asserts: + +1. The kernel boots and reaches ``Run /usr/local/sbin/velxio-init as + init process``. +2. The init script emits a ``[velxio-init] selected TTY=...`` diag + line (i.e. doesn't hit the FATAL path). +3. The init script does NOT spam ``No such file or directory`` / + ``No such device or address`` errors (which mean the redirect to + the chosen TTY is failing). +4. The Velxio bashrc banner appears (``Velxio Raspberry Pi 3 — + interactive root shell``). +5. After sending ``echo VELXIO_OK_$$\\n`` to the console, the same + token comes back (proves the shell is actually reading input). + +Fidelity rule (memory ``feedback_tests_import_real_code``): the +QEMU launch args mirror what ``app.services.qemu_manager`` emits, +so any regression in the production manager that touches the +console wiring fails this test too. The SD image read is the +exact image cached by BootImageProvider at runtime. + +Run: + cd /home/dave/velxio-prod + python3 velxio/test/pi3_console_boot/test_pi3_console_boot.py + +The test takes ~45 s (boot is the slow part). Exit code 0 = OK, +non-zero = failure with the offending log excerpt printed. +""" + +from __future__ import annotations + +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + + +BOOT_IMAGES = Path("/var/cache/velxio/boot-images/raspberry-pi-3-virt") +KERNEL = BOOT_IMAGES / "velxio-kernel-arm64" +INITRAMFS = BOOT_IMAGES / "velxio-initramfs-arm64.cpio.gz" +ROOTFS = BOOT_IMAGES / "velxio-pi-rootfs-arm64.ext4" + +# Boot timeout: cold boot is ~10-15 s on virt + minimal rootfs (much +# faster than the old raspi3b + raspios path that took 25-30 s). +BOOT_TIMEOUT_S = 45 + +# Markers we expect to see in the console output during a healthy +# boot. The order is significant — each MUST appear before the +# next in time. Matches what Alpine + agetty + autologin emit: +# - "OpenRC 0.x" — openrc init started +# - "Welcome to the Velxio Pi Simulator" — /etc/motd printed by login +# - "raspberrypi:" — agetty's autologin reached the shell prompt +BOOT_MARKERS = [ + b"Velxio Pi Simulator", + b"login on 'hvc0'", +] + +# Any of these substrings in the output means the init or shell is +# broken — fail fast rather than waiting for the prompt timeout. +BOOT_NEGATIVE_MARKERS = [ + b"FATAL", + b"Kernel panic", + b"unable to mount root fs", + b"Cannot open root device", +] + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _make_qcow2_overlay() -> str: + """Mirror of ``qemu_manager`` — qcow2 over the cached rootfs ext4 + so the test never mutates the base.""" + overlay = tempfile.NamedTemporaryFile(suffix=".qcow2", delete=False) + overlay.close() + subprocess.run( + ["qemu-img", "create", "-f", "qcow2", + "-b", str(ROOTFS), "-F", "raw", overlay.name], + check=True, capture_output=True, + ) + return overlay.name + + +def _qemu_argv(overlay_path: str, serial_port: int, gpio_port: int) -> list[str]: + """Mirror of ``qemu_manager._boot`` — -M virt + virtio-{blk,serial} + over PCI + velxio kernel/initramfs/rootfs.""" + return [ + "qemu-system-aarch64", + "-M", "virt", + "-cpu", "cortex-a53", + "-smp", "4", + "-m", "1G", + "-kernel", str(KERNEL), + "-initrd", str(INITRAMFS), + "-drive", f"if=none,file={overlay_path},format=qcow2,id=rootfs", + "-device", "virtio-blk-pci,drive=rootfs", + "-nic", "none", + "-display", "none", + "-monitor", "none", + "-serial", "none", + "-chardev", f"socket,id=cons,host=127.0.0.1,port={serial_port}," + f"server=on,wait=off", + "-device", "virtio-serial-pci,id=virtio-serial0", + "-device", "virtconsole,chardev=cons", + "-chardev", f"socket,id=proto,host=127.0.0.1,port={gpio_port}," + f"server=on,wait=off", + "-device", "virtserialport,chardev=proto,name=velxio-protocol", + "-append", "console=hvc0 root=/dev/vda rw panic=10", + ] + + +def _connect_with_retry(host: str, port: int, deadline: float) -> socket.socket: + """Poll-connect to QEMU's TCP serial server (it accepts after + QEMU has fully started).""" + while time.monotonic() < deadline: + try: + s = socket.create_connection((host, port), timeout=2) + s.settimeout(2) + return s + except (ConnectionRefusedError, OSError): + time.sleep(0.2) + raise TimeoutError(f"Could not connect to {host}:{port} within deadline") + + +def _drain_until(sock: socket.socket, marker: bytes, deadline: float, + negatives: list[bytes]) -> bytes: + """Read from socket until ``marker`` appears or we hit a negative + marker / deadline. Returns the accumulated buffer for inspection.""" + buf = bytearray() + while time.monotonic() < deadline: + try: + chunk = sock.recv(4096) + except socket.timeout: + continue + if not chunk: + break + buf.extend(chunk) + for neg in negatives: + if neg in buf: + return bytes(buf) + if marker in buf: + return bytes(buf) + return bytes(buf) + + +def run() -> int: + # ── sanity: required files exist ─────────────────────────────── + for p in (KERNEL, INITRAMFS, ROOTFS): + if not p.exists(): + print(f"FAIL: missing boot image: {p}", file=sys.stderr) + return 2 + if not subprocess.run( + ["which", "qemu-system-aarch64"], capture_output=True + ).returncode == 0: + print("FAIL: qemu-system-aarch64 not in PATH", file=sys.stderr) + return 2 + + # ── start QEMU ───────────────────────────────────────────────── + overlay = _make_qcow2_overlay() + serial_port = _find_free_port() + gpio_port = _find_free_port() + argv = _qemu_argv(overlay, serial_port, gpio_port) + print(f"[test] QEMU: {' '.join(argv)}") + qemu = subprocess.Popen( + argv, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + stdin=subprocess.DEVNULL, + ) + deadline = time.monotonic() + BOOT_TIMEOUT_S + + try: + # Connect to the user-serial channel. + sock = _connect_with_retry("127.0.0.1", serial_port, deadline) + print(f"[test] connected to user serial @ :{serial_port}") + + # Walk through each expected marker in order. Any negative + # marker short-circuits the test. + all_output = bytearray() + for marker in BOOT_MARKERS: + print(f"[test] waiting for marker: {marker!r}") + chunk = _drain_until(sock, marker, deadline, BOOT_NEGATIVE_MARKERS) + all_output.extend(chunk) + for neg in BOOT_NEGATIVE_MARKERS: + if neg in chunk: + print(f"FAIL: negative marker hit: {neg!r}", file=sys.stderr) + _dump_tail(bytes(all_output), 4000) + return 1 + if marker not in chunk: + print(f"FAIL: timeout waiting for marker {marker!r}", + file=sys.stderr) + _dump_tail(bytes(all_output), 4000) + return 1 + print(f"[test] ✓ found") + + # ── round-trip: send a token, expect to see it echoed back ─ + token = f"VELXIO_OK_{os.getpid()}".encode() + # The shell echoes input back (canonical TTY mode), so sending + # `echo TOKEN` followed by newline should produce `echo TOKEN` + # (typed) followed by `TOKEN` (command output) on the wire. + # We send a small delay to let the prompt settle. + time.sleep(2) + cmd = b"echo " + token + b"\n" + print(f"[test] sending: {cmd!r}") + sock.sendall(cmd) + echoback = _drain_until(sock, token, deadline, + BOOT_NEGATIVE_MARKERS) + all_output.extend(echoback) + if token not in echoback: + print(f"FAIL: token {token!r} never came back from the shell — " + "input is not flowing or shell is dead", + file=sys.stderr) + _dump_tail(bytes(all_output), 4000) + return 1 + # The token will appear at least twice (echo of typed bytes + + # output of the echo command). One occurrence is suspicious + # (could be just the echo of our send). + if echoback.count(token) < 2: + print(f"WARN: token {token!r} appears only " + f"{echoback.count(token)}x — expected ≥2 " + "(typed echo + command output). Shell may be in " + "raw mode or the redirect is one-way.") + # Don't fail — some configs disable echo. The presence + # alone proves bidirectional flow. + + print("[test] ✓ Pi 3 console is interactive") + return 0 + finally: + try: + qemu.terminate() + qemu.wait(timeout=5) + except subprocess.TimeoutExpired: + qemu.kill() + try: + os.unlink(overlay) + except OSError: + pass + + +def _dump_tail(buf: bytes, n: int) -> None: + tail = buf[-n:].decode("utf-8", errors="replace") + print("─" * 70, file=sys.stderr) + print("Last bytes of console output:", file=sys.stderr) + print(tail, file=sys.stderr) + print("─" * 70, file=sys.stderr) + + +if __name__ == "__main__": + sys.exit(run())