diff --git a/backend/app/api/routes/micropython_libs.py b/backend/app/api/routes/micropython_libs.py new file mode 100644 index 00000000..a82bf915 --- /dev/null +++ b/backend/app/api/routes/micropython_libs.py @@ -0,0 +1,256 @@ +"""MicroPython library installation (issue #214). + +MicroPython has no Library Manager step: a library IS a .py file in the +project, copied onto the board's filesystem before main.py runs. What was +missing is a way to GET those files without hunting GitHub by hand. This +router answers it against the official micropython-lib package index — the +same index `mip` uses on real hardware (https://micropython.org/pi/v2) — +proxied server-side so the browser needs no CORS exception and so the +limits below are enforced somewhere the client cannot skip. + +Security model (the concern that kept this feature parked): + - Only the OFFICIAL index is reachable — the package name is looked up in + index.json and the files come from the index's own content-addressed + store. No arbitrary URLs, no GitHub fetch. + - Source only: the `py` arch is requested, every returned path must end in + .py, and native/bytecode (.mpy) is rejected. What lands in the editor is + exactly what will run, readable by the user. + - Bounded: per-package file count / total size caps, bounded dependency + recursion, in-process TTL cache over the index. + +The files land in the user's WORKSPACE (the frontend writes them into the +board's file group), not on any server store — so quota is the project's own +size and the "installed" library is visible, editable and travels with the +project. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from typing import Any + +import httpx +from fastapi import APIRouter, HTTPException, Query +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +router = APIRouter() + +INDEX_URL = "https://micropython.org/pi/v2/index.json" +PACKAGE_URL = "https://micropython.org/pi/v2/package/py/{name}/{version}.json" +FILE_URL = "https://micropython.org/pi/v2/file/{prefix}/{hash}" + +# The index changes when micropython-lib publishes — rarely. 15 min keeps a +# busy classroom from hammering micropython.org through us. +INDEX_TTL_S = 15 * 60 +FETCH_TIMEOUT_S = 20.0 + +# Caps: micropython-lib's largest source packages are a handful of files and +# a few tens of KB. Anything past these limits is not a driver, it is a +# mistake (or an index we should not trust that day). +MAX_FILES_PER_PACKAGE = 24 +MAX_TOTAL_BYTES = 262_144 # 256 KB of source +MAX_DEP_DEPTH = 3 + +_index_cache: tuple[float, list[dict[str, Any]]] | None = None +_index_lock = asyncio.Lock() + + +class MpyPackage(BaseModel): + name: str + version: str + description: str = "" + author: str = "" + license: str = "" + + +class SearchResponse(BaseModel): + success: bool + packages: list[MpyPackage] = [] + error: str | None = None + + +class FetchRequest(BaseModel): + name: str = Field(..., min_length=1, max_length=100) + version: str | None = Field(None, max_length=40) + + +class MpyFile(BaseModel): + path: str + content: str + + +class FetchResponse(BaseModel): + success: bool + name: str + version: str | None = None + files: list[MpyFile] = [] + error: str | None = None + + +async def _get_index() -> list[dict[str, Any]]: + global _index_cache + now = time.monotonic() + if _index_cache is not None and now - _index_cache[0] < INDEX_TTL_S: + return _index_cache[1] + async with _index_lock: + if _index_cache is not None and time.monotonic() - _index_cache[0] < INDEX_TTL_S: + return _index_cache[1] + async with httpx.AsyncClient(timeout=FETCH_TIMEOUT_S) as client: + resp = await client.get(INDEX_URL) + resp.raise_for_status() + data = resp.json() + packages = [p for p in data.get("packages", []) if "py" in p.get("versions", {})] + _index_cache = (time.monotonic(), packages) + return packages + + +def filter_packages(packages: list[dict[str, Any]], q: str) -> list[dict[str, Any]]: + """Name/description substring match, name-prefix hits first. Pure so the + unit tests need no network.""" + needle = q.strip().lower() + if not needle: + return packages + starts, contains = [], [] + for p in packages: + name = str(p.get("name", "")).lower() + desc = str(p.get("description", "")).lower() + if name.startswith(needle): + starts.append(p) + elif needle in name or needle in desc: + contains.append(p) + return starts + contains + + +def validate_package_files( + hashes: list[Any], *, package: str +) -> list[tuple[str, str]]: + """The (path, hash) pairs a package json may install, or raise. + + Enforces the source-only and path-safety rules: every path is a relative + .py with no traversal, and the count cap holds. Pure for unit tests. + """ + if len(hashes) > MAX_FILES_PER_PACKAGE: + raise ValueError(f"{package}: too many files ({len(hashes)})") + out: list[tuple[str, str]] = [] + for entry in hashes: + if not isinstance(entry, (list, tuple)) or len(entry) != 2: + raise ValueError(f"{package}: malformed hashes entry") + path, digest = str(entry[0]), str(entry[1]) + if not path.endswith(".py"): + raise ValueError(f"{package}: non-source file {path!r} (only .py is installable)") + if path.startswith("/") or ".." in path.split("/") or "\\" in path: + raise ValueError(f"{package}: unsafe path {path!r}") + if not all(seg for seg in path.split("/")): + raise ValueError(f"{package}: unsafe path {path!r}") + if not digest.isalnum(): + raise ValueError(f"{package}: malformed hash for {path!r}") + out.append((path, digest)) + return out + + +@router.get("/search", response_model=SearchResponse) +async def search_packages(q: str = Query("", max_length=100)) -> SearchResponse: + """Search the official micropython-lib index (name + description).""" + try: + packages = filter_packages(await _get_index(), q) + except Exception as e: # network / index shape — degrade, don't 500 the modal + logger.warning("[mpy-libs] index fetch failed: %s", e) + return SearchResponse(success=False, error=f"micropython-lib index unavailable: {e}") + return SearchResponse( + success=True, + packages=[ + MpyPackage( + name=str(p.get("name", "")), + version=str(p.get("version", "")), + description=str(p.get("description", "")), + author=str(p.get("author", "")), + license=str(p.get("license", "")), + ) + for p in packages[:60] + ], + ) + + +async def _fetch_package_files( + client: httpx.AsyncClient, + name: str, + version: str, + *, + seen: set[str], + depth: int, + budget: dict[str, int], +) -> list[MpyFile]: + if name in seen: + return [] + seen.add(name) + if depth > MAX_DEP_DEPTH: + raise ValueError(f"dependency chain too deep at {name!r}") + + resp = await client.get(PACKAGE_URL.format(name=name, version=version)) + if resp.status_code == 404: + raise ValueError(f"package {name!r} not found in the micropython-lib index") + resp.raise_for_status() + pkg = resp.json() + + files: list[MpyFile] = [] + for path, digest in validate_package_files(pkg.get("hashes", []), package=name): + fresp = await client.get(FILE_URL.format(prefix=digest[:2], hash=digest)) + fresp.raise_for_status() + content = fresp.text + budget["bytes"] += len(content.encode("utf-8")) + if budget["bytes"] > MAX_TOTAL_BYTES: + raise ValueError( + f"{name}: install exceeds the {MAX_TOTAL_BYTES // 1024} KB source cap" + ) + files.append(MpyFile(path=path, content=content)) + + # mip-style dependencies: [[name, version], ...]. Same store, same rules. + for dep in pkg.get("deps", []) or []: + if isinstance(dep, (list, tuple)) and dep: + dep_name = str(dep[0]) + dep_version = str(dep[1]) if len(dep) > 1 and dep[1] else "latest" + files.extend( + await _fetch_package_files( + client, dep_name, dep_version, seen=seen, depth=depth + 1, budget=budget + ) + ) + return files + + +@router.post("/fetch", response_model=FetchResponse) +async def fetch_package(request: FetchRequest) -> FetchResponse: + """Resolve one package (plus its declared deps) into .py files the + frontend writes into the board's workspace.""" + name = request.name.strip() + version = (request.version or "latest").strip() or "latest" + # The name must exist in the official index — this is what pins the fetch + # to micropython-lib and nothing else. + try: + index = await _get_index() + except Exception as e: + raise HTTPException(status_code=502, detail=f"micropython-lib index unavailable: {e}") + if not any(p.get("name") == name for p in index): + raise HTTPException( + status_code=404, + detail=f"'{name}' is not in the micropython-lib index. Community drivers outside " + "the index can be added as a .py file in the workspace.", + ) + try: + async with httpx.AsyncClient(timeout=FETCH_TIMEOUT_S) as client: + files = await _fetch_package_files( + client, name, version, seen=set(), depth=0, budget={"bytes": 0} + ) + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) + except HTTPException: + raise + except Exception as e: + logger.warning("[mpy-libs] fetch %s failed: %s", name, e) + raise HTTPException(status_code=502, detail=f"fetch failed: {e}") + if not files: + raise HTTPException(status_code=422, detail=f"{name}: package resolved to no .py files") + return FetchResponse(success=True, name=name, version=version, files=files) diff --git a/backend/app/main.py b/backend/app/main.py index e4044774..b7112cb9 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -13,7 +13,7 @@ if sys.platform == 'win32': from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from app.api.routes import compile, compile_chip, compile_rom, flash, intellisense, libraries +from app.api.routes import compile, compile_chip, compile_rom, flash, intellisense, libraries, micropython_libs from app.core.config import settings from app.core.hooks import run_lifespan_startup @@ -94,6 +94,11 @@ app.include_router(compile.router, prefix="/api/compile", tags=["compilation"]) app.include_router(compile_chip.router, prefix="/api/compile-chip", tags=["custom-chips"]) app.include_router(compile_rom.router, prefix="/api/compile-rom", tags=["custom-chips"]) app.include_router(libraries.router, prefix="/api/libraries", tags=["libraries"]) +# MicroPython libraries (issue #214): resolves the official micropython-lib +# index into .py files the frontend writes into the board's workspace. +app.include_router( + micropython_libs.router, prefix="/api/micropython-libs", tags=["micropython-libraries"] +) app.include_router(intellisense.router, prefix="/api/intellisense", tags=["intellisense"]) # Hardware flash: subprocesses arduino-cli upload to write a compiled # sketch to a real USB-attached board. Desktop-only in practice (the diff --git a/frontend/src/__tests__/micropython-libs-service.test.ts b/frontend/src/__tests__/micropython-libs-service.test.ts new file mode 100644 index 00000000..38dddc1a --- /dev/null +++ b/frontend/src/__tests__/micropython-libs-service.test.ts @@ -0,0 +1,80 @@ +/** + * micropythonLibs service (issue #214) — the workspace-write half. + * + * The backend resolves a micropython-lib package into .py files; this side + * upserts them into the board's file group. Pinned here: upsert semantics + * (re-adding = upgrade, not duplicate), package-dir paths, and the + * in-project detection the modal's pills rely on. + */ +import { describe, it, expect, beforeEach } from 'vitest'; +import { useEditorStore } from '../store/useEditorStore'; +import { + packageFilesInGroup, + writeFilesIntoGroup, + BUILTIN_MPY_MODULES, + FEATURED_MPY_PACKAGES, +} from '../services/micropythonLibs'; + +const GROUP = 'group-mpy-test'; + +describe('writeFilesIntoGroup', () => { + beforeEach(() => { + const ed = useEditorStore.getState(); + ed.deleteFileGroup(GROUP); + ed.createFileGroup(GROUP, [{ name: 'main.py', content: 'print(1)\n' }]); + ed.setActiveGroup(GROUP); + }); + + it('adds the package files next to main.py', () => { + const written = writeFilesIntoGroup(GROUP, [ + { path: 'ssd1306.py', content: '# driver\n' }, + ]); + expect(written).toEqual(['ssd1306.py']); + const names = useEditorStore.getState().getGroupFiles(GROUP).map((f) => f.name); + expect(names).toContain('main.py'); + expect(names).toContain('ssd1306.py'); + }); + + it('re-adding overwrites instead of duplicating (that is the upgrade path)', () => { + writeFilesIntoGroup(GROUP, [{ path: 'ssd1306.py', content: 'v1\n' }]); + writeFilesIntoGroup(GROUP, [{ path: 'ssd1306.py', content: 'v2\n' }]); + const files = useEditorStore.getState().getGroupFiles(GROUP); + const matches = files.filter((f) => f.name === 'ssd1306.py'); + expect(matches).toHaveLength(1); + expect(matches[0].content).toBe('v2\n'); + }); + + it('handles package-dir paths the way micropython-lib ships them', () => { + writeFilesIntoGroup(GROUP, [{ path: 'umqtt/simple.py', content: '# mqtt\n' }]); + const names = useEditorStore.getState().getGroupFiles(GROUP).map((f) => f.name); + expect(names).toContain('umqtt/simple.py'); + }); +}); + +describe('packageFilesInGroup', () => { + beforeEach(() => { + const ed = useEditorStore.getState(); + ed.deleteFileGroup(GROUP); + ed.createFileGroup(GROUP, [{ name: 'main.py', content: '' }]); + ed.setActiveGroup(GROUP); + }); + + it('detects a plain module and a dotted package', () => { + expect(packageFilesInGroup(GROUP, 'ssd1306')).toBe(false); + writeFilesIntoGroup(GROUP, [{ path: 'ssd1306.py', content: 'x' }]); + expect(packageFilesInGroup(GROUP, 'ssd1306')).toBe(true); + + expect(packageFilesInGroup(GROUP, 'umqtt.simple')).toBe(false); + writeFilesIntoGroup(GROUP, [{ path: 'umqtt/simple.py', content: 'x' }]); + expect(packageFilesInGroup(GROUP, 'umqtt.simple')).toBe(true); + }); +}); + +describe('curated lists', () => { + it('featured picks never claim firmware builtins (one truth per module)', () => { + const builtins = new Set(BUILTIN_MPY_MODULES.map((b) => b.name)); + for (const f of FEATURED_MPY_PACKAGES) { + expect(builtins.has(f.name)).toBe(false); + } + }); +}); diff --git a/frontend/src/components/simulator/LibraryManagerModal.tsx b/frontend/src/components/simulator/LibraryManagerModal.tsx index 14c3adf6..bc588cd3 100644 --- a/frontend/src/components/simulator/LibraryManagerModal.tsx +++ b/frontend/src/components/simulator/LibraryManagerModal.tsx @@ -10,6 +10,15 @@ import { } from '../../services/libraryService'; import type { ArduinoLibrary, InstalledLibrary } from '../../services/libraryService'; import { trackInstallLibrary } from '../../utils/analytics'; +import { + BUILTIN_MPY_MODULES, + FEATURED_MPY_PACKAGES, + fetchMpyPackage, + packageFilesInGroup, + searchMpyPackages, + writeFilesIntoGroup, +} from '../../services/micropythonLibs'; +import type { MpyPackage } from '../../services/micropythonLibs'; import { useSimulatorStore } from '../../store/useSimulatorStore'; import { boardDisplayName } from '../../types/board'; import './LibraryManagerModal.css'; @@ -57,6 +66,17 @@ export const LibraryManagerModal: React.FC = ({ isOpen const manifestLibs = activeBoard?.libraries ?? null; const declared = manifestLibs ?? []; + // ── MicroPython mode (issue #214) ───────────────────────────────────────── + // Same modal, different backing: a MicroPython library is a .py file in + // the project, so search hits the micropython-lib index (backend proxy) + // and "Add to project" writes the package's files into this board's file + // group — no libraries.json, no arduino-cli. + const isMpy = activeBoard?.languageMode === 'micropython'; + const [mpyResults, setMpyResults] = useState([]); + // Bumped after add/remove so the "In project" pills re-derive from the + // file group (which lives outside React state). + const [mpyTick, setMpyTick] = useState(0); + const [searchQuery, setSearchQuery] = useState(''); const [searchResults, setSearchResults] = useState([]); const [installedLibraries, setInstalledLibraries] = useState([]); @@ -169,10 +189,12 @@ export const LibraryManagerModal: React.FC = ({ isOpen setLoadingSearch(true); setStatusMsg(null); try { - setSearchResults(await searchLibraries(searchQuery)); + if (isMpy) setMpyResults(await searchMpyPackages(searchQuery)); + else setSearchResults(await searchLibraries(searchQuery)); } catch (e: unknown) { setStatusMsg({ type: 'error', text: e instanceof Error ? e.message : 'Search failed' }); - setSearchResults([]); + if (isMpy) setMpyResults([]); + else setSearchResults([]); } finally { setLoadingSearch(false); } @@ -180,7 +202,7 @@ export const LibraryManagerModal: React.FC = ({ isOpen return () => { if (debounceRef.current) clearTimeout(debounceRef.current); }; - }, [searchQuery, isOpen]); + }, [searchQuery, isOpen, isMpy]); // A custom .zip upload (pro) lands in the user's per-user store; auto-declare // it on the active board and refresh the list. The upload BUTTON is injected @@ -277,6 +299,83 @@ export const LibraryManagerModal: React.FC = ({ isOpen })); }, [searchQuery, searchResults, installedLibraries, isInstalled]); + // ── MicroPython actions ─────────────────────────────────────────────────── + const mpyGroupId = activeBoard?.activeFileGroupId ?? null; + const addMpyPackage = useCallback( + async (name: string) => { + if (!mpyGroupId) return; + setBusyLib(name); + setStatusMsg(null); + try { + const files = await fetchMpyPackage(name); + const written = writeFilesIntoGroup(mpyGroupId, files); + trackInstallLibrary(`mpy:${name}`); + setStatusMsg({ + type: 'success', + text: `"${name}" added to the workspace: ${written.join(', ')}. main.py can import it now.`, + }); + setMpyTick((n) => n + 1); + } catch (e: unknown) { + const detail = + (e as { response?: { data?: { detail?: string } } })?.response?.data?.detail; + setStatusMsg({ + type: 'error', + text: detail || (e instanceof Error ? e.message : `Failed to add "${name}"`), + }); + } finally { + setBusyLib(null); + } + }, + [mpyGroupId], + ); + + interface MpyRow { + name: string; + version: string; + desc: string; + builtin: boolean; + inProject: boolean; + } + // eslint-disable-next-line react-hooks/exhaustive-deps -- mpyTick re-derives from the file group + const mpyRows: MpyRow[] = useMemo(() => { + if (!isMpy || !mpyGroupId) return []; + const inProj = (name: string) => packageFilesInGroup(mpyGroupId, name); + if (searchQuery.trim()) { + const needle = searchQuery.trim().toLowerCase(); + const builtins = BUILTIN_MPY_MODULES.filter((b) => b.name.includes(needle)).map((b) => ({ + name: b.name, + version: '', + desc: b.blurb, + builtin: true, + inProject: false, + })); + return builtins.concat( + mpyResults.map((p) => ({ + name: p.name, + version: p.version, + desc: p.description, + builtin: false, + inProject: inProj(p.name), + })), + ); + } + return FEATURED_MPY_PACKAGES.map((f) => ({ + name: f.name, + version: '', + desc: f.blurb, + builtin: false, + inProject: inProj(f.name), + })).concat( + BUILTIN_MPY_MODULES.map((b) => ({ + name: b.name, + version: '', + desc: b.blurb, + builtin: true, + inProject: false, + })), + ); + }, [isMpy, mpyGroupId, searchQuery, mpyResults, mpyTick]); + if (!isOpen) return null; const browsing = !searchQuery.trim(); @@ -305,7 +404,11 @@ export const LibraryManagerModal: React.FC = ({ isOpen {t('editor.libraryManager.title')} {activeBoard && ( = ({ isOpen padding: '1px 8px', }} > - {boardDisplayName(activeBoard)} · {declared.length} + {boardDisplayName(activeBoard)} + {isMpy ? ' · MicroPython' : ` · ${declared.length}`} )} @@ -362,12 +466,82 @@ export const LibraryManagerModal: React.FC = ({ isOpen {/* Single unified list */}
- {browsing && loadingInstalled && ( + {isMpy && browsing && ( +
+

+ A MicroPython library is a .py file in the project — adding one + writes it into this board's workspace, next to main.py. Search + the official micropython-lib index above. +

+
+ )} + {isMpy && + mpyRows.map((row) => { + const busy = busyLib === row.name; + return ( +
+
+
+ {row.name} + {row.builtin && ( + + built into the firmware + + )} +
+ {row.desc &&

{row.desc}

} +
+
+ {row.version && {row.version}} + {row.builtin ? ( + + import {row.name} + + ) : row.inProject ? ( + + In project ✓ + + ) : ( + + )} +
+
+ ); + })} + {isMpy && !browsing && !loadingSearch && mpyRows.length === 0 && ( +
+

{t('editor.libraryManager.noResultsFor', { query: searchQuery })}

+

+ Not in micropython-lib? Paste the driver as a .py file in the + workspace — that is all an install does. +

+
+ )} + {!isMpy && browsing && loadingInstalled && (

{t('editor.libraryManager.loadingInstalled')}

)} - {!loadingSearch && rows.length === 0 && !(browsing && loadingInstalled) && ( + {!isMpy && !loadingSearch && rows.length === 0 && !(browsing && loadingInstalled) && (

{browsing @@ -379,7 +553,7 @@ export const LibraryManagerModal: React.FC = ({ isOpen {browsing &&

Search above to find and add a library.

}
)} - {rows.map((row, i) => { + {!isMpy && rows.map((row, i) => { const inProj = inManifest(row.name); const busy = busyLib === row.name; return ( diff --git a/frontend/src/data/examples.ts b/frontend/src/data/examples.ts index 03134251..b8599ee1 100644 --- a/frontend/src/data/examples.ts +++ b/frontend/src/data/examples.ts @@ -267,6 +267,242 @@ void loop() { }, ], }, + // ── MicroPython libraries are files (no Library Manager, no requirements) ── + { + id: 'esp32-micropython-external-library', + title: 'ESP32 + MicroPython: use an external library', + description: + "How a MicroPython project uses a library in Velxio: there is no requirements.txt and no package manager — the driver is a .py file in the workspace. Every file here is copied onto the board's filesystem before main.py runs, so `import ssd1306` picks up the ssd1306.py next to it and draws on the wired OLED. The toolbar's Library Manager installs ARDUINO libraries and has no effect in MicroPython mode.", + category: 'displays', + difficulty: 'beginner', + boardType: 'esp32', + languageMode: 'micropython', + tags: ['esp32', 'micropython', 'library', 'module', 'import', 'ssd1306', 'oled', 'i2c'], + code: '', + files: [ + { + name: 'main.py', + content: `# ESP32 + MicroPython: use an external library. +# +# MicroPython has no Library Manager here: a library IS a .py file in the +# project. Every file in this workspace is written to the board's filesystem +# before main.py runs, so the "import ssd1306" below resolves to the ssd1306.py +# sitting next to this file (open it in the explorer). To use another driver, +# add a new file and paste it in - that is the whole mechanism. +# +# (The toolbar's Library Manager installs ARDUINO libraries. In MicroPython +# mode it has no effect on the run.) +from machine import Pin, I2C +import ssd1306 +import time + +# Velxio decodes the hardware I2C controller (no bit-banged buses): +# ESP32 SDA = GPIO21, SCL = GPIO22. +i2c = I2C(0, sda=Pin(21), scl=Pin(22)) +oled = ssd1306.SSD1306_I2C(128, 64, i2c) + +n = 0 +while True: + oled.fill(0) + oled.text("ssd1306.py", 0, 0) + oled.text("imported OK", 0, 12) + oled.text("count: {}".format(n), 0, 32) + oled.text("MicroPython", 0, 52) + oled.show() + print("tick", n) + n += 1 + time.sleep(1)`, + }, + { + // The stock MicroPython firmware ships no display drivers, so the + // project carries the one it needs: micropython-lib's ssd1306 driver, + // unmodified. This is exactly what a user would paste in. + name: 'ssd1306.py', + content: `# MicroPython SSD1306 OLED driver, I2C and SPI interfaces + +from micropython import const +import framebuf + + +# register definitions +SET_CONTRAST = const(0x81) +SET_ENTIRE_ON = const(0xA4) +SET_NORM_INV = const(0xA6) +SET_DISP = const(0xAE) +SET_MEM_ADDR = const(0x20) +SET_COL_ADDR = const(0x21) +SET_PAGE_ADDR = const(0x22) +SET_DISP_START_LINE = const(0x40) +SET_SEG_REMAP = const(0xA0) +SET_MUX_RATIO = const(0xA8) +SET_COM_OUT_DIR = const(0xC0) +SET_DISP_OFFSET = const(0xD3) +SET_COM_PIN_CFG = const(0xDA) +SET_DISP_CLK_DIV = const(0xD5) +SET_PRECHARGE = const(0xD9) +SET_VCOM_DESEL = const(0xDB) +SET_CHARGE_PUMP = const(0x8D) + +# Subclassing FrameBuffer provides support for graphics primitives +# http://docs.micropython.org/en/latest/pyboard/library/framebuf.html +class SSD1306(framebuf.FrameBuffer): + def __init__(self, width, height, external_vcc): + self.width = width + self.height = height + self.external_vcc = external_vcc + self.pages = self.height // 8 + self.buffer = bytearray(self.pages * self.width) + super().__init__(self.buffer, self.width, self.height, framebuf.MONO_VLSB) + self.init_display() + + def init_display(self): + for cmd in ( + SET_DISP | 0x00, # off + # address setting + SET_MEM_ADDR, + 0x00, # horizontal + # resolution and layout + SET_DISP_START_LINE | 0x00, + SET_SEG_REMAP | 0x01, # column addr 127 mapped to SEG0 + SET_MUX_RATIO, + self.height - 1, + SET_COM_OUT_DIR | 0x08, # scan from COM[N] to COM0 + SET_DISP_OFFSET, + 0x00, + SET_COM_PIN_CFG, + 0x02 if self.width > 2 * self.height else 0x12, + # timing and driving scheme + SET_DISP_CLK_DIV, + 0x80, + SET_PRECHARGE, + 0x22 if self.external_vcc else 0xF1, + SET_VCOM_DESEL, + 0x30, # 0.83*Vcc + # display + SET_CONTRAST, + 0xFF, # maximum + SET_ENTIRE_ON, # output follows RAM contents + SET_NORM_INV, # not inverted + # charge pump + SET_CHARGE_PUMP, + 0x10 if self.external_vcc else 0x14, + SET_DISP | 0x01, + ): # on + self.write_cmd(cmd) + self.fill(0) + self.show() + + def poweroff(self): + self.write_cmd(SET_DISP | 0x00) + + def poweron(self): + self.write_cmd(SET_DISP | 0x01) + + def contrast(self, contrast): + self.write_cmd(SET_CONTRAST) + self.write_cmd(contrast) + + def invert(self, invert): + self.write_cmd(SET_NORM_INV | (invert & 1)) + + def show(self): + x0 = 0 + x1 = self.width - 1 + if self.width == 64: + # displays with width of 64 pixels are shifted by 32 + x0 += 32 + x1 += 32 + self.write_cmd(SET_COL_ADDR) + self.write_cmd(x0) + self.write_cmd(x1) + self.write_cmd(SET_PAGE_ADDR) + self.write_cmd(0) + self.write_cmd(self.pages - 1) + self.write_data(self.buffer) + + +class SSD1306_I2C(SSD1306): + def __init__(self, width, height, i2c, addr=0x3C, external_vcc=False): + self.i2c = i2c + self.addr = addr + self.temp = bytearray(2) + self.write_list = [b"\\x40", None] # Co=0, D/C#=1 + super().__init__(width, height, external_vcc) + + def write_cmd(self, cmd): + self.temp[0] = 0x80 # Co=1, D/C#=0 + self.temp[1] = cmd + self.i2c.writeto(self.addr, self.temp) + + def write_data(self, buf): + self.write_list[1] = buf + self.i2c.writevto(self.addr, self.write_list) + + +class SSD1306_SPI(SSD1306): + def __init__(self, width, height, spi, dc, res, cs, external_vcc=False): + self.rate = 10 * 1024 * 1024 + dc.init(dc.OUT, value=0) + res.init(res.OUT, value=0) + cs.init(cs.OUT, value=1) + self.spi = spi + self.dc = dc + self.res = res + self.cs = cs + import time + + self.res(1) + time.sleep_ms(1) + self.res(0) + time.sleep_ms(10) + self.res(1) + super().__init__(width, height, external_vcc) + + def write_cmd(self, cmd): + self.spi.init(baudrate=self.rate, polarity=0, phase=0) + self.cs(1) + self.dc(0) + self.cs(0) + self.spi.write(bytearray([cmd])) + self.cs(1) + + def write_data(self, buf): + self.spi.init(baudrate=self.rate, polarity=0, phase=0) + self.cs(1) + self.dc(1) + self.cs(0) + self.spi.write(buf) + self.cs(1)`, + }, + ], + components: [{ type: 'velxio-ssd1306-i2c-4pin', id: 'oled', x: 470, y: 100, properties: {} }], + wires: [ + { + id: 'w-vcc', + start: { componentId: 'esp32', pinName: '3V3' }, + end: { componentId: 'oled', pinName: 'VCC' }, + color: '#ff4444', + }, + { + id: 'w-gnd', + start: { componentId: 'esp32', pinName: 'GND' }, + end: { componentId: 'oled', pinName: 'GND' }, + color: '#000000', + }, + { + id: 'w-sda', + start: { componentId: 'esp32', pinName: '21' }, + end: { componentId: 'oled', pinName: 'SDA' }, + color: '#22aaff', + }, + { + id: 'w-scl', + start: { componentId: 'esp32', pinName: '22' }, + end: { componentId: 'oled', pinName: 'SCL' }, + color: '#ff8800', + }, + ], + }, // ── Pure ESP-IDF (issue #139): user app_main(), no Arduino core. ── { id: 'esp32-idf-blink', diff --git a/frontend/src/services/micropythonLibs.ts b/frontend/src/services/micropythonLibs.ts new file mode 100644 index 00000000..770c1276 --- /dev/null +++ b/frontend/src/services/micropythonLibs.ts @@ -0,0 +1,111 @@ +/** + * micropythonLibs — the MicroPython side of the Library Manager (issue #214). + * + * There is no install step on the board: a MicroPython library IS a .py file + * in the project, copied onto the board's filesystem before main.py runs. So + * "installing" here means resolving a package from the official + * micropython-lib index (via the backend proxy, which enforces source-only + + * size caps) and WRITING its files into the active board's file group — + * visible in the explorer, editable, and travelling with the project. + */ +import axios from 'axios'; +import { getApiBase } from '../lib/apiBase'; +import { useEditorStore } from '../store/useEditorStore'; + +export interface MpyPackage { + name: string; + version: string; + description: string; + author: string; + license: string; +} + +export interface MpyFile { + path: string; + content: string; +} + +/** + * Featured picks shown before the user types — the drivers Velxio actually + * emulates as parts, so "add → wire → run" works end to end. All resolve + * through the same index fetch; nothing here is a separate install path. + */ +export const FEATURED_MPY_PACKAGES: ReadonlyArray<{ name: string; blurb: string }> = [ + { name: 'ssd1306', blurb: 'SSD1306 OLED driver (I2C/SPI) — the 128x64 display part' }, + { name: 'umqtt.simple', blurb: 'Minimal MQTT client (WiFi boards)' }, + { name: 'umqtt.robust', blurb: 'MQTT client with auto-reconnect' }, + { name: 'urequests', blurb: 'HTTP requests (WiFi boards)' }, +]; + +/** + * Modules already frozen into the MicroPython firmware Velxio boots — no + * install needed, `import` just works. Listed so a search for "dht" answers + * "you already have it" instead of a confusing empty result. + */ +export const BUILTIN_MPY_MODULES: ReadonlyArray<{ name: string; blurb: string }> = [ + { name: 'machine', blurb: 'Pins, I2C, SPI, ADC, PWM, timers' }, + { name: 'network', blurb: 'WiFi (WLAN) on ESP32-family boards' }, + { name: 'neopixel', blurb: 'WS2812 RGB LED strips' }, + { name: 'dht', blurb: 'DHT11 / DHT22 temperature & humidity' }, + { name: 'onewire', blurb: '1-Wire bus' }, + { name: 'ds18x20', blurb: 'DS18B20 temperature sensor (over onewire)' }, + { name: 'framebuf', blurb: 'Frame buffer drawing (used by display drivers)' }, +]; + +export async function searchMpyPackages(q: string): Promise { + const { data } = await axios.get<{ success: boolean; packages: MpyPackage[]; error?: string }>( + `${getApiBase()}/micropython-libs/search`, + { params: { q } }, + ); + if (!data.success) throw new Error(data.error || 'search failed'); + return data.packages; +} + +export async function fetchMpyPackage(name: string): Promise { + const { data } = await axios.post<{ success: boolean; files: MpyFile[]; error?: string }>( + `${getApiBase()}/micropython-libs/fetch`, + { name }, + ); + if (!data.success) throw new Error(data.error || 'fetch failed'); + return data.files; +} + +/** + * Upsert the package's files into a board's file group. Existing files with + * the same path are overwritten (that IS the upgrade path); everything else + * is untouched. The group must be the ACTIVE one — the modal operates on the + * active board, whose group the editor is already pointed at. + */ +export function writeFilesIntoGroup(groupId: string, files: MpyFile[]): string[] { + const ed = useEditorStore.getState(); + if (ed.activeGroupId !== groupId) ed.setActiveGroup(groupId); + const written: string[] = []; + for (const f of files) { + const existing = useEditorStore + .getState() + .getGroupFiles(groupId) + .find((x) => x.name === f.path); + if (existing) { + useEditorStore.getState().updateGroupFile(groupId, existing.id, f.content); + } else { + const id = useEditorStore.getState().createFile(f.path); + useEditorStore.getState().setFileContent(id, f.content); + } + written.push(f.path); + } + return written; +} + +/** Which of the package's files are already in the group (by exact path). */ +export function packageFilesInGroup(groupId: string, name: string): boolean { + // Cheap heuristic for list rendering: the package's canonical module file + // ("ssd1306" -> ssd1306.py, "umqtt.simple" -> umqtt/simple.py) is present. + const candidates = [`${name}.py`, `${name.replace(/\./g, '/')}.py`, `${name}/__init__.py`]; + const names = new Set( + useEditorStore + .getState() + .getGroupFiles(groupId) + .map((f) => f.name), + ); + return candidates.some((c) => names.has(c)); +} diff --git a/frontend/src/store/useSimulatorStore.ts b/frontend/src/store/useSimulatorStore.ts index 33f4f266..3f50f582 100644 --- a/frontend/src/store/useSimulatorStore.ts +++ b/frontend/src/store/useSimulatorStore.ts @@ -2002,14 +2002,26 @@ export const useSimulatorStore = create((set, get) => { const auxFiles = files.filter( (f) => f !== mainFile && f.name.endsWith('.py'), ); - const preludeLines = auxFiles.map((f) => { + // A packaged module ("umqtt/simple.py" — how micropython-lib ships + // umqtt.simple) needs its directory created first: MicroPython's + // open() does not mkdir, so without this the prelude died with + // ENOENT before main.py ever ran. + const mkdirs = new Set(); + for (const f of auxFiles) { + const segs = f.name.split('/').slice(0, -1); + for (let i = 1; i <= segs.length; i++) mkdirs.add(segs.slice(0, i).join('/')); + } + const mkdirLines = [...mkdirs].sort().map( + (d) => `try:\n __import__('os').mkdir(${JSON.stringify(d)})\nexcept OSError:\n pass`, + ); + const preludeLines = mkdirLines.concat(auxFiles.map((f) => { // JSON.stringify produces an ASCII-safe Python-compatible // string literal (both languages share the same \n \r \t \" \\ // escapes, and JSON does not emit any escape Python rejects). const lit = JSON.stringify(f.content); const path = JSON.stringify(f.name); return `with open(${path},'w') as _f:\n _f.write(${lit})`; - }); + })); // Does this run get a real, WORKING network driver? Two gates: // - the sketch must want WiFi (must match what startBoard() diff --git a/test/backend/unit/test_micropython_libs.py b/test/backend/unit/test_micropython_libs.py new file mode 100644 index 00000000..a12c45fc --- /dev/null +++ b/test/backend/unit/test_micropython_libs.py @@ -0,0 +1,86 @@ +"""micropython-libs route (issue #214) — the pure validation layer. + +The network calls are exercised end-to-end against the real index by the +deploy smoke; here we pin the rules that make the endpoint safe to expose: +source-only, path-safe, bounded, and honest search ordering. +""" + +import pytest + +from app.api.routes.micropython_libs import ( + MAX_FILES_PER_PACKAGE, + filter_packages, + validate_package_files, +) + + +# ── validate_package_files ────────────────────────────────────────────────── + +def test_accepts_plain_py_files(): + out = validate_package_files( + [["ssd1306.py", "156dcec1"], ["umqtt/simple.py", "abc123"]], package="x" + ) + assert out == [("ssd1306.py", "156dcec1"), ("umqtt/simple.py", "abc123")] + + +@pytest.mark.parametrize( + "path", + [ + "driver.mpy", # bytecode — the user could not read what runs + "native.so", + "README.md", + "/etc/passwd.py", # absolute + "../evil.py", # traversal + "a/../../evil.py", + "a//b.py", # empty segment + "c:\\windows\\x.py", # backslash + ], +) +def test_rejects_unsafe_or_non_source_paths(path): + with pytest.raises(ValueError): + validate_package_files([[path, "abc123"]], package="x") + + +def test_rejects_malformed_hash(): + with pytest.raises(ValueError): + validate_package_files([["ok.py", "../f"]], package="x") + + +def test_rejects_oversized_package(): + entries = [[f"f{i}.py", "abc123"] for i in range(MAX_FILES_PER_PACKAGE + 1)] + with pytest.raises(ValueError): + validate_package_files(entries, package="x") + + +def test_rejects_malformed_entries(): + with pytest.raises(ValueError): + validate_package_files([["only-one-element"]], package="x") + + +# ── filter_packages ───────────────────────────────────────────────────────── + +INDEX = [ + {"name": "ssd1306", "description": "OLED display driver"}, + {"name": "umqtt.simple", "description": "MQTT client"}, + {"name": "aiohttp", "description": "HTTP client/server with websockets"}, +] + + +def test_name_prefix_ranks_before_description_hits(): + got = [p["name"] for p in filter_packages(INDEX, "ss")] + assert got == ["ssd1306"] + got = [p["name"] for p in filter_packages(INDEX, "client")] + assert set(got) == {"umqtt.simple", "aiohttp"} + + +def test_empty_query_returns_everything(): + assert filter_packages(INDEX, " ") == INDEX + + +def test_prefix_hits_come_first(): + idx = [ + {"name": "requests", "description": "umqtt helper"}, + {"name": "umqtt.robust", "description": ""}, + ] + got = [p["name"] for p in filter_packages(idx, "umqtt")] + assert got == ["umqtt.robust", "requests"]