diff --git a/backend/app/api/routes/compile.py b/backend/app/api/routes/compile.py index 687d4798..873c7508 100644 --- a/backend/app/api/routes/compile.py +++ b/backend/app/api/routes/compile.py @@ -55,6 +55,7 @@ def _job_key( board_fqbn: str, board_options: dict | None = None, spiffs_files: list[dict] | None = None, + libraries: list[str] | None = None, ) -> str: """Stable content hash of (files, board, options, spiffs) used as the deduplication key. @@ -84,6 +85,12 @@ def _job_key( h.update(b"\0") h.update(f["content_b64"].encode()) h.update(b"\0") + if libraries: + # Manifest changes the resolved library set → different binary, so it + # must not dedup to a job built with a different manifest. + for name in sorted(libraries): + h.update(name.encode()) + h.update(b"\0") return h.hexdigest() @@ -136,6 +143,11 @@ class CompileRequest(BaseModel): # User-uploaded files to bake into the SPIFFS partition (#162). Empty / # None means the SPIFFS region stays blank (current behaviour). spiffs_files: list[SpiffsFileBody] | None = None + # P2 — project library manifest (declared library names). When provided, + # ESP-IDF library resolution is SCOPED to this set: a user-installed lib is + # merged only if it's declared here, so a sketch never picks up an unrelated + # library from the shared dir. None / omitted = legacy scan-all (unchanged). + libraries: list[str] | None = None class CompileResponse(BaseModel): @@ -202,6 +214,7 @@ async def _run_compile( progress_callback=progress_callback, board_options=request.board_options, spiffs_files=spiffs_dicts, + allowed_libraries=set(request.libraries) if request.libraries is not None else None, ) return CompileResponse( success=result["success"], @@ -470,7 +483,7 @@ async def compile_start( spiffs_dicts = ( [f.model_dump() for f in request.spiffs_files] if request.spiffs_files else None ) - key = _job_key(files, request.board_fqbn, request.board_options, spiffs_dicts) + key = _job_key(files, request.board_fqbn, request.board_options, spiffs_dicts, request.libraries) existing_id = JOB_BY_KEY.get(key) if existing_id is not None: existing = COMPILE_JOBS.get(existing_id) diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index 4ba24d31..65d6ba47 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -703,6 +703,23 @@ class ESPIDFCompiler: arches = {a.strip().lower() for a in arch.split(',') if a.strip()} return '*' in arches or self._ESP32_LIB_ARCH in arches + @staticmethod + def _norm_lib_name(name: str) -> str: + """Normalise a library name for manifest matching: lowercased, only + alphanumerics. So "Adafruit GFX Library", "Adafruit_GFX_Library" and + "adafruitgfxlibrary" all compare equal — the Library Manager display + name and the on-disk folder name differ only by separators/case.""" + return ''.join(ch for ch in (name or '').lower() if ch.isalnum()) + + def _library_in_manifest(self, lib_root: Path, allowed_norm: set[str]) -> bool: + """True if this library is in the project's declared manifest. + Matches on the on-disk folder name OR the library.properties `name=` + (the Library Manager display name), both normalised.""" + if self._norm_lib_name(lib_root.name) in allowed_norm: + return True + props_name = self._parse_library_properties(lib_root).get('name', '') + return bool(props_name) and self._norm_lib_name(props_name) in allowed_norm + def _resolve_library_components( self, ext_headers: list[str], @@ -710,11 +727,22 @@ class ESPIDFCompiler: esp32_libs: Path | None, arduino_comp_name: str, user_libs_dir: Path, + allowed_libraries: set[str] | None = None, ) -> tuple[list[str], dict[str, str]]: """ BFS over ext_headers (and transitive includes) to discover all external Arduino libraries and merge them into a single 'user_libs_all' IDF component. + `allowed_libraries` (P2 — project library manifest / scope): when not + None, a USER-installed library (from arduino_libs) is merged only if its + name is in this set. This makes the project's declared manifest the + resolution SCOPE — the compiler never picks up an unrelated library from + the shared dir (another user's install, or a same-named clash). When + None (no manifest supplied) the behaviour is the legacy scan-all, so + existing callers and un-migrated projects are unaffected. Core + arduino-esp32 libs and bundled esp32_libs are always allowed (they are + platform-provided, not user installs). + All library files are copied flat into one directory, so every header is visible to every other header and source file without any cross-component REQUIRES propagation — which is unreliable in ESP-IDF 4.x for deeply @@ -734,6 +762,14 @@ class ESPIDFCompiler: logger.info(f'[espidf] arduino_libs: {arduino_libs}') logger.info(f'[espidf] esp32_libs: {esp32_libs}') + # P2 manifest scope: normalise the allowed set once (None = scan-all). + allowed_norm: set[str] | None = ( + {self._norm_lib_name(a) for a in allowed_libraries} + if allowed_libraries is not None else None + ) + if allowed_norm is not None: + logger.info(f'[espidf] library manifest scope active: {sorted(allowed_libraries)}') + comp_dir = user_libs_dir / 'user_libs_all' comp_dir.mkdir(exist_ok=True) @@ -784,6 +820,20 @@ class ESPIDFCompiler: ) src_root = None + # P2 manifest scope. A user-installed library is merged only if it's + # in the project's declared manifest. Anything else from the shared + # dir is out of scope — drop it (the header then falls through to the + # core check, or to the "not found" path so install-on-missing can + # offer to add it to the manifest). Core/bundled libs aren't gated. + if src_root is not None and allowed_norm is not None: + _lr = src_root.parent if src_root.name == 'src' else src_root + if not self._library_in_manifest(_lr, allowed_norm): + logger.info( + f'[espidf] <{header}> resolves to "{_lr.name}" but it is not in ' + f'the project library manifest — not merging (scope)' + ) + src_root = None + # Tracks the "resolved to a core lib that's already compiled into # the arduino-esp32 component" case, so we don't fall through to # the scary "not found — build may fail" warning below for a @@ -1574,6 +1624,7 @@ class ESPIDFCompiler: progress_callback: Optional[ProgressCallback] = None, board_options: dict | None = None, spiffs_files: list[dict] | None = None, + allowed_libraries: set[str] | None = None, ) -> dict: """ Compile Arduino sketch using ESP-IDF. @@ -1641,6 +1692,7 @@ class ESPIDFCompiler: return await self._compile_in_dir( project_dir, files, idf_target, is_c3, progress_callback, normalized_opts, spiffs_files, + allowed_libraries=allowed_libraries, ) with tempfile.TemporaryDirectory(prefix='espidf_') as temp_dir: @@ -1650,6 +1702,7 @@ class ESPIDFCompiler: return await self._compile_in_dir( project_dir, files, idf_target, is_c3, progress_callback, normalized_opts, spiffs_files, + allowed_libraries=allowed_libraries, ) async def _compile_in_dir( @@ -1661,6 +1714,7 @@ class ESPIDFCompiler: progress_callback: Optional[ProgressCallback] = None, board_options: dict | None = None, spiffs_files: list[dict] | None = None, + allowed_libraries: set[str] | None = None, ) -> dict: """Inner compile body: writes sketch + libs into `project_dir`, runs cmake + ninja, merges binaries. Caller is responsible for @@ -1774,6 +1828,7 @@ class ESPIDFCompiler: component_names, _ = self._resolve_library_components( ext_headers, arduino_libs, esp32_libs, arduino_comp_name, user_libs_dir, + allowed_libraries=allowed_libraries, ) # Patch main/CMakeLists.txt — REQUIRES and INCLUDE_DIRS for user_libs_all. diff --git a/test/backend/unit/test_espidf_core_first.py b/test/backend/unit/test_espidf_core_first.py index 2d06d878..9a01ac4e 100644 --- a/test/backend/unit/test_espidf_core_first.py +++ b/test/backend/unit/test_espidf_core_first.py @@ -88,5 +88,56 @@ class TestCoreFirstResolution(unittest.TestCase): self.assertNotIn("Foo.h", hdr2comp) +class TestManifestScope(unittest.TestCase): + """P2: when a project declares a library manifest, only declared libraries + are merged — a user-installed lib outside the manifest is never picked up, + even if its header is included. None manifest = legacy scan-all (unchanged).""" + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + self.ulibs = self.tmp / "Arduino" / "libraries" + # In-manifest lib, declared by Library Manager display name; on-disk + # folder name differs by separators/case (the realistic shape). + _mk(self.ulibs / "DHT_sensor_library" / "library.properties", + "name=DHT sensor library\n") + _mk(self.ulibs / "DHT_sensor_library" / "DHT.h") + _mk(self.ulibs / "DHT_sensor_library" / "DHT.cpp") + # Out-of-manifest lib (e.g. another user's install / a clash). + _mk(self.ulibs / "RandomOtherLib" / "Foo.h") + _mk(self.ulibs / "RandomOtherLib" / "Foo.cpp") + self.c = ESPIDFCompiler() + self.c.arduino_path = "" + self.c._core_headers_cache = None + self.out = self.tmp / "project" / "user_libs" + self.out.mkdir(parents=True) + + def tearDown(self) -> None: + shutil.rmtree(self.tmp, ignore_errors=True) + + def _resolve(self, headers, allowed): + return self.c._resolve_library_components( + headers, arduino_libs=self.ulibs, esp32_libs=None, + arduino_comp_name="arduino-esp32", user_libs_dir=self.out, + allowed_libraries=allowed, + ) + + def test_only_manifest_libs_merge(self): + # Manifest declares DHT (by display name) but not Foo's lib. + _, hdr2comp = self._resolve(["DHT.h", "Foo.h"], {"DHT sensor library"}) + self.assertEqual(hdr2comp.get("DHT.h"), "user_libs_all") # declared → merged + self.assertNotIn("Foo.h", hdr2comp) # undeclared → dropped + + def test_none_manifest_is_scan_all(self): + # No manifest → legacy behaviour: both resolve. + _, hdr2comp = self._resolve(["DHT.h", "Foo.h"], None) + self.assertEqual(hdr2comp.get("DHT.h"), "user_libs_all") + self.assertEqual(hdr2comp.get("Foo.h"), "user_libs_all") + + def test_match_by_folder_name(self): + # Manifest may also reference the on-disk folder name directly. + _, hdr2comp = self._resolve(["Foo.h"], {"RandomOtherLib"}) + self.assertEqual(hdr2comp.get("Foo.h"), "user_libs_all") + + if __name__ == "__main__": unittest.main()