diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index 2dfe0134..6d7f1cd3 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -1137,10 +1137,104 @@ class ESPIDFCompiler: ) return ['user_libs_all'], header_to_comp + # Platform macros we can decide with certainty when scanning a source + # for #include directives. Everything NOT listed here is unknown, and an + # expression mentioning an unknown identifier is treated as LIVE — the + # scanner must never hide a header the compiler will really need. + _PP_TRUE = frozenset({ + 'ESP32', 'ESP_PLATFORM', 'ARDUINO_ARCH_ESP32', + }) + _PP_FALSE = frozenset({ + 'ESP8266', 'ARDUINO_ARCH_ESP8266', '__AVR__', 'ARDUINO_ARCH_AVR', + 'ARDUINO_ARCH_SAMD', 'ARDUINO_ARCH_SAM', 'ARDUINO_ARCH_STM32', + 'ARDUINO_ARCH_RENESAS', 'ARDUINO_ARCH_RP2040', 'ARDUINO_ARCH_MBED', + 'TARGET_RP2040', 'TARGET_RP2350', 'PICO_RP2040', 'PICO_RP2350', + 'CORE_TEENSY', '__MBED__', '__SAMD51__', '__SAM3X8E__', + 'ARDUINO_ARCH_NRF52', 'NRF52', 'TARGET_ESP8266', + }) + _PP_IDENT_RE = re.compile(r'[A-Za-z_][A-Za-z_0-9]*') + _PP_COMMENT_BLOCK_RE = re.compile(r'/\*.*?\*/', re.DOTALL) + _PP_COMMENT_LINE_RE = re.compile(r'//[^\n]*') + + def _pp_branch_is_live(self, expr: str) -> bool: + """Best-effort evaluation of a #if / #elif condition. + + Returns False ONLY when every identifier in the expression is a + platform macro we know is undefined on ESP32 (so the branch is + provably dead here). Anything else — unknown macros, arithmetic, + version checks — is reported LIVE. Being wrong in the LIVE + direction only costs us the old behaviour; being wrong the other + way would hide a genuinely needed library. + """ + idents = [ + i for i in self._PP_IDENT_RE.findall(expr) + if i not in ('defined', 'ifdef', 'ifndef') + ] + if not idents: + return True + if any(i in self._PP_TRUE for i in idents): + return True + # Dead only when we recognise EVERY identifier as false-on-ESP32. + return not all(i in self._PP_FALSE for i in idents) + def _detect_external_includes(self, code: str) -> list[str]: - """Return library header names that are likely from external libraries.""" - headers = [] - for m in re.finditer(r'#\s*include\s*<([^>]+)>', code): + """Library header names an ESP32 build could really need. + + Preprocessor-aware: includes that only exist inside a branch which + is provably dead on ESP32 are skipped. Without this, the scanner + reported headers the compiler never sees, and each one was resolved + against the ~1000 installed libraries — dragging an unrelated + library (and ALL of its sources) into user_libs_all. + + The canonical break, reported 2026-08-03: ESPAsyncWebServer has + `#if defined(ESP8266) ... #include ... #endif`. Hash.h was + resolved to stemi-hexapod's src/Hash.h (a hexapod-robot library), + whose Serial.cpp / Server.cpp / BluetoothLowEnergy.cpp then failed + on their own missing includes, breaking every ESP32 build in that + project with errors naming libraries the user never installed. + """ + # Comments first: a commented-out include is not an include. + code = self._PP_COMMENT_BLOCK_RE.sub(' ', code) + code = self._PP_COMMENT_LINE_RE.sub(' ', code) + + headers: list[str] = [] + # Stack of (this_branch_live, any_branch_taken_yet) per #if nesting. + stack: list[tuple[bool, bool]] = [] + for raw in code.splitlines(): + line = raw.strip() + if line.startswith('#'): + dtv = line[1:].lstrip() + if dtv.startswith(('ifdef', 'ifndef', 'if ', 'if(')): + if dtv.startswith('ifdef'): + live = self._pp_branch_is_live(dtv[5:]) + elif dtv.startswith('ifndef'): + # #ifndef X is live unless X is known-defined here. + idents = self._PP_IDENT_RE.findall(dtv[6:]) + live = not any(i in self._PP_TRUE for i in idents) + else: + live = self._pp_branch_is_live(dtv[2:]) + stack.append((live, live)) + continue + if dtv.startswith('elif'): + if stack: + _cur, taken = stack[-1] + live = (not taken) and self._pp_branch_is_live(dtv[4:]) + stack[-1] = (live, taken or live) + continue + if dtv.startswith('else'): + if stack: + _cur, taken = stack[-1] + stack[-1] = (not taken, True) + continue + if dtv.startswith('endif'): + if stack: + stack.pop() + continue + if not all(live for live, _ in stack): + continue # inside a branch that is dead on ESP32 + m = re.search(r'#\s*include\s*<([^>]+)>', line) + if not m: + continue h = m.group(1) if h in self._BUILTIN_HEADERS: continue diff --git a/test/backend/unit/test_espidf_include_scanner.py b/test/backend/unit/test_espidf_include_scanner.py new file mode 100644 index 00000000..8cc50786 --- /dev/null +++ b/test/backend/unit/test_espidf_include_scanner.py @@ -0,0 +1,108 @@ +"""Preprocessor-aware include scanning for the ESP-IDF library resolver. + +Reported 2026-08-03: every ESP32 build in a user's project failed with +errors from libraries he never installed (SoftwareSerial.h, +ArduinoBearSSL.h, Adafruit_GFX.h, AsyncElegantOTA.h) coming out of +user_libs_all. + +Chain: ESPAsyncWebServer guards `#include ` behind +`#if defined(ESP8266) || defined(TARGET_RP2040)`. The old scanner was a +bare regex with no preprocessor state, so it reported Hash.h anyway; the +resolver then searched ~1000 installed libraries, found stemi-hexapod's +src/Hash.h, merged that hexapod-robot library whole, and compiled all of +its sources. + +Design rule under test: a branch is skipped ONLY when every identifier in +its condition is a platform macro known to be undefined on ESP32. +Anything unknown stays LIVE — hiding a genuinely needed header would be a +far worse failure than the old over-reporting. + +No toolchain required: pure scanning logic. +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / 'backend')) + +from app.services.espidf_compiler import ESPIDFCompiler + + +class TestPreprocessorAwareIncludeScan(unittest.TestCase): + def setUp(self) -> None: + self.scan = ESPIDFCompiler()._detect_external_includes + + def test_real_espasyncwebserver_shape(self): + """The exact reported break.""" + got = self.scan( + '#if defined(ESP8266) || defined(TARGET_RP2040)\n' + '#include \n' + '#include \n' + '#else\n' + '#include \n' + '#endif\n' + '#include \n' + ) + self.assertNotIn('Hash.h', got) + self.assertNotIn('ESPAsyncTCP.h', got) + self.assertIn('AsyncTCP.h', got) + self.assertIn('ArduinoJson.h', got) + + def test_unknown_macro_stays_live(self): + """The safety property: we must never hide a header we cannot rule + out. A project-specific macro is not a platform macro.""" + self.assertIn('MyLib.h', self.scan( + '#if USE_CUSTOM_THING\n#include \n#endif')) + self.assertIn('Other.h', self.scan( + '#if LIB_VERSION > 2\n#include \n#endif')) + + def test_esp32_branch_lives_and_foreign_branch_dies(self): + got = self.scan( + '#ifdef ESP32\n#include \n#endif\n' + '#ifdef __AVR__\n#include \n#endif') + self.assertIn('WiFi.h', got) + self.assertNotIn('SoftwareSerial.h', got) + + def test_else_of_a_dead_branch_is_live(self): + self.assertEqual( + self.scan('#ifdef ESP8266\n#include \n' + '#else\n#include \n#endif'), + ['Alive.h']) + + def test_elif_selects_the_esp32_arm(self): + self.assertEqual( + self.scan('#if defined(ESP8266)\n#include \n' + '#elif defined(ESP32)\n#include \n' + '#else\n#include \n#endif'), + ['B.h']) + + def test_nested_dead_block(self): + self.assertEqual( + self.scan('#ifdef ESP8266\n#ifdef FOO\n#include \n' + '#endif\n#endif\n#include '), + ['Top.h']) + + def test_comments_are_not_includes(self): + self.assertEqual( + self.scan('// #include \n' + '/* #include */\n' + '#include '), + ['Real.h']) + + def test_plain_sketch_unchanged(self): + got = self.scan('#include \n' + '#include \n' + '#include ') + self.assertIn('ESPAsyncWebServer.h', got) + self.assertIn('ESP32Servo.h', got) + + def test_idf_internal_and_pathed_headers_still_skipped(self): + got = self.scan('#include \n' + '#include \n' + '#include ') + self.assertEqual(got, ['DHT.h']) + + +if __name__ == '__main__': + unittest.main()