espidf: preprocessor-aware include scanning (root cause of foreign libraries in user_libs_all)
Reported 2026-08-03: every ESP32 build in a user's project failed with errors from libraries he never installed, emitted from user_libs_all -- a folder his file list does not even show, in projects he had recreated from scratch. Root cause: _detect_external_includes was a bare regex with no preprocessor state. ESPAsyncWebServer guards <Hash.h> behind '#if defined(ESP8266) || defined(TARGET_RP2040)', so the compiler never sees it on ESP32 -- but the scanner reported it anyway. The resolver then searched the ~1000 shared-cache libraries, matched stemi-hexapod's src/Hash.h (a hexapod-robot library), merged it WHOLE, and compiled all of its sources; each failed on its own unrelated includes. The scanner now tracks #if/#ifdef/#ifndef/#elif/#else/#endif nesting and strips comments. A branch is skipped ONLY when every identifier in its condition is a platform macro known to be undefined on ESP32; unknown macros keep the branch LIVE, because hiding a genuinely needed header would be a much worse failure than the old over-reporting. Verified against the real ESPAsyncWebServer shape plus nesting/elif/else/comment cases (9 new tests, 15 green in the espidf suites). Complements the platform-header guard from the previous commit, which stays as defence in depth.
This commit is contained in:
David Montero
parent
7b5d30791c
commit
0c65de16ad
|
|
@ -1137,10 +1137,104 @@ class ESPIDFCompiler:
|
|||
)
|
||||
return ['user_libs_all'], header_to_comp
|
||||
|
||||
# Platform macros we can decide with certainty when scanning a source
|
||||
# for #include directives. Everything NOT listed here is unknown, and an
|
||||
# expression mentioning an unknown identifier is treated as LIVE — the
|
||||
# scanner must never hide a header the compiler will really need.
|
||||
_PP_TRUE = frozenset({
|
||||
'ESP32', 'ESP_PLATFORM', 'ARDUINO_ARCH_ESP32',
|
||||
})
|
||||
_PP_FALSE = frozenset({
|
||||
'ESP8266', 'ARDUINO_ARCH_ESP8266', '__AVR__', 'ARDUINO_ARCH_AVR',
|
||||
'ARDUINO_ARCH_SAMD', 'ARDUINO_ARCH_SAM', 'ARDUINO_ARCH_STM32',
|
||||
'ARDUINO_ARCH_RENESAS', 'ARDUINO_ARCH_RP2040', 'ARDUINO_ARCH_MBED',
|
||||
'TARGET_RP2040', 'TARGET_RP2350', 'PICO_RP2040', 'PICO_RP2350',
|
||||
'CORE_TEENSY', '__MBED__', '__SAMD51__', '__SAM3X8E__',
|
||||
'ARDUINO_ARCH_NRF52', 'NRF52', 'TARGET_ESP8266',
|
||||
})
|
||||
_PP_IDENT_RE = re.compile(r'[A-Za-z_][A-Za-z_0-9]*')
|
||||
_PP_COMMENT_BLOCK_RE = re.compile(r'/\*.*?\*/', re.DOTALL)
|
||||
_PP_COMMENT_LINE_RE = re.compile(r'//[^\n]*')
|
||||
|
||||
def _pp_branch_is_live(self, expr: str) -> bool:
|
||||
"""Best-effort evaluation of a #if / #elif condition.
|
||||
|
||||
Returns False ONLY when every identifier in the expression is a
|
||||
platform macro we know is undefined on ESP32 (so the branch is
|
||||
provably dead here). Anything else — unknown macros, arithmetic,
|
||||
version checks — is reported LIVE. Being wrong in the LIVE
|
||||
direction only costs us the old behaviour; being wrong the other
|
||||
way would hide a genuinely needed library.
|
||||
"""
|
||||
idents = [
|
||||
i for i in self._PP_IDENT_RE.findall(expr)
|
||||
if i not in ('defined', 'ifdef', 'ifndef')
|
||||
]
|
||||
if not idents:
|
||||
return True
|
||||
if any(i in self._PP_TRUE for i in idents):
|
||||
return True
|
||||
# Dead only when we recognise EVERY identifier as false-on-ESP32.
|
||||
return not all(i in self._PP_FALSE for i in idents)
|
||||
|
||||
def _detect_external_includes(self, code: str) -> list[str]:
|
||||
"""Return library header names that are likely from external libraries."""
|
||||
headers = []
|
||||
for m in re.finditer(r'#\s*include\s*<([^>]+)>', code):
|
||||
"""Library header names an ESP32 build could really need.
|
||||
|
||||
Preprocessor-aware: includes that only exist inside a branch which
|
||||
is provably dead on ESP32 are skipped. Without this, the scanner
|
||||
reported headers the compiler never sees, and each one was resolved
|
||||
against the ~1000 installed libraries — dragging an unrelated
|
||||
library (and ALL of its sources) into user_libs_all.
|
||||
|
||||
The canonical break, reported 2026-08-03: ESPAsyncWebServer has
|
||||
`#if defined(ESP8266) ... #include <Hash.h> ... #endif`. Hash.h was
|
||||
resolved to stemi-hexapod's src/Hash.h (a hexapod-robot library),
|
||||
whose Serial.cpp / Server.cpp / BluetoothLowEnergy.cpp then failed
|
||||
on their own missing includes, breaking every ESP32 build in that
|
||||
project with errors naming libraries the user never installed.
|
||||
"""
|
||||
# Comments first: a commented-out include is not an include.
|
||||
code = self._PP_COMMENT_BLOCK_RE.sub(' ', code)
|
||||
code = self._PP_COMMENT_LINE_RE.sub(' ', code)
|
||||
|
||||
headers: list[str] = []
|
||||
# Stack of (this_branch_live, any_branch_taken_yet) per #if nesting.
|
||||
stack: list[tuple[bool, bool]] = []
|
||||
for raw in code.splitlines():
|
||||
line = raw.strip()
|
||||
if line.startswith('#'):
|
||||
dtv = line[1:].lstrip()
|
||||
if dtv.startswith(('ifdef', 'ifndef', 'if ', 'if(')):
|
||||
if dtv.startswith('ifdef'):
|
||||
live = self._pp_branch_is_live(dtv[5:])
|
||||
elif dtv.startswith('ifndef'):
|
||||
# #ifndef X is live unless X is known-defined here.
|
||||
idents = self._PP_IDENT_RE.findall(dtv[6:])
|
||||
live = not any(i in self._PP_TRUE for i in idents)
|
||||
else:
|
||||
live = self._pp_branch_is_live(dtv[2:])
|
||||
stack.append((live, live))
|
||||
continue
|
||||
if dtv.startswith('elif'):
|
||||
if stack:
|
||||
_cur, taken = stack[-1]
|
||||
live = (not taken) and self._pp_branch_is_live(dtv[4:])
|
||||
stack[-1] = (live, taken or live)
|
||||
continue
|
||||
if dtv.startswith('else'):
|
||||
if stack:
|
||||
_cur, taken = stack[-1]
|
||||
stack[-1] = (not taken, True)
|
||||
continue
|
||||
if dtv.startswith('endif'):
|
||||
if stack:
|
||||
stack.pop()
|
||||
continue
|
||||
if not all(live for live, _ in stack):
|
||||
continue # inside a branch that is dead on ESP32
|
||||
m = re.search(r'#\s*include\s*<([^>]+)>', line)
|
||||
if not m:
|
||||
continue
|
||||
h = m.group(1)
|
||||
if h in self._BUILTIN_HEADERS:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
"""Preprocessor-aware include scanning for the ESP-IDF library resolver.
|
||||
|
||||
Reported 2026-08-03: every ESP32 build in a user's project failed with
|
||||
errors from libraries he never installed (SoftwareSerial.h,
|
||||
ArduinoBearSSL.h, Adafruit_GFX.h, AsyncElegantOTA.h) coming out of
|
||||
user_libs_all.
|
||||
|
||||
Chain: ESPAsyncWebServer guards `#include <Hash.h>` behind
|
||||
`#if defined(ESP8266) || defined(TARGET_RP2040)`. The old scanner was a
|
||||
bare regex with no preprocessor state, so it reported Hash.h anyway; the
|
||||
resolver then searched ~1000 installed libraries, found stemi-hexapod's
|
||||
src/Hash.h, merged that hexapod-robot library whole, and compiled all of
|
||||
its sources.
|
||||
|
||||
Design rule under test: a branch is skipped ONLY when every identifier in
|
||||
its condition is a platform macro known to be undefined on ESP32.
|
||||
Anything unknown stays LIVE — hiding a genuinely needed header would be a
|
||||
far worse failure than the old over-reporting.
|
||||
|
||||
No toolchain required: pure scanning logic.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / 'backend'))
|
||||
|
||||
from app.services.espidf_compiler import ESPIDFCompiler
|
||||
|
||||
|
||||
class TestPreprocessorAwareIncludeScan(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.scan = ESPIDFCompiler()._detect_external_includes
|
||||
|
||||
def test_real_espasyncwebserver_shape(self):
|
||||
"""The exact reported break."""
|
||||
got = self.scan(
|
||||
'#if defined(ESP8266) || defined(TARGET_RP2040)\n'
|
||||
'#include <Hash.h>\n'
|
||||
'#include <ESPAsyncTCP.h>\n'
|
||||
'#else\n'
|
||||
'#include <AsyncTCP.h>\n'
|
||||
'#endif\n'
|
||||
'#include <ArduinoJson.h>\n'
|
||||
)
|
||||
self.assertNotIn('Hash.h', got)
|
||||
self.assertNotIn('ESPAsyncTCP.h', got)
|
||||
self.assertIn('AsyncTCP.h', got)
|
||||
self.assertIn('ArduinoJson.h', got)
|
||||
|
||||
def test_unknown_macro_stays_live(self):
|
||||
"""The safety property: we must never hide a header we cannot rule
|
||||
out. A project-specific macro is not a platform macro."""
|
||||
self.assertIn('MyLib.h', self.scan(
|
||||
'#if USE_CUSTOM_THING\n#include <MyLib.h>\n#endif'))
|
||||
self.assertIn('Other.h', self.scan(
|
||||
'#if LIB_VERSION > 2\n#include <Other.h>\n#endif'))
|
||||
|
||||
def test_esp32_branch_lives_and_foreign_branch_dies(self):
|
||||
got = self.scan(
|
||||
'#ifdef ESP32\n#include <WiFi.h>\n#endif\n'
|
||||
'#ifdef __AVR__\n#include <SoftwareSerial.h>\n#endif')
|
||||
self.assertIn('WiFi.h', got)
|
||||
self.assertNotIn('SoftwareSerial.h', got)
|
||||
|
||||
def test_else_of_a_dead_branch_is_live(self):
|
||||
self.assertEqual(
|
||||
self.scan('#ifdef ESP8266\n#include <Dead.h>\n'
|
||||
'#else\n#include <Alive.h>\n#endif'),
|
||||
['Alive.h'])
|
||||
|
||||
def test_elif_selects_the_esp32_arm(self):
|
||||
self.assertEqual(
|
||||
self.scan('#if defined(ESP8266)\n#include <A.h>\n'
|
||||
'#elif defined(ESP32)\n#include <B.h>\n'
|
||||
'#else\n#include <C.h>\n#endif'),
|
||||
['B.h'])
|
||||
|
||||
def test_nested_dead_block(self):
|
||||
self.assertEqual(
|
||||
self.scan('#ifdef ESP8266\n#ifdef FOO\n#include <Deep.h>\n'
|
||||
'#endif\n#endif\n#include <Top.h>'),
|
||||
['Top.h'])
|
||||
|
||||
def test_comments_are_not_includes(self):
|
||||
self.assertEqual(
|
||||
self.scan('// #include <Commented.h>\n'
|
||||
'/* #include <Block.h> */\n'
|
||||
'#include <Real.h>'),
|
||||
['Real.h'])
|
||||
|
||||
def test_plain_sketch_unchanged(self):
|
||||
got = self.scan('#include <ESPAsyncWebServer.h>\n'
|
||||
'#include <ESP32Servo.h>\n'
|
||||
'#include <WiFi.h>')
|
||||
self.assertIn('ESPAsyncWebServer.h', got)
|
||||
self.assertIn('ESP32Servo.h', got)
|
||||
|
||||
def test_idf_internal_and_pathed_headers_still_skipped(self):
|
||||
got = self.scan('#include <freertos/FreeRTOS.h>\n'
|
||||
'#include <esp_wifi.h>\n'
|
||||
'#include <DHT.h>')
|
||||
self.assertEqual(got, ['DHT.h'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue