From 9761aad0be3452b8e81a6d8c992186138fbf3760 Mon Sep 17 00:00:00 2001 From: David Montero Crespo Date: Tue, 7 Apr 2026 14:59:51 -0300 Subject: [PATCH] feat: enhance Arduino library handling by detecting external libraries and creating IDF components, add tests for library resolution logic --- .../services/esp-idf-template/CMakeLists.txt | 5 + backend/app/services/espidf_compiler.py | 291 +++++++++++++++- backend/test_espidf_compiler.py | 326 ++++++++++++++++++ .../esp32-external-library-compilation.md | 264 ++++++++++++++ .../components/editor/CompilationConsole.tsx | 10 + frontend/src/utils/compilationLogger.ts | 30 +- 6 files changed, 920 insertions(+), 6 deletions(-) create mode 100644 backend/test_espidf_compiler.py create mode 100644 docs/wiki/esp32-external-library-compilation.md diff --git a/backend/app/services/esp-idf-template/CMakeLists.txt b/backend/app/services/esp-idf-template/CMakeLists.txt index d7340135..64be6a19 100644 --- a/backend/app/services/esp-idf-template/CMakeLists.txt +++ b/backend/app/services/esp-idf-template/CMakeLists.txt @@ -5,5 +5,10 @@ if(DEFINED ENV{ARDUINO_ESP32_PATH}) set(EXTRA_COMPONENT_DIRS $ENV{ARDUINO_ESP32_PATH}) endif() +# User libraries — each subdirectory is a proper IDF component generated at build time +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/user_libs") + list(APPEND EXTRA_COMPONENT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/user_libs") +endif() + include($ENV{IDF_PATH}/tools/cmake/project.cmake) project(velxio-sketch) diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index 3afa7c3d..0a074dea 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -335,6 +335,149 @@ class ESPIDFCompiler: return '\n'.join(lines) + '\n' + def _find_arduino_libraries_dir(self) -> Path | None: + """Find the Arduino global user-libraries directory (installed via arduino-cli).""" + candidates = [ + Path.home() / 'Arduino' / 'libraries', + Path.home() / 'Documents' / 'Arduino' / 'libraries', + Path('/root/Arduino/libraries'), # Docker / CI as root + Path('/home/user/Arduino/libraries'), + Path('/Arduino/libraries'), + ] + # Also check arduino-cli's data directory + for base in [ + Path.home() / '.arduino15', + Path('/root/.arduino15'), + Path('/home/user/.arduino15'), + ]: + candidates.append(base / 'libraries') + + for c in candidates: + if c.is_dir(): + logger.info(f'[espidf] Arduino libraries dir: {c}') + return c + logger.warning('[espidf] Arduino libraries dir not found') + return None + + # Built-in headers that do NOT require external library source + _BUILTIN_HEADERS = frozenset({ + 'Arduino.h', 'Wire.h', 'SPI.h', 'SPI.H', 'WiFi.h', 'EEPROM.h', + 'SD.h', 'Servo.h', 'LiquidCrystal.h', 'Ethernet.h', 'IPAddress.h', + 'HardwareSerial.h', 'Stream.h', 'Print.h', 'WString.h', 'pgmspace.h', + 'math.h', 'stdint.h', 'stdio.h', 'stdlib.h', 'string.h', 'stdarg.h', + 'WebServer.h', 'HTTPClient.h', 'WiFiClient.h', 'WiFiServer.h', + 'BluetoothSerial.h', 'BLEDevice.h', + 'FS.h', 'SPIFFS.h', 'LittleFS.h', + 'esp_system.h', 'esp_wifi.h', 'esp_event.h', 'nvs_flash.h', + 'freertos/FreeRTOS.h', 'freertos/task.h', + 'Adafruit_GFX.h', # bundled with arduino-esp32 sometimes + }) + + def _detect_external_includes(self, code: str) -> list[str]: + """Return library header names that are likely from external libraries.""" + headers = [] + for m in re.finditer(r'#\s*include\s*<([^>]+)>', code): + h = m.group(1) + if h in self._BUILTIN_HEADERS: + continue + # Skip paths with / (esp-idf internal headers like freertos/FreeRTOS.h) + if '/' in h: + continue + # Skip headers that look like esp-idf internal (prefix pattern) + if re.match(r'^(esp_|driver/|soc/|hal/|nvs|rom/)', h): + continue + headers.append(h) + return headers + + def _find_library_for_header(self, header: str, libs_dir: Path) -> Path | None: + """ + Search libs_dir for a library that provides `header`. + Returns the source root of the library (root or src/ subdirectory). + """ + for lib_dir in sorted(libs_dir.iterdir()): + if not lib_dir.is_dir(): + continue + for src_root in [lib_dir, lib_dir / 'src']: + if (src_root / header).exists(): + return src_root + return None + + def _create_idf_component( + self, + header: str, + src_root: Path, + user_libs_dir: Path, + arduino_comp_name: str, + ) -> str: + """ + Create a proper ESP-IDF component for a library in user_libs_dir. + + Each library becomes user_libs// with its own CMakeLists.txt + that calls idf_component_register(). This is the correct ESP-IDF way to + include third-party code and properly handles include paths so that + internal library includes like #include "utility/xyz.h" work correctly. + + Returns the component directory name (used in REQUIRES of main). + """ + # Sanitise name: use the library directory name, not the header name + # src_root may be the library root OR lib/src/ — handle both cases + lib_dir_name = src_root.parent.name if src_root.name == 'src' else src_root.name + safe_name = re.sub(r'[^A-Za-z0-9_]', '_', lib_dir_name) + comp_dir = user_libs_dir / safe_name + comp_dir.mkdir(parents=True, exist_ok=True) + + # Collect all source files — preserve subdirectory structure via INCLUDE_DIRS + # We copy files flat into the component root but add src/ as an include dir + cpp_files: list[str] = [] + seen_names: set[str] = set() + + for pattern in ('*.h', '*.cpp', '*.c', 'src/*.h', 'src/*.cpp', 'src/*.c'): + for f in src_root.parent.glob(pattern) if pattern.startswith('src/') else src_root.glob(pattern): + if not f.is_file(): + continue + dest = comp_dir / f.name + if f.name not in seen_names: + shutil.copy2(f, dest) + seen_names.add(f.name) + if f.suffix in ('.cpp', '.c') and f.name not in cpp_files: + cpp_files.append(f.name) + + # Also copy from src/ subdirectory if present (e.g. Adafruit libraries) + src_sub = src_root / 'src' if (src_root / 'src').is_dir() else None + if src_sub is None and (src_root.parent / 'src').is_dir(): + src_sub = src_root.parent / 'src' + if src_sub: + for f in src_sub.glob('**/*'): + if not f.is_file() or f.suffix not in ('.h', '.cpp', '.c'): + continue + if f.name not in seen_names: + shutil.copy2(f, comp_dir / f.name) + seen_names.add(f.name) + if f.suffix in ('.cpp', '.c') and f.name not in cpp_files: + cpp_files.append(f.name) + + # Generate CMakeLists.txt for this component + if cpp_files: + srcs_line = 'SRCS ' + ' '.join(f'"{f}"' for f in sorted(cpp_files)) + else: + srcs_line = '# header-only library' + + cmake_content = ( + f'# Auto-generated by Velxio for library: {lib_dir_name}\n' + f'idf_component_register(\n' + f' {srcs_line}\n' + f' INCLUDE_DIRS "."\n' + f' REQUIRES {arduino_comp_name}\n' + f')\n' + ) + (comp_dir / 'CMakeLists.txt').write_text(cmake_content, encoding='utf-8') + + logger.info( + f'[espidf] Created IDF component "{safe_name}" for <{header}>' + f' ({len(cpp_files)} source file(s))' + ) + return safe_name + def _build_env(self, idf_target: str) -> dict: """Build environment dict for ESP-IDF subprocess.""" env = os.environ.copy() @@ -506,6 +649,106 @@ class ESPIDFCompiler: sketch_translated = project_dir / 'main' / 'sketch_translated.c' if sketch_translated.exists(): sketch_translated.unlink() + + # ── Resolve external Arduino libraries as IDF components ────── + # arduino-cli installs libraries in ~/Arduino/libraries/ but the + # ESP-IDF build system does not scan that path. We create a + # user_libs/ directory where each external library becomes a + # proper ESP-IDF component with its own CMakeLists.txt and + # INCLUDE_DIRS. The root CMakeLists.txt (template) adds user_libs + # to EXTRA_COMPONENT_DIRS so ESP-IDF discovers them automatically. + ext_headers = self._detect_external_includes(main_content) + component_names: list[str] = [] + # arduino-esp32 component name (directory basename of ARDUINO_ESP32_PATH) + arduino_comp_name = Path(self.arduino_path).name if self.arduino_path else 'arduino-esp32' + + if ext_headers: + user_libs_dir = project_dir / 'user_libs' + user_libs_dir.mkdir(exist_ok=True) + + # Search order: esp32-bundled libraries first, then user-installed + esp32_libs = Path(self.arduino_path) / 'libraries' if self.arduino_path else None + arduino_libs = self._find_arduino_libraries_dir() + search_bases = [b for b in [esp32_libs, arduino_libs] if b and b.is_dir()] + + # Phase 1: BFS to discover all needed libraries including transitives + headers_to_resolve: list[str] = list(ext_headers) + resolved_headers: set[str] = set() + header_to_comp: dict[str, str] = {} # header → component name + + while headers_to_resolve: + header = headers_to_resolve.pop(0) + if header in resolved_headers: + continue + resolved_headers.add(header) + + src_root = None + for search_base in search_bases: + src_root = self._find_library_for_header(header, search_base) + if src_root: + break + + if src_root: + comp_name = self._create_idf_component( + header, src_root, user_libs_dir, arduino_comp_name + ) + if comp_name not in component_names: + component_names.append(comp_name) + header_to_comp[header] = comp_name + + # Scan copied library files for further external includes + comp_dir = user_libs_dir / comp_name + for lib_file in comp_dir.glob('*.h'): + try: + lib_content = lib_file.read_text(encoding='utf-8', errors='ignore') + for th in self._detect_external_includes(lib_content): + if th not in resolved_headers: + headers_to_resolve.append(th) + except OSError: + pass + else: + logger.warning(f'[espidf] Library for <{header}> not found — build may fail') + + # Phase 2: patch inter-component REQUIRES so each component can + # see the headers of libraries it transitively includes + for comp_name in component_names: + comp_dir = user_libs_dir / comp_name + extra_reqs: list[str] = [] + for lib_file in comp_dir.glob('*.h'): + try: + content = lib_file.read_text(encoding='utf-8', errors='ignore') + for dep_h in self._detect_external_includes(content): + dep_comp = header_to_comp.get(dep_h) + if dep_comp and dep_comp != comp_name and dep_comp not in extra_reqs: + extra_reqs.append(dep_comp) + except OSError: + pass + if extra_reqs: + cmake_path = comp_dir / 'CMakeLists.txt' + cmake_text = cmake_path.read_text(encoding='utf-8') + cmake_text = cmake_text.replace( + f'REQUIRES {arduino_comp_name}', + f'REQUIRES {arduino_comp_name} {" ".join(extra_reqs)}', + ) + cmake_path.write_text(cmake_text, encoding='utf-8') + logger.info(f'[espidf] {comp_name} REQUIRES += {extra_reqs}') + + # Patch main/CMakeLists.txt to REQUIRES the library components. + # The template uses CMake variable syntax: REQUIRES ${_arduino_comp_name} + if component_names: + cmake_path = project_dir / 'main' / 'CMakeLists.txt' + cmake_text = cmake_path.read_text(encoding='utf-8') + main_reqs = ' '.join(component_names) + # Replace both possible forms: CMake variable (template) or literal (pre-patched) + for old_req in [r'REQUIRES ${_arduino_comp_name}', f'REQUIRES {arduino_comp_name}']: + if old_req in cmake_text: + cmake_text = cmake_text.replace( + old_req, + f'{old_req} {main_reqs}', + ) + break + cmake_path.write_text(cmake_text, encoding='utf-8') + logger.info(f'[espidf] Added {len(component_names)} library component(s) to REQUIRES') else: # Pure ESP-IDF mode: translate sketch translated = self._translate_sketch_to_espidf(main_content) @@ -606,13 +849,57 @@ class ESPIDFCompiler: all_stderr = '\n'.join(filtered_stderr_lines) if ninja_result.returncode != 0: + # Extract the actual compiler errors from ninja's stdout. + # Ninja prints failed job blocks in stdout: + # FAILED: path/to/file.obj + # + # sketch.ino.cpp:5:10: fatal error: DHT.h: No such file or directory + # compilation terminated. + # ninja: build stopped: subcommand failed. + stdout_lines = ninja_result.stdout.split('\n') + error_lines: list[str] = [] + in_failed_block = False + for line in stdout_lines: + stripped = line.strip() + if stripped.startswith('FAILED:') or stripped == 'ninja: build stopped: subcommand failed.': + in_failed_block = True + error_lines.append(line) + continue + # Next [N/M] progress line ends the block + if in_failed_block and stripped.startswith('[') and '/' in stripped and ']' in stripped: + in_failed_block = False + if in_failed_block: + error_lines.append(line) + elif ': error:' in line or 'fatal error:' in line.lower(): + # Explicit compiler error outside a FAILED block + error_lines.append(line) + + extracted = '\n'.join(l for l in error_lines if l.strip()) + + # First non-FAILED, non-command error line → short summary for toolbar + summary = 'ESP-IDF build failed' + for l in error_lines: + s = l.strip() + if s and not s.startswith('FAILED:') and not s.startswith('ninja:') and not s.startswith('/') and 'error:' in s.lower(): + summary = s + break + if summary == 'ESP-IDF build failed' and error_lines: + # Fall back to first non-empty error line + for l in error_lines: + if l.strip() and not l.strip().startswith('FAILED:'): + summary = l.strip() + break + + # Put extracted errors in stderr so the console highlights them + combined_stderr = (extracted + '\n\n' + all_stderr).strip() if extracted else all_stderr + logger.error(f'[espidf] ninja build failed (stdout):\n{ninja_result.stdout[-4000:]}') logger.error(f'[espidf] ninja build failed (stderr):\n{ninja_result.stderr[-2000:]}') return { 'success': False, - 'error': 'ESP-IDF build failed', + 'error': summary, 'stdout': all_stdout, - 'stderr': all_stderr, + 'stderr': combined_stderr, } # Step 3: Merge binaries into flash image diff --git a/backend/test_espidf_compiler.py b/backend/test_espidf_compiler.py new file mode 100644 index 00000000..5b15b68a --- /dev/null +++ b/backend/test_espidf_compiler.py @@ -0,0 +1,326 @@ +""" +Tests for ESPIDFCompiler library resolution logic. + +Tests the methods that detect, locate, and package external Arduino libraries +as proper ESP-IDF components — without requiring the full ESP-IDF toolchain. + +Run from the backend/ directory: + python test_espidf_compiler.py +""" + +import sys +import tempfile +import shutil +import unittest +from pathlib import Path + +# Ensure backend/app is importable +sys.path.insert(0, str(Path(__file__).parent)) + +from app.services.espidf_compiler import ESPIDFCompiler + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +def make_compiler() -> ESPIDFCompiler: + """Create an ESPIDFCompiler instance without a real IDF path.""" + comp = ESPIDFCompiler.__new__(ESPIDFCompiler) + comp.idf_path = '' + comp.arduino_path = '' + comp.has_arduino = False + return comp + + +def make_library(libs_dir: Path, lib_name: str, headers: list[str], + sources: list[str], use_src_subdir: bool = False) -> Path: + """ + Create a mock Arduino library directory structure. + + libs_dir/ + {lib_name}/ + {lib_name}.h + {lib_name}.cpp + [src/] + ... + """ + lib_dir = libs_dir / lib_name + lib_dir.mkdir(parents=True) + src_dir = lib_dir / 'src' if use_src_subdir else lib_dir + + if use_src_subdir: + src_dir.mkdir() + + for h in headers: + (src_dir / h).write_text(f'// {h}', encoding='utf-8') + for s in sources: + (src_dir / s).write_text(f'// {s}', encoding='utf-8') + + return lib_dir + + +# ── Test: _detect_external_includes ────────────────────────────────────────── + +class TestDetectExternalIncludes(unittest.TestCase): + + def setUp(self): + self.comp = make_compiler() + + def test_detects_dht_header(self): + code = '#include \nvoid setup() {}' + result = self.comp._detect_external_includes(code) + self.assertIn('DHT.h', result) + + def test_skips_arduino_builtins(self): + code = '#include \n#include \n#include \n#include ' + result = self.comp._detect_external_includes(code) + self.assertEqual(result, []) + + def test_skips_esp_idf_headers(self): + code = '#include \n#include \n#include ' + result = self.comp._detect_external_includes(code) + self.assertEqual(result, []) + + def test_skips_path_headers(self): + # Headers with / are internal esp-idf or arduino-esp32 paths + code = '#include \n#include ' + result = self.comp._detect_external_includes(code) + self.assertEqual(result, []) + + def test_detects_multiple_external(self): + code = ( + '#include \n' + '#include \n' + '#include \n' + '#include \n' + ) + result = self.comp._detect_external_includes(code) + self.assertIn('DHT.h', result) + self.assertIn('Adafruit_Sensor.h', result) + self.assertNotIn('Arduino.h', result) + self.assertNotIn('Wire.h', result) + + def test_handles_whitespace_in_include(self): + code = '# include ' + result = self.comp._detect_external_includes(code) + self.assertIn('DHT.h', result) + + def test_no_duplicates_from_multiple_files(self): + code = '#include \n#include \n' + result = self.comp._detect_external_includes(code) + # List may have duplicates — the caller deduplicates; just check presence + self.assertIn('DHT.h', result) + + def test_empty_sketch(self): + result = self.comp._detect_external_includes('void setup() {} void loop() {}') + self.assertEqual(result, []) + + +# ── Test: _find_library_for_header ─────────────────────────────────────────── + +class TestFindLibraryForHeader(unittest.TestCase): + + def setUp(self): + self.comp = make_compiler() + self.tmp = tempfile.mkdtemp() + self.libs_dir = Path(self.tmp) + + def tearDown(self): + shutil.rmtree(self.tmp) + + def test_finds_library_in_root(self): + """Library with header in root (no src/ subdirectory).""" + make_library(self.libs_dir, 'DHT_sensor_library', + headers=['DHT.h', 'DHT_U.h'], sources=['DHT.cpp', 'DHT_U.cpp']) + result = self.comp._find_library_for_header('DHT.h', self.libs_dir) + assert result is not None + self.assertTrue((result / 'DHT.h').exists()) + + def test_finds_library_in_src_subdir(self): + """Library with header in src/ subdirectory (modern Arduino layout).""" + make_library(self.libs_dir, 'Adafruit_Sensor', + headers=['Adafruit_Sensor.h'], sources=['Adafruit_Sensor.cpp'], + use_src_subdir=True) + result = self.comp._find_library_for_header('Adafruit_Sensor.h', self.libs_dir) + assert result is not None + self.assertTrue((result / 'Adafruit_Sensor.h').exists()) + + def test_returns_none_for_missing_library(self): + make_library(self.libs_dir, 'SomeOtherLib', + headers=['Other.h'], sources=['Other.cpp']) + result = self.comp._find_library_for_header('DHT.h', self.libs_dir) + self.assertIsNone(result) + + def test_returns_none_for_empty_dir(self): + result = self.comp._find_library_for_header('DHT.h', self.libs_dir) + self.assertIsNone(result) + + +# ── Test: _create_idf_component ────────────────────────────────────────────── + +class TestCreateIdfComponent(unittest.TestCase): + + def setUp(self): + self.comp = make_compiler() + self.tmp = tempfile.mkdtemp() + self.libs_dir = Path(self.tmp) / 'arduino_libs' + self.libs_dir.mkdir() + self.user_libs_dir = Path(self.tmp) / 'user_libs' + self.user_libs_dir.mkdir() + + def tearDown(self): + shutil.rmtree(self.tmp) + + def _make_dht_library(self, use_src: bool = False) -> Path: + return make_library( + self.libs_dir, 'DHT_sensor_library', + headers=['DHT.h', 'DHT_U.h'], + sources=['DHT.cpp', 'DHT_U.cpp'], + use_src_subdir=use_src, + ) + + def test_creates_component_directory(self): + lib_dir = self._make_dht_library() + src_root = lib_dir # header is in root + self.comp._create_idf_component('DHT.h', src_root, self.user_libs_dir, 'arduino-esp32') + self.assertTrue(self.user_libs_dir.exists()) + comp_dirs = list(self.user_libs_dir.iterdir()) + self.assertEqual(len(comp_dirs), 1) + + def test_component_has_cmake_lists(self): + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32' + ) + cmake_path = self.user_libs_dir / comp_name / 'CMakeLists.txt' + self.assertTrue(cmake_path.exists(), 'CMakeLists.txt not found') + + def test_cmake_contains_idf_component_register(self): + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32' + ) + cmake_text = (self.user_libs_dir / comp_name / 'CMakeLists.txt').read_text() + self.assertIn('idf_component_register', cmake_text) + + def test_cmake_includes_cpp_source(self): + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32' + ) + cmake_text = (self.user_libs_dir / comp_name / 'CMakeLists.txt').read_text() + self.assertIn('DHT.cpp', cmake_text) + + def test_cmake_requires_arduino_component(self): + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32' + ) + cmake_text = (self.user_libs_dir / comp_name / 'CMakeLists.txt').read_text() + self.assertIn('arduino-esp32', cmake_text) + self.assertIn('REQUIRES', cmake_text) + + def test_cmake_sets_include_dirs(self): + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32' + ) + cmake_text = (self.user_libs_dir / comp_name / 'CMakeLists.txt').read_text() + self.assertIn('INCLUDE_DIRS', cmake_text) + self.assertIn('"."', cmake_text) + + def test_header_files_are_copied(self): + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32' + ) + comp_dir = self.user_libs_dir / comp_name + self.assertTrue((comp_dir / 'DHT.h').exists()) + self.assertTrue((comp_dir / 'DHT_U.h').exists()) + + def test_cpp_files_are_copied(self): + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32' + ) + comp_dir = self.user_libs_dir / comp_name + self.assertTrue((comp_dir / 'DHT.cpp').exists()) + self.assertTrue((comp_dir / 'DHT_U.cpp').exists()) + + def test_returns_sanitised_component_name(self): + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32' + ) + # Component name must be a valid C identifier (no spaces, no dots) + self.assertRegex(comp_name, r'^[A-Za-z0-9_]+$') + # Must be based on library directory name, not the parent search dir + self.assertEqual(comp_name, 'DHT_sensor_library') + + def test_custom_arduino_comp_name(self): + """arduino_comp_name is properly forwarded into REQUIRES.""" + lib_dir = self._make_dht_library() + comp_name = self.comp._create_idf_component( + 'DHT.h', lib_dir, self.user_libs_dir, 'arduino-esp32-custom' + ) + cmake_text = (self.user_libs_dir / comp_name / 'CMakeLists.txt').read_text() + self.assertIn('arduino-esp32-custom', cmake_text) + + def test_library_in_src_subdir(self): + """Library with modern src/ layout is handled correctly.""" + lib_dir = self._make_dht_library(use_src=True) + src_root = lib_dir / 'src' + comp_name = self.comp._create_idf_component( + 'DHT.h', src_root, self.user_libs_dir, 'arduino-esp32' + ) + comp_dir = self.user_libs_dir / comp_name + self.assertTrue((comp_dir / 'DHT.h').exists()) + self.assertTrue((comp_dir / 'DHT.cpp').exists()) + + +# ── Test: template CMakeLists.txt content ──────────────────────────────────── + +class TestTemplateCMakeLists(unittest.TestCase): + + def test_root_cmake_has_user_libs_block(self): + template_cmake = ( + Path(__file__).parent + / 'app' / 'services' / 'esp-idf-template' / 'CMakeLists.txt' + ) + self.assertTrue(template_cmake.exists(), 'Template CMakeLists.txt not found') + content = template_cmake.read_text(encoding='utf-8') + self.assertIn('user_libs', content, + 'user_libs not referenced in root CMakeLists.txt') + self.assertIn('EXTRA_COMPONENT_DIRS', content) + self.assertIn('EXISTS', content, + 'user_libs block should use EXISTS guard') + + def test_main_cmake_has_arduino_requires(self): + main_cmake = ( + Path(__file__).parent + / 'app' / 'services' / 'esp-idf-template' / 'main' / 'CMakeLists.txt' + ) + self.assertTrue(main_cmake.exists()) + content = main_cmake.read_text(encoding='utf-8') + self.assertIn('REQUIRES', content) + self.assertIn('_arduino_comp_name', content) + + +# ── Runner ──────────────────────────────────────────────────────────────────── + +if __name__ == '__main__': + print('=' * 60) + print('ESPIDFCompiler Library Resolution Tests') + print('=' * 60) + loader = unittest.TestLoader() + suite = unittest.TestSuite() + for cls in [ + TestDetectExternalIncludes, + TestFindLibraryForHeader, + TestCreateIdfComponent, + TestTemplateCMakeLists, + ]: + suite.addTests(loader.loadTestsFromTestCase(cls)) + + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/docs/wiki/esp32-external-library-compilation.md b/docs/wiki/esp32-external-library-compilation.md new file mode 100644 index 00000000..e4c0a783 --- /dev/null +++ b/docs/wiki/esp32-external-library-compilation.md @@ -0,0 +1,264 @@ +# ESP32 External Arduino Library Compilation — IDF Component Approach + +> **Scope**: This document covers the full investigation and implementation of automatic +> external Arduino library inclusion when compiling ESP32 sketches via ESP-IDF. +> Target audience: future maintainers who need to understand *why* the library resolution +> works the way it does and what bugs were discovered during real compilation testing. + +--- + +## Table of Contents + +1. [Problem Statement](#problem-statement) +2. [Root Cause](#root-cause) +3. [Solution Architecture](#solution-architecture) +4. [Implementation Details](#implementation-details) +5. [Bugs Found During Real Compilation](#bugs-found-during-real-compilation) +6. [Error Visibility Improvements](#error-visibility-improvements) +7. [Test Coverage](#test-coverage) +8. [Files Changed](#files-changed) + +--- + +## Problem Statement + +When compiling an ESP32 sketch that uses an external library (e.g. the DHT22 temperature +sensor example with `#include `), the build failed with: + +``` +fatal error: DHT.h: No such file or directory +``` + +For **Arduino UNO**, `arduino-cli` automatically scans `~/Arduino/libraries/` and adds +the correct `-I` flags. For **ESP32 via ESP-IDF**, that path is never scanned — ESP-IDF +only knows about components explicitly listed in `EXTRA_COMPONENT_DIRS`. + +Additionally, even when an error did occur, it was **invisible in the UI**: ninja build +errors go to **stdout** (not stderr), so the frontend was classifying them as grey `info` +lines instead of red `error` lines. + +--- + +## Root Cause + +ESP-IDF uses a CMake-based component system. Every unit of code must be a registered +**IDF component** with its own `CMakeLists.txt` calling `idf_component_register()`. +The `arduino-esp32` library itself is included this way. External Arduino libraries +(installed via `arduino-cli` Library Manager into `~/Arduino/libraries/`) have no +`idf_component_register()` and are therefore invisible to the build system. + +The naive fix of copying `.h`/`.cpp` files flat into `main/` breaks libraries that use +subdirectory structures for internal includes (e.g. `#include "utility/xyz.h"`). + +--- + +## Solution Architecture + +Each external Arduino library is wrapped as a proper ESP-IDF component: + +``` +project/ + CMakeLists.txt ← adds user_libs/ to EXTRA_COMPONENT_DIRS + user_libs/ + DHT_sensor_library/ + CMakeLists.txt ← idf_component_register(SRCS ... REQUIRES arduino-esp32) + DHT.h + DHT.cpp + DHT_U.h + DHT_U.cpp + Adafruit_Unified_Sensor/ ← transitive dependency, auto-discovered + CMakeLists.txt + Adafruit_Sensor.h + Adafruit_Sensor.cpp + main/ + CMakeLists.txt ← REQUIRES arduino-esp32 DHT_sensor_library Adafruit_Unified_Sensor + main.cpp + sketch.ino.cpp +``` + +ESP-IDF automatically compiles every subdirectory of `EXTRA_COMPONENT_DIRS` that contains +an `idf_component_register()` call. The `INCLUDE_DIRS "."` inside each component makes its +headers available to anything that `REQUIRES` it. + +### Library Search Order + +1. `$ARDUINO_ESP32_PATH/libraries/` — ESP32-native libs bundled with arduino-esp32 (WiFi, + BLE, EEPROM…) — already compiled as part of arduino-esp32 component, so usually skipped +2. `~/Documents/Arduino/libraries/` — user-installed libraries (Windows primary path) +3. `~/Arduino/libraries/` — alternate user path +4. `/root/Arduino/libraries/` — Docker / CI root user path + +--- + +## Implementation Details + +### `_detect_external_includes(code)` + +Scans source code for `#include ` directives and returns those that are NOT: +- Arduino/ESP32 built-ins (e.g. `Arduino.h`, `Wire.h`, `WiFi.h`, `esp_wifi.h`) +- Headers with `/` (ESP-IDF internal paths like `freertos/FreeRTOS.h`) +- ESP-IDF pattern prefixes (`esp_`, `driver/`, `soc/`, `hal/`, `nvs`, `rom/`) + +### `_find_library_for_header(header, libs_dir)` + +Iterates subdirectories of `libs_dir`, checking both the root and `src/` subdirectory +for the requested header. Returns the **source root** (either `lib_dir/` or `lib_dir/src/`). + +### `_create_idf_component(header, src_root, user_libs_dir, arduino_comp_name)` + +Creates `user_libs//` with: +- All `.h`, `.cpp`, `.c` files copied flat from `src_root` +- A generated `CMakeLists.txt`: + ```cmake + idf_component_register( + SRCS "DHT.cpp" "DHT_U.cpp" + INCLUDE_DIRS "." + REQUIRES arduino-esp32 + ) + ``` +- Returns the component directory name (e.g. `DHT_sensor_library`) + +### Transitive Dependency Resolution (BFS) + +The library resolution loop uses **breadth-first search**: + +1. **Phase 1 — BFS discovery**: Start with headers found in the user sketch. After creating + each component, scan its copied `.h` files for further external includes. Enqueue any + new headers not yet resolved. Repeat until the queue is empty. + A `header_to_comp` dict tracks which component provides each header. + +2. **Phase 2 — inter-component REQUIRES**: After all components are created, scan each + component's headers again. For any dependency that maps to another component in + `header_to_comp`, patch that component's `CMakeLists.txt` to add the dep to `REQUIRES`. + +This ensures `DHT_sensor_library/CMakeLists.txt` ends up with: +```cmake +REQUIRES arduino-esp32 Adafruit_Unified_Sensor +``` + +### Main `CMakeLists.txt` Patching + +The template `main/CMakeLists.txt` uses a CMake variable: +```cmake +REQUIRES ${_arduino_comp_name} +``` + +The Python patch looks for this exact string (not the resolved literal `arduino-esp32`) +and appends the user library component names: +```cmake +REQUIRES ${_arduino_comp_name} DHT_sensor_library Adafruit_Unified_Sensor +``` + +--- + +## Bugs Found During Real Compilation + +Three bugs were discovered when running an actual ESP32 compile (vs. unit tests with mocks): + +### Bug 1 — Wrong Component Name (`libraries` instead of `DHT_sensor_library`) + +**Symptom**: Build step showed `esp-idf/libraries/CMakeFiles/...` instead of +`esp-idf/DHT_sensor_library/CMakeFiles/...`. + +**Cause**: `_create_idf_component` used `src_root.parent.name` to get the library name. +When `_find_library_for_header` returns the library root (no `src/` subdir), `src_root` +*is* the library directory, so `.parent.name` gives `libraries` (the parent search dir). + +**Fix**: +```python +# Before +lib_dir_name = src_root.parent.name + +# After +lib_dir_name = src_root.parent.name if src_root.name == 'src' else src_root.name +``` + +### Bug 2 — Missing Transitive Dependency (`Adafruit_Sensor.h`) + +**Symptom**: After fixing Bug 1, the build failed with: +``` +DHT_U.h:36:10: fatal error: Adafruit_Sensor.h: No such file or directory +``` + +**Cause**: `DHT_U.h` (part of DHT library) `#include`s `Adafruit_Sensor.h` from the +`Adafruit_Unified_Sensor` library. The original code only scanned the user sketch for +external includes, not the library headers themselves. + +**Fix**: Added BFS transitive dependency resolution (Phase 1 + Phase 2 described above). + +### Bug 3 — CMake Template Variable Mismatch + +**Symptom**: After fixing Bugs 1 and 2, the `main` component still couldn't find `DHT.h`: +``` +sketch.ino.cpp:3:10: fatal error: DHT.h: No such file or directory +``` + +**Cause**: The Python patch looked for `REQUIRES arduino-esp32` (literal) in +`main/CMakeLists.txt`, but the template uses `REQUIRES ${_arduino_comp_name}` (CMake +variable). The replacement was silently a no-op. + +**Fix**: Updated the patch to match the CMake variable syntax: +```python +for old_req in [r'REQUIRES ${_arduino_comp_name}', f'REQUIRES {arduino_comp_name}']: + if old_req in cmake_text: + cmake_text = cmake_text.replace(old_req, f'{old_req} {main_reqs}') + break +``` + +--- + +## Error Visibility Improvements + +Two frontend/backend changes were made to ensure compilation errors are clearly visible: + +### Backend — Ninja Error Extraction + +When ESP-IDF/ninja fails, compiler errors go to **stdout** (not stderr). The backend now +detects `FAILED:` blocks in stdout and moves them to the `stderr` field of the response, +so the frontend classifies them as errors: + +```python +if stripped.startswith('FAILED:') or stripped == 'ninja: build stopped: subcommand failed.': + in_failed_block = True + # ... extract and move to stderr +``` + +### Frontend — Compilation Console Auto-Filter + +`CompilationConsole.tsx` now: +- Tracks previous log count with `prevLogsLenRef` +- Detects newly arrived error logs via `useEffect` +- Automatically switches the filter to **"Errors"** view when new errors arrive + +`compilationLogger.ts` classifies stdout lines in ninja `FAILED:` blocks as `'error'` +type (not `'info'`), using a state machine (`inFailedBlock` flag). + +--- + +## Test Coverage + +`backend/test_espidf_compiler.py` — 25 unit tests, no ESP-IDF toolchain required: + +| Class | Tests | What it covers | +|---|---|---| +| `TestDetectExternalIncludes` | 8 | DHT.h detected; Arduino.h, Wire.h, esp_* skipped; path headers skipped | +| `TestFindLibraryForHeader` | 4 | Root layout, `src/` layout, missing library, empty dir | +| `TestCreateIdfComponent` | 11 | Dir created, CMakeLists.txt content, files copied, name sanitization, correct library name | +| `TestTemplateCMakeLists` | 2 | Template files contain `user_libs` block and `REQUIRES` placeholder | + +Run from `backend/`: +```bash +python test_espidf_compiler.py +``` + +--- + +## Files Changed + +| File | Change | +|---|---| +| `backend/app/services/espidf_compiler.py` | Added `_detect_external_includes`, `_find_library_for_header`, `_create_idf_component`; BFS transitive dep resolution; CMake patching fixes | +| `backend/app/services/esp-idf-template/CMakeLists.txt` | Added `user_libs/` to `EXTRA_COMPONENT_DIRS` via `EXISTS` guard | +| `frontend/src/utils/compilationLogger.ts` | Ninja `FAILED:` block state machine; classifies lines as `'error'` | +| `frontend/src/components/editor/CompilationConsole.tsx` | Auto-switches to Errors filter when new errors arrive | +| `backend/test_espidf_compiler.py` | 25-test suite covering all library resolution logic | diff --git a/frontend/src/components/editor/CompilationConsole.tsx b/frontend/src/components/editor/CompilationConsole.tsx index 8a25051d..ccd2de35 100644 --- a/frontend/src/components/editor/CompilationConsole.tsx +++ b/frontend/src/components/editor/CompilationConsole.tsx @@ -22,6 +22,7 @@ export const CompilationConsole: React.FC = ({ const outputRef = useRef(null); const [autoscroll, setAutoscroll] = useState(true); const [filter, setFilter] = useState<'all' | 'errors' | 'warnings'>('all'); + const prevLogsLenRef = useRef(0); useEffect(() => { if (autoscroll && outputRef.current) { @@ -29,6 +30,15 @@ export const CompilationConsole: React.FC = ({ } }, [logs, autoscroll]); + // Auto-switch to "Errors" filter when a new batch of logs arrives with errors + useEffect(() => { + if (logs.length === prevLogsLenRef.current) return; + const newLogs = logs.slice(prevLogsLenRef.current); + prevLogsLenRef.current = logs.length; + const hasNewErrors = newLogs.some((l) => l.type === 'error'); + if (hasNewErrors) setFilter('errors'); + }, [logs]); + const filteredLogs = logs.filter((log) => { if (filter === 'errors') return log.type === 'error'; if (filter === 'warnings') return log.type === 'warning' || log.type === 'error'; diff --git a/frontend/src/utils/compilationLogger.ts b/frontend/src/utils/compilationLogger.ts index bc8b9243..b07d96ea 100644 --- a/frontend/src/utils/compilationLogger.ts +++ b/frontend/src/utils/compilationLogger.ts @@ -25,13 +25,35 @@ export function parseCompileResult(result: CompileResult, board: string): Compil } } - // stdout + // stdout — for ESP-IDF/ninja builds, compiler errors appear here if (result.stdout) { + let inFailedBlock = false; for (const line of result.stdout.split('\n')) { - if (line.trim()) { - const type = line.toLowerCase().includes('warning') ? 'warning' : 'info'; - logs.push({ timestamp: now, type, message: line }); + if (!line.trim()) continue; + const stripped = line.trim(); + // Ninja FAILED block start + if (stripped.startsWith('FAILED:') || stripped === 'ninja: build stopped: subcommand failed.') { + inFailedBlock = true; + logs.push({ timestamp: now, type: 'error', message: line }); + continue; } + // Progress line [N/M] ends a FAILED block + if (inFailedBlock && /^\[\d+\/\d+\]/.test(stripped)) { + inFailedBlock = false; + } + // Classify the line + let type: CompilationLog['type']; + if (inFailedBlock) { + // Lines inside a FAILED block: compiler output — detect subcategory + type = /:\s*(fatal )?error:/i.test(line) ? 'error' : 'warning'; + } else if (/:\s*(fatal )?error:/i.test(line) && !/^\[/.test(stripped)) { + type = 'error'; + } else if (/:\s*warning:/i.test(line) || line.toLowerCase().includes('warning')) { + type = 'warning'; + } else { + type = 'info'; + } + logs.push({ timestamp: now, type, message: line }); } }