From b58ecd1c370bea3473d7c2527ddbcd1ad8992e08 Mon Sep 17 00:00:00 2001 From: David Montero Date: Sat, 15 Aug 2026 08:29:39 +0200 Subject: [PATCH] fix(espidf): generate .ino forward declarations + link the task-WDT API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The custom sketch.ino.cpp path never ran arduino-cli's ctags prototype pass, so the universal Arduino idiom (helpers defined after loop()) died with "'foo' was not declared in this scope" on every ESP32 build. A conservative generator (skips templates, default args, methods, function-pointer params) injects prototypes before the first definition with a #line directive so error line numbers stay put. 8 unit tests. IDF 5.5 renamed ESP_TASK_WDT to the EN/INIT pair; the v4.4 template's =n became EN=n, compiling the whole implementation out — every sketch using esp_task_wdt_add/reset (Firebase's client does internally) failed at link. The v5 fixup now emits EN=y INIT=n: the API links, the watchdog never auto-starts under emulation. v4.4 semantics untouched. --- backend/app/services/espidf_compiler.py | 160 +++++++++++++++++++++++- backend/tests/test_espidf_options.py | 12 +- backend/tests/test_ino_prototypes.py | 109 ++++++++++++++++ 3 files changed, 273 insertions(+), 8 deletions(-) create mode 100644 backend/tests/test_ino_prototypes.py diff --git a/backend/app/services/espidf_compiler.py b/backend/app/services/espidf_compiler.py index e9abd7f5..98f744f7 100644 --- a/backend/app/services/espidf_compiler.py +++ b/backend/app/services/espidf_compiler.py @@ -195,6 +195,147 @@ def _evict_cold_variants(target_dir: Path, keep: int) -> None: shutil.rmtree(d, ignore_errors=True) +# ── .ino prototype generation ──────────────────────────────────────────────── +# arduino-cli runs ctags over the sketch and injects forward declarations so +# the classic Arduino idiom — helpers defined AFTER setup()/loop() that use +# them — compiles. This pipeline writes sketch.ino.cpp itself and never did, +# so every such sketch died with "'foo' was not declared in this scope". +# This is a conservative ctags stand-in: it only emits prototypes for +# top-level, single-signature-line function definitions and skips anything +# ambiguous (templates, methods, default args, function-pointer params). + +_INO_CTRL_KEYWORDS = frozenset(( + 'if', 'else', 'for', 'while', 'switch', 'do', 'return', 'sizeof', + 'catch', 'case', 'new', 'delete', 'throw', 'goto', +)) + +_INO_FUNC_RE = re.compile( + r'^[ \t]*([A-Za-z_][A-Za-z0-9_:<>,\*&\s]*?[\s\*&])' # return type part + r'([A-Za-z_]\w*)' # function name + r'[ \t]*\(([^;{}()]*)\)' # params (no nesting) + r'[ \t\r\n]*\{', + re.M, +) + + +def _blank_noncode(source: str) -> str: + """Return a same-length copy with comments, string/char literals and + preprocessor lines replaced by spaces (newlines preserved), so brace + counting and signature matching never trip on their contents.""" + out = list(source) + i, n = 0, len(source) + line_start = True + while i < n: + c = source[i] + if line_start and source[i:].lstrip(' \t')[:1] == '#': + # blank the whole preprocessor line (incl. continuations) + while i < n: + if source[i] == '\n' and source[i - 1] != '\\': + break + if source[i] != '\n': + out[i] = ' ' + i += 1 + line_start = True + i += 1 + continue + line_start = c == '\n' + if c == '/' and i + 1 < n and source[i + 1] == '/': + while i < n and source[i] != '\n': + out[i] = ' ' + i += 1 + continue + if c == '/' and i + 1 < n and source[i + 1] == '*': + out[i] = out[i + 1] = ' ' + i += 2 + while i < n and not (source[i] == '*' and i + 1 < n and source[i + 1] == '/'): + if source[i] != '\n': + out[i] = ' ' + i += 1 + if i < n: + out[i] = out[i + 1] = ' ' + i += 2 + continue + if c in ('"', "'"): + quote = c + i += 1 + while i < n and source[i] != quote: + if source[i] == '\\': + out[i] = ' ' + i += 1 + if i < n and source[i] != '\n': + out[i] = ' ' + i += 1 + continue + if source[i] != '\n': + out[i] = ' ' + i += 1 + i += 1 + continue + i += 1 + return ''.join(out) + + +def generate_ino_prototypes(source: str) -> str: + """Insert forward declarations for top-level function definitions right + before the first one, followed by a #line directive so compiler error + line numbers keep matching the un-inserted text. Returns the source + unchanged when nothing safe to declare is found.""" + scan = _blank_noncode(source) + + # depth prefix: depth[i] = brace depth just before scan[i] + depths = [] + d = 0 + for ch in scan: + depths.append(d) + if ch == '{': + d += 1 + elif ch == '}': + d = max(0, d - 1) + + protos: list[str] = [] + seen: set[str] = set() + first_pos: int | None = None + for m in _INO_FUNC_RE.finditer(scan): + sig_start = m.start() + if depths[m.end() - 1] != 0: + continue # method body / nested — the '{' must open at depth 0 + rtype = ' '.join(m.group(1).split()) + name = m.group(2) + params = ' '.join(m.group(3).split()) + if not rtype or name in _INO_CTRL_KEYWORDS: + continue + first_tok = rtype.split()[0] if rtype.split() else '' + if first_tok in _INO_CTRL_KEYWORDS: + continue + if '::' in rtype or '::' in name or 'operator' in rtype: + continue # class methods / operators declare themselves + if '<' in rtype or '>' in rtype or 'template' in rtype.split(): + continue # templated heads/returns — never guess those + if '=' in params or '{' in params: + continue # default args would be re-stated -> hard error + # a `template<...>` head on a preceding line makes a bare prototype wrong + lookback = scan[max(0, sig_start - 160):sig_start] + if re.search(r'template\s*<[^<>]*>\s*$', lookback): + continue + proto = f'{rtype} {name}({params});' + if proto not in seen: + seen.add(proto) + protos.append(proto) + if first_pos is None: + first_pos = sig_start + + if not protos or first_pos is None: + return source + + insert_line = source.count('\n', 0, first_pos) + 1 + block = ( + '// Velxio: auto-generated forward declarations (.ino semantics)\n' + + '\n'.join(protos) + + f'\n#line {insert_line}\n' + ) + return source[:first_pos] + block + source[first_pos:] + + def _prepare_persistent_project_dir( idf_target: str, variant_key: str = 'default', @@ -2490,11 +2631,18 @@ class ESPIDFCompiler: for line in rendered.splitlines(): if line.startswith(drops): continue - # IDF 5.0 renamed ESP_TASK_WDT → ESP_TASK_WDT_EN. + # IDF 5.0 split ESP_TASK_WDT into ESP_TASK_WDT_EN (compile the + # implementation) and ESP_TASK_WDT_INIT (auto-start at boot). + # The v4.4 template's `=n` used to become `EN=n`, which compiles + # the API out entirely — every sketch calling esp_task_wdt_add/ + # reset (Firebase's ESP32 client does internally) died at link + # with "undefined reference to esp_task_wdt_*". Keep the intent + # (no watchdog running under emulation) while linking the API: + # compile it in, never auto-start it. if line.startswith('CONFIG_ESP_TASK_WDT='): - line = line.replace( - 'CONFIG_ESP_TASK_WDT=', 'CONFIG_ESP_TASK_WDT_EN=', 1 - ) + lines.append('CONFIG_ESP_TASK_WDT_EN=y') + lines.append('CONFIG_ESP_TASK_WDT_INIT=n') + continue lines.append(line) return '\n'.join(lines) + '\n' @@ -3237,6 +3385,10 @@ class ESPIDFCompiler: '#include "Arduino.h"\n' + compat_include.rstrip('\n'), 1, ) + # arduino-cli injects ctags prototypes; this path must too, or + # helpers defined after loop() fail with "not declared in this + # scope". Applied AFTER the prepends so #line stays accurate. + main_content = generate_ino_prototypes(main_content) sketch_cpp.write_text(main_content, encoding='utf-8') # Copy additional files (.h, .cpp) diff --git a/backend/tests/test_espidf_options.py b/backend/tests/test_espidf_options.py index 986aa04d..11976feb 100644 --- a/backend/tests/test_espidf_options.py +++ b/backend/tests/test_espidf_options.py @@ -286,8 +286,10 @@ def test_render_sdkconfig_c6_adapts_to_idf5(compiler: ESPIDFCompiler) -> None: assert 'CONFIG_SPIRAM' not in text assert 'CONFIG_ESPTOOLPY_FLASHFREQ_26M' not in text assert 'CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240' not in text - # IDF 5.0 renamed ESP_TASK_WDT → ESP_TASK_WDT_EN. - assert 'CONFIG_ESP_TASK_WDT_EN=n' in text + # IDF 5.0 split ESP_TASK_WDT into _EN (compile) + _INIT (auto-start): + # the API must link (sketches call esp_task_wdt_add) but never auto-run. + assert 'CONFIG_ESP_TASK_WDT_EN=y' in text + assert 'CONFIG_ESP_TASK_WDT_INIT=n' in text assert 'CONFIG_ESP_TASK_WDT=n' not in text # The clamped CPU frequency is rendered. assert 'CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_160=y' in text @@ -403,7 +405,8 @@ def test_render_sdkconfig_idf5_arduino_keeps_component_symbols( assert 'CONFIG_AUTOSTART_ARDUINO=n' in ard assert 'CONFIG_ARDUHAL_LOG_DEFAULT_LEVEL=0' in ard assert 'CONFIG_BT_ENABLED=y' in ard - assert 'CONFIG_ESP_TASK_WDT_EN=n' in ard # WDT rename still applies + assert 'CONFIG_ESP_TASK_WDT_EN=y' in ard # WDT split still applies + assert 'CONFIG_ESP_TASK_WDT_INIT=n' in ard pure = compiler._render_sdkconfig( opts, _TEMPLATE_DIR, idf_target='esp32', use_idf5=True, arduino_mode=False @@ -435,7 +438,8 @@ def test_render_sdkconfig_idf5_per_target_drops(compiler: ESPIDFCompiler) -> Non assert 'CONFIG_ESPTOOLPY_FLASHFREQ_26M' in text # kept on esp32 assert 'CONFIG_ARDUHAL_' not in text # no Arduino on v5 assert 'CONFIG_BT_ENABLED' not in text # no Bluedroid on v5 - assert 'CONFIG_ESP_TASK_WDT_EN=n' in text # renamed in v5 + assert 'CONFIG_ESP_TASK_WDT_EN=y' in text # split in v5 + assert 'CONFIG_ESP_TASK_WDT_INIT=n' in text s3_text = compiler._render_sdkconfig( compiler._normalize_options(None, idf_target='esp32s3'), diff --git a/backend/tests/test_ino_prototypes.py b/backend/tests/test_ino_prototypes.py new file mode 100644 index 00000000..a44bebd9 --- /dev/null +++ b/backend/tests/test_ino_prototypes.py @@ -0,0 +1,109 @@ +"""Unit tests for the .ino forward-declaration generator (espidf_compiler). + +The generator is a ctags stand-in for the ESP32 sketch.ino.cpp path: the +classic Arduino idiom (helpers defined after setup()/loop()) must compile, +while anything ambiguous (methods, templates, default args) is left alone. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app.services.espidf_compiler import generate_ino_prototypes # noqa: E402 + + +def test_helper_after_loop_gets_prototype(): + src = ( + '#include "Arduino.h"\n' + "void setup() { blinkTwice(); }\n" + "void loop() { blinkTwice(); }\n" + "void blinkTwice() { }\n" + ) + out = generate_ino_prototypes(src) + assert "void blinkTwice();" in out + # inserted before the first function definition, after the include + assert out.index("void blinkTwice();") < out.index("void setup() {") + assert out.index('#include "Arduino.h"') < out.index("void blinkTwice();") + + +def test_line_directive_preserves_error_lines(): + src = "void setup() {}\nvoid loop() {}\n" + out = generate_ino_prototypes(src) + # the original first function was on line 1 + assert "#line 1\n" in out + + +def test_control_flow_and_methods_are_not_declared(): + src = ( + "class Foo {\n" + " public:\n" + " void method() { }\n" + "};\n" + "void Foo2::method2() { }\n" + "void run() {\n" + " if (millis() > 5) { }\n" + " while (true) { }\n" + "}\n" + ) + out = generate_ino_prototypes(src) + assert "method();" not in out + assert "method2();" not in out + assert "if(" not in out and "while(" not in out + assert "void run();" in out + + +def test_default_args_and_templates_skipped(): + src = ( + "int scaled(int v, int f = 2) { return v * f; }\n" + "template \n" + "T biggest(T a, T b) { return a > b ? a : b; }\n" + "void setup() {}\n" + ) + out = generate_ino_prototypes(src) + assert "scaled(int v, int f = 2);" not in out + assert "T biggest(T a, T b);" not in out + assert "void setup();" in out + + +def test_strings_comments_and_preprocessor_ignored(): + src = ( + '#define WEIRD "void fake() {"\n' + '// void alsoFake() {\n' + '/* void broken(int x) { */\n' + 'const char *s = "void inString() {";\n' + "void real() { }\n" + ) + out = generate_ino_prototypes(src) + assert "fake();" not in out + assert "alsoFake();" not in out + assert "broken(int x);" not in out + assert "inString();" not in out + assert "void real();" in out + + +def test_no_functions_returns_unchanged(): + src = "int x = 5;\n#define Y 4\n" + assert generate_ino_prototypes(src) == src + + +def test_multiline_signature_params_kept(): + src = ( + "void setup() { drawStatus(1, 2); }\n" + "void drawStatus(int a,\n" + " int b) {\n" + "}\n" + ) + out = generate_ino_prototypes(src) + assert "void drawStatus(int a, int b);" in out + + +def test_attribute_and_pointer_returns(): + src = ( + "void IRAM_ATTR onTimer() { }\n" + "char *nameFor(int idx) { return 0; }\n" + "void setup() { onTimer(); nameFor(1); }\n" + ) + out = generate_ino_prototypes(src) + assert "void IRAM_ATTR onTimer();" in out + assert "char *nameFor(int idx);" in out or "char * nameFor(int idx);" in out