From f15834d14d7d5275169326fa4cc678a76a852d99 Mon Sep 17 00:00:00 2001 From: David Montero Crespo Date: Thu, 20 Aug 2026 05:29:56 +0200 Subject: [PATCH] perf(compile): memoize ldgen - 41% of every ESP-IDF build was re-parsing linker fragments Profiled a one-file ESP-IDF compile on staging: 36.7 s total, of which ldgen.py alone was 15.2 s - consistently, on every single build. It re-parses all 48 component linker.lf fragments with pyparsing each time (3.8 M _parseNoCache calls; parse_fragment_file 15.6 s under cProfile), even though its output depends only on the sdkconfig and those fragments, never on the user's source. pyparsing's packrat cache was tried first and made it WORSE (15.2 s -> 27.4 s), so the fix is memoizing the whole run: a wrapper hashes the argv plus the CONTENT of every file it references and reuses the generated linker script when they are unchanged. Verified on staging: 15.2 s -> 0.08 s on a hit, byte-identical output, and correctly missing when the sdkconfig or any linker.lf changes (restoring the sdkconfig hits again - it is content-addressed, not mtime-based). Any surprise falls through to the untouched upstream script, so a build can never break because of the cache. New hand-written code on the ESP32-P4 lane: 36.7 s -> 11.5 s. --- Dockerfile.standalone | 11 +++ backend/scripts/ldgen_cache.py | 122 +++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 backend/scripts/ldgen_cache.py diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 30b215a4..2f6e2d07 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -257,6 +257,17 @@ COPY --from=espidf-builder /root/.espressif /root/.espressif COPY --from=espidf-builder /opt/arduino-esp32 /opt/arduino-esp32 +# Memoize ldgen. It re-parses every component's linker.lf with pyparsing on +# EVERY build - measured at 15.2 s here, 41% of a whole ESP-IDF compile of a +# one-file sketch - and its output is a pure function of its inputs. The +# wrapper delegates to the untouched upstream script on a miss. +COPY backend/scripts/ldgen_cache.py /tmp/ldgen_cache.py +RUN if [ -f /opt/esp-idf/tools/ldgen/ldgen.py ]; then \ + mv /opt/esp-idf/tools/ldgen/ldgen.py /opt/esp-idf/tools/ldgen/ldgen_upstream.py && \ + cp /tmp/ldgen_cache.py /opt/esp-idf/tools/ldgen/ldgen.py && \ + chmod +x /opt/esp-idf/tools/ldgen/ldgen.py; \ + fi; rm -f /tmp/ldgen_cache.py + ENV IDF_PATH=/opt/esp-idf ENV IDF_TOOLS_PATH=/root/.espressif ENV ARDUINO_ESP32_PATH=/opt/arduino-esp32 diff --git a/backend/scripts/ldgen_cache.py b/backend/scripts/ldgen_cache.py new file mode 100644 index 00000000..64df7176 --- /dev/null +++ b/backend/scripts/ldgen_cache.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +"""Memoizing front-end for ESP-IDF's ldgen. + +ldgen re-parses every component's `linker.lf` with pyparsing on EVERY build — +measured at 15.2 s on this box (3.8 M pyparsing calls for 48 fragment files), +which was 41 % of a whole ESP-IDF compile of a one-file sketch. Its output is a +pure function of its inputs (verified: byte-identical across runs, and unchanged +when only the user's own source changes), so the result is cached by a hash of +every input file plus the argument vector. + +Safety rules: + * every existing path in argv is hashed by CONTENT, plus the argv itself, so a + changed sdkconfig / fragment / library list misses the cache; + * a cache hit only ever copies a previously produced file into --output; + * anything unexpected (parse trouble, unreadable cache, write failure) falls + through to the real ldgen — the build must never break because of the cache. + +Installed as tools/ldgen/ldgen.py, with the upstream script kept alongside as +ldgen_upstream.py. +""" +import hashlib +import os +import runpy +import shutil +import sys +import tempfile + +CACHE_DIR = os.environ.get("VELXIO_LDGEN_CACHE", "/var/lib/velxio-build/.ldgen-cache") +MAX_ENTRIES = 64 +_HERE = os.path.dirname(os.path.abspath(__file__)) +_UPSTREAM = os.path.join(_HERE, "ldgen_upstream.py") + + +def _run_upstream(): + runpy.run_path(_UPSTREAM, run_name="__main__") + + +def _output_path(argv): + for i, a in enumerate(argv): + if a == "--output" and i + 1 < len(argv): + return argv[i + 1] + if a.startswith("--output="): + return a.split("=", 1)[1] + return None + + +def _input_hash(argv): + """Hash the argv and the CONTENT of every path it references. + + Paths arrive either as plain arguments or inside ';'-separated lists + (--fragments-list), so each token is split on ';' before testing. + """ + h = hashlib.sha256() + h.update(b"velxio-ldgen-cache-v1\0") + out = _output_path(argv) + for arg in argv[1:]: + if arg == out: + continue # the destination must not perturb the key + h.update(arg.encode("utf-8", "replace")) + h.update(b"\0") + for token in arg.split(";"): + token = token.strip() + if not token or token == out or not os.path.isfile(token): + continue + with open(token, "rb") as fh: + h.update(hashlib.sha256(fh.read()).digest()) + h.update(b"\0") + return h.hexdigest() + + +def _prune(): + try: + entries = sorted( + (os.path.join(CACHE_DIR, n) for n in os.listdir(CACHE_DIR) if n.endswith(".ld")), + key=os.path.getmtime, + reverse=True, + ) + for stale in entries[MAX_ENTRIES:]: + os.unlink(stale) + except OSError: + pass + + +def main(): + argv = sys.argv + out = _output_path(argv) + if not out: + _run_upstream() + return + + try: + key = _input_hash(argv) + cached = os.path.join(CACHE_DIR, key + ".ld") + except Exception: + _run_upstream() + return + + if os.path.isfile(cached): + try: + os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True) + shutil.copyfile(cached, out) + os.utime(cached, None) # LRU touch + return + except OSError: + pass # fall through and regenerate + + _run_upstream() + + try: + if os.path.isfile(out): + os.makedirs(CACHE_DIR, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=CACHE_DIR, suffix=".tmp") + os.close(fd) + shutil.copyfile(out, tmp) + os.replace(tmp, cached) + _prune() + except OSError: + pass + + +if __name__ == "__main__": + main()