perf(compile): memoize ldgen - 41% of every ESP-IDF build was re-parsing linker fragments

Profiled a one-file ESP-IDF compile on staging: 36.7 s total, of which
ldgen.py alone was 15.2 s - consistently, on every single build. It
re-parses all 48 component linker.lf fragments with pyparsing each time
(3.8 M _parseNoCache calls; parse_fragment_file 15.6 s under cProfile),
even though its output depends only on the sdkconfig and those
fragments, never on the user's source.

pyparsing's packrat cache was tried first and made it WORSE (15.2 s ->
27.4 s), so the fix is memoizing the whole run: a wrapper hashes the
argv plus the CONTENT of every file it references and reuses the
generated linker script when they are unchanged. Verified on staging:
15.2 s -> 0.08 s on a hit, byte-identical output, and correctly missing
when the sdkconfig or any linker.lf changes (restoring the sdkconfig
hits again - it is content-addressed, not mtime-based). Any surprise
falls through to the untouched upstream script, so a build can never
break because of the cache.

New hand-written code on the ESP32-P4 lane: 36.7 s -> 11.5 s.

Dockerfile.standalone

@@ -257,6 +257,17 @@ COPY --from=espidf-builder /root/.espressif /root/.espressif
 
 COPY --from=espidf-builder /opt/arduino-esp32 /opt/arduino-esp32
 
+# Memoize ldgen. It re-parses every component's linker.lf with pyparsing on
+# EVERY build - measured at 15.2 s here, 41% of a whole ESP-IDF compile of a
+# one-file sketch - and its output is a pure function of its inputs. The
+# wrapper delegates to the untouched upstream script on a miss.
+COPY backend/scripts/ldgen_cache.py /tmp/ldgen_cache.py
+RUN if [ -f /opt/esp-idf/tools/ldgen/ldgen.py ]; then \
+      mv /opt/esp-idf/tools/ldgen/ldgen.py /opt/esp-idf/tools/ldgen/ldgen_upstream.py && \
+      cp /tmp/ldgen_cache.py /opt/esp-idf/tools/ldgen/ldgen.py && \
+      chmod +x /opt/esp-idf/tools/ldgen/ldgen.py; \
+    fi; rm -f /tmp/ldgen_cache.py
+
 ENV IDF_PATH=/opt/esp-idf
 ENV IDF_TOOLS_PATH=/root/.espressif
 ENV ARDUINO_ESP32_PATH=/opt/arduino-esp32

backend/scripts/ldgen_cache.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+"""Memoizing front-end for ESP-IDF's ldgen.
+
+ldgen re-parses every component's `linker.lf` with pyparsing on EVERY build —
+measured at 15.2 s on this box (3.8 M pyparsing calls for 48 fragment files),
+which was 41 % of a whole ESP-IDF compile of a one-file sketch. Its output is a
+pure function of its inputs (verified: byte-identical across runs, and unchanged
+when only the user's own source changes), so the result is cached by a hash of
+every input file plus the argument vector.
+
+Safety rules:
+  * every existing path in argv is hashed by CONTENT, plus the argv itself, so a
+    changed sdkconfig / fragment / library list misses the cache;
+  * a cache hit only ever copies a previously produced file into --output;
+  * anything unexpected (parse trouble, unreadable cache, write failure) falls
+    through to the real ldgen — the build must never break because of the cache.
+
+Installed as tools/ldgen/ldgen.py, with the upstream script kept alongside as
+ldgen_upstream.py.
+"""
+import hashlib
+import os
+import runpy
+import shutil
+import sys
+import tempfile
+
+CACHE_DIR = os.environ.get("VELXIO_LDGEN_CACHE", "/var/lib/velxio-build/.ldgen-cache")
+MAX_ENTRIES = 64
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_UPSTREAM = os.path.join(_HERE, "ldgen_upstream.py")
+
+
+def _run_upstream():
+    runpy.run_path(_UPSTREAM, run_name="__main__")
+
+
+def _output_path(argv):
+    for i, a in enumerate(argv):
+        if a == "--output" and i + 1 < len(argv):
+            return argv[i + 1]
+        if a.startswith("--output="):
+            return a.split("=", 1)[1]
+    return None
+
+
+def _input_hash(argv):
+    """Hash the argv and the CONTENT of every path it references.
+
+    Paths arrive either as plain arguments or inside ';'-separated lists
+    (--fragments-list), so each token is split on ';' before testing.
+    """
+    h = hashlib.sha256()
+    h.update(b"velxio-ldgen-cache-v1\0")
+    out = _output_path(argv)
+    for arg in argv[1:]:
+        if arg == out:
+            continue  # the destination must not perturb the key
+        h.update(arg.encode("utf-8", "replace"))
+        h.update(b"\0")
+        for token in arg.split(";"):
+            token = token.strip()
+            if not token or token == out or not os.path.isfile(token):
+                continue
+            with open(token, "rb") as fh:
+                h.update(hashlib.sha256(fh.read()).digest())
+            h.update(b"\0")
+    return h.hexdigest()
+
+
+def _prune():
+    try:
+        entries = sorted(
+            (os.path.join(CACHE_DIR, n) for n in os.listdir(CACHE_DIR) if n.endswith(".ld")),
+            key=os.path.getmtime,
+            reverse=True,
+        )
+        for stale in entries[MAX_ENTRIES:]:
+            os.unlink(stale)
+    except OSError:
+        pass
+
+
+def main():
+    argv = sys.argv
+    out = _output_path(argv)
+    if not out:
+        _run_upstream()
+        return
+
+    try:
+        key = _input_hash(argv)
+        cached = os.path.join(CACHE_DIR, key + ".ld")
+    except Exception:
+        _run_upstream()
+        return
+
+    if os.path.isfile(cached):
+        try:
+            os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True)
+            shutil.copyfile(cached, out)
+            os.utime(cached, None)  # LRU touch
+            return
+        except OSError:
+            pass  # fall through and regenerate
+
+    _run_upstream()
+
+    try:
+        if os.path.isfile(out):
+            os.makedirs(CACHE_DIR, exist_ok=True)
+            fd, tmp = tempfile.mkstemp(dir=CACHE_DIR, suffix=".tmp")
+            os.close(fd)
+            shutil.copyfile(out, tmp)
+            os.replace(tmp, cached)
+            _prune()
+    except OSError:
+        pass
+
+
+if __name__ == "__main__":
+    main()