feat(intellisense): GET /api/intellisense/type/{name} — members of one class
The completion-safe long-tail path. Feeding whole library symbol sets into the editor's candidate pool was measured to make ranking worse (U8g2 alone injects ~250 module-level class names; ArduinoJson dozens of ARDUINOJSON_* macros). What does help is answering `client.` once tree-sitter has resolved `client` to a declared type the embedded catalog does not know -- and for that the engine needs "the members of type T", not "the symbols of library L". This endpoint returns exactly that: symbols whose owner is the requested class, from whichever cached library declares it, in the same payload shape as /symbols/ so the engine's RemoteSymbolCache stores it unchanged. Resolution: a class-name -> cache-entry index built once over the library cache (each per-entry scan is already disk-cached), persisted under .symbols/_types.json (3,818 classes / 199 KB over the 1,125-entry production cache; ~1 s to rebuild from cached scans, ~60 s cold), refreshed after a few hours or when the cache grows. Many classes are declared by more than one library -- the real one plus forks and bundles vendoring a copy (PubSubClient inside "ESP8266 Microgear", NTPClient inside "ESP8266 Weather Station", AccelStepper inside "Arduino Learning Board") -- so the pick is: the library named after the class, else the one declaring the most members of it (vendored copies are trimmed), else alphabetical. Generic, not pro-specific: any host with a library cache can use it.
This commit is contained in:
David Montero
parent
0e107fafe6
commit
1a112324ab
|
|
@ -17,13 +17,21 @@ from __future__ import annotations
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.services.symbol_extract import find_cache_entry, parse_spec, scan_library
|
||||
from app.services.symbol_extract import (
|
||||
_entry_version_key,
|
||||
find_cache_entry,
|
||||
norm_name,
|
||||
parse_spec,
|
||||
scan_library,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -60,6 +68,117 @@ def _write_symbol_cache(sym_file: Path, payload: dict) -> None:
|
|||
logger.warning("[intellisense] symbol cache write failed for %s: %s", sym_file, e)
|
||||
|
||||
|
||||
def _payload_for(root: Path, entry: Path) -> dict:
|
||||
"""Cached scan of one entry (same disk cache get_symbols uses)."""
|
||||
sym_file = root / ".symbols" / f"{entry.name}.json"
|
||||
if sym_file.is_file():
|
||||
try:
|
||||
return json.loads(sym_file.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
payload = scan_library(entry)
|
||||
_write_symbol_cache(sym_file, payload)
|
||||
return payload
|
||||
|
||||
|
||||
_TYPE_INDEX_NAME = "_types.json"
|
||||
_TYPE_INDEX_MAX_AGE_S = 6 * 3600
|
||||
|
||||
|
||||
def _type_index(root: Path) -> dict[str, str]:
|
||||
"""owner-class name -> cache entry dir name, over the whole library
|
||||
cache. Built by scanning every entry once (each scan is itself disk-
|
||||
cached), persisted under .symbols/_types.json and rebuilt when older than
|
||||
a few hours or when the cache has grown. When two entries declare the
|
||||
same class (a library and its fork, or two versions), the newest entry
|
||||
for the alphabetically-first library wins -- deterministic and, for the
|
||||
common case of one library per class, irrelevant."""
|
||||
idx_file = root / ".symbols" / _TYPE_INDEX_NAME
|
||||
entries = sorted(
|
||||
d.name for d in root.iterdir() if d.is_dir() and not d.name.startswith(".")
|
||||
)
|
||||
if idx_file.is_file():
|
||||
try:
|
||||
cached = json.loads(idx_file.read_text(encoding="utf-8"))
|
||||
fresh = (time.time() - idx_file.stat().st_mtime) < _TYPE_INDEX_MAX_AGE_S
|
||||
if fresh and cached.get("entry_count") == len(entries):
|
||||
return cached["types"]
|
||||
except (OSError, ValueError, KeyError):
|
||||
pass
|
||||
|
||||
# Many classes are declared by more than one cached library: the real
|
||||
# one plus forks and bundles that vendor a copy (PubSubClient ships inside
|
||||
# "ESP8266 Microgear", NTPClient inside "ESP8266 Weather Station",
|
||||
# AccelStepper inside "Arduino Learning Board"). Pick, per class:
|
||||
# 1. the library whose normalized name equals the class name, else
|
||||
# 2. the library declaring the MOST members of that class (the
|
||||
# canonical copy is the complete one; vendored copies are trimmed),
|
||||
# 3. ties: alphabetical, deterministic.
|
||||
best: dict[str, tuple[int, int, str]] = {} # owner -> (name_match, member_count, entry) with a sortable key
|
||||
by_name: dict[str, list[Path]] = {}
|
||||
for name in entries:
|
||||
by_name.setdefault(name.partition("@")[0], []).append(root / name)
|
||||
for lib_name in sorted(by_name):
|
||||
newest = sorted(by_name[lib_name], key=_entry_version_key, reverse=True)[0]
|
||||
try:
|
||||
payload = _payload_for(root, newest)
|
||||
except Exception as e: # noqa: BLE001 - one bad library must not kill the index
|
||||
logger.warning("[intellisense] type index: skipping %s: %s", newest.name, e)
|
||||
continue
|
||||
counts: dict[str, int] = {}
|
||||
for sym in payload.get("symbols", []):
|
||||
owner = sym.get("owner")
|
||||
if owner:
|
||||
counts[owner] = counts.get(owner, 0) + 1
|
||||
for owner, n in counts.items():
|
||||
name_match = 1 if norm_name(owner) == lib_name else 0
|
||||
candidate = (name_match, n, newest.name)
|
||||
cur = best.get(owner)
|
||||
# higher name_match, then higher count, then (already alphabetical
|
||||
# by iteration order) keep the first
|
||||
if cur is None or (candidate[0], candidate[1]) > (cur[0], cur[1]):
|
||||
best[owner] = candidate
|
||||
types = {owner: entry for owner, (_, _, entry) in best.items()}
|
||||
_write_symbol_cache(idx_file, {"entry_count": len(entries), "types": types})
|
||||
return types
|
||||
|
||||
|
||||
@router.get("/type/{name}")
|
||||
def get_type_members(name: str) -> JSONResponse:
|
||||
"""Members of one class/struct, whichever cached library declares it.
|
||||
|
||||
This is the long-tail path that is safe for completion quality: the
|
||||
engine calls it only after tree-sitter has resolved a receiver to a
|
||||
declared type that the embedded catalog does not know (`PubSubClient
|
||||
client(espClient); client.` -> PubSubClient), and it returns ONLY that
|
||||
type's members -- never a library's module-level constants or macros,
|
||||
which are what made feeding whole library sets into the candidate pool
|
||||
measurably worse. Payload shape matches /symbols/ so the engine's
|
||||
RemoteSymbolCache can store it unchanged."""
|
||||
root = _cache_root()
|
||||
if not root.is_dir():
|
||||
raise HTTPException(status_code=404, detail=f"library cache not available at {root}")
|
||||
if not re.fullmatch(r"[A-Za-z_]\w{0,80}", name):
|
||||
raise HTTPException(status_code=400, detail="not a type name")
|
||||
entry_name = _type_index(root).get(name)
|
||||
if entry_name is None:
|
||||
raise HTTPException(status_code=404, detail=f"no cached library declares {name}")
|
||||
entry = root / entry_name
|
||||
if not entry.is_dir():
|
||||
raise HTTPException(status_code=404, detail=f"stale type index for {name}")
|
||||
payload = _payload_for(root, entry)
|
||||
members = [s for s in payload.get("symbols", []) if s.get("owner") == name]
|
||||
return JSONResponse(
|
||||
content={
|
||||
"id": payload.get("id") or entry_name,
|
||||
"triggers": payload.get("triggers", []),
|
||||
"type": name,
|
||||
"symbols": members,
|
||||
},
|
||||
headers=_CACHE_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/symbols/{spec}")
|
||||
def get_symbols(spec: str) -> JSONResponse:
|
||||
"""Symbol table for one library. `spec` is `Name@Version` or a bare
|
||||
|
|
@ -74,14 +193,4 @@ def get_symbols(spec: str) -> JSONResponse:
|
|||
if entry is None:
|
||||
raise HTTPException(status_code=404, detail=f"library not found in cache: {spec}")
|
||||
|
||||
sym_file = root / ".symbols" / f"{entry.name}.json"
|
||||
if sym_file.is_file():
|
||||
try:
|
||||
cached = json.loads(sym_file.read_text(encoding="utf-8"))
|
||||
return JSONResponse(content=cached, headers=_CACHE_HEADERS)
|
||||
except (OSError, ValueError):
|
||||
pass # corrupt/unreadable cache file -> rescan below
|
||||
|
||||
payload = scan_library(entry)
|
||||
_write_symbol_cache(sym_file, payload)
|
||||
return JSONResponse(content=payload, headers=_CACHE_HEADERS)
|
||||
return JSONResponse(content=_payload_for(root, entry), headers=_CACHE_HEADERS)
|
||||
|
|
|
|||
|
|
@ -205,3 +205,63 @@ def test_route_registers_symbols_path() -> None:
|
|||
from app.api.routes import intellisense
|
||||
|
||||
assert any(r.path == "/symbols/{spec}" for r in intellisense.router.routes)
|
||||
|
||||
|
||||
def test_type_endpoint_returns_only_the_declaring_class_members(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""/type/{name} is the completion-safe long-tail path: only the members
|
||||
of that class, never a library's module-level constants. And when two
|
||||
libraries declare the same class, the one NAMED after it wins over a
|
||||
fork/bundle that vendors a trimmed copy."""
|
||||
fastapi = pytest.importorskip("fastapi")
|
||||
from fastapi.testclient import TestClient
|
||||
from app.api.routes import intellisense
|
||||
|
||||
root = tmp_path / "libcache"
|
||||
|
||||
def lib(dirname: str, display: str, header: str) -> None:
|
||||
d = root / dirname / "src"
|
||||
d.mkdir(parents=True)
|
||||
(root / dirname / "library.properties").write_text(
|
||||
f"name={display}\nversion=1.0.0\n", encoding="utf-8"
|
||||
)
|
||||
(d / "Lib.h").write_text(header, encoding="utf-8")
|
||||
|
||||
# The canonical library: named after the class, full API + a macro.
|
||||
lib(
|
||||
"pubsubclient@2.8-aaaaaaaaaaaa",
|
||||
"PubSubClient",
|
||||
"#define MQTT_MAX_PACKET_SIZE 256\n"
|
||||
"class PubSubClient {\npublic:\n void setServer(const char*, int);\n"
|
||||
" bool publish(const char*, const char*);\n bool connected();\n};\n",
|
||||
)
|
||||
# A bundle that vendors a trimmed copy under another name.
|
||||
lib(
|
||||
"espbundle@1.0.0-bbbbbbbbbbbb",
|
||||
"ESP Bundle",
|
||||
"class PubSubClient {\npublic:\n bool publish(const char*, const char*);\n};\n"
|
||||
"class BundleThing {\npublic:\n void go();\n};\n",
|
||||
)
|
||||
monkeypatch.setenv("VELXIO_LIBCACHE_DIR", str(root))
|
||||
|
||||
app = fastapi.FastAPI()
|
||||
app.include_router(intellisense.router, prefix="/api/intellisense")
|
||||
client = TestClient(app)
|
||||
|
||||
r = client.get("/api/intellisense/type/PubSubClient")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["id"] == "PubSubClient" # the canonical one, not the bundle
|
||||
names = {s["name"] for s in body["symbols"]}
|
||||
assert names == {"setServer", "publish", "connected"}
|
||||
assert all(s["owner"] == "PubSubClient" for s in body["symbols"])
|
||||
assert "MQTT_MAX_PACKET_SIZE" not in names # module-level never leaks
|
||||
|
||||
r = client.get("/api/intellisense/type/BundleThing")
|
||||
assert r.status_code == 200 and r.json()["id"] == "ESP Bundle"
|
||||
|
||||
assert client.get("/api/intellisense/type/Nope").status_code == 404
|
||||
assert client.get("/api/intellisense/type/not-a-type!").status_code == 400
|
||||
# the index is persisted and reused
|
||||
assert (root / ".symbols" / "_types.json").is_file()
|
||||
|
|
|
|||
Loading…
Reference in New Issue